IEMAllCImpl-x86.cpp@ 108196

Last change on this file since 108196 was 108196, checked in by vboxsync, 3 months ago
VMM/IEM: Moving x86 target specific files to VMMAll/target-x86/... jiraref:VBP-1431
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 399.7 KB

Line
1	/* $Id: IEMAllCImpl-x86.cpp 108196 2025-02-13 15:18:18Z vboxsync $ */
2	/** @file
3	* IEM - Instruction Implementation in C/C++, x86 target.
4	*/
5
6	/*
7	* Copyright (C) 2011-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_IEM
33	#define VMCPU_INCL_CPUM_GST_CTX
34	#define IEM_WITH_OPAQUE_DECODER_STATE
35	#ifdef IN_RING0
36	# define VBOX_VMM_TARGET_X86
37	#endif
38	#include <VBox/vmm/iem.h>
39	#include <VBox/vmm/cpum.h>
40	#include <VBox/vmm/pdmapic.h>
41	#include <VBox/vmm/pdm.h>
42	#include <VBox/vmm/pgm.h>
43	#include <VBox/vmm/iom.h>
44	#include <VBox/vmm/em.h>
45	#include <VBox/vmm/hm.h>
46	#include <VBox/vmm/nem.h>
47	#include <VBox/vmm/gim.h>
48	#include <VBox/vmm/gcm.h>
49	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
50	# include <VBox/vmm/em.h>
51	# include <VBox/vmm/hm_svm.h>
52	#endif
53	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
54	# include <VBox/vmm/hmvmxinline.h>
55	#endif
56	#ifndef VBOX_WITHOUT_CPUID_HOST_CALL
57	# include <VBox/vmm/cpuidcall.h>
58	#endif
59	#include <VBox/vmm/tm.h>
60	#include <VBox/vmm/dbgf.h>
61	#include <VBox/vmm/dbgftrace.h>
62	#include "IEMInternal.h"
63	#include <VBox/vmm/vmcc.h>
64	#include <VBox/log.h>
65	#include <VBox/err.h>
66	#include <VBox/param.h>
67	#include <VBox/dis.h>
68	#include <iprt/asm-math.h>
69	#include <iprt/assert.h>
70	#include <iprt/string.h>
71	#include <iprt/x86.h>
72
73	#include "IEMInline.h"
74
75
76	/*********************************************************************************************************************************
77	* Defined Constants And Macros *
78	*********************************************************************************************************************************/
79	/**
80	* Flushes the prefetch buffer, light version.
81	* @todo The \#if conditions here must match the ones in iemOpcodeFlushLight().
82	*/
83	#ifndef IEM_WITH_CODE_TLB
84	# define IEM_FLUSH_PREFETCH_LIGHT(a_pVCpu, a_cbInstr) iemOpcodeFlushLight(a_pVCpu, a_cbInstr)
85	#else
86	# define IEM_FLUSH_PREFETCH_LIGHT(a_pVCpu, a_cbInstr) do { } while (0)
87	#endif
88
89	/**
90	* Flushes the prefetch buffer, heavy version.
91	* @todo The \#if conditions here must match the ones in iemOpcodeFlushHeavy().
92	*/
93	#if !defined(IEM_WITH_CODE_TLB) \|\| 1
94	# define IEM_FLUSH_PREFETCH_HEAVY(a_pVCpu, a_cbInstr) iemOpcodeFlushHeavy(a_pVCpu, a_cbInstr)
95	#else
96	# define IEM_FLUSH_PREFETCH_HEAVY(a_pVCpu, a_cbInstr) do { } while (0)
97	#endif
98
99
100	/*********************************************************************************************************************************
101	* Structures and Typedefs *
102	*********************************************************************************************************************************/
103	/**
104	* Branch types - iemCImpl_BranchTaskSegment(), iemCImpl_BranchTaskGate(),
105	* iemCImpl_BranchCallGate() and iemCImpl_BranchSysSel().
106	*/
107	typedef enum IEMBRANCH
108	{
109	IEMBRANCH_JUMP = 1,
110	IEMBRANCH_CALL,
111	IEMBRANCH_TRAP,
112	IEMBRANCH_SOFTWARE_INT,
113	IEMBRANCH_HARDWARE_INT
114	} IEMBRANCH;
115	AssertCompileSize(IEMBRANCH, 4);
116
117
118
119
120	/** @name Misc Helpers
121	* @{
122	*/
123
124
125	/**
126	* Worker function for iemHlpCheckPortIOPermission, don't call directly.
127	*
128	* @returns Strict VBox status code.
129	*
130	* @param pVCpu The cross context virtual CPU structure of the calling thread.
131	* @param u16Port The port number.
132	* @param cbOperand The operand size.
133	*/
134	static VBOXSTRICTRC iemHlpCheckPortIOPermissionBitmap(PVMCPUCC pVCpu, uint16_t u16Port, uint8_t cbOperand)
135	{
136	/* The TSS bits we're interested in are the same on 386 and AMD64. */
137	AssertCompile(AMD64_SEL_TYPE_SYS_TSS_BUSY == X86_SEL_TYPE_SYS_386_TSS_BUSY);
138	AssertCompile(AMD64_SEL_TYPE_SYS_TSS_AVAIL == X86_SEL_TYPE_SYS_386_TSS_AVAIL);
139	AssertCompileMembersAtSameOffset(X86TSS32, offIoBitmap, X86TSS64, offIoBitmap);
140	AssertCompile(sizeof(X86TSS32) == sizeof(X86TSS64));
141
142	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR);
143
144	/*
145	* Check the TSS type, 16-bit TSSes doesn't have any I/O permission bitmap.
146	*/
147	Assert(!pVCpu->cpum.GstCtx.tr.Attr.n.u1DescType);
148	if (RT_UNLIKELY( pVCpu->cpum.GstCtx.tr.Attr.n.u4Type != AMD64_SEL_TYPE_SYS_TSS_BUSY
149	&& pVCpu->cpum.GstCtx.tr.Attr.n.u4Type != AMD64_SEL_TYPE_SYS_TSS_AVAIL))
150	{
151	Log(("iemHlpCheckPortIOPermissionBitmap: Port=%#x cb=%d - TSS type %#x (attr=%#x) has no I/O bitmap -> #GP(0)\n",
152	u16Port, cbOperand, pVCpu->cpum.GstCtx.tr.Attr.n.u4Type, pVCpu->cpum.GstCtx.tr.Attr.u));
153	return iemRaiseGeneralProtectionFault0(pVCpu);
154	}
155
156	/*
157	* Read the bitmap offset (may #PF).
158	*/
159	uint16_t offBitmap;
160	VBOXSTRICTRC rcStrict = iemMemFetchSysU16(pVCpu, &offBitmap, UINT8_MAX,
161	pVCpu->cpum.GstCtx.tr.u64Base + RT_UOFFSETOF(X86TSS64, offIoBitmap));
162	if (rcStrict != VINF_SUCCESS)
163	{
164	Log(("iemHlpCheckPortIOPermissionBitmap: Error reading offIoBitmap (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
165	return rcStrict;
166	}
167
168	/*
169	* The bit range from u16Port to (u16Port + cbOperand - 1), however intel
170	* describes the CPU actually reading two bytes regardless of whether the
171	* bit range crosses a byte boundrary. Thus the + 1 in the test below.
172	*/
173	uint32_t offFirstBit = (uint32_t)u16Port / 8 + offBitmap;
174	/** @todo check if real CPUs ensures that offBitmap has a minimum value of
175	* for instance sizeof(X86TSS32). */
176	if (offFirstBit + 1 > pVCpu->cpum.GstCtx.tr.u32Limit) /* the limit is inclusive */
177	{
178	Log(("iemHlpCheckPortIOPermissionBitmap: offFirstBit=%#x + 1 is beyond u32Limit=%#x -> #GP(0)\n",
179	offFirstBit, pVCpu->cpum.GstCtx.tr.u32Limit));
180	return iemRaiseGeneralProtectionFault0(pVCpu);
181	}
182
183	/*
184	* Read the necessary bits.
185	*/
186	/** @todo Test the assertion in the intel manual that the CPU reads two
187	* bytes. The question is how this works wrt to \#PF and \#GP on the
188	* 2nd byte when it's not required. */
189	uint16_t bmBytes = UINT16_MAX;
190	rcStrict = iemMemFetchSysU16(pVCpu, &bmBytes, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + offFirstBit);
191	if (rcStrict != VINF_SUCCESS)
192	{
193	Log(("iemHlpCheckPortIOPermissionBitmap: Error reading I/O bitmap @%#x (%Rrc)\n", offFirstBit, VBOXSTRICTRC_VAL(rcStrict)));
194	return rcStrict;
195	}
196
197	/*
198	* Perform the check.
199	*/
200	uint16_t fPortMask = (1 << cbOperand) - 1;
201	bmBytes >>= (u16Port & 7);
202	if (bmBytes & fPortMask)
203	{
204	Log(("iemHlpCheckPortIOPermissionBitmap: u16Port=%#x LB %u - access denied (bm=%#x mask=%#x) -> #GP(0)\n",
205	u16Port, cbOperand, bmBytes, fPortMask));
206	return iemRaiseGeneralProtectionFault0(pVCpu);
207	}
208
209	return VINF_SUCCESS;
210	}
211
212
213	/**
214	* Checks if we are allowed to access the given I/O port, raising the
215	* appropriate exceptions if we aren't (or if the I/O bitmap is not
216	* accessible).
217	*
218	* @returns Strict VBox status code.
219	*
220	* @param pVCpu The cross context virtual CPU structure of the calling thread.
221	* @param u16Port The port number.
222	* @param cbOperand The operand size.
223	*/
224	DECLINLINE(VBOXSTRICTRC) iemHlpCheckPortIOPermission(PVMCPUCC pVCpu, uint16_t u16Port, uint8_t cbOperand)
225	{
226	X86EFLAGS Efl;
227	Efl.u = IEMMISC_GET_EFL(pVCpu);
228	if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)
229	&& ( IEM_GET_CPL(pVCpu) > Efl.Bits.u2IOPL
230	\|\| Efl.Bits.u1VM) )
231	return iemHlpCheckPortIOPermissionBitmap(pVCpu, u16Port, cbOperand);
232	return VINF_SUCCESS;
233	}
234
235
236	#if 0
237	/**
238	* Calculates the parity bit.
239	*
240	* @returns true if the bit is set, false if not.
241	* @param u8Result The least significant byte of the result.
242	*/
243	static bool iemHlpCalcParityFlag(uint8_t u8Result)
244	{
245	/*
246	* Parity is set if the number of bits in the least significant byte of
247	* the result is even.
248	*/
249	uint8_t cBits;
250	cBits = u8Result & 1; /* 0 */
251	u8Result >>= 1;
252	cBits += u8Result & 1;
253	u8Result >>= 1;
254	cBits += u8Result & 1;
255	u8Result >>= 1;
256	cBits += u8Result & 1;
257	u8Result >>= 1;
258	cBits += u8Result & 1; /* 4 */
259	u8Result >>= 1;
260	cBits += u8Result & 1;
261	u8Result >>= 1;
262	cBits += u8Result & 1;
263	u8Result >>= 1;
264	cBits += u8Result & 1;
265	return !(cBits & 1);
266	}
267	#endif /* not used */
268
269
270	/**
271	* Updates the specified flags according to a 8-bit result.
272	*
273	* @param pVCpu The cross context virtual CPU structure of the calling thread.
274	* @param u8Result The result to set the flags according to.
275	* @param fToUpdate The flags to update.
276	* @param fUndefined The flags that are specified as undefined.
277	*/
278	static void iemHlpUpdateArithEFlagsU8(PVMCPUCC pVCpu, uint8_t u8Result, uint32_t fToUpdate, uint32_t fUndefined)
279	{
280	uint32_t fEFlags = iemAImpl_test_u8(pVCpu->cpum.GstCtx.eflags.u, &u8Result, u8Result);
281	pVCpu->cpum.GstCtx.eflags.u &= ~(fToUpdate \| fUndefined);
282	pVCpu->cpum.GstCtx.eflags.u \|= (fToUpdate \| fUndefined) & fEFlags;
283	}
284
285
286	/**
287	* Updates the specified flags according to a 16-bit result.
288	*
289	* @param pVCpu The cross context virtual CPU structure of the calling thread.
290	* @param u16Result The result to set the flags according to.
291	* @param fToUpdate The flags to update.
292	* @param fUndefined The flags that are specified as undefined.
293	*/
294	static void iemHlpUpdateArithEFlagsU16(PVMCPUCC pVCpu, uint16_t u16Result, uint32_t fToUpdate, uint32_t fUndefined)
295	{
296	uint32_t fEFlags = iemAImpl_test_u16(pVCpu->cpum.GstCtx.eflags.u, &u16Result, u16Result);
297	pVCpu->cpum.GstCtx.eflags.u &= ~(fToUpdate \| fUndefined);
298	pVCpu->cpum.GstCtx.eflags.u \|= (fToUpdate \| fUndefined) & fEFlags;
299	}
300
301
302	/**
303	* Helper used by iret.
304	*
305	* @param pVCpu The cross context virtual CPU structure of the calling thread.
306	* @param uCpl The new CPL.
307	* @param pSReg Pointer to the segment register.
308	*/
309	static void iemHlpAdjustSelectorForNewCpl(PVMCPUCC pVCpu, uint8_t uCpl, PCPUMSELREG pSReg)
310	{
311	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
312	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_MASK);
313
314	if ( uCpl > pSReg->Attr.n.u2Dpl
315	&& pSReg->Attr.n.u1DescType /* code or data, not system */
316	&& (pSReg->Attr.n.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_CONF))
317	!= (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_CONF)) /* not conforming code */
318	iemHlpLoadNullDataSelectorProt(pVCpu, pSReg, 0);
319	}
320
321
322	/**
323	* Indicates that we have modified the FPU state.
324	*
325	* @param pVCpu The cross context virtual CPU structure of the calling thread.
326	*/
327	DECLINLINE(void) iemHlpUsedFpu(PVMCPUCC pVCpu)
328	{
329	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
330	}
331
332	/** @} */
333
334	/** @name C Implementations
335	* @{
336	*/
337
338
339	/**
340	* Implements a pop [mem16].
341	*/
342	IEM_CIMPL_DEF_2(iemCImpl_pop_mem16, uint16_t, iEffSeg, RTGCPTR, GCPtrEffDst)
343	{
344	uint16_t u16Value;
345	RTUINT64U TmpRsp;
346	TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
347	VBOXSTRICTRC rcStrict = iemMemStackPopU16Ex(pVCpu, &u16Value, &TmpRsp);
348	if (rcStrict == VINF_SUCCESS)
349	{
350	rcStrict = iemMemStoreDataU16(pVCpu, iEffSeg, GCPtrEffDst, u16Value);
351	if (rcStrict == VINF_SUCCESS)
352	{
353	pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
354	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
355	}
356	}
357	return rcStrict;
358
359	}
360
361
362	/**
363	* Implements a pop [mem32].
364	*/
365	IEM_CIMPL_DEF_2(iemCImpl_pop_mem32, uint16_t, iEffSeg, RTGCPTR, GCPtrEffDst)
366	{
367	uint32_t u32Value;
368	RTUINT64U TmpRsp;
369	TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
370	VBOXSTRICTRC rcStrict = iemMemStackPopU32Ex(pVCpu, &u32Value, &TmpRsp);
371	if (rcStrict == VINF_SUCCESS)
372	{
373	rcStrict = iemMemStoreDataU32(pVCpu, iEffSeg, GCPtrEffDst, u32Value);
374	if (rcStrict == VINF_SUCCESS)
375	{
376	pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
377	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
378	}
379	}
380	return rcStrict;
381
382	}
383
384
385	/**
386	* Implements a pop [mem64].
387	*/
388	IEM_CIMPL_DEF_2(iemCImpl_pop_mem64, uint16_t, iEffSeg, RTGCPTR, GCPtrEffDst)
389	{
390	uint64_t u64Value;
391	RTUINT64U TmpRsp;
392	TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
393	VBOXSTRICTRC rcStrict = iemMemStackPopU64Ex(pVCpu, &u64Value, &TmpRsp);
394	if (rcStrict == VINF_SUCCESS)
395	{
396	rcStrict = iemMemStoreDataU64(pVCpu, iEffSeg, GCPtrEffDst, u64Value);
397	if (rcStrict == VINF_SUCCESS)
398	{
399	pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
400	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
401	}
402	}
403	return rcStrict;
404
405	}
406
407
408	/**
409	* Implements a 16-bit popa.
410	*/
411	IEM_CIMPL_DEF_0(iemCImpl_popa_16)
412	{
413	RTGCPTR GCPtrStart = iemRegGetEffRsp(pVCpu);
414	RTGCPTR GCPtrLast = GCPtrStart + 15;
415	VBOXSTRICTRC rcStrict;
416
417	/*
418	* The docs are a bit hard to comprehend here, but it looks like we wrap
419	* around in real mode as long as none of the individual "popa" crosses the
420	* end of the stack segment. In protected mode we check the whole access
421	* in one go. For efficiency, only do the word-by-word thing if we're in
422	* danger of wrapping around.
423	*/
424	/** @todo do popa boundary / wrap-around checks. */
425	if (RT_UNLIKELY( IEM_IS_REAL_OR_V86_MODE(pVCpu)
426	&& (pVCpu->cpum.GstCtx.cs.u32Limit < GCPtrLast)) ) /* ASSUMES 64-bit RTGCPTR */
427	{
428	/* word-by-word */
429	RTUINT64U TmpRsp;
430	TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
431	rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.di, &TmpRsp);
432	if (rcStrict == VINF_SUCCESS)
433	rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.si, &TmpRsp);
434	if (rcStrict == VINF_SUCCESS)
435	rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.bp, &TmpRsp);
436	if (rcStrict == VINF_SUCCESS)
437	{
438	iemRegAddToRspEx(pVCpu, &TmpRsp, 2); /* sp */
439	rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.bx, &TmpRsp);
440	}
441	if (rcStrict == VINF_SUCCESS)
442	rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.dx, &TmpRsp);
443	if (rcStrict == VINF_SUCCESS)
444	rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.cx, &TmpRsp);
445	if (rcStrict == VINF_SUCCESS)
446	rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.ax, &TmpRsp);
447	if (rcStrict == VINF_SUCCESS)
448	{
449	pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
450	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
451	}
452	}
453	else
454	{
455	uint8_t bUnmapInfo;
456	uint16_t const *pau16Mem = NULL;
457	rcStrict = iemMemMap(pVCpu, (void **)&pau16Mem, &bUnmapInfo, 16, X86_SREG_SS, GCPtrStart,
458	IEM_ACCESS_STACK_R, sizeof(*pau16Mem) - 1);
459	if (rcStrict == VINF_SUCCESS)
460	{
461	pVCpu->cpum.GstCtx.di = pau16Mem[7 - X86_GREG_xDI];
462	pVCpu->cpum.GstCtx.si = pau16Mem[7 - X86_GREG_xSI];
463	pVCpu->cpum.GstCtx.bp = pau16Mem[7 - X86_GREG_xBP];
464	/* skip sp */
465	pVCpu->cpum.GstCtx.bx = pau16Mem[7 - X86_GREG_xBX];
466	pVCpu->cpum.GstCtx.dx = pau16Mem[7 - X86_GREG_xDX];
467	pVCpu->cpum.GstCtx.cx = pau16Mem[7 - X86_GREG_xCX];
468	pVCpu->cpum.GstCtx.ax = pau16Mem[7 - X86_GREG_xAX];
469	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
470	if (rcStrict == VINF_SUCCESS)
471	{
472	iemRegAddToRsp(pVCpu, 16);
473	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
474	}
475	}
476	}
477	return rcStrict;
478	}
479
480
481	/**
482	* Implements a 32-bit popa.
483	*/
484	IEM_CIMPL_DEF_0(iemCImpl_popa_32)
485	{
486	RTGCPTR GCPtrStart = iemRegGetEffRsp(pVCpu);
487	RTGCPTR GCPtrLast = GCPtrStart + 31;
488	VBOXSTRICTRC rcStrict;
489
490	/*
491	* The docs are a bit hard to comprehend here, but it looks like we wrap
492	* around in real mode as long as none of the individual "popa" crosses the
493	* end of the stack segment. In protected mode we check the whole access
494	* in one go. For efficiency, only do the word-by-word thing if we're in
495	* danger of wrapping around.
496	*/
497	/** @todo do popa boundary / wrap-around checks. */
498	if (RT_UNLIKELY( IEM_IS_REAL_OR_V86_MODE(pVCpu)
499	&& (pVCpu->cpum.GstCtx.cs.u32Limit < GCPtrLast)) ) /* ASSUMES 64-bit RTGCPTR */
500	{
501	/* word-by-word */
502	RTUINT64U TmpRsp;
503	TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
504	rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.edi, &TmpRsp);
505	if (rcStrict == VINF_SUCCESS)
506	rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.esi, &TmpRsp);
507	if (rcStrict == VINF_SUCCESS)
508	rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.ebp, &TmpRsp);
509	if (rcStrict == VINF_SUCCESS)
510	{
511	iemRegAddToRspEx(pVCpu, &TmpRsp, 2); /* sp */
512	rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.ebx, &TmpRsp);
513	}
514	if (rcStrict == VINF_SUCCESS)
515	rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.edx, &TmpRsp);
516	if (rcStrict == VINF_SUCCESS)
517	rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.ecx, &TmpRsp);
518	if (rcStrict == VINF_SUCCESS)
519	rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.eax, &TmpRsp);
520	if (rcStrict == VINF_SUCCESS)
521	{
522	#if 1 /** @todo what actually happens with the high bits when we're in 16-bit mode? */
523	pVCpu->cpum.GstCtx.rdi &= UINT32_MAX;
524	pVCpu->cpum.GstCtx.rsi &= UINT32_MAX;
525	pVCpu->cpum.GstCtx.rbp &= UINT32_MAX;
526	pVCpu->cpum.GstCtx.rbx &= UINT32_MAX;
527	pVCpu->cpum.GstCtx.rdx &= UINT32_MAX;
528	pVCpu->cpum.GstCtx.rcx &= UINT32_MAX;
529	pVCpu->cpum.GstCtx.rax &= UINT32_MAX;
530	#endif
531	pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
532	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
533	}
534	}
535	else
536	{
537	uint8_t bUnmapInfo;
538	uint32_t const *pau32Mem;
539	rcStrict = iemMemMap(pVCpu, (void **)&pau32Mem, &bUnmapInfo, 32, X86_SREG_SS, GCPtrStart,
540	IEM_ACCESS_STACK_R, sizeof(*pau32Mem) - 1);
541	if (rcStrict == VINF_SUCCESS)
542	{
543	pVCpu->cpum.GstCtx.rdi = pau32Mem[7 - X86_GREG_xDI];
544	pVCpu->cpum.GstCtx.rsi = pau32Mem[7 - X86_GREG_xSI];
545	pVCpu->cpum.GstCtx.rbp = pau32Mem[7 - X86_GREG_xBP];
546	/* skip esp */
547	pVCpu->cpum.GstCtx.rbx = pau32Mem[7 - X86_GREG_xBX];
548	pVCpu->cpum.GstCtx.rdx = pau32Mem[7 - X86_GREG_xDX];
549	pVCpu->cpum.GstCtx.rcx = pau32Mem[7 - X86_GREG_xCX];
550	pVCpu->cpum.GstCtx.rax = pau32Mem[7 - X86_GREG_xAX];
551	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
552	if (rcStrict == VINF_SUCCESS)
553	{
554	iemRegAddToRsp(pVCpu, 32);
555	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
556	}
557	}
558	}
559	return rcStrict;
560	}
561
562
563	/**
564	* Implements a 16-bit pusha.
565	*/
566	IEM_CIMPL_DEF_0(iemCImpl_pusha_16)
567	{
568	RTGCPTR GCPtrTop = iemRegGetEffRsp(pVCpu);
569	RTGCPTR GCPtrBottom = GCPtrTop - 15;
570	VBOXSTRICTRC rcStrict;
571
572	/*
573	* The docs are a bit hard to comprehend here, but it looks like we wrap
574	* around in real mode as long as none of the individual "pushd" crosses the
575	* end of the stack segment. In protected mode we check the whole access
576	* in one go. For efficiency, only do the word-by-word thing if we're in
577	* danger of wrapping around.
578	*/
579	/** @todo do pusha boundary / wrap-around checks. */
580	if (RT_UNLIKELY( GCPtrBottom > GCPtrTop
581	&& IEM_IS_REAL_OR_V86_MODE(pVCpu) ) )
582	{
583	/* word-by-word */
584	RTUINT64U TmpRsp;
585	TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
586	rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.ax, &TmpRsp);
587	if (rcStrict == VINF_SUCCESS)
588	rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.cx, &TmpRsp);
589	if (rcStrict == VINF_SUCCESS)
590	rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.dx, &TmpRsp);
591	if (rcStrict == VINF_SUCCESS)
592	rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.bx, &TmpRsp);
593	if (rcStrict == VINF_SUCCESS)
594	rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.sp, &TmpRsp);
595	if (rcStrict == VINF_SUCCESS)
596	rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.bp, &TmpRsp);
597	if (rcStrict == VINF_SUCCESS)
598	rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.si, &TmpRsp);
599	if (rcStrict == VINF_SUCCESS)
600	rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.di, &TmpRsp);
601	if (rcStrict == VINF_SUCCESS)
602	{
603	pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
604	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
605	}
606	}
607	else
608	{
609	GCPtrBottom--;
610	uint8_t bUnmapInfo;
611	uint16_t *pau16Mem = NULL;
612	rcStrict = iemMemMap(pVCpu, (void **)&pau16Mem, &bUnmapInfo, 16, X86_SREG_SS, GCPtrBottom,
613	IEM_ACCESS_STACK_W, sizeof(*pau16Mem) - 1);
614	if (rcStrict == VINF_SUCCESS)
615	{
616	pau16Mem[7 - X86_GREG_xDI] = pVCpu->cpum.GstCtx.di;
617	pau16Mem[7 - X86_GREG_xSI] = pVCpu->cpum.GstCtx.si;
618	pau16Mem[7 - X86_GREG_xBP] = pVCpu->cpum.GstCtx.bp;
619	pau16Mem[7 - X86_GREG_xSP] = pVCpu->cpum.GstCtx.sp;
620	pau16Mem[7 - X86_GREG_xBX] = pVCpu->cpum.GstCtx.bx;
621	pau16Mem[7 - X86_GREG_xDX] = pVCpu->cpum.GstCtx.dx;
622	pau16Mem[7 - X86_GREG_xCX] = pVCpu->cpum.GstCtx.cx;
623	pau16Mem[7 - X86_GREG_xAX] = pVCpu->cpum.GstCtx.ax;
624	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
625	if (rcStrict == VINF_SUCCESS)
626	{
627	iemRegSubFromRsp(pVCpu, 16);
628	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
629	}
630	}
631	}
632	return rcStrict;
633	}
634
635
636	/**
637	* Implements a 32-bit pusha.
638	*/
639	IEM_CIMPL_DEF_0(iemCImpl_pusha_32)
640	{
641	RTGCPTR GCPtrTop = iemRegGetEffRsp(pVCpu);
642	RTGCPTR GCPtrBottom = GCPtrTop - 31;
643	VBOXSTRICTRC rcStrict;
644
645	/*
646	* The docs are a bit hard to comprehend here, but it looks like we wrap
647	* around in real mode as long as none of the individual "pusha" crosses the
648	* end of the stack segment. In protected mode we check the whole access
649	* in one go. For efficiency, only do the word-by-word thing if we're in
650	* danger of wrapping around.
651	*/
652	/** @todo do pusha boundary / wrap-around checks. */
653	if (RT_UNLIKELY( GCPtrBottom > GCPtrTop
654	&& IEM_IS_REAL_OR_V86_MODE(pVCpu) ) )
655	{
656	/* word-by-word */
657	RTUINT64U TmpRsp;
658	TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
659	rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.eax, &TmpRsp);
660	if (rcStrict == VINF_SUCCESS)
661	rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.ecx, &TmpRsp);
662	if (rcStrict == VINF_SUCCESS)
663	rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.edx, &TmpRsp);
664	if (rcStrict == VINF_SUCCESS)
665	rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.ebx, &TmpRsp);
666	if (rcStrict == VINF_SUCCESS)
667	rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.esp, &TmpRsp);
668	if (rcStrict == VINF_SUCCESS)
669	rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.ebp, &TmpRsp);
670	if (rcStrict == VINF_SUCCESS)
671	rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.esi, &TmpRsp);
672	if (rcStrict == VINF_SUCCESS)
673	rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.edi, &TmpRsp);
674	if (rcStrict == VINF_SUCCESS)
675	{
676	pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
677	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
678	}
679	}
680	else
681	{
682	GCPtrBottom--;
683	uint8_t bUnmapInfo;
684	uint32_t *pau32Mem;
685	rcStrict = iemMemMap(pVCpu, (void **)&pau32Mem, &bUnmapInfo, 32, X86_SREG_SS, GCPtrBottom,
686	IEM_ACCESS_STACK_W, sizeof(*pau32Mem) - 1);
687	if (rcStrict == VINF_SUCCESS)
688	{
689	pau32Mem[7 - X86_GREG_xDI] = pVCpu->cpum.GstCtx.edi;
690	pau32Mem[7 - X86_GREG_xSI] = pVCpu->cpum.GstCtx.esi;
691	pau32Mem[7 - X86_GREG_xBP] = pVCpu->cpum.GstCtx.ebp;
692	pau32Mem[7 - X86_GREG_xSP] = pVCpu->cpum.GstCtx.esp;
693	pau32Mem[7 - X86_GREG_xBX] = pVCpu->cpum.GstCtx.ebx;
694	pau32Mem[7 - X86_GREG_xDX] = pVCpu->cpum.GstCtx.edx;
695	pau32Mem[7 - X86_GREG_xCX] = pVCpu->cpum.GstCtx.ecx;
696	pau32Mem[7 - X86_GREG_xAX] = pVCpu->cpum.GstCtx.eax;
697	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
698	if (rcStrict == VINF_SUCCESS)
699	{
700	iemRegSubFromRsp(pVCpu, 32);
701	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
702	}
703	}
704	}
705	return rcStrict;
706	}
707
708
709	/**
710	* Implements pushf.
711	*
712	*
713	* @param enmEffOpSize The effective operand size.
714	*/
715	IEM_CIMPL_DEF_1(iemCImpl_pushf, IEMMODE, enmEffOpSize)
716	{
717	VBOXSTRICTRC rcStrict;
718
719	if (!IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_PUSHF))
720	{ /* probable */ }
721	else
722	{
723	Log2(("pushf: Guest intercept -> #VMEXIT\n"));
724	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
725	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_PUSHF, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
726	}
727
728	/*
729	* If we're in V8086 mode some care is required (which is why we're in
730	* doing this in a C implementation).
731	*/
732	uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
733	if ( (fEfl & X86_EFL_VM)
734	&& X86_EFL_GET_IOPL(fEfl) != 3 )
735	{
736	Assert(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE);
737	if ( enmEffOpSize != IEMMODE_16BIT
738	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_VME))
739	return iemRaiseGeneralProtectionFault0(pVCpu);
740	fEfl &= ~X86_EFL_IF; /* (RF and VM are out of range) */
741	fEfl \|= (fEfl & X86_EFL_VIF) >> (19 - 9);
742	rcStrict = iemMemStackPushU16(pVCpu, (uint16_t)fEfl);
743	}
744	else
745	{
746
747	/*
748	* Ok, clear RF and VM, adjust for ancient CPUs, and push the flags.
749	*/
750	fEfl &= ~(X86_EFL_RF \| X86_EFL_VM);
751
752	switch (enmEffOpSize)
753	{
754	case IEMMODE_16BIT:
755	AssertCompile(IEMTARGETCPU_8086 <= IEMTARGETCPU_186 && IEMTARGETCPU_V20 <= IEMTARGETCPU_186 && IEMTARGETCPU_286 > IEMTARGETCPU_186);
756	if (IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_186)
757	fEfl \|= UINT16_C(0xf000);
758	rcStrict = iemMemStackPushU16(pVCpu, (uint16_t)fEfl);
759	break;
760	case IEMMODE_32BIT:
761	rcStrict = iemMemStackPushU32(pVCpu, fEfl);
762	break;
763	case IEMMODE_64BIT:
764	rcStrict = iemMemStackPushU64(pVCpu, fEfl);
765	break;
766	IEM_NOT_REACHED_DEFAULT_CASE_RET();
767	}
768	}
769
770	if (rcStrict == VINF_SUCCESS)
771	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
772	return rcStrict;
773	}
774
775
776	/**
777	* Implements popf.
778	*
779	* @param enmEffOpSize The effective operand size.
780	*/
781	IEM_CIMPL_DEF_1(iemCImpl_popf, IEMMODE, enmEffOpSize)
782	{
783	uint32_t const fEflOld = IEMMISC_GET_EFL(pVCpu);
784	VBOXSTRICTRC rcStrict;
785	uint32_t fEflNew;
786
787	if (!IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_POPF))
788	{ /* probable */ }
789	else
790	{
791	Log2(("popf: Guest intercept -> #VMEXIT\n"));
792	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
793	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_POPF, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
794	}
795
796	/*
797	* V8086 is special as usual.
798	*/
799	if (fEflOld & X86_EFL_VM)
800	{
801	/*
802	* Almost anything goes if IOPL is 3.
803	*/
804	if (X86_EFL_GET_IOPL(fEflOld) == 3)
805	{
806	switch (enmEffOpSize)
807	{
808	case IEMMODE_16BIT:
809	{
810	uint16_t u16Value;
811	rcStrict = iemMemStackPopU16(pVCpu, &u16Value);
812	if (rcStrict != VINF_SUCCESS)
813	return rcStrict;
814	fEflNew = u16Value \| (fEflOld & UINT32_C(0xffff0000));
815	break;
816	}
817	case IEMMODE_32BIT:
818	rcStrict = iemMemStackPopU32(pVCpu, &fEflNew);
819	if (rcStrict != VINF_SUCCESS)
820	return rcStrict;
821	break;
822	IEM_NOT_REACHED_DEFAULT_CASE_RET();
823	}
824
825	const uint32_t fPopfBits = pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.enmMicroarch != kCpumMicroarch_Intel_80386
826	? X86_EFL_POPF_BITS : X86_EFL_POPF_BITS_386;
827	fEflNew &= fPopfBits & ~(X86_EFL_IOPL);
828	fEflNew \|= ~(fPopfBits & ~(X86_EFL_IOPL)) & fEflOld;
829	}
830	/*
831	* Interrupt flag virtualization with CR4.VME=1.
832	*/
833	else if ( enmEffOpSize == IEMMODE_16BIT
834	&& (pVCpu->cpum.GstCtx.cr4 & X86_CR4_VME) )
835	{
836	uint16_t u16Value;
837	RTUINT64U TmpRsp;
838	TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
839	rcStrict = iemMemStackPopU16Ex(pVCpu, &u16Value, &TmpRsp);
840	if (rcStrict != VINF_SUCCESS)
841	return rcStrict;
842
843	if ( ( (u16Value & X86_EFL_IF)
844	&& (fEflOld & X86_EFL_VIP))
845	\|\| (u16Value & X86_EFL_TF) )
846	return iemRaiseGeneralProtectionFault0(pVCpu);
847
848	fEflNew = X86_EFL_RA1_MASK
849	\| (u16Value & ~(X86_EFL_IF \| X86_EFL_IOPL \| X86_EFL_RAZ_MASK))
850	\| (fEflOld & (UINT32_C(0xffff0000) \| X86_EFL_IF \| X86_EFL_IOPL) & ~(X86_EFL_VIF \| X86_EFL_RF))
851	\| ((uint32_t)(u16Value & X86_EFL_IF) << (X86_EFL_VIF_BIT - X86_EFL_IF_BIT));
852
853	pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
854	}
855	else
856	return iemRaiseGeneralProtectionFault0(pVCpu);
857
858	}
859	/*
860	* Not in V8086 mode.
861	*/
862	else
863	{
864	/* Pop the flags. */
865	switch (enmEffOpSize)
866	{
867	case IEMMODE_16BIT:
868	{
869	uint16_t u16Value;
870	rcStrict = iemMemStackPopU16(pVCpu, &u16Value);
871	if (rcStrict != VINF_SUCCESS)
872	return rcStrict;
873	fEflNew = u16Value \| (fEflOld & UINT32_C(0xffff0000));
874
875	/*
876	* Ancient CPU adjustments:
877	* - 8086, 80186, V20/30:
878	* Fixed bits 15:12 bits are not kept correctly internally, mostly for
879	* practical reasons (masking below). We add them when pushing flags.
880	* - 80286:
881	* The NT and IOPL flags cannot be popped from real mode and are
882	* therefore always zero (since a 286 can never exit from PM and
883	* their initial value is zero). This changed on a 386 and can
884	* therefore be used to detect 286 or 386 CPU in real mode.
885	*/
886	if ( IEM_GET_TARGET_CPU(pVCpu) == IEMTARGETCPU_286
887	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
888	fEflNew &= ~(X86_EFL_NT \| X86_EFL_IOPL);
889	break;
890	}
891	case IEMMODE_32BIT:
892	rcStrict = iemMemStackPopU32(pVCpu, &fEflNew);
893	if (rcStrict != VINF_SUCCESS)
894	return rcStrict;
895	break;
896	case IEMMODE_64BIT:
897	{
898	uint64_t u64Value;
899	rcStrict = iemMemStackPopU64(pVCpu, &u64Value);
900	if (rcStrict != VINF_SUCCESS)
901	return rcStrict;
902	fEflNew = u64Value; /** @todo testcase: Check exactly what happens if high bits are set. */
903	break;
904	}
905	IEM_NOT_REACHED_DEFAULT_CASE_RET();
906	}
907
908	/* Merge them with the current flags. */
909	const uint32_t fPopfBits = pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.enmMicroarch != kCpumMicroarch_Intel_80386
910	? X86_EFL_POPF_BITS : X86_EFL_POPF_BITS_386;
911	if ( (fEflNew & (X86_EFL_IOPL \| X86_EFL_IF)) == (fEflOld & (X86_EFL_IOPL \| X86_EFL_IF))
912	\|\| IEM_GET_CPL(pVCpu) == 0)
913	{
914	fEflNew &= fPopfBits;
915	fEflNew \|= ~fPopfBits & fEflOld;
916	}
917	else if (IEM_GET_CPL(pVCpu) <= X86_EFL_GET_IOPL(fEflOld))
918	{
919	fEflNew &= fPopfBits & ~(X86_EFL_IOPL);
920	fEflNew \|= ~(fPopfBits & ~(X86_EFL_IOPL)) & fEflOld;
921	}
922	else
923	{
924	fEflNew &= fPopfBits & ~(X86_EFL_IOPL \| X86_EFL_IF);
925	fEflNew \|= ~(fPopfBits & ~(X86_EFL_IOPL \| X86_EFL_IF)) & fEflOld;
926	}
927	}
928
929	/*
930	* Commit the flags.
931	*/
932	Assert(fEflNew & RT_BIT_32(1));
933	IEMMISC_SET_EFL(pVCpu, fEflNew);
934	pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~IEM_F_X86_AC) \| iemCalcExecAcFlag(pVCpu);
935	return iemRegAddToRipAndFinishingClearingRfEx(pVCpu, cbInstr, fEflOld);
936	}
937
938
939	/**
940	* Implements far jumps and calls thru task segments (TSS).
941	*
942	* @returns VBox strict status code.
943	* @param pVCpu The cross context virtual CPU structure of the
944	* calling thread.
945	* @param cbInstr The current instruction length.
946	* @param uSel The selector.
947	* @param enmBranch The kind of branching we're performing.
948	* @param enmEffOpSize The effective operand size.
949	* @param pDesc The descriptor corresponding to @a uSel. The type is
950	* task gate.
951	*/
952	static VBOXSTRICTRC iemCImpl_BranchTaskSegment(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uSel, IEMBRANCH enmBranch,
953	IEMMODE enmEffOpSize, PIEMSELDESC pDesc)
954	{
955	#ifndef IEM_IMPLEMENTS_TASKSWITCH
956	IEM_RETURN_ASPECT_NOT_IMPLEMENTED();
957	#else
958	Assert(enmBranch == IEMBRANCH_JUMP \|\| enmBranch == IEMBRANCH_CALL);
959	Assert( pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL
960	\|\| pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL);
961	RT_NOREF_PV(enmEffOpSize);
962	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
963
964	if ( pDesc->Legacy.Gate.u2Dpl < IEM_GET_CPL(pVCpu)
965	\|\| pDesc->Legacy.Gate.u2Dpl < (uSel & X86_SEL_RPL))
966	{
967	Log(("BranchTaskSegment invalid priv. uSel=%04x TSS DPL=%d CPL=%u Sel RPL=%u -> #GP\n", uSel, pDesc->Legacy.Gate.u2Dpl,
968	IEM_GET_CPL(pVCpu), (uSel & X86_SEL_RPL)));
969	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
970	}
971
972	/** @todo This is checked earlier for far jumps (see iemCImpl_FarJmp) but not
973	* far calls (see iemCImpl_callf). Most likely in both cases it should be
974	* checked here, need testcases. */
975	if (!pDesc->Legacy.Gen.u1Present)
976	{
977	Log(("BranchTaskSegment TSS not present uSel=%04x -> #NP\n", uSel));
978	return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
979	}
980
981	uint32_t uNextEip = pVCpu->cpum.GstCtx.eip + cbInstr;
982	return iemTaskSwitch(pVCpu, enmBranch == IEMBRANCH_JUMP ? IEMTASKSWITCH_JUMP : IEMTASKSWITCH_CALL,
983	uNextEip, 0 /* fFlags /, 0 / uErr /, 0 / uCr2 */, uSel, pDesc);
984	#endif
985	}
986
987
988	/**
989	* Implements far jumps and calls thru task gates.
990	*
991	* @returns VBox strict status code.
992	* @param pVCpu The cross context virtual CPU structure of the
993	* calling thread.
994	* @param cbInstr The current instruction length.
995	* @param uSel The selector.
996	* @param enmBranch The kind of branching we're performing.
997	* @param enmEffOpSize The effective operand size.
998	* @param pDesc The descriptor corresponding to @a uSel. The type is
999	* task gate.
1000	*/
1001	static VBOXSTRICTRC iemCImpl_BranchTaskGate(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uSel, IEMBRANCH enmBranch,
1002	IEMMODE enmEffOpSize, PIEMSELDESC pDesc)
1003	{
1004	#ifndef IEM_IMPLEMENTS_TASKSWITCH
1005	IEM_RETURN_ASPECT_NOT_IMPLEMENTED();
1006	#else
1007	Assert(enmBranch == IEMBRANCH_JUMP \|\| enmBranch == IEMBRANCH_CALL);
1008	RT_NOREF_PV(enmEffOpSize);
1009	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
1010
1011	if ( pDesc->Legacy.Gate.u2Dpl < IEM_GET_CPL(pVCpu)
1012	\|\| pDesc->Legacy.Gate.u2Dpl < (uSel & X86_SEL_RPL))
1013	{
1014	Log(("BranchTaskGate invalid priv. uSel=%04x TSS DPL=%d CPL=%u Sel RPL=%u -> #GP\n", uSel, pDesc->Legacy.Gate.u2Dpl,
1015	IEM_GET_CPL(pVCpu), (uSel & X86_SEL_RPL)));
1016	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
1017	}
1018
1019	/** @todo This is checked earlier for far jumps (see iemCImpl_FarJmp) but not
1020	* far calls (see iemCImpl_callf). Most likely in both cases it should be
1021	* checked here, need testcases. */
1022	if (!pDesc->Legacy.Gen.u1Present)
1023	{
1024	Log(("BranchTaskSegment segment not present uSel=%04x -> #NP\n", uSel));
1025	return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
1026	}
1027
1028	/*
1029	* Fetch the new TSS descriptor from the GDT.
1030	*/
1031	RTSEL uSelTss = pDesc->Legacy.Gate.u16Sel;
1032	if (uSelTss & X86_SEL_LDT)
1033	{
1034	Log(("BranchTaskGate TSS is in LDT. uSel=%04x uSelTss=%04x -> #GP\n", uSel, uSelTss));
1035	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
1036	}
1037
1038	IEMSELDESC TssDesc;
1039	VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &TssDesc, uSelTss, X86_XCPT_GP);
1040	if (rcStrict != VINF_SUCCESS)
1041	return rcStrict;
1042
1043	if (TssDesc.Legacy.Gate.u4Type & X86_SEL_TYPE_SYS_TSS_BUSY_MASK)
1044	{
1045	Log(("BranchTaskGate TSS is busy. uSel=%04x uSelTss=%04x DescType=%#x -> #GP\n", uSel, uSelTss,
1046	TssDesc.Legacy.Gate.u4Type));
1047	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
1048	}
1049
1050	if (!TssDesc.Legacy.Gate.u1Present)
1051	{
1052	Log(("BranchTaskGate TSS is not present. uSel=%04x uSelTss=%04x -> #NP\n", uSel, uSelTss));
1053	return iemRaiseSelectorNotPresentBySelector(pVCpu, uSelTss & X86_SEL_MASK_OFF_RPL);
1054	}
1055
1056	uint32_t uNextEip = pVCpu->cpum.GstCtx.eip + cbInstr;
1057	return iemTaskSwitch(pVCpu, enmBranch == IEMBRANCH_JUMP ? IEMTASKSWITCH_JUMP : IEMTASKSWITCH_CALL,
1058	uNextEip, 0 /* fFlags /, 0 / uErr /, 0 / uCr2 */, uSelTss, &TssDesc);
1059	#endif
1060	}
1061
1062
1063	/**
1064	* Implements far jumps and calls thru call gates.
1065	*
1066	* @returns VBox strict status code.
1067	* @param pVCpu The cross context virtual CPU structure of the
1068	* calling thread.
1069	* @param cbInstr The current instruction length.
1070	* @param uSel The selector.
1071	* @param enmBranch The kind of branching we're performing.
1072	* @param enmEffOpSize The effective operand size.
1073	* @param pDesc The descriptor corresponding to @a uSel. The type is
1074	* call gate.
1075	*/
1076	static VBOXSTRICTRC iemCImpl_BranchCallGate(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uSel, IEMBRANCH enmBranch,
1077	IEMMODE enmEffOpSize, PIEMSELDESC pDesc)
1078	{
1079	#define IEM_IMPLEMENTS_CALLGATE
1080	#ifndef IEM_IMPLEMENTS_CALLGATE
1081	IEM_RETURN_ASPECT_NOT_IMPLEMENTED();
1082	#else
1083	RT_NOREF_PV(enmEffOpSize);
1084	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
1085
1086	/* NB: Far jumps can only do intra-privilege transfers. Far calls support
1087	* inter-privilege calls and are much more complex.
1088	*
1089	* NB: 64-bit call gate has the same type as a 32-bit call gate! If
1090	* EFER.LMA=1, the gate must be 64-bit. Conversely if EFER.LMA=0, the gate
1091	* must be 16-bit or 32-bit.
1092	*/
1093	/** @todo effective operand size is probably irrelevant here, only the
1094	* call gate bitness matters??
1095	*/
1096	VBOXSTRICTRC rcStrict;
1097	RTPTRUNION uPtrRet;
1098	uint64_t uNewRsp;
1099	uint64_t uNewRip;
1100	uint64_t u64Base;
1101	uint32_t cbLimit;
1102	RTSEL uNewCS;
1103	IEMSELDESC DescCS;
1104
1105	AssertCompile(X86_SEL_TYPE_SYS_386_CALL_GATE == AMD64_SEL_TYPE_SYS_CALL_GATE);
1106	Assert(enmBranch == IEMBRANCH_JUMP \|\| enmBranch == IEMBRANCH_CALL);
1107	Assert( pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE
1108	\|\| pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE);
1109
1110	/* Determine the new instruction pointer from the gate descriptor. */
1111	uNewRip = pDesc->Legacy.Gate.u16OffsetLow
1112	\| ((uint32_t)pDesc->Legacy.Gate.u16OffsetHigh << 16)
1113	\| ((uint64_t)pDesc->Long.Gate.u32OffsetTop << 32);
1114
1115	/* Perform DPL checks on the gate descriptor. */
1116	if ( pDesc->Legacy.Gate.u2Dpl < IEM_GET_CPL(pVCpu)
1117	\|\| pDesc->Legacy.Gate.u2Dpl < (uSel & X86_SEL_RPL))
1118	{
1119	Log(("BranchCallGate invalid priv. uSel=%04x Gate DPL=%d CPL=%u Sel RPL=%u -> #GP\n", uSel, pDesc->Legacy.Gate.u2Dpl,
1120	IEM_GET_CPL(pVCpu), (uSel & X86_SEL_RPL)));
1121	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
1122	}
1123
1124	/** @todo does this catch NULL selectors, too? */
1125	if (!pDesc->Legacy.Gen.u1Present)
1126	{
1127	Log(("BranchCallGate Gate not present uSel=%04x -> #NP\n", uSel));
1128	return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel);
1129	}
1130
1131	/*
1132	* Fetch the target CS descriptor from the GDT or LDT.
1133	*/
1134	uNewCS = pDesc->Legacy.Gate.u16Sel;
1135	rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, uNewCS, X86_XCPT_GP);
1136	if (rcStrict != VINF_SUCCESS)
1137	return rcStrict;
1138
1139	/* Target CS must be a code selector. */
1140	if ( !DescCS.Legacy.Gen.u1DescType
1141	\|\| !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE) )
1142	{
1143	Log(("BranchCallGate %04x:%08RX64 -> not a code selector (u1DescType=%u u4Type=%#x).\n",
1144	uNewCS, uNewRip, DescCS.Legacy.Gen.u1DescType, DescCS.Legacy.Gen.u4Type));
1145	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS);
1146	}
1147
1148	/* Privilege checks on target CS. */
1149	if (enmBranch == IEMBRANCH_JUMP)
1150	{
1151	if (DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
1152	{
1153	if (DescCS.Legacy.Gen.u2Dpl > IEM_GET_CPL(pVCpu))
1154	{
1155	Log(("BranchCallGate jump (conforming) bad DPL uNewCS=%04x Gate DPL=%d CPL=%u -> #GP\n",
1156	uNewCS, DescCS.Legacy.Gen.u2Dpl, IEM_GET_CPL(pVCpu)));
1157	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS);
1158	}
1159	}
1160	else
1161	{
1162	if (DescCS.Legacy.Gen.u2Dpl != IEM_GET_CPL(pVCpu))
1163	{
1164	Log(("BranchCallGate jump (non-conforming) bad DPL uNewCS=%04x Gate DPL=%d CPL=%u -> #GP\n",
1165	uNewCS, DescCS.Legacy.Gen.u2Dpl, IEM_GET_CPL(pVCpu)));
1166	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS);
1167	}
1168	}
1169	}
1170	else
1171	{
1172	Assert(enmBranch == IEMBRANCH_CALL);
1173	if (DescCS.Legacy.Gen.u2Dpl > IEM_GET_CPL(pVCpu))
1174	{
1175	Log(("BranchCallGate call invalid priv. uNewCS=%04x Gate DPL=%d CPL=%u -> #GP\n",
1176	uNewCS, DescCS.Legacy.Gen.u2Dpl, IEM_GET_CPL(pVCpu)));
1177	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
1178	}
1179	}
1180
1181	/* Additional long mode checks. */
1182	if (IEM_IS_LONG_MODE(pVCpu))
1183	{
1184	if (!DescCS.Legacy.Gen.u1Long)
1185	{
1186	Log(("BranchCallGate uNewCS %04x -> not a 64-bit code segment.\n", uNewCS));
1187	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS);
1188	}
1189
1190	/* L vs D. */
1191	if ( DescCS.Legacy.Gen.u1Long
1192	&& DescCS.Legacy.Gen.u1DefBig)
1193	{
1194	Log(("BranchCallGate uNewCS %04x -> both L and D are set.\n", uNewCS));
1195	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS);
1196	}
1197	}
1198
1199	if (!DescCS.Legacy.Gate.u1Present)
1200	{
1201	Log(("BranchCallGate target CS is not present. uSel=%04x uNewCS=%04x -> #NP(CS)\n", uSel, uNewCS));
1202	return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewCS);
1203	}
1204
1205	if (enmBranch == IEMBRANCH_JUMP)
1206	{
1207	/** @todo This is very similar to regular far jumps; merge! */
1208	/* Jumps are fairly simple... */
1209
1210	/* Chop the high bits off if 16-bit gate (Intel says so). */
1211	if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE)
1212	uNewRip = (uint16_t)uNewRip;
1213
1214	/* Limit check for non-long segments. */
1215	cbLimit = X86DESC_LIMIT_G(&DescCS.Legacy);
1216	if (DescCS.Legacy.Gen.u1Long)
1217	u64Base = 0;
1218	else
1219	{
1220	if (uNewRip > cbLimit)
1221	{
1222	Log(("BranchCallGate jump %04x:%08RX64 -> out of bounds (%#x) -> #GP(0)\n", uNewCS, uNewRip, cbLimit));
1223	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, 0);
1224	}
1225	u64Base = X86DESC_BASE(&DescCS.Legacy);
1226	}
1227
1228	/* Canonical address check. */
1229	if (!IEM_IS_CANONICAL(uNewRip))
1230	{
1231	Log(("BranchCallGate jump %04x:%016RX64 - not canonical -> #GP\n", uNewCS, uNewRip));
1232	return iemRaiseNotCanonical(pVCpu);
1233	}
1234
1235	/*
1236	* Ok, everything checked out fine. Now set the accessed bit before
1237	* committing the result into CS, CSHID and RIP.
1238	*/
1239	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
1240	{
1241	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCS);
1242	if (rcStrict != VINF_SUCCESS)
1243	return rcStrict;
1244	/** @todo check what VT-x and AMD-V does. */
1245	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
1246	}
1247
1248	/* commit */
1249	pVCpu->cpum.GstCtx.rip = uNewRip;
1250	pVCpu->cpum.GstCtx.cs.Sel = uNewCS & X86_SEL_MASK_OFF_RPL;
1251	pVCpu->cpum.GstCtx.cs.Sel \|= IEM_GET_CPL(pVCpu); /** @todo is this right for conforming segs? or in general? */
1252	pVCpu->cpum.GstCtx.cs.ValidSel = pVCpu->cpum.GstCtx.cs.Sel;
1253	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
1254	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
1255	pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit;
1256	pVCpu->cpum.GstCtx.cs.u64Base = u64Base;
1257	}
1258	else
1259	{
1260	Assert(enmBranch == IEMBRANCH_CALL);
1261	/* Calls are much more complicated. */
1262
1263	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF) && (DescCS.Legacy.Gen.u2Dpl < IEM_GET_CPL(pVCpu)))
1264	{
1265	/* More privilege. This is the fun part. */
1266	Assert(!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)); /* Filtered out above. */
1267
1268	/*
1269	* Determine new SS:rSP from the TSS.
1270	*/
1271	Assert(!pVCpu->cpum.GstCtx.tr.Attr.n.u1DescType);
1272
1273	/* Figure out where the new stack pointer is stored in the TSS. */
1274	uint8_t const uNewCSDpl = DescCS.Legacy.Gen.u2Dpl;
1275	uint16_t offNewStack; /* Offset of new stack in TSS. */
1276	uint16_t cbNewStack; /* Number of bytes the stack information takes up in TSS. */
1277	if (!IEM_IS_LONG_MODE(pVCpu))
1278	{
1279	if (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_BUSY)
1280	{
1281	offNewStack = RT_UOFFSETOF(X86TSS32, esp0) + uNewCSDpl * 8;
1282	cbNewStack = RT_SIZEOFMEMB(X86TSS32, esp0) + RT_SIZEOFMEMB(X86TSS32, ss0);
1283	}
1284	else
1285	{
1286	Assert(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_BUSY);
1287	offNewStack = RT_UOFFSETOF(X86TSS16, sp0) + uNewCSDpl * 4;
1288	cbNewStack = RT_SIZEOFMEMB(X86TSS16, sp0) + RT_SIZEOFMEMB(X86TSS16, ss0);
1289	}
1290	}
1291	else
1292	{
1293	Assert(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == AMD64_SEL_TYPE_SYS_TSS_BUSY);
1294	offNewStack = RT_UOFFSETOF(X86TSS64, rsp0) + uNewCSDpl * RT_SIZEOFMEMB(X86TSS64, rsp0);
1295	cbNewStack = RT_SIZEOFMEMB(X86TSS64, rsp0);
1296	}
1297
1298	/* Check against TSS limit. */
1299	if ((uint16_t)(offNewStack + cbNewStack - 1) > pVCpu->cpum.GstCtx.tr.u32Limit)
1300	{
1301	Log(("BranchCallGate inner stack past TSS limit - %u > %u -> #TS(TSS)\n", offNewStack + cbNewStack - 1, pVCpu->cpum.GstCtx.tr.u32Limit));
1302	return iemRaiseTaskSwitchFaultBySelector(pVCpu, pVCpu->cpum.GstCtx.tr.Sel);
1303	}
1304
1305	uint8_t bUnmapInfo;
1306	RTPTRUNION uPtrTss;
1307	RTGCPTR GCPtrTss = pVCpu->cpum.GstCtx.tr.u64Base + offNewStack;
1308	rcStrict = iemMemMap(pVCpu, &uPtrTss.pv, &bUnmapInfo, cbNewStack, UINT8_MAX, GCPtrTss, IEM_ACCESS_SYS_R, 0);
1309	if (rcStrict != VINF_SUCCESS)
1310	{
1311	Log(("BranchCallGate: TSS mapping failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
1312	return rcStrict;
1313	}
1314
1315	RTSEL uNewSS;
1316	if (!IEM_IS_LONG_MODE(pVCpu))
1317	{
1318	if (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_BUSY)
1319	{
1320	uNewRsp = uPtrTss.pu32[0];
1321	uNewSS = uPtrTss.pu16[2];
1322	}
1323	else
1324	{
1325	Assert(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_BUSY);
1326	uNewRsp = uPtrTss.pu16[0];
1327	uNewSS = uPtrTss.pu16[1];
1328	}
1329	}
1330	else
1331	{
1332	Assert(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == AMD64_SEL_TYPE_SYS_TSS_BUSY);
1333	/* SS will be a NULL selector, but that's valid. */
1334	uNewRsp = uPtrTss.pu64[0];
1335	uNewSS = uNewCSDpl;
1336	}
1337
1338	/* Done with the TSS now. */
1339	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
1340	if (rcStrict != VINF_SUCCESS)
1341	{
1342	Log(("BranchCallGate: TSS unmapping failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
1343	return rcStrict;
1344	}
1345
1346	/* Only used outside of long mode. */
1347	uint8_t const cbWords = pDesc->Legacy.Gate.u5ParmCount;
1348
1349	/* If EFER.LMA is 0, there's extra work to do. */
1350	IEMSELDESC DescSS;
1351	if (!IEM_IS_LONG_MODE(pVCpu))
1352	{
1353	if ((uNewSS & X86_SEL_MASK_OFF_RPL) == 0)
1354	{
1355	Log(("BranchCallGate new SS NULL -> #TS(NewSS)\n"));
1356	return iemRaiseTaskSwitchFaultBySelector(pVCpu, uNewSS);
1357	}
1358
1359	/* Grab the new SS descriptor. */
1360	rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSS, X86_XCPT_SS);
1361	if (rcStrict != VINF_SUCCESS)
1362	return rcStrict;
1363
1364	/* Ensure that CS.DPL == SS.RPL == SS.DPL. */
1365	if ( (DescCS.Legacy.Gen.u2Dpl != (uNewSS & X86_SEL_RPL))
1366	\|\| (DescCS.Legacy.Gen.u2Dpl != DescSS.Legacy.Gen.u2Dpl))
1367	{
1368	Log(("BranchCallGate call bad RPL/DPL uNewSS=%04x SS DPL=%d CS DPL=%u -> #TS(NewSS)\n",
1369	uNewSS, DescCS.Legacy.Gen.u2Dpl, DescCS.Legacy.Gen.u2Dpl));
1370	return iemRaiseTaskSwitchFaultBySelector(pVCpu, uNewSS);
1371	}
1372
1373	/* Ensure new SS is a writable data segment. */
1374	if ((DescSS.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_WRITE)) != X86_SEL_TYPE_WRITE)
1375	{
1376	Log(("BranchCallGate call new SS -> not a writable data selector (u4Type=%#x)\n", DescSS.Legacy.Gen.u4Type));
1377	return iemRaiseTaskSwitchFaultBySelector(pVCpu, uNewSS);
1378	}
1379
1380	if (!DescSS.Legacy.Gen.u1Present)
1381	{
1382	Log(("BranchCallGate New stack not present uSel=%04x -> #SS(NewSS)\n", uNewSS));
1383	return iemRaiseStackSelectorNotPresentBySelector(pVCpu, uNewSS);
1384	}
1385	if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE)
1386	cbNewStack = (uint16_t)sizeof(uint32_t) * (4 + cbWords);
1387	else
1388	cbNewStack = (uint16_t)sizeof(uint16_t) * (4 + cbWords);
1389	}
1390	else
1391	{
1392	/* Just grab the new (NULL) SS descriptor. */
1393	/** @todo testcase: Check whether the zero GDT entry is actually loaded here
1394	* like we do... */
1395	rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSS, X86_XCPT_SS);
1396	if (rcStrict != VINF_SUCCESS)
1397	return rcStrict;
1398
1399	cbNewStack = sizeof(uint64_t) * 4;
1400	}
1401
1402	/** @todo According to Intel, new stack is checked for enough space first,
1403	* then switched. According to AMD, the stack is switched first and
1404	* then pushes might fault!
1405	* NB: OS/2 Warp 3/4 actively relies on the fact that possible
1406	* incoming stack \#PF happens before actual stack switch. AMD is
1407	* either lying or implicitly assumes that new state is committed
1408	* only if and when an instruction doesn't fault.
1409	*/
1410
1411	/** @todo According to AMD, CS is loaded first, then SS.
1412	* According to Intel, it's the other way around!?
1413	*/
1414
1415	/** @todo Intel and AMD disagree on when exactly the CPL changes! */
1416
1417	/* Set the accessed bit before committing new SS. */
1418	if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
1419	{
1420	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewSS);
1421	if (rcStrict != VINF_SUCCESS)
1422	return rcStrict;
1423	DescSS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
1424	}
1425
1426	/* Remember the old SS:rSP and their linear address. */
1427	RTSEL const uOldSS = pVCpu->cpum.GstCtx.ss.Sel;
1428	uint64_t const uOldRsp = pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig ? pVCpu->cpum.GstCtx.rsp : pVCpu->cpum.GstCtx.sp;
1429
1430	RTGCPTR const GCPtrParmWds = pVCpu->cpum.GstCtx.ss.u64Base + uOldRsp;
1431
1432	/* HACK ALERT! Probe if the write to the new stack will succeed. May #SS(NewSS)
1433	or #PF, the former is not implemented in this workaround. */
1434	/** @todo Proper fix callgate target stack exceptions. */
1435	/** @todo testcase: Cover callgates with partially or fully inaccessible
1436	* target stacks. */
1437	void *pvNewFrame;
1438	RTGCPTR GCPtrNewStack = X86DESC_BASE(&DescSS.Legacy) + uNewRsp - cbNewStack;
1439	rcStrict = iemMemMap(pVCpu, &pvNewFrame, &bUnmapInfo, cbNewStack, UINT8_MAX, GCPtrNewStack, IEM_ACCESS_SYS_RW, 0);
1440	if (rcStrict != VINF_SUCCESS)
1441	{
1442	Log(("BranchCallGate: Incoming stack (%04x:%08RX64) not accessible, rc=%Rrc\n", uNewSS, uNewRsp, VBOXSTRICTRC_VAL(rcStrict)));
1443	return rcStrict;
1444	}
1445	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
1446	if (rcStrict != VINF_SUCCESS)
1447	{
1448	Log(("BranchCallGate: New stack probe unmapping failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
1449	return rcStrict;
1450	}
1451
1452	/* Commit new SS:rSP. */
1453	pVCpu->cpum.GstCtx.ss.Sel = uNewSS;
1454	pVCpu->cpum.GstCtx.ss.ValidSel = uNewSS;
1455	pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
1456	pVCpu->cpum.GstCtx.ss.u32Limit = X86DESC_LIMIT_G(&DescSS.Legacy);
1457	pVCpu->cpum.GstCtx.ss.u64Base = X86DESC_BASE(&DescSS.Legacy);
1458	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
1459	pVCpu->cpum.GstCtx.rsp = uNewRsp;
1460	IEM_SET_CPL(pVCpu, uNewCSDpl); /** @todo Are the parameter words accessed using the new CPL or the old CPL? */
1461	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
1462	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
1463
1464	/* At this point the stack access must not fail because new state was already committed. */
1465	/** @todo this can still fail due to SS.LIMIT not check. */
1466	uint8_t bUnmapInfoRet;
1467	rcStrict = iemMemStackPushBeginSpecial(pVCpu, cbNewStack,
1468	IEM_IS_LONG_MODE(pVCpu) ? 7
1469	: pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE ? 3 : 1,
1470	&uPtrRet.pv, &bUnmapInfoRet, &uNewRsp);
1471	AssertMsgReturn(rcStrict == VINF_SUCCESS, ("BranchCallGate: New stack mapping failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)),
1472	VERR_INTERNAL_ERROR_5);
1473
1474	if (!IEM_IS_LONG_MODE(pVCpu))
1475	{
1476	if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE)
1477	{
1478	if (cbWords)
1479	{
1480	/* Map the relevant chunk of the old stack. */
1481	RTPTRUNION uPtrParmWds;
1482	rcStrict = iemMemMap(pVCpu, &uPtrParmWds.pv, &bUnmapInfo, cbWords * 4, UINT8_MAX, GCPtrParmWds,
1483	IEM_ACCESS_DATA_R, 0 /** @todo Can uNewCSDpl == 3? Then we need alignment mask here! */);
1484	if (rcStrict != VINF_SUCCESS)
1485	{
1486	Log(("BranchCallGate: Old stack mapping (32-bit) failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
1487	return rcStrict;
1488	}
1489
1490	/* Copy the parameter (d)words. */
1491	for (int i = 0; i < cbWords; ++i)
1492	uPtrRet.pu32[2 + i] = uPtrParmWds.pu32[i];
1493
1494	/* Unmap the old stack. */
1495	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
1496	if (rcStrict != VINF_SUCCESS)
1497	{
1498	Log(("BranchCallGate: Old stack unmapping (32-bit) failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
1499	return rcStrict;
1500	}
1501	}
1502
1503	/* Push the old CS:rIP. */
1504	uPtrRet.pu32[0] = pVCpu->cpum.GstCtx.eip + cbInstr;
1505	uPtrRet.pu32[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high word when pushing CS? */
1506
1507	/* Push the old SS:rSP. */
1508	uPtrRet.pu32[2 + cbWords + 0] = uOldRsp;
1509	uPtrRet.pu32[2 + cbWords + 1] = uOldSS;
1510	}
1511	else
1512	{
1513	Assert(pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE);
1514
1515	if (cbWords)
1516	{
1517	/* Map the relevant chunk of the old stack. */
1518	RTPTRUNION uPtrParmWds;
1519	rcStrict = iemMemMap(pVCpu, &uPtrParmWds.pv, &bUnmapInfo, cbWords * 2, UINT8_MAX, GCPtrParmWds,
1520	IEM_ACCESS_DATA_R, 0 /** @todo Can uNewCSDpl == 3? Then we need alignment mask here! */);
1521	if (rcStrict != VINF_SUCCESS)
1522	{
1523	Log(("BranchCallGate: Old stack mapping (16-bit) failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
1524	return rcStrict;
1525	}
1526
1527	/* Copy the parameter words. */
1528	for (int i = 0; i < cbWords; ++i)
1529	uPtrRet.pu16[2 + i] = uPtrParmWds.pu16[i];
1530
1531	/* Unmap the old stack. */
1532	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
1533	if (rcStrict != VINF_SUCCESS)
1534	{
1535	Log(("BranchCallGate: Old stack unmapping (32-bit) failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
1536	return rcStrict;
1537	}
1538	}
1539
1540	/* Push the old CS:rIP. */
1541	uPtrRet.pu16[0] = pVCpu->cpum.GstCtx.ip + cbInstr;
1542	uPtrRet.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel;
1543
1544	/* Push the old SS:rSP. */
1545	uPtrRet.pu16[2 + cbWords + 0] = uOldRsp;
1546	uPtrRet.pu16[2 + cbWords + 1] = uOldSS;
1547	}
1548	}
1549	else
1550	{
1551	Assert(pDesc->Legacy.Gate.u4Type == AMD64_SEL_TYPE_SYS_CALL_GATE);
1552
1553	/* For 64-bit gates, no parameters are copied. Just push old SS:rSP and CS:rIP. */
1554	uPtrRet.pu64[0] = pVCpu->cpum.GstCtx.rip + cbInstr;
1555	uPtrRet.pu64[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high words when pushing CS? */
1556	uPtrRet.pu64[2] = uOldRsp;
1557	uPtrRet.pu64[3] = uOldSS; /** @todo Testcase: What is written to the high words when pushing SS? */
1558	}
1559
1560	rcStrict = iemMemStackPushCommitSpecial(pVCpu, bUnmapInfoRet, uNewRsp);
1561	if (rcStrict != VINF_SUCCESS)
1562	{
1563	Log(("BranchCallGate: New stack unmapping failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
1564	return rcStrict;
1565	}
1566
1567	/* Chop the high bits off if 16-bit gate (Intel says so). */
1568	if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE)
1569	uNewRip = (uint16_t)uNewRip;
1570
1571	/* Limit / canonical check. */
1572	cbLimit = X86DESC_LIMIT_G(&DescCS.Legacy);
1573	if (!IEM_IS_LONG_MODE(pVCpu))
1574	{
1575	if (uNewRip > cbLimit)
1576	{
1577	Log(("BranchCallGate %04x:%08RX64 -> out of bounds (%#x)\n", uNewCS, uNewRip, cbLimit));
1578	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, 0);
1579	}
1580	u64Base = X86DESC_BASE(&DescCS.Legacy);
1581	}
1582	else
1583	{
1584	Assert(pDesc->Legacy.Gate.u4Type == AMD64_SEL_TYPE_SYS_CALL_GATE);
1585	if (!IEM_IS_CANONICAL(uNewRip))
1586	{
1587	Log(("BranchCallGate call %04x:%016RX64 - not canonical -> #GP\n", uNewCS, uNewRip));
1588	return iemRaiseNotCanonical(pVCpu);
1589	}
1590	u64Base = 0;
1591	}
1592
1593	/*
1594	* Now set the accessed bit before
1595	* writing the return address to the stack and committing the result into
1596	* CS, CSHID and RIP.
1597	*/
1598	/** @todo Testcase: Need to check WHEN exactly the accessed bit is set. */
1599	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
1600	{
1601	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCS);
1602	if (rcStrict != VINF_SUCCESS)
1603	return rcStrict;
1604	/** @todo check what VT-x and AMD-V does. */
1605	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
1606	}
1607
1608	/* Commit new CS:rIP. */
1609	pVCpu->cpum.GstCtx.rip = uNewRip;
1610	pVCpu->cpum.GstCtx.cs.Sel = uNewCS & X86_SEL_MASK_OFF_RPL;
1611	pVCpu->cpum.GstCtx.cs.Sel \|= IEM_GET_CPL(pVCpu);
1612	pVCpu->cpum.GstCtx.cs.ValidSel = pVCpu->cpum.GstCtx.cs.Sel;
1613	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
1614	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
1615	pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit;
1616	pVCpu->cpum.GstCtx.cs.u64Base = u64Base;
1617	}
1618	else
1619	{
1620	/* Same privilege. */
1621	/** @todo This is very similar to regular far calls; merge! */
1622
1623	/* Check stack first - may #SS(0). */
1624	/** @todo check how gate size affects pushing of CS! Does callf 16:32 in
1625	* 16-bit code cause a two or four byte CS to be pushed? */
1626	uint8_t bUnmapInfoRet;
1627	rcStrict = iemMemStackPushBeginSpecial(pVCpu,
1628	IEM_IS_LONG_MODE(pVCpu) ? 8+8
1629	: pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE ? 4+4 : 2+2,
1630	IEM_IS_LONG_MODE(pVCpu) ? 7
1631	: pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE ? 3 : 2,
1632	&uPtrRet.pv, &bUnmapInfoRet, &uNewRsp);
1633	if (rcStrict != VINF_SUCCESS)
1634	return rcStrict;
1635
1636	/* Chop the high bits off if 16-bit gate (Intel says so). */
1637	if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE)
1638	uNewRip = (uint16_t)uNewRip;
1639
1640	/* Limit / canonical check. */
1641	cbLimit = X86DESC_LIMIT_G(&DescCS.Legacy);
1642	if (!IEM_IS_LONG_MODE(pVCpu))
1643	{
1644	if (uNewRip > cbLimit)
1645	{
1646	Log(("BranchCallGate %04x:%08RX64 -> out of bounds (%#x)\n", uNewCS, uNewRip, cbLimit));
1647	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, 0);
1648	}
1649	u64Base = X86DESC_BASE(&DescCS.Legacy);
1650	}
1651	else
1652	{
1653	if (!IEM_IS_CANONICAL(uNewRip))
1654	{
1655	Log(("BranchCallGate call %04x:%016RX64 - not canonical -> #GP\n", uNewCS, uNewRip));
1656	return iemRaiseNotCanonical(pVCpu);
1657	}
1658	u64Base = 0;
1659	}
1660
1661	/*
1662	* Now set the accessed bit before
1663	* writing the return address to the stack and committing the result into
1664	* CS, CSHID and RIP.
1665	*/
1666	/** @todo Testcase: Need to check WHEN exactly the accessed bit is set. */
1667	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
1668	{
1669	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCS);
1670	if (rcStrict != VINF_SUCCESS)
1671	return rcStrict;
1672	/** @todo check what VT-x and AMD-V does. */
1673	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
1674	}
1675
1676	/* stack */
1677	if (!IEM_IS_LONG_MODE(pVCpu))
1678	{
1679	if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE)
1680	{
1681	uPtrRet.pu32[0] = pVCpu->cpum.GstCtx.eip + cbInstr;
1682	uPtrRet.pu32[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high word when pushing CS? */
1683	}
1684	else
1685	{
1686	Assert(pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE);
1687	uPtrRet.pu16[0] = pVCpu->cpum.GstCtx.ip + cbInstr;
1688	uPtrRet.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel;
1689	}
1690	}
1691	else
1692	{
1693	Assert(pDesc->Legacy.Gate.u4Type == AMD64_SEL_TYPE_SYS_CALL_GATE);
1694	uPtrRet.pu64[0] = pVCpu->cpum.GstCtx.rip + cbInstr;
1695	uPtrRet.pu64[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high words when pushing CS? */
1696	}
1697
1698	rcStrict = iemMemStackPushCommitSpecial(pVCpu, bUnmapInfoRet, uNewRsp);
1699	if (rcStrict != VINF_SUCCESS)
1700	return rcStrict;
1701
1702	/* commit */
1703	pVCpu->cpum.GstCtx.rip = uNewRip;
1704	pVCpu->cpum.GstCtx.cs.Sel = uNewCS & X86_SEL_MASK_OFF_RPL;
1705	pVCpu->cpum.GstCtx.cs.Sel \|= IEM_GET_CPL(pVCpu);
1706	pVCpu->cpum.GstCtx.cs.ValidSel = pVCpu->cpum.GstCtx.cs.Sel;
1707	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
1708	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
1709	pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit;
1710	pVCpu->cpum.GstCtx.cs.u64Base = u64Base;
1711	}
1712	}
1713	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
1714
1715	iemRecalcExecModeAndCplAndAcFlags(pVCpu);
1716
1717	/** @todo single stepping */
1718
1719	/* Flush the prefetch buffer. */
1720	IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr);
1721	return VINF_SUCCESS;
1722	#endif /* IEM_IMPLEMENTS_CALLGATE */
1723	}
1724
1725
1726	/**
1727	* Implements far jumps and calls thru system selectors.
1728	*
1729	* @returns VBox strict status code.
1730	* @param pVCpu The cross context virtual CPU structure of the
1731	* calling thread.
1732	* @param cbInstr The current instruction length.
1733	* @param uSel The selector.
1734	* @param enmBranch The kind of branching we're performing.
1735	* @param enmEffOpSize The effective operand size.
1736	* @param pDesc The descriptor corresponding to @a uSel.
1737	*/
1738	static VBOXSTRICTRC iemCImpl_BranchSysSel(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uSel, IEMBRANCH enmBranch,
1739	IEMMODE enmEffOpSize, PIEMSELDESC pDesc)
1740	{
1741	Assert(enmBranch == IEMBRANCH_JUMP \|\| enmBranch == IEMBRANCH_CALL);
1742	Assert((uSel & X86_SEL_MASK_OFF_RPL));
1743	IEM_CTX_IMPORT_RET(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
1744
1745	if (IEM_IS_LONG_MODE(pVCpu))
1746	switch (pDesc->Legacy.Gen.u4Type)
1747	{
1748	case AMD64_SEL_TYPE_SYS_CALL_GATE:
1749	return iemCImpl_BranchCallGate(pVCpu, cbInstr, uSel, enmBranch, enmEffOpSize, pDesc);
1750
1751	default:
1752	case AMD64_SEL_TYPE_SYS_LDT:
1753	case AMD64_SEL_TYPE_SYS_TSS_BUSY:
1754	case AMD64_SEL_TYPE_SYS_TSS_AVAIL:
1755	case AMD64_SEL_TYPE_SYS_TRAP_GATE:
1756	case AMD64_SEL_TYPE_SYS_INT_GATE:
1757	Log(("branch %04x -> wrong sys selector (64-bit): %d\n", uSel, pDesc->Legacy.Gen.u4Type));
1758	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
1759	}
1760
1761	switch (pDesc->Legacy.Gen.u4Type)
1762	{
1763	case X86_SEL_TYPE_SYS_286_CALL_GATE:
1764	case X86_SEL_TYPE_SYS_386_CALL_GATE:
1765	return iemCImpl_BranchCallGate(pVCpu, cbInstr, uSel, enmBranch, enmEffOpSize, pDesc);
1766
1767	case X86_SEL_TYPE_SYS_TASK_GATE:
1768	return iemCImpl_BranchTaskGate(pVCpu, cbInstr, uSel, enmBranch, enmEffOpSize, pDesc);
1769
1770	case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
1771	case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
1772	return iemCImpl_BranchTaskSegment(pVCpu, cbInstr, uSel, enmBranch, enmEffOpSize, pDesc);
1773
1774	case X86_SEL_TYPE_SYS_286_TSS_BUSY:
1775	Log(("branch %04x -> busy 286 TSS\n", uSel));
1776	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
1777
1778	case X86_SEL_TYPE_SYS_386_TSS_BUSY:
1779	Log(("branch %04x -> busy 386 TSS\n", uSel));
1780	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
1781
1782	default:
1783	case X86_SEL_TYPE_SYS_LDT:
1784	case X86_SEL_TYPE_SYS_286_INT_GATE:
1785	case X86_SEL_TYPE_SYS_286_TRAP_GATE:
1786	case X86_SEL_TYPE_SYS_386_INT_GATE:
1787	case X86_SEL_TYPE_SYS_386_TRAP_GATE:
1788	Log(("branch %04x -> wrong sys selector: %d\n", uSel, pDesc->Legacy.Gen.u4Type));
1789	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
1790	}
1791	}
1792
1793
1794	/**
1795	* Implements far jumps.
1796	*
1797	* @param uSel The selector.
1798	* @param offSeg The segment offset.
1799	* @param enmEffOpSize The effective operand size.
1800	*/
1801	IEM_CIMPL_DEF_3(iemCImpl_FarJmp, uint16_t, uSel, uint64_t, offSeg, IEMMODE, enmEffOpSize)
1802	{
1803	NOREF(cbInstr);
1804	Assert(offSeg <= UINT32_MAX \|\| (!IEM_IS_GUEST_CPU_AMD(pVCpu) && IEM_IS_64BIT_CODE(pVCpu)));
1805
1806	/*
1807	* Real mode and V8086 mode are easy. The only snag seems to be that
1808	* CS.limit doesn't change and the limit check is done against the current
1809	* limit.
1810	*/
1811	/** @todo Robert Collins claims (The Segment Descriptor Cache, DDJ August
1812	* 1998) that up to and including the Intel 486, far control
1813	* transfers in real mode set default CS attributes (0x93) and also
1814	* set a 64K segment limit. Starting with the Pentium, the
1815	* attributes and limit are left alone but the access rights are
1816	* ignored. We only implement the Pentium+ behavior.
1817	* */
1818	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
1819	{
1820	Assert(enmEffOpSize == IEMMODE_16BIT \|\| enmEffOpSize == IEMMODE_32BIT);
1821	if (offSeg > pVCpu->cpum.GstCtx.cs.u32Limit)
1822	{
1823	Log(("iemCImpl_FarJmp: 16-bit limit\n"));
1824	return iemRaiseGeneralProtectionFault0(pVCpu);
1825	}
1826
1827	if (enmEffOpSize == IEMMODE_16BIT) /** @todo WRONG, must pass this. */
1828	pVCpu->cpum.GstCtx.rip = offSeg;
1829	else
1830	pVCpu->cpum.GstCtx.rip = offSeg & UINT16_MAX;
1831	pVCpu->cpum.GstCtx.cs.Sel = uSel;
1832	pVCpu->cpum.GstCtx.cs.ValidSel = uSel;
1833	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
1834	pVCpu->cpum.GstCtx.cs.u64Base = (uint32_t)uSel << 4;
1835
1836	/* Update the FLAT 32-bit mode flag, if we're in 32-bit unreal mode (unlikely): */
1837	if (RT_LIKELY(!IEM_IS_32BIT_CODE(pVCpu)))
1838	{ /* likely */ }
1839	else if (uSel != 0)
1840	pVCpu->iem.s.fExec &= ~IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK;
1841	else
1842	pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK)
1843	\| iemCalc32BitFlatIndicator(pVCpu);
1844
1845	return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);
1846	}
1847
1848	/*
1849	* Protected mode. Need to parse the specified descriptor...
1850	*/
1851	if (!(uSel & X86_SEL_MASK_OFF_RPL))
1852	{
1853	Log(("jmpf %04x:%08RX64 -> invalid selector, #GP(0)\n", uSel, offSeg));
1854	return iemRaiseGeneralProtectionFault0(pVCpu);
1855	}
1856
1857	/* Fetch the descriptor. */
1858	IEMSELDESC Desc;
1859	VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uSel, X86_XCPT_GP);
1860	if (rcStrict != VINF_SUCCESS)
1861	return rcStrict;
1862
1863	/* Is it there? */
1864	if (!Desc.Legacy.Gen.u1Present) /** @todo this is probably checked too early. Testcase! */
1865	{
1866	Log(("jmpf %04x:%08RX64 -> segment not present\n", uSel, offSeg));
1867	return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel);
1868	}
1869
1870	/*
1871	* Deal with it according to its type. We do the standard code selectors
1872	* here and dispatch the system selectors to worker functions.
1873	*/
1874	if (!Desc.Legacy.Gen.u1DescType)
1875	return iemCImpl_BranchSysSel(pVCpu, cbInstr, uSel, IEMBRANCH_JUMP, enmEffOpSize, &Desc);
1876
1877	/* Only code segments. */
1878	if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
1879	{
1880	Log(("jmpf %04x:%08RX64 -> not a code selector (u4Type=%#x).\n", uSel, offSeg, Desc.Legacy.Gen.u4Type));
1881	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
1882	}
1883
1884	/* L vs D. */
1885	if ( Desc.Legacy.Gen.u1Long
1886	&& Desc.Legacy.Gen.u1DefBig
1887	&& IEM_IS_LONG_MODE(pVCpu))
1888	{
1889	Log(("jmpf %04x:%08RX64 -> both L and D are set.\n", uSel, offSeg));
1890	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
1891	}
1892
1893	/* DPL/RPL/CPL check, where conforming segments makes a difference. */
1894	if (Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
1895	{
1896	if (IEM_GET_CPL(pVCpu) < Desc.Legacy.Gen.u2Dpl)
1897	{
1898	Log(("jmpf %04x:%08RX64 -> DPL violation (conforming); DPL=%d CPL=%u\n",
1899	uSel, offSeg, Desc.Legacy.Gen.u2Dpl, IEM_GET_CPL(pVCpu)));
1900	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
1901	}
1902	}
1903	else
1904	{
1905	if (IEM_GET_CPL(pVCpu) != Desc.Legacy.Gen.u2Dpl)
1906	{
1907	Log(("jmpf %04x:%08RX64 -> CPL != DPL; DPL=%d CPL=%u\n", uSel, offSeg, Desc.Legacy.Gen.u2Dpl, IEM_GET_CPL(pVCpu)));
1908	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
1909	}
1910	if ((uSel & X86_SEL_RPL) > IEM_GET_CPL(pVCpu))
1911	{
1912	Log(("jmpf %04x:%08RX64 -> RPL > DPL; RPL=%d CPL=%u\n", uSel, offSeg, (uSel & X86_SEL_RPL), IEM_GET_CPL(pVCpu)));
1913	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
1914	}
1915	}
1916
1917	/* Chop the high bits if 16-bit (Intel says so). */
1918	if (enmEffOpSize == IEMMODE_16BIT)
1919	offSeg &= UINT16_MAX;
1920
1921	/* Limit check and get the base. */
1922	uint64_t u64Base;
1923	uint32_t cbLimit = X86DESC_LIMIT_G(&Desc.Legacy);
1924	if ( !Desc.Legacy.Gen.u1Long
1925	\|\| !IEM_IS_LONG_MODE(pVCpu))
1926	{
1927	if (RT_LIKELY(offSeg <= cbLimit))
1928	u64Base = X86DESC_BASE(&Desc.Legacy);
1929	else
1930	{
1931	Log(("jmpf %04x:%08RX64 -> out of bounds (%#x)\n", uSel, offSeg, cbLimit));
1932	/** @todo Intel says this is \#GP(0)! */
1933	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
1934	}
1935	}
1936	else
1937	u64Base = 0;
1938
1939	/*
1940	* Ok, everything checked out fine. Now set the accessed bit before
1941	* committing the result into CS, CSHID and RIP.
1942	*/
1943	if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
1944	{
1945	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uSel);
1946	if (rcStrict != VINF_SUCCESS)
1947	return rcStrict;
1948	/** @todo check what VT-x and AMD-V does. */
1949	Desc.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
1950	}
1951
1952	/* commit */
1953	pVCpu->cpum.GstCtx.rip = offSeg;
1954	pVCpu->cpum.GstCtx.cs.Sel = uSel & X86_SEL_MASK_OFF_RPL;
1955	pVCpu->cpum.GstCtx.cs.Sel \|= IEM_GET_CPL(pVCpu); /** @todo is this right for conforming segs? or in general? */
1956	pVCpu->cpum.GstCtx.cs.ValidSel = pVCpu->cpum.GstCtx.cs.Sel;
1957	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
1958	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&Desc.Legacy);
1959	pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit;
1960	pVCpu->cpum.GstCtx.cs.u64Base = u64Base;
1961
1962	/** @todo check if the hidden bits are loaded correctly for 64-bit
1963	* mode. */
1964
1965	iemRecalcExecModeAndCplAndAcFlags(pVCpu);
1966
1967	/* Flush the prefetch buffer. */
1968	IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr);
1969
1970	return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);
1971	}
1972
1973
1974	/**
1975	* Implements far calls.
1976	*
1977	* This very similar to iemCImpl_FarJmp.
1978	*
1979	* @param uSel The selector.
1980	* @param offSeg The segment offset.
1981	* @param enmEffOpSize The operand size (in case we need it).
1982	*/
1983	IEM_CIMPL_DEF_3(iemCImpl_callf, uint16_t, uSel, uint64_t, offSeg, IEMMODE, enmEffOpSize)
1984	{
1985	VBOXSTRICTRC rcStrict;
1986	uint64_t uNewRsp;
1987	RTPTRUNION uPtrRet;
1988	uint8_t bUnmapInfo;
1989
1990	/*
1991	* Real mode and V8086 mode are easy. The only snag seems to be that
1992	* CS.limit doesn't change and the limit check is done against the current
1993	* limit.
1994	*/
1995	/** @todo See comment for similar code in iemCImpl_FarJmp */
1996	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
1997	{
1998	Assert(enmEffOpSize == IEMMODE_16BIT \|\| enmEffOpSize == IEMMODE_32BIT);
1999
2000	/* Check stack first - may #SS(0). */
2001	rcStrict = iemMemStackPushBeginSpecial(pVCpu, enmEffOpSize == IEMMODE_32BIT ? 4+4 : 2+2,
2002	enmEffOpSize == IEMMODE_32BIT ? 3 : 1,
2003	&uPtrRet.pv, &bUnmapInfo, &uNewRsp);
2004	if (rcStrict != VINF_SUCCESS)
2005	return rcStrict;
2006
2007	/* Check the target address range. */
2008	/** @todo this must be wrong! Write unreal mode tests! */
2009	if (offSeg > UINT32_MAX)
2010	return iemRaiseGeneralProtectionFault0(pVCpu);
2011
2012	/* Everything is fine, push the return address. */
2013	if (enmEffOpSize == IEMMODE_16BIT)
2014	{
2015	uPtrRet.pu16[0] = pVCpu->cpum.GstCtx.ip + cbInstr;
2016	uPtrRet.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel;
2017	}
2018	else
2019	{
2020	uPtrRet.pu32[0] = pVCpu->cpum.GstCtx.eip + cbInstr;
2021	uPtrRet.pu16[2] = pVCpu->cpum.GstCtx.cs.Sel;
2022	}
2023	rcStrict = iemMemStackPushCommitSpecial(pVCpu, bUnmapInfo, uNewRsp);
2024	if (rcStrict != VINF_SUCCESS)
2025	return rcStrict;
2026
2027	/* Branch. */
2028	pVCpu->cpum.GstCtx.rip = offSeg;
2029	pVCpu->cpum.GstCtx.cs.Sel = uSel;
2030	pVCpu->cpum.GstCtx.cs.ValidSel = uSel;
2031	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
2032	pVCpu->cpum.GstCtx.cs.u64Base = (uint32_t)uSel << 4;
2033
2034	return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);
2035	}
2036
2037	/*
2038	* Protected mode. Need to parse the specified descriptor...
2039	*/
2040	if (!(uSel & X86_SEL_MASK_OFF_RPL))
2041	{
2042	Log(("callf %04x:%08RX64 -> invalid selector, #GP(0)\n", uSel, offSeg));
2043	return iemRaiseGeneralProtectionFault0(pVCpu);
2044	}
2045
2046	/* Fetch the descriptor. */
2047	IEMSELDESC Desc;
2048	rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uSel, X86_XCPT_GP);
2049	if (rcStrict != VINF_SUCCESS)
2050	return rcStrict;
2051
2052	/*
2053	* Deal with it according to its type. We do the standard code selectors
2054	* here and dispatch the system selectors to worker functions.
2055	*/
2056	if (!Desc.Legacy.Gen.u1DescType)
2057	return iemCImpl_BranchSysSel(pVCpu, cbInstr, uSel, IEMBRANCH_CALL, enmEffOpSize, &Desc);
2058
2059	/* Only code segments. */
2060	if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
2061	{
2062	Log(("callf %04x:%08RX64 -> not a code selector (u4Type=%#x).\n", uSel, offSeg, Desc.Legacy.Gen.u4Type));
2063	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
2064	}
2065
2066	/* L vs D. */
2067	if ( Desc.Legacy.Gen.u1Long
2068	&& Desc.Legacy.Gen.u1DefBig
2069	&& IEM_IS_LONG_MODE(pVCpu))
2070	{
2071	Log(("callf %04x:%08RX64 -> both L and D are set.\n", uSel, offSeg));
2072	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
2073	}
2074
2075	/* DPL/RPL/CPL check, where conforming segments makes a difference. */
2076	if (Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
2077	{
2078	if (IEM_GET_CPL(pVCpu) < Desc.Legacy.Gen.u2Dpl)
2079	{
2080	Log(("callf %04x:%08RX64 -> DPL violation (conforming); DPL=%d CPL=%u\n",
2081	uSel, offSeg, Desc.Legacy.Gen.u2Dpl, IEM_GET_CPL(pVCpu)));
2082	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
2083	}
2084	}
2085	else
2086	{
2087	if (IEM_GET_CPL(pVCpu) != Desc.Legacy.Gen.u2Dpl)
2088	{
2089	Log(("callf %04x:%08RX64 -> CPL != DPL; DPL=%d CPL=%u\n", uSel, offSeg, Desc.Legacy.Gen.u2Dpl, IEM_GET_CPL(pVCpu)));
2090	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
2091	}
2092	if ((uSel & X86_SEL_RPL) > IEM_GET_CPL(pVCpu))
2093	{
2094	Log(("callf %04x:%08RX64 -> RPL > DPL; RPL=%d CPL=%u\n", uSel, offSeg, (uSel & X86_SEL_RPL), IEM_GET_CPL(pVCpu)));
2095	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
2096	}
2097	}
2098
2099	/* Is it there? */
2100	if (!Desc.Legacy.Gen.u1Present)
2101	{
2102	Log(("callf %04x:%08RX64 -> segment not present\n", uSel, offSeg));
2103	return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel);
2104	}
2105
2106	/* Check stack first - may #SS(0). */
2107	/** @todo check how operand prefix affects pushing of CS! Does callf 16:32 in
2108	* 16-bit code cause a two or four byte CS to be pushed? */
2109	rcStrict = iemMemStackPushBeginSpecial(pVCpu,
2110	enmEffOpSize == IEMMODE_64BIT ? 8+8 : enmEffOpSize == IEMMODE_32BIT ? 4+4 : 2+2,
2111	enmEffOpSize == IEMMODE_64BIT ? 7 : enmEffOpSize == IEMMODE_32BIT ? 3 : 1,
2112	&uPtrRet.pv, &bUnmapInfo, &uNewRsp);
2113	if (rcStrict != VINF_SUCCESS)
2114	return rcStrict;
2115
2116	/* Chop the high bits if 16-bit (Intel says so). */
2117	if (enmEffOpSize == IEMMODE_16BIT)
2118	offSeg &= UINT16_MAX;
2119
2120	/* Limit / canonical check. */
2121	uint64_t u64Base;
2122	uint32_t cbLimit = X86DESC_LIMIT_G(&Desc.Legacy);
2123	if ( !Desc.Legacy.Gen.u1Long
2124	\|\| !IEM_IS_LONG_MODE(pVCpu))
2125	{
2126	if (RT_LIKELY(offSeg <= cbLimit))
2127	u64Base = X86DESC_BASE(&Desc.Legacy);
2128	else
2129	{
2130	Log(("callf %04x:%08RX64 -> out of bounds (%#x)\n", uSel, offSeg, cbLimit));
2131	/** @todo Intel says this is \#GP(0)! */
2132	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
2133	}
2134	}
2135	else if (IEM_IS_CANONICAL(offSeg))
2136	u64Base = 0;
2137	else
2138	{
2139	Log(("callf %04x:%016RX64 - not canonical -> #GP\n", uSel, offSeg));
2140	return iemRaiseNotCanonical(pVCpu);
2141	}
2142
2143	/*
2144	* Now set the accessed bit before
2145	* writing the return address to the stack and committing the result into
2146	* CS, CSHID and RIP.
2147	*/
2148	/** @todo Testcase: Need to check WHEN exactly the accessed bit is set. */
2149	if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
2150	{
2151	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uSel);
2152	if (rcStrict != VINF_SUCCESS)
2153	return rcStrict;
2154	/** @todo check what VT-x and AMD-V does. */
2155	Desc.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
2156	}
2157
2158	/* stack */
2159	if (enmEffOpSize == IEMMODE_16BIT)
2160	{
2161	uPtrRet.pu16[0] = pVCpu->cpum.GstCtx.ip + cbInstr;
2162	uPtrRet.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel;
2163	}
2164	else if (enmEffOpSize == IEMMODE_32BIT)
2165	{
2166	uPtrRet.pu32[0] = pVCpu->cpum.GstCtx.eip + cbInstr;
2167	uPtrRet.pu32[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high word when callf is pushing CS? */
2168	}
2169	else
2170	{
2171	uPtrRet.pu64[0] = pVCpu->cpum.GstCtx.rip + cbInstr;
2172	uPtrRet.pu64[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high words when callf is pushing CS? */
2173	}
2174	rcStrict = iemMemStackPushCommitSpecial(pVCpu, bUnmapInfo, uNewRsp);
2175	if (rcStrict != VINF_SUCCESS)
2176	return rcStrict;
2177
2178	/* commit */
2179	pVCpu->cpum.GstCtx.rip = offSeg;
2180	pVCpu->cpum.GstCtx.cs.Sel = uSel & X86_SEL_MASK_OFF_RPL;
2181	pVCpu->cpum.GstCtx.cs.Sel \|= IEM_GET_CPL(pVCpu);
2182	pVCpu->cpum.GstCtx.cs.ValidSel = pVCpu->cpum.GstCtx.cs.Sel;
2183	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
2184	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&Desc.Legacy);
2185	pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit;
2186	pVCpu->cpum.GstCtx.cs.u64Base = u64Base;
2187
2188	/** @todo check if the hidden bits are loaded correctly for 64-bit
2189	* mode. */
2190
2191	iemRecalcExecModeAndCplAndAcFlags(pVCpu);
2192
2193	/* Flush the prefetch buffer. */
2194	IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr);
2195
2196	return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);
2197	}
2198
2199
2200	/**
2201	* Implements retf.
2202	*
2203	* @param enmEffOpSize The effective operand size.
2204	* @param cbPop The amount of arguments to pop from the stack
2205	* (bytes).
2206	*/
2207	IEM_CIMPL_DEF_2(iemCImpl_retf, IEMMODE, enmEffOpSize, uint16_t, cbPop)
2208	{
2209	NOREF(cbInstr);
2210
2211	/*
2212	* Read the stack values first.
2213	*/
2214	RTUINT64U NewRsp;
2215	uint8_t bUnmapInfo;
2216	RTCPTRUNION uPtrFrame;
2217	uint32_t cbRetPtr = enmEffOpSize == IEMMODE_16BIT ? 2+2
2218	: enmEffOpSize == IEMMODE_32BIT ? 4+4 : 8+8;
2219	VBOXSTRICTRC rcStrict = iemMemStackPopBeginSpecial(pVCpu, cbRetPtr,
2220	enmEffOpSize == IEMMODE_16BIT ? 1 : enmEffOpSize == IEMMODE_32BIT ? 3 : 7,
2221	&uPtrFrame.pv, &bUnmapInfo, &NewRsp.u);
2222	if (rcStrict != VINF_SUCCESS)
2223	return rcStrict;
2224
2225	uint64_t uNewRip;
2226	uint16_t uNewCs;
2227	if (enmEffOpSize == IEMMODE_16BIT)
2228	{
2229	uNewRip = uPtrFrame.pu16[0];
2230	uNewCs = uPtrFrame.pu16[1];
2231	}
2232	else if (enmEffOpSize == IEMMODE_32BIT)
2233	{
2234	uNewRip = uPtrFrame.pu32[0];
2235	uNewCs = uPtrFrame.pu16[2];
2236	}
2237	else
2238	{
2239	uNewRip = uPtrFrame.pu64[0];
2240	uNewCs = uPtrFrame.pu16[4];
2241	}
2242
2243	rcStrict = iemMemStackPopDoneSpecial(pVCpu, bUnmapInfo);
2244	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
2245	{ /* extremely likely */ }
2246	else
2247	return rcStrict;
2248
2249	/*
2250	* Real mode and V8086 mode are easy.
2251	*/
2252	/** @todo See comment for similar code in iemCImpl_FarJmp */
2253	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
2254	{
2255	Assert(enmEffOpSize == IEMMODE_32BIT \|\| enmEffOpSize == IEMMODE_16BIT);
2256	/** @todo check how this is supposed to work if sp=0xfffe. */
2257
2258	/* Check the limit of the new EIP. */
2259	/** @todo Intel pseudo code only does the limit check for 16-bit
2260	* operands, AMD does not make any distinction. What is right? */
2261	if (uNewRip > pVCpu->cpum.GstCtx.cs.u32Limit)
2262	return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
2263
2264	/* commit the operation. */
2265	if (cbPop)
2266	iemRegAddToRspEx(pVCpu, &NewRsp, cbPop);
2267	pVCpu->cpum.GstCtx.rsp = NewRsp.u;
2268	pVCpu->cpum.GstCtx.rip = uNewRip;
2269	pVCpu->cpum.GstCtx.cs.Sel = uNewCs;
2270	pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs;
2271	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
2272	pVCpu->cpum.GstCtx.cs.u64Base = (uint32_t)uNewCs << 4;
2273	return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);
2274	}
2275
2276	/*
2277	* Protected mode is complicated, of course.
2278	*/
2279	if (!(uNewCs & X86_SEL_MASK_OFF_RPL))
2280	{
2281	Log(("retf %04x:%08RX64 -> invalid selector, #GP(0)\n", uNewCs, uNewRip));
2282	return iemRaiseGeneralProtectionFault0(pVCpu);
2283	}
2284
2285	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_GDTR \| CPUMCTX_EXTRN_LDTR);
2286
2287	/* Fetch the descriptor. */
2288	IEMSELDESC DescCs;
2289	rcStrict = iemMemFetchSelDesc(pVCpu, &DescCs, uNewCs, X86_XCPT_GP);
2290	if (rcStrict != VINF_SUCCESS)
2291	return rcStrict;
2292
2293	/* Can only return to a code selector. */
2294	if ( !DescCs.Legacy.Gen.u1DescType
2295	\|\| !(DescCs.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE) )
2296	{
2297	Log(("retf %04x:%08RX64 -> not a code selector (u1DescType=%u u4Type=%#x).\n",
2298	uNewCs, uNewRip, DescCs.Legacy.Gen.u1DescType, DescCs.Legacy.Gen.u4Type));
2299	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
2300	}
2301
2302	/* L vs D. */
2303	if ( DescCs.Legacy.Gen.u1Long /** @todo Testcase: far return to a selector with both L and D set. */
2304	&& DescCs.Legacy.Gen.u1DefBig
2305	&& IEM_IS_LONG_MODE(pVCpu))
2306	{
2307	Log(("retf %04x:%08RX64 -> both L & D set.\n", uNewCs, uNewRip));
2308	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
2309	}
2310
2311	/* DPL/RPL/CPL checks. */
2312	if ((uNewCs & X86_SEL_RPL) < IEM_GET_CPL(pVCpu))
2313	{
2314	Log(("retf %04x:%08RX64 -> RPL < CPL(%d).\n", uNewCs, uNewRip, IEM_GET_CPL(pVCpu)));
2315	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
2316	}
2317
2318	if (DescCs.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
2319	{
2320	if ((uNewCs & X86_SEL_RPL) < DescCs.Legacy.Gen.u2Dpl)
2321	{
2322	Log(("retf %04x:%08RX64 -> DPL violation (conforming); DPL=%u RPL=%u\n",
2323	uNewCs, uNewRip, DescCs.Legacy.Gen.u2Dpl, (uNewCs & X86_SEL_RPL)));
2324	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
2325	}
2326	}
2327	else
2328	{
2329	if ((uNewCs & X86_SEL_RPL) != DescCs.Legacy.Gen.u2Dpl)
2330	{
2331	Log(("retf %04x:%08RX64 -> RPL != DPL; DPL=%u RPL=%u\n",
2332	uNewCs, uNewRip, DescCs.Legacy.Gen.u2Dpl, (uNewCs & X86_SEL_RPL)));
2333	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
2334	}
2335	}
2336
2337	/* Is it there? */
2338	if (!DescCs.Legacy.Gen.u1Present)
2339	{
2340	Log(("retf %04x:%08RX64 -> segment not present\n", uNewCs, uNewRip));
2341	return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewCs);
2342	}
2343
2344	/*
2345	* Return to outer privilege? (We'll typically have entered via a call gate.)
2346	*/
2347	if ((uNewCs & X86_SEL_RPL) != IEM_GET_CPL(pVCpu))
2348	{
2349	/* Read the outer stack pointer stored after the parameters. */
2350	rcStrict = iemMemStackPopContinueSpecial(pVCpu, cbPop /off/, cbRetPtr, &uPtrFrame.pv, &bUnmapInfo, NewRsp.u);
2351	if (rcStrict != VINF_SUCCESS)
2352	return rcStrict;
2353
2354	uint16_t uNewOuterSs;
2355	RTUINT64U NewOuterRsp;
2356	if (enmEffOpSize == IEMMODE_16BIT)
2357	{
2358	NewOuterRsp.u = uPtrFrame.pu16[0];
2359	uNewOuterSs = uPtrFrame.pu16[1];
2360	}
2361	else if (enmEffOpSize == IEMMODE_32BIT)
2362	{
2363	NewOuterRsp.u = uPtrFrame.pu32[0];
2364	uNewOuterSs = uPtrFrame.pu16[2];
2365	}
2366	else
2367	{
2368	NewOuterRsp.u = uPtrFrame.pu64[0];
2369	uNewOuterSs = uPtrFrame.pu16[4];
2370	}
2371	rcStrict = iemMemStackPopDoneSpecial(pVCpu, bUnmapInfo);
2372	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
2373	{ /* extremely likely */ }
2374	else
2375	return rcStrict;
2376
2377	/* Check for NULL stack selector (invalid in ring-3 and non-long mode)
2378	and read the selector. */
2379	IEMSELDESC DescSs;
2380	if (!(uNewOuterSs & X86_SEL_MASK_OFF_RPL))
2381	{
2382	if ( !DescCs.Legacy.Gen.u1Long
2383	\|\| (uNewOuterSs & X86_SEL_RPL) == 3)
2384	{
2385	Log(("retf %04x:%08RX64 %04x:%08RX64 -> invalid stack selector, #GP\n",
2386	uNewCs, uNewRip, uNewOuterSs, NewOuterRsp.u));
2387	return iemRaiseGeneralProtectionFault0(pVCpu);
2388	}
2389	/** @todo Testcase: Return far to ring-1 or ring-2 with SS=0. */
2390	iemMemFakeStackSelDesc(&DescSs, (uNewOuterSs & X86_SEL_RPL));
2391	}
2392	else
2393	{
2394	/* Fetch the descriptor for the new stack segment. */
2395	rcStrict = iemMemFetchSelDesc(pVCpu, &DescSs, uNewOuterSs, X86_XCPT_GP);
2396	if (rcStrict != VINF_SUCCESS)
2397	return rcStrict;
2398	}
2399
2400	/* Check that RPL of stack and code selectors match. */
2401	if ((uNewCs & X86_SEL_RPL) != (uNewOuterSs & X86_SEL_RPL))
2402	{
2403	Log(("retf %04x:%08RX64 %04x:%08RX64 - SS.RPL != CS.RPL -> #GP(SS)\n", uNewCs, uNewRip, uNewOuterSs, NewOuterRsp.u));
2404	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewOuterSs);
2405	}
2406
2407	/* Must be a writable data segment. */
2408	if ( !DescSs.Legacy.Gen.u1DescType
2409	\|\| (DescSs.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
2410	\|\| !(DescSs.Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE) )
2411	{
2412	Log(("retf %04x:%08RX64 %04x:%08RX64 - SS not a writable data segment (u1DescType=%u u4Type=%#x) -> #GP(SS).\n",
2413	uNewCs, uNewRip, uNewOuterSs, NewOuterRsp.u, DescSs.Legacy.Gen.u1DescType, DescSs.Legacy.Gen.u4Type));
2414	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewOuterSs);
2415	}
2416
2417	/* L vs D. (Not mentioned by intel.) */
2418	if ( DescSs.Legacy.Gen.u1Long /** @todo Testcase: far return to a stack selector with both L and D set. */
2419	&& DescSs.Legacy.Gen.u1DefBig
2420	&& IEM_IS_LONG_MODE(pVCpu))
2421	{
2422	Log(("retf %04x:%08RX64 %04x:%08RX64 - SS has both L & D set -> #GP(SS).\n",
2423	uNewCs, uNewRip, uNewOuterSs, NewOuterRsp.u));
2424	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewOuterSs);
2425	}
2426
2427	/* DPL/RPL/CPL checks. */
2428	if (DescSs.Legacy.Gen.u2Dpl != (uNewCs & X86_SEL_RPL))
2429	{
2430	Log(("retf %04x:%08RX64 %04x:%08RX64 - SS.DPL(%u) != CS.RPL (%u) -> #GP(SS).\n",
2431	uNewCs, uNewRip, uNewOuterSs, NewOuterRsp.u, DescSs.Legacy.Gen.u2Dpl, uNewCs & X86_SEL_RPL));
2432	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewOuterSs);
2433	}
2434
2435	/* Is it there? */
2436	if (!DescSs.Legacy.Gen.u1Present)
2437	{
2438	Log(("retf %04x:%08RX64 %04x:%08RX64 - SS not present -> #NP(SS).\n", uNewCs, uNewRip, uNewOuterSs, NewOuterRsp.u));
2439	return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewCs);
2440	}
2441
2442	/* Calc SS limit.*/
2443	uint64_t u64BaseSs;
2444	uint32_t cbLimitSs = X86DESC_LIMIT_G(&DescSs.Legacy);
2445
2446	/* Is RIP canonical or within CS.limit? */
2447	uint64_t u64BaseCs;
2448	uint32_t cbLimitCs = X86DESC_LIMIT_G(&DescCs.Legacy);
2449
2450	/** @todo Testcase: Is this correct? */
2451	if ( DescCs.Legacy.Gen.u1Long
2452	&& IEM_IS_LONG_MODE(pVCpu) )
2453	{
2454	if (!IEM_IS_CANONICAL(uNewRip))
2455	{
2456	Log(("retf %04x:%08RX64 %04x:%08RX64 - not canonical -> #GP.\n", uNewCs, uNewRip, uNewOuterSs, NewOuterRsp.u));
2457	return iemRaiseNotCanonical(pVCpu);
2458	}
2459	u64BaseCs = 0;
2460	u64BaseSs = 0;
2461	}
2462	else
2463	{
2464	if (uNewRip > cbLimitCs)
2465	{
2466	Log(("retf %04x:%08RX64 %04x:%08RX64 - out of bounds (%#x)-> #GP(CS).\n",
2467	uNewCs, uNewRip, uNewOuterSs, NewOuterRsp.u, cbLimitCs));
2468	/** @todo Intel says this is \#GP(0)! */
2469	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
2470	}
2471	u64BaseCs = X86DESC_BASE(&DescCs.Legacy);
2472	u64BaseSs = X86DESC_BASE(&DescSs.Legacy);
2473	}
2474
2475	/*
2476	* Now set the accessed bit before
2477	* writing the return address to the stack and committing the result into
2478	* CS, CSHID and RIP.
2479	*/
2480	/** @todo Testcase: Need to check WHEN exactly the CS accessed bit is set. */
2481	if (!(DescCs.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
2482	{
2483	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCs);
2484	if (rcStrict != VINF_SUCCESS)
2485	return rcStrict;
2486	/** @todo check what VT-x and AMD-V does. */
2487	DescCs.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
2488	}
2489	/** @todo Testcase: Need to check WHEN exactly the SS accessed bit is set. */
2490	if (!(DescSs.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
2491	{
2492	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewOuterSs);
2493	if (rcStrict != VINF_SUCCESS)
2494	return rcStrict;
2495	/** @todo check what VT-x and AMD-V does. */
2496	DescSs.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
2497	}
2498
2499	/* commit */
2500	if (enmEffOpSize == IEMMODE_16BIT)
2501	pVCpu->cpum.GstCtx.rip = uNewRip & UINT16_MAX; /** @todo Testcase: When exactly does this occur? With call it happens prior to the limit check according to Intel... */
2502	else
2503	pVCpu->cpum.GstCtx.rip = uNewRip;
2504	pVCpu->cpum.GstCtx.cs.Sel = uNewCs;
2505	pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs;
2506	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
2507	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCs.Legacy);
2508	pVCpu->cpum.GstCtx.cs.u32Limit = cbLimitCs;
2509	pVCpu->cpum.GstCtx.cs.u64Base = u64BaseCs;
2510	pVCpu->cpum.GstCtx.ss.Sel = uNewOuterSs;
2511	pVCpu->cpum.GstCtx.ss.ValidSel = uNewOuterSs;
2512	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
2513	pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSs.Legacy);
2514	pVCpu->cpum.GstCtx.ss.u32Limit = cbLimitSs;
2515	pVCpu->cpum.GstCtx.ss.u64Base = u64BaseSs;
2516
2517	iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.ds);
2518	iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.es);
2519	iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.fs);
2520	iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.gs);
2521
2522	iemRecalcExecModeAndCplAndAcFlags(pVCpu); /* Affects iemRegAddToRspEx and the setting of RSP/SP below. */
2523
2524	if (cbPop)
2525	iemRegAddToRspEx(pVCpu, &NewOuterRsp, cbPop);
2526	if (IEM_IS_64BIT_CODE(pVCpu))
2527	pVCpu->cpum.GstCtx.rsp = NewOuterRsp.u;
2528	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2529	pVCpu->cpum.GstCtx.rsp = (uint32_t)NewOuterRsp.u;
2530	else
2531	pVCpu->cpum.GstCtx.sp = (uint16_t)NewOuterRsp.u;
2532
2533	iemRecalcExecModeAndCplAndAcFlags(pVCpu); /* Affects iemRegAddToRspEx and the setting of RSP/SP below. */
2534
2535	/** @todo check if the hidden bits are loaded correctly for 64-bit
2536	* mode. */
2537	}
2538	/*
2539	* Return to the same privilege level
2540	*/
2541	else
2542	{
2543	/* Limit / canonical check. */
2544	uint64_t u64Base;
2545	uint32_t cbLimitCs = X86DESC_LIMIT_G(&DescCs.Legacy);
2546
2547	/** @todo Testcase: Is this correct? */
2548	bool f64BitCs = false;
2549	if ( DescCs.Legacy.Gen.u1Long
2550	&& IEM_IS_LONG_MODE(pVCpu) )
2551	{
2552	if (!IEM_IS_CANONICAL(uNewRip))
2553	{
2554	Log(("retf %04x:%08RX64 - not canonical -> #GP\n", uNewCs, uNewRip));
2555	return iemRaiseNotCanonical(pVCpu);
2556	}
2557	u64Base = 0;
2558	f64BitCs = true;
2559	f64BitCs = true;
2560	}
2561	else
2562	{
2563	if (uNewRip > cbLimitCs)
2564	{
2565	Log(("retf %04x:%08RX64 -> out of bounds (%#x)\n", uNewCs, uNewRip, cbLimitCs));
2566	/** @todo Intel says this is \#GP(0)! */
2567	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
2568	}
2569	u64Base = X86DESC_BASE(&DescCs.Legacy);
2570	}
2571
2572	/*
2573	* Now set the accessed bit before
2574	* writing the return address to the stack and committing the result into
2575	* CS, CSHID and RIP.
2576	*/
2577	/** @todo Testcase: Need to check WHEN exactly the accessed bit is set. */
2578	if (!(DescCs.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
2579	{
2580	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCs);
2581	if (rcStrict != VINF_SUCCESS)
2582	return rcStrict;
2583	/** @todo check what VT-x and AMD-V does. */
2584	DescCs.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
2585	}
2586
2587	/* commit */
2588	if (cbPop)
2589	/** @todo This cannot be right. We're using the old CS mode here, and iemRegAddToRspEx checks fExec. */
2590	iemRegAddToRspEx(pVCpu, &NewRsp, cbPop);
2591	if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig \|\| f64BitCs)
2592	pVCpu->cpum.GstCtx.rsp = NewRsp.u;
2593	else
2594	pVCpu->cpum.GstCtx.sp = (uint16_t)NewRsp.u;
2595	if (enmEffOpSize == IEMMODE_16BIT)
2596	pVCpu->cpum.GstCtx.rip = uNewRip & UINT16_MAX; /** @todo Testcase: When exactly does this occur? With call it happens prior to the limit check according to Intel... */
2597	else
2598	pVCpu->cpum.GstCtx.rip = uNewRip;
2599	pVCpu->cpum.GstCtx.cs.Sel = uNewCs;
2600	pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs;
2601	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
2602	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCs.Legacy);
2603	pVCpu->cpum.GstCtx.cs.u32Limit = cbLimitCs;
2604	pVCpu->cpum.GstCtx.cs.u64Base = u64Base;
2605	/** @todo check if the hidden bits are loaded correctly for 64-bit
2606	* mode. */
2607
2608	iemRecalcExecModeAndCplAndAcFlags(pVCpu);
2609	}
2610
2611	/* Flush the prefetch buffer. */
2612	IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr); /** @todo use light flush for same privilege? */
2613
2614	return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);
2615	}
2616
2617
2618	/**
2619	* Implements enter.
2620	*
2621	* We're doing this in C because the instruction is insane, even for the
2622	* u8NestingLevel=0 case dealing with the stack is tedious.
2623	*
2624	* @param enmEffOpSize The effective operand size.
2625	* @param cbFrame Frame size.
2626	* @param cParameters Frame parameter count.
2627	*/
2628	IEM_CIMPL_DEF_3(iemCImpl_enter, IEMMODE, enmEffOpSize, uint16_t, cbFrame, uint8_t, cParameters)
2629	{
2630	/* Push RBP, saving the old value in TmpRbp. */
2631	RTUINT64U NewRsp; NewRsp.u = pVCpu->cpum.GstCtx.rsp;
2632	RTUINT64U TmpRbp; TmpRbp.u = pVCpu->cpum.GstCtx.rbp;
2633	RTUINT64U NewRbp;
2634	VBOXSTRICTRC rcStrict;
2635	if (enmEffOpSize == IEMMODE_64BIT)
2636	{
2637	rcStrict = iemMemStackPushU64Ex(pVCpu, TmpRbp.u, &NewRsp);
2638	NewRbp = NewRsp;
2639	}
2640	else if (enmEffOpSize == IEMMODE_32BIT)
2641	{
2642	rcStrict = iemMemStackPushU32Ex(pVCpu, TmpRbp.DWords.dw0, &NewRsp);
2643	NewRbp = NewRsp;
2644	}
2645	else
2646	{
2647	rcStrict = iemMemStackPushU16Ex(pVCpu, TmpRbp.Words.w0, &NewRsp);
2648	NewRbp = TmpRbp;
2649	NewRbp.Words.w0 = NewRsp.Words.w0;
2650	}
2651	if (rcStrict != VINF_SUCCESS)
2652	return rcStrict;
2653
2654	/* Copy the parameters (aka nesting levels by Intel). */
2655	cParameters &= 0x1f;
2656	if (cParameters > 0)
2657	{
2658	switch (enmEffOpSize)
2659	{
2660	case IEMMODE_16BIT:
2661	if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2662	TmpRbp.DWords.dw0 -= 2;
2663	else
2664	TmpRbp.Words.w0 -= 2;
2665	do
2666	{
2667	uint16_t u16Tmp;
2668	rcStrict = iemMemStackPopU16Ex(pVCpu, &u16Tmp, &TmpRbp);
2669	if (rcStrict != VINF_SUCCESS)
2670	break;
2671	rcStrict = iemMemStackPushU16Ex(pVCpu, u16Tmp, &NewRsp);
2672	} while (--cParameters > 0 && rcStrict == VINF_SUCCESS);
2673	break;
2674
2675	case IEMMODE_32BIT:
2676	if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2677	TmpRbp.DWords.dw0 -= 4;
2678	else
2679	TmpRbp.Words.w0 -= 4;
2680	do
2681	{
2682	uint32_t u32Tmp;
2683	rcStrict = iemMemStackPopU32Ex(pVCpu, &u32Tmp, &TmpRbp);
2684	if (rcStrict != VINF_SUCCESS)
2685	break;
2686	rcStrict = iemMemStackPushU32Ex(pVCpu, u32Tmp, &NewRsp);
2687	} while (--cParameters > 0 && rcStrict == VINF_SUCCESS);
2688	break;
2689
2690	case IEMMODE_64BIT:
2691	TmpRbp.u -= 8;
2692	do
2693	{
2694	uint64_t u64Tmp;
2695	rcStrict = iemMemStackPopU64Ex(pVCpu, &u64Tmp, &TmpRbp);
2696	if (rcStrict != VINF_SUCCESS)
2697	break;
2698	rcStrict = iemMemStackPushU64Ex(pVCpu, u64Tmp, &NewRsp);
2699	} while (--cParameters > 0 && rcStrict == VINF_SUCCESS);
2700	break;
2701
2702	IEM_NOT_REACHED_DEFAULT_CASE_RET();
2703	}
2704	if (rcStrict != VINF_SUCCESS)
2705	return VINF_SUCCESS;
2706
2707	/* Push the new RBP */
2708	if (enmEffOpSize == IEMMODE_64BIT)
2709	rcStrict = iemMemStackPushU64Ex(pVCpu, NewRbp.u, &NewRsp);
2710	else if (enmEffOpSize == IEMMODE_32BIT)
2711	rcStrict = iemMemStackPushU32Ex(pVCpu, NewRbp.DWords.dw0, &NewRsp);
2712	else
2713	rcStrict = iemMemStackPushU16Ex(pVCpu, NewRbp.Words.w0, &NewRsp);
2714	if (rcStrict != VINF_SUCCESS)
2715	return rcStrict;
2716
2717	}
2718
2719	/* Recalc RSP. */
2720	iemRegSubFromRspEx(pVCpu, &NewRsp, cbFrame);
2721
2722	/** @todo Should probe write access at the new RSP according to AMD. */
2723	/** @todo Should handle accesses to the VMX APIC-access page. */
2724
2725	/* Commit it. */
2726	pVCpu->cpum.GstCtx.rbp = NewRbp.u;
2727	pVCpu->cpum.GstCtx.rsp = NewRsp.u;
2728	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
2729	}
2730
2731
2732
2733	/**
2734	* Implements leave.
2735	*
2736	* We're doing this in C because messing with the stack registers is annoying
2737	* since they depends on SS attributes.
2738	*
2739	* @param enmEffOpSize The effective operand size.
2740	*/
2741	IEM_CIMPL_DEF_1(iemCImpl_leave, IEMMODE, enmEffOpSize)
2742	{
2743	/* Calculate the intermediate RSP from RBP and the stack attributes. */
2744	RTUINT64U NewRsp;
2745	if (IEM_IS_64BIT_CODE(pVCpu))
2746	NewRsp.u = pVCpu->cpum.GstCtx.rbp;
2747	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2748	NewRsp.u = pVCpu->cpum.GstCtx.ebp;
2749	else
2750	{
2751	/** @todo Check that LEAVE actually preserve the high EBP bits. */
2752	NewRsp.u = pVCpu->cpum.GstCtx.rsp;
2753	NewRsp.Words.w0 = pVCpu->cpum.GstCtx.bp;
2754	}
2755
2756	/* Pop RBP according to the operand size. */
2757	VBOXSTRICTRC rcStrict;
2758	RTUINT64U NewRbp;
2759	switch (enmEffOpSize)
2760	{
2761	case IEMMODE_16BIT:
2762	NewRbp.u = pVCpu->cpum.GstCtx.rbp;
2763	rcStrict = iemMemStackPopU16Ex(pVCpu, &NewRbp.Words.w0, &NewRsp);
2764	break;
2765	case IEMMODE_32BIT:
2766	NewRbp.u = 0;
2767	rcStrict = iemMemStackPopU32Ex(pVCpu, &NewRbp.DWords.dw0, &NewRsp);
2768	break;
2769	case IEMMODE_64BIT:
2770	rcStrict = iemMemStackPopU64Ex(pVCpu, &NewRbp.u, &NewRsp);
2771	break;
2772	IEM_NOT_REACHED_DEFAULT_CASE_RET();
2773	}
2774	if (rcStrict != VINF_SUCCESS)
2775	return rcStrict;
2776
2777
2778	/* Commit it. */
2779	pVCpu->cpum.GstCtx.rbp = NewRbp.u;
2780	pVCpu->cpum.GstCtx.rsp = NewRsp.u;
2781	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
2782	}
2783
2784
2785	/**
2786	* Implements int3 and int XX.
2787	*
2788	* @param u8Int The interrupt vector number.
2789	* @param enmInt The int instruction type.
2790	*/
2791	IEM_CIMPL_DEF_2(iemCImpl_int, uint8_t, u8Int, IEMINT, enmInt)
2792	{
2793	Assert(pVCpu->iem.s.cXcptRecursions == 0);
2794
2795	/*
2796	* We must check if this INT3 might belong to DBGF before raising a #BP.
2797	*/
2798	if (u8Int == 3)
2799	{
2800	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2801	if (pVM->dbgf.ro.cEnabledSwBreakpoints == 0)
2802	{ /* likely: No vbox debugger breakpoints */ }
2803	else
2804	{
2805	VBOXSTRICTRC rcStrict = DBGFTrap03Handler(pVM, pVCpu, &pVCpu->cpum.GstCtx);
2806	Log(("iemCImpl_int: DBGFTrap03Handler -> %Rrc\n", VBOXSTRICTRC_VAL(rcStrict) ));
2807	if (rcStrict != VINF_EM_RAW_GUEST_TRAP)
2808	return iemSetPassUpStatus(pVCpu, rcStrict);
2809	}
2810	}
2811	/** @todo single stepping */
2812	return iemRaiseXcptOrInt(pVCpu,
2813	cbInstr,
2814	u8Int,
2815	IEM_XCPT_FLAGS_T_SOFT_INT \| enmInt,
2816	0,
2817	0);
2818	}
2819
2820
2821	/**
2822	* Implements iret for real mode and V8086 mode.
2823	*
2824	* @param enmEffOpSize The effective operand size.
2825	*/
2826	IEM_CIMPL_DEF_1(iemCImpl_iret_real_v8086, IEMMODE, enmEffOpSize)
2827	{
2828	X86EFLAGS Efl;
2829	Efl.u = IEMMISC_GET_EFL(pVCpu);
2830	NOREF(cbInstr);
2831
2832	/*
2833	* iret throws an exception if VME isn't enabled.
2834	*/
2835	if ( Efl.Bits.u1VM
2836	&& Efl.Bits.u2IOPL != 3
2837	&& !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_VME))
2838	return iemRaiseGeneralProtectionFault0(pVCpu);
2839
2840	/*
2841	* Do the stack bits, but don't commit RSP before everything checks
2842	* out right.
2843	*/
2844	Assert(enmEffOpSize == IEMMODE_32BIT \|\| enmEffOpSize == IEMMODE_16BIT);
2845	VBOXSTRICTRC rcStrict;
2846	uint8_t bUnmapInfo;
2847	RTCPTRUNION uFrame;
2848	uint16_t uNewCs;
2849	uint32_t uNewEip;
2850	uint32_t uNewFlags;
2851	uint64_t uNewRsp;
2852	if (enmEffOpSize == IEMMODE_32BIT)
2853	{
2854	rcStrict = iemMemStackPopBeginSpecial(pVCpu, 12, 1, &uFrame.pv, &bUnmapInfo, &uNewRsp);
2855	if (rcStrict != VINF_SUCCESS)
2856	return rcStrict;
2857	uNewEip = uFrame.pu32[0];
2858	if (uNewEip > UINT16_MAX)
2859	return iemRaiseGeneralProtectionFault0(pVCpu);
2860
2861	uNewCs = (uint16_t)uFrame.pu32[1];
2862	uNewFlags = uFrame.pu32[2];
2863	uNewFlags &= X86_EFL_CF \| X86_EFL_PF \| X86_EFL_AF \| X86_EFL_ZF \| X86_EFL_SF
2864	\| X86_EFL_TF \| X86_EFL_IF \| X86_EFL_DF \| X86_EFL_OF \| X86_EFL_IOPL \| X86_EFL_NT
2865	\| X86_EFL_RF /\| X86_EFL_VM/ \| X86_EFL_AC /\|X86_EFL_VIF/ /\|X86_EFL_VIP/
2866	\| X86_EFL_ID;
2867	if (IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_386)
2868	uNewFlags &= ~(X86_EFL_AC \| X86_EFL_ID \| X86_EFL_VIF \| X86_EFL_VIP);
2869	uNewFlags \|= Efl.u & (X86_EFL_VM \| X86_EFL_VIF \| X86_EFL_VIP \| X86_EFL_1);
2870	}
2871	else
2872	{
2873	rcStrict = iemMemStackPopBeginSpecial(pVCpu, 6, 1, &uFrame.pv, &bUnmapInfo, &uNewRsp);
2874	if (rcStrict != VINF_SUCCESS)
2875	return rcStrict;
2876	uNewEip = uFrame.pu16[0];
2877	uNewCs = uFrame.pu16[1];
2878	uNewFlags = uFrame.pu16[2];
2879	uNewFlags &= X86_EFL_CF \| X86_EFL_PF \| X86_EFL_AF \| X86_EFL_ZF \| X86_EFL_SF
2880	\| X86_EFL_TF \| X86_EFL_IF \| X86_EFL_DF \| X86_EFL_OF \| X86_EFL_IOPL \| X86_EFL_NT;
2881	uNewFlags \|= Efl.u & ((UINT32_C(0xffff0000) \| X86_EFL_1) & ~X86_EFL_RF);
2882	/** @todo The intel pseudo code does not indicate what happens to
2883	* reserved flags. We just ignore them. */
2884	/* Ancient CPU adjustments: See iemCImpl_popf. */
2885	if (IEM_GET_TARGET_CPU(pVCpu) == IEMTARGETCPU_286)
2886	uNewFlags &= ~(X86_EFL_NT \| X86_EFL_IOPL);
2887	}
2888	rcStrict = iemMemStackPopDoneSpecial(pVCpu, bUnmapInfo);
2889	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
2890	{ /* extremely likely */ }
2891	else
2892	return rcStrict;
2893
2894	/** @todo Check how this is supposed to work if sp=0xfffe. */
2895	Log7(("iemCImpl_iret_real_v8086: uNewCs=%#06x uNewRip=%#010x uNewFlags=%#x uNewRsp=%#18llx\n",
2896	uNewCs, uNewEip, uNewFlags, uNewRsp));
2897
2898	/*
2899	* Check the limit of the new EIP.
2900	*/
2901	/** @todo Only the AMD pseudo code check the limit here, what's
2902	* right? */
2903	if (uNewEip > pVCpu->cpum.GstCtx.cs.u32Limit)
2904	return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
2905
2906	/*
2907	* V8086 checks and flag adjustments
2908	*/
2909	if (Efl.Bits.u1VM)
2910	{
2911	if (Efl.Bits.u2IOPL == 3)
2912	{
2913	/* Preserve IOPL and clear RF. */
2914	uNewFlags &= ~(X86_EFL_IOPL \| X86_EFL_RF);
2915	uNewFlags \|= Efl.u & (X86_EFL_IOPL);
2916	}
2917	else if ( enmEffOpSize == IEMMODE_16BIT
2918	&& ( !(uNewFlags & X86_EFL_IF)
2919	\|\| !Efl.Bits.u1VIP )
2920	&& !(uNewFlags & X86_EFL_TF) )
2921	{
2922	/* Move IF to VIF, clear RF and preserve IF and IOPL.*/
2923	uNewFlags &= ~X86_EFL_VIF;
2924	uNewFlags \|= (uNewFlags & X86_EFL_IF) << (19 - 9);
2925	uNewFlags &= ~(X86_EFL_IF \| X86_EFL_IOPL \| X86_EFL_RF);
2926	uNewFlags \|= Efl.u & (X86_EFL_IF \| X86_EFL_IOPL);
2927	}
2928	else
2929	return iemRaiseGeneralProtectionFault0(pVCpu);
2930	Log7(("iemCImpl_iret_real_v8086: u1VM=1: adjusted uNewFlags=%#x\n", uNewFlags));
2931	}
2932
2933	/*
2934	* Commit the operation.
2935	*/
2936	IEMTLBTRACE_IRET(pVCpu, uNewCs, uNewEip, uNewFlags);
2937	#ifdef DBGFTRACE_ENABLED
2938	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "iret/rm %04x:%04x -> %04x:%04x %x %04llx",
2939	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, uNewCs, uNewEip, uNewFlags, uNewRsp);
2940	#endif
2941	pVCpu->cpum.GstCtx.rsp = uNewRsp;
2942	pVCpu->cpum.GstCtx.rip = uNewEip;
2943	pVCpu->cpum.GstCtx.cs.Sel = uNewCs;
2944	pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs;
2945	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
2946	pVCpu->cpum.GstCtx.cs.u64Base = (uint32_t)uNewCs << 4;
2947	/** @todo do we load attribs and limit as well? */
2948	Assert(uNewFlags & X86_EFL_1);
2949	IEMMISC_SET_EFL(pVCpu, uNewFlags);
2950	pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~IEM_F_X86_AC) \| iemCalcExecAcFlag(pVCpu);
2951
2952	/* Flush the prefetch buffer. */
2953	IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr); /** @todo can do light flush in real mode at least */
2954
2955	/** @todo single stepping */
2956	return VINF_SUCCESS;
2957	}
2958
2959
2960	/**
2961	* Loads a segment register when entering V8086 mode.
2962	*
2963	* @param pSReg The segment register.
2964	* @param uSeg The segment to load.
2965	*/
2966	static void iemCImplCommonV8086LoadSeg(PCPUMSELREG pSReg, uint16_t uSeg)
2967	{
2968	pSReg->Sel = uSeg;
2969	pSReg->ValidSel = uSeg;
2970	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
2971	pSReg->u64Base = (uint32_t)uSeg << 4;
2972	pSReg->u32Limit = 0xffff;
2973	pSReg->Attr.u = X86_SEL_TYPE_RW_ACC \| RT_BIT(4) /!sys/ \| RT_BIT(7) /P/ \| (3 /DPL/ << 5); /* VT-x wants 0xf3 */
2974	/** @todo Testcase: Check if VT-x really needs this and what it does itself when
2975	* IRET'ing to V8086. */
2976	}
2977
2978
2979	/**
2980	* Implements iret for protected mode returning to V8086 mode.
2981	*
2982	* @param uNewEip The new EIP.
2983	* @param uNewCs The new CS.
2984	* @param uNewFlags The new EFLAGS.
2985	* @param uNewRsp The RSP after the initial IRET frame.
2986	*
2987	* @note This can only be a 32-bit iret du to the X86_EFL_VM position.
2988	*/
2989	IEM_CIMPL_DEF_4(iemCImpl_iret_prot_v8086, uint32_t, uNewEip, uint16_t, uNewCs, uint32_t, uNewFlags, uint64_t, uNewRsp)
2990	{
2991	RT_NOREF_PV(cbInstr);
2992	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_MASK);
2993
2994	/*
2995	* Pop the V8086 specific frame bits off the stack.
2996	*/
2997	uint8_t bUnmapInfo;
2998	RTCPTRUNION uFrame;
2999	VBOXSTRICTRC rcStrict = iemMemStackPopContinueSpecial(pVCpu, 0 /off/, 24 /cbMem/, &uFrame.pv, &bUnmapInfo, uNewRsp);
3000	if (rcStrict != VINF_SUCCESS)
3001	return rcStrict;
3002	uint32_t uNewEsp = uFrame.pu32[0];
3003	uint16_t uNewSs = uFrame.pu32[1];
3004	uint16_t uNewEs = uFrame.pu32[2];
3005	uint16_t uNewDs = uFrame.pu32[3];
3006	uint16_t uNewFs = uFrame.pu32[4];
3007	uint16_t uNewGs = uFrame.pu32[5];
3008	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo); /* don't use iemMemStackPopCommitSpecial here. */
3009	if (rcStrict != VINF_SUCCESS)
3010	return rcStrict;
3011
3012	/*
3013	* Commit the operation.
3014	*/
3015	uNewFlags &= X86_EFL_LIVE_MASK;
3016	uNewFlags \|= X86_EFL_RA1_MASK;
3017	#ifdef DBGFTRACE_ENABLED
3018	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "iret/p/v %04x:%08x -> %04x:%04x %x %04x:%04x",
3019	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, uNewCs, uNewEip, uNewFlags, uNewSs, uNewEsp);
3020	#endif
3021	Log7(("iemCImpl_iret_prot_v8086: %04x:%08x -> %04x:%04x %x %04x:%04x\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, uNewCs, uNewEip, uNewFlags, uNewSs, uNewEsp));
3022
3023	IEMMISC_SET_EFL(pVCpu, uNewFlags);
3024	iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.cs, uNewCs);
3025	iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.ss, uNewSs);
3026	iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.es, uNewEs);
3027	iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.ds, uNewDs);
3028	iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.fs, uNewFs);
3029	iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.gs, uNewGs);
3030	pVCpu->cpum.GstCtx.rip = (uint16_t)uNewEip;
3031	pVCpu->cpum.GstCtx.rsp = uNewEsp; /** @todo check this out! */
3032	pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~(IEM_F_MODE_MASK \| IEM_F_X86_CPL_MASK \| IEM_F_X86_AC))
3033	\| (3 << IEM_F_X86_CPL_SHIFT)
3034	\| IEM_F_MODE_X86_16BIT_PROT_V86
3035	\| iemCalcExecAcFlag(pVCpu);
3036
3037	/* Flush the prefetch buffer. */
3038	IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr);
3039
3040	/** @todo single stepping */
3041	return VINF_SUCCESS;
3042	}
3043
3044
3045	/**
3046	* Implements iret for protected mode returning via a nested task.
3047	*
3048	* @param enmEffOpSize The effective operand size.
3049	*/
3050	IEM_CIMPL_DEF_1(iemCImpl_iret_prot_NestedTask, IEMMODE, enmEffOpSize)
3051	{
3052	Log7(("iemCImpl_iret_prot_NestedTask:\n"));
3053	#ifndef IEM_IMPLEMENTS_TASKSWITCH
3054	IEM_RETURN_ASPECT_NOT_IMPLEMENTED();
3055	#else
3056	RT_NOREF_PV(enmEffOpSize);
3057
3058	/*
3059	* Read the segment selector in the link-field of the current TSS.
3060	*/
3061	RTSEL uSelRet;
3062	VBOXSTRICTRC rcStrict = iemMemFetchSysU16(pVCpu, &uSelRet, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base);
3063	if (rcStrict != VINF_SUCCESS)
3064	return rcStrict;
3065
3066	/*
3067	* Fetch the returning task's TSS descriptor from the GDT.
3068	*/
3069	if (uSelRet & X86_SEL_LDT)
3070	{
3071	Log(("iret_prot_NestedTask TSS not in LDT. uSelRet=%04x -> #TS\n", uSelRet));
3072	return iemRaiseTaskSwitchFaultBySelector(pVCpu, uSelRet);
3073	}
3074
3075	IEMSELDESC TssDesc;
3076	rcStrict = iemMemFetchSelDesc(pVCpu, &TssDesc, uSelRet, X86_XCPT_GP);
3077	if (rcStrict != VINF_SUCCESS)
3078	return rcStrict;
3079
3080	if (TssDesc.Legacy.Gate.u1DescType)
3081	{
3082	Log(("iret_prot_NestedTask Invalid TSS type. uSelRet=%04x -> #TS\n", uSelRet));
3083	return iemRaiseTaskSwitchFaultBySelector(pVCpu, uSelRet & X86_SEL_MASK_OFF_RPL);
3084	}
3085
3086	if ( TssDesc.Legacy.Gate.u4Type != X86_SEL_TYPE_SYS_286_TSS_BUSY
3087	&& TssDesc.Legacy.Gate.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
3088	{
3089	Log(("iret_prot_NestedTask TSS is not busy. uSelRet=%04x DescType=%#x -> #TS\n", uSelRet, TssDesc.Legacy.Gate.u4Type));
3090	return iemRaiseTaskSwitchFaultBySelector(pVCpu, uSelRet & X86_SEL_MASK_OFF_RPL);
3091	}
3092
3093	if (!TssDesc.Legacy.Gate.u1Present)
3094	{
3095	Log(("iret_prot_NestedTask TSS is not present. uSelRet=%04x -> #NP\n", uSelRet));
3096	return iemRaiseSelectorNotPresentBySelector(pVCpu, uSelRet & X86_SEL_MASK_OFF_RPL);
3097	}
3098
3099	uint32_t uNextEip = pVCpu->cpum.GstCtx.eip + cbInstr;
3100	return iemTaskSwitch(pVCpu, IEMTASKSWITCH_IRET, uNextEip, 0 /* fFlags /, 0 / uErr */,
3101	0 /* uCr2 */, uSelRet, &TssDesc);
3102	#endif
3103	}
3104
3105
3106	/**
3107	* Implements iret for protected mode
3108	*
3109	* @param enmEffOpSize The effective operand size.
3110	*/
3111	IEM_CIMPL_DEF_1(iemCImpl_iret_prot, IEMMODE, enmEffOpSize)
3112	{
3113	NOREF(cbInstr);
3114	Assert(enmEffOpSize == IEMMODE_32BIT \|\| enmEffOpSize == IEMMODE_16BIT);
3115
3116	/*
3117	* Nested task return.
3118	*/
3119	if (pVCpu->cpum.GstCtx.eflags.Bits.u1NT)
3120	return IEM_CIMPL_CALL_1(iemCImpl_iret_prot_NestedTask, enmEffOpSize);
3121
3122	/*
3123	* Normal return.
3124	*
3125	* Do the stack bits, but don't commit RSP before everything checks
3126	* out right.
3127	*/
3128	Assert(enmEffOpSize == IEMMODE_32BIT \|\| enmEffOpSize == IEMMODE_16BIT);
3129	uint8_t bUnmapInfo;
3130	VBOXSTRICTRC rcStrict;
3131	RTCPTRUNION uFrame;
3132	uint16_t uNewCs;
3133	uint32_t uNewEip;
3134	uint32_t uNewFlags;
3135	uint64_t uNewRsp;
3136	if (enmEffOpSize == IEMMODE_32BIT)
3137	{
3138	rcStrict = iemMemStackPopBeginSpecial(pVCpu, 12, 3, &uFrame.pv, &bUnmapInfo, &uNewRsp);
3139	if (rcStrict != VINF_SUCCESS)
3140	return rcStrict;
3141	uNewEip = uFrame.pu32[0];
3142	uNewCs = (uint16_t)uFrame.pu32[1];
3143	uNewFlags = uFrame.pu32[2];
3144	}
3145	else
3146	{
3147	rcStrict = iemMemStackPopBeginSpecial(pVCpu, 6, 1, &uFrame.pv, &bUnmapInfo, &uNewRsp);
3148	if (rcStrict != VINF_SUCCESS)
3149	return rcStrict;
3150	uNewEip = uFrame.pu16[0];
3151	uNewCs = uFrame.pu16[1];
3152	uNewFlags = uFrame.pu16[2];
3153	}
3154	rcStrict = iemMemStackPopDoneSpecial(pVCpu, bUnmapInfo); /* don't use iemMemStackPopCommitSpecial here. */
3155	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
3156	{ /* extremely likely */ }
3157	else
3158	return rcStrict;
3159	Log7(("iemCImpl_iret_prot: uNewCs=%#06x uNewEip=%#010x uNewFlags=%#x uNewRsp=%#18llx uCpl=%u\n", uNewCs, uNewEip, uNewFlags, uNewRsp, IEM_GET_CPL(pVCpu)));
3160
3161	/*
3162	* We're hopefully not returning to V8086 mode...
3163	*/
3164	if ( (uNewFlags & X86_EFL_VM)
3165	&& IEM_GET_CPL(pVCpu) == 0)
3166	{
3167	Assert(enmEffOpSize == IEMMODE_32BIT);
3168	return IEM_CIMPL_CALL_4(iemCImpl_iret_prot_v8086, uNewEip, uNewCs, uNewFlags, uNewRsp);
3169	}
3170
3171	/*
3172	* Protected mode.
3173	*/
3174	/* Read the CS descriptor. */
3175	if (!(uNewCs & X86_SEL_MASK_OFF_RPL))
3176	{
3177	Log(("iret %04x:%08x -> invalid CS selector, #GP(0)\n", uNewCs, uNewEip));
3178	return iemRaiseGeneralProtectionFault0(pVCpu);
3179	}
3180
3181	IEMSELDESC DescCS;
3182	rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, uNewCs, X86_XCPT_GP);
3183	if (rcStrict != VINF_SUCCESS)
3184	{
3185	Log(("iret %04x:%08x - rcStrict=%Rrc when fetching CS\n", uNewCs, uNewEip, VBOXSTRICTRC_VAL(rcStrict)));
3186	return rcStrict;
3187	}
3188
3189	/* Must be a code descriptor. */
3190	if (!DescCS.Legacy.Gen.u1DescType)
3191	{
3192	Log(("iret %04x:%08x - CS is system segment (%#x) -> #GP\n", uNewCs, uNewEip, DescCS.Legacy.Gen.u4Type));
3193	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
3194	}
3195	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
3196	{
3197	Log(("iret %04x:%08x - not code segment (%#x) -> #GP\n", uNewCs, uNewEip, DescCS.Legacy.Gen.u4Type));
3198	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
3199	}
3200
3201	/* Privilege checks. */
3202	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF))
3203	{
3204	if ((uNewCs & X86_SEL_RPL) != DescCS.Legacy.Gen.u2Dpl)
3205	{
3206	Log(("iret %04x:%08x - RPL != DPL (%d) -> #GP\n", uNewCs, uNewEip, DescCS.Legacy.Gen.u2Dpl));
3207	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
3208	}
3209	}
3210	else if ((uNewCs & X86_SEL_RPL) < DescCS.Legacy.Gen.u2Dpl)
3211	{
3212	Log(("iret %04x:%08x - RPL < DPL (%d) -> #GP\n", uNewCs, uNewEip, DescCS.Legacy.Gen.u2Dpl));
3213	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
3214	}
3215	if ((uNewCs & X86_SEL_RPL) < IEM_GET_CPL(pVCpu))
3216	{
3217	Log(("iret %04x:%08x - RPL < CPL (%d) -> #GP\n", uNewCs, uNewEip, IEM_GET_CPL(pVCpu)));
3218	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
3219	}
3220
3221	/* Present? */
3222	if (!DescCS.Legacy.Gen.u1Present)
3223	{
3224	Log(("iret %04x:%08x - CS not present -> #NP\n", uNewCs, uNewEip));
3225	return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewCs);
3226	}
3227
3228	uint32_t cbLimitCS = X86DESC_LIMIT_G(&DescCS.Legacy);
3229
3230	/*
3231	* Return to outer level?
3232	*/
3233	if ((uNewCs & X86_SEL_RPL) != IEM_GET_CPL(pVCpu))
3234	{
3235	uint16_t uNewSS;
3236	uint32_t uNewESP;
3237	if (enmEffOpSize == IEMMODE_32BIT)
3238	{
3239	rcStrict = iemMemStackPopContinueSpecial(pVCpu, 0/off/, 8 /cbMem/, &uFrame.pv, &bUnmapInfo, uNewRsp);
3240	if (rcStrict != VINF_SUCCESS)
3241	return rcStrict;
3242	/** @todo We might be popping a 32-bit ESP from the IRET frame, but whether
3243	* 16-bit or 32-bit are being loaded into SP depends on the D/B
3244	* bit of the popped SS selector it turns out. */
3245	uNewESP = uFrame.pu32[0];
3246	uNewSS = (uint16_t)uFrame.pu32[1];
3247	}
3248	else
3249	{
3250	rcStrict = iemMemStackPopContinueSpecial(pVCpu, 0 /off/, 4 /cbMem/, &uFrame.pv, &bUnmapInfo, uNewRsp);
3251	if (rcStrict != VINF_SUCCESS)
3252	return rcStrict;
3253	uNewESP = uFrame.pu16[0];
3254	uNewSS = uFrame.pu16[1];
3255	}
3256	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
3257	if (rcStrict != VINF_SUCCESS)
3258	return rcStrict;
3259	Log7(("iemCImpl_iret_prot: uNewSS=%#06x uNewESP=%#010x\n", uNewSS, uNewESP));
3260
3261	/* Read the SS descriptor. */
3262	if (!(uNewSS & X86_SEL_MASK_OFF_RPL))
3263	{
3264	Log(("iret %04x:%08x/%04x:%08x -> invalid SS selector, #GP(0)\n", uNewCs, uNewEip, uNewSS, uNewESP));
3265	return iemRaiseGeneralProtectionFault0(pVCpu);
3266	}
3267
3268	IEMSELDESC DescSS;
3269	rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSS, X86_XCPT_GP); /** @todo Correct exception? */
3270	if (rcStrict != VINF_SUCCESS)
3271	{
3272	Log(("iret %04x:%08x/%04x:%08x - %Rrc when fetching SS\n",
3273	uNewCs, uNewEip, uNewSS, uNewESP, VBOXSTRICTRC_VAL(rcStrict)));
3274	return rcStrict;
3275	}
3276
3277	/* Privilege checks. */
3278	if ((uNewSS & X86_SEL_RPL) != (uNewCs & X86_SEL_RPL))
3279	{
3280	Log(("iret %04x:%08x/%04x:%08x -> SS.RPL != CS.RPL -> #GP\n", uNewCs, uNewEip, uNewSS, uNewESP));
3281	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSS);
3282	}
3283	if (DescSS.Legacy.Gen.u2Dpl != (uNewCs & X86_SEL_RPL))
3284	{
3285	Log(("iret %04x:%08x/%04x:%08x -> SS.DPL (%d) != CS.RPL -> #GP\n",
3286	uNewCs, uNewEip, uNewSS, uNewESP, DescSS.Legacy.Gen.u2Dpl));
3287	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSS);
3288	}
3289
3290	/* Must be a writeable data segment descriptor. */
3291	if (!DescSS.Legacy.Gen.u1DescType)
3292	{
3293	Log(("iret %04x:%08x/%04x:%08x -> SS is system segment (%#x) -> #GP\n",
3294	uNewCs, uNewEip, uNewSS, uNewESP, DescSS.Legacy.Gen.u4Type));
3295	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSS);
3296	}
3297	if ((DescSS.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_WRITE)) != X86_SEL_TYPE_WRITE)
3298	{
3299	Log(("iret %04x:%08x/%04x:%08x - not writable data segment (%#x) -> #GP\n",
3300	uNewCs, uNewEip, uNewSS, uNewESP, DescSS.Legacy.Gen.u4Type));
3301	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSS);
3302	}
3303
3304	/* Present? */
3305	if (!DescSS.Legacy.Gen.u1Present)
3306	{
3307	Log(("iret %04x:%08x/%04x:%08x -> SS not present -> #SS\n", uNewCs, uNewEip, uNewSS, uNewESP));
3308	return iemRaiseStackSelectorNotPresentBySelector(pVCpu, uNewSS);
3309	}
3310
3311	uint32_t cbLimitSs = X86DESC_LIMIT_G(&DescSS.Legacy);
3312
3313	/* Check EIP. */
3314	if (uNewEip > cbLimitCS)
3315	{
3316	Log(("iret %04x:%08x/%04x:%08x -> EIP is out of bounds (%#x) -> #GP(0)\n",
3317	uNewCs, uNewEip, uNewSS, uNewESP, cbLimitCS));
3318	/** @todo Which is it, \#GP(0) or \#GP(sel)? */
3319	return iemRaiseSelectorBoundsBySelector(pVCpu, uNewCs);
3320	}
3321
3322	/*
3323	* Commit the changes, marking CS and SS accessed first since
3324	* that may fail.
3325	*/
3326	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3327	{
3328	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCs);
3329	if (rcStrict != VINF_SUCCESS)
3330	return rcStrict;
3331	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3332	}
3333	if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3334	{
3335	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewSS);
3336	if (rcStrict != VINF_SUCCESS)
3337	return rcStrict;
3338	DescSS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3339	}
3340
3341	uint32_t fEFlagsMask = X86_EFL_CF \| X86_EFL_PF \| X86_EFL_AF \| X86_EFL_ZF \| X86_EFL_SF
3342	\| X86_EFL_TF \| X86_EFL_DF \| X86_EFL_OF \| X86_EFL_NT;
3343	if (enmEffOpSize != IEMMODE_16BIT)
3344	fEFlagsMask \|= X86_EFL_RF \| X86_EFL_AC \| X86_EFL_ID;
3345	if (IEM_GET_CPL(pVCpu) == 0)
3346	fEFlagsMask \|= X86_EFL_IF \| X86_EFL_IOPL \| X86_EFL_VIF \| X86_EFL_VIP; /* VM is 0 */
3347	else if (IEM_GET_CPL(pVCpu) <= pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL)
3348	fEFlagsMask \|= X86_EFL_IF;
3349	if (IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_386)
3350	fEFlagsMask &= ~(X86_EFL_AC \| X86_EFL_ID \| X86_EFL_VIF \| X86_EFL_VIP);
3351	uint32_t fEFlagsNew = IEMMISC_GET_EFL(pVCpu);
3352	fEFlagsNew &= ~fEFlagsMask;
3353	fEFlagsNew \|= uNewFlags & fEFlagsMask;
3354	IEMTLBTRACE_IRET(pVCpu, uNewCs, uNewEip, fEFlagsNew);
3355	#ifdef DBGFTRACE_ENABLED
3356	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "iret/%up%u %04x:%08x -> %04x:%04x %x %04x:%04x",
3357	IEM_GET_CPL(pVCpu), uNewCs & X86_SEL_RPL, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip,
3358	uNewCs, uNewEip, uNewFlags, uNewSS, uNewESP);
3359	#endif
3360
3361	IEMMISC_SET_EFL(pVCpu, fEFlagsNew);
3362	pVCpu->cpum.GstCtx.rip = uNewEip;
3363	pVCpu->cpum.GstCtx.cs.Sel = uNewCs;
3364	pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs;
3365	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
3366	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
3367	pVCpu->cpum.GstCtx.cs.u32Limit = cbLimitCS;
3368	pVCpu->cpum.GstCtx.cs.u64Base = X86DESC_BASE(&DescCS.Legacy);
3369
3370	pVCpu->cpum.GstCtx.ss.Sel = uNewSS;
3371	pVCpu->cpum.GstCtx.ss.ValidSel = uNewSS;
3372	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
3373	pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
3374	pVCpu->cpum.GstCtx.ss.u32Limit = cbLimitSs;
3375	pVCpu->cpum.GstCtx.ss.u64Base = X86DESC_BASE(&DescSS.Legacy);
3376	if (!pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
3377	pVCpu->cpum.GstCtx.sp = (uint16_t)uNewESP;
3378	else
3379	pVCpu->cpum.GstCtx.rsp = uNewESP;
3380
3381	iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.ds);
3382	iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.es);
3383	iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.fs);
3384	iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.gs);
3385
3386	iemRecalcExecModeAndCplAndAcFlags(pVCpu);
3387
3388	/* Done! */
3389
3390	}
3391	/*
3392	* Return to the same level.
3393	*/
3394	else
3395	{
3396	/* Check EIP. */
3397	if (uNewEip > cbLimitCS)
3398	{
3399	Log(("iret %04x:%08x - EIP is out of bounds (%#x) -> #GP(0)\n", uNewCs, uNewEip, cbLimitCS));
3400	/** @todo Which is it, \#GP(0) or \#GP(sel)? */
3401	return iemRaiseSelectorBoundsBySelector(pVCpu, uNewCs);
3402	}
3403
3404	/*
3405	* Commit the changes, marking CS first since it may fail.
3406	*/
3407	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3408	{
3409	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCs);
3410	if (rcStrict != VINF_SUCCESS)
3411	return rcStrict;
3412	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3413	}
3414
3415	X86EFLAGS NewEfl;
3416	NewEfl.u = IEMMISC_GET_EFL(pVCpu);
3417	uint32_t fEFlagsMask = X86_EFL_CF \| X86_EFL_PF \| X86_EFL_AF \| X86_EFL_ZF \| X86_EFL_SF
3418	\| X86_EFL_TF \| X86_EFL_DF \| X86_EFL_OF \| X86_EFL_NT;
3419	if (enmEffOpSize != IEMMODE_16BIT)
3420	fEFlagsMask \|= X86_EFL_RF \| X86_EFL_AC \| X86_EFL_ID;
3421	if (IEM_GET_CPL(pVCpu) == 0)
3422	fEFlagsMask \|= X86_EFL_IF \| X86_EFL_IOPL \| X86_EFL_VIF \| X86_EFL_VIP; /* VM is 0 */
3423	else if (IEM_GET_CPL(pVCpu) <= NewEfl.Bits.u2IOPL)
3424	fEFlagsMask \|= X86_EFL_IF;
3425	if (IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_386)
3426	fEFlagsMask &= ~(X86_EFL_AC \| X86_EFL_ID \| X86_EFL_VIF \| X86_EFL_VIP);
3427	NewEfl.u &= ~fEFlagsMask;
3428	NewEfl.u \|= fEFlagsMask & uNewFlags;
3429	IEMTLBTRACE_IRET(pVCpu, uNewCs, uNewEip, NewEfl.u);
3430	#ifdef DBGFTRACE_ENABLED
3431	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "iret/%up %04x:%08x -> %04x:%04x %x %04x:%04llx",
3432	IEM_GET_CPL(pVCpu), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip,
3433	uNewCs, uNewEip, uNewFlags, pVCpu->cpum.GstCtx.ss.Sel, uNewRsp);
3434	#endif
3435
3436	IEMMISC_SET_EFL(pVCpu, NewEfl.u);
3437	pVCpu->cpum.GstCtx.rip = uNewEip;
3438	pVCpu->cpum.GstCtx.cs.Sel = uNewCs;
3439	pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs;
3440	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
3441	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
3442	pVCpu->cpum.GstCtx.cs.u32Limit = cbLimitCS;
3443	pVCpu->cpum.GstCtx.cs.u64Base = X86DESC_BASE(&DescCS.Legacy);
3444	if (!pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
3445	pVCpu->cpum.GstCtx.sp = (uint16_t)uNewRsp;
3446	else
3447	pVCpu->cpum.GstCtx.rsp = uNewRsp;
3448
3449	iemRecalcExecModeAndCplAndAcFlags(pVCpu);
3450
3451	/* Done! */
3452	}
3453
3454	/* Flush the prefetch buffer. */
3455	IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr); /** @todo may light flush if same ring? */
3456
3457	/** @todo single stepping */
3458	return VINF_SUCCESS;
3459	}
3460
3461
3462	/**
3463	* Implements iret for long mode
3464	*
3465	* @param enmEffOpSize The effective operand size.
3466	*/
3467	IEM_CIMPL_DEF_1(iemCImpl_iret_64bit, IEMMODE, enmEffOpSize)
3468	{
3469	NOREF(cbInstr);
3470
3471	/*
3472	* Nested task return is not supported in long mode.
3473	*/
3474	if (pVCpu->cpum.GstCtx.eflags.Bits.u1NT)
3475	{
3476	Log(("iret/64 with NT=1 (eflags=%#x) -> #GP(0)\n", pVCpu->cpum.GstCtx.eflags.u));
3477	return iemRaiseGeneralProtectionFault0(pVCpu);
3478	}
3479
3480	/*
3481	* Normal return.
3482	*
3483	* Do the stack bits, but don't commit RSP before everything checks
3484	* out right.
3485	*/
3486	VBOXSTRICTRC rcStrict;
3487	uint8_t bUnmapInfo;
3488	RTCPTRUNION uFrame;
3489	uint64_t uNewRip;
3490	uint16_t uNewCs;
3491	uint16_t uNewSs;
3492	uint32_t uNewFlags;
3493	uint64_t uNewRsp;
3494	if (enmEffOpSize == IEMMODE_64BIT)
3495	{
3496	rcStrict = iemMemStackPopBeginSpecial(pVCpu, 5*8, 7, &uFrame.pv, &bUnmapInfo, &uNewRsp);
3497	if (rcStrict != VINF_SUCCESS)
3498	return rcStrict;
3499	uNewRip = uFrame.pu64[0];
3500	uNewCs = (uint16_t)uFrame.pu64[1];
3501	uNewFlags = (uint32_t)uFrame.pu64[2];
3502	uNewRsp = uFrame.pu64[3];
3503	uNewSs = (uint16_t)uFrame.pu64[4];
3504	}
3505	else if (enmEffOpSize == IEMMODE_32BIT)
3506	{
3507	rcStrict = iemMemStackPopBeginSpecial(pVCpu, 5*4, 3, &uFrame.pv, &bUnmapInfo, &uNewRsp);
3508	if (rcStrict != VINF_SUCCESS)
3509	return rcStrict;
3510	uNewRip = uFrame.pu32[0];
3511	uNewCs = (uint16_t)uFrame.pu32[1];
3512	uNewFlags = uFrame.pu32[2];
3513	uNewRsp = uFrame.pu32[3];
3514	uNewSs = (uint16_t)uFrame.pu32[4];
3515	}
3516	else
3517	{
3518	Assert(enmEffOpSize == IEMMODE_16BIT);
3519	rcStrict = iemMemStackPopBeginSpecial(pVCpu, 5*2, 1, &uFrame.pv, &bUnmapInfo, &uNewRsp);
3520	if (rcStrict != VINF_SUCCESS)
3521	return rcStrict;
3522	uNewRip = uFrame.pu16[0];
3523	uNewCs = uFrame.pu16[1];
3524	uNewFlags = uFrame.pu16[2];
3525	uNewRsp = uFrame.pu16[3];
3526	uNewSs = uFrame.pu16[4];
3527	}
3528	rcStrict = iemMemStackPopDoneSpecial(pVCpu, bUnmapInfo); /* don't use iemMemStackPopCommitSpecial here. */
3529	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
3530	{ /* extremely like */ }
3531	else
3532	return rcStrict;
3533	Log7(("iret/64 stack: cs:rip=%04x:%016RX64 rflags=%016RX64 ss:rsp=%04x:%016RX64\n", uNewCs, uNewRip, uNewFlags, uNewSs, uNewRsp));
3534
3535	/*
3536	* Check stuff.
3537	*/
3538	/* Read the CS descriptor. */
3539	if (!(uNewCs & X86_SEL_MASK_OFF_RPL))
3540	{
3541	Log(("iret/64 %04x:%016RX64/%04x:%016RX64 -> invalid CS selector, #GP(0)\n", uNewCs, uNewRip, uNewSs, uNewRsp));
3542	return iemRaiseGeneralProtectionFault0(pVCpu);
3543	}
3544
3545	IEMSELDESC DescCS;
3546	rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, uNewCs, X86_XCPT_GP);
3547	if (rcStrict != VINF_SUCCESS)
3548	{
3549	Log(("iret/64 %04x:%016RX64/%04x:%016RX64 - rcStrict=%Rrc when fetching CS\n",
3550	uNewCs, uNewRip, uNewSs, uNewRsp, VBOXSTRICTRC_VAL(rcStrict)));
3551	return rcStrict;
3552	}
3553
3554	/* Must be a code descriptor. */
3555	if ( !DescCS.Legacy.Gen.u1DescType
3556	\|\| !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
3557	{
3558	Log(("iret/64 %04x:%016RX64/%04x:%016RX64 - CS is not a code segment T=%u T=%#xu -> #GP\n",
3559	uNewCs, uNewRip, uNewSs, uNewRsp, DescCS.Legacy.Gen.u1DescType, DescCS.Legacy.Gen.u4Type));
3560	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
3561	}
3562
3563	/* Privilege checks. */
3564	uint8_t const uNewCpl = uNewCs & X86_SEL_RPL;
3565	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF))
3566	{
3567	if ((uNewCs & X86_SEL_RPL) != DescCS.Legacy.Gen.u2Dpl)
3568	{
3569	Log(("iret/64 %04x:%016RX64 - RPL != DPL (%d) -> #GP\n", uNewCs, uNewRip, DescCS.Legacy.Gen.u2Dpl));
3570	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
3571	}
3572	}
3573	else if ((uNewCs & X86_SEL_RPL) < DescCS.Legacy.Gen.u2Dpl)
3574	{
3575	Log(("iret/64 %04x:%016RX64 - RPL < DPL (%d) -> #GP\n", uNewCs, uNewRip, DescCS.Legacy.Gen.u2Dpl));
3576	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
3577	}
3578	if ((uNewCs & X86_SEL_RPL) < IEM_GET_CPL(pVCpu))
3579	{
3580	Log(("iret/64 %04x:%016RX64 - RPL < CPL (%d) -> #GP\n", uNewCs, uNewRip, IEM_GET_CPL(pVCpu)));
3581	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
3582	}
3583
3584	/* Present? */
3585	if (!DescCS.Legacy.Gen.u1Present)
3586	{
3587	Log(("iret/64 %04x:%016RX64/%04x:%016RX64 - CS not present -> #NP\n", uNewCs, uNewRip, uNewSs, uNewRsp));
3588	return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewCs);
3589	}
3590
3591	uint32_t cbLimitCS = X86DESC_LIMIT_G(&DescCS.Legacy);
3592
3593	/* Read the SS descriptor. */
3594	IEMSELDESC DescSS;
3595	if (!(uNewSs & X86_SEL_MASK_OFF_RPL))
3596	{
3597	if ( !DescCS.Legacy.Gen.u1Long
3598	\|\| DescCS.Legacy.Gen.u1DefBig /** @todo exactly how does iret (and others) behave with u1Long=1 and u1DefBig=1? \#GP(sel)? */
3599	\|\| uNewCpl > 2) /** @todo verify SS=0 impossible for ring-3. */
3600	{
3601	Log(("iret/64 %04x:%016RX64/%04x:%016RX64 -> invalid SS selector, #GP(0)\n", uNewCs, uNewRip, uNewSs, uNewRsp));
3602	return iemRaiseGeneralProtectionFault0(pVCpu);
3603	}
3604	/* Make sure SS is sensible, marked as accessed etc. */
3605	iemMemFakeStackSelDesc(&DescSS, (uNewSs & X86_SEL_RPL));
3606	}
3607	else
3608	{
3609	rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSs, X86_XCPT_GP); /** @todo Correct exception? */
3610	if (rcStrict != VINF_SUCCESS)
3611	{
3612	Log(("iret/64 %04x:%016RX64/%04x:%016RX64 - %Rrc when fetching SS\n",
3613	uNewCs, uNewRip, uNewSs, uNewRsp, VBOXSTRICTRC_VAL(rcStrict)));
3614	return rcStrict;
3615	}
3616	}
3617
3618	/* Privilege checks. */
3619	if ((uNewSs & X86_SEL_RPL) != (uNewCs & X86_SEL_RPL))
3620	{
3621	Log(("iret/64 %04x:%016RX64/%04x:%016RX64 -> SS.RPL != CS.RPL -> #GP\n", uNewCs, uNewRip, uNewSs, uNewRsp));
3622	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSs);
3623	}
3624
3625	uint32_t cbLimitSs;
3626	if (!(uNewSs & X86_SEL_MASK_OFF_RPL))
3627	cbLimitSs = UINT32_MAX;
3628	else
3629	{
3630	if (DescSS.Legacy.Gen.u2Dpl != (uNewCs & X86_SEL_RPL))
3631	{
3632	Log(("iret/64 %04x:%016RX64/%04x:%016RX64 -> SS.DPL (%d) != CS.RPL -> #GP\n",
3633	uNewCs, uNewRip, uNewSs, uNewRsp, DescSS.Legacy.Gen.u2Dpl));
3634	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSs);
3635	}
3636
3637	/* Must be a writeable data segment descriptor. */
3638	if (!DescSS.Legacy.Gen.u1DescType)
3639	{
3640	Log(("iret/64 %04x:%016RX64/%04x:%016RX64 -> SS is system segment (%#x) -> #GP\n",
3641	uNewCs, uNewRip, uNewSs, uNewRsp, DescSS.Legacy.Gen.u4Type));
3642	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSs);
3643	}
3644	if ((DescSS.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_WRITE)) != X86_SEL_TYPE_WRITE)
3645	{
3646	Log(("iret/64 %04x:%016RX64/%04x:%016RX64 - not writable data segment (%#x) -> #GP\n",
3647	uNewCs, uNewRip, uNewSs, uNewRsp, DescSS.Legacy.Gen.u4Type));
3648	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSs);
3649	}
3650
3651	/* Present? */
3652	if (!DescSS.Legacy.Gen.u1Present)
3653	{
3654	Log(("iret/64 %04x:%016RX64/%04x:%016RX64 -> SS not present -> #SS\n", uNewCs, uNewRip, uNewSs, uNewRsp));
3655	return iemRaiseStackSelectorNotPresentBySelector(pVCpu, uNewSs);
3656	}
3657	cbLimitSs = X86DESC_LIMIT_G(&DescSS.Legacy);
3658	}
3659
3660	/* Check EIP. */
3661	if (DescCS.Legacy.Gen.u1Long)
3662	{
3663	if (!IEM_IS_CANONICAL(uNewRip))
3664	{
3665	Log(("iret/64 %04x:%016RX64/%04x:%016RX64 -> RIP is not canonical -> #GP(0)\n", uNewCs, uNewRip, uNewSs, uNewRsp));
3666	return iemRaiseNotCanonical(pVCpu);
3667	}
3668	/** @todo check the location of this... Testcase. */
3669	if (RT_LIKELY(!DescCS.Legacy.Gen.u1DefBig))
3670	{ /* likely */ }
3671	else
3672	{
3673	Log(("iret/64 %04x:%016RX64/%04x:%016RX64 -> both L and D are set -> #GP(0)\n", uNewCs, uNewRip, uNewSs, uNewRsp));
3674	return iemRaiseGeneralProtectionFault0(pVCpu);
3675	}
3676	}
3677	else
3678	{
3679	if (uNewRip > cbLimitCS)
3680	{
3681	Log(("iret/64 %04x:%016RX64/%04x:%016RX64 -> EIP is out of bounds (%#x) -> #GP(0)\n",
3682	uNewCs, uNewRip, uNewSs, uNewRsp, cbLimitCS));
3683	/** @todo Which is it, \#GP(0) or \#GP(sel)? */
3684	return iemRaiseSelectorBoundsBySelector(pVCpu, uNewCs);
3685	}
3686	}
3687
3688	/*
3689	* Commit the changes, marking CS and SS accessed first since
3690	* that may fail.
3691	*/
3692	/** @todo where exactly are these actually marked accessed by a real CPU? */
3693	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3694	{
3695	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCs);
3696	if (rcStrict != VINF_SUCCESS)
3697	return rcStrict;
3698	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3699	}
3700	if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3701	{
3702	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewSs);
3703	if (rcStrict != VINF_SUCCESS)
3704	return rcStrict;
3705	DescSS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3706	}
3707
3708	uint32_t fEFlagsMask = X86_EFL_CF \| X86_EFL_PF \| X86_EFL_AF \| X86_EFL_ZF \| X86_EFL_SF
3709	\| X86_EFL_TF \| X86_EFL_DF \| X86_EFL_OF \| X86_EFL_NT;
3710	if (enmEffOpSize != IEMMODE_16BIT)
3711	fEFlagsMask \|= X86_EFL_RF \| X86_EFL_AC \| X86_EFL_ID;
3712	if (IEM_GET_CPL(pVCpu) == 0)
3713	fEFlagsMask \|= X86_EFL_IF \| X86_EFL_IOPL \| X86_EFL_VIF \| X86_EFL_VIP; /* VM is ignored */
3714	else if (IEM_GET_CPL(pVCpu) <= pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL)
3715	fEFlagsMask \|= X86_EFL_IF;
3716	uint32_t fEFlagsNew = IEMMISC_GET_EFL(pVCpu);
3717	fEFlagsNew &= ~fEFlagsMask;
3718	fEFlagsNew \|= uNewFlags & fEFlagsMask;
3719	IEMTLBTRACE_IRET(pVCpu, uNewCs, uNewRip, fEFlagsNew);
3720	#ifdef DBGFTRACE_ENABLED
3721	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "iret/64/%ul%u %08llx -> %04x:%04llx %llx %04x:%04llx",
3722	IEM_GET_CPL(pVCpu), uNewCpl, pVCpu->cpum.GstCtx.rip, uNewCs, uNewRip, uNewFlags, uNewSs, uNewRsp);
3723	#endif
3724
3725	IEMMISC_SET_EFL(pVCpu, fEFlagsNew);
3726	pVCpu->cpum.GstCtx.rip = uNewRip;
3727	pVCpu->cpum.GstCtx.cs.Sel = uNewCs;
3728	pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs;
3729	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
3730	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
3731	pVCpu->cpum.GstCtx.cs.u32Limit = cbLimitCS;
3732	pVCpu->cpum.GstCtx.cs.u64Base = X86DESC_BASE(&DescCS.Legacy);
3733	if (pVCpu->cpum.GstCtx.cs.Attr.n.u1Long \|\| pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig)
3734	pVCpu->cpum.GstCtx.rsp = uNewRsp;
3735	else
3736	pVCpu->cpum.GstCtx.sp = (uint16_t)uNewRsp;
3737	pVCpu->cpum.GstCtx.ss.Sel = uNewSs;
3738	pVCpu->cpum.GstCtx.ss.ValidSel = uNewSs;
3739	if (!(uNewSs & X86_SEL_MASK_OFF_RPL))
3740	{
3741	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
3742	pVCpu->cpum.GstCtx.ss.Attr.u = X86DESCATTR_UNUSABLE \| (uNewCpl << X86DESCATTR_DPL_SHIFT);
3743	pVCpu->cpum.GstCtx.ss.u32Limit = UINT32_MAX;
3744	pVCpu->cpum.GstCtx.ss.u64Base = 0;
3745	Log2(("iret/64 new SS: NULL\n"));
3746	}
3747	else
3748	{
3749	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
3750	pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
3751	pVCpu->cpum.GstCtx.ss.u32Limit = cbLimitSs;
3752	pVCpu->cpum.GstCtx.ss.u64Base = X86DESC_BASE(&DescSS.Legacy);
3753	Log2(("iret/64 new SS: base=%#RX64 lim=%#x attr=%#x\n", pVCpu->cpum.GstCtx.ss.u64Base, pVCpu->cpum.GstCtx.ss.u32Limit, pVCpu->cpum.GstCtx.ss.Attr.u));
3754	}
3755
3756	if (IEM_GET_CPL(pVCpu) != uNewCpl)
3757	{
3758	iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCpl, &pVCpu->cpum.GstCtx.ds);
3759	iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCpl, &pVCpu->cpum.GstCtx.es);
3760	iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCpl, &pVCpu->cpum.GstCtx.fs);
3761	iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCpl, &pVCpu->cpum.GstCtx.gs);
3762	}
3763
3764	iemRecalcExecModeAndCplAndAcFlags(pVCpu);
3765
3766	/* Flush the prefetch buffer. */
3767	IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr); /** @todo may light flush if the ring + mode doesn't change */
3768
3769	/** @todo single stepping */
3770	return VINF_SUCCESS;
3771	}
3772
3773
3774	/**
3775	* Implements iret.
3776	*
3777	* @param enmEffOpSize The effective operand size.
3778	*/
3779	IEM_CIMPL_DEF_1(iemCImpl_iret, IEMMODE, enmEffOpSize)
3780	{
3781	bool fBlockingNmi = CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx);
3782
3783	if (!IEM_IS_IN_GUEST(pVCpu))
3784	{ /* probable */ }
3785	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3786	else if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
3787	{
3788	/*
3789	* Record whether NMI (or virtual-NMI) blocking is in effect during the execution
3790	* of this IRET instruction. We need to provide this information as part of some
3791	* VM-exits.
3792	*
3793	* See Intel spec. 27.2.2 "Information for VM Exits Due to Vectored Events".
3794	*/
3795	if (IEM_VMX_IS_PINCTLS_SET(pVCpu, VMX_PIN_CTLS_VIRT_NMI))
3796	pVCpu->cpum.GstCtx.hwvirt.vmx.fNmiUnblockingIret = pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking;
3797	else
3798	pVCpu->cpum.GstCtx.hwvirt.vmx.fNmiUnblockingIret = fBlockingNmi;
3799
3800	/*
3801	* If "NMI exiting" is set, IRET does not affect blocking of NMIs.
3802	* See Intel Spec. 25.3 "Changes To Instruction Behavior In VMX Non-root Operation".
3803	*/
3804	if (IEM_VMX_IS_PINCTLS_SET(pVCpu, VMX_PIN_CTLS_NMI_EXIT))
3805	fBlockingNmi = false;
3806
3807	/* Clear virtual-NMI blocking, if any, before causing any further exceptions. */
3808	pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking = false;
3809	}
3810	#endif
3811	/*
3812	* The SVM nested-guest intercept for IRET takes priority over all exceptions,
3813	* The NMI is still held pending (which I assume means blocking of further NMIs
3814	* is in effect).
3815	*
3816	* See AMD spec. 15.9 "Instruction Intercepts".
3817	* See AMD spec. 15.21.9 "NMI Support".
3818	*/
3819	else if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IRET))
3820	{
3821	Log(("iret: Guest intercept -> #VMEXIT\n"));
3822	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
3823	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_IRET, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
3824	}
3825
3826	/*
3827	* Clear NMI blocking, if any, before causing any further exceptions.
3828	* See Intel spec. 6.7.1 "Handling Multiple NMIs".
3829	*/
3830	if (fBlockingNmi)
3831	CPUMClearInterruptInhibitingByNmi(&pVCpu->cpum.GstCtx);
3832
3833	/*
3834	* Call a mode specific worker.
3835	*/
3836	VBOXSTRICTRC rcStrict;
3837	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
3838	rcStrict = IEM_CIMPL_CALL_1(iemCImpl_iret_real_v8086, enmEffOpSize);
3839	else
3840	{
3841	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_GDTR \| CPUMCTX_EXTRN_LDTR);
3842	if (IEM_IS_64BIT_CODE(pVCpu))
3843	rcStrict = IEM_CIMPL_CALL_1(iemCImpl_iret_64bit, enmEffOpSize);
3844	else
3845	rcStrict = IEM_CIMPL_CALL_1(iemCImpl_iret_prot, enmEffOpSize);
3846	}
3847
3848	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3849	/*
3850	* Clear NMI unblocking IRET state with the completion of IRET.
3851	*/
3852	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
3853	pVCpu->cpum.GstCtx.hwvirt.vmx.fNmiUnblockingIret = false;
3854	#endif
3855	return rcStrict;
3856	}
3857
3858
3859	static void iemLoadallSetSelector(PVMCPUCC pVCpu, uint8_t iSegReg, uint16_t uSel)
3860	{
3861	PCPUMSELREGHID pHid = iemSRegGetHid(pVCpu, iSegReg);
3862
3863	pHid->Sel = uSel;
3864	pHid->ValidSel = uSel;
3865	pHid->fFlags = CPUMSELREG_FLAGS_VALID;
3866	}
3867
3868
3869	static void iemLoadall286SetDescCache(PVMCPUCC pVCpu, uint8_t iSegReg, uint8_t const *pbMem)
3870	{
3871	PCPUMSELREGHID pHid = iemSRegGetHid(pVCpu, iSegReg);
3872
3873	/* The base is in the first three bytes. */
3874	pHid->u64Base = pbMem[0] + (pbMem[1] << 8) + (pbMem[2] << 16);
3875	/* The attributes are in the fourth byte. */
3876	pHid->Attr.u = pbMem[3];
3877	pHid->Attr.u &= ~(X86DESCATTR_L \| X86DESCATTR_D); /* (just to be on the safe side) */
3878	/* The limit is in the last two bytes. */
3879	pHid->u32Limit = pbMem[4] + (pbMem[5] << 8);
3880	}
3881
3882
3883	/**
3884	* Implements 286 LOADALL (286 CPUs only).
3885	*/
3886	IEM_CIMPL_DEF_0(iemCImpl_loadall286)
3887	{
3888	NOREF(cbInstr);
3889
3890	/* Data is loaded from a buffer at 800h. No checks are done on the
3891	* validity of loaded state.
3892	*
3893	* LOADALL only loads the internal CPU state, it does not access any
3894	* GDT, LDT, or similar tables.
3895	*/
3896
3897	if (IEM_GET_CPL(pVCpu) != 0)
3898	{
3899	Log(("loadall286: CPL must be 0 not %u -> #GP(0)\n", IEM_GET_CPL(pVCpu)));
3900	return iemRaiseGeneralProtectionFault0(pVCpu);
3901	}
3902
3903	uint8_t bUnmapInfo;
3904	uint8_t const *pbMem = NULL;
3905	RTGCPHYS GCPtrStart = 0x800; /* Fixed table location. */
3906	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, (void **)&pbMem, &bUnmapInfo, 0x66, UINT8_MAX, GCPtrStart, IEM_ACCESS_SYS_R, 0);
3907	if (rcStrict != VINF_SUCCESS)
3908	return rcStrict;
3909
3910	/* The MSW is at offset 0x06. */
3911	uint16_t const pau16Mem = (uint16_t const )(pbMem + 0x06);
3912	/* Even LOADALL can't clear the MSW.PE bit, though it can set it. */
3913	uint64_t uNewCr0 = pVCpu->cpum.GstCtx.cr0 & ~(X86_CR0_MP \| X86_CR0_EM \| X86_CR0_TS);
3914	uNewCr0 \|= *pau16Mem & (X86_CR0_PE \| X86_CR0_MP \| X86_CR0_EM \| X86_CR0_TS);
3915	uint64_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
3916
3917	CPUMSetGuestCR0(pVCpu, uNewCr0);
3918	Assert(pVCpu->cpum.GstCtx.cr0 == uNewCr0);
3919
3920	/* Inform PGM if mode changed. */
3921	if ((uNewCr0 & X86_CR0_PE) != (uOldCr0 & X86_CR0_PE))
3922	{
3923	int rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, true /* global */);
3924	AssertRCReturn(rc, rc);
3925	/* ignore informational status codes */
3926	}
3927	rcStrict = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER,
3928	false /* fForce */);
3929
3930	/* TR selector is at offset 0x16. */
3931	pau16Mem = (uint16_t const *)(pbMem + 0x16);
3932	pVCpu->cpum.GstCtx.tr.Sel = pau16Mem[0];
3933	pVCpu->cpum.GstCtx.tr.ValidSel = pau16Mem[0];
3934	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
3935
3936	/* Followed by FLAGS... */
3937	pVCpu->cpum.GstCtx.eflags.u = pau16Mem[1] \| X86_EFL_1;
3938	pVCpu->cpum.GstCtx.ip = pau16Mem[2]; /* ...and IP. */
3939
3940	/* LDT is at offset 0x1C. */
3941	pau16Mem = (uint16_t const *)(pbMem + 0x1C);
3942	pVCpu->cpum.GstCtx.ldtr.Sel = pau16Mem[0];
3943	pVCpu->cpum.GstCtx.ldtr.ValidSel = pau16Mem[0];
3944	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
3945
3946	/* Segment registers are at offset 0x1E. */
3947	pau16Mem = (uint16_t const *)(pbMem + 0x1E);
3948	iemLoadallSetSelector(pVCpu, X86_SREG_DS, pau16Mem[0]);
3949	iemLoadallSetSelector(pVCpu, X86_SREG_SS, pau16Mem[1]);
3950	iemLoadallSetSelector(pVCpu, X86_SREG_CS, pau16Mem[2]);
3951	iemLoadallSetSelector(pVCpu, X86_SREG_ES, pau16Mem[3]);
3952
3953	/* GPRs are at offset 0x26. */
3954	pau16Mem = (uint16_t const *)(pbMem + 0x26);
3955	pVCpu->cpum.GstCtx.di = pau16Mem[0];
3956	pVCpu->cpum.GstCtx.si = pau16Mem[1];
3957	pVCpu->cpum.GstCtx.bp = pau16Mem[2];
3958	pVCpu->cpum.GstCtx.sp = pau16Mem[3];
3959	pVCpu->cpum.GstCtx.bx = pau16Mem[4];
3960	pVCpu->cpum.GstCtx.dx = pau16Mem[5];
3961	pVCpu->cpum.GstCtx.cx = pau16Mem[6];
3962	pVCpu->cpum.GstCtx.ax = pau16Mem[7];
3963
3964	/* Descriptor caches are at offset 0x36, 6 bytes per entry. */
3965	iemLoadall286SetDescCache(pVCpu, X86_SREG_ES, pbMem + 0x36);
3966	iemLoadall286SetDescCache(pVCpu, X86_SREG_CS, pbMem + 0x3C);
3967	iemLoadall286SetDescCache(pVCpu, X86_SREG_SS, pbMem + 0x42);
3968	iemLoadall286SetDescCache(pVCpu, X86_SREG_DS, pbMem + 0x48);
3969
3970	/* GDTR contents are at offset 0x4E, 6 bytes. */
3971	uint8_t const *pau8Mem = pbMem + 0x4E;
3972	/* NB: Fourth byte "should be zero"; we are ignoring it. */
3973	RTGCPHYS GCPtrBase = pau8Mem[0] + ((uint32_t)pau8Mem[1] << 8) + ((uint32_t)pau8Mem[2] << 16);
3974	uint16_t cbLimit = pau8Mem[4] + ((uint32_t)pau8Mem[5] << 8);
3975	CPUMSetGuestGDTR(pVCpu, GCPtrBase, cbLimit);
3976
3977	/* IDTR contents are at offset 0x5A, 6 bytes. */
3978	pau8Mem = pbMem + 0x5A;
3979	GCPtrBase = pau8Mem[0] + ((uint32_t)pau8Mem[1] << 8) + ((uint32_t)pau8Mem[2] << 16);
3980	cbLimit = pau8Mem[4] + ((uint32_t)pau8Mem[5] << 8);
3981	CPUMSetGuestIDTR(pVCpu, GCPtrBase, cbLimit);
3982
3983	Log(("LOADALL: GDTR:%08RX64/%04X, IDTR:%08RX64/%04X\n", pVCpu->cpum.GstCtx.gdtr.pGdt, pVCpu->cpum.GstCtx.gdtr.cbGdt, pVCpu->cpum.GstCtx.idtr.pIdt, pVCpu->cpum.GstCtx.idtr.cbIdt));
3984	Log(("LOADALL: CS:%04X, CS base:%08X, limit:%04X, attrs:%02X\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.cs.u64Base, pVCpu->cpum.GstCtx.cs.u32Limit, pVCpu->cpum.GstCtx.cs.Attr.u));
3985	Log(("LOADALL: DS:%04X, DS base:%08X, limit:%04X, attrs:%02X\n", pVCpu->cpum.GstCtx.ds.Sel, pVCpu->cpum.GstCtx.ds.u64Base, pVCpu->cpum.GstCtx.ds.u32Limit, pVCpu->cpum.GstCtx.ds.Attr.u));
3986	Log(("LOADALL: ES:%04X, ES base:%08X, limit:%04X, attrs:%02X\n", pVCpu->cpum.GstCtx.es.Sel, pVCpu->cpum.GstCtx.es.u64Base, pVCpu->cpum.GstCtx.es.u32Limit, pVCpu->cpum.GstCtx.es.Attr.u));
3987	Log(("LOADALL: SS:%04X, SS base:%08X, limit:%04X, attrs:%02X\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.ss.u64Base, pVCpu->cpum.GstCtx.ss.u32Limit, pVCpu->cpum.GstCtx.ss.Attr.u));
3988	Log(("LOADALL: SI:%04X, DI:%04X, AX:%04X, BX:%04X, CX:%04X, DX:%04X\n", pVCpu->cpum.GstCtx.si, pVCpu->cpum.GstCtx.di, pVCpu->cpum.GstCtx.bx, pVCpu->cpum.GstCtx.bx, pVCpu->cpum.GstCtx.cx, pVCpu->cpum.GstCtx.dx));
3989
3990	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
3991	if (rcStrict != VINF_SUCCESS)
3992	return rcStrict;
3993
3994	/*
3995	* The CPL may change and protected mode may change enabled. It is taken
3996	* from the "DPL fields of the SS and CS descriptor caches" but there is no
3997	* word as to what happens if those are not identical (probably bad things).
3998	*/
3999	iemRecalcExecModeAndCplAndAcFlags(pVCpu);
4000	Assert(IEM_IS_16BIT_CODE(pVCpu));
4001
4002	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS \| CPUM_CHANGED_IDTR \| CPUM_CHANGED_GDTR \| CPUM_CHANGED_TR \| CPUM_CHANGED_LDTR);
4003
4004	/* Flush the prefetch buffer. */
4005	IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr);
4006
4007	/** @todo single stepping */
4008	return rcStrict;
4009	}
4010
4011
4012	/**
4013	* Implements SYSCALL (AMD and Intel64).
4014	*/
4015	IEM_CIMPL_DEF_0(iemCImpl_syscall)
4016	{
4017
4018
4019	/*
4020	* Check preconditions.
4021	*
4022	* Note that CPUs described in the documentation may load a few odd values
4023	* into CS and SS than we allow here. This has yet to be checked on real
4024	* hardware.
4025	*/
4026	if (!(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_SCE))
4027	{
4028	Log(("syscall: Not enabled in EFER -> #UD\n"));
4029	return iemRaiseUndefinedOpcode(pVCpu);
4030	}
4031	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
4032	{
4033	Log(("syscall: Protected mode is required -> #GP(0)\n"));
4034	return iemRaiseGeneralProtectionFault0(pVCpu);
4035	}
4036	if ( IEM_IS_GUEST_CPU_INTEL(pVCpu)
4037	&& !IEM_IS_64BIT_CODE(pVCpu)) //&& !CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)))
4038	{
4039	Log(("syscall: Only available in 64-bit mode on intel -> #UD\n"));
4040	return iemRaiseUndefinedOpcode(pVCpu);
4041	}
4042
4043	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS);
4044
4045	/** @todo verify RPL ignoring and CS=0xfff8 (i.e. SS == 0). */
4046	/** @todo what about LDT selectors? Shouldn't matter, really. */
4047	uint16_t uNewCs = (pVCpu->cpum.GstCtx.msrSTAR >> MSR_K6_STAR_SYSCALL_CS_SS_SHIFT) & X86_SEL_MASK_OFF_RPL;
4048	uint16_t uNewSs = uNewCs + 8;
4049	if (uNewCs == 0 \|\| uNewSs == 0)
4050	{
4051	/** @todo Neither Intel nor AMD document this check. */
4052	Log(("syscall: msrSTAR.CS = 0 or SS = 0 -> #GP(0)\n"));
4053	return iemRaiseGeneralProtectionFault0(pVCpu);
4054	}
4055
4056	/*
4057	* Hack alert! Convert incoming debug events to slient on Intel.
4058	* See the dbg+inhibit+ringxfer test in bs3-cpu-weird-1.
4059	*/
4060	if ( !(pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK_NONSILENT)
4061	\|\| !IEM_IS_GUEST_CPU_INTEL(pVCpu))
4062	{ /* ignore */ }
4063	else
4064	{
4065	Log(("iemCImpl_syscall: Converting pending %#x debug events to a silent one (intel hack)\n",
4066	pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK));
4067	pVCpu->cpum.GstCtx.eflags.uBoth = (pVCpu->cpum.GstCtx.eflags.uBoth & ~CPUMCTX_DBG_HIT_DRX_MASK)
4068	\| CPUMCTX_DBG_HIT_DRX_SILENT;
4069	}
4070
4071	/*
4072	* Long mode and legacy mode differs.
4073	*/
4074	if (CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)))
4075	{
4076	uint64_t uNewRip = IEM_IS_64BIT_CODE(pVCpu) ? pVCpu->cpum.GstCtx.msrLSTAR : pVCpu->cpum.GstCtx. msrCSTAR;
4077
4078	/* This test isn't in the docs, but I'm not trusting the guys writing
4079	the MSRs to have validated the values as canonical like they should. */
4080	if (!IEM_IS_CANONICAL(uNewRip))
4081	{
4082	/** @todo Intel claims this can't happen because IA32_LSTAR MSR can't be written with non-canonical address. */
4083	Log(("syscall: New RIP not canonical -> #UD\n"));
4084	return iemRaiseUndefinedOpcode(pVCpu);
4085	}
4086
4087	/*
4088	* Commit it.
4089	*/
4090	Log(("syscall: %04x:%016RX64 [efl=%#llx] -> %04x:%016RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u, uNewCs, uNewRip));
4091	pVCpu->cpum.GstCtx.rcx = pVCpu->cpum.GstCtx.rip + cbInstr;
4092	pVCpu->cpum.GstCtx.rip = uNewRip;
4093
4094	pVCpu->cpum.GstCtx.rflags.u &= ~X86_EFL_RF;
4095	pVCpu->cpum.GstCtx.r11 = pVCpu->cpum.GstCtx.rflags.u;
4096	pVCpu->cpum.GstCtx.rflags.u &= ~pVCpu->cpum.GstCtx.msrSFMASK;
4097	pVCpu->cpum.GstCtx.rflags.u \|= X86_EFL_RA1_MASK;
4098
4099	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_P \| X86DESCATTR_G \| X86DESCATTR_L \| X86DESCATTR_DT \| X86_SEL_TYPE_ER_ACC;
4100	pVCpu->cpum.GstCtx.ss.Attr.u = X86DESCATTR_P \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_DT \| X86_SEL_TYPE_RW_ACC;
4101
4102	pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~(IEM_F_MODE_MASK \| IEM_F_X86_CPL_MASK \| IEM_F_X86_AC))
4103	\| IEM_F_MODE_X86_64BIT;
4104	}
4105	else
4106	{
4107	/*
4108	* Commit it.
4109	*/
4110	Log(("syscall: %04x:%08RX32 [efl=%#x] -> %04x:%08RX32\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.eflags.u, uNewCs, (uint32_t)(pVCpu->cpum.GstCtx.msrSTAR & MSR_K6_STAR_SYSCALL_EIP_MASK)));
4111	pVCpu->cpum.GstCtx.rcx = pVCpu->cpum.GstCtx.eip + cbInstr;
4112	pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.msrSTAR & MSR_K6_STAR_SYSCALL_EIP_MASK;
4113	pVCpu->cpum.GstCtx.rflags.u &= ~(X86_EFL_VM \| X86_EFL_IF \| X86_EFL_RF);
4114
4115	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_P \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_DT \| X86_SEL_TYPE_ER_ACC;
4116	pVCpu->cpum.GstCtx.ss.Attr.u = X86DESCATTR_P \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_DT \| X86_SEL_TYPE_RW_ACC;
4117
4118	pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~(IEM_F_MODE_MASK \| IEM_F_X86_CPL_MASK \| IEM_F_X86_AC))
4119	\| IEM_F_MODE_X86_32BIT_PROT
4120	\| iemCalc32BitFlatIndicatorEsDs(pVCpu);
4121	}
4122	pVCpu->cpum.GstCtx.cs.Sel = uNewCs;
4123	pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs;
4124	pVCpu->cpum.GstCtx.cs.u64Base = 0;
4125	pVCpu->cpum.GstCtx.cs.u32Limit = UINT32_MAX;
4126	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
4127
4128	pVCpu->cpum.GstCtx.ss.Sel = uNewSs;
4129	pVCpu->cpum.GstCtx.ss.ValidSel = uNewSs;
4130	pVCpu->cpum.GstCtx.ss.u64Base = 0;
4131	pVCpu->cpum.GstCtx.ss.u32Limit = UINT32_MAX;
4132	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
4133
4134	/* Flush the prefetch buffer. */
4135	IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr);
4136
4137	/*
4138	* Handle debug events.
4139	* If TF isn't masked, we're supposed to raise a single step #DB.
4140	*/
4141	return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);
4142	}
4143
4144
4145	/**
4146	* Implements SYSRET (AMD and Intel64).
4147	*
4148	* @param enmEffOpSize The effective operand size.
4149	*/
4150	IEM_CIMPL_DEF_1(iemCImpl_sysret, IEMMODE, enmEffOpSize)
4151
4152	{
4153	RT_NOREF_PV(cbInstr);
4154
4155	/*
4156	* Check preconditions.
4157	*
4158	* Note that CPUs described in the documentation may load a few odd values
4159	* into CS and SS than we allow here. This has yet to be checked on real
4160	* hardware.
4161	*/
4162	if (!(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_SCE))
4163	{
4164	Log(("sysret: Not enabled in EFER -> #UD\n"));
4165	return iemRaiseUndefinedOpcode(pVCpu);
4166	}
4167	if (IEM_IS_GUEST_CPU_INTEL(pVCpu) && !CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)))
4168	{
4169	Log(("sysret: Only available in long mode on intel -> #UD\n"));
4170	return iemRaiseUndefinedOpcode(pVCpu);
4171	}
4172	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
4173	{
4174	Log(("sysret: Protected mode is required -> #GP(0)\n"));
4175	return iemRaiseGeneralProtectionFault0(pVCpu);
4176	}
4177	if (IEM_GET_CPL(pVCpu) != 0)
4178	{
4179	Log(("sysret: CPL must be 0 not %u -> #GP(0)\n", IEM_GET_CPL(pVCpu)));
4180	return iemRaiseGeneralProtectionFault0(pVCpu);
4181	}
4182
4183	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS);
4184
4185	/** @todo Does SYSRET verify CS != 0 and SS != 0? Neither is valid in ring-3. */
4186	uint16_t uNewCs = (pVCpu->cpum.GstCtx.msrSTAR >> MSR_K6_STAR_SYSRET_CS_SS_SHIFT) & X86_SEL_MASK_OFF_RPL;
4187	uint16_t uNewSs = uNewCs + 8;
4188	if (enmEffOpSize == IEMMODE_64BIT)
4189	uNewCs += 16;
4190	if (uNewCs == 0 \|\| uNewSs == 0)
4191	{
4192	Log(("sysret: msrSTAR.CS = 0 or SS = 0 -> #GP(0)\n"));
4193	return iemRaiseGeneralProtectionFault0(pVCpu);
4194	}
4195
4196	/*
4197	* Commit it.
4198	*/
4199	bool f32Bit = true;
4200	if (CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)))
4201	{
4202	if (enmEffOpSize == IEMMODE_64BIT)
4203	{
4204	Log(("sysret: %04x:%016RX64 [efl=%#llx] -> %04x:%016RX64 [r11=%#llx]\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u, uNewCs, pVCpu->cpum.GstCtx.rcx, pVCpu->cpum.GstCtx.r11));
4205	/* Note! We disregard intel manual regarding the RCX canonical
4206	check, ask intel+xen why AMD doesn't do it. */
4207	pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.rcx;
4208	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_P \| X86DESCATTR_G \| X86DESCATTR_L \| X86DESCATTR_DT \| X86_SEL_TYPE_ER_ACC
4209	\| (3 << X86DESCATTR_DPL_SHIFT);
4210	f32Bit = false;
4211	}
4212	else
4213	{
4214	Log(("sysret: %04x:%016RX64 [efl=%#llx] -> %04x:%08RX32 [r11=%#llx]\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u, uNewCs, pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.r11));
4215	pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.ecx;
4216	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_P \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_DT \| X86_SEL_TYPE_ER_ACC
4217	\| (3 << X86DESCATTR_DPL_SHIFT);
4218	}
4219	/** @todo testcase: See what kind of flags we can make SYSRET restore and
4220	* what it really ignores. RF and VM are hinted at being zero, by AMD.
4221	* Intel says: RFLAGS := (R11 & 3C7FD7H) \| 2; */
4222	pVCpu->cpum.GstCtx.rflags.u = pVCpu->cpum.GstCtx.r11 & (X86_EFL_POPF_BITS \| X86_EFL_VIF \| X86_EFL_VIP);
4223	pVCpu->cpum.GstCtx.rflags.u \|= X86_EFL_RA1_MASK;
4224	}
4225	else
4226	{
4227	Log(("sysret: %04x:%08RX32 [efl=%#x] -> %04x:%08RX32\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.eflags.u, uNewCs, pVCpu->cpum.GstCtx.ecx));
4228	pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.rcx;
4229	pVCpu->cpum.GstCtx.rflags.u \|= X86_EFL_IF;
4230	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_P \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_DT \| X86_SEL_TYPE_ER_ACC
4231	\| (3 << X86DESCATTR_DPL_SHIFT);
4232	}
4233	pVCpu->cpum.GstCtx.cs.Sel = uNewCs \| 3;
4234	pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs \| 3;
4235	pVCpu->cpum.GstCtx.cs.u64Base = 0;
4236	pVCpu->cpum.GstCtx.cs.u32Limit = UINT32_MAX;
4237	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
4238
4239	/* The SS hidden bits remains unchanged says AMD, we presume they set DPL to 3.
4240	Intel (and presuably VIA) OTOH sets loads valid ring-3 values it seems, see
4241	X86_BUG_SYSRET_SS_ATTRS in linux 5.3. */
4242	if (IEM_IS_GUEST_CPU_AMD(pVCpu))
4243	{
4244	Log(("sysret: ss:rsp=%04x:%08RX64 attr=%x -> %04x:%08RX64 attr=%#x\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.ss.Attr.u, uNewSs \| 3, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.ss.Attr.u \| (3 << X86DESCATTR_DPL_SHIFT) ));
4245	pVCpu->cpum.GstCtx.ss.Attr.u \|= (3 << X86DESCATTR_DPL_SHIFT);
4246	}
4247	else
4248	{
4249	Log(("sysret: ss:rsp=%04x:%08RX64 attr=%x -> %04x:%08RX64 attr=%#x\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.ss.Attr.u, uNewSs \| 3, pVCpu->cpum.GstCtx.rsp, X86DESCATTR_P \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_DT \| X86_SEL_TYPE_RW_ACC \| (3 << X86DESCATTR_DPL_SHIFT) ));
4250	pVCpu->cpum.GstCtx.ss.Attr.u = X86DESCATTR_P \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_DT \| X86_SEL_TYPE_RW_ACC
4251	\| (3 << X86DESCATTR_DPL_SHIFT);
4252	pVCpu->cpum.GstCtx.ss.u64Base = 0;
4253	pVCpu->cpum.GstCtx.ss.u32Limit = UINT32_MAX;
4254	}
4255	pVCpu->cpum.GstCtx.ss.Sel = uNewSs \| 3;
4256	pVCpu->cpum.GstCtx.ss.ValidSel = uNewSs \| 3;
4257	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
4258	/** @todo Testcase: verify that SS.u1Long and SS.u1DefBig are left unchanged
4259	* on sysret on AMD and not on intel. */
4260
4261	if (!f32Bit)
4262	pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~(IEM_F_MODE_MASK \| IEM_F_X86_CPL_MASK \| IEM_F_X86_AC))
4263	\| (3 << IEM_F_X86_CPL_SHIFT)
4264	\| IEM_F_MODE_X86_64BIT
4265	\| iemCalcExecAcFlag(pVCpu);
4266	else
4267	pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~(IEM_F_MODE_MASK \| IEM_F_X86_CPL_MASK \| IEM_F_X86_AC))
4268	\| (3 << IEM_F_X86_CPL_SHIFT)
4269	\| IEM_F_MODE_X86_32BIT_PROT
4270	/** @todo sort out the SS.BASE/LIM/ATTR claim by AMD and maybe we can switch to
4271	* iemCalc32BitFlatIndicatorDsEs and move this up into the above branch. */
4272	\| iemCalc32BitFlatIndicator(pVCpu)
4273	\| iemCalcExecAcFlag(pVCpu);
4274
4275	/* Flush the prefetch buffer. */
4276	IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr);
4277
4278	/** @todo single step */
4279	return VINF_SUCCESS;
4280	}
4281
4282
4283	/**
4284	* Implements SYSENTER (Intel, 32-bit AMD).
4285	*/
4286	IEM_CIMPL_DEF_0(iemCImpl_sysenter)
4287	{
4288	RT_NOREF(cbInstr);
4289
4290	/*
4291	* Check preconditions.
4292	*
4293	* Note that CPUs described in the documentation may load a few odd values
4294	* into CS and SS than we allow here. This has yet to be checked on real
4295	* hardware.
4296	*/
4297	if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSysEnter)
4298	{
4299	Log(("sysenter: not supported -=> #UD\n"));
4300	return iemRaiseUndefinedOpcode(pVCpu);
4301	}
4302	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
4303	{
4304	Log(("sysenter: Protected or long mode is required -> #GP(0)\n"));
4305	return iemRaiseGeneralProtectionFault0(pVCpu);
4306	}
4307	bool fIsLongMode = CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu));
4308	if (IEM_IS_GUEST_CPU_AMD(pVCpu) && fIsLongMode)
4309	{
4310	Log(("sysenter: Only available in protected mode on AMD -> #UD\n"));
4311	return iemRaiseUndefinedOpcode(pVCpu);
4312	}
4313	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4314	uint16_t uNewCs = pVCpu->cpum.GstCtx.SysEnter.cs;
4315	if ((uNewCs & X86_SEL_MASK_OFF_RPL) == 0)
4316	{
4317	Log(("sysenter: SYSENTER_CS = %#x -> #GP(0)\n", uNewCs));
4318	return iemRaiseGeneralProtectionFault0(pVCpu);
4319	}
4320
4321	/* This test isn't in the docs, it's just a safeguard against missing
4322	canonical checks when writing the registers. */
4323	if (RT_LIKELY( !fIsLongMode
4324	\|\| ( IEM_IS_CANONICAL(pVCpu->cpum.GstCtx.SysEnter.eip)
4325	&& IEM_IS_CANONICAL(pVCpu->cpum.GstCtx.SysEnter.esp))))
4326	{ /* likely */ }
4327	else
4328	{
4329	Log(("sysenter: SYSENTER_EIP = %#RX64 or/and SYSENTER_ESP = %#RX64 not canonical -> #GP(0)\n",
4330	pVCpu->cpum.GstCtx.SysEnter.eip, pVCpu->cpum.GstCtx.SysEnter.esp));
4331	return iemRaiseUndefinedOpcode(pVCpu);
4332	}
4333
4334	/** @todo Test: Sysenter from ring-0, ring-1 and ring-2. */
4335
4336	/*
4337	* Update registers and commit.
4338	*/
4339	if (fIsLongMode)
4340	{
4341	Log(("sysenter: %04x:%016RX64 [efl=%#llx] -> %04x:%016RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
4342	pVCpu->cpum.GstCtx.rflags.u, uNewCs & X86_SEL_MASK_OFF_RPL, pVCpu->cpum.GstCtx.SysEnter.eip));
4343	pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.SysEnter.eip;
4344	pVCpu->cpum.GstCtx.rsp = pVCpu->cpum.GstCtx.SysEnter.esp;
4345	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_L \| X86DESCATTR_G \| X86DESCATTR_P \| X86DESCATTR_DT
4346	\| X86DESCATTR_LIMIT_HIGH \| X86_SEL_TYPE_ER_ACC;
4347	pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~(IEM_F_MODE_MASK \| IEM_F_X86_CPL_MASK \| IEM_F_X86_AC))
4348	\| IEM_F_MODE_X86_64BIT;
4349	}
4350	else
4351	{
4352	Log(("sysenter: %04x:%08RX32 [efl=%#llx] -> %04x:%08RX32\n", pVCpu->cpum.GstCtx.cs.Sel, (uint32_t)pVCpu->cpum.GstCtx.rip,
4353	pVCpu->cpum.GstCtx.rflags.u, uNewCs & X86_SEL_MASK_OFF_RPL, (uint32_t)pVCpu->cpum.GstCtx.SysEnter.eip));
4354	pVCpu->cpum.GstCtx.rip = (uint32_t)pVCpu->cpum.GstCtx.SysEnter.eip;
4355	pVCpu->cpum.GstCtx.rsp = (uint32_t)pVCpu->cpum.GstCtx.SysEnter.esp;
4356	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_D \| X86DESCATTR_G \| X86DESCATTR_P \| X86DESCATTR_DT
4357	\| X86DESCATTR_LIMIT_HIGH \| X86_SEL_TYPE_ER_ACC;
4358	pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~(IEM_F_MODE_MASK \| IEM_F_X86_CPL_MASK \| IEM_F_X86_AC))
4359	\| IEM_F_MODE_X86_32BIT_PROT
4360	\| iemCalc32BitFlatIndicatorEsDs(pVCpu);
4361	}
4362	pVCpu->cpum.GstCtx.cs.Sel = uNewCs & X86_SEL_MASK_OFF_RPL;
4363	pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs & X86_SEL_MASK_OFF_RPL;
4364	pVCpu->cpum.GstCtx.cs.u64Base = 0;
4365	pVCpu->cpum.GstCtx.cs.u32Limit = UINT32_MAX;
4366	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
4367
4368	pVCpu->cpum.GstCtx.ss.Sel = (uNewCs & X86_SEL_MASK_OFF_RPL) + 8;
4369	pVCpu->cpum.GstCtx.ss.ValidSel = (uNewCs & X86_SEL_MASK_OFF_RPL) + 8;
4370	pVCpu->cpum.GstCtx.ss.u64Base = 0;
4371	pVCpu->cpum.GstCtx.ss.u32Limit = UINT32_MAX;
4372	pVCpu->cpum.GstCtx.ss.Attr.u = X86DESCATTR_D \| X86DESCATTR_G \| X86DESCATTR_P \| X86DESCATTR_DT
4373	\| X86DESCATTR_LIMIT_HIGH \| X86_SEL_TYPE_RW_ACC;
4374	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
4375
4376	pVCpu->cpum.GstCtx.rflags.Bits.u1IF = 0;
4377	pVCpu->cpum.GstCtx.rflags.Bits.u1VM = 0;
4378	pVCpu->cpum.GstCtx.rflags.Bits.u1RF = 0;
4379
4380	/* Flush the prefetch buffer. */
4381	IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr);
4382
4383	/** @todo single stepping */
4384	return VINF_SUCCESS;
4385	}
4386
4387
4388	/**
4389	* Implements SYSEXIT (Intel, 32-bit AMD).
4390	*
4391	* @param enmEffOpSize The effective operand size.
4392	*/
4393	IEM_CIMPL_DEF_1(iemCImpl_sysexit, IEMMODE, enmEffOpSize)
4394	{
4395	RT_NOREF(cbInstr);
4396
4397	/*
4398	* Check preconditions.
4399	*
4400	* Note that CPUs described in the documentation may load a few odd values
4401	* into CS and SS than we allow here. This has yet to be checked on real
4402	* hardware.
4403	*/
4404	if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSysEnter)
4405	{
4406	Log(("sysexit: not supported -=> #UD\n"));
4407	return iemRaiseUndefinedOpcode(pVCpu);
4408	}
4409	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
4410	{
4411	Log(("sysexit: Protected or long mode is required -> #GP(0)\n"));
4412	return iemRaiseGeneralProtectionFault0(pVCpu);
4413	}
4414	bool fIsLongMode = CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu));
4415	if (IEM_IS_GUEST_CPU_AMD(pVCpu) && fIsLongMode)
4416	{
4417	Log(("sysexit: Only available in protected mode on AMD -> #UD\n"));
4418	return iemRaiseUndefinedOpcode(pVCpu);
4419	}
4420	if (IEM_GET_CPL(pVCpu) != 0)
4421	{
4422	Log(("sysexit: CPL(=%u) != 0 -> #GP(0)\n", IEM_GET_CPL(pVCpu)));
4423	return iemRaiseGeneralProtectionFault0(pVCpu);
4424	}
4425	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4426	uint16_t uNewCs = pVCpu->cpum.GstCtx.SysEnter.cs;
4427	if ((uNewCs & X86_SEL_MASK_OFF_RPL) == 0)
4428	{
4429	Log(("sysexit: SYSENTER_CS = %#x -> #GP(0)\n", uNewCs));
4430	return iemRaiseGeneralProtectionFault0(pVCpu);
4431	}
4432
4433	/*
4434	* Update registers and commit.
4435	*/
4436	if (enmEffOpSize == IEMMODE_64BIT)
4437	{
4438	Log(("sysexit: %04x:%016RX64 [efl=%#llx] -> %04x:%016RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
4439	pVCpu->cpum.GstCtx.rflags.u, (uNewCs \| 3) + 32, pVCpu->cpum.GstCtx.rcx));
4440	pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.rdx;
4441	pVCpu->cpum.GstCtx.rsp = pVCpu->cpum.GstCtx.rcx;
4442	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_L \| X86DESCATTR_G \| X86DESCATTR_P \| X86DESCATTR_DT
4443	\| X86DESCATTR_LIMIT_HIGH \| X86_SEL_TYPE_ER_ACC \| (3 << X86DESCATTR_DPL_SHIFT);
4444	pVCpu->cpum.GstCtx.cs.Sel = (uNewCs \| 3) + 32;
4445	pVCpu->cpum.GstCtx.cs.ValidSel = (uNewCs \| 3) + 32;
4446	pVCpu->cpum.GstCtx.ss.Sel = (uNewCs \| 3) + 40;
4447	pVCpu->cpum.GstCtx.ss.ValidSel = (uNewCs \| 3) + 40;
4448
4449	pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~(IEM_F_MODE_MASK \| IEM_F_X86_CPL_MASK))
4450	\| (3 << IEM_F_X86_CPL_SHIFT)
4451	\| IEM_F_MODE_X86_64BIT
4452	\| iemCalcExecAcFlag(pVCpu);
4453	}
4454	else
4455	{
4456	Log(("sysexit: %04x:%08RX64 [efl=%#llx] -> %04x:%08RX32\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
4457	pVCpu->cpum.GstCtx.rflags.u, (uNewCs \| 3) + 16, (uint32_t)pVCpu->cpum.GstCtx.edx));
4458	pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.edx;
4459	pVCpu->cpum.GstCtx.rsp = pVCpu->cpum.GstCtx.ecx;
4460	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_D \| X86DESCATTR_G \| X86DESCATTR_P \| X86DESCATTR_DT
4461	\| X86DESCATTR_LIMIT_HIGH \| X86_SEL_TYPE_ER_ACC \| (3 << X86DESCATTR_DPL_SHIFT);
4462	pVCpu->cpum.GstCtx.cs.Sel = (uNewCs \| 3) + 16;
4463	pVCpu->cpum.GstCtx.cs.ValidSel = (uNewCs \| 3) + 16;
4464	pVCpu->cpum.GstCtx.ss.Sel = (uNewCs \| 3) + 24;
4465	pVCpu->cpum.GstCtx.ss.ValidSel = (uNewCs \| 3) + 24;
4466
4467	pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~(IEM_F_MODE_MASK \| IEM_F_X86_CPL_MASK))
4468	\| (3 << IEM_F_X86_CPL_SHIFT)
4469	\| IEM_F_MODE_X86_32BIT_PROT
4470	\| iemCalc32BitFlatIndicatorEsDs(pVCpu)
4471	\| iemCalcExecAcFlag(pVCpu);
4472	}
4473	pVCpu->cpum.GstCtx.cs.u64Base = 0;
4474	pVCpu->cpum.GstCtx.cs.u32Limit = UINT32_MAX;
4475	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
4476
4477	pVCpu->cpum.GstCtx.ss.u64Base = 0;
4478	pVCpu->cpum.GstCtx.ss.u32Limit = UINT32_MAX;
4479	pVCpu->cpum.GstCtx.ss.Attr.u = X86DESCATTR_D \| X86DESCATTR_G \| X86DESCATTR_P \| X86DESCATTR_DT
4480	\| X86DESCATTR_LIMIT_HIGH \| X86_SEL_TYPE_RW_ACC \| (3 << X86DESCATTR_DPL_SHIFT);
4481	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
4482	pVCpu->cpum.GstCtx.rflags.Bits.u1RF = 0;
4483
4484	/** @todo single stepping */
4485
4486	/* Flush the prefetch buffer. */
4487	IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr);
4488
4489	return VINF_SUCCESS;
4490	}
4491
4492
4493	/**
4494	* Completes a MOV SReg,XXX or POP SReg instruction.
4495	*
4496	* When not modifying SS or when we're already in an interrupt shadow we
4497	* can update RIP and finish the instruction the normal way.
4498	*
4499	* Otherwise, the MOV/POP SS interrupt shadow that we now enable will block
4500	* both TF and DBx events. The TF will be ignored while the DBx ones will
4501	* be delayed till the next instruction boundrary. For more details see
4502	* @sdmv3{077,200,6.8.3,Masking Exceptions and Interrupts When Switching Stacks}.
4503	*/
4504	DECLINLINE(VBOXSTRICTRC) iemCImpl_LoadSRegFinish(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iSegReg)
4505	{
4506	if (iSegReg != X86_SREG_SS \|\| CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx))
4507	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
4508
4509	iemRegAddToRip(pVCpu, cbInstr);
4510	pVCpu->cpum.GstCtx.eflags.uBoth &= ~X86_EFL_RF; /* Shadow int isn't set and DRx is delayed, so only clear RF. */
4511	CPUMSetInInterruptShadowSs(&pVCpu->cpum.GstCtx);
4512
4513	return VINF_SUCCESS;
4514	}
4515
4516
4517	/**
4518	* Common worker for 'pop SReg', 'mov SReg, GReg' and 'lXs GReg, reg/mem'.
4519	*
4520	* @param pVCpu The cross context virtual CPU structure of the calling
4521	* thread.
4522	* @param iSegReg The segment register number (valid).
4523	* @param uSel The new selector value.
4524	*/
4525	static VBOXSTRICTRC iemCImpl_LoadSRegWorker(PVMCPUCC pVCpu, uint8_t iSegReg, uint16_t uSel)
4526	{
4527	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
4528	uint16_t *pSel = iemSRegRef(pVCpu, iSegReg);
4529	PCPUMSELREGHID pHid = iemSRegGetHid(pVCpu, iSegReg);
4530
4531	Assert(iSegReg <= X86_SREG_GS && iSegReg != X86_SREG_CS);
4532
4533	/*
4534	* Real mode and V8086 mode are easy.
4535	*/
4536	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
4537	{
4538	*pSel = uSel;
4539	pHid->u64Base = (uint32_t)uSel << 4;
4540	pHid->ValidSel = uSel;
4541	pHid->fFlags = CPUMSELREG_FLAGS_VALID;
4542	#if 0 /* AMD Volume 2, chapter 4.1 - "real mode segmentation" - states that limit and attributes are untouched. */
4543	/** @todo Does the CPU actually load limits and attributes in the
4544	* real/V8086 mode segment load case? It doesn't for CS in far
4545	* jumps... Affects unreal mode. */
4546	pHid->u32Limit = 0xffff;
4547	pHid->Attr.u = 0;
4548	pHid->Attr.n.u1Present = 1;
4549	pHid->Attr.n.u1DescType = 1;
4550	pHid->Attr.n.u4Type = iSegReg != X86_SREG_CS
4551	? X86_SEL_TYPE_RW
4552	: X86_SEL_TYPE_READ \| X86_SEL_TYPE_CODE;
4553	#endif
4554
4555	/* Update the FLAT 32-bit mode flag, if we're in 32-bit unreal mode (unlikely): */
4556	if (RT_LIKELY(!IEM_IS_32BIT_CODE(pVCpu)))
4557	{ /* likely */ }
4558	else if (uSel != 0)
4559	pVCpu->iem.s.fExec &= ~IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK;
4560	else
4561	pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK)
4562	\| iemCalc32BitFlatIndicator(pVCpu);
4563	}
4564	/*
4565	* Protected / long mode - null segment.
4566	*
4567	* Check if it's a null segment selector value first, that's OK for DS, ES,
4568	* FS and GS. If not null, then we have to load and parse the descriptor.
4569	*/
4570	else if (!(uSel & X86_SEL_MASK_OFF_RPL))
4571	{
4572	Assert(iSegReg != X86_SREG_CS); /** @todo testcase for \#UD on MOV CS, ax! */
4573	if (iSegReg == X86_SREG_SS)
4574	{
4575	/* In 64-bit kernel mode, the stack can be 0 because of the way
4576	interrupts are dispatched. AMD seems to have a slighly more
4577	relaxed relationship to SS.RPL than intel does. */
4578	/** @todo We cannot 'mov ss, 3' in 64-bit kernel mode, can we? There is a testcase (bs-cpu-xcpt-1), but double check this! */
4579	if ( !IEM_IS_64BIT_CODE(pVCpu)
4580	\|\| IEM_GET_CPL(pVCpu) > 2
4581	\|\| ( uSel != IEM_GET_CPL(pVCpu)
4582	&& !IEM_IS_GUEST_CPU_AMD(pVCpu)) )
4583	{
4584	Log(("load sreg %#x -> invalid stack selector, #GP(0)\n", uSel));
4585	return iemRaiseGeneralProtectionFault0(pVCpu);
4586	}
4587	}
4588
4589	pSel = uSel; / Not RPL, remember :-) */
4590	iemHlpLoadNullDataSelectorProt(pVCpu, pHid, uSel);
4591	if (iSegReg == X86_SREG_SS)
4592	pHid->Attr.u \|= IEM_GET_CPL(pVCpu) << X86DESCATTR_DPL_SHIFT;
4593
4594	/* This will affect the FLAT 32-bit mode flag: */
4595	if ( iSegReg < X86_SREG_FS
4596	&& IEM_IS_32BIT_CODE(pVCpu))
4597	pVCpu->iem.s.fExec &= ~IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK;
4598	}
4599	/*
4600	* Protected / long mode.
4601	*/
4602	else
4603	{
4604	/* Fetch the descriptor. */
4605	IEMSELDESC Desc;
4606	VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uSel, X86_XCPT_GP); /** @todo Correct exception? */
4607	if (rcStrict != VINF_SUCCESS)
4608	return rcStrict;
4609
4610	/* Check GPs first. */
4611	if (!Desc.Legacy.Gen.u1DescType)
4612	{
4613	Log(("load sreg %d (=%#x) - system selector (%#x) -> #GP\n", iSegReg, uSel, Desc.Legacy.Gen.u4Type));
4614	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
4615	}
4616	if (iSegReg == X86_SREG_SS) /* SS gets different treatment */
4617	{
4618	if ( (Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
4619	\|\| !(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE) )
4620	{
4621	Log(("load sreg SS, %#x - code or read only (%#x) -> #GP\n", uSel, Desc.Legacy.Gen.u4Type));
4622	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
4623	}
4624	if ((uSel & X86_SEL_RPL) != IEM_GET_CPL(pVCpu))
4625	{
4626	Log(("load sreg SS, %#x - RPL and CPL (%d) differs -> #GP\n", uSel, IEM_GET_CPL(pVCpu)));
4627	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
4628	}
4629	if (Desc.Legacy.Gen.u2Dpl != IEM_GET_CPL(pVCpu))
4630	{
4631	Log(("load sreg SS, %#x - DPL (%d) and CPL (%d) differs -> #GP\n", uSel, Desc.Legacy.Gen.u2Dpl, IEM_GET_CPL(pVCpu)));
4632	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
4633	}
4634	}
4635	else
4636	{
4637	if ((Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
4638	{
4639	Log(("load sreg%u, %#x - execute only segment -> #GP\n", iSegReg, uSel));
4640	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
4641	}
4642	if ( (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_CONF))
4643	!= (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_CONF))
4644	{
4645	#if 0 /* this is what intel says. */
4646	if ( (uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl
4647	&& IEM_GET_CPL(pVCpu) > Desc.Legacy.Gen.u2Dpl)
4648	{
4649	Log(("load sreg%u, %#x - both RPL (%d) and CPL (%d) are greater than DPL (%d) -> #GP\n",
4650	iSegReg, uSel, (uSel & X86_SEL_RPL), IEM_GET_CPL(pVCpu), Desc.Legacy.Gen.u2Dpl));
4651	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
4652	}
4653	#else /* this is what makes more sense. */
4654	if ((unsigned)(uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl)
4655	{
4656	Log(("load sreg%u, %#x - RPL (%d) is greater than DPL (%d) -> #GP\n",
4657	iSegReg, uSel, (uSel & X86_SEL_RPL), Desc.Legacy.Gen.u2Dpl));
4658	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
4659	}
4660	if (IEM_GET_CPL(pVCpu) > Desc.Legacy.Gen.u2Dpl)
4661	{
4662	Log(("load sreg%u, %#x - CPL (%d) is greater than DPL (%d) -> #GP\n",
4663	iSegReg, uSel, IEM_GET_CPL(pVCpu), Desc.Legacy.Gen.u2Dpl));
4664	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
4665	}
4666	#endif
4667	}
4668	}
4669
4670	/* Is it there? */
4671	if (!Desc.Legacy.Gen.u1Present)
4672	{
4673	Log(("load sreg%d,%#x - segment not present -> #NP\n", iSegReg, uSel));
4674	return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel);
4675	}
4676
4677	/* The base and limit. */
4678	uint32_t cbLimit = X86DESC_LIMIT_G(&Desc.Legacy);
4679	uint64_t u64Base = X86DESC_BASE(&Desc.Legacy);
4680
4681	/*
4682	* Ok, everything checked out fine. Now set the accessed bit before
4683	* committing the result into the registers.
4684	*/
4685	if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
4686	{
4687	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uSel);
4688	if (rcStrict != VINF_SUCCESS)
4689	return rcStrict;
4690	Desc.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
4691	}
4692
4693	/* commit */
4694	*pSel = uSel;
4695	pHid->Attr.u = X86DESC_GET_HID_ATTR(&Desc.Legacy);
4696	pHid->u32Limit = cbLimit;
4697	pHid->u64Base = u64Base;
4698	pHid->ValidSel = uSel;
4699	pHid->fFlags = CPUMSELREG_FLAGS_VALID;
4700
4701	/** @todo check if the hidden bits are loaded correctly for 64-bit
4702	* mode. */
4703
4704	/* This will affect the FLAT 32-bit mode flag: */
4705	if ( iSegReg < X86_SREG_FS
4706	&& IEM_IS_32BIT_CODE(pVCpu))
4707	pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK)
4708	\| iemCalc32BitFlatIndicator(pVCpu);
4709	}
4710
4711	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pHid));
4712	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
4713	return VINF_SUCCESS;
4714	}
4715
4716
4717	/**
4718	* Implements 'mov SReg, r/m'.
4719	*
4720	* @param iSegReg The segment register number (valid).
4721	* @param uSel The new selector value.
4722	*/
4723	IEM_CIMPL_DEF_2(iemCImpl_load_SReg, uint8_t, iSegReg, uint16_t, uSel)
4724	{
4725	VBOXSTRICTRC rcStrict = iemCImpl_LoadSRegWorker(pVCpu, iSegReg, uSel);
4726	if (rcStrict == VINF_SUCCESS)
4727	rcStrict = iemCImpl_LoadSRegFinish(pVCpu, cbInstr, iSegReg);
4728	return rcStrict;
4729	}
4730
4731
4732	/**
4733	* Implements 'pop SReg'.
4734	*
4735	* @param iSegReg The segment register number (valid).
4736	* @param enmEffOpSize The efficient operand size (valid).
4737	*/
4738	IEM_CIMPL_DEF_2(iemCImpl_pop_Sreg, uint8_t, iSegReg, IEMMODE, enmEffOpSize)
4739	{
4740	VBOXSTRICTRC rcStrict;
4741
4742	/*
4743	* Read the selector off the stack and join paths with mov ss, reg.
4744	*/
4745	RTUINT64U TmpRsp;
4746	TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
4747	switch (enmEffOpSize)
4748	{
4749	case IEMMODE_16BIT:
4750	{
4751	uint16_t uSel;
4752	rcStrict = iemMemStackPopU16Ex(pVCpu, &uSel, &TmpRsp);
4753	if (rcStrict == VINF_SUCCESS)
4754	rcStrict = iemCImpl_LoadSRegWorker(pVCpu, iSegReg, uSel);
4755	break;
4756	}
4757
4758	case IEMMODE_32BIT:
4759	{
4760	/* Modern Intel CPU only does a WORD sized access here, both as
4761	segmentation and paging is concerned. So, we have to emulate
4762	this to make bs3-cpu-weird-1 happy. */
4763	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
4764	{
4765	/* We don't have flexible enough stack primitives here, so just
4766	do a word pop and add two bytes to SP/RSP on success. */
4767	uint16_t uSel;
4768	rcStrict = iemMemStackPopU16Ex(pVCpu, &uSel, &TmpRsp);
4769	if (rcStrict == VINF_SUCCESS)
4770	{
4771	iemRegAddToRspEx(pVCpu, &TmpRsp, sizeof(uint32_t) - sizeof(uint16_t));
4772	rcStrict = iemCImpl_LoadSRegWorker(pVCpu, iSegReg, uSel);
4773	}
4774	}
4775	else
4776	{
4777	uint32_t u32Value;
4778	rcStrict = iemMemStackPopU32Ex(pVCpu, &u32Value, &TmpRsp);
4779	if (rcStrict == VINF_SUCCESS)
4780	rcStrict = iemCImpl_LoadSRegWorker(pVCpu, iSegReg, (uint16_t)u32Value);
4781	}
4782	break;
4783	}
4784
4785	case IEMMODE_64BIT:
4786	{
4787	/* Like for the 32-bit case above, intel only does a WORD access. */
4788	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
4789	{
4790	uint16_t uSel;
4791	rcStrict = iemMemStackPopU16Ex(pVCpu, &uSel, &TmpRsp);
4792	if (rcStrict == VINF_SUCCESS)
4793	{
4794	iemRegAddToRspEx(pVCpu, &TmpRsp, sizeof(uint64_t) - sizeof(uint16_t));
4795	rcStrict = iemCImpl_LoadSRegWorker(pVCpu, iSegReg, uSel);
4796	}
4797	}
4798	else
4799	{
4800	uint64_t u64Value;
4801	rcStrict = iemMemStackPopU64Ex(pVCpu, &u64Value, &TmpRsp);
4802	if (rcStrict == VINF_SUCCESS)
4803	rcStrict = iemCImpl_LoadSRegWorker(pVCpu, iSegReg, (uint16_t)u64Value);
4804	}
4805	break;
4806	}
4807	IEM_NOT_REACHED_DEFAULT_CASE_RET();
4808	}
4809
4810	/*
4811	* If the load succeeded, commit the stack change and finish the instruction.
4812	*/
4813	if (rcStrict == VINF_SUCCESS)
4814	{
4815	pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
4816	rcStrict = iemCImpl_LoadSRegFinish(pVCpu, cbInstr, iSegReg);
4817	}
4818
4819	return rcStrict;
4820	}
4821
4822
4823	/**
4824	* Implements lgs, lfs, les, lds & lss.
4825	*/
4826	IEM_CIMPL_DEF_5(iemCImpl_load_SReg_Greg, uint16_t, uSel, uint64_t, offSeg, uint8_t, iSegReg, uint8_t, iGReg, IEMMODE, enmEffOpSize)
4827	{
4828	/*
4829	* Use iemCImpl_LoadSRegWorker to do the tricky segment register loading.
4830	*/
4831	/** @todo verify and test that mov, pop and lXs works the segment
4832	* register loading in the exact same way. */
4833	VBOXSTRICTRC rcStrict = iemCImpl_LoadSRegWorker(pVCpu, iSegReg, uSel);
4834	if (rcStrict == VINF_SUCCESS)
4835	{
4836	switch (enmEffOpSize)
4837	{
4838	case IEMMODE_16BIT:
4839	iemGRegStoreU16(pVCpu, iGReg, offSeg);
4840	break;
4841	case IEMMODE_32BIT:
4842	case IEMMODE_64BIT:
4843	iemGRegStoreU64(pVCpu, iGReg, offSeg);
4844	break;
4845	IEM_NOT_REACHED_DEFAULT_CASE_RET();
4846	}
4847	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
4848	}
4849	return rcStrict;
4850	}
4851
4852
4853	/**
4854	* Helper for VERR, VERW, LAR, and LSL and loads the descriptor into memory.
4855	*
4856	* @retval VINF_SUCCESS on success.
4857	* @retval VINF_IEM_SELECTOR_NOT_OK if the selector isn't ok.
4858	* @retval iemMemFetchSysU64 return value.
4859	*
4860	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4861	* @param uSel The selector value.
4862	* @param fAllowSysDesc Whether system descriptors are OK or not.
4863	* @param pDesc Where to return the descriptor on success.
4864	*/
4865	static VBOXSTRICTRC iemCImpl_LoadDescHelper(PVMCPUCC pVCpu, uint16_t uSel, bool fAllowSysDesc, PIEMSELDESC pDesc)
4866	{
4867	pDesc->Long.au64[0] = 0;
4868	pDesc->Long.au64[1] = 0;
4869
4870	if (!(uSel & X86_SEL_MASK_OFF_RPL)) /** @todo test this on 64-bit. */
4871	return VINF_IEM_SELECTOR_NOT_OK;
4872
4873	/* Within the table limits? */
4874	RTGCPTR GCPtrBase;
4875	if (uSel & X86_SEL_LDT)
4876	{
4877	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_LDTR);
4878	if ( !pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present
4879	\|\| (uSel \| X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.ldtr.u32Limit )
4880	return VINF_IEM_SELECTOR_NOT_OK;
4881	GCPtrBase = pVCpu->cpum.GstCtx.ldtr.u64Base;
4882	}
4883	else
4884	{
4885	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_GDTR);
4886	if ((uSel \| X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.gdtr.cbGdt)
4887	return VINF_IEM_SELECTOR_NOT_OK;
4888	GCPtrBase = pVCpu->cpum.GstCtx.gdtr.pGdt;
4889	}
4890
4891	/* Fetch the descriptor. */
4892	VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Legacy.u, UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK));
4893	if (rcStrict != VINF_SUCCESS)
4894	return rcStrict;
4895	if (!pDesc->Legacy.Gen.u1DescType)
4896	{
4897	if (!fAllowSysDesc)
4898	return VINF_IEM_SELECTOR_NOT_OK;
4899	if (CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)))
4900	{
4901	rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Long.au64[1], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 8);
4902	if (rcStrict != VINF_SUCCESS)
4903	return rcStrict;
4904	}
4905
4906	}
4907
4908	return VINF_SUCCESS;
4909	}
4910
4911
4912	/**
4913	* Implements verr (fWrite = false) and verw (fWrite = true).
4914	*/
4915	IEM_CIMPL_DEF_2(iemCImpl_VerX, uint16_t, uSel, bool, fWrite)
4916	{
4917	Assert(!IEM_IS_REAL_OR_V86_MODE(pVCpu));
4918
4919	/** @todo figure whether the accessed bit is set or not. */
4920
4921	bool fAccessible = true;
4922	IEMSELDESC Desc;
4923	VBOXSTRICTRC rcStrict = iemCImpl_LoadDescHelper(pVCpu, uSel, false /fAllowSysDesc/, &Desc);
4924	if (rcStrict == VINF_SUCCESS)
4925	{
4926	/* Check the descriptor, order doesn't matter much here. */
4927	if ( !Desc.Legacy.Gen.u1DescType
4928	\|\| !Desc.Legacy.Gen.u1Present)
4929	fAccessible = false;
4930	else
4931	{
4932	if ( fWrite
4933	? (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_WRITE)) != X86_SEL_TYPE_WRITE
4934	: (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
4935	fAccessible = false;
4936
4937	/** @todo testcase for the conforming behavior. */
4938	if ( (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_CONF))
4939	!= (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_CONF))
4940	{
4941	if ((unsigned)(uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl)
4942	fAccessible = false;
4943	else if (IEM_GET_CPL(pVCpu) > Desc.Legacy.Gen.u2Dpl)
4944	fAccessible = false;
4945	}
4946	}
4947
4948	}
4949	else if (rcStrict == VINF_IEM_SELECTOR_NOT_OK)
4950	fAccessible = false;
4951	else
4952	return rcStrict;
4953
4954	/* commit */
4955	pVCpu->cpum.GstCtx.eflags.Bits.u1ZF = fAccessible;
4956
4957	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
4958	}
4959
4960
4961	/**
4962	* Implements LAR and LSL with 64-bit operand size.
4963	*
4964	* @returns VINF_SUCCESS.
4965	* @param pu64Dst Pointer to the destination register.
4966	* @param uSel The selector to load details for.
4967	* @param fIsLar true = LAR, false = LSL.
4968	*/
4969	IEM_CIMPL_DEF_3(iemCImpl_LarLsl_u64, uint64_t *, pu64Dst, uint16_t, uSel, bool, fIsLar)
4970	{
4971	Assert(!IEM_IS_REAL_OR_V86_MODE(pVCpu));
4972
4973	/** @todo figure whether the accessed bit is set or not. */
4974
4975	bool fDescOk = true;
4976	IEMSELDESC Desc;
4977	VBOXSTRICTRC rcStrict = iemCImpl_LoadDescHelper(pVCpu, uSel, true /fAllowSysDesc/, &Desc);
4978	if (rcStrict == VINF_SUCCESS)
4979	{
4980	/*
4981	* Check the descriptor type.
4982	*/
4983	if (!Desc.Legacy.Gen.u1DescType)
4984	{
4985	if (CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)))
4986	{
4987	if (Desc.Long.Gen.u5Zeros)
4988	fDescOk = false;
4989	else
4990	switch (Desc.Long.Gen.u4Type)
4991	{
4992	/** @todo Intel lists 0 as valid for LSL, verify whether that's correct */
4993	case AMD64_SEL_TYPE_SYS_TSS_AVAIL:
4994	case AMD64_SEL_TYPE_SYS_TSS_BUSY:
4995	case AMD64_SEL_TYPE_SYS_LDT: /** @todo Intel lists this as invalid for LAR, AMD and 32-bit does otherwise. */
4996	break;
4997	case AMD64_SEL_TYPE_SYS_CALL_GATE:
4998	fDescOk = fIsLar;
4999	break;
5000	default:
5001	fDescOk = false;
5002	break;
5003	}
5004	}
5005	else
5006	{
5007	switch (Desc.Long.Gen.u4Type)
5008	{
5009	case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
5010	case X86_SEL_TYPE_SYS_286_TSS_BUSY:
5011	case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
5012	case X86_SEL_TYPE_SYS_386_TSS_BUSY:
5013	case X86_SEL_TYPE_SYS_LDT:
5014	break;
5015	case X86_SEL_TYPE_SYS_286_CALL_GATE:
5016	case X86_SEL_TYPE_SYS_TASK_GATE:
5017	case X86_SEL_TYPE_SYS_386_CALL_GATE:
5018	fDescOk = fIsLar;
5019	break;
5020	default:
5021	fDescOk = false;
5022	break;
5023	}
5024	}
5025	}
5026	if (fDescOk)
5027	{
5028	/*
5029	* Check the RPL/DPL/CPL interaction..
5030	*/
5031	/** @todo testcase for the conforming behavior. */
5032	if ( (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_CONF)) != (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_CONF)
5033	\|\| !Desc.Legacy.Gen.u1DescType)
5034	{
5035	if ((unsigned)(uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl)
5036	fDescOk = false;
5037	else if (IEM_GET_CPL(pVCpu) > Desc.Legacy.Gen.u2Dpl)
5038	fDescOk = false;
5039	}
5040	}
5041
5042	if (fDescOk)
5043	{
5044	/*
5045	* All fine, start committing the result.
5046	*/
5047	if (fIsLar)
5048	*pu64Dst = Desc.Legacy.au32[1] & UINT32_C(0x00ffff00);
5049	else
5050	*pu64Dst = X86DESC_LIMIT_G(&Desc.Legacy);
5051	}
5052
5053	}
5054	else if (rcStrict == VINF_IEM_SELECTOR_NOT_OK)
5055	fDescOk = false;
5056	else
5057	return rcStrict;
5058
5059	/* commit flags value and advance rip. */
5060	pVCpu->cpum.GstCtx.eflags.Bits.u1ZF = fDescOk;
5061	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5062	}
5063
5064
5065	/**
5066	* Implements LAR and LSL with 16-bit operand size.
5067	*
5068	* @returns VINF_SUCCESS.
5069	* @param pu16Dst Pointer to the destination register.
5070	* @param uSel The selector to load details for.
5071	* @param fIsLar true = LAR, false = LSL.
5072	*/
5073	IEM_CIMPL_DEF_3(iemCImpl_LarLsl_u16, uint16_t *, pu16Dst, uint16_t, uSel, bool, fIsLar)
5074	{
5075	uint64_t u64TmpDst = *pu16Dst;
5076	IEM_CIMPL_CALL_3(iemCImpl_LarLsl_u64, &u64TmpDst, uSel, fIsLar);
5077	*pu16Dst = u64TmpDst;
5078	return VINF_SUCCESS;
5079	}
5080
5081
5082	/**
5083	* Implements lgdt.
5084	*
5085	* @param iEffSeg The segment of the new gdtr contents
5086	* @param GCPtrEffSrc The address of the new gdtr contents.
5087	* @param enmEffOpSize The effective operand size.
5088	*/
5089	IEM_CIMPL_DEF_3(iemCImpl_lgdt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc, IEMMODE, enmEffOpSize)
5090	{
5091	if (IEM_GET_CPL(pVCpu) != 0)
5092	return iemRaiseGeneralProtectionFault0(pVCpu);
5093	Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
5094
5095	if (!IEM_IS_IN_GUEST(pVCpu))
5096	{ /* probable */ }
5097	else if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
5098	&& IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
5099	{
5100	Log(("lgdt: Guest intercept -> VM-exit\n"));
5101	IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_GDTR_IDTR_ACCESS, VMXINSTRID_LGDT, cbInstr);
5102	}
5103	else if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_GDTR_WRITES))
5104	{
5105	Log(("lgdt: Guest intercept -> #VMEXIT\n"));
5106	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
5107	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_GDTR_WRITE, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
5108	}
5109
5110	/*
5111	* Fetch the limit and base address.
5112	*/
5113	uint16_t cbLimit;
5114	RTGCPTR GCPtrBase;
5115	VBOXSTRICTRC rcStrict = iemMemFetchDataXdtr(pVCpu, &cbLimit, &GCPtrBase, iEffSeg, GCPtrEffSrc, enmEffOpSize);
5116	if (rcStrict == VINF_SUCCESS)
5117	{
5118	if ( !IEM_IS_64BIT_CODE(pVCpu)
5119	\|\| X86_IS_CANONICAL(GCPtrBase))
5120	{
5121	rcStrict = CPUMSetGuestGDTR(pVCpu, GCPtrBase, cbLimit);
5122	if (rcStrict == VINF_SUCCESS)
5123	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5124	}
5125	else
5126	{
5127	Log(("iemCImpl_lgdt: Non-canonical base %04x:%RGv\n", cbLimit, GCPtrBase));
5128	return iemRaiseGeneralProtectionFault0(pVCpu);
5129	}
5130	}
5131	return rcStrict;
5132	}
5133
5134
5135	/**
5136	* Implements sgdt.
5137	*
5138	* @param iEffSeg The segment where to store the gdtr content.
5139	* @param GCPtrEffDst The address where to store the gdtr content.
5140	*/
5141	IEM_CIMPL_DEF_2(iemCImpl_sgdt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst)
5142	{
5143	/*
5144	* Join paths with sidt.
5145	* Note! No CPL or V8086 checks here, it's a really sad story, ask Intel if
5146	* you really must know.
5147	*/
5148	if (!IEM_IS_IN_GUEST(pVCpu))
5149	{ /* probable */ }
5150	else if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
5151	&& IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
5152	{
5153	Log(("sgdt: Guest intercept -> VM-exit\n"));
5154	IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_GDTR_IDTR_ACCESS, VMXINSTRID_SGDT, cbInstr);
5155	}
5156	else if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_GDTR_READS))
5157	{
5158	Log(("sgdt: Guest intercept -> #VMEXIT\n"));
5159	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
5160	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_GDTR_READ, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
5161	}
5162
5163	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_GDTR);
5164	VBOXSTRICTRC rcStrict = iemMemStoreDataXdtr(pVCpu, pVCpu->cpum.GstCtx.gdtr.cbGdt, pVCpu->cpum.GstCtx.gdtr.pGdt, iEffSeg, GCPtrEffDst);
5165	if (rcStrict == VINF_SUCCESS)
5166	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5167	return rcStrict;
5168	}
5169
5170
5171	/**
5172	* Implements lidt.
5173	*
5174	* @param iEffSeg The segment of the new idtr contents
5175	* @param GCPtrEffSrc The address of the new idtr contents.
5176	* @param enmEffOpSize The effective operand size.
5177	*/
5178	IEM_CIMPL_DEF_3(iemCImpl_lidt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc, IEMMODE, enmEffOpSize)
5179	{
5180	if (IEM_GET_CPL(pVCpu) != 0)
5181	return iemRaiseGeneralProtectionFault0(pVCpu);
5182	Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
5183
5184	if (!IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IDTR_WRITES))
5185	{ /* probable */ }
5186	else
5187	{
5188	Log(("lidt: Guest intercept -> #VMEXIT\n"));
5189	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
5190	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_IDTR_WRITE, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
5191	}
5192
5193	/*
5194	* Fetch the limit and base address.
5195	*/
5196	uint16_t cbLimit;
5197	RTGCPTR GCPtrBase;
5198	VBOXSTRICTRC rcStrict = iemMemFetchDataXdtr(pVCpu, &cbLimit, &GCPtrBase, iEffSeg, GCPtrEffSrc, enmEffOpSize);
5199	if (rcStrict == VINF_SUCCESS)
5200	{
5201	if ( !IEM_IS_64BIT_CODE(pVCpu)
5202	\|\| X86_IS_CANONICAL(GCPtrBase))
5203	{
5204	CPUMSetGuestIDTR(pVCpu, GCPtrBase, cbLimit);
5205	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5206	}
5207	else
5208	{
5209	Log(("iemCImpl_lidt: Non-canonical base %04x:%RGv\n", cbLimit, GCPtrBase));
5210	return iemRaiseGeneralProtectionFault0(pVCpu);
5211	}
5212	}
5213	return rcStrict;
5214	}
5215
5216
5217	/**
5218	* Implements sidt.
5219	*
5220	* @param iEffSeg The segment where to store the idtr content.
5221	* @param GCPtrEffDst The address where to store the idtr content.
5222	*/
5223	IEM_CIMPL_DEF_2(iemCImpl_sidt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst)
5224	{
5225	/*
5226	* Join paths with sgdt.
5227	* Note! No CPL or V8086 checks here, it's a really sad story, ask Intel if
5228	* you really must know.
5229	*/
5230	if (!IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IDTR_READS))
5231	{ /* probable */ }
5232	else
5233	{
5234	Log(("sidt: Guest intercept -> #VMEXIT\n"));
5235	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
5236	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_IDTR_READ, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
5237	}
5238
5239	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_IDTR);
5240	VBOXSTRICTRC rcStrict = iemMemStoreDataXdtr(pVCpu, pVCpu->cpum.GstCtx.idtr.cbIdt, pVCpu->cpum.GstCtx.idtr.pIdt, iEffSeg, GCPtrEffDst);
5241	if (rcStrict == VINF_SUCCESS)
5242	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5243	return rcStrict;
5244	}
5245
5246
5247	/**
5248	* Implements lldt.
5249	*
5250	* @param uNewLdt The new LDT selector value.
5251	*/
5252	IEM_CIMPL_DEF_1(iemCImpl_lldt, uint16_t, uNewLdt)
5253	{
5254	/*
5255	* Check preconditions.
5256	*/
5257	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
5258	{
5259	Log(("lldt %04x - real or v8086 mode -> #GP(0)\n", uNewLdt));
5260	return iemRaiseUndefinedOpcode(pVCpu);
5261	}
5262	if (IEM_GET_CPL(pVCpu) != 0)
5263	{
5264	Log(("lldt %04x - CPL is %d -> #GP(0)\n", uNewLdt, IEM_GET_CPL(pVCpu)));
5265	return iemRaiseGeneralProtectionFault0(pVCpu);
5266	}
5267
5268	/* Nested-guest VMX intercept (SVM is after all checks). */
5269	/** @todo testcase: exit vs check order. */
5270	if ( !IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
5271	\|\| !IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
5272	{ /* probable */ }
5273	else
5274	{
5275	Log(("lldt: Guest intercept -> VM-exit\n"));
5276	IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_LDTR_TR_ACCESS, VMXINSTRID_LLDT, cbInstr);
5277	}
5278
5279	if (uNewLdt & X86_SEL_LDT)
5280	{
5281	Log(("lldt %04x - LDT selector -> #GP\n", uNewLdt));
5282	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewLdt);
5283	}
5284
5285	/*
5286	* Now, loading a NULL selector is easy.
5287	*/
5288	if (!(uNewLdt & X86_SEL_MASK_OFF_RPL))
5289	{
5290	/* Nested-guest SVM intercept. */
5291	if (!IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_LDTR_WRITES))
5292	{ /* probable */ }
5293	else
5294	{
5295	Log(("lldt: Guest intercept -> #VMEXIT\n"));
5296	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
5297	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_LDTR_WRITE, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
5298	}
5299
5300	Log(("lldt %04x: Loading NULL selector.\n", uNewLdt));
5301	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_LDTR;
5302	CPUMSetGuestLDTR(pVCpu, uNewLdt);
5303	pVCpu->cpum.GstCtx.ldtr.ValidSel = uNewLdt;
5304	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
5305	if (IEM_IS_GUEST_CPU_AMD(pVCpu))
5306	{
5307	/* AMD-V seems to leave the base and limit alone. */
5308	pVCpu->cpum.GstCtx.ldtr.Attr.u = X86DESCATTR_UNUSABLE;
5309	}
5310	else
5311	{
5312	/* VT-x (Intel 3960x) seems to be doing the following. */
5313	pVCpu->cpum.GstCtx.ldtr.Attr.u = X86DESCATTR_UNUSABLE \| X86DESCATTR_G \| X86DESCATTR_D;
5314	pVCpu->cpum.GstCtx.ldtr.u64Base = 0;
5315	pVCpu->cpum.GstCtx.ldtr.u32Limit = UINT32_MAX;
5316	}
5317
5318	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5319	}
5320
5321	/*
5322	* Read the descriptor.
5323	*/
5324	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_LDTR \| CPUMCTX_EXTRN_GDTR);
5325	IEMSELDESC Desc;
5326	VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uNewLdt, X86_XCPT_GP); /** @todo Correct exception? */
5327	if (rcStrict != VINF_SUCCESS)
5328	return rcStrict;
5329
5330	/* Check GPs first. */
5331	if (Desc.Legacy.Gen.u1DescType)
5332	{
5333	Log(("lldt %#x - not system selector (type %x) -> #GP\n", uNewLdt, Desc.Legacy.Gen.u4Type));
5334	return iemRaiseGeneralProtectionFault(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
5335	}
5336	if (Desc.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_LDT)
5337	{
5338	Log(("lldt %#x - not LDT selector (type %x) -> #GP\n", uNewLdt, Desc.Legacy.Gen.u4Type));
5339	return iemRaiseGeneralProtectionFault(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
5340	}
5341	uint64_t u64Base;
5342	if (!IEM_IS_LONG_MODE(pVCpu))
5343	u64Base = X86DESC_BASE(&Desc.Legacy);
5344	else
5345	{
5346	if (Desc.Long.Gen.u5Zeros)
5347	{
5348	Log(("lldt %#x - u5Zeros=%#x -> #GP\n", uNewLdt, Desc.Long.Gen.u5Zeros));
5349	return iemRaiseGeneralProtectionFault(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
5350	}
5351
5352	u64Base = X86DESC64_BASE(&Desc.Long);
5353	if (!IEM_IS_CANONICAL(u64Base))
5354	{
5355	Log(("lldt %#x - non-canonical base address %#llx -> #GP\n", uNewLdt, u64Base));
5356	return iemRaiseGeneralProtectionFault(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
5357	}
5358	}
5359
5360	/* NP */
5361	if (!Desc.Legacy.Gen.u1Present)
5362	{
5363	Log(("lldt %#x - segment not present -> #NP\n", uNewLdt));
5364	return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewLdt);
5365	}
5366
5367	/* Nested-guest SVM intercept. */
5368	if (!IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_LDTR_WRITES))
5369	{ /* probable */ }
5370	else
5371	{
5372	Log(("lldt: Guest intercept -> #VMEXIT\n"));
5373	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
5374	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_LDTR_WRITE, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
5375	}
5376
5377	/*
5378	* It checks out alright, update the registers.
5379	*/
5380	/** @todo check if the actual value is loaded or if the RPL is dropped */
5381	CPUMSetGuestLDTR(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
5382	pVCpu->cpum.GstCtx.ldtr.ValidSel = uNewLdt & X86_SEL_MASK_OFF_RPL;
5383	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
5384	pVCpu->cpum.GstCtx.ldtr.Attr.u = X86DESC_GET_HID_ATTR(&Desc.Legacy);
5385	pVCpu->cpum.GstCtx.ldtr.u32Limit = X86DESC_LIMIT_G(&Desc.Legacy);
5386	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
5387
5388	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5389	}
5390
5391
5392	/**
5393	* Implements sldt GReg
5394	*
5395	* @param iGReg The general register to store the CRx value in.
5396	* @param enmEffOpSize The operand size.
5397	*/
5398	IEM_CIMPL_DEF_2(iemCImpl_sldt_reg, uint8_t, iGReg, uint8_t, enmEffOpSize)
5399	{
5400	if (!IEM_IS_IN_GUEST(pVCpu))
5401	{ /* probable */ }
5402	else if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
5403	&& IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
5404	{
5405	Log(("sldt: Guest intercept -> VM-exit\n"));
5406	IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_LDTR_TR_ACCESS, VMXINSTRID_SLDT, cbInstr);
5407	}
5408	else
5409	IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_LDTR_READS, SVM_EXIT_LDTR_READ, 0, 0, cbInstr);
5410
5411	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_LDTR);
5412	switch (enmEffOpSize)
5413	{
5414	case IEMMODE_16BIT:
5415	iemGRegStoreU16(pVCpu, iGReg, pVCpu->cpum.GstCtx.ldtr.Sel);
5416	break;
5417	case IEMMODE_32BIT:
5418	case IEMMODE_64BIT:
5419	iemGRegStoreU64(pVCpu, iGReg, pVCpu->cpum.GstCtx.ldtr.Sel);
5420	break;
5421	IEM_NOT_REACHED_DEFAULT_CASE_RET();
5422	}
5423	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5424	}
5425
5426
5427	/**
5428	* Implements sldt mem.
5429	*
5430	* @param iEffSeg The effective segment register to use with @a GCPtrMem.
5431	* @param GCPtrEffDst Where to store the 16-bit CR0 value.
5432	*/
5433	IEM_CIMPL_DEF_2(iemCImpl_sldt_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst)
5434	{
5435	IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_LDTR_READS, SVM_EXIT_LDTR_READ, 0, 0, cbInstr);
5436
5437	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_LDTR);
5438	VBOXSTRICTRC rcStrict = iemMemStoreDataU16(pVCpu, iEffSeg, GCPtrEffDst, pVCpu->cpum.GstCtx.ldtr.Sel);
5439	if (rcStrict == VINF_SUCCESS)
5440	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5441	return rcStrict;
5442	}
5443
5444
5445	/**
5446	* Implements ltr.
5447	*
5448	* @param uNewTr The new TSS selector value.
5449	*/
5450	IEM_CIMPL_DEF_1(iemCImpl_ltr, uint16_t, uNewTr)
5451	{
5452	/*
5453	* Check preconditions.
5454	*/
5455	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
5456	{
5457	Log(("ltr %04x - real or v8086 mode -> #GP(0)\n", uNewTr));
5458	return iemRaiseUndefinedOpcode(pVCpu);
5459	}
5460	if (IEM_GET_CPL(pVCpu) != 0)
5461	{
5462	Log(("ltr %04x - CPL is %d -> #GP(0)\n", uNewTr, IEM_GET_CPL(pVCpu)));
5463	return iemRaiseGeneralProtectionFault0(pVCpu);
5464	}
5465	if ( !IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
5466	\|\| !IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
5467	{ /* probable */ }
5468	else
5469	{
5470	Log(("ltr: Guest intercept -> VM-exit\n"));
5471	IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_LDTR_TR_ACCESS, VMXINSTRID_LTR, cbInstr);
5472	}
5473	if (uNewTr & X86_SEL_LDT)
5474	{
5475	Log(("ltr %04x - LDT selector -> #GP\n", uNewTr));
5476	return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewTr);
5477	}
5478	if (!(uNewTr & X86_SEL_MASK_OFF_RPL))
5479	{
5480	Log(("ltr %04x - NULL selector -> #GP(0)\n", uNewTr));
5481	return iemRaiseGeneralProtectionFault0(pVCpu);
5482	}
5483	if (!IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_TR_WRITES))
5484	{ /* probable */ }
5485	else
5486	{
5487	Log(("ltr: Guest intercept -> #VMEXIT\n"));
5488	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
5489	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_TR_WRITE, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
5490	}
5491
5492	/*
5493	* Read the descriptor.
5494	*/
5495	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_LDTR \| CPUMCTX_EXTRN_GDTR \| CPUMCTX_EXTRN_TR);
5496	IEMSELDESC Desc;
5497	VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uNewTr, X86_XCPT_GP); /** @todo Correct exception? */
5498	if (rcStrict != VINF_SUCCESS)
5499	return rcStrict;
5500
5501	/* Check GPs first. */
5502	if (Desc.Legacy.Gen.u1DescType)
5503	{
5504	Log(("ltr %#x - not system selector (type %x) -> #GP\n", uNewTr, Desc.Legacy.Gen.u4Type));
5505	return iemRaiseGeneralProtectionFault(pVCpu, uNewTr & X86_SEL_MASK_OFF_RPL);
5506	}
5507	if ( Desc.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_386_TSS_AVAIL /* same as AMD64_SEL_TYPE_SYS_TSS_AVAIL */
5508	&& ( Desc.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_286_TSS_AVAIL
5509	\|\| IEM_IS_LONG_MODE(pVCpu)) )
5510	{
5511	Log(("ltr %#x - not an available TSS selector (type %x) -> #GP\n", uNewTr, Desc.Legacy.Gen.u4Type));
5512	return iemRaiseGeneralProtectionFault(pVCpu, uNewTr & X86_SEL_MASK_OFF_RPL);
5513	}
5514	uint64_t u64Base;
5515	if (!IEM_IS_LONG_MODE(pVCpu))
5516	u64Base = X86DESC_BASE(&Desc.Legacy);
5517	else
5518	{
5519	if (Desc.Long.Gen.u5Zeros)
5520	{
5521	Log(("ltr %#x - u5Zeros=%#x -> #GP\n", uNewTr, Desc.Long.Gen.u5Zeros));
5522	return iemRaiseGeneralProtectionFault(pVCpu, uNewTr & X86_SEL_MASK_OFF_RPL);
5523	}
5524
5525	u64Base = X86DESC64_BASE(&Desc.Long);
5526	if (!IEM_IS_CANONICAL(u64Base))
5527	{
5528	Log(("ltr %#x - non-canonical base address %#llx -> #GP\n", uNewTr, u64Base));
5529	return iemRaiseGeneralProtectionFault(pVCpu, uNewTr & X86_SEL_MASK_OFF_RPL);
5530	}
5531	}
5532
5533	/* NP */
5534	if (!Desc.Legacy.Gen.u1Present)
5535	{
5536	Log(("ltr %#x - segment not present -> #NP\n", uNewTr));
5537	return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewTr);
5538	}
5539
5540	/*
5541	* Set it busy.
5542	* Note! Intel says this should lock down the whole descriptor, but we'll
5543	* restrict our selves to 32-bit for now due to lack of inline
5544	* assembly and such.
5545	*/
5546	uint8_t bUnmapInfo;
5547	void *pvDesc;
5548	rcStrict = iemMemMap(pVCpu, &pvDesc, &bUnmapInfo, 8, UINT8_MAX,
5549	pVCpu->cpum.GstCtx.gdtr.pGdt + (uNewTr & X86_SEL_MASK_OFF_RPL), IEM_ACCESS_DATA_RW, 0);
5550	if (rcStrict != VINF_SUCCESS)
5551	return rcStrict;
5552	switch ((uintptr_t)pvDesc & 3)
5553	{
5554	case 0: ASMAtomicBitSet(pvDesc, 40 + 1); break;
5555	case 1: ASMAtomicBitSet((uint8_t *)pvDesc + 3, 40 + 1 - 24); break;
5556	case 2: ASMAtomicBitSet((uint8_t *)pvDesc + 2, 40 + 1 - 16); break;
5557	case 3: ASMAtomicBitSet((uint8_t *)pvDesc + 1, 40 + 1 - 8); break;
5558	}
5559	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
5560	if (rcStrict != VINF_SUCCESS)
5561	return rcStrict;
5562	Desc.Legacy.Gen.u4Type \|= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
5563
5564	/*
5565	* It checks out alright, update the registers.
5566	*/
5567	/** @todo check if the actual value is loaded or if the RPL is dropped */
5568	CPUMSetGuestTR(pVCpu, uNewTr & X86_SEL_MASK_OFF_RPL);
5569	pVCpu->cpum.GstCtx.tr.ValidSel = uNewTr & X86_SEL_MASK_OFF_RPL;
5570	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
5571	pVCpu->cpum.GstCtx.tr.Attr.u = X86DESC_GET_HID_ATTR(&Desc.Legacy);
5572	pVCpu->cpum.GstCtx.tr.u32Limit = X86DESC_LIMIT_G(&Desc.Legacy);
5573	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
5574
5575	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5576	}
5577
5578
5579	/**
5580	* Implements str GReg
5581	*
5582	* @param iGReg The general register to store the CRx value in.
5583	* @param enmEffOpSize The operand size.
5584	*/
5585	IEM_CIMPL_DEF_2(iemCImpl_str_reg, uint8_t, iGReg, uint8_t, enmEffOpSize)
5586	{
5587	if (!IEM_IS_IN_GUEST(pVCpu))
5588	{ /* probable */ }
5589	else if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
5590	&& IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
5591	{
5592	Log(("str_reg: Guest intercept -> VM-exit\n"));
5593	IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_LDTR_TR_ACCESS, VMXINSTRID_STR, cbInstr);
5594	}
5595	else
5596	IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_TR_READS, SVM_EXIT_TR_READ, 0, 0, cbInstr);
5597
5598	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR);
5599	switch (enmEffOpSize)
5600	{
5601	case IEMMODE_16BIT:
5602	iemGRegStoreU16(pVCpu, iGReg, pVCpu->cpum.GstCtx.tr.Sel);
5603	break;
5604	case IEMMODE_32BIT:
5605	case IEMMODE_64BIT:
5606	iemGRegStoreU64(pVCpu, iGReg, pVCpu->cpum.GstCtx.tr.Sel);
5607	break;
5608	IEM_NOT_REACHED_DEFAULT_CASE_RET();
5609	}
5610	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5611	}
5612
5613
5614	/**
5615	* Implements str mem.
5616	*
5617	* @param iEffSeg The effective segment register to use with @a GCPtrMem.
5618	* @param GCPtrEffDst Where to store the 16-bit CR0 value.
5619	*/
5620	IEM_CIMPL_DEF_2(iemCImpl_str_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst)
5621	{
5622	if (!IEM_IS_IN_GUEST(pVCpu))
5623	{ /* probable */ }
5624	else if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
5625	&& IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
5626	{
5627	Log(("str_mem: Guest intercept -> VM-exit\n"));
5628	IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_LDTR_TR_ACCESS, VMXINSTRID_STR, cbInstr);
5629	}
5630	else
5631	IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_TR_READS, SVM_EXIT_TR_READ, 0, 0, cbInstr);
5632
5633	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR);
5634	VBOXSTRICTRC rcStrict = iemMemStoreDataU16(pVCpu, iEffSeg, GCPtrEffDst, pVCpu->cpum.GstCtx.tr.Sel);
5635	if (rcStrict == VINF_SUCCESS)
5636	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5637	return rcStrict;
5638	}
5639
5640
5641	/**
5642	* Implements mov GReg,CRx.
5643	*
5644	* @param iGReg The general register to store the CRx value in.
5645	* @param iCrReg The CRx register to read (valid).
5646	*/
5647	IEM_CIMPL_DEF_2(iemCImpl_mov_Rd_Cd, uint8_t, iGReg, uint8_t, iCrReg)
5648	{
5649	if (IEM_GET_CPL(pVCpu) != 0)
5650	return iemRaiseGeneralProtectionFault0(pVCpu);
5651	Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
5652
5653	if (!IEM_SVM_IS_READ_CR_INTERCEPT_SET(pVCpu, iCrReg))
5654	{ /* probable */ }
5655	else
5656	{
5657	Log(("iemCImpl_mov_Rd_Cd: Guest intercept CR%u -> #VMEXIT\n", iCrReg));
5658	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
5659	IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_READ_CR0 + iCrReg, IEMACCESSCRX_MOV_CRX, iGReg);
5660	}
5661
5662	/* Read it. */
5663	uint64_t crX;
5664	switch (iCrReg)
5665	{
5666	case 0:
5667	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5668	crX = pVCpu->cpum.GstCtx.cr0;
5669	if (IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_386)
5670	crX \|= UINT32_C(0x7fffffe0); /* All reserved CR0 flags are set on a 386, just like MSW on 286. */
5671	break;
5672	case 2:
5673	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_CR2);
5674	crX = pVCpu->cpum.GstCtx.cr2;
5675	break;
5676	case 3:
5677	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
5678	crX = pVCpu->cpum.GstCtx.cr3;
5679	break;
5680	case 4:
5681	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5682	crX = pVCpu->cpum.GstCtx.cr4;
5683	break;
5684	case 8:
5685	{
5686	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
5687	if (!IEM_IS_IN_GUEST(pVCpu))
5688	{ /* probable */ }
5689	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5690	else if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
5691	{
5692	VBOXSTRICTRC rcStrict = iemVmxVmexitInstrMovFromCr8(pVCpu, iGReg, cbInstr);
5693	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
5694	return rcStrict;
5695
5696	/*
5697	* If the Mov-from-CR8 doesn't cause a VM-exit, bits 7:4 of the VTPR is copied
5698	* to bits 0:3 of the destination operand. Bits 63:4 of the destination operand
5699	* are cleared.
5700	*
5701	* See Intel Spec. 29.3 "Virtualizing CR8-based TPR Accesses"
5702	*/
5703	if (IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_USE_TPR_SHADOW))
5704	{
5705	uint32_t const uTpr = iemVmxVirtApicReadRaw32(pVCpu, XAPIC_OFF_TPR);
5706	crX = (uTpr >> 4) & 0xf;
5707	break;
5708	}
5709	}
5710	#endif
5711	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
5712	else if (pVCpu->iem.s.fExec & IEM_F_X86_CTX_SVM)
5713	{
5714	PCSVMVMCBCTRL pVmcbCtrl = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.ctrl;
5715	if (CPUMIsGuestSvmVirtIntrMasking(pVCpu, IEM_GET_CTX(pVCpu)))
5716	{
5717	crX = pVmcbCtrl->IntCtrl.n.u8VTPR & 0xf;
5718	break;
5719	}
5720	}
5721	#endif
5722	uint8_t uTpr;
5723	int rc = PDMApicGetTpr(pVCpu, &uTpr, NULL, NULL);
5724	if (RT_SUCCESS(rc))
5725	crX = uTpr >> 4;
5726	else
5727	crX = 0;
5728	break;
5729	}
5730	IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* call checks */
5731	}
5732
5733	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5734	if (!IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
5735	{ /* probable */ }
5736	else
5737	switch (iCrReg)
5738	{
5739	/* CR0/CR4 reads are subject to masking when in VMX non-root mode. */
5740	case 0: crX = CPUMGetGuestVmxMaskedCr0(&pVCpu->cpum.GstCtx, pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr0Mask.u); break;
5741	case 4: crX = CPUMGetGuestVmxMaskedCr4(&pVCpu->cpum.GstCtx, pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr4Mask.u); break;
5742	case 3:
5743	{
5744	VBOXSTRICTRC rcStrict = iemVmxVmexitInstrMovFromCr3(pVCpu, iGReg, cbInstr);
5745	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
5746	return rcStrict;
5747	break;
5748	}
5749	}
5750	#endif
5751
5752	/* Store it. */
5753	if (IEM_IS_64BIT_CODE(pVCpu))
5754	iemGRegStoreU64(pVCpu, iGReg, crX);
5755	else
5756	iemGRegStoreU64(pVCpu, iGReg, (uint32_t)crX);
5757
5758	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5759	}
5760
5761
5762	/**
5763	* Implements smsw GReg.
5764	*
5765	* @param iGReg The general register to store the CRx value in.
5766	* @param enmEffOpSize The operand size.
5767	*/
5768	IEM_CIMPL_DEF_2(iemCImpl_smsw_reg, uint8_t, iGReg, uint8_t, enmEffOpSize)
5769	{
5770	IEM_SVM_CHECK_READ_CR0_INTERCEPT(pVCpu, 0 /* uExitInfo1 /, 0 / uExitInfo2 */, cbInstr);
5771
5772	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5773	uint64_t u64MaskedCr0;
5774	if (!IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
5775	u64MaskedCr0 = pVCpu->cpum.GstCtx.cr0;
5776	else
5777	u64MaskedCr0 = CPUMGetGuestVmxMaskedCr0(&pVCpu->cpum.GstCtx, pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr0Mask.u);
5778	uint64_t const u64GuestCr0 = u64MaskedCr0;
5779	#else
5780	uint64_t const u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5781	#endif
5782
5783	switch (enmEffOpSize)
5784	{
5785	case IEMMODE_16BIT:
5786	if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_386)
5787	iemGRegStoreU16(pVCpu, iGReg, (uint16_t)u64GuestCr0);
5788	/* Unused bits are set on 386 and older CPU: */
5789	else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
5790	iemGRegStoreU16(pVCpu, iGReg, (uint16_t)u64GuestCr0 \| 0xffe0);
5791	else
5792	iemGRegStoreU16(pVCpu, iGReg, (uint16_t)u64GuestCr0 \| 0xfff0);
5793	break;
5794
5795	/** @todo testcase for bits 31:16. We're not doing that correctly. */
5796
5797	case IEMMODE_32BIT:
5798	if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
5799	iemGRegStoreU32(pVCpu, iGReg, (uint32_t)u64GuestCr0);
5800	else /** @todo test this! */
5801	iemGRegStoreU32(pVCpu, iGReg, (uint32_t)u64GuestCr0 \| UINT32_C(0x7fffffe0)); /* Unused bits are set on 386. */
5802	break;
5803
5804	case IEMMODE_64BIT:
5805	iemGRegStoreU64(pVCpu, iGReg, u64GuestCr0);
5806	break;
5807
5808	IEM_NOT_REACHED_DEFAULT_CASE_RET();
5809	}
5810
5811	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5812	}
5813
5814
5815	/**
5816	* Implements smsw mem.
5817	*
5818	* @param iEffSeg The effective segment register to use with @a GCPtrMem.
5819	* @param GCPtrEffDst Where to store the 16-bit CR0 value.
5820	*/
5821	IEM_CIMPL_DEF_2(iemCImpl_smsw_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst)
5822	{
5823	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5824	if (!IEM_IS_IN_GUEST(pVCpu))
5825	{ /* probable */ }
5826	else if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
5827	u64GuestCr0 = CPUMGetGuestVmxMaskedCr0(&pVCpu->cpum.GstCtx, pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr0Mask.u);
5828	else
5829	IEM_SVM_CHECK_READ_CR0_INTERCEPT(pVCpu, 0 /* uExitInfo1 /, 0 / uExitInfo2 */, cbInstr);
5830
5831	uint16_t u16Value;
5832	if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_386)
5833	u16Value = (uint16_t)u64GuestCr0;
5834	else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
5835	u16Value = (uint16_t)u64GuestCr0 \| 0xffe0;
5836	else
5837	u16Value = (uint16_t)u64GuestCr0 \| 0xfff0;
5838
5839	VBOXSTRICTRC rcStrict = iemMemStoreDataU16(pVCpu, iEffSeg, GCPtrEffDst, u16Value);
5840	if (rcStrict == VINF_SUCCESS)
5841	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
5842	return rcStrict;
5843	}
5844
5845
5846	/**
5847	* Helper for mapping CR3 and PAE PDPEs for 'mov CRx,GReg'.
5848	*/
5849	#define IEM_MAP_PAE_PDPES_AT_CR3_RET(a_pVCpu, a_iCrReg, a_uCr3) \
5850	do \
5851	{ \
5852	int const rcX = PGMGstMapPaePdpesAtCr3(a_pVCpu, a_uCr3); \
5853	if (RT_SUCCESS(rcX)) \
5854	{ /* likely */ } \
5855	else \
5856	{ \
5857	/* Either invalid PDPTEs or CR3 second-level translation failed. Raise #GP(0) either way. */ \
5858	Log(("iemCImpl_load_Cr%#x: Trying to load invalid PAE PDPEs\n", a_iCrReg)); \
5859	return iemRaiseGeneralProtectionFault0(a_pVCpu); \
5860	} \
5861	} while (0)
5862
5863
5864	/**
5865	* Used to implemented 'mov CRx,GReg' and 'lmsw r/m16'.
5866	*
5867	* @param iCrReg The CRx register to write (valid).
5868	* @param uNewCrX The new value.
5869	* @param enmAccessCrX The instruction that caused the CrX load.
5870	* @param iGReg The general register in case of a 'mov CRx,GReg'
5871	* instruction.
5872	*/
5873	IEM_CIMPL_DEF_4(iemCImpl_load_CrX, uint8_t, iCrReg, uint64_t, uNewCrX, IEMACCESSCRX, enmAccessCrX, uint8_t, iGReg)
5874	{
5875	VBOXSTRICTRC rcStrict;
5876	int rc;
5877	#ifndef VBOX_WITH_NESTED_HWVIRT_SVM
5878	RT_NOREF2(iGReg, enmAccessCrX);
5879	#endif
5880
5881	/*
5882	* Try store it.
5883	* Unfortunately, CPUM only does a tiny bit of the work.
5884	*/
5885	switch (iCrReg)
5886	{
5887	case 0:
5888	{
5889	/*
5890	* Perform checks.
5891	*/
5892	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5893
5894	uint64_t const uOldCrX = pVCpu->cpum.GstCtx.cr0;
5895	uint32_t const fValid = CPUMGetGuestCR0ValidMask();
5896
5897	/* ET is hardcoded on 486 and later. */
5898	if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_486)
5899	uNewCrX \|= X86_CR0_ET;
5900	/* The 386 and 486 didn't #GP(0) on attempting to set reserved CR0 bits. ET was settable on 386. */
5901	else if (IEM_GET_TARGET_CPU(pVCpu) == IEMTARGETCPU_486)
5902	{
5903	uNewCrX &= fValid;
5904	uNewCrX \|= X86_CR0_ET;
5905	}
5906	else
5907	uNewCrX &= X86_CR0_PE \| X86_CR0_MP \| X86_CR0_EM \| X86_CR0_TS \| X86_CR0_PG \| X86_CR0_ET;
5908
5909	/* Check for reserved bits. */
5910	if (uNewCrX & ~(uint64_t)fValid)
5911	{
5912	Log(("Trying to set reserved CR0 bits: NewCR0=%#llx InvalidBits=%#llx\n", uNewCrX, uNewCrX & ~(uint64_t)fValid));
5913	return iemRaiseGeneralProtectionFault0(pVCpu);
5914	}
5915
5916	/* Check for invalid combinations. */
5917	if ( (uNewCrX & X86_CR0_PG)
5918	&& !(uNewCrX & X86_CR0_PE) )
5919	{
5920	Log(("Trying to set CR0.PG without CR0.PE\n"));
5921	return iemRaiseGeneralProtectionFault0(pVCpu);
5922	}
5923
5924	if ( !(uNewCrX & X86_CR0_CD)
5925	&& (uNewCrX & X86_CR0_NW) )
5926	{
5927	Log(("Trying to clear CR0.CD while leaving CR0.NW set\n"));
5928	return iemRaiseGeneralProtectionFault0(pVCpu);
5929	}
5930
5931	if ( !(uNewCrX & X86_CR0_PG)
5932	&& (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PCIDE))
5933	{
5934	Log(("Trying to clear CR0.PG while leaving CR4.PCID set\n"));
5935	return iemRaiseGeneralProtectionFault0(pVCpu);
5936	}
5937
5938	/* Long mode consistency checks. */
5939	if ( (uNewCrX & X86_CR0_PG)
5940	&& !(uOldCrX & X86_CR0_PG)
5941	&& (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME) )
5942	{
5943	if (!(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE))
5944	{
5945	Log(("Trying to enabled long mode paging without CR4.PAE set\n"));
5946	return iemRaiseGeneralProtectionFault0(pVCpu);
5947	}
5948	if (pVCpu->cpum.GstCtx.cs.Attr.n.u1Long)
5949	{
5950	Log(("Trying to enabled long mode paging with a long CS descriptor loaded.\n"));
5951	return iemRaiseGeneralProtectionFault0(pVCpu);
5952	}
5953	}
5954
5955	/** @todo testcase: what happens if we disable paging while in 64-bit code? */
5956
5957	if (!IEM_IS_IN_GUEST(pVCpu))
5958	{ /* probable */ }
5959	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5960	/* Check for bits that must remain set or cleared in VMX operation,
5961	see Intel spec. 23.8 "Restrictions on VMX operation". */
5962	else if (IEM_VMX_IS_ROOT_MODE(pVCpu))
5963	{
5964	uint64_t const uCr0Fixed0 = iemVmxGetCr0Fixed0(pVCpu, IEM_VMX_IS_NON_ROOT_MODE(pVCpu));
5965	if ((uNewCrX & uCr0Fixed0) != uCr0Fixed0)
5966	{
5967	Log(("Trying to clear reserved CR0 bits in VMX operation: NewCr0=%#llx MB1=%#llx\n", uNewCrX, uCr0Fixed0));
5968	return iemRaiseGeneralProtectionFault0(pVCpu);
5969	}
5970
5971	uint64_t const uCr0Fixed1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr0Fixed1;
5972	if (uNewCrX & ~uCr0Fixed1)
5973	{
5974	Log(("Trying to set reserved CR0 bits in VMX operation: NewCr0=%#llx MB0=%#llx\n", uNewCrX, uCr0Fixed1));
5975	return iemRaiseGeneralProtectionFault0(pVCpu);
5976	}
5977	}
5978	#endif
5979	/*
5980	* SVM nested-guest CR0 write intercepts.
5981	*/
5982	else if (IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(pVCpu, iCrReg))
5983	{
5984	Log(("iemCImpl_load_Cr%#x: Guest intercept -> #VMEXIT\n", iCrReg));
5985	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
5986	IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_CR0, enmAccessCrX, iGReg);
5987	}
5988	else if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_CR0_SEL_WRITE))
5989	{
5990	/* 'lmsw' intercepts regardless of whether the TS/MP bits are actually toggled. */
5991	if ( enmAccessCrX == IEMACCESSCRX_LMSW
5992	\|\| (uNewCrX & ~(X86_CR0_TS \| X86_CR0_MP)) != (uOldCrX & ~(X86_CR0_TS \| X86_CR0_MP)))
5993	{
5994	Assert(enmAccessCrX != IEMACCESSCRX_CLTS);
5995	Log(("iemCImpl_load_Cr%#x: lmsw or bits other than TS/MP changed: Guest intercept -> #VMEXIT\n", iCrReg));
5996	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
5997	IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_CR0_SEL_WRITE, enmAccessCrX, iGReg);
5998	}
5999	}
6000
6001	/*
6002	* Change EFER.LMA if entering or leaving long mode.
6003	*/
6004	uint64_t NewEFER = pVCpu->cpum.GstCtx.msrEFER;
6005	if ( (uNewCrX & X86_CR0_PG) != (uOldCrX & X86_CR0_PG)
6006	&& (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME) )
6007	{
6008	if (uNewCrX & X86_CR0_PG)
6009	NewEFER \|= MSR_K6_EFER_LMA;
6010	else
6011	NewEFER &= ~MSR_K6_EFER_LMA;
6012
6013	CPUMSetGuestEFER(pVCpu, NewEFER);
6014	Assert(pVCpu->cpum.GstCtx.msrEFER == NewEFER);
6015	}
6016
6017	IEMTLBTRACE_LOAD_CR0(pVCpu, uNewCrX, uOldCrX);
6018
6019	/*
6020	* Inform PGM.
6021	*/
6022	if ( (uNewCrX & (X86_CR0_PG \| X86_CR0_WP \| X86_CR0_PE \| X86_CR0_CD \| X86_CR0_NW))
6023	!= (uOldCrX & (X86_CR0_PG \| X86_CR0_WP \| X86_CR0_PE \| X86_CR0_CD \| X86_CR0_NW)) )
6024	{
6025	if ( enmAccessCrX != IEMACCESSCRX_MOV_CRX
6026	\|\| !CPUMIsPaePagingEnabled(uNewCrX, pVCpu->cpum.GstCtx.cr4, NewEFER)
6027	\|\| CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)))
6028	{ /* likely */ }
6029	else
6030	IEM_MAP_PAE_PDPES_AT_CR3_RET(pVCpu, iCrReg, pVCpu->cpum.GstCtx.cr3);
6031	rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, true /* global */);
6032	AssertRCReturn(rc, rc);
6033	/* ignore informational status codes */
6034	}
6035
6036	/*
6037	* Change CR0.
6038	*/
6039	CPUMSetGuestCR0(pVCpu, uNewCrX);
6040	Assert(pVCpu->cpum.GstCtx.cr0 == uNewCrX);
6041
6042	/* Update the fExec flags if PE changed. */
6043	if ((uNewCrX ^ uOldCrX) & X86_CR0_PE)
6044	iemRecalcExecModeAndCplAndAcFlags(pVCpu);
6045
6046	/*
6047	* Inform PGM some more...
6048	*/
6049	rcStrict = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER,
6050	false /* fForce */);
6051	break;
6052	}
6053
6054	/*
6055	* CR2 can be changed without any restrictions.
6056	*/
6057	case 2:
6058	{
6059	if (!IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(pVCpu, /cr/ 2))
6060	{ /* probable */ }
6061	else
6062	{
6063	Log(("iemCImpl_load_Cr%#x: Guest intercept -> #VMEXIT\n", iCrReg));
6064	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
6065	IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_CR2, enmAccessCrX, iGReg);
6066	}
6067	pVCpu->cpum.GstCtx.cr2 = uNewCrX;
6068	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_CR2;
6069	rcStrict = VINF_SUCCESS;
6070	break;
6071	}
6072
6073	/*
6074	* CR3 is relatively simple, although AMD and Intel have different
6075	* accounts of how setting reserved bits are handled. We take intel's
6076	* word for the lower bits and AMD's for the high bits (63:52). The
6077	* lower reserved bits are ignored and left alone; OpenBSD 5.8 relies
6078	* on this.
6079	*/
6080	/** @todo Testcase: Setting reserved bits in CR3, especially before
6081	* enabling paging. */
6082	case 3:
6083	{
6084	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
6085
6086	/* Bit 63 being clear in the source operand with PCIDE indicates no invalidations are required. */
6087	if ( (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PCIDE)
6088	&& (uNewCrX & RT_BIT_64(63)))
6089	{
6090	/** @todo r=ramshankar: avoiding a TLB flush altogether here causes Windows 10
6091	* SMP(w/o nested-paging) to hang during bootup on Skylake systems, see
6092	* Intel spec. 4.10.4.1 "Operations that Invalidate TLBs and
6093	* Paging-Structure Caches". */
6094	uNewCrX &= ~RT_BIT_64(63);
6095	}
6096
6097	/* Check / mask the value. */
6098	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6099	/* See Intel spec. 27.2.2 "EPT Translation Mechanism" footnote. */
6100	uint64_t const fInvPhysMask = !CPUMIsGuestVmxEptPagingEnabledEx(IEM_GET_CTX(pVCpu))
6101	? (UINT64_MAX << IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cMaxPhysAddrWidth)
6102	: (~X86_CR3_EPT_PAGE_MASK & X86_PAGE_4K_BASE_MASK);
6103	#else
6104	uint64_t const fInvPhysMask = UINT64_C(0xfff0000000000000);
6105	#endif
6106	if (uNewCrX & fInvPhysMask)
6107	{
6108	/** @todo Should we raise this only for 64-bit mode like Intel claims? AMD is
6109	* very vague in this area. As mentioned above, need testcase on real
6110	* hardware... Sigh. */
6111	Log(("Trying to load CR3 with invalid high bits set: %#llx\n", uNewCrX));
6112	return iemRaiseGeneralProtectionFault0(pVCpu);
6113	}
6114
6115	uint64_t fValid;
6116	if ( (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE)
6117	&& (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME))
6118	{
6119	/** @todo Redundant? This value has already been validated above. */
6120	fValid = UINT64_C(0x000fffffffffffff);
6121	}
6122	else
6123	fValid = UINT64_C(0xffffffff);
6124	if (uNewCrX & ~fValid)
6125	{
6126	Log(("Automatically clearing reserved MBZ bits in CR3 load: NewCR3=%#llx ClearedBits=%#llx\n",
6127	uNewCrX, uNewCrX & ~fValid));
6128	uNewCrX &= fValid;
6129	}
6130
6131	if (!IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(pVCpu, /cr/ 3))
6132	{ /* probable */ }
6133	else
6134	{
6135	Log(("iemCImpl_load_Cr%#x: Guest intercept -> #VMEXIT\n", iCrReg));
6136	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
6137	IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_CR3, enmAccessCrX, iGReg);
6138	}
6139
6140	IEMTLBTRACE_LOAD_CR3(pVCpu, uNewCrX, pVCpu->cpum.GstCtx.cr3);
6141
6142	/* Inform PGM. */
6143	if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PG)
6144	{
6145	if ( !CPUMIsGuestInPAEModeEx(IEM_GET_CTX(pVCpu))
6146	\|\| CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)))
6147	{ /* likely */ }
6148	else
6149	{
6150	Assert(enmAccessCrX == IEMACCESSCRX_MOV_CRX);
6151	IEM_MAP_PAE_PDPES_AT_CR3_RET(pVCpu, iCrReg, uNewCrX);
6152	}
6153	rc = PGMFlushTLB(pVCpu, uNewCrX, !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PGE));
6154	AssertRCReturn(rc, rc);
6155	/* ignore informational status codes */
6156	}
6157
6158	/* Make the change. */
6159	rc = CPUMSetGuestCR3(pVCpu, uNewCrX);
6160	AssertRCSuccessReturn(rc, rc);
6161
6162	rcStrict = VINF_SUCCESS;
6163	break;
6164	}
6165
6166	/*
6167	* CR4 is a bit more tedious as there are bits which cannot be cleared
6168	* under some circumstances and such.
6169	*/
6170	case 4:
6171	{
6172	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
6173	uint64_t const uOldCrX = pVCpu->cpum.GstCtx.cr4;
6174
6175	/* Reserved bits. */
6176	uint32_t const fValid = CPUMGetGuestCR4ValidMask(pVCpu->CTX_SUFF(pVM));
6177	if (uNewCrX & ~(uint64_t)fValid)
6178	{
6179	Log(("Trying to set reserved CR4 bits: NewCR4=%#llx InvalidBits=%#llx\n", uNewCrX, uNewCrX & ~(uint64_t)fValid));
6180	return iemRaiseGeneralProtectionFault0(pVCpu);
6181	}
6182
6183	bool const fPcide = !(uOldCrX & X86_CR4_PCIDE) && (uNewCrX & X86_CR4_PCIDE);
6184	bool const fLongMode = CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu));
6185
6186	/* PCIDE check. */
6187	if ( fPcide
6188	&& ( !fLongMode
6189	\|\| (pVCpu->cpum.GstCtx.cr3 & UINT64_C(0xfff))))
6190	{
6191	Log(("Trying to set PCIDE with invalid PCID or outside long mode. Pcid=%#x\n", (pVCpu->cpum.GstCtx.cr3 & UINT64_C(0xfff))));
6192	return iemRaiseGeneralProtectionFault0(pVCpu);
6193	}
6194
6195	/* PAE check. */
6196	if ( fLongMode
6197	&& (uOldCrX & X86_CR4_PAE)
6198	&& !(uNewCrX & X86_CR4_PAE))
6199	{
6200	Log(("Trying to set clear CR4.PAE while long mode is active\n"));
6201	return iemRaiseGeneralProtectionFault0(pVCpu);
6202	}
6203
6204	if (!IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(pVCpu, /cr/ 4))
6205	{ /* probable */ }
6206	else
6207	{
6208	Log(("iemCImpl_load_Cr%#x: Guest intercept -> #VMEXIT\n", iCrReg));
6209	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
6210	IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_CR4, enmAccessCrX, iGReg);
6211	}
6212
6213	/* Check for bits that must remain set or cleared in VMX operation,
6214	see Intel spec. 23.8 "Restrictions on VMX operation". */
6215	if (!IEM_VMX_IS_ROOT_MODE(pVCpu))
6216	{ /* probable */ }
6217	else
6218	{
6219	uint64_t const uCr4Fixed0 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed0;
6220	if ((uNewCrX & uCr4Fixed0) != uCr4Fixed0)
6221	{
6222	Log(("Trying to clear reserved CR4 bits in VMX operation: NewCr4=%#llx MB1=%#llx\n", uNewCrX, uCr4Fixed0));
6223	return iemRaiseGeneralProtectionFault0(pVCpu);
6224	}
6225
6226	uint64_t const uCr4Fixed1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed1;
6227	if (uNewCrX & ~uCr4Fixed1)
6228	{
6229	Log(("Trying to set reserved CR4 bits in VMX operation: NewCr4=%#llx MB0=%#llx\n", uNewCrX, uCr4Fixed1));
6230	return iemRaiseGeneralProtectionFault0(pVCpu);
6231	}
6232	}
6233
6234	IEMTLBTRACE_LOAD_CR4(pVCpu, uNewCrX, uOldCrX);
6235
6236	/*
6237	* Notify PGM.
6238	*/
6239	if ((uNewCrX ^ uOldCrX) & (X86_CR4_PSE \| X86_CR4_PAE \| X86_CR4_PGE \| X86_CR4_PCIDE /* \| X86_CR4_SMEP */))
6240	{
6241	if ( !CPUMIsPaePagingEnabled(pVCpu->cpum.GstCtx.cr0, uNewCrX, pVCpu->cpum.GstCtx.msrEFER)
6242	\|\| CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)))
6243	{ /* likely */ }
6244	else
6245	{
6246	Assert(enmAccessCrX == IEMACCESSCRX_MOV_CRX);
6247	IEM_MAP_PAE_PDPES_AT_CR3_RET(pVCpu, iCrReg, pVCpu->cpum.GstCtx.cr3);
6248	}
6249	rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, true /* global */);
6250	AssertRCReturn(rc, rc);
6251	/* ignore informational status codes */
6252	}
6253
6254	/*
6255	* Change it.
6256	*/
6257	rc = CPUMSetGuestCR4(pVCpu, uNewCrX);
6258	AssertRCSuccessReturn(rc, rc);
6259	Assert(pVCpu->cpum.GstCtx.cr4 == uNewCrX);
6260
6261	rcStrict = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER,
6262	false /* fForce */);
6263	break;
6264	}
6265
6266	/*
6267	* CR8 maps to the APIC TPR.
6268	*/
6269	case 8:
6270	{
6271	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
6272	if (uNewCrX & ~(uint64_t)0xf)
6273	{
6274	Log(("Trying to set reserved CR8 bits (%#RX64)\n", uNewCrX));
6275	return iemRaiseGeneralProtectionFault0(pVCpu);
6276	}
6277
6278	if (!IEM_IS_IN_GUEST(pVCpu))
6279	{ /* probable */ }
6280	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6281	else if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
6282	&& IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_USE_TPR_SHADOW))
6283	{
6284	/*
6285	* If the Mov-to-CR8 doesn't cause a VM-exit, bits 0:3 of the source operand
6286	* is copied to bits 7:4 of the VTPR. Bits 0:3 and bits 31:8 of the VTPR are
6287	* cleared. Following this the processor performs TPR virtualization.
6288	*
6289	* However, we should not perform TPR virtualization immediately here but
6290	* after this instruction has completed.
6291	*
6292	* See Intel spec. 29.3 "Virtualizing CR8-based TPR Accesses"
6293	* See Intel spec. 27.1 "Architectural State Before A VM-exit"
6294	*/
6295	uint32_t const uTpr = (uNewCrX & 0xf) << 4;
6296	Log(("iemCImpl_load_Cr%#x: Virtualizing TPR (%#x) write\n", iCrReg, uTpr));
6297	iemVmxVirtApicWriteRaw32(pVCpu, XAPIC_OFF_TPR, uTpr);
6298	iemVmxVirtApicSetPendingWrite(pVCpu, XAPIC_OFF_TPR);
6299	rcStrict = VINF_SUCCESS;
6300	break;
6301	}
6302	#endif
6303	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
6304	else if (pVCpu->iem.s.fExec & IEM_F_X86_CTX_SVM)
6305	{
6306	if (IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(pVCpu, /cr/ 8))
6307	{
6308	Log(("iemCImpl_load_Cr%#x: Guest intercept -> #VMEXIT\n", iCrReg));
6309	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
6310	IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_CR8, enmAccessCrX, iGReg);
6311	}
6312
6313	pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.ctrl.IntCtrl.n.u8VTPR = uNewCrX;
6314	if (CPUMIsGuestSvmVirtIntrMasking(pVCpu, IEM_GET_CTX(pVCpu)))
6315	{
6316	rcStrict = VINF_SUCCESS;
6317	break;
6318	}
6319	}
6320	#endif
6321	uint8_t const u8Tpr = (uint8_t)uNewCrX << 4;
6322	PDMApicSetTpr(pVCpu, u8Tpr);
6323	rcStrict = VINF_SUCCESS;
6324	break;
6325	}
6326
6327	IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* call checks */
6328	}
6329
6330	/*
6331	* Advance the RIP on success.
6332	*/
6333	if (RT_SUCCESS(rcStrict))
6334	{
6335	if (rcStrict != VINF_SUCCESS)
6336	iemSetPassUpStatus(pVCpu, rcStrict);
6337	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
6338	}
6339
6340	return rcStrict;
6341	}
6342
6343
6344	/**
6345	* Implements mov CRx,GReg.
6346	*
6347	* @param iCrReg The CRx register to write (valid).
6348	* @param iGReg The general register to load the CRx value from.
6349	*/
6350	IEM_CIMPL_DEF_2(iemCImpl_mov_Cd_Rd, uint8_t, iCrReg, uint8_t, iGReg)
6351	{
6352	if (IEM_GET_CPL(pVCpu) != 0)
6353	return iemRaiseGeneralProtectionFault0(pVCpu);
6354	Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
6355
6356	/*
6357	* Read the new value from the source register and call common worker.
6358	*/
6359	uint64_t uNewCrX;
6360	if (IEM_IS_64BIT_CODE(pVCpu))
6361	uNewCrX = iemGRegFetchU64(pVCpu, iGReg);
6362	else
6363	uNewCrX = iemGRegFetchU32(pVCpu, iGReg);
6364
6365	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6366	if (!IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
6367	{ /* probable */ }
6368	else
6369	{
6370	VBOXSTRICTRC rcStrict = VINF_VMX_INTERCEPT_NOT_ACTIVE;
6371	switch (iCrReg)
6372	{
6373	case 0:
6374	case 4: rcStrict = iemVmxVmexitInstrMovToCr0Cr4(pVCpu, iCrReg, &uNewCrX, iGReg, cbInstr); break;
6375	case 3: rcStrict = iemVmxVmexitInstrMovToCr3(pVCpu, uNewCrX, iGReg, cbInstr); break;
6376	case 8: rcStrict = iemVmxVmexitInstrMovToCr8(pVCpu, iGReg, cbInstr); break;
6377	}
6378	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
6379	return rcStrict;
6380	}
6381	#endif
6382
6383	return IEM_CIMPL_CALL_4(iemCImpl_load_CrX, iCrReg, uNewCrX, IEMACCESSCRX_MOV_CRX, iGReg);
6384	}
6385
6386
6387	/**
6388	* Implements 'LMSW r/m16'
6389	*
6390	* @param u16NewMsw The new value.
6391	* @param GCPtrEffDst The guest-linear address of the source operand in case
6392	* of a memory operand. For register operand, pass
6393	* NIL_RTGCPTR.
6394	*/
6395	IEM_CIMPL_DEF_2(iemCImpl_lmsw, uint16_t, u16NewMsw, RTGCPTR, GCPtrEffDst)
6396	{
6397	if (IEM_GET_CPL(pVCpu) != 0)
6398	return iemRaiseGeneralProtectionFault0(pVCpu);
6399	Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
6400	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
6401
6402	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6403	/* Check nested-guest VMX intercept and get updated MSW if there's no VM-exit. */
6404	if (!IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
6405	{ /* probable */ }
6406	else
6407	{
6408	VBOXSTRICTRC rcStrict = iemVmxVmexitInstrLmsw(pVCpu, pVCpu->cpum.GstCtx.cr0, &u16NewMsw, GCPtrEffDst, cbInstr);
6409	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
6410	return rcStrict;
6411	}
6412	#else
6413	RT_NOREF_PV(GCPtrEffDst);
6414	#endif
6415
6416	/*
6417	* Compose the new CR0 value and call common worker.
6418	*/
6419	uint64_t uNewCr0 = pVCpu->cpum.GstCtx.cr0 & ~(X86_CR0_MP \| X86_CR0_EM \| X86_CR0_TS);
6420	uNewCr0 \|= u16NewMsw & (X86_CR0_PE \| X86_CR0_MP \| X86_CR0_EM \| X86_CR0_TS);
6421	return IEM_CIMPL_CALL_4(iemCImpl_load_CrX, /cr/ 0, uNewCr0, IEMACCESSCRX_LMSW, UINT8_MAX /* iGReg */);
6422	}
6423
6424
6425	/**
6426	* Implements 'CLTS'.
6427	*/
6428	IEM_CIMPL_DEF_0(iemCImpl_clts)
6429	{
6430	if (IEM_GET_CPL(pVCpu) != 0)
6431	return iemRaiseGeneralProtectionFault0(pVCpu);
6432
6433	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
6434	uint64_t uNewCr0 = pVCpu->cpum.GstCtx.cr0;
6435	uNewCr0 &= ~X86_CR0_TS;
6436
6437	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6438	if (!IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
6439	{ /* probable */ }
6440	else
6441	{
6442	VBOXSTRICTRC rcStrict = iemVmxVmexitInstrClts(pVCpu, cbInstr);
6443	if (rcStrict == VINF_VMX_MODIFIES_BEHAVIOR)
6444	uNewCr0 \|= (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS);
6445	else if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
6446	return rcStrict;
6447	}
6448	#endif
6449
6450	return IEM_CIMPL_CALL_4(iemCImpl_load_CrX, /cr/ 0, uNewCr0, IEMACCESSCRX_CLTS, UINT8_MAX /* iGReg */);
6451	}
6452
6453
6454	/**
6455	* Implements mov GReg,DRx.
6456	*
6457	* @param iGReg The general register to store the DRx value in.
6458	* @param iDrReg The DRx register to read (0-7).
6459	*/
6460	IEM_CIMPL_DEF_2(iemCImpl_mov_Rd_Dd, uint8_t, iGReg, uint8_t, iDrReg)
6461	{
6462	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6463	/*
6464	* Check nested-guest VMX intercept.
6465	* Unlike most other intercepts, the Mov DRx intercept takes preceedence
6466	* over CPL and CR4.DE and even DR4/DR5 checks.
6467	*
6468	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
6469	*/
6470	if (!IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
6471	{ /* probable */ }
6472	else
6473	{
6474	VBOXSTRICTRC rcStrict = iemVmxVmexitInstrMovDrX(pVCpu, VMXINSTRID_MOV_FROM_DRX, iDrReg, iGReg, cbInstr);
6475	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
6476	return rcStrict;
6477	}
6478	#endif
6479
6480	/*
6481	* Check preconditions.
6482	*/
6483	/* Raise GPs. */
6484	if (IEM_GET_CPL(pVCpu) != 0)
6485	return iemRaiseGeneralProtectionFault0(pVCpu);
6486	Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
6487	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
6488
6489	/** @todo \#UD in outside ring-0 too? */
6490	if (iDrReg == 4 \|\| iDrReg == 5)
6491	{
6492	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_CR4);
6493	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_DE)
6494	{
6495	Log(("mov r%u,dr%u: CR4.DE=1 -> #GP(0)\n", iGReg, iDrReg));
6496	return iemRaiseGeneralProtectionFault0(pVCpu);
6497	}
6498	iDrReg += 2;
6499	}
6500
6501	/* Raise #DB if general access detect is enabled. */
6502	if (pVCpu->cpum.GstCtx.dr[7] & X86_DR7_GD)
6503	{
6504	Log(("mov r%u,dr%u: DR7.GD=1 -> #DB\n", iGReg, iDrReg));
6505	return iemRaiseDebugException(pVCpu);
6506	}
6507
6508	/*
6509	* Read the debug register and store it in the specified general register.
6510	*/
6511	uint64_t drX;
6512	switch (iDrReg)
6513	{
6514	case 0:
6515	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
6516	drX = pVCpu->cpum.GstCtx.dr[0];
6517	break;
6518	case 1:
6519	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
6520	drX = pVCpu->cpum.GstCtx.dr[1];
6521	break;
6522	case 2:
6523	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
6524	drX = pVCpu->cpum.GstCtx.dr[2];
6525	break;
6526	case 3:
6527	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
6528	drX = pVCpu->cpum.GstCtx.dr[3];
6529	break;
6530	case 6:
6531	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
6532	drX = pVCpu->cpum.GstCtx.dr[6];
6533	drX \|= X86_DR6_RA1_MASK;
6534	drX &= ~X86_DR6_RAZ_MASK;
6535	break;
6536	case 7:
6537	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
6538	drX = pVCpu->cpum.GstCtx.dr[7];
6539	drX \|=X86_DR7_RA1_MASK;
6540	drX &= ~X86_DR7_RAZ_MASK;
6541	break;
6542	IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* caller checks */
6543	}
6544
6545	/** @todo SVM nested-guest intercept for DR8-DR15? */
6546	/*
6547	* Check for any SVM nested-guest intercepts for the DRx read.
6548	*/
6549	if (!IEM_SVM_IS_READ_DR_INTERCEPT_SET(pVCpu, iDrReg))
6550	{ /* probable */ }
6551	else
6552	{
6553	Log(("mov r%u,dr%u: Guest intercept -> #VMEXIT\n", iGReg, iDrReg));
6554	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
6555	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_READ_DR0 + (iDrReg & 0xf),
6556	IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvmDecodeAssists ? (iGReg & 7) : 0, 0 /* uExitInfo2 */);
6557	}
6558
6559	if (IEM_IS_64BIT_CODE(pVCpu))
6560	iemGRegStoreU64(pVCpu, iGReg, drX);
6561	else
6562	iemGRegStoreU32(pVCpu, iGReg, (uint32_t)drX);
6563
6564	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
6565	}
6566
6567
6568	/**
6569	* Implements mov DRx,GReg.
6570	*
6571	* @param iDrReg The DRx register to write (valid).
6572	* @param iGReg The general register to load the DRx value from.
6573	*/
6574	IEM_CIMPL_DEF_2(iemCImpl_mov_Dd_Rd, uint8_t, iDrReg, uint8_t, iGReg)
6575	{
6576	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6577	/*
6578	* Check nested-guest VMX intercept.
6579	* Unlike most other intercepts, the Mov DRx intercept takes preceedence
6580	* over CPL and CR4.DE and even DR4/DR5 checks.
6581	*
6582	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
6583	*/
6584	if (!IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
6585	{ /* probable */ }
6586	else
6587	{
6588	VBOXSTRICTRC rcStrict = iemVmxVmexitInstrMovDrX(pVCpu, VMXINSTRID_MOV_TO_DRX, iDrReg, iGReg, cbInstr);
6589	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
6590	return rcStrict;
6591	}
6592	#endif
6593
6594	/*
6595	* Check preconditions.
6596	*/
6597	if (IEM_GET_CPL(pVCpu) != 0)
6598	return iemRaiseGeneralProtectionFault0(pVCpu);
6599	Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
6600	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
6601
6602	if (iDrReg == 4 \|\| iDrReg == 5)
6603	{
6604	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_CR4);
6605	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_DE)
6606	{
6607	Log(("mov dr%u,r%u: CR4.DE=1 -> #GP(0)\n", iDrReg, iGReg));
6608	return iemRaiseGeneralProtectionFault0(pVCpu);
6609	}
6610	iDrReg += 2;
6611	}
6612
6613	/* Raise #DB if general access detect is enabled. */
6614	/** @todo is \#DB/DR7.GD raised before any reserved high bits in DR7/DR6
6615	* \#GP? */
6616	if (pVCpu->cpum.GstCtx.dr[7] & X86_DR7_GD)
6617	{
6618	Log(("mov dr%u,r%u: DR7.GD=1 -> #DB\n", iDrReg, iGReg));
6619	return iemRaiseDebugException(pVCpu);
6620	}
6621
6622	/*
6623	* Read the new value from the source register.
6624	*/
6625	uint64_t uNewDrX;
6626	if (IEM_IS_64BIT_CODE(pVCpu))
6627	uNewDrX = iemGRegFetchU64(pVCpu, iGReg);
6628	else
6629	uNewDrX = iemGRegFetchU32(pVCpu, iGReg);
6630
6631	/*
6632	* Adjust it.
6633	*/
6634	switch (iDrReg)
6635	{
6636	case 0:
6637	case 1:
6638	case 2:
6639	case 3:
6640	/* nothing to adjust */
6641	break;
6642
6643	case 6:
6644	if (uNewDrX & X86_DR6_MBZ_MASK)
6645	{
6646	Log(("mov dr%u,%#llx: DR6 high bits are not zero -> #GP(0)\n", iDrReg, uNewDrX));
6647	return iemRaiseGeneralProtectionFault0(pVCpu);
6648	}
6649	uNewDrX \|= X86_DR6_RA1_MASK;
6650	uNewDrX &= ~X86_DR6_RAZ_MASK;
6651	break;
6652
6653	case 7:
6654	if (uNewDrX & X86_DR7_MBZ_MASK)
6655	{
6656	Log(("mov dr%u,%#llx: DR7 high bits are not zero -> #GP(0)\n", iDrReg, uNewDrX));
6657	return iemRaiseGeneralProtectionFault0(pVCpu);
6658	}
6659	uNewDrX \|= X86_DR7_RA1_MASK;
6660	uNewDrX &= ~X86_DR7_RAZ_MASK;
6661	break;
6662
6663	IEM_NOT_REACHED_DEFAULT_CASE_RET();
6664	}
6665
6666	/** @todo SVM nested-guest intercept for DR8-DR15? */
6667	/*
6668	* Check for any SVM nested-guest intercepts for the DRx write.
6669	*/
6670	if (!IEM_SVM_IS_WRITE_DR_INTERCEPT_SET(pVCpu, iDrReg))
6671	{ /* probable */ }
6672	else
6673	{
6674	Log2(("mov dr%u,r%u: Guest intercept -> #VMEXIT\n", iDrReg, iGReg));
6675	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
6676	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_DR0 + (iDrReg & 0xf),
6677	IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvmDecodeAssists ? (iGReg & 7) : 0, 0 /* uExitInfo2 */);
6678	}
6679
6680	/*
6681	* Do the actual setting.
6682	*/
6683	if (iDrReg < 4)
6684	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
6685	else if (iDrReg == 6)
6686	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
6687
6688	int rc = CPUMSetGuestDRx(pVCpu, iDrReg, uNewDrX);
6689	AssertRCSuccessReturn(rc, RT_SUCCESS_NP(rc) ? VERR_IEM_IPE_1 : rc);
6690
6691	/*
6692	* Re-init hardware breakpoint summary if it was DR7 that got changed.
6693	*
6694	* We also do this when an active data breakpoint is updated so that the
6695	* TLB entry can be correctly invalidated.
6696	*/
6697	if ( iDrReg == 7
6698	#ifdef IEM_WITH_DATA_TLB
6699	\|\| ( iDrReg <= 3
6700	&& (X86_DR7_L_G(iDrReg) & pVCpu->cpum.GstCtx.dr[7])
6701	&& X86_DR7_IS_W_CFG(pVCpu->cpum.GstCtx.dr[7], iDrReg) )
6702	#endif
6703	)
6704	iemRecalcExecDbgFlags(pVCpu);
6705
6706	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
6707	}
6708
6709
6710	/**
6711	* Implements mov GReg,TRx.
6712	*
6713	* @param iGReg The general register to store the
6714	* TRx value in.
6715	* @param iTrReg The TRx register to read (6/7).
6716	*/
6717	IEM_CIMPL_DEF_2(iemCImpl_mov_Rd_Td, uint8_t, iGReg, uint8_t, iTrReg)
6718	{
6719	/*
6720	* Check preconditions. NB: This instruction is 386/486 only.
6721	*/
6722
6723	/* Raise GPs. */
6724	if (IEM_GET_CPL(pVCpu) != 0)
6725	return iemRaiseGeneralProtectionFault0(pVCpu);
6726	Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
6727
6728	if (iTrReg < 6 \|\| iTrReg > 7)
6729	{
6730	/** @todo Do Intel CPUs reject this or are the TRs aliased? */
6731	Log(("mov r%u,tr%u: invalid register -> #GP(0)\n", iGReg, iTrReg));
6732	return iemRaiseGeneralProtectionFault0(pVCpu);
6733	}
6734
6735	/*
6736	* Read the test register and store it in the specified general register.
6737	* This is currently a dummy implementation that only exists to satisfy
6738	* old debuggers like WDEB386 or OS/2 KDB which unconditionally read the
6739	* TR6/TR7 registers. Software which actually depends on the TR values
6740	* (different on 386/486) is exceedingly rare.
6741	*/
6742	uint32_t trX;
6743	switch (iTrReg)
6744	{
6745	case 6:
6746	trX = 0; /* Currently a dummy. */
6747	break;
6748	case 7:
6749	trX = 0; /* Currently a dummy. */
6750	break;
6751	IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* call checks */
6752	}
6753
6754	iemGRegStoreU32(pVCpu, iGReg, trX);
6755
6756	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
6757	}
6758
6759
6760	/**
6761	* Implements mov TRx,GReg.
6762	*
6763	* @param iTrReg The TRx register to write (valid).
6764	* @param iGReg The general register to load the TRx
6765	* value from.
6766	*/
6767	IEM_CIMPL_DEF_2(iemCImpl_mov_Td_Rd, uint8_t, iTrReg, uint8_t, iGReg)
6768	{
6769	/*
6770	* Check preconditions. NB: This instruction is 386/486 only.
6771	*/
6772
6773	/* Raise GPs. */
6774	if (IEM_GET_CPL(pVCpu) != 0)
6775	return iemRaiseGeneralProtectionFault0(pVCpu);
6776	Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
6777
6778	if (iTrReg < 6 \|\| iTrReg > 7)
6779	{
6780	/** @todo Do Intel CPUs reject this or are the TRs aliased? */
6781	Log(("mov r%u,tr%u: invalid register -> #GP(0)\n", iGReg, iTrReg));
6782	return iemRaiseGeneralProtectionFault0(pVCpu);
6783	}
6784
6785	/*
6786	* Read the new value from the source register.
6787	*/
6788	uint32_t uNewTrX = iemGRegFetchU32(pVCpu, iGReg);
6789
6790	/*
6791	* Here we would do the actual setting if this weren't a dummy implementation.
6792	* This is currently a dummy implementation that only exists to prevent
6793	* old debuggers like WDEB386 or OS/2 KDB from crashing.
6794	*/
6795	RT_NOREF(uNewTrX);
6796
6797	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
6798	}
6799
6800
6801	/**
6802	* Implements 'INVLPG m'.
6803	*
6804	* @param GCPtrPage The effective address of the page to invalidate.
6805	* @remarks Updates the RIP.
6806	*/
6807	IEM_CIMPL_DEF_1(iemCImpl_invlpg, RTGCPTR, GCPtrPage)
6808	{
6809	/* ring-0 only. */
6810	if (IEM_GET_CPL(pVCpu) != 0)
6811	return iemRaiseGeneralProtectionFault0(pVCpu);
6812	Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
6813	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
6814
6815	if (!IEM_IS_IN_GUEST(pVCpu))
6816	{ /* probable */ }
6817	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6818	else if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
6819	&& IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_INVLPG_EXIT))
6820	{
6821	Log(("invlpg: Guest intercept (%RGp) -> VM-exit\n", GCPtrPage));
6822	return iemVmxVmexitInstrInvlpg(pVCpu, GCPtrPage, cbInstr);
6823	}
6824	#endif
6825	else if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_INVLPG))
6826	{
6827	Log(("invlpg: Guest intercept (%RGp) -> #VMEXIT\n", GCPtrPage));
6828	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
6829	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_INVLPG,
6830	IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvmDecodeAssists ? GCPtrPage : 0, 0 /* uExitInfo2 */);
6831	}
6832
6833	int rc = PGMInvalidatePage(pVCpu, GCPtrPage);
6834	if (rc == VINF_SUCCESS)
6835	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
6836	if (rc == VINF_PGM_SYNC_CR3)
6837	{
6838	iemSetPassUpStatus(pVCpu, rc);
6839	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
6840	}
6841
6842	AssertMsg(RT_FAILURE_NP(rc), ("%Rrc\n", rc));
6843	Log(("PGMInvalidatePage(%RGv) -> %Rrc\n", GCPtrPage, rc));
6844	return rc;
6845	}
6846
6847
6848	/**
6849	* Implements INVPCID.
6850	*
6851	* @param iEffSeg The segment of the invpcid descriptor.
6852	* @param GCPtrInvpcidDesc The address of invpcid descriptor.
6853	* @param uInvpcidType The invalidation type.
6854	* @remarks Updates the RIP.
6855	*/
6856	IEM_CIMPL_DEF_3(iemCImpl_invpcid, uint8_t, iEffSeg, RTGCPTR, GCPtrInvpcidDesc, uint64_t, uInvpcidType)
6857	{
6858	/*
6859	* Check preconditions.
6860	*/
6861	if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fInvpcid)
6862	return iemRaiseUndefinedOpcode(pVCpu);
6863
6864	/* When in VMX non-root mode and INVPCID is not enabled, it results in #UD. */
6865	if (RT_LIKELY( !IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
6866	\|\| IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_INVPCID)))
6867	{ /* likely */ }
6868	else
6869	{
6870	Log(("invpcid: Not enabled for nested-guest execution -> #UD\n"));
6871	return iemRaiseUndefinedOpcode(pVCpu);
6872	}
6873
6874	if (IEM_GET_CPL(pVCpu) != 0)
6875	{
6876	Log(("invpcid: CPL != 0 -> #GP(0)\n"));
6877	return iemRaiseGeneralProtectionFault0(pVCpu);
6878	}
6879
6880	if (IEM_IS_V86_MODE(pVCpu))
6881	{
6882	Log(("invpcid: v8086 mode -> #GP(0)\n"));
6883	return iemRaiseGeneralProtectionFault0(pVCpu);
6884	}
6885
6886	/*
6887	* Check nested-guest intercept.
6888	*
6889	* INVPCID causes a VM-exit if "enable INVPCID" and "INVLPG exiting" are
6890	* both set. We have already checked the former earlier in this function.
6891	*
6892	* CPL and virtual-8086 mode checks take priority over this VM-exit.
6893	* See Intel spec. "25.1.1 Relative Priority of Faults and VM Exits".
6894	*/
6895	if ( !IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
6896	\|\| !IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_INVLPG_EXIT))
6897	{ /* probable */ }
6898	else
6899	{
6900	Log(("invpcid: Guest intercept -> #VM-exit\n"));
6901	IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_INVPCID, VMXINSTRID_NONE, cbInstr);
6902	}
6903
6904	if (uInvpcidType > X86_INVPCID_TYPE_MAX_VALID)
6905	{
6906	Log(("invpcid: invalid/unrecognized invpcid type %#RX64 -> #GP(0)\n", uInvpcidType));
6907	return iemRaiseGeneralProtectionFault0(pVCpu);
6908	}
6909	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
6910
6911	/*
6912	* Fetch the invpcid descriptor from guest memory.
6913	*/
6914	/** @todo Check if the entire 128 bits are always read for all types. Check for invalid types as well. */
6915	RTUINT128U uDesc;
6916	VBOXSTRICTRC rcStrict = iemMemFetchDataU128(pVCpu, &uDesc, iEffSeg, GCPtrInvpcidDesc);
6917	if (rcStrict == VINF_SUCCESS)
6918	{
6919	/*
6920	* Validate the descriptor.
6921	*/
6922	if (uDesc.s.Lo > 0xfff)
6923	{
6924	Log(("invpcid: reserved bits set in invpcid descriptor %#RX64 -> #GP(0)\n", uDesc.s.Lo));
6925	return iemRaiseGeneralProtectionFault0(pVCpu);
6926	}
6927
6928	RTGCUINTPTR64 const GCPtrInvAddr = uDesc.s.Hi;
6929	uint8_t const uPcid = uDesc.s.Lo & UINT64_C(0xfff);
6930	uint32_t const uCr4 = pVCpu->cpum.GstCtx.cr4;
6931	uint64_t const uCr3 = pVCpu->cpum.GstCtx.cr3;
6932	switch (uInvpcidType)
6933	{
6934	case X86_INVPCID_TYPE_INDV_ADDR:
6935	{
6936	if (!IEM_IS_CANONICAL(GCPtrInvAddr))
6937	{
6938	Log(("invpcid: invalidation address %#RGP is not canonical -> #GP(0)\n", GCPtrInvAddr));
6939	return iemRaiseGeneralProtectionFault0(pVCpu);
6940	}
6941	if ( !(uCr4 & X86_CR4_PCIDE)
6942	&& uPcid != 0)
6943	{
6944	Log(("invpcid: invalid pcid %#x\n", uPcid));
6945	return iemRaiseGeneralProtectionFault0(pVCpu);
6946	}
6947
6948	/* Invalidate mappings for the linear address tagged with PCID except global translations. */
6949	/** @todo PGMFlushTLB is overkill for X86_INVPCID_TYPE_INDV_ADDR. Add a fGlobal parameter
6950	* to PGMInvalidatePage or add a new function to support this variation of invlpg. */
6951	PGMFlushTLB(pVCpu, uCr3, false /* fGlobal */);
6952	break;
6953	}
6954
6955	case X86_INVPCID_TYPE_SINGLE_CONTEXT:
6956	{
6957	if ( !(uCr4 & X86_CR4_PCIDE)
6958	&& uPcid != 0)
6959	{
6960	Log(("invpcid: invalid pcid %#x\n", uPcid));
6961	return iemRaiseGeneralProtectionFault0(pVCpu);
6962	}
6963	/* Invalidate all mappings associated with PCID except global translations. */
6964	PGMFlushTLB(pVCpu, uCr3, false /* fGlobal */);
6965	break;
6966	}
6967
6968	case X86_INVPCID_TYPE_ALL_CONTEXT_INCL_GLOBAL:
6969	{
6970	PGMFlushTLB(pVCpu, uCr3, true /* fGlobal */);
6971	break;
6972	}
6973
6974	case X86_INVPCID_TYPE_ALL_CONTEXT_EXCL_GLOBAL:
6975	{
6976	PGMFlushTLB(pVCpu, uCr3, false /* fGlobal */);
6977	break;
6978	}
6979	IEM_NOT_REACHED_DEFAULT_CASE_RET();
6980	}
6981	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
6982	}
6983	return rcStrict;
6984	}
6985
6986
6987	/**
6988	* Implements INVD.
6989	*/
6990	IEM_CIMPL_DEF_0(iemCImpl_invd)
6991	{
6992	if (IEM_GET_CPL(pVCpu) != 0)
6993	{
6994	Log(("invd: CPL != 0 -> #GP(0)\n"));
6995	return iemRaiseGeneralProtectionFault0(pVCpu);
6996	}
6997
6998	if (!IEM_IS_IN_GUEST(pVCpu))
6999	{ /* probable */ }
7000	else if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
7001	IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_INVD, cbInstr);
7002	else
7003	IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_INVD, SVM_EXIT_INVD, 0, 0, cbInstr);
7004
7005	/* We currently take no action here. */
7006	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7007	}
7008
7009
7010	/**
7011	* Implements WBINVD.
7012	*/
7013	IEM_CIMPL_DEF_0(iemCImpl_wbinvd)
7014	{
7015	if (IEM_GET_CPL(pVCpu) != 0)
7016	{
7017	Log(("wbinvd: CPL != 0 -> #GP(0)\n"));
7018	return iemRaiseGeneralProtectionFault0(pVCpu);
7019	}
7020
7021	if (!IEM_IS_IN_GUEST(pVCpu))
7022	{ /* probable */ }
7023	else if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
7024	IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_WBINVD, cbInstr);
7025	else
7026	IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_WBINVD, SVM_EXIT_WBINVD, 0, 0, cbInstr);
7027
7028	/* We currently take no action here. */
7029	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7030	}
7031
7032
7033	/** Opcode 0x0f 0xaa. */
7034	IEM_CIMPL_DEF_0(iemCImpl_rsm)
7035	{
7036	IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_RSM, SVM_EXIT_RSM, 0, 0, cbInstr);
7037	NOREF(cbInstr);
7038	return iemRaiseUndefinedOpcode(pVCpu);
7039	}
7040
7041
7042	/**
7043	* Implements RDTSC.
7044	*/
7045	IEM_CIMPL_DEF_0(iemCImpl_rdtsc)
7046	{
7047	/*
7048	* Check preconditions.
7049	*/
7050	if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fTsc)
7051	return iemRaiseUndefinedOpcode(pVCpu);
7052
7053	if (IEM_GET_CPL(pVCpu) != 0)
7054	{
7055	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
7056	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_TSD)
7057	{
7058	Log(("rdtsc: CR4.TSD and CPL=%u -> #GP(0)\n", IEM_GET_CPL(pVCpu)));
7059	return iemRaiseGeneralProtectionFault0(pVCpu);
7060	}
7061	}
7062
7063	if (!IEM_IS_IN_GUEST(pVCpu))
7064	{ /* probable */ }
7065	else if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
7066	&& IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_RDTSC_EXIT))
7067	{
7068	Log(("rdtsc: Guest intercept -> VM-exit\n"));
7069	IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_RDTSC, cbInstr);
7070	}
7071	else if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_RDTSC))
7072	{
7073	Log(("rdtsc: Guest intercept -> #VMEXIT\n"));
7074	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
7075	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_RDTSC, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
7076	}
7077
7078	/*
7079	* Do the job.
7080	*/
7081	uint64_t uTicks = TMCpuTickGet(pVCpu);
7082	#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
7083	uTicks = CPUMApplyNestedGuestTscOffset(pVCpu, uTicks);
7084	#endif
7085	pVCpu->cpum.GstCtx.rax = RT_LO_U32(uTicks);
7086	pVCpu->cpum.GstCtx.rdx = RT_HI_U32(uTicks);
7087	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX); /* For IEMExecDecodedRdtsc. */
7088	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7089	}
7090
7091
7092	/**
7093	* Implements RDTSC.
7094	*/
7095	IEM_CIMPL_DEF_0(iemCImpl_rdtscp)
7096	{
7097	/*
7098	* Check preconditions.
7099	*/
7100	if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fRdTscP)
7101	return iemRaiseUndefinedOpcode(pVCpu);
7102
7103	if (RT_LIKELY( !IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
7104	\|\| IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_RDTSCP)))
7105	{ /* likely */ }
7106	else
7107	{
7108	Log(("rdtscp: Not enabled for VMX non-root mode -> #UD\n"));
7109	return iemRaiseUndefinedOpcode(pVCpu);
7110	}
7111
7112	if (IEM_GET_CPL(pVCpu) != 0)
7113	{
7114	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
7115	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_TSD)
7116	{
7117	Log(("rdtscp: CR4.TSD and CPL=%u -> #GP(0)\n", IEM_GET_CPL(pVCpu)));
7118	return iemRaiseGeneralProtectionFault0(pVCpu);
7119	}
7120	}
7121
7122	if (!IEM_IS_IN_GUEST(pVCpu))
7123	{ /* probable */ }
7124	else if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
7125	&& IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_RDTSC_EXIT))
7126	{
7127	Log(("rdtscp: Guest intercept -> VM-exit\n"));
7128	IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_RDTSCP, cbInstr);
7129	}
7130	else if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_RDTSCP))
7131	{
7132	Log(("rdtscp: Guest intercept -> #VMEXIT\n"));
7133	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
7134	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_RDTSCP, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
7135	}
7136
7137	/*
7138	* Do the job.
7139	* Query the MSR first in case of trips to ring-3.
7140	*/
7141	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
7142	VBOXSTRICTRC rcStrict = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pVCpu->cpum.GstCtx.rcx);
7143	if (rcStrict == VINF_SUCCESS)
7144	{
7145	/* Low dword of the TSC_AUX msr only. */
7146	pVCpu->cpum.GstCtx.rcx &= UINT32_C(0xffffffff);
7147
7148	uint64_t uTicks = TMCpuTickGet(pVCpu);
7149	#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
7150	uTicks = CPUMApplyNestedGuestTscOffset(pVCpu, uTicks);
7151	#endif
7152	pVCpu->cpum.GstCtx.rax = RT_LO_U32(uTicks);
7153	pVCpu->cpum.GstCtx.rdx = RT_HI_U32(uTicks);
7154	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX); /* For IEMExecDecodedRdtscp. */
7155	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7156	}
7157	return rcStrict;
7158	}
7159
7160
7161	/**
7162	* Implements RDPMC.
7163	*/
7164	IEM_CIMPL_DEF_0(iemCImpl_rdpmc)
7165	{
7166	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
7167
7168	if ( IEM_GET_CPL(pVCpu) != 0
7169	&& !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PCE))
7170	return iemRaiseGeneralProtectionFault0(pVCpu);
7171
7172	if (!IEM_IS_IN_GUEST(pVCpu))
7173	{ /* probable */ }
7174	else if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
7175	&& IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_RDPMC_EXIT))
7176	{
7177	Log(("rdpmc: Guest intercept -> VM-exit\n"));
7178	IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_RDPMC, cbInstr);
7179	}
7180	else if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_RDPMC))
7181	{
7182	Log(("rdpmc: Guest intercept -> #VMEXIT\n"));
7183	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
7184	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_RDPMC, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
7185	}
7186
7187	/** @todo Emulate performance counters, for now just return 0. */
7188	pVCpu->cpum.GstCtx.rax = 0;
7189	pVCpu->cpum.GstCtx.rdx = 0;
7190	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX);
7191	/** @todo We should trigger a \#GP here if the CPU doesn't support the index in
7192	* ecx but see @bugref{3472}! */
7193
7194	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7195	}
7196
7197
7198	/**
7199	* Implements RDMSR.
7200	*/
7201	IEM_CIMPL_DEF_0(iemCImpl_rdmsr)
7202	{
7203	/*
7204	* Check preconditions.
7205	*/
7206	if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMsr)
7207	return iemRaiseUndefinedOpcode(pVCpu);
7208	if (IEM_GET_CPL(pVCpu) != 0)
7209	return iemRaiseGeneralProtectionFault0(pVCpu);
7210
7211	/*
7212	* Check nested-guest intercepts.
7213	*/
7214	if (!IEM_IS_IN_GUEST(pVCpu))
7215	{ /* probable */ }
7216	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7217	else if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
7218	{
7219	if (iemVmxIsRdmsrWrmsrInterceptSet(pVCpu, VMX_EXIT_RDMSR, pVCpu->cpum.GstCtx.ecx))
7220	IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_RDMSR, cbInstr);
7221	}
7222	#endif
7223	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
7224	else if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_MSR_PROT))
7225	{
7226	VBOXSTRICTRC rcStrict = iemSvmHandleMsrIntercept(pVCpu, pVCpu->cpum.GstCtx.ecx, false /* fWrite */, cbInstr);
7227	if (rcStrict == VINF_SVM_VMEXIT)
7228	return VINF_SUCCESS;
7229	if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE)
7230	{
7231	Log(("IEM: SVM intercepted rdmsr(%#x) failed. rc=%Rrc\n", pVCpu->cpum.GstCtx.ecx, VBOXSTRICTRC_VAL(rcStrict)));
7232	return rcStrict;
7233	}
7234	}
7235	#endif
7236
7237	/*
7238	* Do the job.
7239	*/
7240	RTUINT64U uValue;
7241	/** @todo make CPUMAllMsrs.cpp import the necessary MSR state. */
7242	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
7243
7244	VBOXSTRICTRC rcStrict = CPUMQueryGuestMsr(pVCpu, pVCpu->cpum.GstCtx.ecx, &uValue.u);
7245	if (rcStrict == VINF_SUCCESS)
7246	{
7247	pVCpu->cpum.GstCtx.rax = uValue.s.Lo;
7248	pVCpu->cpum.GstCtx.rdx = uValue.s.Hi;
7249	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX);
7250
7251	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7252	}
7253
7254	#ifndef IN_RING3
7255	/* Deferred to ring-3. */
7256	if (rcStrict == VINF_CPUM_R3_MSR_READ)
7257	{
7258	Log(("IEM: rdmsr(%#x) -> ring-3\n", pVCpu->cpum.GstCtx.ecx));
7259	return rcStrict;
7260	}
7261	#endif
7262
7263	/* Often a unimplemented MSR or MSR bit, so worth logging. */
7264	if (pVCpu->iem.s.cLogRelRdMsr < 32)
7265	{
7266	pVCpu->iem.s.cLogRelRdMsr++;
7267	LogRel(("IEM: rdmsr(%#x) -> #GP(0)\n", pVCpu->cpum.GstCtx.ecx));
7268	}
7269	else
7270	Log(( "IEM: rdmsr(%#x) -> #GP(0)\n", pVCpu->cpum.GstCtx.ecx));
7271	AssertMsgReturn(rcStrict == VERR_CPUM_RAISE_GP_0, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), VERR_IPE_UNEXPECTED_STATUS);
7272	return iemRaiseGeneralProtectionFault0(pVCpu);
7273	}
7274
7275
7276	/**
7277	* Implements WRMSR.
7278	*/
7279	IEM_CIMPL_DEF_0(iemCImpl_wrmsr)
7280	{
7281	/*
7282	* Check preconditions.
7283	*/
7284	if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMsr)
7285	return iemRaiseUndefinedOpcode(pVCpu);
7286	if (IEM_GET_CPL(pVCpu) != 0)
7287	return iemRaiseGeneralProtectionFault0(pVCpu);
7288
7289	RTUINT64U uValue;
7290	uValue.s.Lo = pVCpu->cpum.GstCtx.eax;
7291	uValue.s.Hi = pVCpu->cpum.GstCtx.edx;
7292
7293	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
7294
7295	/** @todo make CPUMAllMsrs.cpp import the necessary MSR state. */
7296	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
7297
7298	/*
7299	* Check nested-guest intercepts.
7300	*/
7301	if (!IEM_IS_IN_GUEST(pVCpu))
7302	{ /* probable */ }
7303	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7304	else if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
7305	{
7306	if (iemVmxIsRdmsrWrmsrInterceptSet(pVCpu, VMX_EXIT_WRMSR, idMsr))
7307	IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_WRMSR, cbInstr);
7308	}
7309	#endif
7310	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
7311	else if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_MSR_PROT))
7312	{
7313	VBOXSTRICTRC rcStrict = iemSvmHandleMsrIntercept(pVCpu, idMsr, true /* fWrite */, cbInstr);
7314	if (rcStrict == VINF_SVM_VMEXIT)
7315	return VINF_SUCCESS;
7316	if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE)
7317	{
7318	Log(("IEM: SVM intercepted rdmsr(%#x) failed. rc=%Rrc\n", idMsr, VBOXSTRICTRC_VAL(rcStrict)));
7319	return rcStrict;
7320	}
7321	}
7322	#endif
7323
7324	if (idMsr == MSR_K6_EFER)
7325	IEMTLBTRACE_LOAD_EFER(pVCpu, uValue.u, pVCpu->cpum.GstCtx.msrEFER);
7326
7327	/*
7328	* Do the job.
7329	*/
7330	VBOXSTRICTRC rcStrict = CPUMSetGuestMsr(pVCpu, idMsr, uValue.u);
7331	if (rcStrict == VINF_SUCCESS)
7332	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7333
7334	#ifndef IN_RING3
7335	/* Deferred to ring-3. */
7336	if (rcStrict == VINF_CPUM_R3_MSR_WRITE)
7337	{
7338	Log(("IEM: wrmsr(%#x) -> ring-3\n", idMsr));
7339	return rcStrict;
7340	}
7341	#endif
7342
7343	/* Often a unimplemented MSR or MSR bit, so worth logging. */
7344	if (pVCpu->iem.s.cLogRelWrMsr < 32)
7345	{
7346	pVCpu->iem.s.cLogRelWrMsr++;
7347	LogRel(("IEM: wrmsr(%#x,%#x`%08x) -> #GP(0)\n", idMsr, uValue.s.Hi, uValue.s.Lo));
7348	}
7349	else
7350	Log(( "IEM: wrmsr(%#x,%#x`%08x) -> #GP(0)\n", idMsr, uValue.s.Hi, uValue.s.Lo));
7351	AssertMsgReturn(rcStrict == VERR_CPUM_RAISE_GP_0, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), VERR_IPE_UNEXPECTED_STATUS);
7352	return iemRaiseGeneralProtectionFault0(pVCpu);
7353	}
7354
7355
7356	/**
7357	* Implements 'IN eAX, port'.
7358	*
7359	* @param u16Port The source port.
7360	* @param cbReg The register size.
7361	* @param bImmAndEffAddrMode Bit 7: Whether the port was specified through an
7362	* immediate operand or the implicit DX register.
7363	* Bits 3-0: Effective address mode.
7364	*/
7365	IEM_CIMPL_DEF_3(iemCImpl_in, uint16_t, u16Port, uint8_t, cbReg, uint8_t, bImmAndEffAddrMode)
7366	{
7367	/*
7368	* GCM intercept.
7369	*
7370	* This must be placed before the IOPL check as the mesa driver intercept
7371	* would otherwise trigger a #GP(0).
7372	*/
7373	if (!IEM_IS_IN_GUEST(pVCpu) && GCMIsInterceptingIOPortRead(pVCpu, u16Port, cbReg))
7374	{
7375	VBOXSTRICTRC rcStrict = GCMInterceptedIOPortRead(pVCpu, &pVCpu->cpum.GstCtx, u16Port, cbReg);
7376	if (rcStrict == VINF_GCM_HANDLED_ADVANCE_RIP \|\| rcStrict == VINF_GCM_HANDLED)
7377	{
7378	Log(("iemCImpl_in: u16Port=%#x cbReg=%d was handled by GCMIOPortRead (%d)\n", u16Port, cbReg, VBOXSTRICTRC_VAL(rcStrict)));
7379	if (rcStrict == VINF_GCM_HANDLED_ADVANCE_RIP)
7380	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7381	else
7382	rcStrict = VINF_SUCCESS;
7383	return rcStrict;
7384	}
7385	Assert(rcStrict == VERR_GCM_NOT_HANDLED);
7386	}
7387
7388	/*
7389	* CPL check
7390	*/
7391	VBOXSTRICTRC rcStrict = iemHlpCheckPortIOPermission(pVCpu, u16Port, cbReg);
7392	if (rcStrict != VINF_SUCCESS)
7393	return rcStrict;
7394
7395	if (!IEM_IS_IN_GUEST(pVCpu))
7396	{ /* probable */ }
7397
7398	/*
7399	* Check VMX nested-guest IO intercept.
7400	*/
7401	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7402	else if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
7403	{
7404	rcStrict = iemVmxVmexitInstrIo(pVCpu, VMXINSTRID_IO_IN, u16Port, RT_BOOL(bImmAndEffAddrMode & 0x80), cbReg, cbInstr);
7405	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
7406	return rcStrict;
7407	}
7408	#endif
7409
7410	/*
7411	* Check SVM nested-guest IO intercept.
7412	*/
7413	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
7414	else if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IOIO_PROT))
7415	{
7416	uint8_t cAddrSizeBits;
7417	switch (bImmAndEffAddrMode & 0xf)
7418	{
7419	case IEMMODE_16BIT: cAddrSizeBits = 16; break;
7420	case IEMMODE_32BIT: cAddrSizeBits = 32; break;
7421	case IEMMODE_64BIT: cAddrSizeBits = 64; break;
7422	IEM_NOT_REACHED_DEFAULT_CASE_RET();
7423	}
7424	rcStrict = iemSvmHandleIOIntercept(pVCpu, u16Port, SVMIOIOTYPE_IN, cbReg, cAddrSizeBits, 0 /* N/A - iEffSeg */,
7425	false /* fRep /, false / fStrIo */, cbInstr);
7426	if (rcStrict == VINF_SVM_VMEXIT)
7427	return VINF_SUCCESS;
7428	if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE)
7429	{
7430	Log(("iemCImpl_in: iemSvmHandleIOIntercept failed (u16Port=%#x, cbReg=%u) rc=%Rrc\n", u16Port, cbReg,
7431	VBOXSTRICTRC_VAL(rcStrict)));
7432	return rcStrict;
7433	}
7434	}
7435	#endif
7436	#if !defined(VBOX_WITH_NESTED_HWVIRT_VMX) && !defined(VBOX_WITH_NESTED_HWVIRT_SVM)
7437	RT_NOREF(bImmAndEffAddrMode);
7438	#endif
7439
7440	/*
7441	* Perform the I/O.
7442	*/
7443	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
7444	uint32_t u32Value = 0;
7445	rcStrict = IOMIOPortRead(pVM, pVCpu, u16Port, &u32Value, cbReg);
7446	if (IOM_SUCCESS(rcStrict))
7447	{
7448	switch (cbReg)
7449	{
7450	case 1: pVCpu->cpum.GstCtx.al = (uint8_t)u32Value; break;
7451	case 2: pVCpu->cpum.GstCtx.ax = (uint16_t)u32Value; break;
7452	case 4: pVCpu->cpum.GstCtx.rax = u32Value; break;
7453	default: AssertFailedReturn(VERR_IEM_IPE_3);
7454	}
7455
7456	pVCpu->iem.s.cPotentialExits++;
7457	if (rcStrict != VINF_SUCCESS)
7458	iemSetPassUpStatus(pVCpu, rcStrict);
7459
7460	/*
7461	* Check for I/O breakpoints before we complete the instruction.
7462	*/
7463	uint32_t const fDr7 = pVCpu->cpum.GstCtx.dr[7];
7464	if (RT_UNLIKELY( ( ( (fDr7 & X86_DR7_ENABLED_MASK)
7465	&& X86_DR7_ANY_RW_IO(fDr7)
7466	&& (pVCpu->cpum.GstCtx.cr4 & X86_CR4_DE))
7467	\|\| pVM->dbgf.ro.cEnabledHwIoBreakpoints > 0)
7468	&& rcStrict == VINF_SUCCESS))
7469	{
7470	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3 \| CPUMCTX_EXTRN_DR6);
7471	pVCpu->cpum.GstCtx.eflags.uBoth \|= DBGFBpCheckIo2(pVM, pVCpu, u16Port, cbReg);
7472	}
7473
7474	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7475	}
7476
7477	return rcStrict;
7478	}
7479
7480
7481	/**
7482	* Implements 'IN eAX, DX'.
7483	*
7484	* @param cbReg The register size.
7485	* @param enmEffAddrMode Effective address mode.
7486	*/
7487	IEM_CIMPL_DEF_2(iemCImpl_in_eAX_DX, uint8_t, cbReg, IEMMODE, enmEffAddrMode)
7488	{
7489	return IEM_CIMPL_CALL_3(iemCImpl_in, pVCpu->cpum.GstCtx.dx, cbReg, 0 /* fImm */ \| enmEffAddrMode);
7490	}
7491
7492
7493	/**
7494	* Implements 'OUT port, eAX'.
7495	*
7496	* @param u16Port The destination port.
7497	* @param cbReg The register size.
7498	* @param bImmAndEffAddrMode Bit 7: Whether the port was specified through an
7499	* immediate operand or the implicit DX register.
7500	* Bits 3-0: Effective address mode.
7501	*/
7502	IEM_CIMPL_DEF_3(iemCImpl_out, uint16_t, u16Port, uint8_t, cbReg, uint8_t, bImmAndEffAddrMode)
7503	{
7504	/*
7505	* CPL check
7506	*/
7507	VBOXSTRICTRC rcStrict = iemHlpCheckPortIOPermission(pVCpu, u16Port, cbReg);
7508	if (rcStrict != VINF_SUCCESS)
7509	return rcStrict;
7510
7511	if (!IEM_IS_IN_GUEST(pVCpu))
7512	{ /* probable */ }
7513
7514	/*
7515	* Check VMX nested-guest I/O intercept.
7516	*/
7517	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7518	else if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
7519	{
7520	rcStrict = iemVmxVmexitInstrIo(pVCpu, VMXINSTRID_IO_OUT, u16Port, RT_BOOL(bImmAndEffAddrMode & 0x80), cbReg, cbInstr);
7521	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
7522	return rcStrict;
7523	}
7524	#endif
7525
7526	/*
7527	* Check SVM nested-guest I/O intercept.
7528	*/
7529	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
7530	else if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IOIO_PROT))
7531	{
7532	uint8_t cAddrSizeBits;
7533	switch (bImmAndEffAddrMode & 0xf)
7534	{
7535	case IEMMODE_16BIT: cAddrSizeBits = 16; break;
7536	case IEMMODE_32BIT: cAddrSizeBits = 32; break;
7537	case IEMMODE_64BIT: cAddrSizeBits = 64; break;
7538	IEM_NOT_REACHED_DEFAULT_CASE_RET();
7539	}
7540	rcStrict = iemSvmHandleIOIntercept(pVCpu, u16Port, SVMIOIOTYPE_OUT, cbReg, cAddrSizeBits, 0 /* N/A - iEffSeg */,
7541	false /* fRep /, false / fStrIo */, cbInstr);
7542	if (rcStrict == VINF_SVM_VMEXIT)
7543	return VINF_SUCCESS;
7544	if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE)
7545	{
7546	Log(("iemCImpl_out: iemSvmHandleIOIntercept failed (u16Port=%#x, cbReg=%u) rc=%Rrc\n", u16Port, cbReg,
7547	VBOXSTRICTRC_VAL(rcStrict)));
7548	return rcStrict;
7549	}
7550	}
7551	#endif
7552	#if !defined(VBOX_WITH_NESTED_HWVIRT_VMX) && !defined(VBOX_WITH_NESTED_HWVIRT_SVM)
7553	RT_NOREF(bImmAndEffAddrMode);
7554	#endif
7555
7556	/*
7557	* Perform the I/O.
7558	*/
7559	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
7560	uint32_t u32Value;
7561	switch (cbReg)
7562	{
7563	case 1: u32Value = pVCpu->cpum.GstCtx.al; break;
7564	case 2: u32Value = pVCpu->cpum.GstCtx.ax; break;
7565	case 4: u32Value = pVCpu->cpum.GstCtx.eax; break;
7566	default: AssertFailedReturn(VERR_IEM_IPE_4);
7567	}
7568	rcStrict = IOMIOPortWrite(pVM, pVCpu, u16Port, u32Value, cbReg);
7569	if (IOM_SUCCESS(rcStrict))
7570	{
7571	pVCpu->iem.s.cPotentialExits++;
7572	if (rcStrict != VINF_SUCCESS)
7573	iemSetPassUpStatus(pVCpu, rcStrict);
7574
7575	/*
7576	* Check for I/O breakpoints before we complete the instruction.
7577	*/
7578	uint32_t const fDr7 = pVCpu->cpum.GstCtx.dr[7];
7579	if (RT_UNLIKELY( ( ( (fDr7 & X86_DR7_ENABLED_MASK)
7580	&& X86_DR7_ANY_RW_IO(fDr7)
7581	&& (pVCpu->cpum.GstCtx.cr4 & X86_CR4_DE))
7582	\|\| pVM->dbgf.ro.cEnabledHwIoBreakpoints > 0)
7583	&& rcStrict == VINF_SUCCESS))
7584	{
7585	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3 \| CPUMCTX_EXTRN_DR6);
7586	pVCpu->cpum.GstCtx.eflags.uBoth \|= DBGFBpCheckIo2(pVM, pVCpu, u16Port, cbReg);
7587	}
7588
7589	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7590	}
7591	return rcStrict;
7592	}
7593
7594
7595	/**
7596	* Implements 'OUT DX, eAX'.
7597	*
7598	* @param cbReg The register size.
7599	* @param enmEffAddrMode Effective address mode.
7600	*/
7601	IEM_CIMPL_DEF_2(iemCImpl_out_DX_eAX, uint8_t, cbReg, IEMMODE, enmEffAddrMode)
7602	{
7603	return IEM_CIMPL_CALL_3(iemCImpl_out, pVCpu->cpum.GstCtx.dx, cbReg, 0 /* fImm */ \| enmEffAddrMode);
7604	}
7605
7606
7607	/**
7608	* Implements 'CLI'.
7609	*/
7610	IEM_CIMPL_DEF_0(iemCImpl_cli)
7611	{
7612	uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
7613	#ifdef LOG_ENABLED
7614	uint32_t const fEflOld = fEfl;
7615	#endif
7616
7617	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4);
7618	if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)
7619	{
7620	uint8_t const uIopl = X86_EFL_GET_IOPL(fEfl);
7621	if (!(fEfl & X86_EFL_VM))
7622	{
7623	if (IEM_GET_CPL(pVCpu) <= uIopl)
7624	fEfl &= ~X86_EFL_IF;
7625	else if ( IEM_GET_CPL(pVCpu) == 3
7626	&& (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PVI) )
7627	fEfl &= ~X86_EFL_VIF;
7628	else
7629	return iemRaiseGeneralProtectionFault0(pVCpu);
7630	}
7631	/* V8086 */
7632	else if (uIopl == 3)
7633	fEfl &= ~X86_EFL_IF;
7634	else if ( uIopl < 3
7635	&& (pVCpu->cpum.GstCtx.cr4 & X86_CR4_VME) )
7636	fEfl &= ~X86_EFL_VIF;
7637	else
7638	return iemRaiseGeneralProtectionFault0(pVCpu);
7639	}
7640	/* real mode */
7641	else
7642	fEfl &= ~X86_EFL_IF;
7643
7644	/* Commit. */
7645	IEMMISC_SET_EFL(pVCpu, fEfl);
7646	VBOXSTRICTRC const rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7647	Log2(("CLI: %#x -> %#x\n", fEflOld, fEfl));
7648	return rcStrict;
7649	}
7650
7651
7652	/**
7653	* Implements 'STI'.
7654	*/
7655	IEM_CIMPL_DEF_0(iemCImpl_sti)
7656	{
7657	uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
7658	uint32_t const fEflOld = fEfl;
7659
7660	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4);
7661	if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)
7662	{
7663	uint8_t const uIopl = X86_EFL_GET_IOPL(fEfl);
7664	if (!(fEfl & X86_EFL_VM))
7665	{
7666	if (IEM_GET_CPL(pVCpu) <= uIopl)
7667	fEfl \|= X86_EFL_IF;
7668	else if ( IEM_GET_CPL(pVCpu) == 3
7669	&& (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PVI)
7670	&& !(fEfl & X86_EFL_VIP) )
7671	fEfl \|= X86_EFL_VIF;
7672	else
7673	return iemRaiseGeneralProtectionFault0(pVCpu);
7674	}
7675	/* V8086 */
7676	else if (uIopl == 3)
7677	fEfl \|= X86_EFL_IF;
7678	else if ( uIopl < 3
7679	&& (pVCpu->cpum.GstCtx.cr4 & X86_CR4_VME)
7680	&& !(fEfl & X86_EFL_VIP) )
7681	fEfl \|= X86_EFL_VIF;
7682	else
7683	return iemRaiseGeneralProtectionFault0(pVCpu);
7684	}
7685	/* real mode */
7686	else
7687	fEfl \|= X86_EFL_IF;
7688
7689	/*
7690	* Commit.
7691	*
7692	* Note! Setting the shadow interrupt flag must be done after RIP updating.
7693	*/
7694	IEMMISC_SET_EFL(pVCpu, fEfl);
7695	VBOXSTRICTRC const rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7696	if (!(fEflOld & X86_EFL_IF) && (fEfl & X86_EFL_IF))
7697	{
7698	/** @todo only set it the shadow flag if it was clear before? */
7699	CPUMSetInInterruptShadowSti(&pVCpu->cpum.GstCtx);
7700	}
7701	pVCpu->iem.s.fTbCurInstrIsSti = true;
7702	Log2(("STI: %#x -> %#x\n", fEflOld, fEfl));
7703	return rcStrict;
7704	}
7705
7706
7707	/**
7708	* Implements 'HLT'.
7709	*/
7710	IEM_CIMPL_DEF_0(iemCImpl_hlt)
7711	{
7712	if (IEM_GET_CPL(pVCpu) != 0)
7713	return iemRaiseGeneralProtectionFault0(pVCpu);
7714
7715	if (!IEM_IS_IN_GUEST(pVCpu))
7716	{ /* probable */ }
7717	else if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
7718	&& IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_HLT_EXIT))
7719	{
7720	Log2(("hlt: Guest intercept -> VM-exit\n"));
7721	IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_HLT, cbInstr);
7722	}
7723	else if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_HLT))
7724	{
7725	Log2(("hlt: Guest intercept -> #VMEXIT\n"));
7726	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
7727	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_HLT, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
7728	}
7729
7730	/** @todo finish: This ASSUMES that iemRegAddToRipAndFinishingClearingRF won't
7731	* be returning any status codes relating to non-guest events being raised, as
7732	* we'll mess up the guest HALT otherwise. */
7733	VBOXSTRICTRC rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7734	if (rcStrict == VINF_SUCCESS)
7735	rcStrict = VINF_EM_HALT;
7736	return rcStrict;
7737	}
7738
7739
7740	/**
7741	* Implements 'MONITOR'.
7742	*/
7743	IEM_CIMPL_DEF_1(iemCImpl_monitor, uint8_t, iEffSeg)
7744	{
7745	/*
7746	* Permission checks.
7747	*/
7748	if (IEM_GET_CPL(pVCpu) != 0)
7749	{
7750	Log2(("monitor: CPL != 0\n"));
7751	return iemRaiseUndefinedOpcode(pVCpu); /** @todo MSR[0xC0010015].MonMwaitUserEn if we care. */
7752	}
7753	if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMonitorMWait)
7754	{
7755	Log2(("monitor: Not in CPUID\n"));
7756	return iemRaiseUndefinedOpcode(pVCpu);
7757	}
7758
7759	/*
7760	* Check VMX guest-intercept.
7761	* This should be considered a fault-like VM-exit.
7762	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
7763	*/
7764	if ( !IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
7765	\|\| !IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_MONITOR_EXIT))
7766	{ /* probable */ }
7767	else
7768	{
7769	Log2(("monitor: Guest intercept -> #VMEXIT\n"));
7770	IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_MONITOR, cbInstr);
7771	}
7772
7773	/*
7774	* Gather the operands and validate them.
7775	*/
7776	RTGCPTR GCPtrMem = IEM_IS_64BIT_CODE(pVCpu) ? pVCpu->cpum.GstCtx.rax : pVCpu->cpum.GstCtx.eax;
7777	uint32_t uEcx = pVCpu->cpum.GstCtx.ecx;
7778	uint32_t uEdx = pVCpu->cpum.GstCtx.edx;
7779	/** @todo Test whether EAX or ECX is processed first, i.e. do we get \#PF or
7780	* \#GP first. */
7781	if (uEcx != 0)
7782	{
7783	Log2(("monitor rax=%RX64, ecx=%RX32, edx=%RX32; ECX != 0 -> #GP(0)\n", GCPtrMem, uEcx, uEdx)); NOREF(uEdx);
7784	return iemRaiseGeneralProtectionFault0(pVCpu);
7785	}
7786
7787	VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, IEM_ACCESS_TYPE_READ \| IEM_ACCESS_WHAT_DATA, iEffSeg, 1, &GCPtrMem);
7788	if (rcStrict != VINF_SUCCESS)
7789	return rcStrict;
7790
7791	RTGCPHYS GCPhysMem;
7792	/** @todo access size */
7793	rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrMem, 1, IEM_ACCESS_TYPE_READ \| IEM_ACCESS_WHAT_DATA, &GCPhysMem);
7794	if (rcStrict != VINF_SUCCESS)
7795	return rcStrict;
7796
7797	if (!IEM_IS_IN_GUEST(pVCpu))
7798	{ /* probable */ }
7799	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7800	else if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
7801	&& IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_VIRT_APIC_ACCESS))
7802	{
7803	/*
7804	* MONITOR does not access the memory, just monitors the address. However,
7805	* if the address falls in the APIC-access page, the address monitored must
7806	* instead be the corresponding address in the virtual-APIC page.
7807	*
7808	* See Intel spec. 29.4.4 "Instruction-Specific Considerations".
7809	*/
7810	rcStrict = iemVmxVirtApicAccessUnused(pVCpu, &GCPhysMem, 1, IEM_ACCESS_TYPE_READ \| IEM_ACCESS_WHAT_DATA);
7811	if ( rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE
7812	&& rcStrict != VINF_VMX_MODIFIES_BEHAVIOR)
7813	return rcStrict;
7814	}
7815	#endif
7816	else if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_MONITOR))
7817	{
7818	Log2(("monitor: Guest intercept -> #VMEXIT\n"));
7819	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
7820	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_MONITOR, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
7821	}
7822
7823	/*
7824	* Call EM to prepare the monitor/wait.
7825	*/
7826	rcStrict = EMMonitorWaitPrepare(pVCpu, pVCpu->cpum.GstCtx.rax, pVCpu->cpum.GstCtx.rcx, pVCpu->cpum.GstCtx.rdx, GCPhysMem);
7827	Assert(rcStrict == VINF_SUCCESS);
7828	if (rcStrict == VINF_SUCCESS)
7829	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7830	return rcStrict;
7831	}
7832
7833
7834	/**
7835	* Implements 'MWAIT'.
7836	*/
7837	IEM_CIMPL_DEF_0(iemCImpl_mwait)
7838	{
7839	/*
7840	* Permission checks.
7841	*/
7842	if (IEM_GET_CPL(pVCpu) != 0)
7843	{
7844	Log2(("mwait: CPL != 0\n"));
7845	/** @todo MSR[0xC0010015].MonMwaitUserEn if we care. (Remember to check
7846	* EFLAGS.VM then.) */
7847	return iemRaiseUndefinedOpcode(pVCpu);
7848	}
7849	if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMonitorMWait)
7850	{
7851	Log2(("mwait: Not in CPUID\n"));
7852	return iemRaiseUndefinedOpcode(pVCpu);
7853	}
7854
7855	/* Check VMX nested-guest intercept. */
7856	if ( !IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
7857	\|\| !IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_MWAIT_EXIT))
7858	{ /* probable */ }
7859	else
7860	IEM_VMX_VMEXIT_MWAIT_RET(pVCpu, EMMonitorIsArmed(pVCpu), cbInstr);
7861
7862	/*
7863	* Gather the operands and validate them.
7864	*/
7865	uint32_t const uEax = pVCpu->cpum.GstCtx.eax;
7866	uint32_t const uEcx = pVCpu->cpum.GstCtx.ecx;
7867	if (uEcx != 0)
7868	{
7869	/* Only supported extension is break on IRQ when IF=0. */
7870	if (uEcx > 1)
7871	{
7872	Log2(("mwait eax=%RX32, ecx=%RX32; ECX > 1 -> #GP(0)\n", uEax, uEcx));
7873	return iemRaiseGeneralProtectionFault0(pVCpu);
7874	}
7875	uint32_t fMWaitFeatures = 0;
7876	uint32_t uIgnore = 0;
7877	CPUMGetGuestCpuId(pVCpu, 5, 0, -1 /f64BitMode/, &uIgnore, &uIgnore, &fMWaitFeatures, &uIgnore);
7878	if ( (fMWaitFeatures & (X86_CPUID_MWAIT_ECX_EXT \| X86_CPUID_MWAIT_ECX_BREAKIRQIF0))
7879	!= (X86_CPUID_MWAIT_ECX_EXT \| X86_CPUID_MWAIT_ECX_BREAKIRQIF0))
7880	{
7881	Log2(("mwait eax=%RX32, ecx=%RX32; break-on-IRQ-IF=0 extension not enabled -> #GP(0)\n", uEax, uEcx));
7882	return iemRaiseGeneralProtectionFault0(pVCpu);
7883	}
7884
7885	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7886	/*
7887	* If the interrupt-window exiting control is set or a virtual-interrupt is pending
7888	* for delivery; and interrupts are disabled the processor does not enter its
7889	* mwait state but rather passes control to the next instruction.
7890	*
7891	* See Intel spec. 25.3 "Changes to Instruction Behavior In VMX Non-root Operation".
7892	*/
7893	if ( !IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
7894	\|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF)
7895	{ /* probable */ }
7896	else if ( IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_INT_WINDOW_EXIT)
7897	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST))
7898	/** @todo finish: check up this out after we move int window stuff out of the
7899	* run loop and into the instruction finishing logic here. */
7900	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7901	#endif
7902	}
7903
7904	/*
7905	* Check SVM nested-guest mwait intercepts.
7906	*/
7907	if (!IEM_IS_IN_GUEST(pVCpu))
7908	{ /* probable */ }
7909	else if ( IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_MWAIT_ARMED)
7910	&& EMMonitorIsArmed(pVCpu))
7911	{
7912	Log2(("mwait: Guest intercept (monitor hardware armed) -> #VMEXIT\n"));
7913	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
7914	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_MWAIT_ARMED, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
7915	}
7916	else if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_MWAIT))
7917	{
7918	Log2(("mwait: Guest intercept -> #VMEXIT\n"));
7919	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
7920	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_MWAIT, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
7921	}
7922
7923	/*
7924	* Call EM to prepare the monitor/wait.
7925	*
7926	* This will return VINF_EM_HALT. If there the trap flag is set, we may
7927	* override it when executing iemRegAddToRipAndFinishingClearingRF ASSUMING
7928	* that will only return guest related events.
7929	*/
7930	VBOXSTRICTRC rcStrict = EMMonitorWaitPerform(pVCpu, uEax, uEcx);
7931
7932	/** @todo finish: This needs more thinking as we should suppress internal
7933	* debugger events here, or we'll bugger up the guest state even more than we
7934	* alread do around VINF_EM_HALT. */
7935	VBOXSTRICTRC rcStrict2 = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7936	if (rcStrict2 != VINF_SUCCESS)
7937	{
7938	Log2(("mwait: %Rrc (perform) -> %Rrc (finish)!\n", VBOXSTRICTRC_VAL(rcStrict), VBOXSTRICTRC_VAL(rcStrict2) ));
7939	rcStrict = rcStrict2;
7940	}
7941
7942	return rcStrict;
7943	}
7944
7945
7946	/**
7947	* Implements 'SWAPGS'.
7948	*/
7949	IEM_CIMPL_DEF_0(iemCImpl_swapgs)
7950	{
7951	Assert(IEM_IS_64BIT_CODE(pVCpu)); /* Caller checks this. */
7952
7953	/*
7954	* Permission checks.
7955	*/
7956	if (IEM_GET_CPL(pVCpu) != 0)
7957	{
7958	Log2(("swapgs: CPL != 0\n"));
7959	return iemRaiseUndefinedOpcode(pVCpu);
7960	}
7961
7962	/*
7963	* Do the job.
7964	*/
7965	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_GS);
7966	uint64_t uOtherGsBase = pVCpu->cpum.GstCtx.msrKERNELGSBASE;
7967	pVCpu->cpum.GstCtx.msrKERNELGSBASE = pVCpu->cpum.GstCtx.gs.u64Base;
7968	pVCpu->cpum.GstCtx.gs.u64Base = uOtherGsBase;
7969
7970	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
7971	}
7972
7973
7974	#ifndef VBOX_WITHOUT_CPUID_HOST_CALL
7975	/**
7976	* Handles a CPUID call.
7977	*/
7978	static VBOXSTRICTRC iemCpuIdVBoxCall(PVMCPUCC pVCpu, uint32_t iFunction,
7979	uint32_t pEax, uint32_t pEbx, uint32_t pEcx, uint32_t pEdx)
7980	{
7981	switch (iFunction)
7982	{
7983	case VBOX_CPUID_FN_ID:
7984	LogFlow(("iemCpuIdVBoxCall: VBOX_CPUID_FN_ID\n"));
7985	*pEax = VBOX_CPUID_RESP_ID_EAX;
7986	*pEbx = VBOX_CPUID_RESP_ID_EBX;
7987	*pEcx = VBOX_CPUID_RESP_ID_ECX;
7988	*pEdx = VBOX_CPUID_RESP_ID_EDX;
7989	break;
7990
7991	case VBOX_CPUID_FN_LOG:
7992	{
7993	CPUM_IMPORT_EXTRN_RET(pVCpu, CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RBX \| CPUMCTX_EXTRN_RSI
7994	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
7995
7996	/* Validate input. */
7997	uint32_t cchToLog = *pEdx;
7998	if (cchToLog <= _2M)
7999	{
8000	uint32_t const uLogPicker = *pEbx;
8001	if (uLogPicker <= 1)
8002	{
8003	/* Resolve the logger. */
8004	PRTLOGGER const pLogger = !uLogPicker
8005	? RTLogDefaultInstanceEx(UINT32_MAX) : RTLogRelGetDefaultInstanceEx(UINT32_MAX);
8006	if (pLogger)
8007	{
8008	/* Copy over the data: */
8009	RTGCPTR GCPtrSrc = pVCpu->cpum.GstCtx.rsi;
8010	while (cchToLog > 0)
8011	{
8012	uint32_t cbToMap = GUEST_PAGE_SIZE - (GCPtrSrc & GUEST_PAGE_OFFSET_MASK);
8013	if (cbToMap > cchToLog)
8014	cbToMap = cchToLog;
8015	/** @todo Extend iemMemMap to allowing page size accessing and avoid 7
8016	* unnecessary calls & iterations per pages. */
8017	if (cbToMap > 512)
8018	cbToMap = 512;
8019	uint8_t bUnmapInfo;
8020	void *pvSrc = NULL;
8021	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvSrc, &bUnmapInfo, cbToMap,
8022	UINT8_MAX, GCPtrSrc, IEM_ACCESS_DATA_R, 0);
8023	if (rcStrict == VINF_SUCCESS)
8024	{
8025	RTLogBulkNestedWrite(pLogger, (const char *)pvSrc, cbToMap, "Gst:");
8026	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
8027	AssertRCSuccessReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
8028	}
8029	else
8030	{
8031	Log(("iemCpuIdVBoxCall: %Rrc at %RGp LB %#x\n", VBOXSTRICTRC_VAL(rcStrict), GCPtrSrc, cbToMap));
8032	return rcStrict;
8033	}
8034
8035	/* Advance. */
8036	pVCpu->cpum.GstCtx.rsi = GCPtrSrc += cbToMap;
8037	*pEdx = cchToLog -= cbToMap;
8038	}
8039	*pEax = VINF_SUCCESS;
8040	}
8041	else
8042	*pEax = (uint32_t)VERR_NOT_FOUND;
8043	}
8044	else
8045	*pEax = (uint32_t)VERR_NOT_FOUND;
8046	}
8047	else
8048	*pEax = (uint32_t)VERR_TOO_MUCH_DATA;
8049	*pEdx = VBOX_CPUID_RESP_GEN_EDX;
8050	*pEcx = VBOX_CPUID_RESP_GEN_ECX;
8051	*pEbx = VBOX_CPUID_RESP_GEN_EBX;
8052	break;
8053	}
8054
8055	default:
8056	LogFlow(("iemCpuIdVBoxCall: Invalid function %#x (%#x, %#x)\n", iFunction, pEbx, pEdx));
8057	*pEax = (uint32_t)VERR_INVALID_FUNCTION;
8058	*pEbx = (uint32_t)VERR_INVALID_FUNCTION;
8059	*pEcx = (uint32_t)VERR_INVALID_FUNCTION;
8060	*pEdx = (uint32_t)VERR_INVALID_FUNCTION;
8061	break;
8062	}
8063	return VINF_SUCCESS;
8064	}
8065	#endif /* VBOX_WITHOUT_CPUID_HOST_CALL */
8066
8067	/**
8068	* Implements 'CPUID'.
8069	*/
8070	IEM_CIMPL_DEF_0(iemCImpl_cpuid)
8071	{
8072	if (!IEM_IS_IN_GUEST(pVCpu))
8073	{ /* probable */ }
8074	else if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
8075	{
8076	Log2(("cpuid: Guest intercept -> VM-exit\n"));
8077	IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_CPUID, cbInstr);
8078	}
8079	else if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_CPUID))
8080	{
8081	Log2(("cpuid: Guest intercept -> #VMEXIT\n"));
8082	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
8083	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_CPUID, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
8084	}
8085
8086
8087	uint32_t const uEax = pVCpu->cpum.GstCtx.eax;
8088	uint32_t const uEcx = pVCpu->cpum.GstCtx.ecx;
8089
8090	#ifndef VBOX_WITHOUT_CPUID_HOST_CALL
8091	/*
8092	* CPUID host call backdoor.
8093	*/
8094	if ( uEax == VBOX_CPUID_REQ_EAX_FIXED
8095	&& (uEcx & VBOX_CPUID_REQ_ECX_FIXED_MASK) == VBOX_CPUID_REQ_ECX_FIXED
8096	&& pVCpu->CTX_SUFF(pVM)->iem.s.fCpuIdHostCall)
8097	{
8098	VBOXSTRICTRC rcStrict = iemCpuIdVBoxCall(pVCpu, uEcx & VBOX_CPUID_REQ_ECX_FN_MASK,
8099	&pVCpu->cpum.GstCtx.eax, &pVCpu->cpum.GstCtx.ebx,
8100	&pVCpu->cpum.GstCtx.ecx, &pVCpu->cpum.GstCtx.edx);
8101	if (rcStrict != VINF_SUCCESS)
8102	return rcStrict;
8103	}
8104	/*
8105	* Regular CPUID.
8106	*/
8107	else
8108	#endif
8109	CPUMGetGuestCpuId(pVCpu, uEax, uEcx, pVCpu->cpum.GstCtx.cs.Attr.n.u1Long,
8110	&pVCpu->cpum.GstCtx.eax, &pVCpu->cpum.GstCtx.ebx, &pVCpu->cpum.GstCtx.ecx, &pVCpu->cpum.GstCtx.edx);
8111
8112	pVCpu->cpum.GstCtx.rax &= UINT32_C(0xffffffff);
8113	pVCpu->cpum.GstCtx.rbx &= UINT32_C(0xffffffff);
8114	pVCpu->cpum.GstCtx.rcx &= UINT32_C(0xffffffff);
8115	pVCpu->cpum.GstCtx.rdx &= UINT32_C(0xffffffff);
8116	pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RBX);
8117
8118	pVCpu->iem.s.cPotentialExits++;
8119	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8120	}
8121
8122
8123	/**
8124	* Implements 'AAD'.
8125	*
8126	* @param bImm The immediate operand.
8127	*/
8128	IEM_CIMPL_DEF_1(iemCImpl_aad, uint8_t, bImm)
8129	{
8130	uint16_t const ax = pVCpu->cpum.GstCtx.ax;
8131	uint8_t const al = (uint8_t)ax + (uint8_t)(ax >> 8) * bImm;
8132	pVCpu->cpum.GstCtx.ax = al;
8133	iemHlpUpdateArithEFlagsU8(pVCpu, al,
8134	X86_EFL_SF \| X86_EFL_ZF \| X86_EFL_PF,
8135	X86_EFL_OF \| X86_EFL_AF \| X86_EFL_CF);
8136
8137	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8138	}
8139
8140
8141	/**
8142	* Implements 'AAM'.
8143	*
8144	* @param bImm The immediate operand. Cannot be 0.
8145	*/
8146	IEM_CIMPL_DEF_1(iemCImpl_aam, uint8_t, bImm)
8147	{
8148	Assert(bImm != 0); /* #DE on 0 is handled in the decoder. */
8149
8150	uint16_t const ax = pVCpu->cpum.GstCtx.ax;
8151	uint8_t const al = (uint8_t)ax % bImm;
8152	uint8_t const ah = (uint8_t)ax / bImm;
8153	pVCpu->cpum.GstCtx.ax = (ah << 8) + al;
8154	iemHlpUpdateArithEFlagsU8(pVCpu, al,
8155	X86_EFL_SF \| X86_EFL_ZF \| X86_EFL_PF,
8156	X86_EFL_OF \| X86_EFL_AF \| X86_EFL_CF);
8157
8158	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8159	}
8160
8161
8162	/**
8163	* Implements 'DAA'.
8164	*/
8165	IEM_CIMPL_DEF_0(iemCImpl_daa)
8166	{
8167	uint8_t const al = pVCpu->cpum.GstCtx.al;
8168	bool const fCarry = pVCpu->cpum.GstCtx.eflags.Bits.u1CF;
8169
8170	if ( pVCpu->cpum.GstCtx.eflags.Bits.u1AF
8171	\|\| (al & 0xf) >= 10)
8172	{
8173	pVCpu->cpum.GstCtx.al = al + 6;
8174	pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1;
8175	}
8176	else
8177	pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0;
8178
8179	if (al >= 0x9a \|\| fCarry)
8180	{
8181	pVCpu->cpum.GstCtx.al += 0x60;
8182	pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1;
8183	}
8184	else
8185	pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0;
8186
8187	iemHlpUpdateArithEFlagsU8(pVCpu, pVCpu->cpum.GstCtx.al, X86_EFL_SF \| X86_EFL_ZF \| X86_EFL_PF, X86_EFL_OF);
8188	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8189	}
8190
8191
8192	/**
8193	* Implements 'DAS'.
8194	*/
8195	IEM_CIMPL_DEF_0(iemCImpl_das)
8196	{
8197	uint8_t const uInputAL = pVCpu->cpum.GstCtx.al;
8198	bool const fCarry = pVCpu->cpum.GstCtx.eflags.Bits.u1CF;
8199
8200	if ( pVCpu->cpum.GstCtx.eflags.Bits.u1AF
8201	\|\| (uInputAL & 0xf) >= 10)
8202	{
8203	pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1;
8204	if (uInputAL < 6)
8205	pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1;
8206	pVCpu->cpum.GstCtx.al = uInputAL - 6;
8207	}
8208	else
8209	{
8210	pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0;
8211	pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0;
8212	}
8213
8214	if (uInputAL >= 0x9a \|\| fCarry)
8215	{
8216	pVCpu->cpum.GstCtx.al -= 0x60;
8217	pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1;
8218	}
8219
8220	iemHlpUpdateArithEFlagsU8(pVCpu, pVCpu->cpum.GstCtx.al, X86_EFL_SF \| X86_EFL_ZF \| X86_EFL_PF, X86_EFL_OF);
8221	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8222	}
8223
8224
8225	/**
8226	* Implements 'AAA'.
8227	*/
8228	IEM_CIMPL_DEF_0(iemCImpl_aaa)
8229	{
8230	if (IEM_IS_GUEST_CPU_AMD(pVCpu))
8231	{
8232	if ( pVCpu->cpum.GstCtx.eflags.Bits.u1AF
8233	\|\| (pVCpu->cpum.GstCtx.ax & 0xf) >= 10)
8234	{
8235	pVCpu->cpum.GstCtx.eflags.uBoth = iemAImpl_add_u16(pVCpu->cpum.GstCtx.eflags.uBoth, &pVCpu->cpum.GstCtx.ax, 0x106);
8236	pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1;
8237	pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1;
8238	}
8239	else
8240	{
8241	iemHlpUpdateArithEFlagsU16(pVCpu, pVCpu->cpum.GstCtx.ax, X86_EFL_SF \| X86_EFL_ZF \| X86_EFL_PF, X86_EFL_OF);
8242	pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0;
8243	pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0;
8244	}
8245	pVCpu->cpum.GstCtx.ax &= UINT16_C(0xff0f);
8246	}
8247	else
8248	{
8249	if ( pVCpu->cpum.GstCtx.eflags.Bits.u1AF
8250	\|\| (pVCpu->cpum.GstCtx.ax & 0xf) >= 10)
8251	{
8252	pVCpu->cpum.GstCtx.ax += UINT16_C(0x106);
8253	pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1;
8254	pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1;
8255	}
8256	else
8257	{
8258	pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0;
8259	pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0;
8260	}
8261	pVCpu->cpum.GstCtx.ax &= UINT16_C(0xff0f);
8262	iemHlpUpdateArithEFlagsU8(pVCpu, pVCpu->cpum.GstCtx.al, X86_EFL_SF \| X86_EFL_ZF \| X86_EFL_PF, X86_EFL_OF);
8263	}
8264
8265	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8266	}
8267
8268
8269	/**
8270	* Implements 'AAS'.
8271	*/
8272	IEM_CIMPL_DEF_0(iemCImpl_aas)
8273	{
8274	if (IEM_IS_GUEST_CPU_AMD(pVCpu))
8275	{
8276	if ( pVCpu->cpum.GstCtx.eflags.Bits.u1AF
8277	\|\| (pVCpu->cpum.GstCtx.ax & 0xf) >= 10)
8278	{
8279	pVCpu->cpum.GstCtx.eflags.uBoth = iemAImpl_sub_u16(pVCpu->cpum.GstCtx.eflags.uBoth, &pVCpu->cpum.GstCtx.ax, 0x106);
8280	pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1;
8281	pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1;
8282	}
8283	else
8284	{
8285	iemHlpUpdateArithEFlagsU16(pVCpu, pVCpu->cpum.GstCtx.ax, X86_EFL_SF \| X86_EFL_ZF \| X86_EFL_PF, X86_EFL_OF);
8286	pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0;
8287	pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0;
8288	}
8289	pVCpu->cpum.GstCtx.ax &= UINT16_C(0xff0f);
8290	}
8291	else
8292	{
8293	if ( pVCpu->cpum.GstCtx.eflags.Bits.u1AF
8294	\|\| (pVCpu->cpum.GstCtx.ax & 0xf) >= 10)
8295	{
8296	pVCpu->cpum.GstCtx.ax -= UINT16_C(0x106);
8297	pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1;
8298	pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1;
8299	}
8300	else
8301	{
8302	pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0;
8303	pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0;
8304	}
8305	pVCpu->cpum.GstCtx.ax &= UINT16_C(0xff0f);
8306	iemHlpUpdateArithEFlagsU8(pVCpu, pVCpu->cpum.GstCtx.al, X86_EFL_SF \| X86_EFL_ZF \| X86_EFL_PF, X86_EFL_OF);
8307	}
8308
8309	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8310	}
8311
8312
8313	/**
8314	* Implements the 16-bit version of 'BOUND'.
8315	*
8316	* @note We have separate 16-bit and 32-bit variants of this function due to
8317	* the decoder using unsigned parameters, whereas we want signed one to
8318	* do the job. This is significant for a recompiler.
8319	*/
8320	IEM_CIMPL_DEF_3(iemCImpl_bound_16, int16_t, idxArray, int16_t, idxLowerBound, int16_t, idxUpperBound)
8321	{
8322	/*
8323	* Check if the index is inside the bounds, otherwise raise #BR.
8324	*/
8325	if ( idxArray >= idxLowerBound
8326	&& idxArray <= idxUpperBound)
8327	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8328	return iemRaiseBoundRangeExceeded(pVCpu);
8329	}
8330
8331
8332	/**
8333	* Implements the 32-bit version of 'BOUND'.
8334	*/
8335	IEM_CIMPL_DEF_3(iemCImpl_bound_32, int32_t, idxArray, int32_t, idxLowerBound, int32_t, idxUpperBound)
8336	{
8337	/*
8338	* Check if the index is inside the bounds, otherwise raise #BR.
8339	*/
8340	if ( idxArray >= idxLowerBound
8341	&& idxArray <= idxUpperBound)
8342	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8343	return iemRaiseBoundRangeExceeded(pVCpu);
8344	}
8345
8346
8347
8348	/*
8349	* Instantiate the various string operation combinations.
8350	*/
8351	#define OP_SIZE 8
8352	#define ADDR_SIZE 16
8353	#include "IEMAllCImplStrInstr-x86.cpp.h"
8354	#define OP_SIZE 8
8355	#define ADDR_SIZE 32
8356	#include "IEMAllCImplStrInstr-x86.cpp.h"
8357	#define OP_SIZE 8
8358	#define ADDR_SIZE 64
8359	#include "IEMAllCImplStrInstr-x86.cpp.h"
8360
8361	#define OP_SIZE 16
8362	#define ADDR_SIZE 16
8363	#include "IEMAllCImplStrInstr-x86.cpp.h"
8364	#define OP_SIZE 16
8365	#define ADDR_SIZE 32
8366	#include "IEMAllCImplStrInstr-x86.cpp.h"
8367	#define OP_SIZE 16
8368	#define ADDR_SIZE 64
8369	#include "IEMAllCImplStrInstr-x86.cpp.h"
8370
8371	#define OP_SIZE 32
8372	#define ADDR_SIZE 16
8373	#include "IEMAllCImplStrInstr-x86.cpp.h"
8374	#define OP_SIZE 32
8375	#define ADDR_SIZE 32
8376	#include "IEMAllCImplStrInstr-x86.cpp.h"
8377	#define OP_SIZE 32
8378	#define ADDR_SIZE 64
8379	#include "IEMAllCImplStrInstr-x86.cpp.h"
8380
8381	#define OP_SIZE 64
8382	#define ADDR_SIZE 32
8383	#include "IEMAllCImplStrInstr-x86.cpp.h"
8384	#define OP_SIZE 64
8385	#define ADDR_SIZE 64
8386	#include "IEMAllCImplStrInstr-x86.cpp.h"
8387
8388
8389	/**
8390	* Implements 'XGETBV'.
8391	*/
8392	IEM_CIMPL_DEF_0(iemCImpl_xgetbv)
8393	{
8394	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
8395	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE)
8396	{
8397	uint32_t uEcx = pVCpu->cpum.GstCtx.ecx;
8398	switch (uEcx)
8399	{
8400	case 0:
8401	break;
8402
8403	case 1: /** @todo Implement XCR1 support. */
8404	default:
8405	Log(("xgetbv ecx=%RX32 -> #GP(0)\n", uEcx));
8406	return iemRaiseGeneralProtectionFault0(pVCpu);
8407
8408	}
8409	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_XCRx);
8410	pVCpu->cpum.GstCtx.rax = RT_LO_U32(pVCpu->cpum.GstCtx.aXcr[uEcx]);
8411	pVCpu->cpum.GstCtx.rdx = RT_HI_U32(pVCpu->cpum.GstCtx.aXcr[uEcx]);
8412
8413	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8414	}
8415	Log(("xgetbv CR4.OSXSAVE=0 -> UD\n"));
8416	return iemRaiseUndefinedOpcode(pVCpu);
8417	}
8418
8419
8420	/**
8421	* Implements 'XSETBV'.
8422	*/
8423	IEM_CIMPL_DEF_0(iemCImpl_xsetbv)
8424	{
8425	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE)
8426	{
8427	/** @todo explain why this happens before the CPL check. */
8428	if (!IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_XSETBV))
8429	{ /* probable */ }
8430	else
8431	{
8432	Log2(("xsetbv: Guest intercept -> #VMEXIT\n"));
8433	IEM_SVM_UPDATE_NRIP(pVCpu, cbInstr);
8434	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_XSETBV, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
8435	}
8436
8437	if (IEM_GET_CPL(pVCpu) == 0)
8438	{
8439	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_XCRx);
8440
8441	if (!IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
8442	{ /* probable */ }
8443	else
8444	IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_XSETBV, cbInstr);
8445
8446	uint32_t uEcx = pVCpu->cpum.GstCtx.ecx;
8447	uint64_t uNewValue = RT_MAKE_U64(pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.edx);
8448	switch (uEcx)
8449	{
8450	case 0:
8451	{
8452	int rc = CPUMSetGuestXcr0(pVCpu, uNewValue);
8453	if (rc == VINF_SUCCESS)
8454	break;
8455	Assert(rc == VERR_CPUM_RAISE_GP_0);
8456	Log(("xsetbv ecx=%RX32 (newvalue=%RX64) -> #GP(0)\n", uEcx, uNewValue));
8457	return iemRaiseGeneralProtectionFault0(pVCpu);
8458	}
8459
8460	case 1: /** @todo Implement XCR1 support. */
8461	default:
8462	Log(("xsetbv ecx=%RX32 (newvalue=%RX64) -> #GP(0)\n", uEcx, uNewValue));
8463	return iemRaiseGeneralProtectionFault0(pVCpu);
8464
8465	}
8466
8467	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8468	}
8469
8470	Log(("xsetbv cpl=%u -> GP(0)\n", IEM_GET_CPL(pVCpu)));
8471	return iemRaiseGeneralProtectionFault0(pVCpu);
8472	}
8473	Log(("xsetbv CR4.OSXSAVE=0 -> UD\n"));
8474	return iemRaiseUndefinedOpcode(pVCpu);
8475	}
8476
8477	#ifndef RT_ARCH_ARM64
8478	# ifdef IN_RING3
8479
8480	/** Argument package for iemCImpl_cmpxchg16b_fallback_rendezvous_callback. */
8481	struct IEMCIMPLCX16ARGS
8482	{
8483	PRTUINT128U pu128Dst;
8484	PRTUINT128U pu128RaxRdx;
8485	PRTUINT128U pu128RbxRcx;
8486	uint32_t *pEFlags;
8487	# ifdef VBOX_STRICT
8488	uint32_t cCalls;
8489	# endif
8490	};
8491
8492	/**
8493	* @callback_method_impl{FNVMMEMTRENDEZVOUS,
8494	* Worker for iemCImpl_cmpxchg16b_fallback_rendezvous}
8495	*/
8496	static DECLCALLBACK(VBOXSTRICTRC) iemCImpl_cmpxchg16b_fallback_rendezvous_callback(PVM pVM, PVMCPUCC pVCpu, void *pvUser)
8497	{
8498	RT_NOREF(pVM, pVCpu);
8499	struct IEMCIMPLCX16ARGS pArgs = (struct IEMCIMPLCX16ARGS )pvUser;
8500	# ifdef VBOX_STRICT
8501	Assert(pArgs->cCalls == 0);
8502	pArgs->cCalls++;
8503	# endif
8504
8505	iemAImpl_cmpxchg16b_fallback(pArgs->pu128Dst, pArgs->pu128RaxRdx, pArgs->pu128RbxRcx, pArgs->pEFlags);
8506	return VINF_SUCCESS;
8507	}
8508
8509	# endif /* IN_RING3 */
8510
8511	/**
8512	* Implements 'CMPXCHG16B' fallback using rendezvous.
8513	*/
8514	IEM_CIMPL_DEF_5(iemCImpl_cmpxchg16b_fallback_rendezvous, PRTUINT128U, pu128Dst, PRTUINT128U, pu128RaxRdx,
8515	PRTUINT128U, pu128RbxRcx, uint32_t *, pEFlags, uint8_t, bUnmapInfo)
8516	{
8517	# ifdef IN_RING3
8518	struct IEMCIMPLCX16ARGS Args;
8519	Args.pu128Dst = pu128Dst;
8520	Args.pu128RaxRdx = pu128RaxRdx;
8521	Args.pu128RbxRcx = pu128RbxRcx;
8522	Args.pEFlags = pEFlags;
8523	# ifdef VBOX_STRICT
8524	Args.cCalls = 0;
8525	# endif
8526	VBOXSTRICTRC rcStrict = VMMR3EmtRendezvous(pVCpu->CTX_SUFF(pVM), VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE,
8527	iemCImpl_cmpxchg16b_fallback_rendezvous_callback, &Args);
8528	Assert(Args.cCalls == 1);
8529	if (rcStrict == VINF_SUCCESS)
8530	{
8531	/* Duplicated tail code. */
8532	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
8533	if (rcStrict == VINF_SUCCESS)
8534	{
8535	pVCpu->cpum.GstCtx.eflags.u = pEFlags; / IEM_MC_COMMIT_EFLAGS */
8536	if (!(*pEFlags & X86_EFL_ZF))
8537	{
8538	pVCpu->cpum.GstCtx.rax = pu128RaxRdx->s.Lo;
8539	pVCpu->cpum.GstCtx.rdx = pu128RaxRdx->s.Hi;
8540	}
8541	rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8542	}
8543	}
8544	return rcStrict;
8545	# else
8546	RT_NOREF(pVCpu, cbInstr, pu128Dst, pu128RaxRdx, pu128RbxRcx, pEFlags, bUnmapInfo);
8547	return VERR_IEM_ASPECT_NOT_IMPLEMENTED; /* This should get us to ring-3 for now. Should perhaps be replaced later. */
8548	# endif
8549	}
8550
8551	#endif /* RT_ARCH_ARM64 */
8552
8553	/**
8554	* Implements 'CLFLUSH' and 'CLFLUSHOPT'.
8555	*
8556	* This is implemented in C because it triggers a load like behaviour without
8557	* actually reading anything. Since that's not so common, it's implemented
8558	* here.
8559	*
8560	* @param iEffSeg The effective segment.
8561	* @param GCPtrEff The address of the image.
8562	*/
8563	IEM_CIMPL_DEF_2(iemCImpl_clflush_clflushopt, uint8_t, iEffSeg, RTGCPTR, GCPtrEff)
8564	{
8565	/*
8566	* Pretend to do a load w/o reading (see also iemCImpl_monitor and iemMemMap).
8567	*/
8568	VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, IEM_ACCESS_TYPE_READ \| IEM_ACCESS_WHAT_DATA, iEffSeg, 1, &GCPtrEff);
8569	if (rcStrict == VINF_SUCCESS)
8570	{
8571	RTGCPHYS GCPhysMem;
8572	/** @todo access size. */
8573	rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrEff, 1, IEM_ACCESS_TYPE_READ \| IEM_ACCESS_WHAT_DATA, &GCPhysMem);
8574	if (rcStrict == VINF_SUCCESS)
8575	{
8576	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8577	if ( !IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
8578	\|\| !IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_VIRT_APIC_ACCESS))
8579	{ /* probable */ }
8580	else
8581	{
8582	/*
8583	* CLFLUSH/CLFLUSHOPT does not access the memory, but flushes the cache-line
8584	* that contains the address. However, if the address falls in the APIC-access
8585	* page, the address flushed must instead be the corresponding address in the
8586	* virtual-APIC page.
8587	*
8588	* See Intel spec. 29.4.4 "Instruction-Specific Considerations".
8589	*/
8590	rcStrict = iemVmxVirtApicAccessUnused(pVCpu, &GCPhysMem, 1, IEM_ACCESS_TYPE_READ \| IEM_ACCESS_WHAT_DATA);
8591	if ( rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE
8592	&& rcStrict != VINF_VMX_MODIFIES_BEHAVIOR)
8593	return rcStrict;
8594	}
8595	#endif
8596	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8597	}
8598	}
8599
8600	return rcStrict;
8601	}
8602
8603
8604	/**
8605	* Implements 'FINIT' and 'FNINIT'.
8606	*
8607	* @param fCheckXcpts Whether to check for umasked pending exceptions or
8608	* not.
8609	*/
8610	IEM_CIMPL_DEF_1(iemCImpl_finit, bool, fCheckXcpts)
8611	{
8612	/*
8613	* Exceptions.
8614	*/
8615	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
8616	if (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS))
8617	return iemRaiseDeviceNotAvailable(pVCpu);
8618
8619	iemFpuActualizeStateForChange(pVCpu);
8620	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_X87);
8621
8622	/* FINIT: Raise #MF on pending exception(s): */
8623	if (fCheckXcpts && (pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES))
8624	return iemRaiseMathFault(pVCpu);
8625
8626	/*
8627	* Reset the state.
8628	*/
8629	PX86XSAVEAREA pXState = &pVCpu->cpum.GstCtx.XState;
8630
8631	/* Rotate the stack to account for changed TOS. */
8632	iemFpuRotateStackSetTop(&pXState->x87, 0);
8633
8634	pXState->x87.FCW = 0x37f;
8635	pXState->x87.FSW = 0;
8636	pXState->x87.FTW = 0x00; /* 0 - empty. */
8637	/** @todo Intel says the instruction and data pointers are not cleared on
8638	* 387, presume that 8087 and 287 doesn't do so either. */
8639	/** @todo test this stuff. */
8640	if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_386)
8641	{
8642	pXState->x87.FPUDP = 0;
8643	pXState->x87.DS = 0; //??
8644	pXState->x87.Rsrvd2 = 0;
8645	pXState->x87.FPUIP = 0;
8646	pXState->x87.CS = 0; //??
8647	pXState->x87.Rsrvd1 = 0;
8648	}
8649	pXState->x87.FOP = 0;
8650
8651	iemHlpUsedFpu(pVCpu);
8652	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8653	}
8654
8655
8656	/**
8657	* Implements 'FXSAVE'.
8658	*
8659	* @param iEffSeg The effective segment.
8660	* @param GCPtrEff The address of the image.
8661	* @param enmEffOpSize The operand size (only REX.W really matters).
8662	*/
8663	IEM_CIMPL_DEF_3(iemCImpl_fxsave, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize)
8664	{
8665	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX);
8666
8667	/** @todo check out bugref{1529} and AMD behaviour */
8668
8669	/*
8670	* Raise exceptions.
8671	*/
8672	if (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_TS \| X86_CR0_EM))
8673	return iemRaiseDeviceNotAvailable(pVCpu);
8674
8675	/*
8676	* Access the memory.
8677	*/
8678	uint8_t bUnmapInfo;
8679	void *pvMem512;
8680	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvMem512, &bUnmapInfo, 512,
8681	iEffSeg, GCPtrEff, IEM_ACCESS_DATA_W \| IEM_ACCESS_PARTIAL_WRITE,
8682	15 \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_GP_OR_AC);
8683	if (rcStrict != VINF_SUCCESS)
8684	return rcStrict;
8685	PX86FXSTATE pDst = (PX86FXSTATE)pvMem512;
8686	PCX86FXSTATE pSrc = &pVCpu->cpum.GstCtx.XState.x87;
8687
8688	/*
8689	* Store the registers.
8690	*/
8691	/** @todo CPU/VM detection possible! If CR4.OSFXSR=0 MXCSR it's
8692	* implementation specific whether MXCSR and XMM0-XMM7 are saved. */
8693
8694	/* common for all formats */
8695	pDst->FCW = pSrc->FCW;
8696	pDst->FSW = pSrc->FSW;
8697	pDst->FTW = pSrc->FTW & UINT16_C(0xff);
8698	pDst->FOP = pSrc->FOP;
8699	pDst->MXCSR = pSrc->MXCSR;
8700	pDst->MXCSR_MASK = CPUMGetGuestMxCsrMask(pVCpu->CTX_SUFF(pVM));
8701	for (uint32_t i = 0; i < RT_ELEMENTS(pDst->aRegs); i++)
8702	{
8703	/** @todo Testcase: What actually happens to the 6 reserved bytes? I'm clearing
8704	* them for now... */
8705	pDst->aRegs[i].au32[0] = pSrc->aRegs[i].au32[0];
8706	pDst->aRegs[i].au32[1] = pSrc->aRegs[i].au32[1];
8707	pDst->aRegs[i].au32[2] = pSrc->aRegs[i].au32[2] & UINT32_C(0xffff);
8708	pDst->aRegs[i].au32[3] = 0;
8709	}
8710
8711	/* FPU IP, CS, DP and DS. */
8712	pDst->FPUIP = pSrc->FPUIP;
8713	pDst->CS = pSrc->CS;
8714	pDst->FPUDP = pSrc->FPUDP;
8715	pDst->DS = pSrc->DS;
8716	if (enmEffOpSize == IEMMODE_64BIT)
8717	{
8718	/* Save upper 16-bits of FPUIP (IP:CS:Rsvd1) and FPUDP (DP:DS:Rsvd2). */
8719	pDst->Rsrvd1 = pSrc->Rsrvd1;
8720	pDst->Rsrvd2 = pSrc->Rsrvd2;
8721	}
8722	else
8723	{
8724	pDst->Rsrvd1 = 0;
8725	pDst->Rsrvd2 = 0;
8726	}
8727
8728	/* XMM registers. Skipped in 64-bit CPL0 if EFER.FFXSR (AMD only) is set. */
8729	if ( !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_FFXSR)
8730	\|\| !IEM_IS_64BIT_CODE(pVCpu)
8731	\|\| IEM_GET_CPL(pVCpu) != 0)
8732	{
8733	uint32_t cXmmRegs = IEM_IS_64BIT_CODE(pVCpu) ? 16 : 8;
8734	for (uint32_t i = 0; i < cXmmRegs; i++)
8735	pDst->aXMM[i] = pSrc->aXMM[i];
8736	/** @todo Testcase: What happens to the reserved XMM registers? Untouched,
8737	* right? */
8738	}
8739
8740	/*
8741	* Commit the memory.
8742	*/
8743	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
8744	if (rcStrict != VINF_SUCCESS)
8745	return rcStrict;
8746
8747	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8748	}
8749
8750
8751	/**
8752	* Implements 'FXRSTOR'.
8753	*
8754	* @param iEffSeg The effective segment register for @a GCPtrEff.
8755	* @param GCPtrEff The address of the image.
8756	* @param enmEffOpSize The operand size (only REX.W really matters).
8757	*/
8758	IEM_CIMPL_DEF_3(iemCImpl_fxrstor, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize)
8759	{
8760	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX);
8761
8762	/** @todo check out bugref{1529} and AMD behaviour */
8763
8764	/*
8765	* Raise exceptions.
8766	*/
8767	if (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_TS \| X86_CR0_EM))
8768	return iemRaiseDeviceNotAvailable(pVCpu);
8769
8770	/*
8771	* Access the memory.
8772	*/
8773	uint8_t bUnmapInfo;
8774	void *pvMem512;
8775	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvMem512, &bUnmapInfo, 512, iEffSeg, GCPtrEff, IEM_ACCESS_DATA_R,
8776	15 \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_GP_OR_AC);
8777	if (rcStrict != VINF_SUCCESS)
8778	return rcStrict;
8779	PCX86FXSTATE pSrc = (PCX86FXSTATE)pvMem512;
8780	PX86FXSTATE pDst = &pVCpu->cpum.GstCtx.XState.x87;
8781
8782	/*
8783	* Check the state for stuff which will #GP(0).
8784	*/
8785	uint32_t const fMXCSR = pSrc->MXCSR;
8786	uint32_t const fMXCSR_MASK = CPUMGetGuestMxCsrMask(pVCpu->CTX_SUFF(pVM));
8787	if (fMXCSR & ~fMXCSR_MASK)
8788	{
8789	Log(("fxrstor: MXCSR=%#x (MXCSR_MASK=%#x) -> #GP(0)\n", fMXCSR, fMXCSR_MASK));
8790	return iemRaiseGeneralProtectionFault0(pVCpu);
8791	}
8792
8793	/*
8794	* Load the registers.
8795	*/
8796	/** @todo CPU/VM detection possible! If CR4.OSFXSR=0 MXCSR it's
8797	* implementation specific whether MXCSR and XMM0-XMM7 are
8798	* restored according to Intel.
8799	* AMD says MXCSR and XMM registers are never loaded if
8800	* CR4.OSFXSR=0.
8801	*/
8802
8803	/* common for all formats */
8804	pDst->FCW = pSrc->FCW;
8805	pDst->FSW = pSrc->FSW;
8806	pDst->FTW = pSrc->FTW & UINT16_C(0xff);
8807	pDst->FOP = pSrc->FOP;
8808	pDst->MXCSR = fMXCSR;
8809	/* (MXCSR_MASK is read-only) */
8810	for (uint32_t i = 0; i < RT_ELEMENTS(pSrc->aRegs); i++)
8811	{
8812	pDst->aRegs[i].au32[0] = pSrc->aRegs[i].au32[0];
8813	pDst->aRegs[i].au32[1] = pSrc->aRegs[i].au32[1];
8814	pDst->aRegs[i].au32[2] = pSrc->aRegs[i].au32[2] & UINT32_C(0xffff);
8815	pDst->aRegs[i].au32[3] = 0;
8816	}
8817
8818	/* FPU IP, CS, DP and DS. */
8819	/** @todo AMD says this is only done if FSW.ES is set after loading. */
8820	if (enmEffOpSize == IEMMODE_64BIT)
8821	{
8822	pDst->FPUIP = pSrc->FPUIP;
8823	pDst->CS = pSrc->CS;
8824	pDst->Rsrvd1 = pSrc->Rsrvd1;
8825	pDst->FPUDP = pSrc->FPUDP;
8826	pDst->DS = pSrc->DS;
8827	pDst->Rsrvd2 = pSrc->Rsrvd2;
8828	}
8829	else
8830	{
8831	pDst->FPUIP = pSrc->FPUIP;
8832	pDst->CS = pSrc->CS;
8833	pDst->Rsrvd1 = 0;
8834	pDst->FPUDP = pSrc->FPUDP;
8835	pDst->DS = pSrc->DS;
8836	pDst->Rsrvd2 = 0;
8837	}
8838
8839	/* XMM registers. Skipped in 64-bit CPL0 if EFER.FFXSR (AMD only) is set.
8840	* Does not affect MXCSR, only registers.
8841	*/
8842	if ( !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_FFXSR)
8843	\|\| !IEM_IS_64BIT_CODE(pVCpu)
8844	\|\| IEM_GET_CPL(pVCpu) != 0)
8845	{
8846	uint32_t cXmmRegs = IEM_IS_64BIT_CODE(pVCpu) ? 16 : 8;
8847	for (uint32_t i = 0; i < cXmmRegs; i++)
8848	pDst->aXMM[i] = pSrc->aXMM[i];
8849	}
8850
8851	pDst->FCW &= ~X86_FCW_ZERO_MASK \| X86_FCW_IC_MASK; /* Intel 10980xe allows setting the IC bit. Win 3.11 CALC.EXE sets it. */
8852	iemFpuRecalcExceptionStatus(pDst);
8853
8854	if (pDst->FSW & X86_FSW_ES)
8855	Log11(("fxrstor: %04x:%08RX64: loading state with pending FPU exception (FSW=%#x)\n",
8856	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pSrc->FSW));
8857
8858	/*
8859	* Unmap the memory.
8860	*/
8861	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
8862	if (rcStrict != VINF_SUCCESS)
8863	return rcStrict;
8864
8865	iemHlpUsedFpu(pVCpu);
8866	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
8867	}
8868
8869
8870	/**
8871	* Implements 'XSAVE'.
8872	*
8873	* @param iEffSeg The effective segment.
8874	* @param GCPtrEff The address of the image.
8875	* @param enmEffOpSize The operand size (only REX.W really matters).
8876	*/
8877	IEM_CIMPL_DEF_3(iemCImpl_xsave, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize)
8878	{
8879	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
8880
8881	/*
8882	* Raise exceptions.
8883	*/
8884	if (!(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE))
8885	return iemRaiseUndefinedOpcode(pVCpu);
8886	/* When in VMX non-root mode and XSAVE/XRSTOR is not enabled, it results in #UD. */
8887	if (RT_LIKELY( !IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
8888	\|\| IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_XSAVES_XRSTORS)))
8889	{ /* likely */ }
8890	else
8891	{
8892	Log(("xrstor: Not enabled for nested-guest execution -> #UD\n"));
8893	return iemRaiseUndefinedOpcode(pVCpu);
8894	}
8895	if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS)
8896	return iemRaiseDeviceNotAvailable(pVCpu);
8897
8898	/*
8899	* Calc the requested mask.
8900	*/
8901	uint64_t const fReqComponents = RT_MAKE_U64(pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.edx) & pVCpu->cpum.GstCtx.aXcr[0];
8902	AssertLogRelReturn(!(fReqComponents & ~(XSAVE_C_X87 \| XSAVE_C_SSE \| XSAVE_C_YMM)), VERR_IEM_ASPECT_NOT_IMPLEMENTED);
8903	uint64_t const fXInUse = pVCpu->cpum.GstCtx.aXcr[0];
8904
8905	/** @todo figure out the exact protocol for the memory access. Currently we
8906	* just need this crap to work halfways to make it possible to test
8907	* AVX instructions. */
8908	/** @todo figure out the XINUSE and XMODIFIED */
8909
8910	/*
8911	* Access the x87 memory state.
8912	*/
8913	/* The x87+SSE state. */
8914	uint8_t bUnmapInfoMem512;
8915	void *pvMem512;
8916	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvMem512, &bUnmapInfoMem512, 512,
8917	iEffSeg, GCPtrEff, IEM_ACCESS_DATA_W \| IEM_ACCESS_PARTIAL_WRITE,
8918	63 \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_GP_OR_AC);
8919	if (rcStrict != VINF_SUCCESS)
8920	return rcStrict;
8921	PX86FXSTATE pDst = (PX86FXSTATE)pvMem512;
8922	PCX86FXSTATE pSrc = &pVCpu->cpum.GstCtx.XState.x87;
8923
8924	/* The header. */
8925	uint8_t bUnmapInfoHdr;
8926	PX86XSAVEHDR pHdr;
8927	rcStrict = iemMemMap(pVCpu, (void **)&pHdr, &bUnmapInfoHdr, sizeof(pHdr),
8928	iEffSeg, GCPtrEff + 512, IEM_ACCESS_DATA_RW, 0 /* checked above */);
8929	if (rcStrict != VINF_SUCCESS)
8930	return rcStrict;
8931
8932	/*
8933	* Store the X87 state.
8934	*/
8935	if (fReqComponents & XSAVE_C_X87)
8936	{
8937	/* common for all formats */
8938	pDst->FCW = pSrc->FCW;
8939	pDst->FSW = pSrc->FSW;
8940	pDst->FTW = pSrc->FTW & UINT16_C(0xff);
8941	pDst->FOP = pSrc->FOP;
8942	pDst->FPUIP = pSrc->FPUIP;
8943	pDst->CS = pSrc->CS;
8944	pDst->FPUDP = pSrc->FPUDP;
8945	pDst->DS = pSrc->DS;
8946	if (enmEffOpSize == IEMMODE_64BIT)
8947	{
8948	/* Save upper 16-bits of FPUIP (IP:CS:Rsvd1) and FPUDP (DP:DS:Rsvd2). */
8949	pDst->Rsrvd1 = pSrc->Rsrvd1;
8950	pDst->Rsrvd2 = pSrc->Rsrvd2;
8951	}
8952	else
8953	{
8954	pDst->Rsrvd1 = 0;
8955	pDst->Rsrvd2 = 0;
8956	}
8957	for (uint32_t i = 0; i < RT_ELEMENTS(pDst->aRegs); i++)
8958	{
8959	/** @todo Testcase: What actually happens to the 6 reserved bytes? I'm clearing
8960	* them for now... */
8961	pDst->aRegs[i].au32[0] = pSrc->aRegs[i].au32[0];
8962	pDst->aRegs[i].au32[1] = pSrc->aRegs[i].au32[1];
8963	pDst->aRegs[i].au32[2] = pSrc->aRegs[i].au32[2] & UINT32_C(0xffff);
8964	pDst->aRegs[i].au32[3] = 0;
8965	}
8966
8967	}
8968
8969	if (fReqComponents & (XSAVE_C_SSE \| XSAVE_C_YMM))
8970	{
8971	pDst->MXCSR = pSrc->MXCSR;
8972	pDst->MXCSR_MASK = CPUMGetGuestMxCsrMask(pVCpu->CTX_SUFF(pVM));
8973	}
8974
8975	if (fReqComponents & XSAVE_C_SSE)
8976	{
8977	/* XMM registers. */
8978	uint32_t cXmmRegs = enmEffOpSize == IEMMODE_64BIT ? 16 : 8;
8979	for (uint32_t i = 0; i < cXmmRegs; i++)
8980	pDst->aXMM[i] = pSrc->aXMM[i];
8981	/** @todo Testcase: What happens to the reserved XMM registers? Untouched,
8982	* right? */
8983	}
8984
8985	/* Commit the x87 state bits. (probably wrong) */
8986	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfoMem512);
8987	if (rcStrict != VINF_SUCCESS)
8988	return rcStrict;
8989
8990	/*
8991	* Store AVX state.
8992	*/
8993	if (fReqComponents & XSAVE_C_YMM)
8994	{
8995	/** @todo testcase: xsave64 vs xsave32 wrt XSAVE_C_YMM. */
8996	AssertLogRelReturn(pVCpu->cpum.GstCtx.aoffXState[XSAVE_C_YMM_BIT] != UINT16_MAX, VERR_IEM_IPE_9);
8997	uint8_t bUnmapInfoComp;
8998	PCX86XSAVEYMMHI pCompSrc = CPUMCTX_XSAVE_C_PTR(IEM_GET_CTX(pVCpu), XSAVE_C_YMM_BIT, PCX86XSAVEYMMHI);
8999	PX86XSAVEYMMHI pCompDst;
9000	rcStrict = iemMemMap(pVCpu, (void *)&pCompDst, &bUnmapInfoComp, sizeof(pCompDst), iEffSeg,
9001	GCPtrEff + pVCpu->cpum.GstCtx.aoffXState[XSAVE_C_YMM_BIT],
9002	IEM_ACCESS_DATA_W \| IEM_ACCESS_PARTIAL_WRITE, 0 /* checked above */);
9003	if (rcStrict != VINF_SUCCESS)
9004	return rcStrict;
9005
9006	uint32_t cXmmRegs = enmEffOpSize == IEMMODE_64BIT ? 16 : 8;
9007	for (uint32_t i = 0; i < cXmmRegs; i++)
9008	pCompDst->aYmmHi[i] = pCompSrc->aYmmHi[i];
9009
9010	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfoComp);
9011	if (rcStrict != VINF_SUCCESS)
9012	return rcStrict;
9013	}
9014
9015	/*
9016	* Update the header.
9017	*/
9018	pHdr->bmXState = (pHdr->bmXState & ~fReqComponents)
9019	\| (fReqComponents & fXInUse);
9020
9021	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfoHdr);
9022	if (rcStrict != VINF_SUCCESS)
9023	return rcStrict;
9024
9025	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
9026	}
9027
9028
9029	/**
9030	* Implements 'XRSTOR'.
9031	*
9032	* @param iEffSeg The effective segment.
9033	* @param GCPtrEff The address of the image.
9034	* @param enmEffOpSize The operand size (only REX.W really matters).
9035	*/
9036	IEM_CIMPL_DEF_3(iemCImpl_xrstor, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize)
9037	{
9038	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
9039
9040	/*
9041	* Raise exceptions.
9042	*/
9043	if (!(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE))
9044	return iemRaiseUndefinedOpcode(pVCpu);
9045	/* When in VMX non-root mode and XSAVE/XRSTOR is not enabled, it results in #UD. */
9046	if (RT_LIKELY( !IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
9047	\|\| IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_XSAVES_XRSTORS)))
9048	{ /* likely */ }
9049	else
9050	{
9051	Log(("xrstor: Not enabled for nested-guest execution -> #UD\n"));
9052	return iemRaiseUndefinedOpcode(pVCpu);
9053	}
9054	if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS)
9055	return iemRaiseDeviceNotAvailable(pVCpu);
9056	if (GCPtrEff & 63)
9057	{
9058	/** @todo CPU/VM detection possible! \#AC might not be signal for
9059	* all/any misalignment sizes, intel says its an implementation detail. */
9060	if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_AM)
9061	&& pVCpu->cpum.GstCtx.eflags.Bits.u1AC
9062	&& IEM_GET_CPL(pVCpu) == 3)
9063	return iemRaiseAlignmentCheckException(pVCpu);
9064	return iemRaiseGeneralProtectionFault0(pVCpu);
9065	}
9066
9067	/** @todo figure out the exact protocol for the memory access. Currently we
9068	* just need this crap to work halfways to make it possible to test
9069	* AVX instructions. */
9070	/** @todo figure out the XINUSE and XMODIFIED */
9071
9072	/*
9073	* Access the x87 memory state.
9074	*/
9075	/* The x87+SSE state. */
9076	uint8_t bUnmapInfoMem512;
9077	void *pvMem512;
9078	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvMem512, &bUnmapInfoMem512, 512, iEffSeg, GCPtrEff, IEM_ACCESS_DATA_R,
9079	63 \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_GP_OR_AC);
9080	if (rcStrict != VINF_SUCCESS)
9081	return rcStrict;
9082	PCX86FXSTATE pSrc = (PCX86FXSTATE)pvMem512;
9083	PX86FXSTATE pDst = &pVCpu->cpum.GstCtx.XState.x87;
9084
9085	/*
9086	* Calc the requested mask
9087	*/
9088	uint8_t bUnmapInfoHdr;
9089	PX86XSAVEHDR pHdrDst = &pVCpu->cpum.GstCtx.XState.Hdr;
9090	PCX86XSAVEHDR pHdrSrc;
9091	rcStrict = iemMemMap(pVCpu, (void *)&pHdrSrc, &bUnmapInfoHdr, sizeof(pHdrSrc), iEffSeg, GCPtrEff + 512,
9092	IEM_ACCESS_DATA_R, 0 /* checked above */);
9093	if (rcStrict != VINF_SUCCESS)
9094	return rcStrict;
9095
9096	uint64_t const fReqComponents = RT_MAKE_U64(pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.edx) & pVCpu->cpum.GstCtx.aXcr[0];
9097	AssertLogRelReturn(!(fReqComponents & ~(XSAVE_C_X87 \| XSAVE_C_SSE \| XSAVE_C_YMM)), VERR_IEM_ASPECT_NOT_IMPLEMENTED);
9098	//uint64_t const fXInUse = pVCpu->cpum.GstCtx.aXcr[0];
9099	uint64_t const fRstorMask = pHdrSrc->bmXState;
9100	uint64_t const fCompMask = pHdrSrc->bmXComp;
9101
9102	AssertLogRelReturn(!(fCompMask & XSAVE_C_X), VERR_IEM_ASPECT_NOT_IMPLEMENTED);
9103
9104	uint32_t const cXmmRegs = enmEffOpSize == IEMMODE_64BIT ? 16 : 8;
9105
9106	/* We won't need this any longer. */
9107	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfoHdr);
9108	if (rcStrict != VINF_SUCCESS)
9109	return rcStrict;
9110
9111	/*
9112	* Load the X87 state.
9113	*/
9114	if (fReqComponents & XSAVE_C_X87)
9115	{
9116	if (fRstorMask & XSAVE_C_X87)
9117	{
9118	pDst->FCW = pSrc->FCW;
9119	pDst->FSW = pSrc->FSW;
9120	pDst->FTW = pSrc->FTW & UINT16_C(0xff);
9121	pDst->FOP = pSrc->FOP;
9122	pDst->FPUIP = pSrc->FPUIP;
9123	pDst->CS = pSrc->CS;
9124	pDst->FPUDP = pSrc->FPUDP;
9125	pDst->DS = pSrc->DS;
9126	if (enmEffOpSize == IEMMODE_64BIT)
9127	{
9128	/* Load upper 16-bits of FPUIP (IP:CS:Rsvd1) and FPUDP (DP:DS:Rsvd2). */
9129	pDst->Rsrvd1 = pSrc->Rsrvd1;
9130	pDst->Rsrvd2 = pSrc->Rsrvd2;
9131	}
9132	else
9133	{
9134	pDst->Rsrvd1 = 0;
9135	pDst->Rsrvd2 = 0;
9136	}
9137	for (uint32_t i = 0; i < RT_ELEMENTS(pDst->aRegs); i++)
9138	{
9139	pDst->aRegs[i].au32[0] = pSrc->aRegs[i].au32[0];
9140	pDst->aRegs[i].au32[1] = pSrc->aRegs[i].au32[1];
9141	pDst->aRegs[i].au32[2] = pSrc->aRegs[i].au32[2] & UINT32_C(0xffff);
9142	pDst->aRegs[i].au32[3] = 0;
9143	}
9144
9145	pDst->FCW &= ~X86_FCW_ZERO_MASK \| X86_FCW_IC_MASK; /* Intel 10980xe allows setting the IC bit. Win 3.11 CALC.EXE sets it. */
9146	iemFpuRecalcExceptionStatus(pDst);
9147
9148	if (pDst->FSW & X86_FSW_ES)
9149	Log11(("xrstor: %04x:%08RX64: loading state with pending FPU exception (FSW=%#x)\n",
9150	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pSrc->FSW));
9151	}
9152	else
9153	{
9154	pDst->FCW = 0x37f;
9155	pDst->FSW = 0;
9156	pDst->FTW = 0x00; /* 0 - empty. */
9157	pDst->FPUDP = 0;
9158	pDst->DS = 0; //??
9159	pDst->Rsrvd2= 0;
9160	pDst->FPUIP = 0;
9161	pDst->CS = 0; //??
9162	pDst->Rsrvd1= 0;
9163	pDst->FOP = 0;
9164	for (uint32_t i = 0; i < RT_ELEMENTS(pSrc->aRegs); i++)
9165	{
9166	pDst->aRegs[i].au32[0] = 0;
9167	pDst->aRegs[i].au32[1] = 0;
9168	pDst->aRegs[i].au32[2] = 0;
9169	pDst->aRegs[i].au32[3] = 0;
9170	}
9171	}
9172	pHdrDst->bmXState \|= XSAVE_C_X87; /* playing safe for now */
9173	}
9174
9175	/* MXCSR */
9176	if (fReqComponents & (XSAVE_C_SSE \| XSAVE_C_YMM))
9177	{
9178	if (fRstorMask & (XSAVE_C_SSE \| XSAVE_C_YMM))
9179	pDst->MXCSR = pSrc->MXCSR;
9180	else
9181	pDst->MXCSR = 0x1f80;
9182	}
9183
9184	/* XMM registers. */
9185	if (fReqComponents & XSAVE_C_SSE)
9186	{
9187	if (fRstorMask & XSAVE_C_SSE)
9188	{
9189	for (uint32_t i = 0; i < cXmmRegs; i++)
9190	pDst->aXMM[i] = pSrc->aXMM[i];
9191	/** @todo Testcase: What happens to the reserved XMM registers? Untouched,
9192	* right? */
9193	}
9194	else
9195	{
9196	for (uint32_t i = 0; i < cXmmRegs; i++)
9197	{
9198	pDst->aXMM[i].au64[0] = 0;
9199	pDst->aXMM[i].au64[1] = 0;
9200	}
9201	}
9202	pHdrDst->bmXState \|= XSAVE_C_SSE; /* playing safe for now */
9203	}
9204
9205	/* Unmap the x87 state bits (so we've don't run out of mapping). */
9206	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfoMem512);
9207	if (rcStrict != VINF_SUCCESS)
9208	return rcStrict;
9209
9210	/*
9211	* Restore AVX state.
9212	*/
9213	if (fReqComponents & XSAVE_C_YMM)
9214	{
9215	AssertLogRelReturn(pVCpu->cpum.GstCtx.aoffXState[XSAVE_C_YMM_BIT] != UINT16_MAX, VERR_IEM_IPE_9);
9216	PX86XSAVEYMMHI pCompDst = CPUMCTX_XSAVE_C_PTR(IEM_GET_CTX(pVCpu), XSAVE_C_YMM_BIT, PX86XSAVEYMMHI);
9217
9218	if (fRstorMask & XSAVE_C_YMM)
9219	{
9220	/** @todo testcase: xsave64 vs xsave32 wrt XSAVE_C_YMM. */
9221	uint8_t bUnmapInfoComp;
9222	PCX86XSAVEYMMHI pCompSrc;
9223	rcStrict = iemMemMap(pVCpu, (void *)&pCompSrc, &bUnmapInfoComp, sizeof(pCompDst),
9224	iEffSeg, GCPtrEff + pVCpu->cpum.GstCtx.aoffXState[XSAVE_C_YMM_BIT],
9225	IEM_ACCESS_DATA_R, 0 /* checked above */);
9226	if (rcStrict != VINF_SUCCESS)
9227	return rcStrict;
9228
9229	for (uint32_t i = 0; i < cXmmRegs; i++)
9230	{
9231	pCompDst->aYmmHi[i].au64[0] = pCompSrc->aYmmHi[i].au64[0];
9232	pCompDst->aYmmHi[i].au64[1] = pCompSrc->aYmmHi[i].au64[1];
9233	}
9234
9235	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfoComp);
9236	if (rcStrict != VINF_SUCCESS)
9237	return rcStrict;
9238	}
9239	else
9240	{
9241	for (uint32_t i = 0; i < cXmmRegs; i++)
9242	{
9243	pCompDst->aYmmHi[i].au64[0] = 0;
9244	pCompDst->aYmmHi[i].au64[1] = 0;
9245	}
9246	}
9247	pHdrDst->bmXState \|= XSAVE_C_YMM; /* playing safe for now */
9248	}
9249
9250	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
9251	}
9252
9253
9254
9255
9256	/**
9257	* Implements 'STMXCSR'.
9258	*
9259	* @param iEffSeg The effective segment register for @a GCPtrEff.
9260	* @param GCPtrEff The address of the image.
9261	*/
9262	IEM_CIMPL_DEF_2(iemCImpl_stmxcsr, uint8_t, iEffSeg, RTGCPTR, GCPtrEff)
9263	{
9264	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX);
9265
9266	/*
9267	* Raise exceptions.
9268	*/
9269	if ( !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM)
9270	&& (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR))
9271	{
9272	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS))
9273	{
9274	/*
9275	* Do the job.
9276	*/
9277	VBOXSTRICTRC rcStrict = iemMemStoreDataU32(pVCpu, iEffSeg, GCPtrEff, pVCpu->cpum.GstCtx.XState.x87.MXCSR);
9278	if (rcStrict == VINF_SUCCESS)
9279	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
9280	return rcStrict;
9281	}
9282	return iemRaiseDeviceNotAvailable(pVCpu);
9283	}
9284	return iemRaiseUndefinedOpcode(pVCpu);
9285	}
9286
9287
9288	/**
9289	* Implements 'VSTMXCSR'.
9290	*
9291	* @param iEffSeg The effective segment register for @a GCPtrEff.
9292	* @param GCPtrEff The address of the image.
9293	*/
9294	IEM_CIMPL_DEF_2(iemCImpl_vstmxcsr, uint8_t, iEffSeg, RTGCPTR, GCPtrEff)
9295	{
9296	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_XCRx);
9297
9298	/*
9299	* Raise exceptions.
9300	*/
9301	if ( ( !IEM_IS_GUEST_CPU_AMD(pVCpu)
9302	? (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_SSE \| XSAVE_C_YMM)) == (XSAVE_C_SSE \| XSAVE_C_YMM)
9303	: !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM)) /* AMD Jaguar CPU (f0x16,m0,s1) behaviour */
9304	&& (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE))
9305	{
9306	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS))
9307	{
9308	/*
9309	* Do the job.
9310	*/
9311	VBOXSTRICTRC rcStrict = iemMemStoreDataU32(pVCpu, iEffSeg, GCPtrEff, pVCpu->cpum.GstCtx.XState.x87.MXCSR);
9312	if (rcStrict == VINF_SUCCESS)
9313	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
9314	return rcStrict;
9315	}
9316	return iemRaiseDeviceNotAvailable(pVCpu);
9317	}
9318	return iemRaiseUndefinedOpcode(pVCpu);
9319	}
9320
9321
9322	/**
9323	* Implements 'LDMXCSR'.
9324	*
9325	* @param iEffSeg The effective segment register for @a GCPtrEff.
9326	* @param GCPtrEff The address of the image.
9327	*/
9328	IEM_CIMPL_DEF_2(iemCImpl_ldmxcsr, uint8_t, iEffSeg, RTGCPTR, GCPtrEff)
9329	{
9330	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX);
9331
9332	/*
9333	* Raise exceptions.
9334	*/
9335	/** @todo testcase - order of LDMXCSR faults. Does \#PF, \#GP and \#SS
9336	* happen after or before \#UD and \#EM? */
9337	if ( !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM)
9338	&& (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR))
9339	{
9340	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS))
9341	{
9342	/*
9343	* Do the job.
9344	*/
9345	uint32_t fNewMxCsr;
9346	VBOXSTRICTRC rcStrict = iemMemFetchDataU32(pVCpu, &fNewMxCsr, iEffSeg, GCPtrEff);
9347	if (rcStrict == VINF_SUCCESS)
9348	{
9349	uint32_t const fMxCsrMask = CPUMGetGuestMxCsrMask(pVCpu->CTX_SUFF(pVM));
9350	if (!(fNewMxCsr & ~fMxCsrMask))
9351	{
9352	pVCpu->cpum.GstCtx.XState.x87.MXCSR = fNewMxCsr;
9353	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
9354	}
9355	Log(("ldmxcsr: New MXCSR=%#RX32 & ~MASK=%#RX32 = %#RX32 -> #GP(0)\n",
9356	fNewMxCsr, fMxCsrMask, fNewMxCsr & ~fMxCsrMask));
9357	return iemRaiseGeneralProtectionFault0(pVCpu);
9358	}
9359	return rcStrict;
9360	}
9361	return iemRaiseDeviceNotAvailable(pVCpu);
9362	}
9363	return iemRaiseUndefinedOpcode(pVCpu);
9364	}
9365
9366
9367	/**
9368	* Implements 'VSTMXCSR'.
9369	*
9370	* @param iEffSeg The effective segment register for @a GCPtrEff.
9371	* @param GCPtrEff The address of the image.
9372	*/
9373	IEM_CIMPL_DEF_2(iemCImpl_vldmxcsr, uint8_t, iEffSeg, RTGCPTR, GCPtrEff)
9374	{
9375	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_XCRx);
9376
9377	/*
9378	* Raise exceptions.
9379	*/
9380	if ( ( !IEM_IS_GUEST_CPU_AMD(pVCpu)
9381	? (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_SSE \| XSAVE_C_YMM)) == (XSAVE_C_SSE \| XSAVE_C_YMM)
9382	: !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM))
9383	&& (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE))
9384	{
9385	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS))
9386	{
9387	/*
9388	* Do the job.
9389	*/
9390	uint32_t fNewMxCsr;
9391	VBOXSTRICTRC rcStrict = iemMemFetchDataU32(pVCpu, &fNewMxCsr, iEffSeg, GCPtrEff);
9392	if (rcStrict == VINF_SUCCESS)
9393	{
9394	uint32_t const fMxCsrMask = CPUMGetGuestMxCsrMask(pVCpu->CTX_SUFF(pVM));
9395	if (!(fNewMxCsr & ~fMxCsrMask))
9396	{
9397	pVCpu->cpum.GstCtx.XState.x87.MXCSR = fNewMxCsr;
9398	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
9399	}
9400	Log(("ldmxcsr: New MXCSR=%#RX32 & ~MASK=%#RX32 = %#RX32 -> #GP(0)\n",
9401	fNewMxCsr, fMxCsrMask, fNewMxCsr & ~fMxCsrMask));
9402	return iemRaiseGeneralProtectionFault0(pVCpu);
9403	}
9404	return rcStrict;
9405	}
9406	return iemRaiseDeviceNotAvailable(pVCpu);
9407	}
9408	return iemRaiseUndefinedOpcode(pVCpu);
9409	}
9410
9411
9412	/**
9413	* Commmon routine for fnstenv and fnsave.
9414	*
9415	* @param pVCpu The cross context virtual CPU structure of the calling thread.
9416	* @param enmEffOpSize The effective operand size.
9417	* @param uPtr Where to store the state.
9418	*/
9419	static void iemCImplCommonFpuStoreEnv(PVMCPUCC pVCpu, IEMMODE enmEffOpSize, RTPTRUNION uPtr)
9420	{
9421	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_X87);
9422	PCX86FXSTATE pSrcX87 = &pVCpu->cpum.GstCtx.XState.x87;
9423	if (enmEffOpSize == IEMMODE_16BIT)
9424	{
9425	uPtr.pu16[0] = pSrcX87->FCW;
9426	uPtr.pu16[1] = pSrcX87->FSW;
9427	uPtr.pu16[2] = iemFpuCalcFullFtw(pSrcX87);
9428	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
9429	{
9430	/** @todo Testcase: How does this work when the FPUIP/CS was saved in
9431	* protected mode or long mode and we save it in real mode? And vice
9432	* versa? And with 32-bit operand size? I think CPU is storing the
9433	* effective address ((CS << 4) + IP) in the offset register and not
9434	* doing any address calculations here. */
9435	uPtr.pu16[3] = (uint16_t)pSrcX87->FPUIP;
9436	uPtr.pu16[4] = ((pSrcX87->FPUIP >> 4) & UINT16_C(0xf000)) \| pSrcX87->FOP;
9437	uPtr.pu16[5] = (uint16_t)pSrcX87->FPUDP;
9438	uPtr.pu16[6] = (pSrcX87->FPUDP >> 4) & UINT16_C(0xf000);
9439	}
9440	else
9441	{
9442	uPtr.pu16[3] = pSrcX87->FPUIP;
9443	uPtr.pu16[4] = pSrcX87->CS;
9444	uPtr.pu16[5] = pSrcX87->FPUDP;
9445	uPtr.pu16[6] = pSrcX87->DS;
9446	}
9447	}
9448	else
9449	{
9450	/** @todo Testcase: what is stored in the "gray" areas? (figure 8-9 and 8-10) */
9451	uPtr.pu16[0*2] = pSrcX87->FCW;
9452	uPtr.pu16[02+1] = 0xffff; / (0xffff observed on intel skylake.) */
9453	uPtr.pu16[1*2] = pSrcX87->FSW;
9454	uPtr.pu16[1*2+1] = 0xffff;
9455	uPtr.pu16[2*2] = iemFpuCalcFullFtw(pSrcX87);
9456	uPtr.pu16[2*2+1] = 0xffff;
9457	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
9458	{
9459	uPtr.pu16[3*2] = (uint16_t)pSrcX87->FPUIP;
9460	uPtr.pu32[4] = ((pSrcX87->FPUIP & UINT32_C(0xffff0000)) >> 4) \| pSrcX87->FOP;
9461	uPtr.pu16[5*2] = (uint16_t)pSrcX87->FPUDP;
9462	uPtr.pu32[6] = (pSrcX87->FPUDP & UINT32_C(0xffff0000)) >> 4;
9463	}
9464	else
9465	{
9466	uPtr.pu32[3] = pSrcX87->FPUIP;
9467	uPtr.pu16[4*2] = pSrcX87->CS;
9468	uPtr.pu16[4*2+1] = pSrcX87->FOP;
9469	uPtr.pu32[5] = pSrcX87->FPUDP;
9470	uPtr.pu16[6*2] = pSrcX87->DS;
9471	uPtr.pu16[6*2+1] = 0xffff;
9472	}
9473	}
9474	}
9475
9476
9477	/**
9478	* Commmon routine for fldenv and frstor
9479	*
9480	* @param pVCpu The cross context virtual CPU structure of the calling thread.
9481	* @param enmEffOpSize The effective operand size.
9482	* @param uPtr Where to store the state.
9483	*/
9484	static void iemCImplCommonFpuRestoreEnv(PVMCPUCC pVCpu, IEMMODE enmEffOpSize, RTCPTRUNION uPtr)
9485	{
9486	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_X87);
9487	PX86FXSTATE pDstX87 = &pVCpu->cpum.GstCtx.XState.x87;
9488	if (enmEffOpSize == IEMMODE_16BIT)
9489	{
9490	pDstX87->FCW = uPtr.pu16[0];
9491	pDstX87->FSW = uPtr.pu16[1];
9492	pDstX87->FTW = uPtr.pu16[2];
9493	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
9494	{
9495	pDstX87->FPUIP = uPtr.pu16[3] \| ((uint32_t)(uPtr.pu16[4] & UINT16_C(0xf000)) << 4);
9496	pDstX87->FPUDP = uPtr.pu16[5] \| ((uint32_t)(uPtr.pu16[6] & UINT16_C(0xf000)) << 4);
9497	pDstX87->FOP = uPtr.pu16[4] & UINT16_C(0x07ff);
9498	pDstX87->CS = 0;
9499	pDstX87->Rsrvd1= 0;
9500	pDstX87->DS = 0;
9501	pDstX87->Rsrvd2= 0;
9502	}
9503	else
9504	{
9505	pDstX87->FPUIP = uPtr.pu16[3];
9506	pDstX87->CS = uPtr.pu16[4];
9507	pDstX87->Rsrvd1= 0;
9508	pDstX87->FPUDP = uPtr.pu16[5];
9509	pDstX87->DS = uPtr.pu16[6];
9510	pDstX87->Rsrvd2= 0;
9511	/** @todo Testcase: Is FOP cleared when doing 16-bit protected mode fldenv? */
9512	}
9513	}
9514	else
9515	{
9516	pDstX87->FCW = uPtr.pu16[0*2];
9517	pDstX87->FSW = uPtr.pu16[1*2];
9518	pDstX87->FTW = uPtr.pu16[2*2];
9519	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
9520	{
9521	pDstX87->FPUIP = uPtr.pu16[3*2] \| ((uPtr.pu32[4] & UINT32_C(0x0ffff000)) << 4);
9522	pDstX87->FOP = uPtr.pu32[4] & UINT16_C(0x07ff);
9523	pDstX87->FPUDP = uPtr.pu16[5*2] \| ((uPtr.pu32[6] & UINT32_C(0x0ffff000)) << 4);
9524	pDstX87->CS = 0;
9525	pDstX87->Rsrvd1= 0;
9526	pDstX87->DS = 0;
9527	pDstX87->Rsrvd2= 0;
9528	}
9529	else
9530	{
9531	pDstX87->FPUIP = uPtr.pu32[3];
9532	pDstX87->CS = uPtr.pu16[4*2];
9533	pDstX87->Rsrvd1= 0;
9534	pDstX87->FOP = uPtr.pu16[4*2+1];
9535	pDstX87->FPUDP = uPtr.pu32[5];
9536	pDstX87->DS = uPtr.pu16[6*2];
9537	pDstX87->Rsrvd2= 0;
9538	}
9539	}
9540
9541	/* Make adjustments. */
9542	pDstX87->FTW = iemFpuCompressFtw(pDstX87->FTW);
9543	#ifdef LOG_ENABLED
9544	uint16_t const fOldFsw = pDstX87->FSW;
9545	#endif
9546	pDstX87->FCW &= ~X86_FCW_ZERO_MASK \| X86_FCW_IC_MASK; /* Intel 10980xe allows setting the IC bit. Win 3.11 CALC.EXE sets it. */
9547	iemFpuRecalcExceptionStatus(pDstX87);
9548	#ifdef LOG_ENABLED
9549	if ((pDstX87->FSW & X86_FSW_ES) ^ (fOldFsw & X86_FSW_ES))
9550	Log11(("iemCImplCommonFpuRestoreEnv: %04x:%08RX64: %s FPU exception (FCW=%#x FSW=%#x -> %#x)\n",
9551	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, fOldFsw & X86_FSW_ES ? "Suppressed" : "Raised",
9552	pDstX87->FCW, fOldFsw, pDstX87->FSW));
9553	#endif
9554
9555	/** @todo Testcase: Check if ES and/or B are automatically cleared if no
9556	* exceptions are pending after loading the saved state? */
9557	}
9558
9559
9560	/**
9561	* Implements 'FNSTENV'.
9562	*
9563	* @param enmEffOpSize The operand size (only REX.W really matters).
9564	* @param iEffSeg The effective segment register for @a GCPtrEffDst.
9565	* @param GCPtrEffDst The address of the image.
9566	*/
9567	IEM_CIMPL_DEF_3(iemCImpl_fnstenv, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst)
9568	{
9569	uint8_t bUnmapInfo;
9570	RTPTRUNION uPtr;
9571	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &uPtr.pv, &bUnmapInfo, enmEffOpSize == IEMMODE_16BIT ? 14 : 28,
9572	iEffSeg, GCPtrEffDst, IEM_ACCESS_DATA_W \| IEM_ACCESS_PARTIAL_WRITE,
9573	enmEffOpSize == IEMMODE_16BIT ? 1 : 3 /** @todo ? */);
9574	if (rcStrict != VINF_SUCCESS)
9575	return rcStrict;
9576
9577	iemCImplCommonFpuStoreEnv(pVCpu, enmEffOpSize, uPtr);
9578
9579	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
9580	if (rcStrict != VINF_SUCCESS)
9581	return rcStrict;
9582
9583	/* Mask all math exceptions. Any possibly pending exceptions will be cleared. */
9584	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
9585	pFpuCtx->FCW \|= X86_FCW_XCPT_MASK;
9586	#ifdef LOG_ENABLED
9587	uint16_t fOldFsw = pFpuCtx->FSW;
9588	#endif
9589	iemFpuRecalcExceptionStatus(pFpuCtx);
9590	#ifdef LOG_ENABLED
9591	if ((pFpuCtx->FSW & X86_FSW_ES) ^ (fOldFsw & X86_FSW_ES))
9592	Log11(("fnstenv: %04x:%08RX64: %s FPU exception (FCW=%#x, FSW %#x -> %#x)\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
9593	fOldFsw & X86_FSW_ES ? "Suppressed" : "Raised", pFpuCtx->FCW, fOldFsw, pFpuCtx->FSW));
9594	#endif
9595
9596	iemHlpUsedFpu(pVCpu);
9597
9598	/* Note: C0, C1, C2 and C3 are documented as undefined, we leave them untouched! */
9599	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
9600	}
9601
9602
9603	/**
9604	* Implements 'FNSAVE'.
9605	*
9606	* @param enmEffOpSize The operand size.
9607	* @param iEffSeg The effective segment register for @a GCPtrEffDst.
9608	* @param GCPtrEffDst The address of the image.
9609	*/
9610	IEM_CIMPL_DEF_3(iemCImpl_fnsave, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst)
9611	{
9612	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_X87);
9613
9614	uint8_t bUnmapInfo;
9615	RTPTRUNION uPtr;
9616	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &uPtr.pv, &bUnmapInfo, enmEffOpSize == IEMMODE_16BIT ? 94 : 108,
9617	iEffSeg, GCPtrEffDst, IEM_ACCESS_DATA_W \| IEM_ACCESS_PARTIAL_WRITE, 3 /** @todo ? */);
9618	if (rcStrict != VINF_SUCCESS)
9619	return rcStrict;
9620
9621	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
9622	iemCImplCommonFpuStoreEnv(pVCpu, enmEffOpSize, uPtr);
9623	PRTFLOAT80U paRegs = (PRTFLOAT80U)(uPtr.pu8 + (enmEffOpSize == IEMMODE_16BIT ? 14 : 28));
9624	for (uint32_t i = 0; i < RT_ELEMENTS(pFpuCtx->aRegs); i++)
9625	{
9626	paRegs[i].au32[0] = pFpuCtx->aRegs[i].au32[0];
9627	paRegs[i].au32[1] = pFpuCtx->aRegs[i].au32[1];
9628	paRegs[i].au16[4] = pFpuCtx->aRegs[i].au16[4];
9629	}
9630
9631	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
9632	if (rcStrict != VINF_SUCCESS)
9633	return rcStrict;
9634
9635	/* Rotate the stack to account for changed TOS. */
9636	iemFpuRotateStackSetTop(pFpuCtx, 0);
9637
9638	/*
9639	* Re-initialize the FPU context.
9640	*/
9641	pFpuCtx->FCW = 0x37f;
9642	pFpuCtx->FSW = 0;
9643	pFpuCtx->FTW = 0x00; /* 0 - empty */
9644	pFpuCtx->FPUDP = 0;
9645	pFpuCtx->DS = 0;
9646	pFpuCtx->Rsrvd2= 0;
9647	pFpuCtx->FPUIP = 0;
9648	pFpuCtx->CS = 0;
9649	pFpuCtx->Rsrvd1= 0;
9650	pFpuCtx->FOP = 0;
9651
9652	iemHlpUsedFpu(pVCpu);
9653	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
9654	}
9655
9656
9657
9658	/**
9659	* Implements 'FLDENV'.
9660	*
9661	* @param enmEffOpSize The operand size (only REX.W really matters).
9662	* @param iEffSeg The effective segment register for @a GCPtrEffSrc.
9663	* @param GCPtrEffSrc The address of the image.
9664	*/
9665	IEM_CIMPL_DEF_3(iemCImpl_fldenv, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc)
9666	{
9667	uint8_t bUnmapInfo;
9668	RTCPTRUNION uPtr;
9669	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, (void **)&uPtr.pv, &bUnmapInfo, enmEffOpSize == IEMMODE_16BIT ? 14 : 28,
9670	iEffSeg, GCPtrEffSrc, IEM_ACCESS_DATA_R,
9671	enmEffOpSize == IEMMODE_16BIT ? 1 : 3 /** @todo ?*/);
9672	if (rcStrict != VINF_SUCCESS)
9673	return rcStrict;
9674
9675	iemCImplCommonFpuRestoreEnv(pVCpu, enmEffOpSize, uPtr);
9676
9677	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
9678	if (rcStrict != VINF_SUCCESS)
9679	return rcStrict;
9680
9681	iemHlpUsedFpu(pVCpu);
9682	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
9683	}
9684
9685
9686	/**
9687	* Implements 'FRSTOR'.
9688	*
9689	* @param enmEffOpSize The operand size.
9690	* @param iEffSeg The effective segment register for @a GCPtrEffSrc.
9691	* @param GCPtrEffSrc The address of the image.
9692	*/
9693	IEM_CIMPL_DEF_3(iemCImpl_frstor, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc)
9694	{
9695	uint8_t bUnmapInfo;
9696	RTCPTRUNION uPtr;
9697	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, (void **)&uPtr.pv, &bUnmapInfo, enmEffOpSize == IEMMODE_16BIT ? 94 : 108,
9698	iEffSeg, GCPtrEffSrc, IEM_ACCESS_DATA_R, 3 /** @todo ?*/ );
9699	if (rcStrict != VINF_SUCCESS)
9700	return rcStrict;
9701
9702	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
9703	iemCImplCommonFpuRestoreEnv(pVCpu, enmEffOpSize, uPtr);
9704	PCRTFLOAT80U paRegs = (PCRTFLOAT80U)(uPtr.pu8 + (enmEffOpSize == IEMMODE_16BIT ? 14 : 28));
9705	for (uint32_t i = 0; i < RT_ELEMENTS(pFpuCtx->aRegs); i++)
9706	{
9707	pFpuCtx->aRegs[i].au32[0] = paRegs[i].au32[0];
9708	pFpuCtx->aRegs[i].au32[1] = paRegs[i].au32[1];
9709	pFpuCtx->aRegs[i].au32[2] = paRegs[i].au16[4];
9710	pFpuCtx->aRegs[i].au32[3] = 0;
9711	}
9712
9713	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
9714	if (rcStrict != VINF_SUCCESS)
9715	return rcStrict;
9716
9717	iemHlpUsedFpu(pVCpu);
9718	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
9719	}
9720
9721
9722	/**
9723	* Implements 'FLDCW'.
9724	*
9725	* @param u16Fcw The new FCW.
9726	*/
9727	IEM_CIMPL_DEF_1(iemCImpl_fldcw, uint16_t, u16Fcw)
9728	{
9729	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_X87);
9730
9731	/** @todo Testcase: Check what happens when trying to load X86_FCW_PC_RSVD. */
9732	/** @todo Testcase: Try see what happens when trying to set undefined bits
9733	* (other than 6 and 7). Currently ignoring them. */
9734	/** @todo Testcase: Test that it raises and loweres the FPU exception bits
9735	* according to FSW. (This is what is currently implemented.) */
9736	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
9737	pFpuCtx->FCW = u16Fcw & (~X86_FCW_ZERO_MASK \| X86_FCW_IC_MASK); /* Intel 10980xe allows setting the IC bit. Win 3.11 CALC.EXE sets it. */
9738	#ifdef LOG_ENABLED
9739	uint16_t fOldFsw = pFpuCtx->FSW;
9740	#endif
9741	iemFpuRecalcExceptionStatus(pFpuCtx);
9742	#ifdef LOG_ENABLED
9743	if ((pFpuCtx->FSW & X86_FSW_ES) ^ (fOldFsw & X86_FSW_ES))
9744	Log11(("fldcw: %04x:%08RX64: %s FPU exception (FCW=%#x, FSW %#x -> %#x)\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
9745	fOldFsw & X86_FSW_ES ? "Suppressed" : "Raised", pFpuCtx->FCW, fOldFsw, pFpuCtx->FSW));
9746	#endif
9747
9748	/* Note: C0, C1, C2 and C3 are documented as undefined, we leave them untouched! */
9749	iemHlpUsedFpu(pVCpu);
9750	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
9751	}
9752
9753
9754
9755	/**
9756	* Implements the underflow case of fxch.
9757	*
9758	* @param iStReg The other stack register.
9759	* @param uFpuOpcode The FPU opcode (for simplicity).
9760	*/
9761	IEM_CIMPL_DEF_2(iemCImpl_fxch_underflow, uint8_t, iStReg, uint16_t, uFpuOpcode)
9762	{
9763	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_X87);
9764
9765	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
9766	unsigned const iReg1 = X86_FSW_TOP_GET(pFpuCtx->FSW);
9767	unsigned const iReg2 = (iReg1 + iStReg) & X86_FSW_TOP_SMASK;
9768	Assert(!(RT_BIT(iReg1) & pFpuCtx->FTW) \|\| !(RT_BIT(iReg2) & pFpuCtx->FTW));
9769
9770	/** @todo Testcase: fxch underflow. Making assumptions that underflowed
9771	* registers are read as QNaN and then exchanged. This could be
9772	* wrong... */
9773	if (pFpuCtx->FCW & X86_FCW_IM)
9774	{
9775	if (RT_BIT(iReg1) & pFpuCtx->FTW)
9776	{
9777	if (RT_BIT(iReg2) & pFpuCtx->FTW)
9778	iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80);
9779	else
9780	pFpuCtx->aRegs[0].r80 = pFpuCtx->aRegs[iStReg].r80;
9781	iemFpuStoreQNan(&pFpuCtx->aRegs[iStReg].r80);
9782	}
9783	else
9784	{
9785	pFpuCtx->aRegs[iStReg].r80 = pFpuCtx->aRegs[0].r80;
9786	iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80);
9787	}
9788	pFpuCtx->FSW &= ~X86_FSW_C_MASK;
9789	pFpuCtx->FSW \|= X86_FSW_C1 \| X86_FSW_IE \| X86_FSW_SF;
9790	}
9791	else
9792	{
9793	/* raise underflow exception, don't change anything. */
9794	pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK \| X86_FSW_XCPT_MASK);
9795	pFpuCtx->FSW \|= X86_FSW_C1 \| X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
9796	Log11(("fxch: %04x:%08RX64: Underflow exception (FSW=%#x)\n",
9797	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
9798	}
9799
9800	iemFpuUpdateOpcodeAndIpWorkerEx(pVCpu, pFpuCtx, uFpuOpcode);
9801	iemHlpUsedFpu(pVCpu);
9802	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
9803	}
9804
9805
9806	/**
9807	* Implements 'FCOMI', 'FCOMIP', 'FUCOMI', and 'FUCOMIP'.
9808	*
9809	* @param iStReg The other stack register.
9810	* @param fUCmp true for FUCOMI[P], false for FCOMI[P].
9811	* @param uPopAndFpuOpcode Bits 15-0: The FPU opcode.
9812	* Bit 31: Whether we should pop the stack when
9813	* done or not.
9814	*/
9815	IEM_CIMPL_DEF_3(iemCImpl_fcomi_fucomi, uint8_t, iStReg, bool, fUCmp, uint32_t, uPopAndFpuOpcode)
9816	{
9817	Assert(iStReg < 8);
9818	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_X87);
9819
9820	/*
9821	* Raise exceptions.
9822	*/
9823	if (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS))
9824	return iemRaiseDeviceNotAvailable(pVCpu);
9825
9826	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
9827	uint16_t u16Fsw = pFpuCtx->FSW;
9828	if (u16Fsw & X86_FSW_ES)
9829	return iemRaiseMathFault(pVCpu);
9830
9831	/*
9832	* Check if any of the register accesses causes #SF + #IA.
9833	*/
9834	bool fPop = RT_BOOL(uPopAndFpuOpcode & RT_BIT_32(31));
9835	unsigned const iReg1 = X86_FSW_TOP_GET(u16Fsw);
9836	unsigned const iReg2 = (iReg1 + iStReg) & X86_FSW_TOP_SMASK;
9837	if ((pFpuCtx->FTW & (RT_BIT(iReg1) \| RT_BIT(iReg2))) == (RT_BIT(iReg1) \| RT_BIT(iReg2)))
9838	{
9839	uint32_t u32Eflags;
9840	if (!fUCmp)
9841	u32Eflags = iemAImpl_fcomi_r80_by_r80(pFpuCtx, &u16Fsw, &pFpuCtx->aRegs[0].r80, &pFpuCtx->aRegs[iStReg].r80);
9842	else
9843	u32Eflags = iemAImpl_fucomi_r80_by_r80(pFpuCtx, &u16Fsw, &pFpuCtx->aRegs[0].r80, &pFpuCtx->aRegs[iStReg].r80);
9844
9845	pFpuCtx->FSW &= ~X86_FSW_C1;
9846	pFpuCtx->FSW \|= u16Fsw & ~X86_FSW_TOP_MASK;
9847	if ( !(u16Fsw & X86_FSW_IE)
9848	\|\| (pFpuCtx->FCW & X86_FCW_IM) )
9849	{
9850	pVCpu->cpum.GstCtx.eflags.u &= ~(X86_EFL_OF \| X86_EFL_SF \| X86_EFL_AF \| X86_EFL_ZF \| X86_EFL_PF \| X86_EFL_CF);
9851	pVCpu->cpum.GstCtx.eflags.u \|= u32Eflags & (X86_EFL_ZF \| X86_EFL_PF \| X86_EFL_CF);
9852	}
9853	}
9854	else if (pFpuCtx->FCW & X86_FCW_IM)
9855	{
9856	/* Masked underflow. */
9857	pFpuCtx->FSW &= ~X86_FSW_C1;
9858	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF;
9859	pVCpu->cpum.GstCtx.eflags.u &= ~(X86_EFL_OF \| X86_EFL_SF \| X86_EFL_AF \| X86_EFL_ZF \| X86_EFL_PF \| X86_EFL_CF);
9860	pVCpu->cpum.GstCtx.eflags.u \|= X86_EFL_ZF \| X86_EFL_PF \| X86_EFL_CF;
9861	}
9862	else
9863	{
9864	/* Raise underflow - don't touch EFLAGS or TOP. */
9865	pFpuCtx->FSW &= ~X86_FSW_C1;
9866	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
9867	Log11(("fxch: %04x:%08RX64: Raising IE+SF exception (FSW=%#x)\n",
9868	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
9869	fPop = false;
9870	}
9871
9872	/*
9873	* Pop if necessary.
9874	*/
9875	if (fPop)
9876	{
9877	pFpuCtx->FTW &= ~RT_BIT(iReg1);
9878	iemFpuStackIncTop(pVCpu);
9879	}
9880
9881	iemFpuUpdateOpcodeAndIpWorkerEx(pVCpu, pFpuCtx, (uint16_t)uPopAndFpuOpcode);
9882	iemHlpUsedFpu(pVCpu);
9883	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
9884	}
9885
9886
9887	/**
9888	* Implements 'RDSEED'.
9889	*
9890	* @returns VINF_SUCCESS.
9891	* @param iReg The register.
9892	* @param enmEffOpSize The operand size.
9893	*/
9894	IEM_CIMPL_DEF_2(iemCImpl_rdseed, uint8_t, iReg, IEMMODE, enmEffOpSize)
9895	{
9896	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9897	/* Nested-guest VMX intercept. */
9898	if ( !IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
9899	\|\| !IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_RDSEED_EXIT))
9900	{ /* probable */ }
9901	else
9902	{
9903	Log(("rdseed: Guest intercept -> VM-exit\n"));
9904	IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_RDSEED, VMXINSTRID_RDSEED, cbInstr);
9905	}
9906	#endif
9907
9908	uint32_t *pEFlags = &pVCpu->cpum.GstCtx.eflags.uBoth;
9909	switch (enmEffOpSize)
9910	{
9911	case IEMMODE_16BIT:
9912	{
9913	PFNIEMAIMPLRDRANDSEEDU16 pfnImpl = IEM_SELECT_HOST_OR_FALLBACK(fRdSeed,
9914	&iemAImpl_rdseed_u16,
9915	&iemAImpl_rdseed_u16_fallback);
9916	uint16_t *pu16Dst = iemGRegRefU16(pVCpu, iReg);
9917	(pfnImpl)(pu16Dst, pEFlags);
9918	break;
9919	}
9920	case IEMMODE_32BIT:
9921	{
9922	PFNIEMAIMPLRDRANDSEEDU32 pfnImpl = IEM_SELECT_HOST_OR_FALLBACK(fRdSeed,
9923	&iemAImpl_rdseed_u32,
9924	&iemAImpl_rdseed_u32_fallback);
9925	uint32_t *pu32Dst = iemGRegRefU32(pVCpu, iReg);
9926	(pfnImpl)(pu32Dst, pEFlags);
9927	iemGRegStoreU32(pVCpu, iReg, *pu32Dst);
9928	break;
9929	}
9930	case IEMMODE_64BIT:
9931	{
9932	PFNIEMAIMPLRDRANDSEEDU64 pfnImpl = IEM_SELECT_HOST_OR_FALLBACK(fRdSeed,
9933	&iemAImpl_rdseed_u64,
9934	&iemAImpl_rdseed_u64_fallback);
9935	uint64_t *pu64Dst = iemGRegRefU64(pVCpu, iReg);
9936	(pfnImpl)(pu64Dst, pEFlags);
9937	break;
9938	}
9939	IEM_NOT_REACHED_DEFAULT_CASE_RET();
9940	}
9941	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
9942	}
9943
9944
9945	/**
9946	* Implements 'RDRAND'.
9947	*
9948	* @returns VINF_SUCCESS.
9949	* @param iReg The register.
9950	* @param enmEffOpSize The operand size.
9951	*/
9952	IEM_CIMPL_DEF_2(iemCImpl_rdrand, uint8_t, iReg, IEMMODE, enmEffOpSize)
9953	{
9954	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9955	/* Nested-guest VMX intercept. */
9956	if ( !IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
9957	\|\| !IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_RDRAND_EXIT))
9958	{ /* probable */ }
9959	else
9960	{
9961	Log(("rdrand: Guest intercept -> VM-exit\n"));
9962	IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_RDRAND, VMXINSTRID_RDRAND, cbInstr);
9963	}
9964	#endif
9965
9966	uint32_t *pEFlags = &pVCpu->cpum.GstCtx.eflags.uBoth;
9967	switch (enmEffOpSize)
9968	{
9969	case IEMMODE_16BIT:
9970	{
9971	PFNIEMAIMPLRDRANDSEEDU16 pfnImpl = IEM_SELECT_HOST_OR_FALLBACK(fRdRand, &iemAImpl_rdrand_u16,
9972	&iemAImpl_rdrand_u16_fallback);
9973	uint16_t *pu16Dst = iemGRegRefU16(pVCpu, iReg);
9974	(pfnImpl)(pu16Dst, pEFlags);
9975	break;
9976	}
9977	case IEMMODE_32BIT:
9978	{
9979	PFNIEMAIMPLRDRANDSEEDU32 pfnImpl = IEM_SELECT_HOST_OR_FALLBACK(fRdRand, &iemAImpl_rdrand_u32,
9980	&iemAImpl_rdrand_u32_fallback);
9981	uint32_t *pu32Dst = iemGRegRefU32(pVCpu, iReg);
9982	(pfnImpl)(pu32Dst, pEFlags);
9983	iemGRegStoreU32(pVCpu, iReg, *pu32Dst);
9984	break;
9985	}
9986	case IEMMODE_64BIT:
9987	{
9988	PFNIEMAIMPLRDRANDSEEDU64 pfnImpl = IEM_SELECT_HOST_OR_FALLBACK(fRdRand, &iemAImpl_rdrand_u64,
9989	&iemAImpl_rdrand_u64_fallback);
9990	uint64_t *pu64Dst = iemGRegRefU64(pVCpu, iReg);
9991	(pfnImpl)(pu64Dst, pEFlags);
9992	break;
9993	}
9994	IEM_NOT_REACHED_DEFAULT_CASE_RET();
9995	}
9996	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
9997	}
9998
9999
10000	/**
10001	* Worker for 'VMASKMOVPS / VPMASKMOVD' 128-bit 32-bit-masked load.
10002	*
10003	* @param pVCpu The cross context virtual CPU structure of the calling thread.
10004	* @param cbInstr The current instruction length.
10005	* @param iXRegDst The destination XMM register index.
10006	* @param iXRegMsk The mask XMM register index.
10007	* @param iEffSeg The effective segment.
10008	* @param GCPtrEffSrc The source memory address.
10009	*/
10010	static VBOXSTRICTRC iemCImpl_maskmov_load_u128_32_worker(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iXRegDst, uint8_t iXRegMsk, uint8_t iEffSeg, RTGCPTR GCPtrEffSrc)
10011	{
10012	uint32_t fAccessed = 0;
10013
10014	PRTUINT128U puDst = (PRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDst];
10015	PCRTUINT128U puMsk = (PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegMsk];
10016	PCRTUINT128U puSrc;
10017
10018	for (uint32_t i = 0; i < RT_ELEMENTS(puMsk->au32); i++)
10019	{
10020	fAccessed \|= puMsk->au32[i];
10021	}
10022
10023	if (fAccessed & RT_BIT(31)) {
10024	/*
10025	* Access the source memory.
10026	*/
10027	uint8_t bUnmapInfo;
10028	void *pvMemSrc;
10029	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvMemSrc, &bUnmapInfo, sizeof(*puSrc),
10030	iEffSeg, GCPtrEffSrc, IEM_ACCESS_DATA_R, 0);
10031	if (rcStrict != VINF_SUCCESS)
10032	return rcStrict;
10033
10034	puSrc = (PCRTUINT128U)pvMemSrc;
10035
10036	for (uint32_t i = 0; i < RT_ELEMENTS(puSrc->au32); i++)
10037	{
10038	puDst->au32[i] = (puMsk->au32[i] & RT_BIT(31)) ? puSrc->au32[i] : 0;
10039	}
10040	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iXRegDst].au64[0] = 0;
10041	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iXRegDst].au64[1] = 0;
10042
10043	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
10044	if (rcStrict != VINF_SUCCESS)
10045	return rcStrict;
10046	}
10047	else
10048	{
10049	puDst->au64[0] = 0;
10050	puDst->au64[1] = 0;
10051	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iXRegDst].au64[0] = 0;
10052	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iXRegDst].au64[1] = 0;
10053	}
10054
10055	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
10056	}
10057
10058
10059
10060	/**
10061	* Worker for 'VMASKMOVPS / VPMASKMOVD' 256-bit 32-bit-masked load.
10062	*
10063	* @param pVCpu The cross context virtual CPU structure of the calling thread.
10064	* @param cbInstr The current instruction length.
10065	* @param iYRegDst The destination YMM register index.
10066	* @param iYRegMsk The mask YMM register index.
10067	* @param iEffSeg The effective segment.
10068	* @param GCPtrEffSrc The source memory address.
10069	*/
10070	static VBOXSTRICTRC iemCImpl_maskmov_load_u256_32_worker(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iYRegDst, uint8_t iYRegMsk, uint8_t iEffSeg, RTGCPTR GCPtrEffSrc)
10071	{
10072	uint32_t fAccessed = 0;
10073
10074	PRTUINT128U puDstLo = (PRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDst];
10075	PRTUINT128U puDstHi = (PRTUINT128U)&pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDst];
10076	PCRTUINT128U puMskLo = (PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegMsk];
10077	PCRTUINT128U puMskHi = (PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegMsk];
10078	PCRTUINT256U puSrc;
10079
10080	for (uint32_t i = 0; i < RT_ELEMENTS(puMskLo->au32); i++)
10081	{
10082	fAccessed \|= puMskLo->au32[i] \| puMskHi->au32[i];
10083	}
10084
10085	if (fAccessed & RT_BIT(31)) {
10086	/*
10087	* Access the source memory.
10088	*/
10089	uint8_t bUnmapInfo;
10090	void *pvMemSrc;
10091	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvMemSrc, &bUnmapInfo, sizeof(*puSrc),
10092	iEffSeg, GCPtrEffSrc, IEM_ACCESS_DATA_R, 0);
10093	if (rcStrict != VINF_SUCCESS)
10094	return rcStrict;
10095
10096	puSrc = (PCRTUINT256U)pvMemSrc;
10097
10098	uint8_t const iHalf = RT_ELEMENTS(puSrc->au32) / 2;
10099
10100	for (uint32_t i = 0; i < iHalf; i++)
10101	{
10102	puDstLo->au32[i] = (puMskLo->au32[i] & RT_BIT(31)) ? puSrc->au32[i] : 0;
10103	}
10104	for (uint32_t i = iHalf; i < RT_ELEMENTS(puSrc->au32); i++)
10105	{
10106	puDstHi->au32[i - iHalf] = (puMskHi->au32[i - iHalf] & RT_BIT(31)) ? puSrc->au32[i] : 0;
10107	}
10108
10109	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
10110	if (rcStrict != VINF_SUCCESS)
10111	return rcStrict;
10112	}
10113	else
10114	{
10115	puDstLo->au64[0] = 0;
10116	puDstLo->au64[1] = 0;
10117	puDstHi->au64[0] = 0;
10118	puDstHi->au64[1] = 0;
10119	}
10120
10121	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
10122	}
10123
10124
10125	/**
10126	* Worker for 'VMASKMOVPS / VPMASKMOVD' 128-bit 32-bit-masked store.
10127	*
10128	* @param pVCpu The cross context virtual CPU structure of the calling thread.
10129	* @param cbInstr The current instruction length.
10130	* @param iEffSeg The effective segment.
10131	* @param GCPtrEffDst The destination memory address.
10132	* @param iXRegMsk The mask XMM register index.
10133	* @param iXRegSrc The source XMM register index.
10134	*/
10135	static VBOXSTRICTRC iemCImpl_maskmov_store_u128_32_worker(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPTR GCPtrEffDst, uint8_t iXRegMsk, uint8_t iXRegSrc)
10136	{
10137	uint32_t fAccessed = 0;
10138
10139	PRTUINT128U puDst;
10140	PCRTUINT128U puMsk = (PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegMsk];
10141	PCRTUINT128U puSrc = (PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegSrc];
10142
10143	for (uint32_t i = 0; i < RT_ELEMENTS(puMsk->au32); i++)
10144	{
10145	fAccessed \|= puMsk->au32[i];
10146	}
10147
10148	if (fAccessed & RT_BIT(31)) {
10149	/*
10150	* Access the destination memory.
10151	*/
10152	uint8_t bUnmapInfo;
10153	void *pvMemDst;
10154	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvMemDst, &bUnmapInfo, sizeof(*puDst),
10155	iEffSeg, GCPtrEffDst, IEM_ACCESS_DATA_RW, 0);
10156	if (rcStrict != VINF_SUCCESS)
10157	return rcStrict;
10158
10159	puDst = (PRTUINT128U)pvMemDst;
10160
10161	for (uint32_t i = 0; i < RT_ELEMENTS(puDst->au32); i++)
10162	{
10163	if (puMsk->au32[i] & RT_BIT(31))
10164	puDst->au32[i] = puSrc->au32[i];
10165	}
10166
10167	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
10168	if (rcStrict != VINF_SUCCESS)
10169	return rcStrict;
10170	}
10171
10172	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
10173	}
10174
10175
10176
10177	/**
10178	* Worker for 'VMASKMOVPS / VPMASKMOVD' 256-bit 32-bit-masked store.
10179	*
10180	* @param pVCpu The cross context virtual CPU structure of the calling thread.
10181	* @param cbInstr The current instruction length.
10182	* @param iEffSeg The effective segment.
10183	* @param GCPtrEffDst The destination memory address.
10184	* @param iYRegMsk The mask YMM register index.
10185	* @param iYRegSrc The source YMM register index.
10186	*/
10187	static VBOXSTRICTRC iemCImpl_maskmov_store_u256_32_worker(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPTR GCPtrEffDst, uint8_t iYRegMsk, uint8_t iYRegSrc)
10188	{
10189	uint32_t fAccessed = 0;
10190
10191	PRTUINT256U puDst;
10192	PCRTUINT128U puMskLo = (PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegMsk];
10193	PCRTUINT128U puMskHi = (PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegMsk];
10194	PCRTUINT128U puSrcLo = (PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc];
10195	PCRTUINT128U puSrcHi = (PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrc];
10196
10197	for (uint32_t i = 0; i < RT_ELEMENTS(puMskLo->au32); i++)
10198	{
10199	fAccessed \|= puMskLo->au32[i] \| puMskHi->au32[i];
10200	}
10201
10202	if (fAccessed & RT_BIT(31)) {
10203	/*
10204	* Access the destination memory.
10205	*/
10206	uint8_t bUnmapInfo;
10207	void *pvMemDst;
10208	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvMemDst, &bUnmapInfo, sizeof(*puDst),
10209	iEffSeg, GCPtrEffDst, IEM_ACCESS_DATA_RW, 0);
10210	if (rcStrict != VINF_SUCCESS)
10211	return rcStrict;
10212
10213	puDst = (PRTUINT256U)pvMemDst;
10214
10215	uint8_t const iHalf = RT_ELEMENTS(puDst->au32) / 2;
10216
10217	for (uint32_t i = 0; i < iHalf; i++)
10218	{
10219	if (puMskLo->au32[i] & RT_BIT(31))
10220	puDst->au32[i] = puSrcLo->au32[i];
10221	}
10222	for (uint32_t i = iHalf; i < RT_ELEMENTS(puDst->au32); i++)
10223	{
10224	if (puMskHi->au32[i - iHalf] & RT_BIT(31))
10225	puDst->au32[i] = puSrcHi->au32[i - iHalf];
10226	}
10227
10228	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
10229	if (rcStrict != VINF_SUCCESS)
10230	return rcStrict;
10231	}
10232
10233	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
10234	}
10235
10236
10237	/**
10238	* Worker for 'VMASKMOVPD / VPMASKMOVQ' 128-bit 64-bit-masked load.
10239	*
10240	* @param pVCpu The cross context virtual CPU structure of the calling thread.
10241	* @param cbInstr The current instruction length.
10242	* @param iXRegDst The destination XMM register index.
10243	* @param iXRegMsk The mask XMM register index.
10244	* @param iEffSeg The effective segment.
10245	* @param GCPtrEffSrc The source memory address.
10246	*/
10247	static VBOXSTRICTRC iemCImpl_maskmov_load_u128_64_worker(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iXRegDst, uint8_t iXRegMsk, uint8_t iEffSeg, RTGCPTR GCPtrEffSrc)
10248	{
10249	uint64_t fAccessed = 0;
10250
10251	PRTUINT128U puDst = (PRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDst];
10252	PCRTUINT128U puMsk = (PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegMsk];
10253	PCRTUINT128U puSrc;
10254
10255	for (uint32_t i = 0; i < RT_ELEMENTS(puMsk->au64); i++)
10256	{
10257	fAccessed \|= puMsk->au64[i];
10258	}
10259
10260	if (fAccessed & RT_BIT_64(63)) {
10261	/*
10262	* Access the source memory.
10263	*/
10264	uint8_t bUnmapInfo;
10265	void *pvMemSrc;
10266	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvMemSrc, &bUnmapInfo, sizeof(*puSrc),
10267	iEffSeg, GCPtrEffSrc, IEM_ACCESS_DATA_R, 0);
10268	if (rcStrict != VINF_SUCCESS)
10269	return rcStrict;
10270
10271	puSrc = (PCRTUINT128U)pvMemSrc;
10272
10273	for (uint32_t i = 0; i < RT_ELEMENTS(puSrc->au64); i++)
10274	{
10275	puDst->au64[i] = (puMsk->au64[i] & RT_BIT_64(63)) ? puSrc->au64[i] : 0;
10276	}
10277	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iXRegDst].au64[0] = 0;
10278	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iXRegDst].au64[1] = 0;
10279
10280	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
10281	if (rcStrict != VINF_SUCCESS)
10282	return rcStrict;
10283	}
10284	else
10285	{
10286	puDst->au64[0] = 0;
10287	puDst->au64[1] = 0;
10288	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iXRegDst].au64[0] = 0;
10289	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iXRegDst].au64[1] = 0;
10290	}
10291
10292	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
10293	}
10294
10295
10296
10297	/**
10298	* Worker for 'VMASKMOVPD / VPMASKMOVQ' 256-bit 64-bit-masked load.
10299	*
10300	* @param pVCpu The cross context virtual CPU structure of the calling thread.
10301	* @param cbInstr The current instruction length.
10302	* @param iYRegDst The destination YMM register index.
10303	* @param iYRegMsk The mask YMM register index.
10304	* @param iEffSeg The effective segment.
10305	* @param GCPtrEffSrc The source memory address.
10306	*/
10307	static VBOXSTRICTRC iemCImpl_maskmov_load_u256_64_worker(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iYRegDst, uint8_t iYRegMsk, uint8_t iEffSeg, RTGCPTR GCPtrEffSrc)
10308	{
10309	uint64_t fAccessed = 0;
10310
10311	PRTUINT128U puDstLo = (PRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDst];
10312	PRTUINT128U puDstHi = (PRTUINT128U)&pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDst];
10313	PCRTUINT128U puMskLo = (PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegMsk];
10314	PCRTUINT128U puMskHi = (PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegMsk];
10315	PCRTUINT256U puSrc;
10316
10317	for (uint32_t i = 0; i < RT_ELEMENTS(puMskLo->au64); i++)
10318	{
10319	fAccessed \|= puMskLo->au64[i] \| puMskHi->au64[i];
10320	}
10321
10322	if (fAccessed & RT_BIT_64(63)) {
10323	/*
10324	* Access the source memory.
10325	*/
10326	uint8_t bUnmapInfo;
10327	void *pvMemSrc;
10328	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvMemSrc, &bUnmapInfo, sizeof(*puSrc),
10329	iEffSeg, GCPtrEffSrc, IEM_ACCESS_DATA_R, 0);
10330	if (rcStrict != VINF_SUCCESS)
10331	return rcStrict;
10332
10333	puSrc = (PCRTUINT256U)pvMemSrc;
10334
10335	uint8_t const iHalf = RT_ELEMENTS(puSrc->au64) / 2;
10336
10337	for (uint32_t i = 0; i < iHalf; i++)
10338	{
10339	puDstLo->au64[i] = (puMskLo->au64[i] & RT_BIT_64(63)) ? puSrc->au64[i] : 0;
10340	}
10341	for (uint32_t i = iHalf; i < RT_ELEMENTS(puSrc->au64); i++)
10342	{
10343	puDstHi->au64[i - iHalf] = (puMskHi->au64[i - iHalf] & RT_BIT_64(63)) ? puSrc->au64[i] : 0;
10344	}
10345
10346	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
10347	if (rcStrict != VINF_SUCCESS)
10348	return rcStrict;
10349	}
10350	else
10351	{
10352	puDstLo->au64[0] = 0;
10353	puDstLo->au64[1] = 0;
10354	puDstHi->au64[0] = 0;
10355	puDstHi->au64[1] = 0;
10356	}
10357
10358	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
10359	}
10360
10361
10362	/**
10363	* Worker for 'VMASKMOVPD / VPMASKMOVQ' 128-bit 64-bit-masked store.
10364	*
10365	* @param pVCpu The cross context virtual CPU structure of the calling thread.
10366	* @param cbInstr The current instruction length.
10367	* @param iEffSeg The effective segment.
10368	* @param GCPtrEffDst The destination memory address.
10369	* @param iXRegMsk The mask XMM register index.
10370	* @param iXRegSrc The source XMM register index.
10371	*/
10372	static VBOXSTRICTRC iemCImpl_maskmov_store_u128_64_worker(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPTR GCPtrEffDst, uint8_t iXRegMsk, uint8_t iXRegSrc)
10373	{
10374	uint64_t fAccessed = 0;
10375
10376	PRTUINT128U puDst;
10377	PCRTUINT128U puMsk = (PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegMsk];
10378	PCRTUINT128U puSrc = (PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegSrc];
10379
10380	for (uint32_t i = 0; i < RT_ELEMENTS(puMsk->au64); i++)
10381	{
10382	fAccessed \|= puMsk->au64[i];
10383	}
10384
10385	if (fAccessed & RT_BIT_64(63)) {
10386	/*
10387	* Access the destination memory.
10388	*/
10389	uint8_t bUnmapInfo;
10390	void *pvMemDst;
10391	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvMemDst, &bUnmapInfo, sizeof(*puDst),
10392	iEffSeg, GCPtrEffDst, IEM_ACCESS_DATA_RW, 0);
10393	if (rcStrict != VINF_SUCCESS)
10394	return rcStrict;
10395
10396	puDst = (PRTUINT128U)pvMemDst;
10397
10398	for (uint32_t i = 0; i < RT_ELEMENTS(puDst->au64); i++)
10399	{
10400	if (puMsk->au64[i] & RT_BIT_64(63))
10401	puDst->au64[i] = puSrc->au64[i];
10402	}
10403
10404	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
10405	if (rcStrict != VINF_SUCCESS)
10406	return rcStrict;
10407	}
10408
10409	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
10410	}
10411
10412
10413
10414	/**
10415	* Worker for 'VMASKMOVPD / VPMASKMOVQ' 256-bit 64-bit-masked store.
10416	*
10417	* @param pVCpu The cross context virtual CPU structure of the calling thread.
10418	* @param cbInstr The current instruction length.
10419	* @param iEffSeg The effective segment.
10420	* @param GCPtrEffDst The destination memory address.
10421	* @param iYRegMsk The mask YMM register index.
10422	* @param iYRegSrc The source YMM register index.
10423	*/
10424	static VBOXSTRICTRC iemCImpl_maskmov_store_u256_64_worker(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPTR GCPtrEffDst, uint8_t iYRegMsk, uint8_t iYRegSrc)
10425	{
10426	uint64_t fAccessed = 0;
10427
10428	PRTUINT256U puDst;
10429	PCRTUINT128U puMskLo = (PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegMsk];
10430	PCRTUINT128U puMskHi = (PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegMsk];
10431	PCRTUINT128U puSrcLo = (PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc];
10432	PCRTUINT128U puSrcHi = (PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrc];
10433
10434	for (uint32_t i = 0; i < RT_ELEMENTS(puMskLo->au64); i++)
10435	{
10436	fAccessed \|= puMskLo->au64[i] \| puMskHi->au64[i];
10437	}
10438
10439	if (fAccessed & RT_BIT_64(63)) {
10440	/*
10441	* Access the destination memory.
10442	*/
10443	uint8_t bUnmapInfo;
10444	void *pvMemDst;
10445	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvMemDst, &bUnmapInfo, sizeof(*puDst),
10446	iEffSeg, GCPtrEffDst, IEM_ACCESS_DATA_RW, 0);
10447	if (rcStrict != VINF_SUCCESS)
10448	return rcStrict;
10449
10450	puDst = (PRTUINT256U)pvMemDst;
10451
10452	uint8_t const iHalf = RT_ELEMENTS(puDst->au64) / 2;
10453
10454	for (uint32_t i = 0; i < iHalf; i++)
10455	{
10456	if (puMskLo->au64[i] & RT_BIT_64(63))
10457	puDst->au64[i] = puSrcLo->au64[i];
10458	}
10459	for (uint32_t i = iHalf; i < RT_ELEMENTS(puDst->au64); i++)
10460	{
10461	if (puMskHi->au64[i - iHalf] & RT_BIT_64(63))
10462	puDst->au64[i] = puSrcHi->au64[i - iHalf];
10463	}
10464
10465	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
10466	if (rcStrict != VINF_SUCCESS)
10467	return rcStrict;
10468	}
10469
10470	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
10471	}
10472
10473
10474	/**
10475	* Implements 'VMASKMOVPS' 128-bit 32-bit-masked load.
10476	*
10477	* @param iXRegDst The destination XMM register index.
10478	* @param iXRegMsk The mask XMM register index.
10479	* @param iEffSeg The effective segment.
10480	* @param GCPtrEffSrc The source memory address.
10481	*/
10482	IEM_CIMPL_DEF_4(iemCImpl_vmaskmovps_load_u128, uint8_t, iXRegDst, uint8_t, iXRegMsk, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc)
10483	{
10484	return iemCImpl_maskmov_load_u128_32_worker(pVCpu, cbInstr, iXRegDst, iXRegMsk, iEffSeg, GCPtrEffSrc);
10485	}
10486
10487
10488	/**
10489	* Implements 'VMASKMOVPS' 256-bit 32-bit-masked load.
10490	*
10491	* @param iYRegDst The destination YMM register index.
10492	* @param iYRegMsk The mask YMM register index.
10493	* @param iEffSeg The effective segment.
10494	* @param GCPtrEffSrc The source memory address.
10495	*/
10496	IEM_CIMPL_DEF_4(iemCImpl_vmaskmovps_load_u256, uint8_t, iYRegDst, uint8_t, iYRegMsk, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc)
10497	{
10498	return iemCImpl_maskmov_load_u256_32_worker(pVCpu, cbInstr, iYRegDst, iYRegMsk, iEffSeg, GCPtrEffSrc);
10499	}
10500
10501
10502	/**
10503	* Implements 'VMASKMOVPS' 128-bit 32-bit-masked store.
10504	*
10505	* @param iEffSeg The effective segment.
10506	* @param GCPtrEffDst The destination memory address.
10507	* @param iXRegMsk The mask XMM register index.
10508	* @param iXRegSrc The source XMM register index.
10509	*/
10510	IEM_CIMPL_DEF_4(iemCImpl_vmaskmovps_store_u128, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst, uint8_t, iXRegMsk, uint8_t, iXRegSrc)
10511	{
10512	return iemCImpl_maskmov_store_u128_32_worker(pVCpu, cbInstr, iEffSeg, GCPtrEffDst, iXRegMsk, iXRegSrc);
10513	}
10514
10515
10516	/**
10517	* Implements 'VMASKMOVPS' 256-bit 32-bit-masked store.
10518	*
10519	* @param iEffSeg The effective segment.
10520	* @param GCPtrEffDst The destination memory address.
10521	* @param iYRegMsk The mask YMM register index.
10522	* @param iYRegSrc The source YMM register index.
10523	*/
10524	IEM_CIMPL_DEF_4(iemCImpl_vmaskmovps_store_u256, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst, uint8_t, iYRegMsk, uint8_t, iYRegSrc)
10525	{
10526	return iemCImpl_maskmov_store_u256_32_worker(pVCpu, cbInstr, iEffSeg, GCPtrEffDst, iYRegMsk, iYRegSrc);
10527	}
10528
10529
10530	/**
10531	* Implements 'VPMASKMOVD' 128-bit 32-bit-masked load.
10532	*
10533	* @param iXRegDst The destination XMM register index.
10534	* @param iXRegMsk The mask XMM register index.
10535	* @param iEffSeg The effective segment.
10536	* @param GCPtrEffSrc The source memory address.
10537	*/
10538	IEM_CIMPL_DEF_4(iemCImpl_vpmaskmovd_load_u128, uint8_t, iXRegDst, uint8_t, iXRegMsk, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc)
10539	{
10540	return iemCImpl_maskmov_load_u128_32_worker(pVCpu, cbInstr, iXRegDst, iXRegMsk, iEffSeg, GCPtrEffSrc);
10541	}
10542
10543
10544	/**
10545	* Implements 'VPMASKMOVD' 256-bit 32-bit-masked load.
10546	*
10547	* @param iYRegDst The destination YMM register index.
10548	* @param iYRegMsk The mask YMM register index.
10549	* @param iEffSeg The effective segment.
10550	* @param GCPtrEffSrc The source memory address.
10551	*/
10552	IEM_CIMPL_DEF_4(iemCImpl_vpmaskmovd_load_u256, uint8_t, iYRegDst, uint8_t, iYRegMsk, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc)
10553	{
10554	return iemCImpl_maskmov_load_u256_32_worker(pVCpu, cbInstr, iYRegDst, iYRegMsk, iEffSeg, GCPtrEffSrc);
10555	}
10556
10557
10558	/**
10559	* Implements 'VPMASKMOVD' 128-bit 32-bit-masked store.
10560	*
10561	* @param iEffSeg The effective segment.
10562	* @param GCPtrEffDst The destination memory address.
10563	* @param iXRegMsk The mask XMM register index.
10564	* @param iXRegSrc The source XMM register index.
10565	*/
10566	IEM_CIMPL_DEF_4(iemCImpl_vpmaskmovd_store_u128, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst, uint8_t, iXRegMsk, uint8_t, iXRegSrc)
10567	{
10568	return iemCImpl_maskmov_store_u128_32_worker(pVCpu, cbInstr, iEffSeg, GCPtrEffDst, iXRegMsk, iXRegSrc);
10569	}
10570
10571
10572	/**
10573	* Implements 'VPMASKMOVD' 256-bit 32-bit-masked store.
10574	*
10575	* @param iEffSeg The effective segment.
10576	* @param GCPtrEffDst The destination memory address.
10577	* @param iYRegMsk The mask YMM register index.
10578	* @param iYRegSrc The source YMM register index.
10579	*/
10580	IEM_CIMPL_DEF_4(iemCImpl_vpmaskmovd_store_u256, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst, uint8_t, iYRegMsk, uint8_t, iYRegSrc)
10581	{
10582	return iemCImpl_maskmov_store_u256_32_worker(pVCpu, cbInstr, iEffSeg, GCPtrEffDst, iYRegMsk, iYRegSrc);
10583	}
10584
10585
10586	/**
10587	* Implements 'VMASKMOVPD' 128-bit 64-bit-masked load.
10588	*
10589	* @param iXRegDst The destination XMM register index.
10590	* @param iXRegMsk The mask XMM register index.
10591	* @param iEffSeg The effective segment.
10592	* @param GCPtrEffSrc The source memory address.
10593	*/
10594	IEM_CIMPL_DEF_4(iemCImpl_vmaskmovpd_load_u128, uint8_t, iXRegDst, uint8_t, iXRegMsk, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc)
10595	{
10596	return iemCImpl_maskmov_load_u128_64_worker(pVCpu, cbInstr, iXRegDst, iXRegMsk, iEffSeg, GCPtrEffSrc);
10597	}
10598
10599
10600	/**
10601	* Implements 'VMASKMOVPD' 256-bit 64-bit-masked load.
10602	*
10603	* @param iYRegDst The destination YMM register index.
10604	* @param iYRegMsk The mask YMM register index.
10605	* @param iEffSeg The effective segment.
10606	* @param GCPtrEffSrc The source memory address.
10607	*/
10608	IEM_CIMPL_DEF_4(iemCImpl_vmaskmovpd_load_u256, uint8_t, iYRegDst, uint8_t, iYRegMsk, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc)
10609	{
10610	return iemCImpl_maskmov_load_u256_64_worker(pVCpu, cbInstr, iYRegDst, iYRegMsk, iEffSeg, GCPtrEffSrc);
10611	}
10612
10613
10614	/**
10615	* Implements 'VMASKMOVPD' 128-bit 64-bit-masked store.
10616	*
10617	* @param iEffSeg The effective segment.
10618	* @param GCPtrEffDst The destination memory address.
10619	* @param iXRegMsk The mask XMM register index.
10620	* @param iXRegSrc The source XMM register index.
10621	*/
10622	IEM_CIMPL_DEF_4(iemCImpl_vmaskmovpd_store_u128, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst, uint8_t, iXRegMsk, uint8_t, iXRegSrc)
10623	{
10624	return iemCImpl_maskmov_store_u128_64_worker(pVCpu, cbInstr, iEffSeg, GCPtrEffDst, iXRegMsk, iXRegSrc);
10625	}
10626
10627
10628	/**
10629	* Implements 'VMASKMOVPD' 256-bit 64-bit-masked store.
10630	*
10631	* @param iEffSeg The effective segment.
10632	* @param GCPtrEffDst The destination memory address.
10633	* @param iYRegMsk The mask YMM register index.
10634	* @param iYRegSrc The source YMM register index.
10635	*/
10636	IEM_CIMPL_DEF_4(iemCImpl_vmaskmovpd_store_u256, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst, uint8_t, iYRegMsk, uint8_t, iYRegSrc)
10637	{
10638	return iemCImpl_maskmov_store_u256_64_worker(pVCpu, cbInstr, iEffSeg, GCPtrEffDst, iYRegMsk, iYRegSrc);
10639	}
10640
10641
10642	/**
10643	* Implements 'VPMASKMOVQ' 128-bit 64-bit-masked load.
10644	*
10645	* @param iXRegDst The destination XMM register index.
10646	* @param iXRegMsk The mask XMM register index.
10647	* @param iEffSeg The effective segment.
10648	* @param GCPtrEffSrc The source memory address.
10649	*/
10650	IEM_CIMPL_DEF_4(iemCImpl_vpmaskmovq_load_u128, uint8_t, iXRegDst, uint8_t, iXRegMsk, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc)
10651	{
10652	return iemCImpl_maskmov_load_u128_64_worker(pVCpu, cbInstr, iXRegDst, iXRegMsk, iEffSeg, GCPtrEffSrc);
10653	}
10654
10655
10656	/**
10657	* Implements 'VPMASKMOVQ' 256-bit 64-bit-masked load.
10658	*
10659	* @param iYRegDst The destination YMM register index.
10660	* @param iYRegMsk The mask YMM register index.
10661	* @param iEffSeg The effective segment.
10662	* @param GCPtrEffSrc The source memory address.
10663	*/
10664	IEM_CIMPL_DEF_4(iemCImpl_vpmaskmovq_load_u256, uint8_t, iYRegDst, uint8_t, iYRegMsk, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc)
10665	{
10666	return iemCImpl_maskmov_load_u256_64_worker(pVCpu, cbInstr, iYRegDst, iYRegMsk, iEffSeg, GCPtrEffSrc);
10667	}
10668
10669
10670	/**
10671	* Implements 'VPMASKMOVQ' 128-bit 64-bit-masked store.
10672	*
10673	* @param iEffSeg The effective segment.
10674	* @param GCPtrEffDst The destination memory address.
10675	* @param iXRegMsk The mask XMM register index.
10676	* @param iXRegSrc The source XMM register index.
10677	*/
10678	IEM_CIMPL_DEF_4(iemCImpl_vpmaskmovq_store_u128, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst, uint8_t, iXRegMsk, uint8_t, iXRegSrc)
10679	{
10680	return iemCImpl_maskmov_store_u128_64_worker(pVCpu, cbInstr, iEffSeg, GCPtrEffDst, iXRegMsk, iXRegSrc);
10681	}
10682
10683
10684	/**
10685	* Implements 'VPMASKMOVQ' 256-bit 64-bit-masked store.
10686	*
10687	* @param iEffSeg The effective segment.
10688	* @param GCPtrEffDst The destination memory address.
10689	* @param iYRegMsk The mask YMM register index.
10690	* @param iYRegSrc The source YMM register index.
10691	*/
10692	IEM_CIMPL_DEF_4(iemCImpl_vpmaskmovq_store_u256, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst, uint8_t, iYRegMsk, uint8_t, iYRegSrc)
10693	{
10694	return iemCImpl_maskmov_store_u256_64_worker(pVCpu, cbInstr, iEffSeg, GCPtrEffDst, iYRegMsk, iYRegSrc);
10695	}
10696
10697
10698	/**
10699	* Worker for 'VGATHERcxx' / 'VPGATHERxx' masked loads.
10700	*
10701	* @param u32PackedArgs Arguments packed to the tune of IEMGATHERARGS.
10702	* @param u32Disp The address displacement for the indices.
10703	*/
10704	IEM_CIMPL_DEF_2(iemCImpl_vpgather_worker_xx, uint32_t, u32PackedArgs, uint32_t, u32Disp)
10705	{
10706	IEMGATHERARGS const PackedArgs = { u32PackedArgs };
10707	int32_t const offDisp = (int32_t)u32Disp;
10708
10709	if (PackedArgs.s.iYRegDst == PackedArgs.s.iYRegIdc \|\|
10710	PackedArgs.s.iYRegIdc == PackedArgs.s.iYRegMsk \|\|
10711	PackedArgs.s.iYRegDst == PackedArgs.s.iYRegMsk) return iemRaiseUndefinedOpcode(pVCpu);
10712
10713	Assert(PackedArgs.s.enmEffOpSize <= IEMMODE_64BIT);
10714	Assert(PackedArgs.s.enmEffAddrMode <= IEMMODE_64BIT);
10715
10716	uint32_t const cbMaxWidth = PackedArgs.s.fVex256 ? 32 : 16; /* Width of widest XMM / YMM register we will use: 32 or 16 */
10717	uint32_t const cbIdxWidth = PackedArgs.s.fIdxQword ? 8 : 4; /* Width of one index: 4-byte dword or 8-byte qword */
10718	uint32_t const cbValWidth = PackedArgs.s.fValQword ? 8 : 4; /* Width of one value: 4-byte dword or 8-byte qword */
10719	uint32_t const cMasks = cbMaxWidth / cbValWidth; /* Count of masks: 8 or 4 or 2 */
10720	uint32_t const cIndices = cbMaxWidth / cbIdxWidth; /* Count of indices: 8 or 4 or 2 */
10721	uint32_t const cValues = RT_MIN(cMasks, cIndices); /* Count of values to gather: 8 or 4 or 2 */
10722	Assert(cValues == 2 \|\| cValues == 4 \|\| cValues == 8);
10723	uint32_t const cbDstWidth = cValues * cbValWidth; /* Width of the destination & mask XMM / YMM registers: 32 or 16 or 8 */
10724	Assert(cbDstWidth == 8 \|\| cbDstWidth == 16 \|\| cbDstWidth == 32);
10725
10726	/*
10727	* Get the base pointer.
10728	*/
10729	uint64_t u64Base = iemGRegFetchU64(pVCpu, PackedArgs.s.iGRegBase);
10730	if (PackedArgs.s.enmEffAddrMode != IEMMODE_64BIT)
10731	u64Base &= (PackedArgs.s.enmEffAddrMode == IEMMODE_16BIT ? UINT16_MAX : UINT32_MAX);
10732
10733	PRTUINT128U const apuDst[2] =
10734	{
10735	&pVCpu->cpum.GstCtx.XState.x87.aXMM[PackedArgs.s.iYRegDst].uXmm,
10736	&pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[PackedArgs.s.iYRegDst].uXmm
10737	};
10738	PCRTUINT128U const apuIdc[2] =
10739	{
10740	&pVCpu->cpum.GstCtx.XState.x87.aXMM[PackedArgs.s.iYRegIdc].uXmm,
10741	&pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[PackedArgs.s.iYRegIdc].uXmm
10742	};
10743	PRTUINT128U const apuMsk[2] =
10744	{
10745	&pVCpu->cpum.GstCtx.XState.x87.aXMM[PackedArgs.s.iYRegMsk].uXmm,
10746	&pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[PackedArgs.s.iYRegMsk].uXmm
10747	};
10748
10749	/*
10750	* Convert the masks to all-0s or all-1s, writing back to the mask
10751	* register so it will have the correct value if subsequent memory
10752	* accesses fault. Note that cMasks can be larger than cValues, in
10753	* the Qword-index, Dword-value instructions `vgatherqps' and
10754	* `vpgatherqd'. Updating the masks for as many masks as would
10755	* have been used if the destination register were wide enough --
10756	* is the observed behavior of a Core i7-10700.
10757	*/
10758	if (!PackedArgs.s.fValQword)
10759	for (uint32_t i = 0; i < cMasks; i++)
10760	apuMsk[(i >> 2) & 1]->ai32[i & 3] >>= 31; /* Use arithmetic shift right (SAR/ASR) */
10761	else
10762	for (uint32_t i = 0; i < cMasks; i++)
10763	apuMsk[(i >> 1) & 1]->ai64[i & 1] >>= 63; /* Use arithmetic shift right (SAR/ASR) */
10764
10765	/*
10766	* Zero upper bits of mask if VEX128.
10767	*/
10768	if (!PackedArgs.s.fVex256)
10769	{
10770	apuMsk[1]->au64[0] = 0;
10771	apuMsk[1]->au64[1] = 0;
10772	}
10773
10774	/*
10775	* Gather the individual values, as masked.
10776	*/
10777	for (uint32_t i = 0; i < cValues; i++)
10778	{
10779	/*
10780	* Consult the mask determined above.
10781	*/
10782	if ( !PackedArgs.s.fValQword
10783	? apuMsk[(i >> 2) & 1]->au32[i & 3] != 0
10784	: apuMsk[(i >> 1) & 1]->au64[i & 1] != 0)
10785	{
10786	/*
10787	* Get the index, scale it, add scaled index + offset to the base pointer.
10788	*/
10789	int64_t offIndex;
10790	if (!PackedArgs.s.fIdxQword)
10791	offIndex = apuIdc[(i >> 2) & 1]->ai32[i & 3];
10792	else
10793	offIndex = apuIdc[(i >> 1) & 1]->ai64[i & 1];
10794	offIndex <<= PackedArgs.s.iScale;
10795	offIndex += offDisp;
10796
10797	uint64_t u64Addr = u64Base + offIndex;
10798	if (PackedArgs.s.enmEffAddrMode != IEMMODE_64BIT)
10799	u64Addr &= UINT32_MAX;
10800
10801	/*
10802	* Gather it -- fetch this gather-item from guest memory.
10803	*/
10804	VBOXSTRICTRC rcStrict;
10805	if (!PackedArgs.s.fValQword)
10806	rcStrict = iemMemFetchDataU32NoAc(pVCpu, &apuDst[(i >> 2) & 1]->au32[i & 3], PackedArgs.s.iEffSeg, u64Addr);
10807	else
10808	rcStrict = iemMemFetchDataU64NoAc(pVCpu, &apuDst[(i >> 1) & 1]->au64[i & 1], PackedArgs.s.iEffSeg, u64Addr);
10809	if (rcStrict != VINF_SUCCESS)
10810	return rcStrict;
10811
10812	/*
10813	* Now that we didn't fault, write all-0s to that part of the mask register.
10814	*/
10815	if (!PackedArgs.s.fValQword)
10816	apuMsk[(i >> 2) & 1]->au32[i & 3] = 0;
10817	else
10818	apuMsk[(i >> 1) & 1]->au64[i & 1] = 0;
10819	/** @todo How is data breakpoints handled? The intel docs kind of hints they
10820	* may be raised here... */
10821	}
10822	}
10823
10824	/*
10825	* Zero upper bits of destination and mask.
10826	*/
10827	if (cbDstWidth != 32)
10828	{
10829	apuDst[1]->au64[0] = 0;
10830	apuDst[1]->au64[1] = 0;
10831	apuMsk[1]->au64[0] = 0;
10832	apuMsk[1]->au64[1] = 0;
10833	if (cbDstWidth == 8)
10834	{
10835	apuDst[0]->au64[1] = 0;
10836	apuMsk[0]->au64[1] = 0;
10837	}
10838	}
10839
10840	return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
10841	}
10842
10843	/** @} */
10844

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/target-x86/IEMAllCImpl-x86.cpp@ 108196

Download in other formats: