IEMInline.h@ 99385

Last change on this file since 99385 was 99300, checked in by vboxsync, 21 months ago
VMM/IEM: More work on processing MC blocks and generating threaded functions from them. IEMThreadedFunctions.cpp compiles now. Did some PC update optimizations when doing addressing variations. [scm fix] bugref:10369
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 102.2 KB

Line
1	/* $Id: IEMInline.h 99300 2023-04-06 00:08:28Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Inlined Functions.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMInline_h
29	#define VMM_INCLUDED_SRC_include_IEMInline_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35
36	/**
37	* Makes status code addjustments (pass up from I/O and access handler)
38	* as well as maintaining statistics.
39	*
40	* @returns Strict VBox status code to pass up.
41	* @param pVCpu The cross context virtual CPU structure of the calling thread.
42	* @param rcStrict The status from executing an instruction.
43	*/
44	DECL_FORCE_INLINE(VBOXSTRICTRC) iemExecStatusCodeFiddling(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
45	{
46	if (rcStrict != VINF_SUCCESS)
47	{
48	if (RT_SUCCESS(rcStrict))
49	{
50	AssertMsg( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
51	\|\| rcStrict == VINF_IOM_R3_IOPORT_READ
52	\|\| rcStrict == VINF_IOM_R3_IOPORT_WRITE
53	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
54	\|\| rcStrict == VINF_IOM_R3_MMIO_READ
55	\|\| rcStrict == VINF_IOM_R3_MMIO_READ_WRITE
56	\|\| rcStrict == VINF_IOM_R3_MMIO_WRITE
57	\|\| rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE
58	\|\| rcStrict == VINF_CPUM_R3_MSR_READ
59	\|\| rcStrict == VINF_CPUM_R3_MSR_WRITE
60	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
61	\|\| rcStrict == VINF_EM_RAW_TO_R3
62	\|\| rcStrict == VINF_EM_TRIPLE_FAULT
63	\|\| rcStrict == VINF_GIM_R3_HYPERCALL
64	/* raw-mode / virt handlers only: */
65	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT
66	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT
67	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT
68	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT
69	\|\| rcStrict == VINF_SELM_SYNC_GDT
70	\|\| rcStrict == VINF_CSAM_PENDING_ACTION
71	\|\| rcStrict == VINF_PATM_CHECK_PATCH_PAGE
72	/* nested hw.virt codes: */
73	\|\| rcStrict == VINF_VMX_VMEXIT
74	\|\| rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE
75	\|\| rcStrict == VINF_VMX_MODIFIES_BEHAVIOR
76	\|\| rcStrict == VINF_SVM_VMEXIT
77	, ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
78	/** @todo adjust for VINF_EM_RAW_EMULATE_INSTR. */
79	int32_t const rcPassUp = pVCpu->iem.s.rcPassUp;
80	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
81	if ( rcStrict == VINF_VMX_VMEXIT
82	&& rcPassUp == VINF_SUCCESS)
83	rcStrict = VINF_SUCCESS;
84	else
85	#endif
86	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
87	if ( rcStrict == VINF_SVM_VMEXIT
88	&& rcPassUp == VINF_SUCCESS)
89	rcStrict = VINF_SUCCESS;
90	else
91	#endif
92	if (rcPassUp == VINF_SUCCESS)
93	pVCpu->iem.s.cRetInfStatuses++;
94	else if ( rcPassUp < VINF_EM_FIRST
95	\|\| rcPassUp > VINF_EM_LAST
96	\|\| rcPassUp < VBOXSTRICTRC_VAL(rcStrict))
97	{
98	Log(("IEM: rcPassUp=%Rrc! rcStrict=%Rrc\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
99	pVCpu->iem.s.cRetPassUpStatus++;
100	rcStrict = rcPassUp;
101	}
102	else
103	{
104	Log(("IEM: rcPassUp=%Rrc rcStrict=%Rrc!\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
105	pVCpu->iem.s.cRetInfStatuses++;
106	}
107	}
108	else if (rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED)
109	pVCpu->iem.s.cRetAspectNotImplemented++;
110	else if (rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
111	pVCpu->iem.s.cRetInstrNotImplemented++;
112	else
113	pVCpu->iem.s.cRetErrStatuses++;
114	}
115	else if (pVCpu->iem.s.rcPassUp != VINF_SUCCESS)
116	{
117	pVCpu->iem.s.cRetPassUpStatus++;
118	rcStrict = pVCpu->iem.s.rcPassUp;
119	}
120
121	return rcStrict;
122	}
123
124
125	/**
126	* Sets the pass up status.
127	*
128	* @returns VINF_SUCCESS.
129	* @param pVCpu The cross context virtual CPU structure of the
130	* calling thread.
131	* @param rcPassUp The pass up status. Must be informational.
132	* VINF_SUCCESS is not allowed.
133	*/
134	DECLINLINE(int) iemSetPassUpStatus(PVMCPUCC pVCpu, VBOXSTRICTRC rcPassUp) RT_NOEXCEPT
135	{
136	AssertRC(VBOXSTRICTRC_VAL(rcPassUp)); Assert(rcPassUp != VINF_SUCCESS);
137
138	int32_t const rcOldPassUp = pVCpu->iem.s.rcPassUp;
139	if (rcOldPassUp == VINF_SUCCESS)
140	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
141	/* If both are EM scheduling codes, use EM priority rules. */
142	else if ( rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST
143	&& rcPassUp >= VINF_EM_FIRST && rcPassUp <= VINF_EM_LAST)
144	{
145	if (rcPassUp < rcOldPassUp)
146	{
147	Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
148	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
149	}
150	else
151	Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
152	}
153	/* Override EM scheduling with specific status code. */
154	else if (rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST)
155	{
156	Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
157	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
158	}
159	/* Don't override specific status code, first come first served. */
160	else
161	Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
162	return VINF_SUCCESS;
163	}
164
165
166	/**
167	* Calculates the CPU mode.
168	*
169	* This is mainly for updating IEMCPU::enmCpuMode.
170	*
171	* @returns CPU mode.
172	* @param pVCpu The cross context virtual CPU structure of the
173	* calling thread.
174	*/
175	DECLINLINE(IEMMODE) iemCalcCpuMode(PVMCPUCC pVCpu) RT_NOEXCEPT
176	{
177	if (CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx))
178	return IEMMODE_64BIT;
179	if (pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig) /** @todo check if this is correct... */
180	return IEMMODE_32BIT;
181	return IEMMODE_16BIT;
182	}
183
184	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
185
186	# if defined(VBOX_INCLUDED_vmm_dbgf_h) \|\| defined(DOXYGEN_RUNNING) /* dbgf.ro.cEnabledHwBreakpoints */
187	/**
188	* Initializes the execution state.
189	*
190	* @param pVCpu The cross context virtual CPU structure of the
191	* calling thread.
192	* @param fBypassHandlers Whether to bypass access handlers.
193	*
194	* @remarks Callers of this must call iemUninitExec() to undo potentially fatal
195	* side-effects in strict builds.
196	*/
197	DECLINLINE(void) iemInitExec(PVMCPUCC pVCpu, bool fBypassHandlers) RT_NOEXCEPT
198	{
199	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
200	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
201	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
202	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
203	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
204	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
205	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
206	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
207	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
208	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
209
210	pVCpu->iem.s.uCpl = CPUMGetGuestCPL(pVCpu);
211	pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
212	# ifdef VBOX_STRICT
213	pVCpu->iem.s.enmDefAddrMode = (IEMMODE)0xfe;
214	pVCpu->iem.s.enmEffAddrMode = (IEMMODE)0xfe;
215	pVCpu->iem.s.enmDefOpSize = (IEMMODE)0xfe;
216	pVCpu->iem.s.enmEffOpSize = (IEMMODE)0xfe;
217	pVCpu->iem.s.fPrefixes = 0xfeedbeef;
218	pVCpu->iem.s.uRexReg = 127;
219	pVCpu->iem.s.uRexB = 127;
220	pVCpu->iem.s.offModRm = 127;
221	pVCpu->iem.s.uRexIndex = 127;
222	pVCpu->iem.s.iEffSeg = 127;
223	pVCpu->iem.s.idxPrefix = 127;
224	pVCpu->iem.s.uVex3rdReg = 127;
225	pVCpu->iem.s.uVexLength = 127;
226	pVCpu->iem.s.fEvexStuff = 127;
227	pVCpu->iem.s.uFpuOpcode = UINT16_MAX;
228	# ifdef IEM_WITH_CODE_TLB
229	pVCpu->iem.s.offInstrNextByte = UINT16_MAX;
230	pVCpu->iem.s.pbInstrBuf = NULL;
231	pVCpu->iem.s.cbInstrBuf = UINT16_MAX;
232	pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;
233	pVCpu->iem.s.offCurInstrStart = INT16_MAX;
234	pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);
235	# else
236	pVCpu->iem.s.offOpcode = 127;
237	pVCpu->iem.s.cbOpcode = 127;
238	# endif
239	# endif /* VBOX_STRICT */
240
241	pVCpu->iem.s.cActiveMappings = 0;
242	pVCpu->iem.s.iNextMapping = 0;
243	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
244	pVCpu->iem.s.fBypassHandlers = fBypassHandlers;
245	pVCpu->iem.s.fDisregardLock = false;
246	pVCpu->iem.s.fPendingInstructionBreakpoints = false;
247	pVCpu->iem.s.fPendingDataBreakpoints = false;
248	pVCpu->iem.s.fPendingIoBreakpoints = false;
249	if (RT_LIKELY( !(pVCpu->cpum.GstCtx.dr[7] & X86_DR7_ENABLED_MASK)
250	&& pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledHwBreakpoints == 0))
251	{ /* likely */ }
252	else
253	iemInitPendingBreakpointsSlow(pVCpu);
254	}
255	# endif /* VBOX_INCLUDED_vmm_dbgf_h */
256
257
258	# if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
259	/**
260	* Performs a minimal reinitialization of the execution state.
261	*
262	* This is intended to be used by VM-exits, SMM, LOADALL and other similar
263	* 'world-switch' types operations on the CPU. Currently only nested
264	* hardware-virtualization uses it.
265	*
266	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
267	*/
268	DECLINLINE(void) iemReInitExec(PVMCPUCC pVCpu) RT_NOEXCEPT
269	{
270	IEMMODE const enmMode = iemCalcCpuMode(pVCpu);
271	uint8_t const uCpl = CPUMGetGuestCPL(pVCpu);
272
273	pVCpu->iem.s.uCpl = uCpl;
274	pVCpu->iem.s.enmCpuMode = enmMode;
275	pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
276	pVCpu->iem.s.enmEffAddrMode = enmMode;
277	if (enmMode != IEMMODE_64BIT)
278	{
279	pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
280	pVCpu->iem.s.enmEffOpSize = enmMode;
281	}
282	else
283	{
284	pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
285	pVCpu->iem.s.enmEffOpSize = enmMode;
286	}
287	pVCpu->iem.s.iEffSeg = X86_SREG_DS;
288	# ifndef IEM_WITH_CODE_TLB
289	/** @todo Shouldn't we be doing this in IEMTlbInvalidateAll()? */
290	pVCpu->iem.s.offOpcode = 0;
291	pVCpu->iem.s.cbOpcode = 0;
292	# endif
293	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
294	}
295	# endif
296
297
298	/**
299	* Counterpart to #iemInitExec that undoes evil strict-build stuff.
300	*
301	* @param pVCpu The cross context virtual CPU structure of the
302	* calling thread.
303	*/
304	DECLINLINE(void) iemUninitExec(PVMCPUCC pVCpu) RT_NOEXCEPT
305	{
306	/* Note! do not touch fInPatchCode here! (see iemUninitExecAndFiddleStatusAndMaybeReenter) */
307	# ifdef VBOX_STRICT
308	# ifdef IEM_WITH_CODE_TLB
309	NOREF(pVCpu);
310	# else
311	pVCpu->iem.s.cbOpcode = 0;
312	# endif
313	# else
314	NOREF(pVCpu);
315	# endif
316	}
317
318
319	/**
320	* Calls iemUninitExec, iemExecStatusCodeFiddling and iemRCRawMaybeReenter.
321	*
322	* Only calling iemRCRawMaybeReenter in raw-mode, obviously.
323	*
324	* @returns Fiddled strict vbox status code, ready to return to non-IEM caller.
325	* @param pVCpu The cross context virtual CPU structure of the calling thread.
326	* @param rcStrict The status code to fiddle.
327	*/
328	DECLINLINE(VBOXSTRICTRC) iemUninitExecAndFiddleStatusAndMaybeReenter(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
329	{
330	iemUninitExec(pVCpu);
331	return iemExecStatusCodeFiddling(pVCpu, rcStrict);
332	}
333
334
335	/**
336	* Macro used by the IEMExec* method to check the given instruction length.
337	*
338	* Will return on failure!
339	*
340	* @param a_cbInstr The given instruction length.
341	* @param a_cbMin The minimum length.
342	*/
343	# define IEMEXEC_ASSERT_INSTR_LEN_RETURN(a_cbInstr, a_cbMin) \
344	AssertMsgReturn((unsigned)(a_cbInstr) - (unsigned)(a_cbMin) <= (unsigned)15 - (unsigned)(a_cbMin), \
345	("cbInstr=%u cbMin=%u\n", (a_cbInstr), (a_cbMin)), VERR_IEM_INVALID_INSTR_LENGTH)
346
347
348	# ifndef IEM_WITH_SETJMP
349
350	/**
351	* Fetches the first opcode byte.
352	*
353	* @returns Strict VBox status code.
354	* @param pVCpu The cross context virtual CPU structure of the
355	* calling thread.
356	* @param pu8 Where to return the opcode byte.
357	*/
358	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetFirstU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
359	{
360	/*
361	* Check for hardware instruction breakpoints.
362	*/
363	if (RT_LIKELY(!pVCpu->iem.s.fPendingInstructionBreakpoints))
364	{ /* likely */ }
365	else
366	{
367	VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu,
368	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
369	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
370	{ /* likely */ }
371	else if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
372	return iemRaiseDebugException(pVCpu);
373	else
374	return rcStrict;
375	}
376
377	/*
378	* Fetch the first opcode byte.
379	*/
380	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
381	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
382	{
383	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
384	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
385	return VINF_SUCCESS;
386	}
387	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
388	}
389
390	# else /* IEM_WITH_SETJMP */
391
392	/**
393	* Fetches the first opcode byte, longjmp on error.
394	*
395	* @returns The opcode byte.
396	* @param pVCpu The cross context virtual CPU structure of the calling thread.
397	*/
398	DECL_INLINE_THROW(uint8_t) iemOpcodeGetFirstU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
399	{
400	/*
401	* Check for hardware instruction breakpoints.
402	*/
403	if (RT_LIKELY(!pVCpu->iem.s.fPendingInstructionBreakpoints))
404	{ /* likely */ }
405	else
406	{
407	VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu,
408	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
409	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
410	{ /* likely */ }
411	else
412	{
413	if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
414	rcStrict = iemRaiseDebugException(pVCpu);
415	IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
416	}
417	}
418
419	/*
420	* Fetch the first opcode byte.
421	*/
422	# ifdef IEM_WITH_CODE_TLB
423	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
424	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
425	if (RT_LIKELY( pbBuf != NULL
426	&& offBuf < pVCpu->iem.s.cbInstrBuf))
427	{
428	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
429	return pbBuf[offBuf];
430	}
431	# else
432	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
433	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
434	{
435	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
436	return pVCpu->iem.s.abOpcode[offOpcode];
437	}
438	# endif
439	return iemOpcodeGetNextU8SlowJmp(pVCpu);
440	}
441
442	# endif /* IEM_WITH_SETJMP */
443
444	/**
445	* Fetches the first opcode byte, returns/throws automatically on failure.
446	*
447	* @param a_pu8 Where to return the opcode byte.
448	* @remark Implicitly references pVCpu.
449	*/
450	# ifndef IEM_WITH_SETJMP
451	# define IEM_OPCODE_GET_FIRST_U8(a_pu8) \
452	do \
453	{ \
454	VBOXSTRICTRC rcStrict2 = iemOpcodeGetFirstU8(pVCpu, (a_pu8)); \
455	if (rcStrict2 == VINF_SUCCESS) \
456	{ /* likely */ } \
457	else \
458	return rcStrict2; \
459	} while (0)
460	# else
461	# define IEM_OPCODE_GET_FIRST_U8(a_pu8) (*(a_pu8) = iemOpcodeGetFirstU8Jmp(pVCpu))
462	# endif /* IEM_WITH_SETJMP */
463
464
465	# ifndef IEM_WITH_SETJMP
466
467	/**
468	* Fetches the next opcode byte.
469	*
470	* @returns Strict VBox status code.
471	* @param pVCpu The cross context virtual CPU structure of the
472	* calling thread.
473	* @param pu8 Where to return the opcode byte.
474	*/
475	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
476	{
477	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
478	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
479	{
480	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
481	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
482	return VINF_SUCCESS;
483	}
484	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
485	}
486
487	# else /* IEM_WITH_SETJMP */
488
489	/**
490	* Fetches the next opcode byte, longjmp on error.
491	*
492	* @returns The opcode byte.
493	* @param pVCpu The cross context virtual CPU structure of the calling thread.
494	*/
495	DECL_INLINE_THROW(uint8_t) iemOpcodeGetNextU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
496	{
497	# ifdef IEM_WITH_CODE_TLB
498	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
499	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
500	if (RT_LIKELY( pbBuf != NULL
501	&& offBuf < pVCpu->iem.s.cbInstrBuf))
502	{
503	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
504	return pbBuf[offBuf];
505	}
506	# else
507	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
508	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
509	{
510	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
511	return pVCpu->iem.s.abOpcode[offOpcode];
512	}
513	# endif
514	return iemOpcodeGetNextU8SlowJmp(pVCpu);
515	}
516
517	# endif /* IEM_WITH_SETJMP */
518
519	/**
520	* Fetches the next opcode byte, returns automatically on failure.
521	*
522	* @param a_pu8 Where to return the opcode byte.
523	* @remark Implicitly references pVCpu.
524	*/
525	# ifndef IEM_WITH_SETJMP
526	# define IEM_OPCODE_GET_NEXT_U8(a_pu8) \
527	do \
528	{ \
529	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pVCpu, (a_pu8)); \
530	if (rcStrict2 == VINF_SUCCESS) \
531	{ /* likely */ } \
532	else \
533	return rcStrict2; \
534	} while (0)
535	# else
536	# define IEM_OPCODE_GET_NEXT_U8(a_pu8) (*(a_pu8) = iemOpcodeGetNextU8Jmp(pVCpu))
537	# endif /* IEM_WITH_SETJMP */
538
539
540	# ifndef IEM_WITH_SETJMP
541	/**
542	* Fetches the next signed byte from the opcode stream.
543	*
544	* @returns Strict VBox status code.
545	* @param pVCpu The cross context virtual CPU structure of the calling thread.
546	* @param pi8 Where to return the signed byte.
547	*/
548	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PVMCPUCC pVCpu, int8_t *pi8) RT_NOEXCEPT
549	{
550	return iemOpcodeGetNextU8(pVCpu, (uint8_t *)pi8);
551	}
552	# endif /* !IEM_WITH_SETJMP */
553
554
555	/**
556	* Fetches the next signed byte from the opcode stream, returning automatically
557	* on failure.
558	*
559	* @param a_pi8 Where to return the signed byte.
560	* @remark Implicitly references pVCpu.
561	*/
562	# ifndef IEM_WITH_SETJMP
563	# define IEM_OPCODE_GET_NEXT_S8(a_pi8) \
564	do \
565	{ \
566	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pVCpu, (a_pi8)); \
567	if (rcStrict2 != VINF_SUCCESS) \
568	return rcStrict2; \
569	} while (0)
570	# else /* IEM_WITH_SETJMP */
571	# define IEM_OPCODE_GET_NEXT_S8(a_pi8) (*(a_pi8) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
572
573	# endif /* IEM_WITH_SETJMP */
574
575
576	# ifndef IEM_WITH_SETJMP
577	/**
578	* Fetches the next signed byte from the opcode stream, extending it to
579	* unsigned 16-bit.
580	*
581	* @returns Strict VBox status code.
582	* @param pVCpu The cross context virtual CPU structure of the calling thread.
583	* @param pu16 Where to return the unsigned word.
584	*/
585	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
586	{
587	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
588	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
589	return iemOpcodeGetNextS8SxU16Slow(pVCpu, pu16);
590
591	*pu16 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
592	pVCpu->iem.s.offOpcode = offOpcode + 1;
593	return VINF_SUCCESS;
594	}
595	# endif /* !IEM_WITH_SETJMP */
596
597	/**
598	* Fetches the next signed byte from the opcode stream and sign-extending it to
599	* a word, returning automatically on failure.
600	*
601	* @param a_pu16 Where to return the word.
602	* @remark Implicitly references pVCpu.
603	*/
604	# ifndef IEM_WITH_SETJMP
605	# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \
606	do \
607	{ \
608	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pVCpu, (a_pu16)); \
609	if (rcStrict2 != VINF_SUCCESS) \
610	return rcStrict2; \
611	} while (0)
612	# else
613	# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) (*(a_pu16) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
614	# endif
615
616	# ifndef IEM_WITH_SETJMP
617	/**
618	* Fetches the next signed byte from the opcode stream, extending it to
619	* unsigned 32-bit.
620	*
621	* @returns Strict VBox status code.
622	* @param pVCpu The cross context virtual CPU structure of the calling thread.
623	* @param pu32 Where to return the unsigned dword.
624	*/
625	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
626	{
627	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
628	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
629	return iemOpcodeGetNextS8SxU32Slow(pVCpu, pu32);
630
631	*pu32 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
632	pVCpu->iem.s.offOpcode = offOpcode + 1;
633	return VINF_SUCCESS;
634	}
635	# endif /* !IEM_WITH_SETJMP */
636
637	/**
638	* Fetches the next signed byte from the opcode stream and sign-extending it to
639	* a word, returning automatically on failure.
640	*
641	* @param a_pu32 Where to return the word.
642	* @remark Implicitly references pVCpu.
643	*/
644	# ifndef IEM_WITH_SETJMP
645	# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) \
646	do \
647	{ \
648	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU32(pVCpu, (a_pu32)); \
649	if (rcStrict2 != VINF_SUCCESS) \
650	return rcStrict2; \
651	} while (0)
652	# else
653	# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) (*(a_pu32) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
654	# endif
655
656
657	# ifndef IEM_WITH_SETJMP
658	/**
659	* Fetches the next signed byte from the opcode stream, extending it to
660	* unsigned 64-bit.
661	*
662	* @returns Strict VBox status code.
663	* @param pVCpu The cross context virtual CPU structure of the calling thread.
664	* @param pu64 Where to return the unsigned qword.
665	*/
666	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
667	{
668	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
669	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
670	return iemOpcodeGetNextS8SxU64Slow(pVCpu, pu64);
671
672	*pu64 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
673	pVCpu->iem.s.offOpcode = offOpcode + 1;
674	return VINF_SUCCESS;
675	}
676	# endif /* !IEM_WITH_SETJMP */
677
678	/**
679	* Fetches the next signed byte from the opcode stream and sign-extending it to
680	* a word, returning automatically on failure.
681	*
682	* @param a_pu64 Where to return the word.
683	* @remark Implicitly references pVCpu.
684	*/
685	# ifndef IEM_WITH_SETJMP
686	# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) \
687	do \
688	{ \
689	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU64(pVCpu, (a_pu64)); \
690	if (rcStrict2 != VINF_SUCCESS) \
691	return rcStrict2; \
692	} while (0)
693	# else
694	# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) (*(a_pu64) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
695	# endif
696
697
698	# ifndef IEM_WITH_SETJMP
699	/**
700	* Fetches the next opcode byte.
701	*
702	* @returns Strict VBox status code.
703	* @param pVCpu The cross context virtual CPU structure of the
704	* calling thread.
705	* @param pu8 Where to return the opcode byte.
706	*/
707	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextRm(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
708	{
709	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
710	pVCpu->iem.s.offModRm = offOpcode;
711	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
712	{
713	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
714	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
715	return VINF_SUCCESS;
716	}
717	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
718	}
719	# else /* IEM_WITH_SETJMP */
720	/**
721	* Fetches the next opcode byte, longjmp on error.
722	*
723	* @returns The opcode byte.
724	* @param pVCpu The cross context virtual CPU structure of the calling thread.
725	*/
726	DECL_INLINE_THROW(uint8_t) iemOpcodeGetNextRmJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
727	{
728	# ifdef IEM_WITH_CODE_TLB
729	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
730	pVCpu->iem.s.offModRm = offBuf;
731	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
732	if (RT_LIKELY( pbBuf != NULL
733	&& offBuf < pVCpu->iem.s.cbInstrBuf))
734	{
735	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
736	return pbBuf[offBuf];
737	}
738	# else
739	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
740	pVCpu->iem.s.offModRm = offOpcode;
741	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
742	{
743	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
744	return pVCpu->iem.s.abOpcode[offOpcode];
745	}
746	# endif
747	return iemOpcodeGetNextU8SlowJmp(pVCpu);
748	}
749	# endif /* IEM_WITH_SETJMP */
750
751	/**
752	* Fetches the next opcode byte, which is a ModR/M byte, returns automatically
753	* on failure.
754	*
755	* Will note down the position of the ModR/M byte for VT-x exits.
756	*
757	* @param a_pbRm Where to return the RM opcode byte.
758	* @remark Implicitly references pVCpu.
759	*/
760	# ifndef IEM_WITH_SETJMP
761	# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) \
762	do \
763	{ \
764	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextRm(pVCpu, (a_pbRm)); \
765	if (rcStrict2 == VINF_SUCCESS) \
766	{ /* likely */ } \
767	else \
768	return rcStrict2; \
769	} while (0)
770	# else
771	# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) (*(a_pbRm) = iemOpcodeGetNextRmJmp(pVCpu))
772	# endif /* IEM_WITH_SETJMP */
773
774
775	# ifndef IEM_WITH_SETJMP
776
777	/**
778	* Fetches the next opcode word.
779	*
780	* @returns Strict VBox status code.
781	* @param pVCpu The cross context virtual CPU structure of the calling thread.
782	* @param pu16 Where to return the opcode word.
783	*/
784	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
785	{
786	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
787	if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
788	{
789	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
790	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
791	pu16 = (uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
792	# else
793	*pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
794	# endif
795	return VINF_SUCCESS;
796	}
797	return iemOpcodeGetNextU16Slow(pVCpu, pu16);
798	}
799
800	# else /* IEM_WITH_SETJMP */
801
802	/**
803	* Fetches the next opcode word, longjmp on error.
804	*
805	* @returns The opcode word.
806	* @param pVCpu The cross context virtual CPU structure of the calling thread.
807	*/
808	DECL_INLINE_THROW(uint16_t) iemOpcodeGetNextU16Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
809	{
810	# ifdef IEM_WITH_CODE_TLB
811	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
812	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
813	if (RT_LIKELY( pbBuf != NULL
814	&& offBuf + 2 <= pVCpu->iem.s.cbInstrBuf))
815	{
816	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 2;
817	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
818	return (uint16_t const )&pbBuf[offBuf];
819	# else
820	return RT_MAKE_U16(pbBuf[offBuf], pbBuf[offBuf + 1]);
821	# endif
822	}
823	# else /* !IEM_WITH_CODE_TLB */
824	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
825	if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
826	{
827	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
828	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
829	return (uint16_t const )&pVCpu->iem.s.abOpcode[offOpcode];
830	# else
831	return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
832	# endif
833	}
834	# endif /* !IEM_WITH_CODE_TLB */
835	return iemOpcodeGetNextU16SlowJmp(pVCpu);
836	}
837
838	# endif /* IEM_WITH_SETJMP */
839
840	/**
841	* Fetches the next opcode word, returns automatically on failure.
842	*
843	* @param a_pu16 Where to return the opcode word.
844	* @remark Implicitly references pVCpu.
845	*/
846	# ifndef IEM_WITH_SETJMP
847	# define IEM_OPCODE_GET_NEXT_U16(a_pu16) \
848	do \
849	{ \
850	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pVCpu, (a_pu16)); \
851	if (rcStrict2 != VINF_SUCCESS) \
852	return rcStrict2; \
853	} while (0)
854	# else
855	# define IEM_OPCODE_GET_NEXT_U16(a_pu16) (*(a_pu16) = iemOpcodeGetNextU16Jmp(pVCpu))
856	# endif
857
858	# ifndef IEM_WITH_SETJMP
859	/**
860	* Fetches the next opcode word, zero extending it to a double word.
861	*
862	* @returns Strict VBox status code.
863	* @param pVCpu The cross context virtual CPU structure of the calling thread.
864	* @param pu32 Where to return the opcode double word.
865	*/
866	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
867	{
868	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
869	if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
870	return iemOpcodeGetNextU16ZxU32Slow(pVCpu, pu32);
871
872	*pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
873	pVCpu->iem.s.offOpcode = offOpcode + 2;
874	return VINF_SUCCESS;
875	}
876	# endif /* !IEM_WITH_SETJMP */
877
878	/**
879	* Fetches the next opcode word and zero extends it to a double word, returns
880	* automatically on failure.
881	*
882	* @param a_pu32 Where to return the opcode double word.
883	* @remark Implicitly references pVCpu.
884	*/
885	# ifndef IEM_WITH_SETJMP
886	# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \
887	do \
888	{ \
889	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pVCpu, (a_pu32)); \
890	if (rcStrict2 != VINF_SUCCESS) \
891	return rcStrict2; \
892	} while (0)
893	# else
894	# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU16Jmp(pVCpu))
895	# endif
896
897	# ifndef IEM_WITH_SETJMP
898	/**
899	* Fetches the next opcode word, zero extending it to a quad word.
900	*
901	* @returns Strict VBox status code.
902	* @param pVCpu The cross context virtual CPU structure of the calling thread.
903	* @param pu64 Where to return the opcode quad word.
904	*/
905	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
906	{
907	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
908	if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
909	return iemOpcodeGetNextU16ZxU64Slow(pVCpu, pu64);
910
911	*pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
912	pVCpu->iem.s.offOpcode = offOpcode + 2;
913	return VINF_SUCCESS;
914	}
915	# endif /* !IEM_WITH_SETJMP */
916
917	/**
918	* Fetches the next opcode word and zero extends it to a quad word, returns
919	* automatically on failure.
920	*
921	* @param a_pu64 Where to return the opcode quad word.
922	* @remark Implicitly references pVCpu.
923	*/
924	# ifndef IEM_WITH_SETJMP
925	# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \
926	do \
927	{ \
928	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pVCpu, (a_pu64)); \
929	if (rcStrict2 != VINF_SUCCESS) \
930	return rcStrict2; \
931	} while (0)
932	# else
933	# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU16Jmp(pVCpu))
934	# endif
935
936
937	# ifndef IEM_WITH_SETJMP
938	/**
939	* Fetches the next signed word from the opcode stream.
940	*
941	* @returns Strict VBox status code.
942	* @param pVCpu The cross context virtual CPU structure of the calling thread.
943	* @param pi16 Where to return the signed word.
944	*/
945	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PVMCPUCC pVCpu, int16_t *pi16) RT_NOEXCEPT
946	{
947	return iemOpcodeGetNextU16(pVCpu, (uint16_t *)pi16);
948	}
949	# endif /* !IEM_WITH_SETJMP */
950
951
952	/**
953	* Fetches the next signed word from the opcode stream, returning automatically
954	* on failure.
955	*
956	* @param a_pi16 Where to return the signed word.
957	* @remark Implicitly references pVCpu.
958	*/
959	# ifndef IEM_WITH_SETJMP
960	# define IEM_OPCODE_GET_NEXT_S16(a_pi16) \
961	do \
962	{ \
963	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pVCpu, (a_pi16)); \
964	if (rcStrict2 != VINF_SUCCESS) \
965	return rcStrict2; \
966	} while (0)
967	# else
968	# define IEM_OPCODE_GET_NEXT_S16(a_pi16) (*(a_pi16) = (int16_t)iemOpcodeGetNextU16Jmp(pVCpu))
969	# endif
970
971	# ifndef IEM_WITH_SETJMP
972
973	/**
974	* Fetches the next opcode dword.
975	*
976	* @returns Strict VBox status code.
977	* @param pVCpu The cross context virtual CPU structure of the calling thread.
978	* @param pu32 Where to return the opcode double word.
979	*/
980	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
981	{
982	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
983	if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
984	{
985	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
986	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
987	pu32 = (uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
988	# else
989	*pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
990	pVCpu->iem.s.abOpcode[offOpcode + 1],
991	pVCpu->iem.s.abOpcode[offOpcode + 2],
992	pVCpu->iem.s.abOpcode[offOpcode + 3]);
993	# endif
994	return VINF_SUCCESS;
995	}
996	return iemOpcodeGetNextU32Slow(pVCpu, pu32);
997	}
998
999	# else /* IEM_WITH_SETJMP */
1000
1001	/**
1002	* Fetches the next opcode dword, longjmp on error.
1003	*
1004	* @returns The opcode dword.
1005	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1006	*/
1007	DECL_INLINE_THROW(uint32_t) iemOpcodeGetNextU32Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
1008	{
1009	# ifdef IEM_WITH_CODE_TLB
1010	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1011	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1012	if (RT_LIKELY( pbBuf != NULL
1013	&& offBuf + 4 <= pVCpu->iem.s.cbInstrBuf))
1014	{
1015	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 4;
1016	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1017	return (uint32_t const )&pbBuf[offBuf];
1018	# else
1019	return RT_MAKE_U32_FROM_U8(pbBuf[offBuf],
1020	pbBuf[offBuf + 1],
1021	pbBuf[offBuf + 2],
1022	pbBuf[offBuf + 3]);
1023	# endif
1024	}
1025	# else
1026	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1027	if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
1028	{
1029	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
1030	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1031	return (uint32_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1032	# else
1033	return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1034	pVCpu->iem.s.abOpcode[offOpcode + 1],
1035	pVCpu->iem.s.abOpcode[offOpcode + 2],
1036	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1037	# endif
1038	}
1039	# endif
1040	return iemOpcodeGetNextU32SlowJmp(pVCpu);
1041	}
1042
1043	# endif /* IEM_WITH_SETJMP */
1044
1045	/**
1046	* Fetches the next opcode dword, returns automatically on failure.
1047	*
1048	* @param a_pu32 Where to return the opcode dword.
1049	* @remark Implicitly references pVCpu.
1050	*/
1051	# ifndef IEM_WITH_SETJMP
1052	# define IEM_OPCODE_GET_NEXT_U32(a_pu32) \
1053	do \
1054	{ \
1055	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pVCpu, (a_pu32)); \
1056	if (rcStrict2 != VINF_SUCCESS) \
1057	return rcStrict2; \
1058	} while (0)
1059	# else
1060	# define IEM_OPCODE_GET_NEXT_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU32Jmp(pVCpu))
1061	# endif
1062
1063	# ifndef IEM_WITH_SETJMP
1064	/**
1065	* Fetches the next opcode dword, zero extending it to a quad word.
1066	*
1067	* @returns Strict VBox status code.
1068	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1069	* @param pu64 Where to return the opcode quad word.
1070	*/
1071	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1072	{
1073	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1074	if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1075	return iemOpcodeGetNextU32ZxU64Slow(pVCpu, pu64);
1076
1077	*pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1078	pVCpu->iem.s.abOpcode[offOpcode + 1],
1079	pVCpu->iem.s.abOpcode[offOpcode + 2],
1080	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1081	pVCpu->iem.s.offOpcode = offOpcode + 4;
1082	return VINF_SUCCESS;
1083	}
1084	# endif /* !IEM_WITH_SETJMP */
1085
1086	/**
1087	* Fetches the next opcode dword and zero extends it to a quad word, returns
1088	* automatically on failure.
1089	*
1090	* @param a_pu64 Where to return the opcode quad word.
1091	* @remark Implicitly references pVCpu.
1092	*/
1093	# ifndef IEM_WITH_SETJMP
1094	# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \
1095	do \
1096	{ \
1097	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pVCpu, (a_pu64)); \
1098	if (rcStrict2 != VINF_SUCCESS) \
1099	return rcStrict2; \
1100	} while (0)
1101	# else
1102	# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU32Jmp(pVCpu))
1103	# endif
1104
1105
1106	# ifndef IEM_WITH_SETJMP
1107	/**
1108	* Fetches the next signed double word from the opcode stream.
1109	*
1110	* @returns Strict VBox status code.
1111	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1112	* @param pi32 Where to return the signed double word.
1113	*/
1114	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PVMCPUCC pVCpu, int32_t *pi32) RT_NOEXCEPT
1115	{
1116	return iemOpcodeGetNextU32(pVCpu, (uint32_t *)pi32);
1117	}
1118	# endif
1119
1120	/**
1121	* Fetches the next signed double word from the opcode stream, returning
1122	* automatically on failure.
1123	*
1124	* @param a_pi32 Where to return the signed double word.
1125	* @remark Implicitly references pVCpu.
1126	*/
1127	# ifndef IEM_WITH_SETJMP
1128	# define IEM_OPCODE_GET_NEXT_S32(a_pi32) \
1129	do \
1130	{ \
1131	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pVCpu, (a_pi32)); \
1132	if (rcStrict2 != VINF_SUCCESS) \
1133	return rcStrict2; \
1134	} while (0)
1135	# else
1136	# define IEM_OPCODE_GET_NEXT_S32(a_pi32) (*(a_pi32) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1137	# endif
1138
1139	# ifndef IEM_WITH_SETJMP
1140	/**
1141	* Fetches the next opcode dword, sign extending it into a quad word.
1142	*
1143	* @returns Strict VBox status code.
1144	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1145	* @param pu64 Where to return the opcode quad word.
1146	*/
1147	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1148	{
1149	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1150	if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1151	return iemOpcodeGetNextS32SxU64Slow(pVCpu, pu64);
1152
1153	int32_t i32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1154	pVCpu->iem.s.abOpcode[offOpcode + 1],
1155	pVCpu->iem.s.abOpcode[offOpcode + 2],
1156	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1157	*pu64 = i32;
1158	pVCpu->iem.s.offOpcode = offOpcode + 4;
1159	return VINF_SUCCESS;
1160	}
1161	# endif /* !IEM_WITH_SETJMP */
1162
1163	/**
1164	* Fetches the next opcode double word and sign extends it to a quad word,
1165	* returns automatically on failure.
1166	*
1167	* @param a_pu64 Where to return the opcode quad word.
1168	* @remark Implicitly references pVCpu.
1169	*/
1170	# ifndef IEM_WITH_SETJMP
1171	# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \
1172	do \
1173	{ \
1174	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pVCpu, (a_pu64)); \
1175	if (rcStrict2 != VINF_SUCCESS) \
1176	return rcStrict2; \
1177	} while (0)
1178	# else
1179	# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) (*(a_pu64) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1180	# endif
1181
1182	# ifndef IEM_WITH_SETJMP
1183
1184	/**
1185	* Fetches the next opcode qword.
1186	*
1187	* @returns Strict VBox status code.
1188	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1189	* @param pu64 Where to return the opcode qword.
1190	*/
1191	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1192	{
1193	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1194	if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1195	{
1196	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1197	pu64 = (uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1198	# else
1199	*pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1200	pVCpu->iem.s.abOpcode[offOpcode + 1],
1201	pVCpu->iem.s.abOpcode[offOpcode + 2],
1202	pVCpu->iem.s.abOpcode[offOpcode + 3],
1203	pVCpu->iem.s.abOpcode[offOpcode + 4],
1204	pVCpu->iem.s.abOpcode[offOpcode + 5],
1205	pVCpu->iem.s.abOpcode[offOpcode + 6],
1206	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1207	# endif
1208	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1209	return VINF_SUCCESS;
1210	}
1211	return iemOpcodeGetNextU64Slow(pVCpu, pu64);
1212	}
1213
1214	# else /* IEM_WITH_SETJMP */
1215
1216	/**
1217	* Fetches the next opcode qword, longjmp on error.
1218	*
1219	* @returns The opcode qword.
1220	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1221	*/
1222	DECL_INLINE_THROW(uint64_t) iemOpcodeGetNextU64Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
1223	{
1224	# ifdef IEM_WITH_CODE_TLB
1225	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1226	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1227	if (RT_LIKELY( pbBuf != NULL
1228	&& offBuf + 8 <= pVCpu->iem.s.cbInstrBuf))
1229	{
1230	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 8;
1231	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1232	return (uint64_t const )&pbBuf[offBuf];
1233	# else
1234	return RT_MAKE_U64_FROM_U8(pbBuf[offBuf],
1235	pbBuf[offBuf + 1],
1236	pbBuf[offBuf + 2],
1237	pbBuf[offBuf + 3],
1238	pbBuf[offBuf + 4],
1239	pbBuf[offBuf + 5],
1240	pbBuf[offBuf + 6],
1241	pbBuf[offBuf + 7]);
1242	# endif
1243	}
1244	# else
1245	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1246	if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1247	{
1248	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1249	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1250	return (uint64_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1251	# else
1252	return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1253	pVCpu->iem.s.abOpcode[offOpcode + 1],
1254	pVCpu->iem.s.abOpcode[offOpcode + 2],
1255	pVCpu->iem.s.abOpcode[offOpcode + 3],
1256	pVCpu->iem.s.abOpcode[offOpcode + 4],
1257	pVCpu->iem.s.abOpcode[offOpcode + 5],
1258	pVCpu->iem.s.abOpcode[offOpcode + 6],
1259	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1260	# endif
1261	}
1262	# endif
1263	return iemOpcodeGetNextU64SlowJmp(pVCpu);
1264	}
1265
1266	# endif /* IEM_WITH_SETJMP */
1267
1268	/**
1269	* Fetches the next opcode quad word, returns automatically on failure.
1270	*
1271	* @param a_pu64 Where to return the opcode quad word.
1272	* @remark Implicitly references pVCpu.
1273	*/
1274	# ifndef IEM_WITH_SETJMP
1275	# define IEM_OPCODE_GET_NEXT_U64(a_pu64) \
1276	do \
1277	{ \
1278	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pVCpu, (a_pu64)); \
1279	if (rcStrict2 != VINF_SUCCESS) \
1280	return rcStrict2; \
1281	} while (0)
1282	# else
1283	# define IEM_OPCODE_GET_NEXT_U64(a_pu64) ( *(a_pu64) = iemOpcodeGetNextU64Jmp(pVCpu) )
1284	# endif
1285
1286	#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
1287
1288
1289	/** @name Misc Worker Functions.
1290	* @{
1291	*/
1292
1293	/**
1294	* Gets the correct EFLAGS regardless of whether PATM stores parts of them or
1295	* not (kind of obsolete now).
1296	*
1297	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1298	*/
1299	#define IEMMISC_GET_EFL(a_pVCpu) ( (a_pVCpu)->cpum.GstCtx.eflags.u )
1300
1301	/**
1302	* Updates the EFLAGS in the correct manner wrt. PATM (kind of obsolete).
1303	*
1304	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1305	* @param a_fEfl The new EFLAGS.
1306	*/
1307	#define IEMMISC_SET_EFL(a_pVCpu, a_fEfl) do { (a_pVCpu)->cpum.GstCtx.eflags.u = (a_fEfl); } while (0)
1308
1309
1310	/**
1311	* Loads a NULL data selector into a selector register, both the hidden and
1312	* visible parts, in protected mode.
1313	*
1314	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1315	* @param pSReg Pointer to the segment register.
1316	* @param uRpl The RPL.
1317	*/
1318	DECLINLINE(void) iemHlpLoadNullDataSelectorProt(PVMCPUCC pVCpu, PCPUMSELREG pSReg, RTSEL uRpl) RT_NOEXCEPT
1319	{
1320	/** @todo Testcase: write a testcase checking what happends when loading a NULL
1321	* data selector in protected mode. */
1322	pSReg->Sel = uRpl;
1323	pSReg->ValidSel = uRpl;
1324	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
1325	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
1326	{
1327	/* VT-x (Intel 3960x) observed doing something like this. */
1328	pSReg->Attr.u = X86DESCATTR_UNUSABLE \| X86DESCATTR_G \| X86DESCATTR_D \| (pVCpu->iem.s.uCpl << X86DESCATTR_DPL_SHIFT);
1329	pSReg->u32Limit = UINT32_MAX;
1330	pSReg->u64Base = 0;
1331	}
1332	else
1333	{
1334	pSReg->Attr.u = X86DESCATTR_UNUSABLE;
1335	pSReg->u32Limit = 0;
1336	pSReg->u64Base = 0;
1337	}
1338	}
1339
1340	/** @} */
1341
1342
1343	/*
1344	*
1345	* Helpers routines.
1346	* Helpers routines.
1347	* Helpers routines.
1348	*
1349	*/
1350
1351	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1352
1353	/**
1354	* Recalculates the effective operand size.
1355	*
1356	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1357	*/
1358	DECLINLINE(void) iemRecalEffOpSize(PVMCPUCC pVCpu) RT_NOEXCEPT
1359	{
1360	switch (pVCpu->iem.s.enmCpuMode)
1361	{
1362	case IEMMODE_16BIT:
1363	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT;
1364	break;
1365	case IEMMODE_32BIT:
1366	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT;
1367	break;
1368	case IEMMODE_64BIT:
1369	switch (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP))
1370	{
1371	case 0:
1372	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize;
1373	break;
1374	case IEM_OP_PRF_SIZE_OP:
1375	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1376	break;
1377	case IEM_OP_PRF_SIZE_REX_W:
1378	case IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP:
1379	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1380	break;
1381	}
1382	break;
1383	default:
1384	AssertFailed();
1385	}
1386	}
1387
1388
1389	/**
1390	* Sets the default operand size to 64-bit and recalculates the effective
1391	* operand size.
1392	*
1393	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1394	*/
1395	DECLINLINE(void) iemRecalEffOpSize64Default(PVMCPUCC pVCpu) RT_NOEXCEPT
1396	{
1397	Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
1398	pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1399	if ((pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP)
1400	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1401	else
1402	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1403	}
1404
1405
1406	/**
1407	* Sets the default operand size to 64-bit and recalculates the effective
1408	* operand size, with intel ignoring any operand size prefix (AMD respects it).
1409	*
1410	* This is for the relative jumps.
1411	*
1412	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1413	*/
1414	DECLINLINE(void) iemRecalEffOpSize64DefaultAndIntelIgnoresOpSizePrefix(PVMCPUCC pVCpu) RT_NOEXCEPT
1415	{
1416	Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
1417	pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1418	if ( (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP
1419	\|\| pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
1420	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1421	else
1422	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1423	}
1424
1425	#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
1426
1427
1428
1429	/** @name Register Access.
1430	* @{
1431	*/
1432
1433	/**
1434	* Gets a reference (pointer) to the specified hidden segment register.
1435	*
1436	* @returns Hidden register reference.
1437	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1438	* @param iSegReg The segment register.
1439	*/
1440	DECLINLINE(PCPUMSELREG) iemSRegGetHid(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1441	{
1442	Assert(iSegReg < X86_SREG_COUNT);
1443	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1444	PCPUMSELREG pSReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
1445
1446	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1447	return pSReg;
1448	}
1449
1450
1451	/**
1452	* Ensures that the given hidden segment register is up to date.
1453	*
1454	* @returns Hidden register reference.
1455	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1456	* @param pSReg The segment register.
1457	*/
1458	DECLINLINE(PCPUMSELREG) iemSRegUpdateHid(PVMCPUCC pVCpu, PCPUMSELREG pSReg) RT_NOEXCEPT
1459	{
1460	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1461	NOREF(pVCpu);
1462	return pSReg;
1463	}
1464
1465
1466	/**
1467	* Gets a reference (pointer) to the specified segment register (the selector
1468	* value).
1469	*
1470	* @returns Pointer to the selector variable.
1471	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1472	* @param iSegReg The segment register.
1473	*/
1474	DECLINLINE(uint16_t *) iemSRegRef(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1475	{
1476	Assert(iSegReg < X86_SREG_COUNT);
1477	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1478	return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1479	}
1480
1481
1482	/**
1483	* Fetches the selector value of a segment register.
1484	*
1485	* @returns The selector value.
1486	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1487	* @param iSegReg The segment register.
1488	*/
1489	DECLINLINE(uint16_t) iemSRegFetchU16(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1490	{
1491	Assert(iSegReg < X86_SREG_COUNT);
1492	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1493	return pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1494	}
1495
1496
1497	/**
1498	* Fetches the base address value of a segment register.
1499	*
1500	* @returns The selector value.
1501	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1502	* @param iSegReg The segment register.
1503	*/
1504	DECLINLINE(uint64_t) iemSRegBaseFetchU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1505	{
1506	Assert(iSegReg < X86_SREG_COUNT);
1507	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1508	return pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1509	}
1510
1511
1512	/**
1513	* Gets a reference (pointer) to the specified general purpose register.
1514	*
1515	* @returns Register reference.
1516	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1517	* @param iReg The general purpose register.
1518	*/
1519	DECLINLINE(void *) iemGRegRef(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1520	{
1521	Assert(iReg < 16);
1522	return &pVCpu->cpum.GstCtx.aGRegs[iReg];
1523	}
1524
1525
1526	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1527	/**
1528	* Gets a reference (pointer) to the specified 8-bit general purpose register.
1529	*
1530	* Because of AH, CH, DH and BH we cannot use iemGRegRef directly here.
1531	*
1532	* @returns Register reference.
1533	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1534	* @param iReg The register.
1535	*/
1536	DECLINLINE(uint8_t *) iemGRegRefU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1537	{
1538	if (iReg < 4 \|\| (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX))
1539	{
1540	Assert(iReg < 16);
1541	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u8;
1542	}
1543	/* high 8-bit register. */
1544	Assert(iReg < 8);
1545	return &pVCpu->cpum.GstCtx.aGRegs[iReg & 3].bHi;
1546	}
1547	#endif
1548
1549
1550	/**
1551	* Gets a reference (pointer) to the specified 8-bit general purpose register,
1552	* alternative version with extended (20) register index.
1553	*
1554	* @returns Register reference.
1555	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1556	* @param iRegEx The register. The 16 first are regular ones,
1557	* whereas 16 thru 19 maps to AH, CH, DH and BH.
1558	*/
1559	DECLINLINE(uint8_t *) iemGRegRefU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT
1560	{
1561	if (iRegEx < 16)
1562	return &pVCpu->cpum.GstCtx.aGRegs[iRegEx].u8;
1563
1564	/* high 8-bit register. */
1565	Assert(iRegEx < 20);
1566	return &pVCpu->cpum.GstCtx.aGRegs[iRegEx & 3].bHi;
1567	}
1568
1569
1570	/**
1571	* Gets a reference (pointer) to the specified 16-bit general purpose register.
1572	*
1573	* @returns Register reference.
1574	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1575	* @param iReg The register.
1576	*/
1577	DECLINLINE(uint16_t *) iemGRegRefU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1578	{
1579	Assert(iReg < 16);
1580	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1581	}
1582
1583
1584	/**
1585	* Gets a reference (pointer) to the specified 32-bit general purpose register.
1586	*
1587	* @returns Register reference.
1588	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1589	* @param iReg The register.
1590	*/
1591	DECLINLINE(uint32_t *) iemGRegRefU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1592	{
1593	Assert(iReg < 16);
1594	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1595	}
1596
1597
1598	/**
1599	* Gets a reference (pointer) to the specified signed 32-bit general purpose register.
1600	*
1601	* @returns Register reference.
1602	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1603	* @param iReg The register.
1604	*/
1605	DECLINLINE(int32_t *) iemGRegRefI32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1606	{
1607	Assert(iReg < 16);
1608	return (int32_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1609	}
1610
1611
1612	/**
1613	* Gets a reference (pointer) to the specified 64-bit general purpose register.
1614	*
1615	* @returns Register reference.
1616	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1617	* @param iReg The register.
1618	*/
1619	DECLINLINE(uint64_t *) iemGRegRefU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1620	{
1621	Assert(iReg < 64);
1622	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1623	}
1624
1625
1626	/**
1627	* Gets a reference (pointer) to the specified signed 64-bit general purpose register.
1628	*
1629	* @returns Register reference.
1630	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1631	* @param iReg The register.
1632	*/
1633	DECLINLINE(int64_t *) iemGRegRefI64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1634	{
1635	Assert(iReg < 16);
1636	return (int64_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1637	}
1638
1639
1640	/**
1641	* Gets a reference (pointer) to the specified segment register's base address.
1642	*
1643	* @returns Segment register base address reference.
1644	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1645	* @param iSegReg The segment selector.
1646	*/
1647	DECLINLINE(uint64_t *) iemSRegBaseRefU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1648	{
1649	Assert(iSegReg < X86_SREG_COUNT);
1650	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1651	return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1652	}
1653
1654
1655	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1656	/**
1657	* Fetches the value of a 8-bit general purpose register.
1658	*
1659	* @returns The register value.
1660	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1661	* @param iReg The register.
1662	*/
1663	DECLINLINE(uint8_t) iemGRegFetchU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1664	{
1665	return *iemGRegRefU8(pVCpu, iReg);
1666	}
1667	#endif
1668
1669
1670	/**
1671	* Fetches the value of a 8-bit general purpose register, alternative version
1672	* with extended (20) register index.
1673
1674	* @returns The register value.
1675	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1676	* @param iRegEx The register. The 16 first are regular ones,
1677	* whereas 16 thru 19 maps to AH, CH, DH and BH.
1678	*/
1679	DECLINLINE(uint8_t) iemGRegFetchU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT
1680	{
1681	return *iemGRegRefU8Ex(pVCpu, iRegEx);
1682	}
1683
1684
1685	/**
1686	* Fetches the value of a 16-bit general purpose register.
1687	*
1688	* @returns The register value.
1689	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1690	* @param iReg The register.
1691	*/
1692	DECLINLINE(uint16_t) iemGRegFetchU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1693	{
1694	Assert(iReg < 16);
1695	return pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1696	}
1697
1698
1699	/**
1700	* Fetches the value of a 32-bit general purpose register.
1701	*
1702	* @returns The register value.
1703	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1704	* @param iReg The register.
1705	*/
1706	DECLINLINE(uint32_t) iemGRegFetchU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1707	{
1708	Assert(iReg < 16);
1709	return pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1710	}
1711
1712
1713	/**
1714	* Fetches the value of a 64-bit general purpose register.
1715	*
1716	* @returns The register value.
1717	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1718	* @param iReg The register.
1719	*/
1720	DECLINLINE(uint64_t) iemGRegFetchU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1721	{
1722	Assert(iReg < 16);
1723	return pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1724	}
1725
1726
1727	/**
1728	* Get the address of the top of the stack.
1729	*
1730	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1731	*/
1732	DECLINLINE(RTGCPTR) iemRegGetEffRsp(PCVMCPU pVCpu) RT_NOEXCEPT
1733	{
1734	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1735	return pVCpu->cpum.GstCtx.rsp;
1736	if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1737	return pVCpu->cpum.GstCtx.esp;
1738	return pVCpu->cpum.GstCtx.sp;
1739	}
1740
1741
1742	/**
1743	* Updates the RIP/EIP/IP to point to the next instruction.
1744	*
1745	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1746	* @param cbInstr The number of bytes to add.
1747	*/
1748	DECL_FORCE_INLINE(void) iemRegAddToRip(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
1749	{
1750	/*
1751	* Advance RIP.
1752	*
1753	* When we're targetting 8086/8, 80186/8 or 80286 mode the updates are 16-bit,
1754	* while in all other modes except LM64 the updates are 32-bit. This means
1755	* we need to watch for both 32-bit and 16-bit "carry" situations, i.e.
1756	* 4GB and 64KB rollovers, and decide whether anything needs masking.
1757	*
1758	* See PC wrap around tests in bs3-cpu-weird-1.
1759	*/
1760	uint64_t const uRipPrev = pVCpu->cpum.GstCtx.rip;
1761	uint64_t const uRipNext = uRipPrev + cbInstr;
1762	if (RT_LIKELY( !((uRipNext ^ uRipPrev) & (RT_BIT_64(32) \| RT_BIT_64(16)))
1763	\|\| pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT))
1764	pVCpu->cpum.GstCtx.rip = uRipNext;
1765	else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
1766	pVCpu->cpum.GstCtx.rip = (uint32_t)uRipNext;
1767	else
1768	pVCpu->cpum.GstCtx.rip = (uint16_t)uRipNext;
1769	}
1770
1771
1772	/**
1773	* Updates the EIP/IP to point to the next instruction - only for 32-bit and
1774	* 16-bit code.
1775	*
1776	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1777	* @param cbInstr The number of bytes to add.
1778	*/
1779	DECL_FORCE_INLINE(void) iemRegAddToEip32(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
1780	{
1781	/* See comment in iemRegAddToRip. */
1782	uint32_t const uEipPrev = pVCpu->cpum.GstCtx.eip;
1783	uint32_t const uEipNext = uEipPrev + cbInstr;
1784	if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
1785	pVCpu->cpum.GstCtx.rip = (uint32_t)uEipNext;
1786	else
1787	pVCpu->cpum.GstCtx.rip = (uint16_t)uEipNext;
1788	}
1789
1790
1791	/**
1792	* Called by iemRegAddToRipAndFinishingClearingRF and others when any of the
1793	* following EFLAGS bits are set:
1794	* - X86_EFL_RF - clear it.
1795	* - CPUMCTX_INHIBIT_SHADOW (_SS/_STI) - clear them.
1796	* - X86_EFL_TF - generate single step \#DB trap.
1797	* - CPUMCTX_DBG_HIT_DR0/1/2/3 - generate \#DB trap (data or I/O, not
1798	* instruction).
1799	*
1800	* According to @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events},
1801	* a \#DB due to TF (single stepping) or a DRx non-instruction breakpoint
1802	* takes priority over both NMIs and hardware interrupts. So, neither is
1803	* considered here. (The RESET, \#MC, SMI, INIT, STOPCLK and FLUSH events are
1804	* either unsupported will be triggered on-top of any \#DB raised here.)
1805	*
1806	* The RF flag only needs to be cleared here as it only suppresses instruction
1807	* breakpoints which are not raised here (happens synchronously during
1808	* instruction fetching).
1809	*
1810	* The CPUMCTX_INHIBIT_SHADOW_SS flag will be cleared by this function, so its
1811	* status has no bearing on whether \#DB exceptions are raised.
1812	*
1813	* @note This must NOT be called by the two instructions setting the
1814	* CPUMCTX_INHIBIT_SHADOW_SS flag.
1815	*
1816	* @see @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events}
1817	* @see @sdmv3{077,200,6.8.3,Masking Exceptions and Interrupts When Switching
1818	* Stacks}
1819	*/
1820	static VBOXSTRICTRC iemFinishInstructionWithFlagsSet(PVMCPUCC pVCpu) RT_NOEXCEPT
1821	{
1822	/*
1823	* Normally we're just here to clear RF and/or interrupt shadow bits.
1824	*/
1825	if (RT_LIKELY((pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_TF \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK)) == 0))
1826	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW);
1827	else
1828	{
1829	/*
1830	* Raise a #DB or/and DBGF event.
1831	*/
1832	VBOXSTRICTRC rcStrict;
1833	if (pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_TF \| CPUMCTX_DBG_HIT_DRX_MASK))
1834	{
1835	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
1836	pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
1837	if (pVCpu->cpum.GstCtx.eflags.uBoth & X86_EFL_TF)
1838	pVCpu->cpum.GstCtx.dr[6] \|= X86_DR6_BS;
1839	pVCpu->cpum.GstCtx.dr[6] \|= (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT;
1840	LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64\n",
1841	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
1842	pVCpu->cpum.GstCtx.rflags.uBoth));
1843
1844	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK);
1845	rcStrict = iemRaiseDebugException(pVCpu);
1846
1847	/* A DBGF event/breakpoint trumps the iemRaiseDebugException informational status code. */
1848	if ((pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK) && RT_FAILURE(rcStrict))
1849	{
1850	rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT;
1851	LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));
1852	}
1853	}
1854	else
1855	{
1856	Assert(pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK);
1857	rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT;
1858	LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));
1859	}
1860	pVCpu->cpum.GstCtx.eflags.uBoth &= ~CPUMCTX_DBG_DBGF_MASK;
1861	return rcStrict;
1862	}
1863	return VINF_SUCCESS;
1864	}
1865
1866
1867	/**
1868	* Clears the RF and CPUMCTX_INHIBIT_SHADOW, triggering \#DB if pending.
1869	*
1870	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1871	*/
1872	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT
1873	{
1874	/*
1875	* We assume that most of the time nothing actually needs doing here.
1876	*/
1877	AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX);
1878	if (RT_LIKELY(!( pVCpu->cpum.GstCtx.eflags.uBoth
1879	& (X86_EFL_TF \| X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK)) ))
1880	return VINF_SUCCESS;
1881	return iemFinishInstructionWithFlagsSet(pVCpu);
1882	}
1883
1884
1885	/**
1886	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF
1887	* and CPUMCTX_INHIBIT_SHADOW.
1888	*
1889	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1890	* @param cbInstr The number of bytes to add.
1891	*/
1892	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
1893	{
1894	iemRegAddToRip(pVCpu, cbInstr);
1895	return iemRegFinishClearingRF(pVCpu);
1896	}
1897
1898
1899	/**
1900	* Updates the RIP to point to the next instruction and clears EFLAGS.RF
1901	* and CPUMCTX_INHIBIT_SHADOW.
1902	*
1903	* Only called from 64-code code.
1904	*
1905	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1906	* @param cbInstr The number of bytes to add.
1907	*/
1908	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRip64AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
1909	{
1910	pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.rip + cbInstr;
1911	return iemRegFinishClearingRF(pVCpu);
1912	}
1913
1914
1915	/**
1916	* Updates the EIP to point to the next instruction and clears EFLAGS.RF and
1917	* CPUMCTX_INHIBIT_SHADOW.
1918	*
1919	* This is never from 64-code code.
1920	*
1921	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1922	* @param cbInstr The number of bytes to add.
1923	*/
1924	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToEip32AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
1925	{
1926	iemRegAddToEip32(pVCpu, cbInstr);
1927	return iemRegFinishClearingRF(pVCpu);
1928	}
1929
1930
1931	/**
1932	* Extended version of iemFinishInstructionWithFlagsSet that goes with
1933	* iemRegAddToRipAndFinishingClearingRfEx.
1934	*
1935	* See iemFinishInstructionWithFlagsSet() for details.
1936	*/
1937	static VBOXSTRICTRC iemFinishInstructionWithTfSet(PVMCPUCC pVCpu) RT_NOEXCEPT
1938	{
1939	/*
1940	* Raise a #DB.
1941	*/
1942	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
1943	pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
1944	pVCpu->cpum.GstCtx.dr[6] \|= X86_DR6_BS
1945	\| (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT;
1946	/** @todo Do we set all pending \#DB events, or just one? */
1947	LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64 (popf)\n",
1948	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
1949	pVCpu->cpum.GstCtx.rflags.uBoth));
1950	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK);
1951	return iemRaiseDebugException(pVCpu);
1952	}
1953
1954
1955	/**
1956	* Extended version of iemRegAddToRipAndFinishingClearingRF for use by POPF and
1957	* others potentially updating EFLAGS.TF.
1958	*
1959	* The single step event must be generated using the TF value at the start of
1960	* the instruction, not the new value set by it.
1961	*
1962	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1963	* @param cbInstr The number of bytes to add.
1964	* @param fEflOld The EFLAGS at the start of the instruction
1965	* execution.
1966	*/
1967	DECLINLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRfEx(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t fEflOld) RT_NOEXCEPT
1968	{
1969	iemRegAddToRip(pVCpu, cbInstr);
1970	if (!(fEflOld & X86_EFL_TF))
1971	return iemRegFinishClearingRF(pVCpu);
1972	return iemFinishInstructionWithTfSet(pVCpu);
1973	}
1974
1975
1976	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1977	/**
1978	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
1979	*
1980	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1981	*/
1982	DECLINLINE(VBOXSTRICTRC) iemRegUpdateRipAndFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT
1983	{
1984	return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
1985	}
1986	#endif
1987
1988
1989	/**
1990	* Adds to the stack pointer.
1991	*
1992	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1993	* @param cbToAdd The number of bytes to add (8-bit!).
1994	*/
1995	DECLINLINE(void) iemRegAddToRsp(PVMCPUCC pVCpu, uint8_t cbToAdd) RT_NOEXCEPT
1996	{
1997	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1998	pVCpu->cpum.GstCtx.rsp += cbToAdd;
1999	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2000	pVCpu->cpum.GstCtx.esp += cbToAdd;
2001	else
2002	pVCpu->cpum.GstCtx.sp += cbToAdd;
2003	}
2004
2005
2006	/**
2007	* Subtracts from the stack pointer.
2008	*
2009	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2010	* @param cbToSub The number of bytes to subtract (8-bit!).
2011	*/
2012	DECLINLINE(void) iemRegSubFromRsp(PVMCPUCC pVCpu, uint8_t cbToSub) RT_NOEXCEPT
2013	{
2014	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2015	pVCpu->cpum.GstCtx.rsp -= cbToSub;
2016	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2017	pVCpu->cpum.GstCtx.esp -= cbToSub;
2018	else
2019	pVCpu->cpum.GstCtx.sp -= cbToSub;
2020	}
2021
2022
2023	/**
2024	* Adds to the temporary stack pointer.
2025	*
2026	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2027	* @param pTmpRsp The temporary SP/ESP/RSP to update.
2028	* @param cbToAdd The number of bytes to add (16-bit).
2029	*/
2030	DECLINLINE(void) iemRegAddToRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToAdd) RT_NOEXCEPT
2031	{
2032	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2033	pTmpRsp->u += cbToAdd;
2034	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2035	pTmpRsp->DWords.dw0 += cbToAdd;
2036	else
2037	pTmpRsp->Words.w0 += cbToAdd;
2038	}
2039
2040
2041	/**
2042	* Subtracts from the temporary stack pointer.
2043	*
2044	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2045	* @param pTmpRsp The temporary SP/ESP/RSP to update.
2046	* @param cbToSub The number of bytes to subtract.
2047	* @remarks The @a cbToSub argument MUST be 16-bit, iemCImpl_enter is
2048	* expecting that.
2049	*/
2050	DECLINLINE(void) iemRegSubFromRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToSub) RT_NOEXCEPT
2051	{
2052	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2053	pTmpRsp->u -= cbToSub;
2054	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2055	pTmpRsp->DWords.dw0 -= cbToSub;
2056	else
2057	pTmpRsp->Words.w0 -= cbToSub;
2058	}
2059
2060
2061	/**
2062	* Calculates the effective stack address for a push of the specified size as
2063	* well as the new RSP value (upper bits may be masked).
2064	*
2065	* @returns Effective stack addressf for the push.
2066	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2067	* @param cbItem The size of the stack item to pop.
2068	* @param puNewRsp Where to return the new RSP value.
2069	*/
2070	DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT
2071	{
2072	RTUINT64U uTmpRsp;
2073	RTGCPTR GCPtrTop;
2074	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
2075
2076	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2077	GCPtrTop = uTmpRsp.u -= cbItem;
2078	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2079	GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem;
2080	else
2081	GCPtrTop = uTmpRsp.Words.w0 -= cbItem;
2082	*puNewRsp = uTmpRsp.u;
2083	return GCPtrTop;
2084	}
2085
2086
2087	/**
2088	* Gets the current stack pointer and calculates the value after a pop of the
2089	* specified size.
2090	*
2091	* @returns Current stack pointer.
2092	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2093	* @param cbItem The size of the stack item to pop.
2094	* @param puNewRsp Where to return the new RSP value.
2095	*/
2096	DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT
2097	{
2098	RTUINT64U uTmpRsp;
2099	RTGCPTR GCPtrTop;
2100	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
2101
2102	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2103	{
2104	GCPtrTop = uTmpRsp.u;
2105	uTmpRsp.u += cbItem;
2106	}
2107	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2108	{
2109	GCPtrTop = uTmpRsp.DWords.dw0;
2110	uTmpRsp.DWords.dw0 += cbItem;
2111	}
2112	else
2113	{
2114	GCPtrTop = uTmpRsp.Words.w0;
2115	uTmpRsp.Words.w0 += cbItem;
2116	}
2117	*puNewRsp = uTmpRsp.u;
2118	return GCPtrTop;
2119	}
2120
2121
2122	/**
2123	* Calculates the effective stack address for a push of the specified size as
2124	* well as the new temporary RSP value (upper bits may be masked).
2125	*
2126	* @returns Effective stack addressf for the push.
2127	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2128	* @param pTmpRsp The temporary stack pointer. This is updated.
2129	* @param cbItem The size of the stack item to pop.
2130	*/
2131	DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT
2132	{
2133	RTGCPTR GCPtrTop;
2134
2135	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2136	GCPtrTop = pTmpRsp->u -= cbItem;
2137	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2138	GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem;
2139	else
2140	GCPtrTop = pTmpRsp->Words.w0 -= cbItem;
2141	return GCPtrTop;
2142	}
2143
2144
2145	/**
2146	* Gets the effective stack address for a pop of the specified size and
2147	* calculates and updates the temporary RSP.
2148	*
2149	* @returns Current stack pointer.
2150	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2151	* @param pTmpRsp The temporary stack pointer. This is updated.
2152	* @param cbItem The size of the stack item to pop.
2153	*/
2154	DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT
2155	{
2156	RTGCPTR GCPtrTop;
2157	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2158	{
2159	GCPtrTop = pTmpRsp->u;
2160	pTmpRsp->u += cbItem;
2161	}
2162	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2163	{
2164	GCPtrTop = pTmpRsp->DWords.dw0;
2165	pTmpRsp->DWords.dw0 += cbItem;
2166	}
2167	else
2168	{
2169	GCPtrTop = pTmpRsp->Words.w0;
2170	pTmpRsp->Words.w0 += cbItem;
2171	}
2172	return GCPtrTop;
2173	}
2174
2175	/** @} */
2176
2177
2178	/** @name FPU access and helpers.
2179	*
2180	* @{
2181	*/
2182
2183
2184	/**
2185	* Hook for preparing to use the host FPU.
2186	*
2187	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2188	*
2189	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2190	*/
2191	DECLINLINE(void) iemFpuPrepareUsage(PVMCPUCC pVCpu) RT_NOEXCEPT
2192	{
2193	#ifdef IN_RING3
2194	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2195	#else
2196	CPUMRZFpuStatePrepareHostCpuForUse(pVCpu);
2197	#endif
2198	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2199	}
2200
2201
2202	/**
2203	* Hook for preparing to use the host FPU for SSE.
2204	*
2205	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2206	*
2207	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2208	*/
2209	DECLINLINE(void) iemFpuPrepareUsageSse(PVMCPUCC pVCpu) RT_NOEXCEPT
2210	{
2211	iemFpuPrepareUsage(pVCpu);
2212	}
2213
2214
2215	/**
2216	* Hook for preparing to use the host FPU for AVX.
2217	*
2218	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2219	*
2220	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2221	*/
2222	DECLINLINE(void) iemFpuPrepareUsageAvx(PVMCPUCC pVCpu) RT_NOEXCEPT
2223	{
2224	iemFpuPrepareUsage(pVCpu);
2225	}
2226
2227
2228	/**
2229	* Hook for actualizing the guest FPU state before the interpreter reads it.
2230	*
2231	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2232	*
2233	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2234	*/
2235	DECLINLINE(void) iemFpuActualizeStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2236	{
2237	#ifdef IN_RING3
2238	NOREF(pVCpu);
2239	#else
2240	CPUMRZFpuStateActualizeForRead(pVCpu);
2241	#endif
2242	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2243	}
2244
2245
2246	/**
2247	* Hook for actualizing the guest FPU state before the interpreter changes it.
2248	*
2249	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2250	*
2251	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2252	*/
2253	DECLINLINE(void) iemFpuActualizeStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2254	{
2255	#ifdef IN_RING3
2256	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2257	#else
2258	CPUMRZFpuStateActualizeForChange(pVCpu);
2259	#endif
2260	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2261	}
2262
2263
2264	/**
2265	* Hook for actualizing the guest XMM0..15 and MXCSR register state for read
2266	* only.
2267	*
2268	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2269	*
2270	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2271	*/
2272	DECLINLINE(void) iemFpuActualizeSseStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2273	{
2274	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
2275	NOREF(pVCpu);
2276	#else
2277	CPUMRZFpuStateActualizeSseForRead(pVCpu);
2278	#endif
2279	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2280	}
2281
2282
2283	/**
2284	* Hook for actualizing the guest XMM0..15 and MXCSR register state for
2285	* read+write.
2286	*
2287	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2288	*
2289	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2290	*/
2291	DECLINLINE(void) iemFpuActualizeSseStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2292	{
2293	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
2294	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2295	#else
2296	CPUMRZFpuStateActualizeForChange(pVCpu);
2297	#endif
2298	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2299
2300	/* Make sure any changes are loaded the next time around. */
2301	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_SSE;
2302	}
2303
2304
2305	/**
2306	* Hook for actualizing the guest YMM0..15 and MXCSR register state for read
2307	* only.
2308	*
2309	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2310	*
2311	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2312	*/
2313	DECLINLINE(void) iemFpuActualizeAvxStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2314	{
2315	#ifdef IN_RING3
2316	NOREF(pVCpu);
2317	#else
2318	CPUMRZFpuStateActualizeAvxForRead(pVCpu);
2319	#endif
2320	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2321	}
2322
2323
2324	/**
2325	* Hook for actualizing the guest YMM0..15 and MXCSR register state for
2326	* read+write.
2327	*
2328	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2329	*
2330	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2331	*/
2332	DECLINLINE(void) iemFpuActualizeAvxStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2333	{
2334	#ifdef IN_RING3
2335	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2336	#else
2337	CPUMRZFpuStateActualizeForChange(pVCpu);
2338	#endif
2339	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2340
2341	/* Just assume we're going to make changes to the SSE and YMM_HI parts. */
2342	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_YMM \| XSAVE_C_SSE;
2343	}
2344
2345
2346	/**
2347	* Stores a QNaN value into a FPU register.
2348	*
2349	* @param pReg Pointer to the register.
2350	*/
2351	DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg) RT_NOEXCEPT
2352	{
2353	pReg->au32[0] = UINT32_C(0x00000000);
2354	pReg->au32[1] = UINT32_C(0xc0000000);
2355	pReg->au16[4] = UINT16_C(0xffff);
2356	}
2357
2358
2359	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
2360	/**
2361	* Updates the FOP, FPU.CS and FPUIP registers.
2362	*
2363	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2364	* @param pFpuCtx The FPU context.
2365	*/
2366	DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorker(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx) RT_NOEXCEPT
2367	{
2368	Assert(pVCpu->iem.s.uFpuOpcode != UINT16_MAX);
2369	pFpuCtx->FOP = pVCpu->iem.s.uFpuOpcode;
2370	/** @todo x87.CS and FPUIP needs to be kept seperately. */
2371	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
2372	{
2373	/** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled
2374	* happens in real mode here based on the fnsave and fnstenv images. */
2375	pFpuCtx->CS = 0;
2376	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.eip \| ((uint32_t)pVCpu->cpum.GstCtx.cs.Sel << 4);
2377	}
2378	else if (!IEM_IS_LONG_MODE(pVCpu))
2379	{
2380	pFpuCtx->CS = pVCpu->cpum.GstCtx.cs.Sel;
2381	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2382	}
2383	else
2384	(uint64_t )&pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2385	}
2386	#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
2387
2388
2389
2390
2391	/**
2392	* Marks the specified stack register as free (for FFREE).
2393	*
2394	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2395	* @param iStReg The register to free.
2396	*/
2397	DECLINLINE(void) iemFpuStackFree(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
2398	{
2399	Assert(iStReg < 8);
2400	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2401	uint8_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2402	pFpuCtx->FTW &= ~RT_BIT(iReg);
2403	}
2404
2405
2406	/**
2407	* Increments FSW.TOP, i.e. pops an item off the stack without freeing it.
2408	*
2409	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2410	*/
2411	DECLINLINE(void) iemFpuStackIncTop(PVMCPUCC pVCpu) RT_NOEXCEPT
2412	{
2413	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2414	uint16_t uFsw = pFpuCtx->FSW;
2415	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2416	uTop = (uTop + (1 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2417	uFsw &= ~X86_FSW_TOP_MASK;
2418	uFsw \|= uTop;
2419	pFpuCtx->FSW = uFsw;
2420	}
2421
2422
2423	/**
2424	* Decrements FSW.TOP, i.e. push an item off the stack without storing anything.
2425	*
2426	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2427	*/
2428	DECLINLINE(void) iemFpuStackDecTop(PVMCPUCC pVCpu) RT_NOEXCEPT
2429	{
2430	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2431	uint16_t uFsw = pFpuCtx->FSW;
2432	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2433	uTop = (uTop + (7 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2434	uFsw &= ~X86_FSW_TOP_MASK;
2435	uFsw \|= uTop;
2436	pFpuCtx->FSW = uFsw;
2437	}
2438
2439
2440
2441
2442	DECLINLINE(int) iemFpuStRegNotEmpty(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
2443	{
2444	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2445	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2446	if (pFpuCtx->FTW & RT_BIT(iReg))
2447	return VINF_SUCCESS;
2448	return VERR_NOT_FOUND;
2449	}
2450
2451
2452	DECLINLINE(int) iemFpuStRegNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef) RT_NOEXCEPT
2453	{
2454	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2455	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2456	if (pFpuCtx->FTW & RT_BIT(iReg))
2457	{
2458	*ppRef = &pFpuCtx->aRegs[iStReg].r80;
2459	return VINF_SUCCESS;
2460	}
2461	return VERR_NOT_FOUND;
2462	}
2463
2464
2465	DECLINLINE(int) iemFpu2StRegsNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0,
2466	uint8_t iStReg1, PCRTFLOAT80U *ppRef1) RT_NOEXCEPT
2467	{
2468	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2469	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2470	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2471	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2472	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
2473	{
2474	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2475	*ppRef1 = &pFpuCtx->aRegs[iStReg1].r80;
2476	return VINF_SUCCESS;
2477	}
2478	return VERR_NOT_FOUND;
2479	}
2480
2481
2482	DECLINLINE(int) iemFpu2StRegsNotEmptyRefFirst(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1) RT_NOEXCEPT
2483	{
2484	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2485	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2486	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2487	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2488	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
2489	{
2490	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2491	return VINF_SUCCESS;
2492	}
2493	return VERR_NOT_FOUND;
2494	}
2495
2496
2497	/**
2498	* Rotates the stack registers when setting new TOS.
2499	*
2500	* @param pFpuCtx The FPU context.
2501	* @param iNewTop New TOS value.
2502	* @remarks We only do this to speed up fxsave/fxrstor which
2503	* arrange the FP registers in stack order.
2504	* MUST be done before writing the new TOS (FSW).
2505	*/
2506	DECLINLINE(void) iemFpuRotateStackSetTop(PX86FXSTATE pFpuCtx, uint16_t iNewTop) RT_NOEXCEPT
2507	{
2508	uint16_t iOldTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2509	RTFLOAT80U ar80Temp[8];
2510
2511	if (iOldTop == iNewTop)
2512	return;
2513
2514	/* Unscrew the stack and get it into 'native' order. */
2515	ar80Temp[0] = pFpuCtx->aRegs[(8 - iOldTop + 0) & X86_FSW_TOP_SMASK].r80;
2516	ar80Temp[1] = pFpuCtx->aRegs[(8 - iOldTop + 1) & X86_FSW_TOP_SMASK].r80;
2517	ar80Temp[2] = pFpuCtx->aRegs[(8 - iOldTop + 2) & X86_FSW_TOP_SMASK].r80;
2518	ar80Temp[3] = pFpuCtx->aRegs[(8 - iOldTop + 3) & X86_FSW_TOP_SMASK].r80;
2519	ar80Temp[4] = pFpuCtx->aRegs[(8 - iOldTop + 4) & X86_FSW_TOP_SMASK].r80;
2520	ar80Temp[5] = pFpuCtx->aRegs[(8 - iOldTop + 5) & X86_FSW_TOP_SMASK].r80;
2521	ar80Temp[6] = pFpuCtx->aRegs[(8 - iOldTop + 6) & X86_FSW_TOP_SMASK].r80;
2522	ar80Temp[7] = pFpuCtx->aRegs[(8 - iOldTop + 7) & X86_FSW_TOP_SMASK].r80;
2523
2524	/* Now rotate the stack to the new position. */
2525	pFpuCtx->aRegs[0].r80 = ar80Temp[(iNewTop + 0) & X86_FSW_TOP_SMASK];
2526	pFpuCtx->aRegs[1].r80 = ar80Temp[(iNewTop + 1) & X86_FSW_TOP_SMASK];
2527	pFpuCtx->aRegs[2].r80 = ar80Temp[(iNewTop + 2) & X86_FSW_TOP_SMASK];
2528	pFpuCtx->aRegs[3].r80 = ar80Temp[(iNewTop + 3) & X86_FSW_TOP_SMASK];
2529	pFpuCtx->aRegs[4].r80 = ar80Temp[(iNewTop + 4) & X86_FSW_TOP_SMASK];
2530	pFpuCtx->aRegs[5].r80 = ar80Temp[(iNewTop + 5) & X86_FSW_TOP_SMASK];
2531	pFpuCtx->aRegs[6].r80 = ar80Temp[(iNewTop + 6) & X86_FSW_TOP_SMASK];
2532	pFpuCtx->aRegs[7].r80 = ar80Temp[(iNewTop + 7) & X86_FSW_TOP_SMASK];
2533	}
2534
2535
2536	/**
2537	* Updates the FPU exception status after FCW is changed.
2538	*
2539	* @param pFpuCtx The FPU context.
2540	*/
2541	DECLINLINE(void) iemFpuRecalcExceptionStatus(PX86FXSTATE pFpuCtx) RT_NOEXCEPT
2542	{
2543	uint16_t u16Fsw = pFpuCtx->FSW;
2544	if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pFpuCtx->FCW & X86_FCW_XCPT_MASK))
2545	u16Fsw \|= X86_FSW_ES \| X86_FSW_B;
2546	else
2547	u16Fsw &= ~(X86_FSW_ES \| X86_FSW_B);
2548	pFpuCtx->FSW = u16Fsw;
2549	}
2550
2551
2552	/**
2553	* Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE.
2554	*
2555	* @returns The full FTW.
2556	* @param pFpuCtx The FPU context.
2557	*/
2558	DECLINLINE(uint16_t) iemFpuCalcFullFtw(PCX86FXSTATE pFpuCtx) RT_NOEXCEPT
2559	{
2560	uint8_t const u8Ftw = (uint8_t)pFpuCtx->FTW;
2561	uint16_t u16Ftw = 0;
2562	unsigned const iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2563	for (unsigned iSt = 0; iSt < 8; iSt++)
2564	{
2565	unsigned const iReg = (iSt + iTop) & 7;
2566	if (!(u8Ftw & RT_BIT(iReg)))
2567	u16Ftw \|= 3 << (iReg * 2); /* empty */
2568	else
2569	{
2570	uint16_t uTag;
2571	PCRTFLOAT80U const pr80Reg = &pFpuCtx->aRegs[iSt].r80;
2572	if (pr80Reg->s.uExponent == 0x7fff)
2573	uTag = 2; /* Exponent is all 1's => Special. */
2574	else if (pr80Reg->s.uExponent == 0x0000)
2575	{
2576	if (pr80Reg->s.uMantissa == 0x0000)
2577	uTag = 1; /* All bits are zero => Zero. */
2578	else
2579	uTag = 2; /* Must be special. */
2580	}
2581	else if (pr80Reg->s.uMantissa & RT_BIT_64(63)) /* The J bit. */
2582	uTag = 0; /* Valid. */
2583	else
2584	uTag = 2; /* Must be special. */
2585
2586	u16Ftw \|= uTag << (iReg * 2);
2587	}
2588	}
2589
2590	return u16Ftw;
2591	}
2592
2593
2594	/**
2595	* Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR).
2596	*
2597	* @returns The compressed FTW.
2598	* @param u16FullFtw The full FTW to convert.
2599	*/
2600	DECLINLINE(uint16_t) iemFpuCompressFtw(uint16_t u16FullFtw) RT_NOEXCEPT
2601	{
2602	uint8_t u8Ftw = 0;
2603	for (unsigned i = 0; i < 8; i++)
2604	{
2605	if ((u16FullFtw & 3) != 3 /empty/)
2606	u8Ftw \|= RT_BIT(i);
2607	u16FullFtw >>= 2;
2608	}
2609
2610	return u8Ftw;
2611	}
2612
2613	/** @} */
2614
2615
2616	/** @name Memory access.
2617	*
2618	* @{
2619	*/
2620
2621
2622	/**
2623	* Checks whether alignment checks are enabled or not.
2624	*
2625	* @returns true if enabled, false if not.
2626	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2627	*/
2628	DECLINLINE(bool) iemMemAreAlignmentChecksEnabled(PVMCPUCC pVCpu) RT_NOEXCEPT
2629	{
2630	AssertCompile(X86_CR0_AM == X86_EFL_AC);
2631	return pVCpu->iem.s.uCpl == 3
2632	&& (((uint32_t)pVCpu->cpum.GstCtx.cr0 & pVCpu->cpum.GstCtx.eflags.u) & X86_CR0_AM);
2633	}
2634
2635	/**
2636	* Checks if the given segment can be written to, raise the appropriate
2637	* exception if not.
2638	*
2639	* @returns VBox strict status code.
2640	*
2641	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2642	* @param pHid Pointer to the hidden register.
2643	* @param iSegReg The register number.
2644	* @param pu64BaseAddr Where to return the base address to use for the
2645	* segment. (In 64-bit code it may differ from the
2646	* base in the hidden segment.)
2647	*/
2648	DECLINLINE(VBOXSTRICTRC) iemMemSegCheckWriteAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid,
2649	uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT
2650	{
2651	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2652
2653	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2654	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
2655	else
2656	{
2657	if (!pHid->Attr.n.u1Present)
2658	{
2659	uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
2660	AssertRelease(uSel == 0);
2661	Log(("iemMemSegCheckWriteAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
2662	return iemRaiseGeneralProtectionFault0(pVCpu);
2663	}
2664
2665	if ( ( (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE)
2666	\|\| !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
2667	&& pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT )
2668	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
2669	*pu64BaseAddr = pHid->u64Base;
2670	}
2671	return VINF_SUCCESS;
2672	}
2673
2674
2675	/**
2676	* Checks if the given segment can be read from, raise the appropriate
2677	* exception if not.
2678	*
2679	* @returns VBox strict status code.
2680	*
2681	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2682	* @param pHid Pointer to the hidden register.
2683	* @param iSegReg The register number.
2684	* @param pu64BaseAddr Where to return the base address to use for the
2685	* segment. (In 64-bit code it may differ from the
2686	* base in the hidden segment.)
2687	*/
2688	DECLINLINE(VBOXSTRICTRC) iemMemSegCheckReadAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid,
2689	uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT
2690	{
2691	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2692
2693	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2694	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
2695	else
2696	{
2697	if (!pHid->Attr.n.u1Present)
2698	{
2699	uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
2700	AssertRelease(uSel == 0);
2701	Log(("iemMemSegCheckReadAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
2702	return iemRaiseGeneralProtectionFault0(pVCpu);
2703	}
2704
2705	if ((pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
2706	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
2707	*pu64BaseAddr = pHid->u64Base;
2708	}
2709	return VINF_SUCCESS;
2710	}
2711
2712
2713	/**
2714	* Maps a physical page.
2715	*
2716	* @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr).
2717	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2718	* @param GCPhysMem The physical address.
2719	* @param fAccess The intended access.
2720	* @param ppvMem Where to return the mapping address.
2721	* @param pLock The PGM lock.
2722	*/
2723	DECLINLINE(int) iemMemPageMap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess,
2724	void **ppvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT
2725	{
2726	#ifdef IEM_LOG_MEMORY_WRITES
2727	if (fAccess & IEM_ACCESS_TYPE_WRITE)
2728	return VERR_PGM_PHYS_TLB_CATCH_ALL;
2729	#endif
2730
2731	/** @todo This API may require some improving later. A private deal with PGM
2732	* regarding locking and unlocking needs to be struct. A couple of TLBs
2733	* living in PGM, but with publicly accessible inlined access methods
2734	* could perhaps be an even better solution. */
2735	int rc = PGMPhysIemGCPhys2Ptr(pVCpu->CTX_SUFF(pVM), pVCpu,
2736	GCPhysMem,
2737	RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE),
2738	pVCpu->iem.s.fBypassHandlers,
2739	ppvMem,
2740	pLock);
2741	/Log(("PGMPhysIemGCPhys2Ptr %Rrc pLock=%.Rhxs\n", rc, sizeof(pLock), pLock));/
2742	AssertMsg(rc == VINF_SUCCESS \|\| RT_FAILURE_NP(rc), ("%Rrc\n", rc));
2743
2744	return rc;
2745	}
2746
2747
2748	/**
2749	* Unmap a page previously mapped by iemMemPageMap.
2750	*
2751	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2752	* @param GCPhysMem The physical address.
2753	* @param fAccess The intended access.
2754	* @param pvMem What iemMemPageMap returned.
2755	* @param pLock The PGM lock.
2756	*/
2757	DECLINLINE(void) iemMemPageUnmap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess,
2758	const void *pvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT
2759	{
2760	NOREF(pVCpu);
2761	NOREF(GCPhysMem);
2762	NOREF(fAccess);
2763	NOREF(pvMem);
2764	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), pLock);
2765	}
2766
2767	#ifdef IEM_WITH_SETJMP
2768
2769	/** @todo slim this down */
2770	DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToReadJmp(PVMCPUCC pVCpu, uint8_t iSegReg,
2771	size_t cbMem, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
2772	{
2773	Assert(cbMem >= 1);
2774	Assert(iSegReg < X86_SREG_COUNT);
2775
2776	/*
2777	* 64-bit mode is simpler.
2778	*/
2779	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2780	{
2781	if (iSegReg >= X86_SREG_FS && iSegReg != UINT8_MAX)
2782	{
2783	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2784	PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
2785	GCPtrMem += pSel->u64Base;
2786	}
2787
2788	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
2789	return GCPtrMem;
2790	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
2791	}
2792	/*
2793	* 16-bit and 32-bit segmentation.
2794	*/
2795	else if (iSegReg != UINT8_MAX)
2796	{
2797	/** @todo Does this apply to segments with 4G-1 limit? */
2798	uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
2799	if (RT_LIKELY(GCPtrLast32 >= (uint32_t)GCPtrMem))
2800	{
2801	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2802	PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
2803	switch (pSel->Attr.u & ( X86DESCATTR_P \| X86DESCATTR_UNUSABLE
2804	\| X86_SEL_TYPE_READ \| X86_SEL_TYPE_WRITE /* same as read */
2805	\| X86_SEL_TYPE_DOWN \| X86_SEL_TYPE_CONF /* same as down */
2806	\| X86_SEL_TYPE_CODE))
2807	{
2808	case X86DESCATTR_P: /* readonly data, expand up */
2809	case X86DESCATTR_P \| X86_SEL_TYPE_WRITE: /* writable data, expand up */
2810	case X86DESCATTR_P \| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ: /* code, read-only */
2811	case X86DESCATTR_P \| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ \| X86_SEL_TYPE_CONF: /* conforming code, read-only */
2812	/* expand up */
2813	if (RT_LIKELY(GCPtrLast32 <= pSel->u32Limit))
2814	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2815	Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x vs %#x\n",
2816	(uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit));
2817	break;
2818
2819	case X86DESCATTR_P \| X86_SEL_TYPE_DOWN: /* readonly data, expand down */
2820	case X86DESCATTR_P \| X86_SEL_TYPE_DOWN \| X86_SEL_TYPE_WRITE: /* writable data, expand down */
2821	/* expand down */
2822	if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
2823	&& ( pSel->Attr.n.u1DefBig
2824	\|\| GCPtrLast32 <= UINT32_C(0xffff)) ))
2825	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2826	Log10(("iemMemApplySegmentToReadJmp: expand down out of bounds %#x..%#x vs %#x..%#x\n",
2827	(uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit, pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT16_MAX));
2828	break;
2829
2830	default:
2831	Log10(("iemMemApplySegmentToReadJmp: bad selector %#x\n", pSel->Attr.u));
2832	iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
2833	break;
2834	}
2835	}
2836	Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x\n",(uint32_t)GCPtrMem, GCPtrLast32));
2837	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
2838	}
2839	/*
2840	* 32-bit flat address.
2841	*/
2842	else
2843	return GCPtrMem;
2844	}
2845
2846
2847	/** @todo slim this down */
2848	DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToWriteJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem,
2849	RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
2850	{
2851	Assert(cbMem >= 1);
2852	Assert(iSegReg < X86_SREG_COUNT);
2853
2854	/*
2855	* 64-bit mode is simpler.
2856	*/
2857	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2858	{
2859	if (iSegReg >= X86_SREG_FS)
2860	{
2861	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2862	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
2863	GCPtrMem += pSel->u64Base;
2864	}
2865
2866	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
2867	return GCPtrMem;
2868	}
2869	/*
2870	* 16-bit and 32-bit segmentation.
2871	*/
2872	else
2873	{
2874	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2875	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
2876	uint32_t const fRelevantAttrs = pSel->Attr.u & ( X86DESCATTR_P \| X86DESCATTR_UNUSABLE
2877	\| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_WRITE \| X86_SEL_TYPE_DOWN);
2878	if (fRelevantAttrs == (X86DESCATTR_P \| X86_SEL_TYPE_WRITE)) /* data, expand up */
2879	{
2880	/* expand up */
2881	uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
2882	if (RT_LIKELY( GCPtrLast32 > pSel->u32Limit
2883	&& GCPtrLast32 > (uint32_t)GCPtrMem))
2884	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2885	}
2886	else if (fRelevantAttrs == (X86DESCATTR_P \| X86_SEL_TYPE_WRITE \| X86_SEL_TYPE_DOWN)) /* data, expand up */
2887	{
2888	/* expand down */
2889	uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
2890	if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
2891	&& GCPtrLast32 <= (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
2892	&& GCPtrLast32 > (uint32_t)GCPtrMem))
2893	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2894	}
2895	else
2896	iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
2897	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
2898	}
2899	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
2900	}
2901
2902	#endif /* IEM_WITH_SETJMP */
2903
2904	/**
2905	* Fakes a long mode stack selector for SS = 0.
2906	*
2907	* @param pDescSs Where to return the fake stack descriptor.
2908	* @param uDpl The DPL we want.
2909	*/
2910	DECLINLINE(void) iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl) RT_NOEXCEPT
2911	{
2912	pDescSs->Long.au64[0] = 0;
2913	pDescSs->Long.au64[1] = 0;
2914	pDescSs->Long.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
2915	pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */
2916	pDescSs->Long.Gen.u2Dpl = uDpl;
2917	pDescSs->Long.Gen.u1Present = 1;
2918	pDescSs->Long.Gen.u1Long = 1;
2919	}
2920
2921	/** @} */
2922
2923
2924	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2925
2926	/**
2927	* Gets CR0 fixed-0 bits in VMX operation.
2928	*
2929	* We do this rather than fetching what we report to the guest (in
2930	* IA32_VMX_CR0_FIXED0 MSR) because real hardware (and so do we) report the same
2931	* values regardless of whether unrestricted-guest feature is available on the CPU.
2932	*
2933	* @returns CR0 fixed-0 bits.
2934	* @param pVCpu The cross context virtual CPU structure.
2935	* @param fVmxNonRootMode Whether the CR0 fixed-0 bits for VMX non-root mode
2936	* must be returned. When @c false, the CR0 fixed-0
2937	* bits for VMX root mode is returned.
2938	*
2939	*/
2940	DECLINLINE(uint64_t) iemVmxGetCr0Fixed0(PCVMCPUCC pVCpu, bool fVmxNonRootMode) RT_NOEXCEPT
2941	{
2942	Assert(IEM_VMX_IS_ROOT_MODE(pVCpu));
2943
2944	PCVMXMSRS pMsrs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs;
2945	if ( fVmxNonRootMode
2946	&& (pMsrs->ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST))
2947	return VMX_V_CR0_FIXED0_UX;
2948	return VMX_V_CR0_FIXED0;
2949	}
2950
2951
2952	/**
2953	* Sets virtual-APIC write emulation as pending.
2954	*
2955	* @param pVCpu The cross context virtual CPU structure.
2956	* @param offApic The offset in the virtual-APIC page that was written.
2957	*/
2958	DECLINLINE(void) iemVmxVirtApicSetPendingWrite(PVMCPUCC pVCpu, uint16_t offApic) RT_NOEXCEPT
2959	{
2960	Assert(offApic < XAPIC_OFF_END + 4);
2961
2962	/*
2963	* Record the currently updated APIC offset, as we need this later for figuring
2964	* out whether to perform TPR, EOI or self-IPI virtualization as well as well
2965	* as for supplying the exit qualification when causing an APIC-write VM-exit.
2966	*/
2967	pVCpu->cpum.GstCtx.hwvirt.vmx.offVirtApicWrite = offApic;
2968
2969	/*
2970	* Flag that we need to perform virtual-APIC write emulation (TPR/PPR/EOI/Self-IPI
2971	* virtualization or APIC-write emulation).
2972	*/
2973	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
2974	VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE);
2975	}
2976
2977	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
2978
2979	#endif /* !VMM_INCLUDED_SRC_include_IEMInline_h */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMInline.h@ 99385

Download in other formats: