IEMInline.h@ 99296

Last change on this file since 99296 was 99296, checked in by vboxsync, 19 months ago
VMM/IEM: More work on processing MC blocks and generating threaded functions from them. bugref:10369
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 100.0 KB

Line
1	/* $Id: IEMInline.h 99296 2023-04-05 10:15:47Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Inlined Functions.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMInline_h
29	#define VMM_INCLUDED_SRC_include_IEMInline_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35
36	/**
37	* Makes status code addjustments (pass up from I/O and access handler)
38	* as well as maintaining statistics.
39	*
40	* @returns Strict VBox status code to pass up.
41	* @param pVCpu The cross context virtual CPU structure of the calling thread.
42	* @param rcStrict The status from executing an instruction.
43	*/
44	DECL_FORCE_INLINE(VBOXSTRICTRC) iemExecStatusCodeFiddling(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
45	{
46	if (rcStrict != VINF_SUCCESS)
47	{
48	if (RT_SUCCESS(rcStrict))
49	{
50	AssertMsg( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
51	\|\| rcStrict == VINF_IOM_R3_IOPORT_READ
52	\|\| rcStrict == VINF_IOM_R3_IOPORT_WRITE
53	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
54	\|\| rcStrict == VINF_IOM_R3_MMIO_READ
55	\|\| rcStrict == VINF_IOM_R3_MMIO_READ_WRITE
56	\|\| rcStrict == VINF_IOM_R3_MMIO_WRITE
57	\|\| rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE
58	\|\| rcStrict == VINF_CPUM_R3_MSR_READ
59	\|\| rcStrict == VINF_CPUM_R3_MSR_WRITE
60	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
61	\|\| rcStrict == VINF_EM_RAW_TO_R3
62	\|\| rcStrict == VINF_EM_TRIPLE_FAULT
63	\|\| rcStrict == VINF_GIM_R3_HYPERCALL
64	/* raw-mode / virt handlers only: */
65	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT
66	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT
67	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT
68	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT
69	\|\| rcStrict == VINF_SELM_SYNC_GDT
70	\|\| rcStrict == VINF_CSAM_PENDING_ACTION
71	\|\| rcStrict == VINF_PATM_CHECK_PATCH_PAGE
72	/* nested hw.virt codes: */
73	\|\| rcStrict == VINF_VMX_VMEXIT
74	\|\| rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE
75	\|\| rcStrict == VINF_VMX_MODIFIES_BEHAVIOR
76	\|\| rcStrict == VINF_SVM_VMEXIT
77	, ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
78	/** @todo adjust for VINF_EM_RAW_EMULATE_INSTR. */
79	int32_t const rcPassUp = pVCpu->iem.s.rcPassUp;
80	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
81	if ( rcStrict == VINF_VMX_VMEXIT
82	&& rcPassUp == VINF_SUCCESS)
83	rcStrict = VINF_SUCCESS;
84	else
85	#endif
86	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
87	if ( rcStrict == VINF_SVM_VMEXIT
88	&& rcPassUp == VINF_SUCCESS)
89	rcStrict = VINF_SUCCESS;
90	else
91	#endif
92	if (rcPassUp == VINF_SUCCESS)
93	pVCpu->iem.s.cRetInfStatuses++;
94	else if ( rcPassUp < VINF_EM_FIRST
95	\|\| rcPassUp > VINF_EM_LAST
96	\|\| rcPassUp < VBOXSTRICTRC_VAL(rcStrict))
97	{
98	Log(("IEM: rcPassUp=%Rrc! rcStrict=%Rrc\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
99	pVCpu->iem.s.cRetPassUpStatus++;
100	rcStrict = rcPassUp;
101	}
102	else
103	{
104	Log(("IEM: rcPassUp=%Rrc rcStrict=%Rrc!\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
105	pVCpu->iem.s.cRetInfStatuses++;
106	}
107	}
108	else if (rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED)
109	pVCpu->iem.s.cRetAspectNotImplemented++;
110	else if (rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
111	pVCpu->iem.s.cRetInstrNotImplemented++;
112	else
113	pVCpu->iem.s.cRetErrStatuses++;
114	}
115	else if (pVCpu->iem.s.rcPassUp != VINF_SUCCESS)
116	{
117	pVCpu->iem.s.cRetPassUpStatus++;
118	rcStrict = pVCpu->iem.s.rcPassUp;
119	}
120
121	return rcStrict;
122	}
123
124
125	/**
126	* Sets the pass up status.
127	*
128	* @returns VINF_SUCCESS.
129	* @param pVCpu The cross context virtual CPU structure of the
130	* calling thread.
131	* @param rcPassUp The pass up status. Must be informational.
132	* VINF_SUCCESS is not allowed.
133	*/
134	DECLINLINE(int) iemSetPassUpStatus(PVMCPUCC pVCpu, VBOXSTRICTRC rcPassUp) RT_NOEXCEPT
135	{
136	AssertRC(VBOXSTRICTRC_VAL(rcPassUp)); Assert(rcPassUp != VINF_SUCCESS);
137
138	int32_t const rcOldPassUp = pVCpu->iem.s.rcPassUp;
139	if (rcOldPassUp == VINF_SUCCESS)
140	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
141	/* If both are EM scheduling codes, use EM priority rules. */
142	else if ( rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST
143	&& rcPassUp >= VINF_EM_FIRST && rcPassUp <= VINF_EM_LAST)
144	{
145	if (rcPassUp < rcOldPassUp)
146	{
147	Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
148	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
149	}
150	else
151	Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
152	}
153	/* Override EM scheduling with specific status code. */
154	else if (rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST)
155	{
156	Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
157	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
158	}
159	/* Don't override specific status code, first come first served. */
160	else
161	Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
162	return VINF_SUCCESS;
163	}
164
165
166	/**
167	* Calculates the CPU mode.
168	*
169	* This is mainly for updating IEMCPU::enmCpuMode.
170	*
171	* @returns CPU mode.
172	* @param pVCpu The cross context virtual CPU structure of the
173	* calling thread.
174	*/
175	DECLINLINE(IEMMODE) iemCalcCpuMode(PVMCPUCC pVCpu) RT_NOEXCEPT
176	{
177	if (CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx))
178	return IEMMODE_64BIT;
179	if (pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig) /** @todo check if this is correct... */
180	return IEMMODE_32BIT;
181	return IEMMODE_16BIT;
182	}
183
184	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
185
186	# if defined(VBOX_INCLUDED_vmm_dbgf_h) \|\| defined(DOXYGEN_RUNNING) /* dbgf.ro.cEnabledHwBreakpoints */
187	/**
188	* Initializes the execution state.
189	*
190	* @param pVCpu The cross context virtual CPU structure of the
191	* calling thread.
192	* @param fBypassHandlers Whether to bypass access handlers.
193	*
194	* @remarks Callers of this must call iemUninitExec() to undo potentially fatal
195	* side-effects in strict builds.
196	*/
197	DECLINLINE(void) iemInitExec(PVMCPUCC pVCpu, bool fBypassHandlers) RT_NOEXCEPT
198	{
199	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
200	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
201	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
202	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
203	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
204	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
205	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
206	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
207	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
208	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
209
210	pVCpu->iem.s.uCpl = CPUMGetGuestCPL(pVCpu);
211	pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
212	# ifdef VBOX_STRICT
213	pVCpu->iem.s.enmDefAddrMode = (IEMMODE)0xfe;
214	pVCpu->iem.s.enmEffAddrMode = (IEMMODE)0xfe;
215	pVCpu->iem.s.enmDefOpSize = (IEMMODE)0xfe;
216	pVCpu->iem.s.enmEffOpSize = (IEMMODE)0xfe;
217	pVCpu->iem.s.fPrefixes = 0xfeedbeef;
218	pVCpu->iem.s.uRexReg = 127;
219	pVCpu->iem.s.uRexB = 127;
220	pVCpu->iem.s.offModRm = 127;
221	pVCpu->iem.s.uRexIndex = 127;
222	pVCpu->iem.s.iEffSeg = 127;
223	pVCpu->iem.s.idxPrefix = 127;
224	pVCpu->iem.s.uVex3rdReg = 127;
225	pVCpu->iem.s.uVexLength = 127;
226	pVCpu->iem.s.fEvexStuff = 127;
227	pVCpu->iem.s.uFpuOpcode = UINT16_MAX;
228	# ifdef IEM_WITH_CODE_TLB
229	pVCpu->iem.s.offInstrNextByte = UINT16_MAX;
230	pVCpu->iem.s.pbInstrBuf = NULL;
231	pVCpu->iem.s.cbInstrBuf = UINT16_MAX;
232	pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;
233	pVCpu->iem.s.offCurInstrStart = INT16_MAX;
234	pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);
235	# else
236	pVCpu->iem.s.offOpcode = 127;
237	pVCpu->iem.s.cbOpcode = 127;
238	# endif
239	# endif /* VBOX_STRICT */
240
241	pVCpu->iem.s.cActiveMappings = 0;
242	pVCpu->iem.s.iNextMapping = 0;
243	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
244	pVCpu->iem.s.fBypassHandlers = fBypassHandlers;
245	pVCpu->iem.s.fDisregardLock = false;
246	pVCpu->iem.s.fPendingInstructionBreakpoints = false;
247	pVCpu->iem.s.fPendingDataBreakpoints = false;
248	pVCpu->iem.s.fPendingIoBreakpoints = false;
249	if (RT_LIKELY( !(pVCpu->cpum.GstCtx.dr[7] & X86_DR7_ENABLED_MASK)
250	&& pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledHwBreakpoints == 0))
251	{ /* likely */ }
252	else
253	iemInitPendingBreakpointsSlow(pVCpu);
254	}
255	# endif /* VBOX_INCLUDED_vmm_dbgf_h */
256
257
258	# if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
259	/**
260	* Performs a minimal reinitialization of the execution state.
261	*
262	* This is intended to be used by VM-exits, SMM, LOADALL and other similar
263	* 'world-switch' types operations on the CPU. Currently only nested
264	* hardware-virtualization uses it.
265	*
266	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
267	*/
268	DECLINLINE(void) iemReInitExec(PVMCPUCC pVCpu) RT_NOEXCEPT
269	{
270	IEMMODE const enmMode = iemCalcCpuMode(pVCpu);
271	uint8_t const uCpl = CPUMGetGuestCPL(pVCpu);
272
273	pVCpu->iem.s.uCpl = uCpl;
274	pVCpu->iem.s.enmCpuMode = enmMode;
275	pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
276	pVCpu->iem.s.enmEffAddrMode = enmMode;
277	if (enmMode != IEMMODE_64BIT)
278	{
279	pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
280	pVCpu->iem.s.enmEffOpSize = enmMode;
281	}
282	else
283	{
284	pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
285	pVCpu->iem.s.enmEffOpSize = enmMode;
286	}
287	pVCpu->iem.s.iEffSeg = X86_SREG_DS;
288	# ifndef IEM_WITH_CODE_TLB
289	/** @todo Shouldn't we be doing this in IEMTlbInvalidateAll()? */
290	pVCpu->iem.s.offOpcode = 0;
291	pVCpu->iem.s.cbOpcode = 0;
292	# endif
293	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
294	}
295	# endif
296
297
298	/**
299	* Counterpart to #iemInitExec that undoes evil strict-build stuff.
300	*
301	* @param pVCpu The cross context virtual CPU structure of the
302	* calling thread.
303	*/
304	DECLINLINE(void) iemUninitExec(PVMCPUCC pVCpu) RT_NOEXCEPT
305	{
306	/* Note! do not touch fInPatchCode here! (see iemUninitExecAndFiddleStatusAndMaybeReenter) */
307	# ifdef VBOX_STRICT
308	# ifdef IEM_WITH_CODE_TLB
309	NOREF(pVCpu);
310	# else
311	pVCpu->iem.s.cbOpcode = 0;
312	# endif
313	# else
314	NOREF(pVCpu);
315	# endif
316	}
317
318
319	/**
320	* Calls iemUninitExec, iemExecStatusCodeFiddling and iemRCRawMaybeReenter.
321	*
322	* Only calling iemRCRawMaybeReenter in raw-mode, obviously.
323	*
324	* @returns Fiddled strict vbox status code, ready to return to non-IEM caller.
325	* @param pVCpu The cross context virtual CPU structure of the calling thread.
326	* @param rcStrict The status code to fiddle.
327	*/
328	DECLINLINE(VBOXSTRICTRC) iemUninitExecAndFiddleStatusAndMaybeReenter(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
329	{
330	iemUninitExec(pVCpu);
331	return iemExecStatusCodeFiddling(pVCpu, rcStrict);
332	}
333
334
335	/**
336	* Macro used by the IEMExec* method to check the given instruction length.
337	*
338	* Will return on failure!
339	*
340	* @param a_cbInstr The given instruction length.
341	* @param a_cbMin The minimum length.
342	*/
343	# define IEMEXEC_ASSERT_INSTR_LEN_RETURN(a_cbInstr, a_cbMin) \
344	AssertMsgReturn((unsigned)(a_cbInstr) - (unsigned)(a_cbMin) <= (unsigned)15 - (unsigned)(a_cbMin), \
345	("cbInstr=%u cbMin=%u\n", (a_cbInstr), (a_cbMin)), VERR_IEM_INVALID_INSTR_LENGTH)
346
347
348	# ifndef IEM_WITH_SETJMP
349
350	/**
351	* Fetches the first opcode byte.
352	*
353	* @returns Strict VBox status code.
354	* @param pVCpu The cross context virtual CPU structure of the
355	* calling thread.
356	* @param pu8 Where to return the opcode byte.
357	*/
358	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetFirstU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
359	{
360	/*
361	* Check for hardware instruction breakpoints.
362	*/
363	if (RT_LIKELY(!pVCpu->iem.s.fPendingInstructionBreakpoints))
364	{ /* likely */ }
365	else
366	{
367	VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu,
368	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
369	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
370	{ /* likely */ }
371	else if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
372	return iemRaiseDebugException(pVCpu);
373	else
374	return rcStrict;
375	}
376
377	/*
378	* Fetch the first opcode byte.
379	*/
380	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
381	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
382	{
383	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
384	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
385	return VINF_SUCCESS;
386	}
387	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
388	}
389
390	# else /* IEM_WITH_SETJMP */
391
392	/**
393	* Fetches the first opcode byte, longjmp on error.
394	*
395	* @returns The opcode byte.
396	* @param pVCpu The cross context virtual CPU structure of the calling thread.
397	*/
398	DECL_INLINE_THROW(uint8_t) iemOpcodeGetFirstU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
399	{
400	/*
401	* Check for hardware instruction breakpoints.
402	*/
403	if (RT_LIKELY(!pVCpu->iem.s.fPendingInstructionBreakpoints))
404	{ /* likely */ }
405	else
406	{
407	VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu,
408	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
409	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
410	{ /* likely */ }
411	else
412	{
413	if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
414	rcStrict = iemRaiseDebugException(pVCpu);
415	IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
416	}
417	}
418
419	/*
420	* Fetch the first opcode byte.
421	*/
422	# ifdef IEM_WITH_CODE_TLB
423	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
424	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
425	if (RT_LIKELY( pbBuf != NULL
426	&& offBuf < pVCpu->iem.s.cbInstrBuf))
427	{
428	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
429	return pbBuf[offBuf];
430	}
431	# else
432	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
433	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
434	{
435	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
436	return pVCpu->iem.s.abOpcode[offOpcode];
437	}
438	# endif
439	return iemOpcodeGetNextU8SlowJmp(pVCpu);
440	}
441
442	# endif /* IEM_WITH_SETJMP */
443
444	/**
445	* Fetches the first opcode byte, returns/throws automatically on failure.
446	*
447	* @param a_pu8 Where to return the opcode byte.
448	* @remark Implicitly references pVCpu.
449	*/
450	# ifndef IEM_WITH_SETJMP
451	# define IEM_OPCODE_GET_FIRST_U8(a_pu8) \
452	do \
453	{ \
454	VBOXSTRICTRC rcStrict2 = iemOpcodeGetFirstU8(pVCpu, (a_pu8)); \
455	if (rcStrict2 == VINF_SUCCESS) \
456	{ /* likely */ } \
457	else \
458	return rcStrict2; \
459	} while (0)
460	# else
461	# define IEM_OPCODE_GET_FIRST_U8(a_pu8) (*(a_pu8) = iemOpcodeGetFirstU8Jmp(pVCpu))
462	# endif /* IEM_WITH_SETJMP */
463
464
465	# ifndef IEM_WITH_SETJMP
466
467	/**
468	* Fetches the next opcode byte.
469	*
470	* @returns Strict VBox status code.
471	* @param pVCpu The cross context virtual CPU structure of the
472	* calling thread.
473	* @param pu8 Where to return the opcode byte.
474	*/
475	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
476	{
477	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
478	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
479	{
480	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
481	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
482	return VINF_SUCCESS;
483	}
484	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
485	}
486
487	# else /* IEM_WITH_SETJMP */
488
489	/**
490	* Fetches the next opcode byte, longjmp on error.
491	*
492	* @returns The opcode byte.
493	* @param pVCpu The cross context virtual CPU structure of the calling thread.
494	*/
495	DECL_INLINE_THROW(uint8_t) iemOpcodeGetNextU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
496	{
497	# ifdef IEM_WITH_CODE_TLB
498	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
499	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
500	if (RT_LIKELY( pbBuf != NULL
501	&& offBuf < pVCpu->iem.s.cbInstrBuf))
502	{
503	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
504	return pbBuf[offBuf];
505	}
506	# else
507	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
508	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
509	{
510	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
511	return pVCpu->iem.s.abOpcode[offOpcode];
512	}
513	# endif
514	return iemOpcodeGetNextU8SlowJmp(pVCpu);
515	}
516
517	# endif /* IEM_WITH_SETJMP */
518
519	/**
520	* Fetches the next opcode byte, returns automatically on failure.
521	*
522	* @param a_pu8 Where to return the opcode byte.
523	* @remark Implicitly references pVCpu.
524	*/
525	# ifndef IEM_WITH_SETJMP
526	# define IEM_OPCODE_GET_NEXT_U8(a_pu8) \
527	do \
528	{ \
529	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pVCpu, (a_pu8)); \
530	if (rcStrict2 == VINF_SUCCESS) \
531	{ /* likely */ } \
532	else \
533	return rcStrict2; \
534	} while (0)
535	# else
536	# define IEM_OPCODE_GET_NEXT_U8(a_pu8) (*(a_pu8) = iemOpcodeGetNextU8Jmp(pVCpu))
537	# endif /* IEM_WITH_SETJMP */
538
539
540	# ifndef IEM_WITH_SETJMP
541	/**
542	* Fetches the next signed byte from the opcode stream.
543	*
544	* @returns Strict VBox status code.
545	* @param pVCpu The cross context virtual CPU structure of the calling thread.
546	* @param pi8 Where to return the signed byte.
547	*/
548	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PVMCPUCC pVCpu, int8_t *pi8) RT_NOEXCEPT
549	{
550	return iemOpcodeGetNextU8(pVCpu, (uint8_t *)pi8);
551	}
552	# endif /* !IEM_WITH_SETJMP */
553
554
555	/**
556	* Fetches the next signed byte from the opcode stream, returning automatically
557	* on failure.
558	*
559	* @param a_pi8 Where to return the signed byte.
560	* @remark Implicitly references pVCpu.
561	*/
562	# ifndef IEM_WITH_SETJMP
563	# define IEM_OPCODE_GET_NEXT_S8(a_pi8) \
564	do \
565	{ \
566	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pVCpu, (a_pi8)); \
567	if (rcStrict2 != VINF_SUCCESS) \
568	return rcStrict2; \
569	} while (0)
570	# else /* IEM_WITH_SETJMP */
571	# define IEM_OPCODE_GET_NEXT_S8(a_pi8) (*(a_pi8) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
572
573	# endif /* IEM_WITH_SETJMP */
574
575
576	# ifndef IEM_WITH_SETJMP
577	/**
578	* Fetches the next signed byte from the opcode stream, extending it to
579	* unsigned 16-bit.
580	*
581	* @returns Strict VBox status code.
582	* @param pVCpu The cross context virtual CPU structure of the calling thread.
583	* @param pu16 Where to return the unsigned word.
584	*/
585	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
586	{
587	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
588	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
589	return iemOpcodeGetNextS8SxU16Slow(pVCpu, pu16);
590
591	*pu16 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
592	pVCpu->iem.s.offOpcode = offOpcode + 1;
593	return VINF_SUCCESS;
594	}
595	# endif /* !IEM_WITH_SETJMP */
596
597	/**
598	* Fetches the next signed byte from the opcode stream and sign-extending it to
599	* a word, returning automatically on failure.
600	*
601	* @param a_pu16 Where to return the word.
602	* @remark Implicitly references pVCpu.
603	*/
604	# ifndef IEM_WITH_SETJMP
605	# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \
606	do \
607	{ \
608	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pVCpu, (a_pu16)); \
609	if (rcStrict2 != VINF_SUCCESS) \
610	return rcStrict2; \
611	} while (0)
612	# else
613	# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) (*(a_pu16) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
614	# endif
615
616	# ifndef IEM_WITH_SETJMP
617	/**
618	* Fetches the next signed byte from the opcode stream, extending it to
619	* unsigned 32-bit.
620	*
621	* @returns Strict VBox status code.
622	* @param pVCpu The cross context virtual CPU structure of the calling thread.
623	* @param pu32 Where to return the unsigned dword.
624	*/
625	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
626	{
627	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
628	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
629	return iemOpcodeGetNextS8SxU32Slow(pVCpu, pu32);
630
631	*pu32 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
632	pVCpu->iem.s.offOpcode = offOpcode + 1;
633	return VINF_SUCCESS;
634	}
635	# endif /* !IEM_WITH_SETJMP */
636
637	/**
638	* Fetches the next signed byte from the opcode stream and sign-extending it to
639	* a word, returning automatically on failure.
640	*
641	* @param a_pu32 Where to return the word.
642	* @remark Implicitly references pVCpu.
643	*/
644	# ifndef IEM_WITH_SETJMP
645	# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) \
646	do \
647	{ \
648	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU32(pVCpu, (a_pu32)); \
649	if (rcStrict2 != VINF_SUCCESS) \
650	return rcStrict2; \
651	} while (0)
652	# else
653	# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) (*(a_pu32) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
654	# endif
655
656
657	# ifndef IEM_WITH_SETJMP
658	/**
659	* Fetches the next signed byte from the opcode stream, extending it to
660	* unsigned 64-bit.
661	*
662	* @returns Strict VBox status code.
663	* @param pVCpu The cross context virtual CPU structure of the calling thread.
664	* @param pu64 Where to return the unsigned qword.
665	*/
666	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
667	{
668	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
669	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
670	return iemOpcodeGetNextS8SxU64Slow(pVCpu, pu64);
671
672	*pu64 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
673	pVCpu->iem.s.offOpcode = offOpcode + 1;
674	return VINF_SUCCESS;
675	}
676	# endif /* !IEM_WITH_SETJMP */
677
678	/**
679	* Fetches the next signed byte from the opcode stream and sign-extending it to
680	* a word, returning automatically on failure.
681	*
682	* @param a_pu64 Where to return the word.
683	* @remark Implicitly references pVCpu.
684	*/
685	# ifndef IEM_WITH_SETJMP
686	# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) \
687	do \
688	{ \
689	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU64(pVCpu, (a_pu64)); \
690	if (rcStrict2 != VINF_SUCCESS) \
691	return rcStrict2; \
692	} while (0)
693	# else
694	# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) (*(a_pu64) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
695	# endif
696
697
698	# ifndef IEM_WITH_SETJMP
699	/**
700	* Fetches the next opcode byte.
701	*
702	* @returns Strict VBox status code.
703	* @param pVCpu The cross context virtual CPU structure of the
704	* calling thread.
705	* @param pu8 Where to return the opcode byte.
706	*/
707	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextRm(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
708	{
709	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
710	pVCpu->iem.s.offModRm = offOpcode;
711	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
712	{
713	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
714	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
715	return VINF_SUCCESS;
716	}
717	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
718	}
719	# else /* IEM_WITH_SETJMP */
720	/**
721	* Fetches the next opcode byte, longjmp on error.
722	*
723	* @returns The opcode byte.
724	* @param pVCpu The cross context virtual CPU structure of the calling thread.
725	*/
726	DECL_INLINE_THROW(uint8_t) iemOpcodeGetNextRmJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
727	{
728	# ifdef IEM_WITH_CODE_TLB
729	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
730	pVCpu->iem.s.offModRm = offBuf;
731	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
732	if (RT_LIKELY( pbBuf != NULL
733	&& offBuf < pVCpu->iem.s.cbInstrBuf))
734	{
735	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
736	return pbBuf[offBuf];
737	}
738	# else
739	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
740	pVCpu->iem.s.offModRm = offOpcode;
741	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
742	{
743	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
744	return pVCpu->iem.s.abOpcode[offOpcode];
745	}
746	# endif
747	return iemOpcodeGetNextU8SlowJmp(pVCpu);
748	}
749	# endif /* IEM_WITH_SETJMP */
750
751	/**
752	* Fetches the next opcode byte, which is a ModR/M byte, returns automatically
753	* on failure.
754	*
755	* Will note down the position of the ModR/M byte for VT-x exits.
756	*
757	* @param a_pbRm Where to return the RM opcode byte.
758	* @remark Implicitly references pVCpu.
759	*/
760	# ifndef IEM_WITH_SETJMP
761	# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) \
762	do \
763	{ \
764	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextRm(pVCpu, (a_pbRm)); \
765	if (rcStrict2 == VINF_SUCCESS) \
766	{ /* likely */ } \
767	else \
768	return rcStrict2; \
769	} while (0)
770	# else
771	# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) (*(a_pbRm) = iemOpcodeGetNextRmJmp(pVCpu))
772	# endif /* IEM_WITH_SETJMP */
773
774
775	# ifndef IEM_WITH_SETJMP
776
777	/**
778	* Fetches the next opcode word.
779	*
780	* @returns Strict VBox status code.
781	* @param pVCpu The cross context virtual CPU structure of the calling thread.
782	* @param pu16 Where to return the opcode word.
783	*/
784	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
785	{
786	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
787	if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
788	{
789	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
790	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
791	pu16 = (uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
792	# else
793	*pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
794	# endif
795	return VINF_SUCCESS;
796	}
797	return iemOpcodeGetNextU16Slow(pVCpu, pu16);
798	}
799
800	# else /* IEM_WITH_SETJMP */
801
802	/**
803	* Fetches the next opcode word, longjmp on error.
804	*
805	* @returns The opcode word.
806	* @param pVCpu The cross context virtual CPU structure of the calling thread.
807	*/
808	DECL_INLINE_THROW(uint16_t) iemOpcodeGetNextU16Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
809	{
810	# ifdef IEM_WITH_CODE_TLB
811	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
812	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
813	if (RT_LIKELY( pbBuf != NULL
814	&& offBuf + 2 <= pVCpu->iem.s.cbInstrBuf))
815	{
816	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 2;
817	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
818	return (uint16_t const )&pbBuf[offBuf];
819	# else
820	return RT_MAKE_U16(pbBuf[offBuf], pbBuf[offBuf + 1]);
821	# endif
822	}
823	# else /* !IEM_WITH_CODE_TLB */
824	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
825	if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
826	{
827	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
828	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
829	return (uint16_t const )&pVCpu->iem.s.abOpcode[offOpcode];
830	# else
831	return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
832	# endif
833	}
834	# endif /* !IEM_WITH_CODE_TLB */
835	return iemOpcodeGetNextU16SlowJmp(pVCpu);
836	}
837
838	# endif /* IEM_WITH_SETJMP */
839
840	/**
841	* Fetches the next opcode word, returns automatically on failure.
842	*
843	* @param a_pu16 Where to return the opcode word.
844	* @remark Implicitly references pVCpu.
845	*/
846	# ifndef IEM_WITH_SETJMP
847	# define IEM_OPCODE_GET_NEXT_U16(a_pu16) \
848	do \
849	{ \
850	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pVCpu, (a_pu16)); \
851	if (rcStrict2 != VINF_SUCCESS) \
852	return rcStrict2; \
853	} while (0)
854	# else
855	# define IEM_OPCODE_GET_NEXT_U16(a_pu16) (*(a_pu16) = iemOpcodeGetNextU16Jmp(pVCpu))
856	# endif
857
858	# ifndef IEM_WITH_SETJMP
859	/**
860	* Fetches the next opcode word, zero extending it to a double word.
861	*
862	* @returns Strict VBox status code.
863	* @param pVCpu The cross context virtual CPU structure of the calling thread.
864	* @param pu32 Where to return the opcode double word.
865	*/
866	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
867	{
868	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
869	if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
870	return iemOpcodeGetNextU16ZxU32Slow(pVCpu, pu32);
871
872	*pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
873	pVCpu->iem.s.offOpcode = offOpcode + 2;
874	return VINF_SUCCESS;
875	}
876	# endif /* !IEM_WITH_SETJMP */
877
878	/**
879	* Fetches the next opcode word and zero extends it to a double word, returns
880	* automatically on failure.
881	*
882	* @param a_pu32 Where to return the opcode double word.
883	* @remark Implicitly references pVCpu.
884	*/
885	# ifndef IEM_WITH_SETJMP
886	# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \
887	do \
888	{ \
889	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pVCpu, (a_pu32)); \
890	if (rcStrict2 != VINF_SUCCESS) \
891	return rcStrict2; \
892	} while (0)
893	# else
894	# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU16Jmp(pVCpu))
895	# endif
896
897	# ifndef IEM_WITH_SETJMP
898	/**
899	* Fetches the next opcode word, zero extending it to a quad word.
900	*
901	* @returns Strict VBox status code.
902	* @param pVCpu The cross context virtual CPU structure of the calling thread.
903	* @param pu64 Where to return the opcode quad word.
904	*/
905	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
906	{
907	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
908	if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
909	return iemOpcodeGetNextU16ZxU64Slow(pVCpu, pu64);
910
911	*pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
912	pVCpu->iem.s.offOpcode = offOpcode + 2;
913	return VINF_SUCCESS;
914	}
915	# endif /* !IEM_WITH_SETJMP */
916
917	/**
918	* Fetches the next opcode word and zero extends it to a quad word, returns
919	* automatically on failure.
920	*
921	* @param a_pu64 Where to return the opcode quad word.
922	* @remark Implicitly references pVCpu.
923	*/
924	# ifndef IEM_WITH_SETJMP
925	# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \
926	do \
927	{ \
928	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pVCpu, (a_pu64)); \
929	if (rcStrict2 != VINF_SUCCESS) \
930	return rcStrict2; \
931	} while (0)
932	# else
933	# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU16Jmp(pVCpu))
934	# endif
935
936
937	# ifndef IEM_WITH_SETJMP
938	/**
939	* Fetches the next signed word from the opcode stream.
940	*
941	* @returns Strict VBox status code.
942	* @param pVCpu The cross context virtual CPU structure of the calling thread.
943	* @param pi16 Where to return the signed word.
944	*/
945	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PVMCPUCC pVCpu, int16_t *pi16) RT_NOEXCEPT
946	{
947	return iemOpcodeGetNextU16(pVCpu, (uint16_t *)pi16);
948	}
949	# endif /* !IEM_WITH_SETJMP */
950
951
952	/**
953	* Fetches the next signed word from the opcode stream, returning automatically
954	* on failure.
955	*
956	* @param a_pi16 Where to return the signed word.
957	* @remark Implicitly references pVCpu.
958	*/
959	# ifndef IEM_WITH_SETJMP
960	# define IEM_OPCODE_GET_NEXT_S16(a_pi16) \
961	do \
962	{ \
963	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pVCpu, (a_pi16)); \
964	if (rcStrict2 != VINF_SUCCESS) \
965	return rcStrict2; \
966	} while (0)
967	# else
968	# define IEM_OPCODE_GET_NEXT_S16(a_pi16) (*(a_pi16) = (int16_t)iemOpcodeGetNextU16Jmp(pVCpu))
969	# endif
970
971	# ifndef IEM_WITH_SETJMP
972
973	/**
974	* Fetches the next opcode dword.
975	*
976	* @returns Strict VBox status code.
977	* @param pVCpu The cross context virtual CPU structure of the calling thread.
978	* @param pu32 Where to return the opcode double word.
979	*/
980	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
981	{
982	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
983	if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
984	{
985	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
986	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
987	pu32 = (uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
988	# else
989	*pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
990	pVCpu->iem.s.abOpcode[offOpcode + 1],
991	pVCpu->iem.s.abOpcode[offOpcode + 2],
992	pVCpu->iem.s.abOpcode[offOpcode + 3]);
993	# endif
994	return VINF_SUCCESS;
995	}
996	return iemOpcodeGetNextU32Slow(pVCpu, pu32);
997	}
998
999	# else /* IEM_WITH_SETJMP */
1000
1001	/**
1002	* Fetches the next opcode dword, longjmp on error.
1003	*
1004	* @returns The opcode dword.
1005	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1006	*/
1007	DECL_INLINE_THROW(uint32_t) iemOpcodeGetNextU32Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
1008	{
1009	# ifdef IEM_WITH_CODE_TLB
1010	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1011	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1012	if (RT_LIKELY( pbBuf != NULL
1013	&& offBuf + 4 <= pVCpu->iem.s.cbInstrBuf))
1014	{
1015	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 4;
1016	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1017	return (uint32_t const )&pbBuf[offBuf];
1018	# else
1019	return RT_MAKE_U32_FROM_U8(pbBuf[offBuf],
1020	pbBuf[offBuf + 1],
1021	pbBuf[offBuf + 2],
1022	pbBuf[offBuf + 3]);
1023	# endif
1024	}
1025	# else
1026	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1027	if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
1028	{
1029	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
1030	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1031	return (uint32_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1032	# else
1033	return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1034	pVCpu->iem.s.abOpcode[offOpcode + 1],
1035	pVCpu->iem.s.abOpcode[offOpcode + 2],
1036	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1037	# endif
1038	}
1039	# endif
1040	return iemOpcodeGetNextU32SlowJmp(pVCpu);
1041	}
1042
1043	# endif /* IEM_WITH_SETJMP */
1044
1045	/**
1046	* Fetches the next opcode dword, returns automatically on failure.
1047	*
1048	* @param a_pu32 Where to return the opcode dword.
1049	* @remark Implicitly references pVCpu.
1050	*/
1051	# ifndef IEM_WITH_SETJMP
1052	# define IEM_OPCODE_GET_NEXT_U32(a_pu32) \
1053	do \
1054	{ \
1055	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pVCpu, (a_pu32)); \
1056	if (rcStrict2 != VINF_SUCCESS) \
1057	return rcStrict2; \
1058	} while (0)
1059	# else
1060	# define IEM_OPCODE_GET_NEXT_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU32Jmp(pVCpu))
1061	# endif
1062
1063	# ifndef IEM_WITH_SETJMP
1064	/**
1065	* Fetches the next opcode dword, zero extending it to a quad word.
1066	*
1067	* @returns Strict VBox status code.
1068	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1069	* @param pu64 Where to return the opcode quad word.
1070	*/
1071	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1072	{
1073	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1074	if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1075	return iemOpcodeGetNextU32ZxU64Slow(pVCpu, pu64);
1076
1077	*pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1078	pVCpu->iem.s.abOpcode[offOpcode + 1],
1079	pVCpu->iem.s.abOpcode[offOpcode + 2],
1080	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1081	pVCpu->iem.s.offOpcode = offOpcode + 4;
1082	return VINF_SUCCESS;
1083	}
1084	# endif /* !IEM_WITH_SETJMP */
1085
1086	/**
1087	* Fetches the next opcode dword and zero extends it to a quad word, returns
1088	* automatically on failure.
1089	*
1090	* @param a_pu64 Where to return the opcode quad word.
1091	* @remark Implicitly references pVCpu.
1092	*/
1093	# ifndef IEM_WITH_SETJMP
1094	# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \
1095	do \
1096	{ \
1097	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pVCpu, (a_pu64)); \
1098	if (rcStrict2 != VINF_SUCCESS) \
1099	return rcStrict2; \
1100	} while (0)
1101	# else
1102	# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU32Jmp(pVCpu))
1103	# endif
1104
1105
1106	# ifndef IEM_WITH_SETJMP
1107	/**
1108	* Fetches the next signed double word from the opcode stream.
1109	*
1110	* @returns Strict VBox status code.
1111	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1112	* @param pi32 Where to return the signed double word.
1113	*/
1114	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PVMCPUCC pVCpu, int32_t *pi32) RT_NOEXCEPT
1115	{
1116	return iemOpcodeGetNextU32(pVCpu, (uint32_t *)pi32);
1117	}
1118	# endif
1119
1120	/**
1121	* Fetches the next signed double word from the opcode stream, returning
1122	* automatically on failure.
1123	*
1124	* @param a_pi32 Where to return the signed double word.
1125	* @remark Implicitly references pVCpu.
1126	*/
1127	# ifndef IEM_WITH_SETJMP
1128	# define IEM_OPCODE_GET_NEXT_S32(a_pi32) \
1129	do \
1130	{ \
1131	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pVCpu, (a_pi32)); \
1132	if (rcStrict2 != VINF_SUCCESS) \
1133	return rcStrict2; \
1134	} while (0)
1135	# else
1136	# define IEM_OPCODE_GET_NEXT_S32(a_pi32) (*(a_pi32) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1137	# endif
1138
1139	# ifndef IEM_WITH_SETJMP
1140	/**
1141	* Fetches the next opcode dword, sign extending it into a quad word.
1142	*
1143	* @returns Strict VBox status code.
1144	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1145	* @param pu64 Where to return the opcode quad word.
1146	*/
1147	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1148	{
1149	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1150	if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1151	return iemOpcodeGetNextS32SxU64Slow(pVCpu, pu64);
1152
1153	int32_t i32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1154	pVCpu->iem.s.abOpcode[offOpcode + 1],
1155	pVCpu->iem.s.abOpcode[offOpcode + 2],
1156	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1157	*pu64 = i32;
1158	pVCpu->iem.s.offOpcode = offOpcode + 4;
1159	return VINF_SUCCESS;
1160	}
1161	# endif /* !IEM_WITH_SETJMP */
1162
1163	/**
1164	* Fetches the next opcode double word and sign extends it to a quad word,
1165	* returns automatically on failure.
1166	*
1167	* @param a_pu64 Where to return the opcode quad word.
1168	* @remark Implicitly references pVCpu.
1169	*/
1170	# ifndef IEM_WITH_SETJMP
1171	# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \
1172	do \
1173	{ \
1174	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pVCpu, (a_pu64)); \
1175	if (rcStrict2 != VINF_SUCCESS) \
1176	return rcStrict2; \
1177	} while (0)
1178	# else
1179	# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) (*(a_pu64) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1180	# endif
1181
1182	# ifndef IEM_WITH_SETJMP
1183
1184	/**
1185	* Fetches the next opcode qword.
1186	*
1187	* @returns Strict VBox status code.
1188	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1189	* @param pu64 Where to return the opcode qword.
1190	*/
1191	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1192	{
1193	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1194	if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1195	{
1196	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1197	pu64 = (uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1198	# else
1199	*pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1200	pVCpu->iem.s.abOpcode[offOpcode + 1],
1201	pVCpu->iem.s.abOpcode[offOpcode + 2],
1202	pVCpu->iem.s.abOpcode[offOpcode + 3],
1203	pVCpu->iem.s.abOpcode[offOpcode + 4],
1204	pVCpu->iem.s.abOpcode[offOpcode + 5],
1205	pVCpu->iem.s.abOpcode[offOpcode + 6],
1206	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1207	# endif
1208	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1209	return VINF_SUCCESS;
1210	}
1211	return iemOpcodeGetNextU64Slow(pVCpu, pu64);
1212	}
1213
1214	# else /* IEM_WITH_SETJMP */
1215
1216	/**
1217	* Fetches the next opcode qword, longjmp on error.
1218	*
1219	* @returns The opcode qword.
1220	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1221	*/
1222	DECL_INLINE_THROW(uint64_t) iemOpcodeGetNextU64Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
1223	{
1224	# ifdef IEM_WITH_CODE_TLB
1225	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1226	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1227	if (RT_LIKELY( pbBuf != NULL
1228	&& offBuf + 8 <= pVCpu->iem.s.cbInstrBuf))
1229	{
1230	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 8;
1231	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1232	return (uint64_t const )&pbBuf[offBuf];
1233	# else
1234	return RT_MAKE_U64_FROM_U8(pbBuf[offBuf],
1235	pbBuf[offBuf + 1],
1236	pbBuf[offBuf + 2],
1237	pbBuf[offBuf + 3],
1238	pbBuf[offBuf + 4],
1239	pbBuf[offBuf + 5],
1240	pbBuf[offBuf + 6],
1241	pbBuf[offBuf + 7]);
1242	# endif
1243	}
1244	# else
1245	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1246	if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1247	{
1248	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1249	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1250	return (uint64_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1251	# else
1252	return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1253	pVCpu->iem.s.abOpcode[offOpcode + 1],
1254	pVCpu->iem.s.abOpcode[offOpcode + 2],
1255	pVCpu->iem.s.abOpcode[offOpcode + 3],
1256	pVCpu->iem.s.abOpcode[offOpcode + 4],
1257	pVCpu->iem.s.abOpcode[offOpcode + 5],
1258	pVCpu->iem.s.abOpcode[offOpcode + 6],
1259	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1260	# endif
1261	}
1262	# endif
1263	return iemOpcodeGetNextU64SlowJmp(pVCpu);
1264	}
1265
1266	# endif /* IEM_WITH_SETJMP */
1267
1268	/**
1269	* Fetches the next opcode quad word, returns automatically on failure.
1270	*
1271	* @param a_pu64 Where to return the opcode quad word.
1272	* @remark Implicitly references pVCpu.
1273	*/
1274	# ifndef IEM_WITH_SETJMP
1275	# define IEM_OPCODE_GET_NEXT_U64(a_pu64) \
1276	do \
1277	{ \
1278	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pVCpu, (a_pu64)); \
1279	if (rcStrict2 != VINF_SUCCESS) \
1280	return rcStrict2; \
1281	} while (0)
1282	# else
1283	# define IEM_OPCODE_GET_NEXT_U64(a_pu64) ( *(a_pu64) = iemOpcodeGetNextU64Jmp(pVCpu) )
1284	# endif
1285
1286	#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
1287
1288
1289	/** @name Misc Worker Functions.
1290	* @{
1291	*/
1292
1293	/**
1294	* Gets the correct EFLAGS regardless of whether PATM stores parts of them or
1295	* not (kind of obsolete now).
1296	*
1297	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1298	*/
1299	#define IEMMISC_GET_EFL(a_pVCpu) ( (a_pVCpu)->cpum.GstCtx.eflags.u )
1300
1301	/**
1302	* Updates the EFLAGS in the correct manner wrt. PATM (kind of obsolete).
1303	*
1304	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1305	* @param a_fEfl The new EFLAGS.
1306	*/
1307	#define IEMMISC_SET_EFL(a_pVCpu, a_fEfl) do { (a_pVCpu)->cpum.GstCtx.eflags.u = (a_fEfl); } while (0)
1308
1309
1310	/**
1311	* Loads a NULL data selector into a selector register, both the hidden and
1312	* visible parts, in protected mode.
1313	*
1314	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1315	* @param pSReg Pointer to the segment register.
1316	* @param uRpl The RPL.
1317	*/
1318	DECLINLINE(void) iemHlpLoadNullDataSelectorProt(PVMCPUCC pVCpu, PCPUMSELREG pSReg, RTSEL uRpl) RT_NOEXCEPT
1319	{
1320	/** @todo Testcase: write a testcase checking what happends when loading a NULL
1321	* data selector in protected mode. */
1322	pSReg->Sel = uRpl;
1323	pSReg->ValidSel = uRpl;
1324	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
1325	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
1326	{
1327	/* VT-x (Intel 3960x) observed doing something like this. */
1328	pSReg->Attr.u = X86DESCATTR_UNUSABLE \| X86DESCATTR_G \| X86DESCATTR_D \| (pVCpu->iem.s.uCpl << X86DESCATTR_DPL_SHIFT);
1329	pSReg->u32Limit = UINT32_MAX;
1330	pSReg->u64Base = 0;
1331	}
1332	else
1333	{
1334	pSReg->Attr.u = X86DESCATTR_UNUSABLE;
1335	pSReg->u32Limit = 0;
1336	pSReg->u64Base = 0;
1337	}
1338	}
1339
1340	/** @} */
1341
1342
1343	/*
1344	*
1345	* Helpers routines.
1346	* Helpers routines.
1347	* Helpers routines.
1348	*
1349	*/
1350
1351	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1352
1353	/**
1354	* Recalculates the effective operand size.
1355	*
1356	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1357	*/
1358	DECLINLINE(void) iemRecalEffOpSize(PVMCPUCC pVCpu) RT_NOEXCEPT
1359	{
1360	switch (pVCpu->iem.s.enmCpuMode)
1361	{
1362	case IEMMODE_16BIT:
1363	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT;
1364	break;
1365	case IEMMODE_32BIT:
1366	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT;
1367	break;
1368	case IEMMODE_64BIT:
1369	switch (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP))
1370	{
1371	case 0:
1372	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize;
1373	break;
1374	case IEM_OP_PRF_SIZE_OP:
1375	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1376	break;
1377	case IEM_OP_PRF_SIZE_REX_W:
1378	case IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP:
1379	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1380	break;
1381	}
1382	break;
1383	default:
1384	AssertFailed();
1385	}
1386	}
1387
1388
1389	/**
1390	* Sets the default operand size to 64-bit and recalculates the effective
1391	* operand size.
1392	*
1393	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1394	*/
1395	DECLINLINE(void) iemRecalEffOpSize64Default(PVMCPUCC pVCpu) RT_NOEXCEPT
1396	{
1397	Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
1398	pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1399	if ((pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP)
1400	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1401	else
1402	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1403	}
1404
1405
1406	/**
1407	* Sets the default operand size to 64-bit and recalculates the effective
1408	* operand size, with intel ignoring any operand size prefix (AMD respects it).
1409	*
1410	* This is for the relative jumps.
1411	*
1412	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1413	*/
1414	DECLINLINE(void) iemRecalEffOpSize64DefaultAndIntelIgnoresOpSizePrefix(PVMCPUCC pVCpu) RT_NOEXCEPT
1415	{
1416	Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
1417	pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1418	if ( (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP
1419	\|\| pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
1420	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1421	else
1422	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1423	}
1424
1425	#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
1426
1427
1428
1429	/** @name Register Access.
1430	* @{
1431	*/
1432
1433	/**
1434	* Gets a reference (pointer) to the specified hidden segment register.
1435	*
1436	* @returns Hidden register reference.
1437	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1438	* @param iSegReg The segment register.
1439	*/
1440	DECLINLINE(PCPUMSELREG) iemSRegGetHid(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1441	{
1442	Assert(iSegReg < X86_SREG_COUNT);
1443	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1444	PCPUMSELREG pSReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
1445
1446	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1447	return pSReg;
1448	}
1449
1450
1451	/**
1452	* Ensures that the given hidden segment register is up to date.
1453	*
1454	* @returns Hidden register reference.
1455	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1456	* @param pSReg The segment register.
1457	*/
1458	DECLINLINE(PCPUMSELREG) iemSRegUpdateHid(PVMCPUCC pVCpu, PCPUMSELREG pSReg) RT_NOEXCEPT
1459	{
1460	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1461	NOREF(pVCpu);
1462	return pSReg;
1463	}
1464
1465
1466	/**
1467	* Gets a reference (pointer) to the specified segment register (the selector
1468	* value).
1469	*
1470	* @returns Pointer to the selector variable.
1471	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1472	* @param iSegReg The segment register.
1473	*/
1474	DECLINLINE(uint16_t *) iemSRegRef(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1475	{
1476	Assert(iSegReg < X86_SREG_COUNT);
1477	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1478	return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1479	}
1480
1481
1482	/**
1483	* Fetches the selector value of a segment register.
1484	*
1485	* @returns The selector value.
1486	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1487	* @param iSegReg The segment register.
1488	*/
1489	DECLINLINE(uint16_t) iemSRegFetchU16(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1490	{
1491	Assert(iSegReg < X86_SREG_COUNT);
1492	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1493	return pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1494	}
1495
1496
1497	/**
1498	* Fetches the base address value of a segment register.
1499	*
1500	* @returns The selector value.
1501	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1502	* @param iSegReg The segment register.
1503	*/
1504	DECLINLINE(uint64_t) iemSRegBaseFetchU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1505	{
1506	Assert(iSegReg < X86_SREG_COUNT);
1507	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1508	return pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1509	}
1510
1511
1512	/**
1513	* Gets a reference (pointer) to the specified general purpose register.
1514	*
1515	* @returns Register reference.
1516	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1517	* @param iReg The general purpose register.
1518	*/
1519	DECLINLINE(void *) iemGRegRef(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1520	{
1521	Assert(iReg < 16);
1522	return &pVCpu->cpum.GstCtx.aGRegs[iReg];
1523	}
1524
1525
1526	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1527	/**
1528	* Gets a reference (pointer) to the specified 8-bit general purpose register.
1529	*
1530	* Because of AH, CH, DH and BH we cannot use iemGRegRef directly here.
1531	*
1532	* @returns Register reference.
1533	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1534	* @param iReg The register.
1535	*/
1536	DECLINLINE(uint8_t *) iemGRegRefU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1537	{
1538	if (iReg < 4 \|\| (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX))
1539	{
1540	Assert(iReg < 16);
1541	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u8;
1542	}
1543	/* high 8-bit register. */
1544	Assert(iReg < 8);
1545	return &pVCpu->cpum.GstCtx.aGRegs[iReg & 3].bHi;
1546	}
1547	#endif
1548
1549
1550	/**
1551	* Gets a reference (pointer) to the specified 8-bit general purpose register,
1552	* alternative version with extended register index.
1553	*
1554	* @returns Register reference.
1555	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1556	* @param iRegEx The register. The 16 first are regular ones,
1557	* whereas 16 thru 19 maps to AH, CH, DH and BH.
1558	*/
1559	DECLINLINE(uint8_t *) iemGRegRefU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT
1560	{
1561	if (iRegEx < 16)
1562	return &pVCpu->cpum.GstCtx.aGRegs[iRegEx].u8;
1563
1564	/* high 8-bit register. */
1565	Assert(iRegEx < 20);
1566	return &pVCpu->cpum.GstCtx.aGRegs[iRegEx & 3].bHi;
1567	}
1568
1569
1570	/**
1571	* Gets a reference (pointer) to the specified 16-bit general purpose register.
1572	*
1573	* @returns Register reference.
1574	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1575	* @param iReg The register.
1576	*/
1577	DECLINLINE(uint16_t *) iemGRegRefU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1578	{
1579	Assert(iReg < 16);
1580	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1581	}
1582
1583
1584	/**
1585	* Gets a reference (pointer) to the specified 32-bit general purpose register.
1586	*
1587	* @returns Register reference.
1588	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1589	* @param iReg The register.
1590	*/
1591	DECLINLINE(uint32_t *) iemGRegRefU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1592	{
1593	Assert(iReg < 16);
1594	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1595	}
1596
1597
1598	/**
1599	* Gets a reference (pointer) to the specified signed 32-bit general purpose register.
1600	*
1601	* @returns Register reference.
1602	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1603	* @param iReg The register.
1604	*/
1605	DECLINLINE(int32_t *) iemGRegRefI32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1606	{
1607	Assert(iReg < 16);
1608	return (int32_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1609	}
1610
1611
1612	/**
1613	* Gets a reference (pointer) to the specified 64-bit general purpose register.
1614	*
1615	* @returns Register reference.
1616	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1617	* @param iReg The register.
1618	*/
1619	DECLINLINE(uint64_t *) iemGRegRefU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1620	{
1621	Assert(iReg < 64);
1622	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1623	}
1624
1625
1626	/**
1627	* Gets a reference (pointer) to the specified signed 64-bit general purpose register.
1628	*
1629	* @returns Register reference.
1630	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1631	* @param iReg The register.
1632	*/
1633	DECLINLINE(int64_t *) iemGRegRefI64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1634	{
1635	Assert(iReg < 16);
1636	return (int64_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1637	}
1638
1639
1640	/**
1641	* Gets a reference (pointer) to the specified segment register's base address.
1642	*
1643	* @returns Segment register base address reference.
1644	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1645	* @param iSegReg The segment selector.
1646	*/
1647	DECLINLINE(uint64_t *) iemSRegBaseRefU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1648	{
1649	Assert(iSegReg < X86_SREG_COUNT);
1650	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1651	return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1652	}
1653
1654
1655	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1656	/**
1657	* Fetches the value of a 8-bit general purpose register.
1658	*
1659	* @returns The register value.
1660	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1661	* @param iReg The register.
1662	*/
1663	DECLINLINE(uint8_t) iemGRegFetchU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1664	{
1665	return *iemGRegRefU8(pVCpu, iReg);
1666	}
1667	#endif
1668
1669
1670	/**
1671	* Fetches the value of a 16-bit general purpose register.
1672	*
1673	* @returns The register value.
1674	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1675	* @param iReg The register.
1676	*/
1677	DECLINLINE(uint16_t) iemGRegFetchU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1678	{
1679	Assert(iReg < 16);
1680	return pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1681	}
1682
1683
1684	/**
1685	* Fetches the value of a 32-bit general purpose register.
1686	*
1687	* @returns The register value.
1688	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1689	* @param iReg The register.
1690	*/
1691	DECLINLINE(uint32_t) iemGRegFetchU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1692	{
1693	Assert(iReg < 16);
1694	return pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1695	}
1696
1697
1698	/**
1699	* Fetches the value of a 64-bit general purpose register.
1700	*
1701	* @returns The register value.
1702	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1703	* @param iReg The register.
1704	*/
1705	DECLINLINE(uint64_t) iemGRegFetchU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1706	{
1707	Assert(iReg < 16);
1708	return pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1709	}
1710
1711
1712	/**
1713	* Get the address of the top of the stack.
1714	*
1715	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1716	*/
1717	DECLINLINE(RTGCPTR) iemRegGetEffRsp(PCVMCPU pVCpu) RT_NOEXCEPT
1718	{
1719	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1720	return pVCpu->cpum.GstCtx.rsp;
1721	if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1722	return pVCpu->cpum.GstCtx.esp;
1723	return pVCpu->cpum.GstCtx.sp;
1724	}
1725
1726
1727	/**
1728	* Updates the RIP/EIP/IP to point to the next instruction.
1729	*
1730	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1731	* @param cbInstr The number of bytes to add.
1732	*/
1733	DECL_FORCE_INLINE(void) iemRegAddToRip(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
1734	{
1735	/*
1736	* Advance RIP.
1737	*
1738	* When we're targetting 8086/8, 80186/8 or 80286 mode the updates are 16-bit,
1739	* while in all other modes except LM64 the updates are 32-bit. This means
1740	* we need to watch for both 32-bit and 16-bit "carry" situations, i.e.
1741	* 4GB and 64KB rollovers, and decide whether anything needs masking.
1742	*
1743	* See PC wrap around tests in bs3-cpu-weird-1.
1744	*/
1745	uint64_t const uRipPrev = pVCpu->cpum.GstCtx.rip;
1746	uint64_t const uRipNext = uRipPrev + cbInstr;
1747	if (RT_LIKELY( !((uRipNext ^ uRipPrev) & (RT_BIT_64(32) \| RT_BIT_64(16)))
1748	\|\| pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT))
1749	pVCpu->cpum.GstCtx.rip = uRipNext;
1750	else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
1751	pVCpu->cpum.GstCtx.rip = (uint32_t)uRipNext;
1752	else
1753	pVCpu->cpum.GstCtx.rip = (uint16_t)uRipNext;
1754	}
1755
1756
1757	/**
1758	* Called by iemRegAddToRipAndFinishingClearingRF and others when any of the
1759	* following EFLAGS bits are set:
1760	* - X86_EFL_RF - clear it.
1761	* - CPUMCTX_INHIBIT_SHADOW (_SS/_STI) - clear them.
1762	* - X86_EFL_TF - generate single step \#DB trap.
1763	* - CPUMCTX_DBG_HIT_DR0/1/2/3 - generate \#DB trap (data or I/O, not
1764	* instruction).
1765	*
1766	* According to @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events},
1767	* a \#DB due to TF (single stepping) or a DRx non-instruction breakpoint
1768	* takes priority over both NMIs and hardware interrupts. So, neither is
1769	* considered here. (The RESET, \#MC, SMI, INIT, STOPCLK and FLUSH events are
1770	* either unsupported will be triggered on-top of any \#DB raised here.)
1771	*
1772	* The RF flag only needs to be cleared here as it only suppresses instruction
1773	* breakpoints which are not raised here (happens synchronously during
1774	* instruction fetching).
1775	*
1776	* The CPUMCTX_INHIBIT_SHADOW_SS flag will be cleared by this function, so its
1777	* status has no bearing on whether \#DB exceptions are raised.
1778	*
1779	* @note This must NOT be called by the two instructions setting the
1780	* CPUMCTX_INHIBIT_SHADOW_SS flag.
1781	*
1782	* @see @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events}
1783	* @see @sdmv3{077,200,6.8.3,Masking Exceptions and Interrupts When Switching
1784	* Stacks}
1785	*/
1786	static VBOXSTRICTRC iemFinishInstructionWithFlagsSet(PVMCPUCC pVCpu) RT_NOEXCEPT
1787	{
1788	/*
1789	* Normally we're just here to clear RF and/or interrupt shadow bits.
1790	*/
1791	if (RT_LIKELY((pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_TF \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK)) == 0))
1792	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW);
1793	else
1794	{
1795	/*
1796	* Raise a #DB or/and DBGF event.
1797	*/
1798	VBOXSTRICTRC rcStrict;
1799	if (pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_TF \| CPUMCTX_DBG_HIT_DRX_MASK))
1800	{
1801	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
1802	pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
1803	if (pVCpu->cpum.GstCtx.eflags.uBoth & X86_EFL_TF)
1804	pVCpu->cpum.GstCtx.dr[6] \|= X86_DR6_BS;
1805	pVCpu->cpum.GstCtx.dr[6] \|= (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT;
1806	LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64\n",
1807	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
1808	pVCpu->cpum.GstCtx.rflags.uBoth));
1809
1810	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK);
1811	rcStrict = iemRaiseDebugException(pVCpu);
1812
1813	/* A DBGF event/breakpoint trumps the iemRaiseDebugException informational status code. */
1814	if ((pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK) && RT_FAILURE(rcStrict))
1815	{
1816	rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT;
1817	LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));
1818	}
1819	}
1820	else
1821	{
1822	Assert(pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK);
1823	rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT;
1824	LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));
1825	}
1826	pVCpu->cpum.GstCtx.eflags.uBoth &= ~CPUMCTX_DBG_DBGF_MASK;
1827	return rcStrict;
1828	}
1829	return VINF_SUCCESS;
1830	}
1831
1832
1833	/**
1834	* Clears the RF and CPUMCTX_INHIBIT_SHADOW, triggering \#DB if pending.
1835	*
1836	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1837	*/
1838	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT
1839	{
1840	/*
1841	* We assume that most of the time nothing actually needs doing here.
1842	*/
1843	AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX);
1844	if (RT_LIKELY(!( pVCpu->cpum.GstCtx.eflags.uBoth
1845	& (X86_EFL_TF \| X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK)) ))
1846	return VINF_SUCCESS;
1847	return iemFinishInstructionWithFlagsSet(pVCpu);
1848	}
1849
1850
1851	/**
1852	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF
1853	* and CPUMCTX_INHIBIT_SHADOW.
1854	*
1855	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1856	* @param cbInstr The number of bytes to add.
1857	*/
1858	DECLINLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
1859	{
1860	iemRegAddToRip(pVCpu, cbInstr);
1861	return iemRegFinishClearingRF(pVCpu);
1862	}
1863
1864
1865	/**
1866	* Extended version of iemFinishInstructionWithFlagsSet that goes with
1867	* iemRegAddToRipAndFinishingClearingRfEx.
1868	*
1869	* See iemFinishInstructionWithFlagsSet() for details.
1870	*/
1871	static VBOXSTRICTRC iemFinishInstructionWithTfSet(PVMCPUCC pVCpu) RT_NOEXCEPT
1872	{
1873	/*
1874	* Raise a #DB.
1875	*/
1876	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
1877	pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
1878	pVCpu->cpum.GstCtx.dr[6] \|= X86_DR6_BS
1879	\| (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT;
1880	/** @todo Do we set all pending \#DB events, or just one? */
1881	LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64 (popf)\n",
1882	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
1883	pVCpu->cpum.GstCtx.rflags.uBoth));
1884	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK);
1885	return iemRaiseDebugException(pVCpu);
1886	}
1887
1888
1889	/**
1890	* Extended version of iemRegAddToRipAndFinishingClearingRF for use by POPF and
1891	* others potentially updating EFLAGS.TF.
1892	*
1893	* The single step event must be generated using the TF value at the start of
1894	* the instruction, not the new value set by it.
1895	*
1896	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1897	* @param cbInstr The number of bytes to add.
1898	* @param fEflOld The EFLAGS at the start of the instruction
1899	* execution.
1900	*/
1901	DECLINLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRfEx(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t fEflOld) RT_NOEXCEPT
1902	{
1903	iemRegAddToRip(pVCpu, cbInstr);
1904	if (!(fEflOld & X86_EFL_TF))
1905	return iemRegFinishClearingRF(pVCpu);
1906	return iemFinishInstructionWithTfSet(pVCpu);
1907	}
1908
1909
1910	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1911	/**
1912	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
1913	*
1914	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1915	*/
1916	DECLINLINE(VBOXSTRICTRC) iemRegUpdateRipAndFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT
1917	{
1918	return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
1919	}
1920	#endif
1921
1922
1923	/**
1924	* Adds to the stack pointer.
1925	*
1926	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1927	* @param cbToAdd The number of bytes to add (8-bit!).
1928	*/
1929	DECLINLINE(void) iemRegAddToRsp(PVMCPUCC pVCpu, uint8_t cbToAdd) RT_NOEXCEPT
1930	{
1931	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1932	pVCpu->cpum.GstCtx.rsp += cbToAdd;
1933	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1934	pVCpu->cpum.GstCtx.esp += cbToAdd;
1935	else
1936	pVCpu->cpum.GstCtx.sp += cbToAdd;
1937	}
1938
1939
1940	/**
1941	* Subtracts from the stack pointer.
1942	*
1943	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1944	* @param cbToSub The number of bytes to subtract (8-bit!).
1945	*/
1946	DECLINLINE(void) iemRegSubFromRsp(PVMCPUCC pVCpu, uint8_t cbToSub) RT_NOEXCEPT
1947	{
1948	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1949	pVCpu->cpum.GstCtx.rsp -= cbToSub;
1950	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1951	pVCpu->cpum.GstCtx.esp -= cbToSub;
1952	else
1953	pVCpu->cpum.GstCtx.sp -= cbToSub;
1954	}
1955
1956
1957	/**
1958	* Adds to the temporary stack pointer.
1959	*
1960	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1961	* @param pTmpRsp The temporary SP/ESP/RSP to update.
1962	* @param cbToAdd The number of bytes to add (16-bit).
1963	*/
1964	DECLINLINE(void) iemRegAddToRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToAdd) RT_NOEXCEPT
1965	{
1966	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1967	pTmpRsp->u += cbToAdd;
1968	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1969	pTmpRsp->DWords.dw0 += cbToAdd;
1970	else
1971	pTmpRsp->Words.w0 += cbToAdd;
1972	}
1973
1974
1975	/**
1976	* Subtracts from the temporary stack pointer.
1977	*
1978	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1979	* @param pTmpRsp The temporary SP/ESP/RSP to update.
1980	* @param cbToSub The number of bytes to subtract.
1981	* @remarks The @a cbToSub argument MUST be 16-bit, iemCImpl_enter is
1982	* expecting that.
1983	*/
1984	DECLINLINE(void) iemRegSubFromRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToSub) RT_NOEXCEPT
1985	{
1986	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1987	pTmpRsp->u -= cbToSub;
1988	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1989	pTmpRsp->DWords.dw0 -= cbToSub;
1990	else
1991	pTmpRsp->Words.w0 -= cbToSub;
1992	}
1993
1994
1995	/**
1996	* Calculates the effective stack address for a push of the specified size as
1997	* well as the new RSP value (upper bits may be masked).
1998	*
1999	* @returns Effective stack addressf for the push.
2000	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2001	* @param cbItem The size of the stack item to pop.
2002	* @param puNewRsp Where to return the new RSP value.
2003	*/
2004	DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT
2005	{
2006	RTUINT64U uTmpRsp;
2007	RTGCPTR GCPtrTop;
2008	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
2009
2010	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2011	GCPtrTop = uTmpRsp.u -= cbItem;
2012	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2013	GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem;
2014	else
2015	GCPtrTop = uTmpRsp.Words.w0 -= cbItem;
2016	*puNewRsp = uTmpRsp.u;
2017	return GCPtrTop;
2018	}
2019
2020
2021	/**
2022	* Gets the current stack pointer and calculates the value after a pop of the
2023	* specified size.
2024	*
2025	* @returns Current stack pointer.
2026	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2027	* @param cbItem The size of the stack item to pop.
2028	* @param puNewRsp Where to return the new RSP value.
2029	*/
2030	DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT
2031	{
2032	RTUINT64U uTmpRsp;
2033	RTGCPTR GCPtrTop;
2034	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
2035
2036	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2037	{
2038	GCPtrTop = uTmpRsp.u;
2039	uTmpRsp.u += cbItem;
2040	}
2041	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2042	{
2043	GCPtrTop = uTmpRsp.DWords.dw0;
2044	uTmpRsp.DWords.dw0 += cbItem;
2045	}
2046	else
2047	{
2048	GCPtrTop = uTmpRsp.Words.w0;
2049	uTmpRsp.Words.w0 += cbItem;
2050	}
2051	*puNewRsp = uTmpRsp.u;
2052	return GCPtrTop;
2053	}
2054
2055
2056	/**
2057	* Calculates the effective stack address for a push of the specified size as
2058	* well as the new temporary RSP value (upper bits may be masked).
2059	*
2060	* @returns Effective stack addressf for the push.
2061	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2062	* @param pTmpRsp The temporary stack pointer. This is updated.
2063	* @param cbItem The size of the stack item to pop.
2064	*/
2065	DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT
2066	{
2067	RTGCPTR GCPtrTop;
2068
2069	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2070	GCPtrTop = pTmpRsp->u -= cbItem;
2071	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2072	GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem;
2073	else
2074	GCPtrTop = pTmpRsp->Words.w0 -= cbItem;
2075	return GCPtrTop;
2076	}
2077
2078
2079	/**
2080	* Gets the effective stack address for a pop of the specified size and
2081	* calculates and updates the temporary RSP.
2082	*
2083	* @returns Current stack pointer.
2084	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2085	* @param pTmpRsp The temporary stack pointer. This is updated.
2086	* @param cbItem The size of the stack item to pop.
2087	*/
2088	DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT
2089	{
2090	RTGCPTR GCPtrTop;
2091	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2092	{
2093	GCPtrTop = pTmpRsp->u;
2094	pTmpRsp->u += cbItem;
2095	}
2096	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2097	{
2098	GCPtrTop = pTmpRsp->DWords.dw0;
2099	pTmpRsp->DWords.dw0 += cbItem;
2100	}
2101	else
2102	{
2103	GCPtrTop = pTmpRsp->Words.w0;
2104	pTmpRsp->Words.w0 += cbItem;
2105	}
2106	return GCPtrTop;
2107	}
2108
2109	/** @} */
2110
2111
2112	/** @name FPU access and helpers.
2113	*
2114	* @{
2115	*/
2116
2117
2118	/**
2119	* Hook for preparing to use the host FPU.
2120	*
2121	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2122	*
2123	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2124	*/
2125	DECLINLINE(void) iemFpuPrepareUsage(PVMCPUCC pVCpu) RT_NOEXCEPT
2126	{
2127	#ifdef IN_RING3
2128	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2129	#else
2130	CPUMRZFpuStatePrepareHostCpuForUse(pVCpu);
2131	#endif
2132	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2133	}
2134
2135
2136	/**
2137	* Hook for preparing to use the host FPU for SSE.
2138	*
2139	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2140	*
2141	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2142	*/
2143	DECLINLINE(void) iemFpuPrepareUsageSse(PVMCPUCC pVCpu) RT_NOEXCEPT
2144	{
2145	iemFpuPrepareUsage(pVCpu);
2146	}
2147
2148
2149	/**
2150	* Hook for preparing to use the host FPU for AVX.
2151	*
2152	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2153	*
2154	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2155	*/
2156	DECLINLINE(void) iemFpuPrepareUsageAvx(PVMCPUCC pVCpu) RT_NOEXCEPT
2157	{
2158	iemFpuPrepareUsage(pVCpu);
2159	}
2160
2161
2162	/**
2163	* Hook for actualizing the guest FPU state before the interpreter reads it.
2164	*
2165	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2166	*
2167	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2168	*/
2169	DECLINLINE(void) iemFpuActualizeStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2170	{
2171	#ifdef IN_RING3
2172	NOREF(pVCpu);
2173	#else
2174	CPUMRZFpuStateActualizeForRead(pVCpu);
2175	#endif
2176	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2177	}
2178
2179
2180	/**
2181	* Hook for actualizing the guest FPU state before the interpreter changes it.
2182	*
2183	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2184	*
2185	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2186	*/
2187	DECLINLINE(void) iemFpuActualizeStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2188	{
2189	#ifdef IN_RING3
2190	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2191	#else
2192	CPUMRZFpuStateActualizeForChange(pVCpu);
2193	#endif
2194	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2195	}
2196
2197
2198	/**
2199	* Hook for actualizing the guest XMM0..15 and MXCSR register state for read
2200	* only.
2201	*
2202	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2203	*
2204	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2205	*/
2206	DECLINLINE(void) iemFpuActualizeSseStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2207	{
2208	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
2209	NOREF(pVCpu);
2210	#else
2211	CPUMRZFpuStateActualizeSseForRead(pVCpu);
2212	#endif
2213	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2214	}
2215
2216
2217	/**
2218	* Hook for actualizing the guest XMM0..15 and MXCSR register state for
2219	* read+write.
2220	*
2221	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2222	*
2223	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2224	*/
2225	DECLINLINE(void) iemFpuActualizeSseStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2226	{
2227	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
2228	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2229	#else
2230	CPUMRZFpuStateActualizeForChange(pVCpu);
2231	#endif
2232	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2233
2234	/* Make sure any changes are loaded the next time around. */
2235	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_SSE;
2236	}
2237
2238
2239	/**
2240	* Hook for actualizing the guest YMM0..15 and MXCSR register state for read
2241	* only.
2242	*
2243	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2244	*
2245	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2246	*/
2247	DECLINLINE(void) iemFpuActualizeAvxStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2248	{
2249	#ifdef IN_RING3
2250	NOREF(pVCpu);
2251	#else
2252	CPUMRZFpuStateActualizeAvxForRead(pVCpu);
2253	#endif
2254	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2255	}
2256
2257
2258	/**
2259	* Hook for actualizing the guest YMM0..15 and MXCSR register state for
2260	* read+write.
2261	*
2262	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2263	*
2264	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2265	*/
2266	DECLINLINE(void) iemFpuActualizeAvxStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2267	{
2268	#ifdef IN_RING3
2269	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2270	#else
2271	CPUMRZFpuStateActualizeForChange(pVCpu);
2272	#endif
2273	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2274
2275	/* Just assume we're going to make changes to the SSE and YMM_HI parts. */
2276	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_YMM \| XSAVE_C_SSE;
2277	}
2278
2279
2280	/**
2281	* Stores a QNaN value into a FPU register.
2282	*
2283	* @param pReg Pointer to the register.
2284	*/
2285	DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg) RT_NOEXCEPT
2286	{
2287	pReg->au32[0] = UINT32_C(0x00000000);
2288	pReg->au32[1] = UINT32_C(0xc0000000);
2289	pReg->au16[4] = UINT16_C(0xffff);
2290	}
2291
2292
2293	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
2294	/**
2295	* Updates the FOP, FPU.CS and FPUIP registers.
2296	*
2297	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2298	* @param pFpuCtx The FPU context.
2299	*/
2300	DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorker(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx) RT_NOEXCEPT
2301	{
2302	Assert(pVCpu->iem.s.uFpuOpcode != UINT16_MAX);
2303	pFpuCtx->FOP = pVCpu->iem.s.uFpuOpcode;
2304	/** @todo x87.CS and FPUIP needs to be kept seperately. */
2305	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
2306	{
2307	/** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled
2308	* happens in real mode here based on the fnsave and fnstenv images. */
2309	pFpuCtx->CS = 0;
2310	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.eip \| ((uint32_t)pVCpu->cpum.GstCtx.cs.Sel << 4);
2311	}
2312	else if (!IEM_IS_LONG_MODE(pVCpu))
2313	{
2314	pFpuCtx->CS = pVCpu->cpum.GstCtx.cs.Sel;
2315	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2316	}
2317	else
2318	(uint64_t )&pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2319	}
2320	#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
2321
2322
2323
2324
2325	/**
2326	* Marks the specified stack register as free (for FFREE).
2327	*
2328	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2329	* @param iStReg The register to free.
2330	*/
2331	DECLINLINE(void) iemFpuStackFree(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
2332	{
2333	Assert(iStReg < 8);
2334	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2335	uint8_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2336	pFpuCtx->FTW &= ~RT_BIT(iReg);
2337	}
2338
2339
2340	/**
2341	* Increments FSW.TOP, i.e. pops an item off the stack without freeing it.
2342	*
2343	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2344	*/
2345	DECLINLINE(void) iemFpuStackIncTop(PVMCPUCC pVCpu) RT_NOEXCEPT
2346	{
2347	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2348	uint16_t uFsw = pFpuCtx->FSW;
2349	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2350	uTop = (uTop + (1 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2351	uFsw &= ~X86_FSW_TOP_MASK;
2352	uFsw \|= uTop;
2353	pFpuCtx->FSW = uFsw;
2354	}
2355
2356
2357	/**
2358	* Decrements FSW.TOP, i.e. push an item off the stack without storing anything.
2359	*
2360	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2361	*/
2362	DECLINLINE(void) iemFpuStackDecTop(PVMCPUCC pVCpu) RT_NOEXCEPT
2363	{
2364	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2365	uint16_t uFsw = pFpuCtx->FSW;
2366	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2367	uTop = (uTop + (7 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2368	uFsw &= ~X86_FSW_TOP_MASK;
2369	uFsw \|= uTop;
2370	pFpuCtx->FSW = uFsw;
2371	}
2372
2373
2374
2375
2376	DECLINLINE(int) iemFpuStRegNotEmpty(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
2377	{
2378	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2379	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2380	if (pFpuCtx->FTW & RT_BIT(iReg))
2381	return VINF_SUCCESS;
2382	return VERR_NOT_FOUND;
2383	}
2384
2385
2386	DECLINLINE(int) iemFpuStRegNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef) RT_NOEXCEPT
2387	{
2388	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2389	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2390	if (pFpuCtx->FTW & RT_BIT(iReg))
2391	{
2392	*ppRef = &pFpuCtx->aRegs[iStReg].r80;
2393	return VINF_SUCCESS;
2394	}
2395	return VERR_NOT_FOUND;
2396	}
2397
2398
2399	DECLINLINE(int) iemFpu2StRegsNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0,
2400	uint8_t iStReg1, PCRTFLOAT80U *ppRef1) RT_NOEXCEPT
2401	{
2402	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2403	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2404	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2405	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2406	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
2407	{
2408	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2409	*ppRef1 = &pFpuCtx->aRegs[iStReg1].r80;
2410	return VINF_SUCCESS;
2411	}
2412	return VERR_NOT_FOUND;
2413	}
2414
2415
2416	DECLINLINE(int) iemFpu2StRegsNotEmptyRefFirst(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1) RT_NOEXCEPT
2417	{
2418	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2419	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2420	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2421	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2422	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
2423	{
2424	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2425	return VINF_SUCCESS;
2426	}
2427	return VERR_NOT_FOUND;
2428	}
2429
2430
2431	/**
2432	* Rotates the stack registers when setting new TOS.
2433	*
2434	* @param pFpuCtx The FPU context.
2435	* @param iNewTop New TOS value.
2436	* @remarks We only do this to speed up fxsave/fxrstor which
2437	* arrange the FP registers in stack order.
2438	* MUST be done before writing the new TOS (FSW).
2439	*/
2440	DECLINLINE(void) iemFpuRotateStackSetTop(PX86FXSTATE pFpuCtx, uint16_t iNewTop) RT_NOEXCEPT
2441	{
2442	uint16_t iOldTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2443	RTFLOAT80U ar80Temp[8];
2444
2445	if (iOldTop == iNewTop)
2446	return;
2447
2448	/* Unscrew the stack and get it into 'native' order. */
2449	ar80Temp[0] = pFpuCtx->aRegs[(8 - iOldTop + 0) & X86_FSW_TOP_SMASK].r80;
2450	ar80Temp[1] = pFpuCtx->aRegs[(8 - iOldTop + 1) & X86_FSW_TOP_SMASK].r80;
2451	ar80Temp[2] = pFpuCtx->aRegs[(8 - iOldTop + 2) & X86_FSW_TOP_SMASK].r80;
2452	ar80Temp[3] = pFpuCtx->aRegs[(8 - iOldTop + 3) & X86_FSW_TOP_SMASK].r80;
2453	ar80Temp[4] = pFpuCtx->aRegs[(8 - iOldTop + 4) & X86_FSW_TOP_SMASK].r80;
2454	ar80Temp[5] = pFpuCtx->aRegs[(8 - iOldTop + 5) & X86_FSW_TOP_SMASK].r80;
2455	ar80Temp[6] = pFpuCtx->aRegs[(8 - iOldTop + 6) & X86_FSW_TOP_SMASK].r80;
2456	ar80Temp[7] = pFpuCtx->aRegs[(8 - iOldTop + 7) & X86_FSW_TOP_SMASK].r80;
2457
2458	/* Now rotate the stack to the new position. */
2459	pFpuCtx->aRegs[0].r80 = ar80Temp[(iNewTop + 0) & X86_FSW_TOP_SMASK];
2460	pFpuCtx->aRegs[1].r80 = ar80Temp[(iNewTop + 1) & X86_FSW_TOP_SMASK];
2461	pFpuCtx->aRegs[2].r80 = ar80Temp[(iNewTop + 2) & X86_FSW_TOP_SMASK];
2462	pFpuCtx->aRegs[3].r80 = ar80Temp[(iNewTop + 3) & X86_FSW_TOP_SMASK];
2463	pFpuCtx->aRegs[4].r80 = ar80Temp[(iNewTop + 4) & X86_FSW_TOP_SMASK];
2464	pFpuCtx->aRegs[5].r80 = ar80Temp[(iNewTop + 5) & X86_FSW_TOP_SMASK];
2465	pFpuCtx->aRegs[6].r80 = ar80Temp[(iNewTop + 6) & X86_FSW_TOP_SMASK];
2466	pFpuCtx->aRegs[7].r80 = ar80Temp[(iNewTop + 7) & X86_FSW_TOP_SMASK];
2467	}
2468
2469
2470	/**
2471	* Updates the FPU exception status after FCW is changed.
2472	*
2473	* @param pFpuCtx The FPU context.
2474	*/
2475	DECLINLINE(void) iemFpuRecalcExceptionStatus(PX86FXSTATE pFpuCtx) RT_NOEXCEPT
2476	{
2477	uint16_t u16Fsw = pFpuCtx->FSW;
2478	if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pFpuCtx->FCW & X86_FCW_XCPT_MASK))
2479	u16Fsw \|= X86_FSW_ES \| X86_FSW_B;
2480	else
2481	u16Fsw &= ~(X86_FSW_ES \| X86_FSW_B);
2482	pFpuCtx->FSW = u16Fsw;
2483	}
2484
2485
2486	/**
2487	* Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE.
2488	*
2489	* @returns The full FTW.
2490	* @param pFpuCtx The FPU context.
2491	*/
2492	DECLINLINE(uint16_t) iemFpuCalcFullFtw(PCX86FXSTATE pFpuCtx) RT_NOEXCEPT
2493	{
2494	uint8_t const u8Ftw = (uint8_t)pFpuCtx->FTW;
2495	uint16_t u16Ftw = 0;
2496	unsigned const iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2497	for (unsigned iSt = 0; iSt < 8; iSt++)
2498	{
2499	unsigned const iReg = (iSt + iTop) & 7;
2500	if (!(u8Ftw & RT_BIT(iReg)))
2501	u16Ftw \|= 3 << (iReg * 2); /* empty */
2502	else
2503	{
2504	uint16_t uTag;
2505	PCRTFLOAT80U const pr80Reg = &pFpuCtx->aRegs[iSt].r80;
2506	if (pr80Reg->s.uExponent == 0x7fff)
2507	uTag = 2; /* Exponent is all 1's => Special. */
2508	else if (pr80Reg->s.uExponent == 0x0000)
2509	{
2510	if (pr80Reg->s.uMantissa == 0x0000)
2511	uTag = 1; /* All bits are zero => Zero. */
2512	else
2513	uTag = 2; /* Must be special. */
2514	}
2515	else if (pr80Reg->s.uMantissa & RT_BIT_64(63)) /* The J bit. */
2516	uTag = 0; /* Valid. */
2517	else
2518	uTag = 2; /* Must be special. */
2519
2520	u16Ftw \|= uTag << (iReg * 2);
2521	}
2522	}
2523
2524	return u16Ftw;
2525	}
2526
2527
2528	/**
2529	* Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR).
2530	*
2531	* @returns The compressed FTW.
2532	* @param u16FullFtw The full FTW to convert.
2533	*/
2534	DECLINLINE(uint16_t) iemFpuCompressFtw(uint16_t u16FullFtw) RT_NOEXCEPT
2535	{
2536	uint8_t u8Ftw = 0;
2537	for (unsigned i = 0; i < 8; i++)
2538	{
2539	if ((u16FullFtw & 3) != 3 /empty/)
2540	u8Ftw \|= RT_BIT(i);
2541	u16FullFtw >>= 2;
2542	}
2543
2544	return u8Ftw;
2545	}
2546
2547	/** @} */
2548
2549
2550	/** @name Memory access.
2551	*
2552	* @{
2553	*/
2554
2555
2556	/**
2557	* Checks whether alignment checks are enabled or not.
2558	*
2559	* @returns true if enabled, false if not.
2560	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2561	*/
2562	DECLINLINE(bool) iemMemAreAlignmentChecksEnabled(PVMCPUCC pVCpu) RT_NOEXCEPT
2563	{
2564	AssertCompile(X86_CR0_AM == X86_EFL_AC);
2565	return pVCpu->iem.s.uCpl == 3
2566	&& (((uint32_t)pVCpu->cpum.GstCtx.cr0 & pVCpu->cpum.GstCtx.eflags.u) & X86_CR0_AM);
2567	}
2568
2569	/**
2570	* Checks if the given segment can be written to, raise the appropriate
2571	* exception if not.
2572	*
2573	* @returns VBox strict status code.
2574	*
2575	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2576	* @param pHid Pointer to the hidden register.
2577	* @param iSegReg The register number.
2578	* @param pu64BaseAddr Where to return the base address to use for the
2579	* segment. (In 64-bit code it may differ from the
2580	* base in the hidden segment.)
2581	*/
2582	DECLINLINE(VBOXSTRICTRC) iemMemSegCheckWriteAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid,
2583	uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT
2584	{
2585	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2586
2587	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2588	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
2589	else
2590	{
2591	if (!pHid->Attr.n.u1Present)
2592	{
2593	uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
2594	AssertRelease(uSel == 0);
2595	Log(("iemMemSegCheckWriteAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
2596	return iemRaiseGeneralProtectionFault0(pVCpu);
2597	}
2598
2599	if ( ( (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE)
2600	\|\| !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
2601	&& pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT )
2602	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
2603	*pu64BaseAddr = pHid->u64Base;
2604	}
2605	return VINF_SUCCESS;
2606	}
2607
2608
2609	/**
2610	* Checks if the given segment can be read from, raise the appropriate
2611	* exception if not.
2612	*
2613	* @returns VBox strict status code.
2614	*
2615	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2616	* @param pHid Pointer to the hidden register.
2617	* @param iSegReg The register number.
2618	* @param pu64BaseAddr Where to return the base address to use for the
2619	* segment. (In 64-bit code it may differ from the
2620	* base in the hidden segment.)
2621	*/
2622	DECLINLINE(VBOXSTRICTRC) iemMemSegCheckReadAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid,
2623	uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT
2624	{
2625	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2626
2627	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2628	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
2629	else
2630	{
2631	if (!pHid->Attr.n.u1Present)
2632	{
2633	uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
2634	AssertRelease(uSel == 0);
2635	Log(("iemMemSegCheckReadAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
2636	return iemRaiseGeneralProtectionFault0(pVCpu);
2637	}
2638
2639	if ((pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
2640	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
2641	*pu64BaseAddr = pHid->u64Base;
2642	}
2643	return VINF_SUCCESS;
2644	}
2645
2646
2647	/**
2648	* Maps a physical page.
2649	*
2650	* @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr).
2651	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2652	* @param GCPhysMem The physical address.
2653	* @param fAccess The intended access.
2654	* @param ppvMem Where to return the mapping address.
2655	* @param pLock The PGM lock.
2656	*/
2657	DECLINLINE(int) iemMemPageMap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess,
2658	void **ppvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT
2659	{
2660	#ifdef IEM_LOG_MEMORY_WRITES
2661	if (fAccess & IEM_ACCESS_TYPE_WRITE)
2662	return VERR_PGM_PHYS_TLB_CATCH_ALL;
2663	#endif
2664
2665	/** @todo This API may require some improving later. A private deal with PGM
2666	* regarding locking and unlocking needs to be struct. A couple of TLBs
2667	* living in PGM, but with publicly accessible inlined access methods
2668	* could perhaps be an even better solution. */
2669	int rc = PGMPhysIemGCPhys2Ptr(pVCpu->CTX_SUFF(pVM), pVCpu,
2670	GCPhysMem,
2671	RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE),
2672	pVCpu->iem.s.fBypassHandlers,
2673	ppvMem,
2674	pLock);
2675	/Log(("PGMPhysIemGCPhys2Ptr %Rrc pLock=%.Rhxs\n", rc, sizeof(pLock), pLock));/
2676	AssertMsg(rc == VINF_SUCCESS \|\| RT_FAILURE_NP(rc), ("%Rrc\n", rc));
2677
2678	return rc;
2679	}
2680
2681
2682	/**
2683	* Unmap a page previously mapped by iemMemPageMap.
2684	*
2685	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2686	* @param GCPhysMem The physical address.
2687	* @param fAccess The intended access.
2688	* @param pvMem What iemMemPageMap returned.
2689	* @param pLock The PGM lock.
2690	*/
2691	DECLINLINE(void) iemMemPageUnmap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess,
2692	const void *pvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT
2693	{
2694	NOREF(pVCpu);
2695	NOREF(GCPhysMem);
2696	NOREF(fAccess);
2697	NOREF(pvMem);
2698	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), pLock);
2699	}
2700
2701	#ifdef IEM_WITH_SETJMP
2702
2703	/** @todo slim this down */
2704	DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToReadJmp(PVMCPUCC pVCpu, uint8_t iSegReg,
2705	size_t cbMem, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
2706	{
2707	Assert(cbMem >= 1);
2708	Assert(iSegReg < X86_SREG_COUNT);
2709
2710	/*
2711	* 64-bit mode is simpler.
2712	*/
2713	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2714	{
2715	if (iSegReg >= X86_SREG_FS && iSegReg != UINT8_MAX)
2716	{
2717	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2718	PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
2719	GCPtrMem += pSel->u64Base;
2720	}
2721
2722	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
2723	return GCPtrMem;
2724	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
2725	}
2726	/*
2727	* 16-bit and 32-bit segmentation.
2728	*/
2729	else if (iSegReg != UINT8_MAX)
2730	{
2731	/** @todo Does this apply to segments with 4G-1 limit? */
2732	uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
2733	if (RT_LIKELY(GCPtrLast32 >= (uint32_t)GCPtrMem))
2734	{
2735	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2736	PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
2737	switch (pSel->Attr.u & ( X86DESCATTR_P \| X86DESCATTR_UNUSABLE
2738	\| X86_SEL_TYPE_READ \| X86_SEL_TYPE_WRITE /* same as read */
2739	\| X86_SEL_TYPE_DOWN \| X86_SEL_TYPE_CONF /* same as down */
2740	\| X86_SEL_TYPE_CODE))
2741	{
2742	case X86DESCATTR_P: /* readonly data, expand up */
2743	case X86DESCATTR_P \| X86_SEL_TYPE_WRITE: /* writable data, expand up */
2744	case X86DESCATTR_P \| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ: /* code, read-only */
2745	case X86DESCATTR_P \| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ \| X86_SEL_TYPE_CONF: /* conforming code, read-only */
2746	/* expand up */
2747	if (RT_LIKELY(GCPtrLast32 <= pSel->u32Limit))
2748	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2749	Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x vs %#x\n",
2750	(uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit));
2751	break;
2752
2753	case X86DESCATTR_P \| X86_SEL_TYPE_DOWN: /* readonly data, expand down */
2754	case X86DESCATTR_P \| X86_SEL_TYPE_DOWN \| X86_SEL_TYPE_WRITE: /* writable data, expand down */
2755	/* expand down */
2756	if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
2757	&& ( pSel->Attr.n.u1DefBig
2758	\|\| GCPtrLast32 <= UINT32_C(0xffff)) ))
2759	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2760	Log10(("iemMemApplySegmentToReadJmp: expand down out of bounds %#x..%#x vs %#x..%#x\n",
2761	(uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit, pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT16_MAX));
2762	break;
2763
2764	default:
2765	Log10(("iemMemApplySegmentToReadJmp: bad selector %#x\n", pSel->Attr.u));
2766	iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
2767	break;
2768	}
2769	}
2770	Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x\n",(uint32_t)GCPtrMem, GCPtrLast32));
2771	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
2772	}
2773	/*
2774	* 32-bit flat address.
2775	*/
2776	else
2777	return GCPtrMem;
2778	}
2779
2780
2781	/** @todo slim this down */
2782	DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToWriteJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem,
2783	RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
2784	{
2785	Assert(cbMem >= 1);
2786	Assert(iSegReg < X86_SREG_COUNT);
2787
2788	/*
2789	* 64-bit mode is simpler.
2790	*/
2791	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2792	{
2793	if (iSegReg >= X86_SREG_FS)
2794	{
2795	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2796	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
2797	GCPtrMem += pSel->u64Base;
2798	}
2799
2800	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
2801	return GCPtrMem;
2802	}
2803	/*
2804	* 16-bit and 32-bit segmentation.
2805	*/
2806	else
2807	{
2808	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2809	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
2810	uint32_t const fRelevantAttrs = pSel->Attr.u & ( X86DESCATTR_P \| X86DESCATTR_UNUSABLE
2811	\| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_WRITE \| X86_SEL_TYPE_DOWN);
2812	if (fRelevantAttrs == (X86DESCATTR_P \| X86_SEL_TYPE_WRITE)) /* data, expand up */
2813	{
2814	/* expand up */
2815	uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
2816	if (RT_LIKELY( GCPtrLast32 > pSel->u32Limit
2817	&& GCPtrLast32 > (uint32_t)GCPtrMem))
2818	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2819	}
2820	else if (fRelevantAttrs == (X86DESCATTR_P \| X86_SEL_TYPE_WRITE \| X86_SEL_TYPE_DOWN)) /* data, expand up */
2821	{
2822	/* expand down */
2823	uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
2824	if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
2825	&& GCPtrLast32 <= (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
2826	&& GCPtrLast32 > (uint32_t)GCPtrMem))
2827	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2828	}
2829	else
2830	iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
2831	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
2832	}
2833	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
2834	}
2835
2836	#endif /* IEM_WITH_SETJMP */
2837
2838	/**
2839	* Fakes a long mode stack selector for SS = 0.
2840	*
2841	* @param pDescSs Where to return the fake stack descriptor.
2842	* @param uDpl The DPL we want.
2843	*/
2844	DECLINLINE(void) iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl) RT_NOEXCEPT
2845	{
2846	pDescSs->Long.au64[0] = 0;
2847	pDescSs->Long.au64[1] = 0;
2848	pDescSs->Long.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
2849	pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */
2850	pDescSs->Long.Gen.u2Dpl = uDpl;
2851	pDescSs->Long.Gen.u1Present = 1;
2852	pDescSs->Long.Gen.u1Long = 1;
2853	}
2854
2855	/** @} */
2856
2857
2858	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2859
2860	/**
2861	* Gets CR0 fixed-0 bits in VMX operation.
2862	*
2863	* We do this rather than fetching what we report to the guest (in
2864	* IA32_VMX_CR0_FIXED0 MSR) because real hardware (and so do we) report the same
2865	* values regardless of whether unrestricted-guest feature is available on the CPU.
2866	*
2867	* @returns CR0 fixed-0 bits.
2868	* @param pVCpu The cross context virtual CPU structure.
2869	* @param fVmxNonRootMode Whether the CR0 fixed-0 bits for VMX non-root mode
2870	* must be returned. When @c false, the CR0 fixed-0
2871	* bits for VMX root mode is returned.
2872	*
2873	*/
2874	DECLINLINE(uint64_t) iemVmxGetCr0Fixed0(PCVMCPUCC pVCpu, bool fVmxNonRootMode) RT_NOEXCEPT
2875	{
2876	Assert(IEM_VMX_IS_ROOT_MODE(pVCpu));
2877
2878	PCVMXMSRS pMsrs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs;
2879	if ( fVmxNonRootMode
2880	&& (pMsrs->ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST))
2881	return VMX_V_CR0_FIXED0_UX;
2882	return VMX_V_CR0_FIXED0;
2883	}
2884
2885
2886	/**
2887	* Sets virtual-APIC write emulation as pending.
2888	*
2889	* @param pVCpu The cross context virtual CPU structure.
2890	* @param offApic The offset in the virtual-APIC page that was written.
2891	*/
2892	DECLINLINE(void) iemVmxVirtApicSetPendingWrite(PVMCPUCC pVCpu, uint16_t offApic) RT_NOEXCEPT
2893	{
2894	Assert(offApic < XAPIC_OFF_END + 4);
2895
2896	/*
2897	* Record the currently updated APIC offset, as we need this later for figuring
2898	* out whether to perform TPR, EOI or self-IPI virtualization as well as well
2899	* as for supplying the exit qualification when causing an APIC-write VM-exit.
2900	*/
2901	pVCpu->cpum.GstCtx.hwvirt.vmx.offVirtApicWrite = offApic;
2902
2903	/*
2904	* Flag that we need to perform virtual-APIC write emulation (TPR/PPR/EOI/Self-IPI
2905	* virtualization or APIC-write emulation).
2906	*/
2907	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
2908	VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE);
2909	}
2910
2911	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
2912
2913	#endif /* !VMM_INCLUDED_SRC_include_IEMInline_h */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMInline.h@ 99296

Download in other formats: