IEMInline.h@ 97441

Last change on this file since 97441 was 97441, checked in by vboxsync, 2 years ago
VMM/IEM: Single stepping for short and near jumps (relative) and corrected o16 prefix behaviour on intel CPUs in 64-bit mode (ignored). Also, #DB seems to implicitly clear, or at least not set, the resume flag (RF). bugref:9898
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 92.8 KB

Line
1	/* $Id: IEMInline.h 97441 2022-11-08 00:07:49Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Inlined Functions.
4	*/
5
6	/*
7	* Copyright (C) 2011-2022 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMInline_h
29	#define VMM_INCLUDED_SRC_include_IEMInline_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35
36	/**
37	* Makes status code addjustments (pass up from I/O and access handler)
38	* as well as maintaining statistics.
39	*
40	* @returns Strict VBox status code to pass up.
41	* @param pVCpu The cross context virtual CPU structure of the calling thread.
42	* @param rcStrict The status from executing an instruction.
43	*/
44	DECL_FORCE_INLINE(VBOXSTRICTRC) iemExecStatusCodeFiddling(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
45	{
46	if (rcStrict != VINF_SUCCESS)
47	{
48	if (RT_SUCCESS(rcStrict))
49	{
50	AssertMsg( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
51	\|\| rcStrict == VINF_IOM_R3_IOPORT_READ
52	\|\| rcStrict == VINF_IOM_R3_IOPORT_WRITE
53	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
54	\|\| rcStrict == VINF_IOM_R3_MMIO_READ
55	\|\| rcStrict == VINF_IOM_R3_MMIO_READ_WRITE
56	\|\| rcStrict == VINF_IOM_R3_MMIO_WRITE
57	\|\| rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE
58	\|\| rcStrict == VINF_CPUM_R3_MSR_READ
59	\|\| rcStrict == VINF_CPUM_R3_MSR_WRITE
60	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
61	\|\| rcStrict == VINF_EM_RAW_TO_R3
62	\|\| rcStrict == VINF_EM_TRIPLE_FAULT
63	\|\| rcStrict == VINF_GIM_R3_HYPERCALL
64	/* raw-mode / virt handlers only: */
65	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT
66	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT
67	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT
68	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT
69	\|\| rcStrict == VINF_SELM_SYNC_GDT
70	\|\| rcStrict == VINF_CSAM_PENDING_ACTION
71	\|\| rcStrict == VINF_PATM_CHECK_PATCH_PAGE
72	/* nested hw.virt codes: */
73	\|\| rcStrict == VINF_VMX_VMEXIT
74	\|\| rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE
75	\|\| rcStrict == VINF_VMX_MODIFIES_BEHAVIOR
76	\|\| rcStrict == VINF_SVM_VMEXIT
77	, ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
78	/** @todo adjust for VINF_EM_RAW_EMULATE_INSTR. */
79	int32_t const rcPassUp = pVCpu->iem.s.rcPassUp;
80	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
81	if ( rcStrict == VINF_VMX_VMEXIT
82	&& rcPassUp == VINF_SUCCESS)
83	rcStrict = VINF_SUCCESS;
84	else
85	#endif
86	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
87	if ( rcStrict == VINF_SVM_VMEXIT
88	&& rcPassUp == VINF_SUCCESS)
89	rcStrict = VINF_SUCCESS;
90	else
91	#endif
92	if (rcPassUp == VINF_SUCCESS)
93	pVCpu->iem.s.cRetInfStatuses++;
94	else if ( rcPassUp < VINF_EM_FIRST
95	\|\| rcPassUp > VINF_EM_LAST
96	\|\| rcPassUp < VBOXSTRICTRC_VAL(rcStrict))
97	{
98	Log(("IEM: rcPassUp=%Rrc! rcStrict=%Rrc\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
99	pVCpu->iem.s.cRetPassUpStatus++;
100	rcStrict = rcPassUp;
101	}
102	else
103	{
104	Log(("IEM: rcPassUp=%Rrc rcStrict=%Rrc!\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
105	pVCpu->iem.s.cRetInfStatuses++;
106	}
107	}
108	else if (rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED)
109	pVCpu->iem.s.cRetAspectNotImplemented++;
110	else if (rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
111	pVCpu->iem.s.cRetInstrNotImplemented++;
112	else
113	pVCpu->iem.s.cRetErrStatuses++;
114	}
115	else if (pVCpu->iem.s.rcPassUp != VINF_SUCCESS)
116	{
117	pVCpu->iem.s.cRetPassUpStatus++;
118	rcStrict = pVCpu->iem.s.rcPassUp;
119	}
120
121	return rcStrict;
122	}
123
124
125	/**
126	* Sets the pass up status.
127	*
128	* @returns VINF_SUCCESS.
129	* @param pVCpu The cross context virtual CPU structure of the
130	* calling thread.
131	* @param rcPassUp The pass up status. Must be informational.
132	* VINF_SUCCESS is not allowed.
133	*/
134	DECLINLINE(int) iemSetPassUpStatus(PVMCPUCC pVCpu, VBOXSTRICTRC rcPassUp)
135	{
136	AssertRC(VBOXSTRICTRC_VAL(rcPassUp)); Assert(rcPassUp != VINF_SUCCESS);
137
138	int32_t const rcOldPassUp = pVCpu->iem.s.rcPassUp;
139	if (rcOldPassUp == VINF_SUCCESS)
140	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
141	/* If both are EM scheduling codes, use EM priority rules. */
142	else if ( rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST
143	&& rcPassUp >= VINF_EM_FIRST && rcPassUp <= VINF_EM_LAST)
144	{
145	if (rcPassUp < rcOldPassUp)
146	{
147	Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
148	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
149	}
150	else
151	Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
152	}
153	/* Override EM scheduling with specific status code. */
154	else if (rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST)
155	{
156	Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
157	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
158	}
159	/* Don't override specific status code, first come first served. */
160	else
161	Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
162	return VINF_SUCCESS;
163	}
164
165
166	/**
167	* Calculates the CPU mode.
168	*
169	* This is mainly for updating IEMCPU::enmCpuMode.
170	*
171	* @returns CPU mode.
172	* @param pVCpu The cross context virtual CPU structure of the
173	* calling thread.
174	*/
175	DECLINLINE(IEMMODE) iemCalcCpuMode(PVMCPUCC pVCpu)
176	{
177	if (CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx))
178	return IEMMODE_64BIT;
179	if (pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig) /** @todo check if this is correct... */
180	return IEMMODE_32BIT;
181	return IEMMODE_16BIT;
182	}
183
184
185	/**
186	* Initializes the execution state.
187	*
188	* @param pVCpu The cross context virtual CPU structure of the
189	* calling thread.
190	* @param fBypassHandlers Whether to bypass access handlers.
191	*
192	* @remarks Callers of this must call iemUninitExec() to undo potentially fatal
193	* side-effects in strict builds.
194	*/
195	DECLINLINE(void) iemInitExec(PVMCPUCC pVCpu, bool fBypassHandlers)
196	{
197	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
198	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
199	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
200	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
201	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
202	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
203	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
204	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
205	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
206	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
207
208	pVCpu->iem.s.uCpl = CPUMGetGuestCPL(pVCpu);
209	pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
210	#ifdef VBOX_STRICT
211	pVCpu->iem.s.enmDefAddrMode = (IEMMODE)0xfe;
212	pVCpu->iem.s.enmEffAddrMode = (IEMMODE)0xfe;
213	pVCpu->iem.s.enmDefOpSize = (IEMMODE)0xfe;
214	pVCpu->iem.s.enmEffOpSize = (IEMMODE)0xfe;
215	pVCpu->iem.s.fPrefixes = 0xfeedbeef;
216	pVCpu->iem.s.uRexReg = 127;
217	pVCpu->iem.s.uRexB = 127;
218	pVCpu->iem.s.offModRm = 127;
219	pVCpu->iem.s.uRexIndex = 127;
220	pVCpu->iem.s.iEffSeg = 127;
221	pVCpu->iem.s.idxPrefix = 127;
222	pVCpu->iem.s.uVex3rdReg = 127;
223	pVCpu->iem.s.uVexLength = 127;
224	pVCpu->iem.s.fEvexStuff = 127;
225	pVCpu->iem.s.uFpuOpcode = UINT16_MAX;
226	# ifdef IEM_WITH_CODE_TLB
227	pVCpu->iem.s.offInstrNextByte = UINT16_MAX;
228	pVCpu->iem.s.pbInstrBuf = NULL;
229	pVCpu->iem.s.cbInstrBuf = UINT16_MAX;
230	pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;
231	pVCpu->iem.s.offCurInstrStart = INT16_MAX;
232	pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);
233	# else
234	pVCpu->iem.s.offOpcode = 127;
235	pVCpu->iem.s.cbOpcode = 127;
236	# endif
237	#endif
238
239	pVCpu->iem.s.cActiveMappings = 0;
240	pVCpu->iem.s.iNextMapping = 0;
241	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
242	pVCpu->iem.s.fBypassHandlers = fBypassHandlers;
243	#if 0
244	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
245	if ( CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
246	&& CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS))
247	{
248	PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
249	Assert(pVmcs);
250	RTGCPHYS const GCPhysApicAccess = pVmcs->u64AddrApicAccess.u;
251	if (!PGMHandlerPhysicalIsRegistered(pVCpu->CTX_SUFF(pVM), GCPhysApicAccess))
252	{
253	int rc = PGMHandlerPhysicalRegister(pVCpu->CTX_SUFF(pVM), GCPhysApicAccess, GCPhysApicAccess + X86_PAGE_4K_SIZE - 1,
254	pVCpu->iem.s.hVmxApicAccessPage, NIL_RTR3PTR /* pvUserR3 */,
255	NIL_RTR0PTR /* pvUserR0 /, NIL_RTRCPTR / pvUserRC /, NULL / pszDesc */);
256	AssertRC(rc);
257	}
258	}
259	#endif
260	#endif
261	}
262
263	#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
264	/**
265	* Performs a minimal reinitialization of the execution state.
266	*
267	* This is intended to be used by VM-exits, SMM, LOADALL and other similar
268	* 'world-switch' types operations on the CPU. Currently only nested
269	* hardware-virtualization uses it.
270	*
271	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
272	*/
273	DECLINLINE(void) iemReInitExec(PVMCPUCC pVCpu)
274	{
275	IEMMODE const enmMode = iemCalcCpuMode(pVCpu);
276	uint8_t const uCpl = CPUMGetGuestCPL(pVCpu);
277
278	pVCpu->iem.s.uCpl = uCpl;
279	pVCpu->iem.s.enmCpuMode = enmMode;
280	pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
281	pVCpu->iem.s.enmEffAddrMode = enmMode;
282	if (enmMode != IEMMODE_64BIT)
283	{
284	pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
285	pVCpu->iem.s.enmEffOpSize = enmMode;
286	}
287	else
288	{
289	pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
290	pVCpu->iem.s.enmEffOpSize = enmMode;
291	}
292	pVCpu->iem.s.iEffSeg = X86_SREG_DS;
293	#ifndef IEM_WITH_CODE_TLB
294	/** @todo Shouldn't we be doing this in IEMTlbInvalidateAll()? */
295	pVCpu->iem.s.offOpcode = 0;
296	pVCpu->iem.s.cbOpcode = 0;
297	#endif
298	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
299	}
300	#endif
301
302	/**
303	* Counterpart to #iemInitExec that undoes evil strict-build stuff.
304	*
305	* @param pVCpu The cross context virtual CPU structure of the
306	* calling thread.
307	*/
308	DECLINLINE(void) iemUninitExec(PVMCPUCC pVCpu)
309	{
310	/* Note! do not touch fInPatchCode here! (see iemUninitExecAndFiddleStatusAndMaybeReenter) */
311	#ifdef VBOX_STRICT
312	# ifdef IEM_WITH_CODE_TLB
313	NOREF(pVCpu);
314	# else
315	pVCpu->iem.s.cbOpcode = 0;
316	# endif
317	#else
318	NOREF(pVCpu);
319	#endif
320	}
321
322
323	/**
324	* Calls iemUninitExec, iemExecStatusCodeFiddling and iemRCRawMaybeReenter.
325	*
326	* Only calling iemRCRawMaybeReenter in raw-mode, obviously.
327	*
328	* @returns Fiddled strict vbox status code, ready to return to non-IEM caller.
329	* @param pVCpu The cross context virtual CPU structure of the calling thread.
330	* @param rcStrict The status code to fiddle.
331	*/
332	DECLINLINE(VBOXSTRICTRC) iemUninitExecAndFiddleStatusAndMaybeReenter(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
333	{
334	iemUninitExec(pVCpu);
335	return iemExecStatusCodeFiddling(pVCpu, rcStrict);
336	}
337
338
339	/**
340	* Macro used by the IEMExec* method to check the given instruction length.
341	*
342	* Will return on failure!
343	*
344	* @param a_cbInstr The given instruction length.
345	* @param a_cbMin The minimum length.
346	*/
347	#define IEMEXEC_ASSERT_INSTR_LEN_RETURN(a_cbInstr, a_cbMin) \
348	AssertMsgReturn((unsigned)(a_cbInstr) - (unsigned)(a_cbMin) <= (unsigned)15 - (unsigned)(a_cbMin), \
349	("cbInstr=%u cbMin=%u\n", (a_cbInstr), (a_cbMin)), VERR_IEM_INVALID_INSTR_LENGTH)
350
351
352
353	#ifndef IEM_WITH_SETJMP
354
355	/**
356	* Fetches the next opcode byte.
357	*
358	* @returns Strict VBox status code.
359	* @param pVCpu The cross context virtual CPU structure of the
360	* calling thread.
361	* @param pu8 Where to return the opcode byte.
362	*/
363	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PVMCPUCC pVCpu, uint8_t *pu8)
364	{
365	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
366	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
367	{
368	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
369	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
370	return VINF_SUCCESS;
371	}
372	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
373	}
374
375	#else /* IEM_WITH_SETJMP */
376
377	/**
378	* Fetches the next opcode byte, longjmp on error.
379	*
380	* @returns The opcode byte.
381	* @param pVCpu The cross context virtual CPU structure of the calling thread.
382	*/
383	DECLINLINE(uint8_t) iemOpcodeGetNextU8Jmp(PVMCPUCC pVCpu)
384	{
385	# ifdef IEM_WITH_CODE_TLB
386	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
387	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
388	if (RT_LIKELY( pbBuf != NULL
389	&& offBuf < pVCpu->iem.s.cbInstrBuf))
390	{
391	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
392	return pbBuf[offBuf];
393	}
394	# else
395	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
396	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
397	{
398	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
399	return pVCpu->iem.s.abOpcode[offOpcode];
400	}
401	# endif
402	return iemOpcodeGetNextU8SlowJmp(pVCpu);
403	}
404
405	#endif /* IEM_WITH_SETJMP */
406
407	/**
408	* Fetches the next opcode byte, returns automatically on failure.
409	*
410	* @param a_pu8 Where to return the opcode byte.
411	* @remark Implicitly references pVCpu.
412	*/
413	#ifndef IEM_WITH_SETJMP
414	# define IEM_OPCODE_GET_NEXT_U8(a_pu8) \
415	do \
416	{ \
417	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pVCpu, (a_pu8)); \
418	if (rcStrict2 == VINF_SUCCESS) \
419	{ /* likely */ } \
420	else \
421	return rcStrict2; \
422	} while (0)
423	#else
424	# define IEM_OPCODE_GET_NEXT_U8(a_pu8) (*(a_pu8) = iemOpcodeGetNextU8Jmp(pVCpu))
425	#endif /* IEM_WITH_SETJMP */
426
427
428	#ifndef IEM_WITH_SETJMP
429	/**
430	* Fetches the next signed byte from the opcode stream.
431	*
432	* @returns Strict VBox status code.
433	* @param pVCpu The cross context virtual CPU structure of the calling thread.
434	* @param pi8 Where to return the signed byte.
435	*/
436	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PVMCPUCC pVCpu, int8_t *pi8)
437	{
438	return iemOpcodeGetNextU8(pVCpu, (uint8_t *)pi8);
439	}
440	#endif /* !IEM_WITH_SETJMP */
441
442
443	/**
444	* Fetches the next signed byte from the opcode stream, returning automatically
445	* on failure.
446	*
447	* @param a_pi8 Where to return the signed byte.
448	* @remark Implicitly references pVCpu.
449	*/
450	#ifndef IEM_WITH_SETJMP
451	# define IEM_OPCODE_GET_NEXT_S8(a_pi8) \
452	do \
453	{ \
454	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pVCpu, (a_pi8)); \
455	if (rcStrict2 != VINF_SUCCESS) \
456	return rcStrict2; \
457	} while (0)
458	#else /* IEM_WITH_SETJMP */
459	# define IEM_OPCODE_GET_NEXT_S8(a_pi8) (*(a_pi8) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
460
461	#endif /* IEM_WITH_SETJMP */
462
463
464	#ifndef IEM_WITH_SETJMP
465	/**
466	* Fetches the next signed byte from the opcode stream, extending it to
467	* unsigned 16-bit.
468	*
469	* @returns Strict VBox status code.
470	* @param pVCpu The cross context virtual CPU structure of the calling thread.
471	* @param pu16 Where to return the unsigned word.
472	*/
473	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PVMCPUCC pVCpu, uint16_t *pu16)
474	{
475	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
476	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
477	return iemOpcodeGetNextS8SxU16Slow(pVCpu, pu16);
478
479	*pu16 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
480	pVCpu->iem.s.offOpcode = offOpcode + 1;
481	return VINF_SUCCESS;
482	}
483	#endif /* !IEM_WITH_SETJMP */
484
485	/**
486	* Fetches the next signed byte from the opcode stream and sign-extending it to
487	* a word, returning automatically on failure.
488	*
489	* @param a_pu16 Where to return the word.
490	* @remark Implicitly references pVCpu.
491	*/
492	#ifndef IEM_WITH_SETJMP
493	# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \
494	do \
495	{ \
496	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pVCpu, (a_pu16)); \
497	if (rcStrict2 != VINF_SUCCESS) \
498	return rcStrict2; \
499	} while (0)
500	#else
501	# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) (*(a_pu16) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
502	#endif
503
504	#ifndef IEM_WITH_SETJMP
505	/**
506	* Fetches the next signed byte from the opcode stream, extending it to
507	* unsigned 32-bit.
508	*
509	* @returns Strict VBox status code.
510	* @param pVCpu The cross context virtual CPU structure of the calling thread.
511	* @param pu32 Where to return the unsigned dword.
512	*/
513	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU32(PVMCPUCC pVCpu, uint32_t *pu32)
514	{
515	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
516	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
517	return iemOpcodeGetNextS8SxU32Slow(pVCpu, pu32);
518
519	*pu32 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
520	pVCpu->iem.s.offOpcode = offOpcode + 1;
521	return VINF_SUCCESS;
522	}
523	#endif /* !IEM_WITH_SETJMP */
524
525	/**
526	* Fetches the next signed byte from the opcode stream and sign-extending it to
527	* a word, returning automatically on failure.
528	*
529	* @param a_pu32 Where to return the word.
530	* @remark Implicitly references pVCpu.
531	*/
532	#ifndef IEM_WITH_SETJMP
533	#define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) \
534	do \
535	{ \
536	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU32(pVCpu, (a_pu32)); \
537	if (rcStrict2 != VINF_SUCCESS) \
538	return rcStrict2; \
539	} while (0)
540	#else
541	# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) (*(a_pu32) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
542	#endif
543
544
545	#ifndef IEM_WITH_SETJMP
546	/**
547	* Fetches the next signed byte from the opcode stream, extending it to
548	* unsigned 64-bit.
549	*
550	* @returns Strict VBox status code.
551	* @param pVCpu The cross context virtual CPU structure of the calling thread.
552	* @param pu64 Where to return the unsigned qword.
553	*/
554	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU64(PVMCPUCC pVCpu, uint64_t *pu64)
555	{
556	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
557	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
558	return iemOpcodeGetNextS8SxU64Slow(pVCpu, pu64);
559
560	*pu64 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
561	pVCpu->iem.s.offOpcode = offOpcode + 1;
562	return VINF_SUCCESS;
563	}
564	#endif /* !IEM_WITH_SETJMP */
565
566	/**
567	* Fetches the next signed byte from the opcode stream and sign-extending it to
568	* a word, returning automatically on failure.
569	*
570	* @param a_pu64 Where to return the word.
571	* @remark Implicitly references pVCpu.
572	*/
573	#ifndef IEM_WITH_SETJMP
574	# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) \
575	do \
576	{ \
577	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU64(pVCpu, (a_pu64)); \
578	if (rcStrict2 != VINF_SUCCESS) \
579	return rcStrict2; \
580	} while (0)
581	#else
582	# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) (*(a_pu64) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
583	#endif
584
585
586	#ifndef IEM_WITH_SETJMP
587	/**
588	* Fetches the next opcode byte.
589	*
590	* @returns Strict VBox status code.
591	* @param pVCpu The cross context virtual CPU structure of the
592	* calling thread.
593	* @param pu8 Where to return the opcode byte.
594	*/
595	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextRm(PVMCPUCC pVCpu, uint8_t *pu8)
596	{
597	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
598	pVCpu->iem.s.offModRm = offOpcode;
599	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
600	{
601	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
602	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
603	return VINF_SUCCESS;
604	}
605	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
606	}
607	#else /* IEM_WITH_SETJMP */
608	/**
609	* Fetches the next opcode byte, longjmp on error.
610	*
611	* @returns The opcode byte.
612	* @param pVCpu The cross context virtual CPU structure of the calling thread.
613	*/
614	DECLINLINE(uint8_t) iemOpcodeGetNextRmJmp(PVMCPUCC pVCpu)
615	{
616	# ifdef IEM_WITH_CODE_TLB
617	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
618	pVCpu->iem.s.offModRm = offBuf;
619	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
620	if (RT_LIKELY( pbBuf != NULL
621	&& offBuf < pVCpu->iem.s.cbInstrBuf))
622	{
623	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
624	return pbBuf[offBuf];
625	}
626	# else
627	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
628	pVCpu->iem.s.offModRm = offOpcode;
629	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
630	{
631	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
632	return pVCpu->iem.s.abOpcode[offOpcode];
633	}
634	# endif
635	return iemOpcodeGetNextU8SlowJmp(pVCpu);
636	}
637	#endif /* IEM_WITH_SETJMP */
638
639	/**
640	* Fetches the next opcode byte, which is a ModR/M byte, returns automatically
641	* on failure.
642	*
643	* Will note down the position of the ModR/M byte for VT-x exits.
644	*
645	* @param a_pbRm Where to return the RM opcode byte.
646	* @remark Implicitly references pVCpu.
647	*/
648	#ifndef IEM_WITH_SETJMP
649	# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) \
650	do \
651	{ \
652	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextRm(pVCpu, (a_pbRm)); \
653	if (rcStrict2 == VINF_SUCCESS) \
654	{ /* likely */ } \
655	else \
656	return rcStrict2; \
657	} while (0)
658	#else
659	# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) (*(a_pbRm) = iemOpcodeGetNextRmJmp(pVCpu))
660	#endif /* IEM_WITH_SETJMP */
661
662
663	#ifndef IEM_WITH_SETJMP
664
665	/**
666	* Fetches the next opcode word.
667	*
668	* @returns Strict VBox status code.
669	* @param pVCpu The cross context virtual CPU structure of the calling thread.
670	* @param pu16 Where to return the opcode word.
671	*/
672	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PVMCPUCC pVCpu, uint16_t *pu16)
673	{
674	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
675	if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
676	{
677	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
678	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
679	pu16 = (uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
680	# else
681	*pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
682	# endif
683	return VINF_SUCCESS;
684	}
685	return iemOpcodeGetNextU16Slow(pVCpu, pu16);
686	}
687
688	#else /* IEM_WITH_SETJMP */
689
690	/**
691	* Fetches the next opcode word, longjmp on error.
692	*
693	* @returns The opcode word.
694	* @param pVCpu The cross context virtual CPU structure of the calling thread.
695	*/
696	DECLINLINE(uint16_t) iemOpcodeGetNextU16Jmp(PVMCPUCC pVCpu)
697	{
698	# ifdef IEM_WITH_CODE_TLB
699	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
700	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
701	if (RT_LIKELY( pbBuf != NULL
702	&& offBuf + 2 <= pVCpu->iem.s.cbInstrBuf))
703	{
704	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 2;
705	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
706	return (uint16_t const )&pbBuf[offBuf];
707	# else
708	return RT_MAKE_U16(pbBuf[offBuf], pbBuf[offBuf + 1]);
709	# endif
710	}
711	# else
712	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
713	if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
714	{
715	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
716	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
717	return (uint16_t const )&pVCpu->iem.s.abOpcode[offOpcode];
718	# else
719	return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
720	# endif
721	}
722	# endif
723	return iemOpcodeGetNextU16SlowJmp(pVCpu);
724	}
725
726	#endif /* IEM_WITH_SETJMP */
727
728	/**
729	* Fetches the next opcode word, returns automatically on failure.
730	*
731	* @param a_pu16 Where to return the opcode word.
732	* @remark Implicitly references pVCpu.
733	*/
734	#ifndef IEM_WITH_SETJMP
735	# define IEM_OPCODE_GET_NEXT_U16(a_pu16) \
736	do \
737	{ \
738	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pVCpu, (a_pu16)); \
739	if (rcStrict2 != VINF_SUCCESS) \
740	return rcStrict2; \
741	} while (0)
742	#else
743	# define IEM_OPCODE_GET_NEXT_U16(a_pu16) (*(a_pu16) = iemOpcodeGetNextU16Jmp(pVCpu))
744	#endif
745
746	#ifndef IEM_WITH_SETJMP
747	/**
748	* Fetches the next opcode word, zero extending it to a double word.
749	*
750	* @returns Strict VBox status code.
751	* @param pVCpu The cross context virtual CPU structure of the calling thread.
752	* @param pu32 Where to return the opcode double word.
753	*/
754	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PVMCPUCC pVCpu, uint32_t *pu32)
755	{
756	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
757	if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
758	return iemOpcodeGetNextU16ZxU32Slow(pVCpu, pu32);
759
760	*pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
761	pVCpu->iem.s.offOpcode = offOpcode + 2;
762	return VINF_SUCCESS;
763	}
764	#endif /* !IEM_WITH_SETJMP */
765
766	/**
767	* Fetches the next opcode word and zero extends it to a double word, returns
768	* automatically on failure.
769	*
770	* @param a_pu32 Where to return the opcode double word.
771	* @remark Implicitly references pVCpu.
772	*/
773	#ifndef IEM_WITH_SETJMP
774	# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \
775	do \
776	{ \
777	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pVCpu, (a_pu32)); \
778	if (rcStrict2 != VINF_SUCCESS) \
779	return rcStrict2; \
780	} while (0)
781	#else
782	# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU16Jmp(pVCpu))
783	#endif
784
785	#ifndef IEM_WITH_SETJMP
786	/**
787	* Fetches the next opcode word, zero extending it to a quad word.
788	*
789	* @returns Strict VBox status code.
790	* @param pVCpu The cross context virtual CPU structure of the calling thread.
791	* @param pu64 Where to return the opcode quad word.
792	*/
793	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PVMCPUCC pVCpu, uint64_t *pu64)
794	{
795	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
796	if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
797	return iemOpcodeGetNextU16ZxU64Slow(pVCpu, pu64);
798
799	*pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
800	pVCpu->iem.s.offOpcode = offOpcode + 2;
801	return VINF_SUCCESS;
802	}
803	#endif /* !IEM_WITH_SETJMP */
804
805	/**
806	* Fetches the next opcode word and zero extends it to a quad word, returns
807	* automatically on failure.
808	*
809	* @param a_pu64 Where to return the opcode quad word.
810	* @remark Implicitly references pVCpu.
811	*/
812	#ifndef IEM_WITH_SETJMP
813	# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \
814	do \
815	{ \
816	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pVCpu, (a_pu64)); \
817	if (rcStrict2 != VINF_SUCCESS) \
818	return rcStrict2; \
819	} while (0)
820	#else
821	# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU16Jmp(pVCpu))
822	#endif
823
824
825	#ifndef IEM_WITH_SETJMP
826	/**
827	* Fetches the next signed word from the opcode stream.
828	*
829	* @returns Strict VBox status code.
830	* @param pVCpu The cross context virtual CPU structure of the calling thread.
831	* @param pi16 Where to return the signed word.
832	*/
833	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PVMCPUCC pVCpu, int16_t *pi16)
834	{
835	return iemOpcodeGetNextU16(pVCpu, (uint16_t *)pi16);
836	}
837	#endif /* !IEM_WITH_SETJMP */
838
839
840	/**
841	* Fetches the next signed word from the opcode stream, returning automatically
842	* on failure.
843	*
844	* @param a_pi16 Where to return the signed word.
845	* @remark Implicitly references pVCpu.
846	*/
847	#ifndef IEM_WITH_SETJMP
848	# define IEM_OPCODE_GET_NEXT_S16(a_pi16) \
849	do \
850	{ \
851	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pVCpu, (a_pi16)); \
852	if (rcStrict2 != VINF_SUCCESS) \
853	return rcStrict2; \
854	} while (0)
855	#else
856	# define IEM_OPCODE_GET_NEXT_S16(a_pi16) (*(a_pi16) = (int16_t)iemOpcodeGetNextU16Jmp(pVCpu))
857	#endif
858
859	#ifndef IEM_WITH_SETJMP
860
861	/**
862	* Fetches the next opcode dword.
863	*
864	* @returns Strict VBox status code.
865	* @param pVCpu The cross context virtual CPU structure of the calling thread.
866	* @param pu32 Where to return the opcode double word.
867	*/
868	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PVMCPUCC pVCpu, uint32_t *pu32)
869	{
870	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
871	if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
872	{
873	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
874	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
875	pu32 = (uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
876	# else
877	*pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
878	pVCpu->iem.s.abOpcode[offOpcode + 1],
879	pVCpu->iem.s.abOpcode[offOpcode + 2],
880	pVCpu->iem.s.abOpcode[offOpcode + 3]);
881	# endif
882	return VINF_SUCCESS;
883	}
884	return iemOpcodeGetNextU32Slow(pVCpu, pu32);
885	}
886
887	#else /* IEM_WITH_SETJMP */
888
889	/**
890	* Fetches the next opcode dword, longjmp on error.
891	*
892	* @returns The opcode dword.
893	* @param pVCpu The cross context virtual CPU structure of the calling thread.
894	*/
895	DECLINLINE(uint32_t) iemOpcodeGetNextU32Jmp(PVMCPUCC pVCpu)
896	{
897	# ifdef IEM_WITH_CODE_TLB
898	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
899	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
900	if (RT_LIKELY( pbBuf != NULL
901	&& offBuf + 4 <= pVCpu->iem.s.cbInstrBuf))
902	{
903	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 4;
904	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
905	return (uint32_t const )&pbBuf[offBuf];
906	# else
907	return RT_MAKE_U32_FROM_U8(pbBuf[offBuf],
908	pbBuf[offBuf + 1],
909	pbBuf[offBuf + 2],
910	pbBuf[offBuf + 3]);
911	# endif
912	}
913	# else
914	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
915	if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
916	{
917	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
918	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
919	return (uint32_t const )&pVCpu->iem.s.abOpcode[offOpcode];
920	# else
921	return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
922	pVCpu->iem.s.abOpcode[offOpcode + 1],
923	pVCpu->iem.s.abOpcode[offOpcode + 2],
924	pVCpu->iem.s.abOpcode[offOpcode + 3]);
925	# endif
926	}
927	# endif
928	return iemOpcodeGetNextU32SlowJmp(pVCpu);
929	}
930
931	#endif /* IEM_WITH_SETJMP */
932
933	/**
934	* Fetches the next opcode dword, returns automatically on failure.
935	*
936	* @param a_pu32 Where to return the opcode dword.
937	* @remark Implicitly references pVCpu.
938	*/
939	#ifndef IEM_WITH_SETJMP
940	# define IEM_OPCODE_GET_NEXT_U32(a_pu32) \
941	do \
942	{ \
943	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pVCpu, (a_pu32)); \
944	if (rcStrict2 != VINF_SUCCESS) \
945	return rcStrict2; \
946	} while (0)
947	#else
948	# define IEM_OPCODE_GET_NEXT_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU32Jmp(pVCpu))
949	#endif
950
951	#ifndef IEM_WITH_SETJMP
952	/**
953	* Fetches the next opcode dword, zero extending it to a quad word.
954	*
955	* @returns Strict VBox status code.
956	* @param pVCpu The cross context virtual CPU structure of the calling thread.
957	* @param pu64 Where to return the opcode quad word.
958	*/
959	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PVMCPUCC pVCpu, uint64_t *pu64)
960	{
961	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
962	if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
963	return iemOpcodeGetNextU32ZxU64Slow(pVCpu, pu64);
964
965	*pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
966	pVCpu->iem.s.abOpcode[offOpcode + 1],
967	pVCpu->iem.s.abOpcode[offOpcode + 2],
968	pVCpu->iem.s.abOpcode[offOpcode + 3]);
969	pVCpu->iem.s.offOpcode = offOpcode + 4;
970	return VINF_SUCCESS;
971	}
972	#endif /* !IEM_WITH_SETJMP */
973
974	/**
975	* Fetches the next opcode dword and zero extends it to a quad word, returns
976	* automatically on failure.
977	*
978	* @param a_pu64 Where to return the opcode quad word.
979	* @remark Implicitly references pVCpu.
980	*/
981	#ifndef IEM_WITH_SETJMP
982	# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \
983	do \
984	{ \
985	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pVCpu, (a_pu64)); \
986	if (rcStrict2 != VINF_SUCCESS) \
987	return rcStrict2; \
988	} while (0)
989	#else
990	# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU32Jmp(pVCpu))
991	#endif
992
993
994	#ifndef IEM_WITH_SETJMP
995	/**
996	* Fetches the next signed double word from the opcode stream.
997	*
998	* @returns Strict VBox status code.
999	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1000	* @param pi32 Where to return the signed double word.
1001	*/
1002	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PVMCPUCC pVCpu, int32_t *pi32)
1003	{
1004	return iemOpcodeGetNextU32(pVCpu, (uint32_t *)pi32);
1005	}
1006	#endif
1007
1008	/**
1009	* Fetches the next signed double word from the opcode stream, returning
1010	* automatically on failure.
1011	*
1012	* @param a_pi32 Where to return the signed double word.
1013	* @remark Implicitly references pVCpu.
1014	*/
1015	#ifndef IEM_WITH_SETJMP
1016	# define IEM_OPCODE_GET_NEXT_S32(a_pi32) \
1017	do \
1018	{ \
1019	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pVCpu, (a_pi32)); \
1020	if (rcStrict2 != VINF_SUCCESS) \
1021	return rcStrict2; \
1022	} while (0)
1023	#else
1024	# define IEM_OPCODE_GET_NEXT_S32(a_pi32) (*(a_pi32) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1025	#endif
1026
1027	#ifndef IEM_WITH_SETJMP
1028	/**
1029	* Fetches the next opcode dword, sign extending it into a quad word.
1030	*
1031	* @returns Strict VBox status code.
1032	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1033	* @param pu64 Where to return the opcode quad word.
1034	*/
1035	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64)
1036	{
1037	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1038	if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1039	return iemOpcodeGetNextS32SxU64Slow(pVCpu, pu64);
1040
1041	int32_t i32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1042	pVCpu->iem.s.abOpcode[offOpcode + 1],
1043	pVCpu->iem.s.abOpcode[offOpcode + 2],
1044	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1045	*pu64 = i32;
1046	pVCpu->iem.s.offOpcode = offOpcode + 4;
1047	return VINF_SUCCESS;
1048	}
1049	#endif /* !IEM_WITH_SETJMP */
1050
1051	/**
1052	* Fetches the next opcode double word and sign extends it to a quad word,
1053	* returns automatically on failure.
1054	*
1055	* @param a_pu64 Where to return the opcode quad word.
1056	* @remark Implicitly references pVCpu.
1057	*/
1058	#ifndef IEM_WITH_SETJMP
1059	# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \
1060	do \
1061	{ \
1062	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pVCpu, (a_pu64)); \
1063	if (rcStrict2 != VINF_SUCCESS) \
1064	return rcStrict2; \
1065	} while (0)
1066	#else
1067	# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) (*(a_pu64) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1068	#endif
1069
1070	#ifndef IEM_WITH_SETJMP
1071
1072	/**
1073	* Fetches the next opcode qword.
1074	*
1075	* @returns Strict VBox status code.
1076	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1077	* @param pu64 Where to return the opcode qword.
1078	*/
1079	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PVMCPUCC pVCpu, uint64_t *pu64)
1080	{
1081	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1082	if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1083	{
1084	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1085	pu64 = (uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1086	# else
1087	*pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1088	pVCpu->iem.s.abOpcode[offOpcode + 1],
1089	pVCpu->iem.s.abOpcode[offOpcode + 2],
1090	pVCpu->iem.s.abOpcode[offOpcode + 3],
1091	pVCpu->iem.s.abOpcode[offOpcode + 4],
1092	pVCpu->iem.s.abOpcode[offOpcode + 5],
1093	pVCpu->iem.s.abOpcode[offOpcode + 6],
1094	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1095	# endif
1096	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1097	return VINF_SUCCESS;
1098	}
1099	return iemOpcodeGetNextU64Slow(pVCpu, pu64);
1100	}
1101
1102	#else /* IEM_WITH_SETJMP */
1103
1104	/**
1105	* Fetches the next opcode qword, longjmp on error.
1106	*
1107	* @returns The opcode qword.
1108	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1109	*/
1110	DECLINLINE(uint64_t) iemOpcodeGetNextU64Jmp(PVMCPUCC pVCpu)
1111	{
1112	# ifdef IEM_WITH_CODE_TLB
1113	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1114	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1115	if (RT_LIKELY( pbBuf != NULL
1116	&& offBuf + 8 <= pVCpu->iem.s.cbInstrBuf))
1117	{
1118	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 8;
1119	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1120	return (uint64_t const )&pbBuf[offBuf];
1121	# else
1122	return RT_MAKE_U64_FROM_U8(pbBuf[offBuf],
1123	pbBuf[offBuf + 1],
1124	pbBuf[offBuf + 2],
1125	pbBuf[offBuf + 3],
1126	pbBuf[offBuf + 4],
1127	pbBuf[offBuf + 5],
1128	pbBuf[offBuf + 6],
1129	pbBuf[offBuf + 7]);
1130	# endif
1131	}
1132	# else
1133	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1134	if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1135	{
1136	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1137	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1138	return (uint64_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1139	# else
1140	return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1141	pVCpu->iem.s.abOpcode[offOpcode + 1],
1142	pVCpu->iem.s.abOpcode[offOpcode + 2],
1143	pVCpu->iem.s.abOpcode[offOpcode + 3],
1144	pVCpu->iem.s.abOpcode[offOpcode + 4],
1145	pVCpu->iem.s.abOpcode[offOpcode + 5],
1146	pVCpu->iem.s.abOpcode[offOpcode + 6],
1147	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1148	# endif
1149	}
1150	# endif
1151	return iemOpcodeGetNextU64SlowJmp(pVCpu);
1152	}
1153
1154	#endif /* IEM_WITH_SETJMP */
1155
1156	/**
1157	* Fetches the next opcode quad word, returns automatically on failure.
1158	*
1159	* @param a_pu64 Where to return the opcode quad word.
1160	* @remark Implicitly references pVCpu.
1161	*/
1162	#ifndef IEM_WITH_SETJMP
1163	# define IEM_OPCODE_GET_NEXT_U64(a_pu64) \
1164	do \
1165	{ \
1166	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pVCpu, (a_pu64)); \
1167	if (rcStrict2 != VINF_SUCCESS) \
1168	return rcStrict2; \
1169	} while (0)
1170	#else
1171	# define IEM_OPCODE_GET_NEXT_U64(a_pu64) ( *(a_pu64) = iemOpcodeGetNextU64Jmp(pVCpu) )
1172	#endif
1173
1174
1175	/** @name Misc Worker Functions.
1176	* @{
1177	*/
1178
1179	/**
1180	* Gets the correct EFLAGS regardless of whether PATM stores parts of them or
1181	* not (kind of obsolete now).
1182	*
1183	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1184	*/
1185	#define IEMMISC_GET_EFL(a_pVCpu) ( (a_pVCpu)->cpum.GstCtx.eflags.u )
1186
1187	/**
1188	* Updates the EFLAGS in the correct manner wrt. PATM (kind of obsolete).
1189	*
1190	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1191	* @param a_fEfl The new EFLAGS.
1192	*/
1193	#define IEMMISC_SET_EFL(a_pVCpu, a_fEfl) do { (a_pVCpu)->cpum.GstCtx.eflags.u = (a_fEfl); } while (0)
1194
1195
1196	/**
1197	* Loads a NULL data selector into a selector register, both the hidden and
1198	* visible parts, in protected mode.
1199	*
1200	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1201	* @param pSReg Pointer to the segment register.
1202	* @param uRpl The RPL.
1203	*/
1204	DECLINLINE(void) iemHlpLoadNullDataSelectorProt(PVMCPUCC pVCpu, PCPUMSELREG pSReg, RTSEL uRpl)
1205	{
1206	/** @todo Testcase: write a testcase checking what happends when loading a NULL
1207	* data selector in protected mode. */
1208	pSReg->Sel = uRpl;
1209	pSReg->ValidSel = uRpl;
1210	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
1211	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
1212	{
1213	/* VT-x (Intel 3960x) observed doing something like this. */
1214	pSReg->Attr.u = X86DESCATTR_UNUSABLE \| X86DESCATTR_G \| X86DESCATTR_D \| (pVCpu->iem.s.uCpl << X86DESCATTR_DPL_SHIFT);
1215	pSReg->u32Limit = UINT32_MAX;
1216	pSReg->u64Base = 0;
1217	}
1218	else
1219	{
1220	pSReg->Attr.u = X86DESCATTR_UNUSABLE;
1221	pSReg->u32Limit = 0;
1222	pSReg->u64Base = 0;
1223	}
1224	}
1225
1226	/** @} */
1227
1228
1229	/*
1230	*
1231	* Helpers routines.
1232	* Helpers routines.
1233	* Helpers routines.
1234	*
1235	*/
1236
1237	/**
1238	* Recalculates the effective operand size.
1239	*
1240	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1241	*/
1242	DECLINLINE(void) iemRecalEffOpSize(PVMCPUCC pVCpu)
1243	{
1244	switch (pVCpu->iem.s.enmCpuMode)
1245	{
1246	case IEMMODE_16BIT:
1247	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT;
1248	break;
1249	case IEMMODE_32BIT:
1250	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT;
1251	break;
1252	case IEMMODE_64BIT:
1253	switch (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP))
1254	{
1255	case 0:
1256	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize;
1257	break;
1258	case IEM_OP_PRF_SIZE_OP:
1259	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1260	break;
1261	case IEM_OP_PRF_SIZE_REX_W:
1262	case IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP:
1263	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1264	break;
1265	}
1266	break;
1267	default:
1268	AssertFailed();
1269	}
1270	}
1271
1272
1273	/**
1274	* Sets the default operand size to 64-bit and recalculates the effective
1275	* operand size.
1276	*
1277	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1278	*/
1279	DECLINLINE(void) iemRecalEffOpSize64Default(PVMCPUCC pVCpu)
1280	{
1281	Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
1282	pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1283	if ((pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP)
1284	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1285	else
1286	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1287	}
1288
1289
1290	/**
1291	* Sets the default operand size to 64-bit and recalculates the effective
1292	* operand size, with intel ignoring any operand size prefix (AMD respects it).
1293	*
1294	* This is for the relative jumps.
1295	*
1296	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1297	*/
1298	DECLINLINE(void) iemRecalEffOpSize64DefaultAndIntelIgnoresOpSizePrefix(PVMCPUCC pVCpu)
1299	{
1300	Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
1301	pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1302	if ( (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP
1303	\|\| pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
1304	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1305	else
1306	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1307	}
1308
1309
1310
1311
1312	/** @name Register Access.
1313	* @{
1314	*/
1315
1316	/**
1317	* Gets a reference (pointer) to the specified hidden segment register.
1318	*
1319	* @returns Hidden register reference.
1320	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1321	* @param iSegReg The segment register.
1322	*/
1323	DECLINLINE(PCPUMSELREG) iemSRegGetHid(PVMCPUCC pVCpu, uint8_t iSegReg)
1324	{
1325	Assert(iSegReg < X86_SREG_COUNT);
1326	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1327	PCPUMSELREG pSReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
1328
1329	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1330	return pSReg;
1331	}
1332
1333
1334	/**
1335	* Ensures that the given hidden segment register is up to date.
1336	*
1337	* @returns Hidden register reference.
1338	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1339	* @param pSReg The segment register.
1340	*/
1341	DECLINLINE(PCPUMSELREG) iemSRegUpdateHid(PVMCPUCC pVCpu, PCPUMSELREG pSReg)
1342	{
1343	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1344	NOREF(pVCpu);
1345	return pSReg;
1346	}
1347
1348
1349	/**
1350	* Gets a reference (pointer) to the specified segment register (the selector
1351	* value).
1352	*
1353	* @returns Pointer to the selector variable.
1354	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1355	* @param iSegReg The segment register.
1356	*/
1357	DECLINLINE(uint16_t *) iemSRegRef(PVMCPUCC pVCpu, uint8_t iSegReg)
1358	{
1359	Assert(iSegReg < X86_SREG_COUNT);
1360	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1361	return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1362	}
1363
1364
1365	/**
1366	* Fetches the selector value of a segment register.
1367	*
1368	* @returns The selector value.
1369	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1370	* @param iSegReg The segment register.
1371	*/
1372	DECLINLINE(uint16_t) iemSRegFetchU16(PVMCPUCC pVCpu, uint8_t iSegReg)
1373	{
1374	Assert(iSegReg < X86_SREG_COUNT);
1375	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1376	return pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1377	}
1378
1379
1380	/**
1381	* Fetches the base address value of a segment register.
1382	*
1383	* @returns The selector value.
1384	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1385	* @param iSegReg The segment register.
1386	*/
1387	DECLINLINE(uint64_t) iemSRegBaseFetchU64(PVMCPUCC pVCpu, uint8_t iSegReg)
1388	{
1389	Assert(iSegReg < X86_SREG_COUNT);
1390	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1391	return pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1392	}
1393
1394
1395	/**
1396	* Gets a reference (pointer) to the specified general purpose register.
1397	*
1398	* @returns Register reference.
1399	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1400	* @param iReg The general purpose register.
1401	*/
1402	DECLINLINE(void *) iemGRegRef(PVMCPUCC pVCpu, uint8_t iReg)
1403	{
1404	Assert(iReg < 16);
1405	return &pVCpu->cpum.GstCtx.aGRegs[iReg];
1406	}
1407
1408
1409	/**
1410	* Gets a reference (pointer) to the specified 8-bit general purpose register.
1411	*
1412	* Because of AH, CH, DH and BH we cannot use iemGRegRef directly here.
1413	*
1414	* @returns Register reference.
1415	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1416	* @param iReg The register.
1417	*/
1418	DECLINLINE(uint8_t *) iemGRegRefU8(PVMCPUCC pVCpu, uint8_t iReg)
1419	{
1420	if (iReg < 4 \|\| (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX))
1421	{
1422	Assert(iReg < 16);
1423	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u8;
1424	}
1425	/* high 8-bit register. */
1426	Assert(iReg < 8);
1427	return &pVCpu->cpum.GstCtx.aGRegs[iReg & 3].bHi;
1428	}
1429
1430
1431	/**
1432	* Gets a reference (pointer) to the specified 16-bit general purpose register.
1433	*
1434	* @returns Register reference.
1435	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1436	* @param iReg The register.
1437	*/
1438	DECLINLINE(uint16_t *) iemGRegRefU16(PVMCPUCC pVCpu, uint8_t iReg)
1439	{
1440	Assert(iReg < 16);
1441	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1442	}
1443
1444
1445	/**
1446	* Gets a reference (pointer) to the specified 32-bit general purpose register.
1447	*
1448	* @returns Register reference.
1449	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1450	* @param iReg The register.
1451	*/
1452	DECLINLINE(uint32_t *) iemGRegRefU32(PVMCPUCC pVCpu, uint8_t iReg)
1453	{
1454	Assert(iReg < 16);
1455	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1456	}
1457
1458
1459	/**
1460	* Gets a reference (pointer) to the specified signed 32-bit general purpose register.
1461	*
1462	* @returns Register reference.
1463	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1464	* @param iReg The register.
1465	*/
1466	DECLINLINE(int32_t *) iemGRegRefI32(PVMCPUCC pVCpu, uint8_t iReg)
1467	{
1468	Assert(iReg < 16);
1469	return (int32_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1470	}
1471
1472
1473	/**
1474	* Gets a reference (pointer) to the specified 64-bit general purpose register.
1475	*
1476	* @returns Register reference.
1477	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1478	* @param iReg The register.
1479	*/
1480	DECLINLINE(uint64_t *) iemGRegRefU64(PVMCPUCC pVCpu, uint8_t iReg)
1481	{
1482	Assert(iReg < 64);
1483	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1484	}
1485
1486
1487	/**
1488	* Gets a reference (pointer) to the specified signed 64-bit general purpose register.
1489	*
1490	* @returns Register reference.
1491	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1492	* @param iReg The register.
1493	*/
1494	DECLINLINE(int64_t *) iemGRegRefI64(PVMCPUCC pVCpu, uint8_t iReg)
1495	{
1496	Assert(iReg < 16);
1497	return (int64_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1498	}
1499
1500
1501	/**
1502	* Gets a reference (pointer) to the specified segment register's base address.
1503	*
1504	* @returns Segment register base address reference.
1505	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1506	* @param iSegReg The segment selector.
1507	*/
1508	DECLINLINE(uint64_t *) iemSRegBaseRefU64(PVMCPUCC pVCpu, uint8_t iSegReg)
1509	{
1510	Assert(iSegReg < X86_SREG_COUNT);
1511	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1512	return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1513	}
1514
1515
1516	/**
1517	* Fetches the value of a 8-bit general purpose register.
1518	*
1519	* @returns The register value.
1520	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1521	* @param iReg The register.
1522	*/
1523	DECLINLINE(uint8_t) iemGRegFetchU8(PVMCPUCC pVCpu, uint8_t iReg)
1524	{
1525	return *iemGRegRefU8(pVCpu, iReg);
1526	}
1527
1528
1529	/**
1530	* Fetches the value of a 16-bit general purpose register.
1531	*
1532	* @returns The register value.
1533	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1534	* @param iReg The register.
1535	*/
1536	DECLINLINE(uint16_t) iemGRegFetchU16(PVMCPUCC pVCpu, uint8_t iReg)
1537	{
1538	Assert(iReg < 16);
1539	return pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1540	}
1541
1542
1543	/**
1544	* Fetches the value of a 32-bit general purpose register.
1545	*
1546	* @returns The register value.
1547	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1548	* @param iReg The register.
1549	*/
1550	DECLINLINE(uint32_t) iemGRegFetchU32(PVMCPUCC pVCpu, uint8_t iReg)
1551	{
1552	Assert(iReg < 16);
1553	return pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1554	}
1555
1556
1557	/**
1558	* Fetches the value of a 64-bit general purpose register.
1559	*
1560	* @returns The register value.
1561	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1562	* @param iReg The register.
1563	*/
1564	DECLINLINE(uint64_t) iemGRegFetchU64(PVMCPUCC pVCpu, uint8_t iReg)
1565	{
1566	Assert(iReg < 16);
1567	return pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1568	}
1569
1570
1571	/**
1572	* Get the address of the top of the stack.
1573	*
1574	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1575	*/
1576	DECLINLINE(RTGCPTR) iemRegGetEffRsp(PCVMCPU pVCpu)
1577	{
1578	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1579	return pVCpu->cpum.GstCtx.rsp;
1580	if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1581	return pVCpu->cpum.GstCtx.esp;
1582	return pVCpu->cpum.GstCtx.sp;
1583	}
1584
1585
1586	/**
1587	* Updates the RIP/EIP/IP to point to the next instruction.
1588	*
1589	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1590	* @param cbInstr The number of bytes to add.
1591	*/
1592	DECL_FORCE_INLINE(void) iemRegAddToRip(PVMCPUCC pVCpu, uint8_t cbInstr)
1593	{
1594	/*
1595	* Advance RIP.
1596	*
1597	* When we're targetting 8086/8, 80186/8 or 80286 mode the updates are 16-bit,
1598	* while in all other modes except LM64 the updates are 32-bit. This means
1599	* we need to watch for both 32-bit and 16-bit "carry" situations, i.e.
1600	* 4GB and 64KB rollovers, and decide whether anything needs masking.
1601	*
1602	* See PC wrap around tests in bs3-cpu-weird-1.
1603	*/
1604	uint64_t const uRipPrev = pVCpu->cpum.GstCtx.rip;
1605	uint64_t const uRipNext = uRipPrev + cbInstr;
1606	if (RT_LIKELY( !((uRipNext ^ uRipPrev) & (RT_BIT_64(32) \| RT_BIT_64(16)))
1607	\|\| pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT))
1608	pVCpu->cpum.GstCtx.rip = uRipNext;
1609	else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
1610	pVCpu->cpum.GstCtx.rip = (uint32_t)uRipNext;
1611	else
1612	pVCpu->cpum.GstCtx.rip = (uint16_t)uRipNext;
1613	}
1614
1615
1616	/**
1617	* Called by iemRegAddToRipAndFinishingClearingRF and others when any of the
1618	* following EFLAGS bits are set:
1619	* - X86_EFL_RF - clear it.
1620	* - CPUMCTX_INHIBIT_SHADOW (_SS/_STI) - clear them.
1621	* - X86_EFL_TF - generate single step \#DB trap.
1622	* - CPUMCTX_DBG_HIT_DR0/1/2/3 - generate \#DB trap (data or I/O, not
1623	* instruction).
1624	*
1625	* According to @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events},
1626	* a \#DB due to TF (single stepping) or a DRx non-instruction breakpoint
1627	* takes priority over both NMIs and hardware interrupts. So, neither is
1628	* considered here. (The RESET, \#MC, SMI, INIT, STOPCLK and FLUSH events are
1629	* either unsupported will be triggered on-top of any \#DB raised here.)
1630	*
1631	* The RF flag only needs to be cleared here as it only suppresses instruction
1632	* breakpoints which are not raised here (happens synchronously during
1633	* instruction fetching).
1634	*
1635	* The CPUMCTX_INHIBIT_SHADOW_SS flag will be cleared by this function, so its
1636	* status has no bearing on whether \#DB exceptions are raised.
1637	*
1638	* @note This must NOT be called by the two instructions setting the
1639	* CPUMCTX_INHIBIT_SHADOW_SS flag.
1640	*
1641	* @see @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events}
1642	* @see @sdmv3{077,200,6.8.3,Masking Exceptions and Interrupts When Switching
1643	* Stacks}
1644	*/
1645	static VBOXSTRICTRC iemFinishInstructionWithFlagsSet(PVMCPUCC pVCpu)
1646	{
1647	/*
1648	* Normally we're just here to clear RF and/or interrupt shadow bits.
1649	*/
1650	if (RT_LIKELY((pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_TF \| CPUMCTX_DBG_HIT_DRX_MASK)) == 0))
1651	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW);
1652	else
1653	{
1654	#if 1
1655	/*
1656	* Raise a #DB.
1657	*/
1658	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
1659	if (pVCpu->cpum.GstCtx.eflags.uBoth & X86_EFL_TF)
1660	pVCpu->cpum.GstCtx.dr[6] \|= X86_DR6_BS;
1661	pVCpu->cpum.GstCtx.dr[6] \|= (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT;
1662	/** @todo Do we set all pending \#DB events, or just one? */
1663	LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64\n",
1664	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
1665	pVCpu->cpum.GstCtx.rflags.uBoth));
1666	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK);
1667	return iemRaiseDebugException(pVCpu);
1668	#else
1669	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK);
1670	#endif
1671	}
1672	return VINF_SUCCESS;
1673	}
1674
1675
1676	/**
1677	* Clears the RF and CPUMCTX_INHIBIT_SHADOW, triggering \#DB if pending.
1678	*
1679	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1680	*/
1681	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegFinishClearingRF(PVMCPUCC pVCpu)
1682	{
1683	/*
1684	* We assume that most of the time nothing actually needs doing here.
1685	*/
1686	AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX);
1687	if (RT_LIKELY(!( pVCpu->cpum.GstCtx.eflags.uBoth
1688	& (X86_EFL_TF \| X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK)) ))
1689	return VINF_SUCCESS;
1690	return iemFinishInstructionWithFlagsSet(pVCpu);
1691	}
1692
1693
1694	/**
1695	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF
1696	* and CPUMCTX_INHIBIT_SHADOW.
1697	*
1698	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1699	* @param cbInstr The number of bytes to add.
1700	*/
1701	DECLINLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr)
1702	{
1703	iemRegAddToRip(pVCpu, cbInstr);
1704	return iemRegFinishClearingRF(pVCpu);
1705	}
1706
1707
1708	/**
1709	* Extended version of iemFinishInstructionWithFlagsSet that goes with
1710	* iemRegAddToRipAndFinishingClearingRfEx.
1711	*
1712	* See iemFinishInstructionWithFlagsSet() for details.
1713	*/
1714	static VBOXSTRICTRC iemFinishInstructionWithTfSet(PVMCPUCC pVCpu)
1715	{
1716	/*
1717	* Raise a #DB.
1718	*/
1719	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
1720	pVCpu->cpum.GstCtx.dr[6] \|= X86_DR6_BS
1721	\| (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT;
1722	/** @todo Do we set all pending \#DB events, or just one? */
1723	LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64 (popf)\n",
1724	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
1725	pVCpu->cpum.GstCtx.rflags.uBoth));
1726	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK);
1727	return iemRaiseDebugException(pVCpu);
1728	}
1729
1730
1731	/**
1732	* Extended version of iemRegAddToRipAndFinishingClearingRF for use by POPF and
1733	* others potentially updating EFLAGS.TF.
1734	*
1735	* The single step event must be generated using the TF value at the start of
1736	* the instruction, not the new value set by it.
1737	*
1738	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1739	* @param cbInstr The number of bytes to add.
1740	* @param fEflOld The EFLAGS at the start of the instruction
1741	* execution.
1742	*/
1743	DECLINLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRfEx(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t fEflOld)
1744	{
1745	iemRegAddToRip(pVCpu, cbInstr);
1746	if (!(fEflOld & X86_EFL_TF))
1747	return iemRegFinishClearingRF(pVCpu);
1748	return iemFinishInstructionWithTfSet(pVCpu);
1749	}
1750
1751
1752	/**
1753	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
1754	*
1755	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1756	*/
1757	DECLINLINE(VBOXSTRICTRC) iemRegUpdateRipAndFinishClearingRF(PVMCPUCC pVCpu)
1758	{
1759	return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
1760	}
1761
1762
1763	/**
1764	* Adds to the stack pointer.
1765	*
1766	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1767	* @param cbToAdd The number of bytes to add (8-bit!).
1768	*/
1769	DECLINLINE(void) iemRegAddToRsp(PVMCPUCC pVCpu, uint8_t cbToAdd)
1770	{
1771	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1772	pVCpu->cpum.GstCtx.rsp += cbToAdd;
1773	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1774	pVCpu->cpum.GstCtx.esp += cbToAdd;
1775	else
1776	pVCpu->cpum.GstCtx.sp += cbToAdd;
1777	}
1778
1779
1780	/**
1781	* Subtracts from the stack pointer.
1782	*
1783	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1784	* @param cbToSub The number of bytes to subtract (8-bit!).
1785	*/
1786	DECLINLINE(void) iemRegSubFromRsp(PVMCPUCC pVCpu, uint8_t cbToSub)
1787	{
1788	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1789	pVCpu->cpum.GstCtx.rsp -= cbToSub;
1790	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1791	pVCpu->cpum.GstCtx.esp -= cbToSub;
1792	else
1793	pVCpu->cpum.GstCtx.sp -= cbToSub;
1794	}
1795
1796
1797	/**
1798	* Adds to the temporary stack pointer.
1799	*
1800	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1801	* @param pTmpRsp The temporary SP/ESP/RSP to update.
1802	* @param cbToAdd The number of bytes to add (16-bit).
1803	*/
1804	DECLINLINE(void) iemRegAddToRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToAdd)
1805	{
1806	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1807	pTmpRsp->u += cbToAdd;
1808	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1809	pTmpRsp->DWords.dw0 += cbToAdd;
1810	else
1811	pTmpRsp->Words.w0 += cbToAdd;
1812	}
1813
1814
1815	/**
1816	* Subtracts from the temporary stack pointer.
1817	*
1818	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1819	* @param pTmpRsp The temporary SP/ESP/RSP to update.
1820	* @param cbToSub The number of bytes to subtract.
1821	* @remarks The @a cbToSub argument MUST be 16-bit, iemCImpl_enter is
1822	* expecting that.
1823	*/
1824	DECLINLINE(void) iemRegSubFromRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToSub)
1825	{
1826	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1827	pTmpRsp->u -= cbToSub;
1828	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1829	pTmpRsp->DWords.dw0 -= cbToSub;
1830	else
1831	pTmpRsp->Words.w0 -= cbToSub;
1832	}
1833
1834
1835	/**
1836	* Calculates the effective stack address for a push of the specified size as
1837	* well as the new RSP value (upper bits may be masked).
1838	*
1839	* @returns Effective stack addressf for the push.
1840	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1841	* @param cbItem The size of the stack item to pop.
1842	* @param puNewRsp Where to return the new RSP value.
1843	*/
1844	DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp)
1845	{
1846	RTUINT64U uTmpRsp;
1847	RTGCPTR GCPtrTop;
1848	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
1849
1850	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1851	GCPtrTop = uTmpRsp.u -= cbItem;
1852	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1853	GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem;
1854	else
1855	GCPtrTop = uTmpRsp.Words.w0 -= cbItem;
1856	*puNewRsp = uTmpRsp.u;
1857	return GCPtrTop;
1858	}
1859
1860
1861	/**
1862	* Gets the current stack pointer and calculates the value after a pop of the
1863	* specified size.
1864	*
1865	* @returns Current stack pointer.
1866	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1867	* @param cbItem The size of the stack item to pop.
1868	* @param puNewRsp Where to return the new RSP value.
1869	*/
1870	DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp)
1871	{
1872	RTUINT64U uTmpRsp;
1873	RTGCPTR GCPtrTop;
1874	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
1875
1876	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1877	{
1878	GCPtrTop = uTmpRsp.u;
1879	uTmpRsp.u += cbItem;
1880	}
1881	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1882	{
1883	GCPtrTop = uTmpRsp.DWords.dw0;
1884	uTmpRsp.DWords.dw0 += cbItem;
1885	}
1886	else
1887	{
1888	GCPtrTop = uTmpRsp.Words.w0;
1889	uTmpRsp.Words.w0 += cbItem;
1890	}
1891	*puNewRsp = uTmpRsp.u;
1892	return GCPtrTop;
1893	}
1894
1895
1896	/**
1897	* Calculates the effective stack address for a push of the specified size as
1898	* well as the new temporary RSP value (upper bits may be masked).
1899	*
1900	* @returns Effective stack addressf for the push.
1901	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1902	* @param pTmpRsp The temporary stack pointer. This is updated.
1903	* @param cbItem The size of the stack item to pop.
1904	*/
1905	DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem)
1906	{
1907	RTGCPTR GCPtrTop;
1908
1909	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1910	GCPtrTop = pTmpRsp->u -= cbItem;
1911	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1912	GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem;
1913	else
1914	GCPtrTop = pTmpRsp->Words.w0 -= cbItem;
1915	return GCPtrTop;
1916	}
1917
1918
1919	/**
1920	* Gets the effective stack address for a pop of the specified size and
1921	* calculates and updates the temporary RSP.
1922	*
1923	* @returns Current stack pointer.
1924	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1925	* @param pTmpRsp The temporary stack pointer. This is updated.
1926	* @param cbItem The size of the stack item to pop.
1927	*/
1928	DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem)
1929	{
1930	RTGCPTR GCPtrTop;
1931	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1932	{
1933	GCPtrTop = pTmpRsp->u;
1934	pTmpRsp->u += cbItem;
1935	}
1936	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1937	{
1938	GCPtrTop = pTmpRsp->DWords.dw0;
1939	pTmpRsp->DWords.dw0 += cbItem;
1940	}
1941	else
1942	{
1943	GCPtrTop = pTmpRsp->Words.w0;
1944	pTmpRsp->Words.w0 += cbItem;
1945	}
1946	return GCPtrTop;
1947	}
1948
1949	/** @} */
1950
1951
1952	/** @name FPU access and helpers.
1953	*
1954	* @{
1955	*/
1956
1957
1958	/**
1959	* Hook for preparing to use the host FPU.
1960	*
1961	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1962	*
1963	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1964	*/
1965	DECLINLINE(void) iemFpuPrepareUsage(PVMCPUCC pVCpu)
1966	{
1967	#ifdef IN_RING3
1968	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
1969	#else
1970	CPUMRZFpuStatePrepareHostCpuForUse(pVCpu);
1971	#endif
1972	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
1973	}
1974
1975
1976	/**
1977	* Hook for preparing to use the host FPU for SSE.
1978	*
1979	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1980	*
1981	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1982	*/
1983	DECLINLINE(void) iemFpuPrepareUsageSse(PVMCPUCC pVCpu)
1984	{
1985	iemFpuPrepareUsage(pVCpu);
1986	}
1987
1988
1989	/**
1990	* Hook for preparing to use the host FPU for AVX.
1991	*
1992	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1993	*
1994	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1995	*/
1996	DECLINLINE(void) iemFpuPrepareUsageAvx(PVMCPUCC pVCpu)
1997	{
1998	iemFpuPrepareUsage(pVCpu);
1999	}
2000
2001
2002	/**
2003	* Hook for actualizing the guest FPU state before the interpreter reads it.
2004	*
2005	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2006	*
2007	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2008	*/
2009	DECLINLINE(void) iemFpuActualizeStateForRead(PVMCPUCC pVCpu)
2010	{
2011	#ifdef IN_RING3
2012	NOREF(pVCpu);
2013	#else
2014	CPUMRZFpuStateActualizeForRead(pVCpu);
2015	#endif
2016	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2017	}
2018
2019
2020	/**
2021	* Hook for actualizing the guest FPU state before the interpreter changes it.
2022	*
2023	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2024	*
2025	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2026	*/
2027	DECLINLINE(void) iemFpuActualizeStateForChange(PVMCPUCC pVCpu)
2028	{
2029	#ifdef IN_RING3
2030	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2031	#else
2032	CPUMRZFpuStateActualizeForChange(pVCpu);
2033	#endif
2034	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2035	}
2036
2037
2038	/**
2039	* Hook for actualizing the guest XMM0..15 and MXCSR register state for read
2040	* only.
2041	*
2042	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2043	*
2044	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2045	*/
2046	DECLINLINE(void) iemFpuActualizeSseStateForRead(PVMCPUCC pVCpu)
2047	{
2048	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
2049	NOREF(pVCpu);
2050	#else
2051	CPUMRZFpuStateActualizeSseForRead(pVCpu);
2052	#endif
2053	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2054	}
2055
2056
2057	/**
2058	* Hook for actualizing the guest XMM0..15 and MXCSR register state for
2059	* read+write.
2060	*
2061	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2062	*
2063	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2064	*/
2065	DECLINLINE(void) iemFpuActualizeSseStateForChange(PVMCPUCC pVCpu)
2066	{
2067	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
2068	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2069	#else
2070	CPUMRZFpuStateActualizeForChange(pVCpu);
2071	#endif
2072	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2073
2074	/* Make sure any changes are loaded the next time around. */
2075	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_SSE;
2076	}
2077
2078
2079	/**
2080	* Hook for actualizing the guest YMM0..15 and MXCSR register state for read
2081	* only.
2082	*
2083	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2084	*
2085	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2086	*/
2087	DECLINLINE(void) iemFpuActualizeAvxStateForRead(PVMCPUCC pVCpu)
2088	{
2089	#ifdef IN_RING3
2090	NOREF(pVCpu);
2091	#else
2092	CPUMRZFpuStateActualizeAvxForRead(pVCpu);
2093	#endif
2094	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2095	}
2096
2097
2098	/**
2099	* Hook for actualizing the guest YMM0..15 and MXCSR register state for
2100	* read+write.
2101	*
2102	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2103	*
2104	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2105	*/
2106	DECLINLINE(void) iemFpuActualizeAvxStateForChange(PVMCPUCC pVCpu)
2107	{
2108	#ifdef IN_RING3
2109	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2110	#else
2111	CPUMRZFpuStateActualizeForChange(pVCpu);
2112	#endif
2113	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2114
2115	/* Just assume we're going to make changes to the SSE and YMM_HI parts. */
2116	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_YMM \| XSAVE_C_SSE;
2117	}
2118
2119
2120	/**
2121	* Stores a QNaN value into a FPU register.
2122	*
2123	* @param pReg Pointer to the register.
2124	*/
2125	DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg)
2126	{
2127	pReg->au32[0] = UINT32_C(0x00000000);
2128	pReg->au32[1] = UINT32_C(0xc0000000);
2129	pReg->au16[4] = UINT16_C(0xffff);
2130	}
2131
2132
2133	/**
2134	* Updates the FOP, FPU.CS and FPUIP registers.
2135	*
2136	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2137	* @param pFpuCtx The FPU context.
2138	*/
2139	DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorker(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx)
2140	{
2141	Assert(pVCpu->iem.s.uFpuOpcode != UINT16_MAX);
2142	pFpuCtx->FOP = pVCpu->iem.s.uFpuOpcode;
2143	/** @todo x87.CS and FPUIP needs to be kept seperately. */
2144	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
2145	{
2146	/** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled
2147	* happens in real mode here based on the fnsave and fnstenv images. */
2148	pFpuCtx->CS = 0;
2149	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.eip \| ((uint32_t)pVCpu->cpum.GstCtx.cs.Sel << 4);
2150	}
2151	else if (!IEM_IS_LONG_MODE(pVCpu))
2152	{
2153	pFpuCtx->CS = pVCpu->cpum.GstCtx.cs.Sel;
2154	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2155	}
2156	else
2157	(uint64_t )&pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2158	}
2159
2160
2161
2162
2163
2164	/**
2165	* Marks the specified stack register as free (for FFREE).
2166	*
2167	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2168	* @param iStReg The register to free.
2169	*/
2170	DECLINLINE(void) iemFpuStackFree(PVMCPUCC pVCpu, uint8_t iStReg)
2171	{
2172	Assert(iStReg < 8);
2173	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2174	uint8_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2175	pFpuCtx->FTW &= ~RT_BIT(iReg);
2176	}
2177
2178
2179	/**
2180	* Increments FSW.TOP, i.e. pops an item off the stack without freeing it.
2181	*
2182	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2183	*/
2184	DECLINLINE(void) iemFpuStackIncTop(PVMCPUCC pVCpu)
2185	{
2186	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2187	uint16_t uFsw = pFpuCtx->FSW;
2188	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2189	uTop = (uTop + (1 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2190	uFsw &= ~X86_FSW_TOP_MASK;
2191	uFsw \|= uTop;
2192	pFpuCtx->FSW = uFsw;
2193	}
2194
2195
2196	/**
2197	* Decrements FSW.TOP, i.e. push an item off the stack without storing anything.
2198	*
2199	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2200	*/
2201	DECLINLINE(void) iemFpuStackDecTop(PVMCPUCC pVCpu)
2202	{
2203	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2204	uint16_t uFsw = pFpuCtx->FSW;
2205	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2206	uTop = (uTop + (7 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2207	uFsw &= ~X86_FSW_TOP_MASK;
2208	uFsw \|= uTop;
2209	pFpuCtx->FSW = uFsw;
2210	}
2211
2212
2213
2214
2215	DECLINLINE(int) iemFpuStRegNotEmpty(PVMCPUCC pVCpu, uint8_t iStReg)
2216	{
2217	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2218	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2219	if (pFpuCtx->FTW & RT_BIT(iReg))
2220	return VINF_SUCCESS;
2221	return VERR_NOT_FOUND;
2222	}
2223
2224
2225	DECLINLINE(int) iemFpuStRegNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef)
2226	{
2227	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2228	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2229	if (pFpuCtx->FTW & RT_BIT(iReg))
2230	{
2231	*ppRef = &pFpuCtx->aRegs[iStReg].r80;
2232	return VINF_SUCCESS;
2233	}
2234	return VERR_NOT_FOUND;
2235	}
2236
2237
2238	DECLINLINE(int) iemFpu2StRegsNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0,
2239	uint8_t iStReg1, PCRTFLOAT80U *ppRef1)
2240	{
2241	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2242	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2243	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2244	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2245	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
2246	{
2247	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2248	*ppRef1 = &pFpuCtx->aRegs[iStReg1].r80;
2249	return VINF_SUCCESS;
2250	}
2251	return VERR_NOT_FOUND;
2252	}
2253
2254
2255	DECLINLINE(int) iemFpu2StRegsNotEmptyRefFirst(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1)
2256	{
2257	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2258	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2259	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2260	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2261	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
2262	{
2263	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2264	return VINF_SUCCESS;
2265	}
2266	return VERR_NOT_FOUND;
2267	}
2268
2269
2270	/**
2271	* Rotates the stack registers when setting new TOS.
2272	*
2273	* @param pFpuCtx The FPU context.
2274	* @param iNewTop New TOS value.
2275	* @remarks We only do this to speed up fxsave/fxrstor which
2276	* arrange the FP registers in stack order.
2277	* MUST be done before writing the new TOS (FSW).
2278	*/
2279	DECLINLINE(void) iemFpuRotateStackSetTop(PX86FXSTATE pFpuCtx, uint16_t iNewTop)
2280	{
2281	uint16_t iOldTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2282	RTFLOAT80U ar80Temp[8];
2283
2284	if (iOldTop == iNewTop)
2285	return;
2286
2287	/* Unscrew the stack and get it into 'native' order. */
2288	ar80Temp[0] = pFpuCtx->aRegs[(8 - iOldTop + 0) & X86_FSW_TOP_SMASK].r80;
2289	ar80Temp[1] = pFpuCtx->aRegs[(8 - iOldTop + 1) & X86_FSW_TOP_SMASK].r80;
2290	ar80Temp[2] = pFpuCtx->aRegs[(8 - iOldTop + 2) & X86_FSW_TOP_SMASK].r80;
2291	ar80Temp[3] = pFpuCtx->aRegs[(8 - iOldTop + 3) & X86_FSW_TOP_SMASK].r80;
2292	ar80Temp[4] = pFpuCtx->aRegs[(8 - iOldTop + 4) & X86_FSW_TOP_SMASK].r80;
2293	ar80Temp[5] = pFpuCtx->aRegs[(8 - iOldTop + 5) & X86_FSW_TOP_SMASK].r80;
2294	ar80Temp[6] = pFpuCtx->aRegs[(8 - iOldTop + 6) & X86_FSW_TOP_SMASK].r80;
2295	ar80Temp[7] = pFpuCtx->aRegs[(8 - iOldTop + 7) & X86_FSW_TOP_SMASK].r80;
2296
2297	/* Now rotate the stack to the new position. */
2298	pFpuCtx->aRegs[0].r80 = ar80Temp[(iNewTop + 0) & X86_FSW_TOP_SMASK];
2299	pFpuCtx->aRegs[1].r80 = ar80Temp[(iNewTop + 1) & X86_FSW_TOP_SMASK];
2300	pFpuCtx->aRegs[2].r80 = ar80Temp[(iNewTop + 2) & X86_FSW_TOP_SMASK];
2301	pFpuCtx->aRegs[3].r80 = ar80Temp[(iNewTop + 3) & X86_FSW_TOP_SMASK];
2302	pFpuCtx->aRegs[4].r80 = ar80Temp[(iNewTop + 4) & X86_FSW_TOP_SMASK];
2303	pFpuCtx->aRegs[5].r80 = ar80Temp[(iNewTop + 5) & X86_FSW_TOP_SMASK];
2304	pFpuCtx->aRegs[6].r80 = ar80Temp[(iNewTop + 6) & X86_FSW_TOP_SMASK];
2305	pFpuCtx->aRegs[7].r80 = ar80Temp[(iNewTop + 7) & X86_FSW_TOP_SMASK];
2306	}
2307
2308
2309	/**
2310	* Updates the FPU exception status after FCW is changed.
2311	*
2312	* @param pFpuCtx The FPU context.
2313	*/
2314	DECLINLINE(void) iemFpuRecalcExceptionStatus(PX86FXSTATE pFpuCtx)
2315	{
2316	uint16_t u16Fsw = pFpuCtx->FSW;
2317	if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pFpuCtx->FCW & X86_FCW_XCPT_MASK))
2318	u16Fsw \|= X86_FSW_ES \| X86_FSW_B;
2319	else
2320	u16Fsw &= ~(X86_FSW_ES \| X86_FSW_B);
2321	pFpuCtx->FSW = u16Fsw;
2322	}
2323
2324
2325	/**
2326	* Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE.
2327	*
2328	* @returns The full FTW.
2329	* @param pFpuCtx The FPU context.
2330	*/
2331	DECLINLINE(uint16_t) iemFpuCalcFullFtw(PCX86FXSTATE pFpuCtx)
2332	{
2333	uint8_t const u8Ftw = (uint8_t)pFpuCtx->FTW;
2334	uint16_t u16Ftw = 0;
2335	unsigned const iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2336	for (unsigned iSt = 0; iSt < 8; iSt++)
2337	{
2338	unsigned const iReg = (iSt + iTop) & 7;
2339	if (!(u8Ftw & RT_BIT(iReg)))
2340	u16Ftw \|= 3 << (iReg * 2); /* empty */
2341	else
2342	{
2343	uint16_t uTag;
2344	PCRTFLOAT80U const pr80Reg = &pFpuCtx->aRegs[iSt].r80;
2345	if (pr80Reg->s.uExponent == 0x7fff)
2346	uTag = 2; /* Exponent is all 1's => Special. */
2347	else if (pr80Reg->s.uExponent == 0x0000)
2348	{
2349	if (pr80Reg->s.uMantissa == 0x0000)
2350	uTag = 1; /* All bits are zero => Zero. */
2351	else
2352	uTag = 2; /* Must be special. */
2353	}
2354	else if (pr80Reg->s.uMantissa & RT_BIT_64(63)) /* The J bit. */
2355	uTag = 0; /* Valid. */
2356	else
2357	uTag = 2; /* Must be special. */
2358
2359	u16Ftw \|= uTag << (iReg * 2);
2360	}
2361	}
2362
2363	return u16Ftw;
2364	}
2365
2366
2367	/**
2368	* Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR).
2369	*
2370	* @returns The compressed FTW.
2371	* @param u16FullFtw The full FTW to convert.
2372	*/
2373	DECLINLINE(uint16_t) iemFpuCompressFtw(uint16_t u16FullFtw)
2374	{
2375	uint8_t u8Ftw = 0;
2376	for (unsigned i = 0; i < 8; i++)
2377	{
2378	if ((u16FullFtw & 3) != 3 /empty/)
2379	u8Ftw \|= RT_BIT(i);
2380	u16FullFtw >>= 2;
2381	}
2382
2383	return u8Ftw;
2384	}
2385
2386	/** @} */
2387
2388
2389	/** @name Memory access.
2390	*
2391	* @{
2392	*/
2393
2394
2395	/**
2396	* Checks whether alignment checks are enabled or not.
2397	*
2398	* @returns true if enabled, false if not.
2399	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2400	*/
2401	DECLINLINE(bool) iemMemAreAlignmentChecksEnabled(PVMCPUCC pVCpu)
2402	{
2403	AssertCompile(X86_CR0_AM == X86_EFL_AC);
2404	return pVCpu->iem.s.uCpl == 3
2405	&& (((uint32_t)pVCpu->cpum.GstCtx.cr0 & pVCpu->cpum.GstCtx.eflags.u) & X86_CR0_AM);
2406	}
2407
2408	/**
2409	* Checks if the given segment can be written to, raise the appropriate
2410	* exception if not.
2411	*
2412	* @returns VBox strict status code.
2413	*
2414	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2415	* @param pHid Pointer to the hidden register.
2416	* @param iSegReg The register number.
2417	* @param pu64BaseAddr Where to return the base address to use for the
2418	* segment. (In 64-bit code it may differ from the
2419	* base in the hidden segment.)
2420	*/
2421	DECLINLINE(VBOXSTRICTRC) iemMemSegCheckWriteAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid, uint8_t iSegReg, uint64_t *pu64BaseAddr)
2422	{
2423	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2424
2425	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2426	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
2427	else
2428	{
2429	if (!pHid->Attr.n.u1Present)
2430	{
2431	uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
2432	AssertRelease(uSel == 0);
2433	Log(("iemMemSegCheckWriteAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
2434	return iemRaiseGeneralProtectionFault0(pVCpu);
2435	}
2436
2437	if ( ( (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE)
2438	\|\| !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
2439	&& pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT )
2440	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
2441	*pu64BaseAddr = pHid->u64Base;
2442	}
2443	return VINF_SUCCESS;
2444	}
2445
2446
2447	/**
2448	* Checks if the given segment can be read from, raise the appropriate
2449	* exception if not.
2450	*
2451	* @returns VBox strict status code.
2452	*
2453	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2454	* @param pHid Pointer to the hidden register.
2455	* @param iSegReg The register number.
2456	* @param pu64BaseAddr Where to return the base address to use for the
2457	* segment. (In 64-bit code it may differ from the
2458	* base in the hidden segment.)
2459	*/
2460	DECLINLINE(VBOXSTRICTRC) iemMemSegCheckReadAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid, uint8_t iSegReg, uint64_t *pu64BaseAddr)
2461	{
2462	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2463
2464	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2465	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
2466	else
2467	{
2468	if (!pHid->Attr.n.u1Present)
2469	{
2470	uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
2471	AssertRelease(uSel == 0);
2472	Log(("iemMemSegCheckReadAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
2473	return iemRaiseGeneralProtectionFault0(pVCpu);
2474	}
2475
2476	if ((pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
2477	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
2478	*pu64BaseAddr = pHid->u64Base;
2479	}
2480	return VINF_SUCCESS;
2481	}
2482
2483
2484	/**
2485	* Maps a physical page.
2486	*
2487	* @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr).
2488	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2489	* @param GCPhysMem The physical address.
2490	* @param fAccess The intended access.
2491	* @param ppvMem Where to return the mapping address.
2492	* @param pLock The PGM lock.
2493	*/
2494	DECLINLINE(int) iemMemPageMap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, void **ppvMem, PPGMPAGEMAPLOCK pLock)
2495	{
2496	#ifdef IEM_LOG_MEMORY_WRITES
2497	if (fAccess & IEM_ACCESS_TYPE_WRITE)
2498	return VERR_PGM_PHYS_TLB_CATCH_ALL;
2499	#endif
2500
2501	/** @todo This API may require some improving later. A private deal with PGM
2502	* regarding locking and unlocking needs to be struct. A couple of TLBs
2503	* living in PGM, but with publicly accessible inlined access methods
2504	* could perhaps be an even better solution. */
2505	int rc = PGMPhysIemGCPhys2Ptr(pVCpu->CTX_SUFF(pVM), pVCpu,
2506	GCPhysMem,
2507	RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE),
2508	pVCpu->iem.s.fBypassHandlers,
2509	ppvMem,
2510	pLock);
2511	/Log(("PGMPhysIemGCPhys2Ptr %Rrc pLock=%.Rhxs\n", rc, sizeof(pLock), pLock));/
2512	AssertMsg(rc == VINF_SUCCESS \|\| RT_FAILURE_NP(rc), ("%Rrc\n", rc));
2513
2514	return rc;
2515	}
2516
2517
2518	/**
2519	* Unmap a page previously mapped by iemMemPageMap.
2520	*
2521	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2522	* @param GCPhysMem The physical address.
2523	* @param fAccess The intended access.
2524	* @param pvMem What iemMemPageMap returned.
2525	* @param pLock The PGM lock.
2526	*/
2527	DECLINLINE(void) iemMemPageUnmap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, const void *pvMem, PPGMPAGEMAPLOCK pLock)
2528	{
2529	NOREF(pVCpu);
2530	NOREF(GCPhysMem);
2531	NOREF(fAccess);
2532	NOREF(pvMem);
2533	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), pLock);
2534	}
2535
2536	#ifdef IEM_WITH_SETJMP
2537
2538	/** @todo slim this down */
2539	DECLINLINE(RTGCPTR) iemMemApplySegmentToReadJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem, RTGCPTR GCPtrMem)
2540	{
2541	Assert(cbMem >= 1);
2542	Assert(iSegReg < X86_SREG_COUNT);
2543
2544	/*
2545	* 64-bit mode is simpler.
2546	*/
2547	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2548	{
2549	if (iSegReg >= X86_SREG_FS && iSegReg != UINT8_MAX)
2550	{
2551	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2552	PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
2553	GCPtrMem += pSel->u64Base;
2554	}
2555
2556	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
2557	return GCPtrMem;
2558	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
2559	}
2560	/*
2561	* 16-bit and 32-bit segmentation.
2562	*/
2563	else if (iSegReg != UINT8_MAX)
2564	{
2565	/** @todo Does this apply to segments with 4G-1 limit? */
2566	uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
2567	if (RT_LIKELY(GCPtrLast32 >= (uint32_t)GCPtrMem))
2568	{
2569	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2570	PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
2571	switch (pSel->Attr.u & ( X86DESCATTR_P \| X86DESCATTR_UNUSABLE
2572	\| X86_SEL_TYPE_READ \| X86_SEL_TYPE_WRITE /* same as read */
2573	\| X86_SEL_TYPE_DOWN \| X86_SEL_TYPE_CONF /* same as down */
2574	\| X86_SEL_TYPE_CODE))
2575	{
2576	case X86DESCATTR_P: /* readonly data, expand up */
2577	case X86DESCATTR_P \| X86_SEL_TYPE_WRITE: /* writable data, expand up */
2578	case X86DESCATTR_P \| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ: /* code, read-only */
2579	case X86DESCATTR_P \| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ \| X86_SEL_TYPE_CONF: /* conforming code, read-only */
2580	/* expand up */
2581	if (RT_LIKELY(GCPtrLast32 <= pSel->u32Limit))
2582	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2583	Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x vs %#x\n",
2584	(uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit));
2585	break;
2586
2587	case X86DESCATTR_P \| X86_SEL_TYPE_DOWN: /* readonly data, expand down */
2588	case X86DESCATTR_P \| X86_SEL_TYPE_DOWN \| X86_SEL_TYPE_WRITE: /* writable data, expand down */
2589	/* expand down */
2590	if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
2591	&& ( pSel->Attr.n.u1DefBig
2592	\|\| GCPtrLast32 <= UINT32_C(0xffff)) ))
2593	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2594	Log10(("iemMemApplySegmentToReadJmp: expand down out of bounds %#x..%#x vs %#x..%#x\n",
2595	(uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit, pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT16_MAX));
2596	break;
2597
2598	default:
2599	Log10(("iemMemApplySegmentToReadJmp: bad selector %#x\n", pSel->Attr.u));
2600	iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
2601	break;
2602	}
2603	}
2604	Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x\n",(uint32_t)GCPtrMem, GCPtrLast32));
2605	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
2606	}
2607	/*
2608	* 32-bit flat address.
2609	*/
2610	else
2611	return GCPtrMem;
2612	}
2613
2614
2615	/** @todo slim this down */
2616	DECLINLINE(RTGCPTR) iemMemApplySegmentToWriteJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem, RTGCPTR GCPtrMem)
2617	{
2618	Assert(cbMem >= 1);
2619	Assert(iSegReg < X86_SREG_COUNT);
2620
2621	/*
2622	* 64-bit mode is simpler.
2623	*/
2624	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2625	{
2626	if (iSegReg >= X86_SREG_FS)
2627	{
2628	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2629	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
2630	GCPtrMem += pSel->u64Base;
2631	}
2632
2633	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
2634	return GCPtrMem;
2635	}
2636	/*
2637	* 16-bit and 32-bit segmentation.
2638	*/
2639	else
2640	{
2641	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2642	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
2643	uint32_t const fRelevantAttrs = pSel->Attr.u & ( X86DESCATTR_P \| X86DESCATTR_UNUSABLE
2644	\| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_WRITE \| X86_SEL_TYPE_DOWN);
2645	if (fRelevantAttrs == (X86DESCATTR_P \| X86_SEL_TYPE_WRITE)) /* data, expand up */
2646	{
2647	/* expand up */
2648	uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
2649	if (RT_LIKELY( GCPtrLast32 > pSel->u32Limit
2650	&& GCPtrLast32 > (uint32_t)GCPtrMem))
2651	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2652	}
2653	else if (fRelevantAttrs == (X86DESCATTR_P \| X86_SEL_TYPE_WRITE \| X86_SEL_TYPE_DOWN)) /* data, expand up */
2654	{
2655	/* expand down */
2656	uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
2657	if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
2658	&& GCPtrLast32 <= (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
2659	&& GCPtrLast32 > (uint32_t)GCPtrMem))
2660	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2661	}
2662	else
2663	iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
2664	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
2665	}
2666	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
2667	}
2668
2669	#endif /* IEM_WITH_SETJMP */
2670
2671	/**
2672	* Fakes a long mode stack selector for SS = 0.
2673	*
2674	* @param pDescSs Where to return the fake stack descriptor.
2675	* @param uDpl The DPL we want.
2676	*/
2677	DECLINLINE(void) iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl)
2678	{
2679	pDescSs->Long.au64[0] = 0;
2680	pDescSs->Long.au64[1] = 0;
2681	pDescSs->Long.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
2682	pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */
2683	pDescSs->Long.Gen.u2Dpl = uDpl;
2684	pDescSs->Long.Gen.u1Present = 1;
2685	pDescSs->Long.Gen.u1Long = 1;
2686	}
2687
2688	/** @} */
2689
2690
2691	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2692
2693	/**
2694	* Gets CR0 fixed-0 bits in VMX non-root mode.
2695	*
2696	* We do this rather than fetching what we report to the guest (in
2697	* IA32_VMX_CR0_FIXED0 MSR) because real hardware (and so do we) report the same
2698	* values regardless of whether unrestricted-guest feature is available on the CPU.
2699	*
2700	* @returns CR0 fixed-0 bits.
2701	* @param pVCpu The cross context virtual CPU structure.
2702	*/
2703	DECLINLINE(uint64_t) iemVmxGetCr0Fixed0(PCVMCPUCC pVCpu)
2704	{
2705	Assert(IEM_VMX_IS_ROOT_MODE(pVCpu));
2706	Assert(IEM_VMX_HAS_CURRENT_VMCS(pVCpu));
2707
2708	static uint64_t const s_auCr0Fixed0[2] = { VMX_V_CR0_FIXED0, VMX_V_CR0_FIXED0_UX };
2709	PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
2710	uint8_t const fUnrestrictedGuest = !!(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST);
2711	uint64_t const uCr0Fixed0 = s_auCr0Fixed0[fUnrestrictedGuest];
2712	Assert(!(uCr0Fixed0 & (X86_CR0_NW \| X86_CR0_CD)));
2713	return uCr0Fixed0;
2714	}
2715
2716
2717	/**
2718	* Sets virtual-APIC write emulation as pending.
2719	*
2720	* @param pVCpu The cross context virtual CPU structure.
2721	* @param offApic The offset in the virtual-APIC page that was written.
2722	*/
2723	DECLINLINE(void) iemVmxVirtApicSetPendingWrite(PVMCPUCC pVCpu, uint16_t offApic)
2724	{
2725	Assert(offApic < XAPIC_OFF_END + 4);
2726
2727	/*
2728	* Record the currently updated APIC offset, as we need this later for figuring
2729	* out whether to perform TPR, EOI or self-IPI virtualization as well as well
2730	* as for supplying the exit qualification when causing an APIC-write VM-exit.
2731	*/
2732	pVCpu->cpum.GstCtx.hwvirt.vmx.offVirtApicWrite = offApic;
2733
2734	/*
2735	* Flag that we need to perform virtual-APIC write emulation (TPR/PPR/EOI/Self-IPI
2736	* virtualization or APIC-write emulation).
2737	*/
2738	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
2739	VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE);
2740	}
2741
2742	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
2743
2744	#endif /* !VMM_INCLUDED_SRC_include_IEMInline_h */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMInline.h@ 97441

Download in other formats: