IEMInline.h@ 95512

Last change on this file since 95512 was 95410, checked in by vboxsync, 2 years ago
VMM/IEM: Alignment checks (#AC(0)/#GP(0)). bugref:9898
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 84.6 KB

Line
1	/* $Id: IEMInline.h 95410 2022-06-28 18:33:26Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Inlined Functions.
4	*/
5
6	/*
7	* Copyright (C) 2011-2022 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	#ifndef VMM_INCLUDED_SRC_include_IEMInline_h
19	#define VMM_INCLUDED_SRC_include_IEMInline_h
20	#ifndef RT_WITHOUT_PRAGMA_ONCE
21	# pragma once
22	#endif
23
24
25
26	/**
27	* Makes status code addjustments (pass up from I/O and access handler)
28	* as well as maintaining statistics.
29	*
30	* @returns Strict VBox status code to pass up.
31	* @param pVCpu The cross context virtual CPU structure of the calling thread.
32	* @param rcStrict The status from executing an instruction.
33	*/
34	DECL_FORCE_INLINE(VBOXSTRICTRC) iemExecStatusCodeFiddling(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
35	{
36	if (rcStrict != VINF_SUCCESS)
37	{
38	if (RT_SUCCESS(rcStrict))
39	{
40	AssertMsg( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
41	\|\| rcStrict == VINF_IOM_R3_IOPORT_READ
42	\|\| rcStrict == VINF_IOM_R3_IOPORT_WRITE
43	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
44	\|\| rcStrict == VINF_IOM_R3_MMIO_READ
45	\|\| rcStrict == VINF_IOM_R3_MMIO_READ_WRITE
46	\|\| rcStrict == VINF_IOM_R3_MMIO_WRITE
47	\|\| rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE
48	\|\| rcStrict == VINF_CPUM_R3_MSR_READ
49	\|\| rcStrict == VINF_CPUM_R3_MSR_WRITE
50	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
51	\|\| rcStrict == VINF_EM_RAW_TO_R3
52	\|\| rcStrict == VINF_EM_TRIPLE_FAULT
53	\|\| rcStrict == VINF_GIM_R3_HYPERCALL
54	/* raw-mode / virt handlers only: */
55	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT
56	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT
57	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT
58	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT
59	\|\| rcStrict == VINF_SELM_SYNC_GDT
60	\|\| rcStrict == VINF_CSAM_PENDING_ACTION
61	\|\| rcStrict == VINF_PATM_CHECK_PATCH_PAGE
62	/* nested hw.virt codes: */
63	\|\| rcStrict == VINF_VMX_VMEXIT
64	\|\| rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE
65	\|\| rcStrict == VINF_VMX_MODIFIES_BEHAVIOR
66	\|\| rcStrict == VINF_SVM_VMEXIT
67	, ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
68	/** @todo adjust for VINF_EM_RAW_EMULATE_INSTR. */
69	int32_t const rcPassUp = pVCpu->iem.s.rcPassUp;
70	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
71	if ( rcStrict == VINF_VMX_VMEXIT
72	&& rcPassUp == VINF_SUCCESS)
73	rcStrict = VINF_SUCCESS;
74	else
75	#endif
76	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
77	if ( rcStrict == VINF_SVM_VMEXIT
78	&& rcPassUp == VINF_SUCCESS)
79	rcStrict = VINF_SUCCESS;
80	else
81	#endif
82	if (rcPassUp == VINF_SUCCESS)
83	pVCpu->iem.s.cRetInfStatuses++;
84	else if ( rcPassUp < VINF_EM_FIRST
85	\|\| rcPassUp > VINF_EM_LAST
86	\|\| rcPassUp < VBOXSTRICTRC_VAL(rcStrict))
87	{
88	Log(("IEM: rcPassUp=%Rrc! rcStrict=%Rrc\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
89	pVCpu->iem.s.cRetPassUpStatus++;
90	rcStrict = rcPassUp;
91	}
92	else
93	{
94	Log(("IEM: rcPassUp=%Rrc rcStrict=%Rrc!\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
95	pVCpu->iem.s.cRetInfStatuses++;
96	}
97	}
98	else if (rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED)
99	pVCpu->iem.s.cRetAspectNotImplemented++;
100	else if (rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
101	pVCpu->iem.s.cRetInstrNotImplemented++;
102	else
103	pVCpu->iem.s.cRetErrStatuses++;
104	}
105	else if (pVCpu->iem.s.rcPassUp != VINF_SUCCESS)
106	{
107	pVCpu->iem.s.cRetPassUpStatus++;
108	rcStrict = pVCpu->iem.s.rcPassUp;
109	}
110
111	return rcStrict;
112	}
113
114
115	/**
116	* Sets the pass up status.
117	*
118	* @returns VINF_SUCCESS.
119	* @param pVCpu The cross context virtual CPU structure of the
120	* calling thread.
121	* @param rcPassUp The pass up status. Must be informational.
122	* VINF_SUCCESS is not allowed.
123	*/
124	DECLINLINE(int) iemSetPassUpStatus(PVMCPUCC pVCpu, VBOXSTRICTRC rcPassUp)
125	{
126	AssertRC(VBOXSTRICTRC_VAL(rcPassUp)); Assert(rcPassUp != VINF_SUCCESS);
127
128	int32_t const rcOldPassUp = pVCpu->iem.s.rcPassUp;
129	if (rcOldPassUp == VINF_SUCCESS)
130	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
131	/* If both are EM scheduling codes, use EM priority rules. */
132	else if ( rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST
133	&& rcPassUp >= VINF_EM_FIRST && rcPassUp <= VINF_EM_LAST)
134	{
135	if (rcPassUp < rcOldPassUp)
136	{
137	Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
138	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
139	}
140	else
141	Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
142	}
143	/* Override EM scheduling with specific status code. */
144	else if (rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST)
145	{
146	Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
147	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
148	}
149	/* Don't override specific status code, first come first served. */
150	else
151	Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
152	return VINF_SUCCESS;
153	}
154
155
156	/**
157	* Calculates the CPU mode.
158	*
159	* This is mainly for updating IEMCPU::enmCpuMode.
160	*
161	* @returns CPU mode.
162	* @param pVCpu The cross context virtual CPU structure of the
163	* calling thread.
164	*/
165	DECLINLINE(IEMMODE) iemCalcCpuMode(PVMCPUCC pVCpu)
166	{
167	if (CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx))
168	return IEMMODE_64BIT;
169	if (pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig) /** @todo check if this is correct... */
170	return IEMMODE_32BIT;
171	return IEMMODE_16BIT;
172	}
173
174
175	/**
176	* Initializes the execution state.
177	*
178	* @param pVCpu The cross context virtual CPU structure of the
179	* calling thread.
180	* @param fBypassHandlers Whether to bypass access handlers.
181	*
182	* @remarks Callers of this must call iemUninitExec() to undo potentially fatal
183	* side-effects in strict builds.
184	*/
185	DECLINLINE(void) iemInitExec(PVMCPUCC pVCpu, bool fBypassHandlers)
186	{
187	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
188	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
189	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
190	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
191	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
192	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
193	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
194	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
195	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
196	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
197
198	pVCpu->iem.s.uCpl = CPUMGetGuestCPL(pVCpu);
199	pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
200	#ifdef VBOX_STRICT
201	pVCpu->iem.s.enmDefAddrMode = (IEMMODE)0xfe;
202	pVCpu->iem.s.enmEffAddrMode = (IEMMODE)0xfe;
203	pVCpu->iem.s.enmDefOpSize = (IEMMODE)0xfe;
204	pVCpu->iem.s.enmEffOpSize = (IEMMODE)0xfe;
205	pVCpu->iem.s.fPrefixes = 0xfeedbeef;
206	pVCpu->iem.s.uRexReg = 127;
207	pVCpu->iem.s.uRexB = 127;
208	pVCpu->iem.s.offModRm = 127;
209	pVCpu->iem.s.uRexIndex = 127;
210	pVCpu->iem.s.iEffSeg = 127;
211	pVCpu->iem.s.idxPrefix = 127;
212	pVCpu->iem.s.uVex3rdReg = 127;
213	pVCpu->iem.s.uVexLength = 127;
214	pVCpu->iem.s.fEvexStuff = 127;
215	pVCpu->iem.s.uFpuOpcode = UINT16_MAX;
216	# ifdef IEM_WITH_CODE_TLB
217	pVCpu->iem.s.offInstrNextByte = UINT16_MAX;
218	pVCpu->iem.s.pbInstrBuf = NULL;
219	pVCpu->iem.s.cbInstrBuf = UINT16_MAX;
220	pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;
221	pVCpu->iem.s.offCurInstrStart = INT16_MAX;
222	pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);
223	# else
224	pVCpu->iem.s.offOpcode = 127;
225	pVCpu->iem.s.cbOpcode = 127;
226	# endif
227	#endif
228
229	pVCpu->iem.s.cActiveMappings = 0;
230	pVCpu->iem.s.iNextMapping = 0;
231	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
232	pVCpu->iem.s.fBypassHandlers = fBypassHandlers;
233	#if 0
234	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
235	if ( CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
236	&& CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS))
237	{
238	PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
239	Assert(pVmcs);
240	RTGCPHYS const GCPhysApicAccess = pVmcs->u64AddrApicAccess.u;
241	if (!PGMHandlerPhysicalIsRegistered(pVCpu->CTX_SUFF(pVM), GCPhysApicAccess))
242	{
243	int rc = PGMHandlerPhysicalRegister(pVCpu->CTX_SUFF(pVM), GCPhysApicAccess, GCPhysApicAccess + X86_PAGE_4K_SIZE - 1,
244	pVCpu->iem.s.hVmxApicAccessPage, NIL_RTR3PTR /* pvUserR3 */,
245	NIL_RTR0PTR /* pvUserR0 /, NIL_RTRCPTR / pvUserRC /, NULL / pszDesc */);
246	AssertRC(rc);
247	}
248	}
249	#endif
250	#endif
251	}
252
253	#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
254	/**
255	* Performs a minimal reinitialization of the execution state.
256	*
257	* This is intended to be used by VM-exits, SMM, LOADALL and other similar
258	* 'world-switch' types operations on the CPU. Currently only nested
259	* hardware-virtualization uses it.
260	*
261	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
262	*/
263	DECLINLINE(void) iemReInitExec(PVMCPUCC pVCpu)
264	{
265	IEMMODE const enmMode = iemCalcCpuMode(pVCpu);
266	uint8_t const uCpl = CPUMGetGuestCPL(pVCpu);
267
268	pVCpu->iem.s.uCpl = uCpl;
269	pVCpu->iem.s.enmCpuMode = enmMode;
270	pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
271	pVCpu->iem.s.enmEffAddrMode = enmMode;
272	if (enmMode != IEMMODE_64BIT)
273	{
274	pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
275	pVCpu->iem.s.enmEffOpSize = enmMode;
276	}
277	else
278	{
279	pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
280	pVCpu->iem.s.enmEffOpSize = enmMode;
281	}
282	pVCpu->iem.s.iEffSeg = X86_SREG_DS;
283	#ifndef IEM_WITH_CODE_TLB
284	/** @todo Shouldn't we be doing this in IEMTlbInvalidateAll()? */
285	pVCpu->iem.s.offOpcode = 0;
286	pVCpu->iem.s.cbOpcode = 0;
287	#endif
288	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
289	}
290	#endif
291
292	/**
293	* Counterpart to #iemInitExec that undoes evil strict-build stuff.
294	*
295	* @param pVCpu The cross context virtual CPU structure of the
296	* calling thread.
297	*/
298	DECLINLINE(void) iemUninitExec(PVMCPUCC pVCpu)
299	{
300	/* Note! do not touch fInPatchCode here! (see iemUninitExecAndFiddleStatusAndMaybeReenter) */
301	#ifdef VBOX_STRICT
302	# ifdef IEM_WITH_CODE_TLB
303	NOREF(pVCpu);
304	# else
305	pVCpu->iem.s.cbOpcode = 0;
306	# endif
307	#else
308	NOREF(pVCpu);
309	#endif
310	}
311
312
313	/**
314	* Calls iemUninitExec, iemExecStatusCodeFiddling and iemRCRawMaybeReenter.
315	*
316	* Only calling iemRCRawMaybeReenter in raw-mode, obviously.
317	*
318	* @returns Fiddled strict vbox status code, ready to return to non-IEM caller.
319	* @param pVCpu The cross context virtual CPU structure of the calling thread.
320	* @param rcStrict The status code to fiddle.
321	*/
322	DECLINLINE(VBOXSTRICTRC) iemUninitExecAndFiddleStatusAndMaybeReenter(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
323	{
324	iemUninitExec(pVCpu);
325	return iemExecStatusCodeFiddling(pVCpu, rcStrict);
326	}
327
328
329	/**
330	* Macro used by the IEMExec* method to check the given instruction length.
331	*
332	* Will return on failure!
333	*
334	* @param a_cbInstr The given instruction length.
335	* @param a_cbMin The minimum length.
336	*/
337	#define IEMEXEC_ASSERT_INSTR_LEN_RETURN(a_cbInstr, a_cbMin) \
338	AssertMsgReturn((unsigned)(a_cbInstr) - (unsigned)(a_cbMin) <= (unsigned)15 - (unsigned)(a_cbMin), \
339	("cbInstr=%u cbMin=%u\n", (a_cbInstr), (a_cbMin)), VERR_IEM_INVALID_INSTR_LENGTH)
340
341
342
343	#ifndef IEM_WITH_SETJMP
344
345	/**
346	* Fetches the next opcode byte.
347	*
348	* @returns Strict VBox status code.
349	* @param pVCpu The cross context virtual CPU structure of the
350	* calling thread.
351	* @param pu8 Where to return the opcode byte.
352	*/
353	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PVMCPUCC pVCpu, uint8_t *pu8)
354	{
355	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
356	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
357	{
358	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
359	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
360	return VINF_SUCCESS;
361	}
362	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
363	}
364
365	#else /* IEM_WITH_SETJMP */
366
367	/**
368	* Fetches the next opcode byte, longjmp on error.
369	*
370	* @returns The opcode byte.
371	* @param pVCpu The cross context virtual CPU structure of the calling thread.
372	*/
373	DECLINLINE(uint8_t) iemOpcodeGetNextU8Jmp(PVMCPUCC pVCpu)
374	{
375	# ifdef IEM_WITH_CODE_TLB
376	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
377	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
378	if (RT_LIKELY( pbBuf != NULL
379	&& offBuf < pVCpu->iem.s.cbInstrBuf))
380	{
381	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
382	return pbBuf[offBuf];
383	}
384	# else
385	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
386	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
387	{
388	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
389	return pVCpu->iem.s.abOpcode[offOpcode];
390	}
391	# endif
392	return iemOpcodeGetNextU8SlowJmp(pVCpu);
393	}
394
395	#endif /* IEM_WITH_SETJMP */
396
397	/**
398	* Fetches the next opcode byte, returns automatically on failure.
399	*
400	* @param a_pu8 Where to return the opcode byte.
401	* @remark Implicitly references pVCpu.
402	*/
403	#ifndef IEM_WITH_SETJMP
404	# define IEM_OPCODE_GET_NEXT_U8(a_pu8) \
405	do \
406	{ \
407	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pVCpu, (a_pu8)); \
408	if (rcStrict2 == VINF_SUCCESS) \
409	{ /* likely */ } \
410	else \
411	return rcStrict2; \
412	} while (0)
413	#else
414	# define IEM_OPCODE_GET_NEXT_U8(a_pu8) (*(a_pu8) = iemOpcodeGetNextU8Jmp(pVCpu))
415	#endif /* IEM_WITH_SETJMP */
416
417
418	#ifndef IEM_WITH_SETJMP
419	/**
420	* Fetches the next signed byte from the opcode stream.
421	*
422	* @returns Strict VBox status code.
423	* @param pVCpu The cross context virtual CPU structure of the calling thread.
424	* @param pi8 Where to return the signed byte.
425	*/
426	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PVMCPUCC pVCpu, int8_t *pi8)
427	{
428	return iemOpcodeGetNextU8(pVCpu, (uint8_t *)pi8);
429	}
430	#endif /* !IEM_WITH_SETJMP */
431
432
433	/**
434	* Fetches the next signed byte from the opcode stream, returning automatically
435	* on failure.
436	*
437	* @param a_pi8 Where to return the signed byte.
438	* @remark Implicitly references pVCpu.
439	*/
440	#ifndef IEM_WITH_SETJMP
441	# define IEM_OPCODE_GET_NEXT_S8(a_pi8) \
442	do \
443	{ \
444	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pVCpu, (a_pi8)); \
445	if (rcStrict2 != VINF_SUCCESS) \
446	return rcStrict2; \
447	} while (0)
448	#else /* IEM_WITH_SETJMP */
449	# define IEM_OPCODE_GET_NEXT_S8(a_pi8) (*(a_pi8) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
450
451	#endif /* IEM_WITH_SETJMP */
452
453
454	#ifndef IEM_WITH_SETJMP
455	/**
456	* Fetches the next signed byte from the opcode stream, extending it to
457	* unsigned 16-bit.
458	*
459	* @returns Strict VBox status code.
460	* @param pVCpu The cross context virtual CPU structure of the calling thread.
461	* @param pu16 Where to return the unsigned word.
462	*/
463	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PVMCPUCC pVCpu, uint16_t *pu16)
464	{
465	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
466	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
467	return iemOpcodeGetNextS8SxU16Slow(pVCpu, pu16);
468
469	*pu16 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
470	pVCpu->iem.s.offOpcode = offOpcode + 1;
471	return VINF_SUCCESS;
472	}
473	#endif /* !IEM_WITH_SETJMP */
474
475	/**
476	* Fetches the next signed byte from the opcode stream and sign-extending it to
477	* a word, returning automatically on failure.
478	*
479	* @param a_pu16 Where to return the word.
480	* @remark Implicitly references pVCpu.
481	*/
482	#ifndef IEM_WITH_SETJMP
483	# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \
484	do \
485	{ \
486	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pVCpu, (a_pu16)); \
487	if (rcStrict2 != VINF_SUCCESS) \
488	return rcStrict2; \
489	} while (0)
490	#else
491	# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) (*(a_pu16) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
492	#endif
493
494	#ifndef IEM_WITH_SETJMP
495	/**
496	* Fetches the next signed byte from the opcode stream, extending it to
497	* unsigned 32-bit.
498	*
499	* @returns Strict VBox status code.
500	* @param pVCpu The cross context virtual CPU structure of the calling thread.
501	* @param pu32 Where to return the unsigned dword.
502	*/
503	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU32(PVMCPUCC pVCpu, uint32_t *pu32)
504	{
505	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
506	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
507	return iemOpcodeGetNextS8SxU32Slow(pVCpu, pu32);
508
509	*pu32 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
510	pVCpu->iem.s.offOpcode = offOpcode + 1;
511	return VINF_SUCCESS;
512	}
513	#endif /* !IEM_WITH_SETJMP */
514
515	/**
516	* Fetches the next signed byte from the opcode stream and sign-extending it to
517	* a word, returning automatically on failure.
518	*
519	* @param a_pu32 Where to return the word.
520	* @remark Implicitly references pVCpu.
521	*/
522	#ifndef IEM_WITH_SETJMP
523	#define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) \
524	do \
525	{ \
526	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU32(pVCpu, (a_pu32)); \
527	if (rcStrict2 != VINF_SUCCESS) \
528	return rcStrict2; \
529	} while (0)
530	#else
531	# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) (*(a_pu32) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
532	#endif
533
534
535	#ifndef IEM_WITH_SETJMP
536	/**
537	* Fetches the next signed byte from the opcode stream, extending it to
538	* unsigned 64-bit.
539	*
540	* @returns Strict VBox status code.
541	* @param pVCpu The cross context virtual CPU structure of the calling thread.
542	* @param pu64 Where to return the unsigned qword.
543	*/
544	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU64(PVMCPUCC pVCpu, uint64_t *pu64)
545	{
546	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
547	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
548	return iemOpcodeGetNextS8SxU64Slow(pVCpu, pu64);
549
550	*pu64 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
551	pVCpu->iem.s.offOpcode = offOpcode + 1;
552	return VINF_SUCCESS;
553	}
554	#endif /* !IEM_WITH_SETJMP */
555
556	/**
557	* Fetches the next signed byte from the opcode stream and sign-extending it to
558	* a word, returning automatically on failure.
559	*
560	* @param a_pu64 Where to return the word.
561	* @remark Implicitly references pVCpu.
562	*/
563	#ifndef IEM_WITH_SETJMP
564	# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) \
565	do \
566	{ \
567	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU64(pVCpu, (a_pu64)); \
568	if (rcStrict2 != VINF_SUCCESS) \
569	return rcStrict2; \
570	} while (0)
571	#else
572	# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) (*(a_pu64) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
573	#endif
574
575
576	#ifndef IEM_WITH_SETJMP
577	/**
578	* Fetches the next opcode byte.
579	*
580	* @returns Strict VBox status code.
581	* @param pVCpu The cross context virtual CPU structure of the
582	* calling thread.
583	* @param pu8 Where to return the opcode byte.
584	*/
585	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextRm(PVMCPUCC pVCpu, uint8_t *pu8)
586	{
587	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
588	pVCpu->iem.s.offModRm = offOpcode;
589	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
590	{
591	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
592	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
593	return VINF_SUCCESS;
594	}
595	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
596	}
597	#else /* IEM_WITH_SETJMP */
598	/**
599	* Fetches the next opcode byte, longjmp on error.
600	*
601	* @returns The opcode byte.
602	* @param pVCpu The cross context virtual CPU structure of the calling thread.
603	*/
604	DECLINLINE(uint8_t) iemOpcodeGetNextRmJmp(PVMCPUCC pVCpu)
605	{
606	# ifdef IEM_WITH_CODE_TLB
607	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
608	pVCpu->iem.s.offModRm = offBuf;
609	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
610	if (RT_LIKELY( pbBuf != NULL
611	&& offBuf < pVCpu->iem.s.cbInstrBuf))
612	{
613	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
614	return pbBuf[offBuf];
615	}
616	# else
617	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
618	pVCpu->iem.s.offModRm = offOpcode;
619	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
620	{
621	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
622	return pVCpu->iem.s.abOpcode[offOpcode];
623	}
624	# endif
625	return iemOpcodeGetNextU8SlowJmp(pVCpu);
626	}
627	#endif /* IEM_WITH_SETJMP */
628
629	/**
630	* Fetches the next opcode byte, which is a ModR/M byte, returns automatically
631	* on failure.
632	*
633	* Will note down the position of the ModR/M byte for VT-x exits.
634	*
635	* @param a_pbRm Where to return the RM opcode byte.
636	* @remark Implicitly references pVCpu.
637	*/
638	#ifndef IEM_WITH_SETJMP
639	# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) \
640	do \
641	{ \
642	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextRm(pVCpu, (a_pbRm)); \
643	if (rcStrict2 == VINF_SUCCESS) \
644	{ /* likely */ } \
645	else \
646	return rcStrict2; \
647	} while (0)
648	#else
649	# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) (*(a_pbRm) = iemOpcodeGetNextRmJmp(pVCpu))
650	#endif /* IEM_WITH_SETJMP */
651
652
653	#ifndef IEM_WITH_SETJMP
654
655	/**
656	* Fetches the next opcode word.
657	*
658	* @returns Strict VBox status code.
659	* @param pVCpu The cross context virtual CPU structure of the calling thread.
660	* @param pu16 Where to return the opcode word.
661	*/
662	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PVMCPUCC pVCpu, uint16_t *pu16)
663	{
664	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
665	if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
666	{
667	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
668	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
669	pu16 = (uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
670	# else
671	*pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
672	# endif
673	return VINF_SUCCESS;
674	}
675	return iemOpcodeGetNextU16Slow(pVCpu, pu16);
676	}
677
678	#else /* IEM_WITH_SETJMP */
679
680	/**
681	* Fetches the next opcode word, longjmp on error.
682	*
683	* @returns The opcode word.
684	* @param pVCpu The cross context virtual CPU structure of the calling thread.
685	*/
686	DECLINLINE(uint16_t) iemOpcodeGetNextU16Jmp(PVMCPUCC pVCpu)
687	{
688	# ifdef IEM_WITH_CODE_TLB
689	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
690	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
691	if (RT_LIKELY( pbBuf != NULL
692	&& offBuf + 2 <= pVCpu->iem.s.cbInstrBuf))
693	{
694	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 2;
695	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
696	return (uint16_t const )&pbBuf[offBuf];
697	# else
698	return RT_MAKE_U16(pbBuf[offBuf], pbBuf[offBuf + 1]);
699	# endif
700	}
701	# else
702	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
703	if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
704	{
705	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
706	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
707	return (uint16_t const )&pVCpu->iem.s.abOpcode[offOpcode];
708	# else
709	return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
710	# endif
711	}
712	# endif
713	return iemOpcodeGetNextU16SlowJmp(pVCpu);
714	}
715
716	#endif /* IEM_WITH_SETJMP */
717
718	/**
719	* Fetches the next opcode word, returns automatically on failure.
720	*
721	* @param a_pu16 Where to return the opcode word.
722	* @remark Implicitly references pVCpu.
723	*/
724	#ifndef IEM_WITH_SETJMP
725	# define IEM_OPCODE_GET_NEXT_U16(a_pu16) \
726	do \
727	{ \
728	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pVCpu, (a_pu16)); \
729	if (rcStrict2 != VINF_SUCCESS) \
730	return rcStrict2; \
731	} while (0)
732	#else
733	# define IEM_OPCODE_GET_NEXT_U16(a_pu16) (*(a_pu16) = iemOpcodeGetNextU16Jmp(pVCpu))
734	#endif
735
736	#ifndef IEM_WITH_SETJMP
737	/**
738	* Fetches the next opcode word, zero extending it to a double word.
739	*
740	* @returns Strict VBox status code.
741	* @param pVCpu The cross context virtual CPU structure of the calling thread.
742	* @param pu32 Where to return the opcode double word.
743	*/
744	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PVMCPUCC pVCpu, uint32_t *pu32)
745	{
746	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
747	if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
748	return iemOpcodeGetNextU16ZxU32Slow(pVCpu, pu32);
749
750	*pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
751	pVCpu->iem.s.offOpcode = offOpcode + 2;
752	return VINF_SUCCESS;
753	}
754	#endif /* !IEM_WITH_SETJMP */
755
756	/**
757	* Fetches the next opcode word and zero extends it to a double word, returns
758	* automatically on failure.
759	*
760	* @param a_pu32 Where to return the opcode double word.
761	* @remark Implicitly references pVCpu.
762	*/
763	#ifndef IEM_WITH_SETJMP
764	# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \
765	do \
766	{ \
767	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pVCpu, (a_pu32)); \
768	if (rcStrict2 != VINF_SUCCESS) \
769	return rcStrict2; \
770	} while (0)
771	#else
772	# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU16Jmp(pVCpu))
773	#endif
774
775	#ifndef IEM_WITH_SETJMP
776	/**
777	* Fetches the next opcode word, zero extending it to a quad word.
778	*
779	* @returns Strict VBox status code.
780	* @param pVCpu The cross context virtual CPU structure of the calling thread.
781	* @param pu64 Where to return the opcode quad word.
782	*/
783	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PVMCPUCC pVCpu, uint64_t *pu64)
784	{
785	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
786	if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
787	return iemOpcodeGetNextU16ZxU64Slow(pVCpu, pu64);
788
789	*pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
790	pVCpu->iem.s.offOpcode = offOpcode + 2;
791	return VINF_SUCCESS;
792	}
793	#endif /* !IEM_WITH_SETJMP */
794
795	/**
796	* Fetches the next opcode word and zero extends it to a quad word, returns
797	* automatically on failure.
798	*
799	* @param a_pu64 Where to return the opcode quad word.
800	* @remark Implicitly references pVCpu.
801	*/
802	#ifndef IEM_WITH_SETJMP
803	# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \
804	do \
805	{ \
806	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pVCpu, (a_pu64)); \
807	if (rcStrict2 != VINF_SUCCESS) \
808	return rcStrict2; \
809	} while (0)
810	#else
811	# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU16Jmp(pVCpu))
812	#endif
813
814
815	#ifndef IEM_WITH_SETJMP
816	/**
817	* Fetches the next signed word from the opcode stream.
818	*
819	* @returns Strict VBox status code.
820	* @param pVCpu The cross context virtual CPU structure of the calling thread.
821	* @param pi16 Where to return the signed word.
822	*/
823	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PVMCPUCC pVCpu, int16_t *pi16)
824	{
825	return iemOpcodeGetNextU16(pVCpu, (uint16_t *)pi16);
826	}
827	#endif /* !IEM_WITH_SETJMP */
828
829
830	/**
831	* Fetches the next signed word from the opcode stream, returning automatically
832	* on failure.
833	*
834	* @param a_pi16 Where to return the signed word.
835	* @remark Implicitly references pVCpu.
836	*/
837	#ifndef IEM_WITH_SETJMP
838	# define IEM_OPCODE_GET_NEXT_S16(a_pi16) \
839	do \
840	{ \
841	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pVCpu, (a_pi16)); \
842	if (rcStrict2 != VINF_SUCCESS) \
843	return rcStrict2; \
844	} while (0)
845	#else
846	# define IEM_OPCODE_GET_NEXT_S16(a_pi16) (*(a_pi16) = (int16_t)iemOpcodeGetNextU16Jmp(pVCpu))
847	#endif
848
849	#ifndef IEM_WITH_SETJMP
850
851	/**
852	* Fetches the next opcode dword.
853	*
854	* @returns Strict VBox status code.
855	* @param pVCpu The cross context virtual CPU structure of the calling thread.
856	* @param pu32 Where to return the opcode double word.
857	*/
858	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PVMCPUCC pVCpu, uint32_t *pu32)
859	{
860	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
861	if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
862	{
863	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
864	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
865	pu32 = (uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
866	# else
867	*pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
868	pVCpu->iem.s.abOpcode[offOpcode + 1],
869	pVCpu->iem.s.abOpcode[offOpcode + 2],
870	pVCpu->iem.s.abOpcode[offOpcode + 3]);
871	# endif
872	return VINF_SUCCESS;
873	}
874	return iemOpcodeGetNextU32Slow(pVCpu, pu32);
875	}
876
877	#else /* IEM_WITH_SETJMP */
878
879	/**
880	* Fetches the next opcode dword, longjmp on error.
881	*
882	* @returns The opcode dword.
883	* @param pVCpu The cross context virtual CPU structure of the calling thread.
884	*/
885	DECLINLINE(uint32_t) iemOpcodeGetNextU32Jmp(PVMCPUCC pVCpu)
886	{
887	# ifdef IEM_WITH_CODE_TLB
888	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
889	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
890	if (RT_LIKELY( pbBuf != NULL
891	&& offBuf + 4 <= pVCpu->iem.s.cbInstrBuf))
892	{
893	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 4;
894	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
895	return (uint32_t const )&pbBuf[offBuf];
896	# else
897	return RT_MAKE_U32_FROM_U8(pbBuf[offBuf],
898	pbBuf[offBuf + 1],
899	pbBuf[offBuf + 2],
900	pbBuf[offBuf + 3]);
901	# endif
902	}
903	# else
904	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
905	if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
906	{
907	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
908	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
909	return (uint32_t const )&pVCpu->iem.s.abOpcode[offOpcode];
910	# else
911	return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
912	pVCpu->iem.s.abOpcode[offOpcode + 1],
913	pVCpu->iem.s.abOpcode[offOpcode + 2],
914	pVCpu->iem.s.abOpcode[offOpcode + 3]);
915	# endif
916	}
917	# endif
918	return iemOpcodeGetNextU32SlowJmp(pVCpu);
919	}
920
921	#endif /* IEM_WITH_SETJMP */
922
923	/**
924	* Fetches the next opcode dword, returns automatically on failure.
925	*
926	* @param a_pu32 Where to return the opcode dword.
927	* @remark Implicitly references pVCpu.
928	*/
929	#ifndef IEM_WITH_SETJMP
930	# define IEM_OPCODE_GET_NEXT_U32(a_pu32) \
931	do \
932	{ \
933	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pVCpu, (a_pu32)); \
934	if (rcStrict2 != VINF_SUCCESS) \
935	return rcStrict2; \
936	} while (0)
937	#else
938	# define IEM_OPCODE_GET_NEXT_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU32Jmp(pVCpu))
939	#endif
940
941	#ifndef IEM_WITH_SETJMP
942	/**
943	* Fetches the next opcode dword, zero extending it to a quad word.
944	*
945	* @returns Strict VBox status code.
946	* @param pVCpu The cross context virtual CPU structure of the calling thread.
947	* @param pu64 Where to return the opcode quad word.
948	*/
949	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PVMCPUCC pVCpu, uint64_t *pu64)
950	{
951	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
952	if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
953	return iemOpcodeGetNextU32ZxU64Slow(pVCpu, pu64);
954
955	*pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
956	pVCpu->iem.s.abOpcode[offOpcode + 1],
957	pVCpu->iem.s.abOpcode[offOpcode + 2],
958	pVCpu->iem.s.abOpcode[offOpcode + 3]);
959	pVCpu->iem.s.offOpcode = offOpcode + 4;
960	return VINF_SUCCESS;
961	}
962	#endif /* !IEM_WITH_SETJMP */
963
964	/**
965	* Fetches the next opcode dword and zero extends it to a quad word, returns
966	* automatically on failure.
967	*
968	* @param a_pu64 Where to return the opcode quad word.
969	* @remark Implicitly references pVCpu.
970	*/
971	#ifndef IEM_WITH_SETJMP
972	# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \
973	do \
974	{ \
975	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pVCpu, (a_pu64)); \
976	if (rcStrict2 != VINF_SUCCESS) \
977	return rcStrict2; \
978	} while (0)
979	#else
980	# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU32Jmp(pVCpu))
981	#endif
982
983
984	#ifndef IEM_WITH_SETJMP
985	/**
986	* Fetches the next signed double word from the opcode stream.
987	*
988	* @returns Strict VBox status code.
989	* @param pVCpu The cross context virtual CPU structure of the calling thread.
990	* @param pi32 Where to return the signed double word.
991	*/
992	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PVMCPUCC pVCpu, int32_t *pi32)
993	{
994	return iemOpcodeGetNextU32(pVCpu, (uint32_t *)pi32);
995	}
996	#endif
997
998	/**
999	* Fetches the next signed double word from the opcode stream, returning
1000	* automatically on failure.
1001	*
1002	* @param a_pi32 Where to return the signed double word.
1003	* @remark Implicitly references pVCpu.
1004	*/
1005	#ifndef IEM_WITH_SETJMP
1006	# define IEM_OPCODE_GET_NEXT_S32(a_pi32) \
1007	do \
1008	{ \
1009	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pVCpu, (a_pi32)); \
1010	if (rcStrict2 != VINF_SUCCESS) \
1011	return rcStrict2; \
1012	} while (0)
1013	#else
1014	# define IEM_OPCODE_GET_NEXT_S32(a_pi32) (*(a_pi32) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1015	#endif
1016
1017	#ifndef IEM_WITH_SETJMP
1018	/**
1019	* Fetches the next opcode dword, sign extending it into a quad word.
1020	*
1021	* @returns Strict VBox status code.
1022	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1023	* @param pu64 Where to return the opcode quad word.
1024	*/
1025	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64)
1026	{
1027	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1028	if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1029	return iemOpcodeGetNextS32SxU64Slow(pVCpu, pu64);
1030
1031	int32_t i32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1032	pVCpu->iem.s.abOpcode[offOpcode + 1],
1033	pVCpu->iem.s.abOpcode[offOpcode + 2],
1034	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1035	*pu64 = i32;
1036	pVCpu->iem.s.offOpcode = offOpcode + 4;
1037	return VINF_SUCCESS;
1038	}
1039	#endif /* !IEM_WITH_SETJMP */
1040
1041	/**
1042	* Fetches the next opcode double word and sign extends it to a quad word,
1043	* returns automatically on failure.
1044	*
1045	* @param a_pu64 Where to return the opcode quad word.
1046	* @remark Implicitly references pVCpu.
1047	*/
1048	#ifndef IEM_WITH_SETJMP
1049	# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \
1050	do \
1051	{ \
1052	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pVCpu, (a_pu64)); \
1053	if (rcStrict2 != VINF_SUCCESS) \
1054	return rcStrict2; \
1055	} while (0)
1056	#else
1057	# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) (*(a_pu64) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1058	#endif
1059
1060	#ifndef IEM_WITH_SETJMP
1061
1062	/**
1063	* Fetches the next opcode qword.
1064	*
1065	* @returns Strict VBox status code.
1066	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1067	* @param pu64 Where to return the opcode qword.
1068	*/
1069	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PVMCPUCC pVCpu, uint64_t *pu64)
1070	{
1071	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1072	if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1073	{
1074	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1075	pu64 = (uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1076	# else
1077	*pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1078	pVCpu->iem.s.abOpcode[offOpcode + 1],
1079	pVCpu->iem.s.abOpcode[offOpcode + 2],
1080	pVCpu->iem.s.abOpcode[offOpcode + 3],
1081	pVCpu->iem.s.abOpcode[offOpcode + 4],
1082	pVCpu->iem.s.abOpcode[offOpcode + 5],
1083	pVCpu->iem.s.abOpcode[offOpcode + 6],
1084	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1085	# endif
1086	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1087	return VINF_SUCCESS;
1088	}
1089	return iemOpcodeGetNextU64Slow(pVCpu, pu64);
1090	}
1091
1092	#else /* IEM_WITH_SETJMP */
1093
1094	/**
1095	* Fetches the next opcode qword, longjmp on error.
1096	*
1097	* @returns The opcode qword.
1098	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1099	*/
1100	DECLINLINE(uint64_t) iemOpcodeGetNextU64Jmp(PVMCPUCC pVCpu)
1101	{
1102	# ifdef IEM_WITH_CODE_TLB
1103	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1104	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1105	if (RT_LIKELY( pbBuf != NULL
1106	&& offBuf + 8 <= pVCpu->iem.s.cbInstrBuf))
1107	{
1108	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 8;
1109	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1110	return (uint64_t const )&pbBuf[offBuf];
1111	# else
1112	return RT_MAKE_U64_FROM_U8(pbBuf[offBuf],
1113	pbBuf[offBuf + 1],
1114	pbBuf[offBuf + 2],
1115	pbBuf[offBuf + 3],
1116	pbBuf[offBuf + 4],
1117	pbBuf[offBuf + 5],
1118	pbBuf[offBuf + 6],
1119	pbBuf[offBuf + 7]);
1120	# endif
1121	}
1122	# else
1123	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1124	if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1125	{
1126	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1127	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1128	return (uint64_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1129	# else
1130	return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1131	pVCpu->iem.s.abOpcode[offOpcode + 1],
1132	pVCpu->iem.s.abOpcode[offOpcode + 2],
1133	pVCpu->iem.s.abOpcode[offOpcode + 3],
1134	pVCpu->iem.s.abOpcode[offOpcode + 4],
1135	pVCpu->iem.s.abOpcode[offOpcode + 5],
1136	pVCpu->iem.s.abOpcode[offOpcode + 6],
1137	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1138	# endif
1139	}
1140	# endif
1141	return iemOpcodeGetNextU64SlowJmp(pVCpu);
1142	}
1143
1144	#endif /* IEM_WITH_SETJMP */
1145
1146	/**
1147	* Fetches the next opcode quad word, returns automatically on failure.
1148	*
1149	* @param a_pu64 Where to return the opcode quad word.
1150	* @remark Implicitly references pVCpu.
1151	*/
1152	#ifndef IEM_WITH_SETJMP
1153	# define IEM_OPCODE_GET_NEXT_U64(a_pu64) \
1154	do \
1155	{ \
1156	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pVCpu, (a_pu64)); \
1157	if (rcStrict2 != VINF_SUCCESS) \
1158	return rcStrict2; \
1159	} while (0)
1160	#else
1161	# define IEM_OPCODE_GET_NEXT_U64(a_pu64) ( *(a_pu64) = iemOpcodeGetNextU64Jmp(pVCpu) )
1162	#endif
1163
1164
1165	/** @name Misc Worker Functions.
1166	* @{
1167	*/
1168
1169	/**
1170	* Gets the correct EFLAGS regardless of whether PATM stores parts of them or
1171	* not (kind of obsolete now).
1172	*
1173	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1174	*/
1175	#define IEMMISC_GET_EFL(a_pVCpu) ( (a_pVCpu)->cpum.GstCtx.eflags.u )
1176
1177	/**
1178	* Updates the EFLAGS in the correct manner wrt. PATM (kind of obsolete).
1179	*
1180	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1181	* @param a_fEfl The new EFLAGS.
1182	*/
1183	#define IEMMISC_SET_EFL(a_pVCpu, a_fEfl) do { (a_pVCpu)->cpum.GstCtx.eflags.u = (a_fEfl); } while (0)
1184
1185
1186	/**
1187	* Loads a NULL data selector into a selector register, both the hidden and
1188	* visible parts, in protected mode.
1189	*
1190	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1191	* @param pSReg Pointer to the segment register.
1192	* @param uRpl The RPL.
1193	*/
1194	DECLINLINE(void) iemHlpLoadNullDataSelectorProt(PVMCPUCC pVCpu, PCPUMSELREG pSReg, RTSEL uRpl)
1195	{
1196	/** @todo Testcase: write a testcase checking what happends when loading a NULL
1197	* data selector in protected mode. */
1198	pSReg->Sel = uRpl;
1199	pSReg->ValidSel = uRpl;
1200	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
1201	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
1202	{
1203	/* VT-x (Intel 3960x) observed doing something like this. */
1204	pSReg->Attr.u = X86DESCATTR_UNUSABLE \| X86DESCATTR_G \| X86DESCATTR_D \| (pVCpu->iem.s.uCpl << X86DESCATTR_DPL_SHIFT);
1205	pSReg->u32Limit = UINT32_MAX;
1206	pSReg->u64Base = 0;
1207	}
1208	else
1209	{
1210	pSReg->Attr.u = X86DESCATTR_UNUSABLE;
1211	pSReg->u32Limit = 0;
1212	pSReg->u64Base = 0;
1213	}
1214	}
1215
1216	/** @} */
1217
1218
1219	/*
1220	*
1221	* Helpers routines.
1222	* Helpers routines.
1223	* Helpers routines.
1224	*
1225	*/
1226
1227	/**
1228	* Recalculates the effective operand size.
1229	*
1230	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1231	*/
1232	DECLINLINE(void) iemRecalEffOpSize(PVMCPUCC pVCpu)
1233	{
1234	switch (pVCpu->iem.s.enmCpuMode)
1235	{
1236	case IEMMODE_16BIT:
1237	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT;
1238	break;
1239	case IEMMODE_32BIT:
1240	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT;
1241	break;
1242	case IEMMODE_64BIT:
1243	switch (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP))
1244	{
1245	case 0:
1246	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize;
1247	break;
1248	case IEM_OP_PRF_SIZE_OP:
1249	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1250	break;
1251	case IEM_OP_PRF_SIZE_REX_W:
1252	case IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP:
1253	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1254	break;
1255	}
1256	break;
1257	default:
1258	AssertFailed();
1259	}
1260	}
1261
1262
1263	/**
1264	* Sets the default operand size to 64-bit and recalculates the effective
1265	* operand size.
1266	*
1267	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1268	*/
1269	DECLINLINE(void) iemRecalEffOpSize64Default(PVMCPUCC pVCpu)
1270	{
1271	Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
1272	pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1273	if ((pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP)
1274	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1275	else
1276	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1277	}
1278
1279
1280
1281
1282	/** @name Register Access.
1283	* @{
1284	*/
1285
1286	/**
1287	* Gets a reference (pointer) to the specified hidden segment register.
1288	*
1289	* @returns Hidden register reference.
1290	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1291	* @param iSegReg The segment register.
1292	*/
1293	DECLINLINE(PCPUMSELREG) iemSRegGetHid(PVMCPUCC pVCpu, uint8_t iSegReg)
1294	{
1295	Assert(iSegReg < X86_SREG_COUNT);
1296	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1297	PCPUMSELREG pSReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
1298
1299	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1300	return pSReg;
1301	}
1302
1303
1304	/**
1305	* Ensures that the given hidden segment register is up to date.
1306	*
1307	* @returns Hidden register reference.
1308	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1309	* @param pSReg The segment register.
1310	*/
1311	DECLINLINE(PCPUMSELREG) iemSRegUpdateHid(PVMCPUCC pVCpu, PCPUMSELREG pSReg)
1312	{
1313	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1314	NOREF(pVCpu);
1315	return pSReg;
1316	}
1317
1318
1319	/**
1320	* Gets a reference (pointer) to the specified segment register (the selector
1321	* value).
1322	*
1323	* @returns Pointer to the selector variable.
1324	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1325	* @param iSegReg The segment register.
1326	*/
1327	DECLINLINE(uint16_t *) iemSRegRef(PVMCPUCC pVCpu, uint8_t iSegReg)
1328	{
1329	Assert(iSegReg < X86_SREG_COUNT);
1330	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1331	return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1332	}
1333
1334
1335	/**
1336	* Fetches the selector value of a segment register.
1337	*
1338	* @returns The selector value.
1339	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1340	* @param iSegReg The segment register.
1341	*/
1342	DECLINLINE(uint16_t) iemSRegFetchU16(PVMCPUCC pVCpu, uint8_t iSegReg)
1343	{
1344	Assert(iSegReg < X86_SREG_COUNT);
1345	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1346	return pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1347	}
1348
1349
1350	/**
1351	* Fetches the base address value of a segment register.
1352	*
1353	* @returns The selector value.
1354	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1355	* @param iSegReg The segment register.
1356	*/
1357	DECLINLINE(uint64_t) iemSRegBaseFetchU64(PVMCPUCC pVCpu, uint8_t iSegReg)
1358	{
1359	Assert(iSegReg < X86_SREG_COUNT);
1360	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1361	return pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1362	}
1363
1364
1365	/**
1366	* Gets a reference (pointer) to the specified general purpose register.
1367	*
1368	* @returns Register reference.
1369	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1370	* @param iReg The general purpose register.
1371	*/
1372	DECLINLINE(void *) iemGRegRef(PVMCPUCC pVCpu, uint8_t iReg)
1373	{
1374	Assert(iReg < 16);
1375	return &pVCpu->cpum.GstCtx.aGRegs[iReg];
1376	}
1377
1378
1379	/**
1380	* Gets a reference (pointer) to the specified 8-bit general purpose register.
1381	*
1382	* Because of AH, CH, DH and BH we cannot use iemGRegRef directly here.
1383	*
1384	* @returns Register reference.
1385	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1386	* @param iReg The register.
1387	*/
1388	DECLINLINE(uint8_t *) iemGRegRefU8(PVMCPUCC pVCpu, uint8_t iReg)
1389	{
1390	if (iReg < 4 \|\| (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX))
1391	{
1392	Assert(iReg < 16);
1393	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u8;
1394	}
1395	/* high 8-bit register. */
1396	Assert(iReg < 8);
1397	return &pVCpu->cpum.GstCtx.aGRegs[iReg & 3].bHi;
1398	}
1399
1400
1401	/**
1402	* Gets a reference (pointer) to the specified 16-bit general purpose register.
1403	*
1404	* @returns Register reference.
1405	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1406	* @param iReg The register.
1407	*/
1408	DECLINLINE(uint16_t *) iemGRegRefU16(PVMCPUCC pVCpu, uint8_t iReg)
1409	{
1410	Assert(iReg < 16);
1411	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1412	}
1413
1414
1415	/**
1416	* Gets a reference (pointer) to the specified 32-bit general purpose register.
1417	*
1418	* @returns Register reference.
1419	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1420	* @param iReg The register.
1421	*/
1422	DECLINLINE(uint32_t *) iemGRegRefU32(PVMCPUCC pVCpu, uint8_t iReg)
1423	{
1424	Assert(iReg < 16);
1425	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1426	}
1427
1428
1429	/**
1430	* Gets a reference (pointer) to the specified 64-bit general purpose register.
1431	*
1432	* @returns Register reference.
1433	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1434	* @param iReg The register.
1435	*/
1436	DECLINLINE(uint64_t *) iemGRegRefU64(PVMCPUCC pVCpu, uint8_t iReg)
1437	{
1438	Assert(iReg < 64);
1439	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1440	}
1441
1442
1443	/**
1444	* Gets a reference (pointer) to the specified segment register's base address.
1445	*
1446	* @returns Segment register base address reference.
1447	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1448	* @param iSegReg The segment selector.
1449	*/
1450	DECLINLINE(uint64_t *) iemSRegBaseRefU64(PVMCPUCC pVCpu, uint8_t iSegReg)
1451	{
1452	Assert(iSegReg < X86_SREG_COUNT);
1453	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1454	return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1455	}
1456
1457
1458	/**
1459	* Fetches the value of a 8-bit general purpose register.
1460	*
1461	* @returns The register value.
1462	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1463	* @param iReg The register.
1464	*/
1465	DECLINLINE(uint8_t) iemGRegFetchU8(PVMCPUCC pVCpu, uint8_t iReg)
1466	{
1467	return *iemGRegRefU8(pVCpu, iReg);
1468	}
1469
1470
1471	/**
1472	* Fetches the value of a 16-bit general purpose register.
1473	*
1474	* @returns The register value.
1475	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1476	* @param iReg The register.
1477	*/
1478	DECLINLINE(uint16_t) iemGRegFetchU16(PVMCPUCC pVCpu, uint8_t iReg)
1479	{
1480	Assert(iReg < 16);
1481	return pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1482	}
1483
1484
1485	/**
1486	* Fetches the value of a 32-bit general purpose register.
1487	*
1488	* @returns The register value.
1489	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1490	* @param iReg The register.
1491	*/
1492	DECLINLINE(uint32_t) iemGRegFetchU32(PVMCPUCC pVCpu, uint8_t iReg)
1493	{
1494	Assert(iReg < 16);
1495	return pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1496	}
1497
1498
1499	/**
1500	* Fetches the value of a 64-bit general purpose register.
1501	*
1502	* @returns The register value.
1503	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1504	* @param iReg The register.
1505	*/
1506	DECLINLINE(uint64_t) iemGRegFetchU64(PVMCPUCC pVCpu, uint8_t iReg)
1507	{
1508	Assert(iReg < 16);
1509	return pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1510	}
1511
1512
1513	/**
1514	* Get the address of the top of the stack.
1515	*
1516	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1517	*/
1518	DECLINLINE(RTGCPTR) iemRegGetEffRsp(PCVMCPU pVCpu)
1519	{
1520	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1521	return pVCpu->cpum.GstCtx.rsp;
1522	if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1523	return pVCpu->cpum.GstCtx.esp;
1524	return pVCpu->cpum.GstCtx.sp;
1525	}
1526
1527
1528	/**
1529	* Updates the RIP/EIP/IP to point to the next instruction.
1530	*
1531	* This function leaves the EFLAGS.RF flag alone.
1532	*
1533	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1534	* @param cbInstr The number of bytes to add.
1535	*/
1536	DECLINLINE(void) iemRegAddToRipKeepRF(PVMCPUCC pVCpu, uint8_t cbInstr)
1537	{
1538	switch (pVCpu->iem.s.enmCpuMode)
1539	{
1540	case IEMMODE_16BIT:
1541	Assert(pVCpu->cpum.GstCtx.rip <= UINT16_MAX);
1542	pVCpu->cpum.GstCtx.eip += cbInstr;
1543	pVCpu->cpum.GstCtx.eip &= UINT32_C(0xffff);
1544	break;
1545
1546	case IEMMODE_32BIT:
1547	pVCpu->cpum.GstCtx.eip += cbInstr;
1548	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
1549	break;
1550
1551	case IEMMODE_64BIT:
1552	pVCpu->cpum.GstCtx.rip += cbInstr;
1553	break;
1554	default: AssertFailed();
1555	}
1556	}
1557
1558
1559	#if 0
1560	/**
1561	* Updates the RIP/EIP/IP to point to the next instruction.
1562	*
1563	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1564	*/
1565	DECLINLINE(void) iemRegUpdateRipKeepRF(PVMCPUCC pVCpu)
1566	{
1567	return iemRegAddToRipKeepRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
1568	}
1569	#endif
1570
1571
1572
1573	/**
1574	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
1575	*
1576	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1577	* @param cbInstr The number of bytes to add.
1578	*/
1579	DECLINLINE(void) iemRegAddToRipAndClearRF(PVMCPUCC pVCpu, uint8_t cbInstr)
1580	{
1581	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
1582
1583	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
1584	#if ARCH_BITS >= 64
1585	static uint64_t const s_aRipMasks[] = { UINT64_C(0xffffffff), UINT64_C(0xffffffff), UINT64_MAX };
1586	Assert(pVCpu->cpum.GstCtx.rip <= s_aRipMasks[(unsigned)pVCpu->iem.s.enmCpuMode]);
1587	pVCpu->cpum.GstCtx.rip = (pVCpu->cpum.GstCtx.rip + cbInstr) & s_aRipMasks[(unsigned)pVCpu->iem.s.enmCpuMode];
1588	#else
1589	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1590	pVCpu->cpum.GstCtx.rip += cbInstr;
1591	else
1592	pVCpu->cpum.GstCtx.eip += cbInstr;
1593	#endif
1594	}
1595
1596
1597	/**
1598	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
1599	*
1600	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1601	*/
1602	DECLINLINE(void) iemRegUpdateRipAndClearRF(PVMCPUCC pVCpu)
1603	{
1604	return iemRegAddToRipAndClearRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
1605	}
1606
1607
1608	/**
1609	* Adds to the stack pointer.
1610	*
1611	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1612	* @param cbToAdd The number of bytes to add (8-bit!).
1613	*/
1614	DECLINLINE(void) iemRegAddToRsp(PVMCPUCC pVCpu, uint8_t cbToAdd)
1615	{
1616	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1617	pVCpu->cpum.GstCtx.rsp += cbToAdd;
1618	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1619	pVCpu->cpum.GstCtx.esp += cbToAdd;
1620	else
1621	pVCpu->cpum.GstCtx.sp += cbToAdd;
1622	}
1623
1624
1625	/**
1626	* Subtracts from the stack pointer.
1627	*
1628	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1629	* @param cbToSub The number of bytes to subtract (8-bit!).
1630	*/
1631	DECLINLINE(void) iemRegSubFromRsp(PVMCPUCC pVCpu, uint8_t cbToSub)
1632	{
1633	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1634	pVCpu->cpum.GstCtx.rsp -= cbToSub;
1635	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1636	pVCpu->cpum.GstCtx.esp -= cbToSub;
1637	else
1638	pVCpu->cpum.GstCtx.sp -= cbToSub;
1639	}
1640
1641
1642	/**
1643	* Adds to the temporary stack pointer.
1644	*
1645	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1646	* @param pTmpRsp The temporary SP/ESP/RSP to update.
1647	* @param cbToAdd The number of bytes to add (16-bit).
1648	*/
1649	DECLINLINE(void) iemRegAddToRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToAdd)
1650	{
1651	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1652	pTmpRsp->u += cbToAdd;
1653	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1654	pTmpRsp->DWords.dw0 += cbToAdd;
1655	else
1656	pTmpRsp->Words.w0 += cbToAdd;
1657	}
1658
1659
1660	/**
1661	* Subtracts from the temporary stack pointer.
1662	*
1663	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1664	* @param pTmpRsp The temporary SP/ESP/RSP to update.
1665	* @param cbToSub The number of bytes to subtract.
1666	* @remarks The @a cbToSub argument MUST be 16-bit, iemCImpl_enter is
1667	* expecting that.
1668	*/
1669	DECLINLINE(void) iemRegSubFromRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToSub)
1670	{
1671	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1672	pTmpRsp->u -= cbToSub;
1673	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1674	pTmpRsp->DWords.dw0 -= cbToSub;
1675	else
1676	pTmpRsp->Words.w0 -= cbToSub;
1677	}
1678
1679
1680	/**
1681	* Calculates the effective stack address for a push of the specified size as
1682	* well as the new RSP value (upper bits may be masked).
1683	*
1684	* @returns Effective stack addressf for the push.
1685	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1686	* @param cbItem The size of the stack item to pop.
1687	* @param puNewRsp Where to return the new RSP value.
1688	*/
1689	DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp)
1690	{
1691	RTUINT64U uTmpRsp;
1692	RTGCPTR GCPtrTop;
1693	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
1694
1695	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1696	GCPtrTop = uTmpRsp.u -= cbItem;
1697	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1698	GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem;
1699	else
1700	GCPtrTop = uTmpRsp.Words.w0 -= cbItem;
1701	*puNewRsp = uTmpRsp.u;
1702	return GCPtrTop;
1703	}
1704
1705
1706	/**
1707	* Gets the current stack pointer and calculates the value after a pop of the
1708	* specified size.
1709	*
1710	* @returns Current stack pointer.
1711	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1712	* @param cbItem The size of the stack item to pop.
1713	* @param puNewRsp Where to return the new RSP value.
1714	*/
1715	DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp)
1716	{
1717	RTUINT64U uTmpRsp;
1718	RTGCPTR GCPtrTop;
1719	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
1720
1721	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1722	{
1723	GCPtrTop = uTmpRsp.u;
1724	uTmpRsp.u += cbItem;
1725	}
1726	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1727	{
1728	GCPtrTop = uTmpRsp.DWords.dw0;
1729	uTmpRsp.DWords.dw0 += cbItem;
1730	}
1731	else
1732	{
1733	GCPtrTop = uTmpRsp.Words.w0;
1734	uTmpRsp.Words.w0 += cbItem;
1735	}
1736	*puNewRsp = uTmpRsp.u;
1737	return GCPtrTop;
1738	}
1739
1740
1741	/**
1742	* Calculates the effective stack address for a push of the specified size as
1743	* well as the new temporary RSP value (upper bits may be masked).
1744	*
1745	* @returns Effective stack addressf for the push.
1746	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1747	* @param pTmpRsp The temporary stack pointer. This is updated.
1748	* @param cbItem The size of the stack item to pop.
1749	*/
1750	DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem)
1751	{
1752	RTGCPTR GCPtrTop;
1753
1754	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1755	GCPtrTop = pTmpRsp->u -= cbItem;
1756	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1757	GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem;
1758	else
1759	GCPtrTop = pTmpRsp->Words.w0 -= cbItem;
1760	return GCPtrTop;
1761	}
1762
1763
1764	/**
1765	* Gets the effective stack address for a pop of the specified size and
1766	* calculates and updates the temporary RSP.
1767	*
1768	* @returns Current stack pointer.
1769	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1770	* @param pTmpRsp The temporary stack pointer. This is updated.
1771	* @param cbItem The size of the stack item to pop.
1772	*/
1773	DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem)
1774	{
1775	RTGCPTR GCPtrTop;
1776	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1777	{
1778	GCPtrTop = pTmpRsp->u;
1779	pTmpRsp->u += cbItem;
1780	}
1781	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1782	{
1783	GCPtrTop = pTmpRsp->DWords.dw0;
1784	pTmpRsp->DWords.dw0 += cbItem;
1785	}
1786	else
1787	{
1788	GCPtrTop = pTmpRsp->Words.w0;
1789	pTmpRsp->Words.w0 += cbItem;
1790	}
1791	return GCPtrTop;
1792	}
1793
1794	/** @} */
1795
1796
1797	/** @name FPU access and helpers.
1798	*
1799	* @{
1800	*/
1801
1802
1803	/**
1804	* Hook for preparing to use the host FPU.
1805	*
1806	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1807	*
1808	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1809	*/
1810	DECLINLINE(void) iemFpuPrepareUsage(PVMCPUCC pVCpu)
1811	{
1812	#ifdef IN_RING3
1813	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
1814	#else
1815	CPUMRZFpuStatePrepareHostCpuForUse(pVCpu);
1816	#endif
1817	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
1818	}
1819
1820
1821	/**
1822	* Hook for preparing to use the host FPU for SSE.
1823	*
1824	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1825	*
1826	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1827	*/
1828	DECLINLINE(void) iemFpuPrepareUsageSse(PVMCPUCC pVCpu)
1829	{
1830	iemFpuPrepareUsage(pVCpu);
1831	}
1832
1833
1834	/**
1835	* Hook for preparing to use the host FPU for AVX.
1836	*
1837	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1838	*
1839	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1840	*/
1841	DECLINLINE(void) iemFpuPrepareUsageAvx(PVMCPUCC pVCpu)
1842	{
1843	iemFpuPrepareUsage(pVCpu);
1844	}
1845
1846
1847	/**
1848	* Hook for actualizing the guest FPU state before the interpreter reads it.
1849	*
1850	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1851	*
1852	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1853	*/
1854	DECLINLINE(void) iemFpuActualizeStateForRead(PVMCPUCC pVCpu)
1855	{
1856	#ifdef IN_RING3
1857	NOREF(pVCpu);
1858	#else
1859	CPUMRZFpuStateActualizeForRead(pVCpu);
1860	#endif
1861	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
1862	}
1863
1864
1865	/**
1866	* Hook for actualizing the guest FPU state before the interpreter changes it.
1867	*
1868	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1869	*
1870	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1871	*/
1872	DECLINLINE(void) iemFpuActualizeStateForChange(PVMCPUCC pVCpu)
1873	{
1874	#ifdef IN_RING3
1875	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
1876	#else
1877	CPUMRZFpuStateActualizeForChange(pVCpu);
1878	#endif
1879	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
1880	}
1881
1882
1883	/**
1884	* Hook for actualizing the guest XMM0..15 and MXCSR register state for read
1885	* only.
1886	*
1887	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1888	*
1889	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1890	*/
1891	DECLINLINE(void) iemFpuActualizeSseStateForRead(PVMCPUCC pVCpu)
1892	{
1893	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
1894	NOREF(pVCpu);
1895	#else
1896	CPUMRZFpuStateActualizeSseForRead(pVCpu);
1897	#endif
1898	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
1899	}
1900
1901
1902	/**
1903	* Hook for actualizing the guest XMM0..15 and MXCSR register state for
1904	* read+write.
1905	*
1906	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1907	*
1908	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1909	*/
1910	DECLINLINE(void) iemFpuActualizeSseStateForChange(PVMCPUCC pVCpu)
1911	{
1912	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
1913	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
1914	#else
1915	CPUMRZFpuStateActualizeForChange(pVCpu);
1916	#endif
1917	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
1918
1919	/* Make sure any changes are loaded the next time around. */
1920	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_SSE;
1921	}
1922
1923
1924	/**
1925	* Hook for actualizing the guest YMM0..15 and MXCSR register state for read
1926	* only.
1927	*
1928	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1929	*
1930	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1931	*/
1932	DECLINLINE(void) iemFpuActualizeAvxStateForRead(PVMCPUCC pVCpu)
1933	{
1934	#ifdef IN_RING3
1935	NOREF(pVCpu);
1936	#else
1937	CPUMRZFpuStateActualizeAvxForRead(pVCpu);
1938	#endif
1939	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
1940	}
1941
1942
1943	/**
1944	* Hook for actualizing the guest YMM0..15 and MXCSR register state for
1945	* read+write.
1946	*
1947	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1948	*
1949	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1950	*/
1951	DECLINLINE(void) iemFpuActualizeAvxStateForChange(PVMCPUCC pVCpu)
1952	{
1953	#ifdef IN_RING3
1954	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
1955	#else
1956	CPUMRZFpuStateActualizeForChange(pVCpu);
1957	#endif
1958	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
1959
1960	/* Just assume we're going to make changes to the SSE and YMM_HI parts. */
1961	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_YMM \| XSAVE_C_SSE;
1962	}
1963
1964
1965	/**
1966	* Stores a QNaN value into a FPU register.
1967	*
1968	* @param pReg Pointer to the register.
1969	*/
1970	DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg)
1971	{
1972	pReg->au32[0] = UINT32_C(0x00000000);
1973	pReg->au32[1] = UINT32_C(0xc0000000);
1974	pReg->au16[4] = UINT16_C(0xffff);
1975	}
1976
1977
1978	/**
1979	* Updates the FOP, FPU.CS and FPUIP registers.
1980	*
1981	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1982	* @param pFpuCtx The FPU context.
1983	*/
1984	DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorker(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx)
1985	{
1986	Assert(pVCpu->iem.s.uFpuOpcode != UINT16_MAX);
1987	pFpuCtx->FOP = pVCpu->iem.s.uFpuOpcode;
1988	/** @todo x87.CS and FPUIP needs to be kept seperately. */
1989	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
1990	{
1991	/** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled
1992	* happens in real mode here based on the fnsave and fnstenv images. */
1993	pFpuCtx->CS = 0;
1994	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.eip \| ((uint32_t)pVCpu->cpum.GstCtx.cs.Sel << 4);
1995	}
1996	else if (!IEM_IS_LONG_MODE(pVCpu))
1997	{
1998	pFpuCtx->CS = pVCpu->cpum.GstCtx.cs.Sel;
1999	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2000	}
2001	else
2002	(uint64_t )&pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2003	}
2004
2005
2006
2007
2008
2009	/**
2010	* Marks the specified stack register as free (for FFREE).
2011	*
2012	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2013	* @param iStReg The register to free.
2014	*/
2015	DECLINLINE(void) iemFpuStackFree(PVMCPUCC pVCpu, uint8_t iStReg)
2016	{
2017	Assert(iStReg < 8);
2018	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2019	uint8_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2020	pFpuCtx->FTW &= ~RT_BIT(iReg);
2021	}
2022
2023
2024	/**
2025	* Increments FSW.TOP, i.e. pops an item off the stack without freeing it.
2026	*
2027	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2028	*/
2029	DECLINLINE(void) iemFpuStackIncTop(PVMCPUCC pVCpu)
2030	{
2031	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2032	uint16_t uFsw = pFpuCtx->FSW;
2033	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2034	uTop = (uTop + (1 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2035	uFsw &= ~X86_FSW_TOP_MASK;
2036	uFsw \|= uTop;
2037	pFpuCtx->FSW = uFsw;
2038	}
2039
2040
2041	/**
2042	* Decrements FSW.TOP, i.e. push an item off the stack without storing anything.
2043	*
2044	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2045	*/
2046	DECLINLINE(void) iemFpuStackDecTop(PVMCPUCC pVCpu)
2047	{
2048	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2049	uint16_t uFsw = pFpuCtx->FSW;
2050	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2051	uTop = (uTop + (7 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2052	uFsw &= ~X86_FSW_TOP_MASK;
2053	uFsw \|= uTop;
2054	pFpuCtx->FSW = uFsw;
2055	}
2056
2057
2058
2059
2060	DECLINLINE(int) iemFpuStRegNotEmpty(PVMCPUCC pVCpu, uint8_t iStReg)
2061	{
2062	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2063	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2064	if (pFpuCtx->FTW & RT_BIT(iReg))
2065	return VINF_SUCCESS;
2066	return VERR_NOT_FOUND;
2067	}
2068
2069
2070	DECLINLINE(int) iemFpuStRegNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef)
2071	{
2072	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2073	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2074	if (pFpuCtx->FTW & RT_BIT(iReg))
2075	{
2076	*ppRef = &pFpuCtx->aRegs[iStReg].r80;
2077	return VINF_SUCCESS;
2078	}
2079	return VERR_NOT_FOUND;
2080	}
2081
2082
2083	DECLINLINE(int) iemFpu2StRegsNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0,
2084	uint8_t iStReg1, PCRTFLOAT80U *ppRef1)
2085	{
2086	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2087	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2088	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2089	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2090	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
2091	{
2092	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2093	*ppRef1 = &pFpuCtx->aRegs[iStReg1].r80;
2094	return VINF_SUCCESS;
2095	}
2096	return VERR_NOT_FOUND;
2097	}
2098
2099
2100	DECLINLINE(int) iemFpu2StRegsNotEmptyRefFirst(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1)
2101	{
2102	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2103	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2104	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2105	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2106	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
2107	{
2108	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2109	return VINF_SUCCESS;
2110	}
2111	return VERR_NOT_FOUND;
2112	}
2113
2114
2115	/**
2116	* Updates the FPU exception status after FCW is changed.
2117	*
2118	* @param pFpuCtx The FPU context.
2119	*/
2120	DECLINLINE(void) iemFpuRecalcExceptionStatus(PX86FXSTATE pFpuCtx)
2121	{
2122	uint16_t u16Fsw = pFpuCtx->FSW;
2123	if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pFpuCtx->FCW & X86_FCW_XCPT_MASK))
2124	u16Fsw \|= X86_FSW_ES \| X86_FSW_B;
2125	else
2126	u16Fsw &= ~(X86_FSW_ES \| X86_FSW_B);
2127	pFpuCtx->FSW = u16Fsw;
2128	}
2129
2130
2131	/**
2132	* Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE.
2133	*
2134	* @returns The full FTW.
2135	* @param pFpuCtx The FPU context.
2136	*/
2137	DECLINLINE(uint16_t) iemFpuCalcFullFtw(PCX86FXSTATE pFpuCtx)
2138	{
2139	uint8_t const u8Ftw = (uint8_t)pFpuCtx->FTW;
2140	uint16_t u16Ftw = 0;
2141	unsigned const iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2142	for (unsigned iSt = 0; iSt < 8; iSt++)
2143	{
2144	unsigned const iReg = (iSt + iTop) & 7;
2145	if (!(u8Ftw & RT_BIT(iReg)))
2146	u16Ftw \|= 3 << (iReg * 2); /* empty */
2147	else
2148	{
2149	uint16_t uTag;
2150	PCRTFLOAT80U const pr80Reg = &pFpuCtx->aRegs[iSt].r80;
2151	if (pr80Reg->s.uExponent == 0x7fff)
2152	uTag = 2; /* Exponent is all 1's => Special. */
2153	else if (pr80Reg->s.uExponent == 0x0000)
2154	{
2155	if (pr80Reg->s.uMantissa == 0x0000)
2156	uTag = 1; /* All bits are zero => Zero. */
2157	else
2158	uTag = 2; /* Must be special. */
2159	}
2160	else if (pr80Reg->s.uMantissa & RT_BIT_64(63)) /* The J bit. */
2161	uTag = 0; /* Valid. */
2162	else
2163	uTag = 2; /* Must be special. */
2164
2165	u16Ftw \|= uTag << (iReg * 2); /* empty */
2166	}
2167	}
2168
2169	return u16Ftw;
2170	}
2171
2172
2173	/**
2174	* Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR).
2175	*
2176	* @returns The compressed FTW.
2177	* @param u16FullFtw The full FTW to convert.
2178	*/
2179	DECLINLINE(uint16_t) iemFpuCompressFtw(uint16_t u16FullFtw)
2180	{
2181	uint8_t u8Ftw = 0;
2182	for (unsigned i = 0; i < 8; i++)
2183	{
2184	if ((u16FullFtw & 3) != 3 /empty/)
2185	u8Ftw \|= RT_BIT(i);
2186	u16FullFtw >>= 2;
2187	}
2188
2189	return u8Ftw;
2190	}
2191
2192	/** @} */
2193
2194
2195	/** @name Memory access.
2196	*
2197	* @{
2198	*/
2199
2200
2201	/**
2202	* Checks whether alignment checks are enabled or not.
2203	*
2204	* @returns true if enabled, false if not.
2205	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2206	*/
2207	DECLINLINE(bool) iemMemAreAlignmentChecksEnabled(PVMCPUCC pVCpu)
2208	{
2209	AssertCompile(X86_CR0_AM == X86_EFL_AC);
2210	return pVCpu->iem.s.uCpl == 3
2211	&& (((uint32_t)pVCpu->cpum.GstCtx.cr0 & pVCpu->cpum.GstCtx.eflags.u) & X86_CR0_AM);
2212	}
2213
2214	/**
2215	* Checks if the given segment can be written to, raise the appropriate
2216	* exception if not.
2217	*
2218	* @returns VBox strict status code.
2219	*
2220	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2221	* @param pHid Pointer to the hidden register.
2222	* @param iSegReg The register number.
2223	* @param pu64BaseAddr Where to return the base address to use for the
2224	* segment. (In 64-bit code it may differ from the
2225	* base in the hidden segment.)
2226	*/
2227	DECLINLINE(VBOXSTRICTRC) iemMemSegCheckWriteAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid, uint8_t iSegReg, uint64_t *pu64BaseAddr)
2228	{
2229	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2230
2231	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2232	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
2233	else
2234	{
2235	if (!pHid->Attr.n.u1Present)
2236	{
2237	uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
2238	AssertRelease(uSel == 0);
2239	Log(("iemMemSegCheckWriteAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
2240	return iemRaiseGeneralProtectionFault0(pVCpu);
2241	}
2242
2243	if ( ( (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE)
2244	\|\| !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
2245	&& pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT )
2246	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
2247	*pu64BaseAddr = pHid->u64Base;
2248	}
2249	return VINF_SUCCESS;
2250	}
2251
2252
2253	/**
2254	* Checks if the given segment can be read from, raise the appropriate
2255	* exception if not.
2256	*
2257	* @returns VBox strict status code.
2258	*
2259	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2260	* @param pHid Pointer to the hidden register.
2261	* @param iSegReg The register number.
2262	* @param pu64BaseAddr Where to return the base address to use for the
2263	* segment. (In 64-bit code it may differ from the
2264	* base in the hidden segment.)
2265	*/
2266	DECLINLINE(VBOXSTRICTRC) iemMemSegCheckReadAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid, uint8_t iSegReg, uint64_t *pu64BaseAddr)
2267	{
2268	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2269
2270	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2271	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
2272	else
2273	{
2274	if (!pHid->Attr.n.u1Present)
2275	{
2276	uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
2277	AssertRelease(uSel == 0);
2278	Log(("iemMemSegCheckReadAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
2279	return iemRaiseGeneralProtectionFault0(pVCpu);
2280	}
2281
2282	if ((pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
2283	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
2284	*pu64BaseAddr = pHid->u64Base;
2285	}
2286	return VINF_SUCCESS;
2287	}
2288
2289
2290	/**
2291	* Maps a physical page.
2292	*
2293	* @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr).
2294	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2295	* @param GCPhysMem The physical address.
2296	* @param fAccess The intended access.
2297	* @param ppvMem Where to return the mapping address.
2298	* @param pLock The PGM lock.
2299	*/
2300	DECLINLINE(int) iemMemPageMap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, void **ppvMem, PPGMPAGEMAPLOCK pLock)
2301	{
2302	#ifdef IEM_LOG_MEMORY_WRITES
2303	if (fAccess & IEM_ACCESS_TYPE_WRITE)
2304	return VERR_PGM_PHYS_TLB_CATCH_ALL;
2305	#endif
2306
2307	/** @todo This API may require some improving later. A private deal with PGM
2308	* regarding locking and unlocking needs to be struct. A couple of TLBs
2309	* living in PGM, but with publicly accessible inlined access methods
2310	* could perhaps be an even better solution. */
2311	int rc = PGMPhysIemGCPhys2Ptr(pVCpu->CTX_SUFF(pVM), pVCpu,
2312	GCPhysMem,
2313	RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE),
2314	pVCpu->iem.s.fBypassHandlers,
2315	ppvMem,
2316	pLock);
2317	/Log(("PGMPhysIemGCPhys2Ptr %Rrc pLock=%.Rhxs\n", rc, sizeof(pLock), pLock));/
2318	AssertMsg(rc == VINF_SUCCESS \|\| RT_FAILURE_NP(rc), ("%Rrc\n", rc));
2319
2320	return rc;
2321	}
2322
2323
2324	/**
2325	* Unmap a page previously mapped by iemMemPageMap.
2326	*
2327	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2328	* @param GCPhysMem The physical address.
2329	* @param fAccess The intended access.
2330	* @param pvMem What iemMemPageMap returned.
2331	* @param pLock The PGM lock.
2332	*/
2333	DECLINLINE(void) iemMemPageUnmap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, const void *pvMem, PPGMPAGEMAPLOCK pLock)
2334	{
2335	NOREF(pVCpu);
2336	NOREF(GCPhysMem);
2337	NOREF(fAccess);
2338	NOREF(pvMem);
2339	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), pLock);
2340	}
2341
2342	#ifdef IEM_WITH_SETJMP
2343
2344	/** @todo slim this down */
2345	DECLINLINE(RTGCPTR) iemMemApplySegmentToReadJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem, RTGCPTR GCPtrMem)
2346	{
2347	Assert(cbMem >= 1);
2348	Assert(iSegReg < X86_SREG_COUNT);
2349
2350	/*
2351	* 64-bit mode is simpler.
2352	*/
2353	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2354	{
2355	if (iSegReg >= X86_SREG_FS && iSegReg != UINT8_MAX)
2356	{
2357	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2358	PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
2359	GCPtrMem += pSel->u64Base;
2360	}
2361
2362	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
2363	return GCPtrMem;
2364	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
2365	}
2366	/*
2367	* 16-bit and 32-bit segmentation.
2368	*/
2369	else if (iSegReg != UINT8_MAX)
2370	{
2371	/** @todo Does this apply to segments with 4G-1 limit? */
2372	uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
2373	if (RT_LIKELY(GCPtrLast32 >= (uint32_t)GCPtrMem))
2374	{
2375	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2376	PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
2377	switch (pSel->Attr.u & ( X86DESCATTR_P \| X86DESCATTR_UNUSABLE
2378	\| X86_SEL_TYPE_READ \| X86_SEL_TYPE_WRITE /* same as read */
2379	\| X86_SEL_TYPE_DOWN \| X86_SEL_TYPE_CONF /* same as down */
2380	\| X86_SEL_TYPE_CODE))
2381	{
2382	case X86DESCATTR_P: /* readonly data, expand up */
2383	case X86DESCATTR_P \| X86_SEL_TYPE_WRITE: /* writable data, expand up */
2384	case X86DESCATTR_P \| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ: /* code, read-only */
2385	case X86DESCATTR_P \| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ \| X86_SEL_TYPE_CONF: /* conforming code, read-only */
2386	/* expand up */
2387	if (RT_LIKELY(GCPtrLast32 <= pSel->u32Limit))
2388	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2389	Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x vs %#x\n",
2390	(uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit));
2391	break;
2392
2393	case X86DESCATTR_P \| X86_SEL_TYPE_DOWN: /* readonly data, expand down */
2394	case X86DESCATTR_P \| X86_SEL_TYPE_DOWN \| X86_SEL_TYPE_WRITE: /* writable data, expand down */
2395	/* expand down */
2396	if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
2397	&& ( pSel->Attr.n.u1DefBig
2398	\|\| GCPtrLast32 <= UINT32_C(0xffff)) ))
2399	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2400	Log10(("iemMemApplySegmentToReadJmp: expand down out of bounds %#x..%#x vs %#x..%#x\n",
2401	(uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit, pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT16_MAX));
2402	break;
2403
2404	default:
2405	Log10(("iemMemApplySegmentToReadJmp: bad selector %#x\n", pSel->Attr.u));
2406	iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
2407	break;
2408	}
2409	}
2410	Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x\n",(uint32_t)GCPtrMem, GCPtrLast32));
2411	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
2412	}
2413	/*
2414	* 32-bit flat address.
2415	*/
2416	else
2417	return GCPtrMem;
2418	}
2419
2420
2421	/** @todo slim this down */
2422	DECLINLINE(RTGCPTR) iemMemApplySegmentToWriteJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem, RTGCPTR GCPtrMem)
2423	{
2424	Assert(cbMem >= 1);
2425	Assert(iSegReg < X86_SREG_COUNT);
2426
2427	/*
2428	* 64-bit mode is simpler.
2429	*/
2430	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2431	{
2432	if (iSegReg >= X86_SREG_FS)
2433	{
2434	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2435	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
2436	GCPtrMem += pSel->u64Base;
2437	}
2438
2439	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
2440	return GCPtrMem;
2441	}
2442	/*
2443	* 16-bit and 32-bit segmentation.
2444	*/
2445	else
2446	{
2447	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2448	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
2449	uint32_t const fRelevantAttrs = pSel->Attr.u & ( X86DESCATTR_P \| X86DESCATTR_UNUSABLE
2450	\| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_WRITE \| X86_SEL_TYPE_DOWN);
2451	if (fRelevantAttrs == (X86DESCATTR_P \| X86_SEL_TYPE_WRITE)) /* data, expand up */
2452	{
2453	/* expand up */
2454	uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
2455	if (RT_LIKELY( GCPtrLast32 > pSel->u32Limit
2456	&& GCPtrLast32 > (uint32_t)GCPtrMem))
2457	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2458	}
2459	else if (fRelevantAttrs == (X86DESCATTR_P \| X86_SEL_TYPE_WRITE \| X86_SEL_TYPE_DOWN)) /* data, expand up */
2460	{
2461	/* expand down */
2462	uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
2463	if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
2464	&& GCPtrLast32 <= (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
2465	&& GCPtrLast32 > (uint32_t)GCPtrMem))
2466	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2467	}
2468	else
2469	iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
2470	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
2471	}
2472	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
2473	}
2474
2475	#endif /* IEM_WITH_SETJMP */
2476
2477	/**
2478	* Fakes a long mode stack selector for SS = 0.
2479	*
2480	* @param pDescSs Where to return the fake stack descriptor.
2481	* @param uDpl The DPL we want.
2482	*/
2483	DECLINLINE(void) iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl)
2484	{
2485	pDescSs->Long.au64[0] = 0;
2486	pDescSs->Long.au64[1] = 0;
2487	pDescSs->Long.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
2488	pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */
2489	pDescSs->Long.Gen.u2Dpl = uDpl;
2490	pDescSs->Long.Gen.u1Present = 1;
2491	pDescSs->Long.Gen.u1Long = 1;
2492	}
2493
2494	/** @} */
2495
2496
2497	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2498
2499	/**
2500	* Gets CR0 fixed-0 bits in VMX non-root mode.
2501	*
2502	* We do this rather than fetching what we report to the guest (in
2503	* IA32_VMX_CR0_FIXED0 MSR) because real hardware (and so do we) report the same
2504	* values regardless of whether unrestricted-guest feature is available on the CPU.
2505	*
2506	* @returns CR0 fixed-0 bits.
2507	* @param pVCpu The cross context virtual CPU structure.
2508	*/
2509	DECLINLINE(uint64_t) iemVmxGetCr0Fixed0(PCVMCPUCC pVCpu)
2510	{
2511	Assert(IEM_VMX_IS_ROOT_MODE(pVCpu));
2512	Assert(IEM_VMX_HAS_CURRENT_VMCS(pVCpu));
2513
2514	static uint64_t const s_auCr0Fixed0[2] = { VMX_V_CR0_FIXED0, VMX_V_CR0_FIXED0_UX };
2515	PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
2516	uint8_t const fUnrestrictedGuest = !!(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST);
2517	uint64_t const uCr0Fixed0 = s_auCr0Fixed0[fUnrestrictedGuest];
2518	Assert(!(uCr0Fixed0 & (X86_CR0_NW \| X86_CR0_CD)));
2519	return uCr0Fixed0;
2520	}
2521
2522
2523	/**
2524	* Sets virtual-APIC write emulation as pending.
2525	*
2526	* @param pVCpu The cross context virtual CPU structure.
2527	* @param offApic The offset in the virtual-APIC page that was written.
2528	*/
2529	DECLINLINE(void) iemVmxVirtApicSetPendingWrite(PVMCPUCC pVCpu, uint16_t offApic)
2530	{
2531	Assert(offApic < XAPIC_OFF_END + 4);
2532
2533	/*
2534	* Record the currently updated APIC offset, as we need this later for figuring
2535	* out whether to perform TPR, EOI or self-IPI virtualization as well as well
2536	* as for supplying the exit qualification when causing an APIC-write VM-exit.
2537	*/
2538	pVCpu->cpum.GstCtx.hwvirt.vmx.offVirtApicWrite = offApic;
2539
2540	/*
2541	* Flag that we need to perform virtual-APIC write emulation (TPR/PPR/EOI/Self-IPI
2542	* virtualization or APIC-write emulation).
2543	*/
2544	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
2545	VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE);
2546	}
2547
2548	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
2549
2550	#endif /* !VMM_INCLUDED_SRC_include_IEMInline_h */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMInline.h@ 95512

Download in other formats: