VirtualBox

source: vbox/trunk/src/VBox/VMM/include/IEMInline.h@ 100140

Last change on this file since 100140 was 100096, checked in by vboxsync, 17 months ago

VMM/IEM: Adjusted/reworked the relative jump MCs in the threaded function file. bugref:10369
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 119.0 KB
Line 
1/* $Id: IEMInline.h 100096 2023-06-07 15:14:56Z vboxsync $ */
2/** @file
3 * IEM - Interpreted Execution Manager - Inlined Functions.
4 */
5
6/*
7 * Copyright (C) 2011-2023 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28#ifndef VMM_INCLUDED_SRC_include_IEMInline_h
29#define VMM_INCLUDED_SRC_include_IEMInline_h
30#ifndef RT_WITHOUT_PRAGMA_ONCE
31# pragma once
32#endif
33
34
35
36/**
37 * Makes status code addjustments (pass up from I/O and access handler)
38 * as well as maintaining statistics.
39 *
40 * @returns Strict VBox status code to pass up.
41 * @param pVCpu The cross context virtual CPU structure of the calling thread.
42 * @param rcStrict The status from executing an instruction.
43 */
44DECL_FORCE_INLINE(VBOXSTRICTRC) iemExecStatusCodeFiddling(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
45{
46 if (rcStrict != VINF_SUCCESS)
47 {
48 if (RT_SUCCESS(rcStrict))
49 {
50 AssertMsg( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
51 || rcStrict == VINF_IOM_R3_IOPORT_READ
52 || rcStrict == VINF_IOM_R3_IOPORT_WRITE
53 || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
54 || rcStrict == VINF_IOM_R3_MMIO_READ
55 || rcStrict == VINF_IOM_R3_MMIO_READ_WRITE
56 || rcStrict == VINF_IOM_R3_MMIO_WRITE
57 || rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE
58 || rcStrict == VINF_CPUM_R3_MSR_READ
59 || rcStrict == VINF_CPUM_R3_MSR_WRITE
60 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
61 || rcStrict == VINF_EM_RAW_TO_R3
62 || rcStrict == VINF_EM_TRIPLE_FAULT
63 || rcStrict == VINF_GIM_R3_HYPERCALL
64 /* raw-mode / virt handlers only: */
65 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT
66 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT
67 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT
68 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT
69 || rcStrict == VINF_SELM_SYNC_GDT
70 || rcStrict == VINF_CSAM_PENDING_ACTION
71 || rcStrict == VINF_PATM_CHECK_PATCH_PAGE
72 /* nested hw.virt codes: */
73 || rcStrict == VINF_VMX_VMEXIT
74 || rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE
75 || rcStrict == VINF_VMX_MODIFIES_BEHAVIOR
76 || rcStrict == VINF_SVM_VMEXIT
77 , ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
78/** @todo adjust for VINF_EM_RAW_EMULATE_INSTR. */
79 int32_t const rcPassUp = pVCpu->iem.s.rcPassUp;
80#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
81 if ( rcStrict == VINF_VMX_VMEXIT
82 && rcPassUp == VINF_SUCCESS)
83 rcStrict = VINF_SUCCESS;
84 else
85#endif
86#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
87 if ( rcStrict == VINF_SVM_VMEXIT
88 && rcPassUp == VINF_SUCCESS)
89 rcStrict = VINF_SUCCESS;
90 else
91#endif
92 if (rcPassUp == VINF_SUCCESS)
93 pVCpu->iem.s.cRetInfStatuses++;
94 else if ( rcPassUp < VINF_EM_FIRST
95 || rcPassUp > VINF_EM_LAST
96 || rcPassUp < VBOXSTRICTRC_VAL(rcStrict))
97 {
98 Log(("IEM: rcPassUp=%Rrc! rcStrict=%Rrc\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
99 pVCpu->iem.s.cRetPassUpStatus++;
100 rcStrict = rcPassUp;
101 }
102 else
103 {
104 Log(("IEM: rcPassUp=%Rrc rcStrict=%Rrc!\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
105 pVCpu->iem.s.cRetInfStatuses++;
106 }
107 }
108 else if (rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED)
109 pVCpu->iem.s.cRetAspectNotImplemented++;
110 else if (rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
111 pVCpu->iem.s.cRetInstrNotImplemented++;
112 else
113 pVCpu->iem.s.cRetErrStatuses++;
114 }
115 else if (pVCpu->iem.s.rcPassUp != VINF_SUCCESS)
116 {
117 pVCpu->iem.s.cRetPassUpStatus++;
118 rcStrict = pVCpu->iem.s.rcPassUp;
119 }
120
121 return rcStrict;
122}
123
124
125/**
126 * Sets the pass up status.
127 *
128 * @returns VINF_SUCCESS.
129 * @param pVCpu The cross context virtual CPU structure of the
130 * calling thread.
131 * @param rcPassUp The pass up status. Must be informational.
132 * VINF_SUCCESS is not allowed.
133 */
134DECLINLINE(int) iemSetPassUpStatus(PVMCPUCC pVCpu, VBOXSTRICTRC rcPassUp) RT_NOEXCEPT
135{
136 AssertRC(VBOXSTRICTRC_VAL(rcPassUp)); Assert(rcPassUp != VINF_SUCCESS);
137
138 int32_t const rcOldPassUp = pVCpu->iem.s.rcPassUp;
139 if (rcOldPassUp == VINF_SUCCESS)
140 pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
141 /* If both are EM scheduling codes, use EM priority rules. */
142 else if ( rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST
143 && rcPassUp >= VINF_EM_FIRST && rcPassUp <= VINF_EM_LAST)
144 {
145 if (rcPassUp < rcOldPassUp)
146 {
147 Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
148 pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
149 }
150 else
151 Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
152 }
153 /* Override EM scheduling with specific status code. */
154 else if (rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST)
155 {
156 Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
157 pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
158 }
159 /* Don't override specific status code, first come first served. */
160 else
161 Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
162 return VINF_SUCCESS;
163}
164
165
166/**
167 * Calculates the IEM_F_MODE_X86_32BIT_FLAT flag.
168 *
169 * Checks if CS, SS, DS and SS are all wide open flat 32-bit segments. This will
170 * reject expand down data segments and conforming code segments.
171 *
172 * ASSUMES that the CPU is in 32-bit mode.
173 *
174 * @returns IEM_F_MODE_X86_32BIT_FLAT or zero.
175 * @param pVCpu The cross context virtual CPU structure of the
176 * calling thread.
177 * @sa iemCalc32BitFlatIndicatorEsDs
178 */
179DECL_FORCE_INLINE(uint32_t) iemCalc32BitFlatIndicator(PVMCPUCC pVCpu) RT_NOEXCEPT
180{
181 AssertCompile(X86_SEL_TYPE_DOWN == X86_SEL_TYPE_CONF);
182 return ( ( pVCpu->cpum.GstCtx.es.Attr.u
183 | pVCpu->cpum.GstCtx.cs.Attr.u
184 | pVCpu->cpum.GstCtx.ss.Attr.u
185 | pVCpu->cpum.GstCtx.ds.Attr.u)
186 & (X86_SEL_TYPE_ACCESSED | X86_SEL_TYPE_DOWN | X86DESCATTR_UNUSABLE | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_P))
187 == (X86_SEL_TYPE_ACCESSED | X86_SEL_TYPE_DOWN | X86DESCATTR_UNUSABLE | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_P)
188 && ( (pVCpu->cpum.GstCtx.es.u32Limit + 1)
189 | (pVCpu->cpum.GstCtx.cs.u32Limit + 1)
190 | (pVCpu->cpum.GstCtx.ss.u32Limit + 1)
191 | (pVCpu->cpum.GstCtx.ds.u32Limit + 1))
192 == 0
193 && ( pVCpu->cpum.GstCtx.es.u64Base
194 | pVCpu->cpum.GstCtx.cs.u64Base
195 | pVCpu->cpum.GstCtx.ss.u64Base
196 | pVCpu->cpum.GstCtx.ds.u64Base)
197 == 0
198 && !(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ES | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_ES))
199 ? IEM_F_MODE_X86_32BIT_FLAT : 0;
200}
201
202
203/**
204 * Calculates the IEM_F_MODE_X86_32BIT_FLAT flag, ASSUMING the CS and SS are
205 * flat already.
206 *
207 * This is used by sysenter.
208 *
209 * @returns IEM_F_MODE_X86_32BIT_FLAT or zero.
210 * @param pVCpu The cross context virtual CPU structure of the
211 * calling thread.
212 * @sa iemCalc32BitFlatIndicator
213 */
214DECL_FORCE_INLINE(uint32_t) iemCalc32BitFlatIndicatorEsDs(PVMCPUCC pVCpu) RT_NOEXCEPT
215{
216 AssertCompile(X86_SEL_TYPE_DOWN == X86_SEL_TYPE_CONF);
217 return ( ( pVCpu->cpum.GstCtx.es.Attr.u
218 | pVCpu->cpum.GstCtx.ds.Attr.u)
219 & (X86_SEL_TYPE_ACCESSED | X86_SEL_TYPE_DOWN | X86DESCATTR_UNUSABLE | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_P))
220 == (X86_SEL_TYPE_ACCESSED | X86_SEL_TYPE_DOWN | X86DESCATTR_UNUSABLE | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_P)
221 && ( (pVCpu->cpum.GstCtx.es.u32Limit + 1)
222 | (pVCpu->cpum.GstCtx.ds.u32Limit + 1))
223 == 0
224 && ( pVCpu->cpum.GstCtx.es.u64Base
225 | pVCpu->cpum.GstCtx.ds.u64Base)
226 == 0
227 && !(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ES | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_ES))
228 ? IEM_F_MODE_X86_32BIT_FLAT : 0;
229}
230
231
232/**
233 * Calculates the IEM_F_MODE_XXX and CPL flags.
234 *
235 * @returns IEM_F_MODE_XXX
236 * @param pVCpu The cross context virtual CPU structure of the
237 * calling thread.
238 */
239DECL_FORCE_INLINE(uint32_t) iemCalcExecModeAndCplFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
240{
241 /*
242 * We're duplicates code from CPUMGetGuestCPL and CPUMIsGuestIn64BitCodeEx
243 * here to try get this done as efficiently as possible.
244 */
245 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_EFER | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_CS);
246
247 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)
248 {
249 if (!pVCpu->cpum.GstCtx.eflags.Bits.u1VM)
250 {
251 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
252 uint32_t fExec = ((uint32_t)pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl << IEM_F_X86_CPL_SHIFT);
253 if (pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig)
254 {
255 Assert(!pVCpu->cpum.GstCtx.cs.Attr.n.u1Long || !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA));
256 fExec |= IEM_F_MODE_X86_32BIT_PROT | iemCalc32BitFlatIndicator(pVCpu);
257 }
258 else if ( pVCpu->cpum.GstCtx.cs.Attr.n.u1Long
259 && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA))
260 fExec |= IEM_F_MODE_X86_64BIT;
261 else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
262 fExec |= IEM_F_MODE_X86_16BIT_PROT;
263 else
264 fExec |= IEM_F_MODE_X86_16BIT_PROT_PRE_386;
265 return fExec;
266 }
267 return IEM_F_MODE_X86_16BIT_PROT_V86 | (UINT32_C(3) << IEM_F_X86_CPL_SHIFT);
268 }
269
270 /* Real mode is zero; CPL set to 3 for VT-x real-mode emulation. */
271 if (RT_LIKELY(!pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig))
272 {
273 if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
274 return IEM_F_MODE_X86_16BIT;
275 return IEM_F_MODE_X86_16BIT_PRE_386;
276 }
277
278 /* 32-bit unreal mode. */
279 return IEM_F_MODE_X86_32BIT | iemCalc32BitFlatIndicator(pVCpu);
280}
281
282
283/**
284 * Calculates the AMD-V and VT-x related context flags.
285 *
286 * @returns 0 or a combination of IEM_F_X86_CTX_IN_GUEST, IEM_F_X86_CTX_SVM and
287 * IEM_F_X86_CTX_VMX.
288 * @param pVCpu The cross context virtual CPU structure of the
289 * calling thread.
290 */
291DECL_FORCE_INLINE(uint32_t) iemCalcExecHwVirtFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
292{
293 /*
294 * This duplicates code from CPUMIsGuestVmxEnabled, CPUMIsGuestSvmEnabled
295 * and CPUMIsGuestInNestedHwvirtMode to some extent.
296 */
297 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_EFER);
298
299 AssertCompile(X86_CR4_VMXE != MSR_K6_EFER_SVME);
300 uint64_t const fTmp = (pVCpu->cpum.GstCtx.cr4 & X86_CR4_VMXE)
301 | (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_SVME);
302 if (RT_LIKELY(!fTmp))
303 return 0; /* likely */
304
305 if (fTmp & X86_CR4_VMXE)
306 {
307 Assert(pVCpu->cpum.GstCtx.hwvirt.enmHwvirt == CPUMHWVIRT_VMX);
308 if (pVCpu->cpum.GstCtx.hwvirt.vmx.fInVmxNonRootMode)
309 return IEM_F_X86_CTX_VMX | IEM_F_X86_CTX_IN_GUEST;
310 return IEM_F_X86_CTX_VMX;
311 }
312
313 Assert(pVCpu->cpum.GstCtx.hwvirt.enmHwvirt == CPUMHWVIRT_SVM);
314 if (pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_VMRUN)
315 return IEM_F_X86_CTX_SVM | IEM_F_X86_CTX_IN_GUEST;
316 return IEM_F_X86_CTX_SVM;
317}
318
319
320/**
321 * Calculates IEM_F_BRK_PENDING_XXX (IEM_F_PENDING_BRK_MASK) flags.
322 *
323 * @returns IEM_F_BRK_PENDING_XXX or zero.
324 * @param pVCpu The cross context virtual CPU structure of the
325 * calling thread.
326 */
327DECL_FORCE_INLINE(uint32_t) iemCalcExecDbgFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
328{
329 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
330
331 if (RT_LIKELY( !(pVCpu->cpum.GstCtx.dr[7] & X86_DR7_ENABLED_MASK)
332 && pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledHwBreakpoints == 0))
333 return 0;
334 return iemCalcExecDbgFlagsSlow(pVCpu);
335}
336
337/**
338 * Calculates the the IEM_F_XXX flags.
339 *
340 * @returns IEM_F_XXX combination match the current CPU state.
341 * @param pVCpu The cross context virtual CPU structure of the
342 * calling thread.
343 */
344DECL_FORCE_INLINE(uint32_t) iemCalcExecFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
345{
346 return iemCalcExecModeAndCplFlags(pVCpu)
347 | iemCalcExecHwVirtFlags(pVCpu)
348 /* SMM is not yet implemented */
349 | iemCalcExecDbgFlags(pVCpu)
350 ;
351}
352
353
354/**
355 * Re-calculates the MODE and CPL parts of IEMCPU::fExec.
356 *
357 * @param pVCpu The cross context virtual CPU structure of the
358 * calling thread.
359 */
360DECL_FORCE_INLINE(void) iemRecalcExecModeAndCplFlags(PVMCPUCC pVCpu)
361{
362 pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~(IEM_F_MODE_MASK | IEM_F_X86_CPL_MASK))
363 | iemCalcExecModeAndCplFlags(pVCpu);
364}
365
366
367/**
368 * Re-calculates the IEM_F_PENDING_BRK_MASK part of IEMCPU::fExec.
369 *
370 * @param pVCpu The cross context virtual CPU structure of the
371 * calling thread.
372 */
373DECL_FORCE_INLINE(void) iemRecalcExecDbgFlags(PVMCPUCC pVCpu)
374{
375 pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~IEM_F_PENDING_BRK_MASK)
376 | iemCalcExecDbgFlags(pVCpu);
377}
378
379
380#ifndef IEM_WITH_OPAQUE_DECODER_STATE
381
382# if defined(VBOX_INCLUDED_vmm_dbgf_h) || defined(DOXYGEN_RUNNING) /* dbgf.ro.cEnabledHwBreakpoints */
383/**
384 * Initializes the execution state.
385 *
386 * @param pVCpu The cross context virtual CPU structure of the
387 * calling thread.
388 * @param fExecOpts Optional execution flags:
389 * - IEM_F_BYPASS_HANDLERS
390 * - IEM_F_X86_DISREGARD_LOCK
391 *
392 * @remarks Callers of this must call iemUninitExec() to undo potentially fatal
393 * side-effects in strict builds.
394 */
395DECLINLINE(void) iemInitExec(PVMCPUCC pVCpu, uint32_t fExecOpts) RT_NOEXCEPT
396{
397 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
398 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
399 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
400 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
401 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
402 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
403 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
404 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
405 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
406 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
407
408 pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
409 pVCpu->iem.s.fExec = iemCalcExecFlags(pVCpu) | fExecOpts;
410 pVCpu->iem.s.cActiveMappings = 0;
411 pVCpu->iem.s.iNextMapping = 0;
412
413# ifdef VBOX_STRICT
414 pVCpu->iem.s.enmDefAddrMode = (IEMMODE)0xfe;
415 pVCpu->iem.s.enmEffAddrMode = (IEMMODE)0xfe;
416 pVCpu->iem.s.enmDefOpSize = (IEMMODE)0xfe;
417 pVCpu->iem.s.enmEffOpSize = (IEMMODE)0xfe;
418 pVCpu->iem.s.fPrefixes = 0xfeedbeef;
419 pVCpu->iem.s.uRexReg = 127;
420 pVCpu->iem.s.uRexB = 127;
421 pVCpu->iem.s.offModRm = 127;
422 pVCpu->iem.s.uRexIndex = 127;
423 pVCpu->iem.s.iEffSeg = 127;
424 pVCpu->iem.s.idxPrefix = 127;
425 pVCpu->iem.s.uVex3rdReg = 127;
426 pVCpu->iem.s.uVexLength = 127;
427 pVCpu->iem.s.fEvexStuff = 127;
428 pVCpu->iem.s.uFpuOpcode = UINT16_MAX;
429# ifdef IEM_WITH_CODE_TLB
430 pVCpu->iem.s.offInstrNextByte = UINT16_MAX;
431 pVCpu->iem.s.pbInstrBuf = NULL;
432 pVCpu->iem.s.cbInstrBuf = UINT16_MAX;
433 pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;
434 pVCpu->iem.s.offCurInstrStart = INT16_MAX;
435 pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);
436# else
437 pVCpu->iem.s.offOpcode = 127;
438 pVCpu->iem.s.cbOpcode = 127;
439# endif
440# endif /* VBOX_STRICT */
441}
442# endif /* VBOX_INCLUDED_vmm_dbgf_h */
443
444
445# if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
446/**
447 * Performs a minimal reinitialization of the execution state.
448 *
449 * This is intended to be used by VM-exits, SMM, LOADALL and other similar
450 * 'world-switch' types operations on the CPU. Currently only nested
451 * hardware-virtualization uses it.
452 *
453 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
454 * @param cbInstr The instruction length (for flushing).
455 */
456DECLINLINE(void) iemReInitExec(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
457{
458 pVCpu->iem.s.fExec = iemCalcExecFlags(pVCpu) | (pVCpu->iem.s.fExec & IEM_F_USER_OPTS);
459 iemOpcodeFlushHeavy(pVCpu, cbInstr);
460}
461# endif
462
463
464/**
465 * Counterpart to #iemInitExec that undoes evil strict-build stuff.
466 *
467 * @param pVCpu The cross context virtual CPU structure of the
468 * calling thread.
469 */
470DECLINLINE(void) iemUninitExec(PVMCPUCC pVCpu) RT_NOEXCEPT
471{
472 /* Note! do not touch fInPatchCode here! (see iemUninitExecAndFiddleStatusAndMaybeReenter) */
473# ifdef VBOX_STRICT
474# ifdef IEM_WITH_CODE_TLB
475 NOREF(pVCpu);
476# else
477 pVCpu->iem.s.cbOpcode = 0;
478# endif
479# else
480 NOREF(pVCpu);
481# endif
482}
483
484
485/**
486 * Calls iemUninitExec, iemExecStatusCodeFiddling and iemRCRawMaybeReenter.
487 *
488 * Only calling iemRCRawMaybeReenter in raw-mode, obviously.
489 *
490 * @returns Fiddled strict vbox status code, ready to return to non-IEM caller.
491 * @param pVCpu The cross context virtual CPU structure of the calling thread.
492 * @param rcStrict The status code to fiddle.
493 */
494DECLINLINE(VBOXSTRICTRC) iemUninitExecAndFiddleStatusAndMaybeReenter(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
495{
496 iemUninitExec(pVCpu);
497 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
498}
499
500
501/**
502 * Macro used by the IEMExec* method to check the given instruction length.
503 *
504 * Will return on failure!
505 *
506 * @param a_cbInstr The given instruction length.
507 * @param a_cbMin The minimum length.
508 */
509# define IEMEXEC_ASSERT_INSTR_LEN_RETURN(a_cbInstr, a_cbMin) \
510 AssertMsgReturn((unsigned)(a_cbInstr) - (unsigned)(a_cbMin) <= (unsigned)15 - (unsigned)(a_cbMin), \
511 ("cbInstr=%u cbMin=%u\n", (a_cbInstr), (a_cbMin)), VERR_IEM_INVALID_INSTR_LENGTH)
512
513
514# ifndef IEM_WITH_SETJMP
515
516/**
517 * Fetches the first opcode byte.
518 *
519 * @returns Strict VBox status code.
520 * @param pVCpu The cross context virtual CPU structure of the
521 * calling thread.
522 * @param pu8 Where to return the opcode byte.
523 */
524DECLINLINE(VBOXSTRICTRC) iemOpcodeGetFirstU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
525{
526 /*
527 * Check for hardware instruction breakpoints.
528 */
529 if (RT_LIKELY(!(pVCpu->iem.s.fExec & IEM_F_PENDING_BRK_INSTR)))
530 { /* likely */ }
531 else
532 {
533 VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu,
534 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
535 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
536 { /* likely */ }
537 else if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
538 return iemRaiseDebugException(pVCpu);
539 else
540 return rcStrict;
541 }
542
543 /*
544 * Fetch the first opcode byte.
545 */
546 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
547 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
548 {
549 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
550 *pu8 = pVCpu->iem.s.abOpcode[offOpcode];
551 return VINF_SUCCESS;
552 }
553 return iemOpcodeGetNextU8Slow(pVCpu, pu8);
554}
555
556# else /* IEM_WITH_SETJMP */
557
558/**
559 * Fetches the first opcode byte, longjmp on error.
560 *
561 * @returns The opcode byte.
562 * @param pVCpu The cross context virtual CPU structure of the calling thread.
563 */
564DECL_INLINE_THROW(uint8_t) iemOpcodeGetFirstU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
565{
566 /*
567 * Check for hardware instruction breakpoints.
568 */
569 if (RT_LIKELY(!(pVCpu->iem.s.fExec & IEM_F_PENDING_BRK_INSTR)))
570 { /* likely */ }
571 else
572 {
573 VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu,
574 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
575 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
576 { /* likely */ }
577 else
578 {
579 if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
580 rcStrict = iemRaiseDebugException(pVCpu);
581 IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
582 }
583 }
584
585 /*
586 * Fetch the first opcode byte.
587 */
588# ifdef IEM_WITH_CODE_TLB
589 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
590 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
591 if (RT_LIKELY( pbBuf != NULL
592 && offBuf < pVCpu->iem.s.cbInstrBuf))
593 {
594 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
595 return pbBuf[offBuf];
596 }
597# else
598 uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
599 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
600 {
601 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
602 return pVCpu->iem.s.abOpcode[offOpcode];
603 }
604# endif
605 return iemOpcodeGetNextU8SlowJmp(pVCpu);
606}
607
608# endif /* IEM_WITH_SETJMP */
609
610/**
611 * Fetches the first opcode byte, returns/throws automatically on failure.
612 *
613 * @param a_pu8 Where to return the opcode byte.
614 * @remark Implicitly references pVCpu.
615 */
616# ifndef IEM_WITH_SETJMP
617# define IEM_OPCODE_GET_FIRST_U8(a_pu8) \
618 do \
619 { \
620 VBOXSTRICTRC rcStrict2 = iemOpcodeGetFirstU8(pVCpu, (a_pu8)); \
621 if (rcStrict2 == VINF_SUCCESS) \
622 { /* likely */ } \
623 else \
624 return rcStrict2; \
625 } while (0)
626# else
627# define IEM_OPCODE_GET_FIRST_U8(a_pu8) (*(a_pu8) = iemOpcodeGetFirstU8Jmp(pVCpu))
628# endif /* IEM_WITH_SETJMP */
629
630
631# ifndef IEM_WITH_SETJMP
632
633/**
634 * Fetches the next opcode byte.
635 *
636 * @returns Strict VBox status code.
637 * @param pVCpu The cross context virtual CPU structure of the
638 * calling thread.
639 * @param pu8 Where to return the opcode byte.
640 */
641DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
642{
643 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
644 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
645 {
646 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
647 *pu8 = pVCpu->iem.s.abOpcode[offOpcode];
648 return VINF_SUCCESS;
649 }
650 return iemOpcodeGetNextU8Slow(pVCpu, pu8);
651}
652
653# else /* IEM_WITH_SETJMP */
654
655/**
656 * Fetches the next opcode byte, longjmp on error.
657 *
658 * @returns The opcode byte.
659 * @param pVCpu The cross context virtual CPU structure of the calling thread.
660 */
661DECL_INLINE_THROW(uint8_t) iemOpcodeGetNextU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
662{
663# ifdef IEM_WITH_CODE_TLB
664 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
665 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
666 if (RT_LIKELY( pbBuf != NULL
667 && offBuf < pVCpu->iem.s.cbInstrBuf))
668 {
669 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
670 return pbBuf[offBuf];
671 }
672# else
673 uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
674 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
675 {
676 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
677 return pVCpu->iem.s.abOpcode[offOpcode];
678 }
679# endif
680 return iemOpcodeGetNextU8SlowJmp(pVCpu);
681}
682
683# endif /* IEM_WITH_SETJMP */
684
685/**
686 * Fetches the next opcode byte, returns automatically on failure.
687 *
688 * @param a_pu8 Where to return the opcode byte.
689 * @remark Implicitly references pVCpu.
690 */
691# ifndef IEM_WITH_SETJMP
692# define IEM_OPCODE_GET_NEXT_U8(a_pu8) \
693 do \
694 { \
695 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pVCpu, (a_pu8)); \
696 if (rcStrict2 == VINF_SUCCESS) \
697 { /* likely */ } \
698 else \
699 return rcStrict2; \
700 } while (0)
701# else
702# define IEM_OPCODE_GET_NEXT_U8(a_pu8) (*(a_pu8) = iemOpcodeGetNextU8Jmp(pVCpu))
703# endif /* IEM_WITH_SETJMP */
704
705
706# ifndef IEM_WITH_SETJMP
707/**
708 * Fetches the next signed byte from the opcode stream.
709 *
710 * @returns Strict VBox status code.
711 * @param pVCpu The cross context virtual CPU structure of the calling thread.
712 * @param pi8 Where to return the signed byte.
713 */
714DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PVMCPUCC pVCpu, int8_t *pi8) RT_NOEXCEPT
715{
716 return iemOpcodeGetNextU8(pVCpu, (uint8_t *)pi8);
717}
718# endif /* !IEM_WITH_SETJMP */
719
720
721/**
722 * Fetches the next signed byte from the opcode stream, returning automatically
723 * on failure.
724 *
725 * @param a_pi8 Where to return the signed byte.
726 * @remark Implicitly references pVCpu.
727 */
728# ifndef IEM_WITH_SETJMP
729# define IEM_OPCODE_GET_NEXT_S8(a_pi8) \
730 do \
731 { \
732 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pVCpu, (a_pi8)); \
733 if (rcStrict2 != VINF_SUCCESS) \
734 return rcStrict2; \
735 } while (0)
736# else /* IEM_WITH_SETJMP */
737# define IEM_OPCODE_GET_NEXT_S8(a_pi8) (*(a_pi8) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
738
739# endif /* IEM_WITH_SETJMP */
740
741
742# ifndef IEM_WITH_SETJMP
743/**
744 * Fetches the next signed byte from the opcode stream, extending it to
745 * unsigned 16-bit.
746 *
747 * @returns Strict VBox status code.
748 * @param pVCpu The cross context virtual CPU structure of the calling thread.
749 * @param pu16 Where to return the unsigned word.
750 */
751DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
752{
753 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
754 if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
755 return iemOpcodeGetNextS8SxU16Slow(pVCpu, pu16);
756
757 *pu16 = (uint16_t)(int16_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];
758 pVCpu->iem.s.offOpcode = offOpcode + 1;
759 return VINF_SUCCESS;
760}
761# endif /* !IEM_WITH_SETJMP */
762
763/**
764 * Fetches the next signed byte from the opcode stream and sign-extending it to
765 * a word, returning automatically on failure.
766 *
767 * @param a_pu16 Where to return the word.
768 * @remark Implicitly references pVCpu.
769 */
770# ifndef IEM_WITH_SETJMP
771# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \
772 do \
773 { \
774 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pVCpu, (a_pu16)); \
775 if (rcStrict2 != VINF_SUCCESS) \
776 return rcStrict2; \
777 } while (0)
778# else
779# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) (*(a_pu16) = (uint16_t)(int16_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
780# endif
781
782# ifndef IEM_WITH_SETJMP
783/**
784 * Fetches the next signed byte from the opcode stream, extending it to
785 * unsigned 32-bit.
786 *
787 * @returns Strict VBox status code.
788 * @param pVCpu The cross context virtual CPU structure of the calling thread.
789 * @param pu32 Where to return the unsigned dword.
790 */
791DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
792{
793 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
794 if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
795 return iemOpcodeGetNextS8SxU32Slow(pVCpu, pu32);
796
797 *pu32 = (uint32_t)(int32_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];
798 pVCpu->iem.s.offOpcode = offOpcode + 1;
799 return VINF_SUCCESS;
800}
801# endif /* !IEM_WITH_SETJMP */
802
803/**
804 * Fetches the next signed byte from the opcode stream and sign-extending it to
805 * a word, returning automatically on failure.
806 *
807 * @param a_pu32 Where to return the word.
808 * @remark Implicitly references pVCpu.
809 */
810# ifndef IEM_WITH_SETJMP
811# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) \
812 do \
813 { \
814 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU32(pVCpu, (a_pu32)); \
815 if (rcStrict2 != VINF_SUCCESS) \
816 return rcStrict2; \
817 } while (0)
818# else
819# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) (*(a_pu32) = (uint32_t)(int32_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
820# endif
821
822
823# ifndef IEM_WITH_SETJMP
824/**
825 * Fetches the next signed byte from the opcode stream, extending it to
826 * unsigned 64-bit.
827 *
828 * @returns Strict VBox status code.
829 * @param pVCpu The cross context virtual CPU structure of the calling thread.
830 * @param pu64 Where to return the unsigned qword.
831 */
832DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
833{
834 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
835 if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
836 return iemOpcodeGetNextS8SxU64Slow(pVCpu, pu64);
837
838 *pu64 = (uint64_t)(int64_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];
839 pVCpu->iem.s.offOpcode = offOpcode + 1;
840 return VINF_SUCCESS;
841}
842# endif /* !IEM_WITH_SETJMP */
843
844/**
845 * Fetches the next signed byte from the opcode stream and sign-extending it to
846 * a word, returning automatically on failure.
847 *
848 * @param a_pu64 Where to return the word.
849 * @remark Implicitly references pVCpu.
850 */
851# ifndef IEM_WITH_SETJMP
852# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) \
853 do \
854 { \
855 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU64(pVCpu, (a_pu64)); \
856 if (rcStrict2 != VINF_SUCCESS) \
857 return rcStrict2; \
858 } while (0)
859# else
860# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) (*(a_pu64) = (uint64_t)(int64_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
861# endif
862
863
864# ifndef IEM_WITH_SETJMP
865/**
866 * Fetches the next opcode byte.
867 *
868 * @returns Strict VBox status code.
869 * @param pVCpu The cross context virtual CPU structure of the
870 * calling thread.
871 * @param pu8 Where to return the opcode byte.
872 */
873DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextRm(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
874{
875 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
876 pVCpu->iem.s.offModRm = offOpcode;
877 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
878 {
879 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
880 *pu8 = pVCpu->iem.s.abOpcode[offOpcode];
881 return VINF_SUCCESS;
882 }
883 return iemOpcodeGetNextU8Slow(pVCpu, pu8);
884}
885# else /* IEM_WITH_SETJMP */
886/**
887 * Fetches the next opcode byte, longjmp on error.
888 *
889 * @returns The opcode byte.
890 * @param pVCpu The cross context virtual CPU structure of the calling thread.
891 */
892DECL_INLINE_THROW(uint8_t) iemOpcodeGetNextRmJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
893{
894# ifdef IEM_WITH_CODE_TLB
895 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
896 pVCpu->iem.s.offModRm = offBuf;
897 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
898 if (RT_LIKELY( pbBuf != NULL
899 && offBuf < pVCpu->iem.s.cbInstrBuf))
900 {
901 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
902 return pbBuf[offBuf];
903 }
904# else
905 uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
906 pVCpu->iem.s.offModRm = offOpcode;
907 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
908 {
909 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
910 return pVCpu->iem.s.abOpcode[offOpcode];
911 }
912# endif
913 return iemOpcodeGetNextU8SlowJmp(pVCpu);
914}
915# endif /* IEM_WITH_SETJMP */
916
917/**
918 * Fetches the next opcode byte, which is a ModR/M byte, returns automatically
919 * on failure.
920 *
921 * Will note down the position of the ModR/M byte for VT-x exits.
922 *
923 * @param a_pbRm Where to return the RM opcode byte.
924 * @remark Implicitly references pVCpu.
925 */
926# ifndef IEM_WITH_SETJMP
927# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) \
928 do \
929 { \
930 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextRm(pVCpu, (a_pbRm)); \
931 if (rcStrict2 == VINF_SUCCESS) \
932 { /* likely */ } \
933 else \
934 return rcStrict2; \
935 } while (0)
936# else
937# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) (*(a_pbRm) = iemOpcodeGetNextRmJmp(pVCpu))
938# endif /* IEM_WITH_SETJMP */
939
940
941# ifndef IEM_WITH_SETJMP
942
943/**
944 * Fetches the next opcode word.
945 *
946 * @returns Strict VBox status code.
947 * @param pVCpu The cross context virtual CPU structure of the calling thread.
948 * @param pu16 Where to return the opcode word.
949 */
950DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
951{
952 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
953 if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
954 {
955 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
956# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
957 *pu16 = *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
958# else
959 *pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
960# endif
961 return VINF_SUCCESS;
962 }
963 return iemOpcodeGetNextU16Slow(pVCpu, pu16);
964}
965
966# else /* IEM_WITH_SETJMP */
967
968/**
969 * Fetches the next opcode word, longjmp on error.
970 *
971 * @returns The opcode word.
972 * @param pVCpu The cross context virtual CPU structure of the calling thread.
973 */
974DECL_INLINE_THROW(uint16_t) iemOpcodeGetNextU16Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
975{
976# ifdef IEM_WITH_CODE_TLB
977 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
978 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
979 if (RT_LIKELY( pbBuf != NULL
980 && offBuf + 2 <= pVCpu->iem.s.cbInstrBuf))
981 {
982 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 2;
983# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
984 return *(uint16_t const *)&pbBuf[offBuf];
985# else
986 return RT_MAKE_U16(pbBuf[offBuf], pbBuf[offBuf + 1]);
987# endif
988 }
989# else /* !IEM_WITH_CODE_TLB */
990 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
991 if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
992 {
993 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
994# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
995 return *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
996# else
997 return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
998# endif
999 }
1000# endif /* !IEM_WITH_CODE_TLB */
1001 return iemOpcodeGetNextU16SlowJmp(pVCpu);
1002}
1003
1004# endif /* IEM_WITH_SETJMP */
1005
1006/**
1007 * Fetches the next opcode word, returns automatically on failure.
1008 *
1009 * @param a_pu16 Where to return the opcode word.
1010 * @remark Implicitly references pVCpu.
1011 */
1012# ifndef IEM_WITH_SETJMP
1013# define IEM_OPCODE_GET_NEXT_U16(a_pu16) \
1014 do \
1015 { \
1016 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pVCpu, (a_pu16)); \
1017 if (rcStrict2 != VINF_SUCCESS) \
1018 return rcStrict2; \
1019 } while (0)
1020# else
1021# define IEM_OPCODE_GET_NEXT_U16(a_pu16) (*(a_pu16) = iemOpcodeGetNextU16Jmp(pVCpu))
1022# endif
1023
1024# ifndef IEM_WITH_SETJMP
1025/**
1026 * Fetches the next opcode word, zero extending it to a double word.
1027 *
1028 * @returns Strict VBox status code.
1029 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1030 * @param pu32 Where to return the opcode double word.
1031 */
1032DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1033{
1034 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1035 if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
1036 return iemOpcodeGetNextU16ZxU32Slow(pVCpu, pu32);
1037
1038 *pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1039 pVCpu->iem.s.offOpcode = offOpcode + 2;
1040 return VINF_SUCCESS;
1041}
1042# endif /* !IEM_WITH_SETJMP */
1043
1044/**
1045 * Fetches the next opcode word and zero extends it to a double word, returns
1046 * automatically on failure.
1047 *
1048 * @param a_pu32 Where to return the opcode double word.
1049 * @remark Implicitly references pVCpu.
1050 */
1051# ifndef IEM_WITH_SETJMP
1052# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \
1053 do \
1054 { \
1055 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pVCpu, (a_pu32)); \
1056 if (rcStrict2 != VINF_SUCCESS) \
1057 return rcStrict2; \
1058 } while (0)
1059# else
1060# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU16Jmp(pVCpu))
1061# endif
1062
1063# ifndef IEM_WITH_SETJMP
1064/**
1065 * Fetches the next opcode word, zero extending it to a quad word.
1066 *
1067 * @returns Strict VBox status code.
1068 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1069 * @param pu64 Where to return the opcode quad word.
1070 */
1071DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1072{
1073 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1074 if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
1075 return iemOpcodeGetNextU16ZxU64Slow(pVCpu, pu64);
1076
1077 *pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1078 pVCpu->iem.s.offOpcode = offOpcode + 2;
1079 return VINF_SUCCESS;
1080}
1081# endif /* !IEM_WITH_SETJMP */
1082
1083/**
1084 * Fetches the next opcode word and zero extends it to a quad word, returns
1085 * automatically on failure.
1086 *
1087 * @param a_pu64 Where to return the opcode quad word.
1088 * @remark Implicitly references pVCpu.
1089 */
1090# ifndef IEM_WITH_SETJMP
1091# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \
1092 do \
1093 { \
1094 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pVCpu, (a_pu64)); \
1095 if (rcStrict2 != VINF_SUCCESS) \
1096 return rcStrict2; \
1097 } while (0)
1098# else
1099# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU16Jmp(pVCpu))
1100# endif
1101
1102
1103# ifndef IEM_WITH_SETJMP
1104/**
1105 * Fetches the next signed word from the opcode stream.
1106 *
1107 * @returns Strict VBox status code.
1108 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1109 * @param pi16 Where to return the signed word.
1110 */
1111DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PVMCPUCC pVCpu, int16_t *pi16) RT_NOEXCEPT
1112{
1113 return iemOpcodeGetNextU16(pVCpu, (uint16_t *)pi16);
1114}
1115# endif /* !IEM_WITH_SETJMP */
1116
1117
1118/**
1119 * Fetches the next signed word from the opcode stream, returning automatically
1120 * on failure.
1121 *
1122 * @param a_pi16 Where to return the signed word.
1123 * @remark Implicitly references pVCpu.
1124 */
1125# ifndef IEM_WITH_SETJMP
1126# define IEM_OPCODE_GET_NEXT_S16(a_pi16) \
1127 do \
1128 { \
1129 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pVCpu, (a_pi16)); \
1130 if (rcStrict2 != VINF_SUCCESS) \
1131 return rcStrict2; \
1132 } while (0)
1133# else
1134# define IEM_OPCODE_GET_NEXT_S16(a_pi16) (*(a_pi16) = (int16_t)iemOpcodeGetNextU16Jmp(pVCpu))
1135# endif
1136
1137# ifndef IEM_WITH_SETJMP
1138
1139/**
1140 * Fetches the next opcode dword.
1141 *
1142 * @returns Strict VBox status code.
1143 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1144 * @param pu32 Where to return the opcode double word.
1145 */
1146DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1147{
1148 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1149 if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
1150 {
1151 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
1152# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1153 *pu32 = *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1154# else
1155 *pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1156 pVCpu->iem.s.abOpcode[offOpcode + 1],
1157 pVCpu->iem.s.abOpcode[offOpcode + 2],
1158 pVCpu->iem.s.abOpcode[offOpcode + 3]);
1159# endif
1160 return VINF_SUCCESS;
1161 }
1162 return iemOpcodeGetNextU32Slow(pVCpu, pu32);
1163}
1164
1165# else /* IEM_WITH_SETJMP */
1166
1167/**
1168 * Fetches the next opcode dword, longjmp on error.
1169 *
1170 * @returns The opcode dword.
1171 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1172 */
1173DECL_INLINE_THROW(uint32_t) iemOpcodeGetNextU32Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
1174{
1175# ifdef IEM_WITH_CODE_TLB
1176 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1177 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1178 if (RT_LIKELY( pbBuf != NULL
1179 && offBuf + 4 <= pVCpu->iem.s.cbInstrBuf))
1180 {
1181 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 4;
1182# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1183 return *(uint32_t const *)&pbBuf[offBuf];
1184# else
1185 return RT_MAKE_U32_FROM_U8(pbBuf[offBuf],
1186 pbBuf[offBuf + 1],
1187 pbBuf[offBuf + 2],
1188 pbBuf[offBuf + 3]);
1189# endif
1190 }
1191# else
1192 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1193 if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
1194 {
1195 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
1196# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1197 return *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1198# else
1199 return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1200 pVCpu->iem.s.abOpcode[offOpcode + 1],
1201 pVCpu->iem.s.abOpcode[offOpcode + 2],
1202 pVCpu->iem.s.abOpcode[offOpcode + 3]);
1203# endif
1204 }
1205# endif
1206 return iemOpcodeGetNextU32SlowJmp(pVCpu);
1207}
1208
1209# endif /* IEM_WITH_SETJMP */
1210
1211/**
1212 * Fetches the next opcode dword, returns automatically on failure.
1213 *
1214 * @param a_pu32 Where to return the opcode dword.
1215 * @remark Implicitly references pVCpu.
1216 */
1217# ifndef IEM_WITH_SETJMP
1218# define IEM_OPCODE_GET_NEXT_U32(a_pu32) \
1219 do \
1220 { \
1221 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pVCpu, (a_pu32)); \
1222 if (rcStrict2 != VINF_SUCCESS) \
1223 return rcStrict2; \
1224 } while (0)
1225# else
1226# define IEM_OPCODE_GET_NEXT_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU32Jmp(pVCpu))
1227# endif
1228
1229# ifndef IEM_WITH_SETJMP
1230/**
1231 * Fetches the next opcode dword, zero extending it to a quad word.
1232 *
1233 * @returns Strict VBox status code.
1234 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1235 * @param pu64 Where to return the opcode quad word.
1236 */
1237DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1238{
1239 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1240 if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1241 return iemOpcodeGetNextU32ZxU64Slow(pVCpu, pu64);
1242
1243 *pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1244 pVCpu->iem.s.abOpcode[offOpcode + 1],
1245 pVCpu->iem.s.abOpcode[offOpcode + 2],
1246 pVCpu->iem.s.abOpcode[offOpcode + 3]);
1247 pVCpu->iem.s.offOpcode = offOpcode + 4;
1248 return VINF_SUCCESS;
1249}
1250# endif /* !IEM_WITH_SETJMP */
1251
1252/**
1253 * Fetches the next opcode dword and zero extends it to a quad word, returns
1254 * automatically on failure.
1255 *
1256 * @param a_pu64 Where to return the opcode quad word.
1257 * @remark Implicitly references pVCpu.
1258 */
1259# ifndef IEM_WITH_SETJMP
1260# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \
1261 do \
1262 { \
1263 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pVCpu, (a_pu64)); \
1264 if (rcStrict2 != VINF_SUCCESS) \
1265 return rcStrict2; \
1266 } while (0)
1267# else
1268# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU32Jmp(pVCpu))
1269# endif
1270
1271
1272# ifndef IEM_WITH_SETJMP
1273/**
1274 * Fetches the next signed double word from the opcode stream.
1275 *
1276 * @returns Strict VBox status code.
1277 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1278 * @param pi32 Where to return the signed double word.
1279 */
1280DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PVMCPUCC pVCpu, int32_t *pi32) RT_NOEXCEPT
1281{
1282 return iemOpcodeGetNextU32(pVCpu, (uint32_t *)pi32);
1283}
1284# endif
1285
1286/**
1287 * Fetches the next signed double word from the opcode stream, returning
1288 * automatically on failure.
1289 *
1290 * @param a_pi32 Where to return the signed double word.
1291 * @remark Implicitly references pVCpu.
1292 */
1293# ifndef IEM_WITH_SETJMP
1294# define IEM_OPCODE_GET_NEXT_S32(a_pi32) \
1295 do \
1296 { \
1297 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pVCpu, (a_pi32)); \
1298 if (rcStrict2 != VINF_SUCCESS) \
1299 return rcStrict2; \
1300 } while (0)
1301# else
1302# define IEM_OPCODE_GET_NEXT_S32(a_pi32) (*(a_pi32) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1303# endif
1304
1305# ifndef IEM_WITH_SETJMP
1306/**
1307 * Fetches the next opcode dword, sign extending it into a quad word.
1308 *
1309 * @returns Strict VBox status code.
1310 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1311 * @param pu64 Where to return the opcode quad word.
1312 */
1313DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1314{
1315 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1316 if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1317 return iemOpcodeGetNextS32SxU64Slow(pVCpu, pu64);
1318
1319 int32_t i32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1320 pVCpu->iem.s.abOpcode[offOpcode + 1],
1321 pVCpu->iem.s.abOpcode[offOpcode + 2],
1322 pVCpu->iem.s.abOpcode[offOpcode + 3]);
1323 *pu64 = (uint64_t)(int64_t)i32;
1324 pVCpu->iem.s.offOpcode = offOpcode + 4;
1325 return VINF_SUCCESS;
1326}
1327# endif /* !IEM_WITH_SETJMP */
1328
1329/**
1330 * Fetches the next opcode double word and sign extends it to a quad word,
1331 * returns automatically on failure.
1332 *
1333 * @param a_pu64 Where to return the opcode quad word.
1334 * @remark Implicitly references pVCpu.
1335 */
1336# ifndef IEM_WITH_SETJMP
1337# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \
1338 do \
1339 { \
1340 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pVCpu, (a_pu64)); \
1341 if (rcStrict2 != VINF_SUCCESS) \
1342 return rcStrict2; \
1343 } while (0)
1344# else
1345# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) (*(a_pu64) = (uint64_t)(int64_t)(int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1346# endif
1347
1348# ifndef IEM_WITH_SETJMP
1349
1350/**
1351 * Fetches the next opcode qword.
1352 *
1353 * @returns Strict VBox status code.
1354 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1355 * @param pu64 Where to return the opcode qword.
1356 */
1357DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1358{
1359 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1360 if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1361 {
1362# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1363 *pu64 = *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1364# else
1365 *pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1366 pVCpu->iem.s.abOpcode[offOpcode + 1],
1367 pVCpu->iem.s.abOpcode[offOpcode + 2],
1368 pVCpu->iem.s.abOpcode[offOpcode + 3],
1369 pVCpu->iem.s.abOpcode[offOpcode + 4],
1370 pVCpu->iem.s.abOpcode[offOpcode + 5],
1371 pVCpu->iem.s.abOpcode[offOpcode + 6],
1372 pVCpu->iem.s.abOpcode[offOpcode + 7]);
1373# endif
1374 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1375 return VINF_SUCCESS;
1376 }
1377 return iemOpcodeGetNextU64Slow(pVCpu, pu64);
1378}
1379
1380# else /* IEM_WITH_SETJMP */
1381
1382/**
1383 * Fetches the next opcode qword, longjmp on error.
1384 *
1385 * @returns The opcode qword.
1386 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1387 */
1388DECL_INLINE_THROW(uint64_t) iemOpcodeGetNextU64Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
1389{
1390# ifdef IEM_WITH_CODE_TLB
1391 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1392 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1393 if (RT_LIKELY( pbBuf != NULL
1394 && offBuf + 8 <= pVCpu->iem.s.cbInstrBuf))
1395 {
1396 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 8;
1397# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1398 return *(uint64_t const *)&pbBuf[offBuf];
1399# else
1400 return RT_MAKE_U64_FROM_U8(pbBuf[offBuf],
1401 pbBuf[offBuf + 1],
1402 pbBuf[offBuf + 2],
1403 pbBuf[offBuf + 3],
1404 pbBuf[offBuf + 4],
1405 pbBuf[offBuf + 5],
1406 pbBuf[offBuf + 6],
1407 pbBuf[offBuf + 7]);
1408# endif
1409 }
1410# else
1411 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1412 if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1413 {
1414 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1415# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1416 return *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1417# else
1418 return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1419 pVCpu->iem.s.abOpcode[offOpcode + 1],
1420 pVCpu->iem.s.abOpcode[offOpcode + 2],
1421 pVCpu->iem.s.abOpcode[offOpcode + 3],
1422 pVCpu->iem.s.abOpcode[offOpcode + 4],
1423 pVCpu->iem.s.abOpcode[offOpcode + 5],
1424 pVCpu->iem.s.abOpcode[offOpcode + 6],
1425 pVCpu->iem.s.abOpcode[offOpcode + 7]);
1426# endif
1427 }
1428# endif
1429 return iemOpcodeGetNextU64SlowJmp(pVCpu);
1430}
1431
1432# endif /* IEM_WITH_SETJMP */
1433
1434/**
1435 * Fetches the next opcode quad word, returns automatically on failure.
1436 *
1437 * @param a_pu64 Where to return the opcode quad word.
1438 * @remark Implicitly references pVCpu.
1439 */
1440# ifndef IEM_WITH_SETJMP
1441# define IEM_OPCODE_GET_NEXT_U64(a_pu64) \
1442 do \
1443 { \
1444 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pVCpu, (a_pu64)); \
1445 if (rcStrict2 != VINF_SUCCESS) \
1446 return rcStrict2; \
1447 } while (0)
1448# else
1449# define IEM_OPCODE_GET_NEXT_U64(a_pu64) ( *(a_pu64) = iemOpcodeGetNextU64Jmp(pVCpu) )
1450# endif
1451
1452#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
1453
1454
1455/** @name Misc Worker Functions.
1456 * @{
1457 */
1458
1459/**
1460 * Gets the correct EFLAGS regardless of whether PATM stores parts of them or
1461 * not (kind of obsolete now).
1462 *
1463 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1464 */
1465#define IEMMISC_GET_EFL(a_pVCpu) ( (a_pVCpu)->cpum.GstCtx.eflags.u )
1466
1467/**
1468 * Updates the EFLAGS in the correct manner wrt. PATM (kind of obsolete).
1469 *
1470 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1471 * @param a_fEfl The new EFLAGS.
1472 */
1473#define IEMMISC_SET_EFL(a_pVCpu, a_fEfl) do { (a_pVCpu)->cpum.GstCtx.eflags.u = (a_fEfl); } while (0)
1474
1475
1476/**
1477 * Loads a NULL data selector into a selector register, both the hidden and
1478 * visible parts, in protected mode.
1479 *
1480 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1481 * @param pSReg Pointer to the segment register.
1482 * @param uRpl The RPL.
1483 */
1484DECLINLINE(void) iemHlpLoadNullDataSelectorProt(PVMCPUCC pVCpu, PCPUMSELREG pSReg, RTSEL uRpl) RT_NOEXCEPT
1485{
1486 /** @todo Testcase: write a testcase checking what happends when loading a NULL
1487 * data selector in protected mode. */
1488 pSReg->Sel = uRpl;
1489 pSReg->ValidSel = uRpl;
1490 pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
1491 if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
1492 {
1493 /* VT-x (Intel 3960x) observed doing something like this. */
1494 pSReg->Attr.u = X86DESCATTR_UNUSABLE | X86DESCATTR_G | X86DESCATTR_D | (IEM_GET_CPL(pVCpu) << X86DESCATTR_DPL_SHIFT);
1495 pSReg->u32Limit = UINT32_MAX;
1496 pSReg->u64Base = 0;
1497 }
1498 else
1499 {
1500 pSReg->Attr.u = X86DESCATTR_UNUSABLE;
1501 pSReg->u32Limit = 0;
1502 pSReg->u64Base = 0;
1503 }
1504}
1505
1506/** @} */
1507
1508
1509/*
1510 *
1511 * Helpers routines.
1512 * Helpers routines.
1513 * Helpers routines.
1514 *
1515 */
1516
1517#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1518
1519/**
1520 * Recalculates the effective operand size.
1521 *
1522 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1523 */
1524DECLINLINE(void) iemRecalEffOpSize(PVMCPUCC pVCpu) RT_NOEXCEPT
1525{
1526 switch (IEM_GET_CPU_MODE(pVCpu))
1527 {
1528 case IEMMODE_16BIT:
1529 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT;
1530 break;
1531 case IEMMODE_32BIT:
1532 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT;
1533 break;
1534 case IEMMODE_64BIT:
1535 switch (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP))
1536 {
1537 case 0:
1538 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize;
1539 break;
1540 case IEM_OP_PRF_SIZE_OP:
1541 pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1542 break;
1543 case IEM_OP_PRF_SIZE_REX_W:
1544 case IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP:
1545 pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1546 break;
1547 }
1548 break;
1549 default:
1550 AssertFailed();
1551 }
1552}
1553
1554
1555/**
1556 * Sets the default operand size to 64-bit and recalculates the effective
1557 * operand size.
1558 *
1559 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1560 */
1561DECLINLINE(void) iemRecalEffOpSize64Default(PVMCPUCC pVCpu) RT_NOEXCEPT
1562{
1563 Assert(IEM_IS_64BIT_CODE(pVCpu));
1564 pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1565 if ((pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP)
1566 pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1567 else
1568 pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1569}
1570
1571
1572/**
1573 * Sets the default operand size to 64-bit and recalculates the effective
1574 * operand size, with intel ignoring any operand size prefix (AMD respects it).
1575 *
1576 * This is for the relative jumps.
1577 *
1578 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1579 */
1580DECLINLINE(void) iemRecalEffOpSize64DefaultAndIntelIgnoresOpSizePrefix(PVMCPUCC pVCpu) RT_NOEXCEPT
1581{
1582 Assert(IEM_IS_64BIT_CODE(pVCpu));
1583 pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1584 if ( (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP
1585 || pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
1586 pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1587 else
1588 pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1589}
1590
1591#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
1592
1593
1594
1595/** @name Register Access.
1596 * @{
1597 */
1598
1599/**
1600 * Gets a reference (pointer) to the specified hidden segment register.
1601 *
1602 * @returns Hidden register reference.
1603 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1604 * @param iSegReg The segment register.
1605 */
1606DECL_FORCE_INLINE(PCPUMSELREG) iemSRegGetHid(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1607{
1608 Assert(iSegReg < X86_SREG_COUNT);
1609 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1610 PCPUMSELREG pSReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
1611
1612 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1613 return pSReg;
1614}
1615
1616
1617/**
1618 * Ensures that the given hidden segment register is up to date.
1619 *
1620 * @returns Hidden register reference.
1621 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1622 * @param pSReg The segment register.
1623 */
1624DECL_FORCE_INLINE(PCPUMSELREG) iemSRegUpdateHid(PVMCPUCC pVCpu, PCPUMSELREG pSReg) RT_NOEXCEPT
1625{
1626 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1627 NOREF(pVCpu);
1628 return pSReg;
1629}
1630
1631
1632/**
1633 * Gets a reference (pointer) to the specified segment register (the selector
1634 * value).
1635 *
1636 * @returns Pointer to the selector variable.
1637 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1638 * @param iSegReg The segment register.
1639 */
1640DECL_FORCE_INLINE(uint16_t *) iemSRegRef(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1641{
1642 Assert(iSegReg < X86_SREG_COUNT);
1643 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1644 return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1645}
1646
1647
1648/**
1649 * Fetches the selector value of a segment register.
1650 *
1651 * @returns The selector value.
1652 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1653 * @param iSegReg The segment register.
1654 */
1655DECL_FORCE_INLINE(uint16_t) iemSRegFetchU16(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1656{
1657 Assert(iSegReg < X86_SREG_COUNT);
1658 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1659 return pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1660}
1661
1662
1663/**
1664 * Fetches the base address value of a segment register.
1665 *
1666 * @returns The selector value.
1667 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1668 * @param iSegReg The segment register.
1669 */
1670DECL_FORCE_INLINE(uint64_t) iemSRegBaseFetchU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1671{
1672 Assert(iSegReg < X86_SREG_COUNT);
1673 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1674 return pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1675}
1676
1677
1678/**
1679 * Gets a reference (pointer) to the specified general purpose register.
1680 *
1681 * @returns Register reference.
1682 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1683 * @param iReg The general purpose register.
1684 */
1685DECL_FORCE_INLINE(void *) iemGRegRef(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1686{
1687 Assert(iReg < 16);
1688 return &pVCpu->cpum.GstCtx.aGRegs[iReg];
1689}
1690
1691
1692#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1693/**
1694 * Gets a reference (pointer) to the specified 8-bit general purpose register.
1695 *
1696 * Because of AH, CH, DH and BH we cannot use iemGRegRef directly here.
1697 *
1698 * @returns Register reference.
1699 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1700 * @param iReg The register.
1701 */
1702DECL_FORCE_INLINE(uint8_t *) iemGRegRefU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1703{
1704 if (iReg < 4 || (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX))
1705 {
1706 Assert(iReg < 16);
1707 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u8;
1708 }
1709 /* high 8-bit register. */
1710 Assert(iReg < 8);
1711 return &pVCpu->cpum.GstCtx.aGRegs[iReg & 3].bHi;
1712}
1713#endif
1714
1715
1716/**
1717 * Gets a reference (pointer) to the specified 8-bit general purpose register,
1718 * alternative version with extended (20) register index.
1719 *
1720 * @returns Register reference.
1721 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1722 * @param iRegEx The register. The 16 first are regular ones,
1723 * whereas 16 thru 19 maps to AH, CH, DH and BH.
1724 */
1725DECL_FORCE_INLINE(uint8_t *) iemGRegRefU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT
1726{
1727 /** @todo This could be done by double indexing on little endian hosts:
1728 * return &pVCpu->cpum.GstCtx.aGRegs[iRegEx & 15].ab[iRegEx >> 4]; */
1729 if (iRegEx < 16)
1730 return &pVCpu->cpum.GstCtx.aGRegs[iRegEx].u8;
1731
1732 /* high 8-bit register. */
1733 Assert(iRegEx < 20);
1734 return &pVCpu->cpum.GstCtx.aGRegs[iRegEx & 3].bHi;
1735}
1736
1737
1738/**
1739 * Gets a reference (pointer) to the specified 16-bit general purpose register.
1740 *
1741 * @returns Register reference.
1742 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1743 * @param iReg The register.
1744 */
1745DECL_FORCE_INLINE(uint16_t *) iemGRegRefU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1746{
1747 Assert(iReg < 16);
1748 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1749}
1750
1751
1752/**
1753 * Gets a reference (pointer) to the specified 32-bit general purpose register.
1754 *
1755 * @returns Register reference.
1756 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1757 * @param iReg The register.
1758 */
1759DECL_FORCE_INLINE(uint32_t *) iemGRegRefU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1760{
1761 Assert(iReg < 16);
1762 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1763}
1764
1765
1766/**
1767 * Gets a reference (pointer) to the specified signed 32-bit general purpose register.
1768 *
1769 * @returns Register reference.
1770 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1771 * @param iReg The register.
1772 */
1773DECL_FORCE_INLINE(int32_t *) iemGRegRefI32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1774{
1775 Assert(iReg < 16);
1776 return (int32_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1777}
1778
1779
1780/**
1781 * Gets a reference (pointer) to the specified 64-bit general purpose register.
1782 *
1783 * @returns Register reference.
1784 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1785 * @param iReg The register.
1786 */
1787DECL_FORCE_INLINE(uint64_t *) iemGRegRefU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1788{
1789 Assert(iReg < 64);
1790 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1791}
1792
1793
1794/**
1795 * Gets a reference (pointer) to the specified signed 64-bit general purpose register.
1796 *
1797 * @returns Register reference.
1798 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1799 * @param iReg The register.
1800 */
1801DECL_FORCE_INLINE(int64_t *) iemGRegRefI64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1802{
1803 Assert(iReg < 16);
1804 return (int64_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1805}
1806
1807
1808/**
1809 * Gets a reference (pointer) to the specified segment register's base address.
1810 *
1811 * @returns Segment register base address reference.
1812 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1813 * @param iSegReg The segment selector.
1814 */
1815DECL_FORCE_INLINE(uint64_t *) iemSRegBaseRefU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1816{
1817 Assert(iSegReg < X86_SREG_COUNT);
1818 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1819 return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1820}
1821
1822
1823#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1824/**
1825 * Fetches the value of a 8-bit general purpose register.
1826 *
1827 * @returns The register value.
1828 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1829 * @param iReg The register.
1830 */
1831DECL_FORCE_INLINE(uint8_t) iemGRegFetchU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1832{
1833 return *iemGRegRefU8(pVCpu, iReg);
1834}
1835#endif
1836
1837
1838/**
1839 * Fetches the value of a 8-bit general purpose register, alternative version
1840 * with extended (20) register index.
1841
1842 * @returns The register value.
1843 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1844 * @param iRegEx The register. The 16 first are regular ones,
1845 * whereas 16 thru 19 maps to AH, CH, DH and BH.
1846 */
1847DECL_FORCE_INLINE(uint8_t) iemGRegFetchU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT
1848{
1849 return *iemGRegRefU8Ex(pVCpu, iRegEx);
1850}
1851
1852
1853/**
1854 * Fetches the value of a 16-bit general purpose register.
1855 *
1856 * @returns The register value.
1857 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1858 * @param iReg The register.
1859 */
1860DECL_FORCE_INLINE(uint16_t) iemGRegFetchU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1861{
1862 Assert(iReg < 16);
1863 return pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1864}
1865
1866
1867/**
1868 * Fetches the value of a 32-bit general purpose register.
1869 *
1870 * @returns The register value.
1871 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1872 * @param iReg The register.
1873 */
1874DECL_FORCE_INLINE(uint32_t) iemGRegFetchU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1875{
1876 Assert(iReg < 16);
1877 return pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1878}
1879
1880
1881/**
1882 * Fetches the value of a 64-bit general purpose register.
1883 *
1884 * @returns The register value.
1885 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1886 * @param iReg The register.
1887 */
1888DECL_FORCE_INLINE(uint64_t) iemGRegFetchU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1889{
1890 Assert(iReg < 16);
1891 return pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1892}
1893
1894
1895/**
1896 * Stores a 16-bit value to a general purpose register.
1897 *
1898 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1899 * @param iReg The register.
1900 * @param uValue The value to store.
1901 */
1902DECL_FORCE_INLINE(void) iemGRegStoreU16(PVMCPUCC pVCpu, uint8_t iReg, uint16_t uValue) RT_NOEXCEPT
1903{
1904 Assert(iReg < 16);
1905 pVCpu->cpum.GstCtx.aGRegs[iReg].u16 = uValue;
1906}
1907
1908
1909/**
1910 * Stores a 32-bit value to a general purpose register, implicitly clearing high
1911 * values.
1912 *
1913 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1914 * @param iReg The register.
1915 * @param uValue The value to store.
1916 */
1917DECL_FORCE_INLINE(void) iemGRegStoreU32(PVMCPUCC pVCpu, uint8_t iReg, uint32_t uValue) RT_NOEXCEPT
1918{
1919 Assert(iReg < 16);
1920 pVCpu->cpum.GstCtx.aGRegs[iReg].u64 = uValue;
1921}
1922
1923
1924/**
1925 * Stores a 64-bit value to a general purpose register.
1926 *
1927 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1928 * @param iReg The register.
1929 * @param uValue The value to store.
1930 */
1931DECL_FORCE_INLINE(void) iemGRegStoreU64(PVMCPUCC pVCpu, uint8_t iReg, uint64_t uValue) RT_NOEXCEPT
1932{
1933 Assert(iReg < 16);
1934 pVCpu->cpum.GstCtx.aGRegs[iReg].u64 = uValue;
1935}
1936
1937
1938/**
1939 * Get the address of the top of the stack.
1940 *
1941 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1942 */
1943DECL_FORCE_INLINE(RTGCPTR) iemRegGetEffRsp(PCVMCPU pVCpu) RT_NOEXCEPT
1944{
1945 if (IEM_IS_64BIT_CODE(pVCpu))
1946 return pVCpu->cpum.GstCtx.rsp;
1947 if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1948 return pVCpu->cpum.GstCtx.esp;
1949 return pVCpu->cpum.GstCtx.sp;
1950}
1951
1952
1953/**
1954 * Updates the RIP/EIP/IP to point to the next instruction.
1955 *
1956 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1957 * @param cbInstr The number of bytes to add.
1958 */
1959DECL_FORCE_INLINE(void) iemRegAddToRip(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
1960{
1961 /*
1962 * Advance RIP.
1963 *
1964 * When we're targetting 8086/8, 80186/8 or 80286 mode the updates are 16-bit,
1965 * while in all other modes except LM64 the updates are 32-bit. This means
1966 * we need to watch for both 32-bit and 16-bit "carry" situations, i.e.
1967 * 4GB and 64KB rollovers, and decide whether anything needs masking.
1968 *
1969 * See PC wrap around tests in bs3-cpu-weird-1.
1970 */
1971 uint64_t const uRipPrev = pVCpu->cpum.GstCtx.rip;
1972 uint64_t const uRipNext = uRipPrev + cbInstr;
1973 if (RT_LIKELY( !((uRipNext ^ uRipPrev) & (RT_BIT_64(32) | RT_BIT_64(16)))
1974 || IEM_IS_64BIT_CODE(pVCpu)))
1975 pVCpu->cpum.GstCtx.rip = uRipNext;
1976 else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
1977 pVCpu->cpum.GstCtx.rip = (uint32_t)uRipNext;
1978 else
1979 pVCpu->cpum.GstCtx.rip = (uint16_t)uRipNext;
1980}
1981
1982
1983/**
1984 * Called by iemRegAddToRipAndFinishingClearingRF and others when any of the
1985 * following EFLAGS bits are set:
1986 * - X86_EFL_RF - clear it.
1987 * - CPUMCTX_INHIBIT_SHADOW (_SS/_STI) - clear them.
1988 * - X86_EFL_TF - generate single step \#DB trap.
1989 * - CPUMCTX_DBG_HIT_DR0/1/2/3 - generate \#DB trap (data or I/O, not
1990 * instruction).
1991 *
1992 * According to @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events},
1993 * a \#DB due to TF (single stepping) or a DRx non-instruction breakpoint
1994 * takes priority over both NMIs and hardware interrupts. So, neither is
1995 * considered here. (The RESET, \#MC, SMI, INIT, STOPCLK and FLUSH events are
1996 * either unsupported will be triggered on-top of any \#DB raised here.)
1997 *
1998 * The RF flag only needs to be cleared here as it only suppresses instruction
1999 * breakpoints which are not raised here (happens synchronously during
2000 * instruction fetching).
2001 *
2002 * The CPUMCTX_INHIBIT_SHADOW_SS flag will be cleared by this function, so its
2003 * status has no bearing on whether \#DB exceptions are raised.
2004 *
2005 * @note This must *NOT* be called by the two instructions setting the
2006 * CPUMCTX_INHIBIT_SHADOW_SS flag.
2007 *
2008 * @see @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events}
2009 * @see @sdmv3{077,200,6.8.3,Masking Exceptions and Interrupts When Switching
2010 * Stacks}
2011 */
2012static VBOXSTRICTRC iemFinishInstructionWithFlagsSet(PVMCPUCC pVCpu) RT_NOEXCEPT
2013{
2014 /*
2015 * Normally we're just here to clear RF and/or interrupt shadow bits.
2016 */
2017 if (RT_LIKELY((pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_TF | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK)) == 0))
2018 pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW);
2019 else
2020 {
2021 /*
2022 * Raise a #DB or/and DBGF event.
2023 */
2024 VBOXSTRICTRC rcStrict;
2025 if (pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_TF | CPUMCTX_DBG_HIT_DRX_MASK))
2026 {
2027 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
2028 pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
2029 if (pVCpu->cpum.GstCtx.eflags.uBoth & X86_EFL_TF)
2030 pVCpu->cpum.GstCtx.dr[6] |= X86_DR6_BS;
2031 pVCpu->cpum.GstCtx.dr[6] |= (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT;
2032 LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64\n",
2033 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
2034 pVCpu->cpum.GstCtx.rflags.uBoth));
2035
2036 pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW | CPUMCTX_DBG_HIT_DRX_MASK);
2037 rcStrict = iemRaiseDebugException(pVCpu);
2038
2039 /* A DBGF event/breakpoint trumps the iemRaiseDebugException informational status code. */
2040 if ((pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK) && RT_FAILURE(rcStrict))
2041 {
2042 rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT;
2043 LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));
2044 }
2045 }
2046 else
2047 {
2048 Assert(pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK);
2049 rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT;
2050 LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));
2051 }
2052 pVCpu->cpum.GstCtx.eflags.uBoth &= ~CPUMCTX_DBG_DBGF_MASK;
2053 return rcStrict;
2054 }
2055 return VINF_SUCCESS;
2056}
2057
2058
2059/**
2060 * Clears the RF and CPUMCTX_INHIBIT_SHADOW, triggering \#DB if pending.
2061 *
2062 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2063 */
2064DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT
2065{
2066 /*
2067 * We assume that most of the time nothing actually needs doing here.
2068 */
2069 AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX);
2070 if (RT_LIKELY(!( pVCpu->cpum.GstCtx.eflags.uBoth
2071 & (X86_EFL_TF | X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK)) ))
2072 return VINF_SUCCESS;
2073 return iemFinishInstructionWithFlagsSet(pVCpu);
2074}
2075
2076
2077/**
2078 * Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF
2079 * and CPUMCTX_INHIBIT_SHADOW.
2080 *
2081 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2082 * @param cbInstr The number of bytes to add.
2083 */
2084DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2085{
2086 iemRegAddToRip(pVCpu, cbInstr);
2087 return iemRegFinishClearingRF(pVCpu);
2088}
2089
2090
2091/**
2092 * Updates the RIP to point to the next instruction and clears EFLAGS.RF
2093 * and CPUMCTX_INHIBIT_SHADOW.
2094 *
2095 * Only called from 64-bit code.
2096 *
2097 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2098 * @param cbInstr The number of bytes to add.
2099 */
2100DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRip64AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2101{
2102 pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.rip + cbInstr;
2103 return iemRegFinishClearingRF(pVCpu);
2104}
2105
2106
2107/**
2108 * Updates the EIP to point to the next instruction and clears EFLAGS.RF and
2109 * CPUMCTX_INHIBIT_SHADOW.
2110 *
2111 * This is never from 64-bit code.
2112 *
2113 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2114 * @param cbInstr The number of bytes to add.
2115 */
2116DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToEip32AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2117{
2118 pVCpu->cpum.GstCtx.rip = (uint32_t)(pVCpu->cpum.GstCtx.eip + cbInstr);
2119 return iemRegFinishClearingRF(pVCpu);
2120}
2121
2122
2123/**
2124 * Updates the IP to point to the next instruction and clears EFLAGS.RF and
2125 * CPUMCTX_INHIBIT_SHADOW.
2126 *
2127 * This is only ever used from 16-bit code on a pre-386 CPU.
2128 *
2129 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2130 * @param cbInstr The number of bytes to add.
2131 */
2132DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToIp16AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2133{
2134 pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr);
2135 return iemRegFinishClearingRF(pVCpu);
2136}
2137
2138
2139/**
2140 * Adds a 8-bit signed jump offset to RIP from 64-bit code.
2141 *
2142 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2143 * segment limit.
2144 *
2145 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2146 * @param cbInstr Instruction size.
2147 * @param offNextInstr The offset of the next instruction.
2148 * @param enmEffOpSize Effective operand size.
2149 */
2150DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
2151 IEMMODE enmEffOpSize) RT_NOEXCEPT
2152{
2153 Assert(IEM_IS_64BIT_CODE(pVCpu));
2154 Assert(enmEffOpSize == IEMMODE_64BIT || enmEffOpSize == IEMMODE_16BIT);
2155
2156 uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
2157 if (enmEffOpSize == IEMMODE_16BIT)
2158 uNewRip &= UINT16_MAX;
2159
2160 if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
2161 pVCpu->cpum.GstCtx.rip = uNewRip;
2162 else
2163 return iemRaiseGeneralProtectionFault0(pVCpu);
2164
2165#ifndef IEM_WITH_CODE_TLB
2166 iemOpcodeFlushLight(pVCpu, cbInstr);
2167#endif
2168
2169 /*
2170 * Clear RF and finish the instruction (maybe raise #DB).
2171 */
2172 return iemRegFinishClearingRF(pVCpu);
2173}
2174
2175
2176/**
2177 * Adds a 8-bit signed jump offset to EIP, on 386 or later from 16-bit or 32-bit
2178 * code (never 64-bit).
2179 *
2180 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2181 * segment limit.
2182 *
2183 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2184 * @param cbInstr Instruction size.
2185 * @param offNextInstr The offset of the next instruction.
2186 * @param enmEffOpSize Effective operand size.
2187 */
2188DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
2189 IEMMODE enmEffOpSize) RT_NOEXCEPT
2190{
2191 Assert(!IEM_IS_64BIT_CODE(pVCpu));
2192 Assert(enmEffOpSize == IEMMODE_32BIT || enmEffOpSize == IEMMODE_16BIT);
2193
2194 uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr;
2195 if (enmEffOpSize == IEMMODE_16BIT)
2196 uNewEip &= UINT16_MAX;
2197 if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
2198 pVCpu->cpum.GstCtx.rip = uNewEip;
2199 else
2200 return iemRaiseGeneralProtectionFault0(pVCpu);
2201
2202#ifndef IEM_WITH_CODE_TLB
2203 iemOpcodeFlushLight(pVCpu, cbInstr);
2204#endif
2205
2206 /*
2207 * Clear RF and finish the instruction (maybe raise #DB).
2208 */
2209 return iemRegFinishClearingRF(pVCpu);
2210}
2211
2212
2213/**
2214 * Adds a 8-bit signed jump offset to IP, on a pre-386 CPU.
2215 *
2216 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2217 * segment limit.
2218 *
2219 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2220 * @param cbInstr Instruction size.
2221 * @param offNextInstr The offset of the next instruction.
2222 */
2223DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegIp16RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2224 int8_t offNextInstr) RT_NOEXCEPT
2225{
2226 Assert(!IEM_IS_64BIT_CODE(pVCpu));
2227
2228 uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + (int16_t)offNextInstr;
2229 if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))
2230 pVCpu->cpum.GstCtx.rip = uNewIp;
2231 else
2232 return iemRaiseGeneralProtectionFault0(pVCpu);
2233
2234#ifndef IEM_WITH_CODE_TLB
2235 iemOpcodeFlushLight(pVCpu, cbInstr);
2236#endif
2237
2238 /*
2239 * Clear RF and finish the instruction (maybe raise #DB).
2240 */
2241 return iemRegFinishClearingRF(pVCpu);
2242}
2243
2244
2245/**
2246 * Adds a 16-bit signed jump offset to RIP from 64-bit code.
2247 *
2248 * @returns Strict VBox status code.
2249 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2250 * @param cbInstr Instruction size.
2251 * @param offNextInstr The offset of the next instruction.
2252 */
2253DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2254 int16_t offNextInstr) RT_NOEXCEPT
2255{
2256 Assert(IEM_IS_64BIT_CODE(pVCpu));
2257
2258 pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr);
2259
2260#ifndef IEM_WITH_CODE_TLB
2261 iemOpcodeFlushLight(pVCpu, cbInstr);
2262#endif
2263
2264 /*
2265 * Clear RF and finish the instruction (maybe raise #DB).
2266 */
2267 return iemRegFinishClearingRF(pVCpu);
2268}
2269
2270
2271/**
2272 * Adds a 16-bit signed jump offset to EIP from 16-bit or 32-bit code.
2273 *
2274 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2275 * segment limit.
2276 *
2277 * @returns Strict VBox status code.
2278 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2279 * @param cbInstr Instruction size.
2280 * @param offNextInstr The offset of the next instruction.
2281 *
2282 * @note This is also used by 16-bit code in pre-386 mode, as the code is
2283 * identical.
2284 */
2285DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2286 int16_t offNextInstr) RT_NOEXCEPT
2287{
2288 Assert(!IEM_IS_64BIT_CODE(pVCpu));
2289
2290 uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr;
2291 if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))
2292 pVCpu->cpum.GstCtx.rip = uNewIp;
2293 else
2294 return iemRaiseGeneralProtectionFault0(pVCpu);
2295
2296#ifndef IEM_WITH_CODE_TLB
2297 iemOpcodeFlushLight(pVCpu, cbInstr);
2298#endif
2299
2300 /*
2301 * Clear RF and finish the instruction (maybe raise #DB).
2302 */
2303 return iemRegFinishClearingRF(pVCpu);
2304}
2305
2306
2307/**
2308 * Adds a 32-bit signed jump offset to RIP from 64-bit code.
2309 *
2310 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2311 * segment limit.
2312 *
2313 * We ASSUME that the effective operand size is 64-bit here, as 16-bit is the
2314 * only alternative for relative jumps in 64-bit code and that is already
2315 * handled in the decoder stage.
2316 *
2317 * @returns Strict VBox status code.
2318 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2319 * @param cbInstr Instruction size.
2320 * @param offNextInstr The offset of the next instruction.
2321 */
2322DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2323 int32_t offNextInstr) RT_NOEXCEPT
2324{
2325 Assert(IEM_IS_64BIT_CODE(pVCpu));
2326
2327 uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
2328 if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
2329 pVCpu->cpum.GstCtx.rip = uNewRip;
2330 else
2331 return iemRaiseGeneralProtectionFault0(pVCpu);
2332
2333#ifndef IEM_WITH_CODE_TLB
2334 iemOpcodeFlushLight(pVCpu, cbInstr);
2335#endif
2336
2337 /*
2338 * Clear RF and finish the instruction (maybe raise #DB).
2339 */
2340 return iemRegFinishClearingRF(pVCpu);
2341}
2342
2343
2344/**
2345 * Adds a 32-bit signed jump offset to RIP from 64-bit code.
2346 *
2347 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2348 * segment limit.
2349 *
2350 * We ASSUME that the effective operand size is 32-bit here, as 16-bit is the
2351 * only alternative for relative jumps in 32-bit code and that is already
2352 * handled in the decoder stage.
2353 *
2354 * @returns Strict VBox status code.
2355 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2356 * @param cbInstr Instruction size.
2357 * @param offNextInstr The offset of the next instruction.
2358 */
2359DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2360 int32_t offNextInstr) RT_NOEXCEPT
2361{
2362 Assert(!IEM_IS_64BIT_CODE(pVCpu));
2363 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
2364
2365 uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr;
2366 if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
2367 pVCpu->cpum.GstCtx.rip = uNewEip;
2368 else
2369 return iemRaiseGeneralProtectionFault0(pVCpu);
2370
2371#ifndef IEM_WITH_CODE_TLB
2372 iemOpcodeFlushLight(pVCpu, cbInstr);
2373#endif
2374
2375 /*
2376 * Clear RF and finish the instruction (maybe raise #DB).
2377 */
2378 return iemRegFinishClearingRF(pVCpu);
2379}
2380
2381
2382/**
2383 * Extended version of iemFinishInstructionWithFlagsSet that goes with
2384 * iemRegAddToRipAndFinishingClearingRfEx.
2385 *
2386 * See iemFinishInstructionWithFlagsSet() for details.
2387 */
2388static VBOXSTRICTRC iemFinishInstructionWithTfSet(PVMCPUCC pVCpu) RT_NOEXCEPT
2389{
2390 /*
2391 * Raise a #DB.
2392 */
2393 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
2394 pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
2395 pVCpu->cpum.GstCtx.dr[6] |= X86_DR6_BS
2396 | (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT;
2397 /** @todo Do we set all pending \#DB events, or just one? */
2398 LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64 (popf)\n",
2399 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
2400 pVCpu->cpum.GstCtx.rflags.uBoth));
2401 pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK);
2402 return iemRaiseDebugException(pVCpu);
2403}
2404
2405
2406/**
2407 * Extended version of iemRegAddToRipAndFinishingClearingRF for use by POPF and
2408 * others potentially updating EFLAGS.TF.
2409 *
2410 * The single step event must be generated using the TF value at the start of
2411 * the instruction, not the new value set by it.
2412 *
2413 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2414 * @param cbInstr The number of bytes to add.
2415 * @param fEflOld The EFLAGS at the start of the instruction
2416 * execution.
2417 */
2418DECLINLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRfEx(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t fEflOld) RT_NOEXCEPT
2419{
2420 iemRegAddToRip(pVCpu, cbInstr);
2421 if (!(fEflOld & X86_EFL_TF))
2422 return iemRegFinishClearingRF(pVCpu);
2423 return iemFinishInstructionWithTfSet(pVCpu);
2424}
2425
2426
2427#ifndef IEM_WITH_OPAQUE_DECODER_STATE
2428/**
2429 * Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
2430 *
2431 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2432 */
2433DECLINLINE(VBOXSTRICTRC) iemRegUpdateRipAndFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT
2434{
2435 return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
2436}
2437#endif
2438
2439
2440/**
2441 * Adds to the stack pointer.
2442 *
2443 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2444 * @param cbToAdd The number of bytes to add (8-bit!).
2445 */
2446DECLINLINE(void) iemRegAddToRsp(PVMCPUCC pVCpu, uint8_t cbToAdd) RT_NOEXCEPT
2447{
2448 if (IEM_IS_64BIT_CODE(pVCpu))
2449 pVCpu->cpum.GstCtx.rsp += cbToAdd;
2450 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2451 pVCpu->cpum.GstCtx.esp += cbToAdd;
2452 else
2453 pVCpu->cpum.GstCtx.sp += cbToAdd;
2454}
2455
2456
2457/**
2458 * Subtracts from the stack pointer.
2459 *
2460 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2461 * @param cbToSub The number of bytes to subtract (8-bit!).
2462 */
2463DECLINLINE(void) iemRegSubFromRsp(PVMCPUCC pVCpu, uint8_t cbToSub) RT_NOEXCEPT
2464{
2465 if (IEM_IS_64BIT_CODE(pVCpu))
2466 pVCpu->cpum.GstCtx.rsp -= cbToSub;
2467 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2468 pVCpu->cpum.GstCtx.esp -= cbToSub;
2469 else
2470 pVCpu->cpum.GstCtx.sp -= cbToSub;
2471}
2472
2473
2474/**
2475 * Adds to the temporary stack pointer.
2476 *
2477 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2478 * @param pTmpRsp The temporary SP/ESP/RSP to update.
2479 * @param cbToAdd The number of bytes to add (16-bit).
2480 */
2481DECLINLINE(void) iemRegAddToRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToAdd) RT_NOEXCEPT
2482{
2483 if (IEM_IS_64BIT_CODE(pVCpu))
2484 pTmpRsp->u += cbToAdd;
2485 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2486 pTmpRsp->DWords.dw0 += cbToAdd;
2487 else
2488 pTmpRsp->Words.w0 += cbToAdd;
2489}
2490
2491
2492/**
2493 * Subtracts from the temporary stack pointer.
2494 *
2495 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2496 * @param pTmpRsp The temporary SP/ESP/RSP to update.
2497 * @param cbToSub The number of bytes to subtract.
2498 * @remarks The @a cbToSub argument *MUST* be 16-bit, iemCImpl_enter is
2499 * expecting that.
2500 */
2501DECLINLINE(void) iemRegSubFromRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToSub) RT_NOEXCEPT
2502{
2503 if (IEM_IS_64BIT_CODE(pVCpu))
2504 pTmpRsp->u -= cbToSub;
2505 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2506 pTmpRsp->DWords.dw0 -= cbToSub;
2507 else
2508 pTmpRsp->Words.w0 -= cbToSub;
2509}
2510
2511
2512/**
2513 * Calculates the effective stack address for a push of the specified size as
2514 * well as the new RSP value (upper bits may be masked).
2515 *
2516 * @returns Effective stack addressf for the push.
2517 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2518 * @param cbItem The size of the stack item to pop.
2519 * @param puNewRsp Where to return the new RSP value.
2520 */
2521DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT
2522{
2523 RTUINT64U uTmpRsp;
2524 RTGCPTR GCPtrTop;
2525 uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
2526
2527 if (IEM_IS_64BIT_CODE(pVCpu))
2528 GCPtrTop = uTmpRsp.u -= cbItem;
2529 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2530 GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem;
2531 else
2532 GCPtrTop = uTmpRsp.Words.w0 -= cbItem;
2533 *puNewRsp = uTmpRsp.u;
2534 return GCPtrTop;
2535}
2536
2537
2538/**
2539 * Gets the current stack pointer and calculates the value after a pop of the
2540 * specified size.
2541 *
2542 * @returns Current stack pointer.
2543 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2544 * @param cbItem The size of the stack item to pop.
2545 * @param puNewRsp Where to return the new RSP value.
2546 */
2547DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT
2548{
2549 RTUINT64U uTmpRsp;
2550 RTGCPTR GCPtrTop;
2551 uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
2552
2553 if (IEM_IS_64BIT_CODE(pVCpu))
2554 {
2555 GCPtrTop = uTmpRsp.u;
2556 uTmpRsp.u += cbItem;
2557 }
2558 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2559 {
2560 GCPtrTop = uTmpRsp.DWords.dw0;
2561 uTmpRsp.DWords.dw0 += cbItem;
2562 }
2563 else
2564 {
2565 GCPtrTop = uTmpRsp.Words.w0;
2566 uTmpRsp.Words.w0 += cbItem;
2567 }
2568 *puNewRsp = uTmpRsp.u;
2569 return GCPtrTop;
2570}
2571
2572
2573/**
2574 * Calculates the effective stack address for a push of the specified size as
2575 * well as the new temporary RSP value (upper bits may be masked).
2576 *
2577 * @returns Effective stack addressf for the push.
2578 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2579 * @param pTmpRsp The temporary stack pointer. This is updated.
2580 * @param cbItem The size of the stack item to pop.
2581 */
2582DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT
2583{
2584 RTGCPTR GCPtrTop;
2585
2586 if (IEM_IS_64BIT_CODE(pVCpu))
2587 GCPtrTop = pTmpRsp->u -= cbItem;
2588 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2589 GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem;
2590 else
2591 GCPtrTop = pTmpRsp->Words.w0 -= cbItem;
2592 return GCPtrTop;
2593}
2594
2595
2596/**
2597 * Gets the effective stack address for a pop of the specified size and
2598 * calculates and updates the temporary RSP.
2599 *
2600 * @returns Current stack pointer.
2601 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2602 * @param pTmpRsp The temporary stack pointer. This is updated.
2603 * @param cbItem The size of the stack item to pop.
2604 */
2605DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT
2606{
2607 RTGCPTR GCPtrTop;
2608 if (IEM_IS_64BIT_CODE(pVCpu))
2609 {
2610 GCPtrTop = pTmpRsp->u;
2611 pTmpRsp->u += cbItem;
2612 }
2613 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2614 {
2615 GCPtrTop = pTmpRsp->DWords.dw0;
2616 pTmpRsp->DWords.dw0 += cbItem;
2617 }
2618 else
2619 {
2620 GCPtrTop = pTmpRsp->Words.w0;
2621 pTmpRsp->Words.w0 += cbItem;
2622 }
2623 return GCPtrTop;
2624}
2625
2626/** @} */
2627
2628
2629/** @name FPU access and helpers.
2630 *
2631 * @{
2632 */
2633
2634
2635/**
2636 * Hook for preparing to use the host FPU.
2637 *
2638 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
2639 *
2640 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2641 */
2642DECLINLINE(void) iemFpuPrepareUsage(PVMCPUCC pVCpu) RT_NOEXCEPT
2643{
2644#ifdef IN_RING3
2645 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2646#else
2647 CPUMRZFpuStatePrepareHostCpuForUse(pVCpu);
2648#endif
2649 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
2650}
2651
2652
2653/**
2654 * Hook for preparing to use the host FPU for SSE.
2655 *
2656 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
2657 *
2658 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2659 */
2660DECLINLINE(void) iemFpuPrepareUsageSse(PVMCPUCC pVCpu) RT_NOEXCEPT
2661{
2662 iemFpuPrepareUsage(pVCpu);
2663}
2664
2665
2666/**
2667 * Hook for preparing to use the host FPU for AVX.
2668 *
2669 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
2670 *
2671 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2672 */
2673DECLINLINE(void) iemFpuPrepareUsageAvx(PVMCPUCC pVCpu) RT_NOEXCEPT
2674{
2675 iemFpuPrepareUsage(pVCpu);
2676}
2677
2678
2679/**
2680 * Hook for actualizing the guest FPU state before the interpreter reads it.
2681 *
2682 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
2683 *
2684 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2685 */
2686DECLINLINE(void) iemFpuActualizeStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2687{
2688#ifdef IN_RING3
2689 NOREF(pVCpu);
2690#else
2691 CPUMRZFpuStateActualizeForRead(pVCpu);
2692#endif
2693 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
2694}
2695
2696
2697/**
2698 * Hook for actualizing the guest FPU state before the interpreter changes it.
2699 *
2700 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
2701 *
2702 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2703 */
2704DECLINLINE(void) iemFpuActualizeStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2705{
2706#ifdef IN_RING3
2707 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2708#else
2709 CPUMRZFpuStateActualizeForChange(pVCpu);
2710#endif
2711 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
2712}
2713
2714
2715/**
2716 * Hook for actualizing the guest XMM0..15 and MXCSR register state for read
2717 * only.
2718 *
2719 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
2720 *
2721 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2722 */
2723DECLINLINE(void) iemFpuActualizeSseStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2724{
2725#if defined(IN_RING3) || defined(VBOX_WITH_KERNEL_USING_XMM)
2726 NOREF(pVCpu);
2727#else
2728 CPUMRZFpuStateActualizeSseForRead(pVCpu);
2729#endif
2730 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
2731}
2732
2733
2734/**
2735 * Hook for actualizing the guest XMM0..15 and MXCSR register state for
2736 * read+write.
2737 *
2738 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
2739 *
2740 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2741 */
2742DECLINLINE(void) iemFpuActualizeSseStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2743{
2744#if defined(IN_RING3) || defined(VBOX_WITH_KERNEL_USING_XMM)
2745 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2746#else
2747 CPUMRZFpuStateActualizeForChange(pVCpu);
2748#endif
2749 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
2750
2751 /* Make sure any changes are loaded the next time around. */
2752 pVCpu->cpum.GstCtx.XState.Hdr.bmXState |= XSAVE_C_SSE;
2753}
2754
2755
2756/**
2757 * Hook for actualizing the guest YMM0..15 and MXCSR register state for read
2758 * only.
2759 *
2760 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
2761 *
2762 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2763 */
2764DECLINLINE(void) iemFpuActualizeAvxStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2765{
2766#ifdef IN_RING3
2767 NOREF(pVCpu);
2768#else
2769 CPUMRZFpuStateActualizeAvxForRead(pVCpu);
2770#endif
2771 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
2772}
2773
2774
2775/**
2776 * Hook for actualizing the guest YMM0..15 and MXCSR register state for
2777 * read+write.
2778 *
2779 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
2780 *
2781 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2782 */
2783DECLINLINE(void) iemFpuActualizeAvxStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2784{
2785#ifdef IN_RING3
2786 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2787#else
2788 CPUMRZFpuStateActualizeForChange(pVCpu);
2789#endif
2790 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
2791
2792 /* Just assume we're going to make changes to the SSE and YMM_HI parts. */
2793 pVCpu->cpum.GstCtx.XState.Hdr.bmXState |= XSAVE_C_YMM | XSAVE_C_SSE;
2794}
2795
2796
2797/**
2798 * Stores a QNaN value into a FPU register.
2799 *
2800 * @param pReg Pointer to the register.
2801 */
2802DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg) RT_NOEXCEPT
2803{
2804 pReg->au32[0] = UINT32_C(0x00000000);
2805 pReg->au32[1] = UINT32_C(0xc0000000);
2806 pReg->au16[4] = UINT16_C(0xffff);
2807}
2808
2809
2810/**
2811 * Updates the FOP, FPU.CS and FPUIP registers, extended version.
2812 *
2813 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2814 * @param pFpuCtx The FPU context.
2815 * @param uFpuOpcode The FPU opcode value (see IEMCPU::uFpuOpcode).
2816 */
2817DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorkerEx(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx, uint16_t uFpuOpcode) RT_NOEXCEPT
2818{
2819 Assert(uFpuOpcode != UINT16_MAX);
2820 pFpuCtx->FOP = uFpuOpcode;
2821 /** @todo x87.CS and FPUIP needs to be kept seperately. */
2822 if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
2823 {
2824 /** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled
2825 * happens in real mode here based on the fnsave and fnstenv images. */
2826 pFpuCtx->CS = 0;
2827 pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.eip | ((uint32_t)pVCpu->cpum.GstCtx.cs.Sel << 4);
2828 }
2829 else if (!IEM_IS_LONG_MODE(pVCpu))
2830 {
2831 pFpuCtx->CS = pVCpu->cpum.GstCtx.cs.Sel;
2832 pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2833 }
2834 else
2835 *(uint64_t *)&pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2836}
2837
2838
2839#ifndef IEM_WITH_OPAQUE_DECODER_STATE
2840/**
2841 * Updates the FOP, FPU.CS and FPUIP registers.
2842 *
2843 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2844 * @param pFpuCtx The FPU context.
2845 */
2846DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorker(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx) RT_NOEXCEPT
2847{
2848 Assert(pVCpu->iem.s.uFpuOpcode != UINT16_MAX);
2849 iemFpuUpdateOpcodeAndIpWorkerEx(pVCpu, pFpuCtx, pVCpu->iem.s.uFpuOpcode);
2850}
2851#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
2852
2853
2854/**
2855 * Marks the specified stack register as free (for FFREE).
2856 *
2857 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2858 * @param iStReg The register to free.
2859 */
2860DECLINLINE(void) iemFpuStackFree(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
2861{
2862 Assert(iStReg < 8);
2863 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2864 uint8_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2865 pFpuCtx->FTW &= ~RT_BIT(iReg);
2866}
2867
2868
2869/**
2870 * Increments FSW.TOP, i.e. pops an item off the stack without freeing it.
2871 *
2872 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2873 */
2874DECLINLINE(void) iemFpuStackIncTop(PVMCPUCC pVCpu) RT_NOEXCEPT
2875{
2876 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2877 uint16_t uFsw = pFpuCtx->FSW;
2878 uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2879 uTop = (uTop + (1 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2880 uFsw &= ~X86_FSW_TOP_MASK;
2881 uFsw |= uTop;
2882 pFpuCtx->FSW = uFsw;
2883}
2884
2885
2886/**
2887 * Decrements FSW.TOP, i.e. push an item off the stack without storing anything.
2888 *
2889 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2890 */
2891DECLINLINE(void) iemFpuStackDecTop(PVMCPUCC pVCpu) RT_NOEXCEPT
2892{
2893 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2894 uint16_t uFsw = pFpuCtx->FSW;
2895 uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2896 uTop = (uTop + (7 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2897 uFsw &= ~X86_FSW_TOP_MASK;
2898 uFsw |= uTop;
2899 pFpuCtx->FSW = uFsw;
2900}
2901
2902
2903
2904
2905DECLINLINE(int) iemFpuStRegNotEmpty(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
2906{
2907 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2908 uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2909 if (pFpuCtx->FTW & RT_BIT(iReg))
2910 return VINF_SUCCESS;
2911 return VERR_NOT_FOUND;
2912}
2913
2914
2915DECLINLINE(int) iemFpuStRegNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef) RT_NOEXCEPT
2916{
2917 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2918 uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2919 if (pFpuCtx->FTW & RT_BIT(iReg))
2920 {
2921 *ppRef = &pFpuCtx->aRegs[iStReg].r80;
2922 return VINF_SUCCESS;
2923 }
2924 return VERR_NOT_FOUND;
2925}
2926
2927
2928DECLINLINE(int) iemFpu2StRegsNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0,
2929 uint8_t iStReg1, PCRTFLOAT80U *ppRef1) RT_NOEXCEPT
2930{
2931 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2932 uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2933 uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2934 uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2935 if ((pFpuCtx->FTW & (RT_BIT(iReg0) | RT_BIT(iReg1))) == (RT_BIT(iReg0) | RT_BIT(iReg1)))
2936 {
2937 *ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2938 *ppRef1 = &pFpuCtx->aRegs[iStReg1].r80;
2939 return VINF_SUCCESS;
2940 }
2941 return VERR_NOT_FOUND;
2942}
2943
2944
2945DECLINLINE(int) iemFpu2StRegsNotEmptyRefFirst(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1) RT_NOEXCEPT
2946{
2947 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2948 uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2949 uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2950 uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2951 if ((pFpuCtx->FTW & (RT_BIT(iReg0) | RT_BIT(iReg1))) == (RT_BIT(iReg0) | RT_BIT(iReg1)))
2952 {
2953 *ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2954 return VINF_SUCCESS;
2955 }
2956 return VERR_NOT_FOUND;
2957}
2958
2959
2960/**
2961 * Rotates the stack registers when setting new TOS.
2962 *
2963 * @param pFpuCtx The FPU context.
2964 * @param iNewTop New TOS value.
2965 * @remarks We only do this to speed up fxsave/fxrstor which
2966 * arrange the FP registers in stack order.
2967 * MUST be done before writing the new TOS (FSW).
2968 */
2969DECLINLINE(void) iemFpuRotateStackSetTop(PX86FXSTATE pFpuCtx, uint16_t iNewTop) RT_NOEXCEPT
2970{
2971 uint16_t iOldTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2972 RTFLOAT80U ar80Temp[8];
2973
2974 if (iOldTop == iNewTop)
2975 return;
2976
2977 /* Unscrew the stack and get it into 'native' order. */
2978 ar80Temp[0] = pFpuCtx->aRegs[(8 - iOldTop + 0) & X86_FSW_TOP_SMASK].r80;
2979 ar80Temp[1] = pFpuCtx->aRegs[(8 - iOldTop + 1) & X86_FSW_TOP_SMASK].r80;
2980 ar80Temp[2] = pFpuCtx->aRegs[(8 - iOldTop + 2) & X86_FSW_TOP_SMASK].r80;
2981 ar80Temp[3] = pFpuCtx->aRegs[(8 - iOldTop + 3) & X86_FSW_TOP_SMASK].r80;
2982 ar80Temp[4] = pFpuCtx->aRegs[(8 - iOldTop + 4) & X86_FSW_TOP_SMASK].r80;
2983 ar80Temp[5] = pFpuCtx->aRegs[(8 - iOldTop + 5) & X86_FSW_TOP_SMASK].r80;
2984 ar80Temp[6] = pFpuCtx->aRegs[(8 - iOldTop + 6) & X86_FSW_TOP_SMASK].r80;
2985 ar80Temp[7] = pFpuCtx->aRegs[(8 - iOldTop + 7) & X86_FSW_TOP_SMASK].r80;
2986
2987 /* Now rotate the stack to the new position. */
2988 pFpuCtx->aRegs[0].r80 = ar80Temp[(iNewTop + 0) & X86_FSW_TOP_SMASK];
2989 pFpuCtx->aRegs[1].r80 = ar80Temp[(iNewTop + 1) & X86_FSW_TOP_SMASK];
2990 pFpuCtx->aRegs[2].r80 = ar80Temp[(iNewTop + 2) & X86_FSW_TOP_SMASK];
2991 pFpuCtx->aRegs[3].r80 = ar80Temp[(iNewTop + 3) & X86_FSW_TOP_SMASK];
2992 pFpuCtx->aRegs[4].r80 = ar80Temp[(iNewTop + 4) & X86_FSW_TOP_SMASK];
2993 pFpuCtx->aRegs[5].r80 = ar80Temp[(iNewTop + 5) & X86_FSW_TOP_SMASK];
2994 pFpuCtx->aRegs[6].r80 = ar80Temp[(iNewTop + 6) & X86_FSW_TOP_SMASK];
2995 pFpuCtx->aRegs[7].r80 = ar80Temp[(iNewTop + 7) & X86_FSW_TOP_SMASK];
2996}
2997
2998
2999/**
3000 * Updates the FPU exception status after FCW is changed.
3001 *
3002 * @param pFpuCtx The FPU context.
3003 */
3004DECLINLINE(void) iemFpuRecalcExceptionStatus(PX86FXSTATE pFpuCtx) RT_NOEXCEPT
3005{
3006 uint16_t u16Fsw = pFpuCtx->FSW;
3007 if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pFpuCtx->FCW & X86_FCW_XCPT_MASK))
3008 u16Fsw |= X86_FSW_ES | X86_FSW_B;
3009 else
3010 u16Fsw &= ~(X86_FSW_ES | X86_FSW_B);
3011 pFpuCtx->FSW = u16Fsw;
3012}
3013
3014
3015/**
3016 * Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE.
3017 *
3018 * @returns The full FTW.
3019 * @param pFpuCtx The FPU context.
3020 */
3021DECLINLINE(uint16_t) iemFpuCalcFullFtw(PCX86FXSTATE pFpuCtx) RT_NOEXCEPT
3022{
3023 uint8_t const u8Ftw = (uint8_t)pFpuCtx->FTW;
3024 uint16_t u16Ftw = 0;
3025 unsigned const iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
3026 for (unsigned iSt = 0; iSt < 8; iSt++)
3027 {
3028 unsigned const iReg = (iSt + iTop) & 7;
3029 if (!(u8Ftw & RT_BIT(iReg)))
3030 u16Ftw |= 3 << (iReg * 2); /* empty */
3031 else
3032 {
3033 uint16_t uTag;
3034 PCRTFLOAT80U const pr80Reg = &pFpuCtx->aRegs[iSt].r80;
3035 if (pr80Reg->s.uExponent == 0x7fff)
3036 uTag = 2; /* Exponent is all 1's => Special. */
3037 else if (pr80Reg->s.uExponent == 0x0000)
3038 {
3039 if (pr80Reg->s.uMantissa == 0x0000)
3040 uTag = 1; /* All bits are zero => Zero. */
3041 else
3042 uTag = 2; /* Must be special. */
3043 }
3044 else if (pr80Reg->s.uMantissa & RT_BIT_64(63)) /* The J bit. */
3045 uTag = 0; /* Valid. */
3046 else
3047 uTag = 2; /* Must be special. */
3048
3049 u16Ftw |= uTag << (iReg * 2);
3050 }
3051 }
3052
3053 return u16Ftw;
3054}
3055
3056
3057/**
3058 * Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR).
3059 *
3060 * @returns The compressed FTW.
3061 * @param u16FullFtw The full FTW to convert.
3062 */
3063DECLINLINE(uint16_t) iemFpuCompressFtw(uint16_t u16FullFtw) RT_NOEXCEPT
3064{
3065 uint8_t u8Ftw = 0;
3066 for (unsigned i = 0; i < 8; i++)
3067 {
3068 if ((u16FullFtw & 3) != 3 /*empty*/)
3069 u8Ftw |= RT_BIT(i);
3070 u16FullFtw >>= 2;
3071 }
3072
3073 return u8Ftw;
3074}
3075
3076/** @} */
3077
3078
3079/** @name Memory access.
3080 *
3081 * @{
3082 */
3083
3084
3085/**
3086 * Checks whether alignment checks are enabled or not.
3087 *
3088 * @returns true if enabled, false if not.
3089 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3090 */
3091DECLINLINE(bool) iemMemAreAlignmentChecksEnabled(PVMCPUCC pVCpu) RT_NOEXCEPT
3092{
3093 AssertCompile(X86_CR0_AM == X86_EFL_AC);
3094 return IEM_GET_CPL(pVCpu) == 3
3095 && (((uint32_t)pVCpu->cpum.GstCtx.cr0 & pVCpu->cpum.GstCtx.eflags.u) & X86_CR0_AM);
3096}
3097
3098/**
3099 * Checks if the given segment can be written to, raise the appropriate
3100 * exception if not.
3101 *
3102 * @returns VBox strict status code.
3103 *
3104 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3105 * @param pHid Pointer to the hidden register.
3106 * @param iSegReg The register number.
3107 * @param pu64BaseAddr Where to return the base address to use for the
3108 * segment. (In 64-bit code it may differ from the
3109 * base in the hidden segment.)
3110 */
3111DECLINLINE(VBOXSTRICTRC) iemMemSegCheckWriteAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid,
3112 uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT
3113{
3114 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3115
3116 if (IEM_IS_64BIT_CODE(pVCpu))
3117 *pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
3118 else
3119 {
3120 if (!pHid->Attr.n.u1Present)
3121 {
3122 uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
3123 AssertRelease(uSel == 0);
3124 Log(("iemMemSegCheckWriteAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
3125 return iemRaiseGeneralProtectionFault0(pVCpu);
3126 }
3127
3128 if ( ( (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE)
3129 || !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
3130 && !IEM_IS_64BIT_CODE(pVCpu) )
3131 return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3132 *pu64BaseAddr = pHid->u64Base;
3133 }
3134 return VINF_SUCCESS;
3135}
3136
3137
3138/**
3139 * Checks if the given segment can be read from, raise the appropriate
3140 * exception if not.
3141 *
3142 * @returns VBox strict status code.
3143 *
3144 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3145 * @param pHid Pointer to the hidden register.
3146 * @param iSegReg The register number.
3147 * @param pu64BaseAddr Where to return the base address to use for the
3148 * segment. (In 64-bit code it may differ from the
3149 * base in the hidden segment.)
3150 */
3151DECLINLINE(VBOXSTRICTRC) iemMemSegCheckReadAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid,
3152 uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT
3153{
3154 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3155
3156 if (IEM_IS_64BIT_CODE(pVCpu))
3157 *pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
3158 else
3159 {
3160 if (!pHid->Attr.n.u1Present)
3161 {
3162 uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
3163 AssertRelease(uSel == 0);
3164 Log(("iemMemSegCheckReadAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
3165 return iemRaiseGeneralProtectionFault0(pVCpu);
3166 }
3167
3168 if ((pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
3169 return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
3170 *pu64BaseAddr = pHid->u64Base;
3171 }
3172 return VINF_SUCCESS;
3173}
3174
3175
3176/**
3177 * Maps a physical page.
3178 *
3179 * @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr).
3180 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3181 * @param GCPhysMem The physical address.
3182 * @param fAccess The intended access.
3183 * @param ppvMem Where to return the mapping address.
3184 * @param pLock The PGM lock.
3185 */
3186DECLINLINE(int) iemMemPageMap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess,
3187 void **ppvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT
3188{
3189#ifdef IEM_LOG_MEMORY_WRITES
3190 if (fAccess & IEM_ACCESS_TYPE_WRITE)
3191 return VERR_PGM_PHYS_TLB_CATCH_ALL;
3192#endif
3193
3194 /** @todo This API may require some improving later. A private deal with PGM
3195 * regarding locking and unlocking needs to be struct. A couple of TLBs
3196 * living in PGM, but with publicly accessible inlined access methods
3197 * could perhaps be an even better solution. */
3198 int rc = PGMPhysIemGCPhys2Ptr(pVCpu->CTX_SUFF(pVM), pVCpu,
3199 GCPhysMem,
3200 RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE),
3201 RT_BOOL(pVCpu->iem.s.fExec & IEM_F_BYPASS_HANDLERS),
3202 ppvMem,
3203 pLock);
3204 /*Log(("PGMPhysIemGCPhys2Ptr %Rrc pLock=%.*Rhxs\n", rc, sizeof(*pLock), pLock));*/
3205 AssertMsg(rc == VINF_SUCCESS || RT_FAILURE_NP(rc), ("%Rrc\n", rc));
3206
3207 return rc;
3208}
3209
3210
3211/**
3212 * Unmap a page previously mapped by iemMemPageMap.
3213 *
3214 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3215 * @param GCPhysMem The physical address.
3216 * @param fAccess The intended access.
3217 * @param pvMem What iemMemPageMap returned.
3218 * @param pLock The PGM lock.
3219 */
3220DECLINLINE(void) iemMemPageUnmap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess,
3221 const void *pvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT
3222{
3223 NOREF(pVCpu);
3224 NOREF(GCPhysMem);
3225 NOREF(fAccess);
3226 NOREF(pvMem);
3227 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), pLock);
3228}
3229
3230#ifdef IEM_WITH_SETJMP
3231
3232/** @todo slim this down */
3233DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToReadJmp(PVMCPUCC pVCpu, uint8_t iSegReg,
3234 size_t cbMem, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
3235{
3236 Assert(cbMem >= 1);
3237 Assert(iSegReg < X86_SREG_COUNT);
3238
3239 /*
3240 * 64-bit mode is simpler.
3241 */
3242 if (IEM_IS_64BIT_CODE(pVCpu))
3243 {
3244 if (iSegReg >= X86_SREG_FS && iSegReg != UINT8_MAX)
3245 {
3246 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3247 PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
3248 GCPtrMem += pSel->u64Base;
3249 }
3250
3251 if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
3252 return GCPtrMem;
3253 iemRaiseGeneralProtectionFault0Jmp(pVCpu);
3254 }
3255 /*
3256 * 16-bit and 32-bit segmentation.
3257 */
3258 else if (iSegReg != UINT8_MAX)
3259 {
3260 /** @todo Does this apply to segments with 4G-1 limit? */
3261 uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
3262 if (RT_LIKELY(GCPtrLast32 >= (uint32_t)GCPtrMem))
3263 {
3264 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3265 PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
3266 switch (pSel->Attr.u & ( X86DESCATTR_P | X86DESCATTR_UNUSABLE
3267 | X86_SEL_TYPE_READ | X86_SEL_TYPE_WRITE /* same as read */
3268 | X86_SEL_TYPE_DOWN | X86_SEL_TYPE_CONF /* same as down */
3269 | X86_SEL_TYPE_CODE))
3270 {
3271 case X86DESCATTR_P: /* readonly data, expand up */
3272 case X86DESCATTR_P | X86_SEL_TYPE_WRITE: /* writable data, expand up */
3273 case X86DESCATTR_P | X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ: /* code, read-only */
3274 case X86DESCATTR_P | X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ | X86_SEL_TYPE_CONF: /* conforming code, read-only */
3275 /* expand up */
3276 if (RT_LIKELY(GCPtrLast32 <= pSel->u32Limit))
3277 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3278 Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x vs %#x\n",
3279 (uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit));
3280 break;
3281
3282 case X86DESCATTR_P | X86_SEL_TYPE_DOWN: /* readonly data, expand down */
3283 case X86DESCATTR_P | X86_SEL_TYPE_DOWN | X86_SEL_TYPE_WRITE: /* writable data, expand down */
3284 /* expand down */
3285 if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
3286 && ( pSel->Attr.n.u1DefBig
3287 || GCPtrLast32 <= UINT32_C(0xffff)) ))
3288 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3289 Log10(("iemMemApplySegmentToReadJmp: expand down out of bounds %#x..%#x vs %#x..%#x\n",
3290 (uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit, pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT16_MAX));
3291 break;
3292
3293 default:
3294 Log10(("iemMemApplySegmentToReadJmp: bad selector %#x\n", pSel->Attr.u));
3295 iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
3296 break;
3297 }
3298 }
3299 Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x\n",(uint32_t)GCPtrMem, GCPtrLast32));
3300 iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
3301 }
3302 /*
3303 * 32-bit flat address.
3304 */
3305 else
3306 return GCPtrMem;
3307}
3308
3309
3310/** @todo slim this down */
3311DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToWriteJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem,
3312 RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
3313{
3314 Assert(cbMem >= 1);
3315 Assert(iSegReg < X86_SREG_COUNT);
3316
3317 /*
3318 * 64-bit mode is simpler.
3319 */
3320 if (IEM_IS_64BIT_CODE(pVCpu))
3321 {
3322 if (iSegReg >= X86_SREG_FS)
3323 {
3324 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3325 PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
3326 GCPtrMem += pSel->u64Base;
3327 }
3328
3329 if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
3330 return GCPtrMem;
3331 }
3332 /*
3333 * 16-bit and 32-bit segmentation.
3334 */
3335 else
3336 {
3337 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3338 PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
3339 uint32_t const fRelevantAttrs = pSel->Attr.u & ( X86DESCATTR_P | X86DESCATTR_UNUSABLE
3340 | X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE | X86_SEL_TYPE_DOWN);
3341 if (fRelevantAttrs == (X86DESCATTR_P | X86_SEL_TYPE_WRITE)) /* data, expand up */
3342 {
3343 /* expand up */
3344 uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
3345 if (RT_LIKELY( GCPtrLast32 > pSel->u32Limit
3346 && GCPtrLast32 > (uint32_t)GCPtrMem))
3347 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3348 }
3349 else if (fRelevantAttrs == (X86DESCATTR_P | X86_SEL_TYPE_WRITE | X86_SEL_TYPE_DOWN)) /* data, expand up */
3350 {
3351 /* expand down */
3352 uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
3353 if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
3354 && GCPtrLast32 <= (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
3355 && GCPtrLast32 > (uint32_t)GCPtrMem))
3356 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3357 }
3358 else
3359 iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3360 iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3361 }
3362 iemRaiseGeneralProtectionFault0Jmp(pVCpu);
3363}
3364
3365#endif /* IEM_WITH_SETJMP */
3366
3367/**
3368 * Fakes a long mode stack selector for SS = 0.
3369 *
3370 * @param pDescSs Where to return the fake stack descriptor.
3371 * @param uDpl The DPL we want.
3372 */
3373DECLINLINE(void) iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl) RT_NOEXCEPT
3374{
3375 pDescSs->Long.au64[0] = 0;
3376 pDescSs->Long.au64[1] = 0;
3377 pDescSs->Long.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
3378 pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */
3379 pDescSs->Long.Gen.u2Dpl = uDpl;
3380 pDescSs->Long.Gen.u1Present = 1;
3381 pDescSs->Long.Gen.u1Long = 1;
3382}
3383
3384/** @} */
3385
3386
3387#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3388
3389/**
3390 * Gets CR0 fixed-0 bits in VMX operation.
3391 *
3392 * We do this rather than fetching what we report to the guest (in
3393 * IA32_VMX_CR0_FIXED0 MSR) because real hardware (and so do we) report the same
3394 * values regardless of whether unrestricted-guest feature is available on the CPU.
3395 *
3396 * @returns CR0 fixed-0 bits.
3397 * @param pVCpu The cross context virtual CPU structure.
3398 * @param fVmxNonRootMode Whether the CR0 fixed-0 bits for VMX non-root mode
3399 * must be returned. When @c false, the CR0 fixed-0
3400 * bits for VMX root mode is returned.
3401 *
3402 */
3403DECLINLINE(uint64_t) iemVmxGetCr0Fixed0(PCVMCPUCC pVCpu, bool fVmxNonRootMode) RT_NOEXCEPT
3404{
3405 Assert(IEM_VMX_IS_ROOT_MODE(pVCpu));
3406
3407 PCVMXMSRS pMsrs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs;
3408 if ( fVmxNonRootMode
3409 && (pMsrs->ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST))
3410 return VMX_V_CR0_FIXED0_UX;
3411 return VMX_V_CR0_FIXED0;
3412}
3413
3414
3415/**
3416 * Sets virtual-APIC write emulation as pending.
3417 *
3418 * @param pVCpu The cross context virtual CPU structure.
3419 * @param offApic The offset in the virtual-APIC page that was written.
3420 */
3421DECLINLINE(void) iemVmxVirtApicSetPendingWrite(PVMCPUCC pVCpu, uint16_t offApic) RT_NOEXCEPT
3422{
3423 Assert(offApic < XAPIC_OFF_END + 4);
3424
3425 /*
3426 * Record the currently updated APIC offset, as we need this later for figuring
3427 * out whether to perform TPR, EOI or self-IPI virtualization as well as well
3428 * as for supplying the exit qualification when causing an APIC-write VM-exit.
3429 */
3430 pVCpu->cpum.GstCtx.hwvirt.vmx.offVirtApicWrite = offApic;
3431
3432 /*
3433 * Flag that we need to perform virtual-APIC write emulation (TPR/PPR/EOI/Self-IPI
3434 * virtualization or APIC-write emulation).
3435 */
3436 if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
3437 VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE);
3438}
3439
3440#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
3441
3442#endif /* !VMM_INCLUDED_SRC_include_IEMInline_h */
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette