EM.cpp@ 20961

Last change on this file since 20961 was 20871, checked in by vboxsync, 15 years ago
VMM: Make sure there is enough room for a few physical handler notification before we disable ring-3 calls. Partial VMM[GC\|R0]CallHost unification.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 165.2 KB

Line
1	/* $Id: EM.cpp 20871 2009-06-24 01:56:19Z vboxsync $ */
2	/** @file
3	* EM - Execution Monitor / Manager.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22	/** @page pg_em EM - The Execution Monitor / Manager
23	*
24	* The Execution Monitor/Manager is responsible for running the VM, scheduling
25	* the right kind of execution (Raw-mode, Hardware Assisted, Recompiled or
26	* Interpreted), and keeping the CPU states in sync. The function
27	* EMR3ExecuteVM() is the 'main-loop' of the VM, while each of the execution
28	* modes has different inner loops (emR3RawExecute, emR3HwAccExecute, and
29	* emR3RemExecute).
30	*
31	* The interpreted execution is only used to avoid switching between
32	* raw-mode/hwaccm and the recompiler when fielding virtualization traps/faults.
33	* The interpretation is thus implemented as part of EM.
34	*
35	* @see grp_em
36	*/
37
38	/*******************************************************************************
39	* Header Files *
40	*******************************************************************************/
41	#define LOG_GROUP LOG_GROUP_EM
42	#include <VBox/em.h>
43	#include <VBox/vmm.h>
44	#ifdef VBOX_WITH_VMI
45	# include <VBox/parav.h>
46	#endif
47	#include <VBox/patm.h>
48	#include <VBox/csam.h>
49	#include <VBox/selm.h>
50	#include <VBox/trpm.h>
51	#include <VBox/iom.h>
52	#include <VBox/dbgf.h>
53	#include <VBox/pgm.h>
54	#include <VBox/rem.h>
55	#include <VBox/tm.h>
56	#include <VBox/mm.h>
57	#include <VBox/ssm.h>
58	#include <VBox/pdmapi.h>
59	#include <VBox/pdmcritsect.h>
60	#include <VBox/pdmqueue.h>
61	#include <VBox/hwaccm.h>
62	#include <VBox/patm.h>
63	#include "EMInternal.h"
64	#include <VBox/vm.h>
65	#include <VBox/cpumdis.h>
66	#include <VBox/dis.h>
67	#include <VBox/disopcode.h>
68	#include <VBox/dbgf.h>
69
70	#include <VBox/log.h>
71	#include <iprt/thread.h>
72	#include <iprt/assert.h>
73	#include <iprt/asm.h>
74	#include <iprt/semaphore.h>
75	#include <iprt/string.h>
76	#include <iprt/avl.h>
77	#include <iprt/stream.h>
78	#include <VBox/param.h>
79	#include <VBox/err.h>
80
81
82	/*******************************************************************************
83	* Defined Constants And Macros *
84	*******************************************************************************/
85	#if 0 /* Disabled till after 2.1.0 when we've time to test it. */
86	#define EM_NOTIFY_HWACCM
87	#endif
88
89
90	/*******************************************************************************
91	* Internal Functions *
92	*******************************************************************************/
93	static DECLCALLBACK(int) emR3Save(PVM pVM, PSSMHANDLE pSSM);
94	static DECLCALLBACK(int) emR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t u32Version);
95	static int emR3Debug(PVM pVM, PVMCPU pVCpu, int rc);
96	static int emR3RemStep(PVM pVM, PVMCPU pVCpu);
97	static int emR3RemExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone);
98	static int emR3RawResumeHyper(PVM pVM, PVMCPU pVCpu);
99	static int emR3RawStep(PVM pVM, PVMCPU pVCpu);
100	DECLINLINE(int) emR3RawHandleRC(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, int rc);
101	DECLINLINE(int) emR3RawUpdateForceFlag(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, int rc);
102	static int emR3RawForcedActions(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
103	static int emR3RawExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone);
104	DECLINLINE(int) emR3RawExecuteInstruction(PVM pVM, PVMCPU pVCpu, const char *pszPrefix, int rcGC = VINF_SUCCESS);
105	static int emR3HighPriorityPostForcedActions(PVM pVM, PVMCPU pVCpu, int rc);
106	static int emR3ForcedActions(PVM pVM, PVMCPU pVCpu, int rc);
107	static int emR3RawGuestTrap(PVM pVM, PVMCPU pVCpu);
108	static int emR3PatchTrap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, int gcret);
109	static int emR3SingleStepExecRem(PVM pVM, uint32_t cIterations);
110	static EMSTATE emR3Reschedule(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
111
112	/**
113	* Initializes the EM.
114	*
115	* @returns VBox status code.
116	* @param pVM The VM to operate on.
117	*/
118	VMMR3DECL(int) EMR3Init(PVM pVM)
119	{
120	LogFlow(("EMR3Init\n"));
121	/*
122	* Assert alignment and sizes.
123	*/
124	AssertCompileMemberAlignment(VM, em.s, 32);
125	AssertCompile(sizeof(pVM->em.s) <= sizeof(pVM->em.padding));
126	AssertCompile(sizeof(pVM->aCpus[0].em.s.u.FatalLongJump) <= sizeof(pVM->aCpus[0].em.s.u.achPaddingFatalLongJump));
127	AssertCompileMemberAlignment(EM, CritSectREM, sizeof(uintptr_t));
128
129	/*
130	* Init the structure.
131	*/
132	pVM->em.s.offVM = RT_OFFSETOF(VM, em.s);
133	int rc = CFGMR3QueryBool(CFGMR3GetRoot(pVM), "RawR3Enabled", &pVM->fRawR3Enabled);
134	if (RT_FAILURE(rc))
135	pVM->fRawR3Enabled = true;
136	rc = CFGMR3QueryBool(CFGMR3GetRoot(pVM), "RawR0Enabled", &pVM->fRawR0Enabled);
137	if (RT_FAILURE(rc))
138	pVM->fRawR0Enabled = true;
139	Log(("EMR3Init: fRawR3Enabled=%d fRawR0Enabled=%d\n", pVM->fRawR3Enabled, pVM->fRawR0Enabled));
140
141	/*
142	* Initialize the REM critical section.
143	*/
144	rc = PDMR3CritSectInit(pVM, &pVM->em.s.CritSectREM, "EM-REM");
145	AssertRCReturn(rc, rc);
146
147	/*
148	* Saved state.
149	*/
150	rc = SSMR3RegisterInternal(pVM, "em", 0, EM_SAVED_STATE_VERSION, 16,
151	NULL, emR3Save, NULL,
152	NULL, emR3Load, NULL);
153	if (RT_FAILURE(rc))
154	return rc;
155
156	for (unsigned i=0;i<pVM->cCPUs;i++)
157	{
158	PVMCPU pVCpu = &pVM->aCpus[i];
159
160	pVCpu->em.s.offVMCPU = RT_OFFSETOF(VMCPU, em.s);
161
162	pVCpu->em.s.enmState = (i == 0) ? EMSTATE_NONE : EMSTATE_WAIT_SIPI;
163	pVCpu->em.s.enmPrevState = EMSTATE_NONE;
164	pVCpu->em.s.fForceRAW = false;
165
166	pVCpu->em.s.pCtx = CPUMQueryGuestCtxPtr(pVCpu);
167	pVCpu->em.s.pPatmGCState = PATMR3QueryGCStateHC(pVM);
168	AssertMsg(pVCpu->em.s.pPatmGCState, ("PATMR3QueryGCStateHC failed!\n"));
169
170	# define EM_REG_COUNTER(a, b, c) \
171	rc = STAMR3RegisterF(pVM, a, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, c, b, i); \
172	AssertRC(rc);
173
174	# define EM_REG_COUNTER_USED(a, b, c) \
175	rc = STAMR3RegisterF(pVM, a, STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, c, b, i); \
176	AssertRC(rc);
177
178	# define EM_REG_PROFILE(a, b, c) \
179	rc = STAMR3RegisterF(pVM, a, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL, c, b, i); \
180	AssertRC(rc);
181
182	# define EM_REG_PROFILE_ADV(a, b, c) \
183	rc = STAMR3RegisterF(pVM, a, STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL, c, b, i); \
184	AssertRC(rc);
185
186	/*
187	* Statistics.
188	*/
189	#ifdef VBOX_WITH_STATISTICS
190	PEMSTATS pStats;
191	rc = MMHyperAlloc(pVM, sizeof(pStats), 0, MM_TAG_EM, (void *)&pStats);
192	if (RT_FAILURE(rc))
193	return rc;
194
195	pVCpu->em.s.pStatsR3 = pStats;
196	pVCpu->em.s.pStatsR0 = MMHyperR3ToR0(pVM, pStats);
197	pVCpu->em.s.pStatsRC = MMHyperR3ToRC(pVM, pStats);
198
199	EM_REG_PROFILE(&pStats->StatRZEmulate, "/EM/CPU%d/RZ/Interpret", "Profiling of EMInterpretInstruction.");
200	EM_REG_PROFILE(&pStats->StatR3Emulate, "/EM/CPU%d/R3/Interpret", "Profiling of EMInterpretInstruction.");
201
202	EM_REG_PROFILE(&pStats->StatRZInterpretSucceeded, "/EM/CPU%d/RZ/Interpret/Success", "The number of times an instruction was successfully interpreted.");
203	EM_REG_PROFILE(&pStats->StatR3InterpretSucceeded, "/EM/CPU%d/R3/Interpret/Success", "The number of times an instruction was successfully interpreted.");
204
205	EM_REG_COUNTER_USED(&pStats->StatRZAnd, "/EM/CPU%d/RZ/Interpret/Success/And", "The number of times AND was successfully interpreted.");
206	EM_REG_COUNTER_USED(&pStats->StatR3And, "/EM/CPU%d/R3/Interpret/Success/And", "The number of times AND was successfully interpreted.");
207	EM_REG_COUNTER_USED(&pStats->StatRZAdd, "/EM/CPU%d/RZ/Interpret/Success/Add", "The number of times ADD was successfully interpreted.");
208	EM_REG_COUNTER_USED(&pStats->StatR3Add, "/EM/CPU%d/R3/Interpret/Success/Add", "The number of times ADD was successfully interpreted.");
209	EM_REG_COUNTER_USED(&pStats->StatRZAdc, "/EM/CPU%d/RZ/Interpret/Success/Adc", "The number of times ADC was successfully interpreted.");
210	EM_REG_COUNTER_USED(&pStats->StatR3Adc, "/EM/CPU%d/R3/Interpret/Success/Adc", "The number of times ADC was successfully interpreted.");
211	EM_REG_COUNTER_USED(&pStats->StatRZSub, "/EM/CPU%d/RZ/Interpret/Success/Sub", "The number of times SUB was successfully interpreted.");
212	EM_REG_COUNTER_USED(&pStats->StatR3Sub, "/EM/CPU%d/R3/Interpret/Success/Sub", "The number of times SUB was successfully interpreted.");
213	EM_REG_COUNTER_USED(&pStats->StatRZCpuId, "/EM/CPU%d/RZ/Interpret/Success/CpuId", "The number of times CPUID was successfully interpreted.");
214	EM_REG_COUNTER_USED(&pStats->StatR3CpuId, "/EM/CPU%d/R3/Interpret/Success/CpuId", "The number of times CPUID was successfully interpreted.");
215	EM_REG_COUNTER_USED(&pStats->StatRZDec, "/EM/CPU%d/RZ/Interpret/Success/Dec", "The number of times DEC was successfully interpreted.");
216	EM_REG_COUNTER_USED(&pStats->StatR3Dec, "/EM/CPU%d/R3/Interpret/Success/Dec", "The number of times DEC was successfully interpreted.");
217	EM_REG_COUNTER_USED(&pStats->StatRZHlt, "/EM/CPU%d/RZ/Interpret/Success/Hlt", "The number of times HLT was successfully interpreted.");
218	EM_REG_COUNTER_USED(&pStats->StatR3Hlt, "/EM/CPU%d/R3/Interpret/Success/Hlt", "The number of times HLT was successfully interpreted.");
219	EM_REG_COUNTER_USED(&pStats->StatRZInc, "/EM/CPU%d/RZ/Interpret/Success/Inc", "The number of times INC was successfully interpreted.");
220	EM_REG_COUNTER_USED(&pStats->StatR3Inc, "/EM/CPU%d/R3/Interpret/Success/Inc", "The number of times INC was successfully interpreted.");
221	EM_REG_COUNTER_USED(&pStats->StatRZInvlPg, "/EM/CPU%d/RZ/Interpret/Success/Invlpg", "The number of times INVLPG was successfully interpreted.");
222	EM_REG_COUNTER_USED(&pStats->StatR3InvlPg, "/EM/CPU%d/R3/Interpret/Success/Invlpg", "The number of times INVLPG was successfully interpreted.");
223	EM_REG_COUNTER_USED(&pStats->StatRZIret, "/EM/CPU%d/RZ/Interpret/Success/Iret", "The number of times IRET was successfully interpreted.");
224	EM_REG_COUNTER_USED(&pStats->StatR3Iret, "/EM/CPU%d/R3/Interpret/Success/Iret", "The number of times IRET was successfully interpreted.");
225	EM_REG_COUNTER_USED(&pStats->StatRZLLdt, "/EM/CPU%d/RZ/Interpret/Success/LLdt", "The number of times LLDT was successfully interpreted.");
226	EM_REG_COUNTER_USED(&pStats->StatR3LLdt, "/EM/CPU%d/R3/Interpret/Success/LLdt", "The number of times LLDT was successfully interpreted.");
227	EM_REG_COUNTER_USED(&pStats->StatRZLIdt, "/EM/CPU%d/RZ/Interpret/Success/LIdt", "The number of times LIDT was successfully interpreted.");
228	EM_REG_COUNTER_USED(&pStats->StatR3LIdt, "/EM/CPU%d/R3/Interpret/Success/LIdt", "The number of times LIDT was successfully interpreted.");
229	EM_REG_COUNTER_USED(&pStats->StatRZLGdt, "/EM/CPU%d/RZ/Interpret/Success/LGdt", "The number of times LGDT was successfully interpreted.");
230	EM_REG_COUNTER_USED(&pStats->StatR3LGdt, "/EM/CPU%d/R3/Interpret/Success/LGdt", "The number of times LGDT was successfully interpreted.");
231	EM_REG_COUNTER_USED(&pStats->StatRZMov, "/EM/CPU%d/RZ/Interpret/Success/Mov", "The number of times MOV was successfully interpreted.");
232	EM_REG_COUNTER_USED(&pStats->StatR3Mov, "/EM/CPU%d/R3/Interpret/Success/Mov", "The number of times MOV was successfully interpreted.");
233	EM_REG_COUNTER_USED(&pStats->StatRZMovCRx, "/EM/CPU%d/RZ/Interpret/Success/MovCRx", "The number of times MOV CRx was successfully interpreted.");
234	EM_REG_COUNTER_USED(&pStats->StatR3MovCRx, "/EM/CPU%d/R3/Interpret/Success/MovCRx", "The number of times MOV CRx was successfully interpreted.");
235	EM_REG_COUNTER_USED(&pStats->StatRZMovDRx, "/EM/CPU%d/RZ/Interpret/Success/MovDRx", "The number of times MOV DRx was successfully interpreted.");
236	EM_REG_COUNTER_USED(&pStats->StatR3MovDRx, "/EM/CPU%d/R3/Interpret/Success/MovDRx", "The number of times MOV DRx was successfully interpreted.");
237	EM_REG_COUNTER_USED(&pStats->StatRZOr, "/EM/CPU%d/RZ/Interpret/Success/Or", "The number of times OR was successfully interpreted.");
238	EM_REG_COUNTER_USED(&pStats->StatR3Or, "/EM/CPU%d/R3/Interpret/Success/Or", "The number of times OR was successfully interpreted.");
239	EM_REG_COUNTER_USED(&pStats->StatRZPop, "/EM/CPU%d/RZ/Interpret/Success/Pop", "The number of times POP was successfully interpreted.");
240	EM_REG_COUNTER_USED(&pStats->StatR3Pop, "/EM/CPU%d/R3/Interpret/Success/Pop", "The number of times POP was successfully interpreted.");
241	EM_REG_COUNTER_USED(&pStats->StatRZRdtsc, "/EM/CPU%d/RZ/Interpret/Success/Rdtsc", "The number of times RDTSC was successfully interpreted.");
242	EM_REG_COUNTER_USED(&pStats->StatR3Rdtsc, "/EM/CPU%d/R3/Interpret/Success/Rdtsc", "The number of times RDTSC was successfully interpreted.");
243	EM_REG_COUNTER_USED(&pStats->StatRZRdpmc, "/EM/CPU%d/RZ/Interpret/Success/Rdpmc", "The number of times RDPMC was successfully interpreted.");
244	EM_REG_COUNTER_USED(&pStats->StatR3Rdpmc, "/EM/CPU%d/R3/Interpret/Success/Rdpmc", "The number of times RDPMC was successfully interpreted.");
245	EM_REG_COUNTER_USED(&pStats->StatRZSti, "/EM/CPU%d/RZ/Interpret/Success/Sti", "The number of times STI was successfully interpreted.");
246	EM_REG_COUNTER_USED(&pStats->StatR3Sti, "/EM/CPU%d/R3/Interpret/Success/Sti", "The number of times STI was successfully interpreted.");
247	EM_REG_COUNTER_USED(&pStats->StatRZXchg, "/EM/CPU%d/RZ/Interpret/Success/Xchg", "The number of times XCHG was successfully interpreted.");
248	EM_REG_COUNTER_USED(&pStats->StatR3Xchg, "/EM/CPU%d/R3/Interpret/Success/Xchg", "The number of times XCHG was successfully interpreted.");
249	EM_REG_COUNTER_USED(&pStats->StatRZXor, "/EM/CPU%d/RZ/Interpret/Success/Xor", "The number of times XOR was successfully interpreted.");
250	EM_REG_COUNTER_USED(&pStats->StatR3Xor, "/EM/CPU%d/R3/Interpret/Success/Xor", "The number of times XOR was successfully interpreted.");
251	EM_REG_COUNTER_USED(&pStats->StatRZMonitor, "/EM/CPU%d/RZ/Interpret/Success/Monitor", "The number of times MONITOR was successfully interpreted.");
252	EM_REG_COUNTER_USED(&pStats->StatR3Monitor, "/EM/CPU%d/R3/Interpret/Success/Monitor", "The number of times MONITOR was successfully interpreted.");
253	EM_REG_COUNTER_USED(&pStats->StatRZMWait, "/EM/CPU%d/RZ/Interpret/Success/MWait", "The number of times MWAIT was successfully interpreted.");
254	EM_REG_COUNTER_USED(&pStats->StatR3MWait, "/EM/CPU%d/R3/Interpret/Success/MWait", "The number of times MWAIT was successfully interpreted.");
255	EM_REG_COUNTER_USED(&pStats->StatRZBtr, "/EM/CPU%d/RZ/Interpret/Success/Btr", "The number of times BTR was successfully interpreted.");
256	EM_REG_COUNTER_USED(&pStats->StatR3Btr, "/EM/CPU%d/R3/Interpret/Success/Btr", "The number of times BTR was successfully interpreted.");
257	EM_REG_COUNTER_USED(&pStats->StatRZBts, "/EM/CPU%d/RZ/Interpret/Success/Bts", "The number of times BTS was successfully interpreted.");
258	EM_REG_COUNTER_USED(&pStats->StatR3Bts, "/EM/CPU%d/R3/Interpret/Success/Bts", "The number of times BTS was successfully interpreted.");
259	EM_REG_COUNTER_USED(&pStats->StatRZBtc, "/EM/CPU%d/RZ/Interpret/Success/Btc", "The number of times BTC was successfully interpreted.");
260	EM_REG_COUNTER_USED(&pStats->StatR3Btc, "/EM/CPU%d/R3/Interpret/Success/Btc", "The number of times BTC was successfully interpreted.");
261	EM_REG_COUNTER_USED(&pStats->StatRZCmpXchg, "/EM/CPU%d/RZ/Interpret/Success/CmpXchg", "The number of times CMPXCHG was successfully interpreted.");
262	EM_REG_COUNTER_USED(&pStats->StatR3CmpXchg, "/EM/CPU%d/R3/Interpret/Success/CmpXchg", "The number of times CMPXCHG was successfully interpreted.");
263	EM_REG_COUNTER_USED(&pStats->StatRZCmpXchg8b, "/EM/CPU%d/RZ/Interpret/Success/CmpXchg8b", "The number of times CMPXCHG8B was successfully interpreted.");
264	EM_REG_COUNTER_USED(&pStats->StatR3CmpXchg8b, "/EM/CPU%d/R3/Interpret/Success/CmpXchg8b", "The number of times CMPXCHG8B was successfully interpreted.");
265	EM_REG_COUNTER_USED(&pStats->StatRZXAdd, "/EM/CPU%d/RZ/Interpret/Success/XAdd", "The number of times XADD was successfully interpreted.");
266	EM_REG_COUNTER_USED(&pStats->StatR3XAdd, "/EM/CPU%d/R3/Interpret/Success/XAdd", "The number of times XADD was successfully interpreted.");
267	EM_REG_COUNTER_USED(&pStats->StatR3Rdmsr, "/EM/CPU%d/R3/Interpret/Success/Rdmsr", "The number of times RDMSR was successfully interpreted.");
268	EM_REG_COUNTER_USED(&pStats->StatRZRdmsr, "/EM/CPU%d/RZ/Interpret/Success/Rdmsr", "The number of times RDMSR was successfully interpreted.");
269	EM_REG_COUNTER_USED(&pStats->StatR3Wrmsr, "/EM/CPU%d/R3/Interpret/Success/Wrmsr", "The number of times WRMSR was successfully interpreted.");
270	EM_REG_COUNTER_USED(&pStats->StatRZWrmsr, "/EM/CPU%d/RZ/Interpret/Success/Wrmsr", "The number of times WRMSR was successfully interpreted.");
271	EM_REG_COUNTER_USED(&pStats->StatR3StosWD, "/EM/CPU%d/R3/Interpret/Success/Stoswd", "The number of times STOSWD was successfully interpreted.");
272	EM_REG_COUNTER_USED(&pStats->StatRZStosWD, "/EM/CPU%d/RZ/Interpret/Success/Stoswd", "The number of times STOSWD was successfully interpreted.");
273	EM_REG_COUNTER_USED(&pStats->StatRZWbInvd, "/EM/CPU%d/RZ/Interpret/Success/WbInvd", "The number of times WBINVD was successfully interpreted.");
274	EM_REG_COUNTER_USED(&pStats->StatR3WbInvd, "/EM/CPU%d/R3/Interpret/Success/WbInvd", "The number of times WBINVD was successfully interpreted.");
275	EM_REG_COUNTER_USED(&pStats->StatRZLmsw, "/EM/CPU%d/RZ/Interpret/Success/Lmsw", "The number of times LMSW was successfully interpreted.");
276	EM_REG_COUNTER_USED(&pStats->StatR3Lmsw, "/EM/CPU%d/R3/Interpret/Success/Lmsw", "The number of times LMSW was successfully interpreted.");
277
278	EM_REG_COUNTER(&pStats->StatRZInterpretFailed, "/EM/CPU%d/RZ/Interpret/Failed", "The number of times an instruction was not interpreted.");
279	EM_REG_COUNTER(&pStats->StatR3InterpretFailed, "/EM/CPU%d/R3/Interpret/Failed", "The number of times an instruction was not interpreted.");
280
281	EM_REG_COUNTER_USED(&pStats->StatRZFailedAnd, "/EM/CPU%d/RZ/Interpret/Failed/And", "The number of times AND was not interpreted.");
282	EM_REG_COUNTER_USED(&pStats->StatR3FailedAnd, "/EM/CPU%d/R3/Interpret/Failed/And", "The number of times AND was not interpreted.");
283	EM_REG_COUNTER_USED(&pStats->StatRZFailedCpuId, "/EM/CPU%d/RZ/Interpret/Failed/CpuId", "The number of times CPUID was not interpreted.");
284	EM_REG_COUNTER_USED(&pStats->StatR3FailedCpuId, "/EM/CPU%d/R3/Interpret/Failed/CpuId", "The number of times CPUID was not interpreted.");
285	EM_REG_COUNTER_USED(&pStats->StatRZFailedDec, "/EM/CPU%d/RZ/Interpret/Failed/Dec", "The number of times DEC was not interpreted.");
286	EM_REG_COUNTER_USED(&pStats->StatR3FailedDec, "/EM/CPU%d/R3/Interpret/Failed/Dec", "The number of times DEC was not interpreted.");
287	EM_REG_COUNTER_USED(&pStats->StatRZFailedHlt, "/EM/CPU%d/RZ/Interpret/Failed/Hlt", "The number of times HLT was not interpreted.");
288	EM_REG_COUNTER_USED(&pStats->StatR3FailedHlt, "/EM/CPU%d/R3/Interpret/Failed/Hlt", "The number of times HLT was not interpreted.");
289	EM_REG_COUNTER_USED(&pStats->StatRZFailedInc, "/EM/CPU%d/RZ/Interpret/Failed/Inc", "The number of times INC was not interpreted.");
290	EM_REG_COUNTER_USED(&pStats->StatR3FailedInc, "/EM/CPU%d/R3/Interpret/Failed/Inc", "The number of times INC was not interpreted.");
291	EM_REG_COUNTER_USED(&pStats->StatRZFailedInvlPg, "/EM/CPU%d/RZ/Interpret/Failed/InvlPg", "The number of times INVLPG was not interpreted.");
292	EM_REG_COUNTER_USED(&pStats->StatR3FailedInvlPg, "/EM/CPU%d/R3/Interpret/Failed/InvlPg", "The number of times INVLPG was not interpreted.");
293	EM_REG_COUNTER_USED(&pStats->StatRZFailedIret, "/EM/CPU%d/RZ/Interpret/Failed/Iret", "The number of times IRET was not interpreted.");
294	EM_REG_COUNTER_USED(&pStats->StatR3FailedIret, "/EM/CPU%d/R3/Interpret/Failed/Iret", "The number of times IRET was not interpreted.");
295	EM_REG_COUNTER_USED(&pStats->StatRZFailedLLdt, "/EM/CPU%d/RZ/Interpret/Failed/LLdt", "The number of times LLDT was not interpreted.");
296	EM_REG_COUNTER_USED(&pStats->StatR3FailedLLdt, "/EM/CPU%d/R3/Interpret/Failed/LLdt", "The number of times LLDT was not interpreted.");
297	EM_REG_COUNTER_USED(&pStats->StatRZFailedLIdt, "/EM/CPU%d/RZ/Interpret/Failed/LIdt", "The number of times LIDT was not interpreted.");
298	EM_REG_COUNTER_USED(&pStats->StatR3FailedLIdt, "/EM/CPU%d/R3/Interpret/Failed/LIdt", "The number of times LIDT was not interpreted.");
299	EM_REG_COUNTER_USED(&pStats->StatRZFailedLGdt, "/EM/CPU%d/RZ/Interpret/Failed/LGdt", "The number of times LGDT was not interpreted.");
300	EM_REG_COUNTER_USED(&pStats->StatR3FailedLGdt, "/EM/CPU%d/R3/Interpret/Failed/LGdt", "The number of times LGDT was not interpreted.");
301	EM_REG_COUNTER_USED(&pStats->StatRZFailedMov, "/EM/CPU%d/RZ/Interpret/Failed/Mov", "The number of times MOV was not interpreted.");
302	EM_REG_COUNTER_USED(&pStats->StatR3FailedMov, "/EM/CPU%d/R3/Interpret/Failed/Mov", "The number of times MOV was not interpreted.");
303	EM_REG_COUNTER_USED(&pStats->StatRZFailedMovCRx, "/EM/CPU%d/RZ/Interpret/Failed/MovCRx", "The number of times MOV CRx was not interpreted.");
304	EM_REG_COUNTER_USED(&pStats->StatR3FailedMovCRx, "/EM/CPU%d/R3/Interpret/Failed/MovCRx", "The number of times MOV CRx was not interpreted.");
305	EM_REG_COUNTER_USED(&pStats->StatRZFailedMovDRx, "/EM/CPU%d/RZ/Interpret/Failed/MovDRx", "The number of times MOV DRx was not interpreted.");
306	EM_REG_COUNTER_USED(&pStats->StatR3FailedMovDRx, "/EM/CPU%d/R3/Interpret/Failed/MovDRx", "The number of times MOV DRx was not interpreted.");
307	EM_REG_COUNTER_USED(&pStats->StatRZFailedOr, "/EM/CPU%d/RZ/Interpret/Failed/Or", "The number of times OR was not interpreted.");
308	EM_REG_COUNTER_USED(&pStats->StatR3FailedOr, "/EM/CPU%d/R3/Interpret/Failed/Or", "The number of times OR was not interpreted.");
309	EM_REG_COUNTER_USED(&pStats->StatRZFailedPop, "/EM/CPU%d/RZ/Interpret/Failed/Pop", "The number of times POP was not interpreted.");
310	EM_REG_COUNTER_USED(&pStats->StatR3FailedPop, "/EM/CPU%d/R3/Interpret/Failed/Pop", "The number of times POP was not interpreted.");
311	EM_REG_COUNTER_USED(&pStats->StatRZFailedSti, "/EM/CPU%d/RZ/Interpret/Failed/Sti", "The number of times STI was not interpreted.");
312	EM_REG_COUNTER_USED(&pStats->StatR3FailedSti, "/EM/CPU%d/R3/Interpret/Failed/Sti", "The number of times STI was not interpreted.");
313	EM_REG_COUNTER_USED(&pStats->StatRZFailedXchg, "/EM/CPU%d/RZ/Interpret/Failed/Xchg", "The number of times XCHG was not interpreted.");
314	EM_REG_COUNTER_USED(&pStats->StatR3FailedXchg, "/EM/CPU%d/R3/Interpret/Failed/Xchg", "The number of times XCHG was not interpreted.");
315	EM_REG_COUNTER_USED(&pStats->StatRZFailedXor, "/EM/CPU%d/RZ/Interpret/Failed/Xor", "The number of times XOR was not interpreted.");
316	EM_REG_COUNTER_USED(&pStats->StatR3FailedXor, "/EM/CPU%d/R3/Interpret/Failed/Xor", "The number of times XOR was not interpreted.");
317	EM_REG_COUNTER_USED(&pStats->StatRZFailedMonitor, "/EM/CPU%d/RZ/Interpret/Failed/Monitor", "The number of times MONITOR was not interpreted.");
318	EM_REG_COUNTER_USED(&pStats->StatR3FailedMonitor, "/EM/CPU%d/R3/Interpret/Failed/Monitor", "The number of times MONITOR was not interpreted.");
319	EM_REG_COUNTER_USED(&pStats->StatRZFailedMWait, "/EM/CPU%d/RZ/Interpret/Failed/MWait", "The number of times MONITOR was not interpreted.");
320	EM_REG_COUNTER_USED(&pStats->StatR3FailedMWait, "/EM/CPU%d/R3/Interpret/Failed/MWait", "The number of times MONITOR was not interpreted.");
321	EM_REG_COUNTER_USED(&pStats->StatRZFailedRdtsc, "/EM/CPU%d/RZ/Interpret/Failed/Rdtsc", "The number of times RDTSC was not interpreted.");
322	EM_REG_COUNTER_USED(&pStats->StatR3FailedRdtsc, "/EM/CPU%d/R3/Interpret/Failed/Rdtsc", "The number of times RDTSC was not interpreted.");
323	EM_REG_COUNTER_USED(&pStats->StatRZFailedRdpmc, "/EM/CPU%d/RZ/Interpret/Failed/Rdpmc", "The number of times RDPMC was not interpreted.");
324	EM_REG_COUNTER_USED(&pStats->StatR3FailedRdpmc, "/EM/CPU%d/R3/Interpret/Failed/Rdpmc", "The number of times RDPMC was not interpreted.");
325	EM_REG_COUNTER_USED(&pStats->StatRZFailedRdmsr, "/EM/CPU%d/RZ/Interpret/Failed/Rdmsr", "The number of times RDMSR was not interpreted.");
326	EM_REG_COUNTER_USED(&pStats->StatR3FailedRdmsr, "/EM/CPU%d/R3/Interpret/Failed/Rdmsr", "The number of times RDMSR was not interpreted.");
327	EM_REG_COUNTER_USED(&pStats->StatRZFailedWrmsr, "/EM/CPU%d/RZ/Interpret/Failed/Wrmsr", "The number of times WRMSR was not interpreted.");
328	EM_REG_COUNTER_USED(&pStats->StatR3FailedWrmsr, "/EM/CPU%d/R3/Interpret/Failed/Wrmsr", "The number of times WRMSR was not interpreted.");
329	EM_REG_COUNTER_USED(&pStats->StatRZFailedLmsw, "/EM/CPU%d/RZ/Interpret/Failed/Lmsw", "The number of times LMSW was not interpreted.");
330	EM_REG_COUNTER_USED(&pStats->StatR3FailedLmsw, "/EM/CPU%d/R3/Interpret/Failed/Lmsw", "The number of times LMSW was not interpreted.");
331
332	EM_REG_COUNTER_USED(&pStats->StatRZFailedMisc, "/EM/CPU%d/RZ/Interpret/Failed/Misc", "The number of times some misc instruction was encountered.");
333	EM_REG_COUNTER_USED(&pStats->StatR3FailedMisc, "/EM/CPU%d/R3/Interpret/Failed/Misc", "The number of times some misc instruction was encountered.");
334	EM_REG_COUNTER_USED(&pStats->StatRZFailedAdd, "/EM/CPU%d/RZ/Interpret/Failed/Add", "The number of times ADD was not interpreted.");
335	EM_REG_COUNTER_USED(&pStats->StatR3FailedAdd, "/EM/CPU%d/R3/Interpret/Failed/Add", "The number of times ADD was not interpreted.");
336	EM_REG_COUNTER_USED(&pStats->StatRZFailedAdc, "/EM/CPU%d/RZ/Interpret/Failed/Adc", "The number of times ADC was not interpreted.");
337	EM_REG_COUNTER_USED(&pStats->StatR3FailedAdc, "/EM/CPU%d/R3/Interpret/Failed/Adc", "The number of times ADC was not interpreted.");
338	EM_REG_COUNTER_USED(&pStats->StatRZFailedBtr, "/EM/CPU%d/RZ/Interpret/Failed/Btr", "The number of times BTR was not interpreted.");
339	EM_REG_COUNTER_USED(&pStats->StatR3FailedBtr, "/EM/CPU%d/R3/Interpret/Failed/Btr", "The number of times BTR was not interpreted.");
340	EM_REG_COUNTER_USED(&pStats->StatRZFailedBts, "/EM/CPU%d/RZ/Interpret/Failed/Bts", "The number of times BTS was not interpreted.");
341	EM_REG_COUNTER_USED(&pStats->StatR3FailedBts, "/EM/CPU%d/R3/Interpret/Failed/Bts", "The number of times BTS was not interpreted.");
342	EM_REG_COUNTER_USED(&pStats->StatRZFailedBtc, "/EM/CPU%d/RZ/Interpret/Failed/Btc", "The number of times BTC was not interpreted.");
343	EM_REG_COUNTER_USED(&pStats->StatR3FailedBtc, "/EM/CPU%d/R3/Interpret/Failed/Btc", "The number of times BTC was not interpreted.");
344	EM_REG_COUNTER_USED(&pStats->StatRZFailedCli, "/EM/CPU%d/RZ/Interpret/Failed/Cli", "The number of times CLI was not interpreted.");
345	EM_REG_COUNTER_USED(&pStats->StatR3FailedCli, "/EM/CPU%d/R3/Interpret/Failed/Cli", "The number of times CLI was not interpreted.");
346	EM_REG_COUNTER_USED(&pStats->StatRZFailedCmpXchg, "/EM/CPU%d/RZ/Interpret/Failed/CmpXchg", "The number of times CMPXCHG was not interpreted.");
347	EM_REG_COUNTER_USED(&pStats->StatR3FailedCmpXchg, "/EM/CPU%d/R3/Interpret/Failed/CmpXchg", "The number of times CMPXCHG was not interpreted.");
348	EM_REG_COUNTER_USED(&pStats->StatRZFailedCmpXchg8b, "/EM/CPU%d/RZ/Interpret/Failed/CmpXchg8b", "The number of times CMPXCHG8B was not interpreted.");
349	EM_REG_COUNTER_USED(&pStats->StatR3FailedCmpXchg8b, "/EM/CPU%d/R3/Interpret/Failed/CmpXchg8b", "The number of times CMPXCHG8B was not interpreted.");
350	EM_REG_COUNTER_USED(&pStats->StatRZFailedXAdd, "/EM/CPU%d/RZ/Interpret/Failed/XAdd", "The number of times XADD was not interpreted.");
351	EM_REG_COUNTER_USED(&pStats->StatR3FailedXAdd, "/EM/CPU%d/R3/Interpret/Failed/XAdd", "The number of times XADD was not interpreted.");
352	EM_REG_COUNTER_USED(&pStats->StatRZFailedMovNTPS, "/EM/CPU%d/RZ/Interpret/Failed/MovNTPS", "The number of times MOVNTPS was not interpreted.");
353	EM_REG_COUNTER_USED(&pStats->StatR3FailedMovNTPS, "/EM/CPU%d/R3/Interpret/Failed/MovNTPS", "The number of times MOVNTPS was not interpreted.");
354	EM_REG_COUNTER_USED(&pStats->StatRZFailedStosWD, "/EM/CPU%d/RZ/Interpret/Failed/StosWD", "The number of times STOSWD was not interpreted.");
355	EM_REG_COUNTER_USED(&pStats->StatR3FailedStosWD, "/EM/CPU%d/R3/Interpret/Failed/StosWD", "The number of times STOSWD was not interpreted.");
356	EM_REG_COUNTER_USED(&pStats->StatRZFailedSub, "/EM/CPU%d/RZ/Interpret/Failed/Sub", "The number of times SUB was not interpreted.");
357	EM_REG_COUNTER_USED(&pStats->StatR3FailedSub, "/EM/CPU%d/R3/Interpret/Failed/Sub", "The number of times SUB was not interpreted.");
358	EM_REG_COUNTER_USED(&pStats->StatRZFailedWbInvd, "/EM/CPU%d/RZ/Interpret/Failed/WbInvd", "The number of times WBINVD was not interpreted.");
359	EM_REG_COUNTER_USED(&pStats->StatR3FailedWbInvd, "/EM/CPU%d/R3/Interpret/Failed/WbInvd", "The number of times WBINVD was not interpreted.");
360
361	EM_REG_COUNTER_USED(&pStats->StatRZFailedUserMode, "/EM/CPU%d/RZ/Interpret/Failed/UserMode", "The number of rejections because of CPL.");
362	EM_REG_COUNTER_USED(&pStats->StatR3FailedUserMode, "/EM/CPU%d/R3/Interpret/Failed/UserMode", "The number of rejections because of CPL.");
363	EM_REG_COUNTER_USED(&pStats->StatRZFailedPrefix, "/EM/CPU%d/RZ/Interpret/Failed/Prefix", "The number of rejections because of prefix .");
364	EM_REG_COUNTER_USED(&pStats->StatR3FailedPrefix, "/EM/CPU%d/R3/Interpret/Failed/Prefix", "The number of rejections because of prefix .");
365
366	EM_REG_COUNTER_USED(&pStats->StatCli, "/EM/CPU%d/R3/PrivInst/Cli", "Number of cli instructions.");
367	EM_REG_COUNTER_USED(&pStats->StatSti, "/EM/CPU%d/R3/PrivInst/Sti", "Number of sli instructions.");
368	EM_REG_COUNTER_USED(&pStats->StatIn, "/EM/CPU%d/R3/PrivInst/In", "Number of in instructions.");
369	EM_REG_COUNTER_USED(&pStats->StatOut, "/EM/CPU%d/R3/PrivInst/Out", "Number of out instructions.");
370	EM_REG_COUNTER_USED(&pStats->StatHlt, "/EM/CPU%d/R3/PrivInst/Hlt", "Number of hlt instructions not handled in GC because of PATM.");
371	EM_REG_COUNTER_USED(&pStats->StatInvlpg, "/EM/CPU%d/R3/PrivInst/Invlpg", "Number of invlpg instructions.");
372	EM_REG_COUNTER_USED(&pStats->StatMisc, "/EM/CPU%d/R3/PrivInst/Misc", "Number of misc. instructions.");
373	EM_REG_COUNTER_USED(&pStats->StatMovWriteCR[0], "/EM/CPU%d/R3/PrivInst/Mov CR0, X", "Number of mov CR0 read instructions.");
374	EM_REG_COUNTER_USED(&pStats->StatMovWriteCR[1], "/EM/CPU%d/R3/PrivInst/Mov CR1, X", "Number of mov CR1 read instructions.");
375	EM_REG_COUNTER_USED(&pStats->StatMovWriteCR[2], "/EM/CPU%d/R3/PrivInst/Mov CR2, X", "Number of mov CR2 read instructions.");
376	EM_REG_COUNTER_USED(&pStats->StatMovWriteCR[3], "/EM/CPU%d/R3/PrivInst/Mov CR3, X", "Number of mov CR3 read instructions.");
377	EM_REG_COUNTER_USED(&pStats->StatMovWriteCR[4], "/EM/CPU%d/R3/PrivInst/Mov CR4, X", "Number of mov CR4 read instructions.");
378	EM_REG_COUNTER_USED(&pStats->StatMovReadCR[0], "/EM/CPU%d/R3/PrivInst/Mov X, CR0", "Number of mov CR0 write instructions.");
379	EM_REG_COUNTER_USED(&pStats->StatMovReadCR[1], "/EM/CPU%d/R3/PrivInst/Mov X, CR1", "Number of mov CR1 write instructions.");
380	EM_REG_COUNTER_USED(&pStats->StatMovReadCR[2], "/EM/CPU%d/R3/PrivInst/Mov X, CR2", "Number of mov CR2 write instructions.");
381	EM_REG_COUNTER_USED(&pStats->StatMovReadCR[3], "/EM/CPU%d/R3/PrivInst/Mov X, CR3", "Number of mov CR3 write instructions.");
382	EM_REG_COUNTER_USED(&pStats->StatMovReadCR[4], "/EM/CPU%d/R3/PrivInst/Mov X, CR4", "Number of mov CR4 write instructions.");
383	EM_REG_COUNTER_USED(&pStats->StatMovDRx, "/EM/CPU%d/R3/PrivInst/MovDRx", "Number of mov DRx instructions.");
384	EM_REG_COUNTER_USED(&pStats->StatIret, "/EM/CPU%d/R3/PrivInst/Iret", "Number of iret instructions.");
385	EM_REG_COUNTER_USED(&pStats->StatMovLgdt, "/EM/CPU%d/R3/PrivInst/Lgdt", "Number of lgdt instructions.");
386	EM_REG_COUNTER_USED(&pStats->StatMovLidt, "/EM/CPU%d/R3/PrivInst/Lidt", "Number of lidt instructions.");
387	EM_REG_COUNTER_USED(&pStats->StatMovLldt, "/EM/CPU%d/R3/PrivInst/Lldt", "Number of lldt instructions.");
388	EM_REG_COUNTER_USED(&pStats->StatSysEnter, "/EM/CPU%d/R3/PrivInst/Sysenter", "Number of sysenter instructions.");
389	EM_REG_COUNTER_USED(&pStats->StatSysExit, "/EM/CPU%d/R3/PrivInst/Sysexit", "Number of sysexit instructions.");
390	EM_REG_COUNTER_USED(&pStats->StatSysCall, "/EM/CPU%d/R3/PrivInst/Syscall", "Number of syscall instructions.");
391	EM_REG_COUNTER_USED(&pStats->StatSysRet, "/EM/CPU%d/R3/PrivInst/Sysret", "Number of sysret instructions.");
392
393	EM_REG_COUNTER(&pVCpu->em.s.StatTotalClis, "/EM/CPU%d/Cli/Total", "Total number of cli instructions executed.");
394	pVCpu->em.s.pCliStatTree = 0;
395
396	/* these should be considered for release statistics. */
397	EM_REG_COUNTER(&pVCpu->em.s.StatIOEmu, "/PROF/CPU%d/EM/Emulation/IO", "Profiling of emR3RawExecuteIOInstruction.");
398	EM_REG_COUNTER(&pVCpu->em.s.StatPrivEmu, "/PROF/CPU%d/EM/Emulation/Priv", "Profiling of emR3RawPrivileged.");
399	EM_REG_COUNTER(&pVCpu->em.s.StatMiscEmu, "/PROF/CPU%d/EM/Emulation/Misc", "Profiling of emR3RawExecuteInstruction.");
400	EM_REG_PROFILE(&pVCpu->em.s.StatHwAccEntry, "/PROF/CPU%d/EM/HwAccEnter", "Profiling Hardware Accelerated Mode entry overhead.");
401	EM_REG_PROFILE(&pVCpu->em.s.StatHwAccExec, "/PROF/CPU%d/EM/HwAccExec", "Profiling Hardware Accelerated Mode execution.");
402	EM_REG_PROFILE(&pVCpu->em.s.StatREMEmu, "/PROF/CPU%d/EM/REMEmuSingle", "Profiling single instruction REM execution.");
403	EM_REG_PROFILE(&pVCpu->em.s.StatREMExec, "/PROF/CPU%d/EM/REMExec", "Profiling REM execution.");
404	EM_REG_PROFILE(&pVCpu->em.s.StatREMSync, "/PROF/CPU%d/EM/REMSync", "Profiling REM context syncing.");
405	EM_REG_PROFILE(&pVCpu->em.s.StatRAWEntry, "/PROF/CPU%d/EM/RAWEnter", "Profiling Raw Mode entry overhead.");
406	EM_REG_PROFILE(&pVCpu->em.s.StatRAWExec, "/PROF/CPU%d/EM/RAWExec", "Profiling Raw Mode execution.");
407	EM_REG_PROFILE(&pVCpu->em.s.StatRAWTail, "/PROF/CPU%d/EM/RAWTail", "Profiling Raw Mode tail overhead.");
408
409	#endif /* VBOX_WITH_STATISTICS */
410
411	EM_REG_COUNTER(&pVCpu->em.s.StatForcedActions, "/PROF/CPU%d/EM/ForcedActions", "Profiling forced action execution.");
412	EM_REG_COUNTER(&pVCpu->em.s.StatHalted, "/PROF/CPU%d/EM/Halted", "Profiling halted state (VMR3WaitHalted).");
413	EM_REG_COUNTER(&pVCpu->em.s.StatREMTotal, "/PROF/CPU%d/EM/REMTotal", "Profiling emR3RemExecute (excluding FFs).");
414	EM_REG_COUNTER(&pVCpu->em.s.StatRAWTotal, "/PROF/CPU%d/EM/RAWTotal", "Profiling emR3RawExecute (excluding FFs).");
415
416	EM_REG_PROFILE_ADV(&pVCpu->em.s.StatTotal, "/PROF/CPU%d/EM/Total", "Profiling EMR3ExecuteVM.");
417	}
418
419	return VINF_SUCCESS;
420	}
421
422
423	/**
424	* Initializes the per-VCPU EM.
425	*
426	* @returns VBox status code.
427	* @param pVM The VM to operate on.
428	*/
429	VMMR3DECL(int) EMR3InitCPU(PVM pVM)
430	{
431	LogFlow(("EMR3InitCPU\n"));
432	return VINF_SUCCESS;
433	}
434
435
436	/**
437	* Applies relocations to data and code managed by this
438	* component. This function will be called at init and
439	* whenever the VMM need to relocate it self inside the GC.
440	*
441	* @param pVM The VM.
442	*/
443	VMMR3DECL(void) EMR3Relocate(PVM pVM)
444	{
445	LogFlow(("EMR3Relocate\n"));
446	for (unsigned i=0;i<pVM->cCPUs;i++)
447	{
448	PVMCPU pVCpu = &pVM->aCpus[i];
449
450	if (pVCpu->em.s.pStatsR3)
451	pVCpu->em.s.pStatsRC = MMHyperR3ToRC(pVM, pVCpu->em.s.pStatsR3);
452	}
453	}
454
455
456	/**
457	* Reset notification.
458	*
459	* @param pVM
460	*/
461	VMMR3DECL(void) EMR3Reset(PVM pVM)
462	{
463	LogFlow(("EMR3Reset: \n"));
464	for (unsigned i=0;i<pVM->cCPUs;i++)
465	{
466	PVMCPU pVCpu = &pVM->aCpus[i];
467
468	pVCpu->em.s.fForceRAW = false;
469	}
470	}
471
472
473	/**
474	* Terminates the EM.
475	*
476	* Termination means cleaning up and freeing all resources,
477	* the VM it self is at this point powered off or suspended.
478	*
479	* @returns VBox status code.
480	* @param pVM The VM to operate on.
481	*/
482	VMMR3DECL(int) EMR3Term(PVM pVM)
483	{
484	AssertMsg(pVM->em.s.offVM, ("bad init order!\n"));
485
486	PDMR3CritSectDelete(&pVM->em.s.CritSectREM);
487	return VINF_SUCCESS;
488	}
489
490	/**
491	* Terminates the per-VCPU EM.
492	*
493	* Termination means cleaning up and freeing all resources,
494	* the VM it self is at this point powered off or suspended.
495	*
496	* @returns VBox status code.
497	* @param pVM The VM to operate on.
498	*/
499	VMMR3DECL(int) EMR3TermCPU(PVM pVM)
500	{
501	return 0;
502	}
503
504	/**
505	* Execute state save operation.
506	*
507	* @returns VBox status code.
508	* @param pVM VM Handle.
509	* @param pSSM SSM operation handle.
510	*/
511	static DECLCALLBACK(int) emR3Save(PVM pVM, PSSMHANDLE pSSM)
512	{
513	for (VMCPUID i = 0; i < pVM->cCPUs; i++)
514	{
515	PVMCPU pVCpu = &pVM->aCpus[i];
516
517	int rc = SSMR3PutBool(pSSM, pVCpu->em.s.fForceRAW);
518	AssertRCReturn(rc, rc);
519
520	Assert(pVCpu->em.s.enmState == EMSTATE_SUSPENDED);
521	Assert(pVCpu->em.s.enmPrevState != EMSTATE_SUSPENDED);
522	rc = SSMR3PutU32(pSSM, pVCpu->em.s.enmPrevState);
523	AssertRCReturn(rc, rc);
524	}
525	return VINF_SUCCESS;
526	}
527
528
529	/**
530	* Execute state load operation.
531	*
532	* @returns VBox status code.
533	* @param pVM VM Handle.
534	* @param pSSM SSM operation handle.
535	* @param u32Version Data layout version.
536	*/
537	static DECLCALLBACK(int) emR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t u32Version)
538	{
539	int rc = VINF_SUCCESS;
540
541	/*
542	* Validate version.
543	*/
544	if ( u32Version != EM_SAVED_STATE_VERSION
545	&& u32Version != EM_SAVED_STATE_VERSION_PRE_SMP)
546	{
547	AssertMsgFailed(("emR3Load: Invalid version u32Version=%d (current %d)!\n", u32Version, EM_SAVED_STATE_VERSION));
548	return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
549	}
550
551	/*
552	* Load the saved state.
553	*/
554	for (VMCPUID i = 0; i < pVM->cCPUs; i++)
555	{
556	PVMCPU pVCpu = &pVM->aCpus[i];
557
558	rc = SSMR3GetBool(pSSM, &pVCpu->em.s.fForceRAW);
559	if (RT_FAILURE(rc))
560	pVCpu->em.s.fForceRAW = false;
561
562	if (u32Version > EM_SAVED_STATE_VERSION_PRE_SMP)
563	{
564	AssertCompile(sizeof(pVCpu->em.s.enmPrevState) == sizeof(uint32_t));
565	rc = SSMR3GetU32(pSSM, (uint32_t *)&pVCpu->em.s.enmPrevState);
566	AssertRCReturn(rc, rc);
567	Assert(pVCpu->em.s.enmPrevState != EMSTATE_SUSPENDED);
568
569	pVCpu->em.s.enmState = EMSTATE_SUSPENDED;
570	}
571	Assert(!pVCpu->em.s.pCliStatTree);
572	}
573	return rc;
574	}
575
576
577	/**
578	* Enables or disables a set of raw-mode execution modes.
579	*
580	* @returns VINF_SUCCESS on success.
581	* @returns VINF_RESCHEDULE if a rescheduling might be required.
582	* @returns VERR_INVALID_PARAMETER on an invalid enmMode value.
583	*
584	* @param pVM The VM to operate on.
585	* @param enmMode The execution mode change.
586	* @thread The emulation thread.
587	*/
588	VMMR3DECL(int) EMR3RawSetMode(PVM pVM, EMRAWMODE enmMode)
589	{
590	switch (enmMode)
591	{
592	case EMRAW_NONE:
593	pVM->fRawR3Enabled = false;
594	pVM->fRawR0Enabled = false;
595	break;
596	case EMRAW_RING3_ENABLE:
597	pVM->fRawR3Enabled = true;
598	break;
599	case EMRAW_RING3_DISABLE:
600	pVM->fRawR3Enabled = false;
601	break;
602	case EMRAW_RING0_ENABLE:
603	pVM->fRawR0Enabled = true;
604	break;
605	case EMRAW_RING0_DISABLE:
606	pVM->fRawR0Enabled = false;
607	break;
608	default:
609	AssertMsgFailed(("Invalid enmMode=%d\n", enmMode));
610	return VERR_INVALID_PARAMETER;
611	}
612	Log(("EMR3SetRawMode: fRawR3Enabled=%RTbool fRawR0Enabled=%RTbool\n",
613	pVM->fRawR3Enabled, pVM->fRawR0Enabled));
614	return pVM->aCpus[0].em.s.enmState == EMSTATE_RAW ? VINF_EM_RESCHEDULE : VINF_SUCCESS;
615	}
616
617
618	/**
619	* Raise a fatal error.
620	*
621	* Safely terminate the VM with full state report and stuff. This function
622	* will naturally never return.
623	*
624	* @param pVCpu VMCPU handle.
625	* @param rc VBox status code.
626	*/
627	VMMR3DECL(void) EMR3FatalError(PVMCPU pVCpu, int rc)
628	{
629	pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
630	longjmp(pVCpu->em.s.u.FatalLongJump, rc);
631	AssertReleaseMsgFailed(("longjmp returned!\n"));
632	}
633
634
635	/**
636	* Gets the EM state name.
637	*
638	* @returns pointer to read only state name,
639	* @param enmState The state.
640	*/
641	VMMR3DECL(const char *) EMR3GetStateName(EMSTATE enmState)
642	{
643	switch (enmState)
644	{
645	case EMSTATE_NONE: return "EMSTATE_NONE";
646	case EMSTATE_RAW: return "EMSTATE_RAW";
647	case EMSTATE_HWACC: return "EMSTATE_HWACC";
648	case EMSTATE_REM: return "EMSTATE_REM";
649	case EMSTATE_PARAV: return "EMSTATE_PARAV";
650	case EMSTATE_HALTED: return "EMSTATE_HALTED";
651	case EMSTATE_WAIT_SIPI: return "EMSTATE_WAIT_SIPI";
652	case EMSTATE_SUSPENDED: return "EMSTATE_SUSPENDED";
653	case EMSTATE_TERMINATING: return "EMSTATE_TERMINATING";
654	case EMSTATE_DEBUG_GUEST_RAW: return "EMSTATE_DEBUG_GUEST_RAW";
655	case EMSTATE_DEBUG_GUEST_REM: return "EMSTATE_DEBUG_GUEST_REM";
656	case EMSTATE_DEBUG_HYPER: return "EMSTATE_DEBUG_HYPER";
657	case EMSTATE_GURU_MEDITATION: return "EMSTATE_GURU_MEDITATION";
658	default: return "Unknown!";
659	}
660	}
661
662
663	#ifdef VBOX_WITH_STATISTICS
664	/**
665	* Just a braindead function to keep track of cli addresses.
666	* @param pVM VM handle.
667	* @param pVMCPU VMCPU handle.
668	* @param GCPtrInstr The EIP of the cli instruction.
669	*/
670	static void emR3RecordCli(PVM pVM, PVMCPU pVCpu, RTGCPTR GCPtrInstr)
671	{
672	PCLISTAT pRec;
673
674	pRec = (PCLISTAT)RTAvlPVGet(&pVCpu->em.s.pCliStatTree, (AVLPVKEY)GCPtrInstr);
675	if (!pRec)
676	{
677	/* New cli instruction; insert into the tree. */
678	pRec = (PCLISTAT)MMR3HeapAllocZ(pVM, MM_TAG_EM, sizeof(*pRec));
679	Assert(pRec);
680	if (!pRec)
681	return;
682	pRec->Core.Key = (AVLPVKEY)GCPtrInstr;
683
684	char szCliStatName[32];
685	RTStrPrintf(szCliStatName, sizeof(szCliStatName), "/EM/Cli/0x%RGv", GCPtrInstr);
686	STAM_REG(pVM, &pRec->Counter, STAMTYPE_COUNTER, szCliStatName, STAMUNIT_OCCURENCES, "Number of times cli was executed.");
687
688	bool fRc = RTAvlPVInsert(&pVCpu->em.s.pCliStatTree, &pRec->Core);
689	Assert(fRc); NOREF(fRc);
690	}
691	STAM_COUNTER_INC(&pRec->Counter);
692	STAM_COUNTER_INC(&pVCpu->em.s.StatTotalClis);
693	}
694	#endif /* VBOX_WITH_STATISTICS */
695
696
697	/**
698	* Debug loop.
699	*
700	* @returns VBox status code for EM.
701	* @param pVM VM handle.
702	* @param pVCpu VMCPU handle.
703	* @param rc Current EM VBox status code..
704	*/
705	static int emR3Debug(PVM pVM, PVMCPU pVCpu, int rc)
706	{
707	for (;;)
708	{
709	Log(("emR3Debug: rc=%Rrc\n", rc));
710	const int rcLast = rc;
711
712	/*
713	* Debug related RC.
714	*/
715	switch (rc)
716	{
717	/*
718	* Single step an instruction.
719	*/
720	case VINF_EM_DBG_STEP:
721	if ( pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_RAW
722	\|\| pVCpu->em.s.enmState == EMSTATE_DEBUG_HYPER
723	\|\| pVCpu->em.s.fForceRAW /* paranoia */)
724	rc = emR3RawStep(pVM, pVCpu);
725	else
726	{
727	Assert(pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_REM);
728	rc = emR3RemStep(pVM, pVCpu);
729	}
730	break;
731
732	/*
733	* Simple events: stepped, breakpoint, stop/assertion.
734	*/
735	case VINF_EM_DBG_STEPPED:
736	rc = DBGFR3Event(pVM, DBGFEVENT_STEPPED);
737	break;
738
739	case VINF_EM_DBG_BREAKPOINT:
740	rc = DBGFR3EventBreakpoint(pVM, DBGFEVENT_BREAKPOINT);
741	break;
742
743	case VINF_EM_DBG_STOP:
744	rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, NULL, 0, NULL, NULL);
745	break;
746
747	case VINF_EM_DBG_HYPER_STEPPED:
748	rc = DBGFR3Event(pVM, DBGFEVENT_STEPPED_HYPER);
749	break;
750
751	case VINF_EM_DBG_HYPER_BREAKPOINT:
752	rc = DBGFR3EventBreakpoint(pVM, DBGFEVENT_BREAKPOINT_HYPER);
753	break;
754
755	case VINF_EM_DBG_HYPER_ASSERTION:
756	RTPrintf("\nVINF_EM_DBG_HYPER_ASSERTION:\n%s%s\n", VMMR3GetRZAssertMsg1(pVM), VMMR3GetRZAssertMsg2(pVM));
757	rc = DBGFR3EventAssertion(pVM, DBGFEVENT_ASSERTION_HYPER, VMMR3GetRZAssertMsg1(pVM), VMMR3GetRZAssertMsg2(pVM));
758	break;
759
760	/*
761	* Guru meditation.
762	*/
763	case VERR_VMM_RING0_ASSERTION: /** @todo Make a guru meditation event! */
764	rc = DBGFR3EventSrc(pVM, DBGFEVENT_FATAL_ERROR, "VERR_VMM_RING0_ASSERTION", 0, NULL, NULL);
765	break;
766	case VERR_REM_TOO_MANY_TRAPS: /** @todo Make a guru meditation event! */
767	rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, "VERR_REM_TOO_MANY_TRAPS", 0, NULL, NULL);
768	break;
769
770	default: /** @todo don't use default for guru, but make special errors code! */
771	rc = DBGFR3Event(pVM, DBGFEVENT_FATAL_ERROR);
772	break;
773	}
774
775	/*
776	* Process the result.
777	*/
778	do
779	{
780	switch (rc)
781	{
782	/*
783	* Continue the debugging loop.
784	*/
785	case VINF_EM_DBG_STEP:
786	case VINF_EM_DBG_STOP:
787	case VINF_EM_DBG_STEPPED:
788	case VINF_EM_DBG_BREAKPOINT:
789	case VINF_EM_DBG_HYPER_STEPPED:
790	case VINF_EM_DBG_HYPER_BREAKPOINT:
791	case VINF_EM_DBG_HYPER_ASSERTION:
792	break;
793
794	/*
795	* Resuming execution (in some form) has to be done here if we got
796	* a hypervisor debug event.
797	*/
798	case VINF_SUCCESS:
799	case VINF_EM_RESUME:
800	case VINF_EM_SUSPEND:
801	case VINF_EM_RESCHEDULE:
802	case VINF_EM_RESCHEDULE_RAW:
803	case VINF_EM_RESCHEDULE_REM:
804	case VINF_EM_HALT:
805	if (pVCpu->em.s.enmState == EMSTATE_DEBUG_HYPER)
806	{
807	rc = emR3RawResumeHyper(pVM, pVCpu);
808	if (rc != VINF_SUCCESS && RT_SUCCESS(rc))
809	continue;
810	}
811	if (rc == VINF_SUCCESS)
812	rc = VINF_EM_RESCHEDULE;
813	return rc;
814
815	/*
816	* The debugger isn't attached.
817	* We'll simply turn the thing off since that's the easiest thing to do.
818	*/
819	case VERR_DBGF_NOT_ATTACHED:
820	switch (rcLast)
821	{
822	case VINF_EM_DBG_HYPER_STEPPED:
823	case VINF_EM_DBG_HYPER_BREAKPOINT:
824	case VINF_EM_DBG_HYPER_ASSERTION:
825	case VERR_TRPM_PANIC:
826	case VERR_TRPM_DONT_PANIC:
827	case VERR_VMM_RING0_ASSERTION:
828	case VERR_VMM_HYPER_CR3_MISMATCH:
829	case VERR_VMM_RING3_CALL_DISABLED:
830	return rcLast;
831	}
832	return VINF_EM_OFF;
833
834	/*
835	* Status codes terminating the VM in one or another sense.
836	*/
837	case VINF_EM_TERMINATE:
838	case VINF_EM_OFF:
839	case VINF_EM_RESET:
840	case VINF_EM_NO_MEMORY:
841	case VINF_EM_RAW_STALE_SELECTOR:
842	case VINF_EM_RAW_IRET_TRAP:
843	case VERR_TRPM_PANIC:
844	case VERR_TRPM_DONT_PANIC:
845	case VERR_VMM_RING0_ASSERTION:
846	case VERR_VMM_HYPER_CR3_MISMATCH:
847	case VERR_VMM_RING3_CALL_DISABLED:
848	case VERR_INTERNAL_ERROR:
849	case VERR_INTERNAL_ERROR_2:
850	case VERR_INTERNAL_ERROR_3:
851	case VERR_INTERNAL_ERROR_4:
852	case VERR_INTERNAL_ERROR_5:
853	case VERR_IPE_UNEXPECTED_STATUS:
854	case VERR_IPE_UNEXPECTED_INFO_STATUS:
855	case VERR_IPE_UNEXPECTED_ERROR_STATUS:
856	return rc;
857
858	/*
859	* The rest is unexpected, and will keep us here.
860	*/
861	default:
862	AssertMsgFailed(("Unxpected rc %Rrc!\n", rc));
863	break;
864	}
865	} while (false);
866	} /* debug for ever */
867	}
868
869	/**
870	* Steps recompiled code.
871	*
872	* @returns VBox status code. The most important ones are: VINF_EM_STEP_EVENT,
873	* VINF_EM_RESCHEDULE, VINF_EM_SUSPEND, VINF_EM_RESET and VINF_EM_TERMINATE.
874	*
875	* @param pVM VM handle.
876	* @param pVCpu VMCPU handle.
877	*/
878	static int emR3RemStep(PVM pVM, PVMCPU pVCpu)
879	{
880	LogFlow(("emR3RemStep: cs:eip=%04x:%08x\n", CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
881
882	EMRemLock(pVM);
883
884	/*
885	* Switch to REM, step instruction, switch back.
886	*/
887	int rc = REMR3State(pVM, pVCpu);
888	if (RT_SUCCESS(rc))
889	{
890	rc = REMR3Step(pVM, pVCpu);
891	REMR3StateBack(pVM, pVCpu);
892	}
893	EMRemUnlock(pVM);
894
895	LogFlow(("emR3RemStep: returns %Rrc cs:eip=%04x:%08x\n", rc, CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
896	return rc;
897	}
898
899
900	/**
901	* Executes recompiled code.
902	*
903	* This function contains the recompiler version of the inner
904	* execution loop (the outer loop being in EMR3ExecuteVM()).
905	*
906	* @returns VBox status code. The most important ones are: VINF_EM_RESCHEDULE,
907	* VINF_EM_SUSPEND, VINF_EM_RESET and VINF_EM_TERMINATE.
908	*
909	* @param pVM VM handle.
910	* @param pVCpu VMCPU handle.
911	* @param pfFFDone Where to store an indicator telling wheter or not
912	* FFs were done before returning.
913	*
914	*/
915	static int emR3RemExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone)
916	{
917	#ifdef LOG_ENABLED
918	PCPUMCTX pCtx = pVCpu->em.s.pCtx;
919	uint32_t cpl = CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(pCtx));
920
921	if (pCtx->eflags.Bits.u1VM)
922	Log(("EMV86: %04X:%08X IF=%d\n", pCtx->cs, pCtx->eip, pCtx->eflags.Bits.u1IF));
923	else
924	Log(("EMR%d: %04X:%08X ESP=%08X IF=%d CR0=%x\n", cpl, pCtx->cs, pCtx->eip, pCtx->esp, pCtx->eflags.Bits.u1IF, (uint32_t)pCtx->cr0));
925	#endif
926	STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatREMTotal, a);
927
928	#if defined(VBOX_STRICT) && defined(DEBUG_bird)
929	AssertMsg( VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
930	\|\| !MMHyperIsInsideArea(pVM, CPUMGetGuestEIP(pVCpu)), /** @todo #1419 - get flat address. */
931	("cs:eip=%RX16:%RX32\n", CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
932	#endif
933
934	/* Big lock, but you are not supposed to own any lock when coming in here. */
935	EMRemLock(pVM);
936
937	/*
938	* Spin till we get a forced action which returns anything but VINF_SUCCESS
939	* or the REM suggests raw-mode execution.
940	*/
941	*pfFFDone = false;
942	bool fInREMState = false;
943	int rc = VINF_SUCCESS;
944
945	/* Flush the recompiler TLB if the VCPU has changed. */
946	if (pVM->em.s.idLastRemCpu != pVCpu->idCpu)
947	{
948	REMFlushTBs(pVM);
949	/* Also sync the entire state. */
950	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_ALL);
951	}
952	pVM->em.s.idLastRemCpu = pVCpu->idCpu;
953
954	for (;;)
955	{
956	/*
957	* Update REM state if not already in sync.
958	*/
959	if (!fInREMState)
960	{
961	STAM_PROFILE_START(&pVCpu->em.s.StatREMSync, b);
962	rc = REMR3State(pVM, pVCpu);
963	STAM_PROFILE_STOP(&pVCpu->em.s.StatREMSync, b);
964	if (RT_FAILURE(rc))
965	break;
966	fInREMState = true;
967
968	/*
969	* We might have missed the raising of VMREQ, TIMER and some other
970	* imporant FFs while we were busy switching the state. So, check again.
971	*/
972	if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST \| VM_FF_PDM_QUEUES \| VM_FF_DBGF \| VM_FF_TERMINATE \| VM_FF_RESET)
973	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TIMER \| VMCPU_FF_REQUEST))
974	{
975	LogFlow(("emR3RemExecute: Skipping run, because FF is set. %#x\n", pVM->fGlobalForcedActions));
976	goto l_REMDoForcedActions;
977	}
978	}
979
980
981	/*
982	* Execute REM.
983	*/
984	STAM_PROFILE_START(&pVCpu->em.s.StatREMExec, c);
985	rc = REMR3Run(pVM, pVCpu);
986	STAM_PROFILE_STOP(&pVCpu->em.s.StatREMExec, c);
987
988
989	/*
990	* Deal with high priority post execution FFs before doing anything else.
991	*/
992	if ( VM_FF_ISPENDING(pVM, VM_FF_HIGH_PRIORITY_POST_MASK)
993	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_POST_MASK))
994	rc = emR3HighPriorityPostForcedActions(pVM, pVCpu, rc);
995
996	/*
997	* Process the returned status code.
998	* (Try keep this short! Call functions!)
999	*/
1000	if (rc != VINF_SUCCESS)
1001	{
1002	if (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST)
1003	break;
1004	if (rc != VINF_REM_INTERRUPED_FF)
1005	{
1006	/*
1007	* Anything which is not known to us means an internal error
1008	* and the termination of the VM!
1009	*/
1010	AssertMsg(rc == VERR_REM_TOO_MANY_TRAPS, ("Unknown GC return code: %Rra\n", rc));
1011	break;
1012	}
1013	}
1014
1015
1016	/*
1017	* Check and execute forced actions.
1018	* Sync back the VM state before calling any of these.
1019	*/
1020	#ifdef VBOX_HIGH_RES_TIMERS_HACK
1021	TMTimerPollVoid(pVM, pVCpu);
1022	#endif
1023	AssertCompile((VMCPU_FF_ALL_BUT_RAW_MASK & ~(VMCPU_FF_CSAM_PENDING_ACTION \| VMCPU_FF_CSAM_SCAN_PAGE)) & VMCPU_FF_TIMER);
1024	if ( VM_FF_ISPENDING(pVM, VM_FF_ALL_BUT_RAW_MASK)
1025	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_ALL_BUT_RAW_MASK & ~(VMCPU_FF_CSAM_PENDING_ACTION \| VMCPU_FF_CSAM_SCAN_PAGE)))
1026	{
1027	l_REMDoForcedActions:
1028	if (fInREMState)
1029	{
1030	STAM_PROFILE_START(&pVCpu->em.s.StatREMSync, d);
1031	REMR3StateBack(pVM, pVCpu);
1032	STAM_PROFILE_STOP(&pVCpu->em.s.StatREMSync, d);
1033	fInREMState = false;
1034	}
1035	STAM_REL_PROFILE_ADV_SUSPEND(&pVCpu->em.s.StatREMTotal, a);
1036	rc = emR3ForcedActions(pVM, pVCpu, rc);
1037	STAM_REL_PROFILE_ADV_RESUME(&pVCpu->em.s.StatREMTotal, a);
1038	if ( rc != VINF_SUCCESS
1039	&& rc != VINF_EM_RESCHEDULE_REM)
1040	{
1041	*pfFFDone = true;
1042	break;
1043	}
1044	}
1045
1046	} /* The Inner Loop, recompiled execution mode version. */
1047
1048
1049	/*
1050	* Returning. Sync back the VM state if required.
1051	*/
1052	if (fInREMState)
1053	{
1054	STAM_PROFILE_START(&pVCpu->em.s.StatREMSync, e);
1055	REMR3StateBack(pVM, pVCpu);
1056	STAM_PROFILE_STOP(&pVCpu->em.s.StatREMSync, e);
1057	}
1058	EMRemUnlock(pVM);
1059
1060	STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatREMTotal, a);
1061	return rc;
1062	}
1063
1064
1065	/**
1066	* Resumes executing hypervisor after a debug event.
1067	*
1068	* This is kind of special since our current guest state is
1069	* potentially out of sync.
1070	*
1071	* @returns VBox status code.
1072	* @param pVM The VM handle.
1073	* @param pVCpu The VMCPU handle.
1074	*/
1075	static int emR3RawResumeHyper(PVM pVM, PVMCPU pVCpu)
1076	{
1077	int rc;
1078	PCPUMCTX pCtx = pVCpu->em.s.pCtx;
1079	Assert(pVCpu->em.s.enmState == EMSTATE_DEBUG_HYPER);
1080	Log(("emR3RawResumeHyper: cs:eip=%RTsel:%RGr efl=%RGr\n", pCtx->cs, pCtx->eip, pCtx->eflags));
1081
1082	/*
1083	* Resume execution.
1084	*/
1085	CPUMRawEnter(pVCpu, NULL);
1086	CPUMSetHyperEFlags(pVCpu, CPUMGetHyperEFlags(pVCpu) \| X86_EFL_RF);
1087	rc = VMMR3ResumeHyper(pVM, pVCpu);
1088	Log(("emR3RawStep: cs:eip=%RTsel:%RGr efl=%RGr - returned from GC with rc=%Rrc\n", pCtx->cs, pCtx->eip, pCtx->eflags, rc));
1089	rc = CPUMRawLeave(pVCpu, NULL, rc);
1090	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_RESUME_GUEST_MASK);
1091
1092	/*
1093	* Deal with the return code.
1094	*/
1095	rc = emR3HighPriorityPostForcedActions(pVM, pVCpu, rc);
1096	rc = emR3RawHandleRC(pVM, pVCpu, pCtx, rc);
1097	rc = emR3RawUpdateForceFlag(pVM, pVCpu, pCtx, rc);
1098	return rc;
1099	}
1100
1101
1102	/**
1103	* Steps rawmode.
1104	*
1105	* @returns VBox status code.
1106	* @param pVM The VM handle.
1107	* @param pVCpu The VMCPU handle.
1108	*/
1109	static int emR3RawStep(PVM pVM, PVMCPU pVCpu)
1110	{
1111	Assert( pVCpu->em.s.enmState == EMSTATE_DEBUG_HYPER
1112	\|\| pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_RAW
1113	\|\| pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_REM);
1114	int rc;
1115	PCPUMCTX pCtx = pVCpu->em.s.pCtx;
1116	bool fGuest = pVCpu->em.s.enmState != EMSTATE_DEBUG_HYPER;
1117	#ifndef DEBUG_sandervl
1118	Log(("emR3RawStep: cs:eip=%RTsel:%RGr efl=%RGr\n", fGuest ? CPUMGetGuestCS(pVCpu) : CPUMGetHyperCS(pVCpu),
1119	fGuest ? CPUMGetGuestEIP(pVCpu) : CPUMGetHyperEIP(pVCpu), fGuest ? CPUMGetGuestEFlags(pVCpu) : CPUMGetHyperEFlags(pVCpu)));
1120	#endif
1121	if (fGuest)
1122	{
1123	/*
1124	* Check vital forced actions, but ignore pending interrupts and timers.
1125	*/
1126	if ( VM_FF_ISPENDING(pVM, VM_FF_HIGH_PRIORITY_PRE_RAW_MASK)
1127	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_PRE_RAW_MASK))
1128	{
1129	rc = emR3RawForcedActions(pVM, pVCpu, pCtx);
1130	if (rc != VINF_SUCCESS)
1131	return rc;
1132	}
1133
1134	/*
1135	* Set flags for single stepping.
1136	*/
1137	CPUMSetGuestEFlags(pVCpu, CPUMGetGuestEFlags(pVCpu) \| X86_EFL_TF \| X86_EFL_RF);
1138	}
1139	else
1140	CPUMSetHyperEFlags(pVCpu, CPUMGetHyperEFlags(pVCpu) \| X86_EFL_TF \| X86_EFL_RF);
1141
1142	/*
1143	* Single step.
1144	* We do not start time or anything, if anything we should just do a few nanoseconds.
1145	*/
1146	CPUMRawEnter(pVCpu, NULL);
1147	do
1148	{
1149	if (pVCpu->em.s.enmState == EMSTATE_DEBUG_HYPER)
1150	rc = VMMR3ResumeHyper(pVM, pVCpu);
1151	else
1152	rc = VMMR3RawRunGC(pVM, pVCpu);
1153	#ifndef DEBUG_sandervl
1154	Log(("emR3RawStep: cs:eip=%RTsel:%RGr efl=%RGr - GC rc %Rrc\n", fGuest ? CPUMGetGuestCS(pVCpu) : CPUMGetHyperCS(pVCpu),
1155	fGuest ? CPUMGetGuestEIP(pVCpu) : CPUMGetHyperEIP(pVCpu), fGuest ? CPUMGetGuestEFlags(pVCpu) : CPUMGetHyperEFlags(pVCpu), rc));
1156	#endif
1157	} while ( rc == VINF_SUCCESS
1158	\|\| rc == VINF_EM_RAW_INTERRUPT);
1159	rc = CPUMRawLeave(pVCpu, NULL, rc);
1160	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_RESUME_GUEST_MASK);
1161
1162	/*
1163	* Make sure the trap flag is cleared.
1164	* (Too bad if the guest is trying to single step too.)
1165	*/
1166	if (fGuest)
1167	CPUMSetGuestEFlags(pVCpu, CPUMGetGuestEFlags(pVCpu) & ~X86_EFL_TF);
1168	else
1169	CPUMSetHyperEFlags(pVCpu, CPUMGetHyperEFlags(pVCpu) & ~X86_EFL_TF);
1170
1171	/*
1172	* Deal with the return codes.
1173	*/
1174	rc = emR3HighPriorityPostForcedActions(pVM, pVCpu, rc);
1175	rc = emR3RawHandleRC(pVM, pVCpu, pCtx, rc);
1176	rc = emR3RawUpdateForceFlag(pVM, pVCpu, pCtx, rc);
1177	return rc;
1178	}
1179
1180
1181	#ifdef DEBUG
1182
1183	/**
1184	* Steps hardware accelerated mode.
1185	*
1186	* @returns VBox status code.
1187	* @param pVM The VM handle.
1188	* @param pVCpu The VMCPU handle.
1189	*/
1190	static int emR3HwAccStep(PVM pVM, PVMCPU pVCpu)
1191	{
1192	Assert(pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_HWACC);
1193
1194	int rc;
1195	PCPUMCTX pCtx = pVCpu->em.s.pCtx;
1196	VMCPU_FF_CLEAR(pVCpu, (VMCPU_FF_SELM_SYNC_GDT \| VMCPU_FF_SELM_SYNC_LDT \| VMCPU_FF_TRPM_SYNC_IDT \| VMCPU_FF_SELM_SYNC_TSS));
1197
1198	/*
1199	* Check vital forced actions, but ignore pending interrupts and timers.
1200	*/
1201	if ( VM_FF_ISPENDING(pVM, VM_FF_HIGH_PRIORITY_PRE_RAW_MASK)
1202	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_PRE_RAW_MASK))
1203	{
1204	rc = emR3RawForcedActions(pVM, pVCpu, pCtx);
1205	if (rc != VINF_SUCCESS)
1206	return rc;
1207	}
1208	/*
1209	* Set flags for single stepping.
1210	*/
1211	CPUMSetGuestEFlags(pVCpu, CPUMGetGuestEFlags(pVCpu) \| X86_EFL_TF \| X86_EFL_RF);
1212
1213	/*
1214	* Single step.
1215	* We do not start time or anything, if anything we should just do a few nanoseconds.
1216	*/
1217	do
1218	{
1219	rc = VMMR3HwAccRunGC(pVM, pVCpu);
1220	} while ( rc == VINF_SUCCESS
1221	\|\| rc == VINF_EM_RAW_INTERRUPT);
1222	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_RESUME_GUEST_MASK);
1223
1224	/*
1225	* Make sure the trap flag is cleared.
1226	* (Too bad if the guest is trying to single step too.)
1227	*/
1228	CPUMSetGuestEFlags(pVCpu, CPUMGetGuestEFlags(pVCpu) & ~X86_EFL_TF);
1229
1230	/*
1231	* Deal with the return codes.
1232	*/
1233	rc = emR3HighPriorityPostForcedActions(pVM, pVCpu, rc);
1234	rc = emR3RawHandleRC(pVM, pVCpu, pCtx, rc);
1235	rc = emR3RawUpdateForceFlag(pVM, pVCpu, pCtx, rc);
1236	return rc;
1237	}
1238
1239
1240	int emR3SingleStepExecRaw(PVM pVM, PVMCPU pVCpu, uint32_t cIterations)
1241	{
1242	int rc = VINF_SUCCESS;
1243	EMSTATE enmOldState = pVCpu->em.s.enmState;
1244	pVCpu->em.s.enmState = EMSTATE_DEBUG_GUEST_RAW;
1245
1246	Log(("Single step BEGIN:\n"));
1247	for (uint32_t i = 0; i < cIterations; i++)
1248	{
1249	DBGFR3PrgStep(pVCpu);
1250	DBGFR3DisasInstrCurrentLog(pVCpu, "RSS: ");
1251	rc = emR3RawStep(pVM, pVCpu);
1252	if (rc != VINF_SUCCESS)
1253	break;
1254	}
1255	Log(("Single step END: rc=%Rrc\n", rc));
1256	CPUMSetGuestEFlags(pVCpu, CPUMGetGuestEFlags(pVCpu) & ~X86_EFL_TF);
1257	pVCpu->em.s.enmState = enmOldState;
1258	return rc;
1259	}
1260
1261
1262	static int emR3SingleStepExecHwAcc(PVM pVM, PVMCPU pVCpu, uint32_t cIterations)
1263	{
1264	int rc = VINF_SUCCESS;
1265	EMSTATE enmOldState = pVCpu->em.s.enmState;
1266	pVCpu->em.s.enmState = EMSTATE_DEBUG_GUEST_HWACC;
1267
1268	Log(("Single step BEGIN:\n"));
1269	for (uint32_t i = 0; i < cIterations; i++)
1270	{
1271	DBGFR3PrgStep(pVCpu);
1272	DBGFR3DisasInstrCurrentLog(pVCpu, "RSS: ");
1273	rc = emR3HwAccStep(pVM, pVCpu);
1274	if ( rc != VINF_SUCCESS
1275	\|\| !HWACCMR3CanExecuteGuest(pVM, pVCpu->em.s.pCtx))
1276	break;
1277	}
1278	Log(("Single step END: rc=%Rrc\n", rc));
1279	CPUMSetGuestEFlags(pVCpu, CPUMGetGuestEFlags(pVCpu) & ~X86_EFL_TF);
1280	pVCpu->em.s.enmState = enmOldState;
1281	return rc == VINF_SUCCESS ? VINF_EM_RESCHEDULE_REM : rc;
1282	}
1283
1284
1285	static int emR3SingleStepExecRem(PVM pVM, PVMCPU pVCpu, uint32_t cIterations)
1286	{
1287	EMSTATE enmOldState = pVCpu->em.s.enmState;
1288
1289	pVCpu->em.s.enmState = EMSTATE_DEBUG_GUEST_REM;
1290
1291	Log(("Single step BEGIN:\n"));
1292	for (uint32_t i = 0; i < cIterations; i++)
1293	{
1294	DBGFR3PrgStep(pVCpu);
1295	DBGFR3DisasInstrCurrentLog(pVCpu, "RSS: ");
1296	emR3RemStep(pVM, pVCpu);
1297	if (emR3Reschedule(pVM, pVCpu, pVCpu->em.s.pCtx) != EMSTATE_REM)
1298	break;
1299	}
1300	Log(("Single step END:\n"));
1301	CPUMSetGuestEFlags(pVCpu, CPUMGetGuestEFlags(pVCpu) & ~X86_EFL_TF);
1302	pVCpu->em.s.enmState = enmOldState;
1303	return VINF_EM_RESCHEDULE;
1304	}
1305
1306	#endif /* DEBUG */
1307
1308
1309	/**
1310	* Executes one (or perhaps a few more) instruction(s).
1311	*
1312	* @returns VBox status code suitable for EM.
1313	*
1314	* @param pVM VM handle.
1315	* @param pVCpu VMCPU handle
1316	* @param rcGC GC return code
1317	* @param pszPrefix Disassembly prefix. If not NULL we'll disassemble the
1318	* instruction and prefix the log output with this text.
1319	*/
1320	#ifdef LOG_ENABLED
1321	static int emR3RawExecuteInstructionWorker(PVM pVM, PVMCPU pVCpu, int rcGC, const char *pszPrefix)
1322	#else
1323	static int emR3RawExecuteInstructionWorker(PVM pVM, PVMCPU pVCpu, int rcGC)
1324	#endif
1325	{
1326	PCPUMCTX pCtx = pVCpu->em.s.pCtx;
1327	int rc;
1328
1329	/*
1330	*
1331	* The simple solution is to use the recompiler.
1332	* The better solution is to disassemble the current instruction and
1333	* try handle as many as possible without using REM.
1334	*
1335	*/
1336
1337	#ifdef LOG_ENABLED
1338	/*
1339	* Disassemble the instruction if requested.
1340	*/
1341	if (pszPrefix)
1342	{
1343	DBGFR3InfoLog(pVM, "cpumguest", pszPrefix);
1344	DBGFR3DisasInstrCurrentLog(pVCpu, pszPrefix);
1345	}
1346	#endif /* LOG_ENABLED */
1347
1348	/*
1349	* PATM is making life more interesting.
1350	* We cannot hand anything to REM which has an EIP inside patch code. So, we'll
1351	* tell PATM there is a trap in this code and have it take the appropriate actions
1352	* to allow us execute the code in REM.
1353	*/
1354	if (PATMIsPatchGCAddr(pVM, pCtx->eip))
1355	{
1356	Log(("emR3RawExecuteInstruction: In patch block. eip=%RRv\n", (RTRCPTR)pCtx->eip));
1357
1358	RTGCPTR pNewEip;
1359	rc = PATMR3HandleTrap(pVM, pCtx, pCtx->eip, &pNewEip);
1360	switch (rc)
1361	{
1362	/*
1363	* It's not very useful to emulate a single instruction and then go back to raw
1364	* mode; just execute the whole block until IF is set again.
1365	*/
1366	case VINF_SUCCESS:
1367	Log(("emR3RawExecuteInstruction: Executing instruction starting at new address %RGv IF=%d VMIF=%x\n",
1368	pNewEip, pCtx->eflags.Bits.u1IF, pVCpu->em.s.pPatmGCState->uVMFlags));
1369	pCtx->eip = pNewEip;
1370	Assert(pCtx->eip);
1371
1372	if (pCtx->eflags.Bits.u1IF)
1373	{
1374	/*
1375	* The last instruction in the patch block needs to be executed!! (sti/sysexit for example)
1376	*/
1377	Log(("PATCH: IF=1 -> emulate last instruction as it can't be interrupted!!\n"));
1378	return emR3RawExecuteInstruction(pVM, pVCpu, "PATCHIR");
1379	}
1380	else if (rcGC == VINF_PATM_PENDING_IRQ_AFTER_IRET)
1381	{
1382	/* special case: iret, that sets IF, detected a pending irq/event */
1383	return emR3RawExecuteInstruction(pVM, pVCpu, "PATCHIRET");
1384	}
1385	return VINF_EM_RESCHEDULE_REM;
1386
1387	/*
1388	* One instruction.
1389	*/
1390	case VINF_PATCH_EMULATE_INSTR:
1391	Log(("emR3RawExecuteInstruction: Emulate patched instruction at %RGv IF=%d VMIF=%x\n",
1392	pNewEip, pCtx->eflags.Bits.u1IF, pVCpu->em.s.pPatmGCState->uVMFlags));
1393	pCtx->eip = pNewEip;
1394	return emR3RawExecuteInstruction(pVM, pVCpu, "PATCHIR");
1395
1396	/*
1397	* The patch was disabled, hand it to the REM.
1398	*/
1399	case VERR_PATCH_DISABLED:
1400	Log(("emR3RawExecuteInstruction: Disabled patch -> new eip %RGv IF=%d VMIF=%x\n",
1401	pNewEip, pCtx->eflags.Bits.u1IF, pVCpu->em.s.pPatmGCState->uVMFlags));
1402	pCtx->eip = pNewEip;
1403	if (pCtx->eflags.Bits.u1IF)
1404	{
1405	/*
1406	* The last instruction in the patch block needs to be executed!! (sti/sysexit for example)
1407	*/
1408	Log(("PATCH: IF=1 -> emulate last instruction as it can't be interrupted!!\n"));
1409	return emR3RawExecuteInstruction(pVM, pVCpu, "PATCHIR");
1410	}
1411	return VINF_EM_RESCHEDULE_REM;
1412
1413	/* Force continued patch exection; usually due to write monitored stack. */
1414	case VINF_PATCH_CONTINUE:
1415	return VINF_SUCCESS;
1416
1417	default:
1418	AssertReleaseMsgFailed(("Unknown return code %Rrc from PATMR3HandleTrap\n", rc));
1419	return VERR_IPE_UNEXPECTED_STATUS;
1420	}
1421	}
1422
1423	#if 0
1424	/* Try our own instruction emulator before falling back to the recompiler. */
1425	DISCPUSTATE Cpu;
1426	rc = CPUMR3DisasmInstrCPU(pVM, pVCpu, pCtx, pCtx->rip, &Cpu, "GEN EMU");
1427	if (RT_SUCCESS(rc))
1428	{
1429	uint32_t size;
1430
1431	switch (Cpu.pCurInstr->opcode)
1432	{
1433	/* @todo we can do more now */
1434	case OP_MOV:
1435	case OP_AND:
1436	case OP_OR:
1437	case OP_XOR:
1438	case OP_POP:
1439	case OP_INC:
1440	case OP_DEC:
1441	case OP_XCHG:
1442	STAM_PROFILE_START(&pVCpu->em.s.StatMiscEmu, a);
1443	rc = EMInterpretInstructionCPU(pVM, &Cpu, CPUMCTX2CORE(pCtx), 0, &size);
1444	if (RT_SUCCESS(rc))
1445	{
1446	pCtx->rip += Cpu.opsize;
1447	#ifdef EM_NOTIFY_HWACCM
1448	if (pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_HWACC)
1449	HWACCMR3NotifyEmulated(pVCpu);
1450	#endif
1451	STAM_PROFILE_STOP(&pVCpu->em.s.StatMiscEmu, a);
1452	return rc;
1453	}
1454	if (rc != VERR_EM_INTERPRETER)
1455	AssertMsgFailedReturn(("rc=%Rrc\n", rc), rc);
1456	STAM_PROFILE_STOP(&pVCpu->em.s.StatMiscEmu, a);
1457	break;
1458	}
1459	}
1460	#endif /* 0 */
1461	STAM_PROFILE_START(&pVCpu->em.s.StatREMEmu, a);
1462	Log(("EMINS: %04x:%RGv RSP=%RGv\n", pCtx->cs, (RTGCPTR)pCtx->rip, (RTGCPTR)pCtx->rsp));
1463	EMRemLock(pVM);
1464	/* Flush the recompiler TLB if the VCPU has changed. */
1465	if (pVM->em.s.idLastRemCpu != pVCpu->idCpu)
1466	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_ALL);
1467	pVM->em.s.idLastRemCpu = pVCpu->idCpu;
1468
1469	rc = REMR3EmulateInstruction(pVM, pVCpu);
1470	EMRemUnlock(pVM);
1471	STAM_PROFILE_STOP(&pVCpu->em.s.StatREMEmu, a);
1472
1473	#ifdef EM_NOTIFY_HWACCM
1474	if (pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_HWACC)
1475	HWACCMR3NotifyEmulated(pVCpu);
1476	#endif
1477	return rc;
1478	}
1479
1480
1481	/**
1482	* Executes one (or perhaps a few more) instruction(s).
1483	* This is just a wrapper for discarding pszPrefix in non-logging builds.
1484	*
1485	* @returns VBox status code suitable for EM.
1486	* @param pVM VM handle.
1487	* @param pVCpu VMCPU handle.
1488	* @param pszPrefix Disassembly prefix. If not NULL we'll disassemble the
1489	* instruction and prefix the log output with this text.
1490	* @param rcGC GC return code
1491	*/
1492	DECLINLINE(int) emR3RawExecuteInstruction(PVM pVM, PVMCPU pVCpu, const char *pszPrefix, int rcGC)
1493	{
1494	#ifdef LOG_ENABLED
1495	return emR3RawExecuteInstructionWorker(pVM, pVCpu, rcGC, pszPrefix);
1496	#else
1497	return emR3RawExecuteInstructionWorker(pVM, pVCpu, rcGC);
1498	#endif
1499	}
1500
1501	/**
1502	* Executes one (or perhaps a few more) IO instruction(s).
1503	*
1504	* @returns VBox status code suitable for EM.
1505	* @param pVM VM handle.
1506	* @param pVCpu VMCPU handle.
1507	*/
1508	int emR3RawExecuteIOInstruction(PVM pVM, PVMCPU pVCpu)
1509	{
1510	int rc;
1511	PCPUMCTX pCtx = pVCpu->em.s.pCtx;
1512
1513	STAM_PROFILE_START(&pVCpu->em.s.StatIOEmu, a);
1514
1515	/** @todo probably we should fall back to the recompiler; otherwise we'll go back and forth between HC & GC
1516	* as io instructions tend to come in packages of more than one
1517	*/
1518	DISCPUSTATE Cpu;
1519	rc = CPUMR3DisasmInstrCPU(pVM, pVCpu, pCtx, pCtx->rip, &Cpu, "IO EMU");
1520	if (RT_SUCCESS(rc))
1521	{
1522	rc = VINF_EM_RAW_EMULATE_INSTR;
1523
1524	if (!(Cpu.prefix & (PREFIX_REP \| PREFIX_REPNE)))
1525	{
1526	switch (Cpu.pCurInstr->opcode)
1527	{
1528	case OP_IN:
1529	{
1530	STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatIn);
1531	rc = IOMInterpretIN(pVM, CPUMCTX2CORE(pCtx), &Cpu);
1532	break;
1533	}
1534
1535	case OP_OUT:
1536	{
1537	STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatOut);
1538	rc = IOMInterpretOUT(pVM, CPUMCTX2CORE(pCtx), &Cpu);
1539	break;
1540	}
1541	}
1542	}
1543	else if (Cpu.prefix & PREFIX_REP)
1544	{
1545	switch (Cpu.pCurInstr->opcode)
1546	{
1547	case OP_INSB:
1548	case OP_INSWD:
1549	{
1550	STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatIn);
1551	rc = IOMInterpretINS(pVM, CPUMCTX2CORE(pCtx), &Cpu);
1552	break;
1553	}
1554
1555	case OP_OUTSB:
1556	case OP_OUTSWD:
1557	{
1558	STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatOut);
1559	rc = IOMInterpretOUTS(pVM, CPUMCTX2CORE(pCtx), &Cpu);
1560	break;
1561	}
1562	}
1563	}
1564
1565	/*
1566	* Handled the I/O return codes.
1567	* (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
1568	*/
1569	if (IOM_SUCCESS(rc))
1570	{
1571	pCtx->rip += Cpu.opsize;
1572	STAM_PROFILE_STOP(&pVCpu->em.s.StatIOEmu, a);
1573	return rc;
1574	}
1575
1576	if (rc == VINF_EM_RAW_GUEST_TRAP)
1577	{
1578	STAM_PROFILE_STOP(&pVCpu->em.s.StatIOEmu, a);
1579	rc = emR3RawGuestTrap(pVM, pVCpu);
1580	return rc;
1581	}
1582	AssertMsg(rc != VINF_TRPM_XCPT_DISPATCHED, ("Handle VINF_TRPM_XCPT_DISPATCHED\n"));
1583
1584	if (RT_FAILURE(rc))
1585	{
1586	STAM_PROFILE_STOP(&pVCpu->em.s.StatIOEmu, a);
1587	return rc;
1588	}
1589	AssertMsg(rc == VINF_EM_RAW_EMULATE_INSTR \|\| rc == VINF_EM_RESCHEDULE_REM, ("rc=%Rrc\n", rc));
1590	}
1591	STAM_PROFILE_STOP(&pVCpu->em.s.StatIOEmu, a);
1592	return emR3RawExecuteInstruction(pVM, pVCpu, "IO: ");
1593	}
1594
1595
1596	/**
1597	* Handle a guest context trap.
1598	*
1599	* @returns VBox status code suitable for EM.
1600	* @param pVM VM handle.
1601	* @param pVCpu VMCPU handle.
1602	*/
1603	static int emR3RawGuestTrap(PVM pVM, PVMCPU pVCpu)
1604	{
1605	PCPUMCTX pCtx = pVCpu->em.s.pCtx;
1606
1607	/*
1608	* Get the trap info.
1609	*/
1610	uint8_t u8TrapNo;
1611	TRPMEVENT enmType;
1612	RTGCUINT uErrorCode;
1613	RTGCUINTPTR uCR2;
1614	int rc = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrorCode, &uCR2);
1615	if (RT_FAILURE(rc))
1616	{
1617	AssertReleaseMsgFailed(("No trap! (rc=%Rrc)\n", rc));
1618	return rc;
1619	}
1620
1621	/*
1622	* Traps can be directly forwarded in hardware accelerated mode.
1623	*/
1624	if (HWACCMIsEnabled(pVM))
1625	{
1626	#ifdef LOGGING_ENABLED
1627	DBGFR3InfoLog(pVM, "cpumguest", "Guest trap");
1628	DBGFR3DisasInstrCurrentLog(pVCpu, "Guest trap");
1629	#endif
1630	return VINF_EM_RESCHEDULE_HWACC;
1631	}
1632
1633	#if 1 /* Experimental: Review, disable if it causes trouble. */
1634	/*
1635	* Handle traps in patch code first.
1636	*
1637	* We catch a few of these cases in RC before returning to R3 (#PF, #GP, #BP)
1638	* but several traps isn't handled specially by TRPM in RC and we end up here
1639	* instead. One example is #DE.
1640	*/
1641	uint32_t uCpl = CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(pCtx));
1642	if ( uCpl == 0
1643	&& PATMIsPatchGCAddr(pVM, (RTGCPTR)pCtx->eip))
1644	{
1645	LogFlow(("emR3RawGuestTrap: trap %#x in patch code; eip=%08x\n", u8TrapNo, pCtx->eip));
1646	return emR3PatchTrap(pVM, pVCpu, pCtx, rc);
1647	}
1648	#endif
1649
1650	/*
1651	* If the guest gate is marked unpatched, then we will check again if we can patch it.
1652	* (This assumes that we've already tried and failed to dispatch the trap in
1653	* RC for the gates that already has been patched. Which is true for most high
1654	* volume traps, because these are handled specially, but not for odd ones like #DE.)
1655	*/
1656	if (TRPMR3GetGuestTrapHandler(pVM, u8TrapNo) == TRPM_INVALID_HANDLER)
1657	{
1658	CSAMR3CheckGates(pVM, u8TrapNo, 1);
1659	Log(("emR3RawHandleRC: recheck gate %x -> valid=%d\n", u8TrapNo, TRPMR3GetGuestTrapHandler(pVM, u8TrapNo) != TRPM_INVALID_HANDLER));
1660
1661	/* If it was successful, then we could go back to raw mode. */
1662	if (TRPMR3GetGuestTrapHandler(pVM, u8TrapNo) != TRPM_INVALID_HANDLER)
1663	{
1664	/* Must check pending forced actions as our IDT or GDT might be out of sync. */
1665	rc = EMR3CheckRawForcedActions(pVM, pVCpu);
1666	AssertRCReturn(rc, rc);
1667
1668	TRPMERRORCODE enmError = uErrorCode != ~0U
1669	? TRPM_TRAP_HAS_ERRORCODE
1670	: TRPM_TRAP_NO_ERRORCODE;
1671	rc = TRPMForwardTrap(pVCpu, CPUMCTX2CORE(pCtx), u8TrapNo, uErrorCode, enmError, TRPM_TRAP, -1);
1672	if (rc == VINF_SUCCESS /* Don't use RT_SUCCESS */)
1673	{
1674	TRPMResetTrap(pVCpu);
1675	return VINF_EM_RESCHEDULE_RAW;
1676	}
1677	AssertMsg(rc == VINF_EM_RAW_GUEST_TRAP, ("%Rrc\n", rc));
1678	}
1679	}
1680
1681	/*
1682	* Scan kernel code that traps; we might not get another chance.
1683	*/
1684	/** @todo move this up before the dispatching? */
1685	if ( (pCtx->ss & X86_SEL_RPL) <= 1
1686	&& !pCtx->eflags.Bits.u1VM)
1687	{
1688	Assert(!PATMIsPatchGCAddr(pVM, pCtx->eip));
1689	CSAMR3CheckCodeEx(pVM, CPUMCTX2CORE(pCtx), pCtx->eip);
1690	}
1691
1692	/*
1693	* Trap specific handling.
1694	*/
1695	if (u8TrapNo == 6) /* (#UD) Invalid opcode. */
1696	{
1697	/*
1698	* If MONITOR & MWAIT are supported, then interpret them here.
1699	*/
1700	DISCPUSTATE cpu;
1701	rc = CPUMR3DisasmInstrCPU(pVM, pVCpu, pCtx, pCtx->rip, &cpu, "Guest Trap (#UD): ");
1702	if ( RT_SUCCESS(rc)
1703	&& (cpu.pCurInstr->opcode == OP_MONITOR \|\| cpu.pCurInstr->opcode == OP_MWAIT))
1704	{
1705	uint32_t u32Dummy, u32Features, u32ExtFeatures;
1706	CPUMGetGuestCpuId(pVCpu, 1, &u32Dummy, &u32Dummy, &u32ExtFeatures, &u32Features);
1707	if (u32ExtFeatures & X86_CPUID_FEATURE_ECX_MONITOR)
1708	{
1709	rc = TRPMResetTrap(pVCpu);
1710	AssertRC(rc);
1711
1712	uint32_t opsize;
1713	rc = EMInterpretInstructionCPU(pVM, pVCpu, &cpu, CPUMCTX2CORE(pCtx), 0, &opsize);
1714	if (RT_SUCCESS(rc))
1715	{
1716	pCtx->rip += cpu.opsize;
1717	#ifdef EM_NOTIFY_HWACCM
1718	if (pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_HWACC)
1719	HWACCMR3NotifyEmulated(pVCpu);
1720	#endif
1721	return rc;
1722	}
1723	return emR3RawExecuteInstruction(pVM, pVCpu, "Monitor: ");
1724	}
1725	}
1726	}
1727	else if (u8TrapNo == 13) /* (#GP) Privileged exception */
1728	{
1729	/*
1730	* Handle I/O bitmap?
1731	*/
1732	/** @todo We're not supposed to be here with a false guest trap concerning
1733	* I/O access. We can easily handle those in RC. */
1734	DISCPUSTATE cpu;
1735	rc = CPUMR3DisasmInstrCPU(pVM, pVCpu, pCtx, pCtx->rip, &cpu, "Guest Trap: ");
1736	if ( RT_SUCCESS(rc)
1737	&& (cpu.pCurInstr->optype & OPTYPE_PORTIO))
1738	{
1739	/*
1740	* We should really check the TSS for the IO bitmap, but it's not like this
1741	* lazy approach really makes things worse.
1742	*/
1743	rc = TRPMResetTrap(pVCpu);
1744	AssertRC(rc);
1745	return emR3RawExecuteInstruction(pVM, pVCpu, "IO Guest Trap: ");
1746	}
1747	}
1748
1749	#ifdef LOG_ENABLED
1750	DBGFR3InfoLog(pVM, "cpumguest", "Guest trap");
1751	DBGFR3DisasInstrCurrentLog(pVCpu, "Guest trap");
1752
1753	/* Get guest page information. */
1754	uint64_t fFlags = 0;
1755	RTGCPHYS GCPhys = 0;
1756	int rc2 = PGMGstGetPage(pVCpu, uCR2, &fFlags, &GCPhys);
1757	Log(("emR3RawGuestTrap: cs:eip=%04x:%08x: trap=%02x err=%08x cr2=%08x cr0=%08x%s: Phys=%RGp fFlags=%08llx %s %s %s%s rc2=%d\n",
1758	pCtx->cs, pCtx->eip, u8TrapNo, uErrorCode, uCR2, (uint32_t)pCtx->cr0, (enmType == TRPM_SOFTWARE_INT) ? " software" : "", GCPhys, fFlags,
1759	fFlags & X86_PTE_P ? "P " : "NP", fFlags & X86_PTE_US ? "U" : "S",
1760	fFlags & X86_PTE_RW ? "RW" : "R0", fFlags & X86_PTE_G ? " G" : "", rc2));
1761	#endif
1762
1763	/*
1764	* #PG has CR2.
1765	* (Because of stuff like above we must set CR2 in a delayed fashion.)
1766	*/
1767	if (u8TrapNo == 14 /* #PG */)
1768	pCtx->cr2 = uCR2;
1769
1770	return VINF_EM_RESCHEDULE_REM;
1771	}
1772
1773
1774	/**
1775	* Handle a ring switch trap.
1776	* Need to do statistics and to install patches. The result is going to REM.
1777	*
1778	* @returns VBox status code suitable for EM.
1779	* @param pVM VM handle.
1780	* @param pVCpu VMCPU handle.
1781	*/
1782	int emR3RawRingSwitch(PVM pVM, PVMCPU pVCpu)
1783	{
1784	int rc;
1785	DISCPUSTATE Cpu;
1786	PCPUMCTX pCtx = pVCpu->em.s.pCtx;
1787
1788	/*
1789	* sysenter, syscall & callgate
1790	*/
1791	rc = CPUMR3DisasmInstrCPU(pVM, pVCpu, pCtx, pCtx->rip, &Cpu, "RSWITCH: ");
1792	if (RT_SUCCESS(rc))
1793	{
1794	if (Cpu.pCurInstr->opcode == OP_SYSENTER)
1795	{
1796	if (pCtx->SysEnter.cs != 0)
1797	{
1798	rc = PATMR3InstallPatch(pVM, SELMToFlat(pVM, DIS_SELREG_CS, CPUMCTX2CORE(pCtx), pCtx->eip),
1799	(SELMGetCpuModeFromSelector(pVM, pCtx->eflags, pCtx->cs, &pCtx->csHid) == CPUMODE_32BIT) ? PATMFL_CODE32 : 0);
1800	if (RT_SUCCESS(rc))
1801	{
1802	DBGFR3DisasInstrCurrentLog(pVCpu, "Patched sysenter instruction");
1803	return VINF_EM_RESCHEDULE_RAW;
1804	}
1805	}
1806	}
1807
1808	#ifdef VBOX_WITH_STATISTICS
1809	switch (Cpu.pCurInstr->opcode)
1810	{
1811	case OP_SYSENTER:
1812	STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatSysEnter);
1813	break;
1814	case OP_SYSEXIT:
1815	STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatSysExit);
1816	break;
1817	case OP_SYSCALL:
1818	STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatSysCall);
1819	break;
1820	case OP_SYSRET:
1821	STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatSysRet);
1822	break;
1823	}
1824	#endif
1825	}
1826	else
1827	AssertRC(rc);
1828
1829	/* go to the REM to emulate a single instruction */
1830	return emR3RawExecuteInstruction(pVM, pVCpu, "RSWITCH: ");
1831	}
1832
1833
1834	/**
1835	* Handle a trap (\#PF or \#GP) in patch code
1836	*
1837	* @returns VBox status code suitable for EM.
1838	* @param pVM VM handle.
1839	* @param pVCpu VMCPU handle.
1840	* @param pCtx CPU context
1841	* @param gcret GC return code
1842	*/
1843	static int emR3PatchTrap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, int gcret)
1844	{
1845	uint8_t u8TrapNo;
1846	int rc;
1847	TRPMEVENT enmType;
1848	RTGCUINT uErrorCode;
1849	RTGCUINTPTR uCR2;
1850
1851	Assert(PATMIsPatchGCAddr(pVM, pCtx->eip));
1852
1853	if (gcret == VINF_PATM_PATCH_INT3)
1854	{
1855	u8TrapNo = 3;
1856	uCR2 = 0;
1857	uErrorCode = 0;
1858	}
1859	else if (gcret == VINF_PATM_PATCH_TRAP_GP)
1860	{
1861	/* No active trap in this case. Kind of ugly. */
1862	u8TrapNo = X86_XCPT_GP;
1863	uCR2 = 0;
1864	uErrorCode = 0;
1865	}
1866	else
1867	{
1868	rc = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrorCode, &uCR2);
1869	if (RT_FAILURE(rc))
1870	{
1871	AssertReleaseMsgFailed(("emR3PatchTrap: no trap! (rc=%Rrc) gcret=%Rrc\n", rc, gcret));
1872	return rc;
1873	}
1874	/* Reset the trap as we'll execute the original instruction again. */
1875	TRPMResetTrap(pVCpu);
1876	}
1877
1878	/*
1879	* Deal with traps inside patch code.
1880	* (This code won't run outside GC.)
1881	*/
1882	if (u8TrapNo != 1)
1883	{
1884	#ifdef LOG_ENABLED
1885	DBGFR3InfoLog(pVM, "cpumguest", "Trap in patch code");
1886	DBGFR3DisasInstrCurrentLog(pVCpu, "Patch code");
1887
1888	DISCPUSTATE Cpu;
1889	int rc;
1890
1891	rc = CPUMR3DisasmInstrCPU(pVM, pVCpu, pCtx, pCtx->eip, &Cpu, "Patch code: ");
1892	if ( RT_SUCCESS(rc)
1893	&& Cpu.pCurInstr->opcode == OP_IRET)
1894	{
1895	uint32_t eip, selCS, uEFlags;
1896
1897	/* Iret crashes are bad as we have already changed the flags on the stack */
1898	rc = PGMPhysSimpleReadGCPtr(pVCpu, &eip, pCtx->esp, 4);
1899	rc \|= PGMPhysSimpleReadGCPtr(pVCpu, &selCS, pCtx->esp+4, 4);
1900	rc \|= PGMPhysSimpleReadGCPtr(pVCpu, &uEFlags, pCtx->esp+8, 4);
1901	if (rc == VINF_SUCCESS)
1902	{
1903	if ( (uEFlags & X86_EFL_VM)
1904	\|\| (selCS & X86_SEL_RPL) == 3)
1905	{
1906	uint32_t selSS, esp;
1907
1908	rc \|= PGMPhysSimpleReadGCPtr(pVCpu, &esp, pCtx->esp + 12, 4);
1909	rc \|= PGMPhysSimpleReadGCPtr(pVCpu, &selSS, pCtx->esp + 16, 4);
1910
1911	if (uEFlags & X86_EFL_VM)
1912	{
1913	uint32_t selDS, selES, selFS, selGS;
1914	rc = PGMPhysSimpleReadGCPtr(pVCpu, &selES, pCtx->esp + 20, 4);
1915	rc \|= PGMPhysSimpleReadGCPtr(pVCpu, &selDS, pCtx->esp + 24, 4);
1916	rc \|= PGMPhysSimpleReadGCPtr(pVCpu, &selFS, pCtx->esp + 28, 4);
1917	rc \|= PGMPhysSimpleReadGCPtr(pVCpu, &selGS, pCtx->esp + 32, 4);
1918	if (rc == VINF_SUCCESS)
1919	{
1920	Log(("Patch code: IRET->VM stack frame: return address %04X:%08RX32 eflags=%08x ss:esp=%04X:%08RX32\n", selCS, eip, uEFlags, selSS, esp));
1921	Log(("Patch code: IRET->VM stack frame: DS=%04X ES=%04X FS=%04X GS=%04X\n", selDS, selES, selFS, selGS));
1922	}
1923	}
1924	else
1925	Log(("Patch code: IRET stack frame: return address %04X:%08RX32 eflags=%08x ss:esp=%04X:%08RX32\n", selCS, eip, uEFlags, selSS, esp));
1926	}
1927	else
1928	Log(("Patch code: IRET stack frame: return address %04X:%08RX32 eflags=%08x\n", selCS, eip, uEFlags));
1929	}
1930	}
1931	#endif /* LOG_ENABLED */
1932	Log(("emR3PatchTrap: in patch: eip=%08x: trap=%02x err=%08x cr2=%08x cr0=%08x\n",
1933	pCtx->eip, u8TrapNo, uErrorCode, uCR2, (uint32_t)pCtx->cr0));
1934
1935	RTGCPTR pNewEip;
1936	rc = PATMR3HandleTrap(pVM, pCtx, pCtx->eip, &pNewEip);
1937	switch (rc)
1938	{
1939	/*
1940	* Execute the faulting instruction.
1941	*/
1942	case VINF_SUCCESS:
1943	{
1944	/** @todo execute a whole block */
1945	Log(("emR3PatchTrap: Executing faulting instruction at new address %RGv\n", pNewEip));
1946	if (!(pVCpu->em.s.pPatmGCState->uVMFlags & X86_EFL_IF))
1947	Log(("emR3PatchTrap: Virtual IF flag disabled!!\n"));
1948
1949	pCtx->eip = pNewEip;
1950	AssertRelease(pCtx->eip);
1951
1952	if (pCtx->eflags.Bits.u1IF)
1953	{
1954	/* Windows XP lets irets fault intentionally and then takes action based on the opcode; an
1955	* int3 patch overwrites it and leads to blue screens. Remove the patch in this case.
1956	*/
1957	if ( u8TrapNo == X86_XCPT_GP
1958	&& PATMIsInt3Patch(pVM, pCtx->eip, NULL, NULL))
1959	{
1960	/** @todo move to PATMR3HandleTrap */
1961	Log(("Possible Windows XP iret fault at %08RX32\n", pCtx->eip));
1962	PATMR3RemovePatch(pVM, pCtx->eip);
1963	}
1964
1965	/** @todo Knoppix 5 regression when returning VINF_SUCCESS here and going back to raw mode. */
1966	/* Note: possibly because a reschedule is required (e.g. iret to V86 code) */
1967
1968	return emR3RawExecuteInstruction(pVM, pVCpu, "PATCHIR");
1969	/* Interrupts are enabled; just go back to the original instruction.
1970	return VINF_SUCCESS; */
1971	}
1972	return VINF_EM_RESCHEDULE_REM;
1973	}
1974
1975	/*
1976	* One instruction.
1977	*/
1978	case VINF_PATCH_EMULATE_INSTR:
1979	Log(("emR3PatchTrap: Emulate patched instruction at %RGv IF=%d VMIF=%x\n",
1980	pNewEip, pCtx->eflags.Bits.u1IF, pVCpu->em.s.pPatmGCState->uVMFlags));
1981	pCtx->eip = pNewEip;
1982	AssertRelease(pCtx->eip);
1983	return emR3RawExecuteInstruction(pVM, pVCpu, "PATCHEMUL: ");
1984
1985	/*
1986	* The patch was disabled, hand it to the REM.
1987	*/
1988	case VERR_PATCH_DISABLED:
1989	if (!(pVCpu->em.s.pPatmGCState->uVMFlags & X86_EFL_IF))
1990	Log(("emR3PatchTrap: Virtual IF flag disabled!!\n"));
1991	pCtx->eip = pNewEip;
1992	AssertRelease(pCtx->eip);
1993
1994	if (pCtx->eflags.Bits.u1IF)
1995	{
1996	/*
1997	* The last instruction in the patch block needs to be executed!! (sti/sysexit for example)
1998	*/
1999	Log(("PATCH: IF=1 -> emulate last instruction as it can't be interrupted!!\n"));
2000	return emR3RawExecuteInstruction(pVM, pVCpu, "PATCHIR");
2001	}
2002	return VINF_EM_RESCHEDULE_REM;
2003
2004	/* Force continued patch exection; usually due to write monitored stack. */
2005	case VINF_PATCH_CONTINUE:
2006	return VINF_SUCCESS;
2007
2008	/*
2009	* Anything else is fatal.
2010	*/
2011	default:
2012	AssertReleaseMsgFailed(("Unknown return code %Rrc from PATMR3HandleTrap!\n", rc));
2013	return VERR_IPE_UNEXPECTED_STATUS;
2014	}
2015	}
2016	return VINF_SUCCESS;
2017	}
2018
2019
2020	/**
2021	* Handle a privileged instruction.
2022	*
2023	* @returns VBox status code suitable for EM.
2024	* @param pVM VM handle.
2025	* @param pVCpu VMCPU handle;
2026	*/
2027	int emR3RawPrivileged(PVM pVM, PVMCPU pVCpu)
2028	{
2029	STAM_PROFILE_START(&pVCpu->em.s.StatPrivEmu, a);
2030	PCPUMCTX pCtx = pVCpu->em.s.pCtx;
2031
2032	Assert(!pCtx->eflags.Bits.u1VM);
2033
2034	if (PATMIsEnabled(pVM))
2035	{
2036	/*
2037	* Check if in patch code.
2038	*/
2039	if (PATMR3IsInsidePatchJump(pVM, pCtx->eip, NULL))
2040	{
2041	#ifdef LOG_ENABLED
2042	DBGFR3InfoLog(pVM, "cpumguest", "PRIV");
2043	#endif
2044	AssertMsgFailed(("FATAL ERROR: executing random instruction inside generated patch jump %08X\n", pCtx->eip));
2045	return VERR_EM_RAW_PATCH_CONFLICT;
2046	}
2047	if ( (pCtx->ss & X86_SEL_RPL) == 0
2048	&& !pCtx->eflags.Bits.u1VM
2049	&& !PATMIsPatchGCAddr(pVM, pCtx->eip))
2050	{
2051	int rc = PATMR3InstallPatch(pVM, SELMToFlat(pVM, DIS_SELREG_CS, CPUMCTX2CORE(pCtx), pCtx->eip),
2052	(SELMGetCpuModeFromSelector(pVM, pCtx->eflags, pCtx->cs, &pCtx->csHid) == CPUMODE_32BIT) ? PATMFL_CODE32 : 0);
2053	if (RT_SUCCESS(rc))
2054	{
2055	#ifdef LOG_ENABLED
2056	DBGFR3InfoLog(pVM, "cpumguest", "PRIV");
2057	#endif
2058	DBGFR3DisasInstrCurrentLog(pVCpu, "Patched privileged instruction");
2059	return VINF_SUCCESS;
2060	}
2061	}
2062	}
2063
2064	#ifdef LOG_ENABLED
2065	if (!PATMIsPatchGCAddr(pVM, pCtx->eip))
2066	{
2067	DBGFR3InfoLog(pVM, "cpumguest", "PRIV");
2068	DBGFR3DisasInstrCurrentLog(pVCpu, "Privileged instr: ");
2069	}
2070	#endif
2071
2072	/*
2073	* Instruction statistics and logging.
2074	*/
2075	DISCPUSTATE Cpu;
2076	int rc;
2077
2078	rc = CPUMR3DisasmInstrCPU(pVM, pVCpu, pCtx, pCtx->rip, &Cpu, "PRIV: ");
2079	if (RT_SUCCESS(rc))
2080	{
2081	#ifdef VBOX_WITH_STATISTICS
2082	PEMSTATS pStats = pVCpu->em.s.CTX_SUFF(pStats);
2083	switch (Cpu.pCurInstr->opcode)
2084	{
2085	case OP_INVLPG:
2086	STAM_COUNTER_INC(&pStats->StatInvlpg);
2087	break;
2088	case OP_IRET:
2089	STAM_COUNTER_INC(&pStats->StatIret);
2090	break;
2091	case OP_CLI:
2092	STAM_COUNTER_INC(&pStats->StatCli);
2093	emR3RecordCli(pVM, pVCpu, pCtx->rip);
2094	break;
2095	case OP_STI:
2096	STAM_COUNTER_INC(&pStats->StatSti);
2097	break;
2098	case OP_INSB:
2099	case OP_INSWD:
2100	case OP_IN:
2101	case OP_OUTSB:
2102	case OP_OUTSWD:
2103	case OP_OUT:
2104	AssertMsgFailed(("Unexpected privileged exception due to port IO\n"));
2105	break;
2106
2107	case OP_MOV_CR:
2108	if (Cpu.param1.flags & USE_REG_GEN32)
2109	{
2110	//read
2111	Assert(Cpu.param2.flags & USE_REG_CR);
2112	Assert(Cpu.param2.base.reg_ctrl <= USE_REG_CR4);
2113	STAM_COUNTER_INC(&pStats->StatMovReadCR[Cpu.param2.base.reg_ctrl]);
2114	}
2115	else
2116	{
2117	//write
2118	Assert(Cpu.param1.flags & USE_REG_CR);
2119	Assert(Cpu.param1.base.reg_ctrl <= USE_REG_CR4);
2120	STAM_COUNTER_INC(&pStats->StatMovWriteCR[Cpu.param1.base.reg_ctrl]);
2121	}
2122	break;
2123
2124	case OP_MOV_DR:
2125	STAM_COUNTER_INC(&pStats->StatMovDRx);
2126	break;
2127	case OP_LLDT:
2128	STAM_COUNTER_INC(&pStats->StatMovLldt);
2129	break;
2130	case OP_LIDT:
2131	STAM_COUNTER_INC(&pStats->StatMovLidt);
2132	break;
2133	case OP_LGDT:
2134	STAM_COUNTER_INC(&pStats->StatMovLgdt);
2135	break;
2136	case OP_SYSENTER:
2137	STAM_COUNTER_INC(&pStats->StatSysEnter);
2138	break;
2139	case OP_SYSEXIT:
2140	STAM_COUNTER_INC(&pStats->StatSysExit);
2141	break;
2142	case OP_SYSCALL:
2143	STAM_COUNTER_INC(&pStats->StatSysCall);
2144	break;
2145	case OP_SYSRET:
2146	STAM_COUNTER_INC(&pStats->StatSysRet);
2147	break;
2148	case OP_HLT:
2149	STAM_COUNTER_INC(&pStats->StatHlt);
2150	break;
2151	default:
2152	STAM_COUNTER_INC(&pStats->StatMisc);
2153	Log4(("emR3RawPrivileged: opcode=%d\n", Cpu.pCurInstr->opcode));
2154	break;
2155	}
2156	#endif /* VBOX_WITH_STATISTICS */
2157	if ( (pCtx->ss & X86_SEL_RPL) == 0
2158	&& !pCtx->eflags.Bits.u1VM
2159	&& SELMGetCpuModeFromSelector(pVM, pCtx->eflags, pCtx->cs, &pCtx->csHid) == CPUMODE_32BIT)
2160	{
2161	uint32_t size;
2162
2163	STAM_PROFILE_START(&pVCpu->em.s.StatPrivEmu, a);
2164	switch (Cpu.pCurInstr->opcode)
2165	{
2166	case OP_CLI:
2167	pCtx->eflags.u32 &= ~X86_EFL_IF;
2168	Assert(Cpu.opsize == 1);
2169	pCtx->rip += Cpu.opsize;
2170	STAM_PROFILE_STOP(&pVCpu->em.s.StatPrivEmu, a);
2171	return VINF_EM_RESCHEDULE_REM; /* must go to the recompiler now! */
2172
2173	case OP_STI:
2174	pCtx->eflags.u32 \|= X86_EFL_IF;
2175	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + Cpu.opsize);
2176	Assert(Cpu.opsize == 1);
2177	pCtx->rip += Cpu.opsize;
2178	STAM_PROFILE_STOP(&pVCpu->em.s.StatPrivEmu, a);
2179	return VINF_SUCCESS;
2180
2181	case OP_HLT:
2182	if (PATMIsPatchGCAddr(pVM, (RTGCPTR)pCtx->eip))
2183	{
2184	PATMTRANSSTATE enmState;
2185	RTGCPTR pOrgInstrGC = PATMR3PatchToGCPtr(pVM, pCtx->eip, &enmState);
2186
2187	if (enmState == PATMTRANS_OVERWRITTEN)
2188	{
2189	rc = PATMR3DetectConflict(pVM, pOrgInstrGC, pOrgInstrGC);
2190	Assert(rc == VERR_PATCH_DISABLED);
2191	/* Conflict detected, patch disabled */
2192	Log(("emR3RawPrivileged: detected conflict -> disabled patch at %08RX32\n", pCtx->eip));
2193
2194	enmState = PATMTRANS_SAFE;
2195	}
2196
2197	/* The translation had better be successful. Otherwise we can't recover. */
2198	AssertReleaseMsg(pOrgInstrGC && enmState != PATMTRANS_OVERWRITTEN, ("Unable to translate instruction address at %08RX32\n", pCtx->eip));
2199	if (enmState != PATMTRANS_OVERWRITTEN)
2200	pCtx->eip = pOrgInstrGC;
2201	}
2202	/* no break; we could just return VINF_EM_HALT here */
2203
2204	case OP_MOV_CR:
2205	case OP_MOV_DR:
2206	#ifdef LOG_ENABLED
2207	if (PATMIsPatchGCAddr(pVM, pCtx->eip))
2208	{
2209	DBGFR3InfoLog(pVM, "cpumguest", "PRIV");
2210	DBGFR3DisasInstrCurrentLog(pVCpu, "Privileged instr: ");
2211	}
2212	#endif
2213
2214	rc = EMInterpretInstructionCPU(pVM, pVCpu, &Cpu, CPUMCTX2CORE(pCtx), 0, &size);
2215	if (RT_SUCCESS(rc))
2216	{
2217	pCtx->rip += Cpu.opsize;
2218	#ifdef EM_NOTIFY_HWACCM
2219	if (pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_HWACC)
2220	HWACCMR3NotifyEmulated(pVCpu);
2221	#endif
2222	STAM_PROFILE_STOP(&pVCpu->em.s.StatPrivEmu, a);
2223
2224	if ( Cpu.pCurInstr->opcode == OP_MOV_CR
2225	&& Cpu.param1.flags == USE_REG_CR /* write */
2226	)
2227	{
2228	/* Deal with CR0 updates inside patch code that force
2229	* us to go to the recompiler.
2230	*/
2231	if ( PATMIsPatchGCAddr(pVM, pCtx->rip)
2232	&& (pCtx->cr0 & (X86_CR0_WP\|X86_CR0_PG\|X86_CR0_PE)) != (X86_CR0_WP\|X86_CR0_PG\|X86_CR0_PE))
2233	{
2234	PATMTRANSSTATE enmState;
2235	RTGCPTR pOrgInstrGC = PATMR3PatchToGCPtr(pVM, pCtx->rip, &enmState);
2236
2237	Log(("Force recompiler switch due to cr0 (%RGp) update rip=%RGv -> %RGv (enmState=%d)\n", pCtx->cr0, pCtx->rip, pOrgInstrGC, enmState));
2238	if (enmState == PATMTRANS_OVERWRITTEN)
2239	{
2240	rc = PATMR3DetectConflict(pVM, pOrgInstrGC, pOrgInstrGC);
2241	Assert(rc == VERR_PATCH_DISABLED);
2242	/* Conflict detected, patch disabled */
2243	Log(("emR3RawPrivileged: detected conflict -> disabled patch at %RGv\n", (RTGCPTR)pCtx->rip));
2244	enmState = PATMTRANS_SAFE;
2245	}
2246	/* The translation had better be successful. Otherwise we can't recover. */
2247	AssertReleaseMsg(pOrgInstrGC && enmState != PATMTRANS_OVERWRITTEN, ("Unable to translate instruction address at %RGv\n", (RTGCPTR)pCtx->rip));
2248	if (enmState != PATMTRANS_OVERWRITTEN)
2249	pCtx->rip = pOrgInstrGC;
2250	}
2251
2252	/* Reschedule is necessary as the execution/paging mode might have changed. */
2253	return VINF_EM_RESCHEDULE;
2254	}
2255	return rc; /* can return VINF_EM_HALT as well. */
2256	}
2257	AssertMsgReturn(rc == VERR_EM_INTERPRETER, ("%Rrc\n", rc), rc);
2258	break; /* fall back to the recompiler */
2259	}
2260	STAM_PROFILE_STOP(&pVCpu->em.s.StatPrivEmu, a);
2261	}
2262	}
2263
2264	if (PATMIsPatchGCAddr(pVM, pCtx->eip))
2265	return emR3PatchTrap(pVM, pVCpu, pCtx, VINF_PATM_PATCH_TRAP_GP);
2266
2267	return emR3RawExecuteInstruction(pVM, pVCpu, "PRIV");
2268	}
2269
2270
2271	/**
2272	* Update the forced rawmode execution modifier.
2273	*
2274	* This function is called when we're returning from the raw-mode loop(s). If we're
2275	* in patch code, it will set a flag forcing execution to be resumed in raw-mode,
2276	* if not in patch code, the flag will be cleared.
2277	*
2278	* We should never interrupt patch code while it's being executed. Cli patches can
2279	* contain big code blocks, but they are always executed with IF=0. Other patches
2280	* replace single instructions and should be atomic.
2281	*
2282	* @returns Updated rc.
2283	*
2284	* @param pVM The VM handle.
2285	* @param pVCpu The VMCPU handle.
2286	* @param pCtx The guest CPU context.
2287	* @param rc The result code.
2288	*/
2289	DECLINLINE(int) emR3RawUpdateForceFlag(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, int rc)
2290	{
2291	if (PATMIsPatchGCAddr(pVM, pCtx->eip)) /** @todo check cs selector base/type */
2292	{
2293	/* ignore reschedule attempts. */
2294	switch (rc)
2295	{
2296	case VINF_EM_RESCHEDULE:
2297	case VINF_EM_RESCHEDULE_REM:
2298	LogFlow(("emR3RawUpdateForceFlag: patch address -> force raw reschedule\n"));
2299	rc = VINF_SUCCESS;
2300	break;
2301	}
2302	pVCpu->em.s.fForceRAW = true;
2303	}
2304	else
2305	pVCpu->em.s.fForceRAW = false;
2306	return rc;
2307	}
2308
2309
2310	/**
2311	* Process a subset of the raw-mode return code.
2312	*
2313	* Since we have to share this with raw-mode single stepping, this inline
2314	* function has been created to avoid code duplication.
2315	*
2316	* @returns VINF_SUCCESS if it's ok to continue raw mode.
2317	* @returns VBox status code to return to the EM main loop.
2318	*
2319	* @param pVM The VM handle
2320	* @param pVCpu The VMCPU handle
2321	* @param rc The return code.
2322	* @param pCtx The guest cpu context.
2323	*/
2324	DECLINLINE(int) emR3RawHandleRC(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, int rc)
2325	{
2326	switch (rc)
2327	{
2328	/*
2329	* Common & simple ones.
2330	*/
2331	case VINF_SUCCESS:
2332	break;
2333	case VINF_EM_RESCHEDULE_RAW:
2334	case VINF_EM_RESCHEDULE_HWACC:
2335	case VINF_EM_RAW_INTERRUPT:
2336	case VINF_EM_RAW_TO_R3:
2337	case VINF_EM_RAW_TIMER_PENDING:
2338	case VINF_EM_PENDING_REQUEST:
2339	rc = VINF_SUCCESS;
2340	break;
2341
2342	/*
2343	* Privileged instruction.
2344	*/
2345	case VINF_EM_RAW_EXCEPTION_PRIVILEGED:
2346	case VINF_PATM_PATCH_TRAP_GP:
2347	rc = emR3RawPrivileged(pVM, pVCpu);
2348	break;
2349
2350	/*
2351	* Got a trap which needs dispatching.
2352	*/
2353	case VINF_EM_RAW_GUEST_TRAP:
2354	if (PATMR3IsInsidePatchJump(pVM, pCtx->eip, NULL))
2355	{
2356	AssertReleaseMsgFailed(("FATAL ERROR: executing random instruction inside generated patch jump %08X\n", CPUMGetGuestEIP(pVCpu)));
2357	rc = VERR_EM_RAW_PATCH_CONFLICT;
2358	break;
2359	}
2360	rc = emR3RawGuestTrap(pVM, pVCpu);
2361	break;
2362
2363	/*
2364	* Trap in patch code.
2365	*/
2366	case VINF_PATM_PATCH_TRAP_PF:
2367	case VINF_PATM_PATCH_INT3:
2368	rc = emR3PatchTrap(pVM, pVCpu, pCtx, rc);
2369	break;
2370
2371	case VINF_PATM_DUPLICATE_FUNCTION:
2372	Assert(PATMIsPatchGCAddr(pVM, (RTGCPTR)pCtx->eip));
2373	rc = PATMR3DuplicateFunctionRequest(pVM, pCtx);
2374	AssertRC(rc);
2375	rc = VINF_SUCCESS;
2376	break;
2377
2378	case VINF_PATM_CHECK_PATCH_PAGE:
2379	rc = PATMR3HandleMonitoredPage(pVM);
2380	AssertRC(rc);
2381	rc = VINF_SUCCESS;
2382	break;
2383
2384	/*
2385	* Patch manager.
2386	*/
2387	case VERR_EM_RAW_PATCH_CONFLICT:
2388	AssertReleaseMsgFailed(("%Rrc handling is not yet implemented\n", rc));
2389	break;
2390
2391	#ifdef VBOX_WITH_VMI
2392	/*
2393	* PARAV function.
2394	*/
2395	case VINF_EM_RESCHEDULE_PARAV:
2396	rc = PARAVCallFunction(pVM);
2397	break;
2398	#endif
2399
2400	/*
2401	* Memory mapped I/O access - attempt to patch the instruction
2402	*/
2403	case VINF_PATM_HC_MMIO_PATCH_READ:
2404	rc = PATMR3InstallPatch(pVM, SELMToFlat(pVM, DIS_SELREG_CS, CPUMCTX2CORE(pCtx), pCtx->eip),
2405	PATMFL_MMIO_ACCESS \| ((SELMGetCpuModeFromSelector(pVM, pCtx->eflags, pCtx->cs, &pCtx->csHid) == CPUMODE_32BIT) ? PATMFL_CODE32 : 0));
2406	if (RT_FAILURE(rc))
2407	rc = emR3RawExecuteInstruction(pVM, pVCpu, "MMIO");
2408	break;
2409
2410	case VINF_PATM_HC_MMIO_PATCH_WRITE:
2411	AssertFailed(); /* not yet implemented. */
2412	rc = emR3RawExecuteInstruction(pVM, pVCpu, "MMIO");
2413	break;
2414
2415	/*
2416	* Conflict or out of page tables.
2417	*
2418	* VM_FF_PGM_SYNC_CR3 is set by the hypervisor and all we need to
2419	* do here is to execute the pending forced actions.
2420	*/
2421	case VINF_PGM_SYNC_CR3:
2422	AssertMsg(VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL),
2423	("VINF_PGM_SYNC_CR3 and no VMCPU_FF_PGM_SYNC_CR3*!\n"));
2424	rc = VINF_SUCCESS;
2425	break;
2426
2427	/*
2428	* Paging mode change.
2429	*/
2430	case VINF_PGM_CHANGE_MODE:
2431	rc = PGMChangeMode(pVCpu, pCtx->cr0, pCtx->cr4, pCtx->msrEFER);
2432	if (rc == VINF_SUCCESS)
2433	rc = VINF_EM_RESCHEDULE;
2434	AssertMsg(RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST), ("%Rrc\n", rc));
2435	break;
2436
2437	/*
2438	* CSAM wants to perform a task in ring-3. It has set an FF action flag.
2439	*/
2440	case VINF_CSAM_PENDING_ACTION:
2441	rc = VINF_SUCCESS;
2442	break;
2443
2444	/*
2445	* Invoked Interrupt gate - must directly (!) go to the recompiler.
2446	*/
2447	case VINF_EM_RAW_INTERRUPT_PENDING:
2448	case VINF_EM_RAW_RING_SWITCH_INT:
2449	Assert(TRPMHasTrap(pVCpu));
2450	Assert(!PATMIsPatchGCAddr(pVM, (RTGCPTR)pCtx->eip));
2451
2452	if (TRPMHasTrap(pVCpu))
2453	{
2454	/* If the guest gate is marked unpatched, then we will check again if we can patch it. */
2455	uint8_t u8Interrupt = TRPMGetTrapNo(pVCpu);
2456	if (TRPMR3GetGuestTrapHandler(pVM, u8Interrupt) == TRPM_INVALID_HANDLER)
2457	{
2458	CSAMR3CheckGates(pVM, u8Interrupt, 1);
2459	Log(("emR3RawHandleRC: recheck gate %x -> valid=%d\n", u8Interrupt, TRPMR3GetGuestTrapHandler(pVM, u8Interrupt) != TRPM_INVALID_HANDLER));
2460	/* Note: If it was successful, then we could go back to raw mode, but let's keep things simple for now. */
2461	}
2462	}
2463	rc = VINF_EM_RESCHEDULE_REM;
2464	break;
2465
2466	/*
2467	* Other ring switch types.
2468	*/
2469	case VINF_EM_RAW_RING_SWITCH:
2470	rc = emR3RawRingSwitch(pVM, pVCpu);
2471	break;
2472
2473	/*
2474	* I/O Port access - emulate the instruction.
2475	*/
2476	case VINF_IOM_HC_IOPORT_READ:
2477	case VINF_IOM_HC_IOPORT_WRITE:
2478	rc = emR3RawExecuteIOInstruction(pVM, pVCpu);
2479	break;
2480
2481	/*
2482	* Memory mapped I/O access - emulate the instruction.
2483	*/
2484	case VINF_IOM_HC_MMIO_READ:
2485	case VINF_IOM_HC_MMIO_WRITE:
2486	case VINF_IOM_HC_MMIO_READ_WRITE:
2487	rc = emR3RawExecuteInstruction(pVM, pVCpu, "MMIO");
2488	break;
2489
2490	/*
2491	* (MM)IO intensive code block detected; fall back to the recompiler for better performance
2492	*/
2493	case VINF_EM_RAW_EMULATE_IO_BLOCK:
2494	rc = HWACCMR3EmulateIoBlock(pVM, pCtx);
2495	break;
2496
2497	/*
2498	* Execute instruction.
2499	*/
2500	case VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT:
2501	rc = emR3RawExecuteInstruction(pVM, pVCpu, "LDT FAULT: ");
2502	break;
2503	case VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT:
2504	rc = emR3RawExecuteInstruction(pVM, pVCpu, "GDT FAULT: ");
2505	break;
2506	case VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT:
2507	rc = emR3RawExecuteInstruction(pVM, pVCpu, "IDT FAULT: ");
2508	break;
2509	case VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT:
2510	rc = emR3RawExecuteInstruction(pVM, pVCpu, "TSS FAULT: ");
2511	break;
2512	case VINF_EM_RAW_EMULATE_INSTR_PD_FAULT:
2513	rc = emR3RawExecuteInstruction(pVM, pVCpu, "PD FAULT: ");
2514	break;
2515
2516	case VINF_EM_RAW_EMULATE_INSTR_HLT:
2517	/** @todo skip instruction and go directly to the halt state. (see REM for implementation details) */
2518	rc = emR3RawPrivileged(pVM, pVCpu);
2519	break;
2520
2521	case VINF_PATM_PENDING_IRQ_AFTER_IRET:
2522	rc = emR3RawExecuteInstruction(pVM, pVCpu, "EMUL: ", VINF_PATM_PENDING_IRQ_AFTER_IRET);
2523	break;
2524
2525	case VINF_EM_RAW_EMULATE_INSTR:
2526	case VINF_PATCH_EMULATE_INSTR:
2527	rc = emR3RawExecuteInstruction(pVM, pVCpu, "EMUL: ");
2528	break;
2529
2530	/*
2531	* Stale selector and iret traps => REM.
2532	*/
2533	case VINF_EM_RAW_STALE_SELECTOR:
2534	case VINF_EM_RAW_IRET_TRAP:
2535	/* We will not go to the recompiler if EIP points to patch code. */
2536	if (PATMIsPatchGCAddr(pVM, pCtx->eip))
2537	{
2538	pCtx->eip = PATMR3PatchToGCPtr(pVM, (RTGCPTR)pCtx->eip, 0);
2539	}
2540	LogFlow(("emR3RawHandleRC: %Rrc -> %Rrc\n", rc, VINF_EM_RESCHEDULE_REM));
2541	rc = VINF_EM_RESCHEDULE_REM;
2542	break;
2543
2544	/*
2545	* Up a level.
2546	*/
2547	case VINF_EM_TERMINATE:
2548	case VINF_EM_OFF:
2549	case VINF_EM_RESET:
2550	case VINF_EM_SUSPEND:
2551	case VINF_EM_HALT:
2552	case VINF_EM_RESUME:
2553	case VINF_EM_NO_MEMORY:
2554	case VINF_EM_RESCHEDULE:
2555	case VINF_EM_RESCHEDULE_REM:
2556	case VINF_EM_WAIT_SIPI:
2557	break;
2558
2559	/*
2560	* Up a level and invoke the debugger.
2561	*/
2562	case VINF_EM_DBG_STEPPED:
2563	case VINF_EM_DBG_BREAKPOINT:
2564	case VINF_EM_DBG_STEP:
2565	case VINF_EM_DBG_HYPER_BREAKPOINT:
2566	case VINF_EM_DBG_HYPER_STEPPED:
2567	case VINF_EM_DBG_HYPER_ASSERTION:
2568	case VINF_EM_DBG_STOP:
2569	break;
2570
2571	/*
2572	* Up a level, dump and debug.
2573	*/
2574	case VERR_TRPM_DONT_PANIC:
2575	case VERR_TRPM_PANIC:
2576	case VERR_VMM_RING0_ASSERTION:
2577	case VERR_VMM_HYPER_CR3_MISMATCH:
2578	case VERR_VMM_RING3_CALL_DISABLED:
2579	break;
2580
2581	/*
2582	* Up a level, after HwAccM have done some release logging.
2583	*/
2584	case VERR_VMX_INVALID_VMCS_FIELD:
2585	case VERR_VMX_INVALID_VMCS_PTR:
2586	case VERR_VMX_INVALID_VMXON_PTR:
2587	case VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE:
2588	case VERR_VMX_UNEXPECTED_EXCEPTION:
2589	case VERR_VMX_UNEXPECTED_EXIT_CODE:
2590	case VERR_VMX_INVALID_GUEST_STATE:
2591	case VERR_VMX_UNABLE_TO_START_VM:
2592	case VERR_VMX_UNABLE_TO_RESUME_VM:
2593	HWACCMR3CheckError(pVM, rc);
2594	break;
2595	/*
2596	* Anything which is not known to us means an internal error
2597	* and the termination of the VM!
2598	*/
2599	default:
2600	AssertMsgFailed(("Unknown GC return code: %Rra\n", rc));
2601	break;
2602	}
2603	return rc;
2604	}
2605
2606
2607	/**
2608	* Check for pending raw actions
2609	*
2610	* @returns VBox status code. May return VINF_EM_NO_MEMORY but none of the other
2611	* EM statuses.
2612	* @param pVM The VM to operate on.
2613	* @param pVCpu The VMCPU handle.
2614	*/
2615	VMMR3DECL(int) EMR3CheckRawForcedActions(PVM pVM, PVMCPU pVCpu)
2616	{
2617	return emR3RawForcedActions(pVM, pVCpu, pVCpu->em.s.pCtx);
2618	}
2619
2620
2621	/**
2622	* Process raw-mode specific forced actions.
2623	*
2624	* This function is called when any FFs in the VM_FF_HIGH_PRIORITY_PRE_RAW_MASK is pending.
2625	*
2626	* @returns VBox status code. May return VINF_EM_NO_MEMORY but none of the other
2627	* EM statuses.
2628	* @param pVM The VM handle.
2629	* @param pVCpu The VMCPU handle.
2630	* @param pCtx The guest CPUM register context.
2631	*/
2632	static int emR3RawForcedActions(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2633	{
2634	/*
2635	* Note that the order is vitally important!
2636	* Also note that SELMR3UpdateFromCPUM may trigger VM_FF_SELM_SYNC_TSS.
2637	*/
2638
2639
2640	/*
2641	* Sync selector tables.
2642	*/
2643	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_SELM_SYNC_GDT \| VMCPU_FF_SELM_SYNC_LDT))
2644	{
2645	int rc = SELMR3UpdateFromCPUM(pVM, pVCpu);
2646	if (RT_FAILURE(rc))
2647	return rc;
2648	}
2649
2650	/*
2651	* Sync IDT.
2652	*
2653	* The CSAMR3CheckGates call in TRPMR3SyncIDT may call PGMPrefetchPage
2654	* and PGMShwModifyPage, so we're in for trouble if for instance a
2655	* PGMSyncCR3+pgmPoolClearAll is pending.
2656	*/
2657	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TRPM_SYNC_IDT))
2658	{
2659	if ( VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
2660	&& EMIsRawRing0Enabled(pVM)
2661	&& CSAMIsEnabled(pVM))
2662	{
2663	int rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
2664	if (RT_FAILURE(rc))
2665	return rc;
2666	}
2667
2668	int rc = TRPMR3SyncIDT(pVM, pVCpu);
2669	if (RT_FAILURE(rc))
2670	return rc;
2671	}
2672
2673	/*
2674	* Sync TSS.
2675	*/
2676	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_SELM_SYNC_TSS))
2677	{
2678	int rc = SELMR3SyncTSS(pVM, pVCpu);
2679	if (RT_FAILURE(rc))
2680	return rc;
2681	}
2682
2683	/*
2684	* Sync page directory.
2685	*/
2686	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
2687	{
2688	Assert(pVCpu->em.s.enmState != EMSTATE_WAIT_SIPI);
2689	int rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
2690	if (RT_FAILURE(rc))
2691	return rc;
2692
2693	Assert(!VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_SELM_SYNC_GDT \| VMCPU_FF_SELM_SYNC_LDT));
2694
2695	/* Prefetch pages for EIP and ESP. */
2696	/** @todo This is rather expensive. Should investigate if it really helps at all. */
2697	rc = PGMPrefetchPage(pVCpu, SELMToFlat(pVM, DIS_SELREG_CS, CPUMCTX2CORE(pCtx), pCtx->rip));
2698	if (rc == VINF_SUCCESS)
2699	rc = PGMPrefetchPage(pVCpu, SELMToFlat(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->rsp));
2700	if (rc != VINF_SUCCESS)
2701	{
2702	if (rc != VINF_PGM_SYNC_CR3)
2703	{
2704	AssertLogRelMsgReturn(RT_FAILURE(rc), ("%Rrc\n", rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
2705	return rc;
2706	}
2707	rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
2708	if (RT_FAILURE(rc))
2709	return rc;
2710	}
2711	/** @todo maybe prefetch the supervisor stack page as well */
2712	Assert(!VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_SELM_SYNC_GDT \| VMCPU_FF_SELM_SYNC_LDT));
2713	}
2714
2715	/*
2716	* Allocate handy pages (just in case the above actions have consumed some pages).
2717	*/
2718	if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_PGM_NEED_HANDY_PAGES, VM_FF_PGM_NO_MEMORY))
2719	{
2720	int rc = PGMR3PhysAllocateHandyPages(pVM);
2721	if (RT_FAILURE(rc))
2722	return rc;
2723	}
2724
2725	/*
2726	* Check whether we're out of memory now.
2727	*
2728	* This may stem from some of the above actions or operations that has been executed
2729	* since we ran FFs. The allocate handy pages must for instance always be followed by
2730	* this check.
2731	*/
2732	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2733	return VINF_EM_NO_MEMORY;
2734
2735	return VINF_SUCCESS;
2736	}
2737
2738
2739	/**
2740	* Executes raw code.
2741	*
2742	* This function contains the raw-mode version of the inner
2743	* execution loop (the outer loop being in EMR3ExecuteVM()).
2744	*
2745	* @returns VBox status code. The most important ones are: VINF_EM_RESCHEDULE,
2746	* VINF_EM_RESCHEDULE_REM, VINF_EM_SUSPEND, VINF_EM_RESET and VINF_EM_TERMINATE.
2747	*
2748	* @param pVM VM handle.
2749	* @param pVCpu VMCPU handle.
2750	* @param pfFFDone Where to store an indicator telling whether or not
2751	* FFs were done before returning.
2752	*/
2753	static int emR3RawExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone)
2754	{
2755	STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatRAWTotal, a);
2756
2757	int rc = VERR_INTERNAL_ERROR;
2758	PCPUMCTX pCtx = pVCpu->em.s.pCtx;
2759	LogFlow(("emR3RawExecute: (cs:eip=%04x:%08x)\n", pCtx->cs, pCtx->eip));
2760	pVCpu->em.s.fForceRAW = false;
2761	*pfFFDone = false;
2762
2763
2764	/*
2765	*
2766	* Spin till we get a forced action or raw mode status code resulting in
2767	* in anything but VINF_SUCCESS or VINF_EM_RESCHEDULE_RAW.
2768	*
2769	*/
2770	for (;;)
2771	{
2772	STAM_PROFILE_ADV_START(&pVCpu->em.s.StatRAWEntry, b);
2773
2774	/*
2775	* Check various preconditions.
2776	*/
2777	#ifdef VBOX_STRICT
2778	Assert(REMR3QueryPendingInterrupt(pVM, pVCpu) == REM_NO_PENDING_IRQ);
2779	Assert(pCtx->eflags.Bits.u1VM \|\| (pCtx->ss & X86_SEL_RPL) == 3 \|\| (pCtx->ss & X86_SEL_RPL) == 0);
2780	AssertMsg( (pCtx->eflags.u32 & X86_EFL_IF)
2781	\|\| PATMShouldUseRawMode(pVM, (RTGCPTR)pCtx->eip),
2782	("Tried to execute code with IF at EIP=%08x!\n", pCtx->eip));
2783	if ( !VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
2784	&& PGMMapHasConflicts(pVM))
2785	{
2786	PGMMapCheck(pVM);
2787	AssertMsgFailed(("We should not get conflicts any longer!!!\n"));
2788	return VERR_INTERNAL_ERROR;
2789	}
2790	#endif /* VBOX_STRICT */
2791
2792	/*
2793	* Process high priority pre-execution raw-mode FFs.
2794	*/
2795	if ( VM_FF_ISPENDING(pVM, VM_FF_HIGH_PRIORITY_PRE_RAW_MASK)
2796	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_PRE_RAW_MASK))
2797	{
2798	rc = emR3RawForcedActions(pVM, pVCpu, pCtx);
2799	if (rc != VINF_SUCCESS)
2800	break;
2801	}
2802
2803	/*
2804	* If we're going to execute ring-0 code, the guest state needs to
2805	* be modified a bit and some of the state components (IF, SS/CS RPL,
2806	* and perhaps EIP) needs to be stored with PATM.
2807	*/
2808	rc = CPUMRawEnter(pVCpu, NULL);
2809	if (rc != VINF_SUCCESS)
2810	{
2811	STAM_PROFILE_ADV_STOP(&pVCpu->em.s.StatRAWEntry, b);
2812	break;
2813	}
2814
2815	/*
2816	* Scan code before executing it. Don't bother with user mode or V86 code
2817	*/
2818	if ( (pCtx->ss & X86_SEL_RPL) <= 1
2819	&& !pCtx->eflags.Bits.u1VM
2820	&& !PATMIsPatchGCAddr(pVM, pCtx->eip))
2821	{
2822	STAM_PROFILE_ADV_SUSPEND(&pVCpu->em.s.StatRAWEntry, b);
2823	CSAMR3CheckCodeEx(pVM, CPUMCTX2CORE(pCtx), pCtx->eip);
2824	STAM_PROFILE_ADV_RESUME(&pVCpu->em.s.StatRAWEntry, b);
2825	if ( VM_FF_ISPENDING(pVM, VM_FF_HIGH_PRIORITY_PRE_RAW_MASK)
2826	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_PRE_RAW_MASK))
2827	{
2828	rc = emR3RawForcedActions(pVM, pVCpu, pCtx);
2829	if (rc != VINF_SUCCESS)
2830	{
2831	rc = CPUMRawLeave(pVCpu, NULL, rc);
2832	break;
2833	}
2834	}
2835	}
2836
2837	#ifdef LOG_ENABLED
2838	/*
2839	* Log important stuff before entering GC.
2840	*/
2841	PPATMGCSTATE pGCState = PATMR3QueryGCStateHC(pVM);
2842	if (pCtx->eflags.Bits.u1VM)
2843	Log(("RV86: %04X:%08X IF=%d VMFlags=%x\n", pCtx->cs, pCtx->eip, pCtx->eflags.Bits.u1IF, pGCState->uVMFlags));
2844	else if ((pCtx->ss & X86_SEL_RPL) == 1)
2845	{
2846	bool fCSAMScanned = CSAMIsPageScanned(pVM, (RTGCPTR)pCtx->eip);
2847	Log(("RR0: %08X ESP=%08X IF=%d VMFlags=%x PIF=%d CPL=%d (Scanned=%d)\n", pCtx->eip, pCtx->esp, pCtx->eflags.Bits.u1IF, pGCState->uVMFlags, pGCState->fPIF, (pCtx->ss & X86_SEL_RPL), fCSAMScanned));
2848	}
2849	else if ((pCtx->ss & X86_SEL_RPL) == 3)
2850	Log(("RR3: %08X ESP=%08X IF=%d VMFlags=%x\n", pCtx->eip, pCtx->esp, pCtx->eflags.Bits.u1IF, pGCState->uVMFlags));
2851	#endif /* LOG_ENABLED */
2852
2853
2854
2855	/*
2856	* Execute the code.
2857	*/
2858	STAM_PROFILE_ADV_STOP(&pVCpu->em.s.StatRAWEntry, b);
2859	STAM_PROFILE_START(&pVCpu->em.s.StatRAWExec, c);
2860	rc = VMMR3RawRunGC(pVM, pVCpu);
2861	STAM_PROFILE_STOP(&pVCpu->em.s.StatRAWExec, c);
2862	STAM_PROFILE_ADV_START(&pVCpu->em.s.StatRAWTail, d);
2863
2864	LogFlow(("RR0-E: %08X ESP=%08X IF=%d VMFlags=%x PIF=%d CPL=%d\n", pCtx->eip, pCtx->esp, pCtx->eflags.Bits.u1IF, pGCState->uVMFlags, pGCState->fPIF, (pCtx->ss & X86_SEL_RPL)));
2865	LogFlow(("VMMR3RawRunGC returned %Rrc\n", rc));
2866
2867
2868
2869	/*
2870	* Restore the real CPU state and deal with high priority post
2871	* execution FFs before doing anything else.
2872	*/
2873	rc = CPUMRawLeave(pVCpu, NULL, rc);
2874	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_RESUME_GUEST_MASK);
2875	if ( VM_FF_ISPENDING(pVM, VM_FF_HIGH_PRIORITY_POST_MASK)
2876	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_POST_MASK))
2877	rc = emR3HighPriorityPostForcedActions(pVM, pVCpu, rc);
2878
2879	#ifdef VBOX_STRICT
2880	/*
2881	* Assert TSS consistency & rc vs patch code.
2882	*/
2883	if ( !VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_SELM_SYNC_TSS \| VMCPU_FF_SELM_SYNC_GDT) /* GDT implies TSS at the moment. */
2884	&& EMIsRawRing0Enabled(pVM))
2885	SELMR3CheckTSS(pVM);
2886	switch (rc)
2887	{
2888	case VINF_SUCCESS:
2889	case VINF_EM_RAW_INTERRUPT:
2890	case VINF_PATM_PATCH_TRAP_PF:
2891	case VINF_PATM_PATCH_TRAP_GP:
2892	case VINF_PATM_PATCH_INT3:
2893	case VINF_PATM_CHECK_PATCH_PAGE:
2894	case VINF_EM_RAW_EXCEPTION_PRIVILEGED:
2895	case VINF_EM_RAW_GUEST_TRAP:
2896	case VINF_EM_RESCHEDULE_RAW:
2897	break;
2898
2899	default:
2900	if (PATMIsPatchGCAddr(pVM, pCtx->eip) && !(pCtx->eflags.u32 & X86_EFL_TF))
2901	LogIt(NULL, 0, LOG_GROUP_PATM, ("Patch code interrupted at %RRv for reason %Rrc\n", (RTRCPTR)CPUMGetGuestEIP(pVCpu), rc));
2902	break;
2903	}
2904	/*
2905	* Let's go paranoid!
2906	*/
2907	if ( !VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
2908	&& PGMMapHasConflicts(pVM))
2909	{
2910	PGMMapCheck(pVM);
2911	AssertMsgFailed(("We should not get conflicts any longer!!! rc=%Rrc\n", rc));
2912	return VERR_INTERNAL_ERROR;
2913	}
2914	#endif /* VBOX_STRICT */
2915
2916	/*
2917	* Process the returned status code.
2918	*/
2919	if (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST)
2920	{
2921	STAM_PROFILE_ADV_STOP(&pVCpu->em.s.StatRAWTail, d);
2922	break;
2923	}
2924	rc = emR3RawHandleRC(pVM, pVCpu, pCtx, rc);
2925	if (rc != VINF_SUCCESS)
2926	{
2927	rc = emR3RawUpdateForceFlag(pVM, pVCpu, pCtx, rc);
2928	if (rc != VINF_SUCCESS)
2929	{
2930	STAM_PROFILE_ADV_STOP(&pVCpu->em.s.StatRAWTail, d);
2931	break;
2932	}
2933	}
2934
2935	/*
2936	* Check and execute forced actions.
2937	*/
2938	#ifdef VBOX_HIGH_RES_TIMERS_HACK
2939	TMTimerPollVoid(pVM, pVCpu);
2940	#endif
2941	STAM_PROFILE_ADV_STOP(&pVCpu->em.s.StatRAWTail, d);
2942	if ( VM_FF_ISPENDING(pVM, ~VM_FF_HIGH_PRIORITY_PRE_RAW_MASK \| VM_FF_PGM_NO_MEMORY)
2943	\|\| VMCPU_FF_ISPENDING(pVCpu, ~VMCPU_FF_HIGH_PRIORITY_PRE_RAW_MASK))
2944	{
2945	Assert(pCtx->eflags.Bits.u1VM \|\| (pCtx->ss & X86_SEL_RPL) != 1);
2946
2947	STAM_REL_PROFILE_ADV_SUSPEND(&pVCpu->em.s.StatRAWTotal, a);
2948	rc = emR3ForcedActions(pVM, pVCpu, rc);
2949	STAM_REL_PROFILE_ADV_RESUME(&pVCpu->em.s.StatRAWTotal, a);
2950	if ( rc != VINF_SUCCESS
2951	&& rc != VINF_EM_RESCHEDULE_RAW)
2952	{
2953	rc = emR3RawUpdateForceFlag(pVM, pVCpu, pCtx, rc);
2954	if (rc != VINF_SUCCESS)
2955	{
2956	*pfFFDone = true;
2957	break;
2958	}
2959	}
2960	}
2961	}
2962
2963	/*
2964	* Return to outer loop.
2965	*/
2966	#if defined(LOG_ENABLED) && defined(DEBUG)
2967	RTLogFlush(NULL);
2968	#endif
2969	STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatRAWTotal, a);
2970	return rc;
2971	}
2972
2973
2974	/**
2975	* Executes hardware accelerated raw code. (Intel VMX & AMD SVM)
2976	*
2977	* This function contains the raw-mode version of the inner
2978	* execution loop (the outer loop being in EMR3ExecuteVM()).
2979	*
2980	* @returns VBox status code. The most important ones are: VINF_EM_RESCHEDULE, VINF_EM_RESCHEDULE_RAW,
2981	* VINF_EM_RESCHEDULE_REM, VINF_EM_SUSPEND, VINF_EM_RESET and VINF_EM_TERMINATE.
2982	*
2983	* @param pVM VM handle.
2984	* @param pVCpu VMCPU handle.
2985	* @param pfFFDone Where to store an indicator telling whether or not
2986	* FFs were done before returning.
2987	*/
2988	static int emR3HwAccExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone)
2989	{
2990	int rc = VERR_INTERNAL_ERROR;
2991	PCPUMCTX pCtx = pVCpu->em.s.pCtx;
2992
2993	LogFlow(("emR3HwAccExecute%d: (cs:eip=%04x:%RGv)\n", pVCpu->idCpu, pCtx->cs, (RTGCPTR)pCtx->rip));
2994	*pfFFDone = false;
2995
2996	STAM_COUNTER_INC(&pVCpu->em.s.StatHwAccExecuteEntry);
2997
2998	#ifdef EM_NOTIFY_HWACCM
2999	HWACCMR3NotifyScheduled(pVCpu);
3000	#endif
3001
3002	/*
3003	* Spin till we get a forced action which returns anything but VINF_SUCCESS.
3004	*/
3005	for (;;)
3006	{
3007	STAM_PROFILE_ADV_START(&pVCpu->em.s.StatHwAccEntry, a);
3008
3009	/*
3010	* Process high priority pre-execution raw-mode FFs.
3011	*/
3012	VMCPU_FF_CLEAR(pVCpu, (VMCPU_FF_SELM_SYNC_GDT \| VMCPU_FF_SELM_SYNC_LDT \| VMCPU_FF_TRPM_SYNC_IDT \| VMCPU_FF_SELM_SYNC_TSS)); /* not relevant in HWACCM mode; shouldn't be set really. */
3013	if ( VM_FF_ISPENDING(pVM, VM_FF_HIGH_PRIORITY_PRE_RAW_MASK)
3014	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_PRE_RAW_MASK))
3015	{
3016	rc = emR3RawForcedActions(pVM, pVCpu, pCtx);
3017	if (rc != VINF_SUCCESS)
3018	break;
3019	}
3020
3021	#ifdef LOG_ENABLED
3022	/*
3023	* Log important stuff before entering GC.
3024	*/
3025	if (TRPMHasTrap(pVCpu))
3026	Log(("CPU%d: Pending hardware interrupt=0x%x cs:rip=%04X:%RGv\n", pVCpu->idCpu, TRPMGetTrapNo(pVCpu), pCtx->cs, (RTGCPTR)pCtx->rip));
3027
3028	uint32_t cpl = CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(pCtx));
3029
3030	if (pVM->cCPUs == 1)
3031	{
3032	if (pCtx->eflags.Bits.u1VM)
3033	Log(("HWV86: %08X IF=%d\n", pCtx->eip, pCtx->eflags.Bits.u1IF));
3034	else if (CPUMIsGuestIn64BitCodeEx(pCtx))
3035	Log(("HWR%d: %04X:%RGv ESP=%RGv IF=%d IOPL=%d CR0=%x CR4=%x EFER=%x\n", cpl, pCtx->cs, (RTGCPTR)pCtx->rip, pCtx->rsp, pCtx->eflags.Bits.u1IF, pCtx->eflags.Bits.u2IOPL, (uint32_t)pCtx->cr0, (uint32_t)pCtx->cr4, (uint32_t)pCtx->msrEFER));
3036	else
3037	Log(("HWR%d: %04X:%08X ESP=%08X IF=%d IOPL=%d CR0=%x CR4=%x EFER=%x\n", cpl, pCtx->cs, pCtx->eip, pCtx->esp, pCtx->eflags.Bits.u1IF, pCtx->eflags.Bits.u2IOPL, (uint32_t)pCtx->cr0, (uint32_t)pCtx->cr4, (uint32_t)pCtx->msrEFER));
3038	}
3039	else
3040	{
3041	if (pCtx->eflags.Bits.u1VM)
3042	Log(("HWV86-CPU%d: %08X IF=%d\n", pVCpu->idCpu, pCtx->eip, pCtx->eflags.Bits.u1IF));
3043	else if (CPUMIsGuestIn64BitCodeEx(pCtx))
3044	Log(("HWR%d-CPU%d: %04X:%RGv ESP=%RGv IF=%d IOPL=%d CR0=%x CR4=%x EFER=%x\n", cpl, pVCpu->idCpu, pCtx->cs, (RTGCPTR)pCtx->rip, pCtx->rsp, pCtx->eflags.Bits.u1IF, pCtx->eflags.Bits.u2IOPL, (uint32_t)pCtx->cr0, (uint32_t)pCtx->cr4, (uint32_t)pCtx->msrEFER));
3045	else
3046	Log(("HWR%d-CPU%d: %04X:%08X ESP=%08X IF=%d IOPL=%d CR0=%x CR4=%x EFER=%x\n", cpl, pVCpu->idCpu, pCtx->cs, pCtx->eip, pCtx->esp, pCtx->eflags.Bits.u1IF, pCtx->eflags.Bits.u2IOPL, (uint32_t)pCtx->cr0, (uint32_t)pCtx->cr4, (uint32_t)pCtx->msrEFER));
3047	}
3048	#endif /* LOG_ENABLED */
3049
3050	/*
3051	* Execute the code.
3052	*/
3053	STAM_PROFILE_ADV_STOP(&pVCpu->em.s.StatHwAccEntry, a);
3054	STAM_PROFILE_START(&pVCpu->em.s.StatHwAccExec, x);
3055	rc = VMMR3HwAccRunGC(pVM, pVCpu);
3056	STAM_PROFILE_STOP(&pVCpu->em.s.StatHwAccExec, x);
3057
3058	/*
3059	* Deal with high priority post execution FFs before doing anything else.
3060	*/
3061	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_RESUME_GUEST_MASK);
3062	if ( VM_FF_ISPENDING(pVM, VM_FF_HIGH_PRIORITY_POST_MASK)
3063	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_POST_MASK))
3064	rc = emR3HighPriorityPostForcedActions(pVM, pVCpu, rc);
3065
3066	/*
3067	* Process the returned status code.
3068	*/
3069	if (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST)
3070	break;
3071
3072	rc = emR3RawHandleRC(pVM, pVCpu, pCtx, rc);
3073	if (rc != VINF_SUCCESS)
3074	break;
3075
3076	/*
3077	* Check and execute forced actions.
3078	*/
3079	#ifdef VBOX_HIGH_RES_TIMERS_HACK
3080	TMTimerPollVoid(pVM, pVCpu);
3081	#endif
3082	if ( VM_FF_ISPENDING(pVM, VM_FF_ALL_MASK)
3083	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_ALL_MASK))
3084	{
3085	rc = emR3ForcedActions(pVM, pVCpu, rc);
3086	if ( rc != VINF_SUCCESS
3087	&& rc != VINF_EM_RESCHEDULE_HWACC)
3088	{
3089	*pfFFDone = true;
3090	break;
3091	}
3092	}
3093	}
3094
3095	/*
3096	* Return to outer loop.
3097	*/
3098	#if defined(LOG_ENABLED) && defined(DEBUG)
3099	RTLogFlush(NULL);
3100	#endif
3101	return rc;
3102	}
3103
3104
3105	/**
3106	* Decides whether to execute RAW, HWACC or REM.
3107	*
3108	* @returns new EM state
3109	* @param pVM The VM.
3110	* @param pVCpu The VMCPU handle.
3111	* @param pCtx The CPU context.
3112	*/
3113	static EMSTATE emR3Reschedule(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3114	{
3115	/*
3116	* When forcing raw-mode execution, things are simple.
3117	*/
3118	if (pVCpu->em.s.fForceRAW)
3119	return EMSTATE_RAW;
3120
3121	/*
3122	* We stay in the wait for SIPI state unless explicitly told otherwise.
3123	*/
3124	if (pVCpu->em.s.enmState == EMSTATE_WAIT_SIPI)
3125	return EMSTATE_WAIT_SIPI;
3126
3127	/* !!! THIS MUST BE IN SYNC WITH remR3CanExecuteRaw !!! */
3128	/* !!! THIS MUST BE IN SYNC WITH remR3CanExecuteRaw !!! */
3129	/* !!! THIS MUST BE IN SYNC WITH remR3CanExecuteRaw !!! */
3130
3131	X86EFLAGS EFlags = pCtx->eflags;
3132	if (HWACCMIsEnabled(pVM))
3133	{
3134	/* Hardware accelerated raw-mode:
3135	*
3136	* Typically only 32-bits protected mode, with paging enabled, code is allowed here.
3137	*/
3138	if (HWACCMR3CanExecuteGuest(pVM, pCtx) == true)
3139	return EMSTATE_HWACC;
3140
3141	/* Note: Raw mode and hw accelerated mode are incompatible. The latter turns
3142	* off monitoring features essential for raw mode! */
3143	return EMSTATE_REM;
3144	}
3145
3146	/*
3147	* Standard raw-mode:
3148	*
3149	* Here we only support 16 & 32 bits protected mode ring 3 code that has no IO privileges
3150	* or 32 bits protected mode ring 0 code
3151	*
3152	* The tests are ordered by the likelyhood of being true during normal execution.
3153	*/
3154	if (EFlags.u32 & (X86_EFL_TF /* \| HF_INHIBIT_IRQ_MASK*/))
3155	{
3156	Log2(("raw mode refused: EFlags=%#x\n", EFlags.u32));
3157	return EMSTATE_REM;
3158	}
3159
3160	#ifndef VBOX_RAW_V86
3161	if (EFlags.u32 & X86_EFL_VM) {
3162	Log2(("raw mode refused: VM_MASK\n"));
3163	return EMSTATE_REM;
3164	}
3165	#endif
3166
3167	/** @todo check up the X86_CR0_AM flag in respect to raw mode!!! We're probably not emulating it right! */
3168	uint32_t u32CR0 = pCtx->cr0;
3169	if ((u32CR0 & (X86_CR0_PG \| X86_CR0_PE)) != (X86_CR0_PG \| X86_CR0_PE))
3170	{
3171	//Log2(("raw mode refused: %s%s%s\n", (u32CR0 & X86_CR0_PG) ? "" : " !PG", (u32CR0 & X86_CR0_PE) ? "" : " !PE", (u32CR0 & X86_CR0_AM) ? "" : " !AM"));
3172	return EMSTATE_REM;
3173	}
3174
3175	if (pCtx->cr4 & X86_CR4_PAE)
3176	{
3177	uint32_t u32Dummy, u32Features;
3178
3179	CPUMGetGuestCpuId(pVCpu, 1, &u32Dummy, &u32Dummy, &u32Dummy, &u32Features);
3180	if (!(u32Features & X86_CPUID_FEATURE_EDX_PAE))
3181	return EMSTATE_REM;
3182	}
3183
3184	unsigned uSS = pCtx->ss;
3185	if ( pCtx->eflags.Bits.u1VM
3186	\|\| (uSS & X86_SEL_RPL) == 3)
3187	{
3188	if (!EMIsRawRing3Enabled(pVM))
3189	return EMSTATE_REM;
3190
3191	if (!(EFlags.u32 & X86_EFL_IF))
3192	{
3193	Log2(("raw mode refused: IF (RawR3)\n"));
3194	return EMSTATE_REM;
3195	}
3196
3197	if (!(u32CR0 & X86_CR0_WP) && EMIsRawRing0Enabled(pVM))
3198	{
3199	Log2(("raw mode refused: CR0.WP + RawR0\n"));
3200	return EMSTATE_REM;
3201	}
3202	}
3203	else
3204	{
3205	if (!EMIsRawRing0Enabled(pVM))
3206	return EMSTATE_REM;
3207
3208	/* Only ring 0 supervisor code. */
3209	if ((uSS & X86_SEL_RPL) != 0)
3210	{
3211	Log2(("raw r0 mode refused: CPL %d\n", uSS & X86_SEL_RPL));
3212	return EMSTATE_REM;
3213	}
3214
3215	// Let's start with pure 32 bits ring 0 code first
3216	/** @todo What's pure 32-bit mode? flat? */
3217	if ( !(pCtx->ssHid.Attr.n.u1DefBig)
3218	\|\| !(pCtx->csHid.Attr.n.u1DefBig))
3219	{
3220	Log2(("raw r0 mode refused: SS/CS not 32bit\n"));
3221	return EMSTATE_REM;
3222	}
3223
3224	/* Write protection must be turned on, or else the guest can overwrite our hypervisor code and data. */
3225	if (!(u32CR0 & X86_CR0_WP))
3226	{
3227	Log2(("raw r0 mode refused: CR0.WP=0!\n"));
3228	return EMSTATE_REM;
3229	}
3230
3231	if (PATMShouldUseRawMode(pVM, (RTGCPTR)pCtx->eip))
3232	{
3233	Log2(("raw r0 mode forced: patch code\n"));
3234	return EMSTATE_RAW;
3235	}
3236
3237	#if !defined(VBOX_ALLOW_IF0) && !defined(VBOX_RUN_INTERRUPT_GATE_HANDLERS)
3238	if (!(EFlags.u32 & X86_EFL_IF))
3239	{
3240	////Log2(("R0: IF=0 VIF=%d %08X\n", eip, pVMeflags));
3241	//Log2(("RR0: Interrupts turned off; fall back to emulation\n"));
3242	return EMSTATE_REM;
3243	}
3244	#endif
3245
3246	/** @todo still necessary??? */
3247	if (EFlags.Bits.u2IOPL != 0)
3248	{
3249	Log2(("raw r0 mode refused: IOPL %d\n", EFlags.Bits.u2IOPL));
3250	return EMSTATE_REM;
3251	}
3252	}
3253
3254	Assert(PGMPhysIsA20Enabled(pVCpu));
3255	return EMSTATE_RAW;
3256	}
3257
3258
3259	/**
3260	* Executes all high priority post execution force actions.
3261	*
3262	* @returns rc or a fatal status code.
3263	*
3264	* @param pVM VM handle.
3265	* @param pVCpu VMCPU handle.
3266	* @param rc The current rc.
3267	*/
3268	static int emR3HighPriorityPostForcedActions(PVM pVM, PVMCPU pVCpu, int rc)
3269	{
3270	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PDM_CRITSECT))
3271	PDMCritSectFF(pVCpu);
3272
3273	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_CSAM_PENDING_ACTION))
3274	CSAMR3DoPendingAction(pVM, pVCpu);
3275
3276	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
3277	{
3278	if ( rc > VINF_EM_NO_MEMORY
3279	&& rc <= VINF_EM_LAST)
3280	rc = VINF_EM_NO_MEMORY;
3281	}
3282
3283	return rc;
3284	}
3285
3286
3287	/**
3288	* Executes all pending forced actions.
3289	*
3290	* Forced actions can cause execution delays and execution
3291	* rescheduling. The first we deal with using action priority, so
3292	* that for instance pending timers aren't scheduled and ran until
3293	* right before execution. The rescheduling we deal with using
3294	* return codes. The same goes for VM termination, only in that case
3295	* we exit everything.
3296	*
3297	* @returns VBox status code of equal or greater importance/severity than rc.
3298	* The most important ones are: VINF_EM_RESCHEDULE,
3299	* VINF_EM_SUSPEND, VINF_EM_RESET and VINF_EM_TERMINATE.
3300	*
3301	* @param pVM VM handle.
3302	* @param pVCpu VMCPU handle.
3303	* @param rc The current rc.
3304	*
3305	*/
3306	static int emR3ForcedActions(PVM pVM, PVMCPU pVCpu, int rc)
3307	{
3308	STAM_REL_PROFILE_START(&pVCpu->em.s.StatForcedActions, a);
3309	#ifdef VBOX_STRICT
3310	int rcIrq = VINF_SUCCESS;
3311	#endif
3312	int rc2;
3313	#define UPDATE_RC() \
3314	do { \
3315	AssertMsg(rc2 <= 0 \|\| (rc2 >= VINF_EM_FIRST && rc2 <= VINF_EM_LAST), ("Invalid FF return code: %Rra\n", rc2)); \
3316	if (rc2 == VINF_SUCCESS \|\| rc < VINF_SUCCESS) \
3317	break; \
3318	if (!rc \|\| rc2 < rc) \
3319	rc = rc2; \
3320	} while (0)
3321
3322	/*
3323	* Post execution chunk first.
3324	*/
3325	if ( VM_FF_ISPENDING(pVM, VM_FF_NORMAL_PRIORITY_POST_MASK)
3326	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_NORMAL_PRIORITY_POST_MASK))
3327	{
3328	/*
3329	* EMT Rendezvous (must be serviced before termination).
3330	*/
3331	if (VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS))
3332	VMMR3EmtRendezvousFF(pVM, pVCpu);
3333
3334	/*
3335	* Termination request.
3336	*/
3337	if (VM_FF_ISPENDING(pVM, VM_FF_TERMINATE))
3338	{
3339	Log2(("emR3ForcedActions: returns VINF_EM_TERMINATE\n"));
3340	STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
3341	return VINF_EM_TERMINATE;
3342	}
3343
3344	/*
3345	* Debugger Facility polling.
3346	*/
3347	if (VM_FF_ISPENDING(pVM, VM_FF_DBGF))
3348	{
3349	rc2 = DBGFR3VMMForcedAction(pVM);
3350	UPDATE_RC();
3351	}
3352
3353	/*
3354	* Postponed reset request.
3355	*/
3356	if (VM_FF_TESTANDCLEAR(pVM, VM_FF_RESET_BIT))
3357	{
3358	rc2 = VMR3Reset(pVM);
3359	UPDATE_RC();
3360	}
3361
3362	/*
3363	* CSAM page scanning.
3364	*/
3365	if ( !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)
3366	&& VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_CSAM_SCAN_PAGE))
3367	{
3368	PCPUMCTX pCtx = pVCpu->em.s.pCtx;
3369
3370	/** @todo: check for 16 or 32 bits code! (D bit in the code selector) */
3371	Log(("Forced action VMCPU_FF_CSAM_SCAN_PAGE\n"));
3372
3373	CSAMR3CheckCodeEx(pVM, CPUMCTX2CORE(pCtx), pCtx->eip);
3374	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_CSAM_SCAN_PAGE);
3375	}
3376
3377	/*
3378	* Out of memory? Putting this after CSAM as it may in theory cause us to run out of memory.
3379	*/
3380	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
3381	{
3382	rc2 = PGMR3PhysAllocateHandyPages(pVM);
3383	UPDATE_RC();
3384	if (rc == VINF_EM_NO_MEMORY)
3385	return rc;
3386	}
3387
3388	/* check that we got them all */
3389	AssertCompile(VM_FF_NORMAL_PRIORITY_POST_MASK == (VM_FF_TERMINATE \| VM_FF_DBGF \| VM_FF_RESET \| VM_FF_PGM_NO_MEMORY \| VM_FF_EMT_RENDEZVOUS));
3390	AssertCompile(VMCPU_FF_NORMAL_PRIORITY_POST_MASK == VMCPU_FF_CSAM_SCAN_PAGE);
3391	}
3392
3393	/*
3394	* Normal priority then.
3395	* (Executed in no particular order.)
3396	*/
3397	if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_NORMAL_PRIORITY_MASK, VM_FF_PGM_NO_MEMORY))
3398	{
3399	/*
3400	* PDM Queues are pending.
3401	*/
3402	if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_PDM_QUEUES, VM_FF_PGM_NO_MEMORY))
3403	PDMR3QueueFlushAll(pVM);
3404
3405	/*
3406	* PDM DMA transfers are pending.
3407	*/
3408	if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_PDM_DMA, VM_FF_PGM_NO_MEMORY))
3409	PDMR3DmaRun(pVM);
3410
3411	/*
3412	* EMT Rendezvous (make sure they are handled before the requests).
3413	*/
3414	if (VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS))
3415	VMMR3EmtRendezvousFF(pVM, pVCpu);
3416
3417	/*
3418	* Requests from other threads.
3419	*/
3420	if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_REQUEST, VM_FF_PGM_NO_MEMORY))
3421	{
3422	rc2 = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY);
3423	Assert(rc2 != VINF_EM_RESET); /* should be per-VCPU */
3424	if (rc2 == VINF_EM_OFF \|\| rc2 == VINF_EM_TERMINATE)
3425	{
3426	Log2(("emR3ForcedActions: returns %Rrc\n", rc2));
3427	STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
3428	return rc2;
3429	}
3430	UPDATE_RC();
3431	}
3432
3433	/* Replay the handler notification changes. */
3434	if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_REM_HANDLER_NOTIFY, VM_FF_PGM_NO_MEMORY))
3435	{
3436	/* Try not to cause deadlocks. */
3437	if ( pVM->cCPUs == 1
3438	\|\| ( !PGMIsLockOwner(pVM)
3439	&& !IOMIsLockOwner(pVM))
3440	)
3441	{
3442	EMRemLock(pVM);
3443	REMR3ReplayHandlerNotifications(pVM);
3444	EMRemUnlock(pVM);
3445	}
3446	}
3447
3448	/* check that we got them all */
3449	AssertCompile(VM_FF_NORMAL_PRIORITY_MASK == (VM_FF_REQUEST \| VM_FF_PDM_QUEUES \| VM_FF_PDM_DMA \| VM_FF_REM_HANDLER_NOTIFY \| VM_FF_EMT_RENDEZVOUS));
3450	}
3451
3452	/*
3453	* Normal priority then. (per-VCPU)
3454	* (Executed in no particular order.)
3455	*/
3456	if ( !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)
3457	&& VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_NORMAL_PRIORITY_MASK))
3458	{
3459	/*
3460	* Requests from other threads.
3461	*/
3462	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
3463	{
3464	rc2 = VMR3ReqProcessU(pVM->pUVM, pVCpu->idCpu);
3465	if (rc2 == VINF_EM_OFF \|\| rc2 == VINF_EM_TERMINATE \|\| rc2 == VINF_EM_RESET)
3466	{
3467	Log2(("emR3ForcedActions: returns %Rrc\n", rc2));
3468	STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
3469	return rc2;
3470	}
3471	UPDATE_RC();
3472	}
3473
3474	/* check that we got them all */
3475	Assert(!(VMCPU_FF_NORMAL_PRIORITY_MASK & ~(VMCPU_FF_REQUEST)));
3476	}
3477
3478	/*
3479	* High priority pre execution chunk last.
3480	* (Executed in ascending priority order.)
3481	*/
3482	if ( VM_FF_ISPENDING(pVM, VM_FF_HIGH_PRIORITY_PRE_MASK)
3483	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_PRE_MASK))
3484	{
3485	/*
3486	* Timers before interrupts.
3487	*/
3488	if ( VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TIMER)
3489	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
3490	TMR3TimerQueuesDo(pVM);
3491
3492	/*
3493	* The instruction following an emulated STI should always be executed!
3494	*/
3495	if ( VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
3496	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
3497	{
3498	Log(("VM_FF_EMULATED_STI at %RGv successor %RGv\n", (RTGCPTR)CPUMGetGuestRIP(pVCpu), EMGetInhibitInterruptsPC(pVCpu)));
3499	if (CPUMGetGuestEIP(pVCpu) != EMGetInhibitInterruptsPC(pVCpu))
3500	{
3501	/* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here if the eip is the same as the inhibited instr address.
3502	* Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
3503	* force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
3504	* break the guest. Sounds very unlikely, but such timing sensitive problem are not as rare as you might think.
3505	*/
3506	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3507	}
3508	if (HWACCMR3IsActive(pVCpu))
3509	rc2 = VINF_EM_RESCHEDULE_HWACC;
3510	else
3511	rc2 = PATMAreInterruptsEnabled(pVM) ? VINF_EM_RESCHEDULE_RAW : VINF_EM_RESCHEDULE_REM;
3512
3513	UPDATE_RC();
3514	}
3515
3516	/*
3517	* Interrupts.
3518	*/
3519	if ( !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)
3520	&& !VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
3521	&& (!rc \|\| rc >= VINF_EM_RESCHEDULE_HWACC)
3522	&& !TRPMHasTrap(pVCpu) /* an interrupt could already be scheduled for dispatching in the recompiler. */
3523	&& PATMAreInterruptsEnabled(pVM)
3524	&& !HWACCMR3IsEventPending(pVCpu))
3525	{
3526	Assert(pVCpu->em.s.enmState != EMSTATE_WAIT_SIPI);
3527	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
3528	{
3529	/* Note: it's important to make sure the return code from TRPMR3InjectEvent isn't ignored! */
3530	/** @todo this really isn't nice, should properly handle this */
3531	rc2 = TRPMR3InjectEvent(pVM, pVCpu, TRPM_HARDWARE_INT);
3532	#ifdef VBOX_STRICT
3533	rcIrq = rc2;
3534	#endif
3535	UPDATE_RC();
3536	}
3537	/** @todo really ugly; if we entered the hlt state when exiting the recompiler and an interrupt was pending, we previously got stuck in the halted state. */
3538	else if (REMR3QueryPendingInterrupt(pVM, pVCpu) != REM_NO_PENDING_IRQ)
3539	{
3540	rc2 = VINF_EM_RESCHEDULE_REM;
3541	UPDATE_RC();
3542	}
3543	}
3544
3545	/*
3546	* Allocate handy pages.
3547	*/
3548	if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_PGM_NEED_HANDY_PAGES, VM_FF_PGM_NO_MEMORY))
3549	{
3550	rc2 = PGMR3PhysAllocateHandyPages(pVM);
3551	UPDATE_RC();
3552	}
3553
3554	/*
3555	* Debugger Facility request.
3556	*/
3557	if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_DBGF, VM_FF_PGM_NO_MEMORY))
3558	{
3559	rc2 = DBGFR3VMMForcedAction(pVM);
3560	UPDATE_RC();
3561	}
3562
3563	/*
3564	* EMT Rendezvous (must be serviced before termination).
3565	*/
3566	if (VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS))
3567	VMMR3EmtRendezvousFF(pVM, pVCpu);
3568
3569	/*
3570	* Termination request.
3571	*/
3572	if (VM_FF_ISPENDING(pVM, VM_FF_TERMINATE))
3573	{
3574	Log2(("emR3ForcedActions: returns VINF_EM_TERMINATE\n"));
3575	STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
3576	return VINF_EM_TERMINATE;
3577	}
3578
3579	/*
3580	* Out of memory? Since most of our fellow high priority actions may cause us
3581	* to run out of memory, we're employing VM_FF_IS_PENDING_EXCEPT and putting this
3582	* at the end rather than the start. Also, VM_FF_TERMINATE has higher priority
3583	* than us since we can terminate without allocating more memory.
3584	*/
3585	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
3586	{
3587	rc2 = PGMR3PhysAllocateHandyPages(pVM);
3588	UPDATE_RC();
3589	if (rc == VINF_EM_NO_MEMORY)
3590	return rc;
3591	}
3592
3593	/*
3594	* If the virtual sync clock is still stopped, make TM restart it.
3595	*/
3596	if (VM_FF_ISPENDING(pVM, VM_FF_TM_VIRTUAL_SYNC))
3597	TMR3VirtualSyncFF(pVM, pVCpu);
3598
3599	#ifdef DEBUG
3600	/*
3601	* Debug, pause the VM.
3602	*/
3603	if (VM_FF_ISPENDING(pVM, VM_FF_DEBUG_SUSPEND))
3604	{
3605	VM_FF_CLEAR(pVM, VM_FF_DEBUG_SUSPEND);
3606	Log(("emR3ForcedActions: returns VINF_EM_SUSPEND\n"));
3607	return VINF_EM_SUSPEND;
3608	}
3609	#endif
3610
3611	/* check that we got them all */
3612	AssertCompile(VM_FF_HIGH_PRIORITY_PRE_MASK == (VM_FF_TM_VIRTUAL_SYNC \| VM_FF_DBGF \| VM_FF_TERMINATE \| VM_FF_DEBUG_SUSPEND \| VM_FF_PGM_NEED_HANDY_PAGES \| VM_FF_PGM_NO_MEMORY \| VM_FF_EMT_RENDEZVOUS));
3613	AssertCompile(VMCPU_FF_HIGH_PRIORITY_PRE_MASK == (VMCPU_FF_TIMER \| VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL \| VMCPU_FF_SELM_SYNC_TSS \| VMCPU_FF_TRPM_SYNC_IDT \| VMCPU_FF_SELM_SYNC_GDT \| VMCPU_FF_SELM_SYNC_LDT \| VMCPU_FF_INHIBIT_INTERRUPTS));
3614	}
3615
3616	#undef UPDATE_RC
3617	Log2(("emR3ForcedActions: returns %Rrc\n", rc));
3618	STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
3619	Assert(rcIrq == VINF_SUCCESS \|\| rcIrq == rc);
3620	return rc;
3621	}
3622
3623	/**
3624	* Release the IOM lock if owned by the current VCPU
3625	*
3626	* @param pVM The VM to operate on.
3627	*/
3628	VMMR3DECL(void) EMR3ReleaseOwnedLocks(PVM pVM)
3629	{
3630	while (PDMCritSectIsOwner(&pVM->em.s.CritSectREM))
3631	PDMCritSectLeave(&pVM->em.s.CritSectREM);
3632	}
3633
3634
3635	/**
3636	* Execute VM.
3637	*
3638	* This function is the main loop of the VM. The emulation thread
3639	* calls this function when the VM has been successfully constructed
3640	* and we're ready for executing the VM.
3641	*
3642	* Returning from this function means that the VM is turned off or
3643	* suspended (state already saved) and deconstruction in next in line.
3644	*
3645	* All interaction from other thread are done using forced actions
3646	* and signaling of the wait object.
3647	*
3648	* @returns VBox status code, informational status codes may indicate failure.
3649	* @param pVM The VM to operate on.
3650	* @param pVCpu The VMCPU to operate on.
3651	*/
3652	VMMR3DECL(int) EMR3ExecuteVM(PVM pVM, PVMCPU pVCpu)
3653	{
3654	LogFlow(("EMR3ExecuteVM: pVM=%p enmVMState=%d enmState=%d (%s) fForceRAW=%d\n", pVM, pVM->enmVMState,
3655	pVCpu->em.s.enmState, EMR3GetStateName(pVCpu->em.s.enmState), pVCpu->em.s.fForceRAW));
3656	VM_ASSERT_EMT(pVM);
3657	AssertMsg( pVCpu->em.s.enmState == EMSTATE_NONE
3658	\|\| pVCpu->em.s.enmState == EMSTATE_WAIT_SIPI
3659	\|\| pVCpu->em.s.enmState == EMSTATE_SUSPENDED,
3660	("%s\n", EMR3GetStateName(pVCpu->em.s.enmState)));
3661
3662	int rc = setjmp(pVCpu->em.s.u.FatalLongJump);
3663	if (rc == 0)
3664	{
3665	/*
3666	* Start the virtual time.
3667	*/
3668	TMR3NotifyResume(pVM, pVCpu);
3669
3670	/*
3671	* The Outer Main Loop.
3672	*/
3673	bool fFFDone = false;
3674
3675	/* Reschedule right away to start in the right state. */
3676	rc = VINF_SUCCESS;
3677
3678	/* If resuming after a pause or a state load, restore the previous
3679	state or else we'll start executing code. Else, just reschedule. */
3680	if ( pVCpu->em.s.enmState == EMSTATE_SUSPENDED
3681	&& ( pVCpu->em.s.enmPrevState == EMSTATE_WAIT_SIPI
3682	\|\| pVCpu->em.s.enmPrevState == EMSTATE_HALTED))
3683	pVCpu->em.s.enmState = pVCpu->em.s.enmPrevState;
3684	else
3685	pVCpu->em.s.enmState = emR3Reschedule(pVM, pVCpu, pVCpu->em.s.pCtx);
3686
3687	STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatTotal, x);
3688	for (;;)
3689	{
3690	/*
3691	* Before we can schedule anything (we're here because
3692	* scheduling is required) we must service any pending
3693	* forced actions to avoid any pending action causing
3694	* immediate rescheduling upon entering an inner loop
3695	*
3696	* Do forced actions.
3697	*/
3698	if ( !fFFDone
3699	&& rc != VINF_EM_TERMINATE
3700	&& rc != VINF_EM_OFF
3701	&& ( VM_FF_ISPENDING(pVM, VM_FF_ALL_BUT_RAW_MASK)
3702	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_ALL_BUT_RAW_MASK)))
3703	{
3704	rc = emR3ForcedActions(pVM, pVCpu, rc);
3705	if ( ( rc == VINF_EM_RESCHEDULE_REM
3706	\|\| rc == VINF_EM_RESCHEDULE_HWACC)
3707	&& pVCpu->em.s.fForceRAW)
3708	rc = VINF_EM_RESCHEDULE_RAW;
3709	}
3710	else if (fFFDone)
3711	fFFDone = false;
3712
3713	/*
3714	* Now what to do?
3715	*/
3716	Log2(("EMR3ExecuteVM: rc=%Rrc\n", rc));
3717	switch (rc)
3718	{
3719	/*
3720	* Keep doing what we're currently doing.
3721	*/
3722	case VINF_SUCCESS:
3723	break;
3724
3725	/*
3726	* Reschedule - to raw-mode execution.
3727	*/
3728	case VINF_EM_RESCHEDULE_RAW:
3729	Log2(("EMR3ExecuteVM: VINF_EM_RESCHEDULE_RAW: %d -> %d (EMSTATE_RAW)\n", pVCpu->em.s.enmState, EMSTATE_RAW));
3730	pVCpu->em.s.enmState = EMSTATE_RAW;
3731	break;
3732
3733	/*
3734	* Reschedule - to hardware accelerated raw-mode execution.
3735	*/
3736	case VINF_EM_RESCHEDULE_HWACC:
3737	Log2(("EMR3ExecuteVM: VINF_EM_RESCHEDULE_HWACC: %d -> %d (EMSTATE_HWACC)\n", pVCpu->em.s.enmState, EMSTATE_HWACC));
3738	Assert(!pVCpu->em.s.fForceRAW);
3739	pVCpu->em.s.enmState = EMSTATE_HWACC;
3740	break;
3741
3742	/*
3743	* Reschedule - to recompiled execution.
3744	*/
3745	case VINF_EM_RESCHEDULE_REM:
3746	Log2(("EMR3ExecuteVM: VINF_EM_RESCHEDULE_REM: %d -> %d (EMSTATE_REM)\n", pVCpu->em.s.enmState, EMSTATE_REM));
3747	pVCpu->em.s.enmState = EMSTATE_REM;
3748	break;
3749
3750	#ifdef VBOX_WITH_VMI
3751	/*
3752	* Reschedule - parav call.
3753	*/
3754	case VINF_EM_RESCHEDULE_PARAV:
3755	Log2(("EMR3ExecuteVM: VINF_EM_RESCHEDULE_PARAV: %d -> %d (EMSTATE_PARAV)\n", pVCpu->em.s.enmState, EMSTATE_PARAV));
3756	pVCpu->em.s.enmState = EMSTATE_PARAV;
3757	break;
3758	#endif
3759
3760	/*
3761	* Resume.
3762	*/
3763	case VINF_EM_RESUME:
3764	Log2(("EMR3ExecuteVM: VINF_EM_RESUME: %d -> VINF_EM_RESCHEDULE\n", pVCpu->em.s.enmState));
3765	/* Don't reschedule in the halted or wait for SIPI case. */
3766	if ( pVCpu->em.s.enmPrevState == EMSTATE_WAIT_SIPI
3767	\|\| pVCpu->em.s.enmPrevState == EMSTATE_HALTED)
3768	break;
3769	/* fall through and get scheduled. */
3770
3771	/*
3772	* Reschedule.
3773	*/
3774	case VINF_EM_RESCHEDULE:
3775	{
3776	EMSTATE enmState = emR3Reschedule(pVM, pVCpu, pVCpu->em.s.pCtx);
3777	Log2(("EMR3ExecuteVM: VINF_EM_RESCHEDULE: %d -> %d (%s)\n", pVCpu->em.s.enmState, enmState, EMR3GetStateName(enmState)));
3778	pVCpu->em.s.enmState = enmState;
3779	break;
3780	}
3781
3782	/*
3783	* Halted.
3784	*/
3785	case VINF_EM_HALT:
3786	Log2(("EMR3ExecuteVM: VINF_EM_HALT: %d -> %d\n", pVCpu->em.s.enmState, EMSTATE_HALTED));
3787	pVCpu->em.s.enmState = EMSTATE_HALTED;
3788	break;
3789
3790	/*
3791	* Switch to the wait for SIPI state (application processor only)
3792	*/
3793	case VINF_EM_WAIT_SIPI:
3794	Assert(pVCpu->idCpu != 0);
3795	Log2(("EMR3ExecuteVM: VINF_EM_WAIT_SIPI: %d -> %d\n", pVCpu->em.s.enmState, EMSTATE_WAIT_SIPI));
3796	pVCpu->em.s.enmState = EMSTATE_WAIT_SIPI;
3797	break;
3798
3799
3800	/*
3801	* Suspend.
3802	*/
3803	case VINF_EM_SUSPEND:
3804	Log2(("EMR3ExecuteVM: VINF_EM_SUSPEND: %d -> %d\n", pVCpu->em.s.enmState, EMSTATE_SUSPENDED));
3805	pVCpu->em.s.enmPrevState = pVCpu->em.s.enmState;
3806	pVCpu->em.s.enmState = EMSTATE_SUSPENDED;
3807	break;
3808
3809	/*
3810	* Reset.
3811	* We might end up doing a double reset for now, we'll have to clean up the mess later.
3812	*/
3813	case VINF_EM_RESET:
3814	{
3815	if (pVCpu->idCpu == 0)
3816	{
3817	EMSTATE enmState = emR3Reschedule(pVM, pVCpu, pVCpu->em.s.pCtx);
3818	Log2(("EMR3ExecuteVM: VINF_EM_RESET: %d -> %d (%s)\n", pVCpu->em.s.enmState, enmState, EMR3GetStateName(enmState)));
3819	pVCpu->em.s.enmState = enmState;
3820	}
3821	else
3822	{
3823	/* All other VCPUs go into the wait for SIPI state. */
3824	pVCpu->em.s.enmState = EMSTATE_WAIT_SIPI;
3825	}
3826	break;
3827	}
3828
3829	/*
3830	* Power Off.
3831	*/
3832	case VINF_EM_OFF:
3833	pVCpu->em.s.enmState = EMSTATE_TERMINATING;
3834	Log2(("EMR3ExecuteVM: returns VINF_EM_OFF (%d -> %d)\n", pVCpu->em.s.enmState, EMSTATE_TERMINATING));
3835	TMR3NotifySuspend(pVM, pVCpu);
3836	STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
3837	return rc;
3838
3839	/*
3840	* Terminate the VM.
3841	*/
3842	case VINF_EM_TERMINATE:
3843	pVCpu->em.s.enmState = EMSTATE_TERMINATING;
3844	Log(("EMR3ExecuteVM returns VINF_EM_TERMINATE (%d -> %d)\n", pVCpu->em.s.enmState, EMSTATE_TERMINATING));
3845	TMR3NotifySuspend(pVM, pVCpu);
3846	STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
3847	return rc;
3848
3849
3850	/*
3851	* Out of memory, suspend the VM and stuff.
3852	*/
3853	case VINF_EM_NO_MEMORY:
3854	Log2(("EMR3ExecuteVM: VINF_EM_NO_MEMORY: %d -> %d\n", pVCpu->em.s.enmState, EMSTATE_SUSPENDED));
3855	pVCpu->em.s.enmState = EMSTATE_SUSPENDED;
3856	TMR3NotifySuspend(pVM, pVCpu);
3857	STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
3858
3859	rc = VMSetRuntimeError(pVM, VMSETRTERR_FLAGS_SUSPEND, "HostMemoryLow",
3860	N_("Unable to allocate and lock memory. The virtual machine will be paused. Please close applications to free up memory or close the VM"));
3861	if (rc != VINF_EM_SUSPEND)
3862	{
3863	if (RT_SUCCESS_NP(rc))
3864	{
3865	AssertLogRelMsgFailed(("%Rrc\n", rc));
3866	rc = VERR_EM_INTERNAL_ERROR;
3867	}
3868	pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
3869	}
3870	return rc;
3871
3872	/*
3873	* Guest debug events.
3874	*/
3875	case VINF_EM_DBG_STEPPED:
3876	AssertMsgFailed(("VINF_EM_DBG_STEPPED cannot be here!"));
3877	case VINF_EM_DBG_STOP:
3878	case VINF_EM_DBG_BREAKPOINT:
3879	case VINF_EM_DBG_STEP:
3880	if (pVCpu->em.s.enmState == EMSTATE_RAW)
3881	{
3882	Log2(("EMR3ExecuteVM: %Rrc: %d -> %d\n", rc, pVCpu->em.s.enmState, EMSTATE_DEBUG_GUEST_RAW));
3883	pVCpu->em.s.enmState = EMSTATE_DEBUG_GUEST_RAW;
3884	}
3885	else
3886	{
3887	Log2(("EMR3ExecuteVM: %Rrc: %d -> %d\n", rc, pVCpu->em.s.enmState, EMSTATE_DEBUG_GUEST_REM));
3888	pVCpu->em.s.enmState = EMSTATE_DEBUG_GUEST_REM;
3889	}
3890	break;
3891
3892	/*
3893	* Hypervisor debug events.
3894	*/
3895	case VINF_EM_DBG_HYPER_STEPPED:
3896	case VINF_EM_DBG_HYPER_BREAKPOINT:
3897	case VINF_EM_DBG_HYPER_ASSERTION:
3898	Log2(("EMR3ExecuteVM: %Rrc: %d -> %d\n", rc, pVCpu->em.s.enmState, EMSTATE_DEBUG_HYPER));
3899	pVCpu->em.s.enmState = EMSTATE_DEBUG_HYPER;
3900	break;
3901
3902	/*
3903	* Guru mediations.
3904	*/
3905	case VERR_VMM_RING0_ASSERTION:
3906	Log(("EMR3ExecuteVM: %Rrc: %d -> %d (EMSTATE_GURU_MEDITATION)\n", rc, pVCpu->em.s.enmState, EMSTATE_GURU_MEDITATION));
3907	pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
3908	break;
3909
3910	/*
3911	* Any error code showing up here other than the ones we
3912	* know and process above are considered to be FATAL.
3913	*
3914	* Unknown warnings and informational status codes are also
3915	* included in this.
3916	*/
3917	default:
3918	if (RT_SUCCESS_NP(rc))
3919	{
3920	AssertMsgFailed(("Unexpected warning or informational status code %Rra!\n", rc));
3921	rc = VERR_EM_INTERNAL_ERROR;
3922	}
3923	pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
3924	Log(("EMR3ExecuteVM returns %d\n", rc));
3925	break;
3926	}
3927
3928	STAM_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x); /* (skip this in release) */
3929	STAM_PROFILE_ADV_START(&pVCpu->em.s.StatTotal, x);
3930
3931	/*
3932	* Act on the state.
3933	*/
3934	switch (pVCpu->em.s.enmState)
3935	{
3936	/*
3937	* Execute raw.
3938	*/
3939	case EMSTATE_RAW:
3940	rc = emR3RawExecute(pVM, pVCpu, &fFFDone);
3941	break;
3942
3943	/*
3944	* Execute hardware accelerated raw.
3945	*/
3946	case EMSTATE_HWACC:
3947	rc = emR3HwAccExecute(pVM, pVCpu, &fFFDone);
3948	break;
3949
3950	/*
3951	* Execute recompiled.
3952	*/
3953	case EMSTATE_REM:
3954	rc = emR3RemExecute(pVM, pVCpu, &fFFDone);
3955	Log2(("EMR3ExecuteVM: emR3RemExecute -> %Rrc\n", rc));
3956	break;
3957
3958	#ifdef VBOX_WITH_VMI
3959	/*
3960	* Execute PARAV function.
3961	*/
3962	case EMSTATE_PARAV:
3963	rc = PARAVCallFunction(pVM);
3964	pVCpu->em.s.enmState = EMSTATE_REM;
3965	break;
3966	#endif
3967
3968	/*
3969	* Application processor execution halted until SIPI.
3970	*/
3971	case EMSTATE_WAIT_SIPI:
3972	/* no break */
3973	/*
3974	* hlt - execution halted until interrupt.
3975	*/
3976	case EMSTATE_HALTED:
3977	{
3978	STAM_REL_PROFILE_START(&pVCpu->em.s.StatHalted, y);
3979	rc = VMR3WaitHalted(pVM, pVCpu, !(CPUMGetGuestEFlags(pVCpu) & X86_EFL_IF));
3980	STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatHalted, y);
3981	break;
3982	}
3983
3984	/*
3985	* Suspended - return to VM.cpp.
3986	*/
3987	case EMSTATE_SUSPENDED:
3988	TMR3NotifySuspend(pVM, pVCpu);
3989	STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
3990	return VINF_EM_SUSPEND;
3991
3992	/*
3993	* Debugging in the guest.
3994	*/
3995	case EMSTATE_DEBUG_GUEST_REM:
3996	case EMSTATE_DEBUG_GUEST_RAW:
3997	TMR3NotifySuspend(pVM, pVCpu);
3998	rc = emR3Debug(pVM, pVCpu, rc);
3999	TMR3NotifyResume(pVM, pVCpu);
4000	Log2(("EMR3ExecuteVM: enmr3Debug -> %Rrc (state %d)\n", rc, pVCpu->em.s.enmState));
4001	break;
4002
4003	/*
4004	* Debugging in the hypervisor.
4005	*/
4006	case EMSTATE_DEBUG_HYPER:
4007	{
4008	TMR3NotifySuspend(pVM, pVCpu);
4009	STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
4010
4011	rc = emR3Debug(pVM, pVCpu, rc);
4012	Log2(("EMR3ExecuteVM: enmr3Debug -> %Rrc (state %d)\n", rc, pVCpu->em.s.enmState));
4013	if (rc != VINF_SUCCESS)
4014	{
4015	/* switch to guru meditation mode */
4016	pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
4017	VMMR3FatalDump(pVM, pVCpu, rc);
4018	return rc;
4019	}
4020
4021	STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatTotal, x);
4022	TMR3NotifyResume(pVM, pVCpu);
4023	break;
4024	}
4025
4026	/*
4027	* Guru meditation takes place in the debugger.
4028	*/
4029	case EMSTATE_GURU_MEDITATION:
4030	{
4031	TMR3NotifySuspend(pVM, pVCpu);
4032	VMMR3FatalDump(pVM, pVCpu, rc);
4033	emR3Debug(pVM, pVCpu, rc);
4034	STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
4035	return rc;
4036	}
4037
4038	/*
4039	* The states we don't expect here.
4040	*/
4041	case EMSTATE_NONE:
4042	case EMSTATE_TERMINATING:
4043	default:
4044	AssertMsgFailed(("EMR3ExecuteVM: Invalid state %d!\n", pVCpu->em.s.enmState));
4045	pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
4046	TMR3NotifySuspend(pVM, pVCpu);
4047	STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
4048	return VERR_EM_INTERNAL_ERROR;
4049	}
4050	} /* The Outer Main Loop */
4051	}
4052	else
4053	{
4054	/*
4055	* Fatal error.
4056	*/
4057	LogFlow(("EMR3ExecuteVM: returns %Rrc (longjmp / fatal error)\n", rc));
4058	TMR3NotifySuspend(pVM, pVCpu);
4059	VMMR3FatalDump(pVM, pVCpu, rc);
4060	emR3Debug(pVM, pVCpu, rc);
4061	STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
4062	/** @todo change the VM state! */
4063	return rc;
4064	}
4065
4066	/* (won't ever get here). */
4067	AssertFailed();
4068	}
4069

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/EM.cpp@ 20961

Download in other formats: