EM.cpp@ 61072

Last change on this file since 61072 was 61027, checked in by vboxsync, 8 years ago
darwin build fix
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 124.4 KB

Line
1	/* $Id: EM.cpp 61027 2016-05-18 08:12:31Z vboxsync $ */
2	/** @file
3	* EM - Execution Monitor / Manager.
4	*/
5
6	/*
7	* Copyright (C) 2006-2015 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/** @page pg_em EM - The Execution Monitor / Manager
19	*
20	* The Execution Monitor/Manager is responsible for running the VM, scheduling
21	* the right kind of execution (Raw-mode, Hardware Assisted, Recompiled or
22	* Interpreted), and keeping the CPU states in sync. The function
23	* EMR3ExecuteVM() is the 'main-loop' of the VM, while each of the execution
24	* modes has different inner loops (emR3RawExecute, emR3HmExecute, and
25	* emR3RemExecute).
26	*
27	* The interpreted execution is only used to avoid switching between
28	* raw-mode/hm and the recompiler when fielding virtualization traps/faults.
29	* The interpretation is thus implemented as part of EM.
30	*
31	* @see grp_em
32	*/
33
34
35	/*********************************************************************************************************************************
36	* Header Files *
37	*********************************************************************************************************************************/
38	#define LOG_GROUP LOG_GROUP_EM
39	#include <VBox/vmm/em.h>
40	#include <VBox/vmm/vmm.h>
41	#include <VBox/vmm/patm.h>
42	#include <VBox/vmm/csam.h>
43	#include <VBox/vmm/selm.h>
44	#include <VBox/vmm/trpm.h>
45	#include <VBox/vmm/iem.h>
46	#include <VBox/vmm/iom.h>
47	#include <VBox/vmm/dbgf.h>
48	#include <VBox/vmm/pgm.h>
49	#ifdef VBOX_WITH_REM
50	# include <VBox/vmm/rem.h>
51	#endif
52	#ifdef VBOX_WITH_NEW_APIC
53	# include <VBox/vmm/apic.h>
54	#endif
55	#include <VBox/vmm/tm.h>
56	#include <VBox/vmm/mm.h>
57	#include <VBox/vmm/ssm.h>
58	#include <VBox/vmm/pdmapi.h>
59	#include <VBox/vmm/pdmcritsect.h>
60	#include <VBox/vmm/pdmqueue.h>
61	#include <VBox/vmm/hm.h>
62	#include <VBox/vmm/patm.h>
63	#include "EMInternal.h"
64	#include <VBox/vmm/vm.h>
65	#include <VBox/vmm/uvm.h>
66	#include <VBox/vmm/cpumdis.h>
67	#include <VBox/dis.h>
68	#include <VBox/disopcode.h>
69	#include "VMMTracing.h"
70
71	#include <iprt/asm.h>
72	#include <iprt/string.h>
73	#include <iprt/stream.h>
74	#include <iprt/thread.h>
75
76
77	/*********************************************************************************************************************************
78	* Defined Constants And Macros *
79	*********************************************************************************************************************************/
80	#if 0 /* Disabled till after 2.1.0 when we've time to test it. */
81	#define EM_NOTIFY_HM
82	#endif
83
84
85	/*********************************************************************************************************************************
86	* Internal Functions *
87	*********************************************************************************************************************************/
88	static DECLCALLBACK(int) emR3Save(PVM pVM, PSSMHANDLE pSSM);
89	static DECLCALLBACK(int) emR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
90	#if defined(LOG_ENABLED) \|\| defined(VBOX_STRICT)
91	static const char *emR3GetStateName(EMSTATE enmState);
92	#endif
93	static VBOXSTRICTRC emR3Debug(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc);
94	static int emR3RemStep(PVM pVM, PVMCPU pVCpu);
95	static int emR3RemExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone);
96	int emR3HighPriorityPostForcedActions(PVM pVM, PVMCPU pVCpu, int rc);
97
98
99	/**
100	* Initializes the EM.
101	*
102	* @returns VBox status code.
103	* @param pVM The cross context VM structure.
104	*/
105	VMMR3_INT_DECL(int) EMR3Init(PVM pVM)
106	{
107	LogFlow(("EMR3Init\n"));
108	/*
109	* Assert alignment and sizes.
110	*/
111	AssertCompileMemberAlignment(VM, em.s, 32);
112	AssertCompile(sizeof(pVM->em.s) <= sizeof(pVM->em.padding));
113	AssertCompile(sizeof(pVM->aCpus[0].em.s.u.FatalLongJump) <= sizeof(pVM->aCpus[0].em.s.u.achPaddingFatalLongJump));
114
115	/*
116	* Init the structure.
117	*/
118	pVM->em.s.offVM = RT_OFFSETOF(VM, em.s);
119	PCFGMNODE pCfgRoot = CFGMR3GetRoot(pVM);
120	PCFGMNODE pCfgEM = CFGMR3GetChild(pCfgRoot, "EM");
121
122	bool fEnabled;
123	int rc = CFGMR3QueryBoolDef(pCfgRoot, "RawR3Enabled", &fEnabled, true);
124	AssertLogRelRCReturn(rc, rc);
125	pVM->fRecompileUser = !fEnabled;
126
127	rc = CFGMR3QueryBoolDef(pCfgRoot, "RawR0Enabled", &fEnabled, true);
128	AssertLogRelRCReturn(rc, rc);
129	pVM->fRecompileSupervisor = !fEnabled;
130
131	#ifdef VBOX_WITH_RAW_RING1
132	rc = CFGMR3QueryBoolDef(pCfgRoot, "RawR1Enabled", &pVM->fRawRing1Enabled, false);
133	AssertLogRelRCReturn(rc, rc);
134	#else
135	pVM->fRawRing1Enabled = false; /* Disabled by default. */
136	#endif
137
138	rc = CFGMR3QueryBoolDef(pCfgEM, "IemExecutesAll", &pVM->em.s.fIemExecutesAll, false);
139	AssertLogRelRCReturn(rc, rc);
140
141	rc = CFGMR3QueryBoolDef(pCfgEM, "TripleFaultReset", &fEnabled, false);
142	AssertLogRelRCReturn(rc, rc);
143	pVM->em.s.fGuruOnTripleFault = !fEnabled;
144	if (!pVM->em.s.fGuruOnTripleFault && pVM->cCpus > 1)
145	{
146	LogRel(("EM: Overriding /EM/TripleFaultReset, must be false on SMP.\n"));
147	pVM->em.s.fGuruOnTripleFault = true;
148	}
149
150	Log(("EMR3Init: fRecompileUser=%RTbool fRecompileSupervisor=%RTbool fRawRing1Enabled=%RTbool fIemExecutesAll=%RTbool fGuruOnTripleFault=%RTbool\n",
151	pVM->fRecompileUser, pVM->fRecompileSupervisor, pVM->fRawRing1Enabled, pVM->em.s.fIemExecutesAll, pVM->em.s.fGuruOnTripleFault));
152
153	#ifdef VBOX_WITH_REM
154	/*
155	* Initialize the REM critical section.
156	*/
157	AssertCompileMemberAlignment(EM, CritSectREM, sizeof(uintptr_t));
158	rc = PDMR3CritSectInit(pVM, &pVM->em.s.CritSectREM, RT_SRC_POS, "EM-REM");
159	AssertRCReturn(rc, rc);
160	#endif
161
162	/*
163	* Saved state.
164	*/
165	rc = SSMR3RegisterInternal(pVM, "em", 0, EM_SAVED_STATE_VERSION, 16,
166	NULL, NULL, NULL,
167	NULL, emR3Save, NULL,
168	NULL, emR3Load, NULL);
169	if (RT_FAILURE(rc))
170	return rc;
171
172	for (VMCPUID i = 0; i < pVM->cCpus; i++)
173	{
174	PVMCPU pVCpu = &pVM->aCpus[i];
175
176	pVCpu->em.s.enmState = (i == 0) ? EMSTATE_NONE : EMSTATE_WAIT_SIPI;
177	pVCpu->em.s.enmPrevState = EMSTATE_NONE;
178	pVCpu->em.s.fForceRAW = false;
179
180	pVCpu->em.s.pCtx = CPUMQueryGuestCtxPtr(pVCpu);
181	#ifdef VBOX_WITH_RAW_MODE
182	if (!HMIsEnabled(pVM))
183	{
184	pVCpu->em.s.pPatmGCState = PATMR3QueryGCStateHC(pVM);
185	AssertMsg(pVCpu->em.s.pPatmGCState, ("PATMR3QueryGCStateHC failed!\n"));
186	}
187	#endif
188
189	/* Force reset of the time slice. */
190	pVCpu->em.s.u64TimeSliceStart = 0;
191
192	# define EM_REG_COUNTER(a, b, c) \
193	rc = STAMR3RegisterF(pVM, a, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, c, b, i); \
194	AssertRC(rc);
195
196	# define EM_REG_COUNTER_USED(a, b, c) \
197	rc = STAMR3RegisterF(pVM, a, STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, c, b, i); \
198	AssertRC(rc);
199
200	# define EM_REG_PROFILE(a, b, c) \
201	rc = STAMR3RegisterF(pVM, a, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL, c, b, i); \
202	AssertRC(rc);
203
204	# define EM_REG_PROFILE_ADV(a, b, c) \
205	rc = STAMR3RegisterF(pVM, a, STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL, c, b, i); \
206	AssertRC(rc);
207
208	/*
209	* Statistics.
210	*/
211	#ifdef VBOX_WITH_STATISTICS
212	PEMSTATS pStats;
213	rc = MMHyperAlloc(pVM, sizeof(pStats), 0, MM_TAG_EM, (void *)&pStats);
214	if (RT_FAILURE(rc))
215	return rc;
216
217	pVCpu->em.s.pStatsR3 = pStats;
218	pVCpu->em.s.pStatsR0 = MMHyperR3ToR0(pVM, pStats);
219	pVCpu->em.s.pStatsRC = MMHyperR3ToRC(pVM, pStats);
220
221	EM_REG_PROFILE(&pStats->StatRZEmulate, "/EM/CPU%d/RZ/Interpret", "Profiling of EMInterpretInstruction.");
222	EM_REG_PROFILE(&pStats->StatR3Emulate, "/EM/CPU%d/R3/Interpret", "Profiling of EMInterpretInstruction.");
223
224	EM_REG_PROFILE(&pStats->StatRZInterpretSucceeded, "/EM/CPU%d/RZ/Interpret/Success", "The number of times an instruction was successfully interpreted.");
225	EM_REG_PROFILE(&pStats->StatR3InterpretSucceeded, "/EM/CPU%d/R3/Interpret/Success", "The number of times an instruction was successfully interpreted.");
226
227	EM_REG_COUNTER_USED(&pStats->StatRZAnd, "/EM/CPU%d/RZ/Interpret/Success/And", "The number of times AND was successfully interpreted.");
228	EM_REG_COUNTER_USED(&pStats->StatR3And, "/EM/CPU%d/R3/Interpret/Success/And", "The number of times AND was successfully interpreted.");
229	EM_REG_COUNTER_USED(&pStats->StatRZAdd, "/EM/CPU%d/RZ/Interpret/Success/Add", "The number of times ADD was successfully interpreted.");
230	EM_REG_COUNTER_USED(&pStats->StatR3Add, "/EM/CPU%d/R3/Interpret/Success/Add", "The number of times ADD was successfully interpreted.");
231	EM_REG_COUNTER_USED(&pStats->StatRZAdc, "/EM/CPU%d/RZ/Interpret/Success/Adc", "The number of times ADC was successfully interpreted.");
232	EM_REG_COUNTER_USED(&pStats->StatR3Adc, "/EM/CPU%d/R3/Interpret/Success/Adc", "The number of times ADC was successfully interpreted.");
233	EM_REG_COUNTER_USED(&pStats->StatRZSub, "/EM/CPU%d/RZ/Interpret/Success/Sub", "The number of times SUB was successfully interpreted.");
234	EM_REG_COUNTER_USED(&pStats->StatR3Sub, "/EM/CPU%d/R3/Interpret/Success/Sub", "The number of times SUB was successfully interpreted.");
235	EM_REG_COUNTER_USED(&pStats->StatRZCpuId, "/EM/CPU%d/RZ/Interpret/Success/CpuId", "The number of times CPUID was successfully interpreted.");
236	EM_REG_COUNTER_USED(&pStats->StatR3CpuId, "/EM/CPU%d/R3/Interpret/Success/CpuId", "The number of times CPUID was successfully interpreted.");
237	EM_REG_COUNTER_USED(&pStats->StatRZDec, "/EM/CPU%d/RZ/Interpret/Success/Dec", "The number of times DEC was successfully interpreted.");
238	EM_REG_COUNTER_USED(&pStats->StatR3Dec, "/EM/CPU%d/R3/Interpret/Success/Dec", "The number of times DEC was successfully interpreted.");
239	EM_REG_COUNTER_USED(&pStats->StatRZHlt, "/EM/CPU%d/RZ/Interpret/Success/Hlt", "The number of times HLT was successfully interpreted.");
240	EM_REG_COUNTER_USED(&pStats->StatR3Hlt, "/EM/CPU%d/R3/Interpret/Success/Hlt", "The number of times HLT was successfully interpreted.");
241	EM_REG_COUNTER_USED(&pStats->StatRZInc, "/EM/CPU%d/RZ/Interpret/Success/Inc", "The number of times INC was successfully interpreted.");
242	EM_REG_COUNTER_USED(&pStats->StatR3Inc, "/EM/CPU%d/R3/Interpret/Success/Inc", "The number of times INC was successfully interpreted.");
243	EM_REG_COUNTER_USED(&pStats->StatRZInvlPg, "/EM/CPU%d/RZ/Interpret/Success/Invlpg", "The number of times INVLPG was successfully interpreted.");
244	EM_REG_COUNTER_USED(&pStats->StatR3InvlPg, "/EM/CPU%d/R3/Interpret/Success/Invlpg", "The number of times INVLPG was successfully interpreted.");
245	EM_REG_COUNTER_USED(&pStats->StatRZIret, "/EM/CPU%d/RZ/Interpret/Success/Iret", "The number of times IRET was successfully interpreted.");
246	EM_REG_COUNTER_USED(&pStats->StatR3Iret, "/EM/CPU%d/R3/Interpret/Success/Iret", "The number of times IRET was successfully interpreted.");
247	EM_REG_COUNTER_USED(&pStats->StatRZLLdt, "/EM/CPU%d/RZ/Interpret/Success/LLdt", "The number of times LLDT was successfully interpreted.");
248	EM_REG_COUNTER_USED(&pStats->StatR3LLdt, "/EM/CPU%d/R3/Interpret/Success/LLdt", "The number of times LLDT was successfully interpreted.");
249	EM_REG_COUNTER_USED(&pStats->StatRZLIdt, "/EM/CPU%d/RZ/Interpret/Success/LIdt", "The number of times LIDT was successfully interpreted.");
250	EM_REG_COUNTER_USED(&pStats->StatR3LIdt, "/EM/CPU%d/R3/Interpret/Success/LIdt", "The number of times LIDT was successfully interpreted.");
251	EM_REG_COUNTER_USED(&pStats->StatRZLGdt, "/EM/CPU%d/RZ/Interpret/Success/LGdt", "The number of times LGDT was successfully interpreted.");
252	EM_REG_COUNTER_USED(&pStats->StatR3LGdt, "/EM/CPU%d/R3/Interpret/Success/LGdt", "The number of times LGDT was successfully interpreted.");
253	EM_REG_COUNTER_USED(&pStats->StatRZMov, "/EM/CPU%d/RZ/Interpret/Success/Mov", "The number of times MOV was successfully interpreted.");
254	EM_REG_COUNTER_USED(&pStats->StatR3Mov, "/EM/CPU%d/R3/Interpret/Success/Mov", "The number of times MOV was successfully interpreted.");
255	EM_REG_COUNTER_USED(&pStats->StatRZMovCRx, "/EM/CPU%d/RZ/Interpret/Success/MovCRx", "The number of times MOV CRx was successfully interpreted.");
256	EM_REG_COUNTER_USED(&pStats->StatR3MovCRx, "/EM/CPU%d/R3/Interpret/Success/MovCRx", "The number of times MOV CRx was successfully interpreted.");
257	EM_REG_COUNTER_USED(&pStats->StatRZMovDRx, "/EM/CPU%d/RZ/Interpret/Success/MovDRx", "The number of times MOV DRx was successfully interpreted.");
258	EM_REG_COUNTER_USED(&pStats->StatR3MovDRx, "/EM/CPU%d/R3/Interpret/Success/MovDRx", "The number of times MOV DRx was successfully interpreted.");
259	EM_REG_COUNTER_USED(&pStats->StatRZOr, "/EM/CPU%d/RZ/Interpret/Success/Or", "The number of times OR was successfully interpreted.");
260	EM_REG_COUNTER_USED(&pStats->StatR3Or, "/EM/CPU%d/R3/Interpret/Success/Or", "The number of times OR was successfully interpreted.");
261	EM_REG_COUNTER_USED(&pStats->StatRZPop, "/EM/CPU%d/RZ/Interpret/Success/Pop", "The number of times POP was successfully interpreted.");
262	EM_REG_COUNTER_USED(&pStats->StatR3Pop, "/EM/CPU%d/R3/Interpret/Success/Pop", "The number of times POP was successfully interpreted.");
263	EM_REG_COUNTER_USED(&pStats->StatRZRdtsc, "/EM/CPU%d/RZ/Interpret/Success/Rdtsc", "The number of times RDTSC was successfully interpreted.");
264	EM_REG_COUNTER_USED(&pStats->StatR3Rdtsc, "/EM/CPU%d/R3/Interpret/Success/Rdtsc", "The number of times RDTSC was successfully interpreted.");
265	EM_REG_COUNTER_USED(&pStats->StatRZRdpmc, "/EM/CPU%d/RZ/Interpret/Success/Rdpmc", "The number of times RDPMC was successfully interpreted.");
266	EM_REG_COUNTER_USED(&pStats->StatR3Rdpmc, "/EM/CPU%d/R3/Interpret/Success/Rdpmc", "The number of times RDPMC was successfully interpreted.");
267	EM_REG_COUNTER_USED(&pStats->StatRZSti, "/EM/CPU%d/RZ/Interpret/Success/Sti", "The number of times STI was successfully interpreted.");
268	EM_REG_COUNTER_USED(&pStats->StatR3Sti, "/EM/CPU%d/R3/Interpret/Success/Sti", "The number of times STI was successfully interpreted.");
269	EM_REG_COUNTER_USED(&pStats->StatRZXchg, "/EM/CPU%d/RZ/Interpret/Success/Xchg", "The number of times XCHG was successfully interpreted.");
270	EM_REG_COUNTER_USED(&pStats->StatR3Xchg, "/EM/CPU%d/R3/Interpret/Success/Xchg", "The number of times XCHG was successfully interpreted.");
271	EM_REG_COUNTER_USED(&pStats->StatRZXor, "/EM/CPU%d/RZ/Interpret/Success/Xor", "The number of times XOR was successfully interpreted.");
272	EM_REG_COUNTER_USED(&pStats->StatR3Xor, "/EM/CPU%d/R3/Interpret/Success/Xor", "The number of times XOR was successfully interpreted.");
273	EM_REG_COUNTER_USED(&pStats->StatRZMonitor, "/EM/CPU%d/RZ/Interpret/Success/Monitor", "The number of times MONITOR was successfully interpreted.");
274	EM_REG_COUNTER_USED(&pStats->StatR3Monitor, "/EM/CPU%d/R3/Interpret/Success/Monitor", "The number of times MONITOR was successfully interpreted.");
275	EM_REG_COUNTER_USED(&pStats->StatRZMWait, "/EM/CPU%d/RZ/Interpret/Success/MWait", "The number of times MWAIT was successfully interpreted.");
276	EM_REG_COUNTER_USED(&pStats->StatR3MWait, "/EM/CPU%d/R3/Interpret/Success/MWait", "The number of times MWAIT was successfully interpreted.");
277	EM_REG_COUNTER_USED(&pStats->StatRZBtr, "/EM/CPU%d/RZ/Interpret/Success/Btr", "The number of times BTR was successfully interpreted.");
278	EM_REG_COUNTER_USED(&pStats->StatR3Btr, "/EM/CPU%d/R3/Interpret/Success/Btr", "The number of times BTR was successfully interpreted.");
279	EM_REG_COUNTER_USED(&pStats->StatRZBts, "/EM/CPU%d/RZ/Interpret/Success/Bts", "The number of times BTS was successfully interpreted.");
280	EM_REG_COUNTER_USED(&pStats->StatR3Bts, "/EM/CPU%d/R3/Interpret/Success/Bts", "The number of times BTS was successfully interpreted.");
281	EM_REG_COUNTER_USED(&pStats->StatRZBtc, "/EM/CPU%d/RZ/Interpret/Success/Btc", "The number of times BTC was successfully interpreted.");
282	EM_REG_COUNTER_USED(&pStats->StatR3Btc, "/EM/CPU%d/R3/Interpret/Success/Btc", "The number of times BTC was successfully interpreted.");
283	EM_REG_COUNTER_USED(&pStats->StatRZCmpXchg, "/EM/CPU%d/RZ/Interpret/Success/CmpXchg", "The number of times CMPXCHG was successfully interpreted.");
284	EM_REG_COUNTER_USED(&pStats->StatR3CmpXchg, "/EM/CPU%d/R3/Interpret/Success/CmpXchg", "The number of times CMPXCHG was successfully interpreted.");
285	EM_REG_COUNTER_USED(&pStats->StatRZCmpXchg8b, "/EM/CPU%d/RZ/Interpret/Success/CmpXchg8b", "The number of times CMPXCHG8B was successfully interpreted.");
286	EM_REG_COUNTER_USED(&pStats->StatR3CmpXchg8b, "/EM/CPU%d/R3/Interpret/Success/CmpXchg8b", "The number of times CMPXCHG8B was successfully interpreted.");
287	EM_REG_COUNTER_USED(&pStats->StatRZXAdd, "/EM/CPU%d/RZ/Interpret/Success/XAdd", "The number of times XADD was successfully interpreted.");
288	EM_REG_COUNTER_USED(&pStats->StatR3XAdd, "/EM/CPU%d/R3/Interpret/Success/XAdd", "The number of times XADD was successfully interpreted.");
289	EM_REG_COUNTER_USED(&pStats->StatR3Rdmsr, "/EM/CPU%d/R3/Interpret/Success/Rdmsr", "The number of times RDMSR was successfully interpreted.");
290	EM_REG_COUNTER_USED(&pStats->StatRZRdmsr, "/EM/CPU%d/RZ/Interpret/Success/Rdmsr", "The number of times RDMSR was successfully interpreted.");
291	EM_REG_COUNTER_USED(&pStats->StatR3Wrmsr, "/EM/CPU%d/R3/Interpret/Success/Wrmsr", "The number of times WRMSR was successfully interpreted.");
292	EM_REG_COUNTER_USED(&pStats->StatRZWrmsr, "/EM/CPU%d/RZ/Interpret/Success/Wrmsr", "The number of times WRMSR was successfully interpreted.");
293	EM_REG_COUNTER_USED(&pStats->StatR3StosWD, "/EM/CPU%d/R3/Interpret/Success/Stoswd", "The number of times STOSWD was successfully interpreted.");
294	EM_REG_COUNTER_USED(&pStats->StatRZStosWD, "/EM/CPU%d/RZ/Interpret/Success/Stoswd", "The number of times STOSWD was successfully interpreted.");
295	EM_REG_COUNTER_USED(&pStats->StatRZWbInvd, "/EM/CPU%d/RZ/Interpret/Success/WbInvd", "The number of times WBINVD was successfully interpreted.");
296	EM_REG_COUNTER_USED(&pStats->StatR3WbInvd, "/EM/CPU%d/R3/Interpret/Success/WbInvd", "The number of times WBINVD was successfully interpreted.");
297	EM_REG_COUNTER_USED(&pStats->StatRZLmsw, "/EM/CPU%d/RZ/Interpret/Success/Lmsw", "The number of times LMSW was successfully interpreted.");
298	EM_REG_COUNTER_USED(&pStats->StatR3Lmsw, "/EM/CPU%d/R3/Interpret/Success/Lmsw", "The number of times LMSW was successfully interpreted.");
299	EM_REG_COUNTER_USED(&pStats->StatRZSmsw, "/EM/CPU%d/RZ/Interpret/Success/Smsw", "The number of times SMSW was successfully interpreted.");
300	EM_REG_COUNTER_USED(&pStats->StatR3Smsw, "/EM/CPU%d/R3/Interpret/Success/Smsw", "The number of times SMSW was successfully interpreted.");
301
302	EM_REG_COUNTER(&pStats->StatRZInterpretFailed, "/EM/CPU%d/RZ/Interpret/Failed", "The number of times an instruction was not interpreted.");
303	EM_REG_COUNTER(&pStats->StatR3InterpretFailed, "/EM/CPU%d/R3/Interpret/Failed", "The number of times an instruction was not interpreted.");
304
305	EM_REG_COUNTER_USED(&pStats->StatRZFailedAnd, "/EM/CPU%d/RZ/Interpret/Failed/And", "The number of times AND was not interpreted.");
306	EM_REG_COUNTER_USED(&pStats->StatR3FailedAnd, "/EM/CPU%d/R3/Interpret/Failed/And", "The number of times AND was not interpreted.");
307	EM_REG_COUNTER_USED(&pStats->StatRZFailedCpuId, "/EM/CPU%d/RZ/Interpret/Failed/CpuId", "The number of times CPUID was not interpreted.");
308	EM_REG_COUNTER_USED(&pStats->StatR3FailedCpuId, "/EM/CPU%d/R3/Interpret/Failed/CpuId", "The number of times CPUID was not interpreted.");
309	EM_REG_COUNTER_USED(&pStats->StatRZFailedDec, "/EM/CPU%d/RZ/Interpret/Failed/Dec", "The number of times DEC was not interpreted.");
310	EM_REG_COUNTER_USED(&pStats->StatR3FailedDec, "/EM/CPU%d/R3/Interpret/Failed/Dec", "The number of times DEC was not interpreted.");
311	EM_REG_COUNTER_USED(&pStats->StatRZFailedHlt, "/EM/CPU%d/RZ/Interpret/Failed/Hlt", "The number of times HLT was not interpreted.");
312	EM_REG_COUNTER_USED(&pStats->StatR3FailedHlt, "/EM/CPU%d/R3/Interpret/Failed/Hlt", "The number of times HLT was not interpreted.");
313	EM_REG_COUNTER_USED(&pStats->StatRZFailedInc, "/EM/CPU%d/RZ/Interpret/Failed/Inc", "The number of times INC was not interpreted.");
314	EM_REG_COUNTER_USED(&pStats->StatR3FailedInc, "/EM/CPU%d/R3/Interpret/Failed/Inc", "The number of times INC was not interpreted.");
315	EM_REG_COUNTER_USED(&pStats->StatRZFailedInvlPg, "/EM/CPU%d/RZ/Interpret/Failed/InvlPg", "The number of times INVLPG was not interpreted.");
316	EM_REG_COUNTER_USED(&pStats->StatR3FailedInvlPg, "/EM/CPU%d/R3/Interpret/Failed/InvlPg", "The number of times INVLPG was not interpreted.");
317	EM_REG_COUNTER_USED(&pStats->StatRZFailedIret, "/EM/CPU%d/RZ/Interpret/Failed/Iret", "The number of times IRET was not interpreted.");
318	EM_REG_COUNTER_USED(&pStats->StatR3FailedIret, "/EM/CPU%d/R3/Interpret/Failed/Iret", "The number of times IRET was not interpreted.");
319	EM_REG_COUNTER_USED(&pStats->StatRZFailedLLdt, "/EM/CPU%d/RZ/Interpret/Failed/LLdt", "The number of times LLDT was not interpreted.");
320	EM_REG_COUNTER_USED(&pStats->StatR3FailedLLdt, "/EM/CPU%d/R3/Interpret/Failed/LLdt", "The number of times LLDT was not interpreted.");
321	EM_REG_COUNTER_USED(&pStats->StatRZFailedLIdt, "/EM/CPU%d/RZ/Interpret/Failed/LIdt", "The number of times LIDT was not interpreted.");
322	EM_REG_COUNTER_USED(&pStats->StatR3FailedLIdt, "/EM/CPU%d/R3/Interpret/Failed/LIdt", "The number of times LIDT was not interpreted.");
323	EM_REG_COUNTER_USED(&pStats->StatRZFailedLGdt, "/EM/CPU%d/RZ/Interpret/Failed/LGdt", "The number of times LGDT was not interpreted.");
324	EM_REG_COUNTER_USED(&pStats->StatR3FailedLGdt, "/EM/CPU%d/R3/Interpret/Failed/LGdt", "The number of times LGDT was not interpreted.");
325	EM_REG_COUNTER_USED(&pStats->StatRZFailedMov, "/EM/CPU%d/RZ/Interpret/Failed/Mov", "The number of times MOV was not interpreted.");
326	EM_REG_COUNTER_USED(&pStats->StatR3FailedMov, "/EM/CPU%d/R3/Interpret/Failed/Mov", "The number of times MOV was not interpreted.");
327	EM_REG_COUNTER_USED(&pStats->StatRZFailedMovCRx, "/EM/CPU%d/RZ/Interpret/Failed/MovCRx", "The number of times MOV CRx was not interpreted.");
328	EM_REG_COUNTER_USED(&pStats->StatR3FailedMovCRx, "/EM/CPU%d/R3/Interpret/Failed/MovCRx", "The number of times MOV CRx was not interpreted.");
329	EM_REG_COUNTER_USED(&pStats->StatRZFailedMovDRx, "/EM/CPU%d/RZ/Interpret/Failed/MovDRx", "The number of times MOV DRx was not interpreted.");
330	EM_REG_COUNTER_USED(&pStats->StatR3FailedMovDRx, "/EM/CPU%d/R3/Interpret/Failed/MovDRx", "The number of times MOV DRx was not interpreted.");
331	EM_REG_COUNTER_USED(&pStats->StatRZFailedOr, "/EM/CPU%d/RZ/Interpret/Failed/Or", "The number of times OR was not interpreted.");
332	EM_REG_COUNTER_USED(&pStats->StatR3FailedOr, "/EM/CPU%d/R3/Interpret/Failed/Or", "The number of times OR was not interpreted.");
333	EM_REG_COUNTER_USED(&pStats->StatRZFailedPop, "/EM/CPU%d/RZ/Interpret/Failed/Pop", "The number of times POP was not interpreted.");
334	EM_REG_COUNTER_USED(&pStats->StatR3FailedPop, "/EM/CPU%d/R3/Interpret/Failed/Pop", "The number of times POP was not interpreted.");
335	EM_REG_COUNTER_USED(&pStats->StatRZFailedSti, "/EM/CPU%d/RZ/Interpret/Failed/Sti", "The number of times STI was not interpreted.");
336	EM_REG_COUNTER_USED(&pStats->StatR3FailedSti, "/EM/CPU%d/R3/Interpret/Failed/Sti", "The number of times STI was not interpreted.");
337	EM_REG_COUNTER_USED(&pStats->StatRZFailedXchg, "/EM/CPU%d/RZ/Interpret/Failed/Xchg", "The number of times XCHG was not interpreted.");
338	EM_REG_COUNTER_USED(&pStats->StatR3FailedXchg, "/EM/CPU%d/R3/Interpret/Failed/Xchg", "The number of times XCHG was not interpreted.");
339	EM_REG_COUNTER_USED(&pStats->StatRZFailedXor, "/EM/CPU%d/RZ/Interpret/Failed/Xor", "The number of times XOR was not interpreted.");
340	EM_REG_COUNTER_USED(&pStats->StatR3FailedXor, "/EM/CPU%d/R3/Interpret/Failed/Xor", "The number of times XOR was not interpreted.");
341	EM_REG_COUNTER_USED(&pStats->StatRZFailedMonitor, "/EM/CPU%d/RZ/Interpret/Failed/Monitor", "The number of times MONITOR was not interpreted.");
342	EM_REG_COUNTER_USED(&pStats->StatR3FailedMonitor, "/EM/CPU%d/R3/Interpret/Failed/Monitor", "The number of times MONITOR was not interpreted.");
343	EM_REG_COUNTER_USED(&pStats->StatRZFailedMWait, "/EM/CPU%d/RZ/Interpret/Failed/MWait", "The number of times MWAIT was not interpreted.");
344	EM_REG_COUNTER_USED(&pStats->StatR3FailedMWait, "/EM/CPU%d/R3/Interpret/Failed/MWait", "The number of times MWAIT was not interpreted.");
345	EM_REG_COUNTER_USED(&pStats->StatRZFailedRdtsc, "/EM/CPU%d/RZ/Interpret/Failed/Rdtsc", "The number of times RDTSC was not interpreted.");
346	EM_REG_COUNTER_USED(&pStats->StatR3FailedRdtsc, "/EM/CPU%d/R3/Interpret/Failed/Rdtsc", "The number of times RDTSC was not interpreted.");
347	EM_REG_COUNTER_USED(&pStats->StatRZFailedRdpmc, "/EM/CPU%d/RZ/Interpret/Failed/Rdpmc", "The number of times RDPMC was not interpreted.");
348	EM_REG_COUNTER_USED(&pStats->StatR3FailedRdpmc, "/EM/CPU%d/R3/Interpret/Failed/Rdpmc", "The number of times RDPMC was not interpreted.");
349	EM_REG_COUNTER_USED(&pStats->StatRZFailedRdmsr, "/EM/CPU%d/RZ/Interpret/Failed/Rdmsr", "The number of times RDMSR was not interpreted.");
350	EM_REG_COUNTER_USED(&pStats->StatR3FailedRdmsr, "/EM/CPU%d/R3/Interpret/Failed/Rdmsr", "The number of times RDMSR was not interpreted.");
351	EM_REG_COUNTER_USED(&pStats->StatRZFailedWrmsr, "/EM/CPU%d/RZ/Interpret/Failed/Wrmsr", "The number of times WRMSR was not interpreted.");
352	EM_REG_COUNTER_USED(&pStats->StatR3FailedWrmsr, "/EM/CPU%d/R3/Interpret/Failed/Wrmsr", "The number of times WRMSR was not interpreted.");
353	EM_REG_COUNTER_USED(&pStats->StatRZFailedLmsw, "/EM/CPU%d/RZ/Interpret/Failed/Lmsw", "The number of times LMSW was not interpreted.");
354	EM_REG_COUNTER_USED(&pStats->StatR3FailedLmsw, "/EM/CPU%d/R3/Interpret/Failed/Lmsw", "The number of times LMSW was not interpreted.");
355	EM_REG_COUNTER_USED(&pStats->StatRZFailedSmsw, "/EM/CPU%d/RZ/Interpret/Failed/Smsw", "The number of times SMSW was not interpreted.");
356	EM_REG_COUNTER_USED(&pStats->StatR3FailedSmsw, "/EM/CPU%d/R3/Interpret/Failed/Smsw", "The number of times SMSW was not interpreted.");
357
358	EM_REG_COUNTER_USED(&pStats->StatRZFailedMisc, "/EM/CPU%d/RZ/Interpret/Failed/Misc", "The number of times some misc instruction was encountered.");
359	EM_REG_COUNTER_USED(&pStats->StatR3FailedMisc, "/EM/CPU%d/R3/Interpret/Failed/Misc", "The number of times some misc instruction was encountered.");
360	EM_REG_COUNTER_USED(&pStats->StatRZFailedAdd, "/EM/CPU%d/RZ/Interpret/Failed/Add", "The number of times ADD was not interpreted.");
361	EM_REG_COUNTER_USED(&pStats->StatR3FailedAdd, "/EM/CPU%d/R3/Interpret/Failed/Add", "The number of times ADD was not interpreted.");
362	EM_REG_COUNTER_USED(&pStats->StatRZFailedAdc, "/EM/CPU%d/RZ/Interpret/Failed/Adc", "The number of times ADC was not interpreted.");
363	EM_REG_COUNTER_USED(&pStats->StatR3FailedAdc, "/EM/CPU%d/R3/Interpret/Failed/Adc", "The number of times ADC was not interpreted.");
364	EM_REG_COUNTER_USED(&pStats->StatRZFailedBtr, "/EM/CPU%d/RZ/Interpret/Failed/Btr", "The number of times BTR was not interpreted.");
365	EM_REG_COUNTER_USED(&pStats->StatR3FailedBtr, "/EM/CPU%d/R3/Interpret/Failed/Btr", "The number of times BTR was not interpreted.");
366	EM_REG_COUNTER_USED(&pStats->StatRZFailedBts, "/EM/CPU%d/RZ/Interpret/Failed/Bts", "The number of times BTS was not interpreted.");
367	EM_REG_COUNTER_USED(&pStats->StatR3FailedBts, "/EM/CPU%d/R3/Interpret/Failed/Bts", "The number of times BTS was not interpreted.");
368	EM_REG_COUNTER_USED(&pStats->StatRZFailedBtc, "/EM/CPU%d/RZ/Interpret/Failed/Btc", "The number of times BTC was not interpreted.");
369	EM_REG_COUNTER_USED(&pStats->StatR3FailedBtc, "/EM/CPU%d/R3/Interpret/Failed/Btc", "The number of times BTC was not interpreted.");
370	EM_REG_COUNTER_USED(&pStats->StatRZFailedCli, "/EM/CPU%d/RZ/Interpret/Failed/Cli", "The number of times CLI was not interpreted.");
371	EM_REG_COUNTER_USED(&pStats->StatR3FailedCli, "/EM/CPU%d/R3/Interpret/Failed/Cli", "The number of times CLI was not interpreted.");
372	EM_REG_COUNTER_USED(&pStats->StatRZFailedCmpXchg, "/EM/CPU%d/RZ/Interpret/Failed/CmpXchg", "The number of times CMPXCHG was not interpreted.");
373	EM_REG_COUNTER_USED(&pStats->StatR3FailedCmpXchg, "/EM/CPU%d/R3/Interpret/Failed/CmpXchg", "The number of times CMPXCHG was not interpreted.");
374	EM_REG_COUNTER_USED(&pStats->StatRZFailedCmpXchg8b, "/EM/CPU%d/RZ/Interpret/Failed/CmpXchg8b", "The number of times CMPXCHG8B was not interpreted.");
375	EM_REG_COUNTER_USED(&pStats->StatR3FailedCmpXchg8b, "/EM/CPU%d/R3/Interpret/Failed/CmpXchg8b", "The number of times CMPXCHG8B was not interpreted.");
376	EM_REG_COUNTER_USED(&pStats->StatRZFailedXAdd, "/EM/CPU%d/RZ/Interpret/Failed/XAdd", "The number of times XADD was not interpreted.");
377	EM_REG_COUNTER_USED(&pStats->StatR3FailedXAdd, "/EM/CPU%d/R3/Interpret/Failed/XAdd", "The number of times XADD was not interpreted.");
378	EM_REG_COUNTER_USED(&pStats->StatRZFailedMovNTPS, "/EM/CPU%d/RZ/Interpret/Failed/MovNTPS", "The number of times MOVNTPS was not interpreted.");
379	EM_REG_COUNTER_USED(&pStats->StatR3FailedMovNTPS, "/EM/CPU%d/R3/Interpret/Failed/MovNTPS", "The number of times MOVNTPS was not interpreted.");
380	EM_REG_COUNTER_USED(&pStats->StatRZFailedStosWD, "/EM/CPU%d/RZ/Interpret/Failed/StosWD", "The number of times STOSWD was not interpreted.");
381	EM_REG_COUNTER_USED(&pStats->StatR3FailedStosWD, "/EM/CPU%d/R3/Interpret/Failed/StosWD", "The number of times STOSWD was not interpreted.");
382	EM_REG_COUNTER_USED(&pStats->StatRZFailedSub, "/EM/CPU%d/RZ/Interpret/Failed/Sub", "The number of times SUB was not interpreted.");
383	EM_REG_COUNTER_USED(&pStats->StatR3FailedSub, "/EM/CPU%d/R3/Interpret/Failed/Sub", "The number of times SUB was not interpreted.");
384	EM_REG_COUNTER_USED(&pStats->StatRZFailedWbInvd, "/EM/CPU%d/RZ/Interpret/Failed/WbInvd", "The number of times WBINVD was not interpreted.");
385	EM_REG_COUNTER_USED(&pStats->StatR3FailedWbInvd, "/EM/CPU%d/R3/Interpret/Failed/WbInvd", "The number of times WBINVD was not interpreted.");
386
387	EM_REG_COUNTER_USED(&pStats->StatRZFailedUserMode, "/EM/CPU%d/RZ/Interpret/Failed/UserMode", "The number of rejections because of CPL.");
388	EM_REG_COUNTER_USED(&pStats->StatR3FailedUserMode, "/EM/CPU%d/R3/Interpret/Failed/UserMode", "The number of rejections because of CPL.");
389	EM_REG_COUNTER_USED(&pStats->StatRZFailedPrefix, "/EM/CPU%d/RZ/Interpret/Failed/Prefix", "The number of rejections because of prefix .");
390	EM_REG_COUNTER_USED(&pStats->StatR3FailedPrefix, "/EM/CPU%d/R3/Interpret/Failed/Prefix", "The number of rejections because of prefix .");
391
392	EM_REG_COUNTER_USED(&pStats->StatIoRestarted, "/EM/CPU%d/R3/PrivInst/IoRestarted", "I/O instructions restarted in ring-3.");
393	EM_REG_COUNTER_USED(&pStats->StatIoIem, "/EM/CPU%d/R3/PrivInst/IoIem", "I/O instructions end to IEM in ring-3.");
394	EM_REG_COUNTER_USED(&pStats->StatCli, "/EM/CPU%d/R3/PrivInst/Cli", "Number of cli instructions.");
395	EM_REG_COUNTER_USED(&pStats->StatSti, "/EM/CPU%d/R3/PrivInst/Sti", "Number of sli instructions.");
396	EM_REG_COUNTER_USED(&pStats->StatHlt, "/EM/CPU%d/R3/PrivInst/Hlt", "Number of hlt instructions not handled in GC because of PATM.");
397	EM_REG_COUNTER_USED(&pStats->StatInvlpg, "/EM/CPU%d/R3/PrivInst/Invlpg", "Number of invlpg instructions.");
398	EM_REG_COUNTER_USED(&pStats->StatMisc, "/EM/CPU%d/R3/PrivInst/Misc", "Number of misc. instructions.");
399	EM_REG_COUNTER_USED(&pStats->StatMovWriteCR[0], "/EM/CPU%d/R3/PrivInst/Mov CR0, X", "Number of mov CR0 write instructions.");
400	EM_REG_COUNTER_USED(&pStats->StatMovWriteCR[1], "/EM/CPU%d/R3/PrivInst/Mov CR1, X", "Number of mov CR1 write instructions.");
401	EM_REG_COUNTER_USED(&pStats->StatMovWriteCR[2], "/EM/CPU%d/R3/PrivInst/Mov CR2, X", "Number of mov CR2 write instructions.");
402	EM_REG_COUNTER_USED(&pStats->StatMovWriteCR[3], "/EM/CPU%d/R3/PrivInst/Mov CR3, X", "Number of mov CR3 write instructions.");
403	EM_REG_COUNTER_USED(&pStats->StatMovWriteCR[4], "/EM/CPU%d/R3/PrivInst/Mov CR4, X", "Number of mov CR4 write instructions.");
404	EM_REG_COUNTER_USED(&pStats->StatMovReadCR[0], "/EM/CPU%d/R3/PrivInst/Mov X, CR0", "Number of mov CR0 read instructions.");
405	EM_REG_COUNTER_USED(&pStats->StatMovReadCR[1], "/EM/CPU%d/R3/PrivInst/Mov X, CR1", "Number of mov CR1 read instructions.");
406	EM_REG_COUNTER_USED(&pStats->StatMovReadCR[2], "/EM/CPU%d/R3/PrivInst/Mov X, CR2", "Number of mov CR2 read instructions.");
407	EM_REG_COUNTER_USED(&pStats->StatMovReadCR[3], "/EM/CPU%d/R3/PrivInst/Mov X, CR3", "Number of mov CR3 read instructions.");
408	EM_REG_COUNTER_USED(&pStats->StatMovReadCR[4], "/EM/CPU%d/R3/PrivInst/Mov X, CR4", "Number of mov CR4 read instructions.");
409	EM_REG_COUNTER_USED(&pStats->StatMovDRx, "/EM/CPU%d/R3/PrivInst/MovDRx", "Number of mov DRx instructions.");
410	EM_REG_COUNTER_USED(&pStats->StatIret, "/EM/CPU%d/R3/PrivInst/Iret", "Number of iret instructions.");
411	EM_REG_COUNTER_USED(&pStats->StatMovLgdt, "/EM/CPU%d/R3/PrivInst/Lgdt", "Number of lgdt instructions.");
412	EM_REG_COUNTER_USED(&pStats->StatMovLidt, "/EM/CPU%d/R3/PrivInst/Lidt", "Number of lidt instructions.");
413	EM_REG_COUNTER_USED(&pStats->StatMovLldt, "/EM/CPU%d/R3/PrivInst/Lldt", "Number of lldt instructions.");
414	EM_REG_COUNTER_USED(&pStats->StatSysEnter, "/EM/CPU%d/R3/PrivInst/Sysenter", "Number of sysenter instructions.");
415	EM_REG_COUNTER_USED(&pStats->StatSysExit, "/EM/CPU%d/R3/PrivInst/Sysexit", "Number of sysexit instructions.");
416	EM_REG_COUNTER_USED(&pStats->StatSysCall, "/EM/CPU%d/R3/PrivInst/Syscall", "Number of syscall instructions.");
417	EM_REG_COUNTER_USED(&pStats->StatSysRet, "/EM/CPU%d/R3/PrivInst/Sysret", "Number of sysret instructions.");
418
419	EM_REG_COUNTER(&pVCpu->em.s.StatTotalClis, "/EM/CPU%d/Cli/Total", "Total number of cli instructions executed.");
420	pVCpu->em.s.pCliStatTree = 0;
421
422	/* these should be considered for release statistics. */
423	EM_REG_COUNTER(&pVCpu->em.s.StatIOEmu, "/PROF/CPU%d/EM/Emulation/IO", "Profiling of emR3RawExecuteIOInstruction.");
424	EM_REG_COUNTER(&pVCpu->em.s.StatPrivEmu, "/PROF/CPU%d/EM/Emulation/Priv", "Profiling of emR3RawPrivileged.");
425	EM_REG_PROFILE(&pVCpu->em.s.StatHmEntry, "/PROF/CPU%d/EM/HmEnter", "Profiling Hardware Accelerated Mode entry overhead.");
426	EM_REG_PROFILE(&pVCpu->em.s.StatHmExec, "/PROF/CPU%d/EM/HmExec", "Profiling Hardware Accelerated Mode execution.");
427	EM_REG_PROFILE(&pVCpu->em.s.StatIEMEmu, "/PROF/CPU%d/EM/IEMEmuSingle", "Profiling single instruction IEM execution.");
428	EM_REG_PROFILE(&pVCpu->em.s.StatIEMThenREM, "/PROF/CPU%d/EM/IEMThenRem", "Profiling IEM-then-REM instruction execution (by IEM).");
429	EM_REG_PROFILE(&pVCpu->em.s.StatREMEmu, "/PROF/CPU%d/EM/REMEmuSingle", "Profiling single instruction REM execution.");
430	EM_REG_PROFILE(&pVCpu->em.s.StatREMExec, "/PROF/CPU%d/EM/REMExec", "Profiling REM execution.");
431	EM_REG_PROFILE(&pVCpu->em.s.StatREMSync, "/PROF/CPU%d/EM/REMSync", "Profiling REM context syncing.");
432	EM_REG_PROFILE(&pVCpu->em.s.StatRAWEntry, "/PROF/CPU%d/EM/RAWEnter", "Profiling Raw Mode entry overhead.");
433	EM_REG_PROFILE(&pVCpu->em.s.StatRAWExec, "/PROF/CPU%d/EM/RAWExec", "Profiling Raw Mode execution.");
434	EM_REG_PROFILE(&pVCpu->em.s.StatRAWTail, "/PROF/CPU%d/EM/RAWTail", "Profiling Raw Mode tail overhead.");
435
436	#endif /* VBOX_WITH_STATISTICS */
437
438	EM_REG_COUNTER(&pVCpu->em.s.StatForcedActions, "/PROF/CPU%d/EM/ForcedActions", "Profiling forced action execution.");
439	EM_REG_COUNTER(&pVCpu->em.s.StatHalted, "/PROF/CPU%d/EM/Halted", "Profiling halted state (VMR3WaitHalted).");
440	EM_REG_PROFILE_ADV(&pVCpu->em.s.StatCapped, "/PROF/CPU%d/EM/Capped", "Profiling capped state (sleep).");
441	EM_REG_COUNTER(&pVCpu->em.s.StatREMTotal, "/PROF/CPU%d/EM/REMTotal", "Profiling emR3RemExecute (excluding FFs).");
442	EM_REG_COUNTER(&pVCpu->em.s.StatRAWTotal, "/PROF/CPU%d/EM/RAWTotal", "Profiling emR3RawExecute (excluding FFs).");
443
444	EM_REG_PROFILE_ADV(&pVCpu->em.s.StatTotal, "/PROF/CPU%d/EM/Total", "Profiling EMR3ExecuteVM.");
445	}
446
447	emR3InitDbg(pVM);
448	return VINF_SUCCESS;
449	}
450
451
452	/**
453	* Applies relocations to data and code managed by this
454	* component. This function will be called at init and
455	* whenever the VMM need to relocate it self inside the GC.
456	*
457	* @param pVM The cross context VM structure.
458	*/
459	VMMR3_INT_DECL(void) EMR3Relocate(PVM pVM)
460	{
461	LogFlow(("EMR3Relocate\n"));
462	for (VMCPUID i = 0; i < pVM->cCpus; i++)
463	{
464	PVMCPU pVCpu = &pVM->aCpus[i];
465	if (pVCpu->em.s.pStatsR3)
466	pVCpu->em.s.pStatsRC = MMHyperR3ToRC(pVM, pVCpu->em.s.pStatsR3);
467	}
468	}
469
470
471	/**
472	* Reset the EM state for a CPU.
473	*
474	* Called by EMR3Reset and hot plugging.
475	*
476	* @param pVCpu The cross context virtual CPU structure.
477	*/
478	VMMR3_INT_DECL(void) EMR3ResetCpu(PVMCPU pVCpu)
479	{
480	pVCpu->em.s.fForceRAW = false;
481
482	/* VMR3ResetFF may return VINF_EM_RESET or VINF_EM_SUSPEND, so transition
483	out of the HALTED state here so that enmPrevState doesn't end up as
484	HALTED when EMR3Execute returns. */
485	if (pVCpu->em.s.enmState == EMSTATE_HALTED)
486	{
487	Log(("EMR3ResetCpu: Cpu#%u %s -> %s\n", pVCpu->idCpu, emR3GetStateName(pVCpu->em.s.enmState), pVCpu->idCpu == 0 ? "EMSTATE_NONE" : "EMSTATE_WAIT_SIPI"));
488	pVCpu->em.s.enmState = pVCpu->idCpu == 0 ? EMSTATE_NONE : EMSTATE_WAIT_SIPI;
489	}
490	}
491
492
493	/**
494	* Reset notification.
495	*
496	* @param pVM The cross context VM structure.
497	*/
498	VMMR3_INT_DECL(void) EMR3Reset(PVM pVM)
499	{
500	Log(("EMR3Reset: \n"));
501	for (VMCPUID i = 0; i < pVM->cCpus; i++)
502	EMR3ResetCpu(&pVM->aCpus[i]);
503	}
504
505
506	/**
507	* Terminates the EM.
508	*
509	* Termination means cleaning up and freeing all resources,
510	* the VM it self is at this point powered off or suspended.
511	*
512	* @returns VBox status code.
513	* @param pVM The cross context VM structure.
514	*/
515	VMMR3_INT_DECL(int) EMR3Term(PVM pVM)
516	{
517	AssertMsg(pVM->em.s.offVM, ("bad init order!\n"));
518
519	#ifdef VBOX_WITH_REM
520	PDMR3CritSectDelete(&pVM->em.s.CritSectREM);
521	#endif
522	return VINF_SUCCESS;
523	}
524
525
526	/**
527	* Execute state save operation.
528	*
529	* @returns VBox status code.
530	* @param pVM The cross context VM structure.
531	* @param pSSM SSM operation handle.
532	*/
533	static DECLCALLBACK(int) emR3Save(PVM pVM, PSSMHANDLE pSSM)
534	{
535	for (VMCPUID i = 0; i < pVM->cCpus; i++)
536	{
537	PVMCPU pVCpu = &pVM->aCpus[i];
538
539	int rc = SSMR3PutBool(pSSM, pVCpu->em.s.fForceRAW);
540	AssertRCReturn(rc, rc);
541
542	Assert(pVCpu->em.s.enmState == EMSTATE_SUSPENDED);
543	Assert(pVCpu->em.s.enmPrevState != EMSTATE_SUSPENDED);
544	rc = SSMR3PutU32(pSSM, pVCpu->em.s.enmPrevState);
545	AssertRCReturn(rc, rc);
546
547	/* Save mwait state. */
548	rc = SSMR3PutU32(pSSM, pVCpu->em.s.MWait.fWait);
549	AssertRCReturn(rc, rc);
550	rc = SSMR3PutGCPtr(pSSM, pVCpu->em.s.MWait.uMWaitRAX);
551	AssertRCReturn(rc, rc);
552	rc = SSMR3PutGCPtr(pSSM, pVCpu->em.s.MWait.uMWaitRCX);
553	AssertRCReturn(rc, rc);
554	rc = SSMR3PutGCPtr(pSSM, pVCpu->em.s.MWait.uMonitorRAX);
555	AssertRCReturn(rc, rc);
556	rc = SSMR3PutGCPtr(pSSM, pVCpu->em.s.MWait.uMonitorRCX);
557	AssertRCReturn(rc, rc);
558	rc = SSMR3PutGCPtr(pSSM, pVCpu->em.s.MWait.uMonitorRDX);
559	AssertRCReturn(rc, rc);
560	}
561	return VINF_SUCCESS;
562	}
563
564
565	/**
566	* Execute state load operation.
567	*
568	* @returns VBox status code.
569	* @param pVM The cross context VM structure.
570	* @param pSSM SSM operation handle.
571	* @param uVersion Data layout version.
572	* @param uPass The data pass.
573	*/
574	static DECLCALLBACK(int) emR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
575	{
576	/*
577	* Validate version.
578	*/
579	if ( uVersion > EM_SAVED_STATE_VERSION
580	\|\| uVersion < EM_SAVED_STATE_VERSION_PRE_SMP)
581	{
582	AssertMsgFailed(("emR3Load: Invalid version uVersion=%d (current %d)!\n", uVersion, EM_SAVED_STATE_VERSION));
583	return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
584	}
585	Assert(uPass == SSM_PASS_FINAL); NOREF(uPass);
586
587	/*
588	* Load the saved state.
589	*/
590	for (VMCPUID i = 0; i < pVM->cCpus; i++)
591	{
592	PVMCPU pVCpu = &pVM->aCpus[i];
593
594	int rc = SSMR3GetBool(pSSM, &pVCpu->em.s.fForceRAW);
595	if (RT_FAILURE(rc))
596	pVCpu->em.s.fForceRAW = false;
597	AssertRCReturn(rc, rc);
598
599	if (uVersion > EM_SAVED_STATE_VERSION_PRE_SMP)
600	{
601	AssertCompile(sizeof(pVCpu->em.s.enmPrevState) == sizeof(uint32_t));
602	rc = SSMR3GetU32(pSSM, (uint32_t *)&pVCpu->em.s.enmPrevState);
603	AssertRCReturn(rc, rc);
604	Assert(pVCpu->em.s.enmPrevState != EMSTATE_SUSPENDED);
605
606	pVCpu->em.s.enmState = EMSTATE_SUSPENDED;
607	}
608	if (uVersion > EM_SAVED_STATE_VERSION_PRE_MWAIT)
609	{
610	/* Load mwait state. */
611	rc = SSMR3GetU32(pSSM, &pVCpu->em.s.MWait.fWait);
612	AssertRCReturn(rc, rc);
613	rc = SSMR3GetGCPtr(pSSM, &pVCpu->em.s.MWait.uMWaitRAX);
614	AssertRCReturn(rc, rc);
615	rc = SSMR3GetGCPtr(pSSM, &pVCpu->em.s.MWait.uMWaitRCX);
616	AssertRCReturn(rc, rc);
617	rc = SSMR3GetGCPtr(pSSM, &pVCpu->em.s.MWait.uMonitorRAX);
618	AssertRCReturn(rc, rc);
619	rc = SSMR3GetGCPtr(pSSM, &pVCpu->em.s.MWait.uMonitorRCX);
620	AssertRCReturn(rc, rc);
621	rc = SSMR3GetGCPtr(pSSM, &pVCpu->em.s.MWait.uMonitorRDX);
622	AssertRCReturn(rc, rc);
623	}
624
625	Assert(!pVCpu->em.s.pCliStatTree);
626	}
627	return VINF_SUCCESS;
628	}
629
630
631	/**
632	* Argument packet for emR3SetExecutionPolicy.
633	*/
634	struct EMR3SETEXECPOLICYARGS
635	{
636	EMEXECPOLICY enmPolicy;
637	bool fEnforce;
638	};
639
640
641	/**
642	* @callback_method_impl{FNVMMEMTRENDEZVOUS, Rendezvous callback for EMR3SetExecutionPolicy.}
643	*/
644	static DECLCALLBACK(VBOXSTRICTRC) emR3SetExecutionPolicy(PVM pVM, PVMCPU pVCpu, void *pvUser)
645	{
646	/*
647	* Only the first CPU changes the variables.
648	*/
649	if (pVCpu->idCpu == 0)
650	{
651	struct EMR3SETEXECPOLICYARGS pArgs = (struct EMR3SETEXECPOLICYARGS )pvUser;
652	switch (pArgs->enmPolicy)
653	{
654	case EMEXECPOLICY_RECOMPILE_RING0:
655	pVM->fRecompileSupervisor = pArgs->fEnforce;
656	break;
657	case EMEXECPOLICY_RECOMPILE_RING3:
658	pVM->fRecompileUser = pArgs->fEnforce;
659	break;
660	case EMEXECPOLICY_IEM_ALL:
661	pVM->em.s.fIemExecutesAll = pArgs->fEnforce;
662	break;
663	default:
664	AssertFailedReturn(VERR_INVALID_PARAMETER);
665	}
666	Log(("emR3SetExecutionPolicy: fRecompileUser=%RTbool fRecompileSupervisor=%RTbool fIemExecutesAll=%RTbool\n",
667	pVM->fRecompileUser, pVM->fRecompileSupervisor, pVM->em.s.fIemExecutesAll));
668	}
669
670	/*
671	* Force rescheduling if in RAW, HM, IEM, or REM.
672	*/
673	return pVCpu->em.s.enmState == EMSTATE_RAW
674	\|\| pVCpu->em.s.enmState == EMSTATE_HM
675	\|\| pVCpu->em.s.enmState == EMSTATE_IEM
676	\|\| pVCpu->em.s.enmState == EMSTATE_REM
677	\|\| pVCpu->em.s.enmState == EMSTATE_IEM_THEN_REM
678	? VINF_EM_RESCHEDULE
679	: VINF_SUCCESS;
680	}
681
682
683	/**
684	* Changes an execution scheduling policy parameter.
685	*
686	* This is used to enable or disable raw-mode / hardware-virtualization
687	* execution of user and supervisor code.
688	*
689	* @returns VINF_SUCCESS on success.
690	* @returns VINF_RESCHEDULE if a rescheduling might be required.
691	* @returns VERR_INVALID_PARAMETER on an invalid enmMode value.
692	*
693	* @param pUVM The user mode VM handle.
694	* @param enmPolicy The scheduling policy to change.
695	* @param fEnforce Whether to enforce the policy or not.
696	*/
697	VMMR3DECL(int) EMR3SetExecutionPolicy(PUVM pUVM, EMEXECPOLICY enmPolicy, bool fEnforce)
698	{
699	UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
700	VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
701	AssertReturn(enmPolicy > EMEXECPOLICY_INVALID && enmPolicy < EMEXECPOLICY_END, VERR_INVALID_PARAMETER);
702
703	struct EMR3SETEXECPOLICYARGS Args = { enmPolicy, fEnforce };
704	return VMMR3EmtRendezvous(pUVM->pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING, emR3SetExecutionPolicy, &Args);
705	}
706
707
708	/**
709	* Queries an execution scheduling policy parameter.
710	*
711	* @returns VBox status code
712	* @param pUVM The user mode VM handle.
713	* @param enmPolicy The scheduling policy to query.
714	* @param pfEnforced Where to return the current value.
715	*/
716	VMMR3DECL(int) EMR3QueryExecutionPolicy(PUVM pUVM, EMEXECPOLICY enmPolicy, bool *pfEnforced)
717	{
718	AssertReturn(enmPolicy > EMEXECPOLICY_INVALID && enmPolicy < EMEXECPOLICY_END, VERR_INVALID_PARAMETER);
719	AssertPtrReturn(pfEnforced, VERR_INVALID_POINTER);
720	UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
721	PVM pVM = pUVM->pVM;
722	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
723
724	/* No need to bother EMTs with a query. */
725	switch (enmPolicy)
726	{
727	case EMEXECPOLICY_RECOMPILE_RING0:
728	*pfEnforced = pVM->fRecompileSupervisor;
729	break;
730	case EMEXECPOLICY_RECOMPILE_RING3:
731	*pfEnforced = pVM->fRecompileUser;
732	break;
733	case EMEXECPOLICY_IEM_ALL:
734	*pfEnforced = pVM->em.s.fIemExecutesAll;
735	break;
736	default:
737	AssertFailedReturn(VERR_INTERNAL_ERROR_2);
738	}
739
740	return VINF_SUCCESS;
741	}
742
743
744	/**
745	* Raise a fatal error.
746	*
747	* Safely terminate the VM with full state report and stuff. This function
748	* will naturally never return.
749	*
750	* @param pVCpu The cross context virtual CPU structure.
751	* @param rc VBox status code.
752	*/
753	VMMR3DECL(void) EMR3FatalError(PVMCPU pVCpu, int rc)
754	{
755	pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
756	longjmp(pVCpu->em.s.u.FatalLongJump, rc);
757	AssertReleaseMsgFailed(("longjmp returned!\n"));
758	}
759
760
761	#if defined(LOG_ENABLED) \|\| defined(VBOX_STRICT)
762	/**
763	* Gets the EM state name.
764	*
765	* @returns pointer to read only state name,
766	* @param enmState The state.
767	*/
768	static const char *emR3GetStateName(EMSTATE enmState)
769	{
770	switch (enmState)
771	{
772	case EMSTATE_NONE: return "EMSTATE_NONE";
773	case EMSTATE_RAW: return "EMSTATE_RAW";
774	case EMSTATE_HM: return "EMSTATE_HM";
775	case EMSTATE_IEM: return "EMSTATE_IEM";
776	case EMSTATE_REM: return "EMSTATE_REM";
777	case EMSTATE_HALTED: return "EMSTATE_HALTED";
778	case EMSTATE_WAIT_SIPI: return "EMSTATE_WAIT_SIPI";
779	case EMSTATE_SUSPENDED: return "EMSTATE_SUSPENDED";
780	case EMSTATE_TERMINATING: return "EMSTATE_TERMINATING";
781	case EMSTATE_DEBUG_GUEST_RAW: return "EMSTATE_DEBUG_GUEST_RAW";
782	case EMSTATE_DEBUG_GUEST_HM: return "EMSTATE_DEBUG_GUEST_HM";
783	case EMSTATE_DEBUG_GUEST_IEM: return "EMSTATE_DEBUG_GUEST_IEM";
784	case EMSTATE_DEBUG_GUEST_REM: return "EMSTATE_DEBUG_GUEST_REM";
785	case EMSTATE_DEBUG_HYPER: return "EMSTATE_DEBUG_HYPER";
786	case EMSTATE_GURU_MEDITATION: return "EMSTATE_GURU_MEDITATION";
787	case EMSTATE_IEM_THEN_REM: return "EMSTATE_IEM_THEN_REM";
788	default: return "Unknown!";
789	}
790	}
791	#endif /* LOG_ENABLED \|\| VBOX_STRICT */
792
793
794	/**
795	* Debug loop.
796	*
797	* @returns VBox status code for EM.
798	* @param pVM The cross context VM structure.
799	* @param pVCpu The cross context virtual CPU structure.
800	* @param rc Current EM VBox status code.
801	*/
802	static VBOXSTRICTRC emR3Debug(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc)
803	{
804	for (;;)
805	{
806	Log(("emR3Debug: rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
807	const VBOXSTRICTRC rcLast = rc;
808
809	/*
810	* Debug related RC.
811	*/
812	switch (VBOXSTRICTRC_VAL(rc))
813	{
814	/*
815	* Single step an instruction.
816	*/
817	case VINF_EM_DBG_STEP:
818	if ( pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_RAW
819	\|\| pVCpu->em.s.enmState == EMSTATE_DEBUG_HYPER
820	\|\| pVCpu->em.s.fForceRAW /* paranoia */)
821	#ifdef VBOX_WITH_RAW_MODE
822	rc = emR3RawStep(pVM, pVCpu);
823	#else
824	AssertLogRelMsgFailedStmt(("Bad EM state."), VERR_EM_INTERNAL_ERROR);
825	#endif
826	else if (pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_HM)
827	rc = EMR3HmSingleInstruction(pVM, pVCpu, 0 /fFlags/);
828	#ifdef VBOX_WITH_REM
829	else if (pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_REM)
830	rc = emR3RemStep(pVM, pVCpu);
831	#endif
832	else
833	{
834	rc = IEMExecOne(pVCpu); /** @todo add dedicated interface... */
835	if (rc == VINF_SUCCESS \|\| rc == VINF_EM_RESCHEDULE)
836	rc = VINF_EM_DBG_STEPPED;
837	}
838	break;
839
840	/*
841	* Simple events: stepped, breakpoint, stop/assertion.
842	*/
843	case VINF_EM_DBG_STEPPED:
844	rc = DBGFR3Event(pVM, DBGFEVENT_STEPPED);
845	break;
846
847	case VINF_EM_DBG_BREAKPOINT:
848	rc = DBGFR3EventBreakpoint(pVM, DBGFEVENT_BREAKPOINT);
849	break;
850
851	case VINF_EM_DBG_STOP:
852	rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, NULL, 0, NULL, NULL);
853	break;
854
855	case VINF_EM_DBG_EVENT:
856	rc = DBGFR3EventHandlePending(pVM, pVCpu);
857	break;
858
859	case VINF_EM_DBG_HYPER_STEPPED:
860	rc = DBGFR3Event(pVM, DBGFEVENT_STEPPED_HYPER);
861	break;
862
863	case VINF_EM_DBG_HYPER_BREAKPOINT:
864	rc = DBGFR3EventBreakpoint(pVM, DBGFEVENT_BREAKPOINT_HYPER);
865	break;
866
867	case VINF_EM_DBG_HYPER_ASSERTION:
868	RTPrintf("\nVINF_EM_DBG_HYPER_ASSERTION:\n%s%s\n", VMMR3GetRZAssertMsg1(pVM), VMMR3GetRZAssertMsg2(pVM));
869	RTLogFlush(NULL);
870	rc = DBGFR3EventAssertion(pVM, DBGFEVENT_ASSERTION_HYPER, VMMR3GetRZAssertMsg1(pVM), VMMR3GetRZAssertMsg2(pVM));
871	break;
872
873	/*
874	* Guru meditation.
875	*/
876	case VERR_VMM_RING0_ASSERTION: /** @todo Make a guru meditation event! */
877	rc = DBGFR3EventSrc(pVM, DBGFEVENT_FATAL_ERROR, "VERR_VMM_RING0_ASSERTION", 0, NULL, NULL);
878	break;
879	case VERR_REM_TOO_MANY_TRAPS: /** @todo Make a guru meditation event! */
880	rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, "VERR_REM_TOO_MANY_TRAPS", 0, NULL, NULL);
881	break;
882	case VINF_EM_TRIPLE_FAULT: /** @todo Make a guru meditation event! */
883	rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, "VINF_EM_TRIPLE_FAULT", 0, NULL, NULL);
884	break;
885
886	default: /** @todo don't use default for guru, but make special errors code! */
887	{
888	LogRel(("emR3Debug: rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
889	rc = DBGFR3Event(pVM, DBGFEVENT_FATAL_ERROR);
890	break;
891	}
892	}
893
894	/*
895	* Process the result.
896	*/
897	do
898	{
899	switch (VBOXSTRICTRC_VAL(rc))
900	{
901	/*
902	* Continue the debugging loop.
903	*/
904	case VINF_EM_DBG_STEP:
905	case VINF_EM_DBG_STOP:
906	case VINF_EM_DBG_EVENT:
907	case VINF_EM_DBG_STEPPED:
908	case VINF_EM_DBG_BREAKPOINT:
909	case VINF_EM_DBG_HYPER_STEPPED:
910	case VINF_EM_DBG_HYPER_BREAKPOINT:
911	case VINF_EM_DBG_HYPER_ASSERTION:
912	break;
913
914	/*
915	* Resuming execution (in some form) has to be done here if we got
916	* a hypervisor debug event.
917	*/
918	case VINF_SUCCESS:
919	case VINF_EM_RESUME:
920	case VINF_EM_SUSPEND:
921	case VINF_EM_RESCHEDULE:
922	case VINF_EM_RESCHEDULE_RAW:
923	case VINF_EM_RESCHEDULE_REM:
924	case VINF_EM_HALT:
925	if (pVCpu->em.s.enmState == EMSTATE_DEBUG_HYPER)
926	{
927	#ifdef VBOX_WITH_RAW_MODE
928	rc = emR3RawResumeHyper(pVM, pVCpu);
929	if (rc != VINF_SUCCESS && RT_SUCCESS(rc))
930	continue;
931	#else
932	AssertLogRelMsgFailedReturn(("Not implemented\n"), VERR_EM_INTERNAL_ERROR);
933	#endif
934	}
935	if (rc == VINF_SUCCESS)
936	rc = VINF_EM_RESCHEDULE;
937	return rc;
938
939	/*
940	* The debugger isn't attached.
941	* We'll simply turn the thing off since that's the easiest thing to do.
942	*/
943	case VERR_DBGF_NOT_ATTACHED:
944	switch (VBOXSTRICTRC_VAL(rcLast))
945	{
946	case VINF_EM_DBG_HYPER_STEPPED:
947	case VINF_EM_DBG_HYPER_BREAKPOINT:
948	case VINF_EM_DBG_HYPER_ASSERTION:
949	case VERR_TRPM_PANIC:
950	case VERR_TRPM_DONT_PANIC:
951	case VERR_VMM_RING0_ASSERTION:
952	case VERR_VMM_HYPER_CR3_MISMATCH:
953	case VERR_VMM_RING3_CALL_DISABLED:
954	return rcLast;
955	}
956	return VINF_EM_OFF;
957
958	/*
959	* Status codes terminating the VM in one or another sense.
960	*/
961	case VINF_EM_TERMINATE:
962	case VINF_EM_OFF:
963	case VINF_EM_RESET:
964	case VINF_EM_NO_MEMORY:
965	case VINF_EM_RAW_STALE_SELECTOR:
966	case VINF_EM_RAW_IRET_TRAP:
967	case VERR_TRPM_PANIC:
968	case VERR_TRPM_DONT_PANIC:
969	case VERR_IEM_INSTR_NOT_IMPLEMENTED:
970	case VERR_IEM_ASPECT_NOT_IMPLEMENTED:
971	case VERR_VMM_RING0_ASSERTION:
972	case VERR_VMM_HYPER_CR3_MISMATCH:
973	case VERR_VMM_RING3_CALL_DISABLED:
974	case VERR_INTERNAL_ERROR:
975	case VERR_INTERNAL_ERROR_2:
976	case VERR_INTERNAL_ERROR_3:
977	case VERR_INTERNAL_ERROR_4:
978	case VERR_INTERNAL_ERROR_5:
979	case VERR_IPE_UNEXPECTED_STATUS:
980	case VERR_IPE_UNEXPECTED_INFO_STATUS:
981	case VERR_IPE_UNEXPECTED_ERROR_STATUS:
982	return rc;
983
984	/*
985	* The rest is unexpected, and will keep us here.
986	*/
987	default:
988	AssertMsgFailed(("Unexpected rc %Rrc!\n", VBOXSTRICTRC_VAL(rc)));
989	break;
990	}
991	} while (false);
992	} /* debug for ever */
993	}
994
995
996	/**
997	* Steps recompiled code.
998	*
999	* @returns VBox status code. The most important ones are: VINF_EM_STEP_EVENT,
1000	* VINF_EM_RESCHEDULE, VINF_EM_SUSPEND, VINF_EM_RESET and VINF_EM_TERMINATE.
1001	*
1002	* @param pVM The cross context VM structure.
1003	* @param pVCpu The cross context virtual CPU structure.
1004	*/
1005	static int emR3RemStep(PVM pVM, PVMCPU pVCpu)
1006	{
1007	Log3(("emR3RemStep: cs:eip=%04x:%08x\n", CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
1008
1009	#ifdef VBOX_WITH_REM
1010	EMRemLock(pVM);
1011
1012	/*
1013	* Switch to REM, step instruction, switch back.
1014	*/
1015	int rc = REMR3State(pVM, pVCpu);
1016	if (RT_SUCCESS(rc))
1017	{
1018	rc = REMR3Step(pVM, pVCpu);
1019	REMR3StateBack(pVM, pVCpu);
1020	}
1021	EMRemUnlock(pVM);
1022
1023	#else
1024	int rc = VBOXSTRICTRC_TODO(IEMExecOne(pVCpu)); NOREF(pVM);
1025	#endif
1026
1027	Log3(("emR3RemStep: returns %Rrc cs:eip=%04x:%08x\n", rc, CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
1028	return rc;
1029	}
1030
1031
1032	/**
1033	* emR3RemExecute helper that syncs the state back from REM and leave the REM
1034	* critical section.
1035	*
1036	* @returns false - new fInREMState value.
1037	* @param pVM The cross context VM structure.
1038	* @param pVCpu The cross context virtual CPU structure.
1039	*/
1040	DECLINLINE(bool) emR3RemExecuteSyncBack(PVM pVM, PVMCPU pVCpu)
1041	{
1042	#ifdef VBOX_WITH_REM
1043	STAM_PROFILE_START(&pVCpu->em.s.StatREMSync, a);
1044	REMR3StateBack(pVM, pVCpu);
1045	STAM_PROFILE_STOP(&pVCpu->em.s.StatREMSync, a);
1046
1047	EMRemUnlock(pVM);
1048	#endif
1049	return false;
1050	}
1051
1052
1053	/**
1054	* Executes recompiled code.
1055	*
1056	* This function contains the recompiler version of the inner
1057	* execution loop (the outer loop being in EMR3ExecuteVM()).
1058	*
1059	* @returns VBox status code. The most important ones are: VINF_EM_RESCHEDULE,
1060	* VINF_EM_SUSPEND, VINF_EM_RESET and VINF_EM_TERMINATE.
1061	*
1062	* @param pVM The cross context VM structure.
1063	* @param pVCpu The cross context virtual CPU structure.
1064	* @param pfFFDone Where to store an indicator telling whether or not
1065	* FFs were done before returning.
1066	*
1067	*/
1068	static int emR3RemExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone)
1069	{
1070	#ifdef LOG_ENABLED
1071	PCPUMCTX pCtx = pVCpu->em.s.pCtx;
1072	uint32_t cpl = CPUMGetGuestCPL(pVCpu);
1073
1074	if (pCtx->eflags.Bits.u1VM)
1075	Log(("EMV86: %04X:%08X IF=%d\n", pCtx->cs.Sel, pCtx->eip, pCtx->eflags.Bits.u1IF));
1076	else
1077	Log(("EMR%d: %04X:%08X ESP=%08X IF=%d CR0=%x eflags=%x\n", cpl, pCtx->cs.Sel, pCtx->eip, pCtx->esp, pCtx->eflags.Bits.u1IF, (uint32_t)pCtx->cr0, pCtx->eflags.u));
1078	#endif
1079	STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatREMTotal, a);
1080
1081	#if defined(VBOX_STRICT) && defined(DEBUG_bird)
1082	AssertMsg( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
1083	\|\| !MMHyperIsInsideArea(pVM, CPUMGetGuestEIP(pVCpu)), /** @todo @bugref{1419} - get flat address. */
1084	("cs:eip=%RX16:%RX32\n", CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
1085	#endif
1086
1087	/*
1088	* Spin till we get a forced action which returns anything but VINF_SUCCESS
1089	* or the REM suggests raw-mode execution.
1090	*/
1091	*pfFFDone = false;
1092	#ifdef VBOX_WITH_REM
1093	bool fInREMState = false;
1094	#else
1095	uint32_t cLoops = 0;
1096	#endif
1097	int rc = VINF_SUCCESS;
1098	for (;;)
1099	{
1100	#ifdef VBOX_WITH_REM
1101	/*
1102	* Lock REM and update the state if not already in sync.
1103	*
1104	* Note! Big lock, but you are not supposed to own any lock when
1105	* coming in here.
1106	*/
1107	if (!fInREMState)
1108	{
1109	EMRemLock(pVM);
1110	STAM_PROFILE_START(&pVCpu->em.s.StatREMSync, b);
1111
1112	/* Flush the recompiler translation blocks if the VCPU has changed,
1113	also force a full CPU state resync. */
1114	if (pVM->em.s.idLastRemCpu != pVCpu->idCpu)
1115	{
1116	REMFlushTBs(pVM);
1117	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_ALL);
1118	}
1119	pVM->em.s.idLastRemCpu = pVCpu->idCpu;
1120
1121	rc = REMR3State(pVM, pVCpu);
1122
1123	STAM_PROFILE_STOP(&pVCpu->em.s.StatREMSync, b);
1124	if (RT_FAILURE(rc))
1125	break;
1126	fInREMState = true;
1127
1128	/*
1129	* We might have missed the raising of VMREQ, TIMER and some other
1130	* important FFs while we were busy switching the state. So, check again.
1131	*/
1132	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST \| VM_FF_PDM_QUEUES \| VM_FF_DBGF \| VM_FF_CHECK_VM_STATE \| VM_FF_RESET)
1133	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TIMER \| VMCPU_FF_REQUEST))
1134	{
1135	LogFlow(("emR3RemExecute: Skipping run, because FF is set. %#x\n", pVM->fGlobalForcedActions));
1136	goto l_REMDoForcedActions;
1137	}
1138	}
1139	#endif
1140
1141	/*
1142	* Execute REM.
1143	*/
1144	if (RT_LIKELY(emR3IsExecutionAllowed(pVM, pVCpu)))
1145	{
1146	STAM_PROFILE_START(&pVCpu->em.s.StatREMExec, c);
1147	#ifdef VBOX_WITH_REM
1148	rc = REMR3Run(pVM, pVCpu);
1149	#else
1150	rc = VBOXSTRICTRC_TODO(IEMExecLots(pVCpu));
1151	#endif
1152	STAM_PROFILE_STOP(&pVCpu->em.s.StatREMExec, c);
1153	}
1154	else
1155	{
1156	/* Give up this time slice; virtual time continues */
1157	STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatCapped, u);
1158	RTThreadSleep(5);
1159	STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatCapped, u);
1160	rc = VINF_SUCCESS;
1161	}
1162
1163	/*
1164	* Deal with high priority post execution FFs before doing anything
1165	* else. Sync back the state and leave the lock to be on the safe side.
1166	*/
1167	if ( VM_FF_IS_PENDING(pVM, VM_FF_HIGH_PRIORITY_POST_MASK)
1168	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_POST_MASK))
1169	{
1170	#ifdef VBOX_WITH_REM
1171	fInREMState = emR3RemExecuteSyncBack(pVM, pVCpu);
1172	#endif
1173	rc = emR3HighPriorityPostForcedActions(pVM, pVCpu, rc);
1174	}
1175
1176	/*
1177	* Process the returned status code.
1178	*/
1179	if (rc != VINF_SUCCESS)
1180	{
1181	if (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST)
1182	break;
1183	if (rc != VINF_REM_INTERRUPED_FF)
1184	{
1185	/*
1186	* Anything which is not known to us means an internal error
1187	* and the termination of the VM!
1188	*/
1189	AssertMsg(rc == VERR_REM_TOO_MANY_TRAPS, ("Unknown GC return code: %Rra\n", rc));
1190	break;
1191	}
1192	}
1193
1194
1195	/*
1196	* Check and execute forced actions.
1197	*
1198	* Sync back the VM state and leave the lock before calling any of
1199	* these, you never know what's going to happen here.
1200	*/
1201	#ifdef VBOX_HIGH_RES_TIMERS_HACK
1202	TMTimerPollVoid(pVM, pVCpu);
1203	#endif
1204	AssertCompile(VMCPU_FF_ALL_REM_MASK & VMCPU_FF_TIMER);
1205	if ( VM_FF_IS_PENDING(pVM, VM_FF_ALL_REM_MASK)
1206	\|\| VMCPU_FF_IS_PENDING(pVCpu,
1207	VMCPU_FF_ALL_REM_MASK
1208	& VM_WHEN_RAW_MODE(~(VMCPU_FF_CSAM_PENDING_ACTION \| VMCPU_FF_CSAM_SCAN_PAGE), UINT32_MAX)) )
1209	{
1210	#ifdef VBOX_WITH_REM
1211	l_REMDoForcedActions:
1212	if (fInREMState)
1213	fInREMState = emR3RemExecuteSyncBack(pVM, pVCpu);
1214	#endif
1215	STAM_REL_PROFILE_ADV_SUSPEND(&pVCpu->em.s.StatREMTotal, a);
1216	rc = emR3ForcedActions(pVM, pVCpu, rc);
1217	VBOXVMM_EM_FF_ALL_RET(pVCpu, rc);
1218	STAM_REL_PROFILE_ADV_RESUME(&pVCpu->em.s.StatREMTotal, a);
1219	if ( rc != VINF_SUCCESS
1220	&& rc != VINF_EM_RESCHEDULE_REM)
1221	{
1222	*pfFFDone = true;
1223	break;
1224	}
1225	}
1226
1227	#ifndef VBOX_WITH_REM
1228	/*
1229	* Have to check if we can get back to fast execution mode every so often.
1230	*/
1231	if (!(++cLoops & 7))
1232	{
1233	EMSTATE enmCheck = emR3Reschedule(pVM, pVCpu, pVCpu->em.s.pCtx);
1234	if ( enmCheck != EMSTATE_REM
1235	&& enmCheck != EMSTATE_IEM_THEN_REM)
1236	return VINF_EM_RESCHEDULE;
1237	}
1238	#endif
1239
1240	} /* The Inner Loop, recompiled execution mode version. */
1241
1242
1243	#ifdef VBOX_WITH_REM
1244	/*
1245	* Returning. Sync back the VM state if required.
1246	*/
1247	if (fInREMState)
1248	fInREMState = emR3RemExecuteSyncBack(pVM, pVCpu);
1249	#endif
1250
1251	STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatREMTotal, a);
1252	return rc;
1253	}
1254
1255
1256	#ifdef DEBUG
1257
1258	int emR3SingleStepExecRem(PVM pVM, PVMCPU pVCpu, uint32_t cIterations)
1259	{
1260	EMSTATE enmOldState = pVCpu->em.s.enmState;
1261
1262	pVCpu->em.s.enmState = EMSTATE_DEBUG_GUEST_REM;
1263
1264	Log(("Single step BEGIN:\n"));
1265	for (uint32_t i = 0; i < cIterations; i++)
1266	{
1267	DBGFR3PrgStep(pVCpu);
1268	DBGFR3_DISAS_INSTR_CUR_LOG(pVCpu, "RSS");
1269	emR3RemStep(pVM, pVCpu);
1270	if (emR3Reschedule(pVM, pVCpu, pVCpu->em.s.pCtx) != EMSTATE_REM)
1271	break;
1272	}
1273	Log(("Single step END:\n"));
1274	CPUMSetGuestEFlags(pVCpu, CPUMGetGuestEFlags(pVCpu) & ~X86_EFL_TF);
1275	pVCpu->em.s.enmState = enmOldState;
1276	return VINF_EM_RESCHEDULE;
1277	}
1278
1279	#endif /* DEBUG */
1280
1281
1282	/**
1283	* Try execute the problematic code in IEM first, then fall back on REM if there
1284	* is too much of it or if IEM doesn't implement something.
1285	*
1286	* @returns Strict VBox status code from IEMExecLots.
1287	* @param pVM The cross context VM structure.
1288	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1289	* @param pfFFDone Force flags done indicator.
1290	*
1291	* @thread EMT(pVCpu)
1292	*/
1293	static VBOXSTRICTRC emR3ExecuteIemThenRem(PVM pVM, PVMCPU pVCpu, bool *pfFFDone)
1294	{
1295	LogFlow(("emR3ExecuteIemThenRem: %04x:%RGv\n", CPUMGetGuestCS(pVCpu), CPUMGetGuestRIP(pVCpu)));
1296	*pfFFDone = false;
1297
1298	/*
1299	* Execute in IEM for a while.
1300	*/
1301	while (pVCpu->em.s.cIemThenRemInstructions < 1024)
1302	{
1303	VBOXSTRICTRC rcStrict = IEMExecLots(pVCpu);
1304	if (rcStrict != VINF_SUCCESS)
1305	{
1306	if ( rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED
1307	\|\| rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
1308	break;
1309
1310	pVCpu->em.s.cIemThenRemInstructions++;
1311	Log(("emR3ExecuteIemThenRem: returns %Rrc after %u instructions\n",
1312	VBOXSTRICTRC_VAL(rcStrict), pVCpu->em.s.cIemThenRemInstructions));
1313	return rcStrict;
1314	}
1315	pVCpu->em.s.cIemThenRemInstructions++;
1316
1317	EMSTATE enmNewState = emR3Reschedule(pVM, pVCpu, pVCpu->em.s.pCtx);
1318	if (enmNewState != EMSTATE_REM && enmNewState != EMSTATE_IEM_THEN_REM)
1319	{
1320	LogFlow(("emR3ExecuteIemThenRem: -> %d (%s) after %u instructions\n",
1321	enmNewState, emR3GetStateName(enmNewState), pVCpu->em.s.cIemThenRemInstructions));
1322	pVCpu->em.s.enmPrevState = pVCpu->em.s.enmState;
1323	pVCpu->em.s.enmState = enmNewState;
1324	return VINF_SUCCESS;
1325	}
1326
1327	/*
1328	* Check for pending actions.
1329	*/
1330	if ( VM_FF_IS_PENDING(pVM, VM_FF_ALL_REM_MASK)
1331	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_ALL_REM_MASK))
1332	return VINF_SUCCESS;
1333	}
1334
1335	/*
1336	* Switch to REM.
1337	*/
1338	Log(("emR3ExecuteIemThenRem: -> EMSTATE_REM (after %u instructions)\n", pVCpu->em.s.cIemThenRemInstructions));
1339	pVCpu->em.s.enmState = EMSTATE_REM;
1340	return VINF_SUCCESS;
1341	}
1342
1343
1344	/**
1345	* Decides whether to execute RAW, HWACC or REM.
1346	*
1347	* @returns new EM state
1348	* @param pVM The cross context VM structure.
1349	* @param pVCpu The cross context virtual CPU structure.
1350	* @param pCtx Pointer to the guest CPU context.
1351	*/
1352	EMSTATE emR3Reschedule(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1353	{
1354	/*
1355	* When forcing raw-mode execution, things are simple.
1356	*/
1357	if (pVCpu->em.s.fForceRAW)
1358	return EMSTATE_RAW;
1359
1360	/*
1361	* We stay in the wait for SIPI state unless explicitly told otherwise.
1362	*/
1363	if (pVCpu->em.s.enmState == EMSTATE_WAIT_SIPI)
1364	return EMSTATE_WAIT_SIPI;
1365
1366	/*
1367	* Execute everything in IEM?
1368	*/
1369	if (pVM->em.s.fIemExecutesAll)
1370	return EMSTATE_IEM;
1371
1372	/* !!! THIS MUST BE IN SYNC WITH remR3CanExecuteRaw !!! */
1373	/* !!! THIS MUST BE IN SYNC WITH remR3CanExecuteRaw !!! */
1374	/* !!! THIS MUST BE IN SYNC WITH remR3CanExecuteRaw !!! */
1375
1376	X86EFLAGS EFlags = pCtx->eflags;
1377	if (HMIsEnabled(pVM))
1378	{
1379	/*
1380	* Hardware accelerated raw-mode:
1381	*/
1382	if ( EMIsHwVirtExecutionEnabled(pVM)
1383	&& HMR3CanExecuteGuest(pVM, pCtx))
1384	return EMSTATE_HM;
1385
1386	/*
1387	* Note! Raw mode and hw accelerated mode are incompatible. The latter
1388	* turns off monitoring features essential for raw mode!
1389	*/
1390	return EMSTATE_IEM_THEN_REM;
1391	}
1392
1393	/*
1394	* Standard raw-mode:
1395	*
1396	* Here we only support 16 & 32 bits protected mode ring 3 code that has no IO privileges
1397	* or 32 bits protected mode ring 0 code
1398	*
1399	* The tests are ordered by the likelihood of being true during normal execution.
1400	*/
1401	if (EFlags.u32 & (X86_EFL_TF /* \| HF_INHIBIT_IRQ_MASK*/))
1402	{
1403	Log2(("raw mode refused: EFlags=%#x\n", EFlags.u32));
1404	return EMSTATE_REM;
1405	}
1406
1407	# ifndef VBOX_RAW_V86
1408	if (EFlags.u32 & X86_EFL_VM) {
1409	Log2(("raw mode refused: VM_MASK\n"));
1410	return EMSTATE_REM;
1411	}
1412	# endif
1413
1414	/** @todo check up the X86_CR0_AM flag in respect to raw mode!!! We're probably not emulating it right! */
1415	uint32_t u32CR0 = pCtx->cr0;
1416	if ((u32CR0 & (X86_CR0_PG \| X86_CR0_PE)) != (X86_CR0_PG \| X86_CR0_PE))
1417	{
1418	//Log2(("raw mode refused: %s%s%s\n", (u32CR0 & X86_CR0_PG) ? "" : " !PG", (u32CR0 & X86_CR0_PE) ? "" : " !PE", (u32CR0 & X86_CR0_AM) ? "" : " !AM"));
1419	return EMSTATE_REM;
1420	}
1421
1422	if (pCtx->cr4 & X86_CR4_PAE)
1423	{
1424	uint32_t u32Dummy, u32Features;
1425
1426	CPUMGetGuestCpuId(pVCpu, 1, 0, &u32Dummy, &u32Dummy, &u32Dummy, &u32Features);
1427	if (!(u32Features & X86_CPUID_FEATURE_EDX_PAE))
1428	return EMSTATE_REM;
1429	}
1430
1431	unsigned uSS = pCtx->ss.Sel;
1432	if ( pCtx->eflags.Bits.u1VM
1433	\|\| (uSS & X86_SEL_RPL) == 3)
1434	{
1435	if (!EMIsRawRing3Enabled(pVM))
1436	return EMSTATE_REM;
1437
1438	if (!(EFlags.u32 & X86_EFL_IF))
1439	{
1440	Log2(("raw mode refused: IF (RawR3)\n"));
1441	return EMSTATE_REM;
1442	}
1443
1444	if (!(u32CR0 & X86_CR0_WP) && EMIsRawRing0Enabled(pVM))
1445	{
1446	Log2(("raw mode refused: CR0.WP + RawR0\n"));
1447	return EMSTATE_REM;
1448	}
1449	}
1450	else
1451	{
1452	if (!EMIsRawRing0Enabled(pVM))
1453	return EMSTATE_REM;
1454
1455	if (EMIsRawRing1Enabled(pVM))
1456	{
1457	/* Only ring 0 and 1 supervisor code. */
1458	if ((uSS & X86_SEL_RPL) == 2) /* ring 1 code is moved into ring 2, so we can't support ring-2 in that case. */
1459	{
1460	Log2(("raw r0 mode refused: CPL %d\n", uSS & X86_SEL_RPL));
1461	return EMSTATE_REM;
1462	}
1463	}
1464	/* Only ring 0 supervisor code. */
1465	else if ((uSS & X86_SEL_RPL) != 0)
1466	{
1467	Log2(("raw r0 mode refused: CPL %d\n", uSS & X86_SEL_RPL));
1468	return EMSTATE_REM;
1469	}
1470
1471	// Let's start with pure 32 bits ring 0 code first
1472	/** @todo What's pure 32-bit mode? flat? */
1473	if ( !(pCtx->ss.Attr.n.u1DefBig)
1474	\|\| !(pCtx->cs.Attr.n.u1DefBig))
1475	{
1476	Log2(("raw r0 mode refused: SS/CS not 32bit\n"));
1477	return EMSTATE_REM;
1478	}
1479
1480	/* Write protection must be turned on, or else the guest can overwrite our hypervisor code and data. */
1481	if (!(u32CR0 & X86_CR0_WP))
1482	{
1483	Log2(("raw r0 mode refused: CR0.WP=0!\n"));
1484	return EMSTATE_REM;
1485	}
1486
1487	# ifdef VBOX_WITH_RAW_MODE
1488	if (PATMShouldUseRawMode(pVM, (RTGCPTR)pCtx->eip))
1489	{
1490	Log2(("raw r0 mode forced: patch code\n"));
1491	# ifdef VBOX_WITH_SAFE_STR
1492	Assert(pCtx->tr.Sel);
1493	# endif
1494	return EMSTATE_RAW;
1495	}
1496	# endif /* VBOX_WITH_RAW_MODE */
1497
1498	# if !defined(VBOX_ALLOW_IF0) && !defined(VBOX_RUN_INTERRUPT_GATE_HANDLERS)
1499	if (!(EFlags.u32 & X86_EFL_IF))
1500	{
1501	////Log2(("R0: IF=0 VIF=%d %08X\n", eip, pVMeflags));
1502	//Log2(("RR0: Interrupts turned off; fall back to emulation\n"));
1503	return EMSTATE_REM;
1504	}
1505	# endif
1506
1507	# ifndef VBOX_WITH_RAW_RING1
1508	/** @todo still necessary??? */
1509	if (EFlags.Bits.u2IOPL != 0)
1510	{
1511	Log2(("raw r0 mode refused: IOPL %d\n", EFlags.Bits.u2IOPL));
1512	return EMSTATE_REM;
1513	}
1514	# endif
1515	}
1516
1517	/*
1518	* Stale hidden selectors means raw-mode is unsafe (being very careful).
1519	*/
1520	if (pCtx->cs.fFlags & CPUMSELREG_FLAGS_STALE)
1521	{
1522	Log2(("raw mode refused: stale CS\n"));
1523	return EMSTATE_REM;
1524	}
1525	if (pCtx->ss.fFlags & CPUMSELREG_FLAGS_STALE)
1526	{
1527	Log2(("raw mode refused: stale SS\n"));
1528	return EMSTATE_REM;
1529	}
1530	if (pCtx->ds.fFlags & CPUMSELREG_FLAGS_STALE)
1531	{
1532	Log2(("raw mode refused: stale DS\n"));
1533	return EMSTATE_REM;
1534	}
1535	if (pCtx->es.fFlags & CPUMSELREG_FLAGS_STALE)
1536	{
1537	Log2(("raw mode refused: stale ES\n"));
1538	return EMSTATE_REM;
1539	}
1540	if (pCtx->fs.fFlags & CPUMSELREG_FLAGS_STALE)
1541	{
1542	Log2(("raw mode refused: stale FS\n"));
1543	return EMSTATE_REM;
1544	}
1545	if (pCtx->gs.fFlags & CPUMSELREG_FLAGS_STALE)
1546	{
1547	Log2(("raw mode refused: stale GS\n"));
1548	return EMSTATE_REM;
1549	}
1550
1551	# ifdef VBOX_WITH_SAFE_STR
1552	if (pCtx->tr.Sel == 0)
1553	{
1554	Log(("Raw mode refused -> TR=0\n"));
1555	return EMSTATE_REM;
1556	}
1557	# endif
1558
1559	/Assert(PGMPhysIsA20Enabled(pVCpu));/
1560	return EMSTATE_RAW;
1561	}
1562
1563
1564	/**
1565	* Executes all high priority post execution force actions.
1566	*
1567	* @returns rc or a fatal status code.
1568	*
1569	* @param pVM The cross context VM structure.
1570	* @param pVCpu The cross context virtual CPU structure.
1571	* @param rc The current rc.
1572	*/
1573	int emR3HighPriorityPostForcedActions(PVM pVM, PVMCPU pVCpu, int rc)
1574	{
1575	VBOXVMM_EM_FF_HIGH(pVCpu, pVM->fGlobalForcedActions, pVCpu->fLocalForcedActions, rc);
1576
1577	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PDM_CRITSECT))
1578	PDMCritSectBothFF(pVCpu);
1579
1580	/* Update CR3 (Nested Paging case for HM). */
1581	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
1582	{
1583	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
1584	if (RT_FAILURE(rc2))
1585	return rc2;
1586	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
1587	}
1588
1589	/* Update PAE PDPEs. This must be done after PGMUpdateCR3() and used only by the Nested Paging case for HM. */
1590	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
1591	{
1592	if (CPUMIsGuestInPAEMode(pVCpu))
1593	{
1594	PX86PDPE pPdpes = HMGetPaePdpes(pVCpu);
1595	AssertPtr(pPdpes);
1596
1597	PGMGstUpdatePaePdpes(pVCpu, pPdpes);
1598	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
1599	}
1600	else
1601	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
1602	}
1603
1604	/* IEM has pending work (typically memory write after INS instruction). */
1605	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_IEM))
1606	rc = VBOXSTRICTRC_TODO(IEMR3ProcessForceFlag(pVM, pVCpu, rc));
1607
1608	/* IOM has pending work (comitting an I/O or MMIO write). */
1609	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_IOM))
1610	rc = VBOXSTRICTRC_TODO(IOMR3ProcessForceFlag(pVM, pVCpu, rc));
1611
1612	#ifdef VBOX_WITH_RAW_MODE
1613	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_CSAM_PENDING_ACTION))
1614	CSAMR3DoPendingAction(pVM, pVCpu);
1615	#endif
1616
1617	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY))
1618	{
1619	if ( rc > VINF_EM_NO_MEMORY
1620	&& rc <= VINF_EM_LAST)
1621	rc = VINF_EM_NO_MEMORY;
1622	}
1623
1624	return rc;
1625	}
1626
1627
1628	/**
1629	* Executes all pending forced actions.
1630	*
1631	* Forced actions can cause execution delays and execution
1632	* rescheduling. The first we deal with using action priority, so
1633	* that for instance pending timers aren't scheduled and ran until
1634	* right before execution. The rescheduling we deal with using
1635	* return codes. The same goes for VM termination, only in that case
1636	* we exit everything.
1637	*
1638	* @returns VBox status code of equal or greater importance/severity than rc.
1639	* The most important ones are: VINF_EM_RESCHEDULE,
1640	* VINF_EM_SUSPEND, VINF_EM_RESET and VINF_EM_TERMINATE.
1641	*
1642	* @param pVM The cross context VM structure.
1643	* @param pVCpu The cross context virtual CPU structure.
1644	* @param rc The current rc.
1645	*
1646	*/
1647	int emR3ForcedActions(PVM pVM, PVMCPU pVCpu, int rc)
1648	{
1649	STAM_REL_PROFILE_START(&pVCpu->em.s.StatForcedActions, a);
1650	#ifdef VBOX_STRICT
1651	int rcIrq = VINF_SUCCESS;
1652	#endif
1653	int rc2;
1654	#define UPDATE_RC() \
1655	do { \
1656	AssertMsg(rc2 <= 0 \|\| (rc2 >= VINF_EM_FIRST && rc2 <= VINF_EM_LAST), ("Invalid FF return code: %Rra\n", rc2)); \
1657	if (rc2 == VINF_SUCCESS \|\| rc < VINF_SUCCESS) \
1658	break; \
1659	if (!rc \|\| rc2 < rc) \
1660	rc = rc2; \
1661	} while (0)
1662	VBOXVMM_EM_FF_ALL(pVCpu, pVM->fGlobalForcedActions, pVCpu->fLocalForcedActions, rc);
1663
1664	/*
1665	* Post execution chunk first.
1666	*/
1667	if ( VM_FF_IS_PENDING(pVM, VM_FF_NORMAL_PRIORITY_POST_MASK)
1668	\|\| (VMCPU_FF_NORMAL_PRIORITY_POST_MASK && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_NORMAL_PRIORITY_POST_MASK)) )
1669	{
1670	/*
1671	* EMT Rendezvous (must be serviced before termination).
1672	*/
1673	if (VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS))
1674	{
1675	rc2 = VMMR3EmtRendezvousFF(pVM, pVCpu);
1676	UPDATE_RC();
1677	/** @todo HACK ALERT! The following test is to make sure EM+TM
1678	* thinks the VM is stopped/reset before the next VM state change
1679	* is made. We need a better solution for this, or at least make it
1680	* possible to do: (rc >= VINF_EM_FIRST && rc <=
1681	* VINF_EM_SUSPEND). */
1682	if (RT_UNLIKELY(rc == VINF_EM_SUSPEND \|\| rc == VINF_EM_RESET \|\| rc == VINF_EM_OFF))
1683	{
1684	Log2(("emR3ForcedActions: returns %Rrc\n", rc));
1685	STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
1686	return rc;
1687	}
1688	}
1689
1690	/*
1691	* State change request (cleared by vmR3SetStateLocked).
1692	*/
1693	if (VM_FF_IS_PENDING(pVM, VM_FF_CHECK_VM_STATE))
1694	{
1695	VMSTATE enmState = VMR3GetState(pVM);
1696	switch (enmState)
1697	{
1698	case VMSTATE_FATAL_ERROR:
1699	case VMSTATE_FATAL_ERROR_LS:
1700	Log2(("emR3ForcedActions: %s -> VINF_EM_SUSPEND\n", VMGetStateName(enmState) ));
1701	STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
1702	return VINF_EM_SUSPEND;
1703
1704	case VMSTATE_DESTROYING:
1705	Log2(("emR3ForcedActions: %s -> VINF_EM_TERMINATE\n", VMGetStateName(enmState) ));
1706	STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
1707	return VINF_EM_TERMINATE;
1708
1709	default:
1710	AssertMsgFailed(("%s\n", VMGetStateName(enmState)));
1711	}
1712	}
1713
1714	/*
1715	* Debugger Facility polling.
1716	*/
1717	if (VM_FF_IS_PENDING(pVM, VM_FF_DBGF))
1718	{
1719	rc2 = DBGFR3VMMForcedAction(pVM);
1720	UPDATE_RC();
1721	}
1722
1723	/*
1724	* Postponed reset request.
1725	*/
1726	if (VM_FF_TEST_AND_CLEAR(pVM, VM_FF_RESET))
1727	{
1728	rc2 = VBOXSTRICTRC_TODO(VMR3ResetFF(pVM));
1729	UPDATE_RC();
1730	}
1731
1732	#ifdef VBOX_WITH_RAW_MODE
1733	/*
1734	* CSAM page scanning.
1735	*/
1736	if ( !VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)
1737	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_CSAM_SCAN_PAGE))
1738	{
1739	PCPUMCTX pCtx = pVCpu->em.s.pCtx;
1740
1741	/** @todo: check for 16 or 32 bits code! (D bit in the code selector) */
1742	Log(("Forced action VMCPU_FF_CSAM_SCAN_PAGE\n"));
1743
1744	CSAMR3CheckCodeEx(pVM, pCtx, pCtx->eip);
1745	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_CSAM_SCAN_PAGE);
1746	}
1747	#endif
1748
1749	/*
1750	* Out of memory? Putting this after CSAM as it may in theory cause us to run out of memory.
1751	*/
1752	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY))
1753	{
1754	rc2 = PGMR3PhysAllocateHandyPages(pVM);
1755	UPDATE_RC();
1756	if (rc == VINF_EM_NO_MEMORY)
1757	return rc;
1758	}
1759
1760	/* check that we got them all */
1761	AssertCompile(VM_FF_NORMAL_PRIORITY_POST_MASK == (VM_FF_CHECK_VM_STATE \| VM_FF_DBGF \| VM_FF_RESET \| VM_FF_PGM_NO_MEMORY \| VM_FF_EMT_RENDEZVOUS));
1762	AssertCompile(VMCPU_FF_NORMAL_PRIORITY_POST_MASK == VM_WHEN_RAW_MODE(VMCPU_FF_CSAM_SCAN_PAGE, 0));
1763	}
1764
1765	/*
1766	* Normal priority then.
1767	* (Executed in no particular order.)
1768	*/
1769	if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_NORMAL_PRIORITY_MASK, VM_FF_PGM_NO_MEMORY))
1770	{
1771	/*
1772	* PDM Queues are pending.
1773	*/
1774	if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_PDM_QUEUES, VM_FF_PGM_NO_MEMORY))
1775	PDMR3QueueFlushAll(pVM);
1776
1777	/*
1778	* PDM DMA transfers are pending.
1779	*/
1780	if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_PDM_DMA, VM_FF_PGM_NO_MEMORY))
1781	PDMR3DmaRun(pVM);
1782
1783	/*
1784	* EMT Rendezvous (make sure they are handled before the requests).
1785	*/
1786	if (VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS))
1787	{
1788	rc2 = VMMR3EmtRendezvousFF(pVM, pVCpu);
1789	UPDATE_RC();
1790	/** @todo HACK ALERT! The following test is to make sure EM+TM
1791	* thinks the VM is stopped/reset before the next VM state change
1792	* is made. We need a better solution for this, or at least make it
1793	* possible to do: (rc >= VINF_EM_FIRST && rc <=
1794	* VINF_EM_SUSPEND). */
1795	if (RT_UNLIKELY(rc == VINF_EM_SUSPEND \|\| rc == VINF_EM_RESET \|\| rc == VINF_EM_OFF))
1796	{
1797	Log2(("emR3ForcedActions: returns %Rrc\n", rc));
1798	STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
1799	return rc;
1800	}
1801	}
1802
1803	/*
1804	* Requests from other threads.
1805	*/
1806	if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_REQUEST, VM_FF_PGM_NO_MEMORY))
1807	{
1808	rc2 = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY, false /fPriorityOnly/);
1809	if (rc2 == VINF_EM_OFF \|\| rc2 == VINF_EM_TERMINATE) /** @todo this shouldn't be necessary */
1810	{
1811	Log2(("emR3ForcedActions: returns %Rrc\n", rc2));
1812	STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
1813	return rc2;
1814	}
1815	UPDATE_RC();
1816	/** @todo HACK ALERT! The following test is to make sure EM+TM
1817	* thinks the VM is stopped/reset before the next VM state change
1818	* is made. We need a better solution for this, or at least make it
1819	* possible to do: (rc >= VINF_EM_FIRST && rc <=
1820	* VINF_EM_SUSPEND). */
1821	if (RT_UNLIKELY(rc == VINF_EM_SUSPEND \|\| rc == VINF_EM_RESET \|\| rc == VINF_EM_OFF))
1822	{
1823	Log2(("emR3ForcedActions: returns %Rrc\n", rc));
1824	STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
1825	return rc;
1826	}
1827	}
1828
1829	#ifdef VBOX_WITH_REM
1830	/* Replay the handler notification changes. */
1831	if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_REM_HANDLER_NOTIFY, VM_FF_PGM_NO_MEMORY))
1832	{
1833	/* Try not to cause deadlocks. */
1834	if ( pVM->cCpus == 1
1835	\|\| ( !PGMIsLockOwner(pVM)
1836	&& !IOMIsLockWriteOwner(pVM))
1837	)
1838	{
1839	EMRemLock(pVM);
1840	REMR3ReplayHandlerNotifications(pVM);
1841	EMRemUnlock(pVM);
1842	}
1843	}
1844	#endif
1845
1846	/* check that we got them all */
1847	AssertCompile(VM_FF_NORMAL_PRIORITY_MASK == (VM_FF_REQUEST \| VM_FF_PDM_QUEUES \| VM_FF_PDM_DMA \| VM_FF_REM_HANDLER_NOTIFY \| VM_FF_EMT_RENDEZVOUS));
1848	}
1849
1850	/*
1851	* Normal priority then. (per-VCPU)
1852	* (Executed in no particular order.)
1853	*/
1854	if ( !VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)
1855	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_NORMAL_PRIORITY_MASK))
1856	{
1857	/*
1858	* Requests from other threads.
1859	*/
1860	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
1861	{
1862	rc2 = VMR3ReqProcessU(pVM->pUVM, pVCpu->idCpu, false /fPriorityOnly/);
1863	if (rc2 == VINF_EM_OFF \|\| rc2 == VINF_EM_TERMINATE \|\| rc2 == VINF_EM_RESET)
1864	{
1865	Log2(("emR3ForcedActions: returns %Rrc\n", rc2));
1866	STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
1867	return rc2;
1868	}
1869	UPDATE_RC();
1870	/** @todo HACK ALERT! The following test is to make sure EM+TM
1871	* thinks the VM is stopped/reset before the next VM state change
1872	* is made. We need a better solution for this, or at least make it
1873	* possible to do: (rc >= VINF_EM_FIRST && rc <=
1874	* VINF_EM_SUSPEND). */
1875	if (RT_UNLIKELY(rc == VINF_EM_SUSPEND \|\| rc == VINF_EM_RESET \|\| rc == VINF_EM_OFF))
1876	{
1877	Log2(("emR3ForcedActions: returns %Rrc\n", rc));
1878	STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
1879	return rc;
1880	}
1881	}
1882
1883	/*
1884	* Forced unhalting of EMT.
1885	*/
1886	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_UNHALT))
1887	{
1888	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_UNHALT);
1889	if (rc == VINF_EM_HALT)
1890	rc = VINF_EM_RESCHEDULE;
1891	else
1892	{
1893	rc2 = VINF_EM_RESCHEDULE;
1894	UPDATE_RC();
1895	}
1896	}
1897
1898	/* check that we got them all */
1899	Assert(!(VMCPU_FF_NORMAL_PRIORITY_MASK & ~(VMCPU_FF_REQUEST \| VMCPU_FF_UNHALT)));
1900	}
1901
1902	/*
1903	* High priority pre execution chunk last.
1904	* (Executed in ascending priority order.)
1905	*/
1906	if ( VM_FF_IS_PENDING(pVM, VM_FF_HIGH_PRIORITY_PRE_MASK)
1907	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_PRE_MASK))
1908	{
1909	/*
1910	* Timers before interrupts.
1911	*/
1912	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TIMER)
1913	&& !VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY))
1914	TMR3TimerQueuesDo(pVM);
1915
1916	#ifdef VBOX_WITH_NEW_APIC
1917	/*
1918	* Pick up asynchronously posted interrupts into the APIC.
1919	*/
1920	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
1921	APICUpdatePendingInterrupts(pVCpu);
1922	#endif
1923
1924	/*
1925	* The instruction following an emulated STI should always be executed!
1926	*
1927	* Note! We intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here if
1928	* the eip is the same as the inhibited instr address. Before we
1929	* are able to execute this instruction in raw mode (iret to
1930	* guest code) an external interrupt might force a world switch
1931	* again. Possibly allowing a guest interrupt to be dispatched
1932	* in the process. This could break the guest. Sounds very
1933	* unlikely, but such timing sensitive problem are not as rare as
1934	* you might think.
1935	*/
1936	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
1937	&& !VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY))
1938	{
1939	if (CPUMGetGuestRIP(pVCpu) != EMGetInhibitInterruptsPC(pVCpu))
1940	{
1941	Log(("Clearing VMCPU_FF_INHIBIT_INTERRUPTS at %RGv - successor %RGv\n", (RTGCPTR)CPUMGetGuestRIP(pVCpu), EMGetInhibitInterruptsPC(pVCpu)));
1942	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1943	}
1944	else
1945	Log(("Leaving VMCPU_FF_INHIBIT_INTERRUPTS set at %RGv\n", (RTGCPTR)CPUMGetGuestRIP(pVCpu)));
1946	}
1947
1948	/*
1949	* Interrupts.
1950	*/
1951	bool fWakeupPending = false;
1952	if ( !VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)
1953	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
1954	&& (!rc \|\| rc >= VINF_EM_RESCHEDULE_HM)
1955	&& !TRPMHasTrap(pVCpu) /* an interrupt could already be scheduled for dispatching in the recompiler. */
1956	#ifdef VBOX_WITH_RAW_MODE
1957	&& PATMAreInterruptsEnabled(pVM)
1958	#else
1959	&& (pVCpu->em.s.pCtx->eflags.u32 & X86_EFL_IF)
1960	#endif
1961	&& !HMR3IsEventPending(pVCpu))
1962	{
1963	Assert(pVCpu->em.s.enmState != EMSTATE_WAIT_SIPI);
1964	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
1965	{
1966	/* Note: it's important to make sure the return code from TRPMR3InjectEvent isn't ignored! */
1967	/** @todo this really isn't nice, should properly handle this */
1968	rc2 = TRPMR3InjectEvent(pVM, pVCpu, TRPM_HARDWARE_INT);
1969	if (pVM->em.s.fIemExecutesAll && (rc2 == VINF_EM_RESCHEDULE_REM \|\| rc2 == VINF_EM_RESCHEDULE_HM \|\| rc2 == VINF_EM_RESCHEDULE_RAW))
1970	rc2 = VINF_EM_RESCHEDULE;
1971	#ifdef VBOX_STRICT
1972	rcIrq = rc2;
1973	#endif
1974	UPDATE_RC();
1975	/* Reschedule required: We must not miss the wakeup below! */
1976	fWakeupPending = true;
1977	}
1978	#ifdef VBOX_WITH_REM
1979	/** @todo really ugly; if we entered the hlt state when exiting the recompiler and an interrupt was pending, we previously got stuck in the halted state. */
1980	else if (REMR3QueryPendingInterrupt(pVM, pVCpu) != REM_NO_PENDING_IRQ)
1981	{
1982	Log2(("REMR3QueryPendingInterrupt -> %#x\n", REMR3QueryPendingInterrupt(pVM, pVCpu)));
1983	rc2 = VINF_EM_RESCHEDULE_REM;
1984	UPDATE_RC();
1985	}
1986	#endif
1987	}
1988
1989	/*
1990	* Allocate handy pages.
1991	*/
1992	if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_PGM_NEED_HANDY_PAGES, VM_FF_PGM_NO_MEMORY))
1993	{
1994	rc2 = PGMR3PhysAllocateHandyPages(pVM);
1995	UPDATE_RC();
1996	}
1997
1998	/*
1999	* Debugger Facility request.
2000	*/
2001	if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_DBGF, VM_FF_PGM_NO_MEMORY))
2002	{
2003	rc2 = DBGFR3VMMForcedAction(pVM);
2004	UPDATE_RC();
2005	}
2006
2007	/*
2008	* EMT Rendezvous (must be serviced before termination).
2009	*/
2010	if ( !fWakeupPending /* don't miss the wakeup from EMSTATE_HALTED! */
2011	&& VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS))
2012	{
2013	rc2 = VMMR3EmtRendezvousFF(pVM, pVCpu);
2014	UPDATE_RC();
2015	/** @todo HACK ALERT! The following test is to make sure EM+TM thinks the VM is
2016	* stopped/reset before the next VM state change is made. We need a better
2017	* solution for this, or at least make it possible to do: (rc >= VINF_EM_FIRST
2018	* && rc >= VINF_EM_SUSPEND). */
2019	if (RT_UNLIKELY(rc == VINF_EM_SUSPEND \|\| rc == VINF_EM_RESET \|\| rc == VINF_EM_OFF))
2020	{
2021	Log2(("emR3ForcedActions: returns %Rrc\n", rc));
2022	STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
2023	return rc;
2024	}
2025	}
2026
2027	/*
2028	* State change request (cleared by vmR3SetStateLocked).
2029	*/
2030	if ( !fWakeupPending /* don't miss the wakeup from EMSTATE_HALTED! */
2031	&& VM_FF_IS_PENDING(pVM, VM_FF_CHECK_VM_STATE))
2032	{
2033	VMSTATE enmState = VMR3GetState(pVM);
2034	switch (enmState)
2035	{
2036	case VMSTATE_FATAL_ERROR:
2037	case VMSTATE_FATAL_ERROR_LS:
2038	Log2(("emR3ForcedActions: %s -> VINF_EM_SUSPEND\n", VMGetStateName(enmState) ));
2039	STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
2040	return VINF_EM_SUSPEND;
2041
2042	case VMSTATE_DESTROYING:
2043	Log2(("emR3ForcedActions: %s -> VINF_EM_TERMINATE\n", VMGetStateName(enmState) ));
2044	STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
2045	return VINF_EM_TERMINATE;
2046
2047	default:
2048	AssertMsgFailed(("%s\n", VMGetStateName(enmState)));
2049	}
2050	}
2051
2052	/*
2053	* Out of memory? Since most of our fellow high priority actions may cause us
2054	* to run out of memory, we're employing VM_FF_IS_PENDING_EXCEPT and putting this
2055	* at the end rather than the start. Also, VM_FF_TERMINATE has higher priority
2056	* than us since we can terminate without allocating more memory.
2057	*/
2058	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY))
2059	{
2060	rc2 = PGMR3PhysAllocateHandyPages(pVM);
2061	UPDATE_RC();
2062	if (rc == VINF_EM_NO_MEMORY)
2063	return rc;
2064	}
2065
2066	/*
2067	* If the virtual sync clock is still stopped, make TM restart it.
2068	*/
2069	if (VM_FF_IS_PENDING(pVM, VM_FF_TM_VIRTUAL_SYNC))
2070	TMR3VirtualSyncFF(pVM, pVCpu);
2071
2072	#ifdef DEBUG
2073	/*
2074	* Debug, pause the VM.
2075	*/
2076	if (VM_FF_IS_PENDING(pVM, VM_FF_DEBUG_SUSPEND))
2077	{
2078	VM_FF_CLEAR(pVM, VM_FF_DEBUG_SUSPEND);
2079	Log(("emR3ForcedActions: returns VINF_EM_SUSPEND\n"));
2080	return VINF_EM_SUSPEND;
2081	}
2082	#endif
2083
2084	/* check that we got them all */
2085	AssertCompile(VM_FF_HIGH_PRIORITY_PRE_MASK == (VM_FF_TM_VIRTUAL_SYNC \| VM_FF_DBGF \| VM_FF_CHECK_VM_STATE \| VM_FF_DEBUG_SUSPEND \| VM_FF_PGM_NEED_HANDY_PAGES \| VM_FF_PGM_NO_MEMORY \| VM_FF_EMT_RENDEZVOUS));
2086	AssertCompile(VMCPU_FF_HIGH_PRIORITY_PRE_MASK == (VMCPU_FF_TIMER \| VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_UPDATE_APIC \| VMCPU_FF_INTERRUPT_PIC \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL \| VMCPU_FF_INHIBIT_INTERRUPTS \| VM_WHEN_RAW_MODE(VMCPU_FF_SELM_SYNC_TSS \| VMCPU_FF_TRPM_SYNC_IDT \| VMCPU_FF_SELM_SYNC_GDT \| VMCPU_FF_SELM_SYNC_LDT, 0)));
2087	}
2088
2089	#undef UPDATE_RC
2090	Log2(("emR3ForcedActions: returns %Rrc\n", rc));
2091	STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
2092	Assert(rcIrq == VINF_SUCCESS \|\| rcIrq == rc);
2093	return rc;
2094	}
2095
2096
2097	/**
2098	* Check if the preset execution time cap restricts guest execution scheduling.
2099	*
2100	* @returns true if allowed, false otherwise
2101	* @param pVM The cross context VM structure.
2102	* @param pVCpu The cross context virtual CPU structure.
2103	*/
2104	bool emR3IsExecutionAllowed(PVM pVM, PVMCPU pVCpu)
2105	{
2106	uint64_t u64UserTime, u64KernelTime;
2107
2108	if ( pVM->uCpuExecutionCap != 100
2109	&& RT_SUCCESS(RTThreadGetExecutionTimeMilli(&u64KernelTime, &u64UserTime)))
2110	{
2111	uint64_t u64TimeNow = RTTimeMilliTS();
2112	if (pVCpu->em.s.u64TimeSliceStart + EM_TIME_SLICE < u64TimeNow)
2113	{
2114	/* New time slice. */
2115	pVCpu->em.s.u64TimeSliceStart = u64TimeNow;
2116	pVCpu->em.s.u64TimeSliceStartExec = u64KernelTime + u64UserTime;
2117	pVCpu->em.s.u64TimeSliceExec = 0;
2118	}
2119	pVCpu->em.s.u64TimeSliceExec = u64KernelTime + u64UserTime - pVCpu->em.s.u64TimeSliceStartExec;
2120
2121	Log2(("emR3IsExecutionAllowed: start=%RX64 startexec=%RX64 exec=%RX64 (cap=%x)\n", pVCpu->em.s.u64TimeSliceStart, pVCpu->em.s.u64TimeSliceStartExec, pVCpu->em.s.u64TimeSliceExec, (EM_TIME_SLICE * pVM->uCpuExecutionCap) / 100));
2122	if (pVCpu->em.s.u64TimeSliceExec >= (EM_TIME_SLICE * pVM->uCpuExecutionCap) / 100)
2123	return false;
2124	}
2125	return true;
2126	}
2127
2128
2129	/**
2130	* Execute VM.
2131	*
2132	* This function is the main loop of the VM. The emulation thread
2133	* calls this function when the VM has been successfully constructed
2134	* and we're ready for executing the VM.
2135	*
2136	* Returning from this function means that the VM is turned off or
2137	* suspended (state already saved) and deconstruction is next in line.
2138	*
2139	* All interaction from other thread are done using forced actions
2140	* and signaling of the wait object.
2141	*
2142	* @returns VBox status code, informational status codes may indicate failure.
2143	* @param pVM The cross context VM structure.
2144	* @param pVCpu The cross context virtual CPU structure.
2145	*/
2146	VMMR3_INT_DECL(int) EMR3ExecuteVM(PVM pVM, PVMCPU pVCpu)
2147	{
2148	Log(("EMR3ExecuteVM: pVM=%p enmVMState=%d (%s) enmState=%d (%s) enmPrevState=%d (%s) fForceRAW=%RTbool\n",
2149	pVM,
2150	pVM->enmVMState, VMR3GetStateName(pVM->enmVMState),
2151	pVCpu->em.s.enmState, emR3GetStateName(pVCpu->em.s.enmState),
2152	pVCpu->em.s.enmPrevState, emR3GetStateName(pVCpu->em.s.enmPrevState),
2153	pVCpu->em.s.fForceRAW));
2154	VM_ASSERT_EMT(pVM);
2155	AssertMsg( pVCpu->em.s.enmState == EMSTATE_NONE
2156	\|\| pVCpu->em.s.enmState == EMSTATE_WAIT_SIPI
2157	\|\| pVCpu->em.s.enmState == EMSTATE_SUSPENDED,
2158	("%s\n", emR3GetStateName(pVCpu->em.s.enmState)));
2159
2160	int rc = setjmp(pVCpu->em.s.u.FatalLongJump);
2161	if (rc == 0)
2162	{
2163	/*
2164	* Start the virtual time.
2165	*/
2166	TMR3NotifyResume(pVM, pVCpu);
2167
2168	/*
2169	* The Outer Main Loop.
2170	*/
2171	bool fFFDone = false;
2172
2173	/* Reschedule right away to start in the right state. */
2174	rc = VINF_SUCCESS;
2175
2176	/* If resuming after a pause or a state load, restore the previous
2177	state or else we'll start executing code. Else, just reschedule. */
2178	if ( pVCpu->em.s.enmState == EMSTATE_SUSPENDED
2179	&& ( pVCpu->em.s.enmPrevState == EMSTATE_WAIT_SIPI
2180	\|\| pVCpu->em.s.enmPrevState == EMSTATE_HALTED))
2181	pVCpu->em.s.enmState = pVCpu->em.s.enmPrevState;
2182	else
2183	pVCpu->em.s.enmState = emR3Reschedule(pVM, pVCpu, pVCpu->em.s.pCtx);
2184	pVCpu->em.s.cIemThenRemInstructions = 0;
2185	Log(("EMR3ExecuteVM: enmState=%s\n", emR3GetStateName(pVCpu->em.s.enmState)));
2186
2187	STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatTotal, x);
2188	for (;;)
2189	{
2190	/*
2191	* Before we can schedule anything (we're here because
2192	* scheduling is required) we must service any pending
2193	* forced actions to avoid any pending action causing
2194	* immediate rescheduling upon entering an inner loop
2195	*
2196	* Do forced actions.
2197	*/
2198	if ( !fFFDone
2199	&& RT_SUCCESS(rc)
2200	&& rc != VINF_EM_TERMINATE
2201	&& rc != VINF_EM_OFF
2202	&& ( VM_FF_IS_PENDING(pVM, VM_FF_ALL_REM_MASK)
2203	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_ALL_REM_MASK)))
2204	{
2205	rc = emR3ForcedActions(pVM, pVCpu, rc);
2206	VBOXVMM_EM_FF_ALL_RET(pVCpu, rc);
2207	if ( ( rc == VINF_EM_RESCHEDULE_REM
2208	\|\| rc == VINF_EM_RESCHEDULE_HM)
2209	&& pVCpu->em.s.fForceRAW)
2210	rc = VINF_EM_RESCHEDULE_RAW;
2211	}
2212	else if (fFFDone)
2213	fFFDone = false;
2214
2215	/*
2216	* Now what to do?
2217	*/
2218	Log2(("EMR3ExecuteVM: rc=%Rrc\n", rc));
2219	EMSTATE const enmOldState = pVCpu->em.s.enmState;
2220	switch (rc)
2221	{
2222	/*
2223	* Keep doing what we're currently doing.
2224	*/
2225	case VINF_SUCCESS:
2226	break;
2227
2228	/*
2229	* Reschedule - to raw-mode execution.
2230	*/
2231	case VINF_EM_RESCHEDULE_RAW:
2232	Log2(("EMR3ExecuteVM: VINF_EM_RESCHEDULE_RAW: %d -> %d (EMSTATE_RAW)\n", enmOldState, EMSTATE_RAW));
2233	Assert(!pVM->em.s.fIemExecutesAll \|\| pVCpu->em.s.enmState != EMSTATE_IEM);
2234	pVCpu->em.s.enmState = EMSTATE_RAW;
2235	break;
2236
2237	/*
2238	* Reschedule - to hardware accelerated raw-mode execution.
2239	*/
2240	case VINF_EM_RESCHEDULE_HM:
2241	Log2(("EMR3ExecuteVM: VINF_EM_RESCHEDULE_HM: %d -> %d (EMSTATE_HM)\n", enmOldState, EMSTATE_HM));
2242	Assert(!pVM->em.s.fIemExecutesAll \|\| pVCpu->em.s.enmState != EMSTATE_IEM);
2243	Assert(!pVCpu->em.s.fForceRAW);
2244	pVCpu->em.s.enmState = EMSTATE_HM;
2245	break;
2246
2247	/*
2248	* Reschedule - to recompiled execution.
2249	*/
2250	case VINF_EM_RESCHEDULE_REM:
2251	Assert(!pVM->em.s.fIemExecutesAll \|\| pVCpu->em.s.enmState != EMSTATE_IEM);
2252	if (HMIsEnabled(pVM))
2253	{
2254	Log2(("EMR3ExecuteVM: VINF_EM_RESCHEDULE_REM: %d -> %d (EMSTATE_IEM_THEN_REM)\n",
2255	enmOldState, EMSTATE_IEM_THEN_REM));
2256	if (pVCpu->em.s.enmState != EMSTATE_IEM_THEN_REM)
2257	{
2258	pVCpu->em.s.enmState = EMSTATE_IEM_THEN_REM;
2259	pVCpu->em.s.cIemThenRemInstructions = 0;
2260	}
2261	}
2262	else
2263	{
2264	Log2(("EMR3ExecuteVM: VINF_EM_RESCHEDULE_REM: %d -> %d (EMSTATE_REM)\n", enmOldState, EMSTATE_REM));
2265	pVCpu->em.s.enmState = EMSTATE_REM;
2266	}
2267	break;
2268
2269	/*
2270	* Resume.
2271	*/
2272	case VINF_EM_RESUME:
2273	Log2(("EMR3ExecuteVM: VINF_EM_RESUME: %d -> VINF_EM_RESCHEDULE\n", enmOldState));
2274	/* Don't reschedule in the halted or wait for SIPI case. */
2275	if ( pVCpu->em.s.enmPrevState == EMSTATE_WAIT_SIPI
2276	\|\| pVCpu->em.s.enmPrevState == EMSTATE_HALTED)
2277	{
2278	pVCpu->em.s.enmState = pVCpu->em.s.enmPrevState;
2279	break;
2280	}
2281	/* fall through and get scheduled. */
2282
2283	/*
2284	* Reschedule.
2285	*/
2286	case VINF_EM_RESCHEDULE:
2287	{
2288	EMSTATE enmState = emR3Reschedule(pVM, pVCpu, pVCpu->em.s.pCtx);
2289	Log2(("EMR3ExecuteVM: VINF_EM_RESCHEDULE: %d -> %d (%s)\n", enmOldState, enmState, emR3GetStateName(enmState)));
2290	if (pVCpu->em.s.enmState != enmState && enmState == EMSTATE_IEM_THEN_REM)
2291	pVCpu->em.s.cIemThenRemInstructions = 0;
2292	pVCpu->em.s.enmState = enmState;
2293	break;
2294	}
2295
2296	/*
2297	* Halted.
2298	*/
2299	case VINF_EM_HALT:
2300	Log2(("EMR3ExecuteVM: VINF_EM_HALT: %d -> %d\n", enmOldState, EMSTATE_HALTED));
2301	pVCpu->em.s.enmState = EMSTATE_HALTED;
2302	break;
2303
2304	/*
2305	* Switch to the wait for SIPI state (application processor only)
2306	*/
2307	case VINF_EM_WAIT_SIPI:
2308	Assert(pVCpu->idCpu != 0);
2309	Log2(("EMR3ExecuteVM: VINF_EM_WAIT_SIPI: %d -> %d\n", enmOldState, EMSTATE_WAIT_SIPI));
2310	pVCpu->em.s.enmState = EMSTATE_WAIT_SIPI;
2311	break;
2312
2313
2314	/*
2315	* Suspend.
2316	*/
2317	case VINF_EM_SUSPEND:
2318	Log2(("EMR3ExecuteVM: VINF_EM_SUSPEND: %d -> %d\n", enmOldState, EMSTATE_SUSPENDED));
2319	Assert(enmOldState != EMSTATE_SUSPENDED);
2320	pVCpu->em.s.enmPrevState = enmOldState;
2321	pVCpu->em.s.enmState = EMSTATE_SUSPENDED;
2322	break;
2323
2324	/*
2325	* Reset.
2326	* We might end up doing a double reset for now, we'll have to clean up the mess later.
2327	*/
2328	case VINF_EM_RESET:
2329	{
2330	if (pVCpu->idCpu == 0)
2331	{
2332	EMSTATE enmState = emR3Reschedule(pVM, pVCpu, pVCpu->em.s.pCtx);
2333	Log2(("EMR3ExecuteVM: VINF_EM_RESET: %d -> %d (%s)\n", enmOldState, enmState, emR3GetStateName(enmState)));
2334	if (pVCpu->em.s.enmState != enmState && enmState == EMSTATE_IEM_THEN_REM)
2335	pVCpu->em.s.cIemThenRemInstructions = 0;
2336	pVCpu->em.s.enmState = enmState;
2337	}
2338	else
2339	{
2340	/* All other VCPUs go into the wait for SIPI state. */
2341	pVCpu->em.s.enmState = EMSTATE_WAIT_SIPI;
2342	}
2343	break;
2344	}
2345
2346	/*
2347	* Power Off.
2348	*/
2349	case VINF_EM_OFF:
2350	pVCpu->em.s.enmState = EMSTATE_TERMINATING;
2351	Log2(("EMR3ExecuteVM: returns VINF_EM_OFF (%d -> %d)\n", enmOldState, EMSTATE_TERMINATING));
2352	TMR3NotifySuspend(pVM, pVCpu);
2353	STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
2354	return rc;
2355
2356	/*
2357	* Terminate the VM.
2358	*/
2359	case VINF_EM_TERMINATE:
2360	pVCpu->em.s.enmState = EMSTATE_TERMINATING;
2361	Log(("EMR3ExecuteVM returns VINF_EM_TERMINATE (%d -> %d)\n", enmOldState, EMSTATE_TERMINATING));
2362	if (pVM->enmVMState < VMSTATE_DESTROYING) /* ugly */
2363	TMR3NotifySuspend(pVM, pVCpu);
2364	STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
2365	return rc;
2366
2367
2368	/*
2369	* Out of memory, suspend the VM and stuff.
2370	*/
2371	case VINF_EM_NO_MEMORY:
2372	Log2(("EMR3ExecuteVM: VINF_EM_NO_MEMORY: %d -> %d\n", enmOldState, EMSTATE_SUSPENDED));
2373	Assert(enmOldState != EMSTATE_SUSPENDED);
2374	pVCpu->em.s.enmPrevState = enmOldState;
2375	pVCpu->em.s.enmState = EMSTATE_SUSPENDED;
2376	TMR3NotifySuspend(pVM, pVCpu);
2377	STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
2378
2379	rc = VMSetRuntimeError(pVM, VMSETRTERR_FLAGS_SUSPEND, "HostMemoryLow",
2380	N_("Unable to allocate and lock memory. The virtual machine will be paused. Please close applications to free up memory or close the VM"));
2381	if (rc != VINF_EM_SUSPEND)
2382	{
2383	if (RT_SUCCESS_NP(rc))
2384	{
2385	AssertLogRelMsgFailed(("%Rrc\n", rc));
2386	rc = VERR_EM_INTERNAL_ERROR;
2387	}
2388	pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
2389	}
2390	return rc;
2391
2392	/*
2393	* Guest debug events.
2394	*/
2395	case VINF_EM_DBG_STEPPED:
2396	case VINF_EM_DBG_STOP:
2397	case VINF_EM_DBG_EVENT:
2398	case VINF_EM_DBG_BREAKPOINT:
2399	case VINF_EM_DBG_STEP:
2400	if (enmOldState == EMSTATE_RAW)
2401	{
2402	Log2(("EMR3ExecuteVM: %Rrc: %d -> %d\n", rc, enmOldState, EMSTATE_DEBUG_GUEST_RAW));
2403	pVCpu->em.s.enmState = EMSTATE_DEBUG_GUEST_RAW;
2404	}
2405	else if (enmOldState == EMSTATE_HM)
2406	{
2407	Log2(("EMR3ExecuteVM: %Rrc: %d -> %d\n", rc, enmOldState, EMSTATE_DEBUG_GUEST_HM));
2408	pVCpu->em.s.enmState = EMSTATE_DEBUG_GUEST_HM;
2409	}
2410	else if (enmOldState == EMSTATE_REM)
2411	{
2412	Log2(("EMR3ExecuteVM: %Rrc: %d -> %d\n", rc, enmOldState, EMSTATE_DEBUG_GUEST_REM));
2413	pVCpu->em.s.enmState = EMSTATE_DEBUG_GUEST_REM;
2414	}
2415	else
2416	{
2417	Log2(("EMR3ExecuteVM: %Rrc: %d -> %d\n", rc, enmOldState, EMSTATE_DEBUG_GUEST_IEM));
2418	pVCpu->em.s.enmState = EMSTATE_DEBUG_GUEST_IEM;
2419	}
2420	break;
2421
2422	/*
2423	* Hypervisor debug events.
2424	*/
2425	case VINF_EM_DBG_HYPER_STEPPED:
2426	case VINF_EM_DBG_HYPER_BREAKPOINT:
2427	case VINF_EM_DBG_HYPER_ASSERTION:
2428	Log2(("EMR3ExecuteVM: %Rrc: %d -> %d\n", rc, enmOldState, EMSTATE_DEBUG_HYPER));
2429	pVCpu->em.s.enmState = EMSTATE_DEBUG_HYPER;
2430	break;
2431
2432	/*
2433	* Triple fault.
2434	*/
2435	case VINF_EM_TRIPLE_FAULT:
2436	if (!pVM->em.s.fGuruOnTripleFault)
2437	{
2438	Log(("EMR3ExecuteVM: VINF_EM_TRIPLE_FAULT: CPU reset...\n"));
2439	rc = VBOXSTRICTRC_TODO(VMR3ResetTripleFault(pVM));
2440	Log2(("EMR3ExecuteVM: VINF_EM_TRIPLE_FAULT: %d -> %d (rc=%Rrc)\n", enmOldState, pVCpu->em.s.enmState, rc));
2441	continue;
2442	}
2443	/* Else fall through and trigger a guru. */
2444	case VERR_VMM_RING0_ASSERTION:
2445	Log(("EMR3ExecuteVM: %Rrc: %d -> %d (EMSTATE_GURU_MEDITATION)\n", rc, enmOldState, EMSTATE_GURU_MEDITATION));
2446	pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
2447	break;
2448
2449	/*
2450	* Any error code showing up here other than the ones we
2451	* know and process above are considered to be FATAL.
2452	*
2453	* Unknown warnings and informational status codes are also
2454	* included in this.
2455	*/
2456	default:
2457	if (RT_SUCCESS_NP(rc))
2458	{
2459	AssertMsgFailed(("Unexpected warning or informational status code %Rra!\n", rc));
2460	rc = VERR_EM_INTERNAL_ERROR;
2461	}
2462	Log(("EMR3ExecuteVM: %Rrc: %d -> %d (EMSTATE_GURU_MEDITATION)\n", rc, enmOldState, EMSTATE_GURU_MEDITATION));
2463	pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
2464	break;
2465	}
2466
2467	/*
2468	* Act on state transition.
2469	*/
2470	EMSTATE const enmNewState = pVCpu->em.s.enmState;
2471	if (enmOldState != enmNewState)
2472	{
2473	VBOXVMM_EM_STATE_CHANGED(pVCpu, enmOldState, enmNewState, rc);
2474
2475	/* Clear MWait flags. */
2476	if ( enmOldState == EMSTATE_HALTED
2477	&& (pVCpu->em.s.MWait.fWait & EMMWAIT_FLAG_ACTIVE)
2478	&& ( enmNewState == EMSTATE_RAW
2479	\|\| enmNewState == EMSTATE_HM
2480	\|\| enmNewState == EMSTATE_REM
2481	\|\| enmNewState == EMSTATE_IEM_THEN_REM
2482	\|\| enmNewState == EMSTATE_DEBUG_GUEST_RAW
2483	\|\| enmNewState == EMSTATE_DEBUG_GUEST_HM
2484	\|\| enmNewState == EMSTATE_DEBUG_GUEST_IEM
2485	\|\| enmNewState == EMSTATE_DEBUG_GUEST_REM) )
2486	{
2487	LogFlow(("EMR3ExecuteVM: Clearing MWAIT\n"));
2488	pVCpu->em.s.MWait.fWait &= ~(EMMWAIT_FLAG_ACTIVE \| EMMWAIT_FLAG_BREAKIRQIF0);
2489	}
2490	}
2491	else
2492	VBOXVMM_EM_STATE_UNCHANGED(pVCpu, enmNewState, rc);
2493
2494	STAM_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x); /* (skip this in release) */
2495	STAM_PROFILE_ADV_START(&pVCpu->em.s.StatTotal, x);
2496
2497	/*
2498	* Act on the new state.
2499	*/
2500	switch (enmNewState)
2501	{
2502	/*
2503	* Execute raw.
2504	*/
2505	case EMSTATE_RAW:
2506	#ifdef VBOX_WITH_RAW_MODE
2507	rc = emR3RawExecute(pVM, pVCpu, &fFFDone);
2508	#else
2509	AssertLogRelMsgFailed(("%Rrc\n", rc));
2510	rc = VERR_EM_INTERNAL_ERROR;
2511	#endif
2512	break;
2513
2514	/*
2515	* Execute hardware accelerated raw.
2516	*/
2517	case EMSTATE_HM:
2518	rc = emR3HmExecute(pVM, pVCpu, &fFFDone);
2519	break;
2520
2521	/*
2522	* Execute recompiled.
2523	*/
2524	case EMSTATE_REM:
2525	rc = emR3RemExecute(pVM, pVCpu, &fFFDone);
2526	Log2(("EMR3ExecuteVM: emR3RemExecute -> %Rrc\n", rc));
2527	break;
2528
2529	/*
2530	* Execute in the interpreter.
2531	*/
2532	case EMSTATE_IEM:
2533	{
2534	#if 0 /* For testing purposes. */
2535	STAM_PROFILE_START(&pVCpu->em.s.StatHmExec, x1);
2536	rc = VBOXSTRICTRC_TODO(EMR3HmSingleInstruction(pVM, pVCpu, EM_ONE_INS_FLAGS_RIP_CHANGE));
2537	STAM_PROFILE_STOP(&pVCpu->em.s.StatHmExec, x1);
2538	if (rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_RESCHEDULE_HM \|\| rc == VINF_EM_RESCHEDULE_REM \|\| rc == VINF_EM_RESCHEDULE_RAW)
2539	rc = VINF_SUCCESS;
2540	else if (rc == VERR_EM_CANNOT_EXEC_GUEST)
2541	#endif
2542	rc = VBOXSTRICTRC_TODO(IEMExecLots(pVCpu));
2543	if (pVM->em.s.fIemExecutesAll)
2544	{
2545	Assert(rc != VINF_EM_RESCHEDULE_REM);
2546	Assert(rc != VINF_EM_RESCHEDULE_RAW);
2547	Assert(rc != VINF_EM_RESCHEDULE_HM);
2548	}
2549	fFFDone = false;
2550	break;
2551	}
2552
2553	/*
2554	* Execute in IEM, hoping we can quickly switch aback to HM
2555	* or RAW execution. If our hopes fail, we go to REM.
2556	*/
2557	case EMSTATE_IEM_THEN_REM:
2558	{
2559	STAM_PROFILE_START(&pVCpu->em.s.StatIEMThenREM, pIemThenRem);
2560	rc = VBOXSTRICTRC_TODO(emR3ExecuteIemThenRem(pVM, pVCpu, &fFFDone));
2561	STAM_PROFILE_STOP(&pVCpu->em.s.StatIEMThenREM, pIemThenRem);
2562	break;
2563	}
2564
2565	/*
2566	* Application processor execution halted until SIPI.
2567	*/
2568	case EMSTATE_WAIT_SIPI:
2569	/* no break */
2570	/*
2571	* hlt - execution halted until interrupt.
2572	*/
2573	case EMSTATE_HALTED:
2574	{
2575	STAM_REL_PROFILE_START(&pVCpu->em.s.StatHalted, y);
2576	/* If HM (or someone else) store a pending interrupt in
2577	TRPM, it must be dispatched ASAP without any halting.
2578	Anything pending in TRPM has been accepted and the CPU
2579	should already be the right state to receive it. */
2580	if (TRPMHasTrap(pVCpu))
2581	rc = VINF_EM_RESCHEDULE;
2582	/* MWAIT has a special extension where it's woken up when
2583	an interrupt is pending even when IF=0. */
2584	else if ( (pVCpu->em.s.MWait.fWait & (EMMWAIT_FLAG_ACTIVE \| EMMWAIT_FLAG_BREAKIRQIF0))
2585	== (EMMWAIT_FLAG_ACTIVE \| EMMWAIT_FLAG_BREAKIRQIF0))
2586	{
2587	rc = VMR3WaitHalted(pVM, pVCpu, false /fIgnoreInterrupts/);
2588	if (rc == VINF_SUCCESS)
2589	{
2590	#ifdef VBOX_WITH_NEW_APIC
2591	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
2592	APICUpdatePendingInterrupts(pVCpu);
2593	#endif
2594	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC
2595	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI \| VMCPU_FF_UNHALT))
2596	{
2597	Log(("EMR3ExecuteVM: Triggering reschedule on pending IRQ after MWAIT\n"));
2598	rc = VINF_EM_RESCHEDULE;
2599	}
2600	}
2601	}
2602	else
2603	{
2604	rc = VMR3WaitHalted(pVM, pVCpu, !(CPUMGetGuestEFlags(pVCpu) & X86_EFL_IF));
2605	if ( rc == VINF_SUCCESS
2606	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI \| VMCPU_FF_UNHALT))
2607	{
2608	Log(("EMR3ExecuteVM: Triggering reschedule on pending NMI/SMI/UNHALT after HLT\n"));
2609	rc = VINF_EM_RESCHEDULE;
2610	}
2611	}
2612
2613	STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatHalted, y);
2614	break;
2615	}
2616
2617	/*
2618	* Suspended - return to VM.cpp.
2619	*/
2620	case EMSTATE_SUSPENDED:
2621	TMR3NotifySuspend(pVM, pVCpu);
2622	STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
2623	Log(("EMR3ExecuteVM: actually returns %Rrc (state %s / %s)\n", rc, emR3GetStateName(pVCpu->em.s.enmState), emR3GetStateName(enmOldState)));
2624	return VINF_EM_SUSPEND;
2625
2626	/*
2627	* Debugging in the guest.
2628	*/
2629	case EMSTATE_DEBUG_GUEST_RAW:
2630	case EMSTATE_DEBUG_GUEST_HM:
2631	case EMSTATE_DEBUG_GUEST_IEM:
2632	case EMSTATE_DEBUG_GUEST_REM:
2633	TMR3NotifySuspend(pVM, pVCpu);
2634	rc = VBOXSTRICTRC_TODO(emR3Debug(pVM, pVCpu, rc));
2635	TMR3NotifyResume(pVM, pVCpu);
2636	Log2(("EMR3ExecuteVM: emR3Debug -> %Rrc (state %d)\n", rc, pVCpu->em.s.enmState));
2637	break;
2638
2639	/*
2640	* Debugging in the hypervisor.
2641	*/
2642	case EMSTATE_DEBUG_HYPER:
2643	{
2644	TMR3NotifySuspend(pVM, pVCpu);
2645	STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
2646
2647	rc = VBOXSTRICTRC_TODO(emR3Debug(pVM, pVCpu, rc));
2648	Log2(("EMR3ExecuteVM: emR3Debug -> %Rrc (state %d)\n", rc, pVCpu->em.s.enmState));
2649	if (rc != VINF_SUCCESS)
2650	{
2651	if (rc == VINF_EM_OFF \|\| rc == VINF_EM_TERMINATE)
2652	pVCpu->em.s.enmState = EMSTATE_TERMINATING;
2653	else
2654	{
2655	/* switch to guru meditation mode */
2656	pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
2657	VMMR3FatalDump(pVM, pVCpu, rc);
2658	}
2659	Log(("EMR3ExecuteVM: actually returns %Rrc (state %s / %s)\n", rc, emR3GetStateName(pVCpu->em.s.enmState), emR3GetStateName(enmOldState)));
2660	return rc;
2661	}
2662
2663	STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatTotal, x);
2664	TMR3NotifyResume(pVM, pVCpu);
2665	break;
2666	}
2667
2668	/*
2669	* Guru meditation takes place in the debugger.
2670	*/
2671	case EMSTATE_GURU_MEDITATION:
2672	{
2673	TMR3NotifySuspend(pVM, pVCpu);
2674	VMMR3FatalDump(pVM, pVCpu, rc);
2675	emR3Debug(pVM, pVCpu, rc);
2676	STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
2677	Log(("EMR3ExecuteVM: actually returns %Rrc (state %s / %s)\n", rc, emR3GetStateName(pVCpu->em.s.enmState), emR3GetStateName(enmOldState)));
2678	return rc;
2679	}
2680
2681	/*
2682	* The states we don't expect here.
2683	*/
2684	case EMSTATE_NONE:
2685	case EMSTATE_TERMINATING:
2686	default:
2687	AssertMsgFailed(("EMR3ExecuteVM: Invalid state %d!\n", pVCpu->em.s.enmState));
2688	pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
2689	TMR3NotifySuspend(pVM, pVCpu);
2690	STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
2691	Log(("EMR3ExecuteVM: actually returns %Rrc (state %s / %s)\n", rc, emR3GetStateName(pVCpu->em.s.enmState), emR3GetStateName(enmOldState)));
2692	return VERR_EM_INTERNAL_ERROR;
2693	}
2694	} /* The Outer Main Loop */
2695	}
2696	else
2697	{
2698	/*
2699	* Fatal error.
2700	*/
2701	Log(("EMR3ExecuteVM: returns %Rrc because of longjmp / fatal error; (state %s / %s)\n", rc, emR3GetStateName(pVCpu->em.s.enmState), emR3GetStateName(pVCpu->em.s.enmPrevState)));
2702	TMR3NotifySuspend(pVM, pVCpu);
2703	VMMR3FatalDump(pVM, pVCpu, rc);
2704	emR3Debug(pVM, pVCpu, rc);
2705	STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
2706	/** @todo change the VM state! */
2707	return rc;
2708	}
2709
2710	/* (won't ever get here). */
2711	AssertFailed();
2712	}
2713
2714	/**
2715	* Notify EM of a state change (used by FTM)
2716	*
2717	* @param pVM The cross context VM structure.
2718	*/
2719	VMMR3_INT_DECL(int) EMR3NotifySuspend(PVM pVM)
2720	{
2721	PVMCPU pVCpu = VMMGetCpu(pVM);
2722
2723	TMR3NotifySuspend(pVM, pVCpu); /* Stop the virtual time. */
2724	pVCpu->em.s.enmPrevState = pVCpu->em.s.enmState;
2725	pVCpu->em.s.enmState = EMSTATE_SUSPENDED;
2726	return VINF_SUCCESS;
2727	}
2728
2729	/**
2730	* Notify EM of a state change (used by FTM)
2731	*
2732	* @param pVM The cross context VM structure.
2733	*/
2734	VMMR3_INT_DECL(int) EMR3NotifyResume(PVM pVM)
2735	{
2736	PVMCPU pVCpu = VMMGetCpu(pVM);
2737	EMSTATE enmCurState = pVCpu->em.s.enmState;
2738
2739	TMR3NotifyResume(pVM, pVCpu); /* Resume the virtual time. */
2740	pVCpu->em.s.enmState = pVCpu->em.s.enmPrevState;
2741	pVCpu->em.s.enmPrevState = enmCurState;
2742	return VINF_SUCCESS;
2743	}

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR3/EM.cpp@ 61072

Download in other formats: