VMM.cpp@ 22309

Last change on this file since 22309 was 21963, checked in by vboxsync, 15 years ago
VMM: When we're switching to our own private stack, it must have a ring-0 mapping. fixes darwin.amd64 #DFs and #PFs when using VT-x and sometimes with raw-mode.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 74.8 KB

Line
1	/* $Id: VMM.cpp 21963 2009-08-04 15:12:59Z vboxsync $ */
2	/** @file
3	* VMM - The Virtual Machine Monitor Core.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22	//#define NO_SUPCALLR0VMM
23
24	/** @page pg_vmm VMM - The Virtual Machine Monitor
25	*
26	* The VMM component is two things at the moment, it's a component doing a few
27	* management and routing tasks, and it's the whole virtual machine monitor
28	* thing. For hysterical reasons, it is not doing all the management that one
29	* would expect, this is instead done by @ref pg_vm. We'll address this
30	* misdesign eventually.
31	*
32	* @see grp_vmm, grp_vm
33	*
34	*
35	* @section sec_vmmstate VMM State
36	*
37	* @image html VM_Statechart_Diagram.gif
38	*
39	* To be written.
40	*
41	*
42	* @subsection subsec_vmm_init VMM Initialization
43	*
44	* To be written.
45	*
46	*
47	* @subsection subsec_vmm_term VMM Termination
48	*
49	* To be written.
50	*
51	*/
52
53	/*******************************************************************************
54	* Header Files *
55	*******************************************************************************/
56	#define LOG_GROUP LOG_GROUP_VMM
57	#include <VBox/vmm.h>
58	#include <VBox/vmapi.h>
59	#include <VBox/pgm.h>
60	#include <VBox/cfgm.h>
61	#include <VBox/pdmqueue.h>
62	#include <VBox/pdmcritsect.h>
63	#include <VBox/pdmapi.h>
64	#include <VBox/cpum.h>
65	#include <VBox/mm.h>
66	#include <VBox/iom.h>
67	#include <VBox/trpm.h>
68	#include <VBox/selm.h>
69	#include <VBox/em.h>
70	#include <VBox/sup.h>
71	#include <VBox/dbgf.h>
72	#include <VBox/csam.h>
73	#include <VBox/patm.h>
74	#include <VBox/rem.h>
75	#include <VBox/ssm.h>
76	#include <VBox/tm.h>
77	#include "VMMInternal.h"
78	#include "VMMSwitcher/VMMSwitcher.h"
79	#include <VBox/vm.h>
80
81	#include <VBox/err.h>
82	#include <VBox/param.h>
83	#include <VBox/version.h>
84	#include <VBox/x86.h>
85	#include <VBox/hwaccm.h>
86	#include <iprt/assert.h>
87	#include <iprt/alloc.h>
88	#include <iprt/asm.h>
89	#include <iprt/time.h>
90	#include <iprt/semaphore.h>
91	#include <iprt/stream.h>
92	#include <iprt/string.h>
93	#include <iprt/stdarg.h>
94	#include <iprt/ctype.h>
95
96
97
98	/*******************************************************************************
99	* Defined Constants And Macros *
100	*******************************************************************************/
101	/** The saved state version. */
102	#define VMM_SAVED_STATE_VERSION 3
103
104
105	/*******************************************************************************
106	* Internal Functions *
107	*******************************************************************************/
108	static int vmmR3InitStacks(PVM pVM);
109	static int vmmR3InitLoggers(PVM pVM);
110	static void vmmR3InitRegisterStats(PVM pVM);
111	static DECLCALLBACK(int) vmmR3Save(PVM pVM, PSSMHANDLE pSSM);
112	static DECLCALLBACK(int) vmmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t u32Version);
113	static DECLCALLBACK(void) vmmR3YieldEMT(PVM pVM, PTMTIMER pTimer, void *pvUser);
114	static int vmmR3ServiceCallRing3Request(PVM pVM, PVMCPU pVCpu);
115	static DECLCALLBACK(void) vmmR3InfoFF(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
116
117
118	/**
119	* Initializes the VMM.
120	*
121	* @returns VBox status code.
122	* @param pVM The VM to operate on.
123	*/
124	VMMR3DECL(int) VMMR3Init(PVM pVM)
125	{
126	LogFlow(("VMMR3Init\n"));
127
128	/*
129	* Assert alignment, sizes and order.
130	*/
131	AssertMsg(pVM->vmm.s.offVM == 0, ("Already initialized!\n"));
132	AssertCompile(sizeof(pVM->vmm.s) <= sizeof(pVM->vmm.padding));
133	AssertCompile(sizeof(pVM->aCpus[0].vmm.s) <= sizeof(pVM->aCpus[0].vmm.padding));
134
135	/*
136	* Init basic VM VMM members.
137	*/
138	pVM->vmm.s.offVM = RT_OFFSETOF(VM, vmm);
139	pVM->vmm.s.hEvtRendezvousEnterOneByOne = NIL_RTSEMEVENT;
140	pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce = NIL_RTSEMEVENTMULTI;
141	pVM->vmm.s.hEvtMulRendezvousDone = NIL_RTSEMEVENTMULTI;
142	pVM->vmm.s.hEvtRendezvousDoneCaller = NIL_RTSEMEVENT;
143	int rc = CFGMR3QueryU32(CFGMR3GetRoot(pVM), "YieldEMTInterval", &pVM->vmm.s.cYieldEveryMillies);
144	if (rc == VERR_CFGM_VALUE_NOT_FOUND)
145	pVM->vmm.s.cYieldEveryMillies = 23; /* Value arrived at after experimenting with the grub boot prompt. */
146	//pVM->vmm.s.cYieldEveryMillies = 8; //debugging
147	else
148	AssertMsgRCReturn(rc, ("Configuration error. Failed to query \"YieldEMTInterval\", rc=%Rrc\n", rc), rc);
149
150	/*
151	* Initialize the VMM sync critical section and semaphores.
152	*/
153	rc = RTCritSectInit(&pVM->vmm.s.CritSectSync);
154	AssertRCReturn(rc, rc);
155	rc = RTSemEventCreate(&pVM->vmm.s.hEvtRendezvousEnterOneByOne);
156	AssertRCReturn(rc, rc);
157	rc = RTSemEventMultiCreate(&pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
158	AssertRCReturn(rc, rc);
159	rc = RTSemEventMultiCreate(&pVM->vmm.s.hEvtMulRendezvousDone);
160	AssertRCReturn(rc, rc);
161	rc = RTSemEventCreate(&pVM->vmm.s.hEvtRendezvousDoneCaller);
162	AssertRCReturn(rc, rc);
163
164	/* GC switchers are enabled by default. Turned off by HWACCM. */
165	pVM->vmm.s.fSwitcherDisabled = false;
166
167	/*
168	* Register the saved state data unit.
169	*/
170	rc = SSMR3RegisterInternal(pVM, "vmm", 1, VMM_SAVED_STATE_VERSION, VMM_STACK_SIZE + sizeof(RTGCPTR),
171	NULL, vmmR3Save, NULL,
172	NULL, vmmR3Load, NULL);
173	if (RT_FAILURE(rc))
174	return rc;
175
176	/*
177	* Register the Ring-0 VM handle with the session for fast ioctl calls.
178	*/
179	rc = SUPR3SetVMForFastIOCtl(pVM->pVMR0);
180	if (RT_FAILURE(rc))
181	return rc;
182
183	/*
184	* Init various sub-components.
185	*/
186	rc = vmmR3SwitcherInit(pVM);
187	if (RT_SUCCESS(rc))
188	{
189	rc = vmmR3InitStacks(pVM);
190	if (RT_SUCCESS(rc))
191	{
192	rc = vmmR3InitLoggers(pVM);
193
194	#ifdef VBOX_WITH_NMI
195	/*
196	* Allocate mapping for the host APIC.
197	*/
198	if (RT_SUCCESS(rc))
199	{
200	rc = MMR3HyperReserve(pVM, PAGE_SIZE, "Host APIC", &pVM->vmm.s.GCPtrApicBase);
201	AssertRC(rc);
202	}
203	#endif
204	if (RT_SUCCESS(rc))
205	{
206	/*
207	* Debug info and statistics.
208	*/
209	DBGFR3InfoRegisterInternal(pVM, "ff", "Displays the current Forced actions Flags.", vmmR3InfoFF);
210	vmmR3InitRegisterStats(pVM);
211
212	return VINF_SUCCESS;
213	}
214	}
215	/** @todo: Need failure cleanup. */
216
217	//more todo in here?
218	//if (RT_SUCCESS(rc))
219	//{
220	//}
221	//int rc2 = vmmR3TermCoreCode(pVM);
222	//AssertRC(rc2));
223	}
224
225	return rc;
226	}
227
228
229	/**
230	* Allocate & setup the VMM RC stack(s) (for EMTs).
231	*
232	* The stacks are also used for long jumps in Ring-0.
233	*
234	* @returns VBox status code.
235	* @param pVM Pointer to the shared VM structure.
236	*
237	* @remarks The optional guard page gets it protection setup up during R3 init
238	* completion because of init order issues.
239	*/
240	static int vmmR3InitStacks(PVM pVM)
241	{
242	int rc = VINF_SUCCESS;
243	#ifdef VMM_R0_SWITCH_STACK
244	uint32_t fFlags = MMHYPER_AONR_FLAGS_KERNEL_MAPPING;
245	#else
246	uint32_t fFlags = 0;
247	#endif
248
249	for (VMCPUID idCpu = 0; idCpu < pVM->cCPUs; idCpu++)
250	{
251	PVMCPU pVCpu = &pVM->aCpus[idCpu];
252
253	#ifdef VBOX_STRICT_VMM_STACK
254	rc = MMR3HyperAllocOnceNoRelEx(pVM, PAGE_SIZE + VMM_STACK_SIZE + PAGE_SIZE,
255	#else
256	rc = MMR3HyperAllocOnceNoRelEx(pVM, VMM_STACK_SIZE,
257	#endif
258	PAGE_SIZE, MM_TAG_VMM, fFlags, (void **)&pVCpu->vmm.s.pbEMTStackR3);
259	if (RT_SUCCESS(rc))
260	{
261	#ifdef VBOX_STRICT_VMM_STACK
262	pVCpu->vmm.s.pbEMTStackR3 += PAGE_SIZE;
263	#endif
264	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
265	/* MMHyperR3ToR0 returns R3 when not doing hardware assisted virtualization. */
266	if (!VMMIsHwVirtExtForced(pVM))
267	pVCpu->vmm.s.CallRing3JmpBufR0.pvSavedStack = NIL_RTR0PTR;
268	else
269	#endif
270	pVCpu->vmm.s.CallRing3JmpBufR0.pvSavedStack = MMHyperR3ToR0(pVM, pVCpu->vmm.s.pbEMTStackR3);
271	pVCpu->vmm.s.pbEMTStackRC = MMHyperR3ToRC(pVM, pVCpu->vmm.s.pbEMTStackR3);
272	pVCpu->vmm.s.pbEMTStackBottomRC = pVCpu->vmm.s.pbEMTStackRC + VMM_STACK_SIZE;
273	AssertRelease(pVCpu->vmm.s.pbEMTStackRC);
274
275	CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC);
276	}
277	}
278
279	return rc;
280	}
281
282
283	/**
284	* Initialize the loggers.
285	*
286	* @returns VBox status code.
287	* @param pVM Pointer to the shared VM structure.
288	*/
289	static int vmmR3InitLoggers(PVM pVM)
290	{
291	int rc;
292
293	/*
294	* Allocate RC & R0 Logger instances (they are finalized in the relocator).
295	*/
296	#ifdef LOG_ENABLED
297	PRTLOGGER pLogger = RTLogDefaultInstance();
298	if (pLogger)
299	{
300	pVM->vmm.s.cbRCLogger = RT_OFFSETOF(RTLOGGERRC, afGroups[pLogger->cGroups]);
301	rc = MMR3HyperAllocOnceNoRel(pVM, pVM->vmm.s.cbRCLogger, 0, MM_TAG_VMM, (void **)&pVM->vmm.s.pRCLoggerR3);
302	if (RT_FAILURE(rc))
303	return rc;
304	pVM->vmm.s.pRCLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCLoggerR3);
305
306	# ifdef VBOX_WITH_R0_LOGGING
307	for (VMCPUID i = 0; i < pVM->cCPUs; i++)
308	{
309	PVMCPU pVCpu = &pVM->aCpus[i];
310
311	rc = MMR3HyperAllocOnceNoRelEx(pVM, RT_OFFSETOF(VMMR0LOGGER, Logger.afGroups[pLogger->cGroups]),
312	0, MM_TAG_VMM, MMHYPER_AONR_FLAGS_KERNEL_MAPPING,
313	(void **)&pVCpu->vmm.s.pR0LoggerR3);
314	if (RT_FAILURE(rc))
315	return rc;
316	pVCpu->vmm.s.pR0LoggerR3->pVM = pVM->pVMR0;
317	//pVCpu->vmm.s.pR0LoggerR3->fCreated = false;
318	pVCpu->vmm.s.pR0LoggerR3->cbLogger = RT_OFFSETOF(RTLOGGER, afGroups[pLogger->cGroups]);
319	pVCpu->vmm.s.pR0LoggerR0 = MMHyperR3ToR0(pVM, pVCpu->vmm.s.pR0LoggerR3);
320	}
321	# endif
322	}
323	#endif /* LOG_ENABLED */
324
325	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
326	/*
327	* Allocate RC release logger instances (finalized in the relocator).
328	*/
329	PRTLOGGER pRelLogger = RTLogRelDefaultInstance();
330	if (pRelLogger)
331	{
332	pVM->vmm.s.cbRCRelLogger = RT_OFFSETOF(RTLOGGERRC, afGroups[pRelLogger->cGroups]);
333	rc = MMR3HyperAllocOnceNoRel(pVM, pVM->vmm.s.cbRCRelLogger, 0, MM_TAG_VMM, (void **)&pVM->vmm.s.pRCRelLoggerR3);
334	if (RT_FAILURE(rc))
335	return rc;
336	pVM->vmm.s.pRCRelLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCRelLoggerR3);
337	}
338	#endif /* VBOX_WITH_RC_RELEASE_LOGGING */
339	return VINF_SUCCESS;
340	}
341
342
343	/**
344	* VMMR3Init worker that register the statistics with STAM.
345	*
346	* @param pVM The shared VM structure.
347	*/
348	static void vmmR3InitRegisterStats(PVM pVM)
349	{
350	/*
351	* Statistics.
352	*/
353	STAM_REG(pVM, &pVM->vmm.s.StatRunRC, STAMTYPE_COUNTER, "/VMM/RunRC", STAMUNIT_OCCURENCES, "Number of context switches.");
354	STAM_REG(pVM, &pVM->vmm.s.StatRZRetNormal, STAMTYPE_COUNTER, "/VMM/RZRet/Normal", STAMUNIT_OCCURENCES, "Number of VINF_SUCCESS returns.");
355	STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterrupt, STAMTYPE_COUNTER, "/VMM/RZRet/Interrupt", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT returns.");
356	STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterruptHyper, STAMTYPE_COUNTER, "/VMM/RZRet/InterruptHyper", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT_HYPER returns.");
357	STAM_REG(pVM, &pVM->vmm.s.StatRZRetGuestTrap, STAMTYPE_COUNTER, "/VMM/RZRet/GuestTrap", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_GUEST_TRAP returns.");
358	STAM_REG(pVM, &pVM->vmm.s.StatRZRetRingSwitch, STAMTYPE_COUNTER, "/VMM/RZRet/RingSwitch", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_RING_SWITCH returns.");
359	STAM_REG(pVM, &pVM->vmm.s.StatRZRetRingSwitchInt, STAMTYPE_COUNTER, "/VMM/RZRet/RingSwitchInt", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_RING_SWITCH_INT returns.");
360	STAM_REG(pVM, &pVM->vmm.s.StatRZRetStaleSelector, STAMTYPE_COUNTER, "/VMM/RZRet/StaleSelector", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_STALE_SELECTOR returns.");
361	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIRETTrap, STAMTYPE_COUNTER, "/VMM/RZRet/IRETTrap", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_IRET_TRAP returns.");
362	STAM_REG(pVM, &pVM->vmm.s.StatRZRetEmulate, STAMTYPE_COUNTER, "/VMM/RZRet/Emulate", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION returns.");
363	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIOBlockEmulate, STAMTYPE_COUNTER, "/VMM/RZRet/EmulateIOBlock", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_EMULATE_IO_BLOCK returns.");
364	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchEmulate, STAMTYPE_COUNTER, "/VMM/RZRet/PatchEmulate", STAMUNIT_OCCURENCES, "Number of VINF_PATCH_EMULATE_INSTR returns.");
365	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIORead, STAMTYPE_COUNTER, "/VMM/RZRet/IORead", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_IOPORT_READ returns.");
366	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIOWrite, STAMTYPE_COUNTER, "/VMM/RZRet/IOWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_IOPORT_WRITE returns.");
367	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIORead, STAMTYPE_COUNTER, "/VMM/RZRet/MMIORead", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_READ returns.");
368	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOWrite, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_WRITE returns.");
369	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOReadWrite, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOReadWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_READ_WRITE returns.");
370	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOPatchRead, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOPatchRead", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_PATCH_READ returns.");
371	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOPatchWrite, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOPatchWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_PATCH_WRITE returns.");
372	STAM_REG(pVM, &pVM->vmm.s.StatRZRetLDTFault, STAMTYPE_COUNTER, "/VMM/RZRet/LDTFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_GDT_FAULT returns.");
373	STAM_REG(pVM, &pVM->vmm.s.StatRZRetGDTFault, STAMTYPE_COUNTER, "/VMM/RZRet/GDTFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_LDT_FAULT returns.");
374	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIDTFault, STAMTYPE_COUNTER, "/VMM/RZRet/IDTFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_IDT_FAULT returns.");
375	STAM_REG(pVM, &pVM->vmm.s.StatRZRetTSSFault, STAMTYPE_COUNTER, "/VMM/RZRet/TSSFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_TSS_FAULT returns.");
376	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPDFault, STAMTYPE_COUNTER, "/VMM/RZRet/PDFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_PD_FAULT returns.");
377	STAM_REG(pVM, &pVM->vmm.s.StatRZRetCSAMTask, STAMTYPE_COUNTER, "/VMM/RZRet/CSAMTask", STAMUNIT_OCCURENCES, "Number of VINF_CSAM_PENDING_ACTION returns.");
378	STAM_REG(pVM, &pVM->vmm.s.StatRZRetSyncCR3, STAMTYPE_COUNTER, "/VMM/RZRet/SyncCR", STAMUNIT_OCCURENCES, "Number of VINF_PGM_SYNC_CR3 returns.");
379	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMisc, STAMTYPE_COUNTER, "/VMM/RZRet/Misc", STAMUNIT_OCCURENCES, "Number of misc returns.");
380	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchInt3, STAMTYPE_COUNTER, "/VMM/RZRet/PatchInt3", STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_INT3 returns.");
381	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchPF, STAMTYPE_COUNTER, "/VMM/RZRet/PatchPF", STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_TRAP_PF returns.");
382	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchGP, STAMTYPE_COUNTER, "/VMM/RZRet/PatchGP", STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_TRAP_GP returns.");
383	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchIretIRQ, STAMTYPE_COUNTER, "/VMM/RZRet/PatchIret", STAMUNIT_OCCURENCES, "Number of VINF_PATM_PENDING_IRQ_AFTER_IRET returns.");
384	STAM_REG(pVM, &pVM->vmm.s.StatRZRetRescheduleREM, STAMTYPE_COUNTER, "/VMM/RZRet/ScheduleREM", STAMUNIT_OCCURENCES, "Number of VINF_EM_RESCHEDULE_REM returns.");
385	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
386	STAM_REG(pVM, &pVM->vmm.s.StatRZRetTimerPending, STAMTYPE_COUNTER, "/VMM/RZRet/TimerPending", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TIMER_PENDING returns.");
387	STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterruptPending, STAMTYPE_COUNTER, "/VMM/RZRet/InterruptPending", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT_PENDING returns.");
388	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPATMDuplicateFn, STAMTYPE_COUNTER, "/VMM/RZRet/PATMDuplicateFn", STAMUNIT_OCCURENCES, "Number of VINF_PATM_DUPLICATE_FUNCTION returns.");
389	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPGMChangeMode, STAMTYPE_COUNTER, "/VMM/RZRet/PGMChangeMode", STAMUNIT_OCCURENCES, "Number of VINF_PGM_CHANGE_MODE returns.");
390	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPendingRequest, STAMTYPE_COUNTER, "/VMM/RZRet/PendingRequest", STAMUNIT_OCCURENCES, "Number of VINF_EM_PENDING_REQUEST returns.");
391	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchTPR, STAMTYPE_COUNTER, "/VMM/RZRet/PatchTPR", STAMUNIT_OCCURENCES, "Number of VINF_EM_HWACCM_PATCH_TPR_INSTR returns.");
392	STAM_REG(pVM, &pVM->vmm.s.StatRZRetCallRing3, STAMTYPE_COUNTER, "/VMM/RZCallR3/Misc", STAMUNIT_OCCURENCES, "Number of Other ring-3 calls.");
393	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPDMLock, STAMTYPE_COUNTER, "/VMM/RZCallR3/PDMLock", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PDM_LOCK calls.");
394	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPDMQueueFlush, STAMTYPE_COUNTER, "/VMM/RZCallR3/PDMQueueFlush", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PDM_QUEUE_FLUSH calls.");
395	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMLock, STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMLock", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_LOCK calls.");
396	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMPoolGrow, STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMPoolGrow", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_POOL_GROW calls.");
397	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMMapChunk, STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMMapChunk", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_MAP_CHUNK calls.");
398	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMAllocHandy, STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMAllocHandy", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES calls.");
399	STAM_REG(pVM, &pVM->vmm.s.StatRZCallRemReplay, STAMTYPE_COUNTER, "/VMM/RZCallR3/REMReplay", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_REM_REPLAY_HANDLER_NOTIFICATIONS calls.");
400	STAM_REG(pVM, &pVM->vmm.s.StatRZCallLogFlush, STAMTYPE_COUNTER, "/VMM/RZCallR3/VMMLogFlush", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VMM_LOGGER_FLUSH calls.");
401	STAM_REG(pVM, &pVM->vmm.s.StatRZCallVMSetError, STAMTYPE_COUNTER, "/VMM/RZCallR3/VMSetError", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VM_SET_ERROR calls.");
402	STAM_REG(pVM, &pVM->vmm.s.StatRZCallVMSetRuntimeError, STAMTYPE_COUNTER, "/VMM/RZCallR3/VMRuntimeError", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VM_SET_RUNTIME_ERROR calls.");
403
404	#ifdef VBOX_WITH_STATISTICS
405	for (VMCPUID i = 0; i < pVM->cCPUs; i++)
406	{
407	STAMR3RegisterF(pVM, &pVM->aCpus[i].vmm.s.CallRing3JmpBufR0.cbUsedMax, STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Max amount of stack used.", "/VMM/Stack/CPU%u/Max", i);
408	STAMR3RegisterF(pVM, &pVM->aCpus[i].vmm.s.CallRing3JmpBufR0.cbUsedAvg, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Average stack usage.", "/VMM/Stack/CPU%u/Avg", i);
409	STAMR3RegisterF(pVM, &pVM->aCpus[i].vmm.s.CallRing3JmpBufR0.cUsedTotal, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of stack usages.", "/VMM/Stack/CPU%u/Uses", i);
410	}
411	#endif
412	}
413
414
415	/**
416	* Initializes the per-VCPU VMM.
417	*
418	* @returns VBox status code.
419	* @param pVM The VM to operate on.
420	*/
421	VMMR3DECL(int) VMMR3InitCPU(PVM pVM)
422	{
423	LogFlow(("VMMR3InitCPU\n"));
424	return VINF_SUCCESS;
425	}
426
427
428	/**
429	* Ring-3 init finalizing.
430	*
431	* @returns VBox status code.
432	* @param pVM The VM handle.
433	*/
434	VMMR3DECL(int) VMMR3InitFinalize(PVM pVM)
435	{
436	int rc = VINF_SUCCESS;
437
438	for (VMCPUID idCpu = 0; idCpu < pVM->cCPUs; idCpu++)
439	{
440	PVMCPU pVCpu = &pVM->aCpus[idCpu];
441
442	#ifdef VBOX_STRICT_VMM_STACK
443	/*
444	* Two inaccessible pages at each sides of the stack to catch over/under-flows.
445	*/
446	memset(pVCpu->vmm.s.pbEMTStackR3 - PAGE_SIZE, 0xcc, PAGE_SIZE);
447	MMR3HyperSetGuard(pVM, pVCpu->vmm.s.pbEMTStackR3 - PAGE_SIZE, PAGE_SIZE, true /fSet/);
448
449	memset(pVCpu->vmm.s.pbEMTStackR3 + VMM_STACK_SIZE, 0xcc, PAGE_SIZE);
450	MMR3HyperSetGuard(pVM, pVCpu->vmm.s.pbEMTStackR3 + VMM_STACK_SIZE, PAGE_SIZE, true /fSet/);
451	#endif
452
453	/*
454	* Set page attributes to r/w for stack pages.
455	*/
456	rc = PGMMapSetPage(pVM, pVCpu->vmm.s.pbEMTStackRC, VMM_STACK_SIZE, X86_PTE_P \| X86_PTE_A \| X86_PTE_D \| X86_PTE_RW);
457	AssertRC(rc);
458	if (RT_FAILURE(rc))
459	break;
460	}
461	if (RT_SUCCESS(rc))
462	{
463	/*
464	* Create the EMT yield timer.
465	*/
466	rc = TMR3TimerCreateInternal(pVM, TMCLOCK_REAL, vmmR3YieldEMT, NULL, "EMT Yielder", &pVM->vmm.s.pYieldTimer);
467	if (RT_SUCCESS(rc))
468	rc = TMTimerSetMillies(pVM->vmm.s.pYieldTimer, pVM->vmm.s.cYieldEveryMillies);
469	}
470
471	#ifdef VBOX_WITH_NMI
472	/*
473	* Map the host APIC into GC - This is AMD/Intel + Host OS specific!
474	*/
475	if (RT_SUCCESS(rc))
476	rc = PGMMap(pVM, pVM->vmm.s.GCPtrApicBase, 0xfee00000, PAGE_SIZE,
477	X86_PTE_P \| X86_PTE_RW \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_A \| X86_PTE_D);
478	#endif
479	return rc;
480	}
481
482
483	/**
484	* Initializes the R0 VMM.
485	*
486	* @returns VBox status code.
487	* @param pVM The VM to operate on.
488	*/
489	VMMR3DECL(int) VMMR3InitR0(PVM pVM)
490	{
491	int rc;
492	PVMCPU pVCpu = VMMGetCpu(pVM);
493	Assert(pVCpu && pVCpu->idCpu == 0);
494
495	#ifdef LOG_ENABLED
496	/*
497	* Initialize the ring-0 logger if we haven't done so yet.
498	*/
499	if ( pVCpu->vmm.s.pR0LoggerR3
500	&& !pVCpu->vmm.s.pR0LoggerR3->fCreated)
501	{
502	rc = VMMR3UpdateLoggers(pVM);
503	if (RT_FAILURE(rc))
504	return rc;
505	}
506	#endif
507
508	/*
509	* Call Ring-0 entry with init code.
510	*/
511	for (;;)
512	{
513	#ifdef NO_SUPCALLR0VMM
514	//rc = VERR_GENERAL_FAILURE;
515	rc = VINF_SUCCESS;
516	#else
517	rc = SUPR3CallVMMR0Ex(pVM->pVMR0, 0 /idCpu/, VMMR0_DO_VMMR0_INIT, VMMGetSvnRev(), NULL);
518	#endif
519	/*
520	* Flush the logs.
521	*/
522	#ifdef LOG_ENABLED
523	if ( pVCpu->vmm.s.pR0LoggerR3
524	&& pVCpu->vmm.s.pR0LoggerR3->Logger.offScratch > 0)
525	RTLogFlushToLogger(&pVCpu->vmm.s.pR0LoggerR3->Logger, NULL);
526	#endif
527	if (rc != VINF_VMM_CALL_HOST)
528	break;
529	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
530	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
531	break;
532	/* Resume R0 */
533	}
534
535	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
536	{
537	LogRel(("R0 init failed, rc=%Rra\n", rc));
538	if (RT_SUCCESS(rc))
539	rc = VERR_INTERNAL_ERROR;
540	}
541	return rc;
542	}
543
544
545	/**
546	* Initializes the RC VMM.
547	*
548	* @returns VBox status code.
549	* @param pVM The VM to operate on.
550	*/
551	VMMR3DECL(int) VMMR3InitRC(PVM pVM)
552	{
553	PVMCPU pVCpu = VMMGetCpu(pVM);
554	Assert(pVCpu && pVCpu->idCpu == 0);
555
556	/* In VMX mode, there's no need to init RC. */
557	if (pVM->vmm.s.fSwitcherDisabled)
558	return VINF_SUCCESS;
559
560	AssertReturn(pVM->cCPUs == 1, VERR_RAW_MODE_INVALID_SMP);
561
562	/*
563	* Call VMMGCInit():
564	* -# resolve the address.
565	* -# setup stackframe and EIP to use the trampoline.
566	* -# do a generic hypervisor call.
567	*/
568	RTRCPTR RCPtrEP;
569	int rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "VMMGCEntry", &RCPtrEP);
570	if (RT_SUCCESS(rc))
571	{
572	CPUMHyperSetCtxCore(pVCpu, NULL);
573	CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC); /* Clear the stack. */
574	uint64_t u64TS = RTTimeProgramStartNanoTS();
575	CPUMPushHyper(pVCpu, (uint32_t)(u64TS >> 32)); /* Param 3: The program startup TS - Hi. */
576	CPUMPushHyper(pVCpu, (uint32_t)u64TS); /* Param 3: The program startup TS - Lo. */
577	CPUMPushHyper(pVCpu, VMMGetSvnRev()); /* Param 2: Version argument. */
578	CPUMPushHyper(pVCpu, VMMGC_DO_VMMGC_INIT); /* Param 1: Operation. */
579	CPUMPushHyper(pVCpu, pVM->pVMRC); /* Param 0: pVM */
580	CPUMPushHyper(pVCpu, 5 * sizeof(RTRCPTR)); /* trampoline param: stacksize. */
581	CPUMPushHyper(pVCpu, RCPtrEP); /* Call EIP. */
582	CPUMSetHyperEIP(pVCpu, pVM->vmm.s.pfnCallTrampolineRC);
583	Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
584
585	for (;;)
586	{
587	#ifdef NO_SUPCALLR0VMM
588	//rc = VERR_GENERAL_FAILURE;
589	rc = VINF_SUCCESS;
590	#else
591	rc = SUPR3CallVMMR0(pVM->pVMR0, 0 /* VCPU 0 */, VMMR0_DO_CALL_HYPERVISOR, NULL);
592	#endif
593	#ifdef LOG_ENABLED
594	PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
595	if ( pLogger
596	&& pLogger->offScratch > 0)
597	RTLogFlushRC(NULL, pLogger);
598	#endif
599	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
600	PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
601	if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
602	RTLogFlushRC(RTLogRelDefaultInstance(), pRelLogger);
603	#endif
604	if (rc != VINF_VMM_CALL_HOST)
605	break;
606	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
607	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
608	break;
609	}
610
611	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
612	{
613	VMMR3FatalDump(pVM, pVCpu, rc);
614	if (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST)
615	rc = VERR_INTERNAL_ERROR;
616	}
617	AssertRC(rc);
618	}
619	return rc;
620	}
621
622
623	/**
624	* Terminate the VMM bits.
625	*
626	* @returns VINF_SUCCESS.
627	* @param pVM The VM handle.
628	*/
629	VMMR3DECL(int) VMMR3Term(PVM pVM)
630	{
631	PVMCPU pVCpu = VMMGetCpu(pVM);
632	Assert(pVCpu && pVCpu->idCpu == 0);
633
634	/*
635	* Call Ring-0 entry with termination code.
636	*/
637	int rc;
638	for (;;)
639	{
640	#ifdef NO_SUPCALLR0VMM
641	//rc = VERR_GENERAL_FAILURE;
642	rc = VINF_SUCCESS;
643	#else
644	rc = SUPR3CallVMMR0Ex(pVM->pVMR0, 0 /idCpu/, VMMR0_DO_VMMR0_TERM, 0, NULL);
645	#endif
646	/*
647	* Flush the logs.
648	*/
649	#ifdef LOG_ENABLED
650	if ( pVCpu->vmm.s.pR0LoggerR3
651	&& pVCpu->vmm.s.pR0LoggerR3->Logger.offScratch > 0)
652	RTLogFlushToLogger(&pVCpu->vmm.s.pR0LoggerR3->Logger, NULL);
653	#endif
654	if (rc != VINF_VMM_CALL_HOST)
655	break;
656	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
657	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
658	break;
659	/* Resume R0 */
660	}
661	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
662	{
663	LogRel(("VMMR3Term: R0 term failed, rc=%Rra. (warning)\n", rc));
664	if (RT_SUCCESS(rc))
665	rc = VERR_INTERNAL_ERROR;
666	}
667
668	RTCritSectDelete(&pVM->vmm.s.CritSectSync);
669	RTSemEventDestroy(pVM->vmm.s.hEvtRendezvousEnterOneByOne);
670	pVM->vmm.s.hEvtRendezvousEnterOneByOne = NIL_RTSEMEVENT;
671	RTSemEventMultiDestroy(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
672	pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce = NIL_RTSEMEVENTMULTI;
673	RTSemEventMultiDestroy(pVM->vmm.s.hEvtMulRendezvousDone);
674	pVM->vmm.s.hEvtMulRendezvousDone = NIL_RTSEMEVENTMULTI;
675	RTSemEventDestroy(pVM->vmm.s.hEvtRendezvousDoneCaller);
676	pVM->vmm.s.hEvtRendezvousDoneCaller = NIL_RTSEMEVENT;
677
678	#ifdef VBOX_STRICT_VMM_STACK
679	/*
680	* Make the two stack guard pages present again.
681	*/
682	for (VMCPUID i = 0; i < pVM->cCPUs; i++)
683	{
684	MMR3HyperSetGuard(pVM, pVM->aCpus[i].vmm.s.pbEMTStackR3 - PAGE_SIZE, PAGE_SIZE, false /fSet/);
685	MMR3HyperSetGuard(pVM, pVM->aCpus[i].vmm.s.pbEMTStackR3 + VMM_STACK_SIZE, PAGE_SIZE, false /fSet/);
686	}
687	#endif
688	return rc;
689	}
690
691
692	/**
693	* Terminates the per-VCPU VMM.
694	*
695	* Termination means cleaning up and freeing all resources,
696	* the VM it self is at this point powered off or suspended.
697	*
698	* @returns VBox status code.
699	* @param pVM The VM to operate on.
700	*/
701	VMMR3DECL(int) VMMR3TermCPU(PVM pVM)
702	{
703	return VINF_SUCCESS;
704	}
705
706
707	/**
708	* Applies relocations to data and code managed by this
709	* component. This function will be called at init and
710	* whenever the VMM need to relocate it self inside the GC.
711	*
712	* The VMM will need to apply relocations to the core code.
713	*
714	* @param pVM The VM handle.
715	* @param offDelta The relocation delta.
716	*/
717	VMMR3DECL(void) VMMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
718	{
719	LogFlow(("VMMR3Relocate: offDelta=%RGv\n", offDelta));
720
721	/*
722	* Recalc the RC address.
723	*/
724	pVM->vmm.s.pvCoreCodeRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pvCoreCodeR3);
725
726	/*
727	* The stack.
728	*/
729	for (VMCPUID i = 0; i < pVM->cCPUs; i++)
730	{
731	PVMCPU pVCpu = &pVM->aCpus[i];
732
733	CPUMSetHyperESP(pVCpu, CPUMGetHyperESP(pVCpu) + offDelta);
734
735	pVCpu->vmm.s.pbEMTStackRC = MMHyperR3ToRC(pVM, pVCpu->vmm.s.pbEMTStackR3);
736	pVCpu->vmm.s.pbEMTStackBottomRC = pVCpu->vmm.s.pbEMTStackRC + VMM_STACK_SIZE;
737	}
738
739	/*
740	* All the switchers.
741	*/
742	vmmR3SwitcherRelocate(pVM, offDelta);
743
744	/*
745	* Get other RC entry points.
746	*/
747	int rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "CPUMGCResumeGuest", &pVM->vmm.s.pfnCPUMRCResumeGuest);
748	AssertReleaseMsgRC(rc, ("CPUMGCResumeGuest not found! rc=%Rra\n", rc));
749
750	rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "CPUMGCResumeGuestV86", &pVM->vmm.s.pfnCPUMRCResumeGuestV86);
751	AssertReleaseMsgRC(rc, ("CPUMGCResumeGuestV86 not found! rc=%Rra\n", rc));
752
753	/*
754	* Update the logger.
755	*/
756	VMMR3UpdateLoggers(pVM);
757	}
758
759
760	/**
761	* Updates the settings for the RC and R0 loggers.
762	*
763	* @returns VBox status code.
764	* @param pVM The VM handle.
765	*/
766	VMMR3DECL(int) VMMR3UpdateLoggers(PVM pVM)
767	{
768	/*
769	* Simply clone the logger instance (for RC).
770	*/
771	int rc = VINF_SUCCESS;
772	RTRCPTR RCPtrLoggerFlush = 0;
773
774	if (pVM->vmm.s.pRCLoggerR3
775	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
776	\|\| pVM->vmm.s.pRCRelLoggerR3
777	#endif
778	)
779	{
780	rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "vmmGCLoggerFlush", &RCPtrLoggerFlush);
781	AssertReleaseMsgRC(rc, ("vmmGCLoggerFlush not found! rc=%Rra\n", rc));
782	}
783
784	if (pVM->vmm.s.pRCLoggerR3)
785	{
786	RTRCPTR RCPtrLoggerWrapper = 0;
787	rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "vmmGCLoggerWrapper", &RCPtrLoggerWrapper);
788	AssertReleaseMsgRC(rc, ("vmmGCLoggerWrapper not found! rc=%Rra\n", rc));
789
790	pVM->vmm.s.pRCLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCLoggerR3);
791	rc = RTLogCloneRC(NULL /* default */, pVM->vmm.s.pRCLoggerR3, pVM->vmm.s.cbRCLogger,
792	RCPtrLoggerWrapper, RCPtrLoggerFlush, RTLOGFLAGS_BUFFERED);
793	AssertReleaseMsgRC(rc, ("RTLogCloneRC failed! rc=%Rra\n", rc));
794	}
795
796	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
797	if (pVM->vmm.s.pRCRelLoggerR3)
798	{
799	RTRCPTR RCPtrLoggerWrapper = 0;
800	rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "vmmGCRelLoggerWrapper", &RCPtrLoggerWrapper);
801	AssertReleaseMsgRC(rc, ("vmmGCRelLoggerWrapper not found! rc=%Rra\n", rc));
802
803	pVM->vmm.s.pRCRelLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCRelLoggerR3);
804	rc = RTLogCloneRC(RTLogRelDefaultInstance(), pVM->vmm.s.pRCRelLoggerR3, pVM->vmm.s.cbRCRelLogger,
805	RCPtrLoggerWrapper, RCPtrLoggerFlush, RTLOGFLAGS_BUFFERED);
806	AssertReleaseMsgRC(rc, ("RTLogCloneRC failed! rc=%Rra\n", rc));
807	}
808	#endif /* VBOX_WITH_RC_RELEASE_LOGGING */
809
810	#ifdef LOG_ENABLED
811	/*
812	* For the ring-0 EMT logger, we use a per-thread logger instance
813	* in ring-0. Only initialize it once.
814	*/
815	for (VMCPUID i = 0; i < pVM->cCPUs; i++)
816	{
817	PVMCPU pVCpu = &pVM->aCpus[i];
818	PVMMR0LOGGER pR0LoggerR3 = pVCpu->vmm.s.pR0LoggerR3;
819	if (pR0LoggerR3)
820	{
821	if (!pR0LoggerR3->fCreated)
822	{
823	RTR0PTR pfnLoggerWrapper = NIL_RTR0PTR;
824	rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerWrapper", &pfnLoggerWrapper);
825	AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerWrapper not found! rc=%Rra\n", rc), rc);
826
827	RTR0PTR pfnLoggerFlush = NIL_RTR0PTR;
828	rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerFlush", &pfnLoggerFlush);
829	AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerFlush not found! rc=%Rra\n", rc), rc);
830
831	rc = RTLogCreateForR0(&pR0LoggerR3->Logger, pR0LoggerR3->cbLogger,
832	(PFNRTLOGGER )&pfnLoggerWrapper, (PFNRTLOGFLUSH )&pfnLoggerFlush,
833	RTLOGFLAGS_BUFFERED, RTLOGDEST_DUMMY);
834	AssertReleaseMsgRCReturn(rc, ("RTLogCreateForR0 failed! rc=%Rra\n", rc), rc);
835
836	RTR0PTR pfnLoggerPrefix = NIL_RTR0PTR;
837	rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerPrefix", &pfnLoggerPrefix);
838	AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerPrefix not found! rc=%Rra\n", rc), rc);
839	rc = RTLogSetCustomPrefixCallback(&pR0LoggerR3->Logger, (PFNRTLOGPREFIX )&pfnLoggerPrefix, NULL);
840	AssertReleaseMsgRCReturn(rc, ("RTLogSetCustomPrefixCallback failed! rc=%Rra\n", rc), rc);
841
842	pR0LoggerR3->idCpu = i;
843	pR0LoggerR3->fCreated = true;
844	pR0LoggerR3->fFlushingDisabled = false;
845
846	}
847
848	rc = RTLogCopyGroupsAndFlags(&pR0LoggerR3->Logger, NULL /* default */, pVM->vmm.s.pRCLoggerR3->fFlags, RTLOGFLAGS_BUFFERED);
849	AssertRC(rc);
850	}
851	}
852	#endif
853	return rc;
854	}
855
856
857	/**
858	* Gets the pointer to a buffer containing the R0/RC AssertMsg1 output.
859	*
860	* @returns Pointer to the buffer.
861	* @param pVM The VM handle.
862	*/
863	VMMR3DECL(const char *) VMMR3GetRZAssertMsg1(PVM pVM)
864	{
865	if (HWACCMIsEnabled(pVM))
866	return pVM->vmm.s.szRing0AssertMsg1;
867
868	RTRCPTR RCPtr;
869	int rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_szRTAssertMsg1", &RCPtr);
870	if (RT_SUCCESS(rc))
871	return (const char *)MMHyperRCToR3(pVM, RCPtr);
872
873	return NULL;
874	}
875
876
877	/**
878	* Gets the pointer to a buffer containing the R0/RC AssertMsg2 output.
879	*
880	* @returns Pointer to the buffer.
881	* @param pVM The VM handle.
882	*/
883	VMMR3DECL(const char *) VMMR3GetRZAssertMsg2(PVM pVM)
884	{
885	if (HWACCMIsEnabled(pVM))
886	return pVM->vmm.s.szRing0AssertMsg2;
887
888	RTRCPTR RCPtr;
889	int rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_szRTAssertMsg2", &RCPtr);
890	if (RT_SUCCESS(rc))
891	return (const char *)MMHyperRCToR3(pVM, RCPtr);
892
893	return NULL;
894	}
895
896
897	/**
898	* Execute state save operation.
899	*
900	* @returns VBox status code.
901	* @param pVM VM Handle.
902	* @param pSSM SSM operation handle.
903	*/
904	static DECLCALLBACK(int) vmmR3Save(PVM pVM, PSSMHANDLE pSSM)
905	{
906	LogFlow(("vmmR3Save:\n"));
907
908	/*
909	* The hypervisor stack.
910	* Note! See note in vmmR3Load (remove this on version change).
911	*/
912	PVMCPU pVCpu0 = &pVM->aCpus[0];
913	SSMR3PutRCPtr(pSSM, pVCpu0->vmm.s.pbEMTStackBottomRC);
914	RTRCPTR RCPtrESP = CPUMGetHyperESP(pVCpu0);
915	AssertMsg(pVCpu0->vmm.s.pbEMTStackBottomRC - RCPtrESP <= VMM_STACK_SIZE, ("Bottom %RRv ESP=%RRv\n", pVCpu0->vmm.s.pbEMTStackBottomRC, RCPtrESP));
916	SSMR3PutRCPtr(pSSM, RCPtrESP);
917	SSMR3PutMem(pSSM, pVCpu0->vmm.s.pbEMTStackR3, VMM_STACK_SIZE);
918
919	/*
920	* Save the started/stopped state of all CPUs except 0 as it will always
921	* be running. This avoids breaking the saved state version. :-)
922	*/
923	for (VMCPUID i = 1; i < pVM->cCPUs; i++)
924	SSMR3PutBool(pSSM, VMCPUSTATE_IS_STARTED(VMCPU_GET_STATE(&pVM->aCpus[i])));
925
926	return SSMR3PutU32(pSSM, ~0); /* terminator */
927	}
928
929
930	/**
931	* Execute state load operation.
932	*
933	* @returns VBox status code.
934	* @param pVM VM Handle.
935	* @param pSSM SSM operation handle.
936	* @param u32Version Data layout version.
937	*/
938	static DECLCALLBACK(int) vmmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t u32Version)
939	{
940	LogFlow(("vmmR3Load:\n"));
941
942	/*
943	* Validate version.
944	*/
945	if (u32Version != VMM_SAVED_STATE_VERSION)
946	{
947	AssertMsgFailed(("vmmR3Load: Invalid version u32Version=%d!\n", u32Version));
948	return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
949	}
950
951	/*
952	* Check that the stack is in the same place, or that it's fearly empty.
953	*
954	* Note! This can be skipped next time we update saved state as we will
955	* never be in a R0/RC -> ring-3 call when saving the state. The
956	* stack and the two associated pointers are not required.
957	*/
958	RTRCPTR RCPtrStackBottom;
959	SSMR3GetRCPtr(pSSM, &RCPtrStackBottom);
960	RTRCPTR RCPtrESP;
961	int rc = SSMR3GetRCPtr(pSSM, &RCPtrESP);
962	if (RT_FAILURE(rc))
963	return rc;
964	SSMR3GetMem(pSSM, pVM->aCpus[0].vmm.s.pbEMTStackR3, VMM_STACK_SIZE);
965
966	/* Restore the VMCPU states. VCPU 0 is always started. */
967	VMCPU_SET_STATE(&pVM->aCpus[0], VMCPUSTATE_STARTED);
968	for (VMCPUID i = 1; i < pVM->cCPUs; i++)
969	{
970	bool fStarted;
971	rc = SSMR3GetBool(pSSM, &fStarted);
972	if (RT_FAILURE(rc))
973	return rc;
974	VMCPU_SET_STATE(&pVM->aCpus[i], fStarted ? VMCPUSTATE_STARTED : VMCPUSTATE_STOPPED);
975	}
976
977	/* terminator */
978	uint32_t u32;
979	rc = SSMR3GetU32(pSSM, &u32);
980	if (RT_FAILURE(rc))
981	return rc;
982	if (u32 != ~0U)
983	{
984	AssertMsgFailed(("u32=%#x\n", u32));
985	return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
986	}
987	return VINF_SUCCESS;
988	}
989
990
991	/**
992	* Resolve a builtin RC symbol.
993	*
994	* Called by PDM when loading or relocating RC modules.
995	*
996	* @returns VBox status
997	* @param pVM VM Handle.
998	* @param pszSymbol Symbol to resolv
999	* @param pRCPtrValue Where to store the symbol value.
1000	*
1001	* @remark This has to work before VMMR3Relocate() is called.
1002	*/
1003	VMMR3DECL(int) VMMR3GetImportRC(PVM pVM, const char *pszSymbol, PRTRCPTR pRCPtrValue)
1004	{
1005	if (!strcmp(pszSymbol, "g_Logger"))
1006	{
1007	if (pVM->vmm.s.pRCLoggerR3)
1008	pVM->vmm.s.pRCLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCLoggerR3);
1009	*pRCPtrValue = pVM->vmm.s.pRCLoggerRC;
1010	}
1011	else if (!strcmp(pszSymbol, "g_RelLogger"))
1012	{
1013	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
1014	if (pVM->vmm.s.pRCRelLoggerR3)
1015	pVM->vmm.s.pRCRelLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCRelLoggerR3);
1016	*pRCPtrValue = pVM->vmm.s.pRCRelLoggerRC;
1017	#else
1018	*pRCPtrValue = NIL_RTRCPTR;
1019	#endif
1020	}
1021	else
1022	return VERR_SYMBOL_NOT_FOUND;
1023	return VINF_SUCCESS;
1024	}
1025
1026
1027	/**
1028	* Suspends the CPU yielder.
1029	*
1030	* @param pVM The VM handle.
1031	*/
1032	VMMR3DECL(void) VMMR3YieldSuspend(PVM pVM)
1033	{
1034	VMCPU_ASSERT_EMT(&pVM->aCpus[0]);
1035	if (!pVM->vmm.s.cYieldResumeMillies)
1036	{
1037	uint64_t u64Now = TMTimerGet(pVM->vmm.s.pYieldTimer);
1038	uint64_t u64Expire = TMTimerGetExpire(pVM->vmm.s.pYieldTimer);
1039	if (u64Now >= u64Expire \|\| u64Expire == ~(uint64_t)0)
1040	pVM->vmm.s.cYieldResumeMillies = pVM->vmm.s.cYieldEveryMillies;
1041	else
1042	pVM->vmm.s.cYieldResumeMillies = TMTimerToMilli(pVM->vmm.s.pYieldTimer, u64Expire - u64Now);
1043	TMTimerStop(pVM->vmm.s.pYieldTimer);
1044	}
1045	pVM->vmm.s.u64LastYield = RTTimeNanoTS();
1046	}
1047
1048
1049	/**
1050	* Stops the CPU yielder.
1051	*
1052	* @param pVM The VM handle.
1053	*/
1054	VMMR3DECL(void) VMMR3YieldStop(PVM pVM)
1055	{
1056	if (!pVM->vmm.s.cYieldResumeMillies)
1057	TMTimerStop(pVM->vmm.s.pYieldTimer);
1058	pVM->vmm.s.cYieldResumeMillies = pVM->vmm.s.cYieldEveryMillies;
1059	pVM->vmm.s.u64LastYield = RTTimeNanoTS();
1060	}
1061
1062
1063	/**
1064	* Resumes the CPU yielder when it has been a suspended or stopped.
1065	*
1066	* @param pVM The VM handle.
1067	*/
1068	VMMR3DECL(void) VMMR3YieldResume(PVM pVM)
1069	{
1070	if (pVM->vmm.s.cYieldResumeMillies)
1071	{
1072	TMTimerSetMillies(pVM->vmm.s.pYieldTimer, pVM->vmm.s.cYieldResumeMillies);
1073	pVM->vmm.s.cYieldResumeMillies = 0;
1074	}
1075	}
1076
1077
1078	/**
1079	* Internal timer callback function.
1080	*
1081	* @param pVM The VM.
1082	* @param pTimer The timer handle.
1083	* @param pvUser User argument specified upon timer creation.
1084	*/
1085	static DECLCALLBACK(void) vmmR3YieldEMT(PVM pVM, PTMTIMER pTimer, void *pvUser)
1086	{
1087	/*
1088	* This really needs some careful tuning. While we shouldn't be too greedy since
1089	* that'll cause the rest of the system to stop up, we shouldn't be too nice either
1090	* because that'll cause us to stop up.
1091	*
1092	* The current logic is to use the default interval when there is no lag worth
1093	* mentioning, but when we start accumulating lag we don't bother yielding at all.
1094	*
1095	* (This depends on the TMCLOCK_VIRTUAL_SYNC to be scheduled before TMCLOCK_REAL
1096	* so the lag is up to date.)
1097	*/
1098	const uint64_t u64Lag = TMVirtualSyncGetLag(pVM);
1099	if ( u64Lag < 50000000 /* 50ms */
1100	\|\| ( u64Lag < 1000000000 /* 1s */
1101	&& RTTimeNanoTS() - pVM->vmm.s.u64LastYield < 500000000 /* 500 ms */)
1102	)
1103	{
1104	uint64_t u64Elapsed = RTTimeNanoTS();
1105	pVM->vmm.s.u64LastYield = u64Elapsed;
1106
1107	RTThreadYield();
1108
1109	#ifdef LOG_ENABLED
1110	u64Elapsed = RTTimeNanoTS() - u64Elapsed;
1111	Log(("vmmR3YieldEMT: %RI64 ns\n", u64Elapsed));
1112	#endif
1113	}
1114	TMTimerSetMillies(pTimer, pVM->vmm.s.cYieldEveryMillies);
1115	}
1116
1117
1118	/**
1119	* Executes guest code in the raw-mode context.
1120	*
1121	* @param pVM VM handle.
1122	* @param pVCpu The VMCPU to operate on.
1123	*/
1124	VMMR3DECL(int) VMMR3RawRunGC(PVM pVM, PVMCPU pVCpu)
1125	{
1126	Log2(("VMMR3RawRunGC: (cs:eip=%04x:%08x)\n", CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
1127
1128	AssertReturn(pVM->cCPUs == 1, VERR_RAW_MODE_INVALID_SMP);
1129
1130	/*
1131	* Set the EIP and ESP.
1132	*/
1133	CPUMSetHyperEIP(pVCpu, CPUMGetGuestEFlags(pVCpu) & X86_EFL_VM
1134	? pVM->vmm.s.pfnCPUMRCResumeGuestV86
1135	: pVM->vmm.s.pfnCPUMRCResumeGuest);
1136	CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC);
1137
1138	/*
1139	* We hide log flushes (outer) and hypervisor interrupts (inner).
1140	*/
1141	for (;;)
1142	{
1143	#ifdef VBOX_STRICT
1144	if (RT_UNLIKELY(!CPUMGetHyperCR3(pVCpu) \|\| CPUMGetHyperCR3(pVCpu) != PGMGetHyperCR3(pVCpu)))
1145	EMR3FatalError(pVCpu, VERR_VMM_HYPER_CR3_MISMATCH);
1146	PGMMapCheck(pVM);
1147	#endif
1148	int rc;
1149	do
1150	{
1151	#ifdef NO_SUPCALLR0VMM
1152	rc = VERR_GENERAL_FAILURE;
1153	#else
1154	rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
1155	if (RT_LIKELY(rc == VINF_SUCCESS))
1156	rc = pVCpu->vmm.s.iLastGZRc;
1157	#endif
1158	} while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
1159
1160	/*
1161	* Flush the logs.
1162	*/
1163	#ifdef LOG_ENABLED
1164	PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
1165	if ( pLogger
1166	&& pLogger->offScratch > 0)
1167	RTLogFlushRC(NULL, pLogger);
1168	#endif
1169	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
1170	PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
1171	if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
1172	RTLogFlushRC(RTLogRelDefaultInstance(), pRelLogger);
1173	#endif
1174	if (rc != VINF_VMM_CALL_HOST)
1175	{
1176	Log2(("VMMR3RawRunGC: returns %Rrc (cs:eip=%04x:%08x)\n", rc, CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
1177	return rc;
1178	}
1179	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
1180	if (RT_FAILURE(rc))
1181	return rc;
1182	/* Resume GC */
1183	}
1184	}
1185
1186
1187	/**
1188	* Executes guest code (Intel VT-x and AMD-V).
1189	*
1190	* @param pVM VM handle.
1191	* @param pVCpu The VMCPU to operate on.
1192	*/
1193	VMMR3DECL(int) VMMR3HwAccRunGC(PVM pVM, PVMCPU pVCpu)
1194	{
1195	Log2(("VMMR3HwAccRunGC: (cs:eip=%04x:%08x)\n", CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
1196
1197	for (;;)
1198	{
1199	int rc;
1200	do
1201	{
1202	#ifdef NO_SUPCALLR0VMM
1203	rc = VERR_GENERAL_FAILURE;
1204	#else
1205	rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_HWACC_RUN, pVCpu->idCpu);
1206	if (RT_LIKELY(rc == VINF_SUCCESS))
1207	rc = pVCpu->vmm.s.iLastGZRc;
1208	#endif
1209	} while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
1210
1211	#ifdef LOG_ENABLED
1212	/*
1213	* Flush the log
1214	*/
1215	PVMMR0LOGGER pR0LoggerR3 = pVCpu->vmm.s.pR0LoggerR3;
1216	if ( pR0LoggerR3
1217	&& pR0LoggerR3->Logger.offScratch > 0)
1218	RTLogFlushToLogger(&pR0LoggerR3->Logger, NULL);
1219	#endif /* !LOG_ENABLED */
1220	if (rc != VINF_VMM_CALL_HOST)
1221	{
1222	Log2(("VMMR3HwAccRunGC: returns %Rrc (cs:eip=%04x:%08x)\n", rc, CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
1223	return rc;
1224	}
1225	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
1226	if (RT_FAILURE(rc))
1227	return rc;
1228	/* Resume R0 */
1229	}
1230	}
1231
1232	/**
1233	* VCPU worker for VMMSendSipi.
1234	*
1235	* @param pVM The VM to operate on.
1236	* @param idCpu Virtual CPU to perform SIPI on
1237	* @param uVector SIPI vector
1238	*/
1239	DECLCALLBACK(int) vmmR3SendSipi(PVM pVM, VMCPUID idCpu, uint32_t uVector)
1240	{
1241	PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
1242	VMCPU_ASSERT_EMT(pVCpu);
1243
1244	/** @todo what are we supposed to do if the processor is already running? */
1245	if (EMGetState(pVCpu) != EMSTATE_WAIT_SIPI)
1246	return VERR_ACCESS_DENIED;
1247
1248
1249	PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
1250
1251	pCtx->cs = uVector << 8;
1252	pCtx->csHid.u64Base = uVector << 12;
1253	pCtx->csHid.u32Limit = 0x0000ffff;
1254	pCtx->rip = 0;
1255
1256	Log(("vmmR3SendSipi for VCPU %d with vector %x\n", uVector));
1257
1258	# if 1 /* If we keep the EMSTATE_WAIT_SIPI method, then move this to EM.cpp. */
1259	EMSetState(pVCpu, EMSTATE_HALTED);
1260	return VINF_EM_RESCHEDULE;
1261	# else /* And if we go the VMCPU::enmState way it can stay here. */
1262	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STOPPED);
1263	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
1264	return VINF_SUCCESS;
1265	# endif
1266	}
1267
1268	DECLCALLBACK(int) vmmR3SendInitIpi(PVM pVM, VMCPUID idCpu)
1269	{
1270	PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
1271	VMCPU_ASSERT_EMT(pVCpu);
1272
1273	Log(("vmmR3SendInitIpi for VCPU %d\n", idCpu));
1274	CPUMR3ResetCpu(pVCpu);
1275	return VINF_EM_WAIT_SIPI;
1276	}
1277
1278	/**
1279	* Sends SIPI to the virtual CPU by setting CS:EIP into vector-dependent state
1280	* and unhalting processor
1281	*
1282	* @param pVM The VM to operate on.
1283	* @param idCpu Virtual CPU to perform SIPI on
1284	* @param uVector SIPI vector
1285	*/
1286	VMMR3DECL(void) VMMR3SendSipi(PVM pVM, VMCPUID idCpu, uint32_t uVector)
1287	{
1288	AssertReturnVoid(idCpu < pVM->cCPUs);
1289
1290	PVMREQ pReq;
1291	int rc = VMR3ReqCallU(pVM->pUVM, idCpu, &pReq, 0, VMREQFLAGS_NO_WAIT,
1292	(PFNRT)vmmR3SendSipi, 3, pVM, idCpu, uVector);
1293	AssertRC(rc);
1294	}
1295
1296	/**
1297	* Sends init IPI to the virtual CPU.
1298	*
1299	* @param pVM The VM to operate on.
1300	* @param idCpu Virtual CPU to perform int IPI on
1301	*/
1302	VMMR3DECL(void) VMMR3SendInitIpi(PVM pVM, VMCPUID idCpu)
1303	{
1304	AssertReturnVoid(idCpu < pVM->cCPUs);
1305
1306	PVMREQ pReq;
1307	int rc = VMR3ReqCallU(pVM->pUVM, idCpu, &pReq, 0, VMREQFLAGS_NO_WAIT,
1308	(PFNRT)vmmR3SendInitIpi, 2, pVM, idCpu);
1309	AssertRC(rc);
1310	}
1311
1312	/**
1313	* Registers the guest memory range that can be used for patching
1314	*
1315	* @returns VBox status code.
1316	* @param pVM The VM to operate on.
1317	* @param pPatchMem Patch memory range
1318	* @param cbPatchMem Size of the memory range
1319	*/
1320	VMMR3DECL(int) VMMR3RegisterPatchMemory(PVM pVM, RTGCPTR pPatchMem, unsigned cbPatchMem)
1321	{
1322	if (HWACCMIsEnabled(pVM))
1323	return HWACMMR3EnablePatching(pVM, pPatchMem, cbPatchMem);
1324
1325	return VERR_NOT_SUPPORTED;
1326	}
1327
1328	/**
1329	* Deregisters the guest memory range that can be used for patching
1330	*
1331	* @returns VBox status code.
1332	* @param pVM The VM to operate on.
1333	* @param pPatchMem Patch memory range
1334	* @param cbPatchMem Size of the memory range
1335	*/
1336	VMMR3DECL(int) VMMR3DeregisterPatchMemory(PVM pVM, RTGCPTR pPatchMem, unsigned cbPatchMem)
1337	{
1338	if (HWACCMIsEnabled(pVM))
1339	return HWACMMR3DisablePatching(pVM, pPatchMem, cbPatchMem);
1340
1341	return VINF_SUCCESS;
1342	}
1343
1344
1345	/**
1346	* VCPU worker for VMMR3SynchronizeAllVCpus.
1347	*
1348	* @param pVM The VM to operate on.
1349	* @param idCpu Virtual CPU to perform SIPI on
1350	* @param uVector SIPI vector
1351	*/
1352	DECLCALLBACK(int) vmmR3SyncVCpu(PVM pVM)
1353	{
1354	/* Block until the job in the caller has finished. */
1355	RTCritSectEnter(&pVM->vmm.s.CritSectSync);
1356	RTCritSectLeave(&pVM->vmm.s.CritSectSync);
1357	return VINF_SUCCESS;
1358	}
1359
1360
1361	/**
1362	* Atomically execute a callback handler
1363	* Note: This is very expensive; avoid using it frequently!
1364	*
1365	* @param pVM The VM to operate on.
1366	* @param pfnHandler Callback handler
1367	* @param pvUser User specified parameter
1368	*
1369	* @thread EMT
1370	*/
1371	VMMR3DECL(int) VMMR3AtomicExecuteHandler(PVM pVM, PFNATOMICHANDLER pfnHandler, void *pvUser)
1372	{
1373	int rc;
1374	PVMCPU pVCpu = VMMGetCpu(pVM);
1375	AssertReturn(pVCpu, VERR_VM_THREAD_NOT_EMT);
1376
1377	/* Shortcut for the uniprocessor case. */
1378	if (pVM->cCPUs == 1)
1379	return pfnHandler(pVM, pvUser);
1380
1381	RTCritSectEnter(&pVM->vmm.s.CritSectSync);
1382	for (VMCPUID idCpu = 0; idCpu < pVM->cCPUs; idCpu++)
1383	{
1384	if (idCpu != pVCpu->idCpu)
1385	{
1386	rc = VMR3ReqCallU(pVM->pUVM, idCpu, NULL, 0, VMREQFLAGS_NO_WAIT,
1387	(PFNRT)vmmR3SyncVCpu, 1, pVM);
1388	AssertRC(rc);
1389	}
1390	}
1391	/* Wait until all other VCPUs are waiting for us. */
1392	while (RTCritSectGetWaiters(&pVM->vmm.s.CritSectSync) != (int32_t)(pVM->cCPUs - 1))
1393	RTThreadSleep(1);
1394
1395	rc = pfnHandler(pVM, pvUser);
1396	RTCritSectLeave(&pVM->vmm.s.CritSectSync);
1397	return rc;
1398	}
1399
1400
1401	/**
1402	* Count returns and have the last non-caller EMT wake up the caller.
1403	*
1404	* @param pVM The VM handle.
1405	*/
1406	DECL_FORCE_INLINE(void) vmmR3EmtRendezvousNonCallerReturn(PVM pVM)
1407	{
1408	uint32_t cReturned = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsReturned);
1409	if (cReturned == pVM->cCPUs - 1U)
1410	{
1411	int rc = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousDoneCaller);
1412	AssertLogRelRC(rc);
1413	}
1414	}
1415
1416
1417	/**
1418	* Common worker for VMMR3EmtRendezvous and VMMR3EmtRendezvousFF.
1419	*
1420	* @param pVM The VM handle.
1421	* @param pVCpu The VMCPU structure for the calling EMT.
1422	* @param fIsCaller Whether we're the VMMR3EmtRendezvous caller or
1423	* not.
1424	* @param fFlags The flags.
1425	* @param pfnRendezvous The callback.
1426	* @param pvUser The user argument for the callback.
1427	*/
1428	static void vmmR3EmtRendezvousCommon(PVM pVM, PVMCPU pVCpu, bool fIsCaller,
1429	uint32_t fFlags, PFNVMMEMTRENDEZVOUS pfnRendezvous, void *pvUser)
1430	{
1431	int rc;
1432
1433	/*
1434	* Enter, the last EMT triggers the next callback phase.
1435	*/
1436	uint32_t cEntered = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsEntered);
1437	if (cEntered != pVM->cCPUs)
1438	{
1439	if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE)
1440	{
1441	/* Wait for our turn. */
1442	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousEnterOneByOne, RT_INDEFINITE_WAIT);
1443	AssertLogRelRC(rc);
1444	}
1445	else if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE)
1446	{
1447	/* Wait for the last EMT to arrive and wake everyone up. */
1448	rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce, RT_INDEFINITE_WAIT);
1449	AssertLogRelRC(rc);
1450	}
1451	else
1452	{
1453	Assert((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE);
1454
1455	/*
1456	* The execute once is handled specially to optimize the code flow.
1457	*
1458	* The last EMT to arrive will perform the callback and the other
1459	* EMTs will wait on the Done/DoneCaller semaphores (instead of
1460	* the EnterOneByOne/AllAtOnce) in the meanwhile. When the callback
1461	* returns, that EMT will initiate the normal return sequence.
1462	*/
1463	if (!fIsCaller)
1464	{
1465	rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousDone, RT_INDEFINITE_WAIT);
1466	AssertLogRelRC(rc);
1467
1468	vmmR3EmtRendezvousNonCallerReturn(pVM);
1469	}
1470	return;
1471	}
1472	}
1473	else
1474	{
1475	/*
1476	* All EMTs are waiting, clear the FF and take action according to the
1477	* execution method.
1478	*/
1479	VM_FF_CLEAR(pVM, VM_FF_EMT_RENDEZVOUS);
1480
1481	if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE)
1482	{
1483	/* Wake up everyone. */
1484	rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
1485	AssertLogRelRC(rc);
1486	}
1487	/* else: execute the handler on the current EMT and wake up one or more threads afterwards. */
1488	}
1489
1490
1491	/*
1492	* Do the callback and update the status if necessary.
1493	*/
1494	rc = pfnRendezvous(pVM, pVCpu, pvUser);
1495	if (rc != VINF_SUCCESS)
1496	{
1497	int32_t i32RendezvousStatus;
1498	do
1499	{
1500	i32RendezvousStatus = ASMAtomicUoReadS32(&pVM->vmm.s.i32RendezvousStatus);
1501	if ( RT_FAILURE(i32RendezvousStatus)
1502	\|\| ( i32RendezvousStatus != VINF_SUCCESS
1503	&& RT_SUCCESS(rc)))
1504	break;
1505	} while (!ASMAtomicCmpXchgS32(&pVM->vmm.s.i32RendezvousStatus, rc, i32RendezvousStatus));
1506	}
1507
1508	/*
1509	* Increment the done counter and take action depending on whether we're
1510	* the last to finish callback execution.
1511	*/
1512	uint32_t cDone = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsDone);
1513	if ( cDone != pVM->cCPUs
1514	&& (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE)
1515	{
1516	/* Signal the next EMT? */
1517	if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE)
1518	{
1519	rc = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousEnterOneByOne);
1520	AssertLogRelRC(rc);
1521	}
1522
1523	/* Wait for the rest to finish (the caller waits on hEvtRendezvousDoneCaller). */
1524	if (!fIsCaller)
1525	{
1526	rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousDone, RT_INDEFINITE_WAIT);
1527	AssertLogRelRC(rc);
1528	}
1529	}
1530	else
1531	{
1532	/* Callback execution is all done, tell the rest to return. */
1533	rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousDone);
1534	AssertLogRelRC(rc);
1535	}
1536
1537	if (!fIsCaller)
1538	vmmR3EmtRendezvousNonCallerReturn(pVM);
1539	}
1540
1541
1542	/**
1543	* Called in response to VM_FF_EMT_RENDEZVOUS.
1544	*
1545	* @param pVM The VM handle
1546	* @param pVCpu The handle of the calling EMT.
1547	*
1548	* @thread EMT
1549	*/
1550	VMMR3DECL(void) VMMR3EmtRendezvousFF(PVM pVM, PVMCPU pVCpu)
1551	{
1552	vmmR3EmtRendezvousCommon(pVM, pVCpu, false /* fIsCaller */, pVM->vmm.s.fRendezvousFlags,
1553	pVM->vmm.s.pfnRendezvous, pVM->vmm.s.pvRendezvousUser);
1554	}
1555
1556
1557	/**
1558	* EMT rendezvous.
1559	*
1560	* Gathers all the EMTs and execute some code on each of them, either in a one
1561	* by one fashion or all at once.
1562	*
1563	* @returns VBox status code. This will be the first error or, if all succeed,
1564	* the first informational status code.
1565	* @retval VERR_VM_THREAD_NOT_EMT if the caller is not an EMT.
1566	*
1567	* @param pVM The VM handle.
1568	* @param fFlags Flags indicating execution methods. See
1569	* grp_VMMR3EmtRendezvous_fFlags.
1570	* @param pfnRendezvous The callback.
1571	* @param pvUser User argument for the callback.
1572	*
1573	* @thread EMT
1574	*/
1575	VMMR3DECL(int) VMMR3EmtRendezvous(PVM pVM, uint32_t fFlags, PFNVMMEMTRENDEZVOUS pfnRendezvous, void *pvUser)
1576	{
1577	/*
1578	* Validate input.
1579	*/
1580	PVMCPU pVCpu = VMMGetCpu(pVM);
1581	AssertReturn(pVCpu, VERR_VM_THREAD_NOT_EMT);
1582	AssertMsg( (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_INVALID
1583	&& !(fFlags & ~VMMEMTRENDEZVOUS_FLAGS_VALID_MASK), ("%#x\n", fFlags));
1584
1585	int rc;
1586	if (pVM->cCPUs == 1)
1587	/*
1588	* Shortcut for the single EMT case.
1589	*/
1590	rc = pfnRendezvous(pVM, pVCpu, pvUser);
1591	else
1592	{
1593	/*
1594	* Spin lock. If busy, wait for the other EMT to finish while keeping a
1595	* lookout of the RENDEZVOUS FF.
1596	*/
1597	while (!ASMAtomicCmpXchgU32(&pVM->vmm.s.u32RendezvousLock, 0x77778888, 0))
1598	{
1599	if (VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS))
1600	VMMR3EmtRendezvousFF(pVM, pVCpu);
1601	}
1602	Assert(!VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS));
1603
1604	/*
1605	* Clear the slate. This is a semaphore ping-pong orgy. :-)
1606	*/
1607	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousEnterOneByOne, 0); AssertLogRelMsg(rc == VERR_TIMEOUT \|\| rc == VINF_SUCCESS, ("%Rrc\n", rc));
1608	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce); AssertLogRelRC(rc);
1609	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousDone); AssertLogRelRC(rc);
1610	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousDoneCaller, 0); AssertLogRelMsg(rc == VERR_TIMEOUT \|\| rc == VINF_SUCCESS, ("%Rrc\n", rc));
1611	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsEntered, 0);
1612	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsDone, 0);
1613	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsReturned, 0);
1614	ASMAtomicWriteS32(&pVM->vmm.s.i32RendezvousStatus, VINF_SUCCESS);
1615	ASMAtomicWritePtr((void * volatile )&pVM->vmm.s.pfnRendezvous, (void )(uintptr_t)pfnRendezvous);
1616	ASMAtomicWritePtr(&pVM->vmm.s.pvRendezvousUser, pvUser);
1617	ASMAtomicWriteU32(&pVM->vmm.s.fRendezvousFlags, fFlags);
1618
1619	/*
1620	* Set the FF and poke the other EMTs.
1621	*/
1622	VM_FF_SET(pVM, VM_FF_EMT_RENDEZVOUS);
1623	VMR3NotifyGlobalFFU(pVM->pUVM, VMNOTIFYFF_FLAGS_POKE);
1624
1625	/*
1626	* Do the same ourselves.
1627	*/
1628	vmmR3EmtRendezvousCommon(pVM, pVCpu, true /* fIsCaller */, fFlags, pfnRendezvous, pvUser);
1629
1630	/*
1631	* The caller waits for the other EMTs to be done and return before doing
1632	* the cleanup. This makes away with wakeup / reset races we would otherwise
1633	* risk in the multiple release event semaphore code (hEvtRendezvousDoneCaller).
1634	*/
1635	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousDoneCaller, RT_INDEFINITE_WAIT);
1636	AssertLogRelRC(rc);
1637
1638	/*
1639	* Get the return code and clean up a little bit.
1640	*/
1641	rc = pVM->vmm.s.i32RendezvousStatus;
1642	ASMAtomicWritePtr((void * volatile *)&pVM->vmm.s.pfnRendezvous, NULL);
1643
1644	ASMAtomicWriteU32(&pVM->vmm.s.u32RendezvousLock, 0);
1645	}
1646
1647	return rc;
1648	}
1649
1650
1651	/**
1652	* Read from the ring 0 jump buffer stack
1653	*
1654	* @returns VBox status code.
1655	*
1656	* @param pVM Pointer to the shared VM structure.
1657	* @param idCpu The ID of the source CPU context (for the address).
1658	* @param pAddress Where to start reading.
1659	* @param pvBuf Where to store the data we've read.
1660	* @param cbRead The number of bytes to read.
1661	*/
1662	VMMR3DECL(int) VMMR3ReadR0Stack(PVM pVM, VMCPUID idCpu, RTHCUINTPTR pAddress, void *pvBuf, size_t cbRead)
1663	{
1664	PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
1665	AssertReturn(pVCpu, VERR_INVALID_PARAMETER);
1666
1667	RTHCUINTPTR offset = pVCpu->vmm.s.CallRing3JmpBufR0.SpCheck - pAddress;
1668	if (offset >= pVCpu->vmm.s.CallRing3JmpBufR0.cbSavedStack)
1669	return VERR_INVALID_POINTER;
1670
1671	memcpy(pvBuf, pVCpu->vmm.s.pbEMTStackR3 + pVCpu->vmm.s.CallRing3JmpBufR0.cbSavedStack - offset, cbRead);
1672	return VINF_SUCCESS;
1673	}
1674
1675
1676	/**
1677	* Calls a RC function.
1678	*
1679	* @param pVM The VM handle.
1680	* @param RCPtrEntry The address of the RC function.
1681	* @param cArgs The number of arguments in the ....
1682	* @param ... Arguments to the function.
1683	*/
1684	VMMR3DECL(int) VMMR3CallRC(PVM pVM, RTRCPTR RCPtrEntry, unsigned cArgs, ...)
1685	{
1686	va_list args;
1687	va_start(args, cArgs);
1688	int rc = VMMR3CallRCV(pVM, RCPtrEntry, cArgs, args);
1689	va_end(args);
1690	return rc;
1691	}
1692
1693
1694	/**
1695	* Calls a RC function.
1696	*
1697	* @param pVM The VM handle.
1698	* @param RCPtrEntry The address of the RC function.
1699	* @param cArgs The number of arguments in the ....
1700	* @param args Arguments to the function.
1701	*/
1702	VMMR3DECL(int) VMMR3CallRCV(PVM pVM, RTRCPTR RCPtrEntry, unsigned cArgs, va_list args)
1703	{
1704	/* Raw mode implies 1 VCPU. */
1705	AssertReturn(pVM->cCPUs == 1, VERR_RAW_MODE_INVALID_SMP);
1706	PVMCPU pVCpu = &pVM->aCpus[0];
1707
1708	Log2(("VMMR3CallGCV: RCPtrEntry=%RRv cArgs=%d\n", RCPtrEntry, cArgs));
1709
1710	/*
1711	* Setup the call frame using the trampoline.
1712	*/
1713	CPUMHyperSetCtxCore(pVCpu, NULL);
1714	memset(pVCpu->vmm.s.pbEMTStackR3, 0xaa, VMM_STACK_SIZE); /* Clear the stack. */
1715	CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC - cArgs * sizeof(RTGCUINTPTR32));
1716	PRTGCUINTPTR32 pFrame = (PRTGCUINTPTR32)(pVCpu->vmm.s.pbEMTStackR3 + VMM_STACK_SIZE) - cArgs;
1717	int i = cArgs;
1718	while (i-- > 0)
1719	*pFrame++ = va_arg(args, RTGCUINTPTR32);
1720
1721	CPUMPushHyper(pVCpu, cArgs * sizeof(RTGCUINTPTR32)); /* stack frame size */
1722	CPUMPushHyper(pVCpu, RCPtrEntry); /* what to call */
1723	CPUMSetHyperEIP(pVCpu, pVM->vmm.s.pfnCallTrampolineRC);
1724
1725	/*
1726	* We hide log flushes (outer) and hypervisor interrupts (inner).
1727	*/
1728	for (;;)
1729	{
1730	int rc;
1731	Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
1732	do
1733	{
1734	#ifdef NO_SUPCALLR0VMM
1735	rc = VERR_GENERAL_FAILURE;
1736	#else
1737	rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
1738	if (RT_LIKELY(rc == VINF_SUCCESS))
1739	rc = pVCpu->vmm.s.iLastGZRc;
1740	#endif
1741	} while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
1742
1743	/*
1744	* Flush the logs.
1745	*/
1746	#ifdef LOG_ENABLED
1747	PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
1748	if ( pLogger
1749	&& pLogger->offScratch > 0)
1750	RTLogFlushRC(NULL, pLogger);
1751	#endif
1752	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
1753	PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
1754	if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
1755	RTLogFlushRC(RTLogRelDefaultInstance(), pRelLogger);
1756	#endif
1757	if (rc == VERR_TRPM_PANIC \|\| rc == VERR_TRPM_DONT_PANIC)
1758	VMMR3FatalDump(pVM, pVCpu, rc);
1759	if (rc != VINF_VMM_CALL_HOST)
1760	{
1761	Log2(("VMMR3CallGCV: returns %Rrc (cs:eip=%04x:%08x)\n", rc, CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
1762	return rc;
1763	}
1764	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
1765	if (RT_FAILURE(rc))
1766	return rc;
1767	}
1768	}
1769
1770
1771	/**
1772	* Wrapper for SUPR3CallVMMR0Ex which will deal with VINF_VMM_CALL_HOST returns.
1773	*
1774	* @returns VBox status code.
1775	* @param pVM The VM to operate on.
1776	* @param uOperation Operation to execute.
1777	* @param u64Arg Constant argument.
1778	* @param pReqHdr Pointer to a request header. See SUPR3CallVMMR0Ex for
1779	* details.
1780	*/
1781	VMMR3DECL(int) VMMR3CallR0(PVM pVM, uint32_t uOperation, uint64_t u64Arg, PSUPVMMR0REQHDR pReqHdr)
1782	{
1783	PVMCPU pVCpu = VMMGetCpu(pVM);
1784	AssertReturn(pVCpu, VERR_VM_THREAD_NOT_EMT);
1785
1786	/*
1787	* Call Ring-0 entry with init code.
1788	*/
1789	int rc;
1790	for (;;)
1791	{
1792	#ifdef NO_SUPCALLR0VMM
1793	rc = VERR_GENERAL_FAILURE;
1794	#else
1795	rc = SUPR3CallVMMR0Ex(pVM->pVMR0, pVCpu->idCpu, uOperation, u64Arg, pReqHdr);
1796	#endif
1797	/*
1798	* Flush the logs.
1799	*/
1800	#ifdef LOG_ENABLED
1801	if ( pVCpu->vmm.s.pR0LoggerR3
1802	&& pVCpu->vmm.s.pR0LoggerR3->Logger.offScratch > 0)
1803	RTLogFlushToLogger(&pVCpu->vmm.s.pR0LoggerR3->Logger, NULL);
1804	#endif
1805	if (rc != VINF_VMM_CALL_HOST)
1806	break;
1807	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
1808	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
1809	break;
1810	/* Resume R0 */
1811	}
1812
1813	AssertLogRelMsgReturn(rc == VINF_SUCCESS \|\| VBOX_FAILURE(rc),
1814	("uOperation=%u rc=%Rrc\n", uOperation, rc),
1815	VERR_INTERNAL_ERROR);
1816	return rc;
1817	}
1818
1819
1820	/**
1821	* Resumes executing hypervisor code when interrupted by a queue flush or a
1822	* debug event.
1823	*
1824	* @returns VBox status code.
1825	* @param pVM VM handle.
1826	* @param pVCpu VMCPU handle.
1827	*/
1828	VMMR3DECL(int) VMMR3ResumeHyper(PVM pVM, PVMCPU pVCpu)
1829	{
1830	Log(("VMMR3ResumeHyper: eip=%RRv esp=%RRv\n", CPUMGetHyperEIP(pVCpu), CPUMGetHyperESP(pVCpu)));
1831	AssertReturn(pVM->cCPUs == 1, VERR_RAW_MODE_INVALID_SMP);
1832
1833	/*
1834	* We hide log flushes (outer) and hypervisor interrupts (inner).
1835	*/
1836	for (;;)
1837	{
1838	int rc;
1839	Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
1840	do
1841	{
1842	#ifdef NO_SUPCALLR0VMM
1843	rc = VERR_GENERAL_FAILURE;
1844	#else
1845	rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
1846	if (RT_LIKELY(rc == VINF_SUCCESS))
1847	rc = pVCpu->vmm.s.iLastGZRc;
1848	#endif
1849	} while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
1850
1851	/*
1852	* Flush the loggers,
1853	*/
1854	#ifdef LOG_ENABLED
1855	PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
1856	if ( pLogger
1857	&& pLogger->offScratch > 0)
1858	RTLogFlushRC(NULL, pLogger);
1859	#endif
1860	#ifdef VBOX_WITH_RC_RELEASE_LOGGING
1861	PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
1862	if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
1863	RTLogFlushRC(RTLogRelDefaultInstance(), pRelLogger);
1864	#endif
1865	if (rc == VERR_TRPM_PANIC \|\| rc == VERR_TRPM_DONT_PANIC)
1866	VMMR3FatalDump(pVM, pVCpu, rc);
1867	if (rc != VINF_VMM_CALL_HOST)
1868	{
1869	Log(("VMMR3ResumeHyper: returns %Rrc\n", rc));
1870	return rc;
1871	}
1872	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
1873	if (RT_FAILURE(rc))
1874	return rc;
1875	}
1876	}
1877
1878
1879	/**
1880	* Service a call to the ring-3 host code.
1881	*
1882	* @returns VBox status code.
1883	* @param pVM VM handle.
1884	* @param pVCpu VMCPU handle
1885	* @remark Careful with critsects.
1886	*/
1887	static int vmmR3ServiceCallRing3Request(PVM pVM, PVMCPU pVCpu)
1888	{
1889	/*
1890	* We must also check for pending critsect exits or else we can deadlock
1891	* when entering other critsects here.
1892	*/
1893	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PDM_CRITSECT))
1894	PDMCritSectFF(pVCpu);
1895
1896	switch (pVCpu->vmm.s.enmCallRing3Operation)
1897	{
1898	/*
1899	* Acquire the PDM lock.
1900	*/
1901	case VMMCALLRING3_PDM_LOCK:
1902	{
1903	pVCpu->vmm.s.rcCallRing3 = PDMR3LockCall(pVM);
1904	break;
1905	}
1906
1907	/*
1908	* Flush a PDM queue.
1909	*/
1910	case VMMCALLRING3_PDM_QUEUE_FLUSH:
1911	{
1912	PDMR3QueueFlushWorker(pVM, NULL);
1913	pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
1914	break;
1915	}
1916
1917	/*
1918	* Grow the PGM pool.
1919	*/
1920	case VMMCALLRING3_PGM_POOL_GROW:
1921	{
1922	pVCpu->vmm.s.rcCallRing3 = PGMR3PoolGrow(pVM);
1923	break;
1924	}
1925
1926	/*
1927	* Maps an page allocation chunk into ring-3 so ring-0 can use it.
1928	*/
1929	case VMMCALLRING3_PGM_MAP_CHUNK:
1930	{
1931	pVCpu->vmm.s.rcCallRing3 = PGMR3PhysChunkMap(pVM, pVCpu->vmm.s.u64CallRing3Arg);
1932	break;
1933	}
1934
1935	/*
1936	* Allocates more handy pages.
1937	*/
1938	case VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES:
1939	{
1940	pVCpu->vmm.s.rcCallRing3 = PGMR3PhysAllocateHandyPages(pVM);
1941	break;
1942	}
1943
1944	/*
1945	* Acquire the PGM lock.
1946	*/
1947	case VMMCALLRING3_PGM_LOCK:
1948	{
1949	pVCpu->vmm.s.rcCallRing3 = PGMR3LockCall(pVM);
1950	break;
1951	}
1952
1953	/*
1954	* Acquire the MM hypervisor heap lock.
1955	*/
1956	case VMMCALLRING3_MMHYPER_LOCK:
1957	{
1958	pVCpu->vmm.s.rcCallRing3 = MMR3LockCall(pVM);
1959	break;
1960	}
1961
1962	/*
1963	* Flush REM handler notifications.
1964	*/
1965	case VMMCALLRING3_REM_REPLAY_HANDLER_NOTIFICATIONS:
1966	{
1967	REMR3ReplayHandlerNotifications(pVM);
1968	pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
1969	break;
1970	}
1971
1972	/*
1973	* This is a noop. We just take this route to avoid unnecessary
1974	* tests in the loops.
1975	*/
1976	case VMMCALLRING3_VMM_LOGGER_FLUSH:
1977	pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
1978	LogAlways(("FLUSH\n"));
1979	break;
1980
1981	/*
1982	* Set the VM error message.
1983	*/
1984	case VMMCALLRING3_VM_SET_ERROR:
1985	VMR3SetErrorWorker(pVM);
1986	pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
1987	break;
1988
1989	/*
1990	* Set the VM runtime error message.
1991	*/
1992	case VMMCALLRING3_VM_SET_RUNTIME_ERROR:
1993	pVCpu->vmm.s.rcCallRing3 = VMR3SetRuntimeErrorWorker(pVM);
1994	break;
1995
1996	/*
1997	* Signal a ring 0 hypervisor assertion.
1998	* Cancel the longjmp operation that's in progress.
1999	*/
2000	case VMMCALLRING3_VM_R0_ASSERTION:
2001	pVCpu->vmm.s.enmCallRing3Operation = VMMCALLRING3_INVALID;
2002	pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call = false;
2003	#ifdef RT_ARCH_X86
2004	pVCpu->vmm.s.CallRing3JmpBufR0.eip = 0;
2005	#else
2006	pVCpu->vmm.s.CallRing3JmpBufR0.rip = 0;
2007	#endif
2008	LogRel((pVM->vmm.s.szRing0AssertMsg1));
2009	LogRel((pVM->vmm.s.szRing0AssertMsg2));
2010	return VERR_VMM_RING0_ASSERTION;
2011
2012	/*
2013	* A forced switch to ring 0 for preemption purposes.
2014	*/
2015	case VMMCALLRING3_VM_R0_PREEMPT:
2016	pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
2017	break;
2018
2019	default:
2020	AssertMsgFailed(("enmCallRing3Operation=%d\n", pVCpu->vmm.s.enmCallRing3Operation));
2021	return VERR_INTERNAL_ERROR;
2022	}
2023
2024	pVCpu->vmm.s.enmCallRing3Operation = VMMCALLRING3_INVALID;
2025	return VINF_SUCCESS;
2026	}
2027
2028
2029	/**
2030	* Displays the Force action Flags.
2031	*
2032	* @param pVM The VM handle.
2033	* @param pHlp The output helpers.
2034	* @param pszArgs The additional arguments (ignored).
2035	*/
2036	static DECLCALLBACK(void) vmmR3InfoFF(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
2037	{
2038	int c;
2039	uint32_t f;
2040	#define PRINT_FLAG(prf,flag) do { \
2041	if (f & (prf##flag)) \
2042	{ \
2043	static const char *s_psz = #flag; \
2044	if (!(c % 6)) \
2045	pHlp->pfnPrintf(pHlp, "%s\n %s", c ? "," : "", s_psz); \
2046	else \
2047	pHlp->pfnPrintf(pHlp, ", %s", s_psz); \
2048	c++; \
2049	f &= ~(prf##flag); \
2050	} \
2051	} while (0)
2052
2053	#define PRINT_GROUP(prf,grp,sfx) do { \
2054	if (f & (prf##grp##sfx)) \
2055	{ \
2056	static const char *s_psz = #grp; \
2057	if (!(c % 5)) \
2058	pHlp->pfnPrintf(pHlp, "%s %s", c ? ",\n" : " Groups:\n", s_psz); \
2059	else \
2060	pHlp->pfnPrintf(pHlp, ", %s", s_psz); \
2061	c++; \
2062	} \
2063	} while (0)
2064
2065	/*
2066	* The global flags.
2067	*/
2068	const uint32_t fGlobalForcedActions = pVM->fGlobalForcedActions;
2069	pHlp->pfnPrintf(pHlp, "Global FFs: %#RX32", fGlobalForcedActions);
2070
2071	/* show the flag mnemonics */
2072	c = 0;
2073	f = fGlobalForcedActions;
2074	PRINT_FLAG(VM_FF_,TM_VIRTUAL_SYNC);
2075	PRINT_FLAG(VM_FF_,PDM_QUEUES);
2076	PRINT_FLAG(VM_FF_,PDM_DMA);
2077	PRINT_FLAG(VM_FF_,DBGF);
2078	PRINT_FLAG(VM_FF_,REQUEST);
2079	PRINT_FLAG(VM_FF_,TERMINATE);
2080	PRINT_FLAG(VM_FF_,RESET);
2081	PRINT_FLAG(VM_FF_,EMT_RENDEZVOUS);
2082	PRINT_FLAG(VM_FF_,PGM_NEED_HANDY_PAGES);
2083	PRINT_FLAG(VM_FF_,PGM_NO_MEMORY);
2084	PRINT_FLAG(VM_FF_,REM_HANDLER_NOTIFY);
2085	PRINT_FLAG(VM_FF_,DEBUG_SUSPEND);
2086	if (f)
2087	pHlp->pfnPrintf(pHlp, "%s\n Unknown bits: %#RX32\n", c ? "," : "", f);
2088	else
2089	pHlp->pfnPrintf(pHlp, "\n");
2090
2091	/* the groups */
2092	c = 0;
2093	f = fGlobalForcedActions;
2094	PRINT_GROUP(VM_FF_,EXTERNAL_SUSPENDED,_MASK);
2095	PRINT_GROUP(VM_FF_,EXTERNAL_HALTED,_MASK);
2096	PRINT_GROUP(VM_FF_,HIGH_PRIORITY_PRE,_MASK);
2097	PRINT_GROUP(VM_FF_,HIGH_PRIORITY_PRE_RAW,_MASK);
2098	PRINT_GROUP(VM_FF_,HIGH_PRIORITY_POST,_MASK);
2099	PRINT_GROUP(VM_FF_,NORMAL_PRIORITY_POST,_MASK);
2100	PRINT_GROUP(VM_FF_,NORMAL_PRIORITY,_MASK);
2101	PRINT_GROUP(VM_FF_,ALL_BUT_RAW,_MASK);
2102	if (c)
2103	pHlp->pfnPrintf(pHlp, "\n");
2104
2105	/*
2106	* Per CPU flags.
2107	*/
2108	for (VMCPUID i = 0; i < pVM->cCPUs; i++)
2109	{
2110	const uint32_t fLocalForcedActions = pVM->aCpus[i].fLocalForcedActions;
2111	pHlp->pfnPrintf(pHlp, "CPU %u FFs: %#RX32", i, fLocalForcedActions);
2112
2113	/* show the flag mnemonics */
2114	c = 0;
2115	f = fLocalForcedActions;
2116	PRINT_FLAG(VMCPU_FF_,INTERRUPT_APIC);
2117	PRINT_FLAG(VMCPU_FF_,INTERRUPT_PIC);
2118	PRINT_FLAG(VMCPU_FF_,TIMER);
2119	PRINT_FLAG(VMCPU_FF_,PDM_CRITSECT);
2120	PRINT_FLAG(VMCPU_FF_,PGM_SYNC_CR3);
2121	PRINT_FLAG(VMCPU_FF_,PGM_SYNC_CR3_NON_GLOBAL);
2122	PRINT_FLAG(VMCPU_FF_,TRPM_SYNC_IDT);
2123	PRINT_FLAG(VMCPU_FF_,SELM_SYNC_TSS);
2124	PRINT_FLAG(VMCPU_FF_,SELM_SYNC_GDT);
2125	PRINT_FLAG(VMCPU_FF_,SELM_SYNC_LDT);
2126	PRINT_FLAG(VMCPU_FF_,INHIBIT_INTERRUPTS);
2127	PRINT_FLAG(VMCPU_FF_,CSAM_SCAN_PAGE);
2128	PRINT_FLAG(VMCPU_FF_,CSAM_PENDING_ACTION);
2129	PRINT_FLAG(VMCPU_FF_,TO_R3);
2130	if (f)
2131	pHlp->pfnPrintf(pHlp, "%s\n Unknown bits: %#RX32\n", c ? "," : "", f);
2132	else
2133	pHlp->pfnPrintf(pHlp, "\n");
2134
2135	/* the groups */
2136	c = 0;
2137	f = fLocalForcedActions;
2138	PRINT_GROUP(VMCPU_FF_,EXTERNAL_SUSPENDED,_MASK);
2139	PRINT_GROUP(VMCPU_FF_,EXTERNAL_HALTED,_MASK);
2140	PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_PRE,_MASK);
2141	PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_PRE_RAW,_MASK);
2142	PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_POST,_MASK);
2143	PRINT_GROUP(VMCPU_FF_,NORMAL_PRIORITY_POST,_MASK);
2144	PRINT_GROUP(VMCPU_FF_,NORMAL_PRIORITY,_MASK);
2145	PRINT_GROUP(VMCPU_FF_,RESUME_GUEST,_MASK);
2146	PRINT_GROUP(VMCPU_FF_,HWACCM_TO_R3,_MASK);
2147	PRINT_GROUP(VMCPU_FF_,ALL_BUT_RAW,_MASK);
2148	if (c)
2149	pHlp->pfnPrintf(pHlp, "\n");
2150	}
2151
2152	#undef PRINT_FLAG
2153	#undef PRINT_GROUP
2154	}
2155

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMM.cpp@ 22309

Download in other formats: