VMMR0.cpp

Last change on this file was 106920, checked in by vboxsync, 3 weeks ago
/Config.kmk,Devices/Makefile.kmk,VMM/*: Introducing VBOX_WITH_MINIMAL_R0 for win.arm64 and similar build configurations not really needing much from VMMR0.r0/VBoxSup.sys. jiraref:VBP-1449
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 138.9 KB

Line
1	/* $Id: VMMR0.cpp 106920 2024-11-11 01:09:38Z vboxsync $ */
2	/** @file
3	* VMM - Host Context Ring 0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_VMM
33	#include <VBox/vmm/vmm.h>
34	#include <VBox/sup.h>
35	#include <VBox/vmm/iem.h>
36	#include <VBox/vmm/iom.h>
37	#include <VBox/vmm/trpm.h>
38	#include <VBox/vmm/cpum.h>
39	#include <VBox/vmm/pdmapi.h>
40	#include <VBox/vmm/pgm.h>
41	#ifdef VBOX_WITH_NEM_R0
42	# include <VBox/vmm/nem.h>
43	#endif
44	#include <VBox/vmm/em.h>
45	#include <VBox/vmm/stam.h>
46	#include <VBox/vmm/tm.h>
47	#include "VMMInternal.h"
48	#include <VBox/vmm/vmcc.h>
49	#include <VBox/vmm/gvm.h>
50	#ifdef VBOX_WITH_PCI_PASSTHROUGH
51	# include <VBox/vmm/pdmpci.h>
52	#endif
53	#include <VBox/vmm/apic.h>
54
55	#include <VBox/vmm/gvmm.h>
56	#include <VBox/vmm/gmm.h>
57	#include <VBox/vmm/gim.h>
58	#include <VBox/intnet.h>
59	#include <VBox/vmm/hm.h>
60	#include <VBox/param.h>
61	#include <VBox/err.h>
62	#include <VBox/version.h>
63	#include <VBox/log.h>
64
65	#ifdef RT_ARCH_AMD64
66	# include <iprt/asm-amd64-x86.h>
67	#endif
68	#include <iprt/assert.h>
69	#include <iprt/crc.h>
70	#include <iprt/initterm.h>
71	#include <iprt/mem.h>
72	#include <iprt/memobj.h>
73	#include <iprt/mp.h>
74	#include <iprt/once.h>
75	#include <iprt/semaphore.h>
76	#include <iprt/spinlock.h>
77	#include <iprt/stdarg.h>
78	#include <iprt/string.h>
79	#include <iprt/thread.h>
80	#include <iprt/timer.h>
81	#include <iprt/time.h>
82
83	#include "dtrace/VBoxVMM.h"
84
85
86	#if defined(_MSC_VER) && defined(RT_ARCH_AMD64) /** @todo check this with with VC7! */
87	# pragma intrinsic(_AddressOfReturnAddress)
88	#endif
89
90	#if defined(RT_OS_DARWIN) && ARCH_BITS == 32
91	# error "32-bit darwin is no longer supported. Go back to 4.3 or earlier!"
92	#endif
93
94
95	/*********************************************************************************************************************************
96	* Internal Functions *
97	*********************************************************************************************************************************/
98	RT_C_DECLS_BEGIN
99	#if defined(RT_ARCH_X86) && (defined(RT_OS_SOLARIS) \|\| defined(RT_OS_FREEBSD))
100	extern uint64_t __udivdi3(uint64_t, uint64_t);
101	extern uint64_t __umoddi3(uint64_t, uint64_t);
102	#endif
103	RT_C_DECLS_END
104	static int vmmR0UpdateLoggers(PGVM pGVM, VMCPUID idCpu, PVMMR0UPDATELOGGERSREQ pReq, uint64_t fFlags);
105	static int vmmR0LogFlusher(PGVM pGVM);
106	static int vmmR0LogWaitFlushed(PGVM pGVM, VMCPUID idCpu, size_t idxLogger);
107	static int vmmR0InitLoggers(PGVM pGVM);
108	static void vmmR0CleanupLoggers(PGVM pGVM);
109
110
111	/*********************************************************************************************************************************
112	* Global Variables *
113	*********************************************************************************************************************************/
114	/** Drag in necessary library bits.
115	* The runtime lives here (in VMMR0.r0) and VBoxDDR0.r0 links against us. /
116	struct CLANG11WEIRDNOTHROW { PFNRT pfn; } g_VMMR0Deps[] =
117	{
118	{ (PFNRT)RTCrc32 },
119	{ (PFNRT)RTOnce },
120	#if defined(RT_ARCH_X86) && (defined(RT_OS_SOLARIS) \|\| defined(RT_OS_FREEBSD))
121	{ (PFNRT)__udivdi3 },
122	{ (PFNRT)__umoddi3 },
123	#endif
124	{ NULL }
125	};
126
127	#ifdef RT_OS_SOLARIS
128	/* Dependency information for the native solaris loader. */
129	extern "C" { char _depends_on[] = "vboxdrv"; }
130	#endif
131
132
133	/**
134	* Initialize the module.
135	* This is called when we're first loaded.
136	*
137	* @returns 0 on success.
138	* @returns VBox status on failure.
139	* @param hMod Image handle for use in APIs.
140	*/
141	DECLEXPORT(int) ModuleInit(void *hMod)
142	{
143	RT_NOREF_PV(hMod);
144
145	#ifdef VBOX_WITH_DTRACE_R0
146	/*
147	* The first thing to do is register the static tracepoints.
148	* (Deregistration is automatic.)
149	*/
150	int rc2 = SUPR0TracerRegisterModule(hMod, &g_VTGObjHeader);
151	if (RT_FAILURE(rc2))
152	return rc2;
153	#endif
154	LogFlow(("ModuleInit:\n"));
155
156	#ifdef VBOX_WITH_64ON32_CMOS_DEBUG
157	/*
158	* Display the CMOS debug code.
159	*/
160	ASMOutU8(0x72, 0x03);
161	uint8_t bDebugCode = ASMInU8(0x73);
162	LogRel(("CMOS Debug Code: %#x (%d)\n", bDebugCode, bDebugCode));
163	RTLogComPrintf("CMOS Debug Code: %#x (%d)\n", bDebugCode, bDebugCode);
164	#endif
165
166	/*
167	* Initialize the VMM, GVMM, GMM, HM, PGM (Darwin) and INTNET.
168	*/
169	int rc = vmmInitFormatTypes();
170	if (RT_SUCCESS(rc))
171	{
172	rc = GVMMR0Init();
173	if (RT_SUCCESS(rc))
174	{
175	#ifndef VBOX_WITH_MINIMAL_R0
176	rc = GMMR0Init();
177	if (RT_SUCCESS(rc))
178	{
179	rc = HMR0Init();
180	if (RT_SUCCESS(rc))
181	{
182	PDMR0Init(hMod);
183
184	rc = PGMRegisterStringFormatTypes();
185	if (RT_SUCCESS(rc))
186	{
187	#endif /* !VBOX_WITH_MINIMAL_R0 */
188	rc = IntNetR0Init();
189	if (RT_SUCCESS(rc))
190	{
191	#ifndef VBOX_WITH_MINIMAL_R0
192	# ifdef VBOX_WITH_PCI_PASSTHROUGH
193	rc = PciRawR0Init();
194	# endif
195	if (RT_SUCCESS(rc))
196	{
197	rc = CPUMR0ModuleInit();
198	if (RT_SUCCESS(rc))
199	{
200	# ifdef VBOX_WITH_TRIPLE_FAULT_HACK
201	rc = vmmR0TripleFaultHackInit();
202	if (RT_SUCCESS(rc))
203	# endif
204	{
205	# ifdef VBOX_WITH_NEM_R0
206	rc = NEMR0Init();
207	if (RT_SUCCESS(rc))
208	# endif
209	#endif /* !VBOX_WITH_MINIMAL_R0 */
210	{
211	LogFlow(("ModuleInit: returns success\n"));
212	return VINF_SUCCESS;
213	}
214
215	/*
216	* Bail out.
217	*/
218	#ifndef VBOX_WITH_MINIMAL_R0
219	}
220	# ifdef VBOX_WITH_TRIPLE_FAULT_HACK
221	vmmR0TripleFaultHackTerm();
222	# endif
223	}
224	else
225	LogRel(("ModuleInit: CPUMR0ModuleInit -> %Rrc\n", rc));
226	# ifdef VBOX_WITH_PCI_PASSTHROUGH
227	PciRawR0Term();
228	# endif
229	}
230	else
231	LogRel(("ModuleInit: PciRawR0Init -> %Rrc\n", rc));
232	IntNetR0Term();
233	#endif /* !VBOX_WITH_MINIMAL_R0 */
234	}
235	else
236	LogRel(("ModuleInit: IntNetR0Init -> %Rrc\n", rc));
237	#ifndef VBOX_WITH_MINIMAL_R0
238	PGMDeregisterStringFormatTypes();
239	}
240	else
241	LogRel(("ModuleInit: PGMRegisterStringFormatTypes -> %Rrc\n", rc));
242	HMR0Term();
243	}
244	else
245	LogRel(("ModuleInit: HMR0Init -> %Rrc\n", rc));
246	GMMR0Term();
247	}
248	else
249	LogRel(("ModuleInit: GMMR0Init -> %Rrc\n", rc));
250	#endif /* !VBOX_WITH_MINIMAL_R0 */
251	GVMMR0Term();
252	}
253	else
254	LogRel(("ModuleInit: GVMMR0Init -> %Rrc\n", rc));
255	vmmTermFormatTypes();
256	}
257	else
258	LogRel(("ModuleInit: vmmInitFormatTypes -> %Rrc\n", rc));
259
260	LogFlow(("ModuleInit: failed %Rrc\n", rc));
261	return rc;
262	}
263
264
265	/**
266	* Terminate the module.
267	* This is called when we're finally unloaded.
268	*
269	* @param hMod Image handle for use in APIs.
270	*/
271	DECLEXPORT(void) ModuleTerm(void *hMod)
272	{
273	LogFlow(("ModuleTerm:\n"));
274	RT_NOREF_PV(hMod);
275
276	#ifndef VBOX_WITH_MINIMAL_R0
277	/*
278	* Terminate the CPUM module (Local APIC cleanup).
279	*/
280	CPUMR0ModuleTerm();
281	#endif
282
283	/*
284	* Terminate the internal network service.
285	*/
286	IntNetR0Term();
287
288	#ifndef VBOX_WITH_MINIMAL_R0
289	/*
290	* PGM (Darwin), HM and PciRaw global cleanup.
291	*/
292	# ifdef VBOX_WITH_PCI_PASSTHROUGH
293	PciRawR0Term();
294	# endif
295	PGMDeregisterStringFormatTypes();
296	HMR0Term();
297	# ifdef VBOX_WITH_TRIPLE_FAULT_HACK
298	vmmR0TripleFaultHackTerm();
299	# endif
300	# ifdef VBOX_WITH_NEM_R0
301	NEMR0Term();
302	# endif
303	#endif /* !VBOX_WITH_MINIMAL_R0 */
304
305	/*
306	* Destroy the GMM and GVMM instances.
307	*/
308	#ifndef VBOX_WITH_MINIMAL_R0
309	GMMR0Term();
310	#endif
311	GVMMR0Term();
312
313	vmmTermFormatTypes();
314	RTTermRunCallbacks(RTTERMREASON_UNLOAD, 0);
315
316	LogFlow(("ModuleTerm: returns\n"));
317	}
318
319
320	/**
321	* Initializes VMM specific members when the GVM structure is created,
322	* allocating loggers and stuff.
323	*
324	* The loggers are allocated here so that we can update their settings before
325	* doing VMMR0_DO_VMMR0_INIT and have correct logging at that time.
326	*
327	* @returns VBox status code.
328	* @param pGVM The global (ring-0) VM structure.
329	*/
330	VMMR0_INT_DECL(int) VMMR0InitPerVMData(PGVM pGVM)
331	{
332	AssertCompile(sizeof(pGVM->vmmr0.s) <= sizeof(pGVM->vmmr0.padding));
333
334	/*
335	* Initialize all members first.
336	*/
337	pGVM->vmmr0.s.fCalledInitVm = false;
338	pGVM->vmmr0.s.hMemObjLogger = NIL_RTR0MEMOBJ;
339	pGVM->vmmr0.s.hMapObjLogger = NIL_RTR0MEMOBJ;
340	pGVM->vmmr0.s.hMemObjReleaseLogger = NIL_RTR0MEMOBJ;
341	pGVM->vmmr0.s.hMapObjReleaseLogger = NIL_RTR0MEMOBJ;
342	pGVM->vmmr0.s.LogFlusher.hSpinlock = NIL_RTSPINLOCK;
343	pGVM->vmmr0.s.LogFlusher.hThread = NIL_RTNATIVETHREAD;
344	pGVM->vmmr0.s.LogFlusher.hEvent = NIL_RTSEMEVENT;
345	pGVM->vmmr0.s.LogFlusher.idxRingHead = 0;
346	pGVM->vmmr0.s.LogFlusher.idxRingTail = 0;
347	pGVM->vmmr0.s.LogFlusher.fThreadWaiting = false;
348
349	for (VMCPUID idCpu = 0; idCpu < pGVM->cCpus; idCpu++)
350	{
351	PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
352	Assert(pGVCpu->idHostCpu == NIL_RTCPUID);
353	Assert(pGVCpu->iHostCpuSet == UINT32_MAX);
354	pGVCpu->vmmr0.s.pPreemptState = NULL;
355	pGVCpu->vmmr0.s.hCtxHook = NIL_RTTHREADCTXHOOK;
356	pGVCpu->vmmr0.s.AssertJmpBuf.pMirrorBuf = &pGVCpu->vmm.s.AssertJmpBuf;
357	pGVCpu->vmmr0.s.AssertJmpBuf.pvStackBuf = &pGVCpu->vmm.s.abAssertStack[0];
358	pGVCpu->vmmr0.s.AssertJmpBuf.cbStackBuf = sizeof(pGVCpu->vmm.s.abAssertStack);
359
360	for (size_t iLogger = 0; iLogger < RT_ELEMENTS(pGVCpu->vmmr0.s.u.aLoggers); iLogger++)
361	pGVCpu->vmmr0.s.u.aLoggers[iLogger].hEventFlushWait = NIL_RTSEMEVENT;
362	}
363
364	/*
365	* Create the loggers.
366	*/
367	return vmmR0InitLoggers(pGVM);
368	}
369
370
371	/**
372	* Initiates the R0 driver for a particular VM instance.
373	*
374	* @returns VBox status code.
375	*
376	* @param pGVM The global (ring-0) VM structure.
377	* @param uSvnRev The SVN revision of the ring-3 part.
378	* @param uBuildType Build type indicator.
379	* @thread EMT(0)
380	*/
381	static int vmmR0InitVM(PGVM pGVM, uint32_t uSvnRev, uint32_t uBuildType)
382	{
383	/*
384	* Match the SVN revisions and build type.
385	*/
386	if (uSvnRev != VMMGetSvnRev())
387	{
388	LogRel(("VMMR0InitVM: Revision mismatch, r3=%d r0=%d\n", uSvnRev, VMMGetSvnRev()));
389	SUPR0Printf("VMMR0InitVM: Revision mismatch, r3=%d r0=%d\n", uSvnRev, VMMGetSvnRev());
390	return VERR_VMM_R0_VERSION_MISMATCH;
391	}
392	if (uBuildType != vmmGetBuildType())
393	{
394	LogRel(("VMMR0InitVM: Build type mismatch, r3=%#x r0=%#x\n", uBuildType, vmmGetBuildType()));
395	SUPR0Printf("VMMR0InitVM: Build type mismatch, r3=%#x r0=%#x\n", uBuildType, vmmGetBuildType());
396	return VERR_VMM_R0_VERSION_MISMATCH;
397	}
398
399	int rc = GVMMR0ValidateGVMandEMT(pGVM, 0 /idCpu/);
400	if (RT_FAILURE(rc))
401	return rc;
402
403	/* Don't allow this to be called more than once. */
404	if (!pGVM->vmmr0.s.fCalledInitVm)
405	pGVM->vmmr0.s.fCalledInitVm = true;
406	else
407	return VERR_ALREADY_INITIALIZED;
408
409	#ifdef LOG_ENABLED
410
411	/*
412	* Register the EMT R0 logger instance for VCPU 0.
413	*/
414	PVMCPUCC pVCpu = VMCC_GET_CPU_0(pGVM);
415	if (pVCpu->vmmr0.s.u.s.Logger.pLogger)
416	{
417	# if 0 /* testing of the logger. */
418	LogCom(("vmmR0InitVM: before %p\n", RTLogDefaultInstance()));
419	LogCom(("vmmR0InitVM: pfnFlush=%p actual=%p\n", pR0Logger->Logger.pfnFlush, vmmR0LoggerFlush));
420	LogCom(("vmmR0InitVM: pfnLogger=%p actual=%p\n", pR0Logger->Logger.pfnLogger, vmmR0LoggerWrapper));
421	LogCom(("vmmR0InitVM: offScratch=%d fFlags=%#x fDestFlags=%#x\n", pR0Logger->Logger.offScratch, pR0Logger->Logger.fFlags, pR0Logger->Logger.fDestFlags));
422
423	RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pGVM->pSession);
424	LogCom(("vmmR0InitVM: after %p reg\n", RTLogDefaultInstance()));
425	RTLogSetDefaultInstanceThread(NULL, pGVM->pSession);
426	LogCom(("vmmR0InitVM: after %p dereg\n", RTLogDefaultInstance()));
427
428	pR0Logger->Logger.pfnLogger("hello ring-0 logger\n");
429	LogCom(("vmmR0InitVM: returned successfully from direct logger call.\n"));
430	pR0Logger->Logger.pfnFlush(&pR0Logger->Logger);
431	LogCom(("vmmR0InitVM: returned successfully from direct flush call.\n"));
432
433	RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pGVM->pSession);
434	LogCom(("vmmR0InitVM: after %p reg2\n", RTLogDefaultInstance()));
435	pR0Logger->Logger.pfnLogger("hello ring-0 logger\n");
436	LogCom(("vmmR0InitVM: returned successfully from direct logger call (2). offScratch=%d\n", pR0Logger->Logger.offScratch));
437	RTLogSetDefaultInstanceThread(NULL, pGVM->pSession);
438	LogCom(("vmmR0InitVM: after %p dereg2\n", RTLogDefaultInstance()));
439
440	RTLogLoggerEx(&pR0Logger->Logger, 0, ~0U, "hello ring-0 logger (RTLogLoggerEx)\n");
441	LogCom(("vmmR0InitVM: RTLogLoggerEx returned fine offScratch=%d\n", pR0Logger->Logger.offScratch));
442
443	RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pGVM->pSession);
444	RTLogPrintf("hello ring-0 logger (RTLogPrintf)\n");
445	LogCom(("vmmR0InitVM: RTLogPrintf returned fine offScratch=%d\n", pR0Logger->Logger.offScratch));
446	# endif
447	# ifdef VBOX_WITH_R0_LOGGING
448	Log(("Switching to per-thread logging instance %p (key=%p)\n", pVCpu->vmmr0.s.u.s.Logger.pLogger, pGVM->pSession));
449	RTLogSetDefaultInstanceThread(pVCpu->vmmr0.s.u.s.Logger.pLogger, (uintptr_t)pGVM->pSession);
450	pVCpu->vmmr0.s.u.s.Logger.fRegistered = true;
451	# endif
452	}
453	#endif /* LOG_ENABLED */
454
455	/*
456	* Check if the host supports high resolution timers or not.
457	*/
458	if ( pGVM->vmm.s.fUsePeriodicPreemptionTimers
459	&& !RTTimerCanDoHighResolution())
460	pGVM->vmm.s.fUsePeriodicPreemptionTimers = false;
461
462	/*
463	* Initialize the per VM data for GVMM and GMM.
464	*/
465	rc = GVMMR0InitVM(pGVM);
466	if (RT_SUCCESS(rc))
467	{
468	#ifndef VBOX_WITH_MINIMAL_R0
469	/*
470	* Init HM, CPUM and PGM.
471	*/
472	rc = HMR0InitVM(pGVM);
473	if (RT_SUCCESS(rc))
474	{
475	rc = CPUMR0InitVM(pGVM);
476	if (RT_SUCCESS(rc))
477	{
478	rc = PGMR0InitVM(pGVM);
479	if (RT_SUCCESS(rc))
480	{
481	rc = EMR0InitVM(pGVM);
482	if (RT_SUCCESS(rc))
483	{
484	rc = IEMR0InitVM(pGVM);
485	if (RT_SUCCESS(rc))
486	{
487	rc = IOMR0InitVM(pGVM);
488	if (RT_SUCCESS(rc))
489	{
490	# ifdef VBOX_WITH_PCI_PASSTHROUGH
491	rc = PciRawR0InitVM(pGVM);
492	# endif
493	if (RT_SUCCESS(rc))
494	{
495	rc = GIMR0InitVM(pGVM);
496	if (RT_SUCCESS(rc))
497	{
498	#endif /* !VBOX_WITH_MINIMAL_R0 */
499	GVMMR0DoneInitVM(pGVM);
500	#ifndef VBOX_WITH_MINIMAL_R0
501	PGMR0DoneInitVM(pGVM);
502	#endif
503
504	/*
505	* Collect a bit of info for the VM release log.
506	*/
507	pGVM->vmm.s.fIsPreemptPendingApiTrusty = RTThreadPreemptIsPendingTrusty();
508	pGVM->vmm.s.fIsPreemptPossible = RTThreadPreemptIsPossible();;
509	return rc;
510
511	/* bail out*/
512	#ifndef VBOX_WITH_MINIMAL_R0
513	//GIMR0TermVM(pGVM);
514	}
515	# ifdef VBOX_WITH_PCI_PASSTHROUGH
516	PciRawR0TermVM(pGVM);
517	# endif
518	}
519	}
520	}
521	}
522	}
523	}
524	HMR0TermVM(pGVM);
525	}
526	#endif /* !VBOX_WITH_MINIMAL_R0 */
527	}
528
529	RTLogSetDefaultInstanceThread(NULL, (uintptr_t)pGVM->pSession);
530	return rc;
531	}
532
533
534	/**
535	* Does EMT specific VM initialization.
536	*
537	* @returns VBox status code.
538	* @param pGVM The ring-0 VM structure.
539	* @param idCpu The EMT that's calling.
540	*/
541	static int vmmR0InitVMEmt(PGVM pGVM, VMCPUID idCpu)
542	{
543	/* Paranoia (caller checked these already). */
544	AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_CPU_ID);
545	AssertReturn(pGVM->aCpus[idCpu].hEMT == RTThreadNativeSelf(), VERR_INVALID_CPU_ID);
546
547	#if defined(LOG_ENABLED) && defined(VBOX_WITH_R0_LOGGING)
548	/*
549	* Registration of ring 0 loggers.
550	*/
551	PVMCPUCC pVCpu = &pGVM->aCpus[idCpu];
552	if ( pVCpu->vmmr0.s.u.s.Logger.pLogger
553	&& !pVCpu->vmmr0.s.u.s.Logger.fRegistered)
554	{
555	RTLogSetDefaultInstanceThread(pVCpu->vmmr0.s.u.s.Logger.pLogger, (uintptr_t)pGVM->pSession);
556	pVCpu->vmmr0.s.u.s.Logger.fRegistered = true;
557	}
558	#endif
559
560	return VINF_SUCCESS;
561	}
562
563
564
565	/**
566	* Terminates the R0 bits for a particular VM instance.
567	*
568	* This is normally called by ring-3 as part of the VM termination process, but
569	* may alternatively be called during the support driver session cleanup when
570	* the VM object is destroyed (see GVMM).
571	*
572	* @returns VBox status code.
573	*
574	* @param pGVM The global (ring-0) VM structure.
575	* @param idCpu Set to 0 if EMT(0) or NIL_VMCPUID if session cleanup
576	* thread.
577	* @thread EMT(0) or session clean up thread.
578	*/
579	VMMR0_INT_DECL(int) VMMR0TermVM(PGVM pGVM, VMCPUID idCpu)
580	{
581	/*
582	* Check EMT(0) claim if we're called from userland.
583	*/
584	if (idCpu != NIL_VMCPUID)
585	{
586	AssertReturn(idCpu == 0, VERR_INVALID_CPU_ID);
587	int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
588	if (RT_FAILURE(rc))
589	return rc;
590	}
591
592	#ifndef VBOX_WITH_MINIMAL_R0
593	# ifdef VBOX_WITH_PCI_PASSTHROUGH
594	PciRawR0TermVM(pGVM);
595	# endif
596	#endif
597
598	/*
599	* Tell GVMM what we're up to and check that we only do this once.
600	*/
601	if (GVMMR0DoingTermVM(pGVM))
602	{
603	#ifndef VBOX_WITH_MINIMAL_R0
604	GIMR0TermVM(pGVM);
605
606	/** @todo I wish to call PGMR0PhysFlushHandyPages(pGVM, &pGVM->aCpus[idCpu])
607	* here to make sure we don't leak any shared pages if we crash... */
608	HMR0TermVM(pGVM);
609	#endif
610	}
611
612	/*
613	* Deregister the logger for this EMT.
614	*/
615	RTLogSetDefaultInstanceThread(NULL, (uintptr_t)pGVM->pSession);
616
617	/*
618	* Start log flusher thread termination.
619	*/
620	ASMAtomicWriteBool(&pGVM->vmmr0.s.LogFlusher.fThreadShutdown, true);
621	if (pGVM->vmmr0.s.LogFlusher.hEvent != NIL_RTSEMEVENT)
622	RTSemEventSignal(pGVM->vmmr0.s.LogFlusher.hEvent);
623
624	return VINF_SUCCESS;
625	}
626
627
628	/**
629	* This is called at the end of gvmmR0CleanupVM().
630	*
631	* @param pGVM The global (ring-0) VM structure.
632	*/
633	VMMR0_INT_DECL(void) VMMR0CleanupVM(PGVM pGVM)
634	{
635	AssertCompile(NIL_RTTHREADCTXHOOK == (RTTHREADCTXHOOK)0); /* Depends on zero initialized memory working for NIL at the moment. */
636	for (VMCPUID idCpu = 0; idCpu < pGVM->cCpus; idCpu++)
637	{
638	PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
639
640	/** @todo Can we busy wait here for all thread-context hooks to be
641	* deregistered before releasing (destroying) it? Only until we find a
642	* solution for not deregistering hooks everytime we're leaving HMR0
643	* context. */
644	VMMR0ThreadCtxHookDestroyForEmt(pGVCpu);
645	}
646
647	vmmR0CleanupLoggers(pGVM);
648	}
649
650
651	#ifndef VBOX_WITH_MINIMAL_R0
652
653	/**
654	* An interrupt or unhalt force flag is set, deal with it.
655	*
656	* @returns VINF_SUCCESS (or VINF_EM_HALT).
657	* @param pVCpu The cross context virtual CPU structure.
658	* @param uMWait Result from EMMonitorWaitIsActive().
659	* @param enmInterruptibility Guest CPU interruptbility level.
660	*/
661	static int vmmR0DoHaltInterrupt(PVMCPUCC pVCpu, unsigned uMWait, CPUMINTERRUPTIBILITY enmInterruptibility)
662	{
663	Assert(!TRPMHasTrap(pVCpu));
664	Assert( enmInterruptibility > CPUMINTERRUPTIBILITY_INVALID
665	&& enmInterruptibility < CPUMINTERRUPTIBILITY_END);
666
667	/*
668	* Pending interrupts w/o any SMIs or NMIs? That the usual case.
669	*/
670	if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
671	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI \| VMCPU_FF_INTERRUPT_NMI))
672	{
673	if (enmInterruptibility <= CPUMINTERRUPTIBILITY_UNRESTRAINED)
674	{
675	uint8_t u8Interrupt = 0;
676	int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
677	Log(("vmmR0DoHaltInterrupt: CPU%d u8Interrupt=%d (%#x) rc=%Rrc\n", pVCpu->idCpu, u8Interrupt, u8Interrupt, rc));
678	if (RT_SUCCESS(rc))
679	{
680	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_UNHALT);
681
682	rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
683	AssertRCSuccess(rc);
684	STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltExec);
685	return rc;
686	}
687	}
688	}
689	/*
690	* SMI is not implemented yet, at least not here.
691	*/
692	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI))
693	{
694	Log12(("vmmR0DoHaltInterrupt: CPU%d failed #3\n", pVCpu->idCpu));
695	STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltToR3);
696	return VINF_EM_HALT;
697	}
698	/*
699	* NMI.
700	*/
701	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
702	{
703	if (enmInterruptibility < CPUMINTERRUPTIBILITY_NMI_INHIBIT)
704	{
705	/** @todo later. */
706	Log12(("vmmR0DoHaltInterrupt: CPU%d failed #2 (uMWait=%u enmInt=%d)\n", pVCpu->idCpu, uMWait, enmInterruptibility));
707	STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltToR3);
708	return VINF_EM_HALT;
709	}
710	}
711	/*
712	* Nested-guest virtual interrupt.
713	*/
714	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST))
715	{
716	if (enmInterruptibility < CPUMINTERRUPTIBILITY_VIRT_INT_DISABLED)
717	{
718	/** @todo NSTVMX: NSTSVM: Remember, we might have to check and perform VM-exits
719	* here before injecting the virtual interrupt. See emR3ForcedActions
720	* for details. */
721	Log12(("vmmR0DoHaltInterrupt: CPU%d failed #1 (uMWait=%u enmInt=%d)\n", pVCpu->idCpu, uMWait, enmInterruptibility));
722	STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltToR3);
723	return VINF_EM_HALT;
724	}
725	}
726
727	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UNHALT))
728	{
729	STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltExec);
730	Log11(("vmmR0DoHaltInterrupt: CPU%d success VINF_SUCCESS (UNHALT)\n", pVCpu->idCpu));
731	return VINF_SUCCESS;
732	}
733	if (uMWait > 1)
734	{
735	STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltExec);
736	Log11(("vmmR0DoHaltInterrupt: CPU%d success VINF_SUCCESS (uMWait=%u > 1)\n", pVCpu->idCpu, uMWait));
737	return VINF_SUCCESS;
738	}
739
740	Log12(("vmmR0DoHaltInterrupt: CPU%d failed #0 (uMWait=%u enmInt=%d)\n", pVCpu->idCpu, uMWait, enmInterruptibility));
741	STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltToR3);
742	return VINF_EM_HALT;
743	}
744
745
746	/**
747	* This does one round of vmR3HaltGlobal1Halt().
748	*
749	* The rational here is that we'll reduce latency in interrupt situations if we
750	* don't go to ring-3 immediately on a VINF_EM_HALT (guest executed HLT or
751	* MWAIT), but do one round of blocking here instead and hope the interrupt is
752	* raised in the meanwhile.
753	*
754	* If we go to ring-3 we'll quit the inner HM/NEM loop in EM and end up in the
755	* outer loop, which will then call VMR3WaitHalted() and that in turn will do a
756	* ring-0 call (unless we're too close to a timer event). When the interrupt
757	* wakes us up, we'll return from ring-0 and EM will by instinct do a
758	* rescheduling (because of raw-mode) before it resumes the HM/NEM loop and gets
759	* back to VMMR0EntryFast().
760	*
761	* @returns VINF_SUCCESS or VINF_EM_HALT.
762	* @param pGVM The ring-0 VM structure.
763	* @param pGVCpu The ring-0 virtual CPU structure.
764	*
765	* @todo r=bird: All the blocking/waiting and EMT managment should move out of
766	* the VM module, probably to VMM. Then this would be more weird wrt
767	* parameters and statistics.
768	*/
769	static int vmmR0DoHalt(PGVM pGVM, PGVMCPU pGVCpu)
770	{
771	/*
772	* Do spin stat historization.
773	*/
774	if (++pGVCpu->vmm.s.cR0Halts & 0xff)
775	{ /* likely */ }
776	else if (pGVCpu->vmm.s.cR0HaltsSucceeded > pGVCpu->vmm.s.cR0HaltsToRing3)
777	{
778	pGVCpu->vmm.s.cR0HaltsSucceeded = 2;
779	pGVCpu->vmm.s.cR0HaltsToRing3 = 0;
780	}
781	else
782	{
783	pGVCpu->vmm.s.cR0HaltsSucceeded = 0;
784	pGVCpu->vmm.s.cR0HaltsToRing3 = 2;
785	}
786
787	/*
788	* Flags that makes us go to ring-3.
789	*/
790	uint32_t const fVmFFs = VM_FF_TM_VIRTUAL_SYNC \| VM_FF_PDM_QUEUES \| VM_FF_PDM_DMA
791	\| VM_FF_DBGF \| VM_FF_REQUEST \| VM_FF_CHECK_VM_STATE
792	\| VM_FF_RESET \| VM_FF_EMT_RENDEZVOUS \| VM_FF_PGM_NEED_HANDY_PAGES
793	\| VM_FF_PGM_NO_MEMORY \| VM_FF_DEBUG_SUSPEND;
794	uint64_t const fCpuFFs = VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT \| VMCPU_FF_IEM
795	\| VMCPU_FF_REQUEST \| VMCPU_FF_DBGF \| VMCPU_FF_HM_UPDATE_CR3
796	\| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
797	\| VMCPU_FF_TO_R3 \| VMCPU_FF_IOM;
798
799	/*
800	* Check preconditions.
801	*/
802	unsigned const uMWait = EMMonitorWaitIsActive(pGVCpu);
803	CPUMINTERRUPTIBILITY const enmInterruptibility = CPUMGetGuestInterruptibility(pGVCpu);
804	if ( pGVCpu->vmm.s.fMayHaltInRing0
805	&& !TRPMHasTrap(pGVCpu)
806	&& ( enmInterruptibility == CPUMINTERRUPTIBILITY_UNRESTRAINED
807	\|\| uMWait > 1))
808	{
809	if ( !VM_FF_IS_ANY_SET(pGVM, fVmFFs)
810	&& !VMCPU_FF_IS_ANY_SET(pGVCpu, fCpuFFs))
811	{
812	/*
813	* Interrupts pending already?
814	*/
815	if (VMCPU_FF_TEST_AND_CLEAR(pGVCpu, VMCPU_FF_UPDATE_APIC))
816	APICUpdatePendingInterrupts(pGVCpu);
817
818	/*
819	* Flags that wake up from the halted state.
820	*/
821	uint64_t const fIntMask = VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC \| VMCPU_FF_INTERRUPT_NESTED_GUEST
822	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI \| VMCPU_FF_UNHALT;
823
824	if (VMCPU_FF_IS_ANY_SET(pGVCpu, fIntMask))
825	return vmmR0DoHaltInterrupt(pGVCpu, uMWait, enmInterruptibility);
826	ASMNopPause();
827
828	/*
829	* Check out how long till the next timer event.
830	*/
831	uint64_t u64Delta;
832	uint64_t u64GipTime = TMTimerPollGIP(pGVM, pGVCpu, &u64Delta);
833
834	if ( !VM_FF_IS_ANY_SET(pGVM, fVmFFs)
835	&& !VMCPU_FF_IS_ANY_SET(pGVCpu, fCpuFFs))
836	{
837	if (VMCPU_FF_TEST_AND_CLEAR(pGVCpu, VMCPU_FF_UPDATE_APIC))
838	APICUpdatePendingInterrupts(pGVCpu);
839
840	if (VMCPU_FF_IS_ANY_SET(pGVCpu, fIntMask))
841	return vmmR0DoHaltInterrupt(pGVCpu, uMWait, enmInterruptibility);
842
843	/*
844	* Wait if there is enough time to the next timer event.
845	*/
846	if (u64Delta >= pGVCpu->vmm.s.cNsSpinBlockThreshold)
847	{
848	/* If there are few other CPU cores around, we will procrastinate a
849	little before going to sleep, hoping for some device raising an
850	interrupt or similar. Though, the best thing here would be to
851	dynamically adjust the spin count according to its usfulness or
852	something... */
853	if ( pGVCpu->vmm.s.cR0HaltsSucceeded > pGVCpu->vmm.s.cR0HaltsToRing3
854	&& RTMpGetOnlineCount() >= 4)
855	{
856	/** @todo Figure out how we can skip this if it hasn't help recently...
857	* @bugref{9172#c12} */
858	uint32_t cSpinLoops = 42;
859	while (cSpinLoops-- > 0)
860	{
861	ASMNopPause();
862	if (VMCPU_FF_TEST_AND_CLEAR(pGVCpu, VMCPU_FF_UPDATE_APIC))
863	APICUpdatePendingInterrupts(pGVCpu);
864	ASMNopPause();
865	if (VM_FF_IS_ANY_SET(pGVM, fVmFFs))
866	{
867	STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3FromSpin);
868	STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3);
869	return VINF_EM_HALT;
870	}
871	ASMNopPause();
872	if (VMCPU_FF_IS_ANY_SET(pGVCpu, fCpuFFs))
873	{
874	STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3FromSpin);
875	STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3);
876	return VINF_EM_HALT;
877	}
878	ASMNopPause();
879	if (VMCPU_FF_IS_ANY_SET(pGVCpu, fIntMask))
880	{
881	STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltExecFromSpin);
882	return vmmR0DoHaltInterrupt(pGVCpu, uMWait, enmInterruptibility);
883	}
884	ASMNopPause();
885	}
886	}
887
888	/*
889	* We have to set the state to VMCPUSTATE_STARTED_HALTED here so ring-3
890	* knows when to notify us (cannot access VMINTUSERPERVMCPU::fWait from here).
891	* After changing the state we must recheck the force flags of course.
892	*/
893	if (VMCPU_CMPXCHG_STATE(pGVCpu, VMCPUSTATE_STARTED_HALTED, VMCPUSTATE_STARTED))
894	{
895	if ( !VM_FF_IS_ANY_SET(pGVM, fVmFFs)
896	&& !VMCPU_FF_IS_ANY_SET(pGVCpu, fCpuFFs))
897	{
898	if (VMCPU_FF_TEST_AND_CLEAR(pGVCpu, VMCPU_FF_UPDATE_APIC))
899	APICUpdatePendingInterrupts(pGVCpu);
900
901	if (VMCPU_FF_IS_ANY_SET(pGVCpu, fIntMask))
902	{
903	VMCPU_CMPXCHG_STATE(pGVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_HALTED);
904	return vmmR0DoHaltInterrupt(pGVCpu, uMWait, enmInterruptibility);
905	}
906
907	/* Okay, block! */
908	uint64_t const u64StartSchedHalt = RTTimeNanoTS();
909	int rc = GVMMR0SchedHalt(pGVM, pGVCpu, u64GipTime);
910	uint64_t const u64EndSchedHalt = RTTimeNanoTS();
911	uint64_t const cNsElapsedSchedHalt = u64EndSchedHalt - u64StartSchedHalt;
912	Log10(("vmmR0DoHalt: CPU%d: halted %llu ns\n", pGVCpu->idCpu, cNsElapsedSchedHalt));
913
914	VMCPU_CMPXCHG_STATE(pGVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_HALTED);
915	STAM_REL_PROFILE_ADD_PERIOD(&pGVCpu->vmm.s.StatR0HaltBlock, cNsElapsedSchedHalt);
916	if ( rc == VINF_SUCCESS
917	\|\| rc == VERR_INTERRUPTED)
918	{
919	/* Keep some stats like ring-3 does. */
920	int64_t const cNsOverslept = u64EndSchedHalt - u64GipTime;
921	if (cNsOverslept > 50000)
922	STAM_REL_PROFILE_ADD_PERIOD(&pGVCpu->vmm.s.StatR0HaltBlockOverslept, cNsOverslept);
923	else if (cNsOverslept < -50000)
924	STAM_REL_PROFILE_ADD_PERIOD(&pGVCpu->vmm.s.StatR0HaltBlockInsomnia, cNsElapsedSchedHalt);
925	else
926	STAM_REL_PROFILE_ADD_PERIOD(&pGVCpu->vmm.s.StatR0HaltBlockOnTime, cNsElapsedSchedHalt);
927
928	/*
929	* Recheck whether we can resume execution or have to go to ring-3.
930	*/
931	if ( !VM_FF_IS_ANY_SET(pGVM, fVmFFs)
932	&& !VMCPU_FF_IS_ANY_SET(pGVCpu, fCpuFFs))
933	{
934	if (VMCPU_FF_TEST_AND_CLEAR(pGVCpu, VMCPU_FF_UPDATE_APIC))
935	APICUpdatePendingInterrupts(pGVCpu);
936	if (VMCPU_FF_IS_ANY_SET(pGVCpu, fIntMask))
937	{
938	STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltExecFromBlock);
939	return vmmR0DoHaltInterrupt(pGVCpu, uMWait, enmInterruptibility);
940	}
941	STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3PostNoInt);
942	Log12(("vmmR0DoHalt: CPU%d post #2 - No pending interrupt\n", pGVCpu->idCpu));
943	}
944	else
945	{
946	STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3PostPendingFF);
947	Log12(("vmmR0DoHalt: CPU%d post #1 - Pending FF\n", pGVCpu->idCpu));
948	}
949	}
950	else
951	{
952	STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3Other);
953	Log12(("vmmR0DoHalt: CPU%d GVMMR0SchedHalt failed: %Rrc\n", pGVCpu->idCpu, rc));
954	}
955	}
956	else
957	{
958	VMCPU_CMPXCHG_STATE(pGVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_HALTED);
959	STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3PendingFF);
960	Log12(("vmmR0DoHalt: CPU%d failed #5 - Pending FF\n", pGVCpu->idCpu));
961	}
962	}
963	else
964	{
965	STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3Other);
966	Log12(("vmmR0DoHalt: CPU%d failed #4 - enmState=%d\n", pGVCpu->idCpu, VMCPU_GET_STATE(pGVCpu)));
967	}
968	}
969	else
970	{
971	STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3SmallDelta);
972	Log12(("vmmR0DoHalt: CPU%d failed #3 - delta too small: %RU64\n", pGVCpu->idCpu, u64Delta));
973	}
974	}
975	else
976	{
977	STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3PendingFF);
978	Log12(("vmmR0DoHalt: CPU%d failed #2 - Pending FF\n", pGVCpu->idCpu));
979	}
980	}
981	else
982	{
983	STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3PendingFF);
984	Log12(("vmmR0DoHalt: CPU%d failed #1 - Pending FF\n", pGVCpu->idCpu));
985	}
986	}
987	else
988	{
989	STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3Other);
990	Log12(("vmmR0DoHalt: CPU%d failed #0 - fMayHaltInRing0=%d TRPMHasTrap=%d enmInt=%d uMWait=%u\n",
991	pGVCpu->idCpu, pGVCpu->vmm.s.fMayHaltInRing0, TRPMHasTrap(pGVCpu), enmInterruptibility, uMWait));
992	}
993
994	STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3);
995	return VINF_EM_HALT;
996	}
997
998	#endif /* !VBOX_WITH_MINIMAL_R0 */
999
1000	/**
1001	* VMM ring-0 thread-context callback.
1002	*
1003	* This does common HM state updating and calls the HM-specific thread-context
1004	* callback.
1005	*
1006	* This is used together with RTThreadCtxHookCreate() on platforms which
1007	* supports it, and directly from VMMR0EmtPrepareForBlocking() and
1008	* VMMR0EmtResumeAfterBlocking() on platforms which don't.
1009	*
1010	* @param enmEvent The thread-context event.
1011	* @param pvUser Opaque pointer to the VMCPU.
1012	*
1013	* @thread EMT(pvUser)
1014	*/
1015	static DECLCALLBACK(void) vmmR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, void *pvUser)
1016	{
1017	PVMCPUCC pVCpu = (PVMCPUCC)pvUser;
1018
1019	switch (enmEvent)
1020	{
1021	case RTTHREADCTXEVENT_IN:
1022	{
1023	/*
1024	* Linux may call us with preemption enabled (really!) but technically we
1025	* cannot get preempted here, otherwise we end up in an infinite recursion
1026	* scenario (i.e. preempted in resume hook -> preempt hook -> resume hook...
1027	* ad infinitum). Let's just disable preemption for now...
1028	*/
1029	/** @todo r=bird: I don't believe the above. The linux code is clearly enabling
1030	* preemption after doing the callout (one or two functions up the
1031	* call chain). */
1032	/** @todo r=ramshankar: See @bugref{5313#c30}. */
1033	RTTHREADPREEMPTSTATE ParanoidPreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
1034	RTThreadPreemptDisable(&ParanoidPreemptState);
1035
1036	/* We need to update the VCPU <-> host CPU mapping. */
1037	RTCPUID idHostCpu;
1038	uint32_t iHostCpuSet = RTMpCurSetIndexAndId(&idHostCpu);
1039	pVCpu->iHostCpuSet = iHostCpuSet;
1040	ASMAtomicWriteU32(&pVCpu->idHostCpu, idHostCpu);
1041
1042	/* In the very unlikely event that the GIP delta for the CPU we're
1043	rescheduled needs calculating, try force a return to ring-3.
1044	We unfortunately cannot do the measurements right here. */
1045	if (RT_LIKELY(!SUPIsTscDeltaAvailableForCpuSetIndex(iHostCpuSet)))
1046	{ /* likely */ }
1047	else
1048	VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
1049
1050	#ifndef VBOX_WITH_MINIMAL_R0
1051	/* Invoke the HM-specific thread-context callback. */
1052	HMR0ThreadCtxCallback(enmEvent, pvUser);
1053	#endif
1054
1055	/* Restore preemption. */
1056	RTThreadPreemptRestore(&ParanoidPreemptState);
1057	break;
1058	}
1059
1060	case RTTHREADCTXEVENT_OUT:
1061	{
1062	#ifndef VBOX_WITH_MINIMAL_R0
1063	/* Invoke the HM-specific thread-context callback. */
1064	HMR0ThreadCtxCallback(enmEvent, pvUser);
1065	#endif
1066
1067	/*
1068	* Sigh. See VMMGetCpu() used by VMCPU_ASSERT_EMT(). We cannot let several VCPUs
1069	* have the same host CPU associated with it.
1070	*/
1071	pVCpu->iHostCpuSet = UINT32_MAX;
1072	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
1073	break;
1074	}
1075
1076	default:
1077	#ifndef VBOX_WITH_MINIMAL_R0
1078	/* Invoke the HM-specific thread-context callback. */
1079	HMR0ThreadCtxCallback(enmEvent, pvUser);
1080	#endif
1081	break;
1082	}
1083	}
1084
1085
1086	/**
1087	* Creates thread switching hook for the current EMT thread.
1088	*
1089	* This is called by GVMMR0CreateVM and GVMMR0RegisterVCpu. If the host
1090	* platform does not implement switcher hooks, no hooks will be create and the
1091	* member set to NIL_RTTHREADCTXHOOK.
1092	*
1093	* @returns VBox status code.
1094	* @param pVCpu The cross context virtual CPU structure.
1095	* @thread EMT(pVCpu)
1096	*/
1097	VMMR0_INT_DECL(int) VMMR0ThreadCtxHookCreateForEmt(PVMCPUCC pVCpu)
1098	{
1099	VMCPU_ASSERT_EMT(pVCpu);
1100	Assert(pVCpu->vmmr0.s.hCtxHook == NIL_RTTHREADCTXHOOK);
1101
1102	#ifndef VBOX_WITH_MINIMAL_R0
1103
1104	# if 1 /* To disable this stuff change to zero. */
1105	int rc = RTThreadCtxHookCreate(&pVCpu->vmmr0.s.hCtxHook, 0, vmmR0ThreadCtxCallback, pVCpu);
1106	if (RT_SUCCESS(rc))
1107	{
1108	pVCpu->pGVM->vmm.s.fIsUsingContextHooks = true;
1109	return rc;
1110	}
1111	# else
1112	RT_NOREF(vmmR0ThreadCtxCallback);
1113	int rc = VERR_NOT_SUPPORTED;
1114	# endif
1115	#endif
1116
1117	pVCpu->vmmr0.s.hCtxHook = NIL_RTTHREADCTXHOOK;
1118	pVCpu->pGVM->vmm.s.fIsUsingContextHooks = false;
1119	#ifndef VBOX_WITH_MINIMAL_R0
1120	if (rc != VERR_NOT_SUPPORTED) /* Just ignore it, we can live without context hooks. */
1121	LogRelMax(32, ("RTThreadCtxHookCreate failed! rc=%Rrc pVCpu=%p idCpu=%RU32\n", rc, pVCpu, pVCpu->idCpu));
1122	#endif
1123	return VINF_SUCCESS;
1124	}
1125
1126
1127	/**
1128	* Destroys the thread switching hook for the specified VCPU.
1129	*
1130	* @param pVCpu The cross context virtual CPU structure.
1131	* @remarks Can be called from any thread.
1132	*/
1133	VMMR0_INT_DECL(void) VMMR0ThreadCtxHookDestroyForEmt(PVMCPUCC pVCpu)
1134	{
1135	#ifndef VBOX_WITH_MINIMAL_R0
1136	int rc = RTThreadCtxHookDestroy(pVCpu->vmmr0.s.hCtxHook);
1137	AssertRC(rc);
1138	#endif
1139	pVCpu->vmmr0.s.hCtxHook = NIL_RTTHREADCTXHOOK;
1140	}
1141
1142	#ifndef VBOX_WITH_MINIMAL_R0
1143
1144	/**
1145	* Disables the thread switching hook for this VCPU (if we got one).
1146	*
1147	* @param pVCpu The cross context virtual CPU structure.
1148	* @thread EMT(pVCpu)
1149	*
1150	* @remarks This also clears GVMCPU::idHostCpu, so the mapping is invalid after
1151	* this call. This means you have to be careful with what you do!
1152	*/
1153	VMMR0_INT_DECL(void) VMMR0ThreadCtxHookDisable(PVMCPUCC pVCpu)
1154	{
1155	/*
1156	* Clear the VCPU <-> host CPU mapping as we've left HM context.
1157	* @bugref{7726#c19} explains the need for this trick:
1158	*
1159	* VMXR0CallRing3Callback/SVMR0CallRing3Callback &
1160	* hmR0VmxLeaveSession/hmR0SvmLeaveSession disables context hooks during
1161	* longjmp & normal return to ring-3, which opens a window where we may be
1162	* rescheduled without changing GVMCPUID::idHostCpu and cause confusion if
1163	* the CPU starts executing a different EMT. Both functions first disables
1164	* preemption and then calls HMR0LeaveCpu which invalids idHostCpu, leaving
1165	* an opening for getting preempted.
1166	*/
1167	/** @todo Make HM not need this API! Then we could leave the hooks enabled
1168	* all the time. */
1169
1170	/*
1171	* Disable the context hook, if we got one.
1172	*/
1173	if (pVCpu->vmmr0.s.hCtxHook != NIL_RTTHREADCTXHOOK)
1174	{
1175	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1176	ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
1177	int rc = RTThreadCtxHookDisable(pVCpu->vmmr0.s.hCtxHook);
1178	AssertRC(rc);
1179	}
1180	}
1181
1182
1183	/**
1184	* Internal version of VMMR0ThreadCtxHooksAreRegistered.
1185	*
1186	* @returns true if registered, false otherwise.
1187	* @param pVCpu The cross context virtual CPU structure.
1188	*/
1189	DECLINLINE(bool) vmmR0ThreadCtxHookIsEnabled(PVMCPUCC pVCpu)
1190	{
1191	return RTThreadCtxHookIsEnabled(pVCpu->vmmr0.s.hCtxHook);
1192	}
1193
1194
1195	/**
1196	* Whether thread-context hooks are registered for this VCPU.
1197	*
1198	* @returns true if registered, false otherwise.
1199	* @param pVCpu The cross context virtual CPU structure.
1200	*/
1201	VMMR0_INT_DECL(bool) VMMR0ThreadCtxHookIsEnabled(PVMCPUCC pVCpu)
1202	{
1203	return vmmR0ThreadCtxHookIsEnabled(pVCpu);
1204	}
1205
1206	#endif /* !VBOX_WITH_MINIMAL_R0 */
1207
1208	/**
1209	* Returns the ring-0 release logger instance.
1210	*
1211	* @returns Pointer to release logger, NULL if not configured.
1212	* @param pVCpu The cross context virtual CPU structure of the caller.
1213	* @thread EMT(pVCpu)
1214	*/
1215	VMMR0_INT_DECL(PRTLOGGER) VMMR0GetReleaseLogger(PVMCPUCC pVCpu)
1216	{
1217	return pVCpu->vmmr0.s.u.s.RelLogger.pLogger;
1218	}
1219
1220
1221	#ifdef VBOX_WITH_STATISTICS
1222	/**
1223	* Record return code statistics
1224	* @param pVM The cross context VM structure.
1225	* @param pVCpu The cross context virtual CPU structure.
1226	* @param rc The status code.
1227	*/
1228	static void vmmR0RecordRC(PVMCC pVM, PVMCPUCC pVCpu, int rc)
1229	{
1230	/*
1231	* Collect statistics.
1232	*/
1233	switch (rc)
1234	{
1235	case VINF_SUCCESS:
1236	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetNormal);
1237	break;
1238	case VINF_EM_RAW_INTERRUPT:
1239	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetInterrupt);
1240	break;
1241	case VINF_EM_RAW_INTERRUPT_HYPER:
1242	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetInterruptHyper);
1243	break;
1244	case VINF_EM_RAW_GUEST_TRAP:
1245	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetGuestTrap);
1246	break;
1247	case VINF_EM_RAW_RING_SWITCH:
1248	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetRingSwitch);
1249	break;
1250	case VINF_EM_RAW_RING_SWITCH_INT:
1251	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetRingSwitchInt);
1252	break;
1253	case VINF_EM_RAW_STALE_SELECTOR:
1254	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetStaleSelector);
1255	break;
1256	case VINF_EM_RAW_IRET_TRAP:
1257	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIRETTrap);
1258	break;
1259	case VINF_IOM_R3_IOPORT_READ:
1260	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIORead);
1261	break;
1262	case VINF_IOM_R3_IOPORT_WRITE:
1263	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIOWrite);
1264	break;
1265	case VINF_IOM_R3_IOPORT_COMMIT_WRITE:
1266	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIOCommitWrite);
1267	break;
1268	case VINF_IOM_R3_MMIO_READ:
1269	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIORead);
1270	break;
1271	case VINF_IOM_R3_MMIO_WRITE:
1272	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOWrite);
1273	break;
1274	case VINF_IOM_R3_MMIO_COMMIT_WRITE:
1275	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOCommitWrite);
1276	break;
1277	case VINF_IOM_R3_MMIO_READ_WRITE:
1278	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOReadWrite);
1279	break;
1280	case VINF_PATM_HC_MMIO_PATCH_READ:
1281	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOPatchRead);
1282	break;
1283	case VINF_PATM_HC_MMIO_PATCH_WRITE:
1284	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOPatchWrite);
1285	break;
1286	case VINF_CPUM_R3_MSR_READ:
1287	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMSRRead);
1288	break;
1289	case VINF_CPUM_R3_MSR_WRITE:
1290	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMSRWrite);
1291	break;
1292	case VINF_EM_RAW_EMULATE_INSTR:
1293	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetEmulate);
1294	break;
1295	case VINF_PATCH_EMULATE_INSTR:
1296	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchEmulate);
1297	break;
1298	case VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT:
1299	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetLDTFault);
1300	break;
1301	case VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT:
1302	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetGDTFault);
1303	break;
1304	case VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT:
1305	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIDTFault);
1306	break;
1307	case VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT:
1308	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetTSSFault);
1309	break;
1310	case VINF_CSAM_PENDING_ACTION:
1311	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetCSAMTask);
1312	break;
1313	case VINF_PGM_SYNC_CR3:
1314	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetSyncCR3);
1315	break;
1316	case VINF_PATM_PATCH_INT3:
1317	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchInt3);
1318	break;
1319	case VINF_PATM_PATCH_TRAP_PF:
1320	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchPF);
1321	break;
1322	case VINF_PATM_PATCH_TRAP_GP:
1323	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchGP);
1324	break;
1325	case VINF_PATM_PENDING_IRQ_AFTER_IRET:
1326	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchIretIRQ);
1327	break;
1328	case VINF_EM_RESCHEDULE_REM:
1329	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetRescheduleREM);
1330	break;
1331	case VINF_EM_RAW_TO_R3:
1332	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Total);
1333	if (VM_FF_IS_SET(pVM, VM_FF_TM_VIRTUAL_SYNC))
1334	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3TMVirt);
1335	else if (VM_FF_IS_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES))
1336	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3HandyPages);
1337	else if (VM_FF_IS_SET(pVM, VM_FF_PDM_QUEUES))
1338	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3PDMQueues);
1339	else if (VM_FF_IS_SET(pVM, VM_FF_EMT_RENDEZVOUS))
1340	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Rendezvous);
1341	else if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
1342	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3DMA);
1343	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TIMER))
1344	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Timer);
1345	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PDM_CRITSECT))
1346	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3CritSect);
1347	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TO_R3))
1348	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3FF);
1349	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM))
1350	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Iem);
1351	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IOM))
1352	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Iom);
1353	else
1354	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Unknown);
1355	break;
1356
1357	case VINF_EM_RAW_TIMER_PENDING:
1358	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetTimerPending);
1359	break;
1360	case VINF_EM_RAW_INTERRUPT_PENDING:
1361	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetInterruptPending);
1362	break;
1363	case VINF_PATM_DUPLICATE_FUNCTION:
1364	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPATMDuplicateFn);
1365	break;
1366	case VINF_PGM_POOL_FLUSH_PENDING:
1367	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPGMFlushPending);
1368	break;
1369	case VINF_EM_PENDING_REQUEST:
1370	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPendingRequest);
1371	break;
1372	case VINF_EM_HM_PATCH_TPR_INSTR:
1373	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchTPR);
1374	break;
1375	default:
1376	STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMisc);
1377	break;
1378	}
1379	}
1380	#endif /* VBOX_WITH_STATISTICS */
1381
1382
1383	/**
1384	* The Ring 0 entry point, called by the fast-ioctl path.
1385	*
1386	* @param pGVM The global (ring-0) VM structure.
1387	* @param pVMIgnored The cross context VM structure. The return code is
1388	* stored in pVM->vmm.s.iLastGZRc.
1389	* @param idCpu The Virtual CPU ID of the calling EMT.
1390	* @param enmOperation Which operation to execute.
1391	* @remarks Assume called with interrupts _enabled_.
1392	*/
1393	VMMR0DECL(void) VMMR0EntryFast(PGVM pGVM, PVMCC pVMIgnored, VMCPUID idCpu, VMMR0OPERATION enmOperation)
1394	{
1395	RT_NOREF(pVMIgnored);
1396
1397	/*
1398	* Validation.
1399	*/
1400	if ( idCpu < pGVM->cCpus
1401	&& pGVM->cCpus == pGVM->cCpusUnsafe)
1402	{ /likely/ }
1403	else
1404	{
1405	SUPR0Printf("VMMR0EntryFast: Bad idCpu=%#x cCpus=%#x cCpusUnsafe=%#x\n", idCpu, pGVM->cCpus, pGVM->cCpusUnsafe);
1406	return;
1407	}
1408
1409	PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
1410	RTNATIVETHREAD const hNativeThread = RTThreadNativeSelf();
1411	if (RT_LIKELY( pGVCpu->hEMT == hNativeThread
1412	&& pGVCpu->hNativeThreadR0 == hNativeThread))
1413	{ /* likely */ }
1414	else
1415	{
1416	SUPR0Printf("VMMR0EntryFast: Bad thread idCpu=%#x hNativeSelf=%p pGVCpu->hEmt=%p pGVCpu->hNativeThreadR0=%p\n",
1417	idCpu, hNativeThread, pGVCpu->hEMT, pGVCpu->hNativeThreadR0);
1418	return;
1419	}
1420
1421	/*
1422	* Perform requested operation.
1423	*/
1424	switch (enmOperation)
1425	{
1426	#ifndef VBOX_WITH_MINIMAL_R0
1427	/*
1428	* Run guest code using the available hardware acceleration technology.
1429	*/
1430	case VMMR0_DO_HM_RUN:
1431	{
1432	for (;;) /* hlt loop */
1433	{
1434	/*
1435	* Disable ring-3 calls & blocking till we've successfully entered HM.
1436	* Otherwise we sometimes end up blocking at the finall Log4 statement
1437	* in VMXR0Enter, while still in a somewhat inbetween state.
1438	*/
1439	VMMRZCallRing3Disable(pGVCpu);
1440
1441	/*
1442	* Disable preemption.
1443	*/
1444	Assert(!vmmR0ThreadCtxHookIsEnabled(pGVCpu));
1445	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
1446	RTThreadPreemptDisable(&PreemptState);
1447	pGVCpu->vmmr0.s.pPreemptState = &PreemptState;
1448
1449	/*
1450	* Get the host CPU identifiers, make sure they are valid and that
1451	* we've got a TSC delta for the CPU.
1452	*/
1453	RTCPUID idHostCpu;
1454	uint32_t iHostCpuSet = RTMpCurSetIndexAndId(&idHostCpu);
1455	if (RT_LIKELY( iHostCpuSet < RTCPUSET_MAX_CPUS
1456	&& SUPIsTscDeltaAvailableForCpuSetIndex(iHostCpuSet)))
1457	{
1458	pGVCpu->iHostCpuSet = iHostCpuSet;
1459	ASMAtomicWriteU32(&pGVCpu->idHostCpu, idHostCpu);
1460
1461	/*
1462	* Update the periodic preemption timer if it's active.
1463	*/
1464	if (pGVM->vmm.s.fUsePeriodicPreemptionTimers)
1465	GVMMR0SchedUpdatePeriodicPreemptionTimer(pGVM, pGVCpu->idHostCpu, TMCalcHostTimerFrequency(pGVM, pGVCpu));
1466
1467	# ifdef VMM_R0_TOUCH_FPU
1468	/*
1469	* Make sure we've got the FPU state loaded so and we don't need to clear
1470	* CR0.TS and get out of sync with the host kernel when loading the guest
1471	* FPU state. @ref sec_cpum_fpu (CPUM.cpp) and @bugref{4053}.
1472	*/
1473	CPUMR0TouchHostFpu();
1474	# endif
1475	int rc;
1476	bool fPreemptRestored = false;
1477	if (!HMR0SuspendPending())
1478	{
1479	/*
1480	* Enable the context switching hook.
1481	*/
1482	if (pGVCpu->vmmr0.s.hCtxHook != NIL_RTTHREADCTXHOOK)
1483	{
1484	Assert(!RTThreadCtxHookIsEnabled(pGVCpu->vmmr0.s.hCtxHook));
1485	int rc2 = RTThreadCtxHookEnable(pGVCpu->vmmr0.s.hCtxHook); AssertRC(rc2);
1486	}
1487
1488	/*
1489	* Enter HM context.
1490	*/
1491	rc = HMR0Enter(pGVCpu);
1492	if (RT_SUCCESS(rc))
1493	{
1494	VMCPU_SET_STATE(pGVCpu, VMCPUSTATE_STARTED_HM);
1495
1496	/*
1497	* When preemption hooks are in place, enable preemption now that
1498	* we're in HM context.
1499	*/
1500	if (vmmR0ThreadCtxHookIsEnabled(pGVCpu))
1501	{
1502	fPreemptRestored = true;
1503	pGVCpu->vmmr0.s.pPreemptState = NULL;
1504	RTThreadPreemptRestore(&PreemptState);
1505	}
1506	VMMRZCallRing3Enable(pGVCpu);
1507
1508	/*
1509	* Setup the longjmp machinery and execute guest code (calls HMR0RunGuestCode).
1510	*/
1511	rc = vmmR0CallRing3SetJmp(&pGVCpu->vmmr0.s.AssertJmpBuf, HMR0RunGuestCode, pGVM, pGVCpu);
1512
1513	/*
1514	* Assert sanity on the way out. Using manual assertions code here as normal
1515	* assertions are going to panic the host since we're outside the setjmp/longjmp zone.
1516	*/
1517	if (RT_UNLIKELY( VMCPU_GET_STATE(pGVCpu) != VMCPUSTATE_STARTED_HM
1518	&& RT_SUCCESS_NP(rc)
1519	&& rc != VERR_VMM_RING0_ASSERTION ))
1520	{
1521	pGVM->vmm.s.szRing0AssertMsg1[0] = '\0';
1522	RTStrPrintf(pGVM->vmm.s.szRing0AssertMsg2, sizeof(pGVM->vmm.s.szRing0AssertMsg2),
1523	"Got VMCPU state %d expected %d.\n", VMCPU_GET_STATE(pGVCpu), VMCPUSTATE_STARTED_HM);
1524	rc = VERR_VMM_WRONG_HM_VMCPU_STATE;
1525	}
1526	# if 0
1527	/** @todo Get rid of this. HM shouldn't disable the context hook. */
1528	else if (RT_UNLIKELY(vmmR0ThreadCtxHookIsEnabled(pGVCpu)))
1529	{
1530	pGVM->vmm.s.szRing0AssertMsg1[0] = '\0';
1531	RTStrPrintf(pGVM->vmm.s.szRing0AssertMsg2, sizeof(pGVM->vmm.s.szRing0AssertMsg2),
1532	"Thread-context hooks still enabled! VCPU=%p Id=%u rc=%d.\n", pGVCpu, pGVCpu->idCpu, rc);
1533	rc = VERR_VMM_CONTEXT_HOOK_STILL_ENABLED;
1534	}
1535	# endif
1536
1537	VMMRZCallRing3Disable(pGVCpu); /* Lazy bird: Simpler just disabling it again... */
1538	VMCPU_SET_STATE(pGVCpu, VMCPUSTATE_STARTED);
1539	}
1540	STAM_COUNTER_INC(&pGVM->vmm.s.StatRunGC);
1541
1542	/*
1543	* Invalidate the host CPU identifiers before we disable the context
1544	* hook / restore preemption.
1545	*/
1546	pGVCpu->iHostCpuSet = UINT32_MAX;
1547	ASMAtomicWriteU32(&pGVCpu->idHostCpu, NIL_RTCPUID);
1548
1549	/*
1550	* Disable context hooks. Due to unresolved cleanup issues, we
1551	* cannot leave the hooks enabled when we return to ring-3.
1552	*
1553	* Note! At the moment HM may also have disabled the hook
1554	* when we get here, but the IPRT API handles that.
1555	*/
1556	if (pGVCpu->vmmr0.s.hCtxHook != NIL_RTTHREADCTXHOOK)
1557	RTThreadCtxHookDisable(pGVCpu->vmmr0.s.hCtxHook);
1558	}
1559	/*
1560	* The system is about to go into suspend mode; go back to ring 3.
1561	*/
1562	else
1563	{
1564	pGVCpu->iHostCpuSet = UINT32_MAX;
1565	ASMAtomicWriteU32(&pGVCpu->idHostCpu, NIL_RTCPUID);
1566	rc = VINF_EM_RAW_INTERRUPT;
1567	}
1568
1569	/** @todo When HM stops messing with the context hook state, we'll disable
1570	* preemption again before the RTThreadCtxHookDisable call. */
1571	if (!fPreemptRestored)
1572	{
1573	pGVCpu->vmmr0.s.pPreemptState = NULL;
1574	RTThreadPreemptRestore(&PreemptState);
1575	}
1576
1577	pGVCpu->vmm.s.iLastGZRc = rc;
1578
1579	/* Fire dtrace probe and collect statistics. */
1580	VBOXVMM_R0_VMM_RETURN_TO_RING3_HM(pGVCpu, CPUMQueryGuestCtxPtr(pGVCpu), rc);
1581	# ifdef VBOX_WITH_STATISTICS
1582	vmmR0RecordRC(pGVM, pGVCpu, rc);
1583	# endif
1584	VMMRZCallRing3Enable(pGVCpu);
1585
1586	/*
1587	* If this is a halt.
1588	*/
1589	if (rc != VINF_EM_HALT)
1590	{ /* we're not in a hurry for a HLT, so prefer this path */ }
1591	else
1592	{
1593	pGVCpu->vmm.s.iLastGZRc = rc = vmmR0DoHalt(pGVM, pGVCpu);
1594	if (rc == VINF_SUCCESS)
1595	{
1596	pGVCpu->vmm.s.cR0HaltsSucceeded++;
1597	continue;
1598	}
1599	pGVCpu->vmm.s.cR0HaltsToRing3++;
1600	}
1601	}
1602	/*
1603	* Invalid CPU set index or TSC delta in need of measuring.
1604	*/
1605	else
1606	{
1607	pGVCpu->vmmr0.s.pPreemptState = NULL;
1608	pGVCpu->iHostCpuSet = UINT32_MAX;
1609	ASMAtomicWriteU32(&pGVCpu->idHostCpu, NIL_RTCPUID);
1610	RTThreadPreemptRestore(&PreemptState);
1611
1612	VMMRZCallRing3Enable(pGVCpu);
1613
1614	if (iHostCpuSet < RTCPUSET_MAX_CPUS)
1615	{
1616	int rc = SUPR0TscDeltaMeasureBySetIndex(pGVM->pSession, iHostCpuSet, 0 /fFlags/,
1617	2 /cMsWaitRetry/, 5RT_MS_1SEC /cMsWaitThread*/,
1618	0 /default cTries/);
1619	if (RT_SUCCESS(rc) \|\| rc == VERR_CPU_OFFLINE)
1620	pGVCpu->vmm.s.iLastGZRc = VINF_EM_RAW_TO_R3;
1621	else
1622	pGVCpu->vmm.s.iLastGZRc = rc;
1623	}
1624	else
1625	pGVCpu->vmm.s.iLastGZRc = VERR_INVALID_CPU_INDEX;
1626	}
1627	break;
1628	} /* halt loop. */
1629	break;
1630	}
1631
1632	# ifdef VBOX_WITH_NEM_R0
1633	# if defined(RT_ARCH_AMD64) && defined(RT_OS_WINDOWS)
1634	case VMMR0_DO_NEM_RUN:
1635	{
1636	/*
1637	* Setup the longjmp machinery and execute guest code (calls NEMR0RunGuestCode).
1638	*/
1639	# ifdef VBOXSTRICTRC_STRICT_ENABLED
1640	int rc = vmmR0CallRing3SetJmp2(&pGVCpu->vmmr0.s.AssertJmpBuf, (PFNVMMR0SETJMP2)NEMR0RunGuestCode, pGVM, idCpu);
1641	# else
1642	int rc = vmmR0CallRing3SetJmp2(&pGVCpu->vmmr0.s.AssertJmpBuf, NEMR0RunGuestCode, pGVM, idCpu);
1643	# endif
1644	STAM_COUNTER_INC(&pGVM->vmm.s.StatRunGC);
1645
1646	pGVCpu->vmm.s.iLastGZRc = rc;
1647
1648	/*
1649	* Fire dtrace probe and collect statistics.
1650	*/
1651	VBOXVMM_R0_VMM_RETURN_TO_RING3_NEM(pGVCpu, CPUMQueryGuestCtxPtr(pGVCpu), rc);
1652	# ifdef VBOX_WITH_STATISTICS
1653	vmmR0RecordRC(pGVM, pGVCpu, rc);
1654	# endif
1655	break;
1656	}
1657	# endif
1658	# endif
1659
1660	#endif /* !VBOX_WITH_MINIMAL_R0 */
1661
1662	/*
1663	* For profiling.
1664	*/
1665	case VMMR0_DO_NOP:
1666	pGVCpu->vmm.s.iLastGZRc = VINF_SUCCESS;
1667	break;
1668
1669	/*
1670	* Shouldn't happen.
1671	*/
1672	default:
1673	AssertMsgFailed(("%#x\n", enmOperation));
1674	pGVCpu->vmm.s.iLastGZRc = VERR_NOT_SUPPORTED;
1675	break;
1676	}
1677	}
1678
1679
1680	/**
1681	* Validates a session or VM session argument.
1682	*
1683	* @returns true / false accordingly.
1684	* @param pGVM The global (ring-0) VM structure.
1685	* @param pClaimedSession The session claim to validate.
1686	* @param pSession The session argument.
1687	*/
1688	DECLINLINE(bool) vmmR0IsValidSession(PGVM pGVM, PSUPDRVSESSION pClaimedSession, PSUPDRVSESSION pSession)
1689	{
1690	/* This must be set! */
1691	if (!pSession)
1692	return false;
1693
1694	/* Only one out of the two. */
1695	if (pGVM && pClaimedSession)
1696	return false;
1697	if (pGVM)
1698	pClaimedSession = pGVM->pSession;
1699	return pClaimedSession == pSession;
1700	}
1701
1702
1703	/**
1704	* VMMR0EntryEx worker function, either called directly or when ever possible
1705	* called thru a longjmp so we can exit safely on failure.
1706	*
1707	* @returns VBox status code.
1708	* @param pGVM The global (ring-0) VM structure.
1709	* @param idCpu Virtual CPU ID argument. Must be NIL_VMCPUID if pVM
1710	* is NIL_RTR0PTR, and may be NIL_VMCPUID if it isn't
1711	* @param enmOperation Which operation to execute.
1712	* @param pReqHdr This points to a SUPVMMR0REQHDR packet. Optional.
1713	* The support driver validates this if it's present.
1714	* @param u64Arg Some simple constant argument.
1715	* @param pSession The session of the caller.
1716	*
1717	* @remarks Assume called with interrupts _enabled_.
1718	*/
1719	DECL_NO_INLINE(static, int) vmmR0EntryExWorker(PGVM pGVM, VMCPUID idCpu, VMMR0OPERATION enmOperation,
1720	PSUPVMMR0REQHDR pReqHdr, uint64_t u64Arg, PSUPDRVSESSION pSession)
1721	{
1722	/*
1723	* Validate pGVM and idCpu for consistency and validity.
1724	*/
1725	if (pGVM != NULL)
1726	{
1727	if (RT_LIKELY(((uintptr_t)pGVM & HOST_PAGE_OFFSET_MASK) == 0))
1728	{ /* likely */ }
1729	else
1730	{
1731	SUPR0Printf("vmmR0EntryExWorker: Invalid pGVM=%p! (op=%d)\n", pGVM, enmOperation);
1732	return VERR_INVALID_POINTER;
1733	}
1734
1735	if (RT_LIKELY(idCpu == NIL_VMCPUID \|\| idCpu < pGVM->cCpus))
1736	{ /* likely */ }
1737	else
1738	{
1739	SUPR0Printf("vmmR0EntryExWorker: Invalid idCpu %#x (cCpus=%#x)\n", idCpu, pGVM->cCpus);
1740	return VERR_INVALID_PARAMETER;
1741	}
1742
1743	if (RT_LIKELY( pGVM->enmVMState >= VMSTATE_CREATING
1744	&& pGVM->enmVMState <= VMSTATE_TERMINATED
1745	&& pGVM->pSession == pSession
1746	&& pGVM->pSelf == pGVM))
1747	{ /* likely */ }
1748	else
1749	{
1750	SUPR0Printf("vmmR0EntryExWorker: Invalid pGVM=%p:{.enmVMState=%d, .cCpus=%#x, .pSession=%p(==%p), .pSelf=%p(==%p)}! (op=%d)\n",
1751	pGVM, pGVM->enmVMState, pGVM->cCpus, pGVM->pSession, pSession, pGVM->pSelf, pGVM, enmOperation);
1752	return VERR_INVALID_POINTER;
1753	}
1754	}
1755	else if (RT_LIKELY(idCpu == NIL_VMCPUID))
1756	{ /* likely */ }
1757	else
1758	{
1759	SUPR0Printf("vmmR0EntryExWorker: Invalid idCpu=%u\n", idCpu);
1760	return VERR_INVALID_PARAMETER;
1761	}
1762
1763	/*
1764	* Process the request.
1765	*/
1766	int rc;
1767	switch (enmOperation)
1768	{
1769	/*
1770	* GVM requests
1771	*/
1772	case VMMR0_DO_GVMM_CREATE_VM:
1773	if (pGVM == NULL && u64Arg == 0 && idCpu == NIL_VMCPUID)
1774	rc = GVMMR0CreateVMReq((PGVMMCREATEVMREQ)pReqHdr, pSession);
1775	else
1776	rc = VERR_INVALID_PARAMETER;
1777	break;
1778
1779	case VMMR0_DO_GVMM_DESTROY_VM:
1780	if (pReqHdr == NULL && u64Arg == 0)
1781	rc = GVMMR0DestroyVM(pGVM);
1782	else
1783	rc = VERR_INVALID_PARAMETER;
1784	break;
1785
1786	case VMMR0_DO_GVMM_REGISTER_VMCPU:
1787	if (pGVM != NULL)
1788	rc = GVMMR0RegisterVCpu(pGVM, idCpu);
1789	else
1790	rc = VERR_INVALID_PARAMETER;
1791	break;
1792
1793	case VMMR0_DO_GVMM_DEREGISTER_VMCPU:
1794	if (pGVM != NULL)
1795	rc = GVMMR0DeregisterVCpu(pGVM, idCpu);
1796	else
1797	rc = VERR_INVALID_PARAMETER;
1798	break;
1799
1800	case VMMR0_DO_GVMM_REGISTER_WORKER_THREAD:
1801	if (pGVM != NULL && pReqHdr && pReqHdr->cbReq == sizeof(GVMMREGISTERWORKERTHREADREQ))
1802	rc = GVMMR0RegisterWorkerThread(pGVM, (GVMMWORKERTHREAD)(unsigned)u64Arg,
1803	((PGVMMREGISTERWORKERTHREADREQ)(pReqHdr))->hNativeThreadR3);
1804	else
1805	rc = VERR_INVALID_PARAMETER;
1806	break;
1807
1808	case VMMR0_DO_GVMM_DEREGISTER_WORKER_THREAD:
1809	if (pGVM != NULL)
1810	rc = GVMMR0DeregisterWorkerThread(pGVM, (GVMMWORKERTHREAD)(unsigned)u64Arg);
1811	else
1812	rc = VERR_INVALID_PARAMETER;
1813	break;
1814
1815	case VMMR0_DO_GVMM_SCHED_HALT:
1816	if (pReqHdr)
1817	return VERR_INVALID_PARAMETER;
1818	rc = GVMMR0SchedHaltReq(pGVM, idCpu, u64Arg);
1819	break;
1820
1821	case VMMR0_DO_GVMM_SCHED_WAKE_UP:
1822	if (pReqHdr \|\| u64Arg)
1823	return VERR_INVALID_PARAMETER;
1824	rc = GVMMR0SchedWakeUp(pGVM, idCpu);
1825	break;
1826
1827	case VMMR0_DO_GVMM_SCHED_POKE:
1828	if (pReqHdr \|\| u64Arg)
1829	return VERR_INVALID_PARAMETER;
1830	rc = GVMMR0SchedPoke(pGVM, idCpu);
1831	break;
1832
1833	case VMMR0_DO_GVMM_SCHED_WAKE_UP_AND_POKE_CPUS:
1834	if (u64Arg)
1835	return VERR_INVALID_PARAMETER;
1836	rc = GVMMR0SchedWakeUpAndPokeCpusReq(pGVM, (PGVMMSCHEDWAKEUPANDPOKECPUSREQ)pReqHdr);
1837	break;
1838
1839	case VMMR0_DO_GVMM_SCHED_POLL:
1840	if (pReqHdr \|\| u64Arg > 1)
1841	return VERR_INVALID_PARAMETER;
1842	rc = GVMMR0SchedPoll(pGVM, idCpu, !!u64Arg);
1843	break;
1844
1845	case VMMR0_DO_GVMM_QUERY_STATISTICS:
1846	if (u64Arg)
1847	return VERR_INVALID_PARAMETER;
1848	rc = GVMMR0QueryStatisticsReq(pGVM, (PGVMMQUERYSTATISTICSSREQ)pReqHdr, pSession);
1849	break;
1850
1851	case VMMR0_DO_GVMM_RESET_STATISTICS:
1852	if (u64Arg)
1853	return VERR_INVALID_PARAMETER;
1854	rc = GVMMR0ResetStatisticsReq(pGVM, (PGVMMRESETSTATISTICSSREQ)pReqHdr, pSession);
1855	break;
1856
1857	/*
1858	* Initialize the R0 part of a VM instance.
1859	*/
1860	case VMMR0_DO_VMMR0_INIT:
1861	rc = vmmR0InitVM(pGVM, RT_LODWORD(u64Arg), RT_HIDWORD(u64Arg));
1862	break;
1863
1864	/*
1865	* Does EMT specific ring-0 init.
1866	*/
1867	case VMMR0_DO_VMMR0_INIT_EMT:
1868	if (idCpu == NIL_VMCPUID)
1869	return VERR_INVALID_CPU_ID;
1870	rc = vmmR0InitVMEmt(pGVM, idCpu);
1871	break;
1872
1873	/*
1874	* Terminate the R0 part of a VM instance.
1875	*/
1876	case VMMR0_DO_VMMR0_TERM:
1877	rc = VMMR0TermVM(pGVM, 0 /idCpu/);
1878	break;
1879
1880	/*
1881	* Update release or debug logger instances.
1882	*/
1883	case VMMR0_DO_VMMR0_UPDATE_LOGGERS:
1884	if (idCpu == NIL_VMCPUID)
1885	return VERR_INVALID_CPU_ID;
1886	if (!(u64Arg & ~VMMR0UPDATELOGGER_F_VALID_MASK) && pReqHdr != NULL)
1887	rc = vmmR0UpdateLoggers(pGVM, idCpu /idCpu/, (PVMMR0UPDATELOGGERSREQ)pReqHdr, u64Arg);
1888	else
1889	return VERR_INVALID_PARAMETER;
1890	break;
1891
1892	/*
1893	* Log flusher thread.
1894	*/
1895	case VMMR0_DO_VMMR0_LOG_FLUSHER:
1896	if (idCpu != NIL_VMCPUID)
1897	return VERR_INVALID_CPU_ID;
1898	if (pReqHdr == NULL && pGVM != NULL)
1899	rc = vmmR0LogFlusher(pGVM);
1900	else
1901	return VERR_INVALID_PARAMETER;
1902	break;
1903
1904	/*
1905	* Wait for the flush to finish with all the buffers for the given logger.
1906	*/
1907	case VMMR0_DO_VMMR0_LOG_WAIT_FLUSHED:
1908	if (idCpu == NIL_VMCPUID)
1909	return VERR_INVALID_CPU_ID;
1910	if (u64Arg < VMMLOGGER_IDX_MAX && pReqHdr == NULL)
1911	rc = vmmR0LogWaitFlushed(pGVM, idCpu /idCpu/, (size_t)u64Arg);
1912	else
1913	return VERR_INVALID_PARAMETER;
1914	break;
1915
1916	#ifndef VBOX_WITH_MINIMAL_R0
1917
1918	/*
1919	* Attempt to enable hm mode and check the current setting.
1920	*/
1921	case VMMR0_DO_HM_ENABLE:
1922	rc = HMR0EnableAllCpus(pGVM);
1923	break;
1924
1925	/*
1926	* Setup the hardware accelerated session.
1927	*/
1928	case VMMR0_DO_HM_SETUP_VM:
1929	rc = HMR0SetupVM(pGVM);
1930	break;
1931
1932	/*
1933	* PGM wrappers.
1934	*/
1935	case VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES:
1936	if (idCpu == NIL_VMCPUID)
1937	return VERR_INVALID_CPU_ID;
1938	rc = PGMR0PhysAllocateHandyPages(pGVM, idCpu);
1939	break;
1940
1941	case VMMR0_DO_PGM_FLUSH_HANDY_PAGES:
1942	if (idCpu == NIL_VMCPUID)
1943	return VERR_INVALID_CPU_ID;
1944	rc = PGMR0PhysFlushHandyPages(pGVM, idCpu);
1945	break;
1946
1947	case VMMR0_DO_PGM_ALLOCATE_LARGE_PAGE:
1948	if (idCpu == NIL_VMCPUID)
1949	return VERR_INVALID_CPU_ID;
1950	rc = PGMR0PhysAllocateLargePage(pGVM, idCpu, u64Arg);
1951	break;
1952
1953	case VMMR0_DO_PGM_PHYS_SETUP_IOMMU:
1954	if (idCpu != 0)
1955	return VERR_INVALID_CPU_ID;
1956	rc = PGMR0PhysSetupIoMmu(pGVM);
1957	break;
1958
1959	case VMMR0_DO_PGM_POOL_GROW:
1960	if (idCpu == NIL_VMCPUID)
1961	return VERR_INVALID_CPU_ID;
1962	rc = PGMR0PoolGrow(pGVM, idCpu);
1963	break;
1964
1965	case VMMR0_DO_PGM_PHYS_HANDLER_INIT:
1966	if (idCpu != 0 \|\| pReqHdr != NULL \|\| u64Arg > UINT32_MAX)
1967	return VERR_INVALID_PARAMETER;
1968	rc = PGMR0PhysHandlerInitReqHandler(pGVM, (uint32_t)u64Arg);
1969	break;
1970
1971	case VMMR0_DO_PGM_PHYS_ALLOCATE_RAM_RANGE:
1972	if (idCpu != 0 \|\| u64Arg)
1973	return VERR_INVALID_PARAMETER;
1974	rc = PGMR0PhysAllocateRamRangeReq(pGVM, (PPGMPHYSALLOCATERAMRANGEREQ)pReqHdr);
1975	break;
1976
1977	case VMMR0_DO_PGM_PHYS_MMIO2_REGISTER:
1978	if (idCpu != 0 \|\| u64Arg)
1979	return VERR_INVALID_PARAMETER;
1980	rc = PGMR0PhysMmio2RegisterReq(pGVM, (PPGMPHYSMMIO2REGISTERREQ)pReqHdr);
1981	break;
1982
1983	case VMMR0_DO_PGM_PHYS_MMIO2_DEREGISTER:
1984	if (idCpu != 0 \|\| u64Arg)
1985	return VERR_INVALID_PARAMETER;
1986	rc = PGMR0PhysMmio2DeregisterReq(pGVM, (PPGMPHYSMMIO2DEREGISTERREQ)pReqHdr);
1987	break;
1988
1989	case VMMR0_DO_PGM_PHYS_ROM_ALLOCATE_RANGE:
1990	if (idCpu != 0 \|\| u64Arg)
1991	return VERR_INVALID_PARAMETER;
1992	rc = PGMR0PhysRomAllocateRangeReq(pGVM, (PPGMPHYSROMALLOCATERANGEREQ)pReqHdr);
1993	break;
1994
1995	/*
1996	* GMM wrappers.
1997	*/
1998	case VMMR0_DO_GMM_INITIAL_RESERVATION:
1999	if (u64Arg)
2000	return VERR_INVALID_PARAMETER;
2001	rc = GMMR0InitialReservationReq(pGVM, idCpu, (PGMMINITIALRESERVATIONREQ)pReqHdr);
2002	break;
2003
2004	case VMMR0_DO_GMM_UPDATE_RESERVATION:
2005	if (u64Arg)
2006	return VERR_INVALID_PARAMETER;
2007	rc = GMMR0UpdateReservationReq(pGVM, idCpu, (PGMMUPDATERESERVATIONREQ)pReqHdr);
2008	break;
2009
2010	case VMMR0_DO_GMM_ALLOCATE_PAGES:
2011	if (u64Arg)
2012	return VERR_INVALID_PARAMETER;
2013	rc = GMMR0AllocatePagesReq(pGVM, idCpu, (PGMMALLOCATEPAGESREQ)pReqHdr);
2014	break;
2015
2016	case VMMR0_DO_GMM_FREE_PAGES:
2017	if (u64Arg)
2018	return VERR_INVALID_PARAMETER;
2019	rc = GMMR0FreePagesReq(pGVM, idCpu, (PGMMFREEPAGESREQ)pReqHdr);
2020	break;
2021
2022	case VMMR0_DO_GMM_FREE_LARGE_PAGE:
2023	if (u64Arg)
2024	return VERR_INVALID_PARAMETER;
2025	rc = GMMR0FreeLargePageReq(pGVM, idCpu, (PGMMFREELARGEPAGEREQ)pReqHdr);
2026	break;
2027
2028	case VMMR0_DO_GMM_QUERY_HYPERVISOR_MEM_STATS:
2029	if (u64Arg)
2030	return VERR_INVALID_PARAMETER;
2031	rc = GMMR0QueryHypervisorMemoryStatsReq((PGMMMEMSTATSREQ)pReqHdr);
2032	break;
2033
2034	case VMMR0_DO_GMM_QUERY_MEM_STATS:
2035	if (idCpu == NIL_VMCPUID)
2036	return VERR_INVALID_CPU_ID;
2037	if (u64Arg)
2038	return VERR_INVALID_PARAMETER;
2039	rc = GMMR0QueryMemoryStatsReq(pGVM, idCpu, (PGMMMEMSTATSREQ)pReqHdr);
2040	break;
2041
2042	case VMMR0_DO_GMM_BALLOONED_PAGES:
2043	if (u64Arg)
2044	return VERR_INVALID_PARAMETER;
2045	rc = GMMR0BalloonedPagesReq(pGVM, idCpu, (PGMMBALLOONEDPAGESREQ)pReqHdr);
2046	break;
2047
2048	case VMMR0_DO_GMM_MAP_UNMAP_CHUNK:
2049	if (u64Arg)
2050	return VERR_INVALID_PARAMETER;
2051	rc = GMMR0MapUnmapChunkReq(pGVM, (PGMMMAPUNMAPCHUNKREQ)pReqHdr);
2052	break;
2053
2054	case VMMR0_DO_GMM_REGISTER_SHARED_MODULE:
2055	if (idCpu == NIL_VMCPUID)
2056	return VERR_INVALID_CPU_ID;
2057	if (u64Arg)
2058	return VERR_INVALID_PARAMETER;
2059	rc = GMMR0RegisterSharedModuleReq(pGVM, idCpu, (PGMMREGISTERSHAREDMODULEREQ)pReqHdr);
2060	break;
2061
2062	case VMMR0_DO_GMM_UNREGISTER_SHARED_MODULE:
2063	if (idCpu == NIL_VMCPUID)
2064	return VERR_INVALID_CPU_ID;
2065	if (u64Arg)
2066	return VERR_INVALID_PARAMETER;
2067	rc = GMMR0UnregisterSharedModuleReq(pGVM, idCpu, (PGMMUNREGISTERSHAREDMODULEREQ)pReqHdr);
2068	break;
2069
2070	case VMMR0_DO_GMM_RESET_SHARED_MODULES:
2071	if (idCpu == NIL_VMCPUID)
2072	return VERR_INVALID_CPU_ID;
2073	if ( u64Arg
2074	\|\| pReqHdr)
2075	return VERR_INVALID_PARAMETER;
2076	rc = GMMR0ResetSharedModules(pGVM, idCpu);
2077	break;
2078
2079	# ifdef VBOX_WITH_PAGE_SHARING
2080	case VMMR0_DO_GMM_CHECK_SHARED_MODULES:
2081	{
2082	if (idCpu == NIL_VMCPUID)
2083	return VERR_INVALID_CPU_ID;
2084	if ( u64Arg
2085	\|\| pReqHdr)
2086	return VERR_INVALID_PARAMETER;
2087	rc = GMMR0CheckSharedModules(pGVM, idCpu);
2088	break;
2089	}
2090	# endif
2091
2092	# if defined(VBOX_STRICT) && HC_ARCH_BITS == 64
2093	case VMMR0_DO_GMM_FIND_DUPLICATE_PAGE:
2094	if (u64Arg)
2095	return VERR_INVALID_PARAMETER;
2096	rc = GMMR0FindDuplicatePageReq(pGVM, (PGMMFINDDUPLICATEPAGEREQ)pReqHdr);
2097	break;
2098	# endif
2099
2100	case VMMR0_DO_GMM_QUERY_STATISTICS:
2101	if (u64Arg)
2102	return VERR_INVALID_PARAMETER;
2103	rc = GMMR0QueryStatisticsReq(pGVM, (PGMMQUERYSTATISTICSSREQ)pReqHdr);
2104	break;
2105
2106	case VMMR0_DO_GMM_RESET_STATISTICS:
2107	if (u64Arg)
2108	return VERR_INVALID_PARAMETER;
2109	rc = GMMR0ResetStatisticsReq(pGVM, (PGMMRESETSTATISTICSSREQ)pReqHdr);
2110	break;
2111
2112	#endif /* !VBOX_WITH_MINIMAL_R0 */
2113
2114	/*
2115	* A quick GCFGM mock-up.
2116	*/
2117	/** @todo GCFGM with proper access control, ring-3 management interface and all that. */
2118	case VMMR0_DO_GCFGM_SET_VALUE:
2119	case VMMR0_DO_GCFGM_QUERY_VALUE:
2120	{
2121	if (pGVM \|\| !pReqHdr \|\| u64Arg \|\| idCpu != NIL_VMCPUID)
2122	return VERR_INVALID_PARAMETER;
2123	PGCFGMVALUEREQ pReq = (PGCFGMVALUEREQ)pReqHdr;
2124	if (pReq->Hdr.cbReq != sizeof(*pReq))
2125	return VERR_INVALID_PARAMETER;
2126	if (enmOperation == VMMR0_DO_GCFGM_SET_VALUE)
2127	{
2128	rc = GVMMR0SetConfig(pReq->pSession, &pReq->szName[0], pReq->u64Value);
2129	//if (rc == VERR_CFGM_VALUE_NOT_FOUND)
2130	// rc = GMMR0SetConfig(pReq->pSession, &pReq->szName[0], pReq->u64Value);
2131	}
2132	else
2133	{
2134	rc = GVMMR0QueryConfig(pReq->pSession, &pReq->szName[0], &pReq->u64Value);
2135	//if (rc == VERR_CFGM_VALUE_NOT_FOUND)
2136	// rc = GMMR0QueryConfig(pReq->pSession, &pReq->szName[0], &pReq->u64Value);
2137	}
2138	break;
2139	}
2140
2141	#ifndef VBOX_WITH_MINIMAL_R0
2142	/*
2143	* PDM Wrappers.
2144	*/
2145	case VMMR0_DO_PDM_DRIVER_CALL_REQ_HANDLER:
2146	{
2147	if (!pReqHdr \|\| u64Arg \|\| idCpu != NIL_VMCPUID)
2148	return VERR_INVALID_PARAMETER;
2149	rc = PDMR0DriverCallReqHandler(pGVM, (PPDMDRIVERCALLREQHANDLERREQ)pReqHdr);
2150	break;
2151	}
2152
2153	case VMMR0_DO_PDM_DEVICE_CREATE:
2154	{
2155	if (!pReqHdr \|\| u64Arg \|\| idCpu != 0)
2156	return VERR_INVALID_PARAMETER;
2157	rc = PDMR0DeviceCreateReqHandler(pGVM, (PPDMDEVICECREATEREQ)pReqHdr);
2158	break;
2159	}
2160
2161	case VMMR0_DO_PDM_DEVICE_GEN_CALL:
2162	{
2163	if (!pReqHdr \|\| u64Arg)
2164	return VERR_INVALID_PARAMETER;
2165	rc = PDMR0DeviceGenCallReqHandler(pGVM, (PPDMDEVICEGENCALLREQ)pReqHdr, idCpu);
2166	break;
2167	}
2168
2169	/** @todo Remove the once all devices has been converted to new style! @bugref{9218} */
2170	case VMMR0_DO_PDM_DEVICE_COMPAT_SET_CRITSECT:
2171	{
2172	if (!pReqHdr \|\| u64Arg \|\| idCpu != 0)
2173	return VERR_INVALID_PARAMETER;
2174	rc = PDMR0DeviceCompatSetCritSectReqHandler(pGVM, (PPDMDEVICECOMPATSETCRITSECTREQ)pReqHdr);
2175	break;
2176	}
2177
2178	case VMMR0_DO_PDM_QUEUE_CREATE:
2179	{
2180	if (!pReqHdr \|\| u64Arg \|\| idCpu != 0)
2181	return VERR_INVALID_PARAMETER;
2182	rc = PDMR0QueueCreateReqHandler(pGVM, (PPDMQUEUECREATEREQ)pReqHdr);
2183	break;
2184	}
2185	#endif /* !VBOX_WITH_MINIMAL_R0 */
2186
2187	/*
2188	* Requests to the internal networking service.
2189	*/
2190	case VMMR0_DO_INTNET_OPEN:
2191	{
2192	PINTNETOPENREQ pReq = (PINTNETOPENREQ)pReqHdr;
2193	if (u64Arg \|\| !pReq \|\| !vmmR0IsValidSession(pGVM, pReq->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2194	return VERR_INVALID_PARAMETER;
2195	rc = IntNetR0OpenReq(pSession, pReq);
2196	break;
2197	}
2198
2199	case VMMR0_DO_INTNET_IF_CLOSE:
2200	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pGVM, ((PINTNETIFCLOSEREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2201	return VERR_INVALID_PARAMETER;
2202	rc = IntNetR0IfCloseReq(pSession, (PINTNETIFCLOSEREQ)pReqHdr);
2203	break;
2204
2205
2206	case VMMR0_DO_INTNET_IF_GET_BUFFER_PTRS:
2207	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pGVM, ((PINTNETIFGETBUFFERPTRSREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2208	return VERR_INVALID_PARAMETER;
2209	rc = IntNetR0IfGetBufferPtrsReq(pSession, (PINTNETIFGETBUFFERPTRSREQ)pReqHdr);
2210	break;
2211
2212	case VMMR0_DO_INTNET_IF_SET_PROMISCUOUS_MODE:
2213	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pGVM, ((PINTNETIFSETPROMISCUOUSMODEREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2214	return VERR_INVALID_PARAMETER;
2215	rc = IntNetR0IfSetPromiscuousModeReq(pSession, (PINTNETIFSETPROMISCUOUSMODEREQ)pReqHdr);
2216	break;
2217
2218	case VMMR0_DO_INTNET_IF_SET_MAC_ADDRESS:
2219	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pGVM, ((PINTNETIFSETMACADDRESSREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2220	return VERR_INVALID_PARAMETER;
2221	rc = IntNetR0IfSetMacAddressReq(pSession, (PINTNETIFSETMACADDRESSREQ)pReqHdr);
2222	break;
2223
2224	case VMMR0_DO_INTNET_IF_SET_ACTIVE:
2225	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pGVM, ((PINTNETIFSETACTIVEREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2226	return VERR_INVALID_PARAMETER;
2227	rc = IntNetR0IfSetActiveReq(pSession, (PINTNETIFSETACTIVEREQ)pReqHdr);
2228	break;
2229
2230	case VMMR0_DO_INTNET_IF_SEND:
2231	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pGVM, ((PINTNETIFSENDREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2232	return VERR_INVALID_PARAMETER;
2233	rc = IntNetR0IfSendReq(pSession, (PINTNETIFSENDREQ)pReqHdr);
2234	break;
2235
2236	case VMMR0_DO_INTNET_IF_WAIT:
2237	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pGVM, ((PINTNETIFWAITREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2238	return VERR_INVALID_PARAMETER;
2239	rc = IntNetR0IfWaitReq(pSession, (PINTNETIFWAITREQ)pReqHdr);
2240	break;
2241
2242	case VMMR0_DO_INTNET_IF_ABORT_WAIT:
2243	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pGVM, ((PINTNETIFWAITREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2244	return VERR_INVALID_PARAMETER;
2245	rc = IntNetR0IfAbortWaitReq(pSession, (PINTNETIFABORTWAITREQ)pReqHdr);
2246	break;
2247
2248	#if 0 //defined(VBOX_WITH_PCI_PASSTHROUGH) && !defined(VBOX_WITH_MINIMAL_R0)
2249	/*
2250	* Requests to host PCI driver service.
2251	*/
2252	case VMMR0_DO_PCIRAW_REQ:
2253	if (u64Arg \|\| !pReqHdr \|\| !vmmR0IsValidSession(pGVM, ((PPCIRAWSENDREQ)pReqHdr)->pSession, pSession) \|\| idCpu != NIL_VMCPUID)
2254	return VERR_INVALID_PARAMETER;
2255	rc = PciRawR0ProcessReq(pGVM, pSession, (PPCIRAWSENDREQ)pReqHdr);
2256	break;
2257	#endif
2258
2259	#ifndef VBOX_WITH_MINIMAL_R0
2260
2261	/*
2262	* NEM requests.
2263	*/
2264	# ifdef VBOX_WITH_NEM_R0
2265	# if defined(RT_ARCH_AMD64) && defined(RT_OS_WINDOWS)
2266	case VMMR0_DO_NEM_INIT_VM:
2267	if (u64Arg \|\| pReqHdr \|\| idCpu != 0)
2268	return VERR_INVALID_PARAMETER;
2269	rc = NEMR0InitVM(pGVM);
2270	break;
2271
2272	case VMMR0_DO_NEM_INIT_VM_PART_2:
2273	if (u64Arg \|\| pReqHdr \|\| idCpu != 0)
2274	return VERR_INVALID_PARAMETER;
2275	rc = NEMR0InitVMPart2(pGVM);
2276	break;
2277
2278	case VMMR0_DO_NEM_MAP_PAGES:
2279	if (u64Arg \|\| pReqHdr \|\| idCpu == NIL_VMCPUID)
2280	return VERR_INVALID_PARAMETER;
2281	rc = NEMR0MapPages(pGVM, idCpu);
2282	break;
2283
2284	case VMMR0_DO_NEM_UNMAP_PAGES:
2285	if (u64Arg \|\| pReqHdr \|\| idCpu == NIL_VMCPUID)
2286	return VERR_INVALID_PARAMETER;
2287	rc = NEMR0UnmapPages(pGVM, idCpu);
2288	break;
2289
2290	case VMMR0_DO_NEM_EXPORT_STATE:
2291	if (u64Arg \|\| pReqHdr \|\| idCpu == NIL_VMCPUID)
2292	return VERR_INVALID_PARAMETER;
2293	rc = NEMR0ExportState(pGVM, idCpu);
2294	break;
2295
2296	case VMMR0_DO_NEM_IMPORT_STATE:
2297	if (pReqHdr \|\| idCpu == NIL_VMCPUID)
2298	return VERR_INVALID_PARAMETER;
2299	rc = NEMR0ImportState(pGVM, idCpu, u64Arg);
2300	break;
2301
2302	case VMMR0_DO_NEM_QUERY_CPU_TICK:
2303	if (u64Arg \|\| pReqHdr \|\| idCpu == NIL_VMCPUID)
2304	return VERR_INVALID_PARAMETER;
2305	rc = NEMR0QueryCpuTick(pGVM, idCpu);
2306	break;
2307
2308	case VMMR0_DO_NEM_RESUME_CPU_TICK_ON_ALL:
2309	if (pReqHdr \|\| idCpu == NIL_VMCPUID)
2310	return VERR_INVALID_PARAMETER;
2311	rc = NEMR0ResumeCpuTickOnAll(pGVM, idCpu, u64Arg);
2312	break;
2313
2314	case VMMR0_DO_NEM_UPDATE_STATISTICS:
2315	if (u64Arg \|\| pReqHdr)
2316	return VERR_INVALID_PARAMETER;
2317	rc = NEMR0UpdateStatistics(pGVM, idCpu);
2318	break;
2319
2320	# if 1 && defined(DEBUG_bird)
2321	case VMMR0_DO_NEM_EXPERIMENT:
2322	if (pReqHdr)
2323	return VERR_INVALID_PARAMETER;
2324	rc = NEMR0DoExperiment(pGVM, idCpu, u64Arg);
2325	break;
2326	# endif
2327	# endif
2328	# endif
2329
2330	/*
2331	* IOM requests.
2332	*/
2333	case VMMR0_DO_IOM_GROW_IO_PORTS:
2334	{
2335	if (pReqHdr \|\| idCpu != 0)
2336	return VERR_INVALID_PARAMETER;
2337	rc = IOMR0IoPortGrowRegistrationTables(pGVM, u64Arg);
2338	break;
2339	}
2340
2341	case VMMR0_DO_IOM_GROW_IO_PORT_STATS:
2342	{
2343	if (pReqHdr \|\| idCpu != 0)
2344	return VERR_INVALID_PARAMETER;
2345	rc = IOMR0IoPortGrowStatisticsTable(pGVM, u64Arg);
2346	break;
2347	}
2348
2349	case VMMR0_DO_IOM_GROW_MMIO_REGS:
2350	{
2351	if (pReqHdr \|\| idCpu != 0)
2352	return VERR_INVALID_PARAMETER;
2353	rc = IOMR0MmioGrowRegistrationTables(pGVM, u64Arg);
2354	break;
2355	}
2356
2357	case VMMR0_DO_IOM_GROW_MMIO_STATS:
2358	{
2359	if (pReqHdr \|\| idCpu != 0)
2360	return VERR_INVALID_PARAMETER;
2361	rc = IOMR0MmioGrowStatisticsTable(pGVM, u64Arg);
2362	break;
2363	}
2364
2365	case VMMR0_DO_IOM_SYNC_STATS_INDICES:
2366	{
2367	if (pReqHdr \|\| idCpu != 0)
2368	return VERR_INVALID_PARAMETER;
2369	rc = IOMR0IoPortSyncStatisticsIndices(pGVM);
2370	if (RT_SUCCESS(rc))
2371	rc = IOMR0MmioSyncStatisticsIndices(pGVM);
2372	break;
2373	}
2374
2375	/*
2376	* DBGF requests.
2377	*/
2378	# ifdef VBOX_WITH_DBGF_TRACING
2379	case VMMR0_DO_DBGF_TRACER_CREATE:
2380	{
2381	if (!pReqHdr \|\| u64Arg \|\| idCpu != 0)
2382	return VERR_INVALID_PARAMETER;
2383	rc = DBGFR0TracerCreateReqHandler(pGVM, (PDBGFTRACERCREATEREQ)pReqHdr);
2384	break;
2385	}
2386
2387	case VMMR0_DO_DBGF_TRACER_CALL_REQ_HANDLER:
2388	{
2389	if (!pReqHdr \|\| u64Arg)
2390	return VERR_INVALID_PARAMETER;
2391	# if 0 /** @todo */
2392	rc = DBGFR0TracerGenCallReqHandler(pGVM, (PDBGFTRACERGENCALLREQ)pReqHdr, idCpu);
2393	# else
2394	rc = VERR_NOT_IMPLEMENTED;
2395	# endif
2396	break;
2397	}
2398	# endif
2399
2400	case VMMR0_DO_DBGF_BP_INIT:
2401	{
2402	if (!pReqHdr \|\| u64Arg \|\| idCpu != 0)
2403	return VERR_INVALID_PARAMETER;
2404	rc = DBGFR0BpInitReqHandler(pGVM, (PDBGFBPINITREQ)pReqHdr);
2405	break;
2406	}
2407
2408	case VMMR0_DO_DBGF_BP_CHUNK_ALLOC:
2409	{
2410	if (!pReqHdr \|\| u64Arg \|\| idCpu != 0)
2411	return VERR_INVALID_PARAMETER;
2412	rc = DBGFR0BpChunkAllocReqHandler(pGVM, (PDBGFBPCHUNKALLOCREQ)pReqHdr);
2413	break;
2414	}
2415
2416	case VMMR0_DO_DBGF_BP_L2_TBL_CHUNK_ALLOC:
2417	{
2418	if (!pReqHdr \|\| u64Arg \|\| idCpu != 0)
2419	return VERR_INVALID_PARAMETER;
2420	rc = DBGFR0BpL2TblChunkAllocReqHandler(pGVM, (PDBGFBPL2TBLCHUNKALLOCREQ)pReqHdr);
2421	break;
2422	}
2423
2424	case VMMR0_DO_DBGF_BP_OWNER_INIT:
2425	{
2426	if (!pReqHdr \|\| u64Arg \|\| idCpu != 0)
2427	return VERR_INVALID_PARAMETER;
2428	rc = DBGFR0BpOwnerInitReqHandler(pGVM, (PDBGFBPOWNERINITREQ)pReqHdr);
2429	break;
2430	}
2431
2432	case VMMR0_DO_DBGF_BP_PORTIO_INIT:
2433	{
2434	if (!pReqHdr \|\| u64Arg \|\| idCpu != 0)
2435	return VERR_INVALID_PARAMETER;
2436	rc = DBGFR0BpPortIoInitReqHandler(pGVM, (PDBGFBPINITREQ)pReqHdr);
2437	break;
2438	}
2439
2440
2441	/*
2442	* TM requests.
2443	*/
2444	case VMMR0_DO_TM_GROW_TIMER_QUEUE:
2445	{
2446	if (pReqHdr \|\| idCpu == NIL_VMCPUID)
2447	return VERR_INVALID_PARAMETER;
2448	rc = TMR0TimerQueueGrow(pGVM, RT_HI_U32(u64Arg), RT_LO_U32(u64Arg));
2449	break;
2450	}
2451
2452	#endif /* n!VBOX_WITH_MINIMAL_R0 */
2453
2454	/*
2455	* For profiling.
2456	*/
2457	case VMMR0_DO_NOP:
2458	case VMMR0_DO_SLOW_NOP:
2459	return VINF_SUCCESS;
2460
2461	/*
2462	* For testing Ring-0 APIs invoked in this environment.
2463	*/
2464	case VMMR0_DO_TESTS:
2465	/** @todo make new test */
2466	return VINF_SUCCESS;
2467
2468	default:
2469	/*
2470	* We're returning VERR_NOT_SUPPORT here so we've got something else
2471	* than -1 which the interrupt gate glue code might return.
2472	*/
2473	Log(("operation %#x is not supported\n", enmOperation));
2474	return VERR_NOT_SUPPORTED;
2475	}
2476	return rc;
2477	}
2478
2479
2480	#ifndef RT_ARCH_ARM64 /** @todo port vmmR0CallRing3SetJmpEx to ARM64 */
2481	/**
2482	* This is just a longjmp wrapper function for VMMR0EntryEx calls.
2483	*
2484	* @returns VBox status code.
2485	* @param pvArgs The argument package
2486	*/
2487	static DECLCALLBACK(int) vmmR0EntryExWrapper(void *pvArgs)
2488	{
2489	PGVMCPU pGVCpu = (PGVMCPU)pvArgs;
2490	return vmmR0EntryExWorker(pGVCpu->vmmr0.s.pGVM,
2491	pGVCpu->vmmr0.s.idCpu,
2492	pGVCpu->vmmr0.s.enmOperation,
2493	pGVCpu->vmmr0.s.pReq,
2494	pGVCpu->vmmr0.s.u64Arg,
2495	pGVCpu->vmmr0.s.pSession);
2496	}
2497	#endif
2498
2499
2500	/**
2501	* The Ring 0 entry point, called by the support library (SUP).
2502	*
2503	* @returns VBox status code.
2504	* @param pGVM The global (ring-0) VM structure.
2505	* @param pVM The cross context VM structure.
2506	* @param idCpu Virtual CPU ID argument. Must be NIL_VMCPUID if pVM
2507	* is NIL_RTR0PTR, and may be NIL_VMCPUID if it isn't
2508	* @param enmOperation Which operation to execute.
2509	* @param pReq Pointer to the SUPVMMR0REQHDR packet. Optional.
2510	* @param u64Arg Some simple constant argument.
2511	* @param pSession The session of the caller.
2512	* @remarks Assume called with interrupts _enabled_.
2513	*/
2514	VMMR0DECL(int) VMMR0EntryEx(PGVM pGVM, PVMCC pVM, VMCPUID idCpu, VMMR0OPERATION enmOperation,
2515	PSUPVMMR0REQHDR pReq, uint64_t u64Arg, PSUPDRVSESSION pSession)
2516	{
2517	#ifndef RT_ARCH_ARM64 /** @todo port vmmR0CallRing3SetJmpEx to ARM64 - see RTAssertShouldPanic */
2518	/*
2519	* Requests that should only happen on the EMT thread will be
2520	* wrapped in a setjmp so we can assert without causing too much trouble.
2521	*/
2522	if ( pVM != NULL
2523	&& pGVM != NULL
2524	&& pVM == pGVM /** @todo drop pVM or pGVM */
2525	&& idCpu < pGVM->cCpus
2526	&& pGVM->pSession == pSession
2527	&& pGVM->pSelf == pGVM
2528	&& enmOperation != VMMR0_DO_GVMM_DESTROY_VM
2529	&& enmOperation != VMMR0_DO_GVMM_REGISTER_VMCPU
2530	&& enmOperation != VMMR0_DO_GVMM_SCHED_WAKE_UP /* idCpu is not caller but target. Sigh. / /* @todo fix*/
2531	&& enmOperation != VMMR0_DO_GVMM_SCHED_POKE /* idCpu is not caller but target. Sigh. / /* @todo fix*/
2532	)
2533	{
2534	PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
2535	RTNATIVETHREAD hNativeThread = RTThreadNativeSelf();
2536	if (RT_LIKELY( pGVCpu->hEMT == hNativeThread
2537	&& pGVCpu->hNativeThreadR0 == hNativeThread))
2538	{
2539	pGVCpu->vmmr0.s.pGVM = pGVM;
2540	pGVCpu->vmmr0.s.idCpu = idCpu;
2541	pGVCpu->vmmr0.s.enmOperation = enmOperation;
2542	pGVCpu->vmmr0.s.pReq = pReq;
2543	pGVCpu->vmmr0.s.u64Arg = u64Arg;
2544	pGVCpu->vmmr0.s.pSession = pSession;
2545	return vmmR0CallRing3SetJmpEx(&pGVCpu->vmmr0.s.AssertJmpBuf, vmmR0EntryExWrapper, pGVCpu,
2546	((uintptr_t)u64Arg << 16) \| (uintptr_t)enmOperation);
2547	}
2548	return VERR_VM_THREAD_NOT_EMT;
2549	}
2550	#else
2551	RT_NOREF(pVM);
2552	#endif
2553	return vmmR0EntryExWorker(pGVM, idCpu, enmOperation, pReq, u64Arg, pSession);
2554	}
2555
2556
2557	/*********************************************************************************************************************************
2558	* EMT Blocking *
2559	*********************************************************************************************************************************/
2560
2561	/**
2562	* Checks whether we've armed the ring-0 long jump machinery.
2563	*
2564	* @returns @c true / @c false
2565	* @param pVCpu The cross context virtual CPU structure.
2566	* @thread EMT
2567	* @sa VMMIsLongJumpArmed
2568	*/
2569	VMMR0_INT_DECL(bool) VMMR0IsLongJumpArmed(PVMCPUCC pVCpu)
2570	{
2571	#ifdef RT_ARCH_X86
2572	return pVCpu->vmmr0.s.AssertJmpBuf.eip != 0;
2573	#else
2574	return pVCpu->vmmr0.s.AssertJmpBuf.rip != 0;
2575	#endif
2576	}
2577
2578
2579	/**
2580	* Locking helper that deals with HM context and checks if the thread can block.
2581	*
2582	* @returns VINF_SUCCESS if we can block. Returns @a rcBusy or
2583	* VERR_VMM_CANNOT_BLOCK if not able to block.
2584	* @param pVCpu The cross context virtual CPU structure of the calling
2585	* thread.
2586	* @param rcBusy What to return in case of a blocking problem. Will IPE
2587	* if VINF_SUCCESS and we cannot block.
2588	* @param pszCaller The caller (for logging problems).
2589	* @param pvLock The lock address (for logging problems).
2590	* @param pCtx Where to return context info for the resume call.
2591	* @thread EMT(pVCpu)
2592	*/
2593	VMMR0_INT_DECL(int) VMMR0EmtPrepareToBlock(PVMCPUCC pVCpu, int rcBusy, const char pszCaller, void pvLock,
2594	PVMMR0EMTBLOCKCTX pCtx)
2595	{
2596	const char *pszMsg;
2597
2598	/*
2599	* Check that we are allowed to block.
2600	*/
2601	if (RT_LIKELY(VMMRZCallRing3IsEnabled(pVCpu)))
2602	{
2603	/*
2604	* Are we in HM context and w/o a context hook? If so work the context hook.
2605	*/
2606	if (pVCpu->idHostCpu != NIL_RTCPUID)
2607	{
2608	Assert(pVCpu->iHostCpuSet != UINT32_MAX);
2609
2610	if (pVCpu->vmmr0.s.hCtxHook == NIL_RTTHREADCTXHOOK)
2611	{
2612	vmmR0ThreadCtxCallback(RTTHREADCTXEVENT_OUT, pVCpu);
2613	if (pVCpu->vmmr0.s.pPreemptState)
2614	RTThreadPreemptRestore(pVCpu->vmmr0.s.pPreemptState);
2615
2616	pCtx->uMagic = VMMR0EMTBLOCKCTX_MAGIC;
2617	pCtx->fWasInHmContext = true;
2618	return VINF_SUCCESS;
2619	}
2620	}
2621
2622	if (RT_LIKELY(!pVCpu->vmmr0.s.pPreemptState))
2623	{
2624	/*
2625	* Not in HM context or we've got hooks, so just check that preemption
2626	* is enabled.
2627	*/
2628	if (RT_LIKELY(RTThreadPreemptIsEnabled(NIL_RTTHREAD)))
2629	{
2630	pCtx->uMagic = VMMR0EMTBLOCKCTX_MAGIC;
2631	pCtx->fWasInHmContext = false;
2632	return VINF_SUCCESS;
2633	}
2634	pszMsg = "Preemption is disabled!";
2635	}
2636	else
2637	pszMsg = "Preemption state w/o HM state!";
2638	}
2639	else
2640	pszMsg = "Ring-3 calls are disabled!";
2641
2642	static uint32_t volatile s_cWarnings = 0;
2643	if (++s_cWarnings < 50)
2644	SUPR0Printf("VMMR0EmtPrepareToBlock: %s pvLock=%p pszCaller=%s rcBusy=%p\n", pszMsg, pvLock, pszCaller, rcBusy);
2645	pCtx->uMagic = VMMR0EMTBLOCKCTX_MAGIC_DEAD;
2646	pCtx->fWasInHmContext = false;
2647	return rcBusy != VINF_SUCCESS ? rcBusy : VERR_VMM_CANNOT_BLOCK;
2648	}
2649
2650
2651	/**
2652	* Counterpart to VMMR0EmtPrepareToBlock.
2653	*
2654	* @param pVCpu The cross context virtual CPU structure of the calling
2655	* thread.
2656	* @param pCtx The context structure used with VMMR0EmtPrepareToBlock.
2657	* @thread EMT(pVCpu)
2658	*/
2659	VMMR0_INT_DECL(void) VMMR0EmtResumeAfterBlocking(PVMCPUCC pVCpu, PVMMR0EMTBLOCKCTX pCtx)
2660	{
2661	AssertReturnVoid(pCtx->uMagic == VMMR0EMTBLOCKCTX_MAGIC);
2662	if (pCtx->fWasInHmContext)
2663	{
2664	if (pVCpu->vmmr0.s.pPreemptState)
2665	RTThreadPreemptDisable(pVCpu->vmmr0.s.pPreemptState);
2666
2667	pCtx->fWasInHmContext = false;
2668	vmmR0ThreadCtxCallback(RTTHREADCTXEVENT_IN, pVCpu);
2669	}
2670	pCtx->uMagic = VMMR0EMTBLOCKCTX_MAGIC_DEAD;
2671	}
2672
2673
2674	/**
2675	* Helper for waiting on an RTSEMEVENT, caller did VMMR0EmtPrepareToBlock.
2676	*
2677	* @returns
2678	* @retval VERR_THREAD_IS_TERMINATING
2679	* @retval VERR_TIMEOUT if we ended up waiting too long, either according to
2680	* @a cMsTimeout or to maximum wait values.
2681	*
2682	* @param pGVCpu The ring-0 virtual CPU structure.
2683	* @param fFlags VMMR0EMTWAIT_F_XXX.
2684	* @param hEvent The event to wait on.
2685	* @param cMsTimeout The timeout or RT_INDEFINITE_WAIT.
2686	*/
2687	VMMR0_INT_DECL(int) VMMR0EmtWaitEventInner(PGVMCPU pGVCpu, uint32_t fFlags, RTSEMEVENT hEvent, RTMSINTERVAL cMsTimeout)
2688	{
2689	AssertReturn(pGVCpu->hEMT == RTThreadNativeSelf(), VERR_VM_THREAD_NOT_EMT);
2690
2691	/*
2692	* Note! Similar code is found in the PDM critical sections too.
2693	*/
2694	uint64_t const nsStart = RTTimeNanoTS();
2695	uint64_t cNsMaxTotal = cMsTimeout == RT_INDEFINITE_WAIT
2696	? RT_NS_5MIN : RT_MIN(RT_NS_5MIN, RT_NS_1MS_64 * cMsTimeout);
2697	uint32_t cMsMaxOne = RT_MS_5SEC;
2698	bool fNonInterruptible = false;
2699	for (;;)
2700	{
2701	/* Wait. */
2702	int rcWait = !fNonInterruptible
2703	? RTSemEventWaitNoResume(hEvent, cMsMaxOne)
2704	: RTSemEventWait(hEvent, cMsMaxOne);
2705	if (RT_SUCCESS(rcWait))
2706	return rcWait;
2707
2708	if (rcWait == VERR_TIMEOUT \|\| rcWait == VERR_INTERRUPTED)
2709	{
2710	uint64_t const cNsElapsed = RTTimeNanoTS() - nsStart;
2711
2712	/*
2713	* Check the thread termination status.
2714	*/
2715	int const rcTerm = RTThreadQueryTerminationStatus(NIL_RTTHREAD);
2716	AssertMsg(rcTerm == VINF_SUCCESS \|\| rcTerm == VERR_NOT_SUPPORTED \|\| rcTerm == VINF_THREAD_IS_TERMINATING,
2717	("rcTerm=%Rrc\n", rcTerm));
2718	if ( rcTerm == VERR_NOT_SUPPORTED
2719	&& !fNonInterruptible
2720	&& cNsMaxTotal > RT_NS_1MIN)
2721	cNsMaxTotal = RT_NS_1MIN;
2722
2723	/* We return immediately if it looks like the thread is terminating. */
2724	if (rcTerm == VINF_THREAD_IS_TERMINATING)
2725	return VERR_THREAD_IS_TERMINATING;
2726
2727	/* We may suppress VERR_INTERRUPTED if VMMR0EMTWAIT_F_TRY_SUPPRESS_INTERRUPTED was
2728	specified, otherwise we'll just return it. */
2729	if (rcWait == VERR_INTERRUPTED)
2730	{
2731	if (!(fFlags & VMMR0EMTWAIT_F_TRY_SUPPRESS_INTERRUPTED))
2732	return VERR_INTERRUPTED;
2733	if (!fNonInterruptible)
2734	{
2735	/* First time: Adjust down the wait parameters and make sure we get at least
2736	one non-interruptible wait before timing out. */
2737	fNonInterruptible = true;
2738	cMsMaxOne = 32;
2739	uint64_t const cNsLeft = cNsMaxTotal - cNsElapsed;
2740	if (cNsLeft > RT_NS_10SEC)
2741	cNsMaxTotal = cNsElapsed + RT_NS_10SEC;
2742	continue;
2743	}
2744	}
2745
2746	/* Check for timeout. */
2747	if (cNsElapsed > cNsMaxTotal)
2748	return VERR_TIMEOUT;
2749	}
2750	else
2751	return rcWait;
2752	}
2753	/* not reached */
2754	}
2755
2756
2757	/**
2758	* Helper for signalling an SUPSEMEVENT.
2759	*
2760	* This may temporarily leave the HM context if the host requires that for
2761	* signalling SUPSEMEVENT objects.
2762	*
2763	* @returns VBox status code (see VMMR0EmtPrepareToBlock)
2764	* @param pGVM The ring-0 VM structure.
2765	* @param pGVCpu The ring-0 virtual CPU structure.
2766	* @param hEvent The event to signal.
2767	*/
2768	VMMR0_INT_DECL(int) VMMR0EmtSignalSupEvent(PGVM pGVM, PGVMCPU pGVCpu, SUPSEMEVENT hEvent)
2769	{
2770	AssertReturn(pGVCpu->hEMT == RTThreadNativeSelf(), VERR_VM_THREAD_NOT_EMT);
2771	if (RTSemEventIsSignalSafe())
2772	return SUPSemEventSignal(pGVM->pSession, hEvent);
2773
2774	VMMR0EMTBLOCKCTX Ctx;
2775	int rc = VMMR0EmtPrepareToBlock(pGVCpu, VINF_SUCCESS, __FUNCTION__, (void *)(uintptr_t)hEvent, &Ctx);
2776	if (RT_SUCCESS(rc))
2777	{
2778	rc = SUPSemEventSignal(pGVM->pSession, hEvent);
2779	VMMR0EmtResumeAfterBlocking(pGVCpu, &Ctx);
2780	}
2781	return rc;
2782	}
2783
2784
2785	/**
2786	* Helper for signalling an SUPSEMEVENT, variant supporting non-EMTs.
2787	*
2788	* This may temporarily leave the HM context if the host requires that for
2789	* signalling SUPSEMEVENT objects.
2790	*
2791	* @returns VBox status code (see VMMR0EmtPrepareToBlock)
2792	* @param pGVM The ring-0 VM structure.
2793	* @param hEvent The event to signal.
2794	*/
2795	VMMR0_INT_DECL(int) VMMR0EmtSignalSupEventByGVM(PGVM pGVM, SUPSEMEVENT hEvent)
2796	{
2797	if (!RTSemEventIsSignalSafe())
2798	{
2799	PGVMCPU pGVCpu = GVMMR0GetGVCpuByGVMandEMT(pGVM, NIL_RTNATIVETHREAD);
2800	if (pGVCpu)
2801	{
2802	VMMR0EMTBLOCKCTX Ctx;
2803	int rc = VMMR0EmtPrepareToBlock(pGVCpu, VINF_SUCCESS, __FUNCTION__, (void *)(uintptr_t)hEvent, &Ctx);
2804	if (RT_SUCCESS(rc))
2805	{
2806	rc = SUPSemEventSignal(pGVM->pSession, hEvent);
2807	VMMR0EmtResumeAfterBlocking(pGVCpu, &Ctx);
2808	}
2809	return rc;
2810	}
2811	}
2812	return SUPSemEventSignal(pGVM->pSession, hEvent);
2813	}
2814
2815
2816	/*********************************************************************************************************************************
2817	* Logging. *
2818	*********************************************************************************************************************************/
2819
2820	/**
2821	* VMMR0_DO_VMMR0_UPDATE_LOGGERS: Updates the EMT loggers for the VM.
2822	*
2823	* @returns VBox status code.
2824	* @param pGVM The global (ring-0) VM structure.
2825	* @param idCpu The ID of the calling EMT.
2826	* @param pReq The request data.
2827	* @param fFlags Flags, see VMMR0UPDATELOGGER_F_XXX.
2828	* @thread EMT(idCpu)
2829	*/
2830	static int vmmR0UpdateLoggers(PGVM pGVM, VMCPUID idCpu, PVMMR0UPDATELOGGERSREQ pReq, uint64_t fFlags)
2831	{
2832	/*
2833	* Check sanity. First we require EMT to be calling us.
2834	*/
2835	AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_CPU_ID);
2836	AssertReturn(pGVM->aCpus[idCpu].hEMT == RTThreadNativeSelf(), VERR_INVALID_CPU_ID);
2837
2838	AssertReturn(pReq->Hdr.cbReq >= RT_UOFFSETOF_DYN(VMMR0UPDATELOGGERSREQ, afGroups[0]), VERR_INVALID_PARAMETER);
2839	AssertReturn(pReq->cGroups < _8K, VERR_INVALID_PARAMETER);
2840	AssertReturn(pReq->Hdr.cbReq == RT_UOFFSETOF_DYN(VMMR0UPDATELOGGERSREQ, afGroups[pReq->cGroups]), VERR_INVALID_PARAMETER);
2841
2842	size_t const idxLogger = (size_t)(fFlags & VMMR0UPDATELOGGER_F_LOGGER_MASK);
2843	AssertReturn(idxLogger < VMMLOGGER_IDX_MAX, VERR_OUT_OF_RANGE);
2844
2845	/*
2846	* Adjust flags.
2847	*/
2848	/* Always buffered, unless logging directly to parent VMM: */
2849	if (!(fFlags & (VMMR0UPDATELOGGER_F_TO_PARENT_VMM_DBG \| VMMR0UPDATELOGGER_F_TO_PARENT_VMM_REL)))
2850	pReq->fFlags \|= RTLOGFLAGS_BUFFERED;
2851	/* These doesn't make sense at present: */
2852	pReq->fFlags &= ~(RTLOGFLAGS_FLUSH \| RTLOGFLAGS_WRITE_THROUGH);
2853	/* We've traditionally skipped the group restrictions. */
2854	pReq->fFlags &= ~RTLOGFLAGS_RESTRICT_GROUPS;
2855
2856	/*
2857	* Do the updating.
2858	*/
2859	int rc = VINF_SUCCESS;
2860	for (idCpu = 0; idCpu < pGVM->cCpus; idCpu++)
2861	{
2862	PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
2863	PRTLOGGER pLogger = pGVCpu->vmmr0.s.u.aLoggers[idxLogger].pLogger;
2864	if (pLogger)
2865	{
2866	pGVCpu->vmmr0.s.u.aLoggers[idxLogger].fFlushToParentVmmDbg = RT_BOOL(fFlags & VMMR0UPDATELOGGER_F_TO_PARENT_VMM_DBG);
2867	pGVCpu->vmmr0.s.u.aLoggers[idxLogger].fFlushToParentVmmRel = RT_BOOL(fFlags & VMMR0UPDATELOGGER_F_TO_PARENT_VMM_REL);
2868
2869	RTLogSetR0ProgramStart(pLogger, pGVM->vmm.s.nsProgramStart);
2870	rc = RTLogBulkUpdate(pLogger, pReq->fFlags, pReq->uGroupCrc32, pReq->cGroups, pReq->afGroups);
2871	}
2872	}
2873
2874	return rc;
2875	}
2876
2877
2878	/**
2879	* VMMR0_DO_VMMR0_LOG_FLUSHER: Get the next log flushing job.
2880	*
2881	* The job info is copied into VMM::LogFlusherItem.
2882	*
2883	* @returns VBox status code.
2884	* @retval VERR_OBJECT_DESTROYED if we're shutting down.
2885	* @retval VERR_NOT_OWNER if the calling thread is not the flusher thread.
2886	* @param pGVM The global (ring-0) VM structure.
2887	* @thread The log flusher thread (first caller automatically becomes the log
2888	* flusher).
2889	*/
2890	static int vmmR0LogFlusher(PGVM pGVM)
2891	{
2892	/*
2893	* Check that this really is the flusher thread.
2894	*/
2895	RTNATIVETHREAD const hNativeSelf = RTThreadNativeSelf();
2896	AssertReturn(hNativeSelf != NIL_RTNATIVETHREAD, VERR_INTERNAL_ERROR_3);
2897	if (RT_LIKELY(pGVM->vmmr0.s.LogFlusher.hThread == hNativeSelf))
2898	{ /* likely */ }
2899	else
2900	{
2901	/* The first caller becomes the flusher thread. */
2902	bool fOk;
2903	ASMAtomicCmpXchgHandle(&pGVM->vmmr0.s.LogFlusher.hThread, hNativeSelf, NIL_RTNATIVETHREAD, fOk);
2904	if (!fOk)
2905	return VERR_NOT_OWNER;
2906	pGVM->vmmr0.s.LogFlusher.fThreadRunning = true;
2907	}
2908
2909	/*
2910	* Acknowledge flush, waking up waiting EMT.
2911	*/
2912	RTSpinlockAcquire(pGVM->vmmr0.s.LogFlusher.hSpinlock);
2913
2914	uint32_t idxTail = pGVM->vmmr0.s.LogFlusher.idxRingTail % RT_ELEMENTS(pGVM->vmmr0.s.LogFlusher.aRing);
2915	uint32_t idxHead = pGVM->vmmr0.s.LogFlusher.idxRingHead % RT_ELEMENTS(pGVM->vmmr0.s.LogFlusher.aRing);
2916	if ( idxTail != idxHead
2917	&& pGVM->vmmr0.s.LogFlusher.aRing[idxHead].s.fProcessing)
2918	{
2919	/* Pop the head off the ring buffer. */
2920	uint32_t const idCpu = pGVM->vmmr0.s.LogFlusher.aRing[idxHead].s.idCpu;
2921	uint32_t const idxLogger = pGVM->vmmr0.s.LogFlusher.aRing[idxHead].s.idxLogger;
2922	uint32_t const idxBuffer = pGVM->vmmr0.s.LogFlusher.aRing[idxHead].s.idxBuffer;
2923
2924	pGVM->vmmr0.s.LogFlusher.aRing[idxHead].u32 = UINT32_MAX >> 1; /* invalidate the entry */
2925	pGVM->vmmr0.s.LogFlusher.idxRingHead = (idxHead + 1) % RT_ELEMENTS(pGVM->vmmr0.s.LogFlusher.aRing);
2926
2927	/* Validate content. */
2928	if ( idCpu < pGVM->cCpus
2929	&& idxLogger < VMMLOGGER_IDX_MAX
2930	&& idxBuffer < VMMLOGGER_BUFFER_COUNT)
2931	{
2932	PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
2933	PVMMR0PERVCPULOGGER pR0Log = &pGVCpu->vmmr0.s.u.aLoggers[idxLogger];
2934	PVMMR3CPULOGGER pShared = &pGVCpu->vmm.s.u.aLoggers[idxLogger];
2935
2936	/*
2937	* Accounting.
2938	*/
2939	uint32_t cFlushing = pR0Log->cFlushing - 1;
2940	if (RT_LIKELY(cFlushing < VMMLOGGER_BUFFER_COUNT))
2941	{ /likely/ }
2942	else
2943	cFlushing = 0;
2944	pR0Log->cFlushing = cFlushing;
2945	ASMAtomicWriteU32(&pShared->cFlushing, cFlushing);
2946
2947	/*
2948	* Wake up the EMT if it's waiting.
2949	*/
2950	if (!pR0Log->fEmtWaiting)
2951	RTSpinlockRelease(pGVM->vmmr0.s.LogFlusher.hSpinlock);
2952	else
2953	{
2954	pR0Log->fEmtWaiting = false;
2955	RTSpinlockRelease(pGVM->vmmr0.s.LogFlusher.hSpinlock);
2956
2957	int rc = RTSemEventSignal(pR0Log->hEventFlushWait);
2958	if (RT_FAILURE(rc))
2959	LogRelMax(64, ("vmmR0LogFlusher: RTSemEventSignal failed ACKing entry #%u (%u/%u/%u): %Rrc!\n",
2960	idxHead, idCpu, idxLogger, idxBuffer, rc));
2961	}
2962	}
2963	else
2964	{
2965	RTSpinlockRelease(pGVM->vmmr0.s.LogFlusher.hSpinlock);
2966	LogRelMax(64, ("vmmR0LogFlusher: Bad ACK entry #%u: %u/%u/%u!\n", idxHead, idCpu, idxLogger, idxBuffer));
2967	}
2968
2969	RTSpinlockAcquire(pGVM->vmmr0.s.LogFlusher.hSpinlock);
2970	}
2971
2972	/*
2973	* The wait loop.
2974	*/
2975	int rc;
2976	for (;;)
2977	{
2978	/*
2979	* Work pending?
2980	*/
2981	idxTail = pGVM->vmmr0.s.LogFlusher.idxRingTail % RT_ELEMENTS(pGVM->vmmr0.s.LogFlusher.aRing);
2982	idxHead = pGVM->vmmr0.s.LogFlusher.idxRingHead % RT_ELEMENTS(pGVM->vmmr0.s.LogFlusher.aRing);
2983	if (idxTail != idxHead)
2984	{
2985	pGVM->vmmr0.s.LogFlusher.aRing[idxHead].s.fProcessing = true;
2986	pGVM->vmm.s.LogFlusherItem.u32 = pGVM->vmmr0.s.LogFlusher.aRing[idxHead].u32;
2987
2988	RTSpinlockRelease(pGVM->vmmr0.s.LogFlusher.hSpinlock);
2989	return VINF_SUCCESS;
2990	}
2991
2992	/*
2993	* Nothing to do, so, check for termination and go to sleep.
2994	*/
2995	if (!pGVM->vmmr0.s.LogFlusher.fThreadShutdown)
2996	{ /* likely */ }
2997	else
2998	{
2999	rc = VERR_OBJECT_DESTROYED;
3000	break;
3001	}
3002
3003	pGVM->vmmr0.s.LogFlusher.fThreadWaiting = true;
3004	RTSpinlockRelease(pGVM->vmmr0.s.LogFlusher.hSpinlock);
3005
3006	rc = RTSemEventWaitNoResume(pGVM->vmmr0.s.LogFlusher.hEvent, RT_MS_5MIN);
3007
3008	RTSpinlockAcquire(pGVM->vmmr0.s.LogFlusher.hSpinlock);
3009	pGVM->vmmr0.s.LogFlusher.fThreadWaiting = false;
3010
3011	if (RT_SUCCESS(rc) \|\| rc == VERR_TIMEOUT)
3012	{ /* likely */ }
3013	else if (rc == VERR_INTERRUPTED)
3014	{
3015	RTSpinlockRelease(pGVM->vmmr0.s.LogFlusher.hSpinlock);
3016	return rc;
3017	}
3018	else if (rc == VERR_SEM_DESTROYED \|\| rc == VERR_INVALID_HANDLE)
3019	break;
3020	else
3021	{
3022	LogRel(("vmmR0LogFlusher: RTSemEventWaitNoResume returned unexpected status %Rrc\n", rc));
3023	break;
3024	}
3025	}
3026
3027	/*
3028	* Terminating - prevent further calls and indicate to the EMTs that we're no longer around.
3029	*/
3030	pGVM->vmmr0.s.LogFlusher.hThread = ~pGVM->vmmr0.s.LogFlusher.hThread; /* (should be reasonably safe) */
3031	pGVM->vmmr0.s.LogFlusher.fThreadRunning = false;
3032
3033	RTSpinlockRelease(pGVM->vmmr0.s.LogFlusher.hSpinlock);
3034	return rc;
3035	}
3036
3037
3038	/**
3039	* VMMR0_DO_VMMR0_LOG_WAIT_FLUSHED: Waits for the flusher thread to finish all
3040	* buffers for logger @a idxLogger.
3041	*
3042	* @returns VBox status code.
3043	* @param pGVM The global (ring-0) VM structure.
3044	* @param idCpu The ID of the calling EMT.
3045	* @param idxLogger Which logger to wait on.
3046	* @thread EMT(idCpu)
3047	*/
3048	static int vmmR0LogWaitFlushed(PGVM pGVM, VMCPUID idCpu, size_t idxLogger)
3049	{
3050	/*
3051	* Check sanity. First we require EMT to be calling us.
3052	*/
3053	AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_CPU_ID);
3054	PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
3055	AssertReturn(pGVCpu->hEMT == RTThreadNativeSelf(), VERR_INVALID_CPU_ID);
3056	AssertReturn(idxLogger < VMMLOGGER_IDX_MAX, VERR_OUT_OF_RANGE);
3057	PVMMR0PERVCPULOGGER const pR0Log = &pGVCpu->vmmr0.s.u.aLoggers[idxLogger];
3058
3059	/*
3060	* Do the waiting.
3061	*/
3062	int rc = VINF_SUCCESS;
3063	RTSpinlockAcquire(pGVM->vmmr0.s.LogFlusher.hSpinlock);
3064	uint32_t cFlushing = pR0Log->cFlushing;
3065	while (cFlushing > 0)
3066	{
3067	pR0Log->fEmtWaiting = true;
3068	RTSpinlockRelease(pGVM->vmmr0.s.LogFlusher.hSpinlock);
3069
3070	rc = RTSemEventWaitNoResume(pR0Log->hEventFlushWait, RT_MS_5MIN);
3071
3072	RTSpinlockAcquire(pGVM->vmmr0.s.LogFlusher.hSpinlock);
3073	pR0Log->fEmtWaiting = false;
3074	if (RT_SUCCESS(rc))
3075	{
3076	/* Read the new count, make sure it decreased before looping. That
3077	way we can guarentee that we will only wait more than 5 min * buffers. */
3078	uint32_t const cPrevFlushing = cFlushing;
3079	cFlushing = pR0Log->cFlushing;
3080	if (cFlushing < cPrevFlushing)
3081	continue;
3082	rc = VERR_INTERNAL_ERROR_3;
3083	}
3084	break;
3085	}
3086	RTSpinlockRelease(pGVM->vmmr0.s.LogFlusher.hSpinlock);
3087	return rc;
3088	}
3089
3090
3091	/**
3092	* Inner worker for vmmR0LoggerFlushCommon for flushing to ring-3.
3093	*/
3094	static bool vmmR0LoggerFlushInnerToRing3(PGVM pGVM, PGVMCPU pGVCpu, uint32_t idxLogger, size_t idxBuffer, uint32_t cbToFlush)
3095	{
3096	PVMMR0PERVCPULOGGER const pR0Log = &pGVCpu->vmmr0.s.u.aLoggers[idxLogger];
3097	PVMMR3CPULOGGER const pShared = &pGVCpu->vmm.s.u.aLoggers[idxLogger];
3098
3099	/*
3100	* Figure out what we need to do and whether we can.
3101	*/
3102	enum { kJustSignal, kPrepAndSignal, kPrepSignalAndWait } enmAction;
3103	#if VMMLOGGER_BUFFER_COUNT >= 2
3104	if (pR0Log->cFlushing < VMMLOGGER_BUFFER_COUNT - 1)
3105	{
3106	if (RTSemEventIsSignalSafe())
3107	enmAction = kJustSignal;
3108	else if (VMMRZCallRing3IsEnabled(pGVCpu))
3109	enmAction = kPrepAndSignal;
3110	else
3111	{
3112	/** @todo This is a bit simplistic. We could introduce a FF to signal the
3113	* thread or similar. */
3114	STAM_REL_COUNTER_INC(&pShared->StatCannotBlock);
3115	# if defined(RT_OS_LINUX)
3116	SUP_DPRINTF(("vmmR0LoggerFlush: Signalling not safe and EMT blocking disabled! (%u bytes)\n", cbToFlush));
3117	# endif
3118	pShared->cbDropped += cbToFlush;
3119	return true;
3120	}
3121	}
3122	else
3123	#endif
3124	if (VMMRZCallRing3IsEnabled(pGVCpu))
3125	enmAction = kPrepSignalAndWait;
3126	else
3127	{
3128	STAM_REL_COUNTER_INC(&pShared->StatCannotBlock);
3129	# if defined(RT_OS_LINUX)
3130	SUP_DPRINTF(("vmmR0LoggerFlush: EMT blocking disabled! (%u bytes)\n", cbToFlush));
3131	# endif
3132	pShared->cbDropped += cbToFlush;
3133	return true;
3134	}
3135
3136	/*
3137	* Prepare for blocking if necessary.
3138	*/
3139	VMMR0EMTBLOCKCTX Ctx;
3140	if (enmAction != kJustSignal)
3141	{
3142	int rc = VMMR0EmtPrepareToBlock(pGVCpu, VINF_SUCCESS, "vmmR0LoggerFlushInnerToRing3", pR0Log->hEventFlushWait, &Ctx);
3143	if (RT_SUCCESS(rc))
3144	{ /* likely */ }
3145	else
3146	{
3147	STAM_REL_COUNTER_INC(&pShared->StatCannotBlock);
3148	SUP_DPRINTF(("vmmR0LoggerFlush: VMMR0EmtPrepareToBlock failed! rc=%d\n", rc));
3149	return false;
3150	}
3151	}
3152
3153	/*
3154	* Queue the flush job.
3155	*/
3156	bool fFlushedBuffer;
3157	RTSpinlockAcquire(pGVM->vmmr0.s.LogFlusher.hSpinlock);
3158	if (pGVM->vmmr0.s.LogFlusher.fThreadRunning)
3159	{
3160	uint32_t const idxHead = pGVM->vmmr0.s.LogFlusher.idxRingHead % RT_ELEMENTS(pGVM->vmmr0.s.LogFlusher.aRing);
3161	uint32_t const idxTail = pGVM->vmmr0.s.LogFlusher.idxRingTail % RT_ELEMENTS(pGVM->vmmr0.s.LogFlusher.aRing);
3162	uint32_t const idxNewTail = (idxTail + 1) % RT_ELEMENTS(pGVM->vmmr0.s.LogFlusher.aRing);
3163	if (idxNewTail != idxHead)
3164	{
3165	/* Queue it. */
3166	pGVM->vmmr0.s.LogFlusher.aRing[idxTail].s.idCpu = pGVCpu->idCpu;
3167	pGVM->vmmr0.s.LogFlusher.aRing[idxTail].s.idxLogger = idxLogger;
3168	pGVM->vmmr0.s.LogFlusher.aRing[idxTail].s.idxBuffer = (uint32_t)idxBuffer;
3169	pGVM->vmmr0.s.LogFlusher.aRing[idxTail].s.fProcessing = 0;
3170	pGVM->vmmr0.s.LogFlusher.idxRingTail = idxNewTail;
3171
3172	/* Update the number of buffers currently being flushed. */
3173	uint32_t cFlushing = pR0Log->cFlushing;
3174	cFlushing = RT_MIN(cFlushing + 1, VMMLOGGER_BUFFER_COUNT);
3175	pShared->cFlushing = pR0Log->cFlushing = cFlushing;
3176
3177	/* We must wait if all buffers are currently being flushed. */
3178	bool const fEmtWaiting = cFlushing >= VMMLOGGER_BUFFER_COUNT && enmAction != kJustSignal /* paranoia */;
3179	pR0Log->fEmtWaiting = fEmtWaiting;
3180
3181	/* Stats. */
3182	STAM_REL_COUNTER_INC(&pShared->StatFlushes);
3183	STAM_REL_COUNTER_INC(&pGVM->vmm.s.StatLogFlusherFlushes);
3184
3185	/* Signal the worker thread. */
3186	if (pGVM->vmmr0.s.LogFlusher.fThreadWaiting)
3187	{
3188	RTSpinlockRelease(pGVM->vmmr0.s.LogFlusher.hSpinlock);
3189	RTSemEventSignal(pGVM->vmmr0.s.LogFlusher.hEvent);
3190	}
3191	else
3192	{
3193	STAM_REL_COUNTER_INC(&pGVM->vmm.s.StatLogFlusherNoWakeUp);
3194	RTSpinlockRelease(pGVM->vmmr0.s.LogFlusher.hSpinlock);
3195	}
3196
3197	/*
3198	* Wait for a buffer to finish flushing.
3199	*
3200	* Note! Lazy bird is ignoring the status code here. The result is
3201	* that we might end up with an extra even signalling and the
3202	* next time we need to wait we won't and end up with some log
3203	* corruption. However, it's too much hazzle right now for
3204	* a scenario which would most likely end the process rather
3205	* than causing log corruption.
3206	*/
3207	if (fEmtWaiting)
3208	{
3209	STAM_REL_PROFILE_START(&pShared->StatWait, a);
3210	VMMR0EmtWaitEventInner(pGVCpu, VMMR0EMTWAIT_F_TRY_SUPPRESS_INTERRUPTED,
3211	pR0Log->hEventFlushWait, RT_INDEFINITE_WAIT);
3212	STAM_REL_PROFILE_STOP(&pShared->StatWait, a);
3213	}
3214
3215	/*
3216	* We always switch buffer if we have more than one.
3217	*/
3218	#if VMMLOGGER_BUFFER_COUNT == 1
3219	fFlushedBuffer = true;
3220	#else
3221	AssertCompile(VMMLOGGER_BUFFER_COUNT >= 1);
3222	pShared->idxBuf = (idxBuffer + 1) % VMMLOGGER_BUFFER_COUNT;
3223	fFlushedBuffer = false;
3224	#endif
3225	}
3226	else
3227	{
3228	RTSpinlockRelease(pGVM->vmmr0.s.LogFlusher.hSpinlock);
3229	SUP_DPRINTF(("vmmR0LoggerFlush: ring buffer is full!\n"));
3230	fFlushedBuffer = true;
3231	}
3232	}
3233	else
3234	{
3235	RTSpinlockRelease(pGVM->vmmr0.s.LogFlusher.hSpinlock);
3236	SUP_DPRINTF(("vmmR0LoggerFlush: flusher not active - dropping %u bytes\n", cbToFlush));
3237	fFlushedBuffer = true;
3238	}
3239
3240	/*
3241	* Restore the HM context.
3242	*/
3243	if (enmAction != kJustSignal)
3244	VMMR0EmtResumeAfterBlocking(pGVCpu, &Ctx);
3245
3246	return fFlushedBuffer;
3247	}
3248
3249
3250	/**
3251	* Inner worker for vmmR0LoggerFlushCommon when only flushing to the parent
3252	* VMM's logs.
3253	*/
3254	static bool vmmR0LoggerFlushInnerToParent(PVMMR0PERVCPULOGGER pR0Log, PRTLOGBUFFERDESC pBufDesc)
3255	{
3256	#ifdef RT_ARCH_AMD64
3257	uint32_t const cbToFlush = pBufDesc->offBuf;
3258	if (pR0Log->fFlushToParentVmmDbg)
3259	RTLogWriteVmm(pBufDesc->pchBuf, cbToFlush, false /fRelease/);
3260	if (pR0Log->fFlushToParentVmmRel)
3261	RTLogWriteVmm(pBufDesc->pchBuf, cbToFlush, true /fRelease/);
3262	#else
3263	RT_NOREF(pR0Log, pBufDesc);
3264	#endif
3265	return true;
3266	}
3267
3268
3269
3270	/**
3271	* Common worker for vmmR0LogFlush and vmmR0LogRelFlush.
3272	*/
3273	static bool vmmR0LoggerFlushCommon(PRTLOGGER pLogger, PRTLOGBUFFERDESC pBufDesc, uint32_t idxLogger)
3274	{
3275	/*
3276	* Convert the pLogger into a GVMCPU handle and 'call' back to Ring-3.
3277	* (This is a bit paranoid code.)
3278	*/
3279	if (RT_VALID_PTR(pLogger))
3280	{
3281	if ( pLogger->u32Magic == RTLOGGER_MAGIC
3282	&& (pLogger->u32UserValue1 & VMMR0_LOGGER_FLAGS_MAGIC_MASK) == VMMR0_LOGGER_FLAGS_MAGIC_VALUE
3283	&& pLogger->u64UserValue2 == pLogger->u64UserValue3)
3284	{
3285	PGVMCPU const pGVCpu = (PGVMCPU)(uintptr_t)pLogger->u64UserValue2;
3286	if ( RT_VALID_PTR(pGVCpu)
3287	&& ((uintptr_t)pGVCpu & HOST_PAGE_OFFSET_MASK) == 0)
3288	{
3289	RTNATIVETHREAD const hNativeSelf = RTThreadNativeSelf();
3290	PGVM const pGVM = pGVCpu->pGVM;
3291	if ( hNativeSelf == pGVCpu->hEMT
3292	&& RT_VALID_PTR(pGVM))
3293	{
3294	PVMMR0PERVCPULOGGER const pR0Log = &pGVCpu->vmmr0.s.u.aLoggers[idxLogger];
3295	size_t const idxBuffer = pBufDesc - &pR0Log->aBufDescs[0];
3296	if (idxBuffer < VMMLOGGER_BUFFER_COUNT)
3297	{
3298	/*
3299	* Make sure we don't recurse forever here should something in the
3300	* following code trigger logging or an assertion. Do the rest in
3301	* an inner work to avoid hitting the right margin too hard.
3302	*/
3303	if (!pR0Log->fFlushing)
3304	{
3305	pR0Log->fFlushing = true;
3306	bool fFlushed;
3307	if ( !pR0Log->fFlushToParentVmmDbg
3308	&& !pR0Log->fFlushToParentVmmRel)
3309	fFlushed = vmmR0LoggerFlushInnerToRing3(pGVM, pGVCpu, idxLogger, idxBuffer, pBufDesc->offBuf);
3310	else
3311	fFlushed = vmmR0LoggerFlushInnerToParent(pR0Log, pBufDesc);
3312	pR0Log->fFlushing = false;
3313	return fFlushed;
3314	}
3315
3316	SUP_DPRINTF(("vmmR0LoggerFlush: Recursive flushing!\n"));
3317	}
3318	else
3319	SUP_DPRINTF(("vmmR0LoggerFlush: pLogger=%p pGVCpu=%p: idxBuffer=%#zx\n", pLogger, pGVCpu, idxBuffer));
3320	}
3321	else
3322	SUP_DPRINTF(("vmmR0LoggerFlush: pLogger=%p pGVCpu=%p hEMT=%p hNativeSelf=%p!\n",
3323	pLogger, pGVCpu, pGVCpu->hEMT, hNativeSelf));
3324	}
3325	else
3326	SUP_DPRINTF(("vmmR0LoggerFlush: pLogger=%p pGVCpu=%p!\n", pLogger, pGVCpu));
3327	}
3328	else
3329	SUP_DPRINTF(("vmmR0LoggerFlush: pLogger=%p u32Magic=%#x u32UserValue1=%#x u64UserValue2=%#RX64 u64UserValue3=%#RX64!\n",
3330	pLogger, pLogger->u32Magic, pLogger->u32UserValue1, pLogger->u64UserValue2, pLogger->u64UserValue3));
3331	}
3332	else
3333	SUP_DPRINTF(("vmmR0LoggerFlush: pLogger=%p!\n", pLogger));
3334	return true;
3335	}
3336
3337
3338	/**
3339	* @callback_method_impl{FNRTLOGFLUSH, Release logger buffer flush callback.}
3340	*/
3341	static DECLCALLBACK(bool) vmmR0LogRelFlush(PRTLOGGER pLogger, PRTLOGBUFFERDESC pBufDesc)
3342	{
3343	return vmmR0LoggerFlushCommon(pLogger, pBufDesc, VMMLOGGER_IDX_RELEASE);
3344	}
3345
3346
3347	/**
3348	* @callback_method_impl{FNRTLOGFLUSH, Logger (debug) buffer flush callback.}
3349	*/
3350	static DECLCALLBACK(bool) vmmR0LogFlush(PRTLOGGER pLogger, PRTLOGBUFFERDESC pBufDesc)
3351	{
3352	#ifdef LOG_ENABLED
3353	return vmmR0LoggerFlushCommon(pLogger, pBufDesc, VMMLOGGER_IDX_REGULAR);
3354	#else
3355	RT_NOREF(pLogger, pBufDesc);
3356	return true;
3357	#endif
3358	}
3359
3360
3361	/*
3362	* Override RTLogDefaultInstanceEx so we can do logging from EMTs in ring-0.
3363	*/
3364	DECLEXPORT(PRTLOGGER) RTLogDefaultInstanceEx(uint32_t fFlagsAndGroup)
3365	{
3366	#ifdef LOG_ENABLED
3367	PGVMCPU pGVCpu = GVMMR0GetGVCpuByEMT(NIL_RTNATIVETHREAD);
3368	if (pGVCpu)
3369	{
3370	PRTLOGGER pLogger = pGVCpu->vmmr0.s.u.s.Logger.pLogger;
3371	if (RT_VALID_PTR(pLogger))
3372	{
3373	if ( pLogger->u64UserValue2 == (uintptr_t)pGVCpu
3374	&& pLogger->u64UserValue3 == (uintptr_t)pGVCpu)
3375	{
3376	if (!pGVCpu->vmmr0.s.u.s.Logger.fFlushing)
3377	return RTLogCheckGroupFlags(pLogger, fFlagsAndGroup);
3378
3379	/*
3380	* When we're flushing we _must_ return NULL here to suppress any
3381	* attempts at using the logger while in vmmR0LoggerFlushCommon.
3382	* The VMMR0EmtPrepareToBlock code may trigger logging in HM,
3383	* which will reset the buffer content before we even get to queue
3384	* the flush request. (Only an issue when VBOX_WITH_R0_LOGGING
3385	* is enabled.)
3386	*/
3387	return NULL;
3388	}
3389	}
3390	}
3391	#endif
3392	return SUPR0DefaultLogInstanceEx(fFlagsAndGroup);
3393	}
3394
3395
3396	/*
3397	* Override RTLogRelGetDefaultInstanceEx so we can do LogRel to VBox.log from EMTs in ring-0.
3398	*/
3399	DECLEXPORT(PRTLOGGER) RTLogRelGetDefaultInstanceEx(uint32_t fFlagsAndGroup)
3400	{
3401	PGVMCPU pGVCpu = GVMMR0GetGVCpuByEMT(NIL_RTNATIVETHREAD);
3402	if (pGVCpu)
3403	{
3404	PRTLOGGER pLogger = pGVCpu->vmmr0.s.u.s.RelLogger.pLogger;
3405	if (RT_VALID_PTR(pLogger))
3406	{
3407	if ( pLogger->u64UserValue2 == (uintptr_t)pGVCpu
3408	&& pLogger->u64UserValue3 == (uintptr_t)pGVCpu)
3409	{
3410	if (!pGVCpu->vmmr0.s.u.s.RelLogger.fFlushing)
3411	return RTLogCheckGroupFlags(pLogger, fFlagsAndGroup);
3412
3413	/* ASSUMES no LogRels hidden within the VMMR0EmtPrepareToBlock code
3414	path, so we don't return NULL here like for the debug logger... */
3415	}
3416	}
3417	}
3418	return SUPR0GetDefaultLogRelInstanceEx(fFlagsAndGroup);
3419	}
3420
3421
3422	/**
3423	* Helper for vmmR0InitLoggerSet
3424	*/
3425	static int vmmR0InitLoggerOne(PGVMCPU pGVCpu, bool fRelease, PVMMR0PERVCPULOGGER pR0Log, PVMMR3CPULOGGER pShared,
3426	uint32_t cbBuf, char *pchBuf, RTR3PTR pchBufR3)
3427	{
3428	/*
3429	* Create and configure the logger.
3430	*/
3431	for (size_t i = 0; i < VMMLOGGER_BUFFER_COUNT; i++)
3432	{
3433	pR0Log->aBufDescs[i].u32Magic = RTLOGBUFFERDESC_MAGIC;
3434	pR0Log->aBufDescs[i].uReserved = 0;
3435	pR0Log->aBufDescs[i].cbBuf = cbBuf;
3436	pR0Log->aBufDescs[i].offBuf = 0;
3437	pR0Log->aBufDescs[i].pchBuf = pchBuf + i * cbBuf;
3438	pR0Log->aBufDescs[i].pAux = &pShared->aBufs[i].AuxDesc;
3439
3440	pShared->aBufs[i].AuxDesc.fFlushedIndicator = false;
3441	pShared->aBufs[i].AuxDesc.afPadding[0] = 0;
3442	pShared->aBufs[i].AuxDesc.afPadding[1] = 0;
3443	pShared->aBufs[i].AuxDesc.afPadding[2] = 0;
3444	pShared->aBufs[i].AuxDesc.offBuf = 0;
3445	pShared->aBufs[i].pchBufR3 = pchBufR3 + i * cbBuf;
3446	}
3447	pShared->cbBuf = cbBuf;
3448
3449	static const char * const s_apszGroups[] = VBOX_LOGGROUP_NAMES;
3450	int rc = RTLogCreateEx(&pR0Log->pLogger, fRelease ? "VBOX_RELEASE_LOG" : "VBOX_LOG", RTLOG_F_NO_LOCKING \| RTLOGFLAGS_BUFFERED,
3451	"all", RT_ELEMENTS(s_apszGroups), s_apszGroups, UINT32_MAX,
3452	VMMLOGGER_BUFFER_COUNT, pR0Log->aBufDescs, RTLOGDEST_DUMMY,
3453	NULL /pfnPhase/, 0 /cHistory/, 0 /cbHistoryFileMax/, 0 /cSecsHistoryTimeSlot/,
3454	NULL /pOutputIf/, NULL /pvOutputIfUser/,
3455	NULL /pErrInfo/, NULL /pszFilenameFmt/);
3456	if (RT_SUCCESS(rc))
3457	{
3458	PRTLOGGER pLogger = pR0Log->pLogger;
3459	pLogger->u32UserValue1 = VMMR0_LOGGER_FLAGS_MAGIC_VALUE;
3460	pLogger->u64UserValue2 = (uintptr_t)pGVCpu;
3461	pLogger->u64UserValue3 = (uintptr_t)pGVCpu;
3462
3463	rc = RTLogSetFlushCallback(pLogger, fRelease ? vmmR0LogRelFlush : vmmR0LogFlush);
3464	if (RT_SUCCESS(rc))
3465	{
3466	RTLogSetR0ThreadNameF(pLogger, "EMT-%u-R0", pGVCpu->idCpu);
3467
3468	/*
3469	* Create the event sem the EMT waits on while flushing is happening.
3470	*/
3471	rc = RTSemEventCreate(&pR0Log->hEventFlushWait);
3472	if (RT_SUCCESS(rc))
3473	return VINF_SUCCESS;
3474	pR0Log->hEventFlushWait = NIL_RTSEMEVENT;
3475	}
3476	RTLogDestroy(pLogger);
3477	}
3478	pR0Log->pLogger = NULL;
3479	return rc;
3480	}
3481
3482
3483	/**
3484	* Worker for VMMR0CleanupVM and vmmR0InitLoggerSet that destroys one logger.
3485	*/
3486	static void vmmR0TermLoggerOne(PVMMR0PERVCPULOGGER pR0Log, PVMMR3CPULOGGER pShared)
3487	{
3488	RTLogDestroy(pR0Log->pLogger);
3489	pR0Log->pLogger = NULL;
3490
3491	for (size_t i = 0; i < VMMLOGGER_BUFFER_COUNT; i++)
3492	pShared->aBufs[i].pchBufR3 = NIL_RTR3PTR;
3493
3494	RTSemEventDestroy(pR0Log->hEventFlushWait);
3495	pR0Log->hEventFlushWait = NIL_RTSEMEVENT;
3496	}
3497
3498
3499	/**
3500	* Initializes one type of loggers for each EMT.
3501	*/
3502	static int vmmR0InitLoggerSet(PGVM pGVM, uint8_t idxLogger, uint32_t cbBuf, PRTR0MEMOBJ phMemObj, PRTR0MEMOBJ phMapObj)
3503	{
3504	/* Allocate buffers first. */
3505	int rc = RTR0MemObjAllocPage(phMemObj, cbBuf * pGVM->cCpus * VMMLOGGER_BUFFER_COUNT, false /fExecutable/);
3506	if (RT_SUCCESS(rc))
3507	{
3508	rc = RTR0MemObjMapUser(phMapObj, phMemObj, (RTR3PTR)-1, 0 /uAlignment*/, RTMEM_PROT_READ, NIL_RTR0PROCESS);
3509	if (RT_SUCCESS(rc))
3510	{
3511	char * const pchBuf = (char )RTR0MemObjAddress(phMemObj);
3512	AssertPtrReturn(pchBuf, VERR_INTERNAL_ERROR_2);
3513
3514	RTR3PTR const pchBufR3 = RTR0MemObjAddressR3(*phMapObj);
3515	AssertReturn(pchBufR3 != NIL_RTR3PTR, VERR_INTERNAL_ERROR_3);
3516
3517	/* Initialize the per-CPU loggers. */
3518	for (uint32_t i = 0; i < pGVM->cCpus; i++)
3519	{
3520	PGVMCPU pGVCpu = &pGVM->aCpus[i];
3521	PVMMR0PERVCPULOGGER pR0Log = &pGVCpu->vmmr0.s.u.aLoggers[idxLogger];
3522	PVMMR3CPULOGGER pShared = &pGVCpu->vmm.s.u.aLoggers[idxLogger];
3523	rc = vmmR0InitLoggerOne(pGVCpu, idxLogger == VMMLOGGER_IDX_RELEASE, pR0Log, pShared, cbBuf,
3524	pchBuf + i * cbBuf * VMMLOGGER_BUFFER_COUNT,
3525	pchBufR3 + i * cbBuf * VMMLOGGER_BUFFER_COUNT);
3526	if (RT_FAILURE(rc))
3527	{
3528	vmmR0TermLoggerOne(pR0Log, pShared);
3529	while (i-- > 0)
3530	{
3531	pGVCpu = &pGVM->aCpus[i];
3532	vmmR0TermLoggerOne(&pGVCpu->vmmr0.s.u.aLoggers[idxLogger], &pGVCpu->vmm.s.u.aLoggers[idxLogger]);
3533	}
3534	break;
3535	}
3536	}
3537	if (RT_SUCCESS(rc))
3538	return VINF_SUCCESS;
3539
3540	/* Bail out. */
3541	RTR0MemObjFree(phMapObj, false /fFreeMappings*/);
3542	*phMapObj = NIL_RTR0MEMOBJ;
3543	}
3544	RTR0MemObjFree(phMemObj, true /fFreeMappings*/);
3545	*phMemObj = NIL_RTR0MEMOBJ;
3546	}
3547	return rc;
3548	}
3549
3550
3551	/**
3552	* Worker for VMMR0InitPerVMData that initializes all the logging related stuff.
3553	*
3554	* @returns VBox status code.
3555	* @param pGVM The global (ring-0) VM structure.
3556	*/
3557	static int vmmR0InitLoggers(PGVM pGVM)
3558	{
3559	/*
3560	* Invalidate the ring buffer (not really necessary).
3561	*/
3562	for (size_t idx = 0; idx < RT_ELEMENTS(pGVM->vmmr0.s.LogFlusher.aRing); idx++)
3563	pGVM->vmmr0.s.LogFlusher.aRing[idx].u32 = UINT32_MAX >> 1; /* (all bits except fProcessing set) */
3564
3565	/*
3566	* Create the spinlock and flusher event semaphore.
3567	*/
3568	int rc = RTSpinlockCreate(&pGVM->vmmr0.s.LogFlusher.hSpinlock, RTSPINLOCK_FLAGS_INTERRUPT_SAFE, "VM-Log-Flusher");
3569	if (RT_SUCCESS(rc))
3570	{
3571	rc = RTSemEventCreate(&pGVM->vmmr0.s.LogFlusher.hEvent);
3572	if (RT_SUCCESS(rc))
3573	{
3574	/*
3575	* Create the ring-0 release loggers.
3576	*/
3577	rc = vmmR0InitLoggerSet(pGVM, VMMLOGGER_IDX_RELEASE, _4K,
3578	&pGVM->vmmr0.s.hMemObjReleaseLogger, &pGVM->vmmr0.s.hMapObjReleaseLogger);
3579	#ifdef LOG_ENABLED
3580	if (RT_SUCCESS(rc))
3581	{
3582	/*
3583	* Create debug loggers.
3584	*/
3585	rc = vmmR0InitLoggerSet(pGVM, VMMLOGGER_IDX_REGULAR, _64K,
3586	&pGVM->vmmr0.s.hMemObjLogger, &pGVM->vmmr0.s.hMapObjLogger);
3587	}
3588	#endif
3589	}
3590	}
3591	return rc;
3592	}
3593
3594
3595	/**
3596	* Worker for VMMR0InitPerVMData that initializes all the logging related stuff.
3597	*
3598	* @param pGVM The global (ring-0) VM structure.
3599	*/
3600	static void vmmR0CleanupLoggers(PGVM pGVM)
3601	{
3602	for (VMCPUID idCpu = 0; idCpu < pGVM->cCpus; idCpu++)
3603	{
3604	PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
3605	for (size_t iLogger = 0; iLogger < RT_ELEMENTS(pGVCpu->vmmr0.s.u.aLoggers); iLogger++)
3606	vmmR0TermLoggerOne(&pGVCpu->vmmr0.s.u.aLoggers[iLogger], &pGVCpu->vmm.s.u.aLoggers[iLogger]);
3607	}
3608
3609	/*
3610	* Free logger buffer memory.
3611	*/
3612	RTR0MemObjFree(pGVM->vmmr0.s.hMapObjReleaseLogger, false /fFreeMappings/);
3613	pGVM->vmmr0.s.hMapObjReleaseLogger = NIL_RTR0MEMOBJ;
3614	RTR0MemObjFree(pGVM->vmmr0.s.hMemObjReleaseLogger, true /fFreeMappings/);
3615	pGVM->vmmr0.s.hMemObjReleaseLogger = NIL_RTR0MEMOBJ;
3616
3617	RTR0MemObjFree(pGVM->vmmr0.s.hMapObjLogger, false /fFreeMappings/);
3618	pGVM->vmmr0.s.hMapObjLogger = NIL_RTR0MEMOBJ;
3619	RTR0MemObjFree(pGVM->vmmr0.s.hMemObjLogger, true /fFreeMappings/);
3620	pGVM->vmmr0.s.hMemObjLogger = NIL_RTR0MEMOBJ;
3621
3622	/*
3623	* Free log flusher related stuff.
3624	*/
3625	RTSpinlockDestroy(pGVM->vmmr0.s.LogFlusher.hSpinlock);
3626	pGVM->vmmr0.s.LogFlusher.hSpinlock = NIL_RTSPINLOCK;
3627	RTSemEventDestroy(pGVM->vmmr0.s.LogFlusher.hEvent);
3628	pGVM->vmmr0.s.LogFlusher.hEvent = NIL_RTSEMEVENT;
3629	}
3630
3631
3632	/*********************************************************************************************************************************
3633	* Assertions *
3634	*********************************************************************************************************************************/
3635
3636	/**
3637	* Installs a notification callback for ring-0 assertions.
3638	*
3639	* @param pVCpu The cross context virtual CPU structure.
3640	* @param pfnCallback Pointer to the callback.
3641	* @param pvUser The user argument.
3642	*
3643	* @return VBox status code.
3644	*/
3645	VMMR0_INT_DECL(int) VMMR0AssertionSetNotification(PVMCPUCC pVCpu, PFNVMMR0ASSERTIONNOTIFICATION pfnCallback, RTR0PTR pvUser)
3646	{
3647	AssertPtrReturn(pVCpu, VERR_INVALID_POINTER);
3648	AssertPtrReturn(pfnCallback, VERR_INVALID_POINTER);
3649
3650	if (!pVCpu->vmmr0.s.pfnAssertCallback)
3651	{
3652	pVCpu->vmmr0.s.pfnAssertCallback = pfnCallback;
3653	pVCpu->vmmr0.s.pvAssertCallbackUser = pvUser;
3654	return VINF_SUCCESS;
3655	}
3656	return VERR_ALREADY_EXISTS;
3657	}
3658
3659
3660	/**
3661	* Removes the ring-0 callback.
3662	*
3663	* @param pVCpu The cross context virtual CPU structure.
3664	*/
3665	VMMR0_INT_DECL(void) VMMR0AssertionRemoveNotification(PVMCPUCC pVCpu)
3666	{
3667	pVCpu->vmmr0.s.pfnAssertCallback = NULL;
3668	pVCpu->vmmr0.s.pvAssertCallbackUser = NULL;
3669	}
3670
3671
3672	/**
3673	* Checks whether there is a ring-0 callback notification active.
3674	*
3675	* @param pVCpu The cross context virtual CPU structure.
3676	* @returns true if there the notification is active, false otherwise.
3677	*/
3678	VMMR0_INT_DECL(bool) VMMR0AssertionIsNotificationSet(PVMCPUCC pVCpu)
3679	{
3680	return pVCpu->vmmr0.s.pfnAssertCallback != NULL;
3681	}
3682
3683
3684	/*
3685	* Jump back to ring-3 if we're the EMT and the longjmp is armed.
3686	*
3687	* @returns true if the breakpoint should be hit, false if it should be ignored.
3688	*/
3689	DECLEXPORT(bool) RTCALL RTAssertShouldPanic(void)
3690	{
3691	#ifdef RT_ARCH_ARM64 /** @todo port vmmR0CallRing3SetJmpEx/vmmR0CallRing3LongJmp to ARM64 */
3692	return true;
3693	#else
3694	PVMCC pVM = GVMMR0GetVMByEMT(NIL_RTNATIVETHREAD);
3695	if (pVM)
3696	{
3697	PVMCPUCC pVCpu = VMMGetCpu(pVM);
3698
3699	if (pVCpu)
3700	{
3701	# ifdef RT_ARCH_X86
3702	if (pVCpu->vmmr0.s.AssertJmpBuf.eip)
3703	# else
3704	if (pVCpu->vmmr0.s.AssertJmpBuf.rip)
3705	# endif
3706	{
3707	if (pVCpu->vmmr0.s.pfnAssertCallback)
3708	pVCpu->vmmr0.s.pfnAssertCallback(pVCpu, pVCpu->vmmr0.s.pvAssertCallbackUser);
3709	int rc = vmmR0CallRing3LongJmp(&pVCpu->vmmr0.s.AssertJmpBuf, VERR_VMM_RING0_ASSERTION);
3710	return RT_FAILURE_NP(rc);
3711	}
3712	}
3713	}
3714	# ifdef RT_OS_LINUX
3715	return true;
3716	# else
3717	return false;
3718	# endif
3719	#endif
3720	}
3721
3722
3723	/*
3724	* Override this so we can push it up to ring-3.
3725	*/
3726	DECLEXPORT(void) RTCALL RTAssertMsg1Weak(const char pszExpr, unsigned uLine, const char pszFile, const char *pszFunction)
3727	{
3728	/*
3729	* To host kernel log/whatever.
3730	*/
3731	SUPR0Printf("!!R0-Assertion Failed!!\n"
3732	"Expression: %s\n"
3733	"Location : %s(%d) %s\n",
3734	pszExpr, pszFile, uLine, pszFunction);
3735
3736	/*
3737	* To the log.
3738	*/
3739	LogAlways(("\n!!R0-Assertion Failed!!\n"
3740	"Expression: %s\n"
3741	"Location : %s(%d) %s\n",
3742	pszExpr, pszFile, uLine, pszFunction));
3743
3744	/*
3745	* To the global VMM buffer.
3746	*/
3747	PVMCC pVM = GVMMR0GetVMByEMT(NIL_RTNATIVETHREAD);
3748	if (pVM)
3749	RTStrPrintf(pVM->vmm.s.szRing0AssertMsg1, sizeof(pVM->vmm.s.szRing0AssertMsg1),
3750	"\n!!R0-Assertion Failed!!\n"
3751	"Expression: %.*s\n"
3752	"Location : %s(%d) %s\n",
3753	sizeof(pVM->vmm.s.szRing0AssertMsg1) / 4 * 3, pszExpr,
3754	pszFile, uLine, pszFunction);
3755
3756	/*
3757	* Continue the normal way.
3758	*/
3759	RTAssertMsg1(pszExpr, uLine, pszFile, pszFunction);
3760	}
3761
3762
3763	/**
3764	* Callback for RTLogFormatV which writes to the ring-3 log port.
3765	* See PFNLOGOUTPUT() for details.
3766	*/
3767	static DECLCALLBACK(size_t) rtLogOutput(void pv, const char pachChars, size_t cbChars)
3768	{
3769	for (size_t i = 0; i < cbChars; i++)
3770	{
3771	LogAlways(("%c", pachChars[i])); NOREF(pachChars);
3772	}
3773
3774	NOREF(pv);
3775	return cbChars;
3776	}
3777
3778
3779	/*
3780	* Override this so we can push it up to ring-3.
3781	*/
3782	DECLEXPORT(void) RTCALL RTAssertMsg2WeakV(const char *pszFormat, va_list va)
3783	{
3784	va_list vaCopy;
3785
3786	/*
3787	* Push the message to the loggers.
3788	*/
3789	PRTLOGGER pLog = RTLogRelGetDefaultInstance();
3790	if (pLog)
3791	{
3792	va_copy(vaCopy, va);
3793	RTLogFormatV(rtLogOutput, pLog, pszFormat, vaCopy);
3794	va_end(vaCopy);
3795	}
3796	pLog = RTLogGetDefaultInstance(); /* Don't initialize it here... */
3797	if (pLog)
3798	{
3799	va_copy(vaCopy, va);
3800	RTLogFormatV(rtLogOutput, pLog, pszFormat, vaCopy);
3801	va_end(vaCopy);
3802	}
3803
3804	/*
3805	* Push it to the global VMM buffer.
3806	*/
3807	PVMCC pVM = GVMMR0GetVMByEMT(NIL_RTNATIVETHREAD);
3808	if (pVM)
3809	{
3810	va_copy(vaCopy, va);
3811	RTStrPrintfV(pVM->vmm.s.szRing0AssertMsg2, sizeof(pVM->vmm.s.szRing0AssertMsg2), pszFormat, vaCopy);
3812	va_end(vaCopy);
3813	}
3814
3815	/*
3816	* Continue the normal way.
3817	*/
3818	RTAssertMsg2V(pszFormat, va);
3819	}
3820

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/VMMR0.cpp

Download in other formats: