SUPDrvGip.cpp@ 100108

Last change on this file since 100108 was 98103, checked in by vboxsync, 23 months ago
Copyright year updates by scm.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 186.7 KB

Line
1	/* $Id: SUPDrvGip.cpp 98103 2023-01-17 14:15:46Z vboxsync $ */
2	/** @file
3	* VBoxDrv - The VirtualBox Support Driver - Common code for GIP.
4	*/
5
6	/*
7	* Copyright (C) 2006-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* The contents of this file may alternatively be used under the terms
26	* of the Common Development and Distribution License Version 1.0
27	* (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
28	* in the VirtualBox distribution, in which case the provisions of the
29	* CDDL are applicable instead of those of the GPL.
30	*
31	* You may elect to license modified versions of this file under the
32	* terms and conditions of either the GPL or the CDDL or both.
33	*
34	* SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
35	*/
36
37
38	/*********************************************************************************************************************************
39	* Header Files *
40	*********************************************************************************************************************************/
41	#define LOG_GROUP LOG_GROUP_SUP_DRV
42	#define SUPDRV_AGNOSTIC
43	#include "SUPDrvInternal.h"
44	#ifndef PAGE_SHIFT
45	# include <iprt/param.h>
46	#endif
47	#include <iprt/asm.h>
48	#include <iprt/asm-amd64-x86.h>
49	#include <iprt/asm-math.h>
50	#include <iprt/cpuset.h>
51	#include <iprt/handletable.h>
52	#include <iprt/mem.h>
53	#include <iprt/mp.h>
54	#include <iprt/power.h>
55	#include <iprt/process.h>
56	#include <iprt/semaphore.h>
57	#include <iprt/spinlock.h>
58	#include <iprt/thread.h>
59	#include <iprt/uuid.h>
60	#include <iprt/net.h>
61	#include <iprt/crc.h>
62	#include <iprt/string.h>
63	#include <iprt/timer.h>
64	#if defined(RT_OS_DARWIN) \|\| defined(RT_OS_SOLARIS) \|\| defined(RT_OS_FREEBSD)
65	# include <iprt/rand.h>
66	# include <iprt/path.h>
67	#endif
68	#include <iprt/uint128.h>
69	#include <iprt/x86.h>
70
71	#include <VBox/param.h>
72	#include <VBox/log.h>
73	#include <VBox/err.h>
74
75	#if defined(RT_OS_SOLARIS) \|\| defined(RT_OS_DARWIN)
76	# include "dtrace/SUPDrv.h"
77	#else
78	/* ... */
79	#endif
80
81
82	/*********************************************************************************************************************************
83	* Defined Constants And Macros *
84	*********************************************************************************************************************************/
85	/** The frequency by which we recalculate the u32UpdateHz and
86	* u32UpdateIntervalNS GIP members. The value must be a power of 2.
87	*
88	* Warning: Bumping this too high might overflow u32UpdateIntervalNS.
89	*/
90	#define GIP_UPDATEHZ_RECALC_FREQ 0x800
91
92	/** A reserved TSC value used for synchronization as well as measurement of
93	* TSC deltas. */
94	#define GIP_TSC_DELTA_RSVD UINT64_MAX
95	/** The number of TSC delta measurement loops in total (includes primer and
96	* read-time loops). */
97	#define GIP_TSC_DELTA_LOOPS 96
98	/** The number of cache primer loops. */
99	#define GIP_TSC_DELTA_PRIMER_LOOPS 4
100	/** The number of loops until we keep computing the minumum read time. */
101	#define GIP_TSC_DELTA_READ_TIME_LOOPS 24
102
103	/** The TSC frequency refinement period in seconds.
104	* The timer fires after 200ms, then every second, this value just says when
105	* to stop it after that. */
106	#define GIP_TSC_REFINE_PERIOD_IN_SECS 12
107	/** The TSC-delta threshold for the SUPGIPUSETSCDELTA_PRACTICALLY_ZERO rating */
108	#define GIP_TSC_DELTA_THRESHOLD_PRACTICALLY_ZERO 32
109	/** The TSC-delta threshold for the SUPGIPUSETSCDELTA_ROUGHLY_ZERO rating */
110	#define GIP_TSC_DELTA_THRESHOLD_ROUGHLY_ZERO 448
111	/** The TSC delta value for the initial GIP master - 0 in regular builds.
112	* To test the delta code this can be set to a non-zero value. */
113	#if 0
114	# define GIP_TSC_DELTA_INITIAL_MASTER_VALUE INT64_C(170139095182512) /* 0x00009abd9854acb0 */
115	#else
116	# define GIP_TSC_DELTA_INITIAL_MASTER_VALUE INT64_C(0)
117	#endif
118
119	AssertCompile(GIP_TSC_DELTA_PRIMER_LOOPS < GIP_TSC_DELTA_READ_TIME_LOOPS);
120	AssertCompile(GIP_TSC_DELTA_PRIMER_LOOPS + GIP_TSC_DELTA_READ_TIME_LOOPS < GIP_TSC_DELTA_LOOPS);
121
122	/** @def VBOX_SVN_REV
123	* The makefile should define this if it can. */
124	#ifndef VBOX_SVN_REV
125	# define VBOX_SVN_REV 0
126	#endif
127
128	#if 0 /* Don't start the GIP timers. Useful when debugging the IPRT timer code. */
129	# define DO_NOT_START_GIP
130	#endif
131
132
133	/*********************************************************************************************************************************
134	* Internal Functions *
135	*********************************************************************************************************************************/
136	static DECLCALLBACK(void) supdrvGipSyncAndInvariantTimer(PRTTIMER pTimer, void *pvUser, uint64_t iTick);
137	static DECLCALLBACK(void) supdrvGipAsyncTimer(PRTTIMER pTimer, void *pvUser, uint64_t iTick);
138	static int supdrvGipSetFlags(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession, uint32_t fOrMask, uint32_t fAndMask);
139	static void supdrvGipInitCpu(PSUPGLOBALINFOPAGE pGip, PSUPGIPCPU pCpu, uint64_t u64NanoTS, uint64_t uCpuHz);
140	static void supdrvTscResetSamples(PSUPDRVDEVEXT pDevExt, bool fClearDeltas);
141	#ifdef SUPDRV_USE_TSC_DELTA_THREAD
142	static int supdrvTscDeltaThreadInit(PSUPDRVDEVEXT pDevExt);
143	static void supdrvTscDeltaTerm(PSUPDRVDEVEXT pDevExt);
144	static void supdrvTscDeltaThreadStartMeasurement(PSUPDRVDEVEXT pDevExt, bool fForceAll);
145	#else
146	static int supdrvTscMeasureInitialDeltas(PSUPDRVDEVEXT pDevExt);
147	static int supdrvTscMeasureDeltaOne(PSUPDRVDEVEXT pDevExt, uint32_t idxWorker);
148	#endif
149
150
151	/*********************************************************************************************************************************
152	* Global Variables *
153	*********************************************************************************************************************************/
154	DECLEXPORT(PSUPGLOBALINFOPAGE) g_pSUPGlobalInfoPage = NULL;
155	SUPR0_EXPORT_SYMBOL(g_pSUPGlobalInfoPage);
156
157
158
159	/*
160	*
161	* Misc Common GIP Code
162	* Misc Common GIP Code
163	* Misc Common GIP Code
164	*
165	*
166	*/
167
168
169	/**
170	* Finds the GIP CPU index corresponding to @a idCpu.
171	*
172	* @returns GIP CPU array index, UINT32_MAX if not found.
173	* @param pGip The GIP.
174	* @param idCpu The CPU ID.
175	*/
176	static uint32_t supdrvGipFindCpuIndexForCpuId(PSUPGLOBALINFOPAGE pGip, RTCPUID idCpu)
177	{
178	uint32_t i;
179	for (i = 0; i < pGip->cCpus; i++)
180	if (pGip->aCPUs[i].idCpu == idCpu)
181	return i;
182	return UINT32_MAX;
183	}
184
185
186	/**
187	* Gets the APIC ID using the best available method.
188	*
189	* @returns APIC ID.
190	* @param pGip The GIP, for SUPGIPGETCPU_XXX.
191	*/
192	DECLINLINE(uint32_t) supdrvGipGetApicId(PSUPGLOBALINFOPAGE pGip)
193	{
194	if (pGip->fGetGipCpu & SUPGIPGETCPU_APIC_ID_EXT_0B)
195	return ASMGetApicIdExt0B();
196	if (pGip->fGetGipCpu & SUPGIPGETCPU_APIC_ID_EXT_8000001E)
197	return ASMGetApicIdExt8000001E();
198	return ASMGetApicId();
199	}
200
201
202	/**
203	* Gets the APIC ID using the best available method, slow version.
204	*/
205	static uint32_t supdrvGipGetApicIdSlow(void)
206	{
207	uint32_t const idApic = ASMGetApicId();
208
209	/* The Intel CPU topology leaf: */
210	uint32_t uOther = ASMCpuId_EAX(0);
211	if (uOther >= UINT32_C(0xb) && RTX86IsValidStdRange(uOther))
212	{
213	uint32_t uEax = 0;
214	uint32_t uEbx = 0;
215	uint32_t uEcx = 0;
216	uint32_t uEdx = 0;
217	#if defined(RT_OS_LINUX) \|\| defined(RT_OS_FREEBSD)
218	ASMCpuId_Idx_ECX(0xb, 0, &uEax, &uEbx, &uEcx, &uEdx);
219	#else
220	ASMCpuIdExSlow(0xb, 0, 0, 0, &uEax, &uEbx, &uEcx, &uEdx);
221	#endif
222	if ((uEcx >> 8) != 0) /* level type != invalid */
223	{
224	if ((uEdx & 0xff) == idApic)
225	return uEdx;
226	AssertMsgFailed(("ASMGetApicIdExt0B=>%#x idApic=%#x\n", uEdx, idApic));
227	}
228	}
229
230	/* The AMD leaf: */
231	uOther = ASMCpuId_EAX(UINT32_C(0x80000000));
232	if (uOther >= UINT32_C(0x8000001e) && RTX86IsValidExtRange(uOther))
233	{
234	uOther = ASMGetApicIdExt8000001E();
235	if ((uOther & 0xff) == idApic)
236	return uOther;
237	AssertMsgFailed(("ASMGetApicIdExt8000001E=>%#x idApic=%#x\n", uOther, idApic));
238	}
239	return idApic;
240	}
241
242
243	/*
244	*
245	* GIP Mapping and Unmapping Related Code.
246	* GIP Mapping and Unmapping Related Code.
247	* GIP Mapping and Unmapping Related Code.
248	*
249	*
250	*/
251
252
253	/**
254	* (Re-)initializes the per-cpu structure prior to starting or resuming the GIP
255	* updating.
256	*
257	* @param pGipCpu The per CPU structure for this CPU.
258	* @param u64NanoTS The current time.
259	*/
260	static void supdrvGipReInitCpu(PSUPGIPCPU pGipCpu, uint64_t u64NanoTS)
261	{
262	/*
263	* Here we don't really care about applying the TSC delta. The re-initialization of this
264	* value is not relevant especially while (re)starting the GIP as the first few ones will
265	* be ignored anyway, see supdrvGipDoUpdateCpu().
266	*/
267	pGipCpu->u64TSC = ASMReadTSC() - pGipCpu->u32UpdateIntervalTSC;
268	pGipCpu->u64NanoTS = u64NanoTS;
269	}
270
271
272	/**
273	* Set the current TSC and NanoTS value for the CPU.
274	*
275	* @param idCpu The CPU ID. Unused - we have to use the APIC ID.
276	* @param pvUser1 Pointer to the ring-0 GIP mapping.
277	* @param pvUser2 Pointer to the variable holding the current time.
278	*/
279	static DECLCALLBACK(void) supdrvGipReInitCpuCallback(RTCPUID idCpu, void pvUser1, void pvUser2)
280	{
281	PSUPGLOBALINFOPAGE pGip = (PSUPGLOBALINFOPAGE)pvUser1;
282	uint32_t const idApic = supdrvGipGetApicId(pGip);
283	if (idApic < RT_ELEMENTS(pGip->aiCpuFromApicId))
284	{
285	unsigned const iCpu = pGip->aiCpuFromApicId[idApic];
286
287	if (RT_LIKELY(iCpu < pGip->cCpus && pGip->aCPUs[iCpu].idCpu == idCpu))
288	supdrvGipReInitCpu(&pGip->aCPUs[iCpu], (uint64_t )pvUser2);
289	else
290	LogRelMax(64, ("supdrvGipReInitCpuCallback: iCpu=%#x out of bounds (%#zx, idApic=%#x)\n",
291	iCpu, RT_ELEMENTS(pGip->aiCpuFromApicId), idApic));
292	}
293	else
294	LogRelMax(64, ("supdrvGipReInitCpuCallback: idApic=%#x out of bounds (%#zx)\n",
295	idApic, RT_ELEMENTS(pGip->aiCpuFromApicId)));
296
297	NOREF(pvUser2);
298	}
299
300
301	/**
302	* State structure for supdrvGipDetectGetGipCpuCallback.
303	*/
304	typedef struct SUPDRVGIPDETECTGETCPU
305	{
306	/** Bitmap of APIC IDs that has been seen (initialized to zero).
307	* Used to detect duplicate APIC IDs (paranoia). */
308	uint8_t volatile bmApicId[4096 / 8];
309	/** Mask of supported GIP CPU getter methods (SUPGIPGETCPU_XXX) (all bits set
310	* initially). The callback clears the methods not detected. */
311	uint32_t volatile fSupported;
312	/** The first callback detecting any kind of range issues (initialized to
313	* NIL_RTCPUID). */
314	RTCPUID volatile idCpuProblem;
315	} SUPDRVGIPDETECTGETCPU;
316	/** Pointer to state structure for supdrvGipDetectGetGipCpuCallback. */
317	typedef SUPDRVGIPDETECTGETCPU *PSUPDRVGIPDETECTGETCPU;
318
319
320	/**
321	* Checks for alternative ways of getting the CPU ID.
322	*
323	* This also checks the APIC ID, CPU ID and CPU set index values against the
324	* GIP tables.
325	*
326	* @param idCpu The CPU ID. Unused - we have to use the APIC ID.
327	* @param pvUser1 Pointer to the state structure.
328	* @param pvUser2 Pointer to the GIP.
329	*/
330	static DECLCALLBACK(void) supdrvGipDetectGetGipCpuCallback(RTCPUID idCpu, void pvUser1, void pvUser2)
331	{
332	PSUPDRVGIPDETECTGETCPU pState = (PSUPDRVGIPDETECTGETCPU)pvUser1;
333	PSUPGLOBALINFOPAGE pGip = (PSUPGLOBALINFOPAGE)pvUser2;
334	uint32_t fSupported = 0;
335	uint32_t idApic;
336	uint32_t uEax, uEbx, uEcx, uEdx;
337	int iCpuSet;
338	NOREF(pGip);
339
340	AssertMsg(idCpu == RTMpCpuId(), ("idCpu=%#x RTMpCpuId()=%#x\n", idCpu, RTMpCpuId())); /* paranoia^3 */
341
342	/*
343	* Check that the CPU ID and CPU set index are interchangable.
344	*/
345	iCpuSet = RTMpCpuIdToSetIndex(idCpu);
346	if ((RTCPUID)iCpuSet == idCpu)
347	{
348	AssertCompile(RT_IS_POWER_OF_TWO(RTCPUSET_MAX_CPUS));
349	if ( iCpuSet >= 0
350	&& iCpuSet < RTCPUSET_MAX_CPUS
351	&& RT_IS_POWER_OF_TWO(RTCPUSET_MAX_CPUS))
352	{
353	PSUPGIPCPU pGipCpu = SUPGetGipCpuBySetIndex(pGip, iCpuSet);
354
355	/*
356	* Check whether the IDTR.LIMIT contains a CPU number.
357	*/
358	#ifdef RT_ARCH_X86
359	uint16_t const cbIdt = sizeof(X86DESC64SYSTEM) * 256;
360	#else
361	uint16_t const cbIdt = sizeof(X86DESCGATE) * 256;
362	#endif
363	RTIDTR Idtr;
364	ASMGetIDTR(&Idtr);
365	if (Idtr.cbIdt >= cbIdt)
366	{
367	uint32_t uTmp = Idtr.cbIdt - cbIdt;
368	uTmp &= RTCPUSET_MAX_CPUS - 1;
369	if (uTmp == idCpu)
370	{
371	RTIDTR Idtr2;
372	ASMGetIDTR(&Idtr2);
373	if (Idtr2.cbIdt == Idtr.cbIdt)
374	fSupported \|= SUPGIPGETCPU_IDTR_LIMIT_MASK_MAX_SET_CPUS;
375	}
376	}
377
378	/*
379	* Check whether RDTSCP is an option.
380	*/
381	if (ASMHasCpuId())
382	{
383	if ( RTX86IsValidExtRange(ASMCpuId_EAX(UINT32_C(0x80000000)))
384	&& (ASMCpuId_EDX(UINT32_C(0x80000001)) & X86_CPUID_EXT_FEATURE_EDX_RDTSCP) )
385	{
386	uint32_t uAux;
387	ASMReadTscWithAux(&uAux);
388	if ((uAux & (RTCPUSET_MAX_CPUS - 1)) == idCpu)
389	{
390	ASMNopPause();
391	ASMReadTscWithAux(&uAux);
392	if ((uAux & (RTCPUSET_MAX_CPUS - 1)) == idCpu)
393	fSupported \|= SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS;
394	}
395
396	if (pGipCpu)
397	{
398	uint32_t const uGroupedAux = (uint8_t)pGipCpu->iCpuGroupMember \| ((uint32_t)pGipCpu->iCpuGroup << 8);
399	if ( (uAux & UINT16_MAX) == uGroupedAux
400	&& pGipCpu->iCpuGroupMember <= UINT8_MAX)
401	{
402	ASMNopPause();
403	ASMReadTscWithAux(&uAux);
404	if ((uAux & UINT16_MAX) == uGroupedAux)
405	fSupported \|= SUPGIPGETCPU_RDTSCP_GROUP_IN_CH_NUMBER_IN_CL;
406	}
407	}
408	}
409	}
410	}
411	}
412
413	/*
414	* Check for extended APIC ID methods.
415	*/
416	idApic = UINT32_MAX;
417	uEax = ASMCpuId_EAX(0);
418	if (uEax >= UINT32_C(0xb) && RTX86IsValidStdRange(uEax))
419	{
420	#if defined(RT_OS_LINUX) \|\| defined(RT_OS_FREEBSD)
421	ASMCpuId_Idx_ECX(0xb, 0, &uEax, &uEbx, &uEcx, &uEdx);
422	#else
423	ASMCpuIdExSlow(0xb, 0, 0, 0, &uEax, &uEbx, &uEcx, &uEdx);
424	#endif
425	if ((uEcx >> 8) != 0) /* level type != invalid */
426	{
427	if (RT_LIKELY( uEdx < RT_ELEMENTS(pGip->aiCpuFromApicId)
428	&& !ASMBitTest(pState->bmApicId, uEdx)))
429	{
430	if (uEdx == ASMGetApicIdExt0B())
431	{
432	idApic = uEdx;
433	fSupported \|= SUPGIPGETCPU_APIC_ID_EXT_0B;
434	}
435	else
436	AssertMsgFailed(("%#x vs %#x\n", uEdx, ASMGetApicIdExt0B()));
437	}
438	}
439	}
440
441	uEax = ASMCpuId_EAX(UINT32_C(0x80000000));
442	if (uEax >= UINT32_C(0x8000001e) && RTX86IsValidExtRange(uEax))
443	{
444	#if defined(RT_OS_LINUX) \|\| defined(RT_OS_FREEBSD)
445	ASMCpuId_Idx_ECX(UINT32_C(0x8000001e), 0, &uEax, &uEbx, &uEcx, &uEdx);
446	#else
447	ASMCpuIdExSlow(UINT32_C(0x8000001e), 0, 0, 0, &uEax, &uEbx, &uEcx, &uEdx);
448	#endif
449	if (uEax \|\| uEbx \|\| uEcx \|\| uEdx)
450	{
451	if (RT_LIKELY( uEax < RT_ELEMENTS(pGip->aiCpuFromApicId)
452	&& ( idApic == UINT32_MAX
453	\|\| idApic == uEax)
454	&& !ASMBitTest(pState->bmApicId, uEax)))
455	{
456	if (uEax == ASMGetApicIdExt8000001E())
457	{
458	idApic = uEax;
459	fSupported \|= SUPGIPGETCPU_APIC_ID_EXT_8000001E;
460	}
461	else
462	AssertMsgFailed(("%#x vs %#x\n", uEax, ASMGetApicIdExt8000001E()));
463	}
464	}
465	}
466
467	/*
468	* Check that the APIC ID is unique.
469	*/
470	uEax = ASMGetApicId();
471	if (RT_LIKELY( uEax < RT_ELEMENTS(pGip->aiCpuFromApicId)
472	&& ( idApic == UINT32_MAX
473	\|\| idApic == uEax)
474	&& !ASMAtomicBitTestAndSet(pState->bmApicId, uEax)))
475	{
476	idApic = uEax;
477	fSupported \|= SUPGIPGETCPU_APIC_ID;
478	}
479	else if ( idApic == UINT32_MAX
480	\|\| idApic >= RT_ELEMENTS(pGip->aiCpuFromApicId) /* parnaoia */
481	\|\| ASMAtomicBitTestAndSet(pState->bmApicId, idApic))
482	{
483	AssertCompile(sizeof(pState->bmApicId) * 8 == RT_ELEMENTS(pGip->aiCpuFromApicId));
484	ASMAtomicCmpXchgU32(&pState->idCpuProblem, idCpu, NIL_RTCPUID);
485	LogRel(("supdrvGipDetectGetGipCpuCallback: idCpu=%#x iCpuSet=%d idApic=%#x/%#x - duplicate APIC ID.\n",
486	idCpu, iCpuSet, uEax, idApic));
487	}
488
489	/*
490	* Check that the iCpuSet is within the expected range.
491	*/
492	if (RT_UNLIKELY( iCpuSet < 0
493	\|\| (unsigned)iCpuSet >= RTCPUSET_MAX_CPUS
494	\|\| (unsigned)iCpuSet >= RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)))
495	{
496	ASMAtomicCmpXchgU32(&pState->idCpuProblem, idCpu, NIL_RTCPUID);
497	LogRel(("supdrvGipDetectGetGipCpuCallback: idCpu=%#x iCpuSet=%d idApic=%#x - CPU set index is out of range.\n",
498	idCpu, iCpuSet, idApic));
499	}
500	else
501	{
502	RTCPUID idCpu2 = RTMpCpuIdFromSetIndex(iCpuSet);
503	if (RT_UNLIKELY(idCpu2 != idCpu))
504	{
505	ASMAtomicCmpXchgU32(&pState->idCpuProblem, idCpu, NIL_RTCPUID);
506	LogRel(("supdrvGipDetectGetGipCpuCallback: idCpu=%#x iCpuSet=%d idApic=%#x - CPU id/index roundtrip problem: %#x\n",
507	idCpu, iCpuSet, idApic, idCpu2));
508	}
509	}
510
511	/*
512	* Update the supported feature mask before we return.
513	*/
514	ASMAtomicAndU32(&pState->fSupported, fSupported);
515
516	NOREF(pvUser2);
517	}
518
519
520	/**
521	* Increase the timer freqency on hosts where this is possible (NT).
522	*
523	* The idea is that more interrupts is better for us... Also, it's better than
524	* we increase the timer frequence, because we might end up getting inaccurate
525	* callbacks if someone else does it.
526	*
527	* @param pDevExt Sets u32SystemTimerGranularityGrant if increased.
528	*/
529	static void supdrvGipRequestHigherTimerFrequencyFromSystem(PSUPDRVDEVEXT pDevExt)
530	{
531	if (pDevExt->u32SystemTimerGranularityGrant == 0)
532	{
533	uint32_t u32SystemResolution;
534	if ( RT_SUCCESS_NP(RTTimerRequestSystemGranularity( 976563 /* 1024 HZ */, &u32SystemResolution))
535	\|\| RT_SUCCESS_NP(RTTimerRequestSystemGranularity( 1000000 /* 1000 HZ */, &u32SystemResolution))
536	\|\| RT_SUCCESS_NP(RTTimerRequestSystemGranularity( 1953125 /* 512 HZ */, &u32SystemResolution))
537	\|\| RT_SUCCESS_NP(RTTimerRequestSystemGranularity( 2000000 /* 500 HZ */, &u32SystemResolution))
538	)
539	{
540	#if 0 /* def VBOX_STRICT - this is somehow triggers bogus assertions on windows 10 */
541	uint32_t u32After = RTTimerGetSystemGranularity();
542	AssertMsg(u32After <= u32SystemResolution, ("u32After=%u u32SystemResolution=%u\n", u32After, u32SystemResolution));
543	#endif
544	pDevExt->u32SystemTimerGranularityGrant = u32SystemResolution;
545	}
546	}
547	}
548
549
550	/**
551	* Undoes supdrvGipRequestHigherTimerFrequencyFromSystem.
552	*
553	* @param pDevExt Clears u32SystemTimerGranularityGrant.
554	*/
555	static void supdrvGipReleaseHigherTimerFrequencyFromSystem(PSUPDRVDEVEXT pDevExt)
556	{
557	if (pDevExt->u32SystemTimerGranularityGrant)
558	{
559	int rc2 = RTTimerReleaseSystemGranularity(pDevExt->u32SystemTimerGranularityGrant);
560	AssertRC(rc2);
561	pDevExt->u32SystemTimerGranularityGrant = 0;
562	}
563	}
564
565
566	/**
567	* Maps the GIP into userspace and/or get the physical address of the GIP.
568	*
569	* @returns IPRT status code.
570	* @param pSession Session to which the GIP mapping should belong.
571	* @param ppGipR3 Where to store the address of the ring-3 mapping. (optional)
572	* @param pHCPhysGip Where to store the physical address. (optional)
573	*
574	* @remark There is no reference counting on the mapping, so one call to this function
575	* count globally as one reference. One call to SUPR0GipUnmap() is will unmap GIP
576	* and remove the session as a GIP user.
577	*/
578	SUPR0DECL(int) SUPR0GipMap(PSUPDRVSESSION pSession, PRTR3PTR ppGipR3, PRTHCPHYS pHCPhysGip)
579	{
580	int rc;
581	PSUPDRVDEVEXT pDevExt = pSession->pDevExt;
582	RTR3PTR pGipR3 = NIL_RTR3PTR;
583	RTHCPHYS HCPhys = NIL_RTHCPHYS;
584	LogFlow(("SUPR0GipMap: pSession=%p ppGipR3=%p pHCPhysGip=%p\n", pSession, ppGipR3, pHCPhysGip));
585
586	/*
587	* Validate
588	*/
589	AssertReturn(SUP_IS_SESSION_VALID(pSession), VERR_INVALID_PARAMETER);
590	AssertPtrNullReturn(ppGipR3, VERR_INVALID_POINTER);
591	AssertPtrNullReturn(pHCPhysGip, VERR_INVALID_POINTER);
592
593	#ifdef SUPDRV_USE_MUTEX_FOR_GIP
594	RTSemMutexRequest(pDevExt->mtxGip, RT_INDEFINITE_WAIT);
595	#else
596	RTSemFastMutexRequest(pDevExt->mtxGip);
597	#endif
598	if (pDevExt->pGip)
599	{
600	/*
601	* Map it?
602	*/
603	rc = VINF_SUCCESS;
604	if (ppGipR3)
605	{
606	if (pSession->GipMapObjR3 == NIL_RTR0MEMOBJ)
607	rc = RTR0MemObjMapUser(&pSession->GipMapObjR3, pDevExt->GipMemObj, (RTR3PTR)-1, 0,
608	RTMEM_PROT_READ, NIL_RTR0PROCESS);
609	if (RT_SUCCESS(rc))
610	pGipR3 = RTR0MemObjAddressR3(pSession->GipMapObjR3);
611	}
612
613	/*
614	* Get physical address.
615	*/
616	if (pHCPhysGip && RT_SUCCESS(rc))
617	HCPhys = pDevExt->HCPhysGip;
618
619	/*
620	* Reference globally.
621	*/
622	if (!pSession->fGipReferenced && RT_SUCCESS(rc))
623	{
624	pSession->fGipReferenced = 1;
625	pDevExt->cGipUsers++;
626	if (pDevExt->cGipUsers == 1)
627	{
628	PSUPGLOBALINFOPAGE pGipR0 = pDevExt->pGip;
629	uint64_t u64NanoTS;
630
631	/*
632	* GIP starts/resumes updating again. On windows we bump the
633	* host timer frequency to make sure we don't get stuck in guest
634	* mode and to get better timer (and possibly clock) accuracy.
635	*/
636	LogFlow(("SUPR0GipMap: Resumes GIP updating\n"));
637
638	supdrvGipRequestHigherTimerFrequencyFromSystem(pDevExt);
639
640	/*
641	* document me
642	*/
643	if (pGipR0->aCPUs[0].u32TransactionId != 2 /* not the first time */)
644	{
645	unsigned i;
646	for (i = 0; i < pGipR0->cCpus; i++)
647	ASMAtomicUoWriteU32(&pGipR0->aCPUs[i].u32TransactionId,
648	(pGipR0->aCPUs[i].u32TransactionId + GIP_UPDATEHZ_RECALC_FREQ * 2)
649	& ~(GIP_UPDATEHZ_RECALC_FREQ * 2 - 1));
650	ASMAtomicWriteU64(&pGipR0->u64NanoTSLastUpdateHz, 0);
651	}
652
653	/*
654	* document me
655	*/
656	u64NanoTS = RTTimeSystemNanoTS() - pGipR0->u32UpdateIntervalNS;
657	if ( pGipR0->u32Mode == SUPGIPMODE_INVARIANT_TSC
658	\|\| pGipR0->u32Mode == SUPGIPMODE_SYNC_TSC
659	\|\| RTMpGetOnlineCount() == 1)
660	supdrvGipReInitCpu(&pGipR0->aCPUs[0], u64NanoTS);
661	else
662	RTMpOnAll(supdrvGipReInitCpuCallback, pGipR0, &u64NanoTS);
663
664	/*
665	* Detect alternative ways to figure the CPU ID in ring-3 and
666	* raw-mode context. Check the sanity of the APIC IDs, CPU IDs,
667	* and CPU set indexes while we're at it.
668	*/
669	if (RT_SUCCESS(rc))
670	{
671	PSUPDRVGIPDETECTGETCPU pDetectState = (PSUPDRVGIPDETECTGETCPU)RTMemTmpAllocZ(sizeof(*pDetectState));
672	if (pDetectState)
673	{
674	pDetectState->fSupported = UINT32_MAX;
675	pDetectState->idCpuProblem = NIL_RTCPUID;
676	rc = RTMpOnAll(supdrvGipDetectGetGipCpuCallback, pDetectState, pGipR0);
677	if (pDetectState->idCpuProblem == NIL_RTCPUID)
678	{
679	if ( pDetectState->fSupported != UINT32_MAX
680	&& pDetectState->fSupported != 0)
681	{
682	if (pGipR0->fGetGipCpu != pDetectState->fSupported)
683	{
684	pGipR0->fGetGipCpu = pDetectState->fSupported;
685	LogRel(("SUPR0GipMap: fGetGipCpu=%#x\n", pDetectState->fSupported));
686	}
687	}
688	else
689	{
690	LogRel(("SUPR0GipMap: No supported ways of getting the APIC ID or CPU number in ring-3! (%#x)\n",
691	pDetectState->fSupported));
692	rc = VERR_UNSUPPORTED_CPU;
693	}
694	}
695	else
696	{
697	LogRel(("SUPR0GipMap: APIC ID, CPU ID or CPU set index problem detected on CPU #%u (%#x)!\n",
698	pDetectState->idCpuProblem, pDetectState->idCpuProblem));
699	rc = VERR_INVALID_CPU_ID;
700	}
701	RTMemTmpFree(pDetectState);
702	}
703	else
704	rc = VERR_NO_TMP_MEMORY;
705	}
706
707	/*
708	* Start the GIP timer if all is well..
709	*/
710	if (RT_SUCCESS(rc))
711	{
712	#ifndef DO_NOT_START_GIP
713	rc = RTTimerStart(pDevExt->pGipTimer, 0 /* fire ASAP */); AssertRC(rc);
714	#endif
715	rc = VINF_SUCCESS;
716	}
717
718	/*
719	* Bail out on error.
720	*/
721	if (RT_FAILURE(rc))
722	{
723	LogRel(("SUPR0GipMap: failed rc=%Rrc\n", rc));
724	pDevExt->cGipUsers = 0;
725	pSession->fGipReferenced = 0;
726	if (pSession->GipMapObjR3 != NIL_RTR0MEMOBJ)
727	{
728	int rc2 = RTR0MemObjFree(pSession->GipMapObjR3, false); AssertRC(rc2);
729	if (RT_SUCCESS(rc2))
730	pSession->GipMapObjR3 = NIL_RTR0MEMOBJ;
731	}
732	HCPhys = NIL_RTHCPHYS;
733	pGipR3 = NIL_RTR3PTR;
734	}
735	}
736	}
737	}
738	else
739	{
740	rc = VERR_GENERAL_FAILURE;
741	Log(("SUPR0GipMap: GIP is not available!\n"));
742	}
743	#ifdef SUPDRV_USE_MUTEX_FOR_GIP
744	RTSemMutexRelease(pDevExt->mtxGip);
745	#else
746	RTSemFastMutexRelease(pDevExt->mtxGip);
747	#endif
748
749	/*
750	* Write returns.
751	*/
752	if (pHCPhysGip)
753	*pHCPhysGip = HCPhys;
754	if (ppGipR3)
755	*ppGipR3 = pGipR3;
756
757	#ifdef DEBUG_DARWIN_GIP
758	OSDBGPRINT(("SUPR0GipMap: returns %d pHCPhysGip=%lx pGipR3=%p\n", rc, (unsigned long)HCPhys, (void )pGipR3));
759	#else
760	LogFlow(( "SUPR0GipMap: returns %d pHCPhysGip=%lx pGipR3=%p\n", rc, (unsigned long)HCPhys, (void )pGipR3));
761	#endif
762	return rc;
763	}
764	SUPR0_EXPORT_SYMBOL(SUPR0GipMap);
765
766
767	/**
768	* Unmaps any user mapping of the GIP and terminates all GIP access
769	* from this session.
770	*
771	* @returns IPRT status code.
772	* @param pSession Session to which the GIP mapping should belong.
773	*/
774	SUPR0DECL(int) SUPR0GipUnmap(PSUPDRVSESSION pSession)
775	{
776	int rc = VINF_SUCCESS;
777	PSUPDRVDEVEXT pDevExt = pSession->pDevExt;
778	#ifdef DEBUG_DARWIN_GIP
779	OSDBGPRINT(("SUPR0GipUnmap: pSession=%p pGip=%p GipMapObjR3=%p\n",
780	pSession,
781	pSession->GipMapObjR3 != NIL_RTR0MEMOBJ ? RTR0MemObjAddress(pSession->GipMapObjR3) : NULL,
782	pSession->GipMapObjR3));
783	#else
784	LogFlow(("SUPR0GipUnmap: pSession=%p\n", pSession));
785	#endif
786	AssertReturn(SUP_IS_SESSION_VALID(pSession), VERR_INVALID_PARAMETER);
787
788	#ifdef SUPDRV_USE_MUTEX_FOR_GIP
789	RTSemMutexRequest(pDevExt->mtxGip, RT_INDEFINITE_WAIT);
790	#else
791	RTSemFastMutexRequest(pDevExt->mtxGip);
792	#endif
793
794	/*
795	* GIP test-mode session?
796	*/
797	if ( pSession->fGipTestMode
798	&& pDevExt->pGip)
799	{
800	supdrvGipSetFlags(pDevExt, pSession, 0, ~SUPGIP_FLAGS_TESTING_ENABLE);
801	Assert(!pSession->fGipTestMode);
802	}
803
804	/*
805	* Unmap anything?
806	*/
807	if (pSession->GipMapObjR3 != NIL_RTR0MEMOBJ)
808	{
809	rc = RTR0MemObjFree(pSession->GipMapObjR3, false);
810	AssertRC(rc);
811	if (RT_SUCCESS(rc))
812	pSession->GipMapObjR3 = NIL_RTR0MEMOBJ;
813	}
814
815	/*
816	* Dereference global GIP.
817	*/
818	if (pSession->fGipReferenced && !rc)
819	{
820	pSession->fGipReferenced = 0;
821	if ( pDevExt->cGipUsers > 0
822	&& !--pDevExt->cGipUsers)
823	{
824	LogFlow(("SUPR0GipUnmap: Suspends GIP updating\n"));
825	#ifndef DO_NOT_START_GIP
826	rc = RTTimerStop(pDevExt->pGipTimer); AssertRC(rc); rc = VINF_SUCCESS;
827	#endif
828	supdrvGipReleaseHigherTimerFrequencyFromSystem(pDevExt);
829	}
830	}
831
832	#ifdef SUPDRV_USE_MUTEX_FOR_GIP
833	RTSemMutexRelease(pDevExt->mtxGip);
834	#else
835	RTSemFastMutexRelease(pDevExt->mtxGip);
836	#endif
837
838	return rc;
839	}
840	SUPR0_EXPORT_SYMBOL(SUPR0GipUnmap);
841
842
843	/**
844	* Gets the GIP pointer.
845	*
846	* @returns Pointer to the GIP or NULL.
847	*/
848	SUPDECL(PSUPGLOBALINFOPAGE) SUPGetGIP(void)
849	{
850	return g_pSUPGlobalInfoPage;
851	}
852
853
854
855
856
857	/*
858	*
859	*
860	* GIP Initialization, Termination and CPU Offline / Online Related Code.
861	* GIP Initialization, Termination and CPU Offline / Online Related Code.
862	* GIP Initialization, Termination and CPU Offline / Online Related Code.
863	*
864	*
865	*/
866
867	/**
868	* Used by supdrvGipInitRefineInvariantTscFreqTimer and supdrvGipInitMeasureTscFreq
869	* to update the TSC frequency related GIP variables.
870	*
871	* @param pGip The GIP.
872	* @param nsElapsed The number of nanoseconds elapsed.
873	* @param cElapsedTscTicks The corresponding number of TSC ticks.
874	* @param iTick The tick number for debugging.
875	*/
876	static void supdrvGipInitSetCpuFreq(PSUPGLOBALINFOPAGE pGip, uint64_t nsElapsed, uint64_t cElapsedTscTicks, uint32_t iTick)
877	{
878	/*
879	* Calculate the frequency.
880	*/
881	uint64_t uCpuHz;
882	if ( cElapsedTscTicks < UINT64_MAX / RT_NS_1SEC
883	&& nsElapsed < UINT32_MAX)
884	uCpuHz = ASMMultU64ByU32DivByU32(cElapsedTscTicks, RT_NS_1SEC, (uint32_t)nsElapsed);
885	else
886	{
887	RTUINT128U CpuHz, Tmp, Divisor;
888	CpuHz.s.Lo = CpuHz.s.Hi = 0;
889	RTUInt128MulU64ByU64(&Tmp, cElapsedTscTicks, RT_NS_1SEC_64);
890	RTUInt128Div(&CpuHz, &Tmp, RTUInt128AssignU64(&Divisor, nsElapsed));
891	uCpuHz = CpuHz.s.Lo;
892	}
893
894	/*
895	* Update the GIP.
896	*/
897	ASMAtomicWriteU64(&pGip->u64CpuHz, uCpuHz);
898	if (pGip->u32Mode != SUPGIPMODE_ASYNC_TSC)
899	{
900	ASMAtomicWriteU64(&pGip->aCPUs[0].u64CpuHz, uCpuHz);
901
902	/* For inspecting the frequency calcs using tstGIP-2, debugger or similar. */
903	if (iTick + 1 < pGip->cCpus)
904	ASMAtomicWriteU64(&pGip->aCPUs[iTick + 1].u64CpuHz, uCpuHz);
905	}
906	}
907
908
909	/**
910	* Timer callback function for TSC frequency refinement in invariant GIP mode.
911	*
912	* This is started during driver init and fires once
913	* GIP_TSC_REFINE_PERIOD_IN_SECS seconds later.
914	*
915	* @param pTimer The timer.
916	* @param pvUser Opaque pointer to the device instance data.
917	* @param iTick The timer tick.
918	*/
919	static DECLCALLBACK(void) supdrvGipInitRefineInvariantTscFreqTimer(PRTTIMER pTimer, void *pvUser, uint64_t iTick)
920	{
921	PSUPDRVDEVEXT pDevExt = (PSUPDRVDEVEXT)pvUser;
922	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
923	RTCPUID idCpu;
924	uint64_t cNsElapsed;
925	uint64_t cTscTicksElapsed;
926	uint64_t nsNow;
927	uint64_t uTsc;
928	RTCCUINTREG fEFlags;
929
930	/* Paranoia. */
931	AssertReturnVoid(pGip);
932	AssertReturnVoid(pGip->u32Mode == SUPGIPMODE_INVARIANT_TSC);
933
934	/*
935	* If we got a power event, stop the refinement process.
936	*/
937	if (pDevExt->fInvTscRefinePowerEvent)
938	{
939	int rc = RTTimerStop(pTimer); AssertRC(rc);
940	return;
941	}
942
943	/*
944	* Read the TSC and time, noting which CPU we are on.
945	*
946	* Don't bother spinning until RTTimeSystemNanoTS changes, since on
947	* systems where it matters we're in a context where we cannot waste that
948	* much time (DPC watchdog, called from clock interrupt).
949	*/
950	fEFlags = ASMIntDisableFlags();
951	uTsc = ASMReadTSC();
952	nsNow = RTTimeSystemNanoTS();
953	idCpu = RTMpCpuId();
954	ASMSetFlags(fEFlags);
955
956	cNsElapsed = nsNow - pDevExt->nsStartInvarTscRefine;
957	cTscTicksElapsed = uTsc - pDevExt->uTscStartInvarTscRefine;
958
959	/*
960	* If the above measurement was taken on a different CPU than the one we
961	* started the process on, cTscTicksElapsed will need to be adjusted with
962	* the TSC deltas of both the CPUs.
963	*
964	* We ASSUME that the delta calculation process takes less time than the
965	* TSC frequency refinement timer. If it doesn't, we'll complain and
966	* drop the frequency refinement.
967	*
968	* Note! We cannot entirely trust enmUseTscDelta here because it's
969	* downgraded after each delta calculation.
970	*/
971	if ( idCpu != pDevExt->idCpuInvarTscRefine
972	&& pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
973	{
974	uint32_t iStartCpuSet = RTMpCpuIdToSetIndex(pDevExt->idCpuInvarTscRefine);
975	uint32_t iStopCpuSet = RTMpCpuIdToSetIndex(idCpu);
976	uint16_t iStartGipCpu = iStartCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)
977	? pGip->aiCpuFromCpuSetIdx[iStartCpuSet] : UINT16_MAX;
978	uint16_t iStopGipCpu = iStopCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)
979	? pGip->aiCpuFromCpuSetIdx[iStopCpuSet] : UINT16_MAX;
980	int64_t iStartTscDelta = iStartGipCpu < pGip->cCpus ? pGip->aCPUs[iStartGipCpu].i64TSCDelta : INT64_MAX;
981	int64_t iStopTscDelta = iStopGipCpu < pGip->cCpus ? pGip->aCPUs[iStopGipCpu].i64TSCDelta : INT64_MAX;
982	if (RT_LIKELY(iStartTscDelta != INT64_MAX && iStopTscDelta != INT64_MAX))
983	{
984	if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_PRACTICALLY_ZERO)
985	{
986	/* cTscTicksElapsed = (uTsc - iStopTscDelta) - (pDevExt->uTscStartInvarTscRefine - iStartTscDelta); */
987	cTscTicksElapsed += iStartTscDelta - iStopTscDelta;
988	}
989	}
990	/*
991	* Allow 5 times the refinement period to elapse before we give up on the TSC delta
992	* calculations.
993	*/
994	else if (cNsElapsed > GIP_TSC_REFINE_PERIOD_IN_SECS * 5 * RT_NS_1SEC_64)
995	{
996	SUPR0Printf("vboxdrv: Failed to refine invariant TSC frequency because deltas are unavailable after %u (%u) seconds\n",
997	(uint32_t)(cNsElapsed / RT_NS_1SEC), GIP_TSC_REFINE_PERIOD_IN_SECS);
998	SUPR0Printf("vboxdrv: start: %u, %u, %#llx stop: %u, %u, %#llx\n",
999	iStartCpuSet, iStartGipCpu, iStartTscDelta, iStopCpuSet, iStopGipCpu, iStopTscDelta);
1000	int rc = RTTimerStop(pTimer); AssertRC(rc);
1001	return;
1002	}
1003	}
1004
1005	/*
1006	* Calculate and update the CPU frequency variables in GIP.
1007	*
1008	* If there is a GIP user already and we've already refined the frequency
1009	* a couple of times, don't update it as we want a stable frequency value
1010	* for all VMs.
1011	*/
1012	if ( pDevExt->cGipUsers == 0
1013	\|\| cNsElapsed < RT_NS_1SEC * 2)
1014	{
1015	supdrvGipInitSetCpuFreq(pGip, cNsElapsed, cTscTicksElapsed, (uint32_t)iTick);
1016
1017	/*
1018	* Stop the timer once we've reached the defined refinement period.
1019	*/
1020	if (cNsElapsed > GIP_TSC_REFINE_PERIOD_IN_SECS * RT_NS_1SEC_64)
1021	{
1022	int rc = RTTimerStop(pTimer);
1023	AssertRC(rc);
1024	}
1025	}
1026	else
1027	{
1028	int rc = RTTimerStop(pTimer);
1029	AssertRC(rc);
1030	}
1031	}
1032
1033
1034	/**
1035	* @callback_method_impl{FNRTPOWERNOTIFICATION}
1036	*/
1037	static DECLCALLBACK(void) supdrvGipPowerNotificationCallback(RTPOWEREVENT enmEvent, void *pvUser)
1038	{
1039	PSUPDRVDEVEXT pDevExt = (PSUPDRVDEVEXT)pvUser;
1040	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
1041
1042	/*
1043	* If the TSC frequency refinement timer is running, we need to cancel it so it
1044	* doesn't screw up the frequency after a long suspend.
1045	*
1046	* Recalculate all TSC-deltas on host resume as it may have changed, seen
1047	* on Windows 7 running on the Dell Optiplex Intel Core i5-3570.
1048	*/
1049	if (enmEvent == RTPOWEREVENT_RESUME)
1050	{
1051	ASMAtomicWriteBool(&pDevExt->fInvTscRefinePowerEvent, true);
1052	if ( RT_LIKELY(pGip)
1053	&& pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED
1054	&& !supdrvOSAreCpusOfflinedOnSuspend())
1055	{
1056	#ifdef SUPDRV_USE_TSC_DELTA_THREAD
1057	supdrvTscDeltaThreadStartMeasurement(pDevExt, true /* fForceAll */);
1058	#else
1059	RTCpuSetCopy(&pDevExt->TscDeltaCpuSet, &pGip->OnlineCpuSet);
1060	supdrvTscMeasureInitialDeltas(pDevExt);
1061	#endif
1062	}
1063	}
1064	else if (enmEvent == RTPOWEREVENT_SUSPEND)
1065	ASMAtomicWriteBool(&pDevExt->fInvTscRefinePowerEvent, true);
1066	}
1067
1068
1069	/**
1070	* Start the TSC-frequency refinment timer for the invariant TSC GIP mode.
1071	*
1072	* We cannot use this in the synchronous and asynchronous tsc GIP modes because
1073	* the CPU may change the TSC frequence between now and when the timer fires
1074	* (supdrvInitAsyncRefineTscTimer).
1075	*
1076	* @param pDevExt Pointer to the device instance data.
1077	*/
1078	static void supdrvGipInitStartTimerForRefiningInvariantTscFreq(PSUPDRVDEVEXT pDevExt)
1079	{
1080	uint64_t u64NanoTS;
1081	RTCCUINTREG fEFlags;
1082	int rc;
1083
1084	/*
1085	* Register a power management callback.
1086	*/
1087	pDevExt->fInvTscRefinePowerEvent = false;
1088	rc = RTPowerNotificationRegister(supdrvGipPowerNotificationCallback, pDevExt);
1089	AssertRC(rc); /* ignore */
1090
1091	/*
1092	* Record the TSC and NanoTS as the starting anchor point for refinement
1093	* of the TSC. We try get as close to a clock tick as possible on systems
1094	* which does not provide high resolution time.
1095	*/
1096	u64NanoTS = RTTimeSystemNanoTS();
1097	while (RTTimeSystemNanoTS() == u64NanoTS)
1098	ASMNopPause();
1099
1100	fEFlags = ASMIntDisableFlags();
1101	pDevExt->uTscStartInvarTscRefine = ASMReadTSC();
1102	pDevExt->nsStartInvarTscRefine = RTTimeSystemNanoTS();
1103	pDevExt->idCpuInvarTscRefine = RTMpCpuId();
1104	ASMSetFlags(fEFlags);
1105
1106	/*
1107	* Create a timer that runs on the same CPU so we won't have a depencency
1108	* on the TSC-delta and can run in parallel to it. On systems that does not
1109	* implement CPU specific timers we'll apply deltas in the timer callback,
1110	* just like we do for CPUs going offline.
1111	*
1112	* The longer the refinement interval the better the accuracy, at least in
1113	* theory. If it's too long though, ring-3 may already be starting its
1114	* first VMs before we're done. On most systems we will be loading the
1115	* support driver during boot and VMs won't be started for a while yet,
1116	* it is really only a problem during development (especially with
1117	* on-demand driver starting on windows).
1118	*
1119	* To avoid wasting time doing a long supdrvGipInitMeasureTscFreq() call
1120	* to calculate the frequency during driver loading, the timer is set
1121	* to fire after 200 ms the first time. It will then reschedule itself
1122	* to fire every second until GIP_TSC_REFINE_PERIOD_IN_SECS has been
1123	* reached or it notices that there is a user land client with GIP
1124	* mapped (we want a stable frequency for all VMs).
1125	*/
1126	rc = RTTimerCreateEx(&pDevExt->pInvarTscRefineTimer, RT_NS_1SEC,
1127	RTTIMER_FLAGS_CPU(RTMpCpuIdToSetIndex(pDevExt->idCpuInvarTscRefine)),
1128	supdrvGipInitRefineInvariantTscFreqTimer, pDevExt);
1129	if (RT_SUCCESS(rc))
1130	{
1131	rc = RTTimerStart(pDevExt->pInvarTscRefineTimer, 2*RT_NS_100MS);
1132	if (RT_SUCCESS(rc))
1133	return;
1134	RTTimerDestroy(pDevExt->pInvarTscRefineTimer);
1135	}
1136
1137	if (rc == VERR_CPU_OFFLINE \|\| rc == VERR_NOT_SUPPORTED)
1138	{
1139	rc = RTTimerCreateEx(&pDevExt->pInvarTscRefineTimer, RT_NS_1SEC, RTTIMER_FLAGS_CPU_ANY,
1140	supdrvGipInitRefineInvariantTscFreqTimer, pDevExt);
1141	if (RT_SUCCESS(rc))
1142	{
1143	rc = RTTimerStart(pDevExt->pInvarTscRefineTimer, 2*RT_NS_100MS);
1144	if (RT_SUCCESS(rc))
1145	return;
1146	RTTimerDestroy(pDevExt->pInvarTscRefineTimer);
1147	}
1148	}
1149
1150	pDevExt->pInvarTscRefineTimer = NULL;
1151	OSDBGPRINT(("vboxdrv: Failed to create or start TSC frequency refinement timer: rc=%Rrc\n", rc));
1152	}
1153
1154
1155	/**
1156	* @callback_method_impl{PFNRTMPWORKER,
1157	* RTMpOnSpecific callback for reading TSC and time on the CPU we started
1158	* the measurements on.}
1159	*/
1160	static DECLCALLBACK(void) supdrvGipInitReadTscAndNanoTsOnCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
1161	{
1162	RTCCUINTREG fEFlags = ASMIntDisableFlags();
1163	uint64_t puTscStop = (uint64_t )pvUser1;
1164	uint64_t pnsStop = (uint64_t )pvUser2;
1165	RT_NOREF1(idCpu);
1166
1167	*puTscStop = ASMReadTSC();
1168	*pnsStop = RTTimeSystemNanoTS();
1169
1170	ASMSetFlags(fEFlags);
1171	}
1172
1173
1174	/**
1175	* Measures the TSC frequency of the system.
1176	*
1177	* The TSC frequency can vary on systems which are not reported as invariant.
1178	* On such systems the object of this function is to find out what the nominal,
1179	* maximum TSC frequency under 'normal' CPU operation.
1180	*
1181	* @returns VBox status code.
1182	* @param pGip Pointer to the GIP.
1183	* @param fRough Set if we're doing the rough calculation that the
1184	* TSC measuring code needs, where accuracy isn't all
1185	* that important (too high is better than too low).
1186	* When clear we try for best accuracy that we can
1187	* achieve in reasonably short time.
1188	*/
1189	static int supdrvGipInitMeasureTscFreq(PSUPGLOBALINFOPAGE pGip, bool fRough)
1190	{
1191	uint32_t nsTimerIncr = RTTimerGetSystemGranularity();
1192	int cTriesLeft = fRough ? 4 : 2;
1193	while (cTriesLeft-- > 0)
1194	{
1195	RTCCUINTREG fEFlags;
1196	uint64_t nsStart;
1197	uint64_t nsStop;
1198	uint64_t uTscStart;
1199	uint64_t uTscStop;
1200	RTCPUID idCpuStart;
1201	RTCPUID idCpuStop;
1202
1203	/*
1204	* Synchronize with the host OS clock tick on systems without high
1205	* resolution time API (older Windows version for example).
1206	*/
1207	nsStart = RTTimeSystemNanoTS();
1208	while (RTTimeSystemNanoTS() == nsStart)
1209	ASMNopPause();
1210
1211	/*
1212	* Read the TSC and current time, noting which CPU we're on.
1213	*/
1214	fEFlags = ASMIntDisableFlags();
1215	uTscStart = ASMReadTSC();
1216	nsStart = RTTimeSystemNanoTS();
1217	idCpuStart = RTMpCpuId();
1218	ASMSetFlags(fEFlags);
1219
1220	/*
1221	* Delay for a while.
1222	*/
1223	if (pGip->u32Mode == SUPGIPMODE_INVARIANT_TSC)
1224	{
1225	/*
1226	* Sleep-wait since the TSC frequency is constant, it eases host load.
1227	* Shorter interval produces more variance in the frequency (esp. Windows).
1228	*/
1229	uint64_t msElapsed = 0;
1230	uint64_t msDelay = ( ((fRough ? 16 : 200) * RT_NS_1MS + nsTimerIncr - 1) / nsTimerIncr * nsTimerIncr - RT_NS_100US )
1231	/ RT_NS_1MS;
1232	do
1233	{
1234	RTThreadSleep((RTMSINTERVAL)(msDelay - msElapsed));
1235	nsStop = RTTimeSystemNanoTS();
1236	msElapsed = (nsStop - nsStart) / RT_NS_1MS;
1237	} while (msElapsed < msDelay);
1238
1239	while (RTTimeSystemNanoTS() == nsStop)
1240	ASMNopPause();
1241	}
1242	else
1243	{
1244	/*
1245	* Busy-wait keeping the frequency up.
1246	*/
1247	do
1248	{
1249	ASMNopPause();
1250	nsStop = RTTimeSystemNanoTS();
1251	} while (nsStop - nsStart < RT_NS_100MS);
1252	}
1253
1254	/*
1255	* Read the TSC and time again.
1256	*/
1257	fEFlags = ASMIntDisableFlags();
1258	uTscStop = ASMReadTSC();
1259	nsStop = RTTimeSystemNanoTS();
1260	idCpuStop = RTMpCpuId();
1261	ASMSetFlags(fEFlags);
1262
1263	/*
1264	* If the CPU changes, things get a bit complicated and what we
1265	* can get away with depends on the GIP mode / TSC reliability.
1266	*/
1267	if (idCpuStop != idCpuStart)
1268	{
1269	bool fDoXCall = false;
1270
1271	/*
1272	* Synchronous TSC mode: we're probably fine as it's unlikely
1273	* that we were rescheduled because of TSC throttling or power
1274	* management reasons, so just go ahead.
1275	*/
1276	if (pGip->u32Mode == SUPGIPMODE_SYNC_TSC)
1277	{
1278	/* Probably ok, maybe we should retry once?. */
1279	Assert(pGip->enmUseTscDelta == SUPGIPUSETSCDELTA_NOT_APPLICABLE);
1280	}
1281	/*
1282	* If we're just doing the rough measurement, do the cross call and
1283	* get on with things (we don't have deltas!).
1284	*/
1285	else if (fRough)
1286	fDoXCall = true;
1287	/*
1288	* Invariant TSC mode: It doesn't matter if we have delta available
1289	* for both CPUs. That is not something we can assume at this point.
1290	*
1291	* Note! We cannot necessarily trust enmUseTscDelta here because it's
1292	* downgraded after each delta calculation and the delta
1293	* calculations may not be complete yet.
1294	*/
1295	else if (pGip->u32Mode == SUPGIPMODE_INVARIANT_TSC)
1296	{
1297	/** @todo This section of code is never reached atm, consider dropping it later on... */
1298	if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
1299	{
1300	uint32_t iStartCpuSet = RTMpCpuIdToSetIndex(idCpuStart);
1301	uint32_t iStopCpuSet = RTMpCpuIdToSetIndex(idCpuStop);
1302	uint16_t iStartGipCpu = iStartCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)
1303	? pGip->aiCpuFromCpuSetIdx[iStartCpuSet] : UINT16_MAX;
1304	uint16_t iStopGipCpu = iStopCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)
1305	? pGip->aiCpuFromCpuSetIdx[iStopCpuSet] : UINT16_MAX;
1306	int64_t iStartTscDelta = iStartGipCpu < pGip->cCpus ? pGip->aCPUs[iStartGipCpu].i64TSCDelta : INT64_MAX;
1307	int64_t iStopTscDelta = iStopGipCpu < pGip->cCpus ? pGip->aCPUs[iStopGipCpu].i64TSCDelta : INT64_MAX;
1308	if (RT_LIKELY(iStartTscDelta != INT64_MAX && iStopTscDelta != INT64_MAX))
1309	{
1310	if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_PRACTICALLY_ZERO)
1311	{
1312	uTscStart -= iStartTscDelta;
1313	uTscStop -= iStopTscDelta;
1314	}
1315	}
1316	/*
1317	* Invalid CPU indexes are not caused by online/offline races, so
1318	* we have to trigger driver load failure if that happens as GIP
1319	* and IPRT assumptions are busted on this system.
1320	*/
1321	else if (iStopGipCpu >= pGip->cCpus \|\| iStartGipCpu >= pGip->cCpus)
1322	{
1323	SUPR0Printf("vboxdrv: Unexpected CPU index in supdrvGipInitMeasureTscFreq.\n");
1324	SUPR0Printf("vboxdrv: start: %u, %u, %#llx stop: %u, %u, %#llx\n",
1325	iStartCpuSet, iStartGipCpu, iStartTscDelta, iStopCpuSet, iStopGipCpu, iStopTscDelta);
1326	return VERR_INVALID_CPU_INDEX;
1327	}
1328	/*
1329	* No valid deltas. We retry, if we're on our last retry
1330	* we do the cross call instead just to get a result. The
1331	* frequency will be refined in a few seconds anyway.
1332	*/
1333	else if (cTriesLeft > 0)
1334	continue;
1335	else
1336	fDoXCall = true;
1337	}
1338	}
1339	/*
1340	* Asynchronous TSC mode: This is bad, as the reason we usually
1341	* use this mode is to deal with variable TSC frequencies and
1342	* deltas. So, we need to get the TSC from the same CPU as
1343	* started it, we also need to keep that CPU busy. So, retry
1344	* and fall back to the cross call on the last attempt.
1345	*/
1346	else
1347	{
1348	Assert(pGip->u32Mode == SUPGIPMODE_ASYNC_TSC);
1349	if (cTriesLeft > 0)
1350	continue;
1351	fDoXCall = true;
1352	}
1353
1354	if (fDoXCall)
1355	{
1356	/*
1357	* Try read the TSC and timestamp on the start CPU.
1358	*/
1359	int rc = RTMpOnSpecific(idCpuStart, supdrvGipInitReadTscAndNanoTsOnCpu, &uTscStop, &nsStop);
1360	if (RT_FAILURE(rc) && (!fRough \|\| cTriesLeft > 0))
1361	continue;
1362	}
1363	}
1364
1365	/*
1366	* Calculate the TSC frequency and update it (shared with the refinement timer).
1367	*/
1368	supdrvGipInitSetCpuFreq(pGip, nsStop - nsStart, uTscStop - uTscStart, 0);
1369	return VINF_SUCCESS;
1370	}
1371
1372	Assert(!fRough);
1373	return VERR_SUPDRV_TSC_FREQ_MEASUREMENT_FAILED;
1374	}
1375
1376
1377	/**
1378	* Finds our (@a idCpu) entry, or allocates a new one if not found.
1379	*
1380	* @returns Index of the CPU in the cache set.
1381	* @param pGip The GIP.
1382	* @param idCpu The CPU ID.
1383	*/
1384	static uint32_t supdrvGipFindOrAllocCpuIndexForCpuId(PSUPGLOBALINFOPAGE pGip, RTCPUID idCpu)
1385	{
1386	uint32_t i, cTries;
1387
1388	/*
1389	* ASSUMES that CPU IDs are constant.
1390	*/
1391	for (i = 0; i < pGip->cCpus; i++)
1392	if (pGip->aCPUs[i].idCpu == idCpu)
1393	return i;
1394
1395	cTries = 0;
1396	do
1397	{
1398	for (i = 0; i < pGip->cCpus; i++)
1399	{
1400	bool fRc;
1401	ASMAtomicCmpXchgSize(&pGip->aCPUs[i].idCpu, idCpu, NIL_RTCPUID, fRc);
1402	if (fRc)
1403	return i;
1404	}
1405	} while (cTries++ < 32);
1406	AssertReleaseFailed();
1407	return i - 1;
1408	}
1409
1410
1411	/**
1412	* The calling CPU should be accounted as online, update GIP accordingly.
1413	*
1414	* This is used by supdrvGipCreate() as well as supdrvGipMpEvent().
1415	*
1416	* @param pDevExt The device extension.
1417	* @param idCpu The CPU ID.
1418	*/
1419	static void supdrvGipMpEventOnlineOrInitOnCpu(PSUPDRVDEVEXT pDevExt, RTCPUID idCpu)
1420	{
1421	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
1422	int iCpuSet = 0;
1423	uint32_t idApic;
1424	uint32_t i = 0;
1425	uint64_t u64NanoTS = 0;
1426
1427	AssertPtrReturnVoid(pGip);
1428	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1429	AssertRelease(idCpu == RTMpCpuId());
1430	Assert(pGip->cPossibleCpus == RTMpGetCount());
1431
1432	/*
1433	* Do this behind a spinlock with interrupts disabled as this can fire
1434	* on all CPUs simultaneously, see @bugref{6110}.
1435	*/
1436	RTSpinlockAcquire(pDevExt->hGipSpinlock);
1437
1438	/*
1439	* Update the globals.
1440	*/
1441	ASMAtomicWriteU16(&pGip->cPresentCpus, RTMpGetPresentCount());
1442	ASMAtomicWriteU16(&pGip->cOnlineCpus, RTMpGetOnlineCount());
1443	iCpuSet = RTMpCpuIdToSetIndex(idCpu);
1444	if (iCpuSet >= 0)
1445	{
1446	Assert(RTCpuSetIsMemberByIndex(&pGip->PossibleCpuSet, iCpuSet));
1447	RTCpuSetAddByIndex(&pGip->OnlineCpuSet, iCpuSet);
1448	RTCpuSetAddByIndex(&pGip->PresentCpuSet, iCpuSet);
1449	}
1450
1451	/*
1452	* Update the entry.
1453	*/
1454	u64NanoTS = RTTimeSystemNanoTS() - pGip->u32UpdateIntervalNS;
1455	i = supdrvGipFindOrAllocCpuIndexForCpuId(pGip, idCpu);
1456
1457	supdrvGipInitCpu(pGip, &pGip->aCPUs[i], u64NanoTS, pGip->u64CpuHz);
1458
1459	idApic = supdrvGipGetApicIdSlow();
1460	ASMAtomicWriteU16(&pGip->aCPUs[i].idApic, idApic);
1461	ASMAtomicWriteS16(&pGip->aCPUs[i].iCpuSet, (int16_t)iCpuSet);
1462	ASMAtomicWriteSize(&pGip->aCPUs[i].idCpu, idCpu);
1463
1464	pGip->aCPUs[i].iCpuGroup = 0;
1465	pGip->aCPUs[i].iCpuGroupMember = iCpuSet;
1466	#ifdef RT_OS_WINDOWS
1467	supdrvOSGipInitGroupBitsForCpu(pDevExt, pGip, &pGip->aCPUs[i]);
1468	#endif
1469
1470	/*
1471	* Update the APIC ID and CPU set index mappings.
1472	*/
1473	if (idApic < RT_ELEMENTS(pGip->aiCpuFromApicId))
1474	ASMAtomicWriteU16(&pGip->aiCpuFromApicId[idApic], i);
1475	else
1476	LogRelMax(64, ("supdrvGipMpEventOnlineOrInitOnCpu: idApic=%#x is out of bounds (%#zx, i=%u, iCpuSet=%d)\n",
1477	idApic, RT_ELEMENTS(pGip->aiCpuFromApicId), i, iCpuSet));
1478	if ((unsigned)iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx))
1479	ASMAtomicWriteU16(&pGip->aiCpuFromCpuSetIdx[iCpuSet], i);
1480	else
1481	LogRelMax(64, ("supdrvGipMpEventOnlineOrInitOnCpu: iCpuSet=%d is out of bounds (%#zx, i=%u, idApic=%d)\n",
1482	iCpuSet, RT_ELEMENTS(pGip->aiCpuFromApicId), i, idApic));
1483
1484	/* Add this CPU to this set of CPUs we need to calculate the TSC-delta for. */
1485	RTCpuSetAddByIndex(&pDevExt->TscDeltaCpuSet, RTMpCpuIdToSetIndex(idCpu));
1486
1487	/* Update the Mp online/offline counter. */
1488	ASMAtomicIncU32(&pDevExt->cMpOnOffEvents);
1489
1490	/* Commit it. */
1491	ASMAtomicWriteSize(&pGip->aCPUs[i].enmState, SUPGIPCPUSTATE_ONLINE);
1492
1493	RTSpinlockRelease(pDevExt->hGipSpinlock);
1494	}
1495
1496
1497	/**
1498	* RTMpOnSpecific callback wrapper for supdrvGipMpEventOnlineOrInitOnCpu().
1499	*
1500	* @param idCpu The CPU ID we are running on.
1501	* @param pvUser1 Opaque pointer to the device instance data.
1502	* @param pvUser2 Not used.
1503	*/
1504	static DECLCALLBACK(void) supdrvGipMpEventOnlineCallback(RTCPUID idCpu, void pvUser1, void pvUser2)
1505	{
1506	PSUPDRVDEVEXT pDevExt = (PSUPDRVDEVEXT)pvUser1;
1507	NOREF(pvUser2);
1508	supdrvGipMpEventOnlineOrInitOnCpu(pDevExt, idCpu);
1509	}
1510
1511
1512	/**
1513	* The CPU should be accounted as offline, update the GIP accordingly.
1514	*
1515	* This is used by supdrvGipMpEvent.
1516	*
1517	* @param pDevExt The device extension.
1518	* @param idCpu The CPU ID.
1519	*/
1520	static void supdrvGipMpEventOffline(PSUPDRVDEVEXT pDevExt, RTCPUID idCpu)
1521	{
1522	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
1523	int iCpuSet;
1524	unsigned i;
1525
1526	AssertPtrReturnVoid(pGip);
1527	RTSpinlockAcquire(pDevExt->hGipSpinlock);
1528
1529	iCpuSet = RTMpCpuIdToSetIndex(idCpu);
1530	AssertReturnVoid(iCpuSet >= 0);
1531
1532	i = pGip->aiCpuFromCpuSetIdx[iCpuSet];
1533	AssertReturnVoid(i < pGip->cCpus);
1534	AssertReturnVoid(pGip->aCPUs[i].idCpu == idCpu);
1535
1536	Assert(RTCpuSetIsMemberByIndex(&pGip->PossibleCpuSet, iCpuSet));
1537	RTCpuSetDelByIndex(&pGip->OnlineCpuSet, iCpuSet);
1538
1539	/* Update the Mp online/offline counter. */
1540	ASMAtomicIncU32(&pDevExt->cMpOnOffEvents);
1541
1542	if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
1543	{
1544	/* Reset the TSC delta, we will recalculate it lazily. */
1545	ASMAtomicWriteS64(&pGip->aCPUs[i].i64TSCDelta, INT64_MAX);
1546	/* Remove this CPU from the set of CPUs that we have obtained the TSC deltas. */
1547	RTCpuSetDelByIndex(&pDevExt->TscDeltaObtainedCpuSet, iCpuSet);
1548	}
1549
1550	/* Commit it. */
1551	ASMAtomicWriteSize(&pGip->aCPUs[i].enmState, SUPGIPCPUSTATE_OFFLINE);
1552
1553	RTSpinlockRelease(pDevExt->hGipSpinlock);
1554	}
1555
1556
1557	/**
1558	* Multiprocessor event notification callback.
1559	*
1560	* This is used to make sure that the GIP master gets passed on to
1561	* another CPU. It also updates the associated CPU data.
1562	*
1563	* @param enmEvent The event.
1564	* @param idCpu The cpu it applies to.
1565	* @param pvUser Pointer to the device extension.
1566	*/
1567	static DECLCALLBACK(void) supdrvGipMpEvent(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvUser)
1568	{
1569	PSUPDRVDEVEXT pDevExt = (PSUPDRVDEVEXT)pvUser;
1570	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
1571
1572	if (pGip)
1573	{
1574	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
1575	switch (enmEvent)
1576	{
1577	case RTMPEVENT_ONLINE:
1578	{
1579	RTThreadPreemptDisable(&PreemptState);
1580	if (idCpu == RTMpCpuId())
1581	{
1582	supdrvGipMpEventOnlineOrInitOnCpu(pDevExt, idCpu);
1583	RTThreadPreemptRestore(&PreemptState);
1584	}
1585	else
1586	{
1587	RTThreadPreemptRestore(&PreemptState);
1588	RTMpOnSpecific(idCpu, supdrvGipMpEventOnlineCallback, pDevExt, NULL /* pvUser2 */);
1589	}
1590
1591	/*
1592	* Recompute TSC-delta for the newly online'd CPU.
1593	*/
1594	if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
1595	{
1596	#ifdef SUPDRV_USE_TSC_DELTA_THREAD
1597	supdrvTscDeltaThreadStartMeasurement(pDevExt, false /* fForceAll */);
1598	#else
1599	uint32_t iCpu = supdrvGipFindOrAllocCpuIndexForCpuId(pGip, idCpu);
1600	supdrvTscMeasureDeltaOne(pDevExt, iCpu);
1601	#endif
1602	}
1603	break;
1604	}
1605
1606	case RTMPEVENT_OFFLINE:
1607	supdrvGipMpEventOffline(pDevExt, idCpu);
1608	break;
1609	}
1610	}
1611
1612	/*
1613	* Make sure there is a master GIP.
1614	*/
1615	if (enmEvent == RTMPEVENT_OFFLINE)
1616	{
1617	RTCPUID idGipMaster = ASMAtomicReadU32(&pDevExt->idGipMaster);
1618	if (idGipMaster == idCpu)
1619	{
1620	/*
1621	* The GIP master is going offline, find a new one.
1622	*/
1623	bool fIgnored;
1624	unsigned i;
1625	RTCPUID idNewGipMaster = NIL_RTCPUID;
1626	RTCPUSET OnlineCpus;
1627	RTMpGetOnlineSet(&OnlineCpus);
1628
1629	for (i = 0; i < RTCPUSET_MAX_CPUS; i++)
1630	if (RTCpuSetIsMemberByIndex(&OnlineCpus, i))
1631	{
1632	RTCPUID idCurCpu = RTMpCpuIdFromSetIndex(i);
1633	if (idCurCpu != idGipMaster)
1634	{
1635	idNewGipMaster = idCurCpu;
1636	break;
1637	}
1638	}
1639
1640	Log(("supdrvGipMpEvent: Gip master %#lx -> %#lx\n", (long)idGipMaster, (long)idNewGipMaster));
1641	ASMAtomicCmpXchgSize(&pDevExt->idGipMaster, idNewGipMaster, idGipMaster, fIgnored);
1642	NOREF(fIgnored);
1643	}
1644	}
1645	}
1646
1647
1648	/**
1649	* On CPU initialization callback for RTMpOnAll.
1650	*
1651	* @param idCpu The CPU ID.
1652	* @param pvUser1 The device extension.
1653	* @param pvUser2 The GIP.
1654	*/
1655	static DECLCALLBACK(void) supdrvGipInitOnCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
1656	{
1657	/* This is good enough, even though it will update some of the globals a
1658	bit to much. */
1659	supdrvGipMpEventOnlineOrInitOnCpu((PSUPDRVDEVEXT)pvUser1, idCpu);
1660	NOREF(pvUser2);
1661	}
1662
1663
1664	/**
1665	* Callback used by supdrvDetermineAsyncTSC to read the TSC on a CPU.
1666	*
1667	* @param idCpu Ignored.
1668	* @param pvUser1 Where to put the TSC.
1669	* @param pvUser2 Ignored.
1670	*/
1671	static DECLCALLBACK(void) supdrvGipInitDetermineAsyncTscWorker(RTCPUID idCpu, void pvUser1, void pvUser2)
1672	{
1673	Assert(RTMpCpuIdToSetIndex(idCpu) == (intptr_t)pvUser2);
1674	ASMAtomicWriteU64((uint64_t volatile *)pvUser1, ASMReadTSC());
1675	RT_NOREF2(idCpu, pvUser2);
1676	}
1677
1678
1679	/**
1680	* Determine if Async GIP mode is required because of TSC drift.
1681	*
1682	* When using the default/normal timer code it is essential that the time stamp counter
1683	* (TSC) runs never backwards, that is, a read operation to the counter should return
1684	* a bigger value than any previous read operation. This is guaranteed by the latest
1685	* AMD CPUs and by newer Intel CPUs which never enter the C2 state (P4). In any other
1686	* case we have to choose the asynchronous timer mode.
1687	*
1688	* @param poffMin Pointer to the determined difference between different
1689	* cores (optional, can be NULL).
1690	* @return false if the time stamp counters appear to be synchronized, true otherwise.
1691	*/
1692	static bool supdrvGipInitDetermineAsyncTsc(uint64_t *poffMin)
1693	{
1694	/*
1695	* Just iterate all the cpus 8 times and make sure that the TSC is
1696	* ever increasing. We don't bother taking TSC rollover into account.
1697	*/
1698	int iEndCpu = RTMpGetArraySize();
1699	int iCpu;
1700	int cLoops = 8;
1701	bool fAsync = false;
1702	int rc = VINF_SUCCESS;
1703	uint64_t offMax = 0;
1704	uint64_t offMin = ~(uint64_t)0;
1705	uint64_t PrevTsc = ASMReadTSC();
1706
1707	while (cLoops-- > 0)
1708	{
1709	for (iCpu = 0; iCpu < iEndCpu; iCpu++)
1710	{
1711	uint64_t CurTsc;
1712	rc = RTMpOnSpecific(RTMpCpuIdFromSetIndex(iCpu), supdrvGipInitDetermineAsyncTscWorker,
1713	&CurTsc, (void *)(uintptr_t)iCpu);
1714	if (RT_SUCCESS(rc))
1715	{
1716	if (CurTsc <= PrevTsc)
1717	{
1718	fAsync = true;
1719	offMin = offMax = PrevTsc - CurTsc;
1720	Log(("supdrvGipInitDetermineAsyncTsc: iCpu=%d cLoops=%d CurTsc=%llx PrevTsc=%llx\n",
1721	iCpu, cLoops, CurTsc, PrevTsc));
1722	break;
1723	}
1724
1725	/* Gather statistics (except the first time). */
1726	if (iCpu != 0 \|\| cLoops != 7)
1727	{
1728	uint64_t off = CurTsc - PrevTsc;
1729	if (off < offMin)
1730	offMin = off;
1731	if (off > offMax)
1732	offMax = off;
1733	Log2(("%d/%d: off=%llx\n", cLoops, iCpu, off));
1734	}
1735
1736	/* Next */
1737	PrevTsc = CurTsc;
1738	}
1739	else if (rc == VERR_NOT_SUPPORTED)
1740	break;
1741	else
1742	AssertMsg(rc == VERR_CPU_NOT_FOUND \|\| rc == VERR_CPU_OFFLINE, ("%d\n", rc));
1743	}
1744
1745	/* broke out of the loop. */
1746	if (iCpu < iEndCpu)
1747	break;
1748	}
1749
1750	if (poffMin)
1751	poffMin = offMin; / Almost RTMpOnSpecific profiling. */
1752	Log(("supdrvGipInitDetermineAsyncTsc: returns %d; iEndCpu=%d rc=%d offMin=%llx offMax=%llx\n",
1753	fAsync, iEndCpu, rc, offMin, offMax));
1754	#if !defined(RT_OS_SOLARIS) && !defined(RT_OS_OS2) && !defined(RT_OS_WINDOWS)
1755	OSDBGPRINT(("vboxdrv: fAsync=%d offMin=%#lx offMax=%#lx\n", fAsync, (long)offMin, (long)offMax));
1756	#endif
1757	return fAsync;
1758	}
1759
1760
1761	/**
1762	* supdrvGipInit() worker that determines the GIP TSC mode.
1763	*
1764	* @returns The most suitable TSC mode.
1765	* @param pDevExt Pointer to the device instance data.
1766	*/
1767	static SUPGIPMODE supdrvGipInitDetermineTscMode(PSUPDRVDEVEXT pDevExt)
1768	{
1769	uint64_t u64DiffCoresIgnored;
1770	uint32_t uEAX, uEBX, uECX, uEDX;
1771
1772	/*
1773	* Establish whether the CPU advertises TSC as invariant, we need that in
1774	* a couple of places below.
1775	*/
1776	bool fInvariantTsc = false;
1777	if (ASMHasCpuId())
1778	{
1779	uEAX = ASMCpuId_EAX(0x80000000);
1780	if (RTX86IsValidExtRange(uEAX) && uEAX >= 0x80000007)
1781	{
1782	uEDX = ASMCpuId_EDX(0x80000007);
1783	if (uEDX & X86_CPUID_AMD_ADVPOWER_EDX_TSCINVAR)
1784	fInvariantTsc = true;
1785	}
1786	}
1787
1788	/*
1789	* On single CPU systems, we don't need to consider ASYNC mode.
1790	*/
1791	if (RTMpGetCount() <= 1)
1792	return fInvariantTsc ? SUPGIPMODE_INVARIANT_TSC : SUPGIPMODE_SYNC_TSC;
1793
1794	/*
1795	* Allow the user and/or OS specific bits to force async mode.
1796	*/
1797	if (supdrvOSGetForcedAsyncTscMode(pDevExt))
1798	return SUPGIPMODE_ASYNC_TSC;
1799
1800	/*
1801	* Use invariant mode if the CPU says TSC is invariant.
1802	*/
1803	if (fInvariantTsc)
1804	return SUPGIPMODE_INVARIANT_TSC;
1805
1806	/*
1807	* TSC is not invariant and we're on SMP, this presents two problems:
1808	*
1809	* (1) There might be a skew between the CPU, so that cpu0
1810	* returns a TSC that is slightly different from cpu1.
1811	* This screw may be due to (2), bad TSC initialization
1812	* or slightly different TSC rates.
1813	*
1814	* (2) Power management (and other things) may cause the TSC
1815	* to run at a non-constant speed, and cause the speed
1816	* to be different on the cpus. This will result in (1).
1817	*
1818	* If any of the above is detected, we will have to use ASYNC mode.
1819	*/
1820	/* (1). Try check for current differences between the cpus. */
1821	if (supdrvGipInitDetermineAsyncTsc(&u64DiffCoresIgnored))
1822	return SUPGIPMODE_ASYNC_TSC;
1823
1824	/* (2) If it's an AMD CPU with power management, we won't trust its TSC. */
1825	ASMCpuId(0, &uEAX, &uEBX, &uECX, &uEDX);
1826	if ( RTX86IsValidStdRange(uEAX)
1827	&& (RTX86IsAmdCpu(uEBX, uECX, uEDX) \|\| RTX86IsHygonCpu(uEBX, uECX, uEDX)) )
1828	{
1829	/* Check for APM support. */
1830	uEAX = ASMCpuId_EAX(0x80000000);
1831	if (RTX86IsValidExtRange(uEAX) && uEAX >= 0x80000007)
1832	{
1833	uEDX = ASMCpuId_EDX(0x80000007);
1834	if (uEDX & 0x3e) /* STC\|TM\|THERMTRIP\|VID\|FID. Ignore TS. */
1835	return SUPGIPMODE_ASYNC_TSC;
1836	}
1837	}
1838
1839	return SUPGIPMODE_SYNC_TSC;
1840	}
1841
1842
1843	/**
1844	* Initializes per-CPU GIP information.
1845	*
1846	* @param pGip Pointer to the GIP.
1847	* @param pCpu Pointer to which GIP CPU to initialize.
1848	* @param u64NanoTS The current nanosecond timestamp.
1849	* @param uCpuHz The CPU frequency to set, 0 if the caller doesn't know.
1850	*/
1851	static void supdrvGipInitCpu(PSUPGLOBALINFOPAGE pGip, PSUPGIPCPU pCpu, uint64_t u64NanoTS, uint64_t uCpuHz)
1852	{
1853	pCpu->u32TransactionId = 2;
1854	pCpu->u64NanoTS = u64NanoTS;
1855	pCpu->u64TSC = ASMReadTSC();
1856	pCpu->u64TSCSample = GIP_TSC_DELTA_RSVD;
1857	pCpu->i64TSCDelta = pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED ? INT64_MAX : 0;
1858
1859	ASMAtomicWriteSize(&pCpu->enmState, SUPGIPCPUSTATE_INVALID);
1860	ASMAtomicWriteU32(&pCpu->idCpu, NIL_RTCPUID);
1861	ASMAtomicWriteS16(&pCpu->iCpuSet, -1);
1862	ASMAtomicWriteU16(&pCpu->iCpuGroup, 0);
1863	ASMAtomicWriteU16(&pCpu->iCpuGroupMember, UINT16_MAX);
1864	ASMAtomicWriteU16(&pCpu->idApic, UINT16_MAX);
1865	ASMAtomicWriteU32(&pCpu->iReservedForNumaNode, 0);
1866
1867	/*
1868	* The first time we're called, we don't have a CPU frequency handy,
1869	* so pretend it's a 4 GHz CPU. On CPUs that are online, we'll get
1870	* called again and at that point we have a more plausible CPU frequency
1871	* value handy. The frequency history will also be adjusted again on
1872	* the 2nd timer callout (maybe we can skip that now?).
1873	*/
1874	if (!uCpuHz)
1875	{
1876	pCpu->u64CpuHz = _4G - 1;
1877	pCpu->u32UpdateIntervalTSC = (uint32_t)((_4G - 1) / pGip->u32UpdateHz);
1878	}
1879	else
1880	{
1881	pCpu->u64CpuHz = uCpuHz;
1882	pCpu->u32UpdateIntervalTSC = (uint32_t)(uCpuHz / pGip->u32UpdateHz);
1883	}
1884	pCpu->au32TSCHistory[0]
1885	= pCpu->au32TSCHistory[1]
1886	= pCpu->au32TSCHistory[2]
1887	= pCpu->au32TSCHistory[3]
1888	= pCpu->au32TSCHistory[4]
1889	= pCpu->au32TSCHistory[5]
1890	= pCpu->au32TSCHistory[6]
1891	= pCpu->au32TSCHistory[7]
1892	= pCpu->u32UpdateIntervalTSC;
1893	}
1894
1895
1896	/**
1897	* Initializes the GIP data.
1898	*
1899	* @returns VBox status code.
1900	* @param pDevExt Pointer to the device instance data.
1901	* @param pGip Pointer to the read-write kernel mapping of the GIP.
1902	* @param HCPhys The physical address of the GIP.
1903	* @param u64NanoTS The current nanosecond timestamp.
1904	* @param uUpdateHz The update frequency.
1905	* @param uUpdateIntervalNS The update interval in nanoseconds.
1906	* @param cCpus The CPU count.
1907	* @param cbGipCpuGroups The supdrvOSGipGetGroupTableSize return value we
1908	* used when allocating the GIP structure.
1909	*/
1910	static int supdrvGipInit(PSUPDRVDEVEXT pDevExt, PSUPGLOBALINFOPAGE pGip, RTHCPHYS HCPhys,
1911	uint64_t u64NanoTS, unsigned uUpdateHz, unsigned uUpdateIntervalNS,
1912	unsigned cCpus, size_t cbGipCpuGroups)
1913	{
1914	size_t const cbGip = RT_ALIGN_Z(RT_UOFFSETOF_DYN(SUPGLOBALINFOPAGE, aCPUs[cCpus]) + cbGipCpuGroups, PAGE_SIZE);
1915	unsigned i;
1916	#ifdef DEBUG_DARWIN_GIP
1917	OSDBGPRINT(("supdrvGipInit: pGip=%p HCPhys=%lx u64NanoTS=%llu uUpdateHz=%d cCpus=%u\n", pGip, (long)HCPhys, u64NanoTS, uUpdateHz, cCpus));
1918	#else
1919	LogFlow(("supdrvGipInit: pGip=%p HCPhys=%lx u64NanoTS=%llu uUpdateHz=%d cCpus=%u\n", pGip, (long)HCPhys, u64NanoTS, uUpdateHz, cCpus));
1920	#endif
1921
1922	/*
1923	* Initialize the structure.
1924	*/
1925	memset(pGip, 0, cbGip);
1926
1927	pGip->u32Magic = SUPGLOBALINFOPAGE_MAGIC;
1928	pGip->u32Version = SUPGLOBALINFOPAGE_VERSION;
1929	pGip->u32Mode = supdrvGipInitDetermineTscMode(pDevExt);
1930	if ( pGip->u32Mode == SUPGIPMODE_INVARIANT_TSC
1931	/\|\| pGip->u32Mode == SUPGIPMODE_SYNC_TSC /)
1932	pGip->enmUseTscDelta = supdrvOSAreTscDeltasInSync() /* Allow OS override (windows). */
1933	? SUPGIPUSETSCDELTA_ZERO_CLAIMED : SUPGIPUSETSCDELTA_PRACTICALLY_ZERO /* downgrade later */;
1934	else
1935	pGip->enmUseTscDelta = SUPGIPUSETSCDELTA_NOT_APPLICABLE;
1936	pGip->cCpus = (uint16_t)cCpus;
1937	pGip->cPages = (uint16_t)(cbGip / PAGE_SIZE);
1938	pGip->u32UpdateHz = uUpdateHz;
1939	pGip->u32UpdateIntervalNS = uUpdateIntervalNS;
1940	pGip->fGetGipCpu = SUPGIPGETCPU_APIC_ID;
1941	RTCpuSetEmpty(&pGip->OnlineCpuSet);
1942	RTCpuSetEmpty(&pGip->PresentCpuSet);
1943	RTMpGetSet(&pGip->PossibleCpuSet);
1944	pGip->cOnlineCpus = RTMpGetOnlineCount();
1945	pGip->cPresentCpus = RTMpGetPresentCount();
1946	pGip->cPossibleCpus = RTMpGetCount();
1947	pGip->cPossibleCpuGroups = 1;
1948	pGip->idCpuMax = RTMpGetMaxCpuId();
1949	for (i = 0; i < RT_ELEMENTS(pGip->aiCpuFromApicId); i++)
1950	pGip->aiCpuFromApicId[i] = UINT16_MAX;
1951	for (i = 0; i < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx); i++)
1952	pGip->aiCpuFromCpuSetIdx[i] = UINT16_MAX;
1953	for (i = 0; i < RT_ELEMENTS(pGip->aoffCpuGroup); i++)
1954	pGip->aoffCpuGroup[i] = UINT32_MAX;
1955	for (i = 0; i < cCpus; i++)
1956	supdrvGipInitCpu(pGip, &pGip->aCPUs[i], u64NanoTS, 0 /uCpuHz/);
1957	#ifdef RT_OS_WINDOWS
1958	int rc = supdrvOSInitGipGroupTable(pDevExt, pGip, cbGipCpuGroups);
1959	AssertRCReturn(rc, rc);
1960	#endif
1961
1962	/*
1963	* Link it to the device extension.
1964	*/
1965	pDevExt->pGip = pGip;
1966	pDevExt->HCPhysGip = HCPhys;
1967	pDevExt->cGipUsers = 0;
1968
1969	return VINF_SUCCESS;
1970	}
1971
1972
1973	/**
1974	* Creates the GIP.
1975	*
1976	* @returns VBox status code.
1977	* @param pDevExt Instance data. GIP stuff may be updated.
1978	*/
1979	int VBOXCALL supdrvGipCreate(PSUPDRVDEVEXT pDevExt)
1980	{
1981	PSUPGLOBALINFOPAGE pGip;
1982	size_t cbGip;
1983	size_t cbGipCpuGroups;
1984	RTHCPHYS HCPhysGip;
1985	uint32_t u32SystemResolution;
1986	uint32_t u32Interval;
1987	uint32_t u32MinInterval;
1988	uint32_t uMod;
1989	unsigned cCpus;
1990	int rc;
1991
1992	LogFlow(("supdrvGipCreate:\n"));
1993
1994	/*
1995	* Assert order.
1996	*/
1997	Assert(pDevExt->u32SystemTimerGranularityGrant == 0);
1998	Assert(pDevExt->GipMemObj == NIL_RTR0MEMOBJ);
1999	Assert(!pDevExt->pGipTimer);
2000	#ifdef SUPDRV_USE_MUTEX_FOR_GIP
2001	Assert(pDevExt->mtxGip != NIL_RTSEMMUTEX);
2002	Assert(pDevExt->mtxTscDelta != NIL_RTSEMMUTEX);
2003	#else
2004	Assert(pDevExt->mtxGip != NIL_RTSEMFASTMUTEX);
2005	Assert(pDevExt->mtxTscDelta != NIL_RTSEMFASTMUTEX);
2006	#endif
2007
2008	/*
2009	* Check the CPU count.
2010	*/
2011	cCpus = RTMpGetArraySize();
2012	if (cCpus > RT_MIN(RTCPUSET_MAX_CPUS, RT_ELEMENTS(pGip->aiCpuFromApicId)))
2013	{
2014	SUPR0Printf("VBoxDrv: Too many CPUs (%u) for the GIP (max %u)\n", cCpus, RT_MIN(RTCPUSET_MAX_CPUS, RT_ELEMENTS(pGip->aiCpuFromApicId)));
2015	return VERR_TOO_MANY_CPUS;
2016	}
2017
2018	/*
2019	* Allocate a contiguous set of pages with a default kernel mapping.
2020	*/
2021	#ifdef RT_OS_WINDOWS
2022	cbGipCpuGroups = supdrvOSGipGetGroupTableSize(pDevExt);
2023	#else
2024	cbGipCpuGroups = 0;
2025	#endif
2026	cbGip = RT_UOFFSETOF_DYN(SUPGLOBALINFOPAGE, aCPUs[cCpus]) + cbGipCpuGroups;
2027	rc = RTR0MemObjAllocCont(&pDevExt->GipMemObj, cbGip, false /fExecutable/);
2028	if (RT_FAILURE(rc))
2029	{
2030	OSDBGPRINT(("supdrvGipCreate: failed to allocate the GIP page. rc=%d\n", rc));
2031	return rc;
2032	}
2033	pGip = (PSUPGLOBALINFOPAGE)RTR0MemObjAddress(pDevExt->GipMemObj); AssertPtr(pGip);
2034	HCPhysGip = RTR0MemObjGetPagePhysAddr(pDevExt->GipMemObj, 0); Assert(HCPhysGip != NIL_RTHCPHYS);
2035
2036	/*
2037	* Find a reasonable update interval and initialize the structure.
2038	*/
2039	supdrvGipRequestHigherTimerFrequencyFromSystem(pDevExt);
2040	/** @todo figure out why using a 100Ms interval upsets timekeeping in VMs.
2041	* See @bugref{6710}. */
2042	u32MinInterval = RT_NS_10MS;
2043	u32SystemResolution = RTTimerGetSystemGranularity();
2044	u32Interval = u32MinInterval;
2045	uMod = u32MinInterval % u32SystemResolution;
2046	if (uMod)
2047	u32Interval += u32SystemResolution - uMod;
2048
2049	rc = supdrvGipInit(pDevExt, pGip, HCPhysGip, RTTimeSystemNanoTS(), RT_NS_1SEC / u32Interval /=Hz/, u32Interval,
2050	cCpus, cbGipCpuGroups);
2051
2052	/*
2053	* Important sanity check... (Sets rc)
2054	*/
2055	if (RT_UNLIKELY( pGip->enmUseTscDelta == SUPGIPUSETSCDELTA_ZERO_CLAIMED
2056	&& pGip->u32Mode == SUPGIPMODE_ASYNC_TSC
2057	&& !supdrvOSGetForcedAsyncTscMode(pDevExt)))
2058	{
2059	OSDBGPRINT(("supdrvGipCreate: Host-OS/user claims the TSC-deltas are zero but we detected async. TSC! Bad.\n"));
2060	rc = VERR_INTERNAL_ERROR_2;
2061	}
2062
2063	/* It doesn't make sense to do TSC-delta detection on systems we detect as async. */
2064	AssertStmt( pGip->u32Mode != SUPGIPMODE_ASYNC_TSC
2065	\|\| pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_ZERO_CLAIMED,
2066	rc = VERR_INTERNAL_ERROR_3);
2067
2068	/*
2069	* Do the TSC frequency measurements.
2070	*
2071	* If we're in invariant TSC mode, just to a quick preliminary measurement
2072	* that the TSC-delta measurement code can use to yield cross calls.
2073	*
2074	* If we're in any of the other two modes, neither which require MP init,
2075	* notifications or deltas for the job, do the full measurement now so
2076	* that supdrvGipInitOnCpu() can populate the TSC interval and history
2077	* array with more reasonable values.
2078	*/
2079	if (RT_SUCCESS(rc))
2080	{
2081	if (pGip->u32Mode == SUPGIPMODE_INVARIANT_TSC)
2082	{
2083	rc = supdrvGipInitMeasureTscFreq(pGip, true /fRough/); /* cannot fail */
2084	supdrvGipInitStartTimerForRefiningInvariantTscFreq(pDevExt);
2085	}
2086	else
2087	rc = supdrvGipInitMeasureTscFreq(pGip, false /fRough/);
2088	if (RT_SUCCESS(rc))
2089	{
2090	/*
2091	* Start TSC-delta measurement thread before we start getting MP
2092	* events that will try kick it into action (includes the
2093	* RTMpOnAll/supdrvGipInitOnCpu call below).
2094	*/
2095	RTCpuSetEmpty(&pDevExt->TscDeltaCpuSet);
2096	RTCpuSetEmpty(&pDevExt->TscDeltaObtainedCpuSet);
2097	#ifdef SUPDRV_USE_TSC_DELTA_THREAD
2098	if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
2099	rc = supdrvTscDeltaThreadInit(pDevExt);
2100	#endif
2101	if (RT_SUCCESS(rc))
2102	{
2103	rc = RTMpNotificationRegister(supdrvGipMpEvent, pDevExt);
2104	if (RT_SUCCESS(rc))
2105	{
2106	/*
2107	* Do GIP initialization on all online CPUs. Wake up the
2108	* TSC-delta thread afterwards.
2109	*/
2110	rc = RTMpOnAll(supdrvGipInitOnCpu, pDevExt, pGip);
2111	if (RT_SUCCESS(rc))
2112	{
2113	#ifdef SUPDRV_USE_TSC_DELTA_THREAD
2114	supdrvTscDeltaThreadStartMeasurement(pDevExt, true /* fForceAll */);
2115	#else
2116	uint16_t iCpu;
2117	if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
2118	{
2119	/*
2120	* Measure the TSC deltas now that we have MP notifications.
2121	*/
2122	int cTries = 5;
2123	do
2124	{
2125	rc = supdrvTscMeasureInitialDeltas(pDevExt);
2126	if ( rc != VERR_TRY_AGAIN
2127	&& rc != VERR_CPU_OFFLINE)
2128	break;
2129	} while (--cTries > 0);
2130	for (iCpu = 0; iCpu < pGip->cCpus; iCpu++)
2131	Log(("supdrvTscDeltaInit: cpu[%u] delta %lld\n", iCpu, pGip->aCPUs[iCpu].i64TSCDelta));
2132	}
2133	else
2134	{
2135	for (iCpu = 0; iCpu < pGip->cCpus; iCpu++)
2136	AssertMsg(!pGip->aCPUs[iCpu].i64TSCDelta, ("iCpu=%u %lld mode=%d\n", iCpu, pGip->aCPUs[iCpu].i64TSCDelta, pGip->u32Mode));
2137	}
2138	if (RT_SUCCESS(rc))
2139	#endif
2140	{
2141	/*
2142	* Create the timer.
2143	* If CPU_ALL isn't supported we'll have to fall back to synchronous mode.
2144	*/
2145	if (pGip->u32Mode == SUPGIPMODE_ASYNC_TSC)
2146	{
2147	rc = RTTimerCreateEx(&pDevExt->pGipTimer, u32Interval, RTTIMER_FLAGS_CPU_ALL,
2148	supdrvGipAsyncTimer, pDevExt);
2149	if (rc == VERR_NOT_SUPPORTED)
2150	{
2151	OSDBGPRINT(("supdrvGipCreate: omni timer not supported, falling back to synchronous mode\n"));
2152	pGip->u32Mode = SUPGIPMODE_SYNC_TSC;
2153	}
2154	}
2155	if (pGip->u32Mode != SUPGIPMODE_ASYNC_TSC)
2156	rc = RTTimerCreateEx(&pDevExt->pGipTimer, u32Interval, 0 /* fFlags */,
2157	supdrvGipSyncAndInvariantTimer, pDevExt);
2158	if (RT_SUCCESS(rc))
2159	{
2160	/*
2161	* We're good.
2162	*/
2163	Log(("supdrvGipCreate: %u ns interval.\n", u32Interval));
2164	supdrvGipReleaseHigherTimerFrequencyFromSystem(pDevExt);
2165
2166	g_pSUPGlobalInfoPage = pGip;
2167	return VINF_SUCCESS;
2168	}
2169
2170	OSDBGPRINT(("supdrvGipCreate: failed create GIP timer at %u ns interval. rc=%Rrc\n", u32Interval, rc));
2171	Assert(!pDevExt->pGipTimer);
2172	}
2173	}
2174	else
2175	OSDBGPRINT(("supdrvGipCreate: RTMpOnAll failed. rc=%Rrc\n", rc));
2176	}
2177	else
2178	OSDBGPRINT(("supdrvGipCreate: failed to register MP event notfication. rc=%Rrc\n", rc));
2179	}
2180	else
2181	OSDBGPRINT(("supdrvGipCreate: supdrvTscDeltaInit failed. rc=%Rrc\n", rc));
2182	}
2183	else
2184	OSDBGPRINT(("supdrvGipCreate: supdrvTscMeasureInitialDeltas failed. rc=%Rrc\n", rc));
2185	}
2186
2187	/* Releases timer frequency increase too. */
2188	supdrvGipDestroy(pDevExt);
2189	return rc;
2190	}
2191
2192
2193	/**
2194	* Invalidates the GIP data upon termination.
2195	*
2196	* @param pGip Pointer to the read-write kernel mapping of the GIP.
2197	*/
2198	static void supdrvGipTerm(PSUPGLOBALINFOPAGE pGip)
2199	{
2200	unsigned i;
2201	pGip->u32Magic = 0;
2202	for (i = 0; i < pGip->cCpus; i++)
2203	{
2204	pGip->aCPUs[i].u64NanoTS = 0;
2205	pGip->aCPUs[i].u64TSC = 0;
2206	pGip->aCPUs[i].iTSCHistoryHead = 0;
2207	pGip->aCPUs[i].u64TSCSample = 0;
2208	pGip->aCPUs[i].i64TSCDelta = INT64_MAX;
2209	}
2210	}
2211
2212
2213	/**
2214	* Terminates the GIP.
2215	*
2216	* @param pDevExt Instance data. GIP stuff may be updated.
2217	*/
2218	void VBOXCALL supdrvGipDestroy(PSUPDRVDEVEXT pDevExt)
2219	{
2220	int rc;
2221	#ifdef DEBUG_DARWIN_GIP
2222	OSDBGPRINT(("supdrvGipDestroy: pDevExt=%p pGip=%p pGipTimer=%p GipMemObj=%p\n", pDevExt,
2223	pDevExt->GipMemObj != NIL_RTR0MEMOBJ ? RTR0MemObjAddress(pDevExt->GipMemObj) : NULL,
2224	pDevExt->pGipTimer, pDevExt->GipMemObj));
2225	#endif
2226
2227	/*
2228	* Stop receiving MP notifications before tearing anything else down.
2229	*/
2230	RTMpNotificationDeregister(supdrvGipMpEvent, pDevExt);
2231
2232	#ifdef SUPDRV_USE_TSC_DELTA_THREAD
2233	/*
2234	* Terminate the TSC-delta measurement thread and resources.
2235	*/
2236	supdrvTscDeltaTerm(pDevExt);
2237	#endif
2238
2239	/*
2240	* Destroy the TSC-refinement timer.
2241	*/
2242	if (pDevExt->pInvarTscRefineTimer)
2243	{
2244	RTTimerDestroy(pDevExt->pInvarTscRefineTimer);
2245	pDevExt->pInvarTscRefineTimer = NULL;
2246	}
2247
2248	/*
2249	* Invalid the GIP data.
2250	*/
2251	if (pDevExt->pGip)
2252	{
2253	supdrvGipTerm(pDevExt->pGip);
2254	pDevExt->pGip = NULL;
2255	}
2256	g_pSUPGlobalInfoPage = NULL;
2257
2258	/*
2259	* Destroy the timer and free the GIP memory object.
2260	*/
2261	if (pDevExt->pGipTimer)
2262	{
2263	rc = RTTimerDestroy(pDevExt->pGipTimer); AssertRC(rc);
2264	pDevExt->pGipTimer = NULL;
2265	}
2266
2267	if (pDevExt->GipMemObj != NIL_RTR0MEMOBJ)
2268	{
2269	rc = RTR0MemObjFree(pDevExt->GipMemObj, true /* free mappings */); AssertRC(rc);
2270	pDevExt->GipMemObj = NIL_RTR0MEMOBJ;
2271	}
2272
2273	/*
2274	* Finally, make sure we've release the system timer resolution request
2275	* if one actually succeeded and is still pending.
2276	*/
2277	supdrvGipReleaseHigherTimerFrequencyFromSystem(pDevExt);
2278	}
2279
2280
2281
2282
2283	/*
2284	*
2285	*
2286	* GIP Update Timer Related Code
2287	* GIP Update Timer Related Code
2288	* GIP Update Timer Related Code
2289	*
2290	*
2291	*/
2292
2293
2294	/**
2295	* Worker routine for supdrvGipUpdate() and supdrvGipUpdatePerCpu() that
2296	* updates all the per cpu data except the transaction id.
2297	*
2298	* @param pDevExt The device extension.
2299	* @param pGipCpu Pointer to the per cpu data.
2300	* @param u64NanoTS The current time stamp.
2301	* @param u64TSC The current TSC.
2302	* @param iTick The current timer tick.
2303	*
2304	* @remarks Can be called with interrupts disabled!
2305	*/
2306	static void supdrvGipDoUpdateCpu(PSUPDRVDEVEXT pDevExt, PSUPGIPCPU pGipCpu, uint64_t u64NanoTS, uint64_t u64TSC, uint64_t iTick)
2307	{
2308	uint64_t u64TSCDelta;
2309	bool fUpdateCpuHz;
2310	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
2311	AssertPtrReturnVoid(pGip);
2312
2313	/* Delta between this and the previous update. */
2314	ASMAtomicUoWriteU32(&pGipCpu->u32PrevUpdateIntervalNS, (uint32_t)(u64NanoTS - pGipCpu->u64NanoTS));
2315
2316	/*
2317	* Update the NanoTS.
2318	*/
2319	ASMAtomicWriteU64(&pGipCpu->u64NanoTS, u64NanoTS);
2320
2321	/*
2322	* Calc TSC delta.
2323	*/
2324	u64TSCDelta = u64TSC - pGipCpu->u64TSC;
2325	ASMAtomicWriteU64(&pGipCpu->u64TSC, u64TSC);
2326
2327	/*
2328	* Determine if we need to update the CPU (TSC) frequency calculation.
2329	*
2330	* We don't need to keep recalculating the frequency when it's invariant,
2331	* unless the special tstGIP-2 testing mode is enabled.
2332	*/
2333	fUpdateCpuHz = pGip->u32Mode != SUPGIPMODE_INVARIANT_TSC;
2334	if (!(pGip->fFlags & SUPGIP_FLAGS_TESTING))
2335	{ /* likely*/ }
2336	else
2337	{
2338	uint32_t fGipFlags = pGip->fFlags;
2339	if (fGipFlags & (SUPGIP_FLAGS_TESTING_ENABLE \| SUPGIP_FLAGS_TESTING_START))
2340	{
2341	if (fGipFlags & SUPGIP_FLAGS_TESTING_START)
2342	{
2343	/* Cache the TSC frequency before forcing updates due to test mode. */
2344	if (!fUpdateCpuHz)
2345	pDevExt->uGipTestModeInvariantCpuHz = pGip->aCPUs[0].u64CpuHz;
2346	ASMAtomicAndU32(&pGip->fFlags, ~SUPGIP_FLAGS_TESTING_START);
2347	}
2348	fUpdateCpuHz = true;
2349	}
2350	else if (fGipFlags & SUPGIP_FLAGS_TESTING_STOP)
2351	{
2352	/* Restore the cached TSC frequency if any. */
2353	if (!fUpdateCpuHz)
2354	{
2355	Assert(pDevExt->uGipTestModeInvariantCpuHz);
2356	ASMAtomicWriteU64(&pGip->aCPUs[0].u64CpuHz, pDevExt->uGipTestModeInvariantCpuHz);
2357	}
2358	ASMAtomicAndU32(&pGip->fFlags, ~(SUPGIP_FLAGS_TESTING_STOP \| SUPGIP_FLAGS_TESTING));
2359	}
2360	}
2361
2362	/*
2363	* Calculate the CPU (TSC) frequency if necessary.
2364	*/
2365	if (fUpdateCpuHz)
2366	{
2367	uint64_t u64CpuHz;
2368	uint32_t u32UpdateIntervalTSC;
2369	uint32_t u32UpdateIntervalTSCSlack;
2370	uint32_t u32TransactionId;
2371	unsigned iTSCHistoryHead;
2372
2373	if (u64TSCDelta >> 32)
2374	{
2375	u64TSCDelta = pGipCpu->u32UpdateIntervalTSC;
2376	pGipCpu->cErrors++;
2377	}
2378
2379	/*
2380	* On the 2nd and 3rd callout, reset the history with the current TSC
2381	* interval since the values entered by supdrvGipInit are totally off.
2382	* The interval on the 1st callout completely unreliable, the 2nd is a bit
2383	* better, while the 3rd should be most reliable.
2384	*/
2385	/** @todo Could we drop this now that we initializes the history
2386	* with nominal TSC frequency values? */
2387	u32TransactionId = pGipCpu->u32TransactionId;
2388	if (RT_UNLIKELY( ( u32TransactionId == 5
2389	\|\| u32TransactionId == 7)
2390	&& ( iTick == 2
2391	\|\| iTick == 3) ))
2392	{
2393	unsigned i;
2394	for (i = 0; i < RT_ELEMENTS(pGipCpu->au32TSCHistory); i++)
2395	ASMAtomicUoWriteU32(&pGipCpu->au32TSCHistory[i], (uint32_t)u64TSCDelta);
2396	}
2397
2398	/*
2399	* Validate the NanoTS deltas between timer fires with an arbitrary threshold of 0.5%.
2400	* Wait until we have at least one full history since the above history reset. The
2401	* assumption is that the majority of the previous history values will be tolerable.
2402	* See @bugref{6710#c67}.
2403	*/
2404	/** @todo Could we drop the fudging there now that we initializes the history
2405	* with nominal TSC frequency values? */
2406	if ( u32TransactionId > 23 /* 7 + (8 * 2) */
2407	&& pGip->u32Mode != SUPGIPMODE_ASYNC_TSC)
2408	{
2409	uint32_t uNanoTsThreshold = pGip->u32UpdateIntervalNS / 200;
2410	if ( pGipCpu->u32PrevUpdateIntervalNS > pGip->u32UpdateIntervalNS + uNanoTsThreshold
2411	\|\| pGipCpu->u32PrevUpdateIntervalNS < pGip->u32UpdateIntervalNS - uNanoTsThreshold)
2412	{
2413	uint32_t u32;
2414	u32 = pGipCpu->au32TSCHistory[0];
2415	u32 += pGipCpu->au32TSCHistory[1];
2416	u32 += pGipCpu->au32TSCHistory[2];
2417	u32 += pGipCpu->au32TSCHistory[3];
2418	u32 >>= 2;
2419	u64TSCDelta = pGipCpu->au32TSCHistory[4];
2420	u64TSCDelta += pGipCpu->au32TSCHistory[5];
2421	u64TSCDelta += pGipCpu->au32TSCHistory[6];
2422	u64TSCDelta += pGipCpu->au32TSCHistory[7];
2423	u64TSCDelta >>= 2;
2424	u64TSCDelta += u32;
2425	u64TSCDelta >>= 1;
2426	}
2427	}
2428
2429	/*
2430	* TSC History.
2431	*/
2432	Assert(RT_ELEMENTS(pGipCpu->au32TSCHistory) == 8);
2433	iTSCHistoryHead = (pGipCpu->iTSCHistoryHead + 1) & 7;
2434	ASMAtomicWriteU32(&pGipCpu->iTSCHistoryHead, iTSCHistoryHead);
2435	ASMAtomicWriteU32(&pGipCpu->au32TSCHistory[iTSCHistoryHead], (uint32_t)u64TSCDelta);
2436
2437	/*
2438	* UpdateIntervalTSC = average of last 8,2,1 intervals depending on update HZ.
2439	*
2440	* On Windows, we have an occasional (but recurring) sour value that messed up
2441	* the history but taking only 1 interval reduces the precision overall.
2442	*/
2443	if ( pGip->u32Mode == SUPGIPMODE_INVARIANT_TSC
2444	\|\| pGip->u32UpdateHz >= 1000)
2445	{
2446	uint32_t u32;
2447	u32 = pGipCpu->au32TSCHistory[0];
2448	u32 += pGipCpu->au32TSCHistory[1];
2449	u32 += pGipCpu->au32TSCHistory[2];
2450	u32 += pGipCpu->au32TSCHistory[3];
2451	u32 >>= 2;
2452	u32UpdateIntervalTSC = pGipCpu->au32TSCHistory[4];
2453	u32UpdateIntervalTSC += pGipCpu->au32TSCHistory[5];
2454	u32UpdateIntervalTSC += pGipCpu->au32TSCHistory[6];
2455	u32UpdateIntervalTSC += pGipCpu->au32TSCHistory[7];
2456	u32UpdateIntervalTSC >>= 2;
2457	u32UpdateIntervalTSC += u32;
2458	u32UpdateIntervalTSC >>= 1;
2459
2460	/* Value chosen for a 2GHz Athlon64 running linux 2.6.10/11. */
2461	u32UpdateIntervalTSCSlack = u32UpdateIntervalTSC >> 14;
2462	}
2463	else if (pGip->u32UpdateHz >= 90)
2464	{
2465	u32UpdateIntervalTSC = (uint32_t)u64TSCDelta;
2466	u32UpdateIntervalTSC += pGipCpu->au32TSCHistory[(iTSCHistoryHead - 1) & 7];
2467	u32UpdateIntervalTSC >>= 1;
2468
2469	/* value chosen on a 2GHz thinkpad running windows */
2470	u32UpdateIntervalTSCSlack = u32UpdateIntervalTSC >> 7;
2471	}
2472	else
2473	{
2474	u32UpdateIntervalTSC = (uint32_t)u64TSCDelta;
2475
2476	/* This value hasn't be checked yet.. waiting for OS/2 and 33Hz timers.. :-) */
2477	u32UpdateIntervalTSCSlack = u32UpdateIntervalTSC >> 6;
2478	}
2479	ASMAtomicWriteU32(&pGipCpu->u32UpdateIntervalTSC, u32UpdateIntervalTSC + u32UpdateIntervalTSCSlack);
2480
2481	/*
2482	* CpuHz.
2483	*/
2484	u64CpuHz = ASMMult2xU32RetU64(u32UpdateIntervalTSC, RT_NS_1SEC);
2485	u64CpuHz /= pGip->u32UpdateIntervalNS;
2486	ASMAtomicWriteU64(&pGipCpu->u64CpuHz, u64CpuHz);
2487	}
2488	}
2489
2490
2491	/**
2492	* Updates the GIP.
2493	*
2494	* @param pDevExt The device extension.
2495	* @param u64NanoTS The current nanosecond timestamp.
2496	* @param u64TSC The current TSC timestamp.
2497	* @param idCpu The CPU ID.
2498	* @param iTick The current timer tick.
2499	*
2500	* @remarks Can be called with interrupts disabled!
2501	*/
2502	static void supdrvGipUpdate(PSUPDRVDEVEXT pDevExt, uint64_t u64NanoTS, uint64_t u64TSC, RTCPUID idCpu, uint64_t iTick)
2503	{
2504	/*
2505	* Determine the relevant CPU data.
2506	*/
2507	PSUPGIPCPU pGipCpu;
2508	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
2509	AssertPtrReturnVoid(pGip);
2510
2511	if (pGip->u32Mode != SUPGIPMODE_ASYNC_TSC)
2512	pGipCpu = &pGip->aCPUs[0];
2513	else
2514	{
2515	unsigned iCpu;
2516	uint32_t idApic = supdrvGipGetApicId(pGip);
2517	if (RT_LIKELY(idApic < RT_ELEMENTS(pGip->aiCpuFromApicId)))
2518	{ /* likely */ }
2519	else
2520	return;
2521	iCpu = pGip->aiCpuFromApicId[idApic];
2522	if (RT_LIKELY(iCpu < pGip->cCpus))
2523	{ /* likely */ }
2524	else
2525	return;
2526	pGipCpu = &pGip->aCPUs[iCpu];
2527	if (RT_LIKELY(pGipCpu->idCpu == idCpu))
2528	{ /* likely */ }
2529	else
2530	return;
2531	}
2532
2533	/*
2534	* Start update transaction.
2535	*/
2536	if (!(ASMAtomicIncU32(&pGipCpu->u32TransactionId) & 1))
2537	{
2538	/* this can happen on win32 if we're taking to long and there are more CPUs around. shouldn't happen though. */
2539	AssertMsgFailed(("Invalid transaction id, %#x, not odd!\n", pGipCpu->u32TransactionId));
2540	ASMAtomicIncU32(&pGipCpu->u32TransactionId);
2541	pGipCpu->cErrors++;
2542	return;
2543	}
2544
2545	/*
2546	* Recalc the update frequency every 0x800th time.
2547	*/
2548	if ( pGip->u32Mode != SUPGIPMODE_INVARIANT_TSC /* cuz we're not recalculating the frequency on invariant hosts. */
2549	&& !(pGipCpu->u32TransactionId & (GIP_UPDATEHZ_RECALC_FREQ * 2 - 2)))
2550	{
2551	if (pGip->u64NanoTSLastUpdateHz)
2552	{
2553	#ifdef RT_ARCH_AMD64 /** @todo fix 64-bit div here to work on x86 linux. */
2554	uint64_t u64Delta = u64NanoTS - pGip->u64NanoTSLastUpdateHz;
2555	uint32_t u32UpdateHz = (uint32_t)((RT_NS_1SEC_64 * GIP_UPDATEHZ_RECALC_FREQ) / u64Delta);
2556	if (u32UpdateHz <= 2000 && u32UpdateHz >= 30)
2557	{
2558	/** @todo r=ramshankar: Changing u32UpdateHz might screw up TSC frequency
2559	* calculation on non-invariant hosts if it changes the history decision
2560	* taken in supdrvGipDoUpdateCpu(). */
2561	uint64_t u64Interval = u64Delta / GIP_UPDATEHZ_RECALC_FREQ;
2562	ASMAtomicWriteU32(&pGip->u32UpdateHz, u32UpdateHz);
2563	ASMAtomicWriteU32(&pGip->u32UpdateIntervalNS, (uint32_t)u64Interval);
2564	}
2565	#endif
2566	}
2567	ASMAtomicWriteU64(&pGip->u64NanoTSLastUpdateHz, u64NanoTS \| 1);
2568	}
2569
2570	/*
2571	* Update the data.
2572	*/
2573	supdrvGipDoUpdateCpu(pDevExt, pGipCpu, u64NanoTS, u64TSC, iTick);
2574
2575	/*
2576	* Complete transaction.
2577	*/
2578	ASMAtomicIncU32(&pGipCpu->u32TransactionId);
2579	}
2580
2581
2582	/**
2583	* Updates the per cpu GIP data for the calling cpu.
2584	*
2585	* @param pDevExt The device extension.
2586	* @param u64NanoTS The current nanosecond timestamp.
2587	* @param u64TSC The current TSC timesaver.
2588	* @param idCpu The CPU ID.
2589	* @param idApic The APIC id for the CPU index.
2590	* @param iTick The current timer tick.
2591	*
2592	* @remarks Can be called with interrupts disabled!
2593	*/
2594	static void supdrvGipUpdatePerCpu(PSUPDRVDEVEXT pDevExt, uint64_t u64NanoTS, uint64_t u64TSC,
2595	RTCPUID idCpu, uint8_t idApic, uint64_t iTick)
2596	{
2597	uint32_t iCpu;
2598	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
2599
2600	/*
2601	* Avoid a potential race when a CPU online notification doesn't fire on
2602	* the onlined CPU but the tick creeps in before the event notification is
2603	* run.
2604	*/
2605	if (RT_LIKELY(iTick != 1))
2606	{ /* likely*/ }
2607	else
2608	{
2609	iCpu = supdrvGipFindOrAllocCpuIndexForCpuId(pGip, idCpu);
2610	if (pGip->aCPUs[iCpu].enmState == SUPGIPCPUSTATE_OFFLINE)
2611	supdrvGipMpEventOnlineOrInitOnCpu(pDevExt, idCpu);
2612	}
2613
2614	iCpu = pGip->aiCpuFromApicId[idApic];
2615	if (RT_LIKELY(iCpu < pGip->cCpus))
2616	{
2617	PSUPGIPCPU pGipCpu = &pGip->aCPUs[iCpu];
2618	if (pGipCpu->idCpu == idCpu)
2619	{
2620	/*
2621	* Start update transaction.
2622	*/
2623	if (!(ASMAtomicIncU32(&pGipCpu->u32TransactionId) & 1))
2624	{
2625	AssertMsgFailed(("Invalid transaction id, %#x, not odd!\n", pGipCpu->u32TransactionId));
2626	ASMAtomicIncU32(&pGipCpu->u32TransactionId);
2627	pGipCpu->cErrors++;
2628	return;
2629	}
2630
2631	/*
2632	* Update the data.
2633	*/
2634	supdrvGipDoUpdateCpu(pDevExt, pGipCpu, u64NanoTS, u64TSC, iTick);
2635
2636	/*
2637	* Complete transaction.
2638	*/
2639	ASMAtomicIncU32(&pGipCpu->u32TransactionId);
2640	}
2641	}
2642	}
2643
2644
2645	/**
2646	* Timer callback function for the sync and invariant GIP modes.
2647	*
2648	* @param pTimer The timer.
2649	* @param pvUser Opaque pointer to the device extension.
2650	* @param iTick The timer tick.
2651	*/
2652	static DECLCALLBACK(void) supdrvGipSyncAndInvariantTimer(PRTTIMER pTimer, void *pvUser, uint64_t iTick)
2653	{
2654	PSUPDRVDEVEXT pDevExt = (PSUPDRVDEVEXT)pvUser;
2655	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
2656	RTCCUINTREG fEFlags = ASMIntDisableFlags(); /* No interruptions please (real problem on S10). */
2657	uint64_t u64TSC = ASMReadTSC();
2658	uint64_t u64NanoTS = RTTimeSystemNanoTS();
2659	RT_NOREF1(pTimer);
2660
2661	if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_PRACTICALLY_ZERO)
2662	{
2663	/*
2664	* The calculations in supdrvGipUpdate() is somewhat timing sensitive,
2665	* missing timer ticks is not an option for GIP because the GIP users
2666	* will end up incrementing the time in 1ns per time getter call until
2667	* there is a complete timer update. So, if the delta has yet to be
2668	* calculated, we just pretend it is zero for now (the GIP users
2669	* probably won't have it for a wee while either and will do the same).
2670	*
2671	* We could maybe on some platforms try cross calling a CPU with a
2672	* working delta here, but it's not worth the hassle since the
2673	* likelihood of this happening is really low. On Windows, Linux, and
2674	* Solaris timers fire on the CPU they were registered/started on.
2675	* Darwin timers doesn't necessarily (they are high priority threads).
2676	*/
2677	uint32_t iCpuSet = RTMpCpuIdToSetIndex(RTMpCpuId());
2678	uint16_t iGipCpu = RT_LIKELY(iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx))
2679	? pGip->aiCpuFromCpuSetIdx[iCpuSet] : UINT16_MAX;
2680	Assert(!ASMIntAreEnabled());
2681	if (RT_LIKELY(iGipCpu < pGip->cCpus))
2682	{
2683	int64_t iTscDelta = pGip->aCPUs[iGipCpu].i64TSCDelta;
2684	if (iTscDelta != INT64_MAX)
2685	u64TSC -= iTscDelta;
2686	}
2687	}
2688
2689	supdrvGipUpdate(pDevExt, u64NanoTS, u64TSC, NIL_RTCPUID, iTick);
2690
2691	ASMSetFlags(fEFlags);
2692	}
2693
2694
2695	/**
2696	* Timer callback function for async GIP mode.
2697	* @param pTimer The timer.
2698	* @param pvUser Opaque pointer to the device extension.
2699	* @param iTick The timer tick.
2700	*/
2701	static DECLCALLBACK(void) supdrvGipAsyncTimer(PRTTIMER pTimer, void *pvUser, uint64_t iTick)
2702	{
2703	PSUPDRVDEVEXT pDevExt = (PSUPDRVDEVEXT)pvUser;
2704	RTCCUINTREG fEFlags = ASMIntDisableFlags(); /* No interruptions please (real problem on S10). */
2705	RTCPUID idCpu = RTMpCpuId();
2706	uint64_t u64TSC = ASMReadTSC();
2707	uint64_t NanoTS = RTTimeSystemNanoTS();
2708	RT_NOREF1(pTimer);
2709
2710	/** @todo reset the transaction number and whatnot when iTick == 1. */
2711	if (pDevExt->idGipMaster == idCpu)
2712	supdrvGipUpdate(pDevExt, NanoTS, u64TSC, idCpu, iTick);
2713	else
2714	supdrvGipUpdatePerCpu(pDevExt, NanoTS, u64TSC, idCpu, supdrvGipGetApicId(pDevExt->pGip), iTick);
2715
2716	ASMSetFlags(fEFlags);
2717	}
2718
2719
2720
2721
2722	/*
2723	*
2724	*
2725	* TSC Delta Measurements And Related Code
2726	* TSC Delta Measurements And Related Code
2727	* TSC Delta Measurements And Related Code
2728	*
2729	*
2730	*/
2731
2732
2733	/*
2734	* Select TSC delta measurement algorithm.
2735	*/
2736	#if 0
2737	# define GIP_TSC_DELTA_METHOD_1
2738	#else
2739	# define GIP_TSC_DELTA_METHOD_2
2740	#endif
2741
2742	/** For padding variables to keep them away from other cache lines. Better too
2743	* large than too small!
2744	* @remarks Current AMD64 and x86 CPUs seems to use 64 bytes. There are claims
2745	* that NetBurst had 128 byte cache lines while the 486 thru Pentium
2746	* III had 32 bytes cache lines. */
2747	#define GIP_TSC_DELTA_CACHE_LINE_SIZE 128
2748
2749
2750	/**
2751	* TSC delta measurement algorithm \#2 result entry.
2752	*/
2753	typedef struct SUPDRVTSCDELTAMETHOD2ENTRY
2754	{
2755	uint32_t iSeqMine;
2756	uint32_t iSeqOther;
2757	uint64_t uTsc;
2758	} SUPDRVTSCDELTAMETHOD2ENTRY;
2759
2760	/**
2761	* TSC delta measurement algorithm \#2 Data.
2762	*/
2763	typedef struct SUPDRVTSCDELTAMETHOD2
2764	{
2765	/** Padding to make sure the iCurSeqNo is in its own cache line. */
2766	uint64_t au64CacheLinePaddingBefore[GIP_TSC_DELTA_CACHE_LINE_SIZE / sizeof(uint64_t)];
2767	/** The current sequence number of this worker. */
2768	uint32_t volatile iCurSeqNo;
2769	/** Padding to make sure the iCurSeqNo is in its own cache line. */
2770	uint32_t au64CacheLinePaddingAfter[GIP_TSC_DELTA_CACHE_LINE_SIZE / sizeof(uint32_t) - 1];
2771	/** Result table. */
2772	SUPDRVTSCDELTAMETHOD2ENTRY aResults[64];
2773	} SUPDRVTSCDELTAMETHOD2;
2774	/** Pointer to the data for TSC delta measurement algorithm \#2 .*/
2775	typedef SUPDRVTSCDELTAMETHOD2 *PSUPDRVTSCDELTAMETHOD2;
2776
2777
2778	/**
2779	* The TSC delta synchronization struct, version 2.
2780	*
2781	* The synchronization variable is completely isolated in its own cache line
2782	* (provided our max cache line size estimate is correct).
2783	*/
2784	typedef struct SUPTSCDELTASYNC2
2785	{
2786	/** Padding to make sure the uVar1 is in its own cache line. */
2787	uint64_t au64CacheLinePaddingBefore[GIP_TSC_DELTA_CACHE_LINE_SIZE / sizeof(uint64_t)];
2788
2789	/** The synchronization variable, holds values GIP_TSC_DELTA_SYNC_. /
2790	volatile uint32_t uSyncVar;
2791	/** Sequence synchronizing variable used for post 'GO' synchronization. */
2792	volatile uint32_t uSyncSeq;
2793
2794	/** Padding to make sure the uVar1 is in its own cache line. */
2795	uint64_t au64CacheLinePaddingAfter[GIP_TSC_DELTA_CACHE_LINE_SIZE / sizeof(uint64_t) - 2];
2796
2797	/** Start RDTSC value. Put here mainly to save stack space. */
2798	uint64_t uTscStart;
2799	/** Copy of SUPDRVGIPTSCDELTARGS::cMaxTscTicks. */
2800	uint64_t cMaxTscTicks;
2801	} SUPTSCDELTASYNC2;
2802	AssertCompileSize(SUPTSCDELTASYNC2, GIP_TSC_DELTA_CACHE_LINE_SIZE * 2 + sizeof(uint64_t));
2803	typedef SUPTSCDELTASYNC2 *PSUPTSCDELTASYNC2;
2804
2805	/** Prestart wait. */
2806	#define GIP_TSC_DELTA_SYNC2_PRESTART_WAIT UINT32_C(0x0ffe)
2807	/** Prestart aborted. */
2808	#define GIP_TSC_DELTA_SYNC2_PRESTART_ABORT UINT32_C(0x0fff)
2809	/** Ready (on your mark). */
2810	#define GIP_TSC_DELTA_SYNC2_READY UINT32_C(0x1000)
2811	/** Steady (get set). */
2812	#define GIP_TSC_DELTA_SYNC2_STEADY UINT32_C(0x1001)
2813	/** Go! */
2814	#define GIP_TSC_DELTA_SYNC2_GO UINT32_C(0x1002)
2815	/** Used by the verification test. */
2816	#define GIP_TSC_DELTA_SYNC2_GO_GO UINT32_C(0x1003)
2817
2818	/** We reached the time limit. */
2819	#define GIP_TSC_DELTA_SYNC2_TIMEOUT UINT32_C(0x1ffe)
2820	/** The other party won't touch the sync struct ever again. */
2821	#define GIP_TSC_DELTA_SYNC2_FINAL UINT32_C(0x1fff)
2822
2823
2824	/**
2825	* Argument package/state passed by supdrvTscMeasureDeltaOne() to the RTMpOn
2826	* callback worker.
2827	* @todo add
2828	*/
2829	typedef struct SUPDRVGIPTSCDELTARGS
2830	{
2831	/** The device extension. */
2832	PSUPDRVDEVEXT pDevExt;
2833	/** Pointer to the GIP CPU array entry for the worker. */
2834	PSUPGIPCPU pWorker;
2835	/** Pointer to the GIP CPU array entry for the master. */
2836	PSUPGIPCPU pMaster;
2837	/** The maximum number of ticks to spend in supdrvTscMeasureDeltaCallback.
2838	* (This is what we need a rough TSC frequency for.) */
2839	uint64_t cMaxTscTicks;
2840	/** Used to abort synchronization setup. */
2841	bool volatile fAbortSetup;
2842
2843	/** Padding to make sure the master variables live in its own cache lines. */
2844	uint64_t au64CacheLinePaddingBefore[GIP_TSC_DELTA_CACHE_LINE_SIZE / sizeof(uint64_t)];
2845
2846	/** @name Master
2847	* @{ */
2848	/** The time the master spent in the MP worker. */
2849	uint64_t cElapsedMasterTscTicks;
2850	/** The iTry value when stopped at. */
2851	uint32_t iTry;
2852	/** Set if the run timed out. */
2853	bool volatile fTimedOut;
2854	/** Pointer to the master's synchronization struct (on stack). */
2855	PSUPTSCDELTASYNC2 volatile pSyncMaster;
2856	/** Master data union. */
2857	union
2858	{
2859	/** Data (master) for delta verification. */
2860	struct
2861	{
2862	/** Verification test TSC values for the master. */
2863	uint64_t volatile auTscs[32];
2864	} Verify;
2865	/** Data (master) for measurement method \#2. */
2866	struct
2867	{
2868	/** Data and sequence number. */
2869	SUPDRVTSCDELTAMETHOD2 Data;
2870	/** The lag setting for the next run. */
2871	bool fLag;
2872	/** Number of hits. */
2873	uint32_t cHits;
2874	} M2;
2875	} uMaster;
2876	/** The verifier verdict, VINF_SUCCESS if ok, VERR_OUT_OF_RANGE if not,
2877	* VERR_TRY_AGAIN on timeout. */
2878	int32_t rcVerify;
2879	#ifdef TSCDELTA_VERIFY_WITH_STATS
2880	/** The maximum difference between TSC read during delta verification. */
2881	int64_t cMaxVerifyTscTicks;
2882	/** The minimum difference between two TSC reads during verification. */
2883	int64_t cMinVerifyTscTicks;
2884	/** The bad TSC diff, worker relative to master (= worker - master).
2885	* Negative value means the worker is behind the master. */
2886	int64_t iVerifyBadTscDiff;
2887	#endif
2888	/** @} */
2889
2890	/** Padding to make sure the worker variables live is in its own cache line. */
2891	uint64_t au64CacheLinePaddingBetween[GIP_TSC_DELTA_CACHE_LINE_SIZE / sizeof(uint64_t)];
2892
2893	/** @name Proletarian
2894	* @{ */
2895	/** Pointer to the worker's synchronization struct (on stack). */
2896	PSUPTSCDELTASYNC2 volatile pSyncWorker;
2897	/** The time the worker spent in the MP worker. */
2898	uint64_t cElapsedWorkerTscTicks;
2899	/** Worker data union. */
2900	union
2901	{
2902	/** Data (worker) for delta verification. */
2903	struct
2904	{
2905	/** Verification test TSC values for the worker. */
2906	uint64_t volatile auTscs[32];
2907	} Verify;
2908	/** Data (worker) for measurement method \#2. */
2909	struct
2910	{
2911	/** Data and sequence number. */
2912	SUPDRVTSCDELTAMETHOD2 Data;
2913	/** The lag setting for the next run (set by master). */
2914	bool fLag;
2915	} M2;
2916	} uWorker;
2917	/** @} */
2918
2919	/** Padding to make sure the above is in its own cache line. */
2920	uint64_t au64CacheLinePaddingAfter[GIP_TSC_DELTA_CACHE_LINE_SIZE / sizeof(uint64_t)];
2921	} SUPDRVGIPTSCDELTARGS;
2922	typedef SUPDRVGIPTSCDELTARGS *PSUPDRVGIPTSCDELTARGS;
2923
2924
2925	/** @name Macros that implements the basic synchronization steps common to
2926	* the algorithms.
2927	*
2928	* Must be used from loop as the timeouts are implemented via 'break' statements
2929	* at the moment.
2930	*
2931	* @{
2932	*/
2933	#if defined(DEBUG_bird) /* \|\| defined(VBOX_STRICT) */
2934	# define TSCDELTA_DBG_VARS() uint32_t iDbgCounter
2935	# define TSCDELTA_DBG_START_LOOP() do { iDbgCounter = 0; } while (0)
2936	# define TSCDELTA_DBG_CHECK_LOOP() \
2937	do { iDbgCounter++; if ((iDbgCounter & UINT32_C(0x01ffffff)) == 0) RT_BREAKPOINT(); } while (0)
2938	#else
2939	# define TSCDELTA_DBG_VARS() ((void)0)
2940	# define TSCDELTA_DBG_START_LOOP() ((void)0)
2941	# define TSCDELTA_DBG_CHECK_LOOP() ((void)0)
2942	#endif
2943	#if 0
2944	# define TSCDELTA_DBG_SYNC_MSG(a_Args) SUPR0Printf a_Args
2945	#else
2946	# define TSCDELTA_DBG_SYNC_MSG(a_Args) ((void)0)
2947	#endif
2948	#if 0
2949	# define TSCDELTA_DBG_SYNC_MSG2(a_Args) SUPR0Printf a_Args
2950	#else
2951	# define TSCDELTA_DBG_SYNC_MSG2(a_Args) ((void)0)
2952	#endif
2953	#if 0
2954	# define TSCDELTA_DBG_SYNC_MSG9(a_Args) SUPR0Printf a_Args
2955	#else
2956	# define TSCDELTA_DBG_SYNC_MSG9(a_Args) ((void)0)
2957	#endif
2958
2959
2960	static bool supdrvTscDeltaSync2_Before(PSUPTSCDELTASYNC2 pMySync, PSUPTSCDELTASYNC2 pOtherSync,
2961	bool fIsMaster, PRTCCUINTREG pfEFlags, PSUPDRVGIPTSCDELTARGS pArgs)
2962	{
2963	uint32_t iMySeq = fIsMaster ? 0 : 256;
2964	uint32_t const iMaxSeq = iMySeq + 16; /* For the last loop, darn linux/freebsd C-ishness. */
2965	uint32_t u32Tmp;
2966	uint32_t iSync2Loops = 0;
2967	RTCCUINTREG fEFlags;
2968	TSCDELTA_DBG_VARS();
2969
2970	pfEFlags = X86_EFL_IF \| X86_EFL_1; / should shut up most nagging compilers. */
2971
2972	/*
2973	* The master tells the worker to get on it's mark.
2974	*/
2975	if (fIsMaster)
2976	{
2977	if (RT_LIKELY(ASMAtomicCmpXchgU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_STEADY, GIP_TSC_DELTA_SYNC2_READY)))
2978	{ /* likely*/ }
2979	else
2980	{
2981	TSCDELTA_DBG_SYNC_MSG(("sync/before/%s: #1 uSyncVar=%#x\n", fIsMaster ? "master" : "worker", pOtherSync->uSyncVar));
2982	return false;
2983	}
2984	}
2985
2986	/*
2987	* Wait for the on your mark signal (ack in the master case). We process timeouts here.
2988	*/
2989	ASMAtomicWriteU32(&(pMySync)->uSyncSeq, 0);
2990	for (;;)
2991	{
2992	fEFlags = ASMIntDisableFlags();
2993	u32Tmp = ASMAtomicReadU32(&pMySync->uSyncVar);
2994	if (u32Tmp == GIP_TSC_DELTA_SYNC2_STEADY)
2995	break;
2996	ASMSetFlags(fEFlags);
2997	ASMNopPause();
2998
2999	/* Abort? */
3000	if (u32Tmp != GIP_TSC_DELTA_SYNC2_READY)
3001	{
3002	TSCDELTA_DBG_SYNC_MSG(("sync/before/%s: #2 u32Tmp=%#x\n", fIsMaster ? "master" : "worker", u32Tmp));
3003	return false;
3004	}
3005
3006	/* Check for timeouts every so often (not every loop in case RDTSC is
3007	trapping or something). Must check the first time around. */
3008	#if 0 /* For debugging the timeout paths. */
3009	static uint32_t volatile xxx;
3010	#endif
3011	if ( ( (iSync2Loops & 0x3ff) == 0
3012	&& ASMReadTSC() - pMySync->uTscStart > pMySync->cMaxTscTicks)
3013	#if 0 /* This is crazy, I know, but enable this code and the results are markedly better when enabled on the 1.4GHz AMD (debug). */
3014	\|\| (!fIsMaster && (++xxx & 0xf) == 0)
3015	#endif
3016	)
3017	{
3018	/* Try switch our own state into timeout mode so the master cannot tell us to 'GO',
3019	ignore the timeout if we've got the go ahead already (simpler). */
3020	if (ASMAtomicCmpXchgU32(&pMySync->uSyncVar, GIP_TSC_DELTA_SYNC2_TIMEOUT, GIP_TSC_DELTA_SYNC2_READY))
3021	{
3022	TSCDELTA_DBG_SYNC_MSG(("sync/before/%s: timeout\n", fIsMaster ? "master" : "worker"));
3023	ASMAtomicCmpXchgU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_TIMEOUT, GIP_TSC_DELTA_SYNC2_STEADY);
3024	ASMAtomicWriteBool(&pArgs->fTimedOut, true);
3025	return false;
3026	}
3027	}
3028	iSync2Loops++;
3029	}
3030
3031	/*
3032	* Interrupts are now disabled and will remain disabled until we do
3033	* TSCDELTA_MASTER_SYNC_AFTER / TSCDELTA_OTHER_SYNC_AFTER.
3034	*/
3035	*pfEFlags = fEFlags;
3036
3037	/*
3038	* The worker tells the master that it is on its mark and that the master
3039	* need to get into position as well.
3040	*/
3041	if (!fIsMaster)
3042	{
3043	if (RT_LIKELY(ASMAtomicCmpXchgU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_STEADY, GIP_TSC_DELTA_SYNC2_READY)))
3044	{ /* likely */ }
3045	else
3046	{
3047	ASMSetFlags(fEFlags);
3048	TSCDELTA_DBG_SYNC_MSG(("sync/before/%s: #3 uSyncVar=%#x\n", fIsMaster ? "master" : "worker", pOtherSync->uSyncVar));
3049	return false;
3050	}
3051	}
3052
3053	/*
3054	* The master sends the 'go' to the worker and wait for ACK.
3055	*/
3056	if (fIsMaster)
3057	{
3058	if (RT_LIKELY(ASMAtomicCmpXchgU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_GO, GIP_TSC_DELTA_SYNC2_STEADY)))
3059	{ /* likely */ }
3060	else
3061	{
3062	ASMSetFlags(fEFlags);
3063	TSCDELTA_DBG_SYNC_MSG(("sync/before/%s: #4 uSyncVar=%#x\n", fIsMaster ? "master" : "worker", pOtherSync->uSyncVar));
3064	return false;
3065	}
3066	}
3067
3068	/*
3069	* Wait for the 'go' signal (ack in the master case).
3070	*/
3071	TSCDELTA_DBG_START_LOOP();
3072	for (;;)
3073	{
3074	u32Tmp = ASMAtomicReadU32(&pMySync->uSyncVar);
3075	if (u32Tmp == GIP_TSC_DELTA_SYNC2_GO)
3076	break;
3077	if (RT_LIKELY(u32Tmp == GIP_TSC_DELTA_SYNC2_STEADY))
3078	{ /* likely */ }
3079	else
3080	{
3081	ASMSetFlags(fEFlags);
3082	TSCDELTA_DBG_SYNC_MSG(("sync/before/%s: #5 u32Tmp=%#x\n", fIsMaster ? "master" : "worker", u32Tmp));
3083	return false;
3084	}
3085
3086	TSCDELTA_DBG_CHECK_LOOP();
3087	ASMNopPause();
3088	}
3089
3090	/*
3091	* The worker acks the 'go' (shouldn't fail).
3092	*/
3093	if (!fIsMaster)
3094	{
3095	if (RT_LIKELY(ASMAtomicCmpXchgU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_GO, GIP_TSC_DELTA_SYNC2_STEADY)))
3096	{ /* likely */ }
3097	else
3098	{
3099	ASMSetFlags(fEFlags);
3100	TSCDELTA_DBG_SYNC_MSG(("sync/before/%s: #6 uSyncVar=%#x\n", fIsMaster ? "master" : "worker", pOtherSync->uSyncVar));
3101	return false;
3102	}
3103	}
3104
3105	/*
3106	* Try enter mostly lockstep execution with it.
3107	*/
3108	for (;;)
3109	{
3110	uint32_t iOtherSeq1, iOtherSeq2;
3111	ASMCompilerBarrier();
3112	ASMSerializeInstruction();
3113
3114	ASMAtomicWriteU32(&pMySync->uSyncSeq, iMySeq);
3115	ASMNopPause();
3116	iOtherSeq1 = ASMAtomicXchgU32(&pOtherSync->uSyncSeq, iMySeq);
3117	ASMNopPause();
3118	iOtherSeq2 = ASMAtomicReadU32(&pMySync->uSyncSeq);
3119
3120	ASMCompilerBarrier();
3121	if (iOtherSeq1 == iOtherSeq2)
3122	return true;
3123
3124	/* Did the other guy give up? Should we give up? */
3125	if ( iOtherSeq1 == UINT32_MAX
3126	\|\| iOtherSeq2 == UINT32_MAX)
3127	return true;
3128	if (++iMySeq >= iMaxSeq)
3129	{
3130	ASMAtomicWriteU32(&pMySync->uSyncSeq, UINT32_MAX);
3131	return true;
3132	}
3133	ASMNopPause();
3134	}
3135	}
3136
3137	#define TSCDELTA_MASTER_SYNC_BEFORE(a_pMySync, a_pOtherSync, a_pfEFlags, a_pArgs) \
3138	if (RT_LIKELY(supdrvTscDeltaSync2_Before(a_pMySync, a_pOtherSync, true /fIsMaster/, a_pfEFlags, a_pArgs))) \
3139	{ /likely/ } \
3140	else if (true) \
3141	{ \
3142	TSCDELTA_DBG_SYNC_MSG9(("sync/before/master: #89\n")); \
3143	break; \
3144	} else do {} while (0)
3145	#define TSCDELTA_OTHER_SYNC_BEFORE(a_pMySync, a_pOtherSync, a_pfEFlags, a_pArgs) \
3146	if (RT_LIKELY(supdrvTscDeltaSync2_Before(a_pMySync, a_pOtherSync, false /fIsMaster/, a_pfEFlags, a_pArgs))) \
3147	{ /likely/ } \
3148	else if (true) \
3149	{ \
3150	TSCDELTA_DBG_SYNC_MSG9(("sync/before/other: #89\n")); \
3151	break; \
3152	} else do {} while (0)
3153
3154
3155	static bool supdrvTscDeltaSync2_After(PSUPTSCDELTASYNC2 pMySync, PSUPTSCDELTASYNC2 pOtherSync,
3156	bool fIsMaster, RTCCUINTREG fEFlags)
3157	{
3158	TSCDELTA_DBG_VARS();
3159	RT_NOREF1(pOtherSync);
3160
3161	/*
3162	* Wait for the 'ready' signal. In the master's case, this means the
3163	* worker has completed its data collection, while in the worker's case it
3164	* means the master is done processing the data and it's time for the next
3165	* loop iteration (or whatever).
3166	*/
3167	ASMSetFlags(fEFlags);
3168	TSCDELTA_DBG_START_LOOP();
3169	for (;;)
3170	{
3171	uint32_t u32Tmp = ASMAtomicReadU32(&pMySync->uSyncVar);
3172	if ( u32Tmp == GIP_TSC_DELTA_SYNC2_READY
3173	\|\| (u32Tmp == GIP_TSC_DELTA_SYNC2_STEADY && !fIsMaster) /* kicked twice => race */ )
3174	return true;
3175	ASMNopPause();
3176	if (RT_LIKELY(u32Tmp == GIP_TSC_DELTA_SYNC2_GO))
3177	{ /* likely */}
3178	else
3179	{
3180	TSCDELTA_DBG_SYNC_MSG(("sync/after/other: #1 u32Tmp=%#x\n", u32Tmp));
3181	return false; /* shouldn't ever happen! */
3182	}
3183	TSCDELTA_DBG_CHECK_LOOP();
3184	ASMNopPause();
3185	}
3186	}
3187
3188	#define TSCDELTA_MASTER_SYNC_AFTER(a_pMySync, a_pOtherSync, a_fEFlags) \
3189	if (RT_LIKELY(supdrvTscDeltaSync2_After(a_pMySync, a_pOtherSync, true /fIsMaster/, a_fEFlags))) \
3190	{ /* likely */ } \
3191	else if (true) \
3192	{ \
3193	TSCDELTA_DBG_SYNC_MSG9(("sync/after/master: #97\n")); \
3194	break; \
3195	} else do {} while (0)
3196
3197	#define TSCDELTA_MASTER_KICK_OTHER_OUT_OF_AFTER(a_pMySync, a_pOtherSync) \
3198	/* \
3199	* Tell the worker that we're done processing the data and ready for the next round. \
3200	*/ \
3201	if (RT_LIKELY(ASMAtomicCmpXchgU32(&(a_pOtherSync)->uSyncVar, GIP_TSC_DELTA_SYNC2_READY, GIP_TSC_DELTA_SYNC2_GO))) \
3202	{ /* likely */ } \
3203	else if (true)\
3204	{ \
3205	TSCDELTA_DBG_SYNC_MSG(("sync/after/master: #99 uSyncVar=%#x\n", (a_pOtherSync)->uSyncVar)); \
3206	break; \
3207	} else do {} while (0)
3208
3209	#define TSCDELTA_OTHER_SYNC_AFTER(a_pMySync, a_pOtherSync, a_fEFlags) \
3210	if (true) { \
3211	/* \
3212	* Tell the master that we're done collecting data and wait for the next round to start. \
3213	*/ \
3214	if (RT_LIKELY(ASMAtomicCmpXchgU32(&(a_pOtherSync)->uSyncVar, GIP_TSC_DELTA_SYNC2_READY, GIP_TSC_DELTA_SYNC2_GO))) \
3215	{ /* likely */ } \
3216	else \
3217	{ \
3218	ASMSetFlags(a_fEFlags); \
3219	TSCDELTA_DBG_SYNC_MSG(("sync/after/other: #0 uSyncVar=%#x\n", (a_pOtherSync)->uSyncVar)); \
3220	break; \
3221	} \
3222	if (RT_LIKELY(supdrvTscDeltaSync2_After(a_pMySync, a_pOtherSync, false /fIsMaster/, a_fEFlags))) \
3223	{ /* likely */ } \
3224	else \
3225	{ \
3226	TSCDELTA_DBG_SYNC_MSG9(("sync/after/other: #98\n")); \
3227	break; \
3228	} \
3229	} else do {} while (0)
3230	/** @} */
3231
3232
3233	#ifdef GIP_TSC_DELTA_METHOD_1
3234	/**
3235	* TSC delta measurement algorithm \#1 (GIP_TSC_DELTA_METHOD_1).
3236	*
3237	*
3238	* We ignore the first few runs of the loop in order to prime the
3239	* cache. Also, we need to be careful about using 'pause' instruction
3240	* in critical busy-wait loops in this code - it can cause undesired
3241	* behaviour with hyperthreading.
3242	*
3243	* We try to minimize the measurement error by computing the minimum
3244	* read time of the compare statement in the worker by taking TSC
3245	* measurements across it.
3246	*
3247	* It must be noted that the computed minimum read time is mostly to
3248	* eliminate huge deltas when the worker is too early and doesn't by
3249	* itself help produce more accurate deltas. We allow two times the
3250	* computed minimum as an arbitrary acceptable threshold. Therefore,
3251	* it is still possible to get negative deltas where there are none
3252	* when the worker is earlier. As long as these occasional negative
3253	* deltas are lower than the time it takes to exit guest-context and
3254	* the OS to reschedule EMT on a different CPU, we won't expose a TSC
3255	* that jumped backwards. It is due to the existence of the negative
3256	* deltas that we don't recompute the delta with the master and
3257	* worker interchanged to eliminate the remaining measurement error.
3258	*
3259	*
3260	* @param pArgs The argument/state data.
3261	* @param pMySync My synchronization structure.
3262	* @param pOtherSync My partner's synchronization structure.
3263	* @param fIsMaster Set if master, clear if worker.
3264	* @param iTry The attempt number.
3265	*/
3266	static void supdrvTscDeltaMethod1Loop(PSUPDRVGIPTSCDELTARGS pArgs, PSUPTSCDELTASYNC2 pMySync, PSUPTSCDELTASYNC2 pOtherSync,
3267	bool fIsMaster, uint32_t iTry)
3268	{
3269	PSUPGIPCPU pGipCpuWorker = pArgs->pWorker;
3270	PSUPGIPCPU pGipCpuMaster = pArgs->pMaster;
3271	uint64_t uMinCmpReadTime = UINT64_MAX;
3272	unsigned iLoop;
3273	NOREF(iTry);
3274
3275	for (iLoop = 0; iLoop < GIP_TSC_DELTA_LOOPS; iLoop++)
3276	{
3277	RTCCUINTREG fEFlags;
3278	if (fIsMaster)
3279	{
3280	/*
3281	* The master.
3282	*/
3283	AssertMsg(pGipCpuMaster->u64TSCSample == GIP_TSC_DELTA_RSVD,
3284	("%#llx idMaster=%#x idWorker=%#x (idGipMaster=%#x)\n",
3285	pGipCpuMaster->u64TSCSample, pGipCpuMaster->idCpu, pGipCpuWorker->idCpu, pArgs->pDevExt->idGipMaster));
3286	TSCDELTA_MASTER_SYNC_BEFORE(pMySync, pOtherSync, &fEFlags, pArgs);
3287
3288	do
3289	{
3290	ASMSerializeInstruction();
3291	ASMAtomicWriteU64(&pGipCpuMaster->u64TSCSample, ASMReadTSC());
3292	} while (pGipCpuMaster->u64TSCSample == GIP_TSC_DELTA_RSVD);
3293
3294	TSCDELTA_MASTER_SYNC_AFTER(pMySync, pOtherSync, fEFlags);
3295
3296	/* Process the data. */
3297	if (iLoop > GIP_TSC_DELTA_PRIMER_LOOPS + GIP_TSC_DELTA_READ_TIME_LOOPS)
3298	{
3299	if (pGipCpuWorker->u64TSCSample != GIP_TSC_DELTA_RSVD)
3300	{
3301	int64_t iDelta = pGipCpuWorker->u64TSCSample
3302	- (pGipCpuMaster->u64TSCSample - pGipCpuMaster->i64TSCDelta);
3303	if ( iDelta >= GIP_TSC_DELTA_INITIAL_MASTER_VALUE
3304	? iDelta < pGipCpuWorker->i64TSCDelta
3305	: iDelta > pGipCpuWorker->i64TSCDelta \|\| pGipCpuWorker->i64TSCDelta == INT64_MAX)
3306	pGipCpuWorker->i64TSCDelta = iDelta;
3307	}
3308	}
3309
3310	/* Reset our TSC sample and tell the worker to move on. */
3311	ASMAtomicWriteU64(&pGipCpuMaster->u64TSCSample, GIP_TSC_DELTA_RSVD);
3312	TSCDELTA_MASTER_KICK_OTHER_OUT_OF_AFTER(pMySync, pOtherSync);
3313	}
3314	else
3315	{
3316	/*
3317	* The worker.
3318	*/
3319	uint64_t uTscWorker;
3320	uint64_t uTscWorkerFlushed;
3321	uint64_t uCmpReadTime;
3322
3323	ASMAtomicReadU64(&pGipCpuMaster->u64TSCSample); /* Warm the cache line. */
3324	TSCDELTA_OTHER_SYNC_BEFORE(pMySync, pOtherSync, &fEFlags, pArgs);
3325
3326	/*
3327	* Keep reading the TSC until we notice that the master has read his. Reading
3328	* the TSC -after- the master has updated the memory is way too late. We thus
3329	* compensate by trying to measure how long it took for the worker to notice
3330	* the memory flushed from the master.
3331	*/
3332	do
3333	{
3334	ASMSerializeInstruction();
3335	uTscWorker = ASMReadTSC();
3336	} while (pGipCpuMaster->u64TSCSample == GIP_TSC_DELTA_RSVD);
3337	ASMSerializeInstruction();
3338	uTscWorkerFlushed = ASMReadTSC();
3339
3340	uCmpReadTime = uTscWorkerFlushed - uTscWorker;
3341	if (iLoop > GIP_TSC_DELTA_PRIMER_LOOPS + GIP_TSC_DELTA_READ_TIME_LOOPS)
3342	{
3343	/* This is totally arbitrary a.k.a I don't like it but I have no better ideas for now. */
3344	if (uCmpReadTime < (uMinCmpReadTime << 1))
3345	{
3346	ASMAtomicWriteU64(&pGipCpuWorker->u64TSCSample, uTscWorker);
3347	if (uCmpReadTime < uMinCmpReadTime)
3348	uMinCmpReadTime = uCmpReadTime;
3349	}
3350	else
3351	ASMAtomicWriteU64(&pGipCpuWorker->u64TSCSample, GIP_TSC_DELTA_RSVD);
3352	}
3353	else if (iLoop > GIP_TSC_DELTA_PRIMER_LOOPS)
3354	{
3355	if (uCmpReadTime < uMinCmpReadTime)
3356	uMinCmpReadTime = uCmpReadTime;
3357	}
3358
3359	TSCDELTA_OTHER_SYNC_AFTER(pMySync, pOtherSync, fEFlags);
3360	}
3361	}
3362
3363	TSCDELTA_DBG_SYNC_MSG9(("sync/method1loop/%s: #92 iLoop=%u MyState=%#x\n", fIsMaster ? "master" : "worker", iLoop,
3364	pMySync->uSyncVar));
3365
3366	/*
3367	* We must reset the worker TSC sample value in case it gets picked as a
3368	* GIP master later on (it's trashed above, naturally).
3369	*/
3370	if (!fIsMaster)
3371	ASMAtomicWriteU64(&pGipCpuWorker->u64TSCSample, GIP_TSC_DELTA_RSVD);
3372	}
3373	#endif /* GIP_TSC_DELTA_METHOD_1 */
3374
3375
3376	#ifdef GIP_TSC_DELTA_METHOD_2
3377	/*
3378	* TSC delta measurement algorithm \#2 configuration and code - Experimental!!
3379	*/
3380
3381	# define GIP_TSC_DELTA_M2_LOOPS (7 + GIP_TSC_DELTA_M2_PRIMER_LOOPS)
3382	# define GIP_TSC_DELTA_M2_PRIMER_LOOPS 0
3383
3384
3385	static void supdrvTscDeltaMethod2ProcessDataOnMaster(PSUPDRVGIPTSCDELTARGS pArgs)
3386	{
3387	int64_t iMasterTscDelta = pArgs->pMaster->i64TSCDelta;
3388	int64_t iBestDelta = pArgs->pWorker->i64TSCDelta;
3389	uint32_t idxResult;
3390	uint32_t cHits = 0;
3391
3392	/*
3393	* Look for matching entries in the master and worker tables.
3394	*/
3395	for (idxResult = 0; idxResult < RT_ELEMENTS(pArgs->uMaster.M2.Data.aResults); idxResult++)
3396	{
3397	uint32_t idxOther = pArgs->uMaster.M2.Data.aResults[idxResult].iSeqOther;
3398	if (idxOther & 1)
3399	{
3400	idxOther >>= 1;
3401	if (idxOther < RT_ELEMENTS(pArgs->uWorker.M2.Data.aResults))
3402	{
3403	if (pArgs->uWorker.M2.Data.aResults[idxOther].iSeqOther == pArgs->uMaster.M2.Data.aResults[idxResult].iSeqMine)
3404	{
3405	int64_t iDelta;
3406	iDelta = pArgs->uWorker.M2.Data.aResults[idxOther].uTsc
3407	- (pArgs->uMaster.M2.Data.aResults[idxResult].uTsc - iMasterTscDelta);
3408	if ( iDelta >= GIP_TSC_DELTA_INITIAL_MASTER_VALUE
3409	? iDelta < iBestDelta
3410	: iDelta > iBestDelta \|\| iBestDelta == INT64_MAX)
3411	iBestDelta = iDelta;
3412	cHits++;
3413	}
3414	}
3415	}
3416	}
3417
3418	/*
3419	* Save the results.
3420	*/
3421	if (cHits > 2)
3422	pArgs->pWorker->i64TSCDelta = iBestDelta;
3423	pArgs->uMaster.M2.cHits += cHits;
3424	}
3425
3426
3427	/**
3428	* The core function of the 2nd TSC delta measurement algorithm.
3429	*
3430	* The idea here is that we have the two CPUs execute the exact same code
3431	* collecting a largish set of TSC samples. The code has one data dependency on
3432	* the other CPU which intention it is to synchronize the execution as well as
3433	* help cross references the two sets of TSC samples (the sequence numbers).
3434	*
3435	* The @a fLag parameter is used to modify the execution a tiny bit on one or
3436	* both of the CPUs. When @a fLag differs between the CPUs, it is thought that
3437	* it will help with making the CPUs enter lock step execution occasionally.
3438	*
3439	*/
3440	static void supdrvTscDeltaMethod2CollectData(PSUPDRVTSCDELTAMETHOD2 pMyData, uint32_t volatile *piOtherSeqNo, bool fLag)
3441	{
3442	SUPDRVTSCDELTAMETHOD2ENTRY *pEntry = &pMyData->aResults[0];
3443	uint32_t cLeft = RT_ELEMENTS(pMyData->aResults);
3444
3445	ASMAtomicWriteU32(&pMyData->iCurSeqNo, 0);
3446	ASMSerializeInstruction();
3447	while (cLeft-- > 0)
3448	{
3449	uint64_t uTsc;
3450	uint32_t iSeqMine = ASMAtomicIncU32(&pMyData->iCurSeqNo);
3451	uint32_t iSeqOther = ASMAtomicReadU32(piOtherSeqNo);
3452	ASMCompilerBarrier();
3453	ASMSerializeInstruction(); /* Way better result than with ASMMemoryFenceSSE2() in this position! */
3454	uTsc = ASMReadTSC();
3455	ASMAtomicIncU32(&pMyData->iCurSeqNo);
3456	ASMCompilerBarrier();
3457	ASMSerializeInstruction();
3458	pEntry->iSeqMine = iSeqMine;
3459	pEntry->iSeqOther = iSeqOther;
3460	pEntry->uTsc = uTsc;
3461	pEntry++;
3462	ASMSerializeInstruction();
3463	if (fLag)
3464	ASMNopPause();
3465	}
3466	}
3467
3468
3469	/**
3470	* TSC delta measurement algorithm \#2 (GIP_TSC_DELTA_METHOD_2).
3471	*
3472	* See supdrvTscDeltaMethod2CollectData for algorithm details.
3473	*
3474	* @param pArgs The argument/state data.
3475	* @param pMySync My synchronization structure.
3476	* @param pOtherSync My partner's synchronization structure.
3477	* @param fIsMaster Set if master, clear if worker.
3478	* @param iTry The attempt number.
3479	*/
3480	static void supdrvTscDeltaMethod2Loop(PSUPDRVGIPTSCDELTARGS pArgs, PSUPTSCDELTASYNC2 pMySync, PSUPTSCDELTASYNC2 pOtherSync,
3481	bool fIsMaster, uint32_t iTry)
3482	{
3483	unsigned iLoop;
3484	RT_NOREF1(iTry);
3485
3486	for (iLoop = 0; iLoop < GIP_TSC_DELTA_M2_LOOPS; iLoop++)
3487	{
3488	RTCCUINTREG fEFlags;
3489	if (fIsMaster)
3490	{
3491	/*
3492	* Adjust the loop lag fudge.
3493	*/
3494	# if GIP_TSC_DELTA_M2_PRIMER_LOOPS > 0
3495	if (iLoop < GIP_TSC_DELTA_M2_PRIMER_LOOPS)
3496	{
3497	/* Lag during the priming to be nice to everyone.. */
3498	pArgs->uMaster.M2.fLag = true;
3499	pArgs->uWorker.M2.fLag = true;
3500	}
3501	else
3502	# endif
3503	if (iLoop < (GIP_TSC_DELTA_M2_LOOPS - GIP_TSC_DELTA_M2_PRIMER_LOOPS) / 4)
3504	{
3505	/* 25 % of the body without lagging. */
3506	pArgs->uMaster.M2.fLag = false;
3507	pArgs->uWorker.M2.fLag = false;
3508	}
3509	else if (iLoop < (GIP_TSC_DELTA_M2_LOOPS - GIP_TSC_DELTA_M2_PRIMER_LOOPS) / 4 * 2)
3510	{
3511	/* 25 % of the body with both lagging. */
3512	pArgs->uMaster.M2.fLag = true;
3513	pArgs->uWorker.M2.fLag = true;
3514	}
3515	else
3516	{
3517	/* 50% of the body with alternating lag. */
3518	pArgs->uMaster.M2.fLag = (iLoop & 1) == 0;
3519	pArgs->uWorker.M2.fLag= (iLoop & 1) == 1;
3520	}
3521
3522	/*
3523	* Sync up with the worker and collect data.
3524	*/
3525	TSCDELTA_MASTER_SYNC_BEFORE(pMySync, pOtherSync, &fEFlags, pArgs);
3526	supdrvTscDeltaMethod2CollectData(&pArgs->uMaster.M2.Data, &pArgs->uWorker.M2.Data.iCurSeqNo, pArgs->uMaster.M2.fLag);
3527	TSCDELTA_MASTER_SYNC_AFTER(pMySync, pOtherSync, fEFlags);
3528
3529	/*
3530	* Process the data.
3531	*/
3532	# if GIP_TSC_DELTA_M2_PRIMER_LOOPS > 0
3533	if (iLoop >= GIP_TSC_DELTA_M2_PRIMER_LOOPS)
3534	# endif
3535	supdrvTscDeltaMethod2ProcessDataOnMaster(pArgs);
3536
3537	TSCDELTA_MASTER_KICK_OTHER_OUT_OF_AFTER(pMySync, pOtherSync);
3538	}
3539	else
3540	{
3541	/*
3542	* The worker.
3543	*/
3544	TSCDELTA_OTHER_SYNC_BEFORE(pMySync, pOtherSync, &fEFlags, pArgs);
3545	supdrvTscDeltaMethod2CollectData(&pArgs->uWorker.M2.Data, &pArgs->uMaster.M2.Data.iCurSeqNo, pArgs->uWorker.M2.fLag);
3546	TSCDELTA_OTHER_SYNC_AFTER(pMySync, pOtherSync, fEFlags);
3547	}
3548	}
3549	}
3550
3551	#endif /* GIP_TSC_DELTA_METHOD_2 */
3552
3553
3554
3555	static int supdrvTscDeltaVerify(PSUPDRVGIPTSCDELTARGS pArgs, PSUPTSCDELTASYNC2 pMySync,
3556	PSUPTSCDELTASYNC2 pOtherSync, bool fIsMaster, int64_t iWorkerTscDelta)
3557	{
3558	/PSUPGIPCPU pGipCpuWorker = pArgs->pWorker; - unused /
3559	PSUPGIPCPU pGipCpuMaster = pArgs->pMaster;
3560	uint32_t i;
3561	TSCDELTA_DBG_VARS();
3562
3563	for (;;)
3564	{
3565	RTCCUINTREG fEFlags;
3566	AssertCompile((RT_ELEMENTS(pArgs->uMaster.Verify.auTscs) & 1) == 0);
3567	AssertCompile(RT_ELEMENTS(pArgs->uMaster.Verify.auTscs) == RT_ELEMENTS(pArgs->uWorker.Verify.auTscs));
3568
3569	if (fIsMaster)
3570	{
3571	uint64_t uTscWorker;
3572	TSCDELTA_MASTER_SYNC_BEFORE(pMySync, pOtherSync, &fEFlags, pArgs);
3573
3574	/*
3575	* Collect TSC, master goes first.
3576	*/
3577	for (i = 0; i < RT_ELEMENTS(pArgs->uMaster.Verify.auTscs); i += 2)
3578	{
3579	/* Read, kick & wait #1. */
3580	uint64_t uTsc = ASMReadTSC();
3581	ASMAtomicWriteU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_GO_GO);
3582	ASMSerializeInstruction();
3583	pArgs->uMaster.Verify.auTscs[i] = uTsc;
3584	TSCDELTA_DBG_START_LOOP();
3585	while (ASMAtomicReadU32(&pMySync->uSyncVar) == GIP_TSC_DELTA_SYNC2_GO)
3586	{
3587	TSCDELTA_DBG_CHECK_LOOP();
3588	ASMNopPause();
3589	}
3590
3591	/* Read, kick & wait #2. */
3592	uTsc = ASMReadTSC();
3593	ASMAtomicWriteU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_GO);
3594	ASMSerializeInstruction();
3595	pArgs->uMaster.Verify.auTscs[i + 1] = uTsc;
3596	TSCDELTA_DBG_START_LOOP();
3597	while (ASMAtomicReadU32(&pMySync->uSyncVar) == GIP_TSC_DELTA_SYNC2_GO_GO)
3598	{
3599	TSCDELTA_DBG_CHECK_LOOP();
3600	ASMNopPause();
3601	}
3602	}
3603
3604	TSCDELTA_MASTER_SYNC_AFTER(pMySync, pOtherSync, fEFlags);
3605
3606	/*
3607	* Process the data.
3608	*/
3609	#ifdef TSCDELTA_VERIFY_WITH_STATS
3610	pArgs->cMaxVerifyTscTicks = INT64_MIN;
3611	pArgs->cMinVerifyTscTicks = INT64_MAX;
3612	pArgs->iVerifyBadTscDiff = 0;
3613	#endif
3614	ASMAtomicWriteS32(&pArgs->rcVerify, VINF_SUCCESS);
3615	uTscWorker = 0;
3616	for (i = 0; i < RT_ELEMENTS(pArgs->uMaster.Verify.auTscs); i++)
3617	{
3618	/* Master vs previous worker entry. */
3619	uint64_t uTscMaster = pArgs->uMaster.Verify.auTscs[i] - pGipCpuMaster->i64TSCDelta;
3620	int64_t iDiff;
3621	if (i > 0)
3622	{
3623	iDiff = uTscMaster - uTscWorker;
3624	#ifdef TSCDELTA_VERIFY_WITH_STATS
3625	if (iDiff > pArgs->cMaxVerifyTscTicks)
3626	pArgs->cMaxVerifyTscTicks = iDiff;
3627	if (iDiff < pArgs->cMinVerifyTscTicks)
3628	pArgs->cMinVerifyTscTicks = iDiff;
3629	#endif
3630	if (iDiff < 0)
3631	{
3632	#ifdef TSCDELTA_VERIFY_WITH_STATS
3633	pArgs->iVerifyBadTscDiff = -iDiff;
3634	#endif
3635	ASMAtomicWriteS32(&pArgs->rcVerify, VERR_OUT_OF_RANGE);
3636	break;
3637	}
3638	}
3639
3640	/* Worker vs master. */
3641	uTscWorker = pArgs->uWorker.Verify.auTscs[i] - iWorkerTscDelta;
3642	iDiff = uTscWorker - uTscMaster;
3643	#ifdef TSCDELTA_VERIFY_WITH_STATS
3644	if (iDiff > pArgs->cMaxVerifyTscTicks)
3645	pArgs->cMaxVerifyTscTicks = iDiff;
3646	if (iDiff < pArgs->cMinVerifyTscTicks)
3647	pArgs->cMinVerifyTscTicks = iDiff;
3648	#endif
3649	if (iDiff < 0)
3650	{
3651	#ifdef TSCDELTA_VERIFY_WITH_STATS
3652	pArgs->iVerifyBadTscDiff = iDiff;
3653	#endif
3654	ASMAtomicWriteS32(&pArgs->rcVerify, VERR_OUT_OF_RANGE);
3655	break;
3656	}
3657	}
3658
3659	/* Done. */
3660	TSCDELTA_MASTER_KICK_OTHER_OUT_OF_AFTER(pMySync, pOtherSync);
3661	}
3662	else
3663	{
3664	/*
3665	* The worker, master leads.
3666	*/
3667	TSCDELTA_OTHER_SYNC_BEFORE(pMySync, pOtherSync, &fEFlags, pArgs);
3668
3669	for (i = 0; i < RT_ELEMENTS(pArgs->uWorker.Verify.auTscs); i += 2)
3670	{
3671	uint64_t uTsc;
3672
3673	/* Wait, Read and Kick #1. */
3674	TSCDELTA_DBG_START_LOOP();
3675	while (ASMAtomicReadU32(&pMySync->uSyncVar) == GIP_TSC_DELTA_SYNC2_GO)
3676	{
3677	TSCDELTA_DBG_CHECK_LOOP();
3678	ASMNopPause();
3679	}
3680	uTsc = ASMReadTSC();
3681	ASMAtomicWriteU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_GO_GO);
3682	ASMSerializeInstruction();
3683	pArgs->uWorker.Verify.auTscs[i] = uTsc;
3684
3685	/* Wait, Read and Kick #2. */
3686	TSCDELTA_DBG_START_LOOP();
3687	while (ASMAtomicReadU32(&pMySync->uSyncVar) == GIP_TSC_DELTA_SYNC2_GO_GO)
3688	{
3689	TSCDELTA_DBG_CHECK_LOOP();
3690	ASMNopPause();
3691	}
3692	uTsc = ASMReadTSC();
3693	ASMAtomicWriteU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_GO);
3694	ASMSerializeInstruction();
3695	pArgs->uWorker.Verify.auTscs[i + 1] = uTsc;
3696	}
3697
3698	TSCDELTA_OTHER_SYNC_AFTER(pMySync, pOtherSync, fEFlags);
3699	}
3700	return pArgs->rcVerify;
3701	}
3702
3703	/*
3704	* Timed out, please retry.
3705	*/
3706	ASMAtomicWriteS32(&pArgs->rcVerify, VERR_TRY_AGAIN);
3707	return VERR_TIMEOUT;
3708	}
3709
3710
3711
3712	/**
3713	* Handles the special abort procedure during synchronization setup in
3714	* supdrvTscMeasureDeltaCallbackUnwrapped().
3715	*
3716	* @returns 0 (dummy, ignored)
3717	* @param pArgs Pointer to argument/state data.
3718	* @param pMySync Pointer to my sync structure.
3719	* @param fIsMaster Set if we're the master, clear if worker.
3720	* @param fTimeout Set if it's a timeout.
3721	*/
3722	DECL_NO_INLINE(static, int)
3723	supdrvTscMeasureDeltaCallbackAbortSyncSetup(PSUPDRVGIPTSCDELTARGS pArgs, PSUPTSCDELTASYNC2 pMySync, bool fIsMaster, bool fTimeout)
3724	{
3725	PSUPTSCDELTASYNC2 volatile *ppMySync = fIsMaster ? &pArgs->pSyncMaster : &pArgs->pSyncWorker;
3726	PSUPTSCDELTASYNC2 volatile *ppOtherSync = fIsMaster ? &pArgs->pSyncWorker : &pArgs->pSyncMaster;
3727	TSCDELTA_DBG_VARS();
3728	RT_NOREF1(pMySync);
3729
3730	/*
3731	* Clear our sync pointer and make sure the abort flag is set.
3732	*/
3733	ASMAtomicWriteNullPtr(ppMySync);
3734	ASMAtomicWriteBool(&pArgs->fAbortSetup, true);
3735	if (fTimeout)
3736	ASMAtomicWriteBool(&pArgs->fTimedOut, true);
3737
3738	/*
3739	* Make sure the other party is out of there and won't be touching our
3740	* sync state again (would cause stack corruption).
3741	*/
3742	TSCDELTA_DBG_START_LOOP();
3743	while (ASMAtomicReadPtrT(ppOtherSync, PSUPTSCDELTASYNC2) != NULL)
3744	{
3745	ASMNopPause();
3746	ASMNopPause();
3747	ASMNopPause();
3748	TSCDELTA_DBG_CHECK_LOOP();
3749	}
3750
3751	return 0;
3752	}
3753
3754
3755	/**
3756	* This is used by supdrvTscMeasureInitialDeltas() to read the TSC on two CPUs
3757	* and compute the delta between them.
3758	*
3759	* To reduce code size a good when timeout handling was added, a dummy return
3760	* value had to be added (saves 1-3 lines per timeout case), thus this
3761	* 'Unwrapped' function and the dummy 0 return value.
3762	*
3763	* @returns 0 (dummy, ignored)
3764	* @param idCpu The CPU we are current scheduled on.
3765	* @param pArgs Pointer to a parameter package.
3766	*
3767	* @remarks Measuring TSC deltas between the CPUs is tricky because we need to
3768	* read the TSC at exactly the same time on both the master and the
3769	* worker CPUs. Due to DMA, bus arbitration, cache locality,
3770	* contention, SMI, pipelining etc. there is no guaranteed way of
3771	* doing this on x86 CPUs.
3772	*/
3773	static int supdrvTscMeasureDeltaCallbackUnwrapped(RTCPUID idCpu, PSUPDRVGIPTSCDELTARGS pArgs)
3774	{
3775	PSUPDRVDEVEXT pDevExt = pArgs->pDevExt;
3776	PSUPGIPCPU pGipCpuWorker = pArgs->pWorker;
3777	PSUPGIPCPU pGipCpuMaster = pArgs->pMaster;
3778	bool const fIsMaster = idCpu == pGipCpuMaster->idCpu;
3779	uint32_t iTry;
3780	PSUPTSCDELTASYNC2 volatile *ppMySync = fIsMaster ? &pArgs->pSyncMaster : &pArgs->pSyncWorker;
3781	PSUPTSCDELTASYNC2 volatile *ppOtherSync = fIsMaster ? &pArgs->pSyncWorker : &pArgs->pSyncMaster;
3782	SUPTSCDELTASYNC2 MySync;
3783	PSUPTSCDELTASYNC2 pOtherSync;
3784	int rc;
3785	TSCDELTA_DBG_VARS();
3786
3787	/* A bit of paranoia first. */
3788	if (!pGipCpuMaster \|\| !pGipCpuWorker)
3789	return 0;
3790
3791	/*
3792	* If the CPU isn't part of the measurement, return immediately.
3793	*/
3794	if ( !fIsMaster
3795	&& idCpu != pGipCpuWorker->idCpu)
3796	return 0;
3797
3798	/*
3799	* Set up my synchronization stuff and wait for the other party to show up.
3800	*
3801	* We don't wait forever since the other party may be off fishing (offline,
3802	* spinning with ints disables, whatever), we must play nice to the rest of
3803	* the system as this context generally isn't one in which we will get
3804	* preempted and we may hold up a number of lower priority interrupts.
3805	*/
3806	ASMAtomicWriteU32(&MySync.uSyncVar, GIP_TSC_DELTA_SYNC2_PRESTART_WAIT);
3807	ASMAtomicWritePtr(ppMySync, &MySync);
3808	MySync.uTscStart = ASMReadTSC();
3809	MySync.cMaxTscTicks = pArgs->cMaxTscTicks;
3810
3811	/* Look for the partner, might not be here yet... Special abort considerations. */
3812	iTry = 0;
3813	TSCDELTA_DBG_START_LOOP();
3814	while ((pOtherSync = ASMAtomicReadPtrT(ppOtherSync, PSUPTSCDELTASYNC2)) == NULL)
3815	{
3816	ASMNopPause();
3817	if ( ASMAtomicReadBool(&pArgs->fAbortSetup)
3818	\|\| !RTMpIsCpuOnline(fIsMaster ? pGipCpuWorker->idCpu : pGipCpuMaster->idCpu) )
3819	return supdrvTscMeasureDeltaCallbackAbortSyncSetup(pArgs, &MySync, fIsMaster, false /fTimeout/);
3820	if ( (iTry++ & 0xff) == 0
3821	&& ASMReadTSC() - MySync.uTscStart > pArgs->cMaxTscTicks)
3822	return supdrvTscMeasureDeltaCallbackAbortSyncSetup(pArgs, &MySync, fIsMaster, true /fTimeout/);
3823	TSCDELTA_DBG_CHECK_LOOP();
3824	ASMNopPause();
3825	}
3826
3827	/* I found my partner, waiting to be found... Special abort considerations. */
3828	if (fIsMaster)
3829	if (!ASMAtomicCmpXchgU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_READY, GIP_TSC_DELTA_SYNC2_PRESTART_WAIT)) /* parnaoia */
3830	return supdrvTscMeasureDeltaCallbackAbortSyncSetup(pArgs, &MySync, fIsMaster, false /fTimeout/);
3831
3832	iTry = 0;
3833	TSCDELTA_DBG_START_LOOP();
3834	while (ASMAtomicReadU32(&MySync.uSyncVar) == GIP_TSC_DELTA_SYNC2_PRESTART_WAIT)
3835	{
3836	ASMNopPause();
3837	if (ASMAtomicReadBool(&pArgs->fAbortSetup))
3838	return supdrvTscMeasureDeltaCallbackAbortSyncSetup(pArgs, &MySync, fIsMaster, false /fTimeout/);
3839	if ( (iTry++ & 0xff) == 0
3840	&& ASMReadTSC() - MySync.uTscStart > pArgs->cMaxTscTicks)
3841	{
3842	if ( fIsMaster
3843	&& !ASMAtomicCmpXchgU32(&MySync.uSyncVar, GIP_TSC_DELTA_SYNC2_PRESTART_ABORT, GIP_TSC_DELTA_SYNC2_PRESTART_WAIT))
3844	break; /* race #1: slave has moved on, handle timeout in loop instead. */
3845	return supdrvTscMeasureDeltaCallbackAbortSyncSetup(pArgs, &MySync, fIsMaster, true /fTimeout/);
3846	}
3847	TSCDELTA_DBG_CHECK_LOOP();
3848	}
3849
3850	if (!fIsMaster)
3851	if (!ASMAtomicCmpXchgU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_READY, GIP_TSC_DELTA_SYNC2_PRESTART_WAIT)) /* race #1 */
3852	return supdrvTscMeasureDeltaCallbackAbortSyncSetup(pArgs, &MySync, fIsMaster, false /fTimeout/);
3853
3854	/** @todo Add a resumable state to pArgs so we don't waste time if we time
3855	* out or something. Timeouts are legit, any of the two CPUs may get
3856	* interrupted. */
3857
3858	/*
3859	* Start by seeing if we have a zero delta between the two CPUs.
3860	* This should normally be the case.
3861	*/
3862	rc = supdrvTscDeltaVerify(pArgs, &MySync, pOtherSync, fIsMaster, GIP_TSC_DELTA_INITIAL_MASTER_VALUE);
3863	if (RT_SUCCESS(rc))
3864	{
3865	if (fIsMaster)
3866	{
3867	ASMAtomicWriteS64(&pGipCpuWorker->i64TSCDelta, GIP_TSC_DELTA_INITIAL_MASTER_VALUE);
3868	RTCpuSetDelByIndex(&pDevExt->TscDeltaCpuSet, pGipCpuWorker->iCpuSet);
3869	RTCpuSetAddByIndex(&pDevExt->TscDeltaObtainedCpuSet, pGipCpuWorker->iCpuSet);
3870	}
3871	}
3872	/*
3873	* If the verification didn't time out, do regular delta measurements.
3874	* We retry this until we get a reasonable value.
3875	*/
3876	else if (rc != VERR_TIMEOUT)
3877	{
3878	Assert(pGipCpuWorker->i64TSCDelta == INT64_MAX);
3879	for (iTry = 0; iTry < 12; iTry++)
3880	{
3881	/*
3882	* Check the state before we start.
3883	*/
3884	uint32_t u32Tmp = ASMAtomicReadU32(&MySync.uSyncVar);
3885	if ( u32Tmp != GIP_TSC_DELTA_SYNC2_READY
3886	&& (fIsMaster \|\| u32Tmp != GIP_TSC_DELTA_SYNC2_STEADY) /* worker may be late prepping for the next round */ )
3887	{
3888	TSCDELTA_DBG_SYNC_MSG(("sync/loop/%s: #0 iTry=%u MyState=%#x\n", fIsMaster ? "master" : "worker", iTry, u32Tmp));
3889	break;
3890	}
3891
3892	/*
3893	* Do the measurements.
3894	*/
3895	#ifdef GIP_TSC_DELTA_METHOD_1
3896	supdrvTscDeltaMethod1Loop(pArgs, &MySync, pOtherSync, fIsMaster, iTry);
3897	#elif defined(GIP_TSC_DELTA_METHOD_2)
3898	supdrvTscDeltaMethod2Loop(pArgs, &MySync, pOtherSync, fIsMaster, iTry);
3899	#else
3900	# error "huh??"
3901	#endif
3902
3903	/*
3904	* Check the state.
3905	*/
3906	u32Tmp = ASMAtomicReadU32(&MySync.uSyncVar);
3907	if ( u32Tmp != GIP_TSC_DELTA_SYNC2_READY
3908	&& (fIsMaster \|\| u32Tmp != GIP_TSC_DELTA_SYNC2_STEADY) /* worker may be late prepping for the next round */ )
3909	{
3910	if (fIsMaster)
3911	TSCDELTA_DBG_SYNC_MSG(("sync/loop/master: #1 iTry=%u MyState=%#x\n", iTry, u32Tmp));
3912	else
3913	TSCDELTA_DBG_SYNC_MSG2(("sync/loop/worker: #1 iTry=%u MyState=%#x\n", iTry, u32Tmp));
3914	break;
3915	}
3916
3917	/*
3918	* Success? If so, stop trying. Master decides.
3919	*/
3920	if (fIsMaster)
3921	{
3922	if (pGipCpuWorker->i64TSCDelta != INT64_MAX)
3923	{
3924	RTCpuSetDelByIndex(&pDevExt->TscDeltaCpuSet, pGipCpuWorker->iCpuSet);
3925	RTCpuSetAddByIndex(&pDevExt->TscDeltaObtainedCpuSet, pGipCpuWorker->iCpuSet);
3926	TSCDELTA_DBG_SYNC_MSG2(("sync/loop/master: #9 iTry=%u MyState=%#x\n", iTry, MySync.uSyncVar));
3927	break;
3928	}
3929	}
3930	}
3931	if (fIsMaster)
3932	pArgs->iTry = iTry;
3933	}
3934
3935	/*
3936	* End the synchronization dance. We tell the other that we're done,
3937	* then wait for the same kind of reply.
3938	*/
3939	ASMAtomicWriteU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_FINAL);
3940	ASMAtomicWriteNullPtr(ppMySync);
3941	iTry = 0;
3942	TSCDELTA_DBG_START_LOOP();
3943	while (ASMAtomicReadU32(&MySync.uSyncVar) != GIP_TSC_DELTA_SYNC2_FINAL)
3944	{
3945	iTry++;
3946	if ( iTry == 0
3947	&& !RTMpIsCpuOnline(fIsMaster ? pGipCpuWorker->idCpu : pGipCpuMaster->idCpu))
3948	break; /* this really shouldn't happen. */
3949	TSCDELTA_DBG_CHECK_LOOP();
3950	ASMNopPause();
3951	}
3952
3953	/*
3954	* Collect some runtime stats.
3955	*/
3956	if (fIsMaster)
3957	pArgs->cElapsedMasterTscTicks = ASMReadTSC() - MySync.uTscStart;
3958	else
3959	pArgs->cElapsedWorkerTscTicks = ASMReadTSC() - MySync.uTscStart;
3960	return 0;
3961	}
3962
3963	/**
3964	* Callback used by supdrvTscMeasureInitialDeltas() to read the TSC on two CPUs
3965	* and compute the delta between them.
3966	*
3967	* @param idCpu The CPU we are current scheduled on.
3968	* @param pvUser1 Pointer to a parameter package (SUPDRVGIPTSCDELTARGS).
3969	* @param pvUser2 Unused.
3970	*/
3971	static DECLCALLBACK(void) supdrvTscMeasureDeltaCallback(RTCPUID idCpu, void pvUser1, void pvUser2)
3972	{
3973	supdrvTscMeasureDeltaCallbackUnwrapped(idCpu, (PSUPDRVGIPTSCDELTARGS)pvUser1);
3974	RT_NOREF1(pvUser2);
3975	}
3976
3977
3978	/**
3979	* Measures the TSC delta between the master GIP CPU and one specified worker
3980	* CPU.
3981	*
3982	* @returns VBox status code.
3983	* @retval VERR_SUPDRV_TSC_DELTA_MEASUREMENT_FAILED on pure measurement
3984	* failure.
3985	* @param pDevExt Pointer to the device instance data.
3986	* @param idxWorker The index of the worker CPU from the GIP's array of
3987	* CPUs.
3988	*
3989	* @remarks This must be called with preemption enabled!
3990	*/
3991	static int supdrvTscMeasureDeltaOne(PSUPDRVDEVEXT pDevExt, uint32_t idxWorker)
3992	{
3993	int rc;
3994	int rc2;
3995	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
3996	RTCPUID idMaster = pDevExt->idGipMaster;
3997	PSUPGIPCPU pGipCpuWorker = &pGip->aCPUs[idxWorker];
3998	PSUPGIPCPU pGipCpuMaster;
3999	uint32_t iGipCpuMaster;
4000	uint32_t u32Tmp;
4001
4002	/* Validate input a bit. */
4003	AssertReturn(pGip, VERR_INVALID_PARAMETER);
4004	Assert(pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED);
4005	Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4006
4007	/*
4008	* Don't attempt measuring the delta for the GIP master.
4009	*/
4010	if (pGipCpuWorker->idCpu == idMaster)
4011	{
4012	if (pGipCpuWorker->i64TSCDelta == INT64_MAX) /* This shouldn't happen, but just in case. */
4013	ASMAtomicWriteS64(&pGipCpuWorker->i64TSCDelta, GIP_TSC_DELTA_INITIAL_MASTER_VALUE);
4014	return VINF_SUCCESS;
4015	}
4016
4017	/*
4018	* One measurement at a time, at least for now. We might be using
4019	* broadcast IPIs so, so be nice to the rest of the system.
4020	*/
4021	#ifdef SUPDRV_USE_MUTEX_FOR_GIP
4022	rc = RTSemMutexRequest(pDevExt->mtxTscDelta, RT_INDEFINITE_WAIT);
4023	#else
4024	rc = RTSemFastMutexRequest(pDevExt->mtxTscDelta);
4025	#endif
4026	if (RT_FAILURE(rc))
4027	return rc;
4028
4029	/*
4030	* If the CPU has hyper-threading and the APIC IDs of the master and worker are adjacent,
4031	* try pick a different master. (This fudge only works with multi core systems.)
4032	* ASSUMES related threads have adjacent APIC IDs. ASSUMES two threads per core.
4033	*
4034	* We skip this on AMDs for now as their HTT is different from Intel's and
4035	* it doesn't seem to have any favorable effect on the results.
4036	*
4037	* If the master is offline, we need a new master too, so share the code.
4038	*/
4039	iGipCpuMaster = supdrvGipFindCpuIndexForCpuId(pGip, idMaster);
4040	AssertReturn(iGipCpuMaster < pGip->cCpus, VERR_INVALID_CPU_ID);
4041	pGipCpuMaster = &pGip->aCPUs[iGipCpuMaster];
4042	if ( ( (pGipCpuMaster->idApic & ~1) == (pGipCpuWorker->idApic & ~1)
4043	&& pGip->cOnlineCpus > 2
4044	&& ASMHasCpuId()
4045	&& RTX86IsValidStdRange(ASMCpuId_EAX(0))
4046	&& (ASMCpuId_EDX(1) & X86_CPUID_FEATURE_EDX_HTT)
4047	&& ( !ASMIsAmdCpu()
4048	\|\| RTX86GetCpuFamily(u32Tmp = ASMCpuId_EAX(1)) > 0x15
4049	\|\| ( RTX86GetCpuFamily(u32Tmp) == 0x15 /* Piledriver+, not bulldozer (FX-4150 didn't like it). */
4050	&& RTX86GetCpuModelAMD(u32Tmp) >= 0x02) ) )
4051	\|\| !RTMpIsCpuOnline(idMaster) )
4052	{
4053	uint32_t i;
4054	for (i = 0; i < pGip->cCpus; i++)
4055	if ( i != iGipCpuMaster
4056	&& i != idxWorker
4057	&& pGip->aCPUs[i].enmState == SUPGIPCPUSTATE_ONLINE
4058	&& pGip->aCPUs[i].i64TSCDelta != INT64_MAX
4059	&& pGip->aCPUs[i].idCpu != NIL_RTCPUID
4060	&& pGip->aCPUs[i].idCpu != idMaster /* paranoia starts here... */
4061	&& pGip->aCPUs[i].idCpu != pGipCpuWorker->idCpu
4062	&& pGip->aCPUs[i].idApic != pGipCpuWorker->idApic
4063	&& pGip->aCPUs[i].idApic != pGipCpuMaster->idApic
4064	&& RTMpIsCpuOnline(pGip->aCPUs[i].idCpu))
4065	{
4066	iGipCpuMaster = i;
4067	pGipCpuMaster = &pGip->aCPUs[i];
4068	idMaster = pGipCpuMaster->idCpu;
4069	break;
4070	}
4071	}
4072
4073	if (RTCpuSetIsMemberByIndex(&pGip->OnlineCpuSet, pGipCpuWorker->iCpuSet))
4074	{
4075	/*
4076	* Initialize data package for the RTMpOnPair callback.
4077	*/
4078	PSUPDRVGIPTSCDELTARGS pArgs = (PSUPDRVGIPTSCDELTARGS)RTMemAllocZ(sizeof(*pArgs));
4079	if (pArgs)
4080	{
4081	pArgs->pWorker = pGipCpuWorker;
4082	pArgs->pMaster = pGipCpuMaster;
4083	pArgs->pDevExt = pDevExt;
4084	pArgs->pSyncMaster = NULL;
4085	pArgs->pSyncWorker = NULL;
4086	pArgs->cMaxTscTicks = ASMAtomicReadU64(&pGip->u64CpuHz) / 512; /* 1953 us */
4087
4088	/*
4089	* Do the RTMpOnPair call. We reset i64TSCDelta first so we
4090	* and supdrvTscMeasureDeltaCallback can use it as a success check.
4091	*/
4092	/** @todo Store the i64TSCDelta result in pArgs first? Perhaps deals with
4093	* that when doing the restart loop reorg. */
4094	ASMAtomicWriteS64(&pGipCpuWorker->i64TSCDelta, INT64_MAX);
4095	rc = RTMpOnPair(pGipCpuMaster->idCpu, pGipCpuWorker->idCpu, RTMPON_F_CONCURRENT_EXEC,
4096	supdrvTscMeasureDeltaCallback, pArgs, NULL);
4097	if (RT_SUCCESS(rc))
4098	{
4099	#if 0
4100	SUPR0Printf("mponpair ticks: %9llu %9llu max: %9llu iTry: %u%s\n", pArgs->cElapsedMasterTscTicks,
4101	pArgs->cElapsedWorkerTscTicks, pArgs->cMaxTscTicks, pArgs->iTry,
4102	pArgs->fTimedOut ? " timed out" :"");
4103	#endif
4104	#if 0
4105	SUPR0Printf("rcVerify=%d iVerifyBadTscDiff=%lld cMinVerifyTscTicks=%lld cMaxVerifyTscTicks=%lld\n",
4106	pArgs->rcVerify, pArgs->iVerifyBadTscDiff, pArgs->cMinVerifyTscTicks, pArgs->cMaxVerifyTscTicks);
4107	#endif
4108	if (RT_LIKELY(pGipCpuWorker->i64TSCDelta != INT64_MAX))
4109	{
4110	/*
4111	* Work the TSC delta applicability rating. It starts
4112	* optimistic in supdrvGipInit, we downgrade it here.
4113	*/
4114	SUPGIPUSETSCDELTA enmRating;
4115	if ( pGipCpuWorker->i64TSCDelta > GIP_TSC_DELTA_THRESHOLD_ROUGHLY_ZERO
4116	\|\| pGipCpuWorker->i64TSCDelta < -GIP_TSC_DELTA_THRESHOLD_ROUGHLY_ZERO)
4117	enmRating = SUPGIPUSETSCDELTA_NOT_ZERO;
4118	else if ( pGipCpuWorker->i64TSCDelta > GIP_TSC_DELTA_THRESHOLD_PRACTICALLY_ZERO
4119	\|\| pGipCpuWorker->i64TSCDelta < -GIP_TSC_DELTA_THRESHOLD_PRACTICALLY_ZERO)
4120	enmRating = SUPGIPUSETSCDELTA_ROUGHLY_ZERO;
4121	else
4122	enmRating = SUPGIPUSETSCDELTA_PRACTICALLY_ZERO;
4123	if (pGip->enmUseTscDelta < enmRating)
4124	{
4125	AssertCompile(sizeof(pGip->enmUseTscDelta) == sizeof(uint32_t));
4126	ASMAtomicWriteU32((uint32_t volatile *)&pGip->enmUseTscDelta, enmRating);
4127	}
4128	}
4129	else
4130	rc = VERR_SUPDRV_TSC_DELTA_MEASUREMENT_FAILED;
4131	}
4132	/** @todo return try-again if we get an offline CPU error. */
4133
4134	RTMemFree(pArgs);
4135	}
4136	else
4137	rc = VERR_NO_MEMORY;
4138	}
4139	else
4140	rc = VERR_CPU_OFFLINE;
4141
4142	/*
4143	* We're done now.
4144	*/
4145	#ifdef SUPDRV_USE_MUTEX_FOR_GIP
4146	rc2 = RTSemMutexRelease(pDevExt->mtxTscDelta); AssertRC(rc2);
4147	#else
4148	rc2 = RTSemFastMutexRelease(pDevExt->mtxTscDelta); AssertRC(rc2);
4149	#endif
4150	return rc;
4151	}
4152
4153
4154	/**
4155	* Resets the TSC-delta related TSC samples and optionally the deltas
4156	* themselves.
4157	*
4158	* @param pDevExt Pointer to the device instance data.
4159	* @param fResetTscDeltas Whether the TSC-deltas are also to be reset.
4160	*
4161	* @remarks This might be called while holding a spinlock!
4162	*/
4163	static void supdrvTscResetSamples(PSUPDRVDEVEXT pDevExt, bool fResetTscDeltas)
4164	{
4165	unsigned iCpu;
4166	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
4167	for (iCpu = 0; iCpu < pGip->cCpus; iCpu++)
4168	{
4169	PSUPGIPCPU pGipCpu = &pGip->aCPUs[iCpu];
4170	ASMAtomicWriteU64(&pGipCpu->u64TSCSample, GIP_TSC_DELTA_RSVD);
4171	if (fResetTscDeltas)
4172	{
4173	RTCpuSetDelByIndex(&pDevExt->TscDeltaObtainedCpuSet, pGipCpu->iCpuSet);
4174	ASMAtomicWriteS64(&pGipCpu->i64TSCDelta, INT64_MAX);
4175	}
4176	}
4177	}
4178
4179
4180	/**
4181	* Picks an online CPU as the master TSC for TSC-delta computations.
4182	*
4183	* @returns VBox status code.
4184	* @param pDevExt Pointer to the device instance data.
4185	* @param pidxMaster Where to store the CPU array index of the chosen
4186	* master. Optional, can be NULL.
4187	*/
4188	static int supdrvTscPickMaster(PSUPDRVDEVEXT pDevExt, uint32_t *pidxMaster)
4189	{
4190	/*
4191	* Pick the first CPU online as the master TSC and make it the new GIP master based
4192	* on the APIC ID.
4193	*
4194	* Technically we can simply use "idGipMaster" but doing this gives us master as CPU 0
4195	* in most cases making it nicer/easier for comparisons. It is safe to update the GIP
4196	* master as this point since the sync/async timer isn't created yet.
4197	*/
4198	unsigned iCpu;
4199	uint32_t idxMaster = UINT32_MAX;
4200	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
4201	for (iCpu = 0; iCpu < RT_ELEMENTS(pGip->aiCpuFromApicId); iCpu++)
4202	{
4203	uint16_t idxCpu = pGip->aiCpuFromApicId[iCpu];
4204	if (idxCpu != UINT16_MAX)
4205	{
4206	PSUPGIPCPU pGipCpu = &pGip->aCPUs[idxCpu];
4207	if (RTCpuSetIsMemberByIndex(&pGip->OnlineCpuSet, pGipCpu->iCpuSet))
4208	{
4209	idxMaster = idxCpu;
4210	pGipCpu->i64TSCDelta = GIP_TSC_DELTA_INITIAL_MASTER_VALUE;
4211	ASMAtomicWriteSize(&pDevExt->idGipMaster, pGipCpu->idCpu);
4212	if (pidxMaster)
4213	*pidxMaster = idxMaster;
4214	return VINF_SUCCESS;
4215	}
4216	}
4217	}
4218	return VERR_CPU_OFFLINE;
4219	}
4220
4221
4222	/**
4223	* Performs the initial measurements of the TSC deltas between CPUs.
4224	*
4225	* This is called by supdrvGipCreate(), supdrvGipPowerNotificationCallback() or
4226	* triggered by it if threaded.
4227	*
4228	* @returns VBox status code.
4229	* @param pDevExt Pointer to the device instance data.
4230	*
4231	* @remarks Must be called only after supdrvGipInitOnCpu() as this function uses
4232	* idCpu, GIP's online CPU set which are populated in
4233	* supdrvGipInitOnCpu().
4234	*/
4235	static int supdrvTscMeasureInitialDeltas(PSUPDRVDEVEXT pDevExt)
4236	{
4237	PSUPGIPCPU pGipCpuMaster;
4238	unsigned iCpu;
4239	unsigned iOddEven;
4240	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
4241	uint32_t idxMaster = UINT32_MAX;
4242	uint32_t cMpOnOffEvents = ASMAtomicReadU32(&pDevExt->cMpOnOffEvents);
4243
4244	Assert(pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED);
4245	supdrvTscResetSamples(pDevExt, true /* fClearDeltas */);
4246	int rc = supdrvTscPickMaster(pDevExt, &idxMaster);
4247	if (RT_FAILURE(rc))
4248	{
4249	SUPR0Printf("Failed to pick a CPU master for TSC-delta measurements rc=%Rrc\n", rc);
4250	return rc;
4251	}
4252	AssertReturn(idxMaster < pGip->cCpus, VERR_INVALID_CPU_INDEX);
4253	pGipCpuMaster = &pGip->aCPUs[idxMaster];
4254	Assert(pDevExt->idGipMaster == pGipCpuMaster->idCpu);
4255
4256	/*
4257	* If there is only a single CPU online we have nothing to do.
4258	*/
4259	if (pGip->cOnlineCpus <= 1)
4260	{
4261	AssertReturn(pGip->cOnlineCpus > 0, VERR_INTERNAL_ERROR_5);
4262	return VINF_SUCCESS;
4263	}
4264
4265	/*
4266	* Loop thru the GIP CPU array and get deltas for each CPU (except the
4267	* master). We do the CPUs with the even numbered APIC IDs first so that
4268	* we've got alternative master CPUs to pick from on hyper-threaded systems.
4269	*/
4270	for (iOddEven = 0; iOddEven < 2; iOddEven++)
4271	{
4272	for (iCpu = 0; iCpu < pGip->cCpus; iCpu++)
4273	{
4274	PSUPGIPCPU pGipCpuWorker = &pGip->aCPUs[iCpu];
4275	if ( iCpu != idxMaster
4276	&& (iOddEven > 0 \|\| (pGipCpuWorker->idApic & 1) == 0)
4277	&& RTCpuSetIsMemberByIndex(&pDevExt->TscDeltaCpuSet, pGipCpuWorker->iCpuSet))
4278	{
4279	rc = supdrvTscMeasureDeltaOne(pDevExt, iCpu);
4280	if (RT_FAILURE(rc))
4281	{
4282	SUPR0Printf("supdrvTscMeasureDeltaOne failed. rc=%d CPU[%u].idCpu=%u Master[%u].idCpu=%u\n", rc, iCpu,
4283	pGipCpuWorker->idCpu, idxMaster, pDevExt->idGipMaster, pGipCpuMaster->idCpu);
4284	break;
4285	}
4286
4287	if (ASMAtomicReadU32(&pDevExt->cMpOnOffEvents) != cMpOnOffEvents)
4288	{
4289	SUPR0Printf("One or more CPUs transitioned between online & offline states. I'm confused, retry...\n");
4290	rc = VERR_TRY_AGAIN;
4291	break;
4292	}
4293	}
4294	}
4295	}
4296
4297	return rc;
4298	}
4299
4300
4301	#ifdef SUPDRV_USE_TSC_DELTA_THREAD
4302
4303	/**
4304	* Switches the TSC-delta measurement thread into the butchered state.
4305	*
4306	* @returns VBox status code.
4307	* @param pDevExt Pointer to the device instance data.
4308	* @param fSpinlockHeld Whether the TSC-delta spinlock is held or not.
4309	* @param pszFailed An error message to log.
4310	* @param rcFailed The error code to exit the thread with.
4311	*/
4312	static int supdrvTscDeltaThreadButchered(PSUPDRVDEVEXT pDevExt, bool fSpinlockHeld, const char *pszFailed, int rcFailed)
4313	{
4314	if (!fSpinlockHeld)
4315	RTSpinlockAcquire(pDevExt->hTscDeltaSpinlock);
4316
4317	pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_Butchered;
4318	RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
4319	OSDBGPRINT(("supdrvTscDeltaThreadButchered: %s. rc=%Rrc\n", pszFailed, rcFailed));
4320	return rcFailed;
4321	}
4322
4323
4324	/**
4325	* The TSC-delta measurement thread.
4326	*
4327	* @returns VBox status code.
4328	* @param hThread The thread handle.
4329	* @param pvUser Opaque pointer to the device instance data.
4330	*/
4331	static DECLCALLBACK(int) supdrvTscDeltaThread(RTTHREAD hThread, void *pvUser)
4332	{
4333	PSUPDRVDEVEXT pDevExt = (PSUPDRVDEVEXT)pvUser;
4334	int rc = VERR_INTERNAL_ERROR_2;
4335	for (;;)
4336	{
4337	/*
4338	* Switch on the current state.
4339	*/
4340	SUPDRVTSCDELTATHREADSTATE enmState;
4341	RTSpinlockAcquire(pDevExt->hTscDeltaSpinlock);
4342	enmState = pDevExt->enmTscDeltaThreadState;
4343	switch (enmState)
4344	{
4345	case kTscDeltaThreadState_Creating:
4346	{
4347	pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_Listening;
4348	rc = RTSemEventSignal(pDevExt->hTscDeltaEvent);
4349	if (RT_FAILURE(rc))
4350	return supdrvTscDeltaThreadButchered(pDevExt, true /* fSpinlockHeld */, "RTSemEventSignal", rc);
4351	RT_FALL_THRU();
4352	}
4353
4354	case kTscDeltaThreadState_Listening:
4355	{
4356	RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
4357
4358	/*
4359	* Linux counts uninterruptible sleeps as load, hence we shall do a
4360	* regular, interruptible sleep here and ignore wake ups due to signals.
4361	* See task_contributes_to_load() in include/linux/sched.h in the Linux sources.
4362	*/
4363	rc = RTThreadUserWaitNoResume(hThread, pDevExt->cMsTscDeltaTimeout);
4364	if ( RT_FAILURE(rc)
4365	&& rc != VERR_TIMEOUT
4366	&& rc != VERR_INTERRUPTED)
4367	return supdrvTscDeltaThreadButchered(pDevExt, false /* fSpinlockHeld */, "RTThreadUserWait", rc);
4368	RTThreadUserReset(hThread);
4369	break;
4370	}
4371
4372	case kTscDeltaThreadState_WaitAndMeasure:
4373	{
4374	pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_Measuring;
4375	rc = RTSemEventSignal(pDevExt->hTscDeltaEvent); /* (Safe on windows as long as spinlock isn't IRQ safe.) */
4376	if (RT_FAILURE(rc))
4377	return supdrvTscDeltaThreadButchered(pDevExt, true /* fSpinlockHeld */, "RTSemEventSignal", rc);
4378	RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
4379	RTThreadSleep(1);
4380	RT_FALL_THRU();
4381	}
4382
4383	case kTscDeltaThreadState_Measuring:
4384	{
4385	if (pDevExt->fTscThreadRecomputeAllDeltas)
4386	{
4387	int cTries = 8;
4388	int cMsWaitPerTry = 10;
4389	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
4390	Assert(pGip);
4391	do
4392	{
4393	RTCpuSetCopy(&pDevExt->TscDeltaCpuSet, &pGip->OnlineCpuSet);
4394	rc = supdrvTscMeasureInitialDeltas(pDevExt);
4395	if ( RT_SUCCESS(rc)
4396	\|\| ( RT_FAILURE(rc)
4397	&& rc != VERR_TRY_AGAIN
4398	&& rc != VERR_CPU_OFFLINE))
4399	{
4400	break;
4401	}
4402	RTThreadSleep(cMsWaitPerTry);
4403	} while (cTries-- > 0);
4404	pDevExt->fTscThreadRecomputeAllDeltas = false;
4405	}
4406	else
4407	{
4408	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
4409	unsigned iCpu;
4410
4411	/* Measure TSC-deltas only for the CPUs that are in the set. */
4412	rc = VINF_SUCCESS;
4413	for (iCpu = 0; iCpu < pGip->cCpus; iCpu++)
4414	{
4415	PSUPGIPCPU pGipCpuWorker = &pGip->aCPUs[iCpu];
4416	if (RTCpuSetIsMemberByIndex(&pDevExt->TscDeltaCpuSet, pGipCpuWorker->iCpuSet))
4417	{
4418	if (pGipCpuWorker->i64TSCDelta == INT64_MAX)
4419	{
4420	int rc2 = supdrvTscMeasureDeltaOne(pDevExt, iCpu);
4421	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
4422	rc = rc2;
4423	}
4424	else
4425	{
4426	/*
4427	* The thread/someone must've called SUPR0TscDeltaMeasureBySetIndex(),
4428	* mark the delta as fine to get the timer thread off our back.
4429	*/
4430	RTCpuSetDelByIndex(&pDevExt->TscDeltaCpuSet, pGipCpuWorker->iCpuSet);
4431	RTCpuSetAddByIndex(&pDevExt->TscDeltaObtainedCpuSet, pGipCpuWorker->iCpuSet);
4432	}
4433	}
4434	}
4435	}
4436	RTSpinlockAcquire(pDevExt->hTscDeltaSpinlock);
4437	if (pDevExt->enmTscDeltaThreadState == kTscDeltaThreadState_Measuring)
4438	pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_Listening;
4439	RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
4440	Assert(rc != VERR_NOT_AVAILABLE); /* VERR_NOT_AVAILABLE is used as init value, see supdrvTscDeltaThreadInit(). */
4441	ASMAtomicWriteS32(&pDevExt->rcTscDelta, rc);
4442	break;
4443	}
4444
4445	case kTscDeltaThreadState_Terminating:
4446	pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_Destroyed;
4447	RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
4448	return VINF_SUCCESS;
4449
4450	case kTscDeltaThreadState_Butchered:
4451	default:
4452	return supdrvTscDeltaThreadButchered(pDevExt, true /* fSpinlockHeld */, "Invalid state", VERR_INVALID_STATE);
4453	}
4454	}
4455	/* not reached */
4456	}
4457
4458
4459	/**
4460	* Waits for the TSC-delta measurement thread to respond to a state change.
4461	*
4462	* @returns VINF_SUCCESS on success, VERR_TIMEOUT if it doesn't respond in time,
4463	* other error code on internal error.
4464	*
4465	* @param pDevExt The device instance data.
4466	* @param enmCurState The current state.
4467	* @param enmNewState The new state we're waiting for it to enter.
4468	*/
4469	static int supdrvTscDeltaThreadWait(PSUPDRVDEVEXT pDevExt, SUPDRVTSCDELTATHREADSTATE enmCurState,
4470	SUPDRVTSCDELTATHREADSTATE enmNewState)
4471	{
4472	SUPDRVTSCDELTATHREADSTATE enmActualState;
4473	int rc;
4474
4475	/*
4476	* Wait a short while for the expected state transition.
4477	*/
4478	RTSemEventWait(pDevExt->hTscDeltaEvent, RT_MS_1SEC);
4479	RTSpinlockAcquire(pDevExt->hTscDeltaSpinlock);
4480	enmActualState = pDevExt->enmTscDeltaThreadState;
4481	if (enmActualState == enmNewState)
4482	{
4483	RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
4484	rc = VINF_SUCCESS;
4485	}
4486	else if (enmActualState == enmCurState)
4487	{
4488	/*
4489	* Wait longer if the state has not yet transitioned to the one we want.
4490	*/
4491	RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
4492	rc = RTSemEventWait(pDevExt->hTscDeltaEvent, 50 * RT_MS_1SEC);
4493	if ( RT_SUCCESS(rc)
4494	\|\| rc == VERR_TIMEOUT)
4495	{
4496	/*
4497	* Check the state whether we've succeeded.
4498	*/
4499	RTSpinlockAcquire(pDevExt->hTscDeltaSpinlock);
4500	enmActualState = pDevExt->enmTscDeltaThreadState;
4501	RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
4502	if (enmActualState == enmNewState)
4503	rc = VINF_SUCCESS;
4504	else if (enmActualState == enmCurState)
4505	{
4506	rc = VERR_TIMEOUT;
4507	OSDBGPRINT(("supdrvTscDeltaThreadWait: timed out state transition. enmActualState=%d enmNewState=%d\n",
4508	enmActualState, enmNewState));
4509	}
4510	else
4511	{
4512	rc = VERR_INTERNAL_ERROR;
4513	OSDBGPRINT(("supdrvTscDeltaThreadWait: invalid state transition from %d to %d, expected %d\n", enmCurState,
4514	enmActualState, enmNewState));
4515	}
4516	}
4517	else
4518	OSDBGPRINT(("supdrvTscDeltaThreadWait: RTSemEventWait failed. rc=%Rrc\n", rc));
4519	}
4520	else
4521	{
4522	RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
4523	OSDBGPRINT(("supdrvTscDeltaThreadWait: invalid state %d when transitioning from %d to %d\n",
4524	enmActualState, enmCurState, enmNewState));
4525	rc = VERR_INTERNAL_ERROR;
4526	}
4527
4528	return rc;
4529	}
4530
4531
4532	/**
4533	* Signals the TSC-delta thread to start measuring TSC-deltas.
4534	*
4535	* @param pDevExt Pointer to the device instance data.
4536	* @param fForceAll Force re-calculating TSC-deltas on all CPUs.
4537	*/
4538	static void supdrvTscDeltaThreadStartMeasurement(PSUPDRVDEVEXT pDevExt, bool fForceAll)
4539	{
4540	if (pDevExt->hTscDeltaThread != NIL_RTTHREAD)
4541	{
4542	RTSpinlockAcquire(pDevExt->hTscDeltaSpinlock);
4543	if ( pDevExt->enmTscDeltaThreadState == kTscDeltaThreadState_Listening
4544	\|\| pDevExt->enmTscDeltaThreadState == kTscDeltaThreadState_Measuring)
4545	{
4546	pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_WaitAndMeasure;
4547	if (fForceAll)
4548	pDevExt->fTscThreadRecomputeAllDeltas = true;
4549	}
4550	else if ( pDevExt->enmTscDeltaThreadState == kTscDeltaThreadState_WaitAndMeasure
4551	&& fForceAll)
4552	pDevExt->fTscThreadRecomputeAllDeltas = true;
4553	RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
4554	RTThreadUserSignal(pDevExt->hTscDeltaThread);
4555	}
4556	}
4557
4558
4559	/**
4560	* Terminates the actual thread running supdrvTscDeltaThread().
4561	*
4562	* This is an internal worker function for supdrvTscDeltaThreadInit() and
4563	* supdrvTscDeltaTerm().
4564	*
4565	* @param pDevExt Pointer to the device instance data.
4566	*/
4567	static void supdrvTscDeltaThreadTerminate(PSUPDRVDEVEXT pDevExt)
4568	{
4569	int rc;
4570	RTSpinlockAcquire(pDevExt->hTscDeltaSpinlock);
4571	pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_Terminating;
4572	RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
4573	RTThreadUserSignal(pDevExt->hTscDeltaThread);
4574	rc = RTThreadWait(pDevExt->hTscDeltaThread, 50 * RT_MS_1SEC, NULL /* prc */);
4575	if (RT_FAILURE(rc))
4576	{
4577	/* Signal a few more times before giving up. */
4578	int cTriesLeft = 5;
4579	while (--cTriesLeft > 0)
4580	{
4581	RTThreadUserSignal(pDevExt->hTscDeltaThread);
4582	rc = RTThreadWait(pDevExt->hTscDeltaThread, 2 * RT_MS_1SEC, NULL /* prc */);
4583	if (rc != VERR_TIMEOUT)
4584	break;
4585	}
4586	}
4587	}
4588
4589
4590	/**
4591	* Initializes and spawns the TSC-delta measurement thread.
4592	*
4593	* A thread is required for servicing re-measurement requests from events like
4594	* CPUs coming online, suspend/resume etc. as it cannot be done synchronously
4595	* under all contexts on all OSs.
4596	*
4597	* @returns VBox status code.
4598	* @param pDevExt Pointer to the device instance data.
4599	*
4600	* @remarks Must only be called -after- initializing GIP and setting up MP
4601	* notifications!
4602	*/
4603	static int supdrvTscDeltaThreadInit(PSUPDRVDEVEXT pDevExt)
4604	{
4605	int rc;
4606	Assert(pDevExt->pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED);
4607	rc = RTSpinlockCreate(&pDevExt->hTscDeltaSpinlock, RTSPINLOCK_FLAGS_INTERRUPT_UNSAFE, "VBoxTscSpnLck");
4608	if (RT_SUCCESS(rc))
4609	{
4610	rc = RTSemEventCreate(&pDevExt->hTscDeltaEvent);
4611	if (RT_SUCCESS(rc))
4612	{
4613	pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_Creating;
4614	pDevExt->cMsTscDeltaTimeout = 60000;
4615	rc = RTThreadCreate(&pDevExt->hTscDeltaThread, supdrvTscDeltaThread, pDevExt, 0 /* cbStack */,
4616	RTTHREADTYPE_DEFAULT, RTTHREADFLAGS_WAITABLE, "VBoxTscThread");
4617	if (RT_SUCCESS(rc))
4618	{
4619	rc = supdrvTscDeltaThreadWait(pDevExt, kTscDeltaThreadState_Creating, kTscDeltaThreadState_Listening);
4620	if (RT_SUCCESS(rc))
4621	{
4622	ASMAtomicWriteS32(&pDevExt->rcTscDelta, VERR_NOT_AVAILABLE);
4623	return rc;
4624	}
4625
4626	OSDBGPRINT(("supdrvTscDeltaInit: supdrvTscDeltaThreadWait failed. rc=%Rrc\n", rc));
4627	supdrvTscDeltaThreadTerminate(pDevExt);
4628	}
4629	else
4630	OSDBGPRINT(("supdrvTscDeltaInit: RTThreadCreate failed. rc=%Rrc\n", rc));
4631	RTSemEventDestroy(pDevExt->hTscDeltaEvent);
4632	pDevExt->hTscDeltaEvent = NIL_RTSEMEVENT;
4633	}
4634	else
4635	OSDBGPRINT(("supdrvTscDeltaInit: RTSemEventCreate failed. rc=%Rrc\n", rc));
4636	RTSpinlockDestroy(pDevExt->hTscDeltaSpinlock);
4637	pDevExt->hTscDeltaSpinlock = NIL_RTSPINLOCK;
4638	}
4639	else
4640	OSDBGPRINT(("supdrvTscDeltaInit: RTSpinlockCreate failed. rc=%Rrc\n", rc));
4641
4642	return rc;
4643	}
4644
4645
4646	/**
4647	* Terminates the TSC-delta measurement thread and cleanup.
4648	*
4649	* @param pDevExt Pointer to the device instance data.
4650	*/
4651	static void supdrvTscDeltaTerm(PSUPDRVDEVEXT pDevExt)
4652	{
4653	if ( pDevExt->hTscDeltaSpinlock != NIL_RTSPINLOCK
4654	&& pDevExt->hTscDeltaEvent != NIL_RTSEMEVENT)
4655	{
4656	supdrvTscDeltaThreadTerminate(pDevExt);
4657	}
4658
4659	if (pDevExt->hTscDeltaSpinlock != NIL_RTSPINLOCK)
4660	{
4661	RTSpinlockDestroy(pDevExt->hTscDeltaSpinlock);
4662	pDevExt->hTscDeltaSpinlock = NIL_RTSPINLOCK;
4663	}
4664
4665	if (pDevExt->hTscDeltaEvent != NIL_RTSEMEVENT)
4666	{
4667	RTSemEventDestroy(pDevExt->hTscDeltaEvent);
4668	pDevExt->hTscDeltaEvent = NIL_RTSEMEVENT;
4669	}
4670
4671	ASMAtomicWriteS32(&pDevExt->rcTscDelta, VERR_NOT_AVAILABLE);
4672	}
4673
4674	#endif /* SUPDRV_USE_TSC_DELTA_THREAD */
4675
4676	/**
4677	* Measure the TSC delta for the CPU given by its CPU set index.
4678	*
4679	* @returns VBox status code.
4680	* @retval VERR_INTERRUPTED if interrupted while waiting.
4681	* @retval VERR_SUPDRV_TSC_DELTA_MEASUREMENT_FAILED if we were unable to get a
4682	* measurement.
4683	* @retval VERR_CPU_OFFLINE if the specified CPU is offline.
4684	*
4685	* @param pSession The caller's session. GIP must've been mapped.
4686	* @param iCpuSet The CPU set index of the CPU to measure.
4687	* @param fFlags Flags, SUP_TSCDELTA_MEASURE_F_XXX.
4688	* @param cMsWaitRetry Number of milliseconds to wait between each retry.
4689	* @param cMsWaitThread Number of milliseconds to wait for the thread to get
4690	* ready.
4691	* @param cTries Number of times to try, pass 0 for the default.
4692	*/
4693	SUPR0DECL(int) SUPR0TscDeltaMeasureBySetIndex(PSUPDRVSESSION pSession, uint32_t iCpuSet, uint32_t fFlags,
4694	RTMSINTERVAL cMsWaitRetry, RTMSINTERVAL cMsWaitThread, uint32_t cTries)
4695	{
4696	PSUPDRVDEVEXT pDevExt;
4697	PSUPGLOBALINFOPAGE pGip;
4698	uint16_t iGipCpu;
4699	int rc;
4700	#ifdef SUPDRV_USE_TSC_DELTA_THREAD
4701	uint64_t msTsStartWait;
4702	uint32_t iWaitLoop;
4703	#endif
4704
4705	/*
4706	* Validate and adjust the input.
4707	*/
4708	AssertReturn(SUP_IS_SESSION_VALID(pSession), VERR_INVALID_PARAMETER);
4709	if (!pSession->fGipReferenced)
4710	return VERR_WRONG_ORDER;
4711
4712	pDevExt = pSession->pDevExt;
4713	AssertReturn(SUP_IS_DEVEXT_VALID(pDevExt), VERR_INVALID_PARAMETER);
4714
4715	pGip = pDevExt->pGip;
4716	AssertPtrReturn(pGip, VERR_INTERNAL_ERROR_2);
4717
4718	AssertReturn(iCpuSet < RTCPUSET_MAX_CPUS, VERR_INVALID_CPU_INDEX);
4719	AssertReturn(iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx), VERR_INVALID_CPU_INDEX);
4720	iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet];
4721	AssertReturn(iGipCpu < pGip->cCpus, VERR_INVALID_CPU_INDEX);
4722
4723	if (fFlags & ~SUP_TSCDELTA_MEASURE_F_VALID_MASK)
4724	return VERR_INVALID_FLAGS;
4725
4726	/*
4727	* The request is a noop if the TSC delta isn't being used.
4728	*/
4729	if (pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_ZERO_CLAIMED)
4730	return VINF_SUCCESS;
4731
4732	if (cTries == 0)
4733	cTries = 12;
4734	else if (cTries > 256)
4735	cTries = 256;
4736
4737	if (cMsWaitRetry == 0)
4738	cMsWaitRetry = 2;
4739	else if (cMsWaitRetry > 1000)
4740	cMsWaitRetry = 1000;
4741
4742	#ifdef SUPDRV_USE_TSC_DELTA_THREAD
4743	/*
4744	* Has the TSC already been measured and we're not forced to redo it?
4745	*/
4746	if ( pGip->aCPUs[iGipCpu].i64TSCDelta != INT64_MAX
4747	&& !(fFlags & SUP_TSCDELTA_MEASURE_F_FORCE))
4748	return VINF_SUCCESS;
4749
4750	/*
4751	* Asynchronous request? Forward it to the thread, no waiting.
4752	*/
4753	if (fFlags & SUP_TSCDELTA_MEASURE_F_ASYNC)
4754	{
4755	/** @todo Async. doesn't implement options like retries, waiting. We'll need
4756	* to pass those options to the thread somehow and implement it in the
4757	* thread. Check if anyone uses/needs fAsync before implementing this. */
4758	RTSpinlockAcquire(pDevExt->hTscDeltaSpinlock);
4759	RTCpuSetAddByIndex(&pDevExt->TscDeltaCpuSet, iCpuSet);
4760	if ( pDevExt->enmTscDeltaThreadState == kTscDeltaThreadState_Listening
4761	\|\| pDevExt->enmTscDeltaThreadState == kTscDeltaThreadState_Measuring)
4762	{
4763	pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_WaitAndMeasure;
4764	rc = VINF_SUCCESS;
4765	}
4766	else if (pDevExt->enmTscDeltaThreadState != kTscDeltaThreadState_WaitAndMeasure)
4767	rc = VERR_THREAD_IS_DEAD;
4768	RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
4769	RTThreadUserSignal(pDevExt->hTscDeltaThread);
4770	return VINF_SUCCESS;
4771	}
4772
4773	/*
4774	* If a TSC-delta measurement request is already being serviced by the thread,
4775	* wait 'cTries' times if a retry-timeout is provided, otherwise bail as busy.
4776	*/
4777	msTsStartWait = RTTimeSystemMilliTS();
4778	for (iWaitLoop = 0;; iWaitLoop++)
4779	{
4780	uint64_t cMsElapsed;
4781	SUPDRVTSCDELTATHREADSTATE enmState;
4782	RTSpinlockAcquire(pDevExt->hTscDeltaSpinlock);
4783	enmState = pDevExt->enmTscDeltaThreadState;
4784	RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
4785
4786	if (enmState == kTscDeltaThreadState_Measuring)
4787	{ /* Must wait, the thread is busy. */ }
4788	else if (enmState == kTscDeltaThreadState_WaitAndMeasure)
4789	{ /* Must wait, this state only says what will happen next. */ }
4790	else if (enmState == kTscDeltaThreadState_Terminating)
4791	{ /* Must wait, this state only says what should happen next. */ }
4792	else
4793	break; /* All other states, the thread is either idly listening or dead. */
4794
4795	/* Wait or fail. */
4796	if (cMsWaitThread == 0)
4797	return VERR_SUPDRV_TSC_DELTA_MEASUREMENT_BUSY;
4798	cMsElapsed = RTTimeSystemMilliTS() - msTsStartWait;
4799	if (cMsElapsed >= cMsWaitThread)
4800	return VERR_SUPDRV_TSC_DELTA_MEASUREMENT_BUSY;
4801
4802	rc = RTThreadSleep(RT_MIN((RTMSINTERVAL)(cMsWaitThread - cMsElapsed), RT_MIN(iWaitLoop + 1, 10)));
4803	if (rc == VERR_INTERRUPTED)
4804	return rc;
4805	}
4806	#endif /* SUPDRV_USE_TSC_DELTA_THREAD */
4807
4808	/*
4809	* Try measure the TSC delta the given number of times.
4810	*/
4811	for (;;)
4812	{
4813	/* Unless we're forced to measure the delta, check whether it's done already. */
4814	if ( !(fFlags & SUP_TSCDELTA_MEASURE_F_FORCE)
4815	&& pGip->aCPUs[iGipCpu].i64TSCDelta != INT64_MAX)
4816	{
4817	rc = VINF_SUCCESS;
4818	break;
4819	}
4820
4821	/* Measure it. */
4822	rc = supdrvTscMeasureDeltaOne(pDevExt, iGipCpu);
4823	if (rc != VERR_SUPDRV_TSC_DELTA_MEASUREMENT_FAILED)
4824	{
4825	Assert(pGip->aCPUs[iGipCpu].i64TSCDelta != INT64_MAX \|\| RT_FAILURE_NP(rc));
4826	break;
4827	}
4828
4829	/* Retry? */
4830	if (cTries <= 1)
4831	break;
4832	cTries--;
4833
4834	/* Always delay between retries (be nice to the rest of the system
4835	and avoid the BSOD hounds). */
4836	rc = RTThreadSleep(cMsWaitRetry);
4837	if (rc == VERR_INTERRUPTED)
4838	break;
4839	}
4840
4841	return rc;
4842	}
4843	SUPR0_EXPORT_SYMBOL(SUPR0TscDeltaMeasureBySetIndex);
4844
4845
4846	/**
4847	* Service a TSC-delta measurement request.
4848	*
4849	* @returns VBox status code.
4850	* @param pDevExt Pointer to the device instance data.
4851	* @param pSession The support driver session.
4852	* @param pReq Pointer to the TSC-delta measurement request.
4853	*/
4854	int VBOXCALL supdrvIOCtl_TscDeltaMeasure(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession, PSUPTSCDELTAMEASURE pReq)
4855	{
4856	uint32_t cTries;
4857	uint32_t iCpuSet;
4858	uint32_t fFlags;
4859	RTMSINTERVAL cMsWaitRetry;
4860	RT_NOREF1(pDevExt);
4861
4862	/*
4863	* Validate and adjust/resolve the input so they can be passed onto SUPR0TscDeltaMeasureBySetIndex.
4864	*/
4865	AssertPtr(pDevExt); AssertPtr(pSession); AssertPtr(pReq); /* paranoia^2 */
4866
4867	if (pReq->u.In.idCpu == NIL_RTCPUID)
4868	return VERR_INVALID_CPU_ID;
4869	iCpuSet = RTMpCpuIdToSetIndex(pReq->u.In.idCpu);
4870	if (iCpuSet >= RTCPUSET_MAX_CPUS)
4871	return VERR_INVALID_CPU_ID;
4872
4873	cTries = pReq->u.In.cRetries == 0 ? 0 : (uint32_t)pReq->u.In.cRetries + 1;
4874
4875	cMsWaitRetry = RT_MAX(pReq->u.In.cMsWaitRetry, 5);
4876
4877	fFlags = 0;
4878	if (pReq->u.In.fAsync)
4879	fFlags \|= SUP_TSCDELTA_MEASURE_F_ASYNC;
4880	if (pReq->u.In.fForce)
4881	fFlags \|= SUP_TSCDELTA_MEASURE_F_FORCE;
4882
4883	return SUPR0TscDeltaMeasureBySetIndex(pSession, iCpuSet, fFlags, cMsWaitRetry,
4884	cTries == 0 ? 5 * RT_MS_1SEC : cMsWaitRetry * cTries /cMsWaitThread/,
4885	cTries);
4886	}
4887
4888
4889	/**
4890	* Reads TSC with delta applied.
4891	*
4892	* Will try to resolve delta value INT64_MAX before applying it. This is the
4893	* main purpose of this function, to handle the case where the delta needs to be
4894	* determined.
4895	*
4896	* @returns VBox status code.
4897	* @param pDevExt Pointer to the device instance data.
4898	* @param pSession The support driver session.
4899	* @param pReq Pointer to the TSC-read request.
4900	*/
4901	int VBOXCALL supdrvIOCtl_TscRead(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession, PSUPTSCREAD pReq)
4902	{
4903	PSUPGLOBALINFOPAGE pGip;
4904	int rc;
4905
4906	/*
4907	* Validate. We require the client to have mapped GIP (no asserting on
4908	* ring-3 preconditions).
4909	*/
4910	AssertPtr(pDevExt); AssertPtr(pReq); AssertPtr(pSession); /* paranoia^2 */
4911	if (pSession->GipMapObjR3 == NIL_RTR0MEMOBJ)
4912	return VERR_WRONG_ORDER;
4913	pGip = pDevExt->pGip;
4914	AssertReturn(pGip, VERR_INTERNAL_ERROR_2);
4915
4916	/*
4917	* We're usually here because we need to apply delta, but we shouldn't be
4918	* upset if the GIP is some different mode.
4919	*/
4920	if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
4921	{
4922	uint32_t cTries = 0;
4923	for (;;)
4924	{
4925	/*
4926	* Start by gathering the data, using CLI for disabling preemption
4927	* while we do that.
4928	*/
4929	RTCCUINTREG fEFlags = ASMIntDisableFlags();
4930	int iCpuSet = RTMpCpuIdToSetIndex(RTMpCpuId());
4931	int iGipCpu = 0; /* gcc maybe used uninitialized */
4932	if (RT_LIKELY( (unsigned)iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)
4933	&& (iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet]) < pGip->cCpus ))
4934	{
4935	int64_t i64Delta = pGip->aCPUs[iGipCpu].i64TSCDelta;
4936	pReq->u.Out.idApic = pGip->aCPUs[iGipCpu].idApic;
4937	pReq->u.Out.u64AdjustedTsc = ASMReadTSC();
4938	ASMSetFlags(fEFlags);
4939
4940	/*
4941	* If we're lucky we've got a delta, but no predictions here
4942	* as this I/O control is normally only used when the TSC delta
4943	* is set to INT64_MAX.
4944	*/
4945	if (i64Delta != INT64_MAX)
4946	{
4947	pReq->u.Out.u64AdjustedTsc -= i64Delta;
4948	rc = VINF_SUCCESS;
4949	break;
4950	}
4951
4952	/* Give up after a few times. */
4953	if (cTries >= 4)
4954	{
4955	rc = VWRN_SUPDRV_TSC_DELTA_MEASUREMENT_FAILED;
4956	break;
4957	}
4958
4959	/* Need to measure the delta an try again. */
4960	rc = supdrvTscMeasureDeltaOne(pDevExt, iGipCpu);
4961	Assert(pGip->aCPUs[iGipCpu].i64TSCDelta != INT64_MAX \|\| RT_FAILURE_NP(rc));
4962	/** @todo should probably delay on failure... dpc watchdogs */
4963	}
4964	else
4965	{
4966	/* This really shouldn't happen. */
4967	AssertMsgFailed(("idCpu=%#x iCpuSet=%#x (%d)\n", RTMpCpuId(), iCpuSet, iCpuSet));
4968	pReq->u.Out.idApic = supdrvGipGetApicIdSlow();
4969	pReq->u.Out.u64AdjustedTsc = ASMReadTSC();
4970	ASMSetFlags(fEFlags);
4971	rc = VERR_INTERNAL_ERROR_5; /** @todo change to warning. */
4972	break;
4973	}
4974	}
4975	}
4976	else
4977	{
4978	/*
4979	* No delta to apply. Easy. Deal with preemption the lazy way.
4980	*/
4981	RTCCUINTREG fEFlags = ASMIntDisableFlags();
4982	int iCpuSet = RTMpCpuIdToSetIndex(RTMpCpuId());
4983	int iGipCpu = 0; /* gcc may be used uninitialized */
4984	if (RT_LIKELY( (unsigned)iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)
4985	&& (iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet]) < pGip->cCpus ))
4986	pReq->u.Out.idApic = pGip->aCPUs[iGipCpu].idApic;
4987	else
4988	pReq->u.Out.idApic = supdrvGipGetApicIdSlow();
4989	pReq->u.Out.u64AdjustedTsc = ASMReadTSC();
4990	ASMSetFlags(fEFlags);
4991	rc = VINF_SUCCESS;
4992	}
4993
4994	return rc;
4995	}
4996
4997
4998	/**
4999	* Worker for supdrvIOCtl_GipSetFlags.
5000	*
5001	* @returns VBox status code.
5002	* @retval VERR_WRONG_ORDER if an enable-once-per-session flag is set again for
5003	* a session.
5004	*
5005	* @param pDevExt Pointer to the device instance data.
5006	* @param pSession The support driver session.
5007	* @param fOrMask The OR mask of the GIP flags, see SUPGIP_FLAGS_XXX.
5008	* @param fAndMask The AND mask of the GIP flags, see SUPGIP_FLAGS_XXX.
5009	*
5010	* @remarks Caller must own the GIP mutex.
5011	*
5012	* @remarks This function doesn't validate any of the flags.
5013	*/
5014	static int supdrvGipSetFlags(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession, uint32_t fOrMask, uint32_t fAndMask)
5015	{
5016	uint32_t cRefs;
5017	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
5018	AssertMsg((fOrMask & fAndMask) == fOrMask, ("%#x & %#x\n", fOrMask, fAndMask)); /* ASSUMED by code below */
5019
5020	/*
5021	* Compute GIP test-mode flags.
5022	*/
5023	if (fOrMask & SUPGIP_FLAGS_TESTING_ENABLE)
5024	{
5025	if (!pSession->fGipTestMode)
5026	{
5027	Assert(pDevExt->cGipTestModeRefs < _64K);
5028	pSession->fGipTestMode = true;
5029	cRefs = ++pDevExt->cGipTestModeRefs;
5030	if (cRefs == 1)
5031	{
5032	fOrMask \|= SUPGIP_FLAGS_TESTING \| SUPGIP_FLAGS_TESTING_START;
5033	fAndMask &= ~SUPGIP_FLAGS_TESTING_STOP;
5034	}
5035	}
5036	else
5037	{
5038	LogRelMax(10, ("supdrvGipSetFlags: SUPGIP_FLAGS_TESTING_ENABLE already set for this session\n"));
5039	return VERR_WRONG_ORDER;
5040	}
5041	}
5042	else if ( !(fAndMask & SUPGIP_FLAGS_TESTING_ENABLE)
5043	&& pSession->fGipTestMode)
5044	{
5045	Assert(pDevExt->cGipTestModeRefs > 0);
5046	Assert(pDevExt->cGipTestModeRefs < _64K);
5047	pSession->fGipTestMode = false;
5048	cRefs = --pDevExt->cGipTestModeRefs;
5049	if (!cRefs)
5050	fOrMask \|= SUPGIP_FLAGS_TESTING_STOP;
5051	else
5052	fAndMask \|= SUPGIP_FLAGS_TESTING_ENABLE;
5053	}
5054
5055	/*
5056	* Commit the flags. This should be done as atomically as possible
5057	* since the flag consumers won't be holding the GIP mutex.
5058	*/
5059	ASMAtomicOrU32(&pGip->fFlags, fOrMask);
5060	ASMAtomicAndU32(&pGip->fFlags, fAndMask);
5061
5062	return VINF_SUCCESS;
5063	}
5064
5065
5066	/**
5067	* Sets GIP test mode parameters.
5068	*
5069	* @returns VBox status code.
5070	* @param pDevExt Pointer to the device instance data.
5071	* @param pSession The support driver session.
5072	* @param fOrMask The OR mask of the GIP flags, see SUPGIP_FLAGS_XXX.
5073	* @param fAndMask The AND mask of the GIP flags, see SUPGIP_FLAGS_XXX.
5074	*/
5075	int VBOXCALL supdrvIOCtl_GipSetFlags(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession, uint32_t fOrMask, uint32_t fAndMask)
5076	{
5077	PSUPGLOBALINFOPAGE pGip;
5078	int rc;
5079
5080	/*
5081	* Validate. We require the client to have mapped GIP (no asserting on
5082	* ring-3 preconditions).
5083	*/
5084	AssertPtr(pDevExt); AssertPtr(pSession); /* paranoia^2 */
5085	if (pSession->GipMapObjR3 == NIL_RTR0MEMOBJ)
5086	return VERR_WRONG_ORDER;
5087	pGip = pDevExt->pGip;
5088	AssertReturn(pGip, VERR_INTERNAL_ERROR_3);
5089
5090	if (fOrMask & ~SUPGIP_FLAGS_VALID_MASK)
5091	return VERR_INVALID_PARAMETER;
5092	if ((fAndMask & ~SUPGIP_FLAGS_VALID_MASK) != ~SUPGIP_FLAGS_VALID_MASK)
5093	return VERR_INVALID_PARAMETER;
5094
5095	/*
5096	* Don't confuse supdrvGipSetFlags or anyone else by both setting
5097	* and clearing the same flags. AND takes precedence.
5098	*/
5099	fOrMask &= fAndMask;
5100
5101	/*
5102	* Take the loader lock to avoid having to think about races between two
5103	* clients changing the flags at the same time (state is not simple).
5104	*/
5105	#ifdef SUPDRV_USE_MUTEX_FOR_GIP
5106	RTSemMutexRequest(pDevExt->mtxGip, RT_INDEFINITE_WAIT);
5107	#else
5108	RTSemFastMutexRequest(pDevExt->mtxGip);
5109	#endif
5110
5111	rc = supdrvGipSetFlags(pDevExt, pSession, fOrMask, fAndMask);
5112
5113	#ifdef SUPDRV_USE_MUTEX_FOR_GIP
5114	RTSemMutexRelease(pDevExt->mtxGip);
5115	#else
5116	RTSemFastMutexRelease(pDevExt->mtxGip);
5117	#endif
5118	return rc;
5119	}
5120

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/HostDrivers/Support/SUPDrvGip.cpp@ 100108

Download in other formats: