mp-r0drv-nt.cpp@ 96407

Last change on this file since 96407 was 96407, checked in by vboxsync, 2 years ago
scm copyright and license note update
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 77.2 KB

Line
1	/* $Id: mp-r0drv-nt.cpp 96407 2022-08-22 17:43:14Z vboxsync $ */
2	/** @file
3	* IPRT - Multiprocessor, Ring-0 Driver, NT.
4	*/
5
6	/*
7	* Copyright (C) 2008-2022 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* The contents of this file may alternatively be used under the terms
26	* of the Common Development and Distribution License Version 1.0
27	* (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
28	* in the VirtualBox distribution, in which case the provisions of the
29	* CDDL are applicable instead of those of the GPL.
30	*
31	* You may elect to license modified versions of this file under the
32	* terms and conditions of either the GPL or the CDDL or both.
33	*
34	* SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
35	*/
36
37
38	/*********************************************************************************************************************************
39	* Header Files *
40	*********************************************************************************************************************************/
41	#include "the-nt-kernel.h"
42
43	#include <iprt/mp.h>
44	#include <iprt/cpuset.h>
45	#include <iprt/err.h>
46	#include <iprt/asm.h>
47	#include <iprt/log.h>
48	#include <iprt/mem.h>
49	#include <iprt/time.h>
50	#include "r0drv/mp-r0drv.h"
51	#include "symdb.h"
52	#include "internal-r0drv-nt.h"
53	#include "internal/mp.h"
54
55
56	/*********************************************************************************************************************************
57	* Structures and Typedefs *
58	*********************************************************************************************************************************/
59	typedef enum
60	{
61	RT_NT_CPUID_SPECIFIC,
62	RT_NT_CPUID_PAIR,
63	RT_NT_CPUID_OTHERS,
64	RT_NT_CPUID_ALL
65	} RT_NT_CPUID;
66
67
68	/**
69	* Used by the RTMpOnSpecific.
70	*/
71	typedef struct RTMPNTONSPECIFICARGS
72	{
73	/** Set if we're executing. */
74	bool volatile fExecuting;
75	/** Set when done executing. */
76	bool volatile fDone;
77	/** Number of references to this heap block. */
78	uint32_t volatile cRefs;
79	/** Event that the calling thread is waiting on. */
80	KEVENT DoneEvt;
81	/** The deferred procedure call object. */
82	KDPC Dpc;
83	/** The callback argument package. */
84	RTMPARGS CallbackArgs;
85	} RTMPNTONSPECIFICARGS;
86	/** Pointer to an argument/state structure for RTMpOnSpecific on NT. */
87	typedef RTMPNTONSPECIFICARGS *PRTMPNTONSPECIFICARGS;
88
89
90	/*********************************************************************************************************************************
91	* Defined Constants And Macros *
92	*********************************************************************************************************************************/
93	/** Inactive bit for g_aidRtMpNtByCpuSetIdx. */
94	#define RTMPNT_ID_F_INACTIVE RT_BIT_32(31)
95
96
97	/*********************************************************************************************************************************
98	* Global Variables *
99	*********************************************************************************************************************************/
100	/** Maximum number of processor groups. */
101	uint32_t g_cRtMpNtMaxGroups;
102	/** Maximum number of processors. */
103	uint32_t g_cRtMpNtMaxCpus;
104	/** Number of active processors. */
105	uint32_t volatile g_cRtMpNtActiveCpus;
106	/** The NT CPU set.
107	* KeQueryActiveProcssors() cannot be called at all IRQLs and therefore we'll
108	* have to cache it. Fortunately, NT doesn't really support taking CPUs offline,
109	* and taking them online was introduced with W2K8 where it is intended for virtual
110	* machines and not real HW. We update this, g_cRtMpNtActiveCpus and
111	* g_aidRtMpNtByCpuSetIdx from the rtR0NtMpProcessorChangeCallback.
112	*/
113	RTCPUSET g_rtMpNtCpuSet;
114
115	/** Static per group info.
116	* @remarks With 256 groups this takes up 33KB. */
117	static struct
118	{
119	/** The max CPUs in the group. */
120	uint16_t cMaxCpus;
121	/** The number of active CPUs at the time of initialization. */
122	uint16_t cActiveCpus;
123	/** CPU set indexes for each CPU in the group. */
124	int16_t aidxCpuSetMembers[64];
125	} g_aRtMpNtCpuGroups[256];
126	/** Maps CPU set indexes to RTCPUID.
127	* Inactive CPUs has bit 31 set (RTMPNT_ID_F_INACTIVE) so we can identify them
128	* and shuffle duplicates during CPU hotplugging. We assign temporary IDs to
129	* the inactive CPUs starting at g_cRtMpNtMaxCpus - 1, ASSUMING that active
130	* CPUs has IDs from 0 to g_cRtMpNtActiveCpus. */
131	RTCPUID g_aidRtMpNtByCpuSetIdx[RTCPUSET_MAX_CPUS];
132	/** The handle of the rtR0NtMpProcessorChangeCallback registration. */
133	static PVOID g_pvMpCpuChangeCallback = NULL;
134	/** Size of the KAFFINITY_EX structure.
135	* This increased from 20 to 32 bitmap words in the 2020 H2 windows 10 release
136	* (i.e. 1280 to 2048 CPUs). We expect this to increase in the future. */
137	static size_t g_cbRtMpNtKaffinityEx = RT_UOFFSETOF(KAFFINITY_EX, Bitmap)
138	+ RT_SIZEOFMEMB(KAFFINITY_EX, Bitmap[0]) * 256;
139	/** The size value of the KAFFINITY_EX structure. */
140	static uint16_t g_cRtMpNtKaffinityExEntries = 256;
141
142
143	/*********************************************************************************************************************************
144	* Internal Functions *
145	*********************************************************************************************************************************/
146	static VOID __stdcall rtR0NtMpProcessorChangeCallback(void *pvUser, PKE_PROCESSOR_CHANGE_NOTIFY_CONTEXT pChangeCtx,
147	PNTSTATUS prcOperationStatus);
148	static int rtR0NtInitQueryGroupRelations(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX **ppInfo);
149
150
151
152	/**
153	* Initalizes multiprocessor globals (called by rtR0InitNative).
154	*
155	* @returns IPRT status code.
156	* @param pOsVerInfo Version information.
157	*/
158	DECLHIDDEN(int) rtR0MpNtInit(RTNTSDBOSVER const *pOsVerInfo)
159	{
160	#define MY_CHECK_BREAK(a_Check, a_DbgPrintArgs) \
161	AssertMsgBreakStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs; rc = VERR_INTERNAL_ERROR_4 )
162	#define MY_CHECK_RETURN(a_Check, a_DbgPrintArgs, a_rcRet) \
163	AssertMsgReturnStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs, a_rcRet)
164	#define MY_CHECK(a_Check, a_DbgPrintArgs) \
165	AssertMsgStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs; rc = VERR_INTERNAL_ERROR_4 )
166
167	/*
168	* API combination checks.
169	*/
170	MY_CHECK_RETURN(!g_pfnrtKeSetTargetProcessorDpcEx \|\| g_pfnrtKeGetProcessorNumberFromIndex,
171	("IPRT: Fatal: Missing KeSetTargetProcessorDpcEx without KeGetProcessorNumberFromIndex!\n"),
172	VERR_SYMBOL_NOT_FOUND);
173
174	/*
175	* Get max number of processor groups.
176	*
177	* We may need to upadjust this number below, because windows likes to keep
178	* all options open when it comes to hotplugged CPU group assignments. A
179	* server advertising up to 64 CPUs in the ACPI table will get a result of
180	* 64 from KeQueryMaximumGroupCount. That makes sense. However, when windows
181	* server 2012 does a two processor group setup for it, the sum of the
182	* GroupInfo[*].MaximumProcessorCount members below is 128. This is probably
183	* because windows doesn't want to make decisions grouping of hotpluggable CPUs.
184	* So, we need to bump the maximum count to 128 below do deal with this as we
185	* want to have valid CPU set indexes for all potential CPUs - how could we
186	* otherwise use the RTMpGetSet() result and also RTCpuSetCount(RTMpGetSet())
187	* should equal RTMpGetCount().
188	*/
189	if (g_pfnrtKeQueryMaximumGroupCount)
190	{
191	g_cRtMpNtMaxGroups = g_pfnrtKeQueryMaximumGroupCount();
192	MY_CHECK_RETURN(g_cRtMpNtMaxGroups <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxGroups > 0,
193	("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u\n", g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
194	VERR_MP_TOO_MANY_CPUS);
195	}
196	else
197	g_cRtMpNtMaxGroups = 1;
198
199	/*
200	* Get max number CPUs.
201	* This also defines the range of NT CPU indexes, RTCPUID and index into RTCPUSET.
202	*/
203	if (g_pfnrtKeQueryMaximumProcessorCountEx)
204	{
205	g_cRtMpNtMaxCpus = g_pfnrtKeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS);
206	MY_CHECK_RETURN(g_cRtMpNtMaxCpus <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxCpus > 0,
207	("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, max %u [KeQueryMaximumProcessorCountEx]\n",
208	g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
209	VERR_MP_TOO_MANY_CPUS);
210	}
211	else if (g_pfnrtKeQueryMaximumProcessorCount)
212	{
213	g_cRtMpNtMaxCpus = g_pfnrtKeQueryMaximumProcessorCount();
214	MY_CHECK_RETURN(g_cRtMpNtMaxCpus <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxCpus > 0,
215	("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, max %u [KeQueryMaximumProcessorCount]\n",
216	g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
217	VERR_MP_TOO_MANY_CPUS);
218	}
219	else if (g_pfnrtKeQueryActiveProcessors)
220	{
221	KAFFINITY fActiveProcessors = g_pfnrtKeQueryActiveProcessors();
222	MY_CHECK_RETURN(fActiveProcessors != 0,
223	("IPRT: Fatal: KeQueryActiveProcessors returned 0!\n"),
224	VERR_INTERNAL_ERROR_2);
225	g_cRtMpNtMaxCpus = 0;
226	do
227	{
228	g_cRtMpNtMaxCpus++;
229	fActiveProcessors >>= 1;
230	} while (fActiveProcessors);
231	}
232	else
233	g_cRtMpNtMaxCpus = KeNumberProcessors;
234
235	/*
236	* Just because we're a bit paranoid about getting something wrong wrt to the
237	* kernel interfaces, we try 16 times to get the KeQueryActiveProcessorCountEx
238	* and KeQueryLogicalProcessorRelationship information to match up.
239	*/
240	for (unsigned cTries = 0;; cTries++)
241	{
242	/*
243	* Get number of active CPUs.
244	*/
245	if (g_pfnrtKeQueryActiveProcessorCountEx)
246	{
247	g_cRtMpNtActiveCpus = g_pfnrtKeQueryActiveProcessorCountEx(ALL_PROCESSOR_GROUPS);
248	MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus && g_cRtMpNtActiveCpus > 0,
249	("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u [KeQueryActiveProcessorCountEx]\n",
250	g_cRtMpNtMaxGroups, g_cRtMpNtMaxCpus),
251	VERR_MP_TOO_MANY_CPUS);
252	}
253	else if (g_pfnrtKeQueryActiveProcessorCount)
254	{
255	g_cRtMpNtActiveCpus = g_pfnrtKeQueryActiveProcessorCount(NULL);
256	MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus && g_cRtMpNtActiveCpus > 0,
257	("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u [KeQueryActiveProcessorCount]\n",
258	g_cRtMpNtMaxGroups, g_cRtMpNtMaxCpus),
259	VERR_MP_TOO_MANY_CPUS);
260	}
261	else
262	g_cRtMpNtActiveCpus = g_cRtMpNtMaxCpus;
263
264	/*
265	* Query the details for the groups to figure out which CPUs are online as
266	* well as the NT index limit.
267	*/
268	for (unsigned i = 0; i < RT_ELEMENTS(g_aidRtMpNtByCpuSetIdx); i++)
269	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
270	g_aidRtMpNtByCpuSetIdx[i] = NIL_RTCPUID;
271	#else
272	g_aidRtMpNtByCpuSetIdx[i] = i < g_cRtMpNtMaxCpus ? i : NIL_RTCPUID;
273	#endif
274	for (unsigned idxGroup = 0; idxGroup < RT_ELEMENTS(g_aRtMpNtCpuGroups); idxGroup++)
275	{
276	g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = 0;
277	g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = 0;
278	for (unsigned idxMember = 0; idxMember < RT_ELEMENTS(g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers); idxMember++)
279	g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = -1;
280	}
281
282	if (g_pfnrtKeQueryLogicalProcessorRelationship)
283	{
284	MY_CHECK_RETURN(g_pfnrtKeGetProcessorIndexFromNumber,
285	("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeGetProcessorIndexFromNumber!\n"),
286	VERR_SYMBOL_NOT_FOUND);
287	MY_CHECK_RETURN(g_pfnrtKeGetProcessorNumberFromIndex,
288	("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeGetProcessorIndexFromNumber!\n"),
289	VERR_SYMBOL_NOT_FOUND);
290	MY_CHECK_RETURN(g_pfnrtKeSetTargetProcessorDpcEx,
291	("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeSetTargetProcessorDpcEx!\n"),
292	VERR_SYMBOL_NOT_FOUND);
293
294	SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *pInfo = NULL;
295	int rc = rtR0NtInitQueryGroupRelations(&pInfo);
296	if (RT_FAILURE(rc))
297	return rc;
298
299	MY_CHECK(pInfo->Group.MaximumGroupCount == g_cRtMpNtMaxGroups,
300	("IPRT: Fatal: MaximumGroupCount=%u != g_cRtMpNtMaxGroups=%u!\n",
301	pInfo->Group.MaximumGroupCount, g_cRtMpNtMaxGroups));
302	MY_CHECK(pInfo->Group.ActiveGroupCount > 0 && pInfo->Group.ActiveGroupCount <= g_cRtMpNtMaxGroups,
303	("IPRT: Fatal: ActiveGroupCount=%u != g_cRtMpNtMaxGroups=%u!\n",
304	pInfo->Group.ActiveGroupCount, g_cRtMpNtMaxGroups));
305
306	/*
307	* First we need to recalc g_cRtMpNtMaxCpus (see above).
308	*/
309	uint32_t cMaxCpus = 0;
310	uint32_t idxGroup;
311	for (idxGroup = 0; RT_SUCCESS(rc) && idxGroup < pInfo->Group.ActiveGroupCount; idxGroup++)
312	{
313	const PROCESSOR_GROUP_INFO *pGrpInfo = &pInfo->Group.GroupInfo[idxGroup];
314	MY_CHECK_BREAK(pGrpInfo->MaximumProcessorCount <= MAXIMUM_PROC_PER_GROUP,
315	("IPRT: Fatal: MaximumProcessorCount=%u\n", pGrpInfo->MaximumProcessorCount));
316	MY_CHECK_BREAK(pGrpInfo->ActiveProcessorCount <= pGrpInfo->MaximumProcessorCount,
317	("IPRT: Fatal: ActiveProcessorCount=%u > MaximumProcessorCount=%u\n",
318	pGrpInfo->ActiveProcessorCount, pGrpInfo->MaximumProcessorCount));
319	cMaxCpus += pGrpInfo->MaximumProcessorCount;
320	}
321	if (cMaxCpus > g_cRtMpNtMaxCpus && RT_SUCCESS(rc))
322	{
323	DbgPrint("IPRT: g_cRtMpNtMaxCpus=%u -> %u\n", g_cRtMpNtMaxCpus, cMaxCpus);
324	#ifndef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
325	uint32_t i = RT_MIN(cMaxCpus, RT_ELEMENTS(g_aidRtMpNtByCpuSetIdx));
326	while (i-- > g_cRtMpNtMaxCpus)
327	g_aidRtMpNtByCpuSetIdx[i] = i;
328	#endif
329	g_cRtMpNtMaxCpus = cMaxCpus;
330	if (g_cRtMpNtMaxGroups > RTCPUSET_MAX_CPUS)
331	{
332	MY_CHECK(g_cRtMpNtMaxGroups <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxGroups > 0,
333	("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u\n", g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS));
334	rc = VERR_MP_TOO_MANY_CPUS;
335	}
336	}
337
338	/*
339	* Calc online mask, partition IDs and such.
340	*
341	* Also check ASSUMPTIONS:
342	*
343	* 1. Processor indexes going from 0 and up to
344	* KeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS) - 1.
345	*
346	* 2. Currently valid processor indexes, i.e. accepted by
347	* KeGetProcessorIndexFromNumber & KeGetProcessorNumberFromIndex, goes
348	* from 0 thru KeQueryActiveProcessorCountEx(ALL_PROCESSOR_GROUPS) - 1.
349	*
350	* 3. PROCESSOR_GROUP_INFO::MaximumProcessorCount gives the number of
351	* relevant bits in the ActiveProcessorMask (from LSB).
352	*
353	* 4. Active processor count found in KeQueryLogicalProcessorRelationship
354	* output matches what KeQueryActiveProcessorCountEx(ALL) returns.
355	*
356	* 5. Active + inactive processor counts in same does not exceed
357	* KeQueryMaximumProcessorCountEx(ALL).
358	*
359	* Note! Processor indexes are assigned as CPUs come online and are not
360	* preallocated according to group maximums. Since CPUS are only taken
361	* online and never offlined, this means that internal CPU bitmaps are
362	* never sparse and no time is wasted scanning unused bits.
363	*
364	* Unfortunately, it means that ring-3 cannot easily guess the index
365	* assignments when hotswapping is used, and must use GIP when available.
366	*/
367	RTCpuSetEmpty(&g_rtMpNtCpuSet);
368	uint32_t cInactive = 0;
369	uint32_t cActive = 0;
370	uint32_t idxCpuMax = 0;
371	uint32_t idxCpuSetNextInactive = g_cRtMpNtMaxCpus - 1;
372	for (idxGroup = 0; RT_SUCCESS(rc) && idxGroup < pInfo->Group.ActiveGroupCount; idxGroup++)
373	{
374	const PROCESSOR_GROUP_INFO *pGrpInfo = &pInfo->Group.GroupInfo[idxGroup];
375	MY_CHECK_BREAK(pGrpInfo->MaximumProcessorCount <= MAXIMUM_PROC_PER_GROUP,
376	("IPRT: Fatal: MaximumProcessorCount=%u\n", pGrpInfo->MaximumProcessorCount));
377	MY_CHECK_BREAK(pGrpInfo->ActiveProcessorCount <= pGrpInfo->MaximumProcessorCount,
378	("IPRT: Fatal: ActiveProcessorCount=%u > MaximumProcessorCount=%u\n",
379	pGrpInfo->ActiveProcessorCount, pGrpInfo->MaximumProcessorCount));
380
381	g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = pGrpInfo->MaximumProcessorCount;
382	g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = pGrpInfo->ActiveProcessorCount;
383
384	for (uint32_t idxMember = 0; idxMember < pGrpInfo->MaximumProcessorCount; idxMember++)
385	{
386	PROCESSOR_NUMBER ProcNum;
387	ProcNum.Group = (USHORT)idxGroup;
388	ProcNum.Number = (UCHAR)idxMember;
389	ProcNum.Reserved = 0;
390	ULONG idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);
391	if (idxCpu != INVALID_PROCESSOR_INDEX)
392	{
393	MY_CHECK_BREAK(idxCpu < g_cRtMpNtMaxCpus && idxCpu < RTCPUSET_MAX_CPUS, /* ASSUMPTION #1 */
394	("IPRT: Fatal: idxCpu=%u >= g_cRtMpNtMaxCpus=%u (RTCPUSET_MAX_CPUS=%u)\n",
395	idxCpu, g_cRtMpNtMaxCpus, RTCPUSET_MAX_CPUS));
396	if (idxCpu > idxCpuMax)
397	idxCpuMax = idxCpu;
398	g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpu;
399	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
400	g_aidRtMpNtByCpuSetIdx[idxCpu] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember);
401	#endif
402
403	ProcNum.Group = UINT16_MAX;
404	ProcNum.Number = UINT8_MAX;
405	ProcNum.Reserved = UINT8_MAX;
406	NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(idxCpu, &ProcNum);
407	MY_CHECK_BREAK(NT_SUCCESS(rcNt),
408	("IPRT: Fatal: KeGetProcessorNumberFromIndex(%u,) -> %#x!\n", idxCpu, rcNt));
409	MY_CHECK_BREAK(ProcNum.Group == idxGroup && ProcNum.Number == idxMember,
410	("IPRT: Fatal: KeGetProcessorXxxxFromYyyy roundtrip error for %#x! Group: %u vs %u, Number: %u vs %u\n",
411	idxCpu, ProcNum.Group, idxGroup, ProcNum.Number, idxMember));
412
413	if (pGrpInfo->ActiveProcessorMask & RT_BIT_64(idxMember))
414	{
415	RTCpuSetAddByIndex(&g_rtMpNtCpuSet, idxCpu);
416	cActive++;
417	}
418	else
419	cInactive++; /* (This is a little unexpected, but not important as long as things add up below.) */
420	}
421	else
422	{
423	/* Must be not present / inactive when KeGetProcessorIndexFromNumber fails. */
424	MY_CHECK_BREAK(!(pGrpInfo->ActiveProcessorMask & RT_BIT_64(idxMember)),
425	("IPRT: Fatal: KeGetProcessorIndexFromNumber(%u/%u) failed but CPU is active! cMax=%u cActive=%u fActive=%p\n",
426	idxGroup, idxMember, pGrpInfo->MaximumProcessorCount, pGrpInfo->ActiveProcessorCount,
427	pGrpInfo->ActiveProcessorMask));
428	cInactive++;
429	if (idxCpuSetNextInactive >= g_cRtMpNtActiveCpus)
430	{
431	g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpuSetNextInactive;
432	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
433	g_aidRtMpNtByCpuSetIdx[idxCpuSetNextInactive] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember)
434	\| RTMPNT_ID_F_INACTIVE;
435	#endif
436	idxCpuSetNextInactive--;
437	}
438	}
439	}
440	}
441
442	MY_CHECK(cInactive + cActive <= g_cRtMpNtMaxCpus, /* ASSUMPTION #5 (not '==' because of inactive groups) */
443	("IPRT: Fatal: cInactive=%u + cActive=%u > g_cRtMpNtMaxCpus=%u\n", cInactive, cActive, g_cRtMpNtMaxCpus));
444
445	/* Deal with inactive groups using KeQueryMaximumProcessorCountEx or as
446	best as we can by as best we can by stipulating maximum member counts
447	from the previous group. */
448	if ( RT_SUCCESS(rc)
449	&& idxGroup < pInfo->Group.MaximumGroupCount)
450	{
451	uint16_t cInactiveLeft = g_cRtMpNtMaxCpus - (cInactive + cActive);
452	while (idxGroup < pInfo->Group.MaximumGroupCount)
453	{
454	uint32_t cMaxMembers = 0;
455	if (g_pfnrtKeQueryMaximumProcessorCountEx)
456	cMaxMembers = g_pfnrtKeQueryMaximumProcessorCountEx(idxGroup);
457	if (cMaxMembers != 0 \|\| cInactiveLeft == 0)
458	AssertStmt(cMaxMembers <= cInactiveLeft, cMaxMembers = cInactiveLeft);
459	else
460	{
461	uint16_t cGroupsLeft = pInfo->Group.MaximumGroupCount - idxGroup;
462	cMaxMembers = pInfo->Group.GroupInfo[idxGroup - 1].MaximumProcessorCount;
463	while (cMaxMembers * cGroupsLeft < cInactiveLeft)
464	cMaxMembers++;
465	if (cMaxMembers > cInactiveLeft)
466	cMaxMembers = cInactiveLeft;
467	}
468
469	g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = (uint16_t)cMaxMembers;
470	g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = 0;
471	for (uint16_t idxMember = 0; idxMember < cMaxMembers; idxMember++)
472	if (idxCpuSetNextInactive >= g_cRtMpNtActiveCpus)
473	{
474	g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpuSetNextInactive;
475	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
476	g_aidRtMpNtByCpuSetIdx[idxCpuSetNextInactive] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember)
477	\| RTMPNT_ID_F_INACTIVE;
478	#endif
479	idxCpuSetNextInactive--;
480	}
481	cInactiveLeft -= cMaxMembers;
482	idxGroup++;
483	}
484	}
485
486	/* We're done with pInfo now, free it so we can start returning when assertions fail. */
487	RTMemFree(pInfo);
488	if (RT_FAILURE(rc)) /* MY_CHECK_BREAK sets rc. */
489	return rc;
490	MY_CHECK_RETURN(cActive >= g_cRtMpNtActiveCpus,
491	("IPRT: Fatal: cActive=%u < g_cRtMpNtActiveCpus=%u - CPUs removed?\n", cActive, g_cRtMpNtActiveCpus),
492	VERR_INTERNAL_ERROR_3);
493	MY_CHECK_RETURN(idxCpuMax < cActive, /* ASSUMPTION #2 */
494	("IPRT: Fatal: idCpuMax=%u >= cActive=%u! Unexpected CPU index allocation. CPUs removed?\n",
495	idxCpuMax, cActive),
496	VERR_INTERNAL_ERROR_4);
497
498	/* Retry if CPUs were added. */
499	if ( cActive != g_cRtMpNtActiveCpus
500	&& cTries < 16)
501	continue;
502	MY_CHECK_RETURN(cActive == g_cRtMpNtActiveCpus, /* ASSUMPTION #4 */
503	("IPRT: Fatal: cActive=%u != g_cRtMpNtActiveCpus=%u\n", cActive, g_cRtMpNtActiveCpus),
504	VERR_INTERNAL_ERROR_5);
505	}
506	else
507	{
508	/* Legacy: */
509	MY_CHECK_RETURN(g_cRtMpNtMaxGroups == 1, ("IPRT: Fatal: Missing KeQueryLogicalProcessorRelationship!\n"),
510	VERR_SYMBOL_NOT_FOUND);
511
512	/** @todo Is it possible that the affinity mask returned by
513	* KeQueryActiveProcessors is sparse? */
514	if (g_pfnrtKeQueryActiveProcessors)
515	RTCpuSetFromU64(&g_rtMpNtCpuSet, g_pfnrtKeQueryActiveProcessors());
516	else if (g_cRtMpNtMaxCpus < 64)
517	RTCpuSetFromU64(&g_rtMpNtCpuSet, (UINT64_C(1) << g_cRtMpNtMaxCpus) - 1);
518	else
519	{
520	MY_CHECK_RETURN(g_cRtMpNtMaxCpus == 64, ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, expect 64 or less\n", g_cRtMpNtMaxCpus),
521	VERR_MP_TOO_MANY_CPUS);
522	RTCpuSetFromU64(&g_rtMpNtCpuSet, UINT64_MAX);
523	}
524
525	g_aRtMpNtCpuGroups[0].cMaxCpus = g_cRtMpNtMaxCpus;
526	g_aRtMpNtCpuGroups[0].cActiveCpus = g_cRtMpNtMaxCpus;
527	for (unsigned i = 0; i < g_cRtMpNtMaxCpus; i++)
528	{
529	g_aRtMpNtCpuGroups[0].aidxCpuSetMembers[i] = i;
530	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
531	g_aidRtMpNtByCpuSetIdx[i] = RTMPCPUID_FROM_GROUP_AND_NUMBER(0, i);
532	#endif
533	}
534	}
535
536	/*
537	* Register CPU hot plugging callback (it also counts active CPUs).
538	*/
539	Assert(g_pvMpCpuChangeCallback == NULL);
540	if (g_pfnrtKeRegisterProcessorChangeCallback)
541	{
542	MY_CHECK_RETURN(g_pfnrtKeDeregisterProcessorChangeCallback,
543	("IPRT: Fatal: KeRegisterProcessorChangeCallback without KeDeregisterProcessorChangeCallback!\n"),
544	VERR_SYMBOL_NOT_FOUND);
545
546	RTCPUSET const ActiveSetCopy = g_rtMpNtCpuSet;
547	RTCpuSetEmpty(&g_rtMpNtCpuSet);
548	uint32_t const cActiveCpus = g_cRtMpNtActiveCpus;
549	g_cRtMpNtActiveCpus = 0;
550
551	g_pvMpCpuChangeCallback = g_pfnrtKeRegisterProcessorChangeCallback(rtR0NtMpProcessorChangeCallback, NULL /pvUser/,
552	KE_PROCESSOR_CHANGE_ADD_EXISTING);
553	if (g_pvMpCpuChangeCallback)
554	{
555	if (cActiveCpus == g_cRtMpNtActiveCpus)
556	{ /* likely */ }
557	else
558	{
559	g_pfnrtKeDeregisterProcessorChangeCallback(g_pvMpCpuChangeCallback);
560	if (cTries < 16)
561	{
562	/* Retry if CPUs were added. */
563	MY_CHECK_RETURN(g_cRtMpNtActiveCpus >= cActiveCpus,
564	("IPRT: Fatal: g_cRtMpNtActiveCpus=%u < cActiveCpus=%u! CPUs removed?\n",
565	g_cRtMpNtActiveCpus, cActiveCpus),
566	VERR_INTERNAL_ERROR_2);
567	MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus,
568	("IPRT: Fatal: g_cRtMpNtActiveCpus=%u > g_cRtMpNtMaxCpus=%u!\n",
569	g_cRtMpNtActiveCpus, g_cRtMpNtMaxCpus),
570	VERR_INTERNAL_ERROR_2);
571	continue;
572	}
573	MY_CHECK_RETURN(0, ("IPRT: Fatal: g_cRtMpNtActiveCpus=%u cActiveCpus=%u\n", g_cRtMpNtActiveCpus, cActiveCpus),
574	VERR_INTERNAL_ERROR_3);
575	}
576	}
577	else
578	{
579	AssertFailed();
580	g_rtMpNtCpuSet = ActiveSetCopy;
581	g_cRtMpNtActiveCpus = cActiveCpus;
582	}
583	}
584	break;
585	} /* Retry loop for stable active CPU count. */
586
587	#undef MY_CHECK_RETURN
588
589	/*
590	* Special IPI fun for RTMpPokeCpu.
591	*
592	* On Vista and later the DPC method doesn't seem to reliably send IPIs,
593	* so we have to use alternative methods.
594	*
595	* On AMD64 We used to use the HalSendSoftwareInterrupt API (also x86 on
596	* W10+), it looks faster and more convenient to use, however we're either
597	* using it wrong or it doesn't reliably do what we want (see @bugref{8343}).
598	*
599	* The HalRequestIpip API is thus far the only alternative to KeInsertQueueDpc
600	* for doing targetted IPIs. Trouble with this API is that it changed
601	* fundamentally in Window 7 when they added support for lots of processors.
602	*
603	* If we really think we cannot use KeInsertQueueDpc, we use the broadcast IPI
604	* API KeIpiGenericCall.
605	*/
606	if ( pOsVerInfo->uMajorVer > 6
607	\|\| (pOsVerInfo->uMajorVer == 6 && pOsVerInfo->uMinorVer > 0))
608	g_pfnrtHalRequestIpiPreW7 = NULL;
609	else
610	g_pfnrtHalRequestIpiW7Plus = NULL;
611
612	if ( g_pfnrtHalRequestIpiW7Plus
613	&& g_pfnrtKeInitializeAffinityEx
614	&& g_pfnrtKeAddProcessorAffinityEx
615	&& g_pfnrtKeGetProcessorIndexFromNumber)
616	{
617	/* Determine the real size of the KAFFINITY_EX structure. */
618	size_t const cbAffinity = _8K;
619	PKAFFINITY_EX pAffinity = (PKAFFINITY_EX)RTMemAllocZ(cbAffinity);
620	AssertReturn(pAffinity, VERR_NO_MEMORY);
621	size_t const cMaxEntries = (cbAffinity - RT_UOFFSETOF(KAFFINITY_EX, Bitmap[0])) / sizeof(pAffinity->Bitmap[0]);
622	g_pfnrtKeInitializeAffinityEx(pAffinity);
623	if (pAffinity->Size > 1 && pAffinity->Size <= cMaxEntries)
624	{
625	g_cRtMpNtKaffinityExEntries = pAffinity->Size;
626	g_cbRtMpNtKaffinityEx = pAffinity->Size * sizeof(pAffinity->Bitmap[0]) + RT_UOFFSETOF(KAFFINITY_EX, Bitmap[0]);
627	g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingHalRequestIpiW7Plus;
628	RTMemFree(pAffinity);
629	DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingHalRequestIpiW7Plus\n");
630	return VINF_SUCCESS;
631	}
632	DbgPrint("IPRT: RTMpPoke can't use rtMpPokeCpuUsingHalRequestIpiW7Plus! pAffinity->Size=%u\n", pAffinity->Size);
633	AssertReleaseMsg(pAffinity->Size <= cMaxEntries, ("%#x\n", pAffinity->Size)); /* stack is toast if larger (32768 CPUs). */
634	RTMemFree(pAffinity);
635	}
636
637	if (pOsVerInfo->uMajorVer >= 6 && g_pfnrtKeIpiGenericCall)
638	{
639	DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingBroadcastIpi\n");
640	g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingBroadcastIpi;
641	}
642	else if (g_pfnrtKeSetTargetProcessorDpc)
643	{
644	DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingDpc\n");
645	g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingDpc;
646	/* Windows XP should send always send an IPI -> VERIFY */
647	}
648	else
649	{
650	DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingFailureNotSupported\n");
651	Assert(pOsVerInfo->uMajorVer == 3 && pOsVerInfo->uMinorVer <= 50);
652	g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingFailureNotSupported;
653	}
654
655	return VINF_SUCCESS;
656	}
657
658
659	/**
660	* Called by rtR0TermNative.
661	*/
662	DECLHIDDEN(void) rtR0MpNtTerm(void)
663	{
664	/*
665	* Deregister the processor change callback.
666	*/
667	PVOID pvMpCpuChangeCallback = g_pvMpCpuChangeCallback;
668	g_pvMpCpuChangeCallback = NULL;
669	if (pvMpCpuChangeCallback)
670	{
671	AssertReturnVoid(g_pfnrtKeDeregisterProcessorChangeCallback);
672	g_pfnrtKeDeregisterProcessorChangeCallback(pvMpCpuChangeCallback);
673	}
674	}
675
676
677	DECLHIDDEN(int) rtR0MpNotificationNativeInit(void)
678	{
679	return VINF_SUCCESS;
680	}
681
682
683	DECLHIDDEN(void) rtR0MpNotificationNativeTerm(void)
684	{
685	}
686
687
688	/**
689	* Implements the NT PROCESSOR_CALLBACK_FUNCTION callback function.
690	*
691	* This maintains the g_rtMpNtCpuSet and works MP notification callbacks. When
692	* registered, it's called for each active CPU in the system, avoiding racing
693	* CPU hotplugging (as well as testing the callback).
694	*
695	* @param pvUser User context (not used).
696	* @param pChangeCtx Change context (in).
697	* @param prcOperationStatus Operation status (in/out).
698	*
699	* @remarks ASSUMES no concurrent execution of KeProcessorAddCompleteNotify
700	* notification callbacks. At least during callback registration
701	* callout, we're owning KiDynamicProcessorLock.
702	*
703	* @remarks When registering the handler, we first get KeProcessorAddStartNotify
704	* callbacks for all active CPUs, and after they all succeed we get the
705	* KeProcessorAddCompleteNotify callbacks.
706	*/
707	static VOID __stdcall rtR0NtMpProcessorChangeCallback(void *pvUser, PKE_PROCESSOR_CHANGE_NOTIFY_CONTEXT pChangeCtx,
708	PNTSTATUS prcOperationStatus)
709	{
710	RT_NOREF(pvUser, prcOperationStatus);
711	switch (pChangeCtx->State)
712	{
713	/*
714	* Check whether we can deal with the CPU, failing the start operation if we
715	* can't. The checks we are doing here are to avoid complicated/impossible
716	* cases in KeProcessorAddCompleteNotify. They are really just verify specs.
717	*/
718	case KeProcessorAddStartNotify:
719	{
720	NTSTATUS rcNt = STATUS_SUCCESS;
721	if (pChangeCtx->NtNumber < RTCPUSET_MAX_CPUS)
722	{
723	if (pChangeCtx->NtNumber >= g_cRtMpNtMaxCpus)
724	{
725	DbgPrint("IPRT: KeProcessorAddStartNotify failure: NtNumber=%u is higher than the max CPU count (%u)!\n",
726	pChangeCtx->NtNumber, g_cRtMpNtMaxCpus);
727	rcNt = STATUS_INTERNAL_ERROR;
728	}
729
730	/* The ProcessNumber field was introduced in Windows 7. */
731	PROCESSOR_NUMBER ProcNum;
732	if (g_pfnrtKeGetProcessorIndexFromNumber)
733	{
734	ProcNum = pChangeCtx->ProcNumber;
735	KEPROCESSORINDEX idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);
736	if (idxCpu != pChangeCtx->NtNumber)
737	{
738	DbgPrint("IPRT: KeProcessorAddStartNotify failure: g_pfnrtKeGetProcessorIndexFromNumber(%u.%u) -> %u, expected %u!\n",
739	ProcNum.Group, ProcNum.Number, idxCpu, pChangeCtx->NtNumber);
740	rcNt = STATUS_INTERNAL_ERROR;
741	}
742	}
743	else
744	{
745	ProcNum.Group = 0;
746	ProcNum.Number = pChangeCtx->NtNumber;
747	}
748
749	if ( ProcNum.Group < RT_ELEMENTS(g_aRtMpNtCpuGroups)
750	&& ProcNum.Number < RT_ELEMENTS(g_aRtMpNtCpuGroups[0].aidxCpuSetMembers))
751	{
752	if (ProcNum.Group >= g_cRtMpNtMaxGroups)
753	{
754	DbgPrint("IPRT: KeProcessorAddStartNotify failure: %u.%u is out of range - max groups: %u!\n",
755	ProcNum.Group, ProcNum.Number, g_cRtMpNtMaxGroups);
756	rcNt = STATUS_INTERNAL_ERROR;
757	}
758
759	if (ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus)
760	{
761	Assert(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] != -1);
762	if (g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] == -1)
763	{
764	DbgPrint("IPRT: KeProcessorAddStartNotify failure: Internal error! %u.%u was assigned -1 as set index!\n",
765	ProcNum.Group, ProcNum.Number);
766	rcNt = STATUS_INTERNAL_ERROR;
767	}
768
769	Assert(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] != NIL_RTCPUID);
770	if (g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] == NIL_RTCPUID)
771	{
772	DbgPrint("IPRT: KeProcessorAddStartNotify failure: Internal error! %u (%u.%u) translates to NIL_RTCPUID!\n",
773	pChangeCtx->NtNumber, ProcNum.Group, ProcNum.Number);
774	rcNt = STATUS_INTERNAL_ERROR;
775	}
776	}
777	else
778	{
779	DbgPrint("IPRT: KeProcessorAddStartNotify failure: max processors in group %u is %u, cannot add %u to it!\n",
780	ProcNum.Group, g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus, ProcNum.Group, ProcNum.Number);
781	rcNt = STATUS_INTERNAL_ERROR;
782	}
783	}
784	else
785	{
786	DbgPrint("IPRT: KeProcessorAddStartNotify failure: %u.%u is out of range (max %u.%u)!\n",
787	ProcNum.Group, ProcNum.Number, RT_ELEMENTS(g_aRtMpNtCpuGroups), RT_ELEMENTS(g_aRtMpNtCpuGroups[0].aidxCpuSetMembers));
788	rcNt = STATUS_INTERNAL_ERROR;
789	}
790	}
791	else
792	{
793	DbgPrint("IPRT: KeProcessorAddStartNotify failure: NtNumber=%u is outside RTCPUSET_MAX_CPUS (%u)!\n",
794	pChangeCtx->NtNumber, RTCPUSET_MAX_CPUS);
795	rcNt = STATUS_INTERNAL_ERROR;
796	}
797	if (!NT_SUCCESS(rcNt))
798	*prcOperationStatus = rcNt;
799	break;
800	}
801
802	/*
803	* Update the globals. Since we've checked out range limits and other
804	* limitations already we just AssertBreak here.
805	*/
806	case KeProcessorAddCompleteNotify:
807	{
808	/*
809	* Calc the processor number and assert conditions checked in KeProcessorAddStartNotify.
810	*/
811	AssertBreak(pChangeCtx->NtNumber < RTCPUSET_MAX_CPUS);
812	AssertBreak(pChangeCtx->NtNumber < g_cRtMpNtMaxCpus);
813	Assert(pChangeCtx->NtNumber == g_cRtMpNtActiveCpus); /* light assumption */
814	PROCESSOR_NUMBER ProcNum;
815	if (g_pfnrtKeGetProcessorIndexFromNumber)
816	{
817	ProcNum = pChangeCtx->ProcNumber;
818	AssertBreak(g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum) == pChangeCtx->NtNumber);
819	AssertBreak(ProcNum.Group < RT_ELEMENTS(g_aRtMpNtCpuGroups));
820	AssertBreak(ProcNum.Group < g_cRtMpNtMaxGroups);
821	}
822	else
823	{
824	ProcNum.Group = 0;
825	ProcNum.Number = pChangeCtx->NtNumber;
826	}
827	AssertBreak(ProcNum.Number < RT_ELEMENTS(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers));
828	AssertBreak(ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus);
829	AssertBreak(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] != -1);
830	AssertBreak(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] != NIL_RTCPUID);
831
832	/*
833	* Add ourselves to the online CPU set and update the active CPU count.
834	*/
835	RTCpuSetAddByIndex(&g_rtMpNtCpuSet, pChangeCtx->NtNumber);
836	ASMAtomicIncU32(&g_cRtMpNtActiveCpus);
837
838	/*
839	* Update the group info.
840	*
841	* If the index prediction failed (real hotplugging callbacks only) we
842	* have to switch it around. This is particularly annoying when we
843	* use the index as the ID.
844	*/
845	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
846	RTCPUID idCpu = RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
847	RTCPUID idOld = g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber];
848	if ((idOld & ~RTMPNT_ID_F_INACTIVE) != idCpu)
849	{
850	Assert(idOld & RTMPNT_ID_F_INACTIVE);
851	int idxDest = g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number];
852	g_aRtMpNtCpuGroups[rtMpCpuIdGetGroup(idOld)].aidxCpuSetMembers[rtMpCpuIdGetGroupMember(idOld)] = idxDest;
853	g_aidRtMpNtByCpuSetIdx[idxDest] = idOld;
854	}
855	g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] = idCpu;
856	#else
857	Assert(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] == pChangeCtx->NtNumber);
858	int idxDest = g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number];
859	if ((ULONG)idxDest != pChangeCtx->NtNumber)
860	{
861	bool fFound = false;
862	uint32_t idxOldGroup = g_cRtMpNtMaxGroups;
863	while (idxOldGroup-- > 0 && !fFound)
864	{
865	uint32_t idxMember = g_aRtMpNtCpuGroups[idxOldGroup].cMaxCpus;
866	while (idxMember-- > 0)
867	if (g_aRtMpNtCpuGroups[idxOldGroup].aidxCpuSetMembers[idxMember] == (int)pChangeCtx->NtNumber)
868	{
869	g_aRtMpNtCpuGroups[idxOldGroup].aidxCpuSetMembers[idxMember] = idxDest;
870	fFound = true;
871	break;
872	}
873	}
874	Assert(fFound);
875	}
876	#endif
877	g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] = pChangeCtx->NtNumber;
878
879	/*
880	* Do MP notification callbacks.
881	*/
882	rtMpNotificationDoCallbacks(RTMPEVENT_ONLINE, pChangeCtx->NtNumber);
883	break;
884	}
885
886	case KeProcessorAddFailureNotify:
887	/* ignore */
888	break;
889
890	default:
891	AssertMsgFailed(("State=%u\n", pChangeCtx->State));
892	}
893	}
894
895
896	/**
897	* Wrapper around KeQueryLogicalProcessorRelationship.
898	*
899	* @returns IPRT status code.
900	* @param ppInfo Where to return the info. Pass to RTMemFree when done.
901	*/
902	static int rtR0NtInitQueryGroupRelations(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX **ppInfo)
903	{
904	ULONG cbInfo = sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)
905	+ g_cRtMpNtMaxGroups * sizeof(GROUP_RELATIONSHIP);
906	NTSTATUS rcNt;
907	do
908	{
909	SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX pInfo = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX )RTMemAlloc(cbInfo);
910	if (pInfo)
911	{
912	rcNt = g_pfnrtKeQueryLogicalProcessorRelationship(NULL /pProcNumber/, RelationGroup, pInfo, &cbInfo);
913	if (NT_SUCCESS(rcNt))
914	{
915	*ppInfo = pInfo;
916	return VINF_SUCCESS;
917	}
918
919	RTMemFree(pInfo);
920	pInfo = NULL;
921	}
922	else
923	rcNt = STATUS_NO_MEMORY;
924	} while (rcNt == STATUS_INFO_LENGTH_MISMATCH);
925	DbgPrint("IPRT: Fatal: KeQueryLogicalProcessorRelationship failed: %#x\n", rcNt);
926	AssertMsgFailed(("KeQueryLogicalProcessorRelationship failed: %#x\n", rcNt));
927	return RTErrConvertFromNtStatus(rcNt);
928	}
929
930
931
932
933
934	RTDECL(RTCPUID) RTMpCpuId(void)
935	{
936	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
937
938	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
939	PROCESSOR_NUMBER ProcNum;
940	ProcNum.Group = 0;
941	if (g_pfnrtKeGetCurrentProcessorNumberEx)
942	{
943	ProcNum.Number = 0;
944	g_pfnrtKeGetCurrentProcessorNumberEx(&ProcNum);
945	}
946	else
947	ProcNum.Number = KeGetCurrentProcessorNumber(); /* Number is 8-bit, so we're not subject to BYTE -> WORD upgrade in WDK. */
948	return RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
949
950	#else
951
952	if (g_pfnrtKeGetCurrentProcessorNumberEx)
953	{
954	KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(NULL);
955	Assert(idxCpu < RTCPUSET_MAX_CPUS);
956	return idxCpu;
957	}
958
959	return (uint8_t)KeGetCurrentProcessorNumber(); /* PCR->Number was changed from BYTE to WORD in the WDK, thus the cast. */
960	#endif
961	}
962
963
964	RTDECL(int) RTMpCurSetIndex(void)
965	{
966	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
967	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
968
969	if (g_pfnrtKeGetCurrentProcessorNumberEx)
970	{
971	KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(NULL);
972	Assert(idxCpu < RTCPUSET_MAX_CPUS);
973	return idxCpu;
974	}
975	return (uint8_t)KeGetCurrentProcessorNumber(); /* PCR->Number was changed from BYTE to WORD in the WDK, thus the cast. */
976	#else
977	return (int)RTMpCpuId();
978	#endif
979	}
980
981
982	RTDECL(int) RTMpCurSetIndexAndId(PRTCPUID pidCpu)
983	{
984	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
985	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
986
987	PROCESSOR_NUMBER ProcNum = { 0 , 0, 0 };
988	KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(&ProcNum);
989	Assert(idxCpu < RTCPUSET_MAX_CPUS);
990	*pidCpu = RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
991	return idxCpu;
992	#else
993	return *pidCpu = RTMpCpuId();
994	#endif
995	}
996
997
998	RTDECL(int) RTMpCpuIdToSetIndex(RTCPUID idCpu)
999	{
1000	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
1001	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1002
1003	if (idCpu != NIL_RTCPUID)
1004	{
1005	if (g_pfnrtKeGetProcessorIndexFromNumber)
1006	{
1007	PROCESSOR_NUMBER ProcNum;
1008	ProcNum.Group = rtMpCpuIdGetGroup(idCpu);
1009	ProcNum.Number = rtMpCpuIdGetGroupMember(idCpu);
1010	ProcNum.Reserved = 0;
1011	KEPROCESSORINDEX idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);
1012	if (idxCpu != INVALID_PROCESSOR_INDEX)
1013	{
1014	Assert(idxCpu < g_cRtMpNtMaxCpus);
1015	Assert((ULONG)g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] == idxCpu);
1016	return idxCpu;
1017	}
1018
1019	/* Since NT assigned indexes as the CPUs come online, we cannot produce an ID <-> index
1020	mapping for not-yet-onlined CPUS that is consistent. We just have to do our best... */
1021	if ( ProcNum.Group < g_cRtMpNtMaxGroups
1022	&& ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus)
1023	return g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number];
1024	}
1025	else if (rtMpCpuIdGetGroup(idCpu) == 0)
1026	return rtMpCpuIdGetGroupMember(idCpu);
1027	}
1028	return -1;
1029	#else
1030	/* 1:1 mapping, just do range checks. */
1031	return idCpu < RTCPUSET_MAX_CPUS ? (int)idCpu : -1;
1032	#endif
1033	}
1034
1035
1036	RTDECL(RTCPUID) RTMpCpuIdFromSetIndex(int iCpu)
1037	{
1038	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
1039	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1040
1041	if ((unsigned)iCpu < g_cRtMpNtMaxCpus)
1042	{
1043	if (g_pfnrtKeGetProcessorIndexFromNumber)
1044	{
1045	PROCESSOR_NUMBER ProcNum = { 0, 0, 0 };
1046	NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(iCpu, &ProcNum);
1047	if (NT_SUCCESS(rcNt))
1048	{
1049	Assert(ProcNum.Group <= g_cRtMpNtMaxGroups);
1050	Assert( (g_aidRtMpNtByCpuSetIdx[iCpu] & ~RTMPNT_ID_F_INACTIVE)
1051	== RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number));
1052	return RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
1053	}
1054	}
1055	return g_aidRtMpNtByCpuSetIdx[iCpu];
1056	}
1057	return NIL_RTCPUID;
1058	#else
1059	/* 1:1 mapping, just do range checks. */
1060	return (unsigned)iCpu < RTCPUSET_MAX_CPUS ? iCpu : NIL_RTCPUID;
1061	#endif
1062	}
1063
1064
1065	RTDECL(int) RTMpSetIndexFromCpuGroupMember(uint32_t idxGroup, uint32_t idxMember)
1066	{
1067	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1068
1069	if (idxGroup < g_cRtMpNtMaxGroups)
1070	if (idxMember < g_aRtMpNtCpuGroups[idxGroup].cMaxCpus)
1071	return g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember];
1072	return -1;
1073	}
1074
1075
1076	RTDECL(uint32_t) RTMpGetCpuGroupCounts(uint32_t idxGroup, uint32_t *pcActive)
1077	{
1078	if (idxGroup < g_cRtMpNtMaxGroups)
1079	{
1080	if (pcActive)
1081	*pcActive = g_aRtMpNtCpuGroups[idxGroup].cActiveCpus;
1082	return g_aRtMpNtCpuGroups[idxGroup].cMaxCpus;
1083	}
1084	if (pcActive)
1085	*pcActive = 0;
1086	return 0;
1087	}
1088
1089
1090	RTDECL(uint32_t) RTMpGetMaxCpuGroupCount(void)
1091	{
1092	return g_cRtMpNtMaxGroups;
1093	}
1094
1095
1096	RTDECL(RTCPUID) RTMpGetMaxCpuId(void)
1097	{
1098	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1099
1100	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
1101	return RTMPCPUID_FROM_GROUP_AND_NUMBER(g_cRtMpNtMaxGroups - 1, g_aRtMpNtCpuGroups[g_cRtMpNtMaxGroups - 1].cMaxCpus - 1);
1102	#else
1103	/* According to MSDN the processor indexes goes from 0 to the maximum
1104	number of CPUs in the system. We've check this in initterm-r0drv-nt.cpp. */
1105	return g_cRtMpNtMaxCpus - 1;
1106	#endif
1107	}
1108
1109
1110	RTDECL(bool) RTMpIsCpuOnline(RTCPUID idCpu)
1111	{
1112	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1113	return RTCpuSetIsMember(&g_rtMpNtCpuSet, idCpu);
1114	}
1115
1116
1117	RTDECL(bool) RTMpIsCpuPossible(RTCPUID idCpu)
1118	{
1119	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1120
1121	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
1122	if (idCpu != NIL_RTCPUID)
1123	{
1124	unsigned idxGroup = rtMpCpuIdGetGroup(idCpu);
1125	if (idxGroup < g_cRtMpNtMaxGroups)
1126	return rtMpCpuIdGetGroupMember(idCpu) < g_aRtMpNtCpuGroups[idxGroup].cMaxCpus;
1127	}
1128	return false;
1129
1130	#else
1131	/* A possible CPU ID is one with a value lower than g_cRtMpNtMaxCpus (see
1132	comment in RTMpGetMaxCpuId). */
1133	return idCpu < g_cRtMpNtMaxCpus;
1134	#endif
1135	}
1136
1137
1138
1139	RTDECL(PRTCPUSET) RTMpGetSet(PRTCPUSET pSet)
1140	{
1141	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1142
1143	/* The set of possible CPU IDs(/indexes) are from 0 up to
1144	g_cRtMpNtMaxCpus (see comment in RTMpGetMaxCpuId). */
1145	RTCpuSetEmpty(pSet);
1146	int idxCpu = g_cRtMpNtMaxCpus;
1147	while (idxCpu-- > 0)
1148	RTCpuSetAddByIndex(pSet, idxCpu);
1149	return pSet;
1150	}
1151
1152
1153	RTDECL(RTCPUID) RTMpGetCount(void)
1154	{
1155	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1156	return g_cRtMpNtMaxCpus;
1157	}
1158
1159
1160	RTDECL(PRTCPUSET) RTMpGetOnlineSet(PRTCPUSET pSet)
1161	{
1162	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1163
1164	*pSet = g_rtMpNtCpuSet;
1165	return pSet;
1166	}
1167
1168
1169	RTDECL(RTCPUID) RTMpGetOnlineCount(void)
1170	{
1171	RTCPUSET Set;
1172	RTMpGetOnlineSet(&Set);
1173	return RTCpuSetCount(&Set);
1174	}
1175
1176
1177	RTDECL(RTCPUID) RTMpGetOnlineCoreCount(void)
1178	{
1179	/** @todo fix me */
1180	return RTMpGetOnlineCount();
1181	}
1182
1183
1184
1185	#if 0
1186	/* Experiment with checking the undocumented KPRCB structure
1187	* 'dt nt!_kprcb 0xaddress' shows the layout
1188	*/
1189	typedef struct
1190	{
1191	LIST_ENTRY DpcListHead;
1192	ULONG_PTR DpcLock;
1193	volatile ULONG DpcQueueDepth;
1194	ULONG DpcQueueCount;
1195	} KDPC_DATA, *PKDPC_DATA;
1196
1197	RTDECL(bool) RTMpIsCpuWorkPending(void)
1198	{
1199	uint8_t *pkprcb;
1200	PKDPC_DATA pDpcData;
1201
1202	_asm {
1203	mov eax, fs:0x20
1204	mov pkprcb, eax
1205	}
1206	pDpcData = (PKDPC_DATA)(pkprcb + 0x19e0);
1207	if (pDpcData->DpcQueueDepth)
1208	return true;
1209
1210	pDpcData++;
1211	if (pDpcData->DpcQueueDepth)
1212	return true;
1213	return false;
1214	}
1215	#else
1216	RTDECL(bool) RTMpIsCpuWorkPending(void)
1217	{
1218	/** @todo not implemented */
1219	return false;
1220	}
1221	#endif
1222
1223
1224	/**
1225	* Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1226	* the RTMpOnAll case.
1227	*
1228	* @param uUserCtx The user context argument (PRTMPARGS).
1229	*/
1230	static ULONG_PTR rtmpNtOnAllBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1231	{
1232	PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1233	/ASMAtomicIncU32(&pArgs->cHits); - not needed /
1234	pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2);
1235	return 0;
1236	}
1237
1238
1239	/**
1240	* Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1241	* the RTMpOnOthers case.
1242	*
1243	* @param uUserCtx The user context argument (PRTMPARGS).
1244	*/
1245	static ULONG_PTR rtmpNtOnOthersBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1246	{
1247	PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1248	RTCPUID idCpu = RTMpCpuId();
1249	if (pArgs->idCpu != idCpu)
1250	{
1251	/ASMAtomicIncU32(&pArgs->cHits); - not needed /
1252	pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
1253	}
1254	return 0;
1255	}
1256
1257
1258	/**
1259	* Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1260	* the RTMpOnPair case.
1261	*
1262	* @param uUserCtx The user context argument (PRTMPARGS).
1263	*/
1264	static ULONG_PTR rtmpNtOnPairBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1265	{
1266	PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1267	RTCPUID idCpu = RTMpCpuId();
1268	if ( pArgs->idCpu == idCpu
1269	\|\| pArgs->idCpu2 == idCpu)
1270	{
1271	ASMAtomicIncU32(&pArgs->cHits);
1272	pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
1273	}
1274	return 0;
1275	}
1276
1277
1278	/**
1279	* Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1280	* the RTMpOnSpecific case.
1281	*
1282	* @param uUserCtx The user context argument (PRTMPARGS).
1283	*/
1284	static ULONG_PTR rtmpNtOnSpecificBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1285	{
1286	PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1287	RTCPUID idCpu = RTMpCpuId();
1288	if (pArgs->idCpu == idCpu)
1289	{
1290	ASMAtomicIncU32(&pArgs->cHits);
1291	pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
1292	}
1293	return 0;
1294	}
1295
1296
1297	/**
1298	* Internal worker for the RTMpOn* APIs using KeIpiGenericCall.
1299	*
1300	* @returns VINF_SUCCESS.
1301	* @param pfnWorker The callback.
1302	* @param pvUser1 User argument 1.
1303	* @param pvUser2 User argument 2.
1304	* @param pfnNativeWrapper The wrapper between the NT and IPRT callbacks.
1305	* @param idCpu First CPU to match, ultimately specific to the
1306	* pfnNativeWrapper used.
1307	* @param idCpu2 Second CPU to match, ultimately specific to the
1308	* pfnNativeWrapper used.
1309	* @param pcHits Where to return the number of this. Optional.
1310	*/
1311	static int rtMpCallUsingBroadcastIpi(PFNRTMPWORKER pfnWorker, void pvUser1, void pvUser2,
1312	PKIPI_BROADCAST_WORKER pfnNativeWrapper, RTCPUID idCpu, RTCPUID idCpu2,
1313	uint32_t *pcHits)
1314	{
1315	RTMPARGS Args;
1316	Args.pfnWorker = pfnWorker;
1317	Args.pvUser1 = pvUser1;
1318	Args.pvUser2 = pvUser2;
1319	Args.idCpu = idCpu;
1320	Args.idCpu2 = idCpu2;
1321	Args.cRefs = 0;
1322	Args.cHits = 0;
1323
1324	AssertPtr(g_pfnrtKeIpiGenericCall);
1325	g_pfnrtKeIpiGenericCall(pfnNativeWrapper, (uintptr_t)&Args);
1326	if (pcHits)
1327	*pcHits = Args.cHits;
1328	return VINF_SUCCESS;
1329	}
1330
1331
1332	/**
1333	* Wrapper between the native nt per-cpu callbacks and PFNRTWORKER
1334	*
1335	* @param Dpc DPC object
1336	* @param DeferredContext Context argument specified by KeInitializeDpc
1337	* @param SystemArgument1 Argument specified by KeInsertQueueDpc
1338	* @param SystemArgument2 Argument specified by KeInsertQueueDpc
1339	*/
1340	static VOID rtmpNtDPCWrapper(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2)
1341	{
1342	PRTMPARGS pArgs = (PRTMPARGS)DeferredContext;
1343	RT_NOREF3(Dpc, SystemArgument1, SystemArgument2);
1344
1345	ASMAtomicIncU32(&pArgs->cHits);
1346	pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2);
1347
1348	/* Dereference the argument structure. */
1349	int32_t cRefs = ASMAtomicDecS32(&pArgs->cRefs);
1350	Assert(cRefs >= 0);
1351	if (cRefs == 0)
1352	RTMemFree(pArgs);
1353	}
1354
1355
1356	/**
1357	* Wrapper around KeSetTargetProcessorDpcEx / KeSetTargetProcessorDpc.
1358	*
1359	* This is shared with the timer code.
1360	*
1361	* @returns IPRT status code (errors are asserted).
1362	* @retval VERR_CPU_NOT_FOUND if impossible CPU. Not asserted.
1363	* @param pDpc The DPC.
1364	* @param idCpu The ID of the new target CPU.
1365	* @note Callable at any IRQL.
1366	*/
1367	DECLHIDDEN(int) rtMpNtSetTargetProcessorDpc(KDPC *pDpc, RTCPUID idCpu)
1368	{
1369	if (g_pfnrtKeSetTargetProcessorDpcEx)
1370	{
1371	/* Convert to stupid process number (bet KeSetTargetProcessorDpcEx does
1372	the reverse conversion internally). */
1373	PROCESSOR_NUMBER ProcNum;
1374	NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(RTMpCpuIdToSetIndex(idCpu), &ProcNum);
1375	if (NT_SUCCESS(rcNt))
1376	{
1377	rcNt = g_pfnrtKeSetTargetProcessorDpcEx(pDpc, &ProcNum);
1378	AssertLogRelMsgReturn(NT_SUCCESS(rcNt),
1379	("KeSetTargetProcessorDpcEx(,%u(%u/%u)) -> %#x\n", idCpu, ProcNum.Group, ProcNum.Number, rcNt),
1380	RTErrConvertFromNtStatus(rcNt));
1381	}
1382	else if (rcNt == STATUS_INVALID_PARAMETER)
1383	return VERR_CPU_NOT_FOUND;
1384	else
1385	AssertLogRelMsgReturn(NT_SUCCESS(rcNt), ("KeGetProcessorNumberFromIndex(%u) -> %#x\n", idCpu, rcNt),
1386	RTErrConvertFromNtStatus(rcNt));
1387
1388	}
1389	else if (g_pfnrtKeSetTargetProcessorDpc)
1390	g_pfnrtKeSetTargetProcessorDpc(pDpc, RTMpCpuIdToSetIndex(idCpu));
1391	else
1392	return VERR_NOT_SUPPORTED;
1393	return VINF_SUCCESS;
1394	}
1395
1396
1397	/**
1398	* Internal worker for the RTMpOn* APIs.
1399	*
1400	* @returns IPRT status code.
1401	* @param pfnWorker The callback.
1402	* @param pvUser1 User argument 1.
1403	* @param pvUser2 User argument 2.
1404	* @param enmCpuid What to do / is idCpu valid.
1405	* @param idCpu Used if enmCpuid is RT_NT_CPUID_SPECIFIC or
1406	* RT_NT_CPUID_PAIR, otherwise ignored.
1407	* @param idCpu2 Used if enmCpuid is RT_NT_CPUID_PAIR, otherwise ignored.
1408	* @param pcHits Where to return the number of this. Optional.
1409	*/
1410	static int rtMpCallUsingDpcs(PFNRTMPWORKER pfnWorker, void pvUser1, void pvUser2,
1411	RT_NT_CPUID enmCpuid, RTCPUID idCpu, RTCPUID idCpu2, uint32_t *pcHits)
1412	{
1413	#if 0
1414	/* KeFlushQueuedDpcs must be run at IRQL PASSIVE_LEVEL according to MSDN, but the
1415	* driver verifier doesn't complain...
1416	*/
1417	AssertMsg(KeGetCurrentIrql() == PASSIVE_LEVEL, ("%d != %d (PASSIVE_LEVEL)\n", KeGetCurrentIrql(), PASSIVE_LEVEL));
1418	#endif
1419	/* KeFlushQueuedDpcs is not present in Windows 2000; import it dynamically so we can just fail this call. */
1420	if (!g_pfnrtNtKeFlushQueuedDpcs)
1421	return VERR_NOT_SUPPORTED;
1422
1423	/*
1424	* Make a copy of the active CPU set and figure out how many KDPCs we really need.
1425	* We must not try setup DPCs for CPUs which aren't there, because that may fail.
1426	*/
1427	RTCPUSET OnlineSet = g_rtMpNtCpuSet;
1428	uint32_t cDpcsNeeded;
1429	switch (enmCpuid)
1430	{
1431	case RT_NT_CPUID_SPECIFIC:
1432	cDpcsNeeded = 1;
1433	break;
1434	case RT_NT_CPUID_PAIR:
1435	cDpcsNeeded = 2;
1436	break;
1437	default:
1438	do
1439	{
1440	cDpcsNeeded = g_cRtMpNtActiveCpus;
1441	OnlineSet = g_rtMpNtCpuSet;
1442	} while (cDpcsNeeded != g_cRtMpNtActiveCpus);
1443	break;
1444	}
1445
1446	/*
1447	* Allocate an RTMPARGS structure followed by cDpcsNeeded KDPCs
1448	* and initialize them.
1449	*/
1450	PRTMPARGS pArgs = (PRTMPARGS)RTMemAllocZ(sizeof(RTMPARGS) + cDpcsNeeded * sizeof(KDPC));
1451	if (!pArgs)
1452	return VERR_NO_MEMORY;
1453
1454	pArgs->pfnWorker = pfnWorker;
1455	pArgs->pvUser1 = pvUser1;
1456	pArgs->pvUser2 = pvUser2;
1457	pArgs->idCpu = NIL_RTCPUID;
1458	pArgs->idCpu2 = NIL_RTCPUID;
1459	pArgs->cHits = 0;
1460	pArgs->cRefs = 1;
1461
1462	int rc;
1463	KDPC paExecCpuDpcs = (KDPC )(pArgs + 1);
1464	if (enmCpuid == RT_NT_CPUID_SPECIFIC)
1465	{
1466	KeInitializeDpc(&paExecCpuDpcs[0], rtmpNtDPCWrapper, pArgs);
1467	if (g_pfnrtKeSetImportanceDpc)
1468	g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[0], HighImportance);
1469	rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[0], idCpu);
1470	pArgs->idCpu = idCpu;
1471	}
1472	else if (enmCpuid == RT_NT_CPUID_PAIR)
1473	{
1474	KeInitializeDpc(&paExecCpuDpcs[0], rtmpNtDPCWrapper, pArgs);
1475	if (g_pfnrtKeSetImportanceDpc)
1476	g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[0], HighImportance);
1477	rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[0], idCpu);
1478	pArgs->idCpu = idCpu;
1479
1480	KeInitializeDpc(&paExecCpuDpcs[1], rtmpNtDPCWrapper, pArgs);
1481	if (g_pfnrtKeSetImportanceDpc)
1482	g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[1], HighImportance);
1483	if (RT_SUCCESS(rc))
1484	rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[1], (int)idCpu2);
1485	pArgs->idCpu2 = idCpu2;
1486	}
1487	else
1488	{
1489	rc = VINF_SUCCESS;
1490	for (uint32_t i = 0; i < cDpcsNeeded && RT_SUCCESS(rc); i++)
1491	if (RTCpuSetIsMemberByIndex(&OnlineSet, i))
1492	{
1493	KeInitializeDpc(&paExecCpuDpcs[i], rtmpNtDPCWrapper, pArgs);
1494	if (g_pfnrtKeSetImportanceDpc)
1495	g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[i], HighImportance);
1496	rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[i], RTMpCpuIdFromSetIndex(i));
1497	}
1498	}
1499	if (RT_FAILURE(rc))
1500	{
1501	RTMemFree(pArgs);
1502	return rc;
1503	}
1504
1505	/*
1506	* Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu.
1507	* KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL.
1508	*/
1509	KIRQL oldIrql;
1510	KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
1511
1512	/*
1513	* We cannot do other than assume a 1:1 relationship between the
1514	* affinity mask and the process despite the warnings in the docs.
1515	* If someone knows a better way to get this done, please let bird know.
1516	*/
1517	ASMCompilerBarrier(); /* paranoia */
1518	if (enmCpuid == RT_NT_CPUID_SPECIFIC)
1519	{
1520	ASMAtomicIncS32(&pArgs->cRefs);
1521	BOOLEAN fRc = KeInsertQueueDpc(&paExecCpuDpcs[0], 0, 0);
1522	Assert(fRc); NOREF(fRc);
1523	}
1524	else if (enmCpuid == RT_NT_CPUID_PAIR)
1525	{
1526	ASMAtomicIncS32(&pArgs->cRefs);
1527	BOOLEAN fRc = KeInsertQueueDpc(&paExecCpuDpcs[0], 0, 0);
1528	Assert(fRc); NOREF(fRc);
1529
1530	ASMAtomicIncS32(&pArgs->cRefs);
1531	fRc = KeInsertQueueDpc(&paExecCpuDpcs[1], 0, 0);
1532	Assert(fRc); NOREF(fRc);
1533	}
1534	else
1535	{
1536	uint32_t iSelf = RTMpCurSetIndex();
1537	for (uint32_t i = 0; i < cDpcsNeeded; i++)
1538	{
1539	if ( (i != iSelf)
1540	&& RTCpuSetIsMemberByIndex(&OnlineSet, i))
1541	{
1542	ASMAtomicIncS32(&pArgs->cRefs);
1543	BOOLEAN fRc = KeInsertQueueDpc(&paExecCpuDpcs[i], 0, 0);
1544	Assert(fRc); NOREF(fRc);
1545	}
1546	}
1547	if (enmCpuid != RT_NT_CPUID_OTHERS)
1548	pfnWorker(iSelf, pvUser1, pvUser2);
1549	}
1550
1551	KeLowerIrql(oldIrql);
1552
1553	/*
1554	* Flush all DPCs and wait for completion. (can take long!)
1555	*/
1556	/** @todo Consider changing this to an active wait using some atomic inc/dec
1557	* stuff (and check for the current cpu above in the specific case). */
1558	/** @todo Seems KeFlushQueuedDpcs doesn't wait for the DPCs to be completely
1559	* executed. Seen pArgs being freed while some CPU was using it before
1560	* cRefs was added. */
1561	if (g_pfnrtNtKeFlushQueuedDpcs)
1562	g_pfnrtNtKeFlushQueuedDpcs();
1563
1564	if (pcHits)
1565	*pcHits = pArgs->cHits;
1566
1567	/* Dereference the argument structure. */
1568	int32_t cRefs = ASMAtomicDecS32(&pArgs->cRefs);
1569	Assert(cRefs >= 0);
1570	if (cRefs == 0)
1571	RTMemFree(pArgs);
1572
1573	return VINF_SUCCESS;
1574	}
1575
1576
1577	RTDECL(int) RTMpOnAll(PFNRTMPWORKER pfnWorker, void pvUser1, void pvUser2)
1578	{
1579	if (g_pfnrtKeIpiGenericCall)
1580	return rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnAllBroadcastIpiWrapper,
1581	NIL_RTCPUID, NIL_RTCPUID, NULL);
1582	return rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_ALL, NIL_RTCPUID, NIL_RTCPUID, NULL);
1583	}
1584
1585
1586	RTDECL(int) RTMpOnOthers(PFNRTMPWORKER pfnWorker, void pvUser1, void pvUser2)
1587	{
1588	if (g_pfnrtKeIpiGenericCall)
1589	return rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnOthersBroadcastIpiWrapper,
1590	NIL_RTCPUID, NIL_RTCPUID, NULL);
1591	return rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_OTHERS, NIL_RTCPUID, NIL_RTCPUID, NULL);
1592	}
1593
1594
1595	RTDECL(int) RTMpOnPair(RTCPUID idCpu1, RTCPUID idCpu2, uint32_t fFlags, PFNRTMPWORKER pfnWorker, void pvUser1, void pvUser2)
1596	{
1597	int rc;
1598	AssertReturn(idCpu1 != idCpu2, VERR_INVALID_PARAMETER);
1599	AssertReturn(!(fFlags & RTMPON_F_VALID_MASK), VERR_INVALID_FLAGS);
1600	if ((fFlags & RTMPON_F_CONCURRENT_EXEC) && !g_pfnrtKeIpiGenericCall)
1601	return VERR_NOT_SUPPORTED;
1602
1603	/*
1604	* Check that both CPUs are online before doing the broadcast call.
1605	*/
1606	if ( RTMpIsCpuOnline(idCpu1)
1607	&& RTMpIsCpuOnline(idCpu2))
1608	{
1609	/*
1610	* The broadcast IPI isn't quite as bad as it could have been, because
1611	* it looks like windows doesn't synchronize CPUs on the way out, they
1612	* seems to get back to normal work while the pair is still busy.
1613	*/
1614	uint32_t cHits = 0;
1615	if (g_pfnrtKeIpiGenericCall)
1616	rc = rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnPairBroadcastIpiWrapper, idCpu1, idCpu2, &cHits);
1617	else
1618	rc = rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_PAIR, idCpu1, idCpu2, &cHits);
1619	if (RT_SUCCESS(rc))
1620	{
1621	Assert(cHits <= 2);
1622	if (cHits == 2)
1623	rc = VINF_SUCCESS;
1624	else if (cHits == 1)
1625	rc = VERR_NOT_ALL_CPUS_SHOWED;
1626	else if (cHits == 0)
1627	rc = VERR_CPU_OFFLINE;
1628	else
1629	rc = VERR_CPU_IPE_1;
1630	}
1631	}
1632	/*
1633	* A CPU must be present to be considered just offline.
1634	*/
1635	else if ( RTMpIsCpuPresent(idCpu1)
1636	&& RTMpIsCpuPresent(idCpu2))
1637	rc = VERR_CPU_OFFLINE;
1638	else
1639	rc = VERR_CPU_NOT_FOUND;
1640	return rc;
1641	}
1642
1643
1644	RTDECL(bool) RTMpOnPairIsConcurrentExecSupported(void)
1645	{
1646	return g_pfnrtKeIpiGenericCall != NULL;
1647	}
1648
1649
1650	/**
1651	* Releases a reference to a RTMPNTONSPECIFICARGS heap allocation, freeing it
1652	* when the last reference is released.
1653	*/
1654	DECLINLINE(void) rtMpNtOnSpecificRelease(PRTMPNTONSPECIFICARGS pArgs)
1655	{
1656	uint32_t cRefs = ASMAtomicDecU32(&pArgs->cRefs);
1657	AssertMsg(cRefs <= 1, ("cRefs=%#x\n", cRefs));
1658	if (cRefs == 0)
1659	RTMemFree(pArgs);
1660	}
1661
1662
1663	/**
1664	* Wrapper between the native nt per-cpu callbacks and PFNRTWORKER
1665	*
1666	* @param Dpc DPC object
1667	* @param DeferredContext Context argument specified by KeInitializeDpc
1668	* @param SystemArgument1 Argument specified by KeInsertQueueDpc
1669	* @param SystemArgument2 Argument specified by KeInsertQueueDpc
1670	*/
1671	static VOID rtMpNtOnSpecificDpcWrapper(IN PKDPC Dpc, IN PVOID DeferredContext,
1672	IN PVOID SystemArgument1, IN PVOID SystemArgument2)
1673	{
1674	PRTMPNTONSPECIFICARGS pArgs = (PRTMPNTONSPECIFICARGS)DeferredContext;
1675	RT_NOREF3(Dpc, SystemArgument1, SystemArgument2);
1676
1677	ASMAtomicWriteBool(&pArgs->fExecuting, true);
1678
1679	pArgs->CallbackArgs.pfnWorker(RTMpCpuId(), pArgs->CallbackArgs.pvUser1, pArgs->CallbackArgs.pvUser2);
1680
1681	ASMAtomicWriteBool(&pArgs->fDone, true);
1682	KeSetEvent(&pArgs->DoneEvt, 1 /PriorityIncrement/, FALSE /Wait/);
1683
1684	rtMpNtOnSpecificRelease(pArgs);
1685	}
1686
1687
1688	RTDECL(int) RTMpOnSpecific(RTCPUID idCpu, PFNRTMPWORKER pfnWorker, void pvUser1, void pvUser2)
1689	{
1690	/*
1691	* Don't try mess with an offline CPU.
1692	*/
1693	if (!RTMpIsCpuOnline(idCpu))
1694	return !RTMpIsCpuPossible(idCpu)
1695	? VERR_CPU_NOT_FOUND
1696	: VERR_CPU_OFFLINE;
1697
1698	/*
1699	* Use the broadcast IPI routine if there are no more than two CPUs online,
1700	* or if the current IRQL is unsuitable for KeWaitForSingleObject.
1701	*/
1702	int rc;
1703	uint32_t cHits = 0;
1704	if ( g_pfnrtKeIpiGenericCall
1705	&& ( RTMpGetOnlineCount() <= 2
1706	\|\| KeGetCurrentIrql() > APC_LEVEL)
1707	)
1708	{
1709	rc = rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnSpecificBroadcastIpiWrapper,
1710	idCpu, NIL_RTCPUID, &cHits);
1711	if (RT_SUCCESS(rc))
1712	{
1713	if (cHits == 1)
1714	return VINF_SUCCESS;
1715	rc = cHits == 0 ? VERR_CPU_OFFLINE : VERR_CPU_IPE_1;
1716	}
1717	return rc;
1718	}
1719
1720	#if 0
1721	rc = rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_SPECIFIC, idCpu, NIL_RTCPUID, &cHits);
1722	if (RT_SUCCESS(rc))
1723	{
1724	if (cHits == 1)
1725	return VINF_SUCCESS;
1726	rc = cHits == 0 ? VERR_CPU_OFFLINE : VERR_CPU_IPE_1;
1727	}
1728	return rc;
1729
1730	#else
1731	/*
1732	* Initialize the argument package and the objects within it.
1733	* The package is referenced counted to avoid unnecessary spinning to
1734	* synchronize cleanup and prevent stack corruption.
1735	*/
1736	PRTMPNTONSPECIFICARGS pArgs = (PRTMPNTONSPECIFICARGS)RTMemAllocZ(sizeof(*pArgs));
1737	if (!pArgs)
1738	return VERR_NO_MEMORY;
1739	pArgs->cRefs = 2;
1740	pArgs->fExecuting = false;
1741	pArgs->fDone = false;
1742	pArgs->CallbackArgs.pfnWorker = pfnWorker;
1743	pArgs->CallbackArgs.pvUser1 = pvUser1;
1744	pArgs->CallbackArgs.pvUser2 = pvUser2;
1745	pArgs->CallbackArgs.idCpu = idCpu;
1746	pArgs->CallbackArgs.cHits = 0;
1747	pArgs->CallbackArgs.cRefs = 2;
1748	KeInitializeEvent(&pArgs->DoneEvt, SynchronizationEvent, FALSE /* not signalled */);
1749	KeInitializeDpc(&pArgs->Dpc, rtMpNtOnSpecificDpcWrapper, pArgs);
1750	if (g_pfnrtKeSetImportanceDpc)
1751	g_pfnrtKeSetImportanceDpc(&pArgs->Dpc, HighImportance);
1752	rc = rtMpNtSetTargetProcessorDpc(&pArgs->Dpc, idCpu);
1753	if (RT_FAILURE(rc))
1754	{
1755	RTMemFree(pArgs);
1756	return rc;
1757	}
1758
1759	/*
1760	* Disable preemption while we check the current processor and inserts the DPC.
1761	*/
1762	KIRQL bOldIrql;
1763	KeRaiseIrql(DISPATCH_LEVEL, &bOldIrql);
1764	ASMCompilerBarrier(); /* paranoia */
1765
1766	if (RTMpCpuId() == idCpu)
1767	{
1768	/* Just execute the callback on the current CPU. */
1769	pfnWorker(idCpu, pvUser1, pvUser2);
1770	KeLowerIrql(bOldIrql);
1771
1772	RTMemFree(pArgs);
1773	return VINF_SUCCESS;
1774	}
1775
1776	/* Different CPU, so queue it if the CPU is still online. */
1777	if (RTMpIsCpuOnline(idCpu))
1778	{
1779	BOOLEAN fRc = KeInsertQueueDpc(&pArgs->Dpc, 0, 0);
1780	Assert(fRc); NOREF(fRc);
1781	KeLowerIrql(bOldIrql);
1782
1783	uint64_t const nsRealWaitTS = RTTimeNanoTS();
1784
1785	/*
1786	* Wait actively for a while in case the CPU/thread responds quickly.
1787	*/
1788	uint32_t cLoopsLeft = 0x20000;
1789	while (cLoopsLeft-- > 0)
1790	{
1791	if (pArgs->fDone)
1792	{
1793	rtMpNtOnSpecificRelease(pArgs);
1794	return VINF_SUCCESS;
1795	}
1796	ASMNopPause();
1797	}
1798
1799	/*
1800	* It didn't respond, so wait on the event object, poking the CPU if it's slow.
1801	*/
1802	LARGE_INTEGER Timeout;
1803	Timeout.QuadPart = -10000; /* 1ms */
1804	NTSTATUS rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
1805	if (rcNt == STATUS_SUCCESS)
1806	{
1807	rtMpNtOnSpecificRelease(pArgs);
1808	return VINF_SUCCESS;
1809	}
1810
1811	/* If it hasn't respondend yet, maybe poke it and wait some more. */
1812	if (rcNt == STATUS_TIMEOUT)
1813	{
1814	if ( !pArgs->fExecuting
1815	&& ( g_pfnrtMpPokeCpuWorker == rtMpPokeCpuUsingHalRequestIpiW7Plus
1816	\|\| g_pfnrtMpPokeCpuWorker == rtMpPokeCpuUsingHalRequestIpiPreW7))
1817	RTMpPokeCpu(idCpu);
1818
1819	Timeout.QuadPart = -1280000; /* 128ms */
1820	rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
1821	if (rcNt == STATUS_SUCCESS)
1822	{
1823	rtMpNtOnSpecificRelease(pArgs);
1824	return VINF_SUCCESS;
1825	}
1826	}
1827
1828	/*
1829	* Something weird is happening, try bail out.
1830	*/
1831	if (KeRemoveQueueDpc(&pArgs->Dpc))
1832	{
1833	RTMemFree(pArgs); /* DPC was still queued, so we can return without further ado. */
1834	LogRel(("RTMpOnSpecific(%#x): Not processed after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
1835	}
1836	else
1837	{
1838	/* DPC is running, wait a good while for it to complete. */
1839	LogRel(("RTMpOnSpecific(%#x): Still running after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
1840
1841	Timeout.QuadPart = -301000100010; / 30 seconds */
1842	rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
1843	if (rcNt != STATUS_SUCCESS)
1844	LogRel(("RTMpOnSpecific(%#x): Giving up on running worker after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
1845	}
1846	rc = RTErrConvertFromNtStatus(rcNt);
1847	}
1848	else
1849	{
1850	/* CPU is offline.*/
1851	KeLowerIrql(bOldIrql);
1852	rc = !RTMpIsCpuPossible(idCpu) ? VERR_CPU_NOT_FOUND : VERR_CPU_OFFLINE;
1853	}
1854
1855	rtMpNtOnSpecificRelease(pArgs);
1856	return rc;
1857	#endif
1858	}
1859
1860
1861
1862
1863	static VOID rtMpNtPokeCpuDummy(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2)
1864	{
1865	NOREF(Dpc);
1866	NOREF(DeferredContext);
1867	NOREF(SystemArgument1);
1868	NOREF(SystemArgument2);
1869	}
1870
1871
1872	/** Callback used by rtMpPokeCpuUsingBroadcastIpi. */
1873	static ULONG_PTR rtMpIpiGenericCall(ULONG_PTR Argument)
1874	{
1875	NOREF(Argument);
1876	return 0;
1877	}
1878
1879
1880	/**
1881	* RTMpPokeCpu worker that uses broadcast IPIs for doing the work.
1882	*
1883	* @returns VINF_SUCCESS
1884	* @param idCpu The CPU identifier.
1885	*/
1886	int rtMpPokeCpuUsingBroadcastIpi(RTCPUID idCpu)
1887	{
1888	NOREF(idCpu);
1889	g_pfnrtKeIpiGenericCall(rtMpIpiGenericCall, 0);
1890	return VINF_SUCCESS;
1891	}
1892
1893
1894	/**
1895	* RTMpPokeCpu worker that uses the Windows 7 and later version of
1896	* HalRequestIpip to get the job done.
1897	*
1898	* @returns VINF_SUCCESS
1899	* @param idCpu The CPU identifier.
1900	*/
1901	int rtMpPokeCpuUsingHalRequestIpiW7Plus(RTCPUID idCpu)
1902	{
1903	/* idCpu is an HAL processor index, so we can use it directly. */
1904	PKAFFINITY_EX pTarget = (PKAFFINITY_EX)alloca(g_cbRtMpNtKaffinityEx);
1905	pTarget->Size = g_cRtMpNtKaffinityExEntries; /* (just in case KeInitializeAffinityEx starts using it) */
1906	g_pfnrtKeInitializeAffinityEx(pTarget);
1907	g_pfnrtKeAddProcessorAffinityEx(pTarget, idCpu);
1908
1909	g_pfnrtHalRequestIpiW7Plus(0, pTarget);
1910	return VINF_SUCCESS;
1911	}
1912
1913
1914	/**
1915	* RTMpPokeCpu worker that uses the Vista and earlier version of HalRequestIpip
1916	* to get the job done.
1917	*
1918	* @returns VINF_SUCCESS
1919	* @param idCpu The CPU identifier.
1920	*/
1921	int rtMpPokeCpuUsingHalRequestIpiPreW7(RTCPUID idCpu)
1922	{
1923	__debugbreak(); /** @todo this code needs testing!! */
1924	KAFFINITY Target = 1;
1925	Target <<= idCpu;
1926	g_pfnrtHalRequestIpiPreW7(Target);
1927	return VINF_SUCCESS;
1928	}
1929
1930
1931	int rtMpPokeCpuUsingFailureNotSupported(RTCPUID idCpu)
1932	{
1933	NOREF(idCpu);
1934	return VERR_NOT_SUPPORTED;
1935	}
1936
1937
1938	int rtMpPokeCpuUsingDpc(RTCPUID idCpu)
1939	{
1940	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1941
1942	/*
1943	* APC fallback.
1944	*/
1945	static KDPC s_aPokeDpcs[RTCPUSET_MAX_CPUS] = {{0}};
1946	static bool s_fPokeDPCsInitialized = false;
1947
1948	if (!s_fPokeDPCsInitialized)
1949	{
1950	for (unsigned i = 0; i < g_cRtMpNtMaxCpus; i++)
1951	{
1952	KeInitializeDpc(&s_aPokeDpcs[i], rtMpNtPokeCpuDummy, NULL);
1953	if (g_pfnrtKeSetImportanceDpc)
1954	g_pfnrtKeSetImportanceDpc(&s_aPokeDpcs[i], HighImportance);
1955	int rc = rtMpNtSetTargetProcessorDpc(&s_aPokeDpcs[i], idCpu);
1956	if (RT_FAILURE(rc) && rc != VERR_CPU_NOT_FOUND)
1957	return rc;
1958	}
1959
1960	s_fPokeDPCsInitialized = true;
1961	}
1962
1963	/* Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu.
1964	KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL. */
1965	KIRQL oldIrql;
1966	KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
1967
1968	if (g_pfnrtKeSetImportanceDpc)
1969	g_pfnrtKeSetImportanceDpc(&s_aPokeDpcs[idCpu], HighImportance);
1970	g_pfnrtKeSetTargetProcessorDpc(&s_aPokeDpcs[idCpu], (int)idCpu);
1971
1972	/* Assuming here that high importance DPCs will be delivered immediately; or at least an IPI will be sent immediately.
1973	Note! Not true on at least Vista & Windows 7 */
1974	BOOLEAN fRet = KeInsertQueueDpc(&s_aPokeDpcs[idCpu], 0, 0);
1975
1976	KeLowerIrql(oldIrql);
1977	return fRet == TRUE ? VINF_SUCCESS : VERR_ACCESS_DENIED /* already queued */;
1978	}
1979
1980
1981	RTDECL(int) RTMpPokeCpu(RTCPUID idCpu)
1982	{
1983	if (!RTMpIsCpuOnline(idCpu))
1984	return !RTMpIsCpuPossible(idCpu)
1985	? VERR_CPU_NOT_FOUND
1986	: VERR_CPU_OFFLINE;
1987	/* Calls rtMpPokeCpuUsingDpc, rtMpPokeCpuUsingHalRequestIpiW7Plus or rtMpPokeCpuUsingBroadcastIpi. */
1988	return g_pfnrtMpPokeCpuWorker(idCpu);
1989	}
1990
1991
1992	RTDECL(bool) RTMpOnAllIsConcurrentSafe(void)
1993	{
1994	return false;
1995	}
1996

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/nt/mp-r0drv-nt.cpp@ 96407

Download in other formats: