VirtualBox

source: vbox/trunk/src/VBox/Runtime/r0drv/nt/mp-r0drv-nt.cpp@ 94155

Last change on this file since 94155 was 93301, checked in by vboxsync, 3 years ago

IPRT: VC++ 19.2 update 11 build adjustments. bugref:8489
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 76.9 KB
Line 
1/* $Id: mp-r0drv-nt.cpp 93301 2022-01-18 11:24:43Z vboxsync $ */
2/** @file
3 * IPRT - Multiprocessor, Ring-0 Driver, NT.
4 */
5
6/*
7 * Copyright (C) 2008-2022 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * The contents of this file may alternatively be used under the terms
18 * of the Common Development and Distribution License Version 1.0
19 * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20 * VirtualBox OSE distribution, in which case the provisions of the
21 * CDDL are applicable instead of those of the GPL.
22 *
23 * You may elect to license modified versions of this file under the
24 * terms and conditions of either the GPL or the CDDL or both.
25 */
26
27
28/*********************************************************************************************************************************
29* Header Files *
30*********************************************************************************************************************************/
31#include "the-nt-kernel.h"
32
33#include <iprt/mp.h>
34#include <iprt/cpuset.h>
35#include <iprt/err.h>
36#include <iprt/asm.h>
37#include <iprt/log.h>
38#include <iprt/mem.h>
39#include <iprt/time.h>
40#include "r0drv/mp-r0drv.h"
41#include "symdb.h"
42#include "internal-r0drv-nt.h"
43#include "internal/mp.h"
44
45
46/*********************************************************************************************************************************
47* Structures and Typedefs *
48*********************************************************************************************************************************/
49typedef enum
50{
51 RT_NT_CPUID_SPECIFIC,
52 RT_NT_CPUID_PAIR,
53 RT_NT_CPUID_OTHERS,
54 RT_NT_CPUID_ALL
55} RT_NT_CPUID;
56
57
58/**
59 * Used by the RTMpOnSpecific.
60 */
61typedef struct RTMPNTONSPECIFICARGS
62{
63 /** Set if we're executing. */
64 bool volatile fExecuting;
65 /** Set when done executing. */
66 bool volatile fDone;
67 /** Number of references to this heap block. */
68 uint32_t volatile cRefs;
69 /** Event that the calling thread is waiting on. */
70 KEVENT DoneEvt;
71 /** The deferred procedure call object. */
72 KDPC Dpc;
73 /** The callback argument package. */
74 RTMPARGS CallbackArgs;
75} RTMPNTONSPECIFICARGS;
76/** Pointer to an argument/state structure for RTMpOnSpecific on NT. */
77typedef RTMPNTONSPECIFICARGS *PRTMPNTONSPECIFICARGS;
78
79
80/*********************************************************************************************************************************
81* Defined Constants And Macros *
82*********************************************************************************************************************************/
83/** Inactive bit for g_aidRtMpNtByCpuSetIdx. */
84#define RTMPNT_ID_F_INACTIVE RT_BIT_32(31)
85
86
87/*********************************************************************************************************************************
88* Global Variables *
89*********************************************************************************************************************************/
90/** Maximum number of processor groups. */
91uint32_t g_cRtMpNtMaxGroups;
92/** Maximum number of processors. */
93uint32_t g_cRtMpNtMaxCpus;
94/** Number of active processors. */
95uint32_t volatile g_cRtMpNtActiveCpus;
96/** The NT CPU set.
97 * KeQueryActiveProcssors() cannot be called at all IRQLs and therefore we'll
98 * have to cache it. Fortunately, NT doesn't really support taking CPUs offline,
99 * and taking them online was introduced with W2K8 where it is intended for virtual
100 * machines and not real HW. We update this, g_cRtMpNtActiveCpus and
101 * g_aidRtMpNtByCpuSetIdx from the rtR0NtMpProcessorChangeCallback.
102 */
103RTCPUSET g_rtMpNtCpuSet;
104
105/** Static per group info.
106 * @remarks With 256 groups this takes up 33KB. */
107static struct
108{
109 /** The max CPUs in the group. */
110 uint16_t cMaxCpus;
111 /** The number of active CPUs at the time of initialization. */
112 uint16_t cActiveCpus;
113 /** CPU set indexes for each CPU in the group. */
114 int16_t aidxCpuSetMembers[64];
115} g_aRtMpNtCpuGroups[256];
116/** Maps CPU set indexes to RTCPUID.
117 * Inactive CPUs has bit 31 set (RTMPNT_ID_F_INACTIVE) so we can identify them
118 * and shuffle duplicates during CPU hotplugging. We assign temporary IDs to
119 * the inactive CPUs starting at g_cRtMpNtMaxCpus - 1, ASSUMING that active
120 * CPUs has IDs from 0 to g_cRtMpNtActiveCpus. */
121RTCPUID g_aidRtMpNtByCpuSetIdx[RTCPUSET_MAX_CPUS];
122/** The handle of the rtR0NtMpProcessorChangeCallback registration. */
123static PVOID g_pvMpCpuChangeCallback = NULL;
124/** Size of the KAFFINITY_EX structure.
125 * This increased from 20 to 32 bitmap words in the 2020 H2 windows 10 release
126 * (i.e. 1280 to 2048 CPUs). We expect this to increase in the future. */
127static size_t g_cbRtMpNtKaffinityEx = RT_UOFFSETOF(KAFFINITY_EX, Bitmap)
128 + RT_SIZEOFMEMB(KAFFINITY_EX, Bitmap[0]) * 256;
129/** The size value of the KAFFINITY_EX structure. */
130static uint16_t g_cRtMpNtKaffinityExEntries = 256;
131
132
133/*********************************************************************************************************************************
134* Internal Functions *
135*********************************************************************************************************************************/
136static VOID __stdcall rtR0NtMpProcessorChangeCallback(void *pvUser, PKE_PROCESSOR_CHANGE_NOTIFY_CONTEXT pChangeCtx,
137 PNTSTATUS prcOperationStatus);
138static int rtR0NtInitQueryGroupRelations(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX **ppInfo);
139
140
141
142/**
143 * Initalizes multiprocessor globals (called by rtR0InitNative).
144 *
145 * @returns IPRT status code.
146 * @param pOsVerInfo Version information.
147 */
148DECLHIDDEN(int) rtR0MpNtInit(RTNTSDBOSVER const *pOsVerInfo)
149{
150#define MY_CHECK_BREAK(a_Check, a_DbgPrintArgs) \
151 AssertMsgBreakStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs; rc = VERR_INTERNAL_ERROR_4 )
152#define MY_CHECK_RETURN(a_Check, a_DbgPrintArgs, a_rcRet) \
153 AssertMsgReturnStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs, a_rcRet)
154#define MY_CHECK(a_Check, a_DbgPrintArgs) \
155 AssertMsgStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs; rc = VERR_INTERNAL_ERROR_4 )
156
157 /*
158 * API combination checks.
159 */
160 MY_CHECK_RETURN(!g_pfnrtKeSetTargetProcessorDpcEx || g_pfnrtKeGetProcessorNumberFromIndex,
161 ("IPRT: Fatal: Missing KeSetTargetProcessorDpcEx without KeGetProcessorNumberFromIndex!\n"),
162 VERR_SYMBOL_NOT_FOUND);
163
164 /*
165 * Get max number of processor groups.
166 *
167 * We may need to upadjust this number below, because windows likes to keep
168 * all options open when it comes to hotplugged CPU group assignments. A
169 * server advertising up to 64 CPUs in the ACPI table will get a result of
170 * 64 from KeQueryMaximumGroupCount. That makes sense. However, when windows
171 * server 2012 does a two processor group setup for it, the sum of the
172 * GroupInfo[*].MaximumProcessorCount members below is 128. This is probably
173 * because windows doesn't want to make decisions grouping of hotpluggable CPUs.
174 * So, we need to bump the maximum count to 128 below do deal with this as we
175 * want to have valid CPU set indexes for all potential CPUs - how could we
176 * otherwise use the RTMpGetSet() result and also RTCpuSetCount(RTMpGetSet())
177 * should equal RTMpGetCount().
178 */
179 if (g_pfnrtKeQueryMaximumGroupCount)
180 {
181 g_cRtMpNtMaxGroups = g_pfnrtKeQueryMaximumGroupCount();
182 MY_CHECK_RETURN(g_cRtMpNtMaxGroups <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxGroups > 0,
183 ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u\n", g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
184 VERR_MP_TOO_MANY_CPUS);
185 }
186 else
187 g_cRtMpNtMaxGroups = 1;
188
189 /*
190 * Get max number CPUs.
191 * This also defines the range of NT CPU indexes, RTCPUID and index into RTCPUSET.
192 */
193 if (g_pfnrtKeQueryMaximumProcessorCountEx)
194 {
195 g_cRtMpNtMaxCpus = g_pfnrtKeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS);
196 MY_CHECK_RETURN(g_cRtMpNtMaxCpus <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxCpus > 0,
197 ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, max %u [KeQueryMaximumProcessorCountEx]\n",
198 g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
199 VERR_MP_TOO_MANY_CPUS);
200 }
201 else if (g_pfnrtKeQueryMaximumProcessorCount)
202 {
203 g_cRtMpNtMaxCpus = g_pfnrtKeQueryMaximumProcessorCount();
204 MY_CHECK_RETURN(g_cRtMpNtMaxCpus <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxCpus > 0,
205 ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, max %u [KeQueryMaximumProcessorCount]\n",
206 g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
207 VERR_MP_TOO_MANY_CPUS);
208 }
209 else if (g_pfnrtKeQueryActiveProcessors)
210 {
211 KAFFINITY fActiveProcessors = g_pfnrtKeQueryActiveProcessors();
212 MY_CHECK_RETURN(fActiveProcessors != 0,
213 ("IPRT: Fatal: KeQueryActiveProcessors returned 0!\n"),
214 VERR_INTERNAL_ERROR_2);
215 g_cRtMpNtMaxCpus = 0;
216 do
217 {
218 g_cRtMpNtMaxCpus++;
219 fActiveProcessors >>= 1;
220 } while (fActiveProcessors);
221 }
222 else
223 g_cRtMpNtMaxCpus = KeNumberProcessors;
224
225 /*
226 * Just because we're a bit paranoid about getting something wrong wrt to the
227 * kernel interfaces, we try 16 times to get the KeQueryActiveProcessorCountEx
228 * and KeQueryLogicalProcessorRelationship information to match up.
229 */
230 for (unsigned cTries = 0;; cTries++)
231 {
232 /*
233 * Get number of active CPUs.
234 */
235 if (g_pfnrtKeQueryActiveProcessorCountEx)
236 {
237 g_cRtMpNtActiveCpus = g_pfnrtKeQueryActiveProcessorCountEx(ALL_PROCESSOR_GROUPS);
238 MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus && g_cRtMpNtActiveCpus > 0,
239 ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u [KeQueryActiveProcessorCountEx]\n",
240 g_cRtMpNtMaxGroups, g_cRtMpNtMaxCpus),
241 VERR_MP_TOO_MANY_CPUS);
242 }
243 else if (g_pfnrtKeQueryActiveProcessorCount)
244 {
245 g_cRtMpNtActiveCpus = g_pfnrtKeQueryActiveProcessorCount(NULL);
246 MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus && g_cRtMpNtActiveCpus > 0,
247 ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u [KeQueryActiveProcessorCount]\n",
248 g_cRtMpNtMaxGroups, g_cRtMpNtMaxCpus),
249 VERR_MP_TOO_MANY_CPUS);
250 }
251 else
252 g_cRtMpNtActiveCpus = g_cRtMpNtMaxCpus;
253
254 /*
255 * Query the details for the groups to figure out which CPUs are online as
256 * well as the NT index limit.
257 */
258 for (unsigned i = 0; i < RT_ELEMENTS(g_aidRtMpNtByCpuSetIdx); i++)
259#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
260 g_aidRtMpNtByCpuSetIdx[i] = NIL_RTCPUID;
261#else
262 g_aidRtMpNtByCpuSetIdx[i] = i < g_cRtMpNtMaxCpus ? i : NIL_RTCPUID;
263#endif
264 for (unsigned idxGroup = 0; idxGroup < RT_ELEMENTS(g_aRtMpNtCpuGroups); idxGroup++)
265 {
266 g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = 0;
267 g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = 0;
268 for (unsigned idxMember = 0; idxMember < RT_ELEMENTS(g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers); idxMember++)
269 g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = -1;
270 }
271
272 if (g_pfnrtKeQueryLogicalProcessorRelationship)
273 {
274 MY_CHECK_RETURN(g_pfnrtKeGetProcessorIndexFromNumber,
275 ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeGetProcessorIndexFromNumber!\n"),
276 VERR_SYMBOL_NOT_FOUND);
277 MY_CHECK_RETURN(g_pfnrtKeGetProcessorNumberFromIndex,
278 ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeGetProcessorIndexFromNumber!\n"),
279 VERR_SYMBOL_NOT_FOUND);
280 MY_CHECK_RETURN(g_pfnrtKeSetTargetProcessorDpcEx,
281 ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeSetTargetProcessorDpcEx!\n"),
282 VERR_SYMBOL_NOT_FOUND);
283
284 SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *pInfo = NULL;
285 int rc = rtR0NtInitQueryGroupRelations(&pInfo);
286 if (RT_FAILURE(rc))
287 return rc;
288
289 MY_CHECK(pInfo->Group.MaximumGroupCount == g_cRtMpNtMaxGroups,
290 ("IPRT: Fatal: MaximumGroupCount=%u != g_cRtMpNtMaxGroups=%u!\n",
291 pInfo->Group.MaximumGroupCount, g_cRtMpNtMaxGroups));
292 MY_CHECK(pInfo->Group.ActiveGroupCount > 0 && pInfo->Group.ActiveGroupCount <= g_cRtMpNtMaxGroups,
293 ("IPRT: Fatal: ActiveGroupCount=%u != g_cRtMpNtMaxGroups=%u!\n",
294 pInfo->Group.ActiveGroupCount, g_cRtMpNtMaxGroups));
295
296 /*
297 * First we need to recalc g_cRtMpNtMaxCpus (see above).
298 */
299 uint32_t cMaxCpus = 0;
300 uint32_t idxGroup;
301 for (idxGroup = 0; RT_SUCCESS(rc) && idxGroup < pInfo->Group.ActiveGroupCount; idxGroup++)
302 {
303 const PROCESSOR_GROUP_INFO *pGrpInfo = &pInfo->Group.GroupInfo[idxGroup];
304 MY_CHECK_BREAK(pGrpInfo->MaximumProcessorCount <= MAXIMUM_PROC_PER_GROUP,
305 ("IPRT: Fatal: MaximumProcessorCount=%u\n", pGrpInfo->MaximumProcessorCount));
306 MY_CHECK_BREAK(pGrpInfo->ActiveProcessorCount <= pGrpInfo->MaximumProcessorCount,
307 ("IPRT: Fatal: ActiveProcessorCount=%u > MaximumProcessorCount=%u\n",
308 pGrpInfo->ActiveProcessorCount, pGrpInfo->MaximumProcessorCount));
309 cMaxCpus += pGrpInfo->MaximumProcessorCount;
310 }
311 if (cMaxCpus > g_cRtMpNtMaxCpus && RT_SUCCESS(rc))
312 {
313 DbgPrint("IPRT: g_cRtMpNtMaxCpus=%u -> %u\n", g_cRtMpNtMaxCpus, cMaxCpus);
314#ifndef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
315 uint32_t i = RT_MIN(cMaxCpus, RT_ELEMENTS(g_aidRtMpNtByCpuSetIdx));
316 while (i-- > g_cRtMpNtMaxCpus)
317 g_aidRtMpNtByCpuSetIdx[i] = i;
318#endif
319 g_cRtMpNtMaxCpus = cMaxCpus;
320 if (g_cRtMpNtMaxGroups > RTCPUSET_MAX_CPUS)
321 {
322 MY_CHECK(g_cRtMpNtMaxGroups <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxGroups > 0,
323 ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u\n", g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS));
324 rc = VERR_MP_TOO_MANY_CPUS;
325 }
326 }
327
328 /*
329 * Calc online mask, partition IDs and such.
330 *
331 * Also check ASSUMPTIONS:
332 *
333 * 1. Processor indexes going from 0 and up to
334 * KeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS) - 1.
335 *
336 * 2. Currently valid processor indexes, i.e. accepted by
337 * KeGetProcessorIndexFromNumber & KeGetProcessorNumberFromIndex, goes
338 * from 0 thru KeQueryActiveProcessorCountEx(ALL_PROCESSOR_GROUPS) - 1.
339 *
340 * 3. PROCESSOR_GROUP_INFO::MaximumProcessorCount gives the number of
341 * relevant bits in the ActiveProcessorMask (from LSB).
342 *
343 * 4. Active processor count found in KeQueryLogicalProcessorRelationship
344 * output matches what KeQueryActiveProcessorCountEx(ALL) returns.
345 *
346 * 5. Active + inactive processor counts in same does not exceed
347 * KeQueryMaximumProcessorCountEx(ALL).
348 *
349 * Note! Processor indexes are assigned as CPUs come online and are not
350 * preallocated according to group maximums. Since CPUS are only taken
351 * online and never offlined, this means that internal CPU bitmaps are
352 * never sparse and no time is wasted scanning unused bits.
353 *
354 * Unfortunately, it means that ring-3 cannot easily guess the index
355 * assignments when hotswapping is used, and must use GIP when available.
356 */
357 RTCpuSetEmpty(&g_rtMpNtCpuSet);
358 uint32_t cInactive = 0;
359 uint32_t cActive = 0;
360 uint32_t idxCpuMax = 0;
361 uint32_t idxCpuSetNextInactive = g_cRtMpNtMaxCpus - 1;
362 for (idxGroup = 0; RT_SUCCESS(rc) && idxGroup < pInfo->Group.ActiveGroupCount; idxGroup++)
363 {
364 const PROCESSOR_GROUP_INFO *pGrpInfo = &pInfo->Group.GroupInfo[idxGroup];
365 MY_CHECK_BREAK(pGrpInfo->MaximumProcessorCount <= MAXIMUM_PROC_PER_GROUP,
366 ("IPRT: Fatal: MaximumProcessorCount=%u\n", pGrpInfo->MaximumProcessorCount));
367 MY_CHECK_BREAK(pGrpInfo->ActiveProcessorCount <= pGrpInfo->MaximumProcessorCount,
368 ("IPRT: Fatal: ActiveProcessorCount=%u > MaximumProcessorCount=%u\n",
369 pGrpInfo->ActiveProcessorCount, pGrpInfo->MaximumProcessorCount));
370
371 g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = pGrpInfo->MaximumProcessorCount;
372 g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = pGrpInfo->ActiveProcessorCount;
373
374 for (uint32_t idxMember = 0; idxMember < pGrpInfo->MaximumProcessorCount; idxMember++)
375 {
376 PROCESSOR_NUMBER ProcNum;
377 ProcNum.Group = (USHORT)idxGroup;
378 ProcNum.Number = (UCHAR)idxMember;
379 ProcNum.Reserved = 0;
380 ULONG idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);
381 if (idxCpu != INVALID_PROCESSOR_INDEX)
382 {
383 MY_CHECK_BREAK(idxCpu < g_cRtMpNtMaxCpus && idxCpu < RTCPUSET_MAX_CPUS, /* ASSUMPTION #1 */
384 ("IPRT: Fatal: idxCpu=%u >= g_cRtMpNtMaxCpus=%u (RTCPUSET_MAX_CPUS=%u)\n",
385 idxCpu, g_cRtMpNtMaxCpus, RTCPUSET_MAX_CPUS));
386 if (idxCpu > idxCpuMax)
387 idxCpuMax = idxCpu;
388 g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpu;
389#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
390 g_aidRtMpNtByCpuSetIdx[idxCpu] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember);
391#endif
392
393 ProcNum.Group = UINT16_MAX;
394 ProcNum.Number = UINT8_MAX;
395 ProcNum.Reserved = UINT8_MAX;
396 NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(idxCpu, &ProcNum);
397 MY_CHECK_BREAK(NT_SUCCESS(rcNt),
398 ("IPRT: Fatal: KeGetProcessorNumberFromIndex(%u,) -> %#x!\n", idxCpu, rcNt));
399 MY_CHECK_BREAK(ProcNum.Group == idxGroup && ProcNum.Number == idxMember,
400 ("IPRT: Fatal: KeGetProcessorXxxxFromYyyy roundtrip error for %#x! Group: %u vs %u, Number: %u vs %u\n",
401 idxCpu, ProcNum.Group, idxGroup, ProcNum.Number, idxMember));
402
403 if (pGrpInfo->ActiveProcessorMask & RT_BIT_64(idxMember))
404 {
405 RTCpuSetAddByIndex(&g_rtMpNtCpuSet, idxCpu);
406 cActive++;
407 }
408 else
409 cInactive++; /* (This is a little unexpected, but not important as long as things add up below.) */
410 }
411 else
412 {
413 /* Must be not present / inactive when KeGetProcessorIndexFromNumber fails. */
414 MY_CHECK_BREAK(!(pGrpInfo->ActiveProcessorMask & RT_BIT_64(idxMember)),
415 ("IPRT: Fatal: KeGetProcessorIndexFromNumber(%u/%u) failed but CPU is active! cMax=%u cActive=%u fActive=%p\n",
416 idxGroup, idxMember, pGrpInfo->MaximumProcessorCount, pGrpInfo->ActiveProcessorCount,
417 pGrpInfo->ActiveProcessorMask));
418 cInactive++;
419 if (idxCpuSetNextInactive >= g_cRtMpNtActiveCpus)
420 {
421 g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpuSetNextInactive;
422#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
423 g_aidRtMpNtByCpuSetIdx[idxCpuSetNextInactive] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember)
424 | RTMPNT_ID_F_INACTIVE;
425#endif
426 idxCpuSetNextInactive--;
427 }
428 }
429 }
430 }
431
432 MY_CHECK(cInactive + cActive <= g_cRtMpNtMaxCpus, /* ASSUMPTION #5 (not '==' because of inactive groups) */
433 ("IPRT: Fatal: cInactive=%u + cActive=%u > g_cRtMpNtMaxCpus=%u\n", cInactive, cActive, g_cRtMpNtMaxCpus));
434
435 /* Deal with inactive groups using KeQueryMaximumProcessorCountEx or as
436 best as we can by as best we can by stipulating maximum member counts
437 from the previous group. */
438 if ( RT_SUCCESS(rc)
439 && idxGroup < pInfo->Group.MaximumGroupCount)
440 {
441 uint16_t cInactiveLeft = g_cRtMpNtMaxCpus - (cInactive + cActive);
442 while (idxGroup < pInfo->Group.MaximumGroupCount)
443 {
444 uint32_t cMaxMembers = 0;
445 if (g_pfnrtKeQueryMaximumProcessorCountEx)
446 cMaxMembers = g_pfnrtKeQueryMaximumProcessorCountEx(idxGroup);
447 if (cMaxMembers != 0 || cInactiveLeft == 0)
448 AssertStmt(cMaxMembers <= cInactiveLeft, cMaxMembers = cInactiveLeft);
449 else
450 {
451 uint16_t cGroupsLeft = pInfo->Group.MaximumGroupCount - idxGroup;
452 cMaxMembers = pInfo->Group.GroupInfo[idxGroup - 1].MaximumProcessorCount;
453 while (cMaxMembers * cGroupsLeft < cInactiveLeft)
454 cMaxMembers++;
455 if (cMaxMembers > cInactiveLeft)
456 cMaxMembers = cInactiveLeft;
457 }
458
459 g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = (uint16_t)cMaxMembers;
460 g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = 0;
461 for (uint16_t idxMember = 0; idxMember < cMaxMembers; idxMember++)
462 if (idxCpuSetNextInactive >= g_cRtMpNtActiveCpus)
463 {
464 g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpuSetNextInactive;
465#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
466 g_aidRtMpNtByCpuSetIdx[idxCpuSetNextInactive] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember)
467 | RTMPNT_ID_F_INACTIVE;
468#endif
469 idxCpuSetNextInactive--;
470 }
471 cInactiveLeft -= cMaxMembers;
472 idxGroup++;
473 }
474 }
475
476 /* We're done with pInfo now, free it so we can start returning when assertions fail. */
477 RTMemFree(pInfo);
478 if (RT_FAILURE(rc)) /* MY_CHECK_BREAK sets rc. */
479 return rc;
480 MY_CHECK_RETURN(cActive >= g_cRtMpNtActiveCpus,
481 ("IPRT: Fatal: cActive=%u < g_cRtMpNtActiveCpus=%u - CPUs removed?\n", cActive, g_cRtMpNtActiveCpus),
482 VERR_INTERNAL_ERROR_3);
483 MY_CHECK_RETURN(idxCpuMax < cActive, /* ASSUMPTION #2 */
484 ("IPRT: Fatal: idCpuMax=%u >= cActive=%u! Unexpected CPU index allocation. CPUs removed?\n",
485 idxCpuMax, cActive),
486 VERR_INTERNAL_ERROR_4);
487
488 /* Retry if CPUs were added. */
489 if ( cActive != g_cRtMpNtActiveCpus
490 && cTries < 16)
491 continue;
492 MY_CHECK_RETURN(cActive == g_cRtMpNtActiveCpus, /* ASSUMPTION #4 */
493 ("IPRT: Fatal: cActive=%u != g_cRtMpNtActiveCpus=%u\n", cActive, g_cRtMpNtActiveCpus),
494 VERR_INTERNAL_ERROR_5);
495 }
496 else
497 {
498 /* Legacy: */
499 MY_CHECK_RETURN(g_cRtMpNtMaxGroups == 1, ("IPRT: Fatal: Missing KeQueryLogicalProcessorRelationship!\n"),
500 VERR_SYMBOL_NOT_FOUND);
501
502 /** @todo Is it possible that the affinity mask returned by
503 * KeQueryActiveProcessors is sparse? */
504 if (g_pfnrtKeQueryActiveProcessors)
505 RTCpuSetFromU64(&g_rtMpNtCpuSet, g_pfnrtKeQueryActiveProcessors());
506 else if (g_cRtMpNtMaxCpus < 64)
507 RTCpuSetFromU64(&g_rtMpNtCpuSet, (UINT64_C(1) << g_cRtMpNtMaxCpus) - 1);
508 else
509 {
510 MY_CHECK_RETURN(g_cRtMpNtMaxCpus == 64, ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, expect 64 or less\n", g_cRtMpNtMaxCpus),
511 VERR_MP_TOO_MANY_CPUS);
512 RTCpuSetFromU64(&g_rtMpNtCpuSet, UINT64_MAX);
513 }
514
515 g_aRtMpNtCpuGroups[0].cMaxCpus = g_cRtMpNtMaxCpus;
516 g_aRtMpNtCpuGroups[0].cActiveCpus = g_cRtMpNtMaxCpus;
517 for (unsigned i = 0; i < g_cRtMpNtMaxCpus; i++)
518 {
519 g_aRtMpNtCpuGroups[0].aidxCpuSetMembers[i] = i;
520#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
521 g_aidRtMpNtByCpuSetIdx[i] = RTMPCPUID_FROM_GROUP_AND_NUMBER(0, i);
522#endif
523 }
524 }
525
526 /*
527 * Register CPU hot plugging callback (it also counts active CPUs).
528 */
529 Assert(g_pvMpCpuChangeCallback == NULL);
530 if (g_pfnrtKeRegisterProcessorChangeCallback)
531 {
532 MY_CHECK_RETURN(g_pfnrtKeDeregisterProcessorChangeCallback,
533 ("IPRT: Fatal: KeRegisterProcessorChangeCallback without KeDeregisterProcessorChangeCallback!\n"),
534 VERR_SYMBOL_NOT_FOUND);
535
536 RTCPUSET const ActiveSetCopy = g_rtMpNtCpuSet;
537 RTCpuSetEmpty(&g_rtMpNtCpuSet);
538 uint32_t const cActiveCpus = g_cRtMpNtActiveCpus;
539 g_cRtMpNtActiveCpus = 0;
540
541 g_pvMpCpuChangeCallback = g_pfnrtKeRegisterProcessorChangeCallback(rtR0NtMpProcessorChangeCallback, NULL /*pvUser*/,
542 KE_PROCESSOR_CHANGE_ADD_EXISTING);
543 if (g_pvMpCpuChangeCallback)
544 {
545 if (cActiveCpus == g_cRtMpNtActiveCpus)
546 { /* likely */ }
547 else
548 {
549 g_pfnrtKeDeregisterProcessorChangeCallback(g_pvMpCpuChangeCallback);
550 if (cTries < 16)
551 {
552 /* Retry if CPUs were added. */
553 MY_CHECK_RETURN(g_cRtMpNtActiveCpus >= cActiveCpus,
554 ("IPRT: Fatal: g_cRtMpNtActiveCpus=%u < cActiveCpus=%u! CPUs removed?\n",
555 g_cRtMpNtActiveCpus, cActiveCpus),
556 VERR_INTERNAL_ERROR_2);
557 MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus,
558 ("IPRT: Fatal: g_cRtMpNtActiveCpus=%u > g_cRtMpNtMaxCpus=%u!\n",
559 g_cRtMpNtActiveCpus, g_cRtMpNtMaxCpus),
560 VERR_INTERNAL_ERROR_2);
561 continue;
562 }
563 MY_CHECK_RETURN(0, ("IPRT: Fatal: g_cRtMpNtActiveCpus=%u cActiveCpus=%u\n", g_cRtMpNtActiveCpus, cActiveCpus),
564 VERR_INTERNAL_ERROR_3);
565 }
566 }
567 else
568 {
569 AssertFailed();
570 g_rtMpNtCpuSet = ActiveSetCopy;
571 g_cRtMpNtActiveCpus = cActiveCpus;
572 }
573 }
574 break;
575 } /* Retry loop for stable active CPU count. */
576
577#undef MY_CHECK_RETURN
578
579 /*
580 * Special IPI fun for RTMpPokeCpu.
581 *
582 * On Vista and later the DPC method doesn't seem to reliably send IPIs,
583 * so we have to use alternative methods.
584 *
585 * On AMD64 We used to use the HalSendSoftwareInterrupt API (also x86 on
586 * W10+), it looks faster and more convenient to use, however we're either
587 * using it wrong or it doesn't reliably do what we want (see @bugref{8343}).
588 *
589 * The HalRequestIpip API is thus far the only alternative to KeInsertQueueDpc
590 * for doing targetted IPIs. Trouble with this API is that it changed
591 * fundamentally in Window 7 when they added support for lots of processors.
592 *
593 * If we really think we cannot use KeInsertQueueDpc, we use the broadcast IPI
594 * API KeIpiGenericCall.
595 */
596 if ( pOsVerInfo->uMajorVer > 6
597 || (pOsVerInfo->uMajorVer == 6 && pOsVerInfo->uMinorVer > 0))
598 g_pfnrtHalRequestIpiPreW7 = NULL;
599 else
600 g_pfnrtHalRequestIpiW7Plus = NULL;
601
602 if ( g_pfnrtHalRequestIpiW7Plus
603 && g_pfnrtKeInitializeAffinityEx
604 && g_pfnrtKeAddProcessorAffinityEx
605 && g_pfnrtKeGetProcessorIndexFromNumber)
606 {
607 /* Determine the real size of the KAFFINITY_EX structure. */
608 size_t const cbAffinity = _8K;
609 PKAFFINITY_EX pAffinity = (PKAFFINITY_EX)RTMemAllocZ(cbAffinity);
610 AssertReturn(pAffinity, VERR_NO_MEMORY);
611 size_t const cMaxEntries = (cbAffinity - RT_UOFFSETOF(KAFFINITY_EX, Bitmap[0])) / sizeof(pAffinity->Bitmap[0]);
612 g_pfnrtKeInitializeAffinityEx(pAffinity);
613 if (pAffinity->Size > 1 && pAffinity->Size <= cMaxEntries)
614 {
615 g_cRtMpNtKaffinityExEntries = pAffinity->Size;
616 g_cbRtMpNtKaffinityEx = pAffinity->Size * sizeof(pAffinity->Bitmap[0]) + RT_UOFFSETOF(KAFFINITY_EX, Bitmap[0]);
617 g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingHalRequestIpiW7Plus;
618 RTMemFree(pAffinity);
619 DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingHalRequestIpiW7Plus\n");
620 return VINF_SUCCESS;
621 }
622 DbgPrint("IPRT: RTMpPoke can't use rtMpPokeCpuUsingHalRequestIpiW7Plus! pAffinity->Size=%u\n", pAffinity->Size);
623 AssertReleaseMsg(pAffinity->Size <= cMaxEntries, ("%#x\n", pAffinity->Size)); /* stack is toast if larger (32768 CPUs). */
624 RTMemFree(pAffinity);
625 }
626
627 if (pOsVerInfo->uMajorVer >= 6 && g_pfnrtKeIpiGenericCall)
628 {
629 DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingBroadcastIpi\n");
630 g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingBroadcastIpi;
631 }
632 else if (g_pfnrtKeSetTargetProcessorDpc)
633 {
634 DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingDpc\n");
635 g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingDpc;
636 /* Windows XP should send always send an IPI -> VERIFY */
637 }
638 else
639 {
640 DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingFailureNotSupported\n");
641 Assert(pOsVerInfo->uMajorVer == 3 && pOsVerInfo->uMinorVer <= 50);
642 g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingFailureNotSupported;
643 }
644
645 return VINF_SUCCESS;
646}
647
648
649/**
650 * Called by rtR0TermNative.
651 */
652DECLHIDDEN(void) rtR0MpNtTerm(void)
653{
654 /*
655 * Deregister the processor change callback.
656 */
657 PVOID pvMpCpuChangeCallback = g_pvMpCpuChangeCallback;
658 g_pvMpCpuChangeCallback = NULL;
659 if (pvMpCpuChangeCallback)
660 {
661 AssertReturnVoid(g_pfnrtKeDeregisterProcessorChangeCallback);
662 g_pfnrtKeDeregisterProcessorChangeCallback(pvMpCpuChangeCallback);
663 }
664}
665
666
667DECLHIDDEN(int) rtR0MpNotificationNativeInit(void)
668{
669 return VINF_SUCCESS;
670}
671
672
673DECLHIDDEN(void) rtR0MpNotificationNativeTerm(void)
674{
675}
676
677
678/**
679 * Implements the NT PROCESSOR_CALLBACK_FUNCTION callback function.
680 *
681 * This maintains the g_rtMpNtCpuSet and works MP notification callbacks. When
682 * registered, it's called for each active CPU in the system, avoiding racing
683 * CPU hotplugging (as well as testing the callback).
684 *
685 * @param pvUser User context (not used).
686 * @param pChangeCtx Change context (in).
687 * @param prcOperationStatus Operation status (in/out).
688 *
689 * @remarks ASSUMES no concurrent execution of KeProcessorAddCompleteNotify
690 * notification callbacks. At least during callback registration
691 * callout, we're owning KiDynamicProcessorLock.
692 *
693 * @remarks When registering the handler, we first get KeProcessorAddStartNotify
694 * callbacks for all active CPUs, and after they all succeed we get the
695 * KeProcessorAddCompleteNotify callbacks.
696 */
697static VOID __stdcall rtR0NtMpProcessorChangeCallback(void *pvUser, PKE_PROCESSOR_CHANGE_NOTIFY_CONTEXT pChangeCtx,
698 PNTSTATUS prcOperationStatus)
699{
700 RT_NOREF(pvUser, prcOperationStatus);
701 switch (pChangeCtx->State)
702 {
703 /*
704 * Check whether we can deal with the CPU, failing the start operation if we
705 * can't. The checks we are doing here are to avoid complicated/impossible
706 * cases in KeProcessorAddCompleteNotify. They are really just verify specs.
707 */
708 case KeProcessorAddStartNotify:
709 {
710 NTSTATUS rcNt = STATUS_SUCCESS;
711 if (pChangeCtx->NtNumber < RTCPUSET_MAX_CPUS)
712 {
713 if (pChangeCtx->NtNumber >= g_cRtMpNtMaxCpus)
714 {
715 DbgPrint("IPRT: KeProcessorAddStartNotify failure: NtNumber=%u is higher than the max CPU count (%u)!\n",
716 pChangeCtx->NtNumber, g_cRtMpNtMaxCpus);
717 rcNt = STATUS_INTERNAL_ERROR;
718 }
719
720 /* The ProcessNumber field was introduced in Windows 7. */
721 PROCESSOR_NUMBER ProcNum;
722 if (g_pfnrtKeGetProcessorIndexFromNumber)
723 {
724 ProcNum = pChangeCtx->ProcNumber;
725 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);
726 if (idxCpu != pChangeCtx->NtNumber)
727 {
728 DbgPrint("IPRT: KeProcessorAddStartNotify failure: g_pfnrtKeGetProcessorIndexFromNumber(%u.%u) -> %u, expected %u!\n",
729 ProcNum.Group, ProcNum.Number, idxCpu, pChangeCtx->NtNumber);
730 rcNt = STATUS_INTERNAL_ERROR;
731 }
732 }
733 else
734 {
735 ProcNum.Group = 0;
736 ProcNum.Number = pChangeCtx->NtNumber;
737 }
738
739 if ( ProcNum.Group < RT_ELEMENTS(g_aRtMpNtCpuGroups)
740 && ProcNum.Number < RT_ELEMENTS(g_aRtMpNtCpuGroups[0].aidxCpuSetMembers))
741 {
742 if (ProcNum.Group >= g_cRtMpNtMaxGroups)
743 {
744 DbgPrint("IPRT: KeProcessorAddStartNotify failure: %u.%u is out of range - max groups: %u!\n",
745 ProcNum.Group, ProcNum.Number, g_cRtMpNtMaxGroups);
746 rcNt = STATUS_INTERNAL_ERROR;
747 }
748
749 if (ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus)
750 {
751 Assert(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] != -1);
752 if (g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] == -1)
753 {
754 DbgPrint("IPRT: KeProcessorAddStartNotify failure: Internal error! %u.%u was assigned -1 as set index!\n",
755 ProcNum.Group, ProcNum.Number);
756 rcNt = STATUS_INTERNAL_ERROR;
757 }
758
759 Assert(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] != NIL_RTCPUID);
760 if (g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] == NIL_RTCPUID)
761 {
762 DbgPrint("IPRT: KeProcessorAddStartNotify failure: Internal error! %u (%u.%u) translates to NIL_RTCPUID!\n",
763 pChangeCtx->NtNumber, ProcNum.Group, ProcNum.Number);
764 rcNt = STATUS_INTERNAL_ERROR;
765 }
766 }
767 else
768 {
769 DbgPrint("IPRT: KeProcessorAddStartNotify failure: max processors in group %u is %u, cannot add %u to it!\n",
770 ProcNum.Group, g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus, ProcNum.Group, ProcNum.Number);
771 rcNt = STATUS_INTERNAL_ERROR;
772 }
773 }
774 else
775 {
776 DbgPrint("IPRT: KeProcessorAddStartNotify failure: %u.%u is out of range (max %u.%u)!\n",
777 ProcNum.Group, ProcNum.Number, RT_ELEMENTS(g_aRtMpNtCpuGroups), RT_ELEMENTS(g_aRtMpNtCpuGroups[0].aidxCpuSetMembers));
778 rcNt = STATUS_INTERNAL_ERROR;
779 }
780 }
781 else
782 {
783 DbgPrint("IPRT: KeProcessorAddStartNotify failure: NtNumber=%u is outside RTCPUSET_MAX_CPUS (%u)!\n",
784 pChangeCtx->NtNumber, RTCPUSET_MAX_CPUS);
785 rcNt = STATUS_INTERNAL_ERROR;
786 }
787 if (!NT_SUCCESS(rcNt))
788 *prcOperationStatus = rcNt;
789 break;
790 }
791
792 /*
793 * Update the globals. Since we've checked out range limits and other
794 * limitations already we just AssertBreak here.
795 */
796 case KeProcessorAddCompleteNotify:
797 {
798 /*
799 * Calc the processor number and assert conditions checked in KeProcessorAddStartNotify.
800 */
801 AssertBreak(pChangeCtx->NtNumber < RTCPUSET_MAX_CPUS);
802 AssertBreak(pChangeCtx->NtNumber < g_cRtMpNtMaxCpus);
803 Assert(pChangeCtx->NtNumber == g_cRtMpNtActiveCpus); /* light assumption */
804 PROCESSOR_NUMBER ProcNum;
805 if (g_pfnrtKeGetProcessorIndexFromNumber)
806 {
807 ProcNum = pChangeCtx->ProcNumber;
808 AssertBreak(g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum) == pChangeCtx->NtNumber);
809 AssertBreak(ProcNum.Group < RT_ELEMENTS(g_aRtMpNtCpuGroups));
810 AssertBreak(ProcNum.Group < g_cRtMpNtMaxGroups);
811 }
812 else
813 {
814 ProcNum.Group = 0;
815 ProcNum.Number = pChangeCtx->NtNumber;
816 }
817 AssertBreak(ProcNum.Number < RT_ELEMENTS(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers));
818 AssertBreak(ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus);
819 AssertBreak(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] != -1);
820 AssertBreak(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] != NIL_RTCPUID);
821
822 /*
823 * Add ourselves to the online CPU set and update the active CPU count.
824 */
825 RTCpuSetAddByIndex(&g_rtMpNtCpuSet, pChangeCtx->NtNumber);
826 ASMAtomicIncU32(&g_cRtMpNtActiveCpus);
827
828 /*
829 * Update the group info.
830 *
831 * If the index prediction failed (real hotplugging callbacks only) we
832 * have to switch it around. This is particularly annoying when we
833 * use the index as the ID.
834 */
835#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
836 RTCPUID idCpu = RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
837 RTCPUID idOld = g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber];
838 if ((idOld & ~RTMPNT_ID_F_INACTIVE) != idCpu)
839 {
840 Assert(idOld & RTMPNT_ID_F_INACTIVE);
841 int idxDest = g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number];
842 g_aRtMpNtCpuGroups[rtMpCpuIdGetGroup(idOld)].aidxCpuSetMembers[rtMpCpuIdGetGroupMember(idOld)] = idxDest;
843 g_aidRtMpNtByCpuSetIdx[idxDest] = idOld;
844 }
845 g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] = idCpu;
846#else
847 Assert(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] == pChangeCtx->NtNumber);
848 int idxDest = g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number];
849 if ((ULONG)idxDest != pChangeCtx->NtNumber)
850 {
851 bool fFound = false;
852 uint32_t idxOldGroup = g_cRtMpNtMaxGroups;
853 while (idxOldGroup-- > 0 && !fFound)
854 {
855 uint32_t idxMember = g_aRtMpNtCpuGroups[idxOldGroup].cMaxCpus;
856 while (idxMember-- > 0)
857 if (g_aRtMpNtCpuGroups[idxOldGroup].aidxCpuSetMembers[idxMember] == (int)pChangeCtx->NtNumber)
858 {
859 g_aRtMpNtCpuGroups[idxOldGroup].aidxCpuSetMembers[idxMember] = idxDest;
860 fFound = true;
861 break;
862 }
863 }
864 Assert(fFound);
865 }
866#endif
867 g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] = pChangeCtx->NtNumber;
868
869 /*
870 * Do MP notification callbacks.
871 */
872 rtMpNotificationDoCallbacks(RTMPEVENT_ONLINE, pChangeCtx->NtNumber);
873 break;
874 }
875
876 case KeProcessorAddFailureNotify:
877 /* ignore */
878 break;
879
880 default:
881 AssertMsgFailed(("State=%u\n", pChangeCtx->State));
882 }
883}
884
885
886/**
887 * Wrapper around KeQueryLogicalProcessorRelationship.
888 *
889 * @returns IPRT status code.
890 * @param ppInfo Where to return the info. Pass to RTMemFree when done.
891 */
892static int rtR0NtInitQueryGroupRelations(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX **ppInfo)
893{
894 ULONG cbInfo = sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)
895 + g_cRtMpNtMaxGroups * sizeof(GROUP_RELATIONSHIP);
896 NTSTATUS rcNt;
897 do
898 {
899 SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *pInfo = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)RTMemAlloc(cbInfo);
900 if (pInfo)
901 {
902 rcNt = g_pfnrtKeQueryLogicalProcessorRelationship(NULL /*pProcNumber*/, RelationGroup, pInfo, &cbInfo);
903 if (NT_SUCCESS(rcNt))
904 {
905 *ppInfo = pInfo;
906 return VINF_SUCCESS;
907 }
908
909 RTMemFree(pInfo);
910 pInfo = NULL;
911 }
912 else
913 rcNt = STATUS_NO_MEMORY;
914 } while (rcNt == STATUS_INFO_LENGTH_MISMATCH);
915 DbgPrint("IPRT: Fatal: KeQueryLogicalProcessorRelationship failed: %#x\n", rcNt);
916 AssertMsgFailed(("KeQueryLogicalProcessorRelationship failed: %#x\n", rcNt));
917 return RTErrConvertFromNtStatus(rcNt);
918}
919
920
921
922
923
924RTDECL(RTCPUID) RTMpCpuId(void)
925{
926 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
927
928#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
929 PROCESSOR_NUMBER ProcNum;
930 ProcNum.Group = 0;
931 if (g_pfnrtKeGetCurrentProcessorNumberEx)
932 {
933 ProcNum.Number = 0;
934 g_pfnrtKeGetCurrentProcessorNumberEx(&ProcNum);
935 }
936 else
937 ProcNum.Number = KeGetCurrentProcessorNumber(); /* Number is 8-bit, so we're not subject to BYTE -> WORD upgrade in WDK. */
938 return RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
939
940#else
941
942 if (g_pfnrtKeGetCurrentProcessorNumberEx)
943 {
944 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(NULL);
945 Assert(idxCpu < RTCPUSET_MAX_CPUS);
946 return idxCpu;
947 }
948
949 return (uint8_t)KeGetCurrentProcessorNumber(); /* PCR->Number was changed from BYTE to WORD in the WDK, thus the cast. */
950#endif
951}
952
953
954RTDECL(int) RTMpCurSetIndex(void)
955{
956#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
957 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
958
959 if (g_pfnrtKeGetCurrentProcessorNumberEx)
960 {
961 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(NULL);
962 Assert(idxCpu < RTCPUSET_MAX_CPUS);
963 return idxCpu;
964 }
965 return (uint8_t)KeGetCurrentProcessorNumber(); /* PCR->Number was changed from BYTE to WORD in the WDK, thus the cast. */
966#else
967 return (int)RTMpCpuId();
968#endif
969}
970
971
972RTDECL(int) RTMpCurSetIndexAndId(PRTCPUID pidCpu)
973{
974#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
975 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
976
977 PROCESSOR_NUMBER ProcNum = { 0 , 0, 0 };
978 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(&ProcNum);
979 Assert(idxCpu < RTCPUSET_MAX_CPUS);
980 *pidCpu = RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
981 return idxCpu;
982#else
983 return *pidCpu = RTMpCpuId();
984#endif
985}
986
987
988RTDECL(int) RTMpCpuIdToSetIndex(RTCPUID idCpu)
989{
990#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
991 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
992
993 if (idCpu != NIL_RTCPUID)
994 {
995 if (g_pfnrtKeGetProcessorIndexFromNumber)
996 {
997 PROCESSOR_NUMBER ProcNum;
998 ProcNum.Group = rtMpCpuIdGetGroup(idCpu);
999 ProcNum.Number = rtMpCpuIdGetGroupMember(idCpu);
1000 ProcNum.Reserved = 0;
1001 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);
1002 if (idxCpu != INVALID_PROCESSOR_INDEX)
1003 {
1004 Assert(idxCpu < g_cRtMpNtMaxCpus);
1005 Assert((ULONG)g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] == idxCpu);
1006 return idxCpu;
1007 }
1008
1009 /* Since NT assigned indexes as the CPUs come online, we cannot produce an ID <-> index
1010 mapping for not-yet-onlined CPUS that is consistent. We just have to do our best... */
1011 if ( ProcNum.Group < g_cRtMpNtMaxGroups
1012 && ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus)
1013 return g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number];
1014 }
1015 else if (rtMpCpuIdGetGroup(idCpu) == 0)
1016 return rtMpCpuIdGetGroupMember(idCpu);
1017 }
1018 return -1;
1019#else
1020 /* 1:1 mapping, just do range checks. */
1021 return idCpu < RTCPUSET_MAX_CPUS ? (int)idCpu : -1;
1022#endif
1023}
1024
1025
1026RTDECL(RTCPUID) RTMpCpuIdFromSetIndex(int iCpu)
1027{
1028#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
1029 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1030
1031 if ((unsigned)iCpu < g_cRtMpNtMaxCpus)
1032 {
1033 if (g_pfnrtKeGetProcessorIndexFromNumber)
1034 {
1035 PROCESSOR_NUMBER ProcNum = { 0, 0, 0 };
1036 NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(iCpu, &ProcNum);
1037 if (NT_SUCCESS(rcNt))
1038 {
1039 Assert(ProcNum.Group <= g_cRtMpNtMaxGroups);
1040 Assert( (g_aidRtMpNtByCpuSetIdx[iCpu] & ~RTMPNT_ID_F_INACTIVE)
1041 == RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number));
1042 return RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
1043 }
1044 }
1045 return g_aidRtMpNtByCpuSetIdx[iCpu];
1046 }
1047 return NIL_RTCPUID;
1048#else
1049 /* 1:1 mapping, just do range checks. */
1050 return (unsigned)iCpu < RTCPUSET_MAX_CPUS ? iCpu : NIL_RTCPUID;
1051#endif
1052}
1053
1054
1055RTDECL(int) RTMpSetIndexFromCpuGroupMember(uint32_t idxGroup, uint32_t idxMember)
1056{
1057 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1058
1059 if (idxGroup < g_cRtMpNtMaxGroups)
1060 if (idxMember < g_aRtMpNtCpuGroups[idxGroup].cMaxCpus)
1061 return g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember];
1062 return -1;
1063}
1064
1065
1066RTDECL(uint32_t) RTMpGetCpuGroupCounts(uint32_t idxGroup, uint32_t *pcActive)
1067{
1068 if (idxGroup < g_cRtMpNtMaxGroups)
1069 {
1070 if (pcActive)
1071 *pcActive = g_aRtMpNtCpuGroups[idxGroup].cActiveCpus;
1072 return g_aRtMpNtCpuGroups[idxGroup].cMaxCpus;
1073 }
1074 if (pcActive)
1075 *pcActive = 0;
1076 return 0;
1077}
1078
1079
1080RTDECL(uint32_t) RTMpGetMaxCpuGroupCount(void)
1081{
1082 return g_cRtMpNtMaxGroups;
1083}
1084
1085
1086RTDECL(RTCPUID) RTMpGetMaxCpuId(void)
1087{
1088 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1089
1090#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
1091 return RTMPCPUID_FROM_GROUP_AND_NUMBER(g_cRtMpNtMaxGroups - 1, g_aRtMpNtCpuGroups[g_cRtMpNtMaxGroups - 1].cMaxCpus - 1);
1092#else
1093 /* According to MSDN the processor indexes goes from 0 to the maximum
1094 number of CPUs in the system. We've check this in initterm-r0drv-nt.cpp. */
1095 return g_cRtMpNtMaxCpus - 1;
1096#endif
1097}
1098
1099
1100RTDECL(bool) RTMpIsCpuOnline(RTCPUID idCpu)
1101{
1102 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1103 return RTCpuSetIsMember(&g_rtMpNtCpuSet, idCpu);
1104}
1105
1106
1107RTDECL(bool) RTMpIsCpuPossible(RTCPUID idCpu)
1108{
1109 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1110
1111#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
1112 if (idCpu != NIL_RTCPUID)
1113 {
1114 unsigned idxGroup = rtMpCpuIdGetGroup(idCpu);
1115 if (idxGroup < g_cRtMpNtMaxGroups)
1116 return rtMpCpuIdGetGroupMember(idCpu) < g_aRtMpNtCpuGroups[idxGroup].cMaxCpus;
1117 }
1118 return false;
1119
1120#else
1121 /* A possible CPU ID is one with a value lower than g_cRtMpNtMaxCpus (see
1122 comment in RTMpGetMaxCpuId). */
1123 return idCpu < g_cRtMpNtMaxCpus;
1124#endif
1125}
1126
1127
1128
1129RTDECL(PRTCPUSET) RTMpGetSet(PRTCPUSET pSet)
1130{
1131 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1132
1133 /* The set of possible CPU IDs(/indexes) are from 0 up to
1134 g_cRtMpNtMaxCpus (see comment in RTMpGetMaxCpuId). */
1135 RTCpuSetEmpty(pSet);
1136 int idxCpu = g_cRtMpNtMaxCpus;
1137 while (idxCpu-- > 0)
1138 RTCpuSetAddByIndex(pSet, idxCpu);
1139 return pSet;
1140}
1141
1142
1143RTDECL(RTCPUID) RTMpGetCount(void)
1144{
1145 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1146 return g_cRtMpNtMaxCpus;
1147}
1148
1149
1150RTDECL(PRTCPUSET) RTMpGetOnlineSet(PRTCPUSET pSet)
1151{
1152 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1153
1154 *pSet = g_rtMpNtCpuSet;
1155 return pSet;
1156}
1157
1158
1159RTDECL(RTCPUID) RTMpGetOnlineCount(void)
1160{
1161 RTCPUSET Set;
1162 RTMpGetOnlineSet(&Set);
1163 return RTCpuSetCount(&Set);
1164}
1165
1166
1167RTDECL(RTCPUID) RTMpGetOnlineCoreCount(void)
1168{
1169 /** @todo fix me */
1170 return RTMpGetOnlineCount();
1171}
1172
1173
1174
1175#if 0
1176/* Experiment with checking the undocumented KPRCB structure
1177 * 'dt nt!_kprcb 0xaddress' shows the layout
1178 */
1179typedef struct
1180{
1181 LIST_ENTRY DpcListHead;
1182 ULONG_PTR DpcLock;
1183 volatile ULONG DpcQueueDepth;
1184 ULONG DpcQueueCount;
1185} KDPC_DATA, *PKDPC_DATA;
1186
1187RTDECL(bool) RTMpIsCpuWorkPending(void)
1188{
1189 uint8_t *pkprcb;
1190 PKDPC_DATA pDpcData;
1191
1192 _asm {
1193 mov eax, fs:0x20
1194 mov pkprcb, eax
1195 }
1196 pDpcData = (PKDPC_DATA)(pkprcb + 0x19e0);
1197 if (pDpcData->DpcQueueDepth)
1198 return true;
1199
1200 pDpcData++;
1201 if (pDpcData->DpcQueueDepth)
1202 return true;
1203 return false;
1204}
1205#else
1206RTDECL(bool) RTMpIsCpuWorkPending(void)
1207{
1208 /** @todo not implemented */
1209 return false;
1210}
1211#endif
1212
1213
1214/**
1215 * Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1216 * the RTMpOnAll case.
1217 *
1218 * @param uUserCtx The user context argument (PRTMPARGS).
1219 */
1220static ULONG_PTR rtmpNtOnAllBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1221{
1222 PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1223 /*ASMAtomicIncU32(&pArgs->cHits); - not needed */
1224 pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2);
1225 return 0;
1226}
1227
1228
1229/**
1230 * Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1231 * the RTMpOnOthers case.
1232 *
1233 * @param uUserCtx The user context argument (PRTMPARGS).
1234 */
1235static ULONG_PTR rtmpNtOnOthersBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1236{
1237 PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1238 RTCPUID idCpu = RTMpCpuId();
1239 if (pArgs->idCpu != idCpu)
1240 {
1241 /*ASMAtomicIncU32(&pArgs->cHits); - not needed */
1242 pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
1243 }
1244 return 0;
1245}
1246
1247
1248/**
1249 * Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1250 * the RTMpOnPair case.
1251 *
1252 * @param uUserCtx The user context argument (PRTMPARGS).
1253 */
1254static ULONG_PTR rtmpNtOnPairBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1255{
1256 PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1257 RTCPUID idCpu = RTMpCpuId();
1258 if ( pArgs->idCpu == idCpu
1259 || pArgs->idCpu2 == idCpu)
1260 {
1261 ASMAtomicIncU32(&pArgs->cHits);
1262 pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
1263 }
1264 return 0;
1265}
1266
1267
1268/**
1269 * Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1270 * the RTMpOnSpecific case.
1271 *
1272 * @param uUserCtx The user context argument (PRTMPARGS).
1273 */
1274static ULONG_PTR rtmpNtOnSpecificBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1275{
1276 PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1277 RTCPUID idCpu = RTMpCpuId();
1278 if (pArgs->idCpu == idCpu)
1279 {
1280 ASMAtomicIncU32(&pArgs->cHits);
1281 pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
1282 }
1283 return 0;
1284}
1285
1286
1287/**
1288 * Internal worker for the RTMpOn* APIs using KeIpiGenericCall.
1289 *
1290 * @returns VINF_SUCCESS.
1291 * @param pfnWorker The callback.
1292 * @param pvUser1 User argument 1.
1293 * @param pvUser2 User argument 2.
1294 * @param pfnNativeWrapper The wrapper between the NT and IPRT callbacks.
1295 * @param idCpu First CPU to match, ultimately specific to the
1296 * pfnNativeWrapper used.
1297 * @param idCpu2 Second CPU to match, ultimately specific to the
1298 * pfnNativeWrapper used.
1299 * @param pcHits Where to return the number of this. Optional.
1300 */
1301static int rtMpCallUsingBroadcastIpi(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2,
1302 PKIPI_BROADCAST_WORKER pfnNativeWrapper, RTCPUID idCpu, RTCPUID idCpu2,
1303 uint32_t *pcHits)
1304{
1305 RTMPARGS Args;
1306 Args.pfnWorker = pfnWorker;
1307 Args.pvUser1 = pvUser1;
1308 Args.pvUser2 = pvUser2;
1309 Args.idCpu = idCpu;
1310 Args.idCpu2 = idCpu2;
1311 Args.cRefs = 0;
1312 Args.cHits = 0;
1313
1314 AssertPtr(g_pfnrtKeIpiGenericCall);
1315 g_pfnrtKeIpiGenericCall(pfnNativeWrapper, (uintptr_t)&Args);
1316 if (pcHits)
1317 *pcHits = Args.cHits;
1318 return VINF_SUCCESS;
1319}
1320
1321
1322/**
1323 * Wrapper between the native nt per-cpu callbacks and PFNRTWORKER
1324 *
1325 * @param Dpc DPC object
1326 * @param DeferredContext Context argument specified by KeInitializeDpc
1327 * @param SystemArgument1 Argument specified by KeInsertQueueDpc
1328 * @param SystemArgument2 Argument specified by KeInsertQueueDpc
1329 */
1330static VOID rtmpNtDPCWrapper(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2)
1331{
1332 PRTMPARGS pArgs = (PRTMPARGS)DeferredContext;
1333 RT_NOREF3(Dpc, SystemArgument1, SystemArgument2);
1334
1335 ASMAtomicIncU32(&pArgs->cHits);
1336 pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2);
1337
1338 /* Dereference the argument structure. */
1339 int32_t cRefs = ASMAtomicDecS32(&pArgs->cRefs);
1340 Assert(cRefs >= 0);
1341 if (cRefs == 0)
1342 RTMemFree(pArgs);
1343}
1344
1345
1346/**
1347 * Wrapper around KeSetTargetProcessorDpcEx / KeSetTargetProcessorDpc.
1348 *
1349 * This is shared with the timer code.
1350 *
1351 * @returns IPRT status code (errors are asserted).
1352 * @retval VERR_CPU_NOT_FOUND if impossible CPU. Not asserted.
1353 * @param pDpc The DPC.
1354 * @param idCpu The ID of the new target CPU.
1355 * @note Callable at any IRQL.
1356 */
1357DECLHIDDEN(int) rtMpNtSetTargetProcessorDpc(KDPC *pDpc, RTCPUID idCpu)
1358{
1359 if (g_pfnrtKeSetTargetProcessorDpcEx)
1360 {
1361 /* Convert to stupid process number (bet KeSetTargetProcessorDpcEx does
1362 the reverse conversion internally). */
1363 PROCESSOR_NUMBER ProcNum;
1364 NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(RTMpCpuIdToSetIndex(idCpu), &ProcNum);
1365 if (NT_SUCCESS(rcNt))
1366 {
1367 rcNt = g_pfnrtKeSetTargetProcessorDpcEx(pDpc, &ProcNum);
1368 AssertLogRelMsgReturn(NT_SUCCESS(rcNt),
1369 ("KeSetTargetProcessorDpcEx(,%u(%u/%u)) -> %#x\n", idCpu, ProcNum.Group, ProcNum.Number, rcNt),
1370 RTErrConvertFromNtStatus(rcNt));
1371 }
1372 else if (rcNt == STATUS_INVALID_PARAMETER)
1373 return VERR_CPU_NOT_FOUND;
1374 else
1375 AssertLogRelMsgReturn(NT_SUCCESS(rcNt), ("KeGetProcessorNumberFromIndex(%u) -> %#x\n", idCpu, rcNt),
1376 RTErrConvertFromNtStatus(rcNt));
1377
1378 }
1379 else if (g_pfnrtKeSetTargetProcessorDpc)
1380 g_pfnrtKeSetTargetProcessorDpc(pDpc, RTMpCpuIdToSetIndex(idCpu));
1381 else
1382 return VERR_NOT_SUPPORTED;
1383 return VINF_SUCCESS;
1384}
1385
1386
1387/**
1388 * Internal worker for the RTMpOn* APIs.
1389 *
1390 * @returns IPRT status code.
1391 * @param pfnWorker The callback.
1392 * @param pvUser1 User argument 1.
1393 * @param pvUser2 User argument 2.
1394 * @param enmCpuid What to do / is idCpu valid.
1395 * @param idCpu Used if enmCpuid is RT_NT_CPUID_SPECIFIC or
1396 * RT_NT_CPUID_PAIR, otherwise ignored.
1397 * @param idCpu2 Used if enmCpuid is RT_NT_CPUID_PAIR, otherwise ignored.
1398 * @param pcHits Where to return the number of this. Optional.
1399 */
1400static int rtMpCallUsingDpcs(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2,
1401 RT_NT_CPUID enmCpuid, RTCPUID idCpu, RTCPUID idCpu2, uint32_t *pcHits)
1402{
1403#if 0
1404 /* KeFlushQueuedDpcs must be run at IRQL PASSIVE_LEVEL according to MSDN, but the
1405 * driver verifier doesn't complain...
1406 */
1407 AssertMsg(KeGetCurrentIrql() == PASSIVE_LEVEL, ("%d != %d (PASSIVE_LEVEL)\n", KeGetCurrentIrql(), PASSIVE_LEVEL));
1408#endif
1409 /* KeFlushQueuedDpcs is not present in Windows 2000; import it dynamically so we can just fail this call. */
1410 if (!g_pfnrtNtKeFlushQueuedDpcs)
1411 return VERR_NOT_SUPPORTED;
1412
1413 /*
1414 * Make a copy of the active CPU set and figure out how many KDPCs we really need.
1415 * We must not try setup DPCs for CPUs which aren't there, because that may fail.
1416 */
1417 RTCPUSET OnlineSet = g_rtMpNtCpuSet;
1418 uint32_t cDpcsNeeded;
1419 switch (enmCpuid)
1420 {
1421 case RT_NT_CPUID_SPECIFIC:
1422 cDpcsNeeded = 1;
1423 break;
1424 case RT_NT_CPUID_PAIR:
1425 cDpcsNeeded = 2;
1426 break;
1427 default:
1428 do
1429 {
1430 cDpcsNeeded = g_cRtMpNtActiveCpus;
1431 OnlineSet = g_rtMpNtCpuSet;
1432 } while (cDpcsNeeded != g_cRtMpNtActiveCpus);
1433 break;
1434 }
1435
1436 /*
1437 * Allocate an RTMPARGS structure followed by cDpcsNeeded KDPCs
1438 * and initialize them.
1439 */
1440 PRTMPARGS pArgs = (PRTMPARGS)RTMemAllocZ(sizeof(RTMPARGS) + cDpcsNeeded * sizeof(KDPC));
1441 if (!pArgs)
1442 return VERR_NO_MEMORY;
1443
1444 pArgs->pfnWorker = pfnWorker;
1445 pArgs->pvUser1 = pvUser1;
1446 pArgs->pvUser2 = pvUser2;
1447 pArgs->idCpu = NIL_RTCPUID;
1448 pArgs->idCpu2 = NIL_RTCPUID;
1449 pArgs->cHits = 0;
1450 pArgs->cRefs = 1;
1451
1452 int rc;
1453 KDPC *paExecCpuDpcs = (KDPC *)(pArgs + 1);
1454 if (enmCpuid == RT_NT_CPUID_SPECIFIC)
1455 {
1456 KeInitializeDpc(&paExecCpuDpcs[0], rtmpNtDPCWrapper, pArgs);
1457 if (g_pfnrtKeSetImportanceDpc)
1458 g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[0], HighImportance);
1459 rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[0], idCpu);
1460 pArgs->idCpu = idCpu;
1461 }
1462 else if (enmCpuid == RT_NT_CPUID_PAIR)
1463 {
1464 KeInitializeDpc(&paExecCpuDpcs[0], rtmpNtDPCWrapper, pArgs);
1465 if (g_pfnrtKeSetImportanceDpc)
1466 g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[0], HighImportance);
1467 rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[0], idCpu);
1468 pArgs->idCpu = idCpu;
1469
1470 KeInitializeDpc(&paExecCpuDpcs[1], rtmpNtDPCWrapper, pArgs);
1471 if (g_pfnrtKeSetImportanceDpc)
1472 g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[1], HighImportance);
1473 if (RT_SUCCESS(rc))
1474 rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[1], (int)idCpu2);
1475 pArgs->idCpu2 = idCpu2;
1476 }
1477 else
1478 {
1479 rc = VINF_SUCCESS;
1480 for (uint32_t i = 0; i < cDpcsNeeded && RT_SUCCESS(rc); i++)
1481 if (RTCpuSetIsMemberByIndex(&OnlineSet, i))
1482 {
1483 KeInitializeDpc(&paExecCpuDpcs[i], rtmpNtDPCWrapper, pArgs);
1484 if (g_pfnrtKeSetImportanceDpc)
1485 g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[i], HighImportance);
1486 rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[i], RTMpCpuIdFromSetIndex(i));
1487 }
1488 }
1489 if (RT_FAILURE(rc))
1490 {
1491 RTMemFree(pArgs);
1492 return rc;
1493 }
1494
1495 /*
1496 * Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu.
1497 * KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL.
1498 */
1499 KIRQL oldIrql;
1500 KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
1501
1502 /*
1503 * We cannot do other than assume a 1:1 relationship between the
1504 * affinity mask and the process despite the warnings in the docs.
1505 * If someone knows a better way to get this done, please let bird know.
1506 */
1507 ASMCompilerBarrier(); /* paranoia */
1508 if (enmCpuid == RT_NT_CPUID_SPECIFIC)
1509 {
1510 ASMAtomicIncS32(&pArgs->cRefs);
1511 BOOLEAN fRc = KeInsertQueueDpc(&paExecCpuDpcs[0], 0, 0);
1512 Assert(fRc); NOREF(fRc);
1513 }
1514 else if (enmCpuid == RT_NT_CPUID_PAIR)
1515 {
1516 ASMAtomicIncS32(&pArgs->cRefs);
1517 BOOLEAN fRc = KeInsertQueueDpc(&paExecCpuDpcs[0], 0, 0);
1518 Assert(fRc); NOREF(fRc);
1519
1520 ASMAtomicIncS32(&pArgs->cRefs);
1521 fRc = KeInsertQueueDpc(&paExecCpuDpcs[1], 0, 0);
1522 Assert(fRc); NOREF(fRc);
1523 }
1524 else
1525 {
1526 uint32_t iSelf = RTMpCurSetIndex();
1527 for (uint32_t i = 0; i < cDpcsNeeded; i++)
1528 {
1529 if ( (i != iSelf)
1530 && RTCpuSetIsMemberByIndex(&OnlineSet, i))
1531 {
1532 ASMAtomicIncS32(&pArgs->cRefs);
1533 BOOLEAN fRc = KeInsertQueueDpc(&paExecCpuDpcs[i], 0, 0);
1534 Assert(fRc); NOREF(fRc);
1535 }
1536 }
1537 if (enmCpuid != RT_NT_CPUID_OTHERS)
1538 pfnWorker(iSelf, pvUser1, pvUser2);
1539 }
1540
1541 KeLowerIrql(oldIrql);
1542
1543 /*
1544 * Flush all DPCs and wait for completion. (can take long!)
1545 */
1546 /** @todo Consider changing this to an active wait using some atomic inc/dec
1547 * stuff (and check for the current cpu above in the specific case). */
1548 /** @todo Seems KeFlushQueuedDpcs doesn't wait for the DPCs to be completely
1549 * executed. Seen pArgs being freed while some CPU was using it before
1550 * cRefs was added. */
1551 if (g_pfnrtNtKeFlushQueuedDpcs)
1552 g_pfnrtNtKeFlushQueuedDpcs();
1553
1554 if (pcHits)
1555 *pcHits = pArgs->cHits;
1556
1557 /* Dereference the argument structure. */
1558 int32_t cRefs = ASMAtomicDecS32(&pArgs->cRefs);
1559 Assert(cRefs >= 0);
1560 if (cRefs == 0)
1561 RTMemFree(pArgs);
1562
1563 return VINF_SUCCESS;
1564}
1565
1566
1567RTDECL(int) RTMpOnAll(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
1568{
1569 if (g_pfnrtKeIpiGenericCall)
1570 return rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnAllBroadcastIpiWrapper,
1571 NIL_RTCPUID, NIL_RTCPUID, NULL);
1572 return rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_ALL, NIL_RTCPUID, NIL_RTCPUID, NULL);
1573}
1574
1575
1576RTDECL(int) RTMpOnOthers(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
1577{
1578 if (g_pfnrtKeIpiGenericCall)
1579 return rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnOthersBroadcastIpiWrapper,
1580 NIL_RTCPUID, NIL_RTCPUID, NULL);
1581 return rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_OTHERS, NIL_RTCPUID, NIL_RTCPUID, NULL);
1582}
1583
1584
1585RTDECL(int) RTMpOnPair(RTCPUID idCpu1, RTCPUID idCpu2, uint32_t fFlags, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
1586{
1587 int rc;
1588 AssertReturn(idCpu1 != idCpu2, VERR_INVALID_PARAMETER);
1589 AssertReturn(!(fFlags & RTMPON_F_VALID_MASK), VERR_INVALID_FLAGS);
1590 if ((fFlags & RTMPON_F_CONCURRENT_EXEC) && !g_pfnrtKeIpiGenericCall)
1591 return VERR_NOT_SUPPORTED;
1592
1593 /*
1594 * Check that both CPUs are online before doing the broadcast call.
1595 */
1596 if ( RTMpIsCpuOnline(idCpu1)
1597 && RTMpIsCpuOnline(idCpu2))
1598 {
1599 /*
1600 * The broadcast IPI isn't quite as bad as it could have been, because
1601 * it looks like windows doesn't synchronize CPUs on the way out, they
1602 * seems to get back to normal work while the pair is still busy.
1603 */
1604 uint32_t cHits = 0;
1605 if (g_pfnrtKeIpiGenericCall)
1606 rc = rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnPairBroadcastIpiWrapper, idCpu1, idCpu2, &cHits);
1607 else
1608 rc = rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_PAIR, idCpu1, idCpu2, &cHits);
1609 if (RT_SUCCESS(rc))
1610 {
1611 Assert(cHits <= 2);
1612 if (cHits == 2)
1613 rc = VINF_SUCCESS;
1614 else if (cHits == 1)
1615 rc = VERR_NOT_ALL_CPUS_SHOWED;
1616 else if (cHits == 0)
1617 rc = VERR_CPU_OFFLINE;
1618 else
1619 rc = VERR_CPU_IPE_1;
1620 }
1621 }
1622 /*
1623 * A CPU must be present to be considered just offline.
1624 */
1625 else if ( RTMpIsCpuPresent(idCpu1)
1626 && RTMpIsCpuPresent(idCpu2))
1627 rc = VERR_CPU_OFFLINE;
1628 else
1629 rc = VERR_CPU_NOT_FOUND;
1630 return rc;
1631}
1632
1633
1634RTDECL(bool) RTMpOnPairIsConcurrentExecSupported(void)
1635{
1636 return g_pfnrtKeIpiGenericCall != NULL;
1637}
1638
1639
1640/**
1641 * Releases a reference to a RTMPNTONSPECIFICARGS heap allocation, freeing it
1642 * when the last reference is released.
1643 */
1644DECLINLINE(void) rtMpNtOnSpecificRelease(PRTMPNTONSPECIFICARGS pArgs)
1645{
1646 uint32_t cRefs = ASMAtomicDecU32(&pArgs->cRefs);
1647 AssertMsg(cRefs <= 1, ("cRefs=%#x\n", cRefs));
1648 if (cRefs == 0)
1649 RTMemFree(pArgs);
1650}
1651
1652
1653/**
1654 * Wrapper between the native nt per-cpu callbacks and PFNRTWORKER
1655 *
1656 * @param Dpc DPC object
1657 * @param DeferredContext Context argument specified by KeInitializeDpc
1658 * @param SystemArgument1 Argument specified by KeInsertQueueDpc
1659 * @param SystemArgument2 Argument specified by KeInsertQueueDpc
1660 */
1661static VOID rtMpNtOnSpecificDpcWrapper(IN PKDPC Dpc, IN PVOID DeferredContext,
1662 IN PVOID SystemArgument1, IN PVOID SystemArgument2)
1663{
1664 PRTMPNTONSPECIFICARGS pArgs = (PRTMPNTONSPECIFICARGS)DeferredContext;
1665 RT_NOREF3(Dpc, SystemArgument1, SystemArgument2);
1666
1667 ASMAtomicWriteBool(&pArgs->fExecuting, true);
1668
1669 pArgs->CallbackArgs.pfnWorker(RTMpCpuId(), pArgs->CallbackArgs.pvUser1, pArgs->CallbackArgs.pvUser2);
1670
1671 ASMAtomicWriteBool(&pArgs->fDone, true);
1672 KeSetEvent(&pArgs->DoneEvt, 1 /*PriorityIncrement*/, FALSE /*Wait*/);
1673
1674 rtMpNtOnSpecificRelease(pArgs);
1675}
1676
1677
1678RTDECL(int) RTMpOnSpecific(RTCPUID idCpu, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
1679{
1680 /*
1681 * Don't try mess with an offline CPU.
1682 */
1683 if (!RTMpIsCpuOnline(idCpu))
1684 return !RTMpIsCpuPossible(idCpu)
1685 ? VERR_CPU_NOT_FOUND
1686 : VERR_CPU_OFFLINE;
1687
1688 /*
1689 * Use the broadcast IPI routine if there are no more than two CPUs online,
1690 * or if the current IRQL is unsuitable for KeWaitForSingleObject.
1691 */
1692 int rc;
1693 uint32_t cHits = 0;
1694 if ( g_pfnrtKeIpiGenericCall
1695 && ( RTMpGetOnlineCount() <= 2
1696 || KeGetCurrentIrql() > APC_LEVEL)
1697 )
1698 {
1699 rc = rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnSpecificBroadcastIpiWrapper,
1700 idCpu, NIL_RTCPUID, &cHits);
1701 if (RT_SUCCESS(rc))
1702 {
1703 if (cHits == 1)
1704 return VINF_SUCCESS;
1705 rc = cHits == 0 ? VERR_CPU_OFFLINE : VERR_CPU_IPE_1;
1706 }
1707 return rc;
1708 }
1709
1710#if 0
1711 rc = rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_SPECIFIC, idCpu, NIL_RTCPUID, &cHits);
1712 if (RT_SUCCESS(rc))
1713 {
1714 if (cHits == 1)
1715 return VINF_SUCCESS;
1716 rc = cHits == 0 ? VERR_CPU_OFFLINE : VERR_CPU_IPE_1;
1717 }
1718 return rc;
1719
1720#else
1721 /*
1722 * Initialize the argument package and the objects within it.
1723 * The package is referenced counted to avoid unnecessary spinning to
1724 * synchronize cleanup and prevent stack corruption.
1725 */
1726 PRTMPNTONSPECIFICARGS pArgs = (PRTMPNTONSPECIFICARGS)RTMemAllocZ(sizeof(*pArgs));
1727 if (!pArgs)
1728 return VERR_NO_MEMORY;
1729 pArgs->cRefs = 2;
1730 pArgs->fExecuting = false;
1731 pArgs->fDone = false;
1732 pArgs->CallbackArgs.pfnWorker = pfnWorker;
1733 pArgs->CallbackArgs.pvUser1 = pvUser1;
1734 pArgs->CallbackArgs.pvUser2 = pvUser2;
1735 pArgs->CallbackArgs.idCpu = idCpu;
1736 pArgs->CallbackArgs.cHits = 0;
1737 pArgs->CallbackArgs.cRefs = 2;
1738 KeInitializeEvent(&pArgs->DoneEvt, SynchronizationEvent, FALSE /* not signalled */);
1739 KeInitializeDpc(&pArgs->Dpc, rtMpNtOnSpecificDpcWrapper, pArgs);
1740 if (g_pfnrtKeSetImportanceDpc)
1741 g_pfnrtKeSetImportanceDpc(&pArgs->Dpc, HighImportance);
1742 rc = rtMpNtSetTargetProcessorDpc(&pArgs->Dpc, idCpu);
1743 if (RT_FAILURE(rc))
1744 {
1745 RTMemFree(pArgs);
1746 return rc;
1747 }
1748
1749 /*
1750 * Disable preemption while we check the current processor and inserts the DPC.
1751 */
1752 KIRQL bOldIrql;
1753 KeRaiseIrql(DISPATCH_LEVEL, &bOldIrql);
1754 ASMCompilerBarrier(); /* paranoia */
1755
1756 if (RTMpCpuId() == idCpu)
1757 {
1758 /* Just execute the callback on the current CPU. */
1759 pfnWorker(idCpu, pvUser1, pvUser2);
1760 KeLowerIrql(bOldIrql);
1761
1762 RTMemFree(pArgs);
1763 return VINF_SUCCESS;
1764 }
1765
1766 /* Different CPU, so queue it if the CPU is still online. */
1767 if (RTMpIsCpuOnline(idCpu))
1768 {
1769 BOOLEAN fRc = KeInsertQueueDpc(&pArgs->Dpc, 0, 0);
1770 Assert(fRc); NOREF(fRc);
1771 KeLowerIrql(bOldIrql);
1772
1773 uint64_t const nsRealWaitTS = RTTimeNanoTS();
1774
1775 /*
1776 * Wait actively for a while in case the CPU/thread responds quickly.
1777 */
1778 uint32_t cLoopsLeft = 0x20000;
1779 while (cLoopsLeft-- > 0)
1780 {
1781 if (pArgs->fDone)
1782 {
1783 rtMpNtOnSpecificRelease(pArgs);
1784 return VINF_SUCCESS;
1785 }
1786 ASMNopPause();
1787 }
1788
1789 /*
1790 * It didn't respond, so wait on the event object, poking the CPU if it's slow.
1791 */
1792 LARGE_INTEGER Timeout;
1793 Timeout.QuadPart = -10000; /* 1ms */
1794 NTSTATUS rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
1795 if (rcNt == STATUS_SUCCESS)
1796 {
1797 rtMpNtOnSpecificRelease(pArgs);
1798 return VINF_SUCCESS;
1799 }
1800
1801 /* If it hasn't respondend yet, maybe poke it and wait some more. */
1802 if (rcNt == STATUS_TIMEOUT)
1803 {
1804 if ( !pArgs->fExecuting
1805 && ( g_pfnrtMpPokeCpuWorker == rtMpPokeCpuUsingHalRequestIpiW7Plus
1806 || g_pfnrtMpPokeCpuWorker == rtMpPokeCpuUsingHalRequestIpiPreW7))
1807 RTMpPokeCpu(idCpu);
1808
1809 Timeout.QuadPart = -1280000; /* 128ms */
1810 rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
1811 if (rcNt == STATUS_SUCCESS)
1812 {
1813 rtMpNtOnSpecificRelease(pArgs);
1814 return VINF_SUCCESS;
1815 }
1816 }
1817
1818 /*
1819 * Something weird is happening, try bail out.
1820 */
1821 if (KeRemoveQueueDpc(&pArgs->Dpc))
1822 {
1823 RTMemFree(pArgs); /* DPC was still queued, so we can return without further ado. */
1824 LogRel(("RTMpOnSpecific(%#x): Not processed after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
1825 }
1826 else
1827 {
1828 /* DPC is running, wait a good while for it to complete. */
1829 LogRel(("RTMpOnSpecific(%#x): Still running after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
1830
1831 Timeout.QuadPart = -30*1000*1000*10; /* 30 seconds */
1832 rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
1833 if (rcNt != STATUS_SUCCESS)
1834 LogRel(("RTMpOnSpecific(%#x): Giving up on running worker after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
1835 }
1836 rc = RTErrConvertFromNtStatus(rcNt);
1837 }
1838 else
1839 {
1840 /* CPU is offline.*/
1841 KeLowerIrql(bOldIrql);
1842 rc = !RTMpIsCpuPossible(idCpu) ? VERR_CPU_NOT_FOUND : VERR_CPU_OFFLINE;
1843 }
1844
1845 rtMpNtOnSpecificRelease(pArgs);
1846 return rc;
1847#endif
1848}
1849
1850
1851
1852
1853static VOID rtMpNtPokeCpuDummy(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2)
1854{
1855 NOREF(Dpc);
1856 NOREF(DeferredContext);
1857 NOREF(SystemArgument1);
1858 NOREF(SystemArgument2);
1859}
1860
1861
1862/** Callback used by rtMpPokeCpuUsingBroadcastIpi. */
1863static ULONG_PTR rtMpIpiGenericCall(ULONG_PTR Argument)
1864{
1865 NOREF(Argument);
1866 return 0;
1867}
1868
1869
1870/**
1871 * RTMpPokeCpu worker that uses broadcast IPIs for doing the work.
1872 *
1873 * @returns VINF_SUCCESS
1874 * @param idCpu The CPU identifier.
1875 */
1876int rtMpPokeCpuUsingBroadcastIpi(RTCPUID idCpu)
1877{
1878 NOREF(idCpu);
1879 g_pfnrtKeIpiGenericCall(rtMpIpiGenericCall, 0);
1880 return VINF_SUCCESS;
1881}
1882
1883
1884/**
1885 * RTMpPokeCpu worker that uses the Windows 7 and later version of
1886 * HalRequestIpip to get the job done.
1887 *
1888 * @returns VINF_SUCCESS
1889 * @param idCpu The CPU identifier.
1890 */
1891int rtMpPokeCpuUsingHalRequestIpiW7Plus(RTCPUID idCpu)
1892{
1893 /* idCpu is an HAL processor index, so we can use it directly. */
1894 PKAFFINITY_EX pTarget = (PKAFFINITY_EX)alloca(g_cbRtMpNtKaffinityEx);
1895 pTarget->Size = g_cRtMpNtKaffinityExEntries; /* (just in case KeInitializeAffinityEx starts using it) */
1896 g_pfnrtKeInitializeAffinityEx(pTarget);
1897 g_pfnrtKeAddProcessorAffinityEx(pTarget, idCpu);
1898
1899 g_pfnrtHalRequestIpiW7Plus(0, pTarget);
1900 return VINF_SUCCESS;
1901}
1902
1903
1904/**
1905 * RTMpPokeCpu worker that uses the Vista and earlier version of HalRequestIpip
1906 * to get the job done.
1907 *
1908 * @returns VINF_SUCCESS
1909 * @param idCpu The CPU identifier.
1910 */
1911int rtMpPokeCpuUsingHalRequestIpiPreW7(RTCPUID idCpu)
1912{
1913 __debugbreak(); /** @todo this code needs testing!! */
1914 KAFFINITY Target = 1;
1915 Target <<= idCpu;
1916 g_pfnrtHalRequestIpiPreW7(Target);
1917 return VINF_SUCCESS;
1918}
1919
1920
1921int rtMpPokeCpuUsingFailureNotSupported(RTCPUID idCpu)
1922{
1923 NOREF(idCpu);
1924 return VERR_NOT_SUPPORTED;
1925}
1926
1927
1928int rtMpPokeCpuUsingDpc(RTCPUID idCpu)
1929{
1930 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1931
1932 /*
1933 * APC fallback.
1934 */
1935 static KDPC s_aPokeDpcs[RTCPUSET_MAX_CPUS] = {{0}};
1936 static bool s_fPokeDPCsInitialized = false;
1937
1938 if (!s_fPokeDPCsInitialized)
1939 {
1940 for (unsigned i = 0; i < g_cRtMpNtMaxCpus; i++)
1941 {
1942 KeInitializeDpc(&s_aPokeDpcs[i], rtMpNtPokeCpuDummy, NULL);
1943 if (g_pfnrtKeSetImportanceDpc)
1944 g_pfnrtKeSetImportanceDpc(&s_aPokeDpcs[i], HighImportance);
1945 int rc = rtMpNtSetTargetProcessorDpc(&s_aPokeDpcs[i], idCpu);
1946 if (RT_FAILURE(rc) && rc != VERR_CPU_NOT_FOUND)
1947 return rc;
1948 }
1949
1950 s_fPokeDPCsInitialized = true;
1951 }
1952
1953 /* Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu.
1954 KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL. */
1955 KIRQL oldIrql;
1956 KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
1957
1958 if (g_pfnrtKeSetImportanceDpc)
1959 g_pfnrtKeSetImportanceDpc(&s_aPokeDpcs[idCpu], HighImportance);
1960 g_pfnrtKeSetTargetProcessorDpc(&s_aPokeDpcs[idCpu], (int)idCpu);
1961
1962 /* Assuming here that high importance DPCs will be delivered immediately; or at least an IPI will be sent immediately.
1963 Note! Not true on at least Vista & Windows 7 */
1964 BOOLEAN fRet = KeInsertQueueDpc(&s_aPokeDpcs[idCpu], 0, 0);
1965
1966 KeLowerIrql(oldIrql);
1967 return fRet == TRUE ? VINF_SUCCESS : VERR_ACCESS_DENIED /* already queued */;
1968}
1969
1970
1971RTDECL(int) RTMpPokeCpu(RTCPUID idCpu)
1972{
1973 if (!RTMpIsCpuOnline(idCpu))
1974 return !RTMpIsCpuPossible(idCpu)
1975 ? VERR_CPU_NOT_FOUND
1976 : VERR_CPU_OFFLINE;
1977 /* Calls rtMpPokeCpuUsingDpc, rtMpPokeCpuUsingHalRequestIpiW7Plus or rtMpPokeCpuUsingBroadcastIpi. */
1978 return g_pfnrtMpPokeCpuWorker(idCpu);
1979}
1980
1981
1982RTDECL(bool) RTMpOnAllIsConcurrentSafe(void)
1983{
1984 return false;
1985}
1986
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette