VirtualBox

source: vbox/trunk/src/VBox/Runtime/r0drv/nt/mp-r0drv-nt.cpp

Last change on this file was 106563, checked in by vboxsync, 5 weeks ago

IPRT/mp-r0drv-nt.cpp: Missing iprt/alloca.h include. jiraref:VBP-1171
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 77.2 KB
Line 
1/* $Id: mp-r0drv-nt.cpp 106563 2024-10-21 12:34:40Z vboxsync $ */
2/** @file
3 * IPRT - Multiprocessor, Ring-0 Driver, NT.
4 */
5
6/*
7 * Copyright (C) 2008-2024 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * The contents of this file may alternatively be used under the terms
26 * of the Common Development and Distribution License Version 1.0
27 * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
28 * in the VirtualBox distribution, in which case the provisions of the
29 * CDDL are applicable instead of those of the GPL.
30 *
31 * You may elect to license modified versions of this file under the
32 * terms and conditions of either the GPL or the CDDL or both.
33 *
34 * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
35 */
36
37
38/*********************************************************************************************************************************
39* Header Files *
40*********************************************************************************************************************************/
41#include "the-nt-kernel.h"
42
43#include <iprt/mp.h>
44#include <iprt/alloca.h>
45#include <iprt/cpuset.h>
46#include <iprt/err.h>
47#include <iprt/asm.h>
48#include <iprt/log.h>
49#include <iprt/mem.h>
50#include <iprt/time.h>
51#include "r0drv/mp-r0drv.h"
52#include "symdb.h"
53#include "internal-r0drv-nt.h"
54#include "internal/mp.h"
55
56
57/*********************************************************************************************************************************
58* Structures and Typedefs *
59*********************************************************************************************************************************/
60typedef enum
61{
62 RT_NT_CPUID_SPECIFIC,
63 RT_NT_CPUID_PAIR,
64 RT_NT_CPUID_OTHERS,
65 RT_NT_CPUID_ALL
66} RT_NT_CPUID;
67
68
69/**
70 * Used by the RTMpOnSpecific.
71 */
72typedef struct RTMPNTONSPECIFICARGS
73{
74 /** Set if we're executing. */
75 bool volatile fExecuting;
76 /** Set when done executing. */
77 bool volatile fDone;
78 /** Number of references to this heap block. */
79 uint32_t volatile cRefs;
80 /** Event that the calling thread is waiting on. */
81 KEVENT DoneEvt;
82 /** The deferred procedure call object. */
83 KDPC Dpc;
84 /** The callback argument package. */
85 RTMPARGS CallbackArgs;
86} RTMPNTONSPECIFICARGS;
87/** Pointer to an argument/state structure for RTMpOnSpecific on NT. */
88typedef RTMPNTONSPECIFICARGS *PRTMPNTONSPECIFICARGS;
89
90
91/*********************************************************************************************************************************
92* Defined Constants And Macros *
93*********************************************************************************************************************************/
94/** Inactive bit for g_aidRtMpNtByCpuSetIdx. */
95#define RTMPNT_ID_F_INACTIVE RT_BIT_32(31)
96
97
98/*********************************************************************************************************************************
99* Global Variables *
100*********************************************************************************************************************************/
101/** Maximum number of processor groups. */
102uint32_t g_cRtMpNtMaxGroups;
103/** Maximum number of processors. */
104uint32_t g_cRtMpNtMaxCpus;
105/** Number of active processors. */
106uint32_t volatile g_cRtMpNtActiveCpus;
107/** The NT CPU set.
108 * KeQueryActiveProcssors() cannot be called at all IRQLs and therefore we'll
109 * have to cache it. Fortunately, NT doesn't really support taking CPUs offline,
110 * and taking them online was introduced with W2K8 where it is intended for virtual
111 * machines and not real HW. We update this, g_cRtMpNtActiveCpus and
112 * g_aidRtMpNtByCpuSetIdx from the rtR0NtMpProcessorChangeCallback.
113 */
114RTCPUSET g_rtMpNtCpuSet;
115
116/** Static per group info.
117 * @remarks With 256 groups this takes up 33KB. */
118static struct
119{
120 /** The max CPUs in the group. */
121 uint16_t cMaxCpus;
122 /** The number of active CPUs at the time of initialization. */
123 uint16_t cActiveCpus;
124 /** CPU set indexes for each CPU in the group. */
125 int16_t aidxCpuSetMembers[64];
126} g_aRtMpNtCpuGroups[256];
127/** Maps CPU set indexes to RTCPUID.
128 * Inactive CPUs has bit 31 set (RTMPNT_ID_F_INACTIVE) so we can identify them
129 * and shuffle duplicates during CPU hotplugging. We assign temporary IDs to
130 * the inactive CPUs starting at g_cRtMpNtMaxCpus - 1, ASSUMING that active
131 * CPUs has IDs from 0 to g_cRtMpNtActiveCpus. */
132RTCPUID g_aidRtMpNtByCpuSetIdx[RTCPUSET_MAX_CPUS];
133/** The handle of the rtR0NtMpProcessorChangeCallback registration. */
134static PVOID g_pvMpCpuChangeCallback = NULL;
135/** Size of the KAFFINITY_EX structure.
136 * This increased from 20 to 32 bitmap words in the 2020 H2 windows 10 release
137 * (i.e. 1280 to 2048 CPUs). We expect this to increase in the future. */
138static size_t g_cbRtMpNtKaffinityEx = RT_UOFFSETOF(KAFFINITY_EX, Bitmap)
139 + RT_SIZEOFMEMB(KAFFINITY_EX, Bitmap[0]) * 256;
140/** The size value of the KAFFINITY_EX structure. */
141static uint16_t g_cRtMpNtKaffinityExEntries = 256;
142
143
144/*********************************************************************************************************************************
145* Internal Functions *
146*********************************************************************************************************************************/
147static VOID __stdcall rtR0NtMpProcessorChangeCallback(void *pvUser, PKE_PROCESSOR_CHANGE_NOTIFY_CONTEXT pChangeCtx,
148 PNTSTATUS prcOperationStatus);
149static int rtR0NtInitQueryGroupRelations(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX **ppInfo);
150
151
152
153/**
154 * Initalizes multiprocessor globals (called by rtR0InitNative).
155 *
156 * @returns IPRT status code.
157 * @param pOsVerInfo Version information.
158 */
159DECLHIDDEN(int) rtR0MpNtInit(RTNTSDBOSVER const *pOsVerInfo)
160{
161#define MY_CHECK_BREAK(a_Check, a_DbgPrintArgs) \
162 AssertMsgBreakStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs; rc = VERR_INTERNAL_ERROR_4 )
163#define MY_CHECK_RETURN(a_Check, a_DbgPrintArgs, a_rcRet) \
164 AssertMsgReturnStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs, a_rcRet)
165#define MY_CHECK(a_Check, a_DbgPrintArgs) \
166 AssertMsgStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs; rc = VERR_INTERNAL_ERROR_4 )
167
168 /*
169 * API combination checks.
170 */
171 MY_CHECK_RETURN(!g_pfnrtKeSetTargetProcessorDpcEx || g_pfnrtKeGetProcessorNumberFromIndex,
172 ("IPRT: Fatal: Missing KeSetTargetProcessorDpcEx without KeGetProcessorNumberFromIndex!\n"),
173 VERR_SYMBOL_NOT_FOUND);
174
175 /*
176 * Get max number of processor groups.
177 *
178 * We may need to upadjust this number below, because windows likes to keep
179 * all options open when it comes to hotplugged CPU group assignments. A
180 * server advertising up to 64 CPUs in the ACPI table will get a result of
181 * 64 from KeQueryMaximumGroupCount. That makes sense. However, when windows
182 * server 2012 does a two processor group setup for it, the sum of the
183 * GroupInfo[*].MaximumProcessorCount members below is 128. This is probably
184 * because windows doesn't want to make decisions grouping of hotpluggable CPUs.
185 * So, we need to bump the maximum count to 128 below do deal with this as we
186 * want to have valid CPU set indexes for all potential CPUs - how could we
187 * otherwise use the RTMpGetSet() result and also RTCpuSetCount(RTMpGetSet())
188 * should equal RTMpGetCount().
189 */
190 if (g_pfnrtKeQueryMaximumGroupCount)
191 {
192 g_cRtMpNtMaxGroups = g_pfnrtKeQueryMaximumGroupCount();
193 MY_CHECK_RETURN(g_cRtMpNtMaxGroups <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxGroups > 0,
194 ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u\n", g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
195 VERR_MP_TOO_MANY_CPUS);
196 }
197 else
198 g_cRtMpNtMaxGroups = 1;
199
200 /*
201 * Get max number CPUs.
202 * This also defines the range of NT CPU indexes, RTCPUID and index into RTCPUSET.
203 */
204 if (g_pfnrtKeQueryMaximumProcessorCountEx)
205 {
206 g_cRtMpNtMaxCpus = g_pfnrtKeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS);
207 MY_CHECK_RETURN(g_cRtMpNtMaxCpus <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxCpus > 0,
208 ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, max %u [KeQueryMaximumProcessorCountEx]\n",
209 g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
210 VERR_MP_TOO_MANY_CPUS);
211 }
212 else if (g_pfnrtKeQueryMaximumProcessorCount)
213 {
214 g_cRtMpNtMaxCpus = g_pfnrtKeQueryMaximumProcessorCount();
215 MY_CHECK_RETURN(g_cRtMpNtMaxCpus <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxCpus > 0,
216 ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, max %u [KeQueryMaximumProcessorCount]\n",
217 g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
218 VERR_MP_TOO_MANY_CPUS);
219 }
220 else if (g_pfnrtKeQueryActiveProcessors)
221 {
222 KAFFINITY fActiveProcessors = g_pfnrtKeQueryActiveProcessors();
223 MY_CHECK_RETURN(fActiveProcessors != 0,
224 ("IPRT: Fatal: KeQueryActiveProcessors returned 0!\n"),
225 VERR_INTERNAL_ERROR_2);
226 g_cRtMpNtMaxCpus = 0;
227 do
228 {
229 g_cRtMpNtMaxCpus++;
230 fActiveProcessors >>= 1;
231 } while (fActiveProcessors);
232 }
233 else
234 g_cRtMpNtMaxCpus = KeNumberProcessors;
235
236 /*
237 * Just because we're a bit paranoid about getting something wrong wrt to the
238 * kernel interfaces, we try 16 times to get the KeQueryActiveProcessorCountEx
239 * and KeQueryLogicalProcessorRelationship information to match up.
240 */
241 for (unsigned cTries = 0;; cTries++)
242 {
243 /*
244 * Get number of active CPUs.
245 */
246 if (g_pfnrtKeQueryActiveProcessorCountEx)
247 {
248 g_cRtMpNtActiveCpus = g_pfnrtKeQueryActiveProcessorCountEx(ALL_PROCESSOR_GROUPS);
249 MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus && g_cRtMpNtActiveCpus > 0,
250 ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u [KeQueryActiveProcessorCountEx]\n",
251 g_cRtMpNtMaxGroups, g_cRtMpNtMaxCpus),
252 VERR_MP_TOO_MANY_CPUS);
253 }
254 else if (g_pfnrtKeQueryActiveProcessorCount)
255 {
256 g_cRtMpNtActiveCpus = g_pfnrtKeQueryActiveProcessorCount(NULL);
257 MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus && g_cRtMpNtActiveCpus > 0,
258 ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u [KeQueryActiveProcessorCount]\n",
259 g_cRtMpNtMaxGroups, g_cRtMpNtMaxCpus),
260 VERR_MP_TOO_MANY_CPUS);
261 }
262 else
263 g_cRtMpNtActiveCpus = g_cRtMpNtMaxCpus;
264
265 /*
266 * Query the details for the groups to figure out which CPUs are online as
267 * well as the NT index limit.
268 */
269 for (unsigned i = 0; i < RT_ELEMENTS(g_aidRtMpNtByCpuSetIdx); i++)
270#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
271 g_aidRtMpNtByCpuSetIdx[i] = NIL_RTCPUID;
272#else
273 g_aidRtMpNtByCpuSetIdx[i] = i < g_cRtMpNtMaxCpus ? i : NIL_RTCPUID;
274#endif
275 for (unsigned idxGroup = 0; idxGroup < RT_ELEMENTS(g_aRtMpNtCpuGroups); idxGroup++)
276 {
277 g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = 0;
278 g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = 0;
279 for (unsigned idxMember = 0; idxMember < RT_ELEMENTS(g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers); idxMember++)
280 g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = -1;
281 }
282
283 if (g_pfnrtKeQueryLogicalProcessorRelationship)
284 {
285 MY_CHECK_RETURN(g_pfnrtKeGetProcessorIndexFromNumber,
286 ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeGetProcessorIndexFromNumber!\n"),
287 VERR_SYMBOL_NOT_FOUND);
288 MY_CHECK_RETURN(g_pfnrtKeGetProcessorNumberFromIndex,
289 ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeGetProcessorIndexFromNumber!\n"),
290 VERR_SYMBOL_NOT_FOUND);
291 MY_CHECK_RETURN(g_pfnrtKeSetTargetProcessorDpcEx,
292 ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeSetTargetProcessorDpcEx!\n"),
293 VERR_SYMBOL_NOT_FOUND);
294
295 SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *pInfo = NULL;
296 int rc = rtR0NtInitQueryGroupRelations(&pInfo);
297 if (RT_FAILURE(rc))
298 return rc;
299
300 MY_CHECK(pInfo->Group.MaximumGroupCount == g_cRtMpNtMaxGroups,
301 ("IPRT: Fatal: MaximumGroupCount=%u != g_cRtMpNtMaxGroups=%u!\n",
302 pInfo->Group.MaximumGroupCount, g_cRtMpNtMaxGroups));
303 MY_CHECK(pInfo->Group.ActiveGroupCount > 0 && pInfo->Group.ActiveGroupCount <= g_cRtMpNtMaxGroups,
304 ("IPRT: Fatal: ActiveGroupCount=%u != g_cRtMpNtMaxGroups=%u!\n",
305 pInfo->Group.ActiveGroupCount, g_cRtMpNtMaxGroups));
306
307 /*
308 * First we need to recalc g_cRtMpNtMaxCpus (see above).
309 */
310 uint32_t cMaxCpus = 0;
311 uint32_t idxGroup;
312 for (idxGroup = 0; RT_SUCCESS(rc) && idxGroup < pInfo->Group.ActiveGroupCount; idxGroup++)
313 {
314 const PROCESSOR_GROUP_INFO *pGrpInfo = &pInfo->Group.GroupInfo[idxGroup];
315 MY_CHECK_BREAK(pGrpInfo->MaximumProcessorCount <= MAXIMUM_PROC_PER_GROUP,
316 ("IPRT: Fatal: MaximumProcessorCount=%u\n", pGrpInfo->MaximumProcessorCount));
317 MY_CHECK_BREAK(pGrpInfo->ActiveProcessorCount <= pGrpInfo->MaximumProcessorCount,
318 ("IPRT: Fatal: ActiveProcessorCount=%u > MaximumProcessorCount=%u\n",
319 pGrpInfo->ActiveProcessorCount, pGrpInfo->MaximumProcessorCount));
320 cMaxCpus += pGrpInfo->MaximumProcessorCount;
321 }
322 if (cMaxCpus > g_cRtMpNtMaxCpus && RT_SUCCESS(rc))
323 {
324 DbgPrint("IPRT: g_cRtMpNtMaxCpus=%u -> %u\n", g_cRtMpNtMaxCpus, cMaxCpus);
325#ifndef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
326 uint32_t i = RT_MIN(cMaxCpus, RT_ELEMENTS(g_aidRtMpNtByCpuSetIdx));
327 while (i-- > g_cRtMpNtMaxCpus)
328 g_aidRtMpNtByCpuSetIdx[i] = i;
329#endif
330 g_cRtMpNtMaxCpus = cMaxCpus;
331 if (g_cRtMpNtMaxGroups > RTCPUSET_MAX_CPUS)
332 {
333 MY_CHECK(g_cRtMpNtMaxGroups <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxGroups > 0,
334 ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u\n", g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS));
335 rc = VERR_MP_TOO_MANY_CPUS;
336 }
337 }
338
339 /*
340 * Calc online mask, partition IDs and such.
341 *
342 * Also check ASSUMPTIONS:
343 *
344 * 1. Processor indexes going from 0 and up to
345 * KeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS) - 1.
346 *
347 * 2. Currently valid processor indexes, i.e. accepted by
348 * KeGetProcessorIndexFromNumber & KeGetProcessorNumberFromIndex, goes
349 * from 0 thru KeQueryActiveProcessorCountEx(ALL_PROCESSOR_GROUPS) - 1.
350 *
351 * 3. PROCESSOR_GROUP_INFO::MaximumProcessorCount gives the number of
352 * relevant bits in the ActiveProcessorMask (from LSB).
353 *
354 * 4. Active processor count found in KeQueryLogicalProcessorRelationship
355 * output matches what KeQueryActiveProcessorCountEx(ALL) returns.
356 *
357 * 5. Active + inactive processor counts in same does not exceed
358 * KeQueryMaximumProcessorCountEx(ALL).
359 *
360 * Note! Processor indexes are assigned as CPUs come online and are not
361 * preallocated according to group maximums. Since CPUS are only taken
362 * online and never offlined, this means that internal CPU bitmaps are
363 * never sparse and no time is wasted scanning unused bits.
364 *
365 * Unfortunately, it means that ring-3 cannot easily guess the index
366 * assignments when hotswapping is used, and must use GIP when available.
367 */
368 RTCpuSetEmpty(&g_rtMpNtCpuSet);
369 uint32_t cInactive = 0;
370 uint32_t cActive = 0;
371 uint32_t idxCpuMax = 0;
372 uint32_t idxCpuSetNextInactive = g_cRtMpNtMaxCpus - 1;
373 for (idxGroup = 0; RT_SUCCESS(rc) && idxGroup < pInfo->Group.ActiveGroupCount; idxGroup++)
374 {
375 const PROCESSOR_GROUP_INFO *pGrpInfo = &pInfo->Group.GroupInfo[idxGroup];
376 MY_CHECK_BREAK(pGrpInfo->MaximumProcessorCount <= MAXIMUM_PROC_PER_GROUP,
377 ("IPRT: Fatal: MaximumProcessorCount=%u\n", pGrpInfo->MaximumProcessorCount));
378 MY_CHECK_BREAK(pGrpInfo->ActiveProcessorCount <= pGrpInfo->MaximumProcessorCount,
379 ("IPRT: Fatal: ActiveProcessorCount=%u > MaximumProcessorCount=%u\n",
380 pGrpInfo->ActiveProcessorCount, pGrpInfo->MaximumProcessorCount));
381
382 g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = pGrpInfo->MaximumProcessorCount;
383 g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = pGrpInfo->ActiveProcessorCount;
384
385 for (uint32_t idxMember = 0; idxMember < pGrpInfo->MaximumProcessorCount; idxMember++)
386 {
387 PROCESSOR_NUMBER ProcNum;
388 ProcNum.Group = (USHORT)idxGroup;
389 ProcNum.Number = (UCHAR)idxMember;
390 ProcNum.Reserved = 0;
391 ULONG idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);
392 if (idxCpu != INVALID_PROCESSOR_INDEX)
393 {
394 MY_CHECK_BREAK(idxCpu < g_cRtMpNtMaxCpus && idxCpu < RTCPUSET_MAX_CPUS, /* ASSUMPTION #1 */
395 ("IPRT: Fatal: idxCpu=%u >= g_cRtMpNtMaxCpus=%u (RTCPUSET_MAX_CPUS=%u)\n",
396 idxCpu, g_cRtMpNtMaxCpus, RTCPUSET_MAX_CPUS));
397 if (idxCpu > idxCpuMax)
398 idxCpuMax = idxCpu;
399 g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpu;
400#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
401 g_aidRtMpNtByCpuSetIdx[idxCpu] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember);
402#endif
403
404 ProcNum.Group = UINT16_MAX;
405 ProcNum.Number = UINT8_MAX;
406 ProcNum.Reserved = UINT8_MAX;
407 NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(idxCpu, &ProcNum);
408 MY_CHECK_BREAK(NT_SUCCESS(rcNt),
409 ("IPRT: Fatal: KeGetProcessorNumberFromIndex(%u,) -> %#x!\n", idxCpu, rcNt));
410 MY_CHECK_BREAK(ProcNum.Group == idxGroup && ProcNum.Number == idxMember,
411 ("IPRT: Fatal: KeGetProcessorXxxxFromYyyy roundtrip error for %#x! Group: %u vs %u, Number: %u vs %u\n",
412 idxCpu, ProcNum.Group, idxGroup, ProcNum.Number, idxMember));
413
414 if (pGrpInfo->ActiveProcessorMask & RT_BIT_64(idxMember))
415 {
416 RTCpuSetAddByIndex(&g_rtMpNtCpuSet, idxCpu);
417 cActive++;
418 }
419 else
420 cInactive++; /* (This is a little unexpected, but not important as long as things add up below.) */
421 }
422 else
423 {
424 /* Must be not present / inactive when KeGetProcessorIndexFromNumber fails. */
425 MY_CHECK_BREAK(!(pGrpInfo->ActiveProcessorMask & RT_BIT_64(idxMember)),
426 ("IPRT: Fatal: KeGetProcessorIndexFromNumber(%u/%u) failed but CPU is active! cMax=%u cActive=%u fActive=%p\n",
427 idxGroup, idxMember, pGrpInfo->MaximumProcessorCount, pGrpInfo->ActiveProcessorCount,
428 pGrpInfo->ActiveProcessorMask));
429 cInactive++;
430 if (idxCpuSetNextInactive >= g_cRtMpNtActiveCpus)
431 {
432 g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpuSetNextInactive;
433#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
434 g_aidRtMpNtByCpuSetIdx[idxCpuSetNextInactive] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember)
435 | RTMPNT_ID_F_INACTIVE;
436#endif
437 idxCpuSetNextInactive--;
438 }
439 }
440 }
441 }
442
443 MY_CHECK(cInactive + cActive <= g_cRtMpNtMaxCpus, /* ASSUMPTION #5 (not '==' because of inactive groups) */
444 ("IPRT: Fatal: cInactive=%u + cActive=%u > g_cRtMpNtMaxCpus=%u\n", cInactive, cActive, g_cRtMpNtMaxCpus));
445
446 /* Deal with inactive groups using KeQueryMaximumProcessorCountEx or as
447 best as we can by as best we can by stipulating maximum member counts
448 from the previous group. */
449 if ( RT_SUCCESS(rc)
450 && idxGroup < pInfo->Group.MaximumGroupCount)
451 {
452 uint16_t cInactiveLeft = g_cRtMpNtMaxCpus - (cInactive + cActive);
453 while (idxGroup < pInfo->Group.MaximumGroupCount)
454 {
455 uint32_t cMaxMembers = 0;
456 if (g_pfnrtKeQueryMaximumProcessorCountEx)
457 cMaxMembers = g_pfnrtKeQueryMaximumProcessorCountEx(idxGroup);
458 if (cMaxMembers != 0 || cInactiveLeft == 0)
459 AssertStmt(cMaxMembers <= cInactiveLeft, cMaxMembers = cInactiveLeft);
460 else
461 {
462 uint16_t cGroupsLeft = pInfo->Group.MaximumGroupCount - idxGroup;
463 cMaxMembers = pInfo->Group.GroupInfo[idxGroup - 1].MaximumProcessorCount;
464 while (cMaxMembers * cGroupsLeft < cInactiveLeft)
465 cMaxMembers++;
466 if (cMaxMembers > cInactiveLeft)
467 cMaxMembers = cInactiveLeft;
468 }
469
470 g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = (uint16_t)cMaxMembers;
471 g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = 0;
472 for (uint16_t idxMember = 0; idxMember < cMaxMembers; idxMember++)
473 if (idxCpuSetNextInactive >= g_cRtMpNtActiveCpus)
474 {
475 g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpuSetNextInactive;
476#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
477 g_aidRtMpNtByCpuSetIdx[idxCpuSetNextInactive] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember)
478 | RTMPNT_ID_F_INACTIVE;
479#endif
480 idxCpuSetNextInactive--;
481 }
482 cInactiveLeft -= cMaxMembers;
483 idxGroup++;
484 }
485 }
486
487 /* We're done with pInfo now, free it so we can start returning when assertions fail. */
488 RTMemFree(pInfo);
489 if (RT_FAILURE(rc)) /* MY_CHECK_BREAK sets rc. */
490 return rc;
491 MY_CHECK_RETURN(cActive >= g_cRtMpNtActiveCpus,
492 ("IPRT: Fatal: cActive=%u < g_cRtMpNtActiveCpus=%u - CPUs removed?\n", cActive, g_cRtMpNtActiveCpus),
493 VERR_INTERNAL_ERROR_3);
494 MY_CHECK_RETURN(idxCpuMax < cActive, /* ASSUMPTION #2 */
495 ("IPRT: Fatal: idCpuMax=%u >= cActive=%u! Unexpected CPU index allocation. CPUs removed?\n",
496 idxCpuMax, cActive),
497 VERR_INTERNAL_ERROR_4);
498
499 /* Retry if CPUs were added. */
500 if ( cActive != g_cRtMpNtActiveCpus
501 && cTries < 16)
502 continue;
503 MY_CHECK_RETURN(cActive == g_cRtMpNtActiveCpus, /* ASSUMPTION #4 */
504 ("IPRT: Fatal: cActive=%u != g_cRtMpNtActiveCpus=%u\n", cActive, g_cRtMpNtActiveCpus),
505 VERR_INTERNAL_ERROR_5);
506 }
507 else
508 {
509 /* Legacy: */
510 MY_CHECK_RETURN(g_cRtMpNtMaxGroups == 1, ("IPRT: Fatal: Missing KeQueryLogicalProcessorRelationship!\n"),
511 VERR_SYMBOL_NOT_FOUND);
512
513 /** @todo Is it possible that the affinity mask returned by
514 * KeQueryActiveProcessors is sparse? */
515 if (g_pfnrtKeQueryActiveProcessors)
516 RTCpuSetFromU64(&g_rtMpNtCpuSet, g_pfnrtKeQueryActiveProcessors());
517 else if (g_cRtMpNtMaxCpus < 64)
518 RTCpuSetFromU64(&g_rtMpNtCpuSet, (UINT64_C(1) << g_cRtMpNtMaxCpus) - 1);
519 else
520 {
521 MY_CHECK_RETURN(g_cRtMpNtMaxCpus == 64, ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, expect 64 or less\n", g_cRtMpNtMaxCpus),
522 VERR_MP_TOO_MANY_CPUS);
523 RTCpuSetFromU64(&g_rtMpNtCpuSet, UINT64_MAX);
524 }
525
526 g_aRtMpNtCpuGroups[0].cMaxCpus = g_cRtMpNtMaxCpus;
527 g_aRtMpNtCpuGroups[0].cActiveCpus = g_cRtMpNtMaxCpus;
528 for (unsigned i = 0; i < g_cRtMpNtMaxCpus; i++)
529 {
530 g_aRtMpNtCpuGroups[0].aidxCpuSetMembers[i] = i;
531#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
532 g_aidRtMpNtByCpuSetIdx[i] = RTMPCPUID_FROM_GROUP_AND_NUMBER(0, i);
533#endif
534 }
535 }
536
537 /*
538 * Register CPU hot plugging callback (it also counts active CPUs).
539 */
540 Assert(g_pvMpCpuChangeCallback == NULL);
541 if (g_pfnrtKeRegisterProcessorChangeCallback)
542 {
543 MY_CHECK_RETURN(g_pfnrtKeDeregisterProcessorChangeCallback,
544 ("IPRT: Fatal: KeRegisterProcessorChangeCallback without KeDeregisterProcessorChangeCallback!\n"),
545 VERR_SYMBOL_NOT_FOUND);
546
547 RTCPUSET const ActiveSetCopy = g_rtMpNtCpuSet;
548 RTCpuSetEmpty(&g_rtMpNtCpuSet);
549 uint32_t const cActiveCpus = g_cRtMpNtActiveCpus;
550 g_cRtMpNtActiveCpus = 0;
551
552 g_pvMpCpuChangeCallback = g_pfnrtKeRegisterProcessorChangeCallback(rtR0NtMpProcessorChangeCallback, NULL /*pvUser*/,
553 KE_PROCESSOR_CHANGE_ADD_EXISTING);
554 if (g_pvMpCpuChangeCallback)
555 {
556 if (cActiveCpus == g_cRtMpNtActiveCpus)
557 { /* likely */ }
558 else
559 {
560 g_pfnrtKeDeregisterProcessorChangeCallback(g_pvMpCpuChangeCallback);
561 if (cTries < 16)
562 {
563 /* Retry if CPUs were added. */
564 MY_CHECK_RETURN(g_cRtMpNtActiveCpus >= cActiveCpus,
565 ("IPRT: Fatal: g_cRtMpNtActiveCpus=%u < cActiveCpus=%u! CPUs removed?\n",
566 g_cRtMpNtActiveCpus, cActiveCpus),
567 VERR_INTERNAL_ERROR_2);
568 MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus,
569 ("IPRT: Fatal: g_cRtMpNtActiveCpus=%u > g_cRtMpNtMaxCpus=%u!\n",
570 g_cRtMpNtActiveCpus, g_cRtMpNtMaxCpus),
571 VERR_INTERNAL_ERROR_2);
572 continue;
573 }
574 MY_CHECK_RETURN(0, ("IPRT: Fatal: g_cRtMpNtActiveCpus=%u cActiveCpus=%u\n", g_cRtMpNtActiveCpus, cActiveCpus),
575 VERR_INTERNAL_ERROR_3);
576 }
577 }
578 else
579 {
580 AssertFailed();
581 g_rtMpNtCpuSet = ActiveSetCopy;
582 g_cRtMpNtActiveCpus = cActiveCpus;
583 }
584 }
585 break;
586 } /* Retry loop for stable active CPU count. */
587
588#undef MY_CHECK_RETURN
589
590 /*
591 * Special IPI fun for RTMpPokeCpu.
592 *
593 * On Vista and later the DPC method doesn't seem to reliably send IPIs,
594 * so we have to use alternative methods.
595 *
596 * On AMD64 We used to use the HalSendSoftwareInterrupt API (also x86 on
597 * W10+), it looks faster and more convenient to use, however we're either
598 * using it wrong or it doesn't reliably do what we want (see @bugref{8343}).
599 *
600 * The HalRequestIpip API is thus far the only alternative to KeInsertQueueDpc
601 * for doing targetted IPIs. Trouble with this API is that it changed
602 * fundamentally in Window 7 when they added support for lots of processors.
603 *
604 * If we really think we cannot use KeInsertQueueDpc, we use the broadcast IPI
605 * API KeIpiGenericCall.
606 */
607 if ( pOsVerInfo->uMajorVer > 6
608 || (pOsVerInfo->uMajorVer == 6 && pOsVerInfo->uMinorVer > 0))
609 g_pfnrtHalRequestIpiPreW7 = NULL;
610 else
611 g_pfnrtHalRequestIpiW7Plus = NULL;
612
613 if ( g_pfnrtHalRequestIpiW7Plus
614 && g_pfnrtKeInitializeAffinityEx
615 && g_pfnrtKeAddProcessorAffinityEx
616 && g_pfnrtKeGetProcessorIndexFromNumber)
617 {
618 /* Determine the real size of the KAFFINITY_EX structure. */
619 size_t const cbAffinity = _8K;
620 PKAFFINITY_EX pAffinity = (PKAFFINITY_EX)RTMemAllocZ(cbAffinity);
621 AssertReturn(pAffinity, VERR_NO_MEMORY);
622 size_t const cMaxEntries = (cbAffinity - RT_UOFFSETOF(KAFFINITY_EX, Bitmap[0])) / sizeof(pAffinity->Bitmap[0]);
623 g_pfnrtKeInitializeAffinityEx(pAffinity);
624 if (pAffinity->Size > 1 && pAffinity->Size <= cMaxEntries)
625 {
626 g_cRtMpNtKaffinityExEntries = pAffinity->Size;
627 g_cbRtMpNtKaffinityEx = pAffinity->Size * sizeof(pAffinity->Bitmap[0]) + RT_UOFFSETOF(KAFFINITY_EX, Bitmap[0]);
628 g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingHalRequestIpiW7Plus;
629 RTMemFree(pAffinity);
630 DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingHalRequestIpiW7Plus\n");
631 return VINF_SUCCESS;
632 }
633 DbgPrint("IPRT: RTMpPoke can't use rtMpPokeCpuUsingHalRequestIpiW7Plus! pAffinity->Size=%u\n", pAffinity->Size);
634 AssertReleaseMsg(pAffinity->Size <= cMaxEntries, ("%#x\n", pAffinity->Size)); /* stack is toast if larger (32768 CPUs). */
635 RTMemFree(pAffinity);
636 }
637
638 if (pOsVerInfo->uMajorVer >= 6 && g_pfnrtKeIpiGenericCall)
639 {
640 DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingBroadcastIpi\n");
641 g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingBroadcastIpi;
642 }
643 else if (g_pfnrtKeSetTargetProcessorDpc)
644 {
645 DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingDpc\n");
646 g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingDpc;
647 /* Windows XP should send always send an IPI -> VERIFY */
648 }
649 else
650 {
651 DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingFailureNotSupported\n");
652 Assert(pOsVerInfo->uMajorVer == 3 && pOsVerInfo->uMinorVer <= 50);
653 g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingFailureNotSupported;
654 }
655
656 return VINF_SUCCESS;
657}
658
659
660/**
661 * Called by rtR0TermNative.
662 */
663DECLHIDDEN(void) rtR0MpNtTerm(void)
664{
665 /*
666 * Deregister the processor change callback.
667 */
668 PVOID pvMpCpuChangeCallback = g_pvMpCpuChangeCallback;
669 g_pvMpCpuChangeCallback = NULL;
670 if (pvMpCpuChangeCallback)
671 {
672 AssertReturnVoid(g_pfnrtKeDeregisterProcessorChangeCallback);
673 g_pfnrtKeDeregisterProcessorChangeCallback(pvMpCpuChangeCallback);
674 }
675}
676
677
678DECLHIDDEN(int) rtR0MpNotificationNativeInit(void)
679{
680 return VINF_SUCCESS;
681}
682
683
684DECLHIDDEN(void) rtR0MpNotificationNativeTerm(void)
685{
686}
687
688
689/**
690 * Implements the NT PROCESSOR_CALLBACK_FUNCTION callback function.
691 *
692 * This maintains the g_rtMpNtCpuSet and works MP notification callbacks. When
693 * registered, it's called for each active CPU in the system, avoiding racing
694 * CPU hotplugging (as well as testing the callback).
695 *
696 * @param pvUser User context (not used).
697 * @param pChangeCtx Change context (in).
698 * @param prcOperationStatus Operation status (in/out).
699 *
700 * @remarks ASSUMES no concurrent execution of KeProcessorAddCompleteNotify
701 * notification callbacks. At least during callback registration
702 * callout, we're owning KiDynamicProcessorLock.
703 *
704 * @remarks When registering the handler, we first get KeProcessorAddStartNotify
705 * callbacks for all active CPUs, and after they all succeed we get the
706 * KeProcessorAddCompleteNotify callbacks.
707 */
708static VOID __stdcall rtR0NtMpProcessorChangeCallback(void *pvUser, PKE_PROCESSOR_CHANGE_NOTIFY_CONTEXT pChangeCtx,
709 PNTSTATUS prcOperationStatus)
710{
711 RT_NOREF(pvUser, prcOperationStatus);
712 switch (pChangeCtx->State)
713 {
714 /*
715 * Check whether we can deal with the CPU, failing the start operation if we
716 * can't. The checks we are doing here are to avoid complicated/impossible
717 * cases in KeProcessorAddCompleteNotify. They are really just verify specs.
718 */
719 case KeProcessorAddStartNotify:
720 {
721 NTSTATUS rcNt = STATUS_SUCCESS;
722 if (pChangeCtx->NtNumber < RTCPUSET_MAX_CPUS)
723 {
724 if (pChangeCtx->NtNumber >= g_cRtMpNtMaxCpus)
725 {
726 DbgPrint("IPRT: KeProcessorAddStartNotify failure: NtNumber=%u is higher than the max CPU count (%u)!\n",
727 pChangeCtx->NtNumber, g_cRtMpNtMaxCpus);
728 rcNt = STATUS_INTERNAL_ERROR;
729 }
730
731 /* The ProcessNumber field was introduced in Windows 7. */
732 PROCESSOR_NUMBER ProcNum;
733 if (g_pfnrtKeGetProcessorIndexFromNumber)
734 {
735 ProcNum = pChangeCtx->ProcNumber;
736 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);
737 if (idxCpu != pChangeCtx->NtNumber)
738 {
739 DbgPrint("IPRT: KeProcessorAddStartNotify failure: g_pfnrtKeGetProcessorIndexFromNumber(%u.%u) -> %u, expected %u!\n",
740 ProcNum.Group, ProcNum.Number, idxCpu, pChangeCtx->NtNumber);
741 rcNt = STATUS_INTERNAL_ERROR;
742 }
743 }
744 else
745 {
746 ProcNum.Group = 0;
747 ProcNum.Number = pChangeCtx->NtNumber;
748 }
749
750 if ( ProcNum.Group < RT_ELEMENTS(g_aRtMpNtCpuGroups)
751 && ProcNum.Number < RT_ELEMENTS(g_aRtMpNtCpuGroups[0].aidxCpuSetMembers))
752 {
753 if (ProcNum.Group >= g_cRtMpNtMaxGroups)
754 {
755 DbgPrint("IPRT: KeProcessorAddStartNotify failure: %u.%u is out of range - max groups: %u!\n",
756 ProcNum.Group, ProcNum.Number, g_cRtMpNtMaxGroups);
757 rcNt = STATUS_INTERNAL_ERROR;
758 }
759
760 if (ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus)
761 {
762 Assert(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] != -1);
763 if (g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] == -1)
764 {
765 DbgPrint("IPRT: KeProcessorAddStartNotify failure: Internal error! %u.%u was assigned -1 as set index!\n",
766 ProcNum.Group, ProcNum.Number);
767 rcNt = STATUS_INTERNAL_ERROR;
768 }
769
770 Assert(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] != NIL_RTCPUID);
771 if (g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] == NIL_RTCPUID)
772 {
773 DbgPrint("IPRT: KeProcessorAddStartNotify failure: Internal error! %u (%u.%u) translates to NIL_RTCPUID!\n",
774 pChangeCtx->NtNumber, ProcNum.Group, ProcNum.Number);
775 rcNt = STATUS_INTERNAL_ERROR;
776 }
777 }
778 else
779 {
780 DbgPrint("IPRT: KeProcessorAddStartNotify failure: max processors in group %u is %u, cannot add %u to it!\n",
781 ProcNum.Group, g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus, ProcNum.Group, ProcNum.Number);
782 rcNt = STATUS_INTERNAL_ERROR;
783 }
784 }
785 else
786 {
787 DbgPrint("IPRT: KeProcessorAddStartNotify failure: %u.%u is out of range (max %u.%u)!\n",
788 ProcNum.Group, ProcNum.Number, RT_ELEMENTS(g_aRtMpNtCpuGroups), RT_ELEMENTS(g_aRtMpNtCpuGroups[0].aidxCpuSetMembers));
789 rcNt = STATUS_INTERNAL_ERROR;
790 }
791 }
792 else
793 {
794 DbgPrint("IPRT: KeProcessorAddStartNotify failure: NtNumber=%u is outside RTCPUSET_MAX_CPUS (%u)!\n",
795 pChangeCtx->NtNumber, RTCPUSET_MAX_CPUS);
796 rcNt = STATUS_INTERNAL_ERROR;
797 }
798 if (!NT_SUCCESS(rcNt))
799 *prcOperationStatus = rcNt;
800 break;
801 }
802
803 /*
804 * Update the globals. Since we've checked out range limits and other
805 * limitations already we just AssertBreak here.
806 */
807 case KeProcessorAddCompleteNotify:
808 {
809 /*
810 * Calc the processor number and assert conditions checked in KeProcessorAddStartNotify.
811 */
812 AssertBreak(pChangeCtx->NtNumber < RTCPUSET_MAX_CPUS);
813 AssertBreak(pChangeCtx->NtNumber < g_cRtMpNtMaxCpus);
814 Assert(pChangeCtx->NtNumber == g_cRtMpNtActiveCpus); /* light assumption */
815 PROCESSOR_NUMBER ProcNum;
816 if (g_pfnrtKeGetProcessorIndexFromNumber)
817 {
818 ProcNum = pChangeCtx->ProcNumber;
819 AssertBreak(g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum) == pChangeCtx->NtNumber);
820 AssertBreak(ProcNum.Group < RT_ELEMENTS(g_aRtMpNtCpuGroups));
821 AssertBreak(ProcNum.Group < g_cRtMpNtMaxGroups);
822 }
823 else
824 {
825 ProcNum.Group = 0;
826 ProcNum.Number = pChangeCtx->NtNumber;
827 }
828 AssertBreak(ProcNum.Number < RT_ELEMENTS(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers));
829 AssertBreak(ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus);
830 AssertBreak(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] != -1);
831 AssertBreak(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] != NIL_RTCPUID);
832
833 /*
834 * Add ourselves to the online CPU set and update the active CPU count.
835 */
836 RTCpuSetAddByIndex(&g_rtMpNtCpuSet, pChangeCtx->NtNumber);
837 ASMAtomicIncU32(&g_cRtMpNtActiveCpus);
838
839 /*
840 * Update the group info.
841 *
842 * If the index prediction failed (real hotplugging callbacks only) we
843 * have to switch it around. This is particularly annoying when we
844 * use the index as the ID.
845 */
846#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
847 RTCPUID idCpu = RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
848 RTCPUID idOld = g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber];
849 if ((idOld & ~RTMPNT_ID_F_INACTIVE) != idCpu)
850 {
851 Assert(idOld & RTMPNT_ID_F_INACTIVE);
852 int idxDest = g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number];
853 g_aRtMpNtCpuGroups[rtMpCpuIdGetGroup(idOld)].aidxCpuSetMembers[rtMpCpuIdGetGroupMember(idOld)] = idxDest;
854 g_aidRtMpNtByCpuSetIdx[idxDest] = idOld;
855 }
856 g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] = idCpu;
857#else
858 Assert(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] == pChangeCtx->NtNumber);
859 int idxDest = g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number];
860 if ((ULONG)idxDest != pChangeCtx->NtNumber)
861 {
862 bool fFound = false;
863 uint32_t idxOldGroup = g_cRtMpNtMaxGroups;
864 while (idxOldGroup-- > 0 && !fFound)
865 {
866 uint32_t idxMember = g_aRtMpNtCpuGroups[idxOldGroup].cMaxCpus;
867 while (idxMember-- > 0)
868 if (g_aRtMpNtCpuGroups[idxOldGroup].aidxCpuSetMembers[idxMember] == (int)pChangeCtx->NtNumber)
869 {
870 g_aRtMpNtCpuGroups[idxOldGroup].aidxCpuSetMembers[idxMember] = idxDest;
871 fFound = true;
872 break;
873 }
874 }
875 Assert(fFound);
876 }
877#endif
878 g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] = pChangeCtx->NtNumber;
879
880 /*
881 * Do MP notification callbacks.
882 */
883 rtMpNotificationDoCallbacks(RTMPEVENT_ONLINE, pChangeCtx->NtNumber);
884 break;
885 }
886
887 case KeProcessorAddFailureNotify:
888 /* ignore */
889 break;
890
891 default:
892 AssertMsgFailed(("State=%u\n", pChangeCtx->State));
893 }
894}
895
896
897/**
898 * Wrapper around KeQueryLogicalProcessorRelationship.
899 *
900 * @returns IPRT status code.
901 * @param ppInfo Where to return the info. Pass to RTMemFree when done.
902 */
903static int rtR0NtInitQueryGroupRelations(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX **ppInfo)
904{
905 ULONG cbInfo = sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)
906 + g_cRtMpNtMaxGroups * sizeof(GROUP_RELATIONSHIP);
907 NTSTATUS rcNt;
908 do
909 {
910 SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *pInfo = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)RTMemAlloc(cbInfo);
911 if (pInfo)
912 {
913 rcNt = g_pfnrtKeQueryLogicalProcessorRelationship(NULL /*pProcNumber*/, RelationGroup, pInfo, &cbInfo);
914 if (NT_SUCCESS(rcNt))
915 {
916 *ppInfo = pInfo;
917 return VINF_SUCCESS;
918 }
919
920 RTMemFree(pInfo);
921 pInfo = NULL;
922 }
923 else
924 rcNt = STATUS_NO_MEMORY;
925 } while (rcNt == STATUS_INFO_LENGTH_MISMATCH);
926 DbgPrint("IPRT: Fatal: KeQueryLogicalProcessorRelationship failed: %#x\n", rcNt);
927 AssertMsgFailed(("KeQueryLogicalProcessorRelationship failed: %#x\n", rcNt));
928 return RTErrConvertFromNtStatus(rcNt);
929}
930
931
932
933
934
935RTDECL(RTCPUID) RTMpCpuId(void)
936{
937 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
938
939#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
940 PROCESSOR_NUMBER ProcNum;
941 ProcNum.Group = 0;
942 if (g_pfnrtKeGetCurrentProcessorNumberEx)
943 {
944 ProcNum.Number = 0;
945 g_pfnrtKeGetCurrentProcessorNumberEx(&ProcNum);
946 }
947 else
948 ProcNum.Number = KeGetCurrentProcessorNumber(); /* Number is 8-bit, so we're not subject to BYTE -> WORD upgrade in WDK. */
949 return RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
950
951#else
952
953 if (g_pfnrtKeGetCurrentProcessorNumberEx)
954 {
955 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(NULL);
956 Assert(idxCpu < RTCPUSET_MAX_CPUS);
957 return idxCpu;
958 }
959
960 return (uint8_t)KeGetCurrentProcessorNumber(); /* PCR->Number was changed from BYTE to WORD in the WDK, thus the cast. */
961#endif
962}
963
964
965RTDECL(int) RTMpCurSetIndex(void)
966{
967#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
968 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
969
970 if (g_pfnrtKeGetCurrentProcessorNumberEx)
971 {
972 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(NULL);
973 Assert(idxCpu < RTCPUSET_MAX_CPUS);
974 return idxCpu;
975 }
976 return (uint8_t)KeGetCurrentProcessorNumber(); /* PCR->Number was changed from BYTE to WORD in the WDK, thus the cast. */
977#else
978 return (int)RTMpCpuId();
979#endif
980}
981
982
983RTDECL(int) RTMpCurSetIndexAndId(PRTCPUID pidCpu)
984{
985#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
986 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
987
988 PROCESSOR_NUMBER ProcNum = { 0 , 0, 0 };
989 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(&ProcNum);
990 Assert(idxCpu < RTCPUSET_MAX_CPUS);
991 *pidCpu = RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
992 return idxCpu;
993#else
994 return *pidCpu = RTMpCpuId();
995#endif
996}
997
998
999RTDECL(int) RTMpCpuIdToSetIndex(RTCPUID idCpu)
1000{
1001#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
1002 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1003
1004 if (idCpu != NIL_RTCPUID)
1005 {
1006 if (g_pfnrtKeGetProcessorIndexFromNumber)
1007 {
1008 PROCESSOR_NUMBER ProcNum;
1009 ProcNum.Group = rtMpCpuIdGetGroup(idCpu);
1010 ProcNum.Number = rtMpCpuIdGetGroupMember(idCpu);
1011 ProcNum.Reserved = 0;
1012 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);
1013 if (idxCpu != INVALID_PROCESSOR_INDEX)
1014 {
1015 Assert(idxCpu < g_cRtMpNtMaxCpus);
1016 Assert((ULONG)g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] == idxCpu);
1017 return idxCpu;
1018 }
1019
1020 /* Since NT assigned indexes as the CPUs come online, we cannot produce an ID <-> index
1021 mapping for not-yet-onlined CPUS that is consistent. We just have to do our best... */
1022 if ( ProcNum.Group < g_cRtMpNtMaxGroups
1023 && ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus)
1024 return g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number];
1025 }
1026 else if (rtMpCpuIdGetGroup(idCpu) == 0)
1027 return rtMpCpuIdGetGroupMember(idCpu);
1028 }
1029 return -1;
1030#else
1031 /* 1:1 mapping, just do range checks. */
1032 return idCpu < RTCPUSET_MAX_CPUS ? (int)idCpu : -1;
1033#endif
1034}
1035
1036
1037RTDECL(RTCPUID) RTMpCpuIdFromSetIndex(int iCpu)
1038{
1039#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
1040 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1041
1042 if ((unsigned)iCpu < g_cRtMpNtMaxCpus)
1043 {
1044 if (g_pfnrtKeGetProcessorIndexFromNumber)
1045 {
1046 PROCESSOR_NUMBER ProcNum = { 0, 0, 0 };
1047 NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(iCpu, &ProcNum);
1048 if (NT_SUCCESS(rcNt))
1049 {
1050 Assert(ProcNum.Group <= g_cRtMpNtMaxGroups);
1051 Assert( (g_aidRtMpNtByCpuSetIdx[iCpu] & ~RTMPNT_ID_F_INACTIVE)
1052 == RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number));
1053 return RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
1054 }
1055 }
1056 return g_aidRtMpNtByCpuSetIdx[iCpu];
1057 }
1058 return NIL_RTCPUID;
1059#else
1060 /* 1:1 mapping, just do range checks. */
1061 return (unsigned)iCpu < RTCPUSET_MAX_CPUS ? iCpu : NIL_RTCPUID;
1062#endif
1063}
1064
1065
1066RTDECL(int) RTMpSetIndexFromCpuGroupMember(uint32_t idxGroup, uint32_t idxMember)
1067{
1068 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1069
1070 if (idxGroup < g_cRtMpNtMaxGroups)
1071 if (idxMember < g_aRtMpNtCpuGroups[idxGroup].cMaxCpus)
1072 return g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember];
1073 return -1;
1074}
1075
1076
1077RTDECL(uint32_t) RTMpGetCpuGroupCounts(uint32_t idxGroup, uint32_t *pcActive)
1078{
1079 if (idxGroup < g_cRtMpNtMaxGroups)
1080 {
1081 if (pcActive)
1082 *pcActive = g_aRtMpNtCpuGroups[idxGroup].cActiveCpus;
1083 return g_aRtMpNtCpuGroups[idxGroup].cMaxCpus;
1084 }
1085 if (pcActive)
1086 *pcActive = 0;
1087 return 0;
1088}
1089
1090
1091RTDECL(uint32_t) RTMpGetMaxCpuGroupCount(void)
1092{
1093 return g_cRtMpNtMaxGroups;
1094}
1095
1096
1097RTDECL(RTCPUID) RTMpGetMaxCpuId(void)
1098{
1099 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1100
1101#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
1102 return RTMPCPUID_FROM_GROUP_AND_NUMBER(g_cRtMpNtMaxGroups - 1, g_aRtMpNtCpuGroups[g_cRtMpNtMaxGroups - 1].cMaxCpus - 1);
1103#else
1104 /* According to MSDN the processor indexes goes from 0 to the maximum
1105 number of CPUs in the system. We've check this in initterm-r0drv-nt.cpp. */
1106 return g_cRtMpNtMaxCpus - 1;
1107#endif
1108}
1109
1110
1111RTDECL(bool) RTMpIsCpuOnline(RTCPUID idCpu)
1112{
1113 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1114 return RTCpuSetIsMember(&g_rtMpNtCpuSet, idCpu);
1115}
1116
1117
1118RTDECL(bool) RTMpIsCpuPossible(RTCPUID idCpu)
1119{
1120 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1121
1122#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
1123 if (idCpu != NIL_RTCPUID)
1124 {
1125 unsigned idxGroup = rtMpCpuIdGetGroup(idCpu);
1126 if (idxGroup < g_cRtMpNtMaxGroups)
1127 return rtMpCpuIdGetGroupMember(idCpu) < g_aRtMpNtCpuGroups[idxGroup].cMaxCpus;
1128 }
1129 return false;
1130
1131#else
1132 /* A possible CPU ID is one with a value lower than g_cRtMpNtMaxCpus (see
1133 comment in RTMpGetMaxCpuId). */
1134 return idCpu < g_cRtMpNtMaxCpus;
1135#endif
1136}
1137
1138
1139
1140RTDECL(PRTCPUSET) RTMpGetSet(PRTCPUSET pSet)
1141{
1142 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1143
1144 /* The set of possible CPU IDs(/indexes) are from 0 up to
1145 g_cRtMpNtMaxCpus (see comment in RTMpGetMaxCpuId). */
1146 RTCpuSetEmpty(pSet);
1147 int idxCpu = g_cRtMpNtMaxCpus;
1148 while (idxCpu-- > 0)
1149 RTCpuSetAddByIndex(pSet, idxCpu);
1150 return pSet;
1151}
1152
1153
1154RTDECL(RTCPUID) RTMpGetCount(void)
1155{
1156 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1157 return g_cRtMpNtMaxCpus;
1158}
1159
1160
1161RTDECL(PRTCPUSET) RTMpGetOnlineSet(PRTCPUSET pSet)
1162{
1163 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1164
1165 *pSet = g_rtMpNtCpuSet;
1166 return pSet;
1167}
1168
1169
1170RTDECL(RTCPUID) RTMpGetOnlineCount(void)
1171{
1172 RTCPUSET Set;
1173 RTMpGetOnlineSet(&Set);
1174 return RTCpuSetCount(&Set);
1175}
1176
1177
1178RTDECL(RTCPUID) RTMpGetOnlineCoreCount(void)
1179{
1180 /** @todo fix me */
1181 return RTMpGetOnlineCount();
1182}
1183
1184
1185
1186#if 0
1187/* Experiment with checking the undocumented KPRCB structure
1188 * 'dt nt!_kprcb 0xaddress' shows the layout
1189 */
1190typedef struct
1191{
1192 LIST_ENTRY DpcListHead;
1193 ULONG_PTR DpcLock;
1194 volatile ULONG DpcQueueDepth;
1195 ULONG DpcQueueCount;
1196} KDPC_DATA, *PKDPC_DATA;
1197
1198RTDECL(bool) RTMpIsCpuWorkPending(void)
1199{
1200 uint8_t *pkprcb;
1201 PKDPC_DATA pDpcData;
1202
1203 _asm {
1204 mov eax, fs:0x20
1205 mov pkprcb, eax
1206 }
1207 pDpcData = (PKDPC_DATA)(pkprcb + 0x19e0);
1208 if (pDpcData->DpcQueueDepth)
1209 return true;
1210
1211 pDpcData++;
1212 if (pDpcData->DpcQueueDepth)
1213 return true;
1214 return false;
1215}
1216#else
1217RTDECL(bool) RTMpIsCpuWorkPending(void)
1218{
1219 /** @todo not implemented */
1220 return false;
1221}
1222#endif
1223
1224
1225/**
1226 * Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1227 * the RTMpOnAll case.
1228 *
1229 * @param uUserCtx The user context argument (PRTMPARGS).
1230 */
1231static ULONG_PTR rtmpNtOnAllBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1232{
1233 PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1234 /*ASMAtomicIncU32(&pArgs->cHits); - not needed */
1235 pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2);
1236 return 0;
1237}
1238
1239
1240/**
1241 * Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1242 * the RTMpOnOthers case.
1243 *
1244 * @param uUserCtx The user context argument (PRTMPARGS).
1245 */
1246static ULONG_PTR rtmpNtOnOthersBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1247{
1248 PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1249 RTCPUID idCpu = RTMpCpuId();
1250 if (pArgs->idCpu != idCpu)
1251 {
1252 /*ASMAtomicIncU32(&pArgs->cHits); - not needed */
1253 pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
1254 }
1255 return 0;
1256}
1257
1258
1259/**
1260 * Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1261 * the RTMpOnPair case.
1262 *
1263 * @param uUserCtx The user context argument (PRTMPARGS).
1264 */
1265static ULONG_PTR rtmpNtOnPairBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1266{
1267 PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1268 RTCPUID idCpu = RTMpCpuId();
1269 if ( pArgs->idCpu == idCpu
1270 || pArgs->idCpu2 == idCpu)
1271 {
1272 ASMAtomicIncU32(&pArgs->cHits);
1273 pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
1274 }
1275 return 0;
1276}
1277
1278
1279/**
1280 * Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1281 * the RTMpOnSpecific case.
1282 *
1283 * @param uUserCtx The user context argument (PRTMPARGS).
1284 */
1285static ULONG_PTR rtmpNtOnSpecificBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1286{
1287 PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1288 RTCPUID idCpu = RTMpCpuId();
1289 if (pArgs->idCpu == idCpu)
1290 {
1291 ASMAtomicIncU32(&pArgs->cHits);
1292 pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
1293 }
1294 return 0;
1295}
1296
1297
1298/**
1299 * Internal worker for the RTMpOn* APIs using KeIpiGenericCall.
1300 *
1301 * @returns VINF_SUCCESS.
1302 * @param pfnWorker The callback.
1303 * @param pvUser1 User argument 1.
1304 * @param pvUser2 User argument 2.
1305 * @param pfnNativeWrapper The wrapper between the NT and IPRT callbacks.
1306 * @param idCpu First CPU to match, ultimately specific to the
1307 * pfnNativeWrapper used.
1308 * @param idCpu2 Second CPU to match, ultimately specific to the
1309 * pfnNativeWrapper used.
1310 * @param pcHits Where to return the number of this. Optional.
1311 */
1312static int rtMpCallUsingBroadcastIpi(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2,
1313 PKIPI_BROADCAST_WORKER pfnNativeWrapper, RTCPUID idCpu, RTCPUID idCpu2,
1314 uint32_t *pcHits)
1315{
1316 RTMPARGS Args;
1317 Args.pfnWorker = pfnWorker;
1318 Args.pvUser1 = pvUser1;
1319 Args.pvUser2 = pvUser2;
1320 Args.idCpu = idCpu;
1321 Args.idCpu2 = idCpu2;
1322 Args.cRefs = 0;
1323 Args.cHits = 0;
1324
1325 AssertPtr(g_pfnrtKeIpiGenericCall);
1326 g_pfnrtKeIpiGenericCall(pfnNativeWrapper, (uintptr_t)&Args);
1327 if (pcHits)
1328 *pcHits = Args.cHits;
1329 return VINF_SUCCESS;
1330}
1331
1332
1333/**
1334 * Wrapper between the native nt per-cpu callbacks and PFNRTWORKER
1335 *
1336 * @param Dpc DPC object
1337 * @param DeferredContext Context argument specified by KeInitializeDpc
1338 * @param SystemArgument1 Argument specified by KeInsertQueueDpc
1339 * @param SystemArgument2 Argument specified by KeInsertQueueDpc
1340 */
1341static VOID rtmpNtDPCWrapper(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2)
1342{
1343 PRTMPARGS pArgs = (PRTMPARGS)DeferredContext;
1344 RT_NOREF3(Dpc, SystemArgument1, SystemArgument2);
1345
1346 ASMAtomicIncU32(&pArgs->cHits);
1347 pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2);
1348
1349 /* Dereference the argument structure. */
1350 int32_t cRefs = ASMAtomicDecS32(&pArgs->cRefs);
1351 Assert(cRefs >= 0);
1352 if (cRefs == 0)
1353 RTMemFree(pArgs);
1354}
1355
1356
1357/**
1358 * Wrapper around KeSetTargetProcessorDpcEx / KeSetTargetProcessorDpc.
1359 *
1360 * This is shared with the timer code.
1361 *
1362 * @returns IPRT status code (errors are asserted).
1363 * @retval VERR_CPU_NOT_FOUND if impossible CPU. Not asserted.
1364 * @param pDpc The DPC.
1365 * @param idCpu The ID of the new target CPU.
1366 * @note Callable at any IRQL.
1367 */
1368DECLHIDDEN(int) rtMpNtSetTargetProcessorDpc(KDPC *pDpc, RTCPUID idCpu)
1369{
1370 if (g_pfnrtKeSetTargetProcessorDpcEx)
1371 {
1372 /* Convert to stupid process number (bet KeSetTargetProcessorDpcEx does
1373 the reverse conversion internally). */
1374 PROCESSOR_NUMBER ProcNum;
1375 NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(RTMpCpuIdToSetIndex(idCpu), &ProcNum);
1376 if (NT_SUCCESS(rcNt))
1377 {
1378 rcNt = g_pfnrtKeSetTargetProcessorDpcEx(pDpc, &ProcNum);
1379 AssertLogRelMsgReturn(NT_SUCCESS(rcNt),
1380 ("KeSetTargetProcessorDpcEx(,%u(%u/%u)) -> %#x\n", idCpu, ProcNum.Group, ProcNum.Number, rcNt),
1381 RTErrConvertFromNtStatus(rcNt));
1382 }
1383 else if (rcNt == STATUS_INVALID_PARAMETER)
1384 return VERR_CPU_NOT_FOUND;
1385 else
1386 AssertLogRelMsgReturn(NT_SUCCESS(rcNt), ("KeGetProcessorNumberFromIndex(%u) -> %#x\n", idCpu, rcNt),
1387 RTErrConvertFromNtStatus(rcNt));
1388
1389 }
1390 else if (g_pfnrtKeSetTargetProcessorDpc)
1391 g_pfnrtKeSetTargetProcessorDpc(pDpc, RTMpCpuIdToSetIndex(idCpu));
1392 else
1393 return VERR_NOT_SUPPORTED;
1394 return VINF_SUCCESS;
1395}
1396
1397
1398/**
1399 * Internal worker for the RTMpOn* APIs.
1400 *
1401 * @returns IPRT status code.
1402 * @param pfnWorker The callback.
1403 * @param pvUser1 User argument 1.
1404 * @param pvUser2 User argument 2.
1405 * @param enmCpuid What to do / is idCpu valid.
1406 * @param idCpu Used if enmCpuid is RT_NT_CPUID_SPECIFIC or
1407 * RT_NT_CPUID_PAIR, otherwise ignored.
1408 * @param idCpu2 Used if enmCpuid is RT_NT_CPUID_PAIR, otherwise ignored.
1409 * @param pcHits Where to return the number of this. Optional.
1410 */
1411static int rtMpCallUsingDpcs(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2,
1412 RT_NT_CPUID enmCpuid, RTCPUID idCpu, RTCPUID idCpu2, uint32_t *pcHits)
1413{
1414#if 0
1415 /* KeFlushQueuedDpcs must be run at IRQL PASSIVE_LEVEL according to MSDN, but the
1416 * driver verifier doesn't complain...
1417 */
1418 AssertMsg(KeGetCurrentIrql() == PASSIVE_LEVEL, ("%d != %d (PASSIVE_LEVEL)\n", KeGetCurrentIrql(), PASSIVE_LEVEL));
1419#endif
1420 /* KeFlushQueuedDpcs is not present in Windows 2000; import it dynamically so we can just fail this call. */
1421 if (!g_pfnrtNtKeFlushQueuedDpcs)
1422 return VERR_NOT_SUPPORTED;
1423
1424 /*
1425 * Make a copy of the active CPU set and figure out how many KDPCs we really need.
1426 * We must not try setup DPCs for CPUs which aren't there, because that may fail.
1427 */
1428 RTCPUSET OnlineSet = g_rtMpNtCpuSet;
1429 uint32_t cDpcsNeeded;
1430 switch (enmCpuid)
1431 {
1432 case RT_NT_CPUID_SPECIFIC:
1433 cDpcsNeeded = 1;
1434 break;
1435 case RT_NT_CPUID_PAIR:
1436 cDpcsNeeded = 2;
1437 break;
1438 default:
1439 do
1440 {
1441 cDpcsNeeded = g_cRtMpNtActiveCpus;
1442 OnlineSet = g_rtMpNtCpuSet;
1443 } while (cDpcsNeeded != g_cRtMpNtActiveCpus);
1444 break;
1445 }
1446
1447 /*
1448 * Allocate an RTMPARGS structure followed by cDpcsNeeded KDPCs
1449 * and initialize them.
1450 */
1451 PRTMPARGS pArgs = (PRTMPARGS)RTMemAllocZ(sizeof(RTMPARGS) + cDpcsNeeded * sizeof(KDPC));
1452 if (!pArgs)
1453 return VERR_NO_MEMORY;
1454
1455 pArgs->pfnWorker = pfnWorker;
1456 pArgs->pvUser1 = pvUser1;
1457 pArgs->pvUser2 = pvUser2;
1458 pArgs->idCpu = NIL_RTCPUID;
1459 pArgs->idCpu2 = NIL_RTCPUID;
1460 pArgs->cHits = 0;
1461 pArgs->cRefs = 1;
1462
1463 int rc;
1464 KDPC *paExecCpuDpcs = (KDPC *)(pArgs + 1);
1465 if (enmCpuid == RT_NT_CPUID_SPECIFIC)
1466 {
1467 KeInitializeDpc(&paExecCpuDpcs[0], rtmpNtDPCWrapper, pArgs);
1468 if (g_pfnrtKeSetImportanceDpc)
1469 g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[0], HighImportance);
1470 rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[0], idCpu);
1471 pArgs->idCpu = idCpu;
1472 }
1473 else if (enmCpuid == RT_NT_CPUID_PAIR)
1474 {
1475 KeInitializeDpc(&paExecCpuDpcs[0], rtmpNtDPCWrapper, pArgs);
1476 if (g_pfnrtKeSetImportanceDpc)
1477 g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[0], HighImportance);
1478 rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[0], idCpu);
1479 pArgs->idCpu = idCpu;
1480
1481 KeInitializeDpc(&paExecCpuDpcs[1], rtmpNtDPCWrapper, pArgs);
1482 if (g_pfnrtKeSetImportanceDpc)
1483 g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[1], HighImportance);
1484 if (RT_SUCCESS(rc))
1485 rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[1], (int)idCpu2);
1486 pArgs->idCpu2 = idCpu2;
1487 }
1488 else
1489 {
1490 rc = VINF_SUCCESS;
1491 for (uint32_t i = 0; i < cDpcsNeeded && RT_SUCCESS(rc); i++)
1492 if (RTCpuSetIsMemberByIndex(&OnlineSet, i))
1493 {
1494 KeInitializeDpc(&paExecCpuDpcs[i], rtmpNtDPCWrapper, pArgs);
1495 if (g_pfnrtKeSetImportanceDpc)
1496 g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[i], HighImportance);
1497 rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[i], RTMpCpuIdFromSetIndex(i));
1498 }
1499 }
1500 if (RT_FAILURE(rc))
1501 {
1502 RTMemFree(pArgs);
1503 return rc;
1504 }
1505
1506 /*
1507 * Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu.
1508 * KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL.
1509 */
1510 KIRQL oldIrql;
1511 KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
1512
1513 /*
1514 * We cannot do other than assume a 1:1 relationship between the
1515 * affinity mask and the process despite the warnings in the docs.
1516 * If someone knows a better way to get this done, please let bird know.
1517 */
1518 ASMCompilerBarrier(); /* paranoia */
1519 if (enmCpuid == RT_NT_CPUID_SPECIFIC)
1520 {
1521 ASMAtomicIncS32(&pArgs->cRefs);
1522 BOOLEAN fRc = KeInsertQueueDpc(&paExecCpuDpcs[0], 0, 0);
1523 Assert(fRc); NOREF(fRc);
1524 }
1525 else if (enmCpuid == RT_NT_CPUID_PAIR)
1526 {
1527 ASMAtomicIncS32(&pArgs->cRefs);
1528 BOOLEAN fRc = KeInsertQueueDpc(&paExecCpuDpcs[0], 0, 0);
1529 Assert(fRc); NOREF(fRc);
1530
1531 ASMAtomicIncS32(&pArgs->cRefs);
1532 fRc = KeInsertQueueDpc(&paExecCpuDpcs[1], 0, 0);
1533 Assert(fRc); NOREF(fRc);
1534 }
1535 else
1536 {
1537 uint32_t iSelf = RTMpCurSetIndex();
1538 for (uint32_t i = 0; i < cDpcsNeeded; i++)
1539 {
1540 if ( (i != iSelf)
1541 && RTCpuSetIsMemberByIndex(&OnlineSet, i))
1542 {
1543 ASMAtomicIncS32(&pArgs->cRefs);
1544 BOOLEAN fRc = KeInsertQueueDpc(&paExecCpuDpcs[i], 0, 0);
1545 Assert(fRc); NOREF(fRc);
1546 }
1547 }
1548 if (enmCpuid != RT_NT_CPUID_OTHERS)
1549 pfnWorker(iSelf, pvUser1, pvUser2);
1550 }
1551
1552 KeLowerIrql(oldIrql);
1553
1554 /*
1555 * Flush all DPCs and wait for completion. (can take long!)
1556 */
1557 /** @todo Consider changing this to an active wait using some atomic inc/dec
1558 * stuff (and check for the current cpu above in the specific case). */
1559 /** @todo Seems KeFlushQueuedDpcs doesn't wait for the DPCs to be completely
1560 * executed. Seen pArgs being freed while some CPU was using it before
1561 * cRefs was added. */
1562 if (g_pfnrtNtKeFlushQueuedDpcs)
1563 g_pfnrtNtKeFlushQueuedDpcs();
1564
1565 if (pcHits)
1566 *pcHits = pArgs->cHits;
1567
1568 /* Dereference the argument structure. */
1569 int32_t cRefs = ASMAtomicDecS32(&pArgs->cRefs);
1570 Assert(cRefs >= 0);
1571 if (cRefs == 0)
1572 RTMemFree(pArgs);
1573
1574 return VINF_SUCCESS;
1575}
1576
1577
1578RTDECL(int) RTMpOnAll(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
1579{
1580 if (g_pfnrtKeIpiGenericCall)
1581 return rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnAllBroadcastIpiWrapper,
1582 NIL_RTCPUID, NIL_RTCPUID, NULL);
1583 return rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_ALL, NIL_RTCPUID, NIL_RTCPUID, NULL);
1584}
1585
1586
1587RTDECL(int) RTMpOnOthers(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
1588{
1589 if (g_pfnrtKeIpiGenericCall)
1590 return rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnOthersBroadcastIpiWrapper,
1591 NIL_RTCPUID, NIL_RTCPUID, NULL);
1592 return rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_OTHERS, NIL_RTCPUID, NIL_RTCPUID, NULL);
1593}
1594
1595
1596RTDECL(int) RTMpOnPair(RTCPUID idCpu1, RTCPUID idCpu2, uint32_t fFlags, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
1597{
1598 int rc;
1599 AssertReturn(idCpu1 != idCpu2, VERR_INVALID_PARAMETER);
1600 AssertReturn(!(fFlags & RTMPON_F_VALID_MASK), VERR_INVALID_FLAGS);
1601 if ((fFlags & RTMPON_F_CONCURRENT_EXEC) && !g_pfnrtKeIpiGenericCall)
1602 return VERR_NOT_SUPPORTED;
1603
1604 /*
1605 * Check that both CPUs are online before doing the broadcast call.
1606 */
1607 if ( RTMpIsCpuOnline(idCpu1)
1608 && RTMpIsCpuOnline(idCpu2))
1609 {
1610 /*
1611 * The broadcast IPI isn't quite as bad as it could have been, because
1612 * it looks like windows doesn't synchronize CPUs on the way out, they
1613 * seems to get back to normal work while the pair is still busy.
1614 */
1615 uint32_t cHits = 0;
1616 if (g_pfnrtKeIpiGenericCall)
1617 rc = rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnPairBroadcastIpiWrapper, idCpu1, idCpu2, &cHits);
1618 else
1619 rc = rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_PAIR, idCpu1, idCpu2, &cHits);
1620 if (RT_SUCCESS(rc))
1621 {
1622 Assert(cHits <= 2);
1623 if (cHits == 2)
1624 rc = VINF_SUCCESS;
1625 else if (cHits == 1)
1626 rc = VERR_NOT_ALL_CPUS_SHOWED;
1627 else if (cHits == 0)
1628 rc = VERR_CPU_OFFLINE;
1629 else
1630 rc = VERR_CPU_IPE_1;
1631 }
1632 }
1633 /*
1634 * A CPU must be present to be considered just offline.
1635 */
1636 else if ( RTMpIsCpuPresent(idCpu1)
1637 && RTMpIsCpuPresent(idCpu2))
1638 rc = VERR_CPU_OFFLINE;
1639 else
1640 rc = VERR_CPU_NOT_FOUND;
1641 return rc;
1642}
1643
1644
1645RTDECL(bool) RTMpOnPairIsConcurrentExecSupported(void)
1646{
1647 return g_pfnrtKeIpiGenericCall != NULL;
1648}
1649
1650
1651/**
1652 * Releases a reference to a RTMPNTONSPECIFICARGS heap allocation, freeing it
1653 * when the last reference is released.
1654 */
1655DECLINLINE(void) rtMpNtOnSpecificRelease(PRTMPNTONSPECIFICARGS pArgs)
1656{
1657 uint32_t cRefs = ASMAtomicDecU32(&pArgs->cRefs);
1658 AssertMsg(cRefs <= 1, ("cRefs=%#x\n", cRefs));
1659 if (cRefs == 0)
1660 RTMemFree(pArgs);
1661}
1662
1663
1664/**
1665 * Wrapper between the native nt per-cpu callbacks and PFNRTWORKER
1666 *
1667 * @param Dpc DPC object
1668 * @param DeferredContext Context argument specified by KeInitializeDpc
1669 * @param SystemArgument1 Argument specified by KeInsertQueueDpc
1670 * @param SystemArgument2 Argument specified by KeInsertQueueDpc
1671 */
1672static VOID rtMpNtOnSpecificDpcWrapper(IN PKDPC Dpc, IN PVOID DeferredContext,
1673 IN PVOID SystemArgument1, IN PVOID SystemArgument2)
1674{
1675 PRTMPNTONSPECIFICARGS pArgs = (PRTMPNTONSPECIFICARGS)DeferredContext;
1676 RT_NOREF3(Dpc, SystemArgument1, SystemArgument2);
1677
1678 ASMAtomicWriteBool(&pArgs->fExecuting, true);
1679
1680 pArgs->CallbackArgs.pfnWorker(RTMpCpuId(), pArgs->CallbackArgs.pvUser1, pArgs->CallbackArgs.pvUser2);
1681
1682 ASMAtomicWriteBool(&pArgs->fDone, true);
1683 KeSetEvent(&pArgs->DoneEvt, 1 /*PriorityIncrement*/, FALSE /*Wait*/);
1684
1685 rtMpNtOnSpecificRelease(pArgs);
1686}
1687
1688
1689RTDECL(int) RTMpOnSpecific(RTCPUID idCpu, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
1690{
1691 /*
1692 * Don't try mess with an offline CPU.
1693 */
1694 if (!RTMpIsCpuOnline(idCpu))
1695 return !RTMpIsCpuPossible(idCpu)
1696 ? VERR_CPU_NOT_FOUND
1697 : VERR_CPU_OFFLINE;
1698
1699 /*
1700 * Use the broadcast IPI routine if there are no more than two CPUs online,
1701 * or if the current IRQL is unsuitable for KeWaitForSingleObject.
1702 */
1703 int rc;
1704 uint32_t cHits = 0;
1705 if ( g_pfnrtKeIpiGenericCall
1706 && ( RTMpGetOnlineCount() <= 2
1707 || KeGetCurrentIrql() > APC_LEVEL)
1708 )
1709 {
1710 rc = rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnSpecificBroadcastIpiWrapper,
1711 idCpu, NIL_RTCPUID, &cHits);
1712 if (RT_SUCCESS(rc))
1713 {
1714 if (cHits == 1)
1715 return VINF_SUCCESS;
1716 rc = cHits == 0 ? VERR_CPU_OFFLINE : VERR_CPU_IPE_1;
1717 }
1718 return rc;
1719 }
1720
1721#if 0
1722 rc = rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_SPECIFIC, idCpu, NIL_RTCPUID, &cHits);
1723 if (RT_SUCCESS(rc))
1724 {
1725 if (cHits == 1)
1726 return VINF_SUCCESS;
1727 rc = cHits == 0 ? VERR_CPU_OFFLINE : VERR_CPU_IPE_1;
1728 }
1729 return rc;
1730
1731#else
1732 /*
1733 * Initialize the argument package and the objects within it.
1734 * The package is referenced counted to avoid unnecessary spinning to
1735 * synchronize cleanup and prevent stack corruption.
1736 */
1737 PRTMPNTONSPECIFICARGS pArgs = (PRTMPNTONSPECIFICARGS)RTMemAllocZ(sizeof(*pArgs));
1738 if (!pArgs)
1739 return VERR_NO_MEMORY;
1740 pArgs->cRefs = 2;
1741 pArgs->fExecuting = false;
1742 pArgs->fDone = false;
1743 pArgs->CallbackArgs.pfnWorker = pfnWorker;
1744 pArgs->CallbackArgs.pvUser1 = pvUser1;
1745 pArgs->CallbackArgs.pvUser2 = pvUser2;
1746 pArgs->CallbackArgs.idCpu = idCpu;
1747 pArgs->CallbackArgs.cHits = 0;
1748 pArgs->CallbackArgs.cRefs = 2;
1749 KeInitializeEvent(&pArgs->DoneEvt, SynchronizationEvent, FALSE /* not signalled */);
1750 KeInitializeDpc(&pArgs->Dpc, rtMpNtOnSpecificDpcWrapper, pArgs);
1751 if (g_pfnrtKeSetImportanceDpc)
1752 g_pfnrtKeSetImportanceDpc(&pArgs->Dpc, HighImportance);
1753 rc = rtMpNtSetTargetProcessorDpc(&pArgs->Dpc, idCpu);
1754 if (RT_FAILURE(rc))
1755 {
1756 RTMemFree(pArgs);
1757 return rc;
1758 }
1759
1760 /*
1761 * Disable preemption while we check the current processor and inserts the DPC.
1762 */
1763 KIRQL bOldIrql;
1764 KeRaiseIrql(DISPATCH_LEVEL, &bOldIrql);
1765 ASMCompilerBarrier(); /* paranoia */
1766
1767 if (RTMpCpuId() == idCpu)
1768 {
1769 /* Just execute the callback on the current CPU. */
1770 pfnWorker(idCpu, pvUser1, pvUser2);
1771 KeLowerIrql(bOldIrql);
1772
1773 RTMemFree(pArgs);
1774 return VINF_SUCCESS;
1775 }
1776
1777 /* Different CPU, so queue it if the CPU is still online. */
1778 if (RTMpIsCpuOnline(idCpu))
1779 {
1780 BOOLEAN fRc = KeInsertQueueDpc(&pArgs->Dpc, 0, 0);
1781 Assert(fRc); NOREF(fRc);
1782 KeLowerIrql(bOldIrql);
1783
1784 uint64_t const nsRealWaitTS = RTTimeNanoTS();
1785
1786 /*
1787 * Wait actively for a while in case the CPU/thread responds quickly.
1788 */
1789 uint32_t cLoopsLeft = 0x20000;
1790 while (cLoopsLeft-- > 0)
1791 {
1792 if (pArgs->fDone)
1793 {
1794 rtMpNtOnSpecificRelease(pArgs);
1795 return VINF_SUCCESS;
1796 }
1797 ASMNopPause();
1798 }
1799
1800 /*
1801 * It didn't respond, so wait on the event object, poking the CPU if it's slow.
1802 */
1803 LARGE_INTEGER Timeout;
1804 Timeout.QuadPart = -10000; /* 1ms */
1805 NTSTATUS rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
1806 if (rcNt == STATUS_SUCCESS)
1807 {
1808 rtMpNtOnSpecificRelease(pArgs);
1809 return VINF_SUCCESS;
1810 }
1811
1812 /* If it hasn't respondend yet, maybe poke it and wait some more. */
1813 if (rcNt == STATUS_TIMEOUT)
1814 {
1815 if ( !pArgs->fExecuting
1816 && ( g_pfnrtMpPokeCpuWorker == rtMpPokeCpuUsingHalRequestIpiW7Plus
1817 || g_pfnrtMpPokeCpuWorker == rtMpPokeCpuUsingHalRequestIpiPreW7))
1818 RTMpPokeCpu(idCpu);
1819
1820 Timeout.QuadPart = -1280000; /* 128ms */
1821 rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
1822 if (rcNt == STATUS_SUCCESS)
1823 {
1824 rtMpNtOnSpecificRelease(pArgs);
1825 return VINF_SUCCESS;
1826 }
1827 }
1828
1829 /*
1830 * Something weird is happening, try bail out.
1831 */
1832 if (KeRemoveQueueDpc(&pArgs->Dpc))
1833 {
1834 RTMemFree(pArgs); /* DPC was still queued, so we can return without further ado. */
1835 LogRel(("RTMpOnSpecific(%#x): Not processed after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
1836 }
1837 else
1838 {
1839 /* DPC is running, wait a good while for it to complete. */
1840 LogRel(("RTMpOnSpecific(%#x): Still running after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
1841
1842 Timeout.QuadPart = -30*1000*1000*10; /* 30 seconds */
1843 rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
1844 if (rcNt != STATUS_SUCCESS)
1845 LogRel(("RTMpOnSpecific(%#x): Giving up on running worker after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
1846 }
1847 rc = RTErrConvertFromNtStatus(rcNt);
1848 }
1849 else
1850 {
1851 /* CPU is offline.*/
1852 KeLowerIrql(bOldIrql);
1853 rc = !RTMpIsCpuPossible(idCpu) ? VERR_CPU_NOT_FOUND : VERR_CPU_OFFLINE;
1854 }
1855
1856 rtMpNtOnSpecificRelease(pArgs);
1857 return rc;
1858#endif
1859}
1860
1861
1862
1863
1864static VOID rtMpNtPokeCpuDummy(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2)
1865{
1866 NOREF(Dpc);
1867 NOREF(DeferredContext);
1868 NOREF(SystemArgument1);
1869 NOREF(SystemArgument2);
1870}
1871
1872
1873/** Callback used by rtMpPokeCpuUsingBroadcastIpi. */
1874static ULONG_PTR rtMpIpiGenericCall(ULONG_PTR Argument)
1875{
1876 NOREF(Argument);
1877 return 0;
1878}
1879
1880
1881/**
1882 * RTMpPokeCpu worker that uses broadcast IPIs for doing the work.
1883 *
1884 * @returns VINF_SUCCESS
1885 * @param idCpu The CPU identifier.
1886 */
1887int rtMpPokeCpuUsingBroadcastIpi(RTCPUID idCpu)
1888{
1889 NOREF(idCpu);
1890 g_pfnrtKeIpiGenericCall(rtMpIpiGenericCall, 0);
1891 return VINF_SUCCESS;
1892}
1893
1894
1895/**
1896 * RTMpPokeCpu worker that uses the Windows 7 and later version of
1897 * HalRequestIpip to get the job done.
1898 *
1899 * @returns VINF_SUCCESS
1900 * @param idCpu The CPU identifier.
1901 */
1902int rtMpPokeCpuUsingHalRequestIpiW7Plus(RTCPUID idCpu)
1903{
1904 /* idCpu is an HAL processor index, so we can use it directly. */
1905 PKAFFINITY_EX pTarget = (PKAFFINITY_EX)alloca(g_cbRtMpNtKaffinityEx);
1906 pTarget->Size = g_cRtMpNtKaffinityExEntries; /* (just in case KeInitializeAffinityEx starts using it) */
1907 g_pfnrtKeInitializeAffinityEx(pTarget);
1908 g_pfnrtKeAddProcessorAffinityEx(pTarget, idCpu);
1909
1910 g_pfnrtHalRequestIpiW7Plus(0, pTarget);
1911 return VINF_SUCCESS;
1912}
1913
1914
1915/**
1916 * RTMpPokeCpu worker that uses the Vista and earlier version of HalRequestIpip
1917 * to get the job done.
1918 *
1919 * @returns VINF_SUCCESS
1920 * @param idCpu The CPU identifier.
1921 */
1922int rtMpPokeCpuUsingHalRequestIpiPreW7(RTCPUID idCpu)
1923{
1924 __debugbreak(); /** @todo this code needs testing!! */
1925 KAFFINITY Target = 1;
1926 Target <<= idCpu;
1927 g_pfnrtHalRequestIpiPreW7(Target);
1928 return VINF_SUCCESS;
1929}
1930
1931
1932int rtMpPokeCpuUsingFailureNotSupported(RTCPUID idCpu)
1933{
1934 NOREF(idCpu);
1935 return VERR_NOT_SUPPORTED;
1936}
1937
1938
1939int rtMpPokeCpuUsingDpc(RTCPUID idCpu)
1940{
1941 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1942
1943 /*
1944 * APC fallback.
1945 */
1946 static KDPC s_aPokeDpcs[RTCPUSET_MAX_CPUS] = {{0}};
1947 static bool s_fPokeDPCsInitialized = false;
1948
1949 if (!s_fPokeDPCsInitialized)
1950 {
1951 for (unsigned i = 0; i < g_cRtMpNtMaxCpus; i++)
1952 {
1953 KeInitializeDpc(&s_aPokeDpcs[i], rtMpNtPokeCpuDummy, NULL);
1954 if (g_pfnrtKeSetImportanceDpc)
1955 g_pfnrtKeSetImportanceDpc(&s_aPokeDpcs[i], HighImportance);
1956 int rc = rtMpNtSetTargetProcessorDpc(&s_aPokeDpcs[i], idCpu);
1957 if (RT_FAILURE(rc) && rc != VERR_CPU_NOT_FOUND)
1958 return rc;
1959 }
1960
1961 s_fPokeDPCsInitialized = true;
1962 }
1963
1964 /* Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu.
1965 KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL. */
1966 KIRQL oldIrql;
1967 KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
1968
1969 if (g_pfnrtKeSetImportanceDpc)
1970 g_pfnrtKeSetImportanceDpc(&s_aPokeDpcs[idCpu], HighImportance);
1971 g_pfnrtKeSetTargetProcessorDpc(&s_aPokeDpcs[idCpu], (int)idCpu);
1972
1973 /* Assuming here that high importance DPCs will be delivered immediately; or at least an IPI will be sent immediately.
1974 Note! Not true on at least Vista & Windows 7 */
1975 BOOLEAN fRet = KeInsertQueueDpc(&s_aPokeDpcs[idCpu], 0, 0);
1976
1977 KeLowerIrql(oldIrql);
1978 return fRet == TRUE ? VINF_SUCCESS : VERR_ACCESS_DENIED /* already queued */;
1979}
1980
1981
1982RTDECL(int) RTMpPokeCpu(RTCPUID idCpu)
1983{
1984 if (!RTMpIsCpuOnline(idCpu))
1985 return !RTMpIsCpuPossible(idCpu)
1986 ? VERR_CPU_NOT_FOUND
1987 : VERR_CPU_OFFLINE;
1988 /* Calls rtMpPokeCpuUsingDpc, rtMpPokeCpuUsingHalRequestIpiW7Plus or rtMpPokeCpuUsingBroadcastIpi. */
1989 return g_pfnrtMpPokeCpuWorker(idCpu);
1990}
1991
1992
1993RTDECL(bool) RTMpOnAllIsConcurrentSafe(void)
1994{
1995 return false;
1996}
1997
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette