VirtualBox

source: vbox/trunk/src/VBox/Runtime/r0drv/linux/mp-r0drv-linux.c@ 56290

Last change on this file since 56290 was 56290, checked in by vboxsync, 10 years ago

IPRT: Updated (C) year.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 15.0 KB
Line 
1/* $Id: mp-r0drv-linux.c 56290 2015-06-09 14:01:31Z vboxsync $ */
2/** @file
3 * IPRT - Multiprocessor, Ring-0 Driver, Linux.
4 */
5
6/*
7 * Copyright (C) 2008-2015 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * The contents of this file may alternatively be used under the terms
18 * of the Common Development and Distribution License Version 1.0
19 * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20 * VirtualBox OSE distribution, in which case the provisions of the
21 * CDDL are applicable instead of those of the GPL.
22 *
23 * You may elect to license modified versions of this file under the
24 * terms and conditions of either the GPL or the CDDL or both.
25 */
26
27
28/*******************************************************************************
29* Header Files *
30*******************************************************************************/
31#include "the-linux-kernel.h"
32#include "internal/iprt.h"
33
34#include <iprt/mp.h>
35#include <iprt/cpuset.h>
36#include <iprt/err.h>
37#include <iprt/asm.h>
38#include <iprt/thread.h>
39#include "r0drv/mp-r0drv.h"
40
41
42RTDECL(RTCPUID) RTMpCpuId(void)
43{
44 return smp_processor_id();
45}
46RT_EXPORT_SYMBOL(RTMpCpuId);
47
48
49RTDECL(int) RTMpCurSetIndex(void)
50{
51 return smp_processor_id();
52}
53RT_EXPORT_SYMBOL(RTMpCurSetIndex);
54
55
56RTDECL(int) RTMpCurSetIndexAndId(PRTCPUID pidCpu)
57{
58 return *pidCpu = smp_processor_id();
59}
60RT_EXPORT_SYMBOL(RTMpCurSetIndexAndId);
61
62
63RTDECL(int) RTMpCpuIdToSetIndex(RTCPUID idCpu)
64{
65 return idCpu < RTCPUSET_MAX_CPUS && idCpu < NR_CPUS ? (int)idCpu : -1;
66}
67RT_EXPORT_SYMBOL(RTMpCpuIdToSetIndex);
68
69
70RTDECL(RTCPUID) RTMpCpuIdFromSetIndex(int iCpu)
71{
72 return iCpu < NR_CPUS ? (RTCPUID)iCpu : NIL_RTCPUID;
73}
74RT_EXPORT_SYMBOL(RTMpCpuIdFromSetIndex);
75
76
77RTDECL(RTCPUID) RTMpGetMaxCpuId(void)
78{
79 return NR_CPUS - 1; //???
80}
81RT_EXPORT_SYMBOL(RTMpGetMaxCpuId);
82
83
84RTDECL(bool) RTMpIsCpuPossible(RTCPUID idCpu)
85{
86#if defined(CONFIG_SMP)
87 if (RT_UNLIKELY(idCpu >= NR_CPUS))
88 return false;
89
90# if defined(cpu_possible)
91 return cpu_possible(idCpu);
92# else /* < 2.5.29 */
93 return idCpu < (RTCPUID)smp_num_cpus;
94# endif
95#else
96 return idCpu == RTMpCpuId();
97#endif
98}
99RT_EXPORT_SYMBOL(RTMpIsCpuPossible);
100
101
102RTDECL(PRTCPUSET) RTMpGetSet(PRTCPUSET pSet)
103{
104 RTCPUID idCpu;
105
106 RTCpuSetEmpty(pSet);
107 idCpu = RTMpGetMaxCpuId();
108 do
109 {
110 if (RTMpIsCpuPossible(idCpu))
111 RTCpuSetAdd(pSet, idCpu);
112 } while (idCpu-- > 0);
113 return pSet;
114}
115RT_EXPORT_SYMBOL(RTMpGetSet);
116
117
118RTDECL(RTCPUID) RTMpGetCount(void)
119{
120#ifdef CONFIG_SMP
121# if defined(CONFIG_HOTPLUG_CPU) /* introduced & uses cpu_present */
122 return num_present_cpus();
123# elif defined(num_possible_cpus)
124 return num_possible_cpus();
125# elif LINUX_VERSION_CODE < KERNEL_VERSION(2, 5, 0)
126 return smp_num_cpus;
127# else
128 RTCPUSET Set;
129 RTMpGetSet(&Set);
130 return RTCpuSetCount(&Set);
131# endif
132#else
133 return 1;
134#endif
135}
136RT_EXPORT_SYMBOL(RTMpGetCount);
137
138
139RTDECL(bool) RTMpIsCpuOnline(RTCPUID idCpu)
140{
141#ifdef CONFIG_SMP
142 if (RT_UNLIKELY(idCpu >= NR_CPUS))
143 return false;
144# ifdef cpu_online
145 return cpu_online(idCpu);
146# else /* 2.4: */
147 return cpu_online_map & RT_BIT_64(idCpu);
148# endif
149#else
150 return idCpu == RTMpCpuId();
151#endif
152}
153RT_EXPORT_SYMBOL(RTMpIsCpuOnline);
154
155
156RTDECL(PRTCPUSET) RTMpGetOnlineSet(PRTCPUSET pSet)
157{
158#ifdef CONFIG_SMP
159 RTCPUID idCpu;
160
161 RTCpuSetEmpty(pSet);
162 idCpu = RTMpGetMaxCpuId();
163 do
164 {
165 if (RTMpIsCpuOnline(idCpu))
166 RTCpuSetAdd(pSet, idCpu);
167 } while (idCpu-- > 0);
168#else
169 RTCpuSetEmpty(pSet);
170 RTCpuSetAdd(pSet, RTMpCpuId());
171#endif
172 return pSet;
173}
174RT_EXPORT_SYMBOL(RTMpGetOnlineSet);
175
176
177RTDECL(RTCPUID) RTMpGetOnlineCount(void)
178{
179#ifdef CONFIG_SMP
180# if defined(num_online_cpus)
181 return num_online_cpus();
182# else
183 RTCPUSET Set;
184 RTMpGetOnlineSet(&Set);
185 return RTCpuSetCount(&Set);
186# endif
187#else
188 return 1;
189#endif
190}
191RT_EXPORT_SYMBOL(RTMpGetOnlineCount);
192
193
194RTDECL(bool) RTMpIsCpuWorkPending(void)
195{
196 /** @todo (not used on non-Windows platforms yet). */
197 return false;
198}
199RT_EXPORT_SYMBOL(RTMpIsCpuWorkPending);
200
201
202/**
203 * Wrapper between the native linux per-cpu callbacks and PFNRTWORKER.
204 *
205 * @param pvInfo Pointer to the RTMPARGS package.
206 */
207static void rtmpLinuxWrapper(void *pvInfo)
208{
209 PRTMPARGS pArgs = (PRTMPARGS)pvInfo;
210 ASMAtomicIncU32(&pArgs->cHits);
211 pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2);
212}
213
214
215/**
216 * Wrapper between the native linux per-cpu callbacks and PFNRTWORKER, does hit
217 * increment after calling the worker.
218 *
219 * @param pvInfo Pointer to the RTMPARGS package.
220 */
221static void rtmpLinuxWrapperPostInc(void *pvInfo)
222{
223 PRTMPARGS pArgs = (PRTMPARGS)pvInfo;
224 pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2);
225 ASMAtomicIncU32(&pArgs->cHits);
226}
227
228
229/**
230 * Wrapper between the native linux all-cpu callbacks and PFNRTWORKER.
231 *
232 * @param pvInfo Pointer to the RTMPARGS package.
233 */
234static void rtmpLinuxAllWrapper(void *pvInfo)
235{
236 PRTMPARGS pArgs = (PRTMPARGS)pvInfo;
237 PRTCPUSET pWorkerSet = pArgs->pWorkerSet;
238 RTCPUID idCpu = RTMpCpuId();
239 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
240
241 if (RTCpuSetIsMember(pWorkerSet, idCpu))
242 {
243 pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
244 RTCpuSetDel(pWorkerSet, idCpu);
245 }
246}
247
248
249RTDECL(int) RTMpOnAll(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
250{
251 int rc;
252 RTMPARGS Args;
253 RTCPUSET OnlineSet;
254 RTCPUID idCpu;
255 uint32_t cLoops;
256
257 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
258
259 Args.pfnWorker = pfnWorker;
260 Args.pvUser1 = pvUser1;
261 Args.pvUser2 = pvUser2;
262 Args.idCpu = NIL_RTCPUID;
263 Args.cHits = 0;
264
265 RTThreadPreemptDisable(&PreemptState);
266 RTMpGetOnlineSet(&OnlineSet);
267 Args.pWorkerSet = &OnlineSet;
268 idCpu = RTMpCpuId();
269
270 if (RTCpuSetCount(&OnlineSet) > 1)
271 {
272 /* Fire the function on all other CPUs without waiting for completion. */
273#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
274 rc = smp_call_function(rtmpLinuxAllWrapper, &Args, 0 /* wait */);
275#else
276 rc = smp_call_function(rtmpLinuxAllWrapper, &Args, 0 /* retry */, 0 /* wait */);
277#endif
278 Assert(!rc); NOREF(rc);
279 }
280
281 /* Fire the function on this CPU. */
282 Args.pfnWorker(idCpu, Args.pvUser1, Args.pvUser2);
283 RTCpuSetDel(Args.pWorkerSet, idCpu);
284
285 /* Wait for all of them finish. */
286 cLoops = 64000;
287 while (!RTCpuSetIsEmpty(Args.pWorkerSet))
288 {
289 /* Periodically check if any CPU in the wait set has gone offline, if so update the wait set. */
290 if (!cLoops--)
291 {
292 RTCPUSET OnlineSetNow;
293 RTMpGetOnlineSet(&OnlineSetNow);
294 RTCpuSetAnd(Args.pWorkerSet, &OnlineSetNow);
295
296 cLoops = 64000;
297 }
298
299 ASMNopPause();
300 }
301
302 RTThreadPreemptRestore(&PreemptState);
303 return VINF_SUCCESS;
304}
305RT_EXPORT_SYMBOL(RTMpOnAll);
306
307
308RTDECL(int) RTMpOnOthers(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
309{
310 int rc;
311 RTMPARGS Args;
312
313 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
314 Args.pfnWorker = pfnWorker;
315 Args.pvUser1 = pvUser1;
316 Args.pvUser2 = pvUser2;
317 Args.idCpu = NIL_RTCPUID;
318 Args.cHits = 0;
319
320 RTThreadPreemptDisable(&PreemptState);
321#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
322 rc = smp_call_function(rtmpLinuxWrapper, &Args, 1 /* wait */);
323#else /* older kernels */
324 rc = smp_call_function(rtmpLinuxWrapper, &Args, 0 /* retry */, 1 /* wait */);
325#endif /* older kernels */
326 RTThreadPreemptRestore(&PreemptState);
327
328 Assert(rc == 0); NOREF(rc);
329 return VINF_SUCCESS;
330}
331RT_EXPORT_SYMBOL(RTMpOnOthers);
332
333
334#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 27)
335/**
336 * Wrapper between the native linux per-cpu callbacks and PFNRTWORKER
337 * employed by RTMpOnPair on older kernels that lacks smp_call_function_many.
338 *
339 * @param pvInfo Pointer to the RTMPARGS package.
340 */
341static void rtMpLinuxOnPairWrapper(void *pvInfo)
342{
343 PRTMPARGS pArgs = (PRTMPARGS)pvInfo;
344 RTCPUID idCpu = RTMpCpuId();
345
346 if ( idCpu == pArgs->idCpu
347 || idCpu == pArgs->idCpu2)
348 {
349 pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
350 ASMAtomicIncU32(&pArgs->cHits);
351 }
352}
353#endif
354
355
356RTDECL(int) RTMpOnPair(RTCPUID idCpu1, RTCPUID idCpu2, uint32_t fFlags, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
357{
358 int rc;
359 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
360
361 AssertReturn(idCpu1 != idCpu2, VERR_INVALID_PARAMETER);
362 AssertReturn(!(fFlags & RTMPON_F_VALID_MASK), VERR_INVALID_FLAGS);
363
364 /*
365 * Check that both CPUs are online before doing the broadcast call.
366 */
367 RTThreadPreemptDisable(&PreemptState);
368 if ( RTMpIsCpuOnline(idCpu1)
369 && RTMpIsCpuOnline(idCpu2))
370 {
371 /*
372 * Use the smp_call_function variant taking a cpu mask where available,
373 * falling back on broadcast with filter. Slight snag if one of the
374 * CPUs is the one we're running on, we must do the call and the post
375 * call wait ourselves.
376 */
377#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
378 cpumask_t DstCpuMask;
379#endif
380 RTCPUID idCpuSelf = RTMpCpuId();
381 bool const fCallSelf = idCpuSelf == idCpu1 || idCpuSelf == idCpu2;
382 RTMPARGS Args;
383 Args.pfnWorker = pfnWorker;
384 Args.pvUser1 = pvUser1;
385 Args.pvUser2 = pvUser2;
386 Args.idCpu = idCpu1;
387 Args.idCpu2 = idCpu2;
388 Args.cHits = 0;
389
390#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 28)
391 cpumask_clear(&DstCpuMask);
392 cpumask_set_cpu(idCpu1, &DstCpuMask);
393 cpumask_set_cpu(idCpu2, &DstCpuMask);
394#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
395 cpus_clear(DstCpuMask);
396 cpu_set(idCpu1, DstCpuMask);
397 cpu_set(idCpu2, DstCpuMask);
398#endif
399
400#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29)
401 smp_call_function_many(&DstCpuMask, rtmpLinuxWrapperPostInc, &Args, !fCallSelf /* wait */);
402 rc = 0;
403#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 28)
404 rc = smp_call_function_many(&DstCpuMask, rtmpLinuxWrapperPostInc, &Args, !fCallSelf /* wait */);
405#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
406 rc = smp_call_function_mask(DstCpuMask, rtmpLinuxWrapperPostInc, &Args, !fCallSelf /* wait */);
407#else /* older kernels */
408 rc = smp_call_function(rtMpLinuxOnPairWrapper, &Args, 0 /* retry */, !fCallSelf /* wait */);
409#endif /* older kernels */
410 Assert(rc == 0);
411
412 /* Call ourselves if necessary and wait for the other party to be done. */
413 if (fCallSelf)
414 {
415 uint32_t cLoops = 0;
416 rtmpLinuxWrapper(&Args);
417 while (ASMAtomicReadU32(&Args.cHits) < 2)
418 {
419 if ((cLoops & 0x1ff) == 0 && !RTMpIsCpuOnline(idCpuSelf == idCpu1 ? idCpu2 : idCpu1))
420 break;
421 cLoops++;
422 ASMNopPause();
423 }
424 }
425
426 Assert(Args.cHits <= 2);
427 if (Args.cHits == 2)
428 rc = VINF_SUCCESS;
429 else if (Args.cHits == 1)
430 rc = VERR_NOT_ALL_CPUS_SHOWED;
431 else if (Args.cHits == 0)
432 rc = VERR_CPU_OFFLINE;
433 else
434 rc = VERR_CPU_IPE_1;
435 }
436 /*
437 * A CPU must be present to be considered just offline.
438 */
439 else if ( RTMpIsCpuPresent(idCpu1)
440 && RTMpIsCpuPresent(idCpu2))
441 rc = VERR_CPU_OFFLINE;
442 else
443 rc = VERR_CPU_NOT_FOUND;
444 RTThreadPreemptRestore(&PreemptState);;
445 return rc;
446}
447RT_EXPORT_SYMBOL(RTMpOnPair);
448
449
450RTDECL(bool) RTMpOnPairIsConcurrentExecSupported(void)
451{
452 return true;
453}
454RT_EXPORT_SYMBOL(RTMpOnPairIsConcurrentExecSupported);
455
456
457#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 19)
458/**
459 * Wrapper between the native linux per-cpu callbacks and PFNRTWORKER
460 * employed by RTMpOnSpecific on older kernels that lacks smp_call_function_single.
461 *
462 * @param pvInfo Pointer to the RTMPARGS package.
463 */
464static void rtmpOnSpecificLinuxWrapper(void *pvInfo)
465{
466 PRTMPARGS pArgs = (PRTMPARGS)pvInfo;
467 RTCPUID idCpu = RTMpCpuId();
468
469 if (idCpu == pArgs->idCpu)
470 {
471 pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
472 ASMAtomicIncU32(&pArgs->cHits);
473 }
474}
475#endif
476
477
478RTDECL(int) RTMpOnSpecific(RTCPUID idCpu, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
479{
480 int rc;
481 RTMPARGS Args;
482
483 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
484 Args.pfnWorker = pfnWorker;
485 Args.pvUser1 = pvUser1;
486 Args.pvUser2 = pvUser2;
487 Args.idCpu = idCpu;
488 Args.cHits = 0;
489
490 if (!RTMpIsCpuPossible(idCpu))
491 return VERR_CPU_NOT_FOUND;
492
493 RTThreadPreemptDisable(&PreemptState);
494 if (idCpu != RTMpCpuId())
495 {
496 if (RTMpIsCpuOnline(idCpu))
497 {
498#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
499 rc = smp_call_function_single(idCpu, rtmpLinuxWrapper, &Args, 1 /* wait */);
500#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19)
501 rc = smp_call_function_single(idCpu, rtmpLinuxWrapper, &Args, 0 /* retry */, 1 /* wait */);
502#else /* older kernels */
503 rc = smp_call_function(rtmpOnSpecificLinuxWrapper, &Args, 0 /* retry */, 1 /* wait */);
504#endif /* older kernels */
505 Assert(rc == 0);
506 rc = Args.cHits ? VINF_SUCCESS : VERR_CPU_OFFLINE;
507 }
508 else
509 rc = VERR_CPU_OFFLINE;
510 }
511 else
512 {
513 rtmpLinuxWrapper(&Args);
514 rc = VINF_SUCCESS;
515 }
516 RTThreadPreemptRestore(&PreemptState);;
517
518 NOREF(rc);
519 return rc;
520}
521RT_EXPORT_SYMBOL(RTMpOnSpecific);
522
523
524#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19)
525/**
526 * Dummy callback used by RTMpPokeCpu.
527 *
528 * @param pvInfo Ignored.
529 */
530static void rtmpLinuxPokeCpuCallback(void *pvInfo)
531{
532 NOREF(pvInfo);
533}
534#endif
535
536
537RTDECL(int) RTMpPokeCpu(RTCPUID idCpu)
538{
539#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19)
540 int rc;
541
542 if (!RTMpIsCpuPossible(idCpu))
543 return VERR_CPU_NOT_FOUND;
544 if (!RTMpIsCpuOnline(idCpu))
545 return VERR_CPU_OFFLINE;
546
547# if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
548 rc = smp_call_function_single(idCpu, rtmpLinuxPokeCpuCallback, NULL, 0 /* wait */);
549# elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19)
550 rc = smp_call_function_single(idCpu, rtmpLinuxPokeCpuCallback, NULL, 0 /* retry */, 0 /* wait */);
551# else /* older kernels */
552# error oops
553# endif /* older kernels */
554 NOREF(rc);
555 Assert(rc == 0);
556 return VINF_SUCCESS;
557
558#else /* older kernels */
559 /* no unicast here? */
560 return VERR_NOT_SUPPORTED;
561#endif /* older kernels */
562}
563RT_EXPORT_SYMBOL(RTMpPokeCpu);
564
565
566RTDECL(bool) RTMpOnAllIsConcurrentSafe(void)
567{
568 return true;
569}
570RT_EXPORT_SYMBOL(RTMpOnAllIsConcurrentSafe);
571
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette