timer-r0drv-linux.c@ 24696

Last change on this file since 24696 was 24181, checked in by vboxsync, 15 years ago
VMM,SUPDrv,IPRT: Always initialize RTSPINLOCKTMP structures.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 30.0 KB

Line
1	/* $Id: timer-r0drv-linux.c 24181 2009-10-30 10:51:56Z vboxsync $ */
2	/** @file
3	* IPRT - Timers, Ring-0 Driver, Linux.
4	*/
5
6	/*
7	* Copyright (C) 2006-2008 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*
26	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
27	* Clara, CA 95054 USA or visit http://www.sun.com if you need
28	* additional information or have any questions.
29	*/
30
31
32	/*******************************************************************************
33	* Header Files *
34	*******************************************************************************/
35	#include "the-linux-kernel.h"
36	#include "internal/iprt.h"
37
38	#include <iprt/timer.h>
39	#include <iprt/time.h>
40	#include <iprt/mp.h>
41	#include <iprt/cpuset.h>
42	#include <iprt/spinlock.h>
43	#include <iprt/err.h>
44	#include <iprt/asm.h>
45	#include <iprt/assert.h>
46	#include <iprt/alloc.h>
47
48	#include "internal/magics.h"
49
50	#if !defined(RT_USE_LINUX_HRTIMER) \
51	&& LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 23) \
52	&& 0 /* disabled because it somehow sucks. */
53	# define RT_USE_LINUX_HRTIMER
54	#endif
55
56	/* This check must match the ktime usage in rtTimeGetSystemNanoTS() / time-r0drv-linux.c. */
57	#if defined(RT_USE_LINUX_HRTIMER) \
58	&& LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 16)
59	# error "RT_USE_LINUX_HRTIMER requires 2.6.16 or later, sorry."
60	#endif
61
62
63	/*******************************************************************************
64	* Structures and Typedefs *
65	*******************************************************************************/
66	/**
67	* Timer state machine.
68	*
69	* This is used to try handle the issues with MP events and
70	* timers that runs on all CPUs. It's relatively nasty :-/
71	*/
72	typedef enum RTTIMERLNXSTATE
73	{
74	/** Stopped. */
75	RTTIMERLNXSTATE_STOPPED = 0,
76	/** Transient state; next ACTIVE. */
77	RTTIMERLNXSTATE_STARTING,
78	/** Transient state; next ACTIVE. (not really necessary) */
79	RTTIMERLNXSTATE_MP_STARTING,
80	/** Active. */
81	RTTIMERLNXSTATE_ACTIVE,
82	/** Transient state; next STOPPED. */
83	RTTIMERLNXSTATE_STOPPING,
84	/** Transient state; next STOPPED. */
85	RTTIMERLNXSTATE_MP_STOPPING,
86	/** The usual 32-bit hack. */
87	RTTIMERLNXSTATE_32BIT_HACK = 0x7fffffff
88	} RTTIMERLNXSTATE;
89
90
91	/**
92	* A Linux sub-timer.
93	*/
94	typedef struct RTTIMERLNXSUBTIMER
95	{
96	/** The linux timer structure. */
97	#ifdef RT_USE_LINUX_HRTIMER
98	struct hrtimer LnxTimer;
99	#else
100	struct timer_list LnxTimer;
101	#endif
102	/** The start of the current run (ns).
103	* This is used to calculate when the timer ought to fire the next time. */
104	uint64_t u64StartTS;
105	/** The start of the current run (ns).
106	* This is used to calculate when the timer ought to fire the next time. */
107	uint64_t u64NextTS;
108	/** The current tick number (since u64StartTS). */
109	uint64_t iTick;
110	/** Pointer to the parent timer. */
111	PRTTIMER pParent;
112	#ifndef RT_USE_LINUX_HRTIMER
113	/** The u64NextTS in jiffies. */
114	unsigned long ulNextJiffies;
115	#endif
116	/** The current sub-timer state. */
117	RTTIMERLNXSTATE volatile enmState;
118	} RTTIMERLNXSUBTIMER;
119	/** Pointer to a linux sub-timer. */
120	typedef RTTIMERLNXSUBTIMER *PRTTIMERLNXSUBTIMER;
121	AssertCompileMemberOffset(RTTIMERLNXSUBTIMER, LnxTimer, 0);
122
123
124	/**
125	* The internal representation of an Linux timer handle.
126	*/
127	typedef struct RTTIMER
128	{
129	/** Magic.
130	* This is RTTIMER_MAGIC, but changes to something else before the timer
131	* is destroyed to indicate clearly that thread should exit. */
132	uint32_t volatile u32Magic;
133	/** Spinlock synchronizing the fSuspended and MP event handling.
134	* This is NIL_RTSPINLOCK if cCpus == 1. */
135	RTSPINLOCK hSpinlock;
136	/** Flag indicating that the timer is suspended. */
137	bool volatile fSuspended;
138	/** Whether the timer must run on one specific CPU or not. */
139	bool fSpecificCpu;
140	#ifdef CONFIG_SMP
141	/** Whether the timer must run on all CPUs or not. */
142	bool fAllCpus;
143	#endif /* else: All -> specific on non-SMP kernels */
144	/** The CPU it must run on if fSpecificCpu is set. */
145	RTCPUID idCpu;
146	/** The number of CPUs this timer should run on. */
147	RTCPUID cCpus;
148	/** Callback. */
149	PFNRTTIMER pfnTimer;
150	/** User argument. */
151	void *pvUser;
152	/** The timer interval. 0 if one-shot. */
153	uint64_t u64NanoInterval;
154	#ifndef RT_USE_LINUX_HRTIMER
155	/** This is set to the number of jiffies between ticks if the interval is
156	* an exact number of jiffies. */
157	unsigned long cJiffies;
158	#endif
159	/** Sub-timers.
160	* Normally there is just one, but for RTTIMER_FLAGS_CPU_ALL this will contain
161	* an entry for all possible cpus. In that case the index will be the same as
162	* for the RTCpuSet. */
163	RTTIMERLNXSUBTIMER aSubTimers[1];
164	} RTTIMER;
165
166
167	/**
168	* A rtTimerLinuxStartOnCpu and rtTimerLinuxStartOnCpu argument package.
169	*/
170	typedef struct RTTIMERLINUXSTARTONCPUARGS
171	{
172	/** The current time (RTTimeNanoTS). */
173	uint64_t u64Now;
174	/** When to start firing (delta). */
175	uint64_t u64First;
176	} RTTIMERLINUXSTARTONCPUARGS;
177	/** Pointer to a rtTimerLinuxStartOnCpu argument package. */
178	typedef RTTIMERLINUXSTARTONCPUARGS *PRTTIMERLINUXSTARTONCPUARGS;
179
180
181	/**
182	* Sets the state.
183	*/
184	DECLINLINE(void) rtTimerLnxSetState(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState)
185	{
186	ASMAtomicWriteU32((uint32_t volatile *)penmState, enmNewState);
187	}
188
189
190	/**
191	* Sets the state if it has a certain value.
192	*
193	* @return true if xchg was done.
194	* @return false if xchg wasn't done.
195	*/
196	DECLINLINE(bool) rtTimerLnxCmpXchgState(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState, RTTIMERLNXSTATE enmCurState)
197	{
198	return ASMAtomicCmpXchgU32((uint32_t volatile *)penmState, enmNewState, enmCurState);
199	}
200
201
202	/**
203	* Gets the state.
204	*/
205	DECLINLINE(RTTIMERLNXSTATE) rtTimerLnxGetState(RTTIMERLNXSTATE volatile *penmState)
206	{
207	return (RTTIMERLNXSTATE)ASMAtomicUoReadU32((uint32_t volatile *)penmState);
208	}
209
210
211	#ifdef RT_USE_LINUX_HRTIMER
212	/**
213	* Converts a nano second time stamp to ktime_t.
214	*
215	* ASSUMES RTTimeNanoTS() is implemented using ktime_get_ts().
216	*
217	* @returns ktime_t.
218	* @param cNanoSecs Nanoseconds.
219	*/
220	DECLINLINE(ktime_t) rtTimerLnxNanoToKt(uint64_t cNanoSecs)
221	{
222	/* With some luck the compiler optimizes the division out of this... (Bet it doesn't.) */
223	return ktime_set(cNanoSecs / 1000000000, cNanoSecs % 1000000000);
224	}
225
226	/**
227	* Converts ktime_t to a nano second time stamp.
228	*
229	* ASSUMES RTTimeNanoTS() is implemented using ktime_get_ts().
230	*
231	* @returns nano second time stamp.
232	* @param Kt ktime_t.
233	*/
234	DECLINLINE(uint64_t) rtTimerLnxKtToNano(ktime_t Kt)
235	{
236	return ktime_to_ns(Kt);
237	}
238
239	#else /* ! RT_USE_LINUX_HRTIMER */
240
241	/**
242	* Converts a nano second interval to jiffies.
243	*
244	* @returns Jiffies.
245	* @param cNanoSecs Nanoseconds.
246	*/
247	DECLINLINE(unsigned long) rtTimerLnxNanoToJiffies(uint64_t cNanoSecs)
248	{
249	/* this can be made even better... */
250	if (cNanoSecs > (uint64_t)TICK_NSEC * MAX_JIFFY_OFFSET)
251	return MAX_JIFFY_OFFSET;
252	#if ARCH_BITS == 32
253	if (RT_LIKELY(cNanoSecs <= UINT32_MAX))
254	return ((uint32_t)cNanoSecs + (TICK_NSEC-1)) / TICK_NSEC;
255	#endif
256	return (cNanoSecs + (TICK_NSEC-1)) / TICK_NSEC;
257	}
258	#endif
259
260
261	/**
262	* Starts a sub-timer (RTTimerStart).
263	*
264	* @param pSubTimer The sub-timer to start.
265	* @param u64Now The current timestamp (RTTimeNanoTS()).
266	* @param u64First The interval from u64Now to the first time the timer should fire.
267	*/
268	static void rtTimerLnxStartSubTimer(PRTTIMERLNXSUBTIMER pSubTimer, uint64_t u64Now, uint64_t u64First)
269	{
270	/*
271	* Calc when it should start firing.
272	*/
273	uint64_t u64NextTS = u64Now + u64First;
274	pSubTimer->u64StartTS = u64NextTS;
275	pSubTimer->u64NextTS = u64NextTS;
276	pSubTimer->iTick = 0;
277
278	#ifdef RT_USE_LINUX_HRTIMER
279	hrtimer_start(&pSubTimer->LnxTimer, rtTimerLnxNanoToKt(u64NextTS), HRTIMER_MODE_ABS);
280	#else
281	{
282	unsigned long cJiffies = !u64First ? 0 : rtTimerLnxNanoToJiffies(u64First);
283	pSubTimer->ulNextJiffies = jiffies + cJiffies;
284	mod_timer(&pSubTimer->LnxTimer, pSubTimer->ulNextJiffies);
285	}
286	#endif
287
288	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE);
289	}
290
291
292	/**
293	* Stops a sub-timer (RTTimerStart and rtTimerLinuxMpEvent()).
294	*
295	* @param pSubTimer The sub-timer.
296	*/
297	static void rtTimerLnxStopSubTimer(PRTTIMERLNXSUBTIMER pSubTimer)
298	{
299	#ifdef RT_USE_LINUX_HRTIMER
300	hrtimer_cancel(&pSubTimer->LnxTimer);
301	#else
302	if (timer_pending(&pSubTimer->LnxTimer))
303	del_timer_sync(&pSubTimer->LnxTimer);
304	#endif
305
306	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED);
307	}
308
309
310	#ifdef RT_USE_LINUX_HRTIMER
311	/**
312	* Timer callback function.
313	* @returns HRTIMER_NORESTART or HRTIMER_RESTART depending on whether it's a one-shot or interval timer.
314	* @param pHrTimer Pointer to the sub-timer structure.
315	*/
316	static enum hrtimer_restart rtTimerLinuxCallback(struct hrtimer *pHrTimer)
317	#else
318	/**
319	* Timer callback function.
320	* @param ulUser Address of the sub-timer structure.
321	*/
322	static void rtTimerLinuxCallback(unsigned long ulUser)
323	#endif
324	{
325	#ifdef RT_USE_LINUX_HRTIMER
326	enum hrtimer_restart rc;
327	PRTTIMERLNXSUBTIMER pSubTimer = (PRTTIMERLNXSUBTIMER)pHrTimer;
328	#else
329	PRTTIMERLNXSUBTIMER pSubTimer = (PRTTIMERLNXSUBTIMER)ulUser;
330	#endif
331	PRTTIMER pTimer = pSubTimer->pParent;
332
333	/*
334	* Don't call the handler if the timer has been suspended.
335	* Also, when running on all CPUS, make sure we don't call out twice
336	* on a CPU because of timer migration.
337	*
338	* For the specific cpu case, we're just ignoring timer migration for now... (bad)
339	*/
340	if ( ASMAtomicUoReadBool(&pTimer->fSuspended)
341	#ifdef CONFIG_SMP
342	\|\| ( pTimer->fAllCpus
343	&& (RTCPUID)(pSubTimer - &pTimer->aSubTimers[0]) != RTMpCpuId())
344	#endif
345	)
346	{
347	rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_ACTIVE);
348	# ifdef RT_USE_LINUX_HRTIMER
349	rc = HRTIMER_NORESTART;
350	# endif
351	}
352	else if (!pTimer->u64NanoInterval)
353	{
354	/*
355	* One shot timer, stop it before dispatching it.
356	*/
357	if (pTimer->cCpus == 1)
358	ASMAtomicWriteBool(&pTimer->fSuspended, true);
359	rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_ACTIVE);
360	#ifdef RT_USE_LINUX_HRTIMER
361	rc = HRTIMER_NORESTART;
362	#else
363	/* detached before we're called, nothing to do for this case. */
364	#endif
365
366	pTimer->pfnTimer(pTimer, pTimer->pvUser, ++pSubTimer->iTick);
367	}
368	else
369	{
370	/*
371	* Interval timer, calculate the next timeout and re-arm it.
372	*
373	* The first time around, we'll re-adjust the u64StartTS to
374	* try prevent some jittering if we were started at a bad time.
375	* This may of course backfire with highres timers...
376	*/
377	const uint64_t u64NanoTS = RTTimeNanoTS();
378	const uint64_t iTick = ++pSubTimer->iTick;
379
380	if (RT_UNLIKELY(iTick == 1))
381	{
382	#ifdef RT_USE_LINUX_HRTIMER
383	pSubTimer->u64StartTS = pSubTimer->u64NextTS = u64NanoTS;//rtTimerLnxKtToNano(pSubTimer->LnxTimer.base->softirq_time);
384	#else
385	pSubTimer->u64StartTS = pSubTimer->u64NextTS = u64NanoTS;
386	pSubTimer->ulNextJiffies = jiffies;
387	#endif
388	}
389
390	pSubTimer->u64NextTS += pTimer->u64NanoInterval;
391
392	#ifdef RT_USE_LINUX_HRTIMER
393	while (pSubTimer->u64NextTS < u64NanoTS)
394	pSubTimer->u64NextTS += pTimer->u64NanoInterval;
395
396	pSubTimer->LnxTimer.expires = rtTimerLnxNanoToKt(pSubTimer->u64NextTS);
397	rc = HRTIMER_RESTART;
398	#else
399	if (pTimer->cJiffies)
400	{
401	pSubTimer->ulNextJiffies += pTimer->cJiffies;
402	/* Prevent overflows when the jiffies counter wraps around.
403	* Special thanks to Ken Preslan for helping debugging! */
404	while (time_before(pSubTimer->ulNextJiffies, jiffies))
405	{
406	pSubTimer->ulNextJiffies += pTimer->cJiffies;
407	pSubTimer->u64NextTS += pTimer->u64NanoInterval;
408	}
409	}
410	else
411	{
412	while (pSubTimer->u64NextTS < u64NanoTS)
413	pSubTimer->u64NextTS += pTimer->u64NanoInterval;
414	pSubTimer->ulNextJiffies = jiffies + rtTimerLnxNanoToJiffies(pSubTimer->u64NextTS - u64NanoTS);
415	}
416
417	mod_timer(&pSubTimer->LnxTimer, pSubTimer->ulNextJiffies);
418	#endif
419
420	/*
421	* Run the timer.
422	*/
423	pTimer->pfnTimer(pTimer, pTimer->pvUser, iTick);
424	}
425
426	#ifdef RT_USE_LINUX_HRTIMER
427	return rc;
428	#endif
429	}
430
431
432	#ifdef CONFIG_SMP
433
434	/**
435	* Per-cpu callback function (RTMpOnAll/RTMpOnSpecific).
436	*
437	* @param idCpu The current CPU.
438	* @param pvUser1 Pointer to the timer.
439	* @param pvUser2 Pointer to the argument structure.
440	*/
441	static DECLCALLBACK(void) rtTimerLnxStartAllOnCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
442	{
443	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
444	PRTTIMER pTimer = (PRTTIMER)pvUser1;
445	Assert(idCpu < pTimer->cCpus);
446	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[idCpu], pArgs->u64Now, pArgs->u64First);
447	}
448
449
450	/**
451	* Worker for RTTimerStart() that takes care of the ugly bit.s
452	*
453	* @returns RTTimerStart() return value.
454	* @param pTimer The timer.
455	* @param pArgs The argument structure.
456	*/
457	static int rtTimerLnxStartAll(PRTTIMER pTimer, PRTTIMERLINUXSTARTONCPUARGS pArgs)
458	{
459	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
460	RTCPUID iCpu;
461	RTCPUSET OnlineSet;
462	RTCPUSET OnlineSet2;
463	int rc2;
464
465	/*
466	* Prepare all the sub-timers for the startup and then flag the timer
467	* as a whole as non-suspended, make sure we get them all before
468	* clearing fSuspended as the MP handler will be waiting on this
469	* should something happen while we're looping.
470	*/
471	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
472
473	do
474	{
475	RTMpGetOnlineSet(&OnlineSet);
476	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
477	{
478	Assert(pTimer->aSubTimers[iCpu].enmState != RTTIMERLNXSTATE_MP_STOPPING);
479	rtTimerLnxSetState(&pTimer->aSubTimers[iCpu].enmState,
480	RTCpuSetIsMember(&OnlineSet, iCpu)
481	? RTTIMERLNXSTATE_STARTING
482	: RTTIMERLNXSTATE_STOPPED);
483	}
484	} while (!RTCpuSetIsEqual(&OnlineSet, RTMpGetOnlineSet(&OnlineSet2)));
485
486	ASMAtomicWriteBool(&pTimer->fSuspended, false);
487
488	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
489
490	/*
491	* Start them (can't find any exported function that allows me to
492	* do this without the cross calls).
493	*/
494	pArgs->u64Now = RTTimeNanoTS();
495	rc2 = RTMpOnAll(rtTimerLnxStartAllOnCpu, pTimer, pArgs);
496	AssertRC(rc2); /* screw this if it fails. */
497
498	/*
499	* Reset the sub-timers who didn't start up (ALL CPUs case).
500	* CPUs that comes online between the
501	*/
502	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
503
504	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
505	if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_STARTING))
506	{
507	/** @todo very odd case for a rainy day. Cpus that temporarily went offline while
508	* we were between calls needs to nudged as the MP handler will ignore events for
509	* them because of the STARTING state. This is an extremely unlikely case - not that
510	* that means anything in my experience... ;-) */
511	}
512
513	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
514
515	return VINF_SUCCESS;
516	}
517
518
519	/**
520	* Worker for RTTimerStop() that takes care of the ugly SMP bits.
521	*
522	* @returns RTTimerStop() return value.
523	* @param pTimer The timer (valid).
524	*/
525	static int rtTimerLnxStopAll(PRTTIMER pTimer)
526	{
527	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
528	RTCPUID iCpu;
529
530
531	/*
532	* Mark the timer as suspended and flag all timers as stopping, except
533	* for those being stopped by an MP event.
534	*/
535	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
536
537	ASMAtomicWriteBool(&pTimer->fSuspended, true);
538	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
539	{
540	RTTIMERLNXSTATE enmState;
541	do
542	{
543	enmState = rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState);
544	if ( enmState == RTTIMERLNXSTATE_STOPPED
545	\|\| enmState == RTTIMERLNXSTATE_MP_STOPPING)
546	break;
547	Assert(enmState == RTTIMERLNXSTATE_ACTIVE);
548	} while (!rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_STOPPING, enmState));
549	}
550
551	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
552
553	/*
554	* Do the actual stopping. Fortunately, this doesn't require any IPIs.
555	* Unfortunately it cannot be done synchronously from within the spinlock,
556	* because we might end up in an active waiting for a handler to complete.
557	*/
558	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
559	if (rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState) == RTTIMERLNXSTATE_STOPPING)
560	rtTimerLnxStopSubTimer(&pTimer->aSubTimers[iCpu]);
561
562	return VINF_SUCCESS;
563	}
564
565
566	/**
567	* Per-cpu callback function (RTMpOnSpecific) used by rtTimerLinuxMpEvent()
568	* to start a sub-timer on a cpu that just have come online.
569	*
570	* @param idCpu The current CPU.
571	* @param pvUser1 Pointer to the timer.
572	* @param pvUser2 Pointer to the argument structure.
573	*/
574	static DECLCALLBACK(void) rtTimerLinuxMpStartOnCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
575	{
576	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
577	PRTTIMER pTimer = (PRTTIMER)pvUser1;
578	RTSPINLOCK hSpinlock;
579	Assert(idCpu < pTimer->cCpus);
580
581	/*
582	* We have to be kind of careful here as we might be racing RTTimerStop
583	* (and/or RTTimerDestroy, thus the paranoia.
584	*/
585	hSpinlock = pTimer->hSpinlock;
586	if ( hSpinlock != NIL_RTSPINLOCK
587	&& pTimer->u32Magic == RTTIMER_MAGIC)
588	{
589	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
590	RTSpinlockAcquire(hSpinlock, &Tmp);
591
592	if ( !ASMAtomicUoReadBool(&pTimer->fSuspended)
593	&& pTimer->u32Magic == RTTIMER_MAGIC)
594	{
595	/* We're sane and the timer is not suspended yet. */
596	PRTTIMERLNXSUBTIMER pSubTimer = &pTimer->aSubTimers[idCpu];
597	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STARTING, RTTIMERLNXSTATE_STOPPED))
598	rtTimerLnxStartSubTimer(pSubTimer, pArgs->u64Now, pArgs->u64First);
599	}
600
601	RTSpinlockRelease(hSpinlock, &Tmp);
602	}
603	}
604
605
606	/**
607	* MP event notification callback.
608	*
609	* @param enmEvent The event.
610	* @param idCpu The cpu it applies to.
611	* @param pvUser The timer.
612	*/
613	static DECLCALLBACK(void) rtTimerLinuxMpEvent(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvUser)
614	{
615	PRTTIMER pTimer = (PRTTIMER)pvUser;
616	PRTTIMERLNXSUBTIMER pSubTimer = &pTimer->aSubTimers[idCpu];
617	RTSPINLOCK hSpinlock;
618	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
619
620	Assert(idCpu < pTimer->cCpus);
621
622	/*
623	* Some initial paranoia.
624	*/
625	if (pTimer->u32Magic != RTTIMER_MAGIC)
626	return;
627	hSpinlock = pTimer->hSpinlock;
628	if (hSpinlock == NIL_RTSPINLOCK)
629	return;
630
631	RTSpinlockAcquire(hSpinlock, &Tmp);
632
633	/* Is it active? */
634	if ( !ASMAtomicUoReadBool(&pTimer->fSuspended)
635	&& pTimer->u32Magic == RTTIMER_MAGIC)
636	{
637	switch (enmEvent)
638	{
639	/*
640	* Try do it without leaving the spin lock, but if we have to, retake it
641	* when we're on the right cpu.
642	*/
643	case RTMPEVENT_ONLINE:
644	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STARTING, RTTIMERLNXSTATE_STOPPED))
645	{
646	RTTIMERLINUXSTARTONCPUARGS Args;
647	Args.u64Now = RTTimeNanoTS();
648	Args.u64First = 0;
649
650	if (RTMpCpuId() == idCpu)
651	rtTimerLnxStartSubTimer(pSubTimer, Args.u64Now, Args.u64First);
652	else
653	{
654	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED); /* we'll recheck it. */
655	RTSpinlockRelease(hSpinlock, &Tmp);
656
657	RTMpOnSpecific(idCpu, rtTimerLinuxMpStartOnCpu, pTimer, &Args);
658	return; /* we've left the spinlock */
659	}
660	}
661	break;
662
663	/*
664	* The CPU is (going) offline, make sure the sub-timer is stopped.
665	*
666	* Linux will migrate it to a different CPU, but we don't want this. The
667	* timer function is checking for this.
668	*/
669	case RTMPEVENT_OFFLINE:
670	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STOPPING, RTTIMERLNXSTATE_ACTIVE))
671	{
672	RTSpinlockRelease(hSpinlock, &Tmp);
673
674	rtTimerLnxStopSubTimer(pSubTimer);
675	return; /* we've left the spinlock */
676	}
677	break;
678	}
679	}
680
681	RTSpinlockRelease(hSpinlock, &Tmp);
682	}
683
684	#endif /* CONFIG_SMP */
685
686
687	/**
688	* Callback function use by RTTimerStart via RTMpOnSpecific to start
689	* a timer running on a specific CPU.
690	*
691	* @param idCpu The current CPU.
692	* @param pvUser1 Pointer to the timer.
693	* @param pvUser2 Pointer to the argument structure.
694	*/
695	static DECLCALLBACK(void) rtTimerLnxStartOnSpecificCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
696	{
697	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
698	PRTTIMER pTimer = (PRTTIMER)pvUser1;
699	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[0], pArgs->u64Now, pArgs->u64First);
700	}
701
702
703	RTDECL(int) RTTimerStart(PRTTIMER pTimer, uint64_t u64First)
704	{
705	RTTIMERLINUXSTARTONCPUARGS Args;
706	int rc2;
707
708	/*
709	* Validate.
710	*/
711	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
712	AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
713
714	if (!ASMAtomicUoReadBool(&pTimer->fSuspended))
715	return VERR_TIMER_ACTIVE;
716
717	Args.u64First = u64First;
718	#ifdef CONFIG_SMP
719	/*
720	* Omnit timer?
721	*/
722	if (pTimer->fAllCpus)
723	return rtTimerLnxStartAll(pTimer, &Args);
724	#endif
725
726	/*
727	* Simple timer - Pretty straight forward.
728	*/
729	Args.u64Now = RTTimeNanoTS();
730	rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STARTING);
731	ASMAtomicWriteBool(&pTimer->fSuspended, false);
732	if (!pTimer->fSpecificCpu)
733	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[0], Args.u64Now, Args.u64First);
734	else
735	{
736	rc2 = RTMpOnSpecific(pTimer->idCpu, rtTimerLnxStartOnSpecificCpu, pTimer, &Args);
737	if (RT_FAILURE(rc2))
738	{
739	/* Suspend it, the cpu id is probably invalid or offline. */
740	ASMAtomicWriteBool(&pTimer->fSuspended, true);
741	rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STOPPED);
742	return rc2;
743	}
744	}
745
746	return VINF_SUCCESS;
747	}
748	RT_EXPORT_SYMBOL(RTTimerStart);
749
750
751	RTDECL(int) RTTimerStop(PRTTIMER pTimer)
752	{
753
754	/*
755	* Validate.
756	*/
757	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
758	AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
759
760	if (ASMAtomicUoReadBool(&pTimer->fSuspended))
761	return VERR_TIMER_SUSPENDED;
762
763	#ifdef CONFIG_SMP
764	/*
765	* Omni timer?
766	*/
767	if (pTimer->fAllCpus)
768	return rtTimerLnxStopAll(pTimer);
769	#endif
770
771	/*
772	* Simple timer.
773	*/
774	ASMAtomicWriteBool(&pTimer->fSuspended, true);
775	rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STOPPING);
776	rtTimerLnxStopSubTimer(&pTimer->aSubTimers[0]);
777
778	return VINF_SUCCESS;
779	}
780	RT_EXPORT_SYMBOL(RTTimerStop);
781
782
783	RTDECL(int) RTTimerDestroy(PRTTIMER pTimer)
784	{
785	RTSPINLOCK hSpinlock;
786
787	/* It's ok to pass NULL pointer. */
788	if (pTimer == /NIL_RTTIMER/ NULL)
789	return VINF_SUCCESS;
790	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
791	AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
792
793	/*
794	* Remove the MP notifications first because it'll reduce the risk of
795	* us overtaking any MP event that might theoretically be racing us here.
796	*/
797	hSpinlock = pTimer->hSpinlock;
798	#ifdef CONFIG_SMP
799	if ( pTimer->cCpus > 1
800	&& hSpinlock != NIL_RTSPINLOCK)
801	{
802	int rc = RTMpNotificationDeregister(rtTimerLinuxMpEvent, pTimer);
803	AssertRC(rc);
804	}
805	#endif /* CONFIG_SMP */
806
807	/*
808	* Stop the timer if it's running.
809	*/
810	if (!ASMAtomicUoReadBool(&pTimer->fSuspended))
811	RTTimerStop(pTimer);
812
813	/*
814	* Uninitialize the structure and free the associated resources.
815	* The spinlock goes last.
816	*/
817	ASMAtomicWriteU32(&pTimer->u32Magic, ~RTTIMER_MAGIC);
818	RTMemFree(pTimer);
819	if (hSpinlock != NIL_RTSPINLOCK)
820	RTSpinlockDestroy(hSpinlock);
821
822	return VINF_SUCCESS;
823	}
824	RT_EXPORT_SYMBOL(RTTimerDestroy);
825
826
827	RTDECL(int) RTTimerCreateEx(PRTTIMER ppTimer, uint64_t u64NanoInterval, unsigned fFlags, PFNRTTIMER pfnTimer, void pvUser)
828	{
829	PRTTIMER pTimer;
830	RTCPUID iCpu;
831	unsigned cCpus;
832
833	*ppTimer = NULL;
834
835	/*
836	* Validate flags.
837	*/
838	if (!RTTIMER_FLAGS_ARE_VALID(fFlags))
839	return VERR_INVALID_PARAMETER;
840	if ( (fFlags & RTTIMER_FLAGS_CPU_SPECIFIC)
841	&& (fFlags & RTTIMER_FLAGS_CPU_ALL) != RTTIMER_FLAGS_CPU_ALL
842	&& !RTMpIsCpuOnline(fFlags & RTTIMER_FLAGS_CPU_MASK))
843	return (fFlags & RTTIMER_FLAGS_CPU_MASK) > RTMpGetMaxCpuId()
844	? VERR_CPU_NOT_FOUND
845	: VERR_CPU_OFFLINE;
846
847	/*
848	* Allocate the timer handler.
849	*/
850	cCpus = 1;
851	#ifdef CONFIG_SMP
852	if ((fFlags & RTTIMER_FLAGS_CPU_ALL) == RTTIMER_FLAGS_CPU_ALL)
853	{
854	cCpus = RTMpGetMaxCpuId() + 1;
855	Assert(cCpus <= RTCPUSET_MAX_CPUS); /* On linux we have a 1:1 relationship between cpuid and set index. */
856	AssertReturn(u64NanoInterval, VERR_NOT_IMPLEMENTED); /* We don't implement single shot on all cpus, sorry. */
857	}
858	#endif
859
860	pTimer = (PRTTIMER)RTMemAllocZ(RT_OFFSETOF(RTTIMER, aSubTimers[cCpus]));
861	if (!pTimer)
862	return VERR_NO_MEMORY;
863
864	/*
865	* Initialize it.
866	*/
867	pTimer->u32Magic = RTTIMER_MAGIC;
868	pTimer->hSpinlock = NIL_RTSPINLOCK;
869	pTimer->fSuspended = true;
870	#ifdef CONFIG_SMP
871	pTimer->fSpecificCpu = (fFlags & RTTIMER_FLAGS_CPU_SPECIFIC) && (fFlags & RTTIMER_FLAGS_CPU_ALL) != RTTIMER_FLAGS_CPU_ALL;
872	pTimer->fAllCpus = (fFlags & RTTIMER_FLAGS_CPU_ALL) == RTTIMER_FLAGS_CPU_ALL;
873	pTimer->idCpu = fFlags & RTTIMER_FLAGS_CPU_MASK;
874	#else
875	pTimer->fSpecificCpu = !!(fFlags & RTTIMER_FLAGS_CPU_SPECIFIC);
876	pTimer->idCpu = RTMpCpuId();
877	#endif
878	pTimer->cCpus = cCpus;
879	pTimer->pfnTimer = pfnTimer;
880	pTimer->pvUser = pvUser;
881	pTimer->u64NanoInterval = u64NanoInterval;
882	#ifndef RT_USE_LINUX_HRTIMER
883	pTimer->cJiffies = u64NanoInterval / RTTimerGetSystemGranularity();
884	if (pTimer->cJiffies * RTTimerGetSystemGranularity() != u64NanoInterval)
885	pTimer->cJiffies = 0;
886	#endif
887
888	for (iCpu = 0; iCpu < cCpus; iCpu++)
889	{
890	#ifdef RT_USE_LINUX_HRTIMER
891	hrtimer_init(&pTimer->aSubTimers[iCpu].LnxTimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
892	pTimer->aSubTimers[iCpu].LnxTimer.function = rtTimerLinuxCallback;
893	#else
894	init_timer(&pTimer->aSubTimers[iCpu].LnxTimer);
895	pTimer->aSubTimers[iCpu].LnxTimer.data = (unsigned long)&pTimer->aSubTimers[iCpu];
896	pTimer->aSubTimers[iCpu].LnxTimer.function = rtTimerLinuxCallback;
897	pTimer->aSubTimers[iCpu].LnxTimer.expires = jiffies;
898	#endif
899	pTimer->aSubTimers[iCpu].u64StartTS = 0;
900	pTimer->aSubTimers[iCpu].u64NextTS = 0;
901	pTimer->aSubTimers[iCpu].iTick = 0;
902	pTimer->aSubTimers[iCpu].pParent = pTimer;
903	pTimer->aSubTimers[iCpu].enmState = RTTIMERLNXSTATE_STOPPED;
904	}
905
906	#ifdef CONFIG_SMP
907	/*
908	* If this is running on ALL cpus, we'll have to register a callback
909	* for MP events (so timers can be started/stopped on cpus going
910	* online/offline). We also create the spinlock for syncrhonizing
911	* stop/start/mp-event.
912	*/
913	if (cCpus > 1)
914	{
915	int rc = RTSpinlockCreate(&pTimer->hSpinlock);
916	if (RT_SUCCESS(rc))
917	rc = RTMpNotificationRegister(rtTimerLinuxMpEvent, pTimer);
918	else
919	pTimer->hSpinlock = NIL_RTSPINLOCK;
920	if (RT_FAILURE(rc))
921	{
922	RTTimerDestroy(pTimer);
923	return rc;
924	}
925	}
926	#endif /* CONFIG_SMP */
927
928	*ppTimer = pTimer;
929	return VINF_SUCCESS;
930	}
931	RT_EXPORT_SYMBOL(RTTimerCreateEx);
932
933
934	RTDECL(uint32_t) RTTimerGetSystemGranularity(void)
935	{
936	#ifdef RT_USE_LINUX_HRTIMER
937	struct timespec Ts;
938	int rc = hrtimer_get_res(CLOCK_MONOTONIC, &Ts);
939	if (!rc)
940	{
941	Assert(!Ts.tv_sec);
942	return Ts.tv_nsec;
943	}
944	#endif
945	return 1000000000 / HZ; /* ns */
946	}
947	RT_EXPORT_SYMBOL(RTTimerGetSystemGranularity);
948
949
950	RTDECL(int) RTTimerRequestSystemGranularity(uint32_t u32Request, uint32_t *pu32Granted)
951	{
952	return VERR_NOT_SUPPORTED;
953	}
954	RT_EXPORT_SYMBOL(RTTimerRequestSystemGranularity);
955
956
957	RTDECL(int) RTTimerReleaseSystemGranularity(uint32_t u32Granted)
958	{
959	return VERR_NOT_SUPPORTED;
960	}
961	RT_EXPORT_SYMBOL(RTTimerReleaseSystemGranularity);
962

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/linux/timer-r0drv-linux.c@ 24696

Download in other formats: