VirtualBox

source: vbox/trunk/src/VBox/Runtime/generic/critsectrw-generic.cpp@ 58829

Last change on this file since 58829 was 58829, checked in by vboxsync, 9 years ago

IPRT: Make read-write critical sections available to ring-0 code.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 32.6 KB
Line 
1/* $Id: critsectrw-generic.cpp 58829 2015-11-23 17:13:18Z vboxsync $ */
2/** @file
3 * IPRT - Read/Write Critical Section, Generic.
4 */
5
6/*
7 * Copyright (C) 2009-2015 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * The contents of this file may alternatively be used under the terms
18 * of the Common Development and Distribution License Version 1.0
19 * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20 * VirtualBox OSE distribution, in which case the provisions of the
21 * CDDL are applicable instead of those of the GPL.
22 *
23 * You may elect to license modified versions of this file under the
24 * terms and conditions of either the GPL or the CDDL or both.
25 */
26
27
28/*********************************************************************************************************************************
29* Header Files *
30*********************************************************************************************************************************/
31#define RTCRITSECTRW_WITHOUT_REMAPPING
32#define RTASSERT_QUIET
33#include <iprt/critsect.h>
34#include "internal/iprt.h"
35
36#include <iprt/asm.h>
37#include <iprt/assert.h>
38#include <iprt/err.h>
39#include <iprt/lockvalidator.h>
40#include <iprt/mem.h>
41#include <iprt/semaphore.h>
42#include <iprt/thread.h>
43
44#include "internal/magics.h"
45#include "internal/strict.h"
46
47
48
49RTDECL(int) RTCritSectRwInit(PRTCRITSECTRW pThis)
50{
51 return RTCritSectRwInitEx(pThis, 0, NIL_RTLOCKVALCLASS, RTLOCKVAL_SUB_CLASS_NONE, "RTCritSectRw");
52}
53RT_EXPORT_SYMBOL(RTCritSectRwInit);
54
55
56RTDECL(int) RTCritSectRwInitEx(PRTCRITSECTRW pThis, uint32_t fFlags,
57 RTLOCKVALCLASS hClass, uint32_t uSubClass, const char *pszNameFmt, ...)
58{
59 int rc;
60 AssertReturn(!(fFlags & ~( RTCRITSECT_FLAGS_NO_NESTING | RTCRITSECT_FLAGS_NO_LOCK_VAL | RTCRITSECT_FLAGS_BOOTSTRAP_HACK
61 | RTCRITSECT_FLAGS_NOP )),
62 VERR_INVALID_PARAMETER);
63
64 /*
65 * Initialize the structure, allocate the lock validator stuff and sems.
66 */
67 pThis->u32Magic = RTCRITSECTRW_MAGIC_DEAD;
68 pThis->fNeedReset = false;
69#ifdef IN_RING0
70 pThis->fFlags = (uint16_t)(fFlags | RTCRITSECT_FLAGS_RING0);
71#else
72 pThis->fFlags = (uint16_t)(fFlags & ~RTCRITSECT_FLAGS_RING0);
73#endif
74 pThis->u64State = 0;
75 pThis->hNativeWriter = NIL_RTNATIVETHREAD;
76 pThis->cWriterReads = 0;
77 pThis->cWriteRecursions = 0;
78 pThis->hEvtWrite = NIL_RTSEMEVENT;
79 pThis->hEvtRead = NIL_RTSEMEVENTMULTI;
80 pThis->pValidatorWrite = NULL;
81 pThis->pValidatorRead = NULL;
82#if HC_ARCH_BITS == 32
83 pThis->HCPtrPadding = NIL_RTHCPTR;
84#endif
85
86#ifdef RTCRITSECTRW_STRICT
87 bool const fLVEnabled = !(fFlags & RTCRITSECT_FLAGS_NO_LOCK_VAL);
88 if (!pszNameFmt)
89 {
90 static uint32_t volatile s_iAnon = 0;
91 uint32_t i = ASMAtomicIncU32(&s_iAnon) - 1;
92 rc = RTLockValidatorRecExclCreate(&pThis->pValidatorWrite, hClass, uSubClass, pThis,
93 fLVEnabled, "RTCritSectRw-%u", i);
94 if (RT_SUCCESS(rc))
95 rc = RTLockValidatorRecSharedCreate(&pThis->pValidatorRead, hClass, uSubClass, pThis,
96 false /*fSignaller*/, fLVEnabled, "RTCritSectRw-%u", i);
97 }
98 else
99 {
100 va_list va;
101 va_start(va, pszNameFmt);
102 rc = RTLockValidatorRecExclCreateV(&pThis->pValidatorWrite, hClass, uSubClass, pThis,
103 fLVEnabled, pszNameFmt, va);
104 va_end(va);
105 if (RT_SUCCESS(rc))
106 {
107 va_start(va, pszNameFmt);
108 RTLockValidatorRecSharedCreateV(&pThis->pValidatorRead, hClass, uSubClass, pThis,
109 false /*fSignaller*/, fLVEnabled, pszNameFmt, va);
110 va_end(va);
111 }
112 }
113 if (RT_SUCCESS(rc))
114 rc = RTLockValidatorRecMakeSiblings(&pThis->pValidatorWrite->Core, &pThis->pValidatorRead->Core);
115
116 if (RT_SUCCESS(rc))
117#endif
118 {
119 rc = RTSemEventMultiCreate(&pThis->hEvtRead);
120 if (RT_SUCCESS(rc))
121 {
122 rc = RTSemEventCreate(&pThis->hEvtWrite);
123 if (RT_SUCCESS(rc))
124 {
125 pThis->u32Magic = RTCRITSECTRW_MAGIC;
126 return VINF_SUCCESS;
127 }
128 RTSemEventMultiDestroy(pThis->hEvtRead);
129 }
130 }
131
132#ifdef RTCRITSECTRW_STRICT
133 RTLockValidatorRecSharedDestroy(&pThis->pValidatorRead);
134 RTLockValidatorRecExclDestroy(&pThis->pValidatorWrite);
135#endif
136 return rc;
137}
138RT_EXPORT_SYMBOL(RTCritSectRwInitEx);
139
140
141RTDECL(uint32_t) RTCritSectRwSetSubClass(PRTCRITSECTRW pThis, uint32_t uSubClass)
142{
143 AssertPtrReturn(pThis, RTLOCKVAL_SUB_CLASS_INVALID);
144 AssertReturn(pThis->u32Magic == RTCRITSECTRW_MAGIC, RTLOCKVAL_SUB_CLASS_INVALID);
145#ifdef IN_RING0
146 Assert(pThis->fFlags & RTCRITSECT_FLAGS_RING0);
147#else
148 Assert(!(pThis->fFlags & RTCRITSECT_FLAGS_RING0));
149#endif
150#ifdef RTCRITSECTRW_STRICT
151 AssertReturn(!(pThis->fFlags & RTCRITSECT_FLAGS_NOP), RTLOCKVAL_SUB_CLASS_INVALID);
152
153 RTLockValidatorRecSharedSetSubClass(pThis->pValidatorRead, uSubClass);
154 return RTLockValidatorRecExclSetSubClass(pThis->pValidatorWrite, uSubClass);
155#else
156 NOREF(uSubClass);
157 return RTLOCKVAL_SUB_CLASS_INVALID;
158#endif
159}
160RT_EXPORT_SYMBOL(RTCritSectRwSetSubClass);
161
162
163static int rtCritSectRwEnterShared(PRTCRITSECTRW pThis, PCRTLOCKVALSRCPOS pSrcPos, bool fTryOnly)
164{
165 /*
166 * Validate input.
167 */
168 AssertPtr(pThis);
169 AssertReturn(pThis->u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
170#ifdef IN_RING0
171 Assert(pThis->fFlags & RTCRITSECT_FLAGS_RING0);
172#else
173 Assert(!(pThis->fFlags & RTCRITSECT_FLAGS_RING0));
174#endif
175
176#ifdef RTCRITSECTRW_STRICT
177 RTTHREAD hThreadSelf = RTThreadSelfAutoAdopt();
178 if (!fTryOnly)
179 {
180 int rc9;
181 RTNATIVETHREAD hNativeWriter;
182 ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter);
183 if (hNativeWriter != NIL_RTTHREAD && hNativeWriter == RTThreadNativeSelf())
184 rc9 = RTLockValidatorRecExclCheckOrder(pThis->pValidatorWrite, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT);
185 else
186 rc9 = RTLockValidatorRecSharedCheckOrder(pThis->pValidatorRead, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT);
187 if (RT_FAILURE(rc9))
188 return rc9;
189 }
190#endif
191
192 /*
193 * Get cracking...
194 */
195 uint64_t u64State = ASMAtomicReadU64(&pThis->u64State);
196 uint64_t u64OldState = u64State;
197
198 for (;;)
199 {
200 if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT))
201 {
202 /* It flows in the right direction, try follow it before it changes. */
203 uint64_t c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT;
204 c++;
205 Assert(c < RTCSRW_CNT_MASK / 2);
206 u64State &= ~RTCSRW_CNT_RD_MASK;
207 u64State |= c << RTCSRW_CNT_RD_SHIFT;
208 if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
209 {
210#ifdef RTCRITSECTRW_STRICT
211 RTLockValidatorRecSharedAddOwner(pThis->pValidatorRead, hThreadSelf, pSrcPos);
212#endif
213 break;
214 }
215 }
216 else if ((u64State & (RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK)) == 0)
217 {
218 /* Wrong direction, but we're alone here and can simply try switch the direction. */
219 u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK | RTCSRW_DIR_MASK);
220 u64State |= (UINT64_C(1) << RTCSRW_CNT_RD_SHIFT) | (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT);
221 if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
222 {
223 Assert(!pThis->fNeedReset);
224#ifdef RTCRITSECTRW_STRICT
225 RTLockValidatorRecSharedAddOwner(pThis->pValidatorRead, hThreadSelf, pSrcPos);
226#endif
227 break;
228 }
229 }
230 else
231 {
232 /* Is the writer perhaps doing a read recursion? */
233 RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf();
234 RTNATIVETHREAD hNativeWriter;
235 ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter);
236 if (hNativeSelf == hNativeWriter)
237 {
238#ifdef RTCRITSECTRW_STRICT
239 int rc9 = RTLockValidatorRecExclRecursionMixed(pThis->pValidatorWrite, &pThis->pValidatorRead->Core, pSrcPos);
240 if (RT_FAILURE(rc9))
241 return rc9;
242#endif
243 Assert(pThis->cWriterReads < UINT32_MAX / 2);
244 ASMAtomicIncU32(&pThis->cWriterReads);
245 return VINF_SUCCESS; /* don't break! */
246 }
247
248 /* If we're only trying, return already. */
249 if (fTryOnly)
250 return VERR_SEM_BUSY;
251
252 /* Add ourselves to the queue and wait for the direction to change. */
253 uint64_t c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT;
254 c++;
255 Assert(c < RTCSRW_CNT_MASK / 2);
256
257 uint64_t cWait = (u64State & RTCSRW_WAIT_CNT_RD_MASK) >> RTCSRW_WAIT_CNT_RD_SHIFT;
258 cWait++;
259 Assert(cWait <= c);
260 Assert(cWait < RTCSRW_CNT_MASK / 2);
261
262 u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_WAIT_CNT_RD_MASK);
263 u64State |= (c << RTCSRW_CNT_RD_SHIFT) | (cWait << RTCSRW_WAIT_CNT_RD_SHIFT);
264
265 if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
266 {
267 for (uint32_t iLoop = 0; ; iLoop++)
268 {
269 int rc;
270#ifdef RTCRITSECTRW_STRICT
271 rc = RTLockValidatorRecSharedCheckBlocking(pThis->pValidatorRead, hThreadSelf, pSrcPos, true,
272 RT_INDEFINITE_WAIT, RTTHREADSTATE_RW_READ, false);
273 if (RT_SUCCESS(rc))
274#elif defined(IN_RING3)
275 RTTHREAD hThreadSelf = RTThreadSelf();
276 RTThreadBlocking(hThreadSelf, RTTHREADSTATE_RW_READ, false);
277#endif
278 {
279 rc = RTSemEventMultiWait(pThis->hEvtRead, RT_INDEFINITE_WAIT);
280#ifdef IN_RING3
281 RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_READ);
282#endif
283 if (pThis->u32Magic != RTCRITSECTRW_MAGIC)
284 return VERR_SEM_DESTROYED;
285 }
286 if (RT_FAILURE(rc))
287 {
288 /* Decrement the counts and return the error. */
289 for (;;)
290 {
291 u64OldState = u64State = ASMAtomicReadU64(&pThis->u64State);
292 c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT; Assert(c > 0);
293 c--;
294 cWait = (u64State & RTCSRW_WAIT_CNT_RD_MASK) >> RTCSRW_WAIT_CNT_RD_SHIFT; Assert(cWait > 0);
295 cWait--;
296 u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_WAIT_CNT_RD_MASK);
297 u64State |= (c << RTCSRW_CNT_RD_SHIFT) | (cWait << RTCSRW_WAIT_CNT_RD_SHIFT);
298 if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
299 break;
300 }
301 return rc;
302 }
303
304 Assert(pThis->fNeedReset);
305 u64State = ASMAtomicReadU64(&pThis->u64State);
306 if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT))
307 break;
308 AssertMsg(iLoop < 1, ("%u\n", iLoop));
309 }
310
311 /* Decrement the wait count and maybe reset the semaphore (if we're last). */
312 for (;;)
313 {
314 u64OldState = u64State;
315
316 cWait = (u64State & RTCSRW_WAIT_CNT_RD_MASK) >> RTCSRW_WAIT_CNT_RD_SHIFT;
317 Assert(cWait > 0);
318 cWait--;
319 u64State &= ~RTCSRW_WAIT_CNT_RD_MASK;
320 u64State |= cWait << RTCSRW_WAIT_CNT_RD_SHIFT;
321
322 if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
323 {
324 if (cWait == 0)
325 {
326 if (ASMAtomicXchgBool(&pThis->fNeedReset, false))
327 {
328 int rc = RTSemEventMultiReset(pThis->hEvtRead);
329 AssertRCReturn(rc, rc);
330 }
331 }
332 break;
333 }
334 u64State = ASMAtomicReadU64(&pThis->u64State);
335 }
336
337#ifdef RTCRITSECTRW_STRICT
338 RTLockValidatorRecSharedAddOwner(pThis->pValidatorRead, hThreadSelf, pSrcPos);
339#endif
340 break;
341 }
342 }
343
344 if (pThis->u32Magic != RTCRITSECTRW_MAGIC)
345 return VERR_SEM_DESTROYED;
346
347 ASMNopPause();
348 u64State = ASMAtomicReadU64(&pThis->u64State);
349 u64OldState = u64State;
350 }
351
352 /* got it! */
353 Assert((ASMAtomicReadU64(&pThis->u64State) & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT));
354 return VINF_SUCCESS;
355
356}
357
358
359RTDECL(int) RTCritSectRwEnterShared(PRTCRITSECTRW pThis)
360{
361#ifndef RTCRITSECTRW_STRICT
362 return rtCritSectRwEnterShared(pThis, NULL, false /*fTryOnly*/);
363#else
364 RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
365 return rtCritSectRwEnterShared(pThis, &SrcPos, false /*fTryOnly*/);
366#endif
367}
368RT_EXPORT_SYMBOL(RTCritSectRwEnterShared);
369
370
371RTDECL(int) RTCritSectRwEnterSharedDebug(PRTCRITSECTRW pThis, RTHCUINTPTR uId, RT_SRC_POS_DECL)
372{
373 RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
374 return rtCritSectRwEnterShared(pThis, &SrcPos, false /*fTryOnly*/);
375}
376RT_EXPORT_SYMBOL(RTCritSectRwEnterSharedDebug);
377
378
379RTDECL(int) RTCritSectRwTryEnterShared(PRTCRITSECTRW pThis)
380{
381#ifndef RTCRITSECTRW_STRICT
382 return rtCritSectRwEnterShared(pThis, NULL, true /*fTryOnly*/);
383#else
384 RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
385 return rtCritSectRwEnterShared(pThis, &SrcPos, true /*fTryOnly*/);
386#endif
387}
388RT_EXPORT_SYMBOL(RTCritSectRwEnterShared);
389
390
391RTDECL(int) RTCritSectRwTryEnterSharedDebug(PRTCRITSECTRW pThis, RTHCUINTPTR uId, RT_SRC_POS_DECL)
392{
393 RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
394 return rtCritSectRwEnterShared(pThis, &SrcPos, true /*fTryOnly*/);
395}
396RT_EXPORT_SYMBOL(RTCritSectRwEnterSharedDebug);
397
398
399
400RTDECL(int) RTCritSectRwLeaveShared(PRTCRITSECTRW pThis)
401{
402 /*
403 * Validate handle.
404 */
405 AssertPtr(pThis);
406 AssertReturn(pThis->u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
407#ifdef IN_RING0
408 Assert(pThis->fFlags & RTCRITSECT_FLAGS_RING0);
409#else
410 Assert(!(pThis->fFlags & RTCRITSECT_FLAGS_RING0));
411#endif
412
413 /*
414 * Check the direction and take action accordingly.
415 */
416 uint64_t u64State = ASMAtomicReadU64(&pThis->u64State);
417 uint64_t u64OldState = u64State;
418 if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT))
419 {
420#ifdef RTCRITSECTRW_STRICT
421 int rc9 = RTLockValidatorRecSharedCheckAndRelease(pThis->pValidatorRead, NIL_RTTHREAD);
422 if (RT_FAILURE(rc9))
423 return rc9;
424#endif
425 for (;;)
426 {
427 uint64_t c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT;
428 AssertReturn(c > 0, VERR_NOT_OWNER);
429 c--;
430
431 if ( c > 0
432 || (u64State & RTCSRW_CNT_WR_MASK) == 0)
433 {
434 /* Don't change the direction. */
435 u64State &= ~RTCSRW_CNT_RD_MASK;
436 u64State |= c << RTCSRW_CNT_RD_SHIFT;
437 if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
438 break;
439 }
440 else
441 {
442 /* Reverse the direction and signal the reader threads. */
443 u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_DIR_MASK);
444 u64State |= RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT;
445 if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
446 {
447 int rc = RTSemEventSignal(pThis->hEvtWrite);
448 AssertRC(rc);
449 break;
450 }
451 }
452
453 ASMNopPause();
454 u64State = ASMAtomicReadU64(&pThis->u64State);
455 u64OldState = u64State;
456 }
457 }
458 else
459 {
460 RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf();
461 RTNATIVETHREAD hNativeWriter;
462 ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter);
463 AssertReturn(hNativeSelf == hNativeWriter, VERR_NOT_OWNER);
464 AssertReturn(pThis->cWriterReads > 0, VERR_NOT_OWNER);
465#ifdef RTCRITSECTRW_STRICT
466 int rc = RTLockValidatorRecExclUnwindMixed(pThis->pValidatorWrite, &pThis->pValidatorRead->Core);
467 if (RT_FAILURE(rc))
468 return rc;
469#endif
470 ASMAtomicDecU32(&pThis->cWriterReads);
471 }
472
473 return VINF_SUCCESS;
474}
475RT_EXPORT_SYMBOL(RTCritSectRwLeaveShared);
476
477
478static int rtCritSectRwEnterExcl(PRTCRITSECTRW pThis, PCRTLOCKVALSRCPOS pSrcPos, bool fTryOnly)
479{
480 /*
481 * Validate input.
482 */
483 AssertPtr(pThis);
484 AssertReturn(pThis->u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
485#ifdef IN_RING0
486 Assert(pThis->fFlags & RTCRITSECT_FLAGS_RING0);
487#else
488 Assert(!(pThis->fFlags & RTCRITSECT_FLAGS_RING0));
489#endif
490
491#ifdef RTCRITSECTRW_STRICT
492 RTTHREAD hThreadSelf = NIL_RTTHREAD;
493 if (!fTryOnly)
494 {
495 hThreadSelf = RTThreadSelfAutoAdopt();
496 int rc9 = RTLockValidatorRecExclCheckOrder(pThis->pValidatorWrite, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT);
497 if (RT_FAILURE(rc9))
498 return rc9;
499 }
500#endif
501
502 /*
503 * Check if we're already the owner and just recursing.
504 */
505 RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf();
506 RTNATIVETHREAD hNativeWriter;
507 ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter);
508 if (hNativeSelf == hNativeWriter)
509 {
510 Assert((ASMAtomicReadU64(&pThis->u64State) & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT));
511#ifdef RTCRITSECTRW_STRICT
512 int rc9 = RTLockValidatorRecExclRecursion(pThis->pValidatorWrite, pSrcPos);
513 if (RT_FAILURE(rc9))
514 return rc9;
515#endif
516 Assert(pThis->cWriteRecursions < UINT32_MAX / 2);
517 ASMAtomicIncU32(&pThis->cWriteRecursions);
518 return VINF_SUCCESS;
519 }
520
521 /*
522 * Get cracking.
523 */
524 uint64_t u64State = ASMAtomicReadU64(&pThis->u64State);
525 uint64_t u64OldState = u64State;
526
527 for (;;)
528 {
529 if ( (u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT)
530 || (u64State & (RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK)) != 0)
531 {
532 /* It flows in the right direction, try follow it before it changes. */
533 uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT;
534 c++;
535 Assert(c < RTCSRW_CNT_MASK / 2);
536 u64State &= ~RTCSRW_CNT_WR_MASK;
537 u64State |= c << RTCSRW_CNT_WR_SHIFT;
538 if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
539 break;
540 }
541 else if ((u64State & (RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK)) == 0)
542 {
543 /* Wrong direction, but we're alone here and can simply try switch the direction. */
544 u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK | RTCSRW_DIR_MASK);
545 u64State |= (UINT64_C(1) << RTCSRW_CNT_WR_SHIFT) | (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT);
546 if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
547 break;
548 }
549 else if (fTryOnly)
550 /* Wrong direction and we're not supposed to wait, just return. */
551 return VERR_SEM_BUSY;
552 else
553 {
554 /* Add ourselves to the write count and break out to do the wait. */
555 uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT;
556 c++;
557 Assert(c < RTCSRW_CNT_MASK / 2);
558 u64State &= ~RTCSRW_CNT_WR_MASK;
559 u64State |= c << RTCSRW_CNT_WR_SHIFT;
560 if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
561 break;
562 }
563
564 if (pThis->u32Magic != RTCRITSECTRW_MAGIC)
565 return VERR_SEM_DESTROYED;
566
567 ASMNopPause();
568 u64State = ASMAtomicReadU64(&pThis->u64State);
569 u64OldState = u64State;
570 }
571
572 /*
573 * If we're in write mode now try grab the ownership. Play fair if there
574 * are threads already waiting.
575 */
576 bool fDone = (u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT)
577 && ( ((u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT) == 1
578 || fTryOnly);
579 if (fDone)
580 ASMAtomicCmpXchgHandle(&pThis->hNativeWriter, hNativeSelf, NIL_RTNATIVETHREAD, fDone);
581 if (!fDone)
582 {
583 /*
584 * If only trying, undo the above writer incrementation and return.
585 */
586 if (fTryOnly)
587 {
588 for (;;)
589 {
590 u64OldState = u64State = ASMAtomicReadU64(&pThis->u64State);
591 uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT; Assert(c > 0);
592 c--;
593 u64State &= ~RTCSRW_CNT_WR_MASK;
594 u64State |= c << RTCSRW_CNT_WR_SHIFT;
595 if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
596 break;
597 }
598 return VERR_SEM_BUSY;
599 }
600
601 /*
602 * Wait for our turn.
603 */
604 for (uint32_t iLoop = 0; ; iLoop++)
605 {
606 int rc;
607#ifdef RTCRITSECTRW_STRICT
608 if (hThreadSelf == NIL_RTTHREAD)
609 hThreadSelf = RTThreadSelfAutoAdopt();
610 rc = RTLockValidatorRecExclCheckBlocking(pThis->pValidatorWrite, hThreadSelf, pSrcPos, true,
611 RT_INDEFINITE_WAIT, RTTHREADSTATE_RW_WRITE, false);
612 if (RT_SUCCESS(rc))
613#elif defined(IN_RING3)
614 RTTHREAD hThreadSelf = RTThreadSelf();
615 RTThreadBlocking(hThreadSelf, RTTHREADSTATE_RW_WRITE, false);
616#endif
617 {
618 rc = RTSemEventWait(pThis->hEvtWrite, RT_INDEFINITE_WAIT);
619#ifdef IN_RING3
620 RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE);
621#endif
622 if (pThis->u32Magic != RTCRITSECTRW_MAGIC)
623 return VERR_SEM_DESTROYED;
624 }
625 if (RT_FAILURE(rc))
626 {
627 /* Decrement the counts and return the error. */
628 for (;;)
629 {
630 u64OldState = u64State = ASMAtomicReadU64(&pThis->u64State);
631 uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT; Assert(c > 0);
632 c--;
633 u64State &= ~RTCSRW_CNT_WR_MASK;
634 u64State |= c << RTCSRW_CNT_WR_SHIFT;
635 if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
636 break;
637 }
638 return rc;
639 }
640
641 u64State = ASMAtomicReadU64(&pThis->u64State);
642 if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT))
643 {
644 ASMAtomicCmpXchgHandle(&pThis->hNativeWriter, hNativeSelf, NIL_RTNATIVETHREAD, fDone);
645 if (fDone)
646 break;
647 }
648 AssertMsg(iLoop < 1000, ("%u\n", iLoop)); /* may loop a few times here... */
649 }
650 }
651
652 /*
653 * Got it!
654 */
655 Assert((ASMAtomicReadU64(&pThis->u64State) & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT));
656 ASMAtomicWriteU32(&pThis->cWriteRecursions, 1);
657 Assert(pThis->cWriterReads == 0);
658#ifdef RTCRITSECTRW_STRICT
659 RTLockValidatorRecExclSetOwner(pThis->pValidatorWrite, hThreadSelf, pSrcPos, true);
660#endif
661
662 return VINF_SUCCESS;
663}
664
665
666RTDECL(int) RTCritSectRwEnterExcl(PRTCRITSECTRW pThis)
667{
668#ifndef RTCRITSECTRW_STRICT
669 return rtCritSectRwEnterExcl(pThis, NULL, false /*fTryAgain*/);
670#else
671 RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
672 return rtCritSectRwEnterExcl(pThis, &SrcPos, false /*fTryAgain*/);
673#endif
674}
675RT_EXPORT_SYMBOL(RTCritSectRwEnterExcl);
676
677
678RTDECL(int) RTCritSectRwEnterExclDebug(PRTCRITSECTRW pThis, RTHCUINTPTR uId, RT_SRC_POS_DECL)
679{
680 RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
681 return rtCritSectRwEnterExcl(pThis, &SrcPos, false /*fTryAgain*/);
682}
683RT_EXPORT_SYMBOL(RTCritSectRwEnterExclDebug);
684
685
686RTDECL(int) RTCritSectRwTryEnterExcl(PRTCRITSECTRW pThis)
687{
688#ifndef RTCRITSECTRW_STRICT
689 return rtCritSectRwEnterExcl(pThis, NULL, true /*fTryAgain*/);
690#else
691 RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
692 return rtCritSectRwEnterExcl(pThis, &SrcPos, true /*fTryAgain*/);
693#endif
694}
695RT_EXPORT_SYMBOL(RTCritSectRwTryEnterExcl);
696
697
698RTDECL(int) RTCritSectRwTryEnterExclDebug(PRTCRITSECTRW pThis, RTHCUINTPTR uId, RT_SRC_POS_DECL)
699{
700 RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
701 return rtCritSectRwEnterExcl(pThis, &SrcPos, true /*fTryAgain*/);
702}
703RT_EXPORT_SYMBOL(RTCritSectRwTryEnterExclDebug);
704
705
706RTDECL(int) RTCritSectRwLeaveExcl(PRTCRITSECTRW pThis)
707{
708 /*
709 * Validate handle.
710 */
711 AssertPtr(pThis);
712 AssertReturn(pThis->u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
713#ifdef IN_RING0
714 Assert(pThis->fFlags & RTCRITSECT_FLAGS_RING0);
715#else
716 Assert(!(pThis->fFlags & RTCRITSECT_FLAGS_RING0));
717#endif
718
719 RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf();
720 RTNATIVETHREAD hNativeWriter;
721 ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter);
722 AssertReturn(hNativeSelf == hNativeWriter, VERR_NOT_OWNER);
723
724 /*
725 * Unwind a recursion.
726 */
727 if (pThis->cWriteRecursions == 1)
728 {
729 AssertReturn(pThis->cWriterReads == 0, VERR_WRONG_ORDER); /* (must release all read recursions before the final write.) */
730#ifdef RTCRITSECTRW_STRICT
731 int rc9 = RTLockValidatorRecExclReleaseOwner(pThis->pValidatorWrite, true);
732 if (RT_FAILURE(rc9))
733 return rc9;
734#endif
735 /*
736 * Update the state.
737 */
738 ASMAtomicWriteU32(&pThis->cWriteRecursions, 0);
739 ASMAtomicWriteHandle(&pThis->hNativeWriter, NIL_RTNATIVETHREAD);
740
741 for (;;)
742 {
743 uint64_t u64State = ASMAtomicReadU64(&pThis->u64State);
744 uint64_t u64OldState = u64State;
745
746 uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT;
747 Assert(c > 0);
748 c--;
749
750 if ( c > 0
751 || (u64State & RTCSRW_CNT_RD_MASK) == 0)
752 {
753 /* Don't change the direction, wait up the next writer if any. */
754 u64State &= ~RTCSRW_CNT_WR_MASK;
755 u64State |= c << RTCSRW_CNT_WR_SHIFT;
756 if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
757 {
758 if (c > 0)
759 {
760 int rc = RTSemEventSignal(pThis->hEvtWrite);
761 AssertRC(rc);
762 }
763 break;
764 }
765 }
766 else
767 {
768 /* Reverse the direction and signal the reader threads. */
769 u64State &= ~(RTCSRW_CNT_WR_MASK | RTCSRW_DIR_MASK);
770 u64State |= RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT;
771 if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
772 {
773 Assert(!pThis->fNeedReset);
774 ASMAtomicWriteBool(&pThis->fNeedReset, true);
775 int rc = RTSemEventMultiSignal(pThis->hEvtRead);
776 AssertRC(rc);
777 break;
778 }
779 }
780
781 ASMNopPause();
782 if (pThis->u32Magic != RTCRITSECTRW_MAGIC)
783 return VERR_SEM_DESTROYED;
784 }
785 }
786 else
787 {
788 Assert(pThis->cWriteRecursions != 0);
789#ifdef RTCRITSECTRW_STRICT
790 int rc9 = RTLockValidatorRecExclUnwind(pThis->pValidatorWrite);
791 if (RT_FAILURE(rc9))
792 return rc9;
793#endif
794 ASMAtomicDecU32(&pThis->cWriteRecursions);
795 }
796
797 return VINF_SUCCESS;
798}
799RT_EXPORT_SYMBOL(RTCritSectRwLeaveExcl);
800
801
802RTDECL(bool) RTCritSectRwIsWriteOwner(PRTCRITSECTRW pThis)
803{
804 /*
805 * Validate handle.
806 */
807 AssertPtr(pThis);
808 AssertReturn(pThis->u32Magic == RTCRITSECTRW_MAGIC, false);
809#ifdef IN_RING0
810 Assert(pThis->fFlags & RTCRITSECT_FLAGS_RING0);
811#else
812 Assert(!(pThis->fFlags & RTCRITSECT_FLAGS_RING0));
813#endif
814
815 /*
816 * Check ownership.
817 */
818 RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf();
819 RTNATIVETHREAD hNativeWriter;
820 ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter);
821 return hNativeWriter == hNativeSelf;
822}
823RT_EXPORT_SYMBOL(RTCritSectRwIsWriteOwner);
824
825
826RTDECL(bool) RTCritSectRwIsReadOwner(PRTCRITSECTRW pThis, bool fWannaHear)
827{
828 /*
829 * Validate handle.
830 */
831 AssertPtr(pThis);
832 AssertReturn(pThis->u32Magic == RTCRITSECTRW_MAGIC, false);
833#ifdef IN_RING0
834 Assert(pThis->fFlags & RTCRITSECT_FLAGS_RING0);
835#else
836 Assert(!(pThis->fFlags & RTCRITSECT_FLAGS_RING0));
837#endif
838
839 /*
840 * Inspect the state.
841 */
842 uint64_t u64State = ASMAtomicReadU64(&pThis->u64State);
843 if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT))
844 {
845 /*
846 * It's in write mode, so we can only be a reader if we're also the
847 * current writer.
848 */
849 RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf();
850 RTNATIVETHREAD hWriter;
851 ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hWriter);
852 return hWriter == hNativeSelf;
853 }
854
855 /*
856 * Read mode. If there are no current readers, then we cannot be a reader.
857 */
858 if (!(u64State & RTCSRW_CNT_RD_MASK))
859 return false;
860
861#ifdef RTCRITSECTRW_STRICT
862 /*
863 * Ask the lock validator.
864 */
865 return RTLockValidatorRecSharedIsOwner(pThis->pValidatorRead, NIL_RTTHREAD);
866#else
867 /*
868 * Ok, we don't know, just tell the caller what he want to hear.
869 */
870 return fWannaHear;
871#endif
872}
873RT_EXPORT_SYMBOL(RTCritSectRwIsReadOwner);
874
875
876RTDECL(uint32_t) RTCritSectRwGetWriteRecursion(PRTCRITSECTRW pThis)
877{
878 /*
879 * Validate handle.
880 */
881 AssertPtr(pThis);
882 AssertReturn(pThis->u32Magic == RTCRITSECTRW_MAGIC, 0);
883
884 /*
885 * Return the requested data.
886 */
887 return pThis->cWriteRecursions;
888}
889RT_EXPORT_SYMBOL(RTCritSectRwGetWriteRecursion);
890
891
892RTDECL(uint32_t) RTCritSectRwGetWriterReadRecursion(PRTCRITSECTRW pThis)
893{
894 /*
895 * Validate handle.
896 */
897 AssertPtr(pThis);
898 AssertReturn(pThis->u32Magic == RTCRITSECTRW_MAGIC, 0);
899
900 /*
901 * Return the requested data.
902 */
903 return pThis->cWriterReads;
904}
905RT_EXPORT_SYMBOL(RTCritSectRwGetWriterReadRecursion);
906
907
908RTDECL(uint32_t) RTCritSectRwGetReadCount(PRTCRITSECTRW pThis)
909{
910 /*
911 * Validate input.
912 */
913 AssertPtr(pThis);
914 AssertReturn(pThis->u32Magic == RTCRITSECTRW_MAGIC, 0);
915
916 /*
917 * Return the requested data.
918 */
919 uint64_t u64State = ASMAtomicReadU64(&pThis->u64State);
920 if ((u64State & RTCSRW_DIR_MASK) != (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT))
921 return 0;
922 return (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT;
923}
924RT_EXPORT_SYMBOL(RTCritSectRwGetReadCount);
925
926
927RTDECL(int) RTCritSectRwDelete(PRTCRITSECTRW pThis)
928{
929 /*
930 * Assert free waiters and so on.
931 */
932 AssertPtr(pThis);
933 Assert(pThis->u32Magic == RTCRITSECTRW_MAGIC);
934 //Assert(pThis->cNestings == 0);
935 //Assert(pThis->cLockers == -1);
936 Assert(pThis->hNativeWriter == NIL_RTNATIVETHREAD);
937#ifdef IN_RING0
938 Assert(pThis->fFlags & RTCRITSECT_FLAGS_RING0);
939#else
940 Assert(!(pThis->fFlags & RTCRITSECT_FLAGS_RING0));
941#endif
942
943 /*
944 * Invalidate the structure and free the semaphores.
945 */
946 if (!ASMAtomicCmpXchgU32(&pThis->u32Magic, RTCRITSECTRW_MAGIC_DEAD, RTCRITSECTRW_MAGIC))
947 return VERR_INVALID_PARAMETER;
948
949 pThis->fFlags = 0;
950 pThis->u64State = 0;
951
952 RTSEMEVENT hEvtWrite = pThis->hEvtWrite;
953 pThis->hEvtWrite = NIL_RTSEMEVENT;
954 RTSEMEVENTMULTI hEvtRead = pThis->hEvtRead;
955 pThis->hEvtRead = NIL_RTSEMEVENTMULTI;
956
957 int rc1 = RTSemEventDestroy(hEvtWrite); AssertRC(rc1);
958 int rc2 = RTSemEventMultiDestroy(hEvtRead); AssertRC(rc2);
959
960#ifndef IN_RING0
961 RTLockValidatorRecSharedDestroy(&pThis->pValidatorRead);
962 RTLockValidatorRecExclDestroy(&pThis->pValidatorWrite);
963#endif
964
965 return RT_SUCCESS(rc1) ? rc2 : rc1;
966}
967RT_EXPORT_SYMBOL(RTCritSectRwDelete);
968
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette