VirtualBox

source: vbox/trunk/src/VBox/Runtime/common/time/timesupA.mac@ 80346

Last change on this file since 80346 was 76553, checked in by vboxsync, 6 years ago

scm --update-copyright-year

  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Revision
File size: 23.3 KB
Line 
1; $Id: timesupA.mac 76553 2019-01-01 01:45:53Z vboxsync $
2;; @file
3; IPRT - Time using SUPLib, the Assembly Code Template.
4;
5
6;
7; Copyright (C) 2006-2019 Oracle Corporation
8;
9; This file is part of VirtualBox Open Source Edition (OSE), as
10; available from http://www.virtualbox.org. This file is free software;
11; you can redistribute it and/or modify it under the terms of the GNU
12; General Public License (GPL) as published by the Free Software
13; Foundation, in version 2 as it comes in the "COPYING" file of the
14; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16;
17; The contents of this file may alternatively be used under the terms
18; of the Common Development and Distribution License Version 1.0
19; (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20; VirtualBox OSE distribution, in which case the provisions of the
21; CDDL are applicable instead of those of the GPL.
22;
23; You may elect to license modified versions of this file under the
24; terms and conditions of either the GPL or the CDDL or both.
25;
26
27%ifdef RT_ARCH_X86
28;;
29; The x86 assembly implementation of the assembly routines.
30;
31; @returns Nanosecond timestamp.
32; @param pData Pointer to the nanosecond timestamp data.
33;
34BEGINPROC rtTimeNanoTSInternalAsm
35 ;
36 ; Variable definitions.
37 ;
38%define pData [ebp + 08h]
39%define u64RetNanoTS_Hi [ebp - 04h]
40%define u64RetNanoTS [ebp - 08h]
41%define u32UpdateIntervalNS [ebp - 0ch]
42%define u32UpdateIntervalTSC [ebp - 10h]
43%define u64TSC_Hi [ebp - 14h]
44%define u64TSC [ebp - 18h]
45%define u64CurNanoTS_Hi [ebp - 1ch]
46%define u64CurNanoTS [ebp - 20h]
47%define u64PrevNanoTS_Hi [ebp - 24h]
48%define u64PrevNanoTS [ebp - 28h]
49%define u32TransactionId [ebp - 2ch]
50%define u32ApicIdPlus [ebp - 30h]
51%define TmpVar [ebp - 34h]
52%define SavedEBX [ebp - 38h]
53%define SavedEDI [ebp - 3ch]
54%define SavedESI [ebp - 40h]
55
56 ;
57 ; Prolog.
58 ;
59 push ebp
60 mov ebp, esp
61 sub esp, 40h
62 mov SavedEBX, ebx
63 mov SavedEDI, edi
64 mov SavedESI, esi
65
66
67 ;;
68 ;; Read the GIP data and the previous value.
69 ;;
70.ReadGip:
71
72
73 ;
74 ; Load pGip.
75 ;
76%ifdef IMPORTED_SUPLIB
77 %ifdef IN_RING0
78 mov esi, IMP(g_SUPGlobalInfoPage)
79 %else
80 mov esi, IMP(g_pSUPGlobalInfoPage)
81 mov esi, [esi]
82 %endif
83%else
84 mov esi, [NAME(g_pSUPGlobalInfoPage)]
85%endif
86 or esi, esi
87 jz .Rediscover
88 cmp dword [esi + SUPGLOBALINFOPAGE.u32Magic], SUPGLOBALINFOPAGE_MAGIC
89 jne .Rediscover
90
91 ;
92 ; Calc pGipCPU, setting u32ApicIdPlus if necessary.
93 ;
94%ifdef NEED_APIC_ID
95 ; u8ApicId = ASMGetApicId();
96 mov eax, 1
97 cpuid ; expensive
98 %ifdef NEED_TRANSACTION_ID
99 mov u32ApicIdPlus, ebx
100 %endif
101 ; pGipCpu/pGipCpuDelta = &pGip->aCPU[pGip->aiCpuFromApicId[u8ApicId]];
102 shr ebx, 24
103 movzx ebx, word [esi + ebx * 2 + SUPGLOBALINFOPAGE.aiCpuFromApicId]
104 mov eax, SUPGIPCPU_size
105 mul ebx
106 lea edi, [esi + eax + SUPGLOBALINFOPAGE.aCPUs] ; edi == &pGip->aCPU[u8ApicId];
107%endif
108
109%ifdef NEED_TRANSACTION_ID
110 ;
111 ; Serialized loading of u32TransactionId.
112 ;
113 %ifdef ASYNC_GIP
114 mov ebx, [edi + SUPGIPCPU.u32TransactionId]
115 %else
116 mov ebx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32TransactionId]
117 %endif
118 mov u32TransactionId, ebx
119 %ifdef USE_LFENCE
120 lfence
121 %else
122 lock xor dword TmpVar, 0
123 %endif
124%endif
125
126 ;
127 ; Load the data and TSC with delta applied.
128 ;
129 mov eax, [esi + SUPGLOBALINFOPAGE.u32UpdateIntervalNS]
130 mov u32UpdateIntervalNS, eax
131%ifdef ASYNC_GIP ; esi is now free.
132 mov edx, [edi + SUPGIPCPU.u32UpdateIntervalTSC]
133%else
134 mov edx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32UpdateIntervalTSC]
135%endif
136 mov u32UpdateIntervalTSC, edx
137
138 rdtsc
139%ifdef WITH_TSC_DELTA
140 cmp dword [edi + SUPGIPCPU.i64TSCDelta], 0xffffffff
141 je .TscDeltaPossiblyInvalid
142.TscDeltaValid:
143 sub eax, dword [edi + SUPGIPCPU.i64TSCDelta]
144 sbb edx, dword [edi + SUPGIPCPU.i64TSCDelta + 4]
145.TscDeltaNotValid: ; edi is now free.
146%endif
147
148%ifdef ASYNC_GIP
149 mov ecx, [edi + SUPGIPCPU.u64NanoTS]
150 mov esi, [edi + SUPGIPCPU.u64NanoTS + 4]
151%else
152 mov ecx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64NanoTS]
153 mov ebx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64NanoTS + 4]
154%endif
155 mov u64CurNanoTS, ecx
156 mov u64CurNanoTS_Hi, ebx
157%ifdef ASYNC_GIP
158 mov ecx, [edi + SUPGIPCPU.u64TSC]
159 mov ebx, [edi + SUPGIPCPU.u64TSC + 4]
160%else
161 mov ecx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64TSC]
162 mov ebx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64TSC + 4]
163%endif
164 mov u64TSC, ecx
165 mov u64TSC_Hi, ebx
166
167 ; u64PrevNanoTS = ASMAtomicReadU64(pu64Prev);
168 ; This serializes load/save. And with the dependency on the
169 ; RDTSC result, we try to make sure it has completed as well.
170%ifdef ASYNC_GIP
171 mov esi, pData
172 mov esi, [esi + RTTIMENANOTSDATA.pu64Prev]
173%else
174 mov edi, pData
175 mov edi, [esi + RTTIMENANOTSDATA.pu64Prev]
176%endif
177 mov ebx, eax
178 mov ecx, edx
179%ifdef ASYNC_GIP
180 lock cmpxchg8b [esi]
181%else
182 lock cmpxchg8b [edi]
183%endif
184 mov u64PrevNanoTS, eax
185 mov u64PrevNanoTS_Hi, edx
186
187%undef SAVED_u64RetNanoTS
188%ifdef NEED_TRANSACTION_ID
189 ;
190 ; Check that the GIP and CPU didn't change.
191 ; We've already serialized all the loads and stores at this point.
192 ;
193 %ifdef NEED_APIC_ID
194 mov u64RetNanoTS, ebx
195 mov u64RetNanoTS_Hi, ecx
196 %define SAVED_u64RetNanoTS
197 mov eax, 1
198 cpuid
199 cmp u32ApicIdPlus, ebx
200 jne .ReadGip
201 %endif
202 %ifdef ASYNC_GIP
203 mov esi, [edi + SUPGIPCPU.u32TransactionId]
204 %else
205 mov esi, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32TransactionId]
206 %endif
207 cmp esi, u32TransactionId
208 jne .ReadGip
209 test esi, 1
210 jnz .ReadGip
211%endif ; NEED_TRANSACTION_ID
212%ifdef SAVED_u64RetNanoTS
213 mov ebx, u64RetNanoTS
214 mov ecx, u64RetNanoTS_Hi
215%endif
216
217 ;;
218 ;; Calc the timestamp.
219 ;;
220 ; u64RetNanoTS -= u64TSC;
221 sub ebx, u64TSC
222 sbb ecx, u64TSC_Hi
223
224 ; if (u64RetNanoTS > u32UpdateIntervalTSC) -> jump
225 or ecx, ecx
226 jnz .OverFlow
227 cmp ebx, u32UpdateIntervalTSC
228 ja .OverFlow
229 mov eax, ebx
230.ContinueCalcs: ; eax <= u32UpdateIntervalTSC
231 mul dword u32UpdateIntervalNS
232 div dword u32UpdateIntervalTSC
233 xor edx, edx
234
235 ; u64RetNanoTS += u64CurNanoTS;
236 add eax, u64CurNanoTS
237 adc edx, u64CurNanoTS_Hi
238
239 ;;
240 ;; Compare it with the previous one.
241 ;;
242 ; if (RT_LIKELY( u64RetNanoTS > u64PrevNanoTS
243 ; && u64RetNanoTS < u64PrevNanoTS + UINT64_C(86000000000000) /* 24h */))
244 ;; @todo optimize this compare (/me too tired).
245 mov ecx, u64PrevNanoTS_Hi
246 mov ebx, u64PrevNanoTS
247 cmp edx, ecx
248 ja .Compare2
249 jb .DeltaPrevTooBig
250 cmp eax, ebx
251 jbe .DeltaPrevTooBig
252
253.Compare2:
254 add ebx, 0x6F736000
255 adc ecx, 0x00004E37
256 cmp edx, ecx
257 jb .CompareDone
258 ja .DeltaPrevTooBig
259 cmp eax, ebx
260 jae .DeltaPrevTooBig
261.CompareDone:
262
263
264 ;;
265 ;; Update the previous value with the u64RetNanoTS value.
266 ;;
267.Update:
268 ; if (RT_LIKELY(ASMAtomicCmpXchgU64(&pData->u64Prev, u64RetNanoTS, u64PrevNanoTS)))
269 mov ebx, eax
270 mov ecx, edx
271 mov esi, pData
272 mov esi, [esi + RTTIMENANOTSDATA.pu64Prev]
273 mov eax, u64PrevNanoTS
274 mov edx, u64PrevNanoTS_Hi
275 lock cmpxchg8b [esi]
276 jnz .UpdateFailed
277
278.Updated:
279 mov eax, ebx
280 mov edx, ecx
281
282.Done:
283 mov esi, SavedESI
284 mov edi, SavedEDI
285 mov ebx, SavedEBX
286 leave
287 ret
288
289
290 ;;
291 ;; We've expired the interval, cap it. If we're here for the 2nd
292 ;; time without any GIP update in-between, the checks against
293 ;; pData->u64Prev below will force 1ns stepping.
294 ;;
295.OverFlow:
296 ; u64Delta = u32UpdateIntervalTSC;
297 mov esi, pData
298 inc dword [esi + RTTIMENANOTSDATA.cExpired]
299 mov eax, u32UpdateIntervalTSC
300 jmp .ContinueCalcs
301
302
303 ;;
304 ;; u64DeltaPrev >= 24h
305 ;;
306 ;; eax:edx = u64RetNanoTS (to be adjusted)
307 ;;
308.DeltaPrevTooBig:
309 ; uint64_t u64DeltaPrev = u64RetNanoTS - u64PrevNanoTS;
310 mov ebx, eax
311 sub ebx, u64PrevNanoTS
312 mov ecx, edx
313 sbb ecx, u64PrevNanoTS_Hi ; ebx:ecx = u64DeltaPrev
314
315 ; else if ( (int64_t)u64DeltaPrev <= 0
316 ; && (int64_t)u64DeltaPrev + u32UpdateIntervalNS * 2 >= 0)
317 ; {
318 ; /* Occasional - u64RetNanoTS is in the recent 'past' relative the previous call. */
319 ; pData->c1nsSteps++;
320 ; u64RetNanoTS = u64PrevNanoTS + 1;
321 ; }
322 mov esi, u32UpdateIntervalNS
323 cmp ecx, 0
324 jl .PrevNotZero2ndTest
325 jg .DeltaPrevNotInRecentPast
326 cmp ebx, 0
327 ja .DeltaPrevNotInRecentPast
328
329.PrevNotZero2ndTest:
330 add esi, esi ; ASSUMES: u32UpdateIntervalNS * 2 <= 32-bit.
331 xor edi, edi
332 add esi, ebx
333 adc edi, ecx
334 test edi, edi
335 js .DeltaPrevNotInRecentPast
336
337.DeltaPrevInRecentPast:
338 mov esi, pData
339 inc dword [esi + RTTIMENANOTSDATA.c1nsSteps]
340 mov eax, u64PrevNanoTS
341 mov edx, u64PrevNanoTS_Hi
342 add eax, 1
343 adc edx, 0
344 jmp .Update
345
346.DeltaPrevNotInRecentPast:
347 ; else if (!u64PrevNanoTS) /* We're resuming (see TMVirtualResume). */
348 ; /* do nothing */;
349 cmp dword u64PrevNanoTS, 0
350 jne .DeltaPrevNotZero
351 cmp dword u64PrevNanoTS_Hi, 0
352 jne .DeltaPrevNotZero
353 jmp .Update
354
355.DeltaPrevNotZero:
356 ; else
357 ; {
358 ; /* Something has gone bust, if negative offset it's real bad. */
359 ; rtTimeNanoTSInternalBitch(pVM,
360 ; }
361
362 ; call C function that does the bitching.
363 mov u64RetNanoTS, eax
364 mov u64RetNanoTS_Hi, edx
365
366 mov edi, u64PrevNanoTS_Hi
367 mov esi, u64PrevNanoTS
368 push edi
369 push esi ; 4 - u64PrevNanoTS
370 push ecx
371 push ebx ; 3 - u64DeltaPrev
372 push edx
373 push eax ; 2 - u64RetNanoTS
374 mov eax, pData
375 push eax ; 1 - pData
376 call dword [eax + RTTIMENANOTSDATA.pfnBad]
377 add esp, 4*7
378
379 mov eax, u64RetNanoTS
380 mov edx, u64RetNanoTS_Hi
381 jmp .Update
382
383
384 ;;
385 ;; Attempt updating the previous value, provided we're still ahead of it.
386 ;;
387 ;; There is no point in recalculating u64NanoTS because we got preempted or if
388 ;; we raced somebody while the GIP was updated, since these are events
389 ;; that might occur at any point in the return path as well.
390 ;;
391 ;; eax:edx = *pData->u64Prev
392 ;; ebx:ecx = u64RetNanoTS
393 ;;
394 ALIGNCODE(16)
395.UpdateFailed:
396 mov edi, pData
397 lock inc dword [edi + RTTIMENANOTSDATA.cUpdateRaces]
398 ; for (i = 0; i < 10; i++)
399 mov edi, 10
400.UpdateLoop:
401 ; if (u64PrevNanoTS >= u64NanoTS)
402 ; break;
403 cmp edx, ecx
404 jg .Updated
405 jne .UpdateLoopLess
406 cmp eax, ebx
407 jae .Updated
408.UpdateLoopLess:
409 ; retry
410 lock cmpxchg8b [esi]
411 jz .Updated
412 dec edi
413 jnz .UpdateLoop
414 jmp .Updated
415
416
417 ;;
418 ;; The GIP is seemingly invalid, redo the discovery.
419 ;;
420.Rediscover:
421 mov eax, pData
422 push eax
423 call [eax + RTTIMENANOTSDATA.pfnRediscover]
424 add esp, 4h
425 jmp .Done
426
427
428%ifdef WITH_TSC_DELTA
429 ;;
430 ;; Unlikely branch for when we think the TSC delta might be invalid.
431 ;;
432.TscDeltaPossiblyInvalid:
433 cmp dword [edi + SUPGIPCPU.i64TSCDelta + 4], 0x7fffffff
434 jne .TscDeltaValid
435 jmp .TscDeltaNotValid
436%endif
437
438 ;
439 ; Cleanup variables
440 ;
441%undef pData
442%undef u64Delta_Hi
443%undef u64Delta
444%undef u32UpdateIntervalNS
445%undef u32UpdateIntervalTSC
446%undef u64TSC_Hi
447%undef u64TSC
448%undef u64NanoTS_Hi
449%undef u64NanoTS
450%undef u64PrevNanoTS_Hi
451%undef u64PrevNanoTS
452%undef u32TransactionId
453%undef u8ApicId
454
455%else ; AMD64
456
457;;
458; The AMD64 assembly implementation of the assembly routines.
459;
460; @returns Nanosecond timestamp.
461; @param pData gcc:rdi msc:rcx Pointer to the nanosecond timestamp data.
462;
463BEGINPROC rtTimeNanoTSInternalAsm
464 ;
465 ; Define variables and stack frame.
466 ;
467%define SavedRBX [rbp - 08h]
468%define SavedR12 [rbp - 10h]
469%define SavedR13 [rbp - 18h]
470%define SavedRDI [rbp - 20h]
471%define SavedRSI [rbp - 28h]
472%define TmpVar [rbp - 30h]
473%define TmpVar2 [rbp - 38h]
474%ifdef NEED_TRANSACTION_ID
475 %ifdef NEED_APIC_ID
476 %define SavedR14 [rbp - 40h]
477 %define SavedR15 [rbp - 48h]
478 %endif
479%endif
480
481%define pData rdi
482
483%ifdef ASYNC_GIP
484 %define u64TSC rsi
485 %define pGip rsi
486 %ifdef NEED_APIC_ID
487 %define pGipCPU r8
488 %endif
489%else
490 %define u64TSC r8
491 %define pGip rsi
492 %ifdef NEED_APIC_ID
493 %define pGipCPU r8
494 %endif
495%endif
496%define u32TransactionId r9d
497%define u64CurNanoTS r10
498%define u64PrevNanoTS r11 ; not parameter register
499%define u32UpdateIntervalTSC r12d
500%define u32UpdateIntervalTSC_64 r12
501%define u32UpdateIntervalNS r13d
502%define u32UpdateIntervalNS_64 r13
503%undef u64SavedRetNanoTS
504%undef u32ApicIdPlus
505%ifdef NEED_TRANSACTION_ID
506 %ifdef NEED_APIC_ID
507 %define u64SavedRetNanoTS r14
508 %define u32ApicIdPlus r15d
509 %endif
510%endif
511
512 ;
513 ; The prolog.
514 ;
515 push rbp
516 mov rbp, rsp
517%ifdef ASM_CALL64_MSC
518 sub rsp, 50h+20h
519%else
520 sub rsp, 50h
521%endif
522 mov SavedRBX, rbx
523 mov SavedR12, r12
524 mov SavedR13, r13
525%ifdef ASM_CALL64_MSC
526 mov SavedRDI, rdi
527 mov SavedRSI, rsi
528 mov pData, rcx
529%else
530 ;mov pData, rdi - already in rdi.
531%endif
532%ifdef SavedR14
533 mov SavedR14, r14
534%endif
535%ifdef SavedR15
536 mov SavedR15, r15
537%endif
538
539
540 ;;
541 ;; Data fetch loop.
542 ;; We take great pain ensuring that data consistency here.
543 ;;
544.ReadGip:
545
546 ;
547 ; Load pGip - finding the GIP is fun...
548 ;
549%ifdef RT_OS_WINDOWS
550 %ifdef IMPORTED_SUPLIB
551 %ifdef IN_RING0
552 mov rax, qword IMP(g_SUPGlobalInfoPage)
553 mov pGip, rax
554 %else
555 mov pGip, [IMP(g_pSUPGlobalInfoPage) wrt rip]
556 mov pGip, [pGip]
557 %endif
558 %else
559 mov pGip, [NAME(g_pSUPGlobalInfoPage) wrt rip]
560 %endif
561%else
562 %ifdef IN_RING0
563 mov rax, qword NAME(g_SUPGlobalInfoPage)
564 mov pGip, rax
565 %else
566 mov pGip, [rel NAME(g_pSUPGlobalInfoPage) wrt ..gotpcrel]
567 mov pGip, [pGip]
568 %endif
569%endif
570 or pGip, pGip
571 jz .Rediscover
572 cmp dword [pGip + SUPGLOBALINFOPAGE.u32Magic], SUPGLOBALINFOPAGE_MAGIC
573 jne .Rediscover
574
575 ;
576 ; pGipCPU, setting u32ApicIdPlus if necessary.
577 ;
578%ifdef NEED_APIC_ID
579 ; u8ApicId = ASMGetApicId();
580 mov eax, 1
581 cpuid ; expensive
582 %ifdef NEED_TRANSACTION_ID
583 mov u32ApicIdPlus, ebx
584 %endif
585 ; pGipCPU = &pGip->aCPU[pGip->aiCpuFromApicId[u8ApicId]];
586 shr ebx, 24
587 movzx eax, word [pGip + rbx * 2 + SUPGLOBALINFOPAGE.aiCpuFromApicId]
588 imul eax, SUPGIPCPU_size
589 lea pGipCPU, [pGip + rax + SUPGLOBALINFOPAGE.aCPUs]
590%endif
591
592%ifdef NEED_TRANSACTION_ID
593 ;
594 ; Serialized loading of u32TransactionId.
595 ;
596 %ifdef ASYNC_GIP
597 mov u32TransactionId, [pGipCPU + SUPGIPCPU.u32TransactionId]
598 %else
599 mov u32TransactionId, [pGip + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32TransactionId]
600 %endif
601 %ifdef USE_LFENCE
602 lfence
603 %else
604 lock xor dword TmpVar, 0
605 %endif
606%endif
607
608 ;
609 ; Load the data and TSC.
610 ;
611 mov u32UpdateIntervalNS, [pGip + SUPGLOBALINFOPAGE.u32UpdateIntervalNS]
612%ifdef ASYNC_GIP
613 mov u32UpdateIntervalTSC, [pGipCPU + SUPGIPCPU.u32UpdateIntervalTSC]
614%else
615 mov u32UpdateIntervalTSC, [pGip + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32UpdateIntervalTSC]
616%endif
617
618 rdtsc
619 mov u64PrevNanoTS, [pData + RTTIMENANOTSDATA.pu64Prev]
620 mov u64PrevNanoTS, [u64PrevNanoTS]
621 shl rdx, 32
622 or rax, rdx ; rax is u64RetNanoTS.
623%ifdef WITH_TSC_DELTA
624 mov rdx, [pGipCPU + SUPGIPCPU.i64TSCDelta]
625 mov u64CurNanoTS, 0x7fffffffffffffff ; INT64_MAX - temporarily borrowing u64CurNanoTS
626 cmp rdx, u64CurNanoTS
627 je .TscDeltaNotValid
628 sub rax, rdx
629.TscDeltaNotValid:
630%endif
631%ifdef u64SavedRetNanoTS ; doing this here may save a tick or so?
632 mov u64SavedRetNanoTS, rax
633%endif
634
635%ifdef ASYNC_GIP
636 mov u64CurNanoTS, [pGipCPU + SUPGIPCPU.u64NanoTS]
637 mov u64TSC, [pGipCPU + SUPGIPCPU.u64TSC] ; transhes pGIP!
638%else
639 mov u64CurNanoTS, [pGip + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64NanoTS]
640 mov u64TSC, [pGip + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64TSC] ; trashes pGipCPU!
641%endif
642
643
644%ifdef NEED_TRANSACTION_ID
645 ;
646 ; Check that the GIP and CPU didn't change.
647 ;
648 ; It is crucial that the rdtsc instruction has completed before
649 ; we check the transaction id. The LOCK prefixed instruction with
650 ; dependency on the RDTSC result should do the trick, I think.
651 ; CPUID is serializing, so the async path is safe by default.
652 ;
653 %ifdef NEED_APIC_ID
654 mov eax, 1
655 cpuid
656 cmp u32ApicIdPlus, ebx
657 jne .ReadGip
658 %else
659 lock xor qword TmpVar, rax
660 %endif
661 %ifdef ASYNC_GIP
662 cmp u32TransactionId, [pGipCPU + SUPGIPCPU.u32TransactionId]
663 %else
664 cmp u32TransactionId, [pGip + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32TransactionId]
665 %endif
666 jne .ReadGip
667 test u32TransactionId, 1
668 jnz .ReadGip
669 %ifdef u64SavedRetNanoTS
670 mov rax, u64SavedRetNanoTS ; rax is u64RetNanoTS.
671 %endif
672%endif ; NEED_TRANSACTION_ID
673
674
675 ;;
676 ;; Calc the timestamp.
677 ;;
678 ; u64RetNanoTS -= u64TSC;
679 sub rax, u64TSC
680 xor edx, edx
681
682 ; if (u64RetNanoTS > u32UpdateIntervalTSC) -> jump
683 cmp rax, u32UpdateIntervalTSC_64
684 ja .OverFlow
685.ContinueCalcs: ; edx = 0; eax <= u32UpdateIntervalTSC
686 mul u32UpdateIntervalNS
687 div u32UpdateIntervalTSC
688
689 ; u64RetNanoTS += u64CurNanoTS;
690 add rax, u64CurNanoTS
691
692
693 ;;
694 ;; Compare it with the previous one.
695 ;;
696 ; if (RT_LIKELY( u64RetNanoTS > u64PrevNanoTS
697 ; && u64RetNanoTS < u64PrevNanoTS + UINT64_C(86000000000000) /* 24h */))
698 ; /* Frequent - less than 24h since last call. */;
699 cmp rax, u64PrevNanoTS
700 jbe .DeltaPrevTooBig
701 mov ecx, 5
702 shl rcx, 44 ; close enough
703 add rcx, u64PrevNanoTS
704 cmp rax, rcx
705 jae .DeltaPrevTooBig
706
707
708 ;;
709 ;; Update the previous value.
710 ;;
711.Update:
712 ; if (RT_LIKELY(ASMAtomicCmpXchgU64(&pData->u64Prev, u64RetNanoTS, u64PrevNanoTS)))
713 mov rbx, [pData + RTTIMENANOTSDATA.pu64Prev]
714 mov rcx, rax
715 mov rax, u64PrevNanoTS
716 lock cmpxchg [rbx], rcx
717 jnz .UpdateFailed
718
719.Updated:
720 mov rax, rcx
721
722.Done:
723 mov rbx, SavedRBX
724 mov r12, SavedR12
725 mov r13, SavedR13
726%ifdef SavedR14
727 mov r14, SavedR14
728%endif
729%ifdef SavedR15
730 mov r15, SavedR15
731%endif
732%ifdef ASM_CALL64_MSC
733 mov rdi, SavedRDI
734 mov rsi, SavedRSI
735%endif
736 leave
737 ret
738
739
740 ;;
741 ;; We've expired the interval, cap it. If we're here for the 2nd
742 ;; time without any GIP update in-between, the checks against
743 ;; pData->u64Prev below will force 1ns stepping.
744 ;;
745ALIGNCODE(16)
746.OverFlow:
747 ; u64RetNanoTS = u32UpdateIntervalTSC;
748 inc dword [pData + RTTIMENANOTSDATA.cExpired]
749 mov eax, u32UpdateIntervalTSC
750 jmp .ContinueCalcs
751
752
753 ;;
754 ;; u64DeltaPrev >= 24h
755 ;;
756 ;; rax = u64RetNanoTS (to be adjusted)
757 ;;
758ALIGNCODE(16)
759.DeltaPrevTooBig:
760 ; uint64_t u64DeltaPrev = u64RetNanoTS - u64PrevNanoTS;
761 mov rbx, rax
762 sub rbx, u64PrevNanoTS
763
764 ; else if ( (int64_t)u64DeltaPrev <= 0
765 ; && (int64_t)u64DeltaPrev + u32UpdateIntervalNS * 2 >= 0)
766 ; {
767 ; /* Occasional - u64NanoTS is in the recent 'past' relative the previous call. */
768 ; pData->c1nsSteps++;
769 ; u64RetNanoTS = u64PrevNanoTS + 1;
770 ; }
771 test rbx, rbx
772 jg .DeltaPrevNotInRecentPast
773
774 lea rdx, [u32UpdateIntervalNS_64 + u32UpdateIntervalNS_64]
775 add rdx, rbx
776 js .DeltaPrevNotInRecentPast
777
778 ; body
779 inc dword [pData + RTTIMENANOTSDATA.c1nsSteps]
780 lea rax, [u64PrevNanoTS + 1]
781 jmp .Update
782
783 ; else if (!u64PrevNanoTS) /* We're resuming (see TMVirtualResume) / first call. */
784 ; /* do nothing */;
785.DeltaPrevNotInRecentPast:
786 or u64PrevNanoTS, u64PrevNanoTS
787 jz .Update
788
789 ; else
790 ; {
791 ; /* Something has gone bust, if negative offset it's real bad. */
792 ; rtTimeNanoTSInternalBitch(pVM,
793 ; }
794
795 ; call C function that does the bitching.
796 mov TmpVar, rax
797 mov TmpVar2, pData
798
799%ifdef ASM_CALL64_MSC
800 mov rcx, pData ; param 1 - pData
801 mov rdx, rax ; param 2 - u64RetNanoTS
802 mov r8, rbx ; param 3 - u64DeltaPrev
803 mov r9, u64PrevNanoTS ; param 4 - u64PrevNanoTS
804%else
805 ;mov rdi, pData - already in rdi; param 1 - pData
806 mov rsi, rax ; param 2 - u64RetNanoTS
807 mov rdx, rbx ; param 3 - u64DeltaPrev
808 mov rcx, u64PrevNanoTS ; param 4 - u64PrevNanoTS
809%endif
810 call qword [pData + RTTIMENANOTSDATA.pfnBad]
811
812 mov rax, TmpVar
813 mov pData, TmpVar2
814 jmp .Update
815
816
817 ;;
818 ;; Attempt updating the previous value, provided we're still ahead of it.
819 ;;
820 ;; There is no point in recalculating u64NanoTS because we got preempted or if
821 ;; we raced somebody while the GIP was updated, since these are events
822 ;; that might occur at any point in the return path as well.
823 ;;
824 ;; rax = *pData->u64Prev;
825 ;; rcx = u64RetNanoTS
826 ;;
827ALIGNCODE(16)
828.UpdateFailed:
829 lock inc dword [pData + RTTIMENANOTSDATA.cUpdateRaces]
830 ; for (i = 0; i < 10; i++)
831 mov edx, 10
832.UpdateLoop:
833 ; if (u64PrevNanoTS >= u64RetNanoTS)
834 ; break;
835 cmp rax, rcx
836 jge .Updated
837.UpdateLoopLess:
838 ; retry
839 lock cmpxchg [rbx], rcx
840 jz .Updated
841 dec edx
842 jnz .UpdateLoop
843 jmp .Updated
844
845
846 ;;
847 ;; The GIP is seemingly invalid, redo the discovery.
848 ;;
849.Rediscover:
850%ifdef ASM_CALL64_MSC
851 mov rcx, pData
852%else
853 ; mov rdi, pData - already in rdi
854%endif
855 call [pData + RTTIMENANOTSDATA.pfnRediscover]
856 jmp .Done
857
858
859 ;
860 ; Cleanup variables
861 ;
862%undef SavedRBX
863%undef SavedR12
864%undef SavedR13
865%undef SavedR14
866%undef SavedR15
867%undef SavedRDI
868%undef SavedRSI
869%undef pData
870%undef TmpVar
871%undef u64TSC
872%undef pGip
873%undef pGipCPU
874%undef u32TransactionId
875%undef u64CurNanoTS
876%undef u64PrevNanoTS
877%undef u32UpdateIntervalTSC
878%undef u32UpdateIntervalTSC_64
879%undef u32UpdateIntervalNS
880%undef u64SavedRetNanoTS
881%undef u32ApicIdPlus
882
883%endif ; AMD64
884ENDPROC rtTimeNanoTSInternalAsm
885
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette