1 | ; $Id: timesupA.mac 56290 2015-06-09 14:01:31Z vboxsync $
|
---|
2 | ;; @file
|
---|
3 | ; IPRT - Time using SUPLib, the Assembly Code Template.
|
---|
4 | ;
|
---|
5 |
|
---|
6 | ;
|
---|
7 | ; Copyright (C) 2006-2015 Oracle Corporation
|
---|
8 | ;
|
---|
9 | ; This file is part of VirtualBox Open Source Edition (OSE), as
|
---|
10 | ; available from http://www.virtualbox.org. This file is free software;
|
---|
11 | ; you can redistribute it and/or modify it under the terms of the GNU
|
---|
12 | ; General Public License (GPL) as published by the Free Software
|
---|
13 | ; Foundation, in version 2 as it comes in the "COPYING" file of the
|
---|
14 | ; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
|
---|
15 | ; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
|
---|
16 | ;
|
---|
17 | ; The contents of this file may alternatively be used under the terms
|
---|
18 | ; of the Common Development and Distribution License Version 1.0
|
---|
19 | ; (CDDL) only, as it comes in the "COPYING.CDDL" file of the
|
---|
20 | ; VirtualBox OSE distribution, in which case the provisions of the
|
---|
21 | ; CDDL are applicable instead of those of the GPL.
|
---|
22 | ;
|
---|
23 | ; You may elect to license modified versions of this file under the
|
---|
24 | ; terms and conditions of either the GPL or the CDDL or both.
|
---|
25 | ;
|
---|
26 |
|
---|
27 | %ifdef RT_ARCH_X86
|
---|
28 | ;;
|
---|
29 | ; The x86 assembly implementation of the assembly routines.
|
---|
30 | ;
|
---|
31 | ; @returns Nanosecond timestamp.
|
---|
32 | ; @param pData Pointer to the nanosecond timestamp data.
|
---|
33 | ;
|
---|
34 | BEGINPROC rtTimeNanoTSInternalAsm
|
---|
35 | ;
|
---|
36 | ; Variable definitions.
|
---|
37 | ;
|
---|
38 | %define pData [ebp + 08h]
|
---|
39 | %define u64RetNanoTS_Hi [ebp - 04h]
|
---|
40 | %define u64RetNanoTS [ebp - 08h]
|
---|
41 | %define u32UpdateIntervalNS [ebp - 0ch]
|
---|
42 | %define u32UpdateIntervalTSC [ebp - 10h]
|
---|
43 | %define u64TSC_Hi [ebp - 14h]
|
---|
44 | %define u64TSC [ebp - 18h]
|
---|
45 | %define u64CurNanoTS_Hi [ebp - 1ch]
|
---|
46 | %define u64CurNanoTS [ebp - 20h]
|
---|
47 | %define u64PrevNanoTS_Hi [ebp - 24h]
|
---|
48 | %define u64PrevNanoTS [ebp - 28h]
|
---|
49 | %define u32TransactionId [ebp - 2ch]
|
---|
50 | %define u32ApicIdPlus [ebp - 30h]
|
---|
51 | %define TmpVar [ebp - 34h]
|
---|
52 | %define SavedEBX [ebp - 38h]
|
---|
53 | %define SavedEDI [ebp - 3ch]
|
---|
54 | %define SavedESI [ebp - 40h]
|
---|
55 |
|
---|
56 | ;
|
---|
57 | ; Prolog.
|
---|
58 | ;
|
---|
59 | push ebp
|
---|
60 | mov ebp, esp
|
---|
61 | sub esp, 40h
|
---|
62 | mov SavedEBX, ebx
|
---|
63 | mov SavedEDI, edi
|
---|
64 | mov SavedESI, esi
|
---|
65 |
|
---|
66 |
|
---|
67 | ;;
|
---|
68 | ;; Read the GIP data and the previous value.
|
---|
69 | ;;
|
---|
70 | .ReadGip:
|
---|
71 |
|
---|
72 |
|
---|
73 | ;
|
---|
74 | ; Load pGip.
|
---|
75 | ;
|
---|
76 | %ifdef IMPORTED_SUPLIB
|
---|
77 | %ifdef IN_RING0
|
---|
78 | mov esi, IMP(g_SUPGlobalInfoPage)
|
---|
79 | %else
|
---|
80 | mov esi, IMP(g_pSUPGlobalInfoPage)
|
---|
81 | mov esi, [esi]
|
---|
82 | %endif
|
---|
83 | %else
|
---|
84 | mov esi, [NAME(g_pSUPGlobalInfoPage)]
|
---|
85 | %endif
|
---|
86 | or esi, esi
|
---|
87 | jz .Rediscover
|
---|
88 | cmp dword [esi + SUPGLOBALINFOPAGE.u32Magic], SUPGLOBALINFOPAGE_MAGIC
|
---|
89 | jne .Rediscover
|
---|
90 |
|
---|
91 | ;
|
---|
92 | ; Calc pGipCPU, setting u32ApicIdPlus if necessary.
|
---|
93 | ;
|
---|
94 | %ifdef NEED_APIC_ID
|
---|
95 | ; u8ApicId = ASMGetApicId();
|
---|
96 | mov eax, 1
|
---|
97 | cpuid ; expensive
|
---|
98 | %ifdef NEED_TRANSACTION_ID
|
---|
99 | mov u32ApicIdPlus, ebx
|
---|
100 | %endif
|
---|
101 | ; pGipCpu/pGipCpuDelta = &pGip->aCPU[pGip->aiCpuFromApicId[u8ApicId]];
|
---|
102 | shr ebx, 24
|
---|
103 | movzx ebx, word [esi + ebx * 2 + SUPGLOBALINFOPAGE.aiCpuFromApicId]
|
---|
104 | mov eax, SUPGIPCPU_size
|
---|
105 | mul ebx
|
---|
106 | lea edi, [esi + eax + SUPGLOBALINFOPAGE.aCPUs] ; edi == &pGip->aCPU[u8ApicId];
|
---|
107 | %endif
|
---|
108 |
|
---|
109 | %ifdef NEED_TRANSACTION_ID
|
---|
110 | ;
|
---|
111 | ; Serialized loading of u32TransactionId.
|
---|
112 | ;
|
---|
113 | %ifdef ASYNC_GIP
|
---|
114 | mov ebx, [edi + SUPGIPCPU.u32TransactionId]
|
---|
115 | %else
|
---|
116 | mov ebx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32TransactionId]
|
---|
117 | %endif
|
---|
118 | mov u32TransactionId, ebx
|
---|
119 | %ifdef USE_LFENCE
|
---|
120 | lfence
|
---|
121 | %else
|
---|
122 | lock xor dword TmpVar, 0
|
---|
123 | %endif
|
---|
124 | %endif
|
---|
125 |
|
---|
126 | ;
|
---|
127 | ; Load the data and TSC with delta applied.
|
---|
128 | ;
|
---|
129 | mov eax, [esi + SUPGLOBALINFOPAGE.u32UpdateIntervalNS]
|
---|
130 | mov u32UpdateIntervalNS, eax
|
---|
131 | %ifdef ASYNC_GIP ; esi is now free.
|
---|
132 | mov edx, [edi + SUPGIPCPU.u32UpdateIntervalTSC]
|
---|
133 | %else
|
---|
134 | mov edx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32UpdateIntervalTSC]
|
---|
135 | %endif
|
---|
136 | mov u32UpdateIntervalTSC, edx
|
---|
137 |
|
---|
138 | rdtsc
|
---|
139 | %ifdef WITH_TSC_DELTA
|
---|
140 | cmp dword [edi + SUPGIPCPU.i64TSCDelta], 0xffffffff
|
---|
141 | je .TscDeltaPossiblyInvalid
|
---|
142 | .TscDeltaValid:
|
---|
143 | sub eax, dword [edi + SUPGIPCPU.i64TSCDelta]
|
---|
144 | sbb edx, dword [edi + SUPGIPCPU.i64TSCDelta + 4]
|
---|
145 | .TscDeltaNotValid: ; edi is now free.
|
---|
146 | %endif
|
---|
147 |
|
---|
148 | %ifdef ASYNC_GIP
|
---|
149 | mov ecx, [edi + SUPGIPCPU.u64NanoTS]
|
---|
150 | mov esi, [edi + SUPGIPCPU.u64NanoTS + 4]
|
---|
151 | %else
|
---|
152 | mov ecx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64NanoTS]
|
---|
153 | mov ebx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64NanoTS + 4]
|
---|
154 | %endif
|
---|
155 | mov u64CurNanoTS, ecx
|
---|
156 | mov u64CurNanoTS_Hi, ebx
|
---|
157 | %ifdef ASYNC_GIP
|
---|
158 | mov ecx, [edi + SUPGIPCPU.u64TSC]
|
---|
159 | mov ebx, [edi + SUPGIPCPU.u64TSC + 4]
|
---|
160 | %else
|
---|
161 | mov ecx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64TSC]
|
---|
162 | mov ebx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64TSC + 4]
|
---|
163 | %endif
|
---|
164 | mov u64TSC, ecx
|
---|
165 | mov u64TSC_Hi, ebx
|
---|
166 |
|
---|
167 | ; u64PrevNanoTS = ASMAtomicReadU64(pu64Prev);
|
---|
168 | ; This serializes load/save. And with the dependency on the
|
---|
169 | ; RDTSC result, we try to make sure it has completed as well.
|
---|
170 | %ifdef ASYNC_GIP
|
---|
171 | mov esi, pData
|
---|
172 | mov esi, [esi + RTTIMENANOTSDATA.pu64Prev]
|
---|
173 | %else
|
---|
174 | mov edi, pData
|
---|
175 | mov edi, [esi + RTTIMENANOTSDATA.pu64Prev]
|
---|
176 | %endif
|
---|
177 | mov ebx, eax
|
---|
178 | mov ecx, edx
|
---|
179 | %ifdef ASYNC_GIP
|
---|
180 | lock cmpxchg8b [esi]
|
---|
181 | %else
|
---|
182 | lock cmpxchg8b [edi]
|
---|
183 | %endif
|
---|
184 | mov u64PrevNanoTS, eax
|
---|
185 | mov u64PrevNanoTS_Hi, edx
|
---|
186 |
|
---|
187 | %undef SAVED_u64RetNanoTS
|
---|
188 | %ifdef NEED_TRANSACTION_ID
|
---|
189 | ;
|
---|
190 | ; Check that the GIP and CPU didn't change.
|
---|
191 | ; We've already serialized all the loads and stores at this point.
|
---|
192 | ;
|
---|
193 | %ifdef NEED_APIC_ID
|
---|
194 | mov u64RetNanoTS, ebx
|
---|
195 | mov u64RetNanoTS_Hi, ecx
|
---|
196 | %define SAVED_u64RetNanoTS
|
---|
197 | mov eax, 1
|
---|
198 | cpuid
|
---|
199 | cmp u32ApicIdPlus, ebx
|
---|
200 | jne .ReadGip
|
---|
201 | %endif
|
---|
202 | %ifdef ASYNC_GIP
|
---|
203 | mov esi, [edi + SUPGIPCPU.u32TransactionId]
|
---|
204 | %else
|
---|
205 | mov esi, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32TransactionId]
|
---|
206 | %endif
|
---|
207 | cmp esi, u32TransactionId
|
---|
208 | jne .ReadGip
|
---|
209 | test esi, 1
|
---|
210 | jnz .ReadGip
|
---|
211 | %endif ; NEED_TRANSACTION_ID
|
---|
212 | %ifdef SAVED_u64RetNanoTS
|
---|
213 | mov ebx, u64RetNanoTS
|
---|
214 | mov ecx, u64RetNanoTS_Hi
|
---|
215 | %endif
|
---|
216 |
|
---|
217 | ;;
|
---|
218 | ;; Calc the timestamp.
|
---|
219 | ;;
|
---|
220 | ; u64RetNanoTS -= u64TSC;
|
---|
221 | sub ebx, u64TSC
|
---|
222 | sbb ecx, u64TSC_Hi
|
---|
223 |
|
---|
224 | ; if (u64RetNanoTS > u32UpdateIntervalTSC) -> jump
|
---|
225 | or ecx, ecx
|
---|
226 | jnz .OverFlow
|
---|
227 | cmp ebx, u32UpdateIntervalTSC
|
---|
228 | ja .OverFlow
|
---|
229 | mov eax, ebx
|
---|
230 | .ContinueCalcs: ; eax <= u32UpdateIntervalTSC
|
---|
231 | mul dword u32UpdateIntervalNS
|
---|
232 | div dword u32UpdateIntervalTSC
|
---|
233 | xor edx, edx
|
---|
234 |
|
---|
235 | ; u64RetNanoTS += u64CurNanoTS;
|
---|
236 | add eax, u64CurNanoTS
|
---|
237 | adc edx, u64CurNanoTS_Hi
|
---|
238 |
|
---|
239 | ;;
|
---|
240 | ;; Compare it with the previous one.
|
---|
241 | ;;
|
---|
242 | ; if (RT_LIKELY( u64RetNanoTS > u64PrevNanoTS
|
---|
243 | ; && u64RetNanoTS < u64PrevNanoTS + UINT64_C(86000000000000) /* 24h */))
|
---|
244 | ;; @todo optimize this compare (/me too tired).
|
---|
245 | mov ecx, u64PrevNanoTS_Hi
|
---|
246 | mov ebx, u64PrevNanoTS
|
---|
247 | cmp edx, ecx
|
---|
248 | ja .Compare2
|
---|
249 | jb .DeltaPrevTooBig
|
---|
250 | cmp eax, ebx
|
---|
251 | jbe .DeltaPrevTooBig
|
---|
252 |
|
---|
253 | .Compare2:
|
---|
254 | add ebx, 0x6F736000
|
---|
255 | adc ecx, 0x00004E37
|
---|
256 | cmp edx, ecx
|
---|
257 | jb .CompareDone
|
---|
258 | ja .DeltaPrevTooBig
|
---|
259 | cmp eax, ebx
|
---|
260 | jae .DeltaPrevTooBig
|
---|
261 | .CompareDone:
|
---|
262 |
|
---|
263 |
|
---|
264 | ;;
|
---|
265 | ;; Update the previous value with the u64RetNanoTS value.
|
---|
266 | ;;
|
---|
267 | .Update:
|
---|
268 | ; if (RT_LIKELY(ASMAtomicCmpXchgU64(&pData->u64Prev, u64RetNanoTS, u64PrevNanoTS)))
|
---|
269 | mov ebx, eax
|
---|
270 | mov ecx, edx
|
---|
271 | mov esi, pData
|
---|
272 | mov esi, [esi + RTTIMENANOTSDATA.pu64Prev]
|
---|
273 | mov eax, u64PrevNanoTS
|
---|
274 | mov edx, u64PrevNanoTS_Hi
|
---|
275 | lock cmpxchg8b [esi]
|
---|
276 | jnz .UpdateFailed
|
---|
277 |
|
---|
278 | .Updated:
|
---|
279 | mov eax, ebx
|
---|
280 | mov edx, ecx
|
---|
281 |
|
---|
282 | .Done:
|
---|
283 | mov esi, SavedESI
|
---|
284 | mov edi, SavedEDI
|
---|
285 | mov ebx, SavedEBX
|
---|
286 | leave
|
---|
287 | ret
|
---|
288 |
|
---|
289 |
|
---|
290 | ;;
|
---|
291 | ;; We've expired the interval, cap it. If we're here for the 2nd
|
---|
292 | ;; time without any GIP update in-between, the checks against
|
---|
293 | ;; pData->u64Prev below will force 1ns stepping.
|
---|
294 | ;;
|
---|
295 | .OverFlow:
|
---|
296 | ; u64Delta = u32UpdateIntervalTSC;
|
---|
297 | mov esi, pData
|
---|
298 | inc dword [esi + RTTIMENANOTSDATA.cExpired]
|
---|
299 | mov eax, u32UpdateIntervalTSC
|
---|
300 | jmp .ContinueCalcs
|
---|
301 |
|
---|
302 |
|
---|
303 | ;;
|
---|
304 | ;; u64DeltaPrev >= 24h
|
---|
305 | ;;
|
---|
306 | ;; eax:edx = u64RetNanoTS (to be adjusted)
|
---|
307 | ;;
|
---|
308 | .DeltaPrevTooBig:
|
---|
309 | ; uint64_t u64DeltaPrev = u64RetNanoTS - u64PrevNanoTS;
|
---|
310 | mov ebx, eax
|
---|
311 | sub ebx, u64PrevNanoTS
|
---|
312 | mov ecx, edx
|
---|
313 | sbb ecx, u64PrevNanoTS_Hi ; ebx:ecx = u64DeltaPrev
|
---|
314 |
|
---|
315 | ; else if ( (int64_t)u64DeltaPrev <= 0
|
---|
316 | ; && (int64_t)u64DeltaPrev + u32UpdateIntervalNS * 2 >= 0)
|
---|
317 | ; {
|
---|
318 | ; /* Occasional - u64RetNanoTS is in the recent 'past' relative the previous call. */
|
---|
319 | ; pData->c1nsSteps++;
|
---|
320 | ; u64RetNanoTS = u64PrevNanoTS + 1;
|
---|
321 | ; }
|
---|
322 | mov esi, u32UpdateIntervalNS
|
---|
323 | cmp ecx, 0
|
---|
324 | jl .PrevNotZero2ndTest
|
---|
325 | jg .DeltaPrevNotInRecentPast
|
---|
326 | cmp ebx, 0
|
---|
327 | ja .DeltaPrevNotInRecentPast
|
---|
328 |
|
---|
329 | .PrevNotZero2ndTest:
|
---|
330 | add esi, esi ; ASSUMES: u32UpdateIntervalNS * 2 <= 32-bit.
|
---|
331 | xor edi, edi
|
---|
332 | add esi, ebx
|
---|
333 | adc edi, ecx
|
---|
334 | test edi, edi
|
---|
335 | js .DeltaPrevNotInRecentPast
|
---|
336 |
|
---|
337 | .DeltaPrevInRecentPast:
|
---|
338 | mov esi, pData
|
---|
339 | inc dword [esi + RTTIMENANOTSDATA.c1nsSteps]
|
---|
340 | mov eax, u64PrevNanoTS
|
---|
341 | mov edx, u64PrevNanoTS_Hi
|
---|
342 | add eax, 1
|
---|
343 | adc edx, 0
|
---|
344 | jmp .Update
|
---|
345 |
|
---|
346 | .DeltaPrevNotInRecentPast:
|
---|
347 | ; else if (!u64PrevNanoTS) /* We're resuming (see TMVirtualResume). */
|
---|
348 | ; /* do nothing */;
|
---|
349 | cmp dword u64PrevNanoTS, 0
|
---|
350 | jne .DeltaPrevNotZero
|
---|
351 | cmp dword u64PrevNanoTS_Hi, 0
|
---|
352 | jne .DeltaPrevNotZero
|
---|
353 | jmp .Update
|
---|
354 |
|
---|
355 | .DeltaPrevNotZero:
|
---|
356 | ; else
|
---|
357 | ; {
|
---|
358 | ; /* Something has gone bust, if negative offset it's real bad. */
|
---|
359 | ; rtTimeNanoTSInternalBitch(pVM,
|
---|
360 | ; }
|
---|
361 |
|
---|
362 | ; call C function that does the bitching.
|
---|
363 | mov u64RetNanoTS, eax
|
---|
364 | mov u64RetNanoTS_Hi, edx
|
---|
365 |
|
---|
366 | mov edi, u64PrevNanoTS_Hi
|
---|
367 | mov esi, u64PrevNanoTS
|
---|
368 | push edi
|
---|
369 | push esi ; 4 - u64PrevNanoTS
|
---|
370 | push ecx
|
---|
371 | push ebx ; 3 - u64DeltaPrev
|
---|
372 | push edx
|
---|
373 | push eax ; 2 - u64RetNanoTS
|
---|
374 | mov eax, pData
|
---|
375 | push eax ; 1 - pData
|
---|
376 | call dword [eax + RTTIMENANOTSDATA.pfnBad]
|
---|
377 | add esp, 4*7
|
---|
378 |
|
---|
379 | mov eax, u64RetNanoTS
|
---|
380 | mov edx, u64RetNanoTS_Hi
|
---|
381 | jmp .Update
|
---|
382 |
|
---|
383 |
|
---|
384 | ;;
|
---|
385 | ;; Attempt updating the previous value, provided we're still ahead of it.
|
---|
386 | ;;
|
---|
387 | ;; There is no point in recalculating u64NanoTS because we got preempted or if
|
---|
388 | ;; we raced somebody while the GIP was updated, since these are events
|
---|
389 | ;; that might occur at any point in the return path as well.
|
---|
390 | ;;
|
---|
391 | ;; eax:edx = *pData->u64Prev
|
---|
392 | ;; ebx:ecx = u64RetNanoTS
|
---|
393 | ;;
|
---|
394 | ALIGNCODE(16)
|
---|
395 | .UpdateFailed:
|
---|
396 | mov edi, pData
|
---|
397 | lock inc dword [edi + RTTIMENANOTSDATA.cUpdateRaces]
|
---|
398 | ; for (i = 0; i < 10; i++)
|
---|
399 | mov edi, 10
|
---|
400 | .UpdateLoop:
|
---|
401 | ; if (u64PrevNanoTS >= u64NanoTS)
|
---|
402 | ; break;
|
---|
403 | cmp edx, ecx
|
---|
404 | jg .Updated
|
---|
405 | jne .UpdateLoopLess
|
---|
406 | cmp eax, ebx
|
---|
407 | jae .Updated
|
---|
408 | .UpdateLoopLess:
|
---|
409 | ; retry
|
---|
410 | lock cmpxchg8b [esi]
|
---|
411 | jz .Updated
|
---|
412 | dec edi
|
---|
413 | jnz .UpdateLoop
|
---|
414 | jmp .Updated
|
---|
415 |
|
---|
416 |
|
---|
417 | ;;
|
---|
418 | ;; The GIP is seemingly invalid, redo the discovery.
|
---|
419 | ;;
|
---|
420 | .Rediscover:
|
---|
421 | mov eax, pData
|
---|
422 | push eax
|
---|
423 | call [eax + RTTIMENANOTSDATA.pfnRediscover]
|
---|
424 | add esp, 4h
|
---|
425 | jmp .Done
|
---|
426 |
|
---|
427 |
|
---|
428 | %ifdef WITH_TSC_DELTA
|
---|
429 | ;;
|
---|
430 | ;; Unlikely branch for when we think the TSC delta might be invalid.
|
---|
431 | ;;
|
---|
432 | .TscDeltaPossiblyInvalid:
|
---|
433 | cmp dword [edi + SUPGIPCPU.i64TSCDelta + 4], 0x7fffffff
|
---|
434 | jne .TscDeltaValid
|
---|
435 | jmp .TscDeltaNotValid
|
---|
436 | %endif
|
---|
437 |
|
---|
438 | ;
|
---|
439 | ; Cleanup variables
|
---|
440 | ;
|
---|
441 | %undef pData
|
---|
442 | %undef u64Delta_Hi
|
---|
443 | %undef u64Delta
|
---|
444 | %undef u32UpdateIntervalNS
|
---|
445 | %undef u32UpdateIntervalTSC
|
---|
446 | %undef u64TSC_Hi
|
---|
447 | %undef u64TSC
|
---|
448 | %undef u64NanoTS_Hi
|
---|
449 | %undef u64NanoTS
|
---|
450 | %undef u64PrevNanoTS_Hi
|
---|
451 | %undef u64PrevNanoTS
|
---|
452 | %undef u32TransactionId
|
---|
453 | %undef u8ApicId
|
---|
454 |
|
---|
455 | %else ; AMD64
|
---|
456 |
|
---|
457 | ;;
|
---|
458 | ; The AMD64 assembly implementation of the assembly routines.
|
---|
459 | ;
|
---|
460 | ; @returns Nanosecond timestamp.
|
---|
461 | ; @param pData gcc:rdi msc:rcx Pointer to the nanosecond timestamp data.
|
---|
462 | ;
|
---|
463 | BEGINPROC rtTimeNanoTSInternalAsm
|
---|
464 | ;
|
---|
465 | ; Define variables and stack frame.
|
---|
466 | ;
|
---|
467 | %define SavedRBX [rbp - 08h]
|
---|
468 | %define SavedR12 [rbp - 10h]
|
---|
469 | %define SavedR13 [rbp - 18h]
|
---|
470 | %define SavedRDI [rbp - 20h]
|
---|
471 | %define SavedRSI [rbp - 28h]
|
---|
472 | %define TmpVar [rbp - 30h]
|
---|
473 | %define TmpVar2 [rbp - 38h]
|
---|
474 | %ifdef NEED_TRANSACTION_ID
|
---|
475 | %ifdef NEED_APIC_ID
|
---|
476 | %define SavedR14 [rbp - 40h]
|
---|
477 | %define SavedR15 [rbp - 48h]
|
---|
478 | %endif
|
---|
479 | %endif
|
---|
480 |
|
---|
481 | %define pData rdi
|
---|
482 |
|
---|
483 | %ifdef ASYNC_GIP
|
---|
484 | %define u64TSC rsi
|
---|
485 | %define pGip rsi
|
---|
486 | %ifdef NEED_APIC_ID
|
---|
487 | %define pGipCPU r8
|
---|
488 | %endif
|
---|
489 | %else
|
---|
490 | %define u64TSC r8
|
---|
491 | %define pGip rsi
|
---|
492 | %ifdef NEED_APIC_ID
|
---|
493 | %define pGipCPU r8
|
---|
494 | %endif
|
---|
495 | %endif
|
---|
496 | %define u32TransactionId r9d
|
---|
497 | %define u64CurNanoTS r10
|
---|
498 | %define u64PrevNanoTS r11 ; not parameter register
|
---|
499 | %define u32UpdateIntervalTSC r12d
|
---|
500 | %define u32UpdateIntervalTSC_64 r12
|
---|
501 | %define u32UpdateIntervalNS r13d
|
---|
502 | %define u32UpdateIntervalNS_64 r13
|
---|
503 | %undef u64SavedRetNanoTS
|
---|
504 | %undef u32ApicIdPlus
|
---|
505 | %ifdef NEED_TRANSACTION_ID
|
---|
506 | %ifdef NEED_APIC_ID
|
---|
507 | %define u64SavedRetNanoTS r14
|
---|
508 | %define u32ApicIdPlus r15d
|
---|
509 | %endif
|
---|
510 | %endif
|
---|
511 |
|
---|
512 | ;
|
---|
513 | ; The prolog.
|
---|
514 | ;
|
---|
515 | push rbp
|
---|
516 | mov rbp, rsp
|
---|
517 | %ifdef ASM_CALL64_MSC
|
---|
518 | sub rsp, 50h+20h
|
---|
519 | %else
|
---|
520 | sub rsp, 50h
|
---|
521 | %endif
|
---|
522 | mov SavedRBX, rbx
|
---|
523 | mov SavedR12, r12
|
---|
524 | mov SavedR13, r13
|
---|
525 | %ifdef ASM_CALL64_MSC
|
---|
526 | mov SavedRDI, rdi
|
---|
527 | mov SavedRSI, rsi
|
---|
528 | mov pData, rcx
|
---|
529 | %else
|
---|
530 | ;mov pData, rdi - already in rdi.
|
---|
531 | %endif
|
---|
532 | %ifdef SavedR14
|
---|
533 | mov SavedR14, r14
|
---|
534 | %endif
|
---|
535 | %ifdef SavedR15
|
---|
536 | mov SavedR15, r15
|
---|
537 | %endif
|
---|
538 |
|
---|
539 |
|
---|
540 | ;;
|
---|
541 | ;; Data fetch loop.
|
---|
542 | ;; We take great pain ensuring that data consistency here.
|
---|
543 | ;;
|
---|
544 | .ReadGip:
|
---|
545 |
|
---|
546 | ;
|
---|
547 | ; Load pGip - finding the GIP is fun...
|
---|
548 | ;
|
---|
549 | %ifdef RT_OS_WINDOWS
|
---|
550 | %ifdef IMPORTED_SUPLIB
|
---|
551 | %ifdef IN_RING0
|
---|
552 | mov rax, qword IMP(g_SUPGlobalInfoPage)
|
---|
553 | mov pGip, rax
|
---|
554 | %else
|
---|
555 | mov pGip, [IMP(g_pSUPGlobalInfoPage) wrt rip]
|
---|
556 | mov pGip, [pGip]
|
---|
557 | %endif
|
---|
558 | %else
|
---|
559 | mov pGip, [NAME(g_pSUPGlobalInfoPage) wrt rip]
|
---|
560 | %endif
|
---|
561 | %else
|
---|
562 | %ifdef IN_RING0
|
---|
563 | mov rax, qword NAME(g_SUPGlobalInfoPage)
|
---|
564 | mov pGip, rax
|
---|
565 | %else
|
---|
566 | mov pGip, [rel NAME(g_pSUPGlobalInfoPage) wrt ..gotpcrel]
|
---|
567 | mov pGip, [pGip]
|
---|
568 | %endif
|
---|
569 | %endif
|
---|
570 | or pGip, pGip
|
---|
571 | jz .Rediscover
|
---|
572 | cmp dword [pGip + SUPGLOBALINFOPAGE.u32Magic], SUPGLOBALINFOPAGE_MAGIC
|
---|
573 | jne .Rediscover
|
---|
574 |
|
---|
575 | ;
|
---|
576 | ; pGipCPU, setting u32ApicIdPlus if necessary.
|
---|
577 | ;
|
---|
578 | %ifdef NEED_APIC_ID
|
---|
579 | ; u8ApicId = ASMGetApicId();
|
---|
580 | mov eax, 1
|
---|
581 | cpuid ; expensive
|
---|
582 | %ifdef NEED_TRANSACTION_ID
|
---|
583 | mov u32ApicIdPlus, ebx
|
---|
584 | %endif
|
---|
585 | ; pGipCPU = &pGip->aCPU[pGip->aiCpuFromApicId[u8ApicId]];
|
---|
586 | shr ebx, 24
|
---|
587 | movzx eax, word [pGip + rbx * 2 + SUPGLOBALINFOPAGE.aiCpuFromApicId]
|
---|
588 | imul eax, SUPGIPCPU_size
|
---|
589 | lea pGipCPU, [pGip + rax + SUPGLOBALINFOPAGE.aCPUs]
|
---|
590 | %endif
|
---|
591 |
|
---|
592 | %ifdef NEED_TRANSACTION_ID
|
---|
593 | ;
|
---|
594 | ; Serialized loading of u32TransactionId.
|
---|
595 | ;
|
---|
596 | %ifdef ASYNC_GIP
|
---|
597 | mov u32TransactionId, [pGipCPU + SUPGIPCPU.u32TransactionId]
|
---|
598 | %else
|
---|
599 | mov u32TransactionId, [pGip + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32TransactionId]
|
---|
600 | %endif
|
---|
601 | %ifdef USE_LFENCE
|
---|
602 | lfence
|
---|
603 | %else
|
---|
604 | lock xor dword TmpVar, 0
|
---|
605 | %endif
|
---|
606 | %endif
|
---|
607 |
|
---|
608 | ;
|
---|
609 | ; Load the data and TSC.
|
---|
610 | ;
|
---|
611 | mov u32UpdateIntervalNS, [pGip + SUPGLOBALINFOPAGE.u32UpdateIntervalNS]
|
---|
612 | %ifdef ASYNC_GIP
|
---|
613 | mov u32UpdateIntervalTSC, [pGipCPU + SUPGIPCPU.u32UpdateIntervalTSC]
|
---|
614 | %else
|
---|
615 | mov u32UpdateIntervalTSC, [pGip + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32UpdateIntervalTSC]
|
---|
616 | %endif
|
---|
617 |
|
---|
618 | rdtsc
|
---|
619 | mov u64PrevNanoTS, [pData + RTTIMENANOTSDATA.pu64Prev]
|
---|
620 | mov u64PrevNanoTS, [u64PrevNanoTS]
|
---|
621 | shl rdx, 32
|
---|
622 | or rax, rdx ; rax is u64RetNanoTS.
|
---|
623 | %ifdef WITH_TSC_DELTA
|
---|
624 | mov rdx, [pGipCPU + SUPGIPCPU.i64TSCDelta]
|
---|
625 | mov u64CurNanoTS, 0x7fffffffffffffff ; INT64_MAX - temporarily borrowing u64CurNanoTS
|
---|
626 | cmp rdx, u64CurNanoTS
|
---|
627 | je .TscDeltaNotValid
|
---|
628 | sub rax, rdx
|
---|
629 | .TscDeltaNotValid:
|
---|
630 | %endif
|
---|
631 | %ifdef u64SavedRetNanoTS ; doing this here may save a tick or so?
|
---|
632 | mov u64SavedRetNanoTS, rax
|
---|
633 | %endif
|
---|
634 |
|
---|
635 | %ifdef ASYNC_GIP
|
---|
636 | mov u64CurNanoTS, [pGipCPU + SUPGIPCPU.u64NanoTS]
|
---|
637 | mov u64TSC, [pGipCPU + SUPGIPCPU.u64TSC] ; transhes pGIP!
|
---|
638 | %else
|
---|
639 | mov u64CurNanoTS, [pGip + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64NanoTS]
|
---|
640 | mov u64TSC, [pGip + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64TSC] ; trashes pGipCPU!
|
---|
641 | %endif
|
---|
642 |
|
---|
643 |
|
---|
644 | %ifdef NEED_TRANSACTION_ID
|
---|
645 | ;
|
---|
646 | ; Check that the GIP and CPU didn't change.
|
---|
647 | ;
|
---|
648 | ; It is crucial that the rdtsc instruction has completed before
|
---|
649 | ; we check the transaction id. The LOCK prefixed instruction with
|
---|
650 | ; dependency on the RDTSC result should do the trick, I think.
|
---|
651 | ; CPUID is serializing, so the async path is safe by default.
|
---|
652 | ;
|
---|
653 | %ifdef NEED_APIC_ID
|
---|
654 | mov eax, 1
|
---|
655 | cpuid
|
---|
656 | cmp u32ApicIdPlus, ebx
|
---|
657 | jne .ReadGip
|
---|
658 | %else
|
---|
659 | lock xor qword TmpVar, rax
|
---|
660 | %endif
|
---|
661 | %ifdef ASYNC_GIP
|
---|
662 | cmp u32TransactionId, [pGipCPU + SUPGIPCPU.u32TransactionId]
|
---|
663 | %else
|
---|
664 | cmp u32TransactionId, [pGip + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32TransactionId]
|
---|
665 | %endif
|
---|
666 | jne .ReadGip
|
---|
667 | test u32TransactionId, 1
|
---|
668 | jnz .ReadGip
|
---|
669 | %ifdef u64SavedRetNanoTS
|
---|
670 | mov rax, u64SavedRetNanoTS ; rax is u64RetNanoTS.
|
---|
671 | %endif
|
---|
672 | %endif ; NEED_TRANSACTION_ID
|
---|
673 |
|
---|
674 |
|
---|
675 | ;;
|
---|
676 | ;; Calc the timestamp.
|
---|
677 | ;;
|
---|
678 | ; u64RetNanoTS -= u64TSC;
|
---|
679 | sub rax, u64TSC
|
---|
680 | xor edx, edx
|
---|
681 |
|
---|
682 | ; if (u64RetNanoTS > u32UpdateIntervalTSC) -> jump
|
---|
683 | cmp rax, u32UpdateIntervalTSC_64
|
---|
684 | ja .OverFlow
|
---|
685 | .ContinueCalcs: ; edx = 0; eax <= u32UpdateIntervalTSC
|
---|
686 | mul u32UpdateIntervalNS
|
---|
687 | div u32UpdateIntervalTSC
|
---|
688 |
|
---|
689 | ; u64RetNanoTS += u64CurNanoTS;
|
---|
690 | add rax, u64CurNanoTS
|
---|
691 |
|
---|
692 |
|
---|
693 | ;;
|
---|
694 | ;; Compare it with the previous one.
|
---|
695 | ;;
|
---|
696 | ; if (RT_LIKELY( u64RetNanoTS > u64PrevNanoTS
|
---|
697 | ; && u64RetNanoTS < u64PrevNanoTS + UINT64_C(86000000000000) /* 24h */))
|
---|
698 | ; /* Frequent - less than 24h since last call. */;
|
---|
699 | cmp rax, u64PrevNanoTS
|
---|
700 | jbe .DeltaPrevTooBig
|
---|
701 | mov ecx, 5
|
---|
702 | shl rcx, 44 ; close enough
|
---|
703 | add rcx, u64PrevNanoTS
|
---|
704 | cmp rax, rcx
|
---|
705 | jae .DeltaPrevTooBig
|
---|
706 |
|
---|
707 |
|
---|
708 | ;;
|
---|
709 | ;; Update the previous value.
|
---|
710 | ;;
|
---|
711 | .Update:
|
---|
712 | ; if (RT_LIKELY(ASMAtomicCmpXchgU64(&pData->u64Prev, u64RetNanoTS, u64PrevNanoTS)))
|
---|
713 | mov rbx, [pData + RTTIMENANOTSDATA.pu64Prev]
|
---|
714 | mov rcx, rax
|
---|
715 | mov rax, u64PrevNanoTS
|
---|
716 | lock cmpxchg [rbx], rcx
|
---|
717 | jnz .UpdateFailed
|
---|
718 |
|
---|
719 | .Updated:
|
---|
720 | mov rax, rcx
|
---|
721 |
|
---|
722 | .Done:
|
---|
723 | mov rbx, SavedRBX
|
---|
724 | mov r12, SavedR12
|
---|
725 | mov r13, SavedR13
|
---|
726 | %ifdef SavedR14
|
---|
727 | mov r14, SavedR14
|
---|
728 | %endif
|
---|
729 | %ifdef SavedR15
|
---|
730 | mov r15, SavedR15
|
---|
731 | %endif
|
---|
732 | %ifdef ASM_CALL64_MSC
|
---|
733 | mov rdi, SavedRDI
|
---|
734 | mov rsi, SavedRSI
|
---|
735 | %endif
|
---|
736 | leave
|
---|
737 | ret
|
---|
738 |
|
---|
739 |
|
---|
740 | ;;
|
---|
741 | ;; We've expired the interval, cap it. If we're here for the 2nd
|
---|
742 | ;; time without any GIP update in-between, the checks against
|
---|
743 | ;; pData->u64Prev below will force 1ns stepping.
|
---|
744 | ;;
|
---|
745 | ALIGNCODE(16)
|
---|
746 | .OverFlow:
|
---|
747 | ; u64RetNanoTS = u32UpdateIntervalTSC;
|
---|
748 | inc dword [pData + RTTIMENANOTSDATA.cExpired]
|
---|
749 | mov eax, u32UpdateIntervalTSC
|
---|
750 | jmp .ContinueCalcs
|
---|
751 |
|
---|
752 |
|
---|
753 | ;;
|
---|
754 | ;; u64DeltaPrev >= 24h
|
---|
755 | ;;
|
---|
756 | ;; rax = u64RetNanoTS (to be adjusted)
|
---|
757 | ;;
|
---|
758 | ALIGNCODE(16)
|
---|
759 | .DeltaPrevTooBig:
|
---|
760 | ; uint64_t u64DeltaPrev = u64RetNanoTS - u64PrevNanoTS;
|
---|
761 | mov rbx, rax
|
---|
762 | sub rbx, u64PrevNanoTS
|
---|
763 |
|
---|
764 | ; else if ( (int64_t)u64DeltaPrev <= 0
|
---|
765 | ; && (int64_t)u64DeltaPrev + u32UpdateIntervalNS * 2 >= 0)
|
---|
766 | ; {
|
---|
767 | ; /* Occasional - u64NanoTS is in the recent 'past' relative the previous call. */
|
---|
768 | ; pData->c1nsSteps++;
|
---|
769 | ; u64RetNanoTS = u64PrevNanoTS + 1;
|
---|
770 | ; }
|
---|
771 | test rbx, rbx
|
---|
772 | jg .DeltaPrevNotInRecentPast
|
---|
773 |
|
---|
774 | lea rdx, [u32UpdateIntervalNS_64 + u32UpdateIntervalNS_64]
|
---|
775 | add rdx, rbx
|
---|
776 | js .DeltaPrevNotInRecentPast
|
---|
777 |
|
---|
778 | ; body
|
---|
779 | inc dword [pData + RTTIMENANOTSDATA.c1nsSteps]
|
---|
780 | lea rax, [u64PrevNanoTS + 1]
|
---|
781 | jmp .Update
|
---|
782 |
|
---|
783 | ; else if (!u64PrevNanoTS) /* We're resuming (see TMVirtualResume) / first call. */
|
---|
784 | ; /* do nothing */;
|
---|
785 | .DeltaPrevNotInRecentPast:
|
---|
786 | or u64PrevNanoTS, u64PrevNanoTS
|
---|
787 | jz .Update
|
---|
788 |
|
---|
789 | ; else
|
---|
790 | ; {
|
---|
791 | ; /* Something has gone bust, if negative offset it's real bad. */
|
---|
792 | ; rtTimeNanoTSInternalBitch(pVM,
|
---|
793 | ; }
|
---|
794 |
|
---|
795 | ; call C function that does the bitching.
|
---|
796 | mov TmpVar, rax
|
---|
797 | mov TmpVar2, pData
|
---|
798 |
|
---|
799 | %ifdef ASM_CALL64_MSC
|
---|
800 | mov rcx, pData ; param 1 - pData
|
---|
801 | mov rdx, rax ; param 2 - u64RetNanoTS
|
---|
802 | mov r8, rbx ; param 3 - u64DeltaPrev
|
---|
803 | mov r9, u64PrevNanoTS ; param 4 - u64PrevNanoTS
|
---|
804 | %else
|
---|
805 | ;mov rdi, pData - already in rdi; param 1 - pData
|
---|
806 | mov rsi, rax ; param 2 - u64RetNanoTS
|
---|
807 | mov rdx, rbx ; param 3 - u64DeltaPrev
|
---|
808 | mov rcx, u64PrevNanoTS ; param 4 - u64PrevNanoTS
|
---|
809 | %endif
|
---|
810 | call qword [pData + RTTIMENANOTSDATA.pfnBad]
|
---|
811 |
|
---|
812 | mov rax, TmpVar
|
---|
813 | mov pData, TmpVar2
|
---|
814 | jmp .Update
|
---|
815 |
|
---|
816 |
|
---|
817 | ;;
|
---|
818 | ;; Attempt updating the previous value, provided we're still ahead of it.
|
---|
819 | ;;
|
---|
820 | ;; There is no point in recalculating u64NanoTS because we got preempted or if
|
---|
821 | ;; we raced somebody while the GIP was updated, since these are events
|
---|
822 | ;; that might occur at any point in the return path as well.
|
---|
823 | ;;
|
---|
824 | ;; rax = *pData->u64Prev;
|
---|
825 | ;; rcx = u64RetNanoTS
|
---|
826 | ;;
|
---|
827 | ALIGNCODE(16)
|
---|
828 | .UpdateFailed:
|
---|
829 | lock inc dword [pData + RTTIMENANOTSDATA.cUpdateRaces]
|
---|
830 | ; for (i = 0; i < 10; i++)
|
---|
831 | mov edx, 10
|
---|
832 | .UpdateLoop:
|
---|
833 | ; if (u64PrevNanoTS >= u64RetNanoTS)
|
---|
834 | ; break;
|
---|
835 | cmp rax, rcx
|
---|
836 | jge .Updated
|
---|
837 | .UpdateLoopLess:
|
---|
838 | ; retry
|
---|
839 | lock cmpxchg [rbx], rcx
|
---|
840 | jz .Updated
|
---|
841 | dec edx
|
---|
842 | jnz .UpdateLoop
|
---|
843 | jmp .Updated
|
---|
844 |
|
---|
845 |
|
---|
846 | ;;
|
---|
847 | ;; The GIP is seemingly invalid, redo the discovery.
|
---|
848 | ;;
|
---|
849 | .Rediscover:
|
---|
850 | %ifdef ASM_CALL64_MSC
|
---|
851 | mov rcx, pData
|
---|
852 | %else
|
---|
853 | ; mov rdi, pData - already in rdi
|
---|
854 | %endif
|
---|
855 | call [pData + RTTIMENANOTSDATA.pfnRediscover]
|
---|
856 | jmp .Done
|
---|
857 |
|
---|
858 |
|
---|
859 | ;
|
---|
860 | ; Cleanup variables
|
---|
861 | ;
|
---|
862 | %undef SavedRBX
|
---|
863 | %undef SavedR12
|
---|
864 | %undef SavedR13
|
---|
865 | %undef SavedR14
|
---|
866 | %undef SavedR15
|
---|
867 | %undef SavedRDI
|
---|
868 | %undef SavedRSI
|
---|
869 | %undef pData
|
---|
870 | %undef TmpVar
|
---|
871 | %undef u64TSC
|
---|
872 | %undef pGip
|
---|
873 | %undef pGipCPU
|
---|
874 | %undef u32TransactionId
|
---|
875 | %undef u64CurNanoTS
|
---|
876 | %undef u64PrevNanoTS
|
---|
877 | %undef u32UpdateIntervalTSC
|
---|
878 | %undef u32UpdateIntervalTSC_64
|
---|
879 | %undef u32UpdateIntervalNS
|
---|
880 | %undef u64SavedRetNanoTS
|
---|
881 | %undef u32ApicIdPlus
|
---|
882 |
|
---|
883 | %endif ; AMD64
|
---|
884 | ENDPROC rtTimeNanoTSInternalAsm
|
---|
885 |
|
---|
886 |
|
---|
887 |
|
---|