VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/PGMR0DynMap.cpp@ 18768

Last change on this file since 18768 was 15436, checked in by vboxsync, 16 years ago

PGMR0DynMap: Corrected PGMR0DYNMAP_PAGES_PER_CPU_MIN (>= max set size). Added set size stats.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 78.0 KB
Line 
1/* $Id: PGMR0DynMap.cpp 15436 2008-12-13 11:59:30Z vboxsync $ */
2/** @file
3 * PGM - Page Manager and Monitor, ring-0 dynamic mapping cache.
4 */
5
6/*
7 * Copyright (C) 2008 Sun Microsystems, Inc.
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18 * Clara, CA 95054 USA or visit http://www.sun.com if you need
19 * additional information or have any questions.
20 */
21
22/*******************************************************************************
23* Internal Functions *
24*******************************************************************************/
25#define LOG_GROUP LOG_GROUP_PGM
26#include <VBox/pgm.h>
27#include "../PGMInternal.h"
28#include <VBox/vm.h>
29#include <VBox/sup.h>
30#include <VBox/err.h>
31#include <iprt/asm.h>
32#include <iprt/alloc.h>
33#include <iprt/assert.h>
34#include <iprt/cpuset.h>
35#include <iprt/memobj.h>
36#include <iprt/mp.h>
37#include <iprt/semaphore.h>
38#include <iprt/spinlock.h>
39#include <iprt/string.h>
40
41
42/*******************************************************************************
43* Defined Constants And Macros *
44*******************************************************************************/
45/** The max size of the mapping cache (in pages). */
46#define PGMR0DYNMAP_MAX_PAGES ((16*_1M) >> PAGE_SHIFT)
47/** The small segment size that is adopted on out-of-memory conditions with a
48 * single big segment. */
49#define PGMR0DYNMAP_SMALL_SEG_PAGES 128
50/** The number of pages we reserve per CPU. */
51#define PGMR0DYNMAP_PAGES_PER_CPU 256
52/** The minimum number of pages we reserve per CPU.
53 * This must be equal or larger than the autoset size. */
54#define PGMR0DYNMAP_PAGES_PER_CPU_MIN 64
55/** The number of guard pages.
56 * @remarks Never do tuning of the hashing or whatnot with a strict build! */
57#if defined(VBOX_STRICT)
58# define PGMR0DYNMAP_GUARD_PAGES 1
59#else
60# define PGMR0DYNMAP_GUARD_PAGES 0
61#endif
62/** The dummy physical address of guard pages. */
63#define PGMR0DYNMAP_GUARD_PAGE_HCPHYS UINT32_C(0x7777feed)
64/** The dummy reference count of guard pages. (Must be non-zero.) */
65#define PGMR0DYNMAP_GUARD_PAGE_REF_COUNT INT32_C(0x7777feed)
66#if 0
67/** Define this to just clear the present bit on guard pages.
68 * The alternative is to replace the entire PTE with an bad not-present
69 * PTE. Either way, XNU will screw us. :-/ */
70#define PGMR0DYNMAP_GUARD_NP
71#endif
72/** The dummy PTE value for a page. */
73#define PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE X86_PTE_PG_MASK
74/** The dummy PTE value for a page. */
75#define PGMR0DYNMAP_GUARD_PAGE_PAE_PTE UINT64_MAX /*X86_PTE_PAE_PG_MASK*/
76/** Calcs the overload threshold. Current set at 50%. */
77#define PGMR0DYNMAP_CALC_OVERLOAD(cPages) ((cPages) / 2)
78
79#if 0
80/* Assertions causes panics if preemption is disabled, this can be used to work aroudn that. */
81//#define RTSpinlockAcquire(a,b) do {} while (0)
82//#define RTSpinlockRelease(a,b) do {} while (0)
83#endif
84
85
86/*******************************************************************************
87* Structures and Typedefs *
88*******************************************************************************/
89/**
90 * Ring-0 dynamic mapping cache segment.
91 *
92 * The dynamic mapping cache can be extended with additional segments if the
93 * load is found to be too high. This done the next time a VM is created, under
94 * the protection of the init mutex. The arrays is reallocated and the new
95 * segment is added to the end of these. Nothing is rehashed of course, as the
96 * indexes / addresses must remain unchanged.
97 *
98 * This structure is only modified while owning the init mutex or during module
99 * init / term.
100 */
101typedef struct PGMR0DYNMAPSEG
102{
103 /** Pointer to the next segment. */
104 struct PGMR0DYNMAPSEG *pNext;
105 /** The memory object for the virtual address range that we're abusing. */
106 RTR0MEMOBJ hMemObj;
107 /** The start page in the cache. (I.e. index into the arrays.) */
108 uint16_t iPage;
109 /** The number of pages this segment contributes. */
110 uint16_t cPages;
111 /** The number of page tables. */
112 uint16_t cPTs;
113 /** The memory objects for the page tables. */
114 RTR0MEMOBJ ahMemObjPTs[1];
115} PGMR0DYNMAPSEG;
116/** Pointer to a ring-0 dynamic mapping cache segment. */
117typedef PGMR0DYNMAPSEG *PPGMR0DYNMAPSEG;
118
119
120/**
121 * Ring-0 dynamic mapping cache entry.
122 *
123 * This structure tracks
124 */
125typedef struct PGMR0DYNMAPENTRY
126{
127 /** The physical address of the currently mapped page.
128 * This is duplicate for three reasons: cache locality, cache policy of the PT
129 * mappings and sanity checks. */
130 RTHCPHYS HCPhys;
131 /** Pointer to the page. */
132 void *pvPage;
133 /** The number of references. */
134 int32_t volatile cRefs;
135 /** PTE pointer union. */
136 union PGMR0DYNMAPENTRY_PPTE
137 {
138 /** PTE pointer, 32-bit legacy version. */
139 PX86PTE pLegacy;
140 /** PTE pointer, PAE version. */
141 PX86PTEPAE pPae;
142 /** PTE pointer, the void version. */
143 void *pv;
144 } uPte;
145 /** CPUs that haven't invalidated this entry after it's last update. */
146 RTCPUSET PendingSet;
147} PGMR0DYNMAPENTRY;
148/** Pointer to a ring-0 dynamic mapping cache entry. */
149typedef PGMR0DYNMAPENTRY *PPGMR0DYNMAPENTRY;
150
151
152/**
153 * Ring-0 dynamic mapping cache.
154 *
155 * This is initialized during VMMR0 module init but no segments are allocated at
156 * that time. Segments will be added when the first VM is started and removed
157 * again when the last VM shuts down, thus avoid consuming memory while dormant.
158 * At module termination, the remaining bits will be freed up.
159 */
160typedef struct PGMR0DYNMAP
161{
162 /** The usual magic number / eye catcher (PGMR0DYNMAP_MAGIC). */
163 uint32_t u32Magic;
164 /** Spinlock serializing the normal operation of the cache. */
165 RTSPINLOCK hSpinlock;
166 /** Array for tracking and managing the pages. */
167 PPGMR0DYNMAPENTRY paPages;
168 /** The cache size given as a number of pages. */
169 uint32_t cPages;
170 /** Whether it's 32-bit legacy or PAE/AMD64 paging mode. */
171 bool fLegacyMode;
172 /** The current load.
173 * This does not include guard pages. */
174 uint32_t cLoad;
175 /** The max load ever.
176 * This is maintained to get trigger adding of more mapping space. */
177 uint32_t cMaxLoad;
178 /** Initialization / termination lock. */
179 RTSEMFASTMUTEX hInitLock;
180 /** The number of guard pages. */
181 uint32_t cGuardPages;
182 /** The number of users (protected by hInitLock). */
183 uint32_t cUsers;
184 /** Array containing a copy of the original page tables.
185 * The entries are either X86PTE or X86PTEPAE according to fLegacyMode. */
186 void *pvSavedPTEs;
187 /** List of segments. */
188 PPGMR0DYNMAPSEG pSegHead;
189 /** The paging mode. */
190 SUPPAGINGMODE enmPgMode;
191} PGMR0DYNMAP;
192/** Pointer to the ring-0 dynamic mapping cache */
193typedef PGMR0DYNMAP *PPGMR0DYNMAP;
194
195/** PGMR0DYNMAP::u32Magic. (Jens Christian Bugge Wesseltoft) */
196#define PGMR0DYNMAP_MAGIC 0x19640201
197
198
199/**
200 * Paging level data.
201 */
202typedef struct PGMR0DYNMAPPGLVL
203{
204 uint32_t cLevels; /**< The number of levels. */
205 struct
206 {
207 RTHCPHYS HCPhys; /**< The address of the page for the current level,
208 * i.e. what hMemObj/hMapObj is currently mapping. */
209 RTHCPHYS fPhysMask; /**< Mask for extracting HCPhys from uEntry. */
210 RTR0MEMOBJ hMemObj; /**< Memory object for HCPhys, PAGE_SIZE. */
211 RTR0MEMOBJ hMapObj; /**< Mapping object for hMemObj. */
212 uint32_t fPtrShift; /**< The pointer shift count. */
213 uint64_t fPtrMask; /**< The mask to apply to the shifted pointer to get the table index. */
214 uint64_t fAndMask; /**< And mask to check entry flags. */
215 uint64_t fResMask; /**< The result from applying fAndMask. */
216 union
217 {
218 void *pv; /**< hMapObj address. */
219 PX86PGUINT paLegacy; /**< Legacy table view. */
220 PX86PGPAEUINT paPae; /**< PAE/AMD64 table view. */
221 } u;
222 } a[4];
223} PGMR0DYNMAPPGLVL;
224/** Pointer to paging level data. */
225typedef PGMR0DYNMAPPGLVL *PPGMR0DYNMAPPGLVL;
226
227
228/*******************************************************************************
229* Global Variables *
230*******************************************************************************/
231/** Pointer to the ring-0 dynamic mapping cache. */
232static PPGMR0DYNMAP g_pPGMR0DynMap;
233/** For overflow testing. */
234static bool g_fPGMR0DynMapTestRunning = false;
235
236
237/*******************************************************************************
238* Internal Functions *
239*******************************************************************************/
240static void pgmR0DynMapReleasePage(PPGMR0DYNMAP pThis, uint32_t iPage, uint32_t cRefs);
241static int pgmR0DynMapSetup(PPGMR0DYNMAP pThis);
242static int pgmR0DynMapExpand(PPGMR0DYNMAP pThis);
243static void pgmR0DynMapTearDown(PPGMR0DYNMAP pThis);
244#if 0 /*def DEBUG*/
245static int pgmR0DynMapTest(PVM pVM);
246#endif
247
248
249/**
250 * Initializes the ring-0 dynamic mapping cache.
251 *
252 * @returns VBox status code.
253 */
254VMMR0DECL(int) PGMR0DynMapInit(void)
255{
256 Assert(!g_pPGMR0DynMap);
257
258 /*
259 * Create and initialize the cache instance.
260 */
261 PPGMR0DYNMAP pThis = (PPGMR0DYNMAP)RTMemAllocZ(sizeof(*pThis));
262 AssertLogRelReturn(pThis, VERR_NO_MEMORY);
263 int rc = VINF_SUCCESS;
264 pThis->enmPgMode = SUPR0GetPagingMode();
265 switch (pThis->enmPgMode)
266 {
267 case SUPPAGINGMODE_32_BIT:
268 case SUPPAGINGMODE_32_BIT_GLOBAL:
269 pThis->fLegacyMode = false;
270 break;
271 case SUPPAGINGMODE_PAE:
272 case SUPPAGINGMODE_PAE_GLOBAL:
273 case SUPPAGINGMODE_PAE_NX:
274 case SUPPAGINGMODE_PAE_GLOBAL_NX:
275 case SUPPAGINGMODE_AMD64:
276 case SUPPAGINGMODE_AMD64_GLOBAL:
277 case SUPPAGINGMODE_AMD64_NX:
278 case SUPPAGINGMODE_AMD64_GLOBAL_NX:
279 pThis->fLegacyMode = false;
280 break;
281 default:
282 rc = VERR_INTERNAL_ERROR;
283 break;
284 }
285 if (RT_SUCCESS(rc))
286 {
287 rc = RTSemFastMutexCreate(&pThis->hInitLock);
288 if (RT_SUCCESS(rc))
289 {
290 rc = RTSpinlockCreate(&pThis->hSpinlock);
291 if (RT_SUCCESS(rc))
292 {
293 pThis->u32Magic = PGMR0DYNMAP_MAGIC;
294 g_pPGMR0DynMap = pThis;
295 return VINF_SUCCESS;
296 }
297 RTSemFastMutexDestroy(pThis->hInitLock);
298 }
299 }
300 RTMemFree(pThis);
301 return rc;
302}
303
304
305/**
306 * Terminates the ring-0 dynamic mapping cache.
307 */
308VMMR0DECL(void) PGMR0DynMapTerm(void)
309{
310 /*
311 * Destroy the cache.
312 *
313 * There is not supposed to be any races here, the loader should
314 * make sure about that. So, don't bother locking anything.
315 *
316 * The VM objects should all be destroyed by now, so there is no
317 * dangling users or anything like that to clean up. This routine
318 * is just a mirror image of PGMR0DynMapInit.
319 */
320 PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
321 if (pThis)
322 {
323 AssertPtr(pThis);
324 g_pPGMR0DynMap = NULL;
325
326 /* This should *never* happen, but in case it does try not to leak memory. */
327 AssertLogRelMsg(!pThis->cUsers && !pThis->paPages && !pThis->pvSavedPTEs && !pThis->cPages,
328 ("cUsers=%d paPages=%p pvSavedPTEs=%p cPages=%#x\n",
329 pThis->cUsers, pThis->paPages, pThis->pvSavedPTEs, pThis->cPages));
330 if (pThis->paPages)
331 pgmR0DynMapTearDown(pThis);
332
333 /* Free the associated resources. */
334 RTSemFastMutexDestroy(pThis->hInitLock);
335 pThis->hInitLock = NIL_RTSEMFASTMUTEX;
336 RTSpinlockDestroy(pThis->hSpinlock);
337 pThis->hSpinlock = NIL_RTSPINLOCK;
338 pThis->u32Magic = UINT32_MAX;
339 RTMemFree(pThis);
340 }
341}
342
343
344/**
345 * Initializes the dynamic mapping cache for a new VM.
346 *
347 * @returns VBox status code.
348 * @param pVM Pointer to the shared VM structure.
349 */
350VMMR0DECL(int) PGMR0DynMapInitVM(PVM pVM)
351{
352 AssertMsgReturn(!pVM->pgm.s.pvR0DynMapUsed, ("%p (pThis=%p)\n", pVM->pgm.s.pvR0DynMapUsed, g_pPGMR0DynMap), VERR_WRONG_ORDER);
353
354 /*
355 * Initialize the auto sets.
356 */
357 VMCPUID idCpu = pVM->cCPUs;
358 AssertReturn(idCpu > 0 && idCpu <= VMCPU_MAX_CPU_COUNT, VERR_INTERNAL_ERROR);
359 while (idCpu-- > 0)
360 {
361 PPGMMAPSET pSet = &pVM->aCpus[idCpu].pgm.s.AutoSet;
362 uint32_t j = RT_ELEMENTS(pSet->aEntries);
363 while (j-- > 0)
364 {
365 pSet->aEntries[j].iPage = UINT16_MAX;
366 pSet->aEntries[j].cRefs = 0;
367 pSet->aEntries[j].pvPage = NULL;
368 pSet->aEntries[j].HCPhys = NIL_RTHCPHYS;
369 }
370 pSet->cEntries = PGMMAPSET_CLOSED;
371 pSet->iSubset = UINT32_MAX;
372 pSet->iCpu = -1;
373 memset(&pSet->aiHashTable[0], 0xff, sizeof(pSet->aiHashTable));
374 }
375
376 /*
377 * Do we need the cache? Skip the last bit if we don't.
378 */
379 if (!VMMIsHwVirtExtForced(pVM))
380 return VINF_SUCCESS;
381
382 /*
383 * Reference and if necessary setup or expand the cache.
384 */
385 PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
386 AssertPtrReturn(pThis, VERR_INTERNAL_ERROR);
387 int rc = RTSemFastMutexRequest(pThis->hInitLock);
388 AssertLogRelRCReturn(rc, rc);
389
390 pThis->cUsers++;
391 if (pThis->cUsers == 1)
392 {
393 rc = pgmR0DynMapSetup(pThis);
394#if 0 /*def DEBUG*/
395 if (RT_SUCCESS(rc))
396 {
397 rc = pgmR0DynMapTest(pVM);
398 if (RT_FAILURE(rc))
399 pgmR0DynMapTearDown(pThis);
400 }
401#endif
402 }
403 else if (pThis->cMaxLoad > PGMR0DYNMAP_CALC_OVERLOAD(pThis->cPages - pThis->cGuardPages))
404 rc = pgmR0DynMapExpand(pThis);
405 if (RT_SUCCESS(rc))
406 pVM->pgm.s.pvR0DynMapUsed = pThis;
407 else
408 pThis->cUsers--;
409
410 RTSemFastMutexRelease(pThis->hInitLock);
411 return rc;
412}
413
414
415/**
416 * Terminates the dynamic mapping cache usage for a VM.
417 *
418 * @param pVM Pointer to the shared VM structure.
419 */
420VMMR0DECL(void) PGMR0DynMapTermVM(PVM pVM)
421{
422 /*
423 * Return immediately if we're not using the cache.
424 */
425 if (!pVM->pgm.s.pvR0DynMapUsed)
426 return;
427
428 PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
429 AssertPtrReturnVoid(pThis);
430
431 int rc = RTSemFastMutexRequest(pThis->hInitLock);
432 AssertLogRelRCReturnVoid(rc);
433
434 if (pVM->pgm.s.pvR0DynMapUsed == pThis)
435 {
436 pVM->pgm.s.pvR0DynMapUsed = NULL;
437
438#ifdef VBOX_STRICT
439 PGMR0DynMapAssertIntegrity();
440#endif
441
442 /*
443 * Clean up and check the auto sets.
444 */
445 VMCPUID idCpu = pVM->cCPUs;
446 while (idCpu-- > 0)
447 {
448 PPGMMAPSET pSet = &pVM->aCpus[idCpu].pgm.s.AutoSet;
449 uint32_t j = pSet->cEntries;
450 if (j <= RT_ELEMENTS(pSet->aEntries))
451 {
452 /*
453 * The set is open, close it.
454 */
455 while (j-- > 0)
456 {
457 int32_t cRefs = pSet->aEntries[j].cRefs;
458 uint32_t iPage = pSet->aEntries[j].iPage;
459 LogRel(("PGMR0DynMapTermVM: %d dangling refs to %#x\n", cRefs, iPage));
460 if (iPage < pThis->cPages && cRefs > 0)
461 pgmR0DynMapReleasePage(pThis, iPage, cRefs);
462 else
463 AssertLogRelMsgFailed(("cRefs=%d iPage=%#x cPages=%u\n", cRefs, iPage, pThis->cPages));
464
465 pSet->aEntries[j].iPage = UINT16_MAX;
466 pSet->aEntries[j].cRefs = 0;
467 pSet->aEntries[j].pvPage = NULL;
468 pSet->aEntries[j].HCPhys = NIL_RTHCPHYS;
469 }
470 pSet->cEntries = PGMMAPSET_CLOSED;
471 pSet->iSubset = UINT32_MAX;
472 pSet->iCpu = -1;
473 }
474 else
475 AssertMsg(j == PGMMAPSET_CLOSED, ("cEntries=%#x\n", j));
476
477 j = RT_ELEMENTS(pSet->aEntries);
478 while (j-- > 0)
479 {
480 Assert(pSet->aEntries[j].iPage == UINT16_MAX);
481 Assert(!pSet->aEntries[j].cRefs);
482 }
483 }
484
485 /*
486 * Release our reference to the mapping cache.
487 */
488 Assert(pThis->cUsers > 0);
489 pThis->cUsers--;
490 if (!pThis->cUsers)
491 pgmR0DynMapTearDown(pThis);
492 }
493 else
494 AssertLogRelMsgFailed(("pvR0DynMapUsed=%p pThis=%p\n", pVM->pgm.s.pvR0DynMapUsed, pThis));
495
496 RTSemFastMutexRelease(pThis->hInitLock);
497}
498
499
500/**
501 * Shoots down the TLBs for all the cache pages, pgmR0DynMapTearDown helper.
502 *
503 * @param idCpu The current CPU.
504 * @param pvUser1 The dynamic mapping cache instance.
505 * @param pvUser2 Unused, NULL.
506 */
507static DECLCALLBACK(void) pgmR0DynMapShootDownTlbs(RTCPUID idCpu, void *pvUser1, void *pvUser2)
508{
509 Assert(!pvUser2);
510 PPGMR0DYNMAP pThis = (PPGMR0DYNMAP)pvUser1;
511 Assert(pThis == g_pPGMR0DynMap);
512 PPGMR0DYNMAPENTRY paPages = pThis->paPages;
513 uint32_t iPage = pThis->cPages;
514 while (iPage-- > 0)
515 ASMInvalidatePage(paPages[iPage].pvPage);
516}
517
518
519/**
520 * Shoot down the TLBs for every single cache entry on all CPUs.
521 *
522 * @returns IPRT status code (RTMpOnAll).
523 * @param pThis The dynamic mapping cache instance.
524 */
525static int pgmR0DynMapTlbShootDown(PPGMR0DYNMAP pThis)
526{
527 int rc = RTMpOnAll(pgmR0DynMapShootDownTlbs, pThis, NULL);
528 AssertRC(rc);
529 if (RT_FAILURE(rc))
530 {
531 uint32_t iPage = pThis->cPages;
532 while (iPage-- > 0)
533 ASMInvalidatePage(pThis->paPages[iPage].pvPage);
534 }
535 return rc;
536}
537
538
539/**
540 * Calculate the new cache size based on cMaxLoad statistics.
541 *
542 * @returns Number of pages.
543 * @param pThis The dynamic mapping cache instance.
544 * @param pcMinPages The minimal size in pages.
545 */
546static uint32_t pgmR0DynMapCalcNewSize(PPGMR0DYNMAP pThis, uint32_t *pcMinPages)
547{
548 Assert(pThis->cPages <= PGMR0DYNMAP_MAX_PAGES);
549
550 /* cCpus * PGMR0DYNMAP_PAGES_PER_CPU(_MIN). */
551 RTCPUID cCpus = RTMpGetCount();
552 AssertReturn(cCpus > 0 && cCpus <= RTCPUSET_MAX_CPUS, 0);
553 uint32_t cPages = cCpus * PGMR0DYNMAP_PAGES_PER_CPU;
554 uint32_t cMinPages = cCpus * PGMR0DYNMAP_PAGES_PER_CPU_MIN;
555
556 /* adjust against cMaxLoad. */
557 AssertMsg(pThis->cMaxLoad <= PGMR0DYNMAP_MAX_PAGES, ("%#x\n", pThis->cMaxLoad));
558 if (pThis->cMaxLoad > PGMR0DYNMAP_MAX_PAGES)
559 pThis->cMaxLoad = 0;
560
561 while (pThis->cMaxLoad > PGMR0DYNMAP_CALC_OVERLOAD(cPages))
562 cPages += PGMR0DYNMAP_PAGES_PER_CPU;
563
564 if (pThis->cMaxLoad > cMinPages)
565 cMinPages = pThis->cMaxLoad;
566
567 /* adjust against max and current size. */
568 if (cPages < pThis->cPages)
569 cPages = pThis->cPages;
570 cPages *= PGMR0DYNMAP_GUARD_PAGES + 1;
571 if (cPages > PGMR0DYNMAP_MAX_PAGES)
572 cPages = PGMR0DYNMAP_MAX_PAGES;
573
574 if (cMinPages < pThis->cPages)
575 cMinPages = pThis->cPages;
576 cMinPages *= PGMR0DYNMAP_GUARD_PAGES + 1;
577 if (cMinPages > PGMR0DYNMAP_MAX_PAGES)
578 cMinPages = PGMR0DYNMAP_MAX_PAGES;
579
580 Assert(cMinPages);
581 *pcMinPages = cMinPages;
582 return cPages;
583}
584
585
586/**
587 * Initializes the paging level data.
588 *
589 * @param pThis The dynamic mapping cache instance.
590 * @param pPgLvl The paging level data.
591 */
592void pgmR0DynMapPagingArrayInit(PPGMR0DYNMAP pThis, PPGMR0DYNMAPPGLVL pPgLvl)
593{
594 RTCCUINTREG cr4 = ASMGetCR4();
595 switch (pThis->enmPgMode)
596 {
597 case SUPPAGINGMODE_32_BIT:
598 case SUPPAGINGMODE_32_BIT_GLOBAL:
599 pPgLvl->cLevels = 2;
600 pPgLvl->a[0].fPhysMask = X86_CR3_PAGE_MASK;
601 pPgLvl->a[0].fAndMask = X86_PDE_P | X86_PDE_RW | (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0);
602 pPgLvl->a[0].fResMask = X86_PDE_P | X86_PDE_RW;
603 pPgLvl->a[0].fPtrMask = X86_PD_MASK;
604 pPgLvl->a[0].fPtrShift = X86_PD_SHIFT;
605
606 pPgLvl->a[1].fPhysMask = X86_PDE_PG_MASK;
607 pPgLvl->a[1].fAndMask = X86_PTE_P | X86_PTE_RW;
608 pPgLvl->a[1].fResMask = X86_PTE_P | X86_PTE_RW;
609 pPgLvl->a[1].fPtrMask = X86_PT_MASK;
610 pPgLvl->a[1].fPtrShift = X86_PT_SHIFT;
611 break;
612
613 case SUPPAGINGMODE_PAE:
614 case SUPPAGINGMODE_PAE_GLOBAL:
615 case SUPPAGINGMODE_PAE_NX:
616 case SUPPAGINGMODE_PAE_GLOBAL_NX:
617 pPgLvl->cLevels = 3;
618 pPgLvl->a[0].fPhysMask = X86_CR3_PAE_PAGE_MASK;
619 pPgLvl->a[0].fPtrMask = X86_PDPT_MASK_PAE;
620 pPgLvl->a[0].fPtrShift = X86_PDPT_SHIFT;
621 pPgLvl->a[0].fAndMask = X86_PDPE_P;
622 pPgLvl->a[0].fResMask = X86_PDPE_P;
623
624 pPgLvl->a[1].fPhysMask = X86_PDPE_PG_MASK;
625 pPgLvl->a[1].fPtrMask = X86_PD_PAE_MASK;
626 pPgLvl->a[1].fPtrShift = X86_PD_PAE_SHIFT;
627 pPgLvl->a[1].fAndMask = X86_PDE_P | X86_PDE_RW | (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0);
628 pPgLvl->a[1].fResMask = X86_PDE_P | X86_PDE_RW;
629
630 pPgLvl->a[2].fPhysMask = X86_PDE_PAE_PG_MASK;
631 pPgLvl->a[2].fPtrMask = X86_PT_PAE_MASK;
632 pPgLvl->a[2].fPtrShift = X86_PT_PAE_SHIFT;
633 pPgLvl->a[2].fAndMask = X86_PTE_P | X86_PTE_RW;
634 pPgLvl->a[2].fResMask = X86_PTE_P | X86_PTE_RW;
635 break;
636
637 case SUPPAGINGMODE_AMD64:
638 case SUPPAGINGMODE_AMD64_GLOBAL:
639 case SUPPAGINGMODE_AMD64_NX:
640 case SUPPAGINGMODE_AMD64_GLOBAL_NX:
641 pPgLvl->cLevels = 4;
642 pPgLvl->a[0].fPhysMask = X86_CR3_AMD64_PAGE_MASK;
643 pPgLvl->a[0].fPtrShift = X86_PML4_SHIFT;
644 pPgLvl->a[0].fPtrMask = X86_PML4_MASK;
645 pPgLvl->a[0].fAndMask = X86_PML4E_P | X86_PML4E_RW;
646 pPgLvl->a[0].fResMask = X86_PML4E_P | X86_PML4E_RW;
647
648 pPgLvl->a[1].fPhysMask = X86_PML4E_PG_MASK;
649 pPgLvl->a[1].fPtrShift = X86_PDPT_SHIFT;
650 pPgLvl->a[1].fPtrMask = X86_PDPT_MASK_AMD64;
651 pPgLvl->a[1].fAndMask = X86_PDPE_P | X86_PDPE_RW /** @todo check for X86_PDPT_PS support. */;
652 pPgLvl->a[1].fResMask = X86_PDPE_P | X86_PDPE_RW;
653
654 pPgLvl->a[2].fPhysMask = X86_PDPE_PG_MASK;
655 pPgLvl->a[2].fPtrShift = X86_PD_PAE_SHIFT;
656 pPgLvl->a[2].fPtrMask = X86_PD_PAE_MASK;
657 pPgLvl->a[2].fAndMask = X86_PDE_P | X86_PDE_RW | (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0);
658 pPgLvl->a[2].fResMask = X86_PDE_P | X86_PDE_RW;
659
660 pPgLvl->a[3].fPhysMask = X86_PDE_PAE_PG_MASK;
661 pPgLvl->a[3].fPtrShift = X86_PT_PAE_SHIFT;
662 pPgLvl->a[3].fPtrMask = X86_PT_PAE_MASK;
663 pPgLvl->a[3].fAndMask = X86_PTE_P | X86_PTE_RW;
664 pPgLvl->a[3].fResMask = X86_PTE_P | X86_PTE_RW;
665 break;
666
667 default:
668 AssertFailed();
669 pPgLvl->cLevels = 0;
670 break;
671 }
672
673 for (uint32_t i = 0; i < 4; i++) /* ASSUMING array size. */
674 {
675 pPgLvl->a[i].HCPhys = NIL_RTHCPHYS;
676 pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ;
677 pPgLvl->a[i].hMemObj = NIL_RTR0MEMOBJ;
678 pPgLvl->a[i].u.pv = NULL;
679 }
680}
681
682
683/**
684 * Maps a PTE.
685 *
686 * This will update the segment structure when new PTs are mapped.
687 *
688 * It also assumes that we (for paranoid reasons) wish to establish a mapping
689 * chain from CR3 to the PT that all corresponds to the processor we're
690 * currently running on, and go about this by running with interrupts disabled
691 * and restarting from CR3 for every change.
692 *
693 * @returns VBox status code, VINF_TRY_AGAIN if we changed any mappings and had
694 * to re-enable interrupts.
695 * @param pThis The dynamic mapping cache instance.
696 * @param pPgLvl The paging level structure.
697 * @param pvPage The page.
698 * @param pSeg The segment.
699 * @param cMaxPTs The max number of PTs expected in the segment.
700 * @param ppvPTE Where to store the PTE address.
701 */
702static int pgmR0DynMapPagingArrayMapPte(PPGMR0DYNMAP pThis, PPGMR0DYNMAPPGLVL pPgLvl, void *pvPage,
703 PPGMR0DYNMAPSEG pSeg, uint32_t cMaxPTs, void **ppvPTE)
704{
705 Assert(!(ASMGetFlags() & X86_EFL_IF));
706 void *pvEntry = NULL;
707 X86PGPAEUINT uEntry = ASMGetCR3();
708 for (uint32_t i = 0; i < pPgLvl->cLevels; i++)
709 {
710 RTHCPHYS HCPhys = uEntry & pPgLvl->a[i].fPhysMask;
711 if (pPgLvl->a[i].HCPhys != HCPhys)
712 {
713 /*
714 * Need to remap this level.
715 * The final level, the PT, will not be freed since that is what it's all about.
716 */
717 ASMIntEnable();
718 if (i + 1 == pPgLvl->cLevels)
719 AssertReturn(pSeg->cPTs < cMaxPTs, VERR_INTERNAL_ERROR);
720 else
721 {
722 int rc2 = RTR0MemObjFree(pPgLvl->a[i].hMemObj, true /* fFreeMappings */); AssertRC(rc2);
723 pPgLvl->a[i].hMemObj = pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ;
724 }
725
726 int rc = RTR0MemObjEnterPhys(&pPgLvl->a[i].hMemObj, HCPhys, PAGE_SIZE);
727 if (RT_SUCCESS(rc))
728 {
729 rc = RTR0MemObjMapKernel(&pPgLvl->a[i].hMapObj, pPgLvl->a[i].hMemObj,
730 (void *)-1 /* pvFixed */, 0 /* cbAlignment */,
731 RTMEM_PROT_WRITE | RTMEM_PROT_READ);
732 if (RT_SUCCESS(rc))
733 {
734 pPgLvl->a[i].u.pv = RTR0MemObjAddress(pPgLvl->a[i].hMapObj);
735 AssertMsg(((uintptr_t)pPgLvl->a[i].u.pv & ~(uintptr_t)PAGE_OFFSET_MASK), ("%p\n", pPgLvl->a[i].u.pv));
736 pPgLvl->a[i].HCPhys = HCPhys;
737 if (i + 1 == pPgLvl->cLevels)
738 pSeg->ahMemObjPTs[pSeg->cPTs++] = pPgLvl->a[i].hMemObj;
739 ASMIntDisable();
740 return VINF_TRY_AGAIN;
741 }
742
743 pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ;
744 }
745 else
746 pPgLvl->a[i].hMemObj = NIL_RTR0MEMOBJ;
747 pPgLvl->a[i].HCPhys = NIL_RTHCPHYS;
748 return rc;
749 }
750
751 /*
752 * The next level.
753 */
754 uint32_t iEntry = ((uint64_t)(uintptr_t)pvPage >> pPgLvl->a[i].fPtrShift) & pPgLvl->a[i].fPtrMask;
755 if (pThis->fLegacyMode)
756 {
757 pvEntry = &pPgLvl->a[i].u.paLegacy[iEntry];
758 uEntry = pPgLvl->a[i].u.paLegacy[iEntry];
759 }
760 else
761 {
762 pvEntry = &pPgLvl->a[i].u.paPae[iEntry];
763 uEntry = pPgLvl->a[i].u.paPae[iEntry];
764 }
765
766 if ((uEntry & pPgLvl->a[i].fAndMask) != pPgLvl->a[i].fResMask)
767 {
768 LogRel(("PGMR0DynMap: internal error - iPgLvl=%u cLevels=%u uEntry=%#llx fAnd=%#llx fRes=%#llx got=%#llx\n"
769 "PGMR0DynMap: pv=%p pvPage=%p iEntry=%#x fLegacyMode=%RTbool\n",
770 i, pPgLvl->cLevels, uEntry, pPgLvl->a[i].fAndMask, pPgLvl->a[i].fResMask, uEntry & pPgLvl->a[i].fAndMask,
771 pPgLvl->a[i].u.pv, pvPage, iEntry, pThis->fLegacyMode));
772 return VERR_INTERNAL_ERROR;
773 }
774 /*Log(("#%d: iEntry=%4d uEntry=%#llx pvEntry=%p HCPhys=%RHp \n", i, iEntry, uEntry, pvEntry, pPgLvl->a[i].HCPhys));*/
775 }
776
777 /* made it thru without needing to remap anything. */
778 *ppvPTE = pvEntry;
779 return VINF_SUCCESS;
780}
781
782
783/**
784 * Sets up a guard page.
785 *
786 * @param pThis The dynamic mapping cache instance.
787 * @param pPage The page.
788 */
789DECLINLINE(void) pgmR0DynMapSetupGuardPage(PPGMR0DYNMAP pThis, PPGMR0DYNMAPENTRY pPage)
790{
791 memset(pPage->pvPage, 0xfd, PAGE_SIZE);
792 pPage->cRefs = PGMR0DYNMAP_GUARD_PAGE_REF_COUNT;
793 pPage->HCPhys = PGMR0DYNMAP_GUARD_PAGE_HCPHYS;
794#ifdef PGMR0DYNMAP_GUARD_NP
795 ASMAtomicBitClear(pPage->uPte.pv, X86_PTE_BIT_P);
796#else
797 if (pThis->fLegacyMode)
798 ASMAtomicWriteU32(&pPage->uPte.pLegacy->u, PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE);
799 else
800 ASMAtomicWriteU64(&pPage->uPte.pPae->u, PGMR0DYNMAP_GUARD_PAGE_PAE_PTE);
801#endif
802 pThis->cGuardPages++;
803}
804
805
806/**
807 * Adds a new segment of the specified size.
808 *
809 * @returns VBox status code.
810 * @param pThis The dynamic mapping cache instance.
811 * @param cPages The size of the new segment, give as a page count.
812 */
813static int pgmR0DynMapAddSeg(PPGMR0DYNMAP pThis, uint32_t cPages)
814{
815 int rc2;
816 AssertReturn(ASMGetFlags() & X86_EFL_IF, VERR_PREEMPT_DISABLED);
817
818 /*
819 * Do the array reallocations first.
820 * (The pages array has to be replaced behind the spinlock of course.)
821 */
822 void *pvSavedPTEs = RTMemRealloc(pThis->pvSavedPTEs, (pThis->fLegacyMode ? sizeof(X86PGUINT) : sizeof(X86PGPAEUINT)) * (pThis->cPages + cPages));
823 if (!pvSavedPTEs)
824 return VERR_NO_MEMORY;
825 pThis->pvSavedPTEs = pvSavedPTEs;
826
827 void *pvPages = RTMemAllocZ(sizeof(pThis->paPages[0]) * (pThis->cPages + cPages));
828 if (!pvPages)
829 {
830 pvSavedPTEs = RTMemRealloc(pThis->pvSavedPTEs, (pThis->fLegacyMode ? sizeof(X86PGUINT) : sizeof(X86PGPAEUINT)) * pThis->cPages);
831 if (pvSavedPTEs)
832 pThis->pvSavedPTEs = pvSavedPTEs;
833 return VERR_NO_MEMORY;
834 }
835
836 RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
837 RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
838
839 memcpy(pvPages, pThis->paPages, sizeof(pThis->paPages[0]) * pThis->cPages);
840 void *pvToFree = pThis->paPages;
841 pThis->paPages = (PPGMR0DYNMAPENTRY)pvPages;
842
843 RTSpinlockRelease(pThis->hSpinlock, &Tmp);
844 RTMemFree(pvToFree);
845
846 /*
847 * Allocate the segment structure and pages of memory, then touch all the pages (paranoia).
848 */
849 uint32_t cMaxPTs = cPages / (pThis->fLegacyMode ? X86_PG_ENTRIES : X86_PG_PAE_ENTRIES) + 2;
850 PPGMR0DYNMAPSEG pSeg = (PPGMR0DYNMAPSEG)RTMemAllocZ(RT_UOFFSETOF(PGMR0DYNMAPSEG, ahMemObjPTs[cMaxPTs]));
851 if (!pSeg)
852 return VERR_NO_MEMORY;
853 pSeg->pNext = NULL;
854 pSeg->cPages = cPages;
855 pSeg->iPage = pThis->cPages;
856 pSeg->cPTs = 0;
857 int rc = RTR0MemObjAllocPage(&pSeg->hMemObj, cPages << PAGE_SHIFT, false);
858 if (RT_SUCCESS(rc))
859 {
860 uint8_t *pbPage = (uint8_t *)RTR0MemObjAddress(pSeg->hMemObj);
861 AssertMsg(VALID_PTR(pbPage) && !((uintptr_t)pbPage & PAGE_OFFSET_MASK), ("%p\n", pbPage));
862 memset(pbPage, 0xfe, cPages << PAGE_SHIFT);
863
864 /*
865 * Walk thru the pages and set them up with a mapping of their PTE and everything.
866 */
867 ASMIntDisable();
868 PGMR0DYNMAPPGLVL PgLvl;
869 pgmR0DynMapPagingArrayInit(pThis, &PgLvl);
870 uint32_t const iEndPage = pSeg->iPage + cPages;
871 for (uint32_t iPage = pSeg->iPage;
872 iPage < iEndPage;
873 iPage++, pbPage += PAGE_SIZE)
874 {
875 /* Initialize the page data. */
876 pThis->paPages[iPage].HCPhys = NIL_RTHCPHYS;
877 pThis->paPages[iPage].pvPage = pbPage;
878 pThis->paPages[iPage].cRefs = 0;
879 pThis->paPages[iPage].uPte.pPae = 0;
880 RTCpuSetFill(&pThis->paPages[iPage].PendingSet);
881
882 /* Map its page table, retry until we've got a clean run (paranoia). */
883 do
884 rc = pgmR0DynMapPagingArrayMapPte(pThis, &PgLvl, pbPage, pSeg, cMaxPTs,
885 &pThis->paPages[iPage].uPte.pv);
886 while (rc == VINF_TRY_AGAIN);
887 if (RT_FAILURE(rc))
888 break;
889
890 /* Save the PTE. */
891 if (pThis->fLegacyMode)
892 ((PX86PGUINT)pThis->pvSavedPTEs)[iPage] = pThis->paPages[iPage].uPte.pLegacy->u;
893 else
894 ((PX86PGPAEUINT)pThis->pvSavedPTEs)[iPage] = pThis->paPages[iPage].uPte.pPae->u;
895
896#ifdef VBOX_STRICT
897 /* Check that we've got the right entry. */
898 RTHCPHYS HCPhysPage = RTR0MemObjGetPagePhysAddr(pSeg->hMemObj, iPage - pSeg->iPage);
899 RTHCPHYS HCPhysPte = pThis->fLegacyMode
900 ? pThis->paPages[iPage].uPte.pLegacy->u & X86_PTE_PG_MASK
901 : pThis->paPages[iPage].uPte.pPae->u & X86_PTE_PAE_PG_MASK;
902 if (HCPhysPage != HCPhysPte)
903 {
904 LogRel(("pgmR0DynMapAddSeg: internal error - page #%u HCPhysPage=%RHp HCPhysPte=%RHp pbPage=%p pvPte=%p\n",
905 iPage - pSeg->iPage, HCPhysPage, HCPhysPte, pbPage, pThis->paPages[iPage].uPte.pv));
906 rc = VERR_INTERNAL_ERROR;
907 break;
908 }
909#endif
910 } /* for each page */
911 ASMIntEnable();
912
913 /* cleanup non-PT mappings */
914 for (uint32_t i = 0; i < PgLvl.cLevels - 1; i++)
915 RTR0MemObjFree(PgLvl.a[i].hMemObj, true /* fFreeMappings */);
916
917 if (RT_SUCCESS(rc))
918 {
919#if PGMR0DYNMAP_GUARD_PAGES > 0
920 /*
921 * Setup guard pages.
922 * (Note: TLBs will be shot down later on.)
923 */
924 uint32_t iPage = pSeg->iPage;
925 while (iPage < iEndPage)
926 {
927 for (uint32_t iGPg = 0; iGPg < PGMR0DYNMAP_GUARD_PAGES && iPage < iEndPage; iGPg++, iPage++)
928 pgmR0DynMapSetupGuardPage(pThis, &pThis->paPages[iPage]);
929 iPage++; /* the guarded page */
930 }
931
932 /* Make sure the very last page is a guard page too. */
933 iPage = iEndPage - 1;
934 if (pThis->paPages[iPage].cRefs != PGMR0DYNMAP_GUARD_PAGE_REF_COUNT)
935 pgmR0DynMapSetupGuardPage(pThis, &pThis->paPages[iPage]);
936#endif /* PGMR0DYNMAP_GUARD_PAGES > 0 */
937
938 /*
939 * Commit it by adding the segment to the list and updating the page count.
940 */
941 pSeg->pNext = pThis->pSegHead;
942 pThis->pSegHead = pSeg;
943 pThis->cPages += cPages;
944 return VINF_SUCCESS;
945 }
946
947 /*
948 * Bail out.
949 */
950 while (pSeg->cPTs-- > 0)
951 {
952 rc2 = RTR0MemObjFree(pSeg->ahMemObjPTs[pSeg->cPTs], true /* fFreeMappings */);
953 AssertRC(rc2);
954 pSeg->ahMemObjPTs[pSeg->cPTs] = NIL_RTR0MEMOBJ;
955 }
956
957 rc2 = RTR0MemObjFree(pSeg->hMemObj, true /* fFreeMappings */);
958 AssertRC(rc2);
959 pSeg->hMemObj = NIL_RTR0MEMOBJ;
960 }
961 RTMemFree(pSeg);
962
963 /* Don't bother resizing the arrays, but free them if we're the only user. */
964 if (!pThis->cPages)
965 {
966 RTMemFree(pThis->paPages);
967 pThis->paPages = NULL;
968 RTMemFree(pThis->pvSavedPTEs);
969 pThis->pvSavedPTEs = NULL;
970 }
971 return rc;
972}
973
974
975/**
976 * Called by PGMR0DynMapInitVM under the init lock.
977 *
978 * @returns VBox status code.
979 * @param pThis The dynamic mapping cache instance.
980 */
981static int pgmR0DynMapSetup(PPGMR0DYNMAP pThis)
982{
983 /*
984 * Calc the size and add a segment of that size.
985 */
986 uint32_t cMinPages;
987 uint32_t cPages = pgmR0DynMapCalcNewSize(pThis, &cMinPages);
988 AssertReturn(cPages, VERR_INTERNAL_ERROR);
989 int rc = pgmR0DynMapAddSeg(pThis, cPages);
990 if (rc == VERR_NO_MEMORY)
991 {
992 /*
993 * Try adding smaller segments.
994 */
995 do
996 rc = pgmR0DynMapAddSeg(pThis, PGMR0DYNMAP_SMALL_SEG_PAGES);
997 while (RT_SUCCESS(rc) && pThis->cPages < cPages);
998 if (rc == VERR_NO_MEMORY && pThis->cPages >= cMinPages)
999 rc = VINF_SUCCESS;
1000 if (rc == VERR_NO_MEMORY)
1001 {
1002 if (pThis->cPages)
1003 pgmR0DynMapTearDown(pThis);
1004 rc = VERR_PGM_DYNMAP_SETUP_ERROR;
1005 }
1006 }
1007 Assert(ASMGetFlags() & X86_EFL_IF);
1008
1009#if PGMR0DYNMAP_GUARD_PAGES > 0
1010 /* paranoia */
1011 if (RT_SUCCESS(rc))
1012 pgmR0DynMapTlbShootDown(pThis);
1013#endif
1014 return rc;
1015}
1016
1017
1018/**
1019 * Called by PGMR0DynMapInitVM under the init lock.
1020 *
1021 * @returns VBox status code.
1022 * @param pThis The dynamic mapping cache instance.
1023 */
1024static int pgmR0DynMapExpand(PPGMR0DYNMAP pThis)
1025{
1026 /*
1027 * Calc the new target size and add a segment of the appropriate size.
1028 */
1029 uint32_t cMinPages;
1030 uint32_t cPages = pgmR0DynMapCalcNewSize(pThis, &cMinPages);
1031 AssertReturn(cPages, VERR_INTERNAL_ERROR);
1032 if (pThis->cPages >= cPages)
1033 return VINF_SUCCESS;
1034
1035 uint32_t cAdd = cPages - pThis->cPages;
1036 int rc = pgmR0DynMapAddSeg(pThis, cAdd);
1037 if (rc == VERR_NO_MEMORY)
1038 {
1039 /*
1040 * Try adding smaller segments.
1041 */
1042 do
1043 rc = pgmR0DynMapAddSeg(pThis, PGMR0DYNMAP_SMALL_SEG_PAGES);
1044 while (RT_SUCCESS(rc) && pThis->cPages < cPages);
1045 if (rc == VERR_NO_MEMORY && pThis->cPages >= cMinPages)
1046 rc = VINF_SUCCESS;
1047 if (rc == VERR_NO_MEMORY)
1048 rc = VERR_PGM_DYNMAP_EXPAND_ERROR;
1049 }
1050 Assert(ASMGetFlags() & X86_EFL_IF);
1051
1052#if PGMR0DYNMAP_GUARD_PAGES > 0
1053 /* paranoia */
1054 if (RT_SUCCESS(rc))
1055 pgmR0DynMapTlbShootDown(pThis);
1056#endif
1057 return rc;
1058}
1059
1060
1061/**
1062 * Called by PGMR0DynMapTermVM under the init lock.
1063 *
1064 * @returns VBox status code.
1065 * @param pThis The dynamic mapping cache instance.
1066 */
1067static void pgmR0DynMapTearDown(PPGMR0DYNMAP pThis)
1068{
1069 /*
1070 * Restore the original page table entries
1071 */
1072 PPGMR0DYNMAPENTRY paPages = pThis->paPages;
1073 uint32_t iPage = pThis->cPages;
1074 if (pThis->fLegacyMode)
1075 {
1076 X86PGUINT const *paSavedPTEs = (X86PGUINT const *)pThis->pvSavedPTEs;
1077 while (iPage-- > 0)
1078 {
1079 X86PGUINT uOld = paPages[iPage].uPte.pLegacy->u;
1080 X86PGUINT uOld2 = uOld; NOREF(uOld2);
1081 X86PGUINT uNew = paSavedPTEs[iPage];
1082 while (!ASMAtomicCmpXchgExU32(&paPages[iPage].uPte.pLegacy->u, uNew, uOld, &uOld))
1083 AssertMsgFailed(("uOld=%#x uOld2=%#x uNew=%#x\n", uOld, uOld2, uNew));
1084 Assert(paPages[iPage].uPte.pLegacy->u == paSavedPTEs[iPage]);
1085 }
1086 }
1087 else
1088 {
1089 X86PGPAEUINT const *paSavedPTEs = (X86PGPAEUINT const *)pThis->pvSavedPTEs;
1090 while (iPage-- > 0)
1091 {
1092 X86PGPAEUINT uOld = paPages[iPage].uPte.pPae->u;
1093 X86PGPAEUINT uOld2 = uOld; NOREF(uOld2);
1094 X86PGPAEUINT uNew = paSavedPTEs[iPage];
1095 while (!ASMAtomicCmpXchgExU64(&paPages[iPage].uPte.pPae->u, uNew, uOld, &uOld))
1096 AssertMsgFailed(("uOld=%#llx uOld2=%#llx uNew=%#llx\n", uOld, uOld2, uNew));
1097 Assert(paPages[iPage].uPte.pPae->u == paSavedPTEs[iPage]);
1098 }
1099 }
1100
1101 /*
1102 * Shoot down the TLBs on all CPUs before freeing them.
1103 */
1104 pgmR0DynMapTlbShootDown(pThis);
1105
1106 /*
1107 * Free the segments.
1108 */
1109 while (pThis->pSegHead)
1110 {
1111 int rc;
1112 PPGMR0DYNMAPSEG pSeg = pThis->pSegHead;
1113 pThis->pSegHead = pSeg->pNext;
1114
1115 uint32_t iPT = pSeg->cPTs;
1116 while (iPT-- > 0)
1117 {
1118 rc = RTR0MemObjFree(pSeg->ahMemObjPTs[iPT], true /* fFreeMappings */); AssertRC(rc);
1119 pSeg->ahMemObjPTs[iPT] = NIL_RTR0MEMOBJ;
1120 }
1121 rc = RTR0MemObjFree(pSeg->hMemObj, true /* fFreeMappings */); AssertRC(rc);
1122 pSeg->hMemObj = NIL_RTR0MEMOBJ;
1123 pSeg->pNext = NULL;
1124 pSeg->iPage = UINT16_MAX;
1125 pSeg->cPages = 0;
1126 pSeg->cPTs = 0;
1127 RTMemFree(pSeg);
1128 }
1129
1130 /*
1131 * Free the arrays and restore the initial state.
1132 * The cLoadMax value is left behind for the next setup.
1133 */
1134 RTMemFree(pThis->paPages);
1135 pThis->paPages = NULL;
1136 RTMemFree(pThis->pvSavedPTEs);
1137 pThis->pvSavedPTEs = NULL;
1138 pThis->cPages = 0;
1139 pThis->cLoad = 0;
1140 pThis->cGuardPages = 0;
1141}
1142
1143
1144/**
1145 * Release references to a page, caller owns the spin lock.
1146 *
1147 * @param pThis The dynamic mapping cache instance.
1148 * @param iPage The page.
1149 * @param cRefs The number of references to release.
1150 */
1151DECLINLINE(void) pgmR0DynMapReleasePageLocked(PPGMR0DYNMAP pThis, uint32_t iPage, int32_t cRefs)
1152{
1153 cRefs = ASMAtomicSubS32(&pThis->paPages[iPage].cRefs, cRefs) - cRefs;
1154 AssertMsg(cRefs >= 0, ("%d\n", cRefs));
1155 if (!cRefs)
1156 pThis->cLoad--;
1157}
1158
1159
1160/**
1161 * Release references to a page, caller does not own the spin lock.
1162 *
1163 * @param pThis The dynamic mapping cache instance.
1164 * @param iPage The page.
1165 * @param cRefs The number of references to release.
1166 */
1167static void pgmR0DynMapReleasePage(PPGMR0DYNMAP pThis, uint32_t iPage, uint32_t cRefs)
1168{
1169 RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
1170 RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
1171 pgmR0DynMapReleasePageLocked(pThis, iPage, cRefs);
1172 RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1173}
1174
1175
1176/**
1177 * pgmR0DynMapPage worker that deals with the tedious bits.
1178 *
1179 * @returns The page index on success, UINT32_MAX on failure.
1180 * @param pThis The dynamic mapping cache instance.
1181 * @param HCPhys The address of the page to be mapped.
1182 * @param iPage The page index pgmR0DynMapPage hashed HCPhys to.
1183 * @param pVM The shared VM structure, for statistics only.
1184 */
1185static uint32_t pgmR0DynMapPageSlow(PPGMR0DYNMAP pThis, RTHCPHYS HCPhys, uint32_t iPage, PVM pVM)
1186{
1187 STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageSlow);
1188
1189 /*
1190 * Check if any of the first 3 pages are unreferenced since the caller
1191 * already has made sure they aren't matching.
1192 */
1193#ifdef VBOX_WITH_STATISTICS
1194 bool fLooped = false;
1195#endif
1196 uint32_t const cPages = pThis->cPages;
1197 PPGMR0DYNMAPENTRY paPages = pThis->paPages;
1198 uint32_t iFreePage;
1199 if (!paPages[iPage].cRefs)
1200 iFreePage = iPage;
1201 else if (!paPages[(iPage + 1) % cPages].cRefs)
1202 iFreePage = (iPage + 1) % cPages;
1203 else if (!paPages[(iPage + 2) % cPages].cRefs)
1204 iFreePage = (iPage + 2) % cPages;
1205 else
1206 {
1207 /*
1208 * Search for an unused or matching entry.
1209 */
1210 iFreePage = (iPage + 3) % cPages;
1211 for (;;)
1212 {
1213 if (paPages[iFreePage].HCPhys == HCPhys)
1214 {
1215 STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageSlowLoopHits);
1216 return iFreePage;
1217 }
1218 if (!paPages[iFreePage].cRefs)
1219 break;
1220
1221 /* advance */
1222 iFreePage = (iFreePage + 1) % cPages;
1223 if (RT_UNLIKELY(iFreePage == iPage))
1224 return UINT32_MAX;
1225 }
1226 STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageSlowLoopMisses);
1227#ifdef VBOX_WITH_STATISTICS
1228 fLooped = true;
1229#endif
1230 }
1231 Assert(iFreePage < cPages);
1232
1233#if 0 //def VBOX_WITH_STATISTICS
1234 /* Check for lost hits. */
1235 if (!fLooped)
1236 for (uint32_t iPage2 = (iPage + 3) % cPages; iPage2 != iPage; iPage2 = (iPage2 + 1) % cPages)
1237 if (paPages[iPage2].HCPhys == HCPhys)
1238 STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageSlowLostHits);
1239#endif
1240
1241 /*
1242 * Setup the new entry.
1243 */
1244 /*Log6(("pgmR0DynMapPageSlow: old - %RHp %#x %#llx\n", paPages[iFreePage].HCPhys, paPages[iFreePage].cRefs, paPages[iFreePage].uPte.pPae->u));*/
1245 paPages[iFreePage].HCPhys = HCPhys;
1246 RTCpuSetFill(&paPages[iFreePage].PendingSet);
1247 if (pThis->fLegacyMode)
1248 {
1249 X86PGUINT uOld = paPages[iFreePage].uPte.pLegacy->u;
1250 X86PGUINT uOld2 = uOld; NOREF(uOld2);
1251 X86PGUINT uNew = (uOld & (X86_PTE_G | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT))
1252 | X86_PTE_P | X86_PTE_RW | X86_PTE_A | X86_PTE_D
1253 | (HCPhys & X86_PTE_PG_MASK);
1254 while (!ASMAtomicCmpXchgExU32(&paPages[iFreePage].uPte.pLegacy->u, uNew, uOld, &uOld))
1255 AssertMsgFailed(("uOld=%#x uOld2=%#x uNew=%#x\n", uOld, uOld2, uNew));
1256 Assert(paPages[iFreePage].uPte.pLegacy->u == uNew);
1257 }
1258 else
1259 {
1260 X86PGPAEUINT uOld = paPages[iFreePage].uPte.pPae->u;
1261 X86PGPAEUINT uOld2 = uOld; NOREF(uOld2);
1262 X86PGPAEUINT uNew = (uOld & (X86_PTE_G | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT))
1263 | X86_PTE_P | X86_PTE_RW | X86_PTE_A | X86_PTE_D
1264 | (HCPhys & X86_PTE_PAE_PG_MASK);
1265 while (!ASMAtomicCmpXchgExU64(&paPages[iFreePage].uPte.pPae->u, uNew, uOld, &uOld))
1266 AssertMsgFailed(("uOld=%#llx uOld2=%#llx uNew=%#llx\n", uOld, uOld2, uNew));
1267 Assert(paPages[iFreePage].uPte.pPae->u == uNew);
1268 /*Log6(("pgmR0DynMapPageSlow: #%x - %RHp %p %#llx\n", iFreePage, HCPhys, paPages[iFreePage].pvPage, uNew));*/
1269 }
1270 return iFreePage;
1271}
1272
1273
1274/**
1275 * Maps a page into the pool.
1276 *
1277 * @returns Page index on success, UINT32_MAX on failure.
1278 * @param pThis The dynamic mapping cache instance.
1279 * @param HCPhys The address of the page to be mapped.
1280 * @param iRealCpu The real cpu set index. (optimization)
1281 * @param pVM The shared VM structure, for statistics only.
1282 * @param ppvPage Where to the page address.
1283 */
1284DECLINLINE(uint32_t) pgmR0DynMapPage(PPGMR0DYNMAP pThis, RTHCPHYS HCPhys, int32_t iRealCpu, PVM pVM, void **ppvPage)
1285{
1286 RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
1287 RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
1288 AssertMsg(!(HCPhys & PAGE_OFFSET_MASK), ("HCPhys=%RHp\n", HCPhys));
1289 STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPage);
1290
1291 /*
1292 * Find an entry, if possible a matching one. The HCPhys address is hashed
1293 * down to a page index, collisions are handled by linear searching.
1294 * Optimized for a hit in the first 3 pages.
1295 *
1296 * To the cheap hits here and defer the tedious searching and inserting
1297 * to a helper function.
1298 */
1299 uint32_t const cPages = pThis->cPages;
1300 uint32_t iPage = (HCPhys >> PAGE_SHIFT) % cPages;
1301 PPGMR0DYNMAPENTRY paPages = pThis->paPages;
1302 if (RT_LIKELY(paPages[iPage].HCPhys == HCPhys))
1303 STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageHits0);
1304 else
1305 {
1306 uint32_t iPage2 = (iPage + 1) % cPages;
1307 if (RT_LIKELY(paPages[iPage2].HCPhys == HCPhys))
1308 {
1309 iPage = iPage2;
1310 STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageHits1);
1311 }
1312 else
1313 {
1314 iPage2 = (iPage + 2) % cPages;
1315 if (paPages[iPage2].HCPhys == HCPhys)
1316 {
1317 iPage = iPage2;
1318 STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageHits2);
1319 }
1320 else
1321 {
1322 iPage = pgmR0DynMapPageSlow(pThis, HCPhys, iPage, pVM);
1323 if (RT_UNLIKELY(iPage == UINT32_MAX))
1324 {
1325 RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1326 *ppvPage = NULL;
1327 return iPage;
1328 }
1329 }
1330 }
1331 }
1332
1333 /*
1334 * Reference it, update statistics and get the return address.
1335 */
1336 int32_t cRefs = ASMAtomicIncS32(&paPages[iPage].cRefs);
1337 if (cRefs == 1)
1338 {
1339 pThis->cLoad++;
1340 if (pThis->cLoad > pThis->cMaxLoad)
1341 pThis->cMaxLoad = pThis->cLoad;
1342 AssertMsg(pThis->cLoad <= pThis->cPages - pThis->cGuardPages, ("%d/%d\n", pThis->cLoad, pThis->cPages - pThis->cGuardPages));
1343 }
1344 else if (RT_UNLIKELY(cRefs <= 0))
1345 {
1346 ASMAtomicDecS32(&paPages[iPage].cRefs);
1347 RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1348 *ppvPage = NULL;
1349 AssertLogRelMsgFailedReturn(("cRefs=%d iPage=%p HCPhys=%RHp\n", cRefs, iPage, HCPhys), UINT32_MAX);
1350 }
1351 void *pvPage = paPages[iPage].pvPage;
1352
1353 /*
1354 * Invalidate the entry?
1355 */
1356 bool fInvalidateIt = RTCpuSetIsMemberByIndex(&paPages[iPage].PendingSet, iRealCpu);
1357 if (RT_UNLIKELY(fInvalidateIt))
1358 RTCpuSetDelByIndex(&paPages[iPage].PendingSet, iRealCpu);
1359
1360 RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1361
1362 /*
1363 * Do the actual invalidation outside the spinlock.
1364 */
1365 if (RT_UNLIKELY(fInvalidateIt))
1366 {
1367 STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageInvlPg);
1368 ASMInvalidatePage(pvPage);
1369 }
1370
1371 *ppvPage = pvPage;
1372 return iPage;
1373}
1374
1375
1376/**
1377 * Assert the the integrity of the pool.
1378 *
1379 * @returns VBox status code.
1380 */
1381VMMR0DECL(int) PGMR0DynMapAssertIntegrity(void)
1382{
1383 /*
1384 * Basic pool stuff that doesn't require any lock, just assumes we're a user.
1385 */
1386 PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
1387 if (!pThis)
1388 return VINF_SUCCESS;
1389 AssertPtrReturn(pThis, VERR_INVALID_POINTER);
1390 AssertReturn(pThis->u32Magic == PGMR0DYNMAP_MAGIC, VERR_INVALID_MAGIC);
1391 if (!pThis->cUsers)
1392 return VERR_INVALID_PARAMETER;
1393
1394
1395 int rc = VINF_SUCCESS;
1396 RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
1397 RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
1398
1399#define CHECK_RET(expr, a) \
1400 do { \
1401 if (RT_UNLIKELY(!(expr))) \
1402 { \
1403 RTSpinlockRelease(pThis->hSpinlock, &Tmp); \
1404 AssertMsg1(#expr, __LINE__, __FILE__, __PRETTY_FUNCTION__); \
1405 AssertMsg2 a; \
1406 return VERR_INTERNAL_ERROR; \
1407 } \
1408 } while (0)
1409
1410 /*
1411 * Check that the PTEs are correct.
1412 */
1413 uint32_t cGuard = 0;
1414 uint32_t cLoad = 0;
1415 PPGMR0DYNMAPENTRY paPages = pThis->paPages;
1416 uint32_t iPage = pThis->cPages;
1417 if (pThis->fLegacyMode)
1418 {
1419 PCX86PGUINT paSavedPTEs = (PCX86PGUINT)pThis->pvSavedPTEs; NOREF(paSavedPTEs);
1420 while (iPage-- > 0)
1421 {
1422 CHECK_RET(!((uintptr_t)paPages[iPage].pvPage & PAGE_OFFSET_MASK), ("#%u: %p\n", iPage, paPages[iPage].pvPage));
1423 if ( paPages[iPage].cRefs == PGMR0DYNMAP_GUARD_PAGE_REF_COUNT
1424 && paPages[iPage].HCPhys == PGMR0DYNMAP_GUARD_PAGE_HCPHYS)
1425 {
1426#ifdef PGMR0DYNMAP_GUARD_NP
1427 CHECK_RET(paPages[iPage].uPte.pLegacy->u == (paSavedPTEs[iPage] & ~(X86PGUINT)X86_PTE_P),
1428 ("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, paSavedPTEs[iPage]));
1429#else
1430 CHECK_RET(paPages[iPage].uPte.pLegacy->u == PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE,
1431 ("#%u: %#x", iPage, paPages[iPage].uPte.pLegacy->u));
1432#endif
1433 cGuard++;
1434 }
1435 else if (paPages[iPage].HCPhys != NIL_RTHCPHYS)
1436 {
1437 CHECK_RET(!(paPages[iPage].HCPhys & PAGE_OFFSET_MASK), ("#%u: %RHp\n", iPage, paPages[iPage].HCPhys));
1438 X86PGUINT uPte = (paSavedPTEs[iPage] & (X86_PTE_G | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT))
1439 | X86_PTE_P | X86_PTE_RW | X86_PTE_A | X86_PTE_D
1440 | (paPages[iPage].HCPhys & X86_PTE_PAE_PG_MASK);
1441 CHECK_RET(paPages[iPage].uPte.pLegacy->u == uPte,
1442 ("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, uPte));
1443 if (paPages[iPage].cRefs)
1444 cLoad++;
1445 }
1446 else
1447 CHECK_RET(paPages[iPage].uPte.pLegacy->u == paSavedPTEs[iPage],
1448 ("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, paSavedPTEs[iPage]));
1449 }
1450 }
1451 else
1452 {
1453 PCX86PGPAEUINT paSavedPTEs = (PCX86PGPAEUINT)pThis->pvSavedPTEs; NOREF(paSavedPTEs);
1454 while (iPage-- > 0)
1455 {
1456 CHECK_RET(!((uintptr_t)paPages[iPage].pvPage & PAGE_OFFSET_MASK), ("#%u: %p\n", iPage, paPages[iPage].pvPage));
1457 if ( paPages[iPage].cRefs == PGMR0DYNMAP_GUARD_PAGE_REF_COUNT
1458 && paPages[iPage].HCPhys == PGMR0DYNMAP_GUARD_PAGE_HCPHYS)
1459 {
1460#ifdef PGMR0DYNMAP_GUARD_NP
1461 CHECK_RET(paPages[iPage].uPte.pPae->u == (paSavedPTEs[iPage] & ~(X86PGPAEUINT)X86_PTE_P),
1462 ("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pPae->u, paSavedPTEs[iPage]));
1463#else
1464 CHECK_RET(paPages[iPage].uPte.pPae->u == PGMR0DYNMAP_GUARD_PAGE_PAE_PTE,
1465 ("#%u: %#llx", iPage, paPages[iPage].uPte.pPae->u));
1466#endif
1467 cGuard++;
1468 }
1469 else if (paPages[iPage].HCPhys != NIL_RTHCPHYS)
1470 {
1471 CHECK_RET(!(paPages[iPage].HCPhys & PAGE_OFFSET_MASK), ("#%u: %RHp\n", iPage, paPages[iPage].HCPhys));
1472 X86PGPAEUINT uPte = (paSavedPTEs[iPage] & (X86_PTE_G | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT))
1473 | X86_PTE_P | X86_PTE_RW | X86_PTE_A | X86_PTE_D
1474 | (paPages[iPage].HCPhys & X86_PTE_PAE_PG_MASK);
1475 CHECK_RET(paPages[iPage].uPte.pPae->u == uPte,
1476 ("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pLegacy->u, uPte));
1477 if (paPages[iPage].cRefs)
1478 cLoad++;
1479 }
1480 else
1481 CHECK_RET(paPages[iPage].uPte.pPae->u == paSavedPTEs[iPage],
1482 ("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pPae->u, paSavedPTEs[iPage]));
1483 }
1484 }
1485
1486 CHECK_RET(cLoad == pThis->cLoad, ("%u %u\n", cLoad, pThis->cLoad));
1487 CHECK_RET(cGuard == pThis->cGuardPages, ("%u %u\n", cGuard, pThis->cGuardPages));
1488
1489#undef CHECK_RET
1490 RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1491 return VINF_SUCCESS;
1492}
1493
1494
1495/**
1496 * Signals the start of a new set of mappings.
1497 *
1498 * Mostly for strictness. PGMDynMapHCPage won't work unless this
1499 * API is called.
1500 *
1501 * @param pVCpu The shared data for the current virtual CPU.
1502 */
1503VMMDECL(void) PGMDynMapStartAutoSet(PVMCPU pVCpu)
1504{
1505 Assert(pVCpu->pgm.s.AutoSet.cEntries == PGMMAPSET_CLOSED);
1506 Assert(pVCpu->pgm.s.AutoSet.iSubset == UINT32_MAX);
1507 pVCpu->pgm.s.AutoSet.cEntries = 0;
1508 pVCpu->pgm.s.AutoSet.iCpu = RTMpCpuIdToSetIndex(RTMpCpuId());
1509}
1510
1511
1512/**
1513 * Worker that performs the actual flushing of the set.
1514 *
1515 * @param pSet The set to flush.
1516 * @param cEntries The number of entries.
1517 */
1518DECLINLINE(void) pgmDynMapFlushAutoSetWorker(PPGMMAPSET pSet, uint32_t cEntries)
1519{
1520 /*
1521 * Release any pages it's referencing.
1522 */
1523 if ( cEntries != 0
1524 && RT_LIKELY(cEntries <= RT_ELEMENTS(pSet->aEntries)))
1525 {
1526 PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
1527 RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
1528 RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
1529
1530 uint32_t i = cEntries;
1531 while (i-- > 0)
1532 {
1533 uint32_t iPage = pSet->aEntries[i].iPage;
1534 Assert(iPage < pThis->cPages);
1535 int32_t cRefs = pSet->aEntries[i].cRefs;
1536 Assert(cRefs > 0);
1537 pgmR0DynMapReleasePageLocked(pThis, iPage, cRefs);
1538
1539 pSet->aEntries[i].iPage = UINT16_MAX;
1540 pSet->aEntries[i].cRefs = 0;
1541 }
1542
1543 Assert(pThis->cLoad <= pThis->cPages - pThis->cGuardPages);
1544 RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1545 }
1546}
1547
1548
1549/**
1550 * Releases the dynamic memory mappings made by PGMDynMapHCPage and associates
1551 * since the PGMDynMapStartAutoSet call.
1552 *
1553 * @param pVCpu The shared data for the current virtual CPU.
1554 */
1555VMMDECL(void) PGMDynMapReleaseAutoSet(PVMCPU pVCpu)
1556{
1557 PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
1558
1559 /*
1560 * Close and flush the set.
1561 */
1562 uint32_t cEntries = pSet->cEntries;
1563 AssertReturnVoid(cEntries != PGMMAPSET_CLOSED);
1564 pSet->cEntries = PGMMAPSET_CLOSED;
1565 pSet->iSubset = UINT32_MAX;
1566 pSet->iCpu = -1;
1567
1568 STAM_COUNTER_INC(&pVCpu->pVMR0->pgm.s.aStatR0DynMapSetSize[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
1569 AssertMsg(cEntries < PGMMAPSET_MAX_FILL, ("%u\n", cEntries));
1570 if (cEntries > RT_ELEMENTS(pSet->aEntries) * 50 / 100)
1571 Log(("PGMDynMapReleaseAutoSet: cEntries=%d\n", pSet->cEntries));
1572
1573 pgmDynMapFlushAutoSetWorker(pSet, cEntries);
1574}
1575
1576
1577/**
1578 * Flushes the set if it's above a certain threshold.
1579 *
1580 * @param pVCpu The shared data for the current virtual CPU.
1581 */
1582VMMDECL(void) PGMDynMapFlushAutoSet(PVMCPU pVCpu)
1583{
1584 PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
1585 AssertMsg(pSet->iCpu == RTMpCpuIdToSetIndex(RTMpCpuId()), ("%d %d(%d) efl=%#x\n", pSet->iCpu, RTMpCpuIdToSetIndex(RTMpCpuId()), RTMpCpuId(), ASMGetFlags()));
1586
1587 /*
1588 * Only flush it if it's 45% full.
1589 */
1590 uint32_t cEntries = pSet->cEntries;
1591 AssertReturnVoid(cEntries != PGMMAPSET_CLOSED);
1592 STAM_COUNTER_INC(&pVCpu->pVMR0->pgm.s.aStatR0DynMapSetSize[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
1593 if (cEntries >= RT_ELEMENTS(pSet->aEntries) * 45 / 100)
1594 {
1595 pSet->cEntries = 0;
1596
1597 AssertMsg(cEntries < PGMMAPSET_MAX_FILL, ("%u\n", cEntries));
1598 Log(("PGMDynMapFlushAutoSet: cEntries=%d\n", pSet->cEntries));
1599
1600 pgmDynMapFlushAutoSetWorker(pSet, cEntries);
1601 AssertMsg(pSet->iCpu == RTMpCpuIdToSetIndex(RTMpCpuId()), ("%d %d(%d) efl=%#x\n", pSet->iCpu, RTMpCpuIdToSetIndex(RTMpCpuId()), RTMpCpuId(), ASMGetFlags()));
1602 }
1603}
1604
1605
1606/**
1607 * Migrates the automatic mapping set of the current vCPU if it's active and
1608 * necessary.
1609 *
1610 * This is called when re-entering the hardware assisted execution mode after a
1611 * nip down to ring-3. We run the risk that the CPU might have change and we
1612 * will therefore make sure all the cache entries currently in the auto set will
1613 * be valid on the new CPU. If the cpu didn't change nothing will happen as all
1614 * the entries will have been flagged as invalidated.
1615 *
1616 * @param pVCpu The shared data for the current virtual CPU.
1617 * @thread EMT
1618 */
1619VMMDECL(void) PGMDynMapMigrateAutoSet(PVMCPU pVCpu)
1620{
1621 PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
1622 int32_t iRealCpu = RTMpCpuIdToSetIndex(RTMpCpuId());
1623 if (pSet->iCpu != iRealCpu)
1624 {
1625 uint32_t i = pSet->cEntries;
1626 if (i != PGMMAPSET_CLOSED)
1627 {
1628 AssertMsg(i <= RT_ELEMENTS(pSet->aEntries), ("%#x (%u)\n", i, i));
1629 if (i != 0 && RT_LIKELY(i <= RT_ELEMENTS(pSet->aEntries)))
1630 {
1631 PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
1632 RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
1633 RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
1634
1635 while (i-- > 0)
1636 {
1637 Assert(pSet->aEntries[i].cRefs > 0);
1638 uint32_t iPage = pSet->aEntries[i].iPage;
1639 Assert(iPage < pThis->cPages);
1640 if (RTCpuSetIsMemberByIndex(&pThis->paPages[iPage].PendingSet, iRealCpu))
1641 {
1642 RTCpuSetDelByIndex(&pThis->paPages[iPage].PendingSet, iRealCpu);
1643 RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1644
1645 ASMInvalidatePage(pThis->paPages[iPage].pvPage);
1646 STAM_COUNTER_INC(&pVCpu->pVMR0->pgm.s.StatR0DynMapMigrateInvlPg);
1647
1648 RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
1649 }
1650 }
1651
1652 RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1653 }
1654 }
1655 pSet->iCpu = iRealCpu;
1656 }
1657}
1658
1659
1660/**
1661 * Worker function that flushes the current subset.
1662 *
1663 * This is called when the set is popped or when the set
1664 * hash a too high load. As also pointed out elsewhere, the
1665 * whole subset thing is a hack for working around code that
1666 * accesses too many pages. Like PGMPool.
1667 *
1668 * @param pSet The set which subset to flush.
1669 */
1670static void pgmDynMapFlushSubset(PPGMMAPSET pSet)
1671{
1672 uint32_t iSubset = pSet->iSubset;
1673 uint32_t i = pSet->cEntries;
1674 Assert(i <= RT_ELEMENTS(pSet->aEntries));
1675 if ( i > iSubset
1676 && i <= RT_ELEMENTS(pSet->aEntries))
1677 {
1678 Log(("pgmDynMapFlushSubset: cEntries=%d iSubset=%d\n", pSet->cEntries, iSubset));
1679 pSet->cEntries = iSubset;
1680
1681 PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
1682 RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
1683 RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
1684
1685 while (i-- > iSubset)
1686 {
1687 uint32_t iPage = pSet->aEntries[i].iPage;
1688 Assert(iPage < pThis->cPages);
1689 int32_t cRefs = pSet->aEntries[i].cRefs;
1690 Assert(cRefs > 0);
1691 pgmR0DynMapReleasePageLocked(pThis, iPage, cRefs);
1692
1693 pSet->aEntries[i].iPage = UINT16_MAX;
1694 pSet->aEntries[i].cRefs = 0;
1695 }
1696
1697 RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1698 }
1699}
1700
1701
1702/**
1703 * Creates a subset.
1704 *
1705 * A subset is a hack to avoid having to rewrite code that touches a lot of
1706 * pages. It prevents the mapping set from being overflowed by automatically
1707 * flushing previous mappings when a certain threshold is reached.
1708 *
1709 * Pages mapped after calling this function are only valid until the next page
1710 * is mapped.
1711 *
1712 * @returns The index of the previous subset. Pass this to
1713 * PGMDynMapPopAutoSubset when poping it.
1714 * @param pVCpu Pointer to the virtual cpu data.
1715 */
1716VMMDECL(uint32_t) PGMDynMapPushAutoSubset(PVMCPU pVCpu)
1717{
1718 PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
1719 AssertReturn(pSet->cEntries != PGMMAPSET_CLOSED, UINT32_MAX);
1720 uint32_t iPrevSubset = pSet->iSubset;
1721Assert(iPrevSubset == UINT32_MAX);
1722 pSet->iSubset = pSet->cEntries;
1723 STAM_COUNTER_INC(&pVCpu->pVMR0->pgm.s.StatR0DynMapSubsets);
1724 return iPrevSubset;
1725}
1726
1727
1728/**
1729 * Pops a subset created by a previous call to PGMDynMapPushAutoSubset.
1730 *
1731 * @param pVCpu Pointer to the virtual cpu data.
1732 * @param iPrevSubset What PGMDynMapPushAutoSubset returned.
1733 */
1734VMMDECL(void) PGMDynMapPopAutoSubset(PVMCPU pVCpu, uint32_t iPrevSubset)
1735{
1736 PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
1737 uint32_t cEntries = pSet->cEntries;
1738 AssertReturnVoid(cEntries != PGMMAPSET_CLOSED);
1739 AssertReturnVoid(pSet->iSubset <= iPrevSubset || iPrevSubset == UINT32_MAX);
1740Assert(iPrevSubset == UINT32_MAX);
1741 STAM_COUNTER_INC(&pVCpu->pVMR0->pgm.s.aStatR0DynMapSetSize[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
1742 if ( cEntries >= RT_ELEMENTS(pSet->aEntries) * 40 / 100
1743 && cEntries != pSet->iSubset)
1744 {
1745 AssertMsg(cEntries < PGMMAPSET_MAX_FILL, ("%u\n", cEntries));
1746 pgmDynMapFlushSubset(pSet);
1747 }
1748 pSet->iSubset = iPrevSubset;
1749}
1750
1751
1752/**
1753 * As a final resort for a full auto set, try merge duplicate entries.
1754 *
1755 * @param pSet The set.
1756 */
1757static void pgmDynMapOptimizeAutoSet(PPGMMAPSET pSet)
1758{
1759 for (uint32_t i = 0 ; i < pSet->cEntries; i++)
1760 {
1761 uint16_t const iPage = pSet->aEntries[i].iPage;
1762 uint32_t j = i + 1;
1763 while (j < pSet->cEntries)
1764 {
1765 if (pSet->aEntries[j].iPage != iPage)
1766 j++;
1767 else if ((uint32_t)pSet->aEntries[i].cRefs + (uint32_t)pSet->aEntries[j].cRefs < UINT16_MAX)
1768 {
1769 /* merge j into i removing j. */
1770 pSet->aEntries[i].cRefs += pSet->aEntries[j].cRefs;
1771 pSet->cEntries--;
1772 if (j < pSet->cEntries)
1773 {
1774 pSet->aEntries[j] = pSet->aEntries[pSet->cEntries];
1775 pSet->aEntries[pSet->cEntries].iPage = UINT16_MAX;
1776 pSet->aEntries[pSet->cEntries].cRefs = 0;
1777 }
1778 else
1779 {
1780 pSet->aEntries[j].iPage = UINT16_MAX;
1781 pSet->aEntries[j].cRefs = 0;
1782 }
1783 }
1784 else
1785 {
1786 /* migrate the max number of refs from j into i and quit the inner loop. */
1787 uint32_t cMigrate = UINT16_MAX - 1 - pSet->aEntries[i].cRefs;
1788 Assert(pSet->aEntries[j].cRefs > cMigrate);
1789 pSet->aEntries[j].cRefs -= cMigrate;
1790 pSet->aEntries[i].cRefs = UINT16_MAX - 1;
1791 break;
1792 }
1793 }
1794 }
1795}
1796
1797
1798/**
1799 * Common worker code for PGMDynMapHCPhys, pgmR0DynMapHCPageInlined and
1800 * pgmR0DynMapGCPageInlined.
1801 *
1802 * @returns VINF_SUCCESS, bails out to ring-3 on failure.
1803 * @param pVM The shared VM structure (for statistics).
1804 * @param pSet The set.
1805 * @param HCPhys The physical address of the page.
1806 * @param ppv Where to store the address of the mapping on success.
1807 *
1808 * @remarks This is a very hot path.
1809 */
1810int pgmR0DynMapHCPageCommon(PVM pVM, PPGMMAPSET pSet, RTHCPHYS HCPhys, void **ppv)
1811{
1812 AssertMsg(pSet->iCpu == RTMpCpuIdToSetIndex(RTMpCpuId()), ("%d %d(%d) efl=%#x\n", pSet->iCpu, RTMpCpuIdToSetIndex(RTMpCpuId()), RTMpCpuId(), ASMGetFlags()));
1813
1814 /*
1815 * Map it.
1816 */
1817 void *pvPage;
1818 uint32_t const iPage = pgmR0DynMapPage(g_pPGMR0DynMap, HCPhys, pSet->iCpu, pVM, &pvPage);
1819 if (RT_UNLIKELY(iPage == UINT32_MAX))
1820 {
1821 AssertMsg2("PGMDynMapHCPage: cLoad=%u/%u cPages=%u cGuardPages=%u\n",
1822 g_pPGMR0DynMap->cLoad, g_pPGMR0DynMap->cMaxLoad, g_pPGMR0DynMap->cPages, g_pPGMR0DynMap->cGuardPages);
1823 if (!g_fPGMR0DynMapTestRunning)
1824 VMMR0CallHost(pVM, VMMCALLHOST_VM_R0_ASSERTION, 0);
1825 *ppv = NULL;
1826 return VERR_PGM_DYNMAP_FAILED;
1827 }
1828
1829 /*
1830 * Add the page to the auto reference set.
1831 *
1832 * The typical usage pattern means that the same pages will be mapped
1833 * several times in the same set. We can catch most of these
1834 * remappings by looking a few pages back into the set. (The searching
1835 * and set optimizing path will hardly ever be used when doing this.)
1836 */
1837 AssertCompile(RT_ELEMENTS(pSet->aEntries) >= 8);
1838 int32_t i = pSet->cEntries;
1839 if (i-- < 5)
1840 {
1841 unsigned iEntry = pSet->cEntries++;
1842 pSet->aEntries[iEntry].cRefs = 1;
1843 pSet->aEntries[iEntry].iPage = iPage;
1844 pSet->aEntries[iEntry].pvPage = pvPage;
1845 pSet->aEntries[iEntry].HCPhys = HCPhys;
1846 pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
1847 }
1848 /* Any of the last 5 pages? */
1849 else if ( pSet->aEntries[i - 0].iPage == iPage
1850 && pSet->aEntries[i - 0].cRefs < UINT16_MAX - 1)
1851 pSet->aEntries[i - 0].cRefs++;
1852 else if ( pSet->aEntries[i - 1].iPage == iPage
1853 && pSet->aEntries[i - 1].cRefs < UINT16_MAX - 1)
1854 pSet->aEntries[i - 1].cRefs++;
1855 else if ( pSet->aEntries[i - 2].iPage == iPage
1856 && pSet->aEntries[i - 2].cRefs < UINT16_MAX - 1)
1857 pSet->aEntries[i - 2].cRefs++;
1858 else if ( pSet->aEntries[i - 3].iPage == iPage
1859 && pSet->aEntries[i - 3].cRefs < UINT16_MAX - 1)
1860 pSet->aEntries[i - 3].cRefs++;
1861 else if ( pSet->aEntries[i - 4].iPage == iPage
1862 && pSet->aEntries[i - 4].cRefs < UINT16_MAX - 1)
1863 pSet->aEntries[i - 4].cRefs++;
1864 /* Don't bother searching unless we're above a 60% load. */
1865 else if (RT_LIKELY(i <= (int32_t)RT_ELEMENTS(pSet->aEntries) * 60 / 100))
1866 {
1867 unsigned iEntry = pSet->cEntries++;
1868 pSet->aEntries[iEntry].cRefs = 1;
1869 pSet->aEntries[iEntry].iPage = iPage;
1870 pSet->aEntries[iEntry].pvPage = pvPage;
1871 pSet->aEntries[iEntry].HCPhys = HCPhys;
1872 pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
1873 }
1874 else
1875 {
1876 /* Search the rest of the set. */
1877 Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
1878 i -= 4;
1879 while (i-- > 0)
1880 if ( pSet->aEntries[i].iPage == iPage
1881 && pSet->aEntries[i].cRefs < UINT16_MAX - 1)
1882 {
1883 pSet->aEntries[i].cRefs++;
1884 STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapSetSearchHits);
1885 break;
1886 }
1887 if (i < 0)
1888 {
1889 STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapSetSearchMisses);
1890 if (pSet->iSubset < pSet->cEntries)
1891 {
1892 STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapSetSearchFlushes);
1893 STAM_COUNTER_INC(&pVM->pgm.s.aStatR0DynMapSetSize[(pSet->cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
1894 AssertMsg(pSet->cEntries < PGMMAPSET_MAX_FILL, ("%u\n", pSet->cEntries));
1895 pgmDynMapFlushSubset(pSet);
1896 }
1897
1898 if (RT_UNLIKELY(pSet->cEntries >= RT_ELEMENTS(pSet->aEntries)))
1899 {
1900 STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapSetOptimize);
1901 pgmDynMapOptimizeAutoSet(pSet);
1902 }
1903
1904 if (RT_LIKELY(pSet->cEntries < RT_ELEMENTS(pSet->aEntries)))
1905 {
1906 unsigned iEntry = pSet->cEntries++;
1907 pSet->aEntries[iEntry].cRefs = 1;
1908 pSet->aEntries[iEntry].iPage = iPage;
1909 pSet->aEntries[iEntry].pvPage = pvPage;
1910 pSet->aEntries[iEntry].HCPhys = HCPhys;
1911 pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
1912 }
1913 else
1914 {
1915 /* We're screwed. */
1916 pgmR0DynMapReleasePage(g_pPGMR0DynMap, iPage, 1);
1917
1918 AssertMsg2("PGMDynMapHCPage: set is full!\n");
1919 if (!g_fPGMR0DynMapTestRunning)
1920 VMMR0CallHost(pVM, VMMCALLHOST_VM_R0_ASSERTION, 0);
1921 *ppv = NULL;
1922 return VERR_PGM_DYNMAP_FULL_SET;
1923 }
1924 }
1925 }
1926
1927 *ppv = pvPage;
1928 return VINF_SUCCESS;
1929}
1930
1931
1932#if 0 /* Not used in R0, should internalized the other PGMDynMapHC/GCPage too. */
1933/* documented elsewhere - a bit of a mess. */
1934VMMDECL(int) PGMDynMapHCPage(PVM pVM, RTHCPHYS HCPhys, void **ppv)
1935{
1936 /*
1937 * Validate state.
1938 */
1939 STAM_PROFILE_START(&pVM->pgm.s.StatR0DynMapHCPage, a);
1940 AssertPtr(ppv);
1941 AssertMsg(pVM->pgm.s.pvR0DynMapUsed == g_pPGMR0DynMap,
1942 ("%p != %p\n", pVM->pgm.s.pvR0DynMapUsed, g_pPGMR0DynMap));
1943 AssertMsg(!(HCPhys & PAGE_OFFSET_MASK), ("HCPhys=%RHp\n", HCPhys));
1944 PVMCPU pVCpu = VMMGetCpu(pVM);
1945 AssertPtr(pVCpu);
1946 PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
1947 AssertMsg(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries),
1948 ("%#x (%u)\n", pSet->cEntries, pSet->cEntries));
1949
1950 /*
1951 * Call common code.
1952 */
1953 int rc = pgmR0DynMapHCPageCommon(pVM, pSet, HCPhys, ppv);
1954
1955 STAM_PROFILE_STOP(&pVM->pgm.s.StatR0DynMapHCPage, a);
1956 return rc;
1957}
1958#endif
1959
1960
1961#if 0 /*def DEBUG*/
1962/** For pgmR0DynMapTest3PerCpu. */
1963typedef struct PGMR0DYNMAPTEST
1964{
1965 uint32_t u32Expect;
1966 uint32_t *pu32;
1967 uint32_t volatile cFailures;
1968} PGMR0DYNMAPTEST;
1969typedef PGMR0DYNMAPTEST *PPGMR0DYNMAPTEST;
1970
1971/**
1972 * Checks that the content of the page is the same on all CPUs, i.e. that there
1973 * are no CPU specfic PTs or similar nasty stuff involved.
1974 *
1975 * @param idCpu The current CPU.
1976 * @param pvUser1 Pointer a PGMR0DYNMAPTEST structure.
1977 * @param pvUser2 Unused, ignored.
1978 */
1979static DECLCALLBACK(void) pgmR0DynMapTest3PerCpu(RTCPUID idCpu, void *pvUser1, void *pvUser2)
1980{
1981 PPGMR0DYNMAPTEST pTest = (PPGMR0DYNMAPTEST)pvUser1;
1982 ASMInvalidatePage(pTest->pu32);
1983 if (*pTest->pu32 != pTest->u32Expect)
1984 ASMAtomicIncU32(&pTest->cFailures);
1985 NOREF(pvUser2); NOREF(idCpu);
1986}
1987
1988
1989/**
1990 * Performs some basic tests in debug builds.
1991 */
1992static int pgmR0DynMapTest(PVM pVM)
1993{
1994 LogRel(("pgmR0DynMapTest: ****** START ******\n"));
1995 PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
1996 PPGMMAPSET pSet = &pVM->aCpus[0].pgm.s.AutoSet;
1997 uint32_t i;
1998
1999 /*
2000 * Assert internal integrity first.
2001 */
2002 LogRel(("Test #0\n"));
2003 int rc = PGMR0DynMapAssertIntegrity();
2004 if (RT_FAILURE(rc))
2005 return rc;
2006
2007 void *pvR0DynMapUsedSaved = pVM->pgm.s.pvR0DynMapUsed;
2008 pVM->pgm.s.pvR0DynMapUsed = pThis;
2009 g_fPGMR0DynMapTestRunning = true;
2010
2011 /*
2012 * Simple test, map CR3 twice and check that we're getting the
2013 * same mapping address back.
2014 */
2015 LogRel(("Test #1\n"));
2016 ASMIntDisable();
2017 PGMDynMapStartAutoSet(&pVM->aCpus[0]);
2018
2019 uint64_t cr3 = ASMGetCR3() & ~(uint64_t)PAGE_OFFSET_MASK;
2020 void *pv = (void *)(intptr_t)-1;
2021 void *pv2 = (void *)(intptr_t)-2;
2022 rc = PGMDynMapHCPage(pVM, cr3, &pv);
2023 int rc2 = PGMDynMapHCPage(pVM, cr3, &pv2);
2024 ASMIntEnable();
2025 if ( RT_SUCCESS(rc2)
2026 && RT_SUCCESS(rc)
2027 && pv == pv2)
2028 {
2029 LogRel(("Load=%u/%u/%u Set=%u/%u\n", pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
2030 rc = PGMR0DynMapAssertIntegrity();
2031
2032 /*
2033 * Check that the simple set overflow code works by filling it
2034 * with more CR3 mappings.
2035 */
2036 LogRel(("Test #2\n"));
2037 ASMIntDisable();
2038 PGMDynMapMigrateAutoSet(&pVM->aCpus[0]);
2039 for (i = 0 ; i < UINT16_MAX*2 - 1 && RT_SUCCESS(rc) && pv2 == pv; i++)
2040 {
2041 pv2 = (void *)(intptr_t)-4;
2042 rc = PGMDynMapHCPage(pVM, cr3, &pv2);
2043 }
2044 ASMIntEnable();
2045 if (RT_FAILURE(rc) || pv != pv2)
2046 {
2047 LogRel(("failed(%d): rc=%Rrc; pv=%p pv2=%p i=%p\n", __LINE__, rc, pv, pv2, i));
2048 if (RT_SUCCESS(rc)) rc = VERR_INTERNAL_ERROR;
2049 }
2050 else if (pSet->cEntries != 5)
2051 {
2052 LogRel(("failed(%d): cEntries=%d expected %d\n", __LINE__, pSet->cEntries, RT_ELEMENTS(pSet->aEntries) / 2));
2053 rc = VERR_INTERNAL_ERROR;
2054 }
2055 else if ( pSet->aEntries[4].cRefs != UINT16_MAX - 1
2056 || pSet->aEntries[3].cRefs != UINT16_MAX - 1
2057 || pSet->aEntries[2].cRefs != 1
2058 || pSet->aEntries[1].cRefs != 1
2059 || pSet->aEntries[0].cRefs != 1)
2060 {
2061 LogRel(("failed(%d): bad set dist: ", __LINE__));
2062 for (i = 0; i < pSet->cEntries; i++)
2063 LogRel(("[%d]=%d, ", i, pSet->aEntries[i].cRefs));
2064 LogRel(("\n"));
2065 rc = VERR_INTERNAL_ERROR;
2066 }
2067 if (RT_SUCCESS(rc))
2068 rc = PGMR0DynMapAssertIntegrity();
2069 if (RT_SUCCESS(rc))
2070 {
2071 /*
2072 * Trigger an set optimization run (exactly).
2073 */
2074 LogRel(("Test #3\n"));
2075 ASMIntDisable();
2076 PGMDynMapMigrateAutoSet(&pVM->aCpus[0]);
2077 pv2 = NULL;
2078 for (i = 0 ; i < RT_ELEMENTS(pSet->aEntries) - 5 && RT_SUCCESS(rc) && pv2 != pv; i++)
2079 {
2080 pv2 = (void *)(intptr_t)(-5 - i);
2081 rc = PGMDynMapHCPage(pVM, cr3 + PAGE_SIZE * (i + 5), &pv2);
2082 }
2083 ASMIntEnable();
2084 if (RT_FAILURE(rc) || pv == pv2)
2085 {
2086 LogRel(("failed(%d): rc=%Rrc; pv=%p pv2=%p i=%d\n", __LINE__, rc, pv, pv2, i));
2087 if (RT_SUCCESS(rc)) rc = VERR_INTERNAL_ERROR;
2088 }
2089 else if (pSet->cEntries != RT_ELEMENTS(pSet->aEntries))
2090 {
2091 LogRel(("failed(%d): cEntries=%d expected %d\n", __LINE__, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
2092 rc = VERR_INTERNAL_ERROR;
2093 }
2094 LogRel(("Load=%u/%u/%u Set=%u/%u\n", pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
2095 if (RT_SUCCESS(rc))
2096 rc = PGMR0DynMapAssertIntegrity();
2097 if (RT_SUCCESS(rc))
2098 {
2099 /*
2100 * Trigger an overflow error.
2101 */
2102 LogRel(("Test #4\n"));
2103 ASMIntDisable();
2104 PGMDynMapMigrateAutoSet(&pVM->aCpus[0]);
2105 for (i = 0 ; i < RT_ELEMENTS(pSet->aEntries) + 2; i++)
2106 {
2107 rc = PGMDynMapHCPage(pVM, cr3 - PAGE_SIZE * (i + 5), &pv2);
2108 if (RT_SUCCESS(rc))
2109 rc = PGMR0DynMapAssertIntegrity();
2110 if (RT_FAILURE(rc))
2111 break;
2112 }
2113 ASMIntEnable();
2114 if (rc == VERR_PGM_DYNMAP_FULL_SET)
2115 {
2116 /* flush the set. */
2117 LogRel(("Test #5\n"));
2118 ASMIntDisable();
2119 PGMDynMapMigrateAutoSet(&pVM->aCpus[0]);
2120 PGMDynMapReleaseAutoSet(&pVM->aCpus[0]);
2121 PGMDynMapStartAutoSet(&pVM->aCpus[0]);
2122 ASMIntEnable();
2123
2124 rc = PGMR0DynMapAssertIntegrity();
2125 }
2126 else
2127 {
2128 LogRel(("failed(%d): rc=%Rrc, wanted %d ; pv2=%p Set=%u/%u; i=%d\n", __LINE__,
2129 rc, VERR_PGM_DYNMAP_FULL_SET, pv2, pSet->cEntries, RT_ELEMENTS(pSet->aEntries), i));
2130 if (RT_SUCCESS(rc)) rc = VERR_INTERNAL_ERROR;
2131 }
2132 }
2133 }
2134 }
2135 else
2136 {
2137 LogRel(("failed(%d): rc=%Rrc rc2=%Rrc; pv=%p pv2=%p\n", __LINE__, rc, rc2, pv, pv2));
2138 if (RT_SUCCESS(rc))
2139 rc = rc2;
2140 }
2141
2142 /*
2143 * Check that everyone sees the same stuff.
2144 */
2145 if (RT_SUCCESS(rc))
2146 {
2147 LogRel(("Test #5\n"));
2148 ASMIntDisable();
2149 PGMDynMapMigrateAutoSet(&pVM->aCpus[0]);
2150 RTHCPHYS HCPhysPT = RTR0MemObjGetPagePhysAddr(pThis->pSegHead->ahMemObjPTs[0], 0);
2151 rc = PGMDynMapHCPage(pVM, HCPhysPT, &pv);
2152 if (RT_SUCCESS(rc))
2153 {
2154 PGMR0DYNMAPTEST Test;
2155 uint32_t *pu32Real = &pThis->paPages[pThis->pSegHead->iPage].uPte.pLegacy->u;
2156 Test.pu32 = (uint32_t *)((uintptr_t)pv | ((uintptr_t)pu32Real & PAGE_OFFSET_MASK));
2157 Test.u32Expect = *pu32Real;
2158 ASMAtomicWriteU32(&Test.cFailures, 0);
2159 ASMIntEnable();
2160
2161 rc = RTMpOnAll(pgmR0DynMapTest3PerCpu, &Test, NULL);
2162 if (RT_FAILURE(rc))
2163 LogRel(("failed(%d): RTMpOnAll rc=%Rrc\n", __LINE__, rc));
2164 else if (Test.cFailures)
2165 {
2166 LogRel(("failed(%d): cFailures=%d pu32Real=%p pu32=%p u32Expect=%#x *pu32=%#x\n", __LINE__,
2167 Test.cFailures, pu32Real, Test.pu32, Test.u32Expect, *Test.pu32));
2168 rc = VERR_INTERNAL_ERROR;
2169 }
2170 else
2171 LogRel(("pu32Real=%p pu32=%p u32Expect=%#x *pu32=%#x\n",
2172 pu32Real, Test.pu32, Test.u32Expect, *Test.pu32));
2173 }
2174 else
2175 {
2176 ASMIntEnable();
2177 LogRel(("failed(%d): rc=%Rrc\n", rc));
2178 }
2179 }
2180
2181 /*
2182 * Clean up.
2183 */
2184 LogRel(("Cleanup.\n"));
2185 ASMIntDisable();
2186 PGMDynMapMigrateAutoSet(&pVM->aCpus[0]);
2187 PGMDynMapFlushAutoSet(&pVM->aCpus[0]);
2188 PGMDynMapReleaseAutoSet(&pVM->aCpus[0]);
2189 ASMIntEnable();
2190
2191 if (RT_SUCCESS(rc))
2192 rc = PGMR0DynMapAssertIntegrity();
2193 else
2194 PGMR0DynMapAssertIntegrity();
2195
2196 g_fPGMR0DynMapTestRunning = false;
2197 LogRel(("Result: rc=%Rrc Load=%u/%u/%u Set=%#x/%u\n", rc,
2198 pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
2199 pVM->pgm.s.pvR0DynMapUsed = pvR0DynMapUsedSaved;
2200 LogRel(("pgmR0DynMapTest: ****** END ******\n"));
2201 return rc;
2202}
2203#endif /* DEBUG */
2204
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette