PGMRZDynMap.cpp@ 31834

Last change on this file since 31834 was 31834, checked in by vboxsync, 14 years ago
PGMRZDynMap.cpp: Dropped the PGMMAPSET_MAX_FILL checks and instead make sure we can optimize the better.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 96.6 KB

Line
1	/* $Id: PGMRZDynMap.cpp 31834 2010-08-21 00:52:39Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor, dynamic mapping cache.
4	*/
5
6	/*
7	* Copyright (C) 2008-2010 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Internal Functions *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_PGM_DYNMAP
23	#include <VBox/pgm.h>
24	#include "../PGMInternal.h"
25	#include <VBox/vm.h>
26	#include "../PGMInline.h"
27	#include <VBox/err.h>
28	#include <VBox/param.h>
29	#include <VBox/sup.h>
30	#include <iprt/asm.h>
31	#include <iprt/asm-amd64-x86.h>
32	#include <iprt/assert.h>
33	#ifndef IN_RC
34	# include <iprt/cpuset.h>
35	# include <iprt/mem.h>
36	# include <iprt/memobj.h>
37	# include <iprt/mp.h>
38	# include <iprt/semaphore.h>
39	# include <iprt/spinlock.h>
40	#endif
41	#include <iprt/string.h>
42
43
44	/*******************************************************************************
45	* Defined Constants And Macros *
46	*******************************************************************************/
47	#ifdef IN_RING0
48	/** The max size of the mapping cache (in pages). */
49	# define PGMR0DYNMAP_MAX_PAGES ((16*_1M) >> PAGE_SHIFT)
50	/** The small segment size that is adopted on out-of-memory conditions with a
51	* single big segment. */
52	# define PGMR0DYNMAP_SMALL_SEG_PAGES 128
53	/** The number of pages we reserve per CPU. */
54	# define PGMR0DYNMAP_PAGES_PER_CPU 256
55	/** The minimum number of pages we reserve per CPU.
56	* This must be equal or larger than the autoset size. */
57	# define PGMR0DYNMAP_PAGES_PER_CPU_MIN 64
58	/** Calcs the overload threshold (safety margin). Current set at 50%. */
59	# define PGMR0DYNMAP_CALC_OVERLOAD(cPages) ((cPages) / 2)
60	/** The number of guard pages.
61	* @remarks Never do tuning of the hashing or whatnot with a strict build! */
62	# if defined(VBOX_STRICT)
63	# define PGMR0DYNMAP_GUARD_PAGES 1
64	# else
65	# define PGMR0DYNMAP_GUARD_PAGES 0
66	# endif
67	#endif /* IN_RING0 */
68	/** The dummy physical address of guard pages. */
69	#define PGMR0DYNMAP_GUARD_PAGE_HCPHYS UINT32_C(0x7777feed)
70	/** The dummy reference count of guard pages. (Must be non-zero.) */
71	#define PGMR0DYNMAP_GUARD_PAGE_REF_COUNT INT32_C(0x7777feed)
72	#if 0
73	/** Define this to just clear the present bit on guard pages.
74	* The alternative is to replace the entire PTE with an bad not-present
75	* PTE. Either way, XNU will screw us. :-/ */
76	# define PGMR0DYNMAP_GUARD_NP
77	#endif
78	/** The dummy PTE value for a page. */
79	#define PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE X86_PTE_PG_MASK
80	/** The dummy PTE value for a page. */
81	#define PGMR0DYNMAP_GUARD_PAGE_PAE_PTE UINT64_MAX /X86_PTE_PAE_PG_MASK/
82
83	#ifdef IN_RING0 /* Note! Assertions causes panics if preemption is disabled,
84	* disable this to work around that. */
85	/**
86	* Acquire the spinlock.
87	* This will declare a temporary variable and expands to two statements!
88	*/
89	# define PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis) \
90	RTSPINLOCKTMP MySpinlockTmp = RTSPINLOCKTMP_INITIALIZER; \
91	RTSpinlockAcquire((pThis)->hSpinlock, &MySpinlockTmp)
92	/**
93	* Releases the spinlock.
94	*/
95	# define PGMRZDYNMAP_SPINLOCK_RELEASE(pThis) \
96	RTSpinlockRelease((pThis)->hSpinlock, &MySpinlockTmp)
97
98	/**
99	* Re-acquires the spinlock.
100	*/
101	# define PGMRZDYNMAP_SPINLOCK_REACQUIRE(pThis) \
102	RTSpinlockAcquire((pThis)->hSpinlock, &MySpinlockTmp)
103	#else
104	# define PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis) do { } while (0)
105	# define PGMRZDYNMAP_SPINLOCK_RELEASE(pThis) do { } while (0)
106	# define PGMRZDYNMAP_SPINLOCK_REACQUIRE(pThis) do { } while (0)
107	#endif
108
109
110	/** Converts a PGMCPUM::AutoSet pointer into a PVMCPU. */
111	#define PGMRZDYNMAP_SET_2_VMCPU(pSet) (RT_FROM_MEMBER(pSet, VMCPU, pgm.s.AutoSet))
112
113	/** Converts a PGMCPUM::AutoSet pointer into a PVM. */
114	#define PGMRZDYNMAP_SET_2_VM(pSet) (PGMRZDYNMAP_SET_2_VMCPU(pSet)->CTX_SUFF(pVM))
115
116	/** Converts a PGMCPUM::AutoSet pointer into a PVM. */
117	#ifdef IN_RC
118	# define PGMRZDYNMAP_SET_2_DYNMAP(pSet) (PGMRZDYNMAP_SET_2_VM(pSet)->pgm.s.pRCDynMap)
119	#else
120	# define PGMRZDYNMAP_SET_2_DYNMAP(pSet) (g_pPGMR0DynMap)
121	#endif
122
123	/**
124	* Gets the set index of the current CPU.
125	*
126	* This always returns 0 when in raw-mode context because there is only ever
127	* one EMT in that context (at least presently).
128	*/
129	#ifdef IN_RC
130	# define PGMRZDYNMAP_CUR_CPU() (0)
131	#else
132	# define PGMRZDYNMAP_CUR_CPU() RTMpCpuIdToSetIndex(RTMpCpuId())
133	#endif
134
135	/** PGMRZDYNMAP::u32Magic. (Jens Christian Bugge Wesseltoft) */
136	#define PGMRZDYNMAP_MAGIC UINT32_C(0x19640201)
137
138
139	/** Zaps an set entry. */
140	#define PGMRZDYNMAP_ZAP_ENTRY(pEntry) \
141	do \
142	{ \
143	(pEntry)->iPage = UINT16_MAX; \
144	(pEntry)->cRefs = 0; \
145	(pEntry)->cInlinedRefs = 0; \
146	(pEntry)->cUnrefs = 0; \
147	} while (0)
148
149
150	/** @def PGMRZDYNMAP_STRICT_RELEASE
151	* Define this to force pages to be released and make non-present ASAP after
152	* use. This should not normally be enabled as it is a bit expensive. */
153	#if 0 \|\| defined(DOXYGEN_RUNNING)
154	# define PGMRZDYNMAP_STRICT_RELEASE
155	#endif
156
157
158	/*******************************************************************************
159	* Structures and Typedefs *
160	*******************************************************************************/
161	#ifdef IN_RING0
162	/**
163	* Ring-0 dynamic mapping cache segment.
164	*
165	* The dynamic mapping cache can be extended with additional segments if the
166	* load is found to be too high. This done the next time a VM is created, under
167	* the protection of the init mutex. The arrays is reallocated and the new
168	* segment is added to the end of these. Nothing is rehashed of course, as the
169	* indexes / addresses must remain unchanged.
170	*
171	* This structure is only modified while owning the init mutex or during module
172	* init / term.
173	*/
174	typedef struct PGMR0DYNMAPSEG
175	{
176	/** Pointer to the next segment. */
177	struct PGMR0DYNMAPSEG *pNext;
178	/** The memory object for the virtual address range that we're abusing. */
179	RTR0MEMOBJ hMemObj;
180	/** The start page in the cache. (I.e. index into the arrays.) */
181	uint16_t iPage;
182	/** The number of pages this segment contributes. */
183	uint16_t cPages;
184	/** The number of page tables. */
185	uint16_t cPTs;
186	/** The memory objects for the page tables. */
187	RTR0MEMOBJ ahMemObjPTs[1];
188	} PGMR0DYNMAPSEG;
189	/** Pointer to a ring-0 dynamic mapping cache segment. */
190	typedef PGMR0DYNMAPSEG *PPGMR0DYNMAPSEG;
191
192
193	/**
194	* Ring-0 dynamic mapping cache entry.
195	*
196	* @sa PGMRZDYNMAPENTRY, PGMRCDYNMAPENTRY.
197	*/
198	typedef struct PGMR0DYNMAPENTRY
199	{
200	/** The physical address of the currently mapped page.
201	* This is duplicate for three reasons: cache locality, cache policy of the PT
202	* mappings and sanity checks. */
203	RTHCPHYS HCPhys;
204	/** Pointer to the page. */
205	void *pvPage;
206	/** The number of references. */
207	int32_t volatile cRefs;
208	/** PTE pointer union. */
209	union PGMR0DYNMAPENTRY_PPTE
210	{
211	/** PTE pointer, 32-bit legacy version. */
212	PX86PTE pLegacy;
213	/** PTE pointer, PAE version. */
214	PX86PTEPAE pPae;
215	/** PTE pointer, the void version. */
216	void *pv;
217	} uPte;
218	# ifndef IN_RC
219	/** CPUs that haven't invalidated this entry after it's last update. */
220	RTCPUSET PendingSet;
221	# endif
222	} PGMR0DYNMAPENTRY;
223	/** Pointer a mapping cache entry for the ring-0.
224	* @sa PPGMRZDYNMAPENTRY, PPGMRCDYNMAPENTRY, */
225	typedef PGMR0DYNMAPENTRY *PPGMR0DYNMAPENTRY;
226
227
228	/**
229	* Dynamic mapping cache for ring-0.
230	*
231	* This is initialized during VMMR0 module init but no segments are allocated
232	* at that time. Segments will be added when the first VM is started and
233	* removed again when the last VM shuts down, thus avoid consuming memory while
234	* dormant. At module termination, the remaining bits will be freed up.
235	*
236	* @sa PPGMRZDYNMAP, PGMRCDYNMAP.
237	*/
238	typedef struct PGMR0DYNMAP
239	{
240	/** The usual magic number / eye catcher (PGMRZDYNMAP_MAGIC). */
241	uint32_t u32Magic;
242	# ifndef IN_RC
243	/** Spinlock serializing the normal operation of the cache. */
244	RTSPINLOCK hSpinlock;
245	# endif
246	/** Array for tracking and managing the pages. */
247	PPGMR0DYNMAPENTRY paPages;
248	/** The cache size given as a number of pages. */
249	uint32_t cPages;
250	/** Whether it's 32-bit legacy or PAE/AMD64 paging mode. */
251	bool fLegacyMode;
252	/** The current load.
253	* This does not include guard pages. */
254	uint32_t cLoad;
255	/** The max load ever.
256	* This is maintained to get trigger adding of more mapping space. */
257	uint32_t cMaxLoad;
258	# ifndef IN_RC
259	/** Initialization / termination lock. */
260	RTSEMFASTMUTEX hInitLock;
261	# endif
262	/** The number of guard pages. */
263	uint32_t cGuardPages;
264	/** The number of users (protected by hInitLock). */
265	uint32_t cUsers;
266	# ifndef IN_RC
267	/** Array containing a copy of the original page tables.
268	* The entries are either X86PTE or X86PTEPAE according to fLegacyMode. */
269	void *pvSavedPTEs;
270	/** List of segments. */
271	PPGMR0DYNMAPSEG pSegHead;
272	/** The paging mode. */
273	SUPPAGINGMODE enmPgMode;
274	# endif
275	} PGMR0DYNMAP;
276
277
278	/**
279	* Paging level data.
280	*/
281	typedef struct PGMR0DYNMAPPGLVL
282	{
283	uint32_t cLevels; /*< The number of levels. /
284	struct
285	{
286	RTHCPHYS HCPhys; /**< The address of the page for the current level,
287	* i.e. what hMemObj/hMapObj is currently mapping. */
288	RTHCPHYS fPhysMask; /*< Mask for extracting HCPhys from uEntry. /
289	RTR0MEMOBJ hMemObj; /*< Memory object for HCPhys, PAGE_SIZE. /
290	RTR0MEMOBJ hMapObj; /*< Mapping object for hMemObj. /
291	uint32_t fPtrShift; /*< The pointer shift count. /
292	uint64_t fPtrMask; /*< The mask to apply to the shifted pointer to get the table index. /
293	uint64_t fAndMask; /*< And mask to check entry flags. /
294	uint64_t fResMask; /*< The result from applying fAndMask. /
295	union
296	{
297	void pv; /< hMapObj address. /
298	PX86PGUINT paLegacy; /*< Legacy table view. /
299	PX86PGPAEUINT paPae; /*< PAE/AMD64 table view. /
300	} u;
301	} a[4];
302	} PGMR0DYNMAPPGLVL;
303	/** Pointer to paging level data. */
304	typedef PGMR0DYNMAPPGLVL *PPGMR0DYNMAPPGLVL;
305	#endif
306
307	/** Mapping cache entry for the current context.
308	* @sa PGMR0DYNMAPENTRY, PGMRCDYNMAPENTRY */
309	typedef CTX_MID(PGM,DYNMAPENTRY) PGMRZDYNMAPENTRY;
310	/** Pointer a mapping cache entry for the current context.
311	* @sa PGMR0DYNMAPENTRY, PGMRCDYNMAPENTRY */
312	typedef PGMRZDYNMAPENTRY *PPGMRZDYNMAPENTRY;
313
314	/** Pointer the mapping cache instance for the current context.
315	* @sa PGMR0DYNMAP, PGMRCDYNMAP */
316	typedef CTX_MID(PGM,DYNMAP) *PPGMRZDYNMAP;
317
318
319
320	/*******************************************************************************
321	* Global Variables *
322	*******************************************************************************/
323	#ifdef IN_RING0
324	/** Pointer to the ring-0 dynamic mapping cache. */
325	static PGMR0DYNMAP *g_pPGMR0DynMap;
326	#endif
327	/** For overflow testing. */
328	static bool g_fPGMR0DynMapTestRunning = false;
329
330
331	/*******************************************************************************
332	* Internal Functions *
333	*******************************************************************************/
334	static void pgmRZDynMapReleasePage(PPGMRZDYNMAP pThis, uint32_t iPage, uint32_t cRefs);
335	#ifdef IN_RING0
336	static int pgmR0DynMapSetup(PPGMRZDYNMAP pThis);
337	static int pgmR0DynMapExpand(PPGMRZDYNMAP pThis);
338	static void pgmR0DynMapTearDown(PPGMRZDYNMAP pThis);
339	#endif
340	#if 0 /def DEBUG/
341	static int pgmR0DynMapTest(PVM pVM);
342	#endif
343
344
345	/**
346	* Initializes the auto mapping sets for a VM.
347	*
348	* @returns VINF_SUCCESS on success, VERR_INTERNAL_ERROR on failure.
349	* @param pVM The VM in question.
350	*/
351	static int pgmRZDynMapInitAutoSetsForVM(PVM pVM)
352	{
353	VMCPUID idCpu = pVM->cCpus;
354	AssertReturn(idCpu > 0 && idCpu <= VMM_MAX_CPU_COUNT, VERR_INTERNAL_ERROR);
355	while (idCpu-- > 0)
356	{
357	PPGMMAPSET pSet = &pVM->aCpus[idCpu].pgm.s.AutoSet;
358	uint32_t j = RT_ELEMENTS(pSet->aEntries);
359	while (j-- > 0)
360	{
361	pSet->aEntries[j].pvPage = NULL;
362	pSet->aEntries[j].HCPhys = NIL_RTHCPHYS;
363	PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[j]);
364	}
365	pSet->cEntries = PGMMAPSET_CLOSED;
366	pSet->iSubset = UINT32_MAX;
367	pSet->iCpu = -1;
368	memset(&pSet->aiHashTable[0], 0xff, sizeof(pSet->aiHashTable));
369	}
370
371	return VINF_SUCCESS;
372	}
373
374
375	#ifdef IN_RING0
376
377	/**
378	* Initializes the ring-0 dynamic mapping cache.
379	*
380	* @returns VBox status code.
381	*/
382	VMMR0DECL(int) PGMR0DynMapInit(void)
383	{
384	Assert(!g_pPGMR0DynMap);
385
386	/*
387	* Create and initialize the cache instance.
388	*/
389	PPGMRZDYNMAP pThis = (PPGMRZDYNMAP)RTMemAllocZ(sizeof(*pThis));
390	AssertLogRelReturn(pThis, VERR_NO_MEMORY);
391	int rc = VINF_SUCCESS;
392	pThis->enmPgMode = SUPR0GetPagingMode();
393	switch (pThis->enmPgMode)
394	{
395	case SUPPAGINGMODE_32_BIT:
396	case SUPPAGINGMODE_32_BIT_GLOBAL:
397	pThis->fLegacyMode = false;
398	break;
399	case SUPPAGINGMODE_PAE:
400	case SUPPAGINGMODE_PAE_GLOBAL:
401	case SUPPAGINGMODE_PAE_NX:
402	case SUPPAGINGMODE_PAE_GLOBAL_NX:
403	case SUPPAGINGMODE_AMD64:
404	case SUPPAGINGMODE_AMD64_GLOBAL:
405	case SUPPAGINGMODE_AMD64_NX:
406	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
407	pThis->fLegacyMode = false;
408	break;
409	default:
410	rc = VERR_INTERNAL_ERROR;
411	break;
412	}
413	if (RT_SUCCESS(rc))
414	{
415	rc = RTSemFastMutexCreate(&pThis->hInitLock);
416	if (RT_SUCCESS(rc))
417	{
418	rc = RTSpinlockCreate(&pThis->hSpinlock);
419	if (RT_SUCCESS(rc))
420	{
421	pThis->u32Magic = PGMRZDYNMAP_MAGIC;
422	g_pPGMR0DynMap = pThis;
423	return VINF_SUCCESS;
424	}
425	RTSemFastMutexDestroy(pThis->hInitLock);
426	}
427	}
428	RTMemFree(pThis);
429	return rc;
430	}
431
432
433	/**
434	* Terminates the ring-0 dynamic mapping cache.
435	*/
436	VMMR0DECL(void) PGMR0DynMapTerm(void)
437	{
438	/*
439	* Destroy the cache.
440	*
441	* There is not supposed to be any races here, the loader should
442	* make sure about that. So, don't bother locking anything.
443	*
444	* The VM objects should all be destroyed by now, so there is no
445	* dangling users or anything like that to clean up. This routine
446	* is just a mirror image of PGMR0DynMapInit.
447	*/
448	PPGMRZDYNMAP pThis = g_pPGMR0DynMap;
449	if (pThis)
450	{
451	AssertPtr(pThis);
452	g_pPGMR0DynMap = NULL;
453
454	/* This should never happen, but in case it does try not to leak memory. */
455	AssertLogRelMsg(!pThis->cUsers && !pThis->paPages && !pThis->pvSavedPTEs && !pThis->cPages,
456	("cUsers=%d paPages=%p pvSavedPTEs=%p cPages=%#x\n",
457	pThis->cUsers, pThis->paPages, pThis->pvSavedPTEs, pThis->cPages));
458	if (pThis->paPages)
459	pgmR0DynMapTearDown(pThis);
460
461	/* Free the associated resources. */
462	RTSemFastMutexDestroy(pThis->hInitLock);
463	pThis->hInitLock = NIL_RTSEMFASTMUTEX;
464	RTSpinlockDestroy(pThis->hSpinlock);
465	pThis->hSpinlock = NIL_RTSPINLOCK;
466	pThis->u32Magic = UINT32_MAX;
467	RTMemFree(pThis);
468	}
469	}
470
471
472	/**
473	* Initializes the dynamic mapping cache for a new VM.
474	*
475	* @returns VBox status code.
476	* @param pVM Pointer to the shared VM structure.
477	*/
478	VMMR0DECL(int) PGMR0DynMapInitVM(PVM pVM)
479	{
480	AssertMsgReturn(!pVM->pgm.s.pvR0DynMapUsed, ("%p (pThis=%p)\n", pVM->pgm.s.pvR0DynMapUsed, g_pPGMR0DynMap), VERR_WRONG_ORDER);
481
482	/*
483	* Initialize the auto sets.
484	*/
485	int rc = pgmRZDynMapInitAutoSetsForVM(pVM);
486	if (RT_FAILURE(rc))
487	return rc;
488
489	/*
490	* Do we need the cache? Skip the last bit if we don't.
491	*/
492	if (!VMMIsHwVirtExtForced(pVM))
493	return VINF_SUCCESS;
494
495	/*
496	* Reference and if necessary setup or expand the cache.
497	*/
498	PPGMRZDYNMAP pThis = g_pPGMR0DynMap;
499	AssertPtrReturn(pThis, VERR_INTERNAL_ERROR);
500	rc = RTSemFastMutexRequest(pThis->hInitLock);
501	AssertLogRelRCReturn(rc, rc);
502
503	pThis->cUsers++;
504	if (pThis->cUsers == 1)
505	{
506	rc = pgmR0DynMapSetup(pThis);
507	#if 0 /def DEBUG/
508	if (RT_SUCCESS(rc))
509	{
510	rc = pgmR0DynMapTest(pVM);
511	if (RT_FAILURE(rc))
512	pgmR0DynMapTearDown(pThis);
513	}
514	#endif
515	}
516	else if (pThis->cMaxLoad > PGMR0DYNMAP_CALC_OVERLOAD(pThis->cPages - pThis->cGuardPages))
517	rc = pgmR0DynMapExpand(pThis);
518	if (RT_SUCCESS(rc))
519	pVM->pgm.s.pvR0DynMapUsed = pThis;
520	else
521	pThis->cUsers--;
522
523	RTSemFastMutexRelease(pThis->hInitLock);
524	return rc;
525	}
526
527
528	/**
529	* Terminates the dynamic mapping cache usage for a VM.
530	*
531	* @param pVM Pointer to the shared VM structure.
532	*/
533	VMMR0DECL(void) PGMR0DynMapTermVM(PVM pVM)
534	{
535	/*
536	* Return immediately if we're not using the cache.
537	*/
538	if (!pVM->pgm.s.pvR0DynMapUsed)
539	return;
540
541	PPGMRZDYNMAP pThis = g_pPGMR0DynMap;
542	AssertPtrReturnVoid(pThis);
543
544	int rc = RTSemFastMutexRequest(pThis->hInitLock);
545	AssertLogRelRCReturnVoid(rc);
546
547	if (pVM->pgm.s.pvR0DynMapUsed == pThis)
548	{
549	pVM->pgm.s.pvR0DynMapUsed = NULL;
550
551	#ifdef VBOX_STRICT
552	PGMR0DynMapAssertIntegrity();
553	#endif
554
555	/*
556	* Clean up and check the auto sets.
557	*/
558	VMCPUID idCpu = pVM->cCpus;
559	while (idCpu-- > 0)
560	{
561	PPGMMAPSET pSet = &pVM->aCpus[idCpu].pgm.s.AutoSet;
562	uint32_t j = pSet->cEntries;
563	if (j <= RT_ELEMENTS(pSet->aEntries))
564	{
565	/*
566	* The set is open, close it.
567	*/
568	while (j-- > 0)
569	{
570	int32_t cRefs = pSet->aEntries[j].cRefs;
571	uint32_t iPage = pSet->aEntries[j].iPage;
572	LogRel(("PGMR0DynMapTermVM: %d dangling refs to %#x\n", cRefs, iPage));
573	if (iPage < pThis->cPages && cRefs > 0)
574	pgmRZDynMapReleasePage(pThis, iPage, cRefs);
575	else
576	AssertLogRelMsgFailed(("cRefs=%d iPage=%#x cPages=%u\n", cRefs, iPage, pThis->cPages));
577
578	PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[j]);
579	}
580	pSet->cEntries = PGMMAPSET_CLOSED;
581	pSet->iSubset = UINT32_MAX;
582	pSet->iCpu = -1;
583	}
584	else
585	AssertMsg(j == PGMMAPSET_CLOSED, ("cEntries=%#x\n", j));
586
587	j = RT_ELEMENTS(pSet->aEntries);
588	while (j-- > 0)
589	{
590	Assert(pSet->aEntries[j].iPage == UINT16_MAX);
591	Assert(!pSet->aEntries[j].cRefs);
592	}
593	}
594
595	/*
596	* Release our reference to the mapping cache.
597	*/
598	Assert(pThis->cUsers > 0);
599	pThis->cUsers--;
600	if (!pThis->cUsers)
601	pgmR0DynMapTearDown(pThis);
602	}
603	else
604	AssertLogRelMsgFailed(("pvR0DynMapUsed=%p pThis=%p\n", pVM->pgm.s.pvR0DynMapUsed, pThis));
605
606	RTSemFastMutexRelease(pThis->hInitLock);
607	}
608
609
610	/**
611	* Shoots down the TLBs for all the cache pages, pgmR0DynMapTearDown helper.
612	*
613	* @param idCpu The current CPU.
614	* @param pvUser1 The dynamic mapping cache instance.
615	* @param pvUser2 Unused, NULL.
616	*/
617	static DECLCALLBACK(void) pgmR0DynMapShootDownTlbs(RTCPUID idCpu, void pvUser1, void pvUser2)
618	{
619	Assert(!pvUser2);
620	PPGMRZDYNMAP pThis = (PPGMRZDYNMAP)pvUser1;
621	Assert(pThis == g_pPGMR0DynMap);
622	PPGMRZDYNMAPENTRY paPages = pThis->paPages;
623	uint32_t iPage = pThis->cPages;
624	while (iPage-- > 0)
625	ASMInvalidatePage(paPages[iPage].pvPage);
626	}
627
628
629	/**
630	* Shoot down the TLBs for every single cache entry on all CPUs.
631	*
632	* @returns IPRT status code (RTMpOnAll).
633	* @param pThis The dynamic mapping cache instance.
634	*/
635	static int pgmR0DynMapTlbShootDown(PPGMRZDYNMAP pThis)
636	{
637	int rc = RTMpOnAll(pgmR0DynMapShootDownTlbs, pThis, NULL);
638	AssertRC(rc);
639	if (RT_FAILURE(rc))
640	{
641	uint32_t iPage = pThis->cPages;
642	while (iPage-- > 0)
643	ASMInvalidatePage(pThis->paPages[iPage].pvPage);
644	}
645	return rc;
646	}
647
648
649	/**
650	* Calculate the new cache size based on cMaxLoad statistics.
651	*
652	* @returns Number of pages.
653	* @param pThis The dynamic mapping cache instance.
654	* @param pcMinPages The minimal size in pages.
655	*/
656	static uint32_t pgmR0DynMapCalcNewSize(PPGMRZDYNMAP pThis, uint32_t *pcMinPages)
657	{
658	Assert(pThis->cPages <= PGMR0DYNMAP_MAX_PAGES);
659
660	/* cCpus * PGMR0DYNMAP_PAGES_PER_CPU(_MIN). */
661	RTCPUID cCpus = RTMpGetCount();
662	AssertReturn(cCpus > 0 && cCpus <= RTCPUSET_MAX_CPUS, 0);
663	uint32_t cPages = cCpus * PGMR0DYNMAP_PAGES_PER_CPU;
664	uint32_t cMinPages = cCpus * PGMR0DYNMAP_PAGES_PER_CPU_MIN;
665
666	/* adjust against cMaxLoad. */
667	AssertMsg(pThis->cMaxLoad <= PGMR0DYNMAP_MAX_PAGES, ("%#x\n", pThis->cMaxLoad));
668	if (pThis->cMaxLoad > PGMR0DYNMAP_MAX_PAGES)
669	pThis->cMaxLoad = 0;
670
671	while (pThis->cMaxLoad > PGMR0DYNMAP_CALC_OVERLOAD(cPages))
672	cPages += PGMR0DYNMAP_PAGES_PER_CPU;
673
674	if (pThis->cMaxLoad > cMinPages)
675	cMinPages = pThis->cMaxLoad;
676
677	/* adjust against max and current size. */
678	if (cPages < pThis->cPages)
679	cPages = pThis->cPages;
680	cPages *= PGMR0DYNMAP_GUARD_PAGES + 1;
681	if (cPages > PGMR0DYNMAP_MAX_PAGES)
682	cPages = PGMR0DYNMAP_MAX_PAGES;
683
684	if (cMinPages < pThis->cPages)
685	cMinPages = pThis->cPages;
686	cMinPages *= PGMR0DYNMAP_GUARD_PAGES + 1;
687	if (cMinPages > PGMR0DYNMAP_MAX_PAGES)
688	cMinPages = PGMR0DYNMAP_MAX_PAGES;
689
690	Assert(cMinPages);
691	*pcMinPages = cMinPages;
692	return cPages;
693	}
694
695
696	/**
697	* Initializes the paging level data.
698	*
699	* @param pThis The dynamic mapping cache instance.
700	* @param pPgLvl The paging level data.
701	*/
702	void pgmR0DynMapPagingArrayInit(PPGMRZDYNMAP pThis, PPGMR0DYNMAPPGLVL pPgLvl)
703	{
704	RTCCUINTREG cr4 = ASMGetCR4();
705	switch (pThis->enmPgMode)
706	{
707	case SUPPAGINGMODE_32_BIT:
708	case SUPPAGINGMODE_32_BIT_GLOBAL:
709	pPgLvl->cLevels = 2;
710	pPgLvl->a[0].fPhysMask = X86_CR3_PAGE_MASK;
711	pPgLvl->a[0].fAndMask = X86_PDE_P \| X86_PDE_RW \| (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0);
712	pPgLvl->a[0].fResMask = X86_PDE_P \| X86_PDE_RW;
713	pPgLvl->a[0].fPtrMask = X86_PD_MASK;
714	pPgLvl->a[0].fPtrShift = X86_PD_SHIFT;
715
716	pPgLvl->a[1].fPhysMask = X86_PDE_PG_MASK;
717	pPgLvl->a[1].fAndMask = X86_PTE_P \| X86_PTE_RW;
718	pPgLvl->a[1].fResMask = X86_PTE_P \| X86_PTE_RW;
719	pPgLvl->a[1].fPtrMask = X86_PT_MASK;
720	pPgLvl->a[1].fPtrShift = X86_PT_SHIFT;
721	break;
722
723	case SUPPAGINGMODE_PAE:
724	case SUPPAGINGMODE_PAE_GLOBAL:
725	case SUPPAGINGMODE_PAE_NX:
726	case SUPPAGINGMODE_PAE_GLOBAL_NX:
727	pPgLvl->cLevels = 3;
728	pPgLvl->a[0].fPhysMask = X86_CR3_PAE_PAGE_MASK;
729	pPgLvl->a[0].fPtrMask = X86_PDPT_MASK_PAE;
730	pPgLvl->a[0].fPtrShift = X86_PDPT_SHIFT;
731	pPgLvl->a[0].fAndMask = X86_PDPE_P;
732	pPgLvl->a[0].fResMask = X86_PDPE_P;
733
734	pPgLvl->a[1].fPhysMask = X86_PDPE_PG_MASK;
735	pPgLvl->a[1].fPtrMask = X86_PD_PAE_MASK;
736	pPgLvl->a[1].fPtrShift = X86_PD_PAE_SHIFT;
737	pPgLvl->a[1].fAndMask = X86_PDE_P \| X86_PDE_RW \| (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0);
738	pPgLvl->a[1].fResMask = X86_PDE_P \| X86_PDE_RW;
739
740	pPgLvl->a[2].fPhysMask = X86_PDE_PAE_PG_MASK;
741	pPgLvl->a[2].fPtrMask = X86_PT_PAE_MASK;
742	pPgLvl->a[2].fPtrShift = X86_PT_PAE_SHIFT;
743	pPgLvl->a[2].fAndMask = X86_PTE_P \| X86_PTE_RW;
744	pPgLvl->a[2].fResMask = X86_PTE_P \| X86_PTE_RW;
745	break;
746
747	case SUPPAGINGMODE_AMD64:
748	case SUPPAGINGMODE_AMD64_GLOBAL:
749	case SUPPAGINGMODE_AMD64_NX:
750	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
751	pPgLvl->cLevels = 4;
752	pPgLvl->a[0].fPhysMask = X86_CR3_AMD64_PAGE_MASK;
753	pPgLvl->a[0].fPtrShift = X86_PML4_SHIFT;
754	pPgLvl->a[0].fPtrMask = X86_PML4_MASK;
755	pPgLvl->a[0].fAndMask = X86_PML4E_P \| X86_PML4E_RW;
756	pPgLvl->a[0].fResMask = X86_PML4E_P \| X86_PML4E_RW;
757
758	pPgLvl->a[1].fPhysMask = X86_PML4E_PG_MASK;
759	pPgLvl->a[1].fPtrShift = X86_PDPT_SHIFT;
760	pPgLvl->a[1].fPtrMask = X86_PDPT_MASK_AMD64;
761	pPgLvl->a[1].fAndMask = X86_PDPE_P \| X86_PDPE_RW /** @todo check for X86_PDPT_PS support. */;
762	pPgLvl->a[1].fResMask = X86_PDPE_P \| X86_PDPE_RW;
763
764	pPgLvl->a[2].fPhysMask = X86_PDPE_PG_MASK;
765	pPgLvl->a[2].fPtrShift = X86_PD_PAE_SHIFT;
766	pPgLvl->a[2].fPtrMask = X86_PD_PAE_MASK;
767	pPgLvl->a[2].fAndMask = X86_PDE_P \| X86_PDE_RW \| (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0);
768	pPgLvl->a[2].fResMask = X86_PDE_P \| X86_PDE_RW;
769
770	pPgLvl->a[3].fPhysMask = X86_PDE_PAE_PG_MASK;
771	pPgLvl->a[3].fPtrShift = X86_PT_PAE_SHIFT;
772	pPgLvl->a[3].fPtrMask = X86_PT_PAE_MASK;
773	pPgLvl->a[3].fAndMask = X86_PTE_P \| X86_PTE_RW;
774	pPgLvl->a[3].fResMask = X86_PTE_P \| X86_PTE_RW;
775	break;
776
777	default:
778	AssertFailed();
779	pPgLvl->cLevels = 0;
780	break;
781	}
782
783	for (uint32_t i = 0; i < 4; i++) /* ASSUMING array size. */
784	{
785	pPgLvl->a[i].HCPhys = NIL_RTHCPHYS;
786	pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ;
787	pPgLvl->a[i].hMemObj = NIL_RTR0MEMOBJ;
788	pPgLvl->a[i].u.pv = NULL;
789	}
790	}
791
792
793	/**
794	* Maps a PTE.
795	*
796	* This will update the segment structure when new PTs are mapped.
797	*
798	* It also assumes that we (for paranoid reasons) wish to establish a mapping
799	* chain from CR3 to the PT that all corresponds to the processor we're
800	* currently running on, and go about this by running with interrupts disabled
801	* and restarting from CR3 for every change.
802	*
803	* @returns VBox status code, VINF_TRY_AGAIN if we changed any mappings and had
804	* to re-enable interrupts.
805	* @param pThis The dynamic mapping cache instance.
806	* @param pPgLvl The paging level structure.
807	* @param pvPage The page.
808	* @param pSeg The segment.
809	* @param cMaxPTs The max number of PTs expected in the segment.
810	* @param ppvPTE Where to store the PTE address.
811	*/
812	static int pgmR0DynMapPagingArrayMapPte(PPGMRZDYNMAP pThis, PPGMR0DYNMAPPGLVL pPgLvl, void *pvPage,
813	PPGMR0DYNMAPSEG pSeg, uint32_t cMaxPTs, void **ppvPTE)
814	{
815	Assert(!(ASMGetFlags() & X86_EFL_IF));
816	void *pvEntry = NULL;
817	X86PGPAEUINT uEntry = ASMGetCR3();
818	for (uint32_t i = 0; i < pPgLvl->cLevels; i++)
819	{
820	RTHCPHYS HCPhys = uEntry & pPgLvl->a[i].fPhysMask;
821	if (pPgLvl->a[i].HCPhys != HCPhys)
822	{
823	/*
824	* Need to remap this level.
825	* The final level, the PT, will not be freed since that is what it's all about.
826	*/
827	ASMIntEnable();
828	if (i + 1 == pPgLvl->cLevels)
829	AssertReturn(pSeg->cPTs < cMaxPTs, VERR_INTERNAL_ERROR);
830	else
831	{
832	int rc2 = RTR0MemObjFree(pPgLvl->a[i].hMemObj, true /* fFreeMappings */); AssertRC(rc2);
833	pPgLvl->a[i].hMemObj = pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ;
834	}
835
836	int rc = RTR0MemObjEnterPhys(&pPgLvl->a[i].hMemObj, HCPhys, PAGE_SIZE, RTMEM_CACHE_POLICY_DONT_CARE);
837	if (RT_SUCCESS(rc))
838	{
839	rc = RTR0MemObjMapKernel(&pPgLvl->a[i].hMapObj, pPgLvl->a[i].hMemObj,
840	(void )-1 / pvFixed /, 0 / cbAlignment */,
841	RTMEM_PROT_WRITE \| RTMEM_PROT_READ);
842	if (RT_SUCCESS(rc))
843	{
844	pPgLvl->a[i].u.pv = RTR0MemObjAddress(pPgLvl->a[i].hMapObj);
845	AssertMsg(((uintptr_t)pPgLvl->a[i].u.pv & ~(uintptr_t)PAGE_OFFSET_MASK), ("%p\n", pPgLvl->a[i].u.pv));
846	pPgLvl->a[i].HCPhys = HCPhys;
847	if (i + 1 == pPgLvl->cLevels)
848	pSeg->ahMemObjPTs[pSeg->cPTs++] = pPgLvl->a[i].hMemObj;
849	ASMIntDisable();
850	return VINF_TRY_AGAIN;
851	}
852
853	pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ;
854	}
855	else
856	pPgLvl->a[i].hMemObj = NIL_RTR0MEMOBJ;
857	pPgLvl->a[i].HCPhys = NIL_RTHCPHYS;
858	return rc;
859	}
860
861	/*
862	* The next level.
863	*/
864	uint32_t iEntry = ((uint64_t)(uintptr_t)pvPage >> pPgLvl->a[i].fPtrShift) & pPgLvl->a[i].fPtrMask;
865	if (pThis->fLegacyMode)
866	{
867	pvEntry = &pPgLvl->a[i].u.paLegacy[iEntry];
868	uEntry = pPgLvl->a[i].u.paLegacy[iEntry];
869	}
870	else
871	{
872	pvEntry = &pPgLvl->a[i].u.paPae[iEntry];
873	uEntry = pPgLvl->a[i].u.paPae[iEntry];
874	}
875
876	if ((uEntry & pPgLvl->a[i].fAndMask) != pPgLvl->a[i].fResMask)
877	{
878	LogRel(("PGMR0DynMap: internal error - iPgLvl=%u cLevels=%u uEntry=%#llx fAnd=%#llx fRes=%#llx got=%#llx\n"
879	"PGMR0DynMap: pv=%p pvPage=%p iEntry=%#x fLegacyMode=%RTbool\n",
880	i, pPgLvl->cLevels, uEntry, pPgLvl->a[i].fAndMask, pPgLvl->a[i].fResMask, uEntry & pPgLvl->a[i].fAndMask,
881	pPgLvl->a[i].u.pv, pvPage, iEntry, pThis->fLegacyMode));
882	return VERR_INTERNAL_ERROR;
883	}
884	/Log(("#%d: iEntry=%4d uEntry=%#llx pvEntry=%p HCPhys=%RHp \n", i, iEntry, uEntry, pvEntry, pPgLvl->a[i].HCPhys));/
885	}
886
887	/* made it thru without needing to remap anything. */
888	*ppvPTE = pvEntry;
889	return VINF_SUCCESS;
890	}
891
892
893	/**
894	* Sets up a guard page.
895	*
896	* @param pThis The dynamic mapping cache instance.
897	* @param pPage The page.
898	*/
899	DECLINLINE(void) pgmR0DynMapSetupGuardPage(PPGMRZDYNMAP pThis, PPGMRZDYNMAPENTRY pPage)
900	{
901	memset(pPage->pvPage, 0xfd, PAGE_SIZE);
902	pPage->cRefs = PGMR0DYNMAP_GUARD_PAGE_REF_COUNT;
903	pPage->HCPhys = PGMR0DYNMAP_GUARD_PAGE_HCPHYS;
904	#ifdef PGMR0DYNMAP_GUARD_NP
905	ASMAtomicBitClear(pPage->uPte.pv, X86_PTE_BIT_P);
906	#else
907	if (pThis->fLegacyMode)
908	ASMAtomicWriteU32(&pPage->uPte.pLegacy->u, PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE);
909	else
910	ASMAtomicWriteU64(&pPage->uPte.pPae->u, PGMR0DYNMAP_GUARD_PAGE_PAE_PTE);
911	#endif
912	pThis->cGuardPages++;
913	}
914
915
916	/**
917	* Adds a new segment of the specified size.
918	*
919	* @returns VBox status code.
920	* @param pThis The dynamic mapping cache instance.
921	* @param cPages The size of the new segment, give as a page count.
922	*/
923	static int pgmR0DynMapAddSeg(PPGMRZDYNMAP pThis, uint32_t cPages)
924	{
925	int rc2;
926	AssertReturn(ASMGetFlags() & X86_EFL_IF, VERR_PREEMPT_DISABLED);
927
928	/*
929	* Do the array reallocations first.
930	* (The pages array has to be replaced behind the spinlock of course.)
931	*/
932	void pvSavedPTEs = RTMemRealloc(pThis->pvSavedPTEs, (pThis->fLegacyMode ? sizeof(X86PGUINT) : sizeof(X86PGPAEUINT)) (pThis->cPages + cPages));
933	if (!pvSavedPTEs)
934	return VERR_NO_MEMORY;
935	pThis->pvSavedPTEs = pvSavedPTEs;
936
937	void pvPages = RTMemAllocZ(sizeof(pThis->paPages[0]) (pThis->cPages + cPages));
938	if (!pvPages)
939	{
940	pvSavedPTEs = RTMemRealloc(pThis->pvSavedPTEs, (pThis->fLegacyMode ? sizeof(X86PGUINT) : sizeof(X86PGPAEUINT)) * pThis->cPages);
941	if (pvSavedPTEs)
942	pThis->pvSavedPTEs = pvSavedPTEs;
943	return VERR_NO_MEMORY;
944	}
945
946	PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
947
948	memcpy(pvPages, pThis->paPages, sizeof(pThis->paPages[0]) * pThis->cPages);
949	void *pvToFree = pThis->paPages;
950	pThis->paPages = (PPGMRZDYNMAPENTRY)pvPages;
951
952	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
953	RTMemFree(pvToFree);
954
955	/*
956	* Allocate the segment structure and pages of memory, then touch all the pages (paranoia).
957	*/
958	uint32_t cMaxPTs = cPages / (pThis->fLegacyMode ? X86_PG_ENTRIES : X86_PG_PAE_ENTRIES) + 2;
959	PPGMR0DYNMAPSEG pSeg = (PPGMR0DYNMAPSEG)RTMemAllocZ(RT_UOFFSETOF(PGMR0DYNMAPSEG, ahMemObjPTs[cMaxPTs]));
960	if (!pSeg)
961	return VERR_NO_MEMORY;
962	pSeg->pNext = NULL;
963	pSeg->cPages = cPages;
964	pSeg->iPage = pThis->cPages;
965	pSeg->cPTs = 0;
966	int rc = RTR0MemObjAllocPage(&pSeg->hMemObj, cPages << PAGE_SHIFT, false);
967	if (RT_SUCCESS(rc))
968	{
969	uint8_t pbPage = (uint8_t )RTR0MemObjAddress(pSeg->hMemObj);
970	AssertMsg(VALID_PTR(pbPage) && !((uintptr_t)pbPage & PAGE_OFFSET_MASK), ("%p\n", pbPage));
971	memset(pbPage, 0xfe, cPages << PAGE_SHIFT);
972
973	/*
974	* Walk thru the pages and set them up with a mapping of their PTE and everything.
975	*/
976	ASMIntDisable();
977	PGMR0DYNMAPPGLVL PgLvl;
978	pgmR0DynMapPagingArrayInit(pThis, &PgLvl);
979	uint32_t const iEndPage = pSeg->iPage + cPages;
980	for (uint32_t iPage = pSeg->iPage;
981	iPage < iEndPage;
982	iPage++, pbPage += PAGE_SIZE)
983	{
984	/* Initialize the page data. */
985	pThis->paPages[iPage].HCPhys = NIL_RTHCPHYS;
986	pThis->paPages[iPage].pvPage = pbPage;
987	pThis->paPages[iPage].cRefs = 0;
988	pThis->paPages[iPage].uPte.pPae = 0;
989	#ifndef IN_RC
990	RTCpuSetFill(&pThis->paPages[iPage].PendingSet);
991	#endif
992
993	/* Map its page table, retry until we've got a clean run (paranoia). */
994	do
995	rc = pgmR0DynMapPagingArrayMapPte(pThis, &PgLvl, pbPage, pSeg, cMaxPTs,
996	&pThis->paPages[iPage].uPte.pv);
997	while (rc == VINF_TRY_AGAIN);
998	if (RT_FAILURE(rc))
999	break;
1000
1001	/* Save the PTE. */
1002	if (pThis->fLegacyMode)
1003	((PX86PGUINT)pThis->pvSavedPTEs)[iPage] = pThis->paPages[iPage].uPte.pLegacy->u;
1004	else
1005	((PX86PGPAEUINT)pThis->pvSavedPTEs)[iPage] = pThis->paPages[iPage].uPte.pPae->u;
1006
1007	#ifdef VBOX_STRICT
1008	/* Check that we've got the right entry. */
1009	RTHCPHYS HCPhysPage = RTR0MemObjGetPagePhysAddr(pSeg->hMemObj, iPage - pSeg->iPage);
1010	RTHCPHYS HCPhysPte = pThis->fLegacyMode
1011	? pThis->paPages[iPage].uPte.pLegacy->u & X86_PTE_PG_MASK
1012	: pThis->paPages[iPage].uPte.pPae->u & X86_PTE_PAE_PG_MASK;
1013	if (HCPhysPage != HCPhysPte)
1014	{
1015	LogRel(("pgmR0DynMapAddSeg: internal error - page #%u HCPhysPage=%RHp HCPhysPte=%RHp pbPage=%p pvPte=%p\n",
1016	iPage - pSeg->iPage, HCPhysPage, HCPhysPte, pbPage, pThis->paPages[iPage].uPte.pv));
1017	rc = VERR_INTERNAL_ERROR;
1018	break;
1019	}
1020	#endif
1021	} /* for each page */
1022	ASMIntEnable();
1023
1024	/* cleanup non-PT mappings */
1025	for (uint32_t i = 0; i < PgLvl.cLevels - 1; i++)
1026	RTR0MemObjFree(PgLvl.a[i].hMemObj, true /* fFreeMappings */);
1027
1028	if (RT_SUCCESS(rc))
1029	{
1030	#if PGMR0DYNMAP_GUARD_PAGES > 0
1031	/*
1032	* Setup guard pages.
1033	* (Note: TLBs will be shot down later on.)
1034	*/
1035	uint32_t iPage = pSeg->iPage;
1036	while (iPage < iEndPage)
1037	{
1038	for (uint32_t iGPg = 0; iGPg < PGMR0DYNMAP_GUARD_PAGES && iPage < iEndPage; iGPg++, iPage++)
1039	pgmR0DynMapSetupGuardPage(pThis, &pThis->paPages[iPage]);
1040	iPage++; /* the guarded page */
1041	}
1042
1043	/* Make sure the very last page is a guard page too. */
1044	iPage = iEndPage - 1;
1045	if (pThis->paPages[iPage].cRefs != PGMR0DYNMAP_GUARD_PAGE_REF_COUNT)
1046	pgmR0DynMapSetupGuardPage(pThis, &pThis->paPages[iPage]);
1047	#endif /* PGMR0DYNMAP_GUARD_PAGES > 0 */
1048
1049	/*
1050	* Commit it by adding the segment to the list and updating the page count.
1051	*/
1052	pSeg->pNext = pThis->pSegHead;
1053	pThis->pSegHead = pSeg;
1054	pThis->cPages += cPages;
1055	return VINF_SUCCESS;
1056	}
1057
1058	/*
1059	* Bail out.
1060	*/
1061	while (pSeg->cPTs-- > 0)
1062	{
1063	rc2 = RTR0MemObjFree(pSeg->ahMemObjPTs[pSeg->cPTs], true /* fFreeMappings */);
1064	AssertRC(rc2);
1065	pSeg->ahMemObjPTs[pSeg->cPTs] = NIL_RTR0MEMOBJ;
1066	}
1067
1068	rc2 = RTR0MemObjFree(pSeg->hMemObj, true /* fFreeMappings */);
1069	AssertRC(rc2);
1070	pSeg->hMemObj = NIL_RTR0MEMOBJ;
1071	}
1072	RTMemFree(pSeg);
1073
1074	/* Don't bother resizing the arrays, but free them if we're the only user. */
1075	if (!pThis->cPages)
1076	{
1077	RTMemFree(pThis->paPages);
1078	pThis->paPages = NULL;
1079	RTMemFree(pThis->pvSavedPTEs);
1080	pThis->pvSavedPTEs = NULL;
1081	}
1082	return rc;
1083	}
1084
1085
1086	/**
1087	* Called by PGMR0DynMapInitVM under the init lock.
1088	*
1089	* @returns VBox status code.
1090	* @param pThis The dynamic mapping cache instance.
1091	*/
1092	static int pgmR0DynMapSetup(PPGMRZDYNMAP pThis)
1093	{
1094	/*
1095	* Calc the size and add a segment of that size.
1096	*/
1097	uint32_t cMinPages;
1098	uint32_t cPages = pgmR0DynMapCalcNewSize(pThis, &cMinPages);
1099	AssertReturn(cPages, VERR_INTERNAL_ERROR);
1100	int rc = pgmR0DynMapAddSeg(pThis, cPages);
1101	if (rc == VERR_NO_MEMORY)
1102	{
1103	/*
1104	* Try adding smaller segments.
1105	*/
1106	do
1107	rc = pgmR0DynMapAddSeg(pThis, PGMR0DYNMAP_SMALL_SEG_PAGES);
1108	while (RT_SUCCESS(rc) && pThis->cPages < cPages);
1109	if (rc == VERR_NO_MEMORY && pThis->cPages >= cMinPages)
1110	rc = VINF_SUCCESS;
1111	if (rc == VERR_NO_MEMORY)
1112	{
1113	if (pThis->cPages)
1114	pgmR0DynMapTearDown(pThis);
1115	rc = VERR_PGM_DYNMAP_SETUP_ERROR;
1116	}
1117	}
1118	Assert(ASMGetFlags() & X86_EFL_IF);
1119
1120	#if PGMR0DYNMAP_GUARD_PAGES > 0
1121	/* paranoia */
1122	if (RT_SUCCESS(rc))
1123	pgmR0DynMapTlbShootDown(pThis);
1124	#endif
1125	return rc;
1126	}
1127
1128
1129	/**
1130	* Called by PGMR0DynMapInitVM under the init lock.
1131	*
1132	* @returns VBox status code.
1133	* @param pThis The dynamic mapping cache instance.
1134	*/
1135	static int pgmR0DynMapExpand(PPGMRZDYNMAP pThis)
1136	{
1137	/*
1138	* Calc the new target size and add a segment of the appropriate size.
1139	*/
1140	uint32_t cMinPages;
1141	uint32_t cPages = pgmR0DynMapCalcNewSize(pThis, &cMinPages);
1142	AssertReturn(cPages, VERR_INTERNAL_ERROR);
1143	if (pThis->cPages >= cPages)
1144	return VINF_SUCCESS;
1145
1146	uint32_t cAdd = cPages - pThis->cPages;
1147	int rc = pgmR0DynMapAddSeg(pThis, cAdd);
1148	if (rc == VERR_NO_MEMORY)
1149	{
1150	/*
1151	* Try adding smaller segments.
1152	*/
1153	do
1154	rc = pgmR0DynMapAddSeg(pThis, PGMR0DYNMAP_SMALL_SEG_PAGES);
1155	while (RT_SUCCESS(rc) && pThis->cPages < cPages);
1156	if (rc == VERR_NO_MEMORY && pThis->cPages >= cMinPages)
1157	rc = VINF_SUCCESS;
1158	if (rc == VERR_NO_MEMORY)
1159	rc = VERR_PGM_DYNMAP_EXPAND_ERROR;
1160	}
1161	Assert(ASMGetFlags() & X86_EFL_IF);
1162
1163	#if PGMR0DYNMAP_GUARD_PAGES > 0
1164	/* paranoia */
1165	if (RT_SUCCESS(rc))
1166	pgmR0DynMapTlbShootDown(pThis);
1167	#endif
1168	return rc;
1169	}
1170
1171
1172	/**
1173	* Called by PGMR0DynMapTermVM under the init lock.
1174	*
1175	* @returns VBox status code.
1176	* @param pThis The dynamic mapping cache instance.
1177	*/
1178	static void pgmR0DynMapTearDown(PPGMRZDYNMAP pThis)
1179	{
1180	/*
1181	* Restore the original page table entries
1182	*/
1183	PPGMRZDYNMAPENTRY paPages = pThis->paPages;
1184	uint32_t iPage = pThis->cPages;
1185	if (pThis->fLegacyMode)
1186	{
1187	X86PGUINT const paSavedPTEs = (X86PGUINT const )pThis->pvSavedPTEs;
1188	while (iPage-- > 0)
1189	{
1190	X86PGUINT uOld = paPages[iPage].uPte.pLegacy->u;
1191	X86PGUINT uOld2 = uOld; NOREF(uOld2);
1192	X86PGUINT uNew = paSavedPTEs[iPage];
1193	while (!ASMAtomicCmpXchgExU32(&paPages[iPage].uPte.pLegacy->u, uNew, uOld, &uOld))
1194	AssertMsgFailed(("uOld=%#x uOld2=%#x uNew=%#x\n", uOld, uOld2, uNew));
1195	Assert(paPages[iPage].uPte.pLegacy->u == paSavedPTEs[iPage]);
1196	}
1197	}
1198	else
1199	{
1200	X86PGPAEUINT const paSavedPTEs = (X86PGPAEUINT const )pThis->pvSavedPTEs;
1201	while (iPage-- > 0)
1202	{
1203	X86PGPAEUINT uOld = paPages[iPage].uPte.pPae->u;
1204	X86PGPAEUINT uOld2 = uOld; NOREF(uOld2);
1205	X86PGPAEUINT uNew = paSavedPTEs[iPage];
1206	while (!ASMAtomicCmpXchgExU64(&paPages[iPage].uPte.pPae->u, uNew, uOld, &uOld))
1207	AssertMsgFailed(("uOld=%#llx uOld2=%#llx uNew=%#llx\n", uOld, uOld2, uNew));
1208	Assert(paPages[iPage].uPte.pPae->u == paSavedPTEs[iPage]);
1209	}
1210	}
1211
1212	/*
1213	* Shoot down the TLBs on all CPUs before freeing them.
1214	*/
1215	pgmR0DynMapTlbShootDown(pThis);
1216
1217	/*
1218	* Free the segments.
1219	*/
1220	while (pThis->pSegHead)
1221	{
1222	int rc;
1223	PPGMR0DYNMAPSEG pSeg = pThis->pSegHead;
1224	pThis->pSegHead = pSeg->pNext;
1225
1226	uint32_t iPT = pSeg->cPTs;
1227	while (iPT-- > 0)
1228	{
1229	rc = RTR0MemObjFree(pSeg->ahMemObjPTs[iPT], true /* fFreeMappings */); AssertRC(rc);
1230	pSeg->ahMemObjPTs[iPT] = NIL_RTR0MEMOBJ;
1231	}
1232	rc = RTR0MemObjFree(pSeg->hMemObj, true /* fFreeMappings */); AssertRC(rc);
1233	pSeg->hMemObj = NIL_RTR0MEMOBJ;
1234	pSeg->pNext = NULL;
1235	pSeg->iPage = UINT16_MAX;
1236	pSeg->cPages = 0;
1237	pSeg->cPTs = 0;
1238	RTMemFree(pSeg);
1239	}
1240
1241	/*
1242	* Free the arrays and restore the initial state.
1243	* The cLoadMax value is left behind for the next setup.
1244	*/
1245	RTMemFree(pThis->paPages);
1246	pThis->paPages = NULL;
1247	RTMemFree(pThis->pvSavedPTEs);
1248	pThis->pvSavedPTEs = NULL;
1249	pThis->cPages = 0;
1250	pThis->cLoad = 0;
1251	pThis->cGuardPages = 0;
1252	}
1253
1254	#endif /* IN_RING0 */
1255	#ifdef IN_RC
1256
1257	/**
1258	* Initializes the dynamic mapping cache in raw-mode context.
1259	*
1260	* @returns VBox status code.
1261	* @param pVM The VM handle.
1262	*/
1263	VMMRCDECL(int) PGMRCDynMapInit(PVM pVM)
1264	{
1265	/*
1266	* Allocate and initialize the instance data and page array.
1267	*/
1268	PPGMRZDYNMAP pThis;
1269	size_t const cPages = MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE;
1270	size_t const cb = RT_ALIGN_Z(sizeof(*pThis), 32)
1271	+ sizeof(PGMRZDYNMAPENTRY) * cPages;
1272	int rc = MMHyperAlloc(pVM, cb, 32, MM_TAG_PGM, (void **)&pThis);
1273	if (RT_FAILURE(rc))
1274	return rc;
1275
1276	pThis->u32Magic = PGMRZDYNMAP_MAGIC;
1277	pThis->paPages = RT_ALIGN_PT(pThis + 1, 32, PPGMRZDYNMAPENTRY);
1278	pThis->cPages = cPages;
1279	pThis->fLegacyMode = PGMGetHostMode(pVM) == PGMMODE_32_BIT;
1280	pThis->cLoad = 0;
1281	pThis->cMaxLoad = 0;
1282	pThis->cGuardPages = 0;
1283	pThis->cUsers = 1;
1284
1285	for (size_t iPage = 0; iPage < cPages; iPage++)
1286	{
1287	pThis->paPages[iPage].HCPhys = NIL_RTHCPHYS;
1288	pThis->paPages[iPage].pvPage = pVM->pgm.s.pbDynPageMapBaseGC + iPage * PAGE_SIZE;
1289	pThis->paPages[iPage].cRefs = 0;
1290	if (pThis->fLegacyMode)
1291	pThis->paPages[iPage].uPte.pLegacy = &pVM->pgm.s.paDynPageMap32BitPTEsGC[iPage];
1292	else
1293	pThis->paPages[iPage].uPte.pPae = (PX86PTEPAE)&pVM->pgm.s.paDynPageMapPaePTEsGC[iPage];
1294	}
1295
1296	pVM->pgm.s.pRCDynMap = pThis;
1297
1298	/*
1299	* Initialize the autosets the VM.
1300	*/
1301	rc = pgmRZDynMapInitAutoSetsForVM(pVM);
1302	if (RT_FAILURE(rc))
1303	return rc;
1304
1305	return VINF_SUCCESS;
1306	}
1307
1308	#endif /* IN_RC */
1309
1310	/**
1311	* Release references to a page, caller owns the spin lock.
1312	*
1313	* @param pThis The dynamic mapping cache instance.
1314	* @param iPage The page.
1315	* @param cRefs The number of references to release.
1316	*/
1317	DECLINLINE(void) pgmRZDynMapReleasePageLocked(PPGMRZDYNMAP pThis, uint32_t iPage, int32_t cRefs)
1318	{
1319	cRefs = ASMAtomicSubS32(&pThis->paPages[iPage].cRefs, cRefs) - cRefs;
1320	AssertMsg(cRefs >= 0, ("%d\n", cRefs));
1321	if (!cRefs)
1322	{
1323	pThis->cLoad--;
1324	#ifdef PGMRZDYNMAP_STRICT_RELEASE
1325	pThis->paPages[iPage].HCPhys = NIL_RTHCPHYS;
1326	ASMAtomicBitClear(pThis->paPages[iPage].uPte.pv, X86_PTE_BIT_P);
1327	ASMInvalidatePage(pThis->paPages[iPage].pvPage);
1328	#endif
1329	}
1330	}
1331
1332
1333	/**
1334	* Release references to a page, caller does not own the spin lock.
1335	*
1336	* @param pThis The dynamic mapping cache instance.
1337	* @param iPage The page.
1338	* @param cRefs The number of references to release.
1339	*/
1340	static void pgmRZDynMapReleasePage(PPGMRZDYNMAP pThis, uint32_t iPage, uint32_t cRefs)
1341	{
1342	PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
1343	pgmRZDynMapReleasePageLocked(pThis, iPage, cRefs);
1344	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
1345	}
1346
1347
1348	/**
1349	* pgmR0DynMapPage worker that deals with the tedious bits.
1350	*
1351	* @returns The page index on success, UINT32_MAX on failure.
1352	* @param pThis The dynamic mapping cache instance.
1353	* @param HCPhys The address of the page to be mapped.
1354	* @param iPage The page index pgmR0DynMapPage hashed HCPhys to.
1355	* @param pVCpu The current CPU, for statistics.
1356	* @param pfNew Set to @c true if a new entry was made and @c false if
1357	* an old entry was found and reused.
1358	*/
1359	static uint32_t pgmR0DynMapPageSlow(PPGMRZDYNMAP pThis, RTHCPHYS HCPhys, uint32_t iPage, PVMCPU pVCpu, bool *pfNew)
1360	{
1361	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageSlow);
1362
1363	/*
1364	* Check if any of the first 3 pages are unreferenced since the caller
1365	* already has made sure they aren't matching.
1366	*/
1367	#ifdef VBOX_WITH_STATISTICS
1368	bool fLooped = false;
1369	#endif
1370	uint32_t const cPages = pThis->cPages;
1371	PPGMRZDYNMAPENTRY paPages = pThis->paPages;
1372	uint32_t iFreePage;
1373	if (!paPages[iPage].cRefs)
1374	iFreePage = iPage;
1375	else if (!paPages[(iPage + 1) % cPages].cRefs)
1376	iFreePage = (iPage + 1) % cPages;
1377	else if (!paPages[(iPage + 2) % cPages].cRefs)
1378	iFreePage = (iPage + 2) % cPages;
1379	else
1380	{
1381	/*
1382	* Search for an unused or matching entry.
1383	*/
1384	iFreePage = (iPage + 3) % cPages;
1385	for (;;)
1386	{
1387	if (paPages[iFreePage].HCPhys == HCPhys)
1388	{
1389	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageSlowLoopHits);
1390	*pfNew = false;
1391	return iFreePage;
1392	}
1393	if (!paPages[iFreePage].cRefs)
1394	break;
1395
1396	/* advance */
1397	iFreePage = (iFreePage + 1) % cPages;
1398	if (RT_UNLIKELY(iFreePage == iPage))
1399	return UINT32_MAX;
1400	}
1401	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageSlowLoopMisses);
1402	#ifdef VBOX_WITH_STATISTICS
1403	fLooped = true;
1404	#endif
1405	}
1406	Assert(iFreePage < cPages);
1407
1408	#if 0 //def VBOX_WITH_STATISTICS
1409	/* Check for lost hits. */
1410	if (!fLooped)
1411	for (uint32_t iPage2 = (iPage + 3) % cPages; iPage2 != iPage; iPage2 = (iPage2 + 1) % cPages)
1412	if (paPages[iPage2].HCPhys == HCPhys)
1413	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZDynMapPageSlowLostHits);
1414	#endif
1415
1416	/*
1417	* Setup the new entry.
1418	*/
1419	*pfNew = true;
1420	/Log6(("pgmR0DynMapPageSlow: old - %RHp %#x %#llx\n", paPages[iFreePage].HCPhys, paPages[iFreePage].cRefs, paPages[iFreePage].uPte.pPae->u));/
1421	paPages[iFreePage].HCPhys = HCPhys;
1422	#ifndef IN_RC
1423	RTCpuSetFill(&paPages[iFreePage].PendingSet);
1424	#endif
1425	if (pThis->fLegacyMode)
1426	{
1427	X86PGUINT uOld = paPages[iFreePage].uPte.pLegacy->u;
1428	X86PGUINT uOld2 = uOld; NOREF(uOld2);
1429	X86PGUINT uNew = (uOld & (X86_PTE_G \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1430	\| X86_PTE_P \| X86_PTE_RW \| X86_PTE_A \| X86_PTE_D
1431	\| (HCPhys & X86_PTE_PG_MASK);
1432	while (!ASMAtomicCmpXchgExU32(&paPages[iFreePage].uPte.pLegacy->u, uNew, uOld, &uOld))
1433	AssertMsgFailed(("uOld=%#x uOld2=%#x uNew=%#x\n", uOld, uOld2, uNew));
1434	Assert(paPages[iFreePage].uPte.pLegacy->u == uNew);
1435	}
1436	else
1437	{
1438	X86PGPAEUINT uOld = paPages[iFreePage].uPte.pPae->u;
1439	X86PGPAEUINT uOld2 = uOld; NOREF(uOld2);
1440	X86PGPAEUINT uNew = (uOld & (X86_PTE_G \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1441	\| X86_PTE_P \| X86_PTE_RW \| X86_PTE_A \| X86_PTE_D
1442	\| (HCPhys & X86_PTE_PAE_PG_MASK);
1443	while (!ASMAtomicCmpXchgExU64(&paPages[iFreePage].uPte.pPae->u, uNew, uOld, &uOld))
1444	AssertMsgFailed(("uOld=%#llx uOld2=%#llx uNew=%#llx\n", uOld, uOld2, uNew));
1445	Assert(paPages[iFreePage].uPte.pPae->u == uNew);
1446	/Log6(("pgmR0DynMapPageSlow: #%x - %RHp %p %#llx\n", iFreePage, HCPhys, paPages[iFreePage].pvPage, uNew));/
1447	}
1448	return iFreePage;
1449	}
1450
1451
1452	/**
1453	* Maps a page into the pool.
1454	*
1455	* @returns Page index on success, UINT32_MAX on failure.
1456	* @param pThis The dynamic mapping cache instance.
1457	* @param HCPhys The address of the page to be mapped.
1458	* @param iRealCpu The real cpu set index. (optimization)
1459	* @param pVCpu The current CPU (for statistics).
1460	* @param ppvPage Where to the page address.
1461	*/
1462	DECLINLINE(uint32_t) pgmR0DynMapPage(PPGMRZDYNMAP pThis, RTHCPHYS HCPhys, int32_t iRealCpu, PVMCPU pVCpu, void **ppvPage)
1463	{
1464	PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
1465	AssertMsg(!(HCPhys & PAGE_OFFSET_MASK), ("HCPhys=%RHp\n", HCPhys));
1466	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPage);
1467
1468	/*
1469	* Find an entry, if possible a matching one. The HCPhys address is hashed
1470	* down to a page index, collisions are handled by linear searching.
1471	* Optimized for a hit in the first 3 pages.
1472	*
1473	* Field easy hits here and defer the tedious searching and inserting
1474	* to pgmR0DynMapPageSlow().
1475	*/
1476	bool fNew = false;
1477	uint32_t const cPages = pThis->cPages;
1478	uint32_t iPage = (HCPhys >> PAGE_SHIFT) % cPages;
1479	PPGMRZDYNMAPENTRY paPages = pThis->paPages;
1480	if (RT_LIKELY(paPages[iPage].HCPhys == HCPhys))
1481	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageHits0);
1482	else
1483	{
1484	uint32_t iPage2 = (iPage + 1) % cPages;
1485	if (RT_LIKELY(paPages[iPage2].HCPhys == HCPhys))
1486	{
1487	iPage = iPage2;
1488	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageHits1);
1489	}
1490	else
1491	{
1492	iPage2 = (iPage + 2) % cPages;
1493	if (paPages[iPage2].HCPhys == HCPhys)
1494	{
1495	iPage = iPage2;
1496	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageHits2);
1497	}
1498	else
1499	{
1500	iPage = pgmR0DynMapPageSlow(pThis, HCPhys, iPage, pVCpu, &fNew);
1501	if (RT_UNLIKELY(iPage == UINT32_MAX))
1502	{
1503	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
1504	*ppvPage = NULL;
1505	return iPage;
1506	}
1507	}
1508	}
1509	}
1510
1511	/*
1512	* Reference it, update statistics and get the return address.
1513	*/
1514	int32_t cRefs = ASMAtomicIncS32(&paPages[iPage].cRefs);
1515	if (cRefs == 1)
1516	{
1517	pThis->cLoad++;
1518	if (pThis->cLoad > pThis->cMaxLoad)
1519	pThis->cMaxLoad = pThis->cLoad;
1520	AssertMsg(pThis->cLoad <= pThis->cPages - pThis->cGuardPages, ("%d/%d\n", pThis->cLoad, pThis->cPages - pThis->cGuardPages));
1521	}
1522	else if (RT_UNLIKELY(cRefs <= 0))
1523	{
1524	ASMAtomicDecS32(&paPages[iPage].cRefs);
1525	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
1526	*ppvPage = NULL;
1527	AssertLogRelMsgFailedReturn(("cRefs=%d iPage=%p HCPhys=%RHp\n", cRefs, iPage, HCPhys), UINT32_MAX);
1528	}
1529	void *pvPage = paPages[iPage].pvPage;
1530
1531	#ifndef IN_RC
1532	/*
1533	* Invalidate the entry?
1534	*/
1535	bool fInvalidateIt = RTCpuSetIsMemberByIndex(&paPages[iPage].PendingSet, iRealCpu);
1536	if (RT_UNLIKELY(fInvalidateIt))
1537	RTCpuSetDelByIndex(&paPages[iPage].PendingSet, iRealCpu);
1538	#endif
1539
1540	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
1541
1542	/*
1543	* Do the actual invalidation outside the spinlock.
1544	*/
1545	#ifdef IN_RC
1546	if (RT_UNLIKELY(fNew))
1547	#else
1548	if (RT_UNLIKELY(fInvalidateIt))
1549	#endif
1550	{
1551	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageInvlPg);
1552	ASMInvalidatePage(pvPage);
1553	}
1554
1555	*ppvPage = pvPage;
1556	return iPage;
1557	}
1558
1559
1560	/**
1561	* Assert the the integrity of the pool.
1562	*
1563	* @returns VBox status code.
1564	*/
1565	static int pgmRZDynMapAssertIntegrity(PPGMRZDYNMAP pThis)
1566	{
1567	/*
1568	* Basic pool stuff that doesn't require any lock, just assumes we're a user.
1569	*/
1570	if (!pThis)
1571	return VINF_SUCCESS;
1572	AssertPtrReturn(pThis, VERR_INVALID_POINTER);
1573	AssertReturn(pThis->u32Magic == PGMRZDYNMAP_MAGIC, VERR_INVALID_MAGIC);
1574	if (!pThis->cUsers)
1575	return VERR_INVALID_PARAMETER;
1576
1577
1578	int rc = VINF_SUCCESS;
1579	PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
1580
1581	#define CHECK_RET(expr, a) \
1582	do { \
1583	if (RT_UNLIKELY(!(expr))) \
1584	{ \
1585	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis); \
1586	RTAssertMsg1Weak(#expr, __LINE__, __FILE__, __PRETTY_FUNCTION__); \
1587	RTAssertMsg2Weak a; \
1588	return VERR_INTERNAL_ERROR; \
1589	} \
1590	} while (0)
1591
1592	/*
1593	* Check that the PTEs are correct.
1594	*/
1595	uint32_t cGuard = 0;
1596	uint32_t cLoad = 0;
1597	PPGMRZDYNMAPENTRY paPages = pThis->paPages;
1598	uint32_t iPage = pThis->cPages;
1599	if (pThis->fLegacyMode)
1600	{
1601	#ifdef IN_RING0
1602	PCX86PGUINT paSavedPTEs = (PCX86PGUINT)pThis->pvSavedPTEs; NOREF(paSavedPTEs);
1603	#endif
1604	while (iPage-- > 0)
1605	{
1606	CHECK_RET(!((uintptr_t)paPages[iPage].pvPage & PAGE_OFFSET_MASK), ("#%u: %p\n", iPage, paPages[iPage].pvPage));
1607	if ( paPages[iPage].cRefs == PGMR0DYNMAP_GUARD_PAGE_REF_COUNT
1608	&& paPages[iPage].HCPhys == PGMR0DYNMAP_GUARD_PAGE_HCPHYS)
1609	{
1610	#ifdef PGMR0DYNMAP_GUARD_NP
1611	CHECK_RET(paPages[iPage].uPte.pLegacy->u == (paSavedPTEs[iPage] & ~(X86PGUINT)X86_PTE_P),
1612	("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, paSavedPTEs[iPage]));
1613	#else
1614	CHECK_RET(paPages[iPage].uPte.pLegacy->u == PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE,
1615	("#%u: %#x", iPage, paPages[iPage].uPte.pLegacy->u));
1616	#endif
1617	cGuard++;
1618	}
1619	else if (paPages[iPage].HCPhys != NIL_RTHCPHYS)
1620	{
1621	CHECK_RET(!(paPages[iPage].HCPhys & PAGE_OFFSET_MASK), ("#%u: %RHp\n", iPage, paPages[iPage].HCPhys));
1622	X86PGUINT uPte = X86_PTE_P \| X86_PTE_RW \| X86_PTE_A \| X86_PTE_D
1623	#ifdef IN_RING0
1624	\| (paSavedPTEs[iPage] & (X86_PTE_G \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1625	#endif
1626	\| (paPages[iPage].HCPhys & X86_PTE_PAE_PG_MASK);
1627	CHECK_RET(paPages[iPage].uPte.pLegacy->u == uPte,
1628	("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, uPte));
1629	if (paPages[iPage].cRefs)
1630	cLoad++;
1631	}
1632	#if defined(IN_RING0) && !defined(PGMRZDYNMAP_STRICT_RELEASE)
1633	else
1634	CHECK_RET(paPages[iPage].uPte.pLegacy->u == paSavedPTEs[iPage],
1635	("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, paSavedPTEs[iPage]));
1636	#endif
1637	}
1638	}
1639	else
1640	{
1641	#ifdef IN_RING0
1642	PCX86PGPAEUINT paSavedPTEs = (PCX86PGPAEUINT)pThis->pvSavedPTEs; NOREF(paSavedPTEs);
1643	#endif
1644	while (iPage-- > 0)
1645	{
1646	CHECK_RET(!((uintptr_t)paPages[iPage].pvPage & PAGE_OFFSET_MASK), ("#%u: %p\n", iPage, paPages[iPage].pvPage));
1647	if ( paPages[iPage].cRefs == PGMR0DYNMAP_GUARD_PAGE_REF_COUNT
1648	&& paPages[iPage].HCPhys == PGMR0DYNMAP_GUARD_PAGE_HCPHYS)
1649	{
1650	#ifdef PGMR0DYNMAP_GUARD_NP
1651	CHECK_RET(paPages[iPage].uPte.pPae->u == (paSavedPTEs[iPage] & ~(X86PGPAEUINT)X86_PTE_P),
1652	("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pPae->u, paSavedPTEs[iPage]));
1653	#else
1654	CHECK_RET(paPages[iPage].uPte.pPae->u == PGMR0DYNMAP_GUARD_PAGE_PAE_PTE,
1655	("#%u: %#llx", iPage, paPages[iPage].uPte.pPae->u));
1656	#endif
1657	cGuard++;
1658	}
1659	else if (paPages[iPage].HCPhys != NIL_RTHCPHYS)
1660	{
1661	CHECK_RET(!(paPages[iPage].HCPhys & PAGE_OFFSET_MASK), ("#%u: %RHp\n", iPage, paPages[iPage].HCPhys));
1662	X86PGPAEUINT uPte = X86_PTE_P \| X86_PTE_RW \| X86_PTE_A \| X86_PTE_D
1663	#ifdef IN_RING0
1664	\| (paSavedPTEs[iPage] & (X86_PTE_G \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1665	#endif
1666	\| (paPages[iPage].HCPhys & X86_PTE_PAE_PG_MASK);
1667	CHECK_RET(paPages[iPage].uPte.pPae->u == uPte,
1668	("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pLegacy->u, uPte));
1669	if (paPages[iPage].cRefs)
1670	cLoad++;
1671	}
1672	#ifdef IN_RING0
1673	else
1674	CHECK_RET(paPages[iPage].uPte.pPae->u == paSavedPTEs[iPage],
1675	("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pPae->u, paSavedPTEs[iPage]));
1676	#endif
1677	}
1678	}
1679
1680	CHECK_RET(cLoad == pThis->cLoad, ("%u %u\n", cLoad, pThis->cLoad));
1681	CHECK_RET(cGuard == pThis->cGuardPages, ("%u %u\n", cGuard, pThis->cGuardPages));
1682
1683	#undef CHECK_RET
1684	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
1685	return VINF_SUCCESS;
1686	}
1687
1688	#ifdef IN_RING0
1689	/**
1690	* Assert the the integrity of the pool.
1691	*
1692	* @returns VBox status code.
1693	*/
1694	VMMR0DECL(int) PGMR0DynMapAssertIntegrity(void)
1695	{
1696	return pgmRZDynMapAssertIntegrity(g_pPGMR0DynMap);
1697	}
1698	#endif /* IN_RING0 */
1699
1700	#ifdef IN_RC
1701	/**
1702	* Assert the the integrity of the pool.
1703	*
1704	* @returns VBox status code.
1705	*/
1706	VMMRCDECL(int) PGMRCDynMapAssertIntegrity(PVM pVM)
1707	{
1708	return pgmRZDynMapAssertIntegrity((PPGMRZDYNMAP)pVM->pgm.s.pRCDynMap);
1709	}
1710	#endif /* IN_RC */
1711
1712
1713	/**
1714	* As a final resort for a (somewhat) full auto set or full cache, try merge
1715	* duplicate entries and flush the ones we can.
1716	*
1717	* @param pSet The set.
1718	*/
1719	static void pgmDynMapOptimizeAutoSet(PPGMMAPSET pSet)
1720	{
1721	LogFlow(("pgmDynMapOptimizeAutoSet\n"));
1722
1723	for (uint32_t i = 0 ; i < pSet->cEntries; i++)
1724	{
1725	/*
1726	* Try merge entries.
1727	*/
1728	uint16_t const iPage = pSet->aEntries[i].iPage;
1729	uint32_t j = i + 1;
1730	while ( j < pSet->cEntries
1731	&& ( pSet->iSubset == UINT32_MAX
1732	\|\| pSet->iSubset < pSet->cEntries) )
1733	{
1734	if (pSet->aEntries[j].iPage != iPage)
1735	j++;
1736	else
1737	{
1738	uint32_t const cHardRefs = (uint32_t)pSet->aEntries[i].cRefs
1739	+ (uint32_t)pSet->aEntries[j].cRefs;
1740	uint32_t cInlinedRefs = (uint32_t)pSet->aEntries[i].cInlinedRefs
1741	+ (uint32_t)pSet->aEntries[j].cInlinedRefs;
1742	uint32_t cUnrefs = (uint32_t)pSet->aEntries[i].cUnrefs
1743	+ (uint32_t)pSet->aEntries[j].cUnrefs;
1744	uint32_t cSub = RT_MIN(cUnrefs, cInlinedRefs);
1745	cInlinedRefs -= cSub;
1746	cUnrefs -= cSub;
1747
1748	if ( cHardRefs < UINT16_MAX
1749	&& cInlinedRefs < UINT16_MAX
1750	&& cUnrefs < UINT16_MAX)
1751	{
1752	/* merge j into i removing j. */
1753	Log2(("pgmDynMapOptimizeAutoSet: Merging #%u into #%u\n", j, i));
1754	pSet->aEntries[i].cRefs = cHardRefs;
1755	pSet->aEntries[i].cInlinedRefs = cInlinedRefs;
1756	pSet->aEntries[i].cUnrefs = cUnrefs;
1757	pSet->cEntries--;
1758	if (j < pSet->cEntries)
1759	{
1760	pSet->aEntries[j] = pSet->aEntries[pSet->cEntries];
1761	PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[pSet->cEntries]);
1762	}
1763	else
1764	PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[j]);
1765	}
1766	#if 0 /* too complicated, skip it. */
1767	else
1768	{
1769	/* migrate the max number of refs from j into i and quit the inner loop. */
1770	uint32_t cMigrate = UINT16_MAX - 1 - pSet->aEntries[i].cRefs;
1771	Assert(pSet->aEntries[j].cRefs > cMigrate);
1772	pSet->aEntries[j].cRefs -= cMigrate;
1773	pSet->aEntries[i].cRefs = UINT16_MAX - 1;
1774	break;
1775	}
1776	#endif
1777	}
1778	}
1779
1780	/*
1781	* Try make use of the unused hinting (cUnrefs) to evict entries
1782	* from both the set as well as the mapping cache.
1783	*/
1784
1785	uint32_t const cTotalRefs = (uint32_t)pSet->aEntries[i].cRefs + pSet->aEntries[i].cInlinedRefs;
1786	Log2(("pgmDynMapOptimizeAutoSet: #%u/%u/%u pvPage=%p iPage=%u cRefs=%u cInlinedRefs=%u cUnrefs=%u cTotalRefs=%u\n",
1787	i,
1788	pSet->iSubset,
1789	pSet->cEntries,
1790	pSet->aEntries[i].pvPage,
1791	pSet->aEntries[i].iPage,
1792	pSet->aEntries[i].cRefs,
1793	pSet->aEntries[i].cInlinedRefs,
1794	pSet->aEntries[i].cUnrefs,
1795	cTotalRefs));
1796	Assert(cTotalRefs >= pSet->aEntries[i].cUnrefs);
1797
1798	if ( cTotalRefs == pSet->aEntries[i].cUnrefs
1799	&& ( pSet->iSubset == UINT32_MAX
1800	\|\| pSet->iSubset < pSet->cEntries)
1801	)
1802	{
1803	Log2(("pgmDynMapOptimizeAutoSet: Releasing iPage=%d/%p\n", pSet->aEntries[i].iPage, pSet->aEntries[i].pvPage));
1804	//LogFlow(("pgmDynMapOptimizeAutoSet: Releasing iPage=%d/%p\n", pSet->aEntries[i].iPage, pSet->aEntries[i].pvPage));
1805	pgmRZDynMapReleasePage(PGMRZDYNMAP_SET_2_DYNMAP(pSet),
1806	pSet->aEntries[i].iPage,
1807	pSet->aEntries[i].cRefs);
1808	pSet->cEntries--;
1809	if (i < pSet->cEntries)
1810	{
1811	pSet->aEntries[i] = pSet->aEntries[pSet->cEntries];
1812	PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[pSet->cEntries]);
1813	}
1814
1815	i--;
1816	}
1817	}
1818	}
1819
1820
1821
1822
1823	/**
1824	* Signals the start of a new set of mappings.
1825	*
1826	* Mostly for strictness. PGMDynMapHCPage won't work unless this
1827	* API is called.
1828	*
1829	* @param pVCpu The shared data for the current virtual CPU.
1830	*/
1831	VMMDECL(void) PGMRZDynMapStartAutoSet(PVMCPU pVCpu)
1832	{
1833	LogFlow(("PGMRZDynMapStartAutoSet:\n"));
1834	Assert(pVCpu->pgm.s.AutoSet.cEntries == PGMMAPSET_CLOSED);
1835	Assert(pVCpu->pgm.s.AutoSet.iSubset == UINT32_MAX);
1836	pVCpu->pgm.s.AutoSet.cEntries = 0;
1837	pVCpu->pgm.s.AutoSet.iCpu = PGMRZDYNMAP_CUR_CPU();
1838	}
1839
1840
1841	#ifdef IN_RING0
1842	/**
1843	* Starts or migrates the autoset of a virtual CPU.
1844	*
1845	* This is used by HWACCMR0Enter. When we've longjumped out of the HWACCM
1846	* execution loop with the set open, we'll migrate it when re-entering. While
1847	* under normal circumstances, we'll start it so VMXR0LoadGuestState can access
1848	* guest memory.
1849	*
1850	* @returns @c true if started, @c false if migrated.
1851	* @param pVCpu The shared data for the current virtual CPU.
1852	* @thread EMT
1853	*/
1854	VMMR0DECL(bool) PGMR0DynMapStartOrMigrateAutoSet(PVMCPU pVCpu)
1855	{
1856	bool fStartIt = pVCpu->pgm.s.AutoSet.cEntries == PGMMAPSET_CLOSED;
1857	if (fStartIt)
1858	PGMRZDynMapStartAutoSet(pVCpu);
1859	else
1860	PGMR0DynMapMigrateAutoSet(pVCpu);
1861	return fStartIt;
1862	}
1863	#endif /* IN_RING0 */
1864
1865
1866	/**
1867	* Worker that performs the actual flushing of the set.
1868	*
1869	* @param pSet The set to flush.
1870	* @param cEntries The number of entries.
1871	*/
1872	DECLINLINE(void) pgmDynMapFlushAutoSetWorker(PPGMMAPSET pSet, uint32_t cEntries)
1873	{
1874	/*
1875	* Release any pages it's referencing.
1876	*/
1877	if ( cEntries != 0
1878	&& RT_LIKELY(cEntries <= RT_ELEMENTS(pSet->aEntries)))
1879	{
1880	PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
1881	PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
1882
1883	uint32_t i = cEntries;
1884	while (i-- > 0)
1885	{
1886	uint32_t iPage = pSet->aEntries[i].iPage;
1887	Assert(iPage < pThis->cPages);
1888	int32_t cRefs = pSet->aEntries[i].cRefs;
1889	Assert(cRefs > 0);
1890	pgmRZDynMapReleasePageLocked(pThis, iPage, cRefs);
1891
1892	PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[i]);
1893	}
1894
1895	Assert(pThis->cLoad <= pThis->cPages - pThis->cGuardPages);
1896	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
1897	}
1898	}
1899
1900
1901	/**
1902	* Releases the dynamic memory mappings made by PGMDynMapHCPage and associates
1903	* since the PGMDynMapStartAutoSet call.
1904	*
1905	* @param pVCpu The shared data for the current virtual CPU.
1906	*/
1907	VMMDECL(void) PGMRZDynMapReleaseAutoSet(PVMCPU pVCpu)
1908	{
1909	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
1910
1911	/*
1912	* Close and flush the set.
1913	*/
1914	uint32_t cEntries = pSet->cEntries;
1915	AssertReturnVoid(cEntries != PGMMAPSET_CLOSED);
1916	pSet->cEntries = PGMMAPSET_CLOSED;
1917	pSet->iSubset = UINT32_MAX;
1918	pSet->iCpu = -1;
1919
1920	#ifdef IN_RC
1921	if (RT_ELEMENTS(pSet->aEntries) > MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)
1922	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / (MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)) % 11]);
1923	else
1924	#endif
1925	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
1926	if (cEntries > RT_ELEMENTS(pSet->aEntries) * 50 / 100)
1927	Log(("PGMRZDynMapReleaseAutoSet: cEntries=%d\n", cEntries));
1928	else
1929	LogFlow(("PGMRZDynMapReleaseAutoSet: cEntries=%d\n", cEntries));
1930
1931	pgmDynMapFlushAutoSetWorker(pSet, cEntries);
1932	}
1933
1934
1935	/**
1936	* Flushes the set if it's above a certain threshold.
1937	*
1938	* @param pVCpu The shared data for the current virtual CPU.
1939	*/
1940	VMMDECL(void) PGMRZDynMapFlushAutoSet(PVMCPU pVCpu)
1941	{
1942	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
1943	AssertMsg(pSet->iCpu == PGMRZDYNMAP_CUR_CPU(), ("%d %d efl=%#x\n", pSet->iCpu, PGMRZDYNMAP_CUR_CPU(), ASMGetFlags()));
1944
1945	/*
1946	* Only flush it if it's 45% full.
1947	*/
1948	uint32_t cEntries = pSet->cEntries;
1949	AssertReturnVoid(cEntries != PGMMAPSET_CLOSED);
1950	Assert(pSet->iSubset == UINT32_MAX);
1951	#ifdef IN_RC
1952	if (RT_ELEMENTS(pSet->aEntries) > MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)
1953	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / (MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)) % 11]);
1954	else
1955	#endif
1956	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
1957	if (cEntries >= RT_ELEMENTS(pSet->aEntries) * 45 / 100)
1958	{
1959	pSet->cEntries = 0;
1960	Log(("PGMDynMapFlushAutoSet: cEntries=%d\n", pSet->cEntries));
1961
1962	pgmDynMapFlushAutoSetWorker(pSet, cEntries);
1963	AssertMsg(pSet->iCpu == PGMRZDYNMAP_CUR_CPU(), ("%d %d efl=%#x\n", pSet->iCpu, PGMRZDYNMAP_CUR_CPU(), ASMGetFlags()));
1964	}
1965	}
1966
1967
1968	#ifndef IN_RC
1969	/**
1970	* Migrates the automatic mapping set of the current vCPU if it's active and
1971	* necessary.
1972	*
1973	* This is called when re-entering the hardware assisted execution mode after a
1974	* nip down to ring-3. We run the risk that the CPU might have change and we
1975	* will therefore make sure all the cache entries currently in the auto set will
1976	* be valid on the new CPU. If the cpu didn't change nothing will happen as all
1977	* the entries will have been flagged as invalidated.
1978	*
1979	* @param pVCpu The shared data for the current virtual CPU.
1980	* @thread EMT
1981	*/
1982	VMMR0DECL(void) PGMR0DynMapMigrateAutoSet(PVMCPU pVCpu)
1983	{
1984	LogFlow(("PGMR0DynMapMigrateAutoSet\n"));
1985	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
1986	int32_t iRealCpu = PGMRZDYNMAP_CUR_CPU();
1987	if (pSet->iCpu != iRealCpu)
1988	{
1989	uint32_t i = pSet->cEntries;
1990	if (i != PGMMAPSET_CLOSED)
1991	{
1992	AssertMsg(i <= RT_ELEMENTS(pSet->aEntries), ("%#x (%u)\n", i, i));
1993	if (i != 0 && RT_LIKELY(i <= RT_ELEMENTS(pSet->aEntries)))
1994	{
1995	PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
1996	PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
1997
1998	while (i-- > 0)
1999	{
2000	Assert(pSet->aEntries[i].cRefs > 0);
2001	uint32_t iPage = pSet->aEntries[i].iPage;
2002	Assert(iPage < pThis->cPages);
2003	if (RTCpuSetIsMemberByIndex(&pThis->paPages[iPage].PendingSet, iRealCpu))
2004	{
2005	RTCpuSetDelByIndex(&pThis->paPages[iPage].PendingSet, iRealCpu);
2006	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
2007
2008	ASMInvalidatePage(pThis->paPages[iPage].pvPage);
2009	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapMigrateInvlPg);
2010
2011	PGMRZDYNMAP_SPINLOCK_REACQUIRE(pThis);
2012	}
2013	}
2014
2015	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
2016	}
2017	}
2018	pSet->iCpu = iRealCpu;
2019	}
2020	}
2021	#endif /* !IN_RC */
2022
2023
2024	/**
2025	* Worker function that flushes the current subset.
2026	*
2027	* This is called when the set is popped or when the set
2028	* hash a too high load. As also pointed out elsewhere, the
2029	* whole subset thing is a hack for working around code that
2030	* accesses too many pages. Like PGMPool.
2031	*
2032	* @param pSet The set which subset to flush.
2033	*/
2034	static void pgmDynMapFlushSubset(PPGMMAPSET pSet)
2035	{
2036	uint32_t iSubset = pSet->iSubset;
2037	uint32_t i = pSet->cEntries;
2038	Assert(i <= RT_ELEMENTS(pSet->aEntries));
2039	if ( i > iSubset
2040	&& i <= RT_ELEMENTS(pSet->aEntries))
2041	{
2042	Log(("pgmDynMapFlushSubset: cEntries=%d iSubset=%d\n", pSet->cEntries, iSubset));
2043	pSet->cEntries = iSubset;
2044
2045	PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
2046	PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
2047
2048	while (i-- > iSubset)
2049	{
2050	uint32_t iPage = pSet->aEntries[i].iPage;
2051	Assert(iPage < pThis->cPages);
2052	int32_t cRefs = pSet->aEntries[i].cRefs;
2053	Assert(cRefs > 0);
2054	pgmRZDynMapReleasePageLocked(pThis, iPage, cRefs);
2055
2056	PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[i]);
2057	}
2058
2059	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
2060	}
2061	}
2062
2063
2064	/**
2065	* Creates a subset.
2066	*
2067	* A subset is a hack to avoid having to rewrite code that touches a lot of
2068	* pages. It prevents the mapping set from being overflowed by automatically
2069	* flushing previous mappings when a certain threshold is reached.
2070	*
2071	* Pages mapped after calling this function are only valid until the next page
2072	* is mapped.
2073	*
2074	* @returns The index of the previous subset. Pass this to
2075	* PGMDynMapPopAutoSubset when popping it.
2076	* @param pVCpu Pointer to the virtual cpu data.
2077	*/
2078	VMMDECL(uint32_t) PGMRZDynMapPushAutoSubset(PVMCPU pVCpu)
2079	{
2080	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
2081	AssertReturn(pSet->cEntries != PGMMAPSET_CLOSED, UINT32_MAX);
2082	uint32_t iPrevSubset = pSet->iSubset;
2083	LogFlow(("PGMRZDynMapPushAutoSubset: pVCpu=%p iPrevSubset=%u\n", pVCpu, iPrevSubset));
2084
2085	/*
2086	* If it looks like we're approaching the max set size or mapping space
2087	* optimize the set to drop off unused pages.
2088	*/
2089	if ( pSet->cEntries > RT_ELEMENTS(pSet->aEntries) * 60 / 100
2090	#ifdef IN_RC
2091	\|\| pSet->cEntries > MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE / 2 /** @todo need to do this for r0 as well.*/
2092	#endif
2093	)
2094	{
2095	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetOptimize);
2096	pgmDynMapOptimizeAutoSet(pSet);
2097	}
2098
2099	pSet->iSubset = pSet->cEntries;
2100	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSubsets);
2101
2102	AssertMsg(iPrevSubset <= pSet->iSubset \|\| iPrevSubset == UINT32_MAX, ("iPrevSubset=%#x iSubset=%#x\n", iPrevSubset, pSet->iSubset));
2103	return iPrevSubset;
2104	}
2105
2106
2107	/**
2108	* Pops a subset created by a previous call to PGMDynMapPushAutoSubset.
2109	*
2110	* @param pVCpu Pointer to the virtual cpu data.
2111	* @param iPrevSubset What PGMDynMapPushAutoSubset returned.
2112	*/
2113	VMMDECL(void) PGMRZDynMapPopAutoSubset(PVMCPU pVCpu, uint32_t iPrevSubset)
2114	{
2115	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
2116	uint32_t cEntries = pSet->cEntries;
2117	LogFlow(("PGMRZDynMapPopAutoSubset: pVCpu=%p iPrevSubset=%u iSubset=%u cEntries=%u\n", pVCpu, iPrevSubset, pSet->iSubset, cEntries));
2118	AssertReturnVoid(cEntries != PGMMAPSET_CLOSED);
2119	AssertMsgReturnVoid(pSet->iSubset >= iPrevSubset \|\| iPrevSubset == UINT32_MAX, ("iPrevSubset=%u iSubset=%u cEntries=%u\n", iPrevSubset, pSet->iSubset, cEntries));
2120	#ifdef IN_RC
2121	if (RT_ELEMENTS(pSet->aEntries) > MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)
2122	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / (MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)) % 11]);
2123	else
2124	#endif
2125	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
2126	if ( cEntries >= RT_ELEMENTS(pSet->aEntries) * 40 / 100
2127	&& cEntries != pSet->iSubset)
2128	{
2129	pgmDynMapFlushSubset(pSet);
2130	Assert(pSet->cEntries >= iPrevSubset \|\| iPrevSubset == UINT32_MAX);
2131	}
2132	pSet->iSubset = iPrevSubset;
2133	}
2134
2135
2136	/**
2137	* Indicates that the given page is unused and its mapping can be re-used.
2138	*
2139	* @param pVCpu The current CPU.
2140	* @param pvHint The page that is now unused. This does not have to
2141	* point at the start of the page. NULL is ignored.
2142	*/
2143	#ifdef LOG_ENABLED
2144	void pgmRZDynMapUnusedHint(PVMCPU pVCpu, void *pvHint, RT_SRC_POS_DECL)
2145	#else
2146	void pgmRZDynMapUnusedHint(PVMCPU pVCpu, void *pvHint)
2147	#endif
2148	{
2149	/*
2150	* Ignore NULL pointers and mask off the page offset bits.
2151	*/
2152	if (pvHint == NULL)
2153	return;
2154	pvHint = (void *)((uintptr_t)pvHint & ~(uintptr_t)PAGE_OFFSET_MASK);
2155
2156	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
2157	uint32_t iEntry = pSet->cEntries;
2158	AssertReturnVoid(iEntry > 0);
2159
2160	/*
2161	* Find the entry in the usual unrolled fashion.
2162	*/
2163	/** @todo add a hint to the set which entry was used last since it's not
2164	* always the last entry? */
2165	#define IS_MATCHING_ENTRY(pSet, iEntry, pvHint) \
2166	( (pSet)->aEntries[(iEntry)].pvPage == (pvHint) \
2167	&& (uint32_t)(pSet)->aEntries[(iEntry)].cRefs + (pSet)->aEntries[(iEntry)].cInlinedRefs \
2168	> (pSet)->aEntries[(iEntry)].cUnrefs )
2169	if ( iEntry >= 1 && IS_MATCHING_ENTRY(pSet, iEntry - 1, pvHint))
2170	iEntry = iEntry - 1;
2171	else if (iEntry >= 2 && IS_MATCHING_ENTRY(pSet, iEntry - 2, pvHint))
2172	iEntry = iEntry - 2;
2173	else if (iEntry >= 3 && IS_MATCHING_ENTRY(pSet, iEntry - 3, pvHint))
2174	iEntry = iEntry - 3;
2175	else if (iEntry >= 4 && IS_MATCHING_ENTRY(pSet, iEntry - 4, pvHint))
2176	iEntry = iEntry - 4;
2177	else if (iEntry >= 5 && IS_MATCHING_ENTRY(pSet, iEntry - 5, pvHint))
2178	iEntry = iEntry - 5;
2179	else if (iEntry >= 6 && IS_MATCHING_ENTRY(pSet, iEntry - 6, pvHint))
2180	iEntry = iEntry - 6;
2181	else if (iEntry >= 7 && IS_MATCHING_ENTRY(pSet, iEntry - 7, pvHint))
2182	iEntry = iEntry - 7;
2183	else
2184	{
2185	/*
2186	* Loop till we find it.
2187	*/
2188	bool fFound = false;
2189	if (iEntry > 7)
2190	{
2191	iEntry -= 7;
2192	while (iEntry-- > 0)
2193	if (IS_MATCHING_ENTRY(pSet, iEntry, pvHint))
2194	{
2195	fFound = true;
2196	break;
2197	}
2198	}
2199	AssertMsgReturnVoid(fFound,
2200	("pvHint=%p cEntries=%#x iSubset=%#x\n"
2201	"aEntries[0] = {%#x, %#x, %#x, %#x, %p}\n"
2202	"aEntries[1] = {%#x, %#x, %#x, %#x, %p}\n"
2203	"aEntries[2] = {%#x, %#x, %#x, %#x, %p}\n"
2204	"aEntries[3] = {%#x, %#x, %#x, %#x, %p}\n"
2205	"aEntries[4] = {%#x, %#x, %#x, %#x, %p}\n"
2206	"aEntries[5] = {%#x, %#x, %#x, %#x, %p}\n"
2207	,
2208	pvHint, pSet->cEntries, pSet->iSubset,
2209	pSet->aEntries[0].iPage, pSet->aEntries[0].cRefs, pSet->aEntries[0].cInlinedRefs, pSet->aEntries[0].cUnrefs, pSet->aEntries[0].pvPage,
2210	pSet->aEntries[1].iPage, pSet->aEntries[1].cRefs, pSet->aEntries[1].cInlinedRefs, pSet->aEntries[1].cUnrefs, pSet->aEntries[1].pvPage,
2211	pSet->aEntries[2].iPage, pSet->aEntries[2].cRefs, pSet->aEntries[2].cInlinedRefs, pSet->aEntries[2].cUnrefs, pSet->aEntries[2].pvPage,
2212	pSet->aEntries[3].iPage, pSet->aEntries[3].cRefs, pSet->aEntries[3].cInlinedRefs, pSet->aEntries[3].cUnrefs, pSet->aEntries[3].pvPage,
2213	pSet->aEntries[4].iPage, pSet->aEntries[4].cRefs, pSet->aEntries[4].cInlinedRefs, pSet->aEntries[4].cUnrefs, pSet->aEntries[4].pvPage,
2214	pSet->aEntries[5].iPage, pSet->aEntries[5].cRefs, pSet->aEntries[5].cInlinedRefs, pSet->aEntries[5].cUnrefs, pSet->aEntries[5].pvPage));
2215	}
2216	#undef IS_MATCHING_ENTRY
2217
2218	/*
2219	* Update it.
2220	*/
2221	uint32_t const cTotalRefs = (uint32_t)pSet->aEntries[iEntry].cRefs + pSet->aEntries[iEntry].cInlinedRefs;
2222	uint32_t const cUnrefs = pSet->aEntries[iEntry].cUnrefs;
2223	LogFlow(("pgmRZDynMapUnusedHint: pvHint=%p #%u cRefs=%d cInlinedRefs=%d cUnrefs=%d (+1) cTotalRefs=%d %s(%d) %s\n",
2224	pvHint, iEntry, pSet->aEntries[iEntry].cRefs, pSet->aEntries[iEntry].cInlinedRefs, cUnrefs, cTotalRefs, pszFile, iLine, pszFunction));
2225	AssertReturnVoid(cTotalRefs > cUnrefs);
2226
2227	if (RT_LIKELY(cUnrefs < UINT16_MAX - 1))
2228	pSet->aEntries[iEntry].cUnrefs++;
2229	else if (pSet->aEntries[iEntry].cInlinedRefs)
2230	{
2231	uint32_t cSub = RT_MIN(pSet->aEntries[iEntry].cInlinedRefs, pSet->aEntries[iEntry].cUnrefs);
2232	pSet->aEntries[iEntry].cInlinedRefs -= cSub;
2233	pSet->aEntries[iEntry].cUnrefs -= cSub;
2234	pSet->aEntries[iEntry].cUnrefs++;
2235	}
2236	else
2237	Log(("pgmRZDynMapUnusedHint: pvHint=%p ignored because of overflow! %s(%d) %s\n", pvHint, pszFile, iLine, pszFunction));
2238
2239	#ifdef PGMRZDYNMAP_STRICT_RELEASE
2240	/*
2241	* Optimize the set to trigger the unmapping and invalidation of the page.
2242	*/
2243	if (cUnrefs + 1 == cTotalRefs)
2244	pgmDynMapOptimizeAutoSet(pSet);
2245	#endif
2246	}
2247
2248
2249	/**
2250	* Common worker code for pgmRZDynMapHCPageInlined, pgmRZDynMapHCPageV2Inlined
2251	* and pgmR0DynMapGCPageOffInlined.
2252	*
2253	* @returns VINF_SUCCESS, bails out to ring-3 on failure.
2254	* @param pSet The set.
2255	* @param HCPhys The physical address of the page.
2256	* @param ppv Where to store the address of the mapping on success.
2257	*
2258	* @remarks This is a very hot path.
2259	*/
2260	int pgmRZDynMapHCPageCommon(PPGMMAPSET pSet, RTHCPHYS HCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL)
2261	{
2262	AssertMsg(pSet->iCpu == PGMRZDYNMAP_CUR_CPU(), ("%d %d efl=%#x\n", pSet->iCpu, PGMRZDYNMAP_CUR_CPU(), ASMGetFlags()));
2263	PVMCPU pVCpu = PGMRZDYNMAP_SET_2_VMCPU(pSet);
2264	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPage, a);
2265
2266	/*
2267	* Map it.
2268	*/
2269	void *pvPage;
2270	PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
2271	uint32_t iPage = pgmR0DynMapPage(pThis, HCPhys, pSet->iCpu, pVCpu, &pvPage);
2272	if (RT_UNLIKELY(iPage == UINT32_MAX))
2273	{
2274	/*
2275	* We're out of mapping space, optimize our set to try remedy the
2276	* situation. (Only works if there are unreference hints.)
2277	*/
2278	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetOptimize);
2279	pgmDynMapOptimizeAutoSet(pSet);
2280
2281	iPage = pgmR0DynMapPage(pThis, HCPhys, pSet->iCpu, pVCpu, &pvPage);
2282	if (RT_UNLIKELY(iPage == UINT32_MAX))
2283	{
2284	RTAssertMsg2Weak("pgmRZDynMapHCPageCommon: cLoad=%u/%u cPages=%u cGuardPages=%u\n",
2285	pThis->cLoad, pThis->cMaxLoad, pThis->cPages, pThis->cGuardPages);
2286	if (!g_fPGMR0DynMapTestRunning)
2287	VMMRZCallRing3NoCpu(PGMRZDYNMAP_SET_2_VM(pSet), VMMCALLRING3_VM_R0_ASSERTION, 0);
2288	*ppv = NULL;
2289	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPage, a);
2290	return VERR_PGM_DYNMAP_FAILED;
2291	}
2292	}
2293
2294	/*
2295	* Add the page to the auto reference set.
2296	*
2297	* The typical usage pattern means that the same pages will be mapped
2298	* several times in the same set. We can catch most of these
2299	* remappings by looking a few pages back into the set. (The searching
2300	* and set optimizing path will hardly ever be used when doing this.)
2301	*/
2302	AssertCompile(RT_ELEMENTS(pSet->aEntries) >= 8);
2303	int32_t i = pSet->cEntries;
2304	if (i-- < 5)
2305	{
2306	unsigned iEntry = pSet->cEntries++;
2307	pSet->aEntries[iEntry].cRefs = 1;
2308	pSet->aEntries[iEntry].cUnrefs = 0;
2309	pSet->aEntries[iEntry].cInlinedRefs = 0;
2310	pSet->aEntries[iEntry].iPage = iPage;
2311	pSet->aEntries[iEntry].pvPage = pvPage;
2312	pSet->aEntries[iEntry].HCPhys = HCPhys;
2313	pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
2314	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/0/0 iPage=%#x [a] %s(%d) %s\n",
2315	pSet, HCPhys, iEntry, iEntry + 1, pvPage, 1, iPage, pszFile, iLine, pszFunction));
2316	}
2317	/* Any of the last 5 pages? */
2318	else if ( pSet->aEntries[i - 0].iPage == iPage
2319	&& pSet->aEntries[i - 0].cRefs < UINT16_MAX - 1)
2320	{
2321	pSet->aEntries[i - 0].cRefs++;
2322	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [0] %s(%d) %s\n", pSet, HCPhys, i - 0, pSet->cEntries, pvPage, pSet->aEntries[i - 0].cRefs, pSet->aEntries[i - 0].cInlinedRefs, pSet->aEntries[i - 0].cUnrefs, iPage, pszFile, iLine, pszFunction));
2323	}
2324	else if ( pSet->aEntries[i - 1].iPage == iPage
2325	&& pSet->aEntries[i - 1].cRefs < UINT16_MAX - 1)
2326	{
2327	pSet->aEntries[i - 1].cRefs++;
2328	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [1] %s(%d) %s\n", pSet, HCPhys, i - 1, pSet->cEntries, pvPage, pSet->aEntries[i - 1].cRefs, pSet->aEntries[i - 1].cInlinedRefs, pSet->aEntries[i - 1].cUnrefs, iPage, pszFile, iLine, pszFunction));
2329	}
2330	else if ( pSet->aEntries[i - 2].iPage == iPage
2331	&& pSet->aEntries[i - 2].cRefs < UINT16_MAX - 1)
2332	{
2333	pSet->aEntries[i - 2].cRefs++;
2334	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [2] %s(%d) %s\n", pSet, HCPhys, i - 2, pSet->cEntries, pvPage, pSet->aEntries[i - 2].cRefs, pSet->aEntries[i - 2].cInlinedRefs, pSet->aEntries[i - 2].cUnrefs, iPage, pszFile, iLine, pszFunction));
2335	}
2336	else if ( pSet->aEntries[i - 3].iPage == iPage
2337	&& pSet->aEntries[i - 3].cRefs < UINT16_MAX - 1)
2338	{
2339	pSet->aEntries[i - 3].cRefs++;
2340	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [4] %s(%d) %s\n", pSet, HCPhys, i - 3, pSet->cEntries, pvPage, pSet->aEntries[i - 3].cRefs, pSet->aEntries[i - 3].cInlinedRefs, pSet->aEntries[i - 3].cUnrefs, iPage, pszFile, iLine, pszFunction));
2341	}
2342	else if ( pSet->aEntries[i - 4].iPage == iPage
2343	&& pSet->aEntries[i - 4].cRefs < UINT16_MAX - 1)
2344	{
2345	pSet->aEntries[i - 4].cRefs++;
2346	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [4] %s(%d) %s\n", pSet, HCPhys, i - 4, pSet->cEntries, pvPage, pSet->aEntries[i - 4].cRefs, pSet->aEntries[i - 4].cInlinedRefs, pSet->aEntries[i - 4].cUnrefs, iPage, pszFile, iLine, pszFunction));
2347	}
2348	/* Don't bother searching unless we're above a 60% load. */
2349	else if (RT_LIKELY(i <= (int32_t)RT_ELEMENTS(pSet->aEntries) * 60 / 100))
2350	{
2351	unsigned iEntry = pSet->cEntries++;
2352	pSet->aEntries[iEntry].cRefs = 1;
2353	pSet->aEntries[iEntry].cUnrefs = 0;
2354	pSet->aEntries[iEntry].cInlinedRefs = 0;
2355	pSet->aEntries[iEntry].iPage = iPage;
2356	pSet->aEntries[iEntry].pvPage = pvPage;
2357	pSet->aEntries[iEntry].HCPhys = HCPhys;
2358	pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
2359	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=1/0/0 iPage=%#x [b] %s(%d) %s\n", pSet, HCPhys, iEntry, pSet->cEntries, pvPage, iPage, pszFile, iLine, pszFunction));
2360	}
2361	else
2362	{
2363	/* Search the rest of the set. */
2364	Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
2365	i -= 4;
2366	while (i-- > 0)
2367	if ( pSet->aEntries[i].iPage == iPage
2368	&& pSet->aEntries[i].cRefs < UINT16_MAX - 1)
2369	{
2370	pSet->aEntries[i].cRefs++;
2371	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetSearchHits);
2372	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [c] %s(%d) %s\n", pSet, HCPhys, i, pSet->cEntries, pvPage, pSet->aEntries[i].cRefs, pSet->aEntries[i].cInlinedRefs, pSet->aEntries[i].cUnrefs, iPage, pszFile, iLine, pszFunction));
2373	break;
2374	}
2375	if (i < 0)
2376	{
2377	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetSearchMisses);
2378	#if 0 /* this is very bogus */
2379	if (pSet->iSubset < pSet->cEntries)
2380	{
2381	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetSearchFlushes);
2382	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(pSet->cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
2383	pgmDynMapFlushSubset(pSet);
2384	}
2385	#endif
2386
2387	if (RT_UNLIKELY(pSet->cEntries >= RT_ELEMENTS(pSet->aEntries)))
2388	{
2389	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetOptimize);
2390	pgmDynMapOptimizeAutoSet(pSet);
2391	}
2392
2393	if (RT_LIKELY(pSet->cEntries < RT_ELEMENTS(pSet->aEntries)))
2394	{
2395	unsigned iEntry = pSet->cEntries++;
2396	pSet->aEntries[iEntry].cRefs = 1;
2397	pSet->aEntries[iEntry].cUnrefs = 0;
2398	pSet->aEntries[iEntry].cInlinedRefs = 0;
2399	pSet->aEntries[iEntry].iPage = iPage;
2400	pSet->aEntries[iEntry].pvPage = pvPage;
2401	pSet->aEntries[iEntry].HCPhys = HCPhys;
2402	pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
2403	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=1/0/0 iPage=%#x [d] %s(%d) %s\n", pSet, HCPhys, iEntry, pSet->cEntries, pvPage, iPage, pszFile, iLine, pszFunction));
2404	}
2405	else
2406	{
2407	/* We're screwed. */
2408	pgmRZDynMapReleasePage(pThis, iPage, 1);
2409
2410	RTAssertMsg2Weak("pgmRZDynMapHCPageCommon: set is full!\n");
2411	if (!g_fPGMR0DynMapTestRunning)
2412	VMMRZCallRing3NoCpu(PGMRZDYNMAP_SET_2_VM(pSet), VMMCALLRING3_VM_R0_ASSERTION, 0);
2413	*ppv = NULL;
2414	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPage, a);
2415	return VERR_PGM_DYNMAP_FULL_SET;
2416	}
2417	}
2418	}
2419
2420	*ppv = pvPage;
2421	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPage, a);
2422	return VINF_SUCCESS;
2423	}
2424
2425
2426	#if 0 /def DEBUG/
2427	/** For pgmR0DynMapTest3PerCpu. */
2428	typedef struct PGMR0DYNMAPTEST
2429	{
2430	uint32_t u32Expect;
2431	uint32_t *pu32;
2432	uint32_t volatile cFailures;
2433	} PGMR0DYNMAPTEST;
2434	typedef PGMR0DYNMAPTEST *PPGMR0DYNMAPTEST;
2435
2436	/**
2437	* Checks that the content of the page is the same on all CPUs, i.e. that there
2438	* are no CPU specfic PTs or similar nasty stuff involved.
2439	*
2440	* @param idCpu The current CPU.
2441	* @param pvUser1 Pointer a PGMR0DYNMAPTEST structure.
2442	* @param pvUser2 Unused, ignored.
2443	*/
2444	static DECLCALLBACK(void) pgmR0DynMapTest3PerCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
2445	{
2446	PPGMR0DYNMAPTEST pTest = (PPGMR0DYNMAPTEST)pvUser1;
2447	ASMInvalidatePage(pTest->pu32);
2448	if (*pTest->pu32 != pTest->u32Expect)
2449	ASMAtomicIncU32(&pTest->cFailures);
2450	NOREF(pvUser2); NOREF(idCpu);
2451	}
2452
2453
2454	/**
2455	* Performs some basic tests in debug builds.
2456	*/
2457	static int pgmR0DynMapTest(PVM pVM)
2458	{
2459	LogRel(("pgmR0DynMapTest: **** START ****\n"));
2460	PPGMMAPSET pSet = &pVM->aCpus[0].pgm.s.AutoSet;
2461	PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
2462	uint32_t i;
2463
2464	/*
2465	* Assert internal integrity first.
2466	*/
2467	LogRel(("Test #0\n"));
2468	int rc = PGMR0DynMapAssertIntegrity();
2469	if (RT_FAILURE(rc))
2470	return rc;
2471
2472	void *pvR0DynMapUsedSaved = pVM->pgm.s.pvR0DynMapUsed;
2473	pVM->pgm.s.pvR0DynMapUsed = pThis;
2474	g_fPGMR0DynMapTestRunning = true;
2475
2476	/*
2477	* Simple test, map CR3 twice and check that we're getting the
2478	* same mapping address back.
2479	*/
2480	LogRel(("Test #1\n"));
2481	ASMIntDisable();
2482	PGMRZDynMapStartAutoSet(&pVM->aCpus[0]);
2483
2484	uint64_t cr3 = ASMGetCR3() & ~(uint64_t)PAGE_OFFSET_MASK;
2485	void pv = (void )(intptr_t)-1;
2486	void pv2 = (void )(intptr_t)-2;
2487	rc = pgmRZDynMapHCPageCommon(pVM, cr3, &pv RTLOG_COMMA_SRC_POS);
2488	int rc2 = pgmRZDynMapHCPageCommon(pVM, cr3, &pv2 RTLOG_COMMA_SRC_POS);
2489	ASMIntEnable();
2490	if ( RT_SUCCESS(rc2)
2491	&& RT_SUCCESS(rc)
2492	&& pv == pv2)
2493	{
2494	LogRel(("Load=%u/%u/%u Set=%u/%u\n", pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
2495	rc = PGMR0DynMapAssertIntegrity();
2496
2497	/*
2498	* Check that the simple set overflow code works by filling it
2499	* with more CR3 mappings.
2500	*/
2501	LogRel(("Test #2\n"));
2502	ASMIntDisable();
2503	PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
2504	for (i = 0 ; i < UINT16_MAX*2 - 1 && RT_SUCCESS(rc) && pv2 == pv; i++)
2505	{
2506	pv2 = (void *)(intptr_t)-4;
2507	rc = pgmRZDynMapHCPageCommon(pVM, cr3, &pv2 RTLOG_COMMA_SRC_POS);
2508	}
2509	ASMIntEnable();
2510	if (RT_FAILURE(rc) \|\| pv != pv2)
2511	{
2512	LogRel(("failed(%d): rc=%Rrc; pv=%p pv2=%p i=%p\n", __LINE__, rc, pv, pv2, i));
2513	if (RT_SUCCESS(rc)) rc = VERR_INTERNAL_ERROR;
2514	}
2515	else if (pSet->cEntries != 5)
2516	{
2517	LogRel(("failed(%d): cEntries=%d expected %d\n", __LINE__, pSet->cEntries, RT_ELEMENTS(pSet->aEntries) / 2));
2518	rc = VERR_INTERNAL_ERROR;
2519	}
2520	else if ( pSet->aEntries[4].cRefs != UINT16_MAX - 1
2521	\|\| pSet->aEntries[3].cRefs != UINT16_MAX - 1
2522	\|\| pSet->aEntries[2].cRefs != 1
2523	\|\| pSet->aEntries[1].cRefs != 1
2524	\|\| pSet->aEntries[0].cRefs != 1)
2525	{
2526	LogRel(("failed(%d): bad set dist: ", __LINE__));
2527	for (i = 0; i < pSet->cEntries; i++)
2528	LogRel(("[%d]=%d, ", i, pSet->aEntries[i].cRefs));
2529	LogRel(("\n"));
2530	rc = VERR_INTERNAL_ERROR;
2531	}
2532	if (RT_SUCCESS(rc))
2533	rc = PGMR0DynMapAssertIntegrity();
2534	if (RT_SUCCESS(rc))
2535	{
2536	/*
2537	* Trigger an set optimization run (exactly).
2538	*/
2539	LogRel(("Test #3\n"));
2540	ASMIntDisable();
2541	PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
2542	pv2 = NULL;
2543	for (i = 0 ; i < RT_ELEMENTS(pSet->aEntries) - 5 && RT_SUCCESS(rc) && pv2 != pv; i++)
2544	{
2545	pv2 = (void *)(intptr_t)(-5 - i);
2546	rc = pgmRZDynMapHCPageCommon(pVM, cr3 + PAGE_SIZE * (i + 5), &pv2 RTLOG_COMMA_SRC_POS);
2547	}
2548	ASMIntEnable();
2549	if (RT_FAILURE(rc) \|\| pv == pv2)
2550	{
2551	LogRel(("failed(%d): rc=%Rrc; pv=%p pv2=%p i=%d\n", __LINE__, rc, pv, pv2, i));
2552	if (RT_SUCCESS(rc)) rc = VERR_INTERNAL_ERROR;
2553	}
2554	else if (pSet->cEntries != RT_ELEMENTS(pSet->aEntries))
2555	{
2556	LogRel(("failed(%d): cEntries=%d expected %d\n", __LINE__, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
2557	rc = VERR_INTERNAL_ERROR;
2558	}
2559	LogRel(("Load=%u/%u/%u Set=%u/%u\n", pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
2560	if (RT_SUCCESS(rc))
2561	rc = PGMR0DynMapAssertIntegrity();
2562	if (RT_SUCCESS(rc))
2563	{
2564	/*
2565	* Trigger an overflow error.
2566	*/
2567	LogRel(("Test #4\n"));
2568	ASMIntDisable();
2569	PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
2570	for (i = 0 ; i < RT_ELEMENTS(pSet->aEntries) + 2; i++)
2571	{
2572	rc = pgmRZDynMapHCPageCommon(pVM, cr3 - PAGE_SIZE * (i + 5), &pv2 RTLOG_COMMA_SRC_POS);
2573	if (RT_SUCCESS(rc))
2574	rc = PGMR0DynMapAssertIntegrity();
2575	if (RT_FAILURE(rc))
2576	break;
2577	}
2578	ASMIntEnable();
2579	if (rc == VERR_PGM_DYNMAP_FULL_SET)
2580	{
2581	/* flush the set. */
2582	LogRel(("Test #5\n"));
2583	ASMIntDisable();
2584	PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
2585	PGMRZDynMapReleaseAutoSet(&pVM->aCpus[0]);
2586	PGMRZDynMapStartAutoSet(&pVM->aCpus[0]);
2587	ASMIntEnable();
2588
2589	rc = PGMR0DynMapAssertIntegrity();
2590	}
2591	else
2592	{
2593	LogRel(("failed(%d): rc=%Rrc, wanted %d ; pv2=%p Set=%u/%u; i=%d\n", __LINE__,
2594	rc, VERR_PGM_DYNMAP_FULL_SET, pv2, pSet->cEntries, RT_ELEMENTS(pSet->aEntries), i));
2595	if (RT_SUCCESS(rc)) rc = VERR_INTERNAL_ERROR;
2596	}
2597	}
2598	}
2599	}
2600	else
2601	{
2602	LogRel(("failed(%d): rc=%Rrc rc2=%Rrc; pv=%p pv2=%p\n", __LINE__, rc, rc2, pv, pv2));
2603	if (RT_SUCCESS(rc))
2604	rc = rc2;
2605	}
2606
2607	/*
2608	* Check that everyone sees the same stuff.
2609	*/
2610	if (RT_SUCCESS(rc))
2611	{
2612	LogRel(("Test #5\n"));
2613	ASMIntDisable();
2614	PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
2615	RTHCPHYS HCPhysPT = RTR0MemObjGetPagePhysAddr(pThis->pSegHead->ahMemObjPTs[0], 0);
2616	rc = pgmRZDynMapHCPageCommon(pVM, HCPhysPT, &pv RTLOG_COMMA_SRC_POS);
2617	if (RT_SUCCESS(rc))
2618	{
2619	PGMR0DYNMAPTEST Test;
2620	uint32_t *pu32Real = &pThis->paPages[pThis->pSegHead->iPage].uPte.pLegacy->u;
2621	Test.pu32 = (uint32_t *)((uintptr_t)pv \| ((uintptr_t)pu32Real & PAGE_OFFSET_MASK));
2622	Test.u32Expect = *pu32Real;
2623	ASMAtomicWriteU32(&Test.cFailures, 0);
2624	ASMIntEnable();
2625
2626	rc = RTMpOnAll(pgmR0DynMapTest3PerCpu, &Test, NULL);
2627	if (RT_FAILURE(rc))
2628	LogRel(("failed(%d): RTMpOnAll rc=%Rrc\n", __LINE__, rc));
2629	else if (Test.cFailures)
2630	{
2631	LogRel(("failed(%d): cFailures=%d pu32Real=%p pu32=%p u32Expect=%#x *pu32=%#x\n", __LINE__,
2632	Test.cFailures, pu32Real, Test.pu32, Test.u32Expect, *Test.pu32));
2633	rc = VERR_INTERNAL_ERROR;
2634	}
2635	else
2636	LogRel(("pu32Real=%p pu32=%p u32Expect=%#x *pu32=%#x\n",
2637	pu32Real, Test.pu32, Test.u32Expect, *Test.pu32));
2638	}
2639	else
2640	{
2641	ASMIntEnable();
2642	LogRel(("failed(%d): rc=%Rrc\n", rc));
2643	}
2644	}
2645
2646	/*
2647	* Clean up.
2648	*/
2649	LogRel(("Cleanup.\n"));
2650	ASMIntDisable();
2651	PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
2652	PGMRZDynMapFlushAutoSet(&pVM->aCpus[0]);
2653	PGMRZDynMapReleaseAutoSet(&pVM->aCpus[0]);
2654	ASMIntEnable();
2655
2656	if (RT_SUCCESS(rc))
2657	rc = PGMR0DynMapAssertIntegrity();
2658	else
2659	PGMR0DynMapAssertIntegrity();
2660
2661	g_fPGMR0DynMapTestRunning = false;
2662	LogRel(("Result: rc=%Rrc Load=%u/%u/%u Set=%#x/%u\n", rc,
2663	pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
2664	pVM->pgm.s.pvR0DynMapUsed = pvR0DynMapUsedSaved;
2665	LogRel(("pgmR0DynMapTest: **** END ****\n"));
2666	return rc;
2667	}
2668	#endif /* DEBUG */
2669

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMRZ/PGMRZDynMap.cpp@ 31834

Download in other formats: