PGMPool.cpp@ 29719

Last change on this file since 29719 was 29704, checked in by vboxsync, 14 years ago
Reset all dirty pages to reactivate the page monitoring after clearing all page references and shadow page tables as there might be stale references to recently removed MMIO ranges around that might otherwise end up asserting in pgmPoolTracDerefGCPhysHint.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 45.9 KB

Line
1	/* $Id: PGMPool.cpp 29704 2010-05-20 15:42:28Z vboxsync $ */
2	/** @file
3	* PGM Shadow Page Pool.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/** @page pg_pgm_pool PGM Shadow Page Pool
19	*
20	* Motivations:
21	* -# Relationship between shadow page tables and physical guest pages. This
22	* should allow us to skip most of the global flushes now following access
23	* handler changes. The main expense is flushing shadow pages.
24	* -# Limit the pool size if necessary (default is kind of limitless).
25	* -# Allocate shadow pages from RC. We use to only do this in SyncCR3.
26	* -# Required for 64-bit guests.
27	* -# Combining the PD cache and page pool in order to simplify caching.
28	*
29	*
30	* @section sec_pgm_pool_outline Design Outline
31	*
32	* The shadow page pool tracks pages used for shadowing paging structures (i.e.
33	* page tables, page directory, page directory pointer table and page map
34	* level-4). Each page in the pool has an unique identifier. This identifier is
35	* used to link a guest physical page to a shadow PT. The identifier is a
36	* non-zero value and has a relativly low max value - say 14 bits. This makes it
37	* possible to fit it into the upper bits of the of the aHCPhys entries in the
38	* ram range.
39	*
40	* By restricting host physical memory to the first 48 bits (which is the
41	* announced physical memory range of the K8L chip (scheduled for 2008)), we
42	* can safely use the upper 16 bits for shadow page ID and reference counting.
43	*
44	* Update: The 48 bit assumption will be lifted with the new physical memory
45	* management (PGMPAGE), so we won't have any trouble when someone stuffs 2TB
46	* into a box in some years.
47	*
48	* Now, it's possible for a page to be aliased, i.e. mapped by more than one PT
49	* or PD. This is solved by creating a list of physical cross reference extents
50	* when ever this happens. Each node in the list (extent) is can contain 3 page
51	* pool indexes. The list it self is chained using indexes into the paPhysExt
52	* array.
53	*
54	*
55	* @section sec_pgm_pool_life Life Cycle of a Shadow Page
56	*
57	* -# The SyncPT function requests a page from the pool.
58	* The request includes the kind of page it is (PT/PD, PAE/legacy), the
59	* address of the page it's shadowing, and more.
60	* -# The pool responds to the request by allocating a new page.
61	* When the cache is enabled, it will first check if it's in the cache.
62	* Should the pool be exhausted, one of two things can be done:
63	* -# Flush the whole pool and current CR3.
64	* -# Use the cache to find a page which can be flushed (~age).
65	* -# The SyncPT function will sync one or more pages and insert it into the
66	* shadow PD.
67	* -# The SyncPage function may sync more pages on a later \#PFs.
68	* -# The page is freed / flushed in SyncCR3 (perhaps) and some other cases.
69	* When caching is enabled, the page isn't flush but remains in the cache.
70	*
71	*
72	* @section sec_pgm_pool_impl Monitoring
73	*
74	* We always monitor PAGE_SIZE chunks of memory. When we've got multiple shadow
75	* pages for the same PAGE_SIZE of guest memory (PAE and mixed PD/PT) the pages
76	* sharing the monitor get linked using the iMonitoredNext/Prev. The head page
77	* is the pvUser to the access handlers.
78	*
79	*
80	* @section sec_pgm_pool_impl Implementation
81	*
82	* The pool will take pages from the MM page pool. The tracking data
83	* (attributes, bitmaps and so on) are allocated from the hypervisor heap. The
84	* pool content can be accessed both by using the page id and the physical
85	* address (HC). The former is managed by means of an array, the latter by an
86	* offset based AVL tree.
87	*
88	* Flushing of a pool page means that we iterate the content (we know what kind
89	* it is) and updates the link information in the ram range.
90	*
91	* ...
92	*/
93
94
95	/*******************************************************************************
96	* Header Files *
97	*******************************************************************************/
98	#define LOG_GROUP LOG_GROUP_PGM_POOL
99	#include <VBox/pgm.h>
100	#include <VBox/mm.h>
101	#include "PGMInternal.h"
102	#include <VBox/vm.h>
103	#include "PGMInline.h"
104
105	#include <VBox/log.h>
106	#include <VBox/err.h>
107	#include <iprt/asm.h>
108	#include <iprt/string.h>
109	#include <VBox/dbg.h>
110
111
112	/*******************************************************************************
113	* Internal Functions *
114	*******************************************************************************/
115	static DECLCALLBACK(int) pgmR3PoolAccessHandler(PVM pVM, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser);
116	#ifdef VBOX_WITH_DEBUGGER
117	static DECLCALLBACK(int) pgmR3PoolCmdCheck(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
118	#endif
119
120	#ifdef VBOX_WITH_DEBUGGER
121	/** Command descriptors. */
122	static const DBGCCMD g_aCmds[] =
123	{
124	/* pszCmd, cArgsMin, cArgsMax, paArgDesc, cArgDescs, pResultDesc, fFlags, pfnHandler pszSyntax, ....pszDescription */
125	{ "pgmpoolcheck", 0, 0, NULL, 0, NULL, 0, pgmR3PoolCmdCheck, "", "Check the pgm pool pages." },
126	};
127	#endif
128
129	/**
130	* Initalizes the pool
131	*
132	* @returns VBox status code.
133	* @param pVM The VM handle.
134	*/
135	int pgmR3PoolInit(PVM pVM)
136	{
137	AssertCompile(NIL_PGMPOOL_IDX == 0);
138	/* pPage->cLocked is an unsigned byte. */
139	AssertCompile(VMM_MAX_CPU_COUNT <= 255);
140
141	/*
142	* Query Pool config.
143	*/
144	PCFGMNODE pCfg = CFGMR3GetChild(CFGMR3GetRoot(pVM), "/PGM/Pool");
145
146	/** @cfgm{/PGM/Pool/MaxPages, uint16_t, #pages, 16, 0x3fff, 1024}
147	* The max size of the shadow page pool in pages. The pool will grow dynamically
148	* up to this limit.
149	*/
150	uint16_t cMaxPages;
151	int rc = CFGMR3QueryU16Def(pCfg, "MaxPages", &cMaxPages, 4*_1M >> PAGE_SHIFT);
152	AssertLogRelRCReturn(rc, rc);
153	AssertLogRelMsgReturn(cMaxPages <= PGMPOOL_IDX_LAST && cMaxPages >= RT_ALIGN(PGMPOOL_IDX_FIRST, 16),
154	("cMaxPages=%u (%#x)\n", cMaxPages, cMaxPages), VERR_INVALID_PARAMETER);
155	cMaxPages = RT_ALIGN(cMaxPages, 16);
156
157	/** @cfgm{/PGM/Pool/MaxUsers, uint16_t, #users, MaxUsers, 32K, MaxPages*2}
158	* The max number of shadow page user tracking records. Each shadow page has
159	* zero of other shadow pages (or CR3s) that references it, or uses it if you
160	* like. The structures describing these relationships are allocated from a
161	* fixed sized pool. This configuration variable defines the pool size.
162	*/
163	uint16_t cMaxUsers;
164	rc = CFGMR3QueryU16Def(pCfg, "MaxUsers", &cMaxUsers, cMaxPages * 2);
165	AssertLogRelRCReturn(rc, rc);
166	AssertLogRelMsgReturn(cMaxUsers >= cMaxPages && cMaxPages <= _32K,
167	("cMaxUsers=%u (%#x)\n", cMaxUsers, cMaxUsers), VERR_INVALID_PARAMETER);
168
169	/** @cfgm{/PGM/Pool/MaxPhysExts, uint16_t, #extents, 16, MaxPages * 2, MAX(MaxPages*2,0x3fff)}
170	* The max number of extents for tracking aliased guest pages.
171	*/
172	uint16_t cMaxPhysExts;
173	rc = CFGMR3QueryU16Def(pCfg, "MaxPhysExts", &cMaxPhysExts, RT_MAX(cMaxPages * 2, PGMPOOL_IDX_LAST));
174	AssertLogRelRCReturn(rc, rc);
175	AssertLogRelMsgReturn(cMaxPhysExts >= 16 && cMaxPages <= PGMPOOL_IDX_LAST,
176	("cMaxPhysExts=%u (%#x)\n", cMaxPhysExts, cMaxPhysExts), VERR_INVALID_PARAMETER);
177
178	/** @cfgm{/PGM/Pool/ChacheEnabled, bool, true}
179	* Enables or disabling caching of shadow pages. Chaching means that we will try
180	* reuse shadow pages instead of recreating them everything SyncCR3, SyncPT or
181	* SyncPage requests one. When reusing a shadow page, we can save time
182	* reconstructing it and it's children.
183	*/
184	bool fCacheEnabled;
185	rc = CFGMR3QueryBoolDef(pCfg, "CacheEnabled", &fCacheEnabled, true);
186	AssertLogRelRCReturn(rc, rc);
187
188	Log(("pgmR3PoolInit: cMaxPages=%#RX16 cMaxUsers=%#RX16 cMaxPhysExts=%#RX16 fCacheEnable=%RTbool\n",
189	cMaxPages, cMaxUsers, cMaxPhysExts, fCacheEnabled));
190
191	/*
192	* Allocate the data structures.
193	*/
194	uint32_t cb = RT_OFFSETOF(PGMPOOL, aPages[cMaxPages]);
195	cb += cMaxUsers * sizeof(PGMPOOLUSER);
196	cb += cMaxPhysExts * sizeof(PGMPOOLPHYSEXT);
197	PPGMPOOL pPool;
198	rc = MMR3HyperAllocOnceNoRel(pVM, cb, 0, MM_TAG_PGM_POOL, (void **)&pPool);
199	if (RT_FAILURE(rc))
200	return rc;
201	pVM->pgm.s.pPoolR3 = pPool;
202	pVM->pgm.s.pPoolR0 = MMHyperR3ToR0(pVM, pPool);
203	pVM->pgm.s.pPoolRC = MMHyperR3ToRC(pVM, pPool);
204
205	/*
206	* Initialize it.
207	*/
208	pPool->pVMR3 = pVM;
209	pPool->pVMR0 = pVM->pVMR0;
210	pPool->pVMRC = pVM->pVMRC;
211	pPool->cMaxPages = cMaxPages;
212	pPool->cCurPages = PGMPOOL_IDX_FIRST;
213	pPool->iUserFreeHead = 0;
214	pPool->cMaxUsers = cMaxUsers;
215	PPGMPOOLUSER paUsers = (PPGMPOOLUSER)&pPool->aPages[pPool->cMaxPages];
216	pPool->paUsersR3 = paUsers;
217	pPool->paUsersR0 = MMHyperR3ToR0(pVM, paUsers);
218	pPool->paUsersRC = MMHyperR3ToRC(pVM, paUsers);
219	for (unsigned i = 0; i < cMaxUsers; i++)
220	{
221	paUsers[i].iNext = i + 1;
222	paUsers[i].iUser = NIL_PGMPOOL_IDX;
223	paUsers[i].iUserTable = 0xfffffffe;
224	}
225	paUsers[cMaxUsers - 1].iNext = NIL_PGMPOOL_USER_INDEX;
226	pPool->iPhysExtFreeHead = 0;
227	pPool->cMaxPhysExts = cMaxPhysExts;
228	PPGMPOOLPHYSEXT paPhysExts = (PPGMPOOLPHYSEXT)&paUsers[cMaxUsers];
229	pPool->paPhysExtsR3 = paPhysExts;
230	pPool->paPhysExtsR0 = MMHyperR3ToR0(pVM, paPhysExts);
231	pPool->paPhysExtsRC = MMHyperR3ToRC(pVM, paPhysExts);
232	for (unsigned i = 0; i < cMaxPhysExts; i++)
233	{
234	paPhysExts[i].iNext = i + 1;
235	paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
236	paPhysExts[i].apte[0] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
237	paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
238	paPhysExts[i].apte[1] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
239	paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
240	paPhysExts[i].apte[2] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
241	}
242	paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
243	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aiHash); i++)
244	pPool->aiHash[i] = NIL_PGMPOOL_IDX;
245	pPool->iAgeHead = NIL_PGMPOOL_IDX;
246	pPool->iAgeTail = NIL_PGMPOOL_IDX;
247	pPool->fCacheEnabled = fCacheEnabled;
248	pPool->pfnAccessHandlerR3 = pgmR3PoolAccessHandler;
249	pPool->pszAccessHandler = "Guest Paging Access Handler";
250	pPool->HCPhysTree = 0;
251
252	/* The NIL entry. */
253	Assert(NIL_PGMPOOL_IDX == 0);
254	pPool->aPages[NIL_PGMPOOL_IDX].enmKind = PGMPOOLKIND_INVALID;
255
256	/* The Shadow 32-bit PD. (32 bits guest paging) */
257	pPool->aPages[PGMPOOL_IDX_PD].Core.Key = NIL_RTHCPHYS;
258	pPool->aPages[PGMPOOL_IDX_PD].GCPhys = NIL_RTGCPHYS;
259	pPool->aPages[PGMPOOL_IDX_PD].pvPageR3 = 0;
260	pPool->aPages[PGMPOOL_IDX_PD].enmKind = PGMPOOLKIND_32BIT_PD;
261	pPool->aPages[PGMPOOL_IDX_PD].idx = PGMPOOL_IDX_PD;
262
263	/* The Shadow PDPT. */
264	pPool->aPages[PGMPOOL_IDX_PDPT].Core.Key = NIL_RTHCPHYS;
265	pPool->aPages[PGMPOOL_IDX_PDPT].GCPhys = NIL_RTGCPHYS;
266	pPool->aPages[PGMPOOL_IDX_PDPT].pvPageR3 = 0;
267	pPool->aPages[PGMPOOL_IDX_PDPT].enmKind = PGMPOOLKIND_PAE_PDPT;
268	pPool->aPages[PGMPOOL_IDX_PDPT].idx = PGMPOOL_IDX_PDPT;
269
270	/* The Shadow AMD64 CR3. */
271	pPool->aPages[PGMPOOL_IDX_AMD64_CR3].Core.Key = NIL_RTHCPHYS;
272	pPool->aPages[PGMPOOL_IDX_AMD64_CR3].GCPhys = NIL_RTGCPHYS;
273	pPool->aPages[PGMPOOL_IDX_AMD64_CR3].pvPageR3 = 0;
274	pPool->aPages[PGMPOOL_IDX_AMD64_CR3].enmKind = PGMPOOLKIND_64BIT_PML4;
275	pPool->aPages[PGMPOOL_IDX_AMD64_CR3].idx = PGMPOOL_IDX_AMD64_CR3;
276
277	/* The Nested Paging CR3. */
278	pPool->aPages[PGMPOOL_IDX_NESTED_ROOT].Core.Key = NIL_RTHCPHYS;
279	pPool->aPages[PGMPOOL_IDX_NESTED_ROOT].GCPhys = NIL_RTGCPHYS;
280	pPool->aPages[PGMPOOL_IDX_NESTED_ROOT].pvPageR3 = 0;
281	pPool->aPages[PGMPOOL_IDX_NESTED_ROOT].enmKind = PGMPOOLKIND_ROOT_NESTED;
282	pPool->aPages[PGMPOOL_IDX_NESTED_ROOT].idx = PGMPOOL_IDX_NESTED_ROOT;
283
284	/*
285	* Set common stuff.
286	*/
287	for (unsigned iPage = 1; iPage < PGMPOOL_IDX_FIRST; iPage++)
288	{
289	pPool->aPages[iPage].iNext = NIL_PGMPOOL_IDX;
290	pPool->aPages[iPage].iUserHead = NIL_PGMPOOL_USER_INDEX;
291	pPool->aPages[iPage].iModifiedNext = NIL_PGMPOOL_IDX;
292	pPool->aPages[iPage].iModifiedPrev = NIL_PGMPOOL_IDX;
293	pPool->aPages[iPage].iMonitoredNext = NIL_PGMPOOL_IDX;
294	pPool->aPages[iPage].iMonitoredNext = NIL_PGMPOOL_IDX;
295	pPool->aPages[iPage].iAgeNext = NIL_PGMPOOL_IDX;
296	pPool->aPages[iPage].iAgePrev = NIL_PGMPOOL_IDX;
297	Assert(pPool->aPages[iPage].idx == iPage);
298	Assert(pPool->aPages[iPage].GCPhys == NIL_RTGCPHYS);
299	Assert(!pPool->aPages[iPage].fSeenNonGlobal);
300	Assert(!pPool->aPages[iPage].fMonitored);
301	Assert(!pPool->aPages[iPage].fCached);
302	Assert(!pPool->aPages[iPage].fZeroed);
303	Assert(!pPool->aPages[iPage].fReusedFlushPending);
304	}
305
306	#ifdef VBOX_WITH_STATISTICS
307	/*
308	* Register statistics.
309	*/
310	STAM_REG(pVM, &pPool->cCurPages, STAMTYPE_U16, "/PGM/Pool/cCurPages", STAMUNIT_PAGES, "Current pool size.");
311	STAM_REG(pVM, &pPool->cMaxPages, STAMTYPE_U16, "/PGM/Pool/cMaxPages", STAMUNIT_PAGES, "Max pool size.");
312	STAM_REG(pVM, &pPool->cUsedPages, STAMTYPE_U16, "/PGM/Pool/cUsedPages", STAMUNIT_PAGES, "The number of pages currently in use.");
313	STAM_REG(pVM, &pPool->cUsedPagesHigh, STAMTYPE_U16_RESET, "/PGM/Pool/cUsedPagesHigh", STAMUNIT_PAGES, "The high watermark for cUsedPages.");
314	STAM_REG(pVM, &pPool->StatAlloc, STAMTYPE_PROFILE_ADV, "/PGM/Pool/Alloc", STAMUNIT_TICKS_PER_CALL, "Profiling of pgmPoolAlloc.");
315	STAM_REG(pVM, &pPool->StatClearAll, STAMTYPE_PROFILE, "/PGM/Pool/ClearAll", STAMUNIT_TICKS_PER_CALL, "Profiling of pgmR3PoolClearAll.");
316	STAM_REG(pVM, &pPool->StatR3Reset, STAMTYPE_PROFILE, "/PGM/Pool/R3Reset", STAMUNIT_TICKS_PER_CALL, "Profiling of pgmR3PoolReset.");
317	STAM_REG(pVM, &pPool->StatFlushPage, STAMTYPE_PROFILE, "/PGM/Pool/FlushPage", STAMUNIT_TICKS_PER_CALL, "Profiling of pgmPoolFlushPage.");
318	STAM_REG(pVM, &pPool->StatFree, STAMTYPE_PROFILE, "/PGM/Pool/Free", STAMUNIT_TICKS_PER_CALL, "Profiling of pgmPoolFree.");
319	STAM_REG(pVM, &pPool->StatForceFlushPage, STAMTYPE_COUNTER, "/PGM/Pool/FlushForce", STAMUNIT_OCCURENCES, "Counting explicit flushes by PGMPoolFlushPage().");
320	STAM_REG(pVM, &pPool->StatForceFlushDirtyPage, STAMTYPE_COUNTER, "/PGM/Pool/FlushForceDirty", STAMUNIT_OCCURENCES, "Counting explicit flushes of dirty pages by PGMPoolFlushPage().");
321	STAM_REG(pVM, &pPool->StatForceFlushReused, STAMTYPE_COUNTER, "/PGM/Pool/FlushReused", STAMUNIT_OCCURENCES, "Counting flushes for reused pages.");
322	STAM_REG(pVM, &pPool->StatZeroPage, STAMTYPE_PROFILE, "/PGM/Pool/ZeroPage", STAMUNIT_TICKS_PER_CALL, "Profiling time spent zeroing pages. Overlaps with Alloc.");
323	STAM_REG(pVM, &pPool->cMaxUsers, STAMTYPE_U16, "/PGM/Pool/Track/cMaxUsers", STAMUNIT_COUNT, "Max user tracking records.");
324	STAM_REG(pVM, &pPool->cPresent, STAMTYPE_U32, "/PGM/Pool/Track/cPresent", STAMUNIT_COUNT, "Number of present page table entries.");
325	STAM_REG(pVM, &pPool->StatTrackDeref, STAMTYPE_PROFILE, "/PGM/Pool/Track/Deref", STAMUNIT_TICKS_PER_CALL, "Profiling of pgmPoolTrackDeref.");
326	STAM_REG(pVM, &pPool->StatTrackFlushGCPhysPT, STAMTYPE_PROFILE, "/PGM/Pool/Track/FlushGCPhysPT", STAMUNIT_TICKS_PER_CALL, "Profiling of pgmPoolTrackFlushGCPhysPT.");
327	STAM_REG(pVM, &pPool->StatTrackFlushGCPhysPTs, STAMTYPE_PROFILE, "/PGM/Pool/Track/FlushGCPhysPTs", STAMUNIT_TICKS_PER_CALL, "Profiling of pgmPoolTrackFlushGCPhysPTs.");
328	STAM_REG(pVM, &pPool->StatTrackFlushGCPhysPTsSlow, STAMTYPE_PROFILE, "/PGM/Pool/Track/FlushGCPhysPTsSlow", STAMUNIT_TICKS_PER_CALL, "Profiling of pgmPoolTrackFlushGCPhysPTsSlow.");
329	STAM_REG(pVM, &pPool->StatTrackFlushEntry, STAMTYPE_COUNTER, "/PGM/Pool/Track/Entry/Flush", STAMUNIT_COUNT, "Nr of flushed entries.");
330	STAM_REG(pVM, &pPool->StatTrackFlushEntryKeep, STAMTYPE_COUNTER, "/PGM/Pool/Track/Entry/Update", STAMUNIT_COUNT, "Nr of updated entries.");
331	STAM_REG(pVM, &pPool->StatTrackFreeUpOneUser, STAMTYPE_COUNTER, "/PGM/Pool/Track/FreeUpOneUser", STAMUNIT_TICKS_PER_CALL, "The number of times we were out of user tracking records.");
332	STAM_REG(pVM, &pPool->StatTrackDerefGCPhys, STAMTYPE_PROFILE, "/PGM/Pool/Track/DrefGCPhys", STAMUNIT_TICKS_PER_CALL, "Profiling deref activity related tracking GC physical pages.");
333	STAM_REG(pVM, &pPool->StatTrackLinearRamSearches, STAMTYPE_COUNTER, "/PGM/Pool/Track/LinearRamSearches", STAMUNIT_OCCURENCES, "The number of times we had to do linear ram searches.");
334	STAM_REG(pVM, &pPool->StamTrackPhysExtAllocFailures,STAMTYPE_COUNTER, "/PGM/Pool/Track/PhysExtAllocFailures", STAMUNIT_OCCURENCES, "The number of failing pgmPoolTrackPhysExtAlloc calls.");
335	STAM_REG(pVM, &pPool->StatMonitorRZ, STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ", STAMUNIT_TICKS_PER_CALL, "Profiling the RC/R0 access handler.");
336	STAM_REG(pVM, &pPool->StatMonitorRZEmulateInstr, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/EmulateInstr", STAMUNIT_OCCURENCES, "Times we've failed interpreting the instruction.");
337	STAM_REG(pVM, &pPool->StatMonitorRZFlushPage, STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/FlushPage", STAMUNIT_TICKS_PER_CALL, "Profiling the pgmPoolFlushPage calls made from the RC/R0 access handler.");
338	STAM_REG(pVM, &pPool->StatMonitorRZFlushReinit, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/FlushReinit", STAMUNIT_OCCURENCES, "Times we've detected a page table reinit.");
339	STAM_REG(pVM, &pPool->StatMonitorRZFlushModOverflow,STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/FlushOverflow", STAMUNIT_OCCURENCES, "Counting flushes for pages that are modified too often.");
340	STAM_REG(pVM, &pPool->StatMonitorRZFork, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/Fork", STAMUNIT_OCCURENCES, "Times we've detected fork().");
341	STAM_REG(pVM, &pPool->StatMonitorRZHandled, STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/Handled", STAMUNIT_TICKS_PER_CALL, "Profiling the RC/R0 access we've handled (except REP STOSD).");
342	STAM_REG(pVM, &pPool->StatMonitorRZIntrFailPatch1, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/IntrFailPatch1", STAMUNIT_OCCURENCES, "Times we've failed interpreting a patch code instruction.");
343	STAM_REG(pVM, &pPool->StatMonitorRZIntrFailPatch2, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/IntrFailPatch2", STAMUNIT_OCCURENCES, "Times we've failed interpreting a patch code instruction during flushing.");
344	STAM_REG(pVM, &pPool->StatMonitorRZRepPrefix, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/RepPrefix", STAMUNIT_OCCURENCES, "The number of times we've seen rep prefixes we can't handle.");
345	STAM_REG(pVM, &pPool->StatMonitorRZRepStosd, STAMTYPE_PROFILE, "/PGM/Pool/Monitor/RZ/RepStosd", STAMUNIT_TICKS_PER_CALL, "Profiling the REP STOSD cases we've handled.");
346	STAM_REG(pVM, &pPool->StatMonitorRZFaultPT, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/Fault/PT", STAMUNIT_OCCURENCES, "Nr of handled PT faults.");
347	STAM_REG(pVM, &pPool->StatMonitorRZFaultPD, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/Fault/PD", STAMUNIT_OCCURENCES, "Nr of handled PD faults.");
348	STAM_REG(pVM, &pPool->StatMonitorRZFaultPDPT, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/Fault/PDPT", STAMUNIT_OCCURENCES, "Nr of handled PDPT faults.");
349	STAM_REG(pVM, &pPool->StatMonitorRZFaultPML4, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/RZ/Fault/PML4", STAMUNIT_OCCURENCES, "Nr of handled PML4 faults.");
350	STAM_REG(pVM, &pPool->StatMonitorR3, STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3", STAMUNIT_TICKS_PER_CALL, "Profiling the R3 access handler.");
351	STAM_REG(pVM, &pPool->StatMonitorR3EmulateInstr, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/R3/EmulateInstr", STAMUNIT_OCCURENCES, "Times we've failed interpreting the instruction.");
352	STAM_REG(pVM, &pPool->StatMonitorR3FlushPage, STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/FlushPage", STAMUNIT_TICKS_PER_CALL, "Profiling the pgmPoolFlushPage calls made from the R3 access handler.");
353	STAM_REG(pVM, &pPool->StatMonitorR3FlushReinit, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/R3/FlushReinit", STAMUNIT_OCCURENCES, "Times we've detected a page table reinit.");
354	STAM_REG(pVM, &pPool->StatMonitorR3FlushModOverflow,STAMTYPE_COUNTER, "/PGM/Pool/Monitor/R3/FlushOverflow", STAMUNIT_OCCURENCES, "Counting flushes for pages that are modified too often.");
355	STAM_REG(pVM, &pPool->StatMonitorR3Fork, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/R3/Fork", STAMUNIT_OCCURENCES, "Times we've detected fork().");
356	STAM_REG(pVM, &pPool->StatMonitorR3Handled, STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/Handled", STAMUNIT_TICKS_PER_CALL, "Profiling the R3 access we've handled (except REP STOSD).");
357	STAM_REG(pVM, &pPool->StatMonitorR3RepPrefix, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/R3/RepPrefix", STAMUNIT_OCCURENCES, "The number of times we've seen rep prefixes we can't handle.");
358	STAM_REG(pVM, &pPool->StatMonitorR3RepStosd, STAMTYPE_PROFILE, "/PGM/Pool/Monitor/R3/RepStosd", STAMUNIT_TICKS_PER_CALL, "Profiling the REP STOSD cases we've handled.");
359	STAM_REG(pVM, &pPool->StatMonitorR3FaultPT, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/R3/Fault/PT", STAMUNIT_OCCURENCES, "Nr of handled PT faults.");
360	STAM_REG(pVM, &pPool->StatMonitorR3FaultPD, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/R3/Fault/PD", STAMUNIT_OCCURENCES, "Nr of handled PD faults.");
361	STAM_REG(pVM, &pPool->StatMonitorR3FaultPDPT, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/R3/Fault/PDPT", STAMUNIT_OCCURENCES, "Nr of handled PDPT faults.");
362	STAM_REG(pVM, &pPool->StatMonitorR3FaultPML4, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/R3/Fault/PML4", STAMUNIT_OCCURENCES, "Nr of handled PML4 faults.");
363	STAM_REG(pVM, &pPool->StatMonitorR3Async, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/R3/Async", STAMUNIT_OCCURENCES, "Times we're called in an async thread and need to flush.");
364	STAM_REG(pVM, &pPool->cModifiedPages, STAMTYPE_U16, "/PGM/Pool/Monitor/cModifiedPages", STAMUNIT_PAGES, "The current cModifiedPages value.");
365	STAM_REG(pVM, &pPool->cModifiedPagesHigh, STAMTYPE_U16_RESET, "/PGM/Pool/Monitor/cModifiedPagesHigh", STAMUNIT_PAGES, "The high watermark for cModifiedPages.");
366	STAM_REG(pVM, &pPool->StatResetDirtyPages, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/Dirty/Resets", STAMUNIT_OCCURENCES, "Times we've called pgmPoolResetDirtyPages (and there were dirty page).");
367	STAM_REG(pVM, &pPool->StatDirtyPage, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/Dirty/Pages", STAMUNIT_OCCURENCES, "Times we've called pgmPoolAddDirtyPage.");
368	STAM_REG(pVM, &pPool->StatDirtyPageDupFlush, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/Dirty/FlushDup", STAMUNIT_OCCURENCES, "Times we've had to flush duplicates for dirty page management.");
369	STAM_REG(pVM, &pPool->StatDirtyPageOverFlowFlush, STAMTYPE_COUNTER, "/PGM/Pool/Monitor/Dirty/FlushOverflow",STAMUNIT_OCCURENCES, "Times we've had to flush because of overflow.");
370	STAM_REG(pVM, &pPool->StatCacheHits, STAMTYPE_COUNTER, "/PGM/Pool/Cache/Hits", STAMUNIT_OCCURENCES, "The number of pgmPoolAlloc calls satisfied by the cache.");
371	STAM_REG(pVM, &pPool->StatCacheMisses, STAMTYPE_COUNTER, "/PGM/Pool/Cache/Misses", STAMUNIT_OCCURENCES, "The number of pgmPoolAlloc calls not statisfied by the cache.");
372	STAM_REG(pVM, &pPool->StatCacheKindMismatches, STAMTYPE_COUNTER, "/PGM/Pool/Cache/KindMismatches", STAMUNIT_OCCURENCES, "The number of shadow page kind mismatches. (Better be low, preferably 0!)");
373	STAM_REG(pVM, &pPool->StatCacheFreeUpOne, STAMTYPE_COUNTER, "/PGM/Pool/Cache/FreeUpOne", STAMUNIT_OCCURENCES, "The number of times the cache was asked to free up a page.");
374	STAM_REG(pVM, &pPool->StatCacheCacheable, STAMTYPE_COUNTER, "/PGM/Pool/Cache/Cacheable", STAMUNIT_OCCURENCES, "The number of cacheable allocations.");
375	STAM_REG(pVM, &pPool->StatCacheUncacheable, STAMTYPE_COUNTER, "/PGM/Pool/Cache/Uncacheable", STAMUNIT_OCCURENCES, "The number of uncacheable allocations.");
376	#endif /* VBOX_WITH_STATISTICS */
377
378	#ifdef VBOX_WITH_DEBUGGER
379	/*
380	* Debugger commands.
381	*/
382	static bool s_fRegisteredCmds = false;
383	if (!s_fRegisteredCmds)
384	{
385	rc = DBGCRegisterCommands(&g_aCmds[0], RT_ELEMENTS(g_aCmds));
386	if (RT_SUCCESS(rc))
387	s_fRegisteredCmds = true;
388	}
389	#endif
390
391	return VINF_SUCCESS;
392	}
393
394
395	/**
396	* Relocate the page pool data.
397	*
398	* @param pVM The VM handle.
399	*/
400	void pgmR3PoolRelocate(PVM pVM)
401	{
402	pVM->pgm.s.pPoolRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pPoolR3);
403	pVM->pgm.s.pPoolR3->pVMRC = pVM->pVMRC;
404	pVM->pgm.s.pPoolR3->paUsersRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pPoolR3->paUsersR3);
405	pVM->pgm.s.pPoolR3->paPhysExtsRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pPoolR3->paPhysExtsR3);
406	int rc = PDMR3LdrGetSymbolRC(pVM, NULL, "pgmPoolAccessHandler", &pVM->pgm.s.pPoolR3->pfnAccessHandlerRC);
407	AssertReleaseRC(rc);
408	/* init order hack. */
409	if (!pVM->pgm.s.pPoolR3->pfnAccessHandlerR0)
410	{
411	rc = PDMR3LdrGetSymbolR0(pVM, NULL, "pgmPoolAccessHandler", &pVM->pgm.s.pPoolR3->pfnAccessHandlerR0);
412	AssertReleaseRC(rc);
413	}
414	}
415
416
417	/**
418	* Grows the shadow page pool.
419	*
420	* I.e. adds more pages to it, assuming that hasn't reached cMaxPages yet.
421	*
422	* @returns VBox status code.
423	* @param pVM The VM handle.
424	*/
425	VMMR3DECL(int) PGMR3PoolGrow(PVM pVM)
426	{
427	PPGMPOOL pPool = pVM->pgm.s.pPoolR3;
428	AssertReturn(pPool->cCurPages < pPool->cMaxPages, VERR_INTERNAL_ERROR);
429
430	pgmLock(pVM);
431
432	/*
433	* How much to grow it by?
434	*/
435	uint32_t cPages = pPool->cMaxPages - pPool->cCurPages;
436	cPages = RT_MIN(PGMPOOL_CFG_MAX_GROW, cPages);
437	LogFlow(("PGMR3PoolGrow: Growing the pool by %d (%#x) pages.\n", cPages, cPages));
438
439	for (unsigned i = pPool->cCurPages; cPages-- > 0; i++)
440	{
441	PPGMPOOLPAGE pPage = &pPool->aPages[i];
442
443	/* Allocate all pages in low (below 4 GB) memory as 32 bits guests need a page table root in low memory. */
444	pPage->pvPageR3 = MMR3PageAllocLow(pVM);
445	if (!pPage->pvPageR3)
446	{
447	Log(("We're out of memory!! i=%d\n", i));
448	pgmUnlock(pVM);
449	return i ? VINF_SUCCESS : VERR_NO_PAGE_MEMORY;
450	}
451	pPage->Core.Key = MMPage2Phys(pVM, pPage->pvPageR3);
452	AssertFatal(pPage->Core.Key < _4G);
453	pPage->GCPhys = NIL_RTGCPHYS;
454	pPage->enmKind = PGMPOOLKIND_FREE;
455	pPage->idx = pPage - &pPool->aPages[0];
456	LogFlow(("PGMR3PoolGrow: insert page #%#x - %RHp\n", pPage->idx, pPage->Core.Key));
457	pPage->iNext = pPool->iFreeHead;
458	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
459	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
460	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
461	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
462	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
463	pPage->iAgeNext = NIL_PGMPOOL_IDX;
464	pPage->iAgePrev = NIL_PGMPOOL_IDX;
465	/* commit it */
466	bool fRc = RTAvloHCPhysInsert(&pPool->HCPhysTree, &pPage->Core); Assert(fRc); NOREF(fRc);
467	pPool->iFreeHead = i;
468	pPool->cCurPages = i + 1;
469	}
470
471	pgmUnlock(pVM);
472	Assert(pPool->cCurPages <= pPool->cMaxPages);
473	return VINF_SUCCESS;
474	}
475
476
477
478	/**
479	* Worker used by pgmR3PoolAccessHandler when it's invoked by an async thread.
480	*
481	* @param pPool The pool.
482	* @param pPage The page.
483	*/
484	static DECLCALLBACK(void) pgmR3PoolFlushReusedPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
485	{
486	/* for the present this should be safe enough I think... */
487	pgmLock(pPool->pVMR3);
488	if ( pPage->fReusedFlushPending
489	&& pPage->enmKind != PGMPOOLKIND_FREE)
490	pgmPoolFlushPage(pPool, pPage);
491	pgmUnlock(pPool->pVMR3);
492	}
493
494
495	/**
496	* \#PF Handler callback for PT write accesses.
497	*
498	* The handler can not raise any faults, it's mainly for monitoring write access
499	* to certain pages.
500	*
501	* @returns VINF_SUCCESS if the handler has carried out the operation.
502	* @returns VINF_PGM_HANDLER_DO_DEFAULT if the caller should carry out the access operation.
503	* @param pVM VM Handle.
504	* @param GCPhys The physical address the guest is writing to.
505	* @param pvPhys The HC mapping of that address.
506	* @param pvBuf What the guest is reading/writing.
507	* @param cbBuf How much it's reading/writing.
508	* @param enmAccessType The access type.
509	* @param pvUser User argument.
510	*/
511	static DECLCALLBACK(int) pgmR3PoolAccessHandler(PVM pVM, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser)
512	{
513	STAM_PROFILE_START(&pVM->pgm.s.pPoolR3->StatMonitorR3, a);
514	PPGMPOOL pPool = pVM->pgm.s.pPoolR3;
515	PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)pvUser;
516	LogFlow(("pgmR3PoolAccessHandler: GCPhys=%RGp %p:{.Core=%RHp, .idx=%d, .GCPhys=%RGp, .enmType=%d}\n",
517	GCPhys, pPage, pPage->Core.Key, pPage->idx, pPage->GCPhys, pPage->enmKind));
518
519	PVMCPU pVCpu = VMMGetCpu(pVM);
520
521	/*
522	* We don't have to be very sophisticated about this since there are relativly few calls here.
523	* However, we must try our best to detect any non-cpu accesses (disk / networking).
524	*
525	* Just to make life more interesting, we'll have to deal with the async threads too.
526	* We cannot flush a page if we're in an async thread because of REM notifications.
527	*/
528	pgmLock(pVM);
529	if (PHYS_PAGE_ADDRESS(GCPhys) != PHYS_PAGE_ADDRESS(pPage->GCPhys))
530	{
531	/* Pool page changed while we were waiting for the lock; ignore. */
532	Log(("CPU%d: pgmR3PoolAccessHandler pgm pool page for %RGp changed (to %RGp) while waiting!\n", pVCpu->idCpu, PHYS_PAGE_ADDRESS(GCPhys), PHYS_PAGE_ADDRESS(pPage->GCPhys)));
533	pgmUnlock(pVM);
534	return VINF_PGM_HANDLER_DO_DEFAULT;
535	}
536
537	Assert(pPage->enmKind != PGMPOOLKIND_FREE);
538
539	/* @todo this code doesn't make any sense. remove the if (!pVCpu) block */
540	if (!pVCpu) /** @todo This shouldn't happen any longer, all access handlers will be called on an EMT. All ring-3 handlers, except MMIO, already own the PGM lock. @bugref{3170} */
541	{
542	Log(("pgmR3PoolAccessHandler: async thread, requesting EMT to flush the page: %p:{.Core=%RHp, .idx=%d, .GCPhys=%RGp, .enmType=%d}\n",
543	pPage, pPage->Core.Key, pPage->idx, pPage->GCPhys, pPage->enmKind));
544	STAM_COUNTER_INC(&pPool->StatMonitorR3Async);
545	if (!pPage->fReusedFlushPending)
546	{
547	pgmUnlock(pVM);
548	int rc = VMR3ReqCallVoidNoWait(pPool->pVMR3, VMCPUID_ANY, (PFNRT)pgmR3PoolFlushReusedPage, 2, pPool, pPage);
549	AssertRCReturn(rc, rc);
550	pgmLock(pVM);
551	pPage->fReusedFlushPending = true;
552	pPage->cModifications += 0x1000;
553	}
554
555	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvPhys, 0 /* unknown write size */);
556	/** @todo r=bird: making unsafe assumption about not crossing entries here! */
557	while (cbBuf > 4)
558	{
559	cbBuf -= 4;
560	pvPhys = (uint8_t *)pvPhys + 4;
561	GCPhys += 4;
562	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvPhys, 0 /* unknown write size */);
563	}
564	STAM_PROFILE_STOP(&pPool->StatMonitorR3, a);
565	}
566	else if ( ( pPage->cModifications < 96 /* it's cheaper here. */
567	\|\| pgmPoolIsPageLocked(&pVM->pgm.s, pPage)
568	)
569	&& cbBuf <= 4)
570	{
571	/* Clear the shadow entry. */
572	if (!pPage->cModifications++)
573	pgmPoolMonitorModifiedInsert(pPool, pPage);
574	/** @todo r=bird: making unsafe assumption about not crossing entries here! */
575	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvPhys, 0 /* unknown write size */);
576	STAM_PROFILE_STOP(&pPool->StatMonitorR3, a);
577	}
578	else
579	{
580	pgmPoolMonitorChainFlush(pPool, pPage); /* ASSUME that VERR_PGM_POOL_CLEARED can be ignored here and that FFs will deal with it in due time. */
581	STAM_PROFILE_STOP_EX(&pPool->StatMonitorR3, &pPool->StatMonitorR3FlushPage, a);
582	}
583	pgmUnlock(pVM);
584	return VINF_PGM_HANDLER_DO_DEFAULT;
585	}
586
587
588	/**
589	* Rendezvous callback used by pgmR3PoolClearAll that clears all shadow pages
590	* and all modification counters.
591	*
592	* This is only called on one of the EMTs while the other ones are waiting for
593	* it to complete this function.
594	*
595	* @returns VINF_SUCCESS (VBox strict status code).
596	* @param pVM The VM handle.
597	* @param pVCpu The VMCPU for the EMT we're being called on. Unused.
598	* @param pvUser Unused parameter.
599	*
600	*/
601	DECLCALLBACK(VBOXSTRICTRC) pgmR3PoolClearAllRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
602	{
603	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
604	STAM_PROFILE_START(&pPool->StatClearAll, c);
605
606	pgmLock(pVM);
607	Log(("pgmR3PoolClearAllRendezvous: cUsedPages=%d\n", pPool->cUsedPages));
608
609	/*
610	* Iterate all the pages until we've encountered all that are in use.
611	* This is a simple but not quite optimal solution.
612	*/
613	unsigned cModifiedPages = 0; NOREF(cModifiedPages);
614	unsigned cLeft = pPool->cUsedPages;
615	unsigned iPage = pPool->cCurPages;
616	while (--iPage >= PGMPOOL_IDX_FIRST)
617	{
618	PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
619	if (pPage->GCPhys != NIL_RTGCPHYS)
620	{
621	switch (pPage->enmKind)
622	{
623	/*
624	* We only care about shadow page tables.
625	*/
626	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
627	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
628	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
629	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
630	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
631	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
632	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
633	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
634	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
635	{
636	if (pPage->cPresent)
637	{
638	void *pvShw = PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
639	STAM_PROFILE_START(&pPool->StatZeroPage, z);
640	#if 0
641	/* Useful check for leaking references; very expensive though. */
642	switch (pPage->enmKind)
643	{
644	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
645	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
646	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
647	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
648	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
649	{
650	bool fFoundFirst = false;
651	PX86PTPAE pPT = (PX86PTPAE)pvShw;
652	for (unsigned ptIndex = 0; ptIndex < RT_ELEMENTS(pPT->a); ptIndex++)
653	{
654	if (pPT->a[ptIndex].u)
655	{
656	if (!fFoundFirst)
657	{
658	AssertFatalMsg(pPage->iFirstPresent <= ptIndex, ("ptIndex = %d first present = %d\n", ptIndex, pPage->iFirstPresent));
659	if (pPage->iFirstPresent != ptIndex)
660	Log(("ptIndex = %d first present = %d\n", ptIndex, pPage->iFirstPresent));
661	fFoundFirst = true;
662	}
663	if (pPT->a[ptIndex].n.u1Present)
664	{
665	pgmPoolTracDerefGCPhysHint(pPool, pPage, pPT->a[ptIndex].u & X86_PTE_PAE_PG_MASK, NIL_RTGCPHYS);
666	if (pPage->iFirstPresent == ptIndex)
667	pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
668	}
669	}
670	}
671	AssertFatalMsg(pPage->cPresent == 0, ("cPresent = %d pPage = %RGv\n", pPage->cPresent, pPage->GCPhys));
672	break;
673	}
674	default:
675	break;
676	}
677	#endif
678	ASMMemZeroPage(pvShw);
679	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
680	pPage->cPresent = 0;
681	pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
682	}
683	}
684	/* fall thru */
685
686	default:
687	Assert(!pPage->cModifications \|\| ++cModifiedPages);
688	Assert(pPage->iModifiedNext == NIL_PGMPOOL_IDX \|\| pPage->cModifications);
689	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX \|\| pPage->cModifications);
690	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
691	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
692	pPage->cModifications = 0;
693	break;
694
695	}
696	if (!--cLeft)
697	break;
698	}
699	}
700
701	/* swipe the special pages too. */
702	for (iPage = PGMPOOL_IDX_FIRST_SPECIAL; iPage < PGMPOOL_IDX_FIRST; iPage++)
703	{
704	PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
705	if (pPage->GCPhys != NIL_RTGCPHYS)
706	{
707	Assert(!pPage->cModifications \|\| ++cModifiedPages);
708	Assert(pPage->iModifiedNext == NIL_PGMPOOL_IDX \|\| pPage->cModifications);
709	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX \|\| pPage->cModifications);
710	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
711	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
712	pPage->cModifications = 0;
713	}
714	}
715
716	#ifndef DEBUG_michael
717	AssertMsg(cModifiedPages == pPool->cModifiedPages, ("%d != %d\n", cModifiedPages, pPool->cModifiedPages));
718	#endif
719	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
720	pPool->cModifiedPages = 0;
721
722	/*
723	* Clear all the GCPhys links and rebuild the phys ext free list.
724	*/
725	for (PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
726	pRam;
727	pRam = pRam->CTX_SUFF(pNext))
728	{
729	iPage = pRam->cb >> PAGE_SHIFT;
730	while (iPage-- > 0)
731	PGM_PAGE_SET_TRACKING(&pRam->aPages[iPage], 0);
732	}
733
734	pPool->iPhysExtFreeHead = 0;
735	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
736	const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
737	for (unsigned i = 0; i < cMaxPhysExts; i++)
738	{
739	paPhysExts[i].iNext = i + 1;
740	paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
741	paPhysExts[i].apte[0] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
742	paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
743	paPhysExts[i].apte[1] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
744	paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
745	paPhysExts[i].apte[2] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
746	}
747	paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
748
749
750	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
751	/* Reset all dirty pages to reactivate the page monitoring. */
752	/* Note: we must do this after clearing all page references and shadow page tables as there might be stale references to
753	* recently removed MMIO ranges around that might otherwise end up asserting in pgmPoolTracDerefGCPhysHint
754	*/
755	pgmPoolResetDirtyPages(pVM);
756
757	/* Clear all dirty pages. */
758	pPool->idxFreeDirtyPage = 0;
759	pPool->cDirtyPages = 0;
760	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aIdxDirtyPages); i++)
761	pPool->aIdxDirtyPages[i] = NIL_PGMPOOL_IDX;
762	#endif
763
764	/* Clear the PGM_SYNC_CLEAR_PGM_POOL flag on all VCPUs to prevent redundant flushes. */
765	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
766	pVM->aCpus[idCpu].pgm.s.fSyncFlags &= ~PGM_SYNC_CLEAR_PGM_POOL;
767
768	/* Flush job finished. */
769	VM_FF_CLEAR(pVM, VM_FF_PGM_POOL_FLUSH_PENDING);
770
771	pPool->cPresent = 0;
772	pgmUnlock(pVM);
773	PGM_INVL_ALL_VCPU_TLBS(pVM);
774	STAM_PROFILE_STOP(&pPool->StatClearAll, c);
775	return VINF_SUCCESS;
776	}
777
778
779	/**
780	* Clears the shadow page pool.
781	*
782	* @param pVM The VM handle.
783	*/
784	void pgmR3PoolClearAll(PVM pVM)
785	{
786	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PoolClearAllRendezvous, NULL);
787	AssertRC(rc);
788	}
789
790
791	#ifdef VBOX_WITH_DEBUGGER
792	/**
793	* The '.pgmpoolcheck' command.
794	*
795	* @returns VBox status.
796	* @param pCmd Pointer to the command descriptor (as registered).
797	* @param pCmdHlp Pointer to command helper functions.
798	* @param pVM Pointer to the current VM (if any).
799	* @param paArgs Pointer to (readonly) array of arguments.
800	* @param cArgs Number of arguments in the array.
801	*/
802	static DECLCALLBACK(int) pgmR3PoolCmdCheck(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
803	{
804	/*
805	* Validate input.
806	*/
807	if (!pVM)
808	return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
809
810	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
811
812	for (unsigned i = 0; i < pPool->cCurPages; i++)
813	{
814	PPGMPOOLPAGE pPage = &pPool->aPages[i];
815	bool fFirstMsg = true;
816
817	/* Todo: cover other paging modes too. */
818	if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
819	{
820	PX86PTPAE pShwPT = (PX86PTPAE)PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
821	PX86PTPAE pGstPT;
822	int rc = PGM_GCPHYS_2_PTR(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pGstPT); AssertReleaseRC(rc);
823
824	/* Check if any PTEs are out of sync. */
825	for (unsigned j = 0; j < RT_ELEMENTS(pShwPT->a); j++)
826	{
827	if (pShwPT->a[j].n.u1Present)
828	{
829	RTHCPHYS HCPhys = NIL_RTHCPHYS;
830	rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pGstPT->a[j].u & X86_PTE_PAE_PG_MASK, &HCPhys);
831	if ( rc != VINF_SUCCESS
832	\|\| (pShwPT->a[j].u & X86_PTE_PAE_PG_MASK) != HCPhys)
833	{
834	if (fFirstMsg)
835	{
836	pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Check pool page %RGp\n", pPage->GCPhys);
837	fFirstMsg = false;
838	}
839	pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Mismatch HCPhys: rc=%d idx=%d guest %RX64 shw=%RX64 vs %RHp\n", rc, j, pGstPT->a[j].u, pShwPT->a[j].u, HCPhys);
840	}
841	else
842	if ( pShwPT->a[j].n.u1Write
843	&& !pGstPT->a[j].n.u1Write)
844	{
845	if (fFirstMsg)
846	{
847	pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Check pool page %RGp\n", pPage->GCPhys);
848	fFirstMsg = false;
849	}
850	pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Mismatch r/w gst/shw: idx=%d guest %RX64 shw=%RX64 vs %RHp\n", j, pGstPT->a[j].u, pShwPT->a[j].u, HCPhys);
851	}
852	}
853	}
854
855	/* Make sure this page table can't be written to from any shadow mapping. */
856	RTHCPHYS HCPhysPT = NIL_RTHCPHYS;
857	rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pPage->GCPhys, &HCPhysPT);
858	AssertMsgRC(rc, ("PGMPhysGCPhys2HCPhys failed with rc=%d for %RGp\n", rc, pPage->GCPhys));
859	if (rc == VINF_SUCCESS)
860	{
861	for (unsigned j = 0; j < pPool->cCurPages; j++)
862	{
863	PPGMPOOLPAGE pTempPage = &pPool->aPages[j];
864
865	if (pTempPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
866	{
867	PX86PTPAE pShwPT2 = (PX86PTPAE)PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pTempPage);
868
869	for (unsigned k = 0; k < RT_ELEMENTS(pShwPT->a); k++)
870	{
871	if ( pShwPT2->a[k].n.u1Present
872	&& pShwPT2->a[k].n.u1Write
873	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
874	&& !pPage->fDirty
875	# endif
876	&& ((pShwPT2->a[k].u & X86_PTE_PAE_PG_MASK) == HCPhysPT))
877	{
878	if (fFirstMsg)
879	{
880	pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Check pool page %RGp\n", pPage->GCPhys);
881	fFirstMsg = false;
882	}
883	pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Mismatch: r/w: GCPhys=%RGp idx=%d shw %RX64 %RX64\n", pTempPage->GCPhys, k, pShwPT->a[k].u, pShwPT2->a[k].u);
884	}
885	}
886	}
887	}
888	}
889	}
890	}
891	return VINF_SUCCESS;
892	}
893	#endif /* VBOX_WITH_DEBUGGER */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/PGMPool.cpp@ 29719

Download in other formats: