PGMAllPool.cpp@ 96879

Last change on this file since 96879 was 96879, checked in by vboxsync, 2 years ago
VMM/PGM: Nested VMX: bugref:10092 Nested EPT shadow page-pool handling.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 223.0 KB

Line
1	/* $Id: PGMAllPool.cpp 96879 2022-09-26 17:43:43Z vboxsync $ */
2	/** @file
3	* PGM Shadow Page Pool.
4	*/
5
6	/*
7	* Copyright (C) 2006-2022 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_PGM_POOL
33	#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */
34	#include <VBox/vmm/pgm.h>
35	#include <VBox/vmm/mm.h>
36	#include <VBox/vmm/em.h>
37	#include <VBox/vmm/cpum.h>
38	#include "PGMInternal.h"
39	#include <VBox/vmm/vmcc.h>
40	#include "PGMInline.h"
41	#include <VBox/disopcode.h>
42	#include <VBox/vmm/hm_vmx.h>
43
44	#include <VBox/log.h>
45	#include <VBox/err.h>
46	#include <iprt/asm.h>
47	#include <iprt/string.h>
48
49
50	/*********************************************************************************************************************************
51	* Internal Functions *
52	*********************************************************************************************************************************/
53	RT_C_DECLS_BEGIN
54	#if 0 /* unused */
55	DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind);
56	DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind);
57	#endif /* unused */
58	static void pgmPoolTrackClearPageUsers(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
59	static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
60	static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable);
61	static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
62	#if defined(LOG_ENABLED) \|\| defined(VBOX_STRICT)
63	static const char *pgmPoolPoolKindToStr(uint8_t enmKind);
64	#endif
65	#if 0 /defined(VBOX_STRICT) && defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT)/
66	static void pgmPoolTrackCheckPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT);
67	#endif
68
69	int pgmPoolTrackFlushGCPhysPTsSlow(PVMCC pVM, PPGMPAGE pPhysPage);
70	PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVMCC pVM, uint16_t *piPhysExt);
71	void pgmPoolTrackPhysExtFree(PVMCC pVM, uint16_t iPhysExt);
72	void pgmPoolTrackPhysExtFreeList(PVMCC pVM, uint16_t iPhysExt);
73
74	RT_C_DECLS_END
75
76
77	#if 0 /* unused */
78	/**
79	* Checks if the specified page pool kind is for a 4MB or 2MB guest page.
80	*
81	* @returns true if it's the shadow of a 4MB or 2MB guest page, otherwise false.
82	* @param enmKind The page kind.
83	*/
84	DECLINLINE(bool) pgmPoolIsBigPage(PGMPOOLKIND enmKind)
85	{
86	switch (enmKind)
87	{
88	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
89	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
90	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
91	return true;
92	default:
93	return false;
94	}
95	}
96	#endif /* unused */
97
98
99	/**
100	* Flushes a chain of pages sharing the same access monitor.
101	*
102	* @param pPool The pool.
103	* @param pPage A page in the chain.
104	*/
105	void pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
106	{
107	LogFlow(("pgmPoolMonitorChainFlush: Flush page %RGp type=%d\n", pPage->GCPhys, pPage->enmKind));
108
109	/*
110	* Find the list head.
111	*/
112	uint16_t idx = pPage->idx;
113	if (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
114	{
115	while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
116	{
117	idx = pPage->iMonitoredPrev;
118	Assert(idx != pPage->idx);
119	pPage = &pPool->aPages[idx];
120	}
121	}
122
123	/*
124	* Iterate the list flushing each shadow page.
125	*/
126	for (;;)
127	{
128	idx = pPage->iMonitoredNext;
129	Assert(idx != pPage->idx);
130	if (pPage->idx >= PGMPOOL_IDX_FIRST)
131	{
132	int rc2 = pgmPoolFlushPage(pPool, pPage);
133	AssertRC(rc2);
134	}
135	/* next */
136	if (idx == NIL_PGMPOOL_IDX)
137	break;
138	pPage = &pPool->aPages[idx];
139	}
140	}
141
142
143	/**
144	* Wrapper for getting the current context pointer to the entry being modified.
145	*
146	* @returns VBox status code suitable for scheduling.
147	* @param pVM The cross context VM structure.
148	* @param pvDst Destination address
149	* @param pvSrc Pointer to the mapping of @a GCPhysSrc or NULL depending
150	* on the context (e.g. \#PF in R0 & RC).
151	* @param GCPhysSrc The source guest physical address.
152	* @param cb Size of data to read
153	*/
154	DECLINLINE(int) pgmPoolPhysSimpleReadGCPhys(PVMCC pVM, void pvDst, void const pvSrc, RTGCPHYS GCPhysSrc, size_t cb)
155	{
156	#if defined(IN_RING3)
157	NOREF(pVM); NOREF(GCPhysSrc);
158	memcpy(pvDst, (RTHCPTR)((uintptr_t)pvSrc & ~(RTHCUINTPTR)(cb - 1)), cb);
159	return VINF_SUCCESS;
160	#else
161	/** @todo in RC we could attempt to use the virtual address, although this can cause many faults (PAE Windows XP guest). */
162	NOREF(pvSrc);
163	return PGMPhysSimpleReadGCPhys(pVM, pvDst, GCPhysSrc & ~(RTGCPHYS)(cb - 1), cb);
164	#endif
165	}
166
167
168	/**
169	* Process shadow entries before they are changed by the guest.
170	*
171	* For PT entries we will clear them. For PD entries, we'll simply check
172	* for mapping conflicts and set the SyncCR3 FF if found.
173	*
174	* @param pVCpu The cross context virtual CPU structure.
175	* @param pPool The pool.
176	* @param pPage The head page.
177	* @param GCPhysFault The guest physical fault address.
178	* @param pvAddress Pointer to the mapping of @a GCPhysFault or NULL
179	* depending on the context (e.g. \#PF in R0 & RC).
180	* @param cbWrite Write size; might be zero if the caller knows we're not crossing entry boundaries
181	*/
182	static void pgmPoolMonitorChainChanging(PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault,
183	void const *pvAddress, unsigned cbWrite)
184	{
185	AssertMsg(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX, ("%u (idx=%u)\n", pPage->iMonitoredPrev, pPage->idx));
186	const unsigned off = GCPhysFault & GUEST_PAGE_OFFSET_MASK;
187	PVMCC pVM = pPool->CTX_SUFF(pVM);
188	NOREF(pVCpu);
189
190	LogFlow(("pgmPoolMonitorChainChanging: %RGv phys=%RGp cbWrite=%d\n",
191	(RTGCPTR)(CTXTYPE(RTGCPTR, uintptr_t, RTGCPTR))(uintptr_t)pvAddress, GCPhysFault, cbWrite));
192
193	if (PGMPOOL_PAGE_IS_NESTED(pPage))
194	Log7Func(("%RGv phys=%RGp cbWrite=%d\n",
195	(RTGCPTR)(CTXTYPE(RTGCPTR, uintptr_t, RTGCPTR))(uintptr_t)pvAddress, GCPhysFault, cbWrite));
196
197	for (;;)
198	{
199	union
200	{
201	void *pv;
202	PX86PT pPT;
203	PPGMSHWPTPAE pPTPae;
204	PX86PD pPD;
205	PX86PDPAE pPDPae;
206	PX86PDPT pPDPT;
207	PX86PML4 pPML4;
208	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
209	PEPTPDPT pEptPdpt;
210	PEPTPD pEptPd;
211	PEPTPT pEptPt;
212	#endif
213	} uShw;
214
215	LogFlow(("pgmPoolMonitorChainChanging: page idx=%d phys=%RGp (next=%d) kind=%s write=%#x\n",
216	pPage->idx, pPage->GCPhys, pPage->iMonitoredNext, pgmPoolPoolKindToStr(pPage->enmKind), cbWrite));
217
218	uShw.pv = NULL;
219	switch (pPage->enmKind)
220	{
221	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
222	{
223	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT));
224	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
225	const unsigned iShw = off / sizeof(X86PTE);
226	LogFlow(("PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT iShw=%x\n", iShw));
227	X86PGUINT const uPde = uShw.pPT->a[iShw].u;
228	if (uPde & X86_PTE_P)
229	{
230	X86PTE GstPte;
231	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
232	AssertRC(rc);
233	Log4(("pgmPoolMonitorChainChanging 32_32: deref %016RX64 GCPhys %08RX32\n", uPde & X86_PTE_PG_MASK, GstPte.u & X86_PTE_PG_MASK));
234	pgmPoolTracDerefGCPhysHint(pPool, pPage, uPde & X86_PTE_PG_MASK, GstPte.u & X86_PTE_PG_MASK, iShw);
235	ASMAtomicWriteU32(&uShw.pPT->a[iShw].u, 0);
236	}
237	break;
238	}
239
240	/* page/2 sized */
241	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
242	{
243	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT));
244	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
245	if (!((off ^ pPage->GCPhys) & (PAGE_SIZE / 2)))
246	{
247	const unsigned iShw = (off / sizeof(X86PTE)) & (X86_PG_PAE_ENTRIES - 1);
248	LogFlow(("PGMPOOLKIND_PAE_PT_FOR_32BIT_PT iShw=%x\n", iShw));
249	if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw]))
250	{
251	X86PTE GstPte;
252	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
253	AssertRC(rc);
254
255	Log4(("pgmPoolMonitorChainChanging pae_32: deref %016RX64 GCPhys %08RX32\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PG_MASK));
256	pgmPoolTracDerefGCPhysHint(pPool, pPage,
257	PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw]),
258	GstPte.u & X86_PTE_PG_MASK,
259	iShw);
260	PGMSHWPTEPAE_ATOMIC_SET(uShw.pPTPae->a[iShw], 0);
261	}
262	}
263	break;
264	}
265
266	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
267	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
268	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
269	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
270	{
271	unsigned iGst = off / sizeof(X86PDE);
272	unsigned iShwPdpt = iGst / 256;
273	unsigned iShw = (iGst % 256) * 2;
274	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
275
276	LogFlow(("pgmPoolMonitorChainChanging PAE for 32 bits: iGst=%x iShw=%x idx = %d page idx=%d\n", iGst, iShw, iShwPdpt, pPage->enmKind - PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD));
277	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
278	if (iShwPdpt == pPage->enmKind - (unsigned)PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD)
279	{
280	for (unsigned i = 0; i < 2; i++)
281	{
282	X86PGPAEUINT const uPde = uShw.pPDPae->a[iShw + i].u;
283	if (uPde & X86_PDE_P)
284	{
285	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw + i, uPde));
286	pgmPoolFree(pVM, uPde & X86_PDE_PAE_PG_MASK, pPage->idx, iShw + i);
287	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw + i].u, 0);
288	}
289
290	/* paranoia / a bit assumptive. */
291	if ( (off & 3)
292	&& (off & 3) + cbWrite > 4)
293	{
294	const unsigned iShw2 = iShw + 2 + i;
295	if (iShw2 < RT_ELEMENTS(uShw.pPDPae->a))
296	{
297	X86PGPAEUINT const uPde2 = uShw.pPDPae->a[iShw2].u;
298	if (uPde2 & X86_PDE_P)
299	{
300	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uPde2));
301	pgmPoolFree(pVM, uPde2 & X86_PDE_PAE_PG_MASK, pPage->idx, iShw2);
302	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw2].u, 0);
303	}
304	}
305	}
306	}
307	}
308	break;
309	}
310
311	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
312	{
313	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
314	const unsigned iShw = off / sizeof(X86PTEPAE);
315	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT));
316	if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw]))
317	{
318	X86PTEPAE GstPte;
319	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
320	AssertRC(rc);
321
322	Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw]), GstPte.u & X86_PTE_PAE_PG_MASK));
323	pgmPoolTracDerefGCPhysHint(pPool, pPage,
324	PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw]),
325	GstPte.u & X86_PTE_PAE_PG_MASK,
326	iShw);
327	PGMSHWPTEPAE_ATOMIC_SET(uShw.pPTPae->a[iShw], 0);
328	}
329
330	/* paranoia / a bit assumptive. */
331	if ( (off & 7)
332	&& (off & 7) + cbWrite > sizeof(X86PTEPAE))
333	{
334	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTEPAE);
335	AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPTPae->a));
336
337	if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw2]))
338	{
339	X86PTEPAE GstPte;
340	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte,
341	pvAddress ? (uint8_t const *)pvAddress + sizeof(GstPte) : NULL,
342	GCPhysFault + sizeof(GstPte), sizeof(GstPte));
343	AssertRC(rc);
344	Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw2]), GstPte.u & X86_PTE_PAE_PG_MASK));
345	pgmPoolTracDerefGCPhysHint(pPool, pPage,
346	PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw2]),
347	GstPte.u & X86_PTE_PAE_PG_MASK,
348	iShw2);
349	PGMSHWPTEPAE_ATOMIC_SET(uShw.pPTPae->a[iShw2], 0);
350	}
351	}
352	break;
353	}
354
355	case PGMPOOLKIND_32BIT_PD:
356	{
357	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
358	const unsigned iShw = off / sizeof(X86PTE); // ASSUMING 32-bit guest paging!
359
360	LogFlow(("pgmPoolMonitorChainChanging: PGMPOOLKIND_32BIT_PD %x\n", iShw));
361	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
362	X86PGUINT const uPde = uShw.pPD->a[iShw].u;
363	if (uPde & X86_PDE_P)
364	{
365	LogFlow(("pgmPoolMonitorChainChanging: 32 bit pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uPde));
366	pgmPoolFree(pVM, uPde & X86_PDE_PG_MASK, pPage->idx, iShw);
367	ASMAtomicWriteU32(&uShw.pPD->a[iShw].u, 0);
368	}
369
370	/* paranoia / a bit assumptive. */
371	if ( (off & 3)
372	&& (off & 3) + cbWrite > sizeof(X86PTE))
373	{
374	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTE);
375	if ( iShw2 != iShw
376	&& iShw2 < RT_ELEMENTS(uShw.pPD->a))
377	{
378	X86PGUINT const uPde2 = uShw.pPD->a[iShw2].u;
379	if (uPde2 & X86_PDE_P)
380	{
381	LogFlow(("pgmPoolMonitorChainChanging: 32 bit pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uPde2));
382	pgmPoolFree(pVM, uPde2 & X86_PDE_PG_MASK, pPage->idx, iShw2);
383	ASMAtomicWriteU32(&uShw.pPD->a[iShw2].u, 0);
384	}
385	}
386	}
387	#if 0 /* useful when running PGMAssertCR3(), a bit too troublesome for general use (TLBs). - not working any longer... */
388	if ( uShw.pPD->a[iShw].n.u1Present
389	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
390	{
391	LogFlow(("pgmPoolMonitorChainChanging: iShw=%#x: %RX32 -> freeing it!\n", iShw, uShw.pPD->a[iShw].u));
392	pgmPoolFree(pVM, uShw.pPD->a[iShw].u & X86_PDE_PG_MASK, pPage->idx, iShw);
393	ASMAtomicWriteU32(&uShw.pPD->a[iShw].u, 0);
394	}
395	#endif
396	break;
397	}
398
399	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
400	{
401	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
402	const unsigned iShw = off / sizeof(X86PDEPAE);
403	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
404
405	/*
406	* Causes trouble when the guest uses a PDE to refer to the whole page table level
407	* structure. (Invalidate here; faults later on when it tries to change the page
408	* table entries -> recheck; probably only applies to the RC case.)
409	*/
410	X86PGPAEUINT const uPde = uShw.pPDPae->a[iShw].u;
411	if (uPde & X86_PDE_P)
412	{
413	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uPde));
414	pgmPoolFree(pVM, uPde & X86_PDE_PAE_PG_MASK, pPage->idx, iShw);
415	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw].u, 0);
416	}
417
418	/* paranoia / a bit assumptive. */
419	if ( (off & 7)
420	&& (off & 7) + cbWrite > sizeof(X86PDEPAE))
421	{
422	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE);
423	AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPDPae->a));
424
425	X86PGPAEUINT const uPde2 = uShw.pPDPae->a[iShw2].u;
426	if (uPde2 & X86_PDE_P)
427	{
428	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPde2));
429	pgmPoolFree(pVM, uPde2 & X86_PDE_PAE_PG_MASK, pPage->idx, iShw2);
430	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw2].u, 0);
431	}
432	}
433	break;
434	}
435
436	case PGMPOOLKIND_PAE_PDPT:
437	{
438	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPDPT));
439	/*
440	* Hopefully this doesn't happen very often:
441	* - touching unused parts of the page
442	* - messing with the bits of pd pointers without changing the physical address
443	*/
444	/* PDPT roots are not page aligned; 32 byte only! */
445	const unsigned offPdpt = GCPhysFault - pPage->GCPhys;
446
447	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
448	const unsigned iShw = offPdpt / sizeof(X86PDPE);
449	if (iShw < X86_PG_PAE_PDPE_ENTRIES) /* don't use RT_ELEMENTS(uShw.pPDPT->a), because that's for long mode only */
450	{
451	X86PGPAEUINT const uPdpe = uShw.pPDPT->a[iShw].u;
452	if (uPdpe & X86_PDPE_P)
453	{
454	LogFlow(("pgmPoolMonitorChainChanging: pae pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPT->a[iShw].u));
455	pgmPoolFree(pVM, uPdpe & X86_PDPE_PG_MASK, pPage->idx, iShw);
456	ASMAtomicWriteU64(&uShw.pPDPT->a[iShw].u, 0);
457	}
458
459	/* paranoia / a bit assumptive. */
460	if ( (offPdpt & 7)
461	&& (offPdpt & 7) + cbWrite > sizeof(X86PDPE))
462	{
463	const unsigned iShw2 = (offPdpt + cbWrite - 1) / sizeof(X86PDPE);
464	if ( iShw2 != iShw
465	&& iShw2 < X86_PG_PAE_PDPE_ENTRIES)
466	{
467	X86PGPAEUINT const uPdpe2 = uShw.pPDPT->a[iShw2].u;
468	if (uPdpe2 & X86_PDPE_P)
469	{
470	LogFlow(("pgmPoolMonitorChainChanging: pae pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPT->a[iShw2].u));
471	pgmPoolFree(pVM, uPdpe2 & X86_PDPE_PG_MASK, pPage->idx, iShw2);
472	ASMAtomicWriteU64(&uShw.pPDPT->a[iShw2].u, 0);
473	}
474	}
475	}
476	}
477	break;
478	}
479
480	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
481	{
482	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
483	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
484	const unsigned iShw = off / sizeof(X86PDEPAE);
485	X86PGPAEUINT const uPde = uShw.pPDPae->a[iShw].u;
486	if (uPde & X86_PDE_P)
487	{
488	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uPde));
489	pgmPoolFree(pVM, uPde & X86_PDE_PAE_PG_MASK, pPage->idx, iShw);
490	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw].u, 0);
491	}
492
493	/* paranoia / a bit assumptive. */
494	if ( (off & 7)
495	&& (off & 7) + cbWrite > sizeof(X86PDEPAE))
496	{
497	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE);
498	AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPDPae->a));
499	X86PGPAEUINT const uPde2 = uShw.pPDPae->a[iShw2].u;
500	if (uPde2 & X86_PDE_P)
501	{
502	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPde2));
503	pgmPoolFree(pVM, uPde2 & X86_PDE_PAE_PG_MASK, pPage->idx, iShw2);
504	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw2].u, 0);
505	}
506	}
507	break;
508	}
509
510	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
511	{
512	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPDPT));
513	/*
514	* Hopefully this doesn't happen very often:
515	* - messing with the bits of pd pointers without changing the physical address
516	*/
517	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
518	const unsigned iShw = off / sizeof(X86PDPE);
519	X86PGPAEUINT const uPdpe = uShw.pPDPT->a[iShw].u;
520	if (uPdpe & X86_PDPE_P)
521	{
522	LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uPdpe));
523	pgmPoolFree(pVM, uPdpe & X86_PDPE_PG_MASK, pPage->idx, iShw);
524	ASMAtomicWriteU64(&uShw.pPDPT->a[iShw].u, 0);
525	}
526	/* paranoia / a bit assumptive. */
527	if ( (off & 7)
528	&& (off & 7) + cbWrite > sizeof(X86PDPE))
529	{
530	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDPE);
531	X86PGPAEUINT const uPdpe2 = uShw.pPDPT->a[iShw2].u;
532	if (uPdpe2 & X86_PDPE_P)
533	{
534	LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPdpe2));
535	pgmPoolFree(pVM, uPdpe2 & X86_PDPE_PG_MASK, pPage->idx, iShw2);
536	ASMAtomicWriteU64(&uShw.pPDPT->a[iShw2].u, 0);
537	}
538	}
539	break;
540	}
541
542	case PGMPOOLKIND_64BIT_PML4:
543	{
544	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPML4));
545	/*
546	* Hopefully this doesn't happen very often:
547	* - messing with the bits of pd pointers without changing the physical address
548	*/
549	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
550	const unsigned iShw = off / sizeof(X86PDPE);
551	X86PGPAEUINT const uPml4e = uShw.pPML4->a[iShw].u;
552	if (uPml4e & X86_PML4E_P)
553	{
554	LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw=%#x: %RX64 -> freeing it!\n", iShw, uPml4e));
555	pgmPoolFree(pVM, uPml4e & X86_PML4E_PG_MASK, pPage->idx, iShw);
556	ASMAtomicWriteU64(&uShw.pPML4->a[iShw].u, 0);
557	}
558	/* paranoia / a bit assumptive. */
559	if ( (off & 7)
560	&& (off & 7) + cbWrite > sizeof(X86PDPE))
561	{
562	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PML4E);
563	X86PGPAEUINT const uPml4e2 = uShw.pPML4->a[iShw2].u;
564	if (uPml4e2 & X86_PML4E_P)
565	{
566	LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPml4e2));
567	pgmPoolFree(pVM, uPml4e2 & X86_PML4E_PG_MASK, pPage->idx, iShw2);
568	ASMAtomicWriteU64(&uShw.pPML4->a[iShw2].u, 0);
569	}
570	}
571	break;
572	}
573
574	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
575	case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
576	{
577	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
578	const unsigned iShw = off / sizeof(EPTPML4E);
579	X86PGPAEUINT const uPml4e = uShw.pPML4->a[iShw].u;
580	if (uPml4e & EPT_PRESENT_MASK)
581	{
582	Log7Func(("PML4 iShw=%#x: %RX64 (%RGp) -> freeing it!\n", iShw, uPml4e, pPage->GCPhys));
583	pgmPoolFree(pVM, uPml4e & X86_PML4E_PG_MASK, pPage->idx, iShw);
584	ASMAtomicWriteU64(&uShw.pPML4->a[iShw].u, 0);
585	}
586
587	/* paranoia / a bit assumptive. */
588	if ( (off & 7)
589	&& (off & 7) + cbWrite > sizeof(X86PML4E))
590	{
591	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PML4E);
592	X86PGPAEUINT const uPml4e2 = uShw.pPML4->a[iShw2].u;
593	if (uPml4e2 & EPT_PRESENT_MASK)
594	{
595	Log7Func(("PML4 iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPml4e2));
596	pgmPoolFree(pVM, uPml4e2 & X86_PML4E_PG_MASK, pPage->idx, iShw2);
597	ASMAtomicWriteU64(&uShw.pPML4->a[iShw2].u, 0);
598	}
599	}
600	break;
601	}
602
603	case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
604	{
605	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
606	const unsigned iShw = off / sizeof(EPTPDPTE);
607	X86PGPAEUINT const uPdpte = uShw.pEptPdpt->a[iShw].u;
608	if (uPdpte & EPT_PRESENT_MASK)
609	{
610	Log7Func(("EPT PDPT iShw=%#x: %RX64 (%RGp) -> freeing it!\n", iShw, uPdpte, pPage->GCPhys));
611	pgmPoolFree(pVM, uPdpte & EPT_PDPTE_PG_MASK, pPage->idx, iShw);
612	ASMAtomicWriteU64(&uShw.pEptPdpt->a[iShw].u, 0);
613	}
614
615	/* paranoia / a bit assumptive. */
616	if ( (off & 7)
617	&& (off & 7) + cbWrite > sizeof(EPTPDPTE))
618	{
619	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(EPTPDPTE);
620	X86PGPAEUINT const uPdpte2 = uShw.pEptPdpt->a[iShw2].u;
621	if (uPdpte2 & EPT_PRESENT_MASK)
622	{
623	Log7Func(("EPT PDPT iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPdpte2));
624	pgmPoolFree(pVM, uPdpte2 & EPT_PDPTE_PG_MASK, pPage->idx, iShw2);
625	ASMAtomicWriteU64(&uShw.pEptPdpt->a[iShw2].u, 0);
626	}
627	}
628	break;
629	}
630
631	case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
632	{
633	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
634	const unsigned iShw = off / sizeof(EPTPDE);
635	X86PGPAEUINT const uPde = uShw.pEptPd->a[iShw].u;
636	if (uPde & EPT_PRESENT_MASK)
637	{
638	Log7Func(("EPT PD iShw=%#x: %RX64 (%RGp) -> freeing it!\n", iShw, uPde, pPage->GCPhys));
639	pgmPoolFree(pVM, uPde & EPT_PDE_PG_MASK, pPage->idx, iShw);
640	ASMAtomicWriteU64(&uShw.pEptPd->a[iShw].u, 0);
641	}
642
643	/* paranoia / a bit assumptive. */
644	if ( (off & 7)
645	&& (off & 7) + cbWrite > sizeof(EPTPDE))
646	{
647	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(EPTPDE);
648	AssertBreak(iShw2 < RT_ELEMENTS(uShw.pEptPd->a));
649	X86PGPAEUINT const uPde2 = uShw.pEptPd->a[iShw2].u;
650	if (uPde2 & EPT_PRESENT_MASK)
651	{
652	Log7Func(("EPT PD (2): iShw2=%#x: %RX64 (%RGp) -> freeing it!\n", iShw2, uPde2, pPage->GCPhys));
653	pgmPoolFree(pVM, uPde2 & EPT_PDE_PG_MASK, pPage->idx, iShw2);
654	ASMAtomicWriteU64(&uShw.pEptPd->a[iShw2].u, 0);
655	}
656	}
657	break;
658	}
659
660	case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
661	{
662	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
663	const unsigned iShw = off / sizeof(EPTPTE);
664	X86PGPAEUINT const uPte = uShw.pEptPt->a[iShw].u;
665	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT));
666	if (uPte & EPT_PRESENT_MASK)
667	{
668	EPTPTE GstPte;
669	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
670	AssertRC(rc);
671
672	Log7Func(("EPT PT: iShw=%#x %RX64 (%RGp)\n", iShw, uPte, pPage->GCPhys));
673	pgmPoolTracDerefGCPhysHint(pPool, pPage,
674	uShw.pEptPt->a[iShw].u & EPT_PTE_PG_MASK,
675	GstPte.u & EPT_PTE_PG_MASK,
676	iShw);
677	ASMAtomicWriteU64(&uShw.pEptPt->a[iShw].u, 0);
678	}
679
680	/* paranoia / a bit assumptive. */
681	if ( (off & 7)
682	&& (off & 7) + cbWrite > sizeof(EPTPTE))
683	{
684	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(EPTPTE);
685	AssertBreak(iShw2 < RT_ELEMENTS(uShw.pEptPt->a));
686	X86PGPAEUINT const uPte2 = uShw.pEptPt->a[iShw2].u;
687	if (uPte2 & EPT_PRESENT_MASK)
688	{
689	EPTPTE GstPte;
690	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte,
691	pvAddress ? (uint8_t const *)pvAddress + sizeof(GstPte) : NULL,
692	GCPhysFault + sizeof(GstPte), sizeof(GstPte));
693	AssertRC(rc);
694	Log7Func(("EPT PT (2): iShw=%#x %RX64 (%RGp)\n", iShw2, uPte2, pPage->GCPhys));
695	pgmPoolTracDerefGCPhysHint(pPool, pPage,
696	uShw.pEptPt->a[iShw2].u & EPT_PTE_PG_MASK,
697	GstPte.u & EPT_PTE_PG_MASK,
698	iShw2);
699	ASMAtomicWriteU64(&uShw.pEptPt->a[iShw2].u, 0);
700	}
701	}
702	break;
703	}
704	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
705
706	default:
707	AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
708	}
709	PGM_DYNMAP_UNUSED_HINT_VM(pVM, uShw.pv);
710
711	/* next */
712	if (pPage->iMonitoredNext == NIL_PGMPOOL_IDX)
713	return;
714	pPage = &pPool->aPages[pPage->iMonitoredNext];
715	}
716	}
717
718	#ifndef IN_RING3
719
720	/**
721	* Checks if a access could be a fork operation in progress.
722	*
723	* Meaning, that the guest is setting up the parent process for Copy-On-Write.
724	*
725	* @returns true if it's likely that we're forking, otherwise false.
726	* @param pPool The pool.
727	* @param pDis The disassembled instruction.
728	* @param offFault The access offset.
729	*/
730	DECLINLINE(bool) pgmRZPoolMonitorIsForking(PPGMPOOL pPool, PDISCPUSTATE pDis, unsigned offFault)
731	{
732	/*
733	* i386 linux is using btr to clear X86_PTE_RW.
734	* The functions involved are (2.6.16 source inspection):
735	* clear_bit
736	* ptep_set_wrprotect
737	* copy_one_pte
738	* copy_pte_range
739	* copy_pmd_range
740	* copy_pud_range
741	* copy_page_range
742	* dup_mmap
743	* dup_mm
744	* copy_mm
745	* copy_process
746	* do_fork
747	*/
748	if ( pDis->pCurInstr->uOpcode == OP_BTR
749	&& !(offFault & 4)
750	/** @todo Validate that the bit index is X86_PTE_RW. */
751	)
752	{
753	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,Fork)); RT_NOREF_PV(pPool);
754	return true;
755	}
756	return false;
757	}
758
759
760	/**
761	* Determine whether the page is likely to have been reused.
762	*
763	* @returns true if we consider the page as being reused for a different purpose.
764	* @returns false if we consider it to still be a paging page.
765	* @param pVM The cross context VM structure.
766	* @param pVCpu The cross context virtual CPU structure.
767	* @param pRegFrame Trap register frame.
768	* @param pDis The disassembly info for the faulting instruction.
769	* @param pvFault The fault address.
770	* @param pPage The pool page being accessed.
771	*
772	* @remark The REP prefix check is left to the caller because of STOSD/W.
773	*/
774	DECLINLINE(bool) pgmRZPoolMonitorIsReused(PVMCC pVM, PVMCPUCC pVCpu, PCPUMCTXCORE pRegFrame, PDISCPUSTATE pDis, RTGCPTR pvFault,
775	PPGMPOOLPAGE pPage)
776	{
777	/* Locked (CR3, PDPTR*4) should not be reusable. Considering them as
778	such may cause loops booting tst-ubuntu-15_10-64-efi, ++. */
779	if (pPage->cLocked)
780	{
781	Log2(("pgmRZPoolMonitorIsReused: %RGv (%p) can't have been resued, because it's locked!\n", pvFault, pPage));
782	return false;
783	}
784
785	/** @todo could make this general, faulting close to rsp should be a safe reuse heuristic. */
786	if ( HMHasPendingIrq(pVM)
787	&& pRegFrame->rsp - pvFault < 32)
788	{
789	/* Fault caused by stack writes while trying to inject an interrupt event. */
790	Log(("pgmRZPoolMonitorIsReused: reused %RGv for interrupt stack (rsp=%RGv).\n", pvFault, pRegFrame->rsp));
791	return true;
792	}
793
794	LogFlow(("Reused instr %RGv %d at %RGv param1.fUse=%llx param1.reg=%d\n", pRegFrame->rip, pDis->pCurInstr->uOpcode, pvFault, pDis->Param1.fUse, pDis->Param1.Base.idxGenReg));
795
796	/* Non-supervisor mode write means it's used for something else. */
797	if (CPUMGetGuestCPL(pVCpu) == 3)
798	return true;
799
800	switch (pDis->pCurInstr->uOpcode)
801	{
802	/* call implies the actual push of the return address faulted */
803	case OP_CALL:
804	Log4(("pgmRZPoolMonitorIsReused: CALL\n"));
805	return true;
806	case OP_PUSH:
807	Log4(("pgmRZPoolMonitorIsReused: PUSH\n"));
808	return true;
809	case OP_PUSHF:
810	Log4(("pgmRZPoolMonitorIsReused: PUSHF\n"));
811	return true;
812	case OP_PUSHA:
813	Log4(("pgmRZPoolMonitorIsReused: PUSHA\n"));
814	return true;
815	case OP_FXSAVE:
816	Log4(("pgmRZPoolMonitorIsReused: FXSAVE\n"));
817	return true;
818	case OP_MOVNTI: /* solaris - block_zero_no_xmm */
819	Log4(("pgmRZPoolMonitorIsReused: MOVNTI\n"));
820	return true;
821	case OP_MOVNTDQ: /* solaris - hwblkclr & hwblkpagecopy */
822	Log4(("pgmRZPoolMonitorIsReused: MOVNTDQ\n"));
823	return true;
824	case OP_MOVSWD:
825	case OP_STOSWD:
826	if ( pDis->fPrefix == (DISPREFIX_REP\|DISPREFIX_REX)
827	&& pRegFrame->rcx >= 0x40
828	)
829	{
830	Assert(pDis->uCpuMode == DISCPUMODE_64BIT);
831
832	Log(("pgmRZPoolMonitorIsReused: OP_STOSQ\n"));
833	return true;
834	}
835	break;
836
837	default:
838	/*
839	* Anything having ESP on the left side means stack writes.
840	*/
841	if ( ( (pDis->Param1.fUse & DISUSE_REG_GEN32)
842	\|\| (pDis->Param1.fUse & DISUSE_REG_GEN64))
843	&& (pDis->Param1.Base.idxGenReg == DISGREG_ESP))
844	{
845	Log4(("pgmRZPoolMonitorIsReused: ESP\n"));
846	return true;
847	}
848	break;
849	}
850
851	/*
852	* Page table updates are very very unlikely to be crossing page boundraries,
853	* and we don't want to deal with that in pgmPoolMonitorChainChanging and such.
854	*/
855	uint32_t const cbWrite = DISGetParamSize(pDis, &pDis->Param1);
856	if ( (((uintptr_t)pvFault + cbWrite) >> X86_PAGE_SHIFT) != ((uintptr_t)pvFault >> X86_PAGE_SHIFT) )
857	{
858	Log4(("pgmRZPoolMonitorIsReused: cross page write\n"));
859	return true;
860	}
861
862	/*
863	* Nobody does an unaligned 8 byte write to a page table, right.
864	*/
865	if (cbWrite >= 8 && ((uintptr_t)pvFault & 7) != 0)
866	{
867	Log4(("pgmRZPoolMonitorIsReused: Unaligned 8+ byte write\n"));
868	return true;
869	}
870
871	return false;
872	}
873
874
875	/**
876	* Flushes the page being accessed.
877	*
878	* @returns VBox status code suitable for scheduling.
879	* @param pVM The cross context VM structure.
880	* @param pVCpu The cross context virtual CPU structure.
881	* @param pPool The pool.
882	* @param pPage The pool page (head).
883	* @param pDis The disassembly of the write instruction.
884	* @param pRegFrame The trap register frame.
885	* @param GCPhysFault The fault address as guest physical address.
886	* @param pvFault The fault address.
887	* @todo VBOXSTRICTRC
888	*/
889	static int pgmRZPoolAccessPfHandlerFlush(PVMCC pVM, PVMCPUCC pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
890	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
891	{
892	NOREF(pVM); NOREF(GCPhysFault);
893
894	/*
895	* First, do the flushing.
896	*/
897	pgmPoolMonitorChainFlush(pPool, pPage);
898
899	/*
900	* Emulate the instruction (xp/w2k problem, requires pc/cr2/sp detection).
901	* Must do this in raw mode (!); XP boot will fail otherwise.
902	*/
903	int rc = VINF_SUCCESS;
904	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, pRegFrame, pvFault, EMCODETYPE_ALL);
905	if (rc2 == VINF_SUCCESS)
906	{ /* do nothing */ }
907	else if (rc2 == VINF_EM_RESCHEDULE)
908	{
909	rc = VBOXSTRICTRC_VAL(rc2);
910	# ifndef IN_RING3
911	VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
912	# endif
913	}
914	else if (rc2 == VERR_EM_INTERPRETER)
915	{
916	rc = VINF_EM_RAW_EMULATE_INSTR;
917	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,EmulateInstr));
918	}
919	else if (RT_FAILURE_NP(rc2))
920	rc = VBOXSTRICTRC_VAL(rc2);
921	else
922	AssertMsgFailed(("%Rrc\n", VBOXSTRICTRC_VAL(rc2))); /* ASSUMES no complicated stuff here. */
923
924	LogFlow(("pgmRZPoolAccessPfHandlerFlush: returns %Rrc (flushed)\n", rc));
925	return rc;
926	}
927
928
929	/**
930	* Handles the STOSD write accesses.
931	*
932	* @returns VBox status code suitable for scheduling.
933	* @param pVM The cross context VM structure.
934	* @param pPool The pool.
935	* @param pPage The pool page (head).
936	* @param pDis The disassembly of the write instruction.
937	* @param pRegFrame The trap register frame.
938	* @param GCPhysFault The fault address as guest physical address.
939	* @param pvFault The fault address.
940	*/
941	DECLINLINE(int) pgmRZPoolAccessPfHandlerSTOSD(PVMCC pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
942	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
943	{
944	unsigned uIncrement = pDis->Param1.cb;
945	NOREF(pVM);
946
947	Assert(pDis->uCpuMode == DISCPUMODE_32BIT \|\| pDis->uCpuMode == DISCPUMODE_64BIT);
948	Assert(pRegFrame->rcx <= 0x20);
949
950	# ifdef VBOX_STRICT
951	if (pDis->uOpMode == DISCPUMODE_32BIT)
952	Assert(uIncrement == 4);
953	else
954	Assert(uIncrement == 8);
955	# endif
956
957	Log3(("pgmRZPoolAccessPfHandlerSTOSD\n"));
958
959	/*
960	* Increment the modification counter and insert it into the list
961	* of modified pages the first time.
962	*/
963	if (!pPage->cModifications++)
964	pgmPoolMonitorModifiedInsert(pPool, pPage);
965
966	/*
967	* Execute REP STOSD.
968	*
969	* This ASSUMES that we're not invoked by Trap0e on in a out-of-sync
970	* write situation, meaning that it's safe to write here.
971	*/
972	PVMCPUCC pVCpu = VMMGetCpu(pPool->CTX_SUFF(pVM));
973	RTGCUINTPTR pu32 = (RTGCUINTPTR)pvFault;
974	while (pRegFrame->rcx)
975	{
976	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, uIncrement);
977	PGMPhysSimpleWriteGCPhys(pVM, GCPhysFault, &pRegFrame->rax, uIncrement);
978	pu32 += uIncrement;
979	GCPhysFault += uIncrement;
980	pRegFrame->rdi += uIncrement;
981	pRegFrame->rcx--;
982	}
983	pRegFrame->rip += pDis->cbInstr;
984
985	LogFlow(("pgmRZPoolAccessPfHandlerSTOSD: returns\n"));
986	return VINF_SUCCESS;
987	}
988
989
990	/**
991	* Handles the simple write accesses.
992	*
993	* @returns VBox status code suitable for scheduling.
994	* @param pVM The cross context VM structure.
995	* @param pVCpu The cross context virtual CPU structure.
996	* @param pPool The pool.
997	* @param pPage The pool page (head).
998	* @param pDis The disassembly of the write instruction.
999	* @param pRegFrame The trap register frame.
1000	* @param GCPhysFault The fault address as guest physical address.
1001	* @param pvFault The fault address.
1002	* @param pfReused Reused state (in/out)
1003	*/
1004	DECLINLINE(int) pgmRZPoolAccessPfHandlerSimple(PVMCC pVM, PVMCPUCC pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
1005	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault, bool *pfReused)
1006	{
1007	Log3(("pgmRZPoolAccessPfHandlerSimple\n"));
1008	NOREF(pVM);
1009	NOREF(pfReused); /* initialized by caller */
1010
1011	/*
1012	* Increment the modification counter and insert it into the list
1013	* of modified pages the first time.
1014	*/
1015	if (!pPage->cModifications++)
1016	pgmPoolMonitorModifiedInsert(pPool, pPage);
1017
1018	/*
1019	* Clear all the pages. ASSUMES that pvFault is readable.
1020	*/
1021	uint32_t cbWrite = DISGetParamSize(pDis, &pDis->Param1);
1022	if (cbWrite <= 8)
1023	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, cbWrite);
1024	else if (cbWrite <= 16)
1025	{
1026	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, 8);
1027	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault + 8, NULL, cbWrite - 8);
1028	}
1029	else
1030	{
1031	Assert(cbWrite <= 32);
1032	for (uint32_t off = 0; off < cbWrite; off += 8)
1033	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault + off, NULL, RT_MIN(8, cbWrite - off));
1034	}
1035
1036	/*
1037	* Interpret the instruction.
1038	*/
1039	VBOXSTRICTRC rc = EMInterpretInstructionDisasState(pVCpu, pDis, pRegFrame, pvFault, EMCODETYPE_ALL);
1040	if (RT_SUCCESS(rc))
1041	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", VBOXSTRICTRC_VAL(rc))); /* ASSUMES no complicated stuff here. */
1042	else if (rc == VERR_EM_INTERPRETER)
1043	{
1044	LogFlow(("pgmRZPoolAccessPfHandlerSimple: Interpretation failed for %04x:%RGv - opcode=%d\n",
1045	pRegFrame->cs.Sel, (RTGCPTR)pRegFrame->rip, pDis->pCurInstr->uOpcode));
1046	rc = VINF_EM_RAW_EMULATE_INSTR;
1047	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,EmulateInstr));
1048	}
1049
1050	# if 0 /* experimental code */
1051	if (rc == VINF_SUCCESS)
1052	{
1053	switch (pPage->enmKind)
1054	{
1055	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1056	{
1057	X86PTEPAE GstPte;
1058	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvFault, GCPhysFault, sizeof(GstPte));
1059	AssertRC(rc);
1060
1061	/* Check the new value written by the guest. If present and with a bogus physical address, then
1062	* it's fairly safe to assume the guest is reusing the PT.
1063	*/
1064	if (GstPte.n.u1Present)
1065	{
1066	RTHCPHYS HCPhys = -1;
1067	int rc = PGMPhysGCPhys2HCPhys(pVM, GstPte.u & X86_PTE_PAE_PG_MASK, &HCPhys);
1068	if (rc != VINF_SUCCESS)
1069	{
1070	*pfReused = true;
1071	STAM_COUNTER_INC(&pPool->StatForceFlushReused);
1072	}
1073	}
1074	break;
1075	}
1076	}
1077	}
1078	# endif
1079
1080	LogFlow(("pgmRZPoolAccessPfHandlerSimple: returns %Rrc\n", VBOXSTRICTRC_VAL(rc)));
1081	return VBOXSTRICTRC_VAL(rc);
1082	}
1083
1084
1085	/**
1086	* @callback_method_impl{FNPGMRZPHYSPFHANDLER,
1087	* \#PF access handler callback for page table pages.}
1088	*
1089	* @remarks The @a uUser argument is the index of the PGMPOOLPAGE.
1090	*/
1091	DECLCALLBACK(VBOXSTRICTRC) pgmRZPoolAccessPfHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame,
1092	RTGCPTR pvFault, RTGCPHYS GCPhysFault, uint64_t uUser)
1093	{
1094	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorRZ, a);
1095	PPGMPOOL const pPool = pVM->pgm.s.CTX_SUFF(pPool);
1096	AssertReturn(uUser < pPool->cCurPages, VERR_PGM_POOL_IPE);
1097	PPGMPOOLPAGE const pPage = &pPool->aPages[uUser];
1098	unsigned cMaxModifications;
1099	bool fForcedFlush = false;
1100	RT_NOREF_PV(uErrorCode);
1101
1102	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1103	AssertMsg(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_DIRECT,
1104	("pvFault=%RGv pPage=%p:{.idx=%d} GCPhysFault=%RGp\n", pvFault, pPage, pPage->idx, GCPhysFault));
1105	# endif
1106	LogFlow(("pgmRZPoolAccessPfHandler: pvFault=%RGv pPage=%p:{.idx=%d} GCPhysFault=%RGp\n", pvFault, pPage, pPage->idx, GCPhysFault));
1107
1108	PGM_LOCK_VOID(pVM);
1109	if (PHYS_PAGE_ADDRESS(GCPhysFault) != PHYS_PAGE_ADDRESS(pPage->GCPhys))
1110	{
1111	/* Pool page changed while we were waiting for the lock; ignore. */
1112	Log(("CPU%d: pgmRZPoolAccessPfHandler pgm pool page for %RGp changed (to %RGp) while waiting!\n", pVCpu->idCpu, PHYS_PAGE_ADDRESS(GCPhysFault), PHYS_PAGE_ADDRESS(pPage->GCPhys)));
1113	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZHandled, a);
1114	PGM_UNLOCK(pVM);
1115	return VINF_SUCCESS;
1116	}
1117	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1118	if (pPage->fDirty)
1119	{
1120	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1121	Assert(!PGMPOOL_PAGE_IS_NESTED(pPage));
1122	# endif
1123	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH));
1124	PGM_UNLOCK(pVM);
1125	return VINF_SUCCESS; /* SMP guest case where we were blocking on the pgm lock while the same page was being marked dirty. */
1126	}
1127	# endif
1128
1129	# if 0 /* test code defined(VBOX_STRICT) && defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) */
1130	if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
1131	{
1132	void *pvShw = PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
1133	void *pvGst;
1134	int rc = PGM_GCPHYS_2_PTR(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
1135	pgmPoolTrackCheckPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst);
1136	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
1137	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
1138	}
1139	# endif
1140
1141	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1142	if (PGMPOOL_PAGE_IS_NESTED(pPage))
1143	{
1144	Assert(!CPUMIsGuestInVmxNonRootMode(CPUMQueryGuestCtxPtr(pVCpu)));
1145	Log7Func(("Flushing pvFault=%RGv GCPhysFault=%RGp\n", pvFault, GCPhysFault));
1146	pgmPoolMonitorChainFlush(pPool, pPage);
1147	PGM_UNLOCK(pVM);
1148	return VINF_SUCCESS;
1149	}
1150	# endif
1151
1152	/*
1153	* Disassemble the faulting instruction.
1154	*/
1155	PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
1156	int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
1157	if (RT_UNLIKELY(rc != VINF_SUCCESS))
1158	{
1159	AssertMsg(rc == VERR_PAGE_NOT_PRESENT \|\| rc == VERR_PAGE_TABLE_NOT_PRESENT, ("Unexpected rc %d\n", rc));
1160	PGM_UNLOCK(pVM);
1161	return rc;
1162	}
1163
1164	Assert(pPage->enmKind != PGMPOOLKIND_FREE);
1165
1166	/*
1167	* We should ALWAYS have the list head as user parameter. This
1168	* is because we use that page to record the changes.
1169	*/
1170	Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1171
1172	# ifdef IN_RING0
1173	/* Maximum nr of modifications depends on the page type. */
1174	if ( pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT
1175	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
1176	cMaxModifications = 4;
1177	else
1178	cMaxModifications = 24;
1179	# else
1180	cMaxModifications = 48;
1181	# endif
1182
1183	/*
1184	* Incremental page table updates should weigh more than random ones.
1185	* (Only applies when started from offset 0)
1186	*/
1187	pVCpu->pgm.s.cPoolAccessHandler++;
1188	if ( pPage->GCPtrLastAccessHandlerRip >= pRegFrame->rip - 0x40 /* observed loops in Windows 7 x64 */
1189	&& pPage->GCPtrLastAccessHandlerRip < pRegFrame->rip + 0x40
1190	&& pvFault == (pPage->GCPtrLastAccessHandlerFault + pDis->Param1.cb)
1191	&& pVCpu->pgm.s.cPoolAccessHandler == pPage->cLastAccessHandler + 1)
1192	{
1193	Log(("Possible page reuse cMods=%d -> %d (locked=%d type=%s)\n", pPage->cModifications, pPage->cModifications * 2, pgmPoolIsPageLocked(pPage), pgmPoolPoolKindToStr(pPage->enmKind)));
1194	Assert(pPage->cModifications < 32000);
1195	pPage->cModifications = pPage->cModifications * 2;
1196	pPage->GCPtrLastAccessHandlerFault = pvFault;
1197	pPage->cLastAccessHandler = pVCpu->pgm.s.cPoolAccessHandler;
1198	if (pPage->cModifications >= cMaxModifications)
1199	{
1200	STAM_COUNTER_INC(&pPool->StatMonitorPfRZFlushReinit);
1201	fForcedFlush = true;
1202	}
1203	}
1204
1205	if (pPage->cModifications >= cMaxModifications)
1206	Log(("Mod overflow %RGv cMods=%d (locked=%d type=%s)\n", pvFault, pPage->cModifications, pgmPoolIsPageLocked(pPage), pgmPoolPoolKindToStr(pPage->enmKind)));
1207
1208	/*
1209	* Check if it's worth dealing with.
1210	*/
1211	bool fReused = false;
1212	bool fNotReusedNotForking = false;
1213	if ( ( pPage->cModifications < cMaxModifications /** @todo \#define / /* @todo need to check that it's not mapping EIP. / /* @todo adjust this! */
1214	\|\| pgmPoolIsPageLocked(pPage)
1215	)
1216	&& !(fReused = pgmRZPoolMonitorIsReused(pVM, pVCpu, pRegFrame, pDis, pvFault, pPage))
1217	&& !pgmRZPoolMonitorIsForking(pPool, pDis, GCPhysFault & PAGE_OFFSET_MASK))
1218	{
1219	/*
1220	* Simple instructions, no REP prefix.
1221	*/
1222	if (!(pDis->fPrefix & (DISPREFIX_REP \| DISPREFIX_REPNE)))
1223	{
1224	rc = pgmRZPoolAccessPfHandlerSimple(pVM, pVCpu, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault, &fReused);
1225	if (fReused)
1226	goto flushPage;
1227
1228	/* A mov instruction to change the first page table entry will be remembered so we can detect
1229	* full page table changes early on. This will reduce the amount of unnecessary traps we'll take.
1230	*/
1231	if ( rc == VINF_SUCCESS
1232	&& !pPage->cLocked /* only applies to unlocked pages as we can't free locked ones (e.g. cr3 root). */
1233	&& pDis->pCurInstr->uOpcode == OP_MOV
1234	&& (pvFault & PAGE_OFFSET_MASK) == 0)
1235	{
1236	pPage->GCPtrLastAccessHandlerFault = pvFault;
1237	pPage->cLastAccessHandler = pVCpu->pgm.s.cPoolAccessHandler;
1238	pPage->GCPtrLastAccessHandlerRip = pRegFrame->rip;
1239	/* Make sure we don't kick out a page too quickly. */
1240	if (pPage->cModifications > 8)
1241	pPage->cModifications = 2;
1242	}
1243	else if (pPage->GCPtrLastAccessHandlerFault == pvFault)
1244	{
1245	/* ignore the 2nd write to this page table entry. */
1246	pPage->cLastAccessHandler = pVCpu->pgm.s.cPoolAccessHandler;
1247	}
1248	else
1249	{
1250	pPage->GCPtrLastAccessHandlerFault = NIL_RTGCPTR;
1251	pPage->GCPtrLastAccessHandlerRip = 0;
1252	}
1253
1254	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZHandled, a);
1255	PGM_UNLOCK(pVM);
1256	return rc;
1257	}
1258
1259	/*
1260	* Windows is frequently doing small memset() operations (netio test 4k+).
1261	* We have to deal with these or we'll kill the cache and performance.
1262	*/
1263	if ( pDis->pCurInstr->uOpcode == OP_STOSWD
1264	&& !pRegFrame->eflags.Bits.u1DF
1265	&& pDis->uOpMode == pDis->uCpuMode
1266	&& pDis->uAddrMode == pDis->uCpuMode)
1267	{
1268	bool fValidStosd = false;
1269
1270	if ( pDis->uCpuMode == DISCPUMODE_32BIT
1271	&& pDis->fPrefix == DISPREFIX_REP
1272	&& pRegFrame->ecx <= 0x20
1273	&& pRegFrame->ecx * 4 <= GUEST_PAGE_SIZE - ((uintptr_t)pvFault & GUEST_PAGE_OFFSET_MASK)
1274	&& !((uintptr_t)pvFault & 3)
1275	&& (pRegFrame->eax == 0 \|\| pRegFrame->eax == 0x80) /* the two values observed. */
1276	)
1277	{
1278	fValidStosd = true;
1279	pRegFrame->rcx &= 0xffffffff; /* paranoia */
1280	}
1281	else
1282	if ( pDis->uCpuMode == DISCPUMODE_64BIT
1283	&& pDis->fPrefix == (DISPREFIX_REP \| DISPREFIX_REX)
1284	&& pRegFrame->rcx <= 0x20
1285	&& pRegFrame->rcx * 8 <= GUEST_PAGE_SIZE - ((uintptr_t)pvFault & GUEST_PAGE_OFFSET_MASK)
1286	&& !((uintptr_t)pvFault & 7)
1287	&& (pRegFrame->rax == 0 \|\| pRegFrame->rax == 0x80) /* the two values observed. */
1288	)
1289	{
1290	fValidStosd = true;
1291	}
1292
1293	if (fValidStosd)
1294	{
1295	rc = pgmRZPoolAccessPfHandlerSTOSD(pVM, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault);
1296	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZRepStosd, a);
1297	PGM_UNLOCK(pVM);
1298	return rc;
1299	}
1300	}
1301
1302	/* REP prefix, don't bother. */
1303	STAM_COUNTER_INC(&pPool->StatMonitorPfRZRepPrefix);
1304	Log4(("pgmRZPoolAccessPfHandler: eax=%#x ecx=%#x edi=%#x esi=%#x rip=%RGv opcode=%d prefix=%#x\n",
1305	pRegFrame->eax, pRegFrame->ecx, pRegFrame->edi, pRegFrame->esi, (RTGCPTR)pRegFrame->rip, pDis->pCurInstr->uOpcode, pDis->fPrefix));
1306	fNotReusedNotForking = true;
1307	}
1308
1309	# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) && defined(IN_RING0)
1310	/* E.g. Windows 7 x64 initializes page tables and touches some pages in the table during the process. This
1311	* leads to pgm pool trashing and an excessive amount of write faults due to page monitoring.
1312	*/
1313	if ( pPage->cModifications >= cMaxModifications
1314	&& !fForcedFlush
1315	&& (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT \|\| pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
1316	&& ( fNotReusedNotForking
1317	\|\| ( !pgmRZPoolMonitorIsReused(pVM, pVCpu, pRegFrame, pDis, pvFault, pPage)
1318	&& !pgmRZPoolMonitorIsForking(pPool, pDis, GCPhysFault & PAGE_OFFSET_MASK))
1319	)
1320	)
1321	{
1322	Assert(!pgmPoolIsPageLocked(pPage));
1323	Assert(pPage->fDirty == false);
1324
1325	/* Flush any monitored duplicates as we will disable write protection. */
1326	if ( pPage->iMonitoredNext != NIL_PGMPOOL_IDX
1327	\|\| pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
1328	{
1329	PPGMPOOLPAGE pPageHead = pPage;
1330
1331	/* Find the monitor head. */
1332	while (pPageHead->iMonitoredPrev != NIL_PGMPOOL_IDX)
1333	pPageHead = &pPool->aPages[pPageHead->iMonitoredPrev];
1334
1335	while (pPageHead)
1336	{
1337	unsigned idxNext = pPageHead->iMonitoredNext;
1338
1339	if (pPageHead != pPage)
1340	{
1341	STAM_COUNTER_INC(&pPool->StatDirtyPageDupFlush);
1342	Log(("Flush duplicate page idx=%d GCPhys=%RGp type=%s\n", pPageHead->idx, pPageHead->GCPhys, pgmPoolPoolKindToStr(pPageHead->enmKind)));
1343	int rc2 = pgmPoolFlushPage(pPool, pPageHead);
1344	AssertRC(rc2);
1345	}
1346
1347	if (idxNext == NIL_PGMPOOL_IDX)
1348	break;
1349
1350	pPageHead = &pPool->aPages[idxNext];
1351	}
1352	}
1353
1354	/* The flushing above might fail for locked pages, so double check. */
1355	if ( pPage->iMonitoredNext == NIL_PGMPOOL_IDX
1356	&& pPage->iMonitoredPrev == NIL_PGMPOOL_IDX)
1357	{
1358	pgmPoolAddDirtyPage(pVM, pPool, pPage);
1359
1360	/* Temporarily allow write access to the page table again. */
1361	rc = PGMHandlerPhysicalPageTempOff(pVM,
1362	pPage->GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK,
1363	pPage->GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK);
1364	if (rc == VINF_SUCCESS)
1365	{
1366	rc = PGMShwMakePageWritable(pVCpu, pvFault, PGM_MK_PG_IS_WRITE_FAULT);
1367	AssertMsg(rc == VINF_SUCCESS
1368	/* In the SMP case the page table might be removed while we wait for the PGM lock in the trap handler. */
1369	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
1370	\|\| rc == VERR_PAGE_NOT_PRESENT,
1371	("PGMShwModifyPage -> GCPtr=%RGv rc=%d\n", pvFault, rc));
1372	# ifdef VBOX_STRICT
1373	pPage->GCPtrDirtyFault = pvFault;
1374	# endif
1375
1376	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, a);
1377	PGM_UNLOCK(pVM);
1378	return rc;
1379	}
1380	}
1381	}
1382	# endif /* PGMPOOL_WITH_OPTIMIZED_DIRTY_PT && IN_RING0 */
1383
1384	STAM_COUNTER_INC(&pPool->StatMonitorPfRZFlushModOverflow);
1385	flushPage:
1386	/*
1387	* Not worth it, so flush it.
1388	*
1389	* If we considered it to be reused, don't go back to ring-3
1390	* to emulate failed instructions since we usually cannot
1391	* interpret then. This may be a bit risky, in which case
1392	* the reuse detection must be fixed.
1393	*/
1394	rc = pgmRZPoolAccessPfHandlerFlush(pVM, pVCpu, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault);
1395	if ( rc == VINF_EM_RAW_EMULATE_INSTR
1396	&& fReused)
1397	{
1398	Assert(!PGMPOOL_PAGE_IS_NESTED(pPage)); /* temporary, remove later. */
1399	/* Make sure that the current instruction still has shadow page backing, otherwise we'll end up in a loop. */
1400	if (PGMShwGetPage(pVCpu, pRegFrame->rip, NULL, NULL) == VINF_SUCCESS)
1401	rc = VINF_SUCCESS; /* safe to restart the instruction. */
1402	}
1403	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZFlushPage, a);
1404	PGM_UNLOCK(pVM);
1405	return rc;
1406	}
1407
1408	#endif /* !IN_RING3 */
1409
1410	/**
1411	* @callback_method_impl{FNPGMPHYSHANDLER,
1412	* Access handler for shadowed page table pages.}
1413	*
1414	* @remarks Only uses the VINF_PGM_HANDLER_DO_DEFAULT status.
1415	* @note The @a uUser argument is the index of the PGMPOOLPAGE.
1416	*/
1417	DECLCALLBACK(VBOXSTRICTRC)
1418	pgmPoolAccessHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf,
1419	PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, uint64_t uUser)
1420	{
1421	PPGMPOOL const pPool = pVM->pgm.s.CTX_SUFF(pPool);
1422	STAM_PROFILE_START(&pPool->CTX_SUFF_Z(StatMonitor), a);
1423	AssertReturn(uUser < pPool->cCurPages, VERR_PGM_POOL_IPE);
1424	PPGMPOOLPAGE const pPage = &pPool->aPages[uUser];
1425	LogFlow(("PGM_ALL_CB_DECL: GCPhys=%RGp %p:{.Core=%RHp, .idx=%d, .GCPhys=%RGp, .enmType=%d}\n",
1426	GCPhys, pPage, pPage->Core.Key, pPage->idx, pPage->GCPhys, pPage->enmKind));
1427
1428	NOREF(pvPhys); NOREF(pvBuf); NOREF(enmAccessType);
1429
1430	PGM_LOCK_VOID(pVM);
1431
1432	#ifdef VBOX_WITH_STATISTICS
1433	/*
1434	* Collect stats on the access.
1435	*/
1436	AssertCompile(RT_ELEMENTS(pPool->CTX_MID_Z(aStatMonitor,Sizes)) == 19);
1437	if (cbBuf <= 16 && cbBuf > 0)
1438	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[cbBuf - 1]);
1439	else if (cbBuf >= 17 && cbBuf < 32)
1440	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[16]);
1441	else if (cbBuf >= 32 && cbBuf < 64)
1442	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[17]);
1443	else if (cbBuf >= 64)
1444	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[18]);
1445
1446	uint8_t cbAlign;
1447	switch (pPage->enmKind)
1448	{
1449	default:
1450	cbAlign = 7;
1451	break;
1452	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1453	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1454	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1455	case PGMPOOLKIND_32BIT_PD:
1456	case PGMPOOLKIND_32BIT_PD_PHYS:
1457	cbAlign = 3;
1458	break;
1459	}
1460	AssertCompile(RT_ELEMENTS(pPool->CTX_MID_Z(aStatMonitor,Misaligned)) == 7);
1461	if ((uint8_t)GCPhys & cbAlign)
1462	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Misaligned)[((uint8_t)GCPhys & cbAlign) - 1]);
1463	#endif
1464
1465	/*
1466	* Make sure the pool page wasn't modified by a different CPU.
1467	*/
1468	if (PHYS_PAGE_ADDRESS(GCPhys) == PHYS_PAGE_ADDRESS(pPage->GCPhys))
1469	{
1470	Assert(pPage->enmKind != PGMPOOLKIND_FREE);
1471
1472	/* The max modification count before flushing depends on the context and page type. */
1473	#ifdef IN_RING3
1474	uint16_t const cMaxModifications = 96; /* it's cheaper here, right? */
1475	#else
1476	uint16_t cMaxModifications;
1477	if ( pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT
1478	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
1479	cMaxModifications = 4;
1480	else
1481	cMaxModifications = 24;
1482	#endif
1483
1484	/*
1485	* We don't have to be very sophisticated about this since there are relativly few calls here.
1486	* However, we must try our best to detect any non-cpu accesses (disk / networking).
1487	*/
1488	if ( ( pPage->cModifications < cMaxModifications
1489	\|\| pgmPoolIsPageLocked(pPage) )
1490	&& enmOrigin != PGMACCESSORIGIN_DEVICE
1491	&& cbBuf <= 16)
1492	{
1493	/* Clear the shadow entry. */
1494	if (!pPage->cModifications++)
1495	pgmPoolMonitorModifiedInsert(pPool, pPage);
1496
1497	if (cbBuf <= 8)
1498	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvBuf, (uint32_t)cbBuf);
1499	else
1500	{
1501	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvBuf, 8);
1502	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys + 8, (uint8_t *)pvBuf + 8, (uint32_t)cbBuf - 8);
1503	}
1504	}
1505	else
1506	pgmPoolMonitorChainFlush(pPool, pPage);
1507
1508	STAM_PROFILE_STOP_EX(&pPool->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,FlushPage), a);
1509	}
1510	else
1511	Log(("CPU%d: PGM_ALL_CB_DECL pgm pool page for %RGp changed (to %RGp) while waiting!\n", pVCpu->idCpu, PHYS_PAGE_ADDRESS(GCPhys), PHYS_PAGE_ADDRESS(pPage->GCPhys)));
1512	PGM_UNLOCK(pVM);
1513	return VINF_PGM_HANDLER_DO_DEFAULT;
1514	}
1515
1516
1517	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1518
1519	# if defined(VBOX_STRICT) && !defined(IN_RING3)
1520
1521	/**
1522	* Check references to guest physical memory in a PAE / PAE page table.
1523	*
1524	* @param pPool The pool.
1525	* @param pPage The page.
1526	* @param pShwPT The shadow page table (mapping of the page).
1527	* @param pGstPT The guest page table.
1528	*/
1529	static void pgmPoolTrackCheckPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT)
1530	{
1531	unsigned cErrors = 0;
1532	int LastRc = -1; /* initialized to shut up gcc */
1533	unsigned LastPTE = ~0U; /* initialized to shut up gcc */
1534	RTHCPHYS LastHCPhys = NIL_RTHCPHYS; /* initialized to shut up gcc */
1535	PVMCC pVM = pPool->CTX_SUFF(pVM);
1536
1537	# ifdef VBOX_STRICT
1538	for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
1539	AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), pPage->iFirstPresent));
1540	# endif
1541	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
1542	{
1543	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
1544	{
1545	RTHCPHYS HCPhys = NIL_RTHCPHYS;
1546	int rc = PGMPhysGCPhys2HCPhys(pVM, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK, &HCPhys);
1547	if ( rc != VINF_SUCCESS
1548	\|\| PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) != HCPhys)
1549	{
1550	Log(("rc=%d idx=%d guest %RX64 shw=%RX64 vs %RHp\n", rc, i, pGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
1551	LastPTE = i;
1552	LastRc = rc;
1553	LastHCPhys = HCPhys;
1554	cErrors++;
1555
1556	RTHCPHYS HCPhysPT = NIL_RTHCPHYS;
1557	rc = PGMPhysGCPhys2HCPhys(pVM, pPage->GCPhys, &HCPhysPT);
1558	AssertRC(rc);
1559
1560	for (unsigned iPage = 0; iPage < pPool->cCurPages; iPage++)
1561	{
1562	PPGMPOOLPAGE pTempPage = &pPool->aPages[iPage];
1563
1564	if (pTempPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
1565	{
1566	PPGMSHWPTPAE pShwPT2 = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pTempPage);
1567
1568	for (unsigned j = 0; j < RT_ELEMENTS(pShwPT->a); j++)
1569	{
1570	if ( PGMSHWPTEPAE_IS_P_RW(pShwPT2->a[j])
1571	&& PGMSHWPTEPAE_GET_HCPHYS(pShwPT2->a[j]) == HCPhysPT)
1572	{
1573	Log(("GCPhys=%RGp idx=%d %RX64 vs %RX64\n", pTempPage->GCPhys, j, PGMSHWPTEPAE_GET_LOG(pShwPT->a[j]), PGMSHWPTEPAE_GET_LOG(pShwPT2->a[j])));
1574	}
1575	}
1576
1577	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pShwPT2);
1578	}
1579	}
1580	}
1581	}
1582	}
1583	AssertMsg(!cErrors, ("cErrors=%d: last rc=%d idx=%d guest %RX64 shw=%RX64 vs %RHp\n", cErrors, LastRc, LastPTE, pGstPT->a[LastPTE].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[LastPTE]), LastHCPhys));
1584	}
1585
1586
1587	/**
1588	* Check references to guest physical memory in a PAE / 32-bit page table.
1589	*
1590	* @param pPool The pool.
1591	* @param pPage The page.
1592	* @param pShwPT The shadow page table (mapping of the page).
1593	* @param pGstPT The guest page table.
1594	*/
1595	static void pgmPoolTrackCheckPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PT pGstPT)
1596	{
1597	unsigned cErrors = 0;
1598	int LastRc = -1; /* initialized to shut up gcc */
1599	unsigned LastPTE = ~0U; /* initialized to shut up gcc */
1600	RTHCPHYS LastHCPhys = NIL_RTHCPHYS; /* initialized to shut up gcc */
1601	PVMCC pVM = pPool->CTX_SUFF(pVM);
1602
1603	# ifdef VBOX_STRICT
1604	for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
1605	AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), pPage->iFirstPresent));
1606	# endif
1607	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
1608	{
1609	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
1610	{
1611	RTHCPHYS HCPhys = NIL_RTHCPHYS;
1612	int rc = PGMPhysGCPhys2HCPhys(pVM, pGstPT->a[i].u & X86_PTE_PG_MASK, &HCPhys);
1613	if ( rc != VINF_SUCCESS
1614	\|\| PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) != HCPhys)
1615	{
1616	Log(("rc=%d idx=%d guest %x shw=%RX64 vs %RHp\n", rc, i, pGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
1617	LastPTE = i;
1618	LastRc = rc;
1619	LastHCPhys = HCPhys;
1620	cErrors++;
1621
1622	RTHCPHYS HCPhysPT = NIL_RTHCPHYS;
1623	rc = PGMPhysGCPhys2HCPhys(pVM, pPage->GCPhys, &HCPhysPT);
1624	AssertRC(rc);
1625
1626	for (unsigned iPage = 0; iPage < pPool->cCurPages; iPage++)
1627	{
1628	PPGMPOOLPAGE pTempPage = &pPool->aPages[iPage];
1629
1630	if (pTempPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
1631	{
1632	PPGMSHWPTPAE pShwPT2 = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pTempPage);
1633
1634	for (unsigned j = 0; j < RT_ELEMENTS(pShwPT->a); j++)
1635	{
1636	if ( PGMSHWPTEPAE_IS_P_RW(pShwPT2->a[j])
1637	&& PGMSHWPTEPAE_GET_HCPHYS(pShwPT2->a[j]) == HCPhysPT)
1638	{
1639	Log(("GCPhys=%RGp idx=%d %RX64 vs %RX64\n", pTempPage->GCPhys, j, PGMSHWPTEPAE_GET_LOG(pShwPT->a[j]), PGMSHWPTEPAE_GET_LOG(pShwPT2->a[j])));
1640	}
1641	}
1642
1643	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pShwPT2);
1644	}
1645	}
1646	}
1647	}
1648	}
1649	AssertMsg(!cErrors, ("cErrors=%d: last rc=%d idx=%d guest %x shw=%RX64 vs %RHp\n", cErrors, LastRc, LastPTE, pGstPT->a[LastPTE].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[LastPTE]), LastHCPhys));
1650	}
1651
1652	# endif /* VBOX_STRICT && !IN_RING3 */
1653
1654	/**
1655	* Clear references to guest physical memory in a PAE / PAE page table.
1656	*
1657	* @returns nr of changed PTEs
1658	* @param pPool The pool.
1659	* @param pPage The page.
1660	* @param pShwPT The shadow page table (mapping of the page).
1661	* @param pGstPT The guest page table.
1662	* @param pOldGstPT The old cached guest page table.
1663	* @param fAllowRemoval Bail out as soon as we encounter an invalid PTE
1664	* @param pfFlush Flush reused page table (out)
1665	*/
1666	DECLINLINE(unsigned) pgmPoolTrackFlushPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT,
1667	PCX86PTPAE pOldGstPT, bool fAllowRemoval, bool *pfFlush)
1668	{
1669	unsigned cChanged = 0;
1670
1671	# ifdef VBOX_STRICT
1672	for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
1673	AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), pPage->iFirstPresent));
1674	# endif
1675	*pfFlush = false;
1676
1677	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
1678	{
1679	/* Check the new value written by the guest. If present and with a bogus physical address, then
1680	* it's fairly safe to assume the guest is reusing the PT.
1681	*/
1682	if ( fAllowRemoval
1683	&& (pGstPT->a[i].u & X86_PTE_P))
1684	{
1685	if (!PGMPhysIsGCPhysValid(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK))
1686	{
1687	*pfFlush = true;
1688	return ++cChanged;
1689	}
1690	}
1691	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
1692	{
1693	/* If the old cached PTE is identical, then there's no need to flush the shadow copy. */
1694	if ((pGstPT->a[i].u & X86_PTE_PAE_PG_MASK) == (pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK))
1695	{
1696	# ifdef VBOX_STRICT
1697	RTHCPHYS HCPhys = NIL_RTGCPHYS;
1698	int rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK, &HCPhys);
1699	AssertMsg(rc == VINF_SUCCESS && PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) == HCPhys, ("rc=%d guest %RX64 old %RX64 shw=%RX64 vs %RHp\n", rc, pGstPT->a[i].u, pOldGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
1700	# endif
1701	uint64_t uHostAttr = PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & (X86_PTE_P \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D \| X86_PTE_G \| X86_PTE_PAE_NX);
1702	bool fHostRW = !!(PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & X86_PTE_RW);
1703	uint64_t uGuestAttr = pGstPT->a[i].u & (X86_PTE_P \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D \| X86_PTE_G \| X86_PTE_PAE_NX);
1704	bool fGuestRW = !!(pGstPT->a[i].u & X86_PTE_RW);
1705
1706	if ( uHostAttr == uGuestAttr
1707	&& fHostRW <= fGuestRW)
1708	continue;
1709	}
1710	cChanged++;
1711	/* Something was changed, so flush it. */
1712	Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX64 hint=%RX64\n",
1713	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK));
1714	pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK, i);
1715	PGMSHWPTEPAE_ATOMIC_SET(pShwPT->a[i], 0);
1716	}
1717	}
1718	return cChanged;
1719	}
1720
1721
1722	/**
1723	* Clear references to guest physical memory in a PAE / PAE page table.
1724	*
1725	* @returns nr of changed PTEs
1726	* @param pPool The pool.
1727	* @param pPage The page.
1728	* @param pShwPT The shadow page table (mapping of the page).
1729	* @param pGstPT The guest page table.
1730	* @param pOldGstPT The old cached guest page table.
1731	* @param fAllowRemoval Bail out as soon as we encounter an invalid PTE
1732	* @param pfFlush Flush reused page table (out)
1733	*/
1734	DECLINLINE(unsigned) pgmPoolTrackFlushPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PT pGstPT,
1735	PCX86PT pOldGstPT, bool fAllowRemoval, bool *pfFlush)
1736	{
1737	unsigned cChanged = 0;
1738
1739	# ifdef VBOX_STRICT
1740	for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
1741	AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), pPage->iFirstPresent));
1742	# endif
1743	*pfFlush = false;
1744
1745	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
1746	{
1747	/* Check the new value written by the guest. If present and with a bogus physical address, then
1748	* it's fairly safe to assume the guest is reusing the PT. */
1749	if (fAllowRemoval)
1750	{
1751	X86PGUINT const uPte = pGstPT->a[i].u;
1752	if ( (uPte & X86_PTE_P)
1753	&& !PGMPhysIsGCPhysValid(pPool->CTX_SUFF(pVM), uPte & X86_PTE_PG_MASK))
1754	{
1755	*pfFlush = true;
1756	return ++cChanged;
1757	}
1758	}
1759	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
1760	{
1761	/* If the old cached PTE is identical, then there's no need to flush the shadow copy. */
1762	if ((pGstPT->a[i].u & X86_PTE_PG_MASK) == (pOldGstPT->a[i].u & X86_PTE_PG_MASK))
1763	{
1764	# ifdef VBOX_STRICT
1765	RTHCPHYS HCPhys = NIL_RTGCPHYS;
1766	int rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PG_MASK, &HCPhys);
1767	AssertMsg(rc == VINF_SUCCESS && PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) == HCPhys, ("rc=%d guest %x old %x shw=%RX64 vs %RHp\n", rc, pGstPT->a[i].u, pOldGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
1768	# endif
1769	uint64_t uHostAttr = PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & (X86_PTE_P \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D \| X86_PTE_G);
1770	bool fHostRW = !!(PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & X86_PTE_RW);
1771	uint64_t uGuestAttr = pGstPT->a[i].u & (X86_PTE_P \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D \| X86_PTE_G);
1772	bool fGuestRW = !!(pGstPT->a[i].u & X86_PTE_RW);
1773
1774	if ( uHostAttr == uGuestAttr
1775	&& fHostRW <= fGuestRW)
1776	continue;
1777	}
1778	cChanged++;
1779	/* Something was changed, so flush it. */
1780	Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX64 hint=%x\n",
1781	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PG_MASK));
1782	pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PG_MASK, i);
1783	PGMSHWPTEPAE_ATOMIC_SET(pShwPT->a[i], 0);
1784	}
1785	}
1786	return cChanged;
1787	}
1788
1789
1790	/**
1791	* Flush a dirty page
1792	*
1793	* @param pVM The cross context VM structure.
1794	* @param pPool The pool.
1795	* @param idxSlot Dirty array slot index
1796	* @param fAllowRemoval Allow a reused page table to be removed
1797	*/
1798	static void pgmPoolFlushDirtyPage(PVMCC pVM, PPGMPOOL pPool, unsigned idxSlot, bool fAllowRemoval = false)
1799	{
1800	AssertCompile(RT_ELEMENTS(pPool->aidxDirtyPages) == RT_ELEMENTS(pPool->aDirtyPages));
1801
1802	Assert(idxSlot < RT_ELEMENTS(pPool->aDirtyPages));
1803	unsigned idxPage = pPool->aidxDirtyPages[idxSlot];
1804	if (idxPage == NIL_PGMPOOL_IDX)
1805	return;
1806
1807	PPGMPOOLPAGE pPage = &pPool->aPages[idxPage];
1808	Assert(pPage->idx == idxPage);
1809	Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1810
1811	AssertMsg(pPage->fDirty, ("Page %RGp (slot=%d) not marked dirty!", pPage->GCPhys, idxSlot));
1812	Log(("Flush dirty page %RGp cMods=%d\n", pPage->GCPhys, pPage->cModifications));
1813
1814	/* First write protect the page again to catch all write accesses. (before checking for changes -> SMP) */
1815	int rc = PGMHandlerPhysicalReset(pVM, pPage->GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK);
1816	Assert(rc == VINF_SUCCESS);
1817	pPage->fDirty = false;
1818
1819	# ifdef VBOX_STRICT
1820	uint64_t fFlags = 0;
1821	RTHCPHYS HCPhys;
1822	rc = PGMShwGetPage(VMMGetCpu(pVM), pPage->GCPtrDirtyFault, &fFlags, &HCPhys);
1823	AssertMsg( ( rc == VINF_SUCCESS
1824	&& (!(fFlags & X86_PTE_RW) \|\| HCPhys != pPage->Core.Key))
1825	/* In the SMP case the page table might be removed while we wait for the PGM lock in the trap handler. */
1826	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
1827	\|\| rc == VERR_PAGE_NOT_PRESENT,
1828	("PGMShwGetPage -> GCPtr=%RGv rc=%d flags=%RX64\n", pPage->GCPtrDirtyFault, rc, fFlags));
1829	# endif
1830
1831	/* Flush those PTEs that have changed. */
1832	STAM_PROFILE_START(&pPool->StatTrackDeref,a);
1833	void *pvShw = PGMPOOL_PAGE_2_PTR(pVM, pPage);
1834	void *pvGst;
1835	rc = PGM_GCPHYS_2_PTR_EX(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
1836	bool fFlush;
1837	unsigned cChanges;
1838
1839	if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
1840	cChanges = pgmPoolTrackFlushPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst,
1841	(PCX86PTPAE)&pPool->aDirtyPages[idxSlot].aPage[0], fAllowRemoval, &fFlush);
1842	else
1843	{
1844	Assert(!PGMPOOL_PAGE_IS_NESTED(pPage)); /* temporary, remove later. */
1845	cChanges = pgmPoolTrackFlushPTPae32Bit(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PT)pvGst,
1846	(PCX86PT)&pPool->aDirtyPages[idxSlot].aPage[0], fAllowRemoval, &fFlush);
1847	}
1848
1849	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
1850	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
1851	STAM_PROFILE_STOP(&pPool->StatTrackDeref,a);
1852	/* Note: we might want to consider keeping the dirty page active in case there were many changes. */
1853
1854	/* This page is likely to be modified again, so reduce the nr of modifications just a bit here. */
1855	Assert(pPage->cModifications);
1856	if (cChanges < 4)
1857	pPage->cModifications = 1; /* must use > 0 here */
1858	else
1859	pPage->cModifications = RT_MAX(1, pPage->cModifications / 2);
1860
1861	STAM_COUNTER_INC(&pPool->StatResetDirtyPages);
1862	if (pPool->cDirtyPages == RT_ELEMENTS(pPool->aDirtyPages))
1863	pPool->idxFreeDirtyPage = idxSlot;
1864
1865	pPool->cDirtyPages--;
1866	pPool->aidxDirtyPages[idxSlot] = NIL_PGMPOOL_IDX;
1867	Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
1868	if (fFlush)
1869	{
1870	Assert(fAllowRemoval);
1871	Log(("Flush reused page table!\n"));
1872	pgmPoolFlushPage(pPool, pPage);
1873	STAM_COUNTER_INC(&pPool->StatForceFlushReused);
1874	}
1875	else
1876	Log(("Removed dirty page %RGp cMods=%d cChanges=%d\n", pPage->GCPhys, pPage->cModifications, cChanges));
1877	}
1878
1879
1880	# ifndef IN_RING3
1881	/**
1882	* Add a new dirty page
1883	*
1884	* @param pVM The cross context VM structure.
1885	* @param pPool The pool.
1886	* @param pPage The page.
1887	*/
1888	void pgmPoolAddDirtyPage(PVMCC pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1889	{
1890	PGM_LOCK_ASSERT_OWNER(pVM);
1891	AssertCompile(RT_ELEMENTS(pPool->aDirtyPages) == 8 \|\| RT_ELEMENTS(pPool->aDirtyPages) == 16);
1892	Assert(!pPage->fDirty);
1893	Assert(!PGMPOOL_PAGE_IS_NESTED(pPage));
1894
1895	unsigned idxFree = pPool->idxFreeDirtyPage;
1896	Assert(idxFree < RT_ELEMENTS(pPool->aDirtyPages));
1897	Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1898
1899	if (pPool->cDirtyPages >= RT_ELEMENTS(pPool->aDirtyPages))
1900	{
1901	STAM_COUNTER_INC(&pPool->StatDirtyPageOverFlowFlush);
1902	pgmPoolFlushDirtyPage(pVM, pPool, idxFree, true /* allow removal of reused page tables*/);
1903	}
1904	Assert(pPool->cDirtyPages < RT_ELEMENTS(pPool->aDirtyPages));
1905	AssertMsg(pPool->aidxDirtyPages[idxFree] == NIL_PGMPOOL_IDX, ("idxFree=%d cDirtyPages=%d\n", idxFree, pPool->cDirtyPages));
1906
1907	Log(("Add dirty page %RGp (slot=%d)\n", pPage->GCPhys, idxFree));
1908
1909	/*
1910	* Make a copy of the guest page table as we require valid GCPhys addresses
1911	* when removing references to physical pages.
1912	* (The HCPhys linear lookup is extremely expensive!)
1913	*/
1914	void *pvGst;
1915	int rc = PGM_GCPHYS_2_PTR_EX(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
1916	memcpy(&pPool->aDirtyPages[idxFree].aPage[0], pvGst,
1917	pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT ? PAGE_SIZE : PAGE_SIZE / 2);
1918	# ifdef VBOX_STRICT
1919	void *pvShw = PGMPOOL_PAGE_2_PTR(pVM, pPage);
1920	if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
1921	pgmPoolTrackCheckPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst);
1922	else
1923	pgmPoolTrackCheckPTPae32Bit(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PT)pvGst);
1924	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
1925	# endif
1926	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
1927
1928	STAM_COUNTER_INC(&pPool->StatDirtyPage);
1929	pPage->fDirty = true;
1930	pPage->idxDirtyEntry = (uint8_t)idxFree; Assert(pPage->idxDirtyEntry == idxFree);
1931	pPool->aidxDirtyPages[idxFree] = pPage->idx;
1932	pPool->cDirtyPages++;
1933
1934	pPool->idxFreeDirtyPage = (pPool->idxFreeDirtyPage + 1) & (RT_ELEMENTS(pPool->aDirtyPages) - 1);
1935	if ( pPool->cDirtyPages < RT_ELEMENTS(pPool->aDirtyPages)
1936	&& pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX)
1937	{
1938	unsigned i;
1939	for (i = 1; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
1940	{
1941	idxFree = (pPool->idxFreeDirtyPage + i) & (RT_ELEMENTS(pPool->aDirtyPages) - 1);
1942	if (pPool->aidxDirtyPages[idxFree] == NIL_PGMPOOL_IDX)
1943	{
1944	pPool->idxFreeDirtyPage = idxFree;
1945	break;
1946	}
1947	}
1948	Assert(i != RT_ELEMENTS(pPool->aDirtyPages));
1949	}
1950
1951	Assert(pPool->cDirtyPages == RT_ELEMENTS(pPool->aDirtyPages) \|\| pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] == NIL_PGMPOOL_IDX);
1952
1953	/*
1954	* Clear all references to this shadow table. See @bugref{7298}.
1955	*/
1956	pgmPoolTrackClearPageUsers(pPool, pPage);
1957	}
1958	# endif /* !IN_RING3 */
1959
1960
1961	/**
1962	* Check if the specified page is dirty (not write monitored)
1963	*
1964	* @return dirty or not
1965	* @param pVM The cross context VM structure.
1966	* @param GCPhys Guest physical address
1967	*/
1968	bool pgmPoolIsDirtyPageSlow(PVMCC pVM, RTGCPHYS GCPhys)
1969	{
1970	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1971	PGM_LOCK_ASSERT_OWNER(pVM);
1972	if (!pPool->cDirtyPages)
1973	return false;
1974
1975	GCPhys = GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
1976
1977	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
1978	{
1979	unsigned idxPage = pPool->aidxDirtyPages[i];
1980	if (idxPage != NIL_PGMPOOL_IDX)
1981	{
1982	PPGMPOOLPAGE pPage = &pPool->aPages[idxPage];
1983	if (pPage->GCPhys == GCPhys)
1984	return true;
1985	}
1986	}
1987	return false;
1988	}
1989
1990
1991	/**
1992	* Reset all dirty pages by reinstating page monitoring.
1993	*
1994	* @param pVM The cross context VM structure.
1995	*/
1996	void pgmPoolResetDirtyPages(PVMCC pVM)
1997	{
1998	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1999	PGM_LOCK_ASSERT_OWNER(pVM);
2000	Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
2001
2002	if (!pPool->cDirtyPages)
2003	return;
2004
2005	Log(("pgmPoolResetDirtyPages\n"));
2006	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
2007	pgmPoolFlushDirtyPage(pVM, pPool, i, true /* allow removal of reused page tables*/);
2008
2009	pPool->idxFreeDirtyPage = 0;
2010	if ( pPool->cDirtyPages != RT_ELEMENTS(pPool->aDirtyPages)
2011	&& pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX)
2012	{
2013	unsigned i;
2014	for (i = 1; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
2015	{
2016	if (pPool->aidxDirtyPages[i] == NIL_PGMPOOL_IDX)
2017	{
2018	pPool->idxFreeDirtyPage = i;
2019	break;
2020	}
2021	}
2022	AssertMsg(i != RT_ELEMENTS(pPool->aDirtyPages), ("cDirtyPages %d", pPool->cDirtyPages));
2023	}
2024
2025	Assert(pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] == NIL_PGMPOOL_IDX \|\| pPool->cDirtyPages == RT_ELEMENTS(pPool->aDirtyPages));
2026	return;
2027	}
2028
2029
2030	/**
2031	* Invalidate the PT entry for the specified page
2032	*
2033	* @param pVM The cross context VM structure.
2034	* @param GCPtrPage Guest page to invalidate
2035	*/
2036	void pgmPoolResetDirtyPage(PVMCC pVM, RTGCPTR GCPtrPage)
2037	{
2038	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2039	PGM_LOCK_ASSERT_OWNER(pVM);
2040	Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
2041
2042	if (!pPool->cDirtyPages)
2043	return;
2044
2045	Log(("pgmPoolResetDirtyPage %RGv\n", GCPtrPage)); RT_NOREF_PV(GCPtrPage);
2046	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
2047	{
2048	/** @todo What was intended here??? This looks incomplete... */
2049	}
2050	}
2051
2052
2053	/**
2054	* Reset all dirty pages by reinstating page monitoring.
2055	*
2056	* @param pVM The cross context VM structure.
2057	* @param GCPhysPT Physical address of the page table
2058	*/
2059	void pgmPoolInvalidateDirtyPage(PVMCC pVM, RTGCPHYS GCPhysPT)
2060	{
2061	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2062	PGM_LOCK_ASSERT_OWNER(pVM);
2063	Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
2064	unsigned idxDirtyPage = RT_ELEMENTS(pPool->aDirtyPages);
2065
2066	if (!pPool->cDirtyPages)
2067	return;
2068
2069	GCPhysPT = GCPhysPT & ~(RTGCPHYS)PAGE_OFFSET_MASK;
2070
2071	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
2072	{
2073	unsigned idxPage = pPool->aidxDirtyPages[i];
2074	if (idxPage != NIL_PGMPOOL_IDX)
2075	{
2076	PPGMPOOLPAGE pPage = &pPool->aPages[idxPage];
2077	if (pPage->GCPhys == GCPhysPT)
2078	{
2079	idxDirtyPage = i;
2080	break;
2081	}
2082	}
2083	}
2084
2085	if (idxDirtyPage != RT_ELEMENTS(pPool->aDirtyPages))
2086	{
2087	pgmPoolFlushDirtyPage(pVM, pPool, idxDirtyPage, true /* allow removal of reused page tables*/);
2088	if ( pPool->cDirtyPages != RT_ELEMENTS(pPool->aDirtyPages)
2089	&& pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX)
2090	{
2091	unsigned i;
2092	for (i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
2093	{
2094	if (pPool->aidxDirtyPages[i] == NIL_PGMPOOL_IDX)
2095	{
2096	pPool->idxFreeDirtyPage = i;
2097	break;
2098	}
2099	}
2100	AssertMsg(i != RT_ELEMENTS(pPool->aDirtyPages), ("cDirtyPages %d", pPool->cDirtyPages));
2101	}
2102	}
2103	}
2104
2105	#endif /* PGMPOOL_WITH_OPTIMIZED_DIRTY_PT */
2106
2107	/**
2108	* Inserts a page into the GCPhys hash table.
2109	*
2110	* @param pPool The pool.
2111	* @param pPage The page.
2112	*/
2113	DECLINLINE(void) pgmPoolHashInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2114	{
2115	Log3(("pgmPoolHashInsert: %RGp\n", pPage->GCPhys));
2116	Assert(pPage->GCPhys != NIL_RTGCPHYS); Assert(pPage->iNext == NIL_PGMPOOL_IDX);
2117	uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
2118	pPage->iNext = pPool->aiHash[iHash];
2119	pPool->aiHash[iHash] = pPage->idx;
2120	}
2121
2122
2123	/**
2124	* Removes a page from the GCPhys hash table.
2125	*
2126	* @param pPool The pool.
2127	* @param pPage The page.
2128	*/
2129	DECLINLINE(void) pgmPoolHashRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2130	{
2131	Log3(("pgmPoolHashRemove: %RGp\n", pPage->GCPhys));
2132	uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
2133	if (pPool->aiHash[iHash] == pPage->idx)
2134	pPool->aiHash[iHash] = pPage->iNext;
2135	else
2136	{
2137	uint16_t iPrev = pPool->aiHash[iHash];
2138	for (;;)
2139	{
2140	const int16_t i = pPool->aPages[iPrev].iNext;
2141	if (i == pPage->idx)
2142	{
2143	pPool->aPages[iPrev].iNext = pPage->iNext;
2144	break;
2145	}
2146	if (i == NIL_PGMPOOL_IDX)
2147	{
2148	AssertReleaseMsgFailed(("GCPhys=%RGp idx=%d\n", pPage->GCPhys, pPage->idx));
2149	break;
2150	}
2151	iPrev = i;
2152	}
2153	}
2154	pPage->iNext = NIL_PGMPOOL_IDX;
2155	}
2156
2157
2158	/**
2159	* Frees up one cache page.
2160	*
2161	* @returns VBox status code.
2162	* @retval VINF_SUCCESS on success.
2163	* @param pPool The pool.
2164	* @param iUser The user index.
2165	*/
2166	static int pgmPoolCacheFreeOne(PPGMPOOL pPool, uint16_t iUser)
2167	{
2168	const PVMCC pVM = pPool->CTX_SUFF(pVM);
2169	Assert(pPool->iAgeHead != pPool->iAgeTail); /* We shouldn't be here if there < 2 cached entries! */
2170	STAM_COUNTER_INC(&pPool->StatCacheFreeUpOne);
2171
2172	/*
2173	* Select one page from the tail of the age list.
2174	*/
2175	PPGMPOOLPAGE pPage;
2176	for (unsigned iLoop = 0; ; iLoop++)
2177	{
2178	uint16_t iToFree = pPool->iAgeTail;
2179	if (iToFree == iUser && iUser != NIL_PGMPOOL_IDX)
2180	iToFree = pPool->aPages[iToFree].iAgePrev;
2181	/* This is the alternative to the SyncCR3 pgmPoolCacheUsed calls.
2182	if (pPool->aPages[iToFree].iUserHead != NIL_PGMPOOL_USER_INDEX)
2183	{
2184	uint16_t i = pPool->aPages[iToFree].iAgePrev;
2185	for (unsigned j = 0; j < 10 && i != NIL_PGMPOOL_USER_INDEX; j++, i = pPool->aPages[i].iAgePrev)
2186	{
2187	if (pPool->aPages[iToFree].iUserHead == NIL_PGMPOOL_USER_INDEX)
2188	continue;
2189	iToFree = i;
2190	break;
2191	}
2192	}
2193	*/
2194	Assert(iToFree != iUser);
2195	AssertReleaseMsg(iToFree != NIL_PGMPOOL_IDX,
2196	("iToFree=%#x (iAgeTail=%#x) iUser=%#x iLoop=%u - pPool=%p LB %#zx\n",
2197	iToFree, pPool->iAgeTail, iUser, iLoop, pPool,
2198	RT_UOFFSETOF_DYN(PGMPOOL, aPages[pPool->cMaxPages])
2199	+ pPool->cMaxUsers * sizeof(PGMPOOLUSER)
2200	+ pPool->cMaxPhysExts * sizeof(PGMPOOLPHYSEXT) ));
2201
2202	pPage = &pPool->aPages[iToFree];
2203
2204	/*
2205	* Reject any attempts at flushing the currently active shadow CR3 mapping.
2206	* Call pgmPoolCacheUsed to move the page to the head of the age list.
2207	*/
2208	if ( !pgmPoolIsPageLocked(pPage)
2209	&& pPage->idx >= PGMPOOL_IDX_FIRST /* paranoia (#6349) */)
2210	break;
2211	LogFlow(("pgmPoolCacheFreeOne: refuse CR3 mapping\n"));
2212	pgmPoolCacheUsed(pPool, pPage);
2213	AssertLogRelReturn(iLoop < 8192, VERR_PGM_POOL_TOO_MANY_LOOPS);
2214	}
2215
2216	/*
2217	* Found a usable page, flush it and return.
2218	*/
2219	int rc = pgmPoolFlushPage(pPool, pPage);
2220	/* This flush was initiated by us and not the guest, so explicitly flush the TLB. */
2221	/** @todo find out why this is necessary; pgmPoolFlushPage should trigger a flush if one is really needed. */
2222	if (rc == VINF_SUCCESS)
2223	PGM_INVL_ALL_VCPU_TLBS(pVM);
2224	return rc;
2225	}
2226
2227
2228	/**
2229	* Checks if a kind mismatch is really a page being reused
2230	* or if it's just normal remappings.
2231	*
2232	* @returns true if reused and the cached page (enmKind1) should be flushed
2233	* @returns false if not reused.
2234	* @param enmKind1 The kind of the cached page.
2235	* @param enmKind2 The kind of the requested page.
2236	*/
2237	static bool pgmPoolCacheReusedByKind(PGMPOOLKIND enmKind1, PGMPOOLKIND enmKind2)
2238	{
2239	switch (enmKind1)
2240	{
2241	/*
2242	* Never reuse them. There is no remapping in non-paging mode.
2243	*/
2244	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2245	case PGMPOOLKIND_32BIT_PD_PHYS:
2246	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2247	case PGMPOOLKIND_PAE_PD_PHYS:
2248	case PGMPOOLKIND_PAE_PDPT_PHYS:
2249	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2250	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2251	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2252	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2253	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2254	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT: /* never reuse them for other types */
2255	return false;
2256
2257	/*
2258	* It's perfectly fine to reuse these, except for PAE and non-paging stuff.
2259	*/
2260	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2261	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2262	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2263	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2264	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2265	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2266	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2267	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2268	case PGMPOOLKIND_32BIT_PD:
2269	case PGMPOOLKIND_PAE_PDPT:
2270	Assert(!PGMPOOL_PAGE_IS_KIND_NESTED(enmKind2));
2271	switch (enmKind2)
2272	{
2273	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2274	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2275	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2276	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2277	case PGMPOOLKIND_64BIT_PML4:
2278	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2279	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2280	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2281	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2282	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2283	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2284	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2285	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2286	return true;
2287	default:
2288	return false;
2289	}
2290
2291	/*
2292	* It's perfectly fine to reuse these, except for PAE and non-paging stuff.
2293	*/
2294	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2295	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2296	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2297	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2298	case PGMPOOLKIND_64BIT_PML4:
2299	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2300	Assert(!PGMPOOL_PAGE_IS_KIND_NESTED(enmKind2));
2301	switch (enmKind2)
2302	{
2303	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2304	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2305	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2306	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2307	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2308	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2309	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2310	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2311	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2312	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2313	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2314	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2315	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2316	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2317	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2318	return true;
2319	default:
2320	return false;
2321	}
2322
2323	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2324	case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
2325	case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
2326	case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
2327	return PGMPOOL_PAGE_IS_KIND_NESTED(enmKind2);
2328
2329	case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
2330	return false;
2331	#endif
2332
2333	/*
2334	* These cannot be flushed, and it's common to reuse the PDs as PTs.
2335	*/
2336	case PGMPOOLKIND_ROOT_NESTED:
2337	return false;
2338
2339	default:
2340	AssertFatalMsgFailed(("enmKind1=%d\n", enmKind1));
2341	}
2342	}
2343
2344
2345	/**
2346	* Attempts to satisfy a pgmPoolAlloc request from the cache.
2347	*
2348	* @returns VBox status code.
2349	* @retval VINF_PGM_CACHED_PAGE on success.
2350	* @retval VERR_FILE_NOT_FOUND if not found.
2351	* @param pPool The pool.
2352	* @param GCPhys The GC physical address of the page we're gonna shadow.
2353	* @param enmKind The kind of mapping.
2354	* @param enmAccess Access type for the mapping (only relevant for big pages)
2355	* @param fA20Enabled Whether the CPU has the A20 gate enabled.
2356	* @param iUser The shadow page pool index of the user table. This is
2357	* NIL_PGMPOOL_IDX for root pages.
2358	* @param iUserTable The index into the user table (shadowed). Ignored if
2359	* root page
2360	* @param ppPage Where to store the pointer to the page.
2361	*/
2362	static int pgmPoolCacheAlloc(PPGMPOOL pPool, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, bool fA20Enabled,
2363	uint16_t iUser, uint32_t iUserTable, PPPGMPOOLPAGE ppPage)
2364	{
2365	/*
2366	* Look up the GCPhys in the hash.
2367	*/
2368	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
2369	Log3(("pgmPoolCacheAlloc: %RGp kind %s iUser=%d iUserTable=%x SLOT=%d\n", GCPhys, pgmPoolPoolKindToStr(enmKind), iUser, iUserTable, i));
2370	if (i != NIL_PGMPOOL_IDX)
2371	{
2372	do
2373	{
2374	PPGMPOOLPAGE pPage = &pPool->aPages[i];
2375	Log4(("pgmPoolCacheAlloc: slot %d found page %RGp\n", i, pPage->GCPhys));
2376	if (pPage->GCPhys == GCPhys)
2377	{
2378	if ( (PGMPOOLKIND)pPage->enmKind == enmKind
2379	&& (PGMPOOLACCESS)pPage->enmAccess == enmAccess
2380	&& pPage->fA20Enabled == fA20Enabled)
2381	{
2382	/* Put it at the start of the use list to make sure pgmPoolTrackAddUser
2383	* doesn't flush it in case there are no more free use records.
2384	*/
2385	pgmPoolCacheUsed(pPool, pPage);
2386
2387	int rc = VINF_SUCCESS;
2388	if (iUser != NIL_PGMPOOL_IDX)
2389	rc = pgmPoolTrackAddUser(pPool, pPage, iUser, iUserTable);
2390	if (RT_SUCCESS(rc))
2391	{
2392	Assert((PGMPOOLKIND)pPage->enmKind == enmKind);
2393	*ppPage = pPage;
2394	if (pPage->cModifications)
2395	pPage->cModifications = 1; /* reset counter (can't use 0, or else it will be reinserted in the modified list) */
2396	STAM_COUNTER_INC(&pPool->StatCacheHits);
2397	return VINF_PGM_CACHED_PAGE;
2398	}
2399	return rc;
2400	}
2401
2402	if ((PGMPOOLKIND)pPage->enmKind != enmKind)
2403	{
2404	/*
2405	* The kind is different. In some cases we should now flush the page
2406	* as it has been reused, but in most cases this is normal remapping
2407	* of PDs as PT or big pages using the GCPhys field in a slightly
2408	* different way than the other kinds.
2409	*/
2410	if (pgmPoolCacheReusedByKind((PGMPOOLKIND)pPage->enmKind, enmKind))
2411	{
2412	STAM_COUNTER_INC(&pPool->StatCacheKindMismatches);
2413	pgmPoolFlushPage(pPool, pPage);
2414	break;
2415	}
2416	}
2417	}
2418
2419	/* next */
2420	i = pPage->iNext;
2421	} while (i != NIL_PGMPOOL_IDX);
2422	}
2423
2424	Log3(("pgmPoolCacheAlloc: Missed GCPhys=%RGp enmKind=%s\n", GCPhys, pgmPoolPoolKindToStr(enmKind)));
2425	STAM_COUNTER_INC(&pPool->StatCacheMisses);
2426	return VERR_FILE_NOT_FOUND;
2427	}
2428
2429
2430	/**
2431	* Inserts a page into the cache.
2432	*
2433	* @param pPool The pool.
2434	* @param pPage The cached page.
2435	* @param fCanBeCached Set if the page is fit for caching from the caller's point of view.
2436	*/
2437	static void pgmPoolCacheInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fCanBeCached)
2438	{
2439	/*
2440	* Insert into the GCPhys hash if the page is fit for that.
2441	*/
2442	Assert(!pPage->fCached);
2443	if (fCanBeCached)
2444	{
2445	pPage->fCached = true;
2446	pgmPoolHashInsert(pPool, pPage);
2447	Log3(("pgmPoolCacheInsert: Caching %p:{.Core=%RHp, .idx=%d, .enmKind=%s, GCPhys=%RGp}\n",
2448	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
2449	STAM_COUNTER_INC(&pPool->StatCacheCacheable);
2450	}
2451	else
2452	{
2453	Log3(("pgmPoolCacheInsert: Not caching %p:{.Core=%RHp, .idx=%d, .enmKind=%s, GCPhys=%RGp}\n",
2454	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
2455	STAM_COUNTER_INC(&pPool->StatCacheUncacheable);
2456	}
2457
2458	/*
2459	* Insert at the head of the age list.
2460	*/
2461	pPage->iAgePrev = NIL_PGMPOOL_IDX;
2462	pPage->iAgeNext = pPool->iAgeHead;
2463	if (pPool->iAgeHead != NIL_PGMPOOL_IDX)
2464	pPool->aPages[pPool->iAgeHead].iAgePrev = pPage->idx;
2465	else
2466	pPool->iAgeTail = pPage->idx;
2467	pPool->iAgeHead = pPage->idx;
2468	}
2469
2470
2471	/**
2472	* Flushes a cached page.
2473	*
2474	* @param pPool The pool.
2475	* @param pPage The cached page.
2476	*/
2477	static void pgmPoolCacheFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2478	{
2479	Log3(("pgmPoolCacheFlushPage: %RGp\n", pPage->GCPhys));
2480
2481	/*
2482	* Remove the page from the hash.
2483	*/
2484	if (pPage->fCached)
2485	{
2486	pPage->fCached = false;
2487	pgmPoolHashRemove(pPool, pPage);
2488	}
2489	else
2490	Assert(pPage->iNext == NIL_PGMPOOL_IDX);
2491
2492	/*
2493	* Remove it from the age list.
2494	*/
2495	if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
2496	pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
2497	else
2498	pPool->iAgeTail = pPage->iAgePrev;
2499	if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
2500	pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
2501	else
2502	pPool->iAgeHead = pPage->iAgeNext;
2503	pPage->iAgeNext = NIL_PGMPOOL_IDX;
2504	pPage->iAgePrev = NIL_PGMPOOL_IDX;
2505	}
2506
2507
2508	/**
2509	* Looks for pages sharing the monitor.
2510	*
2511	* @returns Pointer to the head page.
2512	* @returns NULL if not found.
2513	* @param pPool The Pool
2514	* @param pNewPage The page which is going to be monitored.
2515	*/
2516	static PPGMPOOLPAGE pgmPoolMonitorGetPageByGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pNewPage)
2517	{
2518	/*
2519	* Look up the GCPhys in the hash.
2520	*/
2521	RTGCPHYS GCPhys = pNewPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
2522	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
2523	if (i == NIL_PGMPOOL_IDX)
2524	return NULL;
2525	do
2526	{
2527	PPGMPOOLPAGE pPage = &pPool->aPages[i];
2528	if ( pPage->GCPhys - GCPhys < PAGE_SIZE
2529	&& pPage != pNewPage)
2530	{
2531	switch (pPage->enmKind)
2532	{
2533	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2534	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2535	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2536	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2537	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2538	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2539	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2540	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2541	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2542	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2543	case PGMPOOLKIND_64BIT_PML4:
2544	case PGMPOOLKIND_32BIT_PD:
2545	case PGMPOOLKIND_PAE_PDPT:
2546	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2547	case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
2548	case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
2549	case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
2550	#endif
2551	{
2552	/* find the head */
2553	while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
2554	{
2555	Assert(pPage->iMonitoredPrev != pPage->idx);
2556	pPage = &pPool->aPages[pPage->iMonitoredPrev];
2557	}
2558	return pPage;
2559	}
2560
2561	/* ignore, no monitoring. */
2562	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2563	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2564	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2565	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2566	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2567	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2568	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2569	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2570	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2571	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2572	case PGMPOOLKIND_ROOT_NESTED:
2573	case PGMPOOLKIND_PAE_PD_PHYS:
2574	case PGMPOOLKIND_PAE_PDPT_PHYS:
2575	case PGMPOOLKIND_32BIT_PD_PHYS:
2576	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
2577	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2578	case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
2579	#endif
2580	break;
2581	default:
2582	AssertFatalMsgFailed(("enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
2583	}
2584	}
2585
2586	/* next */
2587	i = pPage->iNext;
2588	} while (i != NIL_PGMPOOL_IDX);
2589	return NULL;
2590	}
2591
2592
2593	/**
2594	* Enabled write monitoring of a guest page.
2595	*
2596	* @returns VBox status code.
2597	* @retval VINF_SUCCESS on success.
2598	* @param pPool The pool.
2599	* @param pPage The cached page.
2600	*/
2601	static int pgmPoolMonitorInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2602	{
2603	LogFlow(("pgmPoolMonitorInsert %RGp\n", pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK));
2604
2605	/*
2606	* Filter out the relevant kinds.
2607	*/
2608	switch (pPage->enmKind)
2609	{
2610	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2611	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2612	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2613	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2614	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2615	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2616	case PGMPOOLKIND_64BIT_PML4:
2617	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2618	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2619	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2620	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2621	case PGMPOOLKIND_32BIT_PD:
2622	case PGMPOOLKIND_PAE_PDPT:
2623	break;
2624
2625	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2626	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2627	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2628	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2629	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2630	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2631	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2632	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2633	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2634	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2635	case PGMPOOLKIND_ROOT_NESTED:
2636	/* Nothing to monitor here. */
2637	return VINF_SUCCESS;
2638
2639	case PGMPOOLKIND_32BIT_PD_PHYS:
2640	case PGMPOOLKIND_PAE_PDPT_PHYS:
2641	case PGMPOOLKIND_PAE_PD_PHYS:
2642	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
2643	/* Nothing to monitor here. */
2644	return VINF_SUCCESS;
2645
2646	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2647	case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
2648	case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
2649	case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
2650	break;
2651
2652	case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
2653	/* Nothing to monitor here. */
2654	return VINF_SUCCESS;
2655	#endif
2656
2657	default:
2658	AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
2659	}
2660
2661	/*
2662	* Install handler.
2663	*/
2664	int rc;
2665	PPGMPOOLPAGE pPageHead = pgmPoolMonitorGetPageByGCPhys(pPool, pPage);
2666	if (pPageHead)
2667	{
2668	Assert(pPageHead != pPage); Assert(pPageHead->iMonitoredNext != pPage->idx);
2669	Assert(pPageHead->iMonitoredPrev != pPage->idx);
2670
2671	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2672	if (pPageHead->fDirty)
2673	pgmPoolFlushDirtyPage(pPool->CTX_SUFF(pVM), pPool, pPageHead->idxDirtyEntry, false /* do not remove */);
2674	#endif
2675
2676	pPage->iMonitoredPrev = pPageHead->idx;
2677	pPage->iMonitoredNext = pPageHead->iMonitoredNext;
2678	if (pPageHead->iMonitoredNext != NIL_PGMPOOL_IDX)
2679	pPool->aPages[pPageHead->iMonitoredNext].iMonitoredPrev = pPage->idx;
2680	pPageHead->iMonitoredNext = pPage->idx;
2681	rc = VINF_SUCCESS;
2682	if (PGMPOOL_PAGE_IS_NESTED(pPage))
2683	Log7Func(("Adding to monitoring list GCPhysPage=%RGp\n", pPage->GCPhys));
2684	}
2685	else
2686	{
2687	if (PGMPOOL_PAGE_IS_NESTED(pPage))
2688	Log7Func(("Started monitoring GCPhysPage=%RGp HCPhys=%RHp enmKind=%s\n", pPage->GCPhys, pPage->Core.Key, pgmPoolPoolKindToStr(pPage->enmKind)));
2689
2690	Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX); Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
2691	PVMCC pVM = pPool->CTX_SUFF(pVM);
2692	const RTGCPHYS GCPhysPage = pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
2693	rc = PGMHandlerPhysicalRegister(pVM, GCPhysPage, GCPhysPage + PAGE_OFFSET_MASK, pPool->hAccessHandlerType,
2694	pPage - &pPool->aPages[0], NIL_RTR3PTR /pszDesc/);
2695	/** @todo we should probably deal with out-of-memory conditions here, but for now increasing
2696	* the heap size should suffice. */
2697	AssertFatalMsgRC(rc, ("PGMHandlerPhysicalRegisterEx %RGp failed with %Rrc\n", GCPhysPage, rc));
2698	PVMCPU pVCpu = VMMGetCpu(pVM);
2699	AssertFatalMsg(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL) \|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), ("fSyncFlags=%x syncff=%d\n", pVCpu->pgm.s.fSyncFlags, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)));
2700	}
2701	pPage->fMonitored = true;
2702	return rc;
2703	}
2704
2705
2706	/**
2707	* Disables write monitoring of a guest page.
2708	*
2709	* @returns VBox status code.
2710	* @retval VINF_SUCCESS on success.
2711	* @param pPool The pool.
2712	* @param pPage The cached page.
2713	*/
2714	static int pgmPoolMonitorFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2715	{
2716	/*
2717	* Filter out the relevant kinds.
2718	*/
2719	switch (pPage->enmKind)
2720	{
2721	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2722	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2723	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2724	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2725	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2726	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2727	case PGMPOOLKIND_64BIT_PML4:
2728	case PGMPOOLKIND_32BIT_PD:
2729	case PGMPOOLKIND_PAE_PDPT:
2730	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2731	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2732	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2733	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2734	break;
2735
2736	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2737	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2738	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2739	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2740	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2741	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2742	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2743	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2744	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2745	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2746	case PGMPOOLKIND_ROOT_NESTED:
2747	case PGMPOOLKIND_PAE_PD_PHYS:
2748	case PGMPOOLKIND_PAE_PDPT_PHYS:
2749	case PGMPOOLKIND_32BIT_PD_PHYS:
2750	/* Nothing to monitor here. */
2751	Assert(!pPage->fMonitored);
2752	return VINF_SUCCESS;
2753
2754	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2755	case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
2756	case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
2757	case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
2758	break;
2759
2760	case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
2761	/* Nothing to monitor here. */
2762	Assert(!pPage->fMonitored);
2763	return VINF_SUCCESS;
2764	#endif
2765
2766	default:
2767	AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
2768	}
2769	Assert(pPage->fMonitored);
2770
2771	/*
2772	* Remove the page from the monitored list or uninstall it if last.
2773	*/
2774	const PVMCC pVM = pPool->CTX_SUFF(pVM);
2775	int rc;
2776	if ( pPage->iMonitoredNext != NIL_PGMPOOL_IDX
2777	\|\| pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
2778	{
2779	if (pPage->iMonitoredPrev == NIL_PGMPOOL_IDX)
2780	{
2781	PPGMPOOLPAGE pNewHead = &pPool->aPages[pPage->iMonitoredNext];
2782	pNewHead->iMonitoredPrev = NIL_PGMPOOL_IDX;
2783	rc = PGMHandlerPhysicalChangeUserArg(pVM, pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK, pPage->iMonitoredNext);
2784
2785	AssertFatalRCSuccess(rc);
2786	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
2787	}
2788	else
2789	{
2790	pPool->aPages[pPage->iMonitoredPrev].iMonitoredNext = pPage->iMonitoredNext;
2791	if (pPage->iMonitoredNext != NIL_PGMPOOL_IDX)
2792	{
2793	pPool->aPages[pPage->iMonitoredNext].iMonitoredPrev = pPage->iMonitoredPrev;
2794	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
2795	}
2796	pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
2797	rc = VINF_SUCCESS;
2798	}
2799	}
2800	else
2801	{
2802	rc = PGMHandlerPhysicalDeregister(pVM, pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
2803	AssertFatalRC(rc);
2804	PVMCPU pVCpu = VMMGetCpu(pVM);
2805	AssertFatalMsg(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL) \|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3),
2806	("%#x %#x\n", pVCpu->pgm.s.fSyncFlags, pVM->fGlobalForcedActions));
2807	}
2808	pPage->fMonitored = false;
2809
2810	/*
2811	* Remove it from the list of modified pages (if in it).
2812	*/
2813	pgmPoolMonitorModifiedRemove(pPool, pPage);
2814
2815	if (PGMPOOL_PAGE_IS_NESTED(pPage))
2816	Log7Func(("Stopped monitoring %RGp\n", pPage->GCPhys));
2817
2818	return rc;
2819	}
2820
2821
2822	/**
2823	* Inserts the page into the list of modified pages.
2824	*
2825	* @param pPool The pool.
2826	* @param pPage The page.
2827	*/
2828	void pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2829	{
2830	Log3(("pgmPoolMonitorModifiedInsert: idx=%d\n", pPage->idx));
2831	AssertMsg( pPage->iModifiedNext == NIL_PGMPOOL_IDX
2832	&& pPage->iModifiedPrev == NIL_PGMPOOL_IDX
2833	&& pPool->iModifiedHead != pPage->idx,
2834	("Next=%d Prev=%d idx=%d cModifications=%d Head=%d cModifiedPages=%d\n",
2835	pPage->iModifiedNext, pPage->iModifiedPrev, pPage->idx, pPage->cModifications,
2836	pPool->iModifiedHead, pPool->cModifiedPages));
2837
2838	pPage->iModifiedNext = pPool->iModifiedHead;
2839	if (pPool->iModifiedHead != NIL_PGMPOOL_IDX)
2840	pPool->aPages[pPool->iModifiedHead].iModifiedPrev = pPage->idx;
2841	pPool->iModifiedHead = pPage->idx;
2842	pPool->cModifiedPages++;
2843	#ifdef VBOX_WITH_STATISTICS
2844	if (pPool->cModifiedPages > pPool->cModifiedPagesHigh)
2845	pPool->cModifiedPagesHigh = pPool->cModifiedPages;
2846	#endif
2847	}
2848
2849
2850	/**
2851	* Removes the page from the list of modified pages and resets the
2852	* modification counter.
2853	*
2854	* @param pPool The pool.
2855	* @param pPage The page which is believed to be in the list of modified pages.
2856	*/
2857	static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2858	{
2859	Log3(("pgmPoolMonitorModifiedRemove: idx=%d cModifications=%d\n", pPage->idx, pPage->cModifications));
2860	if (pPool->iModifiedHead == pPage->idx)
2861	{
2862	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
2863	pPool->iModifiedHead = pPage->iModifiedNext;
2864	if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
2865	{
2866	pPool->aPages[pPage->iModifiedNext].iModifiedPrev = NIL_PGMPOOL_IDX;
2867	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2868	}
2869	pPool->cModifiedPages--;
2870	}
2871	else if (pPage->iModifiedPrev != NIL_PGMPOOL_IDX)
2872	{
2873	pPool->aPages[pPage->iModifiedPrev].iModifiedNext = pPage->iModifiedNext;
2874	if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
2875	{
2876	pPool->aPages[pPage->iModifiedNext].iModifiedPrev = pPage->iModifiedPrev;
2877	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2878	}
2879	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2880	pPool->cModifiedPages--;
2881	}
2882	else
2883	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
2884	pPage->cModifications = 0;
2885	}
2886
2887
2888	/**
2889	* Zaps the list of modified pages, resetting their modification counters in the process.
2890	*
2891	* @param pVM The cross context VM structure.
2892	*/
2893	static void pgmPoolMonitorModifiedClearAll(PVMCC pVM)
2894	{
2895	PGM_LOCK_VOID(pVM);
2896	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2897	LogFlow(("pgmPoolMonitorModifiedClearAll: cModifiedPages=%d\n", pPool->cModifiedPages));
2898
2899	unsigned cPages = 0; NOREF(cPages);
2900
2901	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2902	pgmPoolResetDirtyPages(pVM);
2903	#endif
2904
2905	uint16_t idx = pPool->iModifiedHead;
2906	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
2907	while (idx != NIL_PGMPOOL_IDX)
2908	{
2909	PPGMPOOLPAGE pPage = &pPool->aPages[idx];
2910	idx = pPage->iModifiedNext;
2911	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2912	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2913	pPage->cModifications = 0;
2914	Assert(++cPages);
2915	}
2916	AssertMsg(cPages == pPool->cModifiedPages, ("%d != %d\n", cPages, pPool->cModifiedPages));
2917	pPool->cModifiedPages = 0;
2918	PGM_UNLOCK(pVM);
2919	}
2920
2921
2922	/**
2923	* Handle SyncCR3 pool tasks
2924	*
2925	* @returns VBox status code.
2926	* @retval VINF_SUCCESS if successfully added.
2927	* @retval VINF_PGM_SYNC_CR3 is it needs to be deferred to ring 3 (GC only)
2928	* @param pVCpu The cross context virtual CPU structure.
2929	* @remark Should only be used when monitoring is available, thus placed in
2930	* the PGMPOOL_WITH_MONITORING \#ifdef.
2931	*/
2932	int pgmPoolSyncCR3(PVMCPUCC pVCpu)
2933	{
2934	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2935	LogFlow(("pgmPoolSyncCR3 fSyncFlags=%x\n", pVCpu->pgm.s.fSyncFlags));
2936
2937	/*
2938	* When monitoring shadowed pages, we reset the modification counters on CR3 sync.
2939	* Occasionally we will have to clear all the shadow page tables because we wanted
2940	* to monitor a page which was mapped by too many shadowed page tables. This operation
2941	* sometimes referred to as a 'lightweight flush'.
2942	*/
2943	# ifdef IN_RING3 /* Don't flush in ring-0 or raw mode, it's taking too long. */
2944	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2945	pgmR3PoolClearAll(pVM, false /fFlushRemTlb/);
2946	# else /* !IN_RING3 */
2947	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2948	{
2949	Log(("SyncCR3: PGM_SYNC_CLEAR_PGM_POOL is set -> VINF_PGM_SYNC_CR3\n"));
2950	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
2951
2952	/* Make sure all other VCPUs return to ring 3. */
2953	if (pVM->cCpus > 1)
2954	{
2955	VM_FF_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING);
2956	PGM_INVL_ALL_VCPU_TLBS(pVM);
2957	}
2958	return VINF_PGM_SYNC_CR3;
2959	}
2960	# endif /* !IN_RING3 */
2961	else
2962	{
2963	pgmPoolMonitorModifiedClearAll(pVM);
2964
2965	/* pgmPoolMonitorModifiedClearAll can cause a pgm pool flush (dirty page clearing), so make sure we handle this! */
2966	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2967	{
2968	Log(("pgmPoolMonitorModifiedClearAll caused a pgm flush -> call pgmPoolSyncCR3 again!\n"));
2969	return pgmPoolSyncCR3(pVCpu);
2970	}
2971	}
2972	return VINF_SUCCESS;
2973	}
2974
2975
2976	/**
2977	* Frees up at least one user entry.
2978	*
2979	* @returns VBox status code.
2980	* @retval VINF_SUCCESS if successfully added.
2981	*
2982	* @param pPool The pool.
2983	* @param iUser The user index.
2984	*/
2985	static int pgmPoolTrackFreeOneUser(PPGMPOOL pPool, uint16_t iUser)
2986	{
2987	STAM_COUNTER_INC(&pPool->StatTrackFreeUpOneUser);
2988	/*
2989	* Just free cached pages in a braindead fashion.
2990	*/
2991	/** @todo walk the age list backwards and free the first with usage. */
2992	int rc = VINF_SUCCESS;
2993	do
2994	{
2995	int rc2 = pgmPoolCacheFreeOne(pPool, iUser);
2996	if (RT_FAILURE(rc2) && rc == VINF_SUCCESS)
2997	rc = rc2;
2998	} while (pPool->iUserFreeHead == NIL_PGMPOOL_USER_INDEX);
2999	return rc;
3000	}
3001
3002
3003	/**
3004	* Inserts a page into the cache.
3005	*
3006	* This will create user node for the page, insert it into the GCPhys
3007	* hash, and insert it into the age list.
3008	*
3009	* @returns VBox status code.
3010	* @retval VINF_SUCCESS if successfully added.
3011	*
3012	* @param pPool The pool.
3013	* @param pPage The cached page.
3014	* @param GCPhys The GC physical address of the page we're gonna shadow.
3015	* @param iUser The user index.
3016	* @param iUserTable The user table index.
3017	*/
3018	DECLINLINE(int) pgmPoolTrackInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhys, uint16_t iUser, uint32_t iUserTable)
3019	{
3020	int rc = VINF_SUCCESS;
3021	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
3022
3023	LogFlow(("pgmPoolTrackInsert GCPhys=%RGp iUser=%d iUserTable=%x\n", GCPhys, iUser, iUserTable)); RT_NOREF_PV(GCPhys);
3024
3025	if (iUser != NIL_PGMPOOL_IDX)
3026	{
3027	#ifdef VBOX_STRICT
3028	/*
3029	* Check that the entry doesn't already exists.
3030	*/
3031	if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
3032	{
3033	uint16_t i = pPage->iUserHead;
3034	do
3035	{
3036	Assert(i < pPool->cMaxUsers);
3037	AssertMsg(paUsers[i].iUser != iUser \|\| paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
3038	i = paUsers[i].iNext;
3039	} while (i != NIL_PGMPOOL_USER_INDEX);
3040	}
3041	#endif
3042
3043	/*
3044	* Find free a user node.
3045	*/
3046	uint16_t i = pPool->iUserFreeHead;
3047	if (i == NIL_PGMPOOL_USER_INDEX)
3048	{
3049	rc = pgmPoolTrackFreeOneUser(pPool, iUser);
3050	if (RT_FAILURE(rc))
3051	return rc;
3052	i = pPool->iUserFreeHead;
3053	}
3054
3055	/*
3056	* Unlink the user node from the free list,
3057	* initialize and insert it into the user list.
3058	*/
3059	pPool->iUserFreeHead = paUsers[i].iNext;
3060	paUsers[i].iNext = NIL_PGMPOOL_USER_INDEX;
3061	paUsers[i].iUser = iUser;
3062	paUsers[i].iUserTable = iUserTable;
3063	pPage->iUserHead = i;
3064	}
3065	else
3066	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
3067
3068
3069	/*
3070	* Insert into cache and enable monitoring of the guest page if enabled.
3071	*
3072	* Until we implement caching of all levels, including the CR3 one, we'll
3073	* have to make sure we don't try monitor & cache any recursive reuse of
3074	* a monitored CR3 page. Because all windows versions are doing this we'll
3075	* have to be able to do combined access monitoring, CR3 + PT and
3076	* PD + PT (guest PAE).
3077	*
3078	* Update:
3079	* We're now cooperating with the CR3 monitor if an uncachable page is found.
3080	*/
3081	const bool fCanBeMonitored = true;
3082	pgmPoolCacheInsert(pPool, pPage, fCanBeMonitored); /* This can be expanded. */
3083	if (fCanBeMonitored)
3084	{
3085	rc = pgmPoolMonitorInsert(pPool, pPage);
3086	AssertRC(rc);
3087	}
3088	return rc;
3089	}
3090
3091
3092	/**
3093	* Adds a user reference to a page.
3094	*
3095	* This will move the page to the head of the
3096	*
3097	* @returns VBox status code.
3098	* @retval VINF_SUCCESS if successfully added.
3099	*
3100	* @param pPool The pool.
3101	* @param pPage The cached page.
3102	* @param iUser The user index.
3103	* @param iUserTable The user table.
3104	*/
3105	static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
3106	{
3107	Log3(("pgmPoolTrackAddUser: GCPhys=%RGp iUser=%x iUserTable=%x\n", pPage->GCPhys, iUser, iUserTable));
3108	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
3109	Assert(iUser != NIL_PGMPOOL_IDX);
3110
3111	# ifdef VBOX_STRICT
3112	/*
3113	* Check that the entry doesn't already exists. We only allow multiple
3114	* users of top-level paging structures (SHW_POOL_ROOT_IDX).
3115	*/
3116	if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
3117	{
3118	uint16_t i = pPage->iUserHead;
3119	do
3120	{
3121	Assert(i < pPool->cMaxUsers);
3122	/** @todo this assertion looks odd... Shouldn't it be && here? */
3123	AssertMsg(paUsers[i].iUser != iUser \|\| paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
3124	i = paUsers[i].iNext;
3125	} while (i != NIL_PGMPOOL_USER_INDEX);
3126	}
3127	# endif
3128
3129	/*
3130	* Allocate a user node.
3131	*/
3132	uint16_t i = pPool->iUserFreeHead;
3133	if (i == NIL_PGMPOOL_USER_INDEX)
3134	{
3135	int rc = pgmPoolTrackFreeOneUser(pPool, iUser);
3136	if (RT_FAILURE(rc))
3137	return rc;
3138	i = pPool->iUserFreeHead;
3139	}
3140	pPool->iUserFreeHead = paUsers[i].iNext;
3141
3142	/*
3143	* Initialize the user node and insert it.
3144	*/
3145	paUsers[i].iNext = pPage->iUserHead;
3146	paUsers[i].iUser = iUser;
3147	paUsers[i].iUserTable = iUserTable;
3148	pPage->iUserHead = i;
3149
3150	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
3151	if (pPage->fDirty)
3152	pgmPoolFlushDirtyPage(pPool->CTX_SUFF(pVM), pPool, pPage->idxDirtyEntry, false /* do not remove */);
3153	# endif
3154
3155	/*
3156	* Tell the cache to update its replacement stats for this page.
3157	*/
3158	pgmPoolCacheUsed(pPool, pPage);
3159	return VINF_SUCCESS;
3160	}
3161
3162
3163	/**
3164	* Frees a user record associated with a page.
3165	*
3166	* This does not clear the entry in the user table, it simply replaces the
3167	* user record to the chain of free records.
3168	*
3169	* @param pPool The pool.
3170	* @param pPage The shadow page.
3171	* @param iUser The shadow page pool index of the user table.
3172	* @param iUserTable The index into the user table (shadowed).
3173	*
3174	* @remarks Don't call this for root pages.
3175	*/
3176	static void pgmPoolTrackFreeUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
3177	{
3178	Log3(("pgmPoolTrackFreeUser %RGp %x %x\n", pPage->GCPhys, iUser, iUserTable));
3179	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
3180	Assert(iUser != NIL_PGMPOOL_IDX);
3181
3182	/*
3183	* Unlink and free the specified user entry.
3184	*/
3185
3186	/* Special: For PAE and 32-bit paging, there is usually no more than one user. */
3187	uint16_t i = pPage->iUserHead;
3188	if ( i != NIL_PGMPOOL_USER_INDEX
3189	&& paUsers[i].iUser == iUser
3190	&& paUsers[i].iUserTable == iUserTable)
3191	{
3192	pPage->iUserHead = paUsers[i].iNext;
3193
3194	paUsers[i].iUser = NIL_PGMPOOL_IDX;
3195	paUsers[i].iNext = pPool->iUserFreeHead;
3196	pPool->iUserFreeHead = i;
3197	return;
3198	}
3199
3200	/* General: Linear search. */
3201	uint16_t iPrev = NIL_PGMPOOL_USER_INDEX;
3202	while (i != NIL_PGMPOOL_USER_INDEX)
3203	{
3204	if ( paUsers[i].iUser == iUser
3205	&& paUsers[i].iUserTable == iUserTable)
3206	{
3207	if (iPrev != NIL_PGMPOOL_USER_INDEX)
3208	paUsers[iPrev].iNext = paUsers[i].iNext;
3209	else
3210	pPage->iUserHead = paUsers[i].iNext;
3211
3212	paUsers[i].iUser = NIL_PGMPOOL_IDX;
3213	paUsers[i].iNext = pPool->iUserFreeHead;
3214	pPool->iUserFreeHead = i;
3215	return;
3216	}
3217	iPrev = i;
3218	i = paUsers[i].iNext;
3219	}
3220
3221	/* Fatal: didn't find it */
3222	AssertFatalMsgFailed(("Didn't find the user entry! iUser=%d iUserTable=%#x GCPhys=%RGp\n",
3223	iUser, iUserTable, pPage->GCPhys));
3224	}
3225
3226
3227	#if 0 /* unused */
3228	/**
3229	* Gets the entry size of a shadow table.
3230	*
3231	* @param enmKind The kind of page.
3232	*
3233	* @returns The size of the entry in bytes. That is, 4 or 8.
3234	* @returns If the kind is not for a table, an assertion is raised and 0 is
3235	* returned.
3236	*/
3237	DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind)
3238	{
3239	switch (enmKind)
3240	{
3241	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3242	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3243	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3244	case PGMPOOLKIND_32BIT_PD:
3245	case PGMPOOLKIND_32BIT_PD_PHYS:
3246	return 4;
3247
3248	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3249	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3250	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3251	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3252	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3253	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3254	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3255	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3256	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3257	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3258	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3259	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3260	case PGMPOOLKIND_64BIT_PML4:
3261	case PGMPOOLKIND_PAE_PDPT:
3262	case PGMPOOLKIND_ROOT_NESTED:
3263	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3264	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3265	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3266	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3267	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
3268	case PGMPOOLKIND_PAE_PD_PHYS:
3269	case PGMPOOLKIND_PAE_PDPT_PHYS:
3270	return 8;
3271
3272	default:
3273	AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
3274	}
3275	}
3276	#endif /* unused */
3277
3278	#if 0 /* unused */
3279	/**
3280	* Gets the entry size of a guest table.
3281	*
3282	* @param enmKind The kind of page.
3283	*
3284	* @returns The size of the entry in bytes. That is, 0, 4 or 8.
3285	* @returns If the kind is not for a table, an assertion is raised and 0 is
3286	* returned.
3287	*/
3288	DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind)
3289	{
3290	switch (enmKind)
3291	{
3292	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3293	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3294	case PGMPOOLKIND_32BIT_PD:
3295	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3296	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3297	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3298	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3299	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3300	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3301	return 4;
3302
3303	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3304	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3305	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3306	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3307	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3308	case PGMPOOLKIND_64BIT_PML4:
3309	case PGMPOOLKIND_PAE_PDPT:
3310	return 8;
3311
3312	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3313	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3314	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3315	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3316	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3317	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3318	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
3319	case PGMPOOLKIND_ROOT_NESTED:
3320	case PGMPOOLKIND_PAE_PD_PHYS:
3321	case PGMPOOLKIND_PAE_PDPT_PHYS:
3322	case PGMPOOLKIND_32BIT_PD_PHYS:
3323	/** @todo can we return 0? (nobody is calling this...) */
3324	AssertFailed();
3325	return 0;
3326
3327	default:
3328	AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
3329	}
3330	}
3331	#endif /* unused */
3332
3333
3334	/**
3335	* Checks one shadow page table entry for a mapping of a physical page.
3336	*
3337	* @returns true / false indicating removal of all relevant PTEs
3338	*
3339	* @param pVM The cross context VM structure.
3340	* @param pPhysPage The guest page in question.
3341	* @param fFlushPTEs Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
3342	* @param iShw The shadow page table.
3343	* @param iPte Page table entry or NIL_PGMPOOL_PHYSEXT_IDX_PTE if unknown
3344	*/
3345	static bool pgmPoolTrackFlushGCPhysPTInt(PVM pVM, PCPGMPAGE pPhysPage, bool fFlushPTEs, uint16_t iShw, uint16_t iPte)
3346	{
3347	LogFlow(("pgmPoolTrackFlushGCPhysPTInt: pPhysPage=%RHp iShw=%d iPte=%d\n", PGM_PAGE_GET_HCPHYS(pPhysPage), iShw, iPte));
3348	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3349	bool fRet = false;
3350
3351	/*
3352	* Assert sanity.
3353	*/
3354	Assert(iPte != NIL_PGMPOOL_PHYSEXT_IDX_PTE);
3355	AssertFatalMsg(iShw < pPool->cCurPages && iShw != NIL_PGMPOOL_IDX, ("iShw=%d\n", iShw));
3356	PPGMPOOLPAGE pPage = &pPool->aPages[iShw];
3357
3358	/*
3359	* Then, clear the actual mappings to the page in the shadow PT.
3360	*/
3361	switch (pPage->enmKind)
3362	{
3363	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3364	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3365	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3366	{
3367	const uint32_t u32 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
3368	PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3369	uint32_t u32AndMask = 0;
3370	uint32_t u32OrMask = 0;
3371
3372	if (!fFlushPTEs)
3373	{
3374	switch (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage))
3375	{
3376	case PGM_PAGE_HNDL_PHYS_STATE_NONE: /* No handler installed. */
3377	case PGM_PAGE_HNDL_PHYS_STATE_DISABLED: /* Monitoring is temporarily disabled. */
3378	u32OrMask = X86_PTE_RW;
3379	u32AndMask = UINT32_MAX;
3380	fRet = true;
3381	STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
3382	break;
3383
3384	case PGM_PAGE_HNDL_PHYS_STATE_WRITE: /* Write access is monitored. */
3385	u32OrMask = 0;
3386	u32AndMask = ~X86_PTE_RW;
3387	fRet = true;
3388	STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
3389	break;
3390	default:
3391	/* We will end up here when called with an "ALL" access handler. */
3392	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3393	break;
3394	}
3395	}
3396	else
3397	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3398
3399	/* Update the counter if we're removing references. */
3400	if (!u32AndMask)
3401	{
3402	Assert(pPage->cPresent);
3403	Assert(pPool->cPresent);
3404	pPage->cPresent--;
3405	pPool->cPresent--;
3406	}
3407
3408	if ((pPT->a[iPte].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
3409	{
3410	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX32\n", iPte, pPT->a[iPte]));
3411	X86PTE Pte;
3412	Pte.u = (pPT->a[iPte].u & u32AndMask) \| u32OrMask;
3413	if (Pte.u & PGM_PTFLAGS_TRACK_DIRTY)
3414	Pte.u &= ~(X86PGUINT)X86_PTE_RW; /* need to disallow writes when dirty bit tracking is still active. */
3415
3416	ASMAtomicWriteU32(&pPT->a[iPte].u, Pte.u);
3417	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3418	return fRet;
3419	}
3420	#ifdef LOG_ENABLED
3421	Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3422	for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPT->a); i++)
3423	if ((pPT->a[i].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
3424	{
3425	Log(("i=%d cFound=%d\n", i, ++cFound));
3426	}
3427	#endif
3428	AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d u32=%RX32 poolkind=%x\n", pPage->iFirstPresent, pPage->cPresent, u32, pPage->enmKind));
3429	/PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);/
3430	break;
3431	}
3432
3433	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3434	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3435	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3436	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3437	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3438	case PGMPOOLKIND_EPT_PT_FOR_PHYS: /* physical mask the same as PAE; RW bit as well; be careful! */
3439	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3440	case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
3441	#endif
3442	{
3443	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
3444	PPGMSHWPTPAE pPT = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3445	uint64_t u64OrMask = 0;
3446	uint64_t u64AndMask = 0;
3447
3448	if (!fFlushPTEs)
3449	{
3450	switch (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage))
3451	{
3452	case PGM_PAGE_HNDL_PHYS_STATE_NONE: /* No handler installed. */
3453	case PGM_PAGE_HNDL_PHYS_STATE_DISABLED: /* Monitoring is temporarily disabled. */
3454	u64OrMask = X86_PTE_RW;
3455	u64AndMask = UINT64_MAX;
3456	fRet = true;
3457	STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
3458	break;
3459
3460	case PGM_PAGE_HNDL_PHYS_STATE_WRITE: /* Write access is monitored. */
3461	u64OrMask = 0;
3462	u64AndMask = ~(uint64_t)X86_PTE_RW;
3463	fRet = true;
3464	STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
3465	break;
3466
3467	default:
3468	/* We will end up here when called with an "ALL" access handler. */
3469	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3470	break;
3471	}
3472	}
3473	else
3474	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3475
3476	/* Update the counter if we're removing references. */
3477	if (!u64AndMask)
3478	{
3479	Assert(pPage->cPresent);
3480	Assert(pPool->cPresent);
3481	pPage->cPresent--;
3482	pPool->cPresent--;
3483	}
3484
3485	if ((PGMSHWPTEPAE_GET_U(pPT->a[iPte]) & (X86_PTE_PAE_PG_MASK \| X86_PTE_P \| X86_PTE_PAE_MBZ_MASK_NX)) == u64)
3486	{
3487	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX64\n", iPte, PGMSHWPTEPAE_GET_LOG(pPT->a[iPte])));
3488	X86PTEPAE Pte;
3489	Pte.u = (PGMSHWPTEPAE_GET_U(pPT->a[iPte]) & u64AndMask) \| u64OrMask;
3490	if (Pte.u & PGM_PTFLAGS_TRACK_DIRTY)
3491	Pte.u &= ~(X86PGPAEUINT)X86_PTE_RW; /* need to disallow writes when dirty bit tracking is still active. */
3492
3493	PGMSHWPTEPAE_ATOMIC_SET(pPT->a[iPte], Pte.u);
3494	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3495	return fRet;
3496	}
3497	#ifdef LOG_ENABLED
3498	Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3499	Log(("Found %RX64 expected %RX64\n", PGMSHWPTEPAE_GET_U(pPT->a[iPte]) & (X86_PTE_PAE_PG_MASK \| X86_PTE_P \| X86_PTE_PAE_MBZ_MASK_NX), u64));
3500	for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPT->a); i++)
3501	if ((PGMSHWPTEPAE_GET_U(pPT->a[i]) & (X86_PTE_PAE_PG_MASK \| X86_PTE_P \| X86_PTE_PAE_MBZ_MASK_NX)) == u64)
3502	Log(("i=%d cFound=%d\n", i, ++cFound));
3503	#endif
3504	AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d u64=%RX64 poolkind=%x iPte=%d PT=%RX64\n", pPage->iFirstPresent, pPage->cPresent, u64, pPage->enmKind, iPte, PGMSHWPTEPAE_GET_LOG(pPT->a[iPte])));
3505	/PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);/
3506	break;
3507	}
3508
3509	#ifdef PGM_WITH_LARGE_PAGES
3510	/* Large page case only. */
3511	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3512	{
3513	Assert(pVM->pgm.s.fNestedPaging);
3514
3515	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PDE4M_P \| X86_PDE4M_PS;
3516	PEPTPD pPD = (PEPTPD)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3517
3518	if ((pPD->a[iPte].u & (EPT_PDE2M_PG_MASK \| X86_PDE4M_P \| X86_PDE4M_PS)) == u64)
3519	{
3520	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pde=%RX64\n", iPte, pPD->a[iPte]));
3521	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3522	pPD->a[iPte].u = 0;
3523	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);
3524
3525	/* Update the counter as we're removing references. */
3526	Assert(pPage->cPresent);
3527	Assert(pPool->cPresent);
3528	pPage->cPresent--;
3529	pPool->cPresent--;
3530
3531	return fRet;
3532	}
3533	# ifdef LOG_ENABLED
3534	Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3535	for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPD->a); i++)
3536	if ((pPD->a[i].u & (EPT_PDE2M_PG_MASK \| X86_PDE4M_P \| X86_PDE4M_PS)) == u64)
3537	Log(("i=%d cFound=%d\n", i, ++cFound));
3538	# endif
3539	AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3540	/PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);/
3541	break;
3542	}
3543
3544	/* AMD-V nested paging / /* @todo merge with EPT as we only check the parts that are identical. */
3545	case PGMPOOLKIND_PAE_PD_PHYS:
3546	{
3547	Assert(pVM->pgm.s.fNestedPaging);
3548
3549	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PDE4M_P \| X86_PDE4M_PS;
3550	PX86PDPAE pPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3551
3552	if ((pPD->a[iPte].u & (X86_PDE2M_PAE_PG_MASK \| X86_PDE4M_P \| X86_PDE4M_PS)) == u64)
3553	{
3554	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pde=%RX64\n", iPte, pPD->a[iPte]));
3555	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3556	pPD->a[iPte].u = 0;
3557	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);
3558
3559	/* Update the counter as we're removing references. */
3560	Assert(pPage->cPresent);
3561	Assert(pPool->cPresent);
3562	pPage->cPresent--;
3563	pPool->cPresent--;
3564	return fRet;
3565	}
3566	# ifdef LOG_ENABLED
3567	Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3568	for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPD->a); i++)
3569	if ((pPD->a[i].u & (X86_PDE2M_PAE_PG_MASK \| X86_PDE4M_P \| X86_PDE4M_PS)) == u64)
3570	Log(("i=%d cFound=%d\n", i, ++cFound));
3571	# endif
3572	AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3573	/PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);/
3574	break;
3575	}
3576	#endif /* PGM_WITH_LARGE_PAGES */
3577
3578	default:
3579	AssertFatalMsgFailed(("enmKind=%d iShw=%d\n", pPage->enmKind, iShw));
3580	}
3581
3582	/* not reached. */
3583	#ifndef _MSC_VER
3584	return fRet;
3585	#endif
3586	}
3587
3588
3589	/**
3590	* Scans one shadow page table for mappings of a physical page.
3591	*
3592	* @param pVM The cross context VM structure.
3593	* @param pPhysPage The guest page in question.
3594	* @param fFlushPTEs Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
3595	* @param iShw The shadow page table.
3596	*/
3597	static void pgmPoolTrackFlushGCPhysPT(PVM pVM, PPGMPAGE pPhysPage, bool fFlushPTEs, uint16_t iShw)
3598	{
3599	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
3600
3601	/* We should only come here with when there's only one reference to this physical page. */
3602	Assert(PGMPOOL_TD_GET_CREFS(PGM_PAGE_GET_TRACKING(pPhysPage)) == 1);
3603
3604	Log2(("pgmPoolTrackFlushGCPhysPT: pPhysPage=%RHp iShw=%d\n", PGM_PAGE_GET_HCPHYS(pPhysPage), iShw));
3605	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPT, f);
3606	bool fKeptPTEs = pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, fFlushPTEs, iShw, PGM_PAGE_GET_PTE_INDEX(pPhysPage));
3607	if (!fKeptPTEs)
3608	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
3609	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPT, f);
3610	}
3611
3612
3613	/**
3614	* Flushes a list of shadow page tables mapping the same physical page.
3615	*
3616	* @param pVM The cross context VM structure.
3617	* @param pPhysPage The guest page in question.
3618	* @param fFlushPTEs Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
3619	* @param iPhysExt The physical cross reference extent list to flush.
3620	*/
3621	static void pgmPoolTrackFlushGCPhysPTs(PVMCC pVM, PPGMPAGE pPhysPage, bool fFlushPTEs, uint16_t iPhysExt)
3622	{
3623	PGM_LOCK_ASSERT_OWNER(pVM);
3624	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3625	bool fKeepList = false;
3626
3627	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTs, f);
3628	Log2(("pgmPoolTrackFlushGCPhysPTs: pPhysPage=%RHp iPhysExt=%u\n", PGM_PAGE_GET_HCPHYS(pPhysPage), iPhysExt));
3629
3630	const uint16_t iPhysExtStart = iPhysExt;
3631	PPGMPOOLPHYSEXT pPhysExt;
3632	do
3633	{
3634	Assert(iPhysExt < pPool->cMaxPhysExts);
3635	pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3636	for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
3637	{
3638	if (pPhysExt->aidx[i] != NIL_PGMPOOL_IDX)
3639	{
3640	bool fKeptPTEs = pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, fFlushPTEs, pPhysExt->aidx[i], pPhysExt->apte[i]);
3641	if (!fKeptPTEs)
3642	{
3643	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
3644	pPhysExt->apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
3645	}
3646	else
3647	fKeepList = true;
3648	}
3649	}
3650	/* next */
3651	iPhysExt = pPhysExt->iNext;
3652	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
3653
3654	if (!fKeepList)
3655	{
3656	/* insert the list into the free list and clear the ram range entry. */
3657	pPhysExt->iNext = pPool->iPhysExtFreeHead;
3658	pPool->iPhysExtFreeHead = iPhysExtStart;
3659	/* Invalidate the tracking data. */
3660	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
3661	}
3662
3663	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTs, f);
3664	}
3665
3666
3667	/**
3668	* Flushes all shadow page table mappings of the given guest page.
3669	*
3670	* This is typically called when the host page backing the guest one has been
3671	* replaced or when the page protection was changed due to a guest access
3672	* caught by the monitoring.
3673	*
3674	* @returns VBox status code.
3675	* @retval VINF_SUCCESS if all references has been successfully cleared.
3676	* @retval VINF_PGM_SYNC_CR3 if we're better off with a CR3 sync and a page
3677	* pool cleaning. FF and sync flags are set.
3678	*
3679	* @param pVM The cross context VM structure.
3680	* @param GCPhysPage GC physical address of the page in question
3681	* @param pPhysPage The guest page in question.
3682	* @param fFlushPTEs Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
3683	* @param pfFlushTLBs This is set to @a true if the shadow TLBs should be
3684	* flushed, it is NOT touched if this isn't necessary.
3685	* The caller MUST initialized this to @a false.
3686	*/
3687	int pgmPoolTrackUpdateGCPhys(PVMCC pVM, RTGCPHYS GCPhysPage, PPGMPAGE pPhysPage, bool fFlushPTEs, bool *pfFlushTLBs)
3688	{
3689	PVMCPUCC pVCpu = VMMGetCpu(pVM);
3690	PGM_LOCK_VOID(pVM);
3691	int rc = VINF_SUCCESS;
3692
3693	#ifdef PGM_WITH_LARGE_PAGES
3694	/* Is this page part of a large page? */
3695	if (PGM_PAGE_GET_PDE_TYPE(pPhysPage) == PGM_PAGE_PDE_TYPE_PDE)
3696	{
3697	RTGCPHYS GCPhysBase = GCPhysPage & X86_PDE2M_PAE_PG_MASK;
3698	GCPhysPage &= X86_PDE_PAE_PG_MASK;
3699
3700	/* Fetch the large page base. */
3701	PPGMPAGE pLargePage;
3702	if (GCPhysBase != GCPhysPage)
3703	{
3704	pLargePage = pgmPhysGetPage(pVM, GCPhysBase);
3705	AssertFatal(pLargePage);
3706	}
3707	else
3708	pLargePage = pPhysPage;
3709
3710	Log(("pgmPoolTrackUpdateGCPhys: update large page PDE for %RGp (%RGp)\n", GCPhysBase, GCPhysPage));
3711
3712	if (PGM_PAGE_GET_PDE_TYPE(pLargePage) == PGM_PAGE_PDE_TYPE_PDE)
3713	{
3714	/* Mark the large page as disabled as we need to break it up to change a single page in the 2 MB range. */
3715	PGM_PAGE_SET_PDE_TYPE(pVM, pLargePage, PGM_PAGE_PDE_TYPE_PDE_DISABLED);
3716	pVM->pgm.s.cLargePagesDisabled++;
3717
3718	/* Update the base as that only that one has a reference and there's only one PDE to clear. */
3719	rc = pgmPoolTrackUpdateGCPhys(pVM, GCPhysBase, pLargePage, fFlushPTEs, pfFlushTLBs);
3720
3721	*pfFlushTLBs = true;
3722	PGM_UNLOCK(pVM);
3723	return rc;
3724	}
3725	}
3726	#else
3727	NOREF(GCPhysPage);
3728	#endif /* PGM_WITH_LARGE_PAGES */
3729
3730	const uint16_t u16 = PGM_PAGE_GET_TRACKING(pPhysPage);
3731	if (u16)
3732	{
3733	/*
3734	* The zero page is currently screwing up the tracking and we'll
3735	* have to flush the whole shebang. Unless VBOX_WITH_NEW_LAZY_PAGE_ALLOC
3736	* is defined, zero pages won't normally be mapped. Some kind of solution
3737	* will be needed for this problem of course, but it will have to wait...
3738	*/
3739	if ( PGM_PAGE_IS_ZERO(pPhysPage)
3740	\|\| PGM_PAGE_IS_BALLOONED(pPhysPage))
3741	rc = VINF_PGM_GCPHYS_ALIASED;
3742	else
3743	{
3744	if (PGMPOOL_TD_GET_CREFS(u16) != PGMPOOL_TD_CREFS_PHYSEXT)
3745	{
3746	Assert(PGMPOOL_TD_GET_CREFS(u16) == 1);
3747	pgmPoolTrackFlushGCPhysPT(pVM,
3748	pPhysPage,
3749	fFlushPTEs,
3750	PGMPOOL_TD_GET_IDX(u16));
3751	}
3752	else if (u16 != PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED))
3753	pgmPoolTrackFlushGCPhysPTs(pVM, pPhysPage, fFlushPTEs, PGMPOOL_TD_GET_IDX(u16));
3754	else
3755	rc = pgmPoolTrackFlushGCPhysPTsSlow(pVM, pPhysPage);
3756	*pfFlushTLBs = true;
3757	}
3758	}
3759
3760	if (rc == VINF_PGM_GCPHYS_ALIASED)
3761	{
3762	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
3763	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3764	rc = VINF_PGM_SYNC_CR3;
3765	}
3766	PGM_UNLOCK(pVM);
3767	return rc;
3768	}
3769
3770
3771	/**
3772	* Scans all shadow page tables for mappings of a physical page.
3773	*
3774	* This may be slow, but it's most likely more efficient than cleaning
3775	* out the entire page pool / cache.
3776	*
3777	* @returns VBox status code.
3778	* @retval VINF_SUCCESS if all references has been successfully cleared.
3779	* @retval VINF_PGM_GCPHYS_ALIASED if we're better off with a CR3 sync and
3780	* a page pool cleaning.
3781	*
3782	* @param pVM The cross context VM structure.
3783	* @param pPhysPage The guest page in question.
3784	*/
3785	int pgmPoolTrackFlushGCPhysPTsSlow(PVMCC pVM, PPGMPAGE pPhysPage)
3786	{
3787	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3788	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTsSlow, s);
3789	LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: cUsedPages=%d cPresent=%d pPhysPage=%R[pgmpage]\n",
3790	pPool->cUsedPages, pPool->cPresent, pPhysPage));
3791
3792	/*
3793	* There is a limit to what makes sense.
3794	*/
3795	if ( pPool->cPresent > 1024
3796	&& pVM->cCpus == 1)
3797	{
3798	LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: giving up... (cPresent=%d)\n", pPool->cPresent));
3799	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
3800	return VINF_PGM_GCPHYS_ALIASED;
3801	}
3802
3803	/*
3804	* Iterate all the pages until we've encountered all that in use.
3805	* This is simple but not quite optimal solution.
3806	*/
3807	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage);
3808	unsigned cLeft = pPool->cUsedPages;
3809	unsigned iPage = pPool->cCurPages;
3810	while (--iPage >= PGMPOOL_IDX_FIRST)
3811	{
3812	PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
3813	if ( pPage->GCPhys != NIL_RTGCPHYS
3814	&& pPage->cPresent)
3815	{
3816	Assert(!PGMPOOL_PAGE_IS_NESTED(pPage)); /* see if it hits */
3817	switch (pPage->enmKind)
3818	{
3819	/*
3820	* We only care about shadow page tables.
3821	*/
3822	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3823	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3824	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3825	{
3826	const uint32_t u32 = (uint32_t)u64;
3827	unsigned cPresent = pPage->cPresent;
3828	PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3829	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
3830	{
3831	const X86PGUINT uPte = pPT->a[i].u;
3832	if (uPte & X86_PTE_P)
3833	{
3834	if ((uPte & X86_PTE_PG_MASK) == u32)
3835	{
3836	//Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX32\n", iPage, i, pPT->a[i]));
3837	ASMAtomicWriteU32(&pPT->a[i].u, 0);
3838
3839	/* Update the counter as we're removing references. */
3840	Assert(pPage->cPresent);
3841	Assert(pPool->cPresent);
3842	pPage->cPresent--;
3843	pPool->cPresent--;
3844	}
3845	if (!--cPresent)
3846	break;
3847	}
3848	}
3849	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3850	break;
3851	}
3852
3853	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3854	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3855	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3856	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3857	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3858	{
3859	unsigned cPresent = pPage->cPresent;
3860	PPGMSHWPTPAE pPT = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3861	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
3862	if (PGMSHWPTEPAE_IS_P(pPT->a[i]))
3863	{
3864	if ((PGMSHWPTEPAE_GET_U(pPT->a[i]) & X86_PTE_PAE_PG_MASK) == u64)
3865	{
3866	//Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX64\n", iPage, i, pPT->a[i]));
3867	PGMSHWPTEPAE_ATOMIC_SET(pPT->a[i], 0); /// @todo why not atomic?
3868
3869	/* Update the counter as we're removing references. */
3870	Assert(pPage->cPresent);
3871	Assert(pPool->cPresent);
3872	pPage->cPresent--;
3873	pPool->cPresent--;
3874	}
3875	if (!--cPresent)
3876	break;
3877	}
3878	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3879	break;
3880	}
3881
3882	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
3883	{
3884	unsigned cPresent = pPage->cPresent;
3885	PEPTPT pPT = (PEPTPT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3886	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
3887	{
3888	X86PGPAEUINT const uPte = pPT->a[i].u;
3889	if (uPte & EPT_E_READ)
3890	{
3891	if ((uPte & EPT_PTE_PG_MASK) == u64)
3892	{
3893	//Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX64\n", iPage, i, pPT->a[i]));
3894	ASMAtomicWriteU64(&pPT->a[i].u, 0);
3895
3896	/* Update the counter as we're removing references. */
3897	Assert(pPage->cPresent);
3898	Assert(pPool->cPresent);
3899	pPage->cPresent--;
3900	pPool->cPresent--;
3901	}
3902	if (!--cPresent)
3903	break;
3904	}
3905	}
3906	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3907	break;
3908	}
3909	}
3910
3911	if (!--cLeft)
3912	break;
3913	}
3914	}
3915
3916	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
3917	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
3918
3919	/*
3920	* There is a limit to what makes sense. The above search is very expensive, so force a pgm pool flush.
3921	*/
3922	if (pPool->cPresent > 1024)
3923	{
3924	LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: giving up... (cPresent=%d)\n", pPool->cPresent));
3925	return VINF_PGM_GCPHYS_ALIASED;
3926	}
3927
3928	return VINF_SUCCESS;
3929	}
3930
3931
3932	/**
3933	* Clears the user entry in a user table.
3934	*
3935	* This is used to remove all references to a page when flushing it.
3936	*/
3937	static void pgmPoolTrackClearPageUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PCPGMPOOLUSER pUser)
3938	{
3939	Assert(pUser->iUser != NIL_PGMPOOL_IDX);
3940	Assert(pUser->iUser < pPool->cCurPages);
3941	uint32_t iUserTable = pUser->iUserTable;
3942
3943	/*
3944	* Map the user page. Ignore references made by fictitious pages.
3945	*/
3946	PPGMPOOLPAGE pUserPage = &pPool->aPages[pUser->iUser];
3947	LogFlow(("pgmPoolTrackClearPageUser: clear %x in %s (%RGp) (flushing %s)\n", iUserTable, pgmPoolPoolKindToStr(pUserPage->enmKind), pUserPage->Core.Key, pgmPoolPoolKindToStr(pPage->enmKind)));
3948	union
3949	{
3950	uint64_t *pau64;
3951	uint32_t *pau32;
3952	} u;
3953	if (pUserPage->idx < PGMPOOL_IDX_FIRST)
3954	{
3955	Assert(!pUserPage->pvPageR3);
3956	return;
3957	}
3958	u.pau64 = (uint64_t *)PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pUserPage);
3959
3960
3961	/* Safety precaution in case we change the paging for other modes too in the future. */
3962	Assert(!pgmPoolIsPageLocked(pPage)); RT_NOREF_PV(pPage);
3963
3964	#ifdef VBOX_STRICT
3965	/*
3966	* Some sanity checks.
3967	*/
3968	switch (pUserPage->enmKind)
3969	{
3970	case PGMPOOLKIND_32BIT_PD:
3971	case PGMPOOLKIND_32BIT_PD_PHYS:
3972	Assert(iUserTable < X86_PG_ENTRIES);
3973	break;
3974	case PGMPOOLKIND_PAE_PDPT:
3975	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
3976	case PGMPOOLKIND_PAE_PDPT_PHYS:
3977	Assert(iUserTable < 4);
3978	Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
3979	break;
3980	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3981	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3982	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3983	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3984	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3985	case PGMPOOLKIND_PAE_PD_PHYS:
3986	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3987	break;
3988	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3989	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3990	break;
3991	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3992	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3993	Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
3994	break;
3995	case PGMPOOLKIND_64BIT_PML4:
3996	Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
3997	/* GCPhys >> PAGE_SHIFT is the index here */
3998	break;
3999	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
4000	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
4001	Assert(iUserTable < X86_PG_PAE_ENTRIES);
4002	break;
4003
4004	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
4005	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
4006	Assert(iUserTable < X86_PG_PAE_ENTRIES);
4007	break;
4008
4009	case PGMPOOLKIND_ROOT_NESTED:
4010	Assert(iUserTable < X86_PG_PAE_ENTRIES);
4011	break;
4012
4013	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
4014	case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
4015	case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
4016	case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
4017	case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
4018	Assert(iUserTable < EPT_PG_ENTRIES);
4019	break;
4020	# endif
4021
4022	default:
4023	AssertMsgFailed(("enmKind=%d GCPhys=%RGp\n", pUserPage->enmKind, pPage->GCPhys));
4024	break;
4025	}
4026	#endif /* VBOX_STRICT */
4027
4028	/*
4029	* Clear the entry in the user page.
4030	*/
4031	switch (pUserPage->enmKind)
4032	{
4033	/* 32-bit entries */
4034	case PGMPOOLKIND_32BIT_PD:
4035	case PGMPOOLKIND_32BIT_PD_PHYS:
4036	ASMAtomicWriteU32(&u.pau32[iUserTable], 0);
4037	break;
4038
4039	/* 64-bit entries */
4040	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
4041	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
4042	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
4043	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
4044	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
4045	case PGMPOOLKIND_PAE_PD_PHYS:
4046	case PGMPOOLKIND_PAE_PDPT_PHYS:
4047	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
4048	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
4049	case PGMPOOLKIND_64BIT_PML4:
4050	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
4051	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
4052	case PGMPOOLKIND_PAE_PDPT:
4053	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
4054	case PGMPOOLKIND_ROOT_NESTED:
4055	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
4056	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
4057	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
4058	case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
4059	case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
4060	case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
4061	case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
4062	#endif
4063	ASMAtomicWriteU64(&u.pau64[iUserTable], 0);
4064	break;
4065
4066	default:
4067	AssertFatalMsgFailed(("enmKind=%d iUser=%d iUserTable=%#x\n", pUserPage->enmKind, pUser->iUser, pUser->iUserTable));
4068	}
4069	PGM_DYNMAP_UNUSED_HINT_VM(pPool->CTX_SUFF(pVM), u.pau64);
4070	}
4071
4072
4073	/**
4074	* Clears all users of a page.
4075	*/
4076	static void pgmPoolTrackClearPageUsers(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4077	{
4078	/*
4079	* Free all the user records.
4080	*/
4081	LogFlow(("pgmPoolTrackClearPageUsers %RGp\n", pPage->GCPhys));
4082
4083	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
4084	uint16_t i = pPage->iUserHead;
4085	while (i != NIL_PGMPOOL_USER_INDEX)
4086	{
4087	/* Clear enter in user table. */
4088	pgmPoolTrackClearPageUser(pPool, pPage, &paUsers[i]);
4089
4090	/* Free it. */
4091	const uint16_t iNext = paUsers[i].iNext;
4092	paUsers[i].iUser = NIL_PGMPOOL_IDX;
4093	paUsers[i].iNext = pPool->iUserFreeHead;
4094	pPool->iUserFreeHead = i;
4095
4096	/* Next. */
4097	i = iNext;
4098	}
4099	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
4100	}
4101
4102
4103	/**
4104	* Allocates a new physical cross reference extent.
4105	*
4106	* @returns Pointer to the allocated extent on success. NULL if we're out of them.
4107	* @param pVM The cross context VM structure.
4108	* @param piPhysExt Where to store the phys ext index.
4109	*/
4110	PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVMCC pVM, uint16_t *piPhysExt)
4111	{
4112	PGM_LOCK_ASSERT_OWNER(pVM);
4113	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4114	uint16_t iPhysExt = pPool->iPhysExtFreeHead;
4115	if (iPhysExt == NIL_PGMPOOL_PHYSEXT_INDEX)
4116	{
4117	STAM_COUNTER_INC(&pPool->StamTrackPhysExtAllocFailures);
4118	return NULL;
4119	}
4120	PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
4121	pPool->iPhysExtFreeHead = pPhysExt->iNext;
4122	pPhysExt->iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
4123	*piPhysExt = iPhysExt;
4124	return pPhysExt;
4125	}
4126
4127
4128	/**
4129	* Frees a physical cross reference extent.
4130	*
4131	* @param pVM The cross context VM structure.
4132	* @param iPhysExt The extent to free.
4133	*/
4134	void pgmPoolTrackPhysExtFree(PVMCC pVM, uint16_t iPhysExt)
4135	{
4136	PGM_LOCK_ASSERT_OWNER(pVM);
4137	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4138	Assert(iPhysExt < pPool->cMaxPhysExts);
4139	PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
4140	for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
4141	{
4142	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
4143	pPhysExt->apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
4144	}
4145	pPhysExt->iNext = pPool->iPhysExtFreeHead;
4146	pPool->iPhysExtFreeHead = iPhysExt;
4147	}
4148
4149
4150	/**
4151	* Frees a physical cross reference extent.
4152	*
4153	* @param pVM The cross context VM structure.
4154	* @param iPhysExt The extent to free.
4155	*/
4156	void pgmPoolTrackPhysExtFreeList(PVMCC pVM, uint16_t iPhysExt)
4157	{
4158	PGM_LOCK_ASSERT_OWNER(pVM);
4159	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4160
4161	const uint16_t iPhysExtStart = iPhysExt;
4162	PPGMPOOLPHYSEXT pPhysExt;
4163	do
4164	{
4165	Assert(iPhysExt < pPool->cMaxPhysExts);
4166	pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
4167	for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
4168	{
4169	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
4170	pPhysExt->apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
4171	}
4172
4173	/* next */
4174	iPhysExt = pPhysExt->iNext;
4175	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
4176
4177	pPhysExt->iNext = pPool->iPhysExtFreeHead;
4178	pPool->iPhysExtFreeHead = iPhysExtStart;
4179	}
4180
4181
4182	/**
4183	* Insert a reference into a list of physical cross reference extents.
4184	*
4185	* @returns The new tracking data for PGMPAGE.
4186	*
4187	* @param pVM The cross context VM structure.
4188	* @param iPhysExt The physical extent index of the list head.
4189	* @param iShwPT The shadow page table index.
4190	* @param iPte Page table entry
4191	*
4192	*/
4193	static uint16_t pgmPoolTrackPhysExtInsert(PVMCC pVM, uint16_t iPhysExt, uint16_t iShwPT, uint16_t iPte)
4194	{
4195	PGM_LOCK_ASSERT_OWNER(pVM);
4196	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4197	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
4198
4199	/*
4200	* Special common cases.
4201	*/
4202	if (paPhysExts[iPhysExt].aidx[1] == NIL_PGMPOOL_IDX)
4203	{
4204	paPhysExts[iPhysExt].aidx[1] = iShwPT;
4205	paPhysExts[iPhysExt].apte[1] = iPte;
4206	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackAliasedMany);
4207	LogFlow(("pgmPoolTrackPhysExtInsert: %d:{,%d pte %d,}\n", iPhysExt, iShwPT, iPte));
4208	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
4209	}
4210	if (paPhysExts[iPhysExt].aidx[2] == NIL_PGMPOOL_IDX)
4211	{
4212	paPhysExts[iPhysExt].aidx[2] = iShwPT;
4213	paPhysExts[iPhysExt].apte[2] = iPte;
4214	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackAliasedMany);
4215	LogFlow(("pgmPoolTrackPhysExtInsert: %d:{,,%d pte %d}\n", iPhysExt, iShwPT, iPte));
4216	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
4217	}
4218	AssertCompile(RT_ELEMENTS(paPhysExts[iPhysExt].aidx) == 3);
4219
4220	/*
4221	* General treatment.
4222	*/
4223	const uint16_t iPhysExtStart = iPhysExt;
4224	unsigned cMax = 15;
4225	for (;;)
4226	{
4227	Assert(iPhysExt < pPool->cMaxPhysExts);
4228	for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
4229	if (paPhysExts[iPhysExt].aidx[i] == NIL_PGMPOOL_IDX)
4230	{
4231	paPhysExts[iPhysExt].aidx[i] = iShwPT;
4232	paPhysExts[iPhysExt].apte[i] = iPte;
4233	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackAliasedMany);
4234	LogFlow(("pgmPoolTrackPhysExtInsert: %d:{%d pte %d} i=%d cMax=%d\n", iPhysExt, iShwPT, iPte, i, cMax));
4235	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExtStart);
4236	}
4237	if (!--cMax)
4238	{
4239	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackOverflows);
4240	pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
4241	LogFlow(("pgmPoolTrackPhysExtInsert: overflow (1) iShwPT=%d\n", iShwPT));
4242	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
4243	}
4244
4245	/* advance */
4246	iPhysExt = paPhysExts[iPhysExt].iNext;
4247	if (iPhysExt == NIL_PGMPOOL_PHYSEXT_INDEX)
4248	break;
4249	}
4250
4251	/*
4252	* Add another extent to the list.
4253	*/
4254	PPGMPOOLPHYSEXT pNew = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
4255	if (!pNew)
4256	{
4257	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackNoExtentsLeft);
4258	pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
4259	LogFlow(("pgmPoolTrackPhysExtInsert: pgmPoolTrackPhysExtAlloc failed iShwPT=%d\n", iShwPT));
4260	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
4261	}
4262	pNew->iNext = iPhysExtStart;
4263	pNew->aidx[0] = iShwPT;
4264	pNew->apte[0] = iPte;
4265	LogFlow(("pgmPoolTrackPhysExtInsert: added new extent %d:{%d pte %d}->%d\n", iPhysExt, iShwPT, iPte, iPhysExtStart));
4266	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
4267	}
4268
4269
4270	/**
4271	* Add a reference to guest physical page where extents are in use.
4272	*
4273	* @returns The new tracking data for PGMPAGE.
4274	*
4275	* @param pVM The cross context VM structure.
4276	* @param pPhysPage Pointer to the aPages entry in the ram range.
4277	* @param u16 The ram range flags (top 16-bits).
4278	* @param iShwPT The shadow page table index.
4279	* @param iPte Page table entry
4280	*/
4281	uint16_t pgmPoolTrackPhysExtAddref(PVMCC pVM, PPGMPAGE pPhysPage, uint16_t u16, uint16_t iShwPT, uint16_t iPte)
4282	{
4283	PGM_LOCK_VOID(pVM);
4284	if (PGMPOOL_TD_GET_CREFS(u16) != PGMPOOL_TD_CREFS_PHYSEXT)
4285	{
4286	/*
4287	* Convert to extent list.
4288	*/
4289	Assert(PGMPOOL_TD_GET_CREFS(u16) == 1);
4290	uint16_t iPhysExt;
4291	PPGMPOOLPHYSEXT pPhysExt = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
4292	if (pPhysExt)
4293	{
4294	LogFlow(("pgmPoolTrackPhysExtAddref: new extent: %d:{%d, %d}\n", iPhysExt, PGMPOOL_TD_GET_IDX(u16), iShwPT));
4295	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackAliased);
4296	pPhysExt->aidx[0] = PGMPOOL_TD_GET_IDX(u16);
4297	pPhysExt->apte[0] = PGM_PAGE_GET_PTE_INDEX(pPhysPage);
4298	pPhysExt->aidx[1] = iShwPT;
4299	pPhysExt->apte[1] = iPte;
4300	u16 = PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
4301	}
4302	else
4303	u16 = PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
4304	}
4305	else if (u16 != PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED))
4306	{
4307	/*
4308	* Insert into the extent list.
4309	*/
4310	u16 = pgmPoolTrackPhysExtInsert(pVM, PGMPOOL_TD_GET_IDX(u16), iShwPT, iPte);
4311	}
4312	else
4313	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackAliasedLots);
4314	PGM_UNLOCK(pVM);
4315	return u16;
4316	}
4317
4318
4319	/**
4320	* Clear references to guest physical memory.
4321	*
4322	* @param pPool The pool.
4323	* @param pPage The page.
4324	* @param pPhysPage Pointer to the aPages entry in the ram range.
4325	* @param iPte Shadow PTE index
4326	*/
4327	void pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMPAGE pPhysPage, uint16_t iPte)
4328	{
4329	PVMCC pVM = pPool->CTX_SUFF(pVM);
4330	const unsigned cRefs = PGM_PAGE_GET_TD_CREFS(pPhysPage);
4331	AssertFatalMsg(cRefs == PGMPOOL_TD_CREFS_PHYSEXT, ("cRefs=%d pPhysPage=%R[pgmpage] pPage=%p:{.idx=%d}\n", cRefs, pPhysPage, pPage, pPage->idx));
4332
4333	uint16_t iPhysExt = PGM_PAGE_GET_TD_IDX(pPhysPage);
4334	if (iPhysExt != PGMPOOL_TD_IDX_OVERFLOWED)
4335	{
4336	PGM_LOCK_VOID(pVM);
4337
4338	uint16_t iPhysExtPrev = NIL_PGMPOOL_PHYSEXT_INDEX;
4339	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
4340	do
4341	{
4342	Assert(iPhysExt < pPool->cMaxPhysExts);
4343
4344	/*
4345	* Look for the shadow page and check if it's all freed.
4346	*/
4347	for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
4348	{
4349	if ( paPhysExts[iPhysExt].aidx[i] == pPage->idx
4350	&& paPhysExts[iPhysExt].apte[i] == iPte)
4351	{
4352	paPhysExts[iPhysExt].aidx[i] = NIL_PGMPOOL_IDX;
4353	paPhysExts[iPhysExt].apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
4354
4355	for (i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
4356	if (paPhysExts[iPhysExt].aidx[i] != NIL_PGMPOOL_IDX)
4357	{
4358	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d\n", pPhysPage, pPage->idx));
4359	PGM_UNLOCK(pVM);
4360	return;
4361	}
4362
4363	/* we can free the node. */
4364	const uint16_t iPhysExtNext = paPhysExts[iPhysExt].iNext;
4365	if ( iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX
4366	&& iPhysExtNext == NIL_PGMPOOL_PHYSEXT_INDEX)
4367	{
4368	/* lonely node */
4369	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
4370	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d lonely\n", pPhysPage, pPage->idx));
4371	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
4372	}
4373	else if (iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX)
4374	{
4375	/* head */
4376	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d head\n", pPhysPage, pPage->idx));
4377	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExtNext));
4378	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
4379	}
4380	else
4381	{
4382	/* in list */
4383	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d in list\n", pPhysPage, pPage->idx));
4384	paPhysExts[iPhysExtPrev].iNext = iPhysExtNext;
4385	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
4386	}
4387	iPhysExt = iPhysExtNext;
4388	PGM_UNLOCK(pVM);
4389	return;
4390	}
4391	}
4392
4393	/* next */
4394	iPhysExtPrev = iPhysExt;
4395	iPhysExt = paPhysExts[iPhysExt].iNext;
4396	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
4397
4398	PGM_UNLOCK(pVM);
4399	AssertFatalMsgFailed(("not-found! cRefs=%d pPhysPage=%R[pgmpage] pPage=%p:{.idx=%d}\n", cRefs, pPhysPage, pPage, pPage->idx));
4400	}
4401	else /* nothing to do */
4402	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage]\n", pPhysPage));
4403	}
4404
4405	/**
4406	* Clear references to guest physical memory.
4407	*
4408	* This is the same as pgmPoolTracDerefGCPhysHint except that the guest
4409	* physical address is assumed to be correct, so the linear search can be
4410	* skipped and we can assert at an earlier point.
4411	*
4412	* @param pPool The pool.
4413	* @param pPage The page.
4414	* @param HCPhys The host physical address corresponding to the guest page.
4415	* @param GCPhys The guest physical address corresponding to HCPhys.
4416	* @param iPte Shadow PTE index
4417	*/
4418	static void pgmPoolTracDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhys, uint16_t iPte)
4419	{
4420	/*
4421	* Lookup the page and check if it checks out before derefing it.
4422	*/
4423	PVMCC pVM = pPool->CTX_SUFF(pVM);
4424	PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhys);
4425	if (pPhysPage)
4426	{
4427	Assert(PGM_PAGE_GET_HCPHYS(pPhysPage));
4428	#ifdef LOG_ENABLED
4429	RTHCPHYS HCPhysPage = PGM_PAGE_GET_HCPHYS(pPhysPage);
4430	Log2(("pgmPoolTracDerefGCPhys %RHp vs %RHp\n", HCPhysPage, HCPhys));
4431	#endif
4432	if (PGM_PAGE_GET_HCPHYS(pPhysPage) == HCPhys)
4433	{
4434	Assert(pPage->cPresent);
4435	Assert(pPool->cPresent);
4436	pPage->cPresent--;
4437	pPool->cPresent--;
4438	pgmTrackDerefGCPhys(pPool, pPage, pPhysPage, iPte);
4439	return;
4440	}
4441
4442	AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp; found page has HCPhys=%RHp iPte=%u fIsNested=%RTbool\n",
4443	HCPhys, GCPhys, PGM_PAGE_GET_HCPHYS(pPhysPage), iPte, PGMPOOL_PAGE_IS_NESTED(pPage)));
4444	}
4445	AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp\n", HCPhys, GCPhys));
4446	}
4447
4448
4449	/**
4450	* Clear references to guest physical memory.
4451	*
4452	* @param pPool The pool.
4453	* @param pPage The page.
4454	* @param HCPhys The host physical address corresponding to the guest page.
4455	* @param GCPhysHint The guest physical address which may corresponding to HCPhys.
4456	* @param iPte Shadow pte index
4457	*/
4458	void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint, uint16_t iPte)
4459	{
4460	Log4(("pgmPoolTracDerefGCPhysHint %RHp %RGp\n", HCPhys, GCPhysHint));
4461
4462	/*
4463	* Try the hint first.
4464	*/
4465	RTHCPHYS HCPhysHinted;
4466	PVMCC pVM = pPool->CTX_SUFF(pVM);
4467	PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysHint);
4468	if (pPhysPage)
4469	{
4470	HCPhysHinted = PGM_PAGE_GET_HCPHYS(pPhysPage);
4471	Assert(HCPhysHinted);
4472	if (HCPhysHinted == HCPhys)
4473	{
4474	Assert(pPage->cPresent);
4475	Assert(pPool->cPresent);
4476	pPage->cPresent--;
4477	pPool->cPresent--;
4478	pgmTrackDerefGCPhys(pPool, pPage, pPhysPage, iPte);
4479	return;
4480	}
4481	}
4482	else
4483	HCPhysHinted = UINT64_C(0xdeadbeefdeadbeef);
4484
4485	/*
4486	* Damn, the hint didn't work. We'll have to do an expensive linear search.
4487	*/
4488	STAM_COUNTER_INC(&pPool->StatTrackLinearRamSearches);
4489	PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRangesX);
4490	while (pRam)
4491	{
4492	unsigned iPage = pRam->cb >> PAGE_SHIFT;
4493	while (iPage-- > 0)
4494	{
4495	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
4496	{
4497	Log4(("pgmPoolTracDerefGCPhysHint: Linear HCPhys=%RHp GCPhysHint=%RGp GCPhysReal=%RGp\n",
4498	HCPhys, GCPhysHint, pRam->GCPhys + (iPage << PAGE_SHIFT)));
4499	Assert(pPage->cPresent);
4500	Assert(pPool->cPresent);
4501	pPage->cPresent--;
4502	pPool->cPresent--;
4503	pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage], iPte);
4504	return;
4505	}
4506	}
4507	pRam = pRam->CTX_SUFF(pNext);
4508	}
4509
4510	AssertFatalMsgFailed(("HCPhys=%RHp GCPhysHint=%RGp (Hinted page has HCPhys = %RHp)\n", HCPhys, GCPhysHint, HCPhysHinted));
4511	}
4512
4513
4514	/**
4515	* Clear references to guest physical memory in a 32-bit / 32-bit page table.
4516	*
4517	* @param pPool The pool.
4518	* @param pPage The page.
4519	* @param pShwPT The shadow page table (mapping of the page).
4520	* @param pGstPT The guest page table.
4521	*/
4522	DECLINLINE(void) pgmPoolTrackDerefPT32Bit32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT, PCX86PT pGstPT)
4523	{
4524	RTGCPHYS32 const fPgMask = pPage->fA20Enabled ? X86_PTE_PG_MASK : X86_PTE_PG_MASK & ~RT_BIT_32(20);
4525	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
4526	{
4527	const X86PGUINT uPte = pShwPT->a[i].u;
4528	Assert(!(uPte & RT_BIT_32(10)));
4529	if (uPte & X86_PTE_P)
4530	{
4531	Log4(("pgmPoolTrackDerefPT32Bit32Bit: i=%d pte=%RX32 hint=%RX32\n",
4532	i, uPte & X86_PTE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK));
4533	pgmPoolTracDerefGCPhysHint(pPool, pPage, uPte & X86_PTE_PG_MASK, pGstPT->a[i].u & fPgMask, i);
4534	if (!pPage->cPresent)
4535	break;
4536	}
4537	}
4538	}
4539
4540
4541	/**
4542	* Clear references to guest physical memory in a PAE / 32-bit page table.
4543	*
4544	* @param pPool The pool.
4545	* @param pPage The page.
4546	* @param pShwPT The shadow page table (mapping of the page).
4547	* @param pGstPT The guest page table (just a half one).
4548	*/
4549	DECLINLINE(void) pgmPoolTrackDerefPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PT pGstPT)
4550	{
4551	RTGCPHYS32 const fPgMask = pPage->fA20Enabled ? X86_PTE_PG_MASK : X86_PTE_PG_MASK & ~RT_BIT_32(20);
4552	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
4553	{
4554	Assert( (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == 0
4555	\|\| (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == UINT64_C(0x7ff0000000000000));
4556	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
4557	{
4558	Log4(("pgmPoolTrackDerefPTPae32Bit: i=%d pte=%RX64 hint=%RX32\n",
4559	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & X86_PTE_PG_MASK));
4560	pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & fPgMask, i);
4561	if (!pPage->cPresent)
4562	break;
4563	}
4564	}
4565	}
4566
4567
4568	/**
4569	* Clear references to guest physical memory in a PAE / PAE page table.
4570	*
4571	* @param pPool The pool.
4572	* @param pPage The page.
4573	* @param pShwPT The shadow page table (mapping of the page).
4574	* @param pGstPT The guest page table.
4575	*/
4576	DECLINLINE(void) pgmPoolTrackDerefPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT)
4577	{
4578	RTGCPHYS const fPgMask = pPage->fA20Enabled ? X86_PTE_PAE_PG_MASK : X86_PTE_PAE_PG_MASK & ~RT_BIT_64(20);
4579	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
4580	{
4581	Assert( (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == 0
4582	\|\| (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == UINT64_C(0x7ff0000000000000));
4583	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
4584	{
4585	Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX32 hint=%RX32\n",
4586	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK));
4587	pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & fPgMask, i);
4588	if (!pPage->cPresent)
4589	break;
4590	}
4591	}
4592	}
4593
4594
4595	/**
4596	* Clear references to guest physical memory in a 32-bit / 4MB page table.
4597	*
4598	* @param pPool The pool.
4599	* @param pPage The page.
4600	* @param pShwPT The shadow page table (mapping of the page).
4601	*/
4602	DECLINLINE(void) pgmPoolTrackDerefPT32Bit4MB(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT)
4603	{
4604	RTGCPHYS const GCPhysA20Mask = pPage->fA20Enabled ? UINT64_MAX : ~RT_BIT_64(20);
4605	RTGCPHYS GCPhys = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
4606	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
4607	{
4608	const X86PGUINT uPte = pShwPT->a[i].u;
4609	Assert(!(uPte & RT_BIT_32(10)));
4610	if (uPte & X86_PTE_P)
4611	{
4612	Log4(("pgmPoolTrackDerefPT32Bit4MB: i=%d pte=%RX32 GCPhys=%RGp\n",
4613	i, uPte & X86_PTE_PG_MASK, GCPhys));
4614	pgmPoolTracDerefGCPhys(pPool, pPage, uPte & X86_PTE_PG_MASK, GCPhys & GCPhysA20Mask, i);
4615	if (!pPage->cPresent)
4616	break;
4617	}
4618	}
4619	}
4620
4621
4622	/**
4623	* Clear references to guest physical memory in a PAE / 2/4MB page table.
4624	*
4625	* @param pPool The pool.
4626	* @param pPage The page.
4627	* @param pShwPT The shadow page table (mapping of the page).
4628	*/
4629	DECLINLINE(void) pgmPoolTrackDerefPTPaeBig(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT)
4630	{
4631	RTGCPHYS const GCPhysA20Mask = pPage->fA20Enabled ? UINT64_MAX : ~RT_BIT_64(20);
4632	RTGCPHYS GCPhys = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
4633	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
4634	{
4635	Assert( (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == 0
4636	\|\| (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == UINT64_C(0x7ff0000000000000));
4637	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
4638	{
4639	Log4(("pgmPoolTrackDerefPTPaeBig: i=%d pte=%RX64 hint=%RGp\n",
4640	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), GCPhys));
4641	pgmPoolTracDerefGCPhys(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), GCPhys & GCPhysA20Mask, i);
4642	if (!pPage->cPresent)
4643	break;
4644	}
4645	}
4646	}
4647
4648
4649	/**
4650	* Clear references to shadowed pages in an EPT page table.
4651	*
4652	* @param pPool The pool.
4653	* @param pPage The page.
4654	* @param pShwPT The shadow page directory pointer table (mapping of the
4655	* page).
4656	*/
4657	DECLINLINE(void) pgmPoolTrackDerefPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPT pShwPT)
4658	{
4659	RTGCPHYS const GCPhysA20Mask = pPage->fA20Enabled ? UINT64_MAX : ~RT_BIT_64(20);
4660	RTGCPHYS GCPhys = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
4661	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
4662	{
4663	X86PGPAEUINT const uPte = pShwPT->a[i].u;
4664	Assert((uPte & UINT64_C(0xfff0000000000f80)) == 0);
4665	if (uPte & EPT_E_READ)
4666	{
4667	Log4(("pgmPoolTrackDerefPTEPT: i=%d pte=%RX64 GCPhys=%RX64\n",
4668	i, uPte & EPT_PTE_PG_MASK, pPage->GCPhys));
4669	pgmPoolTracDerefGCPhys(pPool, pPage, uPte & EPT_PTE_PG_MASK, GCPhys & GCPhysA20Mask, i);
4670	if (!pPage->cPresent)
4671	break;
4672	}
4673	}
4674	}
4675
4676	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
4677	/**
4678	* Clears references to shadowed pages in a SLAT EPT page table.
4679	*
4680	* @param pPool The pool.
4681	* @param pPage The page.
4682	* @param pShwPT The shadow page table (mapping of the page).
4683	* @param pGstPT The guest page table.
4684	*/
4685	DECLINLINE(void) pgmPoolTrackDerefNestedPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPT pShwPT, PCEPTPT pGstPT)
4686	{
4687	Assert(PGMPOOL_PAGE_IS_NESTED(pPage));
4688	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
4689	{
4690	X86PGPAEUINT const uShwPte = pShwPT->a[i].u;
4691	Assert((uShwPte & UINT64_C(0xfff0000000000f80)) == 0); /* Access, Dirty, UserX (not supported) and ignored bits 7, 11. */
4692	if (uShwPte & EPT_PRESENT_MASK)
4693	{
4694	Log7Func(("Shw=%RX64 GstPte=%RX64\n", uShwPte, pGstPT->a[i].u));
4695	pgmPoolTracDerefGCPhys(pPool, pPage, uShwPte & EPT_PTE_PG_MASK, pGstPT->a[i].u & EPT_PTE_PG_MASK, i);
4696	if (!pPage->cPresent)
4697	break;
4698	}
4699	}
4700	}
4701
4702
4703	# if 0
4704	/**
4705	* Clears refernces to shadowed pages in a SLAT EPT PM4 table.
4706	*
4707	* @param pPool The pool.
4708	* @param pPage The page.
4709	* @param pShwPml4 The shadow PML4 table.
4710	*/
4711	DECLINLINE(void) pgmPoolTrackDerefNestedPML4EPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPML4 pShwPml4)
4712	{
4713	/** @todo later merge this with 64-bit PML and pass the assert and present masks as
4714	* parameters. */
4715	Assert(PGMPOOL_PAGE_IS_NESTED(pPage));
4716	for (unsigned i = 0; i < RT_ELEMENTS(pShwPml4->a); i++)
4717	{
4718	X86PGPAEUINT const uPml4e = pShwPml4->a[i].u;
4719	Assert((uPml4e & (EPT_PML4E_MBZ_MASK \| UINT64_C(0xfff0000000000000))) == 0);
4720	if (uPml4e & EPT_PRESENT_MASK)
4721	{
4722	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPml4e & EPT_PML4E_PG_MASK);
4723	if (pSubPage)
4724	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4725	else
4726	AssertFatalMsgFailed(("%RX64\n", uPml4e & EPT_PML4E_PG_MASK));
4727	}
4728	}
4729	}
4730	# endif
4731	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
4732
4733	/**
4734	* Clear references to shadowed pages in a 32 bits page directory.
4735	*
4736	* @param pPool The pool.
4737	* @param pPage The page.
4738	* @param pShwPD The shadow page directory (mapping of the page).
4739	*/
4740	DECLINLINE(void) pgmPoolTrackDerefPD(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PD pShwPD)
4741	{
4742	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
4743	{
4744	X86PGUINT const uPde = pShwPD->a[i].u;
4745	if (uPde & X86_PDE_P)
4746	{
4747	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & X86_PDE_PG_MASK);
4748	if (pSubPage)
4749	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4750	else
4751	AssertFatalMsgFailed(("%x\n", pShwPD->a[i].u & X86_PDE_PG_MASK));
4752	}
4753	}
4754	}
4755
4756
4757	/**
4758	* Clear references to shadowed pages in a PAE (legacy or 64 bits) page directory.
4759	*
4760	* @param pPool The pool.
4761	* @param pPage The page.
4762	* @param pShwPD The shadow page directory (mapping of the page).
4763	*/
4764	DECLINLINE(void) pgmPoolTrackDerefPDPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPAE pShwPD)
4765	{
4766	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
4767	{
4768	X86PGPAEUINT const uPde = pShwPD->a[i].u;
4769	if (uPde & X86_PDE_P)
4770	{
4771	#ifdef PGM_WITH_LARGE_PAGES
4772	if (uPde & X86_PDE_PS)
4773	{
4774	Log4(("pgmPoolTrackDerefPDPae: i=%d pde=%RX64 GCPhys=%RX64\n",
4775	i, uPde & X86_PDE2M_PAE_PG_MASK, pPage->GCPhys));
4776	pgmPoolTracDerefGCPhys(pPool, pPage, uPde & X86_PDE2M_PAE_PG_MASK,
4777	pPage->GCPhys + i * 2 * _1M /* pPage->GCPhys = base address of the memory described by the PD */,
4778	i);
4779	}
4780	else
4781	#endif
4782	{
4783	Assert((uPde & (X86_PDE_PAE_MBZ_MASK_NX \| UINT64_C(0x7ff0000000000000))) == 0);
4784	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPde & X86_PDE_PAE_PG_MASK);
4785	if (pSubPage)
4786	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4787	else
4788	AssertFatalMsgFailed(("%RX64\n", uPde & X86_PDE_PAE_PG_MASK));
4789	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4790	}
4791	}
4792	}
4793	}
4794
4795
4796	/**
4797	* Clear references to shadowed pages in a PAE page directory pointer table.
4798	*
4799	* @param pPool The pool.
4800	* @param pPage The page.
4801	* @param pShwPDPT The shadow page directory pointer table (mapping of the page).
4802	*/
4803	DECLINLINE(void) pgmPoolTrackDerefPDPTPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
4804	{
4805	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
4806	{
4807	X86PGPAEUINT const uPdpe = pShwPDPT->a[i].u;
4808	Assert((uPdpe & (X86_PDPE_PAE_MBZ_MASK \| UINT64_C(0x7ff0000000000200))) == 0);
4809	if (uPdpe & X86_PDPE_P)
4810	{
4811	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPdpe & X86_PDPE_PG_MASK);
4812	if (pSubPage)
4813	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4814	else
4815	AssertFatalMsgFailed(("%RX64\n", uPdpe & X86_PDPE_PG_MASK));
4816	}
4817	}
4818	}
4819
4820
4821	/**
4822	* Clear references to shadowed pages in a 64-bit page directory pointer table.
4823	*
4824	* @param pPool The pool.
4825	* @param pPage The page.
4826	* @param pShwPDPT The shadow page directory pointer table (mapping of the page).
4827	*/
4828	DECLINLINE(void) pgmPoolTrackDerefPDPT64Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
4829	{
4830	for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++)
4831	{
4832	X86PGPAEUINT const uPdpe = pShwPDPT->a[i].u;
4833	Assert((uPdpe & (X86_PDPE_LM_MBZ_MASK_NX \| UINT64_C(0x7ff0000000000200))) == 0);
4834	if (uPdpe & X86_PDPE_P)
4835	{
4836	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPdpe & X86_PDPE_PG_MASK);
4837	if (pSubPage)
4838	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4839	else
4840	AssertFatalMsgFailed(("%RX64\n", uPdpe & X86_PDPE_PG_MASK));
4841	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4842	}
4843	}
4844	}
4845
4846
4847	/**
4848	* Clear references to shadowed pages in a 64-bit level 4 page table.
4849	*
4850	* @param pPool The pool.
4851	* @param pPage The page.
4852	* @param pShwPML4 The shadow page directory pointer table (mapping of the page).
4853	*/
4854	DECLINLINE(void) pgmPoolTrackDerefPML464Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PML4 pShwPML4)
4855	{
4856	for (unsigned i = 0; i < RT_ELEMENTS(pShwPML4->a); i++)
4857	{
4858	X86PGPAEUINT const uPml4e = pShwPML4->a[i].u;
4859	Assert((uPml4e & (X86_PML4E_MBZ_MASK_NX \| UINT64_C(0x7ff0000000000200))) == 0);
4860	if (uPml4e & X86_PML4E_P)
4861	{
4862	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPml4e & X86_PDPE_PG_MASK);
4863	if (pSubPage)
4864	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4865	else
4866	AssertFatalMsgFailed(("%RX64\n", uPml4e & X86_PML4E_PG_MASK));
4867	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4868	}
4869	}
4870	}
4871
4872
4873	/**
4874	* Clear references to shadowed pages in an EPT page directory.
4875	*
4876	* @param pPool The pool.
4877	* @param pPage The page.
4878	* @param pShwPD The shadow page directory (mapping of the page).
4879	*/
4880	DECLINLINE(void) pgmPoolTrackDerefPDEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPD pShwPD)
4881	{
4882	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
4883	{
4884	X86PGPAEUINT const uPde = pShwPD->a[i].u;
4885	#ifdef PGM_WITH_LARGE_PAGES
4886	AssertMsg((uPde & UINT64_C(0xfff0000000000f00)) == 0, ("uPde=%RX64\n", uPde));
4887	#else
4888	AssertMsg((uPde & UINT64_C(0xfff0000000000f80)) == 0, ("uPde=%RX64\n", uPde));
4889	#endif
4890	if (uPde & EPT_E_READ)
4891	{
4892	#ifdef PGM_WITH_LARGE_PAGES
4893	if (uPde & EPT_E_LEAF)
4894	{
4895	Log4(("pgmPoolTrackDerefPDEPT: i=%d pde=%RX64 GCPhys=%RX64\n",
4896	i, uPde & EPT_PDE2M_PG_MASK, pPage->GCPhys));
4897	Assert(!PGMPOOL_PAGE_IS_NESTED(pPage)); /* We don't support large guest EPT yet. */
4898	pgmPoolTracDerefGCPhys(pPool, pPage, uPde & EPT_PDE2M_PG_MASK,
4899	pPage->GCPhys + i * 2 * _1M /* pPage->GCPhys = base address of the memory described by the PD */,
4900	i);
4901	}
4902	else
4903	#endif
4904	{
4905	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPde & EPT_PDE_PG_MASK);
4906	if (pSubPage)
4907	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4908	else
4909	AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & EPT_PDE_PG_MASK));
4910	}
4911	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4912	}
4913	}
4914	}
4915
4916
4917	/**
4918	* Clear references to shadowed pages in an EPT page directory pointer table.
4919	*
4920	* @param pPool The pool.
4921	* @param pPage The page.
4922	* @param pShwPDPT The shadow page directory pointer table (mapping of the page).
4923	*/
4924	DECLINLINE(void) pgmPoolTrackDerefPDPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPDPT pShwPDPT)
4925	{
4926	for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++)
4927	{
4928	X86PGPAEUINT const uPdpe = pShwPDPT->a[i].u;
4929	Assert((uPdpe & UINT64_C(0xfff0000000000f80)) == 0);
4930	if (uPdpe & EPT_E_READ)
4931	{
4932	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPdpe & EPT_PDPTE_PG_MASK);
4933	if (pSubPage)
4934	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4935	else
4936	AssertFatalMsgFailed(("%RX64\n", uPdpe & EPT_PDPTE_PG_MASK));
4937	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4938	}
4939	}
4940	}
4941
4942
4943	/**
4944	* Clears all references made by this page.
4945	*
4946	* This includes other shadow pages and GC physical addresses.
4947	*
4948	* @param pPool The pool.
4949	* @param pPage The page.
4950	*/
4951	static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4952	{
4953	/*
4954	* Map the shadow page and take action according to the page kind.
4955	*/
4956	PVMCC pVM = pPool->CTX_SUFF(pVM);
4957	void *pvShw = PGMPOOL_PAGE_2_PTR(pVM, pPage);
4958	switch (pPage->enmKind)
4959	{
4960	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
4961	{
4962	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4963	void *pvGst;
4964	int rc = PGM_GCPHYS_2_PTR(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
4965	pgmPoolTrackDerefPT32Bit32Bit(pPool, pPage, (PX86PT)pvShw, (PCX86PT)pvGst);
4966	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
4967	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4968	break;
4969	}
4970
4971	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
4972	{
4973	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4974	void *pvGst;
4975	int rc = PGM_GCPHYS_2_PTR_EX(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
4976	pgmPoolTrackDerefPTPae32Bit(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PT)pvGst);
4977	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
4978	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4979	break;
4980	}
4981
4982	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
4983	{
4984	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4985	void *pvGst;
4986	int rc = PGM_GCPHYS_2_PTR(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
4987	pgmPoolTrackDerefPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst);
4988	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
4989	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4990	break;
4991	}
4992
4993	case PGMPOOLKIND_32BIT_PT_FOR_PHYS: /* treat it like a 4 MB page */
4994	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
4995	{
4996	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4997	pgmPoolTrackDerefPT32Bit4MB(pPool, pPage, (PX86PT)pvShw);
4998	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4999	break;
5000	}
5001
5002	case PGMPOOLKIND_PAE_PT_FOR_PHYS: /* treat it like a 2 MB page */
5003	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
5004	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
5005	{
5006	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
5007	pgmPoolTrackDerefPTPaeBig(pPool, pPage, (PPGMSHWPTPAE)pvShw);
5008	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
5009	break;
5010	}
5011
5012	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
5013	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
5014	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
5015	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
5016	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
5017	case PGMPOOLKIND_PAE_PD_PHYS:
5018	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
5019	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
5020	pgmPoolTrackDerefPDPae(pPool, pPage, (PX86PDPAE)pvShw);
5021	break;
5022
5023	case PGMPOOLKIND_32BIT_PD_PHYS:
5024	case PGMPOOLKIND_32BIT_PD:
5025	pgmPoolTrackDerefPD(pPool, pPage, (PX86PD)pvShw);
5026	break;
5027
5028	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
5029	case PGMPOOLKIND_PAE_PDPT:
5030	case PGMPOOLKIND_PAE_PDPT_PHYS:
5031	pgmPoolTrackDerefPDPTPae(pPool, pPage, (PX86PDPT)pvShw);
5032	break;
5033
5034	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
5035	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
5036	pgmPoolTrackDerefPDPT64Bit(pPool, pPage, (PX86PDPT)pvShw);
5037	break;
5038
5039	case PGMPOOLKIND_64BIT_PML4:
5040	pgmPoolTrackDerefPML464Bit(pPool, pPage, (PX86PML4)pvShw);
5041	break;
5042
5043	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
5044	pgmPoolTrackDerefPTEPT(pPool, pPage, (PEPTPT)pvShw);
5045	break;
5046
5047	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
5048	pgmPoolTrackDerefPDEPT(pPool, pPage, (PEPTPD)pvShw);
5049	break;
5050
5051	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
5052	pgmPoolTrackDerefPDPTEPT(pPool, pPage, (PEPTPDPT)pvShw);
5053	break;
5054
5055	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5056	case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
5057	{
5058	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
5059	void *pvGst;
5060	int const rc = PGM_GCPHYS_2_PTR(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
5061	pgmPoolTrackDerefNestedPTEPT(pPool, pPage, (PEPTPT)pvShw, (PCEPTPT)pvGst);
5062	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
5063	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
5064	break;
5065	}
5066
5067	case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
5068	pgmPoolTrackDerefPDEPT(pPool, pPage, (PEPTPD)pvShw);
5069	break;
5070
5071	case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
5072	pgmPoolTrackDerefPDPTEPT(pPool, pPage, (PEPTPDPT)pvShw);
5073	break;
5074
5075	case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
5076	//pgmPoolTrackDerefNestedPML4EPT(pPool, pPage, (PEPTPML4)pvShw);
5077	RT_FALL_THRU();
5078	#endif
5079
5080	default:
5081	AssertFatalMsgFailed(("enmKind=%d GCPhys=%RGp\n", pPage->enmKind, pPage->GCPhys));
5082	}
5083
5084	/* paranoia, clear the shadow page. Remove this laser (i.e. let Alloc and ClearAll do it). */
5085	STAM_PROFILE_START(&pPool->StatZeroPage, z);
5086	ASMMemZeroPage(pvShw);
5087	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
5088	pPage->fZeroed = true;
5089	Assert(!pPage->cPresent);
5090	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
5091	}
5092
5093
5094	/**
5095	* Flushes a pool page.
5096	*
5097	* This moves the page to the free list after removing all user references to it.
5098	*
5099	* @returns VBox status code.
5100	* @retval VINF_SUCCESS on success.
5101	* @param pPool The pool.
5102	* @param pPage The shadow page.
5103	* @param fFlush Flush the TLBS when required (should only be false in very specific use cases!!)
5104	*/
5105	int pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fFlush)
5106	{
5107	PVMCC pVM = pPool->CTX_SUFF(pVM);
5108	bool fFlushRequired = false;
5109
5110	int rc = VINF_SUCCESS;
5111	STAM_PROFILE_START(&pPool->StatFlushPage, f);
5112	LogFlow(("pgmPoolFlushPage: pPage=%p:{.Key=%RHp, .idx=%d, .enmKind=%s, .GCPhys=%RGp}\n",
5113	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
5114
5115	if (PGMPOOL_PAGE_IS_NESTED(pPage))
5116	Log7Func(("pPage=%p:{.Key=%RHp, .idx=%d, .enmKind=%s, .GCPhys=%RGp}\n",
5117	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
5118
5119	/*
5120	* Reject any attempts at flushing any of the special root pages (shall
5121	* not happen).
5122	*/
5123	AssertMsgReturn(pPage->idx >= PGMPOOL_IDX_FIRST,
5124	("pgmPoolFlushPage: special root page, rejected. enmKind=%s idx=%d\n",
5125	pgmPoolPoolKindToStr(pPage->enmKind), pPage->idx),
5126	VINF_SUCCESS);
5127
5128	PGM_LOCK_VOID(pVM);
5129
5130	/*
5131	* Quietly reject any attempts at flushing the currently active shadow CR3 mapping
5132	*/
5133	if (pgmPoolIsPageLocked(pPage))
5134	{
5135	AssertMsg( pPage->enmKind == PGMPOOLKIND_64BIT_PML4
5136	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PDPT
5137	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PDPT_FOR_32BIT
5138	\|\| pPage->enmKind == PGMPOOLKIND_32BIT_PD
5139	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD_FOR_PAE_PD
5140	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD
5141	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD
5142	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD
5143	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD
5144	\|\| pPage->enmKind == PGMPOOLKIND_ROOT_NESTED,
5145	("Can't free the shadow CR3! (%RHp vs %RHp kind=%d\n", PGMGetHyperCR3(VMMGetCpu(pVM)), pPage->Core.Key, pPage->enmKind));
5146	Log(("pgmPoolFlushPage: current active shadow CR3, rejected. enmKind=%s idx=%d\n", pgmPoolPoolKindToStr(pPage->enmKind), pPage->idx));
5147	PGM_UNLOCK(pVM);
5148	return VINF_SUCCESS;
5149	}
5150
5151	/*
5152	* Mark the page as being in need of an ASMMemZeroPage().
5153	*/
5154	pPage->fZeroed = false;
5155
5156	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
5157	if (pPage->fDirty)
5158	pgmPoolFlushDirtyPage(pVM, pPool, pPage->idxDirtyEntry, false /* do not remove */);
5159	#endif
5160
5161	/* If there are any users of this table, then we must issue a tlb flush on all VCPUs. */
5162	if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
5163	fFlushRequired = true;
5164
5165	/*
5166	* Clear the page.
5167	*/
5168	pgmPoolTrackClearPageUsers(pPool, pPage);
5169	STAM_PROFILE_START(&pPool->StatTrackDeref,a);
5170	pgmPoolTrackDeref(pPool, pPage);
5171	STAM_PROFILE_STOP(&pPool->StatTrackDeref,a);
5172
5173	/*
5174	* Flush it from the cache.
5175	*/
5176	pgmPoolCacheFlushPage(pPool, pPage);
5177
5178	/*
5179	* Deregistering the monitoring.
5180	*/
5181	if (pPage->fMonitored)
5182	rc = pgmPoolMonitorFlush(pPool, pPage);
5183
5184	/*
5185	* Free the page.
5186	*/
5187	Assert(pPage->iNext == NIL_PGMPOOL_IDX);
5188	pPage->iNext = pPool->iFreeHead;
5189	pPool->iFreeHead = pPage->idx;
5190	pPage->enmKind = PGMPOOLKIND_FREE;
5191	pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
5192	pPage->GCPhys = NIL_RTGCPHYS;
5193	pPage->fReusedFlushPending = false;
5194
5195	pPool->cUsedPages--;
5196
5197	/* Flush the TLBs of all VCPUs if required. */
5198	if ( fFlushRequired
5199	&& fFlush)
5200	{
5201	PGM_INVL_ALL_VCPU_TLBS(pVM);
5202	}
5203
5204	PGM_UNLOCK(pVM);
5205	STAM_PROFILE_STOP(&pPool->StatFlushPage, f);
5206	return rc;
5207	}
5208
5209
5210	/**
5211	* Frees a usage of a pool page.
5212	*
5213	* The caller is responsible to updating the user table so that it no longer
5214	* references the shadow page.
5215	*
5216	* @param pPool The pool.
5217	* @param pPage The shadow page.
5218	* @param iUser The shadow page pool index of the user table.
5219	* NIL_PGMPOOL_IDX for root pages.
5220	* @param iUserTable The index into the user table (shadowed). Ignored if
5221	* root page.
5222	*/
5223	void pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
5224	{
5225	PVMCC pVM = pPool->CTX_SUFF(pVM);
5226
5227	STAM_PROFILE_START(&pPool->StatFree, a);
5228	LogFlow(("pgmPoolFreeByPage: pPage=%p:{.Key=%RHp, .idx=%d, enmKind=%s} iUser=%d iUserTable=%#x\n",
5229	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), iUser, iUserTable));
5230	AssertReturnVoid(pPage->idx >= PGMPOOL_IDX_FIRST); /* paranoia (#6349) */
5231
5232	PGM_LOCK_VOID(pVM);
5233	if (iUser != NIL_PGMPOOL_IDX)
5234	pgmPoolTrackFreeUser(pPool, pPage, iUser, iUserTable);
5235	if (!pPage->fCached)
5236	pgmPoolFlushPage(pPool, pPage);
5237	PGM_UNLOCK(pVM);
5238	STAM_PROFILE_STOP(&pPool->StatFree, a);
5239	}
5240
5241
5242	/**
5243	* Makes one or more free page free.
5244	*
5245	* @returns VBox status code.
5246	* @retval VINF_SUCCESS on success.
5247	*
5248	* @param pPool The pool.
5249	* @param enmKind Page table kind
5250	* @param iUser The user of the page.
5251	*/
5252	static int pgmPoolMakeMoreFreePages(PPGMPOOL pPool, PGMPOOLKIND enmKind, uint16_t iUser)
5253	{
5254	PVMCC pVM = pPool->CTX_SUFF(pVM);
5255	LogFlow(("pgmPoolMakeMoreFreePages: enmKind=%d iUser=%d\n", enmKind, iUser));
5256	NOREF(enmKind);
5257
5258	/*
5259	* If the pool isn't full grown yet, expand it.
5260	*/
5261	if (pPool->cCurPages < pPool->cMaxPages)
5262	{
5263	STAM_PROFILE_ADV_SUSPEND(&pPool->StatAlloc, a);
5264	#ifdef IN_RING3
5265	int rc = PGMR3PoolGrow(pVM, VMMGetCpu(pVM));
5266	#else
5267	int rc = PGMR0PoolGrow(pVM, VMMGetCpuId(pVM));
5268	#endif
5269	if (RT_FAILURE(rc))
5270	return rc;
5271	STAM_PROFILE_ADV_RESUME(&pPool->StatAlloc, a);
5272	if (pPool->iFreeHead != NIL_PGMPOOL_IDX)
5273	return VINF_SUCCESS;
5274	}
5275
5276	/*
5277	* Free one cached page.
5278	*/
5279	return pgmPoolCacheFreeOne(pPool, iUser);
5280	}
5281
5282
5283	/**
5284	* Allocates a page from the pool.
5285	*
5286	* This page may actually be a cached page and not in need of any processing
5287	* on the callers part.
5288	*
5289	* @returns VBox status code.
5290	* @retval VINF_SUCCESS if a NEW page was allocated.
5291	* @retval VINF_PGM_CACHED_PAGE if a CACHED page was returned.
5292	*
5293	* @param pVM The cross context VM structure.
5294	* @param GCPhys The GC physical address of the page we're gonna shadow.
5295	* For 4MB and 2MB PD entries, it's the first address the
5296	* shadow PT is covering.
5297	* @param enmKind The kind of mapping.
5298	* @param enmAccess Access type for the mapping (only relevant for big pages)
5299	* @param fA20Enabled Whether the A20 gate is enabled or not.
5300	* @param iUser The shadow page pool index of the user table. Root
5301	* pages should pass NIL_PGMPOOL_IDX.
5302	* @param iUserTable The index into the user table (shadowed). Ignored for
5303	* root pages (iUser == NIL_PGMPOOL_IDX).
5304	* @param fLockPage Lock the page
5305	* @param ppPage Where to store the pointer to the page. NULL is stored here on failure.
5306	*/
5307	int pgmPoolAlloc(PVMCC pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, bool fA20Enabled,
5308	uint16_t iUser, uint32_t iUserTable, bool fLockPage, PPPGMPOOLPAGE ppPage)
5309	{
5310	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5311	STAM_PROFILE_ADV_START(&pPool->StatAlloc, a);
5312	LogFlow(("pgmPoolAlloc: GCPhys=%RGp enmKind=%s iUser=%d iUserTable=%#x\n", GCPhys, pgmPoolPoolKindToStr(enmKind), iUser, iUserTable));
5313	*ppPage = NULL;
5314	/** @todo CSAM/PGMPrefetchPage messes up here during CSAMR3CheckGates
5315	* (TRPMR3SyncIDT) because of FF priority. Try fix that?
5316	* Assert(!(pVM->pgm.s.fGlobalSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)); */
5317
5318	#if defined(VBOX_STRICT) && defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT)
5319	PVMCPUCC pVCpu = VMMGetCpu(pVM);
5320	Assert(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_DIRECT \|\| PGMPOOL_PAGE_IS_KIND_NESTED(enmKind));
5321	#endif
5322
5323	PGM_LOCK_VOID(pVM);
5324
5325	if (pPool->fCacheEnabled)
5326	{
5327	int rc2 = pgmPoolCacheAlloc(pPool, GCPhys, enmKind, enmAccess, fA20Enabled, iUser, iUserTable, ppPage);
5328	if (RT_SUCCESS(rc2))
5329	{
5330	if (fLockPage)
5331	pgmPoolLockPage(pPool, *ppPage);
5332	PGM_UNLOCK(pVM);
5333	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
5334	LogFlow(("pgmPoolAlloc: cached returns %Rrc ppPage=%p:{.Key=%RHp, .idx=%d}\n", rc2, ppPage, (ppPage)->Core.Key, (ppPage)->idx));
5335	return rc2;
5336	}
5337	}
5338
5339	/*
5340	* Allocate a new one.
5341	*/
5342	int rc = VINF_SUCCESS;
5343	uint16_t iNew = pPool->iFreeHead;
5344	if (iNew == NIL_PGMPOOL_IDX)
5345	{
5346	rc = pgmPoolMakeMoreFreePages(pPool, enmKind, iUser);
5347	if (RT_FAILURE(rc))
5348	{
5349	PGM_UNLOCK(pVM);
5350	Log(("pgmPoolAlloc: returns %Rrc (Free)\n", rc));
5351	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
5352	return rc;
5353	}
5354	iNew = pPool->iFreeHead;
5355	AssertReleaseMsgReturn(iNew != NIL_PGMPOOL_IDX, ("iNew=%#x\n", iNew), VERR_PGM_POOL_IPE);
5356	}
5357
5358	/* unlink the free head */
5359	PPGMPOOLPAGE pPage = &pPool->aPages[iNew];
5360	pPool->iFreeHead = pPage->iNext;
5361	pPage->iNext = NIL_PGMPOOL_IDX;
5362
5363	/*
5364	* Initialize it.
5365	*/
5366	pPool->cUsedPages++; /* physical handler registration / pgmPoolTrackFlushGCPhysPTsSlow requirement. */
5367	pPage->enmKind = enmKind;
5368	pPage->enmAccess = enmAccess;
5369	pPage->GCPhys = GCPhys;
5370	pPage->fA20Enabled = fA20Enabled;
5371	pPage->fSeenNonGlobal = false; /* Set this to 'true' to disable this feature. */
5372	pPage->fMonitored = false;
5373	pPage->fCached = false;
5374	pPage->fDirty = false;
5375	pPage->fReusedFlushPending = false;
5376	pPage->cModifications = 0;
5377	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
5378	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
5379	pPage->cPresent = 0;
5380	pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
5381	pPage->idxDirtyEntry = 0;
5382	pPage->GCPtrLastAccessHandlerFault = NIL_RTGCPTR;
5383	pPage->GCPtrLastAccessHandlerRip = NIL_RTGCPTR;
5384	pPage->cLastAccessHandler = 0;
5385	pPage->cLocked = 0;
5386	# ifdef VBOX_STRICT
5387	pPage->GCPtrDirtyFault = NIL_RTGCPTR;
5388	# endif
5389
5390	/*
5391	* Insert into the tracking and cache. If this fails, free the page.
5392	*/
5393	int rc3 = pgmPoolTrackInsert(pPool, pPage, GCPhys, iUser, iUserTable);
5394	if (RT_FAILURE(rc3))
5395	{
5396	pPool->cUsedPages--;
5397	pPage->enmKind = PGMPOOLKIND_FREE;
5398	pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
5399	pPage->GCPhys = NIL_RTGCPHYS;
5400	pPage->iNext = pPool->iFreeHead;
5401	pPool->iFreeHead = pPage->idx;
5402	PGM_UNLOCK(pVM);
5403	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
5404	Log(("pgmPoolAlloc: returns %Rrc (Insert)\n", rc3));
5405	return rc3;
5406	}
5407
5408	/*
5409	* Commit the allocation, clear the page and return.
5410	*/
5411	#ifdef VBOX_WITH_STATISTICS
5412	if (pPool->cUsedPages > pPool->cUsedPagesHigh)
5413	pPool->cUsedPagesHigh = pPool->cUsedPages;
5414	#endif
5415
5416	if (!pPage->fZeroed)
5417	{
5418	STAM_PROFILE_START(&pPool->StatZeroPage, z);
5419	void *pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
5420	ASMMemZeroPage(pv);
5421	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
5422	}
5423
5424	*ppPage = pPage;
5425	if (fLockPage)
5426	pgmPoolLockPage(pPool, pPage);
5427	PGM_UNLOCK(pVM);
5428	LogFlow(("pgmPoolAlloc: returns %Rrc *ppPage=%p:{.Key=%RHp, .idx=%d, .fCached=%RTbool, .fMonitored=%RTbool}\n",
5429	rc, pPage, pPage->Core.Key, pPage->idx, pPage->fCached, pPage->fMonitored));
5430	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
5431	return rc;
5432	}
5433
5434
5435	/**
5436	* Frees a usage of a pool page.
5437	*
5438	* @param pVM The cross context VM structure.
5439	* @param HCPhys The HC physical address of the shadow page.
5440	* @param iUser The shadow page pool index of the user table.
5441	* NIL_PGMPOOL_IDX if root page.
5442	* @param iUserTable The index into the user table (shadowed). Ignored if
5443	* root page.
5444	*/
5445	void pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint32_t iUserTable)
5446	{
5447	LogFlow(("pgmPoolFree: HCPhys=%RHp iUser=%d iUserTable=%#x\n", HCPhys, iUser, iUserTable));
5448	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5449	pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, HCPhys), iUser, iUserTable);
5450	}
5451
5452
5453	/**
5454	* Internal worker for finding a 'in-use' shadow page give by it's physical address.
5455	*
5456	* @returns Pointer to the shadow page structure.
5457	* @param pPool The pool.
5458	* @param HCPhys The HC physical address of the shadow page.
5459	*/
5460	PPGMPOOLPAGE pgmPoolGetPage(PPGMPOOL pPool, RTHCPHYS HCPhys)
5461	{
5462	PGM_LOCK_ASSERT_OWNER(pPool->CTX_SUFF(pVM));
5463
5464	/*
5465	* Look up the page.
5466	*/
5467	PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
5468
5469	AssertFatalMsg(pPage && pPage->enmKind != PGMPOOLKIND_FREE, ("HCPhys=%RHp pPage=%p idx=%d\n", HCPhys, pPage, (pPage) ? pPage->idx : 0));
5470	return pPage;
5471	}
5472
5473
5474	/**
5475	* Internal worker for finding a page for debugging purposes, no assertions.
5476	*
5477	* @returns Pointer to the shadow page structure. NULL on if not found.
5478	* @param pPool The pool.
5479	* @param HCPhys The HC physical address of the shadow page.
5480	*/
5481	PPGMPOOLPAGE pgmPoolQueryPageForDbg(PPGMPOOL pPool, RTHCPHYS HCPhys)
5482	{
5483	PGM_LOCK_ASSERT_OWNER(pPool->CTX_SUFF(pVM));
5484	return (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
5485	}
5486
5487
5488	/**
5489	* Internal worker for PGM_HCPHYS_2_PTR.
5490	*
5491	* @returns VBox status code.
5492	* @param pVM The cross context VM structure.
5493	* @param HCPhys The HC physical address of the shadow page.
5494	* @param ppv Where to return the address.
5495	*/
5496	int pgmPoolHCPhys2Ptr(PVM pVM, RTHCPHYS HCPhys, void **ppv)
5497	{
5498	PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pVM->pgm.s.CTX_SUFF(pPool)->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
5499	AssertMsgReturn(pPage && pPage->enmKind != PGMPOOLKIND_FREE,
5500	("HCPhys=%RHp pPage=%p idx=%d\n", HCPhys, pPage, (pPage) ? pPage->idx : 0),
5501	VERR_PGM_POOL_GET_PAGE_FAILED);
5502	ppv = (uint8_t )pPage->CTX_SUFF(pvPage) + (HCPhys & PAGE_OFFSET_MASK);
5503	return VINF_SUCCESS;
5504	}
5505
5506	#ifdef IN_RING3 /* currently only used in ring 3; save some space in the R0 & GC modules (left it here as we might need it elsewhere later on) */
5507
5508	/**
5509	* Flush the specified page if present
5510	*
5511	* @param pVM The cross context VM structure.
5512	* @param GCPhys Guest physical address of the page to flush
5513	*/
5514	void pgmPoolFlushPageByGCPhys(PVM pVM, RTGCPHYS GCPhys)
5515	{
5516	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5517
5518	VM_ASSERT_EMT(pVM);
5519
5520	/*
5521	* Look up the GCPhys in the hash.
5522	*/
5523	GCPhys = GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
5524	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
5525	if (i == NIL_PGMPOOL_IDX)
5526	return;
5527
5528	do
5529	{
5530	PPGMPOOLPAGE pPage = &pPool->aPages[i];
5531	if (pPage->GCPhys - GCPhys < PAGE_SIZE)
5532	{
5533	Assert(!PGMPOOL_PAGE_IS_NESTED(pPage)); /* Temporary to see if it hits. Remove later. */
5534	switch (pPage->enmKind)
5535	{
5536	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
5537	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
5538	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
5539	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
5540	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
5541	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
5542	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
5543	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
5544	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
5545	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
5546	case PGMPOOLKIND_64BIT_PML4:
5547	case PGMPOOLKIND_32BIT_PD:
5548	case PGMPOOLKIND_PAE_PDPT:
5549	{
5550	Log(("PGMPoolFlushPage: found pgm pool pages for %RGp\n", GCPhys));
5551	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
5552	if (pPage->fDirty)
5553	STAM_COUNTER_INC(&pPool->StatForceFlushDirtyPage);
5554	else
5555	# endif
5556	STAM_COUNTER_INC(&pPool->StatForceFlushPage);
5557	Assert(!pgmPoolIsPageLocked(pPage));
5558	pgmPoolMonitorChainFlush(pPool, pPage);
5559	return;
5560	}
5561
5562	/* ignore, no monitoring. */
5563	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
5564	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
5565	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
5566	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
5567	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
5568	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
5569	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
5570	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
5571	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
5572	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
5573	case PGMPOOLKIND_ROOT_NESTED:
5574	case PGMPOOLKIND_PAE_PD_PHYS:
5575	case PGMPOOLKIND_PAE_PDPT_PHYS:
5576	case PGMPOOLKIND_32BIT_PD_PHYS:
5577	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
5578	break;
5579
5580	default:
5581	AssertFatalMsgFailed(("enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
5582	}
5583	}
5584
5585	/* next */
5586	i = pPage->iNext;
5587	} while (i != NIL_PGMPOOL_IDX);
5588	return;
5589	}
5590
5591
5592	/**
5593	* Reset CPU on hot plugging.
5594	*
5595	* @param pVM The cross context VM structure.
5596	* @param pVCpu The cross context virtual CPU structure.
5597	*/
5598	void pgmR3PoolResetUnpluggedCpu(PVM pVM, PVMCPU pVCpu)
5599	{
5600	pgmR3ExitShadowModeBeforePoolFlush(pVCpu);
5601
5602	pgmR3ReEnterShadowModeAfterPoolFlush(pVM, pVCpu);
5603	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
5604	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5605	}
5606
5607
5608	/**
5609	* Flushes the entire cache.
5610	*
5611	* It will assert a global CR3 flush (FF) and assumes the caller is aware of
5612	* this and execute this CR3 flush.
5613	*
5614	* @param pVM The cross context VM structure.
5615	*/
5616	void pgmR3PoolReset(PVM pVM)
5617	{
5618	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5619
5620	PGM_LOCK_ASSERT_OWNER(pVM);
5621	STAM_PROFILE_START(&pPool->StatR3Reset, a);
5622	LogFlow(("pgmR3PoolReset:\n"));
5623
5624	/*
5625	* If there are no pages in the pool, there is nothing to do.
5626	*/
5627	if (pPool->cCurPages <= PGMPOOL_IDX_FIRST)
5628	{
5629	STAM_PROFILE_STOP(&pPool->StatR3Reset, a);
5630	return;
5631	}
5632
5633	/*
5634	* Exit the shadow mode since we're going to clear everything,
5635	* including the root page.
5636	*/
5637	VMCC_FOR_EACH_VMCPU(pVM)
5638	pgmR3ExitShadowModeBeforePoolFlush(pVCpu);
5639	VMCC_FOR_EACH_VMCPU_END(pVM);
5640
5641
5642	/*
5643	* Nuke the free list and reinsert all pages into it.
5644	*/
5645	for (unsigned i = pPool->cCurPages - 1; i >= PGMPOOL_IDX_FIRST; i--)
5646	{
5647	PPGMPOOLPAGE pPage = &pPool->aPages[i];
5648
5649	if (pPage->fMonitored)
5650	pgmPoolMonitorFlush(pPool, pPage);
5651	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
5652	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
5653	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
5654	pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
5655	pPage->GCPhys = NIL_RTGCPHYS;
5656	pPage->enmKind = PGMPOOLKIND_FREE;
5657	pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
5658	Assert(pPage->idx == i);
5659	pPage->iNext = i + 1;
5660	pPage->fA20Enabled = true;
5661	pPage->fZeroed = false; /* This could probably be optimized, but better safe than sorry. */
5662	pPage->fSeenNonGlobal = false;
5663	pPage->fMonitored = false;
5664	pPage->fDirty = false;
5665	pPage->fCached = false;
5666	pPage->fReusedFlushPending = false;
5667	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
5668	pPage->cPresent = 0;
5669	pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
5670	pPage->cModifications = 0;
5671	pPage->iAgeNext = NIL_PGMPOOL_IDX;
5672	pPage->iAgePrev = NIL_PGMPOOL_IDX;
5673	pPage->idxDirtyEntry = 0;
5674	pPage->GCPtrLastAccessHandlerRip = NIL_RTGCPTR;
5675	pPage->GCPtrLastAccessHandlerFault = NIL_RTGCPTR;
5676	pPage->cLastAccessHandler = 0;
5677	pPage->cLocked = 0;
5678	# ifdef VBOX_STRICT
5679	pPage->GCPtrDirtyFault = NIL_RTGCPTR;
5680	# endif
5681	}
5682	pPool->aPages[pPool->cCurPages - 1].iNext = NIL_PGMPOOL_IDX;
5683	pPool->iFreeHead = PGMPOOL_IDX_FIRST;
5684	pPool->cUsedPages = 0;
5685
5686	/*
5687	* Zap and reinitialize the user records.
5688	*/
5689	pPool->cPresent = 0;
5690	pPool->iUserFreeHead = 0;
5691	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
5692	const unsigned cMaxUsers = pPool->cMaxUsers;
5693	for (unsigned i = 0; i < cMaxUsers; i++)
5694	{
5695	paUsers[i].iNext = i + 1;
5696	paUsers[i].iUser = NIL_PGMPOOL_IDX;
5697	paUsers[i].iUserTable = 0xfffffffe;
5698	}
5699	paUsers[cMaxUsers - 1].iNext = NIL_PGMPOOL_USER_INDEX;
5700
5701	/*
5702	* Clear all the GCPhys links and rebuild the phys ext free list.
5703	*/
5704	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
5705	pRam;
5706	pRam = pRam->CTX_SUFF(pNext))
5707	{
5708	unsigned iPage = pRam->cb >> PAGE_SHIFT;
5709	while (iPage-- > 0)
5710	PGM_PAGE_SET_TRACKING(pVM, &pRam->aPages[iPage], 0);
5711	}
5712
5713	pPool->iPhysExtFreeHead = 0;
5714	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
5715	const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
5716	for (unsigned i = 0; i < cMaxPhysExts; i++)
5717	{
5718	paPhysExts[i].iNext = i + 1;
5719	paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
5720	paPhysExts[i].apte[0] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
5721	paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
5722	paPhysExts[i].apte[1] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
5723	paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
5724	paPhysExts[i].apte[2] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
5725	}
5726	paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
5727
5728	/*
5729	* Just zap the modified list.
5730	*/
5731	pPool->cModifiedPages = 0;
5732	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
5733
5734	/*
5735	* Clear the GCPhys hash and the age list.
5736	*/
5737	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aiHash); i++)
5738	pPool->aiHash[i] = NIL_PGMPOOL_IDX;
5739	pPool->iAgeHead = NIL_PGMPOOL_IDX;
5740	pPool->iAgeTail = NIL_PGMPOOL_IDX;
5741
5742	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
5743	/* Clear all dirty pages. */
5744	pPool->idxFreeDirtyPage = 0;
5745	pPool->cDirtyPages = 0;
5746	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aidxDirtyPages); i++)
5747	pPool->aidxDirtyPages[i] = NIL_PGMPOOL_IDX;
5748	# endif
5749
5750	/*
5751	* Reinsert active pages into the hash and ensure monitoring chains are correct.
5752	*/
5753	VMCC_FOR_EACH_VMCPU(pVM)
5754	{
5755	/*
5756	* Re-enter the shadowing mode and assert Sync CR3 FF.
5757	*/
5758	pgmR3ReEnterShadowModeAfterPoolFlush(pVM, pVCpu);
5759	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
5760	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5761	}
5762	VMCC_FOR_EACH_VMCPU_END(pVM);
5763
5764	STAM_PROFILE_STOP(&pPool->StatR3Reset, a);
5765	}
5766
5767	#endif /* IN_RING3 */
5768
5769	#if defined(LOG_ENABLED) \|\| defined(VBOX_STRICT)
5770	/**
5771	* Stringifies a PGMPOOLKIND value.
5772	*/
5773	static const char *pgmPoolPoolKindToStr(uint8_t enmKind)
5774	{
5775	switch ((PGMPOOLKIND)enmKind)
5776	{
5777	case PGMPOOLKIND_INVALID:
5778	return "PGMPOOLKIND_INVALID";
5779	case PGMPOOLKIND_FREE:
5780	return "PGMPOOLKIND_FREE";
5781	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
5782	return "PGMPOOLKIND_32BIT_PT_FOR_PHYS";
5783	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
5784	return "PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT";
5785	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
5786	return "PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB";
5787	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
5788	return "PGMPOOLKIND_PAE_PT_FOR_PHYS";
5789	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
5790	return "PGMPOOLKIND_PAE_PT_FOR_32BIT_PT";
5791	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
5792	return "PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB";
5793	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
5794	return "PGMPOOLKIND_PAE_PT_FOR_PAE_PT";
5795	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
5796	return "PGMPOOLKIND_PAE_PT_FOR_PAE_2MB";
5797	case PGMPOOLKIND_32BIT_PD:
5798	return "PGMPOOLKIND_32BIT_PD";
5799	case PGMPOOLKIND_32BIT_PD_PHYS:
5800	return "PGMPOOLKIND_32BIT_PD_PHYS";
5801	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
5802	return "PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD";
5803	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
5804	return "PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD";
5805	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
5806	return "PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD";
5807	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
5808	return "PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD";
5809	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
5810	return "PGMPOOLKIND_PAE_PD_FOR_PAE_PD";
5811	case PGMPOOLKIND_PAE_PD_PHYS:
5812	return "PGMPOOLKIND_PAE_PD_PHYS";
5813	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
5814	return "PGMPOOLKIND_PAE_PDPT_FOR_32BIT";
5815	case PGMPOOLKIND_PAE_PDPT:
5816	return "PGMPOOLKIND_PAE_PDPT";
5817	case PGMPOOLKIND_PAE_PDPT_PHYS:
5818	return "PGMPOOLKIND_PAE_PDPT_PHYS";
5819	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
5820	return "PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT";
5821	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
5822	return "PGMPOOLKIND_64BIT_PDPT_FOR_PHYS";
5823	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
5824	return "PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD";
5825	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
5826	return "PGMPOOLKIND_64BIT_PD_FOR_PHYS";
5827	case PGMPOOLKIND_64BIT_PML4:
5828	return "PGMPOOLKIND_64BIT_PML4";
5829	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
5830	return "PGMPOOLKIND_EPT_PDPT_FOR_PHYS";
5831	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
5832	return "PGMPOOLKIND_EPT_PD_FOR_PHYS";
5833	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
5834	return "PGMPOOLKIND_EPT_PT_FOR_PHYS";
5835	case PGMPOOLKIND_ROOT_NESTED:
5836	return "PGMPOOLKIND_ROOT_NESTED";
5837	case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
5838	return "PGMPOOLKIND_EPT_PT_FOR_EPT_PT";
5839	case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
5840	return "PGMPOOLKIND_EPT_PD_FOR_EPT_PD";
5841	case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
5842	return "PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT";
5843	case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
5844	return "PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4";
5845	}
5846	return "Unknown kind!";
5847	}
5848	#endif /* LOG_ENABLED \|\| VBOX_STRICT */
5849

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllPool.cpp@ 96879

Download in other formats: