PGMAllPool.cpp@ 77807

Last change on this file since 77807 was 77240, checked in by vboxsync, 6 years ago
PGMPool: Two optimizations to the dirty page code (PGMPOOL_WITH_OPTIMIZED_DIRTY_PT): Inline the first part of pgmPoolIsDirtyPage so we don't waste time on a full fledged call for nested paging. Split the PGMPOOL::aDirtyPages structure into index and page data so we can scan the indexes without requiring a cache line load for each entry. Also eliminated some double uIdx reads and checks.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 216.0 KB

Line
1	/* $Id: PGMAllPool.cpp 77240 2019-02-10 16:34:51Z vboxsync $ */
2	/** @file
3	* PGM Shadow Page Pool.
4	*/
5
6	/*
7	* Copyright (C) 2006-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_PGM_POOL
23	#include <VBox/vmm/pgm.h>
24	#include <VBox/vmm/mm.h>
25	#include <VBox/vmm/em.h>
26	#include <VBox/vmm/cpum.h>
27	#ifdef IN_RC
28	# include <VBox/vmm/patm.h>
29	#endif
30	#include "PGMInternal.h"
31	#include <VBox/vmm/vm.h>
32	#include "PGMInline.h"
33	#include <VBox/disopcode.h>
34	#include <VBox/vmm/hm_vmx.h>
35
36	#include <VBox/log.h>
37	#include <VBox/err.h>
38	#include <iprt/asm.h>
39	#include <iprt/asm-amd64-x86.h>
40	#include <iprt/string.h>
41
42
43	/*********************************************************************************************************************************
44	* Internal Functions *
45	*********************************************************************************************************************************/
46	RT_C_DECLS_BEGIN
47	#if 0 /* unused */
48	DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind);
49	DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind);
50	#endif /* unused */
51	static void pgmPoolTrackClearPageUsers(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
52	static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
53	static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable);
54	static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
55	#if defined(LOG_ENABLED) \|\| defined(VBOX_STRICT)
56	static const char *pgmPoolPoolKindToStr(uint8_t enmKind);
57	#endif
58	#if 0 /defined(VBOX_STRICT) && defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT)/
59	static void pgmPoolTrackCheckPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT);
60	#endif
61
62	int pgmPoolTrackFlushGCPhysPTsSlow(PVM pVM, PPGMPAGE pPhysPage);
63	PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVM pVM, uint16_t *piPhysExt);
64	void pgmPoolTrackPhysExtFree(PVM pVM, uint16_t iPhysExt);
65	void pgmPoolTrackPhysExtFreeList(PVM pVM, uint16_t iPhysExt);
66
67	RT_C_DECLS_END
68
69
70	#if 0 /* unused */
71	/**
72	* Checks if the specified page pool kind is for a 4MB or 2MB guest page.
73	*
74	* @returns true if it's the shadow of a 4MB or 2MB guest page, otherwise false.
75	* @param enmKind The page kind.
76	*/
77	DECLINLINE(bool) pgmPoolIsBigPage(PGMPOOLKIND enmKind)
78	{
79	switch (enmKind)
80	{
81	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
82	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
83	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
84	return true;
85	default:
86	return false;
87	}
88	}
89	#endif /* unused */
90
91
92	/**
93	* Flushes a chain of pages sharing the same access monitor.
94	*
95	* @param pPool The pool.
96	* @param pPage A page in the chain.
97	*/
98	void pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
99	{
100	LogFlow(("pgmPoolMonitorChainFlush: Flush page %RGp type=%d\n", pPage->GCPhys, pPage->enmKind));
101
102	/*
103	* Find the list head.
104	*/
105	uint16_t idx = pPage->idx;
106	if (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
107	{
108	while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
109	{
110	idx = pPage->iMonitoredPrev;
111	Assert(idx != pPage->idx);
112	pPage = &pPool->aPages[idx];
113	}
114	}
115
116	/*
117	* Iterate the list flushing each shadow page.
118	*/
119	for (;;)
120	{
121	idx = pPage->iMonitoredNext;
122	Assert(idx != pPage->idx);
123	if (pPage->idx >= PGMPOOL_IDX_FIRST)
124	{
125	int rc2 = pgmPoolFlushPage(pPool, pPage);
126	AssertRC(rc2);
127	}
128	/* next */
129	if (idx == NIL_PGMPOOL_IDX)
130	break;
131	pPage = &pPool->aPages[idx];
132	}
133	}
134
135
136	/**
137	* Wrapper for getting the current context pointer to the entry being modified.
138	*
139	* @returns VBox status code suitable for scheduling.
140	* @param pVM The cross context VM structure.
141	* @param pvDst Destination address
142	* @param pvSrc Pointer to the mapping of @a GCPhysSrc or NULL depending
143	* on the context (e.g. \#PF in R0 & RC).
144	* @param GCPhysSrc The source guest physical address.
145	* @param cb Size of data to read
146	*/
147	DECLINLINE(int) pgmPoolPhysSimpleReadGCPhys(PVM pVM, void pvDst, void const pvSrc, RTGCPHYS GCPhysSrc, size_t cb)
148	{
149	#if defined(IN_RING3)
150	NOREF(pVM); NOREF(GCPhysSrc);
151	memcpy(pvDst, (RTHCPTR)((uintptr_t)pvSrc & ~(RTHCUINTPTR)(cb - 1)), cb);
152	return VINF_SUCCESS;
153	#else
154	/** @todo in RC we could attempt to use the virtual address, although this can cause many faults (PAE Windows XP guest). */
155	NOREF(pvSrc);
156	return PGMPhysSimpleReadGCPhys(pVM, pvDst, GCPhysSrc & ~(RTGCPHYS)(cb - 1), cb);
157	#endif
158	}
159
160
161	/**
162	* Process shadow entries before they are changed by the guest.
163	*
164	* For PT entries we will clear them. For PD entries, we'll simply check
165	* for mapping conflicts and set the SyncCR3 FF if found.
166	*
167	* @param pVCpu The cross context virtual CPU structure.
168	* @param pPool The pool.
169	* @param pPage The head page.
170	* @param GCPhysFault The guest physical fault address.
171	* @param pvAddress Pointer to the mapping of @a GCPhysFault or NULL
172	* depending on the context (e.g. \#PF in R0 & RC).
173	* @param cbWrite Write size; might be zero if the caller knows we're not crossing entry boundaries
174	*/
175	static void pgmPoolMonitorChainChanging(PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault,
176	void const *pvAddress, unsigned cbWrite)
177	{
178	AssertMsg(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX, ("%u (idx=%u)\n", pPage->iMonitoredPrev, pPage->idx));
179	const unsigned off = GCPhysFault & PAGE_OFFSET_MASK;
180	PVM pVM = pPool->CTX_SUFF(pVM);
181	NOREF(pVCpu);
182
183	LogFlow(("pgmPoolMonitorChainChanging: %RGv phys=%RGp cbWrite=%d\n",
184	(RTGCPTR)(CTXTYPE(RTGCPTR, uintptr_t, RTGCPTR))(uintptr_t)pvAddress, GCPhysFault, cbWrite));
185
186	for (;;)
187	{
188	union
189	{
190	void *pv;
191	PX86PT pPT;
192	PPGMSHWPTPAE pPTPae;
193	PX86PD pPD;
194	PX86PDPAE pPDPae;
195	PX86PDPT pPDPT;
196	PX86PML4 pPML4;
197	} uShw;
198
199	LogFlow(("pgmPoolMonitorChainChanging: page idx=%d phys=%RGp (next=%d) kind=%s write=%#x\n",
200	pPage->idx, pPage->GCPhys, pPage->iMonitoredNext, pgmPoolPoolKindToStr(pPage->enmKind), cbWrite));
201
202	uShw.pv = NULL;
203	switch (pPage->enmKind)
204	{
205	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
206	{
207	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT));
208	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
209	const unsigned iShw = off / sizeof(X86PTE);
210	LogFlow(("PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT iShw=%x\n", iShw));
211	if (uShw.pPT->a[iShw].n.u1Present)
212	{
213	X86PTE GstPte;
214
215	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
216	AssertRC(rc);
217	Log4(("pgmPoolMonitorChainChanging 32_32: deref %016RX64 GCPhys %08RX32\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PG_MASK));
218	pgmPoolTracDerefGCPhysHint(pPool, pPage,
219	uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK,
220	GstPte.u & X86_PTE_PG_MASK,
221	iShw);
222	ASMAtomicWriteU32(&uShw.pPT->a[iShw].u, 0);
223	}
224	break;
225	}
226
227	/* page/2 sized */
228	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
229	{
230	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT));
231	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
232	if (!((off ^ pPage->GCPhys) & (PAGE_SIZE / 2)))
233	{
234	const unsigned iShw = (off / sizeof(X86PTE)) & (X86_PG_PAE_ENTRIES - 1);
235	LogFlow(("PGMPOOLKIND_PAE_PT_FOR_32BIT_PT iShw=%x\n", iShw));
236	if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw]))
237	{
238	X86PTE GstPte;
239	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
240	AssertRC(rc);
241
242	Log4(("pgmPoolMonitorChainChanging pae_32: deref %016RX64 GCPhys %08RX32\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PG_MASK));
243	pgmPoolTracDerefGCPhysHint(pPool, pPage,
244	PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw]),
245	GstPte.u & X86_PTE_PG_MASK,
246	iShw);
247	PGMSHWPTEPAE_ATOMIC_SET(uShw.pPTPae->a[iShw], 0);
248	}
249	}
250	break;
251	}
252
253	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
254	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
255	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
256	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
257	{
258	unsigned iGst = off / sizeof(X86PDE);
259	unsigned iShwPdpt = iGst / 256;
260	unsigned iShw = (iGst % 256) * 2;
261	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
262
263	LogFlow(("pgmPoolMonitorChainChanging PAE for 32 bits: iGst=%x iShw=%x idx = %d page idx=%d\n", iGst, iShw, iShwPdpt, pPage->enmKind - PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD));
264	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
265	if (iShwPdpt == pPage->enmKind - (unsigned)PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD)
266	{
267	for (unsigned i = 0; i < 2; i++)
268	{
269	# ifdef VBOX_WITH_RAW_MODE_NOT_R0
270	if ((uShw.pPDPae->a[iShw + i].u & (PGM_PDFLAGS_MAPPING \| X86_PDE_P)) == (PGM_PDFLAGS_MAPPING \| X86_PDE_P))
271	{
272	Assert(pgmMapAreMappingsEnabled(pVM));
273	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
274	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShwPdpt=%#x iShw=%#x!\n", iShwPdpt, iShw+i));
275	break;
276	}
277	# endif /* VBOX_WITH_RAW_MODE_NOT_R0 */
278	if (uShw.pPDPae->a[iShw+i].n.u1Present)
279	{
280	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw+i, uShw.pPDPae->a[iShw+i].u));
281	pgmPoolFree(pVM,
282	uShw.pPDPae->a[iShw+i].u & X86_PDE_PAE_PG_MASK,
283	pPage->idx,
284	iShw + i);
285	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw+i].u, 0);
286	}
287
288	/* paranoia / a bit assumptive. */
289	if ( (off & 3)
290	&& (off & 3) + cbWrite > 4)
291	{
292	const unsigned iShw2 = iShw + 2 + i;
293	if (iShw2 < RT_ELEMENTS(uShw.pPDPae->a))
294	{
295	# ifdef VBOX_WITH_RAW_MODE_NOT_R0
296	if ((uShw.pPDPae->a[iShw2].u & (PGM_PDFLAGS_MAPPING \| X86_PDE_P)) == (PGM_PDFLAGS_MAPPING \| X86_PDE_P))
297	{
298	Assert(pgmMapAreMappingsEnabled(pVM));
299	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
300	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShwPdpt=%#x iShw2=%#x!\n", iShwPdpt, iShw2));
301	break;
302	}
303	# endif /* VBOX_WITH_RAW_MODE_NOT_R0 */
304	if (uShw.pPDPae->a[iShw2].n.u1Present)
305	{
306	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPae->a[iShw2].u));
307	pgmPoolFree(pVM,
308	uShw.pPDPae->a[iShw2].u & X86_PDE_PAE_PG_MASK,
309	pPage->idx,
310	iShw2);
311	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw2].u, 0);
312	}
313	}
314	}
315	}
316	}
317	break;
318	}
319
320	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
321	{
322	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
323	const unsigned iShw = off / sizeof(X86PTEPAE);
324	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT));
325	if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw]))
326	{
327	X86PTEPAE GstPte;
328	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
329	AssertRC(rc);
330
331	Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw]), GstPte.u & X86_PTE_PAE_PG_MASK));
332	pgmPoolTracDerefGCPhysHint(pPool, pPage,
333	PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw]),
334	GstPte.u & X86_PTE_PAE_PG_MASK,
335	iShw);
336	PGMSHWPTEPAE_ATOMIC_SET(uShw.pPTPae->a[iShw], 0);
337	}
338
339	/* paranoia / a bit assumptive. */
340	if ( (off & 7)
341	&& (off & 7) + cbWrite > sizeof(X86PTEPAE))
342	{
343	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTEPAE);
344	AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPTPae->a));
345
346	if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw2]))
347	{
348	X86PTEPAE GstPte;
349	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte,
350	pvAddress ? (uint8_t const *)pvAddress + sizeof(GstPte) : NULL,
351	GCPhysFault + sizeof(GstPte), sizeof(GstPte));
352	AssertRC(rc);
353	Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw2]), GstPte.u & X86_PTE_PAE_PG_MASK));
354	pgmPoolTracDerefGCPhysHint(pPool, pPage,
355	PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw2]),
356	GstPte.u & X86_PTE_PAE_PG_MASK,
357	iShw2);
358	PGMSHWPTEPAE_ATOMIC_SET(uShw.pPTPae->a[iShw2], 0);
359	}
360	}
361	break;
362	}
363
364	case PGMPOOLKIND_32BIT_PD:
365	{
366	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
367	const unsigned iShw = off / sizeof(X86PTE); // ASSUMING 32-bit guest paging!
368
369	LogFlow(("pgmPoolMonitorChainChanging: PGMPOOLKIND_32BIT_PD %x\n", iShw));
370	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
371	# ifdef VBOX_WITH_RAW_MODE_NOT_R0
372	if (uShw.pPD->a[iShw].u & PGM_PDFLAGS_MAPPING)
373	{
374	Assert(pgmMapAreMappingsEnabled(pVM));
375	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
376	STAM_COUNTER_INC(&(pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestCR3WriteConflict));
377	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
378	break;
379	}
380	else
381	# endif /* VBOX_WITH_RAW_MODE_NOT_R0 */
382	{
383	if (uShw.pPD->a[iShw].n.u1Present)
384	{
385	LogFlow(("pgmPoolMonitorChainChanging: 32 bit pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPD->a[iShw].u));
386	pgmPoolFree(pVM,
387	uShw.pPD->a[iShw].u & X86_PDE_PAE_PG_MASK,
388	pPage->idx,
389	iShw);
390	ASMAtomicWriteU32(&uShw.pPD->a[iShw].u, 0);
391	}
392	}
393	/* paranoia / a bit assumptive. */
394	if ( (off & 3)
395	&& (off & 3) + cbWrite > sizeof(X86PTE))
396	{
397	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTE);
398	if ( iShw2 != iShw
399	&& iShw2 < RT_ELEMENTS(uShw.pPD->a))
400	{
401	# ifdef VBOX_WITH_RAW_MODE_NOT_R0
402	if (uShw.pPD->a[iShw2].u & PGM_PDFLAGS_MAPPING)
403	{
404	Assert(pgmMapAreMappingsEnabled(pVM));
405	STAM_COUNTER_INC(&(pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestCR3WriteConflict));
406	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
407	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
408	break;
409	}
410	# endif /* VBOX_WITH_RAW_MODE_NOT_R0 */
411	if (uShw.pPD->a[iShw2].n.u1Present)
412	{
413	LogFlow(("pgmPoolMonitorChainChanging: 32 bit pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPD->a[iShw2].u));
414	pgmPoolFree(pVM,
415	uShw.pPD->a[iShw2].u & X86_PDE_PAE_PG_MASK,
416	pPage->idx,
417	iShw2);
418	ASMAtomicWriteU32(&uShw.pPD->a[iShw2].u, 0);
419	}
420	}
421	}
422	#if 0 /* useful when running PGMAssertCR3(), a bit too troublesome for general use (TLBs). - not working any longer... */
423	if ( uShw.pPD->a[iShw].n.u1Present
424	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
425	{
426	LogFlow(("pgmPoolMonitorChainChanging: iShw=%#x: %RX32 -> freeing it!\n", iShw, uShw.pPD->a[iShw].u));
427	# ifdef IN_RC /* TLB load - we're pushing things a bit... */
428	ASMProbeReadByte(pvAddress);
429	# endif
430	pgmPoolFree(pVM, uShw.pPD->a[iShw].u & X86_PDE_PG_MASK, pPage->idx, iShw);
431	ASMAtomicWriteU32(&uShw.pPD->a[iShw].u, 0);
432	}
433	#endif
434	break;
435	}
436
437	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
438	{
439	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
440	const unsigned iShw = off / sizeof(X86PDEPAE);
441	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
442	#ifdef VBOX_WITH_RAW_MODE_NOT_R0
443	if (uShw.pPDPae->a[iShw].u & PGM_PDFLAGS_MAPPING)
444	{
445	Assert(pgmMapAreMappingsEnabled(pVM));
446	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
447	STAM_COUNTER_INC(&(pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestCR3WriteConflict));
448	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
449	break;
450	}
451	#endif /* VBOX_WITH_RAW_MODE_NOT_R0 */
452	/*
453	* Causes trouble when the guest uses a PDE to refer to the whole page table level
454	* structure. (Invalidate here; faults later on when it tries to change the page
455	* table entries -> recheck; probably only applies to the RC case.)
456	*/
457	#ifdef VBOX_WITH_RAW_MODE_NOT_R0
458	else
459	#endif
460	{
461	if (uShw.pPDPae->a[iShw].n.u1Present)
462	{
463	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPae->a[iShw].u));
464	pgmPoolFree(pVM,
465	uShw.pPDPae->a[iShw].u & X86_PDE_PAE_PG_MASK,
466	pPage->idx,
467	iShw);
468	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw].u, 0);
469	}
470	}
471	/* paranoia / a bit assumptive. */
472	if ( (off & 7)
473	&& (off & 7) + cbWrite > sizeof(X86PDEPAE))
474	{
475	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE);
476	AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPDPae->a));
477
478	#ifdef VBOX_WITH_RAW_MODE_NOT_R0
479	if ( iShw2 != iShw
480	&& uShw.pPDPae->a[iShw2].u & PGM_PDFLAGS_MAPPING)
481	{
482	Assert(pgmMapAreMappingsEnabled(pVM));
483	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
484	STAM_COUNTER_INC(&(pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestCR3WriteConflict));
485	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
486	break;
487	}
488	else
489	#endif /* VBOX_WITH_RAW_MODE_NOT_R0 */
490	if (uShw.pPDPae->a[iShw2].n.u1Present)
491	{
492	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPae->a[iShw2].u));
493	pgmPoolFree(pVM,
494	uShw.pPDPae->a[iShw2].u & X86_PDE_PAE_PG_MASK,
495	pPage->idx,
496	iShw2);
497	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw2].u, 0);
498	}
499	}
500	break;
501	}
502
503	case PGMPOOLKIND_PAE_PDPT:
504	{
505	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPDPT));
506	/*
507	* Hopefully this doesn't happen very often:
508	* - touching unused parts of the page
509	* - messing with the bits of pd pointers without changing the physical address
510	*/
511	/* PDPT roots are not page aligned; 32 byte only! */
512	const unsigned offPdpt = GCPhysFault - pPage->GCPhys;
513
514	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
515	const unsigned iShw = offPdpt / sizeof(X86PDPE);
516	if (iShw < X86_PG_PAE_PDPE_ENTRIES) /* don't use RT_ELEMENTS(uShw.pPDPT->a), because that's for long mode only */
517	{
518	# ifdef VBOX_WITH_RAW_MODE_NOT_R0
519	if (uShw.pPDPT->a[iShw].u & PGM_PLXFLAGS_MAPPING)
520	{
521	Assert(pgmMapAreMappingsEnabled(pVM));
522	STAM_COUNTER_INC(&(pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestCR3WriteConflict));
523	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
524	LogFlow(("pgmPoolMonitorChainChanging: Detected pdpt conflict at iShw=%#x!\n", iShw));
525	break;
526	}
527	else
528	# endif /* VBOX_WITH_RAW_MODE_NOT_R0 */
529	if (uShw.pPDPT->a[iShw].n.u1Present)
530	{
531	LogFlow(("pgmPoolMonitorChainChanging: pae pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPT->a[iShw].u));
532	pgmPoolFree(pVM,
533	uShw.pPDPT->a[iShw].u & X86_PDPE_PG_MASK,
534	pPage->idx,
535	iShw);
536	ASMAtomicWriteU64(&uShw.pPDPT->a[iShw].u, 0);
537	}
538
539	/* paranoia / a bit assumptive. */
540	if ( (offPdpt & 7)
541	&& (offPdpt & 7) + cbWrite > sizeof(X86PDPE))
542	{
543	const unsigned iShw2 = (offPdpt + cbWrite - 1) / sizeof(X86PDPE);
544	if ( iShw2 != iShw
545	&& iShw2 < X86_PG_PAE_PDPE_ENTRIES)
546	{
547	# ifdef VBOX_WITH_RAW_MODE_NOT_R0
548	if (uShw.pPDPT->a[iShw2].u & PGM_PLXFLAGS_MAPPING)
549	{
550	Assert(pgmMapAreMappingsEnabled(pVM));
551	STAM_COUNTER_INC(&(pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestCR3WriteConflict));
552	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
553	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
554	break;
555	}
556	else
557	# endif /* VBOX_WITH_RAW_MODE_NOT_R0 */
558	if (uShw.pPDPT->a[iShw2].n.u1Present)
559	{
560	LogFlow(("pgmPoolMonitorChainChanging: pae pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPT->a[iShw2].u));
561	pgmPoolFree(pVM,
562	uShw.pPDPT->a[iShw2].u & X86_PDPE_PG_MASK,
563	pPage->idx,
564	iShw2);
565	ASMAtomicWriteU64(&uShw.pPDPT->a[iShw2].u, 0);
566	}
567	}
568	}
569	}
570	break;
571	}
572
573	#ifndef IN_RC
574	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
575	{
576	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
577	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
578	const unsigned iShw = off / sizeof(X86PDEPAE);
579	Assert(!(uShw.pPDPae->a[iShw].u & PGM_PDFLAGS_MAPPING));
580	if (uShw.pPDPae->a[iShw].n.u1Present)
581	{
582	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPae->a[iShw].u));
583	pgmPoolFree(pVM,
584	uShw.pPDPae->a[iShw].u & X86_PDE_PAE_PG_MASK,
585	pPage->idx,
586	iShw);
587	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw].u, 0);
588	}
589	/* paranoia / a bit assumptive. */
590	if ( (off & 7)
591	&& (off & 7) + cbWrite > sizeof(X86PDEPAE))
592	{
593	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE);
594	AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPDPae->a));
595
596	Assert(!(uShw.pPDPae->a[iShw2].u & PGM_PDFLAGS_MAPPING));
597	if (uShw.pPDPae->a[iShw2].n.u1Present)
598	{
599	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPae->a[iShw2].u));
600	pgmPoolFree(pVM,
601	uShw.pPDPae->a[iShw2].u & X86_PDE_PAE_PG_MASK,
602	pPage->idx,
603	iShw2);
604	ASMAtomicWriteU64(&uShw.pPDPae->a[iShw2].u, 0);
605	}
606	}
607	break;
608	}
609
610	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
611	{
612	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPDPT));
613	/*
614	* Hopefully this doesn't happen very often:
615	* - messing with the bits of pd pointers without changing the physical address
616	*/
617	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
618	const unsigned iShw = off / sizeof(X86PDPE);
619	if (uShw.pPDPT->a[iShw].n.u1Present)
620	{
621	LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPT->a[iShw].u));
622	pgmPoolFree(pVM, uShw.pPDPT->a[iShw].u & X86_PDPE_PG_MASK, pPage->idx, iShw);
623	ASMAtomicWriteU64(&uShw.pPDPT->a[iShw].u, 0);
624	}
625	/* paranoia / a bit assumptive. */
626	if ( (off & 7)
627	&& (off & 7) + cbWrite > sizeof(X86PDPE))
628	{
629	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDPE);
630	if (uShw.pPDPT->a[iShw2].n.u1Present)
631	{
632	LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPT->a[iShw2].u));
633	pgmPoolFree(pVM, uShw.pPDPT->a[iShw2].u & X86_PDPE_PG_MASK, pPage->idx, iShw2);
634	ASMAtomicWriteU64(&uShw.pPDPT->a[iShw2].u, 0);
635	}
636	}
637	break;
638	}
639
640	case PGMPOOLKIND_64BIT_PML4:
641	{
642	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPML4));
643	/*
644	* Hopefully this doesn't happen very often:
645	* - messing with the bits of pd pointers without changing the physical address
646	*/
647	uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
648	const unsigned iShw = off / sizeof(X86PDPE);
649	if (uShw.pPML4->a[iShw].n.u1Present)
650	{
651	LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPML4->a[iShw].u));
652	pgmPoolFree(pVM, uShw.pPML4->a[iShw].u & X86_PML4E_PG_MASK, pPage->idx, iShw);
653	ASMAtomicWriteU64(&uShw.pPML4->a[iShw].u, 0);
654	}
655	/* paranoia / a bit assumptive. */
656	if ( (off & 7)
657	&& (off & 7) + cbWrite > sizeof(X86PDPE))
658	{
659	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PML4E);
660	if (uShw.pPML4->a[iShw2].n.u1Present)
661	{
662	LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPML4->a[iShw2].u));
663	pgmPoolFree(pVM, uShw.pPML4->a[iShw2].u & X86_PML4E_PG_MASK, pPage->idx, iShw2);
664	ASMAtomicWriteU64(&uShw.pPML4->a[iShw2].u, 0);
665	}
666	}
667	break;
668	}
669	#endif /* IN_RING0 */
670
671	default:
672	AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
673	}
674	PGM_DYNMAP_UNUSED_HINT_VM(pVM, uShw.pv);
675
676	/* next */
677	if (pPage->iMonitoredNext == NIL_PGMPOOL_IDX)
678	return;
679	pPage = &pPool->aPages[pPage->iMonitoredNext];
680	}
681	}
682
683	#ifndef IN_RING3
684
685	/**
686	* Checks if a access could be a fork operation in progress.
687	*
688	* Meaning, that the guest is setting up the parent process for Copy-On-Write.
689	*
690	* @returns true if it's likely that we're forking, otherwise false.
691	* @param pPool The pool.
692	* @param pDis The disassembled instruction.
693	* @param offFault The access offset.
694	*/
695	DECLINLINE(bool) pgmRZPoolMonitorIsForking(PPGMPOOL pPool, PDISCPUSTATE pDis, unsigned offFault)
696	{
697	/*
698	* i386 linux is using btr to clear X86_PTE_RW.
699	* The functions involved are (2.6.16 source inspection):
700	* clear_bit
701	* ptep_set_wrprotect
702	* copy_one_pte
703	* copy_pte_range
704	* copy_pmd_range
705	* copy_pud_range
706	* copy_page_range
707	* dup_mmap
708	* dup_mm
709	* copy_mm
710	* copy_process
711	* do_fork
712	*/
713	if ( pDis->pCurInstr->uOpcode == OP_BTR
714	&& !(offFault & 4)
715	/** @todo Validate that the bit index is X86_PTE_RW. */
716	)
717	{
718	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,Fork)); RT_NOREF_PV(pPool);
719	return true;
720	}
721	return false;
722	}
723
724
725	/**
726	* Determine whether the page is likely to have been reused.
727	*
728	* @returns true if we consider the page as being reused for a different purpose.
729	* @returns false if we consider it to still be a paging page.
730	* @param pVM The cross context VM structure.
731	* @param pVCpu The cross context virtual CPU structure.
732	* @param pRegFrame Trap register frame.
733	* @param pDis The disassembly info for the faulting instruction.
734	* @param pvFault The fault address.
735	* @param pPage The pool page being accessed.
736	*
737	* @remark The REP prefix check is left to the caller because of STOSD/W.
738	*/
739	DECLINLINE(bool) pgmRZPoolMonitorIsReused(PVM pVM, PVMCPU pVCpu, PCPUMCTXCORE pRegFrame, PDISCPUSTATE pDis, RTGCPTR pvFault,
740	PPGMPOOLPAGE pPage)
741	{
742	/* Locked (CR3, PDPTR*4) should not be reusable. Considering them as
743	such may cause loops booting tst-ubuntu-15_10-64-efi, ++. */
744	if (pPage->cLocked)
745	{
746	Log2(("pgmRZPoolMonitorIsReused: %RGv (%p) can't have been resued, because it's locked!\n", pvFault, pPage));
747	return false;
748	}
749
750	# ifndef IN_RC
751	/** @todo could make this general, faulting close to rsp should be a safe reuse heuristic. */
752	if ( HMHasPendingIrq(pVM)
753	&& (pRegFrame->rsp - pvFault) < 32)
754	{
755	/* Fault caused by stack writes while trying to inject an interrupt event. */
756	Log(("pgmRZPoolMonitorIsReused: reused %RGv for interrupt stack (rsp=%RGv).\n", pvFault, pRegFrame->rsp));
757	return true;
758	}
759	# else
760	NOREF(pVM); NOREF(pvFault);
761	# endif
762
763	LogFlow(("Reused instr %RGv %d at %RGv param1.fUse=%llx param1.reg=%d\n", pRegFrame->rip, pDis->pCurInstr->uOpcode, pvFault, pDis->Param1.fUse, pDis->Param1.Base.idxGenReg));
764
765	/* Non-supervisor mode write means it's used for something else. */
766	if (CPUMGetGuestCPL(pVCpu) == 3)
767	return true;
768
769	switch (pDis->pCurInstr->uOpcode)
770	{
771	/* call implies the actual push of the return address faulted */
772	case OP_CALL:
773	Log4(("pgmRZPoolMonitorIsReused: CALL\n"));
774	return true;
775	case OP_PUSH:
776	Log4(("pgmRZPoolMonitorIsReused: PUSH\n"));
777	return true;
778	case OP_PUSHF:
779	Log4(("pgmRZPoolMonitorIsReused: PUSHF\n"));
780	return true;
781	case OP_PUSHA:
782	Log4(("pgmRZPoolMonitorIsReused: PUSHA\n"));
783	return true;
784	case OP_FXSAVE:
785	Log4(("pgmRZPoolMonitorIsReused: FXSAVE\n"));
786	return true;
787	case OP_MOVNTI: /* solaris - block_zero_no_xmm */
788	Log4(("pgmRZPoolMonitorIsReused: MOVNTI\n"));
789	return true;
790	case OP_MOVNTDQ: /* solaris - hwblkclr & hwblkpagecopy */
791	Log4(("pgmRZPoolMonitorIsReused: MOVNTDQ\n"));
792	return true;
793	case OP_MOVSWD:
794	case OP_STOSWD:
795	if ( pDis->fPrefix == (DISPREFIX_REP\|DISPREFIX_REX)
796	&& pRegFrame->rcx >= 0x40
797	)
798	{
799	Assert(pDis->uCpuMode == DISCPUMODE_64BIT);
800
801	Log(("pgmRZPoolMonitorIsReused: OP_STOSQ\n"));
802	return true;
803	}
804	break;
805
806	default:
807	/*
808	* Anything having ESP on the left side means stack writes.
809	*/
810	if ( ( (pDis->Param1.fUse & DISUSE_REG_GEN32)
811	\|\| (pDis->Param1.fUse & DISUSE_REG_GEN64))
812	&& (pDis->Param1.Base.idxGenReg == DISGREG_ESP))
813	{
814	Log4(("pgmRZPoolMonitorIsReused: ESP\n"));
815	return true;
816	}
817	break;
818	}
819
820	/*
821	* Page table updates are very very unlikely to be crossing page boundraries,
822	* and we don't want to deal with that in pgmPoolMonitorChainChanging and such.
823	*/
824	uint32_t const cbWrite = DISGetParamSize(pDis, &pDis->Param1);
825	if ( (((uintptr_t)pvFault + cbWrite) >> X86_PAGE_SHIFT) != ((uintptr_t)pvFault >> X86_PAGE_SHIFT) )
826	{
827	Log4(("pgmRZPoolMonitorIsReused: cross page write\n"));
828	return true;
829	}
830
831	/*
832	* Nobody does an unaligned 8 byte write to a page table, right.
833	*/
834	if (cbWrite >= 8 && ((uintptr_t)pvFault & 7) != 0)
835	{
836	Log4(("pgmRZPoolMonitorIsReused: Unaligned 8+ byte write\n"));
837	return true;
838	}
839
840	return false;
841	}
842
843
844	/**
845	* Flushes the page being accessed.
846	*
847	* @returns VBox status code suitable for scheduling.
848	* @param pVM The cross context VM structure.
849	* @param pVCpu The cross context virtual CPU structure.
850	* @param pPool The pool.
851	* @param pPage The pool page (head).
852	* @param pDis The disassembly of the write instruction.
853	* @param pRegFrame The trap register frame.
854	* @param GCPhysFault The fault address as guest physical address.
855	* @param pvFault The fault address.
856	* @todo VBOXSTRICTRC
857	*/
858	static int pgmRZPoolAccessPfHandlerFlush(PVM pVM, PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
859	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
860	{
861	NOREF(pVM); NOREF(GCPhysFault);
862
863	/*
864	* First, do the flushing.
865	*/
866	pgmPoolMonitorChainFlush(pPool, pPage);
867
868	/*
869	* Emulate the instruction (xp/w2k problem, requires pc/cr2/sp detection).
870	* Must do this in raw mode (!); XP boot will fail otherwise.
871	*/
872	int rc = VINF_SUCCESS;
873	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, pRegFrame, pvFault, EMCODETYPE_ALL);
874	if (rc2 == VINF_SUCCESS)
875	{ /* do nothing */ }
876	else if (rc2 == VINF_EM_RESCHEDULE)
877	{
878	rc = VBOXSTRICTRC_VAL(rc2);
879	# ifndef IN_RING3
880	VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
881	# endif
882	}
883	else if (rc2 == VERR_EM_INTERPRETER)
884	{
885	# ifdef IN_RC
886	if (PATMIsPatchGCAddr(pVM, pRegFrame->eip))
887	{
888	LogFlow(("pgmRZPoolAccessPfHandlerFlush: Interpretation failed for patch code %04x:%RGv, ignoring.\n",
889	pRegFrame->cs.Sel, (RTGCPTR)pRegFrame->eip));
890	rc = VINF_SUCCESS;
891	STAM_COUNTER_INC(&pPool->StatMonitorPfRZIntrFailPatch2);
892	}
893	else
894	# endif
895	{
896	rc = VINF_EM_RAW_EMULATE_INSTR;
897	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,EmulateInstr));
898	}
899	}
900	else if (RT_FAILURE_NP(rc2))
901	rc = VBOXSTRICTRC_VAL(rc2);
902	else
903	AssertMsgFailed(("%Rrc\n", VBOXSTRICTRC_VAL(rc2))); /* ASSUMES no complicated stuff here. */
904
905	LogFlow(("pgmRZPoolAccessPfHandlerFlush: returns %Rrc (flushed)\n", rc));
906	return rc;
907	}
908
909
910	/**
911	* Handles the STOSD write accesses.
912	*
913	* @returns VBox status code suitable for scheduling.
914	* @param pVM The cross context VM structure.
915	* @param pPool The pool.
916	* @param pPage The pool page (head).
917	* @param pDis The disassembly of the write instruction.
918	* @param pRegFrame The trap register frame.
919	* @param GCPhysFault The fault address as guest physical address.
920	* @param pvFault The fault address.
921	*/
922	DECLINLINE(int) pgmRZPoolAccessPfHandlerSTOSD(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
923	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
924	{
925	unsigned uIncrement = pDis->Param1.cb;
926	NOREF(pVM);
927
928	Assert(pDis->uCpuMode == DISCPUMODE_32BIT \|\| pDis->uCpuMode == DISCPUMODE_64BIT);
929	Assert(pRegFrame->rcx <= 0x20);
930
931	# ifdef VBOX_STRICT
932	if (pDis->uOpMode == DISCPUMODE_32BIT)
933	Assert(uIncrement == 4);
934	else
935	Assert(uIncrement == 8);
936	# endif
937
938	Log3(("pgmRZPoolAccessPfHandlerSTOSD\n"));
939
940	/*
941	* Increment the modification counter and insert it into the list
942	* of modified pages the first time.
943	*/
944	if (!pPage->cModifications++)
945	pgmPoolMonitorModifiedInsert(pPool, pPage);
946
947	/*
948	* Execute REP STOSD.
949	*
950	* This ASSUMES that we're not invoked by Trap0e on in a out-of-sync
951	* write situation, meaning that it's safe to write here.
952	*/
953	PVMCPU pVCpu = VMMGetCpu(pPool->CTX_SUFF(pVM));
954	RTGCUINTPTR pu32 = (RTGCUINTPTR)pvFault;
955	while (pRegFrame->rcx)
956	{
957	# if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) \|\| defined(IN_RC)
958	uint32_t iPrevSubset = PGMRZDynMapPushAutoSubset(pVCpu);
959	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, uIncrement);
960	PGMRZDynMapPopAutoSubset(pVCpu, iPrevSubset);
961	# else
962	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, uIncrement);
963	# endif
964	# ifdef IN_RC
965	(uint32_t )(uintptr_t)pu32 = pRegFrame->eax;
966	# else
967	PGMPhysSimpleWriteGCPhys(pVM, GCPhysFault, &pRegFrame->rax, uIncrement);
968	# endif
969	pu32 += uIncrement;
970	GCPhysFault += uIncrement;
971	pRegFrame->rdi += uIncrement;
972	pRegFrame->rcx--;
973	}
974	pRegFrame->rip += pDis->cbInstr;
975
976	LogFlow(("pgmRZPoolAccessPfHandlerSTOSD: returns\n"));
977	return VINF_SUCCESS;
978	}
979
980
981	/**
982	* Handles the simple write accesses.
983	*
984	* @returns VBox status code suitable for scheduling.
985	* @param pVM The cross context VM structure.
986	* @param pVCpu The cross context virtual CPU structure.
987	* @param pPool The pool.
988	* @param pPage The pool page (head).
989	* @param pDis The disassembly of the write instruction.
990	* @param pRegFrame The trap register frame.
991	* @param GCPhysFault The fault address as guest physical address.
992	* @param pvFault The fault address.
993	* @param pfReused Reused state (in/out)
994	*/
995	DECLINLINE(int) pgmRZPoolAccessPfHandlerSimple(PVM pVM, PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
996	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault, bool *pfReused)
997	{
998	Log3(("pgmRZPoolAccessPfHandlerSimple\n"));
999	NOREF(pVM);
1000	NOREF(pfReused); /* initialized by caller */
1001
1002	/*
1003	* Increment the modification counter and insert it into the list
1004	* of modified pages the first time.
1005	*/
1006	if (!pPage->cModifications++)
1007	pgmPoolMonitorModifiedInsert(pPool, pPage);
1008
1009	/*
1010	* Clear all the pages. ASSUMES that pvFault is readable.
1011	*/
1012	# if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) \|\| defined(IN_RC)
1013	uint32_t iPrevSubset = PGMRZDynMapPushAutoSubset(pVCpu);
1014	# endif
1015
1016	uint32_t cbWrite = DISGetParamSize(pDis, &pDis->Param1);
1017	if (cbWrite <= 8)
1018	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, cbWrite);
1019	else if (cbWrite <= 16)
1020	{
1021	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, 8);
1022	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault + 8, NULL, cbWrite - 8);
1023	}
1024	else
1025	{
1026	Assert(cbWrite <= 32);
1027	for (uint32_t off = 0; off < cbWrite; off += 8)
1028	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault + off, NULL, RT_MIN(8, cbWrite - off));
1029	}
1030
1031	# if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) \|\| defined(IN_RC)
1032	PGMRZDynMapPopAutoSubset(pVCpu, iPrevSubset);
1033	# endif
1034
1035	/*
1036	* Interpret the instruction.
1037	*/
1038	VBOXSTRICTRC rc = EMInterpretInstructionDisasState(pVCpu, pDis, pRegFrame, pvFault, EMCODETYPE_ALL);
1039	if (RT_SUCCESS(rc))
1040	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", VBOXSTRICTRC_VAL(rc))); /* ASSUMES no complicated stuff here. */
1041	else if (rc == VERR_EM_INTERPRETER)
1042	{
1043	LogFlow(("pgmRZPoolAccessPfHandlerSimple: Interpretation failed for %04x:%RGv - opcode=%d\n",
1044	pRegFrame->cs.Sel, (RTGCPTR)pRegFrame->rip, pDis->pCurInstr->uOpcode));
1045	rc = VINF_EM_RAW_EMULATE_INSTR;
1046	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,EmulateInstr));
1047	}
1048
1049	# if 0 /* experimental code */
1050	if (rc == VINF_SUCCESS)
1051	{
1052	switch (pPage->enmKind)
1053	{
1054	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1055	{
1056	X86PTEPAE GstPte;
1057	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvFault, GCPhysFault, sizeof(GstPte));
1058	AssertRC(rc);
1059
1060	/* Check the new value written by the guest. If present and with a bogus physical address, then
1061	* it's fairly safe to assume the guest is reusing the PT.
1062	*/
1063	if (GstPte.n.u1Present)
1064	{
1065	RTHCPHYS HCPhys = -1;
1066	int rc = PGMPhysGCPhys2HCPhys(pVM, GstPte.u & X86_PTE_PAE_PG_MASK, &HCPhys);
1067	if (rc != VINF_SUCCESS)
1068	{
1069	*pfReused = true;
1070	STAM_COUNTER_INC(&pPool->StatForceFlushReused);
1071	}
1072	}
1073	break;
1074	}
1075	}
1076	}
1077	# endif
1078
1079	LogFlow(("pgmRZPoolAccessPfHandlerSimple: returns %Rrc\n", VBOXSTRICTRC_VAL(rc)));
1080	return VBOXSTRICTRC_VAL(rc);
1081	}
1082
1083
1084	/**
1085	* @callback_method_impl{FNPGMRZPHYSPFHANDLER,
1086	* \#PF access handler callback for page table pages.}
1087	*
1088	* @remarks The @a pvUser argument points to the PGMPOOLPAGE.
1089	*/
1090	DECLEXPORT(VBOXSTRICTRC) pgmRZPoolAccessPfHandler(PVM pVM, PVMCPU pVCpu, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame,
1091	RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
1092	{
1093	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorRZ, a);
1094	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1095	PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)pvUser;
1096	unsigned cMaxModifications;
1097	bool fForcedFlush = false;
1098	NOREF(uErrorCode);
1099
1100	LogFlow(("pgmRZPoolAccessPfHandler: pvFault=%RGv pPage=%p:{.idx=%d} GCPhysFault=%RGp\n", pvFault, pPage, pPage->idx, GCPhysFault));
1101
1102	pgmLock(pVM);
1103	if (PHYS_PAGE_ADDRESS(GCPhysFault) != PHYS_PAGE_ADDRESS(pPage->GCPhys))
1104	{
1105	/* Pool page changed while we were waiting for the lock; ignore. */
1106	Log(("CPU%d: pgmRZPoolAccessPfHandler pgm pool page for %RGp changed (to %RGp) while waiting!\n", pVCpu->idCpu, PHYS_PAGE_ADDRESS(GCPhysFault), PHYS_PAGE_ADDRESS(pPage->GCPhys)));
1107	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZHandled, a);
1108	pgmUnlock(pVM);
1109	return VINF_SUCCESS;
1110	}
1111	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1112	if (pPage->fDirty)
1113	{
1114	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH));
1115	pgmUnlock(pVM);
1116	return VINF_SUCCESS; /* SMP guest case where we were blocking on the pgm lock while the same page was being marked dirty. */
1117	}
1118	# endif
1119
1120	# if 0 /* test code defined(VBOX_STRICT) && defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) */
1121	if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
1122	{
1123	void *pvShw = PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
1124	void *pvGst;
1125	int rc = PGM_GCPHYS_2_PTR(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
1126	pgmPoolTrackCheckPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst);
1127	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
1128	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
1129	}
1130	# endif
1131
1132	/*
1133	* Disassemble the faulting instruction.
1134	*/
1135	PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
1136	int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
1137	if (RT_UNLIKELY(rc != VINF_SUCCESS))
1138	{
1139	AssertMsg(rc == VERR_PAGE_NOT_PRESENT \|\| rc == VERR_PAGE_TABLE_NOT_PRESENT, ("Unexpected rc %d\n", rc));
1140	pgmUnlock(pVM);
1141	return rc;
1142	}
1143
1144	Assert(pPage->enmKind != PGMPOOLKIND_FREE);
1145
1146	/*
1147	* We should ALWAYS have the list head as user parameter. This
1148	* is because we use that page to record the changes.
1149	*/
1150	Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1151
1152	# ifdef IN_RING0
1153	/* Maximum nr of modifications depends on the page type. */
1154	if ( pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT
1155	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
1156	cMaxModifications = 4;
1157	else
1158	cMaxModifications = 24;
1159	# else
1160	cMaxModifications = 48;
1161	# endif
1162
1163	/*
1164	* Incremental page table updates should weigh more than random ones.
1165	* (Only applies when started from offset 0)
1166	*/
1167	pVCpu->pgm.s.cPoolAccessHandler++;
1168	if ( pPage->GCPtrLastAccessHandlerRip >= pRegFrame->rip - 0x40 /* observed loops in Windows 7 x64 */
1169	&& pPage->GCPtrLastAccessHandlerRip < pRegFrame->rip + 0x40
1170	&& pvFault == (pPage->GCPtrLastAccessHandlerFault + pDis->Param1.cb)
1171	&& pVCpu->pgm.s.cPoolAccessHandler == pPage->cLastAccessHandler + 1)
1172	{
1173	Log(("Possible page reuse cMods=%d -> %d (locked=%d type=%s)\n", pPage->cModifications, pPage->cModifications * 2, pgmPoolIsPageLocked(pPage), pgmPoolPoolKindToStr(pPage->enmKind)));
1174	Assert(pPage->cModifications < 32000);
1175	pPage->cModifications = pPage->cModifications * 2;
1176	pPage->GCPtrLastAccessHandlerFault = pvFault;
1177	pPage->cLastAccessHandler = pVCpu->pgm.s.cPoolAccessHandler;
1178	if (pPage->cModifications >= cMaxModifications)
1179	{
1180	STAM_COUNTER_INC(&pPool->StatMonitorPfRZFlushReinit);
1181	fForcedFlush = true;
1182	}
1183	}
1184
1185	if (pPage->cModifications >= cMaxModifications)
1186	Log(("Mod overflow %RGv cMods=%d (locked=%d type=%s)\n", pvFault, pPage->cModifications, pgmPoolIsPageLocked(pPage), pgmPoolPoolKindToStr(pPage->enmKind)));
1187
1188	/*
1189	* Check if it's worth dealing with.
1190	*/
1191	bool fReused = false;
1192	bool fNotReusedNotForking = false;
1193	if ( ( pPage->cModifications < cMaxModifications /** @todo \#define / /* @todo need to check that it's not mapping EIP. / /* @todo adjust this! */
1194	\|\| pgmPoolIsPageLocked(pPage)
1195	)
1196	&& !(fReused = pgmRZPoolMonitorIsReused(pVM, pVCpu, pRegFrame, pDis, pvFault, pPage))
1197	&& !pgmRZPoolMonitorIsForking(pPool, pDis, GCPhysFault & PAGE_OFFSET_MASK))
1198	{
1199	/*
1200	* Simple instructions, no REP prefix.
1201	*/
1202	if (!(pDis->fPrefix & (DISPREFIX_REP \| DISPREFIX_REPNE)))
1203	{
1204	rc = pgmRZPoolAccessPfHandlerSimple(pVM, pVCpu, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault, &fReused);
1205	if (fReused)
1206	goto flushPage;
1207
1208	/* A mov instruction to change the first page table entry will be remembered so we can detect
1209	* full page table changes early on. This will reduce the amount of unnecessary traps we'll take.
1210	*/
1211	if ( rc == VINF_SUCCESS
1212	&& !pPage->cLocked /* only applies to unlocked pages as we can't free locked ones (e.g. cr3 root). */
1213	&& pDis->pCurInstr->uOpcode == OP_MOV
1214	&& (pvFault & PAGE_OFFSET_MASK) == 0)
1215	{
1216	pPage->GCPtrLastAccessHandlerFault = pvFault;
1217	pPage->cLastAccessHandler = pVCpu->pgm.s.cPoolAccessHandler;
1218	pPage->GCPtrLastAccessHandlerRip = pRegFrame->rip;
1219	/* Make sure we don't kick out a page too quickly. */
1220	if (pPage->cModifications > 8)
1221	pPage->cModifications = 2;
1222	}
1223	else if (pPage->GCPtrLastAccessHandlerFault == pvFault)
1224	{
1225	/* ignore the 2nd write to this page table entry. */
1226	pPage->cLastAccessHandler = pVCpu->pgm.s.cPoolAccessHandler;
1227	}
1228	else
1229	{
1230	pPage->GCPtrLastAccessHandlerFault = NIL_RTGCPTR;
1231	pPage->GCPtrLastAccessHandlerRip = 0;
1232	}
1233
1234	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZHandled, a);
1235	pgmUnlock(pVM);
1236	return rc;
1237	}
1238
1239	/*
1240	* Windows is frequently doing small memset() operations (netio test 4k+).
1241	* We have to deal with these or we'll kill the cache and performance.
1242	*/
1243	if ( pDis->pCurInstr->uOpcode == OP_STOSWD
1244	&& !pRegFrame->eflags.Bits.u1DF
1245	&& pDis->uOpMode == pDis->uCpuMode
1246	&& pDis->uAddrMode == pDis->uCpuMode)
1247	{
1248	bool fValidStosd = false;
1249
1250	if ( pDis->uCpuMode == DISCPUMODE_32BIT
1251	&& pDis->fPrefix == DISPREFIX_REP
1252	&& pRegFrame->ecx <= 0x20
1253	&& pRegFrame->ecx * 4 <= PAGE_SIZE - ((uintptr_t)pvFault & PAGE_OFFSET_MASK)
1254	&& !((uintptr_t)pvFault & 3)
1255	&& (pRegFrame->eax == 0 \|\| pRegFrame->eax == 0x80) /* the two values observed. */
1256	)
1257	{
1258	fValidStosd = true;
1259	pRegFrame->rcx &= 0xffffffff; /* paranoia */
1260	}
1261	else
1262	if ( pDis->uCpuMode == DISCPUMODE_64BIT
1263	&& pDis->fPrefix == (DISPREFIX_REP \| DISPREFIX_REX)
1264	&& pRegFrame->rcx <= 0x20
1265	&& pRegFrame->rcx * 8 <= PAGE_SIZE - ((uintptr_t)pvFault & PAGE_OFFSET_MASK)
1266	&& !((uintptr_t)pvFault & 7)
1267	&& (pRegFrame->rax == 0 \|\| pRegFrame->rax == 0x80) /* the two values observed. */
1268	)
1269	{
1270	fValidStosd = true;
1271	}
1272
1273	if (fValidStosd)
1274	{
1275	rc = pgmRZPoolAccessPfHandlerSTOSD(pVM, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault);
1276	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZRepStosd, a);
1277	pgmUnlock(pVM);
1278	return rc;
1279	}
1280	}
1281
1282	/* REP prefix, don't bother. */
1283	STAM_COUNTER_INC(&pPool->StatMonitorPfRZRepPrefix);
1284	Log4(("pgmRZPoolAccessPfHandler: eax=%#x ecx=%#x edi=%#x esi=%#x rip=%RGv opcode=%d prefix=%#x\n",
1285	pRegFrame->eax, pRegFrame->ecx, pRegFrame->edi, pRegFrame->esi, (RTGCPTR)pRegFrame->rip, pDis->pCurInstr->uOpcode, pDis->fPrefix));
1286	fNotReusedNotForking = true;
1287	}
1288
1289	# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) && defined(IN_RING0)
1290	/* E.g. Windows 7 x64 initializes page tables and touches some pages in the table during the process. This
1291	* leads to pgm pool trashing and an excessive amount of write faults due to page monitoring.
1292	*/
1293	if ( pPage->cModifications >= cMaxModifications
1294	&& !fForcedFlush
1295	&& (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT \|\| pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
1296	&& ( fNotReusedNotForking
1297	\|\| ( !pgmRZPoolMonitorIsReused(pVM, pVCpu, pRegFrame, pDis, pvFault, pPage)
1298	&& !pgmRZPoolMonitorIsForking(pPool, pDis, GCPhysFault & PAGE_OFFSET_MASK))
1299	)
1300	)
1301	{
1302	Assert(!pgmPoolIsPageLocked(pPage));
1303	Assert(pPage->fDirty == false);
1304
1305	/* Flush any monitored duplicates as we will disable write protection. */
1306	if ( pPage->iMonitoredNext != NIL_PGMPOOL_IDX
1307	\|\| pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
1308	{
1309	PPGMPOOLPAGE pPageHead = pPage;
1310
1311	/* Find the monitor head. */
1312	while (pPageHead->iMonitoredPrev != NIL_PGMPOOL_IDX)
1313	pPageHead = &pPool->aPages[pPageHead->iMonitoredPrev];
1314
1315	while (pPageHead)
1316	{
1317	unsigned idxNext = pPageHead->iMonitoredNext;
1318
1319	if (pPageHead != pPage)
1320	{
1321	STAM_COUNTER_INC(&pPool->StatDirtyPageDupFlush);
1322	Log(("Flush duplicate page idx=%d GCPhys=%RGp type=%s\n", pPageHead->idx, pPageHead->GCPhys, pgmPoolPoolKindToStr(pPageHead->enmKind)));
1323	int rc2 = pgmPoolFlushPage(pPool, pPageHead);
1324	AssertRC(rc2);
1325	}
1326
1327	if (idxNext == NIL_PGMPOOL_IDX)
1328	break;
1329
1330	pPageHead = &pPool->aPages[idxNext];
1331	}
1332	}
1333
1334	/* The flushing above might fail for locked pages, so double check. */
1335	if ( pPage->iMonitoredNext == NIL_PGMPOOL_IDX
1336	&& pPage->iMonitoredPrev == NIL_PGMPOOL_IDX)
1337	{
1338	pgmPoolAddDirtyPage(pVM, pPool, pPage);
1339
1340	/* Temporarily allow write access to the page table again. */
1341	rc = PGMHandlerPhysicalPageTempOff(pVM, pPage->GCPhys & PAGE_BASE_GC_MASK, pPage->GCPhys & PAGE_BASE_GC_MASK);
1342	if (rc == VINF_SUCCESS)
1343	{
1344	rc = PGMShwMakePageWritable(pVCpu, pvFault, PGM_MK_PG_IS_WRITE_FAULT);
1345	AssertMsg(rc == VINF_SUCCESS
1346	/* In the SMP case the page table might be removed while we wait for the PGM lock in the trap handler. */
1347	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
1348	\|\| rc == VERR_PAGE_NOT_PRESENT,
1349	("PGMShwModifyPage -> GCPtr=%RGv rc=%d\n", pvFault, rc));
1350	# ifdef VBOX_STRICT
1351	pPage->GCPtrDirtyFault = pvFault;
1352	# endif
1353
1354	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, a);
1355	pgmUnlock(pVM);
1356	return rc;
1357	}
1358	}
1359	}
1360	# endif /* PGMPOOL_WITH_OPTIMIZED_DIRTY_PT */
1361
1362	STAM_COUNTER_INC(&pPool->StatMonitorPfRZFlushModOverflow);
1363	flushPage:
1364	/*
1365	* Not worth it, so flush it.
1366	*
1367	* If we considered it to be reused, don't go back to ring-3
1368	* to emulate failed instructions since we usually cannot
1369	* interpret then. This may be a bit risky, in which case
1370	* the reuse detection must be fixed.
1371	*/
1372	rc = pgmRZPoolAccessPfHandlerFlush(pVM, pVCpu, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault);
1373	if ( rc == VINF_EM_RAW_EMULATE_INSTR
1374	&& fReused)
1375	{
1376	/* Make sure that the current instruction still has shadow page backing, otherwise we'll end up in a loop. */
1377	if (PGMShwGetPage(pVCpu, pRegFrame->rip, NULL, NULL) == VINF_SUCCESS)
1378	rc = VINF_SUCCESS; /* safe to restart the instruction. */
1379	}
1380	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZFlushPage, a);
1381	pgmUnlock(pVM);
1382	return rc;
1383	}
1384
1385	#endif /* !IN_RING3 */
1386
1387	/**
1388	* @callback_method_impl{FNPGMPHYSHANDLER,
1389	* Access handler for shadowed page table pages.}
1390	*
1391	* @remarks Only uses the VINF_PGM_HANDLER_DO_DEFAULT status.
1392	*/
1393	PGM_ALL_CB2_DECL(VBOXSTRICTRC)
1394	pgmPoolAccessHandler(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf,
1395	PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, void *pvUser)
1396	{
1397	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1398	STAM_PROFILE_START(&pPool->CTX_SUFF_Z(StatMonitor), a);
1399	PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)pvUser;
1400	LogFlow(("PGM_ALL_CB_DECL: GCPhys=%RGp %p:{.Core=%RHp, .idx=%d, .GCPhys=%RGp, .enmType=%d}\n",
1401	GCPhys, pPage, pPage->Core.Key, pPage->idx, pPage->GCPhys, pPage->enmKind));
1402
1403	NOREF(pvPhys); NOREF(pvBuf); NOREF(enmAccessType);
1404
1405	pgmLock(pVM);
1406
1407	#ifdef VBOX_WITH_STATISTICS
1408	/*
1409	* Collect stats on the access.
1410	*/
1411	AssertCompile(RT_ELEMENTS(pPool->CTX_MID_Z(aStatMonitor,Sizes)) == 19);
1412	if (cbBuf <= 16 && cbBuf > 0)
1413	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[cbBuf - 1]);
1414	else if (cbBuf >= 17 && cbBuf < 32)
1415	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[16]);
1416	else if (cbBuf >= 32 && cbBuf < 64)
1417	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[17]);
1418	else if (cbBuf >= 64)
1419	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[18]);
1420
1421	uint8_t cbAlign;
1422	switch (pPage->enmKind)
1423	{
1424	default:
1425	cbAlign = 7;
1426	break;
1427	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1428	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1429	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1430	case PGMPOOLKIND_32BIT_PD:
1431	case PGMPOOLKIND_32BIT_PD_PHYS:
1432	cbAlign = 3;
1433	break;
1434	}
1435	AssertCompile(RT_ELEMENTS(pPool->CTX_MID_Z(aStatMonitor,Misaligned)) == 7);
1436	if ((uint8_t)GCPhys & cbAlign)
1437	STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Misaligned)[((uint8_t)GCPhys & cbAlign) - 1]);
1438	#endif
1439
1440	/*
1441	* Make sure the pool page wasn't modified by a different CPU.
1442	*/
1443	if (PHYS_PAGE_ADDRESS(GCPhys) == PHYS_PAGE_ADDRESS(pPage->GCPhys))
1444	{
1445	Assert(pPage->enmKind != PGMPOOLKIND_FREE);
1446
1447	/* The max modification count before flushing depends on the context and page type. */
1448	#ifdef IN_RING3
1449	uint16_t const cMaxModifications = 96; /* it's cheaper here, right? */
1450	#else
1451	uint16_t cMaxModifications;
1452	if ( pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT
1453	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
1454	cMaxModifications = 4;
1455	else
1456	cMaxModifications = 24;
1457	# ifdef IN_RC
1458	cMaxModifications = 2; / traps are cheaper than exists. */
1459	# endif
1460	#endif
1461
1462	/*
1463	* We don't have to be very sophisticated about this since there are relativly few calls here.
1464	* However, we must try our best to detect any non-cpu accesses (disk / networking).
1465	*/
1466	if ( ( pPage->cModifications < cMaxModifications
1467	\|\| pgmPoolIsPageLocked(pPage) )
1468	&& enmOrigin != PGMACCESSORIGIN_DEVICE
1469	&& cbBuf <= 16)
1470	{
1471	/* Clear the shadow entry. */
1472	if (!pPage->cModifications++)
1473	pgmPoolMonitorModifiedInsert(pPool, pPage);
1474
1475	if (cbBuf <= 8)
1476	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvBuf, (uint32_t)cbBuf);
1477	else
1478	{
1479	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvBuf, 8);
1480	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys + 8, (uint8_t *)pvBuf + 8, (uint32_t)cbBuf - 8);
1481	}
1482	}
1483	else
1484	pgmPoolMonitorChainFlush(pPool, pPage);
1485
1486	STAM_PROFILE_STOP_EX(&pPool->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,FlushPage), a);
1487	}
1488	else
1489	Log(("CPU%d: PGM_ALL_CB_DECL pgm pool page for %RGp changed (to %RGp) while waiting!\n", pVCpu->idCpu, PHYS_PAGE_ADDRESS(GCPhys), PHYS_PAGE_ADDRESS(pPage->GCPhys)));
1490	pgmUnlock(pVM);
1491	return VINF_PGM_HANDLER_DO_DEFAULT;
1492	}
1493
1494
1495	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1496
1497	# if defined(VBOX_STRICT) && !defined(IN_RING3)
1498
1499	/**
1500	* Check references to guest physical memory in a PAE / PAE page table.
1501	*
1502	* @param pPool The pool.
1503	* @param pPage The page.
1504	* @param pShwPT The shadow page table (mapping of the page).
1505	* @param pGstPT The guest page table.
1506	*/
1507	static void pgmPoolTrackCheckPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT)
1508	{
1509	unsigned cErrors = 0;
1510	int LastRc = -1; /* initialized to shut up gcc */
1511	unsigned LastPTE = ~0U; /* initialized to shut up gcc */
1512	RTHCPHYS LastHCPhys = NIL_RTHCPHYS; /* initialized to shut up gcc */
1513	PVM pVM = pPool->CTX_SUFF(pVM);
1514
1515	# ifdef VBOX_STRICT
1516	for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
1517	AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), pPage->iFirstPresent));
1518	# endif
1519	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
1520	{
1521	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
1522	{
1523	RTHCPHYS HCPhys = NIL_RTHCPHYS;
1524	int rc = PGMPhysGCPhys2HCPhys(pVM, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK, &HCPhys);
1525	if ( rc != VINF_SUCCESS
1526	\|\| PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) != HCPhys)
1527	{
1528	Log(("rc=%d idx=%d guest %RX64 shw=%RX64 vs %RHp\n", rc, i, pGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
1529	LastPTE = i;
1530	LastRc = rc;
1531	LastHCPhys = HCPhys;
1532	cErrors++;
1533
1534	RTHCPHYS HCPhysPT = NIL_RTHCPHYS;
1535	rc = PGMPhysGCPhys2HCPhys(pVM, pPage->GCPhys, &HCPhysPT);
1536	AssertRC(rc);
1537
1538	for (unsigned iPage = 0; iPage < pPool->cCurPages; iPage++)
1539	{
1540	PPGMPOOLPAGE pTempPage = &pPool->aPages[iPage];
1541
1542	if (pTempPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
1543	{
1544	PPGMSHWPTPAE pShwPT2 = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pTempPage);
1545
1546	for (unsigned j = 0; j < RT_ELEMENTS(pShwPT->a); j++)
1547	{
1548	if ( PGMSHWPTEPAE_IS_P_RW(pShwPT2->a[j])
1549	&& PGMSHWPTEPAE_GET_HCPHYS(pShwPT2->a[j]) == HCPhysPT)
1550	{
1551	Log(("GCPhys=%RGp idx=%d %RX64 vs %RX64\n", pTempPage->GCPhys, j, PGMSHWPTEPAE_GET_LOG(pShwPT->a[j]), PGMSHWPTEPAE_GET_LOG(pShwPT2->a[j])));
1552	}
1553	}
1554
1555	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pShwPT2);
1556	}
1557	}
1558	}
1559	}
1560	}
1561	AssertMsg(!cErrors, ("cErrors=%d: last rc=%d idx=%d guest %RX64 shw=%RX64 vs %RHp\n", cErrors, LastRc, LastPTE, pGstPT->a[LastPTE].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[LastPTE]), LastHCPhys));
1562	}
1563
1564
1565	/**
1566	* Check references to guest physical memory in a PAE / 32-bit page table.
1567	*
1568	* @param pPool The pool.
1569	* @param pPage The page.
1570	* @param pShwPT The shadow page table (mapping of the page).
1571	* @param pGstPT The guest page table.
1572	*/
1573	static void pgmPoolTrackCheckPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PT pGstPT)
1574	{
1575	unsigned cErrors = 0;
1576	int LastRc = -1; /* initialized to shut up gcc */
1577	unsigned LastPTE = ~0U; /* initialized to shut up gcc */
1578	RTHCPHYS LastHCPhys = NIL_RTHCPHYS; /* initialized to shut up gcc */
1579	PVM pVM = pPool->CTX_SUFF(pVM);
1580
1581	# ifdef VBOX_STRICT
1582	for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
1583	AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), pPage->iFirstPresent));
1584	# endif
1585	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
1586	{
1587	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
1588	{
1589	RTHCPHYS HCPhys = NIL_RTHCPHYS;
1590	int rc = PGMPhysGCPhys2HCPhys(pVM, pGstPT->a[i].u & X86_PTE_PG_MASK, &HCPhys);
1591	if ( rc != VINF_SUCCESS
1592	\|\| PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) != HCPhys)
1593	{
1594	Log(("rc=%d idx=%d guest %x shw=%RX64 vs %RHp\n", rc, i, pGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
1595	LastPTE = i;
1596	LastRc = rc;
1597	LastHCPhys = HCPhys;
1598	cErrors++;
1599
1600	RTHCPHYS HCPhysPT = NIL_RTHCPHYS;
1601	rc = PGMPhysGCPhys2HCPhys(pVM, pPage->GCPhys, &HCPhysPT);
1602	AssertRC(rc);
1603
1604	for (unsigned iPage = 0; iPage < pPool->cCurPages; iPage++)
1605	{
1606	PPGMPOOLPAGE pTempPage = &pPool->aPages[iPage];
1607
1608	if (pTempPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
1609	{
1610	PPGMSHWPTPAE pShwPT2 = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pTempPage);
1611
1612	for (unsigned j = 0; j < RT_ELEMENTS(pShwPT->a); j++)
1613	{
1614	if ( PGMSHWPTEPAE_IS_P_RW(pShwPT2->a[j])
1615	&& PGMSHWPTEPAE_GET_HCPHYS(pShwPT2->a[j]) == HCPhysPT)
1616	{
1617	Log(("GCPhys=%RGp idx=%d %RX64 vs %RX64\n", pTempPage->GCPhys, j, PGMSHWPTEPAE_GET_LOG(pShwPT->a[j]), PGMSHWPTEPAE_GET_LOG(pShwPT2->a[j])));
1618	}
1619	}
1620
1621	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pShwPT2);
1622	}
1623	}
1624	}
1625	}
1626	}
1627	AssertMsg(!cErrors, ("cErrors=%d: last rc=%d idx=%d guest %x shw=%RX64 vs %RHp\n", cErrors, LastRc, LastPTE, pGstPT->a[LastPTE].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[LastPTE]), LastHCPhys));
1628	}
1629
1630	# endif /* VBOX_STRICT && !IN_RING3 */
1631
1632	/**
1633	* Clear references to guest physical memory in a PAE / PAE page table.
1634	*
1635	* @returns nr of changed PTEs
1636	* @param pPool The pool.
1637	* @param pPage The page.
1638	* @param pShwPT The shadow page table (mapping of the page).
1639	* @param pGstPT The guest page table.
1640	* @param pOldGstPT The old cached guest page table.
1641	* @param fAllowRemoval Bail out as soon as we encounter an invalid PTE
1642	* @param pfFlush Flush reused page table (out)
1643	*/
1644	DECLINLINE(unsigned) pgmPoolTrackFlushPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT,
1645	PCX86PTPAE pOldGstPT, bool fAllowRemoval, bool *pfFlush)
1646	{
1647	unsigned cChanged = 0;
1648
1649	# ifdef VBOX_STRICT
1650	for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
1651	AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), pPage->iFirstPresent));
1652	# endif
1653	*pfFlush = false;
1654
1655	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
1656	{
1657	/* Check the new value written by the guest. If present and with a bogus physical address, then
1658	* it's fairly safe to assume the guest is reusing the PT.
1659	*/
1660	if ( fAllowRemoval
1661	&& pGstPT->a[i].n.u1Present)
1662	{
1663	if (!PGMPhysIsGCPhysValid(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK))
1664	{
1665	*pfFlush = true;
1666	return ++cChanged;
1667	}
1668	}
1669	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
1670	{
1671	/* If the old cached PTE is identical, then there's no need to flush the shadow copy. */
1672	if ((pGstPT->a[i].u & X86_PTE_PAE_PG_MASK) == (pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK))
1673	{
1674	# ifdef VBOX_STRICT
1675	RTHCPHYS HCPhys = NIL_RTGCPHYS;
1676	int rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK, &HCPhys);
1677	AssertMsg(rc == VINF_SUCCESS && PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) == HCPhys, ("rc=%d guest %RX64 old %RX64 shw=%RX64 vs %RHp\n", rc, pGstPT->a[i].u, pOldGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
1678	# endif
1679	uint64_t uHostAttr = PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & (X86_PTE_P \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D \| X86_PTE_G \| X86_PTE_PAE_NX);
1680	bool fHostRW = !!(PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & X86_PTE_RW);
1681	uint64_t uGuestAttr = pGstPT->a[i].u & (X86_PTE_P \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D \| X86_PTE_G \| X86_PTE_PAE_NX);
1682	bool fGuestRW = !!(pGstPT->a[i].u & X86_PTE_RW);
1683
1684	if ( uHostAttr == uGuestAttr
1685	&& fHostRW <= fGuestRW)
1686	continue;
1687	}
1688	cChanged++;
1689	/* Something was changed, so flush it. */
1690	Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX64 hint=%RX64\n",
1691	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK));
1692	pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK, i);
1693	PGMSHWPTEPAE_ATOMIC_SET(pShwPT->a[i], 0);
1694	}
1695	}
1696	return cChanged;
1697	}
1698
1699
1700	/**
1701	* Clear references to guest physical memory in a PAE / PAE page table.
1702	*
1703	* @returns nr of changed PTEs
1704	* @param pPool The pool.
1705	* @param pPage The page.
1706	* @param pShwPT The shadow page table (mapping of the page).
1707	* @param pGstPT The guest page table.
1708	* @param pOldGstPT The old cached guest page table.
1709	* @param fAllowRemoval Bail out as soon as we encounter an invalid PTE
1710	* @param pfFlush Flush reused page table (out)
1711	*/
1712	DECLINLINE(unsigned) pgmPoolTrackFlushPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PT pGstPT,
1713	PCX86PT pOldGstPT, bool fAllowRemoval, bool *pfFlush)
1714	{
1715	unsigned cChanged = 0;
1716
1717	# ifdef VBOX_STRICT
1718	for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
1719	AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), pPage->iFirstPresent));
1720	# endif
1721	*pfFlush = false;
1722
1723	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
1724	{
1725	/* Check the new value written by the guest. If present and with a bogus physical address, then
1726	* it's fairly safe to assume the guest is reusing the PT.
1727	*/
1728	if ( fAllowRemoval
1729	&& pGstPT->a[i].n.u1Present)
1730	{
1731	if (!PGMPhysIsGCPhysValid(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PG_MASK))
1732	{
1733	*pfFlush = true;
1734	return ++cChanged;
1735	}
1736	}
1737	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
1738	{
1739	/* If the old cached PTE is identical, then there's no need to flush the shadow copy. */
1740	if ((pGstPT->a[i].u & X86_PTE_PG_MASK) == (pOldGstPT->a[i].u & X86_PTE_PG_MASK))
1741	{
1742	# ifdef VBOX_STRICT
1743	RTHCPHYS HCPhys = NIL_RTGCPHYS;
1744	int rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PG_MASK, &HCPhys);
1745	AssertMsg(rc == VINF_SUCCESS && PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) == HCPhys, ("rc=%d guest %x old %x shw=%RX64 vs %RHp\n", rc, pGstPT->a[i].u, pOldGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
1746	# endif
1747	uint64_t uHostAttr = PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & (X86_PTE_P \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D \| X86_PTE_G);
1748	bool fHostRW = !!(PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & X86_PTE_RW);
1749	uint64_t uGuestAttr = pGstPT->a[i].u & (X86_PTE_P \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D \| X86_PTE_G);
1750	bool fGuestRW = !!(pGstPT->a[i].u & X86_PTE_RW);
1751
1752	if ( uHostAttr == uGuestAttr
1753	&& fHostRW <= fGuestRW)
1754	continue;
1755	}
1756	cChanged++;
1757	/* Something was changed, so flush it. */
1758	Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX64 hint=%x\n",
1759	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PG_MASK));
1760	pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PG_MASK, i);
1761	PGMSHWPTEPAE_ATOMIC_SET(pShwPT->a[i], 0);
1762	}
1763	}
1764	return cChanged;
1765	}
1766
1767
1768	/**
1769	* Flush a dirty page
1770	*
1771	* @param pVM The cross context VM structure.
1772	* @param pPool The pool.
1773	* @param idxSlot Dirty array slot index
1774	* @param fAllowRemoval Allow a reused page table to be removed
1775	*/
1776	static void pgmPoolFlushDirtyPage(PVM pVM, PPGMPOOL pPool, unsigned idxSlot, bool fAllowRemoval = false)
1777	{
1778	AssertCompile(RT_ELEMENTS(pPool->aidxDirtyPages) == RT_ELEMENTS(pPool->aDirtyPages));
1779
1780	Assert(idxSlot < RT_ELEMENTS(pPool->aDirtyPages));
1781	unsigned idxPage = pPool->aidxDirtyPages[idxSlot];
1782	if (idxPage == NIL_PGMPOOL_IDX)
1783	return;
1784
1785	PPGMPOOLPAGE pPage = &pPool->aPages[idxPage];
1786	Assert(pPage->idx == idxPage);
1787	Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1788
1789	AssertMsg(pPage->fDirty, ("Page %RGp (slot=%d) not marked dirty!", pPage->GCPhys, idxSlot));
1790	Log(("Flush dirty page %RGp cMods=%d\n", pPage->GCPhys, pPage->cModifications));
1791
1792	# if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) \|\| defined(IN_RC)
1793	PVMCPU pVCpu = VMMGetCpu(pVM);
1794	uint32_t iPrevSubset = PGMRZDynMapPushAutoSubset(pVCpu);
1795	# endif
1796
1797	/* First write protect the page again to catch all write accesses. (before checking for changes -> SMP) */
1798	int rc = PGMHandlerPhysicalReset(pVM, pPage->GCPhys & PAGE_BASE_GC_MASK);
1799	Assert(rc == VINF_SUCCESS);
1800	pPage->fDirty = false;
1801
1802	# ifdef VBOX_STRICT
1803	uint64_t fFlags = 0;
1804	RTHCPHYS HCPhys;
1805	rc = PGMShwGetPage(VMMGetCpu(pVM), pPage->GCPtrDirtyFault, &fFlags, &HCPhys);
1806	AssertMsg( ( rc == VINF_SUCCESS
1807	&& (!(fFlags & X86_PTE_RW) \|\| HCPhys != pPage->Core.Key))
1808	/* In the SMP case the page table might be removed while we wait for the PGM lock in the trap handler. */
1809	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
1810	\|\| rc == VERR_PAGE_NOT_PRESENT,
1811	("PGMShwGetPage -> GCPtr=%RGv rc=%d flags=%RX64\n", pPage->GCPtrDirtyFault, rc, fFlags));
1812	# endif
1813
1814	/* Flush those PTEs that have changed. */
1815	STAM_PROFILE_START(&pPool->StatTrackDeref,a);
1816	void *pvShw = PGMPOOL_PAGE_2_PTR(pVM, pPage);
1817	void *pvGst;
1818	rc = PGM_GCPHYS_2_PTR_EX(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
1819	bool fFlush;
1820	unsigned cChanges;
1821
1822	if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
1823	cChanges = pgmPoolTrackFlushPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst,
1824	(PCX86PTPAE)&pPool->aDirtyPages[idxSlot].aPage[0], fAllowRemoval, &fFlush);
1825	else
1826	cChanges = pgmPoolTrackFlushPTPae32Bit(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PT)pvGst,
1827	(PCX86PT)&pPool->aDirtyPages[idxSlot].aPage[0], fAllowRemoval, &fFlush);
1828
1829	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
1830	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
1831	STAM_PROFILE_STOP(&pPool->StatTrackDeref,a);
1832	/* Note: we might want to consider keeping the dirty page active in case there were many changes. */
1833
1834	/* This page is likely to be modified again, so reduce the nr of modifications just a bit here. */
1835	Assert(pPage->cModifications);
1836	if (cChanges < 4)
1837	pPage->cModifications = 1; /* must use > 0 here */
1838	else
1839	pPage->cModifications = RT_MAX(1, pPage->cModifications / 2);
1840
1841	STAM_COUNTER_INC(&pPool->StatResetDirtyPages);
1842	if (pPool->cDirtyPages == RT_ELEMENTS(pPool->aDirtyPages))
1843	pPool->idxFreeDirtyPage = idxSlot;
1844
1845	pPool->cDirtyPages--;
1846	pPool->aidxDirtyPages[idxSlot] = NIL_PGMPOOL_IDX;
1847	Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
1848	if (fFlush)
1849	{
1850	Assert(fAllowRemoval);
1851	Log(("Flush reused page table!\n"));
1852	pgmPoolFlushPage(pPool, pPage);
1853	STAM_COUNTER_INC(&pPool->StatForceFlushReused);
1854	}
1855	else
1856	Log(("Removed dirty page %RGp cMods=%d cChanges=%d\n", pPage->GCPhys, pPage->cModifications, cChanges));
1857
1858	# if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) \|\| defined(IN_RC)
1859	PGMRZDynMapPopAutoSubset(pVCpu, iPrevSubset);
1860	# endif
1861	}
1862
1863
1864	# ifndef IN_RING3
1865	/**
1866	* Add a new dirty page
1867	*
1868	* @param pVM The cross context VM structure.
1869	* @param pPool The pool.
1870	* @param pPage The page.
1871	*/
1872	void pgmPoolAddDirtyPage(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1873	{
1874	PGM_LOCK_ASSERT_OWNER(pVM);
1875	AssertCompile(RT_ELEMENTS(pPool->aDirtyPages) == 8 \|\| RT_ELEMENTS(pPool->aDirtyPages) == 16);
1876	Assert(!pPage->fDirty);
1877
1878	unsigned idxFree = pPool->idxFreeDirtyPage;
1879	Assert(idxFree < RT_ELEMENTS(pPool->aDirtyPages));
1880	Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1881
1882	if (pPool->cDirtyPages >= RT_ELEMENTS(pPool->aDirtyPages))
1883	{
1884	STAM_COUNTER_INC(&pPool->StatDirtyPageOverFlowFlush);
1885	pgmPoolFlushDirtyPage(pVM, pPool, idxFree, true /* allow removal of reused page tables*/);
1886	}
1887	Assert(pPool->cDirtyPages < RT_ELEMENTS(pPool->aDirtyPages));
1888	AssertMsg(pPool->aidxDirtyPages[idxFree] == NIL_PGMPOOL_IDX, ("idxFree=%d cDirtyPages=%d\n", idxFree, pPool->cDirtyPages));
1889
1890	Log(("Add dirty page %RGp (slot=%d)\n", pPage->GCPhys, idxFree));
1891
1892	/*
1893	* Make a copy of the guest page table as we require valid GCPhys addresses
1894	* when removing references to physical pages.
1895	* (The HCPhys linear lookup is extremely expensive!)
1896	*/
1897	void *pvGst;
1898	int rc = PGM_GCPHYS_2_PTR_EX(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
1899	memcpy(&pPool->aDirtyPages[idxFree].aPage[0], pvGst, (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT) ? PAGE_SIZE : PAGE_SIZE/2);
1900	# ifdef VBOX_STRICT
1901	void *pvShw = PGMPOOL_PAGE_2_PTR(pVM, pPage);
1902	if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
1903	pgmPoolTrackCheckPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst);
1904	else
1905	pgmPoolTrackCheckPTPae32Bit(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PT)pvGst);
1906	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
1907	# endif
1908	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
1909
1910	STAM_COUNTER_INC(&pPool->StatDirtyPage);
1911	pPage->fDirty = true;
1912	pPage->idxDirtyEntry = (uint8_t)idxFree; Assert(pPage->idxDirtyEntry == idxFree);
1913	pPool->aidxDirtyPages[idxFree] = pPage->idx;
1914	pPool->cDirtyPages++;
1915
1916	pPool->idxFreeDirtyPage = (pPool->idxFreeDirtyPage + 1) & (RT_ELEMENTS(pPool->aDirtyPages) - 1);
1917	if ( pPool->cDirtyPages < RT_ELEMENTS(pPool->aDirtyPages)
1918	&& pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX)
1919	{
1920	unsigned i;
1921	for (i = 1; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
1922	{
1923	idxFree = (pPool->idxFreeDirtyPage + i) & (RT_ELEMENTS(pPool->aDirtyPages) - 1);
1924	if (pPool->aidxDirtyPages[idxFree] == NIL_PGMPOOL_IDX)
1925	{
1926	pPool->idxFreeDirtyPage = idxFree;
1927	break;
1928	}
1929	}
1930	Assert(i != RT_ELEMENTS(pPool->aDirtyPages));
1931	}
1932
1933	Assert(pPool->cDirtyPages == RT_ELEMENTS(pPool->aDirtyPages) \|\| pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] == NIL_PGMPOOL_IDX);
1934
1935	/*
1936	* Clear all references to this shadow table. See @bugref{7298}.
1937	*/
1938	pgmPoolTrackClearPageUsers(pPool, pPage);
1939	}
1940	# endif /* !IN_RING3 */
1941
1942
1943	/**
1944	* Check if the specified page is dirty (not write monitored)
1945	*
1946	* @return dirty or not
1947	* @param pVM The cross context VM structure.
1948	* @param GCPhys Guest physical address
1949	*/
1950	bool pgmPoolIsDirtyPageSlow(PVM pVM, RTGCPHYS GCPhys)
1951	{
1952	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1953	PGM_LOCK_ASSERT_OWNER(pVM);
1954	if (!pPool->cDirtyPages)
1955	return false;
1956
1957	GCPhys = GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
1958
1959	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
1960	{
1961	unsigned idxPage = pPool->aidxDirtyPages[i];
1962	if (idxPage != NIL_PGMPOOL_IDX)
1963	{
1964	PPGMPOOLPAGE pPage = &pPool->aPages[idxPage];
1965	if (pPage->GCPhys == GCPhys)
1966	return true;
1967	}
1968	}
1969	return false;
1970	}
1971
1972
1973	/**
1974	* Reset all dirty pages by reinstating page monitoring.
1975	*
1976	* @param pVM The cross context VM structure.
1977	*/
1978	void pgmPoolResetDirtyPages(PVM pVM)
1979	{
1980	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1981	PGM_LOCK_ASSERT_OWNER(pVM);
1982	Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
1983
1984	if (!pPool->cDirtyPages)
1985	return;
1986
1987	Log(("pgmPoolResetDirtyPages\n"));
1988	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
1989	pgmPoolFlushDirtyPage(pVM, pPool, i, true /* allow removal of reused page tables*/);
1990
1991	pPool->idxFreeDirtyPage = 0;
1992	if ( pPool->cDirtyPages != RT_ELEMENTS(pPool->aDirtyPages)
1993	&& pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX)
1994	{
1995	unsigned i;
1996	for (i = 1; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
1997	{
1998	if (pPool->aidxDirtyPages[i] == NIL_PGMPOOL_IDX)
1999	{
2000	pPool->idxFreeDirtyPage = i;
2001	break;
2002	}
2003	}
2004	AssertMsg(i != RT_ELEMENTS(pPool->aDirtyPages), ("cDirtyPages %d", pPool->cDirtyPages));
2005	}
2006
2007	Assert(pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] == NIL_PGMPOOL_IDX \|\| pPool->cDirtyPages == RT_ELEMENTS(pPool->aDirtyPages));
2008	return;
2009	}
2010
2011
2012	/**
2013	* Invalidate the PT entry for the specified page
2014	*
2015	* @param pVM The cross context VM structure.
2016	* @param GCPtrPage Guest page to invalidate
2017	*/
2018	void pgmPoolResetDirtyPage(PVM pVM, RTGCPTR GCPtrPage)
2019	{
2020	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2021	PGM_LOCK_ASSERT_OWNER(pVM);
2022	Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
2023
2024	if (!pPool->cDirtyPages)
2025	return;
2026
2027	Log(("pgmPoolResetDirtyPage %RGv\n", GCPtrPage)); RT_NOREF_PV(GCPtrPage);
2028	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
2029	{
2030	/** @todo What was intended here??? This looks incomplete... */
2031	}
2032	}
2033
2034
2035	/**
2036	* Reset all dirty pages by reinstating page monitoring.
2037	*
2038	* @param pVM The cross context VM structure.
2039	* @param GCPhysPT Physical address of the page table
2040	*/
2041	void pgmPoolInvalidateDirtyPage(PVM pVM, RTGCPHYS GCPhysPT)
2042	{
2043	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2044	PGM_LOCK_ASSERT_OWNER(pVM);
2045	Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
2046	unsigned idxDirtyPage = RT_ELEMENTS(pPool->aDirtyPages);
2047
2048	if (!pPool->cDirtyPages)
2049	return;
2050
2051	GCPhysPT = GCPhysPT & ~(RTGCPHYS)PAGE_OFFSET_MASK;
2052
2053	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
2054	{
2055	unsigned idxPage = pPool->aidxDirtyPages[i];
2056	if (idxPage != NIL_PGMPOOL_IDX)
2057	{
2058	PPGMPOOLPAGE pPage = &pPool->aPages[idxPage];
2059	if (pPage->GCPhys == GCPhysPT)
2060	{
2061	idxDirtyPage = i;
2062	break;
2063	}
2064	}
2065	}
2066
2067	if (idxDirtyPage != RT_ELEMENTS(pPool->aDirtyPages))
2068	{
2069	pgmPoolFlushDirtyPage(pVM, pPool, idxDirtyPage, true /* allow removal of reused page tables*/);
2070	if ( pPool->cDirtyPages != RT_ELEMENTS(pPool->aDirtyPages)
2071	&& pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX)
2072	{
2073	unsigned i;
2074	for (i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
2075	{
2076	if (pPool->aidxDirtyPages[i] == NIL_PGMPOOL_IDX)
2077	{
2078	pPool->idxFreeDirtyPage = i;
2079	break;
2080	}
2081	}
2082	AssertMsg(i != RT_ELEMENTS(pPool->aDirtyPages), ("cDirtyPages %d", pPool->cDirtyPages));
2083	}
2084	}
2085	}
2086
2087	#endif /* PGMPOOL_WITH_OPTIMIZED_DIRTY_PT */
2088
2089	/**
2090	* Inserts a page into the GCPhys hash table.
2091	*
2092	* @param pPool The pool.
2093	* @param pPage The page.
2094	*/
2095	DECLINLINE(void) pgmPoolHashInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2096	{
2097	Log3(("pgmPoolHashInsert: %RGp\n", pPage->GCPhys));
2098	Assert(pPage->GCPhys != NIL_RTGCPHYS); Assert(pPage->iNext == NIL_PGMPOOL_IDX);
2099	uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
2100	pPage->iNext = pPool->aiHash[iHash];
2101	pPool->aiHash[iHash] = pPage->idx;
2102	}
2103
2104
2105	/**
2106	* Removes a page from the GCPhys hash table.
2107	*
2108	* @param pPool The pool.
2109	* @param pPage The page.
2110	*/
2111	DECLINLINE(void) pgmPoolHashRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2112	{
2113	Log3(("pgmPoolHashRemove: %RGp\n", pPage->GCPhys));
2114	uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
2115	if (pPool->aiHash[iHash] == pPage->idx)
2116	pPool->aiHash[iHash] = pPage->iNext;
2117	else
2118	{
2119	uint16_t iPrev = pPool->aiHash[iHash];
2120	for (;;)
2121	{
2122	const int16_t i = pPool->aPages[iPrev].iNext;
2123	if (i == pPage->idx)
2124	{
2125	pPool->aPages[iPrev].iNext = pPage->iNext;
2126	break;
2127	}
2128	if (i == NIL_PGMPOOL_IDX)
2129	{
2130	AssertReleaseMsgFailed(("GCPhys=%RGp idx=%d\n", pPage->GCPhys, pPage->idx));
2131	break;
2132	}
2133	iPrev = i;
2134	}
2135	}
2136	pPage->iNext = NIL_PGMPOOL_IDX;
2137	}
2138
2139
2140	/**
2141	* Frees up one cache page.
2142	*
2143	* @returns VBox status code.
2144	* @retval VINF_SUCCESS on success.
2145	* @param pPool The pool.
2146	* @param iUser The user index.
2147	*/
2148	static int pgmPoolCacheFreeOne(PPGMPOOL pPool, uint16_t iUser)
2149	{
2150	#ifndef IN_RC
2151	const PVM pVM = pPool->CTX_SUFF(pVM);
2152	#endif
2153	Assert(pPool->iAgeHead != pPool->iAgeTail); /* We shouldn't be here if there < 2 cached entries! */
2154	STAM_COUNTER_INC(&pPool->StatCacheFreeUpOne);
2155
2156	/*
2157	* Select one page from the tail of the age list.
2158	*/
2159	PPGMPOOLPAGE pPage;
2160	for (unsigned iLoop = 0; ; iLoop++)
2161	{
2162	uint16_t iToFree = pPool->iAgeTail;
2163	if (iToFree == iUser && iUser != NIL_PGMPOOL_IDX)
2164	iToFree = pPool->aPages[iToFree].iAgePrev;
2165	/* This is the alternative to the SyncCR3 pgmPoolCacheUsed calls.
2166	if (pPool->aPages[iToFree].iUserHead != NIL_PGMPOOL_USER_INDEX)
2167	{
2168	uint16_t i = pPool->aPages[iToFree].iAgePrev;
2169	for (unsigned j = 0; j < 10 && i != NIL_PGMPOOL_USER_INDEX; j++, i = pPool->aPages[i].iAgePrev)
2170	{
2171	if (pPool->aPages[iToFree].iUserHead == NIL_PGMPOOL_USER_INDEX)
2172	continue;
2173	iToFree = i;
2174	break;
2175	}
2176	}
2177	*/
2178	Assert(iToFree != iUser);
2179	AssertRelease(iToFree != NIL_PGMPOOL_IDX);
2180	pPage = &pPool->aPages[iToFree];
2181
2182	/*
2183	* Reject any attempts at flushing the currently active shadow CR3 mapping.
2184	* Call pgmPoolCacheUsed to move the page to the head of the age list.
2185	*/
2186	if ( !pgmPoolIsPageLocked(pPage)
2187	&& pPage->idx >= PGMPOOL_IDX_FIRST /* paranoia (#6349) */)
2188	break;
2189	LogFlow(("pgmPoolCacheFreeOne: refuse CR3 mapping\n"));
2190	pgmPoolCacheUsed(pPool, pPage);
2191	AssertLogRelReturn(iLoop < 8192, VERR_PGM_POOL_TOO_MANY_LOOPS);
2192	}
2193
2194	/*
2195	* Found a usable page, flush it and return.
2196	*/
2197	int rc = pgmPoolFlushPage(pPool, pPage);
2198	/* This flush was initiated by us and not the guest, so explicitly flush the TLB. */
2199	/** @todo find out why this is necessary; pgmPoolFlushPage should trigger a flush if one is really needed. */
2200	if (rc == VINF_SUCCESS)
2201	PGM_INVL_ALL_VCPU_TLBS(pVM);
2202	return rc;
2203	}
2204
2205
2206	/**
2207	* Checks if a kind mismatch is really a page being reused
2208	* or if it's just normal remappings.
2209	*
2210	* @returns true if reused and the cached page (enmKind1) should be flushed
2211	* @returns false if not reused.
2212	* @param enmKind1 The kind of the cached page.
2213	* @param enmKind2 The kind of the requested page.
2214	*/
2215	static bool pgmPoolCacheReusedByKind(PGMPOOLKIND enmKind1, PGMPOOLKIND enmKind2)
2216	{
2217	switch (enmKind1)
2218	{
2219	/*
2220	* Never reuse them. There is no remapping in non-paging mode.
2221	*/
2222	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2223	case PGMPOOLKIND_32BIT_PD_PHYS:
2224	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2225	case PGMPOOLKIND_PAE_PD_PHYS:
2226	case PGMPOOLKIND_PAE_PDPT_PHYS:
2227	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2228	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2229	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2230	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2231	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2232	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT: /* never reuse them for other types */
2233	return false;
2234
2235	/*
2236	* It's perfectly fine to reuse these, except for PAE and non-paging stuff.
2237	*/
2238	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2239	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2240	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2241	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2242	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2243	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2244	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2245	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2246	case PGMPOOLKIND_32BIT_PD:
2247	case PGMPOOLKIND_PAE_PDPT:
2248	switch (enmKind2)
2249	{
2250	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2251	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2252	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2253	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2254	case PGMPOOLKIND_64BIT_PML4:
2255	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2256	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2257	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2258	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2259	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2260	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2261	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2262	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2263	return true;
2264	default:
2265	return false;
2266	}
2267
2268	/*
2269	* It's perfectly fine to reuse these, except for PAE and non-paging stuff.
2270	*/
2271	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2272	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2273	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2274	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2275	case PGMPOOLKIND_64BIT_PML4:
2276	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2277	switch (enmKind2)
2278	{
2279	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2280	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2281	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2282	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2283	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2284	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2285	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2286	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2287	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2288	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2289	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2290	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2291	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2292	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2293	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2294	return true;
2295	default:
2296	return false;
2297	}
2298
2299	/*
2300	* These cannot be flushed, and it's common to reuse the PDs as PTs.
2301	*/
2302	case PGMPOOLKIND_ROOT_NESTED:
2303	return false;
2304
2305	default:
2306	AssertFatalMsgFailed(("enmKind1=%d\n", enmKind1));
2307	}
2308	}
2309
2310
2311	/**
2312	* Attempts to satisfy a pgmPoolAlloc request from the cache.
2313	*
2314	* @returns VBox status code.
2315	* @retval VINF_PGM_CACHED_PAGE on success.
2316	* @retval VERR_FILE_NOT_FOUND if not found.
2317	* @param pPool The pool.
2318	* @param GCPhys The GC physical address of the page we're gonna shadow.
2319	* @param enmKind The kind of mapping.
2320	* @param enmAccess Access type for the mapping (only relevant for big pages)
2321	* @param fA20Enabled Whether the CPU has the A20 gate enabled.
2322	* @param iUser The shadow page pool index of the user table. This is
2323	* NIL_PGMPOOL_IDX for root pages.
2324	* @param iUserTable The index into the user table (shadowed). Ignored if
2325	* root page
2326	* @param ppPage Where to store the pointer to the page.
2327	*/
2328	static int pgmPoolCacheAlloc(PPGMPOOL pPool, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, bool fA20Enabled,
2329	uint16_t iUser, uint32_t iUserTable, PPPGMPOOLPAGE ppPage)
2330	{
2331	/*
2332	* Look up the GCPhys in the hash.
2333	*/
2334	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
2335	Log3(("pgmPoolCacheAlloc: %RGp kind %s iUser=%d iUserTable=%x SLOT=%d\n", GCPhys, pgmPoolPoolKindToStr(enmKind), iUser, iUserTable, i));
2336	if (i != NIL_PGMPOOL_IDX)
2337	{
2338	do
2339	{
2340	PPGMPOOLPAGE pPage = &pPool->aPages[i];
2341	Log4(("pgmPoolCacheAlloc: slot %d found page %RGp\n", i, pPage->GCPhys));
2342	if (pPage->GCPhys == GCPhys)
2343	{
2344	if ( (PGMPOOLKIND)pPage->enmKind == enmKind
2345	&& (PGMPOOLACCESS)pPage->enmAccess == enmAccess
2346	&& pPage->fA20Enabled == fA20Enabled)
2347	{
2348	/* Put it at the start of the use list to make sure pgmPoolTrackAddUser
2349	* doesn't flush it in case there are no more free use records.
2350	*/
2351	pgmPoolCacheUsed(pPool, pPage);
2352
2353	int rc = VINF_SUCCESS;
2354	if (iUser != NIL_PGMPOOL_IDX)
2355	rc = pgmPoolTrackAddUser(pPool, pPage, iUser, iUserTable);
2356	if (RT_SUCCESS(rc))
2357	{
2358	Assert((PGMPOOLKIND)pPage->enmKind == enmKind);
2359	*ppPage = pPage;
2360	if (pPage->cModifications)
2361	pPage->cModifications = 1; /* reset counter (can't use 0, or else it will be reinserted in the modified list) */
2362	STAM_COUNTER_INC(&pPool->StatCacheHits);
2363	return VINF_PGM_CACHED_PAGE;
2364	}
2365	return rc;
2366	}
2367
2368	if ((PGMPOOLKIND)pPage->enmKind != enmKind)
2369	{
2370	/*
2371	* The kind is different. In some cases we should now flush the page
2372	* as it has been reused, but in most cases this is normal remapping
2373	* of PDs as PT or big pages using the GCPhys field in a slightly
2374	* different way than the other kinds.
2375	*/
2376	if (pgmPoolCacheReusedByKind((PGMPOOLKIND)pPage->enmKind, enmKind))
2377	{
2378	STAM_COUNTER_INC(&pPool->StatCacheKindMismatches);
2379	pgmPoolFlushPage(pPool, pPage);
2380	break;
2381	}
2382	}
2383	}
2384
2385	/* next */
2386	i = pPage->iNext;
2387	} while (i != NIL_PGMPOOL_IDX);
2388	}
2389
2390	Log3(("pgmPoolCacheAlloc: Missed GCPhys=%RGp enmKind=%s\n", GCPhys, pgmPoolPoolKindToStr(enmKind)));
2391	STAM_COUNTER_INC(&pPool->StatCacheMisses);
2392	return VERR_FILE_NOT_FOUND;
2393	}
2394
2395
2396	/**
2397	* Inserts a page into the cache.
2398	*
2399	* @param pPool The pool.
2400	* @param pPage The cached page.
2401	* @param fCanBeCached Set if the page is fit for caching from the caller's point of view.
2402	*/
2403	static void pgmPoolCacheInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fCanBeCached)
2404	{
2405	/*
2406	* Insert into the GCPhys hash if the page is fit for that.
2407	*/
2408	Assert(!pPage->fCached);
2409	if (fCanBeCached)
2410	{
2411	pPage->fCached = true;
2412	pgmPoolHashInsert(pPool, pPage);
2413	Log3(("pgmPoolCacheInsert: Caching %p:{.Core=%RHp, .idx=%d, .enmKind=%s, GCPhys=%RGp}\n",
2414	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
2415	STAM_COUNTER_INC(&pPool->StatCacheCacheable);
2416	}
2417	else
2418	{
2419	Log3(("pgmPoolCacheInsert: Not caching %p:{.Core=%RHp, .idx=%d, .enmKind=%s, GCPhys=%RGp}\n",
2420	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
2421	STAM_COUNTER_INC(&pPool->StatCacheUncacheable);
2422	}
2423
2424	/*
2425	* Insert at the head of the age list.
2426	*/
2427	pPage->iAgePrev = NIL_PGMPOOL_IDX;
2428	pPage->iAgeNext = pPool->iAgeHead;
2429	if (pPool->iAgeHead != NIL_PGMPOOL_IDX)
2430	pPool->aPages[pPool->iAgeHead].iAgePrev = pPage->idx;
2431	else
2432	pPool->iAgeTail = pPage->idx;
2433	pPool->iAgeHead = pPage->idx;
2434	}
2435
2436
2437	/**
2438	* Flushes a cached page.
2439	*
2440	* @param pPool The pool.
2441	* @param pPage The cached page.
2442	*/
2443	static void pgmPoolCacheFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2444	{
2445	Log3(("pgmPoolCacheFlushPage: %RGp\n", pPage->GCPhys));
2446
2447	/*
2448	* Remove the page from the hash.
2449	*/
2450	if (pPage->fCached)
2451	{
2452	pPage->fCached = false;
2453	pgmPoolHashRemove(pPool, pPage);
2454	}
2455	else
2456	Assert(pPage->iNext == NIL_PGMPOOL_IDX);
2457
2458	/*
2459	* Remove it from the age list.
2460	*/
2461	if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
2462	pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
2463	else
2464	pPool->iAgeTail = pPage->iAgePrev;
2465	if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
2466	pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
2467	else
2468	pPool->iAgeHead = pPage->iAgeNext;
2469	pPage->iAgeNext = NIL_PGMPOOL_IDX;
2470	pPage->iAgePrev = NIL_PGMPOOL_IDX;
2471	}
2472
2473
2474	/**
2475	* Looks for pages sharing the monitor.
2476	*
2477	* @returns Pointer to the head page.
2478	* @returns NULL if not found.
2479	* @param pPool The Pool
2480	* @param pNewPage The page which is going to be monitored.
2481	*/
2482	static PPGMPOOLPAGE pgmPoolMonitorGetPageByGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pNewPage)
2483	{
2484	/*
2485	* Look up the GCPhys in the hash.
2486	*/
2487	RTGCPHYS GCPhys = pNewPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
2488	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
2489	if (i == NIL_PGMPOOL_IDX)
2490	return NULL;
2491	do
2492	{
2493	PPGMPOOLPAGE pPage = &pPool->aPages[i];
2494	if ( pPage->GCPhys - GCPhys < PAGE_SIZE
2495	&& pPage != pNewPage)
2496	{
2497	switch (pPage->enmKind)
2498	{
2499	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2500	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2501	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2502	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2503	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2504	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2505	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2506	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2507	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2508	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2509	case PGMPOOLKIND_64BIT_PML4:
2510	case PGMPOOLKIND_32BIT_PD:
2511	case PGMPOOLKIND_PAE_PDPT:
2512	{
2513	/* find the head */
2514	while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
2515	{
2516	Assert(pPage->iMonitoredPrev != pPage->idx);
2517	pPage = &pPool->aPages[pPage->iMonitoredPrev];
2518	}
2519	return pPage;
2520	}
2521
2522	/* ignore, no monitoring. */
2523	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2524	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2525	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2526	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2527	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2528	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2529	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2530	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2531	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2532	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2533	case PGMPOOLKIND_ROOT_NESTED:
2534	case PGMPOOLKIND_PAE_PD_PHYS:
2535	case PGMPOOLKIND_PAE_PDPT_PHYS:
2536	case PGMPOOLKIND_32BIT_PD_PHYS:
2537	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
2538	break;
2539	default:
2540	AssertFatalMsgFailed(("enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
2541	}
2542	}
2543
2544	/* next */
2545	i = pPage->iNext;
2546	} while (i != NIL_PGMPOOL_IDX);
2547	return NULL;
2548	}
2549
2550
2551	/**
2552	* Enabled write monitoring of a guest page.
2553	*
2554	* @returns VBox status code.
2555	* @retval VINF_SUCCESS on success.
2556	* @param pPool The pool.
2557	* @param pPage The cached page.
2558	*/
2559	static int pgmPoolMonitorInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2560	{
2561	LogFlow(("pgmPoolMonitorInsert %RGp\n", pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK));
2562
2563	/*
2564	* Filter out the relevant kinds.
2565	*/
2566	switch (pPage->enmKind)
2567	{
2568	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2569	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2570	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2571	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2572	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2573	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2574	case PGMPOOLKIND_64BIT_PML4:
2575	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2576	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2577	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2578	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2579	case PGMPOOLKIND_32BIT_PD:
2580	case PGMPOOLKIND_PAE_PDPT:
2581	break;
2582
2583	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2584	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2585	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2586	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2587	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2588	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2589	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2590	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2591	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2592	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2593	case PGMPOOLKIND_ROOT_NESTED:
2594	/* Nothing to monitor here. */
2595	return VINF_SUCCESS;
2596
2597	case PGMPOOLKIND_32BIT_PD_PHYS:
2598	case PGMPOOLKIND_PAE_PDPT_PHYS:
2599	case PGMPOOLKIND_PAE_PD_PHYS:
2600	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
2601	/* Nothing to monitor here. */
2602	return VINF_SUCCESS;
2603	default:
2604	AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
2605	}
2606
2607	/*
2608	* Install handler.
2609	*/
2610	int rc;
2611	PPGMPOOLPAGE pPageHead = pgmPoolMonitorGetPageByGCPhys(pPool, pPage);
2612	if (pPageHead)
2613	{
2614	Assert(pPageHead != pPage); Assert(pPageHead->iMonitoredNext != pPage->idx);
2615	Assert(pPageHead->iMonitoredPrev != pPage->idx);
2616
2617	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2618	if (pPageHead->fDirty)
2619	pgmPoolFlushDirtyPage(pPool->CTX_SUFF(pVM), pPool, pPageHead->idxDirtyEntry, false /* do not remove */);
2620	#endif
2621
2622	pPage->iMonitoredPrev = pPageHead->idx;
2623	pPage->iMonitoredNext = pPageHead->iMonitoredNext;
2624	if (pPageHead->iMonitoredNext != NIL_PGMPOOL_IDX)
2625	pPool->aPages[pPageHead->iMonitoredNext].iMonitoredPrev = pPage->idx;
2626	pPageHead->iMonitoredNext = pPage->idx;
2627	rc = VINF_SUCCESS;
2628	}
2629	else
2630	{
2631	Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX); Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
2632	PVM pVM = pPool->CTX_SUFF(pVM);
2633	const RTGCPHYS GCPhysPage = pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
2634	rc = PGMHandlerPhysicalRegister(pVM, GCPhysPage, GCPhysPage + PAGE_OFFSET_MASK, pPool->hAccessHandlerType,
2635	MMHyperCCToR3(pVM, pPage), MMHyperCCToR0(pVM, pPage), MMHyperCCToRC(pVM, pPage),
2636	NIL_RTR3PTR /pszDesc/);
2637	/** @todo we should probably deal with out-of-memory conditions here, but for now increasing
2638	* the heap size should suffice. */
2639	AssertFatalMsgRC(rc, ("PGMHandlerPhysicalRegisterEx %RGp failed with %Rrc\n", GCPhysPage, rc));
2640	PVMCPU pVCpu = VMMGetCpu(pVM);
2641	AssertFatalMsg(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL) \|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), ("fSyncFlags=%x syncff=%d\n", pVCpu->pgm.s.fSyncFlags, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)));
2642	}
2643	pPage->fMonitored = true;
2644	return rc;
2645	}
2646
2647
2648	/**
2649	* Disables write monitoring of a guest page.
2650	*
2651	* @returns VBox status code.
2652	* @retval VINF_SUCCESS on success.
2653	* @param pPool The pool.
2654	* @param pPage The cached page.
2655	*/
2656	static int pgmPoolMonitorFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2657	{
2658	/*
2659	* Filter out the relevant kinds.
2660	*/
2661	switch (pPage->enmKind)
2662	{
2663	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2664	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2665	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2666	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2667	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2668	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2669	case PGMPOOLKIND_64BIT_PML4:
2670	case PGMPOOLKIND_32BIT_PD:
2671	case PGMPOOLKIND_PAE_PDPT:
2672	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2673	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2674	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2675	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2676	break;
2677
2678	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2679	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2680	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2681	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2682	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2683	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2684	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2685	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2686	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2687	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2688	case PGMPOOLKIND_ROOT_NESTED:
2689	case PGMPOOLKIND_PAE_PD_PHYS:
2690	case PGMPOOLKIND_PAE_PDPT_PHYS:
2691	case PGMPOOLKIND_32BIT_PD_PHYS:
2692	/* Nothing to monitor here. */
2693	Assert(!pPage->fMonitored);
2694	return VINF_SUCCESS;
2695
2696	default:
2697	AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
2698	}
2699	Assert(pPage->fMonitored);
2700
2701	/*
2702	* Remove the page from the monitored list or uninstall it if last.
2703	*/
2704	const PVM pVM = pPool->CTX_SUFF(pVM);
2705	int rc;
2706	if ( pPage->iMonitoredNext != NIL_PGMPOOL_IDX
2707	\|\| pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
2708	{
2709	if (pPage->iMonitoredPrev == NIL_PGMPOOL_IDX)
2710	{
2711	PPGMPOOLPAGE pNewHead = &pPool->aPages[pPage->iMonitoredNext];
2712	pNewHead->iMonitoredPrev = NIL_PGMPOOL_IDX;
2713	rc = PGMHandlerPhysicalChangeUserArgs(pVM, pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK, MMHyperCCToR3(pVM, pNewHead),
2714	MMHyperCCToR0(pVM, pNewHead), MMHyperCCToRC(pVM, pNewHead));
2715
2716	AssertFatalRCSuccess(rc);
2717	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
2718	}
2719	else
2720	{
2721	pPool->aPages[pPage->iMonitoredPrev].iMonitoredNext = pPage->iMonitoredNext;
2722	if (pPage->iMonitoredNext != NIL_PGMPOOL_IDX)
2723	{
2724	pPool->aPages[pPage->iMonitoredNext].iMonitoredPrev = pPage->iMonitoredPrev;
2725	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
2726	}
2727	pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
2728	rc = VINF_SUCCESS;
2729	}
2730	}
2731	else
2732	{
2733	rc = PGMHandlerPhysicalDeregister(pVM, pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
2734	AssertFatalRC(rc);
2735	PVMCPU pVCpu = VMMGetCpu(pVM);
2736	AssertFatalMsg(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL) \|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3),
2737	("%#x %#x\n", pVCpu->pgm.s.fSyncFlags, pVM->fGlobalForcedActions));
2738	}
2739	pPage->fMonitored = false;
2740
2741	/*
2742	* Remove it from the list of modified pages (if in it).
2743	*/
2744	pgmPoolMonitorModifiedRemove(pPool, pPage);
2745
2746	return rc;
2747	}
2748
2749
2750	/**
2751	* Inserts the page into the list of modified pages.
2752	*
2753	* @param pPool The pool.
2754	* @param pPage The page.
2755	*/
2756	void pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2757	{
2758	Log3(("pgmPoolMonitorModifiedInsert: idx=%d\n", pPage->idx));
2759	AssertMsg( pPage->iModifiedNext == NIL_PGMPOOL_IDX
2760	&& pPage->iModifiedPrev == NIL_PGMPOOL_IDX
2761	&& pPool->iModifiedHead != pPage->idx,
2762	("Next=%d Prev=%d idx=%d cModifications=%d Head=%d cModifiedPages=%d\n",
2763	pPage->iModifiedNext, pPage->iModifiedPrev, pPage->idx, pPage->cModifications,
2764	pPool->iModifiedHead, pPool->cModifiedPages));
2765
2766	pPage->iModifiedNext = pPool->iModifiedHead;
2767	if (pPool->iModifiedHead != NIL_PGMPOOL_IDX)
2768	pPool->aPages[pPool->iModifiedHead].iModifiedPrev = pPage->idx;
2769	pPool->iModifiedHead = pPage->idx;
2770	pPool->cModifiedPages++;
2771	#ifdef VBOX_WITH_STATISTICS
2772	if (pPool->cModifiedPages > pPool->cModifiedPagesHigh)
2773	pPool->cModifiedPagesHigh = pPool->cModifiedPages;
2774	#endif
2775	}
2776
2777
2778	/**
2779	* Removes the page from the list of modified pages and resets the
2780	* modification counter.
2781	*
2782	* @param pPool The pool.
2783	* @param pPage The page which is believed to be in the list of modified pages.
2784	*/
2785	static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2786	{
2787	Log3(("pgmPoolMonitorModifiedRemove: idx=%d cModifications=%d\n", pPage->idx, pPage->cModifications));
2788	if (pPool->iModifiedHead == pPage->idx)
2789	{
2790	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
2791	pPool->iModifiedHead = pPage->iModifiedNext;
2792	if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
2793	{
2794	pPool->aPages[pPage->iModifiedNext].iModifiedPrev = NIL_PGMPOOL_IDX;
2795	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2796	}
2797	pPool->cModifiedPages--;
2798	}
2799	else if (pPage->iModifiedPrev != NIL_PGMPOOL_IDX)
2800	{
2801	pPool->aPages[pPage->iModifiedPrev].iModifiedNext = pPage->iModifiedNext;
2802	if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
2803	{
2804	pPool->aPages[pPage->iModifiedNext].iModifiedPrev = pPage->iModifiedPrev;
2805	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2806	}
2807	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2808	pPool->cModifiedPages--;
2809	}
2810	else
2811	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
2812	pPage->cModifications = 0;
2813	}
2814
2815
2816	/**
2817	* Zaps the list of modified pages, resetting their modification counters in the process.
2818	*
2819	* @param pVM The cross context VM structure.
2820	*/
2821	static void pgmPoolMonitorModifiedClearAll(PVM pVM)
2822	{
2823	pgmLock(pVM);
2824	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2825	LogFlow(("pgmPoolMonitorModifiedClearAll: cModifiedPages=%d\n", pPool->cModifiedPages));
2826
2827	unsigned cPages = 0; NOREF(cPages);
2828
2829	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2830	pgmPoolResetDirtyPages(pVM);
2831	#endif
2832
2833	uint16_t idx = pPool->iModifiedHead;
2834	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
2835	while (idx != NIL_PGMPOOL_IDX)
2836	{
2837	PPGMPOOLPAGE pPage = &pPool->aPages[idx];
2838	idx = pPage->iModifiedNext;
2839	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2840	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2841	pPage->cModifications = 0;
2842	Assert(++cPages);
2843	}
2844	AssertMsg(cPages == pPool->cModifiedPages, ("%d != %d\n", cPages, pPool->cModifiedPages));
2845	pPool->cModifiedPages = 0;
2846	pgmUnlock(pVM);
2847	}
2848
2849
2850	/**
2851	* Handle SyncCR3 pool tasks
2852	*
2853	* @returns VBox status code.
2854	* @retval VINF_SUCCESS if successfully added.
2855	* @retval VINF_PGM_SYNC_CR3 is it needs to be deferred to ring 3 (GC only)
2856	* @param pVCpu The cross context virtual CPU structure.
2857	* @remark Should only be used when monitoring is available, thus placed in
2858	* the PGMPOOL_WITH_MONITORING \#ifdef.
2859	*/
2860	int pgmPoolSyncCR3(PVMCPU pVCpu)
2861	{
2862	PVM pVM = pVCpu->CTX_SUFF(pVM);
2863	LogFlow(("pgmPoolSyncCR3 fSyncFlags=%x\n", pVCpu->pgm.s.fSyncFlags));
2864
2865	/*
2866	* When monitoring shadowed pages, we reset the modification counters on CR3 sync.
2867	* Occasionally we will have to clear all the shadow page tables because we wanted
2868	* to monitor a page which was mapped by too many shadowed page tables. This operation
2869	* sometimes referred to as a 'lightweight flush'.
2870	*/
2871	# ifdef IN_RING3 /* Don't flush in ring-0 or raw mode, it's taking too long. */
2872	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2873	pgmR3PoolClearAll(pVM, false /fFlushRemTlb/);
2874	# else /* !IN_RING3 */
2875	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2876	{
2877	Log(("SyncCR3: PGM_SYNC_CLEAR_PGM_POOL is set -> VINF_PGM_SYNC_CR3\n"));
2878	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
2879
2880	/* Make sure all other VCPUs return to ring 3. */
2881	if (pVM->cCpus > 1)
2882	{
2883	VM_FF_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING);
2884	PGM_INVL_ALL_VCPU_TLBS(pVM);
2885	}
2886	return VINF_PGM_SYNC_CR3;
2887	}
2888	# endif /* !IN_RING3 */
2889	else
2890	{
2891	pgmPoolMonitorModifiedClearAll(pVM);
2892
2893	/* pgmPoolMonitorModifiedClearAll can cause a pgm pool flush (dirty page clearing), so make sure we handle this! */
2894	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2895	{
2896	Log(("pgmPoolMonitorModifiedClearAll caused a pgm flush -> call pgmPoolSyncCR3 again!\n"));
2897	return pgmPoolSyncCR3(pVCpu);
2898	}
2899	}
2900	return VINF_SUCCESS;
2901	}
2902
2903
2904	/**
2905	* Frees up at least one user entry.
2906	*
2907	* @returns VBox status code.
2908	* @retval VINF_SUCCESS if successfully added.
2909	*
2910	* @param pPool The pool.
2911	* @param iUser The user index.
2912	*/
2913	static int pgmPoolTrackFreeOneUser(PPGMPOOL pPool, uint16_t iUser)
2914	{
2915	STAM_COUNTER_INC(&pPool->StatTrackFreeUpOneUser);
2916	/*
2917	* Just free cached pages in a braindead fashion.
2918	*/
2919	/** @todo walk the age list backwards and free the first with usage. */
2920	int rc = VINF_SUCCESS;
2921	do
2922	{
2923	int rc2 = pgmPoolCacheFreeOne(pPool, iUser);
2924	if (RT_FAILURE(rc2) && rc == VINF_SUCCESS)
2925	rc = rc2;
2926	} while (pPool->iUserFreeHead == NIL_PGMPOOL_USER_INDEX);
2927	return rc;
2928	}
2929
2930
2931	/**
2932	* Inserts a page into the cache.
2933	*
2934	* This will create user node for the page, insert it into the GCPhys
2935	* hash, and insert it into the age list.
2936	*
2937	* @returns VBox status code.
2938	* @retval VINF_SUCCESS if successfully added.
2939	*
2940	* @param pPool The pool.
2941	* @param pPage The cached page.
2942	* @param GCPhys The GC physical address of the page we're gonna shadow.
2943	* @param iUser The user index.
2944	* @param iUserTable The user table index.
2945	*/
2946	DECLINLINE(int) pgmPoolTrackInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhys, uint16_t iUser, uint32_t iUserTable)
2947	{
2948	int rc = VINF_SUCCESS;
2949	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
2950
2951	LogFlow(("pgmPoolTrackInsert GCPhys=%RGp iUser=%d iUserTable=%x\n", GCPhys, iUser, iUserTable)); RT_NOREF_PV(GCPhys);
2952
2953	if (iUser != NIL_PGMPOOL_IDX)
2954	{
2955	#ifdef VBOX_STRICT
2956	/*
2957	* Check that the entry doesn't already exists.
2958	*/
2959	if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
2960	{
2961	uint16_t i = pPage->iUserHead;
2962	do
2963	{
2964	Assert(i < pPool->cMaxUsers);
2965	AssertMsg(paUsers[i].iUser != iUser \|\| paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
2966	i = paUsers[i].iNext;
2967	} while (i != NIL_PGMPOOL_USER_INDEX);
2968	}
2969	#endif
2970
2971	/*
2972	* Find free a user node.
2973	*/
2974	uint16_t i = pPool->iUserFreeHead;
2975	if (i == NIL_PGMPOOL_USER_INDEX)
2976	{
2977	rc = pgmPoolTrackFreeOneUser(pPool, iUser);
2978	if (RT_FAILURE(rc))
2979	return rc;
2980	i = pPool->iUserFreeHead;
2981	}
2982
2983	/*
2984	* Unlink the user node from the free list,
2985	* initialize and insert it into the user list.
2986	*/
2987	pPool->iUserFreeHead = paUsers[i].iNext;
2988	paUsers[i].iNext = NIL_PGMPOOL_USER_INDEX;
2989	paUsers[i].iUser = iUser;
2990	paUsers[i].iUserTable = iUserTable;
2991	pPage->iUserHead = i;
2992	}
2993	else
2994	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
2995
2996
2997	/*
2998	* Insert into cache and enable monitoring of the guest page if enabled.
2999	*
3000	* Until we implement caching of all levels, including the CR3 one, we'll
3001	* have to make sure we don't try monitor & cache any recursive reuse of
3002	* a monitored CR3 page. Because all windows versions are doing this we'll
3003	* have to be able to do combined access monitoring, CR3 + PT and
3004	* PD + PT (guest PAE).
3005	*
3006	* Update:
3007	* We're now cooperating with the CR3 monitor if an uncachable page is found.
3008	*/
3009	const bool fCanBeMonitored = true;
3010	pgmPoolCacheInsert(pPool, pPage, fCanBeMonitored); /* This can be expanded. */
3011	if (fCanBeMonitored)
3012	{
3013	rc = pgmPoolMonitorInsert(pPool, pPage);
3014	AssertRC(rc);
3015	}
3016	return rc;
3017	}
3018
3019
3020	/**
3021	* Adds a user reference to a page.
3022	*
3023	* This will move the page to the head of the
3024	*
3025	* @returns VBox status code.
3026	* @retval VINF_SUCCESS if successfully added.
3027	*
3028	* @param pPool The pool.
3029	* @param pPage The cached page.
3030	* @param iUser The user index.
3031	* @param iUserTable The user table.
3032	*/
3033	static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
3034	{
3035	Log3(("pgmPoolTrackAddUser: GCPhys=%RGp iUser=%x iUserTable=%x\n", pPage->GCPhys, iUser, iUserTable));
3036	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
3037	Assert(iUser != NIL_PGMPOOL_IDX);
3038
3039	# ifdef VBOX_STRICT
3040	/*
3041	* Check that the entry doesn't already exists. We only allow multiple
3042	* users of top-level paging structures (SHW_POOL_ROOT_IDX).
3043	*/
3044	if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
3045	{
3046	uint16_t i = pPage->iUserHead;
3047	do
3048	{
3049	Assert(i < pPool->cMaxUsers);
3050	/** @todo this assertion looks odd... Shouldn't it be && here? */
3051	AssertMsg(paUsers[i].iUser != iUser \|\| paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
3052	i = paUsers[i].iNext;
3053	} while (i != NIL_PGMPOOL_USER_INDEX);
3054	}
3055	# endif
3056
3057	/*
3058	* Allocate a user node.
3059	*/
3060	uint16_t i = pPool->iUserFreeHead;
3061	if (i == NIL_PGMPOOL_USER_INDEX)
3062	{
3063	int rc = pgmPoolTrackFreeOneUser(pPool, iUser);
3064	if (RT_FAILURE(rc))
3065	return rc;
3066	i = pPool->iUserFreeHead;
3067	}
3068	pPool->iUserFreeHead = paUsers[i].iNext;
3069
3070	/*
3071	* Initialize the user node and insert it.
3072	*/
3073	paUsers[i].iNext = pPage->iUserHead;
3074	paUsers[i].iUser = iUser;
3075	paUsers[i].iUserTable = iUserTable;
3076	pPage->iUserHead = i;
3077
3078	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
3079	if (pPage->fDirty)
3080	pgmPoolFlushDirtyPage(pPool->CTX_SUFF(pVM), pPool, pPage->idxDirtyEntry, false /* do not remove */);
3081	# endif
3082
3083	/*
3084	* Tell the cache to update its replacement stats for this page.
3085	*/
3086	pgmPoolCacheUsed(pPool, pPage);
3087	return VINF_SUCCESS;
3088	}
3089
3090
3091	/**
3092	* Frees a user record associated with a page.
3093	*
3094	* This does not clear the entry in the user table, it simply replaces the
3095	* user record to the chain of free records.
3096	*
3097	* @param pPool The pool.
3098	* @param pPage The shadow page.
3099	* @param iUser The shadow page pool index of the user table.
3100	* @param iUserTable The index into the user table (shadowed).
3101	*
3102	* @remarks Don't call this for root pages.
3103	*/
3104	static void pgmPoolTrackFreeUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
3105	{
3106	Log3(("pgmPoolTrackFreeUser %RGp %x %x\n", pPage->GCPhys, iUser, iUserTable));
3107	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
3108	Assert(iUser != NIL_PGMPOOL_IDX);
3109
3110	/*
3111	* Unlink and free the specified user entry.
3112	*/
3113
3114	/* Special: For PAE and 32-bit paging, there is usually no more than one user. */
3115	uint16_t i = pPage->iUserHead;
3116	if ( i != NIL_PGMPOOL_USER_INDEX
3117	&& paUsers[i].iUser == iUser
3118	&& paUsers[i].iUserTable == iUserTable)
3119	{
3120	pPage->iUserHead = paUsers[i].iNext;
3121
3122	paUsers[i].iUser = NIL_PGMPOOL_IDX;
3123	paUsers[i].iNext = pPool->iUserFreeHead;
3124	pPool->iUserFreeHead = i;
3125	return;
3126	}
3127
3128	/* General: Linear search. */
3129	uint16_t iPrev = NIL_PGMPOOL_USER_INDEX;
3130	while (i != NIL_PGMPOOL_USER_INDEX)
3131	{
3132	if ( paUsers[i].iUser == iUser
3133	&& paUsers[i].iUserTable == iUserTable)
3134	{
3135	if (iPrev != NIL_PGMPOOL_USER_INDEX)
3136	paUsers[iPrev].iNext = paUsers[i].iNext;
3137	else
3138	pPage->iUserHead = paUsers[i].iNext;
3139
3140	paUsers[i].iUser = NIL_PGMPOOL_IDX;
3141	paUsers[i].iNext = pPool->iUserFreeHead;
3142	pPool->iUserFreeHead = i;
3143	return;
3144	}
3145	iPrev = i;
3146	i = paUsers[i].iNext;
3147	}
3148
3149	/* Fatal: didn't find it */
3150	AssertFatalMsgFailed(("Didn't find the user entry! iUser=%d iUserTable=%#x GCPhys=%RGp\n",
3151	iUser, iUserTable, pPage->GCPhys));
3152	}
3153
3154
3155	#if 0 /* unused */
3156	/**
3157	* Gets the entry size of a shadow table.
3158	*
3159	* @param enmKind The kind of page.
3160	*
3161	* @returns The size of the entry in bytes. That is, 4 or 8.
3162	* @returns If the kind is not for a table, an assertion is raised and 0 is
3163	* returned.
3164	*/
3165	DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind)
3166	{
3167	switch (enmKind)
3168	{
3169	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3170	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3171	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3172	case PGMPOOLKIND_32BIT_PD:
3173	case PGMPOOLKIND_32BIT_PD_PHYS:
3174	return 4;
3175
3176	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3177	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3178	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3179	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3180	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3181	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3182	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3183	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3184	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3185	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3186	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3187	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3188	case PGMPOOLKIND_64BIT_PML4:
3189	case PGMPOOLKIND_PAE_PDPT:
3190	case PGMPOOLKIND_ROOT_NESTED:
3191	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3192	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3193	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3194	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3195	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
3196	case PGMPOOLKIND_PAE_PD_PHYS:
3197	case PGMPOOLKIND_PAE_PDPT_PHYS:
3198	return 8;
3199
3200	default:
3201	AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
3202	}
3203	}
3204	#endif /* unused */
3205
3206	#if 0 /* unused */
3207	/**
3208	* Gets the entry size of a guest table.
3209	*
3210	* @param enmKind The kind of page.
3211	*
3212	* @returns The size of the entry in bytes. That is, 0, 4 or 8.
3213	* @returns If the kind is not for a table, an assertion is raised and 0 is
3214	* returned.
3215	*/
3216	DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind)
3217	{
3218	switch (enmKind)
3219	{
3220	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3221	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3222	case PGMPOOLKIND_32BIT_PD:
3223	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3224	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3225	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3226	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3227	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3228	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3229	return 4;
3230
3231	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3232	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3233	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3234	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3235	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3236	case PGMPOOLKIND_64BIT_PML4:
3237	case PGMPOOLKIND_PAE_PDPT:
3238	return 8;
3239
3240	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3241	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3242	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3243	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3244	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3245	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3246	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
3247	case PGMPOOLKIND_ROOT_NESTED:
3248	case PGMPOOLKIND_PAE_PD_PHYS:
3249	case PGMPOOLKIND_PAE_PDPT_PHYS:
3250	case PGMPOOLKIND_32BIT_PD_PHYS:
3251	/** @todo can we return 0? (nobody is calling this...) */
3252	AssertFailed();
3253	return 0;
3254
3255	default:
3256	AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
3257	}
3258	}
3259	#endif /* unused */
3260
3261
3262	/**
3263	* Checks one shadow page table entry for a mapping of a physical page.
3264	*
3265	* @returns true / false indicating removal of all relevant PTEs
3266	*
3267	* @param pVM The cross context VM structure.
3268	* @param pPhysPage The guest page in question.
3269	* @param fFlushPTEs Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
3270	* @param iShw The shadow page table.
3271	* @param iPte Page table entry or NIL_PGMPOOL_PHYSEXT_IDX_PTE if unknown
3272	*/
3273	static bool pgmPoolTrackFlushGCPhysPTInt(PVM pVM, PCPGMPAGE pPhysPage, bool fFlushPTEs, uint16_t iShw, uint16_t iPte)
3274	{
3275	LogFlow(("pgmPoolTrackFlushGCPhysPTInt: pPhysPage=%RHp iShw=%d iPte=%d\n", PGM_PAGE_GET_HCPHYS(pPhysPage), iShw, iPte));
3276	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3277	bool fRet = false;
3278
3279	/*
3280	* Assert sanity.
3281	*/
3282	Assert(iPte != NIL_PGMPOOL_PHYSEXT_IDX_PTE);
3283	AssertFatalMsg(iShw < pPool->cCurPages && iShw != NIL_PGMPOOL_IDX, ("iShw=%d\n", iShw));
3284	PPGMPOOLPAGE pPage = &pPool->aPages[iShw];
3285
3286	/*
3287	* Then, clear the actual mappings to the page in the shadow PT.
3288	*/
3289	switch (pPage->enmKind)
3290	{
3291	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3292	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3293	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3294	{
3295	const uint32_t u32 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
3296	PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3297	uint32_t u32AndMask = 0;
3298	uint32_t u32OrMask = 0;
3299
3300	if (!fFlushPTEs)
3301	{
3302	switch (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage))
3303	{
3304	case PGM_PAGE_HNDL_PHYS_STATE_NONE: /* No handler installed. */
3305	case PGM_PAGE_HNDL_PHYS_STATE_DISABLED: /* Monitoring is temporarily disabled. */
3306	u32OrMask = X86_PTE_RW;
3307	u32AndMask = UINT32_MAX;
3308	fRet = true;
3309	STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
3310	break;
3311
3312	case PGM_PAGE_HNDL_PHYS_STATE_WRITE: /* Write access is monitored. */
3313	u32OrMask = 0;
3314	u32AndMask = ~X86_PTE_RW;
3315	fRet = true;
3316	STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
3317	break;
3318	default:
3319	/* (shouldn't be here, will assert below) */
3320	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3321	break;
3322	}
3323	}
3324	else
3325	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3326
3327	/* Update the counter if we're removing references. */
3328	if (!u32AndMask)
3329	{
3330	Assert(pPage->cPresent);
3331	Assert(pPool->cPresent);
3332	pPage->cPresent--;
3333	pPool->cPresent--;
3334	}
3335
3336	if ((pPT->a[iPte].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
3337	{
3338	X86PTE Pte;
3339
3340	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX32\n", iPte, pPT->a[iPte]));
3341	Pte.u = (pPT->a[iPte].u & u32AndMask) \| u32OrMask;
3342	if (Pte.u & PGM_PTFLAGS_TRACK_DIRTY)
3343	Pte.n.u1Write = 0; /* need to disallow writes when dirty bit tracking is still active. */
3344
3345	ASMAtomicWriteU32(&pPT->a[iPte].u, Pte.u);
3346	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3347	return fRet;
3348	}
3349	#ifdef LOG_ENABLED
3350	Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3351	for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPT->a); i++)
3352	if ((pPT->a[i].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
3353	{
3354	Log(("i=%d cFound=%d\n", i, ++cFound));
3355	}
3356	#endif
3357	AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d u32=%RX32 poolkind=%x\n", pPage->iFirstPresent, pPage->cPresent, u32, pPage->enmKind));
3358	/PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);/
3359	break;
3360	}
3361
3362	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3363	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3364	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3365	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3366	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3367	case PGMPOOLKIND_EPT_PT_FOR_PHYS: /* physical mask the same as PAE; RW bit as well; be careful! */
3368	{
3369	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
3370	PPGMSHWPTPAE pPT = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3371	uint64_t u64OrMask = 0;
3372	uint64_t u64AndMask = 0;
3373
3374	if (!fFlushPTEs)
3375	{
3376	switch (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage))
3377	{
3378	case PGM_PAGE_HNDL_PHYS_STATE_NONE: /* No handler installed. */
3379	case PGM_PAGE_HNDL_PHYS_STATE_DISABLED: /* Monitoring is temporarily disabled. */
3380	u64OrMask = X86_PTE_RW;
3381	u64AndMask = UINT64_MAX;
3382	fRet = true;
3383	STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
3384	break;
3385
3386	case PGM_PAGE_HNDL_PHYS_STATE_WRITE: /* Write access is monitored. */
3387	u64OrMask = 0;
3388	u64AndMask = ~(uint64_t)X86_PTE_RW;
3389	fRet = true;
3390	STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
3391	break;
3392
3393	default:
3394	/* (shouldn't be here, will assert below) */
3395	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3396	break;
3397	}
3398	}
3399	else
3400	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3401
3402	/* Update the counter if we're removing references. */
3403	if (!u64AndMask)
3404	{
3405	Assert(pPage->cPresent);
3406	Assert(pPool->cPresent);
3407	pPage->cPresent--;
3408	pPool->cPresent--;
3409	}
3410
3411	if ((PGMSHWPTEPAE_GET_U(pPT->a[iPte]) & (X86_PTE_PAE_PG_MASK \| X86_PTE_P \| X86_PTE_PAE_MBZ_MASK_NX)) == u64)
3412	{
3413	X86PTEPAE Pte;
3414
3415	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX64\n", iPte, PGMSHWPTEPAE_GET_LOG(pPT->a[iPte])));
3416	Pte.u = (PGMSHWPTEPAE_GET_U(pPT->a[iPte]) & u64AndMask) \| u64OrMask;
3417	if (Pte.u & PGM_PTFLAGS_TRACK_DIRTY)
3418	Pte.n.u1Write = 0; /* need to disallow writes when dirty bit tracking is still active. */
3419
3420	PGMSHWPTEPAE_ATOMIC_SET(pPT->a[iPte], Pte.u);
3421	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3422	return fRet;
3423	}
3424	#ifdef LOG_ENABLED
3425	Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3426	Log(("Found %RX64 expected %RX64\n", PGMSHWPTEPAE_GET_U(pPT->a[iPte]) & (X86_PTE_PAE_PG_MASK \| X86_PTE_P \| X86_PTE_PAE_MBZ_MASK_NX), u64));
3427	for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPT->a); i++)
3428	if ((PGMSHWPTEPAE_GET_U(pPT->a[i]) & (X86_PTE_PAE_PG_MASK \| X86_PTE_P \| X86_PTE_PAE_MBZ_MASK_NX)) == u64)
3429	Log(("i=%d cFound=%d\n", i, ++cFound));
3430	#endif
3431	AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d u64=%RX64 poolkind=%x iPte=%d PT=%RX64\n", pPage->iFirstPresent, pPage->cPresent, u64, pPage->enmKind, iPte, PGMSHWPTEPAE_GET_LOG(pPT->a[iPte])));
3432	/PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);/
3433	break;
3434	}
3435
3436	#ifdef PGM_WITH_LARGE_PAGES
3437	/* Large page case only. */
3438	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3439	{
3440	Assert(pVM->pgm.s.fNestedPaging);
3441
3442	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PDE4M_P \| X86_PDE4M_PS;
3443	PEPTPD pPD = (PEPTPD)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3444
3445	if ((pPD->a[iPte].u & (EPT_PDE2M_PG_MASK \| X86_PDE4M_P \| X86_PDE4M_PS)) == u64)
3446	{
3447	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pde=%RX64\n", iPte, pPD->a[iPte]));
3448	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3449	pPD->a[iPte].u = 0;
3450	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);
3451
3452	/* Update the counter as we're removing references. */
3453	Assert(pPage->cPresent);
3454	Assert(pPool->cPresent);
3455	pPage->cPresent--;
3456	pPool->cPresent--;
3457
3458	return fRet;
3459	}
3460	# ifdef LOG_ENABLED
3461	Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3462	for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPD->a); i++)
3463	if ((pPD->a[i].u & (EPT_PDE2M_PG_MASK \| X86_PDE4M_P \| X86_PDE4M_PS)) == u64)
3464	Log(("i=%d cFound=%d\n", i, ++cFound));
3465	# endif
3466	AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3467	/PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);/
3468	break;
3469	}
3470
3471	/* AMD-V nested paging / /* @todo merge with EPT as we only check the parts that are identical. */
3472	case PGMPOOLKIND_PAE_PD_PHYS:
3473	{
3474	Assert(pVM->pgm.s.fNestedPaging);
3475
3476	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PDE4M_P \| X86_PDE4M_PS;
3477	PX86PD pPD = (PX86PD)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3478
3479	if ((pPD->a[iPte].u & (X86_PDE2M_PAE_PG_MASK \| X86_PDE4M_P \| X86_PDE4M_PS)) == u64)
3480	{
3481	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pde=%RX64\n", iPte, pPD->a[iPte]));
3482	STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
3483	pPD->a[iPte].u = 0;
3484	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);
3485
3486	/* Update the counter as we're removing references. */
3487	Assert(pPage->cPresent);
3488	Assert(pPool->cPresent);
3489	pPage->cPresent--;
3490	pPool->cPresent--;
3491	return fRet;
3492	}
3493	# ifdef LOG_ENABLED
3494	Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3495	for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPD->a); i++)
3496	if ((pPD->a[i].u & (X86_PDE2M_PAE_PG_MASK \| X86_PDE4M_P \| X86_PDE4M_PS)) == u64)
3497	Log(("i=%d cFound=%d\n", i, ++cFound));
3498	# endif
3499	AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
3500	/PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);/
3501	break;
3502	}
3503	#endif /* PGM_WITH_LARGE_PAGES */
3504
3505	default:
3506	AssertFatalMsgFailed(("enmKind=%d iShw=%d\n", pPage->enmKind, iShw));
3507	}
3508
3509	/* not reached. */
3510	#ifndef _MSC_VER
3511	return fRet;
3512	#endif
3513	}
3514
3515
3516	/**
3517	* Scans one shadow page table for mappings of a physical page.
3518	*
3519	* @param pVM The cross context VM structure.
3520	* @param pPhysPage The guest page in question.
3521	* @param fFlushPTEs Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
3522	* @param iShw The shadow page table.
3523	*/
3524	static void pgmPoolTrackFlushGCPhysPT(PVM pVM, PPGMPAGE pPhysPage, bool fFlushPTEs, uint16_t iShw)
3525	{
3526	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
3527
3528	/* We should only come here with when there's only one reference to this physical page. */
3529	Assert(PGMPOOL_TD_GET_CREFS(PGM_PAGE_GET_TRACKING(pPhysPage)) == 1);
3530
3531	Log2(("pgmPoolTrackFlushGCPhysPT: pPhysPage=%RHp iShw=%d\n", PGM_PAGE_GET_HCPHYS(pPhysPage), iShw));
3532	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPT, f);
3533	bool fKeptPTEs = pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, fFlushPTEs, iShw, PGM_PAGE_GET_PTE_INDEX(pPhysPage));
3534	if (!fKeptPTEs)
3535	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
3536	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPT, f);
3537	}
3538
3539
3540	/**
3541	* Flushes a list of shadow page tables mapping the same physical page.
3542	*
3543	* @param pVM The cross context VM structure.
3544	* @param pPhysPage The guest page in question.
3545	* @param fFlushPTEs Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
3546	* @param iPhysExt The physical cross reference extent list to flush.
3547	*/
3548	static void pgmPoolTrackFlushGCPhysPTs(PVM pVM, PPGMPAGE pPhysPage, bool fFlushPTEs, uint16_t iPhysExt)
3549	{
3550	PGM_LOCK_ASSERT_OWNER(pVM);
3551	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3552	bool fKeepList = false;
3553
3554	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTs, f);
3555	Log2(("pgmPoolTrackFlushGCPhysPTs: pPhysPage=%RHp iPhysExt=%u\n", PGM_PAGE_GET_HCPHYS(pPhysPage), iPhysExt));
3556
3557	const uint16_t iPhysExtStart = iPhysExt;
3558	PPGMPOOLPHYSEXT pPhysExt;
3559	do
3560	{
3561	Assert(iPhysExt < pPool->cMaxPhysExts);
3562	pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3563	for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
3564	{
3565	if (pPhysExt->aidx[i] != NIL_PGMPOOL_IDX)
3566	{
3567	bool fKeptPTEs = pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, fFlushPTEs, pPhysExt->aidx[i], pPhysExt->apte[i]);
3568	if (!fKeptPTEs)
3569	{
3570	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
3571	pPhysExt->apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
3572	}
3573	else
3574	fKeepList = true;
3575	}
3576	}
3577	/* next */
3578	iPhysExt = pPhysExt->iNext;
3579	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
3580
3581	if (!fKeepList)
3582	{
3583	/* insert the list into the free list and clear the ram range entry. */
3584	pPhysExt->iNext = pPool->iPhysExtFreeHead;
3585	pPool->iPhysExtFreeHead = iPhysExtStart;
3586	/* Invalidate the tracking data. */
3587	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
3588	}
3589
3590	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTs, f);
3591	}
3592
3593
3594	/**
3595	* Flushes all shadow page table mappings of the given guest page.
3596	*
3597	* This is typically called when the host page backing the guest one has been
3598	* replaced or when the page protection was changed due to a guest access
3599	* caught by the monitoring.
3600	*
3601	* @returns VBox status code.
3602	* @retval VINF_SUCCESS if all references has been successfully cleared.
3603	* @retval VINF_PGM_SYNC_CR3 if we're better off with a CR3 sync and a page
3604	* pool cleaning. FF and sync flags are set.
3605	*
3606	* @param pVM The cross context VM structure.
3607	* @param GCPhysPage GC physical address of the page in question
3608	* @param pPhysPage The guest page in question.
3609	* @param fFlushPTEs Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
3610	* @param pfFlushTLBs This is set to @a true if the shadow TLBs should be
3611	* flushed, it is NOT touched if this isn't necessary.
3612	* The caller MUST initialized this to @a false.
3613	*/
3614	int pgmPoolTrackUpdateGCPhys(PVM pVM, RTGCPHYS GCPhysPage, PPGMPAGE pPhysPage, bool fFlushPTEs, bool *pfFlushTLBs)
3615	{
3616	PVMCPU pVCpu = VMMGetCpu(pVM);
3617	pgmLock(pVM);
3618	int rc = VINF_SUCCESS;
3619
3620	#ifdef PGM_WITH_LARGE_PAGES
3621	/* Is this page part of a large page? */
3622	if (PGM_PAGE_GET_PDE_TYPE(pPhysPage) == PGM_PAGE_PDE_TYPE_PDE)
3623	{
3624	RTGCPHYS GCPhysBase = GCPhysPage & X86_PDE2M_PAE_PG_MASK;
3625	GCPhysPage &= X86_PDE_PAE_PG_MASK;
3626
3627	/* Fetch the large page base. */
3628	PPGMPAGE pLargePage;
3629	if (GCPhysBase != GCPhysPage)
3630	{
3631	pLargePage = pgmPhysGetPage(pVM, GCPhysBase);
3632	AssertFatal(pLargePage);
3633	}
3634	else
3635	pLargePage = pPhysPage;
3636
3637	Log(("pgmPoolTrackUpdateGCPhys: update large page PDE for %RGp (%RGp)\n", GCPhysBase, GCPhysPage));
3638
3639	if (PGM_PAGE_GET_PDE_TYPE(pLargePage) == PGM_PAGE_PDE_TYPE_PDE)
3640	{
3641	/* Mark the large page as disabled as we need to break it up to change a single page in the 2 MB range. */
3642	PGM_PAGE_SET_PDE_TYPE(pVM, pLargePage, PGM_PAGE_PDE_TYPE_PDE_DISABLED);
3643	pVM->pgm.s.cLargePagesDisabled++;
3644
3645	/* Update the base as that only that one has a reference and there's only one PDE to clear. */
3646	rc = pgmPoolTrackUpdateGCPhys(pVM, GCPhysBase, pLargePage, fFlushPTEs, pfFlushTLBs);
3647
3648	*pfFlushTLBs = true;
3649	pgmUnlock(pVM);
3650	return rc;
3651	}
3652	}
3653	#else
3654	NOREF(GCPhysPage);
3655	#endif /* PGM_WITH_LARGE_PAGES */
3656
3657	const uint16_t u16 = PGM_PAGE_GET_TRACKING(pPhysPage);
3658	if (u16)
3659	{
3660	/*
3661	* The zero page is currently screwing up the tracking and we'll
3662	* have to flush the whole shebang. Unless VBOX_WITH_NEW_LAZY_PAGE_ALLOC
3663	* is defined, zero pages won't normally be mapped. Some kind of solution
3664	* will be needed for this problem of course, but it will have to wait...
3665	*/
3666	if ( PGM_PAGE_IS_ZERO(pPhysPage)
3667	\|\| PGM_PAGE_IS_BALLOONED(pPhysPage))
3668	rc = VINF_PGM_GCPHYS_ALIASED;
3669	else
3670	{
3671	# if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) \|\| defined(IN_RC) /** @todo we can drop this now. */
3672	/* Start a subset here because pgmPoolTrackFlushGCPhysPTsSlow and
3673	pgmPoolTrackFlushGCPhysPTs will/may kill the pool otherwise. */
3674	uint32_t iPrevSubset = PGMRZDynMapPushAutoSubset(pVCpu);
3675	# endif
3676
3677	if (PGMPOOL_TD_GET_CREFS(u16) != PGMPOOL_TD_CREFS_PHYSEXT)
3678	{
3679	Assert(PGMPOOL_TD_GET_CREFS(u16) == 1);
3680	pgmPoolTrackFlushGCPhysPT(pVM,
3681	pPhysPage,
3682	fFlushPTEs,
3683	PGMPOOL_TD_GET_IDX(u16));
3684	}
3685	else if (u16 != PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED))
3686	pgmPoolTrackFlushGCPhysPTs(pVM, pPhysPage, fFlushPTEs, PGMPOOL_TD_GET_IDX(u16));
3687	else
3688	rc = pgmPoolTrackFlushGCPhysPTsSlow(pVM, pPhysPage);
3689	*pfFlushTLBs = true;
3690
3691	# if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) \|\| defined(IN_RC)
3692	PGMRZDynMapPopAutoSubset(pVCpu, iPrevSubset);
3693	# endif
3694	}
3695	}
3696
3697	if (rc == VINF_PGM_GCPHYS_ALIASED)
3698	{
3699	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
3700	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3701	rc = VINF_PGM_SYNC_CR3;
3702	}
3703	pgmUnlock(pVM);
3704	return rc;
3705	}
3706
3707
3708	/**
3709	* Scans all shadow page tables for mappings of a physical page.
3710	*
3711	* This may be slow, but it's most likely more efficient than cleaning
3712	* out the entire page pool / cache.
3713	*
3714	* @returns VBox status code.
3715	* @retval VINF_SUCCESS if all references has been successfully cleared.
3716	* @retval VINF_PGM_GCPHYS_ALIASED if we're better off with a CR3 sync and
3717	* a page pool cleaning.
3718	*
3719	* @param pVM The cross context VM structure.
3720	* @param pPhysPage The guest page in question.
3721	*/
3722	int pgmPoolTrackFlushGCPhysPTsSlow(PVM pVM, PPGMPAGE pPhysPage)
3723	{
3724	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3725	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTsSlow, s);
3726	LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: cUsedPages=%d cPresent=%d pPhysPage=%R[pgmpage]\n",
3727	pPool->cUsedPages, pPool->cPresent, pPhysPage));
3728
3729	/*
3730	* There is a limit to what makes sense.
3731	*/
3732	if ( pPool->cPresent > 1024
3733	&& pVM->cCpus == 1)
3734	{
3735	LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: giving up... (cPresent=%d)\n", pPool->cPresent));
3736	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
3737	return VINF_PGM_GCPHYS_ALIASED;
3738	}
3739
3740	/*
3741	* Iterate all the pages until we've encountered all that in use.
3742	* This is simple but not quite optimal solution.
3743	*/
3744	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P; /** @todo drop X86_PTE_P here as we always test if present separately, anyway. */
3745	const uint32_t u32 = u64; /** @todo move into the 32BIT_PT_xx case */
3746	unsigned cLeft = pPool->cUsedPages;
3747	unsigned iPage = pPool->cCurPages;
3748	while (--iPage >= PGMPOOL_IDX_FIRST)
3749	{
3750	PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
3751	if ( pPage->GCPhys != NIL_RTGCPHYS
3752	&& pPage->cPresent)
3753	{
3754	switch (pPage->enmKind)
3755	{
3756	/*
3757	* We only care about shadow page tables.
3758	*/
3759	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3760	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3761	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3762	{
3763	unsigned cPresent = pPage->cPresent;
3764	PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3765	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
3766	if (pPT->a[i].n.u1Present)
3767	{
3768	if ((pPT->a[i].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
3769	{
3770	//Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX32\n", iPage, i, pPT->a[i]));
3771	pPT->a[i].u = 0;
3772
3773	/* Update the counter as we're removing references. */
3774	Assert(pPage->cPresent);
3775	Assert(pPool->cPresent);
3776	pPage->cPresent--;
3777	pPool->cPresent--;
3778	}
3779	if (!--cPresent)
3780	break;
3781	}
3782	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3783	break;
3784	}
3785
3786	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3787	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3788	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3789	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3790	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3791	{
3792	unsigned cPresent = pPage->cPresent;
3793	PPGMSHWPTPAE pPT = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3794	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
3795	if (PGMSHWPTEPAE_IS_P(pPT->a[i]))
3796	{
3797	if ((PGMSHWPTEPAE_GET_U(pPT->a[i]) & (X86_PTE_PAE_PG_MASK \| X86_PTE_P)) == u64)
3798	{
3799	//Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX64\n", iPage, i, pPT->a[i]));
3800	PGMSHWPTEPAE_SET(pPT->a[i], 0); /// @todo why not atomic?
3801
3802	/* Update the counter as we're removing references. */
3803	Assert(pPage->cPresent);
3804	Assert(pPool->cPresent);
3805	pPage->cPresent--;
3806	pPool->cPresent--;
3807	}
3808	if (!--cPresent)
3809	break;
3810	}
3811	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3812	break;
3813	}
3814	#ifndef IN_RC
3815	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
3816	{
3817	unsigned cPresent = pPage->cPresent;
3818	PEPTPT pPT = (PEPTPT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3819	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
3820	if (pPT->a[i].n.u1Present)
3821	{
3822	if ((pPT->a[i].u & (EPT_PTE_PG_MASK \| X86_PTE_P)) == u64)
3823	{
3824	//Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX64\n", iPage, i, pPT->a[i]));
3825	pPT->a[i].u = 0;
3826
3827	/* Update the counter as we're removing references. */
3828	Assert(pPage->cPresent);
3829	Assert(pPool->cPresent);
3830	pPage->cPresent--;
3831	pPool->cPresent--;
3832	}
3833	if (!--cPresent)
3834	break;
3835	}
3836	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
3837	break;
3838	}
3839	#endif
3840	}
3841	if (!--cLeft)
3842	break;
3843	}
3844	}
3845
3846	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
3847	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
3848
3849	/*
3850	* There is a limit to what makes sense. The above search is very expensive, so force a pgm pool flush.
3851	*/
3852	if (pPool->cPresent > 1024)
3853	{
3854	LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: giving up... (cPresent=%d)\n", pPool->cPresent));
3855	return VINF_PGM_GCPHYS_ALIASED;
3856	}
3857
3858	return VINF_SUCCESS;
3859	}
3860
3861
3862	/**
3863	* Clears the user entry in a user table.
3864	*
3865	* This is used to remove all references to a page when flushing it.
3866	*/
3867	static void pgmPoolTrackClearPageUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PCPGMPOOLUSER pUser)
3868	{
3869	Assert(pUser->iUser != NIL_PGMPOOL_IDX);
3870	Assert(pUser->iUser < pPool->cCurPages);
3871	uint32_t iUserTable = pUser->iUserTable;
3872
3873	/*
3874	* Map the user page. Ignore references made by fictitious pages.
3875	*/
3876	PPGMPOOLPAGE pUserPage = &pPool->aPages[pUser->iUser];
3877	LogFlow(("pgmPoolTrackClearPageUser: clear %x in %s (%RGp) (flushing %s)\n", iUserTable, pgmPoolPoolKindToStr(pUserPage->enmKind), pUserPage->Core.Key, pgmPoolPoolKindToStr(pPage->enmKind)));
3878	union
3879	{
3880	uint64_t *pau64;
3881	uint32_t *pau32;
3882	} u;
3883	if (pUserPage->idx < PGMPOOL_IDX_FIRST)
3884	{
3885	Assert(!pUserPage->pvPageR3);
3886	return;
3887	}
3888	u.pau64 = (uint64_t *)PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pUserPage);
3889
3890
3891	/* Safety precaution in case we change the paging for other modes too in the future. */
3892	Assert(!pgmPoolIsPageLocked(pPage)); RT_NOREF_PV(pPage);
3893
3894	#ifdef VBOX_STRICT
3895	/*
3896	* Some sanity checks.
3897	*/
3898	switch (pUserPage->enmKind)
3899	{
3900	case PGMPOOLKIND_32BIT_PD:
3901	case PGMPOOLKIND_32BIT_PD_PHYS:
3902	Assert(iUserTable < X86_PG_ENTRIES);
3903	break;
3904	case PGMPOOLKIND_PAE_PDPT:
3905	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
3906	case PGMPOOLKIND_PAE_PDPT_PHYS:
3907	Assert(iUserTable < 4);
3908	Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
3909	break;
3910	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3911	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3912	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3913	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3914	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3915	case PGMPOOLKIND_PAE_PD_PHYS:
3916	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3917	break;
3918	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3919	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3920	Assert(!(u.pau64[iUserTable] & PGM_PDFLAGS_MAPPING));
3921	break;
3922	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3923	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3924	Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
3925	break;
3926	case PGMPOOLKIND_64BIT_PML4:
3927	Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
3928	/* GCPhys >> PAGE_SHIFT is the index here */
3929	break;
3930	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3931	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3932	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3933	break;
3934
3935	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3936	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3937	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3938	break;
3939
3940	case PGMPOOLKIND_ROOT_NESTED:
3941	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3942	break;
3943
3944	default:
3945	AssertMsgFailed(("enmKind=%d\n", pUserPage->enmKind));
3946	break;
3947	}
3948	#endif /* VBOX_STRICT */
3949
3950	/*
3951	* Clear the entry in the user page.
3952	*/
3953	switch (pUserPage->enmKind)
3954	{
3955	/* 32-bit entries */
3956	case PGMPOOLKIND_32BIT_PD:
3957	case PGMPOOLKIND_32BIT_PD_PHYS:
3958	ASMAtomicWriteU32(&u.pau32[iUserTable], 0);
3959	break;
3960
3961	/* 64-bit entries */
3962	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3963	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3964	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3965	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3966	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3967	#ifdef IN_RC
3968	/*
3969	* In 32 bits PAE mode we must invalidate the TLB when changing a
3970	* PDPT entry; the CPU fetches them only during cr3 load, so any
3971	* non-present PDPT will continue to cause page faults.
3972	*/
3973	ASMReloadCR3();
3974	#endif
3975	RT_FALL_THRU();
3976	case PGMPOOLKIND_PAE_PD_PHYS:
3977	case PGMPOOLKIND_PAE_PDPT_PHYS:
3978	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3979	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3980	case PGMPOOLKIND_64BIT_PML4:
3981	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3982	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3983	case PGMPOOLKIND_PAE_PDPT:
3984	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
3985	case PGMPOOLKIND_ROOT_NESTED:
3986	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3987	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3988	ASMAtomicWriteU64(&u.pau64[iUserTable], 0);
3989	break;
3990
3991	default:
3992	AssertFatalMsgFailed(("enmKind=%d iUser=%d iUserTable=%#x\n", pUserPage->enmKind, pUser->iUser, pUser->iUserTable));
3993	}
3994	PGM_DYNMAP_UNUSED_HINT_VM(pPool->CTX_SUFF(pVM), u.pau64);
3995	}
3996
3997
3998	/**
3999	* Clears all users of a page.
4000	*/
4001	static void pgmPoolTrackClearPageUsers(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4002	{
4003	/*
4004	* Free all the user records.
4005	*/
4006	LogFlow(("pgmPoolTrackClearPageUsers %RGp\n", pPage->GCPhys));
4007
4008	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
4009	uint16_t i = pPage->iUserHead;
4010	while (i != NIL_PGMPOOL_USER_INDEX)
4011	{
4012	/* Clear enter in user table. */
4013	pgmPoolTrackClearPageUser(pPool, pPage, &paUsers[i]);
4014
4015	/* Free it. */
4016	const uint16_t iNext = paUsers[i].iNext;
4017	paUsers[i].iUser = NIL_PGMPOOL_IDX;
4018	paUsers[i].iNext = pPool->iUserFreeHead;
4019	pPool->iUserFreeHead = i;
4020
4021	/* Next. */
4022	i = iNext;
4023	}
4024	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
4025	}
4026
4027
4028	/**
4029	* Allocates a new physical cross reference extent.
4030	*
4031	* @returns Pointer to the allocated extent on success. NULL if we're out of them.
4032	* @param pVM The cross context VM structure.
4033	* @param piPhysExt Where to store the phys ext index.
4034	*/
4035	PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVM pVM, uint16_t *piPhysExt)
4036	{
4037	PGM_LOCK_ASSERT_OWNER(pVM);
4038	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4039	uint16_t iPhysExt = pPool->iPhysExtFreeHead;
4040	if (iPhysExt == NIL_PGMPOOL_PHYSEXT_INDEX)
4041	{
4042	STAM_COUNTER_INC(&pPool->StamTrackPhysExtAllocFailures);
4043	return NULL;
4044	}
4045	PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
4046	pPool->iPhysExtFreeHead = pPhysExt->iNext;
4047	pPhysExt->iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
4048	*piPhysExt = iPhysExt;
4049	return pPhysExt;
4050	}
4051
4052
4053	/**
4054	* Frees a physical cross reference extent.
4055	*
4056	* @param pVM The cross context VM structure.
4057	* @param iPhysExt The extent to free.
4058	*/
4059	void pgmPoolTrackPhysExtFree(PVM pVM, uint16_t iPhysExt)
4060	{
4061	PGM_LOCK_ASSERT_OWNER(pVM);
4062	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4063	Assert(iPhysExt < pPool->cMaxPhysExts);
4064	PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
4065	for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
4066	{
4067	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
4068	pPhysExt->apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
4069	}
4070	pPhysExt->iNext = pPool->iPhysExtFreeHead;
4071	pPool->iPhysExtFreeHead = iPhysExt;
4072	}
4073
4074
4075	/**
4076	* Frees a physical cross reference extent.
4077	*
4078	* @param pVM The cross context VM structure.
4079	* @param iPhysExt The extent to free.
4080	*/
4081	void pgmPoolTrackPhysExtFreeList(PVM pVM, uint16_t iPhysExt)
4082	{
4083	PGM_LOCK_ASSERT_OWNER(pVM);
4084	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4085
4086	const uint16_t iPhysExtStart = iPhysExt;
4087	PPGMPOOLPHYSEXT pPhysExt;
4088	do
4089	{
4090	Assert(iPhysExt < pPool->cMaxPhysExts);
4091	pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
4092	for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
4093	{
4094	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
4095	pPhysExt->apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
4096	}
4097
4098	/* next */
4099	iPhysExt = pPhysExt->iNext;
4100	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
4101
4102	pPhysExt->iNext = pPool->iPhysExtFreeHead;
4103	pPool->iPhysExtFreeHead = iPhysExtStart;
4104	}
4105
4106
4107	/**
4108	* Insert a reference into a list of physical cross reference extents.
4109	*
4110	* @returns The new tracking data for PGMPAGE.
4111	*
4112	* @param pVM The cross context VM structure.
4113	* @param iPhysExt The physical extent index of the list head.
4114	* @param iShwPT The shadow page table index.
4115	* @param iPte Page table entry
4116	*
4117	*/
4118	static uint16_t pgmPoolTrackPhysExtInsert(PVM pVM, uint16_t iPhysExt, uint16_t iShwPT, uint16_t iPte)
4119	{
4120	PGM_LOCK_ASSERT_OWNER(pVM);
4121	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4122	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
4123
4124	/*
4125	* Special common cases.
4126	*/
4127	if (paPhysExts[iPhysExt].aidx[1] == NIL_PGMPOOL_IDX)
4128	{
4129	paPhysExts[iPhysExt].aidx[1] = iShwPT;
4130	paPhysExts[iPhysExt].apte[1] = iPte;
4131	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackAliasedMany);
4132	LogFlow(("pgmPoolTrackPhysExtInsert: %d:{,%d pte %d,}\n", iPhysExt, iShwPT, iPte));
4133	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
4134	}
4135	if (paPhysExts[iPhysExt].aidx[2] == NIL_PGMPOOL_IDX)
4136	{
4137	paPhysExts[iPhysExt].aidx[2] = iShwPT;
4138	paPhysExts[iPhysExt].apte[2] = iPte;
4139	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackAliasedMany);
4140	LogFlow(("pgmPoolTrackPhysExtInsert: %d:{,,%d pte %d}\n", iPhysExt, iShwPT, iPte));
4141	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
4142	}
4143	AssertCompile(RT_ELEMENTS(paPhysExts[iPhysExt].aidx) == 3);
4144
4145	/*
4146	* General treatment.
4147	*/
4148	const uint16_t iPhysExtStart = iPhysExt;
4149	unsigned cMax = 15;
4150	for (;;)
4151	{
4152	Assert(iPhysExt < pPool->cMaxPhysExts);
4153	for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
4154	if (paPhysExts[iPhysExt].aidx[i] == NIL_PGMPOOL_IDX)
4155	{
4156	paPhysExts[iPhysExt].aidx[i] = iShwPT;
4157	paPhysExts[iPhysExt].apte[i] = iPte;
4158	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackAliasedMany);
4159	LogFlow(("pgmPoolTrackPhysExtInsert: %d:{%d pte %d} i=%d cMax=%d\n", iPhysExt, iShwPT, iPte, i, cMax));
4160	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExtStart);
4161	}
4162	if (!--cMax)
4163	{
4164	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackOverflows);
4165	pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
4166	LogFlow(("pgmPoolTrackPhysExtInsert: overflow (1) iShwPT=%d\n", iShwPT));
4167	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
4168	}
4169
4170	/* advance */
4171	iPhysExt = paPhysExts[iPhysExt].iNext;
4172	if (iPhysExt == NIL_PGMPOOL_PHYSEXT_INDEX)
4173	break;
4174	}
4175
4176	/*
4177	* Add another extent to the list.
4178	*/
4179	PPGMPOOLPHYSEXT pNew = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
4180	if (!pNew)
4181	{
4182	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackNoExtentsLeft);
4183	pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
4184	LogFlow(("pgmPoolTrackPhysExtInsert: pgmPoolTrackPhysExtAlloc failed iShwPT=%d\n", iShwPT));
4185	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
4186	}
4187	pNew->iNext = iPhysExtStart;
4188	pNew->aidx[0] = iShwPT;
4189	pNew->apte[0] = iPte;
4190	LogFlow(("pgmPoolTrackPhysExtInsert: added new extent %d:{%d pte %d}->%d\n", iPhysExt, iShwPT, iPte, iPhysExtStart));
4191	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
4192	}
4193
4194
4195	/**
4196	* Add a reference to guest physical page where extents are in use.
4197	*
4198	* @returns The new tracking data for PGMPAGE.
4199	*
4200	* @param pVM The cross context VM structure.
4201	* @param pPhysPage Pointer to the aPages entry in the ram range.
4202	* @param u16 The ram range flags (top 16-bits).
4203	* @param iShwPT The shadow page table index.
4204	* @param iPte Page table entry
4205	*/
4206	uint16_t pgmPoolTrackPhysExtAddref(PVM pVM, PPGMPAGE pPhysPage, uint16_t u16, uint16_t iShwPT, uint16_t iPte)
4207	{
4208	pgmLock(pVM);
4209	if (PGMPOOL_TD_GET_CREFS(u16) != PGMPOOL_TD_CREFS_PHYSEXT)
4210	{
4211	/*
4212	* Convert to extent list.
4213	*/
4214	Assert(PGMPOOL_TD_GET_CREFS(u16) == 1);
4215	uint16_t iPhysExt;
4216	PPGMPOOLPHYSEXT pPhysExt = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
4217	if (pPhysExt)
4218	{
4219	LogFlow(("pgmPoolTrackPhysExtAddref: new extent: %d:{%d, %d}\n", iPhysExt, PGMPOOL_TD_GET_IDX(u16), iShwPT));
4220	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackAliased);
4221	pPhysExt->aidx[0] = PGMPOOL_TD_GET_IDX(u16);
4222	pPhysExt->apte[0] = PGM_PAGE_GET_PTE_INDEX(pPhysPage);
4223	pPhysExt->aidx[1] = iShwPT;
4224	pPhysExt->apte[1] = iPte;
4225	u16 = PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
4226	}
4227	else
4228	u16 = PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
4229	}
4230	else if (u16 != PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED))
4231	{
4232	/*
4233	* Insert into the extent list.
4234	*/
4235	u16 = pgmPoolTrackPhysExtInsert(pVM, PGMPOOL_TD_GET_IDX(u16), iShwPT, iPte);
4236	}
4237	else
4238	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackAliasedLots);
4239	pgmUnlock(pVM);
4240	return u16;
4241	}
4242
4243
4244	/**
4245	* Clear references to guest physical memory.
4246	*
4247	* @param pPool The pool.
4248	* @param pPage The page.
4249	* @param pPhysPage Pointer to the aPages entry in the ram range.
4250	* @param iPte Shadow PTE index
4251	*/
4252	void pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMPAGE pPhysPage, uint16_t iPte)
4253	{
4254	PVM pVM = pPool->CTX_SUFF(pVM);
4255	const unsigned cRefs = PGM_PAGE_GET_TD_CREFS(pPhysPage);
4256	AssertFatalMsg(cRefs == PGMPOOL_TD_CREFS_PHYSEXT, ("cRefs=%d pPhysPage=%R[pgmpage] pPage=%p:{.idx=%d}\n", cRefs, pPhysPage, pPage, pPage->idx));
4257
4258	uint16_t iPhysExt = PGM_PAGE_GET_TD_IDX(pPhysPage);
4259	if (iPhysExt != PGMPOOL_TD_IDX_OVERFLOWED)
4260	{
4261	pgmLock(pVM);
4262
4263	uint16_t iPhysExtPrev = NIL_PGMPOOL_PHYSEXT_INDEX;
4264	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
4265	do
4266	{
4267	Assert(iPhysExt < pPool->cMaxPhysExts);
4268
4269	/*
4270	* Look for the shadow page and check if it's all freed.
4271	*/
4272	for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
4273	{
4274	if ( paPhysExts[iPhysExt].aidx[i] == pPage->idx
4275	&& paPhysExts[iPhysExt].apte[i] == iPte)
4276	{
4277	paPhysExts[iPhysExt].aidx[i] = NIL_PGMPOOL_IDX;
4278	paPhysExts[iPhysExt].apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
4279
4280	for (i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
4281	if (paPhysExts[iPhysExt].aidx[i] != NIL_PGMPOOL_IDX)
4282	{
4283	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d\n", pPhysPage, pPage->idx));
4284	pgmUnlock(pVM);
4285	return;
4286	}
4287
4288	/* we can free the node. */
4289	const uint16_t iPhysExtNext = paPhysExts[iPhysExt].iNext;
4290	if ( iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX
4291	&& iPhysExtNext == NIL_PGMPOOL_PHYSEXT_INDEX)
4292	{
4293	/* lonely node */
4294	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
4295	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d lonely\n", pPhysPage, pPage->idx));
4296	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
4297	}
4298	else if (iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX)
4299	{
4300	/* head */
4301	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d head\n", pPhysPage, pPage->idx));
4302	PGM_PAGE_SET_TRACKING(pVM, pPhysPage, PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExtNext));
4303	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
4304	}
4305	else
4306	{
4307	/* in list */
4308	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d in list\n", pPhysPage, pPage->idx));
4309	paPhysExts[iPhysExtPrev].iNext = iPhysExtNext;
4310	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
4311	}
4312	iPhysExt = iPhysExtNext;
4313	pgmUnlock(pVM);
4314	return;
4315	}
4316	}
4317
4318	/* next */
4319	iPhysExtPrev = iPhysExt;
4320	iPhysExt = paPhysExts[iPhysExt].iNext;
4321	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
4322
4323	pgmUnlock(pVM);
4324	AssertFatalMsgFailed(("not-found! cRefs=%d pPhysPage=%R[pgmpage] pPage=%p:{.idx=%d}\n", cRefs, pPhysPage, pPage, pPage->idx));
4325	}
4326	else /* nothing to do */
4327	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage]\n", pPhysPage));
4328	}
4329
4330	/**
4331	* Clear references to guest physical memory.
4332	*
4333	* This is the same as pgmPoolTracDerefGCPhysHint except that the guest
4334	* physical address is assumed to be correct, so the linear search can be
4335	* skipped and we can assert at an earlier point.
4336	*
4337	* @param pPool The pool.
4338	* @param pPage The page.
4339	* @param HCPhys The host physical address corresponding to the guest page.
4340	* @param GCPhys The guest physical address corresponding to HCPhys.
4341	* @param iPte Shadow PTE index
4342	*/
4343	static void pgmPoolTracDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhys, uint16_t iPte)
4344	{
4345	/*
4346	* Lookup the page and check if it checks out before derefing it.
4347	*/
4348	PVM pVM = pPool->CTX_SUFF(pVM);
4349	PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhys);
4350	if (pPhysPage)
4351	{
4352	Assert(PGM_PAGE_GET_HCPHYS(pPhysPage));
4353	#ifdef LOG_ENABLED
4354	RTHCPHYS HCPhysPage = PGM_PAGE_GET_HCPHYS(pPhysPage);
4355	Log2(("pgmPoolTracDerefGCPhys %RHp vs %RHp\n", HCPhysPage, HCPhys));
4356	#endif
4357	if (PGM_PAGE_GET_HCPHYS(pPhysPage) == HCPhys)
4358	{
4359	Assert(pPage->cPresent);
4360	Assert(pPool->cPresent);
4361	pPage->cPresent--;
4362	pPool->cPresent--;
4363	pgmTrackDerefGCPhys(pPool, pPage, pPhysPage, iPte);
4364	return;
4365	}
4366
4367	AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp; found page has HCPhys=%RHp\n",
4368	HCPhys, GCPhys, PGM_PAGE_GET_HCPHYS(pPhysPage)));
4369	}
4370	AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp\n", HCPhys, GCPhys));
4371	}
4372
4373
4374	/**
4375	* Clear references to guest physical memory.
4376	*
4377	* @param pPool The pool.
4378	* @param pPage The page.
4379	* @param HCPhys The host physical address corresponding to the guest page.
4380	* @param GCPhysHint The guest physical address which may corresponding to HCPhys.
4381	* @param iPte Shadow pte index
4382	*/
4383	void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint, uint16_t iPte)
4384	{
4385	Log4(("pgmPoolTracDerefGCPhysHint %RHp %RGp\n", HCPhys, GCPhysHint));
4386
4387	/*
4388	* Try the hint first.
4389	*/
4390	RTHCPHYS HCPhysHinted;
4391	PVM pVM = pPool->CTX_SUFF(pVM);
4392	PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysHint);
4393	if (pPhysPage)
4394	{
4395	HCPhysHinted = PGM_PAGE_GET_HCPHYS(pPhysPage);
4396	Assert(HCPhysHinted);
4397	if (HCPhysHinted == HCPhys)
4398	{
4399	Assert(pPage->cPresent);
4400	Assert(pPool->cPresent);
4401	pPage->cPresent--;
4402	pPool->cPresent--;
4403	pgmTrackDerefGCPhys(pPool, pPage, pPhysPage, iPte);
4404	return;
4405	}
4406	}
4407	else
4408	HCPhysHinted = UINT64_C(0xdeadbeefdeadbeef);
4409
4410	/*
4411	* Damn, the hint didn't work. We'll have to do an expensive linear search.
4412	*/
4413	STAM_COUNTER_INC(&pPool->StatTrackLinearRamSearches);
4414	PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRangesX);
4415	while (pRam)
4416	{
4417	unsigned iPage = pRam->cb >> PAGE_SHIFT;
4418	while (iPage-- > 0)
4419	{
4420	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
4421	{
4422	Log4(("pgmPoolTracDerefGCPhysHint: Linear HCPhys=%RHp GCPhysHint=%RGp GCPhysReal=%RGp\n",
4423	HCPhys, GCPhysHint, pRam->GCPhys + (iPage << PAGE_SHIFT)));
4424	Assert(pPage->cPresent);
4425	Assert(pPool->cPresent);
4426	pPage->cPresent--;
4427	pPool->cPresent--;
4428	pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage], iPte);
4429	return;
4430	}
4431	}
4432	pRam = pRam->CTX_SUFF(pNext);
4433	}
4434
4435	AssertFatalMsgFailed(("HCPhys=%RHp GCPhysHint=%RGp (Hinted page has HCPhys = %RHp)\n", HCPhys, GCPhysHint, HCPhysHinted));
4436	}
4437
4438
4439	/**
4440	* Clear references to guest physical memory in a 32-bit / 32-bit page table.
4441	*
4442	* @param pPool The pool.
4443	* @param pPage The page.
4444	* @param pShwPT The shadow page table (mapping of the page).
4445	* @param pGstPT The guest page table.
4446	*/
4447	DECLINLINE(void) pgmPoolTrackDerefPT32Bit32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT, PCX86PT pGstPT)
4448	{
4449	RTGCPHYS32 const fPgMask = pPage->fA20Enabled ? X86_PTE_PG_MASK : X86_PTE_PG_MASK & ~RT_BIT_32(20);
4450	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
4451	{
4452	Assert(!(pShwPT->a[i].u & RT_BIT_32(10)));
4453	if (pShwPT->a[i].n.u1Present)
4454	{
4455	Log4(("pgmPoolTrackDerefPT32Bit32Bit: i=%d pte=%RX32 hint=%RX32\n",
4456	i, pShwPT->a[i].u & X86_PTE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK));
4457	pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PG_MASK, pGstPT->a[i].u & fPgMask, i);
4458	if (!pPage->cPresent)
4459	break;
4460	}
4461	}
4462	}
4463
4464
4465	/**
4466	* Clear references to guest physical memory in a PAE / 32-bit page table.
4467	*
4468	* @param pPool The pool.
4469	* @param pPage The page.
4470	* @param pShwPT The shadow page table (mapping of the page).
4471	* @param pGstPT The guest page table (just a half one).
4472	*/
4473	DECLINLINE(void) pgmPoolTrackDerefPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PT pGstPT)
4474	{
4475	RTGCPHYS32 const fPgMask = pPage->fA20Enabled ? X86_PTE_PG_MASK : X86_PTE_PG_MASK & ~RT_BIT_32(20);
4476	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
4477	{
4478	Assert( (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == 0
4479	\|\| (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == UINT64_C(0x7ff0000000000000));
4480	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
4481	{
4482	Log4(("pgmPoolTrackDerefPTPae32Bit: i=%d pte=%RX64 hint=%RX32\n",
4483	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & X86_PTE_PG_MASK));
4484	pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & fPgMask, i);
4485	if (!pPage->cPresent)
4486	break;
4487	}
4488	}
4489	}
4490
4491
4492	/**
4493	* Clear references to guest physical memory in a PAE / PAE page table.
4494	*
4495	* @param pPool The pool.
4496	* @param pPage The page.
4497	* @param pShwPT The shadow page table (mapping of the page).
4498	* @param pGstPT The guest page table.
4499	*/
4500	DECLINLINE(void) pgmPoolTrackDerefPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT)
4501	{
4502	RTGCPHYS const fPgMask = pPage->fA20Enabled ? X86_PTE_PAE_PG_MASK : X86_PTE_PAE_PG_MASK & ~RT_BIT_64(20);
4503	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
4504	{
4505	Assert( (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == 0
4506	\|\| (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == UINT64_C(0x7ff0000000000000));
4507	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
4508	{
4509	Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX32 hint=%RX32\n",
4510	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK));
4511	pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & fPgMask, i);
4512	if (!pPage->cPresent)
4513	break;
4514	}
4515	}
4516	}
4517
4518
4519	/**
4520	* Clear references to guest physical memory in a 32-bit / 4MB page table.
4521	*
4522	* @param pPool The pool.
4523	* @param pPage The page.
4524	* @param pShwPT The shadow page table (mapping of the page).
4525	*/
4526	DECLINLINE(void) pgmPoolTrackDerefPT32Bit4MB(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT)
4527	{
4528	RTGCPHYS const GCPhysA20Mask = pPage->fA20Enabled ? UINT64_MAX : ~RT_BIT_64(20);
4529	RTGCPHYS GCPhys = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
4530	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
4531	{
4532	Assert(!(pShwPT->a[i].u & RT_BIT_32(10)));
4533	if (pShwPT->a[i].n.u1Present)
4534	{
4535	Log4(("pgmPoolTrackDerefPT32Bit4MB: i=%d pte=%RX32 GCPhys=%RGp\n",
4536	i, pShwPT->a[i].u & X86_PTE_PG_MASK, GCPhys));
4537	pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & X86_PTE_PG_MASK, GCPhys & GCPhysA20Mask, i);
4538	if (!pPage->cPresent)
4539	break;
4540	}
4541	}
4542	}
4543
4544
4545	/**
4546	* Clear references to guest physical memory in a PAE / 2/4MB page table.
4547	*
4548	* @param pPool The pool.
4549	* @param pPage The page.
4550	* @param pShwPT The shadow page table (mapping of the page).
4551	*/
4552	DECLINLINE(void) pgmPoolTrackDerefPTPaeBig(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT)
4553	{
4554	RTGCPHYS const GCPhysA20Mask = pPage->fA20Enabled ? UINT64_MAX : ~RT_BIT_64(20);
4555	RTGCPHYS GCPhys = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
4556	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
4557	{
4558	Assert( (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == 0
4559	\|\| (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == UINT64_C(0x7ff0000000000000));
4560	if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
4561	{
4562	Log4(("pgmPoolTrackDerefPTPaeBig: i=%d pte=%RX64 hint=%RGp\n",
4563	i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), GCPhys));
4564	pgmPoolTracDerefGCPhys(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), GCPhys & GCPhysA20Mask, i);
4565	if (!pPage->cPresent)
4566	break;
4567	}
4568	}
4569	}
4570
4571
4572	/**
4573	* Clear references to shadowed pages in an EPT page table.
4574	*
4575	* @param pPool The pool.
4576	* @param pPage The page.
4577	* @param pShwPT The shadow page directory pointer table (mapping of the
4578	* page).
4579	*/
4580	DECLINLINE(void) pgmPoolTrackDerefPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPT pShwPT)
4581	{
4582	RTGCPHYS const GCPhysA20Mask = pPage->fA20Enabled ? UINT64_MAX : ~RT_BIT_64(20);
4583	RTGCPHYS GCPhys = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
4584	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
4585	{
4586	Assert((pShwPT->a[i].u & UINT64_C(0xfff0000000000f80)) == 0);
4587	if (pShwPT->a[i].n.u1Present)
4588	{
4589	Log4(("pgmPoolTrackDerefPTEPT: i=%d pte=%RX64 GCPhys=%RX64\n",
4590	i, pShwPT->a[i].u & EPT_PTE_PG_MASK, pPage->GCPhys));
4591	pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & EPT_PTE_PG_MASK, GCPhys & GCPhysA20Mask, i);
4592	if (!pPage->cPresent)
4593	break;
4594	}
4595	}
4596	}
4597
4598
4599	/**
4600	* Clear references to shadowed pages in a 32 bits page directory.
4601	*
4602	* @param pPool The pool.
4603	* @param pPage The page.
4604	* @param pShwPD The shadow page directory (mapping of the page).
4605	*/
4606	DECLINLINE(void) pgmPoolTrackDerefPD(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PD pShwPD)
4607	{
4608	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
4609	{
4610	if ( pShwPD->a[i].n.u1Present
4611	&& !(pShwPD->a[i].u & PGM_PDFLAGS_MAPPING)
4612	)
4613	{
4614	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & X86_PDE_PG_MASK);
4615	if (pSubPage)
4616	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4617	else
4618	AssertFatalMsgFailed(("%x\n", pShwPD->a[i].u & X86_PDE_PG_MASK));
4619	}
4620	}
4621	}
4622
4623
4624	/**
4625	* Clear references to shadowed pages in a PAE (legacy or 64 bits) page directory.
4626	*
4627	* @param pPool The pool.
4628	* @param pPage The page.
4629	* @param pShwPD The shadow page directory (mapping of the page).
4630	*/
4631	DECLINLINE(void) pgmPoolTrackDerefPDPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPAE pShwPD)
4632	{
4633	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
4634	{
4635	if ( pShwPD->a[i].n.u1Present
4636	&& !(pShwPD->a[i].u & PGM_PDFLAGS_MAPPING))
4637	{
4638	#ifdef PGM_WITH_LARGE_PAGES
4639	if (pShwPD->a[i].b.u1Size)
4640	{
4641	Log4(("pgmPoolTrackDerefPDPae: i=%d pde=%RX64 GCPhys=%RX64\n",
4642	i, pShwPD->a[i].u & X86_PDE2M_PAE_PG_MASK, pPage->GCPhys));
4643	pgmPoolTracDerefGCPhys(pPool, pPage, pShwPD->a[i].u & X86_PDE2M_PAE_PG_MASK,
4644	pPage->GCPhys + i * 2 * _1M /* pPage->GCPhys = base address of the memory described by the PD */,
4645	i);
4646	}
4647	else
4648	#endif
4649	{
4650	Assert((pShwPD->a[i].u & (X86_PDE_PAE_MBZ_MASK_NX \| UINT64_C(0x7ff0000000000000))) == 0);
4651	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & X86_PDE_PAE_PG_MASK);
4652	if (pSubPage)
4653	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4654	else
4655	AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & X86_PDE_PAE_PG_MASK));
4656	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4657	}
4658	}
4659	}
4660	}
4661
4662
4663	/**
4664	* Clear references to shadowed pages in a PAE page directory pointer table.
4665	*
4666	* @param pPool The pool.
4667	* @param pPage The page.
4668	* @param pShwPDPT The shadow page directory pointer table (mapping of the page).
4669	*/
4670	DECLINLINE(void) pgmPoolTrackDerefPDPTPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
4671	{
4672	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
4673	{
4674	Assert((pShwPDPT->a[i].u & (X86_PDPE_PAE_MBZ_MASK \| UINT64_C(0x7ff0000000000200))) == 0);
4675	if ( pShwPDPT->a[i].n.u1Present
4676	&& !(pShwPDPT->a[i].u & PGM_PLXFLAGS_MAPPING)
4677	)
4678	{
4679	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & X86_PDPE_PG_MASK);
4680	if (pSubPage)
4681	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4682	else
4683	AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & X86_PDPE_PG_MASK));
4684	}
4685	}
4686	}
4687
4688
4689	/**
4690	* Clear references to shadowed pages in a 64-bit page directory pointer table.
4691	*
4692	* @param pPool The pool.
4693	* @param pPage The page.
4694	* @param pShwPDPT The shadow page directory pointer table (mapping of the page).
4695	*/
4696	DECLINLINE(void) pgmPoolTrackDerefPDPT64Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
4697	{
4698	for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++)
4699	{
4700	Assert((pShwPDPT->a[i].u & (X86_PDPE_LM_MBZ_MASK_NX \| UINT64_C(0x7ff0000000000200))) == 0);
4701	if (pShwPDPT->a[i].n.u1Present)
4702	{
4703	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & X86_PDPE_PG_MASK);
4704	if (pSubPage)
4705	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4706	else
4707	AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & X86_PDPE_PG_MASK));
4708	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4709	}
4710	}
4711	}
4712
4713
4714	/**
4715	* Clear references to shadowed pages in a 64-bit level 4 page table.
4716	*
4717	* @param pPool The pool.
4718	* @param pPage The page.
4719	* @param pShwPML4 The shadow page directory pointer table (mapping of the page).
4720	*/
4721	DECLINLINE(void) pgmPoolTrackDerefPML464Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PML4 pShwPML4)
4722	{
4723	for (unsigned i = 0; i < RT_ELEMENTS(pShwPML4->a); i++)
4724	{
4725	Assert((pShwPML4->a[i].u & (X86_PML4E_MBZ_MASK_NX \| UINT64_C(0x7ff0000000000200))) == 0);
4726	if (pShwPML4->a[i].n.u1Present)
4727	{
4728	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPML4->a[i].u & X86_PDPE_PG_MASK);
4729	if (pSubPage)
4730	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4731	else
4732	AssertFatalMsgFailed(("%RX64\n", pShwPML4->a[i].u & X86_PML4E_PG_MASK));
4733	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4734	}
4735	}
4736	}
4737
4738
4739	/**
4740	* Clear references to shadowed pages in an EPT page directory.
4741	*
4742	* @param pPool The pool.
4743	* @param pPage The page.
4744	* @param pShwPD The shadow page directory (mapping of the page).
4745	*/
4746	DECLINLINE(void) pgmPoolTrackDerefPDEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPD pShwPD)
4747	{
4748	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
4749	{
4750	Assert((pShwPD->a[i].u & UINT64_C(0xfff0000000000f80)) == 0);
4751	if (pShwPD->a[i].n.u1Present)
4752	{
4753	#ifdef PGM_WITH_LARGE_PAGES
4754	if (pShwPD->a[i].b.u1Size)
4755	{
4756	Log4(("pgmPoolTrackDerefPDEPT: i=%d pde=%RX64 GCPhys=%RX64\n",
4757	i, pShwPD->a[i].u & X86_PDE2M_PAE_PG_MASK, pPage->GCPhys));
4758	pgmPoolTracDerefGCPhys(pPool, pPage, pShwPD->a[i].u & X86_PDE2M_PAE_PG_MASK,
4759	pPage->GCPhys + i * 2 * _1M /* pPage->GCPhys = base address of the memory described by the PD */,
4760	i);
4761	}
4762	else
4763	#endif
4764	{
4765	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & EPT_PDE_PG_MASK);
4766	if (pSubPage)
4767	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4768	else
4769	AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & EPT_PDE_PG_MASK));
4770	}
4771	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4772	}
4773	}
4774	}
4775
4776
4777	/**
4778	* Clear references to shadowed pages in an EPT page directory pointer table.
4779	*
4780	* @param pPool The pool.
4781	* @param pPage The page.
4782	* @param pShwPDPT The shadow page directory pointer table (mapping of the page).
4783	*/
4784	DECLINLINE(void) pgmPoolTrackDerefPDPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPDPT pShwPDPT)
4785	{
4786	for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++)
4787	{
4788	Assert((pShwPDPT->a[i].u & UINT64_C(0xfff0000000000f80)) == 0);
4789	if (pShwPDPT->a[i].n.u1Present)
4790	{
4791	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & EPT_PDPTE_PG_MASK);
4792	if (pSubPage)
4793	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4794	else
4795	AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & EPT_PDPTE_PG_MASK));
4796	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4797	}
4798	}
4799	}
4800
4801
4802	/**
4803	* Clears all references made by this page.
4804	*
4805	* This includes other shadow pages and GC physical addresses.
4806	*
4807	* @param pPool The pool.
4808	* @param pPage The page.
4809	*/
4810	static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4811	{
4812	/*
4813	* Map the shadow page and take action according to the page kind.
4814	*/
4815	PVM pVM = pPool->CTX_SUFF(pVM);
4816	void *pvShw = PGMPOOL_PAGE_2_PTR(pVM, pPage);
4817	switch (pPage->enmKind)
4818	{
4819	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
4820	{
4821	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4822	void *pvGst;
4823	int rc = PGM_GCPHYS_2_PTR(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
4824	pgmPoolTrackDerefPT32Bit32Bit(pPool, pPage, (PX86PT)pvShw, (PCX86PT)pvGst);
4825	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
4826	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4827	break;
4828	}
4829
4830	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
4831	{
4832	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4833	void *pvGst;
4834	int rc = PGM_GCPHYS_2_PTR_EX(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
4835	pgmPoolTrackDerefPTPae32Bit(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PT)pvGst);
4836	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
4837	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4838	break;
4839	}
4840
4841	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
4842	{
4843	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4844	void *pvGst;
4845	int rc = PGM_GCPHYS_2_PTR(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
4846	pgmPoolTrackDerefPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst);
4847	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
4848	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4849	break;
4850	}
4851
4852	case PGMPOOLKIND_32BIT_PT_FOR_PHYS: /* treat it like a 4 MB page */
4853	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
4854	{
4855	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4856	pgmPoolTrackDerefPT32Bit4MB(pPool, pPage, (PX86PT)pvShw);
4857	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4858	break;
4859	}
4860
4861	case PGMPOOLKIND_PAE_PT_FOR_PHYS: /* treat it like a 2 MB page */
4862	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
4863	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
4864	{
4865	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4866	pgmPoolTrackDerefPTPaeBig(pPool, pPage, (PPGMSHWPTPAE)pvShw);
4867	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4868	break;
4869	}
4870
4871	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
4872	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
4873	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
4874	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
4875	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
4876	case PGMPOOLKIND_PAE_PD_PHYS:
4877	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
4878	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
4879	pgmPoolTrackDerefPDPae(pPool, pPage, (PX86PDPAE)pvShw);
4880	break;
4881
4882	case PGMPOOLKIND_32BIT_PD_PHYS:
4883	case PGMPOOLKIND_32BIT_PD:
4884	pgmPoolTrackDerefPD(pPool, pPage, (PX86PD)pvShw);
4885	break;
4886
4887	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
4888	case PGMPOOLKIND_PAE_PDPT:
4889	case PGMPOOLKIND_PAE_PDPT_PHYS:
4890	pgmPoolTrackDerefPDPTPae(pPool, pPage, (PX86PDPT)pvShw);
4891	break;
4892
4893	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
4894	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
4895	pgmPoolTrackDerefPDPT64Bit(pPool, pPage, (PX86PDPT)pvShw);
4896	break;
4897
4898	case PGMPOOLKIND_64BIT_PML4:
4899	pgmPoolTrackDerefPML464Bit(pPool, pPage, (PX86PML4)pvShw);
4900	break;
4901
4902	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
4903	pgmPoolTrackDerefPTEPT(pPool, pPage, (PEPTPT)pvShw);
4904	break;
4905
4906	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
4907	pgmPoolTrackDerefPDEPT(pPool, pPage, (PEPTPD)pvShw);
4908	break;
4909
4910	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
4911	pgmPoolTrackDerefPDPTEPT(pPool, pPage, (PEPTPDPT)pvShw);
4912	break;
4913
4914	default:
4915	AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
4916	}
4917
4918	/* paranoia, clear the shadow page. Remove this laser (i.e. let Alloc and ClearAll do it). */
4919	STAM_PROFILE_START(&pPool->StatZeroPage, z);
4920	ASMMemZeroPage(pvShw);
4921	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
4922	pPage->fZeroed = true;
4923	Assert(!pPage->cPresent);
4924	PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
4925	}
4926
4927
4928	/**
4929	* Flushes a pool page.
4930	*
4931	* This moves the page to the free list after removing all user references to it.
4932	*
4933	* @returns VBox status code.
4934	* @retval VINF_SUCCESS on success.
4935	* @param pPool The pool.
4936	* @param pPage The shadow page.
4937	* @param fFlush Flush the TLBS when required (should only be false in very specific use cases!!)
4938	*/
4939	int pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fFlush)
4940	{
4941	PVM pVM = pPool->CTX_SUFF(pVM);
4942	bool fFlushRequired = false;
4943
4944	int rc = VINF_SUCCESS;
4945	STAM_PROFILE_START(&pPool->StatFlushPage, f);
4946	LogFlow(("pgmPoolFlushPage: pPage=%p:{.Key=%RHp, .idx=%d, .enmKind=%s, .GCPhys=%RGp}\n",
4947	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
4948
4949	/*
4950	* Reject any attempts at flushing any of the special root pages (shall
4951	* not happen).
4952	*/
4953	AssertMsgReturn(pPage->idx >= PGMPOOL_IDX_FIRST,
4954	("pgmPoolFlushPage: special root page, rejected. enmKind=%s idx=%d\n",
4955	pgmPoolPoolKindToStr(pPage->enmKind), pPage->idx),
4956	VINF_SUCCESS);
4957
4958	pgmLock(pVM);
4959
4960	/*
4961	* Quietly reject any attempts at flushing the currently active shadow CR3 mapping
4962	*/
4963	if (pgmPoolIsPageLocked(pPage))
4964	{
4965	AssertMsg( pPage->enmKind == PGMPOOLKIND_64BIT_PML4
4966	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PDPT
4967	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PDPT_FOR_32BIT
4968	\|\| pPage->enmKind == PGMPOOLKIND_32BIT_PD
4969	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD_FOR_PAE_PD
4970	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD
4971	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD
4972	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD
4973	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD
4974	\|\| pPage->enmKind == PGMPOOLKIND_ROOT_NESTED,
4975	("Can't free the shadow CR3! (%RHp vs %RHp kind=%d\n", PGMGetHyperCR3(VMMGetCpu(pVM)), pPage->Core.Key, pPage->enmKind));
4976	Log(("pgmPoolFlushPage: current active shadow CR3, rejected. enmKind=%s idx=%d\n", pgmPoolPoolKindToStr(pPage->enmKind), pPage->idx));
4977	pgmUnlock(pVM);
4978	return VINF_SUCCESS;
4979	}
4980
4981	#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) \|\| defined(IN_RC)
4982	/* Start a subset so we won't run out of mapping space. */
4983	PVMCPU pVCpu = VMMGetCpu(pVM);
4984	uint32_t iPrevSubset = PGMRZDynMapPushAutoSubset(pVCpu);
4985	#endif
4986
4987	/*
4988	* Mark the page as being in need of an ASMMemZeroPage().
4989	*/
4990	pPage->fZeroed = false;
4991
4992	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4993	if (pPage->fDirty)
4994	pgmPoolFlushDirtyPage(pVM, pPool, pPage->idxDirtyEntry, false /* do not remove */);
4995	#endif
4996
4997	/* If there are any users of this table, then we must issue a tlb flush on all VCPUs. */
4998	if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
4999	fFlushRequired = true;
5000
5001	/*
5002	* Clear the page.
5003	*/
5004	pgmPoolTrackClearPageUsers(pPool, pPage);
5005	STAM_PROFILE_START(&pPool->StatTrackDeref,a);
5006	pgmPoolTrackDeref(pPool, pPage);
5007	STAM_PROFILE_STOP(&pPool->StatTrackDeref,a);
5008
5009	/*
5010	* Flush it from the cache.
5011	*/
5012	pgmPoolCacheFlushPage(pPool, pPage);
5013
5014	#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) \|\| defined(IN_RC)
5015	/* Heavy stuff done. */
5016	PGMRZDynMapPopAutoSubset(pVCpu, iPrevSubset);
5017	#endif
5018
5019	/*
5020	* Deregistering the monitoring.
5021	*/
5022	if (pPage->fMonitored)
5023	rc = pgmPoolMonitorFlush(pPool, pPage);
5024
5025	/*
5026	* Free the page.
5027	*/
5028	Assert(pPage->iNext == NIL_PGMPOOL_IDX);
5029	pPage->iNext = pPool->iFreeHead;
5030	pPool->iFreeHead = pPage->idx;
5031	pPage->enmKind = PGMPOOLKIND_FREE;
5032	pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
5033	pPage->GCPhys = NIL_RTGCPHYS;
5034	pPage->fReusedFlushPending = false;
5035
5036	pPool->cUsedPages--;
5037
5038	/* Flush the TLBs of all VCPUs if required. */
5039	if ( fFlushRequired
5040	&& fFlush)
5041	{
5042	PGM_INVL_ALL_VCPU_TLBS(pVM);
5043	}
5044
5045	pgmUnlock(pVM);
5046	STAM_PROFILE_STOP(&pPool->StatFlushPage, f);
5047	return rc;
5048	}
5049
5050
5051	/**
5052	* Frees a usage of a pool page.
5053	*
5054	* The caller is responsible to updating the user table so that it no longer
5055	* references the shadow page.
5056	*
5057	* @param pPool The pool.
5058	* @param pPage The shadow page.
5059	* @param iUser The shadow page pool index of the user table.
5060	* NIL_PGMPOOL_IDX for root pages.
5061	* @param iUserTable The index into the user table (shadowed). Ignored if
5062	* root page.
5063	*/
5064	void pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
5065	{
5066	PVM pVM = pPool->CTX_SUFF(pVM);
5067
5068	STAM_PROFILE_START(&pPool->StatFree, a);
5069	LogFlow(("pgmPoolFreeByPage: pPage=%p:{.Key=%RHp, .idx=%d, enmKind=%s} iUser=%d iUserTable=%#x\n",
5070	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), iUser, iUserTable));
5071	AssertReturnVoid(pPage->idx >= PGMPOOL_IDX_FIRST); /* paranoia (#6349) */
5072
5073	pgmLock(pVM);
5074	if (iUser != NIL_PGMPOOL_IDX)
5075	pgmPoolTrackFreeUser(pPool, pPage, iUser, iUserTable);
5076	if (!pPage->fCached)
5077	pgmPoolFlushPage(pPool, pPage);
5078	pgmUnlock(pVM);
5079	STAM_PROFILE_STOP(&pPool->StatFree, a);
5080	}
5081
5082
5083	/**
5084	* Makes one or more free page free.
5085	*
5086	* @returns VBox status code.
5087	* @retval VINF_SUCCESS on success.
5088	*
5089	* @param pPool The pool.
5090	* @param enmKind Page table kind
5091	* @param iUser The user of the page.
5092	*/
5093	static int pgmPoolMakeMoreFreePages(PPGMPOOL pPool, PGMPOOLKIND enmKind, uint16_t iUser)
5094	{
5095	PVM pVM = pPool->CTX_SUFF(pVM);
5096	LogFlow(("pgmPoolMakeMoreFreePages: enmKind=%d iUser=%d\n", enmKind, iUser));
5097	NOREF(enmKind);
5098
5099	/*
5100	* If the pool isn't full grown yet, expand it.
5101	*/
5102	if ( pPool->cCurPages < pPool->cMaxPages
5103	#if defined(IN_RC)
5104	/* Hack alert: we can't deal with jumps to ring 3 when called from MapCR3 and allocating pages for PAE PDs. */
5105	&& enmKind != PGMPOOLKIND_PAE_PD_FOR_PAE_PD
5106	&& (enmKind < PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD \|\| enmKind > PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD)
5107	#endif
5108	)
5109	{
5110	STAM_PROFILE_ADV_SUSPEND(&pPool->StatAlloc, a);
5111	#ifdef IN_RING3
5112	int rc = PGMR3PoolGrow(pVM);
5113	#else
5114	int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_POOL_GROW, 0);
5115	#endif
5116	if (RT_FAILURE(rc))
5117	return rc;
5118	STAM_PROFILE_ADV_RESUME(&pPool->StatAlloc, a);
5119	if (pPool->iFreeHead != NIL_PGMPOOL_IDX)
5120	return VINF_SUCCESS;
5121	}
5122
5123	/*
5124	* Free one cached page.
5125	*/
5126	return pgmPoolCacheFreeOne(pPool, iUser);
5127	}
5128
5129
5130	/**
5131	* Allocates a page from the pool.
5132	*
5133	* This page may actually be a cached page and not in need of any processing
5134	* on the callers part.
5135	*
5136	* @returns VBox status code.
5137	* @retval VINF_SUCCESS if a NEW page was allocated.
5138	* @retval VINF_PGM_CACHED_PAGE if a CACHED page was returned.
5139	*
5140	* @param pVM The cross context VM structure.
5141	* @param GCPhys The GC physical address of the page we're gonna shadow.
5142	* For 4MB and 2MB PD entries, it's the first address the
5143	* shadow PT is covering.
5144	* @param enmKind The kind of mapping.
5145	* @param enmAccess Access type for the mapping (only relevant for big pages)
5146	* @param fA20Enabled Whether the A20 gate is enabled or not.
5147	* @param iUser The shadow page pool index of the user table. Root
5148	* pages should pass NIL_PGMPOOL_IDX.
5149	* @param iUserTable The index into the user table (shadowed). Ignored for
5150	* root pages (iUser == NIL_PGMPOOL_IDX).
5151	* @param fLockPage Lock the page
5152	* @param ppPage Where to store the pointer to the page. NULL is stored here on failure.
5153	*/
5154	int pgmPoolAlloc(PVM pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, bool fA20Enabled,
5155	uint16_t iUser, uint32_t iUserTable, bool fLockPage, PPPGMPOOLPAGE ppPage)
5156	{
5157	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5158	STAM_PROFILE_ADV_START(&pPool->StatAlloc, a);
5159	LogFlow(("pgmPoolAlloc: GCPhys=%RGp enmKind=%s iUser=%d iUserTable=%#x\n", GCPhys, pgmPoolPoolKindToStr(enmKind), iUser, iUserTable));
5160	*ppPage = NULL;
5161	/** @todo CSAM/PGMPrefetchPage messes up here during CSAMR3CheckGates
5162	* (TRPMR3SyncIDT) because of FF priority. Try fix that?
5163	* Assert(!(pVM->pgm.s.fGlobalSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)); */
5164
5165	pgmLock(pVM);
5166
5167	if (pPool->fCacheEnabled)
5168	{
5169	int rc2 = pgmPoolCacheAlloc(pPool, GCPhys, enmKind, enmAccess, fA20Enabled, iUser, iUserTable, ppPage);
5170	if (RT_SUCCESS(rc2))
5171	{
5172	if (fLockPage)
5173	pgmPoolLockPage(pPool, *ppPage);
5174	pgmUnlock(pVM);
5175	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
5176	LogFlow(("pgmPoolAlloc: cached returns %Rrc ppPage=%p:{.Key=%RHp, .idx=%d}\n", rc2, ppPage, (ppPage)->Core.Key, (ppPage)->idx));
5177	return rc2;
5178	}
5179	}
5180
5181	/*
5182	* Allocate a new one.
5183	*/
5184	int rc = VINF_SUCCESS;
5185	uint16_t iNew = pPool->iFreeHead;
5186	if (iNew == NIL_PGMPOOL_IDX)
5187	{
5188	rc = pgmPoolMakeMoreFreePages(pPool, enmKind, iUser);
5189	if (RT_FAILURE(rc))
5190	{
5191	pgmUnlock(pVM);
5192	Log(("pgmPoolAlloc: returns %Rrc (Free)\n", rc));
5193	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
5194	return rc;
5195	}
5196	iNew = pPool->iFreeHead;
5197	AssertReleaseReturn(iNew != NIL_PGMPOOL_IDX, VERR_PGM_POOL_IPE);
5198	}
5199
5200	/* unlink the free head */
5201	PPGMPOOLPAGE pPage = &pPool->aPages[iNew];
5202	pPool->iFreeHead = pPage->iNext;
5203	pPage->iNext = NIL_PGMPOOL_IDX;
5204
5205	/*
5206	* Initialize it.
5207	*/
5208	pPool->cUsedPages++; /* physical handler registration / pgmPoolTrackFlushGCPhysPTsSlow requirement. */
5209	pPage->enmKind = enmKind;
5210	pPage->enmAccess = enmAccess;
5211	pPage->GCPhys = GCPhys;
5212	pPage->fA20Enabled = fA20Enabled;
5213	pPage->fSeenNonGlobal = false; /* Set this to 'true' to disable this feature. */
5214	pPage->fMonitored = false;
5215	pPage->fCached = false;
5216	pPage->fDirty = false;
5217	pPage->fReusedFlushPending = false;
5218	pPage->cModifications = 0;
5219	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
5220	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
5221	pPage->cPresent = 0;
5222	pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
5223	pPage->idxDirtyEntry = 0;
5224	pPage->GCPtrLastAccessHandlerFault = NIL_RTGCPTR;
5225	pPage->GCPtrLastAccessHandlerRip = NIL_RTGCPTR;
5226	pPage->cLastAccessHandler = 0;
5227	pPage->cLocked = 0;
5228	# ifdef VBOX_STRICT
5229	pPage->GCPtrDirtyFault = NIL_RTGCPTR;
5230	# endif
5231
5232	/*
5233	* Insert into the tracking and cache. If this fails, free the page.
5234	*/
5235	int rc3 = pgmPoolTrackInsert(pPool, pPage, GCPhys, iUser, iUserTable);
5236	if (RT_FAILURE(rc3))
5237	{
5238	pPool->cUsedPages--;
5239	pPage->enmKind = PGMPOOLKIND_FREE;
5240	pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
5241	pPage->GCPhys = NIL_RTGCPHYS;
5242	pPage->iNext = pPool->iFreeHead;
5243	pPool->iFreeHead = pPage->idx;
5244	pgmUnlock(pVM);
5245	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
5246	Log(("pgmPoolAlloc: returns %Rrc (Insert)\n", rc3));
5247	return rc3;
5248	}
5249
5250	/*
5251	* Commit the allocation, clear the page and return.
5252	*/
5253	#ifdef VBOX_WITH_STATISTICS
5254	if (pPool->cUsedPages > pPool->cUsedPagesHigh)
5255	pPool->cUsedPagesHigh = pPool->cUsedPages;
5256	#endif
5257
5258	if (!pPage->fZeroed)
5259	{
5260	STAM_PROFILE_START(&pPool->StatZeroPage, z);
5261	void *pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
5262	ASMMemZeroPage(pv);
5263	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
5264	}
5265
5266	*ppPage = pPage;
5267	if (fLockPage)
5268	pgmPoolLockPage(pPool, pPage);
5269	pgmUnlock(pVM);
5270	LogFlow(("pgmPoolAlloc: returns %Rrc *ppPage=%p:{.Key=%RHp, .idx=%d, .fCached=%RTbool, .fMonitored=%RTbool}\n",
5271	rc, pPage, pPage->Core.Key, pPage->idx, pPage->fCached, pPage->fMonitored));
5272	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
5273	return rc;
5274	}
5275
5276
5277	/**
5278	* Frees a usage of a pool page.
5279	*
5280	* @param pVM The cross context VM structure.
5281	* @param HCPhys The HC physical address of the shadow page.
5282	* @param iUser The shadow page pool index of the user table.
5283	* NIL_PGMPOOL_IDX if root page.
5284	* @param iUserTable The index into the user table (shadowed). Ignored if
5285	* root page.
5286	*/
5287	void pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint32_t iUserTable)
5288	{
5289	LogFlow(("pgmPoolFree: HCPhys=%RHp iUser=%d iUserTable=%#x\n", HCPhys, iUser, iUserTable));
5290	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5291	pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, HCPhys), iUser, iUserTable);
5292	}
5293
5294
5295	/**
5296	* Internal worker for finding a 'in-use' shadow page give by it's physical address.
5297	*
5298	* @returns Pointer to the shadow page structure.
5299	* @param pPool The pool.
5300	* @param HCPhys The HC physical address of the shadow page.
5301	*/
5302	PPGMPOOLPAGE pgmPoolGetPage(PPGMPOOL pPool, RTHCPHYS HCPhys)
5303	{
5304	PGM_LOCK_ASSERT_OWNER(pPool->CTX_SUFF(pVM));
5305
5306	/*
5307	* Look up the page.
5308	*/
5309	PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
5310
5311	AssertFatalMsg(pPage && pPage->enmKind != PGMPOOLKIND_FREE, ("HCPhys=%RHp pPage=%p idx=%d\n", HCPhys, pPage, (pPage) ? pPage->idx : 0));
5312	return pPage;
5313	}
5314
5315
5316	/**
5317	* Internal worker for finding a page for debugging purposes, no assertions.
5318	*
5319	* @returns Pointer to the shadow page structure. NULL on if not found.
5320	* @param pPool The pool.
5321	* @param HCPhys The HC physical address of the shadow page.
5322	*/
5323	PPGMPOOLPAGE pgmPoolQueryPageForDbg(PPGMPOOL pPool, RTHCPHYS HCPhys)
5324	{
5325	PGM_LOCK_ASSERT_OWNER(pPool->CTX_SUFF(pVM));
5326	return (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
5327	}
5328
5329	#ifdef IN_RING3 /* currently only used in ring 3; save some space in the R0 & GC modules (left it here as we might need it elsewhere later on) */
5330
5331	/**
5332	* Flush the specified page if present
5333	*
5334	* @param pVM The cross context VM structure.
5335	* @param GCPhys Guest physical address of the page to flush
5336	*/
5337	void pgmPoolFlushPageByGCPhys(PVM pVM, RTGCPHYS GCPhys)
5338	{
5339	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5340
5341	VM_ASSERT_EMT(pVM);
5342
5343	/*
5344	* Look up the GCPhys in the hash.
5345	*/
5346	GCPhys = GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
5347	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
5348	if (i == NIL_PGMPOOL_IDX)
5349	return;
5350
5351	do
5352	{
5353	PPGMPOOLPAGE pPage = &pPool->aPages[i];
5354	if (pPage->GCPhys - GCPhys < PAGE_SIZE)
5355	{
5356	switch (pPage->enmKind)
5357	{
5358	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
5359	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
5360	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
5361	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
5362	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
5363	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
5364	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
5365	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
5366	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
5367	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
5368	case PGMPOOLKIND_64BIT_PML4:
5369	case PGMPOOLKIND_32BIT_PD:
5370	case PGMPOOLKIND_PAE_PDPT:
5371	{
5372	Log(("PGMPoolFlushPage: found pgm pool pages for %RGp\n", GCPhys));
5373	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
5374	if (pPage->fDirty)
5375	STAM_COUNTER_INC(&pPool->StatForceFlushDirtyPage);
5376	else
5377	#endif
5378	STAM_COUNTER_INC(&pPool->StatForceFlushPage);
5379	Assert(!pgmPoolIsPageLocked(pPage));
5380	pgmPoolMonitorChainFlush(pPool, pPage);
5381	return;
5382	}
5383
5384	/* ignore, no monitoring. */
5385	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
5386	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
5387	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
5388	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
5389	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
5390	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
5391	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
5392	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
5393	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
5394	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
5395	case PGMPOOLKIND_ROOT_NESTED:
5396	case PGMPOOLKIND_PAE_PD_PHYS:
5397	case PGMPOOLKIND_PAE_PDPT_PHYS:
5398	case PGMPOOLKIND_32BIT_PD_PHYS:
5399	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
5400	break;
5401
5402	default:
5403	AssertFatalMsgFailed(("enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
5404	}
5405	}
5406
5407	/* next */
5408	i = pPage->iNext;
5409	} while (i != NIL_PGMPOOL_IDX);
5410	return;
5411	}
5412
5413	#endif /* IN_RING3 */
5414	#ifdef IN_RING3
5415
5416	/**
5417	* Reset CPU on hot plugging.
5418	*
5419	* @param pVM The cross context VM structure.
5420	* @param pVCpu The cross context virtual CPU structure.
5421	*/
5422	void pgmR3PoolResetUnpluggedCpu(PVM pVM, PVMCPU pVCpu)
5423	{
5424	pgmR3ExitShadowModeBeforePoolFlush(pVCpu);
5425
5426	pgmR3ReEnterShadowModeAfterPoolFlush(pVM, pVCpu);
5427	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
5428	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5429	}
5430
5431
5432	/**
5433	* Flushes the entire cache.
5434	*
5435	* It will assert a global CR3 flush (FF) and assumes the caller is aware of
5436	* this and execute this CR3 flush.
5437	*
5438	* @param pVM The cross context VM structure.
5439	*/
5440	void pgmR3PoolReset(PVM pVM)
5441	{
5442	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
5443
5444	PGM_LOCK_ASSERT_OWNER(pVM);
5445	STAM_PROFILE_START(&pPool->StatR3Reset, a);
5446	LogFlow(("pgmR3PoolReset:\n"));
5447
5448	/*
5449	* If there are no pages in the pool, there is nothing to do.
5450	*/
5451	if (pPool->cCurPages <= PGMPOOL_IDX_FIRST)
5452	{
5453	STAM_PROFILE_STOP(&pPool->StatR3Reset, a);
5454	return;
5455	}
5456
5457	/*
5458	* Exit the shadow mode since we're going to clear everything,
5459	* including the root page.
5460	*/
5461	for (VMCPUID i = 0; i < pVM->cCpus; i++)
5462	pgmR3ExitShadowModeBeforePoolFlush(&pVM->aCpus[i]);
5463
5464	/*
5465	* Nuke the free list and reinsert all pages into it.
5466	*/
5467	for (unsigned i = pPool->cCurPages - 1; i >= PGMPOOL_IDX_FIRST; i--)
5468	{
5469	PPGMPOOLPAGE pPage = &pPool->aPages[i];
5470
5471	Assert(pPage->Core.Key == MMPage2Phys(pVM, pPage->pvPageR3));
5472	if (pPage->fMonitored)
5473	pgmPoolMonitorFlush(pPool, pPage);
5474	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
5475	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
5476	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
5477	pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
5478	pPage->GCPhys = NIL_RTGCPHYS;
5479	pPage->enmKind = PGMPOOLKIND_FREE;
5480	pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
5481	Assert(pPage->idx == i);
5482	pPage->iNext = i + 1;
5483	pPage->fA20Enabled = true;
5484	pPage->fZeroed = false; /* This could probably be optimized, but better safe than sorry. */
5485	pPage->fSeenNonGlobal = false;
5486	pPage->fMonitored = false;
5487	pPage->fDirty = false;
5488	pPage->fCached = false;
5489	pPage->fReusedFlushPending = false;
5490	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
5491	pPage->cPresent = 0;
5492	pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
5493	pPage->cModifications = 0;
5494	pPage->iAgeNext = NIL_PGMPOOL_IDX;
5495	pPage->iAgePrev = NIL_PGMPOOL_IDX;
5496	pPage->idxDirtyEntry = 0;
5497	pPage->GCPtrLastAccessHandlerRip = NIL_RTGCPTR;
5498	pPage->GCPtrLastAccessHandlerFault = NIL_RTGCPTR;
5499	pPage->cLastAccessHandler = 0;
5500	pPage->cLocked = 0;
5501	#ifdef VBOX_STRICT
5502	pPage->GCPtrDirtyFault = NIL_RTGCPTR;
5503	#endif
5504	}
5505	pPool->aPages[pPool->cCurPages - 1].iNext = NIL_PGMPOOL_IDX;
5506	pPool->iFreeHead = PGMPOOL_IDX_FIRST;
5507	pPool->cUsedPages = 0;
5508
5509	/*
5510	* Zap and reinitialize the user records.
5511	*/
5512	pPool->cPresent = 0;
5513	pPool->iUserFreeHead = 0;
5514	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
5515	const unsigned cMaxUsers = pPool->cMaxUsers;
5516	for (unsigned i = 0; i < cMaxUsers; i++)
5517	{
5518	paUsers[i].iNext = i + 1;
5519	paUsers[i].iUser = NIL_PGMPOOL_IDX;
5520	paUsers[i].iUserTable = 0xfffffffe;
5521	}
5522	paUsers[cMaxUsers - 1].iNext = NIL_PGMPOOL_USER_INDEX;
5523
5524	/*
5525	* Clear all the GCPhys links and rebuild the phys ext free list.
5526	*/
5527	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
5528	pRam;
5529	pRam = pRam->CTX_SUFF(pNext))
5530	{
5531	unsigned iPage = pRam->cb >> PAGE_SHIFT;
5532	while (iPage-- > 0)
5533	PGM_PAGE_SET_TRACKING(pVM, &pRam->aPages[iPage], 0);
5534	}
5535
5536	pPool->iPhysExtFreeHead = 0;
5537	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
5538	const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
5539	for (unsigned i = 0; i < cMaxPhysExts; i++)
5540	{
5541	paPhysExts[i].iNext = i + 1;
5542	paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
5543	paPhysExts[i].apte[0] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
5544	paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
5545	paPhysExts[i].apte[1] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
5546	paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
5547	paPhysExts[i].apte[2] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
5548	}
5549	paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
5550
5551	/*
5552	* Just zap the modified list.
5553	*/
5554	pPool->cModifiedPages = 0;
5555	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
5556
5557	/*
5558	* Clear the GCPhys hash and the age list.
5559	*/
5560	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aiHash); i++)
5561	pPool->aiHash[i] = NIL_PGMPOOL_IDX;
5562	pPool->iAgeHead = NIL_PGMPOOL_IDX;
5563	pPool->iAgeTail = NIL_PGMPOOL_IDX;
5564
5565	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
5566	/* Clear all dirty pages. */
5567	pPool->idxFreeDirtyPage = 0;
5568	pPool->cDirtyPages = 0;
5569	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aidxDirtyPages); i++)
5570	pPool->aidxDirtyPages[i] = NIL_PGMPOOL_IDX;
5571	#endif
5572
5573	/*
5574	* Reinsert active pages into the hash and ensure monitoring chains are correct.
5575	*/
5576	for (VMCPUID i = 0; i < pVM->cCpus; i++)
5577	{
5578	/*
5579	* Re-enter the shadowing mode and assert Sync CR3 FF.
5580	*/
5581	PVMCPU pVCpu = &pVM->aCpus[i];
5582	pgmR3ReEnterShadowModeAfterPoolFlush(pVM, pVCpu);
5583	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
5584	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5585	}
5586
5587	STAM_PROFILE_STOP(&pPool->StatR3Reset, a);
5588	}
5589
5590	#endif /* IN_RING3 */
5591
5592	#if defined(LOG_ENABLED) \|\| defined(VBOX_STRICT)
5593	/**
5594	* Stringifies a PGMPOOLKIND value.
5595	*/
5596	static const char *pgmPoolPoolKindToStr(uint8_t enmKind)
5597	{
5598	switch ((PGMPOOLKIND)enmKind)
5599	{
5600	case PGMPOOLKIND_INVALID:
5601	return "PGMPOOLKIND_INVALID";
5602	case PGMPOOLKIND_FREE:
5603	return "PGMPOOLKIND_FREE";
5604	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
5605	return "PGMPOOLKIND_32BIT_PT_FOR_PHYS";
5606	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
5607	return "PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT";
5608	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
5609	return "PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB";
5610	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
5611	return "PGMPOOLKIND_PAE_PT_FOR_PHYS";
5612	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
5613	return "PGMPOOLKIND_PAE_PT_FOR_32BIT_PT";
5614	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
5615	return "PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB";
5616	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
5617	return "PGMPOOLKIND_PAE_PT_FOR_PAE_PT";
5618	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
5619	return "PGMPOOLKIND_PAE_PT_FOR_PAE_2MB";
5620	case PGMPOOLKIND_32BIT_PD:
5621	return "PGMPOOLKIND_32BIT_PD";
5622	case PGMPOOLKIND_32BIT_PD_PHYS:
5623	return "PGMPOOLKIND_32BIT_PD_PHYS";
5624	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
5625	return "PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD";
5626	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
5627	return "PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD";
5628	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
5629	return "PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD";
5630	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
5631	return "PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD";
5632	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
5633	return "PGMPOOLKIND_PAE_PD_FOR_PAE_PD";
5634	case PGMPOOLKIND_PAE_PD_PHYS:
5635	return "PGMPOOLKIND_PAE_PD_PHYS";
5636	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
5637	return "PGMPOOLKIND_PAE_PDPT_FOR_32BIT";
5638	case PGMPOOLKIND_PAE_PDPT:
5639	return "PGMPOOLKIND_PAE_PDPT";
5640	case PGMPOOLKIND_PAE_PDPT_PHYS:
5641	return "PGMPOOLKIND_PAE_PDPT_PHYS";
5642	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
5643	return "PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT";
5644	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
5645	return "PGMPOOLKIND_64BIT_PDPT_FOR_PHYS";
5646	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
5647	return "PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD";
5648	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
5649	return "PGMPOOLKIND_64BIT_PD_FOR_PHYS";
5650	case PGMPOOLKIND_64BIT_PML4:
5651	return "PGMPOOLKIND_64BIT_PML4";
5652	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
5653	return "PGMPOOLKIND_EPT_PDPT_FOR_PHYS";
5654	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
5655	return "PGMPOOLKIND_EPT_PD_FOR_PHYS";
5656	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
5657	return "PGMPOOLKIND_EPT_PT_FOR_PHYS";
5658	case PGMPOOLKIND_ROOT_NESTED:
5659	return "PGMPOOLKIND_ROOT_NESTED";
5660	}
5661	return "Unknown kind!";
5662	}
5663	#endif /* LOG_ENABLED \|\| VBOX_STRICT */
5664

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllPool.cpp@ 77807

Download in other formats: