PGMAllBth.h@ 8863

Last change on this file since 8863 was 8739, checked in by vboxsync, 17 years ago
Set the shadow PDE for real and protected mode without paging to read/write & user.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 162.8 KB

Line
1	/* $Id: PGMAllBth.h 8739 2008-05-09 11:21:20Z vboxsync $ */
2	/** @file
3	* VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4	*
5	* This file is a big challenge!
6	*/
7
8	/*
9	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
10	*
11	* This file is part of VirtualBox Open Source Edition (OSE), as
12	* available from http://www.virtualbox.org. This file is free software;
13	* you can redistribute it and/or modify it under the terms of the GNU
14	* General Public License (GPL) as published by the Free Software
15	* Foundation, in version 2 as it comes in the "COPYING" file of the
16	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
17	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
18	*
19	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
20	* Clara, CA 95054 USA or visit http://www.sun.com if you need
21	* additional information or have any questions.
22	*/
23
24	/*******************************************************************************
25	* Internal Functions *
26	*******************************************************************************/
27	__BEGIN_DECLS
28	PGM_BTH_DECL(int, Trap0eHandler)(PVM pVM, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault);
29	PGM_BTH_DECL(int, InvalidatePage)(PVM pVM, RTGCUINTPTR GCPtrPage);
30	PGM_BTH_DECL(int, SyncPage)(PVM pVM, GSTPDE PdeSrc, RTGCUINTPTR GCPtrPage, unsigned cPages, unsigned uErr);
31	PGM_BTH_DECL(int, CheckPageFault)(PVM pVM, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCUINTPTR GCPtrPage);
32	PGM_BTH_DECL(int, SyncPT)(PVM pVM, unsigned iPD, PGSTPD pPDSrc, RTGCUINTPTR GCPtrPage);
33	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVM pVM, RTGCUINTPTR Addr, unsigned fPage, unsigned uErr);
34	PGM_BTH_DECL(int, PrefetchPage)(PVM pVM, RTGCUINTPTR GCPtrPage);
35	PGM_BTH_DECL(int, SyncCR3)(PVM pVM, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
36	#ifdef VBOX_STRICT
37	PGM_BTH_DECL(unsigned, AssertCR3)(PVM pVM, uint64_t cr3, uint64_t cr4, RTGCUINTPTR GCPtr = 0, RTGCUINTPTR cb = ~(RTGCUINTPTR)0);
38	#endif
39	#ifdef PGMPOOL_WITH_USER_TRACKING
40	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVM pVM, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys);
41	#endif
42	__END_DECLS
43
44
45	/* Filter out some illegal combinations of guest and shadow paging, so we can remove redundant checks inside functions. */
46	#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE
47	# error "Invalid combination; PAE guest implies PAE shadow"
48	#endif
49
50	#if (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
51	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE)
52	# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
53	#endif
54
55	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) \
56	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE)
57	# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
58	#endif
59
60	#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64)
61	\|\| (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64)
62	# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
63	#endif
64
65	#ifdef IN_RING0 /* no mappings in VT-x and AMD-V mode */
66	# define PGM_WITHOUT_MAPPINGS
67	#endif
68
69	/**
70	* #PF Handler for raw-mode guest execution.
71	*
72	* @returns VBox status code (appropriate for trap handling and GC return).
73	* @param pVM VM Handle.
74	* @param uErr The trap error code.
75	* @param pRegFrame Trap register frame.
76	* @param pvFault The fault address.
77	*/
78	PGM_BTH_DECL(int, Trap0eHandler)(PVM pVM, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
79	{
80	#if PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64
81
82	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE != PGM_TYPE_PAE
83	/*
84	* Hide the instruction fetch trap indicator for now.
85	*/
86	/** @todo NXE will change this and we must fix NXE in the switcher too! */
87	if (uErr & X86_TRAP_PF_ID)
88	{
89	uErr &= ~X86_TRAP_PF_ID;
90	TRPMSetErrorCode(pVM, uErr);
91	}
92	# endif
93
94	/*
95	* Get PDs.
96	*/
97	int rc;
98	# if PGM_WITH_PAGING(PGM_GST_TYPE)
99	# if PGM_GST_TYPE == PGM_TYPE_32BIT
100	const unsigned iPDSrc = (RTGCUINTPTR)pvFault >> GST_PD_SHIFT;
101	PGSTPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
102
103	# elif PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64
104
105	# if PGM_GST_TYPE == PGM_TYPE_PAE
106	unsigned iPDSrc;
107	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, (RTGCUINTPTR)pvFault, &iPDSrc);
108
109	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
110	unsigned iPDSrc;
111	PX86PML4E pPml4e;
112	X86PDPE Pdpe;
113	PGSTPD pPDSrc;
114
115	pPDSrc = pgmGstGetLongModePDPtr(&pVM->pgm.s, pvFault, &pPml4e, &Pdpe, &iPDSrc);
116	Assert(pPml4e);
117	# endif
118	/* Quick check for a valid guest trap. */
119	if (!pPDSrc)
120	{
121	LogFlow(("Trap0eHandler: guest PDPTR not present CR3=%VGp\n", (CPUMGetGuestCR3(pVM) & X86_CR3_PAGE_MASK)));
122	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eGuestTrap; });
123	TRPMSetErrorCode(pVM, uErr);
124	return VINF_EM_RAW_GUEST_TRAP;
125	}
126	# endif
127	# else
128	PGSTPD pPDSrc = NULL;
129	const unsigned iPDSrc = 0;
130	# endif
131
132	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
133	const unsigned iPDDst = (RTGCUINTPTR)pvFault >> SHW_PD_SHIFT;
134	PX86PD pPDDst = pVM->pgm.s.CTXMID(p,32BitPD);
135	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
136	const unsigned iPDDst = (RTGCUINTPTR)pvFault >> SHW_PD_SHIFT;
137	PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]; /* We treat this as a PD with 2048 entries, so no need to and with SHW_PD_MASK to get iPDDst */
138
139	# if PGM_GST_TYPE == PGM_TYPE_PAE
140	/* Did we mark the PDPT as not present in SyncCR3? */
141	unsigned iPDPTE = ((RTGCUINTPTR)pvFault >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
142	if (!pVM->pgm.s.CTXMID(p,PaePDPT)->a[iPDPTE].n.u1Present)
143	pVM->pgm.s.CTXMID(p,PaePDPT)->a[iPDPTE].n.u1Present = 1;
144
145	# endif
146
147	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
148	const unsigned iPDDst = (((RTGCUINTPTR)pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
149	PX86PDPAE pPDDst;
150
151	rc = PGMShwGetLongModePDPtr(pVM, (RTGCUINTPTR)pvFault, &pPDDst);
152	if (rc != VINF_SUCCESS)
153	{
154	AssertMsg(rc == VINF_PGM_SYNC_CR3, ("Unexpected rc=%Vrc\n", rc));
155	return rc;
156	}
157	Assert(pPDDst);
158	# endif
159
160	# if PGM_WITH_PAGING(PGM_GST_TYPE)
161	/*
162	* If we successfully correct the write protection fault due to dirty bit
163	* tracking, or this page fault is a genuine one, then return immediately.
164	*/
165	STAM_PROFILE_START(&pVM->pgm.s.StatCheckPageFault, e);
166	rc = PGM_BTH_NAME(CheckPageFault)(pVM, uErr, &pPDDst->a[iPDDst], &pPDSrc->a[iPDSrc], (RTGCUINTPTR)pvFault);
167	STAM_PROFILE_STOP(&pVM->pgm.s.StatCheckPageFault, e);
168	if ( rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
169	\|\| rc == VINF_EM_RAW_GUEST_TRAP)
170	{
171	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution)
172	= rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? &pVM->pgm.s.StatTrap0eDirtyAndAccessedBits : &pVM->pgm.s.StatTrap0eGuestTrap; });
173	LogBird(("Trap0eHandler: returns %s\n", rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? "VINF_SUCCESS" : "VINF_EM_RAW_GUEST_TRAP"));
174	return rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? VINF_SUCCESS : rc;
175	}
176
177	STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0ePD[iPDSrc]);
178	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
179
180	/*
181	* A common case is the not-present error caused by lazy page table syncing.
182	*
183	* It is IMPORTANT that we weed out any access to non-present shadow PDEs here
184	* so we can safely assume that the shadow PT is present when calling SyncPage later.
185	*
186	* On failure, we ASSUME that SyncPT is out of memory or detected some kind
187	* of mapping conflict and defer to SyncCR3 in R3.
188	* (Again, we do NOT support access handlers for non-present guest pages.)
189	*
190	*/
191	# if PGM_WITH_PAGING(PGM_GST_TYPE)
192	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
193	# else
194	GSTPDE PdeSrc;
195	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
196	PdeSrc.n.u1Present = 1;
197	PdeSrc.n.u1Write = 1;
198	PdeSrc.n.u1Accessed = 1;
199	PdeSrc.n.u1User = 1;
200	# endif
201	if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
202	&& !pPDDst->a[iPDDst].n.u1Present
203	&& PdeSrc.n.u1Present
204	)
205
206	{
207	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eSyncPT; });
208	STAM_PROFILE_START(&pVM->pgm.s.StatLazySyncPT, f);
209	LogFlow(("=>SyncPT %04x = %08x\n", iPDSrc, PdeSrc.au32[0]));
210	rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, (RTGCUINTPTR)pvFault);
211	if (VBOX_SUCCESS(rc))
212	{
213	STAM_PROFILE_STOP(&pVM->pgm.s.StatLazySyncPT, f);
214	return rc;
215	}
216	Log(("SyncPT: %d failed!! rc=%d\n", iPDSrc, rc));
217	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
218	STAM_PROFILE_STOP(&pVM->pgm.s.StatLazySyncPT, f);
219	return VINF_PGM_SYNC_CR3;
220	}
221
222	# if PGM_WITH_PAGING(PGM_GST_TYPE)
223	/*
224	* Check if this address is within any of our mappings.
225	*
226	* This is very fast and it's gonna save us a bit of effort below and prevent
227	* us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
228	* (BTW, it's impossible to have physical access handlers in a mapping.)
229	*/
230	if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
231	{
232	STAM_PROFILE_START(&pVM->pgm.s.StatMapping, a);
233	PPGMMAPPING pMapping = CTXALLSUFF(pVM->pgm.s.pMappings);
234	for ( ; pMapping; pMapping = CTXALLSUFF(pMapping->pNext))
235	{
236	if ((RTGCUINTPTR)pvFault < (RTGCUINTPTR)pMapping->GCPtr)
237	break;
238	if ((RTGCUINTPTR)pvFault - (RTGCUINTPTR)pMapping->GCPtr < pMapping->cb)
239	{
240	/*
241	* The first thing we check is if we've got an undetected conflict.
242	*/
243	if (!pVM->pgm.s.fMappingsFixed)
244	{
245	unsigned iPT = pMapping->cb >> GST_PD_SHIFT;
246	while (iPT-- > 0)
247	if (pPDSrc->a[iPDSrc + iPT].n.u1Present)
248	{
249	STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0eConflicts);
250	Log(("Trap0e: Detected Conflict %VGv-%VGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
251	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
252	STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
253	return VINF_PGM_SYNC_CR3;
254	}
255	}
256
257	/*
258	* Check if the fault address is in a virtual page access handler range.
259	*/
260	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->HyperVirtHandlers, pvFault);
261	if ( pCur
262	&& (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb
263	&& uErr & X86_TRAP_PF_RW)
264	{
265	# ifdef IN_GC
266	STAM_PROFILE_START(&pCur->Stat, h);
267	rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr);
268	STAM_PROFILE_STOP(&pCur->Stat, h);
269	# else
270	AssertFailed();
271	rc = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
272	# endif
273	STAM_COUNTER_INC(&pVM->pgm.s.StatTrap0eMapHandler);
274	STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
275	return rc;
276	}
277
278	/*
279	* Pretend we're not here and let the guest handle the trap.
280	*/
281	TRPMSetErrorCode(pVM, uErr & ~X86_TRAP_PF_P);
282	STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0eMap);
283	LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
284	STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
285	return VINF_EM_RAW_GUEST_TRAP;
286	}
287	}
288	STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
289	} /* pgmAreMappingsEnabled(&pVM->pgm.s) */
290	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
291
292	/*
293	* Check if this fault address is flagged for special treatment,
294	* which means we'll have to figure out the physical address and
295	* check flags associated with it.
296	*
297	* ASSUME that we can limit any special access handling to pages
298	* in page tables which the guest believes to be present.
299	*/
300	if (PdeSrc.n.u1Present)
301	{
302	RTGCPHYS GCPhys = NIL_RTGCPHYS;
303
304	# if PGM_WITH_PAGING(PGM_GST_TYPE)
305	uint32_t cr4 = CPUMGetGuestCR4(pVM);
306	if ( PdeSrc.b.u1Size
307	&& (cr4 & X86_CR4_PSE))
308	GCPhys = (PdeSrc.u & GST_PDE_BIG_PG_MASK)
309	\| ((RTGCPHYS)pvFault & (GST_BIG_PAGE_OFFSET_MASK ^ PAGE_OFFSET_MASK));
310	else
311	{
312	PGSTPT pPTSrc;
313	rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
314	if (VBOX_SUCCESS(rc))
315	{
316	unsigned iPTESrc = ((RTGCUINTPTR)pvFault >> GST_PT_SHIFT) & GST_PT_MASK;
317	if (pPTSrc->a[iPTESrc].n.u1Present)
318	GCPhys = pPTSrc->a[iPTESrc].u & GST_PTE_PG_MASK;
319	}
320	}
321	# else
322	/* No paging so the fault address is the physical address */
323	GCPhys = (RTGCPHYS)((RTGCUINTPTR)pvFault & ~PAGE_OFFSET_MASK);
324	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
325
326	/*
327	* If we have a GC address we'll check if it has any flags set.
328	*/
329	if (GCPhys != NIL_RTGCPHYS)
330	{
331	STAM_PROFILE_START(&pVM->pgm.s.StatHandlers, b);
332
333	PPGMPAGE pPage;
334	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
335	if (VBOX_SUCCESS(rc))
336	{
337	if (PGM_PAGE_HAS_ANY_HANDLERS(pPage))
338	{
339	if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
340	{
341	/*
342	* Physical page access handler.
343	*/
344	const RTGCPHYS GCPhysFault = GCPhys \| ((RTGCUINTPTR)pvFault & PAGE_OFFSET_MASK);
345	PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->PhysHandlers, GCPhysFault);
346	if (pCur)
347	{
348	# ifdef PGM_SYNC_N_PAGES
349	/*
350	* If the region is write protected and we got a page not present fault, then sync
351	* the pages. If the fault was caused by a read, then restart the instruction.
352	* In case of write access continue to the GC write handler.
353	*
354	* ASSUMES that there is only one handler per page or that they have similar write properties.
355	*/
356	if ( pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
357	&& !(uErr & X86_TRAP_PF_P))
358	{
359	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
360	if ( VBOX_FAILURE(rc)
361	\|\| !(uErr & X86_TRAP_PF_RW)
362	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
363	{
364	AssertRC(rc);
365	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersOutOfSync);
366	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
367	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncHndPhys; });
368	return rc;
369	}
370	}
371	# endif
372
373	AssertMsg( pCur->enmType != PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
374	\|\| (pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE && (uErr & X86_TRAP_PF_RW)),
375	("Unexpected trap for physical handler: %08X (phys=%08x) HCPhys=%X uErr=%X, enum=%d\n", pvFault, GCPhys, pPage->HCPhys, uErr, pCur->enmType));
376
377	#if defined(IN_GC) \|\| defined(IN_RING0)
378	if (CTXALLSUFF(pCur->pfnHandler))
379	{
380	STAM_PROFILE_START(&pCur->Stat, h);
381	rc = pCur->CTXALLSUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, GCPhysFault, CTXALLSUFF(pCur->pvUser));
382	STAM_PROFILE_STOP(&pCur->Stat, h);
383	}
384	else
385	#endif
386	rc = VINF_EM_RAW_EMULATE_INSTR;
387	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersPhysical);
388	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
389	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndPhys; });
390	return rc;
391	}
392	}
393	# if PGM_WITH_PAGING(PGM_GST_TYPE)
394	else
395	{
396	# ifdef PGM_SYNC_N_PAGES
397	/*
398	* If the region is write protected and we got a page not present fault, then sync
399	* the pages. If the fault was caused by a read, then restart the instruction.
400	* In case of write access continue to the GC write handler.
401	*/
402	if ( PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) < PGM_PAGE_HNDL_PHYS_STATE_ALL
403	&& !(uErr & X86_TRAP_PF_P))
404	{
405	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
406	if ( VBOX_FAILURE(rc)
407	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
408	\|\| !(uErr & X86_TRAP_PF_RW))
409	{
410	AssertRC(rc);
411	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersOutOfSync);
412	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
413	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncHndVirt; });
414	return rc;
415	}
416	}
417	# endif
418	/*
419	* Ok, it's an virtual page access handler.
420	*
421	* Since it's faster to search by address, we'll do that first
422	* and then retry by GCPhys if that fails.
423	*/
424	/** @todo r=bird: perhaps we should consider looking up by physical address directly now? */
425	/** @note r=svl: true, but lookup on virtual address should remain as a fallback as phys & virt trees might be out of sync, because the
426	* page was changed without us noticing it (not-present -> present without invlpg or mov cr3, xxx)
427	*/
428	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->VirtHandlers, pvFault);
429	if (pCur)
430	{
431	AssertMsg(!((RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb)
432	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
433	\|\| !(uErr & X86_TRAP_PF_P)
434	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
435	("Unexpected trap for virtual handler: %VGv (phys=%VGp) HCPhys=%HGp uErr=%X, enum=%d\n", pvFault, GCPhys, pPage->HCPhys, uErr, pCur->enmType));
436
437	if ( (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb
438	&& ( uErr & X86_TRAP_PF_RW
439	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
440	{
441	# ifdef IN_GC
442	STAM_PROFILE_START(&pCur->Stat, h);
443	rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr);
444	STAM_PROFILE_STOP(&pCur->Stat, h);
445	# else
446	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
447	# endif
448	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersVirtual);
449	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
450	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndVirt; });
451	return rc;
452	}
453	/* Unhandled part of a monitored page */
454	}
455	else
456	{
457	/* Check by physical address. */
458	PPGMVIRTHANDLER pCur;
459	unsigned iPage;
460	rc = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys + ((RTGCUINTPTR)pvFault & PAGE_OFFSET_MASK),
461	&pCur, &iPage);
462	Assert(VBOX_SUCCESS(rc) \|\| !pCur);
463	if ( pCur
464	&& ( uErr & X86_TRAP_PF_RW
465	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
466	{
467	Assert((pCur->aPhysToVirt[iPage].Core.Key & X86_PTE_PAE_PG_MASK) == GCPhys);
468	# ifdef IN_GC
469	RTGCUINTPTR off = (iPage << PAGE_SHIFT) + ((RTGCUINTPTR)pvFault & PAGE_OFFSET_MASK) - ((RTGCUINTPTR)pCur->GCPtr & PAGE_OFFSET_MASK);
470	Assert(off < pCur->cb);
471	STAM_PROFILE_START(&pCur->Stat, h);
472	rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, off);
473	STAM_PROFILE_STOP(&pCur->Stat, h);
474	# else
475	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
476	# endif
477	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersVirtualByPhys);
478	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
479	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndVirt; });
480	return rc;
481	}
482	}
483	}
484	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
485
486	/*
487	* There is a handled area of the page, but this fault doesn't belong to it.
488	* We must emulate the instruction.
489	*
490	* To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
491	* we first check if this was a page-not-present fault for a page with only
492	* write access handlers. Restart the instruction if it wasn't a write access.
493	*/
494	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersUnhandled);
495
496	if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
497	&& !(uErr & X86_TRAP_PF_P))
498	{
499	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
500	if ( VBOX_FAILURE(rc)
501	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
502	\|\| !(uErr & X86_TRAP_PF_RW))
503	{
504	AssertRC(rc);
505	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersOutOfSync);
506	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
507	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncHndPhys; });
508	return rc;
509	}
510	}
511
512	/** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
513	* It's writing to an unhandled part of the LDT page several million times.
514	*/
515	rc = PGMInterpretInstruction(pVM, pRegFrame, pvFault);
516	LogFlow(("PGM: PGMInterpretInstruction -> rc=%d HCPhys=%RHp%s%s\n",
517	rc, pPage->HCPhys,
518	PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage) ? " phys" : "",
519	PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS(pPage) ? " virt" : ""));
520	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
521	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndUnhandled; });
522	return rc;
523	} /* if any kind of handler */
524
525	# if PGM_WITH_PAGING(PGM_GST_TYPE)
526	if (uErr & X86_TRAP_PF_P)
527	{
528	/*
529	* The page isn't marked, but it might still be monitored by a virtual page access handler.
530	* (ASSUMES no temporary disabling of virtual handlers.)
531	*/
532	/** @todo r=bird: Since the purpose is to catch out of sync pages with virtual handler(s) here,
533	* we should correct both the shadow page table and physical memory flags, and not only check for
534	* accesses within the handler region but for access to pages with virtual handlers. */
535	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->VirtHandlers, pvFault);
536	if (pCur)
537	{
538	AssertMsg( !((RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb)
539	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
540	\|\| !(uErr & X86_TRAP_PF_P)
541	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
542	("Unexpected trap for virtual handler: %08X (phys=%08x) HCPhys=%X uErr=%X, enum=%d\n", pvFault, GCPhys, pPage->HCPhys, uErr, pCur->enmType));
543
544	if ( (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb
545	&& ( uErr & X86_TRAP_PF_RW
546	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
547	{
548	# ifdef IN_GC
549	STAM_PROFILE_START(&pCur->Stat, h);
550	rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr);
551	STAM_PROFILE_STOP(&pCur->Stat, h);
552	# else
553	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
554	# endif
555	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersVirtualUnmarked);
556	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
557	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndVirt; });
558	return rc;
559	}
560	}
561	}
562	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
563	}
564	else
565	{
566	/* When the guest accesses invalid physical memory (e.g. probing of RAM or accessing a remapped MMIO range), then we'll fall
567	* back to the recompiler to emulate the instruction.
568	*/
569	LogFlow(("pgmPhysGetPageEx %VGp failed with %Vrc\n", GCPhys, rc));
570	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersInvalid);
571	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
572	return VINF_EM_RAW_EMULATE_INSTR;
573	}
574
575	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
576
577	# ifdef PGM_OUT_OF_SYNC_IN_GC
578	/*
579	* We are here only if page is present in Guest page tables and trap is not handled
580	* by our handlers.
581	* Check it for page out-of-sync situation.
582	*/
583	STAM_PROFILE_START(&pVM->pgm.s.StatOutOfSync, c);
584
585	if (!(uErr & X86_TRAP_PF_P))
586	{
587	/*
588	* Page is not present in our page tables.
589	* Try to sync it!
590	* BTW, fPageShw is invalid in this branch!
591	*/
592	if (uErr & X86_TRAP_PF_US)
593	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncUser);
594	else /* supervisor */
595	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncSupervisor);
596
597	# if defined(LOG_ENABLED) && !defined(IN_RING0)
598	RTGCPHYS GCPhys;
599	uint64_t fPageGst;
600	PGMGstGetPage(pVM, pvFault, &fPageGst, &GCPhys);
601	Log(("Page out of sync: %p eip=%08x PdeSrc.n.u1User=%d fPageGst=%08llx GCPhys=%VGp scan=%d\n",
602	pvFault, pRegFrame->eip, PdeSrc.n.u1User, fPageGst, GCPhys, CSAMDoesPageNeedScanning(pVM, (RTGCPTR)pRegFrame->eip)));
603	# endif /* LOG_ENABLED */
604
605	# if PGM_WITH_PAGING(PGM_GST_TYPE) && !defined(IN_RING0)
606	if (CPUMGetGuestCPL(pVM, pRegFrame) == 0)
607	{
608	uint64_t fPageGst;
609	rc = PGMGstGetPage(pVM, pvFault, &fPageGst, NULL);
610	if ( VBOX_SUCCESS(rc)
611	&& !(fPageGst & X86_PTE_US))
612	{
613	/* Note: can't check for X86_TRAP_ID bit, because that requires execute disable support on the CPU */
614	if ( pvFault == (RTGCPTR)pRegFrame->eip
615	\|\| (RTGCUINTPTR)pvFault - pRegFrame->eip < 8 /* instruction crossing a page boundary */
616	# ifdef CSAM_DETECT_NEW_CODE_PAGES
617	\|\| ( !PATMIsPatchGCAddr(pVM, (RTGCPTR)pRegFrame->eip)
618	&& CSAMDoesPageNeedScanning(pVM, (RTGCPTR)pRegFrame->eip)) /* any new code we encounter here */
619	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
620	)
621	{
622	LogFlow(("CSAMExecFault %VGv\n", pRegFrame->eip));
623	rc = CSAMExecFault(pVM, (RTGCPTR)pRegFrame->eip);
624	if (rc != VINF_SUCCESS)
625	{
626	/*
627	* CSAM needs to perform a job in ring 3.
628	*
629	* Sync the page before going to the host context; otherwise we'll end up in a loop if
630	* CSAM fails (e.g. instruction crosses a page boundary and the next page is not present)
631	*/
632	LogFlow(("CSAM ring 3 job\n"));
633	int rc2 = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, 1, uErr);
634	AssertRC(rc2);
635
636	STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
637	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eCSAM; });
638	return rc;
639	}
640	}
641	# ifdef CSAM_DETECT_NEW_CODE_PAGES
642	else
643	if ( uErr == X86_TRAP_PF_RW
644	&& pRegFrame->ecx >= 0x100 /* early check for movswd count */
645	&& pRegFrame->ecx < 0x10000
646	)
647	{
648	/* In case of a write to a non-present supervisor shadow page, we'll take special precautions
649	* to detect loading of new code pages.
650	*/
651
652	/*
653	* Decode the instruction.
654	*/
655	RTGCPTR PC;
656	rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &PC);
657	if (rc == VINF_SUCCESS)
658	{
659	DISCPUSTATE Cpu;
660	uint32_t cbOp;
661	rc = EMInterpretDisasOneEx(pVM, (RTGCUINTPTR)PC, pRegFrame, &Cpu, &cbOp);
662
663	/* For now we'll restrict this to rep movsw/d instructions */
664	if ( rc == VINF_SUCCESS
665	&& Cpu.pCurInstr->opcode == OP_MOVSWD
666	&& (Cpu.prefix & PREFIX_REP))
667	{
668	CSAMMarkPossibleCodePage(pVM, pvFault);
669	}
670	}
671	}
672	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
673
674	/*
675	* Mark this page as safe.
676	*/
677	/** @todo not correct for pages that contain both code and data!! */
678	Log2(("CSAMMarkPage %p; scanned=%d\n", pvFault, true));
679	CSAMMarkPage(pVM, pvFault, true);
680	}
681	}
682	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) && !defined(IN_RING0) */
683	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
684	if (VBOX_SUCCESS(rc))
685	{
686	/* The page was successfully synced, return to the guest. */
687	STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
688	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSync; });
689	return VINF_SUCCESS;
690	}
691	}
692	else
693	{
694	/*
695	* A side effect of not flushing global PDEs are out of sync pages due
696	* to physical monitored regions, that are no longer valid.
697	* Assume for now it only applies to the read/write flag
698	*/
699	if (VBOX_SUCCESS(rc) && (uErr & X86_TRAP_PF_RW))
700	{
701	if (uErr & X86_TRAP_PF_US)
702	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncUser);
703	else /* supervisor */
704	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncSupervisor);
705
706
707	/*
708	* Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the page is not present, which is not true in this case.
709	*/
710	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, 1, uErr);
711	if (VBOX_SUCCESS(rc))
712	{
713	/*
714	* Page was successfully synced, return to guest.
715	*/
716	# ifdef VBOX_STRICT
717	RTGCPHYS GCPhys;
718	uint64_t fPageGst;
719	rc = PGMGstGetPage(pVM, pvFault, &fPageGst, &GCPhys);
720	Assert(VBOX_SUCCESS(rc) && fPageGst & X86_PTE_RW);
721	LogFlow(("Obsolete physical monitor page out of sync %VGv - phys %VGp flags=%08llx\n", pvFault, GCPhys, (uint64_t)fPageGst));
722
723	uint64_t fPageShw;
724	rc = PGMShwGetPage(pVM, pvFault, &fPageShw, NULL);
725	AssertMsg(VBOX_SUCCESS(rc) && fPageShw & X86_PTE_RW, ("rc=%Vrc fPageShw=%VX64\n", rc, fPageShw));
726	# endif /* VBOX_STRICT */
727	STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
728	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncObsHnd; });
729	return VINF_SUCCESS;
730	}
731
732	/* Check to see if we need to emulate the instruction as X86_CR0_WP has been cleared. */
733	if ( CPUMGetGuestCPL(pVM, pRegFrame) == 0
734	&& ((CPUMGetGuestCR0(pVM) & (X86_CR0_WP\|X86_CR0_PG)) == X86_CR0_PG)
735	&& (uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RW \| X86_TRAP_PF_P))
736	{
737	uint64_t fPageGst;
738	rc = PGMGstGetPage(pVM, pvFault, &fPageGst, NULL);
739	if ( VBOX_SUCCESS(rc)
740	&& !(fPageGst & X86_PTE_RW))
741	{
742	rc = PGMInterpretInstruction(pVM, pRegFrame, pvFault);
743	if (VBOX_SUCCESS(rc))
744	STAM_COUNTER_INC(&pVM->pgm.s.StatTrap0eWPEmulGC);
745	else
746	STAM_COUNTER_INC(&pVM->pgm.s.StatTrap0eWPEmulR3);
747	return rc;
748	}
749	else
750	AssertMsgFailed(("Unexpected r/w page %x flag=%x\n", pvFault, (uint32_t)fPageGst));
751	}
752
753	}
754
755	# if PGM_WITH_PAGING(PGM_GST_TYPE)
756	# ifdef VBOX_STRICT
757	/*
758	* Check for VMM page flags vs. Guest page flags consistency.
759	* Currently only for debug purposes.
760	*/
761	if (VBOX_SUCCESS(rc))
762	{
763	/* Get guest page flags. */
764	uint64_t fPageGst;
765	rc = PGMGstGetPage(pVM, pvFault, &fPageGst, NULL);
766	if (VBOX_SUCCESS(rc))
767	{
768	uint64_t fPageShw;
769	rc = PGMShwGetPage(pVM, pvFault, &fPageShw, NULL);
770
771	/*
772	* Compare page flags.
773	* Note: we have AVL, A, D bits desynched.
774	*/
775	AssertMsg((fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)),
776	("Page flags mismatch! pvFault=%p GCPhys=%VGp fPageShw=%08llx fPageGst=%08llx\n", pvFault, GCPhys, fPageShw, fPageGst));
777	}
778	else
779	AssertMsgFailed(("PGMGstGetPage rc=%Vrc\n", rc));
780	}
781	else
782	AssertMsgFailed(("PGMGCGetPage rc=%Vrc\n", rc));
783	# endif /* VBOX_STRICT */
784	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
785	}
786	STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
787	# endif /* PGM_OUT_OF_SYNC_IN_GC */
788	}
789	else
790	{
791	/*
792	* Page not present in Guest OS or invalid page table address.
793	* This is potential virtual page access handler food.
794	*
795	* For the present we'll say that our access handlers don't
796	* work for this case - we've already discarded the page table
797	* not present case which is identical to this.
798	*
799	* When we perchance find we need this, we will probably have AVL
800	* trees (offset based) to operate on and we can measure their speed
801	* agains mapping a page table and probably rearrange this handling
802	* a bit. (Like, searching virtual ranges before checking the
803	* physical address.)
804	*/
805	}
806	}
807
808
809	# if PGM_WITH_PAGING(PGM_GST_TYPE)
810	/*
811	* Conclusion, this is a guest trap.
812	*/
813	LogFlow(("PGM: Unhandled #PF -> route trap to recompiler!\n"));
814	STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0eUnhandled);
815	return VINF_EM_RAW_GUEST_TRAP;
816	# else
817	/* present, but not a monitored page; perhaps the guest is probing physical memory */
818	return VINF_EM_RAW_EMULATE_INSTR;
819	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
820
821
822	#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
823
824	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
825	return VERR_INTERNAL_ERROR;
826	#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
827	}
828
829
830	/**
831	* Emulation of the invlpg instruction.
832	*
833	*
834	* @returns VBox status code.
835	*
836	* @param pVM VM handle.
837	* @param GCPtrPage Page to invalidate.
838	*
839	* @remark ASSUMES that the guest is updating before invalidating. This order
840	* isn't required by the CPU, so this is speculative and could cause
841	* trouble.
842	*
843	* @todo Flush page or page directory only if necessary!
844	* @todo Add a #define for simply invalidating the page.
845	*/
846	PGM_BTH_DECL(int, InvalidatePage)(PVM pVM, RTGCUINTPTR GCPtrPage)
847	{
848	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
849	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
850	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
851	int rc;
852
853	LogFlow(("InvalidatePage %x\n", GCPtrPage));
854	/*
855	* Get the shadow PD entry and skip out if this PD isn't present.
856	* (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
857	*/
858	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
859	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
860	PX86PDE pPdeDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst];
861	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
862	PX86PDEPAE pPdeDst = &pVM->pgm.s.CTXMID(ap,PaePDs[0])->a[iPDDst];
863	# else /* AMD64 */
864	/* PML4 */
865	const unsigned iPml4 = ((RTGCUINTPTR64)GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;
866	PX86PML4E pPml4eDst = &CTXMID(pVM->pgm.s.p,PaePML4)->a[iPml4];
867	if (!pPml4eDst->n.u1Present)
868	{
869	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePageSkipped));
870	return VINF_SUCCESS;
871	}
872
873	/* PDPT */
874	PX86PDPT pPDPT;
875	rc = PGM_HCPHYS_2_PTR(pVM, pPml4eDst->u & X86_PML4E_PG_MASK, &pPDPT);
876	if (VBOX_FAILURE(rc))
877	return rc;
878	const unsigned iPDPT = ((RTGCUINTPTR64)GCPtrPage >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
879	PX86PDPE pPdpeDst = &pPDPT->a[iPDPT];
880	if (!pPdpeDst->n.u1Present)
881	{
882	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePageSkipped));
883	return VINF_SUCCESS;
884	}
885
886	/* PD */
887	PX86PDPAE pPd;
888	rc = PGM_HCPHYS_2_PTR(pVM, pPdpeDst->u & X86_PDPE_PG_MASK, &pPd);
889	if (VBOX_FAILURE(rc))
890	return rc;
891	const unsigned iPd = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
892	PX86PDEPAE pPdeDst = &pPd->a[iPd];
893	# endif
894
895	const SHWPDE PdeDst = *pPdeDst;
896	if (!PdeDst.n.u1Present)
897	{
898	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePageSkipped));
899	return VINF_SUCCESS;
900	}
901
902	/*
903	* Get the guest PD entry and calc big page.
904	*/
905	# if PGM_GST_TYPE == PGM_TYPE_32BIT
906	PX86PD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
907	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
908	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
909	# else
910	unsigned iPDSrc;
911	# if PGM_GST_TYPE == PGM_TYPE_PAE
912	PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, GCPtrPage, &iPDSrc);
913	# else /* AMD64 */
914	PX86PML4E pPml4eSrc;
915	X86PDPE PdpeSrc;
916	PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(&pVM->pgm.s, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
917	# endif
918	GSTPDE PdeSrc;
919
920	if (pPDSrc)
921	PdeSrc = pPDSrc->a[iPDSrc];
922	else
923	PdeSrc.u = 0;
924	# endif
925
926	const uint32_t cr4 = CPUMGetGuestCR4(pVM);
927	const bool fIsBigPage = PdeSrc.b.u1Size && (cr4 & X86_CR4_PSE);
928
929	# ifdef IN_RING3
930	/*
931	* If a CR3 Sync is pending we may ignore the invalidate page operation
932	* depending on the kind of sync and if it's a global page or not.
933	* This doesn't make sense in GC/R0 so we'll skip it entirely there.
934	*/
935	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
936	if ( VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3)
937	\|\| ( VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3_NON_GLOBAL)
938	&& fIsBigPage
939	&& PdeSrc.b.u1Global
940	&& (cr4 & X86_CR4_PGE)
941	)
942	)
943	# else
944	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
945	# endif
946	{
947	STAM_COUNTER_INC(&pVM->pgm.s.StatHCInvalidatePageSkipped);
948	return VINF_SUCCESS;
949	}
950	# endif /* IN_RING3 */
951
952
953	# if PGM_GST_TYPE == PGM_TYPE_AMD64
954	Assert(pPml4eDst->n.u1Present && pPml4eDst->u & SHW_PDPT_MASK);
955	if (pPml4eSrc->n.u1Present)
956	{
957	if ( pPml4eSrc->n.u1User != pPml4eDst->n.u1User
958	\|\| (!pPml4eSrc->n.u1Write && pPml4eDst->n.u1Write))
959	{
960	/*
961	* Mark not present so we can resync the PML4E when it's used.
962	*/
963	LogFlow(("InvalidatePage: Out-of-sync PML4E at %VGp Pml4eSrc=%RX64 Pml4eDst=%RX64\n",
964	GCPtrPage, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
965	pgmPoolFree(pVM, pPml4eDst->u & X86_PML4E_PG_MASK, PGMPOOL_IDX_PML4, iPml4);
966	pPml4eDst->u = 0;
967	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDOutOfSync));
968	PGM_INVL_GUEST_TLBS();
969	}
970	else if (!pPml4eSrc->n.u1Accessed)
971	{
972	/*
973	* Mark not present so we can set the accessed bit.
974	*/
975	pgmPoolFree(pVM, pPml4eDst->u & X86_PML4E_PG_MASK, PGMPOOL_IDX_PML4, iPml4);
976	pPml4eDst->u = 0;
977	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNAs));
978	PGM_INVL_GUEST_TLBS();
979	}
980	}
981	else
982	{
983	pgmPoolFree(pVM, pPml4eDst->u & X86_PML4E_PG_MASK, PGMPOOL_IDX_PML4, iPml4);
984	pPml4eDst->u = 0;
985	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNPs));
986	PGM_INVL_PG(GCPtrPage);
987	return VINF_SUCCESS;
988	}
989
990	Assert(pPdpeDst->n.u1Present && pPdpeDst->u & SHW_PDPT_MASK);
991	if (PdpeSrc.n.u1Present)
992	{
993	if ( PdpeSrc.lm.u1User != pPdpeDst->lm.u1User
994	\|\| (!PdpeSrc.lm.u1Write && pPdpeDst->lm.u1Write))
995	{
996	/*
997	* Mark not present so we can resync the PML4E when it's used.
998	*/
999	LogFlow(("InvalidatePage: Out-of-sync PDPE at %VGp PdpeSrc=%RX64 PdpeDst=%RX64\n",
1000	GCPtrPage, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
1001	pgmPoolFree(pVM, pPdpeDst->u & SHW_PDPT_MASK, PGMPOOL_IDX_PML4, iPml4);
1002	pPdpeDst->u = 0;
1003	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDOutOfSync));
1004	PGM_INVL_GUEST_TLBS();
1005	}
1006	else if (!PdpeSrc.lm.u1Accessed)
1007	{
1008	/*
1009	* Mark not present so we can set the accessed bit.
1010	*/
1011	pgmPoolFree(pVM, pPdpeDst->u & SHW_PDPT_MASK, PGMPOOL_IDX_PML4, iPml4);
1012	pPdpeDst->u = 0;
1013	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNAs));
1014	PGM_INVL_GUEST_TLBS();
1015	}
1016	}
1017	else
1018	{
1019	pgmPoolFree(pVM, pPdpeDst->u & SHW_PDPT_MASK, PGMPOOL_IDX_PDPT, iPDDst);
1020	pPdpeDst->u = 0;
1021	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNPs));
1022	PGM_INVL_PG(GCPtrPage);
1023	return VINF_SUCCESS;
1024	}
1025	# endif
1026
1027	/*
1028	* Deal with the Guest PDE.
1029	*/
1030	rc = VINF_SUCCESS;
1031	if (PdeSrc.n.u1Present)
1032	{
1033	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
1034	{
1035	/*
1036	* Conflict - Let SyncPT deal with it to avoid duplicate code.
1037	*/
1038	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1039	Assert(PGMGetGuestMode(pVM) <= PGMMODE_32_BIT);
1040	rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, GCPtrPage);
1041	}
1042	else if ( PdeSrc.n.u1User != PdeDst.n.u1User
1043	\|\| (!PdeSrc.n.u1Write && PdeDst.n.u1Write))
1044	{
1045	/*
1046	* Mark not present so we can resync the PDE when it's used.
1047	*/
1048	LogFlow(("InvalidatePage: Out-of-sync at %VGp PdeSrc=%RX64 PdeDst=%RX64\n",
1049	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1050	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1051	pPdeDst->u = 0;
1052	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDOutOfSync));
1053	PGM_INVL_GUEST_TLBS();
1054	}
1055	else if (!PdeSrc.n.u1Accessed)
1056	{
1057	/*
1058	* Mark not present so we can set the accessed bit.
1059	*/
1060	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1061	pPdeDst->u = 0;
1062	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNAs));
1063	PGM_INVL_GUEST_TLBS();
1064	}
1065	else if (!fIsBigPage)
1066	{
1067	/*
1068	* 4KB - page.
1069	*/
1070	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1071	RTGCPHYS GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1072	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1073	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1074	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE/2);
1075	# endif
1076	if (pShwPage->GCPhys == GCPhys)
1077	{
1078	# if 0 /* likely cause of a major performance regression; must be SyncPageWorkerTrackDeref then */
1079	const unsigned iPTEDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1080	PSHWPT pPT = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1081	if (pPT->a[iPTEDst].n.u1Present)
1082	{
1083	# ifdef PGMPOOL_WITH_USER_TRACKING
1084	/* This is very unlikely with caching/monitoring enabled. */
1085	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPT->a[iPTEDst].u & SHW_PTE_PG_MASK);
1086	# endif
1087	pPT->a[iPTEDst].u = 0;
1088	}
1089	# else /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
1090	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, GCPtrPage, 1, 0);
1091	if (VBOX_SUCCESS(rc))
1092	rc = VINF_SUCCESS;
1093	# endif
1094	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePage4KBPages));
1095	PGM_INVL_PG(GCPtrPage);
1096	}
1097	else
1098	{
1099	/*
1100	* The page table address changed.
1101	*/
1102	LogFlow(("InvalidatePage: Out-of-sync at %VGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%VGp iPDDst=%#x\n",
1103	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
1104	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1105	pPdeDst->u = 0;
1106	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDOutOfSync));
1107	PGM_INVL_GUEST_TLBS();
1108	}
1109	}
1110	else
1111	{
1112	/*
1113	* 2/4MB - page.
1114	*/
1115	/* Before freeing the page, check if anything really changed. */
1116	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1117	RTGCPHYS GCPhys = PdeSrc.u & GST_PDE_BIG_PG_MASK;
1118	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1119	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1120	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1121	# endif
1122	if ( pShwPage->GCPhys == GCPhys
1123	&& pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1124	{
1125	/* ASSUMES a the given bits are identical for 4M and normal PDEs */
1126	/** @todo PAT */
1127	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
1128	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
1129	&& ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1130	\|\| (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1131	{
1132	LogFlow(("Skipping flush for big page containing %VGv (PD=%X .u=%VX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1133	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePage4MBPagesSkip));
1134	return VINF_SUCCESS;
1135	}
1136	}
1137
1138	/*
1139	* Ok, the page table is present and it's been changed in the guest.
1140	* If we're in host context, we'll just mark it as not present taking the lazy approach.
1141	* We could do this for some flushes in GC too, but we need an algorithm for
1142	* deciding which 4MB pages containing code likely to be executed very soon.
1143	*/
1144	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1145	pPdeDst->u = 0;
1146	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePage4MBPages));
1147	PGM_INVL_BIG_PG(GCPtrPage);
1148	}
1149	}
1150	else
1151	{
1152	/*
1153	* Page directory is not present, mark shadow PDE not present.
1154	*/
1155	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
1156	{
1157	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1158	pPdeDst->u = 0;
1159	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNPs));
1160	PGM_INVL_PG(GCPtrPage);
1161	}
1162	else
1163	{
1164	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1165	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDMappings));
1166	}
1167	}
1168
1169	return rc;
1170
1171	#else /* guest real and protected mode */
1172	/* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1173	return VINF_SUCCESS;
1174	#endif
1175	}
1176
1177
1178	#ifdef PGMPOOL_WITH_USER_TRACKING
1179	/**
1180	* Update the tracking of shadowed pages.
1181	*
1182	* @param pVM The VM handle.
1183	* @param pShwPage The shadow page.
1184	* @param HCPhys The physical page we is being dereferenced.
1185	*/
1186	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVM pVM, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys)
1187	{
1188	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1189	STAM_PROFILE_START(&pVM->pgm.s.StatTrackDeref, a);
1190	LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%VHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1191
1192	/** @todo If this turns out to be a bottle neck (very likely) two things can be done:
1193	* 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1194	* 2. write protect all shadowed pages. I.e. implement caching.
1195	*/
1196	/*
1197	* Find the guest address.
1198	*/
1199	for (PPGMRAMRANGE pRam = CTXALLSUFF(pVM->pgm.s.pRamRanges);
1200	pRam;
1201	pRam = CTXALLSUFF(pRam->pNext))
1202	{
1203	unsigned iPage = pRam->cb >> PAGE_SHIFT;
1204	while (iPage-- > 0)
1205	{
1206	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1207	{
1208	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
1209	pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage]);
1210	pShwPage->cPresent--;
1211	pPool->cPresent--;
1212	STAM_PROFILE_STOP(&pVM->pgm.s.StatTrackDeref, a);
1213	return;
1214	}
1215	}
1216	}
1217
1218	for (;;)
1219	AssertReleaseMsgFailed(("HCPhys=%VHp wasn't found!\n", HCPhys));
1220	# else /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1221	pShwPage->cPresent--;
1222	pVM->pgm.s.CTXSUFF(pPool)->cPresent--;
1223	# endif /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1224	}
1225
1226
1227	/**
1228	* Update the tracking of shadowed pages.
1229	*
1230	* @param pVM The VM handle.
1231	* @param pShwPage The shadow page.
1232	* @param u16 The top 16-bit of the pPage->HCPhys.
1233	* @param pPage Pointer to the guest page. this will be modified.
1234	* @param iPTDst The index into the shadow table.
1235	*/
1236	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVM pVM, PPGMPOOLPAGE pShwPage, uint16_t u16, PPGMPAGE pPage, const unsigned iPTDst)
1237	{
1238	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1239	/*
1240	* We're making certain assumptions about the placement of cRef and idx.
1241	*/
1242	Assert(MM_RAM_FLAGS_IDX_SHIFT == 48);
1243	Assert(MM_RAM_FLAGS_CREFS_SHIFT > MM_RAM_FLAGS_IDX_SHIFT);
1244
1245	/*
1246	* Just deal with the simple first time here.
1247	*/
1248	if (!u16)
1249	{
1250	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackVirgin);
1251	u16 = (1 << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT)) \| pShwPage->idx;
1252	}
1253	else
1254	u16 = pgmPoolTrackPhysExtAddref(pVM, u16, pShwPage->idx);
1255
1256	/* write back, trying to be clever... */
1257	Log2(("SyncPageWorkerTrackAddRef: u16=%#x pPage->HCPhys=%VHp->%VHp iPTDst=%#x\n",
1258	u16, pPage->HCPhys, (pPage->HCPhys & MM_RAM_FLAGS_NO_REFS_MASK) \| ((uint64_t)u16 << MM_RAM_FLAGS_CREFS_SHIFT), iPTDst));
1259	((uint16_t )&pPage->HCPhys + 3) = u16; /** @todo PAGE FLAGS */
1260	# endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
1261
1262	/* update statistics. */
1263	pVM->pgm.s.CTXSUFF(pPool)->cPresent++;
1264	pShwPage->cPresent++;
1265	if (pShwPage->iFirstPresent > iPTDst)
1266	pShwPage->iFirstPresent = iPTDst;
1267	}
1268	#endif /* PGMPOOL_WITH_USER_TRACKING */
1269
1270
1271	/**
1272	* Creates a 4K shadow page for a guest page.
1273	*
1274	* For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1275	* physical address. The PdeSrc argument only the flags are used. No page structured
1276	* will be mapped in this function.
1277	*
1278	* @param pVM VM handle.
1279	* @param pPteDst Destination page table entry.
1280	* @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1281	* Can safely assume that only the flags are being used.
1282	* @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1283	* @param pShwPage Pointer to the shadow page.
1284	* @param iPTDst The index into the shadow table.
1285	*
1286	* @remark Not used for 2/4MB pages!
1287	*/
1288	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorker)(PVM pVM, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1289	{
1290	if (PteSrc.n.u1Present)
1291	{
1292	/*
1293	* Find the ram range.
1294	*/
1295	PPGMPAGE pPage;
1296	int rc = pgmPhysGetPageEx(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK, &pPage);
1297	if (VBOX_SUCCESS(rc))
1298	{
1299	/** @todo investiage PWT, PCD and PAT. */
1300	/*
1301	* Make page table entry.
1302	*/
1303	const RTHCPHYS HCPhys = pPage->HCPhys; /** @todo FLAGS */
1304	SHWPTE PteDst;
1305	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1306	{
1307	/** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No. */
1308	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1309	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1310	\| (HCPhys & X86_PTE_PAE_PG_MASK);
1311	else
1312	{
1313	LogFlow(("SyncPageWorker: monitored page (%VGp) -> mark not present\n", HCPhys));
1314	PteDst.u = 0;
1315	}
1316	/** @todo count these two kinds. */
1317	}
1318	else
1319	{
1320	/*
1321	* If the page or page directory entry is not marked accessed,
1322	* we mark the page not present.
1323	*/
1324	if (!PteSrc.n.u1Accessed \|\| !PdeSrc.n.u1Accessed)
1325	{
1326	LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1327	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,AccessedPage));
1328	PteDst.u = 0;
1329	}
1330	else
1331	/*
1332	* If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1333	* when the page is modified.
1334	*/
1335	if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
1336	{
1337	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPage));
1338	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1339	\| (HCPhys & X86_PTE_PAE_PG_MASK)
1340	\| PGM_PTFLAGS_TRACK_DIRTY;
1341	}
1342	else
1343	{
1344	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageSkipped));
1345	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1346	\| (HCPhys & X86_PTE_PAE_PG_MASK);
1347	}
1348	}
1349
1350	#ifdef PGMPOOL_WITH_USER_TRACKING
1351	/*
1352	* Keep user track up to date.
1353	*/
1354	if (PteDst.n.u1Present)
1355	{
1356	if (!pPteDst->n.u1Present)
1357	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst);
1358	else if ((pPteDst->u & SHW_PTE_PG_MASK) != (PteDst.u & SHW_PTE_PG_MASK))
1359	{
1360	Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", (uint64_t)pPteDst->u, (uint64_t)PteDst.u));
1361	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1362	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst);
1363	}
1364	}
1365	else if (pPteDst->n.u1Present)
1366	{
1367	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1368	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1369	}
1370	#endif /* PGMPOOL_WITH_USER_TRACKING */
1371
1372	/*
1373	* Update statistics and commit the entry.
1374	*/
1375	if (!PteSrc.n.u1Global)
1376	pShwPage->fSeenNonGlobal = true;
1377	*pPteDst = PteDst;
1378	}
1379	/* else MMIO or invalid page, we must handle them manually in the #PF handler. */
1380	/** @todo count these. */
1381	}
1382	else
1383	{
1384	/*
1385	* Page not-present.
1386	*/
1387	LogFlow(("SyncPageWorker: page not present in Pte\n"));
1388	#ifdef PGMPOOL_WITH_USER_TRACKING
1389	/* Keep user track up to date. */
1390	if (pPteDst->n.u1Present)
1391	{
1392	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1393	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1394	}
1395	#endif /* PGMPOOL_WITH_USER_TRACKING */
1396	pPteDst->u = 0;
1397	/** @todo count these. */
1398	}
1399	}
1400
1401
1402	/**
1403	* Syncs a guest OS page.
1404	*
1405	* There are no conflicts at this point, neither is there any need for
1406	* page table allocations.
1407	*
1408	* @returns VBox status code.
1409	* @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1410	* @param pVM VM handle.
1411	* @param PdeSrc Page directory entry of the guest.
1412	* @param GCPtrPage Guest context page address.
1413	* @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1414	* @param uErr Fault error (X86_TRAP_PF_*).
1415	*/
1416	PGM_BTH_DECL(int, SyncPage)(PVM pVM, GSTPDE PdeSrc, RTGCUINTPTR GCPtrPage, unsigned cPages, unsigned uErr)
1417	{
1418	LogFlow(("SyncPage: GCPtrPage=%VGv cPages=%d uErr=%#x\n", GCPtrPage, cPages, uErr));
1419
1420	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
1421	\|\| PGM_GST_TYPE == PGM_TYPE_PAE
1422
1423	# if PGM_WITH_NX(PGM_GST_TYPE)
1424	bool fNoExecuteBitValid = !!(CPUMGetGuestEFER(pVM) & MSR_K6_EFER_NXE);
1425	# endif
1426
1427	/*
1428	* Assert preconditions.
1429	*/
1430	STAM_COUNTER_INC(&pVM->pgm.s.StatGCSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
1431	Assert(PdeSrc.n.u1Present);
1432	Assert(cPages);
1433
1434	/*
1435	* Get the shadow PDE, find the shadow page table in the pool.
1436	*/
1437	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
1438	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1439	X86PDE PdeDst = pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst];
1440	# else /* PAE */
1441	X86PDEPAE PdeDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst];
1442	# endif
1443	Assert(PdeDst.n.u1Present);
1444	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1445
1446	/*
1447	* Check that the page is present and that the shadow PDE isn't out of sync.
1448	*/
1449	const bool fBigPage = PdeSrc.b.u1Size && (CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
1450	RTGCPHYS GCPhys;
1451	if (!fBigPage)
1452	{
1453	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1454	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1455	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1456	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE/2);
1457	# endif
1458	}
1459	else
1460	{
1461	GCPhys = PdeSrc.u & GST_PDE_BIG_PG_MASK;
1462	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1463	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1464	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1465	# endif
1466	}
1467	if ( pShwPage->GCPhys == GCPhys
1468	&& PdeSrc.n.u1Present
1469	&& (PdeSrc.n.u1User == PdeDst.n.u1User)
1470	&& (PdeSrc.n.u1Write == PdeDst.n.u1Write \|\| !PdeDst.n.u1Write)
1471	# if PGM_WITH_NX(PGM_GST_TYPE)
1472	&& (!fNoExecuteBitValid \|\| PdeSrc.n.u1NoExecute == PdeDst.n.u1NoExecute)
1473	# endif
1474	)
1475	{
1476	/*
1477	* Check that the PDE is marked accessed already.
1478	* Since we set the accessed bit before getting here on a #PF, this
1479	* check is only meant for dealing with non-#PF'ing paths.
1480	*/
1481	if (PdeSrc.n.u1Accessed)
1482	{
1483	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1484	if (!fBigPage)
1485	{
1486	/*
1487	* 4KB Page - Map the guest page table.
1488	*/
1489	PGSTPT pPTSrc;
1490	int rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
1491	if (VBOX_SUCCESS(rc))
1492	{
1493	# ifdef PGM_SYNC_N_PAGES
1494	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
1495	if (cPages > 1 && !(uErr & X86_TRAP_PF_P))
1496	{
1497	/*
1498	* This code path is currently only taken when the caller is PGMTrap0eHandler
1499	* for non-present pages!
1500	*
1501	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1502	* deal with locality.
1503	*/
1504	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1505	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1506	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1507	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1508	# else
1509	const unsigned offPTSrc = 0;
1510	# endif
1511	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, ELEMENTS(pPTDst->a));
1512	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1513	iPTDst = 0;
1514	else
1515	iPTDst -= PGM_SYNC_NR_PAGES / 2;
1516	for (; iPTDst < iPTDstEnd; iPTDst++)
1517	{
1518	if (!pPTDst->a[iPTDst].n.u1Present)
1519	{
1520	GSTPTE PteSrc = pPTSrc->a[offPTSrc + iPTDst];
1521	RTGCUINTPTR GCPtrCurPage = ((RTGCUINTPTR)GCPtrPage & ~(RTGCUINTPTR)(GST_PT_MASK << GST_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
1522	NOREF(GCPtrCurPage);
1523	#ifndef IN_RING0
1524	/*
1525	* Assuming kernel code will be marked as supervisor - and not as user level
1526	* and executed using a conforming code selector - And marked as readonly.
1527	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
1528	*/
1529	PPGMPAGE pPage;
1530	if ( ((PdeSrc.u & PteSrc.u) & (X86_PTE_RW \| X86_PTE_US))
1531	\|\| iPTDst == ((GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK) /* always sync GCPtrPage */
1532	\|\| !CSAMDoesPageNeedScanning(pVM, (RTGCPTR)GCPtrCurPage)
1533	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
1534	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1535	)
1536	#endif /* else: CSAM not active */
1537	PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1538	Log2(("SyncPage: 4K+ %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1539	GCPtrCurPage, PteSrc.n.u1Present,
1540	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1541	PteSrc.n.u1User & PdeSrc.n.u1User,
1542	(uint64_t)PteSrc.u,
1543	(uint64_t)pPTDst->a[iPTDst].u,
1544	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1545	}
1546	}
1547	}
1548	else
1549	# endif /* PGM_SYNC_N_PAGES */
1550	{
1551	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1552	GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1553	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1554	PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1555	Log2(("SyncPage: 4K %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s\n",
1556	GCPtrPage, PteSrc.n.u1Present,
1557	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1558	PteSrc.n.u1User & PdeSrc.n.u1User,
1559	(uint64_t)PteSrc.u,
1560	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1561	}
1562	}
1563	else /* MMIO or invalid page: emulated in #PF handler. */
1564	{
1565	LogFlow(("PGM_GCPHYS_2_PTR %VGp failed with %Vrc\n", GCPhys, rc));
1566	Assert(!pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK].n.u1Present);
1567	}
1568	}
1569	else
1570	{
1571	/*
1572	* 4/2MB page - lazy syncing shadow 4K pages.
1573	* (There are many causes of getting here, it's no longer only CSAM.)
1574	*/
1575	/* Calculate the GC physical address of this 4KB shadow page. */
1576	RTGCPHYS GCPhys = (PdeSrc.u & GST_PDE_BIG_PG_MASK) \| ((RTGCUINTPTR)GCPtrPage & GST_BIG_PAGE_OFFSET_MASK);
1577	/* Find ram range. */
1578	PPGMPAGE pPage;
1579	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
1580	if (VBOX_SUCCESS(rc))
1581	{
1582	/*
1583	* Make shadow PTE entry.
1584	*/
1585	const RTHCPHYS HCPhys = pPage->HCPhys; /** @todo PAGE FLAGS */
1586	SHWPTE PteDst;
1587	PteDst.u = (PdeSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1588	\| (HCPhys & X86_PTE_PAE_PG_MASK);
1589	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1590	{
1591	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1592	PteDst.n.u1Write = 0;
1593	else
1594	PteDst.u = 0;
1595	}
1596	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1597	# ifdef PGMPOOL_WITH_USER_TRACKING
1598	if (PteDst.n.u1Present && !pPTDst->a[iPTDst].n.u1Present)
1599	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst);
1600	# endif
1601	pPTDst->a[iPTDst] = PteDst;
1602
1603
1604	/*
1605	* If the page is not flagged as dirty and is writable, then make it read-only
1606	* at PD level, so we can set the dirty bit when the page is modified.
1607	*
1608	* ASSUMES that page access handlers are implemented on page table entry level.
1609	* Thus we will first catch the dirty access and set PDE.D and restart. If
1610	* there is an access handler, we'll trap again and let it work on the problem.
1611	*/
1612	/** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
1613	* As for invlpg, it simply frees the whole shadow PT.
1614	* ...It's possibly because the guest clears it and the guest doesn't really tell us... */
1615	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
1616	{
1617	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageBig));
1618	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
1619	PdeDst.n.u1Write = 0;
1620	}
1621	else
1622	{
1623	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
1624	PdeDst.n.u1Write = PdeSrc.n.u1Write;
1625	}
1626	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1627	pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst] = PdeDst;
1628	# else /* PAE */
1629	pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst] = PdeDst;
1630	# endif
1631	Log2(("SyncPage: BIG %VGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%VGp%s\n",
1632	GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
1633	PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1634	}
1635	else
1636	LogFlow(("PGM_GCPHYS_2_PTR %VGp (big) failed with %Vrc\n", GCPhys, rc));
1637	}
1638	return VINF_SUCCESS;
1639	}
1640	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncPagePDNAs));
1641	}
1642	else
1643	{
1644	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncPagePDOutOfSync));
1645	Log2(("SyncPage: Out-Of-Sync PDE at %VGp PdeSrc=%RX64 PdeDst=%RX64\n",
1646	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1647	}
1648
1649	/*
1650	* Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
1651	* Yea, I'm lazy.
1652	*/
1653	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1654	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1655	pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst].u = 0;
1656	# else /* PAE */
1657	pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst].u = 0;
1658	# endif
1659	PGM_INVL_GUEST_TLBS();
1660	return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
1661
1662	#elif PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT
1663
1664	# ifdef PGM_SYNC_N_PAGES
1665	/*
1666	* Get the shadow PDE, find the shadow page table in the pool.
1667	*/
1668	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
1669	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1670	X86PDE PdeDst = pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst];
1671	# else /* PAE */
1672	X86PDEPAE PdeDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst];
1673	# endif
1674	Assert(PdeDst.n.u1Present);
1675	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1676	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1677
1678	# if PGM_SHW_TYPE == PGM_TYPE_PAE
1679	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1680	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1681	# else
1682	const unsigned offPTSrc = 0;
1683	# endif
1684
1685	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
1686	if (cPages > 1 && !(uErr & X86_TRAP_PF_P))
1687	{
1688	/*
1689	* This code path is currently only taken when the caller is PGMTrap0eHandler
1690	* for non-present pages!
1691	*
1692	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1693	* deal with locality.
1694	*/
1695	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1696	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, ELEMENTS(pPTDst->a));
1697	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1698	iPTDst = 0;
1699	else
1700	iPTDst -= PGM_SYNC_NR_PAGES / 2;
1701	for (; iPTDst < iPTDstEnd; iPTDst++)
1702	{
1703	if (!pPTDst->a[iPTDst].n.u1Present)
1704	{
1705	GSTPTE PteSrc;
1706
1707	RTGCUINTPTR GCPtrCurPage = ((RTGCUINTPTR)GCPtrPage & ~(RTGCUINTPTR)(GST_PT_MASK << GST_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
1708
1709	/* Fake the page table entry */
1710	PteSrc.u = GCPtrCurPage;
1711	PteSrc.n.u1Present = 1;
1712	PteSrc.n.u1Dirty = 1;
1713	PteSrc.n.u1Accessed = 1;
1714	PteSrc.n.u1Write = 1;
1715	PteSrc.n.u1User = 1;
1716
1717	PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1718
1719	Log2(("SyncPage: 4K+ %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1720	GCPtrCurPage, PteSrc.n.u1Present,
1721	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1722	PteSrc.n.u1User & PdeSrc.n.u1User,
1723	(uint64_t)PteSrc.u,
1724	(uint64_t)pPTDst->a[iPTDst].u,
1725	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1726	}
1727	}
1728	}
1729	else
1730	# endif /* PGM_SYNC_N_PAGES */
1731	{
1732	GSTPTE PteSrc;
1733	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1734	RTGCUINTPTR GCPtrCurPage = ((RTGCUINTPTR)GCPtrPage & ~(RTGCUINTPTR)(GST_PT_MASK << GST_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
1735
1736	/* Fake the page table entry */
1737	PteSrc.u = GCPtrCurPage;
1738	PteSrc.n.u1Present = 1;
1739	PteSrc.n.u1Dirty = 1;
1740	PteSrc.n.u1Accessed = 1;
1741	PteSrc.n.u1Write = 1;
1742	PteSrc.n.u1User = 1;
1743	PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1744
1745	Log2(("SyncPage: 4K %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}PteDst=%08llx%s\n",
1746	GCPtrPage, PteSrc.n.u1Present,
1747	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1748	PteSrc.n.u1User & PdeSrc.n.u1User,
1749	(uint64_t)PteSrc.u,
1750	(uint64_t)pPTDst->a[iPTDst].u,
1751	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1752	}
1753	return VINF_SUCCESS;
1754
1755	#else /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
1756	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
1757	return VERR_INTERNAL_ERROR;
1758	#endif /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
1759	}
1760
1761
1762
1763	#if PGM_WITH_PAGING(PGM_GST_TYPE)
1764
1765	/**
1766	* Investigate page fault and handle write protection page faults caused by
1767	* dirty bit tracking.
1768	*
1769	* @returns VBox status code.
1770	* @param pVM VM handle.
1771	* @param uErr Page fault error code.
1772	* @param pPdeDst Shadow page directory entry.
1773	* @param pPdeSrc Guest page directory entry.
1774	* @param GCPtrPage Guest context page address.
1775	*/
1776	PGM_BTH_DECL(int, CheckPageFault)(PVM pVM, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCUINTPTR GCPtrPage)
1777	{
1778	bool fWriteProtect = !!(CPUMGetGuestCR0(pVM) & X86_CR0_WP);
1779	bool fUserLevelFault = !!(uErr & X86_TRAP_PF_US);
1780	bool fWriteFault = !!(uErr & X86_TRAP_PF_RW);
1781	bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
1782	# if PGM_WITH_NX(PGM_GST_TYPE)
1783	bool fNoExecuteBitValid = !!(CPUMGetGuestEFER(pVM) & MSR_K6_EFER_NXE);
1784	# endif
1785	unsigned uPageFaultLevel;
1786	int rc;
1787
1788	STAM_PROFILE_START(&pVM->pgm.s.CTXMID(Stat, DirtyBitTracking), a);
1789	LogFlow(("CheckPageFault: GCPtrPage=%VGv uErr=%#x PdeSrc=%08x\n", GCPtrPage, uErr, pPdeSrc->u));
1790
1791	# if PGM_GST_TYPE == PGM_TYPE_PAE \
1792	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
1793
1794	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1795	PX86PML4E pPml4eSrc;
1796	PX86PDPE pPdpeSrc;
1797
1798	pPdpeSrc = pgmGstGetLongModePDPTPtr(&pVM->pgm.s, GCPtrPage, &pPml4eSrc);
1799	Assert(pPml4eSrc);
1800
1801	/*
1802	* Real page fault? (PML4E level)
1803	*/
1804	if ( (uErr & X86_TRAP_PF_RSVD)
1805	\|\| !pPml4eSrc->n.u1Present
1806	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPml4eSrc->n.u1NoExecute)
1807	\|\| (fWriteFault && !pPml4eSrc->n.u1Write && (fUserLevelFault \|\| fWriteProtect))
1808	\|\| (fUserLevelFault && !pPml4eSrc->n.u1User)
1809	)
1810	{
1811	uPageFaultLevel = 0;
1812	goto UpperLevelPageFault;
1813	}
1814	Assert(pPdpeSrc);
1815
1816	# else /* PAE */
1817	PX86PDPE pPdpeSrc = &pVM->pgm.s.CTXSUFF(pGstPaePDPT)->a[(GCPtrPage >> GST_PDPT_SHIFT) & GST_PDPT_MASK];
1818	# endif
1819
1820	/*
1821	* Real page fault? (PDPE level)
1822	*/
1823	if ( (uErr & X86_TRAP_PF_RSVD)
1824	\|\| !pPdpeSrc->n.u1Present
1825	# if PGM_GST_TYPE == PGM_TYPE_AMD64 /* NX, r/w, u/s bits in the PDPE are long mode only */
1826	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPdpeSrc->lm.u1NoExecute)
1827	\|\| (fWriteFault && !pPdpeSrc->lm.u1Write && (fUserLevelFault \|\| fWriteProtect))
1828	\|\| (fUserLevelFault && !pPdpeSrc->lm.u1User)
1829	# endif
1830	)
1831	{
1832	uPageFaultLevel = 1;
1833	goto UpperLevelPageFault;
1834	}
1835	# endif
1836
1837	/*
1838	* Real page fault? (PDE level)
1839	*/
1840	if ( (uErr & X86_TRAP_PF_RSVD)
1841	\|\| !pPdeSrc->n.u1Present
1842	# if PGM_WITH_NX(PGM_GST_TYPE)
1843	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPdeSrc->n.u1NoExecute)
1844	# endif
1845	\|\| (fWriteFault && !pPdeSrc->n.u1Write && (fUserLevelFault \|\| fWriteProtect))
1846	\|\| (fUserLevelFault && !pPdeSrc->n.u1User) )
1847	{
1848	uPageFaultLevel = 2;
1849	goto UpperLevelPageFault;
1850	}
1851
1852	/*
1853	* First check the easy case where the page directory has been marked read-only to track
1854	* the dirty bit of an emulated BIG page
1855	*/
1856	if (pPdeSrc->b.u1Size && fBigPagesSupported)
1857	{
1858	/* Mark guest page directory as accessed */
1859	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1860	pPml4eSrc->n.u1Accessed = 1;
1861	pPdpeSrc->lm.u1Accessed = 1;
1862	# endif
1863	pPdeSrc->b.u1Accessed = 1;
1864
1865	/*
1866	* Only write protection page faults are relevant here.
1867	*/
1868	if (fWriteFault)
1869	{
1870	/* Mark guest page directory as dirty (BIG page only). */
1871	pPdeSrc->b.u1Dirty = 1;
1872
1873	if (pPdeDst->n.u1Present && (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY))
1874	{
1875	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageTrap));
1876
1877	Assert(pPdeSrc->b.u1Write);
1878
1879	pPdeDst->n.u1Write = 1;
1880	pPdeDst->n.u1Accessed = 1;
1881	pPdeDst->au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
1882	PGM_INVL_BIG_PG(GCPtrPage);
1883	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1884	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
1885	}
1886	}
1887	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1888	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
1889	}
1890	/* else: 4KB page table */
1891
1892	/*
1893	* Map the guest page table.
1894	*/
1895	PGSTPT pPTSrc;
1896	rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
1897	if (VBOX_SUCCESS(rc))
1898	{
1899	/*
1900	* Real page fault?
1901	*/
1902	PGSTPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
1903	const GSTPTE PteSrc = *pPteSrc;
1904	if ( !PteSrc.n.u1Present
1905	# if PGM_WITH_NX(PGM_GST_TYPE)
1906	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && PteSrc.n.u1NoExecute)
1907	# endif
1908	\|\| (fWriteFault && !PteSrc.n.u1Write && (fUserLevelFault \|\| fWriteProtect))
1909	\|\| (fUserLevelFault && !PteSrc.n.u1User)
1910	)
1911	{
1912	# ifdef IN_GC
1913	STAM_COUNTER_INC(&pVM->pgm.s.StatGCDirtyTrackRealPF);
1914	# endif
1915	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1916	LogFlow(("CheckPageFault: real page fault at %VGv PteSrc.u=%08x (2)\n", GCPtrPage, PteSrc.u));
1917
1918	/* Check the present bit as the shadow tables can cause different error codes by being out of sync.
1919	* See the 2nd case above as well.
1920	*/
1921	if (pPdeSrc->n.u1Present && pPteSrc->n.u1Present)
1922	TRPMSetErrorCode(pVM, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
1923
1924	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1925	return VINF_EM_RAW_GUEST_TRAP;
1926	}
1927	LogFlow(("CheckPageFault: page fault at %VGv PteSrc.u=%08x\n", GCPtrPage, PteSrc.u));
1928
1929	/*
1930	* Set the accessed bits in the page directory and the page table.
1931	*/
1932	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1933	pPml4eSrc->n.u1Accessed = 1;
1934	pPdpeSrc->lm.u1Accessed = 1;
1935	# endif
1936	pPdeSrc->n.u1Accessed = 1;
1937	pPteSrc->n.u1Accessed = 1;
1938
1939	/*
1940	* Only write protection page faults are relevant here.
1941	*/
1942	if (fWriteFault)
1943	{
1944	/* Write access, so mark guest entry as dirty. */
1945	# if defined(IN_GC) && defined(VBOX_WITH_STATISTICS)
1946	if (!pPteSrc->n.u1Dirty)
1947	STAM_COUNTER_INC(&pVM->pgm.s.StatGCDirtiedPage);
1948	else
1949	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageAlreadyDirty);
1950	# endif
1951
1952	pPteSrc->n.u1Dirty = 1;
1953
1954	if (pPdeDst->n.u1Present)
1955	{
1956	/* Bail out here as pgmPoolGetPageByHCPhys will return NULL and we'll crash below.
1957	* Our individual shadow handlers will provide more information and force a fatal exit.
1958	*/
1959	if (MMHyperIsInsideArea(pVM, (RTGCPTR)GCPtrPage))
1960	{
1961	LogRel(("CheckPageFault: write to hypervisor region %VGv\n", GCPtrPage));
1962	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1963	return VINF_SUCCESS;
1964	}
1965
1966	/*
1967	* Map shadow page table.
1968	*/
1969	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, pPdeDst->u & SHW_PDE_PG_MASK);
1970	if (pShwPage)
1971	{
1972	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1973	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
1974	if ( pPteDst->n.u1Present /** @todo Optimize accessed bit emulation? */
1975	&& (pPteDst->u & PGM_PTFLAGS_TRACK_DIRTY))
1976	{
1977	LogFlow(("DIRTY page trap addr=%VGv\n", GCPtrPage));
1978	# ifdef VBOX_STRICT
1979	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, pPteSrc->u & GST_PTE_PG_MASK);
1980	if (pPage)
1981	AssertMsg(!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage),
1982	("Unexpected dirty bit tracking on monitored page %VGv (phys %VGp)!!!!!!\n", GCPtrPage, pPteSrc->u & X86_PTE_PAE_PG_MASK));
1983	# endif
1984	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageTrap));
1985
1986	Assert(pPteSrc->n.u1Write);
1987
1988	pPteDst->n.u1Write = 1;
1989	pPteDst->n.u1Dirty = 1;
1990	pPteDst->n.u1Accessed = 1;
1991	pPteDst->au32[0] &= ~PGM_PTFLAGS_TRACK_DIRTY;
1992	PGM_INVL_PG(GCPtrPage);
1993
1994	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1995	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
1996	}
1997	}
1998	else
1999	AssertMsgFailed(("pgmPoolGetPageByHCPhys %VGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
2000	}
2001	}
2002	/** @todo Optimize accessed bit emulation? */
2003	# ifdef VBOX_STRICT
2004	/*
2005	* Sanity check.
2006	*/
2007	else if ( !pPteSrc->n.u1Dirty
2008	&& (pPdeSrc->n.u1Write & pPteSrc->n.u1Write)
2009	&& pPdeDst->n.u1Present)
2010	{
2011	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, pPdeDst->u & SHW_PDE_PG_MASK);
2012	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2013	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2014	if ( pPteDst->n.u1Present
2015	&& pPteDst->n.u1Write)
2016	LogFlow(("Writable present page %VGv not marked for dirty bit tracking!!!\n", GCPtrPage));
2017	}
2018	# endif /* VBOX_STRICT */
2019	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
2020	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2021	}
2022	AssertRC(rc);
2023	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
2024	return rc;
2025
2026
2027	UpperLevelPageFault:
2028	/* Pagefault detected while checking the PML4E, PDPE or PDE.
2029	* Single exit handler to get rid of duplicate code paths.
2030	*/
2031	# ifdef IN_GC
2032	STAM_COUNTER_INC(&pVM->pgm.s.StatGCDirtyTrackRealPF);
2033	# endif
2034	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat, DirtyBitTracking), a);
2035	LogFlow(("CheckPageFault: real page fault at %VGv (%d)\n", GCPtrPage, uPageFaultLevel));
2036
2037	if (
2038	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2039	pPml4eSrc->n.u1Present &&
2040	# endif
2041	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
2042	pPdpeSrc->n.u1Present &&
2043	# endif
2044	pPdeSrc->n.u1Present)
2045	{
2046	/* Check the present bit as the shadow tables can cause different error codes by being out of sync. */
2047	if (pPdeSrc->b.u1Size && fBigPagesSupported)
2048	{
2049	TRPMSetErrorCode(pVM, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
2050	}
2051	else
2052	{
2053	/*
2054	* Map the guest page table.
2055	*/
2056	PGSTPT pPTSrc;
2057	rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
2058	if (VBOX_SUCCESS(rc))
2059	{
2060	PGSTPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2061	const GSTPTE PteSrc = *pPteSrc;
2062	if (pPteSrc->n.u1Present)
2063	TRPMSetErrorCode(pVM, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
2064	}
2065	AssertRC(rc);
2066	}
2067	}
2068	return VINF_EM_RAW_GUEST_TRAP;
2069	}
2070
2071	#endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
2072
2073
2074	/**
2075	* Sync a shadow page table.
2076	*
2077	* The shadow page table is not present. This includes the case where
2078	* there is a conflict with a mapping.
2079	*
2080	* @returns VBox status code.
2081	* @param pVM VM handle.
2082	* @param iPD Page directory index.
2083	* @param pPDSrc Source page directory (i.e. Guest OS page directory).
2084	* Assume this is a temporary mapping.
2085	* @param GCPtrPage GC Pointer of the page that caused the fault
2086	*/
2087	PGM_BTH_DECL(int, SyncPT)(PVM pVM, unsigned iPDSrc, PGSTPD pPDSrc, RTGCUINTPTR GCPtrPage)
2088	{
2089	STAM_PROFILE_START(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2090	STAM_COUNTER_INC(&pVM->pgm.s.StatGCSyncPtPD[iPDSrc]);
2091	LogFlow(("SyncPT: GCPtrPage=%VGv\n", GCPtrPage));
2092
2093	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
2094	\|\| PGM_GST_TYPE == PGM_TYPE_PAE
2095
2096	/*
2097	* Validate input a little bit.
2098	*/
2099	AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%VGv\n", iPDSrc, GCPtrPage));
2100	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2101	PX86PD pPDDst = pVM->pgm.s.CTXMID(p,32BitPD);
2102	# else
2103	PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0];
2104	# endif
2105	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2106	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2107	SHWPDE PdeDst = *pPdeDst;
2108
2109	# ifndef PGM_WITHOUT_MAPPINGS
2110	/*
2111	* Check for conflicts.
2112	* GC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
2113	* HC: Simply resolve the conflict.
2114	*/
2115	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
2116	{
2117	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
2118	# ifndef IN_RING3
2119	Log(("SyncPT: Conflict at %VGv\n", GCPtrPage));
2120	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2121	return VERR_ADDRESS_CONFLICT;
2122	# else
2123	PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
2124	Assert(pMapping);
2125	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2126	int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2127	# elif PGM_GST_TYPE == PGM_TYPE_PAE
2128	int rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2129	# endif
2130	if (VBOX_FAILURE(rc))
2131	{
2132	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2133	return rc;
2134	}
2135	PdeDst = *pPdeDst;
2136	# endif
2137	}
2138	# else /* PGM_WITHOUT_MAPPINGS */
2139	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
2140	# endif /* PGM_WITHOUT_MAPPINGS */
2141	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2142
2143	/*
2144	* Sync page directory entry.
2145	*/
2146	int rc = VINF_SUCCESS;
2147	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2148	if (PdeSrc.n.u1Present)
2149	{
2150	/*
2151	* Allocate & map the page table.
2152	*/
2153	PSHWPT pPTDst;
2154	const bool fPageTable = !PdeSrc.b.u1Size \|\| !(CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
2155	PPGMPOOLPAGE pShwPage;
2156	RTGCPHYS GCPhys;
2157	if (fPageTable)
2158	{
2159	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
2160	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2161	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2162	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE / 2);
2163	# endif
2164	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, SHW_POOL_ROOT_IDX, iPDDst, &pShwPage);
2165	}
2166	else
2167	{
2168	GCPhys = PdeSrc.u & GST_PDE_BIG_PG_MASK;
2169	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2170	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2171	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
2172	# endif
2173	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, SHW_POOL_ROOT_IDX, iPDDst, &pShwPage);
2174	}
2175	if (rc == VINF_SUCCESS)
2176	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2177	else if (rc == VINF_PGM_CACHED_PAGE)
2178	{
2179	/*
2180	* The PT was cached, just hook it up.
2181	*/
2182	if (fPageTable)
2183	PdeDst.u = pShwPage->Core.Key
2184	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2185	else
2186	{
2187	PdeDst.u = pShwPage->Core.Key
2188	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2189	/* (see explanation and assumptions further down.) */
2190	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
2191	{
2192	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageBig));
2193	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2194	PdeDst.b.u1Write = 0;
2195	}
2196	}
2197	*pPdeDst = PdeDst;
2198	return VINF_SUCCESS;
2199	}
2200	else if (rc == VERR_PGM_POOL_FLUSHED)
2201	return VINF_PGM_SYNC_CR3;
2202	else
2203	AssertMsgFailedReturn(("rc=%Vrc\n", rc), VERR_INTERNAL_ERROR);
2204	PdeDst.u &= X86_PDE_AVL_MASK;
2205	PdeDst.u \|= pShwPage->Core.Key;
2206
2207	/*
2208	* Page directory has been accessed (this is a fault situation, remember).
2209	*/
2210	pPDSrc->a[iPDSrc].n.u1Accessed = 1;
2211	if (fPageTable)
2212	{
2213	/*
2214	* Page table - 4KB.
2215	*
2216	* Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
2217	*/
2218	Log2(("SyncPT: 4K %VGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
2219	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
2220	PGSTPT pPTSrc;
2221	rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
2222	if (VBOX_SUCCESS(rc))
2223	{
2224	/*
2225	* Start by syncing the page directory entry so CSAM's TLB trick works.
2226	*/
2227	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| X86_PDE_AVL_MASK))
2228	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2229	*pPdeDst = PdeDst;
2230
2231	/*
2232	* Directory/page user or supervisor privilege: (same goes for read/write)
2233	*
2234	* Directory Page Combined
2235	* U/S U/S U/S
2236	* 0 0 0
2237	* 0 1 0
2238	* 1 0 0
2239	* 1 1 1
2240	*
2241	* Simple AND operation. Table listed for completeness.
2242	*
2243	*/
2244	STAM_COUNTER_INC(CTXSUFF(&pVM->pgm.s.StatSynPT4k));
2245	# ifdef PGM_SYNC_N_PAGES
2246	unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2247	unsigned iPTDst = iPTBase;
2248	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, ELEMENTS(pPTDst->a));
2249	if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
2250	iPTDst = 0;
2251	else
2252	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2253	# else /* !PGM_SYNC_N_PAGES */
2254	unsigned iPTDst = 0;
2255	const unsigned iPTDstEnd = ELEMENTS(pPTDst->a);
2256	# endif /* !PGM_SYNC_N_PAGES */
2257	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2258	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2259	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2260	# else
2261	const unsigned offPTSrc = 0;
2262	# endif
2263	for (; iPTDst < iPTDstEnd; iPTDst++)
2264	{
2265	const unsigned iPTSrc = iPTDst + offPTSrc;
2266	const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2267
2268	if (PteSrc.n.u1Present) /* we've already cleared it above */
2269	{
2270	# ifndef IN_RING0
2271	/*
2272	* Assuming kernel code will be marked as supervisor - and not as user level
2273	* and executed using a conforming code selector - And marked as readonly.
2274	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
2275	*/
2276	PPGMPAGE pPage;
2277	if ( ((PdeSrc.u & pPTSrc->a[iPTSrc].u) & (X86_PTE_RW \| X86_PTE_US))
2278	\|\| !CSAMDoesPageNeedScanning(pVM, (RTGCPTR)((iPDSrc << GST_PD_SHIFT) \| (iPTSrc << PAGE_SHIFT)))
2279	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
2280	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2281	)
2282	# endif
2283	PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2284	Log2(("SyncPT: 4K+ %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%VGp\n",
2285	(RTGCPTR)((iPDSrc << GST_PD_SHIFT) \| (iPTSrc << PAGE_SHIFT)),
2286	PteSrc.n.u1Present,
2287	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2288	PteSrc.n.u1User & PdeSrc.n.u1User,
2289	(uint64_t)PteSrc.u,
2290	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : "", pPTDst->a[iPTDst].u, iPTSrc, PdeSrc.au32[0],
2291	(PdeSrc.u & GST_PDE_PG_MASK) + iPTSrc*sizeof(PteSrc)));
2292	}
2293	} /* for PTEs */
2294	}
2295	}
2296	else
2297	{
2298	/*
2299	* Big page - 2/4MB.
2300	*
2301	* We'll walk the ram range list in parallel and optimize lookups.
2302	* We will only sync on shadow page table at a time.
2303	*/
2304	STAM_COUNTER_INC(CTXSUFF(&pVM->pgm.s.StatSynPT4M));
2305
2306	/**
2307	* @todo It might be more efficient to sync only a part of the 4MB page (similar to what we do for 4kb PDs).
2308	*/
2309
2310	/*
2311	* Start by syncing the page directory entry.
2312	*/
2313	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
2314	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2315
2316	/*
2317	* If the page is not flagged as dirty and is writable, then make it read-only
2318	* at PD level, so we can set the dirty bit when the page is modified.
2319	*
2320	* ASSUMES that page access handlers are implemented on page table entry level.
2321	* Thus we will first catch the dirty access and set PDE.D and restart. If
2322	* there is an access handler, we'll trap again and let it work on the problem.
2323	*/
2324	/** @todo move the above stuff to a section in the PGM documentation. */
2325	Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
2326	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
2327	{
2328	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageBig));
2329	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2330	PdeDst.b.u1Write = 0;
2331	}
2332	*pPdeDst = PdeDst;
2333
2334	/*
2335	* Fill the shadow page table.
2336	*/
2337	/* Get address and flags from the source PDE. */
2338	SHWPTE PteDstBase;
2339	PteDstBase.u = PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT);
2340
2341	/* Loop thru the entries in the shadow PT. */
2342	const RTGCUINTPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
2343	Log2(("SyncPT: BIG %VGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%VGv GCPhys=%VGp %s\n",
2344	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
2345	GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2346	PPGMRAMRANGE pRam = CTXALLSUFF(pVM->pgm.s.pRamRanges);
2347	unsigned iPTDst = 0;
2348	while (iPTDst < ELEMENTS(pPTDst->a))
2349	{
2350	/* Advance ram range list. */
2351	while (pRam && GCPhys > pRam->GCPhysLast)
2352	pRam = CTXALLSUFF(pRam->pNext);
2353	if (pRam && GCPhys >= pRam->GCPhys)
2354	{
2355	unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2356	do
2357	{
2358	/* Make shadow PTE. */
2359	PPGMPAGE pPage = &pRam->aPages[iHCPage];
2360	SHWPTE PteDst;
2361
2362	/* Make sure the RAM has already been allocated. */
2363	if (pRam->fFlags & MM_RAM_FLAGS_DYNAMIC_ALLOC) /** @todo PAGE FLAGS */
2364	{
2365	if (RT_UNLIKELY(!PGM_PAGE_GET_HCPHYS(pPage)))
2366	{
2367	# ifdef IN_RING3
2368	int rc = pgmr3PhysGrowRange(pVM, GCPhys);
2369	# else
2370	int rc = CTXALLMID(VMM, CallHost)(pVM, VMMCALLHOST_PGM_RAM_GROW_RANGE, GCPhys);
2371	# endif
2372	if (rc != VINF_SUCCESS)
2373	return rc;
2374	}
2375	}
2376
2377	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2378	{
2379	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
2380	{
2381	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) \| PteDstBase.u;
2382	PteDst.n.u1Write = 0;
2383	}
2384	else
2385	PteDst.u = 0;
2386	}
2387	# ifndef IN_RING0
2388	/*
2389	* Assuming kernel code will be marked as supervisor and not as user level and executed
2390	* using a conforming code selector. Don't check for readonly, as that implies the whole
2391	* 4MB can be code or readonly data. Linux enables write access for its large pages.
2392	*/
2393	else if ( !PdeSrc.n.u1User
2394	&& CSAMDoesPageNeedScanning(pVM, (RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))))
2395	PteDst.u = 0;
2396	# endif
2397	else
2398	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) \| PteDstBase.u;
2399	# ifdef PGMPOOL_WITH_USER_TRACKING
2400	if (PteDst.n.u1Present)
2401	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, pPage->HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst); /** @todo PAGE FLAGS */
2402	# endif
2403	/* commit it */
2404	pPTDst->a[iPTDst] = PteDst;
2405	Log4(("SyncPT: BIG %VGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
2406	(RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT)), PteDst.n.u1Present, PteDst.n.u1Write, PteDst.n.u1User, (uint64_t)PteDst.u,
2407	PteDst.u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2408
2409	/* advance */
2410	GCPhys += PAGE_SIZE;
2411	iHCPage++;
2412	iPTDst++;
2413	} while ( iPTDst < ELEMENTS(pPTDst->a)
2414	&& GCPhys <= pRam->GCPhysLast);
2415	}
2416	else if (pRam)
2417	{
2418	Log(("Invalid pages at %VGp\n", GCPhys));
2419	do
2420	{
2421	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2422	GCPhys += PAGE_SIZE;
2423	iPTDst++;
2424	} while ( iPTDst < ELEMENTS(pPTDst->a)
2425	&& GCPhys < pRam->GCPhys);
2426	}
2427	else
2428	{
2429	Log(("Invalid pages at %VGp (2)\n", GCPhys));
2430	for ( ; iPTDst < ELEMENTS(pPTDst->a); iPTDst++)
2431	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2432	}
2433	} /* while more PTEs */
2434	} /* 4KB / 4MB */
2435	}
2436	else
2437	AssertRelease(!PdeDst.n.u1Present);
2438
2439	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2440	# ifdef IN_GC
2441	if (VBOX_FAILURE(rc))
2442	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncPTFailed));
2443	# endif
2444	return rc;
2445
2446	#elif PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT
2447
2448	int rc = VINF_SUCCESS;
2449
2450	/*
2451	* Validate input a little bit.
2452	*/
2453	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2454	PX86PD pPDDst = pVM->pgm.s.CTXMID(p,32BitPD);
2455	# else
2456	PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0];
2457	# endif
2458	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2459	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2460	SHWPDE PdeDst = *pPdeDst;
2461
2462	Assert(!(PdeDst.u & PGM_PDFLAGS_MAPPING));
2463	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2464
2465	GSTPDE PdeSrc;
2466	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2467	PdeSrc.n.u1Present = 1;
2468	PdeSrc.n.u1Write = 1;
2469	PdeSrc.n.u1Accessed = 1;
2470	PdeSrc.n.u1User = 1;
2471
2472	/*
2473	* Allocate & map the page table.
2474	*/
2475	PSHWPT pPTDst;
2476	PPGMPOOLPAGE pShwPage;
2477	RTGCPHYS GCPhys;
2478
2479	/* Virtual address = physical address */
2480	GCPhys = GCPtrPage & X86_PAGE_4K_BASE_MASK_32;
2481	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, SHW_POOL_ROOT_IDX, iPDDst, &pShwPage);
2482
2483	if ( rc == VINF_SUCCESS
2484	\|\| rc == VINF_PGM_CACHED_PAGE)
2485	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2486	else
2487	AssertMsgFailedReturn(("rc=%Vrc\n", rc), VERR_INTERNAL_ERROR);
2488
2489	PdeDst.u &= X86_PDE_AVL_MASK;
2490	PdeDst.u \|= pShwPage->Core.Key;
2491	PdeDst.n.u1Present = 1;
2492	PdeDst.n.u1Write = 1;
2493	PdeDst.n.u1User = 1;
2494	*pPdeDst = PdeDst;
2495
2496	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)GCPtrPage, PGM_SYNC_NR_PAGES, 0 /* page not present */);
2497	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2498	return rc;
2499
2500	#else /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
2501	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2502	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2503	return VERR_INTERNAL_ERROR;
2504	#endif /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
2505	}
2506
2507
2508
2509	/**
2510	* Prefetch a page/set of pages.
2511	*
2512	* Typically used to sync commonly used pages before entering raw mode
2513	* after a CR3 reload.
2514	*
2515	* @returns VBox status code.
2516	* @param pVM VM handle.
2517	* @param GCPtrPage Page to invalidate.
2518	*/
2519	PGM_BTH_DECL(int, PrefetchPage)(PVM pVM, RTGCUINTPTR GCPtrPage)
2520	{
2521	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) && PGM_SHW_TYPE != PGM_TYPE_AMD64
2522	/*
2523	* Check that all Guest levels thru the PDE are present, getting the
2524	* PD and PDE in the processes.
2525	*/
2526	int rc = VINF_SUCCESS;
2527	# if PGM_WITH_PAGING(PGM_GST_TYPE)
2528	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2529	const unsigned iPDSrc = (RTGCUINTPTR)GCPtrPage >> GST_PD_SHIFT;
2530	PGSTPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
2531	# else /* PAE */
2532	unsigned iPDSrc;
2533	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, GCPtrPage, &iPDSrc);
2534	if (!pPDSrc)
2535	return VINF_SUCCESS; /* not present */
2536	# endif
2537	const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2538	# else
2539	PGSTPD pPDSrc = NULL;
2540	const unsigned iPDSrc = 0;
2541	GSTPDE PdeSrc;
2542
2543	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2544	PdeSrc.n.u1Present = 1;
2545	PdeSrc.n.u1Write = 1;
2546	PdeSrc.n.u1Accessed = 1;
2547	PdeSrc.n.u1User = 1;
2548	# endif
2549
2550	if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
2551	{
2552	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2553	const X86PDE PdeDst = pVM->pgm.s.CTXMID(p,32BitPD)->a[GCPtrPage >> SHW_PD_SHIFT];
2554	# else
2555	const X86PDEPAE PdeDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[GCPtrPage >> SHW_PD_SHIFT];
2556	# endif
2557	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
2558	{
2559	if (!PdeDst.n.u1Present)
2560	/** r=bird: This guy will set the A bit on the PDE, probably harmless. */
2561	rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, GCPtrPage);
2562	else
2563	{
2564	/** @note We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
2565	* R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
2566	* makes no sense to prefetch more than one page.
2567	*/
2568	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, GCPtrPage, 1, 0);
2569	if (VBOX_SUCCESS(rc))
2570	rc = VINF_SUCCESS;
2571	}
2572	}
2573	}
2574	return rc;
2575
2576	#else /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
2577
2578	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
2579	return VERR_INTERNAL_ERROR;
2580	#endif /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
2581	}
2582
2583
2584
2585
2586	/**
2587	* Syncs a page during a PGMVerifyAccess() call.
2588	*
2589	* @returns VBox status code (informational included).
2590	* @param GCPtrPage The address of the page to sync.
2591	* @param fPage The effective guest page flags.
2592	* @param uErr The trap error code.
2593	*/
2594	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVM pVM, RTGCUINTPTR GCPtrPage, unsigned fPage, unsigned uErr)
2595	{
2596	LogFlow(("VerifyAccessSyncPage: GCPtrPage=%VGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
2597
2598	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) && PGM_SHW_TYPE != PGM_TYPE_AMD64
2599
2600	# ifndef IN_RING0
2601	if (!(fPage & X86_PTE_US))
2602	{
2603	/*
2604	* Mark this page as safe.
2605	*/
2606	/** @todo not correct for pages that contain both code and data!! */
2607	Log(("CSAMMarkPage %VGv; scanned=%d\n", GCPtrPage, true));
2608	CSAMMarkPage(pVM, (RTGCPTR)GCPtrPage, true);
2609	}
2610	# endif
2611	/*
2612	* Get guest PD and index.
2613	*/
2614
2615	# if PGM_WITH_PAGING(PGM_GST_TYPE)
2616	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2617	const unsigned iPDSrc = (RTGCUINTPTR)GCPtrPage >> GST_PD_SHIFT;
2618	PGSTPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
2619	# else /* PAE */
2620	unsigned iPDSrc;
2621	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, GCPtrPage, &iPDSrc);
2622
2623	if (pPDSrc)
2624	{
2625	Log(("PGMVerifyAccess: access violation for %VGv due to non-present PDPTR\n", GCPtrPage));
2626	return VINF_EM_RAW_GUEST_TRAP;
2627	}
2628	# endif
2629	# else
2630	PGSTPD pPDSrc = NULL;
2631	const unsigned iPDSrc = 0;
2632	# endif
2633	int rc = VINF_SUCCESS;
2634
2635	/*
2636	* First check if the shadow pd is present.
2637	*/
2638	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2639	PX86PDE pPdeDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[GCPtrPage >> SHW_PD_SHIFT];
2640	# else
2641	PX86PDEPAE pPdeDst = &pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[GCPtrPage >> SHW_PD_SHIFT];
2642	# endif
2643	if (!pPdeDst->n.u1Present)
2644	{
2645	rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, GCPtrPage);
2646	AssertRC(rc);
2647	if (rc != VINF_SUCCESS)
2648	return rc;
2649	}
2650
2651	# if PGM_WITH_PAGING(PGM_GST_TYPE)
2652	/* Check for dirty bit fault */
2653	rc = PGM_BTH_NAME(CheckPageFault)(pVM, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
2654	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
2655	Log(("PGMVerifyAccess: success (dirty)\n"));
2656	else
2657	{
2658	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2659	#else
2660	{
2661	GSTPDE PdeSrc;
2662	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2663	PdeSrc.n.u1Present = 1;
2664	PdeSrc.n.u1Write = 1;
2665	PdeSrc.n.u1Accessed = 1;
2666	PdeSrc.n.u1User = 1;
2667
2668	#endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
2669	Assert(rc != VINF_EM_RAW_GUEST_TRAP);
2670	if (uErr & X86_TRAP_PF_US)
2671	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncUser);
2672	else /* supervisor */
2673	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncSupervisor);
2674
2675	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, GCPtrPage, 1, 0);
2676	if (VBOX_SUCCESS(rc))
2677	{
2678	/* Page was successfully synced */
2679	Log2(("PGMVerifyAccess: success (sync)\n"));
2680	rc = VINF_SUCCESS;
2681	}
2682	else
2683	{
2684	Log(("PGMVerifyAccess: access violation for %VGv rc=%d\n", GCPtrPage, rc));
2685	return VINF_EM_RAW_GUEST_TRAP;
2686	}
2687	}
2688	return rc;
2689
2690	#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2691
2692	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2693	return VERR_INTERNAL_ERROR;
2694	#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2695	}
2696
2697
2698	#if PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE
2699	# if PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE
2700	/**
2701	* Figures out which kind of shadow page this guest PDE warrants.
2702	*
2703	* @returns Shadow page kind.
2704	* @param pPdeSrc The guest PDE in question.
2705	* @param cr4 The current guest cr4 value.
2706	*/
2707	DECLINLINE(PGMPOOLKIND) PGM_BTH_NAME(CalcPageKind)(const GSTPDE *pPdeSrc, uint32_t cr4)
2708	{
2709	if (!pPdeSrc->n.u1Size \|\| !(cr4 & X86_CR4_PSE))
2710	return BTH_PGMPOOLKIND_PT_FOR_PT;
2711	//switch (pPdeSrc->u & (X86_PDE4M_RW \| X86_PDE4M_US /\| X86_PDE4M_PAE_NX/))
2712	//{
2713	// case 0:
2714	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RO;
2715	// case X86_PDE4M_RW:
2716	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW;
2717	// case X86_PDE4M_US:
2718	// return BTH_PGMPOOLKIND_PT_FOR_BIG_US;
2719	// case X86_PDE4M_RW \| X86_PDE4M_US:
2720	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_US;
2721	# if 0
2722	// case X86_PDE4M_PAE_NX:
2723	// return BTH_PGMPOOLKIND_PT_FOR_BIG_NX;
2724	// case X86_PDE4M_RW \| X86_PDE4M_PAE_NX:
2725	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_NX;
2726	// case X86_PDE4M_US \| X86_PDE4M_PAE_NX:
2727	// return BTH_PGMPOOLKIND_PT_FOR_BIG_US_NX;
2728	// case X86_PDE4M_RW \| X86_PDE4M_US \| X86_PDE4M_PAE_NX:
2729	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_US_NX;
2730	# endif
2731	return BTH_PGMPOOLKIND_PT_FOR_BIG;
2732	//}
2733	}
2734	# endif
2735	#endif
2736
2737	#undef MY_STAM_COUNTER_INC
2738	#define MY_STAM_COUNTER_INC(a) do { } while (0)
2739
2740
2741	/**
2742	* Syncs the paging hierarchy starting at CR3.
2743	*
2744	* @returns VBox status code, no specials.
2745	* @param pVM The virtual machine.
2746	* @param cr0 Guest context CR0 register
2747	* @param cr3 Guest context CR3 register
2748	* @param cr4 Guest context CR4 register
2749	* @param fGlobal Including global page directories or not
2750	*/
2751	PGM_BTH_DECL(int, SyncCR3)(PVM pVM, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
2752	{
2753	if (VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3))
2754	fGlobal = true; /* Change this CR3 reload to be a global one. */
2755
2756	/*
2757	* Update page access handlers.
2758	* The virtual are always flushed, while the physical are only on demand.
2759	* WARNING: We are incorrectly not doing global flushing on Virtual Handler updates. We'll
2760	* have to look into that later because it will have a bad influence on the performance.
2761	* @note SvL: There's no need for that. Just invalidate the virtual range(s).
2762	* bird: Yes, but that won't work for aliases.
2763	*/
2764	/** @todo this MUST go away. See #1557. */
2765	STAM_PROFILE_START(&pVM->pgm.s.CTXMID(Stat,SyncCR3Handlers), h);
2766	PGM_GST_NAME(HandlerVirtualUpdate)(pVM, cr4);
2767	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncCR3Handlers), h);
2768
2769	#ifdef PGMPOOL_WITH_MONITORING
2770	/*
2771	* When monitoring shadowed pages, we reset the modification counters on CR3 sync.
2772	* Occationally we will have to clear all the shadow page tables because we wanted
2773	* to monitor a page which was mapped by too many shadowed page tables. This operation
2774	* sometimes refered to as a 'lightweight flush'.
2775	*/
2776	if (!(pVM->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL))
2777	pgmPoolMonitorModifiedClearAll(pVM);
2778	else
2779	{
2780	# ifdef IN_RING3
2781	pVM->pgm.s.fSyncFlags &= ~PGM_SYNC_CLEAR_PGM_POOL;
2782	pgmPoolClearAll(pVM);
2783	# else
2784	LogFlow(("SyncCR3: PGM_SYNC_CLEAR_PGM_POOL is set -> VINF_PGM_SYNC_CR3\n"));
2785	return VINF_PGM_SYNC_CR3;
2786	# endif
2787	}
2788	#endif
2789
2790	Assert(fGlobal \|\| (cr4 & X86_CR4_PGE));
2791	MY_STAM_COUNTER_INC(fGlobal ? &pVM->pgm.s.CTXMID(Stat,SyncCR3Global) : &pVM->pgm.s.CTXMID(Stat,SyncCR3NotGlobal));
2792
2793	#if PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE
2794	/*
2795	* Get page directory addresses.
2796	*/
2797	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2798	PX86PDE pPDEDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[0];
2799	# else /* PGM_SHW_TYPE == PGM_TYPE_PAE */
2800	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2801	PX86PDEPAE pPDEDst = &pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[0];
2802	# endif
2803	# endif
2804
2805	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2806	PGSTPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
2807	Assert(pPDSrc);
2808	# ifndef IN_GC
2809	Assert(MMPhysGCPhys2HCVirt(pVM, (RTGCPHYS)(cr3 & GST_CR3_PAGE_MASK), sizeof(*pPDSrc)) == pPDSrc);
2810	# endif
2811	# endif
2812
2813	/*
2814	* Iterate the page directory.
2815	*/
2816	PPGMMAPPING pMapping;
2817	unsigned iPdNoMapping;
2818	const bool fRawR0Enabled = EMIsRawRing0Enabled(pVM);
2819	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2820
2821	/* Only check mappings if they are supposed to be put into the shadow page table. */
2822	if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
2823	{
2824	pMapping = pVM->pgm.s.CTXALLSUFF(pMappings);
2825	iPdNoMapping = (pMapping) ? (pMapping->GCPtr >> GST_PD_SHIFT) : ~0U;
2826	}
2827	else
2828	{
2829	pMapping = 0;
2830	iPdNoMapping = ~0U;
2831	}
2832	# if PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64
2833	for (unsigned iPDPTE = 0; iPDPTE < GST_PDPE_ENTRIES; iPDPTE++)
2834	{
2835	unsigned iPDSrc;
2836	# if PGM_SHW_TYPE == PGM_TYPE_PAE
2837	PX86PDPAE pPDPAE = pVM->pgm.s.CTXMID(ap,PaePDs)[0];
2838	# else
2839	AssertFailed(); /* @todo */
2840	PX86PDPE pPDPAE = pVM->pgm.s.CTXMID(ap,PaePDs)[iPDPTE * X86_PG_AMD64_ENTRIES];
2841	# endif
2842	PX86PDEPAE pPDEDst = &pPDPAE->a[iPDPTE * X86_PG_PAE_ENTRIES];
2843	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, iPDPTE << X86_PDPT_SHIFT, &iPDSrc);
2844
2845	if (pPDSrc == NULL)
2846	{
2847	/* PDPT not present */
2848	if (pVM->pgm.s.CTXMID(p,PaePDPT)->a[iPDPTE].n.u1Present)
2849	{
2850	for (unsigned iPD = 0; iPD < ELEMENTS(pPDSrc->a); iPD++)
2851	{
2852	if ( pPDEDst[iPD].n.u1Present
2853	&& !(pPDEDst[iPD].u & PGM_PDFLAGS_MAPPING))
2854	{
2855	pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, pPDEDst[iPD].u & SHW_PDE_PG_MASK), SHW_POOL_ROOT_IDX, iPDPTE * X86_PG_PAE_ENTRIES + iPD);
2856	pPDEDst[iPD].u = 0;
2857	}
2858	}
2859	}
2860	if (!(pVM->pgm.s.CTXMID(p,PaePDPT)->a[iPDPTE].u & PGM_PLXFLAGS_MAPPING))
2861	pVM->pgm.s.CTXMID(p,PaePDPT)->a[iPDPTE].n.u1Present = 0;
2862	continue;
2863	}
2864	# else /* PGM_GST_TYPE != PGM_TYPE_PAE && PGM_GST_TYPE != PGM_TYPE_AMD64 */
2865	{
2866	# endif /* PGM_GST_TYPE != PGM_TYPE_PAE && PGM_GST_TYPE != PGM_TYPE_AMD64 */
2867	for (unsigned iPD = 0; iPD < ELEMENTS(pPDSrc->a); iPD++)
2868	{
2869	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2870	Assert(&pVM->pgm.s.CTXMID(p,32BitPD)->a[iPD] == pPDEDst);
2871	# elif PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2872	AssertMsg(&pVM->pgm.s.CTXMID(ap,PaePDs)[iPD * 2 / 512]->a[iPD * 2 % 512] == pPDEDst, ("%p vs %p\n", &pVM->pgm.s.CTXMID(ap,PaePDs)[iPD * 2 / 512]->a[iPD * 2 % 512], pPDEDst));
2873	# endif
2874	register GSTPDE PdeSrc = pPDSrc->a[iPD];
2875	if ( PdeSrc.n.u1Present
2876	&& (PdeSrc.n.u1User \|\| fRawR0Enabled))
2877	{
2878	# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2879	\|\| PGM_GST_TYPE == PGM_TYPE_PAE) \
2880	&& !defined(PGM_WITHOUT_MAPPINGS)
2881
2882	/*
2883	* Check for conflicts with GC mappings.
2884	*/
2885	# if PGM_GST_TYPE == PGM_TYPE_PAE
2886	if (iPD + iPDPTE * X86_PG_PAE_ENTRIES == iPdNoMapping)
2887	# else
2888	if (iPD == iPdNoMapping)
2889	# endif
2890	{
2891	if (pVM->pgm.s.fMappingsFixed)
2892	{
2893	/* It's fixed, just skip the mapping. */
2894	const unsigned cPTs = pMapping->cb >> GST_PD_SHIFT;
2895	iPD += cPTs - 1;
2896	pPDEDst += cPTs + (PGM_GST_TYPE != PGM_SHW_TYPE) * cPTs; /* Only applies to the pae shadow and 32 bits guest case */
2897	pMapping = pMapping->CTXALLSUFF(pNext);
2898	iPdNoMapping = pMapping ? pMapping->GCPtr >> GST_PD_SHIFT : ~0U;
2899	continue;
2900	}
2901	# ifdef IN_RING3
2902	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2903	int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, iPD << GST_PD_SHIFT);
2904	# elif PGM_GST_TYPE == PGM_TYPE_PAE
2905	int rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, (iPDPTE << GST_PDPT_SHIFT) + (iPD << GST_PD_SHIFT));
2906	# endif
2907	if (VBOX_FAILURE(rc))
2908	return rc;
2909
2910	/*
2911	* Update iPdNoMapping and pMapping.
2912	*/
2913	pMapping = pVM->pgm.s.pMappingsR3;
2914	while (pMapping && pMapping->GCPtr < (iPD << GST_PD_SHIFT))
2915	pMapping = pMapping->pNextR3;
2916	iPdNoMapping = pMapping ? pMapping->GCPtr >> GST_PD_SHIFT : ~0U;
2917	# else
2918	LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
2919	return VINF_PGM_SYNC_CR3;
2920	# endif
2921	}
2922	# else /* (PGM_GST_TYPE != PGM_TYPE_32BIT && PGM_GST_TYPE != PGM_TYPE_PAE) \|\| PGM_WITHOUT_MAPPINGS */
2923	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
2924	# endif /* (PGM_GST_TYPE != PGM_TYPE_32BIT && PGM_GST_TYPE != PGM_TYPE_PAE) \|\| PGM_WITHOUT_MAPPINGS */
2925	/*
2926	* Sync page directory entry.
2927	*
2928	* The current approach is to allocated the page table but to set
2929	* the entry to not-present and postpone the page table synching till
2930	* it's actually used.
2931	*/
2932	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2933	for (unsigned i = 0, iPdShw = iPD * 2; i < 2; i++, iPdShw++) /* pray that the compiler unrolls this */
2934	# elif PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64
2935	const unsigned iPdShw = iPD + iPDPTE * X86_PG_PAE_ENTRIES; NOREF(iPdShw);
2936	# else
2937	const unsigned iPdShw = iPD; NOREF(iPdShw);
2938	# endif
2939	{
2940	SHWPDE PdeDst = *pPDEDst;
2941	if (PdeDst.n.u1Present)
2942	{
2943	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2944	RTGCPHYS GCPhys;
2945	if ( !PdeSrc.b.u1Size
2946	\|\| !(cr4 & X86_CR4_PSE))
2947	{
2948	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
2949	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2950	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2951	GCPhys \|= i * (PAGE_SIZE / 2);
2952	# endif
2953	}
2954	else
2955	{
2956	GCPhys = PdeSrc.u & GST_PDE_BIG_PG_MASK;
2957	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2958	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2959	GCPhys \|= i * X86_PAGE_2M_SIZE;
2960	# endif
2961	}
2962
2963	if ( pShwPage->GCPhys == GCPhys
2964	&& pShwPage->enmKind == PGM_BTH_NAME(CalcPageKind)(&PdeSrc, cr4)
2965	&& ( pShwPage->fCached
2966	\|\| ( !fGlobal
2967	&& ( false
2968	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
2969	\|\| ( (PdeSrc.u & (X86_PDE4M_PS \| X86_PDE4M_G)) == (X86_PDE4M_PS \| X86_PDE4M_G)
2970	&& (cr4 & (X86_CR4_PGE \| X86_CR4_PSE)) == (X86_CR4_PGE \| X86_CR4_PSE)) /* global 2/4MB page. */
2971	\|\| ( !pShwPage->fSeenNonGlobal
2972	&& (cr4 & X86_CR4_PGE))
2973	# endif
2974	)
2975	)
2976	)
2977	&& ( (PdeSrc.u & (X86_PDE_US \| X86_PDE_RW)) == (PdeDst.u & (X86_PDE_US \| X86_PDE_RW))
2978	\|\| ( (cr4 & X86_CR4_PSE)
2979	&& ((PdeSrc.u & (X86_PDE_US \| X86_PDE4M_PS \| X86_PDE4M_D)) \| PGM_PDFLAGS_TRACK_DIRTY)
2980	== ((PdeDst.u & (X86_PDE_US \| X86_PDE_RW \| PGM_PDFLAGS_TRACK_DIRTY)) \| X86_PDE4M_PS))
2981	)
2982	)
2983	{
2984	# ifdef VBOX_WITH_STATISTICS
2985	if ( !fGlobal
2986	&& (PdeSrc.u & (X86_PDE4M_PS \| X86_PDE4M_G)) == (X86_PDE4M_PS \| X86_PDE4M_G)
2987	&& (cr4 & (X86_CR4_PGE \| X86_CR4_PSE)) == (X86_CR4_PGE \| X86_CR4_PSE))
2988	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstSkippedGlobalPD));
2989	else if (!fGlobal && !pShwPage->fSeenNonGlobal && (cr4 & X86_CR4_PGE))
2990	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstSkippedGlobalPT));
2991	else
2992	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstCacheHit));
2993	# endif /* VBOX_WITH_STATISTICS */
2994	/** @todo a replacement strategy isn't really needed unless we're using a very small pool < 512 pages.
2995	* The whole ageing stuff should be put in yet another set of #ifdefs. For now, let's just skip it. */
2996	//# ifdef PGMPOOL_WITH_CACHE
2997	// pgmPoolCacheUsed(pPool, pShwPage);
2998	//# endif
2999	}
3000	else
3001	{
3002	pgmPoolFreeByPage(pPool, pShwPage, SHW_POOL_ROOT_IDX, iPdShw);
3003	pPDEDst->u = 0;
3004	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstFreed));
3005	}
3006	}
3007	else
3008	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstNotPresent));
3009	pPDEDst++;
3010	}
3011	}
3012	# if PGM_GST_TYPE == PGM_TYPE_PAE
3013	else if (iPD + iPDPTE * X86_PG_PAE_ENTRIES != iPdNoMapping)
3014	# else
3015	else if (iPD != iPdNoMapping)
3016	# endif
3017	{
3018	/*
3019	* Check if there is any page directory to mark not present here.
3020	*/
3021	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3022	for (unsigned i = 0, iPdShw = iPD * 2; i < 2; i++, iPdShw++) /* pray that the compiler unrolls this */
3023	# elif PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64
3024	const unsigned iPdShw = iPD + iPDPTE * X86_PG_PAE_ENTRIES; NOREF(iPdShw);
3025	# else
3026	const unsigned iPdShw = iPD; NOREF(iPdShw);
3027	# endif
3028	{
3029	if (pPDEDst->n.u1Present)
3030	{
3031	pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, pPDEDst->u & SHW_PDE_PG_MASK), SHW_POOL_ROOT_IDX, iPdShw);
3032	pPDEDst->u = 0;
3033	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstFreedSrcNP));
3034	}
3035	pPDEDst++;
3036	}
3037	}
3038	else
3039	{
3040	# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3041	\|\| PGM_GST_TYPE == PGM_TYPE_PAE) \
3042	&& !defined(PGM_WITHOUT_MAPPINGS)
3043
3044	const unsigned cPTs = pMapping->cb >> GST_PD_SHIFT;
3045
3046	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
3047	if (pVM->pgm.s.fMappingsFixed)
3048	{
3049	/* It's fixed, just skip the mapping. */
3050	pMapping = pMapping->CTXALLSUFF(pNext);
3051	iPdNoMapping = pMapping ? pMapping->GCPtr >> GST_PD_SHIFT : ~0U;
3052	}
3053	else
3054	{
3055	/*
3056	* Check for conflicts for subsequent pagetables
3057	* and advance to the next mapping.
3058	*/
3059	iPdNoMapping = ~0U;
3060	unsigned iPT = cPTs;
3061	while (iPT-- > 1)
3062	{
3063	if ( pPDSrc->a[iPD + iPT].n.u1Present
3064	&& (pPDSrc->a[iPD + iPT].n.u1User \|\| fRawR0Enabled))
3065	{
3066	# ifdef IN_RING3
3067	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3068	int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, iPD << GST_PD_SHIFT);
3069	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3070	int rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, (iPDPTE << GST_PDPT_SHIFT) + (iPD << GST_PD_SHIFT));
3071	# endif
3072	if (VBOX_FAILURE(rc))
3073	return rc;
3074
3075	/*
3076	* Update iPdNoMapping and pMapping.
3077	*/
3078	pMapping = pVM->pgm.s.CTXALLSUFF(pMappings);
3079	while (pMapping && pMapping->GCPtr < (iPD << GST_PD_SHIFT))
3080	pMapping = pMapping->CTXALLSUFF(pNext);
3081	iPdNoMapping = pMapping ? pMapping->GCPtr >> GST_PD_SHIFT : ~0U;
3082	break;
3083	# else
3084	LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
3085	return VINF_PGM_SYNC_CR3;
3086	# endif
3087	}
3088	}
3089	if (iPdNoMapping == ~0U && pMapping)
3090	{
3091	pMapping = pMapping->CTXALLSUFF(pNext);
3092	if (pMapping)
3093	iPdNoMapping = pMapping->GCPtr >> GST_PD_SHIFT;
3094	}
3095	}
3096
3097	/* advance. */
3098	iPD += cPTs - 1;
3099	pPDEDst += cPTs + (PGM_GST_TYPE != PGM_SHW_TYPE) * cPTs; /* Only applies to the pae shadow and 32 bits guest case */
3100	# if PGM_GST_TYPE != PGM_SHW_TYPE
3101	AssertCompile(PGM_GST_TYPE == PGM_TYPE_32BIT && PGM_SHW_TYPE == PGM_TYPE_PAE);
3102	# endif
3103	# else /* (PGM_GST_TYPE != PGM_TYPE_32BIT && PGM_GST_TYPE != PGM_TYPE_PAE) \|\| PGM_WITHOUT_MAPPINGS */
3104	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
3105	# endif /* (PGM_GST_TYPE != PGM_TYPE_32BIT && PGM_GST_TYPE != PGM_TYPE_PAE) \|\| PGM_WITHOUT_MAPPINGS */
3106	}
3107
3108	} /* for iPD */
3109	} /* for each PDPTE (PAE) */
3110
3111	return VINF_SUCCESS;
3112
3113	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
3114	//# error not implemented
3115	return VERR_INTERNAL_ERROR;
3116	#else /* guest real and protected mode */
3117	return VINF_SUCCESS;
3118	#endif
3119	}
3120
3121
3122
3123
3124	#ifdef VBOX_STRICT
3125	#ifdef IN_GC
3126	# undef AssertMsgFailed
3127	# define AssertMsgFailed Log
3128	#endif
3129	#ifdef IN_RING3
3130	# include <VBox/dbgf.h>
3131
3132	/**
3133	* Dumps a page table hierarchy use only physical addresses and cr4/lm flags.
3134	*
3135	* @returns VBox status code (VINF_SUCCESS).
3136	* @param pVM The VM handle.
3137	* @param cr3 The root of the hierarchy.
3138	* @param crr The cr4, only PAE and PSE is currently used.
3139	* @param fLongMode Set if long mode, false if not long mode.
3140	* @param cMaxDepth Number of levels to dump.
3141	* @param pHlp Pointer to the output functions.
3142	*/
3143	__BEGIN_DECLS
3144	PGMR3DECL(int) PGMR3DumpHierarchyHC(PVM pVM, uint32_t cr3, uint32_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp);
3145	__END_DECLS
3146
3147	#endif
3148
3149	/**
3150	* Checks that the shadow page table is in sync with the guest one.
3151	*
3152	* @returns The number of errors.
3153	* @param pVM The virtual machine.
3154	* @param cr3 Guest context CR3 register
3155	* @param cr4 Guest context CR4 register
3156	* @param GCPtr Where to start. Defaults to 0.
3157	* @param cb How much to check. Defaults to everything.
3158	*/
3159	PGM_BTH_DECL(unsigned, AssertCR3)(PVM pVM, uint64_t cr3, uint64_t cr4, RTGCUINTPTR GCPtr, RTGCUINTPTR cb)
3160	{
3161	unsigned cErrors = 0;
3162
3163	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
3164	\|\| PGM_GST_TYPE == PGM_TYPE_PAE
3165
3166	PPGM pPGM = &pVM->pgm.s;
3167	RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
3168	RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
3169	# ifndef IN_RING0
3170	RTHCPHYS HCPhys; /* general usage. */
3171	# endif
3172	int rc;
3173
3174	/*
3175	* Check that the Guest CR3 and all its mappings are correct.
3176	*/
3177	AssertMsgReturn(pPGM->GCPhysCR3 == (cr3 & GST_CR3_PAGE_MASK),
3178	("Invalid GCPhysCR3=%VGp cr3=%VGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
3179	false);
3180	# ifndef IN_RING0
3181	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3182	rc = PGMShwGetPage(pVM, pPGM->pGuestPDGC, NULL, &HCPhysShw);
3183	# else
3184	rc = PGMShwGetPage(pVM, pPGM->pGstPaePDPTGC, NULL, &HCPhysShw);
3185	# endif
3186	AssertRCReturn(rc, 1);
3187	HCPhys = NIL_RTHCPHYS;
3188	rc = pgmRamGCPhys2HCPhys(pPGM, cr3 & GST_CR3_PAGE_MASK, &HCPhys);
3189	AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%VHp HCPhyswShw=%VHp (cr3)\n", HCPhys, HCPhysShw), false);
3190	# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
3191	RTGCPHYS GCPhys;
3192	rc = PGMR3DbgHCPtr2GCPhys(pVM, pPGM->pGuestPDHC, &GCPhys);
3193	AssertRCReturn(rc, 1);
3194	AssertMsgReturn((cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%VGp cr3=%VGp\n", GCPhys, (RTGCPHYS)cr3), false);
3195	# endif
3196	#endif /* !IN_RING0 */
3197
3198	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3199	const GSTPD *pPDSrc = CTXSUFF(pPGM->pGuestPD);
3200	# endif
3201
3202	/*
3203	* Get and check the Shadow CR3.
3204	*/
3205	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3206	const X86PD *pPDDst = pPGM->CTXMID(p,32BitPD);
3207	unsigned cPDEs = ELEMENTS(pPDDst->a);
3208	# else
3209	const X86PDPAE pPDDst = pPGM->CTXMID(ap,PaePDs[0]); / use it as a 2048 entry PD */
3210	unsigned cPDEs = ELEMENTS(pPDDst->a) * ELEMENTS(pPGM->apHCPaePDs);
3211	# endif
3212	if (cb != ~(RTGCUINTPTR)0)
3213	cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
3214
3215	/** @todo call the other two PGMAssert() functions. /
3216
3217	# if PGM_GST_TYPE == PGM_TYPE_PAE
3218	/*
3219	* Check the 4 PDPTs too.
3220	*/
3221	for (unsigned i = 0; i < 4; i++)
3222	{
3223	RTHCPTR HCPtr;
3224	RTHCPHYS HCPhys;
3225	RTGCPHYS GCPhys = pVM->pgm.s.CTXSUFF(pGstPaePDPT)->a[i].u & X86_PDPE_PG_MASK;
3226	int rc2 = pgmRamGCPhys2HCPtrAndHCPhysWithFlags(&pVM->pgm.s, GCPhys, &HCPtr, &HCPhys);
3227	if (VBOX_SUCCESS(rc2))
3228	{
3229	AssertMsg( pVM->pgm.s.apGstPaePDsHC[i] == (R3R0PTRTYPE(PX86PDPAE))HCPtr
3230	&& pVM->pgm.s.aGCPhysGstPaePDs[i] == GCPhys,
3231	("idx %d apGstPaePDsHC %VHv vs %VHv aGCPhysGstPaePDs %VGp vs %VGp\n",
3232	i, pVM->pgm.s.apGstPaePDsHC[i], HCPtr, pVM->pgm.s.aGCPhysGstPaePDs[i], GCPhys));
3233	}
3234	}
3235	# endif
3236
3237	/*
3238	* Iterate the shadow page directory.
3239	*/
3240	GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
3241	unsigned iPDDst = GCPtr >> SHW_PD_SHIFT;
3242	cPDEs += iPDDst;
3243	for (;
3244	iPDDst < cPDEs;
3245	iPDDst++, GCPtr += _4G / cPDEs)
3246	{
3247	# if PGM_GST_TYPE == PGM_TYPE_PAE
3248	uint32_t iPDSrc;
3249	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pPGM, (RTGCUINTPTR)GCPtr, &iPDSrc);
3250	if (!pPDSrc)
3251	{
3252	AssertMsg(!pVM->pgm.s.CTXSUFF(pGstPaePDPT)->a[(GCPtr >> GST_PDPT_SHIFT) & GST_PDPT_MASK].n.u1Present, ("Guest PDTPR not present, shadow PDPTR %VX64\n", pVM->pgm.s.CTXSUFF(pGstPaePDPT)->a[(GCPtr >> GST_PDPT_SHIFT) & GST_PDPT_MASK].u));
3253	continue;
3254	}
3255	#endif
3256
3257	const SHWPDE PdeDst = pPDDst->a[iPDDst];
3258	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
3259	{
3260	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
3261	if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
3262	{
3263	AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
3264	cErrors++;
3265	continue;
3266	}
3267	}
3268	else if ( (PdeDst.u & X86_PDE_P)
3269	\|\| ((PdeDst.u & (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY))
3270	)
3271	{
3272	HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
3273	PPGMPOOLPAGE pPoolPage = pgmPoolGetPageByHCPhys(pVM, HCPhysShw);
3274	if (!pPoolPage)
3275	{
3276	AssertMsgFailed(("Invalid page table address %VGp at %VGv! PdeDst=%#RX64\n",
3277	HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
3278	cErrors++;
3279	continue;
3280	}
3281	const SHWPT pPTDst = (const SHWPT )PGMPOOL_PAGE_2_PTR(pVM, pPoolPage);
3282
3283	if (PdeDst.u & (X86_PDE4M_PWT \| X86_PDE4M_PCD))
3284	{
3285	AssertMsgFailed(("PDE flags PWT and/or PCD is set at %VGv! These flags are not virtualized! PdeDst=%#RX64\n",
3286	GCPtr, (uint64_t)PdeDst.u));
3287	cErrors++;
3288	}
3289
3290	if (PdeDst.u & (X86_PDE4M_G \| X86_PDE4M_D))
3291	{
3292	AssertMsgFailed(("4K PDE reserved flags at %VGv! PdeDst=%#RX64\n",
3293	GCPtr, (uint64_t)PdeDst.u));
3294	cErrors++;
3295	}
3296
3297	const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
3298	if (!PdeSrc.n.u1Present)
3299	{
3300	AssertMsgFailed(("Guest PDE at %VGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
3301	GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
3302	cErrors++;
3303	continue;
3304	}
3305
3306	if ( !PdeSrc.b.u1Size
3307	\|\| !(cr4 & X86_CR4_PSE))
3308	{
3309	GCPhysGst = PdeSrc.u & GST_PDE_PG_MASK;
3310	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3311	GCPhysGst \|= (iPDDst & 1) * (PAGE_SIZE / 2);
3312	# endif
3313	}
3314	else
3315	{
3316	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3317	if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
3318	{
3319	AssertMsgFailed(("Guest PDE at %VGv is using PSE36 or similar! PdeSrc=%#RX64\n",
3320	GCPtr, (uint64_t)PdeSrc.u));
3321	cErrors++;
3322	continue;
3323	}
3324	# endif
3325	GCPhysGst = PdeSrc.u & GST_PDE_BIG_PG_MASK;
3326	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3327	GCPhysGst \|= GCPtr & RT_BIT(X86_PAGE_2M_SHIFT);
3328	# endif
3329	}
3330
3331	if ( pPoolPage->enmKind
3332	!= (!PdeSrc.b.u1Size \|\| !(cr4 & X86_CR4_PSE) ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
3333	{
3334	AssertMsgFailed(("Invalid shadow page table kind %d at %VGv! PdeSrc=%#RX64\n",
3335	pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
3336	cErrors++;
3337	}
3338
3339	PPGMPAGE pPhysPage = pgmPhysGetPage(pPGM, GCPhysGst);
3340	if (!pPhysPage)
3341	{
3342	AssertMsgFailed(("Cannot find guest physical address %VGp in the PDE at %VGv! PdeSrc=%#RX64\n",
3343	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3344	cErrors++;
3345	continue;
3346	}
3347
3348	if (GCPhysGst != pPoolPage->GCPhys)
3349	{
3350	AssertMsgFailed(("GCPhysGst=%VGp != pPage->GCPhys=%VGp at %VGv\n",
3351	GCPhysGst, pPoolPage->GCPhys, GCPtr));
3352	cErrors++;
3353	continue;
3354	}
3355
3356	if ( !PdeSrc.b.u1Size
3357	\|\| !(cr4 & X86_CR4_PSE))
3358	{
3359	/*
3360	* Page Table.
3361	*/
3362	const GSTPT *pPTSrc;
3363	rc = PGM_GCPHYS_2_PTR(pVM, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1), &pPTSrc);
3364	if (VBOX_FAILURE(rc))
3365	{
3366	AssertMsgFailed(("Cannot map/convert guest physical address %VGp in the PDE at %VGv! PdeSrc=%#RX64\n",
3367	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3368	cErrors++;
3369	continue;
3370	}
3371	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/))
3372	!= (PdeDst.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/)))
3373	{
3374	/// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
3375	// (This problem will go away when/if we shadow multiple CR3s.)
3376	AssertMsgFailed(("4K PDE flags mismatch at %VGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3377	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3378	cErrors++;
3379	continue;
3380	}
3381	if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3382	{
3383	AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%VGv PdeDst=%#RX64\n",
3384	GCPtr, (uint64_t)PdeDst.u));
3385	cErrors++;
3386	continue;
3387	}
3388
3389	/* iterate the page table. */
3390	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3391	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3392	const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
3393	# else
3394	const unsigned offPTSrc = 0;
3395	# endif
3396	for (unsigned iPT = 0, off = 0;
3397	iPT < ELEMENTS(pPTDst->a);
3398	iPT++, off += PAGE_SIZE)
3399	{
3400	const SHWPTE PteDst = pPTDst->a[iPT];
3401
3402	/* skip not-present entries. */
3403	if (!(PteDst.u & (X86_PTE_P \| PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
3404	continue;
3405	Assert(PteDst.n.u1Present);
3406
3407	const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
3408	if (!PteSrc.n.u1Present)
3409	{
3410	#ifdef IN_RING3
3411	PGMAssertHandlerAndFlagsInSync(pVM);
3412	PGMR3DumpHierarchyGC(pVM, cr3, cr4, (PdeSrc.u & GST_PDE_PG_MASK));
3413	#endif
3414	AssertMsgFailed(("Out of sync (!P) PTE at %VGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%VGv iPTSrc=%x PdeSrc=%x physpte=%VGp\n",
3415	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u, pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
3416	(PdeSrc.u & GST_PDE_PG_MASK) + (iPT + offPTSrc)*sizeof(PteSrc)));
3417	cErrors++;
3418	continue;
3419	}
3420
3421	uint64_t fIgnoreFlags = GST_PTE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_G \| X86_PTE_D \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
3422	# if 1 /** @todo sync accessed bit properly... */
3423	fIgnoreFlags \|= X86_PTE_A;
3424	# endif
3425
3426	/* match the physical addresses */
3427	HCPhysShw = PteDst.u & SHW_PTE_PG_MASK;
3428	GCPhysGst = PteSrc.u & GST_PTE_PG_MASK;
3429
3430	# ifdef IN_RING3
3431	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3432	if (VBOX_FAILURE(rc))
3433	{
3434	if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3435	{
3436	AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3437	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3438	cErrors++;
3439	continue;
3440	}
3441	}
3442	else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
3443	{
3444	AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PteSrc=%#RX64 PteDst=%#RX64\n",
3445	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3446	cErrors++;
3447	continue;
3448	}
3449	# endif
3450
3451	pPhysPage = pgmPhysGetPage(pPGM, GCPhysGst);
3452	if (!pPhysPage)
3453	{
3454	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
3455	if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3456	{
3457	AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3458	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3459	cErrors++;
3460	continue;
3461	}
3462	# endif
3463	if (PteDst.n.u1Write)
3464	{
3465	AssertMsgFailed(("Invalid guest page at %VGv is writable! GCPhysGst=%VGp PteSrc=%#RX64 PteDst=%#RX64\n",
3466	GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3467	cErrors++;
3468	}
3469	fIgnoreFlags \|= X86_PTE_RW;
3470	}
3471	else if (HCPhysShw != (PGM_PAGE_GET_HCPHYS(pPhysPage) & SHW_PTE_PG_MASK))
3472	{
3473	AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PteSrc=%#RX64 PteDst=%#RX64\n",
3474	GCPtr + off, HCPhysShw, pPhysPage->HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3475	cErrors++;
3476	continue;
3477	}
3478
3479	/* flags */
3480	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
3481	{
3482	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
3483	{
3484	if (PteDst.n.u1Write)
3485	{
3486	AssertMsgFailed(("WRITE access flagged at %VGv but the page is writable! HCPhys=%VGv PteSrc=%#RX64 PteDst=%#RX64\n",
3487	GCPtr + off, pPhysPage->HCPhys, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3488	cErrors++;
3489	continue;
3490	}
3491	fIgnoreFlags \|= X86_PTE_RW;
3492	}
3493	else
3494	{
3495	if (PteDst.n.u1Present)
3496	{
3497	AssertMsgFailed(("ALL access flagged at %VGv but the page is present! HCPhys=%VHp PteSrc=%#RX64 PteDst=%#RX64\n",
3498	GCPtr + off, pPhysPage->HCPhys, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3499	cErrors++;
3500	continue;
3501	}
3502	fIgnoreFlags \|= X86_PTE_P;
3503	}
3504	}
3505	else
3506	{
3507	if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
3508	{
3509	if (PteDst.n.u1Write)
3510	{
3511	AssertMsgFailed(("!DIRTY page at %VGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
3512	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3513	cErrors++;
3514	continue;
3515	}
3516	if (!(PteDst.u & PGM_PTFLAGS_TRACK_DIRTY))
3517	{
3518	AssertMsgFailed(("!DIRTY page at %VGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3519	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3520	cErrors++;
3521	continue;
3522	}
3523	if (PteDst.n.u1Dirty)
3524	{
3525	AssertMsgFailed(("!DIRTY page at %VGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3526	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3527	cErrors++;
3528	}
3529	# if 0 /** @todo sync access bit properly... */
3530	if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
3531	{
3532	AssertMsgFailed(("!DIRTY page at %VGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
3533	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3534	cErrors++;
3535	}
3536	fIgnoreFlags \|= X86_PTE_RW;
3537	# else
3538	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
3539	# endif
3540	}
3541	else if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
3542	{
3543	/* access bit emulation (not implemented). */
3544	if (PteSrc.n.u1Accessed \|\| PteDst.n.u1Present)
3545	{
3546	AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %VGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
3547	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3548	cErrors++;
3549	continue;
3550	}
3551	if (!PteDst.n.u1Accessed)
3552	{
3553	AssertMsgFailed(("!ACCESSED page at %VGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
3554	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3555	cErrors++;
3556	}
3557	fIgnoreFlags \|= X86_PTE_P;
3558	}
3559	# ifdef DEBUG_sandervl
3560	fIgnoreFlags \|= X86_PTE_D \| X86_PTE_A;
3561	# endif
3562	}
3563
3564	if ( (PteSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
3565	&& (PteSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags)
3566	)
3567	{
3568	AssertMsgFailed(("Flags mismatch at %VGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
3569	GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
3570	fIgnoreFlags, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3571	cErrors++;
3572	continue;
3573	}
3574	} /* foreach PTE */
3575	}
3576	else
3577	{
3578	/*
3579	* Big Page.
3580	*/
3581	uint64_t fIgnoreFlags = X86_PDE_AVL_MASK \| GST_PDE_PG_MASK \| X86_PDE4M_G \| X86_PDE4M_D \| X86_PDE4M_PS \| X86_PDE4M_PWT \| X86_PDE4M_PCD;
3582	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
3583	{
3584	if (PdeDst.n.u1Write)
3585	{
3586	AssertMsgFailed(("!DIRTY page at %VGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3587	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3588	cErrors++;
3589	continue;
3590	}
3591	if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
3592	{
3593	AssertMsgFailed(("!DIRTY page at %VGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3594	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3595	cErrors++;
3596	continue;
3597	}
3598	# if 0 /** @todo sync access bit properly... */
3599	if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
3600	{
3601	AssertMsgFailed(("!DIRTY page at %VGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
3602	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3603	cErrors++;
3604	}
3605	fIgnoreFlags \|= X86_PTE_RW;
3606	# else
3607	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
3608	# endif
3609	}
3610	else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3611	{
3612	/* access bit emulation (not implemented). */
3613	if (PdeSrc.b.u1Accessed \|\| PdeDst.n.u1Present)
3614	{
3615	AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %VGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3616	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3617	cErrors++;
3618	continue;
3619	}
3620	if (!PdeDst.n.u1Accessed)
3621	{
3622	AssertMsgFailed(("!ACCESSED page at %VGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3623	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3624	cErrors++;
3625	}
3626	fIgnoreFlags \|= X86_PTE_P;
3627	}
3628
3629	if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
3630	{
3631	AssertMsgFailed(("Flags mismatch (B) at %VGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
3632	GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
3633	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3634	cErrors++;
3635	}
3636
3637	/* iterate the page table. */
3638	for (unsigned iPT = 0, off = 0;
3639	iPT < ELEMENTS(pPTDst->a);
3640	iPT++, off += PAGE_SIZE, GCPhysGst += PAGE_SIZE)
3641	{
3642	const SHWPTE PteDst = pPTDst->a[iPT];
3643
3644	if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
3645	{
3646	AssertMsgFailed(("The PTE at %VGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
3647	GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3648	cErrors++;
3649	}
3650
3651	/* skip not-present entries. */
3652	if (!PteDst.n.u1Present) /** @todo deal with ALL handlers and CSAM !P pages! */
3653	continue;
3654
3655	fIgnoreFlags = X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
3656
3657	/* match the physical addresses */
3658	HCPhysShw = PteDst.u & X86_PTE_PAE_PG_MASK;
3659
3660	# ifdef IN_RING3
3661	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3662	if (VBOX_FAILURE(rc))
3663	{
3664	if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3665	{
3666	AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
3667	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3668	cErrors++;
3669	}
3670	}
3671	else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
3672	{
3673	AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PdeSrc=%#RX64 PteDst=%#RX64\n",
3674	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3675	cErrors++;
3676	continue;
3677	}
3678	# endif
3679
3680	pPhysPage = pgmPhysGetPage(pPGM, GCPhysGst);
3681	if (!pPhysPage)
3682	{
3683	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
3684	if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3685	{
3686	AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
3687	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3688	cErrors++;
3689	continue;
3690	}
3691	# endif
3692	if (PteDst.n.u1Write)
3693	{
3694	AssertMsgFailed(("Invalid guest page at %VGv is writable! GCPhysGst=%VGp PdeSrc=%#RX64 PteDst=%#RX64\n",
3695	GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3696	cErrors++;
3697	}
3698	fIgnoreFlags \|= X86_PTE_RW;
3699	}
3700	else if (HCPhysShw != (pPhysPage->HCPhys & X86_PTE_PAE_PG_MASK))
3701	{
3702	AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PdeSrc=%#RX64 PteDst=%#RX64\n",
3703	GCPtr + off, HCPhysShw, pPhysPage->HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3704	cErrors++;
3705	continue;
3706	}
3707
3708	/* flags */
3709	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
3710	{
3711	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
3712	{
3713	if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
3714	{
3715	if (PteDst.n.u1Write)
3716	{
3717	AssertMsgFailed(("WRITE access flagged at %VGv but the page is writable! HCPhys=%VGv PdeSrc=%#RX64 PteDst=%#RX64\n",
3718	GCPtr + off, pPhysPage->HCPhys, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3719	cErrors++;
3720	continue;
3721	}
3722	fIgnoreFlags \|= X86_PTE_RW;
3723	}
3724	}
3725	else
3726	{
3727	if (PteDst.n.u1Present)
3728	{
3729	AssertMsgFailed(("ALL access flagged at %VGv but the page is present! HCPhys=%VGv PdeSrc=%#RX64 PteDst=%#RX64\n",
3730	GCPtr + off, pPhysPage->HCPhys, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3731	cErrors++;
3732	continue;
3733	}
3734	fIgnoreFlags \|= X86_PTE_P;
3735	}
3736	}
3737
3738	if ( (PdeSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
3739	&& (PdeSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags) /* lazy phys handler dereg. */
3740	)
3741	{
3742	AssertMsgFailed(("Flags mismatch (BT) at %VGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
3743	GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
3744	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3745	cErrors++;
3746	continue;
3747	}
3748	} /* foreach PTE */
3749	}
3750	}
3751	/* not present */
3752
3753	} /* forearch PDE */
3754
3755	# ifdef DEBUG
3756	if (cErrors)
3757	LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
3758	# endif
3759
3760	#elif PGM_GST_TYPE == PGM_TYPE_PAE
3761	//# error not implemented
3762
3763
3764	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
3765	//# error not implemented
3766
3767	/#else: guest real and protected mode /
3768	#endif
3769	return cErrors;
3770	}
3771	#endif /* VBOX_STRICT */
3772

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 8863

Download in other formats: