PGMAllBth.h@ 8564

Last change on this file since 8564 was 8537, checked in by vboxsync, 17 years ago
Spaces
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 159.4 KB

Line
1	/* $Id: PGMAllBth.h 8537 2008-05-02 17:00:34Z vboxsync $ */
2	/** @file
3	* VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4	*
5	* This file is a big challenge!
6	*/
7
8	/*
9	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
10	*
11	* This file is part of VirtualBox Open Source Edition (OSE), as
12	* available from http://www.virtualbox.org. This file is free software;
13	* you can redistribute it and/or modify it under the terms of the GNU
14	* General Public License (GPL) as published by the Free Software
15	* Foundation, in version 2 as it comes in the "COPYING" file of the
16	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
17	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
18	*
19	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
20	* Clara, CA 95054 USA or visit http://www.sun.com if you need
21	* additional information or have any questions.
22	*/
23
24	/*******************************************************************************
25	* Internal Functions *
26	*******************************************************************************/
27	__BEGIN_DECLS
28	PGM_BTH_DECL(int, Trap0eHandler)(PVM pVM, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault);
29	PGM_BTH_DECL(int, InvalidatePage)(PVM pVM, RTGCUINTPTR GCPtrPage);
30	PGM_BTH_DECL(int, SyncPage)(PVM pVM, GSTPDE PdeSrc, RTGCUINTPTR GCPtrPage, unsigned cPages, unsigned uErr);
31	PGM_BTH_DECL(int, CheckPageFault)(PVM pVM, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCUINTPTR GCPtrPage);
32	PGM_BTH_DECL(int, SyncPT)(PVM pVM, unsigned iPD, PGSTPD pPDSrc, RTGCUINTPTR GCPtrPage);
33	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVM pVM, RTGCUINTPTR Addr, unsigned fPage, unsigned uErr);
34	PGM_BTH_DECL(int, PrefetchPage)(PVM pVM, RTGCUINTPTR GCPtrPage);
35	PGM_BTH_DECL(int, SyncCR3)(PVM pVM, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
36	#ifdef VBOX_STRICT
37	PGM_BTH_DECL(unsigned, AssertCR3)(PVM pVM, uint64_t cr3, uint64_t cr4, RTGCUINTPTR GCPtr = 0, RTGCUINTPTR cb = ~(RTGCUINTPTR)0);
38	#endif
39	#ifdef PGMPOOL_WITH_USER_TRACKING
40	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVM pVM, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys);
41	#endif
42	__END_DECLS
43
44
45	/* Filter out some illegal combinations of guest and shadow paging, so we can remove redundant checks inside functions. */
46	#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE
47	# error "Invalid combination; PAE guest implies PAE shadow"
48	#endif
49
50	#if (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
51	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE)
52	# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
53	#endif
54
55	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) \
56	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE)
57	# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
58	#endif
59
60	#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64)
61	\|\| (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64)
62	# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
63	#endif
64
65	#ifdef IN_RING0 /* no mappings in VT-x and AMD-V mode */
66	# define PGM_WITHOUT_MAPPINGS
67	#endif
68
69	/**
70	* #PF Handler for raw-mode guest execution.
71	*
72	* @returns VBox status code (appropriate for trap handling and GC return).
73	* @param pVM VM Handle.
74	* @param uErr The trap error code.
75	* @param pRegFrame Trap register frame.
76	* @param pvFault The fault address.
77	*/
78	PGM_BTH_DECL(int, Trap0eHandler)(PVM pVM, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
79	{
80	#if PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64
81
82	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE != PGM_TYPE_PAE
83	/*
84	* Hide the instruction fetch trap indicator for now.
85	*/
86	/** @todo NXE will change this and we must fix NXE in the switcher too! */
87	if (uErr & X86_TRAP_PF_ID)
88	{
89	uErr &= ~X86_TRAP_PF_ID;
90	TRPMSetErrorCode(pVM, uErr);
91	}
92	# endif
93
94	/*
95	* Get PDs.
96	*/
97	int rc;
98	# if PGM_WITH_PAGING(PGM_GST_TYPE)
99	# if PGM_GST_TYPE == PGM_TYPE_32BIT
100	const unsigned iPDSrc = (RTGCUINTPTR)pvFault >> GST_PD_SHIFT;
101	PGSTPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
102
103	# elif PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64
104
105	# if PGM_GST_TYPE == PGM_TYPE_PAE
106	unsigned iPDSrc;
107	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, (RTGCUINTPTR)pvFault, &iPDSrc);
108
109	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
110	unsigned iPDSrc;
111	PX86PML4E pPml4e;
112	X86PDPE Pdpe;
113	PGSTPD pPDSrc;
114
115	pPDSrc = pgmGstGetLongModePDPtr(&pVM->pgm.s, pvFault, &pPml4e, &Pdpe, &iPDSrc);
116	Assert(pPml4e);
117	# endif
118	/* Quick check for a valid guest trap. */
119	if (!pPDSrc)
120	{
121	LogFlow(("Trap0eHandler: guest PDPTR not present CR3=%VGp\n", (CPUMGetGuestCR3(pVM) & X86_CR3_PAGE_MASK)));
122	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eGuestTrap; });
123	TRPMSetErrorCode(pVM, uErr);
124	return VINF_EM_RAW_GUEST_TRAP;
125	}
126	# endif
127	# else
128	PGSTPD pPDSrc = NULL;
129	const unsigned iPDSrc = 0;
130	# endif
131
132	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
133	const unsigned iPDDst = (RTGCUINTPTR)pvFault >> SHW_PD_SHIFT;
134	PX86PD pPDDst = pVM->pgm.s.CTXMID(p,32BitPD);
135	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
136	const unsigned iPDDst = (RTGCUINTPTR)pvFault >> SHW_PD_SHIFT;
137	PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]; /* We treat this as a PD with 2048 entries, so no need to and with SHW_PD_MASK to get iPDDst */
138
139	# if PGM_GST_TYPE == PGM_TYPE_PAE
140	/* Did we mark the PDPT as not present in SyncCR3? */
141	unsigned iPDPTE = ((RTGCUINTPTR)pvFault >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
142	if (!pVM->pgm.s.CTXMID(p,PaePDPT)->a[iPDPTE].n.u1Present)
143	pVM->pgm.s.CTXMID(p,PaePDPT)->a[iPDPTE].n.u1Present = 1;
144
145	# endif
146
147	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
148	const unsigned iPDDst = (((RTGCUINTPTR)pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
149	PX86PDPAE pPDDst;
150
151	rc = PGMShwGetLongModePDPtr(pVM, (RTGCUINTPTR)pvFault, &pPDDst);
152	if (rc != VINF_SUCCESS)
153	{
154	AssertMsg(rc == VINF_PGM_SYNC_CR3, ("Unexpected rc=%Vrc\n", rc));
155	return rc;
156	}
157	Assert(pPDDst);
158	# endif
159
160	# if PGM_WITH_PAGING(PGM_GST_TYPE)
161	# ifdef PGM_SYNC_DIRTY_BIT
162	/*
163	* If we successfully correct the write protection fault due to dirty bit
164	* tracking, or this page fault is a genuine one, then return immediately.
165	*/
166	STAM_PROFILE_START(&pVM->pgm.s.StatCheckPageFault, e);
167	rc = PGM_BTH_NAME(CheckPageFault)(pVM, uErr, &pPDDst->a[iPDDst], &pPDSrc->a[iPDSrc], (RTGCUINTPTR)pvFault);
168	STAM_PROFILE_STOP(&pVM->pgm.s.StatCheckPageFault, e);
169	if ( rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
170	\|\| rc == VINF_EM_RAW_GUEST_TRAP)
171	{
172	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution)
173	= rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? &pVM->pgm.s.StatTrap0eDirtyAndAccessedBits : &pVM->pgm.s.StatTrap0eGuestTrap; });
174	LogBird(("Trap0eHandler: returns %s\n", rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? "VINF_SUCCESS" : "VINF_EM_RAW_GUEST_TRAP"));
175	return rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? VINF_SUCCESS : rc;
176	}
177	# endif
178
179	STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0ePD[iPDSrc]);
180	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
181
182	/*
183	* A common case is the not-present error caused by lazy page table syncing.
184	*
185	* It is IMPORTANT that we weed out any access to non-present shadow PDEs here
186	* so we can safely assume that the shadow PT is present when calling SyncPage later.
187	*
188	* On failure, we ASSUME that SyncPT is out of memory or detected some kind
189	* of mapping conflict and defer to SyncCR3 in R3.
190	* (Again, we do NOT support access handlers for non-present guest pages.)
191	*
192	*/
193	# if PGM_WITH_PAGING(PGM_GST_TYPE)
194	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
195	# else
196	GSTPDE PdeSrc;
197	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
198	PdeSrc.n.u1Present = 1;
199	PdeSrc.n.u1Write = 1;
200	PdeSrc.n.u1Accessed = 1;
201	PdeSrc.n.u1User = 1;
202	# endif
203	if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
204	&& !pPDDst->a[iPDDst].n.u1Present
205	&& PdeSrc.n.u1Present
206	)
207
208	{
209	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eSyncPT; });
210	STAM_PROFILE_START(&pVM->pgm.s.StatLazySyncPT, f);
211	LogFlow(("=>SyncPT %04x = %08x\n", iPDSrc, PdeSrc.au32[0]));
212	rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, (RTGCUINTPTR)pvFault);
213	if (VBOX_SUCCESS(rc))
214	{
215	STAM_PROFILE_STOP(&pVM->pgm.s.StatLazySyncPT, f);
216	return rc;
217	}
218	Log(("SyncPT: %d failed!! rc=%d\n", iPDSrc, rc));
219	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
220	STAM_PROFILE_STOP(&pVM->pgm.s.StatLazySyncPT, f);
221	return VINF_PGM_SYNC_CR3;
222	}
223
224	# if PGM_WITH_PAGING(PGM_GST_TYPE)
225	/*
226	* Check if this address is within any of our mappings.
227	*
228	* This is very fast and it's gonna save us a bit of effort below and prevent
229	* us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
230	* (BTW, it's impossible to have physical access handlers in a mapping.)
231	*/
232	if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
233	{
234	STAM_PROFILE_START(&pVM->pgm.s.StatMapping, a);
235	PPGMMAPPING pMapping = CTXALLSUFF(pVM->pgm.s.pMappings);
236	for ( ; pMapping; pMapping = CTXALLSUFF(pMapping->pNext))
237	{
238	if ((RTGCUINTPTR)pvFault < (RTGCUINTPTR)pMapping->GCPtr)
239	break;
240	if ((RTGCUINTPTR)pvFault - (RTGCUINTPTR)pMapping->GCPtr < pMapping->cb)
241	{
242	/*
243	* The first thing we check is if we've got an undetected conflict.
244	*/
245	if (!pVM->pgm.s.fMappingsFixed)
246	{
247	unsigned iPT = pMapping->cb >> GST_PD_SHIFT;
248	while (iPT-- > 0)
249	if (pPDSrc->a[iPDSrc + iPT].n.u1Present)
250	{
251	STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0eConflicts);
252	Log(("Trap0e: Detected Conflict %VGv-%VGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
253	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
254	STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
255	return VINF_PGM_SYNC_CR3;
256	}
257	}
258
259	/*
260	* Check if the fault address is in a virtual page access handler range.
261	*/
262	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->HyperVirtHandlers, pvFault);
263	if ( pCur
264	&& (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb
265	&& uErr & X86_TRAP_PF_RW)
266	{
267	# ifdef IN_GC
268	STAM_PROFILE_START(&pCur->Stat, h);
269	rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr);
270	STAM_PROFILE_STOP(&pCur->Stat, h);
271	# else
272	AssertFailed();
273	rc = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
274	# endif
275	STAM_COUNTER_INC(&pVM->pgm.s.StatTrap0eMapHandler);
276	STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
277	return rc;
278	}
279
280	/*
281	* Pretend we're not here and let the guest handle the trap.
282	*/
283	TRPMSetErrorCode(pVM, uErr & ~X86_TRAP_PF_P);
284	STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0eMap);
285	LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
286	STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
287	return VINF_EM_RAW_GUEST_TRAP;
288	}
289	}
290	STAM_PROFILE_STOP(&pVM->pgm.s.StatMapping, a);
291	} /* pgmAreMappingsEnabled(&pVM->pgm.s) */
292	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
293
294	/*
295	* Check if this fault address is flagged for special treatment,
296	* which means we'll have to figure out the physical address and
297	* check flags associated with it.
298	*
299	* ASSUME that we can limit any special access handling to pages
300	* in page tables which the guest believes to be present.
301	*/
302	if (PdeSrc.n.u1Present)
303	{
304	RTGCPHYS GCPhys = NIL_RTGCPHYS;
305
306	# if PGM_WITH_PAGING(PGM_GST_TYPE)
307	uint32_t cr4 = CPUMGetGuestCR4(pVM);
308	if ( PdeSrc.b.u1Size
309	&& (cr4 & X86_CR4_PSE))
310	GCPhys = (PdeSrc.u & GST_PDE_BIG_PG_MASK)
311	\| ((RTGCPHYS)pvFault & (GST_BIG_PAGE_OFFSET_MASK ^ PAGE_OFFSET_MASK));
312	else
313	{
314	PGSTPT pPTSrc;
315	rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
316	if (VBOX_SUCCESS(rc))
317	{
318	unsigned iPTESrc = ((RTGCUINTPTR)pvFault >> GST_PT_SHIFT) & GST_PT_MASK;
319	if (pPTSrc->a[iPTESrc].n.u1Present)
320	GCPhys = pPTSrc->a[iPTESrc].u & GST_PTE_PG_MASK;
321	}
322	}
323	# else
324	/* No paging so the fault address is the physical address */
325	GCPhys = (RTGCPHYS)((RTGCUINTPTR)pvFault & ~PAGE_OFFSET_MASK);
326	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
327
328	/*
329	* If we have a GC address we'll check if it has any flags set.
330	*/
331	if (GCPhys != NIL_RTGCPHYS)
332	{
333	STAM_PROFILE_START(&pVM->pgm.s.StatHandlers, b);
334
335	PPGMPAGE pPage;
336	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
337	if (VBOX_SUCCESS(rc))
338	{
339	if (PGM_PAGE_HAS_ANY_HANDLERS(pPage))
340	{
341	if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
342	{
343	/*
344	* Physical page access handler.
345	*/
346	const RTGCPHYS GCPhysFault = GCPhys \| ((RTGCUINTPTR)pvFault & PAGE_OFFSET_MASK);
347	PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->PhysHandlers, GCPhysFault);
348	if (pCur)
349	{
350	# ifdef PGM_SYNC_N_PAGES
351	/*
352	* If the region is write protected and we got a page not present fault, then sync
353	* the pages. If the fault was caused by a read, then restart the instruction.
354	* In case of write access continue to the GC write handler.
355	*
356	* ASSUMES that there is only one handler per page or that they have similar write properties.
357	*/
358	if ( pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
359	&& !(uErr & X86_TRAP_PF_P))
360	{
361	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
362	if ( VBOX_FAILURE(rc)
363	\|\| !(uErr & X86_TRAP_PF_RW)
364	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
365	{
366	AssertRC(rc);
367	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersOutOfSync);
368	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
369	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncHndPhys; });
370	return rc;
371	}
372	}
373	# endif
374
375	AssertMsg( pCur->enmType != PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
376	\|\| (pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE && (uErr & X86_TRAP_PF_RW)),
377	("Unexpected trap for physical handler: %08X (phys=%08x) HCPhys=%X uErr=%X, enum=%d\n", pvFault, GCPhys, pPage->HCPhys, uErr, pCur->enmType));
378
379	#if defined(IN_GC) \|\| defined(IN_RING0)
380	if (CTXALLSUFF(pCur->pfnHandler))
381	{
382	STAM_PROFILE_START(&pCur->Stat, h);
383	rc = pCur->CTXALLSUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, GCPhysFault, CTXALLSUFF(pCur->pvUser));
384	STAM_PROFILE_STOP(&pCur->Stat, h);
385	}
386	else
387	#endif
388	rc = VINF_EM_RAW_EMULATE_INSTR;
389	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersPhysical);
390	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
391	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndPhys; });
392	return rc;
393	}
394	}
395	# if PGM_WITH_PAGING(PGM_GST_TYPE)
396	else
397	{
398	# ifdef PGM_SYNC_N_PAGES
399	/*
400	* If the region is write protected and we got a page not present fault, then sync
401	* the pages. If the fault was caused by a read, then restart the instruction.
402	* In case of write access continue to the GC write handler.
403	*/
404	if ( PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) < PGM_PAGE_HNDL_PHYS_STATE_ALL
405	&& !(uErr & X86_TRAP_PF_P))
406	{
407	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
408	if ( VBOX_FAILURE(rc)
409	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
410	\|\| !(uErr & X86_TRAP_PF_RW))
411	{
412	AssertRC(rc);
413	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersOutOfSync);
414	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
415	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncHndVirt; });
416	return rc;
417	}
418	}
419	# endif
420	/*
421	* Ok, it's an virtual page access handler.
422	*
423	* Since it's faster to search by address, we'll do that first
424	* and then retry by GCPhys if that fails.
425	*/
426	/** @todo r=bird: perhaps we should consider looking up by physical address directly now? */
427	/** @note r=svl: true, but lookup on virtual address should remain as a fallback as phys & virt trees might be out of sync, because the
428	* page was changed without us noticing it (not-present -> present without invlpg or mov cr3, xxx)
429	*/
430	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->VirtHandlers, pvFault);
431	if (pCur)
432	{
433	AssertMsg(!((RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb)
434	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
435	\|\| !(uErr & X86_TRAP_PF_P)
436	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
437	("Unexpected trap for virtual handler: %VGv (phys=%VGp) HCPhys=%HGp uErr=%X, enum=%d\n", pvFault, GCPhys, pPage->HCPhys, uErr, pCur->enmType));
438
439	if ( (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb
440	&& ( uErr & X86_TRAP_PF_RW
441	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
442	{
443	# ifdef IN_GC
444	STAM_PROFILE_START(&pCur->Stat, h);
445	rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr);
446	STAM_PROFILE_STOP(&pCur->Stat, h);
447	# else
448	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
449	# endif
450	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersVirtual);
451	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
452	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndVirt; });
453	return rc;
454	}
455	/* Unhandled part of a monitored page */
456	}
457	else
458	{
459	/* Check by physical address. */
460	PPGMVIRTHANDLER pCur;
461	unsigned iPage;
462	rc = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys + ((RTGCUINTPTR)pvFault & PAGE_OFFSET_MASK),
463	&pCur, &iPage);
464	Assert(VBOX_SUCCESS(rc) \|\| !pCur);
465	if ( pCur
466	&& ( uErr & X86_TRAP_PF_RW
467	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
468	{
469	Assert((pCur->aPhysToVirt[iPage].Core.Key & X86_PTE_PAE_PG_MASK) == GCPhys);
470	# ifdef IN_GC
471	RTGCUINTPTR off = (iPage << PAGE_SHIFT) + ((RTGCUINTPTR)pvFault & PAGE_OFFSET_MASK) - ((RTGCUINTPTR)pCur->GCPtr & PAGE_OFFSET_MASK);
472	Assert(off < pCur->cb);
473	STAM_PROFILE_START(&pCur->Stat, h);
474	rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, off);
475	STAM_PROFILE_STOP(&pCur->Stat, h);
476	# else
477	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
478	# endif
479	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersVirtualByPhys);
480	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
481	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndVirt; });
482	return rc;
483	}
484	}
485	}
486	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
487
488	/*
489	* There is a handled area of the page, but this fault doesn't belong to it.
490	* We must emulate the instruction.
491	*
492	* To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
493	* we first check if this was a page-not-present fault for a page with only
494	* write access handlers. Restart the instruction if it wasn't a write access.
495	*/
496	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersUnhandled);
497
498	if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
499	&& !(uErr & X86_TRAP_PF_P))
500	{
501	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
502	if ( VBOX_FAILURE(rc)
503	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
504	\|\| !(uErr & X86_TRAP_PF_RW))
505	{
506	AssertRC(rc);
507	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersOutOfSync);
508	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
509	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncHndPhys; });
510	return rc;
511	}
512	}
513
514	/** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
515	* It's writing to an unhandled part of the LDT page several million times.
516	*/
517	rc = PGMInterpretInstruction(pVM, pRegFrame, pvFault);
518	LogFlow(("PGM: PGMInterpretInstruction -> rc=%d HCPhys=%RHp%s%s\n",
519	rc, pPage->HCPhys,
520	PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage) ? " phys" : "",
521	PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS(pPage) ? " virt" : ""));
522	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
523	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndUnhandled; });
524	return rc;
525	} /* if any kind of handler */
526
527	# if PGM_WITH_PAGING(PGM_GST_TYPE)
528	if (uErr & X86_TRAP_PF_P)
529	{
530	/*
531	* The page isn't marked, but it might still be monitored by a virtual page access handler.
532	* (ASSUMES no temporary disabling of virtual handlers.)
533	*/
534	/** @todo r=bird: Since the purpose is to catch out of sync pages with virtual handler(s) here,
535	* we should correct both the shadow page table and physical memory flags, and not only check for
536	* accesses within the handler region but for access to pages with virtual handlers. */
537	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&CTXSUFF(pVM->pgm.s.pTrees)->VirtHandlers, pvFault);
538	if (pCur)
539	{
540	AssertMsg( !((RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb)
541	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
542	\|\| !(uErr & X86_TRAP_PF_P)
543	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
544	("Unexpected trap for virtual handler: %08X (phys=%08x) HCPhys=%X uErr=%X, enum=%d\n", pvFault, GCPhys, pPage->HCPhys, uErr, pCur->enmType));
545
546	if ( (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr < pCur->cb
547	&& ( uErr & X86_TRAP_PF_RW
548	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
549	{
550	# ifdef IN_GC
551	STAM_PROFILE_START(&pCur->Stat, h);
552	rc = CTXSUFF(pCur->pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->GCPtr, (RTGCUINTPTR)pvFault - (RTGCUINTPTR)pCur->GCPtr);
553	STAM_PROFILE_STOP(&pCur->Stat, h);
554	# else
555	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
556	# endif
557	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersVirtualUnmarked);
558	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
559	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eHndVirt; });
560	return rc;
561	}
562	}
563	}
564	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
565	}
566	else
567	{
568	/* When the guest accesses invalid physical memory (e.g. probing of RAM or accessing a remapped MMIO range), then we'll fall
569	* back to the recompiler to emulate the instruction.
570	*/
571	LogFlow(("pgmPhysGetPageEx %VGp failed with %Vrc\n", GCPhys, rc));
572	STAM_COUNTER_INC(&pVM->pgm.s.StatHandlersInvalid);
573	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
574	return VINF_EM_RAW_EMULATE_INSTR;
575	}
576
577	STAM_PROFILE_STOP(&pVM->pgm.s.StatHandlers, b);
578
579	# ifdef PGM_OUT_OF_SYNC_IN_GC
580	/*
581	* We are here only if page is present in Guest page tables and trap is not handled
582	* by our handlers.
583	* Check it for page out-of-sync situation.
584	*/
585	STAM_PROFILE_START(&pVM->pgm.s.StatOutOfSync, c);
586
587	if (!(uErr & X86_TRAP_PF_P))
588	{
589	/*
590	* Page is not present in our page tables.
591	* Try to sync it!
592	* BTW, fPageShw is invalid in this branch!
593	*/
594	if (uErr & X86_TRAP_PF_US)
595	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncUser);
596	else /* supervisor */
597	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncSupervisor);
598
599	# if defined(LOG_ENABLED) && !defined(IN_RING0)
600	RTGCPHYS GCPhys;
601	uint64_t fPageGst;
602	PGMGstGetPage(pVM, pvFault, &fPageGst, &GCPhys);
603	Log(("Page out of sync: %p eip=%08x PdeSrc.n.u1User=%d fPageGst=%08llx GCPhys=%VGp scan=%d\n",
604	pvFault, pRegFrame->eip, PdeSrc.n.u1User, fPageGst, GCPhys, CSAMDoesPageNeedScanning(pVM, (RTGCPTR)pRegFrame->eip)));
605	# endif /* LOG_ENABLED */
606
607	# if PGM_WITH_PAGING(PGM_GST_TYPE) && !defined(IN_RING0)
608	if (CPUMGetGuestCPL(pVM, pRegFrame) == 0)
609	{
610	uint64_t fPageGst;
611	rc = PGMGstGetPage(pVM, pvFault, &fPageGst, NULL);
612	if ( VBOX_SUCCESS(rc)
613	&& !(fPageGst & X86_PTE_US))
614	{
615	/* Note: can't check for X86_TRAP_ID bit, because that requires execute disable support on the CPU */
616	if ( pvFault == (RTGCPTR)pRegFrame->eip
617	\|\| (RTGCUINTPTR)pvFault - pRegFrame->eip < 8 /* instruction crossing a page boundary */
618	# ifdef CSAM_DETECT_NEW_CODE_PAGES
619	\|\| ( !PATMIsPatchGCAddr(pVM, (RTGCPTR)pRegFrame->eip)
620	&& CSAMDoesPageNeedScanning(pVM, (RTGCPTR)pRegFrame->eip)) /* any new code we encounter here */
621	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
622	)
623	{
624	LogFlow(("CSAMExecFault %VGv\n", pRegFrame->eip));
625	rc = CSAMExecFault(pVM, (RTGCPTR)pRegFrame->eip);
626	if (rc != VINF_SUCCESS)
627	{
628	/*
629	* CSAM needs to perform a job in ring 3.
630	*
631	* Sync the page before going to the host context; otherwise we'll end up in a loop if
632	* CSAM fails (e.g. instruction crosses a page boundary and the next page is not present)
633	*/
634	LogFlow(("CSAM ring 3 job\n"));
635	int rc2 = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, 1, uErr);
636	AssertRC(rc2);
637
638	STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
639	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eCSAM; });
640	return rc;
641	}
642	}
643	# ifdef CSAM_DETECT_NEW_CODE_PAGES
644	else
645	if ( uErr == X86_TRAP_PF_RW
646	&& pRegFrame->ecx >= 0x100 /* early check for movswd count */
647	&& pRegFrame->ecx < 0x10000
648	)
649	{
650	/* In case of a write to a non-present supervisor shadow page, we'll take special precautions
651	* to detect loading of new code pages.
652	*/
653
654	/*
655	* Decode the instruction.
656	*/
657	RTGCPTR PC;
658	rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &PC);
659	if (rc == VINF_SUCCESS)
660	{
661	DISCPUSTATE Cpu;
662	uint32_t cbOp;
663	rc = EMInterpretDisasOneEx(pVM, (RTGCUINTPTR)PC, pRegFrame, &Cpu, &cbOp);
664
665	/* For now we'll restrict this to rep movsw/d instructions */
666	if ( rc == VINF_SUCCESS
667	&& Cpu.pCurInstr->opcode == OP_MOVSWD
668	&& (Cpu.prefix & PREFIX_REP))
669	{
670	CSAMMarkPossibleCodePage(pVM, pvFault);
671	}
672	}
673	}
674	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
675
676	/*
677	* Mark this page as safe.
678	*/
679	/** @todo not correct for pages that contain both code and data!! */
680	Log2(("CSAMMarkPage %p; scanned=%d\n", pvFault, true));
681	CSAMMarkPage(pVM, pvFault, true);
682	}
683	}
684	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) && !defined(IN_RING0) */
685	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, PGM_SYNC_NR_PAGES, uErr);
686	if (VBOX_SUCCESS(rc))
687	{
688	/* The page was successfully synced, return to the guest. */
689	STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
690	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSync; });
691	return VINF_SUCCESS;
692	}
693	}
694	else
695	{
696	/*
697	* A side effect of not flushing global PDEs are out of sync pages due
698	* to physical monitored regions, that are no longer valid.
699	* Assume for now it only applies to the read/write flag
700	*/
701	if (VBOX_SUCCESS(rc) && (uErr & X86_TRAP_PF_RW))
702	{
703	if (uErr & X86_TRAP_PF_US)
704	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncUser);
705	else /* supervisor */
706	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncSupervisor);
707
708
709	/*
710	* Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the page is not present, which is not true in this case.
711	*/
712	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)pvFault, 1, uErr);
713	if (VBOX_SUCCESS(rc))
714	{
715	/*
716	* Page was successfully synced, return to guest.
717	*/
718	# ifdef VBOX_STRICT
719	RTGCPHYS GCPhys;
720	uint64_t fPageGst;
721	rc = PGMGstGetPage(pVM, pvFault, &fPageGst, &GCPhys);
722	Assert(VBOX_SUCCESS(rc) && fPageGst & X86_PTE_RW);
723	LogFlow(("Obsolete physical monitor page out of sync %VGv - phys %VGp flags=%08llx\n", pvFault, GCPhys, (uint64_t)fPageGst));
724
725	uint64_t fPageShw;
726	rc = PGMShwGetPage(pVM, pvFault, &fPageShw, NULL);
727	Assert(VBOX_SUCCESS(rc) && fPageShw & X86_PTE_RW);
728	# endif /* VBOX_STRICT */
729	STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
730	STAM_STATS({ pVM->pgm.s.CTXSUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatTrap0eOutOfSyncObsHnd; });
731	return VINF_SUCCESS;
732	}
733
734	/* Check to see if we need to emulate the instruction as X86_CR0_WP has been cleared. */
735	if ( CPUMGetGuestCPL(pVM, pRegFrame) == 0
736	&& ((CPUMGetGuestCR0(pVM) & (X86_CR0_WP\|X86_CR0_PG)) == X86_CR0_PG)
737	&& (uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RW \| X86_TRAP_PF_P))
738	{
739	uint64_t fPageGst;
740	rc = PGMGstGetPage(pVM, pvFault, &fPageGst, NULL);
741	if ( VBOX_SUCCESS(rc)
742	&& !(fPageGst & X86_PTE_RW))
743	{
744	rc = PGMInterpretInstruction(pVM, pRegFrame, pvFault);
745	if (VBOX_SUCCESS(rc))
746	STAM_COUNTER_INC(&pVM->pgm.s.StatTrap0eWPEmulGC);
747	else
748	STAM_COUNTER_INC(&pVM->pgm.s.StatTrap0eWPEmulR3);
749	return rc;
750	}
751	else
752	AssertMsgFailed(("Unexpected r/w page %x flag=%x\n", pvFault, (uint32_t)fPageGst));
753	}
754
755	}
756
757	# if PGM_WITH_PAGING(PGM_GST_TYPE)
758	# ifdef VBOX_STRICT
759	/*
760	* Check for VMM page flags vs. Guest page flags consistency.
761	* Currently only for debug purposes.
762	*/
763	if (VBOX_SUCCESS(rc))
764	{
765	/* Get guest page flags. */
766	uint64_t fPageGst;
767	rc = PGMGstGetPage(pVM, pvFault, &fPageGst, NULL);
768	if (VBOX_SUCCESS(rc))
769	{
770	uint64_t fPageShw;
771	rc = PGMShwGetPage(pVM, pvFault, &fPageShw, NULL);
772
773	/*
774	* Compare page flags.
775	* Note: we have AVL, A, D bits desynched.
776	*/
777	AssertMsg((fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)),
778	("Page flags mismatch! pvFault=%p GCPhys=%VGp fPageShw=%08llx fPageGst=%08llx\n", pvFault, GCPhys, fPageShw, fPageGst));
779	}
780	else
781	AssertMsgFailed(("PGMGstGetPage rc=%Vrc\n", rc));
782	}
783	else
784	AssertMsgFailed(("PGMGCGetPage rc=%Vrc\n", rc));
785	# endif /* VBOX_STRICT */
786	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
787	}
788	STAM_PROFILE_STOP(&pVM->pgm.s.StatOutOfSync, c);
789	# endif /* PGM_OUT_OF_SYNC_IN_GC */
790	}
791	else
792	{
793	/*
794	* Page not present in Guest OS or invalid page table address.
795	* This is potential virtual page access handler food.
796	*
797	* For the present we'll say that our access handlers don't
798	* work for this case - we've already discarded the page table
799	* not present case which is identical to this.
800	*
801	* When we perchance find we need this, we will probably have AVL
802	* trees (offset based) to operate on and we can measure their speed
803	* agains mapping a page table and probably rearrange this handling
804	* a bit. (Like, searching virtual ranges before checking the
805	* physical address.)
806	*/
807	}
808	}
809
810
811	# if PGM_WITH_PAGING(PGM_GST_TYPE)
812	/*
813	* Conclusion, this is a guest trap.
814	*/
815	LogFlow(("PGM: Unhandled #PF -> route trap to recompiler!\n"));
816	STAM_COUNTER_INC(&pVM->pgm.s.StatGCTrap0eUnhandled);
817	return VINF_EM_RAW_GUEST_TRAP;
818	# else
819	/* present, but not a monitored page; perhaps the guest is probing physical memory */
820	return VINF_EM_RAW_EMULATE_INSTR;
821	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
822
823
824	#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
825
826	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
827	return VERR_INTERNAL_ERROR;
828	#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
829	}
830
831
832	/**
833	* Emulation of the invlpg instruction.
834	*
835	*
836	* @returns VBox status code.
837	*
838	* @param pVM VM handle.
839	* @param GCPtrPage Page to invalidate.
840	*
841	* @remark ASSUMES that the guest is updating before invalidating. This order
842	* isn't required by the CPU, so this is speculative and could cause
843	* trouble.
844	*
845	* @todo Flush page or page directory only if necessary!
846	* @todo Add a #define for simply invalidating the page.
847	*/
848	PGM_BTH_DECL(int, InvalidatePage)(PVM pVM, RTGCUINTPTR GCPtrPage)
849	{
850	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
851	\|\| PGM_GST_TYPE == PGM_TYPE_PAE
852
853	LogFlow(("InvalidatePage %x\n", GCPtrPage));
854	/*
855	* Get the shadow PD entry and skip out if this PD isn't present.
856	* (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
857	*/
858	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
859	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
860	PX86PDE pPdeDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst];
861	# else
862	PX86PDEPAE pPdeDst = &pVM->pgm.s.CTXMID(ap,PaePDs[0])->a[iPDDst];
863	# endif
864	const SHWPDE PdeDst = *pPdeDst;
865	if (!PdeDst.n.u1Present)
866	{
867	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePageSkipped));
868	return VINF_SUCCESS;
869	}
870
871	/*
872	* Get the guest PD entry and calc big page.
873	*/
874	# if PGM_GST_TYPE == PGM_TYPE_32BIT
875	PX86PD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
876	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
877	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
878	# else /* PAE */
879	unsigned iPDSrc;
880	PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, GCPtrPage, &iPDSrc);
881	GSTPDE PdeSrc;
882
883	if (pPDSrc)
884	PdeSrc = pPDSrc->a[iPDSrc];
885	else
886	PdeSrc.u = 0;
887	# endif
888
889	const uint32_t cr4 = CPUMGetGuestCR4(pVM);
890	const bool fIsBigPage = PdeSrc.b.u1Size && (cr4 & X86_CR4_PSE);
891
892	# ifdef IN_RING3
893	/*
894	* If a CR3 Sync is pending we may ignore the invalidate page operation
895	* depending on the kind of sync and if it's a global page or not.
896	* This doesn't make sense in GC/R0 so we'll skip it entirely there.
897	*/
898	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
899	if ( VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3)
900	\|\| ( VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3_NON_GLOBAL)
901	&& fIsBigPage
902	&& PdeSrc.b.u1Global
903	&& (cr4 & X86_CR4_PGE)
904	)
905	)
906	# else
907	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
908	# endif
909	{
910	STAM_COUNTER_INC(&pVM->pgm.s.StatHCInvalidatePageSkipped);
911	return VINF_SUCCESS;
912	}
913	# endif /* IN_RING3 */
914
915
916	/*
917	* Deal with the Guest PDE.
918	*/
919	int rc = VINF_SUCCESS;
920	if (PdeSrc.n.u1Present)
921	{
922	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
923	{
924	/*
925	* Conflict - Let SyncPT deal with it to avoid duplicate code.
926	*/
927	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
928	Assert(PGMGetGuestMode(pVM) <= PGMMODE_32_BIT);
929	rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, GCPtrPage);
930	}
931	else if ( PdeSrc.n.u1User != PdeDst.n.u1User
932	\|\| (!PdeSrc.n.u1Write && PdeDst.n.u1Write))
933	{
934	/*
935	* Mark not present so we can resync the PDE when it's used.
936	*/
937	LogFlow(("InvalidatePage: Out-of-sync at %VGp PdeSrc=%RX64 PdeDst=%RX64\n",
938	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
939	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
940	pPdeDst->u = 0;
941	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDOutOfSync));
942	PGM_INVL_GUEST_TLBS();
943	}
944	# ifdef PGM_SYNC_ACCESSED_BIT
945	else if (!PdeSrc.n.u1Accessed)
946	{
947	/*
948	* Mark not present so we can set the accessed bit.
949	*/
950	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
951	pPdeDst->u = 0;
952	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNAs));
953	PGM_INVL_GUEST_TLBS();
954	}
955	# endif
956	else if (!fIsBigPage)
957	{
958	/*
959	* 4KB - page.
960	*/
961	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
962	RTGCPHYS GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
963	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
964	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
965	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE/2);
966	# endif
967	if (pShwPage->GCPhys == GCPhys)
968	{
969	# if 0 /* likely cause of a major performance regression; must be SyncPageWorkerTrackDeref then */
970	const unsigned iPTEDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
971	PSHWPT pPT = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
972	if (pPT->a[iPTEDst].n.u1Present)
973	{
974	# ifdef PGMPOOL_WITH_USER_TRACKING
975	/* This is very unlikely with caching/monitoring enabled. */
976	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPT->a[iPTEDst].u & SHW_PTE_PG_MASK);
977	# endif
978	pPT->a[iPTEDst].u = 0;
979	}
980	# else /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
981	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, GCPtrPage, 1, 0);
982	if (VBOX_SUCCESS(rc))
983	rc = VINF_SUCCESS;
984	# endif
985	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePage4KBPages));
986	PGM_INVL_PG(GCPtrPage);
987	}
988	else
989	{
990	/*
991	* The page table address changed.
992	*/
993	LogFlow(("InvalidatePage: Out-of-sync at %VGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%VGp iPDDst=%#x\n",
994	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
995	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
996	pPdeDst->u = 0;
997	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDOutOfSync));
998	PGM_INVL_GUEST_TLBS();
999	}
1000	}
1001	else
1002	{
1003	/*
1004	* 2/4MB - page.
1005	*/
1006	/* Before freeing the page, check if anything really changed. */
1007	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1008	RTGCPHYS GCPhys = PdeSrc.u & GST_PDE_BIG_PG_MASK;
1009	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1010	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1011	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1012	# endif
1013	if ( pShwPage->GCPhys == GCPhys
1014	&& pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1015	{
1016	/* ASSUMES a the given bits are identical for 4M and normal PDEs */
1017	/** @todo PAT */
1018	# ifdef PGM_SYNC_DIRTY_BIT
1019	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
1020	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
1021	&& ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1022	\|\| (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1023	# else
1024	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
1025	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD)))
1026	# endif
1027	{
1028	LogFlow(("Skipping flush for big page containing %VGv (PD=%X .u=%VX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1029	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePage4MBPagesSkip));
1030	return VINF_SUCCESS;
1031	}
1032	}
1033
1034	/*
1035	* Ok, the page table is present and it's been changed in the guest.
1036	* If we're in host context, we'll just mark it as not present taking the lazy approach.
1037	* We could do this for some flushes in GC too, but we need an algorithm for
1038	* deciding which 4MB pages containing code likely to be executed very soon.
1039	*/
1040	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1041	pPdeDst->u = 0;
1042	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePage4MBPages));
1043	PGM_INVL_BIG_PG(GCPtrPage);
1044	}
1045	}
1046	else
1047	{
1048	/*
1049	* Page directory is not present, mark shadow PDE not present.
1050	*/
1051	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
1052	{
1053	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1054	pPdeDst->u = 0;
1055	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDNPs));
1056	PGM_INVL_PG(GCPtrPage);
1057	}
1058	else
1059	{
1060	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1061	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,InvalidatePagePDMappings));
1062	}
1063	}
1064
1065	return rc;
1066
1067	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
1068	//# error not implemented
1069	return VERR_INTERNAL_ERROR;
1070
1071	#else /* guest real and protected mode */
1072	/* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1073	return VINF_SUCCESS;
1074	#endif
1075	}
1076
1077
1078	#ifdef PGMPOOL_WITH_USER_TRACKING
1079	/**
1080	* Update the tracking of shadowed pages.
1081	*
1082	* @param pVM The VM handle.
1083	* @param pShwPage The shadow page.
1084	* @param HCPhys The physical page we is being dereferenced.
1085	*/
1086	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVM pVM, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys)
1087	{
1088	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1089	STAM_PROFILE_START(&pVM->pgm.s.StatTrackDeref, a);
1090	LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%VHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1091
1092	/** @todo If this turns out to be a bottle neck (very likely) two things can be done:
1093	* 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1094	* 2. write protect all shadowed pages. I.e. implement caching.
1095	*/
1096	/*
1097	* Find the guest address.
1098	*/
1099	for (PPGMRAMRANGE pRam = CTXALLSUFF(pVM->pgm.s.pRamRanges);
1100	pRam;
1101	pRam = CTXALLSUFF(pRam->pNext))
1102	{
1103	unsigned iPage = pRam->cb >> PAGE_SHIFT;
1104	while (iPage-- > 0)
1105	{
1106	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1107	{
1108	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
1109	pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage]);
1110	pShwPage->cPresent--;
1111	pPool->cPresent--;
1112	STAM_PROFILE_STOP(&pVM->pgm.s.StatTrackDeref, a);
1113	return;
1114	}
1115	}
1116	}
1117
1118	for (;;)
1119	AssertReleaseMsgFailed(("HCPhys=%VHp wasn't found!\n", HCPhys));
1120	# else /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1121	pShwPage->cPresent--;
1122	pVM->pgm.s.CTXSUFF(pPool)->cPresent--;
1123	# endif /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1124	}
1125
1126
1127	/**
1128	* Update the tracking of shadowed pages.
1129	*
1130	* @param pVM The VM handle.
1131	* @param pShwPage The shadow page.
1132	* @param u16 The top 16-bit of the pPage->HCPhys.
1133	* @param pPage Pointer to the guest page. this will be modified.
1134	* @param iPTDst The index into the shadow table.
1135	*/
1136	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVM pVM, PPGMPOOLPAGE pShwPage, uint16_t u16, PPGMPAGE pPage, const unsigned iPTDst)
1137	{
1138	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1139	/*
1140	* We're making certain assumptions about the placement of cRef and idx.
1141	*/
1142	Assert(MM_RAM_FLAGS_IDX_SHIFT == 48);
1143	Assert(MM_RAM_FLAGS_CREFS_SHIFT > MM_RAM_FLAGS_IDX_SHIFT);
1144
1145	/*
1146	* Just deal with the simple first time here.
1147	*/
1148	if (!u16)
1149	{
1150	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackVirgin);
1151	u16 = (1 << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT)) \| pShwPage->idx;
1152	}
1153	else
1154	u16 = pgmPoolTrackPhysExtAddref(pVM, u16, pShwPage->idx);
1155
1156	/* write back, trying to be clever... */
1157	Log2(("SyncPageWorkerTrackAddRef: u16=%#x pPage->HCPhys=%VHp->%VHp iPTDst=%#x\n",
1158	u16, pPage->HCPhys, (pPage->HCPhys & MM_RAM_FLAGS_NO_REFS_MASK) \| ((uint64_t)u16 << MM_RAM_FLAGS_CREFS_SHIFT), iPTDst));
1159	((uint16_t )&pPage->HCPhys + 3) = u16; /** @todo PAGE FLAGS */
1160	# endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
1161
1162	/* update statistics. */
1163	pVM->pgm.s.CTXSUFF(pPool)->cPresent++;
1164	pShwPage->cPresent++;
1165	if (pShwPage->iFirstPresent > iPTDst)
1166	pShwPage->iFirstPresent = iPTDst;
1167	}
1168	#endif /* PGMPOOL_WITH_USER_TRACKING */
1169
1170
1171	/**
1172	* Creates a 4K shadow page for a guest page.
1173	*
1174	* For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1175	* physical address. The PdeSrc argument only the flags are used. No page structured
1176	* will be mapped in this function.
1177	*
1178	* @param pVM VM handle.
1179	* @param pPteDst Destination page table entry.
1180	* @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1181	* Can safely assume that only the flags are being used.
1182	* @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1183	* @param pShwPage Pointer to the shadow page.
1184	* @param iPTDst The index into the shadow table.
1185	*
1186	* @remark Not used for 2/4MB pages!
1187	*/
1188	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorker)(PVM pVM, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1189	{
1190	if (PteSrc.n.u1Present)
1191	{
1192	/*
1193	* Find the ram range.
1194	*/
1195	PPGMPAGE pPage;
1196	int rc = pgmPhysGetPageEx(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK, &pPage);
1197	if (VBOX_SUCCESS(rc))
1198	{
1199	/** @todo investiage PWT, PCD and PAT. */
1200	/*
1201	* Make page table entry.
1202	*/
1203	const RTHCPHYS HCPhys = pPage->HCPhys; /** @todo FLAGS */
1204	SHWPTE PteDst;
1205	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1206	{
1207	/** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No. */
1208	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1209	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1210	\| (HCPhys & X86_PTE_PAE_PG_MASK);
1211	else
1212	{
1213	LogFlow(("SyncPageWorker: monitored page (%VGp) -> mark not present\n", HCPhys));
1214	PteDst.u = 0;
1215	}
1216	/** @todo count these two kinds. */
1217	}
1218	else
1219	{
1220	#ifdef PGM_SYNC_DIRTY_BIT
1221	# ifdef PGM_SYNC_ACCESSED_BIT
1222	/*
1223	* If the page or page directory entry is not marked accessed,
1224	* we mark the page not present.
1225	*/
1226	if (!PteSrc.n.u1Accessed \|\| !PdeSrc.n.u1Accessed)
1227	{
1228	LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1229	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,AccessedPage));
1230	PteDst.u = 0;
1231	}
1232	else
1233	# endif
1234	/*
1235	* If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1236	* when the page is modified.
1237	*/
1238	if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
1239	{
1240	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPage));
1241	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1242	\| (HCPhys & X86_PTE_PAE_PG_MASK)
1243	\| PGM_PTFLAGS_TRACK_DIRTY;
1244	}
1245	else
1246	{
1247	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageSkipped));
1248	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1249	\| (HCPhys & X86_PTE_PAE_PG_MASK);
1250	}
1251	#endif
1252	}
1253
1254	#ifdef PGMPOOL_WITH_USER_TRACKING
1255	/*
1256	* Keep user track up to date.
1257	*/
1258	if (PteDst.n.u1Present)
1259	{
1260	if (!pPteDst->n.u1Present)
1261	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst);
1262	else if ((pPteDst->u & SHW_PTE_PG_MASK) != (PteDst.u & SHW_PTE_PG_MASK))
1263	{
1264	Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", (uint64_t)pPteDst->u, (uint64_t)PteDst.u));
1265	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1266	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst);
1267	}
1268	}
1269	else if (pPteDst->n.u1Present)
1270	{
1271	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1272	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1273	}
1274	#endif /* PGMPOOL_WITH_USER_TRACKING */
1275
1276	/*
1277	* Update statistics and commit the entry.
1278	*/
1279	if (!PteSrc.n.u1Global)
1280	pShwPage->fSeenNonGlobal = true;
1281	*pPteDst = PteDst;
1282	}
1283	/* else MMIO or invalid page, we must handle them manually in the #PF handler. */
1284	/** @todo count these. */
1285	}
1286	else
1287	{
1288	/*
1289	* Page not-present.
1290	*/
1291	LogFlow(("SyncPageWorker: page not present in Pte\n"));
1292	#ifdef PGMPOOL_WITH_USER_TRACKING
1293	/* Keep user track up to date. */
1294	if (pPteDst->n.u1Present)
1295	{
1296	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1297	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVM, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1298	}
1299	#endif /* PGMPOOL_WITH_USER_TRACKING */
1300	pPteDst->u = 0;
1301	/** @todo count these. */
1302	}
1303	}
1304
1305
1306	/**
1307	* Syncs a guest OS page.
1308	*
1309	* There are no conflicts at this point, neither is there any need for
1310	* page table allocations.
1311	*
1312	* @returns VBox status code.
1313	* @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1314	* @param pVM VM handle.
1315	* @param PdeSrc Page directory entry of the guest.
1316	* @param GCPtrPage Guest context page address.
1317	* @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1318	* @param uErr Fault error (X86_TRAP_PF_*).
1319	*/
1320	PGM_BTH_DECL(int, SyncPage)(PVM pVM, GSTPDE PdeSrc, RTGCUINTPTR GCPtrPage, unsigned cPages, unsigned uErr)
1321	{
1322	LogFlow(("SyncPage: GCPtrPage=%VGv cPages=%d uErr=%#x\n", GCPtrPage, cPages, uErr));
1323
1324	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
1325	\|\| PGM_GST_TYPE == PGM_TYPE_PAE
1326
1327	# if PGM_WITH_NX(PGM_GST_TYPE)
1328	bool fNoExecuteBitValid = !!(CPUMGetGuestEFER(pVM) & MSR_K6_EFER_NXE);
1329	# endif
1330
1331	/*
1332	* Assert preconditions.
1333	*/
1334	STAM_COUNTER_INC(&pVM->pgm.s.StatGCSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
1335	Assert(PdeSrc.n.u1Present);
1336	Assert(cPages);
1337
1338	/*
1339	* Get the shadow PDE, find the shadow page table in the pool.
1340	*/
1341	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
1342	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1343	X86PDE PdeDst = pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst];
1344	# else /* PAE */
1345	X86PDEPAE PdeDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst];
1346	# endif
1347	Assert(PdeDst.n.u1Present);
1348	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1349
1350	/*
1351	* Check that the page is present and that the shadow PDE isn't out of sync.
1352	*/
1353	const bool fBigPage = PdeSrc.b.u1Size && (CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
1354	RTGCPHYS GCPhys;
1355	if (!fBigPage)
1356	{
1357	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1358	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1359	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1360	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE/2);
1361	# endif
1362	}
1363	else
1364	{
1365	GCPhys = PdeSrc.u & GST_PDE_BIG_PG_MASK;
1366	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1367	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1368	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1369	# endif
1370	}
1371	if ( pShwPage->GCPhys == GCPhys
1372	&& PdeSrc.n.u1Present
1373	&& (PdeSrc.n.u1User == PdeDst.n.u1User)
1374	&& (PdeSrc.n.u1Write == PdeDst.n.u1Write \|\| !PdeDst.n.u1Write)
1375	# if PGM_WITH_NX(PGM_GST_TYPE)
1376	&& (!fNoExecuteBitValid \|\| PdeSrc.n.u1NoExecute == PdeDst.n.u1NoExecute)
1377	# endif
1378	)
1379	{
1380	# ifdef PGM_SYNC_ACCESSED_BIT
1381	/*
1382	* Check that the PDE is marked accessed already.
1383	* Since we set the accessed bit before getting here on a #PF, this
1384	* check is only meant for dealing with non-#PF'ing paths.
1385	*/
1386	if (PdeSrc.n.u1Accessed)
1387	# endif
1388	{
1389	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1390	if (!fBigPage)
1391	{
1392	/*
1393	* 4KB Page - Map the guest page table.
1394	*/
1395	PGSTPT pPTSrc;
1396	int rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
1397	if (VBOX_SUCCESS(rc))
1398	{
1399	# ifdef PGM_SYNC_N_PAGES
1400	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
1401	if (cPages > 1 && !(uErr & X86_TRAP_PF_P))
1402	{
1403	/*
1404	* This code path is currently only taken when the caller is PGMTrap0eHandler
1405	* for non-present pages!
1406	*
1407	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1408	* deal with locality.
1409	*/
1410	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1411	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1412	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1413	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1414	# else
1415	const unsigned offPTSrc = 0;
1416	# endif
1417	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, ELEMENTS(pPTDst->a));
1418	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1419	iPTDst = 0;
1420	else
1421	iPTDst -= PGM_SYNC_NR_PAGES / 2;
1422	for (; iPTDst < iPTDstEnd; iPTDst++)
1423	{
1424	if (!pPTDst->a[iPTDst].n.u1Present)
1425	{
1426	GSTPTE PteSrc = pPTSrc->a[offPTSrc + iPTDst];
1427	RTGCUINTPTR GCPtrCurPage = ((RTGCUINTPTR)GCPtrPage & ~(RTGCUINTPTR)(GST_PT_MASK << GST_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
1428	NOREF(GCPtrCurPage);
1429	#ifndef IN_RING0
1430	/*
1431	* Assuming kernel code will be marked as supervisor - and not as user level
1432	* and executed using a conforming code selector - And marked as readonly.
1433	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
1434	*/
1435	PPGMPAGE pPage;
1436	if ( ((PdeSrc.u & PteSrc.u) & (X86_PTE_RW \| X86_PTE_US))
1437	\|\| iPTDst == ((GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK) /* always sync GCPtrPage */
1438	\|\| !CSAMDoesPageNeedScanning(pVM, (RTGCPTR)GCPtrCurPage)
1439	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
1440	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1441	)
1442	#endif /* else: CSAM not active */
1443	PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1444	Log2(("SyncPage: 4K+ %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1445	GCPtrCurPage, PteSrc.n.u1Present,
1446	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1447	PteSrc.n.u1User & PdeSrc.n.u1User,
1448	(uint64_t)PteSrc.u,
1449	(uint64_t)pPTDst->a[iPTDst].u,
1450	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1451	}
1452	}
1453	}
1454	else
1455	# endif /* PGM_SYNC_N_PAGES */
1456	{
1457	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1458	GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1459	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1460	PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1461	Log2(("SyncPage: 4K %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s\n",
1462	GCPtrPage, PteSrc.n.u1Present,
1463	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1464	PteSrc.n.u1User & PdeSrc.n.u1User,
1465	(uint64_t)PteSrc.u,
1466	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1467	}
1468	}
1469	else /* MMIO or invalid page: emulated in #PF handler. */
1470	{
1471	LogFlow(("PGM_GCPHYS_2_PTR %VGp failed with %Vrc\n", GCPhys, rc));
1472	Assert(!pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK].n.u1Present);
1473	}
1474	}
1475	else
1476	{
1477	/*
1478	* 4/2MB page - lazy syncing shadow 4K pages.
1479	* (There are many causes of getting here, it's no longer only CSAM.)
1480	*/
1481	/* Calculate the GC physical address of this 4KB shadow page. */
1482	RTGCPHYS GCPhys = (PdeSrc.u & GST_PDE_BIG_PG_MASK) \| ((RTGCUINTPTR)GCPtrPage & GST_BIG_PAGE_OFFSET_MASK);
1483	/* Find ram range. */
1484	PPGMPAGE pPage;
1485	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
1486	if (VBOX_SUCCESS(rc))
1487	{
1488	/*
1489	* Make shadow PTE entry.
1490	*/
1491	const RTHCPHYS HCPhys = pPage->HCPhys; /** @todo PAGE FLAGS */
1492	SHWPTE PteDst;
1493	PteDst.u = (PdeSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1494	\| (HCPhys & X86_PTE_PAE_PG_MASK);
1495	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1496	{
1497	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1498	PteDst.n.u1Write = 0;
1499	else
1500	PteDst.u = 0;
1501	}
1502	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1503	# ifdef PGMPOOL_WITH_USER_TRACKING
1504	if (PteDst.n.u1Present && !pPTDst->a[iPTDst].n.u1Present)
1505	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst);
1506	# endif
1507	pPTDst->a[iPTDst] = PteDst;
1508
1509
1510	# ifdef PGM_SYNC_DIRTY_BIT
1511	/*
1512	* If the page is not flagged as dirty and is writable, then make it read-only
1513	* at PD level, so we can set the dirty bit when the page is modified.
1514	*
1515	* ASSUMES that page access handlers are implemented on page table entry level.
1516	* Thus we will first catch the dirty access and set PDE.D and restart. If
1517	* there is an access handler, we'll trap again and let it work on the problem.
1518	*/
1519	/** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
1520	* As for invlpg, it simply frees the whole shadow PT.
1521	* ...It's possibly because the guest clears it and the guest doesn't really tell us... */
1522	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
1523	{
1524	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageBig));
1525	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
1526	PdeDst.n.u1Write = 0;
1527	}
1528	else
1529	{
1530	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
1531	PdeDst.n.u1Write = PdeSrc.n.u1Write;
1532	}
1533	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1534	pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst] = PdeDst;
1535	# else /* PAE */
1536	pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst] = PdeDst;
1537	# endif
1538	# endif /* PGM_SYNC_DIRTY_BIT */
1539	Log2(("SyncPage: BIG %VGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%VGp%s\n",
1540	GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
1541	PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1542	}
1543	else
1544	LogFlow(("PGM_GCPHYS_2_PTR %VGp (big) failed with %Vrc\n", GCPhys, rc));
1545	}
1546	return VINF_SUCCESS;
1547	}
1548	# ifdef PGM_SYNC_ACCESSED_BIT
1549	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncPagePDNAs));
1550	#endif
1551	}
1552	else
1553	{
1554	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncPagePDOutOfSync));
1555	Log2(("SyncPage: Out-Of-Sync PDE at %VGp PdeSrc=%RX64 PdeDst=%RX64\n",
1556	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1557	}
1558
1559	/*
1560	* Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
1561	* Yea, I'm lazy.
1562	*/
1563	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, SHW_POOL_ROOT_IDX, iPDDst);
1564	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1565	pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst].u = 0;
1566	# else /* PAE */
1567	pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst].u = 0;
1568	# endif
1569	PGM_INVL_GUEST_TLBS();
1570	return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
1571
1572	#elif PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT
1573
1574	# ifdef PGM_SYNC_N_PAGES
1575	/*
1576	* Get the shadow PDE, find the shadow page table in the pool.
1577	*/
1578	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
1579	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1580	X86PDE PdeDst = pVM->pgm.s.CTXMID(p,32BitPD)->a[iPDDst];
1581	# else /* PAE */
1582	X86PDEPAE PdeDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[iPDDst];
1583	# endif
1584	Assert(PdeDst.n.u1Present);
1585	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, PdeDst.u & SHW_PDE_PG_MASK);
1586	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1587
1588	# if PGM_SHW_TYPE == PGM_TYPE_PAE
1589	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1590	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1591	# else
1592	const unsigned offPTSrc = 0;
1593	# endif
1594
1595	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
1596	if (cPages > 1 && !(uErr & X86_TRAP_PF_P))
1597	{
1598	/*
1599	* This code path is currently only taken when the caller is PGMTrap0eHandler
1600	* for non-present pages!
1601	*
1602	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1603	* deal with locality.
1604	*/
1605	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1606	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, ELEMENTS(pPTDst->a));
1607	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1608	iPTDst = 0;
1609	else
1610	iPTDst -= PGM_SYNC_NR_PAGES / 2;
1611	for (; iPTDst < iPTDstEnd; iPTDst++)
1612	{
1613	if (!pPTDst->a[iPTDst].n.u1Present)
1614	{
1615	GSTPTE PteSrc;
1616
1617	RTGCUINTPTR GCPtrCurPage = ((RTGCUINTPTR)GCPtrPage & ~(RTGCUINTPTR)(GST_PT_MASK << GST_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
1618
1619	/* Fake the page table entry */
1620	PteSrc.u = GCPtrCurPage;
1621	PteSrc.n.u1Present = 1;
1622	PteSrc.n.u1Dirty = 1;
1623	PteSrc.n.u1Accessed = 1;
1624	PteSrc.n.u1Write = 1;
1625	PteSrc.n.u1User = 1;
1626
1627	PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1628
1629	Log2(("SyncPage: 4K+ %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1630	GCPtrCurPage, PteSrc.n.u1Present,
1631	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1632	PteSrc.n.u1User & PdeSrc.n.u1User,
1633	(uint64_t)PteSrc.u,
1634	(uint64_t)pPTDst->a[iPTDst].u,
1635	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1636	}
1637	}
1638	}
1639	else
1640	# endif /* PGM_SYNC_N_PAGES */
1641	{
1642	GSTPTE PteSrc;
1643	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1644	RTGCUINTPTR GCPtrCurPage = ((RTGCUINTPTR)GCPtrPage & ~(RTGCUINTPTR)(GST_PT_MASK << GST_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
1645
1646	/* Fake the page table entry */
1647	PteSrc.u = GCPtrCurPage;
1648	PteSrc.n.u1Present = 1;
1649	PteSrc.n.u1Dirty = 1;
1650	PteSrc.n.u1Accessed = 1;
1651	PteSrc.n.u1Write = 1;
1652	PteSrc.n.u1User = 1;
1653	PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1654
1655	Log2(("SyncPage: 4K %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s\n",
1656	GCPtrPage, PteSrc.n.u1Present,
1657	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1658	PteSrc.n.u1User & PdeSrc.n.u1User,
1659	(uint64_t)PteSrc.u,
1660	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1661	}
1662	return VINF_SUCCESS;
1663
1664	#else /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
1665	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
1666	return VERR_INTERNAL_ERROR;
1667	#endif /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
1668	}
1669
1670
1671
1672	#if PGM_WITH_PAGING(PGM_GST_TYPE)
1673
1674	/**
1675	* Investigate page fault and handle write protection page faults caused by
1676	* dirty bit tracking.
1677	*
1678	* @returns VBox status code.
1679	* @param pVM VM handle.
1680	* @param uErr Page fault error code.
1681	* @param pPdeDst Shadow page directory entry.
1682	* @param pPdeSrc Guest page directory entry.
1683	* @param GCPtrPage Guest context page address.
1684	*/
1685	PGM_BTH_DECL(int, CheckPageFault)(PVM pVM, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCUINTPTR GCPtrPage)
1686	{
1687	bool fWriteProtect = !!(CPUMGetGuestCR0(pVM) & X86_CR0_WP);
1688	bool fUserLevelFault = !!(uErr & X86_TRAP_PF_US);
1689	bool fWriteFault = !!(uErr & X86_TRAP_PF_RW);
1690	bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
1691	# if PGM_WITH_NX(PGM_GST_TYPE)
1692	bool fNoExecuteBitValid = !!(CPUMGetGuestEFER(pVM) & MSR_K6_EFER_NXE);
1693	# endif
1694	unsigned uPageFaultLevel;
1695	int rc;
1696
1697	STAM_PROFILE_START(&pVM->pgm.s.CTXMID(Stat, DirtyBitTracking), a);
1698	LogFlow(("CheckPageFault: GCPtrPage=%VGv uErr=%#x PdeSrc=%08x\n", GCPtrPage, uErr, pPdeSrc->u));
1699
1700	# if PGM_GST_TYPE == PGM_TYPE_PAE \
1701	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
1702
1703	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1704	PX86PML4E pPml4eSrc;
1705	PX86PDPE pPdpeSrc;
1706
1707	pPdpeSrc = pgmGstGetLongModePDPTPtr(&pVM->pgm.s, GCPtrPage, &pPml4eSrc);
1708	Assert(pPml4eSrc);
1709
1710	/*
1711	* Real page fault? (PML4E level)
1712	*/
1713	if ( (uErr & X86_TRAP_PF_RSVD)
1714	\|\| !pPml4eSrc->n.u1Present
1715	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPml4eSrc->n.u1NoExecute)
1716	\|\| (fWriteFault && !pPml4eSrc->n.u1Write && (fUserLevelFault \|\| fWriteProtect))
1717	\|\| (fUserLevelFault && !pPml4eSrc->n.u1User)
1718	)
1719	{
1720	uPageFaultLevel = 0;
1721	goto UpperLevelPageFault;
1722	}
1723	Assert(pPdpeSrc);
1724
1725	# else /* PAE */
1726	PX86PDPE pPdpeSrc = &pVM->pgm.s.CTXSUFF(pGstPaePDPT)->a[(GCPtrPage >> GST_PDPT_SHIFT) & GST_PDPT_MASK];
1727	# endif
1728
1729	/*
1730	* Real page fault? (PDPE level)
1731	*/
1732	if ( (uErr & X86_TRAP_PF_RSVD)
1733	\|\| !pPdpeSrc->n.u1Present
1734	# if PGM_GST_TYPE == PGM_TYPE_AMD64 /* NX, r/w, u/s bits in the PDPE are long mode only */
1735	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPdpeSrc->lm.u1NoExecute)
1736	\|\| (fWriteFault && !pPdpeSrc->lm.u1Write && (fUserLevelFault \|\| fWriteProtect))
1737	\|\| (fUserLevelFault && !pPdpeSrc->lm.u1User)
1738	# endif
1739	)
1740	{
1741	uPageFaultLevel = 1;
1742	goto UpperLevelPageFault;
1743	}
1744	# endif
1745
1746	/*
1747	* Real page fault? (PDE level)
1748	*/
1749	if ( (uErr & X86_TRAP_PF_RSVD)
1750	\|\| !pPdeSrc->n.u1Present
1751	# if PGM_WITH_NX(PGM_GST_TYPE)
1752	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPdeSrc->n.u1NoExecute)
1753	# endif
1754	\|\| (fWriteFault && !pPdeSrc->n.u1Write && (fUserLevelFault \|\| fWriteProtect))
1755	\|\| (fUserLevelFault && !pPdeSrc->n.u1User) )
1756	{
1757	uPageFaultLevel = 2;
1758	goto UpperLevelPageFault;
1759	}
1760
1761	/*
1762	* First check the easy case where the page directory has been marked read-only to track
1763	* the dirty bit of an emulated BIG page
1764	*/
1765	if (pPdeSrc->b.u1Size && fBigPagesSupported)
1766	{
1767	/* Mark guest page directory as accessed */
1768	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1769	pPml4eSrc->n.u1Accessed = 1;
1770	pPdpeSrc->lm.u1Accessed = 1;
1771	# endif
1772	pPdeSrc->b.u1Accessed = 1;
1773
1774	/*
1775	* Only write protection page faults are relevant here.
1776	*/
1777	if (fWriteFault)
1778	{
1779	/* Mark guest page directory as dirty (BIG page only). */
1780	pPdeSrc->b.u1Dirty = 1;
1781
1782	if (pPdeDst->n.u1Present && (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY))
1783	{
1784	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageTrap));
1785
1786	Assert(pPdeSrc->b.u1Write);
1787
1788	pPdeDst->n.u1Write = 1;
1789	pPdeDst->n.u1Accessed = 1;
1790	pPdeDst->au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
1791	PGM_INVL_BIG_PG(GCPtrPage);
1792	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1793	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
1794	}
1795	}
1796	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1797	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
1798	}
1799	/* else: 4KB page table */
1800
1801	/*
1802	* Map the guest page table.
1803	*/
1804	PGSTPT pPTSrc;
1805	rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
1806	if (VBOX_SUCCESS(rc))
1807	{
1808	/*
1809	* Real page fault?
1810	*/
1811	PGSTPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
1812	const GSTPTE PteSrc = *pPteSrc;
1813	if ( !PteSrc.n.u1Present
1814	# if PGM_WITH_NX(PGM_GST_TYPE)
1815	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && PteSrc.n.u1NoExecute)
1816	# endif
1817	\|\| (fWriteFault && !PteSrc.n.u1Write && (fUserLevelFault \|\| fWriteProtect))
1818	\|\| (fUserLevelFault && !PteSrc.n.u1User)
1819	)
1820	{
1821	# ifdef IN_GC
1822	STAM_COUNTER_INC(&pVM->pgm.s.StatGCDirtyTrackRealPF);
1823	# endif
1824	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1825	LogFlow(("CheckPageFault: real page fault at %VGv PteSrc.u=%08x (2)\n", GCPtrPage, PteSrc.u));
1826
1827	/* Check the present bit as the shadow tables can cause different error codes by being out of sync.
1828	* See the 2nd case above as well.
1829	*/
1830	if (pPdeSrc->n.u1Present && pPteSrc->n.u1Present)
1831	TRPMSetErrorCode(pVM, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
1832
1833	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1834	return VINF_EM_RAW_GUEST_TRAP;
1835	}
1836	LogFlow(("CheckPageFault: page fault at %VGv PteSrc.u=%08x\n", GCPtrPage, PteSrc.u));
1837
1838	/*
1839	* Set the accessed bits in the page directory and the page table.
1840	*/
1841	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1842	pPml4eSrc->n.u1Accessed = 1;
1843	pPdpeSrc->lm.u1Accessed = 1;
1844	# endif
1845	pPdeSrc->n.u1Accessed = 1;
1846	pPteSrc->n.u1Accessed = 1;
1847
1848	/*
1849	* Only write protection page faults are relevant here.
1850	*/
1851	if (fWriteFault)
1852	{
1853	/* Write access, so mark guest entry as dirty. */
1854	# if defined(IN_GC) && defined(VBOX_WITH_STATISTICS)
1855	if (!pPteSrc->n.u1Dirty)
1856	STAM_COUNTER_INC(&pVM->pgm.s.StatGCDirtiedPage);
1857	else
1858	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageAlreadyDirty);
1859	# endif
1860	pPteSrc->n.u1Dirty = 1;
1861
1862	if (pPdeDst->n.u1Present)
1863	{
1864	/* Bail out here as pgmPoolGetPageByHCPhys will return NULL and we'll crash below.
1865	* Our individual shadow handlers will provide more information and force a fatal exit.
1866	*/
1867	if (MMHyperIsInsideArea(pVM, (RTGCPTR)GCPtrPage))
1868	{
1869	LogRel(("CheckPageFault: write to hypervisor region %VGv\n", GCPtrPage));
1870	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1871	return VINF_SUCCESS;
1872	}
1873
1874	/*
1875	* Map shadow page table.
1876	*/
1877	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, pPdeDst->u & SHW_PDE_PG_MASK);
1878	if (pShwPage)
1879	{
1880	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1881	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
1882	if ( pPteDst->n.u1Present /** @todo Optimize accessed bit emulation? */
1883	&& (pPteDst->u & PGM_PTFLAGS_TRACK_DIRTY))
1884	{
1885	LogFlow(("DIRTY page trap addr=%VGv\n", GCPtrPage));
1886	# ifdef VBOX_STRICT
1887	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, pPteSrc->u & GST_PTE_PG_MASK);
1888	if (pPage)
1889	AssertMsg(!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage),
1890	("Unexpected dirty bit tracking on monitored page %VGv (phys %VGp)!!!!!!\n", GCPtrPage, pPteSrc->u & X86_PTE_PAE_PG_MASK));
1891	# endif
1892	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageTrap));
1893
1894	Assert(pPteSrc->n.u1Write);
1895
1896	pPteDst->n.u1Write = 1;
1897	pPteDst->n.u1Dirty = 1;
1898	pPteDst->n.u1Accessed = 1;
1899	pPteDst->au32[0] &= ~PGM_PTFLAGS_TRACK_DIRTY;
1900	PGM_INVL_PG(GCPtrPage);
1901
1902	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1903	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;
1904	}
1905	}
1906	else
1907	AssertMsgFailed(("pgmPoolGetPageByHCPhys %VGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
1908	}
1909	}
1910	/** @todo Optimize accessed bit emulation? */
1911	# ifdef VBOX_STRICT
1912	/*
1913	* Sanity check.
1914	*/
1915	else if ( !pPteSrc->n.u1Dirty
1916	&& (pPdeSrc->n.u1Write & pPteSrc->n.u1Write)
1917	&& pPdeDst->n.u1Present)
1918	{
1919	PPGMPOOLPAGE pShwPage = pgmPoolGetPageByHCPhys(pVM, pPdeDst->u & SHW_PDE_PG_MASK);
1920	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1921	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
1922	if ( pPteDst->n.u1Present
1923	&& pPteDst->n.u1Write)
1924	LogFlow(("Writable present page %VGv not marked for dirty bit tracking!!!\n", GCPtrPage));
1925	}
1926	# endif /* VBOX_STRICT */
1927	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1928	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
1929	}
1930	AssertRC(rc);
1931	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,DirtyBitTracking), a);
1932	return rc;
1933
1934
1935	UpperLevelPageFault:
1936	/* Pagefault detected while checking the PML4E, PDPE or PDE.
1937	* Single exit handler to get rid of duplicate code paths.
1938	*/
1939	# ifdef IN_GC
1940	STAM_COUNTER_INC(&pVM->pgm.s.StatGCDirtyTrackRealPF);
1941	# endif
1942	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat, DirtyBitTracking), a);
1943	LogFlow(("CheckPageFault: real page fault at %VGv (%d)\n", GCPtrPage, uPageFaultLevel));
1944
1945	if (
1946	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1947	pPml4eSrc->n.u1Present &&
1948	# endif
1949	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
1950	pPdpeSrc->n.u1Present &&
1951	# endif
1952	pPdeSrc->n.u1Present)
1953	{
1954	/* Check the present bit as the shadow tables can cause different error codes by being out of sync. */
1955	if (pPdeSrc->b.u1Size && fBigPagesSupported)
1956	{
1957	TRPMSetErrorCode(pVM, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
1958	}
1959	else
1960	{
1961	/*
1962	* Map the guest page table.
1963	*/
1964	PGSTPT pPTSrc;
1965	rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
1966	if (VBOX_SUCCESS(rc))
1967	{
1968	PGSTPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
1969	const GSTPTE PteSrc = *pPteSrc;
1970	if (pPteSrc->n.u1Present)
1971	TRPMSetErrorCode(pVM, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
1972	}
1973	AssertRC(rc);
1974	}
1975	}
1976	return VINF_EM_RAW_GUEST_TRAP;
1977	}
1978
1979	#endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
1980
1981
1982	/**
1983	* Sync a shadow page table.
1984	*
1985	* The shadow page table is not present. This includes the case where
1986	* there is a conflict with a mapping.
1987	*
1988	* @returns VBox status code.
1989	* @param pVM VM handle.
1990	* @param iPD Page directory index.
1991	* @param pPDSrc Source page directory (i.e. Guest OS page directory).
1992	* Assume this is a temporary mapping.
1993	* @param GCPtrPage GC Pointer of the page that caused the fault
1994	*/
1995	PGM_BTH_DECL(int, SyncPT)(PVM pVM, unsigned iPDSrc, PGSTPD pPDSrc, RTGCUINTPTR GCPtrPage)
1996	{
1997	STAM_PROFILE_START(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
1998	STAM_COUNTER_INC(&pVM->pgm.s.StatGCSyncPtPD[iPDSrc]);
1999	LogFlow(("SyncPT: GCPtrPage=%VGv\n", GCPtrPage));
2000
2001	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
2002	\|\| PGM_GST_TYPE == PGM_TYPE_PAE
2003
2004	/*
2005	* Validate input a little bit.
2006	*/
2007	AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%VGv\n", iPDSrc, GCPtrPage));
2008	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2009	PX86PD pPDDst = pVM->pgm.s.CTXMID(p,32BitPD);
2010	# else
2011	PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0];
2012	# endif
2013	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2014	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2015	SHWPDE PdeDst = *pPdeDst;
2016
2017	# ifndef PGM_WITHOUT_MAPPINGS
2018	/*
2019	* Check for conflicts.
2020	* GC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
2021	* HC: Simply resolve the conflict.
2022	*/
2023	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
2024	{
2025	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
2026	# ifndef IN_RING3
2027	Log(("SyncPT: Conflict at %VGv\n", GCPtrPage));
2028	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2029	return VERR_ADDRESS_CONFLICT;
2030	# else
2031	PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
2032	Assert(pMapping);
2033	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2034	int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2035	# elif PGM_GST_TYPE == PGM_TYPE_PAE
2036	int rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2037	# endif
2038	if (VBOX_FAILURE(rc))
2039	{
2040	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2041	return rc;
2042	}
2043	PdeDst = *pPdeDst;
2044	# endif
2045	}
2046	# else /* PGM_WITHOUT_MAPPINGS */
2047	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
2048	# endif /* PGM_WITHOUT_MAPPINGS */
2049	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2050
2051	/*
2052	* Sync page directory entry.
2053	*/
2054	int rc = VINF_SUCCESS;
2055	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2056	if (PdeSrc.n.u1Present)
2057	{
2058	/*
2059	* Allocate & map the page table.
2060	*/
2061	PSHWPT pPTDst;
2062	const bool fPageTable = !PdeSrc.b.u1Size \|\| !(CPUMGetGuestCR4(pVM) & X86_CR4_PSE);
2063	PPGMPOOLPAGE pShwPage;
2064	RTGCPHYS GCPhys;
2065	if (fPageTable)
2066	{
2067	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
2068	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2069	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2070	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE / 2);
2071	# endif
2072	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, SHW_POOL_ROOT_IDX, iPDDst, &pShwPage);
2073	}
2074	else
2075	{
2076	GCPhys = PdeSrc.u & GST_PDE_BIG_PG_MASK;
2077	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2078	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2079	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
2080	# endif
2081	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, SHW_POOL_ROOT_IDX, iPDDst, &pShwPage);
2082	}
2083	if (rc == VINF_SUCCESS)
2084	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2085	else if (rc == VINF_PGM_CACHED_PAGE)
2086	{
2087	/*
2088	* The PT was cached, just hook it up.
2089	*/
2090	if (fPageTable)
2091	PdeDst.u = pShwPage->Core.Key
2092	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2093	else
2094	{
2095	PdeDst.u = pShwPage->Core.Key
2096	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2097	# ifdef PGM_SYNC_DIRTY_BIT /* (see explanation and assumptions further down.) */
2098	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
2099	{
2100	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageBig));
2101	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2102	PdeDst.b.u1Write = 0;
2103	}
2104	# endif
2105	}
2106	*pPdeDst = PdeDst;
2107	return VINF_SUCCESS;
2108	}
2109	else if (rc == VERR_PGM_POOL_FLUSHED)
2110	return VINF_PGM_SYNC_CR3;
2111	else
2112	AssertMsgFailedReturn(("rc=%Vrc\n", rc), VERR_INTERNAL_ERROR);
2113	PdeDst.u &= X86_PDE_AVL_MASK;
2114	PdeDst.u \|= pShwPage->Core.Key;
2115
2116	# ifdef PGM_SYNC_DIRTY_BIT
2117	/*
2118	* Page directory has been accessed (this is a fault situation, remember).
2119	*/
2120	pPDSrc->a[iPDSrc].n.u1Accessed = 1;
2121	# endif
2122	if (fPageTable)
2123	{
2124	/*
2125	* Page table - 4KB.
2126	*
2127	* Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
2128	*/
2129	Log2(("SyncPT: 4K %VGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
2130	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
2131	PGSTPT pPTSrc;
2132	rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
2133	if (VBOX_SUCCESS(rc))
2134	{
2135	/*
2136	* Start by syncing the page directory entry so CSAM's TLB trick works.
2137	*/
2138	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| X86_PDE_AVL_MASK))
2139	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2140	*pPdeDst = PdeDst;
2141
2142	/*
2143	* Directory/page user or supervisor privilege: (same goes for read/write)
2144	*
2145	* Directory Page Combined
2146	* U/S U/S U/S
2147	* 0 0 0
2148	* 0 1 0
2149	* 1 0 0
2150	* 1 1 1
2151	*
2152	* Simple AND operation. Table listed for completeness.
2153	*
2154	*/
2155	STAM_COUNTER_INC(CTXSUFF(&pVM->pgm.s.StatSynPT4k));
2156	# ifdef PGM_SYNC_N_PAGES
2157	unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2158	unsigned iPTDst = iPTBase;
2159	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, ELEMENTS(pPTDst->a));
2160	if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
2161	iPTDst = 0;
2162	else
2163	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2164	# else /* !PGM_SYNC_N_PAGES */
2165	unsigned iPTDst = 0;
2166	const unsigned iPTDstEnd = ELEMENTS(pPTDst->a);
2167	# endif /* !PGM_SYNC_N_PAGES */
2168	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2169	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2170	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2171	# else
2172	const unsigned offPTSrc = 0;
2173	# endif
2174	for (; iPTDst < iPTDstEnd; iPTDst++)
2175	{
2176	const unsigned iPTSrc = iPTDst + offPTSrc;
2177	const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2178
2179	if (PteSrc.n.u1Present) /* we've already cleared it above */
2180	{
2181	# ifndef IN_RING0
2182	/*
2183	* Assuming kernel code will be marked as supervisor - and not as user level
2184	* and executed using a conforming code selector - And marked as readonly.
2185	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
2186	*/
2187	PPGMPAGE pPage;
2188	if ( ((PdeSrc.u & pPTSrc->a[iPTSrc].u) & (X86_PTE_RW \| X86_PTE_US))
2189	\|\| !CSAMDoesPageNeedScanning(pVM, (RTGCPTR)((iPDSrc << GST_PD_SHIFT) \| (iPTSrc << PAGE_SHIFT)))
2190	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
2191	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2192	)
2193	# endif
2194	PGM_BTH_NAME(SyncPageWorker)(pVM, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2195	Log2(("SyncPT: 4K+ %VGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%VGp\n",
2196	(RTGCPTR)((iPDSrc << GST_PD_SHIFT) \| (iPTSrc << PAGE_SHIFT)),
2197	PteSrc.n.u1Present,
2198	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2199	PteSrc.n.u1User & PdeSrc.n.u1User,
2200	(uint64_t)PteSrc.u,
2201	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : "", pPTDst->a[iPTDst].u, iPTSrc, PdeSrc.au32[0],
2202	(PdeSrc.u & GST_PDE_PG_MASK) + iPTSrc*sizeof(PteSrc)));
2203	}
2204	} /* for PTEs */
2205	}
2206	}
2207	else
2208	{
2209	/*
2210	* Big page - 2/4MB.
2211	*
2212	* We'll walk the ram range list in parallel and optimize lookups.
2213	* We will only sync on shadow page table at a time.
2214	*/
2215	STAM_COUNTER_INC(CTXSUFF(&pVM->pgm.s.StatSynPT4M));
2216
2217	/**
2218	* @todo It might be more efficient to sync only a part of the 4MB page (similar to what we do for 4kb PDs).
2219	*/
2220
2221	/*
2222	* Start by syncing the page directory entry.
2223	*/
2224	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
2225	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2226
2227	# ifdef PGM_SYNC_DIRTY_BIT
2228	/*
2229	* If the page is not flagged as dirty and is writable, then make it read-only
2230	* at PD level, so we can set the dirty bit when the page is modified.
2231	*
2232	* ASSUMES that page access handlers are implemented on page table entry level.
2233	* Thus we will first catch the dirty access and set PDE.D and restart. If
2234	* there is an access handler, we'll trap again and let it work on the problem.
2235	*/
2236	/** @todo move the above stuff to a section in the PGM documentation. */
2237	Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
2238	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
2239	{
2240	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,DirtyPageBig));
2241	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2242	PdeDst.b.u1Write = 0;
2243	}
2244	# endif /* PGM_SYNC_DIRTY_BIT */
2245	*pPdeDst = PdeDst;
2246
2247	/*
2248	* Fill the shadow page table.
2249	*/
2250	/* Get address and flags from the source PDE. */
2251	SHWPTE PteDstBase;
2252	PteDstBase.u = PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT);
2253
2254	/* Loop thru the entries in the shadow PT. */
2255	const RTGCUINTPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
2256	Log2(("SyncPT: BIG %VGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%VGv GCPhys=%VGp %s\n",
2257	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
2258	GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2259	PPGMRAMRANGE pRam = CTXALLSUFF(pVM->pgm.s.pRamRanges);
2260	unsigned iPTDst = 0;
2261	while (iPTDst < ELEMENTS(pPTDst->a))
2262	{
2263	/* Advance ram range list. */
2264	while (pRam && GCPhys > pRam->GCPhysLast)
2265	pRam = CTXALLSUFF(pRam->pNext);
2266	if (pRam && GCPhys >= pRam->GCPhys)
2267	{
2268	unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2269	do
2270	{
2271	/* Make shadow PTE. */
2272	PPGMPAGE pPage = &pRam->aPages[iHCPage];
2273	SHWPTE PteDst;
2274
2275	/* Make sure the RAM has already been allocated. */
2276	if (pRam->fFlags & MM_RAM_FLAGS_DYNAMIC_ALLOC) /** @todo PAGE FLAGS */
2277	{
2278	if (RT_UNLIKELY(!PGM_PAGE_GET_HCPHYS(pPage)))
2279	{
2280	# ifdef IN_RING3
2281	int rc = pgmr3PhysGrowRange(pVM, GCPhys);
2282	# else
2283	int rc = CTXALLMID(VMM, CallHost)(pVM, VMMCALLHOST_PGM_RAM_GROW_RANGE, GCPhys);
2284	# endif
2285	if (rc != VINF_SUCCESS)
2286	return rc;
2287	}
2288	}
2289
2290	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2291	{
2292	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
2293	{
2294	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) \| PteDstBase.u;
2295	PteDst.n.u1Write = 0;
2296	}
2297	else
2298	PteDst.u = 0;
2299	}
2300	# ifndef IN_RING0
2301	/*
2302	* Assuming kernel code will be marked as supervisor and not as user level and executed
2303	* using a conforming code selector. Don't check for readonly, as that implies the whole
2304	* 4MB can be code or readonly data. Linux enables write access for its large pages.
2305	*/
2306	else if ( !PdeSrc.n.u1User
2307	&& CSAMDoesPageNeedScanning(pVM, (RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))))
2308	PteDst.u = 0;
2309	# endif
2310	else
2311	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) \| PteDstBase.u;
2312	# ifdef PGMPOOL_WITH_USER_TRACKING
2313	if (PteDst.n.u1Present)
2314	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVM, pShwPage, pPage->HCPhys >> MM_RAM_FLAGS_IDX_SHIFT, pPage, iPTDst); /** @todo PAGE FLAGS */
2315	# endif
2316	/* commit it */
2317	pPTDst->a[iPTDst] = PteDst;
2318	Log4(("SyncPT: BIG %VGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
2319	(RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT)), PteDst.n.u1Present, PteDst.n.u1Write, PteDst.n.u1User, (uint64_t)PteDst.u,
2320	PteDst.u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2321
2322	/* advance */
2323	GCPhys += PAGE_SIZE;
2324	iHCPage++;
2325	iPTDst++;
2326	} while ( iPTDst < ELEMENTS(pPTDst->a)
2327	&& GCPhys <= pRam->GCPhysLast);
2328	}
2329	else if (pRam)
2330	{
2331	Log(("Invalid pages at %VGp\n", GCPhys));
2332	do
2333	{
2334	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2335	GCPhys += PAGE_SIZE;
2336	iPTDst++;
2337	} while ( iPTDst < ELEMENTS(pPTDst->a)
2338	&& GCPhys < pRam->GCPhys);
2339	}
2340	else
2341	{
2342	Log(("Invalid pages at %VGp (2)\n", GCPhys));
2343	for ( ; iPTDst < ELEMENTS(pPTDst->a); iPTDst++)
2344	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2345	}
2346	} /* while more PTEs */
2347	} /* 4KB / 4MB */
2348	}
2349	else
2350	AssertRelease(!PdeDst.n.u1Present);
2351
2352	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2353	# ifdef IN_GC
2354	if (VBOX_FAILURE(rc))
2355	STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncPTFailed));
2356	# endif
2357	return rc;
2358
2359	#elif PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT
2360
2361	int rc = VINF_SUCCESS;
2362
2363	/*
2364	* Validate input a little bit.
2365	*/
2366	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2367	PX86PD pPDDst = pVM->pgm.s.CTXMID(p,32BitPD);
2368	# else
2369	PX86PDPAE pPDDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0];
2370	# endif
2371	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2372	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2373	SHWPDE PdeDst = *pPdeDst;
2374
2375	Assert(!(PdeDst.u & PGM_PDFLAGS_MAPPING));
2376	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2377
2378	GSTPDE PdeSrc;
2379	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2380	PdeSrc.n.u1Present = 1;
2381	PdeSrc.n.u1Write = 1;
2382	PdeSrc.n.u1Accessed = 1;
2383	PdeSrc.n.u1User = 1;
2384
2385	/*
2386	* Allocate & map the page table.
2387	*/
2388	PSHWPT pPTDst;
2389	PPGMPOOLPAGE pShwPage;
2390	RTGCPHYS GCPhys;
2391
2392	/* Virtual address = physical address */
2393	GCPhys = GCPtrPage & X86_PAGE_4K_BASE_MASK_32;
2394	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, SHW_POOL_ROOT_IDX, iPDDst, &pShwPage);
2395
2396	if ( rc == VINF_SUCCESS
2397	\|\| rc == VINF_PGM_CACHED_PAGE)
2398	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2399	else
2400	AssertMsgFailedReturn(("rc=%Vrc\n", rc), VERR_INTERNAL_ERROR);
2401
2402	PdeDst.u &= X86_PDE_AVL_MASK;
2403	PdeDst.u \|= pShwPage->Core.Key;
2404	PdeDst.n.u1Present = 1;
2405	*pPdeDst = PdeDst;
2406
2407	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, (RTGCUINTPTR)GCPtrPage, PGM_SYNC_NR_PAGES, 0 /* page not present */);
2408	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2409	return rc;
2410
2411	#else /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
2412	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2413	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncPT), a);
2414	return VERR_INTERNAL_ERROR;
2415	#endif /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
2416	}
2417
2418
2419
2420	/**
2421	* Prefetch a page/set of pages.
2422	*
2423	* Typically used to sync commonly used pages before entering raw mode
2424	* after a CR3 reload.
2425	*
2426	* @returns VBox status code.
2427	* @param pVM VM handle.
2428	* @param GCPtrPage Page to invalidate.
2429	*/
2430	PGM_BTH_DECL(int, PrefetchPage)(PVM pVM, RTGCUINTPTR GCPtrPage)
2431	{
2432	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) && PGM_SHW_TYPE != PGM_TYPE_AMD64
2433	/*
2434	* Check that all Guest levels thru the PDE are present, getting the
2435	* PD and PDE in the processes.
2436	*/
2437	int rc = VINF_SUCCESS;
2438	# if PGM_WITH_PAGING(PGM_GST_TYPE)
2439	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2440	const unsigned iPDSrc = (RTGCUINTPTR)GCPtrPage >> GST_PD_SHIFT;
2441	PGSTPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
2442	# else /* PAE */
2443	unsigned iPDSrc;
2444	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, GCPtrPage, &iPDSrc);
2445	if (!pPDSrc)
2446	return VINF_SUCCESS; /* not present */
2447	# endif
2448	const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2449	# else
2450	PGSTPD pPDSrc = NULL;
2451	const unsigned iPDSrc = 0;
2452	GSTPDE PdeSrc;
2453
2454	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2455	PdeSrc.n.u1Present = 1;
2456	PdeSrc.n.u1Write = 1;
2457	PdeSrc.n.u1Accessed = 1;
2458	PdeSrc.n.u1User = 1;
2459	# endif
2460
2461	# ifdef PGM_SYNC_ACCESSED_BIT
2462	if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
2463	# else
2464	if (PdeSrc.n.u1Present)
2465	# endif
2466	{
2467	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2468	const X86PDE PdeDst = pVM->pgm.s.CTXMID(p,32BitPD)->a[GCPtrPage >> SHW_PD_SHIFT];
2469	# else
2470	const X86PDEPAE PdeDst = pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[GCPtrPage >> SHW_PD_SHIFT];
2471	# endif
2472	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
2473	{
2474	if (!PdeDst.n.u1Present)
2475	/** r=bird: This guy will set the A bit on the PDE, probably harmless. */
2476	rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, GCPtrPage);
2477	else
2478	{
2479	/** @note We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
2480	* R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
2481	* makes no sense to prefetch more than one page.
2482	*/
2483	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, GCPtrPage, 1, 0);
2484	if (VBOX_SUCCESS(rc))
2485	rc = VINF_SUCCESS;
2486	}
2487	}
2488	}
2489	return rc;
2490
2491	#else /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
2492
2493	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
2494	return VERR_INTERNAL_ERROR;
2495	#endif /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
2496	}
2497
2498
2499
2500
2501	/**
2502	* Syncs a page during a PGMVerifyAccess() call.
2503	*
2504	* @returns VBox status code (informational included).
2505	* @param GCPtrPage The address of the page to sync.
2506	* @param fPage The effective guest page flags.
2507	* @param uErr The trap error code.
2508	*/
2509	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVM pVM, RTGCUINTPTR GCPtrPage, unsigned fPage, unsigned uErr)
2510	{
2511	LogFlow(("VerifyAccessSyncPage: GCPtrPage=%VGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
2512
2513	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) && PGM_SHW_TYPE != PGM_TYPE_AMD64
2514
2515	# ifndef IN_RING0
2516	if (!(fPage & X86_PTE_US))
2517	{
2518	/*
2519	* Mark this page as safe.
2520	*/
2521	/** @todo not correct for pages that contain both code and data!! */
2522	Log(("CSAMMarkPage %VGv; scanned=%d\n", GCPtrPage, true));
2523	CSAMMarkPage(pVM, (RTGCPTR)GCPtrPage, true);
2524	}
2525	# endif
2526	/*
2527	* Get guest PD and index.
2528	*/
2529
2530	# if PGM_WITH_PAGING(PGM_GST_TYPE)
2531	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2532	const unsigned iPDSrc = (RTGCUINTPTR)GCPtrPage >> GST_PD_SHIFT;
2533	PGSTPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
2534	# else /* PAE */
2535	unsigned iPDSrc;
2536	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, GCPtrPage, &iPDSrc);
2537
2538	if (pPDSrc)
2539	{
2540	Log(("PGMVerifyAccess: access violation for %VGv due to non-present PDPTR\n", GCPtrPage));
2541	return VINF_EM_RAW_GUEST_TRAP;
2542	}
2543	# endif
2544	# else
2545	PGSTPD pPDSrc = NULL;
2546	const unsigned iPDSrc = 0;
2547	# endif
2548	int rc = VINF_SUCCESS;
2549
2550	/*
2551	* First check if the shadow pd is present.
2552	*/
2553	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2554	PX86PDE pPdeDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[GCPtrPage >> SHW_PD_SHIFT];
2555	# else
2556	PX86PDEPAE pPdeDst = &pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[GCPtrPage >> SHW_PD_SHIFT];
2557	# endif
2558	if (!pPdeDst->n.u1Present)
2559	{
2560	rc = PGM_BTH_NAME(SyncPT)(pVM, iPDSrc, pPDSrc, GCPtrPage);
2561	AssertRC(rc);
2562	if (rc != VINF_SUCCESS)
2563	return rc;
2564	}
2565
2566	# if PGM_WITH_PAGING(PGM_GST_TYPE)
2567	/* Check for dirty bit fault */
2568	rc = PGM_BTH_NAME(CheckPageFault)(pVM, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
2569	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
2570	Log(("PGMVerifyAccess: success (dirty)\n"));
2571	else
2572	{
2573	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2574	#else
2575	{
2576	GSTPDE PdeSrc;
2577	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
2578	PdeSrc.n.u1Present = 1;
2579	PdeSrc.n.u1Write = 1;
2580	PdeSrc.n.u1Accessed = 1;
2581	PdeSrc.n.u1User = 1;
2582
2583	#endif /* PGM_WITH_PAGING(PGM_GST_TYPE) */
2584	Assert(rc != VINF_EM_RAW_GUEST_TRAP);
2585	if (uErr & X86_TRAP_PF_US)
2586	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncUser);
2587	else /* supervisor */
2588	STAM_COUNTER_INC(&pVM->pgm.s.StatGCPageOutOfSyncSupervisor);
2589
2590	rc = PGM_BTH_NAME(SyncPage)(pVM, PdeSrc, GCPtrPage, 1, 0);
2591	if (VBOX_SUCCESS(rc))
2592	{
2593	/* Page was successfully synced */
2594	Log2(("PGMVerifyAccess: success (sync)\n"));
2595	rc = VINF_SUCCESS;
2596	}
2597	else
2598	{
2599	Log(("PGMVerifyAccess: access violation for %VGv rc=%d\n", GCPtrPage, rc));
2600	return VINF_EM_RAW_GUEST_TRAP;
2601	}
2602	}
2603	return rc;
2604
2605	#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2606
2607	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2608	return VERR_INTERNAL_ERROR;
2609	#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
2610	}
2611
2612
2613	#if PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE
2614	# if PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE
2615	/**
2616	* Figures out which kind of shadow page this guest PDE warrants.
2617	*
2618	* @returns Shadow page kind.
2619	* @param pPdeSrc The guest PDE in question.
2620	* @param cr4 The current guest cr4 value.
2621	*/
2622	DECLINLINE(PGMPOOLKIND) PGM_BTH_NAME(CalcPageKind)(const GSTPDE *pPdeSrc, uint32_t cr4)
2623	{
2624	if (!pPdeSrc->n.u1Size \|\| !(cr4 & X86_CR4_PSE))
2625	return BTH_PGMPOOLKIND_PT_FOR_PT;
2626	//switch (pPdeSrc->u & (X86_PDE4M_RW \| X86_PDE4M_US /\| X86_PDE4M_PAE_NX/))
2627	//{
2628	// case 0:
2629	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RO;
2630	// case X86_PDE4M_RW:
2631	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW;
2632	// case X86_PDE4M_US:
2633	// return BTH_PGMPOOLKIND_PT_FOR_BIG_US;
2634	// case X86_PDE4M_RW \| X86_PDE4M_US:
2635	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_US;
2636	# if 0
2637	// case X86_PDE4M_PAE_NX:
2638	// return BTH_PGMPOOLKIND_PT_FOR_BIG_NX;
2639	// case X86_PDE4M_RW \| X86_PDE4M_PAE_NX:
2640	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_NX;
2641	// case X86_PDE4M_US \| X86_PDE4M_PAE_NX:
2642	// return BTH_PGMPOOLKIND_PT_FOR_BIG_US_NX;
2643	// case X86_PDE4M_RW \| X86_PDE4M_US \| X86_PDE4M_PAE_NX:
2644	// return BTH_PGMPOOLKIND_PT_FOR_BIG_RW_US_NX;
2645	# endif
2646	return BTH_PGMPOOLKIND_PT_FOR_BIG;
2647	//}
2648	}
2649	# endif
2650	#endif
2651
2652	#undef MY_STAM_COUNTER_INC
2653	#define MY_STAM_COUNTER_INC(a) do { } while (0)
2654
2655
2656	/**
2657	* Syncs the paging hierarchy starting at CR3.
2658	*
2659	* @returns VBox status code, no specials.
2660	* @param pVM The virtual machine.
2661	* @param cr0 Guest context CR0 register
2662	* @param cr3 Guest context CR3 register
2663	* @param cr4 Guest context CR4 register
2664	* @param fGlobal Including global page directories or not
2665	*/
2666	PGM_BTH_DECL(int, SyncCR3)(PVM pVM, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
2667	{
2668	if (VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3))
2669	fGlobal = true; /* Change this CR3 reload to be a global one. */
2670
2671	/*
2672	* Update page access handlers.
2673	* The virtual are always flushed, while the physical are only on demand.
2674	* WARNING: We are incorrectly not doing global flushing on Virtual Handler updates. We'll
2675	* have to look into that later because it will have a bad influence on the performance.
2676	* @note SvL: There's no need for that. Just invalidate the virtual range(s).
2677	* bird: Yes, but that won't work for aliases.
2678	*/
2679	/** @todo this MUST go away. See #1557. */
2680	STAM_PROFILE_START(&pVM->pgm.s.CTXMID(Stat,SyncCR3Handlers), h);
2681	PGM_GST_NAME(HandlerVirtualUpdate)(pVM, cr4);
2682	STAM_PROFILE_STOP(&pVM->pgm.s.CTXMID(Stat,SyncCR3Handlers), h);
2683
2684	#ifdef PGMPOOL_WITH_MONITORING
2685	/*
2686	* When monitoring shadowed pages, we reset the modification counters on CR3 sync.
2687	* Occationally we will have to clear all the shadow page tables because we wanted
2688	* to monitor a page which was mapped by too many shadowed page tables. This operation
2689	* sometimes refered to as a 'lightweight flush'.
2690	*/
2691	if (!(pVM->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL))
2692	pgmPoolMonitorModifiedClearAll(pVM);
2693	else
2694	{
2695	# ifdef IN_RING3
2696	pVM->pgm.s.fSyncFlags &= ~PGM_SYNC_CLEAR_PGM_POOL;
2697	pgmPoolClearAll(pVM);
2698	# else
2699	LogFlow(("SyncCR3: PGM_SYNC_CLEAR_PGM_POOL is set -> VINF_PGM_SYNC_CR3\n"));
2700	return VINF_PGM_SYNC_CR3;
2701	# endif
2702	}
2703	#endif
2704
2705	Assert(fGlobal \|\| (cr4 & X86_CR4_PGE));
2706	MY_STAM_COUNTER_INC(fGlobal ? &pVM->pgm.s.CTXMID(Stat,SyncCR3Global) : &pVM->pgm.s.CTXMID(Stat,SyncCR3NotGlobal));
2707
2708	#if PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE
2709	/*
2710	* Get page directory addresses.
2711	*/
2712	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2713	PX86PDE pPDEDst = &pVM->pgm.s.CTXMID(p,32BitPD)->a[0];
2714	# else /* PGM_SHW_TYPE == PGM_TYPE_PAE */
2715	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2716	PX86PDEPAE pPDEDst = &pVM->pgm.s.CTXMID(ap,PaePDs)[0]->a[0];
2717	# endif
2718	# endif
2719
2720	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2721	PGSTPD pPDSrc = CTXSUFF(pVM->pgm.s.pGuestPD);
2722	Assert(pPDSrc);
2723	# ifndef IN_GC
2724	Assert(MMPhysGCPhys2HCVirt(pVM, (RTGCPHYS)(cr3 & GST_CR3_PAGE_MASK), sizeof(*pPDSrc)) == pPDSrc);
2725	# endif
2726	# endif
2727
2728	/*
2729	* Iterate the page directory.
2730	*/
2731	PPGMMAPPING pMapping;
2732	unsigned iPdNoMapping;
2733	const bool fRawR0Enabled = EMIsRawRing0Enabled(pVM);
2734	PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2735
2736	/* Only check mappings if they are supposed to be put into the shadow page table. */
2737	if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
2738	{
2739	pMapping = pVM->pgm.s.CTXALLSUFF(pMappings);
2740	iPdNoMapping = (pMapping) ? (pMapping->GCPtr >> GST_PD_SHIFT) : ~0U;
2741	}
2742	else
2743	{
2744	pMapping = 0;
2745	iPdNoMapping = ~0U;
2746	}
2747	# if PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64
2748	for (unsigned iPDPTE = 0; iPDPTE < GST_PDPE_ENTRIES; iPDPTE++)
2749	{
2750	unsigned iPDSrc;
2751	# if PGM_SHW_TYPE == PGM_TYPE_PAE
2752	PX86PDPAE pPDPAE = pVM->pgm.s.CTXMID(ap,PaePDs)[0];
2753	# else
2754	AssertFailed(); /* @todo */
2755	PX86PDPE pPDPAE = pVM->pgm.s.CTXMID(ap,PaePDs)[iPDPTE * X86_PG_AMD64_ENTRIES];
2756	# endif
2757	PX86PDEPAE pPDEDst = &pPDPAE->a[iPDPTE * X86_PG_PAE_ENTRIES];
2758	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVM->pgm.s, iPDPTE << X86_PDPT_SHIFT, &iPDSrc);
2759
2760	if (pPDSrc == NULL)
2761	{
2762	/* PDPT not present */
2763	if (pVM->pgm.s.CTXMID(p,PaePDPT)->a[iPDPTE].n.u1Present)
2764	{
2765	for (unsigned iPD = 0; iPD < ELEMENTS(pPDSrc->a); iPD++)
2766	{
2767	if ( pPDEDst[iPD].n.u1Present
2768	&& !(pPDEDst[iPD].u & PGM_PDFLAGS_MAPPING))
2769	{
2770	pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, pPDEDst[iPD].u & SHW_PDE_PG_MASK), SHW_POOL_ROOT_IDX, iPDPTE * X86_PG_PAE_ENTRIES + iPD);
2771	pPDEDst[iPD].u = 0;
2772	}
2773	}
2774	}
2775	if (!(pVM->pgm.s.CTXMID(p,PaePDPT)->a[iPDPTE].u & PGM_PLXFLAGS_MAPPING))
2776	pVM->pgm.s.CTXMID(p,PaePDPT)->a[iPDPTE].n.u1Present = 0;
2777	continue;
2778	}
2779	# else /* PGM_GST_TYPE != PGM_TYPE_PAE && PGM_GST_TYPE != PGM_TYPE_AMD64 */
2780	{
2781	# endif /* PGM_GST_TYPE != PGM_TYPE_PAE && PGM_GST_TYPE != PGM_TYPE_AMD64 */
2782	for (unsigned iPD = 0; iPD < ELEMENTS(pPDSrc->a); iPD++)
2783	{
2784	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2785	Assert(&pVM->pgm.s.CTXMID(p,32BitPD)->a[iPD] == pPDEDst);
2786	# elif PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2787	AssertMsg(&pVM->pgm.s.CTXMID(ap,PaePDs)[iPD * 2 / 512]->a[iPD * 2 % 512] == pPDEDst, ("%p vs %p\n", &pVM->pgm.s.CTXMID(ap,PaePDs)[iPD * 2 / 512]->a[iPD * 2 % 512], pPDEDst));
2788	# endif
2789	register GSTPDE PdeSrc = pPDSrc->a[iPD];
2790	if ( PdeSrc.n.u1Present
2791	&& (PdeSrc.n.u1User \|\| fRawR0Enabled))
2792	{
2793	# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2794	\|\| PGM_GST_TYPE == PGM_TYPE_PAE) \
2795	&& !defined(PGM_WITHOUT_MAPPINGS)
2796
2797	/*
2798	* Check for conflicts with GC mappings.
2799	*/
2800	# if PGM_GST_TYPE == PGM_TYPE_PAE
2801	if (iPD + iPDPTE * X86_PG_PAE_ENTRIES == iPdNoMapping)
2802	# else
2803	if (iPD == iPdNoMapping)
2804	# endif
2805	{
2806	if (pVM->pgm.s.fMappingsFixed)
2807	{
2808	/* It's fixed, just skip the mapping. */
2809	const unsigned cPTs = pMapping->cb >> GST_PD_SHIFT;
2810	iPD += cPTs - 1;
2811	pPDEDst += cPTs + (PGM_GST_TYPE != PGM_SHW_TYPE) * cPTs; /* Only applies to the pae shadow and 32 bits guest case */
2812	pMapping = pMapping->CTXALLSUFF(pNext);
2813	iPdNoMapping = pMapping ? pMapping->GCPtr >> GST_PD_SHIFT : ~0U;
2814	continue;
2815	}
2816	# ifdef IN_RING3
2817	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2818	int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, iPD << GST_PD_SHIFT);
2819	# elif PGM_GST_TYPE == PGM_TYPE_PAE
2820	int rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, (iPDPTE << GST_PDPT_SHIFT) + (iPD << GST_PD_SHIFT));
2821	# endif
2822	if (VBOX_FAILURE(rc))
2823	return rc;
2824
2825	/*
2826	* Update iPdNoMapping and pMapping.
2827	*/
2828	pMapping = pVM->pgm.s.pMappingsR3;
2829	while (pMapping && pMapping->GCPtr < (iPD << GST_PD_SHIFT))
2830	pMapping = pMapping->pNextR3;
2831	iPdNoMapping = pMapping ? pMapping->GCPtr >> GST_PD_SHIFT : ~0U;
2832	# else
2833	LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
2834	return VINF_PGM_SYNC_CR3;
2835	# endif
2836	}
2837	# else /* (PGM_GST_TYPE != PGM_TYPE_32BIT && PGM_GST_TYPE != PGM_TYPE_PAE) \|\| PGM_WITHOUT_MAPPINGS */
2838	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
2839	# endif /* (PGM_GST_TYPE != PGM_TYPE_32BIT && PGM_GST_TYPE != PGM_TYPE_PAE) \|\| PGM_WITHOUT_MAPPINGS */
2840	/*
2841	* Sync page directory entry.
2842	*
2843	* The current approach is to allocated the page table but to set
2844	* the entry to not-present and postpone the page table synching till
2845	* it's actually used.
2846	*/
2847	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2848	for (unsigned i = 0, iPdShw = iPD * 2; i < 2; i++, iPdShw++) /* pray that the compiler unrolls this */
2849	# elif PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64
2850	const unsigned iPdShw = iPD + iPDPTE * X86_PG_PAE_ENTRIES; NOREF(iPdShw);
2851	# else
2852	const unsigned iPdShw = iPD; NOREF(iPdShw);
2853	# endif
2854	{
2855	SHWPDE PdeDst = *pPDEDst;
2856	if (PdeDst.n.u1Present)
2857	{
2858	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2859	RTGCPHYS GCPhys;
2860	if ( !PdeSrc.b.u1Size
2861	\|\| !(cr4 & X86_CR4_PSE))
2862	{
2863	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
2864	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2865	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2866	GCPhys \|= i * (PAGE_SIZE / 2);
2867	# endif
2868	}
2869	else
2870	{
2871	GCPhys = PdeSrc.u & GST_PDE_BIG_PG_MASK;
2872	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2873	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2874	GCPhys \|= i * X86_PAGE_2M_SIZE;
2875	# endif
2876	}
2877
2878	if ( pShwPage->GCPhys == GCPhys
2879	&& pShwPage->enmKind == PGM_BTH_NAME(CalcPageKind)(&PdeSrc, cr4)
2880	&& ( pShwPage->fCached
2881	\|\| ( !fGlobal
2882	&& ( false
2883	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
2884	\|\| ( (PdeSrc.u & (X86_PDE4M_PS \| X86_PDE4M_G)) == (X86_PDE4M_PS \| X86_PDE4M_G)
2885	&& (cr4 & (X86_CR4_PGE \| X86_CR4_PSE)) == (X86_CR4_PGE \| X86_CR4_PSE)) /* global 2/4MB page. */
2886	\|\| ( !pShwPage->fSeenNonGlobal
2887	&& (cr4 & X86_CR4_PGE))
2888	# endif
2889	)
2890	)
2891	)
2892	&& ( (PdeSrc.u & (X86_PDE_US \| X86_PDE_RW)) == (PdeDst.u & (X86_PDE_US \| X86_PDE_RW))
2893	\|\| ( (cr4 & X86_CR4_PSE)
2894	&& ((PdeSrc.u & (X86_PDE_US \| X86_PDE4M_PS \| X86_PDE4M_D)) \| PGM_PDFLAGS_TRACK_DIRTY)
2895	== ((PdeDst.u & (X86_PDE_US \| X86_PDE_RW \| PGM_PDFLAGS_TRACK_DIRTY)) \| X86_PDE4M_PS))
2896	)
2897	)
2898	{
2899	# ifdef VBOX_WITH_STATISTICS
2900	if ( !fGlobal
2901	&& (PdeSrc.u & (X86_PDE4M_PS \| X86_PDE4M_G)) == (X86_PDE4M_PS \| X86_PDE4M_G)
2902	&& (cr4 & (X86_CR4_PGE \| X86_CR4_PSE)) == (X86_CR4_PGE \| X86_CR4_PSE))
2903	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstSkippedGlobalPD));
2904	else if (!fGlobal && !pShwPage->fSeenNonGlobal && (cr4 & X86_CR4_PGE))
2905	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstSkippedGlobalPT));
2906	else
2907	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstCacheHit));
2908	# endif /* VBOX_WITH_STATISTICS */
2909	/** @todo a replacement strategy isn't really needed unless we're using a very small pool < 512 pages.
2910	* The whole ageing stuff should be put in yet another set of #ifdefs. For now, let's just skip it. */
2911	//# ifdef PGMPOOL_WITH_CACHE
2912	// pgmPoolCacheUsed(pPool, pShwPage);
2913	//# endif
2914	}
2915	else
2916	{
2917	pgmPoolFreeByPage(pPool, pShwPage, SHW_POOL_ROOT_IDX, iPdShw);
2918	pPDEDst->u = 0;
2919	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstFreed));
2920	}
2921	}
2922	else
2923	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstNotPresent));
2924	pPDEDst++;
2925	}
2926	}
2927	# if PGM_GST_TYPE == PGM_TYPE_PAE
2928	else if (iPD + iPDPTE * X86_PG_PAE_ENTRIES != iPdNoMapping)
2929	# else
2930	else if (iPD != iPdNoMapping)
2931	# endif
2932	{
2933	/*
2934	* Check if there is any page directory to mark not present here.
2935	*/
2936	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2937	for (unsigned i = 0, iPdShw = iPD * 2; i < 2; i++, iPdShw++) /* pray that the compiler unrolls this */
2938	# elif PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64
2939	const unsigned iPdShw = iPD + iPDPTE * X86_PG_PAE_ENTRIES; NOREF(iPdShw);
2940	# else
2941	const unsigned iPdShw = iPD; NOREF(iPdShw);
2942	# endif
2943	{
2944	if (pPDEDst->n.u1Present)
2945	{
2946	pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, pPDEDst->u & SHW_PDE_PG_MASK), SHW_POOL_ROOT_IDX, iPdShw);
2947	pPDEDst->u = 0;
2948	MY_STAM_COUNTER_INC(&pVM->pgm.s.CTXMID(Stat,SyncCR3DstFreedSrcNP));
2949	}
2950	pPDEDst++;
2951	}
2952	}
2953	else
2954	{
2955	# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2956	\|\| PGM_GST_TYPE == PGM_TYPE_PAE) \
2957	&& !defined(PGM_WITHOUT_MAPPINGS)
2958
2959	const unsigned cPTs = pMapping->cb >> GST_PD_SHIFT;
2960
2961	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
2962	if (pVM->pgm.s.fMappingsFixed)
2963	{
2964	/* It's fixed, just skip the mapping. */
2965	pMapping = pMapping->CTXALLSUFF(pNext);
2966	iPdNoMapping = pMapping ? pMapping->GCPtr >> GST_PD_SHIFT : ~0U;
2967	}
2968	else
2969	{
2970	/*
2971	* Check for conflicts for subsequent pagetables
2972	* and advance to the next mapping.
2973	*/
2974	iPdNoMapping = ~0U;
2975	unsigned iPT = cPTs;
2976	while (iPT-- > 1)
2977	{
2978	if ( pPDSrc->a[iPD + iPT].n.u1Present
2979	&& (pPDSrc->a[iPD + iPT].n.u1User \|\| fRawR0Enabled))
2980	{
2981	# ifdef IN_RING3
2982	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2983	int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, iPD << GST_PD_SHIFT);
2984	# elif PGM_GST_TYPE == PGM_TYPE_PAE
2985	int rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, (iPDPTE << GST_PDPT_SHIFT) + (iPD << GST_PD_SHIFT));
2986	# endif
2987	if (VBOX_FAILURE(rc))
2988	return rc;
2989
2990	/*
2991	* Update iPdNoMapping and pMapping.
2992	*/
2993	pMapping = pVM->pgm.s.CTXALLSUFF(pMappings);
2994	while (pMapping && pMapping->GCPtr < (iPD << GST_PD_SHIFT))
2995	pMapping = pMapping->CTXALLSUFF(pNext);
2996	iPdNoMapping = pMapping ? pMapping->GCPtr >> GST_PD_SHIFT : ~0U;
2997	break;
2998	# else
2999	LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
3000	return VINF_PGM_SYNC_CR3;
3001	# endif
3002	}
3003	}
3004	if (iPdNoMapping == ~0U && pMapping)
3005	{
3006	pMapping = pMapping->CTXALLSUFF(pNext);
3007	if (pMapping)
3008	iPdNoMapping = pMapping->GCPtr >> GST_PD_SHIFT;
3009	}
3010	}
3011
3012	/* advance. */
3013	iPD += cPTs - 1;
3014	pPDEDst += cPTs + (PGM_GST_TYPE != PGM_SHW_TYPE) * cPTs; /* Only applies to the pae shadow and 32 bits guest case */
3015	# if PGM_GST_TYPE != PGM_SHW_TYPE
3016	AssertCompile(PGM_GST_TYPE == PGM_TYPE_32BIT && PGM_SHW_TYPE == PGM_TYPE_PAE);
3017	# endif
3018	# else /* (PGM_GST_TYPE != PGM_TYPE_32BIT && PGM_GST_TYPE != PGM_TYPE_PAE) \|\| PGM_WITHOUT_MAPPINGS */
3019	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
3020	# endif /* (PGM_GST_TYPE != PGM_TYPE_32BIT && PGM_GST_TYPE != PGM_TYPE_PAE) \|\| PGM_WITHOUT_MAPPINGS */
3021	}
3022
3023	} /* for iPD */
3024	} /* for each PDPTE (PAE) */
3025
3026	return VINF_SUCCESS;
3027
3028	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
3029	//# error not implemented
3030	return VERR_INTERNAL_ERROR;
3031	#else /* guest real and protected mode */
3032	return VINF_SUCCESS;
3033	#endif
3034	}
3035
3036
3037
3038
3039	#ifdef VBOX_STRICT
3040	#ifdef IN_GC
3041	# undef AssertMsgFailed
3042	# define AssertMsgFailed Log
3043	#endif
3044	#ifdef IN_RING3
3045	# include <VBox/dbgf.h>
3046
3047	/**
3048	* Dumps a page table hierarchy use only physical addresses and cr4/lm flags.
3049	*
3050	* @returns VBox status code (VINF_SUCCESS).
3051	* @param pVM The VM handle.
3052	* @param cr3 The root of the hierarchy.
3053	* @param crr The cr4, only PAE and PSE is currently used.
3054	* @param fLongMode Set if long mode, false if not long mode.
3055	* @param cMaxDepth Number of levels to dump.
3056	* @param pHlp Pointer to the output functions.
3057	*/
3058	__BEGIN_DECLS
3059	PGMR3DECL(int) PGMR3DumpHierarchyHC(PVM pVM, uint32_t cr3, uint32_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp);
3060	__END_DECLS
3061
3062	#endif
3063
3064	/**
3065	* Checks that the shadow page table is in sync with the guest one.
3066	*
3067	* @returns The number of errors.
3068	* @param pVM The virtual machine.
3069	* @param cr3 Guest context CR3 register
3070	* @param cr4 Guest context CR4 register
3071	* @param GCPtr Where to start. Defaults to 0.
3072	* @param cb How much to check. Defaults to everything.
3073	*/
3074	PGM_BTH_DECL(unsigned, AssertCR3)(PVM pVM, uint64_t cr3, uint64_t cr4, RTGCUINTPTR GCPtr, RTGCUINTPTR cb)
3075	{
3076	unsigned cErrors = 0;
3077
3078	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
3079	\|\| PGM_GST_TYPE == PGM_TYPE_PAE
3080
3081	PPGM pPGM = &pVM->pgm.s;
3082	RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
3083	RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
3084	# ifndef IN_RING0
3085	RTHCPHYS HCPhys; /* general usage. */
3086	# endif
3087	int rc;
3088
3089	/*
3090	* Check that the Guest CR3 and all its mappings are correct.
3091	*/
3092	AssertMsgReturn(pPGM->GCPhysCR3 == (cr3 & GST_CR3_PAGE_MASK),
3093	("Invalid GCPhysCR3=%VGp cr3=%VGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
3094	false);
3095	# ifndef IN_RING0
3096	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3097	rc = PGMShwGetPage(pVM, pPGM->pGuestPDGC, NULL, &HCPhysShw);
3098	# else
3099	rc = PGMShwGetPage(pVM, pPGM->pGstPaePDPTGC, NULL, &HCPhysShw);
3100	# endif
3101	AssertRCReturn(rc, 1);
3102	HCPhys = NIL_RTHCPHYS;
3103	rc = pgmRamGCPhys2HCPhys(pPGM, cr3 & GST_CR3_PAGE_MASK, &HCPhys);
3104	AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%VHp HCPhyswShw=%VHp (cr3)\n", HCPhys, HCPhysShw), false);
3105	# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
3106	RTGCPHYS GCPhys;
3107	rc = PGMR3DbgHCPtr2GCPhys(pVM, pPGM->pGuestPDHC, &GCPhys);
3108	AssertRCReturn(rc, 1);
3109	AssertMsgReturn((cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%VGp cr3=%VGp\n", GCPhys, (RTGCPHYS)cr3), false);
3110	# endif
3111	#endif /* !IN_RING0 */
3112
3113	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3114	const GSTPD *pPDSrc = CTXSUFF(pPGM->pGuestPD);
3115	# endif
3116
3117	/*
3118	* Get and check the Shadow CR3.
3119	*/
3120	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3121	const X86PD *pPDDst = pPGM->CTXMID(p,32BitPD);
3122	unsigned cPDEs = ELEMENTS(pPDDst->a);
3123	# else
3124	const X86PDPAE pPDDst = pPGM->CTXMID(ap,PaePDs[0]); / use it as a 2048 entry PD */
3125	unsigned cPDEs = ELEMENTS(pPDDst->a) * ELEMENTS(pPGM->apHCPaePDs);
3126	# endif
3127	if (cb != ~(RTGCUINTPTR)0)
3128	cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
3129
3130	/** @todo call the other two PGMAssert() functions. /
3131
3132	# if PGM_GST_TYPE == PGM_TYPE_PAE
3133	/*
3134	* Check the 4 PDPTs too.
3135	*/
3136	for (unsigned i = 0; i < 4; i++)
3137	{
3138	RTHCPTR HCPtr;
3139	RTHCPHYS HCPhys;
3140	RTGCPHYS GCPhys = pVM->pgm.s.CTXSUFF(pGstPaePDPT)->a[i].u & X86_PDPE_PG_MASK;
3141	int rc2 = pgmRamGCPhys2HCPtrAndHCPhysWithFlags(&pVM->pgm.s, GCPhys, &HCPtr, &HCPhys);
3142	if (VBOX_SUCCESS(rc2))
3143	{
3144	AssertMsg( pVM->pgm.s.apGstPaePDsHC[i] == (R3R0PTRTYPE(PX86PDPAE))HCPtr
3145	&& pVM->pgm.s.aGCPhysGstPaePDs[i] == GCPhys,
3146	("idx %d apGstPaePDsHC %VHv vs %VHv aGCPhysGstPaePDs %VGp vs %VGp\n",
3147	i, pVM->pgm.s.apGstPaePDsHC[i], HCPtr, pVM->pgm.s.aGCPhysGstPaePDs[i], GCPhys));
3148	}
3149	}
3150	# endif
3151
3152	/*
3153	* Iterate the shadow page directory.
3154	*/
3155	GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
3156	unsigned iPDDst = GCPtr >> SHW_PD_SHIFT;
3157	cPDEs += iPDDst;
3158	for (;
3159	iPDDst < cPDEs;
3160	iPDDst++, GCPtr += _4G / cPDEs)
3161	{
3162	# if PGM_GST_TYPE == PGM_TYPE_PAE
3163	uint32_t iPDSrc;
3164	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pPGM, (RTGCUINTPTR)GCPtr, &iPDSrc);
3165	if (!pPDSrc)
3166	{
3167	AssertMsg(!pVM->pgm.s.CTXSUFF(pGstPaePDPT)->a[(GCPtr >> GST_PDPT_SHIFT) & GST_PDPT_MASK].n.u1Present, ("Guest PDTPR not present, shadow PDPTR %VX64\n", pVM->pgm.s.CTXSUFF(pGstPaePDPT)->a[(GCPtr >> GST_PDPT_SHIFT) & GST_PDPT_MASK].u));
3168	continue;
3169	}
3170	#endif
3171
3172	const SHWPDE PdeDst = pPDDst->a[iPDDst];
3173	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
3174	{
3175	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
3176	if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
3177	{
3178	AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
3179	cErrors++;
3180	continue;
3181	}
3182	}
3183	else if ( (PdeDst.u & X86_PDE_P)
3184	\|\| ((PdeDst.u & (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY))
3185	)
3186	{
3187	HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
3188	PPGMPOOLPAGE pPoolPage = pgmPoolGetPageByHCPhys(pVM, HCPhysShw);
3189	if (!pPoolPage)
3190	{
3191	AssertMsgFailed(("Invalid page table address %VGp at %VGv! PdeDst=%#RX64\n",
3192	HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
3193	cErrors++;
3194	continue;
3195	}
3196	const SHWPT pPTDst = (const SHWPT )PGMPOOL_PAGE_2_PTR(pVM, pPoolPage);
3197
3198	if (PdeDst.u & (X86_PDE4M_PWT \| X86_PDE4M_PCD))
3199	{
3200	AssertMsgFailed(("PDE flags PWT and/or PCD is set at %VGv! These flags are not virtualized! PdeDst=%#RX64\n",
3201	GCPtr, (uint64_t)PdeDst.u));
3202	cErrors++;
3203	}
3204
3205	if (PdeDst.u & (X86_PDE4M_G \| X86_PDE4M_D))
3206	{
3207	AssertMsgFailed(("4K PDE reserved flags at %VGv! PdeDst=%#RX64\n",
3208	GCPtr, (uint64_t)PdeDst.u));
3209	cErrors++;
3210	}
3211
3212	const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
3213	if (!PdeSrc.n.u1Present)
3214	{
3215	AssertMsgFailed(("Guest PDE at %VGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
3216	GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
3217	cErrors++;
3218	continue;
3219	}
3220
3221	if ( !PdeSrc.b.u1Size
3222	\|\| !(cr4 & X86_CR4_PSE))
3223	{
3224	GCPhysGst = PdeSrc.u & GST_PDE_PG_MASK;
3225	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3226	GCPhysGst \|= (iPDDst & 1) * (PAGE_SIZE / 2);
3227	# endif
3228	}
3229	else
3230	{
3231	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3232	if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
3233	{
3234	AssertMsgFailed(("Guest PDE at %VGv is using PSE36 or similar! PdeSrc=%#RX64\n",
3235	GCPtr, (uint64_t)PdeSrc.u));
3236	cErrors++;
3237	continue;
3238	}
3239	# endif
3240	GCPhysGst = PdeSrc.u & GST_PDE_BIG_PG_MASK;
3241	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3242	GCPhysGst \|= GCPtr & RT_BIT(X86_PAGE_2M_SHIFT);
3243	# endif
3244	}
3245
3246	if ( pPoolPage->enmKind
3247	!= (!PdeSrc.b.u1Size \|\| !(cr4 & X86_CR4_PSE) ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
3248	{
3249	AssertMsgFailed(("Invalid shadow page table kind %d at %VGv! PdeSrc=%#RX64\n",
3250	pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
3251	cErrors++;
3252	}
3253
3254	PPGMPAGE pPhysPage = pgmPhysGetPage(pPGM, GCPhysGst);
3255	if (!pPhysPage)
3256	{
3257	AssertMsgFailed(("Cannot find guest physical address %VGp in the PDE at %VGv! PdeSrc=%#RX64\n",
3258	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3259	cErrors++;
3260	continue;
3261	}
3262
3263	if (GCPhysGst != pPoolPage->GCPhys)
3264	{
3265	AssertMsgFailed(("GCPhysGst=%VGp != pPage->GCPhys=%VGp at %VGv\n",
3266	GCPhysGst, pPoolPage->GCPhys, GCPtr));
3267	cErrors++;
3268	continue;
3269	}
3270
3271	if ( !PdeSrc.b.u1Size
3272	\|\| !(cr4 & X86_CR4_PSE))
3273	{
3274	/*
3275	* Page Table.
3276	*/
3277	const GSTPT *pPTSrc;
3278	rc = PGM_GCPHYS_2_PTR(pVM, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1), &pPTSrc);
3279	if (VBOX_FAILURE(rc))
3280	{
3281	AssertMsgFailed(("Cannot map/convert guest physical address %VGp in the PDE at %VGv! PdeSrc=%#RX64\n",
3282	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3283	cErrors++;
3284	continue;
3285	}
3286	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/))
3287	!= (PdeDst.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/)))
3288	{
3289	/// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
3290	// (This problem will go away when/if we shadow multiple CR3s.)
3291	AssertMsgFailed(("4K PDE flags mismatch at %VGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3292	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3293	cErrors++;
3294	continue;
3295	}
3296	if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3297	{
3298	AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%VGv PdeDst=%#RX64\n",
3299	GCPtr, (uint64_t)PdeDst.u));
3300	cErrors++;
3301	continue;
3302	}
3303
3304	/* iterate the page table. */
3305	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3306	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3307	const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
3308	# else
3309	const unsigned offPTSrc = 0;
3310	# endif
3311	for (unsigned iPT = 0, off = 0;
3312	iPT < ELEMENTS(pPTDst->a);
3313	iPT++, off += PAGE_SIZE)
3314	{
3315	const SHWPTE PteDst = pPTDst->a[iPT];
3316
3317	/* skip not-present entries. */
3318	if (!(PteDst.u & (X86_PTE_P \| PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
3319	continue;
3320	Assert(PteDst.n.u1Present);
3321
3322	const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
3323	if (!PteSrc.n.u1Present)
3324	{
3325	#ifdef IN_RING3
3326	PGMAssertHandlerAndFlagsInSync(pVM);
3327	PGMR3DumpHierarchyGC(pVM, cr3, cr4, (PdeSrc.u & GST_PDE_PG_MASK));
3328	#endif
3329	AssertMsgFailed(("Out of sync (!P) PTE at %VGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%VGv iPTSrc=%x PdeSrc=%x physpte=%VGp\n",
3330	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u, pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
3331	(PdeSrc.u & GST_PDE_PG_MASK) + (iPT + offPTSrc)*sizeof(PteSrc)));
3332	cErrors++;
3333	continue;
3334	}
3335
3336	uint64_t fIgnoreFlags = GST_PTE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_G \| X86_PTE_D \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
3337	# if 1 /** @todo sync accessed bit properly... */
3338	fIgnoreFlags \|= X86_PTE_A;
3339	# endif
3340
3341	/* match the physical addresses */
3342	HCPhysShw = PteDst.u & SHW_PTE_PG_MASK;
3343	GCPhysGst = PteSrc.u & GST_PTE_PG_MASK;
3344
3345	# ifdef IN_RING3
3346	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3347	if (VBOX_FAILURE(rc))
3348	{
3349	if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3350	{
3351	AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3352	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3353	cErrors++;
3354	continue;
3355	}
3356	}
3357	else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
3358	{
3359	AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PteSrc=%#RX64 PteDst=%#RX64\n",
3360	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3361	cErrors++;
3362	continue;
3363	}
3364	# endif
3365
3366	pPhysPage = pgmPhysGetPage(pPGM, GCPhysGst);
3367	if (!pPhysPage)
3368	{
3369	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
3370	if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3371	{
3372	AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3373	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3374	cErrors++;
3375	continue;
3376	}
3377	# endif
3378	if (PteDst.n.u1Write)
3379	{
3380	AssertMsgFailed(("Invalid guest page at %VGv is writable! GCPhysGst=%VGp PteSrc=%#RX64 PteDst=%#RX64\n",
3381	GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3382	cErrors++;
3383	}
3384	fIgnoreFlags \|= X86_PTE_RW;
3385	}
3386	else if (HCPhysShw != (PGM_PAGE_GET_HCPHYS(pPhysPage) & SHW_PTE_PG_MASK))
3387	{
3388	AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PteSrc=%#RX64 PteDst=%#RX64\n",
3389	GCPtr + off, HCPhysShw, pPhysPage->HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3390	cErrors++;
3391	continue;
3392	}
3393
3394	/* flags */
3395	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
3396	{
3397	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
3398	{
3399	if (PteDst.n.u1Write)
3400	{
3401	AssertMsgFailed(("WRITE access flagged at %VGv but the page is writable! HCPhys=%VGv PteSrc=%#RX64 PteDst=%#RX64\n",
3402	GCPtr + off, pPhysPage->HCPhys, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3403	cErrors++;
3404	continue;
3405	}
3406	fIgnoreFlags \|= X86_PTE_RW;
3407	}
3408	else
3409	{
3410	if (PteDst.n.u1Present)
3411	{
3412	AssertMsgFailed(("ALL access flagged at %VGv but the page is present! HCPhys=%VHp PteSrc=%#RX64 PteDst=%#RX64\n",
3413	GCPtr + off, pPhysPage->HCPhys, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3414	cErrors++;
3415	continue;
3416	}
3417	fIgnoreFlags \|= X86_PTE_P;
3418	}
3419	}
3420	else
3421	{
3422	if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
3423	{
3424	if (PteDst.n.u1Write)
3425	{
3426	AssertMsgFailed(("!DIRTY page at %VGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
3427	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3428	cErrors++;
3429	continue;
3430	}
3431	if (!(PteDst.u & PGM_PTFLAGS_TRACK_DIRTY))
3432	{
3433	AssertMsgFailed(("!DIRTY page at %VGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3434	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3435	cErrors++;
3436	continue;
3437	}
3438	if (PteDst.n.u1Dirty)
3439	{
3440	AssertMsgFailed(("!DIRTY page at %VGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3441	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3442	cErrors++;
3443	}
3444	# if 0 /** @todo sync access bit properly... */
3445	if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
3446	{
3447	AssertMsgFailed(("!DIRTY page at %VGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
3448	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3449	cErrors++;
3450	}
3451	fIgnoreFlags \|= X86_PTE_RW;
3452	# else
3453	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
3454	# endif
3455	}
3456	else if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
3457	{
3458	/* access bit emulation (not implemented). */
3459	if (PteSrc.n.u1Accessed \|\| PteDst.n.u1Present)
3460	{
3461	AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %VGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
3462	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3463	cErrors++;
3464	continue;
3465	}
3466	if (!PteDst.n.u1Accessed)
3467	{
3468	AssertMsgFailed(("!ACCESSED page at %VGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
3469	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3470	cErrors++;
3471	}
3472	fIgnoreFlags \|= X86_PTE_P;
3473	}
3474	# ifdef DEBUG_sandervl
3475	fIgnoreFlags \|= X86_PTE_D \| X86_PTE_A;
3476	# endif
3477	}
3478
3479	if ( (PteSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
3480	&& (PteSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags)
3481	)
3482	{
3483	AssertMsgFailed(("Flags mismatch at %VGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
3484	GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
3485	fIgnoreFlags, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3486	cErrors++;
3487	continue;
3488	}
3489	} /* foreach PTE */
3490	}
3491	else
3492	{
3493	/*
3494	* Big Page.
3495	*/
3496	uint64_t fIgnoreFlags = X86_PDE_AVL_MASK \| GST_PDE_PG_MASK \| X86_PDE4M_G \| X86_PDE4M_D \| X86_PDE4M_PS \| X86_PDE4M_PWT \| X86_PDE4M_PCD;
3497	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
3498	{
3499	if (PdeDst.n.u1Write)
3500	{
3501	AssertMsgFailed(("!DIRTY page at %VGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3502	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3503	cErrors++;
3504	continue;
3505	}
3506	if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
3507	{
3508	AssertMsgFailed(("!DIRTY page at %VGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3509	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3510	cErrors++;
3511	continue;
3512	}
3513	# if 0 /** @todo sync access bit properly... */
3514	if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
3515	{
3516	AssertMsgFailed(("!DIRTY page at %VGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
3517	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3518	cErrors++;
3519	}
3520	fIgnoreFlags \|= X86_PTE_RW;
3521	# else
3522	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
3523	# endif
3524	}
3525	else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3526	{
3527	/* access bit emulation (not implemented). */
3528	if (PdeSrc.b.u1Accessed \|\| PdeDst.n.u1Present)
3529	{
3530	AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %VGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3531	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3532	cErrors++;
3533	continue;
3534	}
3535	if (!PdeDst.n.u1Accessed)
3536	{
3537	AssertMsgFailed(("!ACCESSED page at %VGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3538	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3539	cErrors++;
3540	}
3541	fIgnoreFlags \|= X86_PTE_P;
3542	}
3543
3544	if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
3545	{
3546	AssertMsgFailed(("Flags mismatch (B) at %VGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
3547	GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
3548	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3549	cErrors++;
3550	}
3551
3552	/* iterate the page table. */
3553	for (unsigned iPT = 0, off = 0;
3554	iPT < ELEMENTS(pPTDst->a);
3555	iPT++, off += PAGE_SIZE, GCPhysGst += PAGE_SIZE)
3556	{
3557	const SHWPTE PteDst = pPTDst->a[iPT];
3558
3559	if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
3560	{
3561	AssertMsgFailed(("The PTE at %VGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
3562	GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3563	cErrors++;
3564	}
3565
3566	/* skip not-present entries. */
3567	if (!PteDst.n.u1Present) /** @todo deal with ALL handlers and CSAM !P pages! */
3568	continue;
3569
3570	fIgnoreFlags = X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
3571
3572	/* match the physical addresses */
3573	HCPhysShw = PteDst.u & X86_PTE_PAE_PG_MASK;
3574
3575	# ifdef IN_RING3
3576	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3577	if (VBOX_FAILURE(rc))
3578	{
3579	if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3580	{
3581	AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
3582	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3583	cErrors++;
3584	}
3585	}
3586	else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
3587	{
3588	AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PdeSrc=%#RX64 PteDst=%#RX64\n",
3589	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3590	cErrors++;
3591	continue;
3592	}
3593	# endif
3594
3595	pPhysPage = pgmPhysGetPage(pPGM, GCPhysGst);
3596	if (!pPhysPage)
3597	{
3598	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
3599	if (HCPhysShw != MMR3PageDummyHCPhys(pVM))
3600	{
3601	AssertMsgFailed(("Cannot find guest physical address %VGp at %VGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
3602	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3603	cErrors++;
3604	continue;
3605	}
3606	# endif
3607	if (PteDst.n.u1Write)
3608	{
3609	AssertMsgFailed(("Invalid guest page at %VGv is writable! GCPhysGst=%VGp PdeSrc=%#RX64 PteDst=%#RX64\n",
3610	GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3611	cErrors++;
3612	}
3613	fIgnoreFlags \|= X86_PTE_RW;
3614	}
3615	else if (HCPhysShw != (pPhysPage->HCPhys & X86_PTE_PAE_PG_MASK))
3616	{
3617	AssertMsgFailed(("Out of sync (phys) at %VGv! HCPhysShw=%VHp HCPhys=%VHp GCPhysGst=%VGp PdeSrc=%#RX64 PteDst=%#RX64\n",
3618	GCPtr + off, HCPhysShw, pPhysPage->HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3619	cErrors++;
3620	continue;
3621	}
3622
3623	/* flags */
3624	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
3625	{
3626	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
3627	{
3628	if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
3629	{
3630	if (PteDst.n.u1Write)
3631	{
3632	AssertMsgFailed(("WRITE access flagged at %VGv but the page is writable! HCPhys=%VGv PdeSrc=%#RX64 PteDst=%#RX64\n",
3633	GCPtr + off, pPhysPage->HCPhys, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3634	cErrors++;
3635	continue;
3636	}
3637	fIgnoreFlags \|= X86_PTE_RW;
3638	}
3639	}
3640	else
3641	{
3642	if (PteDst.n.u1Present)
3643	{
3644	AssertMsgFailed(("ALL access flagged at %VGv but the page is present! HCPhys=%VGv PdeSrc=%#RX64 PteDst=%#RX64\n",
3645	GCPtr + off, pPhysPage->HCPhys, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3646	cErrors++;
3647	continue;
3648	}
3649	fIgnoreFlags \|= X86_PTE_P;
3650	}
3651	}
3652
3653	if ( (PdeSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
3654	&& (PdeSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags) /* lazy phys handler dereg. */
3655	)
3656	{
3657	AssertMsgFailed(("Flags mismatch (BT) at %VGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
3658	GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
3659	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
3660	cErrors++;
3661	continue;
3662	}
3663	} /* foreach PTE */
3664	}
3665	}
3666	/* not present */
3667
3668	} /* forearch PDE */
3669
3670	# ifdef DEBUG
3671	if (cErrors)
3672	LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
3673	# endif
3674
3675	#elif PGM_GST_TYPE == PGM_TYPE_PAE
3676	//# error not implemented
3677
3678
3679	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
3680	//# error not implemented
3681
3682	/#else: guest real and protected mode /
3683	#endif
3684	return cErrors;
3685	}
3686	#endif /* VBOX_STRICT */
3687

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 8564

Download in other formats: