PGMAllBth.h@ 25776

Last change on this file since 25776 was 25579, checked in by vboxsync, 15 years ago
COmpile fix
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 198.7 KB

Line
1	/* $Id: PGMAllBth.h 25579 2009-12-23 17:02:29Z vboxsync $ */
2	/** @file
3	* VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4	*
5	* This file is a big challenge!
6	*/
7
8	/*
9	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
10	*
11	* This file is part of VirtualBox Open Source Edition (OSE), as
12	* available from http://www.virtualbox.org. This file is free software;
13	* you can redistribute it and/or modify it under the terms of the GNU
14	* General Public License (GPL) as published by the Free Software
15	* Foundation, in version 2 as it comes in the "COPYING" file of the
16	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
17	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
18	*
19	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
20	* Clara, CA 95054 USA or visit http://www.sun.com if you need
21	* additional information or have any questions.
22	*/
23
24	/*******************************************************************************
25	* Internal Functions *
26	*******************************************************************************/
27	RT_C_DECLS_BEGIN
28	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault);
29	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
30	PGM_BTH_DECL(int, SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr);
31	PGM_BTH_DECL(int, CheckPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCPTR GCPtrPage);
32	PGM_BTH_DECL(int, SyncPT)(PVMCPU pVCpu, unsigned iPD, PGSTPD pPDSrc, RTGCPTR GCPtrPage);
33	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uErr);
34	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
35	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
36	#ifdef VBOX_STRICT
37	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
38	#endif
39	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys);
40	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3);
41	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu);
42	RT_C_DECLS_END
43
44
45	/* Filter out some illegal combinations of guest and shadow paging, so we can remove redundant checks inside functions. */
46	#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
47	# error "Invalid combination; PAE guest implies PAE shadow"
48	#endif
49
50	#if (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
51	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT)
52	# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
53	#endif
54
55	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) \
56	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT)
57	# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
58	#endif
59
60	#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT) \
61	\|\| (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
62	# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
63	#endif
64
65	#ifdef IN_RING0 /* no mappings in VT-x and AMD-V mode */
66	# define PGM_WITHOUT_MAPPINGS
67	#endif
68
69
70	#ifndef IN_RING3
71	/**
72	* #PF Handler for raw-mode guest execution.
73	*
74	* @returns VBox status code (appropriate for trap handling and GC return).
75	*
76	* @param pVCpu VMCPU Handle.
77	* @param uErr The trap error code.
78	* @param pRegFrame Trap register frame.
79	* @param pvFault The fault address.
80	*/
81	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
82	{
83	PVM pVM = pVCpu->CTX_SUFF(pVM);
84
85	# if defined(IN_RC) && defined(VBOX_STRICT)
86	PGMDynCheckLocks(pVM);
87	# endif
88
89	# if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
90	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
91	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT)
92
93	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE != PGM_TYPE_PAE
94	/*
95	* Hide the instruction fetch trap indicator for now.
96	*/
97	/** @todo NXE will change this and we must fix NXE in the switcher too! */
98	if (uErr & X86_TRAP_PF_ID)
99	{
100	uErr &= ~X86_TRAP_PF_ID;
101	TRPMSetErrorCode(pVCpu, uErr);
102	}
103	# endif
104
105	/*
106	* Get PDs.
107	*/
108	int rc;
109	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
110	# if PGM_GST_TYPE == PGM_TYPE_32BIT
111	const unsigned iPDSrc = pvFault >> GST_PD_SHIFT;
112	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
113
114	# elif PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64
115
116	# if PGM_GST_TYPE == PGM_TYPE_PAE
117	unsigned iPDSrc = 0; /* initialized to shut up gcc */
118	X86PDPE PdpeSrc;
119	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, pvFault, &iPDSrc, &PdpeSrc);
120
121	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
122	unsigned iPDSrc = 0; /* initialized to shut up gcc */
123	PX86PML4E pPml4eSrc;
124	X86PDPE PdpeSrc;
125	PGSTPD pPDSrc;
126
127	pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, pvFault, &pPml4eSrc, &PdpeSrc, &iPDSrc);
128	Assert(pPml4eSrc);
129	# endif
130
131	/* Quick check for a valid guest trap. (PAE & AMD64) */
132	if (!pPDSrc)
133	{
134	# if PGM_GST_TYPE == PGM_TYPE_AMD64 && GC_ARCH_BITS == 64
135	LogFlow(("Trap0eHandler: guest PML4 %d not present CR3=%RGp\n", (int)((pvFault >> X86_PML4_SHIFT) & X86_PML4_MASK), CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK));
136	# else
137	LogFlow(("Trap0eHandler: guest iPDSrc=%u not present CR3=%RGp\n", iPDSrc, CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK));
138	# endif
139	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2GuestTrap; });
140	TRPMSetErrorCode(pVCpu, uErr);
141	return VINF_EM_RAW_GUEST_TRAP;
142	}
143	# endif
144
145	# else /* !PGM_WITH_PAGING */
146	PGSTPD pPDSrc = NULL;
147	const unsigned iPDSrc = 0;
148	# endif /* !PGM_WITH_PAGING */
149
150	/* Fetch the guest PDE */
151	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
152	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
153	# else
154	GSTPDE PdeSrc;
155	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
156	PdeSrc.n.u1Present = 1;
157	PdeSrc.n.u1Write = 1;
158	PdeSrc.n.u1Accessed = 1;
159	PdeSrc.n.u1User = 1;
160	# endif
161
162	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
163	const unsigned iPDDst = pvFault >> SHW_PD_SHIFT;
164	PX86PD pPDDst = pgmShwGet32BitPDPtr(&pVCpu->pgm.s);
165
166	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
167	const unsigned iPDDst = (pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK; /* pPDDst index, not used with the pool. */
168
169	PX86PDPAE pPDDst;
170	# if PGM_GST_TYPE != PGM_TYPE_PAE
171	X86PDPE PdpeSrc;
172
173	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
174	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
175	# endif
176	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, &PdpeSrc, &pPDDst);
177	if (rc != VINF_SUCCESS)
178	{
179	AssertRC(rc);
180	return rc;
181	}
182	Assert(pPDDst);
183
184	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
185	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
186	PX86PDPAE pPDDst;
187	# if PGM_GST_TYPE == PGM_TYPE_PROT
188	/* AMD-V nested paging */
189	X86PML4E Pml4eSrc;
190	X86PDPE PdpeSrc;
191	PX86PML4E pPml4eSrc = &Pml4eSrc;
192
193	/* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
194	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
195	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
196	# endif
197
198	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, pPml4eSrc, &PdpeSrc, &pPDDst);
199	if (rc != VINF_SUCCESS)
200	{
201	AssertRC(rc);
202	return rc;
203	}
204	Assert(pPDDst);
205
206	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
207	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
208	PEPTPD pPDDst;
209
210	rc = pgmShwGetEPTPDPtr(pVCpu, pvFault, NULL, &pPDDst);
211	if (rc != VINF_SUCCESS)
212	{
213	AssertRC(rc);
214	return rc;
215	}
216	Assert(pPDDst);
217	# endif
218
219	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
220	/*
221	* If we successfully correct the write protection fault due to dirty bit
222	* tracking, or this page fault is a genuine one, then return immediately.
223	*/
224	STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeCheckPageFault, e);
225	rc = PGM_BTH_NAME(CheckPageFault)(pVCpu, uErr, &pPDDst->a[iPDDst], &pPDSrc->a[iPDSrc], pvFault);
226	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeCheckPageFault, e);
227	if ( rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
228	\|\| rc == VINF_EM_RAW_GUEST_TRAP)
229	{
230	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution)
231	= rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? &pVCpu->pgm.s.StatRZTrap0eTime2DirtyAndAccessed : &pVCpu->pgm.s.StatRZTrap0eTime2GuestTrap; });
232	LogBird(("Trap0eHandler: returns %s\n", rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? "VINF_SUCCESS" : "VINF_EM_RAW_GUEST_TRAP"));
233	return rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? VINF_SUCCESS : rc;
234	}
235
236	# if 0 /* rarely useful; leave for debugging. */
237	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0ePD[iPDSrc]);
238	# endif
239	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
240
241	/*
242	* A common case is the not-present error caused by lazy page table syncing.
243	*
244	* It is IMPORTANT that we weed out any access to non-present shadow PDEs here
245	* so we can safely assume that the shadow PT is present when calling SyncPage later.
246	*
247	* On failure, we ASSUME that SyncPT is out of memory or detected some kind
248	* of mapping conflict and defer to SyncCR3 in R3.
249	* (Again, we do NOT support access handlers for non-present guest pages.)
250	*
251	*/
252	if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
253	&& !pPDDst->a[iPDDst].n.u1Present
254	&& PdeSrc.n.u1Present
255	)
256	{
257	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2SyncPT; });
258	STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeSyncPT, f);
259	LogFlow(("=>SyncPT %04x = %08x\n", iPDSrc, PdeSrc.au32[0]));
260	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, pvFault);
261	if (RT_SUCCESS(rc))
262	{
263	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeSyncPT, f);
264	return rc;
265	}
266	Log(("SyncPT: %d failed!! rc=%d\n", iPDSrc, rc));
267	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
268	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeSyncPT, f);
269	return VINF_PGM_SYNC_CR3;
270	}
271
272	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(PGM_WITHOUT_MAPPINGS)
273	/*
274	* Check if this address is within any of our mappings.
275	*
276	* This is very fast and it's gonna save us a bit of effort below and prevent
277	* us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
278	* (BTW, it's impossible to have physical access handlers in a mapping.)
279	*/
280	if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
281	{
282	STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
283	PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
284	for ( ; pMapping; pMapping = pMapping->CTX_SUFF(pNext))
285	{
286	if (pvFault < pMapping->GCPtr)
287	break;
288	if (pvFault - pMapping->GCPtr < pMapping->cb)
289	{
290	/*
291	* The first thing we check is if we've got an undetected conflict.
292	*/
293	if (!pVM->pgm.s.fMappingsFixed)
294	{
295	unsigned iPT = pMapping->cb >> GST_PD_SHIFT;
296	while (iPT-- > 0)
297	if (pPDSrc->a[iPDSrc + iPT].n.u1Present)
298	{
299	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eConflicts);
300	Log(("Trap0e: Detected Conflict %RGv-%RGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
301	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
302	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
303	return VINF_PGM_SYNC_CR3;
304	}
305	}
306
307	/*
308	* Check if the fault address is in a virtual page access handler range.
309	*/
310	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->HyperVirtHandlers, pvFault);
311	if ( pCur
312	&& pvFault - pCur->Core.Key < pCur->cb
313	&& uErr & X86_TRAP_PF_RW)
314	{
315	# ifdef IN_RC
316	STAM_PROFILE_START(&pCur->Stat, h);
317	pgmUnlock(pVM);
318	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
319	pgmLock(pVM);
320	STAM_PROFILE_STOP(&pCur->Stat, h);
321	# else
322	AssertFailed();
323	rc = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
324	# endif
325	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersMapping);
326	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
327	return rc;
328	}
329
330	/*
331	* Pretend we're not here and let the guest handle the trap.
332	*/
333	TRPMSetErrorCode(pVCpu, uErr & ~X86_TRAP_PF_P);
334	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eGuestPFMapping);
335	LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
336	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
337	return VINF_EM_RAW_GUEST_TRAP;
338	}
339	}
340	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
341	} /* pgmAreMappingsEnabled(&pVM->pgm.s) */
342	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
343
344	/*
345	* Check if this fault address is flagged for special treatment,
346	* which means we'll have to figure out the physical address and
347	* check flags associated with it.
348	*
349	* ASSUME that we can limit any special access handling to pages
350	* in page tables which the guest believes to be present.
351	*/
352	if (PdeSrc.n.u1Present)
353	{
354	RTGCPHYS GCPhys = NIL_RTGCPHYS;
355
356	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
357	# if PGM_GST_TYPE == PGM_TYPE_AMD64
358	bool fBigPagesSupported = true;
359	# else
360	bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
361	# endif
362	if ( PdeSrc.b.u1Size
363	&& fBigPagesSupported)
364	GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc)
365	\| ((RTGCPHYS)pvFault & (GST_BIG_PAGE_OFFSET_MASK ^ PAGE_OFFSET_MASK));
366	else
367	{
368	PGSTPT pPTSrc;
369	rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
370	if (RT_SUCCESS(rc))
371	{
372	unsigned iPTESrc = (pvFault >> GST_PT_SHIFT) & GST_PT_MASK;
373	if (pPTSrc->a[iPTESrc].n.u1Present)
374	GCPhys = pPTSrc->a[iPTESrc].u & GST_PTE_PG_MASK;
375	}
376	}
377	# else
378	/* No paging so the fault address is the physical address */
379	GCPhys = (RTGCPHYS)(pvFault & ~PAGE_OFFSET_MASK);
380	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
381
382	/*
383	* If we have a GC address we'll check if it has any flags set.
384	*/
385	if (GCPhys != NIL_RTGCPHYS)
386	{
387	STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
388
389	PPGMPAGE pPage;
390	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
391	if (RT_SUCCESS(rc)) /** just handle the failure immediate (it returns) and make things easier to read. */
392	{
393	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
394	{
395	if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
396	{
397	/*
398	* Physical page access handler.
399	*/
400	const RTGCPHYS GCPhysFault = GCPhys \| (pvFault & PAGE_OFFSET_MASK);
401	PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhysFault);
402	if (pCur)
403	{
404	# ifdef PGM_SYNC_N_PAGES
405	/*
406	* If the region is write protected and we got a page not present fault, then sync
407	* the pages. If the fault was caused by a read, then restart the instruction.
408	* In case of write access continue to the GC write handler.
409	*
410	* ASSUMES that there is only one handler per page or that they have similar write properties.
411	*/
412	if ( pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
413	&& !(uErr & X86_TRAP_PF_P))
414	{
415	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, PGM_SYNC_NR_PAGES, uErr);
416	if ( RT_FAILURE(rc)
417	\|\| !(uErr & X86_TRAP_PF_RW)
418	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
419	{
420	AssertRC(rc);
421	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersOutOfSync);
422	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
423	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSyncHndPhys; });
424	return rc;
425	}
426	}
427	# endif
428
429	AssertMsg( pCur->enmType != PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
430	\|\| (pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE && (uErr & X86_TRAP_PF_RW)),
431	("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enum=%d\n", pvFault, GCPhys, pPage, uErr, pCur->enmType));
432
433	# if defined(IN_RC) \|\| defined(IN_RING0)
434	if (pCur->CTX_SUFF(pfnHandler))
435	{
436	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
437	# ifdef IN_RING0
438	PFNPGMR0PHYSHANDLER pfnHandler = pCur->CTX_SUFF(pfnHandler);
439	# else
440	PFNPGMRCPHYSHANDLER pfnHandler = pCur->CTX_SUFF(pfnHandler);
441	# endif
442	bool fLeaveLock = (pfnHandler != pPool->CTX_SUFF(pfnAccessHandler));
443	void *pvUser = pCur->CTX_SUFF(pvUser);
444
445	STAM_PROFILE_START(&pCur->Stat, h);
446	if (fLeaveLock)
447	pgmUnlock(pVM); /* @todo: Not entirely safe. */
448
449	rc = pfnHandler(pVM, uErr, pRegFrame, pvFault, GCPhysFault, pvUser);
450	if (fLeaveLock)
451	pgmLock(pVM);
452	# ifdef VBOX_WITH_STATISTICS
453	pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhysFault);
454	if (pCur)
455	STAM_PROFILE_STOP(&pCur->Stat, h);
456	# else
457	pCur = NULL; /* might be invalid by now. */
458	# endif
459
460	}
461	else
462	# endif
463	rc = VINF_EM_RAW_EMULATE_INSTR;
464
465	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersPhysical);
466	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
467	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndPhys; });
468	return rc;
469	}
470	}
471	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
472	else
473	{
474	# ifdef PGM_SYNC_N_PAGES
475	/*
476	* If the region is write protected and we got a page not present fault, then sync
477	* the pages. If the fault was caused by a read, then restart the instruction.
478	* In case of write access continue to the GC write handler.
479	*/
480	if ( PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) < PGM_PAGE_HNDL_PHYS_STATE_ALL
481	&& !(uErr & X86_TRAP_PF_P))
482	{
483	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, PGM_SYNC_NR_PAGES, uErr);
484	if ( RT_FAILURE(rc)
485	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
486	\|\| !(uErr & X86_TRAP_PF_RW))
487	{
488	AssertRC(rc);
489	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersOutOfSync);
490	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
491	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSyncHndVirt; });
492	return rc;
493	}
494	}
495	# endif
496	/*
497	* Ok, it's an virtual page access handler.
498	*
499	* Since it's faster to search by address, we'll do that first
500	* and then retry by GCPhys if that fails.
501	*/
502	/** @todo r=bird: perhaps we should consider looking up by physical address directly now? */
503	/** @note r=svl: true, but lookup on virtual address should remain as a fallback as phys & virt trees might be out of sync, because the
504	* page was changed without us noticing it (not-present -> present without invlpg or mov cr3, xxx)
505	*/
506	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
507	if (pCur)
508	{
509	AssertMsg(!(pvFault - pCur->Core.Key < pCur->cb)
510	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
511	\|\| !(uErr & X86_TRAP_PF_P)
512	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
513	("Unexpected trap for virtual handler: %RGv (phys=%RGp) pPage=%R[pgmpage] uErr=%X, enum=%d\n", pvFault, GCPhys, pPage, uErr, pCur->enmType));
514
515	if ( pvFault - pCur->Core.Key < pCur->cb
516	&& ( uErr & X86_TRAP_PF_RW
517	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
518	{
519	# ifdef IN_RC
520	STAM_PROFILE_START(&pCur->Stat, h);
521	pgmUnlock(pVM);
522	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
523	pgmLock(pVM);
524	STAM_PROFILE_STOP(&pCur->Stat, h);
525	# else
526	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
527	# endif
528	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersVirtual);
529	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
530	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndVirt; });
531	return rc;
532	}
533	/* Unhandled part of a monitored page */
534	}
535	else
536	{
537	/* Check by physical address. */
538	unsigned iPage;
539	rc = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys + (pvFault & PAGE_OFFSET_MASK),
540	&pCur, &iPage);
541	Assert(RT_SUCCESS(rc) \|\| !pCur);
542	if ( pCur
543	&& ( uErr & X86_TRAP_PF_RW
544	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
545	{
546	Assert((pCur->aPhysToVirt[iPage].Core.Key & X86_PTE_PAE_PG_MASK) == GCPhys);
547	# ifdef IN_RC
548	RTGCPTR off = (iPage << PAGE_SHIFT) + (pvFault & PAGE_OFFSET_MASK) - (pCur->Core.Key & PAGE_OFFSET_MASK);
549	Assert(off < pCur->cb);
550	STAM_PROFILE_START(&pCur->Stat, h);
551	pgmUnlock(pVM);
552	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, off);
553	pgmLock(pVM);
554	STAM_PROFILE_STOP(&pCur->Stat, h);
555	# else
556	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
557	# endif
558	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersVirtualByPhys);
559	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
560	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndVirt; });
561	return rc;
562	}
563	}
564	}
565	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
566
567	/*
568	* There is a handled area of the page, but this fault doesn't belong to it.
569	* We must emulate the instruction.
570	*
571	* To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
572	* we first check if this was a page-not-present fault for a page with only
573	* write access handlers. Restart the instruction if it wasn't a write access.
574	*/
575	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersUnhandled);
576
577	if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
578	&& !(uErr & X86_TRAP_PF_P))
579	{
580	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, PGM_SYNC_NR_PAGES, uErr);
581	if ( RT_FAILURE(rc)
582	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
583	\|\| !(uErr & X86_TRAP_PF_RW))
584	{
585	AssertRC(rc);
586	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersOutOfSync);
587	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
588	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSyncHndPhys; });
589	return rc;
590	}
591	}
592
593	/** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
594	* It's writing to an unhandled part of the LDT page several million times.
595	*/
596	rc = PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault);
597	LogFlow(("PGM: PGMInterpretInstruction -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
598	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
599	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndUnhandled; });
600	return rc;
601	} /* if any kind of handler */
602
603	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
604	if (uErr & X86_TRAP_PF_P)
605	{
606	/*
607	* The page isn't marked, but it might still be monitored by a virtual page access handler.
608	* (ASSUMES no temporary disabling of virtual handlers.)
609	*/
610	/** @todo r=bird: Since the purpose is to catch out of sync pages with virtual handler(s) here,
611	* we should correct both the shadow page table and physical memory flags, and not only check for
612	* accesses within the handler region but for access to pages with virtual handlers. */
613	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
614	if (pCur)
615	{
616	AssertMsg( !(pvFault - pCur->Core.Key < pCur->cb)
617	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
618	\|\| !(uErr & X86_TRAP_PF_P)
619	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
620	("Unexpected trap for virtual handler: %08X (phys=%08x) %R[pgmpage] uErr=%X, enum=%d\n", pvFault, GCPhys, pPage, uErr, pCur->enmType));
621
622	if ( pvFault - pCur->Core.Key < pCur->cb
623	&& ( uErr & X86_TRAP_PF_RW
624	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
625	{
626	# ifdef IN_RC
627	STAM_PROFILE_START(&pCur->Stat, h);
628	pgmUnlock(pVM);
629	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
630	pgmLock(pVM);
631	STAM_PROFILE_STOP(&pCur->Stat, h);
632	# else
633	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
634	# endif
635	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersVirtualUnmarked);
636	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
637	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndVirt; });
638	return rc;
639	}
640	}
641	}
642	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
643	}
644	else
645	{
646	/*
647	* When the guest accesses invalid physical memory (e.g. probing
648	* of RAM or accessing a remapped MMIO range), then we'll fall
649	* back to the recompiler to emulate the instruction.
650	*/
651	LogFlow(("PGM #PF: pgmPhysGetPageEx(%RGp) failed with %Rrc\n", GCPhys, rc));
652	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersInvalid);
653	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
654	return VINF_EM_RAW_EMULATE_INSTR;
655	}
656
657	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
658
659	# ifdef PGM_OUT_OF_SYNC_IN_GC /** @todo remove this bugger. */
660	/*
661	* We are here only if page is present in Guest page tables and
662	* trap is not handled by our handlers.
663	*
664	* Check it for page out-of-sync situation.
665	*/
666	STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
667
668	if (!(uErr & X86_TRAP_PF_P))
669	{
670	/*
671	* Page is not present in our page tables.
672	* Try to sync it!
673	* BTW, fPageShw is invalid in this branch!
674	*/
675	if (uErr & X86_TRAP_PF_US)
676	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncUser));
677	else /* supervisor */
678	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
679
680	# if defined(LOG_ENABLED) && !defined(IN_RING0)
681	RTGCPHYS GCPhys2;
682	uint64_t fPageGst2;
683	PGMGstGetPage(pVCpu, pvFault, &fPageGst2, &GCPhys2);
684	Log(("Page out of sync: %RGv eip=%08x PdeSrc.n.u1User=%d fPageGst2=%08llx GCPhys2=%RGp scan=%d\n",
685	pvFault, pRegFrame->eip, PdeSrc.n.u1User, fPageGst2, GCPhys2, CSAMDoesPageNeedScanning(pVM, (RTRCPTR)pRegFrame->eip)));
686	# endif /* LOG_ENABLED */
687
688	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0)
689	if (CPUMGetGuestCPL(pVCpu, pRegFrame) == 0)
690	{
691	uint64_t fPageGst;
692	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
693	if ( RT_SUCCESS(rc)
694	&& !(fPageGst & X86_PTE_US))
695	{
696	/* Note: can't check for X86_TRAP_ID bit, because that requires execute disable support on the CPU */
697	if ( pvFault == (RTGCPTR)pRegFrame->eip
698	\|\| pvFault - pRegFrame->eip < 8 /* instruction crossing a page boundary */
699	# ifdef CSAM_DETECT_NEW_CODE_PAGES
700	\|\| ( !PATMIsPatchGCAddr(pVM, (RTGCPTR)pRegFrame->eip)
701	&& CSAMDoesPageNeedScanning(pVM, (RTRCPTR)pRegFrame->eip)) /* any new code we encounter here */
702	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
703	)
704	{
705	LogFlow(("CSAMExecFault %RX32\n", pRegFrame->eip));
706	rc = CSAMExecFault(pVM, (RTRCPTR)pRegFrame->eip);
707	if (rc != VINF_SUCCESS)
708	{
709	/*
710	* CSAM needs to perform a job in ring 3.
711	*
712	* Sync the page before going to the host context; otherwise we'll end up in a loop if
713	* CSAM fails (e.g. instruction crosses a page boundary and the next page is not present)
714	*/
715	LogFlow(("CSAM ring 3 job\n"));
716	int rc2 = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, 1, uErr);
717	AssertRC(rc2);
718
719	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
720	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2CSAM; });
721	return rc;
722	}
723	}
724	# ifdef CSAM_DETECT_NEW_CODE_PAGES
725	else if ( uErr == X86_TRAP_PF_RW
726	&& pRegFrame->ecx >= 0x100 /* early check for movswd count */
727	&& pRegFrame->ecx < 0x10000)
728	{
729	/* In case of a write to a non-present supervisor shadow page, we'll take special precautions
730	* to detect loading of new code pages.
731	*/
732
733	/*
734	* Decode the instruction.
735	*/
736	RTGCPTR PC;
737	rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &PC);
738	if (rc == VINF_SUCCESS)
739	{
740	PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
741	uint32_t cbOp;
742	rc = EMInterpretDisasOneEx(pVM, pVCpu, PC, pRegFrame, pDis, &cbOp);
743
744	/* For now we'll restrict this to rep movsw/d instructions */
745	if ( rc == VINF_SUCCESS
746	&& pDis->pCurInstr->opcode == OP_MOVSWD
747	&& (pDis->prefix & PREFIX_REP))
748	{
749	CSAMMarkPossibleCodePage(pVM, pvFault);
750	}
751	}
752	}
753	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
754
755	/*
756	* Mark this page as safe.
757	*/
758	/** @todo not correct for pages that contain both code and data!! */
759	Log2(("CSAMMarkPage %RGv; scanned=%d\n", pvFault, true));
760	CSAMMarkPage(pVM, (RTRCPTR)pvFault, true);
761	}
762	}
763	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0) */
764	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, PGM_SYNC_NR_PAGES, uErr);
765	if (RT_SUCCESS(rc))
766	{
767	/* The page was successfully synced, return to the guest. */
768	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
769	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSync; });
770	return VINF_SUCCESS;
771	}
772	}
773	else /* uErr & X86_TRAP_PF_P: */
774	{
775	/*
776	* Write protected pages are make writable when the guest makes the first
777	* write to it. This happens for pages that are shared, write monitored
778	* and not yet allocated.
779	*
780	* Also, a side effect of not flushing global PDEs are out of sync pages due
781	* to physical monitored regions, that are no longer valid.
782	* Assume for now it only applies to the read/write flag.
783	*/
784	if (RT_SUCCESS(rc) && (uErr & X86_TRAP_PF_RW))
785	{
786	if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
787	{
788	Log(("PGM #PF: Make writable: %RGp %R[pgmpage] pvFault=%RGp uErr=%#x\n",
789	GCPhys, pPage, pvFault, uErr));
790	rc = pgmPhysPageMakeWritableUnlocked(pVM, pPage, GCPhys);
791	if (rc != VINF_SUCCESS)
792	{
793	AssertMsg(rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("%Rrc\n", rc));
794	return rc;
795	}
796	if (RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)))
797	return VINF_EM_NO_MEMORY;
798	}
799
800	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
801	/* Check to see if we need to emulate the instruction as X86_CR0_WP has been cleared. */
802	if ( CPUMGetGuestCPL(pVCpu, pRegFrame) == 0
803	&& ((CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG))
804	{
805	Assert((uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RW \| X86_TRAP_PF_P));
806	uint64_t fPageGst;
807	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
808	if ( RT_SUCCESS(rc)
809	&& !(fPageGst & X86_PTE_RW))
810	{
811	rc = PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault);
812	if (RT_SUCCESS(rc))
813	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eWPEmulInRZ);
814	else
815	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eWPEmulToR3);
816	return rc;
817	}
818	AssertMsg(RT_SUCCESS(rc), ("Unexpected r/w page %RGv flag=%x rc=%Rrc\n", pvFault, (uint32_t)fPageGst, rc));
819	}
820	# endif
821	/// @todo count the above case; else
822	if (uErr & X86_TRAP_PF_US)
823	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncUserWrite));
824	else /* supervisor */
825	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
826
827	/*
828	* Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
829	* page is not present, which is not true in this case.
830	*/
831	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, 1, uErr);
832	if (RT_SUCCESS(rc))
833	{
834	/*
835	* Page was successfully synced, return to guest.
836	* First invalidate the page as it might be in the TLB.
837	*/
838	# if PGM_SHW_TYPE == PGM_TYPE_EPT
839	HWACCMInvalidatePhysPage(pVM, (RTGCPHYS)pvFault);
840	# else
841	PGM_INVL_PG(pVCpu, pvFault);
842	# endif
843	# ifdef VBOX_STRICT
844	RTGCPHYS GCPhys2;
845	uint64_t fPageGst;
846	if (!HWACCMIsNestedPagingActive(pVM))
847	{
848	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, &GCPhys2);
849	AssertMsg(RT_SUCCESS(rc) && (fPageGst & X86_PTE_RW), ("rc=%d fPageGst=%RX64\n"));
850	LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GCPhys2, (uint64_t)fPageGst));
851	}
852	uint64_t fPageShw;
853	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
854	AssertMsg((RT_SUCCESS(rc) && (fPageShw & X86_PTE_RW)) \|\| pVM->cCpus > 1 /* new monitor can be installed/page table flushed between the trap exit and PGMTrap0eHandler */, ("rc=%Rrc fPageShw=%RX64\n", rc, fPageShw));
855	# endif /* VBOX_STRICT */
856	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
857	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSyncHndObs; });
858	return VINF_SUCCESS;
859	}
860	}
861
862	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
863	# ifdef VBOX_STRICT
864	/*
865	* Check for VMM page flags vs. Guest page flags consistency.
866	* Currently only for debug purposes.
867	*/
868	if (RT_SUCCESS(rc))
869	{
870	/* Get guest page flags. */
871	uint64_t fPageGst;
872	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
873	if (RT_SUCCESS(rc))
874	{
875	uint64_t fPageShw;
876	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
877
878	/*
879	* Compare page flags.
880	* Note: we have AVL, A, D bits desynched.
881	*/
882	AssertMsg((fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)),
883	("Page flags mismatch! pvFault=%RGv uErr=%x GCPhys=%RGp fPageShw=%RX64 fPageGst=%RX64\n", pvFault, (uint32_t)uErr, GCPhys, fPageShw, fPageGst));
884	}
885	else
886	AssertMsgFailed(("PGMGstGetPage rc=%Rrc\n", rc));
887	}
888	else
889	AssertMsgFailed(("PGMGCGetPage rc=%Rrc\n", rc));
890	# endif /* VBOX_STRICT */
891	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
892	}
893	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
894	# endif /* PGM_OUT_OF_SYNC_IN_GC */
895	}
896	else /* GCPhys == NIL_RTGCPHYS */
897	{
898	/*
899	* Page not present in Guest OS or invalid page table address.
900	* This is potential virtual page access handler food.
901	*
902	* For the present we'll say that our access handlers don't
903	* work for this case - we've already discarded the page table
904	* not present case which is identical to this.
905	*
906	* When we perchance find we need this, we will probably have AVL
907	* trees (offset based) to operate on and we can measure their speed
908	* agains mapping a page table and probably rearrange this handling
909	* a bit. (Like, searching virtual ranges before checking the
910	* physical address.)
911	*/
912	}
913	}
914	/* else: !present (guest) */
915
916
917	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
918	/*
919	* Conclusion, this is a guest trap.
920	*/
921	LogFlow(("PGM: Unhandled #PF -> route trap to recompiler!\n"));
922	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eGuestPFUnh);
923	return VINF_EM_RAW_GUEST_TRAP;
924	# else
925	/* present, but not a monitored page; perhaps the guest is probing physical memory */
926	return VINF_EM_RAW_EMULATE_INSTR;
927	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
928
929
930	# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
931
932	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
933	return VERR_INTERNAL_ERROR;
934	# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
935	}
936	#endif /* !IN_RING3 */
937
938
939	/**
940	* Emulation of the invlpg instruction.
941	*
942	*
943	* @returns VBox status code.
944	*
945	* @param pVCpu The VMCPU handle.
946	* @param GCPtrPage Page to invalidate.
947	*
948	* @remark ASSUMES that the guest is updating before invalidating. This order
949	* isn't required by the CPU, so this is speculative and could cause
950	* trouble.
951	* @remark No TLB shootdown is done on any other VCPU as we assume that
952	* invlpg emulation is the only reason for calling this function.
953	* (The guest has to shoot down TLB entries on other CPUs itself)
954	* Currently true, but keep in mind!
955	*
956	* @todo Clean this up! Most of it is (or should be) no longer necessary as we catch all page table accesses.
957	*/
958	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
959	{
960	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
961	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
962	&& PGM_SHW_TYPE != PGM_TYPE_EPT
963	int rc;
964	PVM pVM = pVCpu->CTX_SUFF(pVM);
965	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
966
967	Assert(PGMIsLockOwner(pVM));
968
969	LogFlow(("InvalidatePage %RGv\n", GCPtrPage));
970
971	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
972	if (pPool->cDirtyPages)
973	pgmPoolResetDirtyPages(pVM);
974	# endif
975
976	/*
977	* Get the shadow PD entry and skip out if this PD isn't present.
978	* (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
979	*/
980	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
981	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
982	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
983
984	/* Fetch the pgm pool shadow descriptor. */
985	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
986	Assert(pShwPde);
987
988	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
989	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT);
990	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(&pVCpu->pgm.s);
991
992	/* If the shadow PDPE isn't present, then skip the invalidate. */
993	if (!pPdptDst->a[iPdpt].n.u1Present)
994	{
995	Assert(!(pPdptDst->a[iPdpt].u & PGM_PLXFLAGS_MAPPING));
996	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
997	return VINF_SUCCESS;
998	}
999
1000	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1001	PPGMPOOLPAGE pShwPde = NULL;
1002	PX86PDPAE pPDDst;
1003
1004	/* Fetch the pgm pool shadow descriptor. */
1005	rc = pgmShwGetPaePoolPagePD(&pVCpu->pgm.s, GCPtrPage, &pShwPde);
1006	AssertRCSuccessReturn(rc, rc);
1007	Assert(pShwPde);
1008
1009	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGM(&pVM->pgm.s, pShwPde);
1010	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1011
1012	# else /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1013	/* PML4 */
1014	const unsigned iPml4 = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;
1015	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1016	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1017	PX86PDPAE pPDDst;
1018	PX86PDPT pPdptDst;
1019	PX86PML4E pPml4eDst;
1020	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eDst, &pPdptDst, &pPDDst);
1021	if (rc != VINF_SUCCESS)
1022	{
1023	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT \|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
1024	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
1025	if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
1026	PGM_INVL_VCPU_TLBS(pVCpu);
1027	return VINF_SUCCESS;
1028	}
1029	Assert(pPDDst);
1030
1031	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1032	PX86PDPE pPdpeDst = &pPdptDst->a[iPdpt];
1033
1034	if (!pPdpeDst->n.u1Present)
1035	{
1036	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
1037	if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
1038	PGM_INVL_VCPU_TLBS(pVCpu);
1039	return VINF_SUCCESS;
1040	}
1041
1042	# endif /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1043
1044	const SHWPDE PdeDst = *pPdeDst;
1045	if (!PdeDst.n.u1Present)
1046	{
1047	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
1048	return VINF_SUCCESS;
1049	}
1050
1051	# if defined(IN_RC)
1052	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1053	PGMDynLockHCPage(pVM, (uint8_t *)pPdeDst);
1054	# endif
1055
1056	/*
1057	* Get the guest PD entry and calc big page.
1058	*/
1059	# if PGM_GST_TYPE == PGM_TYPE_32BIT
1060	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
1061	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
1062	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
1063	# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1064	unsigned iPDSrc = 0;
1065	# if PGM_GST_TYPE == PGM_TYPE_PAE
1066	X86PDPE PdpeSrc;
1067	PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, GCPtrPage, &iPDSrc, &PdpeSrc);
1068	# else /* AMD64 */
1069	PX86PML4E pPml4eSrc;
1070	X86PDPE PdpeSrc;
1071	PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
1072	# endif
1073	GSTPDE PdeSrc;
1074
1075	if (pPDSrc)
1076	PdeSrc = pPDSrc->a[iPDSrc];
1077	else
1078	PdeSrc.u = 0;
1079	# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1080
1081	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1082	const bool fIsBigPage = PdeSrc.b.u1Size;
1083	# else
1084	const bool fIsBigPage = PdeSrc.b.u1Size && (CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
1085	# endif
1086
1087	# ifdef IN_RING3
1088	/*
1089	* If a CR3 Sync is pending we may ignore the invalidate page operation
1090	* depending on the kind of sync and if it's a global page or not.
1091	* This doesn't make sense in GC/R0 so we'll skip it entirely there.
1092	*/
1093	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
1094	if ( VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
1095	\|\| ( VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
1096	&& fIsBigPage
1097	&& PdeSrc.b.u1Global
1098	)
1099	)
1100	# else
1101	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
1102	# endif
1103	{
1104	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
1105	return VINF_SUCCESS;
1106	}
1107	# endif /* IN_RING3 */
1108
1109	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1110	/* Fetch the pgm pool shadow descriptor. */
1111	PPGMPOOLPAGE pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
1112	Assert(pShwPdpt);
1113
1114	/* Fetch the pgm pool shadow descriptor. */
1115	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & SHW_PDPE_PG_MASK);
1116	Assert(pShwPde);
1117
1118	Assert(pPml4eDst->n.u1Present && (pPml4eDst->u & SHW_PDPT_MASK));
1119	RTGCPHYS GCPhysPdpt = pPml4eSrc->u & X86_PML4E_PG_MASK;
1120
1121	if ( !pPml4eSrc->n.u1Present
1122	\|\| pShwPdpt->GCPhys != GCPhysPdpt)
1123	{
1124	LogFlow(("InvalidatePage: Out-of-sync PML4E (P/GCPhys) at %RGv GCPhys=%RGp vs %RGp Pml4eSrc=%RX64 Pml4eDst=%RX64\n",
1125	GCPtrPage, pShwPdpt->GCPhys, GCPhysPdpt, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
1126	pgmPoolFreeByPage(pPool, pShwPdpt, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4);
1127	ASMAtomicWriteSize(pPml4eDst, 0);
1128	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNPs));
1129	PGM_INVL_VCPU_TLBS(pVCpu);
1130	return VINF_SUCCESS;
1131	}
1132	if ( pPml4eSrc->n.u1User != pPml4eDst->n.u1User
1133	\|\| (!pPml4eSrc->n.u1Write && pPml4eDst->n.u1Write))
1134	{
1135	/*
1136	* Mark not present so we can resync the PML4E when it's used.
1137	*/
1138	LogFlow(("InvalidatePage: Out-of-sync PML4E at %RGv Pml4eSrc=%RX64 Pml4eDst=%RX64\n",
1139	GCPtrPage, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
1140	pgmPoolFreeByPage(pPool, pShwPdpt, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4);
1141	ASMAtomicWriteSize(pPml4eDst, 0);
1142	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1143	PGM_INVL_VCPU_TLBS(pVCpu);
1144	}
1145	else if (!pPml4eSrc->n.u1Accessed)
1146	{
1147	/*
1148	* Mark not present so we can set the accessed bit.
1149	*/
1150	LogFlow(("InvalidatePage: Out-of-sync PML4E (A) at %RGv Pml4eSrc=%RX64 Pml4eDst=%RX64\n",
1151	GCPtrPage, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
1152	pgmPoolFreeByPage(pPool, pShwPdpt, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4);
1153	ASMAtomicWriteSize(pPml4eDst, 0);
1154	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNAs));
1155	PGM_INVL_VCPU_TLBS(pVCpu);
1156	}
1157
1158	/* Check if the PDPT entry has changed. */
1159	Assert(pPdpeDst->n.u1Present && pPdpeDst->u & SHW_PDPT_MASK);
1160	RTGCPHYS GCPhysPd = PdpeSrc.u & GST_PDPE_PG_MASK;
1161	if ( !PdpeSrc.n.u1Present
1162	\|\| pShwPde->GCPhys != GCPhysPd)
1163	{
1164	LogFlow(("InvalidatePage: Out-of-sync PDPE (P/GCPhys) at %RGv GCPhys=%RGp vs %RGp PdpeSrc=%RX64 PdpeDst=%RX64\n",
1165	GCPtrPage, pShwPde->GCPhys, GCPhysPd, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
1166	pgmPoolFreeByPage(pPool, pShwPde, pShwPdpt->idx, iPdpt);
1167	ASMAtomicWriteSize(pPdpeDst, 0);
1168	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNPs));
1169	PGM_INVL_VCPU_TLBS(pVCpu);
1170	return VINF_SUCCESS;
1171	}
1172	if ( PdpeSrc.lm.u1User != pPdpeDst->lm.u1User
1173	\|\| (!PdpeSrc.lm.u1Write && pPdpeDst->lm.u1Write))
1174	{
1175	/*
1176	* Mark not present so we can resync the PDPTE when it's used.
1177	*/
1178	LogFlow(("InvalidatePage: Out-of-sync PDPE at %RGv PdpeSrc=%RX64 PdpeDst=%RX64\n",
1179	GCPtrPage, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
1180	pgmPoolFreeByPage(pPool, pShwPde, pShwPdpt->idx, iPdpt);
1181	ASMAtomicWriteSize(pPdpeDst, 0);
1182	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1183	PGM_INVL_VCPU_TLBS(pVCpu);
1184	}
1185	else if (!PdpeSrc.lm.u1Accessed)
1186	{
1187	/*
1188	* Mark not present so we can set the accessed bit.
1189	*/
1190	LogFlow(("InvalidatePage: Out-of-sync PDPE (A) at %RGv PdpeSrc=%RX64 PdpeDst=%RX64\n",
1191	GCPtrPage, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
1192	pgmPoolFreeByPage(pPool, pShwPde, pShwPdpt->idx, iPdpt);
1193	ASMAtomicWriteSize(pPdpeDst, 0);
1194	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNAs));
1195	PGM_INVL_VCPU_TLBS(pVCpu);
1196	}
1197	# endif /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
1198
1199	/*
1200	* Deal with the Guest PDE.
1201	*/
1202	rc = VINF_SUCCESS;
1203	if (PdeSrc.n.u1Present)
1204	{
1205	# ifndef PGM_WITHOUT_MAPPING
1206	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
1207	{
1208	/*
1209	* Conflict - Let SyncPT deal with it to avoid duplicate code.
1210	*/
1211	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1212	Assert(PGMGetGuestMode(pVCpu) <= PGMMODE_PAE);
1213	pgmLock(pVM);
1214	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
1215	pgmUnlock(pVM);
1216	}
1217	else
1218	# endif /* !PGM_WITHOUT_MAPPING */
1219	if ( PdeSrc.n.u1User != PdeDst.n.u1User
1220	\|\| (!PdeSrc.n.u1Write && PdeDst.n.u1Write))
1221	{
1222	/*
1223	* Mark not present so we can resync the PDE when it's used.
1224	*/
1225	LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1226	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1227	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1228	ASMAtomicWriteSize(pPdeDst, 0);
1229	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1230	PGM_INVL_VCPU_TLBS(pVCpu);
1231	}
1232	else if (!PdeSrc.n.u1Accessed)
1233	{
1234	/*
1235	* Mark not present so we can set the accessed bit.
1236	*/
1237	LogFlow(("InvalidatePage: Out-of-sync (A) at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1238	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1239	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1240	ASMAtomicWriteSize(pPdeDst, 0);
1241	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNAs));
1242	PGM_INVL_VCPU_TLBS(pVCpu);
1243	}
1244	else if (!fIsBigPage)
1245	{
1246	/*
1247	* 4KB - page.
1248	*/
1249	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1250	RTGCPHYS GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1251
1252	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1253	/* Reset the modification counter (OpenSolaris trashes tlb entries very often) */
1254	if (pShwPage->cModifications)
1255	pShwPage->cModifications = 1;
1256	# endif
1257
1258	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1259	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1260	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE/2);
1261	# endif
1262	if (pShwPage->GCPhys == GCPhys)
1263	{
1264	# if 0 /* likely cause of a major performance regression; must be SyncPageWorkerTrackDeref then */
1265	const unsigned iPTEDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1266	PSHWPT pPT = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1267	if (pPT->a[iPTEDst].n.u1Present)
1268	{
1269	/* This is very unlikely with caching/monitoring enabled. */
1270	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pShwPage, pPT->a[iPTEDst].u & SHW_PTE_PG_MASK);
1271	ASMAtomicWriteSize(&pPT->a[iPTEDst], 0);
1272	}
1273	# else /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
1274	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
1275	if (RT_SUCCESS(rc))
1276	rc = VINF_SUCCESS;
1277	# endif
1278	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePage4KBPages));
1279	PGM_INVL_PG(pVCpu, GCPtrPage);
1280	}
1281	else
1282	{
1283	/*
1284	* The page table address changed.
1285	*/
1286	LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n",
1287	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
1288	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1289	ASMAtomicWriteSize(pPdeDst, 0);
1290	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1291	PGM_INVL_VCPU_TLBS(pVCpu);
1292	}
1293	}
1294	else
1295	{
1296	/*
1297	* 2/4MB - page.
1298	*/
1299	/* Before freeing the page, check if anything really changed. */
1300	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1301	RTGCPHYS GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
1302	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1303	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1304	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1305	# endif
1306	if ( pShwPage->GCPhys == GCPhys
1307	&& pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1308	{
1309	/* ASSUMES a the given bits are identical for 4M and normal PDEs */
1310	/** @todo PAT */
1311	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
1312	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
1313	&& ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1314	\|\| (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1315	{
1316	LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1317	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePage4MBPagesSkip));
1318	# if defined(IN_RC)
1319	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1320	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
1321	# endif
1322	return VINF_SUCCESS;
1323	}
1324	}
1325
1326	/*
1327	* Ok, the page table is present and it's been changed in the guest.
1328	* If we're in host context, we'll just mark it as not present taking the lazy approach.
1329	* We could do this for some flushes in GC too, but we need an algorithm for
1330	* deciding which 4MB pages containing code likely to be executed very soon.
1331	*/
1332	LogFlow(("InvalidatePage: Out-of-sync PD at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1333	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1334	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1335	ASMAtomicWriteSize(pPdeDst, 0);
1336	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePage4MBPages));
1337	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
1338	}
1339	}
1340	else
1341	{
1342	/*
1343	* Page directory is not present, mark shadow PDE not present.
1344	*/
1345	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
1346	{
1347	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1348	ASMAtomicWriteSize(pPdeDst, 0);
1349	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNPs));
1350	PGM_INVL_PG(pVCpu, GCPtrPage);
1351	}
1352	else
1353	{
1354	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1355	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDMappings));
1356	}
1357	}
1358	# if defined(IN_RC)
1359	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1360	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
1361	# endif
1362	return rc;
1363
1364	#else /* guest real and protected mode */
1365	/* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1366	return VINF_SUCCESS;
1367	#endif
1368	}
1369
1370
1371	/**
1372	* Update the tracking of shadowed pages.
1373	*
1374	* @param pVCpu The VMCPU handle.
1375	* @param pShwPage The shadow page.
1376	* @param HCPhys The physical page we is being dereferenced.
1377	*/
1378	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys)
1379	{
1380	PVM pVM = pVCpu->CTX_SUFF(pVM);
1381
1382	STAM_PROFILE_START(&pVM->pgm.s.StatTrackDeref, a);
1383	LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1384
1385	/** @todo If this turns out to be a bottle neck (very likely) two things can be done:
1386	* 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1387	* 2. write protect all shadowed pages. I.e. implement caching.
1388	*/
1389	/*
1390	* Find the guest address.
1391	*/
1392	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
1393	pRam;
1394	pRam = pRam->CTX_SUFF(pNext))
1395	{
1396	unsigned iPage = pRam->cb >> PAGE_SHIFT;
1397	while (iPage-- > 0)
1398	{
1399	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1400	{
1401	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1402	pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage]);
1403	pShwPage->cPresent--;
1404	pPool->cPresent--;
1405	STAM_PROFILE_STOP(&pVM->pgm.s.StatTrackDeref, a);
1406	return;
1407	}
1408	}
1409	}
1410
1411	for (;;)
1412	AssertReleaseMsgFailed(("HCPhys=%RHp wasn't found!\n", HCPhys));
1413	}
1414
1415
1416	/**
1417	* Update the tracking of shadowed pages.
1418	*
1419	* @param pVCpu The VMCPU handle.
1420	* @param pShwPage The shadow page.
1421	* @param u16 The top 16-bit of the pPage->HCPhys.
1422	* @param pPage Pointer to the guest page. this will be modified.
1423	* @param iPTDst The index into the shadow table.
1424	*/
1425	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, uint16_t u16, PPGMPAGE pPage, const unsigned iPTDst)
1426	{
1427	PVM pVM = pVCpu->CTX_SUFF(pVM);
1428	/*
1429	* Just deal with the simple first time here.
1430	*/
1431	if (!u16)
1432	{
1433	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackVirgin);
1434	u16 = PGMPOOL_TD_MAKE(1, pShwPage->idx);
1435	}
1436	else
1437	u16 = pgmPoolTrackPhysExtAddref(pVM, u16, pShwPage->idx);
1438
1439	/* write back */
1440	Log2(("SyncPageWorkerTrackAddRef: u16=%#x->%#x iPTDst=%#x\n", u16, PGM_PAGE_GET_TRACKING(pPage), iPTDst));
1441	PGM_PAGE_SET_TRACKING(pPage, u16);
1442
1443	/* update statistics. */
1444	pVM->pgm.s.CTX_SUFF(pPool)->cPresent++;
1445	pShwPage->cPresent++;
1446	if (pShwPage->iFirstPresent > iPTDst)
1447	pShwPage->iFirstPresent = iPTDst;
1448	}
1449
1450
1451	/**
1452	* Creates a 4K shadow page for a guest page.
1453	*
1454	* For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1455	* physical address. The PdeSrc argument only the flags are used. No page structured
1456	* will be mapped in this function.
1457	*
1458	* @param pVCpu The VMCPU handle.
1459	* @param pPteDst Destination page table entry.
1460	* @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1461	* Can safely assume that only the flags are being used.
1462	* @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1463	* @param pShwPage Pointer to the shadow page.
1464	* @param iPTDst The index into the shadow table.
1465	*
1466	* @remark Not used for 2/4MB pages!
1467	*/
1468	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1469	{
1470	if (PteSrc.n.u1Present)
1471	{
1472	PVM pVM = pVCpu->CTX_SUFF(pVM);
1473
1474	# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1475	&& PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1476	&& (PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64)
1477	if (pShwPage->fDirty)
1478	{
1479	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1480	PX86PTPAE pGstPT;
1481
1482	pGstPT = (PX86PTPAE)&pPool->aDirtyPages[pShwPage->idxDirty][0];
1483	pGstPT->a[iPTDst].u = PteSrc.u;
1484	}
1485	# endif
1486	/*
1487	* Find the ram range.
1488	*/
1489	PPGMPAGE pPage;
1490	int rc = pgmPhysGetPageEx(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK, &pPage);
1491	if (RT_SUCCESS(rc))
1492	{
1493	#ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1494	/* Try make the page writable if necessary. */
1495	if ( PteSrc.n.u1Write
1496	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1497	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1498	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1499	# endif
1500	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
1501	{
1502	rc = pgmPhysPageMakeWritableUnlocked(pVM, pPage, PteSrc.u & GST_PTE_PG_MASK);
1503	AssertRC(rc);
1504	}
1505	#endif
1506
1507	/** @todo investiage PWT, PCD and PAT. */
1508	/*
1509	* Make page table entry.
1510	*/
1511	SHWPTE PteDst;
1512	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1513	{
1514	/** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No. */
1515	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1516	{
1517	#if PGM_SHW_TYPE == PGM_TYPE_EPT
1518	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage);
1519	PteDst.n.u1Present = 1;
1520	PteDst.n.u1Execute = 1;
1521	PteDst.n.u1IgnorePAT = 1;
1522	PteDst.n.u3EMT = VMX_EPT_MEMTYPE_WB;
1523	/* PteDst.n.u1Write = 0 && PteDst.n.u1Size = 0 */
1524	#else
1525	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1526	\| PGM_PAGE_GET_HCPHYS(pPage);
1527	#endif
1528	}
1529	else
1530	{
1531	LogFlow(("SyncPageWorker: monitored page (%RHp) -> mark not present\n", PGM_PAGE_GET_HCPHYS(pPage)));
1532	PteDst.u = 0;
1533	}
1534	/** @todo count these two kinds. */
1535	}
1536	else
1537	{
1538	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1539	/*
1540	* If the page or page directory entry is not marked accessed,
1541	* we mark the page not present.
1542	*/
1543	if (!PteSrc.n.u1Accessed \|\| !PdeSrc.n.u1Accessed)
1544	{
1545	LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1546	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,AccessedPage));
1547	PteDst.u = 0;
1548	}
1549	else
1550	/*
1551	* If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1552	* when the page is modified.
1553	*/
1554	if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
1555	{
1556	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPage));
1557	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1558	\| PGM_PAGE_GET_HCPHYS(pPage)
1559	\| PGM_PTFLAGS_TRACK_DIRTY;
1560	}
1561	else
1562	#endif
1563	{
1564	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageSkipped));
1565	#if PGM_SHW_TYPE == PGM_TYPE_EPT
1566	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage);
1567	PteDst.n.u1Present = 1;
1568	PteDst.n.u1Write = 1;
1569	PteDst.n.u1Execute = 1;
1570	PteDst.n.u1IgnorePAT = 1;
1571	PteDst.n.u3EMT = VMX_EPT_MEMTYPE_WB;
1572	/* PteDst.n.u1Size = 0 */
1573	#else
1574	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1575	\| PGM_PAGE_GET_HCPHYS(pPage);
1576	#endif
1577	}
1578	}
1579
1580	/*
1581	* Make sure only allocated pages are mapped writable.
1582	*/
1583	if ( PteDst.n.u1Write
1584	&& PteDst.n.u1Present
1585	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1586	{
1587	PteDst.n.u1Write = 0; /** @todo this isn't quite working yet. */
1588	Log3(("SyncPageWorker: write-protecting %RGp pPage=%R[pgmpage]at iPTDst=%d\n", (RTGCPHYS)(PteSrc.u & X86_PTE_PAE_PG_MASK), pPage, iPTDst));
1589	}
1590
1591	/*
1592	* Keep user track up to date.
1593	*/
1594	if (PteDst.n.u1Present)
1595	{
1596	if (!pPteDst->n.u1Present)
1597	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1598	else if ((pPteDst->u & SHW_PTE_PG_MASK) != (PteDst.u & SHW_PTE_PG_MASK))
1599	{
1600	Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", (uint64_t)pPteDst->u, (uint64_t)PteDst.u));
1601	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1602	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1603	}
1604	}
1605	else if (pPteDst->n.u1Present)
1606	{
1607	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1608	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1609	}
1610
1611	/*
1612	* Update statistics and commit the entry.
1613	*/
1614	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1615	if (!PteSrc.n.u1Global)
1616	pShwPage->fSeenNonGlobal = true;
1617	#endif
1618	ASMAtomicWriteSize(pPteDst, PteDst.u);
1619	}
1620	/* else MMIO or invalid page, we must handle them manually in the #PF handler. */
1621	/** @todo count these. */
1622	}
1623	else
1624	{
1625	/*
1626	* Page not-present.
1627	*/
1628	Log2(("SyncPageWorker: page not present in Pte\n"));
1629	/* Keep user track up to date. */
1630	if (pPteDst->n.u1Present)
1631	{
1632	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1633	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1634	}
1635	ASMAtomicWriteSize(pPteDst, 0);
1636	/** @todo count these. */
1637	}
1638	}
1639
1640
1641	/**
1642	* Syncs a guest OS page.
1643	*
1644	* There are no conflicts at this point, neither is there any need for
1645	* page table allocations.
1646	*
1647	* @returns VBox status code.
1648	* @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1649	* @param pVCpu The VMCPU handle.
1650	* @param PdeSrc Page directory entry of the guest.
1651	* @param GCPtrPage Guest context page address.
1652	* @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1653	* @param uErr Fault error (X86_TRAP_PF_*).
1654	*/
1655	PGM_BTH_DECL(int, SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr)
1656	{
1657	PVM pVM = pVCpu->CTX_SUFF(pVM);
1658	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1659	LogFlow(("SyncPage: GCPtrPage=%RGv cPages=%u uErr=%#x\n", GCPtrPage, cPages, uErr));
1660
1661	Assert(PGMIsLockOwner(pVM));
1662
1663	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
1664	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
1665	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
1666	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
1667	&& PGM_SHW_TYPE != PGM_TYPE_EPT
1668
1669	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
1670	bool fNoExecuteBitValid = !!(CPUMGetGuestEFER(pVCpu) & MSR_K6_EFER_NXE);
1671	# endif
1672
1673	/*
1674	* Assert preconditions.
1675	*/
1676	Assert(PdeSrc.n.u1Present);
1677	Assert(cPages);
1678	# if 0 /* rarely useful; leave for debugging. */
1679	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
1680	# endif
1681
1682	/*
1683	* Get the shadow PDE, find the shadow page table in the pool.
1684	*/
1685	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1686	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1687	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
1688
1689	/* Fetch the pgm pool shadow descriptor. */
1690	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1691	Assert(pShwPde);
1692
1693	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1694	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1695	PPGMPOOLPAGE pShwPde = NULL;
1696	PX86PDPAE pPDDst;
1697
1698	/* Fetch the pgm pool shadow descriptor. */
1699	int rc2 = pgmShwGetPaePoolPagePD(&pVCpu->pgm.s, GCPtrPage, &pShwPde);
1700	AssertRCSuccessReturn(rc2, rc2);
1701	Assert(pShwPde);
1702
1703	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGM(&pVM->pgm.s, pShwPde);
1704	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1705
1706	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1707	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1708	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1709	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
1710	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
1711
1712	int rc2 = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
1713	AssertRCSuccessReturn(rc2, rc2);
1714	Assert(pPDDst && pPdptDst);
1715	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1716	# endif
1717	SHWPDE PdeDst = *pPdeDst;
1718	if (!PdeDst.n.u1Present)
1719	{
1720	AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE p=%p/%RX64\n", pPdeDst, (uint64_t)PdeDst.u));
1721	Log(("CPU%d: SyncPage: Pde at %RGv changed behind our back!\n", GCPtrPage));
1722	return VINF_SUCCESS; /* force the instruction to be executed again. */
1723	}
1724
1725	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1726	Assert(pShwPage);
1727
1728	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1729	/* Fetch the pgm pool shadow descriptor. */
1730	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
1731	Assert(pShwPde);
1732	# endif
1733
1734	# if defined(IN_RC)
1735	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1736	PGMDynLockHCPage(pVM, (uint8_t *)pPdeDst);
1737	# endif
1738
1739	/*
1740	* Check that the page is present and that the shadow PDE isn't out of sync.
1741	*/
1742	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1743	const bool fBigPage = PdeSrc.b.u1Size;
1744	# else
1745	const bool fBigPage = PdeSrc.b.u1Size && (CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
1746	# endif
1747	RTGCPHYS GCPhys;
1748	if (!fBigPage)
1749	{
1750	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1751	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1752	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1753	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE / 2);
1754	# endif
1755	}
1756	else
1757	{
1758	GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
1759	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1760	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1761	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1762	# endif
1763	}
1764	if ( pShwPage->GCPhys == GCPhys
1765	&& PdeSrc.n.u1Present
1766	&& (PdeSrc.n.u1User == PdeDst.n.u1User)
1767	&& (PdeSrc.n.u1Write == PdeDst.n.u1Write \|\| !PdeDst.n.u1Write)
1768	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
1769	&& (!fNoExecuteBitValid \|\| PdeSrc.n.u1NoExecute == PdeDst.n.u1NoExecute)
1770	# endif
1771	)
1772	{
1773	/*
1774	* Check that the PDE is marked accessed already.
1775	* Since we set the accessed bit before getting here on a #PF, this
1776	* check is only meant for dealing with non-#PF'ing paths.
1777	*/
1778	if (PdeSrc.n.u1Accessed)
1779	{
1780	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1781	if (!fBigPage)
1782	{
1783	/*
1784	* 4KB Page - Map the guest page table.
1785	*/
1786	PGSTPT pPTSrc;
1787	int rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
1788	if (RT_SUCCESS(rc))
1789	{
1790	# ifdef PGM_SYNC_N_PAGES
1791	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
1792	if ( cPages > 1
1793	&& !(uErr & X86_TRAP_PF_P)
1794	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
1795	{
1796	/*
1797	* This code path is currently only taken when the caller is PGMTrap0eHandler
1798	* for non-present pages!
1799	*
1800	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1801	* deal with locality.
1802	*/
1803	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1804	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1805	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1806	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1807	# else
1808	const unsigned offPTSrc = 0;
1809	# endif
1810	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
1811	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1812	iPTDst = 0;
1813	else
1814	iPTDst -= PGM_SYNC_NR_PAGES / 2;
1815	for (; iPTDst < iPTDstEnd; iPTDst++)
1816	{
1817	if (!pPTDst->a[iPTDst].n.u1Present)
1818	{
1819	GSTPTE PteSrc = pPTSrc->a[offPTSrc + iPTDst];
1820	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(GST_PT_MASK << GST_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
1821	NOREF(GCPtrCurPage);
1822	#ifndef IN_RING0
1823	/*
1824	* Assuming kernel code will be marked as supervisor - and not as user level
1825	* and executed using a conforming code selector - And marked as readonly.
1826	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
1827	*/
1828	PPGMPAGE pPage;
1829	if ( ((PdeSrc.u & PteSrc.u) & (X86_PTE_RW \| X86_PTE_US))
1830	\|\| iPTDst == ((GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK) /* always sync GCPtrPage */
1831	\|\| !CSAMDoesPageNeedScanning(pVM, (RTRCPTR)GCPtrCurPage)
1832	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
1833	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1834	)
1835	#endif /* else: CSAM not active */
1836	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1837	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1838	GCPtrCurPage, PteSrc.n.u1Present,
1839	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1840	PteSrc.n.u1User & PdeSrc.n.u1User,
1841	(uint64_t)PteSrc.u,
1842	(uint64_t)pPTDst->a[iPTDst].u,
1843	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1844	}
1845	}
1846	}
1847	else
1848	# endif /* PGM_SYNC_N_PAGES */
1849	{
1850	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1851	GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1852	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1853	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1854	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
1855	GCPtrPage, PteSrc.n.u1Present,
1856	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1857	PteSrc.n.u1User & PdeSrc.n.u1User,
1858	(uint64_t)PteSrc.u,
1859	(uint64_t)pPTDst->a[iPTDst].u,
1860	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1861	}
1862	}
1863	else /* MMIO or invalid page: emulated in #PF handler. */
1864	{
1865	LogFlow(("PGM_GCPHYS_2_PTR %RGp failed with %Rrc\n", GCPhys, rc));
1866	Assert(!pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK].n.u1Present);
1867	}
1868	}
1869	else
1870	{
1871	/*
1872	* 4/2MB page - lazy syncing shadow 4K pages.
1873	* (There are many causes of getting here, it's no longer only CSAM.)
1874	*/
1875	/* Calculate the GC physical address of this 4KB shadow page. */
1876	GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc) \| (GCPtrPage & GST_BIG_PAGE_OFFSET_MASK);
1877	/* Find ram range. */
1878	PPGMPAGE pPage;
1879	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
1880	if (RT_SUCCESS(rc))
1881	{
1882	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1883	/* Try make the page writable if necessary. */
1884	if ( PdeSrc.n.u1Write
1885	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1886	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1887	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1888	# endif
1889	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
1890	{
1891	rc = pgmPhysPageMakeWritableUnlocked(pVM, pPage, GCPhys);
1892	AssertRC(rc);
1893	}
1894	# endif
1895
1896	/*
1897	* Make shadow PTE entry.
1898	*/
1899	SHWPTE PteDst;
1900	PteDst.u = (PdeSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1901	\| PGM_PAGE_GET_HCPHYS(pPage);
1902	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1903	{
1904	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1905	PteDst.n.u1Write = 0;
1906	else
1907	PteDst.u = 0;
1908	}
1909	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1910	if (PteDst.n.u1Present && !pPTDst->a[iPTDst].n.u1Present)
1911	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1912
1913	/* Make sure only allocated pages are mapped writable. */
1914	if ( PteDst.n.u1Write
1915	&& PteDst.n.u1Present
1916	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1917	{
1918	PteDst.n.u1Write = 0; /** @todo this isn't quite working yet... */
1919	Log3(("SyncPage: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, GCPtrPage));
1920	}
1921
1922	ASMAtomicWriteSize(&pPTDst->a[iPTDst], PteDst.u);
1923
1924	/*
1925	* If the page is not flagged as dirty and is writable, then make it read-only
1926	* at PD level, so we can set the dirty bit when the page is modified.
1927	*
1928	* ASSUMES that page access handlers are implemented on page table entry level.
1929	* Thus we will first catch the dirty access and set PDE.D and restart. If
1930	* there is an access handler, we'll trap again and let it work on the problem.
1931	*/
1932	/** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
1933	* As for invlpg, it simply frees the whole shadow PT.
1934	* ...It's possibly because the guest clears it and the guest doesn't really tell us... */
1935	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
1936	{
1937	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageBig));
1938	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
1939	PdeDst.n.u1Write = 0;
1940	}
1941	else
1942	{
1943	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
1944	PdeDst.n.u1Write = PdeSrc.n.u1Write;
1945	}
1946	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
1947	Log2(("SyncPage: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%RGp%s\n",
1948	GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
1949	PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1950	}
1951	else
1952	LogFlow(("PGM_GCPHYS_2_PTR %RGp (big) failed with %Rrc\n", GCPhys, rc));
1953	}
1954	# if defined(IN_RC)
1955	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1956	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
1957	# endif
1958	return VINF_SUCCESS;
1959	}
1960	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDNAs));
1961	}
1962	else
1963	{
1964	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDOutOfSync));
1965	Log2(("SyncPage: Out-Of-Sync PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
1966	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
1967	}
1968
1969	/*
1970	* Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
1971	* Yea, I'm lazy.
1972	*/
1973	pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst);
1974	ASMAtomicWriteSize(pPdeDst, 0);
1975
1976	# if defined(IN_RC)
1977	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1978	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
1979	# endif
1980	PGM_INVL_VCPU_TLBS(pVCpu);
1981	return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
1982
1983	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
1984	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
1985	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
1986	&& !defined(IN_RC)
1987
1988	# ifdef PGM_SYNC_N_PAGES
1989	/*
1990	* Get the shadow PDE, find the shadow page table in the pool.
1991	*/
1992	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1993	X86PDE PdeDst = pgmShwGet32BitPDE(&pVCpu->pgm.s, GCPtrPage);
1994
1995	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1996	X86PDEPAE PdeDst = pgmShwGetPaePDE(&pVCpu->pgm.s, GCPtrPage);
1997
1998	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1999	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2000	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; NOREF(iPdpt);
2001	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2002	X86PDEPAE PdeDst;
2003	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2004
2005	int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2006	AssertRCSuccessReturn(rc, rc);
2007	Assert(pPDDst && pPdptDst);
2008	PdeDst = pPDDst->a[iPDDst];
2009	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2010	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2011	PEPTPD pPDDst;
2012	EPTPDE PdeDst;
2013
2014	int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, NULL, &pPDDst);
2015	if (rc != VINF_SUCCESS)
2016	{
2017	AssertRC(rc);
2018	return rc;
2019	}
2020	Assert(pPDDst);
2021	PdeDst = pPDDst->a[iPDDst];
2022	# endif
2023	AssertMsg(PdeDst.n.u1Present, ("%#llx\n", (uint64_t)PdeDst.u));
2024	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2025	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2026
2027	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
2028	if ( cPages > 1
2029	&& !(uErr & X86_TRAP_PF_P)
2030	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2031	{
2032	/*
2033	* This code path is currently only taken when the caller is PGMTrap0eHandler
2034	* for non-present pages!
2035	*
2036	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2037	* deal with locality.
2038	*/
2039	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2040	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2041	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2042	iPTDst = 0;
2043	else
2044	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2045	for (; iPTDst < iPTDstEnd; iPTDst++)
2046	{
2047	if (!pPTDst->a[iPTDst].n.u1Present)
2048	{
2049	GSTPTE PteSrc;
2050
2051	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT);
2052
2053	/* Fake the page table entry */
2054	PteSrc.u = GCPtrCurPage;
2055	PteSrc.n.u1Present = 1;
2056	PteSrc.n.u1Dirty = 1;
2057	PteSrc.n.u1Accessed = 1;
2058	PteSrc.n.u1Write = 1;
2059	PteSrc.n.u1User = 1;
2060
2061	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2062
2063	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
2064	GCPtrCurPage, PteSrc.n.u1Present,
2065	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2066	PteSrc.n.u1User & PdeSrc.n.u1User,
2067	(uint64_t)PteSrc.u,
2068	(uint64_t)pPTDst->a[iPTDst].u,
2069	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2070
2071	if (RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)))
2072	break;
2073	}
2074	else
2075	Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n", (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT), iPTDst, pPTDst->a[iPTDst].u));
2076	}
2077	}
2078	else
2079	# endif /* PGM_SYNC_N_PAGES */
2080	{
2081	GSTPTE PteSrc;
2082	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2083	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT);
2084
2085	/* Fake the page table entry */
2086	PteSrc.u = GCPtrCurPage;
2087	PteSrc.n.u1Present = 1;
2088	PteSrc.n.u1Dirty = 1;
2089	PteSrc.n.u1Accessed = 1;
2090	PteSrc.n.u1Write = 1;
2091	PteSrc.n.u1User = 1;
2092	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2093
2094	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}PteDst=%08llx%s\n",
2095	GCPtrPage, PteSrc.n.u1Present,
2096	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2097	PteSrc.n.u1User & PdeSrc.n.u1User,
2098	(uint64_t)PteSrc.u,
2099	(uint64_t)pPTDst->a[iPTDst].u,
2100	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2101	}
2102	return VINF_SUCCESS;
2103
2104	#else
2105	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2106	return VERR_INTERNAL_ERROR;
2107	#endif
2108	}
2109
2110
2111	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
2112	/**
2113	* Investigate page fault and handle write protection page faults caused by
2114	* dirty bit tracking.
2115	*
2116	* @returns VBox status code.
2117	* @param pVCpu The VMCPU handle.
2118	* @param uErr Page fault error code.
2119	* @param pPdeDst Shadow page directory entry.
2120	* @param pPdeSrc Guest page directory entry.
2121	* @param GCPtrPage Guest context page address.
2122	*/
2123	PGM_BTH_DECL(int, CheckPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCPTR GCPtrPage)
2124	{
2125	bool fWriteProtect = !!(CPUMGetGuestCR0(pVCpu) & X86_CR0_WP);
2126	bool fUserLevelFault = !!(uErr & X86_TRAP_PF_US);
2127	bool fWriteFault = !!(uErr & X86_TRAP_PF_RW);
2128	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2129	bool fBigPagesSupported = true;
2130	# else
2131	bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
2132	# endif
2133	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2134	bool fNoExecuteBitValid = !!(CPUMGetGuestEFER(pVCpu) & MSR_K6_EFER_NXE);
2135	# endif
2136	unsigned uPageFaultLevel;
2137	int rc;
2138	PVM pVM = pVCpu->CTX_SUFF(pVM);
2139	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2140
2141	Assert(PGMIsLockOwner(pVM));
2142
2143	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2144	LogFlow(("CheckPageFault: GCPtrPage=%RGv uErr=%#x PdeSrc=%08x\n", GCPtrPage, uErr, pPdeSrc->u));
2145
2146	# if PGM_GST_TYPE == PGM_TYPE_PAE \
2147	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
2148
2149	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2150	PX86PML4E pPml4eSrc;
2151	PX86PDPE pPdpeSrc;
2152
2153	pPdpeSrc = pgmGstGetLongModePDPTPtr(&pVCpu->pgm.s, GCPtrPage, &pPml4eSrc);
2154	Assert(pPml4eSrc);
2155
2156	/*
2157	* Real page fault? (PML4E level)
2158	*/
2159	if ( (uErr & X86_TRAP_PF_RSVD)
2160	\|\| !pPml4eSrc->n.u1Present
2161	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPml4eSrc->n.u1NoExecute)
2162	\|\| (fWriteFault && !pPml4eSrc->n.u1Write && (fUserLevelFault \|\| fWriteProtect))
2163	\|\| (fUserLevelFault && !pPml4eSrc->n.u1User)
2164	)
2165	{
2166	uPageFaultLevel = 0;
2167	goto l_UpperLevelPageFault;
2168	}
2169	Assert(pPdpeSrc);
2170
2171	# else /* PAE */
2172	PX86PDPE pPdpeSrc = pgmGstGetPaePDPEPtr(&pVCpu->pgm.s, GCPtrPage);
2173	# endif /* PAE */
2174
2175	/*
2176	* Real page fault? (PDPE level)
2177	*/
2178	if ( (uErr & X86_TRAP_PF_RSVD)
2179	\|\| !pPdpeSrc->n.u1Present
2180	# if PGM_GST_TYPE == PGM_TYPE_AMD64 /* NX, r/w, u/s bits in the PDPE are long mode only */
2181	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPdpeSrc->lm.u1NoExecute)
2182	\|\| (fWriteFault && !pPdpeSrc->lm.u1Write && (fUserLevelFault \|\| fWriteProtect))
2183	\|\| (fUserLevelFault && !pPdpeSrc->lm.u1User)
2184	# endif
2185	)
2186	{
2187	uPageFaultLevel = 1;
2188	goto l_UpperLevelPageFault;
2189	}
2190	# endif
2191
2192	/*
2193	* Real page fault? (PDE level)
2194	*/
2195	if ( (uErr & X86_TRAP_PF_RSVD)
2196	\|\| !pPdeSrc->n.u1Present
2197	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2198	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPdeSrc->n.u1NoExecute)
2199	# endif
2200	\|\| (fWriteFault && !pPdeSrc->n.u1Write && (fUserLevelFault \|\| fWriteProtect))
2201	\|\| (fUserLevelFault && !pPdeSrc->n.u1User) )
2202	{
2203	uPageFaultLevel = 2;
2204	goto l_UpperLevelPageFault;
2205	}
2206
2207	/*
2208	* First check the easy case where the page directory has been marked read-only to track
2209	* the dirty bit of an emulated BIG page
2210	*/
2211	if (pPdeSrc->b.u1Size && fBigPagesSupported)
2212	{
2213	/* Mark guest page directory as accessed */
2214	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2215	pPml4eSrc->n.u1Accessed = 1;
2216	pPdpeSrc->lm.u1Accessed = 1;
2217	# endif
2218	pPdeSrc->b.u1Accessed = 1;
2219
2220	/*
2221	* Only write protection page faults are relevant here.
2222	*/
2223	if (fWriteFault)
2224	{
2225	/* Mark guest page directory as dirty (BIG page only). */
2226	pPdeSrc->b.u1Dirty = 1;
2227
2228	if (pPdeDst->n.u1Present)
2229	{
2230	if (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY)
2231	{
2232	SHWPDE PdeDst = *pPdeDst;
2233
2234	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageTrap));
2235	Assert(pPdeSrc->b.u1Write);
2236
2237	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
2238	* fault again and take this path to only invalidate the entry.
2239	*/
2240	PdeDst.n.u1Write = 1;
2241	PdeDst.n.u1Accessed = 1;
2242	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2243	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2244	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
2245	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2246	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2247	}
2248	# ifdef IN_RING0
2249	else
2250	/* Check for stale TLB entry; only applies to the SMP guest case. */
2251	if ( pVM->cCpus > 1
2252	&& pPdeDst->n.u1Write
2253	&& pPdeDst->n.u1Accessed)
2254	{
2255	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2256	if (pShwPage)
2257	{
2258	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2259	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2260	if ( pPteDst->n.u1Present
2261	&& pPteDst->n.u1Write)
2262	{
2263	/* Stale TLB entry. */
2264	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageStale));
2265	PGM_INVL_PG(pVCpu, GCPtrPage);
2266
2267	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2268	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2269	}
2270	}
2271	}
2272	# endif /* IN_RING0 */
2273	}
2274	}
2275	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2276	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2277	}
2278	/* else: 4KB page table */
2279
2280	/*
2281	* Map the guest page table.
2282	*/
2283	PGSTPT pPTSrc;
2284	rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
2285	if (RT_SUCCESS(rc))
2286	{
2287	/*
2288	* Real page fault?
2289	*/
2290	PGSTPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2291	const GSTPTE PteSrc = *pPteSrc;
2292	if ( !PteSrc.n.u1Present
2293	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2294	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && PteSrc.n.u1NoExecute)
2295	# endif
2296	\|\| (fWriteFault && !PteSrc.n.u1Write && (fUserLevelFault \|\| fWriteProtect))
2297	\|\| (fUserLevelFault && !PteSrc.n.u1User)
2298	)
2299	{
2300	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyTrackRealPF));
2301	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2302	LogFlow(("CheckPageFault: real page fault at %RGv PteSrc.u=%08x (2)\n", GCPtrPage, PteSrc.u));
2303
2304	/* Check the present bit as the shadow tables can cause different error codes by being out of sync.
2305	* See the 2nd case above as well.
2306	*/
2307	if (pPdeSrc->n.u1Present && pPteSrc->n.u1Present)
2308	TRPMSetErrorCode(pVCpu, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
2309
2310	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2311	return VINF_EM_RAW_GUEST_TRAP;
2312	}
2313	LogFlow(("CheckPageFault: page fault at %RGv PteSrc.u=%08x\n", GCPtrPage, PteSrc.u));
2314
2315	/*
2316	* Set the accessed bits in the page directory and the page table.
2317	*/
2318	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2319	pPml4eSrc->n.u1Accessed = 1;
2320	pPdpeSrc->lm.u1Accessed = 1;
2321	# endif
2322	pPdeSrc->n.u1Accessed = 1;
2323	pPteSrc->n.u1Accessed = 1;
2324
2325	/*
2326	* Only write protection page faults are relevant here.
2327	*/
2328	if (fWriteFault)
2329	{
2330	/* Write access, so mark guest entry as dirty. */
2331	# ifdef VBOX_WITH_STATISTICS
2332	if (!pPteSrc->n.u1Dirty)
2333	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtiedPage));
2334	else
2335	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageAlreadyDirty));
2336	# endif
2337
2338	pPteSrc->n.u1Dirty = 1;
2339
2340	if (pPdeDst->n.u1Present)
2341	{
2342	#ifndef IN_RING0
2343	/* Bail out here as pgmPoolGetPageByHCPhys will return NULL and we'll crash below.
2344	* Our individual shadow handlers will provide more information and force a fatal exit.
2345	*/
2346	if (MMHyperIsInsideArea(pVM, (RTGCPTR)GCPtrPage))
2347	{
2348	LogRel(("CheckPageFault: write to hypervisor region %RGv\n", GCPtrPage));
2349	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2350	return VINF_SUCCESS;
2351	}
2352	#endif
2353	/*
2354	* Map shadow page table.
2355	*/
2356	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2357	if (pShwPage)
2358	{
2359	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2360	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2361	if (pPteDst->n.u1Present) /** @todo Optimize accessed bit emulation? */
2362	{
2363	if (pPteDst->u & PGM_PTFLAGS_TRACK_DIRTY)
2364	{
2365	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, pPteSrc->u & GST_PTE_PG_MASK);
2366	SHWPTE PteDst = *pPteDst;
2367
2368	LogFlow(("DIRTY page trap addr=%RGv\n", GCPtrPage));
2369	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageTrap));
2370
2371	Assert(pPteSrc->n.u1Write);
2372
2373	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
2374	* fault again and take this path to only invalidate the entry.
2375	*/
2376	if (RT_LIKELY(pPage))
2377	{
2378	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2379	/* Assuming write handlers here as the PTE is present (otherwise we wouldn't be here). */
2380	PteDst.n.u1Write = 0;
2381	else
2382	{
2383	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
2384	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
2385	{
2386	rc = pgmPhysPageMakeWritableUnlocked(pVM, pPage, pPteSrc->u & GST_PTE_PG_MASK);
2387	AssertRC(rc);
2388	}
2389	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED)
2390	PteDst.n.u1Write = 1;
2391	else
2392	PteDst.n.u1Write = 0;
2393	}
2394	}
2395	else
2396	PteDst.n.u1Write = 1;
2397
2398	PteDst.n.u1Dirty = 1;
2399	PteDst.n.u1Accessed = 1;
2400	PteDst.au32[0] &= ~PGM_PTFLAGS_TRACK_DIRTY;
2401	ASMAtomicWriteSize(pPteDst, PteDst.u);
2402	PGM_INVL_PG(pVCpu, GCPtrPage);
2403
2404	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2405	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2406	}
2407	# ifdef IN_RING0
2408	else
2409	/* Check for stale TLB entry; only applies to the SMP guest case. */
2410	if ( pVM->cCpus > 1
2411	&& pPteDst->n.u1Write == 1
2412	&& pPteDst->n.u1Accessed == 1)
2413	{
2414	/* Stale TLB entry. */
2415	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageStale));
2416	PGM_INVL_PG(pVCpu, GCPtrPage);
2417
2418	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2419	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2420	}
2421	# endif
2422	}
2423	}
2424	else
2425	AssertMsgFailed(("pgmPoolGetPageByHCPhys %RGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
2426	}
2427	}
2428	/** @todo Optimize accessed bit emulation? */
2429	# ifdef VBOX_STRICT
2430	/*
2431	* Sanity check.
2432	*/
2433	else if ( !pPteSrc->n.u1Dirty
2434	&& (pPdeSrc->n.u1Write & pPteSrc->n.u1Write)
2435	&& pPdeDst->n.u1Present)
2436	{
2437	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2438	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2439	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2440	if ( pPteDst->n.u1Present
2441	&& pPteDst->n.u1Write)
2442	LogFlow(("Writable present page %RGv not marked for dirty bit tracking!!!\n", GCPtrPage));
2443	}
2444	# endif /* VBOX_STRICT */
2445	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2446	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2447	}
2448	AssertRC(rc);
2449	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2450	return rc;
2451
2452
2453	l_UpperLevelPageFault:
2454	/*
2455	* Pagefault detected while checking the PML4E, PDPE or PDE.
2456	* Single exit handler to get rid of duplicate code paths.
2457	*/
2458	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyTrackRealPF));
2459	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2460	Log(("CheckPageFault: real page fault at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2461
2462	if (
2463	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2464	pPml4eSrc->n.u1Present &&
2465	# endif
2466	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
2467	pPdpeSrc->n.u1Present &&
2468	# endif
2469	pPdeSrc->n.u1Present)
2470	{
2471	/* Check the present bit as the shadow tables can cause different error codes by being out of sync. */
2472	if (pPdeSrc->b.u1Size && fBigPagesSupported)
2473	{
2474	TRPMSetErrorCode(pVCpu, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
2475	}
2476	else
2477	{
2478	/*
2479	* Map the guest page table.
2480	*/
2481	PGSTPT pPTSrc2;
2482	rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc2);
2483	if (RT_SUCCESS(rc))
2484	{
2485	PGSTPTE pPteSrc = &pPTSrc2->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2486	const GSTPTE PteSrc = *pPteSrc;
2487	if (pPteSrc->n.u1Present)
2488	TRPMSetErrorCode(pVCpu, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
2489	}
2490	AssertRC(rc);
2491	}
2492	}
2493	return VINF_EM_RAW_GUEST_TRAP;
2494	}
2495	#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
2496
2497
2498	/**
2499	* Sync a shadow page table.
2500	*
2501	* The shadow page table is not present. This includes the case where
2502	* there is a conflict with a mapping.
2503	*
2504	* @returns VBox status code.
2505	* @param pVCpu The VMCPU handle.
2506	* @param iPD Page directory index.
2507	* @param pPDSrc Source page directory (i.e. Guest OS page directory).
2508	* Assume this is a temporary mapping.
2509	* @param GCPtrPage GC Pointer of the page that caused the fault
2510	*/
2511	PGM_BTH_DECL(int, SyncPT)(PVMCPU pVCpu, unsigned iPDSrc, PGSTPD pPDSrc, RTGCPTR GCPtrPage)
2512	{
2513	PVM pVM = pVCpu->CTX_SUFF(pVM);
2514	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2515
2516	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2517	#if 0 /* rarely useful; leave for debugging. */
2518	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPtPD[iPDSrc]);
2519	#endif
2520	LogFlow(("SyncPT: GCPtrPage=%RGv\n", GCPtrPage));
2521
2522	Assert(PGMIsLocked(pVM));
2523
2524	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2525	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
2526	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
2527	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2528	&& PGM_SHW_TYPE != PGM_TYPE_EPT
2529
2530	int rc = VINF_SUCCESS;
2531
2532	/*
2533	* Validate input a little bit.
2534	*/
2535	AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage));
2536	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2537	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2538	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
2539
2540	/* Fetch the pgm pool shadow descriptor. */
2541	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2542	Assert(pShwPde);
2543
2544	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2545	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2546	PPGMPOOLPAGE pShwPde = NULL;
2547	PX86PDPAE pPDDst;
2548	PSHWPDE pPdeDst;
2549
2550	/* Fetch the pgm pool shadow descriptor. */
2551	rc = pgmShwGetPaePoolPagePD(&pVCpu->pgm.s, GCPtrPage, &pShwPde);
2552	AssertRCSuccessReturn(rc, rc);
2553	Assert(pShwPde);
2554
2555	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGM(&pVM->pgm.s, pShwPde);
2556	pPdeDst = &pPDDst->a[iPDDst];
2557
2558	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2559	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2560	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2561	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2562	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2563	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2564	AssertRCSuccessReturn(rc, rc);
2565	Assert(pPDDst);
2566	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2567	# endif
2568	SHWPDE PdeDst = *pPdeDst;
2569
2570	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2571	/* Fetch the pgm pool shadow descriptor. */
2572	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2573	Assert(pShwPde);
2574	# endif
2575
2576	# ifndef PGM_WITHOUT_MAPPINGS
2577	/*
2578	* Check for conflicts.
2579	* GC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
2580	* HC: Simply resolve the conflict.
2581	*/
2582	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
2583	{
2584	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
2585	# ifndef IN_RING3
2586	Log(("SyncPT: Conflict at %RGv\n", GCPtrPage));
2587	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2588	return VERR_ADDRESS_CONFLICT;
2589	# else
2590	PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
2591	Assert(pMapping);
2592	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2593	rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2594	# elif PGM_GST_TYPE == PGM_TYPE_PAE
2595	rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2596	# else
2597	AssertFailed(); /* can't happen for amd64 */
2598	# endif
2599	if (RT_FAILURE(rc))
2600	{
2601	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2602	return rc;
2603	}
2604	PdeDst = *pPdeDst;
2605	# endif
2606	}
2607	# else /* PGM_WITHOUT_MAPPINGS */
2608	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
2609	# endif /* PGM_WITHOUT_MAPPINGS */
2610	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2611
2612	# if defined(IN_RC)
2613	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
2614	PGMDynLockHCPage(pVM, (uint8_t *)pPdeDst);
2615	# endif
2616
2617	/*
2618	* Sync page directory entry.
2619	*/
2620	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2621	if (PdeSrc.n.u1Present)
2622	{
2623	/*
2624	* Allocate & map the page table.
2625	*/
2626	PSHWPT pPTDst;
2627	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2628	const bool fPageTable = !PdeSrc.b.u1Size;
2629	# else
2630	const bool fPageTable = !PdeSrc.b.u1Size \|\| !(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
2631	# endif
2632	PPGMPOOLPAGE pShwPage;
2633	RTGCPHYS GCPhys;
2634	if (fPageTable)
2635	{
2636	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
2637	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2638	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2639	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE / 2);
2640	# endif
2641	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, pShwPde->idx, iPDDst, &pShwPage);
2642	}
2643	else
2644	{
2645	PGMPOOLACCESS enmAccess;
2646
2647	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2648	const bool fNoExecuteBitValid = !!(CPUMGetGuestEFER(pVCpu) & MSR_K6_EFER_NXE);
2649	const bool fNoExecute = fNoExecuteBitValid && PdeSrc.n.u1NoExecute;
2650	# else
2651	const bool fNoExecute = false;
2652	# endif
2653
2654	GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
2655	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2656	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2657	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
2658	# endif
2659	/* Determine the right kind of large page to avoid incorrect cached entry reuse. */
2660	if (PdeSrc.n.u1User)
2661	{
2662	if (PdeSrc.n.u1Write)
2663	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_RW_NX : PGMPOOLACCESS_USER_RW;
2664	else
2665	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_R_NX : PGMPOOLACCESS_USER_R;
2666	}
2667	else
2668	{
2669	if (PdeSrc.n.u1Write)
2670	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW;
2671	else
2672	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_R_NX : PGMPOOLACCESS_SUPERVISOR_R;
2673	}
2674	rc = pgmPoolAllocEx(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, enmAccess, pShwPde->idx, iPDDst, &pShwPage);
2675	}
2676	if (rc == VINF_SUCCESS)
2677	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2678	else if (rc == VINF_PGM_CACHED_PAGE)
2679	{
2680	/*
2681	* The PT was cached, just hook it up.
2682	*/
2683	if (fPageTable)
2684	PdeDst.u = pShwPage->Core.Key
2685	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2686	else
2687	{
2688	PdeDst.u = pShwPage->Core.Key
2689	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2690	/* (see explanation and assumptions further down.) */
2691	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
2692	{
2693	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageBig));
2694	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2695	PdeDst.b.u1Write = 0;
2696	}
2697	}
2698	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2699	# if defined(IN_RC)
2700	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2701	# endif
2702	return VINF_SUCCESS;
2703	}
2704	else if (rc == VERR_PGM_POOL_FLUSHED)
2705	{
2706	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2707	# if defined(IN_RC)
2708	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2709	# endif
2710	return VINF_PGM_SYNC_CR3;
2711	}
2712	else
2713	AssertMsgFailedReturn(("rc=%Rrc\n", rc), VERR_INTERNAL_ERROR);
2714	PdeDst.u &= X86_PDE_AVL_MASK;
2715	PdeDst.u \|= pShwPage->Core.Key;
2716
2717	/*
2718	* Page directory has been accessed (this is a fault situation, remember).
2719	*/
2720	pPDSrc->a[iPDSrc].n.u1Accessed = 1;
2721	if (fPageTable)
2722	{
2723	/*
2724	* Page table - 4KB.
2725	*
2726	* Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
2727	*/
2728	Log2(("SyncPT: 4K %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
2729	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
2730	PGSTPT pPTSrc;
2731	rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
2732	if (RT_SUCCESS(rc))
2733	{
2734	/*
2735	* Start by syncing the page directory entry so CSAM's TLB trick works.
2736	*/
2737	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| X86_PDE_AVL_MASK))
2738	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2739	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2740	# if defined(IN_RC)
2741	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2742	# endif
2743
2744	/*
2745	* Directory/page user or supervisor privilege: (same goes for read/write)
2746	*
2747	* Directory Page Combined
2748	* U/S U/S U/S
2749	* 0 0 0
2750	* 0 1 0
2751	* 1 0 0
2752	* 1 1 1
2753	*
2754	* Simple AND operation. Table listed for completeness.
2755	*
2756	*/
2757	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT4K));
2758	# ifdef PGM_SYNC_N_PAGES
2759	unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2760	unsigned iPTDst = iPTBase;
2761	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2762	if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
2763	iPTDst = 0;
2764	else
2765	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2766	# else /* !PGM_SYNC_N_PAGES */
2767	unsigned iPTDst = 0;
2768	const unsigned iPTDstEnd = RT_ELEMENTS(pPTDst->a);
2769	# endif /* !PGM_SYNC_N_PAGES */
2770	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2771	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2772	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2773	# else
2774	const unsigned offPTSrc = 0;
2775	# endif
2776	for (; iPTDst < iPTDstEnd; iPTDst++)
2777	{
2778	const unsigned iPTSrc = iPTDst + offPTSrc;
2779	const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2780
2781	if (PteSrc.n.u1Present) /* we've already cleared it above */
2782	{
2783	# ifndef IN_RING0
2784	/*
2785	* Assuming kernel code will be marked as supervisor - and not as user level
2786	* and executed using a conforming code selector - And marked as readonly.
2787	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
2788	*/
2789	PPGMPAGE pPage;
2790	if ( ((PdeSrc.u & pPTSrc->a[iPTSrc].u) & (X86_PTE_RW \| X86_PTE_US))
2791	\|\| !CSAMDoesPageNeedScanning(pVM, (RTRCPTR)((iPDSrc << GST_PD_SHIFT) \| (iPTSrc << PAGE_SHIFT)))
2792	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
2793	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2794	)
2795	# endif
2796	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2797	Log2(("SyncPT: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n",
2798	(RTGCPTR)(((RTGCPTR)iPDSrc << GST_PD_SHIFT) \| ((RTGCPTR)iPTSrc << PAGE_SHIFT)),
2799	PteSrc.n.u1Present,
2800	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2801	PteSrc.n.u1User & PdeSrc.n.u1User,
2802	(uint64_t)PteSrc.u,
2803	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : "", pPTDst->a[iPTDst].u, iPTSrc, PdeSrc.au32[0],
2804	(RTGCPHYS)((PdeSrc.u & GST_PDE_PG_MASK) + iPTSrc*sizeof(PteSrc)) ));
2805	}
2806	} /* for PTEs */
2807	}
2808	}
2809	else
2810	{
2811	/*
2812	* Big page - 2/4MB.
2813	*
2814	* We'll walk the ram range list in parallel and optimize lookups.
2815	* We will only sync on shadow page table at a time.
2816	*/
2817	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT4M));
2818
2819	/**
2820	* @todo It might be more efficient to sync only a part of the 4MB page (similar to what we do for 4kb PDs).
2821	*/
2822
2823	/*
2824	* Start by syncing the page directory entry.
2825	*/
2826	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
2827	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2828
2829	/*
2830	* If the page is not flagged as dirty and is writable, then make it read-only
2831	* at PD level, so we can set the dirty bit when the page is modified.
2832	*
2833	* ASSUMES that page access handlers are implemented on page table entry level.
2834	* Thus we will first catch the dirty access and set PDE.D and restart. If
2835	* there is an access handler, we'll trap again and let it work on the problem.
2836	*/
2837	/** @todo move the above stuff to a section in the PGM documentation. */
2838	Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
2839	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
2840	{
2841	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageBig));
2842	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2843	PdeDst.b.u1Write = 0;
2844	}
2845	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2846	# if defined(IN_RC)
2847	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2848	# endif
2849
2850	/*
2851	* Fill the shadow page table.
2852	*/
2853	/* Get address and flags from the source PDE. */
2854	SHWPTE PteDstBase;
2855	PteDstBase.u = PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT);
2856
2857	/* Loop thru the entries in the shadow PT. */
2858	const RTGCPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
2859	Log2(("SyncPT: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%RGv GCPhys=%RGp %s\n",
2860	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
2861	GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2862	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
2863	unsigned iPTDst = 0;
2864	while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2865	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2866	{
2867	/* Advance ram range list. */
2868	while (pRam && GCPhys > pRam->GCPhysLast)
2869	pRam = pRam->CTX_SUFF(pNext);
2870	if (pRam && GCPhys >= pRam->GCPhys)
2871	{
2872	unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2873	do
2874	{
2875	/* Make shadow PTE. */
2876	PPGMPAGE pPage = &pRam->aPages[iHCPage];
2877	SHWPTE PteDst;
2878
2879	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2880	/* Try make the page writable if necessary. */
2881	if ( PteDstBase.n.u1Write
2882	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2883	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
2884	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
2885	# endif
2886	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
2887	{
2888	rc = pgmPhysPageMakeWritableUnlocked(pVM, pPage, GCPhys);
2889	AssertRCReturn(rc, rc);
2890	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2891	break;
2892	}
2893	# endif
2894
2895	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2896	{
2897	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
2898	{
2899	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) \| PteDstBase.u;
2900	PteDst.n.u1Write = 0;
2901	}
2902	else
2903	PteDst.u = 0;
2904	}
2905	# ifndef IN_RING0
2906	/*
2907	* Assuming kernel code will be marked as supervisor and not as user level and executed
2908	* using a conforming code selector. Don't check for readonly, as that implies the whole
2909	* 4MB can be code or readonly data. Linux enables write access for its large pages.
2910	*/
2911	else if ( !PdeSrc.n.u1User
2912	&& CSAMDoesPageNeedScanning(pVM, (RTRCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))))
2913	PteDst.u = 0;
2914	# endif
2915	else
2916	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) \| PteDstBase.u;
2917
2918	/* Only map writable pages writable. */
2919	if ( PteDst.n.u1Write
2920	&& PteDst.n.u1Present
2921	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2922	{
2923	PteDst.n.u1Write = 0; /** @todo this isn't quite working yet... */
2924	Log3(("SyncPT: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, (RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))));
2925	}
2926
2927	if (PteDst.n.u1Present)
2928	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2929
2930	/* commit it */
2931	pPTDst->a[iPTDst] = PteDst;
2932	Log4(("SyncPT: BIG %RGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
2933	(RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT)), PteDst.n.u1Present, PteDst.n.u1Write, PteDst.n.u1User, (uint64_t)PteDst.u,
2934	PteDst.u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2935
2936	/* advance */
2937	GCPhys += PAGE_SIZE;
2938	iHCPage++;
2939	iPTDst++;
2940	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2941	&& GCPhys <= pRam->GCPhysLast);
2942	}
2943	else if (pRam)
2944	{
2945	Log(("Invalid pages at %RGp\n", GCPhys));
2946	do
2947	{
2948	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2949	GCPhys += PAGE_SIZE;
2950	iPTDst++;
2951	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2952	&& GCPhys < pRam->GCPhys);
2953	}
2954	else
2955	{
2956	Log(("Invalid pages at %RGp (2)\n", GCPhys));
2957	for ( ; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
2958	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2959	}
2960	} /* while more PTEs */
2961	} /* 4KB / 4MB */
2962	}
2963	else
2964	AssertRelease(!PdeDst.n.u1Present);
2965
2966	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2967	if (RT_FAILURE(rc))
2968	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPTFailed));
2969	return rc;
2970
2971	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2972	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2973	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2974	&& !defined(IN_RC)
2975
2976	/*
2977	* Validate input a little bit.
2978	*/
2979	int rc = VINF_SUCCESS;
2980	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2981	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2982	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
2983
2984	/* Fetch the pgm pool shadow descriptor. */
2985	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2986	Assert(pShwPde);
2987
2988	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2989	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2990	PPGMPOOLPAGE pShwPde = NULL; /* initialized to shut up gcc */
2991	PX86PDPAE pPDDst;
2992	PSHWPDE pPdeDst;
2993
2994	/* Fetch the pgm pool shadow descriptor. */
2995	rc = pgmShwGetPaePoolPagePD(&pVCpu->pgm.s, GCPtrPage, &pShwPde);
2996	AssertRCSuccessReturn(rc, rc);
2997	Assert(pShwPde);
2998
2999	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGM(&pVM->pgm.s, pShwPde);
3000	pPdeDst = &pPDDst->a[iPDDst];
3001
3002	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3003	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3004	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3005	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
3006	PX86PDPT pPdptDst= NULL; /* initialized to shut up gcc */
3007	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
3008	AssertRCSuccessReturn(rc, rc);
3009	Assert(pPDDst);
3010	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3011
3012	/* Fetch the pgm pool shadow descriptor. */
3013	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
3014	Assert(pShwPde);
3015
3016	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
3017	const unsigned iPdpt = (GCPtrPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
3018	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3019	PEPTPD pPDDst;
3020	PEPTPDPT pPdptDst;
3021
3022	rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, &pPdptDst, &pPDDst);
3023	if (rc != VINF_SUCCESS)
3024	{
3025	AssertRC(rc);
3026	return rc;
3027	}
3028	Assert(pPDDst);
3029	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3030
3031	/* Fetch the pgm pool shadow descriptor. */
3032	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & EPT_PDPTE_PG_MASK);
3033	Assert(pShwPde);
3034	# endif
3035	SHWPDE PdeDst = *pPdeDst;
3036
3037	Assert(!(PdeDst.u & PGM_PDFLAGS_MAPPING));
3038	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
3039
3040	GSTPDE PdeSrc;
3041	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
3042	PdeSrc.n.u1Present = 1;
3043	PdeSrc.n.u1Write = 1;
3044	PdeSrc.n.u1Accessed = 1;
3045	PdeSrc.n.u1User = 1;
3046
3047	/*
3048	* Allocate & map the page table.
3049	*/
3050	PSHWPT pPTDst;
3051	PPGMPOOLPAGE pShwPage;
3052	RTGCPHYS GCPhys;
3053
3054	/* Virtual address = physical address */
3055	GCPhys = GCPtrPage & X86_PAGE_4K_BASE_MASK;
3056	rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, pShwPde->idx, iPDDst, &pShwPage);
3057
3058	if ( rc == VINF_SUCCESS
3059	\|\| rc == VINF_PGM_CACHED_PAGE)
3060	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
3061	else
3062	AssertMsgFailedReturn(("rc=%Rrc\n", rc), VERR_INTERNAL_ERROR);
3063
3064	PdeDst.u &= X86_PDE_AVL_MASK;
3065	PdeDst.u \|= pShwPage->Core.Key;
3066	PdeDst.n.u1Present = 1;
3067	PdeDst.n.u1Write = 1;
3068	# if PGM_SHW_TYPE == PGM_TYPE_EPT
3069	PdeDst.n.u1Execute = 1;
3070	# else
3071	PdeDst.n.u1User = 1;
3072	PdeDst.n.u1Accessed = 1;
3073	# endif
3074	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
3075
3076	pgmLock(pVM);
3077	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, PGM_SYNC_NR_PAGES, 0 /* page not present */);
3078	pgmUnlock(pVM);
3079	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
3080	return rc;
3081
3082	#else
3083	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
3084	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
3085	return VERR_INTERNAL_ERROR;
3086	#endif
3087	}
3088
3089
3090
3091	/**
3092	* Prefetch a page/set of pages.
3093	*
3094	* Typically used to sync commonly used pages before entering raw mode
3095	* after a CR3 reload.
3096	*
3097	* @returns VBox status code.
3098	* @param pVCpu The VMCPU handle.
3099	* @param GCPtrPage Page to invalidate.
3100	*/
3101	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
3102	{
3103	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
3104	&& PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
3105	/*
3106	* Check that all Guest levels thru the PDE are present, getting the
3107	* PD and PDE in the processes.
3108	*/
3109	int rc = VINF_SUCCESS;
3110	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3111	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3112	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
3113	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
3114	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3115	unsigned iPDSrc;
3116	X86PDPE PdpeSrc;
3117	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, GCPtrPage, &iPDSrc, &PdpeSrc);
3118	if (!pPDSrc)
3119	return VINF_SUCCESS; /* not present */
3120	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3121	unsigned iPDSrc;
3122	PX86PML4E pPml4eSrc;
3123	X86PDPE PdpeSrc;
3124	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3125	if (!pPDSrc)
3126	return VINF_SUCCESS; /* not present */
3127	# endif
3128	const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3129	# else
3130	PGSTPD pPDSrc = NULL;
3131	const unsigned iPDSrc = 0;
3132	GSTPDE PdeSrc;
3133
3134	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
3135	PdeSrc.n.u1Present = 1;
3136	PdeSrc.n.u1Write = 1;
3137	PdeSrc.n.u1Accessed = 1;
3138	PdeSrc.n.u1User = 1;
3139	# endif
3140
3141	if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
3142	{
3143	PVM pVM = pVCpu->CTX_SUFF(pVM);
3144	pgmLock(pVM);
3145
3146	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3147	const X86PDE PdeDst = pgmShwGet32BitPDE(&pVCpu->pgm.s, GCPtrPage);
3148	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3149	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3150	PX86PDPAE pPDDst;
3151	X86PDEPAE PdeDst;
3152	# if PGM_GST_TYPE != PGM_TYPE_PAE
3153	X86PDPE PdpeSrc;
3154
3155	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3156	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3157	# endif
3158	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, &PdpeSrc, &pPDDst);
3159	if (rc != VINF_SUCCESS)
3160	{
3161	pgmUnlock(pVM);
3162	AssertRC(rc);
3163	return rc;
3164	}
3165	Assert(pPDDst);
3166	PdeDst = pPDDst->a[iPDDst];
3167
3168	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3169	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3170	PX86PDPAE pPDDst;
3171	X86PDEPAE PdeDst;
3172
3173	# if PGM_GST_TYPE == PGM_TYPE_PROT
3174	/* AMD-V nested paging */
3175	X86PML4E Pml4eSrc;
3176	X86PDPE PdpeSrc;
3177	PX86PML4E pPml4eSrc = &Pml4eSrc;
3178
3179	/* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3180	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_NX \| X86_PML4E_A;
3181	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_NX \| X86_PDPE_A;
3182	# endif
3183
3184	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc, &PdpeSrc, &pPDDst);
3185	if (rc != VINF_SUCCESS)
3186	{
3187	pgmUnlock(pVM);
3188	AssertRC(rc);
3189	return rc;
3190	}
3191	Assert(pPDDst);
3192	PdeDst = pPDDst->a[iPDDst];
3193	# endif
3194	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
3195	{
3196	if (!PdeDst.n.u1Present)
3197	{
3198	/** r=bird: This guy will set the A bit on the PDE, probably harmless. */
3199	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3200	}
3201	else
3202	{
3203	/** @note We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
3204	* R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
3205	* makes no sense to prefetch more than one page.
3206	*/
3207	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3208	if (RT_SUCCESS(rc))
3209	rc = VINF_SUCCESS;
3210	}
3211	}
3212	pgmUnlock(pVM);
3213	}
3214	return rc;
3215
3216	#elif PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3217	return VINF_SUCCESS; /* ignore */
3218	#endif
3219	}
3220
3221
3222
3223
3224	/**
3225	* Syncs a page during a PGMVerifyAccess() call.
3226	*
3227	* @returns VBox status code (informational included).
3228	* @param pVCpu The VMCPU handle.
3229	* @param GCPtrPage The address of the page to sync.
3230	* @param fPage The effective guest page flags.
3231	* @param uErr The trap error code.
3232	*/
3233	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fPage, unsigned uErr)
3234	{
3235	PVM pVM = pVCpu->CTX_SUFF(pVM);
3236
3237	LogFlow(("VerifyAccessSyncPage: GCPtrPage=%RGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
3238
3239	Assert(!HWACCMIsNestedPagingActive(pVM));
3240	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_TYPE_AMD64) \
3241	&& PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
3242
3243	# ifndef IN_RING0
3244	if (!(fPage & X86_PTE_US))
3245	{
3246	/*
3247	* Mark this page as safe.
3248	*/
3249	/** @todo not correct for pages that contain both code and data!! */
3250	Log(("CSAMMarkPage %RGv; scanned=%d\n", GCPtrPage, true));
3251	CSAMMarkPage(pVM, (RTRCPTR)GCPtrPage, true);
3252	}
3253	# endif
3254
3255	/*
3256	* Get guest PD and index.
3257	*/
3258	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3259	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3260	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
3261	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
3262	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3263	unsigned iPDSrc = 0;
3264	X86PDPE PdpeSrc;
3265	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, GCPtrPage, &iPDSrc, &PdpeSrc);
3266
3267	if (pPDSrc)
3268	{
3269	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3270	return VINF_EM_RAW_GUEST_TRAP;
3271	}
3272	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3273	unsigned iPDSrc;
3274	PX86PML4E pPml4eSrc;
3275	X86PDPE PdpeSrc;
3276	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3277	if (!pPDSrc)
3278	{
3279	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3280	return VINF_EM_RAW_GUEST_TRAP;
3281	}
3282	# endif
3283	# else
3284	PGSTPD pPDSrc = NULL;
3285	const unsigned iPDSrc = 0;
3286	# endif
3287	int rc = VINF_SUCCESS;
3288
3289	pgmLock(pVM);
3290
3291	/*
3292	* First check if the shadow pd is present.
3293	*/
3294	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3295	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
3296	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3297	PX86PDEPAE pPdeDst;
3298	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3299	PX86PDPAE pPDDst;
3300	# if PGM_GST_TYPE != PGM_TYPE_PAE
3301	X86PDPE PdpeSrc;
3302
3303	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3304	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3305	# endif
3306	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, &PdpeSrc, &pPDDst);
3307	if (rc != VINF_SUCCESS)
3308	{
3309	pgmUnlock(pVM);
3310	AssertRC(rc);
3311	return rc;
3312	}
3313	Assert(pPDDst);
3314	pPdeDst = &pPDDst->a[iPDDst];
3315
3316	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3317	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3318	PX86PDPAE pPDDst;
3319	PX86PDEPAE pPdeDst;
3320
3321	# if PGM_GST_TYPE == PGM_TYPE_PROT
3322	/* AMD-V nested paging */
3323	X86PML4E Pml4eSrc;
3324	X86PDPE PdpeSrc;
3325	PX86PML4E pPml4eSrc = &Pml4eSrc;
3326
3327	/* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3328	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_NX \| X86_PML4E_A;
3329	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_NX \| X86_PDPE_A;
3330	# endif
3331
3332	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc, &PdpeSrc, &pPDDst);
3333	if (rc != VINF_SUCCESS)
3334	{
3335	pgmUnlock(pVM);
3336	AssertRC(rc);
3337	return rc;
3338	}
3339	Assert(pPDDst);
3340	pPdeDst = &pPDDst->a[iPDDst];
3341	# endif
3342
3343	# if defined(IN_RC)
3344	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
3345	PGMDynLockHCPage(pVM, (uint8_t *)pPdeDst);
3346	# endif
3347
3348	if (!pPdeDst->n.u1Present)
3349	{
3350	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3351	if (rc != VINF_SUCCESS)
3352	{
3353	# if defined(IN_RC)
3354	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
3355	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
3356	# endif
3357	pgmUnlock(pVM);
3358	AssertRC(rc);
3359	return rc;
3360	}
3361	}
3362
3363	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3364	/* Check for dirty bit fault */
3365	rc = PGM_BTH_NAME(CheckPageFault)(pVCpu, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
3366	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
3367	Log(("PGMVerifyAccess: success (dirty)\n"));
3368	else
3369	{
3370	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3371	# else
3372	{
3373	GSTPDE PdeSrc;
3374	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
3375	PdeSrc.n.u1Present = 1;
3376	PdeSrc.n.u1Write = 1;
3377	PdeSrc.n.u1Accessed = 1;
3378	PdeSrc.n.u1User = 1;
3379
3380	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
3381	Assert(rc != VINF_EM_RAW_GUEST_TRAP);
3382	if (uErr & X86_TRAP_PF_US)
3383	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncUser));
3384	else /* supervisor */
3385	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
3386
3387	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3388	if (RT_SUCCESS(rc))
3389	{
3390	/* Page was successfully synced */
3391	Log2(("PGMVerifyAccess: success (sync)\n"));
3392	rc = VINF_SUCCESS;
3393	}
3394	else
3395	{
3396	Log(("PGMVerifyAccess: access violation for %RGv rc=%d\n", GCPtrPage, rc));
3397	rc = VINF_EM_RAW_GUEST_TRAP;
3398	}
3399	}
3400	# if defined(IN_RC)
3401	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
3402	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
3403	# endif
3404	pgmUnlock(pVM);
3405	return rc;
3406
3407	#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
3408
3409	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
3410	return VERR_INTERNAL_ERROR;
3411	#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
3412	}
3413
3414	#undef MY_STAM_COUNTER_INC
3415	#define MY_STAM_COUNTER_INC(a) do { } while (0)
3416
3417
3418	/**
3419	* Syncs the paging hierarchy starting at CR3.
3420	*
3421	* @returns VBox status code, no specials.
3422	* @param pVCpu The VMCPU handle.
3423	* @param cr0 Guest context CR0 register
3424	* @param cr3 Guest context CR3 register
3425	* @param cr4 Guest context CR4 register
3426	* @param fGlobal Including global page directories or not
3427	*/
3428	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
3429	{
3430	PVM pVM = pVCpu->CTX_SUFF(pVM);
3431
3432	if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
3433	fGlobal = true; /* Change this CR3 reload to be a global one. */
3434
3435	LogFlow(("SyncCR3 %d\n", fGlobal));
3436
3437	#if PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
3438
3439	pgmLock(pVM);
3440	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
3441	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3442	if (pPool->cDirtyPages)
3443	pgmPoolResetDirtyPages(pVM);
3444	# endif
3445
3446	/*
3447	* Update page access handlers.
3448	* The virtual are always flushed, while the physical are only on demand.
3449	* WARNING: We are incorrectly not doing global flushing on Virtual Handler updates. We'll
3450	* have to look into that later because it will have a bad influence on the performance.
3451	* @note SvL: There's no need for that. Just invalidate the virtual range(s).
3452	* bird: Yes, but that won't work for aliases.
3453	*/
3454	/** @todo this MUST go away. See #1557. */
3455	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncCR3Handlers), h);
3456	PGM_GST_NAME(HandlerVirtualUpdate)(pVM, cr4);
3457	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncCR3Handlers), h);
3458	pgmUnlock(pVM);
3459	#endif
3460
3461	#if PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3462	/*
3463	* Nested / EPT - almost no work.
3464	*/
3465	/** @todo check if this is really necessary; the call does it as well... */
3466	HWACCMFlushTLB(pVCpu);
3467	return VINF_SUCCESS;
3468
3469	#elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3470	/*
3471	* AMD64 (Shw & Gst) - No need to check all paging levels; we zero
3472	* out the shadow parts when the guest modifies its tables.
3473	*/
3474	return VINF_SUCCESS;
3475
3476	#else /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3477
3478	# ifdef PGM_WITHOUT_MAPPINGS
3479	Assert(pVM->pgm.s.fMappingsFixed);
3480	return VINF_SUCCESS;
3481	# else
3482	/* Nothing to do when mappings are fixed. */
3483	if (pVM->pgm.s.fMappingsFixed)
3484	return VINF_SUCCESS;
3485
3486	int rc = PGMMapResolveConflicts(pVM);
3487	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
3488	if (rc == VINF_PGM_SYNC_CR3)
3489	{
3490	LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
3491	return VINF_PGM_SYNC_CR3;
3492	}
3493	# endif
3494	return VINF_SUCCESS;
3495	#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3496	}
3497
3498
3499
3500
3501	#ifdef VBOX_STRICT
3502	#ifdef IN_RC
3503	# undef AssertMsgFailed
3504	# define AssertMsgFailed Log
3505	#endif
3506	#ifdef IN_RING3
3507	# include <VBox/dbgf.h>
3508
3509	/**
3510	* Dumps a page table hierarchy use only physical addresses and cr4/lm flags.
3511	*
3512	* @returns VBox status code (VINF_SUCCESS).
3513	* @param cr3 The root of the hierarchy.
3514	* @param crr The cr4, only PAE and PSE is currently used.
3515	* @param fLongMode Set if long mode, false if not long mode.
3516	* @param cMaxDepth Number of levels to dump.
3517	* @param pHlp Pointer to the output functions.
3518	*/
3519	RT_C_DECLS_BEGIN
3520	VMMR3DECL(int) PGMR3DumpHierarchyHC(PVM pVM, uint32_t cr3, uint32_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp);
3521	RT_C_DECLS_END
3522
3523	#endif
3524
3525	/**
3526	* Checks that the shadow page table is in sync with the guest one.
3527	*
3528	* @returns The number of errors.
3529	* @param pVM The virtual machine.
3530	* @param pVCpu The VMCPU handle.
3531	* @param cr3 Guest context CR3 register
3532	* @param cr4 Guest context CR4 register
3533	* @param GCPtr Where to start. Defaults to 0.
3534	* @param cb How much to check. Defaults to everything.
3535	*/
3536	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb)
3537	{
3538	#if PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3539	return 0;
3540	#else
3541	unsigned cErrors = 0;
3542	PVM pVM = pVCpu->CTX_SUFF(pVM);
3543	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3544
3545	#if PGM_GST_TYPE == PGM_TYPE_PAE
3546	/** @todo currently broken; crashes below somewhere */
3547	AssertFailed();
3548	#endif
3549
3550	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
3551	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3552	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
3553
3554	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3555	bool fBigPagesSupported = true;
3556	# else
3557	bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
3558	# endif
3559	PPGMCPU pPGM = &pVCpu->pgm.s;
3560	RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
3561	RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
3562	# ifndef IN_RING0
3563	RTHCPHYS HCPhys; /* general usage. */
3564	# endif
3565	int rc;
3566
3567	/*
3568	* Check that the Guest CR3 and all its mappings are correct.
3569	*/
3570	AssertMsgReturn(pPGM->GCPhysCR3 == (cr3 & GST_CR3_PAGE_MASK),
3571	("Invalid GCPhysCR3=%RGp cr3=%RGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
3572	false);
3573	# if !defined(IN_RING0) && PGM_GST_TYPE != PGM_TYPE_AMD64
3574	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3575	rc = PGMShwGetPage(pVCpu, (RTGCPTR)pPGM->pGst32BitPdRC, NULL, &HCPhysShw);
3576	# else
3577	rc = PGMShwGetPage(pVCpu, (RTGCPTR)pPGM->pGstPaePdptRC, NULL, &HCPhysShw);
3578	# endif
3579	AssertRCReturn(rc, 1);
3580	HCPhys = NIL_RTHCPHYS;
3581	rc = pgmRamGCPhys2HCPhys(&pVM->pgm.s, cr3 & GST_CR3_PAGE_MASK, &HCPhys);
3582	AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
3583	# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
3584	pgmGstGet32bitPDPtr(pPGM);
3585	RTGCPHYS GCPhys;
3586	rc = PGMR3DbgR3Ptr2GCPhys(pVM, pPGM->pGst32BitPdR3, &GCPhys);
3587	AssertRCReturn(rc, 1);
3588	AssertMsgReturn((cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false);
3589	# endif
3590	# endif /* !IN_RING0 */
3591
3592	/*
3593	* Get and check the Shadow CR3.
3594	*/
3595	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3596	unsigned cPDEs = X86_PG_ENTRIES;
3597	unsigned cIncrement = X86_PG_ENTRIES * PAGE_SIZE;
3598	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3599	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3600	unsigned cPDEs = X86_PG_PAE_ENTRIES * 4; /* treat it as a 2048 entry table. */
3601	# else
3602	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3603	# endif
3604	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3605	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3606	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3607	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3608	# endif
3609	if (cb != ~(RTGCPTR)0)
3610	cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
3611
3612	/** @todo call the other two PGMAssert() functions. /
3613
3614	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3615	unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3616
3617	for (; iPml4 < X86_PG_PAE_ENTRIES; iPml4++)
3618	{
3619	PPGMPOOLPAGE pShwPdpt = NULL;
3620	PX86PML4E pPml4eSrc;
3621	PX86PML4E pPml4eDst;
3622	RTGCPHYS GCPhysPdptSrc;
3623
3624	pPml4eSrc = pgmGstGetLongModePML4EPtr(&pVCpu->pgm.s, iPml4);
3625	pPml4eDst = pgmShwGetLongModePML4EPtr(&pVCpu->pgm.s, iPml4);
3626
3627	/* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
3628	if (!pPml4eDst->n.u1Present)
3629	{
3630	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3631	continue;
3632	}
3633
3634	pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
3635	GCPhysPdptSrc = pPml4eSrc->u & X86_PML4E_PG_MASK_FULL;
3636
3637	if (pPml4eSrc->n.u1Present != pPml4eDst->n.u1Present)
3638	{
3639	AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3640	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3641	cErrors++;
3642	continue;
3643	}
3644
3645	if (GCPhysPdptSrc != pShwPdpt->GCPhys)
3646	{
3647	AssertMsgFailed(("Physical address doesn't match! iPml4 %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
3648	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3649	cErrors++;
3650	continue;
3651	}
3652
3653	if ( pPml4eDst->n.u1User != pPml4eSrc->n.u1User
3654	\|\| pPml4eDst->n.u1Write != pPml4eSrc->n.u1Write
3655	\|\| pPml4eDst->n.u1NoExecute != pPml4eSrc->n.u1NoExecute)
3656	{
3657	AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3658	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3659	cErrors++;
3660	continue;
3661	}
3662	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3663	{
3664	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3665
3666	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
3667	/*
3668	* Check the PDPTEs too.
3669	*/
3670	unsigned iPdpt = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
3671
3672	for (;iPdpt <= SHW_PDPT_MASK; iPdpt++)
3673	{
3674	unsigned iPDSrc = 0; /* initialized to shut up gcc */
3675	PPGMPOOLPAGE pShwPde = NULL;
3676	PX86PDPE pPdpeDst;
3677	RTGCPHYS GCPhysPdeSrc;
3678	# if PGM_GST_TYPE == PGM_TYPE_PAE
3679	X86PDPE PdpeSrc;
3680	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, GCPtr, &iPDSrc, &PdpeSrc);
3681	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(&pVCpu->pgm.s);
3682	# else
3683	PX86PML4E pPml4eSrcIgn;
3684	X86PDPE PdpeSrc;
3685	PX86PDPT pPdptDst;
3686	PX86PDPAE pPDDst;
3687	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, GCPtr, &pPml4eSrcIgn, &PdpeSrc, &iPDSrc);
3688
3689	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtr, NULL, &pPdptDst, &pPDDst);
3690	if (rc != VINF_SUCCESS)
3691	{
3692	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
3693	GCPtr += 512 * _2M;
3694	continue; /* next PDPTE */
3695	}
3696	Assert(pPDDst);
3697	# endif
3698	Assert(iPDSrc == 0);
3699
3700	pPdpeDst = &pPdptDst->a[iPdpt];
3701
3702	if (!pPdpeDst->n.u1Present)
3703	{
3704	GCPtr += 512 * _2M;
3705	continue; /* next PDPTE */
3706	}
3707
3708	pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
3709	GCPhysPdeSrc = PdpeSrc.u & X86_PDPE_PG_MASK;
3710
3711	if (pPdpeDst->n.u1Present != PdpeSrc.n.u1Present)
3712	{
3713	AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3714	GCPtr += 512 * _2M;
3715	cErrors++;
3716	continue;
3717	}
3718
3719	if (GCPhysPdeSrc != pShwPde->GCPhys)
3720	{
3721	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3722	AssertMsgFailed(("Physical address doesn't match! iPml4 %d iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3723	# else
3724	AssertMsgFailed(("Physical address doesn't match! iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3725	# endif
3726	GCPtr += 512 * _2M;
3727	cErrors++;
3728	continue;
3729	}
3730
3731	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3732	if ( pPdpeDst->lm.u1User != PdpeSrc.lm.u1User
3733	\|\| pPdpeDst->lm.u1Write != PdpeSrc.lm.u1Write
3734	\|\| pPdpeDst->lm.u1NoExecute != PdpeSrc.lm.u1NoExecute)
3735	{
3736	AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3737	GCPtr += 512 * _2M;
3738	cErrors++;
3739	continue;
3740	}
3741	# endif
3742
3743	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3744	{
3745	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3746	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3747	GSTPD const *pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
3748	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3749	PCX86PD pPDDst = pgmShwGet32BitPDPtr(&pVCpu->pgm.s);
3750	# endif
3751	# endif /* PGM_GST_TYPE == PGM_TYPE_32BIT */
3752	/*
3753	* Iterate the shadow page directory.
3754	*/
3755	GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
3756	unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
3757
3758	for (;
3759	iPDDst < cPDEs;
3760	iPDDst++, GCPtr += cIncrement)
3761	{
3762	# if PGM_SHW_TYPE == PGM_TYPE_PAE
3763	const SHWPDE PdeDst = *pgmShwGetPaePDEPtr(pPGM, GCPtr);
3764	# else
3765	const SHWPDE PdeDst = pPDDst->a[iPDDst];
3766	# endif
3767	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
3768	{
3769	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
3770	if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
3771	{
3772	AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
3773	cErrors++;
3774	continue;
3775	}
3776	}
3777	else if ( (PdeDst.u & X86_PDE_P)
3778	\|\| ((PdeDst.u & (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY))
3779	)
3780	{
3781	HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
3782	PPGMPOOLPAGE pPoolPage = pgmPoolGetPage(pPool, HCPhysShw);
3783	if (!pPoolPage)
3784	{
3785	AssertMsgFailed(("Invalid page table address %RHp at %RGv! PdeDst=%#RX64\n",
3786	HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
3787	cErrors++;
3788	continue;
3789	}
3790	const SHWPT pPTDst = (const SHWPT )PGMPOOL_PAGE_2_PTR(pVM, pPoolPage);
3791
3792	if (PdeDst.u & (X86_PDE4M_PWT \| X86_PDE4M_PCD))
3793	{
3794	AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n",
3795	GCPtr, (uint64_t)PdeDst.u));
3796	cErrors++;
3797	}
3798
3799	if (PdeDst.u & (X86_PDE4M_G \| X86_PDE4M_D))
3800	{
3801	AssertMsgFailed(("4K PDE reserved flags at %RGv! PdeDst=%#RX64\n",
3802	GCPtr, (uint64_t)PdeDst.u));
3803	cErrors++;
3804	}
3805
3806	const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
3807	if (!PdeSrc.n.u1Present)
3808	{
3809	AssertMsgFailed(("Guest PDE at %RGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
3810	GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
3811	cErrors++;
3812	continue;
3813	}
3814
3815	if ( !PdeSrc.b.u1Size
3816	\|\| !fBigPagesSupported)
3817	{
3818	GCPhysGst = PdeSrc.u & GST_PDE_PG_MASK;
3819	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3820	GCPhysGst \|= (iPDDst & 1) * (PAGE_SIZE / 2);
3821	# endif
3822	}
3823	else
3824	{
3825	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3826	if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
3827	{
3828	AssertMsgFailed(("Guest PDE at %RGv is using PSE36 or similar! PdeSrc=%#RX64\n",
3829	GCPtr, (uint64_t)PdeSrc.u));
3830	cErrors++;
3831	continue;
3832	}
3833	# endif
3834	GCPhysGst = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
3835	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3836	GCPhysGst \|= GCPtr & RT_BIT(X86_PAGE_2M_SHIFT);
3837	# endif
3838	}
3839
3840	if ( pPoolPage->enmKind
3841	!= (!PdeSrc.b.u1Size \|\| !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
3842	{
3843	AssertMsgFailed(("Invalid shadow page table kind %d at %RGv! PdeSrc=%#RX64\n",
3844	pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
3845	cErrors++;
3846	}
3847
3848	PPGMPAGE pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
3849	if (!pPhysPage)
3850	{
3851	AssertMsgFailed(("Cannot find guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3852	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3853	cErrors++;
3854	continue;
3855	}
3856
3857	if (GCPhysGst != pPoolPage->GCPhys)
3858	{
3859	AssertMsgFailed(("GCPhysGst=%RGp != pPage->GCPhys=%RGp at %RGv\n",
3860	GCPhysGst, pPoolPage->GCPhys, GCPtr));
3861	cErrors++;
3862	continue;
3863	}
3864
3865	if ( !PdeSrc.b.u1Size
3866	\|\| !fBigPagesSupported)
3867	{
3868	/*
3869	* Page Table.
3870	*/
3871	const GSTPT *pPTSrc;
3872	rc = PGM_GCPHYS_2_PTR(pVM, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1), &pPTSrc);
3873	if (RT_FAILURE(rc))
3874	{
3875	AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3876	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3877	cErrors++;
3878	continue;
3879	}
3880	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/))
3881	!= (PdeDst.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/)))
3882	{
3883	/// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
3884	// (This problem will go away when/if we shadow multiple CR3s.)
3885	AssertMsgFailed(("4K PDE flags mismatch at %RGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3886	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3887	cErrors++;
3888	continue;
3889	}
3890	if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3891	{
3892	AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%RGv PdeDst=%#RX64\n",
3893	GCPtr, (uint64_t)PdeDst.u));
3894	cErrors++;
3895	continue;
3896	}
3897
3898	/* iterate the page table. */
3899	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3900	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3901	const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
3902	# else
3903	const unsigned offPTSrc = 0;
3904	# endif
3905	for (unsigned iPT = 0, off = 0;
3906	iPT < RT_ELEMENTS(pPTDst->a);
3907	iPT++, off += PAGE_SIZE)
3908	{
3909	const SHWPTE PteDst = pPTDst->a[iPT];
3910
3911	/* skip not-present entries. */
3912	if (!(PteDst.u & (X86_PTE_P \| PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
3913	continue;
3914	Assert(PteDst.n.u1Present);
3915
3916	const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
3917	if (!PteSrc.n.u1Present)
3918	{
3919	# ifdef IN_RING3
3920	PGMAssertHandlerAndFlagsInSync(pVM);
3921	PGMR3DumpHierarchyGC(pVM, cr3, cr4, (PdeSrc.u & GST_PDE_PG_MASK));
3922	# endif
3923	AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n",
3924	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u, pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
3925	(PdeSrc.u & GST_PDE_PG_MASK) + (iPT + offPTSrc)*sizeof(PteSrc)));
3926	cErrors++;
3927	continue;
3928	}
3929
3930	uint64_t fIgnoreFlags = GST_PTE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_G \| X86_PTE_D \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
3931	# if 1 /** @todo sync accessed bit properly... */
3932	fIgnoreFlags \|= X86_PTE_A;
3933	# endif
3934
3935	/* match the physical addresses */
3936	HCPhysShw = PteDst.u & SHW_PTE_PG_MASK;
3937	GCPhysGst = PteSrc.u & GST_PTE_PG_MASK;
3938
3939	# ifdef IN_RING3
3940	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3941	if (RT_FAILURE(rc))
3942	{
3943	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
3944	{
3945	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3946	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3947	cErrors++;
3948	continue;
3949	}
3950	}
3951	else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
3952	{
3953	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3954	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3955	cErrors++;
3956	continue;
3957	}
3958	# endif
3959
3960	pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
3961	if (!pPhysPage)
3962	{
3963	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
3964	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
3965	{
3966	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3967	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3968	cErrors++;
3969	continue;
3970	}
3971	# endif
3972	if (PteDst.n.u1Write)
3973	{
3974	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3975	GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3976	cErrors++;
3977	}
3978	fIgnoreFlags \|= X86_PTE_RW;
3979	}
3980	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
3981	{
3982	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage:%R[pgmpage] GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3983	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3984	cErrors++;
3985	continue;
3986	}
3987
3988	/* flags */
3989	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
3990	{
3991	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
3992	{
3993	if (PteDst.n.u1Write)
3994	{
3995	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
3996	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3997	cErrors++;
3998	continue;
3999	}
4000	fIgnoreFlags \|= X86_PTE_RW;
4001	}
4002	else
4003	{
4004	if (PteDst.n.u1Present)
4005	{
4006	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4007	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4008	cErrors++;
4009	continue;
4010	}
4011	fIgnoreFlags \|= X86_PTE_P;
4012	}
4013	}
4014	else
4015	{
4016	if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
4017	{
4018	if (PteDst.n.u1Write)
4019	{
4020	AssertMsgFailed(("!DIRTY page at %RGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
4021	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4022	cErrors++;
4023	continue;
4024	}
4025	if (!(PteDst.u & PGM_PTFLAGS_TRACK_DIRTY))
4026	{
4027	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4028	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4029	cErrors++;
4030	continue;
4031	}
4032	if (PteDst.n.u1Dirty)
4033	{
4034	AssertMsgFailed(("!DIRTY page at %RGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4035	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4036	cErrors++;
4037	}
4038	# if 0 /** @todo sync access bit properly... */
4039	if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
4040	{
4041	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4042	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4043	cErrors++;
4044	}
4045	fIgnoreFlags \|= X86_PTE_RW;
4046	# else
4047	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
4048	# endif
4049	}
4050	else if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
4051	{
4052	/* access bit emulation (not implemented). */
4053	if (PteSrc.n.u1Accessed \|\| PteDst.n.u1Present)
4054	{
4055	AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
4056	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4057	cErrors++;
4058	continue;
4059	}
4060	if (!PteDst.n.u1Accessed)
4061	{
4062	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
4063	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4064	cErrors++;
4065	}
4066	fIgnoreFlags \|= X86_PTE_P;
4067	}
4068	# ifdef DEBUG_sandervl
4069	fIgnoreFlags \|= X86_PTE_D \| X86_PTE_A;
4070	# endif
4071	}
4072
4073	if ( (PteSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
4074	&& (PteSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags)
4075	)
4076	{
4077	AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
4078	GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
4079	fIgnoreFlags, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4080	cErrors++;
4081	continue;
4082	}
4083	} /* foreach PTE */
4084	}
4085	else
4086	{
4087	/*
4088	* Big Page.
4089	*/
4090	uint64_t fIgnoreFlags = X86_PDE_AVL_MASK \| GST_PDE_PG_MASK \| X86_PDE4M_G \| X86_PDE4M_D \| X86_PDE4M_PS \| X86_PDE4M_PWT \| X86_PDE4M_PCD;
4091	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
4092	{
4093	if (PdeDst.n.u1Write)
4094	{
4095	AssertMsgFailed(("!DIRTY page at %RGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4096	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4097	cErrors++;
4098	continue;
4099	}
4100	if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
4101	{
4102	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4103	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4104	cErrors++;
4105	continue;
4106	}
4107	# if 0 /** @todo sync access bit properly... */
4108	if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
4109	{
4110	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4111	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4112	cErrors++;
4113	}
4114	fIgnoreFlags \|= X86_PTE_RW;
4115	# else
4116	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
4117	# endif
4118	}
4119	else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4120	{
4121	/* access bit emulation (not implemented). */
4122	if (PdeSrc.b.u1Accessed \|\| PdeDst.n.u1Present)
4123	{
4124	AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4125	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4126	cErrors++;
4127	continue;
4128	}
4129	if (!PdeDst.n.u1Accessed)
4130	{
4131	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4132	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4133	cErrors++;
4134	}
4135	fIgnoreFlags \|= X86_PTE_P;
4136	}
4137
4138	if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
4139	{
4140	AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
4141	GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
4142	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4143	cErrors++;
4144	}
4145
4146	/* iterate the page table. */
4147	for (unsigned iPT = 0, off = 0;
4148	iPT < RT_ELEMENTS(pPTDst->a);
4149	iPT++, off += PAGE_SIZE, GCPhysGst += PAGE_SIZE)
4150	{
4151	const SHWPTE PteDst = pPTDst->a[iPT];
4152
4153	if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
4154	{
4155	AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
4156	GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4157	cErrors++;
4158	}
4159
4160	/* skip not-present entries. */
4161	if (!PteDst.n.u1Present) /** @todo deal with ALL handlers and CSAM !P pages! */
4162	continue;
4163
4164	fIgnoreFlags = X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT \| X86_PTE_D \| X86_PTE_A \| X86_PTE_G \| X86_PTE_PAE_NX;
4165
4166	/* match the physical addresses */
4167	HCPhysShw = PteDst.u & X86_PTE_PAE_PG_MASK;
4168
4169	# ifdef IN_RING3
4170	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4171	if (RT_FAILURE(rc))
4172	{
4173	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4174	{
4175	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4176	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4177	cErrors++;
4178	}
4179	}
4180	else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
4181	{
4182	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4183	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4184	cErrors++;
4185	continue;
4186	}
4187	# endif
4188	pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
4189	if (!pPhysPage)
4190	{
4191	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4192	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4193	{
4194	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4195	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4196	cErrors++;
4197	continue;
4198	}
4199	# endif
4200	if (PteDst.n.u1Write)
4201	{
4202	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4203	GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4204	cErrors++;
4205	}
4206	fIgnoreFlags \|= X86_PTE_RW;
4207	}
4208	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4209	{
4210	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage=%R[pgmpage] GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4211	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4212	cErrors++;
4213	continue;
4214	}
4215
4216	/* flags */
4217	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4218	{
4219	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4220	{
4221	if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
4222	{
4223	if (PteDst.n.u1Write)
4224	{
4225	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4226	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4227	cErrors++;
4228	continue;
4229	}
4230	fIgnoreFlags \|= X86_PTE_RW;
4231	}
4232	}
4233	else
4234	{
4235	if (PteDst.n.u1Present)
4236	{
4237	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4238	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4239	cErrors++;
4240	continue;
4241	}
4242	fIgnoreFlags \|= X86_PTE_P;
4243	}
4244	}
4245
4246	if ( (PdeSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
4247	&& (PdeSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags) /* lazy phys handler dereg. */
4248	)
4249	{
4250	AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
4251	GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
4252	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4253	cErrors++;
4254	continue;
4255	}
4256	} /* for each PTE */
4257	}
4258	}
4259	/* not present */
4260
4261	} /* for each PDE */
4262
4263	} /* for each PDPTE */
4264
4265	} /* for each PML4E */
4266
4267	# ifdef DEBUG
4268	if (cErrors)
4269	LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
4270	# endif
4271
4272	#endif /* GST == 32BIT, PAE or AMD64 */
4273	return cErrors;
4274
4275	#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT */
4276	}
4277	#endif /* VBOX_STRICT */
4278
4279
4280	/**
4281	* Sets up the CR3 for shadow paging
4282	*
4283	* @returns Strict VBox status code.
4284	* @retval VINF_SUCCESS.
4285	*
4286	* @param pVCpu The VMCPU handle.
4287	* @param GCPhysCR3 The physical address in the CR3 register.
4288	*/
4289	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3)
4290	{
4291	PVM pVM = pVCpu->CTX_SUFF(pVM);
4292
4293	/* Update guest paging info. */
4294	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
4295	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
4296	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
4297
4298	LogFlow(("MapCR3: %RGp\n", GCPhysCR3));
4299
4300	/*
4301	* Map the page CR3 points at.
4302	*/
4303	RTHCPTR HCPtrGuestCR3;
4304	RTHCPHYS HCPhysGuestCR3;
4305	pgmLock(pVM);
4306	PPGMPAGE pPageCR3 = pgmPhysGetPage(&pVM->pgm.s, GCPhysCR3);
4307	AssertReturn(pPageCR3, VERR_INTERNAL_ERROR_2);
4308	HCPhysGuestCR3 = PGM_PAGE_GET_HCPHYS(pPageCR3);
4309	/** @todo this needs some reworking wrt. locking. */
4310	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4311	HCPtrGuestCR3 = NIL_RTHCPTR;
4312	int rc = VINF_SUCCESS;
4313	# else
4314	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPageCR3, GCPhysCR3 & GST_CR3_PAGE_MASK, (void )&HCPtrGuestCR3); / @todo r=bird: This GCPhysCR3 masking isn't necessary. */
4315	# endif
4316	pgmUnlock(pVM);
4317	if (RT_SUCCESS(rc))
4318	{
4319	rc = PGMMap(pVM, (RTGCPTR)pVM->pgm.s.GCPtrCR3Mapping, HCPhysGuestCR3, PAGE_SIZE, 0);
4320	if (RT_SUCCESS(rc))
4321	{
4322	# ifdef IN_RC
4323	PGM_INVL_PG(pVCpu, pVM->pgm.s.GCPtrCR3Mapping);
4324	# endif
4325	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4326	pVCpu->pgm.s.pGst32BitPdR3 = (R3PTRTYPE(PX86PD))HCPtrGuestCR3;
4327	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4328	pVCpu->pgm.s.pGst32BitPdR0 = (R0PTRTYPE(PX86PD))HCPtrGuestCR3;
4329	# endif
4330	pVCpu->pgm.s.pGst32BitPdRC = (RCPTRTYPE(PX86PD))pVM->pgm.s.GCPtrCR3Mapping;
4331
4332	# elif PGM_GST_TYPE == PGM_TYPE_PAE
4333	unsigned off = GCPhysCR3 & GST_CR3_PAGE_MASK & PAGE_OFFSET_MASK;
4334	pVCpu->pgm.s.pGstPaePdptR3 = (R3PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4335	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4336	pVCpu->pgm.s.pGstPaePdptR0 = (R0PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4337	# endif
4338	pVCpu->pgm.s.pGstPaePdptRC = (RCPTRTYPE(PX86PDPT))((RCPTRTYPE(uint8_t *))pVM->pgm.s.GCPtrCR3Mapping + off);
4339	Log(("Cached mapping %RRv\n", pVCpu->pgm.s.pGstPaePdptRC));
4340
4341	/*
4342	* Map the 4 PDs too.
4343	*/
4344	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(&pVCpu->pgm.s);
4345	RTGCPTR GCPtr = pVM->pgm.s.GCPtrCR3Mapping + PAGE_SIZE;
4346	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++, GCPtr += PAGE_SIZE)
4347	{
4348	if (pGuestPDPT->a[i].n.u1Present)
4349	{
4350	RTHCPTR HCPtr;
4351	RTHCPHYS HCPhys;
4352	RTGCPHYS GCPhys = pGuestPDPT->a[i].u & X86_PDPE_PG_MASK;
4353	pgmLock(pVM);
4354	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
4355	AssertReturn(pPage, VERR_INTERNAL_ERROR_2);
4356	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
4357	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4358	HCPtr = NIL_RTHCPTR;
4359	int rc2 = VINF_SUCCESS;
4360	# else
4361	int rc2 = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, (void **)&HCPtr);
4362	# endif
4363	pgmUnlock(pVM);
4364	if (RT_SUCCESS(rc2))
4365	{
4366	rc = PGMMap(pVM, GCPtr, HCPhys, PAGE_SIZE, 0);
4367	AssertRCReturn(rc, rc);
4368
4369	pVCpu->pgm.s.apGstPaePDsR3[i] = (R3PTRTYPE(PX86PDPAE))HCPtr;
4370	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4371	pVCpu->pgm.s.apGstPaePDsR0[i] = (R0PTRTYPE(PX86PDPAE))HCPtr;
4372	# endif
4373	pVCpu->pgm.s.apGstPaePDsRC[i] = (RCPTRTYPE(PX86PDPAE))GCPtr;
4374	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys;
4375	# ifdef IN_RC
4376	PGM_INVL_PG(pVCpu, GCPtr);
4377	# endif
4378	continue;
4379	}
4380	AssertMsgFailed(("pgmR3Gst32BitMapCR3: rc2=%d GCPhys=%RGp i=%d\n", rc2, GCPhys, i));
4381	}
4382
4383	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4384	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4385	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4386	# endif
4387	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4388	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4389	# ifdef IN_RC
4390	PGM_INVL_PG(pVCpu, GCPtr); /** @todo this shouldn't be necessary? */
4391	# endif
4392	}
4393
4394	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
4395	pVCpu->pgm.s.pGstAmd64Pml4R3 = (R3PTRTYPE(PX86PML4))HCPtrGuestCR3;
4396	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4397	pVCpu->pgm.s.pGstAmd64Pml4R0 = (R0PTRTYPE(PX86PML4))HCPtrGuestCR3;
4398	# endif
4399	# endif
4400	}
4401	else
4402	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4403	}
4404	else
4405	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4406
4407	#else /* prot/real stub */
4408	int rc = VINF_SUCCESS;
4409	#endif
4410
4411	/* Update shadow paging info for guest modes with paging (32, pae, 64). */
4412	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4413	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4414	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64) \
4415	&& ( PGM_GST_TYPE != PGM_TYPE_REAL \
4416	&& PGM_GST_TYPE != PGM_TYPE_PROT))
4417
4418	Assert(!HWACCMIsNestedPagingActive(pVM));
4419
4420	/*
4421	* Update the shadow root page as well since that's not fixed.
4422	*/
4423	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4424	PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
4425	uint32_t iOldShwUserTable = pVCpu->pgm.s.iShwUserTable;
4426	uint32_t iOldShwUser = pVCpu->pgm.s.iShwUser;
4427	PPGMPOOLPAGE pNewShwPageCR3;
4428
4429	pgmLock(pVM);
4430
4431	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4432	if (pPool->cDirtyPages)
4433	pgmPoolResetDirtyPages(pVM);
4434	# endif
4435
4436	Assert(!(GCPhysCR3 >> (PAGE_SHIFT + 32)));
4437	rc = pgmPoolAlloc(pVM, GCPhysCR3 & GST_CR3_PAGE_MASK, BTH_PGMPOOLKIND_ROOT, SHW_POOL_ROOT_IDX, GCPhysCR3 >> PAGE_SHIFT, &pNewShwPageCR3, true /* lock page */);
4438	AssertFatalRC(rc);
4439	rc = VINF_SUCCESS;
4440
4441	# ifdef IN_RC
4442	/*
4443	* WARNING! We can't deal with jumps to ring 3 in the code below as the
4444	* state will be inconsistent! Flush important things now while
4445	* we still can and then make sure there are no ring-3 calls.
4446	*/
4447	REMNotifyHandlerPhysicalFlushIfAlmostFull(pVM, pVCpu);
4448	VMMRZCallRing3Disable(pVCpu);
4449	# endif
4450
4451	pVCpu->pgm.s.iShwUser = SHW_POOL_ROOT_IDX;
4452	pVCpu->pgm.s.iShwUserTable = GCPhysCR3 >> PAGE_SHIFT;
4453	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3) = pNewShwPageCR3;
4454	# ifdef IN_RING0
4455	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4456	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4457	# elif defined(IN_RC)
4458	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4459	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4460	# else
4461	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4462	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4463	# endif
4464
4465	# ifndef PGM_WITHOUT_MAPPINGS
4466	/*
4467	* Apply all hypervisor mappings to the new CR3.
4468	* Note that SyncCR3 will be executed in case CR3 is changed in a guest paging mode; this will
4469	* make sure we check for conflicts in the new CR3 root.
4470	*/
4471	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
4472	Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL) \|\| VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
4473	# endif
4474	rc = pgmMapActivateCR3(pVM, pNewShwPageCR3);
4475	AssertRCReturn(rc, rc);
4476	# endif
4477
4478	/* Set the current hypervisor CR3. */
4479	CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
4480	SELMShadowCR3Changed(pVM, pVCpu);
4481
4482	# ifdef IN_RC
4483	/* NOTE: The state is consistent again. */
4484	VMMRZCallRing3Enable(pVCpu);
4485	# endif
4486
4487	/* Clean up the old CR3 root. */
4488	if ( pOldShwPageCR3
4489	&& pOldShwPageCR3 != pNewShwPageCR3 /* @todo can happen due to incorrect syncing between REM & PGM; find the real cause */)
4490	{
4491	Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
4492	# ifndef PGM_WITHOUT_MAPPINGS
4493	/* Remove the hypervisor mappings from the shadow page table. */
4494	pgmMapDeactivateCR3(pVM, pOldShwPageCR3);
4495	# endif
4496	/* Mark the page as unlocked; allow flushing again. */
4497	pgmPoolUnlockPage(pPool, pOldShwPageCR3);
4498
4499	pgmPoolFreeByPage(pPool, pOldShwPageCR3, iOldShwUser, iOldShwUserTable);
4500	}
4501	pgmUnlock(pVM);
4502	# endif
4503
4504	return rc;
4505	}
4506
4507	/**
4508	* Unmaps the shadow CR3.
4509	*
4510	* @returns VBox status, no specials.
4511	* @param pVCpu The VMCPU handle.
4512	*/
4513	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu)
4514	{
4515	LogFlow(("UnmapCR3\n"));
4516
4517	int rc = VINF_SUCCESS;
4518	PVM pVM = pVCpu->CTX_SUFF(pVM);
4519
4520	/*
4521	* Update guest paging info.
4522	*/
4523	#if PGM_GST_TYPE == PGM_TYPE_32BIT
4524	pVCpu->pgm.s.pGst32BitPdR3 = 0;
4525	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4526	pVCpu->pgm.s.pGst32BitPdR0 = 0;
4527	# endif
4528	pVCpu->pgm.s.pGst32BitPdRC = 0;
4529
4530	#elif PGM_GST_TYPE == PGM_TYPE_PAE
4531	pVCpu->pgm.s.pGstPaePdptR3 = 0;
4532	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4533	pVCpu->pgm.s.pGstPaePdptR0 = 0;
4534	# endif
4535	pVCpu->pgm.s.pGstPaePdptRC = 0;
4536	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
4537	{
4538	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4539	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4540	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4541	# endif
4542	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4543	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4544	}
4545
4546	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
4547	pVCpu->pgm.s.pGstAmd64Pml4R3 = 0;
4548	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4549	pVCpu->pgm.s.pGstAmd64Pml4R0 = 0;
4550	# endif
4551
4552	#else /* prot/real mode stub */
4553	/* nothing to do */
4554	#endif
4555
4556	#if !defined(IN_RC) /* In RC we rely on MapCR3 to do the shadow part for us at a safe time */
4557	/*
4558	* Update shadow paging info.
4559	*/
4560	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4561	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4562	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64))
4563
4564	# if PGM_GST_TYPE != PGM_TYPE_REAL
4565	Assert(!HWACCMIsNestedPagingActive(pVM));
4566	# endif
4567
4568	pgmLock(pVM);
4569
4570	# ifndef PGM_WITHOUT_MAPPINGS
4571	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4572	/* Remove the hypervisor mappings from the shadow page table. */
4573	pgmMapDeactivateCR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4574	# endif
4575
4576	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4577	{
4578	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4579
4580	Assert(pVCpu->pgm.s.iShwUser != PGMPOOL_IDX_NESTED_ROOT);
4581
4582	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4583	if (pPool->cDirtyPages)
4584	pgmPoolResetDirtyPages(pVM);
4585	# endif
4586
4587	/* Mark the page as unlocked; allow flushing again. */
4588	pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4589
4590	pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), pVCpu->pgm.s.iShwUser, pVCpu->pgm.s.iShwUserTable);
4591	pVCpu->pgm.s.pShwPageCR3R3 = 0;
4592	pVCpu->pgm.s.pShwPageCR3R0 = 0;
4593	pVCpu->pgm.s.pShwPageCR3RC = 0;
4594	pVCpu->pgm.s.iShwUser = 0;
4595	pVCpu->pgm.s.iShwUserTable = 0;
4596	}
4597	pgmUnlock(pVM);
4598	# endif
4599	#endif /* !IN_RC*/
4600
4601	return rc;
4602	}
4603

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 25776

Download in other formats: