PGMAllBth.h@ 24874

Last change on this file since 24874 was 24806, checked in by vboxsync, 15 years ago
PGMAllBth.h: be even more careful.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 199.1 KB

Line
1	/* $Id: PGMAllBth.h 24806 2009-11-19 18:18:38Z vboxsync $ */
2	/** @file
3	* VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4	*
5	* This file is a big challenge!
6	*/
7
8	/*
9	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
10	*
11	* This file is part of VirtualBox Open Source Edition (OSE), as
12	* available from http://www.virtualbox.org. This file is free software;
13	* you can redistribute it and/or modify it under the terms of the GNU
14	* General Public License (GPL) as published by the Free Software
15	* Foundation, in version 2 as it comes in the "COPYING" file of the
16	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
17	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
18	*
19	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
20	* Clara, CA 95054 USA or visit http://www.sun.com if you need
21	* additional information or have any questions.
22	*/
23
24	/*******************************************************************************
25	* Internal Functions *
26	*******************************************************************************/
27	RT_C_DECLS_BEGIN
28	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault);
29	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
30	PGM_BTH_DECL(int, SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr);
31	PGM_BTH_DECL(int, CheckPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCPTR GCPtrPage);
32	PGM_BTH_DECL(int, SyncPT)(PVMCPU pVCpu, unsigned iPD, PGSTPD pPDSrc, RTGCPTR GCPtrPage);
33	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uErr);
34	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
35	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
36	#ifdef VBOX_STRICT
37	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
38	#endif
39	#ifdef PGMPOOL_WITH_USER_TRACKING
40	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys);
41	#endif
42	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3);
43	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu);
44	RT_C_DECLS_END
45
46
47	/* Filter out some illegal combinations of guest and shadow paging, so we can remove redundant checks inside functions. */
48	#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
49	# error "Invalid combination; PAE guest implies PAE shadow"
50	#endif
51
52	#if (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
53	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT)
54	# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
55	#endif
56
57	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) \
58	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT)
59	# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
60	#endif
61
62	#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT) \
63	\|\| (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
64	# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
65	#endif
66
67	#ifdef IN_RING0 /* no mappings in VT-x and AMD-V mode */
68	# define PGM_WITHOUT_MAPPINGS
69	#endif
70
71
72	#ifndef IN_RING3
73	/**
74	* #PF Handler for raw-mode guest execution.
75	*
76	* @returns VBox status code (appropriate for trap handling and GC return).
77	*
78	* @param pVCpu VMCPU Handle.
79	* @param uErr The trap error code.
80	* @param pRegFrame Trap register frame.
81	* @param pvFault The fault address.
82	*/
83	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
84	{
85	PVM pVM = pVCpu->CTX_SUFF(pVM);
86
87	# if defined(IN_RC) && defined(VBOX_STRICT)
88	PGMDynCheckLocks(pVM);
89	# endif
90
91	# if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
92	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
93	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT)
94
95	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE != PGM_TYPE_PAE
96	/*
97	* Hide the instruction fetch trap indicator for now.
98	*/
99	/** @todo NXE will change this and we must fix NXE in the switcher too! */
100	if (uErr & X86_TRAP_PF_ID)
101	{
102	uErr &= ~X86_TRAP_PF_ID;
103	TRPMSetErrorCode(pVCpu, uErr);
104	}
105	# endif
106
107	/*
108	* Get PDs.
109	*/
110	int rc;
111	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
112	# if PGM_GST_TYPE == PGM_TYPE_32BIT
113	const unsigned iPDSrc = pvFault >> GST_PD_SHIFT;
114	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
115
116	# elif PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64
117
118	# if PGM_GST_TYPE == PGM_TYPE_PAE
119	unsigned iPDSrc = 0; /* initialized to shut up gcc */
120	X86PDPE PdpeSrc;
121	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, pvFault, &iPDSrc, &PdpeSrc);
122
123	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
124	unsigned iPDSrc = 0; /* initialized to shut up gcc */
125	PX86PML4E pPml4eSrc;
126	X86PDPE PdpeSrc;
127	PGSTPD pPDSrc;
128
129	pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, pvFault, &pPml4eSrc, &PdpeSrc, &iPDSrc);
130	Assert(pPml4eSrc);
131	# endif
132
133	/* Quick check for a valid guest trap. (PAE & AMD64) */
134	if (!pPDSrc)
135	{
136	# if PGM_GST_TYPE == PGM_TYPE_AMD64 && GC_ARCH_BITS == 64
137	LogFlow(("Trap0eHandler: guest PML4 %d not present CR3=%RGp\n", (int)((pvFault >> X86_PML4_SHIFT) & X86_PML4_MASK), CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK));
138	# else
139	LogFlow(("Trap0eHandler: guest iPDSrc=%u not present CR3=%RGp\n", iPDSrc, CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK));
140	# endif
141	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2GuestTrap; });
142	TRPMSetErrorCode(pVCpu, uErr);
143	return VINF_EM_RAW_GUEST_TRAP;
144	}
145	# endif
146
147	# else /* !PGM_WITH_PAGING */
148	PGSTPD pPDSrc = NULL;
149	const unsigned iPDSrc = 0;
150	# endif /* !PGM_WITH_PAGING */
151
152	/* Fetch the guest PDE */
153	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
154	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
155	# else
156	GSTPDE PdeSrc;
157	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
158	PdeSrc.n.u1Present = 1;
159	PdeSrc.n.u1Write = 1;
160	PdeSrc.n.u1Accessed = 1;
161	PdeSrc.n.u1User = 1;
162	# endif
163
164	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
165	const unsigned iPDDst = pvFault >> SHW_PD_SHIFT;
166	PX86PD pPDDst = pgmShwGet32BitPDPtr(&pVCpu->pgm.s);
167
168	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
169	const unsigned iPDDst = (pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK; /* pPDDst index, not used with the pool. */
170
171	PX86PDPAE pPDDst;
172	# if PGM_GST_TYPE != PGM_TYPE_PAE
173	X86PDPE PdpeSrc;
174
175	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
176	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
177	# endif
178	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, &PdpeSrc, &pPDDst);
179	if (rc != VINF_SUCCESS)
180	{
181	AssertRC(rc);
182	return rc;
183	}
184	Assert(pPDDst);
185
186	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
187	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
188	PX86PDPAE pPDDst;
189	# if PGM_GST_TYPE == PGM_TYPE_PROT
190	/* AMD-V nested paging */
191	X86PML4E Pml4eSrc;
192	X86PDPE PdpeSrc;
193	PX86PML4E pPml4eSrc = &Pml4eSrc;
194
195	/* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
196	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
197	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
198	# endif
199
200	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, pPml4eSrc, &PdpeSrc, &pPDDst);
201	if (rc != VINF_SUCCESS)
202	{
203	AssertRC(rc);
204	return rc;
205	}
206	Assert(pPDDst);
207
208	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
209	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
210	PEPTPD pPDDst;
211
212	rc = pgmShwGetEPTPDPtr(pVCpu, pvFault, NULL, &pPDDst);
213	if (rc != VINF_SUCCESS)
214	{
215	AssertRC(rc);
216	return rc;
217	}
218	Assert(pPDDst);
219	# endif
220
221	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
222	/*
223	* If we successfully correct the write protection fault due to dirty bit
224	* tracking, or this page fault is a genuine one, then return immediately.
225	*/
226	STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeCheckPageFault, e);
227	rc = PGM_BTH_NAME(CheckPageFault)(pVCpu, uErr, &pPDDst->a[iPDDst], &pPDSrc->a[iPDSrc], pvFault);
228	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeCheckPageFault, e);
229	if ( rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
230	\|\| rc == VINF_EM_RAW_GUEST_TRAP)
231	{
232	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution)
233	= rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? &pVCpu->pgm.s.StatRZTrap0eTime2DirtyAndAccessed : &pVCpu->pgm.s.StatRZTrap0eTime2GuestTrap; });
234	LogBird(("Trap0eHandler: returns %s\n", rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? "VINF_SUCCESS" : "VINF_EM_RAW_GUEST_TRAP"));
235	return rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT ? VINF_SUCCESS : rc;
236	}
237
238	# if 0 /* rarely useful; leave for debugging. */
239	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0ePD[iPDSrc]);
240	# endif
241	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
242
243	/*
244	* A common case is the not-present error caused by lazy page table syncing.
245	*
246	* It is IMPORTANT that we weed out any access to non-present shadow PDEs here
247	* so we can safely assume that the shadow PT is present when calling SyncPage later.
248	*
249	* On failure, we ASSUME that SyncPT is out of memory or detected some kind
250	* of mapping conflict and defer to SyncCR3 in R3.
251	* (Again, we do NOT support access handlers for non-present guest pages.)
252	*
253	*/
254	if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
255	&& !pPDDst->a[iPDDst].n.u1Present
256	&& PdeSrc.n.u1Present
257	)
258	{
259	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2SyncPT; });
260	STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeSyncPT, f);
261	LogFlow(("=>SyncPT %04x = %08x\n", iPDSrc, PdeSrc.au32[0]));
262	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, pvFault);
263	if (RT_SUCCESS(rc))
264	{
265	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeSyncPT, f);
266	return rc;
267	}
268	Log(("SyncPT: %d failed!! rc=%d\n", iPDSrc, rc));
269	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
270	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeSyncPT, f);
271	return VINF_PGM_SYNC_CR3;
272	}
273
274	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(PGM_WITHOUT_MAPPINGS)
275	/*
276	* Check if this address is within any of our mappings.
277	*
278	* This is very fast and it's gonna save us a bit of effort below and prevent
279	* us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
280	* (BTW, it's impossible to have physical access handlers in a mapping.)
281	*/
282	if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
283	{
284	STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
285	PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
286	for ( ; pMapping; pMapping = pMapping->CTX_SUFF(pNext))
287	{
288	if (pvFault < pMapping->GCPtr)
289	break;
290	if (pvFault - pMapping->GCPtr < pMapping->cb)
291	{
292	/*
293	* The first thing we check is if we've got an undetected conflict.
294	*/
295	if (!pVM->pgm.s.fMappingsFixed)
296	{
297	unsigned iPT = pMapping->cb >> GST_PD_SHIFT;
298	while (iPT-- > 0)
299	if (pPDSrc->a[iPDSrc + iPT].n.u1Present)
300	{
301	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eConflicts);
302	Log(("Trap0e: Detected Conflict %RGv-%RGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
303	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
304	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
305	return VINF_PGM_SYNC_CR3;
306	}
307	}
308
309	/*
310	* Check if the fault address is in a virtual page access handler range.
311	*/
312	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->HyperVirtHandlers, pvFault);
313	if ( pCur
314	&& pvFault - pCur->Core.Key < pCur->cb
315	&& uErr & X86_TRAP_PF_RW)
316	{
317	# ifdef IN_RC
318	STAM_PROFILE_START(&pCur->Stat, h);
319	pgmUnlock(pVM);
320	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
321	pgmLock(pVM);
322	STAM_PROFILE_STOP(&pCur->Stat, h);
323	# else
324	AssertFailed();
325	rc = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
326	# endif
327	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersMapping);
328	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
329	return rc;
330	}
331
332	/*
333	* Pretend we're not here and let the guest handle the trap.
334	*/
335	TRPMSetErrorCode(pVCpu, uErr & ~X86_TRAP_PF_P);
336	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eGuestPFMapping);
337	LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
338	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
339	return VINF_EM_RAW_GUEST_TRAP;
340	}
341	}
342	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeMapping, a);
343	} /* pgmAreMappingsEnabled(&pVM->pgm.s) */
344	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
345
346	/*
347	* Check if this fault address is flagged for special treatment,
348	* which means we'll have to figure out the physical address and
349	* check flags associated with it.
350	*
351	* ASSUME that we can limit any special access handling to pages
352	* in page tables which the guest believes to be present.
353	*/
354	if (PdeSrc.n.u1Present)
355	{
356	RTGCPHYS GCPhys = NIL_RTGCPHYS;
357
358	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
359	# if PGM_GST_TYPE == PGM_TYPE_AMD64
360	bool fBigPagesSupported = true;
361	# else
362	bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
363	# endif
364	if ( PdeSrc.b.u1Size
365	&& fBigPagesSupported)
366	GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc)
367	\| ((RTGCPHYS)pvFault & (GST_BIG_PAGE_OFFSET_MASK ^ PAGE_OFFSET_MASK));
368	else
369	{
370	PGSTPT pPTSrc;
371	rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
372	if (RT_SUCCESS(rc))
373	{
374	unsigned iPTESrc = (pvFault >> GST_PT_SHIFT) & GST_PT_MASK;
375	if (pPTSrc->a[iPTESrc].n.u1Present)
376	GCPhys = pPTSrc->a[iPTESrc].u & GST_PTE_PG_MASK;
377	}
378	}
379	# else
380	/* No paging so the fault address is the physical address */
381	GCPhys = (RTGCPHYS)(pvFault & ~PAGE_OFFSET_MASK);
382	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
383
384	/*
385	* If we have a GC address we'll check if it has any flags set.
386	*/
387	if (GCPhys != NIL_RTGCPHYS)
388	{
389	STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
390
391	PPGMPAGE pPage;
392	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
393	if (RT_SUCCESS(rc)) /** just handle the failure immediate (it returns) and make things easier to read. */
394	{
395	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
396	{
397	if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
398	{
399	/*
400	* Physical page access handler.
401	*/
402	const RTGCPHYS GCPhysFault = GCPhys \| (pvFault & PAGE_OFFSET_MASK);
403	PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhysFault);
404	if (pCur)
405	{
406	# ifdef PGM_SYNC_N_PAGES
407	/*
408	* If the region is write protected and we got a page not present fault, then sync
409	* the pages. If the fault was caused by a read, then restart the instruction.
410	* In case of write access continue to the GC write handler.
411	*
412	* ASSUMES that there is only one handler per page or that they have similar write properties.
413	*/
414	if ( pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
415	&& !(uErr & X86_TRAP_PF_P))
416	{
417	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, PGM_SYNC_NR_PAGES, uErr);
418	if ( RT_FAILURE(rc)
419	\|\| !(uErr & X86_TRAP_PF_RW)
420	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
421	{
422	AssertRC(rc);
423	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersOutOfSync);
424	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
425	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSyncHndPhys; });
426	return rc;
427	}
428	}
429	# endif
430
431	AssertMsg( pCur->enmType != PGMPHYSHANDLERTYPE_PHYSICAL_WRITE
432	\|\| (pCur->enmType == PGMPHYSHANDLERTYPE_PHYSICAL_WRITE && (uErr & X86_TRAP_PF_RW)),
433	("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enum=%d\n", pvFault, GCPhys, pPage, uErr, pCur->enmType));
434
435	# if defined(IN_RC) \|\| defined(IN_RING0)
436	if (pCur->CTX_SUFF(pfnHandler))
437	{
438	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
439	# ifdef IN_RING0
440	PFNPGMR0PHYSHANDLER pfnHandler = pCur->CTX_SUFF(pfnHandler);
441	# else
442	PFNPGMRCPHYSHANDLER pfnHandler = pCur->CTX_SUFF(pfnHandler);
443	# endif
444	bool fLeaveLock = (pfnHandler != pPool->CTX_SUFF(pfnAccessHandler));
445	void *pvUser = pCur->CTX_SUFF(pvUser);
446
447	STAM_PROFILE_START(&pCur->Stat, h);
448	if (fLeaveLock)
449	pgmUnlock(pVM); /* @todo: Not entirely safe. */
450
451	rc = pfnHandler(pVM, uErr, pRegFrame, pvFault, GCPhysFault, pvUser);
452	if (fLeaveLock)
453	pgmLock(pVM);
454	# ifdef VBOX_WITH_STATISTICS
455	pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhysFault);
456	if (pCur)
457	STAM_PROFILE_STOP(&pCur->Stat, h);
458	# else
459	pCur = NULL; /* might be invalid by now. */
460	# endif
461
462	}
463	else
464	# endif
465	rc = VINF_EM_RAW_EMULATE_INSTR;
466
467	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersPhysical);
468	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
469	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndPhys; });
470	return rc;
471	}
472	}
473	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
474	else
475	{
476	# ifdef PGM_SYNC_N_PAGES
477	/*
478	* If the region is write protected and we got a page not present fault, then sync
479	* the pages. If the fault was caused by a read, then restart the instruction.
480	* In case of write access continue to the GC write handler.
481	*/
482	if ( PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) < PGM_PAGE_HNDL_PHYS_STATE_ALL
483	&& !(uErr & X86_TRAP_PF_P))
484	{
485	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, PGM_SYNC_NR_PAGES, uErr);
486	if ( RT_FAILURE(rc)
487	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
488	\|\| !(uErr & X86_TRAP_PF_RW))
489	{
490	AssertRC(rc);
491	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersOutOfSync);
492	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
493	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSyncHndVirt; });
494	return rc;
495	}
496	}
497	# endif
498	/*
499	* Ok, it's an virtual page access handler.
500	*
501	* Since it's faster to search by address, we'll do that first
502	* and then retry by GCPhys if that fails.
503	*/
504	/** @todo r=bird: perhaps we should consider looking up by physical address directly now? */
505	/** @note r=svl: true, but lookup on virtual address should remain as a fallback as phys & virt trees might be out of sync, because the
506	* page was changed without us noticing it (not-present -> present without invlpg or mov cr3, xxx)
507	*/
508	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
509	if (pCur)
510	{
511	AssertMsg(!(pvFault - pCur->Core.Key < pCur->cb)
512	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
513	\|\| !(uErr & X86_TRAP_PF_P)
514	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
515	("Unexpected trap for virtual handler: %RGv (phys=%RGp) pPage=%R[pgmpage] uErr=%X, enum=%d\n", pvFault, GCPhys, pPage, uErr, pCur->enmType));
516
517	if ( pvFault - pCur->Core.Key < pCur->cb
518	&& ( uErr & X86_TRAP_PF_RW
519	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
520	{
521	# ifdef IN_RC
522	STAM_PROFILE_START(&pCur->Stat, h);
523	pgmUnlock(pVM);
524	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
525	pgmLock(pVM);
526	STAM_PROFILE_STOP(&pCur->Stat, h);
527	# else
528	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
529	# endif
530	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersVirtual);
531	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
532	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndVirt; });
533	return rc;
534	}
535	/* Unhandled part of a monitored page */
536	}
537	else
538	{
539	/* Check by physical address. */
540	PPGMVIRTHANDLER pCur;
541	unsigned iPage;
542	rc = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys + (pvFault & PAGE_OFFSET_MASK),
543	&pCur, &iPage);
544	Assert(RT_SUCCESS(rc) \|\| !pCur);
545	if ( pCur
546	&& ( uErr & X86_TRAP_PF_RW
547	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
548	{
549	Assert((pCur->aPhysToVirt[iPage].Core.Key & X86_PTE_PAE_PG_MASK) == GCPhys);
550	# ifdef IN_RC
551	RTGCPTR off = (iPage << PAGE_SHIFT) + (pvFault & PAGE_OFFSET_MASK) - (pCur->Core.Key & PAGE_OFFSET_MASK);
552	Assert(off < pCur->cb);
553	STAM_PROFILE_START(&pCur->Stat, h);
554	pgmUnlock(pVM);
555	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, off);
556	pgmLock(pVM);
557	STAM_PROFILE_STOP(&pCur->Stat, h);
558	# else
559	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
560	# endif
561	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersVirtualByPhys);
562	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
563	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndVirt; });
564	return rc;
565	}
566	}
567	}
568	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
569
570	/*
571	* There is a handled area of the page, but this fault doesn't belong to it.
572	* We must emulate the instruction.
573	*
574	* To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
575	* we first check if this was a page-not-present fault for a page with only
576	* write access handlers. Restart the instruction if it wasn't a write access.
577	*/
578	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersUnhandled);
579
580	if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
581	&& !(uErr & X86_TRAP_PF_P))
582	{
583	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, PGM_SYNC_NR_PAGES, uErr);
584	if ( RT_FAILURE(rc)
585	\|\| rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE
586	\|\| !(uErr & X86_TRAP_PF_RW))
587	{
588	AssertRC(rc);
589	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersOutOfSync);
590	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
591	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSyncHndPhys; });
592	return rc;
593	}
594	}
595
596	/** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
597	* It's writing to an unhandled part of the LDT page several million times.
598	*/
599	rc = PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault);
600	LogFlow(("PGM: PGMInterpretInstruction -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
601	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
602	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndUnhandled; });
603	return rc;
604	} /* if any kind of handler */
605
606	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
607	if (uErr & X86_TRAP_PF_P)
608	{
609	/*
610	* The page isn't marked, but it might still be monitored by a virtual page access handler.
611	* (ASSUMES no temporary disabling of virtual handlers.)
612	*/
613	/** @todo r=bird: Since the purpose is to catch out of sync pages with virtual handler(s) here,
614	* we should correct both the shadow page table and physical memory flags, and not only check for
615	* accesses within the handler region but for access to pages with virtual handlers. */
616	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
617	if (pCur)
618	{
619	AssertMsg( !(pvFault - pCur->Core.Key < pCur->cb)
620	\|\| ( pCur->enmType != PGMVIRTHANDLERTYPE_WRITE
621	\|\| !(uErr & X86_TRAP_PF_P)
622	\|\| (pCur->enmType == PGMVIRTHANDLERTYPE_WRITE && (uErr & X86_TRAP_PF_RW))),
623	("Unexpected trap for virtual handler: %08X (phys=%08x) %R[pgmpage] uErr=%X, enum=%d\n", pvFault, GCPhys, pPage, uErr, pCur->enmType));
624
625	if ( pvFault - pCur->Core.Key < pCur->cb
626	&& ( uErr & X86_TRAP_PF_RW
627	\|\| pCur->enmType != PGMVIRTHANDLERTYPE_WRITE ) )
628	{
629	# ifdef IN_RC
630	STAM_PROFILE_START(&pCur->Stat, h);
631	pgmUnlock(pVM);
632	rc = pCur->CTX_SUFF(pfnHandler)(pVM, uErr, pRegFrame, pvFault, pCur->Core.Key, pvFault - pCur->Core.Key);
633	pgmLock(pVM);
634	STAM_PROFILE_STOP(&pCur->Stat, h);
635	# else
636	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
637	# endif
638	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersVirtualUnmarked);
639	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
640	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2HndVirt; });
641	return rc;
642	}
643	}
644	}
645	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
646	}
647	else
648	{
649	/*
650	* When the guest accesses invalid physical memory (e.g. probing
651	* of RAM or accessing a remapped MMIO range), then we'll fall
652	* back to the recompiler to emulate the instruction.
653	*/
654	LogFlow(("PGM #PF: pgmPhysGetPageEx(%RGp) failed with %Rrc\n", GCPhys, rc));
655	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eHandlersInvalid);
656	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
657	return VINF_EM_RAW_EMULATE_INSTR;
658	}
659
660	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeHandlers, b);
661
662	# ifdef PGM_OUT_OF_SYNC_IN_GC /** @todo remove this bugger. */
663	/*
664	* We are here only if page is present in Guest page tables and
665	* trap is not handled by our handlers.
666	*
667	* Check it for page out-of-sync situation.
668	*/
669	STAM_PROFILE_START(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
670
671	if (!(uErr & X86_TRAP_PF_P))
672	{
673	/*
674	* Page is not present in our page tables.
675	* Try to sync it!
676	* BTW, fPageShw is invalid in this branch!
677	*/
678	if (uErr & X86_TRAP_PF_US)
679	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncUser));
680	else /* supervisor */
681	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
682
683	# if defined(LOG_ENABLED) && !defined(IN_RING0)
684	RTGCPHYS GCPhys;
685	uint64_t fPageGst;
686	PGMGstGetPage(pVCpu, pvFault, &fPageGst, &GCPhys);
687	Log(("Page out of sync: %RGv eip=%08x PdeSrc.n.u1User=%d fPageGst=%08llx GCPhys=%RGp scan=%d\n",
688	pvFault, pRegFrame->eip, PdeSrc.n.u1User, fPageGst, GCPhys, CSAMDoesPageNeedScanning(pVM, (RTRCPTR)pRegFrame->eip)));
689	# endif /* LOG_ENABLED */
690
691	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0)
692	if (CPUMGetGuestCPL(pVCpu, pRegFrame) == 0)
693	{
694	uint64_t fPageGst;
695	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
696	if ( RT_SUCCESS(rc)
697	&& !(fPageGst & X86_PTE_US))
698	{
699	/* Note: can't check for X86_TRAP_ID bit, because that requires execute disable support on the CPU */
700	if ( pvFault == (RTGCPTR)pRegFrame->eip
701	\|\| pvFault - pRegFrame->eip < 8 /* instruction crossing a page boundary */
702	# ifdef CSAM_DETECT_NEW_CODE_PAGES
703	\|\| ( !PATMIsPatchGCAddr(pVM, (RTGCPTR)pRegFrame->eip)
704	&& CSAMDoesPageNeedScanning(pVM, (RTRCPTR)pRegFrame->eip)) /* any new code we encounter here */
705	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
706	)
707	{
708	LogFlow(("CSAMExecFault %RX32\n", pRegFrame->eip));
709	rc = CSAMExecFault(pVM, (RTRCPTR)pRegFrame->eip);
710	if (rc != VINF_SUCCESS)
711	{
712	/*
713	* CSAM needs to perform a job in ring 3.
714	*
715	* Sync the page before going to the host context; otherwise we'll end up in a loop if
716	* CSAM fails (e.g. instruction crosses a page boundary and the next page is not present)
717	*/
718	LogFlow(("CSAM ring 3 job\n"));
719	int rc2 = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, 1, uErr);
720	AssertRC(rc2);
721
722	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
723	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2CSAM; });
724	return rc;
725	}
726	}
727	# ifdef CSAM_DETECT_NEW_CODE_PAGES
728	else if ( uErr == X86_TRAP_PF_RW
729	&& pRegFrame->ecx >= 0x100 /* early check for movswd count */
730	&& pRegFrame->ecx < 0x10000)
731	{
732	/* In case of a write to a non-present supervisor shadow page, we'll take special precautions
733	* to detect loading of new code pages.
734	*/
735
736	/*
737	* Decode the instruction.
738	*/
739	RTGCPTR PC;
740	rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->eip, &PC);
741	if (rc == VINF_SUCCESS)
742	{
743	PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
744	uint32_t cbOp;
745	rc = EMInterpretDisasOneEx(pVM, pVCpu, PC, pRegFrame, pDis, &cbOp);
746
747	/* For now we'll restrict this to rep movsw/d instructions */
748	if ( rc == VINF_SUCCESS
749	&& pDis->pCurInstr->opcode == OP_MOVSWD
750	&& (pDis->prefix & PREFIX_REP))
751	{
752	CSAMMarkPossibleCodePage(pVM, pvFault);
753	}
754	}
755	}
756	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
757
758	/*
759	* Mark this page as safe.
760	*/
761	/** @todo not correct for pages that contain both code and data!! */
762	Log2(("CSAMMarkPage %RGv; scanned=%d\n", pvFault, true));
763	CSAMMarkPage(pVM, (RTRCPTR)pvFault, true);
764	}
765	}
766	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0) */
767	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, PGM_SYNC_NR_PAGES, uErr);
768	if (RT_SUCCESS(rc))
769	{
770	/* The page was successfully synced, return to the guest. */
771	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
772	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSync; });
773	return VINF_SUCCESS;
774	}
775	}
776	else /* uErr & X86_TRAP_PF_P: */
777	{
778	/*
779	* Write protected pages are make writable when the guest makes the first
780	* write to it. This happens for pages that are shared, write monitored
781	* and not yet allocated.
782	*
783	* Also, a side effect of not flushing global PDEs are out of sync pages due
784	* to physical monitored regions, that are no longer valid.
785	* Assume for now it only applies to the read/write flag.
786	*/
787	if (RT_SUCCESS(rc) && (uErr & X86_TRAP_PF_RW))
788	{
789	if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
790	{
791	Log(("PGM #PF: Make writable: %RGp %R[pgmpage] pvFault=%RGp uErr=%#x\n",
792	GCPhys, pPage, pvFault, uErr));
793	rc = pgmPhysPageMakeWritableUnlocked(pVM, pPage, GCPhys);
794	if (rc != VINF_SUCCESS)
795	{
796	AssertMsg(rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("%Rrc\n", rc));
797	return rc;
798	}
799	if (RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)))
800	return VINF_EM_NO_MEMORY;
801	}
802
803	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
804	/* Check to see if we need to emulate the instruction as X86_CR0_WP has been cleared. */
805	if ( CPUMGetGuestCPL(pVCpu, pRegFrame) == 0
806	&& ((CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG))
807	{
808	Assert((uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RW \| X86_TRAP_PF_P));
809	uint64_t fPageGst;
810	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
811	if ( RT_SUCCESS(rc)
812	&& !(fPageGst & X86_PTE_RW))
813	{
814	rc = PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault);
815	if (RT_SUCCESS(rc))
816	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eWPEmulInRZ);
817	else
818	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eWPEmulToR3);
819	return rc;
820	}
821	AssertMsg(RT_SUCCESS(rc), ("Unexpected r/w page %RGv flag=%x rc=%Rrc\n", pvFault, (uint32_t)fPageGst, rc));
822	}
823	# endif
824	/// @todo count the above case; else
825	if (uErr & X86_TRAP_PF_US)
826	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncUserWrite));
827	else /* supervisor */
828	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
829
830	/*
831	* Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
832	* page is not present, which is not true in this case.
833	*/
834	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, pvFault, 1, uErr);
835	if (RT_SUCCESS(rc))
836	{
837	/*
838	* Page was successfully synced, return to guest.
839	* First invalidate the page as it might be in the TLB.
840	*/
841	# if PGM_SHW_TYPE == PGM_TYPE_EPT
842	HWACCMInvalidatePhysPage(pVM, (RTGCPHYS)pvFault);
843	# else
844	PGM_INVL_PG_ALL_VCPU(pVM, pvFault);
845	# endif
846	# ifdef VBOX_STRICT
847	RTGCPHYS GCPhys;
848	uint64_t fPageGst;
849	if (!HWACCMIsNestedPagingActive(pVM))
850	{
851	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, &GCPhys);
852	AssertMsg(RT_SUCCESS(rc) && (fPageGst & X86_PTE_RW), ("rc=%d fPageGst=%RX64\n"));
853	LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GCPhys, (uint64_t)fPageGst));
854	}
855	uint64_t fPageShw;
856	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
857	AssertMsg((RT_SUCCESS(rc) && (fPageShw & X86_PTE_RW)) \|\| pVM->cCpus > 1 /* new monitor can be installed/page table flushed between the trap exit and PGMTrap0eHandler */, ("rc=%Rrc fPageShw=%RX64\n", rc, fPageShw));
858	# endif /* VBOX_STRICT */
859	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
860	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.StatRZTrap0eTime2OutOfSyncHndObs; });
861	return VINF_SUCCESS;
862	}
863	}
864
865	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
866	# ifdef VBOX_STRICT
867	/*
868	* Check for VMM page flags vs. Guest page flags consistency.
869	* Currently only for debug purposes.
870	*/
871	if (RT_SUCCESS(rc))
872	{
873	/* Get guest page flags. */
874	uint64_t fPageGst;
875	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
876	if (RT_SUCCESS(rc))
877	{
878	uint64_t fPageShw;
879	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
880
881	/*
882	* Compare page flags.
883	* Note: we have AVL, A, D bits desynched.
884	*/
885	AssertMsg((fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)),
886	("Page flags mismatch! pvFault=%RGv uErr=%x GCPhys=%RGp fPageShw=%RX64 fPageGst=%RX64\n", pvFault, (uint32_t)uErr, GCPhys, fPageShw, fPageGst));
887	}
888	else
889	AssertMsgFailed(("PGMGstGetPage rc=%Rrc\n", rc));
890	}
891	else
892	AssertMsgFailed(("PGMGCGetPage rc=%Rrc\n", rc));
893	# endif /* VBOX_STRICT */
894	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
895	}
896	STAM_PROFILE_STOP(&pVCpu->pgm.s.StatRZTrap0eTimeOutOfSync, c);
897	# endif /* PGM_OUT_OF_SYNC_IN_GC */
898	}
899	else /* GCPhys == NIL_RTGCPHYS */
900	{
901	/*
902	* Page not present in Guest OS or invalid page table address.
903	* This is potential virtual page access handler food.
904	*
905	* For the present we'll say that our access handlers don't
906	* work for this case - we've already discarded the page table
907	* not present case which is identical to this.
908	*
909	* When we perchance find we need this, we will probably have AVL
910	* trees (offset based) to operate on and we can measure their speed
911	* agains mapping a page table and probably rearrange this handling
912	* a bit. (Like, searching virtual ranges before checking the
913	* physical address.)
914	*/
915	}
916	}
917	/* else: !present (guest) */
918
919
920	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
921	/*
922	* Conclusion, this is a guest trap.
923	*/
924	LogFlow(("PGM: Unhandled #PF -> route trap to recompiler!\n"));
925	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0eGuestPFUnh);
926	return VINF_EM_RAW_GUEST_TRAP;
927	# else
928	/* present, but not a monitored page; perhaps the guest is probing physical memory */
929	return VINF_EM_RAW_EMULATE_INSTR;
930	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
931
932
933	# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
934
935	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
936	return VERR_INTERNAL_ERROR;
937	# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
938	}
939	#endif /* !IN_RING3 */
940
941
942	/**
943	* Emulation of the invlpg instruction.
944	*
945	*
946	* @returns VBox status code.
947	*
948	* @param pVCpu The VMCPU handle.
949	* @param GCPtrPage Page to invalidate.
950	*
951	* @remark ASSUMES that the guest is updating before invalidating. This order
952	* isn't required by the CPU, so this is speculative and could cause
953	* trouble.
954	* @remark No TLB shootdown is done on any other VCPU as we assume that
955	* invlpg emulation is the only reason for calling this function.
956	* (The guest has to shoot down TLB entries on other CPUs itself)
957	* Currently true, but keep in mind!
958	*
959	* @todo Flush page or page directory only if necessary!
960	* @todo Add a #define for simply invalidating the page.
961	*/
962	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
963	{
964	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
965	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
966	&& PGM_SHW_TYPE != PGM_TYPE_EPT
967	int rc;
968	PVM pVM = pVCpu->CTX_SUFF(pVM);
969	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
970
971	Assert(PGMIsLockOwner(pVM));
972
973	LogFlow(("InvalidatePage %RGv\n", GCPtrPage));
974
975	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
976	if (pPool->cDirtyPages)
977	pgmPoolResetDirtyPages(pVM);
978	# endif
979
980	/*
981	* Get the shadow PD entry and skip out if this PD isn't present.
982	* (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
983	*/
984	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
985	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
986	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
987
988	/* Fetch the pgm pool shadow descriptor. */
989	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
990	Assert(pShwPde);
991
992	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
993	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT);
994	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(&pVCpu->pgm.s);
995
996	/* If the shadow PDPE isn't present, then skip the invalidate. */
997	if (!pPdptDst->a[iPdpt].n.u1Present)
998	{
999	Assert(!(pPdptDst->a[iPdpt].u & PGM_PLXFLAGS_MAPPING));
1000	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
1001	return VINF_SUCCESS;
1002	}
1003
1004	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1005	PPGMPOOLPAGE pShwPde = NULL;
1006	PX86PDPAE pPDDst;
1007
1008	/* Fetch the pgm pool shadow descriptor. */
1009	rc = pgmShwGetPaePoolPagePD(&pVCpu->pgm.s, GCPtrPage, &pShwPde);
1010	AssertRCSuccessReturn(rc, rc);
1011	Assert(pShwPde);
1012
1013	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGM(&pVM->pgm.s, pShwPde);
1014	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1015
1016	# else /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1017	/* PML4 */
1018	const unsigned iPml4 = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;
1019	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1020	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1021	PX86PDPAE pPDDst;
1022	PX86PDPT pPdptDst;
1023	PX86PML4E pPml4eDst;
1024	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eDst, &pPdptDst, &pPDDst);
1025	if (rc != VINF_SUCCESS)
1026	{
1027	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT \|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
1028	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
1029	if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
1030	PGM_INVL_VCPU_TLBS(pVCpu);
1031	return VINF_SUCCESS;
1032	}
1033	Assert(pPDDst);
1034
1035	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1036	PX86PDPE pPdpeDst = &pPdptDst->a[iPdpt];
1037
1038	if (!pPdpeDst->n.u1Present)
1039	{
1040	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
1041	if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
1042	PGM_INVL_VCPU_TLBS(pVCpu);
1043	return VINF_SUCCESS;
1044	}
1045
1046	# endif /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1047
1048	const SHWPDE PdeDst = *pPdeDst;
1049	if (!PdeDst.n.u1Present)
1050	{
1051	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
1052	return VINF_SUCCESS;
1053	}
1054
1055	# if defined(IN_RC)
1056	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1057	PGMDynLockHCPage(pVM, (uint8_t *)pPdeDst);
1058	# endif
1059
1060	/*
1061	* Get the guest PD entry and calc big page.
1062	*/
1063	# if PGM_GST_TYPE == PGM_TYPE_32BIT
1064	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
1065	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
1066	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
1067	# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1068	unsigned iPDSrc = 0;
1069	# if PGM_GST_TYPE == PGM_TYPE_PAE
1070	X86PDPE PdpeSrc;
1071	PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, GCPtrPage, &iPDSrc, &PdpeSrc);
1072	# else /* AMD64 */
1073	PX86PML4E pPml4eSrc;
1074	X86PDPE PdpeSrc;
1075	PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
1076	# endif
1077	GSTPDE PdeSrc;
1078
1079	if (pPDSrc)
1080	PdeSrc = pPDSrc->a[iPDSrc];
1081	else
1082	PdeSrc.u = 0;
1083	# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1084
1085	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1086	const bool fIsBigPage = PdeSrc.b.u1Size;
1087	# else
1088	const bool fIsBigPage = PdeSrc.b.u1Size && (CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
1089	# endif
1090
1091	# ifdef IN_RING3
1092	/*
1093	* If a CR3 Sync is pending we may ignore the invalidate page operation
1094	* depending on the kind of sync and if it's a global page or not.
1095	* This doesn't make sense in GC/R0 so we'll skip it entirely there.
1096	*/
1097	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
1098	if ( VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
1099	\|\| ( VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
1100	&& fIsBigPage
1101	&& PdeSrc.b.u1Global
1102	)
1103	)
1104	# else
1105	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
1106	# endif
1107	{
1108	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePageSkipped));
1109	return VINF_SUCCESS;
1110	}
1111	# endif /* IN_RING3 */
1112
1113	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1114	/* Fetch the pgm pool shadow descriptor. */
1115	PPGMPOOLPAGE pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
1116	Assert(pShwPdpt);
1117
1118	/* Fetch the pgm pool shadow descriptor. */
1119	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & SHW_PDPE_PG_MASK);
1120	Assert(pShwPde);
1121
1122	Assert(pPml4eDst->n.u1Present && (pPml4eDst->u & SHW_PDPT_MASK));
1123	RTGCPHYS GCPhysPdpt = pPml4eSrc->u & X86_PML4E_PG_MASK;
1124
1125	if ( !pPml4eSrc->n.u1Present
1126	\|\| pShwPdpt->GCPhys != GCPhysPdpt)
1127	{
1128	LogFlow(("InvalidatePage: Out-of-sync PML4E (P/GCPhys) at %RGv GCPhys=%RGp vs %RGp Pml4eSrc=%RX64 Pml4eDst=%RX64\n",
1129	GCPtrPage, pShwPdpt->GCPhys, GCPhysPdpt, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
1130	pgmPoolFreeByPage(pPool, pShwPdpt, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4);
1131	ASMAtomicWriteSize(pPml4eDst, 0);
1132	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNPs));
1133	PGM_INVL_VCPU_TLBS(pVCpu);
1134	return VINF_SUCCESS;
1135	}
1136	if ( pPml4eSrc->n.u1User != pPml4eDst->n.u1User
1137	\|\| (!pPml4eSrc->n.u1Write && pPml4eDst->n.u1Write))
1138	{
1139	/*
1140	* Mark not present so we can resync the PML4E when it's used.
1141	*/
1142	LogFlow(("InvalidatePage: Out-of-sync PML4E at %RGv Pml4eSrc=%RX64 Pml4eDst=%RX64\n",
1143	GCPtrPage, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
1144	pgmPoolFreeByPage(pPool, pShwPdpt, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4);
1145	ASMAtomicWriteSize(pPml4eDst, 0);
1146	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1147	PGM_INVL_VCPU_TLBS(pVCpu);
1148	}
1149	else if (!pPml4eSrc->n.u1Accessed)
1150	{
1151	/*
1152	* Mark not present so we can set the accessed bit.
1153	*/
1154	LogFlow(("InvalidatePage: Out-of-sync PML4E (A) at %RGv Pml4eSrc=%RX64 Pml4eDst=%RX64\n",
1155	GCPtrPage, (uint64_t)pPml4eSrc->u, (uint64_t)pPml4eDst->u));
1156	pgmPoolFreeByPage(pPool, pShwPdpt, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4);
1157	ASMAtomicWriteSize(pPml4eDst, 0);
1158	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNAs));
1159	PGM_INVL_VCPU_TLBS(pVCpu);
1160	}
1161
1162	/* Check if the PDPT entry has changed. */
1163	Assert(pPdpeDst->n.u1Present && pPdpeDst->u & SHW_PDPT_MASK);
1164	RTGCPHYS GCPhysPd = PdpeSrc.u & GST_PDPE_PG_MASK;
1165	if ( !PdpeSrc.n.u1Present
1166	\|\| pShwPde->GCPhys != GCPhysPd)
1167	{
1168	LogFlow(("InvalidatePage: Out-of-sync PDPE (P/GCPhys) at %RGv GCPhys=%RGp vs %RGp PdpeSrc=%RX64 PdpeDst=%RX64\n",
1169	GCPtrPage, pShwPde->GCPhys, GCPhysPd, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
1170	pgmPoolFreeByPage(pPool, pShwPde, pShwPdpt->idx, iPdpt);
1171	ASMAtomicWriteSize(pPdpeDst, 0);
1172	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNPs));
1173	PGM_INVL_VCPU_TLBS(pVCpu);
1174	return VINF_SUCCESS;
1175	}
1176	if ( PdpeSrc.lm.u1User != pPdpeDst->lm.u1User
1177	\|\| (!PdpeSrc.lm.u1Write && pPdpeDst->lm.u1Write))
1178	{
1179	/*
1180	* Mark not present so we can resync the PDPTE when it's used.
1181	*/
1182	LogFlow(("InvalidatePage: Out-of-sync PDPE at %RGv PdpeSrc=%RX64 PdpeDst=%RX64\n",
1183	GCPtrPage, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
1184	pgmPoolFreeByPage(pPool, pShwPde, pShwPdpt->idx, iPdpt);
1185	ASMAtomicWriteSize(pPdpeDst, 0);
1186	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1187	PGM_INVL_VCPU_TLBS(pVCpu);
1188	}
1189	else if (!PdpeSrc.lm.u1Accessed)
1190	{
1191	/*
1192	* Mark not present so we can set the accessed bit.
1193	*/
1194	LogFlow(("InvalidatePage: Out-of-sync PDPE (A) at %RGv PdpeSrc=%RX64 PdpeDst=%RX64\n",
1195	GCPtrPage, (uint64_t)PdpeSrc.u, (uint64_t)pPdpeDst->u));
1196	pgmPoolFreeByPage(pPool, pShwPde, pShwPdpt->idx, iPdpt);
1197	ASMAtomicWriteSize(pPdpeDst, 0);
1198	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNAs));
1199	PGM_INVL_VCPU_TLBS(pVCpu);
1200	}
1201	# endif /* PGM_GST_TYPE == PGM_TYPE_AMD64 */
1202
1203	/*
1204	* Deal with the Guest PDE.
1205	*/
1206	rc = VINF_SUCCESS;
1207	if (PdeSrc.n.u1Present)
1208	{
1209	# ifndef PGM_WITHOUT_MAPPING
1210	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
1211	{
1212	/*
1213	* Conflict - Let SyncPT deal with it to avoid duplicate code.
1214	*/
1215	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1216	Assert(PGMGetGuestMode(pVCpu) <= PGMMODE_PAE);
1217	pgmLock(pVM);
1218	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
1219	pgmUnlock(pVM);
1220	}
1221	else
1222	# endif /* !PGM_WITHOUT_MAPPING */
1223	if ( PdeSrc.n.u1User != PdeDst.n.u1User
1224	\|\| (!PdeSrc.n.u1Write && PdeDst.n.u1Write))
1225	{
1226	/*
1227	* Mark not present so we can resync the PDE when it's used.
1228	*/
1229	LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1230	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1231	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1232	ASMAtomicWriteSize(pPdeDst, 0);
1233	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1234	PGM_INVL_VCPU_TLBS(pVCpu);
1235	}
1236	else if (!PdeSrc.n.u1Accessed)
1237	{
1238	/*
1239	* Mark not present so we can set the accessed bit.
1240	*/
1241	LogFlow(("InvalidatePage: Out-of-sync (A) at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1242	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1243	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1244	ASMAtomicWriteSize(pPdeDst, 0);
1245	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNAs));
1246	PGM_INVL_VCPU_TLBS(pVCpu);
1247	}
1248	else if (!fIsBigPage)
1249	{
1250	/*
1251	* 4KB - page.
1252	*/
1253	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1254	RTGCPHYS GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1255	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1256	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1257	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE/2);
1258	# endif
1259	if (pShwPage->GCPhys == GCPhys)
1260	{
1261	# if 0 /* likely cause of a major performance regression; must be SyncPageWorkerTrackDeref then */
1262	const unsigned iPTEDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1263	PSHWPT pPT = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1264	if (pPT->a[iPTEDst].n.u1Present)
1265	{
1266	# ifdef PGMPOOL_WITH_USER_TRACKING
1267	/* This is very unlikely with caching/monitoring enabled. */
1268	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pShwPage, pPT->a[iPTEDst].u & SHW_PTE_PG_MASK);
1269	# endif
1270	ASMAtomicWriteSize(&pPT->a[iPTEDst], 0);
1271	}
1272	# else /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
1273	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
1274	if (RT_SUCCESS(rc))
1275	rc = VINF_SUCCESS;
1276	# endif
1277	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePage4KBPages));
1278	PGM_INVL_PG(pVCpu, GCPtrPage);
1279	}
1280	else
1281	{
1282	/*
1283	* The page table address changed.
1284	*/
1285	LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n",
1286	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
1287	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1288	ASMAtomicWriteSize(pPdeDst, 0);
1289	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1290	PGM_INVL_VCPU_TLBS(pVCpu);
1291	}
1292	}
1293	else
1294	{
1295	/*
1296	* 2/4MB - page.
1297	*/
1298	/* Before freeing the page, check if anything really changed. */
1299	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1300	RTGCPHYS GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
1301	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1302	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1303	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1304	# endif
1305	if ( pShwPage->GCPhys == GCPhys
1306	&& pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1307	{
1308	/* ASSUMES a the given bits are identical for 4M and normal PDEs */
1309	/** @todo PAT */
1310	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
1311	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT \| X86_PDE_PCD))
1312	&& ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1313	\|\| (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1314	{
1315	LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1316	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePage4MBPagesSkip));
1317	# if defined(IN_RC)
1318	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1319	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
1320	# endif
1321	return VINF_SUCCESS;
1322	}
1323	}
1324
1325	/*
1326	* Ok, the page table is present and it's been changed in the guest.
1327	* If we're in host context, we'll just mark it as not present taking the lazy approach.
1328	* We could do this for some flushes in GC too, but we need an algorithm for
1329	* deciding which 4MB pages containing code likely to be executed very soon.
1330	*/
1331	LogFlow(("InvalidatePage: Out-of-sync PD at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1332	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1333	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1334	ASMAtomicWriteSize(pPdeDst, 0);
1335	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePage4MBPages));
1336	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
1337	}
1338	}
1339	else
1340	{
1341	/*
1342	* Page directory is not present, mark shadow PDE not present.
1343	*/
1344	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
1345	{
1346	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1347	ASMAtomicWriteSize(pPdeDst, 0);
1348	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDNPs));
1349	PGM_INVL_PG(pVCpu, GCPtrPage);
1350	}
1351	else
1352	{
1353	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
1354	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,InvalidatePagePDMappings));
1355	}
1356	}
1357	# if defined(IN_RC)
1358	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1359	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
1360	# endif
1361	return rc;
1362
1363	#else /* guest real and protected mode */
1364	/* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1365	return VINF_SUCCESS;
1366	#endif
1367	}
1368
1369
1370	#ifdef PGMPOOL_WITH_USER_TRACKING
1371	/**
1372	* Update the tracking of shadowed pages.
1373	*
1374	* @param pVCpu The VMCPU handle.
1375	* @param pShwPage The shadow page.
1376	* @param HCPhys The physical page we is being dereferenced.
1377	*/
1378	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys)
1379	{
1380	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1381	PVM pVM = pVCpu->CTX_SUFF(pVM);
1382
1383	STAM_PROFILE_START(&pVM->pgm.s.StatTrackDeref, a);
1384	LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1385
1386	/** @todo If this turns out to be a bottle neck (very likely) two things can be done:
1387	* 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1388	* 2. write protect all shadowed pages. I.e. implement caching.
1389	*/
1390	/*
1391	* Find the guest address.
1392	*/
1393	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
1394	pRam;
1395	pRam = pRam->CTX_SUFF(pNext))
1396	{
1397	unsigned iPage = pRam->cb >> PAGE_SHIFT;
1398	while (iPage-- > 0)
1399	{
1400	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1401	{
1402	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1403	pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage]);
1404	pShwPage->cPresent--;
1405	pPool->cPresent--;
1406	STAM_PROFILE_STOP(&pVM->pgm.s.StatTrackDeref, a);
1407	return;
1408	}
1409	}
1410	}
1411
1412	for (;;)
1413	AssertReleaseMsgFailed(("HCPhys=%RHp wasn't found!\n", HCPhys));
1414	# else /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1415	pShwPage->cPresent--;
1416	pVM->pgm.s.CTX_SUFF(pPool)->cPresent--;
1417	# endif /* !PGMPOOL_WITH_GCPHYS_TRACKING */
1418	}
1419
1420
1421	/**
1422	* Update the tracking of shadowed pages.
1423	*
1424	* @param pVCpu The VMCPU handle.
1425	* @param pShwPage The shadow page.
1426	* @param u16 The top 16-bit of the pPage->HCPhys.
1427	* @param pPage Pointer to the guest page. this will be modified.
1428	* @param iPTDst The index into the shadow table.
1429	*/
1430	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, uint16_t u16, PPGMPAGE pPage, const unsigned iPTDst)
1431	{
1432	PVM pVM = pVCpu->CTX_SUFF(pVM);
1433	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1434	/*
1435	* Just deal with the simple first time here.
1436	*/
1437	if (!u16)
1438	{
1439	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackVirgin);
1440	u16 = PGMPOOL_TD_MAKE(1, pShwPage->idx);
1441	}
1442	else
1443	u16 = pgmPoolTrackPhysExtAddref(pVM, u16, pShwPage->idx);
1444
1445	/* write back */
1446	Log2(("SyncPageWorkerTrackAddRef: u16=%#x->%#x iPTDst=%#x\n", u16, PGM_PAGE_GET_TRACKING(pPage), iPTDst));
1447	PGM_PAGE_SET_TRACKING(pPage, u16);
1448
1449	# endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
1450
1451	/* update statistics. */
1452	pVM->pgm.s.CTX_SUFF(pPool)->cPresent++;
1453	pShwPage->cPresent++;
1454	if (pShwPage->iFirstPresent > iPTDst)
1455	pShwPage->iFirstPresent = iPTDst;
1456	}
1457	#endif /* PGMPOOL_WITH_USER_TRACKING */
1458
1459
1460	/**
1461	* Creates a 4K shadow page for a guest page.
1462	*
1463	* For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1464	* physical address. The PdeSrc argument only the flags are used. No page structured
1465	* will be mapped in this function.
1466	*
1467	* @param pVCpu The VMCPU handle.
1468	* @param pPteDst Destination page table entry.
1469	* @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1470	* Can safely assume that only the flags are being used.
1471	* @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1472	* @param pShwPage Pointer to the shadow page.
1473	* @param iPTDst The index into the shadow table.
1474	*
1475	* @remark Not used for 2/4MB pages!
1476	*/
1477	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1478	{
1479	if (PteSrc.n.u1Present)
1480	{
1481	PVM pVM = pVCpu->CTX_SUFF(pVM);
1482
1483	# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1484	&& PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1485	&& (PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64)
1486	if (pShwPage->fDirty)
1487	{
1488	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1489	PX86PTPAE pGstPT;
1490
1491	pGstPT = (PX86PTPAE)&pPool->aDirtyPages[pShwPage->idxDirty][0];
1492	pGstPT->a[iPTDst].u = PteSrc.u;
1493	}
1494	# endif
1495	/*
1496	* Find the ram range.
1497	*/
1498	PPGMPAGE pPage;
1499	int rc = pgmPhysGetPageEx(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK, &pPage);
1500	if (RT_SUCCESS(rc))
1501	{
1502	#ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1503	/* Try make the page writable if necessary. */
1504	if ( PteSrc.n.u1Write
1505	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1506	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1507	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1508	# endif
1509	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
1510	{
1511	rc = pgmPhysPageMakeWritableUnlocked(pVM, pPage, PteSrc.u & GST_PTE_PG_MASK);
1512	AssertRC(rc);
1513	}
1514	#endif
1515
1516	/** @todo investiage PWT, PCD and PAT. */
1517	/*
1518	* Make page table entry.
1519	*/
1520	SHWPTE PteDst;
1521	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1522	{
1523	/** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No. */
1524	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1525	{
1526	#if PGM_SHW_TYPE == PGM_TYPE_EPT
1527	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage);
1528	PteDst.n.u1Present = 1;
1529	PteDst.n.u1Execute = 1;
1530	PteDst.n.u1IgnorePAT = 1;
1531	PteDst.n.u3EMT = VMX_EPT_MEMTYPE_WB;
1532	/* PteDst.n.u1Write = 0 && PteDst.n.u1Size = 0 */
1533	#else
1534	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1535	\| PGM_PAGE_GET_HCPHYS(pPage);
1536	#endif
1537	}
1538	else
1539	{
1540	LogFlow(("SyncPageWorker: monitored page (%RHp) -> mark not present\n", PGM_PAGE_GET_HCPHYS(pPage)));
1541	PteDst.u = 0;
1542	}
1543	/** @todo count these two kinds. */
1544	}
1545	else
1546	{
1547	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1548	/*
1549	* If the page or page directory entry is not marked accessed,
1550	* we mark the page not present.
1551	*/
1552	if (!PteSrc.n.u1Accessed \|\| !PdeSrc.n.u1Accessed)
1553	{
1554	LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1555	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,AccessedPage));
1556	PteDst.u = 0;
1557	}
1558	else
1559	/*
1560	* If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1561	* when the page is modified.
1562	*/
1563	if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
1564	{
1565	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPage));
1566	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT \| X86_PTE_RW))
1567	\| PGM_PAGE_GET_HCPHYS(pPage)
1568	\| PGM_PTFLAGS_TRACK_DIRTY;
1569	}
1570	else
1571	#endif
1572	{
1573	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageSkipped));
1574	#if PGM_SHW_TYPE == PGM_TYPE_EPT
1575	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage);
1576	PteDst.n.u1Present = 1;
1577	PteDst.n.u1Write = 1;
1578	PteDst.n.u1Execute = 1;
1579	PteDst.n.u1IgnorePAT = 1;
1580	PteDst.n.u3EMT = VMX_EPT_MEMTYPE_WB;
1581	/* PteDst.n.u1Size = 0 */
1582	#else
1583	PteDst.u = (PteSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1584	\| PGM_PAGE_GET_HCPHYS(pPage);
1585	#endif
1586	}
1587	}
1588
1589	/*
1590	* Make sure only allocated pages are mapped writable.
1591	*/
1592	if ( PteDst.n.u1Write
1593	&& PteDst.n.u1Present
1594	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1595	{
1596	PteDst.n.u1Write = 0; /** @todo this isn't quite working yet. */
1597	Log3(("SyncPageWorker: write-protecting %RGp pPage=%R[pgmpage]at iPTDst=%d\n", (RTGCPHYS)(PteSrc.u & X86_PTE_PAE_PG_MASK), pPage, iPTDst));
1598	}
1599
1600	#ifdef PGMPOOL_WITH_USER_TRACKING
1601	/*
1602	* Keep user track up to date.
1603	*/
1604	if (PteDst.n.u1Present)
1605	{
1606	if (!pPteDst->n.u1Present)
1607	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1608	else if ((pPteDst->u & SHW_PTE_PG_MASK) != (PteDst.u & SHW_PTE_PG_MASK))
1609	{
1610	Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", (uint64_t)pPteDst->u, (uint64_t)PteDst.u));
1611	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1612	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1613	}
1614	}
1615	else if (pPteDst->n.u1Present)
1616	{
1617	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1618	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1619	}
1620	#endif /* PGMPOOL_WITH_USER_TRACKING */
1621
1622	/*
1623	* Update statistics and commit the entry.
1624	*/
1625	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1626	if (!PteSrc.n.u1Global)
1627	pShwPage->fSeenNonGlobal = true;
1628	#endif
1629	ASMAtomicWriteSize(pPteDst, PteDst.u);
1630	}
1631	/* else MMIO or invalid page, we must handle them manually in the #PF handler. */
1632	/** @todo count these. */
1633	}
1634	else
1635	{
1636	/*
1637	* Page not-present.
1638	*/
1639	Log2(("SyncPageWorker: page not present in Pte\n"));
1640	#ifdef PGMPOOL_WITH_USER_TRACKING
1641	/* Keep user track up to date. */
1642	if (pPteDst->n.u1Present)
1643	{
1644	Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", (uint64_t)pPteDst->u));
1645	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, pPteDst->u & SHW_PTE_PG_MASK);
1646	}
1647	#endif /* PGMPOOL_WITH_USER_TRACKING */
1648	ASMAtomicWriteSize(pPteDst, 0);
1649	/** @todo count these. */
1650	}
1651	}
1652
1653
1654	/**
1655	* Syncs a guest OS page.
1656	*
1657	* There are no conflicts at this point, neither is there any need for
1658	* page table allocations.
1659	*
1660	* @returns VBox status code.
1661	* @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1662	* @param pVCpu The VMCPU handle.
1663	* @param PdeSrc Page directory entry of the guest.
1664	* @param GCPtrPage Guest context page address.
1665	* @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1666	* @param uErr Fault error (X86_TRAP_PF_*).
1667	*/
1668	PGM_BTH_DECL(int, SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr)
1669	{
1670	PVM pVM = pVCpu->CTX_SUFF(pVM);
1671	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1672	LogFlow(("SyncPage: GCPtrPage=%RGv cPages=%u uErr=%#x\n", GCPtrPage, cPages, uErr));
1673
1674	Assert(PGMIsLockOwner(pVM));
1675
1676	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
1677	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
1678	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
1679	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
1680	&& PGM_SHW_TYPE != PGM_TYPE_EPT
1681
1682	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
1683	bool fNoExecuteBitValid = !!(CPUMGetGuestEFER(pVCpu) & MSR_K6_EFER_NXE);
1684	# endif
1685
1686	/*
1687	* Assert preconditions.
1688	*/
1689	Assert(PdeSrc.n.u1Present);
1690	Assert(cPages);
1691	# if 0 /* rarely useful; leave for debugging. */
1692	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
1693	# endif
1694
1695	/*
1696	* Get the shadow PDE, find the shadow page table in the pool.
1697	*/
1698	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1699	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1700	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
1701
1702	/* Fetch the pgm pool shadow descriptor. */
1703	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1704	Assert(pShwPde);
1705
1706	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1707	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1708	PPGMPOOLPAGE pShwPde = NULL;
1709	PX86PDPAE pPDDst;
1710
1711	/* Fetch the pgm pool shadow descriptor. */
1712	int rc = pgmShwGetPaePoolPagePD(&pVCpu->pgm.s, GCPtrPage, &pShwPde);
1713	AssertRCSuccessReturn(rc, rc);
1714	Assert(pShwPde);
1715
1716	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGM(&pVM->pgm.s, pShwPde);
1717	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1718
1719	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1720	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1721	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1722	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
1723	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
1724
1725	int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
1726	AssertRCSuccessReturn(rc, rc);
1727	Assert(pPDDst && pPdptDst);
1728	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1729	# endif
1730	SHWPDE PdeDst = *pPdeDst;
1731	if (!PdeDst.n.u1Present)
1732	{
1733	AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE p=%p/%RX64\n", pPdeDst, (uint64_t)PdeDst.u));
1734	Log(("CPU%d: SyncPage: Pde at %RGv changed behind our back!\n", GCPtrPage));
1735	return VINF_SUCCESS; /* force the instruction to be executed again. */
1736	}
1737
1738	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1739	Assert(pShwPage);
1740
1741	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1742	/* Fetch the pgm pool shadow descriptor. */
1743	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
1744	Assert(pShwPde);
1745	# endif
1746
1747	# if defined(IN_RC)
1748	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1749	PGMDynLockHCPage(pVM, (uint8_t *)pPdeDst);
1750	# endif
1751
1752	/*
1753	* Check that the page is present and that the shadow PDE isn't out of sync.
1754	*/
1755	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1756	const bool fBigPage = PdeSrc.b.u1Size;
1757	# else
1758	const bool fBigPage = PdeSrc.b.u1Size && (CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
1759	# endif
1760	RTGCPHYS GCPhys;
1761	if (!fBigPage)
1762	{
1763	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
1764	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1765	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1766	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE/2);
1767	# endif
1768	}
1769	else
1770	{
1771	GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
1772	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1773	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1774	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
1775	# endif
1776	}
1777	if ( pShwPage->GCPhys == GCPhys
1778	&& PdeSrc.n.u1Present
1779	&& (PdeSrc.n.u1User == PdeDst.n.u1User)
1780	&& (PdeSrc.n.u1Write == PdeDst.n.u1Write \|\| !PdeDst.n.u1Write)
1781	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
1782	&& (!fNoExecuteBitValid \|\| PdeSrc.n.u1NoExecute == PdeDst.n.u1NoExecute)
1783	# endif
1784	)
1785	{
1786	/*
1787	* Check that the PDE is marked accessed already.
1788	* Since we set the accessed bit before getting here on a #PF, this
1789	* check is only meant for dealing with non-#PF'ing paths.
1790	*/
1791	if (PdeSrc.n.u1Accessed)
1792	{
1793	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1794	if (!fBigPage)
1795	{
1796	/*
1797	* 4KB Page - Map the guest page table.
1798	*/
1799	PGSTPT pPTSrc;
1800	int rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
1801	if (RT_SUCCESS(rc))
1802	{
1803	# ifdef PGM_SYNC_N_PAGES
1804	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
1805	if ( cPages > 1
1806	&& !(uErr & X86_TRAP_PF_P)
1807	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
1808	{
1809	/*
1810	* This code path is currently only taken when the caller is PGMTrap0eHandler
1811	* for non-present pages!
1812	*
1813	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1814	* deal with locality.
1815	*/
1816	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1817	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1818	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1819	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1820	# else
1821	const unsigned offPTSrc = 0;
1822	# endif
1823	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
1824	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1825	iPTDst = 0;
1826	else
1827	iPTDst -= PGM_SYNC_NR_PAGES / 2;
1828	for (; iPTDst < iPTDstEnd; iPTDst++)
1829	{
1830	if (!pPTDst->a[iPTDst].n.u1Present)
1831	{
1832	GSTPTE PteSrc = pPTSrc->a[offPTSrc + iPTDst];
1833	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(GST_PT_MASK << GST_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
1834	NOREF(GCPtrCurPage);
1835	#ifndef IN_RING0
1836	/*
1837	* Assuming kernel code will be marked as supervisor - and not as user level
1838	* and executed using a conforming code selector - And marked as readonly.
1839	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
1840	*/
1841	PPGMPAGE pPage;
1842	if ( ((PdeSrc.u & PteSrc.u) & (X86_PTE_RW \| X86_PTE_US))
1843	\|\| iPTDst == ((GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK) /* always sync GCPtrPage */
1844	\|\| !CSAMDoesPageNeedScanning(pVM, (RTRCPTR)GCPtrCurPage)
1845	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
1846	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1847	)
1848	#endif /* else: CSAM not active */
1849	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1850	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1851	GCPtrCurPage, PteSrc.n.u1Present,
1852	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1853	PteSrc.n.u1User & PdeSrc.n.u1User,
1854	(uint64_t)PteSrc.u,
1855	(uint64_t)pPTDst->a[iPTDst].u,
1856	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1857	}
1858	}
1859	}
1860	else
1861	# endif /* PGM_SYNC_N_PAGES */
1862	{
1863	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1864	GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1865	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1866	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1867	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
1868	GCPtrPage, PteSrc.n.u1Present,
1869	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1870	PteSrc.n.u1User & PdeSrc.n.u1User,
1871	(uint64_t)PteSrc.u,
1872	(uint64_t)pPTDst->a[iPTDst].u,
1873	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1874	}
1875	}
1876	else /* MMIO or invalid page: emulated in #PF handler. */
1877	{
1878	LogFlow(("PGM_GCPHYS_2_PTR %RGp failed with %Rrc\n", GCPhys, rc));
1879	Assert(!pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK].n.u1Present);
1880	}
1881	}
1882	else
1883	{
1884	/*
1885	* 4/2MB page - lazy syncing shadow 4K pages.
1886	* (There are many causes of getting here, it's no longer only CSAM.)
1887	*/
1888	/* Calculate the GC physical address of this 4KB shadow page. */
1889	RTGCPHYS GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc) \| (GCPtrPage & GST_BIG_PAGE_OFFSET_MASK);
1890	/* Find ram range. */
1891	PPGMPAGE pPage;
1892	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
1893	if (RT_SUCCESS(rc))
1894	{
1895	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1896	/* Try make the page writable if necessary. */
1897	if ( PdeSrc.n.u1Write
1898	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1899	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1900	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1901	# endif
1902	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
1903	{
1904	rc = pgmPhysPageMakeWritableUnlocked(pVM, pPage, GCPhys);
1905	AssertRC(rc);
1906	}
1907	# endif
1908
1909	/*
1910	* Make shadow PTE entry.
1911	*/
1912	SHWPTE PteDst;
1913	PteDst.u = (PdeSrc.u & ~(X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1914	\| PGM_PAGE_GET_HCPHYS(pPage);
1915	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1916	{
1917	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1918	PteDst.n.u1Write = 0;
1919	else
1920	PteDst.u = 0;
1921	}
1922	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1923	# ifdef PGMPOOL_WITH_USER_TRACKING
1924	if (PteDst.n.u1Present && !pPTDst->a[iPTDst].n.u1Present)
1925	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1926	# endif
1927	/* Make sure only allocated pages are mapped writable. */
1928	if ( PteDst.n.u1Write
1929	&& PteDst.n.u1Present
1930	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1931	{
1932	PteDst.n.u1Write = 0; /** @todo this isn't quite working yet... */
1933	Log3(("SyncPage: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, GCPtrPage));
1934	}
1935
1936	ASMAtomicWriteSize(&pPTDst->a[iPTDst], PteDst.u);
1937
1938	/*
1939	* If the page is not flagged as dirty and is writable, then make it read-only
1940	* at PD level, so we can set the dirty bit when the page is modified.
1941	*
1942	* ASSUMES that page access handlers are implemented on page table entry level.
1943	* Thus we will first catch the dirty access and set PDE.D and restart. If
1944	* there is an access handler, we'll trap again and let it work on the problem.
1945	*/
1946	/** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
1947	* As for invlpg, it simply frees the whole shadow PT.
1948	* ...It's possibly because the guest clears it and the guest doesn't really tell us... */
1949	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
1950	{
1951	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageBig));
1952	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
1953	PdeDst.n.u1Write = 0;
1954	}
1955	else
1956	{
1957	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
1958	PdeDst.n.u1Write = PdeSrc.n.u1Write;
1959	}
1960	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
1961	Log2(("SyncPage: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%RGp%s\n",
1962	GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
1963	PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1964	}
1965	else
1966	LogFlow(("PGM_GCPHYS_2_PTR %RGp (big) failed with %Rrc\n", GCPhys, rc));
1967	}
1968	# if defined(IN_RC)
1969	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1970	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
1971	# endif
1972	return VINF_SUCCESS;
1973	}
1974	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDNAs));
1975	}
1976	else
1977	{
1978	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDOutOfSync));
1979	Log2(("SyncPage: Out-Of-Sync PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
1980	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
1981	}
1982
1983	/*
1984	* Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
1985	* Yea, I'm lazy.
1986	*/
1987	pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst);
1988	ASMAtomicWriteSize(pPdeDst, 0);
1989
1990	# if defined(IN_RC)
1991	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
1992	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
1993	# endif
1994	PGM_INVL_VCPU_TLBS(pVCpu);
1995	return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
1996
1997	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
1998	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
1999	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2000	&& !defined(IN_RC)
2001
2002	# ifdef PGM_SYNC_N_PAGES
2003	/*
2004	* Get the shadow PDE, find the shadow page table in the pool.
2005	*/
2006	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2007	X86PDE PdeDst = pgmShwGet32BitPDE(&pVCpu->pgm.s, GCPtrPage);
2008
2009	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2010	X86PDEPAE PdeDst = pgmShwGetPaePDE(&pVCpu->pgm.s, GCPtrPage);
2011
2012	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2013	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2014	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; NOREF(iPdpt);
2015	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2016	X86PDEPAE PdeDst;
2017	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2018
2019	int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2020	AssertRCSuccessReturn(rc, rc);
2021	Assert(pPDDst && pPdptDst);
2022	PdeDst = pPDDst->a[iPDDst];
2023	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2024	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2025	PEPTPD pPDDst;
2026	EPTPDE PdeDst;
2027
2028	int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, NULL, &pPDDst);
2029	if (rc != VINF_SUCCESS)
2030	{
2031	AssertRC(rc);
2032	return rc;
2033	}
2034	Assert(pPDDst);
2035	PdeDst = pPDDst->a[iPDDst];
2036	# endif
2037	AssertMsg(PdeDst.n.u1Present, ("%#llx\n", (uint64_t)PdeDst.u));
2038	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2039	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2040
2041	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
2042	if ( cPages > 1
2043	&& !(uErr & X86_TRAP_PF_P)
2044	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2045	{
2046	/*
2047	* This code path is currently only taken when the caller is PGMTrap0eHandler
2048	* for non-present pages!
2049	*
2050	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2051	* deal with locality.
2052	*/
2053	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2054	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2055	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2056	iPTDst = 0;
2057	else
2058	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2059	for (; iPTDst < iPTDstEnd; iPTDst++)
2060	{
2061	if (!pPTDst->a[iPTDst].n.u1Present)
2062	{
2063	GSTPTE PteSrc;
2064
2065	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT);
2066
2067	/* Fake the page table entry */
2068	PteSrc.u = GCPtrCurPage;
2069	PteSrc.n.u1Present = 1;
2070	PteSrc.n.u1Dirty = 1;
2071	PteSrc.n.u1Accessed = 1;
2072	PteSrc.n.u1Write = 1;
2073	PteSrc.n.u1User = 1;
2074
2075	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2076
2077	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
2078	GCPtrCurPage, PteSrc.n.u1Present,
2079	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2080	PteSrc.n.u1User & PdeSrc.n.u1User,
2081	(uint64_t)PteSrc.u,
2082	(uint64_t)pPTDst->a[iPTDst].u,
2083	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2084
2085	if (RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)))
2086	break;
2087	}
2088	else
2089	Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n", (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT), iPTDst, pPTDst->a[iPTDst].u));
2090	}
2091	}
2092	else
2093	# endif /* PGM_SYNC_N_PAGES */
2094	{
2095	GSTPTE PteSrc;
2096	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2097	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT);
2098
2099	/* Fake the page table entry */
2100	PteSrc.u = GCPtrCurPage;
2101	PteSrc.n.u1Present = 1;
2102	PteSrc.n.u1Dirty = 1;
2103	PteSrc.n.u1Accessed = 1;
2104	PteSrc.n.u1Write = 1;
2105	PteSrc.n.u1User = 1;
2106	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2107
2108	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}PteDst=%08llx%s\n",
2109	GCPtrPage, PteSrc.n.u1Present,
2110	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2111	PteSrc.n.u1User & PdeSrc.n.u1User,
2112	(uint64_t)PteSrc.u,
2113	(uint64_t)pPTDst->a[iPTDst].u,
2114	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2115	}
2116	return VINF_SUCCESS;
2117
2118	#else
2119	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2120	return VERR_INTERNAL_ERROR;
2121	#endif
2122	}
2123
2124
2125	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
2126	/**
2127	* Investigate page fault and handle write protection page faults caused by
2128	* dirty bit tracking.
2129	*
2130	* @returns VBox status code.
2131	* @param pVCpu The VMCPU handle.
2132	* @param uErr Page fault error code.
2133	* @param pPdeDst Shadow page directory entry.
2134	* @param pPdeSrc Guest page directory entry.
2135	* @param GCPtrPage Guest context page address.
2136	*/
2137	PGM_BTH_DECL(int, CheckPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, PGSTPDE pPdeSrc, RTGCPTR GCPtrPage)
2138	{
2139	bool fWriteProtect = !!(CPUMGetGuestCR0(pVCpu) & X86_CR0_WP);
2140	bool fUserLevelFault = !!(uErr & X86_TRAP_PF_US);
2141	bool fWriteFault = !!(uErr & X86_TRAP_PF_RW);
2142	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2143	bool fBigPagesSupported = true;
2144	# else
2145	bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
2146	# endif
2147	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2148	bool fNoExecuteBitValid = !!(CPUMGetGuestEFER(pVCpu) & MSR_K6_EFER_NXE);
2149	# endif
2150	unsigned uPageFaultLevel;
2151	int rc;
2152	PVM pVM = pVCpu->CTX_SUFF(pVM);
2153	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2154
2155	Assert(PGMIsLockOwner(pVM));
2156
2157	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2158	LogFlow(("CheckPageFault: GCPtrPage=%RGv uErr=%#x PdeSrc=%08x\n", GCPtrPage, uErr, pPdeSrc->u));
2159
2160	# if PGM_GST_TYPE == PGM_TYPE_PAE \
2161	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
2162
2163	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2164	PX86PML4E pPml4eSrc;
2165	PX86PDPE pPdpeSrc;
2166
2167	pPdpeSrc = pgmGstGetLongModePDPTPtr(&pVCpu->pgm.s, GCPtrPage, &pPml4eSrc);
2168	Assert(pPml4eSrc);
2169
2170	/*
2171	* Real page fault? (PML4E level)
2172	*/
2173	if ( (uErr & X86_TRAP_PF_RSVD)
2174	\|\| !pPml4eSrc->n.u1Present
2175	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPml4eSrc->n.u1NoExecute)
2176	\|\| (fWriteFault && !pPml4eSrc->n.u1Write && (fUserLevelFault \|\| fWriteProtect))
2177	\|\| (fUserLevelFault && !pPml4eSrc->n.u1User)
2178	)
2179	{
2180	uPageFaultLevel = 0;
2181	goto l_UpperLevelPageFault;
2182	}
2183	Assert(pPdpeSrc);
2184
2185	# else /* PAE */
2186	PX86PDPE pPdpeSrc = pgmGstGetPaePDPEPtr(&pVCpu->pgm.s, GCPtrPage);
2187	# endif /* PAE */
2188
2189	/*
2190	* Real page fault? (PDPE level)
2191	*/
2192	if ( (uErr & X86_TRAP_PF_RSVD)
2193	\|\| !pPdpeSrc->n.u1Present
2194	# if PGM_GST_TYPE == PGM_TYPE_AMD64 /* NX, r/w, u/s bits in the PDPE are long mode only */
2195	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPdpeSrc->lm.u1NoExecute)
2196	\|\| (fWriteFault && !pPdpeSrc->lm.u1Write && (fUserLevelFault \|\| fWriteProtect))
2197	\|\| (fUserLevelFault && !pPdpeSrc->lm.u1User)
2198	# endif
2199	)
2200	{
2201	uPageFaultLevel = 1;
2202	goto l_UpperLevelPageFault;
2203	}
2204	# endif
2205
2206	/*
2207	* Real page fault? (PDE level)
2208	*/
2209	if ( (uErr & X86_TRAP_PF_RSVD)
2210	\|\| !pPdeSrc->n.u1Present
2211	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2212	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && pPdeSrc->n.u1NoExecute)
2213	# endif
2214	\|\| (fWriteFault && !pPdeSrc->n.u1Write && (fUserLevelFault \|\| fWriteProtect))
2215	\|\| (fUserLevelFault && !pPdeSrc->n.u1User) )
2216	{
2217	uPageFaultLevel = 2;
2218	goto l_UpperLevelPageFault;
2219	}
2220
2221	/*
2222	* First check the easy case where the page directory has been marked read-only to track
2223	* the dirty bit of an emulated BIG page
2224	*/
2225	if (pPdeSrc->b.u1Size && fBigPagesSupported)
2226	{
2227	/* Mark guest page directory as accessed */
2228	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2229	pPml4eSrc->n.u1Accessed = 1;
2230	pPdpeSrc->lm.u1Accessed = 1;
2231	# endif
2232	pPdeSrc->b.u1Accessed = 1;
2233
2234	/*
2235	* Only write protection page faults are relevant here.
2236	*/
2237	if (fWriteFault)
2238	{
2239	/* Mark guest page directory as dirty (BIG page only). */
2240	pPdeSrc->b.u1Dirty = 1;
2241
2242	if (pPdeDst->n.u1Present)
2243	{
2244	if (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY)
2245	{
2246	SHWPDE PdeDst = *pPdeDst;
2247
2248	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageTrap));
2249	Assert(pPdeSrc->b.u1Write);
2250
2251	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
2252	* fault again and take this path to only invalidate the entry.
2253	*/
2254	PdeDst.n.u1Write = 1;
2255	PdeDst.n.u1Accessed = 1;
2256	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2257	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2258	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
2259	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2260	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2261	}
2262	# ifdef IN_RING0
2263	else
2264	/* Check for stale TLB entry; only applies to the SMP guest case. */
2265	if ( pVM->cCpus > 1
2266	&& pPdeDst->n.u1Write
2267	&& pPdeDst->n.u1Accessed)
2268	{
2269	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2270	if (pShwPage)
2271	{
2272	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2273	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2274	if ( pPteDst->n.u1Present
2275	&& pPteDst->n.u1Write)
2276	{
2277	/* Stale TLB entry. */
2278	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageStale));
2279	PGM_INVL_PG(pVCpu, GCPtrPage);
2280
2281	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2282	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2283	}
2284	}
2285	}
2286	# endif /* IN_RING0 */
2287	}
2288	}
2289	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2290	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2291	}
2292	/* else: 4KB page table */
2293
2294	/*
2295	* Map the guest page table.
2296	*/
2297	PGSTPT pPTSrc;
2298	rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
2299	if (RT_SUCCESS(rc))
2300	{
2301	/*
2302	* Real page fault?
2303	*/
2304	PGSTPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2305	const GSTPTE PteSrc = *pPteSrc;
2306	if ( !PteSrc.n.u1Present
2307	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2308	\|\| (fNoExecuteBitValid && (uErr & X86_TRAP_PF_ID) && PteSrc.n.u1NoExecute)
2309	# endif
2310	\|\| (fWriteFault && !PteSrc.n.u1Write && (fUserLevelFault \|\| fWriteProtect))
2311	\|\| (fUserLevelFault && !PteSrc.n.u1User)
2312	)
2313	{
2314	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyTrackRealPF));
2315	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2316	LogFlow(("CheckPageFault: real page fault at %RGv PteSrc.u=%08x (2)\n", GCPtrPage, PteSrc.u));
2317
2318	/* Check the present bit as the shadow tables can cause different error codes by being out of sync.
2319	* See the 2nd case above as well.
2320	*/
2321	if (pPdeSrc->n.u1Present && pPteSrc->n.u1Present)
2322	TRPMSetErrorCode(pVCpu, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
2323
2324	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2325	return VINF_EM_RAW_GUEST_TRAP;
2326	}
2327	LogFlow(("CheckPageFault: page fault at %RGv PteSrc.u=%08x\n", GCPtrPage, PteSrc.u));
2328
2329	/*
2330	* Set the accessed bits in the page directory and the page table.
2331	*/
2332	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2333	pPml4eSrc->n.u1Accessed = 1;
2334	pPdpeSrc->lm.u1Accessed = 1;
2335	# endif
2336	pPdeSrc->n.u1Accessed = 1;
2337	pPteSrc->n.u1Accessed = 1;
2338
2339	/*
2340	* Only write protection page faults are relevant here.
2341	*/
2342	if (fWriteFault)
2343	{
2344	/* Write access, so mark guest entry as dirty. */
2345	# ifdef VBOX_WITH_STATISTICS
2346	if (!pPteSrc->n.u1Dirty)
2347	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtiedPage));
2348	else
2349	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageAlreadyDirty));
2350	# endif
2351
2352	pPteSrc->n.u1Dirty = 1;
2353
2354	if (pPdeDst->n.u1Present)
2355	{
2356	#ifndef IN_RING0
2357	/* Bail out here as pgmPoolGetPageByHCPhys will return NULL and we'll crash below.
2358	* Our individual shadow handlers will provide more information and force a fatal exit.
2359	*/
2360	if (MMHyperIsInsideArea(pVM, (RTGCPTR)GCPtrPage))
2361	{
2362	LogRel(("CheckPageFault: write to hypervisor region %RGv\n", GCPtrPage));
2363	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2364	return VINF_SUCCESS;
2365	}
2366	#endif
2367	/*
2368	* Map shadow page table.
2369	*/
2370	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2371	if (pShwPage)
2372	{
2373	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2374	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2375	if (pPteDst->n.u1Present) /** @todo Optimize accessed bit emulation? */
2376	{
2377	if (pPteDst->u & PGM_PTFLAGS_TRACK_DIRTY)
2378	{
2379	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, pPteSrc->u & GST_PTE_PG_MASK);
2380	SHWPTE PteDst = *pPteDst;
2381
2382	LogFlow(("DIRTY page trap addr=%RGv\n", GCPtrPage));
2383	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageTrap));
2384
2385	Assert(pPteSrc->n.u1Write);
2386
2387	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
2388	* fault again and take this path to only invalidate the entry.
2389	*/
2390	if (RT_LIKELY(pPage))
2391	{
2392	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2393	/* Assuming write handlers here as the PTE is present (otherwise we wouldn't be here). */
2394	PteDst.n.u1Write = 0;
2395	else
2396	{
2397	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
2398	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
2399	{
2400	rc = pgmPhysPageMakeWritableUnlocked(pVM, pPage, pPteSrc->u & GST_PTE_PG_MASK);
2401	AssertRC(rc);
2402	}
2403	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED)
2404	PteDst.n.u1Write = 1;
2405	else
2406	PteDst.n.u1Write = 0;
2407	}
2408	}
2409	else
2410	PteDst.n.u1Write = 1;
2411
2412	PteDst.n.u1Dirty = 1;
2413	PteDst.n.u1Accessed = 1;
2414	PteDst.au32[0] &= ~PGM_PTFLAGS_TRACK_DIRTY;
2415	ASMAtomicWriteSize(pPteDst, PteDst.u);
2416	PGM_INVL_PG(pVCpu, GCPtrPage);
2417
2418	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2419	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2420	}
2421	# ifdef IN_RING0
2422	else
2423	/* Check for stale TLB entry; only applies to the SMP guest case. */
2424	if ( pVM->cCpus > 1
2425	&& pPteDst->n.u1Write == 1
2426	&& pPteDst->n.u1Accessed == 1)
2427	{
2428	/* Stale TLB entry. */
2429	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageStale));
2430	PGM_INVL_PG(pVCpu, GCPtrPage);
2431
2432	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2433	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2434	}
2435	# endif
2436	}
2437	}
2438	else
2439	AssertMsgFailed(("pgmPoolGetPageByHCPhys %RGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
2440	}
2441	}
2442	/** @todo Optimize accessed bit emulation? */
2443	# ifdef VBOX_STRICT
2444	/*
2445	* Sanity check.
2446	*/
2447	else if ( !pPteSrc->n.u1Dirty
2448	&& (pPdeSrc->n.u1Write & pPteSrc->n.u1Write)
2449	&& pPdeDst->n.u1Present)
2450	{
2451	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2452	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2453	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2454	if ( pPteDst->n.u1Present
2455	&& pPteDst->n.u1Write)
2456	LogFlow(("Writable present page %RGv not marked for dirty bit tracking!!!\n", GCPtrPage));
2457	}
2458	# endif /* VBOX_STRICT */
2459	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2460	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2461	}
2462	AssertRC(rc);
2463	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2464	return rc;
2465
2466
2467	l_UpperLevelPageFault:
2468	/*
2469	* Pagefault detected while checking the PML4E, PDPE or PDE.
2470	* Single exit handler to get rid of duplicate code paths.
2471	*/
2472	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyTrackRealPF));
2473	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyBitTracking), a);
2474	Log(("CheckPageFault: real page fault at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2475
2476	if (
2477	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2478	pPml4eSrc->n.u1Present &&
2479	# endif
2480	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
2481	pPdpeSrc->n.u1Present &&
2482	# endif
2483	pPdeSrc->n.u1Present)
2484	{
2485	/* Check the present bit as the shadow tables can cause different error codes by being out of sync. */
2486	if (pPdeSrc->b.u1Size && fBigPagesSupported)
2487	{
2488	TRPMSetErrorCode(pVCpu, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
2489	}
2490	else
2491	{
2492	/*
2493	* Map the guest page table.
2494	*/
2495	PGSTPT pPTSrc;
2496	rc = PGM_GCPHYS_2_PTR(pVM, pPdeSrc->u & GST_PDE_PG_MASK, &pPTSrc);
2497	if (RT_SUCCESS(rc))
2498	{
2499	PGSTPTE pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2500	const GSTPTE PteSrc = *pPteSrc;
2501	if (pPteSrc->n.u1Present)
2502	TRPMSetErrorCode(pVCpu, uErr \| X86_TRAP_PF_P); /* page-level protection violation */
2503	}
2504	AssertRC(rc);
2505	}
2506	}
2507	return VINF_EM_RAW_GUEST_TRAP;
2508	}
2509	#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
2510
2511
2512	/**
2513	* Sync a shadow page table.
2514	*
2515	* The shadow page table is not present. This includes the case where
2516	* there is a conflict with a mapping.
2517	*
2518	* @returns VBox status code.
2519	* @param pVCpu The VMCPU handle.
2520	* @param iPD Page directory index.
2521	* @param pPDSrc Source page directory (i.e. Guest OS page directory).
2522	* Assume this is a temporary mapping.
2523	* @param GCPtrPage GC Pointer of the page that caused the fault
2524	*/
2525	PGM_BTH_DECL(int, SyncPT)(PVMCPU pVCpu, unsigned iPDSrc, PGSTPD pPDSrc, RTGCPTR GCPtrPage)
2526	{
2527	PVM pVM = pVCpu->CTX_SUFF(pVM);
2528	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2529
2530	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2531	#if 0 /* rarely useful; leave for debugging. */
2532	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPtPD[iPDSrc]);
2533	#endif
2534	LogFlow(("SyncPT: GCPtrPage=%RGv\n", GCPtrPage));
2535
2536	Assert(PGMIsLocked(pVM));
2537
2538	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2539	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
2540	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
2541	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2542	&& PGM_SHW_TYPE != PGM_TYPE_EPT
2543
2544	int rc = VINF_SUCCESS;
2545
2546	/*
2547	* Validate input a little bit.
2548	*/
2549	AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage));
2550	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2551	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2552	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
2553
2554	/* Fetch the pgm pool shadow descriptor. */
2555	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2556	Assert(pShwPde);
2557
2558	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2559	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2560	PPGMPOOLPAGE pShwPde = NULL;
2561	PX86PDPAE pPDDst;
2562	PSHWPDE pPdeDst;
2563
2564	/* Fetch the pgm pool shadow descriptor. */
2565	rc = pgmShwGetPaePoolPagePD(&pVCpu->pgm.s, GCPtrPage, &pShwPde);
2566	AssertRCSuccessReturn(rc, rc);
2567	Assert(pShwPde);
2568
2569	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGM(&pVM->pgm.s, pShwPde);
2570	pPdeDst = &pPDDst->a[iPDDst];
2571
2572	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2573	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2574	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2575	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2576	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2577	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2578	AssertRCSuccessReturn(rc, rc);
2579	Assert(pPDDst);
2580	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2581	# endif
2582	SHWPDE PdeDst = *pPdeDst;
2583
2584	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2585	/* Fetch the pgm pool shadow descriptor. */
2586	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2587	Assert(pShwPde);
2588	# endif
2589
2590	# ifndef PGM_WITHOUT_MAPPINGS
2591	/*
2592	* Check for conflicts.
2593	* GC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
2594	* HC: Simply resolve the conflict.
2595	*/
2596	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
2597	{
2598	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
2599	# ifndef IN_RING3
2600	Log(("SyncPT: Conflict at %RGv\n", GCPtrPage));
2601	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2602	return VERR_ADDRESS_CONFLICT;
2603	# else
2604	PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
2605	Assert(pMapping);
2606	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2607	int rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2608	# elif PGM_GST_TYPE == PGM_TYPE_PAE
2609	int rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2610	# else
2611	AssertFailed(); /* can't happen for amd64 */
2612	# endif
2613	if (RT_FAILURE(rc))
2614	{
2615	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2616	return rc;
2617	}
2618	PdeDst = *pPdeDst;
2619	# endif
2620	}
2621	# else /* PGM_WITHOUT_MAPPINGS */
2622	Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
2623	# endif /* PGM_WITHOUT_MAPPINGS */
2624	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2625
2626	# if defined(IN_RC)
2627	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
2628	PGMDynLockHCPage(pVM, (uint8_t *)pPdeDst);
2629	# endif
2630
2631	/*
2632	* Sync page directory entry.
2633	*/
2634	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2635	if (PdeSrc.n.u1Present)
2636	{
2637	/*
2638	* Allocate & map the page table.
2639	*/
2640	PSHWPT pPTDst;
2641	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2642	const bool fPageTable = !PdeSrc.b.u1Size;
2643	# else
2644	const bool fPageTable = !PdeSrc.b.u1Size \|\| !(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
2645	# endif
2646	PPGMPOOLPAGE pShwPage;
2647	RTGCPHYS GCPhys;
2648	if (fPageTable)
2649	{
2650	GCPhys = PdeSrc.u & GST_PDE_PG_MASK;
2651	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2652	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2653	GCPhys \|= (iPDDst & 1) * (PAGE_SIZE / 2);
2654	# endif
2655	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, pShwPde->idx, iPDDst, &pShwPage);
2656	}
2657	else
2658	{
2659	PGMPOOLACCESS enmAccess;
2660
2661	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2662	const bool fNoExecuteBitValid = !!(CPUMGetGuestEFER(pVCpu) & MSR_K6_EFER_NXE);
2663	const bool fNoExecute = fNoExecuteBitValid && PdeSrc.n.u1NoExecute;
2664	# else
2665	const bool fNoExecute = false;
2666	# endif
2667
2668	GCPhys = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
2669	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2670	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2671	GCPhys \|= GCPtrPage & (1 << X86_PD_PAE_SHIFT);
2672	# endif
2673	/* Determine the right kind of large page to avoid incorrect cached entry reuse. */
2674	if (PdeSrc.n.u1User)
2675	{
2676	if (PdeSrc.n.u1Write)
2677	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_RW_NX : PGMPOOLACCESS_USER_RW;
2678	else
2679	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_R_NX : PGMPOOLACCESS_USER_R;
2680	}
2681	else
2682	{
2683	if (PdeSrc.n.u1Write)
2684	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW;
2685	else
2686	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_R_NX : PGMPOOLACCESS_SUPERVISOR_R;
2687	}
2688	rc = pgmPoolAllocEx(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, enmAccess, pShwPde->idx, iPDDst, &pShwPage);
2689	}
2690	if (rc == VINF_SUCCESS)
2691	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
2692	else if (rc == VINF_PGM_CACHED_PAGE)
2693	{
2694	/*
2695	* The PT was cached, just hook it up.
2696	*/
2697	if (fPageTable)
2698	PdeDst.u = pShwPage->Core.Key
2699	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2700	else
2701	{
2702	PdeDst.u = pShwPage->Core.Key
2703	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2704	/* (see explanation and assumptions further down.) */
2705	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
2706	{
2707	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageBig));
2708	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2709	PdeDst.b.u1Write = 0;
2710	}
2711	}
2712	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2713	# if defined(IN_RC)
2714	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2715	# endif
2716	return VINF_SUCCESS;
2717	}
2718	else if (rc == VERR_PGM_POOL_FLUSHED)
2719	{
2720	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2721	# if defined(IN_RC)
2722	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2723	# endif
2724	return VINF_PGM_SYNC_CR3;
2725	}
2726	else
2727	AssertMsgFailedReturn(("rc=%Rrc\n", rc), VERR_INTERNAL_ERROR);
2728	PdeDst.u &= X86_PDE_AVL_MASK;
2729	PdeDst.u \|= pShwPage->Core.Key;
2730
2731	/*
2732	* Page directory has been accessed (this is a fault situation, remember).
2733	*/
2734	pPDSrc->a[iPDSrc].n.u1Accessed = 1;
2735	if (fPageTable)
2736	{
2737	/*
2738	* Page table - 4KB.
2739	*
2740	* Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
2741	*/
2742	Log2(("SyncPT: 4K %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
2743	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
2744	PGSTPT pPTSrc;
2745	rc = PGM_GCPHYS_2_PTR(pVM, PdeSrc.u & GST_PDE_PG_MASK, &pPTSrc);
2746	if (RT_SUCCESS(rc))
2747	{
2748	/*
2749	* Start by syncing the page directory entry so CSAM's TLB trick works.
2750	*/
2751	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| X86_PDE_AVL_MASK))
2752	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2753	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2754	# if defined(IN_RC)
2755	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2756	# endif
2757
2758	/*
2759	* Directory/page user or supervisor privilege: (same goes for read/write)
2760	*
2761	* Directory Page Combined
2762	* U/S U/S U/S
2763	* 0 0 0
2764	* 0 1 0
2765	* 1 0 0
2766	* 1 1 1
2767	*
2768	* Simple AND operation. Table listed for completeness.
2769	*
2770	*/
2771	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT4K));
2772	# ifdef PGM_SYNC_N_PAGES
2773	unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2774	unsigned iPTDst = iPTBase;
2775	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2776	if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
2777	iPTDst = 0;
2778	else
2779	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2780	# else /* !PGM_SYNC_N_PAGES */
2781	unsigned iPTDst = 0;
2782	const unsigned iPTDstEnd = RT_ELEMENTS(pPTDst->a);
2783	# endif /* !PGM_SYNC_N_PAGES */
2784	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2785	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2786	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2787	# else
2788	const unsigned offPTSrc = 0;
2789	# endif
2790	for (; iPTDst < iPTDstEnd; iPTDst++)
2791	{
2792	const unsigned iPTSrc = iPTDst + offPTSrc;
2793	const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2794
2795	if (PteSrc.n.u1Present) /* we've already cleared it above */
2796	{
2797	# ifndef IN_RING0
2798	/*
2799	* Assuming kernel code will be marked as supervisor - and not as user level
2800	* and executed using a conforming code selector - And marked as readonly.
2801	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
2802	*/
2803	PPGMPAGE pPage;
2804	if ( ((PdeSrc.u & pPTSrc->a[iPTSrc].u) & (X86_PTE_RW \| X86_PTE_US))
2805	\|\| !CSAMDoesPageNeedScanning(pVM, (RTRCPTR)((iPDSrc << GST_PD_SHIFT) \| (iPTSrc << PAGE_SHIFT)))
2806	\|\| ( (pPage = pgmPhysGetPage(&pVM->pgm.s, PteSrc.u & GST_PTE_PG_MASK))
2807	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2808	)
2809	# endif
2810	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2811	Log2(("SyncPT: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n",
2812	(RTGCPTR)(((RTGCPTR)iPDSrc << GST_PD_SHIFT) \| ((RTGCPTR)iPTSrc << PAGE_SHIFT)),
2813	PteSrc.n.u1Present,
2814	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2815	PteSrc.n.u1User & PdeSrc.n.u1User,
2816	(uint64_t)PteSrc.u,
2817	pPTDst->a[iPTDst].u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : "", pPTDst->a[iPTDst].u, iPTSrc, PdeSrc.au32[0],
2818	(RTGCPHYS)((PdeSrc.u & GST_PDE_PG_MASK) + iPTSrc*sizeof(PteSrc)) ));
2819	}
2820	} /* for PTEs */
2821	}
2822	}
2823	else
2824	{
2825	/*
2826	* Big page - 2/4MB.
2827	*
2828	* We'll walk the ram range list in parallel and optimize lookups.
2829	* We will only sync on shadow page table at a time.
2830	*/
2831	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT4M));
2832
2833	/**
2834	* @todo It might be more efficient to sync only a part of the 4MB page (similar to what we do for 4kb PDs).
2835	*/
2836
2837	/*
2838	* Start by syncing the page directory entry.
2839	*/
2840	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
2841	\| (PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PDE_AVL_MASK \| X86_PDE_PCD \| X86_PDE_PWT \| X86_PDE_PS \| X86_PDE4M_G \| X86_PDE4M_D));
2842
2843	/*
2844	* If the page is not flagged as dirty and is writable, then make it read-only
2845	* at PD level, so we can set the dirty bit when the page is modified.
2846	*
2847	* ASSUMES that page access handlers are implemented on page table entry level.
2848	* Thus we will first catch the dirty access and set PDE.D and restart. If
2849	* there is an access handler, we'll trap again and let it work on the problem.
2850	*/
2851	/** @todo move the above stuff to a section in the PGM documentation. */
2852	Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
2853	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
2854	{
2855	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,DirtyPageBig));
2856	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2857	PdeDst.b.u1Write = 0;
2858	}
2859	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2860	# if defined(IN_RC)
2861	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
2862	# endif
2863
2864	/*
2865	* Fill the shadow page table.
2866	*/
2867	/* Get address and flags from the source PDE. */
2868	SHWPTE PteDstBase;
2869	PteDstBase.u = PdeSrc.u & ~(GST_PDE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT);
2870
2871	/* Loop thru the entries in the shadow PT. */
2872	const RTGCPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
2873	Log2(("SyncPT: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%RGv GCPhys=%RGp %s\n",
2874	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
2875	GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2876	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
2877	unsigned iPTDst = 0;
2878	while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2879	&& !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2880	{
2881	/* Advance ram range list. */
2882	while (pRam && GCPhys > pRam->GCPhysLast)
2883	pRam = pRam->CTX_SUFF(pNext);
2884	if (pRam && GCPhys >= pRam->GCPhys)
2885	{
2886	unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2887	do
2888	{
2889	/* Make shadow PTE. */
2890	PPGMPAGE pPage = &pRam->aPages[iHCPage];
2891	SHWPTE PteDst;
2892
2893	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2894	/* Try make the page writable if necessary. */
2895	if ( PteDstBase.n.u1Write
2896	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2897	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
2898	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
2899	# endif
2900	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
2901	{
2902	rc = pgmPhysPageMakeWritableUnlocked(pVM, pPage, GCPhys);
2903	AssertRCReturn(rc, rc);
2904	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
2905	break;
2906	}
2907	# endif
2908
2909	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2910	{
2911	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
2912	{
2913	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) \| PteDstBase.u;
2914	PteDst.n.u1Write = 0;
2915	}
2916	else
2917	PteDst.u = 0;
2918	}
2919	# ifndef IN_RING0
2920	/*
2921	* Assuming kernel code will be marked as supervisor and not as user level and executed
2922	* using a conforming code selector. Don't check for readonly, as that implies the whole
2923	* 4MB can be code or readonly data. Linux enables write access for its large pages.
2924	*/
2925	else if ( !PdeSrc.n.u1User
2926	&& CSAMDoesPageNeedScanning(pVM, (RTRCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))))
2927	PteDst.u = 0;
2928	# endif
2929	else
2930	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) \| PteDstBase.u;
2931
2932	/* Only map writable pages writable. */
2933	if ( PteDst.n.u1Write
2934	&& PteDst.n.u1Present
2935	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2936	{
2937	PteDst.n.u1Write = 0; /** @todo this isn't quite working yet... */
2938	Log3(("SyncPT: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, (RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))));
2939	}
2940
2941	# ifdef PGMPOOL_WITH_USER_TRACKING
2942	if (PteDst.n.u1Present)
2943	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2944	# endif
2945	/* commit it */
2946	pPTDst->a[iPTDst] = PteDst;
2947	Log4(("SyncPT: BIG %RGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
2948	(RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT)), PteDst.n.u1Present, PteDst.n.u1Write, PteDst.n.u1User, (uint64_t)PteDst.u,
2949	PteDst.u & PGM_PTFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2950
2951	/* advance */
2952	GCPhys += PAGE_SIZE;
2953	iHCPage++;
2954	iPTDst++;
2955	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2956	&& GCPhys <= pRam->GCPhysLast);
2957	}
2958	else if (pRam)
2959	{
2960	Log(("Invalid pages at %RGp\n", GCPhys));
2961	do
2962	{
2963	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2964	GCPhys += PAGE_SIZE;
2965	iPTDst++;
2966	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2967	&& GCPhys < pRam->GCPhys);
2968	}
2969	else
2970	{
2971	Log(("Invalid pages at %RGp (2)\n", GCPhys));
2972	for ( ; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
2973	pPTDst->a[iPTDst].u = 0; /* MMIO or invalid page, we must handle them manually. */
2974	}
2975	} /* while more PTEs */
2976	} /* 4KB / 4MB */
2977	}
2978	else
2979	AssertRelease(!PdeDst.n.u1Present);
2980
2981	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
2982	if (RT_FAILURE(rc))
2983	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPTFailed));
2984	return rc;
2985
2986	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2987	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2988	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2989	&& !defined(IN_RC)
2990
2991	/*
2992	* Validate input a little bit.
2993	*/
2994	int rc = VINF_SUCCESS;
2995	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2996	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2997	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
2998
2999	/* Fetch the pgm pool shadow descriptor. */
3000	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
3001	Assert(pShwPde);
3002
3003	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3004	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3005	PPGMPOOLPAGE pShwPde = NULL; /* initialized to shut up gcc */
3006	PX86PDPAE pPDDst;
3007	PSHWPDE pPdeDst;
3008
3009	/* Fetch the pgm pool shadow descriptor. */
3010	rc = pgmShwGetPaePoolPagePD(&pVCpu->pgm.s, GCPtrPage, &pShwPde);
3011	AssertRCSuccessReturn(rc, rc);
3012	Assert(pShwPde);
3013
3014	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGM(&pVM->pgm.s, pShwPde);
3015	pPdeDst = &pPDDst->a[iPDDst];
3016
3017	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3018	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3019	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3020	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
3021	PX86PDPT pPdptDst= NULL; /* initialized to shut up gcc */
3022	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
3023	AssertRCSuccessReturn(rc, rc);
3024	Assert(pPDDst);
3025	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3026
3027	/* Fetch the pgm pool shadow descriptor. */
3028	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
3029	Assert(pShwPde);
3030
3031	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
3032	const unsigned iPdpt = (GCPtrPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
3033	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3034	PEPTPD pPDDst;
3035	PEPTPDPT pPdptDst;
3036
3037	rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, &pPdptDst, &pPDDst);
3038	if (rc != VINF_SUCCESS)
3039	{
3040	AssertRC(rc);
3041	return rc;
3042	}
3043	Assert(pPDDst);
3044	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3045
3046	/* Fetch the pgm pool shadow descriptor. */
3047	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & EPT_PDPTE_PG_MASK);
3048	Assert(pShwPde);
3049	# endif
3050	SHWPDE PdeDst = *pPdeDst;
3051
3052	Assert(!(PdeDst.u & PGM_PDFLAGS_MAPPING));
3053	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
3054
3055	GSTPDE PdeSrc;
3056	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
3057	PdeSrc.n.u1Present = 1;
3058	PdeSrc.n.u1Write = 1;
3059	PdeSrc.n.u1Accessed = 1;
3060	PdeSrc.n.u1User = 1;
3061
3062	/*
3063	* Allocate & map the page table.
3064	*/
3065	PSHWPT pPTDst;
3066	PPGMPOOLPAGE pShwPage;
3067	RTGCPHYS GCPhys;
3068
3069	/* Virtual address = physical address */
3070	GCPhys = GCPtrPage & X86_PAGE_4K_BASE_MASK;
3071	rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, pShwPde->idx, iPDDst, &pShwPage);
3072
3073	if ( rc == VINF_SUCCESS
3074	\|\| rc == VINF_PGM_CACHED_PAGE)
3075	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
3076	else
3077	AssertMsgFailedReturn(("rc=%Rrc\n", rc), VERR_INTERNAL_ERROR);
3078
3079	PdeDst.u &= X86_PDE_AVL_MASK;
3080	PdeDst.u \|= pShwPage->Core.Key;
3081	PdeDst.n.u1Present = 1;
3082	PdeDst.n.u1Write = 1;
3083	# if PGM_SHW_TYPE == PGM_TYPE_EPT
3084	PdeDst.n.u1Execute = 1;
3085	# else
3086	PdeDst.n.u1User = 1;
3087	PdeDst.n.u1Accessed = 1;
3088	# endif
3089	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
3090
3091	pgmLock(pVM);
3092	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, PGM_SYNC_NR_PAGES, 0 /* page not present */);
3093	pgmUnlock(pVM);
3094	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
3095	return rc;
3096
3097	#else
3098	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
3099	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPT), a);
3100	return VERR_INTERNAL_ERROR;
3101	#endif
3102	}
3103
3104
3105
3106	/**
3107	* Prefetch a page/set of pages.
3108	*
3109	* Typically used to sync commonly used pages before entering raw mode
3110	* after a CR3 reload.
3111	*
3112	* @returns VBox status code.
3113	* @param pVCpu The VMCPU handle.
3114	* @param GCPtrPage Page to invalidate.
3115	*/
3116	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
3117	{
3118	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
3119	&& PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
3120	/*
3121	* Check that all Guest levels thru the PDE are present, getting the
3122	* PD and PDE in the processes.
3123	*/
3124	int rc = VINF_SUCCESS;
3125	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3126	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3127	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
3128	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
3129	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3130	unsigned iPDSrc;
3131	X86PDPE PdpeSrc;
3132	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, GCPtrPage, &iPDSrc, &PdpeSrc);
3133	if (!pPDSrc)
3134	return VINF_SUCCESS; /* not present */
3135	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3136	unsigned iPDSrc;
3137	PX86PML4E pPml4eSrc;
3138	X86PDPE PdpeSrc;
3139	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3140	if (!pPDSrc)
3141	return VINF_SUCCESS; /* not present */
3142	# endif
3143	const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3144	# else
3145	PGSTPD pPDSrc = NULL;
3146	const unsigned iPDSrc = 0;
3147	GSTPDE PdeSrc;
3148
3149	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
3150	PdeSrc.n.u1Present = 1;
3151	PdeSrc.n.u1Write = 1;
3152	PdeSrc.n.u1Accessed = 1;
3153	PdeSrc.n.u1User = 1;
3154	# endif
3155
3156	if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
3157	{
3158	PVM pVM = pVCpu->CTX_SUFF(pVM);
3159	pgmLock(pVM);
3160
3161	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3162	const X86PDE PdeDst = pgmShwGet32BitPDE(&pVCpu->pgm.s, GCPtrPage);
3163	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3164	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3165	PX86PDPAE pPDDst;
3166	X86PDEPAE PdeDst;
3167	# if PGM_GST_TYPE != PGM_TYPE_PAE
3168	X86PDPE PdpeSrc;
3169
3170	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3171	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3172	# endif
3173	int rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, &PdpeSrc, &pPDDst);
3174	if (rc != VINF_SUCCESS)
3175	{
3176	pgmUnlock(pVM);
3177	AssertRC(rc);
3178	return rc;
3179	}
3180	Assert(pPDDst);
3181	PdeDst = pPDDst->a[iPDDst];
3182
3183	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3184	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3185	PX86PDPAE pPDDst;
3186	X86PDEPAE PdeDst;
3187
3188	# if PGM_GST_TYPE == PGM_TYPE_PROT
3189	/* AMD-V nested paging */
3190	X86PML4E Pml4eSrc;
3191	X86PDPE PdpeSrc;
3192	PX86PML4E pPml4eSrc = &Pml4eSrc;
3193
3194	/* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3195	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_NX \| X86_PML4E_A;
3196	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_NX \| X86_PDPE_A;
3197	# endif
3198
3199	int rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc, &PdpeSrc, &pPDDst);
3200	if (rc != VINF_SUCCESS)
3201	{
3202	pgmUnlock(pVM);
3203	AssertRC(rc);
3204	return rc;
3205	}
3206	Assert(pPDDst);
3207	PdeDst = pPDDst->a[iPDDst];
3208	# endif
3209	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
3210	{
3211	if (!PdeDst.n.u1Present)
3212	{
3213	/** r=bird: This guy will set the A bit on the PDE, probably harmless. */
3214	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3215	}
3216	else
3217	{
3218	/** @note We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
3219	* R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
3220	* makes no sense to prefetch more than one page.
3221	*/
3222	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3223	if (RT_SUCCESS(rc))
3224	rc = VINF_SUCCESS;
3225	}
3226	}
3227	pgmUnlock(pVM);
3228	}
3229	return rc;
3230
3231	#elif PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3232	return VINF_SUCCESS; /* ignore */
3233	#endif
3234	}
3235
3236
3237
3238
3239	/**
3240	* Syncs a page during a PGMVerifyAccess() call.
3241	*
3242	* @returns VBox status code (informational included).
3243	* @param pVCpu The VMCPU handle.
3244	* @param GCPtrPage The address of the page to sync.
3245	* @param fPage The effective guest page flags.
3246	* @param uErr The trap error code.
3247	*/
3248	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fPage, unsigned uErr)
3249	{
3250	PVM pVM = pVCpu->CTX_SUFF(pVM);
3251
3252	LogFlow(("VerifyAccessSyncPage: GCPtrPage=%RGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
3253
3254	Assert(!HWACCMIsNestedPagingActive(pVM));
3255	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_TYPE_AMD64) \
3256	&& PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
3257
3258	# ifndef IN_RING0
3259	if (!(fPage & X86_PTE_US))
3260	{
3261	/*
3262	* Mark this page as safe.
3263	*/
3264	/** @todo not correct for pages that contain both code and data!! */
3265	Log(("CSAMMarkPage %RGv; scanned=%d\n", GCPtrPage, true));
3266	CSAMMarkPage(pVM, (RTRCPTR)GCPtrPage, true);
3267	}
3268	# endif
3269
3270	/*
3271	* Get guest PD and index.
3272	*/
3273	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3274	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3275	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
3276	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
3277	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3278	unsigned iPDSrc = 0;
3279	X86PDPE PdpeSrc;
3280	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, GCPtrPage, &iPDSrc, &PdpeSrc);
3281
3282	if (pPDSrc)
3283	{
3284	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3285	return VINF_EM_RAW_GUEST_TRAP;
3286	}
3287	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3288	unsigned iPDSrc;
3289	PX86PML4E pPml4eSrc;
3290	X86PDPE PdpeSrc;
3291	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3292	if (!pPDSrc)
3293	{
3294	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3295	return VINF_EM_RAW_GUEST_TRAP;
3296	}
3297	# endif
3298	# else
3299	PGSTPD pPDSrc = NULL;
3300	const unsigned iPDSrc = 0;
3301	# endif
3302	int rc = VINF_SUCCESS;
3303
3304	pgmLock(pVM);
3305
3306	/*
3307	* First check if the shadow pd is present.
3308	*/
3309	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3310	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(&pVCpu->pgm.s, GCPtrPage);
3311	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3312	PX86PDEPAE pPdeDst;
3313	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3314	PX86PDPAE pPDDst;
3315	# if PGM_GST_TYPE != PGM_TYPE_PAE
3316	X86PDPE PdpeSrc;
3317
3318	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3319	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3320	# endif
3321	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, &PdpeSrc, &pPDDst);
3322	if (rc != VINF_SUCCESS)
3323	{
3324	pgmUnlock(pVM);
3325	AssertRC(rc);
3326	return rc;
3327	}
3328	Assert(pPDDst);
3329	pPdeDst = &pPDDst->a[iPDDst];
3330
3331	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3332	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3333	PX86PDPAE pPDDst;
3334	PX86PDEPAE pPdeDst;
3335
3336	# if PGM_GST_TYPE == PGM_TYPE_PROT
3337	/* AMD-V nested paging */
3338	X86PML4E Pml4eSrc;
3339	X86PDPE PdpeSrc;
3340	PX86PML4E pPml4eSrc = &Pml4eSrc;
3341
3342	/* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3343	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_NX \| X86_PML4E_A;
3344	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_NX \| X86_PDPE_A;
3345	# endif
3346
3347	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc, &PdpeSrc, &pPDDst);
3348	if (rc != VINF_SUCCESS)
3349	{
3350	pgmUnlock(pVM);
3351	AssertRC(rc);
3352	return rc;
3353	}
3354	Assert(pPDDst);
3355	pPdeDst = &pPDDst->a[iPDDst];
3356	# endif
3357
3358	# if defined(IN_RC)
3359	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
3360	PGMDynLockHCPage(pVM, (uint8_t *)pPdeDst);
3361	# endif
3362
3363	if (!pPdeDst->n.u1Present)
3364	{
3365	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3366	if (rc != VINF_SUCCESS)
3367	{
3368	# if defined(IN_RC)
3369	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
3370	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
3371	# endif
3372	pgmUnlock(pVM);
3373	AssertRC(rc);
3374	return rc;
3375	}
3376	}
3377
3378	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3379	/* Check for dirty bit fault */
3380	rc = PGM_BTH_NAME(CheckPageFault)(pVCpu, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
3381	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
3382	Log(("PGMVerifyAccess: success (dirty)\n"));
3383	else
3384	{
3385	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3386	# else
3387	{
3388	GSTPDE PdeSrc;
3389	PdeSrc.au32[0] = 0; /* faked so we don't have to #ifdef everything */
3390	PdeSrc.n.u1Present = 1;
3391	PdeSrc.n.u1Write = 1;
3392	PdeSrc.n.u1Accessed = 1;
3393	PdeSrc.n.u1User = 1;
3394
3395	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
3396	Assert(rc != VINF_EM_RAW_GUEST_TRAP);
3397	if (uErr & X86_TRAP_PF_US)
3398	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncUser));
3399	else /* supervisor */
3400	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
3401
3402	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3403	if (RT_SUCCESS(rc))
3404	{
3405	/* Page was successfully synced */
3406	Log2(("PGMVerifyAccess: success (sync)\n"));
3407	rc = VINF_SUCCESS;
3408	}
3409	else
3410	{
3411	Log(("PGMVerifyAccess: access violation for %RGv rc=%d\n", GCPtrPage, rc));
3412	rc = VINF_EM_RAW_GUEST_TRAP;
3413	}
3414	}
3415	# if defined(IN_RC)
3416	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
3417	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdeDst);
3418	# endif
3419	pgmUnlock(pVM);
3420	return rc;
3421
3422	#else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
3423
3424	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
3425	return VERR_INTERNAL_ERROR;
3426	#endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
3427	}
3428
3429	#undef MY_STAM_COUNTER_INC
3430	#define MY_STAM_COUNTER_INC(a) do { } while (0)
3431
3432
3433	/**
3434	* Syncs the paging hierarchy starting at CR3.
3435	*
3436	* @returns VBox status code, no specials.
3437	* @param pVCpu The VMCPU handle.
3438	* @param cr0 Guest context CR0 register
3439	* @param cr3 Guest context CR3 register
3440	* @param cr4 Guest context CR4 register
3441	* @param fGlobal Including global page directories or not
3442	*/
3443	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
3444	{
3445	PVM pVM = pVCpu->CTX_SUFF(pVM);
3446
3447	if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
3448	fGlobal = true; /* Change this CR3 reload to be a global one. */
3449
3450	LogFlow(("SyncCR3 %d\n", fGlobal));
3451
3452	#if PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
3453
3454	pgmLock(pVM);
3455	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
3456	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3457	if (pPool->cDirtyPages)
3458	pgmPoolResetDirtyPages(pVM);
3459	# endif
3460
3461	/*
3462	* Update page access handlers.
3463	* The virtual are always flushed, while the physical are only on demand.
3464	* WARNING: We are incorrectly not doing global flushing on Virtual Handler updates. We'll
3465	* have to look into that later because it will have a bad influence on the performance.
3466	* @note SvL: There's no need for that. Just invalidate the virtual range(s).
3467	* bird: Yes, but that won't work for aliases.
3468	*/
3469	/** @todo this MUST go away. See #1557. */
3470	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncCR3Handlers), h);
3471	PGM_GST_NAME(HandlerVirtualUpdate)(pVM, cr4);
3472	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncCR3Handlers), h);
3473	pgmUnlock(pVM);
3474	#endif
3475
3476	#if PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3477	/*
3478	* Nested / EPT - almost no work.
3479	*/
3480	/** @todo check if this is really necessary; the call does it as well... */
3481	HWACCMFlushTLB(pVCpu);
3482	return VINF_SUCCESS;
3483
3484	#elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3485	/*
3486	* AMD64 (Shw & Gst) - No need to check all paging levels; we zero
3487	* out the shadow parts when the guest modifies its tables.
3488	*/
3489	return VINF_SUCCESS;
3490
3491	#else /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3492
3493	# ifdef PGM_WITHOUT_MAPPINGS
3494	Assert(pVM->pgm.s.fMappingsFixed);
3495	return VINF_SUCCESS;
3496	# else
3497	/* Nothing to do when mappings are fixed. */
3498	if (pVM->pgm.s.fMappingsFixed)
3499	return VINF_SUCCESS;
3500
3501	int rc = PGMMapResolveConflicts(pVM);
3502	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
3503	if (rc == VINF_PGM_SYNC_CR3)
3504	{
3505	LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
3506	return VINF_PGM_SYNC_CR3;
3507	}
3508	# endif
3509	return VINF_SUCCESS;
3510	#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3511	}
3512
3513
3514
3515
3516	#ifdef VBOX_STRICT
3517	#ifdef IN_RC
3518	# undef AssertMsgFailed
3519	# define AssertMsgFailed Log
3520	#endif
3521	#ifdef IN_RING3
3522	# include <VBox/dbgf.h>
3523
3524	/**
3525	* Dumps a page table hierarchy use only physical addresses and cr4/lm flags.
3526	*
3527	* @returns VBox status code (VINF_SUCCESS).
3528	* @param cr3 The root of the hierarchy.
3529	* @param crr The cr4, only PAE and PSE is currently used.
3530	* @param fLongMode Set if long mode, false if not long mode.
3531	* @param cMaxDepth Number of levels to dump.
3532	* @param pHlp Pointer to the output functions.
3533	*/
3534	RT_C_DECLS_BEGIN
3535	VMMR3DECL(int) PGMR3DumpHierarchyHC(PVM pVM, uint32_t cr3, uint32_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp);
3536	RT_C_DECLS_END
3537
3538	#endif
3539
3540	/**
3541	* Checks that the shadow page table is in sync with the guest one.
3542	*
3543	* @returns The number of errors.
3544	* @param pVM The virtual machine.
3545	* @param pVCpu The VMCPU handle.
3546	* @param cr3 Guest context CR3 register
3547	* @param cr4 Guest context CR4 register
3548	* @param GCPtr Where to start. Defaults to 0.
3549	* @param cb How much to check. Defaults to everything.
3550	*/
3551	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb)
3552	{
3553	#if PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3554	return 0;
3555	#else
3556	unsigned cErrors = 0;
3557	PVM pVM = pVCpu->CTX_SUFF(pVM);
3558	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3559
3560	#if PGM_GST_TYPE == PGM_TYPE_PAE
3561	/** @todo currently broken; crashes below somewhere */
3562	AssertFailed();
3563	#endif
3564
3565	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
3566	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3567	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
3568
3569	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3570	bool fBigPagesSupported = true;
3571	# else
3572	bool fBigPagesSupported = !!(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
3573	# endif
3574	PPGMCPU pPGM = &pVCpu->pgm.s;
3575	RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
3576	RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
3577	# ifndef IN_RING0
3578	RTHCPHYS HCPhys; /* general usage. */
3579	# endif
3580	int rc;
3581
3582	/*
3583	* Check that the Guest CR3 and all its mappings are correct.
3584	*/
3585	AssertMsgReturn(pPGM->GCPhysCR3 == (cr3 & GST_CR3_PAGE_MASK),
3586	("Invalid GCPhysCR3=%RGp cr3=%RGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
3587	false);
3588	# if !defined(IN_RING0) && PGM_GST_TYPE != PGM_TYPE_AMD64
3589	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3590	rc = PGMShwGetPage(pVCpu, (RTGCPTR)pPGM->pGst32BitPdRC, NULL, &HCPhysShw);
3591	# else
3592	rc = PGMShwGetPage(pVCpu, (RTGCPTR)pPGM->pGstPaePdptRC, NULL, &HCPhysShw);
3593	# endif
3594	AssertRCReturn(rc, 1);
3595	HCPhys = NIL_RTHCPHYS;
3596	rc = pgmRamGCPhys2HCPhys(&pVM->pgm.s, cr3 & GST_CR3_PAGE_MASK, &HCPhys);
3597	AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
3598	# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
3599	pgmGstGet32bitPDPtr(pPGM);
3600	RTGCPHYS GCPhys;
3601	rc = PGMR3DbgR3Ptr2GCPhys(pVM, pPGM->pGst32BitPdR3, &GCPhys);
3602	AssertRCReturn(rc, 1);
3603	AssertMsgReturn((cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false);
3604	# endif
3605	# endif /* !IN_RING0 */
3606
3607	/*
3608	* Get and check the Shadow CR3.
3609	*/
3610	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3611	unsigned cPDEs = X86_PG_ENTRIES;
3612	unsigned cIncrement = X86_PG_ENTRIES * PAGE_SIZE;
3613	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3614	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3615	unsigned cPDEs = X86_PG_PAE_ENTRIES * 4; /* treat it as a 2048 entry table. */
3616	# else
3617	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3618	# endif
3619	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3620	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3621	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3622	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3623	# endif
3624	if (cb != ~(RTGCPTR)0)
3625	cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
3626
3627	/** @todo call the other two PGMAssert() functions. /
3628
3629	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3630	unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3631
3632	for (; iPml4 < X86_PG_PAE_ENTRIES; iPml4++)
3633	{
3634	PPGMPOOLPAGE pShwPdpt = NULL;
3635	PX86PML4E pPml4eSrc;
3636	PX86PML4E pPml4eDst;
3637	RTGCPHYS GCPhysPdptSrc;
3638
3639	pPml4eSrc = pgmGstGetLongModePML4EPtr(&pVCpu->pgm.s, iPml4);
3640	pPml4eDst = pgmShwGetLongModePML4EPtr(&pVCpu->pgm.s, iPml4);
3641
3642	/* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
3643	if (!pPml4eDst->n.u1Present)
3644	{
3645	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3646	continue;
3647	}
3648
3649	pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
3650	GCPhysPdptSrc = pPml4eSrc->u & X86_PML4E_PG_MASK_FULL;
3651
3652	if (pPml4eSrc->n.u1Present != pPml4eDst->n.u1Present)
3653	{
3654	AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3655	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3656	cErrors++;
3657	continue;
3658	}
3659
3660	if (GCPhysPdptSrc != pShwPdpt->GCPhys)
3661	{
3662	AssertMsgFailed(("Physical address doesn't match! iPml4 %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
3663	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3664	cErrors++;
3665	continue;
3666	}
3667
3668	if ( pPml4eDst->n.u1User != pPml4eSrc->n.u1User
3669	\|\| pPml4eDst->n.u1Write != pPml4eSrc->n.u1Write
3670	\|\| pPml4eDst->n.u1NoExecute != pPml4eSrc->n.u1NoExecute)
3671	{
3672	AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3673	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3674	cErrors++;
3675	continue;
3676	}
3677	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3678	{
3679	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3680
3681	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
3682	/*
3683	* Check the PDPTEs too.
3684	*/
3685	unsigned iPdpt = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
3686
3687	for (;iPdpt <= SHW_PDPT_MASK; iPdpt++)
3688	{
3689	unsigned iPDSrc = 0; /* initialized to shut up gcc */
3690	PPGMPOOLPAGE pShwPde = NULL;
3691	PX86PDPE pPdpeDst;
3692	RTGCPHYS GCPhysPdeSrc;
3693	# if PGM_GST_TYPE == PGM_TYPE_PAE
3694	X86PDPE PdpeSrc;
3695	PGSTPD pPDSrc = pgmGstGetPaePDPtr(&pVCpu->pgm.s, GCPtr, &iPDSrc, &PdpeSrc);
3696	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(&pVCpu->pgm.s);
3697	# else
3698	PX86PML4E pPml4eSrc;
3699	X86PDPE PdpeSrc;
3700	PX86PDPT pPdptDst;
3701	PX86PDPAE pPDDst;
3702	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(&pVCpu->pgm.s, GCPtr, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3703
3704	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtr, NULL, &pPdptDst, &pPDDst);
3705	if (rc != VINF_SUCCESS)
3706	{
3707	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
3708	GCPtr += 512 * _2M;
3709	continue; /* next PDPTE */
3710	}
3711	Assert(pPDDst);
3712	# endif
3713	Assert(iPDSrc == 0);
3714
3715	pPdpeDst = &pPdptDst->a[iPdpt];
3716
3717	if (!pPdpeDst->n.u1Present)
3718	{
3719	GCPtr += 512 * _2M;
3720	continue; /* next PDPTE */
3721	}
3722
3723	pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
3724	GCPhysPdeSrc = PdpeSrc.u & X86_PDPE_PG_MASK;
3725
3726	if (pPdpeDst->n.u1Present != PdpeSrc.n.u1Present)
3727	{
3728	AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3729	GCPtr += 512 * _2M;
3730	cErrors++;
3731	continue;
3732	}
3733
3734	if (GCPhysPdeSrc != pShwPde->GCPhys)
3735	{
3736	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3737	AssertMsgFailed(("Physical address doesn't match! iPml4 %d iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3738	# else
3739	AssertMsgFailed(("Physical address doesn't match! iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3740	# endif
3741	GCPtr += 512 * _2M;
3742	cErrors++;
3743	continue;
3744	}
3745
3746	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3747	if ( pPdpeDst->lm.u1User != PdpeSrc.lm.u1User
3748	\|\| pPdpeDst->lm.u1Write != PdpeSrc.lm.u1Write
3749	\|\| pPdpeDst->lm.u1NoExecute != PdpeSrc.lm.u1NoExecute)
3750	{
3751	AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3752	GCPtr += 512 * _2M;
3753	cErrors++;
3754	continue;
3755	}
3756	# endif
3757
3758	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3759	{
3760	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3761	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3762	GSTPD const *pPDSrc = pgmGstGet32bitPDPtr(&pVCpu->pgm.s);
3763	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3764	PCX86PD pPDDst = pgmShwGet32BitPDPtr(&pVCpu->pgm.s);
3765	# endif
3766	# endif /* PGM_GST_TYPE == PGM_TYPE_32BIT */
3767	/*
3768	* Iterate the shadow page directory.
3769	*/
3770	GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
3771	unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
3772
3773	for (;
3774	iPDDst < cPDEs;
3775	iPDDst++, GCPtr += cIncrement)
3776	{
3777	# if PGM_SHW_TYPE == PGM_TYPE_PAE
3778	const SHWPDE PdeDst = *pgmShwGetPaePDEPtr(pPGM, GCPtr);
3779	# else
3780	const SHWPDE PdeDst = pPDDst->a[iPDDst];
3781	# endif
3782	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
3783	{
3784	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
3785	if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
3786	{
3787	AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
3788	cErrors++;
3789	continue;
3790	}
3791	}
3792	else if ( (PdeDst.u & X86_PDE_P)
3793	\|\| ((PdeDst.u & (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY))
3794	)
3795	{
3796	HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
3797	PPGMPOOLPAGE pPoolPage = pgmPoolGetPage(pPool, HCPhysShw);
3798	if (!pPoolPage)
3799	{
3800	AssertMsgFailed(("Invalid page table address %RHp at %RGv! PdeDst=%#RX64\n",
3801	HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
3802	cErrors++;
3803	continue;
3804	}
3805	const SHWPT pPTDst = (const SHWPT )PGMPOOL_PAGE_2_PTR(pVM, pPoolPage);
3806
3807	if (PdeDst.u & (X86_PDE4M_PWT \| X86_PDE4M_PCD))
3808	{
3809	AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n",
3810	GCPtr, (uint64_t)PdeDst.u));
3811	cErrors++;
3812	}
3813
3814	if (PdeDst.u & (X86_PDE4M_G \| X86_PDE4M_D))
3815	{
3816	AssertMsgFailed(("4K PDE reserved flags at %RGv! PdeDst=%#RX64\n",
3817	GCPtr, (uint64_t)PdeDst.u));
3818	cErrors++;
3819	}
3820
3821	const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
3822	if (!PdeSrc.n.u1Present)
3823	{
3824	AssertMsgFailed(("Guest PDE at %RGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
3825	GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
3826	cErrors++;
3827	continue;
3828	}
3829
3830	if ( !PdeSrc.b.u1Size
3831	\|\| !fBigPagesSupported)
3832	{
3833	GCPhysGst = PdeSrc.u & GST_PDE_PG_MASK;
3834	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3835	GCPhysGst \|= (iPDDst & 1) * (PAGE_SIZE / 2);
3836	# endif
3837	}
3838	else
3839	{
3840	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3841	if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
3842	{
3843	AssertMsgFailed(("Guest PDE at %RGv is using PSE36 or similar! PdeSrc=%#RX64\n",
3844	GCPtr, (uint64_t)PdeSrc.u));
3845	cErrors++;
3846	continue;
3847	}
3848	# endif
3849	GCPhysGst = GST_GET_PDE_BIG_PG_GCPHYS(PdeSrc);
3850	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3851	GCPhysGst \|= GCPtr & RT_BIT(X86_PAGE_2M_SHIFT);
3852	# endif
3853	}
3854
3855	if ( pPoolPage->enmKind
3856	!= (!PdeSrc.b.u1Size \|\| !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
3857	{
3858	AssertMsgFailed(("Invalid shadow page table kind %d at %RGv! PdeSrc=%#RX64\n",
3859	pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
3860	cErrors++;
3861	}
3862
3863	PPGMPAGE pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
3864	if (!pPhysPage)
3865	{
3866	AssertMsgFailed(("Cannot find guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3867	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3868	cErrors++;
3869	continue;
3870	}
3871
3872	if (GCPhysGst != pPoolPage->GCPhys)
3873	{
3874	AssertMsgFailed(("GCPhysGst=%RGp != pPage->GCPhys=%RGp at %RGv\n",
3875	GCPhysGst, pPoolPage->GCPhys, GCPtr));
3876	cErrors++;
3877	continue;
3878	}
3879
3880	if ( !PdeSrc.b.u1Size
3881	\|\| !fBigPagesSupported)
3882	{
3883	/*
3884	* Page Table.
3885	*/
3886	const GSTPT *pPTSrc;
3887	rc = PGM_GCPHYS_2_PTR(pVM, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1), &pPTSrc);
3888	if (RT_FAILURE(rc))
3889	{
3890	AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3891	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3892	cErrors++;
3893	continue;
3894	}
3895	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/))
3896	!= (PdeDst.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/)))
3897	{
3898	/// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
3899	// (This problem will go away when/if we shadow multiple CR3s.)
3900	AssertMsgFailed(("4K PDE flags mismatch at %RGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3901	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3902	cErrors++;
3903	continue;
3904	}
3905	if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3906	{
3907	AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%RGv PdeDst=%#RX64\n",
3908	GCPtr, (uint64_t)PdeDst.u));
3909	cErrors++;
3910	continue;
3911	}
3912
3913	/* iterate the page table. */
3914	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3915	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3916	const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
3917	# else
3918	const unsigned offPTSrc = 0;
3919	# endif
3920	for (unsigned iPT = 0, off = 0;
3921	iPT < RT_ELEMENTS(pPTDst->a);
3922	iPT++, off += PAGE_SIZE)
3923	{
3924	const SHWPTE PteDst = pPTDst->a[iPT];
3925
3926	/* skip not-present entries. */
3927	if (!(PteDst.u & (X86_PTE_P \| PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
3928	continue;
3929	Assert(PteDst.n.u1Present);
3930
3931	const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
3932	if (!PteSrc.n.u1Present)
3933	{
3934	# ifdef IN_RING3
3935	PGMAssertHandlerAndFlagsInSync(pVM);
3936	PGMR3DumpHierarchyGC(pVM, cr3, cr4, (PdeSrc.u & GST_PDE_PG_MASK));
3937	# endif
3938	AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n",
3939	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u, pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
3940	(PdeSrc.u & GST_PDE_PG_MASK) + (iPT + offPTSrc)*sizeof(PteSrc)));
3941	cErrors++;
3942	continue;
3943	}
3944
3945	uint64_t fIgnoreFlags = GST_PTE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_G \| X86_PTE_D \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
3946	# if 1 /** @todo sync accessed bit properly... */
3947	fIgnoreFlags \|= X86_PTE_A;
3948	# endif
3949
3950	/* match the physical addresses */
3951	HCPhysShw = PteDst.u & SHW_PTE_PG_MASK;
3952	GCPhysGst = PteSrc.u & GST_PTE_PG_MASK;
3953
3954	# ifdef IN_RING3
3955	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3956	if (RT_FAILURE(rc))
3957	{
3958	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
3959	{
3960	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3961	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3962	cErrors++;
3963	continue;
3964	}
3965	}
3966	else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
3967	{
3968	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3969	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3970	cErrors++;
3971	continue;
3972	}
3973	# endif
3974
3975	pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
3976	if (!pPhysPage)
3977	{
3978	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
3979	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
3980	{
3981	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3982	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3983	cErrors++;
3984	continue;
3985	}
3986	# endif
3987	if (PteDst.n.u1Write)
3988	{
3989	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3990	GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3991	cErrors++;
3992	}
3993	fIgnoreFlags \|= X86_PTE_RW;
3994	}
3995	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
3996	{
3997	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage:%R[pgmpage] GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3998	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
3999	cErrors++;
4000	continue;
4001	}
4002
4003	/* flags */
4004	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4005	{
4006	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4007	{
4008	if (PteDst.n.u1Write)
4009	{
4010	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4011	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4012	cErrors++;
4013	continue;
4014	}
4015	fIgnoreFlags \|= X86_PTE_RW;
4016	}
4017	else
4018	{
4019	if (PteDst.n.u1Present)
4020	{
4021	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4022	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4023	cErrors++;
4024	continue;
4025	}
4026	fIgnoreFlags \|= X86_PTE_P;
4027	}
4028	}
4029	else
4030	{
4031	if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
4032	{
4033	if (PteDst.n.u1Write)
4034	{
4035	AssertMsgFailed(("!DIRTY page at %RGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
4036	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4037	cErrors++;
4038	continue;
4039	}
4040	if (!(PteDst.u & PGM_PTFLAGS_TRACK_DIRTY))
4041	{
4042	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4043	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4044	cErrors++;
4045	continue;
4046	}
4047	if (PteDst.n.u1Dirty)
4048	{
4049	AssertMsgFailed(("!DIRTY page at %RGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4050	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4051	cErrors++;
4052	}
4053	# if 0 /** @todo sync access bit properly... */
4054	if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
4055	{
4056	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4057	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4058	cErrors++;
4059	}
4060	fIgnoreFlags \|= X86_PTE_RW;
4061	# else
4062	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
4063	# endif
4064	}
4065	else if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
4066	{
4067	/* access bit emulation (not implemented). */
4068	if (PteSrc.n.u1Accessed \|\| PteDst.n.u1Present)
4069	{
4070	AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
4071	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4072	cErrors++;
4073	continue;
4074	}
4075	if (!PteDst.n.u1Accessed)
4076	{
4077	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
4078	GCPtr + off, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4079	cErrors++;
4080	}
4081	fIgnoreFlags \|= X86_PTE_P;
4082	}
4083	# ifdef DEBUG_sandervl
4084	fIgnoreFlags \|= X86_PTE_D \| X86_PTE_A;
4085	# endif
4086	}
4087
4088	if ( (PteSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
4089	&& (PteSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags)
4090	)
4091	{
4092	AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
4093	GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
4094	fIgnoreFlags, (uint64_t)PteSrc.u, (uint64_t)PteDst.u));
4095	cErrors++;
4096	continue;
4097	}
4098	} /* foreach PTE */
4099	}
4100	else
4101	{
4102	/*
4103	* Big Page.
4104	*/
4105	uint64_t fIgnoreFlags = X86_PDE_AVL_MASK \| GST_PDE_PG_MASK \| X86_PDE4M_G \| X86_PDE4M_D \| X86_PDE4M_PS \| X86_PDE4M_PWT \| X86_PDE4M_PCD;
4106	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
4107	{
4108	if (PdeDst.n.u1Write)
4109	{
4110	AssertMsgFailed(("!DIRTY page at %RGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4111	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4112	cErrors++;
4113	continue;
4114	}
4115	if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
4116	{
4117	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4118	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4119	cErrors++;
4120	continue;
4121	}
4122	# if 0 /** @todo sync access bit properly... */
4123	if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
4124	{
4125	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4126	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4127	cErrors++;
4128	}
4129	fIgnoreFlags \|= X86_PTE_RW;
4130	# else
4131	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
4132	# endif
4133	}
4134	else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4135	{
4136	/* access bit emulation (not implemented). */
4137	if (PdeSrc.b.u1Accessed \|\| PdeDst.n.u1Present)
4138	{
4139	AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4140	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4141	cErrors++;
4142	continue;
4143	}
4144	if (!PdeDst.n.u1Accessed)
4145	{
4146	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4147	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4148	cErrors++;
4149	}
4150	fIgnoreFlags \|= X86_PTE_P;
4151	}
4152
4153	if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
4154	{
4155	AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
4156	GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
4157	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4158	cErrors++;
4159	}
4160
4161	/* iterate the page table. */
4162	for (unsigned iPT = 0, off = 0;
4163	iPT < RT_ELEMENTS(pPTDst->a);
4164	iPT++, off += PAGE_SIZE, GCPhysGst += PAGE_SIZE)
4165	{
4166	const SHWPTE PteDst = pPTDst->a[iPT];
4167
4168	if (PteDst.u & PGM_PTFLAGS_TRACK_DIRTY)
4169	{
4170	AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
4171	GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4172	cErrors++;
4173	}
4174
4175	/* skip not-present entries. */
4176	if (!PteDst.n.u1Present) /** @todo deal with ALL handlers and CSAM !P pages! */
4177	continue;
4178
4179	fIgnoreFlags = X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT \| X86_PTE_D \| X86_PTE_A \| X86_PTE_G \| X86_PTE_PAE_NX;
4180
4181	/* match the physical addresses */
4182	HCPhysShw = PteDst.u & X86_PTE_PAE_PG_MASK;
4183
4184	# ifdef IN_RING3
4185	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4186	if (RT_FAILURE(rc))
4187	{
4188	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4189	{
4190	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4191	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4192	cErrors++;
4193	}
4194	}
4195	else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
4196	{
4197	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4198	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4199	cErrors++;
4200	continue;
4201	}
4202	# endif
4203	pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
4204	if (!pPhysPage)
4205	{
4206	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4207	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4208	{
4209	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4210	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4211	cErrors++;
4212	continue;
4213	}
4214	# endif
4215	if (PteDst.n.u1Write)
4216	{
4217	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4218	GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4219	cErrors++;
4220	}
4221	fIgnoreFlags \|= X86_PTE_RW;
4222	}
4223	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4224	{
4225	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage=%R[pgmpage] GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4226	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4227	cErrors++;
4228	continue;
4229	}
4230
4231	/* flags */
4232	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4233	{
4234	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4235	{
4236	if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
4237	{
4238	if (PteDst.n.u1Write)
4239	{
4240	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4241	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4242	cErrors++;
4243	continue;
4244	}
4245	fIgnoreFlags \|= X86_PTE_RW;
4246	}
4247	}
4248	else
4249	{
4250	if (PteDst.n.u1Present)
4251	{
4252	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4253	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4254	cErrors++;
4255	continue;
4256	}
4257	fIgnoreFlags \|= X86_PTE_P;
4258	}
4259	}
4260
4261	if ( (PdeSrc.u & ~fIgnoreFlags) != (PteDst.u & ~fIgnoreFlags)
4262	&& (PdeSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (PteDst.u & ~fIgnoreFlags) /* lazy phys handler dereg. */
4263	)
4264	{
4265	AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
4266	GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PteDst.u & ~fIgnoreFlags,
4267	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PteDst.u));
4268	cErrors++;
4269	continue;
4270	}
4271	} /* for each PTE */
4272	}
4273	}
4274	/* not present */
4275
4276	} /* for each PDE */
4277
4278	} /* for each PDPTE */
4279
4280	} /* for each PML4E */
4281
4282	# ifdef DEBUG
4283	if (cErrors)
4284	LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
4285	# endif
4286
4287	#endif /* GST == 32BIT, PAE or AMD64 */
4288	return cErrors;
4289
4290	#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT */
4291	}
4292	#endif /* VBOX_STRICT */
4293
4294
4295	/**
4296	* Sets up the CR3 for shadow paging
4297	*
4298	* @returns Strict VBox status code.
4299	* @retval VINF_SUCCESS.
4300	*
4301	* @param pVCpu The VMCPU handle.
4302	* @param GCPhysCR3 The physical address in the CR3 register.
4303	*/
4304	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3)
4305	{
4306	PVM pVM = pVCpu->CTX_SUFF(pVM);
4307
4308	/* Update guest paging info. */
4309	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
4310	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
4311	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
4312
4313	LogFlow(("MapCR3: %RGp\n", GCPhysCR3));
4314
4315	/*
4316	* Map the page CR3 points at.
4317	*/
4318	RTHCPTR HCPtrGuestCR3;
4319	RTHCPHYS HCPhysGuestCR3;
4320	pgmLock(pVM);
4321	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysCR3);
4322	AssertReturn(pPage, VERR_INTERNAL_ERROR_2);
4323	HCPhysGuestCR3 = PGM_PAGE_GET_HCPHYS(pPage);
4324	/** @todo this needs some reworking wrt. locking. */
4325	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4326	HCPtrGuestCR3 = NIL_RTHCPTR;
4327	int rc = VINF_SUCCESS;
4328	# else
4329	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhysCR3 & GST_CR3_PAGE_MASK, (void )&HCPtrGuestCR3); / @todo r=bird: This GCPhysCR3 masking isn't necessary. */
4330	# endif
4331	pgmUnlock(pVM);
4332	if (RT_SUCCESS(rc))
4333	{
4334	rc = PGMMap(pVM, (RTGCPTR)pVM->pgm.s.GCPtrCR3Mapping, HCPhysGuestCR3, PAGE_SIZE, 0);
4335	if (RT_SUCCESS(rc))
4336	{
4337	# ifdef IN_RC
4338	PGM_INVL_PG(pVCpu, pVM->pgm.s.GCPtrCR3Mapping);
4339	# endif
4340	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4341	pVCpu->pgm.s.pGst32BitPdR3 = (R3PTRTYPE(PX86PD))HCPtrGuestCR3;
4342	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4343	pVCpu->pgm.s.pGst32BitPdR0 = (R0PTRTYPE(PX86PD))HCPtrGuestCR3;
4344	# endif
4345	pVCpu->pgm.s.pGst32BitPdRC = (RCPTRTYPE(PX86PD))pVM->pgm.s.GCPtrCR3Mapping;
4346
4347	# elif PGM_GST_TYPE == PGM_TYPE_PAE
4348	unsigned off = GCPhysCR3 & GST_CR3_PAGE_MASK & PAGE_OFFSET_MASK;
4349	pVCpu->pgm.s.pGstPaePdptR3 = (R3PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4350	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4351	pVCpu->pgm.s.pGstPaePdptR0 = (R0PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4352	# endif
4353	pVCpu->pgm.s.pGstPaePdptRC = (RCPTRTYPE(PX86PDPT))((RCPTRTYPE(uint8_t *))pVM->pgm.s.GCPtrCR3Mapping + off);
4354	Log(("Cached mapping %RRv\n", pVCpu->pgm.s.pGstPaePdptRC));
4355
4356	/*
4357	* Map the 4 PDs too.
4358	*/
4359	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(&pVCpu->pgm.s);
4360	RTGCPTR GCPtr = pVM->pgm.s.GCPtrCR3Mapping + PAGE_SIZE;
4361	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++, GCPtr += PAGE_SIZE)
4362	{
4363	if (pGuestPDPT->a[i].n.u1Present)
4364	{
4365	RTHCPTR HCPtr;
4366	RTHCPHYS HCPhys;
4367	RTGCPHYS GCPhys = pGuestPDPT->a[i].u & X86_PDPE_PG_MASK;
4368	pgmLock(pVM);
4369	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
4370	AssertReturn(pPage, VERR_INTERNAL_ERROR_2);
4371	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
4372	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4373	HCPtr = NIL_RTHCPTR;
4374	int rc2 = VINF_SUCCESS;
4375	# else
4376	int rc2 = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, (void **)&HCPtr);
4377	# endif
4378	pgmUnlock(pVM);
4379	if (RT_SUCCESS(rc2))
4380	{
4381	rc = PGMMap(pVM, GCPtr, HCPhys, PAGE_SIZE, 0);
4382	AssertRCReturn(rc, rc);
4383
4384	pVCpu->pgm.s.apGstPaePDsR3[i] = (R3PTRTYPE(PX86PDPAE))HCPtr;
4385	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4386	pVCpu->pgm.s.apGstPaePDsR0[i] = (R0PTRTYPE(PX86PDPAE))HCPtr;
4387	# endif
4388	pVCpu->pgm.s.apGstPaePDsRC[i] = (RCPTRTYPE(PX86PDPAE))GCPtr;
4389	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys;
4390	# ifdef IN_RC
4391	PGM_INVL_PG(pVCpu, GCPtr);
4392	# endif
4393	continue;
4394	}
4395	AssertMsgFailed(("pgmR3Gst32BitMapCR3: rc2=%d GCPhys=%RGp i=%d\n", rc2, GCPhys, i));
4396	}
4397
4398	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4399	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4400	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4401	# endif
4402	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4403	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4404	# ifdef IN_RC
4405	PGM_INVL_PG(pVCpu, GCPtr); /** @todo this shouldn't be necessary? */
4406	# endif
4407	}
4408
4409	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
4410	pVCpu->pgm.s.pGstAmd64Pml4R3 = (R3PTRTYPE(PX86PML4))HCPtrGuestCR3;
4411	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4412	pVCpu->pgm.s.pGstAmd64Pml4R0 = (R0PTRTYPE(PX86PML4))HCPtrGuestCR3;
4413	# endif
4414	# endif
4415	}
4416	else
4417	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4418	}
4419	else
4420	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4421
4422	#else /* prot/real stub */
4423	int rc = VINF_SUCCESS;
4424	#endif
4425
4426	/* Update shadow paging info for guest modes with paging (32, pae, 64). */
4427	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4428	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4429	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64) \
4430	&& ( PGM_GST_TYPE != PGM_TYPE_REAL \
4431	&& PGM_GST_TYPE != PGM_TYPE_PROT))
4432
4433	Assert(!HWACCMIsNestedPagingActive(pVM));
4434
4435	/*
4436	* Update the shadow root page as well since that's not fixed.
4437	*/
4438	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4439	PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
4440	uint32_t iOldShwUserTable = pVCpu->pgm.s.iShwUserTable;
4441	uint32_t iOldShwUser = pVCpu->pgm.s.iShwUser;
4442	PPGMPOOLPAGE pNewShwPageCR3;
4443
4444	pgmLock(pVM);
4445
4446	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4447	if (pPool->cDirtyPages)
4448	pgmPoolResetDirtyPages(pVM);
4449	# endif
4450
4451	Assert(!(GCPhysCR3 >> (PAGE_SHIFT + 32)));
4452	rc = pgmPoolAlloc(pVM, GCPhysCR3 & GST_CR3_PAGE_MASK, BTH_PGMPOOLKIND_ROOT, SHW_POOL_ROOT_IDX, GCPhysCR3 >> PAGE_SHIFT, &pNewShwPageCR3, true /* lock page */);
4453	AssertFatalRC(rc);
4454	rc = VINF_SUCCESS;
4455
4456	# ifdef IN_RC
4457	/*
4458	* WARNING! We can't deal with jumps to ring 3 in the code below as the
4459	* state will be inconsistent! Flush important things now while
4460	* we still can and then make sure there are no ring-3 calls.
4461	*/
4462	REMNotifyHandlerPhysicalFlushIfAlmostFull(pVM, pVCpu);
4463	VMMRZCallRing3Disable(pVCpu);
4464	# endif
4465
4466	pVCpu->pgm.s.iShwUser = SHW_POOL_ROOT_IDX;
4467	pVCpu->pgm.s.iShwUserTable = GCPhysCR3 >> PAGE_SHIFT;
4468	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3) = pNewShwPageCR3;
4469	# ifdef IN_RING0
4470	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4471	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4472	# elif defined(IN_RC)
4473	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4474	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4475	# else
4476	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4477	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4478	# endif
4479
4480	# ifndef PGM_WITHOUT_MAPPINGS
4481	/*
4482	* Apply all hypervisor mappings to the new CR3.
4483	* Note that SyncCR3 will be executed in case CR3 is changed in a guest paging mode; this will
4484	* make sure we check for conflicts in the new CR3 root.
4485	*/
4486	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
4487	Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL) \|\| VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
4488	# endif
4489	rc = pgmMapActivateCR3(pVM, pNewShwPageCR3);
4490	AssertRCReturn(rc, rc);
4491	# endif
4492
4493	/* Set the current hypervisor CR3. */
4494	CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
4495	SELMShadowCR3Changed(pVM, pVCpu);
4496
4497	# ifdef IN_RC
4498	/* NOTE: The state is consistent again. */
4499	VMMRZCallRing3Enable(pVCpu);
4500	# endif
4501
4502	/* Clean up the old CR3 root. */
4503	if ( pOldShwPageCR3
4504	&& pOldShwPageCR3 != pNewShwPageCR3 /* @todo can happen due to incorrect syncing between REM & PGM; find the real cause */)
4505	{
4506	Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
4507	# ifndef PGM_WITHOUT_MAPPINGS
4508	/* Remove the hypervisor mappings from the shadow page table. */
4509	pgmMapDeactivateCR3(pVM, pOldShwPageCR3);
4510	# endif
4511	/* Mark the page as unlocked; allow flushing again. */
4512	pgmPoolUnlockPage(pPool, pOldShwPageCR3);
4513
4514	pgmPoolFreeByPage(pPool, pOldShwPageCR3, iOldShwUser, iOldShwUserTable);
4515	}
4516	pgmUnlock(pVM);
4517	# endif
4518
4519	return rc;
4520	}
4521
4522	/**
4523	* Unmaps the shadow CR3.
4524	*
4525	* @returns VBox status, no specials.
4526	* @param pVCpu The VMCPU handle.
4527	*/
4528	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu)
4529	{
4530	LogFlow(("UnmapCR3\n"));
4531
4532	int rc = VINF_SUCCESS;
4533	PVM pVM = pVCpu->CTX_SUFF(pVM);
4534
4535	/*
4536	* Update guest paging info.
4537	*/
4538	#if PGM_GST_TYPE == PGM_TYPE_32BIT
4539	pVCpu->pgm.s.pGst32BitPdR3 = 0;
4540	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4541	pVCpu->pgm.s.pGst32BitPdR0 = 0;
4542	# endif
4543	pVCpu->pgm.s.pGst32BitPdRC = 0;
4544
4545	#elif PGM_GST_TYPE == PGM_TYPE_PAE
4546	pVCpu->pgm.s.pGstPaePdptR3 = 0;
4547	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4548	pVCpu->pgm.s.pGstPaePdptR0 = 0;
4549	# endif
4550	pVCpu->pgm.s.pGstPaePdptRC = 0;
4551	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
4552	{
4553	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4554	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4555	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4556	# endif
4557	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4558	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4559	}
4560
4561	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
4562	pVCpu->pgm.s.pGstAmd64Pml4R3 = 0;
4563	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4564	pVCpu->pgm.s.pGstAmd64Pml4R0 = 0;
4565	# endif
4566
4567	#else /* prot/real mode stub */
4568	/* nothing to do */
4569	#endif
4570
4571	#if !defined(IN_RC) /* In RC we rely on MapCR3 to do the shadow part for us at a safe time */
4572	/*
4573	* Update shadow paging info.
4574	*/
4575	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4576	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4577	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64))
4578
4579	# if PGM_GST_TYPE != PGM_TYPE_REAL
4580	Assert(!HWACCMIsNestedPagingActive(pVM));
4581	# endif
4582
4583	pgmLock(pVM);
4584
4585	# ifndef PGM_WITHOUT_MAPPINGS
4586	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4587	/* Remove the hypervisor mappings from the shadow page table. */
4588	pgmMapDeactivateCR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4589	# endif
4590
4591	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4592	{
4593	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4594
4595	Assert(pVCpu->pgm.s.iShwUser != PGMPOOL_IDX_NESTED_ROOT);
4596
4597	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4598	if (pPool->cDirtyPages)
4599	pgmPoolResetDirtyPages(pVM);
4600	# endif
4601
4602	/* Mark the page as unlocked; allow flushing again. */
4603	pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4604
4605	pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), pVCpu->pgm.s.iShwUser, pVCpu->pgm.s.iShwUserTable);
4606	pVCpu->pgm.s.pShwPageCR3R3 = 0;
4607	pVCpu->pgm.s.pShwPageCR3R0 = 0;
4608	pVCpu->pgm.s.pShwPageCR3RC = 0;
4609	pVCpu->pgm.s.iShwUser = 0;
4610	pVCpu->pgm.s.iShwUserTable = 0;
4611	}
4612	pgmUnlock(pVM);
4613	# endif
4614	#endif /* !IN_RC*/
4615
4616	return rc;
4617	}
4618

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 24874

Download in other formats: