PGMAllBth.h@ 93718

Last change on this file since 93718 was 93716, checked in by vboxsync, 3 years ago
VMM/PGM: Moved the physical handler allocation off the hyper heap and into its own slab, changing the it to the 'hardened' avl tree code. bugref:10093
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 189.2 KB

Line
1	/* $Id: PGMAllBth.h 93716 2022-02-14 10:36:21Z vboxsync $ */
2	/** @file
3	* VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4	*
5	* @remarks Extended page tables (intel) are built with PGM_GST_TYPE set to
6	* PGM_TYPE_PROT (and PGM_SHW_TYPE set to PGM_TYPE_EPT).
7	* bird: WTF does this mean these days? Looking at PGMAll.cpp it's
8	*
9	* @remarks This file is one big \#ifdef-orgy!
10	*
11	*/
12
13	/*
14	* Copyright (C) 2006-2022 Oracle Corporation
15	*
16	* This file is part of VirtualBox Open Source Edition (OSE), as
17	* available from http://www.virtualbox.org. This file is free software;
18	* you can redistribute it and/or modify it under the terms of the GNU
19	* General Public License (GPL) as published by the Free Software
20	* Foundation, in version 2 as it comes in the "COPYING" file of the
21	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
22	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
23	*/
24
25	#ifdef _MSC_VER
26	/** @todo we're generating unnecessary code in nested/ept shadow mode and for
27	* real/prot-guest+RC mode. */
28	# pragma warning(disable: 4505)
29	#endif
30
31
32	/*********************************************************************************************************************************
33	* Internal Functions *
34	*********************************************************************************************************************************/
35	RT_C_DECLS_BEGIN
36	PGM_BTH_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3);
37	#ifndef IN_RING3
38	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken);
39	#endif
40	PGM_BTH_DECL(int, InvalidatePage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage);
41	static int PGM_BTH_NAME(SyncPage)(PVMCPUCC pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr);
42	static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPUCC pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc, RTGCPTR GCPtrPage);
43	static int PGM_BTH_NAME(SyncPT)(PVMCPUCC pVCpu, unsigned iPD, PGSTPD pPDSrc, RTGCPTR GCPtrPage);
44	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
45	static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst);
46	#else
47	static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage, unsigned iPTDst);
48	#endif
49	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPUCC pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uErr);
50	PGM_BTH_DECL(int, PrefetchPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage);
51	PGM_BTH_DECL(int, SyncCR3)(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
52	#ifdef VBOX_STRICT
53	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
54	#endif
55	PGM_BTH_DECL(int, MapCR3)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3);
56	PGM_BTH_DECL(int, UnmapCR3)(PVMCPUCC pVCpu);
57
58	#ifdef IN_RING3
59	PGM_BTH_DECL(int, Relocate)(PVMCPUCC pVCpu, RTGCPTR offDelta);
60	#endif
61	RT_C_DECLS_END
62
63
64
65
66	/*
67	* Filter out some illegal combinations of guest and shadow paging, so we can
68	* remove redundant checks inside functions.
69	*/
70	#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE \
71	&& !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
72	# error "Invalid combination; PAE guest implies PAE shadow"
73	#endif
74
75	#if (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
76	&& !( PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64 \
77	\|\| PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \|\| PGM_SHW_TYPE == PGM_TYPE_NONE)
78	# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
79	#endif
80
81	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) \
82	&& !( PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
83	\|\| PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \|\| PGM_SHW_TYPE == PGM_TYPE_NONE)
84	# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
85	#endif
86
87	#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE) \
88	\|\| (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
89	# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
90	#endif
91
92
93	/**
94	* Enters the shadow+guest mode.
95	*
96	* @returns VBox status code.
97	* @param pVCpu The cross context virtual CPU structure.
98	* @param GCPhysCR3 The physical address from the CR3 register.
99	*/
100	PGM_BTH_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3)
101	{
102	/* Here we deal with allocation of the root shadow page table for real and protected mode during mode switches;
103	* Other modes rely on MapCR3/UnmapCR3 to setup the shadow root page tables.
104	*/
105	#if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
106	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
107	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64) \
108	&& ( PGM_GST_TYPE == PGM_TYPE_REAL \
109	\|\| PGM_GST_TYPE == PGM_TYPE_PROT))
110
111	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
112
113	Assert((HMIsNestedPagingActive(pVM) \|\| VM_IS_NEM_ENABLED(pVM)) == pVM->pgm.s.fNestedPaging);
114	Assert(!pVM->pgm.s.fNestedPaging);
115
116	PGM_LOCK_VOID(pVM);
117	/* Note: we only really need shadow paging in real and protected mode for VT-x and AMD-V (excluding nested paging/EPT modes),
118	* but any calls to GC need a proper shadow page setup as well.
119	*/
120	/* Free the previous root mapping if still active. */
121	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
122	PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
123	if (pOldShwPageCR3)
124	{
125	Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
126
127	/* Mark the page as unlocked; allow flushing again. */
128	pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
129
130	pgmPoolFreeByPage(pPool, pOldShwPageCR3, NIL_PGMPOOL_IDX, UINT32_MAX);
131	pVCpu->pgm.s.pShwPageCR3R3 = NIL_RTR3PTR;
132	pVCpu->pgm.s.pShwPageCR3R0 = NIL_RTR0PTR;
133	}
134
135	/* construct a fake address. */
136	GCPhysCR3 = RT_BIT_64(63);
137	PPGMPOOLPAGE pNewShwPageCR3;
138	int rc = pgmPoolAlloc(pVM, GCPhysCR3, BTH_PGMPOOLKIND_ROOT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
139	NIL_PGMPOOL_IDX, UINT32_MAX, false /fLockPage/,
140	&pNewShwPageCR3);
141	AssertRCReturn(rc, rc);
142
143	pVCpu->pgm.s.pShwPageCR3R3 = pgmPoolConvertPageToR3(pPool, pNewShwPageCR3);
144	pVCpu->pgm.s.pShwPageCR3R0 = pgmPoolConvertPageToR0(pPool, pNewShwPageCR3);
145
146	/* Mark the page as locked; disallow flushing. */
147	pgmPoolLockPage(pPool, pNewShwPageCR3);
148
149	/* Set the current hypervisor CR3. */
150	CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
151
152	PGM_UNLOCK(pVM);
153	return rc;
154	#else
155	NOREF(pVCpu); NOREF(GCPhysCR3);
156	return VINF_SUCCESS;
157	#endif
158	}
159
160
161	#ifndef IN_RING3
162
163	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
164	/**
165	* Deal with a guest page fault.
166	*
167	* @returns Strict VBox status code.
168	* @retval VINF_EM_RAW_GUEST_TRAP
169	* @retval VINF_EM_RAW_EMULATE_INSTR
170	*
171	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
172	* @param pWalk The guest page table walk result.
173	* @param uErr The error code.
174	*/
175	PGM_BTH_DECL(VBOXSTRICTRC, Trap0eHandlerGuestFault)(PVMCPUCC pVCpu, PPGMPTWALK pWalk, RTGCUINT uErr)
176	{
177	/*
178	* Calc the error code for the guest trap.
179	*/
180	uint32_t uNewErr = GST_IS_NX_ACTIVE(pVCpu)
181	? uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID)
182	: uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US);
183	if ( pWalk->fRsvdError
184	\|\| pWalk->fBadPhysAddr)
185	{
186	uNewErr \|= X86_TRAP_PF_RSVD \| X86_TRAP_PF_P;
187	Assert(!pWalk->fNotPresent);
188	}
189	else if (!pWalk->fNotPresent)
190	uNewErr \|= X86_TRAP_PF_P;
191	TRPMSetErrorCode(pVCpu, uNewErr);
192
193	LogFlow(("Guest trap; cr2=%RGv uErr=%RGv lvl=%d\n", pWalk->GCPtr, uErr, pWalk->uLevel));
194	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2GuestTrap; });
195	return VINF_EM_RAW_GUEST_TRAP;
196	}
197	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
198
199
200	#if !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
201	/**
202	* Deal with a guest page fault.
203	*
204	* The caller has taken the PGM lock.
205	*
206	* @returns Strict VBox status code.
207	*
208	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
209	* @param uErr The error code.
210	* @param pRegFrame The register frame.
211	* @param pvFault The fault address.
212	* @param pPage The guest page at @a pvFault.
213	* @param pWalk The guest page table walk result.
214	* @param pGstWalk The guest paging-mode specific walk information.
215	* @param pfLockTaken PGM lock taken here or not (out). This is true
216	* when we're called.
217	*/
218	static VBOXSTRICTRC PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame,
219	RTGCPTR pvFault, PPGMPAGE pPage, bool *pfLockTaken
220	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \|\| defined(DOXYGEN_RUNNING)
221	, PPGMPTWALK pWalk
222	, PGSTPTWALK pGstWalk
223	# endif
224	)
225	{
226	# if !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
227	GSTPDE const PdeSrcDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
228	# endif
229	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
230	VBOXSTRICTRC rcStrict;
231
232	if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
233	{
234	/*
235	* Physical page access handler.
236	*/
237	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
238	const RTGCPHYS GCPhysFault = pWalk->GCPhys;
239	# else
240	const RTGCPHYS GCPhysFault = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault);
241	# endif
242	PPGMPHYSHANDLER pCur;
243	rcStrict = pgmHandlerPhysicalLookup(pVM, GCPhysFault, &pCur);
244	if (RT_SUCCESS(rcStrict))
245	{
246	PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
247
248	# ifdef PGM_SYNC_N_PAGES
249	/*
250	* If the region is write protected and we got a page not present fault, then sync
251	* the pages. If the fault was caused by a read, then restart the instruction.
252	* In case of write access continue to the GC write handler.
253	*
254	* ASSUMES that there is only one handler per page or that they have similar write properties.
255	*/
256	if ( !(uErr & X86_TRAP_PF_P)
257	&& pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
258	{
259	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
260	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
261	# else
262	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
263	# endif
264	if ( RT_FAILURE(rcStrict)
265	\|\| !(uErr & X86_TRAP_PF_RW)
266	\|\| rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
267	{
268	AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
269	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
270	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
271	return rcStrict;
272	}
273	}
274	# endif
275	# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
276	/*
277	* If the access was not thru a #PF(RSVD\|...) resync the page.
278	*/
279	if ( !(uErr & X86_TRAP_PF_RSVD)
280	&& pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
281	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
282	&& (pWalk->fEffective & (PGM_PTATTRS_W_MASK \| PGM_PTATTRS_US_MASK))
283	== PGM_PTATTRS_W_MASK /** @todo Remove pGstWalk->Core.fEffectiveUS and X86_PTE_US further down in the sync code. */
284	# endif
285	)
286	{
287	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
288	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
289	# else
290	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
291	# endif
292	if ( RT_FAILURE(rcStrict)
293	\|\| rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
294	{
295	AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
296	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
297	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
298	return rcStrict;
299	}
300	}
301	# endif
302
303	AssertMsg( pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
304	\|\| (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW)),
305	("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enmKind=%d\n",
306	pvFault, GCPhysFault, pPage, uErr, pCurType->enmKind));
307	if (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
308	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysWrite);
309	else
310	{
311	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAll);
312	if (uErr & X86_TRAP_PF_RSVD) STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAllOpt);
313	}
314
315	if (pCurType->pfnPfHandler)
316	{
317	STAM_PROFILE_START(&pCur->Stat, h);
318
319	if (pCurType->fKeepPgmLock)
320	{
321	rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pRegFrame, pvFault, GCPhysFault,
322	!pCurType->fRing0DevInsIdx ? pCur->uUser
323	: (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser));
324
325	STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
326	}
327	else
328	{
329	uint64_t const uUser = !pCurType->fRing0DevInsIdx ? pCur->uUser
330	: (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser);
331	PGM_UNLOCK(pVM);
332	*pfLockTaken = false;
333
334	rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pRegFrame, pvFault, GCPhysFault, uUser);
335
336	STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
337	}
338	}
339	else
340	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
341
342	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2HndPhys; });
343	return rcStrict;
344	}
345	AssertMsgReturn(rcStrict == VERR_NOT_FOUND, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
346	}
347
348	/*
349	* There is a handled area of the page, but this fault doesn't belong to it.
350	* We must emulate the instruction.
351	*
352	* To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
353	* we first check if this was a page-not-present fault for a page with only
354	* write access handlers. Restart the instruction if it wasn't a write access.
355	*/
356	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersUnhandled);
357
358	if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
359	&& !(uErr & X86_TRAP_PF_P))
360	{
361	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
362	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
363	# else
364	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
365	# endif
366	if ( RT_FAILURE(rcStrict)
367	\|\| rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE
368	\|\| !(uErr & X86_TRAP_PF_RW))
369	{
370	AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
371	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
372	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
373	return rcStrict;
374	}
375	}
376
377	/** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
378	* It's writing to an unhandled part of the LDT page several million times.
379	*/
380	rcStrict = PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault);
381	LogFlow(("PGM: PGMInterpretInstruction -> rcStrict=%d pPage=%R[pgmpage]\n", VBOXSTRICTRC_VAL(rcStrict), pPage));
382	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2HndUnhandled; });
383	return rcStrict;
384	} /* if any kind of handler */
385	# endif /* !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE*/
386
387
388	/**
389	* \#PF Handler for raw-mode guest execution.
390	*
391	* @returns VBox status code (appropriate for trap handling and GC return).
392	*
393	* @param pVCpu The cross context virtual CPU structure.
394	* @param uErr The trap error code.
395	* @param pRegFrame Trap register frame.
396	* @param pvFault The fault address.
397	* @param pfLockTaken PGM lock taken here or not (out)
398	*/
399	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken)
400	{
401	PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
402
403	*pfLockTaken = false;
404
405	# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \
406	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
407	&& !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
408	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
409	&& PGM_SHW_TYPE != PGM_TYPE_NONE
410	int rc;
411
412	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
413	/*
414	* Walk the guest page translation tables and check if it's a guest fault.
415	*/
416	PGMPTWALK Walk;
417	GSTPTWALK GstWalk;
418	rc = PGM_GST_NAME(Walk)(pVCpu, pvFault, &Walk, &GstWalk);
419	if (RT_FAILURE_NP(rc))
420	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &Walk, uErr));
421
422	/* assert some GstWalk sanity. */
423	# if PGM_GST_TYPE == PGM_TYPE_AMD64
424	/AssertMsg(GstWalk.Pml4e.u == GstWalk.pPml4e->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pml4e.u, (uint64_t)GstWalk.pPml4e->u)); - not always true with SMP guests. /
425	# endif
426	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
427	/AssertMsg(GstWalk.Pdpe.u == GstWalk.pPdpe->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pdpe.u, (uint64_t)GstWalk.pPdpe->u)); - ditto /
428	# endif
429	/AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); - ditto /
430	/AssertMsg(GstWalk.Core.fBigPage \|\| GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); - ditto /
431	Assert(Walk.fSucceeded);
432	Assert(Walk.fEffective & PGM_PTATTRS_R_MASK);
433
434	if (uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID))
435	{
436	if ( ( (uErr & X86_TRAP_PF_RW)
437	&& !(Walk.fEffective & PGM_PTATTRS_W_MASK)
438	&& ( (uErr & X86_TRAP_PF_US)
439	\|\| CPUMIsGuestR0WriteProtEnabled(pVCpu)) )
440	\|\| ((uErr & X86_TRAP_PF_US) && !(Walk.fEffective & PGM_PTATTRS_US_MASK))
441	\|\| ((uErr & X86_TRAP_PF_ID) && (Walk.fEffective & PGM_PTATTRS_NX_MASK))
442	)
443	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &Walk, uErr));
444	}
445
446	/* Take the big lock now before we update flags. */
447	*pfLockTaken = true;
448	PGM_LOCK_VOID(pVM);
449
450	/*
451	* Set the accessed and dirty flags.
452	*/
453	/** @todo Should probably use cmpxchg logic here as we're potentially racing
454	* other CPUs in SMP configs. (the lock isn't enough, since we take it
455	* after walking and the page tables could be stale already) */
456	# if PGM_GST_TYPE == PGM_TYPE_AMD64
457	if (!(GstWalk.Pml4e.u & X86_PML4E_A))
458	{
459	GstWalk.Pml4e.u \|= X86_PML4E_A;
460	GST_ATOMIC_OR(&GstWalk.pPml4e->u, X86_PML4E_A);
461	}
462	if (!(GstWalk.Pdpe.u & X86_PDPE_A))
463	{
464	GstWalk.Pdpe.u \|= X86_PDPE_A;
465	GST_ATOMIC_OR(&GstWalk.pPdpe->u, X86_PDPE_A);
466	}
467	# endif
468	if (Walk.fBigPage)
469	{
470	Assert(GstWalk.Pde.u & X86_PDE_PS);
471	if (uErr & X86_TRAP_PF_RW)
472	{
473	if ((GstWalk.Pde.u & (X86_PDE4M_A \| X86_PDE4M_D)) != (X86_PDE4M_A \| X86_PDE4M_D))
474	{
475	GstWalk.Pde.u \|= X86_PDE4M_A \| X86_PDE4M_D;
476	GST_ATOMIC_OR(&GstWalk.pPde->u, X86_PDE4M_A \| X86_PDE4M_D);
477	}
478	}
479	else
480	{
481	if (!(GstWalk.Pde.u & X86_PDE4M_A))
482	{
483	GstWalk.Pde.u \|= X86_PDE4M_A;
484	GST_ATOMIC_OR(&GstWalk.pPde->u, X86_PDE4M_A);
485	}
486	}
487	}
488	else
489	{
490	Assert(!(GstWalk.Pde.u & X86_PDE_PS));
491	if (!(GstWalk.Pde.u & X86_PDE_A))
492	{
493	GstWalk.Pde.u \|= X86_PDE_A;
494	GST_ATOMIC_OR(&GstWalk.pPde->u, X86_PDE_A);
495	}
496
497	if (uErr & X86_TRAP_PF_RW)
498	{
499	# ifdef VBOX_WITH_STATISTICS
500	if (GstWalk.Pte.u & X86_PTE_D)
501	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageAlreadyDirty));
502	else
503	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtiedPage));
504	# endif
505	if ((GstWalk.Pte.u & (X86_PTE_A \| X86_PTE_D)) != (X86_PTE_A \| X86_PTE_D))
506	{
507	GstWalk.Pte.u \|= X86_PTE_A \| X86_PTE_D;
508	GST_ATOMIC_OR(&GstWalk.pPte->u, X86_PTE_A \| X86_PTE_D);
509	}
510	}
511	else
512	{
513	if (!(GstWalk.Pte.u & X86_PTE_A))
514	{
515	GstWalk.Pte.u \|= X86_PTE_A;
516	GST_ATOMIC_OR(&GstWalk.pPte->u, X86_PTE_A);
517	}
518	}
519	Assert(GstWalk.Pte.u == GstWalk.pPte->u);
520	}
521	#if 0
522	/* Disabling this since it's not reliable for SMP, see @bugref{10092#c22}. */
523	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u \|\| GstWalk.pPte->u == GstWalk.pPde->u,
524	("%RX64 %RX64 pPte=%p pPde=%p Pte=%RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u, GstWalk.pPte, GstWalk.pPde, (uint64_t)GstWalk.pPte->u));
525	#endif
526
527	# else /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
528	GSTPDE const PdeSrcDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A}; /** @todo eliminate this */
529
530	/* Take the big lock now. */
531	*pfLockTaken = true;
532	PGM_LOCK_VOID(pVM);
533	# endif /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
534
535	# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
536	/*
537	* If it is a reserved bit fault we know that it is an MMIO (access
538	* handler) related fault and can skip some 200 lines of code.
539	*/
540	if (uErr & X86_TRAP_PF_RSVD)
541	{
542	Assert(uErr & X86_TRAP_PF_P);
543	PPGMPAGE pPage;
544	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
545	rc = pgmPhysGetPageEx(pVM, Walk.GCPhys, &pPage);
546	if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
547	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
548	pfLockTaken, &Walk, &GstWalk));
549	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
550	# else
551	rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault), &pPage);
552	if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
553	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
554	pfLockTaken));
555	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
556	# endif
557	AssertRC(rc);
558	PGM_INVL_PG(pVCpu, pvFault);
559	return rc; /* Restart with the corrected entry. */
560	}
561	# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
562
563	/*
564	* Fetch the guest PDE, PDPE and PML4E.
565	*/
566	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
567	const unsigned iPDDst = pvFault >> SHW_PD_SHIFT;
568	PX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
569
570	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
571	const unsigned iPDDst = (pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK; /* pPDDst index, not used with the pool. */
572	PX86PDPAE pPDDst;
573	# if PGM_GST_TYPE == PGM_TYPE_PAE
574	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, GstWalk.Pdpe.u, &pPDDst);
575	# else
576	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, X86_PDPE_P, &pPDDst); /* RW, US and A are reserved in PAE mode. */
577	# endif
578	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
579
580	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
581	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
582	PX86PDPAE pPDDst;
583	# if PGM_GST_TYPE == PGM_TYPE_PROT /* (AMD-V nested paging) */
584	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A,
585	X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A, &pPDDst);
586	# else
587	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, GstWalk.Pml4e.u, GstWalk.Pdpe.u, &pPDDst);
588	# endif
589	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
590
591	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
592	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
593	PEPTPD pPDDst;
594	rc = pgmShwGetEPTPDPtr(pVCpu, pvFault, NULL, &pPDDst);
595	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
596	# endif
597	Assert(pPDDst);
598
599	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
600	/*
601	* Dirty page handling.
602	*
603	* If we successfully correct the write protection fault due to dirty bit
604	* tracking, then return immediately.
605	*/
606	if (uErr & X86_TRAP_PF_RW) /* write fault? */
607	{
608	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyBitTracking), a);
609	rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, &pPDDst->a[iPDDst], GstWalk.pPde, pvFault);
610	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyBitTracking), a);
611	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
612	{
613	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0
614	= rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
615	? &pVCpu->pgm.s.Stats.StatRZTrap0eTime2DirtyAndAccessed
616	: &pVCpu->pgm.s.Stats.StatRZTrap0eTime2GuestTrap; });
617	Log8(("Trap0eHandler: returns VINF_SUCCESS\n"));
618	return VINF_SUCCESS;
619	}
620	#ifdef DEBUG_bird
621	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u \|\| GstWalk.pPte->u == GstWalk.pPde->u \|\| pVM->cCpus > 1, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); // - triggers with smp w7 guests.
622	AssertMsg(Walk.fBigPage \|\| GstWalk.Pte.u == GstWalk.pPte->u \|\| pVM->cCpus > 1, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); // - ditto.
623	#endif
624	}
625
626	# if 0 /* rarely useful; leave for debugging. */
627	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0ePD[iPDSrc]);
628	# endif
629	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
630
631	/*
632	* A common case is the not-present error caused by lazy page table syncing.
633	*
634	* It is IMPORTANT that we weed out any access to non-present shadow PDEs
635	* here so we can safely assume that the shadow PT is present when calling
636	* SyncPage later.
637	*
638	* On failure, we ASSUME that SyncPT is out of memory or detected some kind
639	* of mapping conflict and defer to SyncCR3 in R3.
640	* (Again, we do NOT support access handlers for non-present guest pages.)
641	*
642	*/
643	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
644	Assert(GstWalk.Pde.u & X86_PDE_P);
645	# endif
646	if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
647	&& !SHW_PDE_IS_P(pPDDst->a[iPDDst]))
648	{
649	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2SyncPT; });
650	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
651	LogFlow(("=>SyncPT %04x = %08RX64\n", (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, (uint64_t)GstWalk.Pde.u));
652	rc = PGM_BTH_NAME(SyncPT)(pVCpu, (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, GstWalk.pPd, pvFault);
653	# else
654	LogFlow(("=>SyncPT pvFault=%RGv\n", pvFault));
655	rc = PGM_BTH_NAME(SyncPT)(pVCpu, 0, NULL, pvFault);
656	# endif
657	if (RT_SUCCESS(rc))
658	return rc;
659	Log(("SyncPT: %RGv failed!! rc=%Rrc\n", pvFault, rc));
660	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
661	return VINF_PGM_SYNC_CR3;
662	}
663
664	/*
665	* Check if this fault address is flagged for special treatment,
666	* which means we'll have to figure out the physical address and
667	* check flags associated with it.
668	*
669	* ASSUME that we can limit any special access handling to pages
670	* in page tables which the guest believes to be present.
671	*/
672	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
673	RTGCPHYS GCPhys = Walk.GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
674	# else
675	RTGCPHYS GCPhys = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK);
676	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
677	PPGMPAGE pPage;
678	rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
679	if (RT_FAILURE(rc))
680	{
681	/*
682	* When the guest accesses invalid physical memory (e.g. probing
683	* of RAM or accessing a remapped MMIO range), then we'll fall
684	* back to the recompiler to emulate the instruction.
685	*/
686	LogFlow(("PGM #PF: pgmPhysGetPageEx(%RGp) failed with %Rrc\n", GCPhys, rc));
687	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersInvalid);
688	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2InvalidPhys; });
689	return VINF_EM_RAW_EMULATE_INSTR;
690	}
691
692	/*
693	* Any handlers for this page?
694	*/
695	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
696	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
697	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage, pfLockTaken,
698	&Walk, &GstWalk));
699	# else
700	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage, pfLockTaken));
701	# endif
702
703	/*
704	* We are here only if page is present in Guest page tables and
705	* trap is not handled by our handlers.
706	*
707	* Check it for page out-of-sync situation.
708	*/
709	if (!(uErr & X86_TRAP_PF_P))
710	{
711	/*
712	* Page is not present in our page tables. Try to sync it!
713	*/
714	if (uErr & X86_TRAP_PF_US)
715	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncUser));
716	else /* supervisor */
717	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
718
719	if (PGM_PAGE_IS_BALLOONED(pPage))
720	{
721	/* Emulate reads from ballooned pages as they are not present in
722	our shadow page tables. (Required for e.g. Solaris guests; soft
723	ecc, random nr generator.) */
724	rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
725	LogFlow(("PGM: PGMInterpretInstruction balloon -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
726	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncBallloon));
727	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Ballooned; });
728	return rc;
729	}
730
731	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
732	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
733	# else
734	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
735	# endif
736	if (RT_SUCCESS(rc))
737	{
738	/* The page was successfully synced, return to the guest. */
739	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSync; });
740	return VINF_SUCCESS;
741	}
742	}
743	else /* uErr & X86_TRAP_PF_P: */
744	{
745	/*
746	* Write protected pages are made writable when the guest makes the
747	* first write to it. This happens for pages that are shared, write
748	* monitored or not yet allocated.
749	*
750	* We may also end up here when CR0.WP=0 in the guest.
751	*
752	* Also, a side effect of not flushing global PDEs are out of sync
753	* pages due to physical monitored regions, that are no longer valid.
754	* Assume for now it only applies to the read/write flag.
755	*/
756	if (uErr & X86_TRAP_PF_RW)
757	{
758	/*
759	* Check if it is a read-only page.
760	*/
761	if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
762	{
763	Log(("PGM #PF: Make writable: %RGp %R[pgmpage] pvFault=%RGp uErr=%#x\n", GCPhys, pPage, pvFault, uErr));
764	Assert(!PGM_PAGE_IS_ZERO(pPage));
765	AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
766	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2MakeWritable; });
767
768	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
769	if (rc != VINF_SUCCESS)
770	{
771	AssertMsg(rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("%Rrc\n", rc));
772	return rc;
773	}
774	if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
775	return VINF_EM_NO_MEMORY;
776	}
777
778	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
779	/*
780	* Check to see if we need to emulate the instruction if CR0.WP=0.
781	*/
782	if ( !(Walk.fEffective & PGM_PTATTRS_W_MASK)
783	&& (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG
784	&& CPUMGetGuestCPL(pVCpu) < 3)
785	{
786	Assert((uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RW \| X86_TRAP_PF_P));
787
788	/*
789	* The Netware WP0+RO+US hack.
790	*
791	* Netware sometimes(/always?) runs with WP0. It has been observed doing
792	* excessive write accesses to pages which are mapped with US=1 and RW=0
793	* while WP=0. This causes a lot of exits and extremely slow execution.
794	* To avoid trapping and emulating every write here, we change the shadow
795	* page table entry to map it as US=0 and RW=1 until user mode tries to
796	* access it again (see further below). We count these shadow page table
797	* changes so we can avoid having to clear the page pool every time the WP
798	* bit changes to 1 (see PGMCr0WpEnabled()).
799	*/
800	# if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) && 1
801	if ( (Walk.fEffective & (PGM_PTATTRS_W_MASK \| PGM_PTATTRS_US_MASK)) == PGM_PTATTRS_US_MASK
802	&& (Walk.fBigPage \|\| (GstWalk.Pde.u & X86_PDE_RW))
803	&& pVM->cCpus == 1 /* Sorry, no go on SMP. Add CFGM option? */)
804	{
805	Log(("PGM #PF: Netware WP0+RO+US hack: pvFault=%RGp uErr=%#x (big=%d)\n", pvFault, uErr, Walk.fBigPage));
806	rc = pgmShwMakePageSupervisorAndWritable(pVCpu, pvFault, Walk.fBigPage, PGM_MK_PG_IS_WRITE_FAULT);
807	if (rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3)
808	{
809	PGM_INVL_PG(pVCpu, pvFault);
810	pVCpu->pgm.s.cNetwareWp0Hacks++;
811	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Wp0RoUsHack; });
812	return rc;
813	}
814	AssertMsg(RT_FAILURE_NP(rc), ("%Rrc\n", rc));
815	Log(("pgmShwMakePageSupervisorAndWritable(%RGv) failed with rc=%Rrc - ignored\n", pvFault, rc));
816	}
817	# endif
818
819	/* Interpret the access. */
820	rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
821	Log(("PGM #PF: WP0 emulation (pvFault=%RGp uErr=%#x cpl=%d fBig=%d fEffUs=%d)\n", pvFault, uErr, CPUMGetGuestCPL(pVCpu), Walk.fBigPage, !!(Walk.fEffective & PGM_PTATTRS_US_MASK)));
822	if (RT_SUCCESS(rc))
823	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eWPEmulInRZ);
824	else
825	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eWPEmulToR3);
826	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2WPEmulation; });
827	return rc;
828	}
829	# endif
830	/// @todo count the above case; else
831	if (uErr & X86_TRAP_PF_US)
832	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncUserWrite));
833	else /* supervisor */
834	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
835
836	/*
837	* Sync the page.
838	*
839	* Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
840	* page is not present, which is not true in this case.
841	*/
842	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
843	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
844	# else
845	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
846	# endif
847	if (RT_SUCCESS(rc))
848	{
849	/*
850	* Page was successfully synced, return to guest but invalidate
851	* the TLB first as the page is very likely to be in it.
852	*/
853	# if PGM_SHW_TYPE == PGM_TYPE_EPT
854	HMInvalidatePhysPage(pVM, (RTGCPHYS)pvFault);
855	# else
856	PGM_INVL_PG(pVCpu, pvFault);
857	# endif
858	# ifdef VBOX_STRICT
859	PGMPTWALK GstPageWalk;
860	GstPageWalk.GCPhys = RTGCPHYS_MAX;
861	if (!pVM->pgm.s.fNestedPaging)
862	{
863	rc = PGMGstGetPage(pVCpu, pvFault, &GstPageWalk);
864	AssertMsg(RT_SUCCESS(rc) && ((GstPageWalk.fEffective & X86_PTE_RW) \|\| ((CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG && CPUMGetGuestCPL(pVCpu) < 3)), ("rc=%Rrc fPageGst=%RX64\n", rc, GstPageWalk.fEffective));
865	LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GstPageWalk.GCPhys, GstPageWalk.fEffective));
866	}
867	# if 0 /* Bogus! Triggers incorrectly with w7-64 and later for the SyncPage case: "Pde at %RGv changed behind our back?" */
868	uint64_t fPageShw = 0;
869	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
870	AssertMsg((RT_SUCCESS(rc) && (fPageShw & X86_PTE_RW)) \|\| pVM->cCpus > 1 /* new monitor can be installed/page table flushed between the trap exit and PGMTrap0eHandler */,
871	("rc=%Rrc fPageShw=%RX64 GCPhys2=%RGp fPageGst=%RX64 pvFault=%RGv\n", rc, fPageShw, GstPageWalk.GCPhys, fPageGst, pvFault));
872	# endif
873	# endif /* VBOX_STRICT */
874	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndObs; });
875	return VINF_SUCCESS;
876	}
877	}
878	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
879	/*
880	* Check for Netware WP0+RO+US hack from above and undo it when user
881	* mode accesses the page again.
882	*/
883	else if ( (Walk.fEffective & (PGM_PTATTRS_W_MASK \| PGM_PTATTRS_US_MASK)) == PGM_PTATTRS_US_MASK
884	&& (Walk.fBigPage \|\| (GstWalk.Pde.u & X86_PDE_RW))
885	&& pVCpu->pgm.s.cNetwareWp0Hacks > 0
886	&& (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG
887	&& CPUMGetGuestCPL(pVCpu) == 3
888	&& pVM->cCpus == 1
889	)
890	{
891	Log(("PGM #PF: Undo netware WP0+RO+US hack: pvFault=%RGp uErr=%#x\n", pvFault, uErr));
892	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
893	if (RT_SUCCESS(rc))
894	{
895	PGM_INVL_PG(pVCpu, pvFault);
896	pVCpu->pgm.s.cNetwareWp0Hacks--;
897	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Wp0RoUsUnhack; });
898	return VINF_SUCCESS;
899	}
900	}
901	# endif /* PGM_WITH_PAGING */
902
903	/** @todo else: why are we here? */
904
905	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && defined(VBOX_STRICT)
906	/*
907	* Check for VMM page flags vs. Guest page flags consistency.
908	* Currently only for debug purposes.
909	*/
910	if (RT_SUCCESS(rc))
911	{
912	/* Get guest page flags. */
913	PGMPTWALK GstPageWalk;
914	int rc2 = PGMGstGetPage(pVCpu, pvFault, &GstPageWalk);
915	if (RT_SUCCESS(rc2))
916	{
917	uint64_t fPageShw = 0;
918	rc2 = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
919
920	#if 0
921	/*
922	* Compare page flags.
923	* Note: we have AVL, A, D bits desynced.
924	*/
925	AssertMsg( (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK))
926	== (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK))
927	\|\| ( pVCpu->pgm.s.cNetwareWp0Hacks > 0
928	&& (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US))
929	== (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US))
930	&& (fPageShw & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_RW
931	&& (fPageGst & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_US),
932	("Page flags mismatch! pvFault=%RGv uErr=%x GCPhys=%RGp fPageShw=%RX64 fPageGst=%RX64 rc=%d\n",
933	pvFault, (uint32_t)uErr, GCPhys, fPageShw, fPageGst, rc));
934	01:01:15.623511 00:08:43.266063 Expression: (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) \|\| ( pVCpu->pgm.s.cNetwareWp0Hacks > 0 && (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US)) && (fPageShw & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_RW && (fPageGst & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_US)
935	01:01:15.623511 00:08:43.266064 Location : e:\vbox\svn\trunk\srcPage flags mismatch! pvFault=fffff801b0d7b000 uErr=11 GCPhys=0000000019b52000 fPageShw=0 fPageGst=77b0000000000121 rc=0
936
937	01:01:15.625516 00:08:43.268051 Expression: (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) \|\| ( pVCpu->pgm.s.cNetwareWp0Hacks > 0 && (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US)) && (fPageShw & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_RW && (fPageGst & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_US)
938	01:01:15.625516 00:08:43.268051 Location :
939	e:\vbox\svn\trunk\srcPage flags mismatch!
940	pvFault=fffff801b0d7b000
941	uErr=11 X86_TRAP_PF_ID \| X86_TRAP_PF_P
942	GCPhys=0000000019b52000
943	fPageShw=0
944	fPageGst=77b0000000000121
945	rc=0
946	#endif
947
948	}
949	else
950	AssertMsgFailed(("PGMGstGetPage rc=%Rrc\n", rc));
951	}
952	else
953	AssertMsgFailed(("PGMGCGetPage rc=%Rrc\n", rc));
954	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && VBOX_STRICT */
955	}
956
957
958	/*
959	* If we get here it is because something failed above, i.e. most like guru
960	* meditiation time.
961	*/
962	LogRel(("%s: returns rc=%Rrc pvFault=%RGv uErr=%RX64 cs:rip=%04x:%08RX64\n",
963	__PRETTY_FUNCTION__, rc, pvFault, (uint64_t)uErr, pRegFrame->cs.Sel, pRegFrame->rip));
964	return rc;
965
966	# else /* Nested paging, EPT except PGM_GST_TYPE = PROT, NONE. */
967	NOREF(uErr); NOREF(pRegFrame); NOREF(pvFault);
968	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
969	return VERR_PGM_NOT_USED_IN_MODE;
970	# endif
971	}
972
973	#endif /* !IN_RING3 */
974
975
976	/**
977	* Emulation of the invlpg instruction.
978	*
979	*
980	* @returns VBox status code.
981	*
982	* @param pVCpu The cross context virtual CPU structure.
983	* @param GCPtrPage Page to invalidate.
984	*
985	* @remark ASSUMES that the guest is updating before invalidating. This order
986	* isn't required by the CPU, so this is speculative and could cause
987	* trouble.
988	* @remark No TLB shootdown is done on any other VCPU as we assume that
989	* invlpg emulation is the only reason for calling this function.
990	* (The guest has to shoot down TLB entries on other CPUs itself)
991	* Currently true, but keep in mind!
992	*
993	* @todo Clean this up! Most of it is (or should be) no longer necessary as we catch all page table accesses.
994	* Should only be required when PGMPOOL_WITH_OPTIMIZED_DIRTY_PT is active (PAE or AMD64 (for now))
995	*/
996	PGM_BTH_DECL(int, InvalidatePage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
997	{
998	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
999	&& !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
1000	&& PGM_SHW_TYPE != PGM_TYPE_NONE
1001	int rc;
1002	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1003	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1004
1005	PGM_LOCK_ASSERT_OWNER(pVM);
1006
1007	LogFlow(("InvalidatePage %RGv\n", GCPtrPage));
1008
1009	/*
1010	* Get the shadow PD entry and skip out if this PD isn't present.
1011	* (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
1012	*/
1013	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1014	const unsigned iPDDst = (uint32_t)GCPtrPage >> SHW_PD_SHIFT;
1015	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1016
1017	/* Fetch the pgm pool shadow descriptor. */
1018	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1019	# ifdef IN_RING3 /* Possible we didn't resync yet when called from REM. */
1020	if (!pShwPde)
1021	{
1022	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1023	return VINF_SUCCESS;
1024	}
1025	# else
1026	Assert(pShwPde);
1027	# endif
1028
1029	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1030	const unsigned iPdpt = (uint32_t)GCPtrPage >> X86_PDPT_SHIFT;
1031	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
1032
1033	/* If the shadow PDPE isn't present, then skip the invalidate. */
1034	# ifdef IN_RING3 /* Possible we didn't resync yet when called from REM. */
1035	if (!pPdptDst \|\| !(pPdptDst->a[iPdpt].u & X86_PDPE_P))
1036	# else
1037	if (!(pPdptDst->a[iPdpt].u & X86_PDPE_P))
1038	# endif
1039	{
1040	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1041	PGM_INVL_PG(pVCpu, GCPtrPage);
1042	return VINF_SUCCESS;
1043	}
1044
1045	/* Fetch the pgm pool shadow descriptor. */
1046	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
1047	AssertReturn(pShwPde, VERR_PGM_POOL_GET_PAGE_FAILED);
1048
1049	PX86PDPAE pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1050	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1051	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1052
1053	# else /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1054	/* PML4 */
1055	/const unsigned iPml4 = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;/
1056	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1057	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1058	PX86PDPAE pPDDst;
1059	PX86PDPT pPdptDst;
1060	PX86PML4E pPml4eDst;
1061	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eDst, &pPdptDst, &pPDDst);
1062	if (rc != VINF_SUCCESS)
1063	{
1064	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT \|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
1065	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1066	PGM_INVL_PG(pVCpu, GCPtrPage);
1067	return VINF_SUCCESS;
1068	}
1069	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1070	Assert(pPDDst);
1071	Assert(pPdptDst->a[iPdpt].u & X86_PDPE_P);
1072
1073	/* Fetch the pgm pool shadow descriptor. */
1074	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & SHW_PDPE_PG_MASK);
1075	Assert(pShwPde);
1076
1077	# endif /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1078
1079	const SHWPDE PdeDst = *pPdeDst;
1080	if (!(PdeDst.u & X86_PDE_P))
1081	{
1082	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1083	PGM_INVL_PG(pVCpu, GCPtrPage);
1084	return VINF_SUCCESS;
1085	}
1086
1087	/*
1088	* Get the guest PD entry and calc big page.
1089	*/
1090	# if PGM_GST_TYPE == PGM_TYPE_32BIT
1091	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
1092	const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
1093	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
1094	# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1095	unsigned iPDSrc = 0;
1096	# if PGM_GST_TYPE == PGM_TYPE_PAE
1097	X86PDPE PdpeSrcIgn;
1098	PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrcIgn);
1099	# else /* AMD64 */
1100	PX86PML4E pPml4eSrcIgn;
1101	X86PDPE PdpeSrcIgn;
1102	PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrcIgn, &PdpeSrcIgn, &iPDSrc);
1103	# endif
1104	GSTPDE PdeSrc;
1105
1106	if (pPDSrc)
1107	PdeSrc = pPDSrc->a[iPDSrc];
1108	else
1109	PdeSrc.u = 0;
1110	# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1111	const bool fWasBigPage = RT_BOOL(PdeDst.u & PGM_PDFLAGS_BIG_PAGE);
1112	const bool fIsBigPage = (PdeSrc.u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu);
1113	if (fWasBigPage != fIsBigPage)
1114	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1115
1116	# ifdef IN_RING3
1117	/*
1118	* If a CR3 Sync is pending we may ignore the invalidate page operation
1119	* depending on the kind of sync and if it's a global page or not.
1120	* This doesn't make sense in GC/R0 so we'll skip it entirely there.
1121	*/
1122	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
1123	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
1124	\|\| ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
1125	&& fIsBigPage
1126	&& (PdeSrc.u & X86_PDE4M_G)
1127	)
1128	)
1129	# else
1130	if (VM_FF_IS_ANY_SET(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
1131	# endif
1132	{
1133	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
1134	return VINF_SUCCESS;
1135	}
1136	# endif /* IN_RING3 */
1137
1138	/*
1139	* Deal with the Guest PDE.
1140	*/
1141	rc = VINF_SUCCESS;
1142	if (PdeSrc.u & X86_PDE_P)
1143	{
1144	Assert( (PdeSrc.u & X86_PDE_US) == (PdeDst.u & X86_PDE_US)
1145	&& ((PdeSrc.u & X86_PDE_RW) \|\| !(PdeDst.u & X86_PDE_RW) \|\| pVCpu->pgm.s.cNetwareWp0Hacks > 0));
1146	if (!fIsBigPage)
1147	{
1148	/*
1149	* 4KB - page.
1150	*/
1151	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1152	RTGCPHYS GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
1153
1154	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1155	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1156	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2)));
1157	# endif
1158	if (pShwPage->GCPhys == GCPhys)
1159	{
1160	/* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
1161	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1162
1163	PGSTPT pPTSrc;
1164	rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
1165	if (RT_SUCCESS(rc))
1166	{
1167	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1168	GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1169	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1170	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1171	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
1172	GCPtrPage, PteSrc.u & X86_PTE_P,
1173	(PteSrc.u & PdeSrc.u & X86_PTE_RW),
1174	(PteSrc.u & PdeSrc.u & X86_PTE_US),
1175	(uint64_t)PteSrc.u,
1176	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
1177	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
1178	}
1179	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage4KBPages));
1180	PGM_INVL_PG(pVCpu, GCPtrPage);
1181	}
1182	else
1183	{
1184	/*
1185	* The page table address changed.
1186	*/
1187	LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n",
1188	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
1189	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1190	SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
1191	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1192	PGM_INVL_VCPU_TLBS(pVCpu);
1193	}
1194	}
1195	else
1196	{
1197	/*
1198	* 2/4MB - page.
1199	*/
1200	/* Before freeing the page, check if anything really changed. */
1201	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1202	RTGCPHYS GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
1203	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1204	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1205	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
1206	# endif
1207	if ( pShwPage->GCPhys == GCPhys
1208	&& pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1209	{
1210	/* ASSUMES a the given bits are identical for 4M and normal PDEs */
1211	/** @todo This test is wrong as it cannot check the G bit!
1212	* FIXME */
1213	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US))
1214	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US))
1215	&& ( (PdeSrc.u & X86_PDE4M_D) /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1216	\|\| (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1217	{
1218	LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1219	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage4MBPagesSkip));
1220	return VINF_SUCCESS;
1221	}
1222	}
1223
1224	/*
1225	* Ok, the page table is present and it's been changed in the guest.
1226	* If we're in host context, we'll just mark it as not present taking the lazy approach.
1227	* We could do this for some flushes in GC too, but we need an algorithm for
1228	* deciding which 4MB pages containing code likely to be executed very soon.
1229	*/
1230	LogFlow(("InvalidatePage: Out-of-sync PD at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1231	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1232	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1233	SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
1234	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage4MBPages));
1235	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
1236	}
1237	}
1238	else
1239	{
1240	/*
1241	* Page directory is not present, mark shadow PDE not present.
1242	*/
1243	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1244	SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
1245	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePagePDNPs));
1246	PGM_INVL_PG(pVCpu, GCPtrPage);
1247	}
1248	return rc;
1249
1250	#else /* guest real and protected mode, nested + ept, none. */
1251	/* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1252	NOREF(pVCpu); NOREF(GCPtrPage);
1253	return VINF_SUCCESS;
1254	#endif
1255	}
1256
1257	#if PGM_SHW_TYPE != PGM_TYPE_NONE
1258
1259	/**
1260	* Update the tracking of shadowed pages.
1261	*
1262	* @param pVCpu The cross context virtual CPU structure.
1263	* @param pShwPage The shadow page.
1264	* @param HCPhys The physical page we is being dereferenced.
1265	* @param iPte Shadow PTE index
1266	* @param GCPhysPage Guest physical address (only valid if pShwPage->fDirty is set)
1267	*/
1268	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPUCC pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys, uint16_t iPte,
1269	RTGCPHYS GCPhysPage)
1270	{
1271	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1272
1273	# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1274	&& PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1275	&& (PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
1276
1277	/* Use the hint we retrieved from the cached guest PT. */
1278	if (pShwPage->fDirty)
1279	{
1280	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1281
1282	Assert(pShwPage->cPresent);
1283	Assert(pPool->cPresent);
1284	pShwPage->cPresent--;
1285	pPool->cPresent--;
1286
1287	PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysPage);
1288	AssertRelease(pPhysPage);
1289	pgmTrackDerefGCPhys(pPool, pShwPage, pPhysPage, iPte);
1290	return;
1291	}
1292	# else
1293	NOREF(GCPhysPage);
1294	# endif
1295
1296	STAM_PROFILE_START(&pVM->pgm.s.Stats.StatTrackDeref, a);
1297	LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1298
1299	/** @todo If this turns out to be a bottle neck (very likely) two things can be done:
1300	* 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1301	* 2. write protect all shadowed pages. I.e. implement caching.
1302	*/
1303	/** @todo duplicated in the 2nd half of pgmPoolTracDerefGCPhysHint */
1304
1305	/*
1306	* Find the guest address.
1307	*/
1308	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
1309	pRam;
1310	pRam = pRam->CTX_SUFF(pNext))
1311	{
1312	unsigned iPage = pRam->cb >> GUEST_PAGE_SHIFT;
1313	while (iPage-- > 0)
1314	{
1315	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1316	{
1317	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1318
1319	Assert(pShwPage->cPresent);
1320	Assert(pPool->cPresent);
1321	pShwPage->cPresent--;
1322	pPool->cPresent--;
1323
1324	pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage], iPte);
1325	STAM_PROFILE_STOP(&pVM->pgm.s.Stats.StatTrackDeref, a);
1326	return;
1327	}
1328	}
1329	}
1330
1331	for (;;)
1332	AssertReleaseMsgFailed(("HCPhys=%RHp wasn't found!\n", HCPhys));
1333	}
1334
1335
1336	/**
1337	* Update the tracking of shadowed pages.
1338	*
1339	* @param pVCpu The cross context virtual CPU structure.
1340	* @param pShwPage The shadow page.
1341	* @param u16 The top 16-bit of the pPage->HCPhys.
1342	* @param pPage Pointer to the guest page. this will be modified.
1343	* @param iPTDst The index into the shadow table.
1344	*/
1345	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVMCPUCC pVCpu, PPGMPOOLPAGE pShwPage, uint16_t u16,
1346	PPGMPAGE pPage, const unsigned iPTDst)
1347	{
1348	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1349
1350	/*
1351	* Just deal with the simple first time here.
1352	*/
1353	if (!u16)
1354	{
1355	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackVirgin);
1356	u16 = PGMPOOL_TD_MAKE(1, pShwPage->idx);
1357	/* Save the page table index. */
1358	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, iPTDst);
1359	}
1360	else
1361	u16 = pgmPoolTrackPhysExtAddref(pVM, pPage, u16, pShwPage->idx, iPTDst);
1362
1363	/* write back */
1364	Log2(("SyncPageWorkerTrackAddRef: u16=%#x->%#x iPTDst=%#x\n", u16, PGM_PAGE_GET_TRACKING(pPage), iPTDst));
1365	PGM_PAGE_SET_TRACKING(pVM, pPage, u16);
1366
1367	/* update statistics. */
1368	pVM->pgm.s.CTX_SUFF(pPool)->cPresent++;
1369	pShwPage->cPresent++;
1370	if (pShwPage->iFirstPresent > iPTDst)
1371	pShwPage->iFirstPresent = iPTDst;
1372	}
1373
1374
1375	/**
1376	* Modifies a shadow PTE to account for access handlers.
1377	*
1378	* @param pVM The cross context VM structure.
1379	* @param pPage The page in question.
1380	* @param fPteSrc The shadowed flags of the source PTE. Must include the
1381	* A (accessed) bit so it can be emulated correctly.
1382	* @param pPteDst The shadow PTE (output). This is temporary storage and
1383	* does not need to be set atomically.
1384	*/
1385	DECLINLINE(void) PGM_BTH_NAME(SyncHandlerPte)(PVMCC pVM, PCPGMPAGE pPage, uint64_t fPteSrc, PSHWPTE pPteDst)
1386	{
1387	NOREF(pVM); RT_NOREF_PV(fPteSrc);
1388
1389	/** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No.
1390	* Update: \#PF should deal with this before or after calling the handlers. It has all the info to do the job efficiently. */
1391	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1392	{
1393	LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark read-only\n", pPage));
1394	# if PGM_SHW_TYPE == PGM_TYPE_EPT
1395	pPteDst->u = PGM_PAGE_GET_HCPHYS(pPage) \| EPT_E_READ \| EPT_E_EXECUTE \| EPT_E_MEMTYPE_WB \| EPT_E_IGNORE_PAT;
1396	# else
1397	if (fPteSrc & X86_PTE_A)
1398	{
1399	SHW_PTE_SET(*pPteDst, fPteSrc \| PGM_PAGE_GET_HCPHYS(pPage));
1400	SHW_PTE_SET_RO(*pPteDst);
1401	}
1402	else
1403	SHW_PTE_SET(*pPteDst, 0);
1404	# endif
1405	}
1406	# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
1407	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
1408	else if ( PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
1409	&& ( BTH_IS_NP_ACTIVE(pVM)
1410	\|\| (fPteSrc & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_RW) /** @todo Remove X86_PTE_US here and pGstWalk->Core.fEffectiveUS before the sync page test. */
1411	# if PGM_SHW_TYPE == PGM_TYPE_AMD64
1412	&& pVM->pgm.s.fLessThan52PhysicalAddressBits
1413	# endif
1414	)
1415	{
1416	LogFlow(("SyncHandlerPte: MMIO page -> invalid \n"));
1417	# if PGM_SHW_TYPE == PGM_TYPE_EPT
1418	/* 25.2.3.1: Reserved physical address bit -> EPT Misconfiguration (exit 49) */
1419	pPteDst->u = pVM->pgm.s.HCPhysInvMmioPg
1420	/* 25.2.3.1: bits 2:0 = 010b -> EPT Misconfiguration (exit 49) */
1421	\| EPT_E_WRITE
1422	/* 25.2.3.1: leaf && 2:0 != 0 && u3Emt in {2, 3, 7} -> EPT Misconfiguration */
1423	\| EPT_E_MEMTYPE_INVALID_3;
1424	# else
1425	/* Set high page frame bits that MBZ (bankers on PAE, CPU dependent on AMD64). */
1426	SHW_PTE_SET(*pPteDst, pVM->pgm.s.HCPhysInvMmioPg \| X86_PTE_PAE_MBZ_MASK_NO_NX \| X86_PTE_P);
1427	# endif
1428	}
1429	# endif
1430	# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
1431	else
1432	{
1433	LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark not present\n", pPage));
1434	SHW_PTE_SET(*pPteDst, 0);
1435	}
1436	/** @todo count these kinds of entries. */
1437	}
1438
1439
1440	/**
1441	* Creates a 4K shadow page for a guest page.
1442	*
1443	* For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1444	* physical address. The PdeSrc argument only the flags are used. No page
1445	* structured will be mapped in this function.
1446	*
1447	* @param pVCpu The cross context virtual CPU structure.
1448	* @param pPteDst Destination page table entry.
1449	* @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1450	* Can safely assume that only the flags are being used.
1451	* @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1452	* @param pShwPage Pointer to the shadow page.
1453	* @param iPTDst The index into the shadow table.
1454	*
1455	* @remark Not used for 2/4MB pages!
1456	*/
1457	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \|\| defined(DOXYGEN_RUNNING)
1458	static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc,
1459	PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1460	# else
1461	static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage,
1462	PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1463	# endif
1464	{
1465	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1466	RTGCPHYS GCPhysOldPage = NIL_RTGCPHYS;
1467
1468	# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1469	&& PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1470	&& (PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
1471
1472	if (pShwPage->fDirty)
1473	{
1474	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1475	PGSTPT pGstPT;
1476
1477	/* Note that iPTDst can be used to index the guest PT even in the pae/32bit combo as we copy only half the table; see pgmPoolAddDirtyPage. */
1478	pGstPT = (PGSTPT)&pPool->aDirtyPages[pShwPage->idxDirtyEntry].aPage[0];
1479	GCPhysOldPage = GST_GET_PTE_GCPHYS(pGstPT->a[iPTDst]);
1480	pGstPT->a[iPTDst].u = PteSrc.u;
1481	}
1482	# else
1483	Assert(!pShwPage->fDirty);
1484	# endif
1485
1486	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1487	if ( (PteSrc.u & X86_PTE_P)
1488	&& GST_IS_PTE_VALID(pVCpu, PteSrc))
1489	# endif
1490	{
1491	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1492	RTGCPHYS GCPhysPage = GST_GET_PTE_GCPHYS(PteSrc);
1493	# endif
1494	PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
1495
1496	/*
1497	* Find the ram range.
1498	*/
1499	PPGMPAGE pPage;
1500	int rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
1501	if (RT_SUCCESS(rc))
1502	{
1503	/* Ignore ballooned pages.
1504	Don't return errors or use a fatal assert here as part of a
1505	shadow sync range might included ballooned pages. */
1506	if (PGM_PAGE_IS_BALLOONED(pPage))
1507	{
1508	Assert(!SHW_PTE_IS_P(pPteDst)); /* @todo user tracking needs updating if this triggers. */
1509	return;
1510	}
1511
1512	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1513	/* Make the page writable if necessary. */
1514	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1515	&& ( PGM_PAGE_IS_ZERO(pPage)
1516	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1517	\|\| ( (PteSrc.u & X86_PTE_RW)
1518	# else
1519	\|\| ( 1
1520	# endif
1521	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1522	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1523	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1524	# endif
1525	# ifdef VBOX_WITH_PAGE_SHARING
1526	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
1527	# endif
1528	)
1529	)
1530	)
1531	{
1532	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhysPage);
1533	AssertRC(rc);
1534	}
1535	# endif
1536
1537	/*
1538	* Make page table entry.
1539	*/
1540	SHWPTE PteDst;
1541	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1542	uint64_t fGstShwPteFlags = GST_GET_PTE_SHW_FLAGS(pVCpu, PteSrc);
1543	# else
1544	uint64_t fGstShwPteFlags = X86_PTE_P \| X86_PTE_RW \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D;
1545	# endif
1546	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1547	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, fGstShwPteFlags, &PteDst);
1548	else
1549	{
1550	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1551	/*
1552	* If the page or page directory entry is not marked accessed,
1553	* we mark the page not present.
1554	*/
1555	if (!(PteSrc.u & X86_PTE_A) \|\| !(PdeSrc.u & X86_PDE_A))
1556	{
1557	LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1558	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,AccessedPage));
1559	SHW_PTE_SET(PteDst, 0);
1560	}
1561	/*
1562	* If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1563	* when the page is modified.
1564	*/
1565	else if (!(PteSrc.u & X86_PTE_D) && (PdeSrc.u & PteSrc.u & X86_PTE_RW))
1566	{
1567	AssertCompile(X86_PTE_RW == X86_PDE_RW);
1568	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPage));
1569	SHW_PTE_SET(PteDst,
1570	fGstShwPteFlags
1571	\| PGM_PAGE_GET_HCPHYS(pPage)
1572	\| PGM_PTFLAGS_TRACK_DIRTY);
1573	SHW_PTE_SET_RO(PteDst);
1574	}
1575	else
1576	# endif
1577	{
1578	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageSkipped));
1579	# if PGM_SHW_TYPE == PGM_TYPE_EPT
1580	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage)
1581	\| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE \| EPT_E_MEMTYPE_WB \| EPT_E_IGNORE_PAT;
1582	# else
1583	SHW_PTE_SET(PteDst, fGstShwPteFlags \| PGM_PAGE_GET_HCPHYS(pPage));
1584	# endif
1585	}
1586
1587	/*
1588	* Make sure only allocated pages are mapped writable.
1589	*/
1590	if ( SHW_PTE_IS_P_RW(PteDst)
1591	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1592	{
1593	/* Still applies to shared pages. */
1594	Assert(!PGM_PAGE_IS_ZERO(pPage));
1595	SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet. Why, isn't it? */
1596	Log3(("SyncPageWorker: write-protecting %RGp pPage=%R[pgmpage]at iPTDst=%d\n", GCPhysPage, pPage, iPTDst));
1597	}
1598	}
1599
1600	/*
1601	* Keep user track up to date.
1602	*/
1603	if (SHW_PTE_IS_P(PteDst))
1604	{
1605	if (!SHW_PTE_IS_P(*pPteDst))
1606	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1607	else if (SHW_PTE_GET_HCPHYS(*pPteDst) != SHW_PTE_GET_HCPHYS(PteDst))
1608	{
1609	Log2(("SyncPageWorker: deref! pPteDst=%RX64 PteDst=%RX64\n", SHW_PTE_LOG64(pPteDst), SHW_PTE_LOG64(PteDst)));
1610	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
1611	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1612	}
1613	}
1614	else if (SHW_PTE_IS_P(*pPteDst))
1615	{
1616	Log2(("SyncPageWorker: deref! pPteDst=%RX64\n", SHW_PTE_LOG64(pPteDst)));
1617	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
1618	}
1619
1620	/*
1621	* Update statistics and commit the entry.
1622	*/
1623	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1624	if (!(PteSrc.u & X86_PTE_G))
1625	pShwPage->fSeenNonGlobal = true;
1626	# endif
1627	SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
1628	return;
1629	}
1630
1631	/** @todo count these three different kinds. */
1632	Log2(("SyncPageWorker: invalid address in Pte\n"));
1633	}
1634	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1635	else if (!(PteSrc.u & X86_PTE_P))
1636	Log2(("SyncPageWorker: page not present in Pte\n"));
1637	else
1638	Log2(("SyncPageWorker: invalid Pte\n"));
1639	# endif
1640
1641	/*
1642	* The page is not present or the PTE is bad. Replace the shadow PTE by
1643	* an empty entry, making sure to keep the user tracking up to date.
1644	*/
1645	if (SHW_PTE_IS_P(*pPteDst))
1646	{
1647	Log2(("SyncPageWorker: deref! pPteDst=%RX64\n", SHW_PTE_LOG64(pPteDst)));
1648	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
1649	}
1650	SHW_PTE_ATOMIC_SET(*pPteDst, 0);
1651	}
1652
1653
1654	/**
1655	* Syncs a guest OS page.
1656	*
1657	* There are no conflicts at this point, neither is there any need for
1658	* page table allocations.
1659	*
1660	* When called in PAE or AMD64 guest mode, the guest PDPE shall be valid.
1661	* When called in AMD64 guest mode, the guest PML4E shall be valid.
1662	*
1663	* @returns VBox status code.
1664	* @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1665	* @param pVCpu The cross context virtual CPU structure.
1666	* @param PdeSrc Page directory entry of the guest.
1667	* @param GCPtrPage Guest context page address.
1668	* @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1669	* @param uErr Fault error (X86_TRAP_PF_*).
1670	*/
1671	static int PGM_BTH_NAME(SyncPage)(PVMCPUCC pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr)
1672	{
1673	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1674	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
1675	LogFlow(("SyncPage: GCPtrPage=%RGv cPages=%u uErr=%#x\n", GCPtrPage, cPages, uErr));
1676	RT_NOREF_PV(uErr); RT_NOREF_PV(cPages); RT_NOREF_PV(GCPtrPage);
1677
1678	PGM_LOCK_ASSERT_OWNER(pVM);
1679
1680	# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
1681	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
1682	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
1683	&& !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE)
1684
1685	/*
1686	* Assert preconditions.
1687	*/
1688	Assert(PdeSrc.u & X86_PDE_P);
1689	Assert(cPages);
1690	# if 0 /* rarely useful; leave for debugging. */
1691	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
1692	# endif
1693
1694	/*
1695	* Get the shadow PDE, find the shadow page table in the pool.
1696	*/
1697	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1698	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1699	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1700
1701	/* Fetch the pgm pool shadow descriptor. */
1702	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1703	Assert(pShwPde);
1704
1705	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1706	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1707	PPGMPOOLPAGE pShwPde = NULL;
1708	PX86PDPAE pPDDst;
1709
1710	/* Fetch the pgm pool shadow descriptor. */
1711	int rc2 = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
1712	AssertRCSuccessReturn(rc2, rc2);
1713	Assert(pShwPde);
1714
1715	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1716	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1717
1718	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1719	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1720	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1721	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
1722	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
1723
1724	int rc2 = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
1725	AssertRCSuccessReturn(rc2, rc2);
1726	Assert(pPDDst && pPdptDst);
1727	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1728	# endif
1729	SHWPDE PdeDst = *pPdeDst;
1730
1731	/*
1732	* - In the guest SMP case we could have blocked while another VCPU reused
1733	* this page table.
1734	* - With W7-64 we may also take this path when the A bit is cleared on
1735	* higher level tables (PDPE/PML4E). The guest does not invalidate the
1736	* relevant TLB entries. If we're write monitoring any page mapped by
1737	* the modified entry, we may end up here with a "stale" TLB entry.
1738	*/
1739	if (!(PdeDst.u & X86_PDE_P))
1740	{
1741	Log(("CPU%u: SyncPage: Pde at %RGv changed behind our back? (pPdeDst=%p/%RX64) uErr=%#x\n", pVCpu->idCpu, GCPtrPage, pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
1742	AssertMsg(pVM->cCpus > 1 \|\| (uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RW)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RW),
1743	("Unexpected missing PDE p=%p/%RX64 uErr=%#x\n", pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
1744	if (uErr & X86_TRAP_PF_P)
1745	PGM_INVL_PG(pVCpu, GCPtrPage);
1746	return VINF_SUCCESS; /* force the instruction to be executed again. */
1747	}
1748
1749	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1750	Assert(pShwPage);
1751
1752	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1753	/* Fetch the pgm pool shadow descriptor. */
1754	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
1755	Assert(pShwPde);
1756	# endif
1757
1758	/*
1759	* Check that the page is present and that the shadow PDE isn't out of sync.
1760	*/
1761	const bool fBigPage = (PdeSrc.u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu);
1762	const bool fPdeValid = !fBigPage ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc);
1763	RTGCPHYS GCPhys;
1764	if (!fBigPage)
1765	{
1766	GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
1767	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1768	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1769	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2)));
1770	# endif
1771	}
1772	else
1773	{
1774	GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
1775	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1776	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1777	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
1778	# endif
1779	}
1780	/** @todo This doesn't check the G bit of 2/4MB pages. FIXME */
1781	if ( fPdeValid
1782	&& pShwPage->GCPhys == GCPhys
1783	&& (PdeSrc.u & X86_PDE_P)
1784	&& (PdeSrc.u & X86_PDE_US) == (PdeDst.u & X86_PDE_US)
1785	&& ((PdeSrc.u & X86_PDE_RW) == (PdeDst.u & X86_PDE_RW) \|\| !(PdeDst.u & X86_PDE_RW))
1786	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
1787	&& ((PdeSrc.u & X86_PDE_PAE_NX) == (PdeDst.u & X86_PDE_PAE_NX) \|\| !GST_IS_NX_ACTIVE(pVCpu))
1788	# endif
1789	)
1790	{
1791	/*
1792	* Check that the PDE is marked accessed already.
1793	* Since we set the accessed bit before getting here on a #PF, this
1794	* check is only meant for dealing with non-#PF'ing paths.
1795	*/
1796	if (PdeSrc.u & X86_PDE_A)
1797	{
1798	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1799	if (!fBigPage)
1800	{
1801	/*
1802	* 4KB Page - Map the guest page table.
1803	*/
1804	PGSTPT pPTSrc;
1805	int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
1806	if (RT_SUCCESS(rc))
1807	{
1808	# ifdef PGM_SYNC_N_PAGES
1809	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
1810	if ( cPages > 1
1811	&& !(uErr & X86_TRAP_PF_P)
1812	&& !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
1813	{
1814	/*
1815	* This code path is currently only taken when the caller is PGMTrap0eHandler
1816	* for non-present pages!
1817	*
1818	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
1819	* deal with locality.
1820	*/
1821	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1822	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1823	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1824	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
1825	# else
1826	const unsigned offPTSrc = 0;
1827	# endif
1828	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
1829	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
1830	iPTDst = 0;
1831	else
1832	iPTDst -= PGM_SYNC_NR_PAGES / 2;
1833
1834	for (; iPTDst < iPTDstEnd; iPTDst++)
1835	{
1836	const PGSTPTE pPteSrc = &pPTSrc->a[offPTSrc + iPTDst];
1837
1838	if ( (pPteSrc->u & X86_PTE_P)
1839	&& !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
1840	{
1841	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(GST_PT_MASK << GST_PT_SHIFT))
1842	\| ((offPTSrc + iPTDst) << GUEST_PAGE_SHIFT);
1843	NOREF(GCPtrCurPage);
1844	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, *pPteSrc, pShwPage, iPTDst);
1845	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
1846	GCPtrCurPage, pPteSrc->u & X86_PTE_P,
1847	!!(pPteSrc->u & PdeSrc.u & X86_PTE_RW),
1848	!!(pPteSrc->u & PdeSrc.u & X86_PTE_US),
1849	(uint64_t)pPteSrc->u,
1850	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
1851	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
1852	}
1853	}
1854	}
1855	else
1856	# endif /* PGM_SYNC_N_PAGES */
1857	{
1858	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1859	GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1860	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1861	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1862	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
1863	GCPtrPage, PteSrc.u & X86_PTE_P,
1864	!!(PteSrc.u & PdeSrc.u & X86_PTE_RW),
1865	!!(PteSrc.u & PdeSrc.u & X86_PTE_US),
1866	(uint64_t)PteSrc.u,
1867	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
1868	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
1869	}
1870	}
1871	else /* MMIO or invalid page: emulated in #PF handler. */
1872	{
1873	LogFlow(("PGM_GCPHYS_2_PTR %RGp failed with %Rrc\n", GCPhys, rc));
1874	Assert(!SHW_PTE_IS_P(pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK]));
1875	}
1876	}
1877	else
1878	{
1879	/*
1880	* 4/2MB page - lazy syncing shadow 4K pages.
1881	* (There are many causes of getting here, it's no longer only CSAM.)
1882	*/
1883	/* Calculate the GC physical address of this 4KB shadow page. */
1884	GCPhys = PGM_A20_APPLY(pVCpu, GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc) \| (GCPtrPage & GST_BIG_PAGE_OFFSET_MASK));
1885	/* Find ram range. */
1886	PPGMPAGE pPage;
1887	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
1888	if (RT_SUCCESS(rc))
1889	{
1890	AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
1891
1892	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1893	/* Try to make the page writable if necessary. */
1894	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1895	&& ( PGM_PAGE_IS_ZERO(pPage)
1896	\|\| ( (PdeSrc.u & X86_PDE_RW)
1897	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1898	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1899	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1900	# endif
1901	# ifdef VBOX_WITH_PAGE_SHARING
1902	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
1903	# endif
1904	)
1905	)
1906	)
1907	{
1908	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1909	AssertRC(rc);
1910	}
1911	# endif
1912
1913	/*
1914	* Make shadow PTE entry.
1915	*/
1916	SHWPTE PteDst;
1917	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1918	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc), &PteDst);
1919	else
1920	SHW_PTE_SET(PteDst, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc) \| PGM_PAGE_GET_HCPHYS(pPage));
1921
1922	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1923	if ( SHW_PTE_IS_P(PteDst)
1924	&& !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
1925	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1926
1927	/* Make sure only allocated pages are mapped writable. */
1928	if ( SHW_PTE_IS_P_RW(PteDst)
1929	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1930	{
1931	/* Still applies to shared pages. */
1932	Assert(!PGM_PAGE_IS_ZERO(pPage));
1933	SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */
1934	Log3(("SyncPage: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, GCPtrPage));
1935	}
1936
1937	SHW_PTE_ATOMIC_SET2(pPTDst->a[iPTDst], PteDst);
1938
1939	/*
1940	* If the page is not flagged as dirty and is writable, then make it read-only
1941	* at PD level, so we can set the dirty bit when the page is modified.
1942	*
1943	* ASSUMES that page access handlers are implemented on page table entry level.
1944	* Thus we will first catch the dirty access and set PDE.D and restart. If
1945	* there is an access handler, we'll trap again and let it work on the problem.
1946	*/
1947	/** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
1948	* As for invlpg, it simply frees the whole shadow PT.
1949	* ...It's possibly because the guest clears it and the guest doesn't really tell us... */
1950	if ((PdeSrc.u & (X86_PDE4M_D \| X86_PDE_RW)) == X86_PDE_RW)
1951	{
1952	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageBig));
1953	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
1954	PdeDst.u &= ~(SHWUINT)X86_PDE_RW;
1955	}
1956	else
1957	{
1958	PdeDst.u &= ~(SHWUINT)(PGM_PDFLAGS_TRACK_DIRTY \| X86_PDE_RW);
1959	PdeDst.u \|= PdeSrc.u & X86_PDE_RW;
1960	}
1961	SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
1962	Log2(("SyncPage: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%RGp%s\n",
1963	GCPtrPage, PdeSrc.u & X86_PDE_P, !!(PdeSrc.u & X86_PDE_RW), !!(PdeSrc.u & X86_PDE_US),
1964	(uint64_t)PdeSrc.u, GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
1965	}
1966	else
1967	{
1968	LogFlow(("PGM_GCPHYS_2_PTR %RGp (big) failed with %Rrc\n", GCPhys, rc));
1969	/** @todo must wipe the shadow page table entry in this
1970	* case. */
1971	}
1972	}
1973	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
1974	return VINF_SUCCESS;
1975	}
1976
1977	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPagePDNAs));
1978	}
1979	else if (fPdeValid)
1980	{
1981	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPagePDOutOfSync));
1982	Log2(("SyncPage: Out-Of-Sync PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
1983	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
1984	}
1985	else
1986	{
1987	/// @todo STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDOutOfSyncAndInvalid));
1988	Log2(("SyncPage: Bad PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
1989	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
1990	}
1991
1992	/*
1993	* Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
1994	* Yea, I'm lazy.
1995	*/
1996	pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst);
1997	SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
1998
1999	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2000	PGM_INVL_VCPU_TLBS(pVCpu);
2001	return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
2002
2003
2004	# elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2005	&& !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
2006	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT)
2007	NOREF(PdeSrc);
2008
2009	# ifdef PGM_SYNC_N_PAGES
2010	/*
2011	* Get the shadow PDE, find the shadow page table in the pool.
2012	*/
2013	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2014	X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
2015
2016	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2017	X86PDEPAE PdeDst = pgmShwGetPaePDE(pVCpu, GCPtrPage);
2018
2019	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2020	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2021	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; NOREF(iPdpt);
2022	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2023	X86PDEPAE PdeDst;
2024	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2025
2026	int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2027	AssertRCSuccessReturn(rc, rc);
2028	Assert(pPDDst && pPdptDst);
2029	PdeDst = pPDDst->a[iPDDst];
2030
2031	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2032	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2033	PEPTPD pPDDst;
2034	EPTPDE PdeDst;
2035
2036	int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, NULL, &pPDDst);
2037	if (rc != VINF_SUCCESS)
2038	{
2039	AssertRC(rc);
2040	return rc;
2041	}
2042	Assert(pPDDst);
2043	PdeDst = pPDDst->a[iPDDst];
2044	# endif
2045	/* In the guest SMP case we could have blocked while another VCPU reused this page table. */
2046	if (!SHW_PDE_IS_P(PdeDst))
2047	{
2048	AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE %RX64\n", (uint64_t)PdeDst.u));
2049	Log(("CPU%d: SyncPage: Pde at %RGv changed behind our back!\n", pVCpu->idCpu, GCPtrPage));
2050	return VINF_SUCCESS; /* force the instruction to be executed again. */
2051	}
2052
2053	/* Can happen in the guest SMP case; other VCPU activated this PDE while we were blocking to handle the page fault. */
2054	if (SHW_PDE_IS_BIG(PdeDst))
2055	{
2056	Assert(pVM->pgm.s.fNestedPaging);
2057	Log(("CPU%d: SyncPage: Pde (big:%RX64) at %RGv changed behind our back!\n", pVCpu->idCpu, PdeDst.u, GCPtrPage));
2058	return VINF_SUCCESS;
2059	}
2060
2061	/* Mask away the page offset. */
2062	GCPtrPage &= ~((RTGCPTR)0xfff);
2063
2064	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2065	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2066
2067	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
2068	if ( cPages > 1
2069	&& !(uErr & X86_TRAP_PF_P)
2070	&& !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
2071	{
2072	/*
2073	* This code path is currently only taken when the caller is PGMTrap0eHandler
2074	* for non-present pages!
2075	*
2076	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2077	* deal with locality.
2078	*/
2079	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2080	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2081	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2082	iPTDst = 0;
2083	else
2084	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2085	for (; iPTDst < iPTDstEnd; iPTDst++)
2086	{
2087	if (!SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2088	{
2089	RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
2090	\| (iPTDst << GUEST_PAGE_SHIFT));
2091
2092	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
2093	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n",
2094	GCPtrCurPage,
2095	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2096	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2097
2098	if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
2099	break;
2100	}
2101	else
2102	Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n",
2103	(GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << GUEST_PAGE_SHIFT), iPTDst, SHW_PTE_LOG64(pPTDst->a[iPTDst]) ));
2104	}
2105	}
2106	else
2107	# endif /* PGM_SYNC_N_PAGES */
2108	{
2109	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2110	RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
2111	\| (iPTDst << GUEST_PAGE_SHIFT));
2112
2113	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
2114
2115	Log2(("SyncPage: 4K %RGv PteSrc:{P=1 RW=1 U=1}PteDst=%08llx%s\n",
2116	GCPtrPage,
2117	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2118	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2119	}
2120	return VINF_SUCCESS;
2121
2122	# else
2123	NOREF(PdeSrc);
2124	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2125	return VERR_PGM_NOT_USED_IN_MODE;
2126	# endif
2127	}
2128
2129	#endif /* PGM_SHW_TYPE != PGM_TYPE_NONE */
2130	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
2131
2132	/**
2133	* CheckPageFault helper for returning a page fault indicating a non-present
2134	* (NP) entry in the page translation structures.
2135	*
2136	* @returns VINF_EM_RAW_GUEST_TRAP.
2137	* @param pVCpu The cross context virtual CPU structure.
2138	* @param uErr The error code of the shadow fault. Corrections to
2139	* TRPM's copy will be made if necessary.
2140	* @param GCPtrPage For logging.
2141	* @param uPageFaultLevel For logging.
2142	*/
2143	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnNP)(PVMCPUCC pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2144	{
2145	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyTrackRealPF));
2146	AssertMsg(!(uErr & X86_TRAP_PF_P), ("%#x\n", uErr));
2147	AssertMsg(!(uErr & X86_TRAP_PF_RSVD), ("%#x\n", uErr));
2148	if (uErr & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
2149	TRPMSetErrorCode(pVCpu, uErr & ~(X86_TRAP_PF_RSVD \| X86_TRAP_PF_P));
2150
2151	Log(("CheckPageFault: real page fault (notp) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2152	RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(uPageFaultLevel);
2153	return VINF_EM_RAW_GUEST_TRAP;
2154	}
2155
2156
2157	/**
2158	* CheckPageFault helper for returning a page fault indicating a reserved bit
2159	* (RSVD) error in the page translation structures.
2160	*
2161	* @returns VINF_EM_RAW_GUEST_TRAP.
2162	* @param pVCpu The cross context virtual CPU structure.
2163	* @param uErr The error code of the shadow fault. Corrections to
2164	* TRPM's copy will be made if necessary.
2165	* @param GCPtrPage For logging.
2166	* @param uPageFaultLevel For logging.
2167	*/
2168	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnRSVD)(PVMCPUCC pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2169	{
2170	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyTrackRealPF));
2171	if ((uErr & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) != (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
2172	TRPMSetErrorCode(pVCpu, uErr \| X86_TRAP_PF_RSVD \| X86_TRAP_PF_P);
2173
2174	Log(("CheckPageFault: real page fault (rsvd) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2175	RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(uPageFaultLevel);
2176	return VINF_EM_RAW_GUEST_TRAP;
2177	}
2178
2179
2180	/**
2181	* CheckPageFault helper for returning a page protection fault (P).
2182	*
2183	* @returns VINF_EM_RAW_GUEST_TRAP.
2184	* @param pVCpu The cross context virtual CPU structure.
2185	* @param uErr The error code of the shadow fault. Corrections to
2186	* TRPM's copy will be made if necessary.
2187	* @param GCPtrPage For logging.
2188	* @param uPageFaultLevel For logging.
2189	*/
2190	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnProt)(PVMCPUCC pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2191	{
2192	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyTrackRealPF));
2193	AssertMsg(uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID), ("%#x\n", uErr));
2194	if ((uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) != X86_TRAP_PF_P)
2195	TRPMSetErrorCode(pVCpu, (uErr & ~X86_TRAP_PF_RSVD) \| X86_TRAP_PF_P);
2196
2197	Log(("CheckPageFault: real page fault (prot) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2198	RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(uPageFaultLevel);
2199	return VINF_EM_RAW_GUEST_TRAP;
2200	}
2201
2202
2203	/**
2204	* Handle dirty bit tracking faults.
2205	*
2206	* @returns VBox status code.
2207	* @param pVCpu The cross context virtual CPU structure.
2208	* @param uErr Page fault error code.
2209	* @param pPdeSrc Guest page directory entry.
2210	* @param pPdeDst Shadow page directory entry.
2211	* @param GCPtrPage Guest context page address.
2212	*/
2213	static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPUCC pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc,
2214	RTGCPTR GCPtrPage)
2215	{
2216	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2217	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2218	NOREF(uErr);
2219
2220	PGM_LOCK_ASSERT_OWNER(pVM);
2221
2222	/*
2223	* Handle big page.
2224	*/
2225	if ((pPdeSrc->u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu))
2226	{
2227	if ((pPdeDst->u & (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY))
2228	{
2229	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageTrap));
2230	Assert(pPdeSrc->u & X86_PDE_RW);
2231
2232	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
2233	* fault again and take this path to only invalidate the entry (see below). */
2234	SHWPDE PdeDst = *pPdeDst;
2235	PdeDst.u &= ~(SHWUINT)PGM_PDFLAGS_TRACK_DIRTY;
2236	PdeDst.u \|= X86_PDE_RW \| X86_PDE_A;
2237	SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
2238	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
2239	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2240	}
2241
2242	# ifdef IN_RING0
2243	/* Check for stale TLB entry; only applies to the SMP guest case. */
2244	if ( pVM->cCpus > 1
2245	&& (pPdeDst->u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_A)) == (X86_PDE_P \| X86_PDE_RW \| X86_PDE_A))
2246	{
2247	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2248	if (pShwPage)
2249	{
2250	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2251	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2252	if (SHW_PTE_IS_P_RW(*pPteDst))
2253	{
2254	/* Stale TLB entry. */
2255	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageStale));
2256	PGM_INVL_PG(pVCpu, GCPtrPage);
2257	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2258	}
2259	}
2260	}
2261	# endif /* IN_RING0 */
2262	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2263	}
2264
2265	/*
2266	* Map the guest page table.
2267	*/
2268	PGSTPT pPTSrc;
2269	int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(*pPdeSrc), &pPTSrc);
2270	AssertRCReturn(rc, rc);
2271
2272	if (SHW_PDE_IS_P(*pPdeDst))
2273	{
2274	GSTPTE const *pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2275	const GSTPTE PteSrc = *pPteSrc;
2276
2277	/*
2278	* Map shadow page table.
2279	*/
2280	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2281	if (pShwPage)
2282	{
2283	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2284	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2285	if (SHW_PTE_IS_P(pPteDst)) /* @todo Optimize accessed bit emulation? */
2286	{
2287	if (SHW_PTE_IS_TRACK_DIRTY(*pPteDst))
2288	{
2289	PPGMPAGE pPage = pgmPhysGetPage(pVM, GST_GET_PTE_GCPHYS(PteSrc));
2290	SHWPTE PteDst = *pPteDst;
2291
2292	LogFlow(("DIRTY page trap addr=%RGv\n", GCPtrPage));
2293	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageTrap));
2294
2295	Assert(PteSrc.u & X86_PTE_RW);
2296
2297	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB
2298	* entry will not harm; write access will simply fault again and
2299	* take this path to only invalidate the entry.
2300	*/
2301	if (RT_LIKELY(pPage))
2302	{
2303	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2304	{
2305	//AssertMsgFailed(("%R[pgmpage] - we don't set PGM_PTFLAGS_TRACK_DIRTY for these pages\n", pPage));
2306	Assert(!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage));
2307	/* Assuming write handlers here as the PTE is present (otherwise we wouldn't be here). */
2308	SHW_PTE_SET_RO(PteDst);
2309	}
2310	else
2311	{
2312	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
2313	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
2314	{
2315	rc = pgmPhysPageMakeWritable(pVM, pPage, GST_GET_PTE_GCPHYS(PteSrc));
2316	AssertRC(rc);
2317	}
2318	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED)
2319	SHW_PTE_SET_RW(PteDst);
2320	else
2321	{
2322	/* Still applies to shared pages. */
2323	Assert(!PGM_PAGE_IS_ZERO(pPage));
2324	SHW_PTE_SET_RO(PteDst);
2325	}
2326	}
2327	}
2328	else
2329	SHW_PTE_SET_RW(PteDst); /** @todo r=bird: This doesn't make sense to me. */
2330
2331	SHW_PTE_SET(PteDst, (SHW_PTE_GET_U(PteDst) \| X86_PTE_D \| X86_PTE_A) & ~(uint64_t)PGM_PTFLAGS_TRACK_DIRTY);
2332	SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
2333	PGM_INVL_PG(pVCpu, GCPtrPage);
2334	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2335	}
2336
2337	# ifdef IN_RING0
2338	/* Check for stale TLB entry; only applies to the SMP guest case. */
2339	if ( pVM->cCpus > 1
2340	&& SHW_PTE_IS_RW(*pPteDst)
2341	&& SHW_PTE_IS_A(*pPteDst))
2342	{
2343	/* Stale TLB entry. */
2344	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageStale));
2345	PGM_INVL_PG(pVCpu, GCPtrPage);
2346	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2347	}
2348	# endif
2349	}
2350	}
2351	else
2352	AssertMsgFailed(("pgmPoolGetPageByHCPhys %RGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
2353	}
2354
2355	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2356	}
2357
2358	#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE */
2359
2360	/**
2361	* Sync a shadow page table.
2362	*
2363	* The shadow page table is not present in the shadow PDE.
2364	*
2365	* Handles mapping conflicts.
2366	*
2367	* This is called by VerifyAccessSyncPage, PrefetchPage, InvalidatePage (on
2368	* conflict), and Trap0eHandler.
2369	*
2370	* A precondition for this method is that the shadow PDE is not present. The
2371	* caller must take the PGM lock before checking this and continue to hold it
2372	* when calling this method.
2373	*
2374	* @returns VBox status code.
2375	* @param pVCpu The cross context virtual CPU structure.
2376	* @param iPDSrc Page directory index.
2377	* @param pPDSrc Source page directory (i.e. Guest OS page directory).
2378	* Assume this is a temporary mapping.
2379	* @param GCPtrPage GC Pointer of the page that caused the fault
2380	*/
2381	static int PGM_BTH_NAME(SyncPT)(PVMCPUCC pVCpu, unsigned iPDSrc, PGSTPD pPDSrc, RTGCPTR GCPtrPage)
2382	{
2383	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2384	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
2385
2386	#if 0 /* rarely useful; leave for debugging. */
2387	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPtPD[iPDSrc]);
2388	#endif
2389	LogFlow(("SyncPT: GCPtrPage=%RGv\n", GCPtrPage)); RT_NOREF_PV(GCPtrPage);
2390
2391	PGM_LOCK_ASSERT_OWNER(pVM);
2392
2393	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2394	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
2395	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
2396	&& !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
2397	&& PGM_SHW_TYPE != PGM_TYPE_NONE
2398	int rc = VINF_SUCCESS;
2399
2400	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
2401
2402	/*
2403	* Some input validation first.
2404	*/
2405	AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage));
2406
2407	/*
2408	* Get the relevant shadow PDE entry.
2409	*/
2410	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2411	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2412	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
2413
2414	/* Fetch the pgm pool shadow descriptor. */
2415	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2416	Assert(pShwPde);
2417
2418	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2419	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2420	PPGMPOOLPAGE pShwPde = NULL;
2421	PX86PDPAE pPDDst;
2422	PSHWPDE pPdeDst;
2423
2424	/* Fetch the pgm pool shadow descriptor. */
2425	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
2426	AssertRCSuccessReturn(rc, rc);
2427	Assert(pShwPde);
2428
2429	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
2430	pPdeDst = &pPDDst->a[iPDDst];
2431
2432	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2433	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2434	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2435	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2436	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2437	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2438	AssertRCSuccessReturn(rc, rc);
2439	Assert(pPDDst);
2440	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2441
2442	# endif
2443	SHWPDE PdeDst = *pPdeDst;
2444
2445	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2446	/* Fetch the pgm pool shadow descriptor. */
2447	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2448	Assert(pShwPde);
2449	# endif
2450
2451	Assert(!SHW_PDE_IS_P(PdeDst)); /* We're only supposed to call SyncPT on PDE!P.*/
2452
2453	/*
2454	* Sync the page directory entry.
2455	*/
2456	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2457	const bool fPageTable = !(PdeSrc.u & X86_PDE_PS) \|\| !GST_IS_PSE_ACTIVE(pVCpu);
2458	if ( (PdeSrc.u & X86_PDE_P)
2459	&& (fPageTable ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc)) )
2460	{
2461	/*
2462	* Allocate & map the page table.
2463	*/
2464	PSHWPT pPTDst;
2465	PPGMPOOLPAGE pShwPage;
2466	RTGCPHYS GCPhys;
2467	if (fPageTable)
2468	{
2469	GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
2470	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2471	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2472	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2)));
2473	# endif
2474	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
2475	pShwPde->idx, iPDDst, false /fLockPage/,
2476	&pShwPage);
2477	}
2478	else
2479	{
2480	PGMPOOLACCESS enmAccess;
2481	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2482	const bool fNoExecute = (PdeSrc.u & X86_PDE_PAE_NX) && GST_IS_NX_ACTIVE(pVCpu);
2483	# else
2484	const bool fNoExecute = false;
2485	# endif
2486
2487	GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
2488	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2489	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2490	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
2491	# endif
2492	/* Determine the right kind of large page to avoid incorrect cached entry reuse. */
2493	if (PdeSrc.u & X86_PDE_US)
2494	{
2495	if (PdeSrc.u & X86_PDE_RW)
2496	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_RW_NX : PGMPOOLACCESS_USER_RW;
2497	else
2498	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_R_NX : PGMPOOLACCESS_USER_R;
2499	}
2500	else
2501	{
2502	if (PdeSrc.u & X86_PDE_RW)
2503	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW;
2504	else
2505	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_R_NX : PGMPOOLACCESS_SUPERVISOR_R;
2506	}
2507	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, enmAccess, PGM_A20_IS_ENABLED(pVCpu),
2508	pShwPde->idx, iPDDst, false /fLockPage/,
2509	&pShwPage);
2510	}
2511	if (rc == VINF_SUCCESS)
2512	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2513	else if (rc == VINF_PGM_CACHED_PAGE)
2514	{
2515	/*
2516	* The PT was cached, just hook it up.
2517	*/
2518	if (fPageTable)
2519	PdeDst.u = pShwPage->Core.Key \| GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2520	else
2521	{
2522	PdeDst.u = pShwPage->Core.Key \| GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2523	/* (see explanation and assumptions further down.) */
2524	if ((PdeSrc.u & (X86_PDE_RW \| X86_PDE4M_D)) == X86_PDE_RW)
2525	{
2526	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageBig));
2527	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2528	PdeDst.u &= ~(SHWUINT)X86_PDE_RW;
2529	}
2530	}
2531	SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
2532	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2533	return VINF_SUCCESS;
2534	}
2535	else
2536	AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
2537	/** @todo Why do we bother preserving X86_PDE_AVL_MASK here?
2538	* Both PGM_PDFLAGS_MAPPING and PGM_PDFLAGS_TRACK_DIRTY should be
2539	* irrelevant at this point. */
2540	PdeDst.u &= X86_PDE_AVL_MASK;
2541	PdeDst.u \|= pShwPage->Core.Key;
2542
2543	/*
2544	* Page directory has been accessed (this is a fault situation, remember).
2545	*/
2546	/** @todo
2547	* Well, when the caller is PrefetchPage or InvalidatePage is isn't a
2548	* fault situation. What's more, the Trap0eHandler has already set the
2549	* accessed bit. So, it's actually just VerifyAccessSyncPage which
2550	* might need setting the accessed flag.
2551	*
2552	* The best idea is to leave this change to the caller and add an
2553	* assertion that it's set already. */
2554	pPDSrc->a[iPDSrc].u \|= X86_PDE_A;
2555	if (fPageTable)
2556	{
2557	/*
2558	* Page table - 4KB.
2559	*
2560	* Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
2561	*/
2562	Log2(("SyncPT: 4K %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
2563	GCPtrPage, PdeSrc.u & X86_PTE_P, !!(PdeSrc.u & X86_PTE_RW), !!(PdeSrc.u & X86_PDE_US), (uint64_t)PdeSrc.u));
2564	PGSTPT pPTSrc;
2565	rc = PGM_GCPHYS_2_PTR(pVM, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
2566	if (RT_SUCCESS(rc))
2567	{
2568	/*
2569	* Start by syncing the page directory entry so CSAM's TLB trick works.
2570	*/
2571	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| X86_PDE_AVL_MASK))
2572	\| GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2573	SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
2574	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2575
2576	/*
2577	* Directory/page user or supervisor privilege: (same goes for read/write)
2578	*
2579	* Directory Page Combined
2580	* U/S U/S U/S
2581	* 0 0 0
2582	* 0 1 0
2583	* 1 0 0
2584	* 1 1 1
2585	*
2586	* Simple AND operation. Table listed for completeness.
2587	*
2588	*/
2589	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT4K));
2590	# ifdef PGM_SYNC_N_PAGES
2591	unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2592	unsigned iPTDst = iPTBase;
2593	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2594	if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
2595	iPTDst = 0;
2596	else
2597	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2598	# else /* !PGM_SYNC_N_PAGES */
2599	unsigned iPTDst = 0;
2600	const unsigned iPTDstEnd = RT_ELEMENTS(pPTDst->a);
2601	# endif /* !PGM_SYNC_N_PAGES */
2602	RTGCPTR GCPtrCur = (GCPtrPage & ~(RTGCPTR)((1 << SHW_PD_SHIFT) - 1))
2603	\| ((RTGCPTR)iPTDst << GUEST_PAGE_SHIFT);
2604	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2605	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2606	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2607	# else
2608	const unsigned offPTSrc = 0;
2609	# endif
2610	for (; iPTDst < iPTDstEnd; iPTDst++, GCPtrCur += GUEST_PAGE_SIZE)
2611	{
2612	const unsigned iPTSrc = iPTDst + offPTSrc;
2613	const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2614	if (PteSrc.u & X86_PTE_P)
2615	{
2616	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2617	Log2(("SyncPT: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n",
2618	GCPtrCur,
2619	PteSrc.u & X86_PTE_P,
2620	!!(PteSrc.u & PdeSrc.u & X86_PTE_RW),
2621	!!(PteSrc.u & PdeSrc.u & X86_PTE_US),
2622	(uint64_t)PteSrc.u,
2623	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : "", SHW_PTE_LOG64(pPTDst->a[iPTDst]), iPTSrc, PdeSrc.au32[0],
2624	(RTGCPHYS)(GST_GET_PDE_GCPHYS(PdeSrc) + iPTSrc*sizeof(PteSrc)) ));
2625	}
2626	/* else: the page table was cleared by the pool */
2627	} /* for PTEs */
2628	}
2629	}
2630	else
2631	{
2632	/*
2633	* Big page - 2/4MB.
2634	*
2635	* We'll walk the ram range list in parallel and optimize lookups.
2636	* We will only sync one shadow page table at a time.
2637	*/
2638	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT4M));
2639
2640	/**
2641	* @todo It might be more efficient to sync only a part of the 4MB
2642	* page (similar to what we do for 4KB PDs).
2643	*/
2644
2645	/*
2646	* Start by syncing the page directory entry.
2647	*/
2648	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
2649	\| GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2650
2651	/*
2652	* If the page is not flagged as dirty and is writable, then make it read-only
2653	* at PD level, so we can set the dirty bit when the page is modified.
2654	*
2655	* ASSUMES that page access handlers are implemented on page table entry level.
2656	* Thus we will first catch the dirty access and set PDE.D and restart. If
2657	* there is an access handler, we'll trap again and let it work on the problem.
2658	*/
2659	/** @todo move the above stuff to a section in the PGM documentation. */
2660	Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
2661	if ((PdeSrc.u & (X86_PDE_RW \| X86_PDE4M_D)) == X86_PDE_RW)
2662	{
2663	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageBig));
2664	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2665	PdeDst.u &= ~(SHWUINT)X86_PDE_RW;
2666	}
2667	SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
2668	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2669
2670	/*
2671	* Fill the shadow page table.
2672	*/
2673	/* Get address and flags from the source PDE. */
2674	SHWPTE PteDstBase;
2675	SHW_PTE_SET(PteDstBase, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc));
2676
2677	/* Loop thru the entries in the shadow PT. */
2678	const RTGCPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
2679	Log2(("SyncPT: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%RGv GCPhys=%RGp %s\n",
2680	GCPtrPage, PdeSrc.u & X86_PDE_P, !!(PdeSrc.u & X86_PDE_RW), !!(PdeSrc.u & X86_PDE_US), (uint64_t)PdeSrc.u, GCPtr,
2681	GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2682	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
2683	unsigned iPTDst = 0;
2684	while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2685	&& !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
2686	{
2687	if (pRam && GCPhys >= pRam->GCPhys)
2688	{
2689	# ifndef PGM_WITH_A20
2690	unsigned iHCPage = (GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT;
2691	# endif
2692	do
2693	{
2694	/* Make shadow PTE. */
2695	# ifdef PGM_WITH_A20
2696	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT];
2697	# else
2698	PPGMPAGE pPage = &pRam->aPages[iHCPage];
2699	# endif
2700	SHWPTE PteDst;
2701
2702	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2703	/* Try to make the page writable if necessary. */
2704	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2705	&& ( PGM_PAGE_IS_ZERO(pPage)
2706	\|\| ( SHW_PTE_IS_RW(PteDstBase)
2707	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2708	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
2709	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
2710	# endif
2711	# ifdef VBOX_WITH_PAGE_SHARING
2712	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
2713	# endif
2714	&& !PGM_PAGE_IS_BALLOONED(pPage))
2715	)
2716	)
2717	{
2718	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
2719	AssertRCReturn(rc, rc);
2720	if (VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
2721	break;
2722	}
2723	# endif
2724
2725	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2726	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, SHW_PTE_GET_U(PteDstBase), &PteDst);
2727	else if (PGM_PAGE_IS_BALLOONED(pPage))
2728	SHW_PTE_SET(PteDst, 0); /* Handle ballooned pages at #PF time. */
2729	else
2730	SHW_PTE_SET(PteDst, PGM_PAGE_GET_HCPHYS(pPage) \| SHW_PTE_GET_U(PteDstBase));
2731
2732	/* Only map writable pages writable. */
2733	if ( SHW_PTE_IS_P_RW(PteDst)
2734	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2735	{
2736	/* Still applies to shared pages. */
2737	Assert(!PGM_PAGE_IS_ZERO(pPage));
2738	SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */
2739	Log3(("SyncPT: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, (RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))));
2740	}
2741
2742	if (SHW_PTE_IS_P(PteDst))
2743	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2744
2745	/* commit it (not atomic, new table) */
2746	pPTDst->a[iPTDst] = PteDst;
2747	Log4(("SyncPT: BIG %RGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
2748	(RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT)), SHW_PTE_IS_P(PteDst), SHW_PTE_IS_RW(PteDst), SHW_PTE_IS_US(PteDst), SHW_PTE_LOG64(PteDst),
2749	SHW_PTE_IS_TRACK_DIRTY(PteDst) ? " Track-Dirty" : ""));
2750
2751	/* advance */
2752	GCPhys += GUEST_PAGE_SIZE;
2753	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys);
2754	# ifndef PGM_WITH_A20
2755	iHCPage++;
2756	# endif
2757	iPTDst++;
2758	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2759	&& GCPhys <= pRam->GCPhysLast);
2760
2761	/* Advance ram range list. */
2762	while (pRam && GCPhys > pRam->GCPhysLast)
2763	pRam = pRam->CTX_SUFF(pNext);
2764	}
2765	else if (pRam)
2766	{
2767	Log(("Invalid pages at %RGp\n", GCPhys));
2768	do
2769	{
2770	SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
2771	GCPhys += GUEST_PAGE_SIZE;
2772	iPTDst++;
2773	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2774	&& GCPhys < pRam->GCPhys);
2775	PGM_A20_APPLY_TO_VAR(pVCpu,GCPhys);
2776	}
2777	else
2778	{
2779	Log(("Invalid pages at %RGp (2)\n", GCPhys));
2780	for ( ; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
2781	SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
2782	}
2783	} /* while more PTEs */
2784	} /* 4KB / 4MB */
2785	}
2786	else
2787	AssertRelease(!SHW_PDE_IS_P(PdeDst));
2788
2789	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
2790	if (RT_FAILURE(rc))
2791	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPTFailed));
2792	return rc;
2793
2794	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2795	&& !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
2796	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2797	&& PGM_SHW_TYPE != PGM_TYPE_NONE
2798	NOREF(iPDSrc); NOREF(pPDSrc);
2799
2800	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
2801
2802	/*
2803	* Validate input a little bit.
2804	*/
2805	int rc = VINF_SUCCESS;
2806	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2807	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2808	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
2809
2810	/* Fetch the pgm pool shadow descriptor. */
2811	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2812	Assert(pShwPde);
2813
2814	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2815	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2816	PPGMPOOLPAGE pShwPde = NULL; /* initialized to shut up gcc */
2817	PX86PDPAE pPDDst;
2818	PSHWPDE pPdeDst;
2819
2820	/* Fetch the pgm pool shadow descriptor. */
2821	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
2822	AssertRCSuccessReturn(rc, rc);
2823	Assert(pShwPde);
2824
2825	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
2826	pPdeDst = &pPDDst->a[iPDDst];
2827
2828	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2829	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2830	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2831	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2832	PX86PDPT pPdptDst= NULL; /* initialized to shut up gcc */
2833	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2834	AssertRCSuccessReturn(rc, rc);
2835	Assert(pPDDst);
2836	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2837
2838	/* Fetch the pgm pool shadow descriptor. */
2839	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2840	Assert(pShwPde);
2841
2842	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2843	const unsigned iPdpt = (GCPtrPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
2844	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2845	PEPTPD pPDDst;
2846	PEPTPDPT pPdptDst;
2847
2848	rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, &pPdptDst, &pPDDst);
2849	if (rc != VINF_SUCCESS)
2850	{
2851	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
2852	AssertRC(rc);
2853	return rc;
2854	}
2855	Assert(pPDDst);
2856	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2857
2858	/* Fetch the pgm pool shadow descriptor. */
2859	/** @todo r=bird: didn't pgmShwGetEPTPDPtr just do this lookup already? */
2860	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & EPT_PDPTE_PG_MASK);
2861	Assert(pShwPde);
2862	# endif
2863	SHWPDE PdeDst = *pPdeDst;
2864
2865	Assert(!SHW_PDE_IS_P(PdeDst)); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2866
2867	# if defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
2868	if ( BTH_IS_NP_ACTIVE(pVM)
2869	&& !VM_IS_NEM_ENABLED(pVM)) /** @todo NEM: Large page support. */
2870	{
2871	/* Check if we allocated a big page before for this 2 MB range. */
2872	PPGMPAGE pPage;
2873	rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PDE2M_PAE_PG_MASK), &pPage);
2874	if (RT_SUCCESS(rc))
2875	{
2876	RTHCPHYS HCPhys = NIL_RTHCPHYS;
2877	if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
2878	{
2879	if (PGM_A20_IS_ENABLED(pVCpu))
2880	{
2881	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageReused);
2882	AssertRelease(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
2883	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2884	}
2885	else
2886	{
2887	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PDE_DISABLED);
2888	pVM->pgm.s.cLargePagesDisabled++;
2889	}
2890	}
2891	else if ( PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED
2892	&& PGM_A20_IS_ENABLED(pVCpu))
2893	{
2894	/* Recheck the entire 2 MB range to see if we can use it again as a large page. */
2895	rc = pgmPhysRecheckLargePage(pVM, GCPtrPage, pPage);
2896	if (RT_SUCCESS(rc))
2897	{
2898	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
2899	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
2900	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2901	}
2902	}
2903	else if ( PGMIsUsingLargePages(pVM)
2904	&& PGM_A20_IS_ENABLED(pVCpu))
2905	{
2906	rc = pgmPhysAllocLargePage(pVM, GCPtrPage);
2907	if (RT_SUCCESS(rc))
2908	{
2909	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
2910	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
2911	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2912	}
2913	else
2914	LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
2915	}
2916
2917	if (HCPhys != NIL_RTHCPHYS)
2918	{
2919	# if PGM_SHW_TYPE == PGM_TYPE_EPT
2920	PdeDst.u = HCPhys \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE \| EPT_E_LEAF \| EPT_E_IGNORE_PAT \| EPT_E_MEMTYPE_WB
2921	\| (PdeDst.u & X86_PDE_AVL_MASK) /** @todo do we need this? */;
2922	# else
2923	PdeDst.u = HCPhys \| X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PS
2924	\| (PdeDst.u & X86_PDE_AVL_MASK) /** @todo PGM_PD_FLAGS? */;
2925	# endif
2926	SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
2927
2928	Log(("SyncPT: Use large page at %RGp PDE=%RX64\n", GCPtrPage, PdeDst.u));
2929	/* Add a reference to the first page only. */
2930	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPde, PGM_PAGE_GET_TRACKING(pPage), pPage, iPDDst);
2931
2932	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
2933	return VINF_SUCCESS;
2934	}
2935	}
2936	}
2937	# endif /* defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE */
2938
2939	/*
2940	* Allocate & map the page table.
2941	*/
2942	PSHWPT pPTDst;
2943	PPGMPOOLPAGE pShwPage;
2944	RTGCPHYS GCPhys;
2945
2946	/* Virtual address = physical address */
2947	GCPhys = PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PAGE_4K_BASE_MASK);
2948	rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE,
2949	PGM_A20_IS_ENABLED(pVCpu), pShwPde->idx, iPDDst, false /fLockPage/,
2950	&pShwPage);
2951	if ( rc == VINF_SUCCESS
2952	\|\| rc == VINF_PGM_CACHED_PAGE)
2953	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2954	else
2955	{
2956	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
2957	AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
2958	}
2959
2960	if (rc == VINF_SUCCESS)
2961	{
2962	/* New page table; fully set it up. */
2963	Assert(pPTDst);
2964
2965	/* Mask away the page offset. */
2966	GCPtrPage &= ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK;
2967
2968	for (unsigned iPTDst = 0; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
2969	{
2970	RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
2971	\| (iPTDst << GUEST_PAGE_SHIFT));
2972
2973	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
2974	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n",
2975	GCPtrCurPage,
2976	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2977	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2978
2979	if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
2980	break;
2981	}
2982	}
2983	else
2984	rc = VINF_SUCCESS; /* Cached entry; assume it's still fully valid. */
2985
2986	/* Save the new PDE. */
2987	# if PGM_SHW_TYPE == PGM_TYPE_EPT
2988	PdeDst.u = pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE
2989	\| (PdeDst.u & X86_PDE_AVL_MASK /** @todo do we really need this? */);
2990	# else
2991	PdeDst.u = pShwPage->Core.Key \| X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_A
2992	\| (PdeDst.u & X86_PDE_AVL_MASK /** @todo use a PGM_PD_FLAGS define */);
2993	# endif
2994	SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
2995
2996	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
2997	if (RT_FAILURE(rc))
2998	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPTFailed));
2999	return rc;
3000
3001	#else
3002	NOREF(iPDSrc); NOREF(pPDSrc);
3003	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
3004	return VERR_PGM_NOT_USED_IN_MODE;
3005	#endif
3006	}
3007
3008
3009
3010	/**
3011	* Prefetch a page/set of pages.
3012	*
3013	* Typically used to sync commonly used pages before entering raw mode
3014	* after a CR3 reload.
3015	*
3016	* @returns VBox status code.
3017	* @param pVCpu The cross context virtual CPU structure.
3018	* @param GCPtrPage Page to invalidate.
3019	*/
3020	PGM_BTH_DECL(int, PrefetchPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
3021	{
3022	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3023	\|\| PGM_GST_TYPE == PGM_TYPE_REAL \
3024	\|\| PGM_GST_TYPE == PGM_TYPE_PROT \
3025	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3026	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3027	&& !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
3028	&& PGM_SHW_TYPE != PGM_TYPE_NONE
3029	/*
3030	* Check that all Guest levels thru the PDE are present, getting the
3031	* PD and PDE in the processes.
3032	*/
3033	int rc = VINF_SUCCESS;
3034	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3035	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3036	const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
3037	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3038	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3039	unsigned iPDSrc;
3040	X86PDPE PdpeSrc;
3041	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3042	if (!pPDSrc)
3043	return VINF_SUCCESS; /* not present */
3044	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3045	unsigned iPDSrc;
3046	PX86PML4E pPml4eSrc;
3047	X86PDPE PdpeSrc;
3048	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3049	if (!pPDSrc)
3050	return VINF_SUCCESS; /* not present */
3051	# endif
3052	const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3053	# else
3054	PGSTPD pPDSrc = NULL;
3055	const unsigned iPDSrc = 0;
3056	GSTPDE const PdeSrc = { X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_A }; /* faked so we don't have to #ifdef everything */
3057	# endif
3058
3059	if ((PdeSrc.u & (X86_PDE_P \| X86_PDE_A)) == (X86_PDE_P \| X86_PDE_A))
3060	{
3061	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3062	PGM_LOCK_VOID(pVM);
3063
3064	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3065	const X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
3066	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3067	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3068	PX86PDPAE pPDDst;
3069	X86PDEPAE PdeDst;
3070	# if PGM_GST_TYPE != PGM_TYPE_PAE
3071	X86PDPE PdpeSrc;
3072
3073	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3074	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3075	# endif
3076	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
3077	if (rc != VINF_SUCCESS)
3078	{
3079	PGM_UNLOCK(pVM);
3080	AssertRC(rc);
3081	return rc;
3082	}
3083	Assert(pPDDst);
3084	PdeDst = pPDDst->a[iPDDst];
3085
3086	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3087	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3088	PX86PDPAE pPDDst;
3089	X86PDEPAE PdeDst;
3090
3091	# if PGM_GST_TYPE == PGM_TYPE_PROT
3092	/* AMD-V nested paging */
3093	X86PML4E Pml4eSrc;
3094	X86PDPE PdpeSrc;
3095	PX86PML4E pPml4eSrc = &Pml4eSrc;
3096
3097	/* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3098	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
3099	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
3100	# endif
3101
3102	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
3103	if (rc != VINF_SUCCESS)
3104	{
3105	PGM_UNLOCK(pVM);
3106	AssertRC(rc);
3107	return rc;
3108	}
3109	Assert(pPDDst);
3110	PdeDst = pPDDst->a[iPDDst];
3111	# endif
3112	if (!(PdeDst.u & X86_PDE_P))
3113	{
3114	/** @todo r=bird: This guy will set the A bit on the PDE,
3115	* probably harmless. */
3116	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3117	}
3118	else
3119	{
3120	/* Note! We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
3121	* R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
3122	* makes no sense to prefetch more than one page.
3123	*/
3124	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3125	if (RT_SUCCESS(rc))
3126	rc = VINF_SUCCESS;
3127	}
3128	PGM_UNLOCK(pVM);
3129	}
3130	return rc;
3131
3132	#elif PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \|\| PGM_SHW_TYPE == PGM_TYPE_NONE
3133	NOREF(pVCpu); NOREF(GCPtrPage);
3134	return VINF_SUCCESS; /* ignore */
3135	#else
3136	AssertCompile(0);
3137	#endif
3138	}
3139
3140
3141
3142
3143	/**
3144	* Syncs a page during a PGMVerifyAccess() call.
3145	*
3146	* @returns VBox status code (informational included).
3147	* @param pVCpu The cross context virtual CPU structure.
3148	* @param GCPtrPage The address of the page to sync.
3149	* @param fPage The effective guest page flags.
3150	* @param uErr The trap error code.
3151	* @remarks This will normally never be called on invalid guest page
3152	* translation entries.
3153	*/
3154	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage, unsigned fPage, unsigned uErr)
3155	{
3156	PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
3157
3158	LogFlow(("VerifyAccessSyncPage: GCPtrPage=%RGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
3159	RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(fPage); RT_NOREF_PV(uErr);
3160
3161	Assert(!pVM->pgm.s.fNestedPaging);
3162	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3163	\|\| PGM_GST_TYPE == PGM_TYPE_REAL \
3164	\|\| PGM_GST_TYPE == PGM_TYPE_PROT \
3165	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3166	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3167	&& !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
3168	&& PGM_SHW_TYPE != PGM_TYPE_NONE
3169
3170	/*
3171	* Get guest PD and index.
3172	*/
3173	/** @todo Performance: We've done all this a jiffy ago in the
3174	* PGMGstGetPage call. */
3175	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3176	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3177	const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
3178	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3179
3180	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3181	unsigned iPDSrc = 0;
3182	X86PDPE PdpeSrc;
3183	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3184	if (RT_UNLIKELY(!pPDSrc))
3185	{
3186	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3187	return VINF_EM_RAW_GUEST_TRAP;
3188	}
3189
3190	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3191	unsigned iPDSrc = 0; /* shut up gcc */
3192	PX86PML4E pPml4eSrc = NULL; /* ditto */
3193	X86PDPE PdpeSrc;
3194	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3195	if (RT_UNLIKELY(!pPDSrc))
3196	{
3197	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3198	return VINF_EM_RAW_GUEST_TRAP;
3199	}
3200	# endif
3201
3202	# else /* !PGM_WITH_PAGING */
3203	PGSTPD pPDSrc = NULL;
3204	const unsigned iPDSrc = 0;
3205	# endif /* !PGM_WITH_PAGING */
3206	int rc = VINF_SUCCESS;
3207
3208	PGM_LOCK_VOID(pVM);
3209
3210	/*
3211	* First check if the shadow pd is present.
3212	*/
3213	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3214	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
3215
3216	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3217	PX86PDEPAE pPdeDst;
3218	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3219	PX86PDPAE pPDDst;
3220	# if PGM_GST_TYPE != PGM_TYPE_PAE
3221	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3222	X86PDPE PdpeSrc;
3223	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3224	# endif
3225	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
3226	if (rc != VINF_SUCCESS)
3227	{
3228	PGM_UNLOCK(pVM);
3229	AssertRC(rc);
3230	return rc;
3231	}
3232	Assert(pPDDst);
3233	pPdeDst = &pPDDst->a[iPDDst];
3234
3235	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3236	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3237	PX86PDPAE pPDDst;
3238	PX86PDEPAE pPdeDst;
3239
3240	# if PGM_GST_TYPE == PGM_TYPE_PROT
3241	/* AMD-V nested paging: Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3242	X86PML4E Pml4eSrc;
3243	X86PDPE PdpeSrc;
3244	PX86PML4E pPml4eSrc = &Pml4eSrc;
3245	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
3246	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
3247	# endif
3248
3249	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
3250	if (rc != VINF_SUCCESS)
3251	{
3252	PGM_UNLOCK(pVM);
3253	AssertRC(rc);
3254	return rc;
3255	}
3256	Assert(pPDDst);
3257	pPdeDst = &pPDDst->a[iPDDst];
3258	# endif
3259
3260	if (!(pPdeDst->u & X86_PDE_P))
3261	{
3262	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3263	if (rc != VINF_SUCCESS)
3264	{
3265	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3266	PGM_UNLOCK(pVM);
3267	AssertRC(rc);
3268	return rc;
3269	}
3270	}
3271
3272	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3273	/* Check for dirty bit fault */
3274	rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
3275	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
3276	Log(("PGMVerifyAccess: success (dirty)\n"));
3277	else
3278	# endif
3279	{
3280	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3281	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3282	# else
3283	GSTPDE const PdeSrc = { X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_A }; /* faked so we don't have to #ifdef everything */
3284	# endif
3285
3286	Assert(rc != VINF_EM_RAW_GUEST_TRAP);
3287	if (uErr & X86_TRAP_PF_US)
3288	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncUser));
3289	else /* supervisor */
3290	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
3291
3292	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3293	if (RT_SUCCESS(rc))
3294	{
3295	/* Page was successfully synced */
3296	Log2(("PGMVerifyAccess: success (sync)\n"));
3297	rc = VINF_SUCCESS;
3298	}
3299	else
3300	{
3301	Log(("PGMVerifyAccess: access violation for %RGv rc=%Rrc\n", GCPtrPage, rc));
3302	rc = VINF_EM_RAW_GUEST_TRAP;
3303	}
3304	}
3305	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3306	PGM_UNLOCK(pVM);
3307	return rc;
3308
3309	#else /* PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) */
3310
3311	AssertLogRelMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
3312	return VERR_PGM_NOT_USED_IN_MODE;
3313	#endif /* PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) */
3314	}
3315
3316
3317	/**
3318	* Syncs the paging hierarchy starting at CR3.
3319	*
3320	* @returns VBox status code, R0/RC may return VINF_PGM_SYNC_CR3, no other
3321	* informational status codes.
3322	* @retval VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map
3323	* the VMM into guest context.
3324	* @param pVCpu The cross context virtual CPU structure.
3325	* @param cr0 Guest context CR0 register.
3326	* @param cr3 Guest context CR3 register. Not subjected to the A20
3327	* mask.
3328	* @param cr4 Guest context CR4 register.
3329	* @param fGlobal Including global page directories or not
3330	*/
3331	PGM_BTH_DECL(int, SyncCR3)(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
3332	{
3333	PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
3334	NOREF(cr0); NOREF(cr3); NOREF(cr4); NOREF(fGlobal);
3335
3336	LogFlow(("SyncCR3 FF=%d fGlobal=%d\n", !!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), fGlobal));
3337
3338	#if !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
3339	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
3340	PGM_LOCK_VOID(pVM);
3341	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3342	if (pPool->cDirtyPages)
3343	pgmPoolResetDirtyPages(pVM);
3344	PGM_UNLOCK(pVM);
3345	# endif
3346	#endif /* !NESTED && !EPT */
3347
3348	#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \|\| PGM_SHW_TYPE == PGM_TYPE_NONE
3349	/*
3350	* Nested / EPT / None - No work.
3351	*/
3352	return VINF_SUCCESS;
3353
3354	#elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3355	/*
3356	* AMD64 (Shw & Gst) - No need to check all paging levels; we zero
3357	* out the shadow parts when the guest modifies its tables.
3358	*/
3359	return VINF_SUCCESS;
3360
3361	#else /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3362
3363	return VINF_SUCCESS;
3364	#endif /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3365	}
3366
3367
3368
3369
3370	#ifdef VBOX_STRICT
3371
3372	/**
3373	* Checks that the shadow page table is in sync with the guest one.
3374	*
3375	* @returns The number of errors.
3376	* @param pVCpu The cross context virtual CPU structure.
3377	* @param cr3 Guest context CR3 register.
3378	* @param cr4 Guest context CR4 register.
3379	* @param GCPtr Where to start. Defaults to 0.
3380	* @param cb How much to check. Defaults to everything.
3381	*/
3382	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb)
3383	{
3384	NOREF(pVCpu); NOREF(cr3); NOREF(cr4); NOREF(GCPtr); NOREF(cb);
3385	#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \|\| PGM_SHW_TYPE == PGM_TYPE_NONE
3386	return 0;
3387	#else
3388	unsigned cErrors = 0;
3389	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3390	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
3391
3392	# if PGM_GST_TYPE == PGM_TYPE_PAE
3393	/** @todo currently broken; crashes below somewhere */
3394	AssertFailed();
3395	# endif
3396
3397	# if PGM_GST_TYPE == PGM_TYPE_32BIT \
3398	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3399	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
3400
3401	bool fBigPagesSupported = GST_IS_PSE_ACTIVE(pVCpu);
3402	PPGMCPU pPGM = &pVCpu->pgm.s;
3403	RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
3404	RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
3405	# ifndef IN_RING0
3406	RTHCPHYS HCPhys; /* general usage. */
3407	# endif
3408	int rc;
3409
3410	/*
3411	* Check that the Guest CR3 and all its mappings are correct.
3412	*/
3413	AssertMsgReturn(pPGM->GCPhysCR3 == PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK),
3414	("Invalid GCPhysCR3=%RGp cr3=%RGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
3415	false);
3416	# if !defined(IN_RING0) && PGM_GST_TYPE != PGM_TYPE_AMD64
3417	# if 0
3418	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3419	rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGst32BitPdRC, NULL, &HCPhysShw);
3420	# else
3421	rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGstPaePdptRC, NULL, &HCPhysShw);
3422	# endif
3423	AssertRCReturn(rc, 1);
3424	HCPhys = NIL_RTHCPHYS;
3425	rc = pgmRamGCPhys2HCPhys(pVM, PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK), &HCPhys);
3426	AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
3427	# endif
3428	# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
3429	pgmGstGet32bitPDPtr(pVCpu);
3430	RTGCPHYS GCPhys;
3431	rc = PGMR3DbgR3Ptr2GCPhys(pVM->pUVM, pPGM->pGst32BitPdR3, &GCPhys);
3432	AssertRCReturn(rc, 1);
3433	AssertMsgReturn(PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false);
3434	# endif
3435	# endif /* !IN_RING0 */
3436
3437	/*
3438	* Get and check the Shadow CR3.
3439	*/
3440	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3441	unsigned cPDEs = X86_PG_ENTRIES;
3442	unsigned cIncrement = X86_PG_ENTRIES * GUEST_PAGE_SIZE;
3443	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3444	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3445	unsigned cPDEs = X86_PG_PAE_ENTRIES * 4; /* treat it as a 2048 entry table. */
3446	# else
3447	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3448	# endif
3449	unsigned cIncrement = X86_PG_PAE_ENTRIES * GUEST_PAGE_SIZE;
3450	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3451	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3452	unsigned cIncrement = X86_PG_PAE_ENTRIES * GUEST_PAGE_SIZE;
3453	# endif
3454	if (cb != ~(RTGCPTR)0)
3455	cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
3456
3457	/** @todo call the other two PGMAssert() functions. /
3458
3459	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3460	unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3461
3462	for (; iPml4 < X86_PG_PAE_ENTRIES; iPml4++)
3463	{
3464	PPGMPOOLPAGE pShwPdpt = NULL;
3465	PX86PML4E pPml4eSrc;
3466	PX86PML4E pPml4eDst;
3467	RTGCPHYS GCPhysPdptSrc;
3468
3469	pPml4eSrc = pgmGstGetLongModePML4EPtr(pVCpu, iPml4);
3470	pPml4eDst = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
3471
3472	/* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
3473	if (!(pPml4eDst->u & X86_PML4E_P))
3474	{
3475	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3476	continue;
3477	}
3478
3479	pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
3480	GCPhysPdptSrc = PGM_A20_APPLY(pVCpu, pPml4eSrc->u & X86_PML4E_PG_MASK);
3481
3482	if ((pPml4eSrc->u & X86_PML4E_P) != (pPml4eDst->u & X86_PML4E_P))
3483	{
3484	AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3485	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3486	cErrors++;
3487	continue;
3488	}
3489
3490	if (GCPhysPdptSrc != pShwPdpt->GCPhys)
3491	{
3492	AssertMsgFailed(("Physical address doesn't match! iPml4 %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
3493	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3494	cErrors++;
3495	continue;
3496	}
3497
3498	if ( (pPml4eDst->u & (X86_PML4E_US \| X86_PML4E_RW \| X86_PML4E_NX))
3499	!= (pPml4eSrc->u & (X86_PML4E_US \| X86_PML4E_RW \| X86_PML4E_NX)))
3500	{
3501	AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3502	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3503	cErrors++;
3504	continue;
3505	}
3506	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3507	{
3508	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3509
3510	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
3511	/*
3512	* Check the PDPTEs too.
3513	*/
3514	unsigned iPdpt = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
3515
3516	for (;iPdpt <= SHW_PDPT_MASK; iPdpt++)
3517	{
3518	unsigned iPDSrc = 0; /* initialized to shut up gcc */
3519	PPGMPOOLPAGE pShwPde = NULL;
3520	PX86PDPE pPdpeDst;
3521	RTGCPHYS GCPhysPdeSrc;
3522	X86PDPE PdpeSrc;
3523	PdpeSrc.u = 0; /* initialized to shut up gcc 4.5 */
3524	# if PGM_GST_TYPE == PGM_TYPE_PAE
3525	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtr, &iPDSrc, &PdpeSrc);
3526	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
3527	# else
3528	PX86PML4E pPml4eSrcIgn;
3529	PX86PDPT pPdptDst;
3530	PX86PDPAE pPDDst;
3531	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtr, &pPml4eSrcIgn, &PdpeSrc, &iPDSrc);
3532
3533	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtr, NULL, &pPdptDst, &pPDDst);
3534	if (rc != VINF_SUCCESS)
3535	{
3536	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
3537	GCPtr += 512 * _2M;
3538	continue; /* next PDPTE */
3539	}
3540	Assert(pPDDst);
3541	# endif
3542	Assert(iPDSrc == 0);
3543
3544	pPdpeDst = &pPdptDst->a[iPdpt];
3545
3546	if (!(pPdpeDst->u & X86_PDPE_P))
3547	{
3548	GCPtr += 512 * _2M;
3549	continue; /* next PDPTE */
3550	}
3551
3552	pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
3553	GCPhysPdeSrc = PGM_A20_APPLY(pVCpu, PdpeSrc.u & X86_PDPE_PG_MASK);
3554
3555	if ((pPdpeDst->u & X86_PDPE_P) != (PdpeSrc.u & X86_PDPE_P))
3556	{
3557	AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3558	GCPtr += 512 * _2M;
3559	cErrors++;
3560	continue;
3561	}
3562
3563	if (GCPhysPdeSrc != pShwPde->GCPhys)
3564	{
3565	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3566	AssertMsgFailed(("Physical address doesn't match! iPml4 %d iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3567	# else
3568	AssertMsgFailed(("Physical address doesn't match! iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3569	# endif
3570	GCPtr += 512 * _2M;
3571	cErrors++;
3572	continue;
3573	}
3574
3575	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3576	if ( (pPdpeDst->u & (X86_PDPE_US \| X86_PDPE_RW \| X86_PDPE_LM_NX))
3577	!= (PdpeSrc.u & (X86_PDPE_US \| X86_PDPE_RW \| X86_PDPE_LM_NX)))
3578	{
3579	AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3580	GCPtr += 512 * _2M;
3581	cErrors++;
3582	continue;
3583	}
3584	# endif
3585
3586	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3587	{
3588	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3589	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3590	GSTPD const *pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3591	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3592	PCX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
3593	# endif
3594	# endif /* PGM_GST_TYPE == PGM_TYPE_32BIT */
3595	/*
3596	* Iterate the shadow page directory.
3597	*/
3598	GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
3599	unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
3600
3601	for (;
3602	iPDDst < cPDEs;
3603	iPDDst++, GCPtr += cIncrement)
3604	{
3605	# if PGM_SHW_TYPE == PGM_TYPE_PAE
3606	const SHWPDE PdeDst = *pgmShwGetPaePDEPtr(pVCpu, GCPtr);
3607	# else
3608	const SHWPDE PdeDst = pPDDst->a[iPDDst];
3609	# endif
3610	if ( (PdeDst.u & X86_PDE_P)
3611	\|\| ((PdeDst.u & (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) )
3612	{
3613	HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
3614	PPGMPOOLPAGE pPoolPage = pgmPoolGetPage(pPool, HCPhysShw);
3615	if (!pPoolPage)
3616	{
3617	AssertMsgFailed(("Invalid page table address %RHp at %RGv! PdeDst=%#RX64\n",
3618	HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
3619	cErrors++;
3620	continue;
3621	}
3622	const SHWPT pPTDst = (const SHWPT )PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pPoolPage);
3623
3624	if (PdeDst.u & (X86_PDE4M_PWT \| X86_PDE4M_PCD))
3625	{
3626	AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n",
3627	GCPtr, (uint64_t)PdeDst.u));
3628	cErrors++;
3629	}
3630
3631	if (PdeDst.u & (X86_PDE4M_G \| X86_PDE4M_D))
3632	{
3633	AssertMsgFailed(("4K PDE reserved flags at %RGv! PdeDst=%#RX64\n",
3634	GCPtr, (uint64_t)PdeDst.u));
3635	cErrors++;
3636	}
3637
3638	const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
3639	if (!(PdeSrc.u & X86_PDE_P))
3640	{
3641	AssertMsgFailed(("Guest PDE at %RGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
3642	GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
3643	cErrors++;
3644	continue;
3645	}
3646
3647	if ( !(PdeSrc.u & X86_PDE_PS)
3648	\|\| !fBigPagesSupported)
3649	{
3650	GCPhysGst = GST_GET_PDE_GCPHYS(PdeSrc);
3651	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3652	GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst \| ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2)));
3653	# endif
3654	}
3655	else
3656	{
3657	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3658	if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
3659	{
3660	AssertMsgFailed(("Guest PDE at %RGv is using PSE36 or similar! PdeSrc=%#RX64\n",
3661	GCPtr, (uint64_t)PdeSrc.u));
3662	cErrors++;
3663	continue;
3664	}
3665	# endif
3666	GCPhysGst = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
3667	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3668	GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst \| (GCPtr & RT_BIT(X86_PAGE_2M_SHIFT)));
3669	# endif
3670	}
3671
3672	if ( pPoolPage->enmKind
3673	!= (!(PdeSrc.u & X86_PDE_PS) \|\| !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
3674	{
3675	AssertMsgFailed(("Invalid shadow page table kind %d at %RGv! PdeSrc=%#RX64\n",
3676	pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
3677	cErrors++;
3678	}
3679
3680	PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
3681	if (!pPhysPage)
3682	{
3683	AssertMsgFailed(("Cannot find guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3684	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3685	cErrors++;
3686	continue;
3687	}
3688
3689	if (GCPhysGst != pPoolPage->GCPhys)
3690	{
3691	AssertMsgFailed(("GCPhysGst=%RGp != pPage->GCPhys=%RGp at %RGv\n",
3692	GCPhysGst, pPoolPage->GCPhys, GCPtr));
3693	cErrors++;
3694	continue;
3695	}
3696
3697	if ( !(PdeSrc.u & X86_PDE_PS)
3698	\|\| !fBigPagesSupported)
3699	{
3700	/*
3701	* Page Table.
3702	*/
3703	const GSTPT *pPTSrc;
3704	rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, PGM_A20_APPLY(pVCpu, GCPhysGst & ~(RTGCPHYS)(GUEST_PAGE_SIZE - 1)),
3705	&pPTSrc);
3706	if (RT_FAILURE(rc))
3707	{
3708	AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
3709	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
3710	cErrors++;
3711	continue;
3712	}
3713	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/))
3714	!= (PdeDst.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/)))
3715	{
3716	/// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
3717	// (This problem will go away when/if we shadow multiple CR3s.)
3718	AssertMsgFailed(("4K PDE flags mismatch at %RGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3719	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3720	cErrors++;
3721	continue;
3722	}
3723	if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3724	{
3725	AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%RGv PdeDst=%#RX64\n",
3726	GCPtr, (uint64_t)PdeDst.u));
3727	cErrors++;
3728	continue;
3729	}
3730
3731	/* iterate the page table. */
3732	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
3733	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
3734	const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
3735	# else
3736	const unsigned offPTSrc = 0;
3737	# endif
3738	for (unsigned iPT = 0, off = 0;
3739	iPT < RT_ELEMENTS(pPTDst->a);
3740	iPT++, off += GUEST_PAGE_SIZE)
3741	{
3742	const SHWPTE PteDst = pPTDst->a[iPT];
3743
3744	/* skip not-present and dirty tracked entries. */
3745	if (!(SHW_PTE_GET_U(PteDst) & (X86_PTE_P \| PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
3746	continue;
3747	Assert(SHW_PTE_IS_P(PteDst));
3748
3749	const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
3750	if (!(PteSrc.u & X86_PTE_P))
3751	{
3752	# ifdef IN_RING3
3753	PGMAssertHandlerAndFlagsInSync(pVM);
3754	DBGFR3PagingDumpEx(pVM->pUVM, pVCpu->idCpu, DBGFPGDMP_FLAGS_CURRENT_CR3 \| DBGFPGDMP_FLAGS_CURRENT_MODE
3755	\| DBGFPGDMP_FLAGS_GUEST \| DBGFPGDMP_FLAGS_HEADER \| DBGFPGDMP_FLAGS_PRINT_CR3,
3756	0, 0, UINT64_MAX, 99, NULL);
3757	# endif
3758	AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n",
3759	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst), pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
3760	(uint64_t)GST_GET_PDE_GCPHYS(PdeSrc) + (iPT + offPTSrc) * sizeof(PteSrc)));
3761	cErrors++;
3762	continue;
3763	}
3764
3765	uint64_t fIgnoreFlags = GST_PTE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_G \| X86_PTE_D \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
3766	# if 1 /** @todo sync accessed bit properly... */
3767	fIgnoreFlags \|= X86_PTE_A;
3768	# endif
3769
3770	/* match the physical addresses */
3771	HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
3772	GCPhysGst = GST_GET_PTE_GCPHYS(PteSrc);
3773
3774	# ifdef IN_RING3
3775	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
3776	if (RT_FAILURE(rc))
3777	{
3778	# if 0
3779	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
3780	{
3781	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3782	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3783	cErrors++;
3784	continue;
3785	}
3786	# endif
3787	}
3788	else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
3789	{
3790	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3791	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3792	cErrors++;
3793	continue;
3794	}
3795	# endif
3796
3797	pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
3798	if (!pPhysPage)
3799	{
3800	# if 0
3801	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
3802	{
3803	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
3804	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3805	cErrors++;
3806	continue;
3807	}
3808	# endif
3809	if (SHW_PTE_IS_RW(PteDst))
3810	{
3811	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3812	GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3813	cErrors++;
3814	}
3815	fIgnoreFlags \|= X86_PTE_RW;
3816	}
3817	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
3818	{
3819	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage:%R[pgmpage] GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
3820	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3821	cErrors++;
3822	continue;
3823	}
3824
3825	/* flags */
3826	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
3827	{
3828	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
3829	{
3830	if (SHW_PTE_IS_RW(PteDst))
3831	{
3832	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
3833	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3834	cErrors++;
3835	continue;
3836	}
3837	fIgnoreFlags \|= X86_PTE_RW;
3838	}
3839	else
3840	{
3841	if ( SHW_PTE_IS_P(PteDst)
3842	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
3843	&& !PGM_PAGE_IS_MMIO(pPhysPage)
3844	# endif
3845	)
3846	{
3847	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
3848	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3849	cErrors++;
3850	continue;
3851	}
3852	fIgnoreFlags \|= X86_PTE_P;
3853	}
3854	}
3855	else
3856	{
3857	if ((PteSrc.u & (X86_PTE_RW \| X86_PTE_D)) == X86_PTE_RW)
3858	{
3859	if (SHW_PTE_IS_RW(PteDst))
3860	{
3861	AssertMsgFailed(("!DIRTY page at %RGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
3862	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3863	cErrors++;
3864	continue;
3865	}
3866	if (!SHW_PTE_IS_TRACK_DIRTY(PteDst))
3867	{
3868	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3869	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3870	cErrors++;
3871	continue;
3872	}
3873	if (SHW_PTE_IS_D(PteDst))
3874	{
3875	AssertMsgFailed(("!DIRTY page at %RGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3876	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3877	cErrors++;
3878	}
3879	# if 0 /** @todo sync access bit properly... */
3880	if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
3881	{
3882	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
3883	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3884	cErrors++;
3885	}
3886	fIgnoreFlags \|= X86_PTE_RW;
3887	# else
3888	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
3889	# endif
3890	}
3891	else if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
3892	{
3893	/* access bit emulation (not implemented). */
3894	if ((PteSrc.u & X86_PTE_A) \|\| SHW_PTE_IS_P(PteDst))
3895	{
3896	AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
3897	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3898	cErrors++;
3899	continue;
3900	}
3901	if (!SHW_PTE_IS_A(PteDst))
3902	{
3903	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
3904	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3905	cErrors++;
3906	}
3907	fIgnoreFlags \|= X86_PTE_P;
3908	}
3909	# ifdef DEBUG_sandervl
3910	fIgnoreFlags \|= X86_PTE_D \| X86_PTE_A;
3911	# endif
3912	}
3913
3914	if ( (PteSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
3915	&& (PteSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
3916	)
3917	{
3918	AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
3919	GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
3920	fIgnoreFlags, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
3921	cErrors++;
3922	continue;
3923	}
3924	} /* foreach PTE */
3925	}
3926	else
3927	{
3928	/*
3929	* Big Page.
3930	*/
3931	uint64_t fIgnoreFlags = X86_PDE_AVL_MASK \| GST_PDE_PG_MASK \| X86_PDE4M_G \| X86_PDE4M_D \| X86_PDE4M_PS \| X86_PDE4M_PWT \| X86_PDE4M_PCD;
3932	if ((PdeSrc.u & (X86_PDE_RW \| X86_PDE4M_D)) == X86_PDE_RW)
3933	{
3934	if (PdeDst.u & X86_PDE_RW)
3935	{
3936	AssertMsgFailed(("!DIRTY page at %RGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3937	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3938	cErrors++;
3939	continue;
3940	}
3941	if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
3942	{
3943	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
3944	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3945	cErrors++;
3946	continue;
3947	}
3948	# if 0 /** @todo sync access bit properly... */
3949	if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
3950	{
3951	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
3952	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3953	cErrors++;
3954	}
3955	fIgnoreFlags \|= X86_PTE_RW;
3956	# else
3957	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
3958	# endif
3959	}
3960	else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
3961	{
3962	/* access bit emulation (not implemented). */
3963	if ((PdeSrc.u & X86_PDE_A) \|\| SHW_PDE_IS_P(PdeDst))
3964	{
3965	AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3966	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3967	cErrors++;
3968	continue;
3969	}
3970	if (!SHW_PDE_IS_A(PdeDst))
3971	{
3972	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
3973	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3974	cErrors++;
3975	}
3976	fIgnoreFlags \|= X86_PTE_P;
3977	}
3978
3979	if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
3980	{
3981	AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
3982	GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
3983	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
3984	cErrors++;
3985	}
3986
3987	/* iterate the page table. */
3988	for (unsigned iPT = 0, off = 0;
3989	iPT < RT_ELEMENTS(pPTDst->a);
3990	iPT++, off += GUEST_PAGE_SIZE, GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst + GUEST_PAGE_SIZE))
3991	{
3992	const SHWPTE PteDst = pPTDst->a[iPT];
3993
3994	if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
3995	{
3996	AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
3997	GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
3998	cErrors++;
3999	}
4000
4001	/* skip not-present entries. */
4002	if (!SHW_PTE_IS_P(PteDst)) /** @todo deal with ALL handlers and CSAM !P pages! */
4003	continue;
4004
4005	fIgnoreFlags = X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT \| X86_PTE_D \| X86_PTE_A \| X86_PTE_G \| X86_PTE_PAE_NX;
4006
4007	/* match the physical addresses */
4008	HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
4009
4010	# ifdef IN_RING3
4011	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4012	if (RT_FAILURE(rc))
4013	{
4014	# if 0
4015	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4016	{
4017	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4018	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4019	cErrors++;
4020	}
4021	# endif
4022	}
4023	else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
4024	{
4025	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4026	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4027	cErrors++;
4028	continue;
4029	}
4030	# endif
4031	pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4032	if (!pPhysPage)
4033	{
4034	# if 0 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4035	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4036	{
4037	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4038	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4039	cErrors++;
4040	continue;
4041	}
4042	# endif
4043	if (SHW_PTE_IS_RW(PteDst))
4044	{
4045	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4046	GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4047	cErrors++;
4048	}
4049	fIgnoreFlags \|= X86_PTE_RW;
4050	}
4051	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4052	{
4053	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage=%R[pgmpage] GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4054	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4055	cErrors++;
4056	continue;
4057	}
4058
4059	/* flags */
4060	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4061	{
4062	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4063	{
4064	if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
4065	{
4066	if (SHW_PTE_IS_RW(PteDst))
4067	{
4068	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4069	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4070	cErrors++;
4071	continue;
4072	}
4073	fIgnoreFlags \|= X86_PTE_RW;
4074	}
4075	}
4076	else
4077	{
4078	if ( SHW_PTE_IS_P(PteDst)
4079	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
4080	&& !PGM_PAGE_IS_MMIO(pPhysPage)
4081	# endif
4082	)
4083	{
4084	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4085	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4086	cErrors++;
4087	continue;
4088	}
4089	fIgnoreFlags \|= X86_PTE_P;
4090	}
4091	}
4092
4093	if ( (PdeSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4094	&& (PdeSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags) /* lazy phys handler dereg. */
4095	)
4096	{
4097	AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
4098	GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
4099	fIgnoreFlags, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4100	cErrors++;
4101	continue;
4102	}
4103	} /* for each PTE */
4104	}
4105	}
4106	/* not present */
4107
4108	} /* for each PDE */
4109
4110	} /* for each PDPTE */
4111
4112	} /* for each PML4E */
4113
4114	# ifdef DEBUG
4115	if (cErrors)
4116	LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
4117	# endif
4118	# endif /* GST is in {32BIT, PAE, AMD64} */
4119	return cErrors;
4120	#endif /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE */
4121	}
4122	#endif /* VBOX_STRICT */
4123
4124
4125	/**
4126	* Sets up the CR3 for shadow paging
4127	*
4128	* @returns Strict VBox status code.
4129	* @retval VINF_SUCCESS.
4130	*
4131	* @param pVCpu The cross context virtual CPU structure.
4132	* @param GCPhysCR3 The physical address in the CR3 register. (A20 mask
4133	* already applied.)
4134	*/
4135	PGM_BTH_DECL(int, MapCR3)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3)
4136	{
4137	PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
4138	int rc = VINF_SUCCESS;
4139
4140	/* Update guest paging info. */
4141	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
4142	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
4143	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
4144
4145	LogFlow(("MapCR3: %RGp\n", GCPhysCR3));
4146	PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3);
4147
4148	# if PGM_GST_TYPE == PGM_TYPE_PAE
4149	if (!pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped))
4150	# endif
4151	{
4152	/*
4153	* Map the page CR3 points at.
4154	*/
4155	RTHCPTR HCPtrGuestCR3;
4156	rc = pgmGstMapCr3(pVCpu, GCPhysCR3, &HCPtrGuestCR3);
4157	if (RT_SUCCESS(rc))
4158	{
4159	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4160	# ifdef IN_RING3
4161	pVCpu->pgm.s.pGst32BitPdR3 = (PX86PD)HCPtrGuestCR3;
4162	pVCpu->pgm.s.pGst32BitPdR0 = NIL_RTR0PTR;
4163	# else
4164	pVCpu->pgm.s.pGst32BitPdR3 = NIL_RTR3PTR;
4165	pVCpu->pgm.s.pGst32BitPdR0 = (PX86PD)HCPtrGuestCR3;
4166	# endif
4167
4168	# elif PGM_GST_TYPE == PGM_TYPE_PAE
4169	# ifdef IN_RING3
4170	pVCpu->pgm.s.pGstPaePdptR3 = (PX86PDPT)HCPtrGuestCR3;
4171	pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
4172	# else
4173	pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
4174	pVCpu->pgm.s.pGstPaePdptR0 = (PX86PDPT)HCPtrGuestCR3;
4175	# endif
4176
4177	/*
4178	* Update CPUM and map the 4 PDs too.
4179	*/
4180	X86PDPE aGstPaePdpes[X86_PG_PAE_PDPE_ENTRIES];
4181	memcpy(&aGstPaePdpes, HCPtrGuestCR3, sizeof(aGstPaePdpes));
4182	CPUMSetGuestPaePdpes(pVCpu, &aGstPaePdpes[0]);
4183	PGMGstMapPaePdpes(pVCpu, &aGstPaePdpes[0]);
4184
4185	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
4186	# ifdef IN_RING3
4187	pVCpu->pgm.s.pGstAmd64Pml4R3 = (PX86PML4)HCPtrGuestCR3;
4188	pVCpu->pgm.s.pGstAmd64Pml4R0 = NIL_RTR0PTR;
4189	# else
4190	pVCpu->pgm.s.pGstAmd64Pml4R3 = NIL_RTR3PTR;
4191	pVCpu->pgm.s.pGstAmd64Pml4R0 = (PX86PML4)HCPtrGuestCR3;
4192	# endif
4193	# endif
4194	}
4195	else
4196	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4197	}
4198
4199	/*
4200	* Reset fPaePdpesAndCr3Mapped for all modes as there's no guarantee that
4201	* we were called in the correct sequence of PAE followed by other modes
4202	* without CR3 changing in between.
4203	*/
4204	pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = false;
4205	pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = false;
4206	#endif
4207
4208	/*
4209	* Update shadow paging info for guest modes with paging (32-bit, PAE, AMD64).
4210	*/
4211	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4212	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4213	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64) \
4214	&& ( PGM_GST_TYPE != PGM_TYPE_REAL \
4215	&& PGM_GST_TYPE != PGM_TYPE_PROT))
4216
4217	Assert(!pVM->pgm.s.fNestedPaging);
4218	PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3);
4219
4220	/*
4221	* Update the shadow root page as well since that's not fixed.
4222	*/
4223	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4224	PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
4225	PPGMPOOLPAGE pNewShwPageCR3;
4226
4227	PGM_LOCK_VOID(pVM);
4228
4229	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4230	if (pPool->cDirtyPages)
4231	pgmPoolResetDirtyPages(pVM);
4232	# endif
4233
4234	Assert(!(GCPhysCR3 >> (GUEST_PAGE_SHIFT + 32))); /** @todo what is this for? */
4235	int const rc2 = pgmPoolAlloc(pVM, GCPhysCR3 & GST_CR3_PAGE_MASK, BTH_PGMPOOLKIND_ROOT, PGMPOOLACCESS_DONTCARE,
4236	PGM_A20_IS_ENABLED(pVCpu), NIL_PGMPOOL_IDX, UINT32_MAX, true /fLockPage/, &pNewShwPageCR3);
4237	AssertFatalRC(rc2);
4238
4239	pVCpu->pgm.s.pShwPageCR3R3 = pgmPoolConvertPageToR3(pPool, pNewShwPageCR3);
4240	pVCpu->pgm.s.pShwPageCR3R0 = pgmPoolConvertPageToR0(pPool, pNewShwPageCR3);
4241
4242	/* Set the current hypervisor CR3. */
4243	CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
4244
4245	/* Clean up the old CR3 root. */
4246	if ( pOldShwPageCR3
4247	&& pOldShwPageCR3 != pNewShwPageCR3 /* @todo can happen due to incorrect syncing between REM & PGM; find the real cause */)
4248	{
4249	Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
4250
4251	/* Mark the page as unlocked; allow flushing again. */
4252	pgmPoolUnlockPage(pPool, pOldShwPageCR3);
4253
4254	pgmPoolFreeByPage(pPool, pOldShwPageCR3, NIL_PGMPOOL_IDX, UINT32_MAX);
4255	}
4256	PGM_UNLOCK(pVM);
4257	# else
4258	NOREF(GCPhysCR3);
4259	# endif
4260
4261	return rc;
4262	}
4263
4264	/**
4265	* Unmaps the shadow CR3.
4266	*
4267	* @returns VBox status, no specials.
4268	* @param pVCpu The cross context virtual CPU structure.
4269	*/
4270	PGM_BTH_DECL(int, UnmapCR3)(PVMCPUCC pVCpu)
4271	{
4272	LogFlow(("UnmapCR3\n"));
4273
4274	int rc = VINF_SUCCESS;
4275	PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
4276
4277	/*
4278	* Update guest paging info.
4279	*/
4280	#if PGM_GST_TYPE == PGM_TYPE_32BIT
4281	pVCpu->pgm.s.pGst32BitPdR3 = 0;
4282	pVCpu->pgm.s.pGst32BitPdR0 = 0;
4283
4284	#elif PGM_GST_TYPE == PGM_TYPE_PAE
4285	pVCpu->pgm.s.pGstPaePdptR3 = 0;
4286	pVCpu->pgm.s.pGstPaePdptR0 = 0;
4287	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
4288	{
4289	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4290	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4291	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4292	}
4293
4294	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
4295	pVCpu->pgm.s.pGstAmd64Pml4R3 = 0;
4296	pVCpu->pgm.s.pGstAmd64Pml4R0 = 0;
4297
4298	#else /* prot/real mode stub */
4299	/* nothing to do */
4300	#endif
4301
4302	/*
4303	* Update second-level address translation info.
4304	*/
4305	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
4306	pVCpu->pgm.s.pGstEptPml4R3 = 0;
4307	pVCpu->pgm.s.pGstEptPml4R0 = 0;
4308	#endif
4309
4310	pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = false;
4311	pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = false;
4312
4313	/*
4314	* Update shadow paging info.
4315	*/
4316	#if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4317	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4318	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64))
4319	# if PGM_GST_TYPE != PGM_TYPE_REAL
4320	Assert(!pVM->pgm.s.fNestedPaging);
4321	# endif
4322	PGM_LOCK_VOID(pVM);
4323
4324	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4325	{
4326	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4327
4328	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4329	if (pPool->cDirtyPages)
4330	pgmPoolResetDirtyPages(pVM);
4331	# endif
4332
4333	/* Mark the page as unlocked; allow flushing again. */
4334	pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4335
4336	pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), NIL_PGMPOOL_IDX, UINT32_MAX);
4337	pVCpu->pgm.s.pShwPageCR3R3 = 0;
4338	pVCpu->pgm.s.pShwPageCR3R0 = 0;
4339	}
4340
4341	PGM_UNLOCK(pVM);
4342	#endif
4343
4344	return rc;
4345	}
4346

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 93718

Download in other formats: