PGMAllBth.h@ 62517

Last change on this file since 62517 was 62478, checked in by vboxsync, 8 years ago
(C) 2016
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 211.3 KB

Line
1	/* $Id: PGMAllBth.h 62478 2016-07-22 18:29:06Z vboxsync $ */
2	/** @file
3	* VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4	*
5	* @remarks The nested page tables on AMD makes use of PGM_SHW_TYPE in
6	* {PGM_TYPE_AMD64, PGM_TYPE_PAE and PGM_TYPE_32BIT} and PGM_GST_TYPE
7	* set to PGM_TYPE_PROT. Half of the code in this file is not
8	* exercised with PGM_SHW_TYPE set to PGM_TYPE_NESTED.
9	*
10	* @remarks Extended page tables (intel) are built with PGM_GST_TYPE set to
11	* PGM_TYPE_PROT (and PGM_SHW_TYPE set to PGM_TYPE_EPT).
12	*
13	* @remarks This file is one big \#ifdef-orgy!
14	*
15	*/
16
17	/*
18	* Copyright (C) 2006-2016 Oracle Corporation
19	*
20	* This file is part of VirtualBox Open Source Edition (OSE), as
21	* available from http://www.virtualbox.org. This file is free software;
22	* you can redistribute it and/or modify it under the terms of the GNU
23	* General Public License (GPL) as published by the Free Software
24	* Foundation, in version 2 as it comes in the "COPYING" file of the
25	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
26	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
27	*/
28
29	#ifdef _MSC_VER
30	/** @todo we're generating unnecessary code in nested/ept shadow mode and for
31	* real/prot-guest+RC mode. */
32	# pragma warning(disable: 4505)
33	#endif
34
35	/*******************************************************************************
36	* Internal Functions *
37	*******************************************************************************/
38	RT_C_DECLS_BEGIN
39	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken);
40	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
41	static int PGM_BTH_NAME(SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr);
42	static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc, RTGCPTR GCPtrPage);
43	static int PGM_BTH_NAME(SyncPT)(PVMCPU pVCpu, unsigned iPD, PGSTPD pPDSrc, RTGCPTR GCPtrPage);
44	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
45	static void PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst);
46	# else
47	static void PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage, unsigned iPTDst);
48	#endif
49	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uErr);
50	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
51	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
52	#ifdef VBOX_STRICT
53	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
54	#endif
55	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3);
56	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu);
57	RT_C_DECLS_END
58
59
60	/*
61	* Filter out some illegal combinations of guest and shadow paging, so we can
62	* remove redundant checks inside functions.
63	*/
64	#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
65	# error "Invalid combination; PAE guest implies PAE shadow"
66	#endif
67
68	#if (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
69	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT)
70	# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
71	#endif
72
73	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) \
74	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT)
75	# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
76	#endif
77
78	#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT) \
79	\|\| (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
80	# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
81	#endif
82
83	#ifndef IN_RING3
84
85	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
86	/**
87	* Deal with a guest page fault.
88	*
89	* @returns Strict VBox status code.
90	* @retval VINF_EM_RAW_GUEST_TRAP
91	* @retval VINF_EM_RAW_EMULATE_INSTR
92	*
93	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
94	* @param pGstWalk The guest page table walk result.
95	* @param uErr The error code.
96	*/
97	PGM_BTH_DECL(VBOXSTRICTRC, Trap0eHandlerGuestFault)(PVMCPU pVCpu, PGSTPTWALK pGstWalk, RTGCUINT uErr)
98	{
99	# if !defined(PGM_WITHOUT_MAPPINGS) && (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE)
100	/*
101	* Check for write conflicts with our hypervisor mapping.
102	*
103	* If the guest happens to access a non-present page, where our hypervisor
104	* is currently mapped, then we'll create a #PF storm in the guest.
105	*/
106	if ( (uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RW)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RW)
107	&& pgmMapAreMappingsEnabled(pVCpu->CTX_SUFF(pVM))
108	&& MMHyperIsInsideArea(pVCpu->CTX_SUFF(pVM), pGstWalk->Core.GCPtr))
109	{
110	/* Force a CR3 sync to check for conflicts and emulate the instruction. */
111	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
112	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
113	return VINF_EM_RAW_EMULATE_INSTR;
114	}
115	# endif
116
117	/*
118	* Calc the error code for the guest trap.
119	*/
120	uint32_t uNewErr = GST_IS_NX_ACTIVE(pVCpu)
121	? uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID)
122	: uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US);
123	if (pGstWalk->Core.fBadPhysAddr)
124	{
125	uNewErr \|= X86_TRAP_PF_RSVD \| X86_TRAP_PF_P;
126	Assert(!pGstWalk->Core.fNotPresent);
127	}
128	else if (!pGstWalk->Core.fNotPresent)
129	uNewErr \|= X86_TRAP_PF_P;
130	TRPMSetErrorCode(pVCpu, uNewErr);
131
132	LogFlow(("Guest trap; cr2=%RGv uErr=%RGv lvl=%d\n", pGstWalk->Core.GCPtr, uErr, pGstWalk->Core.uLevel));
133	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
134	return VINF_EM_RAW_GUEST_TRAP;
135	}
136	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
137
138
139	/**
140	* Deal with a guest page fault.
141	*
142	* The caller has taken the PGM lock.
143	*
144	* @returns Strict VBox status code.
145	*
146	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
147	* @param uErr The error code.
148	* @param pRegFrame The register frame.
149	* @param pvFault The fault address.
150	* @param pPage The guest page at @a pvFault.
151	* @param pGstWalk The guest page table walk result.
152	* @param pfLockTaken PGM lock taken here or not (out). This is true
153	* when we're called.
154	*/
155	static VBOXSTRICTRC PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame,
156	RTGCPTR pvFault, PPGMPAGE pPage, bool *pfLockTaken
157	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \|\| defined(DOXYGEN_RUNNING)
158	, PGSTPTWALK pGstWalk
159	# endif
160	)
161	{
162	# if !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
163	GSTPDE const PdeSrcDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
164	#endif
165	PVM pVM = pVCpu->CTX_SUFF(pVM);
166	VBOXSTRICTRC rcStrict;
167
168	if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
169	{
170	/*
171	* Physical page access handler.
172	*/
173	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
174	const RTGCPHYS GCPhysFault = pGstWalk->Core.GCPhys;
175	# else
176	const RTGCPHYS GCPhysFault = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault);
177	# endif
178	PPGMPHYSHANDLER pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault);
179	if (pCur)
180	{
181	PPGMPHYSHANDLERTYPEINT pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
182
183	# ifdef PGM_SYNC_N_PAGES
184	/*
185	* If the region is write protected and we got a page not present fault, then sync
186	* the pages. If the fault was caused by a read, then restart the instruction.
187	* In case of write access continue to the GC write handler.
188	*
189	* ASSUMES that there is only one handler per page or that they have similar write properties.
190	*/
191	if ( !(uErr & X86_TRAP_PF_P)
192	&& pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
193	{
194	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
195	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
196	# else
197	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
198	# endif
199	if ( RT_FAILURE(rcStrict)
200	\|\| !(uErr & X86_TRAP_PF_RW)
201	\|\| rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
202	{
203	AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
204	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
205	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
206	return rcStrict;
207	}
208	}
209	# endif
210	# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
211	/*
212	* If the access was not thru a #PF(RSVD\|...) resync the page.
213	*/
214	if ( !(uErr & X86_TRAP_PF_RSVD)
215	&& pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
216	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
217	&& pGstWalk->Core.fEffectiveRW
218	&& !pGstWalk->Core.fEffectiveUS /** @todo Remove pGstWalk->Core.fEffectiveUS and X86_PTE_US further down in the sync code. */
219	# endif
220	)
221	{
222	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
223	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
224	# else
225	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
226	# endif
227	if ( RT_FAILURE(rcStrict)
228	\|\| rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
229	{
230	AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
231	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
232	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
233	return rcStrict;
234	}
235	}
236	# endif
237
238	AssertMsg( pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
239	\|\| (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW)),
240	("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enmKind=%d\n",
241	pvFault, GCPhysFault, pPage, uErr, pCurType->enmKind));
242	if (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
243	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysWrite);
244	else
245	{
246	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysAll);
247	if (uErr & X86_TRAP_PF_RSVD) STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysAllOpt);
248	}
249
250	if (pCurType->CTX_SUFF(pfnPfHandler))
251	{
252	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
253	void *pvUser = pCur->CTX_SUFF(pvUser);
254
255	STAM_PROFILE_START(&pCur->Stat, h);
256	if (pCur->hType != pPool->hAccessHandlerType)
257	{
258	pgmUnlock(pVM);
259	*pfLockTaken = false;
260	}
261
262	rcStrict = pCurType->CTX_SUFF(pfnPfHandler)(pVM, pVCpu, uErr, pRegFrame, pvFault, GCPhysFault, pvUser);
263
264	# ifdef VBOX_WITH_STATISTICS
265	pgmLock(pVM);
266	pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault);
267	if (pCur)
268	STAM_PROFILE_STOP(&pCur->Stat, h);
269	pgmUnlock(pVM);
270	# endif
271	}
272	else
273	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
274
275	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndPhys; });
276	return rcStrict;
277	}
278	}
279	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0)
280	else
281	{
282	# ifdef PGM_SYNC_N_PAGES
283	/*
284	* If the region is write protected and we got a page not present fault, then sync
285	* the pages. If the fault was caused by a read, then restart the instruction.
286	* In case of write access continue to the GC write handler.
287	*/
288	if ( PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) < PGM_PAGE_HNDL_PHYS_STATE_ALL
289	&& !(uErr & X86_TRAP_PF_P))
290	{
291	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
292	if ( RT_FAILURE(rcStrict)
293	\|\| rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE
294	\|\| !(uErr & X86_TRAP_PF_RW))
295	{
296	AssertRC(rcStrict);
297	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
298	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndVirt; });
299	return rcStrict;
300	}
301	}
302	# endif
303	/*
304	* Ok, it's an virtual page access handler.
305	*
306	* Since it's faster to search by address, we'll do that first
307	* and then retry by GCPhys if that fails.
308	*/
309	/** @todo r=bird: perhaps we should consider looking up by physical address directly now?
310	* r=svl: true, but lookup on virtual address should remain as a fallback as phys & virt trees might be
311	* out of sync, because the page was changed without us noticing it (not-present -> present
312	* without invlpg or mov cr3, xxx).
313	*/
314	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
315	if (pCur)
316	{
317	PPGMVIRTHANDLERTYPEINT pCurType = PGMVIRTANDLER_GET_TYPE(pVM, pCur);
318	AssertMsg(!(pvFault - pCur->Core.Key < pCur->cb)
319	\|\| ( pCurType->enmKind != PGMVIRTHANDLERKIND_WRITE
320	\|\| !(uErr & X86_TRAP_PF_P)
321	\|\| (pCurType->enmKind == PGMVIRTHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW))),
322	("Unexpected trap for virtual handler: %RGv (phys=%RGp) pPage=%R[pgmpage] uErr=%X, enumKind=%d\n",
323	pvFault, pGstWalk->Core.GCPhys, pPage, uErr, pCurType->enmKind));
324
325	if ( pvFault - pCur->Core.Key < pCur->cb
326	&& ( uErr & X86_TRAP_PF_RW
327	\|\| pCurType->enmKind != PGMVIRTHANDLERKIND_WRITE ) )
328	{
329	# ifdef IN_RC
330	STAM_PROFILE_START(&pCur->Stat, h);
331	RTGCPTR GCPtrStart = pCur->Core.Key;
332	void *pvUser = pCur->CTX_SUFF(pvUser);
333	pgmUnlock(pVM);
334	*pfLockTaken = false;
335
336	rcStrict = pCurType->CTX_SUFF(pfnPfHandler)(pVM, pVCpu, uErr, pRegFrame, pvFault, GCPtrStart,
337	pvFault - GCPtrStart, pvUser);
338
339	# ifdef VBOX_WITH_STATISTICS
340	pgmLock(pVM);
341	pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
342	if (pCur)
343	STAM_PROFILE_STOP(&pCur->Stat, h);
344	pgmUnlock(pVM);
345	# endif
346	# else
347	rcStrict = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
348	# endif
349	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersVirtual);
350	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
351	return rcStrict;
352	}
353	/* Unhandled part of a monitored page */
354	Log(("Unhandled part of monitored page %RGv\n", pvFault));
355	}
356	else
357	{
358	/* Check by physical address. */
359	unsigned iPage;
360	pCur = pgmHandlerVirtualFindByPhysAddr(pVM, pGstWalk->Core.GCPhys, &iPage);
361	if (pCur)
362	{
363	PPGMVIRTHANDLERTYPEINT pCurType = PGMVIRTANDLER_GET_TYPE(pVM, pCur);
364	if ( uErr & X86_TRAP_PF_RW
365	\|\| pCurType->enmKind != PGMVIRTHANDLERKIND_WRITE )
366	{
367	Assert( (pCur->aPhysToVirt[iPage].Core.Key & X86_PTE_PAE_PG_MASK)
368	== (pGstWalk->Core.GCPhys & X86_PTE_PAE_PG_MASK));
369	# ifdef IN_RC
370	STAM_PROFILE_START(&pCur->Stat, h);
371	RTGCPTR GCPtrStart = pCur->Core.Key;
372	void *pvUser = pCur->CTX_SUFF(pvUser);
373	pgmUnlock(pVM);
374	*pfLockTaken = false;
375
376	RTGCPTR off = (iPage << PAGE_SHIFT)
377	+ (pvFault & PAGE_OFFSET_MASK)
378	- (GCPtrStart & PAGE_OFFSET_MASK);
379	Assert(off < pCur->cb);
380	rcStrict = pCurType->CTX_SUFF(pfnPfHandler)(pVM, pVCpu, uErr, pRegFrame, pvFault, GCPtrStart, off, pvUser);
381
382	# ifdef VBOX_WITH_STATISTICS
383	pgmLock(pVM);
384	pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, GCPtrStart);
385	if (pCur)
386	STAM_PROFILE_STOP(&pCur->Stat, h);
387	pgmUnlock(pVM);
388	# endif
389	# else
390	rcStrict = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
391	# endif
392	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersVirtualByPhys);
393	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
394	return rcStrict;
395	}
396	}
397	}
398	}
399	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
400
401	/*
402	* There is a handled area of the page, but this fault doesn't belong to it.
403	* We must emulate the instruction.
404	*
405	* To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
406	* we first check if this was a page-not-present fault for a page with only
407	* write access handlers. Restart the instruction if it wasn't a write access.
408	*/
409	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersUnhandled);
410
411	if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
412	&& !(uErr & X86_TRAP_PF_P))
413	{
414	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
415	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
416	# else
417	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
418	# endif
419	if ( RT_FAILURE(rcStrict)
420	\|\| rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE
421	\|\| !(uErr & X86_TRAP_PF_RW))
422	{
423	AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
424	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
425	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
426	return rcStrict;
427	}
428	}
429
430	/** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
431	* It's writing to an unhandled part of the LDT page several million times.
432	*/
433	rcStrict = PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault);
434	LogFlow(("PGM: PGMInterpretInstruction -> rcStrict=%d pPage=%R[pgmpage]\n", VBOXSTRICTRC_VAL(rcStrict), pPage));
435	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndUnhandled; });
436	return rcStrict;
437	} /* if any kind of handler */
438
439
440	/**
441	* \#PF Handler for raw-mode guest execution.
442	*
443	* @returns VBox status code (appropriate for trap handling and GC return).
444	*
445	* @param pVCpu The cross context virtual CPU structure.
446	* @param uErr The trap error code.
447	* @param pRegFrame Trap register frame.
448	* @param pvFault The fault address.
449	* @param pfLockTaken PGM lock taken here or not (out)
450	*/
451	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken)
452	{
453	PVM pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
454
455	*pfLockTaken = false;
456
457	# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \
458	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
459	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
460	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT)
461	int rc;
462
463	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
464	/*
465	* Walk the guest page translation tables and check if it's a guest fault.
466	*/
467	GSTPTWALK GstWalk;
468	rc = PGM_GST_NAME(Walk)(pVCpu, pvFault, &GstWalk);
469	if (RT_FAILURE_NP(rc))
470	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &GstWalk, uErr));
471
472	/* assert some GstWalk sanity. */
473	# if PGM_GST_TYPE == PGM_TYPE_AMD64
474	/AssertMsg(GstWalk.Pml4e.u == GstWalk.pPml4e->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pml4e.u, (uint64_t)GstWalk.pPml4e->u)); - not always true with SMP guests. /
475	# endif
476	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
477	/AssertMsg(GstWalk.Pdpe.u == GstWalk.pPdpe->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pdpe.u, (uint64_t)GstWalk.pPdpe->u)); - ditto /
478	# endif
479	/AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); - ditto /
480	/AssertMsg(GstWalk.Core.fBigPage \|\| GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); - ditto /
481	Assert(GstWalk.Core.fSucceeded);
482
483	if (uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID))
484	{
485	if ( ( (uErr & X86_TRAP_PF_RW)
486	&& !GstWalk.Core.fEffectiveRW
487	&& ( (uErr & X86_TRAP_PF_US)
488	\|\| CPUMIsGuestR0WriteProtEnabled(pVCpu)) )
489	\|\| ((uErr & X86_TRAP_PF_US) && !GstWalk.Core.fEffectiveUS)
490	\|\| ((uErr & X86_TRAP_PF_ID) && GstWalk.Core.fEffectiveNX)
491	)
492	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &GstWalk, uErr));
493	}
494
495	/*
496	* Set the accessed and dirty flags.
497	*/
498	# if PGM_GST_TYPE == PGM_TYPE_AMD64
499	GstWalk.Pml4e.u \|= X86_PML4E_A;
500	GstWalk.pPml4e->u \|= X86_PML4E_A;
501	GstWalk.Pdpe.u \|= X86_PDPE_A;
502	GstWalk.pPdpe->u \|= X86_PDPE_A;
503	# endif
504	if (GstWalk.Core.fBigPage)
505	{
506	Assert(GstWalk.Pde.b.u1Size);
507	if (uErr & X86_TRAP_PF_RW)
508	{
509	GstWalk.Pde.u \|= X86_PDE4M_A \| X86_PDE4M_D;
510	GstWalk.pPde->u \|= X86_PDE4M_A \| X86_PDE4M_D;
511	}
512	else
513	{
514	GstWalk.Pde.u \|= X86_PDE4M_A;
515	GstWalk.pPde->u \|= X86_PDE4M_A;
516	}
517	}
518	else
519	{
520	Assert(!GstWalk.Pde.b.u1Size);
521	GstWalk.Pde.u \|= X86_PDE_A;
522	GstWalk.pPde->u \|= X86_PDE_A;
523	if (uErr & X86_TRAP_PF_RW)
524	{
525	# ifdef VBOX_WITH_STATISTICS
526	if (!GstWalk.Pte.n.u1Dirty)
527	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtiedPage));
528	else
529	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageAlreadyDirty));
530	# endif
531	GstWalk.Pte.u \|= X86_PTE_A \| X86_PTE_D;
532	GstWalk.pPte->u \|= X86_PTE_A \| X86_PTE_D;
533	}
534	else
535	{
536	GstWalk.Pte.u \|= X86_PTE_A;
537	GstWalk.pPte->u \|= X86_PTE_A;
538	}
539	Assert(GstWalk.Pte.u == GstWalk.pPte->u);
540	}
541	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u \|\| GstWalk.pPte->u == GstWalk.pPde->u,
542	("%RX64 %RX64 pPte=%p pPde=%p Pte=%RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u, GstWalk.pPte, GstWalk.pPde, (uint64_t)GstWalk.pPte->u));
543	# else /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
544	GSTPDE const PdeSrcDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A}; /** @todo eliminate this */
545	# endif /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
546
547	/* Take the big lock now. */
548	*pfLockTaken = true;
549	pgmLock(pVM);
550
551	# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
552	/*
553	* If it is a reserved bit fault we know that it is an MMIO (access
554	* handler) related fault and can skip some 200 lines of code.
555	*/
556	if (uErr & X86_TRAP_PF_RSVD)
557	{
558	Assert(uErr & X86_TRAP_PF_P);
559	PPGMPAGE pPage;
560	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
561	rc = pgmPhysGetPageEx(pVM, GstWalk.Core.GCPhys, &pPage);
562	if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
563	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
564	pfLockTaken, &GstWalk));
565	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
566	# else
567	rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault), &pPage);
568	if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
569	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
570	pfLockTaken));
571	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
572	#endif
573	AssertRC(rc);
574	PGM_INVL_PG(pVCpu, pvFault);
575	return rc; /* Restart with the corrected entry. */
576	}
577	# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
578
579	/*
580	* Fetch the guest PDE, PDPE and PML4E.
581	*/
582	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
583	const unsigned iPDDst = pvFault >> SHW_PD_SHIFT;
584	PX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
585
586	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
587	const unsigned iPDDst = (pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK; /* pPDDst index, not used with the pool. */
588	PX86PDPAE pPDDst;
589	# if PGM_GST_TYPE == PGM_TYPE_PAE
590	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, GstWalk.Pdpe.u, &pPDDst);
591	# else
592	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, X86_PDPE_P, &pPDDst); /* RW, US and A are reserved in PAE mode. */
593	# endif
594	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
595
596	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
597	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
598	PX86PDPAE pPDDst;
599	# if PGM_GST_TYPE == PGM_TYPE_PROT /* (AMD-V nested paging) */
600	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A,
601	X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A, &pPDDst);
602	# else
603	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, GstWalk.Pml4e.u, GstWalk.Pdpe.u, &pPDDst);
604	# endif
605	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
606
607	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
608	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
609	PEPTPD pPDDst;
610	rc = pgmShwGetEPTPDPtr(pVCpu, pvFault, NULL, &pPDDst);
611	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
612	# endif
613	Assert(pPDDst);
614
615	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
616	/*
617	* Dirty page handling.
618	*
619	* If we successfully correct the write protection fault due to dirty bit
620	* tracking, then return immediately.
621	*/
622	if (uErr & X86_TRAP_PF_RW) /* write fault? */
623	{
624	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyBitTracking), a);
625	rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, &pPDDst->a[iPDDst], GstWalk.pPde, pvFault);
626	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyBitTracking), a);
627	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
628	{
629	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution)
630	= rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
631	? &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2DirtyAndAccessed
632	: &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
633	Log8(("Trap0eHandler: returns VINF_SUCCESS\n"));
634	return VINF_SUCCESS;
635	}
636	//AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u \|\| GstWalk.pPte->u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); - triggers with smp w7 guests.
637	//AssertMsg(GstWalk.Core.fBigPage \|\| GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); - ditto.
638	}
639
640	# if 0 /* rarely useful; leave for debugging. */
641	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0ePD[iPDSrc]);
642	# endif
643	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
644
645	/*
646	* A common case is the not-present error caused by lazy page table syncing.
647	*
648	* It is IMPORTANT that we weed out any access to non-present shadow PDEs
649	* here so we can safely assume that the shadow PT is present when calling
650	* SyncPage later.
651	*
652	* On failure, we ASSUME that SyncPT is out of memory or detected some kind
653	* of mapping conflict and defer to SyncCR3 in R3.
654	* (Again, we do NOT support access handlers for non-present guest pages.)
655	*
656	*/
657	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
658	Assert(GstWalk.Pde.n.u1Present);
659	# endif
660	if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
661	&& !pPDDst->a[iPDDst].n.u1Present)
662	{
663	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2SyncPT; });
664	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
665	LogFlow(("=>SyncPT %04x = %08RX64\n", (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, (uint64_t)GstWalk.Pde.u));
666	rc = PGM_BTH_NAME(SyncPT)(pVCpu, (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, GstWalk.pPd, pvFault);
667	# else
668	LogFlow(("=>SyncPT pvFault=%RGv\n", pvFault));
669	rc = PGM_BTH_NAME(SyncPT)(pVCpu, 0, NULL, pvFault);
670	# endif
671	if (RT_SUCCESS(rc))
672	return rc;
673	Log(("SyncPT: %RGv failed!! rc=%Rrc\n", pvFault, rc));
674	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
675	return VINF_PGM_SYNC_CR3;
676	}
677
678	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(PGM_WITHOUT_MAPPINGS)
679	/*
680	* Check if this address is within any of our mappings.
681	*
682	* This is very fast and it's gonna save us a bit of effort below and prevent
683	* us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
684	* (BTW, it's impossible to have physical access handlers in a mapping.)
685	*/
686	if (pgmMapAreMappingsEnabled(pVM))
687	{
688	PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
689	for ( ; pMapping; pMapping = pMapping->CTX_SUFF(pNext))
690	{
691	if (pvFault < pMapping->GCPtr)
692	break;
693	if (pvFault - pMapping->GCPtr < pMapping->cb)
694	{
695	/*
696	* The first thing we check is if we've got an undetected conflict.
697	*/
698	if (pgmMapAreMappingsFloating(pVM))
699	{
700	unsigned iPT = pMapping->cb >> GST_PD_SHIFT;
701	while (iPT-- > 0)
702	if (GstWalk.pPde[iPT].n.u1Present)
703	{
704	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eConflicts);
705	Log(("Trap0e: Detected Conflict %RGv-%RGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
706	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
707	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
708	return VINF_PGM_SYNC_CR3;
709	}
710	}
711
712	/*
713	* Check if the fault address is in a virtual page access handler range.
714	*/
715	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->HyperVirtHandlers,
716	pvFault);
717	if ( pCur
718	&& pvFault - pCur->Core.Key < pCur->cb
719	&& uErr & X86_TRAP_PF_RW)
720	{
721	VBOXSTRICTRC rcStrict;
722	# ifdef IN_RC
723	STAM_PROFILE_START(&pCur->Stat, h);
724	PPGMVIRTHANDLERTYPEINT pCurType = PGMVIRTANDLER_GET_TYPE(pVM, pCur);
725	void *pvUser = pCur->CTX_SUFF(pvUser);
726	pgmUnlock(pVM);
727	rcStrict = pCurType->CTX_SUFF(pfnPfHandler)(pVM, pVCpu, uErr, pRegFrame, pvFault, pCur->Core.Key,
728	pvFault - pCur->Core.Key, pvUser);
729	pgmLock(pVM);
730	STAM_PROFILE_STOP(&pCur->Stat, h);
731	# else
732	AssertFailed();
733	rcStrict = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
734	# endif
735	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersMapping);
736	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
737	return VBOXSTRICTRC_TODO(rcStrict);
738	}
739
740	/*
741	* Pretend we're not here and let the guest handle the trap.
742	*/
743	TRPMSetErrorCode(pVCpu, uErr & ~X86_TRAP_PF_P);
744	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eGuestPFMapping);
745	LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
746	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
747	return VINF_EM_RAW_GUEST_TRAP;
748	}
749	}
750	} /* pgmAreMappingsEnabled(&pVM->pgm.s) */
751	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
752
753	/*
754	* Check if this fault address is flagged for special treatment,
755	* which means we'll have to figure out the physical address and
756	* check flags associated with it.
757	*
758	* ASSUME that we can limit any special access handling to pages
759	* in page tables which the guest believes to be present.
760	*/
761	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
762	RTGCPHYS GCPhys = GstWalk.Core.GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
763	# else
764	RTGCPHYS GCPhys = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault & ~(RTGCPHYS)PAGE_OFFSET_MASK);
765	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
766	PPGMPAGE pPage;
767	rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
768	if (RT_FAILURE(rc))
769	{
770	/*
771	* When the guest accesses invalid physical memory (e.g. probing
772	* of RAM or accessing a remapped MMIO range), then we'll fall
773	* back to the recompiler to emulate the instruction.
774	*/
775	LogFlow(("PGM #PF: pgmPhysGetPageEx(%RGp) failed with %Rrc\n", GCPhys, rc));
776	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersInvalid);
777	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2InvalidPhys; });
778	return VINF_EM_RAW_EMULATE_INSTR;
779	}
780
781	/*
782	* Any handlers for this page?
783	*/
784	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
785	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
786	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage, pfLockTaken,
787	&GstWalk));
788	# else
789	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage, pfLockTaken));
790	# endif
791
792	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0)
793	if (uErr & X86_TRAP_PF_P)
794	{
795	/*
796	* The page isn't marked, but it might still be monitored by a virtual page access handler.
797	* (ASSUMES no temporary disabling of virtual handlers.)
798	*/
799	/** @todo r=bird: Since the purpose is to catch out of sync pages with virtual handler(s) here,
800	* we should correct both the shadow page table and physical memory flags, and not only check for
801	* accesses within the handler region but for access to pages with virtual handlers. */
802	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
803	if (pCur)
804	{
805	PPGMVIRTHANDLERTYPEINT pCurType = PGMVIRTANDLER_GET_TYPE(pVM, pCur);
806	AssertMsg( !(pvFault - pCur->Core.Key < pCur->cb)
807	\|\| ( pCurType->enmKind != PGMVIRTHANDLERKIND_WRITE
808	\|\| !(uErr & X86_TRAP_PF_P)
809	\|\| (pCurType->enmKind == PGMVIRTHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW))),
810	("Unexpected trap for virtual handler: %08X (phys=%08x) %R[pgmpage] uErr=%X, enumKind=%d\n",
811	pvFault, GCPhys, pPage, uErr, pCurType->enmKind));
812
813	if ( pvFault - pCur->Core.Key < pCur->cb
814	&& ( uErr & X86_TRAP_PF_RW
815	\|\| pCurType->enmKind != PGMVIRTHANDLERKIND_WRITE ) )
816	{
817	VBOXSTRICTRC rcStrict;
818	# ifdef IN_RC
819	STAM_PROFILE_START(&pCur->Stat, h);
820	void *pvUser = pCur->CTX_SUFF(pvUser);
821	pgmUnlock(pVM);
822	rcStrict = pCurType->CTX_SUFF(pfnPfHandler)(pVM, pVCpu, uErr, pRegFrame, pvFault, pCur->Core.Key,
823	pvFault - pCur->Core.Key, pvUser);
824	pgmLock(pVM);
825	STAM_PROFILE_STOP(&pCur->Stat, h);
826	# else
827	rcStrict = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
828	# endif
829	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
830	return VBOXSTRICTRC_TODO(rcStrict);
831	}
832	}
833	}
834	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
835
836	/*
837	* We are here only if page is present in Guest page tables and
838	* trap is not handled by our handlers.
839	*
840	* Check it for page out-of-sync situation.
841	*/
842	if (!(uErr & X86_TRAP_PF_P))
843	{
844	/*
845	* Page is not present in our page tables. Try to sync it!
846	*/
847	if (uErr & X86_TRAP_PF_US)
848	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUser));
849	else /* supervisor */
850	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
851
852	if (PGM_PAGE_IS_BALLOONED(pPage))
853	{
854	/* Emulate reads from ballooned pages as they are not present in
855	our shadow page tables. (Required for e.g. Solaris guests; soft
856	ecc, random nr generator.) */
857	rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
858	LogFlow(("PGM: PGMInterpretInstruction balloon -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
859	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncBallloon));
860	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Ballooned; });
861	return rc;
862	}
863
864	# if defined(LOG_ENABLED) && !defined(IN_RING0)
865	RTGCPHYS GCPhys2;
866	uint64_t fPageGst2;
867	PGMGstGetPage(pVCpu, pvFault, &fPageGst2, &GCPhys2);
868	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
869	Log(("Page out of sync: %RGv eip=%08x PdeSrc.US=%d fPageGst2=%08llx GCPhys2=%RGp scan=%d\n",
870	pvFault, pRegFrame->eip, GstWalk.Pde.n.u1User, fPageGst2, GCPhys2, CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)));
871	# else
872	Log(("Page out of sync: %RGv eip=%08x fPageGst2=%08llx GCPhys2=%RGp scan=%d\n",
873	pvFault, pRegFrame->eip, fPageGst2, GCPhys2, CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)));
874	# endif
875	# endif /* LOG_ENABLED */
876
877	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0)
878	if ( !GstWalk.Core.fEffectiveUS
879	&& CSAMIsEnabled(pVM)
880	&& CPUMGetGuestCPL(pVCpu) == 0)
881	{
882	/* Note: Can't check for X86_TRAP_ID bit, because that requires execute disable support on the CPU. */
883	if ( pvFault == (RTGCPTR)pRegFrame->eip
884	\|\| pvFault - pRegFrame->eip < 8 /* instruction crossing a page boundary */
885	# ifdef CSAM_DETECT_NEW_CODE_PAGES
886	\|\| ( !PATMIsPatchGCAddr(pVM, pRegFrame->eip)
887	&& CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)) /* any new code we encounter here */
888	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
889	)
890	{
891	LogFlow(("CSAMExecFault %RX32\n", pRegFrame->eip));
892	rc = CSAMExecFault(pVM, (RTRCPTR)pRegFrame->eip);
893	if (rc != VINF_SUCCESS)
894	{
895	/*
896	* CSAM needs to perform a job in ring 3.
897	*
898	* Sync the page before going to the host context; otherwise we'll end up in a loop if
899	* CSAM fails (e.g. instruction crosses a page boundary and the next page is not present)
900	*/
901	LogFlow(("CSAM ring 3 job\n"));
902	int rc2 = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
903	AssertRC(rc2);
904
905	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2CSAM; });
906	return rc;
907	}
908	}
909	# ifdef CSAM_DETECT_NEW_CODE_PAGES
910	else if ( uErr == X86_TRAP_PF_RW
911	&& pRegFrame->ecx >= 0x100 /* early check for movswd count */
912	&& pRegFrame->ecx < 0x10000)
913	{
914	/* In case of a write to a non-present supervisor shadow page, we'll take special precautions
915	* to detect loading of new code pages.
916	*/
917
918	/*
919	* Decode the instruction.
920	*/
921	PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
922	uint32_t cbOp;
923	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
924
925	/* For now we'll restrict this to rep movsw/d instructions */
926	if ( rc == VINF_SUCCESS
927	&& pDis->pCurInstr->opcode == OP_MOVSWD
928	&& (pDis->prefix & DISPREFIX_REP))
929	{
930	CSAMMarkPossibleCodePage(pVM, pvFault);
931	}
932	}
933	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
934
935	/*
936	* Mark this page as safe.
937	*/
938	/** @todo not correct for pages that contain both code and data!! */
939	Log2(("CSAMMarkPage %RGv; scanned=%d\n", pvFault, true));
940	CSAMMarkPage(pVM, pvFault, true);
941	}
942	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0) */
943	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
944	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
945	# else
946	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
947	# endif
948	if (RT_SUCCESS(rc))
949	{
950	/* The page was successfully synced, return to the guest. */
951	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSync; });
952	return VINF_SUCCESS;
953	}
954	}
955	else /* uErr & X86_TRAP_PF_P: */
956	{
957	/*
958	* Write protected pages are made writable when the guest makes the
959	* first write to it. This happens for pages that are shared, write
960	* monitored or not yet allocated.
961	*
962	* We may also end up here when CR0.WP=0 in the guest.
963	*
964	* Also, a side effect of not flushing global PDEs are out of sync
965	* pages due to physical monitored regions, that are no longer valid.
966	* Assume for now it only applies to the read/write flag.
967	*/
968	if (uErr & X86_TRAP_PF_RW)
969	{
970	/*
971	* Check if it is a read-only page.
972	*/
973	if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
974	{
975	Log(("PGM #PF: Make writable: %RGp %R[pgmpage] pvFault=%RGp uErr=%#x\n", GCPhys, pPage, pvFault, uErr));
976	Assert(!PGM_PAGE_IS_ZERO(pPage));
977	AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
978	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2MakeWritable; });
979
980	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
981	if (rc != VINF_SUCCESS)
982	{
983	AssertMsg(rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("%Rrc\n", rc));
984	return rc;
985	}
986	if (RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)))
987	return VINF_EM_NO_MEMORY;
988	}
989
990	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
991	/*
992	* Check to see if we need to emulate the instruction if CR0.WP=0.
993	*/
994	if ( !GstWalk.Core.fEffectiveRW
995	&& (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG
996	&& CPUMGetGuestCPL(pVCpu) < 3)
997	{
998	Assert((uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RW \| X86_TRAP_PF_P));
999
1000	/*
1001	* The Netware WP0+RO+US hack.
1002	*
1003	* Netware sometimes(/always?) runs with WP0. It has been observed doing
1004	* excessive write accesses to pages which are mapped with US=1 and RW=0
1005	* while WP=0. This causes a lot of exits and extremely slow execution.
1006	* To avoid trapping and emulating every write here, we change the shadow
1007	* page table entry to map it as US=0 and RW=1 until user mode tries to
1008	* access it again (see further below). We count these shadow page table
1009	* changes so we can avoid having to clear the page pool every time the WP
1010	* bit changes to 1 (see PGMCr0WpEnabled()).
1011	*/
1012	# if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) && 1
1013	if ( GstWalk.Core.fEffectiveUS
1014	&& !GstWalk.Core.fEffectiveRW
1015	&& (GstWalk.Core.fBigPage \|\| GstWalk.Pde.n.u1Write)
1016	&& pVM->cCpus == 1 /* Sorry, no go on SMP. Add CFGM option? */)
1017	{
1018	Log(("PGM #PF: Netware WP0+RO+US hack: pvFault=%RGp uErr=%#x (big=%d)\n", pvFault, uErr, GstWalk.Core.fBigPage));
1019	rc = pgmShwMakePageSupervisorAndWritable(pVCpu, pvFault, GstWalk.Core.fBigPage, PGM_MK_PG_IS_WRITE_FAULT);
1020	if (rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3)
1021	{
1022	PGM_INVL_PG(pVCpu, pvFault);
1023	pVCpu->pgm.s.cNetwareWp0Hacks++;
1024	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Wp0RoUsHack; });
1025	return rc;
1026	}
1027	AssertMsg(RT_FAILURE_NP(rc), ("%Rrc\n", rc));
1028	Log(("pgmShwMakePageSupervisorAndWritable(%RGv) failed with rc=%Rrc - ignored\n", pvFault, rc));
1029	}
1030	# endif
1031
1032	/* Interpret the access. */
1033	rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
1034	Log(("PGM #PF: WP0 emulation (pvFault=%RGp uErr=%#x cpl=%d fBig=%d fEffUs=%d)\n", pvFault, uErr, CPUMGetGuestCPL(pVCpu), GstWalk.Core.fBigPage, GstWalk.Core.fEffectiveUS));
1035	if (RT_SUCCESS(rc))
1036	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eWPEmulInRZ);
1037	else
1038	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eWPEmulToR3);
1039	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2WPEmulation; });
1040	return rc;
1041	}
1042	# endif
1043	/// @todo count the above case; else
1044	if (uErr & X86_TRAP_PF_US)
1045	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUserWrite));
1046	else /* supervisor */
1047	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
1048
1049	/*
1050	* Sync the page.
1051	*
1052	* Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
1053	* page is not present, which is not true in this case.
1054	*/
1055	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1056	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
1057	# else
1058	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
1059	# endif
1060	if (RT_SUCCESS(rc))
1061	{
1062	/*
1063	* Page was successfully synced, return to guest but invalidate
1064	* the TLB first as the page is very likely to be in it.
1065	*/
1066	# if PGM_SHW_TYPE == PGM_TYPE_EPT
1067	HMInvalidatePhysPage(pVM, (RTGCPHYS)pvFault);
1068	# else
1069	PGM_INVL_PG(pVCpu, pvFault);
1070	# endif
1071	# ifdef VBOX_STRICT
1072	RTGCPHYS GCPhys2 = RTGCPHYS_MAX;
1073	uint64_t fPageGst = UINT64_MAX;
1074	if (!pVM->pgm.s.fNestedPaging)
1075	{
1076	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, &GCPhys2);
1077	AssertMsg(RT_SUCCESS(rc) && ((fPageGst & X86_PTE_RW) \|\| ((CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG && CPUMGetGuestCPL(pVCpu) < 3)), ("rc=%Rrc fPageGst=%RX64\n", rc, fPageGst));
1078	LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GCPhys2, (uint64_t)fPageGst));
1079	}
1080	# if 0 /* Bogus! Triggers incorrectly with w7-64 and later for the SyncPage case: "Pde at %RGv changed behind our back?" */
1081	uint64_t fPageShw = 0;
1082	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
1083	AssertMsg((RT_SUCCESS(rc) && (fPageShw & X86_PTE_RW)) \|\| pVM->cCpus > 1 /* new monitor can be installed/page table flushed between the trap exit and PGMTrap0eHandler */,
1084	("rc=%Rrc fPageShw=%RX64 GCPhys2=%RGp fPageGst=%RX64 pvFault=%RGv\n", rc, fPageShw, GCPhys2, fPageGst, pvFault));
1085	# endif
1086	# endif /* VBOX_STRICT */
1087	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndObs; });
1088	return VINF_SUCCESS;
1089	}
1090	}
1091	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1092	/*
1093	* Check for Netware WP0+RO+US hack from above and undo it when user
1094	* mode accesses the page again.
1095	*/
1096	else if ( GstWalk.Core.fEffectiveUS
1097	&& !GstWalk.Core.fEffectiveRW
1098	&& (GstWalk.Core.fBigPage \|\| GstWalk.Pde.n.u1Write)
1099	&& pVCpu->pgm.s.cNetwareWp0Hacks > 0
1100	&& (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG
1101	&& CPUMGetGuestCPL(pVCpu) == 3
1102	&& pVM->cCpus == 1
1103	)
1104	{
1105	Log(("PGM #PF: Undo netware WP0+RO+US hack: pvFault=%RGp uErr=%#x\n", pvFault, uErr));
1106	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
1107	if (RT_SUCCESS(rc))
1108	{
1109	PGM_INVL_PG(pVCpu, pvFault);
1110	pVCpu->pgm.s.cNetwareWp0Hacks--;
1111	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Wp0RoUsUnhack; });
1112	return VINF_SUCCESS;
1113	}
1114	}
1115	# endif /* PGM_WITH_PAGING */
1116
1117	/** @todo else: why are we here? */
1118
1119	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && defined(VBOX_STRICT)
1120	/*
1121	* Check for VMM page flags vs. Guest page flags consistency.
1122	* Currently only for debug purposes.
1123	*/
1124	if (RT_SUCCESS(rc))
1125	{
1126	/* Get guest page flags. */
1127	uint64_t fPageGst;
1128	int rc2 = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
1129	if (RT_SUCCESS(rc2))
1130	{
1131	uint64_t fPageShw = 0;
1132	rc2 = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
1133
1134	/*
1135	* Compare page flags.
1136	* Note: we have AVL, A, D bits desynced.
1137	*/
1138	AssertMsg( (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK))
1139	== (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK))
1140	\|\| ( pVCpu->pgm.s.cNetwareWp0Hacks > 0
1141	&& (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US))
1142	== (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US))
1143	&& (fPageShw & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_RW
1144	&& (fPageGst & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_US),
1145	("Page flags mismatch! pvFault=%RGv uErr=%x GCPhys=%RGp fPageShw=%RX64 fPageGst=%RX64 rc=%d\n",
1146	pvFault, (uint32_t)uErr, GCPhys, fPageShw, fPageGst, rc));
1147	}
1148	else
1149	AssertMsgFailed(("PGMGstGetPage rc=%Rrc\n", rc));
1150	}
1151	else
1152	AssertMsgFailed(("PGMGCGetPage rc=%Rrc\n", rc));
1153	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && VBOX_STRICT */
1154	}
1155
1156
1157	/*
1158	* If we get here it is because something failed above, i.e. most like guru
1159	* meditiation time.
1160	*/
1161	LogRel(("%s: returns rc=%Rrc pvFault=%RGv uErr=%RX64 cs:rip=%04x:%08RX64\n",
1162	__PRETTY_FUNCTION__, rc, pvFault, (uint64_t)uErr, pRegFrame->cs.Sel, pRegFrame->rip));
1163	return rc;
1164
1165	# else /* Nested paging, EPT except PGM_GST_TYPE = PROT */
1166	NOREF(uErr); NOREF(pRegFrame); NOREF(pvFault);
1167	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
1168	return VERR_PGM_NOT_USED_IN_MODE;
1169	# endif
1170	}
1171	#endif /* !IN_RING3 */
1172
1173
1174	/**
1175	* Emulation of the invlpg instruction.
1176	*
1177	*
1178	* @returns VBox status code.
1179	*
1180	* @param pVCpu The cross context virtual CPU structure.
1181	* @param GCPtrPage Page to invalidate.
1182	*
1183	* @remark ASSUMES that the guest is updating before invalidating. This order
1184	* isn't required by the CPU, so this is speculative and could cause
1185	* trouble.
1186	* @remark No TLB shootdown is done on any other VCPU as we assume that
1187	* invlpg emulation is the only reason for calling this function.
1188	* (The guest has to shoot down TLB entries on other CPUs itself)
1189	* Currently true, but keep in mind!
1190	*
1191	* @todo Clean this up! Most of it is (or should be) no longer necessary as we catch all page table accesses.
1192	* Should only be required when PGMPOOL_WITH_OPTIMIZED_DIRTY_PT is active (PAE or AMD64 (for now))
1193	*/
1194	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
1195	{
1196	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1197	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
1198	&& PGM_SHW_TYPE != PGM_TYPE_EPT
1199	int rc;
1200	PVM pVM = pVCpu->CTX_SUFF(pVM);
1201	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1202
1203	PGM_LOCK_ASSERT_OWNER(pVM);
1204
1205	LogFlow(("InvalidatePage %RGv\n", GCPtrPage));
1206
1207	/*
1208	* Get the shadow PD entry and skip out if this PD isn't present.
1209	* (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
1210	*/
1211	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1212	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1213	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1214
1215	/* Fetch the pgm pool shadow descriptor. */
1216	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1217	Assert(pShwPde);
1218
1219	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1220	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT);
1221	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
1222
1223	/* If the shadow PDPE isn't present, then skip the invalidate. */
1224	if (!pPdptDst->a[iPdpt].n.u1Present)
1225	{
1226	Assert(!(pPdptDst->a[iPdpt].u & PGM_PLXFLAGS_MAPPING));
1227	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1228	PGM_INVL_PG(pVCpu, GCPtrPage);
1229	return VINF_SUCCESS;
1230	}
1231
1232	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1233	PPGMPOOLPAGE pShwPde = NULL;
1234	PX86PDPAE pPDDst;
1235
1236	/* Fetch the pgm pool shadow descriptor. */
1237	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
1238	AssertRCSuccessReturn(rc, rc);
1239	Assert(pShwPde);
1240
1241	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1242	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1243
1244	# else /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1245	/* PML4 */
1246	/const unsigned iPml4 = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;/
1247	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1248	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1249	PX86PDPAE pPDDst;
1250	PX86PDPT pPdptDst;
1251	PX86PML4E pPml4eDst;
1252	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eDst, &pPdptDst, &pPDDst);
1253	if (rc != VINF_SUCCESS)
1254	{
1255	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT \|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
1256	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1257	PGM_INVL_PG(pVCpu, GCPtrPage);
1258	return VINF_SUCCESS;
1259	}
1260	Assert(pPDDst);
1261
1262	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1263	PX86PDPE pPdpeDst = &pPdptDst->a[iPdpt];
1264
1265	if (!pPdpeDst->n.u1Present)
1266	{
1267	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1268	PGM_INVL_PG(pVCpu, GCPtrPage);
1269	return VINF_SUCCESS;
1270	}
1271
1272	/* Fetch the pgm pool shadow descriptor. */
1273	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & SHW_PDPE_PG_MASK);
1274	Assert(pShwPde);
1275
1276	# endif /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1277
1278	const SHWPDE PdeDst = *pPdeDst;
1279	if (!PdeDst.n.u1Present)
1280	{
1281	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1282	PGM_INVL_PG(pVCpu, GCPtrPage);
1283	return VINF_SUCCESS;
1284	}
1285
1286	/*
1287	* Get the guest PD entry and calc big page.
1288	*/
1289	# if PGM_GST_TYPE == PGM_TYPE_32BIT
1290	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
1291	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
1292	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
1293	# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1294	unsigned iPDSrc = 0;
1295	# if PGM_GST_TYPE == PGM_TYPE_PAE
1296	X86PDPE PdpeSrcIgn;
1297	PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrcIgn);
1298	# else /* AMD64 */
1299	PX86PML4E pPml4eSrcIgn;
1300	X86PDPE PdpeSrcIgn;
1301	PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrcIgn, &PdpeSrcIgn, &iPDSrc);
1302	# endif
1303	GSTPDE PdeSrc;
1304
1305	if (pPDSrc)
1306	PdeSrc = pPDSrc->a[iPDSrc];
1307	else
1308	PdeSrc.u = 0;
1309	# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1310	const bool fIsBigPage = PdeSrc.b.u1Size && GST_IS_PSE_ACTIVE(pVCpu);
1311
1312	# ifdef IN_RING3
1313	/*
1314	* If a CR3 Sync is pending we may ignore the invalidate page operation
1315	* depending on the kind of sync and if it's a global page or not.
1316	* This doesn't make sense in GC/R0 so we'll skip it entirely there.
1317	*/
1318	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
1319	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
1320	\|\| ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
1321	&& fIsBigPage
1322	&& PdeSrc.b.u1Global
1323	)
1324	)
1325	# else
1326	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
1327	# endif
1328	{
1329	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1330	return VINF_SUCCESS;
1331	}
1332	# endif /* IN_RING3 */
1333
1334	/*
1335	* Deal with the Guest PDE.
1336	*/
1337	rc = VINF_SUCCESS;
1338	if (PdeSrc.n.u1Present)
1339	{
1340	Assert( PdeSrc.n.u1User == PdeDst.n.u1User
1341	&& (PdeSrc.n.u1Write \|\| !PdeDst.n.u1Write \|\| pVCpu->pgm.s.cNetwareWp0Hacks > 0));
1342	# ifndef PGM_WITHOUT_MAPPING
1343	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
1344	{
1345	/*
1346	* Conflict - Let SyncPT deal with it to avoid duplicate code.
1347	*/
1348	Assert(pgmMapAreMappingsEnabled(pVM));
1349	Assert(PGMGetGuestMode(pVCpu) <= PGMMODE_PAE);
1350	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
1351	}
1352	else
1353	# endif /* !PGM_WITHOUT_MAPPING */
1354	if (!fIsBigPage)
1355	{
1356	/*
1357	* 4KB - page.
1358	*/
1359	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1360	RTGCPHYS GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
1361
1362	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1363	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1364	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| ((iPDDst & 1) * (PAGE_SIZE / 2)));
1365	# endif
1366	if (pShwPage->GCPhys == GCPhys)
1367	{
1368	/* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
1369	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1370
1371	PGSTPT pPTSrc;
1372	rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
1373	if (RT_SUCCESS(rc))
1374	{
1375	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1376	GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1377	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1378	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1379	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
1380	GCPtrPage, PteSrc.n.u1Present,
1381	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1382	PteSrc.n.u1User & PdeSrc.n.u1User,
1383	(uint64_t)PteSrc.u,
1384	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
1385	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
1386	}
1387	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4KBPages));
1388	PGM_INVL_PG(pVCpu, GCPtrPage);
1389	}
1390	else
1391	{
1392	/*
1393	* The page table address changed.
1394	*/
1395	LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n",
1396	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
1397	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1398	ASMAtomicWriteSize(pPdeDst, 0);
1399	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1400	PGM_INVL_VCPU_TLBS(pVCpu);
1401	}
1402	}
1403	else
1404	{
1405	/*
1406	* 2/4MB - page.
1407	*/
1408	/* Before freeing the page, check if anything really changed. */
1409	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1410	RTGCPHYS GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
1411	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1412	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1413	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
1414	# endif
1415	if ( pShwPage->GCPhys == GCPhys
1416	&& pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1417	{
1418	/* ASSUMES a the given bits are identical for 4M and normal PDEs */
1419	/** @todo This test is wrong as it cannot check the G bit!
1420	* FIXME */
1421	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US))
1422	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US))
1423	&& ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1424	\|\| (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1425	{
1426	LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1427	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4MBPagesSkip));
1428	return VINF_SUCCESS;
1429	}
1430	}
1431
1432	/*
1433	* Ok, the page table is present and it's been changed in the guest.
1434	* If we're in host context, we'll just mark it as not present taking the lazy approach.
1435	* We could do this for some flushes in GC too, but we need an algorithm for
1436	* deciding which 4MB pages containing code likely to be executed very soon.
1437	*/
1438	LogFlow(("InvalidatePage: Out-of-sync PD at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1439	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1440	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1441	ASMAtomicWriteSize(pPdeDst, 0);
1442	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4MBPages));
1443	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
1444	}
1445	}
1446	else
1447	{
1448	/*
1449	* Page directory is not present, mark shadow PDE not present.
1450	*/
1451	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
1452	{
1453	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1454	ASMAtomicWriteSize(pPdeDst, 0);
1455	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDNPs));
1456	PGM_INVL_PG(pVCpu, GCPtrPage);
1457	}
1458	else
1459	{
1460	Assert(pgmMapAreMappingsEnabled(pVM));
1461	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDMappings));
1462	}
1463	}
1464	return rc;
1465
1466	#else /* guest real and protected mode */
1467	/* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1468	NOREF(pVCpu); NOREF(GCPtrPage);
1469	return VINF_SUCCESS;
1470	#endif
1471	}
1472
1473
1474	/**
1475	* Update the tracking of shadowed pages.
1476	*
1477	* @param pVCpu The cross context virtual CPU structure.
1478	* @param pShwPage The shadow page.
1479	* @param HCPhys The physical page we is being dereferenced.
1480	* @param iPte Shadow PTE index
1481	* @param GCPhysPage Guest physical address (only valid if pShwPage->fDirty is set)
1482	*/
1483	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys, uint16_t iPte,
1484	RTGCPHYS GCPhysPage)
1485	{
1486	PVM pVM = pVCpu->CTX_SUFF(pVM);
1487
1488	# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1489	&& PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1490	&& (PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
1491
1492	/* Use the hint we retrieved from the cached guest PT. */
1493	if (pShwPage->fDirty)
1494	{
1495	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1496
1497	Assert(pShwPage->cPresent);
1498	Assert(pPool->cPresent);
1499	pShwPage->cPresent--;
1500	pPool->cPresent--;
1501
1502	PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysPage);
1503	AssertRelease(pPhysPage);
1504	pgmTrackDerefGCPhys(pPool, pShwPage, pPhysPage, iPte);
1505	return;
1506	}
1507	# else
1508	NOREF(GCPhysPage);
1509	# endif
1510
1511	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackDeref, a);
1512	LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1513
1514	/** @todo If this turns out to be a bottle neck (very likely) two things can be done:
1515	* 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1516	* 2. write protect all shadowed pages. I.e. implement caching.
1517	*/
1518	/** @todo duplicated in the 2nd half of pgmPoolTracDerefGCPhysHint */
1519
1520	/*
1521	* Find the guest address.
1522	*/
1523	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
1524	pRam;
1525	pRam = pRam->CTX_SUFF(pNext))
1526	{
1527	unsigned iPage = pRam->cb >> PAGE_SHIFT;
1528	while (iPage-- > 0)
1529	{
1530	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1531	{
1532	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1533
1534	Assert(pShwPage->cPresent);
1535	Assert(pPool->cPresent);
1536	pShwPage->cPresent--;
1537	pPool->cPresent--;
1538
1539	pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage], iPte);
1540	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackDeref, a);
1541	return;
1542	}
1543	}
1544	}
1545
1546	for (;;)
1547	AssertReleaseMsgFailed(("HCPhys=%RHp wasn't found!\n", HCPhys));
1548	}
1549
1550
1551	/**
1552	* Update the tracking of shadowed pages.
1553	*
1554	* @param pVCpu The cross context virtual CPU structure.
1555	* @param pShwPage The shadow page.
1556	* @param u16 The top 16-bit of the pPage->HCPhys.
1557	* @param pPage Pointer to the guest page. this will be modified.
1558	* @param iPTDst The index into the shadow table.
1559	*/
1560	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, uint16_t u16, PPGMPAGE pPage, const unsigned iPTDst)
1561	{
1562	PVM pVM = pVCpu->CTX_SUFF(pVM);
1563
1564	/*
1565	* Just deal with the simple first time here.
1566	*/
1567	if (!u16)
1568	{
1569	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackVirgin);
1570	u16 = PGMPOOL_TD_MAKE(1, pShwPage->idx);
1571	/* Save the page table index. */
1572	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, iPTDst);
1573	}
1574	else
1575	u16 = pgmPoolTrackPhysExtAddref(pVM, pPage, u16, pShwPage->idx, iPTDst);
1576
1577	/* write back */
1578	Log2(("SyncPageWorkerTrackAddRef: u16=%#x->%#x iPTDst=%#x\n", u16, PGM_PAGE_GET_TRACKING(pPage), iPTDst));
1579	PGM_PAGE_SET_TRACKING(pVM, pPage, u16);
1580
1581	/* update statistics. */
1582	pVM->pgm.s.CTX_SUFF(pPool)->cPresent++;
1583	pShwPage->cPresent++;
1584	if (pShwPage->iFirstPresent > iPTDst)
1585	pShwPage->iFirstPresent = iPTDst;
1586	}
1587
1588
1589	/**
1590	* Modifies a shadow PTE to account for access handlers.
1591	*
1592	* @param pVM The cross context VM structure.
1593	* @param pPage The page in question.
1594	* @param fPteSrc The shadowed flags of the source PTE. Must include the
1595	* A (accessed) bit so it can be emulated correctly.
1596	* @param pPteDst The shadow PTE (output). This is temporary storage and
1597	* does not need to be set atomically.
1598	*/
1599	DECLINLINE(void) PGM_BTH_NAME(SyncHandlerPte)(PVM pVM, PCPGMPAGE pPage, uint64_t fPteSrc, PSHWPTE pPteDst)
1600	{
1601	NOREF(pVM);
1602	/** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No.
1603	* Update: \#PF should deal with this before or after calling the handlers. It has all the info to do the job efficiently. */
1604	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1605	{
1606	LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark read-only\n", pPage));
1607	#if PGM_SHW_TYPE == PGM_TYPE_EPT
1608	pPteDst->u = PGM_PAGE_GET_HCPHYS(pPage);
1609	pPteDst->n.u1Present = 1;
1610	pPteDst->n.u1Execute = 1;
1611	pPteDst->n.u1IgnorePAT = 1;
1612	pPteDst->n.u3EMT = VMX_EPT_MEMTYPE_WB;
1613	/* PteDst.n.u1Write = 0 && PteDst.n.u1Size = 0 */
1614	#else
1615	if (fPteSrc & X86_PTE_A)
1616	{
1617	SHW_PTE_SET(*pPteDst, fPteSrc \| PGM_PAGE_GET_HCPHYS(pPage));
1618	SHW_PTE_SET_RO(*pPteDst);
1619	}
1620	else
1621	SHW_PTE_SET(*pPteDst, 0);
1622	#endif
1623	}
1624	#ifdef PGM_WITH_MMIO_OPTIMIZATIONS
1625	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
1626	else if ( PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
1627	&& ( BTH_IS_NP_ACTIVE(pVM)
1628	\|\| (fPteSrc & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_RW) /** @todo Remove X86_PTE_US here and pGstWalk->Core.fEffectiveUS before the sync page test. */
1629	# if PGM_SHW_TYPE == PGM_TYPE_AMD64
1630	&& pVM->pgm.s.fLessThan52PhysicalAddressBits
1631	# endif
1632	)
1633	{
1634	LogFlow(("SyncHandlerPte: MMIO page -> invalid \n"));
1635	# if PGM_SHW_TYPE == PGM_TYPE_EPT
1636	/* 25.2.3.1: Reserved physical address bit -> EPT Misconfiguration (exit 49) */
1637	pPteDst->u = pVM->pgm.s.HCPhysInvMmioPg;
1638	/* 25.2.3.1: bits 2:0 = 010b -> EPT Misconfiguration (exit 49) */
1639	pPteDst->n.u1Present = 0;
1640	pPteDst->n.u1Write = 1;
1641	pPteDst->n.u1Execute = 0;
1642	/* 25.2.3.1: leaf && 2:0 != 0 && u3Emt in {2, 3, 7} -> EPT Misconfiguration */
1643	pPteDst->n.u3EMT = 7;
1644	# else
1645	/* Set high page frame bits that MBZ (bankers on PAE, CPU dependent on AMD64). */
1646	SHW_PTE_SET(*pPteDst, pVM->pgm.s.HCPhysInvMmioPg \| X86_PTE_PAE_MBZ_MASK_NO_NX \| X86_PTE_P);
1647	# endif
1648	}
1649	# endif
1650	#endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
1651	else
1652	{
1653	LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark not present\n", pPage));
1654	SHW_PTE_SET(*pPteDst, 0);
1655	}
1656	/** @todo count these kinds of entries. */
1657	}
1658
1659
1660	/**
1661	* Creates a 4K shadow page for a guest page.
1662	*
1663	* For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1664	* physical address. The PdeSrc argument only the flags are used. No page
1665	* structured will be mapped in this function.
1666	*
1667	* @param pVCpu The cross context virtual CPU structure.
1668	* @param pPteDst Destination page table entry.
1669	* @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1670	* Can safely assume that only the flags are being used.
1671	* @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1672	* @param pShwPage Pointer to the shadow page.
1673	* @param iPTDst The index into the shadow table.
1674	*
1675	* @remark Not used for 2/4MB pages!
1676	*/
1677	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \|\| defined(DOXYGEN_RUNNING)
1678	static void PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc,
1679	PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1680	#else
1681	static void PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage,
1682	PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1683	#endif
1684	{
1685	PVM pVM = pVCpu->CTX_SUFF(pVM);
1686	RTGCPHYS GCPhysOldPage = NIL_RTGCPHYS;
1687
1688	#if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1689	&& PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1690	&& (PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
1691
1692	if (pShwPage->fDirty)
1693	{
1694	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1695	PGSTPT pGstPT;
1696
1697	/* Note that iPTDst can be used to index the guest PT even in the pae/32bit combo as we copy only half the table; see pgmPoolAddDirtyPage. */
1698	pGstPT = (PGSTPT)&pPool->aDirtyPages[pShwPage->idxDirtyEntry].aPage[0];
1699	GCPhysOldPage = GST_GET_PTE_GCPHYS(pGstPT->a[iPTDst]);
1700	pGstPT->a[iPTDst].u = PteSrc.u;
1701	}
1702	#else
1703	Assert(!pShwPage->fDirty);
1704	#endif
1705
1706	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1707	if ( PteSrc.n.u1Present
1708	&& GST_IS_PTE_VALID(pVCpu, PteSrc))
1709	#endif
1710	{
1711	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1712	RTGCPHYS GCPhysPage = GST_GET_PTE_GCPHYS(PteSrc);
1713	# endif
1714	PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
1715
1716	/*
1717	* Find the ram range.
1718	*/
1719	PPGMPAGE pPage;
1720	int rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
1721	if (RT_SUCCESS(rc))
1722	{
1723	/* Ignore ballooned pages.
1724	Don't return errors or use a fatal assert here as part of a
1725	shadow sync range might included ballooned pages. */
1726	if (PGM_PAGE_IS_BALLOONED(pPage))
1727	{
1728	Assert(!SHW_PTE_IS_P(pPteDst)); /* @todo user tracking needs updating if this triggers. */
1729	return;
1730	}
1731
1732	#ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1733	/* Make the page writable if necessary. */
1734	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1735	&& ( PGM_PAGE_IS_ZERO(pPage)
1736	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1737	\|\| ( PteSrc.n.u1Write
1738	# else
1739	\|\| ( 1
1740	# endif
1741	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1742	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1743	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1744	# endif
1745	# ifdef VBOX_WITH_PAGE_SHARING
1746	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
1747	# endif
1748	)
1749	)
1750	)
1751	{
1752	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhysPage);
1753	AssertRC(rc);
1754	}
1755	#endif
1756
1757	/*
1758	* Make page table entry.
1759	*/
1760	SHWPTE PteDst;
1761	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1762	uint64_t fGstShwPteFlags = GST_GET_PTE_SHW_FLAGS(pVCpu, PteSrc);
1763	# else
1764	uint64_t fGstShwPteFlags = X86_PTE_P \| X86_PTE_RW \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D;
1765	# endif
1766	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1767	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, fGstShwPteFlags, &PteDst);
1768	else
1769	{
1770	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1771	/*
1772	* If the page or page directory entry is not marked accessed,
1773	* we mark the page not present.
1774	*/
1775	if (!PteSrc.n.u1Accessed \|\| !PdeSrc.n.u1Accessed)
1776	{
1777	LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1778	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,AccessedPage));
1779	SHW_PTE_SET(PteDst, 0);
1780	}
1781	/*
1782	* If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1783	* when the page is modified.
1784	*/
1785	else if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
1786	{
1787	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPage));
1788	SHW_PTE_SET(PteDst,
1789	fGstShwPteFlags
1790	\| PGM_PAGE_GET_HCPHYS(pPage)
1791	\| PGM_PTFLAGS_TRACK_DIRTY);
1792	SHW_PTE_SET_RO(PteDst);
1793	}
1794	else
1795	#endif
1796	{
1797	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageSkipped));
1798	#if PGM_SHW_TYPE == PGM_TYPE_EPT
1799	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage);
1800	PteDst.n.u1Present = 1;
1801	PteDst.n.u1Write = 1;
1802	PteDst.n.u1Execute = 1;
1803	PteDst.n.u1IgnorePAT = 1;
1804	PteDst.n.u3EMT = VMX_EPT_MEMTYPE_WB;
1805	/* PteDst.n.u1Size = 0 */
1806	#else
1807	SHW_PTE_SET(PteDst, fGstShwPteFlags \| PGM_PAGE_GET_HCPHYS(pPage));
1808	#endif
1809	}
1810
1811	/*
1812	* Make sure only allocated pages are mapped writable.
1813	*/
1814	if ( SHW_PTE_IS_P_RW(PteDst)
1815	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1816	{
1817	/* Still applies to shared pages. */
1818	Assert(!PGM_PAGE_IS_ZERO(pPage));
1819	SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet. Why, isn't it? */
1820	Log3(("SyncPageWorker: write-protecting %RGp pPage=%R[pgmpage]at iPTDst=%d\n", GCPhysPage, pPage, iPTDst));
1821	}
1822	}
1823
1824	/*
1825	* Keep user track up to date.
1826	*/
1827	if (SHW_PTE_IS_P(PteDst))
1828	{
1829	if (!SHW_PTE_IS_P(*pPteDst))
1830	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1831	else if (SHW_PTE_GET_HCPHYS(*pPteDst) != SHW_PTE_GET_HCPHYS(PteDst))
1832	{
1833	Log2(("SyncPageWorker: deref! pPteDst=%RX64 PteDst=%RX64\n", SHW_PTE_LOG64(pPteDst), SHW_PTE_LOG64(PteDst)));
1834	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
1835	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1836	}
1837	}
1838	else if (SHW_PTE_IS_P(*pPteDst))
1839	{
1840	Log2(("SyncPageWorker: deref! pPteDst=%RX64\n", SHW_PTE_LOG64(pPteDst)));
1841	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
1842	}
1843
1844	/*
1845	* Update statistics and commit the entry.
1846	*/
1847	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1848	if (!PteSrc.n.u1Global)
1849	pShwPage->fSeenNonGlobal = true;
1850	#endif
1851	SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
1852	return;
1853	}
1854
1855	/** @todo count these three different kinds. */
1856	Log2(("SyncPageWorker: invalid address in Pte\n"));
1857	}
1858	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1859	else if (!PteSrc.n.u1Present)
1860	Log2(("SyncPageWorker: page not present in Pte\n"));
1861	else
1862	Log2(("SyncPageWorker: invalid Pte\n"));
1863	#endif
1864
1865	/*
1866	* The page is not present or the PTE is bad. Replace the shadow PTE by
1867	* an empty entry, making sure to keep the user tracking up to date.
1868	*/
1869	if (SHW_PTE_IS_P(*pPteDst))
1870	{
1871	Log2(("SyncPageWorker: deref! pPteDst=%RX64\n", SHW_PTE_LOG64(pPteDst)));
1872	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
1873	}
1874	SHW_PTE_ATOMIC_SET(*pPteDst, 0);
1875	}
1876
1877
1878	/**
1879	* Syncs a guest OS page.
1880	*
1881	* There are no conflicts at this point, neither is there any need for
1882	* page table allocations.
1883	*
1884	* When called in PAE or AMD64 guest mode, the guest PDPE shall be valid.
1885	* When called in AMD64 guest mode, the guest PML4E shall be valid.
1886	*
1887	* @returns VBox status code.
1888	* @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1889	* @param pVCpu The cross context virtual CPU structure.
1890	* @param PdeSrc Page directory entry of the guest.
1891	* @param GCPtrPage Guest context page address.
1892	* @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1893	* @param uErr Fault error (X86_TRAP_PF_*).
1894	*/
1895	static int PGM_BTH_NAME(SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr)
1896	{
1897	PVM pVM = pVCpu->CTX_SUFF(pVM);
1898	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
1899	LogFlow(("SyncPage: GCPtrPage=%RGv cPages=%u uErr=%#x\n", GCPtrPage, cPages, uErr));
1900
1901	PGM_LOCK_ASSERT_OWNER(pVM);
1902
1903	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
1904	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
1905	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
1906	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
1907	&& PGM_SHW_TYPE != PGM_TYPE_EPT
1908
1909	/*
1910	* Assert preconditions.
1911	*/
1912	Assert(PdeSrc.n.u1Present);
1913	Assert(cPages);
1914	# if 0 /* rarely useful; leave for debugging. */
1915	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
1916	# endif
1917
1918	/*
1919	* Get the shadow PDE, find the shadow page table in the pool.
1920	*/
1921	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1922	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1923	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1924
1925	/* Fetch the pgm pool shadow descriptor. */
1926	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1927	Assert(pShwPde);
1928
1929	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1930	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1931	PPGMPOOLPAGE pShwPde = NULL;
1932	PX86PDPAE pPDDst;
1933
1934	/* Fetch the pgm pool shadow descriptor. */
1935	int rc2 = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
1936	AssertRCSuccessReturn(rc2, rc2);
1937	Assert(pShwPde);
1938
1939	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1940	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1941
1942	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1943	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1944	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1945	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
1946	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
1947
1948	int rc2 = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
1949	AssertRCSuccessReturn(rc2, rc2);
1950	Assert(pPDDst && pPdptDst);
1951	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1952	# endif
1953	SHWPDE PdeDst = *pPdeDst;
1954
1955	/*
1956	* - In the guest SMP case we could have blocked while another VCPU reused
1957	* this page table.
1958	* - With W7-64 we may also take this path when the A bit is cleared on
1959	* higher level tables (PDPE/PML4E). The guest does not invalidate the
1960	* relevant TLB entries. If we're write monitoring any page mapped by
1961	* the modified entry, we may end up here with a "stale" TLB entry.
1962	*/
1963	if (!PdeDst.n.u1Present)
1964	{
1965	Log(("CPU%u: SyncPage: Pde at %RGv changed behind our back? (pPdeDst=%p/%RX64) uErr=%#x\n", pVCpu->idCpu, GCPtrPage, pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
1966	AssertMsg(pVM->cCpus > 1 \|\| (uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RW)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RW),
1967	("Unexpected missing PDE p=%p/%RX64 uErr=%#x\n", pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
1968	if (uErr & X86_TRAP_PF_P)
1969	PGM_INVL_PG(pVCpu, GCPtrPage);
1970	return VINF_SUCCESS; /* force the instruction to be executed again. */
1971	}
1972
1973	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1974	Assert(pShwPage);
1975
1976	# if PGM_GST_TYPE == PGM_TYPE_AMD64
1977	/* Fetch the pgm pool shadow descriptor. */
1978	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
1979	Assert(pShwPde);
1980	# endif
1981
1982	/*
1983	* Check that the page is present and that the shadow PDE isn't out of sync.
1984	*/
1985	const bool fBigPage = PdeSrc.b.u1Size && GST_IS_PSE_ACTIVE(pVCpu);
1986	const bool fPdeValid = !fBigPage ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc);
1987	RTGCPHYS GCPhys;
1988	if (!fBigPage)
1989	{
1990	GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
1991	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1992	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1993	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| ((iPDDst & 1) * (PAGE_SIZE / 2)));
1994	# endif
1995	}
1996	else
1997	{
1998	GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
1999	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2000	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2001	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
2002	# endif
2003	}
2004	/** @todo This doesn't check the G bit of 2/4MB pages. FIXME */
2005	if ( fPdeValid
2006	&& pShwPage->GCPhys == GCPhys
2007	&& PdeSrc.n.u1Present
2008	&& PdeSrc.n.u1User == PdeDst.n.u1User
2009	&& (PdeSrc.n.u1Write == PdeDst.n.u1Write \|\| !PdeDst.n.u1Write)
2010	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2011	&& (PdeSrc.n.u1NoExecute == PdeDst.n.u1NoExecute \|\| !GST_IS_NX_ACTIVE(pVCpu))
2012	# endif
2013	)
2014	{
2015	/*
2016	* Check that the PDE is marked accessed already.
2017	* Since we set the accessed bit before getting here on a #PF, this
2018	* check is only meant for dealing with non-#PF'ing paths.
2019	*/
2020	if (PdeSrc.n.u1Accessed)
2021	{
2022	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2023	if (!fBigPage)
2024	{
2025	/*
2026	* 4KB Page - Map the guest page table.
2027	*/
2028	PGSTPT pPTSrc;
2029	int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
2030	if (RT_SUCCESS(rc))
2031	{
2032	# ifdef PGM_SYNC_N_PAGES
2033	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
2034	if ( cPages > 1
2035	&& !(uErr & X86_TRAP_PF_P)
2036	&& !VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY))
2037	{
2038	/*
2039	* This code path is currently only taken when the caller is PGMTrap0eHandler
2040	* for non-present pages!
2041	*
2042	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2043	* deal with locality.
2044	*/
2045	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2046	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2047	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2048	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2049	# else
2050	const unsigned offPTSrc = 0;
2051	# endif
2052	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2053	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2054	iPTDst = 0;
2055	else
2056	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2057
2058	for (; iPTDst < iPTDstEnd; iPTDst++)
2059	{
2060	const PGSTPTE pPteSrc = &pPTSrc->a[offPTSrc + iPTDst];
2061
2062	if ( pPteSrc->n.u1Present
2063	&& !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2064	{
2065	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(GST_PT_MASK << GST_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
2066	NOREF(GCPtrCurPage);
2067	# ifdef VBOX_WITH_RAW_MODE_NOT_R0
2068	/*
2069	* Assuming kernel code will be marked as supervisor - and not as user level
2070	* and executed using a conforming code selector - And marked as readonly.
2071	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
2072	*/
2073	PPGMPAGE pPage;
2074	if ( ((PdeSrc.u & pPteSrc->u) & (X86_PTE_RW \| X86_PTE_US))
2075	\|\| iPTDst == ((GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK) /* always sync GCPtrPage */
2076	\|\| !CSAMDoesPageNeedScanning(pVM, GCPtrCurPage)
2077	\|\| ( (pPage = pgmPhysGetPage(pVM, pPteSrc->u & GST_PTE_PG_MASK))
2078	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2079	)
2080	# endif /* else: CSAM not active */
2081	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, *pPteSrc, pShwPage, iPTDst);
2082	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
2083	GCPtrCurPage, pPteSrc->n.u1Present,
2084	pPteSrc->n.u1Write & PdeSrc.n.u1Write,
2085	pPteSrc->n.u1User & PdeSrc.n.u1User,
2086	(uint64_t)pPteSrc->u,
2087	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2088	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2089	}
2090	}
2091	}
2092	else
2093	# endif /* PGM_SYNC_N_PAGES */
2094	{
2095	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
2096	GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2097	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2098	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2099	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
2100	GCPtrPage, PteSrc.n.u1Present,
2101	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2102	PteSrc.n.u1User & PdeSrc.n.u1User,
2103	(uint64_t)PteSrc.u,
2104	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2105	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2106	}
2107	}
2108	else /* MMIO or invalid page: emulated in #PF handler. */
2109	{
2110	LogFlow(("PGM_GCPHYS_2_PTR %RGp failed with %Rrc\n", GCPhys, rc));
2111	Assert(!SHW_PTE_IS_P(pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK]));
2112	}
2113	}
2114	else
2115	{
2116	/*
2117	* 4/2MB page - lazy syncing shadow 4K pages.
2118	* (There are many causes of getting here, it's no longer only CSAM.)
2119	*/
2120	/* Calculate the GC physical address of this 4KB shadow page. */
2121	GCPhys = PGM_A20_APPLY(pVCpu, GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc) \| (GCPtrPage & GST_BIG_PAGE_OFFSET_MASK));
2122	/* Find ram range. */
2123	PPGMPAGE pPage;
2124	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
2125	if (RT_SUCCESS(rc))
2126	{
2127	AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
2128
2129	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2130	/* Try to make the page writable if necessary. */
2131	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2132	&& ( PGM_PAGE_IS_ZERO(pPage)
2133	\|\| ( PdeSrc.n.u1Write
2134	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2135	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
2136	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
2137	# endif
2138	# ifdef VBOX_WITH_PAGE_SHARING
2139	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
2140	# endif
2141	)
2142	)
2143	)
2144	{
2145	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
2146	AssertRC(rc);
2147	}
2148	# endif
2149
2150	/*
2151	* Make shadow PTE entry.
2152	*/
2153	SHWPTE PteDst;
2154	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2155	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc), &PteDst);
2156	else
2157	SHW_PTE_SET(PteDst, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc) \| PGM_PAGE_GET_HCPHYS(pPage));
2158
2159	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2160	if ( SHW_PTE_IS_P(PteDst)
2161	&& !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2162	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2163
2164	/* Make sure only allocated pages are mapped writable. */
2165	if ( SHW_PTE_IS_P_RW(PteDst)
2166	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2167	{
2168	/* Still applies to shared pages. */
2169	Assert(!PGM_PAGE_IS_ZERO(pPage));
2170	SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */
2171	Log3(("SyncPage: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, GCPtrPage));
2172	}
2173
2174	SHW_PTE_ATOMIC_SET2(pPTDst->a[iPTDst], PteDst);
2175
2176	/*
2177	* If the page is not flagged as dirty and is writable, then make it read-only
2178	* at PD level, so we can set the dirty bit when the page is modified.
2179	*
2180	* ASSUMES that page access handlers are implemented on page table entry level.
2181	* Thus we will first catch the dirty access and set PDE.D and restart. If
2182	* there is an access handler, we'll trap again and let it work on the problem.
2183	*/
2184	/** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
2185	* As for invlpg, it simply frees the whole shadow PT.
2186	* ...It's possibly because the guest clears it and the guest doesn't really tell us... */
2187	if ( !PdeSrc.b.u1Dirty
2188	&& PdeSrc.b.u1Write)
2189	{
2190	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2191	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2192	PdeDst.n.u1Write = 0;
2193	}
2194	else
2195	{
2196	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2197	PdeDst.n.u1Write = PdeSrc.n.u1Write;
2198	}
2199	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2200	Log2(("SyncPage: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%RGp%s\n",
2201	GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
2202	PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2203	}
2204	else
2205	{
2206	LogFlow(("PGM_GCPHYS_2_PTR %RGp (big) failed with %Rrc\n", GCPhys, rc));
2207	/** @todo must wipe the shadow page table entry in this
2208	* case. */
2209	}
2210	}
2211	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2212	return VINF_SUCCESS;
2213	}
2214
2215	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPagePDNAs));
2216	}
2217	else if (fPdeValid)
2218	{
2219	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPagePDOutOfSync));
2220	Log2(("SyncPage: Out-Of-Sync PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2221	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2222	}
2223	else
2224	{
2225	/// @todo STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDOutOfSyncAndInvalid));
2226	Log2(("SyncPage: Bad PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2227	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2228	}
2229
2230	/*
2231	* Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
2232	* Yea, I'm lazy.
2233	*/
2234	pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst);
2235	ASMAtomicWriteSize(pPdeDst, 0);
2236
2237	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2238	PGM_INVL_VCPU_TLBS(pVCpu);
2239	return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
2240
2241
2242	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2243	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2244	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2245	&& !defined(IN_RC)
2246	NOREF(PdeSrc);
2247
2248	# ifdef PGM_SYNC_N_PAGES
2249	/*
2250	* Get the shadow PDE, find the shadow page table in the pool.
2251	*/
2252	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2253	X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
2254
2255	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2256	X86PDEPAE PdeDst = pgmShwGetPaePDE(pVCpu, GCPtrPage);
2257
2258	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2259	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2260	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; NOREF(iPdpt);
2261	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2262	X86PDEPAE PdeDst;
2263	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2264
2265	int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2266	AssertRCSuccessReturn(rc, rc);
2267	Assert(pPDDst && pPdptDst);
2268	PdeDst = pPDDst->a[iPDDst];
2269	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2270	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2271	PEPTPD pPDDst;
2272	EPTPDE PdeDst;
2273
2274	int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, NULL, &pPDDst);
2275	if (rc != VINF_SUCCESS)
2276	{
2277	AssertRC(rc);
2278	return rc;
2279	}
2280	Assert(pPDDst);
2281	PdeDst = pPDDst->a[iPDDst];
2282	# endif
2283	/* In the guest SMP case we could have blocked while another VCPU reused this page table. */
2284	if (!PdeDst.n.u1Present)
2285	{
2286	AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE %RX64\n", (uint64_t)PdeDst.u));
2287	Log(("CPU%d: SyncPage: Pde at %RGv changed behind our back!\n", pVCpu->idCpu, GCPtrPage));
2288	return VINF_SUCCESS; /* force the instruction to be executed again. */
2289	}
2290
2291	/* Can happen in the guest SMP case; other VCPU activated this PDE while we were blocking to handle the page fault. */
2292	if (PdeDst.n.u1Size)
2293	{
2294	Assert(pVM->pgm.s.fNestedPaging);
2295	Log(("CPU%d: SyncPage: Pde (big:%RX64) at %RGv changed behind our back!\n", pVCpu->idCpu, PdeDst.u, GCPtrPage));
2296	return VINF_SUCCESS;
2297	}
2298
2299	/* Mask away the page offset. */
2300	GCPtrPage &= ~((RTGCPTR)0xfff);
2301
2302	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2303	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2304
2305	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
2306	if ( cPages > 1
2307	&& !(uErr & X86_TRAP_PF_P)
2308	&& !VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY))
2309	{
2310	/*
2311	* This code path is currently only taken when the caller is PGMTrap0eHandler
2312	* for non-present pages!
2313	*
2314	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2315	* deal with locality.
2316	*/
2317	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2318	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2319	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2320	iPTDst = 0;
2321	else
2322	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2323	for (; iPTDst < iPTDstEnd; iPTDst++)
2324	{
2325	if (!SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2326	{
2327	RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
2328	\| (iPTDst << PAGE_SHIFT));
2329
2330	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
2331	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n",
2332	GCPtrCurPage,
2333	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2334	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2335
2336	if (RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)))
2337	break;
2338	}
2339	else
2340	Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n", (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT), iPTDst, SHW_PTE_LOG64(pPTDst->a[iPTDst]) ));
2341	}
2342	}
2343	else
2344	# endif /* PGM_SYNC_N_PAGES */
2345	{
2346	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2347	RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
2348	\| (iPTDst << PAGE_SHIFT));
2349
2350	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
2351
2352	Log2(("SyncPage: 4K %RGv PteSrc:{P=1 RW=1 U=1}PteDst=%08llx%s\n",
2353	GCPtrPage,
2354	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2355	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2356	}
2357	return VINF_SUCCESS;
2358
2359	#else
2360	NOREF(PdeSrc);
2361	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2362	return VERR_PGM_NOT_USED_IN_MODE;
2363	#endif
2364	}
2365
2366
2367	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
2368
2369	/**
2370	* CheckPageFault helper for returning a page fault indicating a non-present
2371	* (NP) entry in the page translation structures.
2372	*
2373	* @returns VINF_EM_RAW_GUEST_TRAP.
2374	* @param pVCpu The cross context virtual CPU structure.
2375	* @param uErr The error code of the shadow fault. Corrections to
2376	* TRPM's copy will be made if necessary.
2377	* @param GCPtrPage For logging.
2378	* @param uPageFaultLevel For logging.
2379	*/
2380	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnNP)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2381	{
2382	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2383	AssertMsg(!(uErr & X86_TRAP_PF_P), ("%#x\n", uErr));
2384	AssertMsg(!(uErr & X86_TRAP_PF_RSVD), ("%#x\n", uErr));
2385	if (uErr & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
2386	TRPMSetErrorCode(pVCpu, uErr & ~(X86_TRAP_PF_RSVD \| X86_TRAP_PF_P));
2387
2388	Log(("CheckPageFault: real page fault (notp) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2389	return VINF_EM_RAW_GUEST_TRAP;
2390	}
2391
2392
2393	/**
2394	* CheckPageFault helper for returning a page fault indicating a reserved bit
2395	* (RSVD) error in the page translation structures.
2396	*
2397	* @returns VINF_EM_RAW_GUEST_TRAP.
2398	* @param pVCpu The cross context virtual CPU structure.
2399	* @param uErr The error code of the shadow fault. Corrections to
2400	* TRPM's copy will be made if necessary.
2401	* @param GCPtrPage For logging.
2402	* @param uPageFaultLevel For logging.
2403	*/
2404	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnRSVD)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2405	{
2406	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2407	if ((uErr & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) != (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
2408	TRPMSetErrorCode(pVCpu, uErr \| X86_TRAP_PF_RSVD \| X86_TRAP_PF_P);
2409
2410	Log(("CheckPageFault: real page fault (rsvd) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2411	return VINF_EM_RAW_GUEST_TRAP;
2412	}
2413
2414
2415	/**
2416	* CheckPageFault helper for returning a page protection fault (P).
2417	*
2418	* @returns VINF_EM_RAW_GUEST_TRAP.
2419	* @param pVCpu The cross context virtual CPU structure.
2420	* @param uErr The error code of the shadow fault. Corrections to
2421	* TRPM's copy will be made if necessary.
2422	* @param GCPtrPage For logging.
2423	* @param uPageFaultLevel For logging.
2424	*/
2425	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnProt)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2426	{
2427	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2428	AssertMsg(uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID), ("%#x\n", uErr));
2429	if ((uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) != X86_TRAP_PF_P)
2430	TRPMSetErrorCode(pVCpu, (uErr & ~X86_TRAP_PF_RSVD) \| X86_TRAP_PF_P);
2431
2432	Log(("CheckPageFault: real page fault (prot) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2433	return VINF_EM_RAW_GUEST_TRAP;
2434	}
2435
2436
2437	/**
2438	* Handle dirty bit tracking faults.
2439	*
2440	* @returns VBox status code.
2441	* @param pVCpu The cross context virtual CPU structure.
2442	* @param uErr Page fault error code.
2443	* @param pPdeSrc Guest page directory entry.
2444	* @param pPdeDst Shadow page directory entry.
2445	* @param GCPtrPage Guest context page address.
2446	*/
2447	static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc,
2448	RTGCPTR GCPtrPage)
2449	{
2450	PVM pVM = pVCpu->CTX_SUFF(pVM);
2451	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2452	NOREF(uErr);
2453
2454	PGM_LOCK_ASSERT_OWNER(pVM);
2455
2456	/*
2457	* Handle big page.
2458	*/
2459	if (pPdeSrc->b.u1Size && GST_IS_PSE_ACTIVE(pVCpu))
2460	{
2461	if ( pPdeDst->n.u1Present
2462	&& (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY))
2463	{
2464	SHWPDE PdeDst = *pPdeDst;
2465
2466	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageTrap));
2467	Assert(pPdeSrc->b.u1Write);
2468
2469	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
2470	* fault again and take this path to only invalidate the entry (see below).
2471	*/
2472	PdeDst.n.u1Write = 1;
2473	PdeDst.n.u1Accessed = 1;
2474	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2475	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2476	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
2477	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2478	}
2479
2480	# ifdef IN_RING0
2481	/* Check for stale TLB entry; only applies to the SMP guest case. */
2482	if ( pVM->cCpus > 1
2483	&& pPdeDst->n.u1Write
2484	&& pPdeDst->n.u1Accessed)
2485	{
2486	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2487	if (pShwPage)
2488	{
2489	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2490	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2491	if (SHW_PTE_IS_P_RW(*pPteDst))
2492	{
2493	/* Stale TLB entry. */
2494	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageStale));
2495	PGM_INVL_PG(pVCpu, GCPtrPage);
2496	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2497	}
2498	}
2499	}
2500	# endif /* IN_RING0 */
2501	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2502	}
2503
2504	/*
2505	* Map the guest page table.
2506	*/
2507	PGSTPT pPTSrc;
2508	int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(*pPdeSrc), &pPTSrc);
2509	if (RT_FAILURE(rc))
2510	{
2511	AssertRC(rc);
2512	return rc;
2513	}
2514
2515	if (pPdeDst->n.u1Present)
2516	{
2517	GSTPTE const *pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2518	const GSTPTE PteSrc = *pPteSrc;
2519
2520	#ifdef VBOX_WITH_RAW_MODE_NOT_R0
2521	/* Bail out here as pgmPoolGetPage will return NULL and we'll crash below.
2522	* Our individual shadow handlers will provide more information and force a fatal exit.
2523	*/
2524	if ( !HMIsEnabled(pVM)
2525	&& MMHyperIsInsideArea(pVM, (RTGCPTR)GCPtrPage))
2526	{
2527	LogRel(("CheckPageFault: write to hypervisor region %RGv\n", GCPtrPage));
2528	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2529	}
2530	#endif
2531	/*
2532	* Map shadow page table.
2533	*/
2534	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2535	if (pShwPage)
2536	{
2537	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2538	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2539	if (SHW_PTE_IS_P(pPteDst)) /* @todo Optimize accessed bit emulation? */
2540	{
2541	if (SHW_PTE_IS_TRACK_DIRTY(*pPteDst))
2542	{
2543	PPGMPAGE pPage = pgmPhysGetPage(pVM, GST_GET_PTE_GCPHYS(PteSrc));
2544	SHWPTE PteDst = *pPteDst;
2545
2546	LogFlow(("DIRTY page trap addr=%RGv\n", GCPtrPage));
2547	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageTrap));
2548
2549	Assert(PteSrc.n.u1Write);
2550
2551	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB
2552	* entry will not harm; write access will simply fault again and
2553	* take this path to only invalidate the entry.
2554	*/
2555	if (RT_LIKELY(pPage))
2556	{
2557	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2558	{
2559	//AssertMsgFailed(("%R[pgmpage] - we don't set PGM_PTFLAGS_TRACK_DIRTY for these pages\n", pPage));
2560	Assert(!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage));
2561	/* Assuming write handlers here as the PTE is present (otherwise we wouldn't be here). */
2562	SHW_PTE_SET_RO(PteDst);
2563	}
2564	else
2565	{
2566	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
2567	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
2568	{
2569	rc = pgmPhysPageMakeWritable(pVM, pPage, GST_GET_PTE_GCPHYS(PteSrc));
2570	AssertRC(rc);
2571	}
2572	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED)
2573	SHW_PTE_SET_RW(PteDst);
2574	else
2575	{
2576	/* Still applies to shared pages. */
2577	Assert(!PGM_PAGE_IS_ZERO(pPage));
2578	SHW_PTE_SET_RO(PteDst);
2579	}
2580	}
2581	}
2582	else
2583	SHW_PTE_SET_RW(PteDst); /** @todo r=bird: This doesn't make sense to me. */
2584
2585	SHW_PTE_SET(PteDst, (SHW_PTE_GET_U(PteDst) \| X86_PTE_D \| X86_PTE_A) & ~(uint64_t)PGM_PTFLAGS_TRACK_DIRTY);
2586	SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
2587	PGM_INVL_PG(pVCpu, GCPtrPage);
2588	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2589	}
2590
2591	# ifdef IN_RING0
2592	/* Check for stale TLB entry; only applies to the SMP guest case. */
2593	if ( pVM->cCpus > 1
2594	&& SHW_PTE_IS_RW(*pPteDst)
2595	&& SHW_PTE_IS_A(*pPteDst))
2596	{
2597	/* Stale TLB entry. */
2598	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageStale));
2599	PGM_INVL_PG(pVCpu, GCPtrPage);
2600	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2601	}
2602	# endif
2603	}
2604	}
2605	else
2606	AssertMsgFailed(("pgmPoolGetPageByHCPhys %RGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
2607	}
2608
2609	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2610	}
2611
2612	#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
2613
2614
2615	/**
2616	* Sync a shadow page table.
2617	*
2618	* The shadow page table is not present in the shadow PDE.
2619	*
2620	* Handles mapping conflicts.
2621	*
2622	* This is called by VerifyAccessSyncPage, PrefetchPage, InvalidatePage (on
2623	* conflict), and Trap0eHandler.
2624	*
2625	* A precondition for this method is that the shadow PDE is not present. The
2626	* caller must take the PGM lock before checking this and continue to hold it
2627	* when calling this method.
2628	*
2629	* @returns VBox status code.
2630	* @param pVCpu The cross context virtual CPU structure.
2631	* @param iPDSrc Page directory index.
2632	* @param pPDSrc Source page directory (i.e. Guest OS page directory).
2633	* Assume this is a temporary mapping.
2634	* @param GCPtrPage GC Pointer of the page that caused the fault
2635	*/
2636	static int PGM_BTH_NAME(SyncPT)(PVMCPU pVCpu, unsigned iPDSrc, PGSTPD pPDSrc, RTGCPTR GCPtrPage)
2637	{
2638	PVM pVM = pVCpu->CTX_SUFF(pVM);
2639	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
2640
2641	#if 0 /* rarely useful; leave for debugging. */
2642	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPtPD[iPDSrc]);
2643	#endif
2644	LogFlow(("SyncPT: GCPtrPage=%RGv\n", GCPtrPage));
2645
2646	PGM_LOCK_ASSERT_OWNER(pVM);
2647
2648	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2649	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
2650	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
2651	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
2652	&& PGM_SHW_TYPE != PGM_TYPE_EPT
2653
2654	int rc = VINF_SUCCESS;
2655
2656	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2657
2658	/*
2659	* Some input validation first.
2660	*/
2661	AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage));
2662
2663	/*
2664	* Get the relevant shadow PDE entry.
2665	*/
2666	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2667	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2668	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
2669
2670	/* Fetch the pgm pool shadow descriptor. */
2671	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2672	Assert(pShwPde);
2673
2674	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2675	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2676	PPGMPOOLPAGE pShwPde = NULL;
2677	PX86PDPAE pPDDst;
2678	PSHWPDE pPdeDst;
2679
2680	/* Fetch the pgm pool shadow descriptor. */
2681	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
2682	AssertRCSuccessReturn(rc, rc);
2683	Assert(pShwPde);
2684
2685	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
2686	pPdeDst = &pPDDst->a[iPDDst];
2687
2688	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2689	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2690	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2691	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2692	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2693	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2694	AssertRCSuccessReturn(rc, rc);
2695	Assert(pPDDst);
2696	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2697	# endif
2698	SHWPDE PdeDst = *pPdeDst;
2699
2700	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2701	/* Fetch the pgm pool shadow descriptor. */
2702	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2703	Assert(pShwPde);
2704	# endif
2705
2706	# ifndef PGM_WITHOUT_MAPPINGS
2707	/*
2708	* Check for conflicts.
2709	* RC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
2710	* R3: Simply resolve the conflict.
2711	*/
2712	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
2713	{
2714	Assert(pgmMapAreMappingsEnabled(pVM));
2715	# ifndef IN_RING3
2716	Log(("SyncPT: Conflict at %RGv\n", GCPtrPage));
2717	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2718	return VERR_ADDRESS_CONFLICT;
2719
2720	# else /* IN_RING3 */
2721	PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
2722	Assert(pMapping);
2723	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2724	rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2725	# elif PGM_GST_TYPE == PGM_TYPE_PAE
2726	rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2727	# else
2728	AssertFailed(); NOREF(pMapping); /* can't happen for amd64 */
2729	# endif
2730	if (RT_FAILURE(rc))
2731	{
2732	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2733	return rc;
2734	}
2735	PdeDst = *pPdeDst;
2736	# endif /* IN_RING3 */
2737	}
2738	# endif /* !PGM_WITHOUT_MAPPINGS */
2739	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2740
2741	/*
2742	* Sync the page directory entry.
2743	*/
2744	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2745	const bool fPageTable = !PdeSrc.b.u1Size \|\| !GST_IS_PSE_ACTIVE(pVCpu);
2746	if ( PdeSrc.n.u1Present
2747	&& (fPageTable ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc)) )
2748	{
2749	/*
2750	* Allocate & map the page table.
2751	*/
2752	PSHWPT pPTDst;
2753	PPGMPOOLPAGE pShwPage;
2754	RTGCPHYS GCPhys;
2755	if (fPageTable)
2756	{
2757	GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
2758	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2759	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2760	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| ((iPDDst & 1) * (PAGE_SIZE / 2)));
2761	# endif
2762	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
2763	pShwPde->idx, iPDDst, false /fLockPage/,
2764	&pShwPage);
2765	}
2766	else
2767	{
2768	PGMPOOLACCESS enmAccess;
2769	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2770	const bool fNoExecute = PdeSrc.n.u1NoExecute && GST_IS_NX_ACTIVE(pVCpu);
2771	# else
2772	const bool fNoExecute = false;
2773	# endif
2774
2775	GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
2776	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2777	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2778	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
2779	# endif
2780	/* Determine the right kind of large page to avoid incorrect cached entry reuse. */
2781	if (PdeSrc.n.u1User)
2782	{
2783	if (PdeSrc.n.u1Write)
2784	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_RW_NX : PGMPOOLACCESS_USER_RW;
2785	else
2786	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_R_NX : PGMPOOLACCESS_USER_R;
2787	}
2788	else
2789	{
2790	if (PdeSrc.n.u1Write)
2791	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW;
2792	else
2793	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_R_NX : PGMPOOLACCESS_SUPERVISOR_R;
2794	}
2795	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, enmAccess, PGM_A20_IS_ENABLED(pVCpu),
2796	pShwPde->idx, iPDDst, false /fLockPage/,
2797	&pShwPage);
2798	}
2799	if (rc == VINF_SUCCESS)
2800	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2801	else if (rc == VINF_PGM_CACHED_PAGE)
2802	{
2803	/*
2804	* The PT was cached, just hook it up.
2805	*/
2806	if (fPageTable)
2807	PdeDst.u = pShwPage->Core.Key \| GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2808	else
2809	{
2810	PdeDst.u = pShwPage->Core.Key \| GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2811	/* (see explanation and assumptions further down.) */
2812	if ( !PdeSrc.b.u1Dirty
2813	&& PdeSrc.b.u1Write)
2814	{
2815	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2816	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2817	PdeDst.b.u1Write = 0;
2818	}
2819	}
2820	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2821	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2822	return VINF_SUCCESS;
2823	}
2824	else if (rc == VERR_PGM_POOL_FLUSHED)
2825	{
2826	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2827	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2828	return VINF_PGM_SYNC_CR3;
2829	}
2830	else
2831	AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
2832	/** @todo Why do we bother preserving X86_PDE_AVL_MASK here?
2833	* Both PGM_PDFLAGS_MAPPING and PGM_PDFLAGS_TRACK_DIRTY should be
2834	* irrelevant at this point. */
2835	PdeDst.u &= X86_PDE_AVL_MASK;
2836	PdeDst.u \|= pShwPage->Core.Key;
2837
2838	/*
2839	* Page directory has been accessed (this is a fault situation, remember).
2840	*/
2841	/** @todo
2842	* Well, when the caller is PrefetchPage or InvalidatePage is isn't a
2843	* fault situation. What's more, the Trap0eHandler has already set the
2844	* accessed bit. So, it's actually just VerifyAccessSyncPage which
2845	* might need setting the accessed flag.
2846	*
2847	* The best idea is to leave this change to the caller and add an
2848	* assertion that it's set already. */
2849	pPDSrc->a[iPDSrc].n.u1Accessed = 1;
2850	if (fPageTable)
2851	{
2852	/*
2853	* Page table - 4KB.
2854	*
2855	* Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
2856	*/
2857	Log2(("SyncPT: 4K %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
2858	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
2859	PGSTPT pPTSrc;
2860	rc = PGM_GCPHYS_2_PTR(pVM, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
2861	if (RT_SUCCESS(rc))
2862	{
2863	/*
2864	* Start by syncing the page directory entry so CSAM's TLB trick works.
2865	*/
2866	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| X86_PDE_AVL_MASK))
2867	\| GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2868	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2869	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2870
2871	/*
2872	* Directory/page user or supervisor privilege: (same goes for read/write)
2873	*
2874	* Directory Page Combined
2875	* U/S U/S U/S
2876	* 0 0 0
2877	* 0 1 0
2878	* 1 0 0
2879	* 1 1 1
2880	*
2881	* Simple AND operation. Table listed for completeness.
2882	*
2883	*/
2884	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT4K));
2885	# ifdef PGM_SYNC_N_PAGES
2886	unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2887	unsigned iPTDst = iPTBase;
2888	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2889	if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
2890	iPTDst = 0;
2891	else
2892	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2893	# else /* !PGM_SYNC_N_PAGES */
2894	unsigned iPTDst = 0;
2895	const unsigned iPTDstEnd = RT_ELEMENTS(pPTDst->a);
2896	# endif /* !PGM_SYNC_N_PAGES */
2897	RTGCPTR GCPtrCur = (GCPtrPage & ~(RTGCPTR)((1 << SHW_PD_SHIFT) - 1))
2898	\| ((RTGCPTR)iPTDst << PAGE_SHIFT);
2899	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2900	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2901	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2902	# else
2903	const unsigned offPTSrc = 0;
2904	# endif
2905	for (; iPTDst < iPTDstEnd; iPTDst++, GCPtrCur += PAGE_SIZE)
2906	{
2907	const unsigned iPTSrc = iPTDst + offPTSrc;
2908	const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2909
2910	if (PteSrc.n.u1Present)
2911	{
2912	# ifdef VBOX_WITH_RAW_MODE_NOT_R0
2913	/*
2914	* Assuming kernel code will be marked as supervisor - and not as user level
2915	* and executed using a conforming code selector - And marked as readonly.
2916	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
2917	*/
2918	PPGMPAGE pPage;
2919	if ( ((PdeSrc.u & pPTSrc->a[iPTSrc].u) & (X86_PTE_RW \| X86_PTE_US))
2920	\|\| !CSAMDoesPageNeedScanning(pVM, GCPtrCur)
2921	\|\| ( (pPage = pgmPhysGetPage(pVM, GST_GET_PTE_GCPHYS(PteSrc)))
2922	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2923	)
2924	# endif
2925	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2926	Log2(("SyncPT: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n",
2927	GCPtrCur,
2928	PteSrc.n.u1Present,
2929	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2930	PteSrc.n.u1User & PdeSrc.n.u1User,
2931	(uint64_t)PteSrc.u,
2932	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : "", SHW_PTE_LOG64(pPTDst->a[iPTDst]), iPTSrc, PdeSrc.au32[0],
2933	(RTGCPHYS)(GST_GET_PDE_GCPHYS(PdeSrc) + iPTSrc*sizeof(PteSrc)) ));
2934	}
2935	/* else: the page table was cleared by the pool */
2936	} /* for PTEs */
2937	}
2938	}
2939	else
2940	{
2941	/*
2942	* Big page - 2/4MB.
2943	*
2944	* We'll walk the ram range list in parallel and optimize lookups.
2945	* We will only sync one shadow page table at a time.
2946	*/
2947	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT4M));
2948
2949	/**
2950	* @todo It might be more efficient to sync only a part of the 4MB
2951	* page (similar to what we do for 4KB PDs).
2952	*/
2953
2954	/*
2955	* Start by syncing the page directory entry.
2956	*/
2957	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
2958	\| GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2959
2960	/*
2961	* If the page is not flagged as dirty and is writable, then make it read-only
2962	* at PD level, so we can set the dirty bit when the page is modified.
2963	*
2964	* ASSUMES that page access handlers are implemented on page table entry level.
2965	* Thus we will first catch the dirty access and set PDE.D and restart. If
2966	* there is an access handler, we'll trap again and let it work on the problem.
2967	*/
2968	/** @todo move the above stuff to a section in the PGM documentation. */
2969	Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
2970	if ( !PdeSrc.b.u1Dirty
2971	&& PdeSrc.b.u1Write)
2972	{
2973	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2974	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2975	PdeDst.b.u1Write = 0;
2976	}
2977	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2978	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2979
2980	/*
2981	* Fill the shadow page table.
2982	*/
2983	/* Get address and flags from the source PDE. */
2984	SHWPTE PteDstBase;
2985	SHW_PTE_SET(PteDstBase, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc));
2986
2987	/* Loop thru the entries in the shadow PT. */
2988	const RTGCPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
2989	Log2(("SyncPT: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%RGv GCPhys=%RGp %s\n",
2990	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
2991	GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2992	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
2993	unsigned iPTDst = 0;
2994	while ( iPTDst < RT_ELEMENTS(pPTDst->a)
2995	&& !VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY))
2996	{
2997	if (pRam && GCPhys >= pRam->GCPhys)
2998	{
2999	# ifndef PGM_WITH_A20
3000	unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
3001	# endif
3002	do
3003	{
3004	/* Make shadow PTE. */
3005	# ifdef PGM_WITH_A20
3006	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
3007	# else
3008	PPGMPAGE pPage = &pRam->aPages[iHCPage];
3009	# endif
3010	SHWPTE PteDst;
3011
3012	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
3013	/* Try to make the page writable if necessary. */
3014	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
3015	&& ( PGM_PAGE_IS_ZERO(pPage)
3016	\|\| ( SHW_PTE_IS_RW(PteDstBase)
3017	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
3018	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
3019	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
3020	# endif
3021	# ifdef VBOX_WITH_PAGE_SHARING
3022	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
3023	# endif
3024	&& !PGM_PAGE_IS_BALLOONED(pPage))
3025	)
3026	)
3027	{
3028	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
3029	AssertRCReturn(rc, rc);
3030	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY))
3031	break;
3032	}
3033	# endif
3034
3035	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
3036	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, SHW_PTE_GET_U(PteDstBase), &PteDst);
3037	else if (PGM_PAGE_IS_BALLOONED(pPage))
3038	SHW_PTE_SET(PteDst, 0); /* Handle ballooned pages at #PF time. */
3039	# ifdef VBOX_WITH_RAW_MODE_NOT_R0
3040	/*
3041	* Assuming kernel code will be marked as supervisor and not as user level and executed
3042	* using a conforming code selector. Don't check for readonly, as that implies the whole
3043	* 4MB can be code or readonly data. Linux enables write access for its large pages.
3044	*/
3045	else if ( !PdeSrc.n.u1User
3046	&& CSAMDoesPageNeedScanning(pVM, GCPtr \| (iPTDst << SHW_PT_SHIFT)))
3047	SHW_PTE_SET(PteDst, 0);
3048	# endif
3049	else
3050	SHW_PTE_SET(PteDst, PGM_PAGE_GET_HCPHYS(pPage) \| SHW_PTE_GET_U(PteDstBase));
3051
3052	/* Only map writable pages writable. */
3053	if ( SHW_PTE_IS_P_RW(PteDst)
3054	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
3055	{
3056	/* Still applies to shared pages. */
3057	Assert(!PGM_PAGE_IS_ZERO(pPage));
3058	SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */
3059	Log3(("SyncPT: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, (RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))));
3060	}
3061
3062	if (SHW_PTE_IS_P(PteDst))
3063	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
3064
3065	/* commit it (not atomic, new table) */
3066	pPTDst->a[iPTDst] = PteDst;
3067	Log4(("SyncPT: BIG %RGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
3068	(RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT)), SHW_PTE_IS_P(PteDst), SHW_PTE_IS_RW(PteDst), SHW_PTE_IS_US(PteDst), SHW_PTE_LOG64(PteDst),
3069	SHW_PTE_IS_TRACK_DIRTY(PteDst) ? " Track-Dirty" : ""));
3070
3071	/* advance */
3072	GCPhys += PAGE_SIZE;
3073	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys);
3074	# ifndef PGM_WITH_A20
3075	iHCPage++;
3076	# endif
3077	iPTDst++;
3078	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
3079	&& GCPhys <= pRam->GCPhysLast);
3080
3081	/* Advance ram range list. */
3082	while (pRam && GCPhys > pRam->GCPhysLast)
3083	pRam = pRam->CTX_SUFF(pNext);
3084	}
3085	else if (pRam)
3086	{
3087	Log(("Invalid pages at %RGp\n", GCPhys));
3088	do
3089	{
3090	SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
3091	GCPhys += PAGE_SIZE;
3092	iPTDst++;
3093	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
3094	&& GCPhys < pRam->GCPhys);
3095	PGM_A20_APPLY_TO_VAR(pVCpu,GCPhys);
3096	}
3097	else
3098	{
3099	Log(("Invalid pages at %RGp (2)\n", GCPhys));
3100	for ( ; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
3101	SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
3102	}
3103	} /* while more PTEs */
3104	} /* 4KB / 4MB */
3105	}
3106	else
3107	AssertRelease(!PdeDst.n.u1Present);
3108
3109	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3110	if (RT_FAILURE(rc))
3111	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPTFailed));
3112	return rc;
3113
3114	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
3115	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
3116	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
3117	&& !defined(IN_RC)
3118	NOREF(iPDSrc); NOREF(pPDSrc);
3119
3120	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3121
3122	/*
3123	* Validate input a little bit.
3124	*/
3125	int rc = VINF_SUCCESS;
3126	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3127	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3128	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
3129
3130	/* Fetch the pgm pool shadow descriptor. */
3131	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
3132	Assert(pShwPde);
3133
3134	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3135	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3136	PPGMPOOLPAGE pShwPde = NULL; /* initialized to shut up gcc */
3137	PX86PDPAE pPDDst;
3138	PSHWPDE pPdeDst;
3139
3140	/* Fetch the pgm pool shadow descriptor. */
3141	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
3142	AssertRCSuccessReturn(rc, rc);
3143	Assert(pShwPde);
3144
3145	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
3146	pPdeDst = &pPDDst->a[iPDDst];
3147
3148	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3149	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3150	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3151	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
3152	PX86PDPT pPdptDst= NULL; /* initialized to shut up gcc */
3153	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
3154	AssertRCSuccessReturn(rc, rc);
3155	Assert(pPDDst);
3156	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3157
3158	/* Fetch the pgm pool shadow descriptor. */
3159	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
3160	Assert(pShwPde);
3161
3162	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
3163	const unsigned iPdpt = (GCPtrPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
3164	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3165	PEPTPD pPDDst;
3166	PEPTPDPT pPdptDst;
3167
3168	rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, &pPdptDst, &pPDDst);
3169	if (rc != VINF_SUCCESS)
3170	{
3171	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3172	AssertRC(rc);
3173	return rc;
3174	}
3175	Assert(pPDDst);
3176	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3177
3178	/* Fetch the pgm pool shadow descriptor. */
3179	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & EPT_PDPTE_PG_MASK);
3180	Assert(pShwPde);
3181	# endif
3182	SHWPDE PdeDst = *pPdeDst;
3183
3184	Assert(!(PdeDst.u & PGM_PDFLAGS_MAPPING));
3185	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
3186
3187	# if defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
3188	if (BTH_IS_NP_ACTIVE(pVM))
3189	{
3190	/* Check if we allocated a big page before for this 2 MB range. */
3191	PPGMPAGE pPage;
3192	rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PDE2M_PAE_PG_MASK), &pPage);
3193	if (RT_SUCCESS(rc))
3194	{
3195	RTHCPHYS HCPhys = NIL_RTHCPHYS;
3196	if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
3197	{
3198	if (PGM_A20_IS_ENABLED(pVCpu))
3199	{
3200	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageReused);
3201	AssertRelease(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3202	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3203	}
3204	else
3205	{
3206	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PDE_DISABLED);
3207	pVM->pgm.s.cLargePagesDisabled++;
3208	}
3209	}
3210	else if ( PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED
3211	&& PGM_A20_IS_ENABLED(pVCpu))
3212	{
3213	/* Recheck the entire 2 MB range to see if we can use it again as a large page. */
3214	rc = pgmPhysRecheckLargePage(pVM, GCPtrPage, pPage);
3215	if (RT_SUCCESS(rc))
3216	{
3217	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3218	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3219	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3220	}
3221	}
3222	else if ( PGMIsUsingLargePages(pVM)
3223	&& PGM_A20_IS_ENABLED(pVCpu))
3224	{
3225	rc = pgmPhysAllocLargePage(pVM, GCPtrPage);
3226	if (RT_SUCCESS(rc))
3227	{
3228	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3229	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3230	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3231	}
3232	else
3233	LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
3234	}
3235
3236	if (HCPhys != NIL_RTHCPHYS)
3237	{
3238	PdeDst.u &= X86_PDE_AVL_MASK;
3239	PdeDst.u \|= HCPhys;
3240	PdeDst.n.u1Present = 1;
3241	PdeDst.n.u1Write = 1;
3242	PdeDst.b.u1Size = 1;
3243	# if PGM_SHW_TYPE == PGM_TYPE_EPT
3244	PdeDst.n.u1Execute = 1;
3245	PdeDst.b.u1IgnorePAT = 1;
3246	PdeDst.b.u3EMT = VMX_EPT_MEMTYPE_WB;
3247	# else
3248	PdeDst.n.u1User = 1;
3249	# endif
3250	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
3251
3252	Log(("SyncPT: Use large page at %RGp PDE=%RX64\n", GCPtrPage, PdeDst.u));
3253	/* Add a reference to the first page only. */
3254	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPde, PGM_PAGE_GET_TRACKING(pPage), pPage, iPDDst);
3255
3256	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3257	return VINF_SUCCESS;
3258	}
3259	}
3260	}
3261	# endif /* HC_ARCH_BITS == 64 */
3262
3263	/*
3264	* Allocate & map the page table.
3265	*/
3266	PSHWPT pPTDst;
3267	PPGMPOOLPAGE pShwPage;
3268	RTGCPHYS GCPhys;
3269
3270	/* Virtual address = physical address */
3271	GCPhys = PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PAGE_4K_BASE_MASK);
3272	rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE,
3273	PGM_A20_IS_ENABLED(pVCpu), pShwPde->idx, iPDDst, false /fLockPage/,
3274	&pShwPage);
3275	if ( rc == VINF_SUCCESS
3276	\|\| rc == VINF_PGM_CACHED_PAGE)
3277	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3278	else
3279	{
3280	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3281	AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
3282	}
3283
3284	if (rc == VINF_SUCCESS)
3285	{
3286	/* New page table; fully set it up. */
3287	Assert(pPTDst);
3288
3289	/* Mask away the page offset. */
3290	GCPtrPage &= ~(RTGCPTR)PAGE_OFFSET_MASK;
3291
3292	for (unsigned iPTDst = 0; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
3293	{
3294	RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
3295	\| (iPTDst << PAGE_SHIFT));
3296
3297	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
3298	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n",
3299	GCPtrCurPage,
3300	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
3301	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
3302
3303	if (RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)))
3304	break;
3305	}
3306	}
3307	else
3308	rc = VINF_SUCCESS; /* Cached entry; assume it's still fully valid. */
3309
3310	/* Save the new PDE. */
3311	PdeDst.u &= X86_PDE_AVL_MASK;
3312	PdeDst.u \|= pShwPage->Core.Key;
3313	PdeDst.n.u1Present = 1;
3314	PdeDst.n.u1Write = 1;
3315	# if PGM_SHW_TYPE == PGM_TYPE_EPT
3316	PdeDst.n.u1Execute = 1;
3317	# else
3318	PdeDst.n.u1User = 1;
3319	PdeDst.n.u1Accessed = 1;
3320	# endif
3321	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
3322
3323	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3324	if (RT_FAILURE(rc))
3325	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPTFailed));
3326	return rc;
3327
3328	#else
3329	NOREF(iPDSrc); NOREF(pPDSrc);
3330	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
3331	return VERR_PGM_NOT_USED_IN_MODE;
3332	#endif
3333	}
3334
3335
3336
3337	/**
3338	* Prefetch a page/set of pages.
3339	*
3340	* Typically used to sync commonly used pages before entering raw mode
3341	* after a CR3 reload.
3342	*
3343	* @returns VBox status code.
3344	* @param pVCpu The cross context virtual CPU structure.
3345	* @param GCPtrPage Page to invalidate.
3346	*/
3347	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
3348	{
3349	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3350	\|\| PGM_GST_TYPE == PGM_TYPE_REAL \
3351	\|\| PGM_GST_TYPE == PGM_TYPE_PROT \
3352	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3353	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3354	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
3355	&& PGM_SHW_TYPE != PGM_TYPE_EPT
3356
3357	/*
3358	* Check that all Guest levels thru the PDE are present, getting the
3359	* PD and PDE in the processes.
3360	*/
3361	int rc = VINF_SUCCESS;
3362	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3363	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3364	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
3365	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3366	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3367	unsigned iPDSrc;
3368	X86PDPE PdpeSrc;
3369	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3370	if (!pPDSrc)
3371	return VINF_SUCCESS; /* not present */
3372	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3373	unsigned iPDSrc;
3374	PX86PML4E pPml4eSrc;
3375	X86PDPE PdpeSrc;
3376	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3377	if (!pPDSrc)
3378	return VINF_SUCCESS; /* not present */
3379	# endif
3380	const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3381	# else
3382	PGSTPD pPDSrc = NULL;
3383	const unsigned iPDSrc = 0;
3384	GSTPDE PdeSrc;
3385
3386	PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3387	PdeSrc.n.u1Present = 1;
3388	PdeSrc.n.u1Write = 1;
3389	PdeSrc.n.u1Accessed = 1;
3390	PdeSrc.n.u1User = 1;
3391	# endif
3392
3393	if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
3394	{
3395	PVM pVM = pVCpu->CTX_SUFF(pVM);
3396	pgmLock(pVM);
3397
3398	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3399	const X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
3400	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3401	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3402	PX86PDPAE pPDDst;
3403	X86PDEPAE PdeDst;
3404	# if PGM_GST_TYPE != PGM_TYPE_PAE
3405	X86PDPE PdpeSrc;
3406
3407	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3408	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3409	# endif
3410	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
3411	if (rc != VINF_SUCCESS)
3412	{
3413	pgmUnlock(pVM);
3414	AssertRC(rc);
3415	return rc;
3416	}
3417	Assert(pPDDst);
3418	PdeDst = pPDDst->a[iPDDst];
3419
3420	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3421	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3422	PX86PDPAE pPDDst;
3423	X86PDEPAE PdeDst;
3424
3425	# if PGM_GST_TYPE == PGM_TYPE_PROT
3426	/* AMD-V nested paging */
3427	X86PML4E Pml4eSrc;
3428	X86PDPE PdpeSrc;
3429	PX86PML4E pPml4eSrc = &Pml4eSrc;
3430
3431	/* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3432	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
3433	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
3434	# endif
3435
3436	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
3437	if (rc != VINF_SUCCESS)
3438	{
3439	pgmUnlock(pVM);
3440	AssertRC(rc);
3441	return rc;
3442	}
3443	Assert(pPDDst);
3444	PdeDst = pPDDst->a[iPDDst];
3445	# endif
3446	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
3447	{
3448	if (!PdeDst.n.u1Present)
3449	{
3450	/** @todo r=bird: This guy will set the A bit on the PDE,
3451	* probably harmless. */
3452	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3453	}
3454	else
3455	{
3456	/* Note! We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
3457	* R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
3458	* makes no sense to prefetch more than one page.
3459	*/
3460	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3461	if (RT_SUCCESS(rc))
3462	rc = VINF_SUCCESS;
3463	}
3464	}
3465	pgmUnlock(pVM);
3466	}
3467	return rc;
3468
3469	#elif PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3470	NOREF(pVCpu); NOREF(GCPtrPage);
3471	return VINF_SUCCESS; /* ignore */
3472	#else
3473	AssertCompile(0);
3474	#endif
3475	}
3476
3477
3478
3479
3480	/**
3481	* Syncs a page during a PGMVerifyAccess() call.
3482	*
3483	* @returns VBox status code (informational included).
3484	* @param pVCpu The cross context virtual CPU structure.
3485	* @param GCPtrPage The address of the page to sync.
3486	* @param fPage The effective guest page flags.
3487	* @param uErr The trap error code.
3488	* @remarks This will normally never be called on invalid guest page
3489	* translation entries.
3490	*/
3491	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fPage, unsigned uErr)
3492	{
3493	PVM pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
3494
3495	LogFlow(("VerifyAccessSyncPage: GCPtrPage=%RGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
3496
3497	Assert(!pVM->pgm.s.fNestedPaging);
3498	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3499	\|\| PGM_GST_TYPE == PGM_TYPE_REAL \
3500	\|\| PGM_GST_TYPE == PGM_TYPE_PROT \
3501	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3502	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3503	&& PGM_SHW_TYPE != PGM_TYPE_NESTED \
3504	&& PGM_SHW_TYPE != PGM_TYPE_EPT
3505
3506	# ifdef VBOX_WITH_RAW_MODE_NOT_R0
3507	if (!(fPage & X86_PTE_US))
3508	{
3509	/*
3510	* Mark this page as safe.
3511	*/
3512	/** @todo not correct for pages that contain both code and data!! */
3513	Log(("CSAMMarkPage %RGv; scanned=%d\n", GCPtrPage, true));
3514	CSAMMarkPage(pVM, GCPtrPage, true);
3515	}
3516	# endif
3517
3518	/*
3519	* Get guest PD and index.
3520	*/
3521	/** @todo Performance: We've done all this a jiffy ago in the
3522	* PGMGstGetPage call. */
3523	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3524	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3525	const unsigned iPDSrc = GCPtrPage >> GST_PD_SHIFT;
3526	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3527
3528	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3529	unsigned iPDSrc = 0;
3530	X86PDPE PdpeSrc;
3531	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3532	if (RT_UNLIKELY(!pPDSrc))
3533	{
3534	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3535	return VINF_EM_RAW_GUEST_TRAP;
3536	}
3537
3538	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3539	unsigned iPDSrc = 0; /* shut up gcc */
3540	PX86PML4E pPml4eSrc = NULL; /* ditto */
3541	X86PDPE PdpeSrc;
3542	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3543	if (RT_UNLIKELY(!pPDSrc))
3544	{
3545	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3546	return VINF_EM_RAW_GUEST_TRAP;
3547	}
3548	# endif
3549
3550	# else /* !PGM_WITH_PAGING */
3551	PGSTPD pPDSrc = NULL;
3552	const unsigned iPDSrc = 0;
3553	# endif /* !PGM_WITH_PAGING */
3554	int rc = VINF_SUCCESS;
3555
3556	pgmLock(pVM);
3557
3558	/*
3559	* First check if the shadow pd is present.
3560	*/
3561	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3562	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
3563
3564	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3565	PX86PDEPAE pPdeDst;
3566	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3567	PX86PDPAE pPDDst;
3568	# if PGM_GST_TYPE != PGM_TYPE_PAE
3569	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3570	X86PDPE PdpeSrc;
3571	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3572	# endif
3573	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
3574	if (rc != VINF_SUCCESS)
3575	{
3576	pgmUnlock(pVM);
3577	AssertRC(rc);
3578	return rc;
3579	}
3580	Assert(pPDDst);
3581	pPdeDst = &pPDDst->a[iPDDst];
3582
3583	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3584	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3585	PX86PDPAE pPDDst;
3586	PX86PDEPAE pPdeDst;
3587
3588	# if PGM_GST_TYPE == PGM_TYPE_PROT
3589	/* AMD-V nested paging: Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3590	X86PML4E Pml4eSrc;
3591	X86PDPE PdpeSrc;
3592	PX86PML4E pPml4eSrc = &Pml4eSrc;
3593	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
3594	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
3595	# endif
3596
3597	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
3598	if (rc != VINF_SUCCESS)
3599	{
3600	pgmUnlock(pVM);
3601	AssertRC(rc);
3602	return rc;
3603	}
3604	Assert(pPDDst);
3605	pPdeDst = &pPDDst->a[iPDDst];
3606	# endif
3607
3608	if (!pPdeDst->n.u1Present)
3609	{
3610	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3611	if (rc != VINF_SUCCESS)
3612	{
3613	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3614	pgmUnlock(pVM);
3615	AssertRC(rc);
3616	return rc;
3617	}
3618	}
3619
3620	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3621	/* Check for dirty bit fault */
3622	rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
3623	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
3624	Log(("PGMVerifyAccess: success (dirty)\n"));
3625	else
3626	# endif
3627	{
3628	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3629	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3630	# else
3631	GSTPDE PdeSrc;
3632	PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3633	PdeSrc.n.u1Present = 1;
3634	PdeSrc.n.u1Write = 1;
3635	PdeSrc.n.u1Accessed = 1;
3636	PdeSrc.n.u1User = 1;
3637	# endif
3638
3639	Assert(rc != VINF_EM_RAW_GUEST_TRAP);
3640	if (uErr & X86_TRAP_PF_US)
3641	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUser));
3642	else /* supervisor */
3643	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
3644
3645	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3646	if (RT_SUCCESS(rc))
3647	{
3648	/* Page was successfully synced */
3649	Log2(("PGMVerifyAccess: success (sync)\n"));
3650	rc = VINF_SUCCESS;
3651	}
3652	else
3653	{
3654	Log(("PGMVerifyAccess: access violation for %RGv rc=%Rrc\n", GCPtrPage, rc));
3655	rc = VINF_EM_RAW_GUEST_TRAP;
3656	}
3657	}
3658	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3659	pgmUnlock(pVM);
3660	return rc;
3661
3662	#else /* PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED */
3663
3664	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
3665	return VERR_PGM_NOT_USED_IN_MODE;
3666	#endif /* PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_NESTED */
3667	}
3668
3669
3670	/**
3671	* Syncs the paging hierarchy starting at CR3.
3672	*
3673	* @returns VBox status code, R0/RC may return VINF_PGM_SYNC_CR3, no other
3674	* informational status codes.
3675	* @retval VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map
3676	* the VMM into guest context.
3677	* @param pVCpu The cross context virtual CPU structure.
3678	* @param cr0 Guest context CR0 register.
3679	* @param cr3 Guest context CR3 register. Not subjected to the A20
3680	* mask.
3681	* @param cr4 Guest context CR4 register.
3682	* @param fGlobal Including global page directories or not
3683	*/
3684	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
3685	{
3686	PVM pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
3687	NOREF(cr0); NOREF(cr3); NOREF(cr4); NOREF(fGlobal);
3688
3689	LogFlow(("SyncCR3 FF=%d fGlobal=%d\n", !!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), fGlobal));
3690
3691	#if PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT
3692
3693	pgmLock(pVM);
3694
3695	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
3696	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3697	if (pPool->cDirtyPages)
3698	pgmPoolResetDirtyPages(pVM);
3699	# endif
3700
3701	/*
3702	* Update page access handlers.
3703	* The virtual are always flushed, while the physical are only on demand.
3704	* WARNING: We are incorrectly not doing global flushing on Virtual Handler updates. We'll
3705	* have to look into that later because it will have a bad influence on the performance.
3706	* @note SvL: There's no need for that. Just invalidate the virtual range(s).
3707	* bird: Yes, but that won't work for aliases.
3708	*/
3709	/** @todo this MUST go away. See @bugref{1557}. */
3710	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3Handlers), h);
3711	PGM_GST_NAME(HandlerVirtualUpdate)(pVM, cr4);
3712	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3Handlers), h);
3713	pgmUnlock(pVM);
3714	#endif /* !NESTED && !EPT */
3715
3716	#if PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3717	/*
3718	* Nested / EPT - almost no work.
3719	*/
3720	Assert(!pgmMapAreMappingsEnabled(pVM));
3721	return VINF_SUCCESS;
3722
3723	#elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3724	/*
3725	* AMD64 (Shw & Gst) - No need to check all paging levels; we zero
3726	* out the shadow parts when the guest modifies its tables.
3727	*/
3728	Assert(!pgmMapAreMappingsEnabled(pVM));
3729	return VINF_SUCCESS;
3730
3731	#else /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3732
3733	# ifndef PGM_WITHOUT_MAPPINGS
3734	/*
3735	* Check for and resolve conflicts with our guest mappings if they
3736	* are enabled and not fixed.
3737	*/
3738	if (pgmMapAreMappingsFloating(pVM))
3739	{
3740	int rc = pgmMapResolveConflicts(pVM);
3741	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
3742	if (rc == VINF_SUCCESS)
3743	{ /* likely */ }
3744	else if (rc == VINF_PGM_SYNC_CR3)
3745	{
3746	LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
3747	return VINF_PGM_SYNC_CR3;
3748	}
3749	else if (RT_FAILURE(rc))
3750	return rc;
3751	else
3752	AssertMsgFailed(("%Rrc\n", rc));
3753	}
3754	# else
3755	Assert(!pgmMapAreMappingsEnabled(pVM));
3756	# endif
3757	return VINF_SUCCESS;
3758	#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3759	}
3760
3761
3762
3763
3764	#ifdef VBOX_STRICT
3765	# ifdef IN_RC
3766	# undef AssertMsgFailed
3767	# define AssertMsgFailed Log
3768	# endif
3769
3770	/**
3771	* Checks that the shadow page table is in sync with the guest one.
3772	*
3773	* @returns The number of errors.
3774	* @param pVCpu The cross context virtual CPU structure.
3775	* @param cr3 Guest context CR3 register.
3776	* @param cr4 Guest context CR4 register.
3777	* @param GCPtr Where to start. Defaults to 0.
3778	* @param cb How much to check. Defaults to everything.
3779	*/
3780	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb)
3781	{
3782	NOREF(pVCpu); NOREF(cr3); NOREF(cr4); NOREF(GCPtr); NOREF(cb);
3783	#if PGM_SHW_TYPE == PGM_TYPE_NESTED \|\| PGM_SHW_TYPE == PGM_TYPE_EPT
3784	return 0;
3785	#else
3786	unsigned cErrors = 0;
3787	PVM pVM = pVCpu->CTX_SUFF(pVM);
3788	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
3789
3790	# if PGM_GST_TYPE == PGM_TYPE_PAE
3791	/** @todo currently broken; crashes below somewhere */
3792	AssertFailed();
3793	# endif
3794
3795	# if PGM_GST_TYPE == PGM_TYPE_32BIT \
3796	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3797	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
3798
3799	bool fBigPagesSupported = GST_IS_PSE_ACTIVE(pVCpu);
3800	PPGMCPU pPGM = &pVCpu->pgm.s;
3801	RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
3802	RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
3803	# ifndef IN_RING0
3804	RTHCPHYS HCPhys; /* general usage. */
3805	# endif
3806	int rc;
3807
3808	/*
3809	* Check that the Guest CR3 and all its mappings are correct.
3810	*/
3811	AssertMsgReturn(pPGM->GCPhysCR3 == PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK),
3812	("Invalid GCPhysCR3=%RGp cr3=%RGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
3813	false);
3814	# if !defined(IN_RING0) && PGM_GST_TYPE != PGM_TYPE_AMD64
3815	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3816	rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGst32BitPdRC, NULL, &HCPhysShw);
3817	# else
3818	rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGstPaePdptRC, NULL, &HCPhysShw);
3819	# endif
3820	AssertRCReturn(rc, 1);
3821	HCPhys = NIL_RTHCPHYS;
3822	rc = pgmRamGCPhys2HCPhys(pVM, PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK), &HCPhys);
3823	AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
3824	# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
3825	pgmGstGet32bitPDPtr(pVCpu);
3826	RTGCPHYS GCPhys;
3827	rc = PGMR3DbgR3Ptr2GCPhys(pVM->pUVM, pPGM->pGst32BitPdR3, &GCPhys);
3828	AssertRCReturn(rc, 1);
3829	AssertMsgReturn(PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false);
3830	# endif
3831	# endif /* !IN_RING0 */
3832
3833	/*
3834	* Get and check the Shadow CR3.
3835	*/
3836	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3837	unsigned cPDEs = X86_PG_ENTRIES;
3838	unsigned cIncrement = X86_PG_ENTRIES * PAGE_SIZE;
3839	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3840	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3841	unsigned cPDEs = X86_PG_PAE_ENTRIES * 4; /* treat it as a 2048 entry table. */
3842	# else
3843	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3844	# endif
3845	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3846	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3847	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3848	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3849	# endif
3850	if (cb != ~(RTGCPTR)0)
3851	cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
3852
3853	/** @todo call the other two PGMAssert() functions. /
3854
3855	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3856	unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3857
3858	for (; iPml4 < X86_PG_PAE_ENTRIES; iPml4++)
3859	{
3860	PPGMPOOLPAGE pShwPdpt = NULL;
3861	PX86PML4E pPml4eSrc;
3862	PX86PML4E pPml4eDst;
3863	RTGCPHYS GCPhysPdptSrc;
3864
3865	pPml4eSrc = pgmGstGetLongModePML4EPtr(pVCpu, iPml4);
3866	pPml4eDst = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
3867
3868	/* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
3869	if (!pPml4eDst->n.u1Present)
3870	{
3871	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3872	continue;
3873	}
3874
3875	pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
3876	GCPhysPdptSrc = PGM_A20_APPLY(pVCpu, pPml4eSrc->u & X86_PML4E_PG_MASK);
3877
3878	if (pPml4eSrc->n.u1Present != pPml4eDst->n.u1Present)
3879	{
3880	AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3881	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3882	cErrors++;
3883	continue;
3884	}
3885
3886	if (GCPhysPdptSrc != pShwPdpt->GCPhys)
3887	{
3888	AssertMsgFailed(("Physical address doesn't match! iPml4 %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
3889	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3890	cErrors++;
3891	continue;
3892	}
3893
3894	if ( pPml4eDst->n.u1User != pPml4eSrc->n.u1User
3895	\|\| pPml4eDst->n.u1Write != pPml4eSrc->n.u1Write
3896	\|\| pPml4eDst->n.u1NoExecute != pPml4eSrc->n.u1NoExecute)
3897	{
3898	AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3899	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3900	cErrors++;
3901	continue;
3902	}
3903	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3904	{
3905	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3906
3907	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
3908	/*
3909	* Check the PDPTEs too.
3910	*/
3911	unsigned iPdpt = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
3912
3913	for (;iPdpt <= SHW_PDPT_MASK; iPdpt++)
3914	{
3915	unsigned iPDSrc = 0; /* initialized to shut up gcc */
3916	PPGMPOOLPAGE pShwPde = NULL;
3917	PX86PDPE pPdpeDst;
3918	RTGCPHYS GCPhysPdeSrc;
3919	X86PDPE PdpeSrc;
3920	PdpeSrc.u = 0; /* initialized to shut up gcc 4.5 */
3921	# if PGM_GST_TYPE == PGM_TYPE_PAE
3922	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtr, &iPDSrc, &PdpeSrc);
3923	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
3924	# else
3925	PX86PML4E pPml4eSrcIgn;
3926	PX86PDPT pPdptDst;
3927	PX86PDPAE pPDDst;
3928	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtr, &pPml4eSrcIgn, &PdpeSrc, &iPDSrc);
3929
3930	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtr, NULL, &pPdptDst, &pPDDst);
3931	if (rc != VINF_SUCCESS)
3932	{
3933	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
3934	GCPtr += 512 * _2M;
3935	continue; /* next PDPTE */
3936	}
3937	Assert(pPDDst);
3938	# endif
3939	Assert(iPDSrc == 0);
3940
3941	pPdpeDst = &pPdptDst->a[iPdpt];
3942
3943	if (!pPdpeDst->n.u1Present)
3944	{
3945	GCPtr += 512 * _2M;
3946	continue; /* next PDPTE */
3947	}
3948
3949	pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
3950	GCPhysPdeSrc = PGM_A20_APPLY(pVCpu, PdpeSrc.u & X86_PDPE_PG_MASK);
3951
3952	if (pPdpeDst->n.u1Present != PdpeSrc.n.u1Present)
3953	{
3954	AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3955	GCPtr += 512 * _2M;
3956	cErrors++;
3957	continue;
3958	}
3959
3960	if (GCPhysPdeSrc != pShwPde->GCPhys)
3961	{
3962	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3963	AssertMsgFailed(("Physical address doesn't match! iPml4 %d iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3964	# else
3965	AssertMsgFailed(("Physical address doesn't match! iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3966	# endif
3967	GCPtr += 512 * _2M;
3968	cErrors++;
3969	continue;
3970	}
3971
3972	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3973	if ( pPdpeDst->lm.u1User != PdpeSrc.lm.u1User
3974	\|\| pPdpeDst->lm.u1Write != PdpeSrc.lm.u1Write
3975	\|\| pPdpeDst->lm.u1NoExecute != PdpeSrc.lm.u1NoExecute)
3976	{
3977	AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3978	GCPtr += 512 * _2M;
3979	cErrors++;
3980	continue;
3981	}
3982	# endif
3983
3984	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3985	{
3986	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
3987	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3988	GSTPD const *pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3989	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3990	PCX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
3991	# endif
3992	# endif /* PGM_GST_TYPE == PGM_TYPE_32BIT */
3993	/*
3994	* Iterate the shadow page directory.
3995	*/
3996	GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
3997	unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
3998
3999	for (;
4000	iPDDst < cPDEs;
4001	iPDDst++, GCPtr += cIncrement)
4002	{
4003	# if PGM_SHW_TYPE == PGM_TYPE_PAE
4004	const SHWPDE PdeDst = *pgmShwGetPaePDEPtr(pVCpu, GCPtr);
4005	# else
4006	const SHWPDE PdeDst = pPDDst->a[iPDDst];
4007	# endif
4008	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
4009	{
4010	Assert(pgmMapAreMappingsEnabled(pVM));
4011	if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
4012	{
4013	AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
4014	cErrors++;
4015	continue;
4016	}
4017	}
4018	else if ( (PdeDst.u & X86_PDE_P)
4019	\|\| ((PdeDst.u & (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY))
4020	)
4021	{
4022	HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
4023	PPGMPOOLPAGE pPoolPage = pgmPoolGetPage(pPool, HCPhysShw);
4024	if (!pPoolPage)
4025	{
4026	AssertMsgFailed(("Invalid page table address %RHp at %RGv! PdeDst=%#RX64\n",
4027	HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
4028	cErrors++;
4029	continue;
4030	}
4031	const SHWPT pPTDst = (const SHWPT )PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pPoolPage);
4032
4033	if (PdeDst.u & (X86_PDE4M_PWT \| X86_PDE4M_PCD))
4034	{
4035	AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n",
4036	GCPtr, (uint64_t)PdeDst.u));
4037	cErrors++;
4038	}
4039
4040	if (PdeDst.u & (X86_PDE4M_G \| X86_PDE4M_D))
4041	{
4042	AssertMsgFailed(("4K PDE reserved flags at %RGv! PdeDst=%#RX64\n",
4043	GCPtr, (uint64_t)PdeDst.u));
4044	cErrors++;
4045	}
4046
4047	const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
4048	if (!PdeSrc.n.u1Present)
4049	{
4050	AssertMsgFailed(("Guest PDE at %RGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
4051	GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
4052	cErrors++;
4053	continue;
4054	}
4055
4056	if ( !PdeSrc.b.u1Size
4057	\|\| !fBigPagesSupported)
4058	{
4059	GCPhysGst = GST_GET_PDE_GCPHYS(PdeSrc);
4060	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
4061	GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst \| ((iPDDst & 1) * (PAGE_SIZE / 2)));
4062	# endif
4063	}
4064	else
4065	{
4066	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4067	if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
4068	{
4069	AssertMsgFailed(("Guest PDE at %RGv is using PSE36 or similar! PdeSrc=%#RX64\n",
4070	GCPtr, (uint64_t)PdeSrc.u));
4071	cErrors++;
4072	continue;
4073	}
4074	# endif
4075	GCPhysGst = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
4076	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
4077	GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst \| (GCPtr & RT_BIT(X86_PAGE_2M_SHIFT)));
4078	# endif
4079	}
4080
4081	if ( pPoolPage->enmKind
4082	!= (!PdeSrc.b.u1Size \|\| !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
4083	{
4084	AssertMsgFailed(("Invalid shadow page table kind %d at %RGv! PdeSrc=%#RX64\n",
4085	pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
4086	cErrors++;
4087	}
4088
4089	PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4090	if (!pPhysPage)
4091	{
4092	AssertMsgFailed(("Cannot find guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
4093	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
4094	cErrors++;
4095	continue;
4096	}
4097
4098	if (GCPhysGst != pPoolPage->GCPhys)
4099	{
4100	AssertMsgFailed(("GCPhysGst=%RGp != pPage->GCPhys=%RGp at %RGv\n",
4101	GCPhysGst, pPoolPage->GCPhys, GCPtr));
4102	cErrors++;
4103	continue;
4104	}
4105
4106	if ( !PdeSrc.b.u1Size
4107	\|\| !fBigPagesSupported)
4108	{
4109	/*
4110	* Page Table.
4111	*/
4112	const GSTPT *pPTSrc;
4113	rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, PGM_A20_APPLY(pVCpu, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1)),
4114	&pPTSrc);
4115	if (RT_FAILURE(rc))
4116	{
4117	AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
4118	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
4119	cErrors++;
4120	continue;
4121	}
4122	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/))
4123	!= (PdeDst.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/)))
4124	{
4125	/// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
4126	// (This problem will go away when/if we shadow multiple CR3s.)
4127	AssertMsgFailed(("4K PDE flags mismatch at %RGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4128	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4129	cErrors++;
4130	continue;
4131	}
4132	if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4133	{
4134	AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%RGv PdeDst=%#RX64\n",
4135	GCPtr, (uint64_t)PdeDst.u));
4136	cErrors++;
4137	continue;
4138	}
4139
4140	/* iterate the page table. */
4141	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
4142	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
4143	const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
4144	# else
4145	const unsigned offPTSrc = 0;
4146	# endif
4147	for (unsigned iPT = 0, off = 0;
4148	iPT < RT_ELEMENTS(pPTDst->a);
4149	iPT++, off += PAGE_SIZE)
4150	{
4151	const SHWPTE PteDst = pPTDst->a[iPT];
4152
4153	/* skip not-present and dirty tracked entries. */
4154	if (!(SHW_PTE_GET_U(PteDst) & (X86_PTE_P \| PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
4155	continue;
4156	Assert(SHW_PTE_IS_P(PteDst));
4157
4158	const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
4159	if (!PteSrc.n.u1Present)
4160	{
4161	# ifdef IN_RING3
4162	PGMAssertHandlerAndFlagsInSync(pVM);
4163	DBGFR3PagingDumpEx(pVM->pUVM, pVCpu->idCpu, DBGFPGDMP_FLAGS_CURRENT_CR3 \| DBGFPGDMP_FLAGS_CURRENT_MODE
4164	\| DBGFPGDMP_FLAGS_GUEST \| DBGFPGDMP_FLAGS_HEADER \| DBGFPGDMP_FLAGS_PRINT_CR3,
4165	0, 0, UINT64_MAX, 99, NULL);
4166	# endif
4167	AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n",
4168	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst), pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
4169	(uint64_t)GST_GET_PDE_GCPHYS(PdeSrc) + (iPT + offPTSrc) * sizeof(PteSrc)));
4170	cErrors++;
4171	continue;
4172	}
4173
4174	uint64_t fIgnoreFlags = GST_PTE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_G \| X86_PTE_D \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
4175	# if 1 /** @todo sync accessed bit properly... */
4176	fIgnoreFlags \|= X86_PTE_A;
4177	# endif
4178
4179	/* match the physical addresses */
4180	HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
4181	GCPhysGst = GST_GET_PTE_GCPHYS(PteSrc);
4182
4183	# ifdef IN_RING3
4184	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4185	if (RT_FAILURE(rc))
4186	{
4187	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4188	{
4189	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
4190	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4191	cErrors++;
4192	continue;
4193	}
4194	}
4195	else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
4196	{
4197	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4198	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4199	cErrors++;
4200	continue;
4201	}
4202	# endif
4203
4204	pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4205	if (!pPhysPage)
4206	{
4207	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4208	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4209	{
4210	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
4211	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4212	cErrors++;
4213	continue;
4214	}
4215	# endif
4216	if (SHW_PTE_IS_RW(PteDst))
4217	{
4218	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4219	GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4220	cErrors++;
4221	}
4222	fIgnoreFlags \|= X86_PTE_RW;
4223	}
4224	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4225	{
4226	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage:%R[pgmpage] GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4227	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4228	cErrors++;
4229	continue;
4230	}
4231
4232	/* flags */
4233	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4234	{
4235	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4236	{
4237	if (SHW_PTE_IS_RW(PteDst))
4238	{
4239	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4240	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4241	cErrors++;
4242	continue;
4243	}
4244	fIgnoreFlags \|= X86_PTE_RW;
4245	}
4246	else
4247	{
4248	if ( SHW_PTE_IS_P(PteDst)
4249	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
4250	&& !PGM_PAGE_IS_MMIO(pPhysPage)
4251	# endif
4252	)
4253	{
4254	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4255	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4256	cErrors++;
4257	continue;
4258	}
4259	fIgnoreFlags \|= X86_PTE_P;
4260	}
4261	}
4262	else
4263	{
4264	if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
4265	{
4266	if (SHW_PTE_IS_RW(PteDst))
4267	{
4268	AssertMsgFailed(("!DIRTY page at %RGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
4269	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4270	cErrors++;
4271	continue;
4272	}
4273	if (!SHW_PTE_IS_TRACK_DIRTY(PteDst))
4274	{
4275	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4276	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4277	cErrors++;
4278	continue;
4279	}
4280	if (SHW_PTE_IS_D(PteDst))
4281	{
4282	AssertMsgFailed(("!DIRTY page at %RGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4283	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4284	cErrors++;
4285	}
4286	# if 0 /** @todo sync access bit properly... */
4287	if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
4288	{
4289	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4290	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4291	cErrors++;
4292	}
4293	fIgnoreFlags \|= X86_PTE_RW;
4294	# else
4295	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
4296	# endif
4297	}
4298	else if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
4299	{
4300	/* access bit emulation (not implemented). */
4301	if (PteSrc.n.u1Accessed \|\| SHW_PTE_IS_P(PteDst))
4302	{
4303	AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
4304	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4305	cErrors++;
4306	continue;
4307	}
4308	if (!SHW_PTE_IS_A(PteDst))
4309	{
4310	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
4311	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4312	cErrors++;
4313	}
4314	fIgnoreFlags \|= X86_PTE_P;
4315	}
4316	# ifdef DEBUG_sandervl
4317	fIgnoreFlags \|= X86_PTE_D \| X86_PTE_A;
4318	# endif
4319	}
4320
4321	if ( (PteSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4322	&& (PteSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4323	)
4324	{
4325	AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
4326	GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
4327	fIgnoreFlags, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4328	cErrors++;
4329	continue;
4330	}
4331	} /* foreach PTE */
4332	}
4333	else
4334	{
4335	/*
4336	* Big Page.
4337	*/
4338	uint64_t fIgnoreFlags = X86_PDE_AVL_MASK \| GST_PDE_PG_MASK \| X86_PDE4M_G \| X86_PDE4M_D \| X86_PDE4M_PS \| X86_PDE4M_PWT \| X86_PDE4M_PCD;
4339	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
4340	{
4341	if (PdeDst.n.u1Write)
4342	{
4343	AssertMsgFailed(("!DIRTY page at %RGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4344	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4345	cErrors++;
4346	continue;
4347	}
4348	if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
4349	{
4350	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4351	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4352	cErrors++;
4353	continue;
4354	}
4355	# if 0 /** @todo sync access bit properly... */
4356	if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
4357	{
4358	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4359	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4360	cErrors++;
4361	}
4362	fIgnoreFlags \|= X86_PTE_RW;
4363	# else
4364	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
4365	# endif
4366	}
4367	else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4368	{
4369	/* access bit emulation (not implemented). */
4370	if (PdeSrc.b.u1Accessed \|\| PdeDst.n.u1Present)
4371	{
4372	AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4373	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4374	cErrors++;
4375	continue;
4376	}
4377	if (!PdeDst.n.u1Accessed)
4378	{
4379	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4380	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4381	cErrors++;
4382	}
4383	fIgnoreFlags \|= X86_PTE_P;
4384	}
4385
4386	if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
4387	{
4388	AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
4389	GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
4390	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4391	cErrors++;
4392	}
4393
4394	/* iterate the page table. */
4395	for (unsigned iPT = 0, off = 0;
4396	iPT < RT_ELEMENTS(pPTDst->a);
4397	iPT++, off += PAGE_SIZE, GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst + PAGE_SIZE))
4398	{
4399	const SHWPTE PteDst = pPTDst->a[iPT];
4400
4401	if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
4402	{
4403	AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
4404	GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4405	cErrors++;
4406	}
4407
4408	/* skip not-present entries. */
4409	if (!SHW_PTE_IS_P(PteDst)) /** @todo deal with ALL handlers and CSAM !P pages! */
4410	continue;
4411
4412	fIgnoreFlags = X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT \| X86_PTE_D \| X86_PTE_A \| X86_PTE_G \| X86_PTE_PAE_NX;
4413
4414	/* match the physical addresses */
4415	HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
4416
4417	# ifdef IN_RING3
4418	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4419	if (RT_FAILURE(rc))
4420	{
4421	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4422	{
4423	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4424	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4425	cErrors++;
4426	}
4427	}
4428	else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
4429	{
4430	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4431	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4432	cErrors++;
4433	continue;
4434	}
4435	# endif
4436	pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4437	if (!pPhysPage)
4438	{
4439	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4440	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4441	{
4442	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4443	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4444	cErrors++;
4445	continue;
4446	}
4447	# endif
4448	if (SHW_PTE_IS_RW(PteDst))
4449	{
4450	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4451	GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4452	cErrors++;
4453	}
4454	fIgnoreFlags \|= X86_PTE_RW;
4455	}
4456	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4457	{
4458	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage=%R[pgmpage] GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4459	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4460	cErrors++;
4461	continue;
4462	}
4463
4464	/* flags */
4465	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4466	{
4467	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4468	{
4469	if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
4470	{
4471	if (SHW_PTE_IS_RW(PteDst))
4472	{
4473	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4474	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4475	cErrors++;
4476	continue;
4477	}
4478	fIgnoreFlags \|= X86_PTE_RW;
4479	}
4480	}
4481	else
4482	{
4483	if ( SHW_PTE_IS_P(PteDst)
4484	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
4485	&& !PGM_PAGE_IS_MMIO(pPhysPage)
4486	# endif
4487	)
4488	{
4489	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4490	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4491	cErrors++;
4492	continue;
4493	}
4494	fIgnoreFlags \|= X86_PTE_P;
4495	}
4496	}
4497
4498	if ( (PdeSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4499	&& (PdeSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags) /* lazy phys handler dereg. */
4500	)
4501	{
4502	AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
4503	GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
4504	fIgnoreFlags, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4505	cErrors++;
4506	continue;
4507	}
4508	} /* for each PTE */
4509	}
4510	}
4511	/* not present */
4512
4513	} /* for each PDE */
4514
4515	} /* for each PDPTE */
4516
4517	} /* for each PML4E */
4518
4519	# ifdef DEBUG
4520	if (cErrors)
4521	LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
4522	# endif
4523	# endif /* GST is in {32BIT, PAE, AMD64} */
4524	return cErrors;
4525	#endif /* PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT */
4526	}
4527	#endif /* VBOX_STRICT */
4528
4529
4530	/**
4531	* Sets up the CR3 for shadow paging
4532	*
4533	* @returns Strict VBox status code.
4534	* @retval VINF_SUCCESS.
4535	*
4536	* @param pVCpu The cross context virtual CPU structure.
4537	* @param GCPhysCR3 The physical address in the CR3 register. (A20
4538	* mask already applied.)
4539	*/
4540	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3)
4541	{
4542	PVM pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
4543
4544	/* Update guest paging info. */
4545	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
4546	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
4547	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
4548
4549	LogFlow(("MapCR3: %RGp\n", GCPhysCR3));
4550	PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3);
4551
4552	/*
4553	* Map the page CR3 points at.
4554	*/
4555	RTHCPTR HCPtrGuestCR3;
4556	RTHCPHYS HCPhysGuestCR3;
4557	pgmLock(pVM);
4558	PPGMPAGE pPageCR3 = pgmPhysGetPage(pVM, GCPhysCR3);
4559	AssertReturn(pPageCR3, VERR_PGM_INVALID_CR3_ADDR);
4560	HCPhysGuestCR3 = PGM_PAGE_GET_HCPHYS(pPageCR3);
4561	/** @todo this needs some reworking wrt. locking? */
4562	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4563	HCPtrGuestCR3 = NIL_RTHCPTR;
4564	int rc = VINF_SUCCESS;
4565	# else
4566	int rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPageCR3, GCPhysCR3 & GST_CR3_PAGE_MASK, (void )&HCPtrGuestCR3); / @todo r=bird: This GCPhysCR3 masking isn't necessary. */
4567	# endif
4568	pgmUnlock(pVM);
4569	if (RT_SUCCESS(rc))
4570	{
4571	rc = PGMMap(pVM, (RTGCPTR)pVM->pgm.s.GCPtrCR3Mapping, HCPhysGuestCR3, PAGE_SIZE, 0);
4572	if (RT_SUCCESS(rc))
4573	{
4574	# ifdef IN_RC
4575	PGM_INVL_PG(pVCpu, pVM->pgm.s.GCPtrCR3Mapping);
4576	# endif
4577	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4578	pVCpu->pgm.s.pGst32BitPdR3 = (R3PTRTYPE(PX86PD))HCPtrGuestCR3;
4579	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4580	pVCpu->pgm.s.pGst32BitPdR0 = (R0PTRTYPE(PX86PD))HCPtrGuestCR3;
4581	# endif
4582	pVCpu->pgm.s.pGst32BitPdRC = (RCPTRTYPE(PX86PD))(RTRCUINTPTR)pVM->pgm.s.GCPtrCR3Mapping;
4583
4584	# elif PGM_GST_TYPE == PGM_TYPE_PAE
4585	unsigned off = GCPhysCR3 & GST_CR3_PAGE_MASK & PAGE_OFFSET_MASK;
4586	pVCpu->pgm.s.pGstPaePdptR3 = (R3PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4587	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4588	pVCpu->pgm.s.pGstPaePdptR0 = (R0PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4589	# endif
4590	pVCpu->pgm.s.pGstPaePdptRC = (RCPTRTYPE(PX86PDPT))((RTRCUINTPTR)pVM->pgm.s.GCPtrCR3Mapping + off);
4591	LogFlow(("Cached mapping %RRv\n", pVCpu->pgm.s.pGstPaePdptRC));
4592
4593	/*
4594	* Map the 4 PDs too.
4595	*/
4596	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pVCpu);
4597	RTGCPTR GCPtr = pVM->pgm.s.GCPtrCR3Mapping + PAGE_SIZE;
4598	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++, GCPtr += PAGE_SIZE)
4599	{
4600	pVCpu->pgm.s.aGstPaePdpeRegs[i].u = pGuestPDPT->a[i].u;
4601	if (pGuestPDPT->a[i].n.u1Present)
4602	{
4603	RTHCPTR HCPtr;
4604	RTHCPHYS HCPhys;
4605	RTGCPHYS GCPhys = PGM_A20_APPLY(pVCpu, pGuestPDPT->a[i].u & X86_PDPE_PG_MASK);
4606	pgmLock(pVM);
4607	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
4608	AssertReturn(pPage, VERR_PGM_INVALID_PDPE_ADDR);
4609	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
4610	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4611	HCPtr = NIL_RTHCPTR;
4612	int rc2 = VINF_SUCCESS;
4613	# else
4614	int rc2 = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)&HCPtr);
4615	# endif
4616	pgmUnlock(pVM);
4617	if (RT_SUCCESS(rc2))
4618	{
4619	rc = PGMMap(pVM, GCPtr, HCPhys, PAGE_SIZE, 0);
4620	AssertRCReturn(rc, rc);
4621
4622	pVCpu->pgm.s.apGstPaePDsR3[i] = (R3PTRTYPE(PX86PDPAE))HCPtr;
4623	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4624	pVCpu->pgm.s.apGstPaePDsR0[i] = (R0PTRTYPE(PX86PDPAE))HCPtr;
4625	# endif
4626	pVCpu->pgm.s.apGstPaePDsRC[i] = (RCPTRTYPE(PX86PDPAE))(RTRCUINTPTR)GCPtr;
4627	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys;
4628	# ifdef IN_RC
4629	PGM_INVL_PG(pVCpu, GCPtr);
4630	# endif
4631	continue;
4632	}
4633	AssertMsgFailed(("pgmR3Gst32BitMapCR3: rc2=%d GCPhys=%RGp i=%d\n", rc2, GCPhys, i));
4634	}
4635
4636	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4637	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4638	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4639	# endif
4640	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4641	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4642	# ifdef IN_RC
4643	PGM_INVL_PG(pVCpu, GCPtr); /** @todo this shouldn't be necessary? */
4644	# endif
4645	}
4646
4647	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
4648	pVCpu->pgm.s.pGstAmd64Pml4R3 = (R3PTRTYPE(PX86PML4))HCPtrGuestCR3;
4649	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4650	pVCpu->pgm.s.pGstAmd64Pml4R0 = (R0PTRTYPE(PX86PML4))HCPtrGuestCR3;
4651	# endif
4652	# endif
4653	}
4654	else
4655	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4656	}
4657	else
4658	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4659
4660	#else /* prot/real stub */
4661	int rc = VINF_SUCCESS;
4662	#endif
4663
4664	/* Update shadow paging info for guest modes with paging (32, pae, 64). */
4665	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4666	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4667	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64) \
4668	&& ( PGM_GST_TYPE != PGM_TYPE_REAL \
4669	&& PGM_GST_TYPE != PGM_TYPE_PROT))
4670
4671	Assert(!pVM->pgm.s.fNestedPaging);
4672	PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3);
4673
4674	/*
4675	* Update the shadow root page as well since that's not fixed.
4676	*/
4677	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4678	PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
4679	PPGMPOOLPAGE pNewShwPageCR3;
4680
4681	pgmLock(pVM);
4682
4683	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4684	if (pPool->cDirtyPages)
4685	pgmPoolResetDirtyPages(pVM);
4686	# endif
4687
4688	Assert(!(GCPhysCR3 >> (PAGE_SHIFT + 32)));
4689	rc = pgmPoolAlloc(pVM, GCPhysCR3 & GST_CR3_PAGE_MASK, BTH_PGMPOOLKIND_ROOT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
4690	NIL_PGMPOOL_IDX, UINT32_MAX, true /fLockPage/,
4691	&pNewShwPageCR3);
4692	AssertFatalRC(rc);
4693	rc = VINF_SUCCESS;
4694
4695	# ifdef IN_RC
4696	/*
4697	* WARNING! We can't deal with jumps to ring 3 in the code below as the
4698	* state will be inconsistent! Flush important things now while
4699	* we still can and then make sure there are no ring-3 calls.
4700	*/
4701	# ifdef VBOX_WITH_REM
4702	REMNotifyHandlerPhysicalFlushIfAlmostFull(pVM, pVCpu);
4703	# endif
4704	VMMRZCallRing3Disable(pVCpu);
4705	# endif
4706
4707	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3) = pNewShwPageCR3;
4708	# ifdef IN_RING0
4709	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4710	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4711	# elif defined(IN_RC)
4712	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4713	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4714	# else
4715	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4716	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4717	# endif
4718
4719	# ifndef PGM_WITHOUT_MAPPINGS
4720	/*
4721	* Apply all hypervisor mappings to the new CR3.
4722	* Note that SyncCR3 will be executed in case CR3 is changed in a guest paging mode; this will
4723	* make sure we check for conflicts in the new CR3 root.
4724	*/
4725	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
4726	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL) \|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
4727	# endif
4728	rc = pgmMapActivateCR3(pVM, pNewShwPageCR3);
4729	AssertRCReturn(rc, rc);
4730	# endif
4731
4732	/* Set the current hypervisor CR3. */
4733	CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
4734	SELMShadowCR3Changed(pVM, pVCpu);
4735
4736	# ifdef IN_RC
4737	/* NOTE: The state is consistent again. */
4738	VMMRZCallRing3Enable(pVCpu);
4739	# endif
4740
4741	/* Clean up the old CR3 root. */
4742	if ( pOldShwPageCR3
4743	&& pOldShwPageCR3 != pNewShwPageCR3 /* @todo can happen due to incorrect syncing between REM & PGM; find the real cause */)
4744	{
4745	Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
4746	# ifndef PGM_WITHOUT_MAPPINGS
4747	/* Remove the hypervisor mappings from the shadow page table. */
4748	pgmMapDeactivateCR3(pVM, pOldShwPageCR3);
4749	# endif
4750	/* Mark the page as unlocked; allow flushing again. */
4751	pgmPoolUnlockPage(pPool, pOldShwPageCR3);
4752
4753	pgmPoolFreeByPage(pPool, pOldShwPageCR3, NIL_PGMPOOL_IDX, UINT32_MAX);
4754	}
4755	pgmUnlock(pVM);
4756	# else
4757	NOREF(GCPhysCR3);
4758	# endif
4759
4760	return rc;
4761	}
4762
4763	/**
4764	* Unmaps the shadow CR3.
4765	*
4766	* @returns VBox status, no specials.
4767	* @param pVCpu The cross context virtual CPU structure.
4768	*/
4769	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu)
4770	{
4771	LogFlow(("UnmapCR3\n"));
4772
4773	int rc = VINF_SUCCESS;
4774	PVM pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
4775
4776	/*
4777	* Update guest paging info.
4778	*/
4779	#if PGM_GST_TYPE == PGM_TYPE_32BIT
4780	pVCpu->pgm.s.pGst32BitPdR3 = 0;
4781	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4782	pVCpu->pgm.s.pGst32BitPdR0 = 0;
4783	# endif
4784	pVCpu->pgm.s.pGst32BitPdRC = 0;
4785
4786	#elif PGM_GST_TYPE == PGM_TYPE_PAE
4787	pVCpu->pgm.s.pGstPaePdptR3 = 0;
4788	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4789	pVCpu->pgm.s.pGstPaePdptR0 = 0;
4790	# endif
4791	pVCpu->pgm.s.pGstPaePdptRC = 0;
4792	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
4793	{
4794	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4795	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4796	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4797	# endif
4798	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4799	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4800	}
4801
4802	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
4803	pVCpu->pgm.s.pGstAmd64Pml4R3 = 0;
4804	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4805	pVCpu->pgm.s.pGstAmd64Pml4R0 = 0;
4806	# endif
4807
4808	#else /* prot/real mode stub */
4809	/* nothing to do */
4810	#endif
4811
4812	#if !defined(IN_RC) /* In RC we rely on MapCR3 to do the shadow part for us at a safe time */
4813	/*
4814	* Update shadow paging info.
4815	*/
4816	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4817	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4818	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64))
4819
4820	# if PGM_GST_TYPE != PGM_TYPE_REAL
4821	Assert(!pVM->pgm.s.fNestedPaging);
4822	# endif
4823
4824	pgmLock(pVM);
4825
4826	# ifndef PGM_WITHOUT_MAPPINGS
4827	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4828	/* Remove the hypervisor mappings from the shadow page table. */
4829	pgmMapDeactivateCR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4830	# endif
4831
4832	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4833	{
4834	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4835
4836	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4837	if (pPool->cDirtyPages)
4838	pgmPoolResetDirtyPages(pVM);
4839	# endif
4840
4841	/* Mark the page as unlocked; allow flushing again. */
4842	pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4843
4844	pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), NIL_PGMPOOL_IDX, UINT32_MAX);
4845	pVCpu->pgm.s.pShwPageCR3R3 = 0;
4846	pVCpu->pgm.s.pShwPageCR3R0 = 0;
4847	pVCpu->pgm.s.pShwPageCR3RC = 0;
4848	}
4849	pgmUnlock(pVM);
4850	# endif
4851	#endif /* !IN_RC*/
4852
4853	return rc;
4854	}

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 62517

Download in other formats: