PGMInternal.h@ 19969

Last change on this file since 19969 was 19903, checked in by vboxsync, 15 years ago
Invalidation cleanup
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 185.6 KB

Line
1	/* $Id: PGMInternal.h 19903 2009-05-22 09:41:32Z vboxsync $ */
2	/** @file
3	* PGM - Internal header file.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22	#ifndef ___PGMInternal_h
23	#define ___PGMInternal_h
24
25	#include <VBox/cdefs.h>
26	#include <VBox/types.h>
27	#include <VBox/err.h>
28	#include <VBox/stam.h>
29	#include <VBox/param.h>
30	#include <VBox/vmm.h>
31	#include <VBox/mm.h>
32	#include <VBox/pdmcritsect.h>
33	#include <VBox/pdmapi.h>
34	#include <VBox/dis.h>
35	#include <VBox/dbgf.h>
36	#include <VBox/log.h>
37	#include <VBox/gmm.h>
38	#include <VBox/hwaccm.h>
39	#include <iprt/avl.h>
40	#include <iprt/assert.h>
41	#include <iprt/critsect.h>
42
43
44
45	/** @defgroup grp_pgm_int Internals
46	* @ingroup grp_pgm
47	* @internal
48	* @{
49	*/
50
51
52	/** @name PGM Compile Time Config
53	* @{
54	*/
55
56	/**
57	* Solve page is out of sync issues inside Guest Context (in PGMGC.cpp).
58	* Comment it if it will break something.
59	*/
60	#define PGM_OUT_OF_SYNC_IN_GC
61
62	/**
63	* Check and skip global PDEs for non-global flushes
64	*/
65	#define PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
66
67	/**
68	* Sync N pages instead of a whole page table
69	*/
70	#define PGM_SYNC_N_PAGES
71
72	/**
73	* Number of pages to sync during a page fault
74	*
75	* When PGMPOOL_WITH_GCPHYS_TRACKING is enabled using high values here
76	* causes a lot of unnecessary extents and also is slower than taking more \#PFs.
77	*/
78	#define PGM_SYNC_NR_PAGES 8
79
80	/**
81	* Number of PGMPhysRead/Write cache entries (must be <= sizeof(uint64_t))
82	*/
83	#define PGM_MAX_PHYSCACHE_ENTRIES 64
84	#define PGM_MAX_PHYSCACHE_ENTRIES_MASK (PGM_MAX_PHYSCACHE_ENTRIES-1)
85
86	/**
87	* Enable caching of PGMR3PhysRead/WriteByte/Word/Dword
88	*/
89	#define PGM_PHYSMEMACCESS_CACHING
90
91	/** @def PGMPOOL_WITH_CACHE
92	* Enable agressive caching using the page pool.
93	*
94	* This requires PGMPOOL_WITH_USER_TRACKING and PGMPOOL_WITH_MONITORING.
95	*/
96	#define PGMPOOL_WITH_CACHE
97
98	/** @def PGMPOOL_WITH_MIXED_PT_CR3
99	* When defined, we'll deal with 'uncachable' pages.
100	*/
101	#ifdef PGMPOOL_WITH_CACHE
102	# define PGMPOOL_WITH_MIXED_PT_CR3
103	#endif
104
105	/** @def PGMPOOL_WITH_MONITORING
106	* Monitor the guest pages which are shadowed.
107	* When this is enabled, PGMPOOL_WITH_CACHE or PGMPOOL_WITH_GCPHYS_TRACKING must
108	* be enabled as well.
109	* @remark doesn't really work without caching now. (Mixed PT/CR3 change.)
110	*/
111	#ifdef PGMPOOL_WITH_CACHE
112	# define PGMPOOL_WITH_MONITORING
113	#endif
114
115	/** @def PGMPOOL_WITH_GCPHYS_TRACKING
116	* Tracking the of shadow pages mapping guest physical pages.
117	*
118	* This is very expensive, the current cache prototype is trying to figure out
119	* whether it will be acceptable with an agressive caching policy.
120	*/
121	#if defined(PGMPOOL_WITH_CACHE) \|\| defined(PGMPOOL_WITH_MONITORING)
122	# define PGMPOOL_WITH_GCPHYS_TRACKING
123	#endif
124
125	/** @def PGMPOOL_WITH_USER_TRACKING
126	* Tracking users of shadow pages. This is required for the linking of shadow page
127	* tables and physical guest addresses.
128	*/
129	#if defined(PGMPOOL_WITH_GCPHYS_TRACKING) \|\| defined(PGMPOOL_WITH_CACHE) \|\| defined(PGMPOOL_WITH_MONITORING)
130	# define PGMPOOL_WITH_USER_TRACKING
131	#endif
132
133	/** @def PGMPOOL_CFG_MAX_GROW
134	* The maximum number of pages to add to the pool in one go.
135	*/
136	#define PGMPOOL_CFG_MAX_GROW (_256K >> PAGE_SHIFT)
137
138	/** @def VBOX_STRICT_PGM_HANDLER_VIRTUAL
139	* Enables some extra assertions for virtual handlers (mainly phys2virt related).
140	*/
141	#ifdef VBOX_STRICT
142	# define VBOX_STRICT_PGM_HANDLER_VIRTUAL
143	#endif
144
145	/** @def VBOX_WITH_NEW_LAZY_PAGE_ALLOC
146	* Enables the experimental lazy page allocation code. */
147	/# define VBOX_WITH_NEW_LAZY_PAGE_ALLOC /
148
149	/** @} */
150
151
152	/** @name PDPT and PML4 flags.
153	* These are placed in the three bits available for system programs in
154	* the PDPT and PML4 entries.
155	* @{ */
156	/** The entry is a permanent one and it's must always be present.
157	* Never free such an entry. */
158	#define PGM_PLXFLAGS_PERMANENT RT_BIT_64(10)
159	/** Mapping (hypervisor allocated pagetable). */
160	#define PGM_PLXFLAGS_MAPPING RT_BIT_64(11)
161	/** @} */
162
163	/** @name Page directory flags.
164	* These are placed in the three bits available for system programs in
165	* the page directory entries.
166	* @{ */
167	/** Mapping (hypervisor allocated pagetable). */
168	#define PGM_PDFLAGS_MAPPING RT_BIT_64(10)
169	/** Made read-only to facilitate dirty bit tracking. */
170	#define PGM_PDFLAGS_TRACK_DIRTY RT_BIT_64(11)
171	/** @} */
172
173	/** @name Page flags.
174	* These are placed in the three bits available for system programs in
175	* the page entries.
176	* @{ */
177	/** Made read-only to facilitate dirty bit tracking. */
178	#define PGM_PTFLAGS_TRACK_DIRTY RT_BIT_64(9)
179
180	#ifndef PGM_PTFLAGS_CSAM_VALIDATED
181	/** Scanned and approved by CSAM (tm).
182	* NOTE: Must be identical to the one defined in CSAMInternal.h!!
183	* @todo Move PGM_PTFLAGS_* and PGM_PDFLAGS_* to VBox/pgm.h. */
184	#define PGM_PTFLAGS_CSAM_VALIDATED RT_BIT_64(11)
185	#endif
186
187	/** @} */
188
189	/** @name Defines used to indicate the shadow and guest paging in the templates.
190	* @{ */
191	#define PGM_TYPE_REAL 1
192	#define PGM_TYPE_PROT 2
193	#define PGM_TYPE_32BIT 3
194	#define PGM_TYPE_PAE 4
195	#define PGM_TYPE_AMD64 5
196	#define PGM_TYPE_NESTED 6
197	#define PGM_TYPE_EPT 7
198	#define PGM_TYPE_MAX PGM_TYPE_EPT
199	/** @} */
200
201	/** Macro for checking if the guest is using paging.
202	* @param uGstType PGM_TYPE_*
203	* @param uShwType PGM_TYPE_*
204	* @remark ASSUMES certain order of the PGM_TYPE_* values.
205	*/
206	#define PGM_WITH_PAGING(uGstType, uShwType) \
207	( (uGstType) >= PGM_TYPE_32BIT \
208	&& (uShwType) != PGM_TYPE_NESTED \
209	&& (uShwType) != PGM_TYPE_EPT)
210
211	/** Macro for checking if the guest supports the NX bit.
212	* @param uGstType PGM_TYPE_*
213	* @param uShwType PGM_TYPE_*
214	* @remark ASSUMES certain order of the PGM_TYPE_* values.
215	*/
216	#define PGM_WITH_NX(uGstType, uShwType) \
217	( (uGstType) >= PGM_TYPE_PAE \
218	&& (uShwType) != PGM_TYPE_NESTED \
219	&& (uShwType) != PGM_TYPE_EPT)
220
221
222	/** @def PGM_HCPHYS_2_PTR
223	* Maps a HC physical page pool address to a virtual address.
224	*
225	* @returns VBox status code.
226	* @param pVM The VM handle.
227	* @param HCPhys The HC physical address to map to a virtual one.
228	* @param ppv Where to store the virtual address. No need to cast this.
229	*
230	* @remark In GC this uses PGMGCDynMapHCPage(), so it will consume of the
231	* small page window employeed by that function. Be careful.
232	* @remark There is no need to assert on the result.
233	*/
234	#ifdef IN_RC
235	# define PGM_HCPHYS_2_PTR(pVM, HCPhys, ppv) \
236	PGMDynMapHCPage(pVM, HCPhys, (void **)(ppv))
237	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
238	# define PGM_HCPHYS_2_PTR(pVM, HCPhys, ppv) \
239	pgmR0DynMapHCPageInlined(&(pVM)->pgm.s, HCPhys, (void **)(ppv))
240	#else
241	# define PGM_HCPHYS_2_PTR(pVM, HCPhys, ppv) \
242	MMPagePhys2PageEx(pVM, HCPhys, (void **)(ppv))
243	#endif
244
245	/** @def PGM_HCPHYS_2_PTR_BY_PGM
246	* Maps a HC physical page pool address to a virtual address.
247	*
248	* @returns VBox status code.
249	* @param pPGM The PGM instance data.
250	* @param HCPhys The HC physical address to map to a virtual one.
251	* @param ppv Where to store the virtual address. No need to cast this.
252	*
253	* @remark In GC this uses PGMGCDynMapHCPage(), so it will consume of the
254	* small page window employeed by that function. Be careful.
255	* @remark There is no need to assert on the result.
256	*/
257	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
258	# define PGM_HCPHYS_2_PTR_BY_PGM(pPGM, HCPhys, ppv) \
259	pgmR0DynMapHCPageInlined(pPGM, HCPhys, (void **)(ppv))
260	#else
261	# define PGM_HCPHYS_2_PTR_BY_PGM(pPGM, HCPhys, ppv) \
262	PGM_HCPHYS_2_PTR(PGM2VM(pPGM), HCPhys, (void **)(ppv))
263	#endif
264
265	/** @def PGM_GCPHYS_2_PTR
266	* Maps a GC physical page address to a virtual address.
267	*
268	* @returns VBox status code.
269	* @param pVM The VM handle.
270	* @param GCPhys The GC physical address to map to a virtual one.
271	* @param ppv Where to store the virtual address. No need to cast this.
272	*
273	* @remark In GC this uses PGMGCDynMapGCPage(), so it will consume of the
274	* small page window employeed by that function. Be careful.
275	* @remark There is no need to assert on the result.
276	*/
277	#ifdef IN_RC
278	# define PGM_GCPHYS_2_PTR(pVM, GCPhys, ppv) \
279	PGMDynMapGCPage(pVM, GCPhys, (void **)(ppv))
280	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
281	# define PGM_GCPHYS_2_PTR(pVM, GCPhys, ppv) \
282	pgmR0DynMapGCPageInlined(&(pVM)->pgm.s, GCPhys, (void **)(ppv))
283	#else
284	# define PGM_GCPHYS_2_PTR(pVM, GCPhys, ppv) \
285	PGMPhysGCPhys2R3Ptr(pVM, GCPhys, 1 /* one page only /, (PRTR3PTR)(ppv)) /* @todo this isn't asserting, use PGMRamGCPhys2HCPtr! */
286	#endif
287
288	/** @def PGM_GCPHYS_2_PTR_BY_PGMCPU
289	* Maps a GC physical page address to a virtual address.
290	*
291	* @returns VBox status code.
292	* @param pPGM Pointer to the PGM instance data.
293	* @param GCPhys The GC physical address to map to a virtual one.
294	* @param ppv Where to store the virtual address. No need to cast this.
295	*
296	* @remark In GC this uses PGMGCDynMapGCPage(), so it will consume of the
297	* small page window employeed by that function. Be careful.
298	* @remark There is no need to assert on the result.
299	*/
300	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
301	# define PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, GCPhys, ppv) \
302	pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), GCPhys, (void **)(ppv))
303	#else
304	# define PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, GCPhys, ppv) \
305	PGM_GCPHYS_2_PTR(PGMCPU2VM(pPGM), GCPhys, ppv)
306	#endif
307
308	/** @def PGM_GCPHYS_2_PTR_EX
309	* Maps a unaligned GC physical page address to a virtual address.
310	*
311	* @returns VBox status code.
312	* @param pVM The VM handle.
313	* @param GCPhys The GC physical address to map to a virtual one.
314	* @param ppv Where to store the virtual address. No need to cast this.
315	*
316	* @remark In GC this uses PGMGCDynMapGCPage(), so it will consume of the
317	* small page window employeed by that function. Be careful.
318	* @remark There is no need to assert on the result.
319	*/
320	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
321	# define PGM_GCPHYS_2_PTR_EX(pVM, GCPhys, ppv) \
322	PGMDynMapGCPageOff(pVM, GCPhys, (void **)(ppv))
323	#else
324	# define PGM_GCPHYS_2_PTR_EX(pVM, GCPhys, ppv) \
325	PGMPhysGCPhys2R3Ptr(pVM, GCPhys, 1 /* one page only /, (PRTR3PTR)(ppv)) /* @todo this isn't asserting, use PGMRamGCPhys2HCPtr! */
326	#endif
327
328	/** @def PGM_INVL_PG
329	* Invalidates a page.
330	*
331	* @param pVCpu The VMCPU handle.
332	* @param GCVirt The virtual address of the page to invalidate.
333	*/
334	#ifdef IN_RC
335	# define PGM_INVL_PG(pVCpu, GCVirt) ASMInvalidatePage((void *)(GCVirt))
336	#elif defined(IN_RING0)
337	# define PGM_INVL_PG(pVCpu, GCVirt) HWACCMInvalidatePage(pVCpu, (RTGCPTR)(GCVirt))
338	#else
339	# define PGM_INVL_PG(pVCpu, GCVirt) HWACCMInvalidatePage(pVCpu, (RTGCPTR)(GCVirt))
340	#endif
341
342	/** @def PGM_INVL_PG
343	* Invalidates a page on all VCPUs
344	*
345	* @param pVM The VM handle.
346	* @param GCVirt The virtual address of the page to invalidate.
347	*/
348	#ifdef IN_RC
349	# define PGM_INVL_ALL_VCPU_PG(pVM, GCVirt) ASMInvalidatePage((void *)(GCVirt))
350	#elif defined(IN_RING0)
351	# define PGM_INVL_ALL_VCPU_PG(pVM, GCVirt) HWACCMInvalidatePageOnAllVCpus(pVM, (RTGCPTR)(GCVirt))
352	#else
353	# define PGM_INVL_ALL_VCPU_PG(pVM, GCVirt) HWACCMInvalidatePageOnAllVCpus(pVM, (RTGCPTR)(GCVirt))
354	#endif
355
356	/** @def PGM_INVL_BIG_PG
357	* Invalidates a 4MB page directory entry.
358	*
359	* @param pVCpu The VMCPU handle.
360	* @param GCVirt The virtual address within the page directory to invalidate.
361	*/
362	#ifdef IN_RC
363	# define PGM_INVL_BIG_PG(pVCpu, GCVirt) ASMReloadCR3()
364	#elif defined(IN_RING0)
365	# define PGM_INVL_BIG_PG(pVCpu, GCVirt) HWACCMFlushTLB(pVCpu)
366	#else
367	# define PGM_INVL_BIG_PG(pVCpu, GCVirt) HWACCMFlushTLB(pVCpu)
368	#endif
369
370	/** @def PGM_INVL_VCPU_TLBS()
371	* Invalidates the TLBs of the specified VCPU
372	*
373	* @param pVCpu The VMCPU handle.
374	*/
375	#ifdef IN_RC
376	# define PGM_INVL_VCPU_TLBS(pVCpu) ASMReloadCR3()
377	#elif defined(IN_RING0)
378	# define PGM_INVL_VCPU_TLBS(pVCpu) HWACCMFlushTLB(pVCpu)
379	#else
380	# define PGM_INVL_VCPU_TLBS(pVCpu) HWACCMFlushTLB(pVCpu)
381	#endif
382
383	/** @def PGM_INVL_ALL_VCPU_TLBS()
384	* Invalidates the TLBs of all VCPUs
385	*
386	* @param pVM The VM handle.
387	*/
388	#ifdef IN_RC
389	# define PGM_INVL_ALL_VCPU_TLBS(pVM) ASMReloadCR3()
390	#elif defined(IN_RING0)
391	# define PGM_INVL_ALL_VCPU_TLBS(pVM) HWACCMFlushTLBOnAllVCpus(pVM)
392	#else
393	# define PGM_INVL_ALL_VCPU_TLBS(pVM) HWACCMFlushTLBOnAllVCpus(pVM)
394	#endif
395
396	/** Size of the GCPtrConflict array in PGMMAPPING.
397	* @remarks Must be a power of two. */
398	#define PGMMAPPING_CONFLICT_MAX 8
399
400	/**
401	* Structure for tracking GC Mappings.
402	*
403	* This structure is used by linked list in both GC and HC.
404	*/
405	typedef struct PGMMAPPING
406	{
407	/** Pointer to next entry. */
408	R3PTRTYPE(struct PGMMAPPING *) pNextR3;
409	/** Pointer to next entry. */
410	R0PTRTYPE(struct PGMMAPPING *) pNextR0;
411	/** Pointer to next entry. */
412	RCPTRTYPE(struct PGMMAPPING *) pNextRC;
413	/** Indicate whether this entry is finalized. */
414	bool fFinalized;
415	/** Start Virtual address. */
416	RTGCPTR GCPtr;
417	/** Last Virtual address (inclusive). */
418	RTGCPTR GCPtrLast;
419	/** Range size (bytes). */
420	RTGCPTR cb;
421	/** Pointer to relocation callback function. */
422	R3PTRTYPE(PFNPGMRELOCATE) pfnRelocate;
423	/** User argument to the callback. */
424	R3PTRTYPE(void *) pvUser;
425	/** Mapping description / name. For easing debugging. */
426	R3PTRTYPE(const char *) pszDesc;
427	/** Last 8 addresses that caused conflicts. */
428	RTGCPTR aGCPtrConflicts[PGMMAPPING_CONFLICT_MAX];
429	/** Number of conflicts for this hypervisor mapping. */
430	uint32_t cConflicts;
431	/** Number of page tables. */
432	uint32_t cPTs;
433
434	/** Array of page table mapping data. Each entry
435	* describes one page table. The array can be longer
436	* than the declared length.
437	*/
438	struct
439	{
440	/** The HC physical address of the page table. */
441	RTHCPHYS HCPhysPT;
442	/** The HC physical address of the first PAE page table. */
443	RTHCPHYS HCPhysPaePT0;
444	/** The HC physical address of the second PAE page table. */
445	RTHCPHYS HCPhysPaePT1;
446	/** The HC virtual address of the 32-bit page table. */
447	R3PTRTYPE(PX86PT) pPTR3;
448	/** The HC virtual address of the two PAE page table. (i.e 1024 entries instead of 512) */
449	R3PTRTYPE(PX86PTPAE) paPaePTsR3;
450	/** The GC virtual address of the 32-bit page table. */
451	RCPTRTYPE(PX86PT) pPTRC;
452	/** The GC virtual address of the two PAE page table. */
453	RCPTRTYPE(PX86PTPAE) paPaePTsRC;
454	/** The GC virtual address of the 32-bit page table. */
455	R0PTRTYPE(PX86PT) pPTR0;
456	/** The GC virtual address of the two PAE page table. */
457	R0PTRTYPE(PX86PTPAE) paPaePTsR0;
458	} aPTs[1];
459	} PGMMAPPING;
460	/** Pointer to structure for tracking GC Mappings. */
461	typedef struct PGMMAPPING *PPGMMAPPING;
462
463
464	/**
465	* Physical page access handler structure.
466	*
467	* This is used to keep track of physical address ranges
468	* which are being monitored in some kind of way.
469	*/
470	typedef struct PGMPHYSHANDLER
471	{
472	AVLROGCPHYSNODECORE Core;
473	/** Access type. */
474	PGMPHYSHANDLERTYPE enmType;
475	/** Number of pages to update. */
476	uint32_t cPages;
477	/** Pointer to R3 callback function. */
478	R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnHandlerR3;
479	/** User argument for R3 handlers. */
480	R3PTRTYPE(void *) pvUserR3;
481	/** Pointer to R0 callback function. */
482	R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnHandlerR0;
483	/** User argument for R0 handlers. */
484	R0PTRTYPE(void *) pvUserR0;
485	/** Pointer to GC callback function. */
486	RCPTRTYPE(PFNPGMRCPHYSHANDLER) pfnHandlerRC;
487	/** User argument for RC handlers. */
488	RCPTRTYPE(void *) pvUserRC;
489	/** Description / Name. For easing debugging. */
490	R3PTRTYPE(const char *) pszDesc;
491	#ifdef VBOX_WITH_STATISTICS
492	/** Profiling of this handler. */
493	STAMPROFILE Stat;
494	#endif
495	} PGMPHYSHANDLER;
496	/** Pointer to a physical page access handler structure. */
497	typedef PGMPHYSHANDLER *PPGMPHYSHANDLER;
498
499
500	/**
501	* Cache node for the physical addresses covered by a virtual handler.
502	*/
503	typedef struct PGMPHYS2VIRTHANDLER
504	{
505	/** Core node for the tree based on physical ranges. */
506	AVLROGCPHYSNODECORE Core;
507	/** Offset from this struct to the PGMVIRTHANDLER structure. */
508	int32_t offVirtHandler;
509	/** Offset of the next alias relative to this one.
510	* Bit 0 is used for indicating whether we're in the tree.
511	* Bit 1 is used for indicating that we're the head node.
512	*/
513	int32_t offNextAlias;
514	} PGMPHYS2VIRTHANDLER;
515	/** Pointer to a phys to virtual handler structure. */
516	typedef PGMPHYS2VIRTHANDLER *PPGMPHYS2VIRTHANDLER;
517
518	/** The bit in PGMPHYS2VIRTHANDLER::offNextAlias used to indicate that the
519	* node is in the tree. */
520	#define PGMPHYS2VIRTHANDLER_IN_TREE RT_BIT(0)
521	/** The bit in PGMPHYS2VIRTHANDLER::offNextAlias used to indicate that the
522	* node is in the head of an alias chain.
523	* The PGMPHYS2VIRTHANDLER_IN_TREE is always set if this bit is set. */
524	#define PGMPHYS2VIRTHANDLER_IS_HEAD RT_BIT(1)
525	/** The mask to apply to PGMPHYS2VIRTHANDLER::offNextAlias to get the offset. */
526	#define PGMPHYS2VIRTHANDLER_OFF_MASK (~(int32_t)3)
527
528
529	/**
530	* Virtual page access handler structure.
531	*
532	* This is used to keep track of virtual address ranges
533	* which are being monitored in some kind of way.
534	*/
535	typedef struct PGMVIRTHANDLER
536	{
537	/** Core node for the tree based on virtual ranges. */
538	AVLROGCPTRNODECORE Core;
539	/** Size of the range (in bytes). */
540	RTGCPTR cb;
541	/** Number of cache pages. */
542	uint32_t cPages;
543	/** Access type. */
544	PGMVIRTHANDLERTYPE enmType;
545	/** Pointer to the RC callback function. */
546	RCPTRTYPE(PFNPGMRCVIRTHANDLER) pfnHandlerRC;
547	#if HC_ARCH_BITS == 64
548	RTRCPTR padding;
549	#endif
550	/** Pointer to the R3 callback function for invalidation. */
551	R3PTRTYPE(PFNPGMR3VIRTINVALIDATE) pfnInvalidateR3;
552	/** Pointer to the R3 callback function. */
553	R3PTRTYPE(PFNPGMR3VIRTHANDLER) pfnHandlerR3;
554	/** Description / Name. For easing debugging. */
555	R3PTRTYPE(const char *) pszDesc;
556	#ifdef VBOX_WITH_STATISTICS
557	/** Profiling of this handler. */
558	STAMPROFILE Stat;
559	#endif
560	/** Array of cached physical addresses for the monitored ranged. */
561	PGMPHYS2VIRTHANDLER aPhysToVirt[HC_ARCH_BITS == 32 ? 1 : 2];
562	} PGMVIRTHANDLER;
563	/** Pointer to a virtual page access handler structure. */
564	typedef PGMVIRTHANDLER *PPGMVIRTHANDLER;
565
566
567	/**
568	* Page type.
569	*
570	* @remarks This enum has to fit in a 3-bit field (see PGMPAGE::u3Type).
571	* @remarks This is used in the saved state, so changes to it requires bumping
572	* the saved state version.
573	* @todo So, convert to \#defines!
574	*/
575	typedef enum PGMPAGETYPE
576	{
577	/** The usual invalid zero entry. */
578	PGMPAGETYPE_INVALID = 0,
579	/** RAM page. (RWX) */
580	PGMPAGETYPE_RAM,
581	/** MMIO2 page. (RWX) */
582	PGMPAGETYPE_MMIO2,
583	/** MMIO2 page aliased over an MMIO page. (RWX)
584	* See PGMHandlerPhysicalPageAlias(). */
585	PGMPAGETYPE_MMIO2_ALIAS_MMIO,
586	/** Shadowed ROM. (RWX) */
587	PGMPAGETYPE_ROM_SHADOW,
588	/** ROM page. (R-X) */
589	PGMPAGETYPE_ROM,
590	/** MMIO page. (---) */
591	PGMPAGETYPE_MMIO,
592	/** End of valid entries. */
593	PGMPAGETYPE_END
594	} PGMPAGETYPE;
595	AssertCompile(PGMPAGETYPE_END <= 7);
596
597	/** @name Page type predicates.
598	* @{ */
599	#define PGMPAGETYPE_IS_READABLE(type) ( (type) <= PGMPAGETYPE_ROM )
600	#define PGMPAGETYPE_IS_WRITEABLE(type) ( (type) <= PGMPAGETYPE_ROM_SHADOW )
601	#define PGMPAGETYPE_IS_RWX(type) ( (type) <= PGMPAGETYPE_ROM_SHADOW )
602	#define PGMPAGETYPE_IS_ROX(type) ( (type) == PGMPAGETYPE_ROM )
603	#define PGMPAGETYPE_IS_NP(type) ( (type) == PGMPAGETYPE_MMIO )
604	/** @} */
605
606
607	/**
608	* A Physical Guest Page tracking structure.
609	*
610	* The format of this structure is complicated because we have to fit a lot
611	* of information into as few bits as possible. The format is also subject
612	* to change (there is one comming up soon). Which means that for we'll be
613	* using PGM_PAGE_GET_, PGM_PAGE_IS_ and PGM_PAGE_SET_ macros for all
614	* accessess to the structure.
615	*/
616	typedef struct PGMPAGE
617	{
618	/** The physical address and a whole lot of other stuff. All bits are used! */
619	RTHCPHYS HCPhysX;
620	/** The page state. */
621	uint32_t u2StateX : 2;
622	/** Flag indicating that a write monitored page was written to when set. */
623	uint32_t fWrittenToX : 1;
624	/** For later. */
625	uint32_t fSomethingElse : 1;
626	/** The Page ID.
627	* @todo Merge with HCPhysX once we've liberated HCPhysX of its stuff.
628	* The HCPhysX will then be 100% static. */
629	uint32_t idPageX : 28;
630	/** The page type (PGMPAGETYPE). */
631	uint32_t u3Type : 3;
632	/** The physical handler state (PGM_PAGE_HNDL_PHYS_STATE) /
633	uint32_t u2HandlerPhysStateX : 2;
634	/** The virtual handler state (PGM_PAGE_HNDL_VIRT_STATE) /
635	uint32_t u2HandlerVirtStateX : 2;
636	uint32_t u29B : 25;
637	} PGMPAGE;
638	AssertCompileSize(PGMPAGE, 16);
639	/** Pointer to a physical guest page. */
640	typedef PGMPAGE *PPGMPAGE;
641	/** Pointer to a const physical guest page. */
642	typedef const PGMPAGE *PCPGMPAGE;
643	/** Pointer to a physical guest page pointer. */
644	typedef PPGMPAGE *PPPGMPAGE;
645
646
647	/**
648	* Clears the page structure.
649	* @param pPage Pointer to the physical guest page tracking structure.
650	*/
651	#define PGM_PAGE_CLEAR(pPage) \
652	do { \
653	(pPage)->HCPhysX = 0; \
654	(pPage)->u2StateX = 0; \
655	(pPage)->fWrittenToX = 0; \
656	(pPage)->fSomethingElse = 0; \
657	(pPage)->idPageX = 0; \
658	(pPage)->u3Type = 0; \
659	(pPage)->u29B = 0; \
660	} while (0)
661
662	/**
663	* Initializes the page structure.
664	* @param pPage Pointer to the physical guest page tracking structure.
665	*/
666	#define PGM_PAGE_INIT(pPage, _HCPhys, _idPage, _uType, _uState) \
667	do { \
668	(pPage)->HCPhysX = (_HCPhys); \
669	(pPage)->u2StateX = (_uState); \
670	(pPage)->fWrittenToX = 0; \
671	(pPage)->fSomethingElse = 0; \
672	(pPage)->idPageX = (_idPage); \
673	/(pPage)->u3Type = (_uType); - later / \
674	PGM_PAGE_SET_TYPE(pPage, _uType); \
675	(pPage)->u29B = 0; \
676	} while (0)
677
678	/**
679	* Initializes the page structure of a ZERO page.
680	* @param pPage Pointer to the physical guest page tracking structure.
681	*/
682	#define PGM_PAGE_INIT_ZERO(pPage, pVM, _uType) \
683	PGM_PAGE_INIT(pPage, (pVM)->pgm.s.HCPhysZeroPg, NIL_GMM_PAGEID, (_uType), PGM_PAGE_STATE_ZERO)
684	/** Temporary hack. Replaced by PGM_PAGE_INIT_ZERO once the old code is kicked out. */
685	# define PGM_PAGE_INIT_ZERO_REAL(pPage, pVM, _uType) \
686	PGM_PAGE_INIT(pPage, (pVM)->pgm.s.HCPhysZeroPg, NIL_GMM_PAGEID, (_uType), PGM_PAGE_STATE_ZERO)
687
688
689	/** @name The Page state, PGMPAGE::u2StateX.
690	* @{ */
691	/** The zero page.
692	* This is a per-VM page that's never ever mapped writable. */
693	#define PGM_PAGE_STATE_ZERO 0
694	/** A allocated page.
695	* This is a per-VM page allocated from the page pool (or wherever
696	* we get MMIO2 pages from if the type is MMIO2).
697	*/
698	#define PGM_PAGE_STATE_ALLOCATED 1
699	/** A allocated page that's being monitored for writes.
700	* The shadow page table mappings are read-only. When a write occurs, the
701	* fWrittenTo member is set, the page remapped as read-write and the state
702	* moved back to allocated. */
703	#define PGM_PAGE_STATE_WRITE_MONITORED 2
704	/** The page is shared, aka. copy-on-write.
705	* This is a page that's shared with other VMs. */
706	#define PGM_PAGE_STATE_SHARED 3
707	/** @} */
708
709
710	/**
711	* Gets the page state.
712	* @returns page state (PGM_PAGE_STATE_*).
713	* @param pPage Pointer to the physical guest page tracking structure.
714	*/
715	#define PGM_PAGE_GET_STATE(pPage) ( (pPage)->u2StateX )
716
717	/**
718	* Sets the page state.
719	* @param pPage Pointer to the physical guest page tracking structure.
720	* @param _uState The new page state.
721	*/
722	#define PGM_PAGE_SET_STATE(pPage, _uState) \
723	do { (pPage)->u2StateX = (_uState); } while (0)
724
725
726	/**
727	* Gets the host physical address of the guest page.
728	* @returns host physical address (RTHCPHYS).
729	* @param pPage Pointer to the physical guest page tracking structure.
730	*/
731	#define PGM_PAGE_GET_HCPHYS(pPage) ( (pPage)->HCPhysX & UINT64_C(0x0000fffffffff000) )
732
733	/**
734	* Sets the host physical address of the guest page.
735	* @param pPage Pointer to the physical guest page tracking structure.
736	* @param _HCPhys The new host physical address.
737	*/
738	#define PGM_PAGE_SET_HCPHYS(pPage, _HCPhys) \
739	do { (pPage)->HCPhysX = (((pPage)->HCPhysX) & UINT64_C(0xffff000000000fff)) \
740	\| ((_HCPhys) & UINT64_C(0x0000fffffffff000)); } while (0)
741
742	/**
743	* Get the Page ID.
744	* @returns The Page ID; NIL_GMM_PAGEID if it's a ZERO page.
745	* @param pPage Pointer to the physical guest page tracking structure.
746	*/
747	#define PGM_PAGE_GET_PAGEID(pPage) ( (pPage)->idPageX )
748	/* later:
749	#define PGM_PAGE_GET_PAGEID(pPage) ( ((uint32_t)(pPage)->HCPhysX >> (48 - 12))
750	\| ((uint32_t)(pPage)->HCPhysX & 0xfff) )
751	*/
752	/**
753	* Sets the Page ID.
754	* @param pPage Pointer to the physical guest page tracking structure.
755	*/
756	#define PGM_PAGE_SET_PAGEID(pPage, _idPage) do { (pPage)->idPageX = (_idPage); } while (0)
757	/* later:
758	#define PGM_PAGE_SET_PAGEID(pPage, _idPage) do { (pPage)->HCPhysX = (((pPage)->HCPhysX) & UINT64_C(0x0000fffffffff000)) \
759	\| ((_idPage) & 0xfff) \
760	\| (((_idPage) & 0x0ffff000) << (48-12)); } while (0)
761	*/
762
763	/**
764	* Get the Chunk ID.
765	* @returns The Chunk ID; NIL_GMM_CHUNKID if it's a ZERO page.
766	* @param pPage Pointer to the physical guest page tracking structure.
767	*/
768	#define PGM_PAGE_GET_CHUNKID(pPage) ( (pPage)->idPageX >> GMM_CHUNKID_SHIFT )
769	/* later:
770	#if GMM_CHUNKID_SHIFT == 12
771	# define PGM_PAGE_GET_CHUNKID(pPage) ( (uint32_t)((pPage)->HCPhysX >> 48) )
772	#elif GMM_CHUNKID_SHIFT > 12
773	# define PGM_PAGE_GET_CHUNKID(pPage) ( (uint32_t)((pPage)->HCPhysX >> (48 + (GMM_CHUNKID_SHIFT - 12)) )
774	#elif GMM_CHUNKID_SHIFT < 12
775	# define PGM_PAGE_GET_CHUNKID(pPage) ( ( (uint32_t)((pPage)->HCPhysX >> 48) << (12 - GMM_CHUNKID_SHIFT) ) \
776	\| ( (uint32_t)((pPage)->HCPhysX & 0xfff) >> GMM_CHUNKID_SHIFT ) )
777	#else
778	# error "GMM_CHUNKID_SHIFT isn't defined or something."
779	#endif
780	*/
781
782	/**
783	* Get the index of the page within the allocaiton chunk.
784	* @returns The page index.
785	* @param pPage Pointer to the physical guest page tracking structure.
786	*/
787	#define PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) ( (pPage)->idPageX & GMM_PAGEID_IDX_MASK )
788	/* later:
789	#if GMM_CHUNKID_SHIFT <= 12
790	# define PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) ( (uint32_t)((pPage)->HCPhysX & GMM_PAGEID_IDX_MASK) )
791	#else
792	# define PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) ( (uint32_t)((pPage)->HCPhysX & 0xfff) \
793	\| ( (uint32_t)((pPage)->HCPhysX >> 48) & (RT_BIT_32(GMM_CHUNKID_SHIFT - 12) - 1) ) )
794	#endif
795	*/
796
797
798	/**
799	* Gets the page type.
800	* @returns The page type.
801	* @param pPage Pointer to the physical guest page tracking structure.
802	*/
803	#define PGM_PAGE_GET_TYPE(pPage) (pPage)->u3Type
804
805	/**
806	* Sets the page type.
807	* @param pPage Pointer to the physical guest page tracking structure.
808	* @param _enmType The new page type (PGMPAGETYPE).
809	*/
810	#define PGM_PAGE_SET_TYPE(pPage, _enmType) \
811	do { (pPage)->u3Type = (_enmType); } while (0)
812
813	/**
814	* Checks if the page is marked for MMIO.
815	* @returns true/false.
816	* @param pPage Pointer to the physical guest page tracking structure.
817	*/
818	#define PGM_PAGE_IS_MMIO(pPage) ( (pPage)->u3Type == PGMPAGETYPE_MMIO )
819
820	/**
821	* Checks if the page is backed by the ZERO page.
822	* @returns true/false.
823	* @param pPage Pointer to the physical guest page tracking structure.
824	*/
825	#define PGM_PAGE_IS_ZERO(pPage) ( (pPage)->u2StateX == PGM_PAGE_STATE_ZERO )
826
827	/**
828	* Checks if the page is backed by a SHARED page.
829	* @returns true/false.
830	* @param pPage Pointer to the physical guest page tracking structure.
831	*/
832	#define PGM_PAGE_IS_SHARED(pPage) ( (pPage)->u2StateX == PGM_PAGE_STATE_SHARED )
833
834
835	/**
836	* Marks the paget as written to (for GMM change monitoring).
837	* @param pPage Pointer to the physical guest page tracking structure.
838	*/
839	#define PGM_PAGE_SET_WRITTEN_TO(pPage) do { (pPage)->fWrittenToX = 1; } while (0)
840
841	/**
842	* Clears the written-to indicator.
843	* @param pPage Pointer to the physical guest page tracking structure.
844	*/
845	#define PGM_PAGE_CLEAR_WRITTEN_TO(pPage) do { (pPage)->fWrittenToX = 0; } while (0)
846
847	/**
848	* Checks if the page was marked as written-to.
849	* @returns true/false.
850	* @param pPage Pointer to the physical guest page tracking structure.
851	*/
852	#define PGM_PAGE_IS_WRITTEN_TO(pPage) ( (pPage)->fWrittenToX )
853
854
855	/** @name Physical Access Handler State values (PGMPAGE::u2HandlerPhysStateX).
856	*
857	* @remarks The values are assigned in order of priority, so we can calculate
858	* the correct state for a page with different handlers installed.
859	* @{ */
860	/** No handler installed. */
861	#define PGM_PAGE_HNDL_PHYS_STATE_NONE 0
862	/** Monitoring is temporarily disabled. */
863	#define PGM_PAGE_HNDL_PHYS_STATE_DISABLED 1
864	/** Write access is monitored. */
865	#define PGM_PAGE_HNDL_PHYS_STATE_WRITE 2
866	/** All access is monitored. */
867	#define PGM_PAGE_HNDL_PHYS_STATE_ALL 3
868	/** @} */
869
870	/**
871	* Gets the physical access handler state of a page.
872	* @returns PGM_PAGE_HNDL_PHYS_STATE_* value.
873	* @param pPage Pointer to the physical guest page tracking structure.
874	*/
875	#define PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) ( (pPage)->u2HandlerPhysStateX )
876
877	/**
878	* Sets the physical access handler state of a page.
879	* @param pPage Pointer to the physical guest page tracking structure.
880	* @param _uState The new state value.
881	*/
882	#define PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, _uState) \
883	do { (pPage)->u2HandlerPhysStateX = (_uState); } while (0)
884
885	/**
886	* Checks if the page has any physical access handlers, including temporariliy disabled ones.
887	* @returns true/false
888	* @param pPage Pointer to the physical guest page tracking structure.
889	*/
890	#define PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage) ( (pPage)->u2HandlerPhysStateX != PGM_PAGE_HNDL_PHYS_STATE_NONE )
891
892	/**
893	* Checks if the page has any active physical access handlers.
894	* @returns true/false
895	* @param pPage Pointer to the physical guest page tracking structure.
896	*/
897	#define PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage) ( (pPage)->u2HandlerPhysStateX >= PGM_PAGE_HNDL_PHYS_STATE_WRITE )
898
899
900	/** @name Virtual Access Handler State values (PGMPAGE::u2HandlerVirtStateX).
901	*
902	* @remarks The values are assigned in order of priority, so we can calculate
903	* the correct state for a page with different handlers installed.
904	* @{ */
905	/** No handler installed. */
906	#define PGM_PAGE_HNDL_VIRT_STATE_NONE 0
907	/* 1 is reserved so the lineup is identical with the physical ones. */
908	/** Write access is monitored. */
909	#define PGM_PAGE_HNDL_VIRT_STATE_WRITE 2
910	/** All access is monitored. */
911	#define PGM_PAGE_HNDL_VIRT_STATE_ALL 3
912	/** @} */
913
914	/**
915	* Gets the virtual access handler state of a page.
916	* @returns PGM_PAGE_HNDL_VIRT_STATE_* value.
917	* @param pPage Pointer to the physical guest page tracking structure.
918	*/
919	#define PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) ( (pPage)->u2HandlerVirtStateX )
920
921	/**
922	* Sets the virtual access handler state of a page.
923	* @param pPage Pointer to the physical guest page tracking structure.
924	* @param _uState The new state value.
925	*/
926	#define PGM_PAGE_SET_HNDL_VIRT_STATE(pPage, _uState) \
927	do { (pPage)->u2HandlerVirtStateX = (_uState); } while (0)
928
929	/**
930	* Checks if the page has any virtual access handlers.
931	* @returns true/false
932	* @param pPage Pointer to the physical guest page tracking structure.
933	*/
934	#define PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS(pPage) ( (pPage)->u2HandlerVirtStateX != PGM_PAGE_HNDL_VIRT_STATE_NONE )
935
936	/**
937	* Same as PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS - can't disable pages in
938	* virtual handlers.
939	* @returns true/false
940	* @param pPage Pointer to the physical guest page tracking structure.
941	*/
942	#define PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(pPage) PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS(pPage)
943
944
945
946	/**
947	* Checks if the page has any access handlers, including temporarily disabled ones.
948	* @returns true/false
949	* @param pPage Pointer to the physical guest page tracking structure.
950	*/
951	#define PGM_PAGE_HAS_ANY_HANDLERS(pPage) \
952	( (pPage)->u2HandlerPhysStateX != PGM_PAGE_HNDL_PHYS_STATE_NONE \
953	\|\| (pPage)->u2HandlerVirtStateX != PGM_PAGE_HNDL_VIRT_STATE_NONE )
954
955	/**
956	* Checks if the page has any active access handlers.
957	* @returns true/false
958	* @param pPage Pointer to the physical guest page tracking structure.
959	*/
960	#define PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) \
961	( (pPage)->u2HandlerPhysStateX >= PGM_PAGE_HNDL_PHYS_STATE_WRITE \
962	\|\| (pPage)->u2HandlerVirtStateX >= PGM_PAGE_HNDL_VIRT_STATE_WRITE )
963
964	/**
965	* Checks if the page has any active access handlers catching all accesses.
966	* @returns true/false
967	* @param pPage Pointer to the physical guest page tracking structure.
968	*/
969	#define PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage) \
970	( (pPage)->u2HandlerPhysStateX == PGM_PAGE_HNDL_PHYS_STATE_ALL \
971	\|\| (pPage)->u2HandlerVirtStateX == PGM_PAGE_HNDL_VIRT_STATE_ALL )
972
973
974
975
976	/** @def PGM_PAGE_GET_TRACKING
977	* Gets the packed shadow page pool tracking data associated with a guest page.
978	* @returns uint16_t containing the data.
979	* @param pPage Pointer to the physical guest page tracking structure.
980	*/
981	#define PGM_PAGE_GET_TRACKING(pPage) \
982	( ((uint16_t )&(pPage)->HCPhysX + 3) )
983
984	/** @def PGM_PAGE_SET_TRACKING
985	* Sets the packed shadow page pool tracking data associated with a guest page.
986	* @param pPage Pointer to the physical guest page tracking structure.
987	* @param u16TrackingData The tracking data to store.
988	*/
989	#define PGM_PAGE_SET_TRACKING(pPage, u16TrackingData) \
990	do { ((uint16_t )&(pPage)->HCPhysX + 3) = (u16TrackingData); } while (0)
991
992	/** @def PGM_PAGE_GET_TD_CREFS
993	* Gets the @a cRefs tracking data member.
994	* @returns cRefs.
995	* @param pPage Pointer to the physical guest page tracking structure.
996	*/
997	#define PGM_PAGE_GET_TD_CREFS(pPage) \
998	((PGM_PAGE_GET_TRACKING(pPage) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK)
999
1000	#define PGM_PAGE_GET_TD_IDX(pPage) \
1001	((PGM_PAGE_GET_TRACKING(pPage) >> PGMPOOL_TD_IDX_SHIFT) & PGMPOOL_TD_IDX_MASK)
1002
1003	/**
1004	* Ram range for GC Phys to HC Phys conversion.
1005	*
1006	* Can be used for HC Virt to GC Phys and HC Virt to HC Phys
1007	* conversions too, but we'll let MM handle that for now.
1008	*
1009	* This structure is used by linked lists in both GC and HC.
1010	*/
1011	typedef struct PGMRAMRANGE
1012	{
1013	/** Start of the range. Page aligned. */
1014	RTGCPHYS GCPhys;
1015	/** Size of the range. (Page aligned of course). */
1016	RTGCPHYS cb;
1017	/** Pointer to the next RAM range - for R3. */
1018	R3PTRTYPE(struct PGMRAMRANGE *) pNextR3;
1019	/** Pointer to the next RAM range - for R0. */
1020	R0PTRTYPE(struct PGMRAMRANGE *) pNextR0;
1021	/** Pointer to the next RAM range - for RC. */
1022	RCPTRTYPE(struct PGMRAMRANGE *) pNextRC;
1023	/** PGM_RAM_RANGE_FLAGS_* flags. */
1024	uint32_t fFlags;
1025	/** Last address in the range (inclusive). Page aligned (-1). */
1026	RTGCPHYS GCPhysLast;
1027	/** Start of the HC mapping of the range. This is only used for MMIO2. */
1028	R3PTRTYPE(void *) pvR3;
1029	/** The range description. */
1030	R3PTRTYPE(const char *) pszDesc;
1031	/** Pointer to self - R0 pointer. */
1032	R0PTRTYPE(struct PGMRAMRANGE *) pSelfR0;
1033	/** Pointer to self - RC pointer. */
1034	RCPTRTYPE(struct PGMRAMRANGE *) pSelfRC;
1035	/** Padding to make aPage aligned on sizeof(PGMPAGE). */
1036	uint32_t au32Alignment2[HC_ARCH_BITS == 32 ? 2 : 1];
1037	/** Array of physical guest page tracking structures. */
1038	PGMPAGE aPages[1];
1039	} PGMRAMRANGE;
1040	/** Pointer to Ram range for GC Phys to HC Phys conversion. */
1041	typedef PGMRAMRANGE *PPGMRAMRANGE;
1042
1043	/** @name PGMRAMRANGE::fFlags
1044	* @{ */
1045	/** The RAM range is floating around as an independent guest mapping. */
1046	#define PGM_RAM_RANGE_FLAGS_FLOATING RT_BIT(20)
1047	/** @} */
1048
1049
1050	/**
1051	* Per page tracking structure for ROM image.
1052	*
1053	* A ROM image may have a shadow page, in which case we may have
1054	* two pages backing it. This structure contains the PGMPAGE for
1055	* both while PGMRAMRANGE have a copy of the active one. It is
1056	* important that these aren't out of sync in any regard other
1057	* than page pool tracking data.
1058	*/
1059	typedef struct PGMROMPAGE
1060	{
1061	/** The page structure for the virgin ROM page. */
1062	PGMPAGE Virgin;
1063	/** The page structure for the shadow RAM page. */
1064	PGMPAGE Shadow;
1065	/** The current protection setting. */
1066	PGMROMPROT enmProt;
1067	/** Pad the structure size to a multiple of 8. */
1068	uint32_t u32Padding;
1069	} PGMROMPAGE;
1070	/** Pointer to a ROM page tracking structure. */
1071	typedef PGMROMPAGE *PPGMROMPAGE;
1072
1073
1074	/**
1075	* A registered ROM image.
1076	*
1077	* This is needed to keep track of ROM image since they generally
1078	* intrude into a PGMRAMRANGE. It also keeps track of additional
1079	* info like the two page sets (read-only virgin and read-write shadow),
1080	* the current state of each page.
1081	*
1082	* Because access handlers cannot easily be executed in a different
1083	* context, the ROM ranges needs to be accessible and in all contexts.
1084	*/
1085	typedef struct PGMROMRANGE
1086	{
1087	/** Pointer to the next range - R3. */
1088	R3PTRTYPE(struct PGMROMRANGE *) pNextR3;
1089	/** Pointer to the next range - R0. */
1090	R0PTRTYPE(struct PGMROMRANGE *) pNextR0;
1091	/** Pointer to the next range - RC. */
1092	RCPTRTYPE(struct PGMROMRANGE *) pNextRC;
1093	/** Pointer alignment */
1094	RTRCPTR GCPtrAlignment;
1095	/** Address of the range. */
1096	RTGCPHYS GCPhys;
1097	/** Address of the last byte in the range. */
1098	RTGCPHYS GCPhysLast;
1099	/** Size of the range. */
1100	RTGCPHYS cb;
1101	/** The flags (PGMPHYS_ROM_FLAG_). /
1102	uint32_t fFlags;
1103	/** Alignment padding ensuring that aPages is sizeof(PGMROMPAGE) aligned. */
1104	uint32_t au32Alignemnt[HC_ARCH_BITS == 32 ? 7 : 3];
1105	/** Pointer to the original bits when PGMPHYS_ROM_FLAGS_PERMANENT_BINARY was specified.
1106	* This is used for strictness checks. */
1107	R3PTRTYPE(const void *) pvOriginal;
1108	/** The ROM description. */
1109	R3PTRTYPE(const char *) pszDesc;
1110	/** The per page tracking structures. */
1111	PGMROMPAGE aPages[1];
1112	} PGMROMRANGE;
1113	/** Pointer to a ROM range. */
1114	typedef PGMROMRANGE *PPGMROMRANGE;
1115
1116
1117	/**
1118	* A registered MMIO2 (= Device RAM) range.
1119	*
1120	* There are a few reason why we need to keep track of these
1121	* registrations. One of them is the deregistration & cleanup
1122	* stuff, while another is that the PGMRAMRANGE associated with
1123	* such a region may have to be removed from the ram range list.
1124	*
1125	* Overlapping with a RAM range has to be 100% or none at all. The
1126	* pages in the existing RAM range must not be ROM nor MMIO. A guru
1127	* meditation will be raised if a partial overlap or an overlap of
1128	* ROM pages is encountered. On an overlap we will free all the
1129	* existing RAM pages and put in the ram range pages instead.
1130	*/
1131	typedef struct PGMMMIO2RANGE
1132	{
1133	/** The owner of the range. (a device) */
1134	PPDMDEVINSR3 pDevInsR3;
1135	/** Pointer to the ring-3 mapping of the allocation. */
1136	RTR3PTR pvR3;
1137	/** Pointer to the next range - R3. */
1138	R3PTRTYPE(struct PGMMMIO2RANGE *) pNextR3;
1139	/** Whether it's mapped or not. */
1140	bool fMapped;
1141	/** Whether it's overlapping or not. */
1142	bool fOverlapping;
1143	/** The PCI region number.
1144	* @remarks This ASSUMES that nobody will ever really need to have multiple
1145	* PCI devices with matching MMIO region numbers on a single device. */
1146	uint8_t iRegion;
1147	/** Alignment padding for putting the ram range on a PGMPAGE alignment boundrary. */
1148	uint8_t abAlignemnt[HC_ARCH_BITS == 32 ? 1 : 5];
1149	/** The associated RAM range. */
1150	PGMRAMRANGE RamRange;
1151	} PGMMMIO2RANGE;
1152	/** Pointer to a MMIO2 range. */
1153	typedef PGMMMIO2RANGE *PPGMMMIO2RANGE;
1154
1155
1156
1157
1158	/**
1159	* PGMPhysRead/Write cache entry
1160	*/
1161	typedef struct PGMPHYSCACHEENTRY
1162	{
1163	/** R3 pointer to physical page. */
1164	R3PTRTYPE(uint8_t *) pbR3;
1165	/** GC Physical address for cache entry */
1166	RTGCPHYS GCPhys;
1167	#if HC_ARCH_BITS == 64 && GC_ARCH_BITS == 32
1168	RTGCPHYS u32Padding0; /*< alignment padding. /
1169	#endif
1170	} PGMPHYSCACHEENTRY;
1171
1172	/**
1173	* PGMPhysRead/Write cache to reduce REM memory access overhead
1174	*/
1175	typedef struct PGMPHYSCACHE
1176	{
1177	/** Bitmap of valid cache entries */
1178	uint64_t aEntries;
1179	/** Cache entries */
1180	PGMPHYSCACHEENTRY Entry[PGM_MAX_PHYSCACHE_ENTRIES];
1181	} PGMPHYSCACHE;
1182
1183
1184	/** Pointer to an allocation chunk ring-3 mapping. */
1185	typedef struct PGMCHUNKR3MAP *PPGMCHUNKR3MAP;
1186	/** Pointer to an allocation chunk ring-3 mapping pointer. */
1187	typedef PPGMCHUNKR3MAP *PPPGMCHUNKR3MAP;
1188
1189	/**
1190	* Ring-3 tracking structore for an allocation chunk ring-3 mapping.
1191	*
1192	* The primary tree (Core) uses the chunk id as key.
1193	* The secondary tree (AgeCore) is used for ageing and uses ageing sequence number as key.
1194	*/
1195	typedef struct PGMCHUNKR3MAP
1196	{
1197	/** The key is the chunk id. */
1198	AVLU32NODECORE Core;
1199	/** The key is the ageing sequence number. */
1200	AVLLU32NODECORE AgeCore;
1201	/** The current age thingy. */
1202	uint32_t iAge;
1203	/** The current reference count. */
1204	uint32_t volatile cRefs;
1205	/** The current permanent reference count. */
1206	uint32_t volatile cPermRefs;
1207	/** The mapping address. */
1208	void *pv;
1209	} PGMCHUNKR3MAP;
1210
1211	/**
1212	* Allocation chunk ring-3 mapping TLB entry.
1213	*/
1214	typedef struct PGMCHUNKR3MAPTLBE
1215	{
1216	/** The chunk id. */
1217	uint32_t volatile idChunk;
1218	#if HC_ARCH_BITS == 64
1219	uint32_t u32Padding; /*< alignment padding. /
1220	#endif
1221	/** The chunk map. */
1222	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1223	R3PTRTYPE(PPGMCHUNKR3MAP) volatile pChunk;
1224	#else
1225	R3R0PTRTYPE(PPGMCHUNKR3MAP) volatile pChunk;
1226	#endif
1227	} PGMCHUNKR3MAPTLBE;
1228	/** Pointer to the an allocation chunk ring-3 mapping TLB entry. */
1229	typedef PGMCHUNKR3MAPTLBE *PPGMCHUNKR3MAPTLBE;
1230
1231	/** The number of TLB entries in PGMCHUNKR3MAPTLB.
1232	* @remark Must be a power of two value. */
1233	#define PGM_CHUNKR3MAPTLB_ENTRIES 32
1234
1235	/**
1236	* Allocation chunk ring-3 mapping TLB.
1237	*
1238	* @remarks We use a TLB to speed up lookups by avoiding walking the AVL.
1239	* At first glance this might look kinda odd since AVL trees are
1240	* supposed to give the most optimial lookup times of all trees
1241	* due to their balancing. However, take a tree with 1023 nodes
1242	* in it, that's 10 levels, meaning that most searches has to go
1243	* down 9 levels before they find what they want. This isn't fast
1244	* compared to a TLB hit. There is the factor of cache misses,
1245	* and of course the problem with trees and branch prediction.
1246	* This is why we use TLBs in front of most of the trees.
1247	*
1248	* @todo Generalize this TLB + AVL stuff, shouldn't be all that
1249	* difficult when we switch to the new inlined AVL trees (from kStuff).
1250	*/
1251	typedef struct PGMCHUNKR3MAPTLB
1252	{
1253	/** The TLB entries. */
1254	PGMCHUNKR3MAPTLBE aEntries[PGM_CHUNKR3MAPTLB_ENTRIES];
1255	} PGMCHUNKR3MAPTLB;
1256
1257	/**
1258	* Calculates the index of a guest page in the Ring-3 Chunk TLB.
1259	* @returns Chunk TLB index.
1260	* @param idChunk The Chunk ID.
1261	*/
1262	#define PGM_CHUNKR3MAPTLB_IDX(idChunk) ( (idChunk) & (PGM_CHUNKR3MAPTLB_ENTRIES - 1) )
1263
1264
1265	/**
1266	* Ring-3 guest page mapping TLB entry.
1267	* @remarks used in ring-0 as well at the moment.
1268	*/
1269	typedef struct PGMPAGER3MAPTLBE
1270	{
1271	/** Address of the page. */
1272	RTGCPHYS volatile GCPhys;
1273	/** The guest page. */
1274	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1275	R3PTRTYPE(PPGMPAGE) volatile pPage;
1276	#else
1277	R3R0PTRTYPE(PPGMPAGE) volatile pPage;
1278	#endif
1279	/** Pointer to the page mapping tracking structure, PGMCHUNKR3MAP. */
1280	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1281	R3PTRTYPE(PPGMCHUNKR3MAP) volatile pMap;
1282	#else
1283	R3R0PTRTYPE(PPGMCHUNKR3MAP) volatile pMap;
1284	#endif
1285	/** The address */
1286	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1287	R3PTRTYPE(void *) volatile pv;
1288	#else
1289	R3R0PTRTYPE(void *) volatile pv;
1290	#endif
1291	#if HC_ARCH_BITS == 32
1292	uint32_t u32Padding; /*< alignment padding. /
1293	#endif
1294	} PGMPAGER3MAPTLBE;
1295	/** Pointer to an entry in the HC physical TLB. */
1296	typedef PGMPAGER3MAPTLBE *PPGMPAGER3MAPTLBE;
1297
1298
1299	/** The number of entries in the ring-3 guest page mapping TLB.
1300	* @remarks The value must be a power of two. */
1301	#define PGM_PAGER3MAPTLB_ENTRIES 64
1302
1303	/**
1304	* Ring-3 guest page mapping TLB.
1305	* @remarks used in ring-0 as well at the moment.
1306	*/
1307	typedef struct PGMPAGER3MAPTLB
1308	{
1309	/** The TLB entries. */
1310	PGMPAGER3MAPTLBE aEntries[PGM_PAGER3MAPTLB_ENTRIES];
1311	} PGMPAGER3MAPTLB;
1312	/** Pointer to the ring-3 guest page mapping TLB. */
1313	typedef PGMPAGER3MAPTLB *PPGMPAGER3MAPTLB;
1314
1315	/**
1316	* Calculates the index of the TLB entry for the specified guest page.
1317	* @returns Physical TLB index.
1318	* @param GCPhys The guest physical address.
1319	*/
1320	#define PGM_PAGER3MAPTLB_IDX(GCPhys) ( ((GCPhys) >> PAGE_SHIFT) & (PGM_PAGER3MAPTLB_ENTRIES - 1) )
1321
1322
1323	/**
1324	* Mapping cache usage set entry.
1325	*
1326	* @remarks 16-bit ints was choosen as the set is not expected to be used beyond
1327	* the dynamic ring-0 and (to some extent) raw-mode context mapping
1328	* cache. If it's extended to include ring-3, well, then something will
1329	* have be changed here...
1330	*/
1331	typedef struct PGMMAPSETENTRY
1332	{
1333	/** The mapping cache index. */
1334	uint16_t iPage;
1335	/** The number of references.
1336	* The max is UINT16_MAX - 1. */
1337	uint16_t cRefs;
1338	#if HC_ARCH_BITS == 64
1339	uint32_t alignment;
1340	#endif
1341	/** Pointer to the page. */
1342	RTR0PTR pvPage;
1343	/** The physical address for this entry. */
1344	RTHCPHYS HCPhys;
1345	} PGMMAPSETENTRY;
1346	/** Pointer to a mapping cache usage set entry. */
1347	typedef PGMMAPSETENTRY *PPGMMAPSETENTRY;
1348
1349	/**
1350	* Mapping cache usage set.
1351	*
1352	* This is used in ring-0 and the raw-mode context to track dynamic mappings
1353	* done during exits / traps. The set is
1354	*/
1355	typedef struct PGMMAPSET
1356	{
1357	/** The number of occupied entries.
1358	* This is PGMMAPSET_CLOSED if the set is closed and we're not supposed to do
1359	* dynamic mappings. */
1360	uint32_t cEntries;
1361	/** The start of the current subset.
1362	* This is UINT32_MAX if no subset is currently open. */
1363	uint32_t iSubset;
1364	/** The index of the current CPU, only valid if the set is open. */
1365	int32_t iCpu;
1366	#if HC_ARCH_BITS == 64
1367	uint32_t alignment;
1368	#endif
1369	/** The entries. */
1370	PGMMAPSETENTRY aEntries[64];
1371	/** HCPhys -> iEntry fast lookup table.
1372	* Use PGMMAPSET_HASH for hashing.
1373	* The entries may or may not be valid, check against cEntries. */
1374	uint8_t aiHashTable[128];
1375	} PGMMAPSET;
1376	/** Pointer to the mapping cache set. */
1377	typedef PGMMAPSET *PPGMMAPSET;
1378
1379	/** PGMMAPSET::cEntries value for a closed set. */
1380	#define PGMMAPSET_CLOSED UINT32_C(0xdeadc0fe)
1381
1382	/** Hash function for aiHashTable. */
1383	#define PGMMAPSET_HASH(HCPhys) (((HCPhys) >> PAGE_SHIFT) & 127)
1384
1385	/** The max fill size (strict builds). */
1386	#define PGMMAPSET_MAX_FILL (64U * 80U / 100U)
1387
1388
1389	/** @name Context neutrual page mapper TLB.
1390	*
1391	* Hoping to avoid some code and bug duplication parts of the GCxxx->CCPtr
1392	* code is writting in a kind of context neutrual way. Time will show whether
1393	* this actually makes sense or not...
1394	*
1395	* @todo this needs to be reconsidered and dropped/redone since the ring-0
1396	* context ends up using a global mapping cache on some platforms
1397	* (darwin).
1398	*
1399	* @{ */
1400	/** @typedef PPGMPAGEMAPTLB
1401	* The page mapper TLB pointer type for the current context. */
1402	/** @typedef PPGMPAGEMAPTLB
1403	* The page mapper TLB entry pointer type for the current context. */
1404	/** @typedef PPGMPAGEMAPTLB
1405	* The page mapper TLB entry pointer pointer type for the current context. */
1406	/** @def PGM_PAGEMAPTLB_ENTRIES
1407	* The number of TLB entries in the page mapper TLB for the current context. */
1408	/** @def PGM_PAGEMAPTLB_IDX
1409	* Calculate the TLB index for a guest physical address.
1410	* @returns The TLB index.
1411	* @param GCPhys The guest physical address. */
1412	/** @typedef PPGMPAGEMAP
1413	* Pointer to a page mapper unit for current context. */
1414	/** @typedef PPPGMPAGEMAP
1415	* Pointer to a page mapper unit pointer for current context. */
1416	#ifdef IN_RC
1417	// typedef PPGMPAGEGCMAPTLB PPGMPAGEMAPTLB;
1418	// typedef PPGMPAGEGCMAPTLBE PPGMPAGEMAPTLBE;
1419	// typedef PPGMPAGEGCMAPTLBE *PPPGMPAGEMAPTLBE;
1420	# define PGM_PAGEMAPTLB_ENTRIES PGM_PAGEGCMAPTLB_ENTRIES
1421	# define PGM_PAGEMAPTLB_IDX(GCPhys) PGM_PAGEGCMAPTLB_IDX(GCPhys)
1422	typedef void * PPGMPAGEMAP;
1423	typedef void ** PPPGMPAGEMAP;
1424	//#elif IN_RING0
1425	// typedef PPGMPAGER0MAPTLB PPGMPAGEMAPTLB;
1426	// typedef PPGMPAGER0MAPTLBE PPGMPAGEMAPTLBE;
1427	// typedef PPGMPAGER0MAPTLBE *PPPGMPAGEMAPTLBE;
1428	//# define PGM_PAGEMAPTLB_ENTRIES PGM_PAGER0MAPTLB_ENTRIES
1429	//# define PGM_PAGEMAPTLB_IDX(GCPhys) PGM_PAGER0MAPTLB_IDX(GCPhys)
1430	// typedef PPGMCHUNKR0MAP PPGMPAGEMAP;
1431	// typedef PPPGMCHUNKR0MAP PPPGMPAGEMAP;
1432	#else
1433	typedef PPGMPAGER3MAPTLB PPGMPAGEMAPTLB;
1434	typedef PPGMPAGER3MAPTLBE PPGMPAGEMAPTLBE;
1435	typedef PPGMPAGER3MAPTLBE *PPPGMPAGEMAPTLBE;
1436	# define PGM_PAGEMAPTLB_ENTRIES PGM_PAGER3MAPTLB_ENTRIES
1437	# define PGM_PAGEMAPTLB_IDX(GCPhys) PGM_PAGER3MAPTLB_IDX(GCPhys)
1438	typedef PPGMCHUNKR3MAP PPGMPAGEMAP;
1439	typedef PPPGMCHUNKR3MAP PPPGMPAGEMAP;
1440	#endif
1441	/** @} */
1442
1443
1444	/** @name PGM Pool Indexes.
1445	* Aka. the unique shadow page identifier.
1446	* @{ */
1447	/** NIL page pool IDX. */
1448	#define NIL_PGMPOOL_IDX 0
1449	/** The first normal index. */
1450	#define PGMPOOL_IDX_FIRST_SPECIAL 1
1451	/** Page directory (32-bit root). */
1452	#define PGMPOOL_IDX_PD 1
1453	/** Page Directory Pointer Table (PAE root). */
1454	#define PGMPOOL_IDX_PDPT 2
1455	/** AMD64 CR3 level index.*/
1456	#define PGMPOOL_IDX_AMD64_CR3 3
1457	/** Nested paging root.*/
1458	#define PGMPOOL_IDX_NESTED_ROOT 4
1459	/** The first normal index. */
1460	#define PGMPOOL_IDX_FIRST 5
1461	/** The last valid index. (inclusive, 14 bits) */
1462	#define PGMPOOL_IDX_LAST 0x3fff
1463	/** @} */
1464
1465	/** The NIL index for the parent chain. */
1466	#define NIL_PGMPOOL_USER_INDEX ((uint16_t)0xffff)
1467
1468	/**
1469	* Node in the chain linking a shadowed page to it's parent (user).
1470	*/
1471	#pragma pack(1)
1472	typedef struct PGMPOOLUSER
1473	{
1474	/** The index to the next item in the chain. NIL_PGMPOOL_USER_INDEX is no next. */
1475	uint16_t iNext;
1476	/** The user page index. */
1477	uint16_t iUser;
1478	/** Index into the user table. */
1479	uint32_t iUserTable;
1480	} PGMPOOLUSER, *PPGMPOOLUSER;
1481	typedef const PGMPOOLUSER *PCPGMPOOLUSER;
1482	#pragma pack()
1483
1484
1485	/** The NIL index for the phys ext chain. */
1486	#define NIL_PGMPOOL_PHYSEXT_INDEX ((uint16_t)0xffff)
1487
1488	/**
1489	* Node in the chain of physical cross reference extents.
1490	* @todo Calling this an 'extent' is not quite right, find a better name.
1491	*/
1492	#pragma pack(1)
1493	typedef struct PGMPOOLPHYSEXT
1494	{
1495	/** The index to the next item in the chain. NIL_PGMPOOL_PHYSEXT_INDEX is no next. */
1496	uint16_t iNext;
1497	/** The user page index. */
1498	uint16_t aidx[3];
1499	} PGMPOOLPHYSEXT, *PPGMPOOLPHYSEXT;
1500	typedef const PGMPOOLPHYSEXT *PCPGMPOOLPHYSEXT;
1501	#pragma pack()
1502
1503
1504	/**
1505	* The kind of page that's being shadowed.
1506	*/
1507	typedef enum PGMPOOLKIND
1508	{
1509	/** The virtual invalid 0 entry. */
1510	PGMPOOLKIND_INVALID = 0,
1511	/** The entry is free (=unused). */
1512	PGMPOOLKIND_FREE,
1513
1514	/** Shw: 32-bit page table; Gst: no paging */
1515	PGMPOOLKIND_32BIT_PT_FOR_PHYS,
1516	/** Shw: 32-bit page table; Gst: 32-bit page table. */
1517	PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT,
1518	/** Shw: 32-bit page table; Gst: 4MB page. */
1519	PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB,
1520	/** Shw: PAE page table; Gst: no paging */
1521	PGMPOOLKIND_PAE_PT_FOR_PHYS,
1522	/** Shw: PAE page table; Gst: 32-bit page table. */
1523	PGMPOOLKIND_PAE_PT_FOR_32BIT_PT,
1524	/** Shw: PAE page table; Gst: Half of a 4MB page. */
1525	PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB,
1526	/** Shw: PAE page table; Gst: PAE page table. */
1527	PGMPOOLKIND_PAE_PT_FOR_PAE_PT,
1528	/** Shw: PAE page table; Gst: 2MB page. */
1529	PGMPOOLKIND_PAE_PT_FOR_PAE_2MB,
1530
1531	/** Shw: 32-bit page directory. Gst: 32-bit page directory. */
1532	PGMPOOLKIND_32BIT_PD,
1533	/** Shw: 32-bit page directory. Gst: no paging. */
1534	PGMPOOLKIND_32BIT_PD_PHYS,
1535	/** Shw: PAE page directory 0; Gst: 32-bit page directory. */
1536	PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD,
1537	/** Shw: PAE page directory 1; Gst: 32-bit page directory. */
1538	PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD,
1539	/** Shw: PAE page directory 2; Gst: 32-bit page directory. */
1540	PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD,
1541	/** Shw: PAE page directory 3; Gst: 32-bit page directory. */
1542	PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD,
1543	/** Shw: PAE page directory; Gst: PAE page directory. */
1544	PGMPOOLKIND_PAE_PD_FOR_PAE_PD,
1545	/** Shw: PAE page directory; Gst: no paging. */
1546	PGMPOOLKIND_PAE_PD_PHYS,
1547
1548	/** Shw: PAE page directory pointer table (legacy, 4 entries); Gst 32 bits paging. */
1549	PGMPOOLKIND_PAE_PDPT_FOR_32BIT,
1550	/** Shw: PAE page directory pointer table (legacy, 4 entries); Gst PAE PDPT. */
1551	PGMPOOLKIND_PAE_PDPT,
1552	/** Shw: PAE page directory pointer table (legacy, 4 entries); Gst: no paging. */
1553	PGMPOOLKIND_PAE_PDPT_PHYS,
1554
1555	/** Shw: 64-bit page directory pointer table; Gst: 64-bit page directory pointer table. */
1556	PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT,
1557	/** Shw: 64-bit page directory pointer table; Gst: no paging */
1558	PGMPOOLKIND_64BIT_PDPT_FOR_PHYS,
1559	/** Shw: 64-bit page directory table; Gst: 64-bit page directory table. */
1560	PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD,
1561	/** Shw: 64-bit page directory table; Gst: no paging */
1562	PGMPOOLKIND_64BIT_PD_FOR_PHYS, /* 22 */
1563
1564	/** Shw: 64-bit PML4; Gst: 64-bit PML4. */
1565	PGMPOOLKIND_64BIT_PML4,
1566
1567	/** Shw: EPT page directory pointer table; Gst: no paging */
1568	PGMPOOLKIND_EPT_PDPT_FOR_PHYS,
1569	/** Shw: EPT page directory table; Gst: no paging */
1570	PGMPOOLKIND_EPT_PD_FOR_PHYS,
1571	/** Shw: EPT page table; Gst: no paging */
1572	PGMPOOLKIND_EPT_PT_FOR_PHYS,
1573
1574	/** Shw: Root Nested paging table. */
1575	PGMPOOLKIND_ROOT_NESTED,
1576
1577	/** The last valid entry. */
1578	PGMPOOLKIND_LAST = PGMPOOLKIND_ROOT_NESTED
1579	} PGMPOOLKIND;
1580
1581
1582	/**
1583	* The tracking data for a page in the pool.
1584	*/
1585	typedef struct PGMPOOLPAGE
1586	{
1587	/** AVL node code with the (R3) physical address of this page. */
1588	AVLOHCPHYSNODECORE Core;
1589	/** Pointer to the R3 mapping of the page. */
1590	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1591	R3PTRTYPE(void *) pvPageR3;
1592	#else
1593	R3R0PTRTYPE(void *) pvPageR3;
1594	#endif
1595	/** The guest physical address. */
1596	#if HC_ARCH_BITS == 32 && GC_ARCH_BITS == 64
1597	uint32_t Alignment0;
1598	#endif
1599	RTGCPHYS GCPhys;
1600	/** The kind of page we're shadowing. (This is really a PGMPOOLKIND enum.) */
1601	uint8_t enmKind;
1602	uint8_t bPadding;
1603	/** The index of this page. */
1604	uint16_t idx;
1605	/** The next entry in the list this page currently resides in.
1606	* It's either in the free list or in the GCPhys hash. */
1607	uint16_t iNext;
1608	#ifdef PGMPOOL_WITH_USER_TRACKING
1609	/** Head of the user chain. NIL_PGMPOOL_USER_INDEX if not currently in use. */
1610	uint16_t iUserHead;
1611	/** The number of present entries. */
1612	uint16_t cPresent;
1613	/** The first entry in the table which is present. */
1614	uint16_t iFirstPresent;
1615	#endif
1616	#ifdef PGMPOOL_WITH_MONITORING
1617	/** The number of modifications to the monitored page. */
1618	uint16_t cModifications;
1619	/** The next modified page. NIL_PGMPOOL_IDX if tail. */
1620	uint16_t iModifiedNext;
1621	/** The previous modified page. NIL_PGMPOOL_IDX if head. */
1622	uint16_t iModifiedPrev;
1623	/** The next page sharing access handler. NIL_PGMPOOL_IDX if tail. */
1624	uint16_t iMonitoredNext;
1625	/** The previous page sharing access handler. NIL_PGMPOOL_IDX if head. */
1626	uint16_t iMonitoredPrev;
1627	#endif
1628	#ifdef PGMPOOL_WITH_CACHE
1629	/** The next page in the age list. */
1630	uint16_t iAgeNext;
1631	/** The previous page in the age list. */
1632	uint16_t iAgePrev;
1633	#endif /* PGMPOOL_WITH_CACHE */
1634	/** Used to indicate that the page is zeroed. */
1635	bool fZeroed;
1636	/** Used to indicate that a PT has non-global entries. */
1637	bool fSeenNonGlobal;
1638	/** Used to indicate that we're monitoring writes to the guest page. */
1639	bool fMonitored;
1640	/** Used to indicate that the page is in the cache (e.g. in the GCPhys hash).
1641	* (All pages are in the age list.) */
1642	bool fCached;
1643	/** This is used by the R3 access handlers when invoked by an async thread.
1644	* It's a hack required because of REMR3NotifyHandlerPhysicalDeregister. */
1645	bool volatile fReusedFlushPending;
1646	/** Used to indicate that this page can't be flushed. Important for cr3 root pages or shadow pae pd pages). */
1647	uint8_t cLocked;
1648	} PGMPOOLPAGE, PPGMPOOLPAGE, *PPPGMPOOLPAGE;
1649	/** Pointer to a const pool page. */
1650	typedef PGMPOOLPAGE const *PCPGMPOOLPAGE;
1651
1652
1653	#ifdef PGMPOOL_WITH_CACHE
1654	/** The hash table size. */
1655	# define PGMPOOL_HASH_SIZE 0x40
1656	/** The hash function. */
1657	# define PGMPOOL_HASH(GCPhys) ( ((GCPhys) >> PAGE_SHIFT) & (PGMPOOL_HASH_SIZE - 1) )
1658	#endif
1659
1660
1661	/**
1662	* The shadow page pool instance data.
1663	*
1664	* It's all one big allocation made at init time, except for the
1665	* pages that is. The user nodes follows immediatly after the
1666	* page structures.
1667	*/
1668	typedef struct PGMPOOL
1669	{
1670	/** The VM handle - R3 Ptr. */
1671	PVMR3 pVMR3;
1672	/** The VM handle - R0 Ptr. */
1673	PVMR0 pVMR0;
1674	/** The VM handle - RC Ptr. */
1675	PVMRC pVMRC;
1676	/** The max pool size. This includes the special IDs. */
1677	uint16_t cMaxPages;
1678	/** The current pool size. */
1679	uint16_t cCurPages;
1680	/** The head of the free page list. */
1681	uint16_t iFreeHead;
1682	/* Padding. */
1683	uint16_t u16Padding;
1684	#ifdef PGMPOOL_WITH_USER_TRACKING
1685	/** Head of the chain of free user nodes. */
1686	uint16_t iUserFreeHead;
1687	/** The number of user nodes we've allocated. */
1688	uint16_t cMaxUsers;
1689	/** The number of present page table entries in the entire pool. */
1690	uint32_t cPresent;
1691	/** Pointer to the array of user nodes - RC pointer. */
1692	RCPTRTYPE(PPGMPOOLUSER) paUsersRC;
1693	/** Pointer to the array of user nodes - R3 pointer. */
1694	R3PTRTYPE(PPGMPOOLUSER) paUsersR3;
1695	/** Pointer to the array of user nodes - R0 pointer. */
1696	R0PTRTYPE(PPGMPOOLUSER) paUsersR0;
1697	#endif /* PGMPOOL_WITH_USER_TRACKING */
1698	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1699	/** Head of the chain of free phys ext nodes. */
1700	uint16_t iPhysExtFreeHead;
1701	/** The number of user nodes we've allocated. */
1702	uint16_t cMaxPhysExts;
1703	/** Pointer to the array of physical xref extent - RC pointer. */
1704	RCPTRTYPE(PPGMPOOLPHYSEXT) paPhysExtsRC;
1705	/** Pointer to the array of physical xref extent nodes - R3 pointer. */
1706	R3PTRTYPE(PPGMPOOLPHYSEXT) paPhysExtsR3;
1707	/** Pointer to the array of physical xref extent nodes - R0 pointer. */
1708	R0PTRTYPE(PPGMPOOLPHYSEXT) paPhysExtsR0;
1709	#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
1710	#ifdef PGMPOOL_WITH_CACHE
1711	/** Hash table for GCPhys addresses. */
1712	uint16_t aiHash[PGMPOOL_HASH_SIZE];
1713	/** The head of the age list. */
1714	uint16_t iAgeHead;
1715	/** The tail of the age list. */
1716	uint16_t iAgeTail;
1717	/** Set if the cache is enabled. */
1718	bool fCacheEnabled;
1719	#endif /* PGMPOOL_WITH_CACHE */
1720	#ifdef PGMPOOL_WITH_MONITORING
1721	/** Head of the list of modified pages. */
1722	uint16_t iModifiedHead;
1723	/** The current number of modified pages. */
1724	uint16_t cModifiedPages;
1725	/** Access handler, RC. */
1726	RCPTRTYPE(PFNPGMRCPHYSHANDLER) pfnAccessHandlerRC;
1727	/** Access handler, R0. */
1728	R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnAccessHandlerR0;
1729	/** Access handler, R3. */
1730	R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnAccessHandlerR3;
1731	/** The access handler description (HC ptr). */
1732	R3PTRTYPE(const char *) pszAccessHandler;
1733	#endif /* PGMPOOL_WITH_MONITORING */
1734	/** The number of pages currently in use. */
1735	uint16_t cUsedPages;
1736	#ifdef VBOX_WITH_STATISTICS
1737	/** The high wather mark for cUsedPages. */
1738	uint16_t cUsedPagesHigh;
1739	uint32_t Alignment1; /*< Align the next member on a 64-bit boundrary. /
1740	/** Profiling pgmPoolAlloc(). */
1741	STAMPROFILEADV StatAlloc;
1742	/** Profiling pgmPoolClearAll(). */
1743	STAMPROFILE StatClearAll;
1744	/** Profiling pgmPoolFlushAllInt(). */
1745	STAMPROFILE StatFlushAllInt;
1746	/** Profiling pgmPoolFlushPage(). */
1747	STAMPROFILE StatFlushPage;
1748	/** Profiling pgmPoolFree(). */
1749	STAMPROFILE StatFree;
1750	/** Profiling time spent zeroing pages. */
1751	STAMPROFILE StatZeroPage;
1752	# ifdef PGMPOOL_WITH_USER_TRACKING
1753	/** Profiling of pgmPoolTrackDeref. */
1754	STAMPROFILE StatTrackDeref;
1755	/** Profiling pgmTrackFlushGCPhysPT. */
1756	STAMPROFILE StatTrackFlushGCPhysPT;
1757	/** Profiling pgmTrackFlushGCPhysPTs. */
1758	STAMPROFILE StatTrackFlushGCPhysPTs;
1759	/** Profiling pgmTrackFlushGCPhysPTsSlow. */
1760	STAMPROFILE StatTrackFlushGCPhysPTsSlow;
1761	/** Number of times we've been out of user records. */
1762	STAMCOUNTER StatTrackFreeUpOneUser;
1763	# endif
1764	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1765	/** Profiling deref activity related tracking GC physical pages. */
1766	STAMPROFILE StatTrackDerefGCPhys;
1767	/** Number of linear searches for a HCPhys in the ram ranges. */
1768	STAMCOUNTER StatTrackLinearRamSearches;
1769	/** The number of failing pgmPoolTrackPhysExtAlloc calls. */
1770	STAMCOUNTER StamTrackPhysExtAllocFailures;
1771	# endif
1772	# ifdef PGMPOOL_WITH_MONITORING
1773	/** Profiling the RC/R0 access handler. */
1774	STAMPROFILE StatMonitorRZ;
1775	/** Times we've failed interpreting the instruction. */
1776	STAMCOUNTER StatMonitorRZEmulateInstr;
1777	/** Profiling the pgmPoolFlushPage calls made from the RC/R0 access handler. */
1778	STAMPROFILE StatMonitorRZFlushPage;
1779	/** Times we've detected fork(). */
1780	STAMCOUNTER StatMonitorRZFork;
1781	/** Profiling the RC/R0 access we've handled (except REP STOSD). */
1782	STAMPROFILE StatMonitorRZHandled;
1783	/** Times we've failed interpreting a patch code instruction. */
1784	STAMCOUNTER StatMonitorRZIntrFailPatch1;
1785	/** Times we've failed interpreting a patch code instruction during flushing. */
1786	STAMCOUNTER StatMonitorRZIntrFailPatch2;
1787	/** The number of times we've seen rep prefixes we can't handle. */
1788	STAMCOUNTER StatMonitorRZRepPrefix;
1789	/** Profiling the REP STOSD cases we've handled. */
1790	STAMPROFILE StatMonitorRZRepStosd;
1791
1792	/** Profiling the R3 access handler. */
1793	STAMPROFILE StatMonitorR3;
1794	/** Times we've failed interpreting the instruction. */
1795	STAMCOUNTER StatMonitorR3EmulateInstr;
1796	/** Profiling the pgmPoolFlushPage calls made from the R3 access handler. */
1797	STAMPROFILE StatMonitorR3FlushPage;
1798	/** Times we've detected fork(). */
1799	STAMCOUNTER StatMonitorR3Fork;
1800	/** Profiling the R3 access we've handled (except REP STOSD). */
1801	STAMPROFILE StatMonitorR3Handled;
1802	/** The number of times we've seen rep prefixes we can't handle. */
1803	STAMCOUNTER StatMonitorR3RepPrefix;
1804	/** Profiling the REP STOSD cases we've handled. */
1805	STAMPROFILE StatMonitorR3RepStosd;
1806	/** The number of times we're called in an async thread an need to flush. */
1807	STAMCOUNTER StatMonitorR3Async;
1808	/** The high wather mark for cModifiedPages. */
1809	uint16_t cModifiedPagesHigh;
1810	uint16_t Alignment2[3]; /*< Align the next member on a 64-bit boundrary. /
1811	# endif
1812	# ifdef PGMPOOL_WITH_CACHE
1813	/** The number of cache hits. */
1814	STAMCOUNTER StatCacheHits;
1815	/** The number of cache misses. */
1816	STAMCOUNTER StatCacheMisses;
1817	/** The number of times we've got a conflict of 'kind' in the cache. */
1818	STAMCOUNTER StatCacheKindMismatches;
1819	/** Number of times we've been out of pages. */
1820	STAMCOUNTER StatCacheFreeUpOne;
1821	/** The number of cacheable allocations. */
1822	STAMCOUNTER StatCacheCacheable;
1823	/** The number of uncacheable allocations. */
1824	STAMCOUNTER StatCacheUncacheable;
1825	# endif
1826	#elif HC_ARCH_BITS == 64
1827	uint32_t Alignment3; /*< Align the next member on a 64-bit boundrary. /
1828	#endif
1829	/** The AVL tree for looking up a page by its HC physical address. */
1830	AVLOHCPHYSTREE HCPhysTree;
1831	uint32_t Alignment4; /*< Align the next member on a 64-bit boundrary. /
1832	/** Array of pages. (cMaxPages in length)
1833	* The Id is the index into thist array.
1834	*/
1835	PGMPOOLPAGE aPages[PGMPOOL_IDX_FIRST];
1836	} PGMPOOL, PPGMPOOL, *PPPGMPOOL;
1837
1838
1839	/** @def PGMPOOL_PAGE_2_PTR
1840	* Maps a pool page pool into the current context.
1841	*
1842	* @returns VBox status code.
1843	* @param pVM The VM handle.
1844	* @param pPage The pool page.
1845	*
1846	* @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
1847	* small page window employeed by that function. Be careful.
1848	* @remark There is no need to assert on the result.
1849	*/
1850	#if defined(IN_RC)
1851	# define PGMPOOL_PAGE_2_PTR(pVM, pPage) pgmPoolMapPageInlined(&(pVM)->pgm.s, (pPage))
1852	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1853	# define PGMPOOL_PAGE_2_PTR(pVM, pPage) pgmPoolMapPageInlined(&(pVM)->pgm.s, (pPage))
1854	#elif defined(VBOX_STRICT)
1855	# define PGMPOOL_PAGE_2_PTR(pVM, pPage) pgmPoolMapPageStrict(pPage)
1856	DECLINLINE(void *) pgmPoolMapPageStrict(PPGMPOOLPAGE pPage)
1857	{
1858	Assert(pPage && pPage->pvPageR3);
1859	return pPage->pvPageR3;
1860	}
1861	#else
1862	# define PGMPOOL_PAGE_2_PTR(pVM, pPage) ((pPage)->pvPageR3)
1863	#endif
1864
1865	/** @def PGMPOOL_PAGE_2_PTR_BY_PGM
1866	* Maps a pool page pool into the current context.
1867	*
1868	* @returns VBox status code.
1869	* @param pPGM Pointer to the PGM instance data.
1870	* @param pPage The pool page.
1871	*
1872	* @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
1873	* small page window employeed by that function. Be careful.
1874	* @remark There is no need to assert on the result.
1875	*/
1876	#if defined(IN_RC)
1877	# define PGMPOOL_PAGE_2_PTR_BY_PGM(pPGM, pPage) pgmPoolMapPageInlined(pPGM, (pPage))
1878	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1879	# define PGMPOOL_PAGE_2_PTR_BY_PGM(pPGM, pPage) pgmPoolMapPageInlined(pPGM, (pPage))
1880	#else
1881	# define PGMPOOL_PAGE_2_PTR_BY_PGM(pPGM, pPage) PGMPOOL_PAGE_2_PTR(PGM2VM(pPGM), pPage)
1882	#endif
1883
1884	/** @def PGMPOOL_PAGE_2_PTR_BY_PGMCPU
1885	* Maps a pool page pool into the current context.
1886	*
1887	* @returns VBox status code.
1888	* @param pPGM Pointer to the PGMCPU instance data.
1889	* @param pPage The pool page.
1890	*
1891	* @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
1892	* small page window employeed by that function. Be careful.
1893	* @remark There is no need to assert on the result.
1894	*/
1895	#if defined(IN_RC)
1896	# define PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPage) pgmPoolMapPageInlined(PGMCPU2PGM(pPGM), (pPage))
1897	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1898	# define PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPage) pgmPoolMapPageInlined(PGMCPU2PGM(pPGM), (pPage))
1899	#else
1900	# define PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPage) PGMPOOL_PAGE_2_PTR(PGMCPU2VM(pPGM), pPage)
1901	#endif
1902
1903
1904	/** @name Per guest page tracking data.
1905	* This is currently as a 16-bit word in the PGMPAGE structure, the idea though
1906	* is to use more bits for it and split it up later on. But for now we'll play
1907	* safe and change as little as possible.
1908	*
1909	* The 16-bit word has two parts:
1910	*
1911	* The first 14-bit forms the @a idx field. It is either the index of a page in
1912	* the shadow page pool, or and index into the extent list.
1913	*
1914	* The 2 topmost bits makes up the @a cRefs field, which counts the number of
1915	* shadow page pool references to the page. If cRefs equals
1916	* PGMPOOL_CREFS_PHYSEXT, then the @a idx field is an indext into the extent
1917	* (misnomer) table and not the shadow page pool.
1918	*
1919	* See PGM_PAGE_GET_TRACKING and PGM_PAGE_SET_TRACKING for how to get and set
1920	* the 16-bit word.
1921	*
1922	* @{ */
1923	/** The shift count for getting to the cRefs part. */
1924	#define PGMPOOL_TD_CREFS_SHIFT 14
1925	/** The mask applied after shifting the tracking data down by
1926	* PGMPOOL_TD_CREFS_SHIFT. */
1927	#define PGMPOOL_TD_CREFS_MASK 0x3
1928	/** The cRef value used to indiciate that the idx is the head of a
1929	* physical cross reference list. */
1930	#define PGMPOOL_TD_CREFS_PHYSEXT PGMPOOL_TD_CREFS_MASK
1931	/** The shift used to get idx. */
1932	#define PGMPOOL_TD_IDX_SHIFT 0
1933	/** The mask applied to the idx after shifting down by PGMPOOL_TD_IDX_SHIFT. */
1934	#define PGMPOOL_TD_IDX_MASK 0x3fff
1935	/** The idx value when we're out of of PGMPOOLPHYSEXT entries or/and there are
1936	* simply too many mappings of this page. */
1937	#define PGMPOOL_TD_IDX_OVERFLOWED PGMPOOL_TD_IDX_MASK
1938
1939	/** @def PGMPOOL_TD_MAKE
1940	* Makes a 16-bit tracking data word.
1941	*
1942	* @returns tracking data.
1943	* @param cRefs The @a cRefs field. Must be within bounds!
1944	* @param idx The @a idx field. Must also be within bounds! */
1945	#define PGMPOOL_TD_MAKE(cRefs, idx) ( ((cRefs) << PGMPOOL_TD_CREFS_SHIFT) \| (idx) )
1946
1947	/** @def PGMPOOL_TD_GET_CREFS
1948	* Get the @a cRefs field from a tracking data word.
1949	*
1950	* @returns The @a cRefs field
1951	* @param u16 The tracking data word. */
1952	#define PGMPOOL_TD_GET_CREFS(u16) ( ((u16) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK )
1953
1954	/** @def PGMPOOL_TD_GET_IDX
1955	* Get the @a idx field from a tracking data word.
1956	*
1957	* @returns The @a idx field
1958	* @param u16 The tracking data word. */
1959	#define PGMPOOL_TD_GET_IDX(u16) ( ((u16) >> PGMPOOL_TD_IDX_SHIFT) & PGMPOOL_TD_IDX_MASK )
1960	/** @} */
1961
1962
1963	/**
1964	* Trees are using self relative offsets as pointers.
1965	* So, all its data, including the root pointer, must be in the heap for HC and GC
1966	* to have the same layout.
1967	*/
1968	typedef struct PGMTREES
1969	{
1970	/** Physical access handlers (AVL range+offsetptr tree). */
1971	AVLROGCPHYSTREE PhysHandlers;
1972	/** Virtual access handlers (AVL range + GC ptr tree). */
1973	AVLROGCPTRTREE VirtHandlers;
1974	/** Virtual access handlers (Phys range AVL range + offsetptr tree). */
1975	AVLROGCPHYSTREE PhysToVirtHandlers;
1976	/** Virtual access handlers for the hypervisor (AVL range + GC ptr tree). */
1977	AVLROGCPTRTREE HyperVirtHandlers;
1978	} PGMTREES;
1979	/** Pointer to PGM trees. */
1980	typedef PGMTREES *PPGMTREES;
1981
1982
1983	/** @name Paging mode macros
1984	* @{ */
1985	#ifdef IN_RC
1986	# define PGM_CTX(a,b) a##RC##b
1987	# define PGM_CTX_STR(a,b) a "GC" b
1988	# define PGM_CTX_DECL(type) VMMRCDECL(type)
1989	#else
1990	# ifdef IN_RING3
1991	# define PGM_CTX(a,b) a##R3##b
1992	# define PGM_CTX_STR(a,b) a "R3" b
1993	# define PGM_CTX_DECL(type) DECLCALLBACK(type)
1994	# else
1995	# define PGM_CTX(a,b) a##R0##b
1996	# define PGM_CTX_STR(a,b) a "R0" b
1997	# define PGM_CTX_DECL(type) VMMDECL(type)
1998	# endif
1999	#endif
2000
2001	#define PGM_GST_NAME_REAL(name) PGM_CTX(pgm,GstReal##name)
2002	#define PGM_GST_NAME_RC_REAL_STR(name) "pgmRCGstReal" #name
2003	#define PGM_GST_NAME_R0_REAL_STR(name) "pgmR0GstReal" #name
2004	#define PGM_GST_NAME_PROT(name) PGM_CTX(pgm,GstProt##name)
2005	#define PGM_GST_NAME_RC_PROT_STR(name) "pgmRCGstProt" #name
2006	#define PGM_GST_NAME_R0_PROT_STR(name) "pgmR0GstProt" #name
2007	#define PGM_GST_NAME_32BIT(name) PGM_CTX(pgm,Gst32Bit##name)
2008	#define PGM_GST_NAME_RC_32BIT_STR(name) "pgmRCGst32Bit" #name
2009	#define PGM_GST_NAME_R0_32BIT_STR(name) "pgmR0Gst32Bit" #name
2010	#define PGM_GST_NAME_PAE(name) PGM_CTX(pgm,GstPAE##name)
2011	#define PGM_GST_NAME_RC_PAE_STR(name) "pgmRCGstPAE" #name
2012	#define PGM_GST_NAME_R0_PAE_STR(name) "pgmR0GstPAE" #name
2013	#define PGM_GST_NAME_AMD64(name) PGM_CTX(pgm,GstAMD64##name)
2014	#define PGM_GST_NAME_RC_AMD64_STR(name) "pgmRCGstAMD64" #name
2015	#define PGM_GST_NAME_R0_AMD64_STR(name) "pgmR0GstAMD64" #name
2016	#define PGM_GST_PFN(name, pVCpu) ((pVCpu)->pgm.s.PGM_CTX(pfn,Gst##name))
2017	#define PGM_GST_DECL(type, name) PGM_CTX_DECL(type) PGM_GST_NAME(name)
2018
2019	#define PGM_SHW_NAME_32BIT(name) PGM_CTX(pgm,Shw32Bit##name)
2020	#define PGM_SHW_NAME_RC_32BIT_STR(name) "pgmRCShw32Bit" #name
2021	#define PGM_SHW_NAME_R0_32BIT_STR(name) "pgmR0Shw32Bit" #name
2022	#define PGM_SHW_NAME_PAE(name) PGM_CTX(pgm,ShwPAE##name)
2023	#define PGM_SHW_NAME_RC_PAE_STR(name) "pgmRCShwPAE" #name
2024	#define PGM_SHW_NAME_R0_PAE_STR(name) "pgmR0ShwPAE" #name
2025	#define PGM_SHW_NAME_AMD64(name) PGM_CTX(pgm,ShwAMD64##name)
2026	#define PGM_SHW_NAME_RC_AMD64_STR(name) "pgmRCShwAMD64" #name
2027	#define PGM_SHW_NAME_R0_AMD64_STR(name) "pgmR0ShwAMD64" #name
2028	#define PGM_SHW_NAME_NESTED(name) PGM_CTX(pgm,ShwNested##name)
2029	#define PGM_SHW_NAME_RC_NESTED_STR(name) "pgmRCShwNested" #name
2030	#define PGM_SHW_NAME_R0_NESTED_STR(name) "pgmR0ShwNested" #name
2031	#define PGM_SHW_NAME_EPT(name) PGM_CTX(pgm,ShwEPT##name)
2032	#define PGM_SHW_NAME_RC_EPT_STR(name) "pgmRCShwEPT" #name
2033	#define PGM_SHW_NAME_R0_EPT_STR(name) "pgmR0ShwEPT" #name
2034	#define PGM_SHW_DECL(type, name) PGM_CTX_DECL(type) PGM_SHW_NAME(name)
2035	#define PGM_SHW_PFN(name, pVCpu) ((pVCpu)->pgm.s.PGM_CTX(pfn,Shw##name))
2036
2037	/* Shw_Gst */
2038	#define PGM_BTH_NAME_32BIT_REAL(name) PGM_CTX(pgm,Bth32BitReal##name)
2039	#define PGM_BTH_NAME_32BIT_PROT(name) PGM_CTX(pgm,Bth32BitProt##name)
2040	#define PGM_BTH_NAME_32BIT_32BIT(name) PGM_CTX(pgm,Bth32Bit32Bit##name)
2041	#define PGM_BTH_NAME_PAE_REAL(name) PGM_CTX(pgm,BthPAEReal##name)
2042	#define PGM_BTH_NAME_PAE_PROT(name) PGM_CTX(pgm,BthPAEProt##name)
2043	#define PGM_BTH_NAME_PAE_32BIT(name) PGM_CTX(pgm,BthPAE32Bit##name)
2044	#define PGM_BTH_NAME_PAE_PAE(name) PGM_CTX(pgm,BthPAEPAE##name)
2045	#define PGM_BTH_NAME_AMD64_PROT(name) PGM_CTX(pgm,BthAMD64Prot##name)
2046	#define PGM_BTH_NAME_AMD64_AMD64(name) PGM_CTX(pgm,BthAMD64AMD64##name)
2047	#define PGM_BTH_NAME_NESTED_REAL(name) PGM_CTX(pgm,BthNestedReal##name)
2048	#define PGM_BTH_NAME_NESTED_PROT(name) PGM_CTX(pgm,BthNestedProt##name)
2049	#define PGM_BTH_NAME_NESTED_32BIT(name) PGM_CTX(pgm,BthNested32Bit##name)
2050	#define PGM_BTH_NAME_NESTED_PAE(name) PGM_CTX(pgm,BthNestedPAE##name)
2051	#define PGM_BTH_NAME_NESTED_AMD64(name) PGM_CTX(pgm,BthNestedAMD64##name)
2052	#define PGM_BTH_NAME_EPT_REAL(name) PGM_CTX(pgm,BthEPTReal##name)
2053	#define PGM_BTH_NAME_EPT_PROT(name) PGM_CTX(pgm,BthEPTProt##name)
2054	#define PGM_BTH_NAME_EPT_32BIT(name) PGM_CTX(pgm,BthEPT32Bit##name)
2055	#define PGM_BTH_NAME_EPT_PAE(name) PGM_CTX(pgm,BthEPTPAE##name)
2056	#define PGM_BTH_NAME_EPT_AMD64(name) PGM_CTX(pgm,BthEPTAMD64##name)
2057
2058	#define PGM_BTH_NAME_RC_32BIT_REAL_STR(name) "pgmRCBth32BitReal" #name
2059	#define PGM_BTH_NAME_RC_32BIT_PROT_STR(name) "pgmRCBth32BitProt" #name
2060	#define PGM_BTH_NAME_RC_32BIT_32BIT_STR(name) "pgmRCBth32Bit32Bit" #name
2061	#define PGM_BTH_NAME_RC_PAE_REAL_STR(name) "pgmRCBthPAEReal" #name
2062	#define PGM_BTH_NAME_RC_PAE_PROT_STR(name) "pgmRCBthPAEProt" #name
2063	#define PGM_BTH_NAME_RC_PAE_32BIT_STR(name) "pgmRCBthPAE32Bit" #name
2064	#define PGM_BTH_NAME_RC_PAE_PAE_STR(name) "pgmRCBthPAEPAE" #name
2065	#define PGM_BTH_NAME_RC_AMD64_AMD64_STR(name) "pgmRCBthAMD64AMD64" #name
2066	#define PGM_BTH_NAME_RC_NESTED_REAL_STR(name) "pgmRCBthNestedReal" #name
2067	#define PGM_BTH_NAME_RC_NESTED_PROT_STR(name) "pgmRCBthNestedProt" #name
2068	#define PGM_BTH_NAME_RC_NESTED_32BIT_STR(name) "pgmRCBthNested32Bit" #name
2069	#define PGM_BTH_NAME_RC_NESTED_PAE_STR(name) "pgmRCBthNestedPAE" #name
2070	#define PGM_BTH_NAME_RC_NESTED_AMD64_STR(name) "pgmRCBthNestedAMD64" #name
2071	#define PGM_BTH_NAME_RC_EPT_REAL_STR(name) "pgmRCBthEPTReal" #name
2072	#define PGM_BTH_NAME_RC_EPT_PROT_STR(name) "pgmRCBthEPTProt" #name
2073	#define PGM_BTH_NAME_RC_EPT_32BIT_STR(name) "pgmRCBthEPT32Bit" #name
2074	#define PGM_BTH_NAME_RC_EPT_PAE_STR(name) "pgmRCBthEPTPAE" #name
2075	#define PGM_BTH_NAME_RC_EPT_AMD64_STR(name) "pgmRCBthEPTAMD64" #name
2076	#define PGM_BTH_NAME_R0_32BIT_REAL_STR(name) "pgmR0Bth32BitReal" #name
2077	#define PGM_BTH_NAME_R0_32BIT_PROT_STR(name) "pgmR0Bth32BitProt" #name
2078	#define PGM_BTH_NAME_R0_32BIT_32BIT_STR(name) "pgmR0Bth32Bit32Bit" #name
2079	#define PGM_BTH_NAME_R0_PAE_REAL_STR(name) "pgmR0BthPAEReal" #name
2080	#define PGM_BTH_NAME_R0_PAE_PROT_STR(name) "pgmR0BthPAEProt" #name
2081	#define PGM_BTH_NAME_R0_PAE_32BIT_STR(name) "pgmR0BthPAE32Bit" #name
2082	#define PGM_BTH_NAME_R0_PAE_PAE_STR(name) "pgmR0BthPAEPAE" #name
2083	#define PGM_BTH_NAME_R0_AMD64_PROT_STR(name) "pgmR0BthAMD64Prot" #name
2084	#define PGM_BTH_NAME_R0_AMD64_AMD64_STR(name) "pgmR0BthAMD64AMD64" #name
2085	#define PGM_BTH_NAME_R0_NESTED_REAL_STR(name) "pgmR0BthNestedReal" #name
2086	#define PGM_BTH_NAME_R0_NESTED_PROT_STR(name) "pgmR0BthNestedProt" #name
2087	#define PGM_BTH_NAME_R0_NESTED_32BIT_STR(name) "pgmR0BthNested32Bit" #name
2088	#define PGM_BTH_NAME_R0_NESTED_PAE_STR(name) "pgmR0BthNestedPAE" #name
2089	#define PGM_BTH_NAME_R0_NESTED_AMD64_STR(name) "pgmR0BthNestedAMD64" #name
2090	#define PGM_BTH_NAME_R0_EPT_REAL_STR(name) "pgmR0BthEPTReal" #name
2091	#define PGM_BTH_NAME_R0_EPT_PROT_STR(name) "pgmR0BthEPTProt" #name
2092	#define PGM_BTH_NAME_R0_EPT_32BIT_STR(name) "pgmR0BthEPT32Bit" #name
2093	#define PGM_BTH_NAME_R0_EPT_PAE_STR(name) "pgmR0BthEPTPAE" #name
2094	#define PGM_BTH_NAME_R0_EPT_AMD64_STR(name) "pgmR0BthEPTAMD64" #name
2095
2096	#define PGM_BTH_DECL(type, name) PGM_CTX_DECL(type) PGM_BTH_NAME(name)
2097	#define PGM_BTH_PFN(name, pVCpu) ((pVCpu)->pgm.s.PGM_CTX(pfn,Bth##name))
2098	/** @} */
2099
2100	/**
2101	* Data for each paging mode.
2102	*/
2103	typedef struct PGMMODEDATA
2104	{
2105	/** The guest mode type. */
2106	uint32_t uGstType;
2107	/** The shadow mode type. */
2108	uint32_t uShwType;
2109
2110	/** @name Function pointers for Shadow paging.
2111	* @{
2112	*/
2113	DECLR3CALLBACKMEMBER(int, pfnR3ShwRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2114	DECLR3CALLBACKMEMBER(int, pfnR3ShwExit,(PVMCPU pVCpu));
2115	DECLR3CALLBACKMEMBER(int, pfnR3ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2116	DECLR3CALLBACKMEMBER(int, pfnR3ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2117
2118	DECLRCCALLBACKMEMBER(int, pfnRCShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2119	DECLRCCALLBACKMEMBER(int, pfnRCShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2120
2121	DECLR0CALLBACKMEMBER(int, pfnR0ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2122	DECLR0CALLBACKMEMBER(int, pfnR0ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2123	/** @} */
2124
2125	/** @name Function pointers for Guest paging.
2126	* @{
2127	*/
2128	DECLR3CALLBACKMEMBER(int, pfnR3GstRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2129	DECLR3CALLBACKMEMBER(int, pfnR3GstExit,(PVMCPU pVCpu));
2130	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2131	DECLR3CALLBACKMEMBER(int, pfnR3GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2132	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2133	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2134	DECLRCCALLBACKMEMBER(int, pfnRCGstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2135	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2136	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2137	DECLR0CALLBACKMEMBER(int, pfnR0GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2138	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2139	/** @} */
2140
2141	/** @name Function pointers for Both Shadow and Guest paging.
2142	* @{
2143	*/
2144	DECLR3CALLBACKMEMBER(int, pfnR3BthRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2145	/* no pfnR3BthTrap0eHandler */
2146	DECLR3CALLBACKMEMBER(int, pfnR3BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2147	DECLR3CALLBACKMEMBER(int, pfnR3BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2148	DECLR3CALLBACKMEMBER(int, pfnR3BthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2149	DECLR3CALLBACKMEMBER(int, pfnR3BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2150	DECLR3CALLBACKMEMBER(int, pfnR3BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2151	#ifdef VBOX_STRICT
2152	DECLR3CALLBACKMEMBER(unsigned, pfnR3BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2153	#endif
2154	DECLR3CALLBACKMEMBER(int, pfnR3BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2155	DECLR3CALLBACKMEMBER(int, pfnR3BthUnmapCR3,(PVMCPU pVCpu));
2156
2157	DECLRCCALLBACKMEMBER(int, pfnRCBthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault));
2158	DECLRCCALLBACKMEMBER(int, pfnRCBthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2159	DECLRCCALLBACKMEMBER(int, pfnRCBthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2160	DECLRCCALLBACKMEMBER(int, pfnRCBthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2161	DECLRCCALLBACKMEMBER(int, pfnRCBthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2162	DECLRCCALLBACKMEMBER(int, pfnRCBthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2163	#ifdef VBOX_STRICT
2164	DECLRCCALLBACKMEMBER(unsigned, pfnRCBthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2165	#endif
2166	DECLRCCALLBACKMEMBER(int, pfnRCBthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2167	DECLRCCALLBACKMEMBER(int, pfnRCBthUnmapCR3,(PVMCPU pVCpu));
2168
2169	DECLR0CALLBACKMEMBER(int, pfnR0BthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault));
2170	DECLR0CALLBACKMEMBER(int, pfnR0BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2171	DECLR0CALLBACKMEMBER(int, pfnR0BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2172	DECLR0CALLBACKMEMBER(int, pfnR0BthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2173	DECLR0CALLBACKMEMBER(int, pfnR0BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2174	DECLR0CALLBACKMEMBER(int, pfnR0BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2175	#ifdef VBOX_STRICT
2176	DECLR0CALLBACKMEMBER(unsigned, pfnR0BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2177	#endif
2178	DECLR0CALLBACKMEMBER(int, pfnR0BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2179	DECLR0CALLBACKMEMBER(int, pfnR0BthUnmapCR3,(PVMCPU pVCpu));
2180	/** @} */
2181	} PGMMODEDATA, *PPGMMODEDATA;
2182
2183
2184
2185	/**
2186	* Converts a PGM pointer into a VM pointer.
2187	* @returns Pointer to the VM structure the PGM is part of.
2188	* @param pPGM Pointer to PGM instance data.
2189	*/
2190	#define PGM2VM(pPGM) ( (PVM)((char*)pPGM - pPGM->offVM) )
2191
2192	/**
2193	* PGM Data (part of VM)
2194	*/
2195	typedef struct PGM
2196	{
2197	/** Offset to the VM structure. */
2198	RTINT offVM;
2199	/** Offset of the PGMCPU structure relative to VMCPU. */
2200	RTINT offVCpuPGM;
2201
2202	/** @cfgm{RamPreAlloc, boolean, false}
2203	* Indicates whether the base RAM should all be allocated before starting
2204	* the VM (default), or if it should be allocated when first written to.
2205	*/
2206	bool fRamPreAlloc;
2207	/** Alignment padding. */
2208	bool afAlignment0[7];
2209
2210	/** What needs syncing (PGM_SYNC_*).
2211	* This is used to queue operations for PGMSyncCR3, PGMInvalidatePage,
2212	* PGMFlushTLB, and PGMR3Load. */
2213	RTUINT fGlobalSyncFlags;
2214
2215	/*
2216	* This will be redefined at least two more times before we're done, I'm sure.
2217	* The current code is only to get on with the coding.
2218	* - 2004-06-10: initial version, bird.
2219	* - 2004-07-02: 1st time, bird.
2220	* - 2004-10-18: 2nd time, bird.
2221	* - 2005-07-xx: 3rd time, bird.
2222	*/
2223
2224	/** Pointer to the page table entries for the dynamic page mapping area - GCPtr. */
2225	RCPTRTYPE(PX86PTE) paDynPageMap32BitPTEsGC;
2226	/** Pointer to the page table entries for the dynamic page mapping area - GCPtr. */
2227	RCPTRTYPE(PX86PTEPAE) paDynPageMapPaePTEsGC;
2228
2229	/** The host paging mode. (This is what SUPLib reports.) */
2230	SUPPAGINGMODE enmHostMode;
2231
2232	/** 4 MB page mask; 32 or 36 bits depending on PSE-36 (identical for all VCPUs) */
2233	RTGCPHYS GCPhys4MBPSEMask;
2234
2235	/** Pointer to the list of RAM ranges (Phys GC -> Phys HC conversion) - for R3.
2236	* This is sorted by physical address and contains no overlapping ranges. */
2237	R3PTRTYPE(PPGMRAMRANGE) pRamRangesR3;
2238	/** R0 pointer corresponding to PGM::pRamRangesR3. */
2239	R0PTRTYPE(PPGMRAMRANGE) pRamRangesR0;
2240	/** RC pointer corresponding to PGM::pRamRangesR3. */
2241	RCPTRTYPE(PPGMRAMRANGE) pRamRangesRC;
2242	RTRCPTR alignment4; /*< structure alignment. /
2243
2244	/** Pointer to the list of ROM ranges - for R3.
2245	* This is sorted by physical address and contains no overlapping ranges. */
2246	R3PTRTYPE(PPGMROMRANGE) pRomRangesR3;
2247	/** R0 pointer corresponding to PGM::pRomRangesR3. */
2248	R0PTRTYPE(PPGMROMRANGE) pRomRangesR0;
2249	/** RC pointer corresponding to PGM::pRomRangesR3. */
2250	RCPTRTYPE(PPGMROMRANGE) pRomRangesRC;
2251	/** Alignment padding. */
2252	RTRCPTR GCPtrPadding2;
2253
2254	/** Pointer to the list of MMIO2 ranges - for R3.
2255	* Registration order. */
2256	R3PTRTYPE(PPGMMMIO2RANGE) pMmio2RangesR3;
2257
2258	/** PGM offset based trees - R3 Ptr. */
2259	R3PTRTYPE(PPGMTREES) pTreesR3;
2260	/** PGM offset based trees - R0 Ptr. */
2261	R0PTRTYPE(PPGMTREES) pTreesR0;
2262	/** PGM offset based trees - RC Ptr. */
2263	RCPTRTYPE(PPGMTREES) pTreesRC;
2264
2265	/** Linked list of GC mappings - for RC.
2266	* The list is sorted ascending on address.
2267	*/
2268	RCPTRTYPE(PPGMMAPPING) pMappingsRC;
2269	/** Linked list of GC mappings - for HC.
2270	* The list is sorted ascending on address.
2271	*/
2272	R3PTRTYPE(PPGMMAPPING) pMappingsR3;
2273	/** Linked list of GC mappings - for R0.
2274	* The list is sorted ascending on address.
2275	*/
2276	R0PTRTYPE(PPGMMAPPING) pMappingsR0;
2277
2278	/** Pointer to the 5 page CR3 content mapping.
2279	* The first page is always the CR3 (in some form) while the 4 other pages
2280	* are used of the PDs in PAE mode. */
2281	RTGCPTR GCPtrCR3Mapping;
2282	#if HC_ARCH_BITS == 64 && GC_ARCH_BITS == 32
2283	uint32_t u32Alignment;
2284	#endif
2285
2286	/** Indicates that PGMR3FinalizeMappings has been called and that further
2287	* PGMR3MapIntermediate calls will be rejected. */
2288	bool fFinalizedMappings;
2289	/** If set no conflict checks are required. (boolean) */
2290	bool fMappingsFixed;
2291	/** If set, then no mappings are put into the shadow page table. (boolean) */
2292	bool fDisableMappings;
2293	/** Size of fixed mapping */
2294	uint32_t cbMappingFixed;
2295	/** Base address (GC) of fixed mapping */
2296	RTGCPTR GCPtrMappingFixed;
2297	/** The address of the previous RAM range mapping. */
2298	RTGCPTR GCPtrPrevRamRangeMapping;
2299
2300	/** @name Intermediate Context
2301	* @{ */
2302	/** Pointer to the intermediate page directory - Normal. */
2303	R3PTRTYPE(PX86PD) pInterPD;
2304	/** Pointer to the intermedate page tables - Normal.
2305	* There are two page tables, one for the identity mapping and one for
2306	* the host context mapping (of the core code). */
2307	R3PTRTYPE(PX86PT) apInterPTs[2];
2308	/** Pointer to the intermedate page tables - PAE. */
2309	R3PTRTYPE(PX86PTPAE) apInterPaePTs[2];
2310	/** Pointer to the intermedate page directory - PAE. */
2311	R3PTRTYPE(PX86PDPAE) apInterPaePDs[4];
2312	/** Pointer to the intermedate page directory - PAE. */
2313	R3PTRTYPE(PX86PDPT) pInterPaePDPT;
2314	/** Pointer to the intermedate page-map level 4 - AMD64. */
2315	R3PTRTYPE(PX86PML4) pInterPaePML4;
2316	/** Pointer to the intermedate page directory - AMD64. */
2317	R3PTRTYPE(PX86PDPT) pInterPaePDPT64;
2318	/** The Physical Address (HC) of the intermediate Page Directory - Normal. */
2319	RTHCPHYS HCPhysInterPD;
2320	/** The Physical Address (HC) of the intermediate Page Directory Pointer Table - PAE. */
2321	RTHCPHYS HCPhysInterPaePDPT;
2322	/** The Physical Address (HC) of the intermediate Page Map Level 4 table - AMD64. */
2323	RTHCPHYS HCPhysInterPaePML4;
2324	/** @} */
2325
2326	/** Base address of the dynamic page mapping area.
2327	* The array is MM_HYPER_DYNAMIC_SIZE bytes big.
2328	*/
2329	RCPTRTYPE(uint8_t *) pbDynPageMapBaseGC;
2330	/** The index of the last entry used in the dynamic page mapping area. */
2331	RTUINT iDynPageMapLast;
2332	/** Cache containing the last entries in the dynamic page mapping area.
2333	* The cache size is covering half of the mapping area. */
2334	RTHCPHYS aHCPhysDynPageMapCache[MM_HYPER_DYNAMIC_SIZE >> (PAGE_SHIFT + 1)];
2335	/** Keep a lock counter for the full (!) mapping area. */
2336	uint32_t aLockedDynPageMapCache[MM_HYPER_DYNAMIC_SIZE >> (PAGE_SHIFT)];
2337
2338	/** The address of the ring-0 mapping cache if we're making use of it. */
2339	RTR0PTR pvR0DynMapUsed;
2340
2341	/** PGM critical section.
2342	* This protects the physical & virtual access handlers, ram ranges,
2343	* and the page flag updating (some of it anyway).
2344	*/
2345	PDMCRITSECT CritSect;
2346
2347	/** Pointer to SHW+GST mode data (function pointers).
2348	* The index into this table is made up from */
2349	R3PTRTYPE(PPGMMODEDATA) paModeData;
2350
2351	/** Shadow Page Pool - R3 Ptr. */
2352	R3PTRTYPE(PPGMPOOL) pPoolR3;
2353	/** Shadow Page Pool - R0 Ptr. */
2354	R0PTRTYPE(PPGMPOOL) pPoolR0;
2355	/** Shadow Page Pool - RC Ptr. */
2356	RCPTRTYPE(PPGMPOOL) pPoolRC;
2357
2358	/** We're not in a state which permits writes to guest memory.
2359	* (Only used in strict builds.) */
2360	bool fNoMorePhysWrites;
2361
2362	/** Flush the cache on the next access. */
2363	bool fPhysCacheFlushPending;
2364	/** @todo r=bird: Fix member names!*/
2365	/** PGMPhysRead cache */
2366	PGMPHYSCACHE pgmphysreadcache;
2367	/** PGMPhysWrite cache */
2368	PGMPHYSCACHE pgmphyswritecache;
2369
2370	/**
2371	* Data associated with managing the ring-3 mappings of the allocation chunks.
2372	*/
2373	struct
2374	{
2375	/** The chunk tree, ordered by chunk id. */
2376	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
2377	R3PTRTYPE(PAVLU32NODECORE) pTree;
2378	#else
2379	R3R0PTRTYPE(PAVLU32NODECORE) pTree;
2380	#endif
2381	/** The chunk mapping TLB. */
2382	PGMCHUNKR3MAPTLB Tlb;
2383	/** The number of mapped chunks. */
2384	uint32_t c;
2385	/** The maximum number of mapped chunks.
2386	* @cfgm PGM/MaxRing3Chunks */
2387	uint32_t cMax;
2388	/** The chunk age tree, ordered by ageing sequence number. */
2389	R3PTRTYPE(PAVLLU32NODECORE) pAgeTree;
2390	/** The current time. */
2391	uint32_t iNow;
2392	/** Number of pgmR3PhysChunkFindUnmapCandidate calls left to the next ageing. */
2393	uint32_t AgeingCountdown;
2394	} ChunkR3Map;
2395
2396	/**
2397	* The page mapping TLB for ring-3 and (for the time being) ring-0.
2398	*/
2399	PGMPAGER3MAPTLB PhysTlbHC;
2400
2401	/** @name The zero page.
2402	* @{ */
2403	/** The host physical address of the zero page. */
2404	RTHCPHYS HCPhysZeroPg;
2405	/** The ring-3 mapping of the zero page. */
2406	RTR3PTR pvZeroPgR3;
2407	/** The ring-0 mapping of the zero page. */
2408	RTR0PTR pvZeroPgR0;
2409	/** The GC mapping of the zero page. */
2410	RTGCPTR pvZeroPgRC;
2411	#if GC_ARCH_BITS != 32
2412	uint32_t u32ZeroAlignment; /*< Alignment padding. /
2413	#endif
2414	/** @}*/
2415
2416	/** The number of handy pages. */
2417	uint32_t cHandyPages;
2418	/**
2419	* Array of handy pages.
2420	*
2421	* This array is used in a two way communication between pgmPhysAllocPage
2422	* and GMMR0AllocateHandyPages, with PGMR3PhysAllocateHandyPages serving as
2423	* an intermediary.
2424	*
2425	* The size of this array is important, see pgmPhysEnsureHandyPage for details.
2426	* (The current size of 32 pages, means 128 KB of handy memory.)
2427	*/
2428	GMMPAGEDESC aHandyPages[PGM_HANDY_PAGES];
2429
2430	/** @name Error injection.
2431	* @{ */
2432	/** Inject handy page allocation errors pretending we're completely out of
2433	* memory. */
2434	bool volatile fErrInjHandyPages;
2435	/** Padding. */
2436	bool afReserved[7];
2437	/** @} */
2438
2439	/** @name Release Statistics
2440	* @{ */
2441	uint32_t cAllPages; /*< The total number of pages. (Should be Private + Shared + Zero.) /
2442	uint32_t cPrivatePages; /*< The number of private pages. /
2443	uint32_t cSharedPages; /*< The number of shared pages. /
2444	uint32_t cZeroPages; /*< The number of zero backed pages. /
2445
2446	/** The number of times we were forced to change the hypervisor region location. */
2447	STAMCOUNTER cRelocations;
2448	/** @} */
2449
2450	#ifdef VBOX_WITH_STATISTICS /** @todo move this chunk to the heap. */
2451	/* R3 only: */
2452	STAMCOUNTER StatR3DetectedConflicts; /*< R3: Number of times PGMR3MapHasConflicts() detected a conflict. /
2453	STAMPROFILE StatR3ResolveConflict; /*< R3: pgmR3SyncPTResolveConflict() profiling (includes the entire relocation). /
2454
2455	STAMCOUNTER StatRZChunkR3MapTlbHits; /*< RC/R0: Ring-3/0 chunk mapper TLB hits. /
2456	STAMCOUNTER StatRZChunkR3MapTlbMisses; /*< RC/R0: Ring-3/0 chunk mapper TLB misses. /
2457	STAMCOUNTER StatRZPageMapTlbHits; /*< RC/R0: Ring-3/0 page mapper TLB hits. /
2458	STAMCOUNTER StatRZPageMapTlbMisses; /*< RC/R0: Ring-3/0 page mapper TLB misses. /
2459	STAMCOUNTER StatR3ChunkR3MapTlbHits; /*< R3: Ring-3/0 chunk mapper TLB hits. /
2460	STAMCOUNTER StatR3ChunkR3MapTlbMisses; /*< R3: Ring-3/0 chunk mapper TLB misses. /
2461	STAMCOUNTER StatR3PageMapTlbHits; /*< R3: Ring-3/0 page mapper TLB hits. /
2462	STAMCOUNTER StatR3PageMapTlbMisses; /*< R3: Ring-3/0 page mapper TLB misses. /
2463	STAMPROFILE StatRZSyncCR3HandlerVirtualReset; /*< RC/R0: Profiling of the virtual handler resets. /
2464	STAMPROFILE StatRZSyncCR3HandlerVirtualUpdate; /*< RC/R0: Profiling of the virtual handler updates. /
2465	STAMPROFILE StatR3SyncCR3HandlerVirtualReset; /*< R3: Profiling of the virtual handler resets. /
2466	STAMPROFILE StatR3SyncCR3HandlerVirtualUpdate; /*< R3: Profiling of the virtual handler updates. /
2467	STAMCOUNTER StatR3PhysHandlerReset; /*< R3: The number of times PGMHandlerPhysicalReset is called. /
2468	STAMCOUNTER StatRZPhysHandlerReset; /*< RC/R0: The number of times PGMHandlerPhysicalReset is called. /
2469	STAMPROFILE StatRZVirtHandlerSearchByPhys; /*< RC/R0: Profiling of pgmHandlerVirtualFindByPhysAddr. /
2470	STAMPROFILE StatR3VirtHandlerSearchByPhys; /*< R3: Profiling of pgmHandlerVirtualFindByPhysAddr. /
2471	STAMCOUNTER StatRZPageReplaceShared; /*< RC/R0: Times a shared page has been replaced by a private one. /
2472	STAMCOUNTER StatRZPageReplaceZero; /*< RC/R0: Times the zero page has been replaced by a private one. /
2473	/// @todo STAMCOUNTER StatRZPageHandyAllocs; /*< RC/R0: The number of times we've executed GMMR3AllocateHandyPages. /
2474	STAMCOUNTER StatR3PageReplaceShared; /*< R3: Times a shared page has been replaced by a private one. /
2475	STAMCOUNTER StatR3PageReplaceZero; /*< R3: Times the zero page has been replaced by a private one. /
2476	/// @todo STAMCOUNTER StatR3PageHandyAllocs; /*< R3: The number of times we've executed GMMR3AllocateHandyPages. /
2477
2478	/* RC only: */
2479	STAMCOUNTER StatRCDynMapCacheMisses; /*< RC: The number of dynamic page mapping cache misses /
2480	STAMCOUNTER StatRCDynMapCacheHits; /*< RC: The number of dynamic page mapping cache hits /
2481	STAMCOUNTER StatRCInvlPgConflict; /*< RC: Number of times PGMInvalidatePage() detected a mapping conflict. /
2482	STAMCOUNTER StatRCInvlPgSyncMonCR3; /*< RC: Number of times PGMInvalidatePage() ran into PGM_SYNC_MONITOR_CR3. /
2483
2484	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2485	STAMCOUNTER StatTrackVirgin; /*< The number of first time shadowings. /
2486	STAMCOUNTER StatTrackAliased; /*< The number of times switching to cRef2, i.e. the page is being shadowed by two PTs. /
2487	STAMCOUNTER StatTrackAliasedMany; /*< The number of times we're tracking using cRef2. /
2488	STAMCOUNTER StatTrackAliasedLots; /*< The number of times we're hitting pages which has overflowed cRef2. /
2489	STAMCOUNTER StatTrackOverflows; /*< The number of times the extent list grows to long. /
2490	STAMPROFILE StatTrackDeref; /*< Profiling of SyncPageWorkerTrackDeref (expensive). /
2491	# endif
2492	#endif
2493	} PGM;
2494	/** Pointer to the PGM instance data. */
2495	typedef PGM *PPGM;
2496
2497
2498	/**
2499	* Converts a PGMCPU pointer into a VM pointer.
2500	* @returns Pointer to the VM structure the PGM is part of.
2501	* @param pPGM Pointer to PGMCPU instance data.
2502	*/
2503	#define PGMCPU2VM(pPGM) ( (PVM)((char*)pPGM - pPGM->offVM) )
2504
2505	/**
2506	* Converts a PGMCPU pointer into a PGM pointer.
2507	* @returns Pointer to the VM structure the PGM is part of.
2508	* @param pPGM Pointer to PGMCPU instance data.
2509	*/
2510	#define PGMCPU2PGM(pPGMCpu) ( (PPGM)((char*)pPGMCpu - pPGMCpu->offPGM) )
2511
2512	/**
2513	* PGMCPU Data (part of VMCPU).
2514	*/
2515	typedef struct PGMCPU
2516	{
2517	/** Offset to the VM structure. */
2518	RTINT offVM;
2519	/** Offset to the VMCPU structure. */
2520	RTINT offVCpu;
2521	/** Offset of the PGM structure relative to VMCPU. */
2522	RTINT offPGM;
2523	RTINT uPadding0; /*< structure size alignment. /
2524
2525	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
2526	/** Automatically tracked physical memory mapping set.
2527	* Ring-0 and strict raw-mode builds. */
2528	PGMMAPSET AutoSet;
2529	#endif
2530
2531	/** A20 gate mask.
2532	* Our current approach to A20 emulation is to let REM do it and don't bother
2533	* anywhere else. The interesting Guests will be operating with it enabled anyway.
2534	* But whould need arrise, we'll subject physical addresses to this mask. */
2535	RTGCPHYS GCPhysA20Mask;
2536	/** A20 gate state - boolean! */
2537	bool fA20Enabled;
2538
2539	/** What needs syncing (PGM_SYNC_*).
2540	* This is used to queue operations for PGMSyncCR3, PGMInvalidatePage,
2541	* PGMFlushTLB, and PGMR3Load. */
2542	RTUINT fSyncFlags;
2543
2544	/** The shadow paging mode. */
2545	PGMMODE enmShadowMode;
2546	/** The guest paging mode. */
2547	PGMMODE enmGuestMode;
2548
2549	/** The current physical address representing in the guest CR3 register. */
2550	RTGCPHYS GCPhysCR3;
2551
2552	/** @name 32-bit Guest Paging.
2553	* @{ */
2554	/** The guest's page directory, R3 pointer. */
2555	R3PTRTYPE(PX86PD) pGst32BitPdR3;
2556	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2557	/** The guest's page directory, R0 pointer. */
2558	R0PTRTYPE(PX86PD) pGst32BitPdR0;
2559	#endif
2560	/** The guest's page directory, static RC mapping. */
2561	RCPTRTYPE(PX86PD) pGst32BitPdRC;
2562	/** @} */
2563
2564	/** @name PAE Guest Paging.
2565	* @{ */
2566	/** The guest's page directory pointer table, static RC mapping. */
2567	RCPTRTYPE(PX86PDPT) pGstPaePdptRC;
2568	/** The guest's page directory pointer table, R3 pointer. */
2569	R3PTRTYPE(PX86PDPT) pGstPaePdptR3;
2570	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2571	/** The guest's page directory pointer table, R0 pointer. */
2572	R0PTRTYPE(PX86PDPT) pGstPaePdptR0;
2573	#endif
2574
2575	/** The guest's page directories, R3 pointers.
2576	* These are individual pointers and don't have to be adjecent.
2577	* These don't have to be up-to-date - use pgmGstGetPaePD() to access them. */
2578	R3PTRTYPE(PX86PDPAE) apGstPaePDsR3[4];
2579	/** The guest's page directories, R0 pointers.
2580	* Same restrictions as apGstPaePDsR3. */
2581	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2582	R0PTRTYPE(PX86PDPAE) apGstPaePDsR0[4];
2583	#endif
2584	/** The guest's page directories, static GC mapping.
2585	* Unlike the R3/R0 array the first entry can be accessed as a 2048 entry PD.
2586	* These don't have to be up-to-date - use pgmGstGetPaePD() to access them. */
2587	RCPTRTYPE(PX86PDPAE) apGstPaePDsRC[4];
2588	/** The physical addresses of the guest page directories (PAE) pointed to by apGstPagePDsHC/GC. */
2589	RTGCPHYS aGCPhysGstPaePDs[4];
2590	/** The physical addresses of the monitored guest page directories (PAE). */
2591	RTGCPHYS aGCPhysGstPaePDsMonitored[4];
2592	/** @} */
2593
2594	/** @name AMD64 Guest Paging.
2595	* @{ */
2596	/** The guest's page directory pointer table, R3 pointer. */
2597	R3PTRTYPE(PX86PML4) pGstAmd64Pml4R3;
2598	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2599	/** The guest's page directory pointer table, R0 pointer. */
2600	R0PTRTYPE(PX86PML4) pGstAmd64Pml4R0;
2601	#endif
2602	/** @} */
2603
2604	/** Pointer to the page of the current active CR3 - R3 Ptr. */
2605	R3PTRTYPE(PPGMPOOLPAGE) pShwPageCR3R3;
2606	/** Pointer to the page of the current active CR3 - R0 Ptr. */
2607	R0PTRTYPE(PPGMPOOLPAGE) pShwPageCR3R0;
2608	/** Pointer to the page of the current active CR3 - RC Ptr. */
2609	RCPTRTYPE(PPGMPOOLPAGE) pShwPageCR3RC;
2610	/* The shadow page pool index of the user table as specified during allocation; useful for freeing root pages */
2611	uint32_t iShwUser;
2612	/* The index into the user table (shadowed) as specified during allocation; useful for freeing root pages. */
2613	uint32_t iShwUserTable;
2614	# if HC_ARCH_BITS == 64
2615	RTRCPTR alignment6; /*< structure size alignment. /
2616	# endif
2617	/** @} */
2618
2619	/** @name Function pointers for Shadow paging.
2620	* @{
2621	*/
2622	DECLR3CALLBACKMEMBER(int, pfnR3ShwRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2623	DECLR3CALLBACKMEMBER(int, pfnR3ShwExit,(PVMCPU pVCpu));
2624	DECLR3CALLBACKMEMBER(int, pfnR3ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2625	DECLR3CALLBACKMEMBER(int, pfnR3ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2626
2627	DECLRCCALLBACKMEMBER(int, pfnRCShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2628	DECLRCCALLBACKMEMBER(int, pfnRCShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2629
2630	DECLR0CALLBACKMEMBER(int, pfnR0ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2631	DECLR0CALLBACKMEMBER(int, pfnR0ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2632
2633	/** @} */
2634
2635	/** @name Function pointers for Guest paging.
2636	* @{
2637	*/
2638	DECLR3CALLBACKMEMBER(int, pfnR3GstRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2639	DECLR3CALLBACKMEMBER(int, pfnR3GstExit,(PVMCPU pVCpu));
2640	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2641	DECLR3CALLBACKMEMBER(int, pfnR3GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2642	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2643	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2644	DECLRCCALLBACKMEMBER(int, pfnRCGstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2645	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2646	#if HC_ARCH_BITS == 64
2647	RTRCPTR alignment3; /*< structure size alignment. /
2648	#endif
2649
2650	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2651	DECLR0CALLBACKMEMBER(int, pfnR0GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2652	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2653	/** @} */
2654
2655	/** @name Function pointers for Both Shadow and Guest paging.
2656	* @{
2657	*/
2658	DECLR3CALLBACKMEMBER(int, pfnR3BthRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2659	/* no pfnR3BthTrap0eHandler */
2660	DECLR3CALLBACKMEMBER(int, pfnR3BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2661	DECLR3CALLBACKMEMBER(int, pfnR3BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2662	DECLR3CALLBACKMEMBER(int, pfnR3BthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2663	DECLR3CALLBACKMEMBER(int, pfnR3BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2664	DECLR3CALLBACKMEMBER(int, pfnR3BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2665	DECLR3CALLBACKMEMBER(unsigned, pfnR3BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2666	DECLR3CALLBACKMEMBER(int, pfnR3BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2667	DECLR3CALLBACKMEMBER(int, pfnR3BthUnmapCR3,(PVMCPU pVCpu));
2668
2669	DECLR0CALLBACKMEMBER(int, pfnR0BthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault));
2670	DECLR0CALLBACKMEMBER(int, pfnR0BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2671	DECLR0CALLBACKMEMBER(int, pfnR0BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2672	DECLR0CALLBACKMEMBER(int, pfnR0BthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2673	DECLR0CALLBACKMEMBER(int, pfnR0BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2674	DECLR0CALLBACKMEMBER(int, pfnR0BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2675	DECLR0CALLBACKMEMBER(unsigned, pfnR0BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2676	DECLR0CALLBACKMEMBER(int, pfnR0BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2677	DECLR0CALLBACKMEMBER(int, pfnR0BthUnmapCR3,(PVMCPU pVCpu));
2678
2679	DECLRCCALLBACKMEMBER(int, pfnRCBthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault));
2680	DECLRCCALLBACKMEMBER(int, pfnRCBthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2681	DECLRCCALLBACKMEMBER(int, pfnRCBthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2682	DECLRCCALLBACKMEMBER(int, pfnRCBthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2683	DECLRCCALLBACKMEMBER(int, pfnRCBthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2684	DECLRCCALLBACKMEMBER(int, pfnRCBthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2685	DECLRCCALLBACKMEMBER(unsigned, pfnRCBthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2686	DECLRCCALLBACKMEMBER(int, pfnRCBthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2687	DECLRCCALLBACKMEMBER(int, pfnRCBthUnmapCR3,(PVMCPU pVCpu));
2688	#if HC_ARCH_BITS == 64
2689	RTRCPTR alignment2; /*< structure size alignment. /
2690	#endif
2691	/** @} */
2692
2693	/** @name Release Statistics
2694	* @{ */
2695	/** The number of times the guest has switched mode since last reset or statistics reset. */
2696	STAMCOUNTER cGuestModeChanges;
2697	/** @} */
2698
2699	#ifdef VBOX_WITH_STATISTICS /** @todo move this chunk to the heap. */
2700	/** @name Statistics
2701	* @{ */
2702	/** RC: Which statistic this \#PF should be attributed to. */
2703	RCPTRTYPE(PSTAMPROFILE) pStatTrap0eAttributionRC;
2704	RTRCPTR padding0;
2705	/** R0: Which statistic this \#PF should be attributed to. */
2706	R0PTRTYPE(PSTAMPROFILE) pStatTrap0eAttributionR0;
2707	RTR0PTR padding1;
2708
2709	/* Common */
2710	STAMCOUNTER StatSyncPtPD[X86_PG_ENTRIES]; /*< SyncPT - PD distribution. /
2711	STAMCOUNTER StatSyncPagePD[X86_PG_ENTRIES]; /*< SyncPage - PD distribution. /
2712
2713	/* R0 only: */
2714	STAMCOUNTER StatR0DynMapMigrateInvlPg; /*< R0: invlpg in PGMDynMapMigrateAutoSet. /
2715	STAMPROFILE StatR0DynMapGCPageInl; /*< R0: Calls to pgmR0DynMapGCPageInlined. /
2716	STAMCOUNTER StatR0DynMapGCPageInlHits; /*< R0: Hash table lookup hits. /
2717	STAMCOUNTER StatR0DynMapGCPageInlMisses; /*< R0: Misses that falls back to code common with PGMDynMapHCPage. /
2718	STAMCOUNTER StatR0DynMapGCPageInlRamHits; /*< R0: 1st ram range hits. /
2719	STAMCOUNTER StatR0DynMapGCPageInlRamMisses; /*< R0: 1st ram range misses, takes slow path. /
2720	STAMPROFILE StatR0DynMapHCPageInl; /*< R0: Calls to pgmR0DynMapHCPageInlined. /
2721	STAMCOUNTER StatR0DynMapHCPageInlHits; /*< R0: Hash table lookup hits. /
2722	STAMCOUNTER StatR0DynMapHCPageInlMisses; /*< R0: Misses that falls back to code common with PGMDynMapHCPage. /
2723	STAMPROFILE StatR0DynMapHCPage; /*< R0: Calls to PGMDynMapHCPage. /
2724	STAMCOUNTER StatR0DynMapSetOptimize; /*< R0: Calls to pgmDynMapOptimizeAutoSet. /
2725	STAMCOUNTER StatR0DynMapSetSearchFlushes; /*< R0: Set search restorting to subset flushes. /
2726	STAMCOUNTER StatR0DynMapSetSearchHits; /*< R0: Set search hits. /
2727	STAMCOUNTER StatR0DynMapSetSearchMisses; /*< R0: Set search misses. /
2728	STAMCOUNTER StatR0DynMapPage; /*< R0: Calls to pgmR0DynMapPage. /
2729	STAMCOUNTER StatR0DynMapPageHits0; /*< R0: Hits at iPage+0. /
2730	STAMCOUNTER StatR0DynMapPageHits1; /*< R0: Hits at iPage+1. /
2731	STAMCOUNTER StatR0DynMapPageHits2; /*< R0: Hits at iPage+2. /
2732	STAMCOUNTER StatR0DynMapPageInvlPg; /*< R0: invlpg. /
2733	STAMCOUNTER StatR0DynMapPageSlow; /*< R0: Calls to pgmR0DynMapPageSlow. /
2734	STAMCOUNTER StatR0DynMapPageSlowLoopHits; /*< R0: Hits in the pgmR0DynMapPageSlow search loop. /
2735	STAMCOUNTER StatR0DynMapPageSlowLoopMisses; /*< R0: Misses in the pgmR0DynMapPageSlow search loop. /
2736	//STAMCOUNTER StatR0DynMapPageSlowLostHits; /*< R0: Lost hits. /
2737	STAMCOUNTER StatR0DynMapSubsets; /*< R0: Times PGMDynMapPushAutoSubset was called. /
2738	STAMCOUNTER StatR0DynMapPopFlushes; /*< R0: Times PGMDynMapPopAutoSubset flushes the subset. /
2739	STAMCOUNTER aStatR0DynMapSetSize[11]; /*< R0: Set size distribution. /
2740
2741	/* RZ only: */
2742	STAMPROFILE StatRZTrap0e; /*< RC/R0: PGMTrap0eHandler() profiling. /
2743	STAMPROFILE StatRZTrap0eTimeCheckPageFault;
2744	STAMPROFILE StatRZTrap0eTimeSyncPT;
2745	STAMPROFILE StatRZTrap0eTimeMapping;
2746	STAMPROFILE StatRZTrap0eTimeOutOfSync;
2747	STAMPROFILE StatRZTrap0eTimeHandlers;
2748	STAMPROFILE StatRZTrap0eTime2CSAM; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is CSAM. /
2749	STAMPROFILE StatRZTrap0eTime2DirtyAndAccessed; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is dirty and/or accessed bit emulation. /
2750	STAMPROFILE StatRZTrap0eTime2GuestTrap; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is a guest trap. /
2751	STAMPROFILE StatRZTrap0eTime2HndPhys; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is a physical handler. /
2752	STAMPROFILE StatRZTrap0eTime2HndVirt; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is a virtual handler. /
2753	STAMPROFILE StatRZTrap0eTime2HndUnhandled; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is access outside the monitored areas of a monitored page. /
2754	STAMPROFILE StatRZTrap0eTime2Misc; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is not known. /
2755	STAMPROFILE StatRZTrap0eTime2OutOfSync; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync page. /
2756	STAMPROFILE StatRZTrap0eTime2OutOfSyncHndPhys; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync physical handler page. /
2757	STAMPROFILE StatRZTrap0eTime2OutOfSyncHndVirt; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync virtual handler page. /
2758	STAMPROFILE StatRZTrap0eTime2OutOfSyncHndObs; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an obsolete handler page. /
2759	STAMPROFILE StatRZTrap0eTime2SyncPT; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is lazy syncing of a PT. /
2760	STAMCOUNTER StatRZTrap0eConflicts; /*< RC/R0: The number of times \#PF was caused by an undetected conflict. /
2761	STAMCOUNTER StatRZTrap0eHandlersMapping; /*< RC/R0: Number of traps due to access handlers in mappings. /
2762	STAMCOUNTER StatRZTrap0eHandlersOutOfSync; /*< RC/R0: Number of out-of-sync handled pages. /
2763	STAMCOUNTER StatRZTrap0eHandlersPhysical; /*< RC/R0: Number of traps due to physical access handlers. /
2764	STAMCOUNTER StatRZTrap0eHandlersVirtual; /*< RC/R0: Number of traps due to virtual access handlers. /
2765	STAMCOUNTER StatRZTrap0eHandlersVirtualByPhys; /*< RC/R0: Number of traps due to virtual access handlers found by physical address. /
2766	STAMCOUNTER StatRZTrap0eHandlersVirtualUnmarked;/*< RC/R0: Number of traps due to virtual access handlers found by virtual address (without proper physical flags). /
2767	STAMCOUNTER StatRZTrap0eHandlersUnhandled; /*< RC/R0: Number of traps due to access outside range of monitored page(s). /
2768	STAMCOUNTER StatRZTrap0eHandlersInvalid; /*< RC/R0: Number of traps due to access to invalid physical memory. /
2769	STAMCOUNTER StatRZTrap0eUSNotPresentRead; /*< RC/R0: #PF err kind /
2770	STAMCOUNTER StatRZTrap0eUSNotPresentWrite; /*< RC/R0: #PF err kind /
2771	STAMCOUNTER StatRZTrap0eUSWrite; /*< RC/R0: #PF err kind /
2772	STAMCOUNTER StatRZTrap0eUSReserved; /*< RC/R0: #PF err kind /
2773	STAMCOUNTER StatRZTrap0eUSNXE; /*< RC/R0: #PF err kind /
2774	STAMCOUNTER StatRZTrap0eUSRead; /*< RC/R0: #PF err kind /
2775	STAMCOUNTER StatRZTrap0eSVNotPresentRead; /*< RC/R0: #PF err kind /
2776	STAMCOUNTER StatRZTrap0eSVNotPresentWrite; /*< RC/R0: #PF err kind /
2777	STAMCOUNTER StatRZTrap0eSVWrite; /*< RC/R0: #PF err kind /
2778	STAMCOUNTER StatRZTrap0eSVReserved; /*< RC/R0: #PF err kind /
2779	STAMCOUNTER StatRZTrap0eSNXE; /*< RC/R0: #PF err kind /
2780	STAMCOUNTER StatRZTrap0eGuestPF; /*< RC/R0: Real guest #PFs. /
2781	STAMCOUNTER StatRZTrap0eGuestPFUnh; /*< RC/R0: Real guest #PF ending up at the end of the #PF code. /
2782	STAMCOUNTER StatRZTrap0eGuestPFMapping; /*< RC/R0: Real guest #PF to HMA or other mapping. /
2783	STAMCOUNTER StatRZTrap0eWPEmulInRZ; /*< RC/R0: WP=0 virtualization trap, handled. /
2784	STAMCOUNTER StatRZTrap0eWPEmulToR3; /*< RC/R0: WP=0 virtualization trap, chickened out. /
2785	STAMCOUNTER StatRZTrap0ePD[X86_PG_ENTRIES]; /*< RC/R0: PD distribution of the #PFs. /
2786	STAMCOUNTER StatRZGuestCR3WriteHandled; /*< RC/R0: The number of times WriteHandlerCR3() was successfully called. /
2787	STAMCOUNTER StatRZGuestCR3WriteUnhandled; /*< RC/R0: The number of times WriteHandlerCR3() was called and we had to fall back to the recompiler. /
2788	STAMCOUNTER StatRZGuestCR3WriteConflict; /*< RC/R0: The number of times WriteHandlerCR3() was called and a conflict was detected. /
2789	STAMCOUNTER StatRZGuestROMWriteHandled; /*< RC/R0: The number of times pgmPhysRomWriteHandler() was successfully called. /
2790	STAMCOUNTER StatRZGuestROMWriteUnhandled; /*< RC/R0: The number of times pgmPhysRomWriteHandler() was called and we had to fall back to the recompiler /
2791
2792	/* HC - R3 and (maybe) R0: */
2793
2794	/* RZ & R3: */
2795	STAMPROFILE StatRZSyncCR3; /*< RC/R0: PGMSyncCR3() profiling. /
2796	STAMPROFILE StatRZSyncCR3Handlers; /*< RC/R0: Profiling of the PGMSyncCR3() update handler section. /
2797	STAMCOUNTER StatRZSyncCR3Global; /*< RC/R0: The number of global CR3 syncs. /
2798	STAMCOUNTER StatRZSyncCR3NotGlobal; /*< RC/R0: The number of non-global CR3 syncs. /
2799	STAMCOUNTER StatRZSyncCR3DstCacheHit; /*< RC/R0: The number of times we got some kind of cache hit on a page table. /
2800	STAMCOUNTER StatRZSyncCR3DstFreed; /*< RC/R0: The number of times we've had to free a shadow entry. /
2801	STAMCOUNTER StatRZSyncCR3DstFreedSrcNP; /*< RC/R0: The number of times we've had to free a shadow entry for which the source entry was not present. /
2802	STAMCOUNTER StatRZSyncCR3DstNotPresent; /*< RC/R0: The number of times we've encountered a not present shadow entry for a present guest entry. /
2803	STAMCOUNTER StatRZSyncCR3DstSkippedGlobalPD; /*< RC/R0: The number of times a global page directory wasn't flushed. /
2804	STAMCOUNTER StatRZSyncCR3DstSkippedGlobalPT; /*< RC/R0: The number of times a page table with only global entries wasn't flushed. /
2805	STAMPROFILE StatRZSyncPT; /*< RC/R0: PGMSyncPT() profiling. /
2806	STAMCOUNTER StatRZSyncPTFailed; /*< RC/R0: The number of times PGMSyncPT() failed. /
2807	STAMCOUNTER StatRZSyncPT4K; /*< RC/R0: Number of 4KB syncs. /
2808	STAMCOUNTER StatRZSyncPT4M; /*< RC/R0: Number of 4MB syncs. /
2809	STAMCOUNTER StatRZSyncPagePDNAs; /*< RC/R0: The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit. /
2810	STAMCOUNTER StatRZSyncPagePDOutOfSync; /*< RC/R0: The number of time we've encountered an out-of-sync PD in SyncPage. /
2811	STAMCOUNTER StatRZAccessedPage; /*< RC/R0: The number of pages marked not present for accessed bit emulation. /
2812	STAMPROFILE StatRZDirtyBitTracking; /*< RC/R0: Profiling the dirty bit tracking in CheckPageFault().. /
2813	STAMCOUNTER StatRZDirtyPage; /*< RC/R0: The number of pages marked read-only for dirty bit tracking. /
2814	STAMCOUNTER StatRZDirtyPageBig; /*< RC/R0: The number of pages marked read-only for dirty bit tracking. /
2815	STAMCOUNTER StatRZDirtyPageSkipped; /*< RC/R0: The number of pages already dirty or readonly. /
2816	STAMCOUNTER StatRZDirtyPageTrap; /*< RC/R0: The number of traps generated for dirty bit tracking. /
2817	STAMCOUNTER StatRZDirtyPageStale; /*< RC/R0: The number of traps generated for dirty bit tracking. (stale tlb entries) /
2818	STAMCOUNTER StatRZDirtyTrackRealPF; /*< RC/R0: The number of real pages faults during dirty bit tracking. /
2819	STAMCOUNTER StatRZDirtiedPage; /*< RC/R0: The number of pages marked dirty because of write accesses. /
2820	STAMCOUNTER StatRZPageAlreadyDirty; /*< RC/R0: The number of pages already marked dirty because of write accesses. /
2821	STAMPROFILE StatRZInvalidatePage; /*< RC/R0: PGMInvalidatePage() profiling. /
2822	STAMCOUNTER StatRZInvalidatePage4KBPages; /*< RC/R0: The number of times PGMInvalidatePage() was called for a 4KB page. /
2823	STAMCOUNTER StatRZInvalidatePage4MBPages; /*< RC/R0: The number of times PGMInvalidatePage() was called for a 4MB page. /
2824	STAMCOUNTER StatRZInvalidatePage4MBPagesSkip; /*< RC/R0: The number of times PGMInvalidatePage() skipped a 4MB page. /
2825	STAMCOUNTER StatRZInvalidatePagePDMappings; /*< RC/R0: The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict). /
2826	STAMCOUNTER StatRZInvalidatePagePDNAs; /*< RC/R0: The number of times PGMInvalidatePage() was called for a not accessed page directory. /
2827	STAMCOUNTER StatRZInvalidatePagePDNPs; /*< RC/R0: The number of times PGMInvalidatePage() was called for a not present page directory. /
2828	STAMCOUNTER StatRZInvalidatePagePDOutOfSync; /*< RC/R0: The number of times PGMInvalidatePage() was called for an out of sync page directory. /
2829	STAMCOUNTER StatRZInvalidatePageSkipped; /*< RC/R0: The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3. /
2830	STAMCOUNTER StatRZPageOutOfSyncUser; /*< RC/R0: The number of times user page is out of sync was detected in #PF or VerifyAccessSyncPage. /
2831	STAMCOUNTER StatRZPageOutOfSyncSupervisor; /*< RC/R0: The number of times supervisor page is out of sync was detected in in #PF or VerifyAccessSyncPage. /
2832	STAMPROFILE StatRZPrefetch; /*< RC/R0: PGMPrefetchPage. /
2833	STAMPROFILE StatRZFlushTLB; /*< RC/R0: Profiling of the PGMFlushTLB() body. /
2834	STAMCOUNTER StatRZFlushTLBNewCR3; /*< RC/R0: The number of times PGMFlushTLB was called with a new CR3, non-global. (switch) /
2835	STAMCOUNTER StatRZFlushTLBNewCR3Global; /*< RC/R0: The number of times PGMFlushTLB was called with a new CR3, global. (switch) /
2836	STAMCOUNTER StatRZFlushTLBSameCR3; /*< RC/R0: The number of times PGMFlushTLB was called with the same CR3, non-global. (flush) /
2837	STAMCOUNTER StatRZFlushTLBSameCR3Global; /*< RC/R0: The number of times PGMFlushTLB was called with the same CR3, global. (flush) /
2838	STAMPROFILE StatRZGstModifyPage; /*< RC/R0: Profiling of the PGMGstModifyPage() body /
2839
2840	STAMPROFILE StatR3SyncCR3; /*< R3: PGMSyncCR3() profiling. /
2841	STAMPROFILE StatR3SyncCR3Handlers; /*< R3: Profiling of the PGMSyncCR3() update handler section. /
2842	STAMCOUNTER StatR3SyncCR3Global; /*< R3: The number of global CR3 syncs. /
2843	STAMCOUNTER StatR3SyncCR3NotGlobal; /*< R3: The number of non-global CR3 syncs. /
2844	STAMCOUNTER StatR3SyncCR3DstFreed; /*< R3: The number of times we've had to free a shadow entry. /
2845	STAMCOUNTER StatR3SyncCR3DstFreedSrcNP; /*< R3: The number of times we've had to free a shadow entry for which the source entry was not present. /
2846	STAMCOUNTER StatR3SyncCR3DstNotPresent; /*< R3: The number of times we've encountered a not present shadow entry for a present guest entry. /
2847	STAMCOUNTER StatR3SyncCR3DstSkippedGlobalPD; /*< R3: The number of times a global page directory wasn't flushed. /
2848	STAMCOUNTER StatR3SyncCR3DstSkippedGlobalPT; /*< R3: The number of times a page table with only global entries wasn't flushed. /
2849	STAMCOUNTER StatR3SyncCR3DstCacheHit; /*< R3: The number of times we got some kind of cache hit on a page table. /
2850	STAMPROFILE StatR3SyncPT; /*< R3: PGMSyncPT() profiling. /
2851	STAMCOUNTER StatR3SyncPTFailed; /*< R3: The number of times PGMSyncPT() failed. /
2852	STAMCOUNTER StatR3SyncPT4K; /*< R3: Number of 4KB syncs. /
2853	STAMCOUNTER StatR3SyncPT4M; /*< R3: Number of 4MB syncs. /
2854	STAMCOUNTER StatR3SyncPagePDNAs; /*< R3: The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit. /
2855	STAMCOUNTER StatR3SyncPagePDOutOfSync; /*< R3: The number of time we've encountered an out-of-sync PD in SyncPage. /
2856	STAMCOUNTER StatR3AccessedPage; /*< R3: The number of pages marked not present for accessed bit emulation. /
2857	STAMPROFILE StatR3DirtyBitTracking; /*< R3: Profiling the dirty bit tracking in CheckPageFault(). /
2858	STAMCOUNTER StatR3DirtyPage; /*< R3: The number of pages marked read-only for dirty bit tracking. /
2859	STAMCOUNTER StatR3DirtyPageBig; /*< R3: The number of pages marked read-only for dirty bit tracking. /
2860	STAMCOUNTER StatR3DirtyPageSkipped; /*< R3: The number of pages already dirty or readonly. /
2861	STAMCOUNTER StatR3DirtyPageTrap; /*< R3: The number of traps generated for dirty bit tracking. /
2862	STAMCOUNTER StatR3DirtyTrackRealPF; /*< R3: The number of real pages faults during dirty bit tracking. /
2863	STAMCOUNTER StatR3DirtiedPage; /*< R3: The number of pages marked dirty because of write accesses. /
2864	STAMCOUNTER StatR3PageAlreadyDirty; /*< R3: The number of pages already marked dirty because of write accesses. /
2865	STAMPROFILE StatR3InvalidatePage; /*< R3: PGMInvalidatePage() profiling. /
2866	STAMCOUNTER StatR3InvalidatePage4KBPages; /*< R3: The number of times PGMInvalidatePage() was called for a 4KB page. /
2867	STAMCOUNTER StatR3InvalidatePage4MBPages; /*< R3: The number of times PGMInvalidatePage() was called for a 4MB page. /
2868	STAMCOUNTER StatR3InvalidatePage4MBPagesSkip; /*< R3: The number of times PGMInvalidatePage() skipped a 4MB page. /
2869	STAMCOUNTER StatR3InvalidatePagePDNAs; /*< R3: The number of times PGMInvalidatePage() was called for a not accessed page directory. /
2870	STAMCOUNTER StatR3InvalidatePagePDNPs; /*< R3: The number of times PGMInvalidatePage() was called for a not present page directory. /
2871	STAMCOUNTER StatR3InvalidatePagePDMappings; /*< R3: The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict). /
2872	STAMCOUNTER StatR3InvalidatePagePDOutOfSync; /*< R3: The number of times PGMInvalidatePage() was called for an out of sync page directory. /
2873	STAMCOUNTER StatR3InvalidatePageSkipped; /*< R3: The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3. /
2874	STAMCOUNTER StatR3PageOutOfSyncUser; /*< R3: The number of times user page is out of sync was detected in #PF or VerifyAccessSyncPage. /
2875	STAMCOUNTER StatR3PageOutOfSyncSupervisor; /*< R3: The number of times supervisor page is out of sync was detected in in #PF or VerifyAccessSyncPage. /
2876	STAMPROFILE StatR3Prefetch; /*< R3: PGMPrefetchPage. /
2877	STAMPROFILE StatR3FlushTLB; /*< R3: Profiling of the PGMFlushTLB() body. /
2878	STAMCOUNTER StatR3FlushTLBNewCR3; /*< R3: The number of times PGMFlushTLB was called with a new CR3, non-global. (switch) /
2879	STAMCOUNTER StatR3FlushTLBNewCR3Global; /*< R3: The number of times PGMFlushTLB was called with a new CR3, global. (switch) /
2880	STAMCOUNTER StatR3FlushTLBSameCR3; /*< R3: The number of times PGMFlushTLB was called with the same CR3, non-global. (flush) /
2881	STAMCOUNTER StatR3FlushTLBSameCR3Global; /*< R3: The number of times PGMFlushTLB was called with the same CR3, global. (flush) /
2882	STAMPROFILE StatR3GstModifyPage; /*< R3: Profiling of the PGMGstModifyPage() body /
2883	/** @} */
2884	#endif /* VBOX_WITH_STATISTICS */
2885	} PGMCPU;
2886	/** Pointer to the per-cpu PGM data. */
2887	typedef PGMCPU *PPGMCPU;
2888
2889
2890	/** @name PGM::fSyncFlags Flags
2891	* @{
2892	*/
2893	/** Updates the virtual access handler state bit in PGMPAGE. */
2894	#define PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL RT_BIT(0)
2895	/** Always sync CR3. */
2896	#define PGM_SYNC_ALWAYS RT_BIT(1)
2897	/** Check monitoring on next CR3 (re)load and invalidate page.
2898	* @todo This is obsolete now. Remove after 2.2.0 is branched off. */
2899	#define PGM_SYNC_MONITOR_CR3 RT_BIT(2)
2900	/** Check guest mapping in SyncCR3. */
2901	#define PGM_SYNC_MAP_CR3 RT_BIT(3)
2902	/** Clear the page pool (a light weight flush). */
2903	#define PGM_GLOBAL_SYNC_CLEAR_PGM_POOL_BIT 8
2904	#define PGM_GLOBAL_SYNC_CLEAR_PGM_POOL RT_BIT(PGM_GLOBAL_SYNC_CLEAR_PGM_POOL_BIT)
2905	/** @} */
2906
2907
2908	__BEGIN_DECLS
2909
2910	int pgmLock(PVM pVM);
2911	void pgmUnlock(PVM pVM);
2912
2913	int pgmR3SyncPTResolveConflict(PVM pVM, PPGMMAPPING pMapping, PX86PD pPDSrc, RTGCPTR GCPtrOldMapping);
2914	int pgmR3SyncPTResolveConflictPAE(PVM pVM, PPGMMAPPING pMapping, RTGCPTR GCPtrOldMapping);
2915	PPGMMAPPING pgmGetMapping(PVM pVM, RTGCPTR GCPtr);
2916	void pgmR3MapRelocate(PVM pVM, PPGMMAPPING pMapping, RTGCPTR GCPtrOldMapping, RTGCPTR GCPtrNewMapping);
2917	DECLCALLBACK(void) pgmR3MapInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
2918
2919	void pgmR3HandlerPhysicalUpdateAll(PVM pVM);
2920	bool pgmHandlerPhysicalIsAll(PVM pVM, RTGCPHYS GCPhys);
2921	void pgmHandlerPhysicalResetAliasedPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhysPage);
2922	int pgmHandlerVirtualFindByPhysAddr(PVM pVM, RTGCPHYS GCPhys, PPGMVIRTHANDLER ppVirt, unsigned piPage);
2923	DECLCALLBACK(int) pgmHandlerVirtualResetOne(PAVLROGCPTRNODECORE pNode, void *pvUser);
2924	#if defined(VBOX_STRICT) \|\| defined(LOG_ENABLED)
2925	void pgmHandlerVirtualDumpPhysPages(PVM pVM);
2926	#else
2927	# define pgmHandlerVirtualDumpPhysPages(a) do { } while (0)
2928	#endif
2929	DECLCALLBACK(void) pgmR3InfoHandlers(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
2930
2931
2932	int pgmPhysAllocPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
2933	int pgmPhysPageLoadIntoTlb(PPGM pPGM, RTGCPHYS GCPhys);
2934	int pgmPhysPageLoadIntoTlbWithPage(PPGM pPGM, PPGMPAGE pPage, RTGCPHYS GCPhys);
2935	int pgmPhysPageMakeWritable(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
2936	int pgmPhysPageMakeWritableUnlocked(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
2937	int pgmPhysPageMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAP ppMap, void **ppv);
2938	int pgmPhysPageMapByPageID(PVM pVM, uint32_t idPage, RTHCPHYS HCPhys, void **ppv);
2939	int pgmPhysGCPhys2CCPtrInternal(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv);
2940	int pgmPhysGCPhys2CCPtrInternalReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, const void **ppv);
2941	VMMDECL(int) pgmPhysRomWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser);
2942	#ifdef IN_RING3
2943	void pgmR3PhysRelinkRamRanges(PVM pVM);
2944	int pgmR3PhysRamPreAllocate(PVM pVM);
2945	int pgmR3PhysRamReset(PVM pVM);
2946	int pgmR3PhysRomReset(PVM pVM);
2947	int pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk);
2948
2949	int pgmR3PoolInit(PVM pVM);
2950	void pgmR3PoolRelocate(PVM pVM);
2951	void pgmR3PoolReset(PVM pVM);
2952
2953	#endif /* IN_RING3 */
2954	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2955	int pgmR0DynMapHCPageCommon(PVM pVM, PPGMMAPSET pSet, RTHCPHYS HCPhys, void **ppv);
2956	#endif
2957	int pgmPoolAlloc(PVM pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, uint16_t iUser, uint32_t iUserTable, PPPGMPOOLPAGE ppPage);
2958	void pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint32_t iUserTable);
2959	void pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable);
2960	int pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
2961	void pgmPoolClearAll(PVM pVM);
2962	PPGMPOOLPAGE pgmPoolGetPage(PPGMPOOL pPool, RTHCPHYS HCPhys);
2963	int pgmPoolSyncCR3(PVM pVM);
2964	int pgmPoolTrackFlushGCPhys(PVM pVM, PPGMPAGE pPhysPage, bool *pfFlushTLBs);
2965	uint16_t pgmPoolTrackPhysExtAddref(PVM pVM, uint16_t u16, uint16_t iShwPT);
2966	void pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPoolPage, PPGMPAGE pPhysPage);
2967	#ifdef PGMPOOL_WITH_MONITORING
2968	void pgmPoolMonitorChainChanging(PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault, CTXTYPE(RTGCPTR, RTHCPTR, RTGCPTR) pvAddress, PDISCPUSTATE pCpu);
2969	int pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
2970	void pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
2971	#endif
2972
2973	int pgmR3ExitShadowModeBeforePoolFlush(PVM pVM, PVMCPU pVCpu);
2974	int pgmR3ReEnterShadowModeAfterPoolFlush(PVM pVM, PVMCPU pVCpu);
2975
2976	void pgmMapSetShadowPDEs(PVM pVM, PPGMMAPPING pMap, unsigned iNewPDE);
2977	void pgmMapClearShadowPDEs(PVM pVM, PPGMPOOLPAGE pShwPageCR3, PPGMMAPPING pMap, unsigned iOldPDE, bool fDeactivateCR3);
2978	int pgmMapActivateCR3(PVM pVM, PPGMPOOLPAGE pShwPageCR3);
2979	int pgmMapDeactivateCR3(PVM pVM, PPGMPOOLPAGE pShwPageCR3);
2980
2981	int pgmShwSyncPaePDPtr(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDPE pGstPdpe, PX86PDPAE *ppPD);
2982	#ifndef IN_RC
2983	int pgmShwSyncLongModePDPtr(PVMCPU pVCpu, RTGCPTR64 GCPtr, PX86PML4E pGstPml4e, PX86PDPE pGstPdpe, PX86PDPAE *ppPD);
2984	#endif
2985	int pgmShwGetEPTPDPtr(PVMCPU pVCpu, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD);
2986
2987	PX86PD pgmGstLazyMap32BitPD(PPGMCPU pPGM);
2988	PX86PDPT pgmGstLazyMapPaePDPT(PPGMCPU pPGM);
2989	PX86PDPAE pgmGstLazyMapPaePD(PPGMCPU pPGM, uint32_t iPdpt);
2990	PX86PML4 pgmGstLazyMapPml4(PPGMCPU pPGM);
2991
2992	__END_DECLS
2993
2994
2995	/**
2996	* Gets the PGMRAMRANGE structure for a guest page.
2997	*
2998	* @returns Pointer to the RAM range on success.
2999	* @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
3000	*
3001	* @param pPGM PGM handle.
3002	* @param GCPhys The GC physical address.
3003	*/
3004	DECLINLINE(PPGMRAMRANGE) pgmPhysGetRange(PPGM pPGM, RTGCPHYS GCPhys)
3005	{
3006	/*
3007	* Optimize for the first range.
3008	*/
3009	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3010	RTGCPHYS off = GCPhys - pRam->GCPhys;
3011	if (RT_UNLIKELY(off >= pRam->cb))
3012	{
3013	do
3014	{
3015	pRam = pRam->CTX_SUFF(pNext);
3016	if (RT_UNLIKELY(!pRam))
3017	break;
3018	off = GCPhys - pRam->GCPhys;
3019	} while (off >= pRam->cb);
3020	}
3021	return pRam;
3022	}
3023
3024
3025	/**
3026	* Gets the PGMPAGE structure for a guest page.
3027	*
3028	* @returns Pointer to the page on success.
3029	* @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
3030	*
3031	* @param pPGM PGM handle.
3032	* @param GCPhys The GC physical address.
3033	*/
3034	DECLINLINE(PPGMPAGE) pgmPhysGetPage(PPGM pPGM, RTGCPHYS GCPhys)
3035	{
3036	/*
3037	* Optimize for the first range.
3038	*/
3039	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3040	RTGCPHYS off = GCPhys - pRam->GCPhys;
3041	if (RT_UNLIKELY(off >= pRam->cb))
3042	{
3043	do
3044	{
3045	pRam = pRam->CTX_SUFF(pNext);
3046	if (RT_UNLIKELY(!pRam))
3047	return NULL;
3048	off = GCPhys - pRam->GCPhys;
3049	} while (off >= pRam->cb);
3050	}
3051	return &pRam->aPages[off >> PAGE_SHIFT];
3052	}
3053
3054
3055	/**
3056	* Gets the PGMPAGE structure for a guest page.
3057	*
3058	* Old Phys code: Will make sure the page is present.
3059	*
3060	* @returns VBox status code.
3061	* @retval VINF_SUCCESS and a valid *ppPage on success.
3062	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if the address isn't valid.
3063	*
3064	* @param pPGM PGM handle.
3065	* @param GCPhys The GC physical address.
3066	* @param ppPage Where to store the page poitner on success.
3067	*/
3068	DECLINLINE(int) pgmPhysGetPageEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage)
3069	{
3070	/*
3071	* Optimize for the first range.
3072	*/
3073	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3074	RTGCPHYS off = GCPhys - pRam->GCPhys;
3075	if (RT_UNLIKELY(off >= pRam->cb))
3076	{
3077	do
3078	{
3079	pRam = pRam->CTX_SUFF(pNext);
3080	if (RT_UNLIKELY(!pRam))
3081	{
3082	ppPage = NULL; / avoid incorrect and very annoying GCC warnings */
3083	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
3084	}
3085	off = GCPhys - pRam->GCPhys;
3086	} while (off >= pRam->cb);
3087	}
3088	*ppPage = &pRam->aPages[off >> PAGE_SHIFT];
3089	return VINF_SUCCESS;
3090	}
3091
3092
3093
3094
3095	/**
3096	* Gets the PGMPAGE structure for a guest page.
3097	*
3098	* Old Phys code: Will make sure the page is present.
3099	*
3100	* @returns VBox status code.
3101	* @retval VINF_SUCCESS and a valid *ppPage on success.
3102	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if the address isn't valid.
3103	*
3104	* @param pPGM PGM handle.
3105	* @param GCPhys The GC physical address.
3106	* @param ppPage Where to store the page poitner on success.
3107	* @param ppRamHint Where to read and store the ram list hint.
3108	* The caller initializes this to NULL before the call.
3109	*/
3110	DECLINLINE(int) pgmPhysGetPageWithHintEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRamHint)
3111	{
3112	RTGCPHYS off;
3113	PPGMRAMRANGE pRam = *ppRamHint;
3114	if ( !pRam
3115	\|\| RT_UNLIKELY((off = GCPhys - pRam->GCPhys) >= pRam->cb))
3116	{
3117	pRam = pPGM->CTX_SUFF(pRamRanges);
3118	off = GCPhys - pRam->GCPhys;
3119	if (RT_UNLIKELY(off >= pRam->cb))
3120	{
3121	do
3122	{
3123	pRam = pRam->CTX_SUFF(pNext);
3124	if (RT_UNLIKELY(!pRam))
3125	{
3126	ppPage = NULL; / Kill the incorrect and extremely annoying GCC warnings. */
3127	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
3128	}
3129	off = GCPhys - pRam->GCPhys;
3130	} while (off >= pRam->cb);
3131	}
3132	*ppRamHint = pRam;
3133	}
3134	*ppPage = &pRam->aPages[off >> PAGE_SHIFT];
3135	return VINF_SUCCESS;
3136	}
3137
3138
3139	/**
3140	* Gets the PGMPAGE structure for a guest page together with the PGMRAMRANGE.
3141	*
3142	* @returns Pointer to the page on success.
3143	* @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
3144	*
3145	* @param pPGM PGM handle.
3146	* @param GCPhys The GC physical address.
3147	* @param ppRam Where to store the pointer to the PGMRAMRANGE.
3148	*/
3149	DECLINLINE(PPGMPAGE) pgmPhysGetPageAndRange(PPGM pPGM, RTGCPHYS GCPhys, PPGMRAMRANGE *ppRam)
3150	{
3151	/*
3152	* Optimize for the first range.
3153	*/
3154	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3155	RTGCPHYS off = GCPhys - pRam->GCPhys;
3156	if (RT_UNLIKELY(off >= pRam->cb))
3157	{
3158	do
3159	{
3160	pRam = pRam->CTX_SUFF(pNext);
3161	if (RT_UNLIKELY(!pRam))
3162	return NULL;
3163	off = GCPhys - pRam->GCPhys;
3164	} while (off >= pRam->cb);
3165	}
3166	*ppRam = pRam;
3167	return &pRam->aPages[off >> PAGE_SHIFT];
3168	}
3169
3170
3171	/**
3172	* Gets the PGMPAGE structure for a guest page together with the PGMRAMRANGE.
3173	*
3174	* @returns Pointer to the page on success.
3175	* @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
3176	*
3177	* @param pPGM PGM handle.
3178	* @param GCPhys The GC physical address.
3179	* @param ppPage Where to store the pointer to the PGMPAGE structure.
3180	* @param ppRam Where to store the pointer to the PGMRAMRANGE structure.
3181	*/
3182	DECLINLINE(int) pgmPhysGetPageAndRangeEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam)
3183	{
3184	/*
3185	* Optimize for the first range.
3186	*/
3187	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3188	RTGCPHYS off = GCPhys - pRam->GCPhys;
3189	if (RT_UNLIKELY(off >= pRam->cb))
3190	{
3191	do
3192	{
3193	pRam = pRam->CTX_SUFF(pNext);
3194	if (RT_UNLIKELY(!pRam))
3195	{
3196	ppRam = NULL; / Shut up silly GCC warnings. */
3197	ppPage = NULL; / ditto */
3198	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
3199	}
3200	off = GCPhys - pRam->GCPhys;
3201	} while (off >= pRam->cb);
3202	}
3203	*ppRam = pRam;
3204	*ppPage = &pRam->aPages[off >> PAGE_SHIFT];
3205	return VINF_SUCCESS;
3206	}
3207
3208
3209	/**
3210	* Convert GC Phys to HC Phys.
3211	*
3212	* @returns VBox status.
3213	* @param pPGM PGM handle.
3214	* @param GCPhys The GC physical address.
3215	* @param pHCPhys Where to store the corresponding HC physical address.
3216	*
3217	* @deprecated Doesn't deal with zero, shared or write monitored pages.
3218	* Avoid when writing new code!
3219	*/
3220	DECLINLINE(int) pgmRamGCPhys2HCPhys(PPGM pPGM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys)
3221	{
3222	PPGMPAGE pPage;
3223	int rc = pgmPhysGetPageEx(pPGM, GCPhys, &pPage);
3224	if (RT_FAILURE(rc))
3225	return rc;
3226	*pHCPhys = PGM_PAGE_GET_HCPHYS(pPage) \| (GCPhys & PAGE_OFFSET_MASK);
3227	return VINF_SUCCESS;
3228	}
3229
3230	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3231
3232	/**
3233	* Inlined version of the ring-0 version of PGMDynMapHCPage that
3234	* optimizes access to pages already in the set.
3235	*
3236	* @returns VINF_SUCCESS. Will bail out to ring-3 on failure.
3237	* @param pPGM Pointer to the PVM instance data.
3238	* @param HCPhys The physical address of the page.
3239	* @param ppv Where to store the mapping address.
3240	*/
3241	DECLINLINE(int) pgmR0DynMapHCPageInlined(PPGM pPGM, RTHCPHYS HCPhys, void **ppv)
3242	{
3243	PVM pVM = PGM2VM(pPGM);
3244	PPGMCPU pPGMCPU = (PPGMCPU)((uint8_t )VMMGetCpu(pVM) + pPGM->offVCpuPGM); / very pretty ;-) */
3245	PPGMMAPSET pSet = &pPGMCPU->AutoSet;
3246
3247	STAM_PROFILE_START(&pPGMCPU->StatR0DynMapHCPageInl, a);
3248	Assert(!(HCPhys & PAGE_OFFSET_MASK));
3249	Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
3250
3251	unsigned iHash = PGMMAPSET_HASH(HCPhys);
3252	unsigned iEntry = pSet->aiHashTable[iHash];
3253	if ( iEntry < pSet->cEntries
3254	&& pSet->aEntries[iEntry].HCPhys == HCPhys)
3255	{
3256	*ppv = pSet->aEntries[iEntry].pvPage;
3257	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapHCPageInlHits);
3258	}
3259	else
3260	{
3261	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapHCPageInlMisses);
3262	pgmR0DynMapHCPageCommon(pVM, pSet, HCPhys, ppv);
3263	}
3264
3265	STAM_PROFILE_STOP(&pPGMCPU->StatR0DynMapHCPageInl, a);
3266	return VINF_SUCCESS;
3267	}
3268
3269
3270	/**
3271	* Inlined version of the ring-0 version of PGMDynMapGCPage that optimizes
3272	* access to pages already in the set.
3273	*
3274	* @returns See PGMDynMapGCPage.
3275	* @param pPGM Pointer to the PVM instance data.
3276	* @param HCPhys The physical address of the page.
3277	* @param ppv Where to store the mapping address.
3278	*/
3279	DECLINLINE(int) pgmR0DynMapGCPageInlined(PPGM pPGM, RTGCPHYS GCPhys, void **ppv)
3280	{
3281	PVM pVM = PGM2VM(pPGM);
3282	PPGMCPU pPGMCPU = (PPGMCPU)((uint8_t )VMMGetCpu(pVM) + pPGM->offVCpuPGM); / very pretty ;-) */
3283
3284	STAM_PROFILE_START(&pPGMCPU->StatR0DynMapGCPageInl, a);
3285	Assert(!(GCPhys & PAGE_OFFSET_MASK));
3286
3287	/*
3288	* Get the ram range.
3289	*/
3290	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3291	RTGCPHYS off = GCPhys - pRam->GCPhys;
3292	if (RT_UNLIKELY(off >= pRam->cb
3293	/** @todo \|\| page state stuff */))
3294	{
3295	/* This case is not counted into StatR0DynMapGCPageInl. */
3296	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlRamMisses);
3297	return PGMDynMapGCPage(pVM, GCPhys, ppv);
3298	}
3299
3300	RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(&pRam->aPages[off >> PAGE_SHIFT]);
3301	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlRamHits);
3302
3303	/*
3304	* pgmR0DynMapHCPageInlined with out stats.
3305	*/
3306	PPGMMAPSET pSet = &pPGMCPU->AutoSet;
3307	Assert(!(HCPhys & PAGE_OFFSET_MASK));
3308	Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
3309
3310	unsigned iHash = PGMMAPSET_HASH(HCPhys);
3311	unsigned iEntry = pSet->aiHashTable[iHash];
3312	if ( iEntry < pSet->cEntries
3313	&& pSet->aEntries[iEntry].HCPhys == HCPhys)
3314	{
3315	*ppv = pSet->aEntries[iEntry].pvPage;
3316	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlHits);
3317	}
3318	else
3319	{
3320	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlMisses);
3321	pgmR0DynMapHCPageCommon(pVM, pSet, HCPhys, ppv);
3322	}
3323
3324	STAM_PROFILE_STOP(&pPGMCPU->StatR0DynMapGCPageInl, a);
3325	return VINF_SUCCESS;
3326	}
3327
3328	#endif /* VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
3329	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
3330
3331	/**
3332	* Maps the page into current context (RC and maybe R0).
3333	*
3334	* @returns pointer to the mapping.
3335	* @param pVM Pointer to the PGM instance data.
3336	* @param pPage The page.
3337	*/
3338	DECLINLINE(void *) pgmPoolMapPageInlined(PPGM pPGM, PPGMPOOLPAGE pPage)
3339	{
3340	if (pPage->idx >= PGMPOOL_IDX_FIRST)
3341	{
3342	Assert(pPage->idx < pPGM->CTX_SUFF(pPool)->cCurPages);
3343	void *pv;
3344	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3345	pgmR0DynMapHCPageInlined(pPGM, pPage->Core.Key, &pv);
3346	# else
3347	PGMDynMapHCPage(PGM2VM(pPGM), pPage->Core.Key, &pv);
3348	# endif
3349	return pv;
3350	}
3351	AssertFatalMsgFailed(("pgmPoolMapPageInlined invalid page index %x\n", pPage->idx));
3352	}
3353
3354	/**
3355	* Temporarily maps one host page specified by HC physical address, returning
3356	* pointer within the page.
3357	*
3358	* Be WARNED that the dynamic page mapping area is small, 8 pages, thus the space is
3359	* reused after 8 mappings (or perhaps a few more if you score with the cache).
3360	*
3361	* @returns The address corresponding to HCPhys.
3362	* @param pPGM Pointer to the PVM instance data.
3363	* @param HCPhys HC Physical address of the page.
3364	*/
3365	DECLINLINE(void *) pgmDynMapHCPageOff(PPGM pPGM, RTHCPHYS HCPhys)
3366	{
3367	void *pv;
3368	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3369	pgmR0DynMapHCPageInlined(pPGM, HCPhys & ~(RTHCPHYS)PAGE_OFFSET_MASK, &pv);
3370	# else
3371	PGMDynMapHCPage(PGM2VM(pPGM), HCPhys & ~(RTHCPHYS)PAGE_OFFSET_MASK, &pv);
3372	# endif
3373	pv = (void *)((uintptr_t)pv \| (HCPhys & PAGE_OFFSET_MASK));
3374	return pv;
3375	}
3376
3377	#endif /* VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 \|\| IN_RC */
3378	#ifndef IN_RC
3379
3380	/**
3381	* Queries the Physical TLB entry for a physical guest page,
3382	* attempting to load the TLB entry if necessary.
3383	*
3384	* @returns VBox status code.
3385	* @retval VINF_SUCCESS on success
3386	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3387	*
3388	* @param pPGM The PGM instance handle.
3389	* @param GCPhys The address of the guest page.
3390	* @param ppTlbe Where to store the pointer to the TLB entry.
3391	*/
3392	DECLINLINE(int) pgmPhysPageQueryTlbe(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe)
3393	{
3394	int rc;
3395	PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
3396	if (pTlbe->GCPhys == (GCPhys & X86_PTE_PAE_PG_MASK))
3397	{
3398	STAM_COUNTER_INC(&pPGM->CTX_MID_Z(Stat,PageMapTlbHits));
3399	rc = VINF_SUCCESS;
3400	}
3401	else
3402	rc = pgmPhysPageLoadIntoTlb(pPGM, GCPhys);
3403	*ppTlbe = pTlbe;
3404	return rc;
3405	}
3406
3407
3408	/**
3409	* Queries the Physical TLB entry for a physical guest page,
3410	* attempting to load the TLB entry if necessary.
3411	*
3412	* @returns VBox status code.
3413	* @retval VINF_SUCCESS on success
3414	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3415	*
3416	* @param pPGM The PGM instance handle.
3417	* @param pPage Pointer to the PGMPAGE structure corresponding to
3418	* GCPhys.
3419	* @param GCPhys The address of the guest page.
3420	* @param ppTlbe Where to store the pointer to the TLB entry.
3421	*/
3422	DECLINLINE(int) pgmPhysPageQueryTlbeWithPage(PPGM pPGM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe)
3423	{
3424	int rc;
3425	PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
3426	if (pTlbe->GCPhys == (GCPhys & X86_PTE_PAE_PG_MASK))
3427	{
3428	STAM_COUNTER_INC(&pPGM->CTX_MID_Z(Stat,PageMapTlbHits));
3429	rc = VINF_SUCCESS;
3430	}
3431	else
3432	rc = pgmPhysPageLoadIntoTlbWithPage(pPGM, pPage, GCPhys);
3433	*ppTlbe = pTlbe;
3434	return rc;
3435	}
3436
3437	#endif /* !IN_RC */
3438
3439	/**
3440	* Calculated the guest physical address of the large (4 MB) page in 32 bits paging mode.
3441	* Takes PSE-36 into account.
3442	*
3443	* @returns guest physical address
3444	* @param pPGM Pointer to the PGM instance data.
3445	* @param Pde Guest Pde
3446	*/
3447	DECLINLINE(RTGCPHYS) pgmGstGet4MBPhysPage(PPGM pPGM, X86PDE Pde)
3448	{
3449	RTGCPHYS GCPhys = Pde.u & X86_PDE4M_PG_MASK;
3450	GCPhys \|= (RTGCPHYS)Pde.b.u8PageNoHigh << 32;
3451
3452	return GCPhys & pPGM->GCPhys4MBPSEMask;
3453	}
3454
3455
3456	/**
3457	* Gets the page directory entry for the specified address (32-bit paging).
3458	*
3459	* @returns The page directory entry in question.
3460	* @param pPGM Pointer to the PGM instance data.
3461	* @param GCPtr The address.
3462	*/
3463	DECLINLINE(X86PDE) pgmGstGet32bitPDE(PPGMCPU pPGM, RTGCPTR GCPtr)
3464	{
3465	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3466	PCX86PD pGuestPD = NULL;
3467	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPD);
3468	if (RT_FAILURE(rc))
3469	{
3470	X86PDE ZeroPde = {0};
3471	AssertMsgFailedReturn(("%Rrc\n", rc), ZeroPde);
3472	}
3473	#else
3474	PX86PD pGuestPD = pPGM->CTX_SUFF(pGst32BitPd);
3475	# ifdef IN_RING3
3476	if (!pGuestPD)
3477	pGuestPD = pgmGstLazyMap32BitPD(pPGM);
3478	# endif
3479	#endif
3480	return pGuestPD->a[GCPtr >> X86_PD_SHIFT];
3481	}
3482
3483
3484	/**
3485	* Gets the address of a specific page directory entry (32-bit paging).
3486	*
3487	* @returns Pointer the page directory entry in question.
3488	* @param pPGM Pointer to the PGM instance data.
3489	* @param GCPtr The address.
3490	*/
3491	DECLINLINE(PX86PDE) pgmGstGet32bitPDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
3492	{
3493	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3494	PX86PD pGuestPD = NULL;
3495	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPD);
3496	AssertRCReturn(rc, NULL);
3497	#else
3498	PX86PD pGuestPD = pPGM->CTX_SUFF(pGst32BitPd);
3499	# ifdef IN_RING3
3500	if (!pGuestPD)
3501	pGuestPD = pgmGstLazyMap32BitPD(pPGM);
3502	# endif
3503	#endif
3504	return &pGuestPD->a[GCPtr >> X86_PD_SHIFT];
3505	}
3506
3507
3508	/**
3509	* Gets the address the guest page directory (32-bit paging).
3510	*
3511	* @returns Pointer the page directory entry in question.
3512	* @param pPGM Pointer to the PGM instance data.
3513	*/
3514	DECLINLINE(PX86PD) pgmGstGet32bitPDPtr(PPGMCPU pPGM)
3515	{
3516	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3517	PX86PD pGuestPD = NULL;
3518	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPD);
3519	AssertRCReturn(rc, NULL);
3520	#else
3521	PX86PD pGuestPD = pPGM->CTX_SUFF(pGst32BitPd);
3522	# ifdef IN_RING3
3523	if (!pGuestPD)
3524	pGuestPD = pgmGstLazyMap32BitPD(pPGM);
3525	# endif
3526	#endif
3527	return pGuestPD;
3528	}
3529
3530
3531	/**
3532	* Gets the guest page directory pointer table.
3533	*
3534	* @returns Pointer to the page directory in question.
3535	* @returns NULL if the page directory is not present or on an invalid page.
3536	* @param pPGM Pointer to the PGM instance data.
3537	*/
3538	DECLINLINE(PX86PDPT) pgmGstGetPaePDPTPtr(PPGMCPU pPGM)
3539	{
3540	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3541	PX86PDPT pGuestPDPT = NULL;
3542	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPDPT);
3543	AssertRCReturn(rc, NULL);
3544	#else
3545	PX86PDPT pGuestPDPT = pPGM->CTX_SUFF(pGstPaePdpt);
3546	# ifdef IN_RING3
3547	if (!pGuestPDPT)
3548	pGuestPDPT = pgmGstLazyMapPaePDPT(pPGM);
3549	# endif
3550	#endif
3551	return pGuestPDPT;
3552	}
3553
3554
3555	/**
3556	* Gets the guest page directory pointer table entry for the specified address.
3557	*
3558	* @returns Pointer to the page directory in question.
3559	* @returns NULL if the page directory is not present or on an invalid page.
3560	* @param pPGM Pointer to the PGM instance data.
3561	* @param GCPtr The address.
3562	*/
3563	DECLINLINE(PX86PDPE) pgmGstGetPaePDPEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
3564	{
3565	AssertGCPtr32(GCPtr);
3566
3567	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3568	PX86PDPT pGuestPDPT = 0;
3569	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPDPT);
3570	AssertRCReturn(rc, 0);
3571	#else
3572	PX86PDPT pGuestPDPT = pPGM->CTX_SUFF(pGstPaePdpt);
3573	# ifdef IN_RING3
3574	if (!pGuestPDPT)
3575	pGuestPDPT = pgmGstLazyMapPaePDPT(pPGM);
3576	# endif
3577	#endif
3578	return &pGuestPDPT->a[(GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE];
3579	}
3580
3581
3582	/**
3583	* Gets the page directory for the specified address.
3584	*
3585	* @returns Pointer to the page directory in question.
3586	* @returns NULL if the page directory is not present or on an invalid page.
3587	* @param pPGM Pointer to the PGM instance data.
3588	* @param GCPtr The address.
3589	*/
3590	DECLINLINE(PX86PDPAE) pgmGstGetPaePD(PPGMCPU pPGM, RTGCPTR GCPtr)
3591	{
3592	AssertGCPtr32(GCPtr);
3593
3594	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
3595	AssertReturn(pGuestPDPT, NULL);
3596	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
3597	if (pGuestPDPT->a[iPdpt].n.u1Present)
3598	{
3599	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3600	PX86PDPAE pGuestPD = NULL;
3601	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
3602	AssertRCReturn(rc, NULL);
3603	#else
3604	PX86PDPAE pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
3605	if ( !pGuestPD
3606	\|\| (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
3607	pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
3608	#endif
3609	return pGuestPD;
3610	/* returning NULL is ok if we assume it's just an invalid page of some kind emulated as all 0s. (not quite true) */
3611	}
3612	return NULL;
3613	}
3614
3615
3616	/**
3617	* Gets the page directory entry for the specified address.
3618	*
3619	* @returns Pointer to the page directory entry in question.
3620	* @returns NULL if the page directory is not present or on an invalid page.
3621	* @param pPGM Pointer to the PGM instance data.
3622	* @param GCPtr The address.
3623	*/
3624	DECLINLINE(PX86PDEPAE) pgmGstGetPaePDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
3625	{
3626	AssertGCPtr32(GCPtr);
3627
3628	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
3629	AssertReturn(pGuestPDPT, NULL);
3630	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
3631	if (pGuestPDPT->a[iPdpt].n.u1Present)
3632	{
3633	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3634	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3635	PX86PDPAE pGuestPD = NULL;
3636	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
3637	AssertRCReturn(rc, NULL);
3638	#else
3639	PX86PDPAE pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
3640	if ( !pGuestPD
3641	\|\| (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
3642	pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
3643	#endif
3644	return &pGuestPD->a[iPD];
3645	/* returning NIL_RTGCPHYS is ok if we assume it's just an invalid page or something which we'll emulate as all 0s. (not quite true) */
3646	}
3647	return NULL;
3648	}
3649
3650
3651	/**
3652	* Gets the page directory entry for the specified address.
3653	*
3654	* @returns The page directory entry in question.
3655	* @returns A non-present entry if the page directory is not present or on an invalid page.
3656	* @param pPGM Pointer to the PGM instance data.
3657	* @param GCPtr The address.
3658	*/
3659	DECLINLINE(X86PDEPAE) pgmGstGetPaePDE(PPGMCPU pPGM, RTGCPTR GCPtr)
3660	{
3661	AssertGCPtr32(GCPtr);
3662	X86PDEPAE ZeroPde = {0};
3663	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
3664	if (RT_LIKELY(pGuestPDPT))
3665	{
3666	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
3667	if (pGuestPDPT->a[iPdpt].n.u1Present)
3668	{
3669	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3670	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3671	PX86PDPAE pGuestPD = NULL;
3672	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
3673	AssertRCReturn(rc, ZeroPde);
3674	#else
3675	PX86PDPAE pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
3676	if ( !pGuestPD
3677	\|\| (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
3678	pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
3679	#endif
3680	return pGuestPD->a[iPD];
3681	}
3682	}
3683	return ZeroPde;
3684	}
3685
3686
3687	/**
3688	* Gets the page directory pointer table entry for the specified address
3689	* and returns the index into the page directory
3690	*
3691	* @returns Pointer to the page directory in question.
3692	* @returns NULL if the page directory is not present or on an invalid page.
3693	* @param pPGM Pointer to the PGM instance data.
3694	* @param GCPtr The address.
3695	* @param piPD Receives the index into the returned page directory
3696	* @param pPdpe Receives the page directory pointer entry. Optional.
3697	*/
3698	DECLINLINE(PX86PDPAE) pgmGstGetPaePDPtr(PPGMCPU pPGM, RTGCPTR GCPtr, unsigned *piPD, PX86PDPE pPdpe)
3699	{
3700	AssertGCPtr32(GCPtr);
3701
3702	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
3703	AssertReturn(pGuestPDPT, NULL);
3704	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
3705	if (pPdpe)
3706	*pPdpe = pGuestPDPT->a[iPdpt];
3707	if (pGuestPDPT->a[iPdpt].n.u1Present)
3708	{
3709	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3710	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3711	PX86PDPAE pGuestPD = NULL;
3712	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
3713	AssertRCReturn(rc, NULL);
3714	#else
3715	PX86PDPAE pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
3716	if ( !pGuestPD
3717	\|\| (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
3718	pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
3719	#endif
3720	*piPD = iPD;
3721	return pGuestPD;
3722	/* returning NIL_RTGCPHYS is ok if we assume it's just an invalid page of some kind emulated as all 0s. */
3723	}
3724	return NULL;
3725	}
3726
3727	#ifndef IN_RC
3728
3729	/**
3730	* Gets the page map level-4 pointer for the guest.
3731	*
3732	* @returns Pointer to the PML4 page.
3733	* @param pPGM Pointer to the PGM instance data.
3734	*/
3735	DECLINLINE(PX86PML4) pgmGstGetLongModePML4Ptr(PPGMCPU pPGM)
3736	{
3737	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3738	PX86PML4 pGuestPml4;
3739	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPml4);
3740	AssertRCReturn(rc, NULL);
3741	#else
3742	PX86PML4 pGuestPml4 = pPGM->CTX_SUFF(pGstAmd64Pml4);
3743	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R3
3744	if (!pGuestPml4)
3745	pGuestPml4 = pgmGstLazyMapPml4(pPGM);
3746	# endif
3747	Assert(pGuestPml4);
3748	#endif
3749	return pGuestPml4;
3750	}
3751
3752
3753	/**
3754	* Gets the pointer to a page map level-4 entry.
3755	*
3756	* @returns Pointer to the PML4 entry.
3757	* @param pPGM Pointer to the PGM instance data.
3758	* @param iPml4 The index.
3759	*/
3760	DECLINLINE(PX86PML4E) pgmGstGetLongModePML4EPtr(PPGMCPU pPGM, unsigned int iPml4)
3761	{
3762	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3763	PX86PML4 pGuestPml4;
3764	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPml4);
3765	AssertRCReturn(rc, NULL);
3766	#else
3767	PX86PML4 pGuestPml4 = pPGM->CTX_SUFF(pGstAmd64Pml4);
3768	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R3
3769	if (!pGuestPml4)
3770	pGuestPml4 = pgmGstLazyMapPml4(pPGM);
3771	# endif
3772	Assert(pGuestPml4);
3773	#endif
3774	return &pGuestPml4->a[iPml4];
3775	}
3776
3777
3778	/**
3779	* Gets a page map level-4 entry.
3780	*
3781	* @returns The PML4 entry.
3782	* @param pPGM Pointer to the PGM instance data.
3783	* @param iPml4 The index.
3784	*/
3785	DECLINLINE(X86PML4E) pgmGstGetLongModePML4E(PPGMCPU pPGM, unsigned int iPml4)
3786	{
3787	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3788	PX86PML4 pGuestPml4;
3789	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPml4);
3790	if (RT_FAILURE(rc))
3791	{
3792	X86PML4E ZeroPml4e = {0};
3793	AssertMsgFailedReturn(("%Rrc\n", rc), ZeroPml4e);
3794	}
3795	#else
3796	PX86PML4 pGuestPml4 = pPGM->CTX_SUFF(pGstAmd64Pml4);
3797	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R3
3798	if (!pGuestPml4)
3799	pGuestPml4 = pgmGstLazyMapPml4(pPGM);
3800	# endif
3801	Assert(pGuestPml4);
3802	#endif
3803	return pGuestPml4->a[iPml4];
3804	}
3805
3806
3807	/**
3808	* Gets the page directory pointer entry for the specified address.
3809	*
3810	* @returns Pointer to the page directory pointer entry in question.
3811	* @returns NULL if the page directory is not present or on an invalid page.
3812	* @param pPGM Pointer to the PGM instance data.
3813	* @param GCPtr The address.
3814	* @param ppPml4e Page Map Level-4 Entry (out)
3815	*/
3816	DECLINLINE(PX86PDPE) pgmGstGetLongModePDPTPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e)
3817	{
3818	PX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
3819	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3820	PCX86PML4E pPml4e = *ppPml4e = &pGuestPml4->a[iPml4];
3821	if (pPml4e->n.u1Present)
3822	{
3823	PX86PDPT pPdpt;
3824	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPml4e->u & X86_PML4E_PG_MASK, &pPdpt);
3825	AssertRCReturn(rc, NULL);
3826
3827	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3828	return &pPdpt->a[iPdpt];
3829	}
3830	return NULL;
3831	}
3832
3833
3834	/**
3835	* Gets the page directory entry for the specified address.
3836	*
3837	* @returns The page directory entry in question.
3838	* @returns A non-present entry if the page directory is not present or on an invalid page.
3839	* @param pPGM Pointer to the PGM instance data.
3840	* @param GCPtr The address.
3841	* @param ppPml4e Page Map Level-4 Entry (out)
3842	* @param pPdpe Page directory pointer table entry (out)
3843	*/
3844	DECLINLINE(X86PDEPAE) pgmGstGetLongModePDEEx(PPGMCPU pPGM, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e, PX86PDPE pPdpe)
3845	{
3846	X86PDEPAE ZeroPde = {0};
3847	PX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
3848	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3849	PCX86PML4E pPml4e = *ppPml4e = &pGuestPml4->a[iPml4];
3850	if (pPml4e->n.u1Present)
3851	{
3852	PCX86PDPT pPdptTemp;
3853	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPml4e->u & X86_PML4E_PG_MASK, &pPdptTemp);
3854	AssertRCReturn(rc, ZeroPde);
3855
3856	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3857	*pPdpe = pPdptTemp->a[iPdpt];
3858	if (pPdptTemp->a[iPdpt].n.u1Present)
3859	{
3860	PCX86PDPAE pPD;
3861	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
3862	AssertRCReturn(rc, ZeroPde);
3863
3864	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3865	return pPD->a[iPD];
3866	}
3867	}
3868
3869	return ZeroPde;
3870	}
3871
3872
3873	/**
3874	* Gets the page directory entry for the specified address.
3875	*
3876	* @returns The page directory entry in question.
3877	* @returns A non-present entry if the page directory is not present or on an invalid page.
3878	* @param pPGM Pointer to the PGM instance data.
3879	* @param GCPtr The address.
3880	*/
3881	DECLINLINE(X86PDEPAE) pgmGstGetLongModePDE(PPGMCPU pPGM, RTGCPTR64 GCPtr)
3882	{
3883	X86PDEPAE ZeroPde = {0};
3884	PCX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
3885	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3886	if (pGuestPml4->a[iPml4].n.u1Present)
3887	{
3888	PCX86PDPT pPdptTemp;
3889	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
3890	AssertRCReturn(rc, ZeroPde);
3891
3892	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3893	if (pPdptTemp->a[iPdpt].n.u1Present)
3894	{
3895	PCX86PDPAE pPD;
3896	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
3897	AssertRCReturn(rc, ZeroPde);
3898
3899	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3900	return pPD->a[iPD];
3901	}
3902	}
3903	return ZeroPde;
3904	}
3905
3906
3907	/**
3908	* Gets the page directory entry for the specified address.
3909	*
3910	* @returns Pointer to the page directory entry in question.
3911	* @returns NULL if the page directory is not present or on an invalid page.
3912	* @param pPGM Pointer to the PGM instance data.
3913	* @param GCPtr The address.
3914	*/
3915	DECLINLINE(PX86PDEPAE) pgmGstGetLongModePDEPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr)
3916	{
3917	PCX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
3918	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3919	if (pGuestPml4->a[iPml4].n.u1Present)
3920	{
3921	PCX86PDPT pPdptTemp;
3922	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
3923	AssertRCReturn(rc, NULL);
3924
3925	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3926	if (pPdptTemp->a[iPdpt].n.u1Present)
3927	{
3928	PX86PDPAE pPD;
3929	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
3930	AssertRCReturn(rc, NULL);
3931
3932	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3933	return &pPD->a[iPD];
3934	}
3935	}
3936	return NULL;
3937	}
3938
3939
3940	/**
3941	* Gets the GUEST page directory pointer for the specified address.
3942	*
3943	* @returns The page directory in question.
3944	* @returns NULL if the page directory is not present or on an invalid page.
3945	* @param pPGM Pointer to the PGM instance data.
3946	* @param GCPtr The address.
3947	* @param ppPml4e Page Map Level-4 Entry (out)
3948	* @param pPdpe Page directory pointer table entry (out)
3949	* @param piPD Receives the index into the returned page directory
3950	*/
3951	DECLINLINE(PX86PDPAE) pgmGstGetLongModePDPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPE pPdpe, unsigned piPD)
3952	{
3953	PX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
3954	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3955	PCX86PML4E pPml4e = *ppPml4e = &pGuestPml4->a[iPml4];
3956	if (pPml4e->n.u1Present)
3957	{
3958	PCX86PDPT pPdptTemp;
3959	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPml4e->u & X86_PML4E_PG_MASK, &pPdptTemp);
3960	AssertRCReturn(rc, NULL);
3961
3962	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3963	*pPdpe = pPdptTemp->a[iPdpt];
3964	if (pPdptTemp->a[iPdpt].n.u1Present)
3965	{
3966	PX86PDPAE pPD;
3967	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
3968	AssertRCReturn(rc, NULL);
3969
3970	*piPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3971	return pPD;
3972	}
3973	}
3974	return 0;
3975	}
3976
3977	#endif /* !IN_RC */
3978
3979	/**
3980	* Gets the shadow page directory, 32-bit.
3981	*
3982	* @returns Pointer to the shadow 32-bit PD.
3983	* @param pPGM Pointer to the PGM instance data.
3984	*/
3985	DECLINLINE(PX86PD) pgmShwGet32BitPDPtr(PPGMCPU pPGM)
3986	{
3987	return (PX86PD)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPGM->CTX_SUFF(pShwPageCR3));
3988	}
3989
3990
3991	/**
3992	* Gets the shadow page directory entry for the specified address, 32-bit.
3993	*
3994	* @returns Shadow 32-bit PDE.
3995	* @param pPGM Pointer to the PGM instance data.
3996	* @param GCPtr The address.
3997	*/
3998	DECLINLINE(X86PDE) pgmShwGet32BitPDE(PPGMCPU pPGM, RTGCPTR GCPtr)
3999	{
4000	const unsigned iPd = (GCPtr >> X86_PD_SHIFT) & X86_PD_MASK;
4001
4002	PX86PD pShwPde = pgmShwGet32BitPDPtr(pPGM);
4003	if (!pShwPde)
4004	{
4005	X86PDE ZeroPde = {0};
4006	return ZeroPde;
4007	}
4008	return pShwPde->a[iPd];
4009	}
4010
4011
4012	/**
4013	* Gets the pointer to the shadow page directory entry for the specified
4014	* address, 32-bit.
4015	*
4016	* @returns Pointer to the shadow 32-bit PDE.
4017	* @param pPGM Pointer to the PGM instance data.
4018	* @param GCPtr The address.
4019	*/
4020	DECLINLINE(PX86PDE) pgmShwGet32BitPDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
4021	{
4022	const unsigned iPd = (GCPtr >> X86_PD_SHIFT) & X86_PD_MASK;
4023
4024	PX86PD pPde = pgmShwGet32BitPDPtr(pPGM);
4025	AssertReturn(pPde, NULL);
4026	return &pPde->a[iPd];
4027	}
4028
4029
4030	/**
4031	* Gets the shadow page pointer table, PAE.
4032	*
4033	* @returns Pointer to the shadow PAE PDPT.
4034	* @param pPGM Pointer to the PGM instance data.
4035	*/
4036	DECLINLINE(PX86PDPT) pgmShwGetPaePDPTPtr(PPGMCPU pPGM)
4037	{
4038	return (PX86PDPT)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPGM->CTX_SUFF(pShwPageCR3));
4039	}
4040
4041
4042	/**
4043	* Gets the shadow page directory for the specified address, PAE.
4044	*
4045	* @returns Pointer to the shadow PD.
4046	* @param pPGM Pointer to the PGM instance data.
4047	* @param GCPtr The address.
4048	*/
4049	DECLINLINE(PX86PDPAE) pgmShwGetPaePDPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
4050	{
4051	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
4052	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pPGM);
4053
4054	if (!pPdpt->a[iPdpt].n.u1Present)
4055	return NULL;
4056
4057	/* Fetch the pgm pool shadow descriptor. */
4058	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(PGMCPU2PGM(pPGM)->CTX_SUFF(pPool), pPdpt->a[iPdpt].u & X86_PDPE_PG_MASK);
4059	AssertReturn(pShwPde, NULL);
4060
4061	return (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pShwPde);
4062	}
4063
4064
4065	/**
4066	* Gets the shadow page directory for the specified address, PAE.
4067	*
4068	* @returns Pointer to the shadow PD.
4069	* @param pPGM Pointer to the PGM instance data.
4070	* @param GCPtr The address.
4071	*/
4072	DECLINLINE(PX86PDPAE) pgmShwGetPaePDPtr(PPGMCPU pPGM, PX86PDPT pPdpt, RTGCPTR GCPtr)
4073	{
4074	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
4075
4076	if (!pPdpt->a[iPdpt].n.u1Present)
4077	return NULL;
4078
4079	/* Fetch the pgm pool shadow descriptor. */
4080	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(PGMCPU2PGM(pPGM)->CTX_SUFF(pPool), pPdpt->a[iPdpt].u & X86_PDPE_PG_MASK);
4081	AssertReturn(pShwPde, NULL);
4082
4083	return (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pShwPde);
4084	}
4085
4086
4087	/**
4088	* Gets the shadow page directory entry, PAE.
4089	*
4090	* @returns PDE.
4091	* @param pPGM Pointer to the PGM instance data.
4092	* @param GCPtr The address.
4093	*/
4094	DECLINLINE(X86PDEPAE) pgmShwGetPaePDE(PPGMCPU pPGM, RTGCPTR GCPtr)
4095	{
4096	const unsigned iPd = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
4097
4098	PX86PDPAE pShwPde = pgmShwGetPaePDPtr(pPGM, GCPtr);
4099	if (!pShwPde)
4100	{
4101	X86PDEPAE ZeroPde = {0};
4102	return ZeroPde;
4103	}
4104	return pShwPde->a[iPd];
4105	}
4106
4107
4108	/**
4109	* Gets the pointer to the shadow page directory entry for an address, PAE.
4110	*
4111	* @returns Pointer to the PDE.
4112	* @param pPGM Pointer to the PGM instance data.
4113	* @param GCPtr The address.
4114	*/
4115	DECLINLINE(PX86PDEPAE) pgmShwGetPaePDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
4116	{
4117	const unsigned iPd = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
4118
4119	PX86PDPAE pPde = pgmShwGetPaePDPtr(pPGM, GCPtr);
4120	AssertReturn(pPde, NULL);
4121	return &pPde->a[iPd];
4122	}
4123
4124	#ifndef IN_RC
4125
4126	/**
4127	* Gets the shadow page map level-4 pointer.
4128	*
4129	* @returns Pointer to the shadow PML4.
4130	* @param pPGM Pointer to the PGM instance data.
4131	*/
4132	DECLINLINE(PX86PML4) pgmShwGetLongModePML4Ptr(PPGMCPU pPGM)
4133	{
4134	return (PX86PML4)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPGM->CTX_SUFF(pShwPageCR3));
4135	}
4136
4137
4138	/**
4139	* Gets the shadow page map level-4 entry for the specified address.
4140	*
4141	* @returns The entry.
4142	* @param pPGM Pointer to the PGM instance data.
4143	* @param GCPtr The address.
4144	*/
4145	DECLINLINE(X86PML4E) pgmShwGetLongModePML4E(PPGMCPU pPGM, RTGCPTR GCPtr)
4146	{
4147	const unsigned iPml4 = ((RTGCUINTPTR64)GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
4148	PX86PML4 pShwPml4 = pgmShwGetLongModePML4Ptr(pPGM);
4149
4150	if (!pShwPml4)
4151	{
4152	X86PML4E ZeroPml4e = {0};
4153	return ZeroPml4e;
4154	}
4155	return pShwPml4->a[iPml4];
4156	}
4157
4158
4159	/**
4160	* Gets the pointer to the specified shadow page map level-4 entry.
4161	*
4162	* @returns The entry.
4163	* @param pPGM Pointer to the PGM instance data.
4164	* @param iPml4 The PML4 index.
4165	*/
4166	DECLINLINE(PX86PML4E) pgmShwGetLongModePML4EPtr(PPGMCPU pPGM, unsigned int iPml4)
4167	{
4168	PX86PML4 pShwPml4 = pgmShwGetLongModePML4Ptr(pPGM);
4169	if (!pShwPml4)
4170	return NULL;
4171	return &pShwPml4->a[iPml4];
4172	}
4173
4174
4175	/**
4176	* Gets the GUEST page directory pointer for the specified address.
4177	*
4178	* @returns The page directory in question.
4179	* @returns NULL if the page directory is not present or on an invalid page.
4180	* @param pPGM Pointer to the PGM instance data.
4181	* @param GCPtr The address.
4182	* @param piPD Receives the index into the returned page directory
4183	*/
4184	DECLINLINE(PX86PDPAE) pgmGstGetLongModePDPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr, unsigned *piPD)
4185	{
4186	PCX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
4187	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
4188	if (pGuestPml4->a[iPml4].n.u1Present)
4189	{
4190	PCX86PDPT pPdptTemp;
4191	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
4192	AssertRCReturn(rc, NULL);
4193
4194	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
4195	if (pPdptTemp->a[iPdpt].n.u1Present)
4196	{
4197	PX86PDPAE pPD;
4198	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
4199	AssertRCReturn(rc, NULL);
4200
4201	*piPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
4202	return pPD;
4203	}
4204	}
4205	return NULL;
4206	}
4207
4208	#endif /* !IN_RC */
4209
4210	/**
4211	* Gets the page state for a physical handler.
4212	*
4213	* @returns The physical handler page state.
4214	* @param pCur The physical handler in question.
4215	*/
4216	DECLINLINE(unsigned) pgmHandlerPhysicalCalcState(PPGMPHYSHANDLER pCur)
4217	{
4218	switch (pCur->enmType)
4219	{
4220	case PGMPHYSHANDLERTYPE_PHYSICAL_WRITE:
4221	return PGM_PAGE_HNDL_PHYS_STATE_WRITE;
4222
4223	case PGMPHYSHANDLERTYPE_MMIO:
4224	case PGMPHYSHANDLERTYPE_PHYSICAL_ALL:
4225	return PGM_PAGE_HNDL_PHYS_STATE_ALL;
4226
4227	default:
4228	AssertFatalMsgFailed(("Invalid type %d\n", pCur->enmType));
4229	}
4230	}
4231
4232
4233	/**
4234	* Gets the page state for a virtual handler.
4235	*
4236	* @returns The virtual handler page state.
4237	* @param pCur The virtual handler in question.
4238	* @remarks This should never be used on a hypervisor access handler.
4239	*/
4240	DECLINLINE(unsigned) pgmHandlerVirtualCalcState(PPGMVIRTHANDLER pCur)
4241	{
4242	switch (pCur->enmType)
4243	{
4244	case PGMVIRTHANDLERTYPE_WRITE:
4245	return PGM_PAGE_HNDL_VIRT_STATE_WRITE;
4246	case PGMVIRTHANDLERTYPE_ALL:
4247	return PGM_PAGE_HNDL_VIRT_STATE_ALL;
4248	default:
4249	AssertFatalMsgFailed(("Invalid type %d\n", pCur->enmType));
4250	}
4251	}
4252
4253
4254	/**
4255	* Clears one physical page of a virtual handler
4256	*
4257	* @param pPGM Pointer to the PGM instance.
4258	* @param pCur Virtual handler structure
4259	* @param iPage Physical page index
4260	*
4261	* @remark Only used when PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL is being set, so no
4262	* need to care about other handlers in the same page.
4263	*/
4264	DECLINLINE(void) pgmHandlerVirtualClearPage(PPGM pPGM, PPGMVIRTHANDLER pCur, unsigned iPage)
4265	{
4266	const PPGMPHYS2VIRTHANDLER pPhys2Virt = &pCur->aPhysToVirt[iPage];
4267
4268	/*
4269	* Remove the node from the tree (it's supposed to be in the tree if we get here!).
4270	*/
4271	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4272	AssertReleaseMsg(pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_IN_TREE,
4273	("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4274	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias));
4275	#endif
4276	if (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_IS_HEAD)
4277	{
4278	/* We're the head of the alias chain. */
4279	PPGMPHYS2VIRTHANDLER pRemove = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysRemove(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, pPhys2Virt->Core.Key); NOREF(pRemove);
4280	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4281	AssertReleaseMsg(pRemove != NULL,
4282	("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4283	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias));
4284	AssertReleaseMsg(pRemove == pPhys2Virt,
4285	("wanted: pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n"
4286	" got: pRemove=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4287	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias,
4288	pRemove, pRemove->Core.Key, pRemove->Core.KeyLast, pRemove->offVirtHandler, pRemove->offNextAlias));
4289	#endif
4290	if (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK)
4291	{
4292	/* Insert the next list in the alias chain into the tree. */
4293	PPGMPHYS2VIRTHANDLER pNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPhys2Virt + (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
4294	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4295	AssertReleaseMsg(pNext->offNextAlias & PGMPHYS2VIRTHANDLER_IN_TREE,
4296	("pNext=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4297	pNext, pNext->Core.Key, pNext->Core.KeyLast, pNext->offVirtHandler, pNext->offNextAlias));
4298	#endif
4299	pNext->offNextAlias \|= PGMPHYS2VIRTHANDLER_IS_HEAD;
4300	bool fRc = RTAvlroGCPhysInsert(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, &pNext->Core);
4301	AssertRelease(fRc);
4302	}
4303	}
4304	else
4305	{
4306	/* Locate the previous node in the alias chain. */
4307	PPGMPHYS2VIRTHANDLER pPrev = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysGet(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, pPhys2Virt->Core.Key);
4308	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4309	AssertReleaseMsg(pPrev != pPhys2Virt,
4310	("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32} pPrev=%p\n",
4311	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias, pPrev));
4312	#endif
4313	for (;;)
4314	{
4315	PPGMPHYS2VIRTHANDLER pNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPrev + (pPrev->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
4316	if (pNext == pPhys2Virt)
4317	{
4318	/* unlink. */
4319	LogFlow(("pgmHandlerVirtualClearPage: removed %p:{.offNextAlias=%#RX32} from alias chain. prev %p:{.offNextAlias=%#RX32} [%RGp-%RGp]\n",
4320	pPhys2Virt, pPhys2Virt->offNextAlias, pPrev, pPrev->offNextAlias, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast));
4321	if (!(pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK))
4322	pPrev->offNextAlias &= ~PGMPHYS2VIRTHANDLER_OFF_MASK;
4323	else
4324	{
4325	PPGMPHYS2VIRTHANDLER pNewNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPhys2Virt + (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
4326	pPrev->offNextAlias = ((intptr_t)pNewNext - (intptr_t)pPrev)
4327	\| (pPrev->offNextAlias & ~PGMPHYS2VIRTHANDLER_OFF_MASK);
4328	}
4329	break;
4330	}
4331
4332	/* next */
4333	if (pNext == pPrev)
4334	{
4335	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4336	AssertReleaseMsg(pNext != pPrev,
4337	("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32} pPrev=%p\n",
4338	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias, pPrev));
4339	#endif
4340	break;
4341	}
4342	pPrev = pNext;
4343	}
4344	}
4345	Log2(("PHYS2VIRT: Removing %RGp-%RGp %#RX32 %s\n",
4346	pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offNextAlias, R3STRING(pCur->pszDesc)));
4347	pPhys2Virt->offNextAlias = 0;
4348	pPhys2Virt->Core.KeyLast = NIL_RTGCPHYS; /* require reinsert */
4349
4350	/*
4351	* Clear the ram flags for this page.
4352	*/
4353	PPGMPAGE pPage = pgmPhysGetPage(pPGM, pPhys2Virt->Core.Key);
4354	AssertReturnVoid(pPage);
4355	PGM_PAGE_SET_HNDL_VIRT_STATE(pPage, PGM_PAGE_HNDL_VIRT_STATE_NONE);
4356	}
4357
4358
4359	/**
4360	* Internal worker for finding a 'in-use' shadow page give by it's physical address.
4361	*
4362	* @returns Pointer to the shadow page structure.
4363	* @param pPool The pool.
4364	* @param idx The pool page index.
4365	*/
4366	DECLINLINE(PPGMPOOLPAGE) pgmPoolGetPageByIdx(PPGMPOOL pPool, unsigned idx)
4367	{
4368	AssertFatalMsg(idx >= PGMPOOL_IDX_FIRST && idx < pPool->cCurPages, ("idx=%d\n", idx));
4369	return &pPool->aPages[idx];
4370	}
4371
4372
4373	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
4374	/**
4375	* Clear references to guest physical memory.
4376	*
4377	* @param pPool The pool.
4378	* @param pPoolPage The pool page.
4379	* @param pPhysPage The physical guest page tracking structure.
4380	*/
4381	DECLINLINE(void) pgmTrackDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPoolPage, PPGMPAGE pPhysPage)
4382	{
4383	/*
4384	* Just deal with the simple case here.
4385	*/
4386	# ifdef LOG_ENABLED
4387	const unsigned uOrg = PGM_PAGE_GET_TRACKING(pPhysPage);
4388	# endif
4389	const unsigned cRefs = PGM_PAGE_GET_TD_CREFS(pPhysPage);
4390	if (cRefs == 1)
4391	{
4392	Assert(pPoolPage->idx == PGM_PAGE_GET_TD_IDX(pPhysPage));
4393	PGM_PAGE_SET_TRACKING(pPhysPage, 0);
4394	}
4395	else
4396	pgmPoolTrackPhysExtDerefGCPhys(pPool, pPoolPage, pPhysPage);
4397	Log2(("pgmTrackDerefGCPhys: %x -> %x pPhysPage=%R[pgmpage]\n", uOrg, PGM_PAGE_GET_TRACKING(pPhysPage), pPhysPage ));
4398	}
4399	#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
4400
4401
4402	#ifdef PGMPOOL_WITH_CACHE
4403	/**
4404	* Moves the page to the head of the age list.
4405	*
4406	* This is done when the cached page is used in one way or another.
4407	*
4408	* @param pPool The pool.
4409	* @param pPage The cached page.
4410	*/
4411	DECLINLINE(void) pgmPoolCacheUsed(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4412	{
4413	PVM pVM = pPool->CTX_SUFF(pVM);
4414	pgmLock(pVM);
4415
4416	/*
4417	* Move to the head of the age list.
4418	*/
4419	if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
4420	{
4421	/* unlink */
4422	pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
4423	if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
4424	pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
4425	else
4426	pPool->iAgeTail = pPage->iAgePrev;
4427
4428	/* insert at head */
4429	pPage->iAgePrev = NIL_PGMPOOL_IDX;
4430	pPage->iAgeNext = pPool->iAgeHead;
4431	Assert(pPage->iAgeNext != NIL_PGMPOOL_IDX); /* we would've already been head then */
4432	pPool->iAgeHead = pPage->idx;
4433	pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->idx;
4434	}
4435	pgmUnlock(pVM);
4436	}
4437	#endif /* PGMPOOL_WITH_CACHE */
4438
4439	/**
4440	* Locks a page to prevent flushing (important for cr3 root pages or shadow pae pd pages).
4441	*
4442	* @param pVM VM Handle.
4443	* @param pPage PGM pool page
4444	*/
4445	DECLINLINE(void) pgmPoolLockPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4446	{
4447	pPage->cLocked++;
4448	}
4449
4450
4451	/**
4452	* Unlocks a page to allow flushing again
4453	*
4454	* @param pVM VM Handle.
4455	* @param pPage PGM pool page
4456	*/
4457	DECLINLINE(void) pgmPoolUnlockPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4458	{
4459	Assert(pPage->cLocked);
4460	pPage->cLocked--;
4461	}
4462
4463
4464	/**
4465	* Checks if the page is locked (e.g. the active CR3 or one of the four PDs of a PAE PDPT)
4466	*
4467	* @returns VBox status code.
4468	* @param pPage PGM pool page
4469	*/
4470	DECLINLINE(bool) pgmPoolIsPageLocked(PPGM pPGM, PPGMPOOLPAGE pPage)
4471	{
4472	if (pPage->cLocked)
4473	{
4474	LogFlow(("pgmPoolIsPageLocked found root page %d\n", pPage->enmKind));
4475	if (pPage->cModifications)
4476	pPage->cModifications = 1; /* reset counter (can't use 0, or else it will be reinserted in the modified list) */
4477	return true;
4478	}
4479	return false;
4480	}
4481
4482	/**
4483	* Tells if mappings are to be put into the shadow page table or not
4484	*
4485	* @returns boolean result
4486	* @param pVM VM handle.
4487	*/
4488	DECLINLINE(bool) pgmMapAreMappingsEnabled(PPGM pPGM)
4489	{
4490	#ifdef IN_RING0
4491	/* There are no mappings in VT-x and AMD-V mode. */
4492	Assert(pPGM->fDisableMappings);
4493	return false;
4494	#else
4495	return !pPGM->fDisableMappings;
4496	#endif
4497	}
4498
4499	/** @} */
4500
4501	#endif
4502
4503

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/PGMInternal.h@ 19969

Download in other formats: