PGMInternal.h@ 41728

Last change on this file since 41728 was 41692, checked in by vboxsync, 12 years ago
DIS: Reducing the DISCPUMODE even more (200 bytes now) and making it have the same layout in all contexts. This is useful since it's used several places in the VM structure. Also a bunch of other cleanups.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 182.6 KB

Line
1	/* $Id: PGMInternal.h 41692 2012-06-13 19:32:54Z vboxsync $ */
2	/** @file
3	* PGM - Internal header file.
4	*/
5
6	/*
7	* Copyright (C) 2006-2010 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	#ifndef ___PGMInternal_h
19	#define ___PGMInternal_h
20
21	#include <VBox/cdefs.h>
22	#include <VBox/types.h>
23	#include <VBox/err.h>
24	#include <VBox/dbg.h>
25	#include <VBox/vmm/stam.h>
26	#include <VBox/param.h>
27	#include <VBox/vmm/vmm.h>
28	#include <VBox/vmm/mm.h>
29	#include <VBox/vmm/pdmcritsect.h>
30	#include <VBox/vmm/pdmapi.h>
31	#include <VBox/dis.h>
32	#include <VBox/vmm/dbgf.h>
33	#include <VBox/log.h>
34	#include <VBox/vmm/gmm.h>
35	#include <VBox/vmm/hwaccm.h>
36	#include <VBox/vmm/hwacc_vmx.h>
37	#include "internal/pgm.h"
38	#include <iprt/asm.h>
39	#include <iprt/assert.h>
40	#include <iprt/avl.h>
41	#include <iprt/critsect.h>
42	#include <iprt/sha.h>
43
44
45
46	/** @defgroup grp_pgm_int Internals
47	* @ingroup grp_pgm
48	* @internal
49	* @{
50	*/
51
52
53	/** @name PGM Compile Time Config
54	* @{
55	*/
56
57	/**
58	* Indicates that there are no guest mappings to care about.
59	* Currently on raw-mode related code uses mappings, i.e. RC and R3 code.
60	*/
61	#if defined(IN_RING0) \|\| !defined(VBOX_WITH_RAW_MODE)
62	# define PGM_WITHOUT_MAPPINGS
63	#endif
64
65	/**
66	* Check and skip global PDEs for non-global flushes
67	*/
68	#define PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
69
70	/**
71	* Optimization for PAE page tables that are modified often
72	*/
73	//#if 0 /* disabled again while debugging */
74	#ifndef IN_RC
75	# define PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
76	#endif
77	//#endif
78
79	/**
80	* Large page support enabled only on 64 bits hosts; applies to nested paging only.
81	*/
82	#if (HC_ARCH_BITS == 64) && !defined(IN_RC)
83	# define PGM_WITH_LARGE_PAGES
84	#endif
85
86	/**
87	* Enables optimizations for MMIO handlers that exploits X86_TRAP_PF_RSVD and
88	* VMX_EXIT_EPT_MISCONFIG.
89	*/
90	#if 1 /* testing */
91	# define PGM_WITH_MMIO_OPTIMIZATIONS
92	#endif
93
94	/**
95	* Sync N pages instead of a whole page table
96	*/
97	#define PGM_SYNC_N_PAGES
98
99	/**
100	* Number of pages to sync during a page fault
101	*
102	* When PGMPOOL_WITH_GCPHYS_TRACKING is enabled using high values here
103	* causes a lot of unnecessary extents and also is slower than taking more \#PFs.
104	*
105	* Note that \#PFs are much more expensive in the VT-x/AMD-V case due to
106	* world switch overhead, so let's sync more.
107	*/
108	# ifdef IN_RING0
109	/* Chose 32 based on the compile test in #4219; 64 shows worse stats.
110	* 32 again shows better results than 16; slightly more overhead in the \#PF handler,
111	* but ~5% fewer faults.
112	*/
113	# define PGM_SYNC_NR_PAGES 32
114	#else
115	# define PGM_SYNC_NR_PAGES 8
116	#endif
117
118	/**
119	* Number of PGMPhysRead/Write cache entries (must be <= sizeof(uint64_t))
120	*/
121	#define PGM_MAX_PHYSCACHE_ENTRIES 64
122	#define PGM_MAX_PHYSCACHE_ENTRIES_MASK (PGM_MAX_PHYSCACHE_ENTRIES-1)
123
124
125	/** @def PGMPOOL_CFG_MAX_GROW
126	* The maximum number of pages to add to the pool in one go.
127	*/
128	#define PGMPOOL_CFG_MAX_GROW (_256K >> PAGE_SHIFT)
129
130	/** @def VBOX_STRICT_PGM_HANDLER_VIRTUAL
131	* Enables some extra assertions for virtual handlers (mainly phys2virt related).
132	*/
133	#ifdef VBOX_STRICT
134	# define VBOX_STRICT_PGM_HANDLER_VIRTUAL
135	#endif
136
137	/** @def VBOX_WITH_NEW_LAZY_PAGE_ALLOC
138	* Enables the experimental lazy page allocation code. */
139	/#define VBOX_WITH_NEW_LAZY_PAGE_ALLOC /
140
141	/** @def VBOX_WITH_REAL_WRITE_MONITORED_PAGES
142	* Enables real write monitoring of pages, i.e. mapping them read-only and
143	* only making them writable when getting a write access #PF. */
144	#define VBOX_WITH_REAL_WRITE_MONITORED_PAGES
145
146	/** @} */
147
148
149	/** @name PDPT and PML4 flags.
150	* These are placed in the three bits available for system programs in
151	* the PDPT and PML4 entries.
152	* @{ */
153	/** The entry is a permanent one and it's must always be present.
154	* Never free such an entry. */
155	#define PGM_PLXFLAGS_PERMANENT RT_BIT_64(10)
156	/** Mapping (hypervisor allocated pagetable). */
157	#define PGM_PLXFLAGS_MAPPING RT_BIT_64(11)
158	/** @} */
159
160	/** @name Page directory flags.
161	* These are placed in the three bits available for system programs in
162	* the page directory entries.
163	* @{ */
164	/** Mapping (hypervisor allocated pagetable). */
165	#define PGM_PDFLAGS_MAPPING RT_BIT_64(10)
166	/** Made read-only to facilitate dirty bit tracking. */
167	#define PGM_PDFLAGS_TRACK_DIRTY RT_BIT_64(11)
168	/** @} */
169
170	/** @name Page flags.
171	* These are placed in the three bits available for system programs in
172	* the page entries.
173	* @{ */
174	/** Made read-only to facilitate dirty bit tracking. */
175	#define PGM_PTFLAGS_TRACK_DIRTY RT_BIT_64(9)
176
177	#ifndef PGM_PTFLAGS_CSAM_VALIDATED
178	/** Scanned and approved by CSAM (tm).
179	* NOTE: Must be identical to the one defined in CSAMInternal.h!!
180	* @todo Move PGM_PTFLAGS_* and PGM_PDFLAGS_* to VBox/vmm/pgm.h. */
181	#define PGM_PTFLAGS_CSAM_VALIDATED RT_BIT_64(11)
182	#endif
183
184	/** @} */
185
186	/** @name Defines used to indicate the shadow and guest paging in the templates.
187	* @{ */
188	#define PGM_TYPE_REAL 1
189	#define PGM_TYPE_PROT 2
190	#define PGM_TYPE_32BIT 3
191	#define PGM_TYPE_PAE 4
192	#define PGM_TYPE_AMD64 5
193	#define PGM_TYPE_NESTED 6
194	#define PGM_TYPE_EPT 7
195	#define PGM_TYPE_MAX PGM_TYPE_EPT
196	/** @} */
197
198	/** Macro for checking if the guest is using paging.
199	* @param uGstType PGM_TYPE_*
200	* @param uShwType PGM_TYPE_*
201	* @remark ASSUMES certain order of the PGM_TYPE_* values.
202	*/
203	#define PGM_WITH_PAGING(uGstType, uShwType) \
204	( (uGstType) >= PGM_TYPE_32BIT \
205	&& (uShwType) != PGM_TYPE_NESTED \
206	&& (uShwType) != PGM_TYPE_EPT)
207
208	/** Macro for checking if the guest supports the NX bit.
209	* @param uGstType PGM_TYPE_*
210	* @param uShwType PGM_TYPE_*
211	* @remark ASSUMES certain order of the PGM_TYPE_* values.
212	*/
213	#define PGM_WITH_NX(uGstType, uShwType) \
214	( (uGstType) >= PGM_TYPE_PAE \
215	&& (uShwType) != PGM_TYPE_NESTED \
216	&& (uShwType) != PGM_TYPE_EPT)
217
218
219	/** @def PGM_HCPHYS_2_PTR
220	* Maps a HC physical page pool address to a virtual address.
221	*
222	* @returns VBox status code.
223	* @param pVM The VM handle.
224	* @param pVCpu The current CPU.
225	* @param HCPhys The HC physical address to map to a virtual one.
226	* @param ppv Where to store the virtual address. No need to cast
227	* this.
228	*
229	* @remark Use with care as we don't have so much dynamic mapping space in
230	* ring-0 on 32-bit darwin and in RC.
231	* @remark There is no need to assert on the result.
232	*/
233	#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) \|\| defined(IN_RC)
234	# define PGM_HCPHYS_2_PTR(pVM, pVCpu, HCPhys, ppv) \
235	pgmRZDynMapHCPageInlined(pVCpu, HCPhys, (void **)(ppv) RTLOG_COMMA_SRC_POS)
236	#else
237	# define PGM_HCPHYS_2_PTR(pVM, pVCpu, HCPhys, ppv) \
238	MMPagePhys2PageEx(pVM, HCPhys, (void **)(ppv))
239	#endif
240
241	/** @def PGM_GCPHYS_2_PTR_V2
242	* Maps a GC physical page address to a virtual address.
243	*
244	* @returns VBox status code.
245	* @param pVM The VM handle.
246	* @param pVCpu The current CPU.
247	* @param GCPhys The GC physical address to map to a virtual one.
248	* @param ppv Where to store the virtual address. No need to cast this.
249	*
250	* @remark Use with care as we don't have so much dynamic mapping space in
251	* ring-0 on 32-bit darwin and in RC.
252	* @remark There is no need to assert on the result.
253	*/
254	#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) \|\| defined(IN_RC)
255	# define PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GCPhys, ppv) \
256	pgmRZDynMapGCPageV2Inlined(pVM, pVCpu, GCPhys, (void **)(ppv) RTLOG_COMMA_SRC_POS)
257	#else
258	# define PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GCPhys, ppv) \
259	pgmPhysGCPhys2R3Ptr(pVM, GCPhys, (PRTR3PTR)(ppv)) /** @todo this isn't asserting! */
260	#endif
261
262	/** @def PGM_GCPHYS_2_PTR
263	* Maps a GC physical page address to a virtual address.
264	*
265	* @returns VBox status code.
266	* @param pVM The VM handle.
267	* @param GCPhys The GC physical address to map to a virtual one.
268	* @param ppv Where to store the virtual address. No need to cast this.
269	*
270	* @remark Use with care as we don't have so much dynamic mapping space in
271	* ring-0 on 32-bit darwin and in RC.
272	* @remark There is no need to assert on the result.
273	*/
274	#define PGM_GCPHYS_2_PTR(pVM, GCPhys, ppv) PGM_GCPHYS_2_PTR_V2(pVM, VMMGetCpu(pVM), GCPhys, ppv)
275
276	/** @def PGM_GCPHYS_2_PTR_BY_VMCPU
277	* Maps a GC physical page address to a virtual address.
278	*
279	* @returns VBox status code.
280	* @param pVCpu The current CPU.
281	* @param GCPhys The GC physical address to map to a virtual one.
282	* @param ppv Where to store the virtual address. No need to cast this.
283	*
284	* @remark Use with care as we don't have so much dynamic mapping space in
285	* ring-0 on 32-bit darwin and in RC.
286	* @remark There is no need to assert on the result.
287	*/
288	#define PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, GCPhys, ppv) PGM_GCPHYS_2_PTR_V2((pVCpu)->CTX_SUFF(pVM), pVCpu, GCPhys, ppv)
289
290	/** @def PGM_GCPHYS_2_PTR_EX
291	* Maps a unaligned GC physical page address to a virtual address.
292	*
293	* @returns VBox status code.
294	* @param pVM The VM handle.
295	* @param GCPhys The GC physical address to map to a virtual one.
296	* @param ppv Where to store the virtual address. No need to cast this.
297	*
298	* @remark Use with care as we don't have so much dynamic mapping space in
299	* ring-0 on 32-bit darwin and in RC.
300	* @remark There is no need to assert on the result.
301	*/
302	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
303	# define PGM_GCPHYS_2_PTR_EX(pVM, GCPhys, ppv) \
304	pgmRZDynMapGCPageOffInlined(VMMGetCpu(pVM), GCPhys, (void **)(ppv) RTLOG_COMMA_SRC_POS)
305	#else
306	# define PGM_GCPHYS_2_PTR_EX(pVM, GCPhys, ppv) \
307	pgmPhysGCPhys2R3Ptr(pVM, GCPhys, (PRTR3PTR)(ppv)) /** @todo this isn't asserting! */
308	#endif
309
310	/** @def PGM_DYNMAP_UNUSED_HINT
311	* Hints to the dynamic mapping code in RC and R0/darwin that the specified page
312	* is no longer used.
313	*
314	* For best effect only apply this to the page that was mapped most recently.
315	*
316	* @param pVCpu The current CPU.
317	* @param pvPage The pool page.
318	*/
319	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
320	# ifdef LOG_ENABLED
321	# define PGM_DYNMAP_UNUSED_HINT(pVCpu, pvPage) pgmRZDynMapUnusedHint(pVCpu, pvPage, RT_SRC_POS)
322	# else
323	# define PGM_DYNMAP_UNUSED_HINT(pVCpu, pvPage) pgmRZDynMapUnusedHint(pVCpu, pvPage)
324	# endif
325	#else
326	# define PGM_DYNMAP_UNUSED_HINT(pVCpu, pvPage) do {} while (0)
327	#endif
328
329	/** @def PGM_DYNMAP_UNUSED_HINT_VM
330	* Hints to the dynamic mapping code in RC and R0/darwin that the specified page
331	* is no longer used.
332	*
333	* For best effect only apply this to the page that was mapped most recently.
334	*
335	* @param pVM The VM handle.
336	* @param pvPage The pool page.
337	*/
338	#define PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvPage) PGM_DYNMAP_UNUSED_HINT(VMMGetCpu(pVM), pvPage)
339
340
341	/** @def PGM_INVL_PG
342	* Invalidates a page.
343	*
344	* @param pVCpu The VMCPU handle.
345	* @param GCVirt The virtual address of the page to invalidate.
346	*/
347	#ifdef IN_RC
348	# define PGM_INVL_PG(pVCpu, GCVirt) ASMInvalidatePage((void *)(uintptr_t)(GCVirt))
349	#elif defined(IN_RING0)
350	# define PGM_INVL_PG(pVCpu, GCVirt) HWACCMInvalidatePage(pVCpu, (RTGCPTR)(GCVirt))
351	#else
352	# define PGM_INVL_PG(pVCpu, GCVirt) HWACCMInvalidatePage(pVCpu, (RTGCPTR)(GCVirt))
353	#endif
354
355	/** @def PGM_INVL_PG_ALL_VCPU
356	* Invalidates a page on all VCPUs
357	*
358	* @param pVM The VM handle.
359	* @param GCVirt The virtual address of the page to invalidate.
360	*/
361	#ifdef IN_RC
362	# define PGM_INVL_PG_ALL_VCPU(pVM, GCVirt) ASMInvalidatePage((void *)(uintptr_t)(GCVirt))
363	#elif defined(IN_RING0)
364	# define PGM_INVL_PG_ALL_VCPU(pVM, GCVirt) HWACCMInvalidatePageOnAllVCpus(pVM, (RTGCPTR)(GCVirt))
365	#else
366	# define PGM_INVL_PG_ALL_VCPU(pVM, GCVirt) HWACCMInvalidatePageOnAllVCpus(pVM, (RTGCPTR)(GCVirt))
367	#endif
368
369	/** @def PGM_INVL_BIG_PG
370	* Invalidates a 4MB page directory entry.
371	*
372	* @param pVCpu The VMCPU handle.
373	* @param GCVirt The virtual address within the page directory to invalidate.
374	*/
375	#ifdef IN_RC
376	# define PGM_INVL_BIG_PG(pVCpu, GCVirt) ASMReloadCR3()
377	#elif defined(IN_RING0)
378	# define PGM_INVL_BIG_PG(pVCpu, GCVirt) HWACCMFlushTLB(pVCpu)
379	#else
380	# define PGM_INVL_BIG_PG(pVCpu, GCVirt) HWACCMFlushTLB(pVCpu)
381	#endif
382
383	/** @def PGM_INVL_VCPU_TLBS()
384	* Invalidates the TLBs of the specified VCPU
385	*
386	* @param pVCpu The VMCPU handle.
387	*/
388	#ifdef IN_RC
389	# define PGM_INVL_VCPU_TLBS(pVCpu) ASMReloadCR3()
390	#elif defined(IN_RING0)
391	# define PGM_INVL_VCPU_TLBS(pVCpu) HWACCMFlushTLB(pVCpu)
392	#else
393	# define PGM_INVL_VCPU_TLBS(pVCpu) HWACCMFlushTLB(pVCpu)
394	#endif
395
396	/** @def PGM_INVL_ALL_VCPU_TLBS()
397	* Invalidates the TLBs of all VCPUs
398	*
399	* @param pVM The VM handle.
400	*/
401	#ifdef IN_RC
402	# define PGM_INVL_ALL_VCPU_TLBS(pVM) ASMReloadCR3()
403	#elif defined(IN_RING0)
404	# define PGM_INVL_ALL_VCPU_TLBS(pVM) HWACCMFlushTLBOnAllVCpus(pVM)
405	#else
406	# define PGM_INVL_ALL_VCPU_TLBS(pVM) HWACCMFlushTLBOnAllVCpus(pVM)
407	#endif
408
409
410	/** @name Safer Shadow PAE PT/PTE
411	* For helping avoid misinterpreting invalid PAE/AMD64 page table entries as
412	* present.
413	*
414	* @{
415	*/
416	#if 1
417	/**
418	* For making sure that u1Present and X86_PTE_P checks doesn't mistake
419	* invalid entries for present.
420	* @sa X86PTEPAE.
421	*/
422	typedef union PGMSHWPTEPAE
423	{
424	/** Unsigned integer view */
425	X86PGPAEUINT uCareful;
426	/* Not other views. */
427	} PGMSHWPTEPAE;
428
429	# define PGMSHWPTEPAE_IS_P(Pte) ( ((Pte).uCareful & (X86_PTE_P \| X86_PTE_PAE_MBZ_MASK_NX)) == X86_PTE_P )
430	# define PGMSHWPTEPAE_IS_RW(Pte) ( !!((Pte).uCareful & X86_PTE_RW))
431	# define PGMSHWPTEPAE_IS_US(Pte) ( !!((Pte).uCareful & X86_PTE_US))
432	# define PGMSHWPTEPAE_IS_A(Pte) ( !!((Pte).uCareful & X86_PTE_A))
433	# define PGMSHWPTEPAE_IS_D(Pte) ( !!((Pte).uCareful & X86_PTE_D))
434	# define PGMSHWPTEPAE_IS_TRACK_DIRTY(Pte) ( !!((Pte).uCareful & PGM_PTFLAGS_TRACK_DIRTY) )
435	# define PGMSHWPTEPAE_IS_P_RW(Pte) ( ((Pte).uCareful & (X86_PTE_P \| X86_PTE_RW \| X86_PTE_PAE_MBZ_MASK_NX)) == (X86_PTE_P \| X86_PTE_RW) )
436	# define PGMSHWPTEPAE_GET_LOG(Pte) ( (Pte).uCareful )
437	# define PGMSHWPTEPAE_GET_HCPHYS(Pte) ( (Pte).uCareful & X86_PTE_PAE_PG_MASK )
438	# define PGMSHWPTEPAE_GET_U(Pte) ( (Pte).uCareful ) /*< Use with care. /
439	# define PGMSHWPTEPAE_SET(Pte, uVal) do { (Pte).uCareful = (uVal); } while (0)
440	# define PGMSHWPTEPAE_SET2(Pte, Pte2) do { (Pte).uCareful = (Pte2).uCareful; } while (0)
441	# define PGMSHWPTEPAE_ATOMIC_SET(Pte, uVal) do { ASMAtomicWriteU64(&(Pte).uCareful, (uVal)); } while (0)
442	# define PGMSHWPTEPAE_ATOMIC_SET2(Pte, Pte2) do { ASMAtomicWriteU64(&(Pte).uCareful, (Pte2).uCareful); } while (0)
443	# define PGMSHWPTEPAE_SET_RO(Pte) do { (Pte).uCareful &= ~(X86PGPAEUINT)X86_PTE_RW; } while (0)
444	# define PGMSHWPTEPAE_SET_RW(Pte) do { (Pte).uCareful \|= X86_PTE_RW; } while (0)
445
446	/**
447	* For making sure that u1Present and X86_PTE_P checks doesn't mistake
448	* invalid entries for present.
449	* @sa X86PTPAE.
450	*/
451	typedef struct PGMSHWPTPAE
452	{
453	PGMSHWPTEPAE a[X86_PG_PAE_ENTRIES];
454	} PGMSHWPTPAE;
455
456	#else
457	typedef X86PTEPAE PGMSHWPTEPAE;
458	typedef X86PTPAE PGMSHWPTPAE;
459	# define PGMSHWPTEPAE_IS_P(Pte) ( (Pte).n.u1Present )
460	# define PGMSHWPTEPAE_IS_RW(Pte) ( (Pte).n.u1Write )
461	# define PGMSHWPTEPAE_IS_US(Pte) ( (Pte).n.u1User )
462	# define PGMSHWPTEPAE_IS_A(Pte) ( (Pte).n.u1Accessed )
463	# define PGMSHWPTEPAE_IS_D(Pte) ( (Pte).n.u1Dirty )
464	# define PGMSHWPTEPAE_IS_TRACK_DIRTY(Pte) ( !!((Pte).u & PGM_PTFLAGS_TRACK_DIRTY) )
465	# define PGMSHWPTEPAE_IS_P_RW(Pte) ( ((Pte).u & (X86_PTE_P \| X86_PTE_RW)) == (X86_PTE_P \| X86_PTE_RW) )
466	# define PGMSHWPTEPAE_GET_LOG(Pte) ( (Pte).u )
467	# define PGMSHWPTEPAE_GET_HCPHYS(Pte) ( (Pte).u & X86_PTE_PAE_PG_MASK )
468	# define PGMSHWPTEPAE_GET_U(Pte) ( (Pte).u ) /*< Use with care. /
469	# define PGMSHWPTEPAE_SET(Pte, uVal) do { (Pte).u = (uVal); } while (0)
470	# define PGMSHWPTEPAE_SET2(Pte, Pte2) do { (Pte).u = (Pte2).u; } while (0)
471	# define PGMSHWPTEPAE_ATOMIC_SET(Pte, uVal) do { ASMAtomicWriteU64(&(Pte).u, (uVal)); } while (0)
472	# define PGMSHWPTEPAE_ATOMIC_SET2(Pte, Pte2) do { ASMAtomicWriteU64(&(Pte).u, (Pte2).u); } while (0)
473	# define PGMSHWPTEPAE_SET_RO(Pte) do { (Pte).u &= ~(X86PGPAEUINT)X86_PTE_RW; } while (0)
474	# define PGMSHWPTEPAE_SET_RW(Pte) do { (Pte).u \|= X86_PTE_RW; } while (0)
475
476	#endif
477
478	/** Pointer to a shadow PAE PTE. */
479	typedef PGMSHWPTEPAE *PPGMSHWPTEPAE;
480	/** Pointer to a const shadow PAE PTE. */
481	typedef PGMSHWPTEPAE const *PCPGMSHWPTEPAE;
482
483	/** Pointer to a shadow PAE page table. */
484	typedef PGMSHWPTPAE *PPGMSHWPTPAE;
485	/** Pointer to a const shadow PAE page table. */
486	typedef PGMSHWPTPAE const *PCPGMSHWPTPAE;
487	/** @} */
488
489
490	/** Size of the GCPtrConflict array in PGMMAPPING.
491	* @remarks Must be a power of two. */
492	#define PGMMAPPING_CONFLICT_MAX 8
493
494	/**
495	* Structure for tracking GC Mappings.
496	*
497	* This structure is used by linked list in both GC and HC.
498	*/
499	typedef struct PGMMAPPING
500	{
501	/** Pointer to next entry. */
502	R3PTRTYPE(struct PGMMAPPING *) pNextR3;
503	/** Pointer to next entry. */
504	R0PTRTYPE(struct PGMMAPPING *) pNextR0;
505	/** Pointer to next entry. */
506	RCPTRTYPE(struct PGMMAPPING *) pNextRC;
507	/** Indicate whether this entry is finalized. */
508	bool fFinalized;
509	/** Start Virtual address. */
510	RTGCPTR GCPtr;
511	/** Last Virtual address (inclusive). */
512	RTGCPTR GCPtrLast;
513	/** Range size (bytes). */
514	RTGCPTR cb;
515	/** Pointer to relocation callback function. */
516	R3PTRTYPE(PFNPGMRELOCATE) pfnRelocate;
517	/** User argument to the callback. */
518	R3PTRTYPE(void *) pvUser;
519	/** Mapping description / name. For easing debugging. */
520	R3PTRTYPE(const char *) pszDesc;
521	/** Last 8 addresses that caused conflicts. */
522	RTGCPTR aGCPtrConflicts[PGMMAPPING_CONFLICT_MAX];
523	/** Number of conflicts for this hypervisor mapping. */
524	uint32_t cConflicts;
525	/** Number of page tables. */
526	uint32_t cPTs;
527
528	/** Array of page table mapping data. Each entry
529	* describes one page table. The array can be longer
530	* than the declared length.
531	*/
532	struct
533	{
534	/** The HC physical address of the page table. */
535	RTHCPHYS HCPhysPT;
536	/** The HC physical address of the first PAE page table. */
537	RTHCPHYS HCPhysPaePT0;
538	/** The HC physical address of the second PAE page table. */
539	RTHCPHYS HCPhysPaePT1;
540	/** The HC virtual address of the 32-bit page table. */
541	R3PTRTYPE(PX86PT) pPTR3;
542	/** The HC virtual address of the two PAE page table. (i.e 1024 entries instead of 512) */
543	R3PTRTYPE(PPGMSHWPTPAE) paPaePTsR3;
544	/** The RC virtual address of the 32-bit page table. */
545	RCPTRTYPE(PX86PT) pPTRC;
546	/** The RC virtual address of the two PAE page table. */
547	RCPTRTYPE(PPGMSHWPTPAE) paPaePTsRC;
548	/** The R0 virtual address of the 32-bit page table. */
549	R0PTRTYPE(PX86PT) pPTR0;
550	/** The R0 virtual address of the two PAE page table. */
551	R0PTRTYPE(PPGMSHWPTPAE) paPaePTsR0;
552	} aPTs[1];
553	} PGMMAPPING;
554	/** Pointer to structure for tracking GC Mappings. */
555	typedef struct PGMMAPPING *PPGMMAPPING;
556
557
558	/**
559	* Physical page access handler structure.
560	*
561	* This is used to keep track of physical address ranges
562	* which are being monitored in some kind of way.
563	*/
564	typedef struct PGMPHYSHANDLER
565	{
566	AVLROGCPHYSNODECORE Core;
567	/** Access type. */
568	PGMPHYSHANDLERTYPE enmType;
569	/** Number of pages to update. */
570	uint32_t cPages;
571	/** Set if we have pages that have been aliased. */
572	uint32_t cAliasedPages;
573	/** Set if we have pages that have temporarily been disabled. */
574	uint32_t cTmpOffPages;
575	/** Pointer to R3 callback function. */
576	R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnHandlerR3;
577	/** User argument for R3 handlers. */
578	R3PTRTYPE(void *) pvUserR3;
579	/** Pointer to R0 callback function. */
580	R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnHandlerR0;
581	/** User argument for R0 handlers. */
582	R0PTRTYPE(void *) pvUserR0;
583	/** Pointer to RC callback function. */
584	RCPTRTYPE(PFNPGMRCPHYSHANDLER) pfnHandlerRC;
585	/** User argument for RC handlers. */
586	RCPTRTYPE(void *) pvUserRC;
587	/** Description / Name. For easing debugging. */
588	R3PTRTYPE(const char *) pszDesc;
589	#ifdef VBOX_WITH_STATISTICS
590	/** Profiling of this handler. */
591	STAMPROFILE Stat;
592	#endif
593	} PGMPHYSHANDLER;
594	/** Pointer to a physical page access handler structure. */
595	typedef PGMPHYSHANDLER *PPGMPHYSHANDLER;
596
597
598	/**
599	* Cache node for the physical addresses covered by a virtual handler.
600	*/
601	typedef struct PGMPHYS2VIRTHANDLER
602	{
603	/** Core node for the tree based on physical ranges. */
604	AVLROGCPHYSNODECORE Core;
605	/** Offset from this struct to the PGMVIRTHANDLER structure. */
606	int32_t offVirtHandler;
607	/** Offset of the next alias relative to this one.
608	* Bit 0 is used for indicating whether we're in the tree.
609	* Bit 1 is used for indicating that we're the head node.
610	*/
611	int32_t offNextAlias;
612	} PGMPHYS2VIRTHANDLER;
613	/** Pointer to a phys to virtual handler structure. */
614	typedef PGMPHYS2VIRTHANDLER *PPGMPHYS2VIRTHANDLER;
615
616	/** The bit in PGMPHYS2VIRTHANDLER::offNextAlias used to indicate that the
617	* node is in the tree. */
618	#define PGMPHYS2VIRTHANDLER_IN_TREE RT_BIT(0)
619	/** The bit in PGMPHYS2VIRTHANDLER::offNextAlias used to indicate that the
620	* node is in the head of an alias chain.
621	* The PGMPHYS2VIRTHANDLER_IN_TREE is always set if this bit is set. */
622	#define PGMPHYS2VIRTHANDLER_IS_HEAD RT_BIT(1)
623	/** The mask to apply to PGMPHYS2VIRTHANDLER::offNextAlias to get the offset. */
624	#define PGMPHYS2VIRTHANDLER_OFF_MASK (~(int32_t)3)
625
626
627	/**
628	* Virtual page access handler structure.
629	*
630	* This is used to keep track of virtual address ranges
631	* which are being monitored in some kind of way.
632	*/
633	typedef struct PGMVIRTHANDLER
634	{
635	/** Core node for the tree based on virtual ranges. */
636	AVLROGCPTRNODECORE Core;
637	/** Size of the range (in bytes). */
638	RTGCPTR cb;
639	/** Number of cache pages. */
640	uint32_t cPages;
641	/** Access type. */
642	PGMVIRTHANDLERTYPE enmType;
643	/** Pointer to the RC callback function. */
644	RCPTRTYPE(PFNPGMRCVIRTHANDLER) pfnHandlerRC;
645	#if HC_ARCH_BITS == 64
646	RTRCPTR padding;
647	#endif
648	/** Pointer to the R3 callback function for invalidation. */
649	R3PTRTYPE(PFNPGMR3VIRTINVALIDATE) pfnInvalidateR3;
650	/** Pointer to the R3 callback function. */
651	R3PTRTYPE(PFNPGMR3VIRTHANDLER) pfnHandlerR3;
652	/** Description / Name. For easing debugging. */
653	R3PTRTYPE(const char *) pszDesc;
654	#ifdef VBOX_WITH_STATISTICS
655	/** Profiling of this handler. */
656	STAMPROFILE Stat;
657	#endif
658	/** Array of cached physical addresses for the monitored ranged. */
659	PGMPHYS2VIRTHANDLER aPhysToVirt[HC_ARCH_BITS == 32 ? 1 : 2];
660	} PGMVIRTHANDLER;
661	/** Pointer to a virtual page access handler structure. */
662	typedef PGMVIRTHANDLER *PPGMVIRTHANDLER;
663
664
665	/** @name Page type predicates.
666	* @{ */
667	#define PGMPAGETYPE_IS_READABLE(type) ( (type) <= PGMPAGETYPE_ROM )
668	#define PGMPAGETYPE_IS_WRITEABLE(type) ( (type) <= PGMPAGETYPE_ROM_SHADOW )
669	#define PGMPAGETYPE_IS_RWX(type) ( (type) <= PGMPAGETYPE_ROM_SHADOW )
670	#define PGMPAGETYPE_IS_ROX(type) ( (type) == PGMPAGETYPE_ROM )
671	#define PGMPAGETYPE_IS_NP(type) ( (type) == PGMPAGETYPE_MMIO )
672	/** @} */
673
674
675	/**
676	* A Physical Guest Page tracking structure.
677	*
678	* The format of this structure is complicated because we have to fit a lot
679	* of information into as few bits as possible. The format is also subject
680	* to change (there is one coming up soon). Which means that for we'll be
681	* using PGM_PAGE_GET_, PGM_PAGE_IS_ and PGM_PAGE_SET_ macros for all
682	* accesses to the structure.
683	*/
684	typedef union PGMPAGE
685	{
686	/** Structured view. */
687	struct
688	{
689	/** 1:0 - The physical handler state (PGM_PAGE_HNDL_PHYS_STATE_). /
690	uint64_t u2HandlerPhysStateY : 2;
691	/** 3:2 - Paging structure needed to map the page
692	* (PGM_PAGE_PDE_TYPE_). /
693	uint64_t u2PDETypeY : 2;
694	/** 4 - Indicator of dirty page for fault tolerance tracking. */
695	uint64_t fFTDirtyY : 1;
696	/** 5 - Flag indicating that a write monitored page was written to
697	* when set. */
698	uint64_t fWrittenToY : 1;
699	/** 7:6 - Unused. */
700	uint64_t u2Unused0 : 2;
701	/** 9:8 - The physical handler state (PGM_PAGE_HNDL_VIRT_STATE_). /
702	uint64_t u2HandlerVirtStateY : 2;
703	/** 11:10 - Unused. */
704	uint64_t u2Unused1 : 2;
705	/** 12:48 - The host physical frame number (shift left to get the
706	* address). */
707	uint64_t HCPhysFN : 36;
708	/** 50:48 - The page state. */
709	uint64_t uStateY : 3;
710	/** 51:53 - The page type (PGMPAGETYPE). */
711	uint64_t uTypeY : 3;
712	/** 63:54 - PTE index for usage tracking (page pool). */
713	uint64_t u10PteIdx : 10;
714
715	/** The GMM page ID. */
716	uint32_t idPage;
717	/** Usage tracking (page pool). */
718	uint16_t u16TrackingY;
719	/** The number of read locks on this page. */
720	uint8_t cReadLocksY;
721	/** The number of write locks on this page. */
722	uint8_t cWriteLocksY;
723	} s;
724
725	/** 64-bit integer view. */
726	uint64_t au64[2];
727	/** 16-bit view. */
728	uint32_t au32[4];
729	/** 16-bit view. */
730	uint16_t au16[8];
731	/** 8-bit view. */
732	uint8_t au8[16];
733	} PGMPAGE;
734	AssertCompileSize(PGMPAGE, 16);
735	/** Pointer to a physical guest page. */
736	typedef PGMPAGE *PPGMPAGE;
737	/** Pointer to a const physical guest page. */
738	typedef const PGMPAGE *PCPGMPAGE;
739	/** Pointer to a physical guest page pointer. */
740	typedef PPGMPAGE *PPPGMPAGE;
741
742
743	/**
744	* Clears the page structure.
745	* @param a_pPage Pointer to the physical guest page tracking structure.
746	*/
747	#define PGM_PAGE_CLEAR(a_pPage) \
748	do { \
749	(a_pPage)->au64[0] = 0; \
750	(a_pPage)->au64[1] = 0; \
751	} while (0)
752
753	/**
754	* Initializes the page structure.
755	* @param a_pPage Pointer to the physical guest page tracking structure.
756	*/
757	#define PGM_PAGE_INIT(a_pPage, a_HCPhys, a_idPage, a_uType, a_uState) \
758	do { \
759	RTHCPHYS SetHCPhysTmp = (a_HCPhys); \
760	AssertFatal(!(SetHCPhysTmp & ~UINT64_C(0x0000fffffffff000))); \
761	(a_pPage)->au64[0] = SetHCPhysTmp; \
762	(a_pPage)->au64[1] = 0; \
763	(a_pPage)->s.idPage = (a_idPage); \
764	(a_pPage)->s.uStateY = (a_uState); \
765	(a_pPage)->s.uTypeY = (a_uType); \
766	} while (0)
767
768	/**
769	* Initializes the page structure of a ZERO page.
770	* @param a_pPage Pointer to the physical guest page tracking structure.
771	* @param a_pVM The VM handle (for getting the zero page address).
772	* @param a_uType The page type (PGMPAGETYPE).
773	*/
774	#define PGM_PAGE_INIT_ZERO(a_pPage, a_pVM, a_uType) \
775	PGM_PAGE_INIT((a_pPage), (a_pVM)->pgm.s.HCPhysZeroPg, NIL_GMM_PAGEID, (a_uType), PGM_PAGE_STATE_ZERO)
776
777
778	/** @name The Page state, PGMPAGE::uStateY.
779	* @{ */
780	/** The zero page.
781	* This is a per-VM page that's never ever mapped writable. */
782	#define PGM_PAGE_STATE_ZERO 0
783	/** A allocated page.
784	* This is a per-VM page allocated from the page pool (or wherever
785	* we get MMIO2 pages from if the type is MMIO2).
786	*/
787	#define PGM_PAGE_STATE_ALLOCATED 1
788	/** A allocated page that's being monitored for writes.
789	* The shadow page table mappings are read-only. When a write occurs, the
790	* fWrittenTo member is set, the page remapped as read-write and the state
791	* moved back to allocated. */
792	#define PGM_PAGE_STATE_WRITE_MONITORED 2
793	/** The page is shared, aka. copy-on-write.
794	* This is a page that's shared with other VMs. */
795	#define PGM_PAGE_STATE_SHARED 3
796	/** The page is ballooned, so no longer available for this VM. */
797	#define PGM_PAGE_STATE_BALLOONED 4
798	/** @} */
799
800
801	/** Asserts lock ownership in some of the PGM_PAGE_XXX macros. */
802	#if defined(VBOX_STRICT) && 0 /** @todo triggers in pgmRZDynMapGCPageV2Inlined */
803	# define PGM_PAGE_ASSERT_LOCK(a_pVM) PGM_LOCK_ASSERT_OWNER(a_pVM)
804	#else
805	# define PGM_PAGE_ASSERT_LOCK(a_pVM) do { } while (0)
806	#endif
807
808	/**
809	* Gets the page state.
810	* @returns page state (PGM_PAGE_STATE_*).
811	* @param a_pPage Pointer to the physical guest page tracking structure.
812	*
813	* @remarks See PGM_PAGE_GET_HCPHYS_NA for remarks about GCC and strict
814	* builds.
815	*/
816	#define PGM_PAGE_GET_STATE_NA(a_pPage) ( (a_pPage)->s.uStateY )
817	#if defined(__GNUC__) && defined(VBOX_STRICT)
818	# define PGM_PAGE_GET_STATE(a_pPage) __extension__ ({ PGM_PAGE_ASSERT_LOCK(pVM); PGM_PAGE_GET_STATE_NA(a_pPage); })
819	#else
820	# define PGM_PAGE_GET_STATE PGM_PAGE_GET_STATE_NA
821	#endif
822
823	/**
824	* Sets the page state.
825	* @param a_pVM The VM handle, only used for lock ownership assertions.
826	* @param a_pPage Pointer to the physical guest page tracking structure.
827	* @param a_uState The new page state.
828	*/
829	#define PGM_PAGE_SET_STATE(a_pVM, a_pPage, a_uState) \
830	do { (a_pPage)->s.uStateY = (a_uState); PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0)
831
832
833	/**
834	* Gets the host physical address of the guest page.
835	* @returns host physical address (RTHCPHYS).
836	* @param a_pPage Pointer to the physical guest page tracking structure.
837	*
838	* @remarks In strict builds on gcc platforms, this macro will make some ugly
839	* assumption about a valid pVM variable/parameter being in the
840	* current context. It will use this pVM variable to assert that the
841	* PGM lock is held. Use the PGM_PAGE_GET_HCPHYS_NA in contexts where
842	* pVM is not around.
843	*/
844	#if 0
845	# define PGM_PAGE_GET_HCPHYS_NA(a_pPage) ( (a_pPage)->s.HCPhysFN << 12 )
846	# define PGM_PAGE_GET_HCPHYS PGM_PAGE_GET_HCPHYS_NA
847	#else
848	# define PGM_PAGE_GET_HCPHYS_NA(a_pPage) ( (a_pPage)->au64[0] & UINT64_C(0x0000fffffffff000) )
849	# if defined(__GNUC__) && defined(VBOX_STRICT)
850	# define PGM_PAGE_GET_HCPHYS(a_pPage) __extension__ ({ PGM_PAGE_ASSERT_LOCK(pVM); PGM_PAGE_GET_HCPHYS_NA(a_pPage); })
851	# else
852	# define PGM_PAGE_GET_HCPHYS PGM_PAGE_GET_HCPHYS_NA
853	# endif
854	#endif
855
856	/**
857	* Sets the host physical address of the guest page.
858	*
859	* @param a_pVM The VM handle, only used for lock ownership assertions.
860	* @param a_pPage Pointer to the physical guest page tracking structure.
861	* @param a_HCPhys The new host physical address.
862	*/
863	#define PGM_PAGE_SET_HCPHYS(a_pVM, a_pPage, a_HCPhys) \
864	do { \
865	RTHCPHYS const SetHCPhysTmp = (a_HCPhys); \
866	AssertFatal(!(SetHCPhysTmp & ~UINT64_C(0x0000fffffffff000))); \
867	(a_pPage)->s.HCPhysFN = SetHCPhysTmp >> 12; \
868	PGM_PAGE_ASSERT_LOCK(a_pVM); \
869	} while (0)
870
871	/**
872	* Get the Page ID.
873	* @returns The Page ID; NIL_GMM_PAGEID if it's a ZERO page.
874	* @param a_pPage Pointer to the physical guest page tracking structure.
875	*/
876	#define PGM_PAGE_GET_PAGEID(a_pPage) ( (uint32_t)(a_pPage)->s.idPage )
877
878	/**
879	* Sets the Page ID.
880	* @param a_pVM The VM handle, only used for lock ownership assertions.
881	* @param a_pPage Pointer to the physical guest page tracking structure.
882	* @param a_idPage The new page ID.
883	*/
884	#define PGM_PAGE_SET_PAGEID(a_pVM, a_pPage, a_idPage) \
885	do { \
886	(a_pPage)->s.idPage = (a_idPage); \
887	PGM_PAGE_ASSERT_LOCK(a_pVM); \
888	} while (0)
889
890	/**
891	* Get the Chunk ID.
892	* @returns The Chunk ID; NIL_GMM_CHUNKID if it's a ZERO page.
893	* @param a_pPage Pointer to the physical guest page tracking structure.
894	*/
895	#define PGM_PAGE_GET_CHUNKID(a_pPage) ( PGM_PAGE_GET_PAGEID(a_pPage) >> GMM_CHUNKID_SHIFT )
896
897	/**
898	* Get the index of the page within the allocation chunk.
899	* @returns The page index.
900	* @param a_pPage Pointer to the physical guest page tracking structure.
901	*/
902	#define PGM_PAGE_GET_PAGE_IN_CHUNK(a_pPage) ( PGM_PAGE_GET_PAGEID(a_pPage) & GMM_PAGEID_IDX_MASK )
903
904	/**
905	* Gets the page type.
906	* @returns The page type.
907	* @param a_pPage Pointer to the physical guest page tracking structure.
908	*
909	* @remarks See PGM_PAGE_GET_HCPHYS_NA for remarks about GCC and strict
910	* builds.
911	*/
912	#define PGM_PAGE_GET_TYPE_NA(a_pPage) ( (a_pPage)->s.uTypeY )
913	#if defined(__GNUC__) && defined(VBOX_STRICT)
914	# define PGM_PAGE_GET_TYPE(a_pPage) __extension__ ({ PGM_PAGE_ASSERT_LOCK(pVM); PGM_PAGE_GET_TYPE_NA(a_pPage); })
915	#else
916	# define PGM_PAGE_GET_TYPE PGM_PAGE_GET_TYPE_NA
917	#endif
918
919	/**
920	* Sets the page type.
921	*
922	* @param a_pVM The VM handle, only used for lock ownership assertions.
923	* @param a_pPage Pointer to the physical guest page tracking structure.
924	* @param a_enmType The new page type (PGMPAGETYPE).
925	*/
926	#define PGM_PAGE_SET_TYPE(a_pVM, a_pPage, a_enmType) \
927	do { (a_pPage)->s.uTypeY = (a_enmType); PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0)
928
929	/**
930	* Gets the page table index
931	* @returns The page table index.
932	* @param a_pPage Pointer to the physical guest page tracking structure.
933	*/
934	#define PGM_PAGE_GET_PTE_INDEX(a_pPage) ( (a_pPage)->s.u10PteIdx )
935
936	/**
937	* Sets the page table index.
938	* @param a_pVM The VM handle, only used for lock ownership assertions.
939	* @param a_pPage Pointer to the physical guest page tracking structure.
940	* @param a_iPte New page table index.
941	*/
942	#define PGM_PAGE_SET_PTE_INDEX(a_pVM, a_pPage, a_iPte) \
943	do { (a_pPage)->s.u10PteIdx = (a_iPte); PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0)
944
945	/**
946	* Checks if the page is marked for MMIO.
947	* @returns true/false.
948	* @param a_pPage Pointer to the physical guest page tracking structure.
949	*/
950	#define PGM_PAGE_IS_MMIO(a_pPage) ( (a_pPage)->s.uTypeY == PGMPAGETYPE_MMIO )
951
952	/**
953	* Checks if the page is backed by the ZERO page.
954	* @returns true/false.
955	* @param a_pPage Pointer to the physical guest page tracking structure.
956	*/
957	#define PGM_PAGE_IS_ZERO(a_pPage) ( (a_pPage)->s.uStateY == PGM_PAGE_STATE_ZERO )
958
959	/**
960	* Checks if the page is backed by a SHARED page.
961	* @returns true/false.
962	* @param a_pPage Pointer to the physical guest page tracking structure.
963	*/
964	#define PGM_PAGE_IS_SHARED(a_pPage) ( (a_pPage)->s.uStateY == PGM_PAGE_STATE_SHARED )
965
966	/**
967	* Checks if the page is ballooned.
968	* @returns true/false.
969	* @param a_pPage Pointer to the physical guest page tracking structure.
970	*/
971	#define PGM_PAGE_IS_BALLOONED(a_pPage) ( (a_pPage)->s.uStateY == PGM_PAGE_STATE_BALLOONED )
972
973	/**
974	* Checks if the page is allocated.
975	* @returns true/false.
976	* @param a_pPage Pointer to the physical guest page tracking structure.
977	*/
978	#define PGM_PAGE_IS_ALLOCATED(a_pPage) ( (a_pPage)->s.uStateY == PGM_PAGE_STATE_ALLOCATED )
979
980	/**
981	* Marks the page as written to (for GMM change monitoring).
982	* @param a_pVM The VM handle, only used for lock ownership assertions.
983	* @param a_pPage Pointer to the physical guest page tracking structure.
984	*/
985	#define PGM_PAGE_SET_WRITTEN_TO(a_pVM, a_pPage) \
986	do { (a_pPage)->au8[1] \|= UINT8_C(0x80); PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0) /// FIXME FIXME
987
988	/**
989	* Clears the written-to indicator.
990	* @param a_pVM The VM handle, only used for lock ownership assertions.
991	* @param a_pPage Pointer to the physical guest page tracking structure.
992	*/
993	#define PGM_PAGE_CLEAR_WRITTEN_TO(a_pVM, a_pPage) \
994	do { (a_pPage)->s.fWrittenToY = 0; PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0)
995
996	/**
997	* Checks if the page was marked as written-to.
998	* @returns true/false.
999	* @param a_pPage Pointer to the physical guest page tracking structure.
1000	*/
1001	#define PGM_PAGE_IS_WRITTEN_TO(a_pPage) ( (a_pPage)->s.fWrittenToY )
1002
1003	/**
1004	* Marks the page as dirty for FTM
1005	* @param a_pPage Pointer to the physical guest page tracking structure.
1006	*/
1007	#define PGM_PAGE_SET_FT_DIRTY(a_pPage) do { (a_pPage)->s.fFTDirtyY = 1; } while (0)
1008
1009	/**
1010	* Clears the FTM dirty indicator
1011	* @param a_pPage Pointer to the physical guest page tracking structure.
1012	*/
1013	#define PGM_PAGE_CLEAR_FT_DIRTY(a_pPage) do { (a_pPage)->s.fFTDirtyY = 0; } while (0)
1014
1015	/**
1016	* Checks if the page was marked as dirty for FTM
1017	* @returns true/false.
1018	* @param a_pPage Pointer to the physical guest page tracking structure.
1019	*/
1020	#define PGM_PAGE_IS_FT_DIRTY(a_pPage) ( (a_pPage)->s.fFTDirtyY )
1021
1022
1023	/** @name PT usage values (PGMPAGE::u2PDEType).
1024	*
1025	* @{ */
1026	/** Either as a PT or PDE. */
1027	#define PGM_PAGE_PDE_TYPE_DONTCARE 0
1028	/** Must use a page table to map the range. */
1029	#define PGM_PAGE_PDE_TYPE_PT 1
1030	/** Can use a page directory entry to map the continuous range. */
1031	#define PGM_PAGE_PDE_TYPE_PDE 2
1032	/** Can use a page directory entry to map the continuous range - temporarily disabled (by page monitoring). */
1033	#define PGM_PAGE_PDE_TYPE_PDE_DISABLED 3
1034	/** @} */
1035
1036	/**
1037	* Set the PDE type of the page
1038	* @param a_pVM The VM handle, only used for lock ownership assertions.
1039	* @param a_pPage Pointer to the physical guest page tracking structure.
1040	* @param a_uType PGM_PAGE_PDE_TYPE_*.
1041	*/
1042	#define PGM_PAGE_SET_PDE_TYPE(a_pVM, a_pPage, a_uType) \
1043	do { (a_pPage)->s.u2PDETypeY = (a_uType); PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0)
1044
1045	/**
1046	* Checks if the page was marked being part of a large page
1047	* @returns true/false.
1048	* @param a_pPage Pointer to the physical guest page tracking structure.
1049	*/
1050	#define PGM_PAGE_GET_PDE_TYPE(a_pPage) ( (a_pPage)->s.u2PDETypeY )
1051
1052	/** Enabled optimized access handler tests.
1053	* These optimizations makes ASSUMPTIONS about the state values and the s1
1054	* layout. When enabled, the compiler should normally generate more compact
1055	* code.
1056	*/
1057	#define PGM_PAGE_WITH_OPTIMIZED_HANDLER_ACCESS 1
1058
1059	/** @name Physical Access Handler State values (PGMPAGE::u2HandlerPhysStateY).
1060	*
1061	* @remarks The values are assigned in order of priority, so we can calculate
1062	* the correct state for a page with different handlers installed.
1063	* @{ */
1064	/** No handler installed. */
1065	#define PGM_PAGE_HNDL_PHYS_STATE_NONE 0
1066	/** Monitoring is temporarily disabled. */
1067	#define PGM_PAGE_HNDL_PHYS_STATE_DISABLED 1
1068	/** Write access is monitored. */
1069	#define PGM_PAGE_HNDL_PHYS_STATE_WRITE 2
1070	/** All access is monitored. */
1071	#define PGM_PAGE_HNDL_PHYS_STATE_ALL 3
1072	/** @} */
1073
1074	/**
1075	* Gets the physical access handler state of a page.
1076	* @returns PGM_PAGE_HNDL_PHYS_STATE_* value.
1077	* @param a_pPage Pointer to the physical guest page tracking structure.
1078	*/
1079	#define PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) ( (a_pPage)->s.u2HandlerPhysStateY )
1080
1081	/**
1082	* Sets the physical access handler state of a page.
1083	* @param a_pPage Pointer to the physical guest page tracking structure.
1084	* @param a_uState The new state value.
1085	*/
1086	#define PGM_PAGE_SET_HNDL_PHYS_STATE(a_pPage, a_uState) \
1087	do { (a_pPage)->s.u2HandlerPhysStateY = (a_uState); } while (0)
1088
1089	/**
1090	* Checks if the page has any physical access handlers, including temporarily disabled ones.
1091	* @returns true/false
1092	* @param a_pPage Pointer to the physical guest page tracking structure.
1093	*/
1094	#define PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(a_pPage) \
1095	( PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE )
1096
1097	/**
1098	* Checks if the page has any active physical access handlers.
1099	* @returns true/false
1100	* @param a_pPage Pointer to the physical guest page tracking structure.
1101	*/
1102	#define PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(a_pPage) \
1103	( PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) >= PGM_PAGE_HNDL_PHYS_STATE_WRITE )
1104
1105
1106	/** @name Virtual Access Handler State values (PGMPAGE::u2HandlerVirtStateY).
1107	*
1108	* @remarks The values are assigned in order of priority, so we can calculate
1109	* the correct state for a page with different handlers installed.
1110	* @{ */
1111	/** No handler installed. */
1112	#define PGM_PAGE_HNDL_VIRT_STATE_NONE 0
1113	/* 1 is reserved so the lineup is identical with the physical ones. */
1114	/** Write access is monitored. */
1115	#define PGM_PAGE_HNDL_VIRT_STATE_WRITE 2
1116	/** All access is monitored. */
1117	#define PGM_PAGE_HNDL_VIRT_STATE_ALL 3
1118	/** @} */
1119
1120	/**
1121	* Gets the virtual access handler state of a page.
1122	* @returns PGM_PAGE_HNDL_VIRT_STATE_* value.
1123	* @param a_pPage Pointer to the physical guest page tracking structure.
1124	*/
1125	#define PGM_PAGE_GET_HNDL_VIRT_STATE(a_pPage) ( (a_pPage)->s.u2HandlerVirtStateY )
1126
1127	/**
1128	* Sets the virtual access handler state of a page.
1129	* @param a_pPage Pointer to the physical guest page tracking structure.
1130	* @param a_uState The new state value.
1131	*/
1132	#define PGM_PAGE_SET_HNDL_VIRT_STATE(a_pPage, a_uState) \
1133	do { (a_pPage)->s.u2HandlerVirtStateY = (a_uState); } while (0)
1134
1135	/**
1136	* Checks if the page has any virtual access handlers.
1137	* @returns true/false
1138	* @param a_pPage Pointer to the physical guest page tracking structure.
1139	*/
1140	#define PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS(a_pPage) \
1141	( PGM_PAGE_GET_HNDL_VIRT_STATE(a_pPage) != PGM_PAGE_HNDL_VIRT_STATE_NONE )
1142
1143	/**
1144	* Same as PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS - can't disable pages in
1145	* virtual handlers.
1146	* @returns true/false
1147	* @param a_pPage Pointer to the physical guest page tracking structure.
1148	*/
1149	#define PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(a_pPage) \
1150	PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS(a_pPage)
1151
1152
1153	/**
1154	* Checks if the page has any access handlers, including temporarily disabled ones.
1155	* @returns true/false
1156	* @param a_pPage Pointer to the physical guest page tracking structure.
1157	*/
1158	#ifdef PGM_PAGE_WITH_OPTIMIZED_HANDLER_ACCESS
1159	# define PGM_PAGE_HAS_ANY_HANDLERS(a_pPage) \
1160	( ((a_pPage)->au32[0] & UINT16_C(0x0303)) != 0 )
1161	#else
1162	# define PGM_PAGE_HAS_ANY_HANDLERS(a_pPage) \
1163	( PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE \
1164	\|\| PGM_PAGE_GET_HNDL_VIRT_STATE(a_pPage) != PGM_PAGE_HNDL_VIRT_STATE_NONE )
1165	#endif
1166
1167	/**
1168	* Checks if the page has any active access handlers.
1169	* @returns true/false
1170	* @param a_pPage Pointer to the physical guest page tracking structure.
1171	*/
1172	#ifdef PGM_PAGE_WITH_OPTIMIZED_HANDLER_ACCESS
1173	# define PGM_PAGE_HAS_ACTIVE_HANDLERS(a_pPage) \
1174	( ((a_pPage)->au32[0] & UINT16_C(0x0202)) != 0 )
1175	#else
1176	# define PGM_PAGE_HAS_ACTIVE_HANDLERS(a_pPage) \
1177	( PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) >= PGM_PAGE_HNDL_PHYS_STATE_WRITE \
1178	\|\| PGM_PAGE_GET_HNDL_VIRT_STATE(a_pPage) >= PGM_PAGE_HNDL_VIRT_STATE_WRITE )
1179	#endif
1180
1181	/**
1182	* Checks if the page has any active access handlers catching all accesses.
1183	* @returns true/false
1184	* @param a_pPage Pointer to the physical guest page tracking structure.
1185	*/
1186	#ifdef PGM_PAGE_WITH_OPTIMIZED_HANDLER_ACCESS
1187	# define PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(a_pPage) \
1188	( ( ((a_pPage)->au8[0] \| (a_pPage)->au8[1]) & UINT8_C(0x3) ) \
1189	== PGM_PAGE_HNDL_PHYS_STATE_ALL )
1190	#else
1191	# define PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(a_pPage) \
1192	( PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) == PGM_PAGE_HNDL_PHYS_STATE_ALL \
1193	\|\| PGM_PAGE_GET_HNDL_VIRT_STATE(a_pPage) == PGM_PAGE_HNDL_VIRT_STATE_ALL )
1194	#endif
1195
1196
1197	/** @def PGM_PAGE_GET_TRACKING
1198	* Gets the packed shadow page pool tracking data associated with a guest page.
1199	* @returns uint16_t containing the data.
1200	* @param a_pPage Pointer to the physical guest page tracking structure.
1201	*/
1202	#define PGM_PAGE_GET_TRACKING_NA(a_pPage) ( (a_pPage)->s.u16TrackingY )
1203	#if defined(__GNUC__) && defined(VBOX_STRICT)
1204	# define PGM_PAGE_GET_TRACKING(a_pPage) __extension__ ({ PGM_PAGE_ASSERT_LOCK(pVM); PGM_PAGE_GET_TRACKING_NA(a_pPage); })
1205	#else
1206	# define PGM_PAGE_GET_TRACKING PGM_PAGE_GET_TRACKING_NA
1207	#endif
1208
1209	/** @def PGM_PAGE_SET_TRACKING
1210	* Sets the packed shadow page pool tracking data associated with a guest page.
1211	* @param a_pVM The VM handle, only used for lock ownership assertions.
1212	* @param a_pPage Pointer to the physical guest page tracking structure.
1213	* @param a_u16TrackingData The tracking data to store.
1214	*/
1215	#define PGM_PAGE_SET_TRACKING(a_pVM, a_pPage, a_u16TrackingData) \
1216	do { (a_pPage)->s.u16TrackingY = (a_u16TrackingData); PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0)
1217
1218	/** @def PGM_PAGE_GET_TD_CREFS
1219	* Gets the @a cRefs tracking data member.
1220	* @returns cRefs.
1221	* @param a_pPage Pointer to the physical guest page tracking structure.
1222	*/
1223	#define PGM_PAGE_GET_TD_CREFS(a_pPage) \
1224	((PGM_PAGE_GET_TRACKING(a_pPage) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK)
1225	#define PGM_PAGE_GET_TD_CREFS_NA(a_pPage) \
1226	((PGM_PAGE_GET_TRACKING_NA(a_pPage) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK)
1227
1228	/** @def PGM_PAGE_GET_TD_IDX
1229	* Gets the @a idx tracking data member.
1230	* @returns idx.
1231	* @param a_pPage Pointer to the physical guest page tracking structure.
1232	*/
1233	#define PGM_PAGE_GET_TD_IDX(a_pPage) \
1234	((PGM_PAGE_GET_TRACKING(a_pPage) >> PGMPOOL_TD_IDX_SHIFT) & PGMPOOL_TD_IDX_MASK)
1235	#define PGM_PAGE_GET_TD_IDX_NA(a_pPage) \
1236	((PGM_PAGE_GET_TRACKING_NA(a_pPage) >> PGMPOOL_TD_IDX_SHIFT) & PGMPOOL_TD_IDX_MASK)
1237
1238
1239	/** Max number of locks on a page. */
1240	#define PGM_PAGE_MAX_LOCKS UINT8_C(254)
1241
1242	/** Get the read lock count.
1243	* @returns count.
1244	* @param a_pPage Pointer to the physical guest page tracking structure.
1245	*/
1246	#define PGM_PAGE_GET_READ_LOCKS(a_pPage) ( (a_pPage)->s.cReadLocksY )
1247
1248	/** Get the write lock count.
1249	* @returns count.
1250	* @param a_pPage Pointer to the physical guest page tracking structure.
1251	*/
1252	#define PGM_PAGE_GET_WRITE_LOCKS(a_pPage) ( (a_pPage)->s.cWriteLocksY )
1253
1254	/** Decrement the read lock counter.
1255	* @param a_pPage Pointer to the physical guest page tracking structure.
1256	*/
1257	#define PGM_PAGE_DEC_READ_LOCKS(a_pPage) do { --(a_pPage)->s.cReadLocksY; } while (0)
1258
1259	/** Decrement the write lock counter.
1260	* @param a_pPage Pointer to the physical guest page tracking structure.
1261	*/
1262	#define PGM_PAGE_DEC_WRITE_LOCKS(a_pPage) do { --(a_pPage)->s.cWriteLocksY; } while (0)
1263
1264	/** Increment the read lock counter.
1265	* @param a_pPage Pointer to the physical guest page tracking structure.
1266	*/
1267	#define PGM_PAGE_INC_READ_LOCKS(a_pPage) do { ++(a_pPage)->s.cReadLocksY; } while (0)
1268
1269	/** Increment the write lock counter.
1270	* @param a_pPage Pointer to the physical guest page tracking structure.
1271	*/
1272	#define PGM_PAGE_INC_WRITE_LOCKS(a_pPage) do { ++(a_pPage)->s.cWriteLocksY; } while (0)
1273
1274
1275	#if 0
1276	/** Enables sanity checking of write monitoring using CRC-32. */
1277	# define PGMLIVESAVERAMPAGE_WITH_CRC32
1278	#endif
1279
1280	/**
1281	* Per page live save tracking data.
1282	*/
1283	typedef struct PGMLIVESAVERAMPAGE
1284	{
1285	/** Number of times it has been dirtied. */
1286	uint32_t cDirtied : 24;
1287	/** Whether it is currently dirty. */
1288	uint32_t fDirty : 1;
1289	/** Ignore the page.
1290	* This is used for pages that has been MMIO, MMIO2 or ROM pages once. We will
1291	* deal with these after pausing the VM and DevPCI have said it bit about
1292	* remappings. */
1293	uint32_t fIgnore : 1;
1294	/** Was a ZERO page last time around. */
1295	uint32_t fZero : 1;
1296	/** Was a SHARED page last time around. */
1297	uint32_t fShared : 1;
1298	/** Whether the page is/was write monitored in a previous pass. */
1299	uint32_t fWriteMonitored : 1;
1300	/** Whether the page is/was write monitored earlier in this pass. */
1301	uint32_t fWriteMonitoredJustNow : 1;
1302	/** Bits reserved for future use. */
1303	uint32_t u2Reserved : 2;
1304	#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
1305	/** CRC-32 for the page. This is for internal consistency checks. */
1306	uint32_t u32Crc;
1307	#endif
1308	} PGMLIVESAVERAMPAGE;
1309	#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
1310	AssertCompileSize(PGMLIVESAVERAMPAGE, 8);
1311	#else
1312	AssertCompileSize(PGMLIVESAVERAMPAGE, 4);
1313	#endif
1314	/** Pointer to the per page live save tracking data. */
1315	typedef PGMLIVESAVERAMPAGE *PPGMLIVESAVERAMPAGE;
1316
1317	/** The max value of PGMLIVESAVERAMPAGE::cDirtied. */
1318	#define PGMLIVSAVEPAGE_MAX_DIRTIED 0x00fffff0
1319
1320
1321	/**
1322	* RAM range for GC Phys to HC Phys conversion.
1323	*
1324	* Can be used for HC Virt to GC Phys and HC Virt to HC Phys
1325	* conversions too, but we'll let MM handle that for now.
1326	*
1327	* This structure is used by linked lists in both GC and HC.
1328	*/
1329	typedef struct PGMRAMRANGE
1330	{
1331	/** Start of the range. Page aligned. */
1332	RTGCPHYS GCPhys;
1333	/** Size of the range. (Page aligned of course). */
1334	RTGCPHYS cb;
1335	/** Pointer to the next RAM range - for R3. */
1336	R3PTRTYPE(struct PGMRAMRANGE *) pNextR3;
1337	/** Pointer to the next RAM range - for R0. */
1338	R0PTRTYPE(struct PGMRAMRANGE *) pNextR0;
1339	/** Pointer to the next RAM range - for RC. */
1340	RCPTRTYPE(struct PGMRAMRANGE *) pNextRC;
1341	/** PGM_RAM_RANGE_FLAGS_* flags. */
1342	uint32_t fFlags;
1343	/** Last address in the range (inclusive). Page aligned (-1). */
1344	RTGCPHYS GCPhysLast;
1345	/** Start of the HC mapping of the range. This is only used for MMIO2. */
1346	R3PTRTYPE(void *) pvR3;
1347	/** Live save per page tracking data. */
1348	R3PTRTYPE(PPGMLIVESAVERAMPAGE) paLSPages;
1349	/** The range description. */
1350	R3PTRTYPE(const char *) pszDesc;
1351	/** Pointer to self - R0 pointer. */
1352	R0PTRTYPE(struct PGMRAMRANGE *) pSelfR0;
1353	/** Pointer to self - RC pointer. */
1354	RCPTRTYPE(struct PGMRAMRANGE *) pSelfRC;
1355
1356	/** Alignment padding. */
1357	RTRCPTR Alignment0;
1358	/** Pointer to the left search three node - ring-3 context. */
1359	R3PTRTYPE(struct PGMRAMRANGE *) pLeftR3;
1360	/** Pointer to the right search three node - ring-3 context. */
1361	R3PTRTYPE(struct PGMRAMRANGE *) pRightR3;
1362	/** Pointer to the left search three node - ring-0 context. */
1363	R0PTRTYPE(struct PGMRAMRANGE *) pLeftR0;
1364	/** Pointer to the right search three node - ring-0 context. */
1365	R0PTRTYPE(struct PGMRAMRANGE *) pRightR0;
1366	/** Pointer to the left search three node - raw-mode context. */
1367	RCPTRTYPE(struct PGMRAMRANGE *) pLeftRC;
1368	/** Pointer to the right search three node - raw-mode context. */
1369	RCPTRTYPE(struct PGMRAMRANGE *) pRightRC;
1370
1371	/** Padding to make aPage aligned on sizeof(PGMPAGE). */
1372	#if HC_ARCH_BITS == 32
1373	uint32_t au32Alignment2[HC_ARCH_BITS == 32 ? 2 : 0];
1374	#endif
1375	/** Array of physical guest page tracking structures. */
1376	PGMPAGE aPages[1];
1377	} PGMRAMRANGE;
1378	/** Pointer to RAM range for GC Phys to HC Phys conversion. */
1379	typedef PGMRAMRANGE *PPGMRAMRANGE;
1380
1381	/** @name PGMRAMRANGE::fFlags
1382	* @{ */
1383	/** The RAM range is floating around as an independent guest mapping. */
1384	#define PGM_RAM_RANGE_FLAGS_FLOATING RT_BIT(20)
1385	/** Ad hoc RAM range for an ROM mapping. */
1386	#define PGM_RAM_RANGE_FLAGS_AD_HOC_ROM RT_BIT(21)
1387	/** Ad hoc RAM range for an MMIO mapping. */
1388	#define PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO RT_BIT(22)
1389	/** Ad hoc RAM range for an MMIO2 mapping. */
1390	#define PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO2 RT_BIT(23)
1391	/** @} */
1392
1393	/** Tests if a RAM range is an ad hoc one or not.
1394	* @returns true/false.
1395	* @param pRam The RAM range.
1396	*/
1397	#define PGM_RAM_RANGE_IS_AD_HOC(pRam) \
1398	(!!( (pRam)->fFlags & (PGM_RAM_RANGE_FLAGS_AD_HOC_ROM \| PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO \| PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO2) ) )
1399
1400	/** The number of entries in the RAM range TLBs (there is one for each
1401	* context). Must be a power of two. */
1402	#define PGM_RAMRANGE_TLB_ENTRIES 8
1403
1404	/**
1405	* Calculates the RAM range TLB index for the physical address.
1406	*
1407	* @returns RAM range TLB index.
1408	* @param GCPhys The guest physical address.
1409	*/
1410	#define PGM_RAMRANGE_TLB_IDX(a_GCPhys) ( ((a_GCPhys) >> 20) & (PGM_RAMRANGE_TLB_ENTRIES - 1) )
1411
1412
1413
1414	/**
1415	* Per page tracking structure for ROM image.
1416	*
1417	* A ROM image may have a shadow page, in which case we may have two pages
1418	* backing it. This structure contains the PGMPAGE for both while
1419	* PGMRAMRANGE have a copy of the active one. It is important that these
1420	* aren't out of sync in any regard other than page pool tracking data.
1421	*/
1422	typedef struct PGMROMPAGE
1423	{
1424	/** The page structure for the virgin ROM page. */
1425	PGMPAGE Virgin;
1426	/** The page structure for the shadow RAM page. */
1427	PGMPAGE Shadow;
1428	/** The current protection setting. */
1429	PGMROMPROT enmProt;
1430	/** Live save status information. Makes use of unused alignment space. */
1431	struct
1432	{
1433	/** The previous protection value. */
1434	uint8_t u8Prot;
1435	/** Written to flag set by the handler. */
1436	bool fWrittenTo;
1437	/** Whether the shadow page is dirty or not. */
1438	bool fDirty;
1439	/** Whether it was dirtied in the recently. */
1440	bool fDirtiedRecently;
1441	} LiveSave;
1442	} PGMROMPAGE;
1443	AssertCompileSizeAlignment(PGMROMPAGE, 8);
1444	/** Pointer to a ROM page tracking structure. */
1445	typedef PGMROMPAGE *PPGMROMPAGE;
1446
1447
1448	/**
1449	* A registered ROM image.
1450	*
1451	* This is needed to keep track of ROM image since they generally intrude
1452	* into a PGMRAMRANGE. It also keeps track of additional info like the
1453	* two page sets (read-only virgin and read-write shadow), the current
1454	* state of each page.
1455	*
1456	* Because access handlers cannot easily be executed in a different
1457	* context, the ROM ranges needs to be accessible and in all contexts.
1458	*/
1459	typedef struct PGMROMRANGE
1460	{
1461	/** Pointer to the next range - R3. */
1462	R3PTRTYPE(struct PGMROMRANGE *) pNextR3;
1463	/** Pointer to the next range - R0. */
1464	R0PTRTYPE(struct PGMROMRANGE *) pNextR0;
1465	/** Pointer to the next range - RC. */
1466	RCPTRTYPE(struct PGMROMRANGE *) pNextRC;
1467	/** Pointer alignment */
1468	RTRCPTR RCPtrAlignment;
1469	/** Address of the range. */
1470	RTGCPHYS GCPhys;
1471	/** Address of the last byte in the range. */
1472	RTGCPHYS GCPhysLast;
1473	/** Size of the range. */
1474	RTGCPHYS cb;
1475	/** The flags (PGMPHYS_ROM_FLAGS_). /
1476	uint32_t fFlags;
1477	/** The saved state range ID. */
1478	uint8_t idSavedState;
1479	/** Alignment padding. */
1480	uint8_t au8Alignment[3];
1481	/** Alignment padding ensuring that aPages is sizeof(PGMROMPAGE) aligned. */
1482	uint32_t au32Alignemnt[HC_ARCH_BITS == 32 ? 5 : 1];
1483	/** The size bits pvOriginal points to. */
1484	uint32_t cbOriginal;
1485	/** Pointer to the original bits when PGMPHYS_ROM_FLAGS_PERMANENT_BINARY was specified.
1486	* This is used for strictness checks. */
1487	R3PTRTYPE(const void *) pvOriginal;
1488	/** The ROM description. */
1489	R3PTRTYPE(const char *) pszDesc;
1490	/** The per page tracking structures. */
1491	PGMROMPAGE aPages[1];
1492	} PGMROMRANGE;
1493	/** Pointer to a ROM range. */
1494	typedef PGMROMRANGE *PPGMROMRANGE;
1495
1496
1497	/**
1498	* Live save per page data for an MMIO2 page.
1499	*
1500	* Not using PGMLIVESAVERAMPAGE here because we cannot use normal write monitoring
1501	* of MMIO2 pages. The current approach is using some optimistic SHA-1 +
1502	* CRC-32 for detecting changes as well as special handling of zero pages. This
1503	* is a TEMPORARY measure which isn't perfect, but hopefully it is good enough
1504	* for speeding things up. (We're using SHA-1 and not SHA-256 or SHA-512
1505	* because of speed (2.5x and 6x slower).)
1506	*
1507	* @todo Implement dirty MMIO2 page reporting that can be enabled during live
1508	* save but normally is disabled. Since we can write monitor guest
1509	* accesses on our own, we only need this for host accesses. Shouldn't be
1510	* too difficult for DevVGA, VMMDev might be doable, the planned
1511	* networking fun will be fun since it involves ring-0.
1512	*/
1513	typedef struct PGMLIVESAVEMMIO2PAGE
1514	{
1515	/** Set if the page is considered dirty. */
1516	bool fDirty;
1517	/** The number of scans this page has remained unchanged for.
1518	* Only updated for dirty pages. */
1519	uint8_t cUnchangedScans;
1520	/** Whether this page was zero at the last scan. */
1521	bool fZero;
1522	/** Alignment padding. */
1523	bool fReserved;
1524	/** CRC-32 for the first half of the page.
1525	* This is used together with u32CrcH2 to quickly detect changes in the page
1526	* during the non-final passes. */
1527	uint32_t u32CrcH1;
1528	/** CRC-32 for the second half of the page. */
1529	uint32_t u32CrcH2;
1530	/** SHA-1 for the saved page.
1531	* This is used in the final pass to skip pages without changes. */
1532	uint8_t abSha1Saved[RTSHA1_HASH_SIZE];
1533	} PGMLIVESAVEMMIO2PAGE;
1534	/** Pointer to a live save status data for an MMIO2 page. */
1535	typedef PGMLIVESAVEMMIO2PAGE *PPGMLIVESAVEMMIO2PAGE;
1536
1537	/**
1538	* A registered MMIO2 (= Device RAM) range.
1539	*
1540	* There are a few reason why we need to keep track of these
1541	* registrations. One of them is the deregistration & cleanup stuff,
1542	* while another is that the PGMRAMRANGE associated with such a region may
1543	* have to be removed from the ram range list.
1544	*
1545	* Overlapping with a RAM range has to be 100% or none at all. The pages
1546	* in the existing RAM range must not be ROM nor MMIO. A guru meditation
1547	* will be raised if a partial overlap or an overlap of ROM pages is
1548	* encountered. On an overlap we will free all the existing RAM pages and
1549	* put in the ram range pages instead.
1550	*/
1551	typedef struct PGMMMIO2RANGE
1552	{
1553	/** The owner of the range. (a device) */
1554	PPDMDEVINSR3 pDevInsR3;
1555	/** Pointer to the ring-3 mapping of the allocation. */
1556	RTR3PTR pvR3;
1557	/** Pointer to the next range - R3. */
1558	R3PTRTYPE(struct PGMMMIO2RANGE *) pNextR3;
1559	/** Whether it's mapped or not. */
1560	bool fMapped;
1561	/** Whether it's overlapping or not. */
1562	bool fOverlapping;
1563	/** The PCI region number.
1564	* @remarks This ASSUMES that nobody will ever really need to have multiple
1565	* PCI devices with matching MMIO region numbers on a single device. */
1566	uint8_t iRegion;
1567	/** The saved state range ID. */
1568	uint8_t idSavedState;
1569	/** Alignment padding for putting the ram range on a PGMPAGE alignment boundary. */
1570	uint8_t abAlignemnt[HC_ARCH_BITS == 32 ? 12 : 12];
1571	/** Live save per page tracking data. */
1572	R3PTRTYPE(PPGMLIVESAVEMMIO2PAGE) paLSPages;
1573	/** The associated RAM range. */
1574	PGMRAMRANGE RamRange;
1575	} PGMMMIO2RANGE;
1576	/** Pointer to a MMIO2 range. */
1577	typedef PGMMMIO2RANGE *PPGMMMIO2RANGE;
1578
1579
1580
1581
1582	/**
1583	* PGMPhysRead/Write cache entry
1584	*/
1585	typedef struct PGMPHYSCACHEENTRY
1586	{
1587	/** R3 pointer to physical page. */
1588	R3PTRTYPE(uint8_t *) pbR3;
1589	/** GC Physical address for cache entry */
1590	RTGCPHYS GCPhys;
1591	#if HC_ARCH_BITS == 64 && GC_ARCH_BITS == 32
1592	RTGCPHYS u32Padding0; /*< alignment padding. /
1593	#endif
1594	} PGMPHYSCACHEENTRY;
1595
1596	/**
1597	* PGMPhysRead/Write cache to reduce REM memory access overhead
1598	*/
1599	typedef struct PGMPHYSCACHE
1600	{
1601	/** Bitmap of valid cache entries */
1602	uint64_t aEntries;
1603	/** Cache entries */
1604	PGMPHYSCACHEENTRY Entry[PGM_MAX_PHYSCACHE_ENTRIES];
1605	} PGMPHYSCACHE;
1606
1607
1608	/** Pointer to an allocation chunk ring-3 mapping. */
1609	typedef struct PGMCHUNKR3MAP *PPGMCHUNKR3MAP;
1610	/** Pointer to an allocation chunk ring-3 mapping pointer. */
1611	typedef PPGMCHUNKR3MAP *PPPGMCHUNKR3MAP;
1612
1613	/**
1614	* Ring-3 tracking structore for an allocation chunk ring-3 mapping.
1615	*
1616	* The primary tree (Core) uses the chunk id as key.
1617	*/
1618	typedef struct PGMCHUNKR3MAP
1619	{
1620	/** The key is the chunk id. */
1621	AVLU32NODECORE Core;
1622	/** The time (ChunkR3Map.iNow) this chunk was last used. Used for unmap
1623	* selection. */
1624	uint32_t iLastUsed;
1625	/** The current reference count. */
1626	uint32_t volatile cRefs;
1627	/** The current permanent reference count. */
1628	uint32_t volatile cPermRefs;
1629	/** The mapping address. */
1630	void *pv;
1631	} PGMCHUNKR3MAP;
1632
1633	/**
1634	* Allocation chunk ring-3 mapping TLB entry.
1635	*/
1636	typedef struct PGMCHUNKR3MAPTLBE
1637	{
1638	/** The chunk id. */
1639	uint32_t volatile idChunk;
1640	#if HC_ARCH_BITS == 64
1641	uint32_t u32Padding; /*< alignment padding. /
1642	#endif
1643	/** The chunk map. */
1644	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1645	R3PTRTYPE(PPGMCHUNKR3MAP) volatile pChunk;
1646	#else
1647	R3R0PTRTYPE(PPGMCHUNKR3MAP) volatile pChunk;
1648	#endif
1649	} PGMCHUNKR3MAPTLBE;
1650	/** Pointer to the an allocation chunk ring-3 mapping TLB entry. */
1651	typedef PGMCHUNKR3MAPTLBE *PPGMCHUNKR3MAPTLBE;
1652
1653	/** The number of TLB entries in PGMCHUNKR3MAPTLB.
1654	* @remark Must be a power of two value. */
1655	#define PGM_CHUNKR3MAPTLB_ENTRIES 64
1656
1657	/**
1658	* Allocation chunk ring-3 mapping TLB.
1659	*
1660	* @remarks We use a TLB to speed up lookups by avoiding walking the AVL.
1661	* At first glance this might look kinda odd since AVL trees are
1662	* supposed to give the most optimal lookup times of all trees
1663	* due to their balancing. However, take a tree with 1023 nodes
1664	* in it, that's 10 levels, meaning that most searches has to go
1665	* down 9 levels before they find what they want. This isn't fast
1666	* compared to a TLB hit. There is the factor of cache misses,
1667	* and of course the problem with trees and branch prediction.
1668	* This is why we use TLBs in front of most of the trees.
1669	*
1670	* @todo Generalize this TLB + AVL stuff, shouldn't be all that
1671	* difficult when we switch to the new inlined AVL trees (from kStuff).
1672	*/
1673	typedef struct PGMCHUNKR3MAPTLB
1674	{
1675	/** The TLB entries. */
1676	PGMCHUNKR3MAPTLBE aEntries[PGM_CHUNKR3MAPTLB_ENTRIES];
1677	} PGMCHUNKR3MAPTLB;
1678
1679	/**
1680	* Calculates the index of a guest page in the Ring-3 Chunk TLB.
1681	* @returns Chunk TLB index.
1682	* @param idChunk The Chunk ID.
1683	*/
1684	#define PGM_CHUNKR3MAPTLB_IDX(idChunk) ( (idChunk) & (PGM_CHUNKR3MAPTLB_ENTRIES - 1) )
1685
1686
1687	/**
1688	* Ring-3 guest page mapping TLB entry.
1689	* @remarks used in ring-0 as well at the moment.
1690	*/
1691	typedef struct PGMPAGER3MAPTLBE
1692	{
1693	/** Address of the page. */
1694	RTGCPHYS volatile GCPhys;
1695	/** The guest page. */
1696	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1697	R3PTRTYPE(PPGMPAGE) volatile pPage;
1698	#else
1699	R3R0PTRTYPE(PPGMPAGE) volatile pPage;
1700	#endif
1701	/** Pointer to the page mapping tracking structure, PGMCHUNKR3MAP. */
1702	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1703	R3PTRTYPE(PPGMCHUNKR3MAP) volatile pMap;
1704	#else
1705	R3R0PTRTYPE(PPGMCHUNKR3MAP) volatile pMap;
1706	#endif
1707	/** The address */
1708	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1709	R3PTRTYPE(void *) volatile pv;
1710	#else
1711	R3R0PTRTYPE(void *) volatile pv;
1712	#endif
1713	#if HC_ARCH_BITS == 32
1714	uint32_t u32Padding; /*< alignment padding. /
1715	#endif
1716	} PGMPAGER3MAPTLBE;
1717	/** Pointer to an entry in the HC physical TLB. */
1718	typedef PGMPAGER3MAPTLBE *PPGMPAGER3MAPTLBE;
1719
1720
1721	/** The number of entries in the ring-3 guest page mapping TLB.
1722	* @remarks The value must be a power of two. */
1723	#define PGM_PAGER3MAPTLB_ENTRIES 256
1724
1725	/**
1726	* Ring-3 guest page mapping TLB.
1727	* @remarks used in ring-0 as well at the moment.
1728	*/
1729	typedef struct PGMPAGER3MAPTLB
1730	{
1731	/** The TLB entries. */
1732	PGMPAGER3MAPTLBE aEntries[PGM_PAGER3MAPTLB_ENTRIES];
1733	} PGMPAGER3MAPTLB;
1734	/** Pointer to the ring-3 guest page mapping TLB. */
1735	typedef PGMPAGER3MAPTLB *PPGMPAGER3MAPTLB;
1736
1737	/**
1738	* Calculates the index of the TLB entry for the specified guest page.
1739	* @returns Physical TLB index.
1740	* @param GCPhys The guest physical address.
1741	*/
1742	#define PGM_PAGER3MAPTLB_IDX(GCPhys) ( ((GCPhys) >> PAGE_SHIFT) & (PGM_PAGER3MAPTLB_ENTRIES - 1) )
1743
1744
1745	/**
1746	* Raw-mode context dynamic mapping cache entry.
1747	*
1748	* Because of raw-mode context being reloctable and all relocations are applied
1749	* in ring-3, this has to be defined here and be RC specific.
1750	*
1751	* @sa PGMRZDYNMAPENTRY, PGMR0DYNMAPENTRY.
1752	*/
1753	typedef struct PGMRCDYNMAPENTRY
1754	{
1755	/** The physical address of the currently mapped page.
1756	* This is duplicate for three reasons: cache locality, cache policy of the PT
1757	* mappings and sanity checks. */
1758	RTHCPHYS HCPhys;
1759	/** Pointer to the page. */
1760	RTRCPTR pvPage;
1761	/** The number of references. */
1762	int32_t volatile cRefs;
1763	/** PTE pointer union. */
1764	struct PGMRCDYNMAPENTRY_PPTE
1765	{
1766	/** PTE pointer, 32-bit legacy version. */
1767	RCPTRTYPE(PX86PTE) pLegacy;
1768	/** PTE pointer, PAE version. */
1769	RCPTRTYPE(PX86PTEPAE) pPae;
1770	} uPte;
1771	} PGMRCDYNMAPENTRY;
1772	/** Pointer to a dynamic mapping cache entry for the raw-mode context. */
1773	typedef PGMRCDYNMAPENTRY *PPGMRCDYNMAPENTRY;
1774
1775
1776	/**
1777	* Dynamic mapping cache for the raw-mode context.
1778	*
1779	* This is initialized during VMMRC init based upon the pbDynPageMapBaseGC and
1780	* paDynPageMap* PGM members. However, it has to be defined in PGMInternal.h
1781	* so that we can perform relocations from PGMR3Relocate. This has the
1782	* consequence that we must have separate ring-0 and raw-mode context versions
1783	* of this struct even if they share the basic elements.
1784	*
1785	* @sa PPGMRZDYNMAP, PGMR0DYNMAP.
1786	*/
1787	typedef struct PGMRCDYNMAP
1788	{
1789	/** The usual magic number / eye catcher (PGMRZDYNMAP_MAGIC). */
1790	uint32_t u32Magic;
1791	/** Array for tracking and managing the pages. */
1792	RCPTRTYPE(PPGMRCDYNMAPENTRY) paPages;
1793	/** The cache size given as a number of pages. */
1794	uint32_t cPages;
1795	/** The current load.
1796	* This does not include guard pages. */
1797	uint32_t cLoad;
1798	/** The max load ever.
1799	* This is maintained to get trigger adding of more mapping space. */
1800	uint32_t cMaxLoad;
1801	/** The number of guard pages. */
1802	uint32_t cGuardPages;
1803	/** The number of users (protected by hInitLock). */
1804	uint32_t cUsers;
1805	} PGMRCDYNMAP;
1806	/** Pointer to the dynamic cache for the raw-mode context. */
1807	typedef PGMRCDYNMAP *PPGMRCDYNMAP;
1808
1809
1810	/**
1811	* Mapping cache usage set entry.
1812	*
1813	* @remarks 16-bit ints was chosen as the set is not expected to be used beyond
1814	* the dynamic ring-0 and (to some extent) raw-mode context mapping
1815	* cache. If it's extended to include ring-3, well, then something
1816	* will have be changed here...
1817	*/
1818	typedef struct PGMMAPSETENTRY
1819	{
1820	/** Pointer to the page. */
1821	#ifndef IN_RC
1822	RTR0PTR pvPage;
1823	#else
1824	RTRCPTR pvPage;
1825	# if HC_ARCH_BITS == 64
1826	uint32_t u32Alignment2;
1827	# endif
1828	#endif
1829	/** The mapping cache index. */
1830	uint16_t iPage;
1831	/** The number of references.
1832	* The max is UINT16_MAX - 1. */
1833	uint16_t cRefs;
1834	/** The number inlined references.
1835	* The max is UINT16_MAX - 1. */
1836	uint16_t cInlinedRefs;
1837	/** Unreferences. */
1838	uint16_t cUnrefs;
1839
1840	#if HC_ARCH_BITS == 32
1841	uint32_t u32Alignment1;
1842	#endif
1843	/** The physical address for this entry. */
1844	RTHCPHYS HCPhys;
1845	} PGMMAPSETENTRY;
1846	AssertCompileMemberOffset(PGMMAPSETENTRY, iPage, RT_MAX(sizeof(RTR0PTR), sizeof(RTRCPTR)));
1847	AssertCompileMemberAlignment(PGMMAPSETENTRY, HCPhys, sizeof(RTHCPHYS));
1848	/** Pointer to a mapping cache usage set entry. */
1849	typedef PGMMAPSETENTRY *PPGMMAPSETENTRY;
1850
1851	/**
1852	* Mapping cache usage set.
1853	*
1854	* This is used in ring-0 and the raw-mode context to track dynamic mappings
1855	* done during exits / traps. The set is
1856	*/
1857	typedef struct PGMMAPSET
1858	{
1859	/** The number of occupied entries.
1860	* This is PGMMAPSET_CLOSED if the set is closed and we're not supposed to do
1861	* dynamic mappings. */
1862	uint32_t cEntries;
1863	/** The start of the current subset.
1864	* This is UINT32_MAX if no subset is currently open. */
1865	uint32_t iSubset;
1866	/** The index of the current CPU, only valid if the set is open. */
1867	int32_t iCpu;
1868	uint32_t alignment;
1869	/** The entries. */
1870	PGMMAPSETENTRY aEntries[64];
1871	/** HCPhys -> iEntry fast lookup table.
1872	* Use PGMMAPSET_HASH for hashing.
1873	* The entries may or may not be valid, check against cEntries. */
1874	uint8_t aiHashTable[128];
1875	} PGMMAPSET;
1876	AssertCompileSizeAlignment(PGMMAPSET, 8);
1877	/** Pointer to the mapping cache set. */
1878	typedef PGMMAPSET *PPGMMAPSET;
1879
1880	/** PGMMAPSET::cEntries value for a closed set. */
1881	#define PGMMAPSET_CLOSED UINT32_C(0xdeadc0fe)
1882
1883	/** Hash function for aiHashTable. */
1884	#define PGMMAPSET_HASH(HCPhys) (((HCPhys) >> PAGE_SHIFT) & 127)
1885
1886
1887	/** @name Context neutral page mapper TLB.
1888	*
1889	* Hoping to avoid some code and bug duplication parts of the GCxxx->CCPtr
1890	* code is writting in a kind of context neutral way. Time will show whether
1891	* this actually makes sense or not...
1892	*
1893	* @todo this needs to be reconsidered and dropped/redone since the ring-0
1894	* context ends up using a global mapping cache on some platforms
1895	* (darwin).
1896	*
1897	* @{ */
1898	/** @typedef PPGMPAGEMAPTLB
1899	* The page mapper TLB pointer type for the current context. */
1900	/** @typedef PPGMPAGEMAPTLB
1901	* The page mapper TLB entry pointer type for the current context. */
1902	/** @typedef PPGMPAGEMAPTLB
1903	* The page mapper TLB entry pointer pointer type for the current context. */
1904	/** @def PGM_PAGEMAPTLB_ENTRIES
1905	* The number of TLB entries in the page mapper TLB for the current context. */
1906	/** @def PGM_PAGEMAPTLB_IDX
1907	* Calculate the TLB index for a guest physical address.
1908	* @returns The TLB index.
1909	* @param GCPhys The guest physical address. */
1910	/** @typedef PPGMPAGEMAP
1911	* Pointer to a page mapper unit for current context. */
1912	/** @typedef PPPGMPAGEMAP
1913	* Pointer to a page mapper unit pointer for current context. */
1914	#ifdef IN_RC
1915	// typedef PPGMPAGEGCMAPTLB PPGMPAGEMAPTLB;
1916	// typedef PPGMPAGEGCMAPTLBE PPGMPAGEMAPTLBE;
1917	// typedef PPGMPAGEGCMAPTLBE *PPPGMPAGEMAPTLBE;
1918	# define PGM_PAGEMAPTLB_ENTRIES PGM_PAGEGCMAPTLB_ENTRIES
1919	# define PGM_PAGEMAPTLB_IDX(GCPhys) PGM_PAGEGCMAPTLB_IDX(GCPhys)
1920	typedef void * PPGMPAGEMAP;
1921	typedef void ** PPPGMPAGEMAP;
1922	//#elif IN_RING0
1923	// typedef PPGMPAGER0MAPTLB PPGMPAGEMAPTLB;
1924	// typedef PPGMPAGER0MAPTLBE PPGMPAGEMAPTLBE;
1925	// typedef PPGMPAGER0MAPTLBE *PPPGMPAGEMAPTLBE;
1926	//# define PGM_PAGEMAPTLB_ENTRIES PGM_PAGER0MAPTLB_ENTRIES
1927	//# define PGM_PAGEMAPTLB_IDX(GCPhys) PGM_PAGER0MAPTLB_IDX(GCPhys)
1928	// typedef PPGMCHUNKR0MAP PPGMPAGEMAP;
1929	// typedef PPPGMCHUNKR0MAP PPPGMPAGEMAP;
1930	#else
1931	typedef PPGMPAGER3MAPTLB PPGMPAGEMAPTLB;
1932	typedef PPGMPAGER3MAPTLBE PPGMPAGEMAPTLBE;
1933	typedef PPGMPAGER3MAPTLBE *PPPGMPAGEMAPTLBE;
1934	# define PGM_PAGEMAPTLB_ENTRIES PGM_PAGER3MAPTLB_ENTRIES
1935	# define PGM_PAGEMAPTLB_IDX(GCPhys) PGM_PAGER3MAPTLB_IDX(GCPhys)
1936	typedef PPGMCHUNKR3MAP PPGMPAGEMAP;
1937	typedef PPPGMCHUNKR3MAP PPPGMPAGEMAP;
1938	#endif
1939	/** @} */
1940
1941
1942	/** @name PGM Pool Indexes.
1943	* Aka. the unique shadow page identifier.
1944	* @{ */
1945	/** NIL page pool IDX. */
1946	#define NIL_PGMPOOL_IDX 0
1947	/** The first normal index. */
1948	#define PGMPOOL_IDX_FIRST_SPECIAL 1
1949	/** Page directory (32-bit root). */
1950	#define PGMPOOL_IDX_PD 1
1951	/** Page Directory Pointer Table (PAE root). */
1952	#define PGMPOOL_IDX_PDPT 2
1953	/** AMD64 CR3 level index.*/
1954	#define PGMPOOL_IDX_AMD64_CR3 3
1955	/** Nested paging root.*/
1956	#define PGMPOOL_IDX_NESTED_ROOT 4
1957	/** The first normal index. */
1958	#define PGMPOOL_IDX_FIRST 5
1959	/** The last valid index. (inclusive, 14 bits) */
1960	#define PGMPOOL_IDX_LAST 0x3fff
1961	/** @} */
1962
1963	/** The NIL index for the parent chain. */
1964	#define NIL_PGMPOOL_USER_INDEX ((uint16_t)0xffff)
1965	#define NIL_PGMPOOL_PRESENT_INDEX ((uint16_t)0xffff)
1966
1967	/**
1968	* Node in the chain linking a shadowed page to it's parent (user).
1969	*/
1970	#pragma pack(1)
1971	typedef struct PGMPOOLUSER
1972	{
1973	/** The index to the next item in the chain. NIL_PGMPOOL_USER_INDEX is no next. */
1974	uint16_t iNext;
1975	/** The user page index. */
1976	uint16_t iUser;
1977	/** Index into the user table. */
1978	uint32_t iUserTable;
1979	} PGMPOOLUSER, *PPGMPOOLUSER;
1980	typedef const PGMPOOLUSER *PCPGMPOOLUSER;
1981	#pragma pack()
1982
1983
1984	/** The NIL index for the phys ext chain. */
1985	#define NIL_PGMPOOL_PHYSEXT_INDEX ((uint16_t)0xffff)
1986	/** The NIL pte index for a phys ext chain slot. */
1987	#define NIL_PGMPOOL_PHYSEXT_IDX_PTE ((uint16_t)0xffff)
1988
1989	/**
1990	* Node in the chain of physical cross reference extents.
1991	* @todo Calling this an 'extent' is not quite right, find a better name.
1992	* @todo find out the optimal size of the aidx array
1993	*/
1994	#pragma pack(1)
1995	typedef struct PGMPOOLPHYSEXT
1996	{
1997	/** The index to the next item in the chain. NIL_PGMPOOL_PHYSEXT_INDEX is no next. */
1998	uint16_t iNext;
1999	/** Alignment. */
2000	uint16_t u16Align;
2001	/** The user page index. */
2002	uint16_t aidx[3];
2003	/** The page table index or NIL_PGMPOOL_PHYSEXT_IDX_PTE if unknown. */
2004	uint16_t apte[3];
2005	} PGMPOOLPHYSEXT, *PPGMPOOLPHYSEXT;
2006	typedef const PGMPOOLPHYSEXT *PCPGMPOOLPHYSEXT;
2007	#pragma pack()
2008
2009
2010	/**
2011	* The kind of page that's being shadowed.
2012	*/
2013	typedef enum PGMPOOLKIND
2014	{
2015	/** The virtual invalid 0 entry. */
2016	PGMPOOLKIND_INVALID = 0,
2017	/** The entry is free (=unused). */
2018	PGMPOOLKIND_FREE,
2019
2020	/** Shw: 32-bit page table; Gst: no paging */
2021	PGMPOOLKIND_32BIT_PT_FOR_PHYS,
2022	/** Shw: 32-bit page table; Gst: 32-bit page table. */
2023	PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT,
2024	/** Shw: 32-bit page table; Gst: 4MB page. */
2025	PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB,
2026	/** Shw: PAE page table; Gst: no paging */
2027	PGMPOOLKIND_PAE_PT_FOR_PHYS,
2028	/** Shw: PAE page table; Gst: 32-bit page table. */
2029	PGMPOOLKIND_PAE_PT_FOR_32BIT_PT,
2030	/** Shw: PAE page table; Gst: Half of a 4MB page. */
2031	PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB,
2032	/** Shw: PAE page table; Gst: PAE page table. */
2033	PGMPOOLKIND_PAE_PT_FOR_PAE_PT,
2034	/** Shw: PAE page table; Gst: 2MB page. */
2035	PGMPOOLKIND_PAE_PT_FOR_PAE_2MB,
2036
2037	/** Shw: 32-bit page directory. Gst: 32-bit page directory. */
2038	PGMPOOLKIND_32BIT_PD,
2039	/** Shw: 32-bit page directory. Gst: no paging. */
2040	PGMPOOLKIND_32BIT_PD_PHYS,
2041	/** Shw: PAE page directory 0; Gst: 32-bit page directory. */
2042	PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD,
2043	/** Shw: PAE page directory 1; Gst: 32-bit page directory. */
2044	PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD,
2045	/** Shw: PAE page directory 2; Gst: 32-bit page directory. */
2046	PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD,
2047	/** Shw: PAE page directory 3; Gst: 32-bit page directory. */
2048	PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD,
2049	/** Shw: PAE page directory; Gst: PAE page directory. */
2050	PGMPOOLKIND_PAE_PD_FOR_PAE_PD,
2051	/** Shw: PAE page directory; Gst: no paging. Note: +NP. */
2052	PGMPOOLKIND_PAE_PD_PHYS,
2053
2054	/** Shw: PAE page directory pointer table (legacy, 4 entries); Gst 32 bits paging. */
2055	PGMPOOLKIND_PAE_PDPT_FOR_32BIT,
2056	/** Shw: PAE page directory pointer table (legacy, 4 entries); Gst PAE PDPT. */
2057	PGMPOOLKIND_PAE_PDPT,
2058	/** Shw: PAE page directory pointer table (legacy, 4 entries); Gst: no paging. */
2059	PGMPOOLKIND_PAE_PDPT_PHYS,
2060
2061	/** Shw: 64-bit page directory pointer table; Gst: 64-bit page directory pointer table. */
2062	PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT,
2063	/** Shw: 64-bit page directory pointer table; Gst: no paging */
2064	PGMPOOLKIND_64BIT_PDPT_FOR_PHYS,
2065	/** Shw: 64-bit page directory table; Gst: 64-bit page directory table. */
2066	PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD,
2067	/** Shw: 64-bit page directory table; Gst: no paging */
2068	PGMPOOLKIND_64BIT_PD_FOR_PHYS, /* 22 */
2069
2070	/** Shw: 64-bit PML4; Gst: 64-bit PML4. */
2071	PGMPOOLKIND_64BIT_PML4,
2072
2073	/** Shw: EPT page directory pointer table; Gst: no paging */
2074	PGMPOOLKIND_EPT_PDPT_FOR_PHYS,
2075	/** Shw: EPT page directory table; Gst: no paging */
2076	PGMPOOLKIND_EPT_PD_FOR_PHYS,
2077	/** Shw: EPT page table; Gst: no paging */
2078	PGMPOOLKIND_EPT_PT_FOR_PHYS,
2079
2080	/** Shw: Root Nested paging table. */
2081	PGMPOOLKIND_ROOT_NESTED,
2082
2083	/** The last valid entry. */
2084	PGMPOOLKIND_LAST = PGMPOOLKIND_ROOT_NESTED
2085	} PGMPOOLKIND;
2086
2087	/**
2088	* The access attributes of the page; only applies to big pages.
2089	*/
2090	typedef enum
2091	{
2092	PGMPOOLACCESS_DONTCARE = 0,
2093	PGMPOOLACCESS_USER_RW,
2094	PGMPOOLACCESS_USER_R,
2095	PGMPOOLACCESS_USER_RW_NX,
2096	PGMPOOLACCESS_USER_R_NX,
2097	PGMPOOLACCESS_SUPERVISOR_RW,
2098	PGMPOOLACCESS_SUPERVISOR_R,
2099	PGMPOOLACCESS_SUPERVISOR_RW_NX,
2100	PGMPOOLACCESS_SUPERVISOR_R_NX
2101	} PGMPOOLACCESS;
2102
2103	/**
2104	* The tracking data for a page in the pool.
2105	*/
2106	typedef struct PGMPOOLPAGE
2107	{
2108	/** AVL node code with the (R3) physical address of this page. */
2109	AVLOHCPHYSNODECORE Core;
2110	/** Pointer to the R3 mapping of the page. */
2111	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
2112	R3PTRTYPE(void *) pvPageR3;
2113	#else
2114	R3R0PTRTYPE(void *) pvPageR3;
2115	#endif
2116	#if HC_ARCH_BITS == 32 && GC_ARCH_BITS == 64
2117	uint32_t Alignment0;
2118	#endif
2119	/** The guest physical address. */
2120	RTGCPHYS GCPhys;
2121	/** The kind of page we're shadowing. (This is really a PGMPOOLKIND enum.) */
2122	uint8_t enmKind;
2123	/** The subkind of page we're shadowing. (This is really a PGMPOOLACCESS enum.) */
2124	uint8_t enmAccess;
2125	/** This supplements enmKind and enmAccess */
2126	bool fA20Enabled : 1;
2127
2128	/** Used to indicate that the page is zeroed. */
2129	bool fZeroed : 1;
2130	/** Used to indicate that a PT has non-global entries. */
2131	bool fSeenNonGlobal : 1;
2132	/** Used to indicate that we're monitoring writes to the guest page. */
2133	bool fMonitored : 1;
2134	/** Used to indicate that the page is in the cache (e.g. in the GCPhys hash).
2135	* (All pages are in the age list.) */
2136	bool fCached : 1;
2137	/** This is used by the R3 access handlers when invoked by an async thread.
2138	* It's a hack required because of REMR3NotifyHandlerPhysicalDeregister. */
2139	bool volatile fReusedFlushPending : 1;
2140	/** Used to mark the page as dirty (write monitoring is temporarily
2141	* off). */
2142	bool fDirty : 1;
2143	bool fPadding1 : 1;
2144	bool fPadding2;
2145
2146	/** The index of this page. */
2147	uint16_t idx;
2148	/** The next entry in the list this page currently resides in.
2149	* It's either in the free list or in the GCPhys hash. */
2150	uint16_t iNext;
2151	/** Head of the user chain. NIL_PGMPOOL_USER_INDEX if not currently in use. */
2152	uint16_t iUserHead;
2153	/** The number of present entries. */
2154	uint16_t cPresent;
2155	/** The first entry in the table which is present. */
2156	uint16_t iFirstPresent;
2157	/** The number of modifications to the monitored page. */
2158	uint16_t cModifications;
2159	/** The next modified page. NIL_PGMPOOL_IDX if tail. */
2160	uint16_t iModifiedNext;
2161	/** The previous modified page. NIL_PGMPOOL_IDX if head. */
2162	uint16_t iModifiedPrev;
2163	/** The next page sharing access handler. NIL_PGMPOOL_IDX if tail. */
2164	uint16_t iMonitoredNext;
2165	/** The previous page sharing access handler. NIL_PGMPOOL_IDX if head. */
2166	uint16_t iMonitoredPrev;
2167	/** The next page in the age list. */
2168	uint16_t iAgeNext;
2169	/** The previous page in the age list. */
2170	uint16_t iAgePrev;
2171	/** Index into PGMPOOL::aDirtyPages if fDirty is set. */
2172	uint8_t idxDirtyEntry;
2173
2174	/** @name Access handler statistics to determine whether the guest is
2175	* (re)initializing a page table.
2176	* @{ */
2177	RTGCPTR GCPtrLastAccessHandlerRip;
2178	RTGCPTR GCPtrLastAccessHandlerFault;
2179	uint64_t cLastAccessHandler;
2180	/** @} */
2181	/** Used to indicate that this page can't be flushed. Important for cr3 root pages or shadow pae pd pages. */
2182	uint32_t volatile cLocked;
2183	#if GC_ARCH_BITS == 64
2184	uint32_t u32Alignment3;
2185	#endif
2186	# ifdef VBOX_STRICT
2187	RTGCPTR GCPtrDirtyFault;
2188	# endif
2189	} PGMPOOLPAGE;
2190	/** Pointer to a pool page. */
2191	typedef PGMPOOLPAGE *PPGMPOOLPAGE;
2192	/** Pointer to a const pool page. */
2193	typedef PGMPOOLPAGE const *PCPGMPOOLPAGE;
2194	/** Pointer to a pool page pointer. */
2195	typedef PGMPOOLPAGE **PPPGMPOOLPAGE;
2196
2197
2198	/** The hash table size. */
2199	# define PGMPOOL_HASH_SIZE 0x40
2200	/** The hash function. */
2201	# define PGMPOOL_HASH(GCPhys) ( ((GCPhys) >> PAGE_SHIFT) & (PGMPOOL_HASH_SIZE - 1) )
2202
2203
2204	/**
2205	* The shadow page pool instance data.
2206	*
2207	* It's all one big allocation made at init time, except for the
2208	* pages that is. The user nodes follows immediately after the
2209	* page structures.
2210	*/
2211	typedef struct PGMPOOL
2212	{
2213	/** The VM handle - R3 Ptr. */
2214	PVMR3 pVMR3;
2215	/** The VM handle - R0 Ptr. */
2216	PVMR0 pVMR0;
2217	/** The VM handle - RC Ptr. */
2218	PVMRC pVMRC;
2219	/** The max pool size. This includes the special IDs. */
2220	uint16_t cMaxPages;
2221	/** The current pool size. */
2222	uint16_t cCurPages;
2223	/** The head of the free page list. */
2224	uint16_t iFreeHead;
2225	/* Padding. */
2226	uint16_t u16Padding;
2227	/** Head of the chain of free user nodes. */
2228	uint16_t iUserFreeHead;
2229	/** The number of user nodes we've allocated. */
2230	uint16_t cMaxUsers;
2231	/** The number of present page table entries in the entire pool. */
2232	uint32_t cPresent;
2233	/** Pointer to the array of user nodes - RC pointer. */
2234	RCPTRTYPE(PPGMPOOLUSER) paUsersRC;
2235	/** Pointer to the array of user nodes - R3 pointer. */
2236	R3PTRTYPE(PPGMPOOLUSER) paUsersR3;
2237	/** Pointer to the array of user nodes - R0 pointer. */
2238	R0PTRTYPE(PPGMPOOLUSER) paUsersR0;
2239	/** Head of the chain of free phys ext nodes. */
2240	uint16_t iPhysExtFreeHead;
2241	/** The number of user nodes we've allocated. */
2242	uint16_t cMaxPhysExts;
2243	/** Pointer to the array of physical xref extent - RC pointer. */
2244	RCPTRTYPE(PPGMPOOLPHYSEXT) paPhysExtsRC;
2245	/** Pointer to the array of physical xref extent nodes - R3 pointer. */
2246	R3PTRTYPE(PPGMPOOLPHYSEXT) paPhysExtsR3;
2247	/** Pointer to the array of physical xref extent nodes - R0 pointer. */
2248	R0PTRTYPE(PPGMPOOLPHYSEXT) paPhysExtsR0;
2249	/** Hash table for GCPhys addresses. */
2250	uint16_t aiHash[PGMPOOL_HASH_SIZE];
2251	/** The head of the age list. */
2252	uint16_t iAgeHead;
2253	/** The tail of the age list. */
2254	uint16_t iAgeTail;
2255	/** Set if the cache is enabled. */
2256	bool fCacheEnabled;
2257	/** Alignment padding. */
2258	bool afPadding1[3];
2259	/** Head of the list of modified pages. */
2260	uint16_t iModifiedHead;
2261	/** The current number of modified pages. */
2262	uint16_t cModifiedPages;
2263	/** Access handler, RC. */
2264	RCPTRTYPE(PFNPGMRCPHYSHANDLER) pfnAccessHandlerRC;
2265	/** Access handler, R0. */
2266	R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnAccessHandlerR0;
2267	/** Access handler, R3. */
2268	R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnAccessHandlerR3;
2269	/** The access handler description (R3 ptr). */
2270	R3PTRTYPE(const char *) pszAccessHandler;
2271	# if HC_ARCH_BITS == 32
2272	/** Alignment padding. */
2273	uint32_t u32Padding2;
2274	# endif
2275	/** Next available slot (in aDirtyPages). */
2276	uint32_t idxFreeDirtyPage;
2277	/** Number of active dirty pages. */
2278	uint32_t cDirtyPages;
2279	/** Array of current dirty pgm pool page indices. */
2280	struct
2281	{
2282	uint16_t uIdx;
2283	uint16_t Alignment[3];
2284	uint64_t aPage[512];
2285	} aDirtyPages[16];
2286	/** The number of pages currently in use. */
2287	uint16_t cUsedPages;
2288	#ifdef VBOX_WITH_STATISTICS
2289	/** The high water mark for cUsedPages. */
2290	uint16_t cUsedPagesHigh;
2291	uint32_t Alignment1; /*< Align the next member on a 64-bit boundary. /
2292	/** Profiling pgmPoolAlloc(). */
2293	STAMPROFILEADV StatAlloc;
2294	/** Profiling pgmR3PoolClearDoIt(). */
2295	STAMPROFILE StatClearAll;
2296	/** Profiling pgmR3PoolReset(). */
2297	STAMPROFILE StatR3Reset;
2298	/** Profiling pgmPoolFlushPage(). */
2299	STAMPROFILE StatFlushPage;
2300	/** Profiling pgmPoolFree(). */
2301	STAMPROFILE StatFree;
2302	/** Counting explicit flushes by PGMPoolFlushPage(). */
2303	STAMCOUNTER StatForceFlushPage;
2304	/** Counting explicit flushes of dirty pages by PGMPoolFlushPage(). */
2305	STAMCOUNTER StatForceFlushDirtyPage;
2306	/** Counting flushes for reused pages. */
2307	STAMCOUNTER StatForceFlushReused;
2308	/** Profiling time spent zeroing pages. */
2309	STAMPROFILE StatZeroPage;
2310	/** Profiling of pgmPoolTrackDeref. */
2311	STAMPROFILE StatTrackDeref;
2312	/** Profiling pgmTrackFlushGCPhysPT. */
2313	STAMPROFILE StatTrackFlushGCPhysPT;
2314	/** Profiling pgmTrackFlushGCPhysPTs. */
2315	STAMPROFILE StatTrackFlushGCPhysPTs;
2316	/** Profiling pgmTrackFlushGCPhysPTsSlow. */
2317	STAMPROFILE StatTrackFlushGCPhysPTsSlow;
2318	/** Number of times we've been out of user records. */
2319	STAMCOUNTER StatTrackFreeUpOneUser;
2320	/** Nr of flushed entries. */
2321	STAMCOUNTER StatTrackFlushEntry;
2322	/** Nr of updated entries. */
2323	STAMCOUNTER StatTrackFlushEntryKeep;
2324	/** Profiling deref activity related tracking GC physical pages. */
2325	STAMPROFILE StatTrackDerefGCPhys;
2326	/** Number of linear searches for a HCPhys in the ram ranges. */
2327	STAMCOUNTER StatTrackLinearRamSearches;
2328	/** The number of failing pgmPoolTrackPhysExtAlloc calls. */
2329	STAMCOUNTER StamTrackPhysExtAllocFailures;
2330	/** Profiling the RC/R0 access handler. */
2331	STAMPROFILE StatMonitorRZ;
2332	/** Times we've failed interpreting the instruction. */
2333	STAMCOUNTER StatMonitorRZEmulateInstr;
2334	/** Profiling the pgmPoolFlushPage calls made from the RC/R0 access handler. */
2335	STAMPROFILE StatMonitorRZFlushPage;
2336	/* Times we've detected a page table reinit. */
2337	STAMCOUNTER StatMonitorRZFlushReinit;
2338	/** Counting flushes for pages that are modified too often. */
2339	STAMCOUNTER StatMonitorRZFlushModOverflow;
2340	/** Times we've detected fork(). */
2341	STAMCOUNTER StatMonitorRZFork;
2342	/** Profiling the RC/R0 access we've handled (except REP STOSD). */
2343	STAMPROFILE StatMonitorRZHandled;
2344	/** Times we've failed interpreting a patch code instruction. */
2345	STAMCOUNTER StatMonitorRZIntrFailPatch1;
2346	/** Times we've failed interpreting a patch code instruction during flushing. */
2347	STAMCOUNTER StatMonitorRZIntrFailPatch2;
2348	/** The number of times we've seen rep prefixes we can't handle. */
2349	STAMCOUNTER StatMonitorRZRepPrefix;
2350	/** Profiling the REP STOSD cases we've handled. */
2351	STAMPROFILE StatMonitorRZRepStosd;
2352	/** Nr of handled PT faults. */
2353	STAMCOUNTER StatMonitorRZFaultPT;
2354	/** Nr of handled PD faults. */
2355	STAMCOUNTER StatMonitorRZFaultPD;
2356	/** Nr of handled PDPT faults. */
2357	STAMCOUNTER StatMonitorRZFaultPDPT;
2358	/** Nr of handled PML4 faults. */
2359	STAMCOUNTER StatMonitorRZFaultPML4;
2360
2361	/** Profiling the R3 access handler. */
2362	STAMPROFILE StatMonitorR3;
2363	/** Times we've failed interpreting the instruction. */
2364	STAMCOUNTER StatMonitorR3EmulateInstr;
2365	/** Profiling the pgmPoolFlushPage calls made from the R3 access handler. */
2366	STAMPROFILE StatMonitorR3FlushPage;
2367	/* Times we've detected a page table reinit. */
2368	STAMCOUNTER StatMonitorR3FlushReinit;
2369	/** Counting flushes for pages that are modified too often. */
2370	STAMCOUNTER StatMonitorR3FlushModOverflow;
2371	/** Times we've detected fork(). */
2372	STAMCOUNTER StatMonitorR3Fork;
2373	/** Profiling the R3 access we've handled (except REP STOSD). */
2374	STAMPROFILE StatMonitorR3Handled;
2375	/** The number of times we've seen rep prefixes we can't handle. */
2376	STAMCOUNTER StatMonitorR3RepPrefix;
2377	/** Profiling the REP STOSD cases we've handled. */
2378	STAMPROFILE StatMonitorR3RepStosd;
2379	/** Nr of handled PT faults. */
2380	STAMCOUNTER StatMonitorR3FaultPT;
2381	/** Nr of handled PD faults. */
2382	STAMCOUNTER StatMonitorR3FaultPD;
2383	/** Nr of handled PDPT faults. */
2384	STAMCOUNTER StatMonitorR3FaultPDPT;
2385	/** Nr of handled PML4 faults. */
2386	STAMCOUNTER StatMonitorR3FaultPML4;
2387	/** The number of times we're called in an async thread an need to flush. */
2388	STAMCOUNTER StatMonitorR3Async;
2389	/** Times we've called pgmPoolResetDirtyPages (and there were dirty page). */
2390	STAMCOUNTER StatResetDirtyPages;
2391	/** Times we've called pgmPoolAddDirtyPage. */
2392	STAMCOUNTER StatDirtyPage;
2393	/** Times we've had to flush duplicates for dirty page management. */
2394	STAMCOUNTER StatDirtyPageDupFlush;
2395	/** Times we've had to flush because of overflow. */
2396	STAMCOUNTER StatDirtyPageOverFlowFlush;
2397
2398	/** The high water mark for cModifiedPages. */
2399	uint16_t cModifiedPagesHigh;
2400	uint16_t Alignment2[3]; /*< Align the next member on a 64-bit boundary. /
2401
2402	/** The number of cache hits. */
2403	STAMCOUNTER StatCacheHits;
2404	/** The number of cache misses. */
2405	STAMCOUNTER StatCacheMisses;
2406	/** The number of times we've got a conflict of 'kind' in the cache. */
2407	STAMCOUNTER StatCacheKindMismatches;
2408	/** Number of times we've been out of pages. */
2409	STAMCOUNTER StatCacheFreeUpOne;
2410	/** The number of cacheable allocations. */
2411	STAMCOUNTER StatCacheCacheable;
2412	/** The number of uncacheable allocations. */
2413	STAMCOUNTER StatCacheUncacheable;
2414	#else
2415	uint32_t Alignment3; /*< Align the next member on a 64-bit boundary. /
2416	#endif
2417	/** The AVL tree for looking up a page by its HC physical address. */
2418	AVLOHCPHYSTREE HCPhysTree;
2419	uint32_t Alignment4; /*< Align the next member on a 64-bit boundary. /
2420	/** Array of pages. (cMaxPages in length)
2421	* The Id is the index into thist array.
2422	*/
2423	PGMPOOLPAGE aPages[PGMPOOL_IDX_FIRST];
2424	} PGMPOOL, PPGMPOOL, *PPPGMPOOL;
2425	AssertCompileMemberAlignment(PGMPOOL, iModifiedHead, 8);
2426	AssertCompileMemberAlignment(PGMPOOL, aDirtyPages, 8);
2427	AssertCompileMemberAlignment(PGMPOOL, cUsedPages, 8);
2428	#ifdef VBOX_WITH_STATISTICS
2429	AssertCompileMemberAlignment(PGMPOOL, StatAlloc, 8);
2430	#endif
2431	AssertCompileMemberAlignment(PGMPOOL, aPages, 8);
2432
2433
2434	/** @def PGMPOOL_PAGE_2_PTR
2435	* Maps a pool page pool into the current context.
2436	*
2437	* @returns VBox status code.
2438	* @param a_pVM The VM handle.
2439	* @param a_pPage The pool page.
2440	*
2441	* @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
2442	* small page window employeed by that function. Be careful.
2443	* @remark There is no need to assert on the result.
2444	*/
2445	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
2446	# define PGMPOOL_PAGE_2_PTR(a_pVM, a_pPage) pgmPoolMapPageInlined((a_pVM), (a_pPage) RTLOG_COMMA_SRC_POS)
2447	#elif defined(VBOX_STRICT)
2448	# define PGMPOOL_PAGE_2_PTR(a_pVM, a_pPage) pgmPoolMapPageStrict(a_pPage)
2449	DECLINLINE(void *) pgmPoolMapPageStrict(PPGMPOOLPAGE a_pPage)
2450	{
2451	Assert(a_pPage && a_pPage->pvPageR3);
2452	return a_pPage->pvPageR3;
2453	}
2454	#else
2455	# define PGMPOOL_PAGE_2_PTR(pVM, a_pPage) ((a_pPage)->pvPageR3)
2456	#endif
2457
2458
2459	/** @def PGMPOOL_PAGE_2_PTR_V2
2460	* Maps a pool page pool into the current context, taking both VM and VMCPU.
2461	*
2462	* @returns VBox status code.
2463	* @param a_pVM The VM handle.
2464	* @param a_pVCpu The current CPU.
2465	* @param a_pPage The pool page.
2466	*
2467	* @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
2468	* small page window employeed by that function. Be careful.
2469	* @remark There is no need to assert on the result.
2470	*/
2471	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
2472	# define PGMPOOL_PAGE_2_PTR_V2(a_pVM, a_pVCpu, a_pPage) pgmPoolMapPageV2Inlined((a_pVM), (a_pVCpu), (a_pPage) RTLOG_COMMA_SRC_POS)
2473	#else
2474	# define PGMPOOL_PAGE_2_PTR_V2(a_pVM, a_pVCpu, a_pPage) PGMPOOL_PAGE_2_PTR((a_pVM), (a_pPage))
2475	#endif
2476
2477
2478	/** @name Per guest page tracking data.
2479	* This is currently as a 16-bit word in the PGMPAGE structure, the idea though
2480	* is to use more bits for it and split it up later on. But for now we'll play
2481	* safe and change as little as possible.
2482	*
2483	* The 16-bit word has two parts:
2484	*
2485	* The first 14-bit forms the @a idx field. It is either the index of a page in
2486	* the shadow page pool, or and index into the extent list.
2487	*
2488	* The 2 topmost bits makes up the @a cRefs field, which counts the number of
2489	* shadow page pool references to the page. If cRefs equals
2490	* PGMPOOL_CREFS_PHYSEXT, then the @a idx field is an indext into the extent
2491	* (misnomer) table and not the shadow page pool.
2492	*
2493	* See PGM_PAGE_GET_TRACKING and PGM_PAGE_SET_TRACKING for how to get and set
2494	* the 16-bit word.
2495	*
2496	* @{ */
2497	/** The shift count for getting to the cRefs part. */
2498	#define PGMPOOL_TD_CREFS_SHIFT 14
2499	/** The mask applied after shifting the tracking data down by
2500	* PGMPOOL_TD_CREFS_SHIFT. */
2501	#define PGMPOOL_TD_CREFS_MASK 0x3
2502	/** The cRefs value used to indicate that the idx is the head of a
2503	* physical cross reference list. */
2504	#define PGMPOOL_TD_CREFS_PHYSEXT PGMPOOL_TD_CREFS_MASK
2505	/** The shift used to get idx. */
2506	#define PGMPOOL_TD_IDX_SHIFT 0
2507	/** The mask applied to the idx after shifting down by PGMPOOL_TD_IDX_SHIFT. */
2508	#define PGMPOOL_TD_IDX_MASK 0x3fff
2509	/** The idx value when we're out of of PGMPOOLPHYSEXT entries or/and there are
2510	* simply too many mappings of this page. */
2511	#define PGMPOOL_TD_IDX_OVERFLOWED PGMPOOL_TD_IDX_MASK
2512
2513	/** @def PGMPOOL_TD_MAKE
2514	* Makes a 16-bit tracking data word.
2515	*
2516	* @returns tracking data.
2517	* @param cRefs The @a cRefs field. Must be within bounds!
2518	* @param idx The @a idx field. Must also be within bounds! */
2519	#define PGMPOOL_TD_MAKE(cRefs, idx) ( ((cRefs) << PGMPOOL_TD_CREFS_SHIFT) \| (idx) )
2520
2521	/** @def PGMPOOL_TD_GET_CREFS
2522	* Get the @a cRefs field from a tracking data word.
2523	*
2524	* @returns The @a cRefs field
2525	* @param u16 The tracking data word.
2526	* @remarks This will only return 1 or PGMPOOL_TD_CREFS_PHYSEXT for a
2527	* non-zero @a u16. */
2528	#define PGMPOOL_TD_GET_CREFS(u16) ( ((u16) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK )
2529
2530	/** @def PGMPOOL_TD_GET_IDX
2531	* Get the @a idx field from a tracking data word.
2532	*
2533	* @returns The @a idx field
2534	* @param u16 The tracking data word. */
2535	#define PGMPOOL_TD_GET_IDX(u16) ( ((u16) >> PGMPOOL_TD_IDX_SHIFT) & PGMPOOL_TD_IDX_MASK )
2536	/** @} */
2537
2538
2539
2540	/** @name A20 gate macros
2541	* @{ */
2542	#define PGM_WITH_A20
2543	#ifdef PGM_WITH_A20
2544	# define PGM_A20_IS_ENABLED(a_pVCpu) ((a_pVCpu)->pgm.s.fA20Enabled)
2545	# define PGM_A20_APPLY(a_pVCpu, a_GCPhys) ((a_GCPhys) & (a_pVCpu)->pgm.s.GCPhysA20Mask)
2546	# define PGM_A20_APPLY_TO_VAR(a_pVCpu, a_GCPhysVar) \
2547	do { a_GCPhysVar &= (a_pVCpu)->pgm.s.GCPhysA20Mask; } while (0)
2548	# define PGM_A20_ASSERT_MASKED(pVCpu, a_GCPhys) Assert(PGM_A20_APPLY(pVCpu, a_GCPhys) == (a_GCPhys))
2549	#else
2550	# define PGM_A20_IS_ENABLED(a_pVCpu) (true)
2551	# define PGM_A20_APPLY(a_pVCpu, a_GCPhys) (a_GCPhys)
2552	# define PGM_A20_APPLY_TO_VAR(a_pVCpu, a_GCPhysVar) do { } while (0)
2553	# define PGM_A20_ASSERT_MASKED(pVCpu, a_GCPhys) do { } while (0)
2554	#endif
2555	/** @} */
2556
2557
2558	/**
2559	* Trees are using self relative offsets as pointers.
2560	* So, all its data, including the root pointer, must be in the heap for HC and GC
2561	* to have the same layout.
2562	*/
2563	typedef struct PGMTREES
2564	{
2565	/** Physical access handlers (AVL range+offsetptr tree). */
2566	AVLROGCPHYSTREE PhysHandlers;
2567	/** Virtual access handlers (AVL range + GC ptr tree). */
2568	AVLROGCPTRTREE VirtHandlers;
2569	/** Virtual access handlers (Phys range AVL range + offsetptr tree). */
2570	AVLROGCPHYSTREE PhysToVirtHandlers;
2571	/** Virtual access handlers for the hypervisor (AVL range + GC ptr tree). */
2572	AVLROGCPTRTREE HyperVirtHandlers;
2573	} PGMTREES;
2574	/** Pointer to PGM trees. */
2575	typedef PGMTREES *PPGMTREES;
2576
2577
2578	/**
2579	* Page fault guest state for the AMD64 paging mode.
2580	*/
2581	typedef struct PGMPTWALKCORE
2582	{
2583	/** The guest virtual address that is being resolved by the walk
2584	* (input). */
2585	RTGCPTR GCPtr;
2586
2587	/** The guest physical address that is the result of the walk.
2588	* @remarks only valid if fSucceeded is set. */
2589	RTGCPHYS GCPhys;
2590
2591	/** Set if the walk succeeded, i.d. GCPhys is valid. */
2592	bool fSucceeded;
2593	/** The level problem arrised at.
2594	* PTE is level 1, PDE is level 2, PDPE is level 3, PML4 is level 4, CR3 is
2595	* level 8. This is 0 on success. */
2596	uint8_t uLevel;
2597	/** Set if the page isn't present. */
2598	bool fNotPresent;
2599	/** Encountered a bad physical address. */
2600	bool fBadPhysAddr;
2601	/** Set if there was reserved bit violations. */
2602	bool fRsvdError;
2603	/** Set if it involves a big page (2/4 MB). */
2604	bool fBigPage;
2605	/** Set if it involves a gigantic page (1 GB). */
2606	bool fGigantPage;
2607	/** The effect X86_PTE_US flag for the address. */
2608	bool fEffectiveUS;
2609	/** The effect X86_PTE_RW flag for the address. */
2610	bool fEffectiveRW;
2611	/** The effect X86_PTE_NX flag for the address. */
2612	bool fEffectiveNX;
2613	} PGMPTWALKCORE;
2614
2615
2616	/**
2617	* Guest page table walk for the AMD64 mode.
2618	*/
2619	typedef struct PGMPTWALKGSTAMD64
2620	{
2621	/** The common core. */
2622	PGMPTWALKCORE Core;
2623
2624	PX86PML4 pPml4;
2625	PX86PML4E pPml4e;
2626	X86PML4E Pml4e;
2627
2628	PX86PDPT pPdpt;
2629	PX86PDPE pPdpe;
2630	X86PDPE Pdpe;
2631
2632	PX86PDPAE pPd;
2633	PX86PDEPAE pPde;
2634	X86PDEPAE Pde;
2635
2636	PX86PTPAE pPt;
2637	PX86PTEPAE pPte;
2638	X86PTEPAE Pte;
2639	} PGMPTWALKGSTAMD64;
2640	/** Pointer to a AMD64 guest page table walk. */
2641	typedef PGMPTWALKGSTAMD64 *PPGMPTWALKGSTAMD64;
2642	/** Pointer to a const AMD64 guest page table walk. */
2643	typedef PGMPTWALKGSTAMD64 const *PCPGMPTWALKGSTAMD64;
2644
2645	/**
2646	* Guest page table walk for the PAE mode.
2647	*/
2648	typedef struct PGMPTWALKGSTPAE
2649	{
2650	/** The common core. */
2651	PGMPTWALKCORE Core;
2652
2653	PX86PDPT pPdpt;
2654	PX86PDPE pPdpe;
2655	X86PDPE Pdpe;
2656
2657	PX86PDPAE pPd;
2658	PX86PDEPAE pPde;
2659	X86PDEPAE Pde;
2660
2661	PX86PTPAE pPt;
2662	PX86PTEPAE pPte;
2663	X86PTEPAE Pte;
2664	} PGMPTWALKGSTPAE;
2665	/** Pointer to a PAE guest page table walk. */
2666	typedef PGMPTWALKGSTPAE *PPGMPTWALKGSTPAE;
2667	/** Pointer to a const AMD64 guest page table walk. */
2668	typedef PGMPTWALKGSTPAE const *PCPGMPTWALKGSTPAE;
2669
2670	/**
2671	* Guest page table walk for the 32-bit mode.
2672	*/
2673	typedef struct PGMPTWALKGST32BIT
2674	{
2675	/** The common core. */
2676	PGMPTWALKCORE Core;
2677
2678	PX86PD pPd;
2679	PX86PDE pPde;
2680	X86PDE Pde;
2681
2682	PX86PT pPt;
2683	PX86PTE pPte;
2684	X86PTE Pte;
2685	} PGMPTWALKGST32BIT;
2686	/** Pointer to a 32-bit guest page table walk. */
2687	typedef PGMPTWALKGST32BIT *PPGMPTWALKGST32BIT;
2688	/** Pointer to a const 32-bit guest page table walk. */
2689	typedef PGMPTWALKGST32BIT const *PCPGMPTWALKGST32BIT;
2690
2691
2692	/** @name Paging mode macros
2693	* @{
2694	*/
2695	#ifdef IN_RC
2696	# define PGM_CTX(a,b) a##RC##b
2697	# define PGM_CTX_STR(a,b) a "GC" b
2698	# define PGM_CTX_DECL(type) VMMRCDECL(type)
2699	#else
2700	# ifdef IN_RING3
2701	# define PGM_CTX(a,b) a##R3##b
2702	# define PGM_CTX_STR(a,b) a "R3" b
2703	# define PGM_CTX_DECL(type) DECLCALLBACK(type)
2704	# else
2705	# define PGM_CTX(a,b) a##R0##b
2706	# define PGM_CTX_STR(a,b) a "R0" b
2707	# define PGM_CTX_DECL(type) VMMDECL(type)
2708	# endif
2709	#endif
2710
2711	#define PGM_GST_NAME_REAL(name) PGM_CTX(pgm,GstReal##name)
2712	#define PGM_GST_NAME_RC_REAL_STR(name) "pgmRCGstReal" #name
2713	#define PGM_GST_NAME_R0_REAL_STR(name) "pgmR0GstReal" #name
2714	#define PGM_GST_NAME_PROT(name) PGM_CTX(pgm,GstProt##name)
2715	#define PGM_GST_NAME_RC_PROT_STR(name) "pgmRCGstProt" #name
2716	#define PGM_GST_NAME_R0_PROT_STR(name) "pgmR0GstProt" #name
2717	#define PGM_GST_NAME_32BIT(name) PGM_CTX(pgm,Gst32Bit##name)
2718	#define PGM_GST_NAME_RC_32BIT_STR(name) "pgmRCGst32Bit" #name
2719	#define PGM_GST_NAME_R0_32BIT_STR(name) "pgmR0Gst32Bit" #name
2720	#define PGM_GST_NAME_PAE(name) PGM_CTX(pgm,GstPAE##name)
2721	#define PGM_GST_NAME_RC_PAE_STR(name) "pgmRCGstPAE" #name
2722	#define PGM_GST_NAME_R0_PAE_STR(name) "pgmR0GstPAE" #name
2723	#define PGM_GST_NAME_AMD64(name) PGM_CTX(pgm,GstAMD64##name)
2724	#define PGM_GST_NAME_RC_AMD64_STR(name) "pgmRCGstAMD64" #name
2725	#define PGM_GST_NAME_R0_AMD64_STR(name) "pgmR0GstAMD64" #name
2726	#define PGM_GST_PFN(name, pVCpu) ((pVCpu)->pgm.s.PGM_CTX(pfn,Gst##name))
2727	#define PGM_GST_DECL(type, name) PGM_CTX_DECL(type) PGM_GST_NAME(name)
2728
2729	#define PGM_SHW_NAME_32BIT(name) PGM_CTX(pgm,Shw32Bit##name)
2730	#define PGM_SHW_NAME_RC_32BIT_STR(name) "pgmRCShw32Bit" #name
2731	#define PGM_SHW_NAME_R0_32BIT_STR(name) "pgmR0Shw32Bit" #name
2732	#define PGM_SHW_NAME_PAE(name) PGM_CTX(pgm,ShwPAE##name)
2733	#define PGM_SHW_NAME_RC_PAE_STR(name) "pgmRCShwPAE" #name
2734	#define PGM_SHW_NAME_R0_PAE_STR(name) "pgmR0ShwPAE" #name
2735	#define PGM_SHW_NAME_AMD64(name) PGM_CTX(pgm,ShwAMD64##name)
2736	#define PGM_SHW_NAME_RC_AMD64_STR(name) "pgmRCShwAMD64" #name
2737	#define PGM_SHW_NAME_R0_AMD64_STR(name) "pgmR0ShwAMD64" #name
2738	#define PGM_SHW_NAME_NESTED(name) PGM_CTX(pgm,ShwNested##name)
2739	#define PGM_SHW_NAME_RC_NESTED_STR(name) "pgmRCShwNested" #name
2740	#define PGM_SHW_NAME_R0_NESTED_STR(name) "pgmR0ShwNested" #name
2741	#define PGM_SHW_NAME_EPT(name) PGM_CTX(pgm,ShwEPT##name)
2742	#define PGM_SHW_NAME_RC_EPT_STR(name) "pgmRCShwEPT" #name
2743	#define PGM_SHW_NAME_R0_EPT_STR(name) "pgmR0ShwEPT" #name
2744	#define PGM_SHW_DECL(type, name) PGM_CTX_DECL(type) PGM_SHW_NAME(name)
2745	#define PGM_SHW_PFN(name, pVCpu) ((pVCpu)->pgm.s.PGM_CTX(pfn,Shw##name))
2746
2747	/* Shw_Gst */
2748	#define PGM_BTH_NAME_32BIT_REAL(name) PGM_CTX(pgm,Bth32BitReal##name)
2749	#define PGM_BTH_NAME_32BIT_PROT(name) PGM_CTX(pgm,Bth32BitProt##name)
2750	#define PGM_BTH_NAME_32BIT_32BIT(name) PGM_CTX(pgm,Bth32Bit32Bit##name)
2751	#define PGM_BTH_NAME_PAE_REAL(name) PGM_CTX(pgm,BthPAEReal##name)
2752	#define PGM_BTH_NAME_PAE_PROT(name) PGM_CTX(pgm,BthPAEProt##name)
2753	#define PGM_BTH_NAME_PAE_32BIT(name) PGM_CTX(pgm,BthPAE32Bit##name)
2754	#define PGM_BTH_NAME_PAE_PAE(name) PGM_CTX(pgm,BthPAEPAE##name)
2755	#define PGM_BTH_NAME_AMD64_PROT(name) PGM_CTX(pgm,BthAMD64Prot##name)
2756	#define PGM_BTH_NAME_AMD64_AMD64(name) PGM_CTX(pgm,BthAMD64AMD64##name)
2757	#define PGM_BTH_NAME_NESTED_REAL(name) PGM_CTX(pgm,BthNestedReal##name)
2758	#define PGM_BTH_NAME_NESTED_PROT(name) PGM_CTX(pgm,BthNestedProt##name)
2759	#define PGM_BTH_NAME_NESTED_32BIT(name) PGM_CTX(pgm,BthNested32Bit##name)
2760	#define PGM_BTH_NAME_NESTED_PAE(name) PGM_CTX(pgm,BthNestedPAE##name)
2761	#define PGM_BTH_NAME_NESTED_AMD64(name) PGM_CTX(pgm,BthNestedAMD64##name)
2762	#define PGM_BTH_NAME_EPT_REAL(name) PGM_CTX(pgm,BthEPTReal##name)
2763	#define PGM_BTH_NAME_EPT_PROT(name) PGM_CTX(pgm,BthEPTProt##name)
2764	#define PGM_BTH_NAME_EPT_32BIT(name) PGM_CTX(pgm,BthEPT32Bit##name)
2765	#define PGM_BTH_NAME_EPT_PAE(name) PGM_CTX(pgm,BthEPTPAE##name)
2766	#define PGM_BTH_NAME_EPT_AMD64(name) PGM_CTX(pgm,BthEPTAMD64##name)
2767
2768	#define PGM_BTH_NAME_RC_32BIT_REAL_STR(name) "pgmRCBth32BitReal" #name
2769	#define PGM_BTH_NAME_RC_32BIT_PROT_STR(name) "pgmRCBth32BitProt" #name
2770	#define PGM_BTH_NAME_RC_32BIT_32BIT_STR(name) "pgmRCBth32Bit32Bit" #name
2771	#define PGM_BTH_NAME_RC_PAE_REAL_STR(name) "pgmRCBthPAEReal" #name
2772	#define PGM_BTH_NAME_RC_PAE_PROT_STR(name) "pgmRCBthPAEProt" #name
2773	#define PGM_BTH_NAME_RC_PAE_32BIT_STR(name) "pgmRCBthPAE32Bit" #name
2774	#define PGM_BTH_NAME_RC_PAE_PAE_STR(name) "pgmRCBthPAEPAE" #name
2775	#define PGM_BTH_NAME_RC_AMD64_AMD64_STR(name) "pgmRCBthAMD64AMD64" #name
2776	#define PGM_BTH_NAME_RC_NESTED_REAL_STR(name) "pgmRCBthNestedReal" #name
2777	#define PGM_BTH_NAME_RC_NESTED_PROT_STR(name) "pgmRCBthNestedProt" #name
2778	#define PGM_BTH_NAME_RC_NESTED_32BIT_STR(name) "pgmRCBthNested32Bit" #name
2779	#define PGM_BTH_NAME_RC_NESTED_PAE_STR(name) "pgmRCBthNestedPAE" #name
2780	#define PGM_BTH_NAME_RC_NESTED_AMD64_STR(name) "pgmRCBthNestedAMD64" #name
2781	#define PGM_BTH_NAME_RC_EPT_REAL_STR(name) "pgmRCBthEPTReal" #name
2782	#define PGM_BTH_NAME_RC_EPT_PROT_STR(name) "pgmRCBthEPTProt" #name
2783	#define PGM_BTH_NAME_RC_EPT_32BIT_STR(name) "pgmRCBthEPT32Bit" #name
2784	#define PGM_BTH_NAME_RC_EPT_PAE_STR(name) "pgmRCBthEPTPAE" #name
2785	#define PGM_BTH_NAME_RC_EPT_AMD64_STR(name) "pgmRCBthEPTAMD64" #name
2786	#define PGM_BTH_NAME_R0_32BIT_REAL_STR(name) "pgmR0Bth32BitReal" #name
2787	#define PGM_BTH_NAME_R0_32BIT_PROT_STR(name) "pgmR0Bth32BitProt" #name
2788	#define PGM_BTH_NAME_R0_32BIT_32BIT_STR(name) "pgmR0Bth32Bit32Bit" #name
2789	#define PGM_BTH_NAME_R0_PAE_REAL_STR(name) "pgmR0BthPAEReal" #name
2790	#define PGM_BTH_NAME_R0_PAE_PROT_STR(name) "pgmR0BthPAEProt" #name
2791	#define PGM_BTH_NAME_R0_PAE_32BIT_STR(name) "pgmR0BthPAE32Bit" #name
2792	#define PGM_BTH_NAME_R0_PAE_PAE_STR(name) "pgmR0BthPAEPAE" #name
2793	#define PGM_BTH_NAME_R0_AMD64_PROT_STR(name) "pgmR0BthAMD64Prot" #name
2794	#define PGM_BTH_NAME_R0_AMD64_AMD64_STR(name) "pgmR0BthAMD64AMD64" #name
2795	#define PGM_BTH_NAME_R0_NESTED_REAL_STR(name) "pgmR0BthNestedReal" #name
2796	#define PGM_BTH_NAME_R0_NESTED_PROT_STR(name) "pgmR0BthNestedProt" #name
2797	#define PGM_BTH_NAME_R0_NESTED_32BIT_STR(name) "pgmR0BthNested32Bit" #name
2798	#define PGM_BTH_NAME_R0_NESTED_PAE_STR(name) "pgmR0BthNestedPAE" #name
2799	#define PGM_BTH_NAME_R0_NESTED_AMD64_STR(name) "pgmR0BthNestedAMD64" #name
2800	#define PGM_BTH_NAME_R0_EPT_REAL_STR(name) "pgmR0BthEPTReal" #name
2801	#define PGM_BTH_NAME_R0_EPT_PROT_STR(name) "pgmR0BthEPTProt" #name
2802	#define PGM_BTH_NAME_R0_EPT_32BIT_STR(name) "pgmR0BthEPT32Bit" #name
2803	#define PGM_BTH_NAME_R0_EPT_PAE_STR(name) "pgmR0BthEPTPAE" #name
2804	#define PGM_BTH_NAME_R0_EPT_AMD64_STR(name) "pgmR0BthEPTAMD64" #name
2805
2806	#define PGM_BTH_DECL(type, name) PGM_CTX_DECL(type) PGM_BTH_NAME(name)
2807	#define PGM_BTH_PFN(name, pVCpu) ((pVCpu)->pgm.s.PGM_CTX(pfn,Bth##name))
2808	/** @} */
2809
2810	/**
2811	* Data for each paging mode.
2812	*/
2813	typedef struct PGMMODEDATA
2814	{
2815	/** The guest mode type. */
2816	uint32_t uGstType;
2817	/** The shadow mode type. */
2818	uint32_t uShwType;
2819
2820	/** @name Function pointers for Shadow paging.
2821	* @{
2822	*/
2823	DECLR3CALLBACKMEMBER(int, pfnR3ShwRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2824	DECLR3CALLBACKMEMBER(int, pfnR3ShwExit,(PVMCPU pVCpu));
2825	DECLR3CALLBACKMEMBER(int, pfnR3ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2826	DECLR3CALLBACKMEMBER(int, pfnR3ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags));
2827
2828	DECLRCCALLBACKMEMBER(int, pfnRCShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2829	DECLRCCALLBACKMEMBER(int, pfnRCShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags));
2830
2831	DECLR0CALLBACKMEMBER(int, pfnR0ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2832	DECLR0CALLBACKMEMBER(int, pfnR0ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags));
2833	/** @} */
2834
2835	/** @name Function pointers for Guest paging.
2836	* @{
2837	*/
2838	DECLR3CALLBACKMEMBER(int, pfnR3GstRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2839	DECLR3CALLBACKMEMBER(int, pfnR3GstExit,(PVMCPU pVCpu));
2840	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2841	DECLR3CALLBACKMEMBER(int, pfnR3GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2842	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2843	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2844	DECLRCCALLBACKMEMBER(int, pfnRCGstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2845	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2846	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2847	DECLR0CALLBACKMEMBER(int, pfnR0GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2848	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2849	/** @} */
2850
2851	/** @name Function pointers for Both Shadow and Guest paging.
2852	* @{
2853	*/
2854	DECLR3CALLBACKMEMBER(int, pfnR3BthRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2855	/* no pfnR3BthTrap0eHandler */
2856	DECLR3CALLBACKMEMBER(int, pfnR3BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2857	DECLR3CALLBACKMEMBER(int, pfnR3BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2858	DECLR3CALLBACKMEMBER(int, pfnR3BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2859	DECLR3CALLBACKMEMBER(int, pfnR3BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2860	#ifdef VBOX_STRICT
2861	DECLR3CALLBACKMEMBER(unsigned, pfnR3BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2862	#endif
2863	DECLR3CALLBACKMEMBER(int, pfnR3BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2864	DECLR3CALLBACKMEMBER(int, pfnR3BthUnmapCR3,(PVMCPU pVCpu));
2865
2866	DECLRCCALLBACKMEMBER(int, pfnRCBthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken));
2867	DECLRCCALLBACKMEMBER(int, pfnRCBthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2868	DECLRCCALLBACKMEMBER(int, pfnRCBthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2869	DECLRCCALLBACKMEMBER(int, pfnRCBthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2870	DECLRCCALLBACKMEMBER(int, pfnRCBthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2871	#ifdef VBOX_STRICT
2872	DECLRCCALLBACKMEMBER(unsigned, pfnRCBthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2873	#endif
2874	DECLRCCALLBACKMEMBER(int, pfnRCBthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2875	DECLRCCALLBACKMEMBER(int, pfnRCBthUnmapCR3,(PVMCPU pVCpu));
2876
2877	DECLR0CALLBACKMEMBER(int, pfnR0BthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken));
2878	DECLR0CALLBACKMEMBER(int, pfnR0BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2879	DECLR0CALLBACKMEMBER(int, pfnR0BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2880	DECLR0CALLBACKMEMBER(int, pfnR0BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2881	DECLR0CALLBACKMEMBER(int, pfnR0BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2882	#ifdef VBOX_STRICT
2883	DECLR0CALLBACKMEMBER(unsigned, pfnR0BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2884	#endif
2885	DECLR0CALLBACKMEMBER(int, pfnR0BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2886	DECLR0CALLBACKMEMBER(int, pfnR0BthUnmapCR3,(PVMCPU pVCpu));
2887	/** @} */
2888	} PGMMODEDATA, *PPGMMODEDATA;
2889
2890
2891	#ifdef VBOX_WITH_STATISTICS
2892	/**
2893	* PGM statistics.
2894	*
2895	* These lives on the heap when compiled in as they would otherwise waste
2896	* unnecessary space in release builds.
2897	*/
2898	typedef struct PGMSTATS
2899	{
2900	/* R3 only: */
2901	STAMCOUNTER StatR3DetectedConflicts; /*< R3: Number of times PGMR3MapHasConflicts() detected a conflict. /
2902	STAMPROFILE StatR3ResolveConflict; /*< R3: pgmR3SyncPTResolveConflict() profiling (includes the entire relocation). /
2903
2904	/* R3+RZ */
2905	STAMCOUNTER StatRZChunkR3MapTlbHits; /*< RC/R0: Ring-3/0 chunk mapper TLB hits. /
2906	STAMCOUNTER StatRZChunkR3MapTlbMisses; /*< RC/R0: Ring-3/0 chunk mapper TLB misses. /
2907	STAMCOUNTER StatRZPageMapTlbHits; /*< RC/R0: Ring-3/0 page mapper TLB hits. /
2908	STAMCOUNTER StatRZPageMapTlbMisses; /*< RC/R0: Ring-3/0 page mapper TLB misses. /
2909	STAMCOUNTER StatPageMapTlbFlushes; /*< ALL: Ring-3/0 page mapper TLB flushes. /
2910	STAMCOUNTER StatPageMapTlbFlushEntry; /*< ALL: Ring-3/0 page mapper TLB flushes. /
2911	STAMCOUNTER StatR3ChunkR3MapTlbHits; /*< R3: Ring-3/0 chunk mapper TLB hits. /
2912	STAMCOUNTER StatR3ChunkR3MapTlbMisses; /*< R3: Ring-3/0 chunk mapper TLB misses. /
2913	STAMCOUNTER StatR3PageMapTlbHits; /*< R3: Ring-3/0 page mapper TLB hits. /
2914	STAMCOUNTER StatR3PageMapTlbMisses; /*< R3: Ring-3/0 page mapper TLB misses. /
2915	STAMCOUNTER StatRZRamRangeTlbHits; /*< RC/R0: RAM range TLB hits. /
2916	STAMCOUNTER StatRZRamRangeTlbMisses; /*< RC/R0: RAM range TLB misses. /
2917	STAMCOUNTER StatR3RamRangeTlbHits; /*< R3: RAM range TLB hits. /
2918	STAMCOUNTER StatR3RamRangeTlbMisses; /*< R3: RAM range TLB misses. /
2919	STAMPROFILE StatRZSyncCR3HandlerVirtualReset; /*< RC/R0: Profiling of the virtual handler resets. /
2920	STAMPROFILE StatRZSyncCR3HandlerVirtualUpdate; /*< RC/R0: Profiling of the virtual handler updates. /
2921	STAMPROFILE StatR3SyncCR3HandlerVirtualReset; /*< R3: Profiling of the virtual handler resets. /
2922	STAMPROFILE StatR3SyncCR3HandlerVirtualUpdate; /*< R3: Profiling of the virtual handler updates. /
2923	STAMCOUNTER StatR3PhysHandlerReset; /*< R3: The number of times PGMHandlerPhysicalReset is called. /
2924	STAMCOUNTER StatRZPhysHandlerReset; /*< RC/R0: The number of times PGMHandlerPhysicalReset is called. /
2925	STAMCOUNTER StatR3PhysHandlerLookupHits; /*< R3: Number of cache hits when looking up physical handlers. /
2926	STAMCOUNTER StatR3PhysHandlerLookupMisses; /*< R3: Number of cache misses when looking up physical handlers. /
2927	STAMCOUNTER StatRZPhysHandlerLookupHits; /*< RC/R0: Number of cache hits when lookup up physical handlers. /
2928	STAMCOUNTER StatRZPhysHandlerLookupMisses; /*< RC/R0: Number of cache misses when looking up physical handlers /
2929	STAMPROFILE StatRZVirtHandlerSearchByPhys; /*< RC/R0: Profiling of pgmHandlerVirtualFindByPhysAddr. /
2930	STAMPROFILE StatR3VirtHandlerSearchByPhys; /*< R3: Profiling of pgmHandlerVirtualFindByPhysAddr. /
2931	STAMCOUNTER StatRZPageReplaceShared; /*< RC/R0: Times a shared page has been replaced by a private one. /
2932	STAMCOUNTER StatRZPageReplaceZero; /*< RC/R0: Times the zero page has been replaced by a private one. /
2933	/// @todo STAMCOUNTER StatRZPageHandyAllocs; /*< RC/R0: The number of times we've executed GMMR3AllocateHandyPages. /
2934	STAMCOUNTER StatR3PageReplaceShared; /*< R3: Times a shared page has been replaced by a private one. /
2935	STAMCOUNTER StatR3PageReplaceZero; /*< R3: Times the zero page has been replaced by a private one. /
2936	/// @todo STAMCOUNTER StatR3PageHandyAllocs; /*< R3: The number of times we've executed GMMR3AllocateHandyPages. /
2937
2938	/* RC only: */
2939	STAMCOUNTER StatRCInvlPgConflict; /*< RC: Number of times PGMInvalidatePage() detected a mapping conflict. /
2940	STAMCOUNTER StatRCInvlPgSyncMonCR3; /*< RC: Number of times PGMInvalidatePage() ran into PGM_SYNC_MONITOR_CR3. /
2941
2942	STAMCOUNTER StatRZPhysRead;
2943	STAMCOUNTER StatRZPhysReadBytes;
2944	STAMCOUNTER StatRZPhysWrite;
2945	STAMCOUNTER StatRZPhysWriteBytes;
2946	STAMCOUNTER StatR3PhysRead;
2947	STAMCOUNTER StatR3PhysReadBytes;
2948	STAMCOUNTER StatR3PhysWrite;
2949	STAMCOUNTER StatR3PhysWriteBytes;
2950	STAMCOUNTER StatRCPhysRead;
2951	STAMCOUNTER StatRCPhysReadBytes;
2952	STAMCOUNTER StatRCPhysWrite;
2953	STAMCOUNTER StatRCPhysWriteBytes;
2954
2955	STAMCOUNTER StatRZPhysSimpleRead;
2956	STAMCOUNTER StatRZPhysSimpleReadBytes;
2957	STAMCOUNTER StatRZPhysSimpleWrite;
2958	STAMCOUNTER StatRZPhysSimpleWriteBytes;
2959	STAMCOUNTER StatR3PhysSimpleRead;
2960	STAMCOUNTER StatR3PhysSimpleReadBytes;
2961	STAMCOUNTER StatR3PhysSimpleWrite;
2962	STAMCOUNTER StatR3PhysSimpleWriteBytes;
2963	STAMCOUNTER StatRCPhysSimpleRead;
2964	STAMCOUNTER StatRCPhysSimpleReadBytes;
2965	STAMCOUNTER StatRCPhysSimpleWrite;
2966	STAMCOUNTER StatRCPhysSimpleWriteBytes;
2967
2968	STAMCOUNTER StatTrackVirgin; /*< The number of first time shadowings. /
2969	STAMCOUNTER StatTrackAliased; /*< The number of times switching to cRef2, i.e. the page is being shadowed by two PTs. /
2970	STAMCOUNTER StatTrackAliasedMany; /*< The number of times we're tracking using cRef2. /
2971	STAMCOUNTER StatTrackAliasedLots; /*< The number of times we're hitting pages which has overflowed cRef2. /
2972	STAMCOUNTER StatTrackNoExtentsLeft; /*< The number of times the extent list was exhausted. /
2973	STAMCOUNTER StatTrackOverflows; /*< The number of times the extent list grows to long. /
2974	STAMPROFILE StatTrackDeref; /*< Profiling of SyncPageWorkerTrackDeref (expensive). /
2975
2976	/** Time spent by the host OS for large page allocation. */
2977	STAMPROFILE StatAllocLargePage;
2978	/** Time spent clearing the newly allocated large pages. */
2979	STAMPROFILE StatClearLargePage;
2980	/** The number of times allocating a large pages takes more than the allowed period. */
2981	STAMCOUNTER StatLargePageOverflow;
2982	/** pgmPhysIsValidLargePage profiling - R3 */
2983	STAMPROFILE StatR3IsValidLargePage;
2984	/** pgmPhysIsValidLargePage profiling - RZ*/
2985	STAMPROFILE StatRZIsValidLargePage;
2986
2987	STAMPROFILE StatChunkAging;
2988	STAMPROFILE StatChunkFindCandidate;
2989	STAMPROFILE StatChunkUnmap;
2990	STAMPROFILE StatChunkMap;
2991	} PGMSTATS;
2992	#endif /* VBOX_WITH_STATISTICS */
2993
2994
2995	/**
2996	* Converts a PGM pointer into a VM pointer.
2997	* @returns Pointer to the VM structure the PGM is part of.
2998	* @param pPGM Pointer to PGM instance data.
2999	*/
3000	#define PGM2VM(pPGM) ( (PVM)((char*)pPGM - pPGM->offVM) )
3001
3002	/**
3003	* PGM Data (part of VM)
3004	*/
3005	typedef struct PGM
3006	{
3007	/** Offset to the VM structure. */
3008	int32_t offVM;
3009	/** Offset of the PGMCPU structure relative to VMCPU. */
3010	int32_t offVCpuPGM;
3011
3012	/** @cfgm{RamPreAlloc, boolean, false}
3013	* Indicates whether the base RAM should all be allocated before starting
3014	* the VM (default), or if it should be allocated when first written to.
3015	*/
3016	bool fRamPreAlloc;
3017	/** Indicates whether write monitoring is currently in use.
3018	* This is used to prevent conflicts between live saving and page sharing
3019	* detection. */
3020	bool fPhysWriteMonitoringEngaged;
3021	/** Set if the CPU has less than 52-bit physical address width.
3022	* This is used */
3023	bool fLessThan52PhysicalAddressBits;
3024	/** Set when nested paging is active.
3025	* This is meant to save calls to HWACCMIsNestedPagingActive and let the
3026	* compilers optimize the code better. Whether we use nested paging or
3027	* not is something we find out during VMM initialization and we won't
3028	* change this later on. */
3029	bool fNestedPaging;
3030	/** The host paging mode. (This is what SUPLib reports.) */
3031	SUPPAGINGMODE enmHostMode;
3032	/** We're not in a state which permits writes to guest memory.
3033	* (Only used in strict builds.) */
3034	bool fNoMorePhysWrites;
3035	/** Set if PCI passthrough is enabled. */
3036	bool fPciPassthrough;
3037	/** Alignment padding that makes the next member start on a 8 byte boundary. */
3038	bool afAlignment1[2];
3039
3040	/** Indicates that PGMR3FinalizeMappings has been called and that further
3041	* PGMR3MapIntermediate calls will be rejected. */
3042	bool fFinalizedMappings;
3043	/** If set no conflict checks are required. */
3044	bool fMappingsFixed;
3045	/** If set if restored as fixed but we were unable to re-fixate at the old
3046	* location because of room or address incompatibilities. */
3047	bool fMappingsFixedRestored;
3048	/** If set, then no mappings are put into the shadow page table.
3049	* Use pgmMapAreMappingsEnabled() instead of direct access. */
3050	bool fMappingsDisabled;
3051	/** Size of fixed mapping.
3052	* This is valid if either fMappingsFixed or fMappingsFixedRestored is set. */
3053	uint32_t cbMappingFixed;
3054	/** Generation ID for the RAM ranges. This member is incremented everytime
3055	* a RAM range is linked or unlinked. */
3056	uint32_t volatile idRamRangesGen;
3057
3058	/** Base address (GC) of fixed mapping.
3059	* This is valid if either fMappingsFixed or fMappingsFixedRestored is set. */
3060	RTGCPTR GCPtrMappingFixed;
3061	/** The address of the previous RAM range mapping. */
3062	RTGCPTR GCPtrPrevRamRangeMapping;
3063
3064	/** 4 MB page mask; 32 or 36 bits depending on PSE-36 (identical for all VCPUs) */
3065	RTGCPHYS GCPhys4MBPSEMask;
3066	/** Mask containing the invalid bits of a guest physical address.
3067	* @remarks this does not stop at bit 52. */
3068	RTGCPHYS GCPhysInvAddrMask;
3069
3070
3071	/** RAM range TLB for R3. */
3072	R3PTRTYPE(PPGMRAMRANGE) apRamRangesTlbR3[PGM_RAMRANGE_TLB_ENTRIES];
3073	/** Pointer to the list of RAM ranges (Phys GC -> Phys HC conversion) - for R3.
3074	* This is sorted by physical address and contains no overlapping ranges. */
3075	R3PTRTYPE(PPGMRAMRANGE) pRamRangesXR3;
3076	/** Root of the RAM range search tree for ring-3. */
3077	R3PTRTYPE(PPGMRAMRANGE) pRamRangeTreeR3;
3078	/** PGM offset based trees - R3 Ptr. */
3079	R3PTRTYPE(PPGMTREES) pTreesR3;
3080	/** Caching the last physical handler we looked up in R3. */
3081	R3PTRTYPE(PPGMPHYSHANDLER) pLastPhysHandlerR3;
3082	/** Shadow Page Pool - R3 Ptr. */
3083	R3PTRTYPE(PPGMPOOL) pPoolR3;
3084	/** Linked list of GC mappings - for HC.
3085	* The list is sorted ascending on address. */
3086	R3PTRTYPE(PPGMMAPPING) pMappingsR3;
3087	/** Pointer to the list of ROM ranges - for R3.
3088	* This is sorted by physical address and contains no overlapping ranges. */
3089	R3PTRTYPE(PPGMROMRANGE) pRomRangesR3;
3090	/** Pointer to the list of MMIO2 ranges - for R3.
3091	* Registration order. */
3092	R3PTRTYPE(PPGMMMIO2RANGE) pMmio2RangesR3;
3093	/** Pointer to SHW+GST mode data (function pointers).
3094	* The index into this table is made up from */
3095	R3PTRTYPE(PPGMMODEDATA) paModeData;
3096	RTR3PTR R3PtrAlignment0;
3097
3098	/** RAM range TLB for R0. */
3099	R0PTRTYPE(PPGMRAMRANGE) apRamRangesTlbR0[PGM_RAMRANGE_TLB_ENTRIES];
3100	/** R0 pointer corresponding to PGM::pRamRangesXR3. */
3101	R0PTRTYPE(PPGMRAMRANGE) pRamRangesXR0;
3102	/** Root of the RAM range search tree for ring-0. */
3103	R0PTRTYPE(PPGMRAMRANGE) pRamRangeTreeR0;
3104	/** PGM offset based trees - R0 Ptr. */
3105	R0PTRTYPE(PPGMTREES) pTreesR0;
3106	/** Caching the last physical handler we looked up in R0. */
3107	R0PTRTYPE(PPGMPHYSHANDLER) pLastPhysHandlerR0;
3108	/** Shadow Page Pool - R0 Ptr. */
3109	R0PTRTYPE(PPGMPOOL) pPoolR0;
3110	/** Linked list of GC mappings - for R0.
3111	* The list is sorted ascending on address. */
3112	R0PTRTYPE(PPGMMAPPING) pMappingsR0;
3113	/** R0 pointer corresponding to PGM::pRomRangesR3. */
3114	R0PTRTYPE(PPGMROMRANGE) pRomRangesR0;
3115	RTR0PTR R0PtrAlignment0;
3116
3117
3118	/** RAM range TLB for RC. */
3119	RCPTRTYPE(PPGMRAMRANGE) apRamRangesTlbRC[PGM_RAMRANGE_TLB_ENTRIES];
3120	/** RC pointer corresponding to PGM::pRamRangesXR3. */
3121	RCPTRTYPE(PPGMRAMRANGE) pRamRangesXRC;
3122	/** Root of the RAM range search tree for raw-mode context. */
3123	RCPTRTYPE(PPGMRAMRANGE) pRamRangeTreeRC;
3124	/** PGM offset based trees - RC Ptr. */
3125	RCPTRTYPE(PPGMTREES) pTreesRC;
3126	/** Caching the last physical handler we looked up in RC. */
3127	RCPTRTYPE(PPGMPHYSHANDLER) pLastPhysHandlerRC;
3128	/** Shadow Page Pool - RC Ptr. */
3129	RCPTRTYPE(PPGMPOOL) pPoolRC;
3130	/** Linked list of GC mappings - for RC.
3131	* The list is sorted ascending on address. */
3132	RCPTRTYPE(PPGMMAPPING) pMappingsRC;
3133	/** RC pointer corresponding to PGM::pRomRangesR3. */
3134	RCPTRTYPE(PPGMROMRANGE) pRomRangesRC;
3135	RTRCPTR RCPtrAlignment0;
3136	/** Pointer to the page table entries for the dynamic page mapping area - GCPtr. */
3137	RCPTRTYPE(PX86PTE) paDynPageMap32BitPTEsGC;
3138	/** Pointer to the page table entries for the dynamic page mapping area - GCPtr. */
3139	RCPTRTYPE(PPGMSHWPTEPAE) paDynPageMapPaePTEsGC;
3140
3141
3142	/** Pointer to the 5 page CR3 content mapping.
3143	* The first page is always the CR3 (in some form) while the 4 other pages
3144	* are used of the PDs in PAE mode. */
3145	RTGCPTR GCPtrCR3Mapping;
3146
3147	/** @name Intermediate Context
3148	* @{ */
3149	/** Pointer to the intermediate page directory - Normal. */
3150	R3PTRTYPE(PX86PD) pInterPD;
3151	/** Pointer to the intermediate page tables - Normal.
3152	* There are two page tables, one for the identity mapping and one for
3153	* the host context mapping (of the core code). */
3154	R3PTRTYPE(PX86PT) apInterPTs[2];
3155	/** Pointer to the intermediate page tables - PAE. */
3156	R3PTRTYPE(PX86PTPAE) apInterPaePTs[2];
3157	/** Pointer to the intermediate page directory - PAE. */
3158	R3PTRTYPE(PX86PDPAE) apInterPaePDs[4];
3159	/** Pointer to the intermediate page directory - PAE. */
3160	R3PTRTYPE(PX86PDPT) pInterPaePDPT;
3161	/** Pointer to the intermediate page-map level 4 - AMD64. */
3162	R3PTRTYPE(PX86PML4) pInterPaePML4;
3163	/** Pointer to the intermediate page directory - AMD64. */
3164	R3PTRTYPE(PX86PDPT) pInterPaePDPT64;
3165	/** The Physical Address (HC) of the intermediate Page Directory - Normal. */
3166	RTHCPHYS HCPhysInterPD;
3167	/** The Physical Address (HC) of the intermediate Page Directory Pointer Table - PAE. */
3168	RTHCPHYS HCPhysInterPaePDPT;
3169	/** The Physical Address (HC) of the intermediate Page Map Level 4 table - AMD64. */
3170	RTHCPHYS HCPhysInterPaePML4;
3171	/** @} */
3172
3173	/** Base address of the dynamic page mapping area.
3174	* The array is MM_HYPER_DYNAMIC_SIZE bytes big.
3175	*
3176	* @todo The plan of keeping PGMRCDYNMAP private to PGMRZDynMap.cpp didn't
3177	* work out. Some cleaning up of the initialization that would
3178	* remove this memory is yet to be done...
3179	*/
3180	RCPTRTYPE(uint8_t *) pbDynPageMapBaseGC;
3181	/** The address of the raw-mode context mapping cache. */
3182	RCPTRTYPE(PPGMRCDYNMAP) pRCDynMap;
3183	/** The address of the ring-0 mapping cache if we're making use of it. */
3184	RTR0PTR pvR0DynMapUsed;
3185
3186	/** Hack: Number of deprecated page mapping locks taken by the current lock
3187	* owner via pgmPhysGCPhys2CCPtrInternalDepr. */
3188	uint32_t cDeprecatedPageLocks;
3189	#if HC_ARCH_BITS == 64
3190	/** Alignment padding. */
3191	uint32_t u32Alignment2;
3192	#endif
3193
3194
3195	/** PGM critical section.
3196	* This protects the physical & virtual access handlers, ram ranges,
3197	* and the page flag updating (some of it anyway).
3198	*/
3199	PDMCRITSECT CritSectX;
3200
3201	/**
3202	* Data associated with managing the ring-3 mappings of the allocation chunks.
3203	*/
3204	struct
3205	{
3206	/** The chunk tree, ordered by chunk id. */
3207	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
3208	R3PTRTYPE(PAVLU32NODECORE) pTree;
3209	#else
3210	R3R0PTRTYPE(PAVLU32NODECORE) pTree;
3211	#endif
3212	#if HC_ARCH_BITS == 32
3213	uint32_t u32Alignment0;
3214	#endif
3215	/** The chunk mapping TLB. */
3216	PGMCHUNKR3MAPTLB Tlb;
3217	/** The number of mapped chunks. */
3218	uint32_t c;
3219	/** The maximum number of mapped chunks.
3220	* @cfgm PGM/MaxRing3Chunks */
3221	uint32_t cMax;
3222	/** The current time. This is incremented whenever a chunk is inserted. */
3223	uint32_t iNow;
3224	/** Alignment padding. */
3225	uint32_t u32Alignment1;
3226	} ChunkR3Map;
3227
3228	/**
3229	* The page mapping TLB for ring-3 and (for the time being) ring-0.
3230	*/
3231	PGMPAGER3MAPTLB PhysTlbHC;
3232
3233	/** @name The zero page.
3234	* @{ */
3235	/** The host physical address of the zero page. */
3236	RTHCPHYS HCPhysZeroPg;
3237	/** The ring-3 mapping of the zero page. */
3238	RTR3PTR pvZeroPgR3;
3239	/** The ring-0 mapping of the zero page. */
3240	RTR0PTR pvZeroPgR0;
3241	/** The GC mapping of the zero page. */
3242	RTRCPTR pvZeroPgRC;
3243	RTRCPTR RCPtrAlignment3;
3244	/** @}*/
3245
3246	/** @name The Invalid MMIO page.
3247	* This page is filled with 0xfeedface.
3248	* @{ */
3249	/** The host physical address of the invalid MMIO page. */
3250	RTHCPHYS HCPhysMmioPg;
3251	/** The host pysical address of the invalid MMIO page plus all invalid
3252	* physical address bits set. This is used to trigger X86_TRAP_PF_RSVD.
3253	* @remarks Check fLessThan52PhysicalAddressBits before use. */
3254	RTHCPHYS HCPhysInvMmioPg;
3255	/** The ring-3 mapping of the invalid MMIO page. */
3256	RTR3PTR pvMmioPgR3;
3257	#if HC_ARCH_BITS == 32
3258	RTR3PTR R3PtrAlignment4;
3259	#endif
3260	/** @} */
3261
3262
3263	/** The number of handy pages. */
3264	uint32_t cHandyPages;
3265
3266	/** The number of large handy pages. */
3267	uint32_t cLargeHandyPages;
3268
3269	/**
3270	* Array of handy pages.
3271	*
3272	* This array is used in a two way communication between pgmPhysAllocPage
3273	* and GMMR0AllocateHandyPages, with PGMR3PhysAllocateHandyPages serving as
3274	* an intermediary.
3275	*
3276	* The size of this array is important, see pgmPhysEnsureHandyPage for details.
3277	* (The current size of 32 pages, means 128 KB of handy memory.)
3278	*/
3279	GMMPAGEDESC aHandyPages[PGM_HANDY_PAGES];
3280
3281	/**
3282	* Array of large handy pages. (currently size 1)
3283	*
3284	* This array is used in a two way communication between pgmPhysAllocLargePage
3285	* and GMMR0AllocateLargePage, with PGMR3PhysAllocateLargePage serving as
3286	* an intermediary.
3287	*/
3288	GMMPAGEDESC aLargeHandyPage[1];
3289
3290	/**
3291	* Live save data.
3292	*/
3293	struct
3294	{
3295	/** Per type statistics. */
3296	struct
3297	{
3298	/** The number of ready pages. */
3299	uint32_t cReadyPages;
3300	/** The number of dirty pages. */
3301	uint32_t cDirtyPages;
3302	/** The number of ready zero pages. */
3303	uint32_t cZeroPages;
3304	/** The number of write monitored pages. */
3305	uint32_t cMonitoredPages;
3306	} Rom,
3307	Mmio2,
3308	Ram;
3309	/** The number of ignored pages in the RAM ranges (i.e. MMIO, MMIO2 and ROM). */
3310	uint32_t cIgnoredPages;
3311	/** Indicates that a live save operation is active. */
3312	bool fActive;
3313	/** Padding. */
3314	bool afReserved[2];
3315	/** The next history index. */
3316	uint8_t iDirtyPagesHistory;
3317	/** History of the total amount of dirty pages. */
3318	uint32_t acDirtyPagesHistory[64];
3319	/** Short term dirty page average. */
3320	uint32_t cDirtyPagesShort;
3321	/** Long term dirty page average. */
3322	uint32_t cDirtyPagesLong;
3323	/** The number of saved pages. This is used to get some kind of estimate of the
3324	* link speed so we can decide when we're done. It is reset after the first
3325	* 7 passes so the speed estimate doesn't get inflated by the initial set of
3326	* zero pages. */
3327	uint64_t cSavedPages;
3328	/** The nanosecond timestamp when cSavedPages was 0. */
3329	uint64_t uSaveStartNS;
3330	/** Pages per second (for statistics). */
3331	uint32_t cPagesPerSecond;
3332	uint32_t cAlignment;
3333	} LiveSave;
3334
3335	/** @name Error injection.
3336	* @{ */
3337	/** Inject handy page allocation errors pretending we're completely out of
3338	* memory. */
3339	bool volatile fErrInjHandyPages;
3340	/** Padding. */
3341	bool afReserved[3];
3342	/** @} */
3343
3344	/** @name Release Statistics
3345	* @{ */
3346	uint32_t cAllPages; /*< The total number of pages. (Should be Private + Shared + Zero + Pure MMIO.) /
3347	uint32_t cPrivatePages; /*< The number of private pages. /
3348	uint32_t cSharedPages; /*< The number of shared pages. /
3349	uint32_t cReusedSharedPages; /*< The number of reused shared pages. /
3350	uint32_t cZeroPages; /*< The number of zero backed pages. /
3351	uint32_t cPureMmioPages; /*< The number of pure MMIO pages. /
3352	uint32_t cMonitoredPages; /*< The number of write monitored pages. /
3353	uint32_t cWrittenToPages; /*< The number of previously write monitored pages. /
3354	uint32_t cWriteLockedPages; /*< The number of write locked pages. /
3355	uint32_t cReadLockedPages; /*< The number of read locked pages. /
3356	uint32_t cBalloonedPages; /*< The number of ballooned pages. /
3357	uint32_t cMappedChunks; /*< Number of times we mapped a chunk. /
3358	uint32_t cUnmappedChunks; /*< Number of times we unmapped a chunk. /
3359	uint32_t cLargePages; /*< The number of large pages. /
3360	uint32_t cLargePagesDisabled; /*< The number of disabled large pages. /
3361	/* uint32_t aAlignment4[1]; */
3362
3363	/** The number of times we were forced to change the hypervisor region location. */
3364	STAMCOUNTER cRelocations;
3365
3366	STAMCOUNTER StatLargePageReused; /*< The number of large pages we've reused./
3367	STAMCOUNTER StatLargePageRefused; /*< The number of times we couldn't use a large page./
3368	STAMCOUNTER StatLargePageRecheck; /*< The number of times we rechecked a disabled large page./
3369
3370	STAMPROFILE StatShModCheck; /*< Profiles shared module checks. /
3371	/** @} */
3372
3373	#ifdef VBOX_WITH_STATISTICS
3374	/** @name Statistics on the heap.
3375	* @{ */
3376	R3PTRTYPE(PGMSTATS *) pStatsR3;
3377	R0PTRTYPE(PGMSTATS *) pStatsR0;
3378	RCPTRTYPE(PGMSTATS *) pStatsRC;
3379	RTRCPTR RCPtrAlignment;
3380	/** @} */
3381	#endif
3382	} PGM;
3383	#ifndef IN_TSTVMSTRUCTGC /* HACK */
3384	AssertCompileMemberAlignment(PGM, paDynPageMap32BitPTEsGC, 8);
3385	AssertCompileMemberAlignment(PGM, GCPtrMappingFixed, sizeof(RTGCPTR));
3386	AssertCompileMemberAlignment(PGM, HCPhysInterPD, 8);
3387	AssertCompileMemberAlignment(PGM, CritSectX, 8);
3388	AssertCompileMemberAlignment(PGM, ChunkR3Map, 8);
3389	AssertCompileMemberAlignment(PGM, PhysTlbHC, 8);
3390	AssertCompileMemberAlignment(PGM, HCPhysZeroPg, 8);
3391	AssertCompileMemberAlignment(PGM, aHandyPages, 8);
3392	AssertCompileMemberAlignment(PGM, cRelocations, 8);
3393	#endif /* !IN_TSTVMSTRUCTGC */
3394	/** Pointer to the PGM instance data. */
3395	typedef PGM *PPGM;
3396
3397
3398
3399	typedef struct PGMCPUSTATS
3400	{
3401	/* Common */
3402	STAMCOUNTER StatSyncPtPD[X86_PG_ENTRIES]; /*< SyncPT - PD distribution. /
3403	STAMCOUNTER StatSyncPagePD[X86_PG_ENTRIES]; /*< SyncPage - PD distribution. /
3404
3405	/* R0 only: */
3406	STAMPROFILE StatR0NpMiscfg; /*< R0: PGMR0Trap0eHandlerNPMisconfig() profiling. /
3407	STAMCOUNTER StatR0NpMiscfgSyncPage; /*< R0: SyncPage calls from PGMR0Trap0eHandlerNPMisconfig(). /
3408
3409	/* RZ only: */
3410	STAMPROFILE StatRZTrap0e; /*< RC/R0: PGMTrap0eHandler() profiling. /
3411	STAMPROFILE StatRZTrap0eTime2Ballooned; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is read access to a ballooned page. /
3412	STAMPROFILE StatRZTrap0eTime2CSAM; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is CSAM. /
3413	STAMPROFILE StatRZTrap0eTime2DirtyAndAccessed; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is dirty and/or accessed bit emulation. /
3414	STAMPROFILE StatRZTrap0eTime2GuestTrap; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is a guest trap. /
3415	STAMPROFILE StatRZTrap0eTime2HndPhys; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is a physical handler. /
3416	STAMPROFILE StatRZTrap0eTime2HndVirt; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is a virtual handler. /
3417	STAMPROFILE StatRZTrap0eTime2HndUnhandled; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is access outside the monitored areas of a monitored page. /
3418	STAMPROFILE StatRZTrap0eTime2InvalidPhys; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is access to an invalid physical guest address. /
3419	STAMPROFILE StatRZTrap0eTime2MakeWritable; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is a page that needed to be made writable. /
3420	STAMPROFILE StatRZTrap0eTime2Mapping; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is the guest mappings. /
3421	STAMPROFILE StatRZTrap0eTime2Misc; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is not known. /
3422	STAMPROFILE StatRZTrap0eTime2OutOfSync; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync page. /
3423	STAMPROFILE StatRZTrap0eTime2OutOfSyncHndPhys; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync physical handler page. /
3424	STAMPROFILE StatRZTrap0eTime2OutOfSyncHndVirt; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync virtual handler page. /
3425	STAMPROFILE StatRZTrap0eTime2OutOfSyncHndObs; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an obsolete handler page. /
3426	STAMPROFILE StatRZTrap0eTime2SyncPT; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is lazy syncing of a PT. /
3427	STAMPROFILE StatRZTrap0eTime2WPEmulation; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is CR0.WP emulation. /
3428	STAMCOUNTER StatRZTrap0eConflicts; /*< RC/R0: The number of times \#PF was caused by an undetected conflict. /
3429	STAMCOUNTER StatRZTrap0eHandlersMapping; /*< RC/R0: Number of traps due to access handlers in mappings. /
3430	STAMCOUNTER StatRZTrap0eHandlersOutOfSync; /*< RC/R0: Number of out-of-sync handled pages. /
3431	STAMCOUNTER StatRZTrap0eHandlersPhysAll; /*< RC/R0: Number of traps due to physical all-access handlers. /
3432	STAMCOUNTER StatRZTrap0eHandlersPhysAllOpt; /*< RC/R0: Number of the physical all-access handler traps using the optimization. /
3433	STAMCOUNTER StatRZTrap0eHandlersPhysWrite; /*< RC/R0: Number of traps due to write-physical access handlers. /
3434	STAMCOUNTER StatRZTrap0eHandlersVirtual; /*< RC/R0: Number of traps due to virtual access handlers. /
3435	STAMCOUNTER StatRZTrap0eHandlersVirtualByPhys; /*< RC/R0: Number of traps due to virtual access handlers found by physical address. /
3436	STAMCOUNTER StatRZTrap0eHandlersVirtualUnmarked;/*< RC/R0: Number of traps due to virtual access handlers found by virtual address (without proper physical flags). /
3437	STAMCOUNTER StatRZTrap0eHandlersUnhandled; /*< RC/R0: Number of traps due to access outside range of monitored page(s). /
3438	STAMCOUNTER StatRZTrap0eHandlersInvalid; /*< RC/R0: Number of traps due to access to invalid physical memory. /
3439	STAMCOUNTER StatRZTrap0eUSNotPresentRead; /*< RC/R0: \#PF err kind /
3440	STAMCOUNTER StatRZTrap0eUSNotPresentWrite; /*< RC/R0: \#PF err kind /
3441	STAMCOUNTER StatRZTrap0eUSWrite; /*< RC/R0: \#PF err kind /
3442	STAMCOUNTER StatRZTrap0eUSReserved; /*< RC/R0: \#PF err kind /
3443	STAMCOUNTER StatRZTrap0eUSNXE; /*< RC/R0: \#PF err kind /
3444	STAMCOUNTER StatRZTrap0eUSRead; /*< RC/R0: \#PF err kind /
3445	STAMCOUNTER StatRZTrap0eSVNotPresentRead; /*< RC/R0: \#PF err kind /
3446	STAMCOUNTER StatRZTrap0eSVNotPresentWrite; /*< RC/R0: \#PF err kind /
3447	STAMCOUNTER StatRZTrap0eSVWrite; /*< RC/R0: \#PF err kind /
3448	STAMCOUNTER StatRZTrap0eSVReserved; /*< RC/R0: \#PF err kind /
3449	STAMCOUNTER StatRZTrap0eSNXE; /*< RC/R0: \#PF err kind /
3450	STAMCOUNTER StatRZTrap0eGuestPF; /*< RC/R0: Real guest \#PFs. /
3451	STAMCOUNTER StatRZTrap0eGuestPFMapping; /*< RC/R0: Real guest \#PF to HMA or other mapping. /
3452	STAMCOUNTER StatRZTrap0eWPEmulInRZ; /*< RC/R0: WP=0 virtualization trap, handled. /
3453	STAMCOUNTER StatRZTrap0eWPEmulToR3; /*< RC/R0: WP=0 virtualization trap, chickened out. /
3454	STAMCOUNTER StatRZTrap0ePD[X86_PG_ENTRIES]; /*< RC/R0: PD distribution of the \#PFs. /
3455	STAMCOUNTER StatRZGuestCR3WriteHandled; /*< RC/R0: The number of times WriteHandlerCR3() was successfully called. /
3456	STAMCOUNTER StatRZGuestCR3WriteUnhandled; /*< RC/R0: The number of times WriteHandlerCR3() was called and we had to fall back to the recompiler. /
3457	STAMCOUNTER StatRZGuestCR3WriteConflict; /*< RC/R0: The number of times WriteHandlerCR3() was called and a conflict was detected. /
3458	STAMCOUNTER StatRZGuestROMWriteHandled; /*< RC/R0: The number of times pgmPhysRomWriteHandler() was successfully called. /
3459	STAMCOUNTER StatRZGuestROMWriteUnhandled; /*< RC/R0: The number of times pgmPhysRomWriteHandler() was called and we had to fall back to the recompiler /
3460	STAMCOUNTER StatRZDynMapMigrateInvlPg; /*< RZ: invlpg in PGMR0DynMapMigrateAutoSet. /
3461	STAMPROFILE StatRZDynMapGCPageInl; /*< RZ: Calls to pgmRZDynMapGCPageInlined. /
3462	STAMCOUNTER StatRZDynMapGCPageInlHits; /*< RZ: Hash table lookup hits. /
3463	STAMCOUNTER StatRZDynMapGCPageInlMisses; /*< RZ: Misses that falls back to the code common. /
3464	STAMCOUNTER StatRZDynMapGCPageInlRamHits; /*< RZ: 1st ram range hits. /
3465	STAMCOUNTER StatRZDynMapGCPageInlRamMisses; /*< RZ: 1st ram range misses, takes slow path. /
3466	STAMPROFILE StatRZDynMapHCPageInl; /*< RZ: Calls to pgmRZDynMapHCPageInlined. /
3467	STAMCOUNTER StatRZDynMapHCPageInlHits; /*< RZ: Hash table lookup hits. /
3468	STAMCOUNTER StatRZDynMapHCPageInlMisses; /*< RZ: Misses that falls back to the code common. /
3469	STAMPROFILE StatRZDynMapHCPage; /*< RZ: Calls to pgmRZDynMapHCPageCommon. /
3470	STAMCOUNTER StatRZDynMapSetOptimize; /*< RZ: Calls to pgmRZDynMapOptimizeAutoSet. /
3471	STAMCOUNTER StatRZDynMapSetSearchFlushes; /*< RZ: Set search restoring to subset flushes. /
3472	STAMCOUNTER StatRZDynMapSetSearchHits; /*< RZ: Set search hits. /
3473	STAMCOUNTER StatRZDynMapSetSearchMisses; /*< RZ: Set search misses. /
3474	STAMCOUNTER StatRZDynMapPage; /*< RZ: Calls to pgmR0DynMapPage. /
3475	STAMCOUNTER StatRZDynMapPageHits0; /*< RZ: Hits at iPage+0. /
3476	STAMCOUNTER StatRZDynMapPageHits1; /*< RZ: Hits at iPage+1. /
3477	STAMCOUNTER StatRZDynMapPageHits2; /*< RZ: Hits at iPage+2. /
3478	STAMCOUNTER StatRZDynMapPageInvlPg; /*< RZ: invlpg. /
3479	STAMCOUNTER StatRZDynMapPageSlow; /*< RZ: Calls to pgmR0DynMapPageSlow. /
3480	STAMCOUNTER StatRZDynMapPageSlowLoopHits; /*< RZ: Hits in the pgmR0DynMapPageSlow search loop. /
3481	STAMCOUNTER StatRZDynMapPageSlowLoopMisses; /*< RZ: Misses in the pgmR0DynMapPageSlow search loop. /
3482	//STAMCOUNTER StatRZDynMapPageSlowLostHits; /*< RZ: Lost hits. /
3483	STAMCOUNTER StatRZDynMapSubsets; /*< RZ: Times PGMDynMapPushAutoSubset was called. /
3484	STAMCOUNTER StatRZDynMapPopFlushes; /*< RZ: Times PGMDynMapPopAutoSubset flushes the subset. /
3485	STAMCOUNTER aStatRZDynMapSetFilledPct[11]; /*< RZ: Set fill distribution, percent. /
3486
3487	/* HC - R3 and (maybe) R0: */
3488
3489	/* RZ & R3: */
3490	STAMPROFILE StatRZSyncCR3; /*< RC/R0: PGMSyncCR3() profiling. /
3491	STAMPROFILE StatRZSyncCR3Handlers; /*< RC/R0: Profiling of the PGMSyncCR3() update handler section. /
3492	STAMCOUNTER StatRZSyncCR3Global; /*< RC/R0: The number of global CR3 syncs. /
3493	STAMCOUNTER StatRZSyncCR3NotGlobal; /*< RC/R0: The number of non-global CR3 syncs. /
3494	STAMCOUNTER StatRZSyncCR3DstCacheHit; /*< RC/R0: The number of times we got some kind of cache hit on a page table. /
3495	STAMCOUNTER StatRZSyncCR3DstFreed; /*< RC/R0: The number of times we've had to free a shadow entry. /
3496	STAMCOUNTER StatRZSyncCR3DstFreedSrcNP; /*< RC/R0: The number of times we've had to free a shadow entry for which the source entry was not present. /
3497	STAMCOUNTER StatRZSyncCR3DstNotPresent; /*< RC/R0: The number of times we've encountered a not present shadow entry for a present guest entry. /
3498	STAMCOUNTER StatRZSyncCR3DstSkippedGlobalPD; /*< RC/R0: The number of times a global page directory wasn't flushed. /
3499	STAMCOUNTER StatRZSyncCR3DstSkippedGlobalPT; /*< RC/R0: The number of times a page table with only global entries wasn't flushed. /
3500	STAMPROFILE StatRZSyncPT; /*< RC/R0: PGMSyncPT() profiling. /
3501	STAMCOUNTER StatRZSyncPTFailed; /*< RC/R0: The number of times PGMSyncPT() failed. /
3502	STAMCOUNTER StatRZSyncPT4K; /*< RC/R0: Number of 4KB syncs. /
3503	STAMCOUNTER StatRZSyncPT4M; /*< RC/R0: Number of 4MB syncs. /
3504	STAMCOUNTER StatRZSyncPagePDNAs; /*< RC/R0: The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit. /
3505	STAMCOUNTER StatRZSyncPagePDOutOfSync; /*< RC/R0: The number of time we've encountered an out-of-sync PD in SyncPage. /
3506	STAMCOUNTER StatRZAccessedPage; /*< RC/R0: The number of pages marked not present for accessed bit emulation. /
3507	STAMPROFILE StatRZDirtyBitTracking; /*< RC/R0: Profiling the dirty bit tracking in CheckPageFault().. /
3508	STAMCOUNTER StatRZDirtyPage; /*< RC/R0: The number of pages marked read-only for dirty bit tracking. /
3509	STAMCOUNTER StatRZDirtyPageBig; /*< RC/R0: The number of pages marked read-only for dirty bit tracking. /
3510	STAMCOUNTER StatRZDirtyPageSkipped; /*< RC/R0: The number of pages already dirty or readonly. /
3511	STAMCOUNTER StatRZDirtyPageTrap; /*< RC/R0: The number of traps generated for dirty bit tracking. /
3512	STAMCOUNTER StatRZDirtyPageStale; /*< RC/R0: The number of traps generated for dirty bit tracking. (stale tlb entries) /
3513	STAMCOUNTER StatRZDirtyTrackRealPF; /*< RC/R0: The number of real pages faults during dirty bit tracking. /
3514	STAMCOUNTER StatRZDirtiedPage; /*< RC/R0: The number of pages marked dirty because of write accesses. /
3515	STAMCOUNTER StatRZPageAlreadyDirty; /*< RC/R0: The number of pages already marked dirty because of write accesses. /
3516	STAMPROFILE StatRZInvalidatePage; /*< RC/R0: PGMInvalidatePage() profiling. /
3517	STAMCOUNTER StatRZInvalidatePage4KBPages; /*< RC/R0: The number of times PGMInvalidatePage() was called for a 4KB page. /
3518	STAMCOUNTER StatRZInvalidatePage4MBPages; /*< RC/R0: The number of times PGMInvalidatePage() was called for a 4MB page. /
3519	STAMCOUNTER StatRZInvalidatePage4MBPagesSkip; /*< RC/R0: The number of times PGMInvalidatePage() skipped a 4MB page. /
3520	STAMCOUNTER StatRZInvalidatePagePDMappings; /*< RC/R0: The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict). /
3521	STAMCOUNTER StatRZInvalidatePagePDNAs; /*< RC/R0: The number of times PGMInvalidatePage() was called for a not accessed page directory. /
3522	STAMCOUNTER StatRZInvalidatePagePDNPs; /*< RC/R0: The number of times PGMInvalidatePage() was called for a not present page directory. /
3523	STAMCOUNTER StatRZInvalidatePagePDOutOfSync; /*< RC/R0: The number of times PGMInvalidatePage() was called for an out of sync page directory. /
3524	STAMCOUNTER StatRZInvalidatePageSkipped; /*< RC/R0: The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3. /
3525	STAMCOUNTER StatRZPageOutOfSyncUser; /*< RC/R0: The number of times user page is out of sync was detected in \#PF or VerifyAccessSyncPage. /
3526	STAMCOUNTER StatRZPageOutOfSyncSupervisor; /*< RC/R0: The number of times supervisor page is out of sync was detected in in \#PF or VerifyAccessSyncPage. /
3527	STAMCOUNTER StatRZPageOutOfSyncUserWrite; /*< RC/R0: The number of times user page is out of sync was detected in \#PF. /
3528	STAMCOUNTER StatRZPageOutOfSyncSupervisorWrite; /*< RC/R0: The number of times supervisor page is out of sync was detected in in \#PF. /
3529	STAMCOUNTER StatRZPageOutOfSyncBallloon; /*< RC/R0: The number of times a ballooned page was accessed (read). /
3530	STAMPROFILE StatRZPrefetch; /*< RC/R0: PGMPrefetchPage. /
3531	STAMPROFILE StatRZFlushTLB; /*< RC/R0: Profiling of the PGMFlushTLB() body. /
3532	STAMCOUNTER StatRZFlushTLBNewCR3; /*< RC/R0: The number of times PGMFlushTLB was called with a new CR3, non-global. (switch) /
3533	STAMCOUNTER StatRZFlushTLBNewCR3Global; /*< RC/R0: The number of times PGMFlushTLB was called with a new CR3, global. (switch) /
3534	STAMCOUNTER StatRZFlushTLBSameCR3; /*< RC/R0: The number of times PGMFlushTLB was called with the same CR3, non-global. (flush) /
3535	STAMCOUNTER StatRZFlushTLBSameCR3Global; /*< RC/R0: The number of times PGMFlushTLB was called with the same CR3, global. (flush) /
3536	STAMPROFILE StatRZGstModifyPage; /*< RC/R0: Profiling of the PGMGstModifyPage() body /
3537
3538	STAMPROFILE StatR3SyncCR3; /*< R3: PGMSyncCR3() profiling. /
3539	STAMPROFILE StatR3SyncCR3Handlers; /*< R3: Profiling of the PGMSyncCR3() update handler section. /
3540	STAMCOUNTER StatR3SyncCR3Global; /*< R3: The number of global CR3 syncs. /
3541	STAMCOUNTER StatR3SyncCR3NotGlobal; /*< R3: The number of non-global CR3 syncs. /
3542	STAMCOUNTER StatR3SyncCR3DstFreed; /*< R3: The number of times we've had to free a shadow entry. /
3543	STAMCOUNTER StatR3SyncCR3DstFreedSrcNP; /*< R3: The number of times we've had to free a shadow entry for which the source entry was not present. /
3544	STAMCOUNTER StatR3SyncCR3DstNotPresent; /*< R3: The number of times we've encountered a not present shadow entry for a present guest entry. /
3545	STAMCOUNTER StatR3SyncCR3DstSkippedGlobalPD; /*< R3: The number of times a global page directory wasn't flushed. /
3546	STAMCOUNTER StatR3SyncCR3DstSkippedGlobalPT; /*< R3: The number of times a page table with only global entries wasn't flushed. /
3547	STAMCOUNTER StatR3SyncCR3DstCacheHit; /*< R3: The number of times we got some kind of cache hit on a page table. /
3548	STAMPROFILE StatR3SyncPT; /*< R3: PGMSyncPT() profiling. /
3549	STAMCOUNTER StatR3SyncPTFailed; /*< R3: The number of times PGMSyncPT() failed. /
3550	STAMCOUNTER StatR3SyncPT4K; /*< R3: Number of 4KB syncs. /
3551	STAMCOUNTER StatR3SyncPT4M; /*< R3: Number of 4MB syncs. /
3552	STAMCOUNTER StatR3SyncPagePDNAs; /*< R3: The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit. /
3553	STAMCOUNTER StatR3SyncPagePDOutOfSync; /*< R3: The number of time we've encountered an out-of-sync PD in SyncPage. /
3554	STAMCOUNTER StatR3AccessedPage; /*< R3: The number of pages marked not present for accessed bit emulation. /
3555	STAMPROFILE StatR3DirtyBitTracking; /*< R3: Profiling the dirty bit tracking in CheckPageFault(). /
3556	STAMCOUNTER StatR3DirtyPage; /*< R3: The number of pages marked read-only for dirty bit tracking. /
3557	STAMCOUNTER StatR3DirtyPageBig; /*< R3: The number of pages marked read-only for dirty bit tracking. /
3558	STAMCOUNTER StatR3DirtyPageSkipped; /*< R3: The number of pages already dirty or readonly. /
3559	STAMCOUNTER StatR3DirtyPageTrap; /*< R3: The number of traps generated for dirty bit tracking. /
3560	STAMCOUNTER StatR3DirtyTrackRealPF; /*< R3: The number of real pages faults during dirty bit tracking. /
3561	STAMCOUNTER StatR3DirtiedPage; /*< R3: The number of pages marked dirty because of write accesses. /
3562	STAMCOUNTER StatR3PageAlreadyDirty; /*< R3: The number of pages already marked dirty because of write accesses. /
3563	STAMPROFILE StatR3InvalidatePage; /*< R3: PGMInvalidatePage() profiling. /
3564	STAMCOUNTER StatR3InvalidatePage4KBPages; /*< R3: The number of times PGMInvalidatePage() was called for a 4KB page. /
3565	STAMCOUNTER StatR3InvalidatePage4MBPages; /*< R3: The number of times PGMInvalidatePage() was called for a 4MB page. /
3566	STAMCOUNTER StatR3InvalidatePage4MBPagesSkip; /*< R3: The number of times PGMInvalidatePage() skipped a 4MB page. /
3567	STAMCOUNTER StatR3InvalidatePagePDNAs; /*< R3: The number of times PGMInvalidatePage() was called for a not accessed page directory. /
3568	STAMCOUNTER StatR3InvalidatePagePDNPs; /*< R3: The number of times PGMInvalidatePage() was called for a not present page directory. /
3569	STAMCOUNTER StatR3InvalidatePagePDMappings; /*< R3: The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict). /
3570	STAMCOUNTER StatR3InvalidatePagePDOutOfSync; /*< R3: The number of times PGMInvalidatePage() was called for an out of sync page directory. /
3571	STAMCOUNTER StatR3InvalidatePageSkipped; /*< R3: The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3. /
3572	STAMCOUNTER StatR3PageOutOfSyncUser; /*< R3: The number of times user page is out of sync was detected in \#PF or VerifyAccessSyncPage. /
3573	STAMCOUNTER StatR3PageOutOfSyncSupervisor; /*< R3: The number of times supervisor page is out of sync was detected in in \#PF or VerifyAccessSyncPage. /
3574	STAMCOUNTER StatR3PageOutOfSyncUserWrite; /*< R3: The number of times user page is out of sync was detected in \#PF. /
3575	STAMCOUNTER StatR3PageOutOfSyncSupervisorWrite; /*< R3: The number of times supervisor page is out of sync was detected in in \#PF. /
3576	STAMCOUNTER StatR3PageOutOfSyncBallloon; /*< R3: The number of times a ballooned page was accessed (read). /
3577	STAMPROFILE StatR3Prefetch; /*< R3: PGMPrefetchPage. /
3578	STAMPROFILE StatR3FlushTLB; /*< R3: Profiling of the PGMFlushTLB() body. /
3579	STAMCOUNTER StatR3FlushTLBNewCR3; /*< R3: The number of times PGMFlushTLB was called with a new CR3, non-global. (switch) /
3580	STAMCOUNTER StatR3FlushTLBNewCR3Global; /*< R3: The number of times PGMFlushTLB was called with a new CR3, global. (switch) /
3581	STAMCOUNTER StatR3FlushTLBSameCR3; /*< R3: The number of times PGMFlushTLB was called with the same CR3, non-global. (flush) /
3582	STAMCOUNTER StatR3FlushTLBSameCR3Global; /*< R3: The number of times PGMFlushTLB was called with the same CR3, global. (flush) /
3583	STAMPROFILE StatR3GstModifyPage; /*< R3: Profiling of the PGMGstModifyPage() body /
3584	/** @} */
3585	} PGMCPUSTATS;
3586
3587
3588	/**
3589	* Converts a PGMCPU pointer into a VM pointer.
3590	* @returns Pointer to the VM structure the PGM is part of.
3591	* @param pPGM Pointer to PGMCPU instance data.
3592	*/
3593	#define PGMCPU2VM(pPGM) ( (PVM)((char*)(pPGM) - (pPGM)->offVM) )
3594
3595	/**
3596	* Converts a PGMCPU pointer into a PGM pointer.
3597	* @returns Pointer to the VM structure the PGM is part of.
3598	* @param pPGM Pointer to PGMCPU instance data.
3599	*/
3600	#define PGMCPU2PGM(pPGMCpu) ( (PPGM)((char *)(pPGMCpu) - (pPGMCpu)->offPGM) )
3601
3602	/**
3603	* PGMCPU Data (part of VMCPU).
3604	*/
3605	typedef struct PGMCPU
3606	{
3607	/** Offset to the VM structure. */
3608	int32_t offVM;
3609	/** Offset to the VMCPU structure. */
3610	int32_t offVCpu;
3611	/** Offset of the PGM structure relative to VMCPU. */
3612	int32_t offPGM;
3613	uint32_t uPadding0; /*< structure size alignment. /
3614
3615	#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE) \|\| defined(VBOX_WITH_RAW_MODE)
3616	/** Automatically tracked physical memory mapping set.
3617	* Ring-0 and strict raw-mode builds. */
3618	PGMMAPSET AutoSet;
3619	#endif
3620
3621	/** A20 gate mask.
3622	* Our current approach to A20 emulation is to let REM do it and don't bother
3623	* anywhere else. The interesting Guests will be operating with it enabled anyway.
3624	* But whould need arrise, we'll subject physical addresses to this mask. */
3625	RTGCPHYS GCPhysA20Mask;
3626	/** A20 gate state - boolean! */
3627	bool fA20Enabled;
3628	/** Mirror of the EFER.NXE bit. Managed by PGMNotifyNxeChanged. */
3629	bool fNoExecuteEnabled;
3630	/** Unused bits. */
3631	bool afUnused[2];
3632
3633	/** What needs syncing (PGM_SYNC_*).
3634	* This is used to queue operations for PGMSyncCR3, PGMInvalidatePage,
3635	* PGMFlushTLB, and PGMR3Load. */
3636	RTUINT fSyncFlags;
3637
3638	/** The shadow paging mode. */
3639	PGMMODE enmShadowMode;
3640	/** The guest paging mode. */
3641	PGMMODE enmGuestMode;
3642
3643	/** The current physical address representing in the guest CR3 register. */
3644	RTGCPHYS GCPhysCR3;
3645
3646	/** @name 32-bit Guest Paging.
3647	* @{ */
3648	/** The guest's page directory, R3 pointer. */
3649	R3PTRTYPE(PX86PD) pGst32BitPdR3;
3650	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
3651	/** The guest's page directory, R0 pointer. */
3652	R0PTRTYPE(PX86PD) pGst32BitPdR0;
3653	#endif
3654	/** The guest's page directory, static RC mapping. */
3655	RCPTRTYPE(PX86PD) pGst32BitPdRC;
3656	/** Mask containing the MBZ bits of a big page PDE. */
3657	uint32_t fGst32BitMbzBigPdeMask;
3658	/** Set if the page size extension (PSE) is enabled. */
3659	bool fGst32BitPageSizeExtension;
3660	/** Alignment padding. */
3661	bool afAlignment2[3];
3662	/** @} */
3663
3664	/** @name PAE Guest Paging.
3665	* @{ */
3666	/** The guest's page directory pointer table, static RC mapping. */
3667	RCPTRTYPE(PX86PDPT) pGstPaePdptRC;
3668	/** The guest's page directory pointer table, R3 pointer. */
3669	R3PTRTYPE(PX86PDPT) pGstPaePdptR3;
3670	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
3671	/** The guest's page directory pointer table, R0 pointer. */
3672	R0PTRTYPE(PX86PDPT) pGstPaePdptR0;
3673	#endif
3674
3675	/** The guest's page directories, R3 pointers.
3676	* These are individual pointers and don't have to be adjacent.
3677	* These don't have to be up-to-date - use pgmGstGetPaePD() to access them. */
3678	R3PTRTYPE(PX86PDPAE) apGstPaePDsR3[4];
3679	/** The guest's page directories, R0 pointers.
3680	* Same restrictions as apGstPaePDsR3. */
3681	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
3682	R0PTRTYPE(PX86PDPAE) apGstPaePDsR0[4];
3683	#endif
3684	/** The guest's page directories, static GC mapping.
3685	* Unlike the R3/R0 array the first entry can be accessed as a 2048 entry PD.
3686	* These don't have to be up-to-date - use pgmGstGetPaePD() to access them. */
3687	RCPTRTYPE(PX86PDPAE) apGstPaePDsRC[4];
3688	/** The physical addresses of the guest page directories (PAE) pointed to by apGstPagePDsHC/GC.
3689	* @todo Remove this and use aGstPaePdpeRegs instead? */
3690	RTGCPHYS aGCPhysGstPaePDs[4];
3691	/** The values of the 4 PDPE CPU registers (PAE). */
3692	X86PDPE aGstPaePdpeRegs[4];
3693	/** The physical addresses of the monitored guest page directories (PAE). */
3694	RTGCPHYS aGCPhysGstPaePDsMonitored[4];
3695	/** Mask containing the MBZ PTE bits. */
3696	uint64_t fGstPaeMbzPteMask;
3697	/** Mask containing the MBZ PDE bits. */
3698	uint64_t fGstPaeMbzPdeMask;
3699	/** Mask containing the MBZ big page PDE bits. */
3700	uint64_t fGstPaeMbzBigPdeMask;
3701	/** Mask containing the MBZ PDPE bits. */
3702	uint64_t fGstPaeMbzPdpeMask;
3703	/** @} */
3704
3705	/** @name AMD64 Guest Paging.
3706	* @{ */
3707	/** The guest's page directory pointer table, R3 pointer. */
3708	R3PTRTYPE(PX86PML4) pGstAmd64Pml4R3;
3709	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
3710	/** The guest's page directory pointer table, R0 pointer. */
3711	R0PTRTYPE(PX86PML4) pGstAmd64Pml4R0;
3712	#else
3713	RTR0PTR alignment6b; /*< alignment equalizer. /
3714	#endif
3715	/** Mask containing the MBZ PTE bits. */
3716	uint64_t fGstAmd64MbzPteMask;
3717	/** Mask containing the MBZ PDE bits. */
3718	uint64_t fGstAmd64MbzPdeMask;
3719	/** Mask containing the MBZ big page PDE bits. */
3720	uint64_t fGstAmd64MbzBigPdeMask;
3721	/** Mask containing the MBZ PDPE bits. */
3722	uint64_t fGstAmd64MbzPdpeMask;
3723	/** Mask containing the MBZ big page PDPE bits. */
3724	uint64_t fGstAmd64MbzBigPdpeMask;
3725	/** Mask containing the MBZ PML4E bits. */
3726	uint64_t fGstAmd64MbzPml4eMask;
3727	/** Mask containing the PDPE bits that we shadow. */
3728	uint64_t fGstAmd64ShadowedPdpeMask;
3729	/** Mask containing the PML4E bits that we shadow. */
3730	uint64_t fGstAmd64ShadowedPml4eMask;
3731	/** @} */
3732
3733	/** @name PAE and AMD64 Guest Paging.
3734	* @{ */
3735	/** Mask containing the PTE bits that we shadow. */
3736	uint64_t fGst64ShadowedPteMask;
3737	/** Mask containing the PDE bits that we shadow. */
3738	uint64_t fGst64ShadowedPdeMask;
3739	/** Mask containing the big page PDE bits that we shadow in the PDE. */
3740	uint64_t fGst64ShadowedBigPdeMask;
3741	/** Mask containing the big page PDE bits that we shadow in the PTE. */
3742	uint64_t fGst64ShadowedBigPde4PteMask;
3743	/** @} */
3744
3745	/** Pointer to the page of the current active CR3 - R3 Ptr. */
3746	R3PTRTYPE(PPGMPOOLPAGE) pShwPageCR3R3;
3747	/** Pointer to the page of the current active CR3 - R0 Ptr. */
3748	R0PTRTYPE(PPGMPOOLPAGE) pShwPageCR3R0;
3749	/** Pointer to the page of the current active CR3 - RC Ptr. */
3750	RCPTRTYPE(PPGMPOOLPAGE) pShwPageCR3RC;
3751	/* The shadow page pool index of the user table as specified during allocation; useful for freeing root pages */
3752	uint32_t iShwUser;
3753	/* The index into the user table (shadowed) as specified during allocation; useful for freeing root pages. */
3754	uint32_t iShwUserTable;
3755	# if HC_ARCH_BITS == 64
3756	RTRCPTR alignment6; /*< structure size alignment. /
3757	# endif
3758	/** @} */
3759
3760	/** @name Function pointers for Shadow paging.
3761	* @{
3762	*/
3763	DECLR3CALLBACKMEMBER(int, pfnR3ShwRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
3764	DECLR3CALLBACKMEMBER(int, pfnR3ShwExit,(PVMCPU pVCpu));
3765	DECLR3CALLBACKMEMBER(int, pfnR3ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
3766	DECLR3CALLBACKMEMBER(int, pfnR3ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags));
3767
3768	DECLRCCALLBACKMEMBER(int, pfnRCShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
3769	DECLRCCALLBACKMEMBER(int, pfnRCShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags));
3770
3771	DECLR0CALLBACKMEMBER(int, pfnR0ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
3772	DECLR0CALLBACKMEMBER(int, pfnR0ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags));
3773
3774	/** @} */
3775
3776	/** @name Function pointers for Guest paging.
3777	* @{
3778	*/
3779	DECLR3CALLBACKMEMBER(int, pfnR3GstRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
3780	DECLR3CALLBACKMEMBER(int, pfnR3GstExit,(PVMCPU pVCpu));
3781	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
3782	DECLR3CALLBACKMEMBER(int, pfnR3GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
3783	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
3784	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
3785	DECLRCCALLBACKMEMBER(int, pfnRCGstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
3786	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
3787	#if HC_ARCH_BITS == 64
3788	RTRCPTR alignment3; /*< structure size alignment. /
3789	#endif
3790
3791	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
3792	DECLR0CALLBACKMEMBER(int, pfnR0GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
3793	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
3794	/** @} */
3795
3796	/** @name Function pointers for Both Shadow and Guest paging.
3797	* @{
3798	*/
3799	DECLR3CALLBACKMEMBER(int, pfnR3BthRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
3800	/* no pfnR3BthTrap0eHandler */
3801	DECLR3CALLBACKMEMBER(int, pfnR3BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
3802	DECLR3CALLBACKMEMBER(int, pfnR3BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
3803	DECLR3CALLBACKMEMBER(int, pfnR3BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
3804	DECLR3CALLBACKMEMBER(int, pfnR3BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
3805	DECLR3CALLBACKMEMBER(unsigned, pfnR3BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
3806	DECLR3CALLBACKMEMBER(int, pfnR3BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
3807	DECLR3CALLBACKMEMBER(int, pfnR3BthUnmapCR3,(PVMCPU pVCpu));
3808
3809	DECLR0CALLBACKMEMBER(int, pfnR0BthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken));
3810	DECLR0CALLBACKMEMBER(int, pfnR0BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
3811	DECLR0CALLBACKMEMBER(int, pfnR0BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
3812	DECLR0CALLBACKMEMBER(int, pfnR0BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
3813	DECLR0CALLBACKMEMBER(int, pfnR0BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
3814	DECLR0CALLBACKMEMBER(unsigned, pfnR0BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
3815	DECLR0CALLBACKMEMBER(int, pfnR0BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
3816	DECLR0CALLBACKMEMBER(int, pfnR0BthUnmapCR3,(PVMCPU pVCpu));
3817
3818	DECLRCCALLBACKMEMBER(int, pfnRCBthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken));
3819	DECLRCCALLBACKMEMBER(int, pfnRCBthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
3820	DECLRCCALLBACKMEMBER(int, pfnRCBthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
3821	DECLRCCALLBACKMEMBER(int, pfnRCBthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
3822	DECLRCCALLBACKMEMBER(int, pfnRCBthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
3823	DECLRCCALLBACKMEMBER(unsigned, pfnRCBthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
3824	DECLRCCALLBACKMEMBER(int, pfnRCBthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
3825	DECLRCCALLBACKMEMBER(int, pfnRCBthUnmapCR3,(PVMCPU pVCpu));
3826	#if 0
3827	RTRCPTR alignment2; /*< structure size alignment. /
3828	#endif
3829	/** @} */
3830
3831	/** For saving stack space, the disassembler state is allocated here instead of
3832	* on the stack. */
3833	DISCPUSTATE DisState;
3834
3835	/** Count the number of pgm pool access handler calls. */
3836	uint64_t cPoolAccessHandler;
3837
3838	/** @name Release Statistics
3839	* @{ */
3840	/** The number of times the guest has switched mode since last reset or statistics reset. */
3841	STAMCOUNTER cGuestModeChanges;
3842	/** The number of times the guest has switched mode since last reset or statistics reset. */
3843	STAMCOUNTER cA20Changes;
3844	/** @} */
3845
3846	#ifdef VBOX_WITH_STATISTICS /** @todo move this chunk to the heap. */
3847	/** @name Statistics
3848	* @{ */
3849	/** RC: Pointer to the statistics. */
3850	RCPTRTYPE(PGMCPUSTATS *) pStatsRC;
3851	/** RC: Which statistic this \#PF should be attributed to. */
3852	RCPTRTYPE(PSTAMPROFILE) pStatTrap0eAttributionRC;
3853	/** R0: Pointer to the statistics. */
3854	R0PTRTYPE(PGMCPUSTATS *) pStatsR0;
3855	/** R0: Which statistic this \#PF should be attributed to. */
3856	R0PTRTYPE(PSTAMPROFILE) pStatTrap0eAttributionR0;
3857	/** R3: Pointer to the statistics. */
3858	R3PTRTYPE(PGMCPUSTATS *) pStatsR3;
3859	/** Alignment padding. */
3860	RTR3PTR pPaddingR3;
3861	/** @} */
3862	#endif /* VBOX_WITH_STATISTICS */
3863	} PGMCPU;
3864	/** Pointer to the per-cpu PGM data. */
3865	typedef PGMCPU *PPGMCPU;
3866
3867
3868	/** @name PGM::fSyncFlags Flags
3869	* @{
3870	*/
3871	/** Updates the virtual access handler state bit in PGMPAGE. */
3872	#define PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL RT_BIT(0)
3873	/** Always sync CR3. */
3874	#define PGM_SYNC_ALWAYS RT_BIT(1)
3875	/** Check monitoring on next CR3 (re)load and invalidate page.
3876	* @todo This is obsolete now. Remove after 2.2.0 is branched off. */
3877	#define PGM_SYNC_MONITOR_CR3 RT_BIT(2)
3878	/** Check guest mapping in SyncCR3. */
3879	#define PGM_SYNC_MAP_CR3 RT_BIT(3)
3880	/** Clear the page pool (a light weight flush). */
3881	#define PGM_SYNC_CLEAR_PGM_POOL_BIT 8
3882	#define PGM_SYNC_CLEAR_PGM_POOL RT_BIT(PGM_SYNC_CLEAR_PGM_POOL_BIT)
3883	/** @} */
3884
3885
3886	RT_C_DECLS_BEGIN
3887
3888	int pgmLock(PVM pVM);
3889	void pgmUnlock(PVM pVM);
3890	/**
3891	* Asserts that the caller owns the PDM lock.
3892	* This is the internal variant of PGMIsLockOwner.
3893	* @param a_pVM The VM handle.
3894	*/
3895	#define PGM_LOCK_ASSERT_OWNER(a_pVM) Assert(PDMCritSectIsOwner(&(a_pVM)->pgm.s.CritSectX))
3896	/**
3897	* Asserts that the caller owns the PDM lock.
3898	* This is the internal variant of PGMIsLockOwner.
3899	* @param a_pVM The VM handle.
3900	* @param a_pVCpu The current CPU handle.
3901	*/
3902	#define PGM_LOCK_ASSERT_OWNER_EX(a_pVM, a_pVCpu) Assert(PDMCritSectIsOwnerEx(&(a_pVM)->pgm.s.CritSectX, pVCpu))
3903
3904	int pgmR3MappingsFixInternal(PVM pVM, RTGCPTR GCPtrBase, uint32_t cb);
3905	int pgmR3SyncPTResolveConflict(PVM pVM, PPGMMAPPING pMapping, PX86PD pPDSrc, RTGCPTR GCPtrOldMapping);
3906	int pgmR3SyncPTResolveConflictPAE(PVM pVM, PPGMMAPPING pMapping, RTGCPTR GCPtrOldMapping);
3907	PPGMMAPPING pgmGetMapping(PVM pVM, RTGCPTR GCPtr);
3908	int pgmMapResolveConflicts(PVM pVM);
3909	DECLCALLBACK(void) pgmR3MapInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
3910
3911	void pgmR3HandlerPhysicalUpdateAll(PVM pVM);
3912	bool pgmHandlerPhysicalIsAll(PVM pVM, RTGCPHYS GCPhys);
3913	void pgmHandlerPhysicalResetAliasedPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhysPage, bool fDoAccounting);
3914	int pgmHandlerVirtualFindByPhysAddr(PVM pVM, RTGCPHYS GCPhys, PPGMVIRTHANDLER ppVirt, unsigned piPage);
3915	DECLCALLBACK(int) pgmHandlerVirtualResetOne(PAVLROGCPTRNODECORE pNode, void *pvUser);
3916	#if defined(VBOX_STRICT) \|\| defined(LOG_ENABLED)
3917	void pgmHandlerVirtualDumpPhysPages(PVM pVM);
3918	#else
3919	# define pgmHandlerVirtualDumpPhysPages(a) do { } while (0)
3920	#endif
3921	DECLCALLBACK(void) pgmR3InfoHandlers(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
3922	int pgmR3InitSavedState(PVM pVM, uint64_t cbRam);
3923
3924	int pgmPhysAllocPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
3925	int pgmPhysAllocLargePage(PVM pVM, RTGCPHYS GCPhys);
3926	int pgmPhysRecheckLargePage(PVM pVM, RTGCPHYS GCPhys, PPGMPAGE pLargePage);
3927	int pgmPhysPageLoadIntoTlb(PVM pVM, RTGCPHYS GCPhys);
3928	int pgmPhysPageLoadIntoTlbWithPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
3929	void pgmPhysPageMakeWriteMonitoredWritable(PVM pVM, PPGMPAGE pPage);
3930	int pgmPhysPageMakeWritable(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
3931	int pgmPhysPageMakeWritableAndMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv);
3932	int pgmPhysPageMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv);
3933	int pgmPhysPageMapReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const **ppv);
3934	int pgmPhysPageMapByPageID(PVM pVM, uint32_t idPage, RTHCPHYS HCPhys, void **ppv);
3935	int pgmPhysGCPhys2R3Ptr(PVM pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr);
3936	int pgmPhysCr3ToHCPtr(PVM pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr);
3937	int pgmPhysGCPhys2CCPtrInternalDepr(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv);
3938	int pgmPhysGCPhys2CCPtrInternal(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock);
3939	int pgmPhysGCPhys2CCPtrInternalReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, const void **ppv, PPGMPAGEMAPLOCK pLock);
3940	void pgmPhysReleaseInternalPageMappingLock(PVM pVM, PPGMPAGEMAPLOCK pLock);
3941	VMMDECL(int) pgmPhysHandlerRedirectToHC(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser);
3942	VMMDECL(int) pgmPhysRomWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser);
3943	int pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys);
3944	void pgmPhysInvalidRamRangeTlbs(PVM pVM);
3945	void pgmPhysInvalidatePageMapTLB(PVM pVM);
3946	void pgmPhysInvalidatePageMapTLBEntry(PVM pVM, RTGCPHYS GCPhys);
3947	PPGMRAMRANGE pgmPhysGetRangeSlow(PVM pVM, RTGCPHYS GCPhys);
3948	PPGMRAMRANGE pgmPhysGetRangeAtOrAboveSlow(PVM pVM, RTGCPHYS GCPhys);
3949	PPGMPAGE pgmPhysGetPageSlow(PVM pVM, RTGCPHYS GCPhys);
3950	int pgmPhysGetPageExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage);
3951	int pgmPhysGetPageAndRangeExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam);
3952
3953	#ifdef IN_RING3
3954	void pgmR3PhysRelinkRamRanges(PVM pVM);
3955	int pgmR3PhysRamPreAllocate(PVM pVM);
3956	int pgmR3PhysRamReset(PVM pVM);
3957	int pgmR3PhysRomReset(PVM pVM);
3958	int pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk);
3959	int pgmR3PhysRamTerm(PVM pVM);
3960	void pgmR3PhysRomTerm(PVM pVM);
3961
3962	int pgmR3PoolInit(PVM pVM);
3963	void pgmR3PoolRelocate(PVM pVM);
3964	void pgmR3PoolResetUnpluggedCpu(PVM pVM, PVMCPU pVCpu);
3965	void pgmR3PoolReset(PVM pVM);
3966	void pgmR3PoolClearAll(PVM pVM, bool fFlushRemTlb);
3967	DECLCALLBACK(VBOXSTRICTRC) pgmR3PoolClearAllRendezvous(PVM pVM, PVMCPU pVCpu, void *fpvFlushRemTbl);
3968	void pgmR3PoolWriteProtectPages(PVM pVM);
3969
3970	#endif /* IN_RING3 */
3971	#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) \|\| defined(IN_RC)
3972	int pgmRZDynMapHCPageCommon(PPGMMAPSET pSet, RTHCPHYS HCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL);
3973	int pgmRZDynMapGCPageCommon(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL);
3974	# ifdef LOG_ENABLED
3975	void pgmRZDynMapUnusedHint(PVMCPU pVCpu, void *pvHint, RT_SRC_POS_DECL);
3976	# else
3977	void pgmRZDynMapUnusedHint(PVMCPU pVCpu, void *pvHint);
3978	# endif
3979	#endif
3980	int pgmPoolAlloc(PVM pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, bool fA20Enabled,
3981	uint16_t iUser, uint32_t iUserTable, bool fLockPage, PPPGMPOOLPAGE ppPage);
3982	void pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint32_t iUserTable);
3983	void pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable);
3984	int pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fFlush = true /* DO NOT USE false UNLESS YOU KNOWN WHAT YOU'RE DOING!! */);
3985	void pgmPoolFlushPageByGCPhys(PVM pVM, RTGCPHYS GCPhys);
3986	PPGMPOOLPAGE pgmPoolGetPage(PPGMPOOL pPool, RTHCPHYS HCPhys);
3987	PPGMPOOLPAGE pgmPoolQueryPageForDbg(PPGMPOOL pPool, RTHCPHYS HCPhys);
3988	int pgmPoolSyncCR3(PVMCPU pVCpu);
3989	bool pgmPoolIsDirtyPage(PVM pVM, RTGCPHYS GCPhys);
3990	void pgmPoolInvalidateDirtyPage(PVM pVM, RTGCPHYS GCPhysPT);
3991	int pgmPoolTrackUpdateGCPhys(PVM pVM, RTGCPHYS GCPhysPage, PPGMPAGE pPhysPage, bool fFlushPTEs, bool *pfFlushTLBs);
3992	void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint, uint16_t iPte);
3993	uint16_t pgmPoolTrackPhysExtAddref(PVM pVM, PPGMPAGE pPhysPage, uint16_t u16, uint16_t iShwPT, uint16_t iPte);
3994	void pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPoolPage, PPGMPAGE pPhysPage, uint16_t iPte);
3995	void pgmPoolMonitorChainChanging(PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault, CTXTYPE(RTGCPTR, RTHCPTR, RTGCPTR) pvAddress, unsigned cbWrite);
3996	int pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
3997	void pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
3998
3999	void pgmPoolAddDirtyPage(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage);
4000	void pgmPoolResetDirtyPages(PVM pVM);
4001	void pgmPoolResetDirtyPage(PVM pVM, RTGCPTR GCPtrPage);
4002
4003	int pgmR3ExitShadowModeBeforePoolFlush(PVMCPU pVCpu);
4004	int pgmR3ReEnterShadowModeAfterPoolFlush(PVM pVM, PVMCPU pVCpu);
4005	void pgmR3RefreshShadowModeAfterA20Change(PVMCPU pVCpu);
4006
4007	void pgmMapSetShadowPDEs(PVM pVM, PPGMMAPPING pMap, unsigned iNewPDE);
4008	void pgmMapClearShadowPDEs(PVM pVM, PPGMPOOLPAGE pShwPageCR3, PPGMMAPPING pMap, unsigned iOldPDE, bool fDeactivateCR3);
4009	int pgmMapActivateCR3(PVM pVM, PPGMPOOLPAGE pShwPageCR3);
4010	int pgmMapDeactivateCR3(PVM pVM, PPGMPOOLPAGE pShwPageCR3);
4011
4012	int pgmShwSyncPaePDPtr(PVMCPU pVCpu, RTGCPTR GCPtr, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD);
4013	int pgmShwSyncNestedPageLocked(PVMCPU pVCpu, RTGCPHYS GCPhysFault, uint32_t cPages, PGMMODE enmShwPagingMode);
4014
4015	int pgmGstLazyMap32BitPD(PVMCPU pVCpu, PX86PD *ppPd);
4016	int pgmGstLazyMapPaePDPT(PVMCPU pVCpu, PX86PDPT *ppPdpt);
4017	int pgmGstLazyMapPaePD(PVMCPU pVCpu, uint32_t iPdpt, PX86PDPAE *ppPd);
4018	int pgmGstLazyMapPml4(PVMCPU pVCpu, PX86PML4 *ppPml4);
4019
4020	# if defined(VBOX_STRICT) && HC_ARCH_BITS == 64 && defined(IN_RING3)
4021	DECLCALLBACK(int) pgmR3CmdCheckDuplicatePages(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs);
4022	DECLCALLBACK(int) pgmR3CmdShowSharedModules(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs);
4023	# endif
4024
4025	RT_C_DECLS_END
4026
4027	/** @} */
4028
4029	#endif
4030

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/PGMInternal.h@ 41728

Download in other formats: