PGMAll.cpp@ 96854

Last change on this file since 96854 was 96407, checked in by vboxsync, 2 years ago
scm copyright and license note update
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 142.1 KB

Line
1	/* $Id: PGMAll.cpp 96407 2022-08-22 17:43:14Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor - All context code.
4	*/
5
6	/*
7	* Copyright (C) 2006-2022 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_PGM
33	#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */
34	#include <VBox/vmm/pgm.h>
35	#include <VBox/vmm/cpum.h>
36	#include <VBox/vmm/selm.h>
37	#include <VBox/vmm/iem.h>
38	#include <VBox/vmm/iom.h>
39	#include <VBox/sup.h>
40	#include <VBox/vmm/mm.h>
41	#include <VBox/vmm/stam.h>
42	#include <VBox/vmm/trpm.h>
43	#include <VBox/vmm/em.h>
44	#include <VBox/vmm/hm.h>
45	#include <VBox/vmm/hm_vmx.h>
46	#include "PGMInternal.h"
47	#include <VBox/vmm/vmcc.h>
48	#include "PGMInline.h"
49	#include <iprt/assert.h>
50	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
51	# include <iprt/asm-amd64-x86.h>
52	#endif
53	#include <iprt/string.h>
54	#include <VBox/log.h>
55	#include <VBox/param.h>
56	#include <VBox/err.h>
57
58
59	/*********************************************************************************************************************************
60	* Internal Functions *
61	*********************************************************************************************************************************/
62	DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPT ppPdpt, PX86PDPAE *ppPD);
63	DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde);
64	DECLINLINE(int) pgmGstMapCr3(PVMCPUCC pVCpu, RTGCPHYS GCPhysCr3, PRTHCPTR pHCPtrGuestCr3);
65	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
66	static int pgmGstSlatWalk(PVMCPUCC pVCpu, RTGCPHYS GCPhysNested, bool fIsLinearAddrValid, RTGCPTR GCPtrNested, PPGMPTWALK pWalk,
67	PPGMPTWALKGST pGstWalk);
68	static int pgmGstSlatWalkPhys(PVMCPUCC pVCpu, PGMSLAT enmSlatMode, RTGCPHYS GCPhysNested, PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk);
69	static int pgmGstSlatTranslateCr3(PVMCPUCC pVCpu, uint64_t uCr3, PRTGCPHYS pGCPhysCr3);
70	#endif
71	static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD);
72	static int pgmShwGetEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD);
73
74
75	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
76	/* Guest - EPT SLAT is identical for all guest paging mode. */
77	# define PGM_SLAT_TYPE PGM_SLAT_TYPE_EPT
78	# define PGM_GST_TYPE PGM_TYPE_EPT
79	# include "PGMGstDefs.h"
80	# include "PGMAllGstSlatEpt.cpp.h"
81	# undef PGM_GST_TYPE
82	#endif
83
84
85	/*
86	* Shadow - 32-bit mode
87	*/
88	#define PGM_SHW_TYPE PGM_TYPE_32BIT
89	#define PGM_SHW_NAME(name) PGM_SHW_NAME_32BIT(name)
90	#include "PGMAllShw.h"
91
92	/* Guest - real mode */
93	#define PGM_GST_TYPE PGM_TYPE_REAL
94	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
95	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_REAL(name)
96	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
97	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
98	#include "PGMGstDefs.h"
99	#include "PGMAllGst.h"
100	#include "PGMAllBth.h"
101	#undef BTH_PGMPOOLKIND_PT_FOR_PT
102	#undef BTH_PGMPOOLKIND_ROOT
103	#undef PGM_BTH_NAME
104	#undef PGM_GST_TYPE
105	#undef PGM_GST_NAME
106
107	/* Guest - protected mode */
108	#define PGM_GST_TYPE PGM_TYPE_PROT
109	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
110	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_PROT(name)
111	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
112	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
113	#include "PGMGstDefs.h"
114	#include "PGMAllGst.h"
115	#include "PGMAllBth.h"
116	#undef BTH_PGMPOOLKIND_PT_FOR_PT
117	#undef BTH_PGMPOOLKIND_ROOT
118	#undef PGM_BTH_NAME
119	#undef PGM_GST_TYPE
120	#undef PGM_GST_NAME
121
122	/* Guest - 32-bit mode */
123	#define PGM_GST_TYPE PGM_TYPE_32BIT
124	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
125	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_32BIT(name)
126	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT
127	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB
128	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD
129	#include "PGMGstDefs.h"
130	#include "PGMAllGst.h"
131	#include "PGMAllBth.h"
132	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
133	#undef BTH_PGMPOOLKIND_PT_FOR_PT
134	#undef BTH_PGMPOOLKIND_ROOT
135	#undef PGM_BTH_NAME
136	#undef PGM_GST_TYPE
137	#undef PGM_GST_NAME
138
139	#undef PGM_SHW_TYPE
140	#undef PGM_SHW_NAME
141
142
143	/*
144	* Shadow - PAE mode
145	*/
146	#define PGM_SHW_TYPE PGM_TYPE_PAE
147	#define PGM_SHW_NAME(name) PGM_SHW_NAME_PAE(name)
148	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
149	#include "PGMAllShw.h"
150
151	/* Guest - real mode */
152	#define PGM_GST_TYPE PGM_TYPE_REAL
153	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
154	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
155	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
156	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
157	#include "PGMGstDefs.h"
158	#include "PGMAllBth.h"
159	#undef BTH_PGMPOOLKIND_PT_FOR_PT
160	#undef BTH_PGMPOOLKIND_ROOT
161	#undef PGM_BTH_NAME
162	#undef PGM_GST_TYPE
163	#undef PGM_GST_NAME
164
165	/* Guest - protected mode */
166	#define PGM_GST_TYPE PGM_TYPE_PROT
167	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
168	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PROT(name)
169	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
170	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
171	#include "PGMGstDefs.h"
172	#include "PGMAllBth.h"
173	#undef BTH_PGMPOOLKIND_PT_FOR_PT
174	#undef BTH_PGMPOOLKIND_ROOT
175	#undef PGM_BTH_NAME
176	#undef PGM_GST_TYPE
177	#undef PGM_GST_NAME
178
179	/* Guest - 32-bit mode */
180	#define PGM_GST_TYPE PGM_TYPE_32BIT
181	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
182	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_32BIT(name)
183	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_32BIT_PT
184	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB
185	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_FOR_32BIT
186	#include "PGMGstDefs.h"
187	#include "PGMAllBth.h"
188	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
189	#undef BTH_PGMPOOLKIND_PT_FOR_PT
190	#undef BTH_PGMPOOLKIND_ROOT
191	#undef PGM_BTH_NAME
192	#undef PGM_GST_TYPE
193	#undef PGM_GST_NAME
194
195
196	/* Guest - PAE mode */
197	#define PGM_GST_TYPE PGM_TYPE_PAE
198	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
199	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PAE(name)
200	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
201	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
202	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT
203	#include "PGMGstDefs.h"
204	#include "PGMAllGst.h"
205	#include "PGMAllBth.h"
206	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
207	#undef BTH_PGMPOOLKIND_PT_FOR_PT
208	#undef BTH_PGMPOOLKIND_ROOT
209	#undef PGM_BTH_NAME
210	#undef PGM_GST_TYPE
211	#undef PGM_GST_NAME
212
213	#undef PGM_SHW_TYPE
214	#undef PGM_SHW_NAME
215
216
217	/*
218	* Shadow - AMD64 mode
219	*/
220	#define PGM_SHW_TYPE PGM_TYPE_AMD64
221	#define PGM_SHW_NAME(name) PGM_SHW_NAME_AMD64(name)
222	#include "PGMAllShw.h"
223
224	/* Guest - protected mode (only used for AMD-V nested paging in 64 bits mode) */
225	/** @todo retire this hack. */
226	#define PGM_GST_TYPE PGM_TYPE_PROT
227	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
228	#define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_PROT(name)
229	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
230	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PD_PHYS
231	#include "PGMGstDefs.h"
232	#include "PGMAllBth.h"
233	#undef BTH_PGMPOOLKIND_PT_FOR_PT
234	#undef BTH_PGMPOOLKIND_ROOT
235	#undef PGM_BTH_NAME
236	#undef PGM_GST_TYPE
237	#undef PGM_GST_NAME
238
239	#ifdef VBOX_WITH_64_BITS_GUESTS
240	/* Guest - AMD64 mode */
241	# define PGM_GST_TYPE PGM_TYPE_AMD64
242	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
243	# define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_AMD64(name)
244	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
245	# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
246	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_64BIT_PML4
247	# include "PGMGstDefs.h"
248	# include "PGMAllGst.h"
249	# include "PGMAllBth.h"
250	# undef BTH_PGMPOOLKIND_PT_FOR_BIG
251	# undef BTH_PGMPOOLKIND_PT_FOR_PT
252	# undef BTH_PGMPOOLKIND_ROOT
253	# undef PGM_BTH_NAME
254	# undef PGM_GST_TYPE
255	# undef PGM_GST_NAME
256	#endif /* VBOX_WITH_64_BITS_GUESTS */
257
258	#undef PGM_SHW_TYPE
259	#undef PGM_SHW_NAME
260
261
262	/*
263	* Shadow - 32-bit nested paging mode.
264	*/
265	#define PGM_SHW_TYPE PGM_TYPE_NESTED_32BIT
266	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_32BIT(name)
267	#include "PGMAllShw.h"
268
269	/* Guest - real mode */
270	#define PGM_GST_TYPE PGM_TYPE_REAL
271	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
272	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_REAL(name)
273	#include "PGMGstDefs.h"
274	#include "PGMAllBth.h"
275	#undef PGM_BTH_NAME
276	#undef PGM_GST_TYPE
277	#undef PGM_GST_NAME
278
279	/* Guest - protected mode */
280	#define PGM_GST_TYPE PGM_TYPE_PROT
281	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
282	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_PROT(name)
283	#include "PGMGstDefs.h"
284	#include "PGMAllBth.h"
285	#undef PGM_BTH_NAME
286	#undef PGM_GST_TYPE
287	#undef PGM_GST_NAME
288
289	/* Guest - 32-bit mode */
290	#define PGM_GST_TYPE PGM_TYPE_32BIT
291	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
292	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_32BIT(name)
293	#include "PGMGstDefs.h"
294	#include "PGMAllBth.h"
295	#undef PGM_BTH_NAME
296	#undef PGM_GST_TYPE
297	#undef PGM_GST_NAME
298
299	/* Guest - PAE mode */
300	#define PGM_GST_TYPE PGM_TYPE_PAE
301	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
302	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_PAE(name)
303	#include "PGMGstDefs.h"
304	#include "PGMAllBth.h"
305	#undef PGM_BTH_NAME
306	#undef PGM_GST_TYPE
307	#undef PGM_GST_NAME
308
309	#ifdef VBOX_WITH_64_BITS_GUESTS
310	/* Guest - AMD64 mode */
311	# define PGM_GST_TYPE PGM_TYPE_AMD64
312	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
313	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_AMD64(name)
314	# include "PGMGstDefs.h"
315	# include "PGMAllBth.h"
316	# undef PGM_BTH_NAME
317	# undef PGM_GST_TYPE
318	# undef PGM_GST_NAME
319	#endif /* VBOX_WITH_64_BITS_GUESTS */
320
321	#undef PGM_SHW_TYPE
322	#undef PGM_SHW_NAME
323
324
325	/*
326	* Shadow - PAE nested paging mode.
327	*/
328	#define PGM_SHW_TYPE PGM_TYPE_NESTED_PAE
329	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_PAE(name)
330	#include "PGMAllShw.h"
331
332	/* Guest - real mode */
333	#define PGM_GST_TYPE PGM_TYPE_REAL
334	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
335	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_REAL(name)
336	#include "PGMGstDefs.h"
337	#include "PGMAllBth.h"
338	#undef PGM_BTH_NAME
339	#undef PGM_GST_TYPE
340	#undef PGM_GST_NAME
341
342	/* Guest - protected mode */
343	#define PGM_GST_TYPE PGM_TYPE_PROT
344	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
345	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_PROT(name)
346	#include "PGMGstDefs.h"
347	#include "PGMAllBth.h"
348	#undef PGM_BTH_NAME
349	#undef PGM_GST_TYPE
350	#undef PGM_GST_NAME
351
352	/* Guest - 32-bit mode */
353	#define PGM_GST_TYPE PGM_TYPE_32BIT
354	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
355	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_32BIT(name)
356	#include "PGMGstDefs.h"
357	#include "PGMAllBth.h"
358	#undef PGM_BTH_NAME
359	#undef PGM_GST_TYPE
360	#undef PGM_GST_NAME
361
362	/* Guest - PAE mode */
363	#define PGM_GST_TYPE PGM_TYPE_PAE
364	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
365	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_PAE(name)
366	#include "PGMGstDefs.h"
367	#include "PGMAllBth.h"
368	#undef PGM_BTH_NAME
369	#undef PGM_GST_TYPE
370	#undef PGM_GST_NAME
371
372	#ifdef VBOX_WITH_64_BITS_GUESTS
373	/* Guest - AMD64 mode */
374	# define PGM_GST_TYPE PGM_TYPE_AMD64
375	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
376	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_AMD64(name)
377	# include "PGMGstDefs.h"
378	# include "PGMAllBth.h"
379	# undef PGM_BTH_NAME
380	# undef PGM_GST_TYPE
381	# undef PGM_GST_NAME
382	#endif /* VBOX_WITH_64_BITS_GUESTS */
383
384	#undef PGM_SHW_TYPE
385	#undef PGM_SHW_NAME
386
387
388	/*
389	* Shadow - AMD64 nested paging mode.
390	*/
391	#define PGM_SHW_TYPE PGM_TYPE_NESTED_AMD64
392	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_AMD64(name)
393	#include "PGMAllShw.h"
394
395	/* Guest - real mode */
396	#define PGM_GST_TYPE PGM_TYPE_REAL
397	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
398	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_REAL(name)
399	#include "PGMGstDefs.h"
400	#include "PGMAllBth.h"
401	#undef PGM_BTH_NAME
402	#undef PGM_GST_TYPE
403	#undef PGM_GST_NAME
404
405	/* Guest - protected mode */
406	#define PGM_GST_TYPE PGM_TYPE_PROT
407	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
408	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_PROT(name)
409	#include "PGMGstDefs.h"
410	#include "PGMAllBth.h"
411	#undef PGM_BTH_NAME
412	#undef PGM_GST_TYPE
413	#undef PGM_GST_NAME
414
415	/* Guest - 32-bit mode */
416	#define PGM_GST_TYPE PGM_TYPE_32BIT
417	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
418	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_32BIT(name)
419	#include "PGMGstDefs.h"
420	#include "PGMAllBth.h"
421	#undef PGM_BTH_NAME
422	#undef PGM_GST_TYPE
423	#undef PGM_GST_NAME
424
425	/* Guest - PAE mode */
426	#define PGM_GST_TYPE PGM_TYPE_PAE
427	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
428	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_PAE(name)
429	#include "PGMGstDefs.h"
430	#include "PGMAllBth.h"
431	#undef PGM_BTH_NAME
432	#undef PGM_GST_TYPE
433	#undef PGM_GST_NAME
434
435	#ifdef VBOX_WITH_64_BITS_GUESTS
436	/* Guest - AMD64 mode */
437	# define PGM_GST_TYPE PGM_TYPE_AMD64
438	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
439	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_AMD64(name)
440	# include "PGMGstDefs.h"
441	# include "PGMAllBth.h"
442	# undef PGM_BTH_NAME
443	# undef PGM_GST_TYPE
444	# undef PGM_GST_NAME
445	#endif /* VBOX_WITH_64_BITS_GUESTS */
446
447	#undef PGM_SHW_TYPE
448	#undef PGM_SHW_NAME
449
450
451	/*
452	* Shadow - EPT.
453	*/
454	#define PGM_SHW_TYPE PGM_TYPE_EPT
455	#define PGM_SHW_NAME(name) PGM_SHW_NAME_EPT(name)
456	#include "PGMAllShw.h"
457
458	/* Guest - real mode */
459	#define PGM_GST_TYPE PGM_TYPE_REAL
460	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
461	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_REAL(name)
462	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
463	#include "PGMGstDefs.h"
464	#include "PGMAllBth.h"
465	#undef BTH_PGMPOOLKIND_PT_FOR_PT
466	#undef PGM_BTH_NAME
467	#undef PGM_GST_TYPE
468	#undef PGM_GST_NAME
469
470	/* Guest - protected mode */
471	#define PGM_GST_TYPE PGM_TYPE_PROT
472	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
473	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PROT(name)
474	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
475	#include "PGMGstDefs.h"
476	#include "PGMAllBth.h"
477	#undef BTH_PGMPOOLKIND_PT_FOR_PT
478	#undef PGM_BTH_NAME
479	#undef PGM_GST_TYPE
480	#undef PGM_GST_NAME
481
482	/* Guest - 32-bit mode */
483	#define PGM_GST_TYPE PGM_TYPE_32BIT
484	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
485	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_32BIT(name)
486	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
487	#include "PGMGstDefs.h"
488	#include "PGMAllBth.h"
489	#undef BTH_PGMPOOLKIND_PT_FOR_PT
490	#undef PGM_BTH_NAME
491	#undef PGM_GST_TYPE
492	#undef PGM_GST_NAME
493
494	/* Guest - PAE mode */
495	#define PGM_GST_TYPE PGM_TYPE_PAE
496	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
497	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PAE(name)
498	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
499	#include "PGMGstDefs.h"
500	#include "PGMAllBth.h"
501	#undef BTH_PGMPOOLKIND_PT_FOR_PT
502	#undef PGM_BTH_NAME
503	#undef PGM_GST_TYPE
504	#undef PGM_GST_NAME
505
506	#ifdef VBOX_WITH_64_BITS_GUESTS
507	/* Guest - AMD64 mode */
508	# define PGM_GST_TYPE PGM_TYPE_AMD64
509	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
510	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_AMD64(name)
511	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
512	# include "PGMGstDefs.h"
513	# include "PGMAllBth.h"
514	# undef BTH_PGMPOOLKIND_PT_FOR_PT
515	# undef PGM_BTH_NAME
516	# undef PGM_GST_TYPE
517	# undef PGM_GST_NAME
518	#endif /* VBOX_WITH_64_BITS_GUESTS */
519
520	#undef PGM_SHW_TYPE
521	#undef PGM_SHW_NAME
522
523
524	/*
525	* Shadow - NEM / None.
526	*/
527	#define PGM_SHW_TYPE PGM_TYPE_NONE
528	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NONE(name)
529	#include "PGMAllShw.h"
530
531	/* Guest - real mode */
532	#define PGM_GST_TYPE PGM_TYPE_REAL
533	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
534	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_REAL(name)
535	#include "PGMGstDefs.h"
536	#include "PGMAllBth.h"
537	#undef PGM_BTH_NAME
538	#undef PGM_GST_TYPE
539	#undef PGM_GST_NAME
540
541	/* Guest - protected mode */
542	#define PGM_GST_TYPE PGM_TYPE_PROT
543	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
544	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_PROT(name)
545	#include "PGMGstDefs.h"
546	#include "PGMAllBth.h"
547	#undef PGM_BTH_NAME
548	#undef PGM_GST_TYPE
549	#undef PGM_GST_NAME
550
551	/* Guest - 32-bit mode */
552	#define PGM_GST_TYPE PGM_TYPE_32BIT
553	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
554	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_32BIT(name)
555	#include "PGMGstDefs.h"
556	#include "PGMAllBth.h"
557	#undef PGM_BTH_NAME
558	#undef PGM_GST_TYPE
559	#undef PGM_GST_NAME
560
561	/* Guest - PAE mode */
562	#define PGM_GST_TYPE PGM_TYPE_PAE
563	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
564	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_PAE(name)
565	#include "PGMGstDefs.h"
566	#include "PGMAllBth.h"
567	#undef PGM_BTH_NAME
568	#undef PGM_GST_TYPE
569	#undef PGM_GST_NAME
570
571	#ifdef VBOX_WITH_64_BITS_GUESTS
572	/* Guest - AMD64 mode */
573	# define PGM_GST_TYPE PGM_TYPE_AMD64
574	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
575	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_AMD64(name)
576	# include "PGMGstDefs.h"
577	# include "PGMAllBth.h"
578	# undef PGM_BTH_NAME
579	# undef PGM_GST_TYPE
580	# undef PGM_GST_NAME
581	#endif /* VBOX_WITH_64_BITS_GUESTS */
582
583	#undef PGM_SHW_TYPE
584	#undef PGM_SHW_NAME
585
586
587
588	/**
589	* Guest mode data array.
590	*/
591	PGMMODEDATAGST const g_aPgmGuestModeData[PGM_GUEST_MODE_DATA_ARRAY_SIZE] =
592	{
593	{ UINT32_MAX, NULL, NULL, NULL, NULL }, /* 0 */
594	{
595	PGM_TYPE_REAL,
596	PGM_GST_NAME_REAL(GetPage),
597	PGM_GST_NAME_REAL(ModifyPage),
598	PGM_GST_NAME_REAL(Enter),
599	PGM_GST_NAME_REAL(Exit),
600	#ifdef IN_RING3
601	PGM_GST_NAME_REAL(Relocate),
602	#endif
603	},
604	{
605	PGM_TYPE_PROT,
606	PGM_GST_NAME_PROT(GetPage),
607	PGM_GST_NAME_PROT(ModifyPage),
608	PGM_GST_NAME_PROT(Enter),
609	PGM_GST_NAME_PROT(Exit),
610	#ifdef IN_RING3
611	PGM_GST_NAME_PROT(Relocate),
612	#endif
613	},
614	{
615	PGM_TYPE_32BIT,
616	PGM_GST_NAME_32BIT(GetPage),
617	PGM_GST_NAME_32BIT(ModifyPage),
618	PGM_GST_NAME_32BIT(Enter),
619	PGM_GST_NAME_32BIT(Exit),
620	#ifdef IN_RING3
621	PGM_GST_NAME_32BIT(Relocate),
622	#endif
623	},
624	{
625	PGM_TYPE_PAE,
626	PGM_GST_NAME_PAE(GetPage),
627	PGM_GST_NAME_PAE(ModifyPage),
628	PGM_GST_NAME_PAE(Enter),
629	PGM_GST_NAME_PAE(Exit),
630	#ifdef IN_RING3
631	PGM_GST_NAME_PAE(Relocate),
632	#endif
633	},
634	#ifdef VBOX_WITH_64_BITS_GUESTS
635	{
636	PGM_TYPE_AMD64,
637	PGM_GST_NAME_AMD64(GetPage),
638	PGM_GST_NAME_AMD64(ModifyPage),
639	PGM_GST_NAME_AMD64(Enter),
640	PGM_GST_NAME_AMD64(Exit),
641	# ifdef IN_RING3
642	PGM_GST_NAME_AMD64(Relocate),
643	# endif
644	},
645	#endif
646	};
647
648
649	/**
650	* The shadow mode data array.
651	*/
652	PGMMODEDATASHW const g_aPgmShadowModeData[PGM_SHADOW_MODE_DATA_ARRAY_SIZE] =
653	{
654	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* 0 */
655	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* PGM_TYPE_REAL */
656	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* PGM_TYPE_PROT */
657	{
658	PGM_TYPE_32BIT,
659	PGM_SHW_NAME_32BIT(GetPage),
660	PGM_SHW_NAME_32BIT(ModifyPage),
661	PGM_SHW_NAME_32BIT(Enter),
662	PGM_SHW_NAME_32BIT(Exit),
663	#ifdef IN_RING3
664	PGM_SHW_NAME_32BIT(Relocate),
665	#endif
666	},
667	{
668	PGM_TYPE_PAE,
669	PGM_SHW_NAME_PAE(GetPage),
670	PGM_SHW_NAME_PAE(ModifyPage),
671	PGM_SHW_NAME_PAE(Enter),
672	PGM_SHW_NAME_PAE(Exit),
673	#ifdef IN_RING3
674	PGM_SHW_NAME_PAE(Relocate),
675	#endif
676	},
677	{
678	PGM_TYPE_AMD64,
679	PGM_SHW_NAME_AMD64(GetPage),
680	PGM_SHW_NAME_AMD64(ModifyPage),
681	PGM_SHW_NAME_AMD64(Enter),
682	PGM_SHW_NAME_AMD64(Exit),
683	#ifdef IN_RING3
684	PGM_SHW_NAME_AMD64(Relocate),
685	#endif
686	},
687	{
688	PGM_TYPE_NESTED_32BIT,
689	PGM_SHW_NAME_NESTED_32BIT(GetPage),
690	PGM_SHW_NAME_NESTED_32BIT(ModifyPage),
691	PGM_SHW_NAME_NESTED_32BIT(Enter),
692	PGM_SHW_NAME_NESTED_32BIT(Exit),
693	#ifdef IN_RING3
694	PGM_SHW_NAME_NESTED_32BIT(Relocate),
695	#endif
696	},
697	{
698	PGM_TYPE_NESTED_PAE,
699	PGM_SHW_NAME_NESTED_PAE(GetPage),
700	PGM_SHW_NAME_NESTED_PAE(ModifyPage),
701	PGM_SHW_NAME_NESTED_PAE(Enter),
702	PGM_SHW_NAME_NESTED_PAE(Exit),
703	#ifdef IN_RING3
704	PGM_SHW_NAME_NESTED_PAE(Relocate),
705	#endif
706	},
707	{
708	PGM_TYPE_NESTED_AMD64,
709	PGM_SHW_NAME_NESTED_AMD64(GetPage),
710	PGM_SHW_NAME_NESTED_AMD64(ModifyPage),
711	PGM_SHW_NAME_NESTED_AMD64(Enter),
712	PGM_SHW_NAME_NESTED_AMD64(Exit),
713	#ifdef IN_RING3
714	PGM_SHW_NAME_NESTED_AMD64(Relocate),
715	#endif
716	},
717	{
718	PGM_TYPE_EPT,
719	PGM_SHW_NAME_EPT(GetPage),
720	PGM_SHW_NAME_EPT(ModifyPage),
721	PGM_SHW_NAME_EPT(Enter),
722	PGM_SHW_NAME_EPT(Exit),
723	#ifdef IN_RING3
724	PGM_SHW_NAME_EPT(Relocate),
725	#endif
726	},
727	{
728	PGM_TYPE_NONE,
729	PGM_SHW_NAME_NONE(GetPage),
730	PGM_SHW_NAME_NONE(ModifyPage),
731	PGM_SHW_NAME_NONE(Enter),
732	PGM_SHW_NAME_NONE(Exit),
733	#ifdef IN_RING3
734	PGM_SHW_NAME_NONE(Relocate),
735	#endif
736	},
737	};
738
739
740	/**
741	* The guest+shadow mode data array.
742	*/
743	PGMMODEDATABTH const g_aPgmBothModeData[PGM_BOTH_MODE_DATA_ARRAY_SIZE] =
744	{
745	#if !defined(IN_RING3) && !defined(VBOX_STRICT)
746	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
747	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
748	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Trap0eHandler), Nm(NestedTrap0eHandler) }
749
750	#elif !defined(IN_RING3) && defined(VBOX_STRICT)
751	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
752	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
753	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Trap0eHandler), Nm(NestedTrap0eHandler), Nm(AssertCR3) }
754
755	#elif defined(IN_RING3) && !defined(VBOX_STRICT)
756	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL }
757	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
758	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), }
759
760	#elif defined(IN_RING3) && defined(VBOX_STRICT)
761	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
762	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
763	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(AssertCR3) }
764
765	#else
766	# error "Misconfig."
767	#endif
768
769	/* 32-bit shadow paging mode: */
770	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
771	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_REAL, PGM_BTH_NAME_32BIT_REAL),
772	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_PROT, PGM_BTH_NAME_32BIT_PROT),
773	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_32BIT_32BIT),
774	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_PAE - illegal */
775	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_AMD64 - illegal */
776	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_32BIT - illegal */
777	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_PAE - illegal */
778	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */
779	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_EPT - illegal */
780	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NONE - illegal */
781
782	/* PAE shadow paging mode: */
783	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
784	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_REAL, PGM_BTH_NAME_PAE_REAL),
785	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PROT, PGM_BTH_NAME_PAE_PROT),
786	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_PAE_32BIT),
787	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PAE, PGM_BTH_NAME_PAE_PAE),
788	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_AMD64 - illegal */
789	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_32BIT - illegal */
790	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_PAE - illegal */
791	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_AMD64 - illegal */
792	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_EPT - illegal */
793	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NONE - illegal */
794
795	/* AMD64 shadow paging mode: */
796	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
797	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_REAL, PGM_BTH_NAME_AMD64_REAL),
798	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PROT, PGM_BTH_NAME_AMD64_PROT),
799	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_AMD64_32BIT),
800	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PAE, PGM_BTH_NAME_AMD64_PAE),
801	#ifdef VBOX_WITH_64_BITS_GUESTS
802	PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_AMD64_AMD64),
803	#else
804	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_AMD64 - illegal */
805	#endif
806	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_32BIT - illegal */
807	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_PAE - illegal */
808	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */
809	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_EPT - illegal */
810	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NONE - illegal */
811
812	/* 32-bit nested paging mode: */
813	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
814	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_32BIT_REAL),
815	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_32BIT_PROT),
816	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_32BIT_32BIT),
817	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_32BIT_PAE),
818	#ifdef VBOX_WITH_64_BITS_GUESTS
819	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_32BIT_AMD64),
820	#else
821	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64 - illegal */
822	#endif
823	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_32BIT - illegal */
824	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_PAE - illegal */
825	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */
826	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_EPT - illegal */
827	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NONE - illegal */
828
829	/* PAE nested paging mode: */
830	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
831	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_PAE_REAL),
832	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_PAE_PROT),
833	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_PAE_32BIT),
834	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_PAE_PAE),
835	#ifdef VBOX_WITH_64_BITS_GUESTS
836	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_PAE_AMD64),
837	#else
838	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64 - illegal */
839	#endif
840	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_32BIT - illegal */
841	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_PAE - illegal */
842	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_AMD64 - illegal */
843	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_EPT - illegal */
844	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NONE - illegal */
845
846	/* AMD64 nested paging mode: */
847	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
848	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_AMD64_REAL),
849	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_AMD64_PROT),
850	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_AMD64_32BIT),
851	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_AMD64_PAE),
852	#ifdef VBOX_WITH_64_BITS_GUESTS
853	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_AMD64_AMD64),
854	#else
855	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64 - illegal */
856	#endif
857	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_32BIT - illegal */
858	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_PAE - illegal */
859	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */
860	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_EPT - illegal */
861	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NONE - illegal */
862
863	/* EPT nested paging mode: */
864	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
865	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_REAL, PGM_BTH_NAME_EPT_REAL),
866	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PROT, PGM_BTH_NAME_EPT_PROT),
867	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT),
868	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PAE, PGM_BTH_NAME_EPT_PAE),
869	#ifdef VBOX_WITH_64_BITS_GUESTS
870	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64),
871	#else
872	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_AMD64 - illegal */
873	#endif
874	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_32BIT - illegal */
875	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_PAE - illegal */
876	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_AMD64 - illegal */
877	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_EPT - illegal */
878	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NONE - illegal */
879
880	/* NONE / NEM: */
881	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
882	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_REAL, PGM_BTH_NAME_EPT_REAL),
883	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_PROT, PGM_BTH_NAME_EPT_PROT),
884	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT),
885	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_PAE, PGM_BTH_NAME_EPT_PAE),
886	#ifdef VBOX_WITH_64_BITS_GUESTS
887	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64),
888	#else
889	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_AMD64 - illegal */
890	#endif
891	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_32BIT - illegal */
892	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_PAE - illegal */
893	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_AMD64 - illegal */
894	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_EPT - illegal */
895	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NONE - illegal */
896
897
898	#undef PGMMODEDATABTH_ENTRY
899	#undef PGMMODEDATABTH_NULL_ENTRY
900	};
901
902
903	#ifdef IN_RING0
904	/**
905	* #PF Handler.
906	*
907	* @returns VBox status code (appropriate for trap handling and GC return).
908	* @param pVCpu The cross context virtual CPU structure.
909	* @param uErr The trap error code.
910	* @param pRegFrame Trap register frame.
911	* @param pvFault The fault address.
912	*/
913	VMMDECL(int) PGMTrap0eHandler(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
914	{
915	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
916
917	Log(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv eip=%04x:%RGv cr3=%RGp\n", uErr, pvFault, pRegFrame->cs.Sel, (RTGCPTR)pRegFrame->rip, (RTGCPHYS)CPUMGetGuestCR3(pVCpu)));
918	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.StatRZTrap0e, a);
919	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = NULL; } );
920
921
922	# ifdef VBOX_WITH_STATISTICS
923	/*
924	* Error code stats.
925	*/
926	if (uErr & X86_TRAP_PF_US)
927	{
928	if (!(uErr & X86_TRAP_PF_P))
929	{
930	if (uErr & X86_TRAP_PF_RW)
931	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNotPresentWrite);
932	else
933	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNotPresentRead);
934	}
935	else if (uErr & X86_TRAP_PF_RW)
936	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSWrite);
937	else if (uErr & X86_TRAP_PF_RSVD)
938	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSReserved);
939	else if (uErr & X86_TRAP_PF_ID)
940	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNXE);
941	else
942	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSRead);
943	}
944	else
945	{ /* Supervisor */
946	if (!(uErr & X86_TRAP_PF_P))
947	{
948	if (uErr & X86_TRAP_PF_RW)
949	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVNotPresentWrite);
950	else
951	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVNotPresentRead);
952	}
953	else if (uErr & X86_TRAP_PF_RW)
954	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVWrite);
955	else if (uErr & X86_TRAP_PF_ID)
956	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSNXE);
957	else if (uErr & X86_TRAP_PF_RSVD)
958	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVReserved);
959	}
960	# endif /* VBOX_WITH_STATISTICS */
961
962	/*
963	* Call the worker.
964	*/
965	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
966	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
967	AssertReturn(g_aPgmBothModeData[idxBth].pfnTrap0eHandler, VERR_PGM_MODE_IPE);
968	bool fLockTaken = false;
969	int rc = g_aPgmBothModeData[idxBth].pfnTrap0eHandler(pVCpu, uErr, pRegFrame, pvFault, &fLockTaken);
970	if (fLockTaken)
971	{
972	PGM_LOCK_ASSERT_OWNER(pVM);
973	PGM_UNLOCK(pVM);
974	}
975	LogFlow(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv rc=%Rrc\n", uErr, pvFault, rc));
976
977	/*
978	* Return code tweaks.
979	*/
980	if (rc != VINF_SUCCESS)
981	{
982	if (rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
983	rc = VINF_SUCCESS;
984
985	/* Note: hack alert for difficult to reproduce problem. */
986	if ( rc == VERR_PAGE_NOT_PRESENT /* SMP only ; disassembly might fail. */
987	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT /* seen with UNI & SMP */
988	\|\| rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT /* seen with SMP */
989	\|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT) /* precaution */
990	{
991	Log(("WARNING: Unexpected VERR_PAGE_TABLE_NOT_PRESENT (%d) for page fault at %RGv error code %x (rip=%RGv)\n", rc, pvFault, uErr, pRegFrame->rip));
992	/* Some kind of inconsistency in the SMP case; it's safe to just execute the instruction again; not sure about single VCPU VMs though. */
993	rc = VINF_SUCCESS;
994	}
995	}
996
997	STAM_STATS({ if (rc == VINF_EM_RAW_GUEST_TRAP) STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eGuestPF); });
998	STAM_STATS({ if (!pVCpu->pgmr0.s.pStatTrap0eAttributionR0)
999	pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Misc; });
1000	STAM_PROFILE_STOP_EX(&pVCpu->pgm.s.Stats.StatRZTrap0e, pVCpu->pgmr0.s.pStatTrap0eAttributionR0, a);
1001	return rc;
1002	}
1003	#endif /* IN_RING0 */
1004
1005
1006	/**
1007	* Prefetch a page
1008	*
1009	* Typically used to sync commonly used pages before entering raw mode
1010	* after a CR3 reload.
1011	*
1012	* @returns VBox status code suitable for scheduling.
1013	* @retval VINF_SUCCESS on success.
1014	* @retval VINF_PGM_SYNC_CR3 if we're out of shadow pages or something like that.
1015	* @param pVCpu The cross context virtual CPU structure.
1016	* @param GCPtrPage Page to invalidate.
1017	*/
1018	VMMDECL(int) PGMPrefetchPage(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
1019	{
1020	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,Prefetch), a);
1021
1022	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1023	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
1024	AssertReturn(g_aPgmBothModeData[idxBth].pfnPrefetchPage, VERR_PGM_MODE_IPE);
1025	int rc = g_aPgmBothModeData[idxBth].pfnPrefetchPage(pVCpu, GCPtrPage);
1026
1027	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,Prefetch), a);
1028	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("rc=%Rrc\n", rc));
1029	return rc;
1030	}
1031
1032
1033	/**
1034	* Emulation of the invlpg instruction (HC only actually).
1035	*
1036	* @returns Strict VBox status code, special care required.
1037	* @retval VINF_PGM_SYNC_CR3 - handled.
1038	* @retval VINF_EM_RAW_EMULATE_INSTR - not handled (RC only).
1039	* @retval VERR_REM_FLUSHED_PAGES_OVERFLOW - not handled.
1040	*
1041	* @param pVCpu The cross context virtual CPU structure.
1042	* @param GCPtrPage Page to invalidate.
1043	*
1044	* @remark ASSUMES the page table entry or page directory is valid. Fairly
1045	* safe, but there could be edge cases!
1046	*
1047	* @todo Flush page or page directory only if necessary!
1048	* @todo VBOXSTRICTRC
1049	*/
1050	VMMDECL(int) PGMInvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
1051	{
1052	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1053	int rc;
1054	Log3(("PGMInvalidatePage: GCPtrPage=%RGv\n", GCPtrPage));
1055
1056	IEMTlbInvalidatePage(pVCpu, GCPtrPage);
1057
1058	/*
1059	* Call paging mode specific worker.
1060	*/
1061	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage), a);
1062	PGM_LOCK_VOID(pVM);
1063
1064	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1065	AssertReturnStmt(idxBth < RT_ELEMENTS(g_aPgmBothModeData), PGM_UNLOCK(pVM), VERR_PGM_MODE_IPE);
1066	AssertReturnStmt(g_aPgmBothModeData[idxBth].pfnInvalidatePage, PGM_UNLOCK(pVM), VERR_PGM_MODE_IPE);
1067	rc = g_aPgmBothModeData[idxBth].pfnInvalidatePage(pVCpu, GCPtrPage);
1068
1069	PGM_UNLOCK(pVM);
1070	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage), a);
1071
1072	/* Ignore all irrelevant error codes. */
1073	if ( rc == VERR_PAGE_NOT_PRESENT
1074	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
1075	\|\| rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT
1076	\|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT)
1077	rc = VINF_SUCCESS;
1078
1079	return rc;
1080	}
1081
1082
1083	/**
1084	* Executes an instruction using the interpreter.
1085	*
1086	* @returns VBox status code (appropriate for trap handling and GC return).
1087	* @param pVM The cross context VM structure.
1088	* @param pVCpu The cross context virtual CPU structure.
1089	* @param pRegFrame Register frame.
1090	* @param pvFault Fault address.
1091	*/
1092	VMMDECL(VBOXSTRICTRC) PGMInterpretInstruction(PVMCC pVM, PVMCPUCC pVCpu, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
1093	{
1094	NOREF(pVM);
1095	VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu, pRegFrame, pvFault);
1096	if (rc == VERR_EM_INTERPRETER)
1097	rc = VINF_EM_RAW_EMULATE_INSTR;
1098	if (rc != VINF_SUCCESS)
1099	Log(("PGMInterpretInstruction: returns %Rrc (pvFault=%RGv)\n", VBOXSTRICTRC_VAL(rc), pvFault));
1100	return rc;
1101	}
1102
1103
1104	/**
1105	* Gets effective page information (from the VMM page directory).
1106	*
1107	* @returns VBox status code.
1108	* @param pVCpu The cross context virtual CPU structure.
1109	* @param GCPtr Guest Context virtual address of the page.
1110	* @param pfFlags Where to store the flags. These are X86_PTE_*.
1111	* @param pHCPhys Where to store the HC physical address of the page.
1112	* This is page aligned.
1113	* @remark You should use PGMMapGetPage() for pages in a mapping.
1114	*/
1115	VMMDECL(int) PGMShwGetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys)
1116	{
1117	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1118	PGM_LOCK_VOID(pVM);
1119
1120	uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
1121	AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
1122	AssertReturn(g_aPgmShadowModeData[idxShw].pfnGetPage, VERR_PGM_MODE_IPE);
1123	int rc = g_aPgmShadowModeData[idxShw].pfnGetPage(pVCpu, GCPtr, pfFlags, pHCPhys);
1124
1125	PGM_UNLOCK(pVM);
1126	return rc;
1127	}
1128
1129
1130	/**
1131	* Modify page flags for a range of pages in the shadow context.
1132	*
1133	* The existing flags are ANDed with the fMask and ORed with the fFlags.
1134	*
1135	* @returns VBox status code.
1136	* @param pVCpu The cross context virtual CPU structure.
1137	* @param GCPtr Virtual address of the first page in the range.
1138	* @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course.
1139	* @param fMask The AND mask - page flags X86_PTE_*.
1140	* Be very CAREFUL when ~'ing constants which could be 32-bit!
1141	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1142	* @remark You must use PGMMapModifyPage() for pages in a mapping.
1143	*/
1144	DECLINLINE(int) pdmShwModifyPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags)
1145	{
1146	AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
1147	Assert(!(fOpFlags & ~(PGM_MK_PG_IS_MMIO2 \| PGM_MK_PG_IS_WRITE_FAULT)));
1148
1149	GCPtr &= ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK; /** @todo this ain't necessary, right... */
1150
1151	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1152	PGM_LOCK_VOID(pVM);
1153
1154	uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
1155	AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
1156	AssertReturn(g_aPgmShadowModeData[idxShw].pfnModifyPage, VERR_PGM_MODE_IPE);
1157	int rc = g_aPgmShadowModeData[idxShw].pfnModifyPage(pVCpu, GCPtr, GUEST_PAGE_SIZE, fFlags, fMask, fOpFlags);
1158
1159	PGM_UNLOCK(pVM);
1160	return rc;
1161	}
1162
1163
1164	/**
1165	* Changing the page flags for a single page in the shadow page tables so as to
1166	* make it read-only.
1167	*
1168	* @returns VBox status code.
1169	* @param pVCpu The cross context virtual CPU structure.
1170	* @param GCPtr Virtual address of the first page in the range.
1171	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1172	*/
1173	VMMDECL(int) PGMShwMakePageReadonly(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1174	{
1175	return pdmShwModifyPage(pVCpu, GCPtr, 0, ~(uint64_t)X86_PTE_RW, fOpFlags);
1176	}
1177
1178
1179	/**
1180	* Changing the page flags for a single page in the shadow page tables so as to
1181	* make it writable.
1182	*
1183	* The call must know with 101% certainty that the guest page tables maps this
1184	* as writable too. This function will deal shared, zero and write monitored
1185	* pages.
1186	*
1187	* @returns VBox status code.
1188	* @param pVCpu The cross context virtual CPU structure.
1189	* @param GCPtr Virtual address of the first page in the range.
1190	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1191	*/
1192	VMMDECL(int) PGMShwMakePageWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1193	{
1194	if (pVCpu->pgm.s.enmShadowMode != PGMMODE_NONE) /* avoid assertions */
1195	return pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)0, fOpFlags);
1196	return VINF_SUCCESS;
1197	}
1198
1199
1200	/**
1201	* Changing the page flags for a single page in the shadow page tables so as to
1202	* make it not present.
1203	*
1204	* @returns VBox status code.
1205	* @param pVCpu The cross context virtual CPU structure.
1206	* @param GCPtr Virtual address of the first page in the range.
1207	* @param fOpFlags A combination of the PGM_MK_PG_XXX flags.
1208	*/
1209	VMMDECL(int) PGMShwMakePageNotPresent(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1210	{
1211	return pdmShwModifyPage(pVCpu, GCPtr, 0, 0, fOpFlags);
1212	}
1213
1214
1215	/**
1216	* Changing the page flags for a single page in the shadow page tables so as to
1217	* make it supervisor and writable.
1218	*
1219	* This if for dealing with CR0.WP=0 and readonly user pages.
1220	*
1221	* @returns VBox status code.
1222	* @param pVCpu The cross context virtual CPU structure.
1223	* @param GCPtr Virtual address of the first page in the range.
1224	* @param fBigPage Whether or not this is a big page. If it is, we have to
1225	* change the shadow PDE as well. If it isn't, the caller
1226	* has checked that the shadow PDE doesn't need changing.
1227	* We ASSUME 4KB pages backing the big page here!
1228	* @param fOpFlags A combination of the PGM_MK_PG_XXX flags.
1229	*/
1230	int pgmShwMakePageSupervisorAndWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, bool fBigPage, uint32_t fOpFlags)
1231	{
1232	int rc = pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)X86_PTE_US, fOpFlags);
1233	if (rc == VINF_SUCCESS && fBigPage)
1234	{
1235	/* this is a bit ugly... */
1236	switch (pVCpu->pgm.s.enmShadowMode)
1237	{
1238	case PGMMODE_32_BIT:
1239	{
1240	PX86PDE pPde = pgmShwGet32BitPDEPtr(pVCpu, GCPtr);
1241	AssertReturn(pPde, VERR_INTERNAL_ERROR_3);
1242	Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u));
1243	pPde->u \|= X86_PDE_RW;
1244	Log(("-> PDE=%#llx (32)\n", pPde->u));
1245	break;
1246	}
1247	case PGMMODE_PAE:
1248	case PGMMODE_PAE_NX:
1249	{
1250	PX86PDEPAE pPde = pgmShwGetPaePDEPtr(pVCpu, GCPtr);
1251	AssertReturn(pPde, VERR_INTERNAL_ERROR_3);
1252	Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u));
1253	pPde->u \|= X86_PDE_RW;
1254	Log(("-> PDE=%#llx (PAE)\n", pPde->u));
1255	break;
1256	}
1257	default:
1258	AssertFailedReturn(VERR_INTERNAL_ERROR_4);
1259	}
1260	}
1261	return rc;
1262	}
1263
1264
1265	/**
1266	* Gets the shadow page directory for the specified address, PAE.
1267	*
1268	* @returns Pointer to the shadow PD.
1269	* @param pVCpu The cross context virtual CPU structure.
1270	* @param GCPtr The address.
1271	* @param uGstPdpe Guest PDPT entry. Valid.
1272	* @param ppPD Receives address of page directory
1273	*/
1274	int pgmShwSyncPaePDPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD)
1275	{
1276	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1277	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1278	PPGMPOOLPAGE pShwPage;
1279	int rc;
1280	PGM_LOCK_ASSERT_OWNER(pVM);
1281
1282
1283	/* Allocate page directory if not present. */
1284	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
1285	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
1286	PX86PDPE pPdpe = &pPdpt->a[iPdPt];
1287	X86PGPAEUINT const uPdpe = pPdpe->u;
1288	if (uPdpe & (X86_PDPE_P \| X86_PDPE_PG_MASK))
1289	{
1290	pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
1291	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1292	Assert((pPdpe->u & X86_PDPE_PG_MASK) == pShwPage->Core.Key);
1293
1294	pgmPoolCacheUsed(pPool, pShwPage);
1295
1296	/* Update the entry if necessary. */
1297	X86PGPAEUINT const uPdpeNew = pShwPage->Core.Key \| (uGstPdpe & (X86_PDPE_P \| X86_PDPE_A)) \| (uPdpe & PGM_PDPT_FLAGS);
1298	if (uPdpeNew == uPdpe)
1299	{ /* likely */ }
1300	else
1301	ASMAtomicWriteU64(&pPdpe->u, uPdpeNew);
1302	}
1303	else
1304	{
1305	RTGCPTR64 GCPdPt;
1306	PGMPOOLKIND enmKind;
1307	if (pVM->pgm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabled(pVCpu))
1308	{
1309	/* AMD-V nested paging or real/protected mode without paging. */
1310	GCPdPt = GCPtr & ~(RT_BIT_64(X86_PDPT_SHIFT) - 1);
1311	enmKind = PGMPOOLKIND_PAE_PD_PHYS;
1312	}
1313	else if (CPUMGetGuestCR4(pVCpu) & X86_CR4_PAE)
1314	{
1315	if (uGstPdpe & X86_PDPE_P)
1316	{
1317	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1318	enmKind = PGMPOOLKIND_PAE_PD_FOR_PAE_PD;
1319	}
1320	else
1321	{
1322	/* PD not present; guest must reload CR3 to change it.
1323	* No need to monitor anything in this case. */
1324	/** @todo r=bird: WTF is hit?!? */
1325	/Assert(VM_IS_RAW_MODE_ENABLED(pVM)); - ??? /
1326	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1327	enmKind = PGMPOOLKIND_PAE_PD_PHYS;
1328	Assert(uGstPdpe & X86_PDPE_P); /* caller should do this already */
1329	}
1330	}
1331	else
1332	{
1333	GCPdPt = CPUMGetGuestCR3(pVCpu);
1334	enmKind = (PGMPOOLKIND)(PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD + iPdPt);
1335	}
1336
1337	/* Create a reference back to the PDPT by using the index in its shadow page. */
1338	rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1339	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPdPt, false /fLockPage/,
1340	&pShwPage);
1341	AssertRCReturn(rc, rc);
1342
1343	/* Hook it up. */
1344	ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key \| (uGstPdpe & (X86_PDPE_P \| X86_PDPE_A)) \| (uPdpe & PGM_PDPT_FLAGS));
1345	}
1346	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdpe);
1347
1348	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1349	return VINF_SUCCESS;
1350	}
1351
1352
1353	/**
1354	* Gets the pointer to the shadow page directory entry for an address, PAE.
1355	*
1356	* @returns Pointer to the PDE.
1357	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1358	* @param GCPtr The address.
1359	* @param ppShwPde Receives the address of the pgm pool page for the shadow page directory
1360	*/
1361	DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde)
1362	{
1363	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1364	PGM_LOCK_ASSERT_OWNER(pVM);
1365
1366	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
1367	AssertReturn(pPdpt, VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT); /* can't happen */
1368	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
1369	X86PGPAEUINT const uPdpe = pPdpt->a[iPdPt].u;
1370	if (!(uPdpe & X86_PDPE_P))
1371	{
1372	LogFlow(("pgmShwGetPaePoolPagePD: PD %d not present (%RX64)\n", iPdPt, uPdpe));
1373	return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
1374	}
1375	AssertMsg(uPdpe & X86_PDPE_PG_MASK, ("GCPtr=%RGv\n", GCPtr));
1376
1377	/* Fetch the pgm pool shadow descriptor. */
1378	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pVM->pgm.s.CTX_SUFF(pPool), uPdpe & X86_PDPE_PG_MASK);
1379	AssertReturn(pShwPde, VERR_PGM_POOL_GET_PAGE_FAILED);
1380
1381	*ppShwPde = pShwPde;
1382	return VINF_SUCCESS;
1383	}
1384
1385
1386	/**
1387	* Syncs the SHADOW page directory pointer for the specified address.
1388	*
1389	* Allocates backing pages in case the PDPT or PML4 entry is missing.
1390	*
1391	* The caller is responsible for making sure the guest has a valid PD before
1392	* calling this function.
1393	*
1394	* @returns VBox status code.
1395	* @param pVCpu The cross context virtual CPU structure.
1396	* @param GCPtr The address.
1397	* @param uGstPml4e Guest PML4 entry (valid).
1398	* @param uGstPdpe Guest PDPT entry (valid).
1399	* @param ppPD Receives address of page directory
1400	*/
1401	static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD)
1402	{
1403	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1404	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1405	bool const fNestedPagingOrNoGstPaging = pVM->pgm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabled(pVCpu);
1406	int rc;
1407
1408	PGM_LOCK_ASSERT_OWNER(pVM);
1409
1410	/*
1411	* PML4.
1412	*/
1413	PPGMPOOLPAGE pShwPage;
1414	{
1415	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
1416	PX86PML4E pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
1417	X86PGPAEUINT const uPml4e = pPml4e->u;
1418
1419	/* Allocate page directory pointer table if not present. */
1420	if (uPml4e & (X86_PML4E_P \| X86_PML4E_PG_MASK))
1421	{
1422	pShwPage = pgmPoolGetPage(pPool, uPml4e & X86_PML4E_PG_MASK);
1423	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1424
1425	pgmPoolCacheUsed(pPool, pShwPage);
1426
1427	/* Update the entry if needed. */
1428	X86PGPAEUINT const uPml4eNew = pShwPage->Core.Key \| (uGstPml4e & pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask)
1429	\| (uPml4e & PGM_PML4_FLAGS);
1430	if (uPml4e == uPml4eNew)
1431	{ /* likely */ }
1432	else
1433	ASMAtomicWriteU64(&pPml4e->u, uPml4eNew);
1434	}
1435	else
1436	{
1437	Assert(pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1438
1439	RTGCPTR64 GCPml4;
1440	PGMPOOLKIND enmKind;
1441	if (fNestedPagingOrNoGstPaging)
1442	{
1443	/* AMD-V nested paging or real/protected mode without paging */
1444	GCPml4 = (RTGCPTR64)iPml4 << X86_PML4_SHIFT; /** @todo bogus calculation for PML5 */
1445	enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_PHYS;
1446	}
1447	else
1448	{
1449	GCPml4 = uGstPml4e & X86_PML4E_PG_MASK;
1450	enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT;
1451	}
1452
1453	/* Create a reference back to the PDPT by using the index in its shadow page. */
1454	rc = pgmPoolAlloc(pVM, GCPml4, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1455	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /fLockPage/,
1456	&pShwPage);
1457	AssertRCReturn(rc, rc);
1458
1459	/* Hook it up. */
1460	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| (uGstPml4e & pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask)
1461	\| (uPml4e & PGM_PML4_FLAGS));
1462	}
1463	}
1464
1465	/*
1466	* PDPT.
1467	*/
1468	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1469	PX86PDPT pPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1470	PX86PDPE pPdpe = &pPdpt->a[iPdPt];
1471	X86PGPAEUINT const uPdpe = pPdpe->u;
1472
1473	/* Allocate page directory if not present. */
1474	if (uPdpe & (X86_PDPE_P \| X86_PDPE_PG_MASK))
1475	{
1476	pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
1477	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1478
1479	pgmPoolCacheUsed(pPool, pShwPage);
1480
1481	/* Update the entry if needed. */
1482	X86PGPAEUINT const uPdpeNew = pShwPage->Core.Key \| (uGstPdpe & pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask)
1483	\| (uPdpe & PGM_PDPT_FLAGS);
1484	if (uPdpe == uPdpeNew)
1485	{ /* likely */ }
1486	else
1487	ASMAtomicWriteU64(&pPdpe->u, uPdpeNew);
1488	}
1489	else
1490	{
1491	RTGCPTR64 GCPdPt;
1492	PGMPOOLKIND enmKind;
1493	if (fNestedPagingOrNoGstPaging)
1494	{
1495	/* AMD-V nested paging or real/protected mode without paging */
1496	GCPdPt = GCPtr & ~(RT_BIT_64(iPdPt << X86_PDPT_SHIFT) - 1);
1497	enmKind = PGMPOOLKIND_64BIT_PD_FOR_PHYS;
1498	}
1499	else
1500	{
1501	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1502	enmKind = PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD;
1503	}
1504
1505	/* Create a reference back to the PDPT by using the index in its shadow page. */
1506	rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1507	pShwPage->idx, iPdPt, false /fLockPage/,
1508	&pShwPage);
1509	AssertRCReturn(rc, rc);
1510
1511	/* Hook it up. */
1512	ASMAtomicWriteU64(&pPdpe->u,
1513	pShwPage->Core.Key \| (uGstPdpe & pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask) \| (uPdpe & PGM_PDPT_FLAGS));
1514	}
1515
1516	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1517	return VINF_SUCCESS;
1518	}
1519
1520
1521	/**
1522	* Gets the SHADOW page directory pointer for the specified address (long mode).
1523	*
1524	* @returns VBox status code.
1525	* @param pVCpu The cross context virtual CPU structure.
1526	* @param GCPtr The address.
1527	* @param ppPml4e Receives the address of the page map level 4 entry.
1528	* @param ppPdpt Receives the address of the page directory pointer table.
1529	* @param ppPD Receives the address of the page directory.
1530	*/
1531	DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPT ppPdpt, PX86PDPAE *ppPD)
1532	{
1533	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1534	PGM_LOCK_ASSERT_OWNER(pVM);
1535
1536	/*
1537	* PML4
1538	*/
1539	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
1540	PCX86PML4E pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
1541	AssertReturn(pPml4e, VERR_PGM_PML4_MAPPING);
1542	if (ppPml4e)
1543	*ppPml4e = (PX86PML4E)pPml4e;
1544	X86PGPAEUINT const uPml4e = pPml4e->u;
1545	Log4(("pgmShwGetLongModePDPtr %RGv (%RHv) %RX64\n", GCPtr, pPml4e, uPml4e));
1546	if (!(uPml4e & X86_PML4E_P)) /** @todo other code is check for NULL page frame number! */
1547	return VERR_PAGE_MAP_LEVEL4_NOT_PRESENT;
1548
1549	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1550	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, uPml4e & X86_PML4E_PG_MASK);
1551	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1552
1553	/*
1554	* PDPT
1555	*/
1556	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1557	PCX86PDPT pPdpt = *ppPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1558	X86PGPAEUINT const uPdpe = pPdpt->a[iPdPt].u;
1559	if (!(uPdpe & X86_PDPE_P)) /** @todo other code is check for NULL page frame number! */
1560	return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
1561
1562	pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
1563	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1564
1565	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1566	Log4(("pgmShwGetLongModePDPtr %RGv -> ppPD=%p PDE=%p/%RX64\n", GCPtr, ppPD, &(ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK], (ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK].u));
1567	return VINF_SUCCESS;
1568	}
1569
1570
1571	/**
1572	* Syncs the SHADOW EPT page directory pointer for the specified address. Allocates
1573	* backing pages in case the PDPT or PML4 entry is missing.
1574	*
1575	* @returns VBox status code.
1576	* @param pVCpu The cross context virtual CPU structure.
1577	* @param GCPtr The address.
1578	* @param ppPdpt Receives address of pdpt
1579	* @param ppPD Receives address of page directory
1580	*/
1581	static int pgmShwGetEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD)
1582	{
1583	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1584	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1585	int rc;
1586
1587	Assert(pVM->pgm.s.fNestedPaging);
1588	PGM_LOCK_ASSERT_OWNER(pVM);
1589
1590	/*
1591	* PML4 level.
1592	*/
1593	PEPTPML4 pPml4 = (PEPTPML4)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1594	Assert(pPml4);
1595
1596	/* Allocate page directory pointer table if not present. */
1597	PPGMPOOLPAGE pShwPage;
1598	{
1599	const unsigned iPml4 = (GCPtr >> EPT_PML4_SHIFT) & EPT_PML4_MASK;
1600	PEPTPML4E pPml4e = &pPml4->a[iPml4];
1601	EPTPML4E Pml4e;
1602	Pml4e.u = pPml4e->u;
1603	if (!(Pml4e.u & (EPT_E_PG_MASK \| EPT_E_READ)))
1604	{
1605	RTGCPTR64 GCPml4 = (RTGCPTR64)iPml4 << EPT_PML4_SHIFT;
1606	rc = pgmPoolAlloc(pVM, GCPml4, PGMPOOLKIND_EPT_PDPT_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1607	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /fLockPage/,
1608	&pShwPage);
1609	AssertRCReturn(rc, rc);
1610
1611	/* Hook up the new PDPT now. */
1612	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1613	}
1614	else
1615	{
1616	pShwPage = pgmPoolGetPage(pPool, pPml4e->u & EPT_PML4E_PG_MASK);
1617	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1618
1619	pgmPoolCacheUsed(pPool, pShwPage);
1620
1621	/* Hook up the cached PDPT if needed (probably not given 512512 PTs to sync). /
1622	if (Pml4e.u == (pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE))
1623	{ }
1624	else
1625	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1626	}
1627	}
1628
1629	/*
1630	* PDPT level.
1631	*/
1632	const unsigned iPdPt = (GCPtr >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
1633	PEPTPDPT pPdpt = (PEPTPDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1634	PEPTPDPTE pPdpe = &pPdpt->a[iPdPt];
1635
1636	if (ppPdpt)
1637	*ppPdpt = pPdpt;
1638
1639	/* Allocate page directory if not present. */
1640	EPTPDPTE Pdpe;
1641	Pdpe.u = pPdpe->u;
1642	if (!(Pdpe.u & (EPT_E_PG_MASK \| EPT_E_READ)))
1643	{
1644	RTGCPTR64 const GCPdPt = GCPtr & ~(RT_BIT_64(EPT_PDPT_SHIFT) - 1);
1645	rc = pgmPoolAlloc(pVM, GCPdPt, PGMPOOLKIND_EPT_PD_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1646	pShwPage->idx, iPdPt, false /fLockPage/,
1647	&pShwPage);
1648	AssertRCReturn(rc, rc);
1649
1650	/* Hook up the new PD now. */
1651	ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1652	}
1653	else
1654	{
1655	pShwPage = pgmPoolGetPage(pPool, pPdpe->u & EPT_PDPTE_PG_MASK);
1656	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1657
1658	pgmPoolCacheUsed(pPool, pShwPage);
1659
1660	/* Hook up the cached PD if needed (probably not given there are 512 PTs we may need sync). */
1661	if (Pdpe.u == (pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE))
1662	{ }
1663	else
1664	ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1665	}
1666
1667	*ppPD = (PEPTPD)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1668	return VINF_SUCCESS;
1669	}
1670
1671
1672	#ifdef IN_RING0
1673	/**
1674	* Synchronizes a range of nested page table entries.
1675	*
1676	* The caller must own the PGM lock.
1677	*
1678	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1679	* @param GCPhys Where to start.
1680	* @param cPages How many pages which entries should be synced.
1681	* @param enmShwPagingMode The shadow paging mode (PGMMODE_EPT for VT-x,
1682	* host paging mode for AMD-V).
1683	*/
1684	int pgmShwSyncNestedPageLocked(PVMCPUCC pVCpu, RTGCPHYS GCPhys, uint32_t cPages, PGMMODE enmShwPagingMode)
1685	{
1686	PGM_LOCK_ASSERT_OWNER(pVCpu->CTX_SUFF(pVM));
1687
1688	/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
1689	int rc;
1690	switch (enmShwPagingMode)
1691	{
1692	case PGMMODE_32_BIT:
1693	{
1694	X86PDE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1695	rc = PGM_BTH_NAME_32BIT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1696	break;
1697	}
1698
1699	case PGMMODE_PAE:
1700	case PGMMODE_PAE_NX:
1701	{
1702	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1703	rc = PGM_BTH_NAME_PAE_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1704	break;
1705	}
1706
1707	case PGMMODE_AMD64:
1708	case PGMMODE_AMD64_NX:
1709	{
1710	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1711	rc = PGM_BTH_NAME_AMD64_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1712	break;
1713	}
1714
1715	case PGMMODE_EPT:
1716	{
1717	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1718	rc = PGM_BTH_NAME_EPT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1719	break;
1720	}
1721
1722	default:
1723	AssertMsgFailedReturn(("%d\n", enmShwPagingMode), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
1724	}
1725	return rc;
1726	}
1727	#endif /* IN_RING0 */
1728
1729
1730	/**
1731	* Gets effective Guest OS page information.
1732	*
1733	* When GCPtr is in a big page, the function will return as if it was a normal
1734	* 4KB page. If the need for distinguishing between big and normal page becomes
1735	* necessary at a later point, a PGMGstGetPage() will be created for that
1736	* purpose.
1737	*
1738	* @returns VBox status code.
1739	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1740	* @param GCPtr Guest Context virtual address of the page.
1741	* @param pWalk Where to store the page walk information.
1742	*/
1743	VMMDECL(int) PGMGstGetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk)
1744	{
1745	VMCPU_ASSERT_EMT(pVCpu);
1746	Assert(pWalk);
1747	uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
1748	AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
1749	AssertReturn(g_aPgmGuestModeData[idx].pfnGetPage, VERR_PGM_MODE_IPE);
1750	return g_aPgmGuestModeData[idx].pfnGetPage(pVCpu, GCPtr, pWalk);
1751	}
1752
1753
1754	/**
1755	* Maps the guest CR3.
1756	*
1757	* @returns VBox status code.
1758	* @param pVCpu The cross context virtual CPU structure.
1759	* @param GCPhysCr3 The guest CR3 value.
1760	* @param pHCPtrGuestCr3 Where to store the mapped memory.
1761	*/
1762	DECLINLINE(int) pgmGstMapCr3(PVMCPUCC pVCpu, RTGCPHYS GCPhysCr3, PRTHCPTR pHCPtrGuestCr3)
1763	{
1764	/** @todo this needs some reworking wrt. locking? */
1765	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1766	PGM_LOCK_VOID(pVM);
1767	PPGMPAGE pPageCr3 = pgmPhysGetPage(pVM, GCPhysCr3);
1768	AssertReturnStmt(pPageCr3, PGM_UNLOCK(pVM), VERR_PGM_INVALID_CR3_ADDR);
1769
1770	RTHCPTR HCPtrGuestCr3;
1771	int rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPageCr3, GCPhysCr3, (void **)&HCPtrGuestCr3);
1772	PGM_UNLOCK(pVM);
1773
1774	*pHCPtrGuestCr3 = HCPtrGuestCr3;
1775	return rc;
1776	}
1777
1778
1779	/**
1780	* Unmaps the guest CR3.
1781	*
1782	* @returns VBox status code.
1783	* @param pVCpu The cross context virtual CPU structure.
1784	*/
1785	DECLINLINE(int) pgmGstUnmapCr3(PVMCPUCC pVCpu)
1786	{
1787	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1788	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
1789	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
1790	return g_aPgmBothModeData[idxBth].pfnUnmapCR3(pVCpu);
1791	}
1792
1793
1794	/**
1795	* Performs a guest page table walk.
1796	*
1797	* The guest should be in paged protect mode or long mode when making a call to
1798	* this function.
1799	*
1800	* @returns VBox status code.
1801	* @retval VINF_SUCCESS on success.
1802	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
1803	* @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
1804	* not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
1805	*
1806	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1807	* @param GCPtr The guest virtual address to walk by.
1808	* @param pWalk Where to return the walk result. This is valid for some
1809	* error codes as well.
1810	* @param pGstWalk The guest mode specific page walk information.
1811	*/
1812	int pgmGstPtWalk(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk)
1813	{
1814	VMCPU_ASSERT_EMT(pVCpu);
1815	switch (pVCpu->pgm.s.enmGuestMode)
1816	{
1817	case PGMMODE_32_BIT:
1818	pGstWalk->enmType = PGMPTWALKGSTTYPE_32BIT;
1819	return PGM_GST_NAME_32BIT(Walk)(pVCpu, GCPtr, pWalk, &pGstWalk->u.Legacy);
1820
1821	case PGMMODE_PAE:
1822	case PGMMODE_PAE_NX:
1823	pGstWalk->enmType = PGMPTWALKGSTTYPE_PAE;
1824	return PGM_GST_NAME_PAE(Walk)(pVCpu, GCPtr, pWalk, &pGstWalk->u.Pae);
1825
1826	case PGMMODE_AMD64:
1827	case PGMMODE_AMD64_NX:
1828	pGstWalk->enmType = PGMPTWALKGSTTYPE_AMD64;
1829	return PGM_GST_NAME_AMD64(Walk)(pVCpu, GCPtr, pWalk, &pGstWalk->u.Amd64);
1830
1831	case PGMMODE_REAL:
1832	case PGMMODE_PROTECTED:
1833	pGstWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
1834	return VERR_PGM_NOT_USED_IN_MODE;
1835
1836	case PGMMODE_EPT:
1837	case PGMMODE_NESTED_32BIT:
1838	case PGMMODE_NESTED_PAE:
1839	case PGMMODE_NESTED_AMD64:
1840	default:
1841	AssertFailed();
1842	pGstWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
1843	return VERR_PGM_NOT_USED_IN_MODE;
1844	}
1845	}
1846
1847
1848	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1849	/**
1850	* Performs a guest second-level address translation (SLAT).
1851	*
1852	* The guest paging mode must be 32-bit, PAE or AMD64 when making a call to this
1853	* function.
1854	*
1855	* @returns VBox status code.
1856	* @retval VINF_SUCCESS on success.
1857	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
1858	* @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
1859	* not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
1860	*
1861	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1862	* @param GCPhysNested The nested-guest physical address being translated
1863	* (input).
1864	* @param fIsLinearAddrValid Whether the linear address in @a GCPtrNested is
1865	* valid. This indicates the SLAT is caused when
1866	* translating a nested-guest linear address.
1867	* @param GCPtrNested The nested-guest virtual address that initiated the
1868	* SLAT. If none, pass NIL_RTGCPTR.
1869	* @param pWalk Where to return the walk result. This is valid for
1870	* some error codes as well.
1871	* @param pGstWalk The second-level paging-mode specific walk
1872	* information.
1873	*/
1874	static int pgmGstSlatWalk(PVMCPUCC pVCpu, RTGCPHYS GCPhysNested, bool fIsLinearAddrValid, RTGCPTR GCPtrNested,
1875	PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk)
1876	{
1877	/* SLAT mode must be valid at this point as this should only be used -after- we have determined SLAT mode. */
1878	Assert( pVCpu->pgm.s.enmGuestSlatMode != PGMSLAT_DIRECT
1879	&& pVCpu->pgm.s.enmGuestSlatMode != PGMSLAT_INVALID);
1880	switch (pVCpu->pgm.s.enmGuestSlatMode)
1881	{
1882	case PGMSLAT_EPT:
1883	pGstWalk->enmType = PGMPTWALKGSTTYPE_EPT;
1884	return PGM_GST_SLAT_NAME_EPT(Walk)(pVCpu, GCPhysNested, fIsLinearAddrValid, GCPtrNested, pWalk, &pGstWalk->u.Ept);
1885
1886	default:
1887	AssertFailed();
1888	pGstWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
1889	return VERR_PGM_NOT_USED_IN_MODE;
1890	}
1891	}
1892
1893
1894	/**
1895	* Performs a guest second-level address translation (SLAT) for a nested-guest
1896	* physical address.
1897	*
1898	* This version requires the SLAT mode to be provided by the caller because we could
1899	* be in the process of switching paging modes (MOV CRX) and cannot presume control
1900	* register values.
1901	*
1902	* @returns VBox status code.
1903	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1904	* @param enmSlatMode The second-level paging mode to use.
1905	* @param GCPhysNested The nested-guest physical address to translate.
1906	* @param pWalk Where to store the walk result.
1907	* @param pGstWalk Where to store the second-level paging-mode specific
1908	* walk information.
1909	*/
1910	static int pgmGstSlatWalkPhys(PVMCPUCC pVCpu, PGMSLAT enmSlatMode, RTGCPHYS GCPhysNested, PPGMPTWALK pWalk,
1911	PPGMPTWALKGST pGstWalk)
1912	{
1913	AssertPtr(pWalk);
1914	AssertPtr(pGstWalk);
1915	switch (enmSlatMode)
1916	{
1917	case PGMSLAT_EPT:
1918	pGstWalk->enmType = PGMPTWALKGSTTYPE_EPT;
1919	return PGM_GST_SLAT_NAME_EPT(Walk)(pVCpu, GCPhysNested, false /* fIsLinearaddrValid /, 0 / GCPtrNested */,
1920	pWalk, &pGstWalk->u.Ept);
1921
1922	default:
1923	AssertFailed();
1924	return VERR_PGM_NOT_USED_IN_MODE;
1925	}
1926	}
1927	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
1928
1929
1930	/**
1931	* Tries to continue the previous walk.
1932	*
1933	* @note Requires the caller to hold the PGM lock from the first
1934	* pgmGstPtWalk() call to the last pgmGstPtWalkNext() call. Otherwise
1935	* we cannot use the pointers.
1936	*
1937	* @returns VBox status code.
1938	* @retval VINF_SUCCESS on success.
1939	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
1940	* @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
1941	* not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
1942	*
1943	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1944	* @param GCPtr The guest virtual address to walk by.
1945	* @param pWalk Pointer to the previous walk result and where to return
1946	* the result of this walk. This is valid for some error
1947	* codes as well.
1948	* @param pGstWalk The guest-mode specific walk information.
1949	*/
1950	int pgmGstPtWalkNext(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk)
1951	{
1952	/*
1953	* We can only handle successfully walks.
1954	* We also limit ourselves to the next page.
1955	*/
1956	if ( pWalk->fSucceeded
1957	&& GCPtr - pWalk->GCPtr == GUEST_PAGE_SIZE)
1958	{
1959	Assert(pWalk->uLevel == 0);
1960	if (pGstWalk->enmType == PGMPTWALKGSTTYPE_AMD64)
1961	{
1962	/*
1963	* AMD64
1964	*/
1965	if (!pWalk->fGigantPage && !pWalk->fBigPage)
1966	{
1967	/*
1968	* We fall back to full walk if the PDE table changes, if any
1969	* reserved bits are set, or if the effective page access changes.
1970	*/
1971	const uint64_t fPteSame = X86_PTE_P \| X86_PTE_RW \| X86_PTE_US \| X86_PTE_PWT
1972	\| X86_PTE_PCD \| X86_PTE_A \| X86_PTE_PAE_NX;
1973	const uint64_t fPdeSame = X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT
1974	\| X86_PDE_PCD \| X86_PDE_A \| X86_PDE_PAE_NX \| X86_PDE_PS;
1975
1976	if ((GCPtr >> X86_PD_PAE_SHIFT) == (pWalk->GCPtr >> X86_PD_PAE_SHIFT))
1977	{
1978	if (pGstWalk->u.Amd64.pPte)
1979	{
1980	X86PTEPAE Pte;
1981	Pte.u = pGstWalk->u.Amd64.pPte[1].u;
1982	if ( (Pte.u & fPteSame) == (pGstWalk->u.Amd64.Pte.u & fPteSame)
1983	&& !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask))
1984	{
1985	pWalk->GCPtr = GCPtr;
1986	pWalk->GCPhys = Pte.u & X86_PTE_PAE_PG_MASK;
1987	pGstWalk->u.Amd64.Pte.u = Pte.u;
1988	pGstWalk->u.Amd64.pPte++;
1989	return VINF_SUCCESS;
1990	}
1991	}
1992	}
1993	else if ((GCPtr >> X86_PDPT_SHIFT) == (pWalk->GCPtr >> X86_PDPT_SHIFT))
1994	{
1995	Assert(!((GCPtr >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK)); /* Must be first PT entry. */
1996	if (pGstWalk->u.Amd64.pPde)
1997	{
1998	X86PDEPAE Pde;
1999	Pde.u = pGstWalk->u.Amd64.pPde[1].u;
2000	if ( (Pde.u & fPdeSame) == (pGstWalk->u.Amd64.Pde.u & fPdeSame)
2001	&& !(Pde.u & (pVCpu)->pgm.s.fGstAmd64MbzPdeMask))
2002	{
2003	/* Get the new PTE and check out the first entry. */
2004	int rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, PGM_A20_APPLY(pVCpu, (Pde.u & X86_PDE_PAE_PG_MASK)),
2005	&pGstWalk->u.Amd64.pPt);
2006	if (RT_SUCCESS(rc))
2007	{
2008	pGstWalk->u.Amd64.pPte = &pGstWalk->u.Amd64.pPt->a[0];
2009	X86PTEPAE Pte;
2010	Pte.u = pGstWalk->u.Amd64.pPte->u;
2011	if ( (Pte.u & fPteSame) == (pGstWalk->u.Amd64.Pte.u & fPteSame)
2012	&& !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask))
2013	{
2014	pWalk->GCPtr = GCPtr;
2015	pWalk->GCPhys = Pte.u & X86_PTE_PAE_PG_MASK;
2016	pGstWalk->u.Amd64.Pte.u = Pte.u;
2017	pGstWalk->u.Amd64.Pde.u = Pde.u;
2018	pGstWalk->u.Amd64.pPde++;
2019	return VINF_SUCCESS;
2020	}
2021	}
2022	}
2023	}
2024	}
2025	}
2026	else if (!pWalk->fGigantPage)
2027	{
2028	if ((GCPtr & X86_PAGE_2M_BASE_MASK) == (pWalk->GCPtr & X86_PAGE_2M_BASE_MASK))
2029	{
2030	pWalk->GCPtr = GCPtr;
2031	pWalk->GCPhys += GUEST_PAGE_SIZE;
2032	return VINF_SUCCESS;
2033	}
2034	}
2035	else
2036	{
2037	if ((GCPtr & X86_PAGE_1G_BASE_MASK) == (pWalk->GCPtr & X86_PAGE_1G_BASE_MASK))
2038	{
2039	pWalk->GCPtr = GCPtr;
2040	pWalk->GCPhys += GUEST_PAGE_SIZE;
2041	return VINF_SUCCESS;
2042	}
2043	}
2044	}
2045	}
2046	/* Case we don't handle. Do full walk. */
2047	return pgmGstPtWalk(pVCpu, GCPtr, pWalk, pGstWalk);
2048	}
2049
2050
2051	/**
2052	* Modify page flags for a range of pages in the guest's tables
2053	*
2054	* The existing flags are ANDed with the fMask and ORed with the fFlags.
2055	*
2056	* @returns VBox status code.
2057	* @param pVCpu The cross context virtual CPU structure.
2058	* @param GCPtr Virtual address of the first page in the range.
2059	* @param cb Size (in bytes) of the range to apply the modification to.
2060	* @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course.
2061	* @param fMask The AND mask - page flags X86_PTE_*, excluding the page mask of course.
2062	* Be very CAREFUL when ~'ing constants which could be 32-bit!
2063	*/
2064	VMMDECL(int) PGMGstModifyPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask)
2065	{
2066	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,GstModifyPage), a);
2067	VMCPU_ASSERT_EMT(pVCpu);
2068
2069	/*
2070	* Validate input.
2071	*/
2072	AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
2073	Assert(cb);
2074
2075	LogFlow(("PGMGstModifyPage %RGv %d bytes fFlags=%08llx fMask=%08llx\n", GCPtr, cb, fFlags, fMask));
2076
2077	/*
2078	* Adjust input.
2079	*/
2080	cb += GCPtr & GUEST_PAGE_OFFSET_MASK;
2081	cb = RT_ALIGN_Z(cb, GUEST_PAGE_SIZE);
2082	GCPtr &= ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK;
2083
2084	/*
2085	* Call worker.
2086	*/
2087	uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
2088	AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
2089	AssertReturn(g_aPgmGuestModeData[idx].pfnModifyPage, VERR_PGM_MODE_IPE);
2090	int rc = g_aPgmGuestModeData[idx].pfnModifyPage(pVCpu, GCPtr, cb, fFlags, fMask);
2091
2092	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,GstModifyPage), a);
2093	return rc;
2094	}
2095
2096
2097	/**
2098	* Checks whether the given PAE PDPEs are potentially valid for the guest.
2099	*
2100	* @returns @c true if the PDPE is valid, @c false otherwise.
2101	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2102	* @param paPaePdpes The PAE PDPEs to validate.
2103	*
2104	* @remarks This function -only- checks the reserved bits in the PDPE entries.
2105	*/
2106	VMM_INT_DECL(bool) PGMGstArePaePdpesValid(PVMCPUCC pVCpu, PCX86PDPE paPaePdpes)
2107	{
2108	Assert(paPaePdpes);
2109	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
2110	{
2111	X86PDPE const PaePdpe = paPaePdpes[i];
2112	if ( !(PaePdpe.u & X86_PDPE_P)
2113	\|\| !(PaePdpe.u & pVCpu->pgm.s.fGstPaeMbzPdpeMask))
2114	{ /* likely */ }
2115	else
2116	return false;
2117	}
2118	return true;
2119	}
2120
2121
2122	/**
2123	* Performs the lazy mapping of the 32-bit guest PD.
2124	*
2125	* @returns VBox status code.
2126	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2127	* @param ppPd Where to return the pointer to the mapping. This is
2128	* always set.
2129	*/
2130	int pgmGstLazyMap32BitPD(PVMCPUCC pVCpu, PX86PD *ppPd)
2131	{
2132	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2133	PGM_LOCK_VOID(pVM);
2134
2135	Assert(!pVCpu->pgm.s.CTX_SUFF(pGst32BitPd));
2136
2137	RTGCPHYS GCPhysCR3 = pVCpu->pgm.s.GCPhysCR3 & X86_CR3_PAGE_MASK;
2138	PPGMPAGE pPage;
2139	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2140	if (RT_SUCCESS(rc))
2141	{
2142	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPd);
2143	if (RT_SUCCESS(rc))
2144	{
2145	# ifdef IN_RING3
2146	pVCpu->pgm.s.pGst32BitPdR0 = NIL_RTR0PTR;
2147	pVCpu->pgm.s.pGst32BitPdR3 = *ppPd;
2148	# else
2149	pVCpu->pgm.s.pGst32BitPdR3 = NIL_RTR0PTR;
2150	pVCpu->pgm.s.pGst32BitPdR0 = *ppPd;
2151	# endif
2152	PGM_UNLOCK(pVM);
2153	return VINF_SUCCESS;
2154	}
2155	AssertRC(rc);
2156	}
2157	PGM_UNLOCK(pVM);
2158
2159	*ppPd = NULL;
2160	return rc;
2161	}
2162
2163
2164	/**
2165	* Performs the lazy mapping of the PAE guest PDPT.
2166	*
2167	* @returns VBox status code.
2168	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2169	* @param ppPdpt Where to return the pointer to the mapping. This is
2170	* always set.
2171	*/
2172	int pgmGstLazyMapPaePDPT(PVMCPUCC pVCpu, PX86PDPT *ppPdpt)
2173	{
2174	Assert(!pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt));
2175	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2176	PGM_LOCK_VOID(pVM);
2177
2178	RTGCPHYS GCPhysCR3 = pVCpu->pgm.s.GCPhysCR3 & X86_CR3_PAE_PAGE_MASK;
2179	PPGMPAGE pPage;
2180	/** @todo Nested VMX: convert GCPhysCR3 from nested-guest physical to
2181	* guest-physical address here. */
2182	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2183	if (RT_SUCCESS(rc))
2184	{
2185	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPdpt);
2186	if (RT_SUCCESS(rc))
2187	{
2188	# ifdef IN_RING3
2189	pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
2190	pVCpu->pgm.s.pGstPaePdptR3 = *ppPdpt;
2191	# else
2192	pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
2193	pVCpu->pgm.s.pGstPaePdptR0 = *ppPdpt;
2194	# endif
2195	PGM_UNLOCK(pVM);
2196	return VINF_SUCCESS;
2197	}
2198	AssertRC(rc);
2199	}
2200
2201	PGM_UNLOCK(pVM);
2202	*ppPdpt = NULL;
2203	return rc;
2204	}
2205
2206
2207	/**
2208	* Performs the lazy mapping / updating of a PAE guest PD.
2209	*
2210	* @returns Pointer to the mapping.
2211	* @returns VBox status code.
2212	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2213	* @param iPdpt Which PD entry to map (0..3).
2214	* @param ppPd Where to return the pointer to the mapping. This is
2215	* always set.
2216	*/
2217	int pgmGstLazyMapPaePD(PVMCPUCC pVCpu, uint32_t iPdpt, PX86PDPAE *ppPd)
2218	{
2219	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2220	PGM_LOCK_VOID(pVM);
2221
2222	PX86PDPT pGuestPDPT = pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt);
2223	Assert(pGuestPDPT);
2224	Assert(pGuestPDPT->a[iPdpt].u & X86_PDPE_P);
2225	RTGCPHYS GCPhys = pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK;
2226	bool const fChanged = pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] != GCPhys;
2227
2228	PPGMPAGE pPage;
2229	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
2230	if (RT_SUCCESS(rc))
2231	{
2232	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)ppPd);
2233	AssertRC(rc);
2234	if (RT_SUCCESS(rc))
2235	{
2236	# ifdef IN_RING3
2237	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = NIL_RTR0PTR;
2238	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = *ppPd;
2239	# else
2240	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = NIL_RTR3PTR;
2241	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = *ppPd;
2242	# endif
2243	if (fChanged)
2244	pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] = GCPhys;
2245	PGM_UNLOCK(pVM);
2246	return VINF_SUCCESS;
2247	}
2248	}
2249
2250	/* Invalid page or some failure, invalidate the entry. */
2251	pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] = NIL_RTGCPHYS;
2252	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = NIL_RTR3PTR;
2253	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = NIL_RTR0PTR;
2254
2255	PGM_UNLOCK(pVM);
2256	return rc;
2257	}
2258
2259
2260	/**
2261	* Performs the lazy mapping of the 32-bit guest PD.
2262	*
2263	* @returns VBox status code.
2264	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2265	* @param ppPml4 Where to return the pointer to the mapping. This will
2266	* always be set.
2267	*/
2268	int pgmGstLazyMapPml4(PVMCPUCC pVCpu, PX86PML4 *ppPml4)
2269	{
2270	Assert(!pVCpu->pgm.s.CTX_SUFF(pGstAmd64Pml4));
2271	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2272	PGM_LOCK_VOID(pVM);
2273
2274	RTGCPHYS GCPhysCR3 = pVCpu->pgm.s.GCPhysCR3 & X86_CR3_AMD64_PAGE_MASK;
2275	PPGMPAGE pPage;
2276	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2277	if (RT_SUCCESS(rc))
2278	{
2279	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPml4);
2280	if (RT_SUCCESS(rc))
2281	{
2282	# ifdef IN_RING3
2283	pVCpu->pgm.s.pGstAmd64Pml4R0 = NIL_RTR0PTR;
2284	pVCpu->pgm.s.pGstAmd64Pml4R3 = *ppPml4;
2285	# else
2286	pVCpu->pgm.s.pGstAmd64Pml4R3 = NIL_RTR3PTR;
2287	pVCpu->pgm.s.pGstAmd64Pml4R0 = *ppPml4;
2288	# endif
2289	PGM_UNLOCK(pVM);
2290	return VINF_SUCCESS;
2291	}
2292	}
2293
2294	PGM_UNLOCK(pVM);
2295	*ppPml4 = NULL;
2296	return rc;
2297	}
2298
2299
2300	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2301	/**
2302	* Performs the lazy mapping of the guest PML4 table when using EPT paging.
2303	*
2304	* @returns VBox status code.
2305	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2306	* @param ppEptPml4 Where to return the pointer to the mapping. This will
2307	* always be set.
2308	*/
2309	int pgmGstLazyMapEptPml4(PVMCPUCC pVCpu, PEPTPML4 *ppEptPml4)
2310	{
2311	Assert(!pVCpu->pgm.s.CTX_SUFF(pGstEptPml4));
2312	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2313	PGM_LOCK_VOID(pVM);
2314
2315	RTGCPHYS const GCPhysEpt = pVCpu->pgm.s.uEptPtr & EPT_EPTP_PG_MASK;
2316	PPGMPAGE pPage;
2317	int rc = pgmPhysGetPageEx(pVM, GCPhysEpt, &pPage);
2318	if (RT_SUCCESS(rc))
2319	{
2320	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysEpt, (void **)ppEptPml4);
2321	if (RT_SUCCESS(rc))
2322	{
2323	# ifdef IN_RING3
2324	pVCpu->pgm.s.pGstEptPml4R0 = NIL_RTR0PTR;
2325	pVCpu->pgm.s.pGstEptPml4R3 = *ppEptPml4;
2326	# else
2327	pVCpu->pgm.s.pGstEptPml4R3 = NIL_RTR3PTR;
2328	pVCpu->pgm.s.pGstEptPml4R0 = *ppEptPml4;
2329	# endif
2330	PGM_UNLOCK(pVM);
2331	return VINF_SUCCESS;
2332	}
2333	}
2334
2335	PGM_UNLOCK(pVM);
2336	*ppEptPml4 = NULL;
2337	return rc;
2338	}
2339	#endif
2340
2341
2342	/**
2343	* Gets the current CR3 register value for the shadow memory context.
2344	* @returns CR3 value.
2345	* @param pVCpu The cross context virtual CPU structure.
2346	*/
2347	VMMDECL(RTHCPHYS) PGMGetHyperCR3(PVMCPU pVCpu)
2348	{
2349	PPGMPOOLPAGE pPoolPage = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2350	AssertPtrReturn(pPoolPage, NIL_RTHCPHYS);
2351	return pPoolPage->Core.Key;
2352	}
2353
2354
2355	/**
2356	* Forces lazy remapping of the guest's PAE page-directory structures.
2357	*
2358	* @param pVCpu The cross context virtual CPU structure.
2359	*/
2360	static void pgmGstFlushPaePdpes(PVMCPU pVCpu)
2361	{
2362	for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->pgm.s.aGCPhysGstPaePDs); i++)
2363	{
2364	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
2365	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
2366	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
2367	}
2368	}
2369
2370
2371	/**
2372	* Gets the CR3 mask corresponding to the given paging mode.
2373	*
2374	* @returns The CR3 mask.
2375	* @param enmMode The paging mode.
2376	*/
2377	DECLINLINE(uint64_t) pgmGetCr3MaskForMode(PGMMODE enmMode)
2378	{
2379	/** @todo This work can be optimized either by storing the masks in
2380	* pVCpu->pgm.s.afGstCr3Masks[] for all PGMMODEs -or- just do this once and
2381	* store the result when entering guest mode since we currently use it only
2382	* for enmGuestMode. */
2383	switch (enmMode)
2384	{
2385	case PGMMODE_PAE:
2386	case PGMMODE_PAE_NX:
2387	return X86_CR3_PAE_PAGE_MASK;
2388	case PGMMODE_AMD64:
2389	case PGMMODE_AMD64_NX:
2390	return X86_CR3_AMD64_PAGE_MASK;
2391	case PGMMODE_EPT:
2392	return X86_CR3_EPT_PAGE_MASK;
2393	default:
2394	return X86_CR3_PAGE_MASK;
2395	}
2396	}
2397
2398
2399	/**
2400	* Gets the masked CR3 value according to the current guest paging mode.
2401	*
2402	* @returns The masked PGM CR3 value.
2403	* @param pVCpu The cross context virtual CPU structure.
2404	* @param uCr3 The raw guest CR3 value.
2405	*/
2406	DECLINLINE(RTGCPHYS) pgmGetGuestMaskedCr3(PVMCPUCC pVCpu, uint64_t uCr3)
2407	{
2408	uint64_t const fCr3Mask = pgmGetCr3MaskForMode(pVCpu->pgm.s.enmGuestMode);
2409	RTGCPHYS GCPhysCR3 = (RTGCPHYS)(uCr3 & fCr3Mask);
2410	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
2411	return GCPhysCR3;
2412	}
2413
2414
2415	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2416	/**
2417	* Performs second-level address translation for the given CR3 and updates the
2418	* nested-guest CR3 when successful.
2419	*
2420	* @returns VBox status code.
2421	* @param pVCpu The cross context virtual CPU structure.
2422	* @param uCr3 The masked nested-guest CR3 value.
2423	* @param pGCPhysCR3 Where to store the translated CR3.
2424	*
2425	* @warning This updates PGMCPU::GCPhysNstGstCR3 when the translation succeeds. Be
2426	* mindful of this in code that's hyper sensitive to the order of
2427	* operations.
2428	*/
2429	static int pgmGstSlatTranslateCr3(PVMCPUCC pVCpu, uint64_t uCr3, PRTGCPHYS pGCPhysCr3)
2430	{
2431	if (uCr3 != pVCpu->pgm.s.GCPhysNstGstCR3)
2432	{
2433	PGMPTWALK Walk;
2434	PGMPTWALKGST GstWalk;
2435	int const rc = pgmGstSlatWalkPhys(pVCpu, PGMSLAT_EPT, uCr3, &Walk, &GstWalk);
2436	if (RT_SUCCESS(rc))
2437	{
2438	/* Update nested-guest CR3. */
2439	pVCpu->pgm.s.GCPhysNstGstCR3 = uCr3;
2440
2441	/* Pass back the translated result. */
2442	*pGCPhysCr3 = Walk.GCPhys;
2443	return VINF_SUCCESS;
2444	}
2445
2446	/* Translation failed. */
2447	*pGCPhysCr3 = NIL_RTGCPHYS;
2448	return rc;
2449	}
2450
2451	/*
2452	* If the nested-guest CR3 has not changed, then the previously
2453	* translated CR3 result (i.e. GCPhysCR3) is passed back.
2454	*/
2455	*pGCPhysCr3 = pVCpu->pgm.s.GCPhysCR3;
2456	return VINF_SUCCESS;
2457	}
2458	#endif
2459
2460
2461	/**
2462	* Performs and schedules necessary updates following a CR3 load or reload.
2463	*
2464	* This will normally involve mapping the guest PD or nPDPT
2465	*
2466	* @returns VBox status code.
2467	* @retval VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync. This can
2468	* safely be ignored and overridden since the FF will be set too then.
2469	* @param pVCpu The cross context virtual CPU structure.
2470	* @param cr3 The new cr3.
2471	* @param fGlobal Indicates whether this is a global flush or not.
2472	*/
2473	VMMDECL(int) PGMFlushTLB(PVMCPUCC pVCpu, uint64_t cr3, bool fGlobal)
2474	{
2475	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLB), a);
2476	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2477
2478	VMCPU_ASSERT_EMT(pVCpu);
2479
2480	/*
2481	* Always flag the necessary updates; necessary for hardware acceleration
2482	*/
2483	/** @todo optimize this, it shouldn't always be necessary. */
2484	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2485	if (fGlobal)
2486	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2487
2488	/*
2489	* Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
2490	*/
2491	RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
2492	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3);
2493	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2494	if ( pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT
2495	&& PGMMODE_WITH_PAGING(pVCpu->pgm.s.enmGuestMode))
2496	{
2497	LogFlowFunc(("nested_cr3=%RX64 old=%RX64\n", GCPhysCR3, pVCpu->pgm.s.GCPhysNstGstCR3));
2498	RTGCPHYS GCPhysOut;
2499	int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
2500	if (RT_SUCCESS(rc))
2501	GCPhysCR3 = GCPhysOut;
2502	else
2503	{
2504	/* CR3 SLAT translation failed but we try to pretend it
2505	succeeded for the reasons mentioned in PGMHCChangeMode(). */
2506	AssertMsgFailed(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", cr3, rc));
2507	int const rc2 = pgmGstUnmapCr3(pVCpu);
2508	pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
2509	pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
2510	return rc2;
2511	}
2512	}
2513	#endif
2514
2515	LogFlowFunc(("cr3=%RX64 old=%RX64 fGlobal=%d\n", cr3, GCPhysOldCR3, fGlobal));
2516	int rc = VINF_SUCCESS;
2517	if (GCPhysOldCR3 != GCPhysCR3)
2518	{
2519	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2520	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2521	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2522
2523	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2524	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
2525	if (RT_LIKELY(rc == VINF_SUCCESS))
2526	{ }
2527	else
2528	{
2529	AssertMsg(rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc));
2530	Assert(VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL \| VMCPU_FF_PGM_SYNC_CR3));
2531	pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped) = false;
2532	pVCpu->pgm.s.GCPhysPaeCR3 = NIL_RTGCPHYS;
2533	pVCpu->pgm.s.GCPhysCR3 = GCPhysOldCR3;
2534	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_MAP_CR3;
2535	}
2536
2537	if (fGlobal)
2538	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBNewCR3Global));
2539	else
2540	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBNewCR3));
2541	}
2542	else
2543	{
2544	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2545	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2546	if (pPool->cDirtyPages)
2547	{
2548	PGM_LOCK_VOID(pVM);
2549	pgmPoolResetDirtyPages(pVM);
2550	PGM_UNLOCK(pVM);
2551	}
2552	#endif
2553	if (fGlobal)
2554	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBSameCR3Global));
2555	else
2556	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBSameCR3));
2557
2558	/*
2559	* Flush PAE PDPTEs.
2560	*/
2561	if (PGMMODE_IS_PAE(pVCpu->pgm.s.enmGuestMode))
2562	pgmGstFlushPaePdpes(pVCpu);
2563	}
2564
2565	IEMTlbInvalidateAll(pVCpu);
2566	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLB), a);
2567	return rc;
2568	}
2569
2570
2571	/**
2572	* Performs and schedules necessary updates following a CR3 load or reload when
2573	* using nested or extended paging.
2574	*
2575	* This API is an alternative to PGMFlushTLB that avoids actually flushing the
2576	* TLB and triggering a SyncCR3.
2577	*
2578	* This will normally involve mapping the guest PD or nPDPT
2579	*
2580	* @returns VBox status code.
2581	* @retval VINF_SUCCESS.
2582	* @retval VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync (not for nested
2583	* paging modes). This can safely be ignored and overridden since the
2584	* FF will be set too then.
2585	* @param pVCpu The cross context virtual CPU structure.
2586	* @param cr3 The new CR3.
2587	*/
2588	VMMDECL(int) PGMUpdateCR3(PVMCPUCC pVCpu, uint64_t cr3)
2589	{
2590	VMCPU_ASSERT_EMT(pVCpu);
2591
2592	/* We assume we're only called in nested paging mode. */
2593	Assert(pVCpu->CTX_SUFF(pVM)->pgm.s.fNestedPaging \|\| pVCpu->pgm.s.enmShadowMode == PGMMODE_EPT);
2594
2595	/*
2596	* Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
2597	*/
2598	RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
2599	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3);
2600	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2601	if (CPUMIsGuestVmxEptPagingEnabled(pVCpu))
2602	{
2603	LogFlowFunc(("nested_cr3=%RX64 old_nested_cr3=%RX64\n", cr3, pVCpu->pgm.s.GCPhysNstGstCR3));
2604	RTGCPHYS GCPhysOut;
2605	int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
2606	if (RT_SUCCESS(rc))
2607	GCPhysCR3 = GCPhysOut;
2608	else
2609	{
2610	/* CR3 SLAT translation failed but we try to pretend it
2611	succeeded for the reasons mentioned in PGMHCChangeMode(). */
2612	AssertMsgFailed(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", cr3, rc));
2613	int const rc2 = pgmGstUnmapCr3(pVCpu);
2614	pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
2615	pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
2616	return rc2;
2617	}
2618	}
2619	#endif
2620
2621	LogFlowFunc(("cr3=%RX64 old=%RX64\n", cr3, GCPhysOldCR3));
2622	int rc = VINF_SUCCESS;
2623	if (GCPhysOldCR3 != GCPhysCR3)
2624	{
2625	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2626	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2627	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2628
2629	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2630	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
2631
2632	AssertRCSuccess(rc); /* Assumes VINF_PGM_SYNC_CR3 doesn't apply to nested paging. / /* @todo this isn't true for the mac, but we need hw to test/fix this. */
2633	}
2634	/*
2635	* Flush PAE PDPTEs.
2636	*/
2637	else if (PGMMODE_IS_PAE(pVCpu->pgm.s.enmGuestMode))
2638	pgmGstFlushPaePdpes(pVCpu);
2639
2640	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2641	return rc;
2642	}
2643
2644
2645	/**
2646	* Synchronize the paging structures.
2647	*
2648	* This function is called in response to the VM_FF_PGM_SYNC_CR3 and
2649	* VM_FF_PGM_SYNC_CR3_NONGLOBAL. Those two force action flags are set
2650	* in several places, most importantly whenever the CR3 is loaded.
2651	*
2652	* @returns VBox status code. May return VINF_PGM_SYNC_CR3 in RC/R0.
2653	* @retval VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map
2654	* the VMM into guest context.
2655	* @param pVCpu The cross context virtual CPU structure.
2656	* @param cr0 Guest context CR0 register
2657	* @param cr3 Guest context CR3 register
2658	* @param cr4 Guest context CR4 register
2659	* @param fGlobal Including global page directories or not
2660	*/
2661	VMMDECL(int) PGMSyncCR3(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
2662	{
2663	int rc;
2664
2665	VMCPU_ASSERT_EMT(pVCpu);
2666
2667	/*
2668	* The pool may have pending stuff and even require a return to ring-3 to
2669	* clear the whole thing.
2670	*/
2671	rc = pgmPoolSyncCR3(pVCpu);
2672	if (rc != VINF_SUCCESS)
2673	return rc;
2674
2675	/*
2676	* We might be called when we shouldn't.
2677	*
2678	* The mode switching will ensure that the PD is resynced after every mode
2679	* switch. So, if we find ourselves here when in protected or real mode
2680	* we can safely clear the FF and return immediately.
2681	*/
2682	if (pVCpu->pgm.s.enmGuestMode <= PGMMODE_PROTECTED)
2683	{
2684	Assert((cr0 & (X86_CR0_PG \| X86_CR0_PE)) != (X86_CR0_PG \| X86_CR0_PE));
2685	Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL));
2686	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2687	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2688	return VINF_SUCCESS;
2689	}
2690
2691	/* If global pages are not supported, then all flushes are global. */
2692	if (!(cr4 & X86_CR4_PGE))
2693	fGlobal = true;
2694	LogFlow(("PGMSyncCR3: cr0=%RX64 cr3=%RX64 cr4=%RX64 fGlobal=%d[%d,%d]\n", cr0, cr3, cr4, fGlobal,
2695	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)));
2696
2697	/*
2698	* Check if we need to finish an aborted MapCR3 call (see PGMFlushTLB).
2699	* This should be done before SyncCR3.
2700	*/
2701	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MAP_CR3)
2702	{
2703	pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MAP_CR3;
2704
2705	RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
2706	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3);
2707	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2708	if (CPUMIsGuestVmxEptPagingEnabled(pVCpu))
2709	{
2710	RTGCPHYS GCPhysOut;
2711	int rc2 = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
2712	if (RT_SUCCESS(rc2))
2713	GCPhysCR3 = GCPhysOut;
2714	else
2715	{
2716	/* CR3 SLAT translation failed but we try to pretend it
2717	succeeded for the reasons mentioned in PGMHCChangeMode(). */
2718	AssertMsgFailed(("Failed to translate CR3 %#RX64. rc=%Rrc\n", cr3, rc2));
2719	rc2 = pgmGstUnmapCr3(pVCpu);
2720	pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
2721	pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
2722	return rc2;
2723	}
2724	}
2725	#endif
2726	Assert(!pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped));
2727	if (GCPhysOldCR3 != GCPhysCR3)
2728	{
2729	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2730	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2731	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2732	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2733	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
2734	}
2735
2736	/* Make sure we check for pending pgm pool syncs as we clear VMCPU_FF_PGM_SYNC_CR3 later on! */
2737	if ( rc == VINF_PGM_SYNC_CR3
2738	\|\| (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL))
2739	{
2740	Log(("PGMSyncCR3: pending pgm pool sync after MapCR3!\n"));
2741	#ifdef IN_RING3
2742	rc = pgmPoolSyncCR3(pVCpu);
2743	#else
2744	if (rc == VINF_PGM_SYNC_CR3)
2745	pVCpu->pgm.s.GCPhysCR3 = GCPhysOldCR3;
2746	return VINF_PGM_SYNC_CR3;
2747	#endif
2748	}
2749	AssertRCReturn(rc, rc);
2750	AssertRCSuccessReturn(rc, VERR_IPE_UNEXPECTED_INFO_STATUS);
2751	}
2752
2753	/*
2754	* Let the 'Bth' function do the work and we'll just keep track of the flags.
2755	*/
2756	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
2757
2758	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2759	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2760	AssertReturn(g_aPgmBothModeData[idxBth].pfnSyncCR3, VERR_PGM_MODE_IPE);
2761	rc = g_aPgmBothModeData[idxBth].pfnSyncCR3(pVCpu, cr0, cr3, cr4, fGlobal);
2762
2763	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
2764	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("rc=%Rrc\n", rc));
2765	if (rc == VINF_SUCCESS)
2766	{
2767	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2768	{
2769	/* Go back to ring 3 if a pgm pool sync is again pending. */
2770	return VINF_PGM_SYNC_CR3;
2771	}
2772
2773	if (!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_ALWAYS))
2774	{
2775	Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL));
2776	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2777	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2778	}
2779	}
2780
2781	/*
2782	* Now flush the CR3 (guest context).
2783	*/
2784	if (rc == VINF_SUCCESS)
2785	PGM_INVL_VCPU_TLBS(pVCpu);
2786	return rc;
2787	}
2788
2789
2790	/**
2791	* Maps all the PAE PDPE entries.
2792	*
2793	* @returns VBox status code.
2794	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2795	* @param paPaePdpes The new PAE PDPE values.
2796	*
2797	* @remarks This function may be invoked during the process of changing the guest
2798	* paging mode to PAE, hence the guest state (CR0, CR4 etc.) may not
2799	* reflect PAE paging just yet.
2800	*/
2801	VMM_INT_DECL(int) PGMGstMapPaePdpes(PVMCPUCC pVCpu, PCX86PDPE paPaePdpes)
2802	{
2803	Assert(paPaePdpes);
2804	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
2805	{
2806	X86PDPE const PaePdpe = paPaePdpes[i];
2807
2808	/*
2809	* In some cases (e.g. in SVM with nested paging) the validation of the PAE PDPEs
2810	* are deferred.[1] Also, different situations require different handling of invalid
2811	* PDPE entries. Here we assume the caller has already validated or doesn't require
2812	* validation of the PDPEs.
2813	*
2814	* [1] -- See AMD spec. 15.25.10 "Legacy PAE Mode".
2815	*/
2816	if ((PaePdpe.u & (pVCpu->pgm.s.fGstPaeMbzPdpeMask \| X86_PDPE_P)) == X86_PDPE_P)
2817	{
2818	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2819	RTHCPTR HCPtr;
2820	RTGCPHYS const GCPhys = PGM_A20_APPLY(pVCpu, PaePdpe.u & X86_PDPE_PG_MASK);
2821
2822	PGM_LOCK_VOID(pVM);
2823	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
2824	AssertReturnStmt(pPage, PGM_UNLOCK(pVM), VERR_PGM_INVALID_PDPE_ADDR);
2825	int const rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)&HCPtr);
2826	PGM_UNLOCK(pVM);
2827	if (RT_SUCCESS(rc))
2828	{
2829	#ifdef IN_RING3
2830	pVCpu->pgm.s.apGstPaePDsR3[i] = (PX86PDPAE)HCPtr;
2831	pVCpu->pgm.s.apGstPaePDsR0[i] = NIL_RTR0PTR;
2832	#else
2833	pVCpu->pgm.s.apGstPaePDsR3[i] = NIL_RTR3PTR;
2834	pVCpu->pgm.s.apGstPaePDsR0[i] = (PX86PDPAE)HCPtr;
2835	#endif
2836	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys;
2837	continue;
2838	}
2839	AssertMsgFailed(("PGMPhysMapPaePdpes: rc2=%d GCPhys=%RGp i=%d\n", rc, GCPhys, i));
2840	}
2841	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
2842	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
2843	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
2844	}
2845
2846	return VINF_SUCCESS;
2847	}
2848
2849
2850	/**
2851	* Validates and maps the PDPT and PAE PDPEs referenced by the given CR3.
2852	*
2853	* @returns VBox status code.
2854	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2855	* @param cr3 The guest CR3 value.
2856	*
2857	* @remarks This function may be invoked during the process of changing the guest
2858	* paging mode to PAE but the guest state (CR0, CR4 etc.) may not reflect
2859	* PAE paging just yet.
2860	*/
2861	VMM_INT_DECL(int) PGMGstMapPaePdpesAtCr3(PVMCPUCC pVCpu, uint64_t cr3)
2862	{
2863	/*
2864	* Read the page-directory-pointer table (PDPT) at CR3.
2865	*/
2866	RTGCPHYS GCPhysCR3 = (cr3 & X86_CR3_PAE_PAGE_MASK);
2867	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
2868
2869	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2870	if (CPUMIsGuestVmxEptPaePagingEnabled(pVCpu))
2871	{
2872	RTGCPHYS GCPhysOut;
2873	int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
2874	if (RT_SUCCESS(rc))
2875	GCPhysCR3 = GCPhysOut;
2876	else
2877	{
2878	AssertMsgFailed(("Failed to load CR3 at %#RX64. rc=%Rrc\n", GCPhysCR3, rc));
2879	return rc;
2880	}
2881	}
2882	#endif
2883
2884	RTHCPTR HCPtrGuestCr3;
2885	int rc = pgmGstMapCr3(pVCpu, GCPhysCR3, &HCPtrGuestCr3);
2886	if (RT_SUCCESS(rc))
2887	{
2888	/*
2889	* Validate the page-directory-pointer table entries (PDPE).
2890	*/
2891	X86PDPE aPaePdpes[X86_PG_PAE_PDPE_ENTRIES];
2892	memcpy(&aPaePdpes[0], HCPtrGuestCr3, sizeof(aPaePdpes));
2893	if (PGMGstArePaePdpesValid(pVCpu, &aPaePdpes[0]))
2894	{
2895	/*
2896	* Map the PDPT.
2897	* We deliberately don't update PGM's GCPhysCR3 here as it's expected
2898	* that PGMFlushTLB will be called soon and only a change to CR3 then
2899	* will cause the shadow page tables to be updated.
2900	*/
2901	#ifdef IN_RING3
2902	pVCpu->pgm.s.pGstPaePdptR3 = (PX86PDPT)HCPtrGuestCr3;
2903	pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
2904	#else
2905	pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
2906	pVCpu->pgm.s.pGstPaePdptR0 = (PX86PDPT)HCPtrGuestCr3;
2907	#endif
2908
2909	/*
2910	* Update CPUM.
2911	* We do this prior to mapping the PDPEs to keep the order consistent
2912	* with what's used in HM. In practice, it doesn't really matter.
2913	*/
2914	CPUMSetGuestPaePdpes(pVCpu, &aPaePdpes[0]);
2915
2916	/*
2917	* Map the PDPEs.
2918	*/
2919	rc = PGMGstMapPaePdpes(pVCpu, &aPaePdpes[0]);
2920	if (RT_SUCCESS(rc))
2921	{
2922	#ifdef IN_RING3
2923	pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = true;
2924	pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = false;
2925	#else
2926	pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = false;
2927	pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = true;
2928	#endif
2929	pVCpu->pgm.s.GCPhysPaeCR3 = GCPhysCR3;
2930	}
2931	}
2932	else
2933	rc = VERR_PGM_PAE_PDPE_RSVD;
2934	}
2935	return rc;
2936	}
2937
2938
2939	/**
2940	* Called whenever CR0 or CR4 in a way which may affect the paging mode.
2941	*
2942	* @returns VBox status code, with the following informational code for
2943	* VM scheduling.
2944	* @retval VINF_SUCCESS if the was no change, or it was successfully dealt with.
2945	* @retval VINF_EM_SUSPEND or VINF_EM_OFF on a fatal runtime error. (R3 only)
2946	*
2947	* @param pVCpu The cross context virtual CPU structure.
2948	* @param cr0 The new cr0.
2949	* @param cr4 The new cr4.
2950	* @param efer The new extended feature enable register.
2951	* @param fForce Whether to force a mode change.
2952	*/
2953	VMMDECL(int) PGMChangeMode(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr4, uint64_t efer, bool fForce)
2954	{
2955	VMCPU_ASSERT_EMT(pVCpu);
2956
2957	/*
2958	* Calc the new guest mode.
2959	*
2960	* Note! We check PG before PE and without requiring PE because of the
2961	* special AMD-V paged real mode (APM vol 2, rev 3.28, 15.9).
2962	*/
2963	PGMMODE enmGuestMode;
2964	if (cr0 & X86_CR0_PG)
2965	{
2966	if (!(cr4 & X86_CR4_PAE))
2967	{
2968	bool const fPse = !!(cr4 & X86_CR4_PSE);
2969	if (pVCpu->pgm.s.fGst32BitPageSizeExtension != fPse)
2970	Log(("PGMChangeMode: CR4.PSE %d -> %d\n", pVCpu->pgm.s.fGst32BitPageSizeExtension, fPse));
2971	pVCpu->pgm.s.fGst32BitPageSizeExtension = fPse;
2972	enmGuestMode = PGMMODE_32_BIT;
2973	}
2974	else if (!(efer & MSR_K6_EFER_LME))
2975	{
2976	if (!(efer & MSR_K6_EFER_NXE))
2977	enmGuestMode = PGMMODE_PAE;
2978	else
2979	enmGuestMode = PGMMODE_PAE_NX;
2980	}
2981	else
2982	{
2983	if (!(efer & MSR_K6_EFER_NXE))
2984	enmGuestMode = PGMMODE_AMD64;
2985	else
2986	enmGuestMode = PGMMODE_AMD64_NX;
2987	}
2988	}
2989	else if (!(cr0 & X86_CR0_PE))
2990	enmGuestMode = PGMMODE_REAL;
2991	else
2992	enmGuestMode = PGMMODE_PROTECTED;
2993
2994	/*
2995	* Did it change?
2996	*/
2997	if ( !fForce
2998	&& pVCpu->pgm.s.enmGuestMode == enmGuestMode)
2999	return VINF_SUCCESS;
3000
3001	/* Flush the TLB */
3002	PGM_INVL_VCPU_TLBS(pVCpu);
3003	return PGMHCChangeMode(pVCpu->CTX_SUFF(pVM), pVCpu, enmGuestMode, fForce);
3004	}
3005
3006
3007	/**
3008	* Converts a PGMMODE value to a PGM_TYPE_* \#define.
3009	*
3010	* @returns PGM_TYPE_*.
3011	* @param pgmMode The mode value to convert.
3012	*/
3013	DECLINLINE(unsigned) pgmModeToType(PGMMODE pgmMode)
3014	{
3015	switch (pgmMode)
3016	{
3017	case PGMMODE_REAL: return PGM_TYPE_REAL;
3018	case PGMMODE_PROTECTED: return PGM_TYPE_PROT;
3019	case PGMMODE_32_BIT: return PGM_TYPE_32BIT;
3020	case PGMMODE_PAE:
3021	case PGMMODE_PAE_NX: return PGM_TYPE_PAE;
3022	case PGMMODE_AMD64:
3023	case PGMMODE_AMD64_NX: return PGM_TYPE_AMD64;
3024	case PGMMODE_NESTED_32BIT: return PGM_TYPE_NESTED_32BIT;
3025	case PGMMODE_NESTED_PAE: return PGM_TYPE_NESTED_PAE;
3026	case PGMMODE_NESTED_AMD64: return PGM_TYPE_NESTED_AMD64;
3027	case PGMMODE_EPT: return PGM_TYPE_EPT;
3028	case PGMMODE_NONE: return PGM_TYPE_NONE;
3029	default:
3030	AssertFatalMsgFailed(("pgmMode=%d\n", pgmMode));
3031	}
3032	}
3033
3034
3035	/**
3036	* Calculates the shadow paging mode.
3037	*
3038	* @returns The shadow paging mode.
3039	* @param pVM The cross context VM structure.
3040	* @param enmGuestMode The guest mode.
3041	* @param enmHostMode The host mode.
3042	* @param enmShadowMode The current shadow mode.
3043	*/
3044	static PGMMODE pgmCalcShadowMode(PVMCC pVM, PGMMODE enmGuestMode, SUPPAGINGMODE enmHostMode, PGMMODE enmShadowMode)
3045	{
3046	switch (enmGuestMode)
3047	{
3048	case PGMMODE_REAL:
3049	case PGMMODE_PROTECTED:
3050	switch (enmHostMode)
3051	{
3052	case SUPPAGINGMODE_32_BIT:
3053	case SUPPAGINGMODE_32_BIT_GLOBAL:
3054	enmShadowMode = PGMMODE_32_BIT;
3055	break;
3056
3057	case SUPPAGINGMODE_PAE:
3058	case SUPPAGINGMODE_PAE_NX:
3059	case SUPPAGINGMODE_PAE_GLOBAL:
3060	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3061	enmShadowMode = PGMMODE_PAE;
3062	break;
3063
3064	case SUPPAGINGMODE_AMD64:
3065	case SUPPAGINGMODE_AMD64_GLOBAL:
3066	case SUPPAGINGMODE_AMD64_NX:
3067	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3068	enmShadowMode = PGMMODE_PAE;
3069	break;
3070
3071	default:
3072	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3073	}
3074	break;
3075
3076	case PGMMODE_32_BIT:
3077	switch (enmHostMode)
3078	{
3079	case SUPPAGINGMODE_32_BIT:
3080	case SUPPAGINGMODE_32_BIT_GLOBAL:
3081	enmShadowMode = PGMMODE_32_BIT;
3082	break;
3083
3084	case SUPPAGINGMODE_PAE:
3085	case SUPPAGINGMODE_PAE_NX:
3086	case SUPPAGINGMODE_PAE_GLOBAL:
3087	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3088	enmShadowMode = PGMMODE_PAE;
3089	break;
3090
3091	case SUPPAGINGMODE_AMD64:
3092	case SUPPAGINGMODE_AMD64_GLOBAL:
3093	case SUPPAGINGMODE_AMD64_NX:
3094	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3095	enmShadowMode = PGMMODE_PAE;
3096	break;
3097
3098	default:
3099	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3100	}
3101	break;
3102
3103	case PGMMODE_PAE:
3104	case PGMMODE_PAE_NX: /** @todo This might require more switchers and guest+both modes. */
3105	switch (enmHostMode)
3106	{
3107	case SUPPAGINGMODE_32_BIT:
3108	case SUPPAGINGMODE_32_BIT_GLOBAL:
3109	enmShadowMode = PGMMODE_PAE;
3110	break;
3111
3112	case SUPPAGINGMODE_PAE:
3113	case SUPPAGINGMODE_PAE_NX:
3114	case SUPPAGINGMODE_PAE_GLOBAL:
3115	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3116	enmShadowMode = PGMMODE_PAE;
3117	break;
3118
3119	case SUPPAGINGMODE_AMD64:
3120	case SUPPAGINGMODE_AMD64_GLOBAL:
3121	case SUPPAGINGMODE_AMD64_NX:
3122	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3123	enmShadowMode = PGMMODE_PAE;
3124	break;
3125
3126	default:
3127	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3128	}
3129	break;
3130
3131	case PGMMODE_AMD64:
3132	case PGMMODE_AMD64_NX:
3133	switch (enmHostMode)
3134	{
3135	case SUPPAGINGMODE_32_BIT:
3136	case SUPPAGINGMODE_32_BIT_GLOBAL:
3137	enmShadowMode = PGMMODE_AMD64;
3138	break;
3139
3140	case SUPPAGINGMODE_PAE:
3141	case SUPPAGINGMODE_PAE_NX:
3142	case SUPPAGINGMODE_PAE_GLOBAL:
3143	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3144	enmShadowMode = PGMMODE_AMD64;
3145	break;
3146
3147	case SUPPAGINGMODE_AMD64:
3148	case SUPPAGINGMODE_AMD64_GLOBAL:
3149	case SUPPAGINGMODE_AMD64_NX:
3150	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3151	enmShadowMode = PGMMODE_AMD64;
3152	break;
3153
3154	default:
3155	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3156	}
3157	break;
3158
3159	default:
3160	AssertLogRelMsgFailedReturn(("enmGuestMode=%d\n", enmGuestMode), PGMMODE_INVALID);
3161	}
3162
3163	/*
3164	* Override the shadow mode when NEM, IEM or nested paging is active.
3165	*/
3166	if (!VM_IS_HM_ENABLED(pVM))
3167	{
3168	Assert(VM_IS_NEM_ENABLED(pVM) \|\| VM_IS_EXEC_ENGINE_IEM(pVM));
3169	pVM->pgm.s.fNestedPaging = true;
3170	enmShadowMode = PGMMODE_NONE;
3171	}
3172	else
3173	{
3174	bool fNestedPaging = HMIsNestedPagingActive(pVM);
3175	pVM->pgm.s.fNestedPaging = fNestedPaging;
3176	if (fNestedPaging)
3177	{
3178	if (HMIsVmxActive(pVM))
3179	enmShadowMode = PGMMODE_EPT;
3180	else
3181	{
3182	/* The nested SVM paging depends on the host one. */
3183	Assert(HMIsSvmActive(pVM));
3184	if ( enmGuestMode == PGMMODE_AMD64
3185	\|\| enmGuestMode == PGMMODE_AMD64_NX)
3186	enmShadowMode = PGMMODE_NESTED_AMD64;
3187	else
3188	switch (pVM->pgm.s.enmHostMode)
3189	{
3190	case SUPPAGINGMODE_32_BIT:
3191	case SUPPAGINGMODE_32_BIT_GLOBAL:
3192	enmShadowMode = PGMMODE_NESTED_32BIT;
3193	break;
3194
3195	case SUPPAGINGMODE_PAE:
3196	case SUPPAGINGMODE_PAE_GLOBAL:
3197	case SUPPAGINGMODE_PAE_NX:
3198	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3199	enmShadowMode = PGMMODE_NESTED_PAE;
3200	break;
3201
3202	case SUPPAGINGMODE_AMD64:
3203	case SUPPAGINGMODE_AMD64_GLOBAL:
3204	case SUPPAGINGMODE_AMD64_NX:
3205	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3206	enmShadowMode = PGMMODE_NESTED_AMD64;
3207	break;
3208
3209	default:
3210	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode), PGMMODE_INVALID);
3211	}
3212	}
3213	}
3214	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3215	else
3216	{
3217	/* Nested paging is a requirement for nested VT-x. */
3218	AssertLogRelMsgReturn(enmGuestMode != PGMMODE_EPT, ("enmHostMode=%d\n", pVM->pgm.s.enmHostMode), PGMMODE_INVALID);
3219	}
3220	#endif
3221	}
3222
3223	return enmShadowMode;
3224	}
3225
3226
3227	/**
3228	* Performs the actual mode change.
3229	* This is called by PGMChangeMode and pgmR3InitPaging().
3230	*
3231	* @returns VBox status code. May suspend or power off the VM on error, but this
3232	* will trigger using FFs and not informational status codes.
3233	*
3234	* @param pVM The cross context VM structure.
3235	* @param pVCpu The cross context virtual CPU structure.
3236	* @param enmGuestMode The new guest mode. This is assumed to be different from
3237	* the current mode.
3238	* @param fForce Whether to force a shadow paging mode change.
3239	*/
3240	VMM_INT_DECL(int) PGMHCChangeMode(PVMCC pVM, PVMCPUCC pVCpu, PGMMODE enmGuestMode, bool fForce)
3241	{
3242	Log(("PGMHCChangeMode: Guest mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmGuestMode), PGMGetModeName(enmGuestMode)));
3243	STAM_REL_COUNTER_INC(&pVCpu->pgm.s.cGuestModeChanges);
3244
3245	/*
3246	* Calc the shadow mode and switcher.
3247	*/
3248	PGMMODE const enmShadowMode = pgmCalcShadowMode(pVM, enmGuestMode, pVM->pgm.s.enmHostMode, pVCpu->pgm.s.enmShadowMode);
3249	bool const fShadowModeChanged = enmShadowMode != pVCpu->pgm.s.enmShadowMode \|\| fForce;
3250
3251	/*
3252	* Exit old mode(s).
3253	*/
3254	/* shadow */
3255	if (fShadowModeChanged)
3256	{
3257	LogFlow(("PGMHCChangeMode: Shadow mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode), PGMGetModeName(enmShadowMode)));
3258	uintptr_t idxOldShw = pVCpu->pgm.s.idxShadowModeData;
3259	if ( idxOldShw < RT_ELEMENTS(g_aPgmShadowModeData)
3260	&& g_aPgmShadowModeData[idxOldShw].pfnExit)
3261	{
3262	int rc = g_aPgmShadowModeData[idxOldShw].pfnExit(pVCpu);
3263	AssertMsgRCReturn(rc, ("Exit failed for shadow mode %d: %Rrc\n", pVCpu->pgm.s.enmShadowMode, rc), rc);
3264	}
3265	}
3266	else
3267	LogFlow(("PGMHCChangeMode: Shadow mode remains: %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode)));
3268
3269	/* guest */
3270	uintptr_t const idxOldGst = pVCpu->pgm.s.idxGuestModeData;
3271	if ( idxOldGst < RT_ELEMENTS(g_aPgmGuestModeData)
3272	&& g_aPgmGuestModeData[idxOldGst].pfnExit)
3273	{
3274	int rc = g_aPgmGuestModeData[idxOldGst].pfnExit(pVCpu);
3275	AssertMsgReturn(RT_SUCCESS(rc), ("Exit failed for guest mode %d: %Rrc\n", pVCpu->pgm.s.enmGuestMode, rc), rc);
3276	}
3277	pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
3278	pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
3279	pVCpu->pgm.s.GCPhysPaeCR3 = NIL_RTGCPHYS;
3280	Assert(!pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped));
3281
3282	/*
3283	* Change the paging mode data indexes.
3284	*/
3285	uintptr_t idxNewGst = pVCpu->pgm.s.idxGuestModeData = pgmModeToType(enmGuestMode);
3286	AssertReturn(idxNewGst < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
3287	AssertReturn(g_aPgmGuestModeData[idxNewGst].uType == idxNewGst, VERR_PGM_MODE_IPE);
3288	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnGetPage, VERR_PGM_MODE_IPE);
3289	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnModifyPage, VERR_PGM_MODE_IPE);
3290	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnExit, VERR_PGM_MODE_IPE);
3291	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnEnter, VERR_PGM_MODE_IPE);
3292	#ifdef IN_RING3
3293	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnRelocate, VERR_PGM_MODE_IPE);
3294	#endif
3295
3296	uintptr_t const idxNewShw = pVCpu->pgm.s.idxShadowModeData = pgmModeToType(enmShadowMode);
3297	AssertReturn(idxNewShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
3298	AssertReturn(g_aPgmShadowModeData[idxNewShw].uType == idxNewShw, VERR_PGM_MODE_IPE);
3299	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnGetPage, VERR_PGM_MODE_IPE);
3300	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnModifyPage, VERR_PGM_MODE_IPE);
3301	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnExit, VERR_PGM_MODE_IPE);
3302	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnEnter, VERR_PGM_MODE_IPE);
3303	#ifdef IN_RING3
3304	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnRelocate, VERR_PGM_MODE_IPE);
3305	#endif
3306
3307	uintptr_t const idxNewBth = pVCpu->pgm.s.idxBothModeData = (idxNewShw - PGM_TYPE_FIRST_SHADOW) * PGM_TYPE_END + idxNewGst;
3308	AssertReturn(g_aPgmBothModeData[idxNewBth].uShwType == idxNewShw, VERR_PGM_MODE_IPE);
3309	AssertReturn(g_aPgmBothModeData[idxNewBth].uGstType == idxNewGst, VERR_PGM_MODE_IPE);
3310	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnInvalidatePage, VERR_PGM_MODE_IPE);
3311	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnSyncCR3, VERR_PGM_MODE_IPE);
3312	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnPrefetchPage, VERR_PGM_MODE_IPE);
3313	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnVerifyAccessSyncPage, VERR_PGM_MODE_IPE);
3314	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnMapCR3, VERR_PGM_MODE_IPE);
3315	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnUnmapCR3, VERR_PGM_MODE_IPE);
3316	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnEnter, VERR_PGM_MODE_IPE);
3317	#ifdef VBOX_STRICT
3318	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnAssertCR3, VERR_PGM_MODE_IPE);
3319	#endif
3320
3321	/*
3322	* Enter new shadow mode (if changed).
3323	*/
3324	if (fShadowModeChanged)
3325	{
3326	pVCpu->pgm.s.enmShadowMode = enmShadowMode;
3327	int rc = g_aPgmShadowModeData[idxNewShw].pfnEnter(pVCpu, enmGuestMode >= PGMMODE_AMD64);
3328	AssertLogRelMsgRCReturnStmt(rc, ("Entering enmShadowMode=%s failed: %Rrc\n", PGMGetModeName(enmShadowMode), rc),
3329	pVCpu->pgm.s.enmShadowMode = PGMMODE_INVALID, rc);
3330	}
3331
3332	/*
3333	* Always flag the necessary updates
3334	*/
3335	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3336
3337	/*
3338	* Enter the new guest and shadow+guest modes.
3339	*/
3340	/* Calc the new CR3 value. */
3341	RTGCPHYS GCPhysCR3;
3342	switch (enmGuestMode)
3343	{
3344	case PGMMODE_REAL:
3345	case PGMMODE_PROTECTED:
3346	GCPhysCR3 = NIL_RTGCPHYS;
3347	break;
3348
3349	case PGMMODE_32_BIT:
3350	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK;
3351	break;
3352
3353	case PGMMODE_PAE_NX:
3354	case PGMMODE_PAE:
3355	if (!pVM->cpum.ro.GuestFeatures.fPae)
3356	#ifdef IN_RING3 /** @todo r=bird: wrong place, probably hasn't really worked for a while. */
3357	return VMSetRuntimeError(pVM, VMSETRTERR_FLAGS_FATAL, "PAEmode",
3358	N_("The guest is trying to switch to the PAE mode which is currently disabled by default in VirtualBox. PAE support can be enabled using the VM settings (System/Processor)"));
3359	#else
3360	AssertLogRelMsgFailedReturn(("enmGuestMode=%s - Try enable PAE for the guest!\n", PGMGetModeName(enmGuestMode)), VERR_PGM_MODE_IPE);
3361
3362	#endif
3363	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAE_PAGE_MASK;
3364	break;
3365
3366	#ifdef VBOX_WITH_64_BITS_GUESTS
3367	case PGMMODE_AMD64_NX:
3368	case PGMMODE_AMD64:
3369	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_AMD64_PAGE_MASK;
3370	break;
3371	#endif
3372	default:
3373	AssertLogRelMsgFailedReturn(("enmGuestMode=%d\n", enmGuestMode), VERR_PGM_MODE_IPE);
3374	}
3375
3376	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3377	/*
3378	* If a nested-guest is using EPT paging:
3379	* - Update the second-level address translation (SLAT) mode.
3380	* - Indicate that the CR3 is nested-guest physical address.
3381	*/
3382	if (CPUMIsGuestVmxEptPagingEnabled(pVCpu))
3383	{
3384	if (PGMMODE_WITH_PAGING(enmGuestMode))
3385	{
3386	/*
3387	* Translate CR3 to its guest-physical address.
3388	* We don't use pgmGstSlatTranslateCr3() here as we want to update GCPhysNstGstCR3 -after-
3389	* switching modes to keep it consistent with how GCPhysCR3 is updated.
3390	*/
3391	PGMPTWALK Walk;
3392	PGMPTWALKGST GstWalk;
3393	int const rc = pgmGstSlatWalkPhys(pVCpu, PGMSLAT_EPT, GCPhysCR3, &Walk, &GstWalk);
3394	if (RT_SUCCESS(rc))
3395	{ /* likely */ }
3396	else
3397	{
3398	/*
3399	* SLAT failed but we avoid reporting this to the caller because the caller
3400	* is not supposed to fail. The only time the caller needs to indicate a
3401	* failure to software is when PAE paging is used by the nested-guest, but
3402	* we handle the PAE case separately (e.g., see VMX transition in IEM).
3403	* In all other cases, the failure will be indicated when CR3 tries to be
3404	* translated on the next linear-address memory access.
3405	* See Intel spec. 27.2.1 "EPT Overview".
3406	*/
3407	AssertMsgFailed(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", GCPhysCR3, rc));
3408
3409	/* Trying to coax PGM to succeed for the time being... */
3410	Assert(pVCpu->pgm.s.GCPhysCR3 == NIL_RTGCPHYS);
3411	pVCpu->pgm.s.GCPhysNstGstCR3 = GCPhysCR3;
3412	pVCpu->pgm.s.enmGuestSlatMode = PGMSLAT_EPT;
3413	pVCpu->pgm.s.enmGuestMode = enmGuestMode;
3414	HMHCChangedPagingMode(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
3415	return VINF_SUCCESS;
3416	}
3417	pVCpu->pgm.s.GCPhysNstGstCR3 = GCPhysCR3;
3418	GCPhysCR3 = Walk.GCPhys;
3419	}
3420	pVCpu->pgm.s.enmGuestSlatMode = PGMSLAT_EPT;
3421	}
3422	else
3423	{
3424	Assert(pVCpu->pgm.s.GCPhysNstGstCR3 == NIL_RTGCPHYS);
3425	pVCpu->pgm.s.enmGuestSlatMode = PGMSLAT_DIRECT;
3426	}
3427	#endif
3428
3429	/*
3430	* Enter the new guest mode.
3431	*/
3432	pVCpu->pgm.s.enmGuestMode = enmGuestMode;
3433	int rc = g_aPgmGuestModeData[idxNewGst].pfnEnter(pVCpu, GCPhysCR3);
3434	int rc2 = g_aPgmBothModeData[idxNewBth].pfnEnter(pVCpu, GCPhysCR3);
3435
3436	/* Set the new guest CR3 (and nested-guest CR3). */
3437	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
3438
3439	/* status codes. */
3440	AssertRC(rc);
3441	AssertRC(rc2);
3442	if (RT_SUCCESS(rc))
3443	{
3444	rc = rc2;
3445	if (RT_SUCCESS(rc)) /* no informational status codes. */
3446	rc = VINF_SUCCESS;
3447	}
3448
3449	/*
3450	* Notify HM.
3451	*/
3452	HMHCChangedPagingMode(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
3453	return rc;
3454	}
3455
3456
3457	/**
3458	* Called by CPUM or REM when CR0.WP changes to 1.
3459	*
3460	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
3461	* @thread EMT
3462	*/
3463	VMMDECL(void) PGMCr0WpEnabled(PVMCPUCC pVCpu)
3464	{
3465	/*
3466	* Netware WP0+RO+US hack cleanup when WP0 -> WP1.
3467	*
3468	* Use the counter to judge whether there might be pool pages with active
3469	* hacks in them. If there are, we will be running the risk of messing up
3470	* the guest by allowing it to write to read-only pages. Thus, we have to
3471	* clear the page pool ASAP if there is the slightest chance.
3472	*/
3473	if (pVCpu->pgm.s.cNetwareWp0Hacks > 0)
3474	{
3475	Assert(pVCpu->CTX_SUFF(pVM)->cCpus == 1);
3476
3477	Log(("PGMCr0WpEnabled: %llu WP0 hacks active - clearing page pool\n", pVCpu->pgm.s.cNetwareWp0Hacks));
3478	pVCpu->pgm.s.cNetwareWp0Hacks = 0;
3479	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
3480	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3481	}
3482	}
3483
3484
3485	/**
3486	* Gets the current guest paging mode.
3487	*
3488	* If you just need the CPU mode (real/protected/long), use CPUMGetGuestMode().
3489	*
3490	* @returns The current paging mode.
3491	* @param pVCpu The cross context virtual CPU structure.
3492	*/
3493	VMMDECL(PGMMODE) PGMGetGuestMode(PVMCPU pVCpu)
3494	{
3495	return pVCpu->pgm.s.enmGuestMode;
3496	}
3497
3498
3499	/**
3500	* Gets the current shadow paging mode.
3501	*
3502	* @returns The current paging mode.
3503	* @param pVCpu The cross context virtual CPU structure.
3504	*/
3505	VMMDECL(PGMMODE) PGMGetShadowMode(PVMCPU pVCpu)
3506	{
3507	return pVCpu->pgm.s.enmShadowMode;
3508	}
3509
3510
3511	/**
3512	* Gets the current host paging mode.
3513	*
3514	* @returns The current paging mode.
3515	* @param pVM The cross context VM structure.
3516	*/
3517	VMMDECL(PGMMODE) PGMGetHostMode(PVM pVM)
3518	{
3519	switch (pVM->pgm.s.enmHostMode)
3520	{
3521	case SUPPAGINGMODE_32_BIT:
3522	case SUPPAGINGMODE_32_BIT_GLOBAL:
3523	return PGMMODE_32_BIT;
3524
3525	case SUPPAGINGMODE_PAE:
3526	case SUPPAGINGMODE_PAE_GLOBAL:
3527	return PGMMODE_PAE;
3528
3529	case SUPPAGINGMODE_PAE_NX:
3530	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3531	return PGMMODE_PAE_NX;
3532
3533	case SUPPAGINGMODE_AMD64:
3534	case SUPPAGINGMODE_AMD64_GLOBAL:
3535	return PGMMODE_AMD64;
3536
3537	case SUPPAGINGMODE_AMD64_NX:
3538	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3539	return PGMMODE_AMD64_NX;
3540
3541	default: AssertMsgFailed(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode)); break;
3542	}
3543
3544	return PGMMODE_INVALID;
3545	}
3546
3547
3548	/**
3549	* Get mode name.
3550	*
3551	* @returns read-only name string.
3552	* @param enmMode The mode which name is desired.
3553	*/
3554	VMMDECL(const char *) PGMGetModeName(PGMMODE enmMode)
3555	{
3556	switch (enmMode)
3557	{
3558	case PGMMODE_REAL: return "Real";
3559	case PGMMODE_PROTECTED: return "Protected";
3560	case PGMMODE_32_BIT: return "32-bit";
3561	case PGMMODE_PAE: return "PAE";
3562	case PGMMODE_PAE_NX: return "PAE+NX";
3563	case PGMMODE_AMD64: return "AMD64";
3564	case PGMMODE_AMD64_NX: return "AMD64+NX";
3565	case PGMMODE_NESTED_32BIT: return "Nested-32";
3566	case PGMMODE_NESTED_PAE: return "Nested-PAE";
3567	case PGMMODE_NESTED_AMD64: return "Nested-AMD64";
3568	case PGMMODE_EPT: return "EPT";
3569	case PGMMODE_NONE: return "None";
3570	default: return "unknown mode value";
3571	}
3572	}
3573
3574
3575	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3576	/**
3577	* Gets the SLAT mode name.
3578	*
3579	* @returns The read-only SLAT mode descriptive string.
3580	* @param enmSlatMode The SLAT mode value.
3581	*/
3582	VMM_INT_DECL(const char *) PGMGetSlatModeName(PGMSLAT enmSlatMode)
3583	{
3584	switch (enmSlatMode)
3585	{
3586	case PGMSLAT_DIRECT: return "Direct";
3587	case PGMSLAT_EPT: return "EPT";
3588	case PGMSLAT_32BIT: return "32-bit";
3589	case PGMSLAT_PAE: return "PAE";
3590	case PGMSLAT_AMD64: return "AMD64";
3591	default: return "Unknown";
3592	}
3593	}
3594	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
3595
3596
3597	/**
3598	* Gets the physical address represented in the guest CR3 as PGM sees it.
3599	*
3600	* This is mainly for logging and debugging.
3601	*
3602	* @returns PGM's guest CR3 value.
3603	* @param pVCpu The cross context virtual CPU structure.
3604	*/
3605	VMM_INT_DECL(RTGCPHYS) PGMGetGuestCR3Phys(PVMCPU pVCpu)
3606	{
3607	return pVCpu->pgm.s.GCPhysCR3;
3608	}
3609
3610
3611
3612	/**
3613	* Notification from CPUM that the EFER.NXE bit has changed.
3614	*
3615	* @param pVCpu The cross context virtual CPU structure of the CPU for
3616	* which EFER changed.
3617	* @param fNxe The new NXE state.
3618	*/
3619	VMM_INT_DECL(void) PGMNotifyNxeChanged(PVMCPU pVCpu, bool fNxe)
3620	{
3621	/** @todo VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu); */
3622	Log(("PGMNotifyNxeChanged: fNxe=%RTbool\n", fNxe));
3623
3624	pVCpu->pgm.s.fNoExecuteEnabled = fNxe;
3625	if (fNxe)
3626	{
3627	/pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A /
3628	pVCpu->pgm.s.fGstPaeMbzPteMask &= ~X86_PTE_PAE_NX;
3629	pVCpu->pgm.s.fGstPaeMbzPdeMask &= ~X86_PDE_PAE_NX;
3630	pVCpu->pgm.s.fGstPaeMbzBigPdeMask &= ~X86_PDE2M_PAE_NX;
3631	/pVCpu->pgm.s.fGstPaeMbzPdpeMask - N/A /
3632	pVCpu->pgm.s.fGstAmd64MbzPteMask &= ~X86_PTE_PAE_NX;
3633	pVCpu->pgm.s.fGstAmd64MbzPdeMask &= ~X86_PDE_PAE_NX;
3634	pVCpu->pgm.s.fGstAmd64MbzBigPdeMask &= ~X86_PDE2M_PAE_NX;
3635	pVCpu->pgm.s.fGstAmd64MbzPdpeMask &= ~X86_PDPE_LM_NX;
3636	pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask &= ~X86_PDPE_LM_NX;
3637	pVCpu->pgm.s.fGstAmd64MbzPml4eMask &= ~X86_PML4E_NX;
3638
3639	pVCpu->pgm.s.fGst64ShadowedPteMask \|= X86_PTE_PAE_NX;
3640	pVCpu->pgm.s.fGst64ShadowedPdeMask \|= X86_PDE_PAE_NX;
3641	pVCpu->pgm.s.fGst64ShadowedBigPdeMask \|= X86_PDE2M_PAE_NX;
3642	pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask \|= X86_PDE2M_PAE_NX;
3643	pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask \|= X86_PDPE_LM_NX;
3644	pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask \|= X86_PML4E_NX;
3645	}
3646	else
3647	{
3648	/pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A /
3649	pVCpu->pgm.s.fGstPaeMbzPteMask \|= X86_PTE_PAE_NX;
3650	pVCpu->pgm.s.fGstPaeMbzPdeMask \|= X86_PDE_PAE_NX;
3651	pVCpu->pgm.s.fGstPaeMbzBigPdeMask \|= X86_PDE2M_PAE_NX;
3652	/pVCpu->pgm.s.fGstPaeMbzPdpeMask -N/A /
3653	pVCpu->pgm.s.fGstAmd64MbzPteMask \|= X86_PTE_PAE_NX;
3654	pVCpu->pgm.s.fGstAmd64MbzPdeMask \|= X86_PDE_PAE_NX;
3655	pVCpu->pgm.s.fGstAmd64MbzBigPdeMask \|= X86_PDE2M_PAE_NX;
3656	pVCpu->pgm.s.fGstAmd64MbzPdpeMask \|= X86_PDPE_LM_NX;
3657	pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask \|= X86_PDPE_LM_NX;
3658	pVCpu->pgm.s.fGstAmd64MbzPml4eMask \|= X86_PML4E_NX;
3659
3660	pVCpu->pgm.s.fGst64ShadowedPteMask &= ~X86_PTE_PAE_NX;
3661	pVCpu->pgm.s.fGst64ShadowedPdeMask &= ~X86_PDE_PAE_NX;
3662	pVCpu->pgm.s.fGst64ShadowedBigPdeMask &= ~X86_PDE2M_PAE_NX;
3663	pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask &= ~X86_PDE2M_PAE_NX;
3664	pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask &= ~X86_PDPE_LM_NX;
3665	pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask &= ~X86_PML4E_NX;
3666	}
3667	}
3668
3669
3670	/**
3671	* Check if any pgm pool pages are marked dirty (not monitored)
3672	*
3673	* @returns bool locked/not locked
3674	* @param pVM The cross context VM structure.
3675	*/
3676	VMMDECL(bool) PGMHasDirtyPages(PVM pVM)
3677	{
3678	return pVM->pgm.s.CTX_SUFF(pPool)->cDirtyPages != 0;
3679	}
3680
3681
3682	/**
3683	* Check if this VCPU currently owns the PGM lock.
3684	*
3685	* @returns bool owner/not owner
3686	* @param pVM The cross context VM structure.
3687	*/
3688	VMMDECL(bool) PGMIsLockOwner(PVMCC pVM)
3689	{
3690	return PDMCritSectIsOwner(pVM, &pVM->pgm.s.CritSectX);
3691	}
3692
3693
3694	/**
3695	* Enable or disable large page usage
3696	*
3697	* @returns VBox status code.
3698	* @param pVM The cross context VM structure.
3699	* @param fUseLargePages Use/not use large pages
3700	*/
3701	VMMDECL(int) PGMSetLargePageUsage(PVMCC pVM, bool fUseLargePages)
3702	{
3703	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
3704
3705	pVM->pgm.s.fUseLargePages = fUseLargePages;
3706	return VINF_SUCCESS;
3707	}
3708
3709
3710	/**
3711	* Acquire the PGM lock.
3712	*
3713	* @returns VBox status code
3714	* @param pVM The cross context VM structure.
3715	* @param fVoid Set if the caller cannot handle failure returns.
3716	* @param SRC_POS The source position of the caller (RT_SRC_POS).
3717	*/
3718	#if defined(VBOX_STRICT) \|\| defined(DOXYGEN_RUNNING)
3719	int pgmLockDebug(PVMCC pVM, bool fVoid, RT_SRC_POS_DECL)
3720	#else
3721	int pgmLock(PVMCC pVM, bool fVoid)
3722	#endif
3723	{
3724	#if defined(VBOX_STRICT)
3725	int rc = PDMCritSectEnterDebug(pVM, &pVM->pgm.s.CritSectX, VINF_SUCCESS, (uintptr_t)ASMReturnAddress(), RT_SRC_POS_ARGS);
3726	#else
3727	int rc = PDMCritSectEnter(pVM, &pVM->pgm.s.CritSectX, VINF_SUCCESS);
3728	#endif
3729	if (RT_SUCCESS(rc))
3730	return rc;
3731	if (fVoid)
3732	PDM_CRITSECT_RELEASE_ASSERT_RC(pVM, &pVM->pgm.s.CritSectX, rc);
3733	else
3734	AssertRC(rc);
3735	return rc;
3736	}
3737
3738
3739	/**
3740	* Release the PGM lock.
3741	*
3742	* @returns VBox status code
3743	* @param pVM The cross context VM structure.
3744	*/
3745	void pgmUnlock(PVMCC pVM)
3746	{
3747	uint32_t cDeprecatedPageLocks = pVM->pgm.s.cDeprecatedPageLocks;
3748	pVM->pgm.s.cDeprecatedPageLocks = 0;
3749	int rc = PDMCritSectLeave(pVM, &pVM->pgm.s.CritSectX);
3750	if (rc == VINF_SEM_NESTED)
3751	pVM->pgm.s.cDeprecatedPageLocks = cDeprecatedPageLocks;
3752	}
3753
3754
3755	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
3756
3757	/** Format handler for PGMPAGE.
3758	* @copydoc FNRTSTRFORMATTYPE */
3759	static DECLCALLBACK(size_t) pgmFormatTypeHandlerPage(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
3760	const char pszType, void const pvValue,
3761	int cchWidth, int cchPrecision, unsigned fFlags,
3762	void *pvUser)
3763	{
3764	size_t cch;
3765	PCPGMPAGE pPage = (PCPGMPAGE)pvValue;
3766	if (RT_VALID_PTR(pPage))
3767	{
3768	char szTmp[64+80];
3769
3770	cch = 0;
3771
3772	/* The single char state stuff. */
3773	static const char s_achPageStates[4] = { 'Z', 'A', 'W', 'S' };
3774	szTmp[cch++] = s_achPageStates[PGM_PAGE_GET_STATE_NA(pPage)];
3775
3776	# define IS_PART_INCLUDED(lvl) ( !(fFlags & RTSTR_F_PRECISION) \|\| cchPrecision == (lvl) \|\| cchPrecision >= (lvl)+10 )
3777	if (IS_PART_INCLUDED(5))
3778	{
3779	static const char s_achHandlerStates[4] = { '-', 't', 'w', 'a' };
3780	szTmp[cch++] = s_achHandlerStates[PGM_PAGE_GET_HNDL_PHYS_STATE(pPage)];
3781	}
3782
3783	/* The type. */
3784	if (IS_PART_INCLUDED(4))
3785	{
3786	szTmp[cch++] = ':';
3787	static const char s_achPageTypes[8][4] = { "INV", "RAM", "MI2", "M2A", "SHA", "ROM", "MIO", "BAD" };
3788	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][0];
3789	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][1];
3790	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][2];
3791	}
3792
3793	/* The numbers. */
3794	if (IS_PART_INCLUDED(3))
3795	{
3796	szTmp[cch++] = ':';
3797	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_HCPHYS_NA(pPage), 16, 12, 0, RTSTR_F_ZEROPAD \| RTSTR_F_64BIT);
3798	}
3799
3800	if (IS_PART_INCLUDED(2))
3801	{
3802	szTmp[cch++] = ':';
3803	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_PAGEID(pPage), 16, 7, 0, RTSTR_F_ZEROPAD \| RTSTR_F_32BIT);
3804	}
3805
3806	if (IS_PART_INCLUDED(6))
3807	{
3808	szTmp[cch++] = ':';
3809	static const char s_achRefs[4] = { '-', 'U', '!', 'L' };
3810	szTmp[cch++] = s_achRefs[PGM_PAGE_GET_TD_CREFS_NA(pPage)];
3811	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_TD_IDX_NA(pPage), 16, 4, 0, RTSTR_F_ZEROPAD \| RTSTR_F_16BIT);
3812	}
3813	# undef IS_PART_INCLUDED
3814
3815	cch = pfnOutput(pvArgOutput, szTmp, cch);
3816	}
3817	else
3818	cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("<bad-pgmpage-ptr>"));
3819	NOREF(pszType); NOREF(cchWidth); NOREF(pvUser);
3820	return cch;
3821	}
3822
3823
3824	/** Format handler for PGMRAMRANGE.
3825	* @copydoc FNRTSTRFORMATTYPE */
3826	static DECLCALLBACK(size_t) pgmFormatTypeHandlerRamRange(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
3827	const char pszType, void const pvValue,
3828	int cchWidth, int cchPrecision, unsigned fFlags,
3829	void *pvUser)
3830	{
3831	size_t cch;
3832	PGMRAMRANGE const pRam = (PGMRAMRANGE const )pvValue;
3833	if (RT_VALID_PTR(pRam))
3834	{
3835	char szTmp[80];
3836	cch = RTStrPrintf(szTmp, sizeof(szTmp), "%RGp-%RGp", pRam->GCPhys, pRam->GCPhysLast);
3837	cch = pfnOutput(pvArgOutput, szTmp, cch);
3838	}
3839	else
3840	cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("<bad-pgmramrange-ptr>"));
3841	NOREF(pszType); NOREF(cchWidth); NOREF(cchPrecision); NOREF(pvUser); NOREF(fFlags);
3842	return cch;
3843	}
3844
3845	/** Format type andlers to be registered/deregistered. */
3846	static const struct
3847	{
3848	char szType[24];
3849	PFNRTSTRFORMATTYPE pfnHandler;
3850	} g_aPgmFormatTypes[] =
3851	{
3852	{ "pgmpage", pgmFormatTypeHandlerPage },
3853	{ "pgmramrange", pgmFormatTypeHandlerRamRange }
3854	};
3855
3856	#endif /* !IN_R0 \|\| LOG_ENABLED */
3857
3858	/**
3859	* Registers the global string format types.
3860	*
3861	* This should be called at module load time or in some other manner that ensure
3862	* that it's called exactly one time.
3863	*
3864	* @returns IPRT status code on RTStrFormatTypeRegister failure.
3865	*/
3866	VMMDECL(int) PGMRegisterStringFormatTypes(void)
3867	{
3868	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
3869	int rc = VINF_SUCCESS;
3870	unsigned i;
3871	for (i = 0; RT_SUCCESS(rc) && i < RT_ELEMENTS(g_aPgmFormatTypes); i++)
3872	{
3873	rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL);
3874	# ifdef IN_RING0
3875	if (rc == VERR_ALREADY_EXISTS)
3876	{
3877	/* in case of cleanup failure in ring-0 */
3878	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
3879	rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL);
3880	}
3881	# endif
3882	}
3883	if (RT_FAILURE(rc))
3884	while (i-- > 0)
3885	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
3886
3887	return rc;
3888	#else
3889	return VINF_SUCCESS;
3890	#endif
3891	}
3892
3893
3894	/**
3895	* Deregisters the global string format types.
3896	*
3897	* This should be called at module unload time or in some other manner that
3898	* ensure that it's called exactly one time.
3899	*/
3900	VMMDECL(void) PGMDeregisterStringFormatTypes(void)
3901	{
3902	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
3903	for (unsigned i = 0; i < RT_ELEMENTS(g_aPgmFormatTypes); i++)
3904	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
3905	#endif
3906	}
3907
3908
3909	#ifdef VBOX_STRICT
3910	/**
3911	* Asserts that everything related to the guest CR3 is correctly shadowed.
3912	*
3913	* This will call PGMAssertNoMappingConflicts() and PGMAssertHandlerAndFlagsInSync(),
3914	* and assert the correctness of the guest CR3 mapping before asserting that the
3915	* shadow page tables is in sync with the guest page tables.
3916	*
3917	* @returns Number of conflicts.
3918	* @param pVM The cross context VM structure.
3919	* @param pVCpu The cross context virtual CPU structure.
3920	* @param cr3 The current guest CR3 register value.
3921	* @param cr4 The current guest CR4 register value.
3922	*/
3923	VMMDECL(unsigned) PGMAssertCR3(PVMCC pVM, PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4)
3924	{
3925	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
3926
3927	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
3928	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), -VERR_PGM_MODE_IPE);
3929	AssertReturn(g_aPgmBothModeData[idxBth].pfnAssertCR3, -VERR_PGM_MODE_IPE);
3930
3931	PGM_LOCK_VOID(pVM);
3932	unsigned cErrors = g_aPgmBothModeData[idxBth].pfnAssertCR3(pVCpu, cr3, cr4, 0, ~(RTGCPTR)0);
3933	PGM_UNLOCK(pVM);
3934
3935	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
3936	return cErrors;
3937	}
3938	#endif /* VBOX_STRICT */
3939
3940
3941	/**
3942	* Updates PGM's copy of the guest's EPT pointer.
3943	*
3944	* @param pVCpu The cross context virtual CPU structure.
3945	* @param uEptPtr The EPT pointer.
3946	*
3947	* @remarks This can be called as part of VM-entry so we might be in the midst of
3948	* switching to VMX non-root mode.
3949	*/
3950	VMM_INT_DECL(void) PGMSetGuestEptPtr(PVMCPUCC pVCpu, uint64_t uEptPtr)
3951	{
3952	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3953	PGM_LOCK_VOID(pVM);
3954	pVCpu->pgm.s.uEptPtr = uEptPtr;
3955	PGM_UNLOCK(pVM);
3956	}
3957

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAll.cpp@ 96854

Download in other formats: