1 | /* $Id: PGMInline.h 37354 2011-06-07 15:05:32Z vboxsync $ */
|
---|
2 | /** @file
|
---|
3 | * PGM - Inlined functions.
|
---|
4 | */
|
---|
5 |
|
---|
6 | /*
|
---|
7 | * Copyright (C) 2006-2010 Oracle Corporation
|
---|
8 | *
|
---|
9 | * This file is part of VirtualBox Open Source Edition (OSE), as
|
---|
10 | * available from http://www.virtualbox.org. This file is free software;
|
---|
11 | * you can redistribute it and/or modify it under the terms of the GNU
|
---|
12 | * General Public License (GPL) as published by the Free Software
|
---|
13 | * Foundation, in version 2 as it comes in the "COPYING" file of the
|
---|
14 | * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
|
---|
15 | * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
|
---|
16 | */
|
---|
17 |
|
---|
18 | #ifndef ___PGMInline_h
|
---|
19 | #define ___PGMInline_h
|
---|
20 |
|
---|
21 | #include <VBox/cdefs.h>
|
---|
22 | #include <VBox/types.h>
|
---|
23 | #include <VBox/err.h>
|
---|
24 | #include <VBox/vmm/stam.h>
|
---|
25 | #include <VBox/param.h>
|
---|
26 | #include <VBox/vmm/vmm.h>
|
---|
27 | #include <VBox/vmm/mm.h>
|
---|
28 | #include <VBox/vmm/pdmcritsect.h>
|
---|
29 | #include <VBox/vmm/pdmapi.h>
|
---|
30 | #include <VBox/dis.h>
|
---|
31 | #include <VBox/vmm/dbgf.h>
|
---|
32 | #include <VBox/log.h>
|
---|
33 | #include <VBox/vmm/gmm.h>
|
---|
34 | #include <VBox/vmm/hwaccm.h>
|
---|
35 | #include <iprt/asm.h>
|
---|
36 | #include <iprt/assert.h>
|
---|
37 | #include <iprt/avl.h>
|
---|
38 | #include <iprt/critsect.h>
|
---|
39 | #include <iprt/sha.h>
|
---|
40 |
|
---|
41 |
|
---|
42 |
|
---|
43 | /** @addtogroup grp_pgm_int Internals
|
---|
44 | * @internal
|
---|
45 | * @{
|
---|
46 | */
|
---|
47 |
|
---|
48 | /**
|
---|
49 | * Gets the PGMRAMRANGE structure for a guest page.
|
---|
50 | *
|
---|
51 | * @returns Pointer to the RAM range on success.
|
---|
52 | * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
|
---|
53 | *
|
---|
54 | * @param pVM The VM handle.
|
---|
55 | * @param GCPhys The GC physical address.
|
---|
56 | */
|
---|
57 | DECLINLINE(PPGMRAMRANGE) pgmPhysGetRange(PVM pVM, RTGCPHYS GCPhys)
|
---|
58 | {
|
---|
59 | PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)];
|
---|
60 | if (!pRam || GCPhys - pRam->GCPhys >= pRam->cb)
|
---|
61 | pRam = pgmPhysGetRangeSlow(pVM, GCPhys);
|
---|
62 | STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbHits));
|
---|
63 | return pRam;
|
---|
64 | }
|
---|
65 |
|
---|
66 |
|
---|
67 | /**
|
---|
68 | * Gets the PGMRAMRANGE structure for a guest page, if unassigned get the ram
|
---|
69 | * range above it.
|
---|
70 | *
|
---|
71 | * @returns Pointer to the RAM range on success.
|
---|
72 | * @returns NULL if the address is located after the last range.
|
---|
73 | *
|
---|
74 | * @param pVM The VM handle.
|
---|
75 | * @param GCPhys The GC physical address.
|
---|
76 | */
|
---|
77 | DECLINLINE(PPGMRAMRANGE) pgmPhysGetRangeAtOrAbove(PVM pVM, RTGCPHYS GCPhys)
|
---|
78 | {
|
---|
79 | PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)];
|
---|
80 | if ( !pRam
|
---|
81 | || (GCPhys - pRam->GCPhys) >= pRam->cb)
|
---|
82 | return pgmPhysGetRangeAtOrAboveSlow(pVM, GCPhys);
|
---|
83 | STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbHits));
|
---|
84 | return pRam;
|
---|
85 | }
|
---|
86 |
|
---|
87 |
|
---|
88 |
|
---|
89 | /**
|
---|
90 | * Gets the PGMPAGE structure for a guest page.
|
---|
91 | *
|
---|
92 | * @returns Pointer to the page on success.
|
---|
93 | * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
|
---|
94 | *
|
---|
95 | * @param pVM The VM handle.
|
---|
96 | * @param GCPhys The GC physical address.
|
---|
97 | */
|
---|
98 | DECLINLINE(PPGMPAGE) pgmPhysGetPage(PVM pVM, RTGCPHYS GCPhys)
|
---|
99 | {
|
---|
100 | PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)];
|
---|
101 | RTGCPHYS off;
|
---|
102 | if ( !pRam
|
---|
103 | || (off = GCPhys - pRam->GCPhys) >= pRam->cb)
|
---|
104 | return pgmPhysGetPageSlow(pVM, GCPhys);
|
---|
105 | STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbHits));
|
---|
106 | return &pRam->aPages[off >> PAGE_SHIFT];
|
---|
107 | }
|
---|
108 |
|
---|
109 |
|
---|
110 | /**
|
---|
111 | * Gets the PGMPAGE structure for a guest page.
|
---|
112 | *
|
---|
113 | * Old Phys code: Will make sure the page is present.
|
---|
114 | *
|
---|
115 | * @returns VBox status code.
|
---|
116 | * @retval VINF_SUCCESS and a valid *ppPage on success.
|
---|
117 | * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if the address isn't valid.
|
---|
118 | *
|
---|
119 | * @param pVM The VM handle.
|
---|
120 | * @param GCPhys The GC physical address.
|
---|
121 | * @param ppPage Where to store the page pointer on success.
|
---|
122 | */
|
---|
123 | DECLINLINE(int) pgmPhysGetPageEx(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage)
|
---|
124 | {
|
---|
125 | PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)];
|
---|
126 | RTGCPHYS off;
|
---|
127 | if ( !pRam
|
---|
128 | || (off = GCPhys - pRam->GCPhys) >= pRam->cb)
|
---|
129 | return pgmPhysGetPageExSlow(pVM, GCPhys, ppPage);
|
---|
130 | *ppPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
|
---|
131 | STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbHits));
|
---|
132 | return VINF_SUCCESS;
|
---|
133 | }
|
---|
134 |
|
---|
135 |
|
---|
136 |
|
---|
137 |
|
---|
138 | /**
|
---|
139 | * Gets the PGMPAGE structure for a guest page.
|
---|
140 | *
|
---|
141 | * Old Phys code: Will make sure the page is present.
|
---|
142 | *
|
---|
143 | * @returns VBox status code.
|
---|
144 | * @retval VINF_SUCCESS and a valid *ppPage on success.
|
---|
145 | * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if the address isn't valid.
|
---|
146 | *
|
---|
147 | * @param pVM The VM handle.
|
---|
148 | * @param GCPhys The GC physical address.
|
---|
149 | * @param ppPage Where to store the page pointer on success.
|
---|
150 | * @param ppRamHint Where to read and store the ram list hint.
|
---|
151 | * The caller initializes this to NULL before the call.
|
---|
152 | */
|
---|
153 | DECLINLINE(int) pgmPhysGetPageWithHintEx(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRamHint)
|
---|
154 | {
|
---|
155 | RTGCPHYS off;
|
---|
156 | PPGMRAMRANGE pRam = *ppRamHint;
|
---|
157 | if ( !pRam
|
---|
158 | || RT_UNLIKELY((off = GCPhys - pRam->GCPhys) >= pRam->cb))
|
---|
159 | {
|
---|
160 | pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)];
|
---|
161 | if ( !pRam
|
---|
162 | || (off = GCPhys - pRam->GCPhys) >= pRam->cb)
|
---|
163 | return pgmPhysGetPageAndRangeExSlow(pVM, GCPhys, ppPage, ppRamHint);
|
---|
164 |
|
---|
165 | STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbHits));
|
---|
166 | *ppRamHint = pRam;
|
---|
167 | }
|
---|
168 | *ppPage = &pRam->aPages[off >> PAGE_SHIFT];
|
---|
169 | return VINF_SUCCESS;
|
---|
170 | }
|
---|
171 |
|
---|
172 |
|
---|
173 | /**
|
---|
174 | * Gets the PGMPAGE structure for a guest page together with the PGMRAMRANGE.
|
---|
175 | *
|
---|
176 | * @returns Pointer to the page on success.
|
---|
177 | * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
|
---|
178 | *
|
---|
179 | * @param pVM The VM handle.
|
---|
180 | * @param GCPhys The GC physical address.
|
---|
181 | * @param ppPage Where to store the pointer to the PGMPAGE structure.
|
---|
182 | * @param ppRam Where to store the pointer to the PGMRAMRANGE structure.
|
---|
183 | */
|
---|
184 | DECLINLINE(int) pgmPhysGetPageAndRangeEx(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam)
|
---|
185 | {
|
---|
186 | PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)];
|
---|
187 | RTGCPHYS off;
|
---|
188 | if ( !pRam
|
---|
189 | || (off = GCPhys - pRam->GCPhys) >= pRam->cb)
|
---|
190 | return pgmPhysGetPageAndRangeExSlow(pVM, GCPhys, ppPage, ppRam);
|
---|
191 |
|
---|
192 | STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbHits));
|
---|
193 | *ppRam = pRam;
|
---|
194 | *ppPage = &pRam->aPages[off >> PAGE_SHIFT];
|
---|
195 | return VINF_SUCCESS;
|
---|
196 | }
|
---|
197 |
|
---|
198 |
|
---|
199 | /**
|
---|
200 | * Convert GC Phys to HC Phys.
|
---|
201 | *
|
---|
202 | * @returns VBox status.
|
---|
203 | * @param pVM The VM handle.
|
---|
204 | * @param GCPhys The GC physical address.
|
---|
205 | * @param pHCPhys Where to store the corresponding HC physical address.
|
---|
206 | *
|
---|
207 | * @deprecated Doesn't deal with zero, shared or write monitored pages.
|
---|
208 | * Avoid when writing new code!
|
---|
209 | */
|
---|
210 | DECLINLINE(int) pgmRamGCPhys2HCPhys(PVM pVM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys)
|
---|
211 | {
|
---|
212 | PPGMPAGE pPage;
|
---|
213 | int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
|
---|
214 | if (RT_FAILURE(rc))
|
---|
215 | return rc;
|
---|
216 | *pHCPhys = PGM_PAGE_GET_HCPHYS(pPage) | (GCPhys & PAGE_OFFSET_MASK);
|
---|
217 | return VINF_SUCCESS;
|
---|
218 | }
|
---|
219 |
|
---|
220 | #if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) || defined(IN_RC)
|
---|
221 |
|
---|
222 | /**
|
---|
223 | * Inlined version of the ring-0 version of the host page mapping code
|
---|
224 | * that optimizes access to pages already in the set.
|
---|
225 | *
|
---|
226 | * @returns VINF_SUCCESS. Will bail out to ring-3 on failure.
|
---|
227 | * @param pVCpu The current CPU.
|
---|
228 | * @param HCPhys The physical address of the page.
|
---|
229 | * @param ppv Where to store the mapping address.
|
---|
230 | */
|
---|
231 | DECLINLINE(int) pgmRZDynMapHCPageInlined(PVMCPU pVCpu, RTHCPHYS HCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL)
|
---|
232 | {
|
---|
233 | PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
|
---|
234 |
|
---|
235 | STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPageInl, a);
|
---|
236 | Assert(!(HCPhys & PAGE_OFFSET_MASK));
|
---|
237 | Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
|
---|
238 |
|
---|
239 | unsigned iHash = PGMMAPSET_HASH(HCPhys);
|
---|
240 | unsigned iEntry = pSet->aiHashTable[iHash];
|
---|
241 | if ( iEntry < pSet->cEntries
|
---|
242 | && pSet->aEntries[iEntry].HCPhys == HCPhys
|
---|
243 | && pSet->aEntries[iEntry].cInlinedRefs < UINT16_MAX - 1)
|
---|
244 | {
|
---|
245 | pSet->aEntries[iEntry].cInlinedRefs++;
|
---|
246 | *ppv = pSet->aEntries[iEntry].pvPage;
|
---|
247 | STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPageInlHits);
|
---|
248 | }
|
---|
249 | else
|
---|
250 | {
|
---|
251 | STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPageInlMisses);
|
---|
252 | pgmRZDynMapHCPageCommon(pSet, HCPhys, ppv RTLOG_COMMA_SRC_POS_ARGS);
|
---|
253 | }
|
---|
254 |
|
---|
255 | STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPageInl, a);
|
---|
256 | return VINF_SUCCESS;
|
---|
257 | }
|
---|
258 |
|
---|
259 |
|
---|
260 | /**
|
---|
261 | * Inlined version of the guest page mapping code that optimizes access to pages
|
---|
262 | * already in the set.
|
---|
263 | *
|
---|
264 | * @returns VBox status code, see pgmRZDynMapGCPageCommon for details.
|
---|
265 | * @param pVM The VM handle.
|
---|
266 | * @param pVCpu The current CPU.
|
---|
267 | * @param GCPhys The guest physical address of the page.
|
---|
268 | * @param ppv Where to store the mapping address.
|
---|
269 | */
|
---|
270 | DECLINLINE(int) pgmRZDynMapGCPageV2Inlined(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL)
|
---|
271 | {
|
---|
272 | STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInl, a);
|
---|
273 | AssertMsg(!(GCPhys & PAGE_OFFSET_MASK), ("%RGp\n", GCPhys));
|
---|
274 |
|
---|
275 | /*
|
---|
276 | * Get the ram range.
|
---|
277 | */
|
---|
278 | PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)];
|
---|
279 | RTGCPHYS off;
|
---|
280 | if ( !pRam
|
---|
281 | || (off = GCPhys - pRam->GCPhys) >= pRam->cb
|
---|
282 | /** @todo || page state stuff */
|
---|
283 | )
|
---|
284 | {
|
---|
285 | /* This case is not counted into StatRZDynMapGCPageInl. */
|
---|
286 | STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlRamMisses);
|
---|
287 | return pgmRZDynMapGCPageCommon(pVM, pVCpu, GCPhys, ppv RTLOG_COMMA_SRC_POS_ARGS);
|
---|
288 | }
|
---|
289 |
|
---|
290 | RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(&pRam->aPages[off >> PAGE_SHIFT]);
|
---|
291 | STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlRamHits);
|
---|
292 |
|
---|
293 | /*
|
---|
294 | * pgmRZDynMapHCPageInlined with out stats.
|
---|
295 | */
|
---|
296 | PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
|
---|
297 | Assert(!(HCPhys & PAGE_OFFSET_MASK));
|
---|
298 | Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
|
---|
299 |
|
---|
300 | unsigned iHash = PGMMAPSET_HASH(HCPhys);
|
---|
301 | unsigned iEntry = pSet->aiHashTable[iHash];
|
---|
302 | if ( iEntry < pSet->cEntries
|
---|
303 | && pSet->aEntries[iEntry].HCPhys == HCPhys
|
---|
304 | && pSet->aEntries[iEntry].cInlinedRefs < UINT16_MAX - 1)
|
---|
305 | {
|
---|
306 | pSet->aEntries[iEntry].cInlinedRefs++;
|
---|
307 | *ppv = pSet->aEntries[iEntry].pvPage;
|
---|
308 | STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlHits);
|
---|
309 | }
|
---|
310 | else
|
---|
311 | {
|
---|
312 | STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlMisses);
|
---|
313 | pgmRZDynMapHCPageCommon(pSet, HCPhys, ppv RTLOG_COMMA_SRC_POS_ARGS);
|
---|
314 | }
|
---|
315 |
|
---|
316 | STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInl, a);
|
---|
317 | return VINF_SUCCESS;
|
---|
318 | }
|
---|
319 |
|
---|
320 |
|
---|
321 | /**
|
---|
322 | * Inlined version of the ring-0 version of guest page mapping that optimizes
|
---|
323 | * access to pages already in the set.
|
---|
324 | *
|
---|
325 | * @returns VBox status code, see pgmRZDynMapGCPageCommon for details.
|
---|
326 | * @param pVCpu The current CPU.
|
---|
327 | * @param GCPhys The guest physical address of the page.
|
---|
328 | * @param ppv Where to store the mapping address.
|
---|
329 | */
|
---|
330 | DECLINLINE(int) pgmRZDynMapGCPageInlined(PVMCPU pVCpu, RTGCPHYS GCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL)
|
---|
331 | {
|
---|
332 | return pgmRZDynMapGCPageV2Inlined(pVCpu->CTX_SUFF(pVM), pVCpu, GCPhys, ppv RTLOG_COMMA_SRC_POS_ARGS);
|
---|
333 | }
|
---|
334 |
|
---|
335 |
|
---|
336 | /**
|
---|
337 | * Inlined version of the ring-0 version of the guest byte mapping code
|
---|
338 | * that optimizes access to pages already in the set.
|
---|
339 | *
|
---|
340 | * @returns VBox status code, see pgmRZDynMapGCPageCommon for details.
|
---|
341 | * @param pVCpu The current CPU.
|
---|
342 | * @param HCPhys The physical address of the page.
|
---|
343 | * @param ppv Where to store the mapping address. The offset is
|
---|
344 | * preserved.
|
---|
345 | */
|
---|
346 | DECLINLINE(int) pgmRZDynMapGCPageOffInlined(PVMCPU pVCpu, RTGCPHYS GCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL)
|
---|
347 | {
|
---|
348 | STAM_PROFILE_START(&pVCpu->pgm.s.StatRZDynMapGCPageInl, a);
|
---|
349 |
|
---|
350 | /*
|
---|
351 | * Get the ram range.
|
---|
352 | */
|
---|
353 | PVM pVM = pVCpu->CTX_SUFF(pVM);
|
---|
354 | PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)];
|
---|
355 | RTGCPHYS off;
|
---|
356 | if ( !pRam
|
---|
357 | || (off = GCPhys - pRam->GCPhys) >= pRam->cb
|
---|
358 | /** @todo || page state stuff */
|
---|
359 | )
|
---|
360 | {
|
---|
361 | /* This case is not counted into StatRZDynMapGCPageInl. */
|
---|
362 | STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlRamMisses);
|
---|
363 | return pgmRZDynMapGCPageCommon(pVM, pVCpu, GCPhys, ppv RTLOG_COMMA_SRC_POS_ARGS);
|
---|
364 | }
|
---|
365 |
|
---|
366 | RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(&pRam->aPages[off >> PAGE_SHIFT]);
|
---|
367 | STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlRamHits);
|
---|
368 |
|
---|
369 | /*
|
---|
370 | * pgmRZDynMapHCPageInlined with out stats.
|
---|
371 | */
|
---|
372 | PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
|
---|
373 | Assert(!(HCPhys & PAGE_OFFSET_MASK));
|
---|
374 | Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
|
---|
375 |
|
---|
376 | unsigned iHash = PGMMAPSET_HASH(HCPhys);
|
---|
377 | unsigned iEntry = pSet->aiHashTable[iHash];
|
---|
378 | if ( iEntry < pSet->cEntries
|
---|
379 | && pSet->aEntries[iEntry].HCPhys == HCPhys
|
---|
380 | && pSet->aEntries[iEntry].cInlinedRefs < UINT16_MAX - 1)
|
---|
381 | {
|
---|
382 | STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlHits);
|
---|
383 | pSet->aEntries[iEntry].cInlinedRefs++;
|
---|
384 | *ppv = (void *)((uintptr_t)pSet->aEntries[iEntry].pvPage | (PAGE_OFFSET_MASK & (uintptr_t)GCPhys));
|
---|
385 | }
|
---|
386 | else
|
---|
387 | {
|
---|
388 | STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlMisses);
|
---|
389 | pgmRZDynMapHCPageCommon(pSet, HCPhys, ppv RTLOG_COMMA_SRC_POS_ARGS);
|
---|
390 | *ppv = (void *)((uintptr_t)*ppv | (PAGE_OFFSET_MASK & (uintptr_t)GCPhys));
|
---|
391 | }
|
---|
392 |
|
---|
393 | STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInl, a);
|
---|
394 | return VINF_SUCCESS;
|
---|
395 | }
|
---|
396 |
|
---|
397 | #endif /* VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
|
---|
398 | #if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
|
---|
399 |
|
---|
400 | /**
|
---|
401 | * Maps the page into current context (RC and maybe R0).
|
---|
402 | *
|
---|
403 | * @returns pointer to the mapping.
|
---|
404 | * @param pVM Pointer to the PGM instance data.
|
---|
405 | * @param pPage The page.
|
---|
406 | */
|
---|
407 | DECLINLINE(void *) pgmPoolMapPageInlined(PVM pVM, PPGMPOOLPAGE pPage RTLOG_COMMA_SRC_POS_DECL)
|
---|
408 | {
|
---|
409 | if (pPage->idx >= PGMPOOL_IDX_FIRST)
|
---|
410 | {
|
---|
411 | Assert(pPage->idx < pVM->pgm.s.CTX_SUFF(pPool)->cCurPages);
|
---|
412 | void *pv;
|
---|
413 | pgmRZDynMapHCPageInlined(VMMGetCpu(pVM), pPage->Core.Key, &pv RTLOG_COMMA_SRC_POS_ARGS);
|
---|
414 | return pv;
|
---|
415 | }
|
---|
416 | AssertFatalMsgFailed(("pgmPoolMapPageInlined invalid page index %x\n", pPage->idx));
|
---|
417 | }
|
---|
418 |
|
---|
419 | /**
|
---|
420 | * Maps the page into current context (RC and maybe R0).
|
---|
421 | *
|
---|
422 | * @returns pointer to the mapping.
|
---|
423 | * @param pVM Pointer to the PGM instance data.
|
---|
424 | * @param pVCpu The current CPU.
|
---|
425 | * @param pPage The page.
|
---|
426 | */
|
---|
427 | DECLINLINE(void *) pgmPoolMapPageV2Inlined(PVM pVM, PVMCPU pVCpu, PPGMPOOLPAGE pPage RTLOG_COMMA_SRC_POS_DECL)
|
---|
428 | {
|
---|
429 | if (pPage->idx >= PGMPOOL_IDX_FIRST)
|
---|
430 | {
|
---|
431 | Assert(pPage->idx < pVM->pgm.s.CTX_SUFF(pPool)->cCurPages);
|
---|
432 | void *pv;
|
---|
433 | Assert(pVCpu == VMMGetCpu(pVM));
|
---|
434 | pgmRZDynMapHCPageInlined(pVCpu, pPage->Core.Key, &pv RTLOG_COMMA_SRC_POS_ARGS);
|
---|
435 | return pv;
|
---|
436 | }
|
---|
437 | AssertFatalMsgFailed(("pgmPoolMapPageV2Inlined invalid page index %x\n", pPage->idx));
|
---|
438 | }
|
---|
439 |
|
---|
440 | #endif /* VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 || IN_RC */
|
---|
441 | #ifndef IN_RC
|
---|
442 |
|
---|
443 | /**
|
---|
444 | * Queries the Physical TLB entry for a physical guest page,
|
---|
445 | * attempting to load the TLB entry if necessary.
|
---|
446 | *
|
---|
447 | * @returns VBox status code.
|
---|
448 | * @retval VINF_SUCCESS on success
|
---|
449 | * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
|
---|
450 | *
|
---|
451 | * @param pVM The VM handle.
|
---|
452 | * @param GCPhys The address of the guest page.
|
---|
453 | * @param ppTlbe Where to store the pointer to the TLB entry.
|
---|
454 | */
|
---|
455 | DECLINLINE(int) pgmPhysPageQueryTlbe(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe)
|
---|
456 | {
|
---|
457 | int rc;
|
---|
458 | PPGMPAGEMAPTLBE pTlbe = &pVM->pgm.s.CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
|
---|
459 | if (pTlbe->GCPhys == (GCPhys & X86_PTE_PAE_PG_MASK))
|
---|
460 | {
|
---|
461 | STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbHits));
|
---|
462 | rc = VINF_SUCCESS;
|
---|
463 | }
|
---|
464 | else
|
---|
465 | rc = pgmPhysPageLoadIntoTlb(pVM, GCPhys);
|
---|
466 | *ppTlbe = pTlbe;
|
---|
467 | return rc;
|
---|
468 | }
|
---|
469 |
|
---|
470 |
|
---|
471 | /**
|
---|
472 | * Queries the Physical TLB entry for a physical guest page,
|
---|
473 | * attempting to load the TLB entry if necessary.
|
---|
474 | *
|
---|
475 | * @returns VBox status code.
|
---|
476 | * @retval VINF_SUCCESS on success
|
---|
477 | * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
|
---|
478 | *
|
---|
479 | * @param pVM The VM handle.
|
---|
480 | * @param pPage Pointer to the PGMPAGE structure corresponding to
|
---|
481 | * GCPhys.
|
---|
482 | * @param GCPhys The address of the guest page.
|
---|
483 | * @param ppTlbe Where to store the pointer to the TLB entry.
|
---|
484 | */
|
---|
485 | DECLINLINE(int) pgmPhysPageQueryTlbeWithPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe)
|
---|
486 | {
|
---|
487 | int rc;
|
---|
488 | PPGMPAGEMAPTLBE pTlbe = &pVM->pgm.s.CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
|
---|
489 | if (pTlbe->GCPhys == (GCPhys & X86_PTE_PAE_PG_MASK))
|
---|
490 | {
|
---|
491 | STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbHits));
|
---|
492 | rc = VINF_SUCCESS;
|
---|
493 | }
|
---|
494 | else
|
---|
495 | rc = pgmPhysPageLoadIntoTlbWithPage(pVM, pPage, GCPhys);
|
---|
496 | *ppTlbe = pTlbe;
|
---|
497 | return rc;
|
---|
498 | }
|
---|
499 |
|
---|
500 | #endif /* !IN_RC */
|
---|
501 |
|
---|
502 |
|
---|
503 | /**
|
---|
504 | * Enables write monitoring for an allocated page.
|
---|
505 | *
|
---|
506 | * The caller is responsible for updating the shadow page tables.
|
---|
507 | *
|
---|
508 | * @param pVM The VM handle.
|
---|
509 | * @param pPage The page to write monitor.
|
---|
510 | * @param GCPhysPage The address of the page.
|
---|
511 | */
|
---|
512 | DECLINLINE(void) pgmPhysPageWriteMonitor(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhysPage)
|
---|
513 | {
|
---|
514 | Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
|
---|
515 | PGM_LOCK_ASSERT_OWNER(pVM);
|
---|
516 |
|
---|
517 | PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_WRITE_MONITORED);
|
---|
518 | pVM->pgm.s.cMonitoredPages++;
|
---|
519 |
|
---|
520 | /* Large pages must disabled. */
|
---|
521 | if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
|
---|
522 | {
|
---|
523 | PPGMPAGE pFirstPage = pgmPhysGetPage(pVM, GCPhysPage & X86_PDE2M_PAE_PG_MASK);
|
---|
524 | AssertFatal(pFirstPage);
|
---|
525 | if (PGM_PAGE_GET_PDE_TYPE(pFirstPage) == PGM_PAGE_PDE_TYPE_PDE)
|
---|
526 | {
|
---|
527 | PGM_PAGE_SET_PDE_TYPE(pVM, pFirstPage, PGM_PAGE_PDE_TYPE_PDE_DISABLED);
|
---|
528 | pVM->pgm.s.cLargePagesDisabled++;
|
---|
529 | }
|
---|
530 | else
|
---|
531 | Assert(PGM_PAGE_GET_PDE_TYPE(pFirstPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED);
|
---|
532 | }
|
---|
533 | }
|
---|
534 |
|
---|
535 |
|
---|
536 | /**
|
---|
537 | * Checks if the no-execute (NX) feature is active (EFER.NXE=1).
|
---|
538 | *
|
---|
539 | * Only used when the guest is in PAE or long mode. This is inlined so that we
|
---|
540 | * can perform consistency checks in debug builds.
|
---|
541 | *
|
---|
542 | * @returns true if it is, false if it isn't.
|
---|
543 | * @param pVCpu The current CPU.
|
---|
544 | */
|
---|
545 | DECL_FORCE_INLINE(bool) pgmGstIsNoExecuteActive(PVMCPU pVCpu)
|
---|
546 | {
|
---|
547 | Assert(pVCpu->pgm.s.fNoExecuteEnabled == CPUMIsGuestNXEnabled(pVCpu));
|
---|
548 | Assert(CPUMIsGuestInPAEMode(pVCpu) || CPUMIsGuestInLongMode(pVCpu));
|
---|
549 | return pVCpu->pgm.s.fNoExecuteEnabled;
|
---|
550 | }
|
---|
551 |
|
---|
552 |
|
---|
553 | /**
|
---|
554 | * Checks if the page size extension (PSE) is currently enabled (CR4.PSE=1).
|
---|
555 | *
|
---|
556 | * Only used when the guest is in paged 32-bit mode. This is inlined so that
|
---|
557 | * we can perform consistency checks in debug builds.
|
---|
558 | *
|
---|
559 | * @returns true if it is, false if it isn't.
|
---|
560 | * @param pVCpu The current CPU.
|
---|
561 | */
|
---|
562 | DECL_FORCE_INLINE(bool) pgmGst32BitIsPageSizeExtActive(PVMCPU pVCpu)
|
---|
563 | {
|
---|
564 | Assert(pVCpu->pgm.s.fGst32BitPageSizeExtension == CPUMIsGuestPageSizeExtEnabled(pVCpu));
|
---|
565 | Assert(!CPUMIsGuestInPAEMode(pVCpu));
|
---|
566 | Assert(!CPUMIsGuestInLongMode(pVCpu));
|
---|
567 | return pVCpu->pgm.s.fGst32BitPageSizeExtension;
|
---|
568 | }
|
---|
569 |
|
---|
570 |
|
---|
571 | /**
|
---|
572 | * Calculated the guest physical address of the large (4 MB) page in 32 bits paging mode.
|
---|
573 | * Takes PSE-36 into account.
|
---|
574 | *
|
---|
575 | * @returns guest physical address
|
---|
576 | * @param pVM The VM handle.
|
---|
577 | * @param Pde Guest Pde
|
---|
578 | */
|
---|
579 | DECLINLINE(RTGCPHYS) pgmGstGet4MBPhysPage(PVM pVM, X86PDE Pde)
|
---|
580 | {
|
---|
581 | RTGCPHYS GCPhys = Pde.u & X86_PDE4M_PG_MASK;
|
---|
582 | GCPhys |= (RTGCPHYS)Pde.b.u8PageNoHigh << 32;
|
---|
583 |
|
---|
584 | return GCPhys & pVM->pgm.s.GCPhys4MBPSEMask;
|
---|
585 | }
|
---|
586 |
|
---|
587 |
|
---|
588 | /**
|
---|
589 | * Gets the address the guest page directory (32-bit paging).
|
---|
590 | *
|
---|
591 | * @returns VBox status code.
|
---|
592 | * @param pVCpu The current CPU.
|
---|
593 | * @param ppPd Where to return the mapping. This is always set.
|
---|
594 | */
|
---|
595 | DECLINLINE(int) pgmGstGet32bitPDPtrEx(PVMCPU pVCpu, PX86PD *ppPd)
|
---|
596 | {
|
---|
597 | #ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
|
---|
598 | int rc = pgmRZDynMapGCPageInlined(pVCpu, pVCpu->pgm.s.GCPhysCR3, (void **)ppPd RTLOG_COMMA_SRC_POS);
|
---|
599 | if (RT_FAILURE(rc))
|
---|
600 | {
|
---|
601 | *ppPd = NULL;
|
---|
602 | return rc;
|
---|
603 | }
|
---|
604 | #else
|
---|
605 | *ppPd = pVCpu->pgm.s.CTX_SUFF(pGst32BitPd);
|
---|
606 | if (RT_UNLIKELY(!*ppPd))
|
---|
607 | return pgmGstLazyMap32BitPD(pVCpu, ppPd);
|
---|
608 | #endif
|
---|
609 | return VINF_SUCCESS;
|
---|
610 | }
|
---|
611 |
|
---|
612 |
|
---|
613 | /**
|
---|
614 | * Gets the address the guest page directory (32-bit paging).
|
---|
615 | *
|
---|
616 | * @returns Pointer the page directory entry in question.
|
---|
617 | * @param pVCpu The current CPU.
|
---|
618 | */
|
---|
619 | DECLINLINE(PX86PD) pgmGstGet32bitPDPtr(PVMCPU pVCpu)
|
---|
620 | {
|
---|
621 | #ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
|
---|
622 | PX86PD pGuestPD = NULL;
|
---|
623 | int rc = pgmRZDynMapGCPageInlined(pVCpu, pVCpu->pgm.s.GCPhysCR3, (void **)&pGuestPD RTLOG_COMMA_SRC_POS);
|
---|
624 | if (RT_FAILURE(rc))
|
---|
625 | {
|
---|
626 | AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc));
|
---|
627 | return NULL;
|
---|
628 | }
|
---|
629 | #else
|
---|
630 | PX86PD pGuestPD = pVCpu->pgm.s.CTX_SUFF(pGst32BitPd);
|
---|
631 | if (RT_UNLIKELY(!pGuestPD))
|
---|
632 | {
|
---|
633 | int rc = pgmGstLazyMap32BitPD(pVCpu, &pGuestPD);
|
---|
634 | if (RT_FAILURE(rc))
|
---|
635 | return NULL;
|
---|
636 | }
|
---|
637 | #endif
|
---|
638 | return pGuestPD;
|
---|
639 | }
|
---|
640 |
|
---|
641 |
|
---|
642 | /**
|
---|
643 | * Gets the guest page directory pointer table.
|
---|
644 | *
|
---|
645 | * @returns VBox status code.
|
---|
646 | * @param pVCpu The current CPU.
|
---|
647 | * @param ppPdpt Where to return the mapping. This is always set.
|
---|
648 | */
|
---|
649 | DECLINLINE(int) pgmGstGetPaePDPTPtrEx(PVMCPU pVCpu, PX86PDPT *ppPdpt)
|
---|
650 | {
|
---|
651 | #ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
|
---|
652 | int rc = pgmRZDynMapGCPageOffInlined(pVCpu, pVCpu->pgm.s.GCPhysCR3, (void **)ppPdpt RTLOG_COMMA_SRC_POS);
|
---|
653 | if (RT_FAILURE(rc))
|
---|
654 | {
|
---|
655 | *ppPdpt = NULL;
|
---|
656 | return rc;
|
---|
657 | }
|
---|
658 | #else
|
---|
659 | *ppPdpt = pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt);
|
---|
660 | if (RT_UNLIKELY(!*ppPdpt))
|
---|
661 | return pgmGstLazyMapPaePDPT(pVCpu, ppPdpt);
|
---|
662 | #endif
|
---|
663 | return VINF_SUCCESS;
|
---|
664 | }
|
---|
665 |
|
---|
666 | /**
|
---|
667 | * Gets the guest page directory pointer table.
|
---|
668 | *
|
---|
669 | * @returns Pointer to the page directory in question.
|
---|
670 | * @returns NULL if the page directory is not present or on an invalid page.
|
---|
671 | * @param pVCpu The current CPU.
|
---|
672 | */
|
---|
673 | DECLINLINE(PX86PDPT) pgmGstGetPaePDPTPtr(PVMCPU pVCpu)
|
---|
674 | {
|
---|
675 | PX86PDPT pGuestPdpt;
|
---|
676 | int rc = pgmGstGetPaePDPTPtrEx(pVCpu, &pGuestPdpt);
|
---|
677 | AssertMsg(RT_SUCCESS(rc) || rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc));
|
---|
678 | return pGuestPdpt;
|
---|
679 | }
|
---|
680 |
|
---|
681 |
|
---|
682 | /**
|
---|
683 | * Gets the guest page directory pointer table entry for the specified address.
|
---|
684 | *
|
---|
685 | * @returns Pointer to the page directory in question.
|
---|
686 | * @returns NULL if the page directory is not present or on an invalid page.
|
---|
687 | * @param pVCpu The current CPU
|
---|
688 | * @param GCPtr The address.
|
---|
689 | */
|
---|
690 | DECLINLINE(PX86PDPE) pgmGstGetPaePDPEPtr(PVMCPU pVCpu, RTGCPTR GCPtr)
|
---|
691 | {
|
---|
692 | AssertGCPtr32(GCPtr);
|
---|
693 |
|
---|
694 | #ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
|
---|
695 | PX86PDPT pGuestPDPT = NULL;
|
---|
696 | int rc = pgmRZDynMapGCPageOffInlined(pVCpu, pVCpu->pgm.s.GCPhysCR3, (void **)&pGuestPDPT RTLOG_COMMA_SRC_POS);
|
---|
697 | AssertRCReturn(rc, NULL);
|
---|
698 | #else
|
---|
699 | PX86PDPT pGuestPDPT = pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt);
|
---|
700 | if (RT_UNLIKELY(!pGuestPDPT))
|
---|
701 | {
|
---|
702 | int rc = pgmGstLazyMapPaePDPT(pVCpu, &pGuestPDPT);
|
---|
703 | if (RT_FAILURE(rc))
|
---|
704 | return NULL;
|
---|
705 | }
|
---|
706 | #endif
|
---|
707 | return &pGuestPDPT->a[(GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE];
|
---|
708 | }
|
---|
709 |
|
---|
710 |
|
---|
711 | /**
|
---|
712 | * Gets the page directory entry for the specified address.
|
---|
713 | *
|
---|
714 | * @returns The page directory entry in question.
|
---|
715 | * @returns A non-present entry if the page directory is not present or on an invalid page.
|
---|
716 | * @param pVCpu The handle of the virtual CPU.
|
---|
717 | * @param GCPtr The address.
|
---|
718 | */
|
---|
719 | DECLINLINE(X86PDEPAE) pgmGstGetPaePDE(PVMCPU pVCpu, RTGCPTR GCPtr)
|
---|
720 | {
|
---|
721 | AssertGCPtr32(GCPtr);
|
---|
722 | PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pVCpu);
|
---|
723 | if (RT_LIKELY(pGuestPDPT))
|
---|
724 | {
|
---|
725 | const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
|
---|
726 | if ( pGuestPDPT->a[iPdpt].n.u1Present
|
---|
727 | && !(pGuestPDPT->a[iPdpt].u & pVCpu->pgm.s.fGstPaeMbzPdpeMask) )
|
---|
728 | {
|
---|
729 | const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
|
---|
730 | #ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
|
---|
731 | PX86PDPAE pGuestPD = NULL;
|
---|
732 | int rc = pgmRZDynMapGCPageInlined(pVCpu,
|
---|
733 | pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK,
|
---|
734 | (void **)&pGuestPD
|
---|
735 | RTLOG_COMMA_SRC_POS);
|
---|
736 | if (RT_SUCCESS(rc))
|
---|
737 | return pGuestPD->a[iPD];
|
---|
738 | AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc));
|
---|
739 | #else
|
---|
740 | PX86PDPAE pGuestPD = pVCpu->pgm.s.CTX_SUFF(apGstPaePDs)[iPdpt];
|
---|
741 | if ( !pGuestPD
|
---|
742 | || (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt])
|
---|
743 | pgmGstLazyMapPaePD(pVCpu, iPdpt, &pGuestPD);
|
---|
744 | if (pGuestPD)
|
---|
745 | return pGuestPD->a[iPD];
|
---|
746 | #endif
|
---|
747 | }
|
---|
748 | }
|
---|
749 |
|
---|
750 | X86PDEPAE ZeroPde = {0};
|
---|
751 | return ZeroPde;
|
---|
752 | }
|
---|
753 |
|
---|
754 |
|
---|
755 | /**
|
---|
756 | * Gets the page directory pointer table entry for the specified address
|
---|
757 | * and returns the index into the page directory
|
---|
758 | *
|
---|
759 | * @returns Pointer to the page directory in question.
|
---|
760 | * @returns NULL if the page directory is not present or on an invalid page.
|
---|
761 | * @param pVCpu The current CPU.
|
---|
762 | * @param GCPtr The address.
|
---|
763 | * @param piPD Receives the index into the returned page directory
|
---|
764 | * @param pPdpe Receives the page directory pointer entry. Optional.
|
---|
765 | */
|
---|
766 | DECLINLINE(PX86PDPAE) pgmGstGetPaePDPtr(PVMCPU pVCpu, RTGCPTR GCPtr, unsigned *piPD, PX86PDPE pPdpe)
|
---|
767 | {
|
---|
768 | AssertGCPtr32(GCPtr);
|
---|
769 |
|
---|
770 | /* The PDPE. */
|
---|
771 | PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pVCpu);
|
---|
772 | if (RT_UNLIKELY(!pGuestPDPT))
|
---|
773 | return NULL;
|
---|
774 | const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
|
---|
775 | if (pPdpe)
|
---|
776 | *pPdpe = pGuestPDPT->a[iPdpt];
|
---|
777 | if (!pGuestPDPT->a[iPdpt].n.u1Present)
|
---|
778 | return NULL;
|
---|
779 | if (RT_UNLIKELY(pVCpu->pgm.s.fGstPaeMbzPdpeMask & pGuestPDPT->a[iPdpt].u))
|
---|
780 | return NULL;
|
---|
781 |
|
---|
782 | /* The PDE. */
|
---|
783 | #ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
|
---|
784 | PX86PDPAE pGuestPD = NULL;
|
---|
785 | int rc = pgmRZDynMapGCPageInlined(pVCpu,
|
---|
786 | pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK,
|
---|
787 | (void **)&pGuestPD
|
---|
788 | RTLOG_COMMA_SRC_POS);
|
---|
789 | if (RT_FAILURE(rc))
|
---|
790 | {
|
---|
791 | AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc));
|
---|
792 | return NULL;
|
---|
793 | }
|
---|
794 | #else
|
---|
795 | PX86PDPAE pGuestPD = pVCpu->pgm.s.CTX_SUFF(apGstPaePDs)[iPdpt];
|
---|
796 | if ( !pGuestPD
|
---|
797 | || (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt])
|
---|
798 | pgmGstLazyMapPaePD(pVCpu, iPdpt, &pGuestPD);
|
---|
799 | #endif
|
---|
800 |
|
---|
801 | *piPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
|
---|
802 | return pGuestPD;
|
---|
803 | }
|
---|
804 |
|
---|
805 | #ifndef IN_RC
|
---|
806 |
|
---|
807 | /**
|
---|
808 | * Gets the page map level-4 pointer for the guest.
|
---|
809 | *
|
---|
810 | * @returns VBox status code.
|
---|
811 | * @param pVCpu The current CPU.
|
---|
812 | * @param ppPml4 Where to return the mapping. Always set.
|
---|
813 | */
|
---|
814 | DECLINLINE(int) pgmGstGetLongModePML4PtrEx(PVMCPU pVCpu, PX86PML4 *ppPml4)
|
---|
815 | {
|
---|
816 | #ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
|
---|
817 | int rc = pgmRZDynMapGCPageInlined(pVCpu, pVCpu->pgm.s.GCPhysCR3, (void **)ppPml4 RTLOG_COMMA_SRC_POS);
|
---|
818 | if (RT_FAILURE(rc))
|
---|
819 | {
|
---|
820 | *ppPml4 = NULL;
|
---|
821 | return rc;
|
---|
822 | }
|
---|
823 | #else
|
---|
824 | *ppPml4 = pVCpu->pgm.s.CTX_SUFF(pGstAmd64Pml4);
|
---|
825 | if (RT_UNLIKELY(!*ppPml4))
|
---|
826 | return pgmGstLazyMapPml4(pVCpu, ppPml4);
|
---|
827 | #endif
|
---|
828 | return VINF_SUCCESS;
|
---|
829 | }
|
---|
830 |
|
---|
831 |
|
---|
832 | /**
|
---|
833 | * Gets the page map level-4 pointer for the guest.
|
---|
834 | *
|
---|
835 | * @returns Pointer to the PML4 page.
|
---|
836 | * @param pVCpu The current CPU.
|
---|
837 | */
|
---|
838 | DECLINLINE(PX86PML4) pgmGstGetLongModePML4Ptr(PVMCPU pVCpu)
|
---|
839 | {
|
---|
840 | PX86PML4 pGuestPml4;
|
---|
841 | int rc = pgmGstGetLongModePML4PtrEx(pVCpu, &pGuestPml4);
|
---|
842 | AssertMsg(RT_SUCCESS(rc) || rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc)); NOREF(rc);
|
---|
843 | return pGuestPml4;
|
---|
844 | }
|
---|
845 |
|
---|
846 |
|
---|
847 | /**
|
---|
848 | * Gets the pointer to a page map level-4 entry.
|
---|
849 | *
|
---|
850 | * @returns Pointer to the PML4 entry.
|
---|
851 | * @param pVCpu The current CPU.
|
---|
852 | * @param iPml4 The index.
|
---|
853 | * @remarks Only used by AssertCR3.
|
---|
854 | */
|
---|
855 | DECLINLINE(PX86PML4E) pgmGstGetLongModePML4EPtr(PVMCPU pVCpu, unsigned int iPml4)
|
---|
856 | {
|
---|
857 | #ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
|
---|
858 | PX86PML4 pGuestPml4;
|
---|
859 | int rc = pgmRZDynMapGCPageInlined(pVCpu, pVCpu->pgm.s.GCPhysCR3, (void **)&pGuestPml4 RTLOG_COMMA_SRC_POS);
|
---|
860 | AssertRCReturn(rc, NULL);
|
---|
861 | #else
|
---|
862 | PX86PML4 pGuestPml4 = pVCpu->pgm.s.CTX_SUFF(pGstAmd64Pml4);
|
---|
863 | if (RT_UNLIKELY(!pGuestPml4))
|
---|
864 | {
|
---|
865 | int rc = pgmGstLazyMapPml4(pVCpu, &pGuestPml4);
|
---|
866 | AssertRCReturn(rc, NULL);
|
---|
867 | }
|
---|
868 | #endif
|
---|
869 | return &pGuestPml4->a[iPml4];
|
---|
870 | }
|
---|
871 |
|
---|
872 |
|
---|
873 | /**
|
---|
874 | * Gets the page directory entry for the specified address.
|
---|
875 | *
|
---|
876 | * @returns The page directory entry in question.
|
---|
877 | * @returns A non-present entry if the page directory is not present or on an invalid page.
|
---|
878 | * @param pVCpu The current CPU.
|
---|
879 | * @param GCPtr The address.
|
---|
880 | */
|
---|
881 | DECLINLINE(X86PDEPAE) pgmGstGetLongModePDE(PVMCPU pVCpu, RTGCPTR64 GCPtr)
|
---|
882 | {
|
---|
883 | /*
|
---|
884 | * Note! To keep things simple, ASSUME invalid physical addresses will
|
---|
885 | * cause X86_TRAP_PF_RSVD. This isn't a problem until we start
|
---|
886 | * supporting 52-bit wide physical guest addresses.
|
---|
887 | */
|
---|
888 | PCX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pVCpu);
|
---|
889 | const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
|
---|
890 | if ( RT_LIKELY(pGuestPml4)
|
---|
891 | && pGuestPml4->a[iPml4].n.u1Present
|
---|
892 | && !(pGuestPml4->a[iPml4].u & pVCpu->pgm.s.fGstAmd64MbzPml4eMask) )
|
---|
893 | {
|
---|
894 | PCX86PDPT pPdptTemp;
|
---|
895 | int rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
|
---|
896 | if (RT_SUCCESS(rc))
|
---|
897 | {
|
---|
898 | const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
|
---|
899 | if ( pPdptTemp->a[iPdpt].n.u1Present
|
---|
900 | && !(pPdptTemp->a[iPdpt].u & pVCpu->pgm.s.fGstAmd64MbzPdpeMask) )
|
---|
901 | {
|
---|
902 | PCX86PDPAE pPD;
|
---|
903 | rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
|
---|
904 | if (RT_SUCCESS(rc))
|
---|
905 | {
|
---|
906 | const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
|
---|
907 | return pPD->a[iPD];
|
---|
908 | }
|
---|
909 | }
|
---|
910 | }
|
---|
911 | AssertMsg(RT_SUCCESS(rc) || rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc));
|
---|
912 | }
|
---|
913 |
|
---|
914 | X86PDEPAE ZeroPde = {0};
|
---|
915 | return ZeroPde;
|
---|
916 | }
|
---|
917 |
|
---|
918 |
|
---|
919 | /**
|
---|
920 | * Gets the GUEST page directory pointer for the specified address.
|
---|
921 | *
|
---|
922 | * @returns The page directory in question.
|
---|
923 | * @returns NULL if the page directory is not present or on an invalid page.
|
---|
924 | * @param pVCpu The current CPU.
|
---|
925 | * @param GCPtr The address.
|
---|
926 | * @param ppPml4e Page Map Level-4 Entry (out)
|
---|
927 | * @param pPdpe Page directory pointer table entry (out)
|
---|
928 | * @param piPD Receives the index into the returned page directory
|
---|
929 | */
|
---|
930 | DECLINLINE(PX86PDPAE) pgmGstGetLongModePDPtr(PVMCPU pVCpu, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e, PX86PDPE pPdpe, unsigned *piPD)
|
---|
931 | {
|
---|
932 | /* The PMLE4. */
|
---|
933 | PX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pVCpu);
|
---|
934 | if (RT_UNLIKELY(!pGuestPml4))
|
---|
935 | return NULL;
|
---|
936 | const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
|
---|
937 | PCX86PML4E pPml4e = *ppPml4e = &pGuestPml4->a[iPml4];
|
---|
938 | if (!pPml4e->n.u1Present)
|
---|
939 | return NULL;
|
---|
940 | if (RT_UNLIKELY(pPml4e->u & pVCpu->pgm.s.fGstAmd64MbzPml4eMask))
|
---|
941 | return NULL;
|
---|
942 |
|
---|
943 | /* The PDPE. */
|
---|
944 | PCX86PDPT pPdptTemp;
|
---|
945 | int rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, pPml4e->u & X86_PML4E_PG_MASK, &pPdptTemp);
|
---|
946 | if (RT_FAILURE(rc))
|
---|
947 | {
|
---|
948 | AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc));
|
---|
949 | return NULL;
|
---|
950 | }
|
---|
951 | const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
|
---|
952 | *pPdpe = pPdptTemp->a[iPdpt];
|
---|
953 | if (!pPdpe->n.u1Present)
|
---|
954 | return NULL;
|
---|
955 | if (RT_UNLIKELY(pPdpe->u & pVCpu->pgm.s.fGstAmd64MbzPdpeMask))
|
---|
956 | return NULL;
|
---|
957 |
|
---|
958 | /* The PDE. */
|
---|
959 | PX86PDPAE pPD;
|
---|
960 | rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
|
---|
961 | if (RT_FAILURE(rc))
|
---|
962 | {
|
---|
963 | AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc));
|
---|
964 | return NULL;
|
---|
965 | }
|
---|
966 |
|
---|
967 | *piPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
|
---|
968 | return pPD;
|
---|
969 | }
|
---|
970 |
|
---|
971 | #endif /* !IN_RC */
|
---|
972 |
|
---|
973 | /**
|
---|
974 | * Gets the shadow page directory, 32-bit.
|
---|
975 | *
|
---|
976 | * @returns Pointer to the shadow 32-bit PD.
|
---|
977 | * @param pVCpu The current CPU.
|
---|
978 | */
|
---|
979 | DECLINLINE(PX86PD) pgmShwGet32BitPDPtr(PVMCPU pVCpu)
|
---|
980 | {
|
---|
981 | return (PX86PD)PGMPOOL_PAGE_2_PTR_V2(pVCpu->CTX_SUFF(pVM), pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
|
---|
982 | }
|
---|
983 |
|
---|
984 |
|
---|
985 | /**
|
---|
986 | * Gets the shadow page directory entry for the specified address, 32-bit.
|
---|
987 | *
|
---|
988 | * @returns Shadow 32-bit PDE.
|
---|
989 | * @param pVCpu The current CPU.
|
---|
990 | * @param GCPtr The address.
|
---|
991 | */
|
---|
992 | DECLINLINE(X86PDE) pgmShwGet32BitPDE(PVMCPU pVCpu, RTGCPTR GCPtr)
|
---|
993 | {
|
---|
994 | const unsigned iPd = (GCPtr >> X86_PD_SHIFT) & X86_PD_MASK;
|
---|
995 |
|
---|
996 | PX86PD pShwPde = pgmShwGet32BitPDPtr(pVCpu);
|
---|
997 | if (!pShwPde)
|
---|
998 | {
|
---|
999 | X86PDE ZeroPde = {0};
|
---|
1000 | return ZeroPde;
|
---|
1001 | }
|
---|
1002 | return pShwPde->a[iPd];
|
---|
1003 | }
|
---|
1004 |
|
---|
1005 |
|
---|
1006 | /**
|
---|
1007 | * Gets the pointer to the shadow page directory entry for the specified
|
---|
1008 | * address, 32-bit.
|
---|
1009 | *
|
---|
1010 | * @returns Pointer to the shadow 32-bit PDE.
|
---|
1011 | * @param pVCpu The current CPU.
|
---|
1012 | * @param GCPtr The address.
|
---|
1013 | */
|
---|
1014 | DECLINLINE(PX86PDE) pgmShwGet32BitPDEPtr(PVMCPU pVCpu, RTGCPTR GCPtr)
|
---|
1015 | {
|
---|
1016 | const unsigned iPd = (GCPtr >> X86_PD_SHIFT) & X86_PD_MASK;
|
---|
1017 |
|
---|
1018 | PX86PD pPde = pgmShwGet32BitPDPtr(pVCpu);
|
---|
1019 | AssertReturn(pPde, NULL);
|
---|
1020 | return &pPde->a[iPd];
|
---|
1021 | }
|
---|
1022 |
|
---|
1023 |
|
---|
1024 | /**
|
---|
1025 | * Gets the shadow page pointer table, PAE.
|
---|
1026 | *
|
---|
1027 | * @returns Pointer to the shadow PAE PDPT.
|
---|
1028 | * @param pVCpu The current CPU.
|
---|
1029 | */
|
---|
1030 | DECLINLINE(PX86PDPT) pgmShwGetPaePDPTPtr(PVMCPU pVCpu)
|
---|
1031 | {
|
---|
1032 | return (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVCpu->CTX_SUFF(pVM), pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
|
---|
1033 | }
|
---|
1034 |
|
---|
1035 |
|
---|
1036 | /**
|
---|
1037 | * Gets the shadow page directory for the specified address, PAE.
|
---|
1038 | *
|
---|
1039 | * @returns Pointer to the shadow PD.
|
---|
1040 | * @param pVCpu The current CPU.
|
---|
1041 | * @param GCPtr The address.
|
---|
1042 | */
|
---|
1043 | DECLINLINE(PX86PDPAE) pgmShwGetPaePDPtr(PVMCPU pVCpu, RTGCPTR GCPtr)
|
---|
1044 | {
|
---|
1045 | const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
|
---|
1046 | PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
|
---|
1047 |
|
---|
1048 | if (!pPdpt->a[iPdpt].n.u1Present)
|
---|
1049 | return NULL;
|
---|
1050 |
|
---|
1051 | /* Fetch the pgm pool shadow descriptor. */
|
---|
1052 | PVM pVM = pVCpu->CTX_SUFF(pVM);
|
---|
1053 | PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pVM->pgm.s.CTX_SUFF(pPool), pPdpt->a[iPdpt].u & X86_PDPE_PG_MASK);
|
---|
1054 | AssertReturn(pShwPde, NULL);
|
---|
1055 |
|
---|
1056 | return (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
|
---|
1057 | }
|
---|
1058 |
|
---|
1059 |
|
---|
1060 | /**
|
---|
1061 | * Gets the shadow page directory for the specified address, PAE.
|
---|
1062 | *
|
---|
1063 | * @returns Pointer to the shadow PD.
|
---|
1064 | * @param pVCpu The current CPU.
|
---|
1065 | * @param GCPtr The address.
|
---|
1066 | */
|
---|
1067 | DECLINLINE(PX86PDPAE) pgmShwGetPaePDPtr(PVMCPU pVCpu, PX86PDPT pPdpt, RTGCPTR GCPtr)
|
---|
1068 | {
|
---|
1069 | const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
|
---|
1070 |
|
---|
1071 | if (!pPdpt->a[iPdpt].n.u1Present)
|
---|
1072 | return NULL;
|
---|
1073 |
|
---|
1074 | /* Fetch the pgm pool shadow descriptor. */
|
---|
1075 | PVM pVM = pVCpu->CTX_SUFF(pVM);
|
---|
1076 | PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pVM->pgm.s.CTX_SUFF(pPool), pPdpt->a[iPdpt].u & X86_PDPE_PG_MASK);
|
---|
1077 | AssertReturn(pShwPde, NULL);
|
---|
1078 |
|
---|
1079 | return (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
|
---|
1080 | }
|
---|
1081 |
|
---|
1082 |
|
---|
1083 | /**
|
---|
1084 | * Gets the shadow page directory entry, PAE.
|
---|
1085 | *
|
---|
1086 | * @returns PDE.
|
---|
1087 | * @param pVCpu The current CPU.
|
---|
1088 | * @param GCPtr The address.
|
---|
1089 | */
|
---|
1090 | DECLINLINE(X86PDEPAE) pgmShwGetPaePDE(PVMCPU pVCpu, RTGCPTR GCPtr)
|
---|
1091 | {
|
---|
1092 | const unsigned iPd = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
|
---|
1093 |
|
---|
1094 | PX86PDPAE pShwPde = pgmShwGetPaePDPtr(pVCpu, GCPtr);
|
---|
1095 | if (!pShwPde)
|
---|
1096 | {
|
---|
1097 | X86PDEPAE ZeroPde = {0};
|
---|
1098 | return ZeroPde;
|
---|
1099 | }
|
---|
1100 | return pShwPde->a[iPd];
|
---|
1101 | }
|
---|
1102 |
|
---|
1103 |
|
---|
1104 | /**
|
---|
1105 | * Gets the pointer to the shadow page directory entry for an address, PAE.
|
---|
1106 | *
|
---|
1107 | * @returns Pointer to the PDE.
|
---|
1108 | * @param pVCpu The current CPU.
|
---|
1109 | * @param GCPtr The address.
|
---|
1110 | * @remarks Only used by AssertCR3.
|
---|
1111 | */
|
---|
1112 | DECLINLINE(PX86PDEPAE) pgmShwGetPaePDEPtr(PVMCPU pVCpu, RTGCPTR GCPtr)
|
---|
1113 | {
|
---|
1114 | const unsigned iPd = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
|
---|
1115 |
|
---|
1116 | PX86PDPAE pPde = pgmShwGetPaePDPtr(pVCpu, GCPtr);
|
---|
1117 | AssertReturn(pPde, NULL);
|
---|
1118 | return &pPde->a[iPd];
|
---|
1119 | }
|
---|
1120 |
|
---|
1121 | #ifndef IN_RC
|
---|
1122 |
|
---|
1123 | /**
|
---|
1124 | * Gets the shadow page map level-4 pointer.
|
---|
1125 | *
|
---|
1126 | * @returns Pointer to the shadow PML4.
|
---|
1127 | * @param pVCpu The current CPU.
|
---|
1128 | */
|
---|
1129 | DECLINLINE(PX86PML4) pgmShwGetLongModePML4Ptr(PVMCPU pVCpu)
|
---|
1130 | {
|
---|
1131 | return (PX86PML4)PGMPOOL_PAGE_2_PTR_V2(pVCpu->CTX_SUFF(pVM), pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
|
---|
1132 | }
|
---|
1133 |
|
---|
1134 |
|
---|
1135 | /**
|
---|
1136 | * Gets the shadow page map level-4 entry for the specified address.
|
---|
1137 | *
|
---|
1138 | * @returns The entry.
|
---|
1139 | * @param pVCpu The current CPU.
|
---|
1140 | * @param GCPtr The address.
|
---|
1141 | */
|
---|
1142 | DECLINLINE(X86PML4E) pgmShwGetLongModePML4E(PVMCPU pVCpu, RTGCPTR GCPtr)
|
---|
1143 | {
|
---|
1144 | const unsigned iPml4 = ((RTGCUINTPTR64)GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
|
---|
1145 | PX86PML4 pShwPml4 = pgmShwGetLongModePML4Ptr(pVCpu);
|
---|
1146 |
|
---|
1147 | if (!pShwPml4)
|
---|
1148 | {
|
---|
1149 | X86PML4E ZeroPml4e = {0};
|
---|
1150 | return ZeroPml4e;
|
---|
1151 | }
|
---|
1152 | return pShwPml4->a[iPml4];
|
---|
1153 | }
|
---|
1154 |
|
---|
1155 |
|
---|
1156 | /**
|
---|
1157 | * Gets the pointer to the specified shadow page map level-4 entry.
|
---|
1158 | *
|
---|
1159 | * @returns The entry.
|
---|
1160 | * @param pVCpu The current CPU.
|
---|
1161 | * @param iPml4 The PML4 index.
|
---|
1162 | */
|
---|
1163 | DECLINLINE(PX86PML4E) pgmShwGetLongModePML4EPtr(PVMCPU pVCpu, unsigned int iPml4)
|
---|
1164 | {
|
---|
1165 | PX86PML4 pShwPml4 = pgmShwGetLongModePML4Ptr(pVCpu);
|
---|
1166 | if (!pShwPml4)
|
---|
1167 | return NULL;
|
---|
1168 | return &pShwPml4->a[iPml4];
|
---|
1169 | }
|
---|
1170 |
|
---|
1171 | #endif /* !IN_RC */
|
---|
1172 |
|
---|
1173 |
|
---|
1174 | /**
|
---|
1175 | * Cached physical handler lookup.
|
---|
1176 | *
|
---|
1177 | * @returns Physical handler covering @a GCPhys.
|
---|
1178 | * @param pVM The VM handle.
|
---|
1179 | * @param GCPhys The lookup address.
|
---|
1180 | */
|
---|
1181 | DECLINLINE(PPGMPHYSHANDLER) pgmHandlerPhysicalLookup(PVM pVM, RTGCPHYS GCPhys)
|
---|
1182 | {
|
---|
1183 | PPGMPHYSHANDLER pHandler = pVM->pgm.s.CTX_SUFF(pLastPhysHandler);
|
---|
1184 | if ( pHandler
|
---|
1185 | && GCPhys >= pHandler->Core.Key
|
---|
1186 | && GCPhys < pHandler->Core.KeyLast)
|
---|
1187 | {
|
---|
1188 | STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysHandlerLookupHits));
|
---|
1189 | return pHandler;
|
---|
1190 | }
|
---|
1191 |
|
---|
1192 | STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysHandlerLookupMisses));
|
---|
1193 | pHandler = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys);
|
---|
1194 | if (pHandler)
|
---|
1195 | pVM->pgm.s.CTX_SUFF(pLastPhysHandler) = pHandler;
|
---|
1196 | return pHandler;
|
---|
1197 | }
|
---|
1198 |
|
---|
1199 |
|
---|
1200 | /**
|
---|
1201 | * Gets the page state for a physical handler.
|
---|
1202 | *
|
---|
1203 | * @returns The physical handler page state.
|
---|
1204 | * @param pCur The physical handler in question.
|
---|
1205 | */
|
---|
1206 | DECLINLINE(unsigned) pgmHandlerPhysicalCalcState(PPGMPHYSHANDLER pCur)
|
---|
1207 | {
|
---|
1208 | switch (pCur->enmType)
|
---|
1209 | {
|
---|
1210 | case PGMPHYSHANDLERTYPE_PHYSICAL_WRITE:
|
---|
1211 | return PGM_PAGE_HNDL_PHYS_STATE_WRITE;
|
---|
1212 |
|
---|
1213 | case PGMPHYSHANDLERTYPE_MMIO:
|
---|
1214 | case PGMPHYSHANDLERTYPE_PHYSICAL_ALL:
|
---|
1215 | return PGM_PAGE_HNDL_PHYS_STATE_ALL;
|
---|
1216 |
|
---|
1217 | default:
|
---|
1218 | AssertFatalMsgFailed(("Invalid type %d\n", pCur->enmType));
|
---|
1219 | }
|
---|
1220 | }
|
---|
1221 |
|
---|
1222 |
|
---|
1223 | /**
|
---|
1224 | * Gets the page state for a virtual handler.
|
---|
1225 | *
|
---|
1226 | * @returns The virtual handler page state.
|
---|
1227 | * @param pCur The virtual handler in question.
|
---|
1228 | * @remarks This should never be used on a hypervisor access handler.
|
---|
1229 | */
|
---|
1230 | DECLINLINE(unsigned) pgmHandlerVirtualCalcState(PPGMVIRTHANDLER pCur)
|
---|
1231 | {
|
---|
1232 | switch (pCur->enmType)
|
---|
1233 | {
|
---|
1234 | case PGMVIRTHANDLERTYPE_WRITE:
|
---|
1235 | return PGM_PAGE_HNDL_VIRT_STATE_WRITE;
|
---|
1236 | case PGMVIRTHANDLERTYPE_ALL:
|
---|
1237 | return PGM_PAGE_HNDL_VIRT_STATE_ALL;
|
---|
1238 | default:
|
---|
1239 | AssertFatalMsgFailed(("Invalid type %d\n", pCur->enmType));
|
---|
1240 | }
|
---|
1241 | }
|
---|
1242 |
|
---|
1243 |
|
---|
1244 | /**
|
---|
1245 | * Clears one physical page of a virtual handler.
|
---|
1246 | *
|
---|
1247 | * @param pVM The VM handle.
|
---|
1248 | * @param pCur Virtual handler structure.
|
---|
1249 | * @param iPage Physical page index.
|
---|
1250 | *
|
---|
1251 | * @remark Only used when PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL is being set, so no
|
---|
1252 | * need to care about other handlers in the same page.
|
---|
1253 | */
|
---|
1254 | DECLINLINE(void) pgmHandlerVirtualClearPage(PVM pVM, PPGMVIRTHANDLER pCur, unsigned iPage)
|
---|
1255 | {
|
---|
1256 | const PPGMPHYS2VIRTHANDLER pPhys2Virt = &pCur->aPhysToVirt[iPage];
|
---|
1257 |
|
---|
1258 | /*
|
---|
1259 | * Remove the node from the tree (it's supposed to be in the tree if we get here!).
|
---|
1260 | */
|
---|
1261 | #ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
|
---|
1262 | AssertReleaseMsg(pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_IN_TREE,
|
---|
1263 | ("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
|
---|
1264 | pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias));
|
---|
1265 | #endif
|
---|
1266 | if (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_IS_HEAD)
|
---|
1267 | {
|
---|
1268 | /* We're the head of the alias chain. */
|
---|
1269 | PPGMPHYS2VIRTHANDLER pRemove = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysRemove(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysToVirtHandlers, pPhys2Virt->Core.Key); NOREF(pRemove);
|
---|
1270 | #ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
|
---|
1271 | AssertReleaseMsg(pRemove != NULL,
|
---|
1272 | ("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
|
---|
1273 | pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias));
|
---|
1274 | AssertReleaseMsg(pRemove == pPhys2Virt,
|
---|
1275 | ("wanted: pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n"
|
---|
1276 | " got: pRemove=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
|
---|
1277 | pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias,
|
---|
1278 | pRemove, pRemove->Core.Key, pRemove->Core.KeyLast, pRemove->offVirtHandler, pRemove->offNextAlias));
|
---|
1279 | #endif
|
---|
1280 | if (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK)
|
---|
1281 | {
|
---|
1282 | /* Insert the next list in the alias chain into the tree. */
|
---|
1283 | PPGMPHYS2VIRTHANDLER pNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPhys2Virt + (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
|
---|
1284 | #ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
|
---|
1285 | AssertReleaseMsg(pNext->offNextAlias & PGMPHYS2VIRTHANDLER_IN_TREE,
|
---|
1286 | ("pNext=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
|
---|
1287 | pNext, pNext->Core.Key, pNext->Core.KeyLast, pNext->offVirtHandler, pNext->offNextAlias));
|
---|
1288 | #endif
|
---|
1289 | pNext->offNextAlias |= PGMPHYS2VIRTHANDLER_IS_HEAD;
|
---|
1290 | bool fRc = RTAvlroGCPhysInsert(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysToVirtHandlers, &pNext->Core);
|
---|
1291 | AssertRelease(fRc);
|
---|
1292 | }
|
---|
1293 | }
|
---|
1294 | else
|
---|
1295 | {
|
---|
1296 | /* Locate the previous node in the alias chain. */
|
---|
1297 | PPGMPHYS2VIRTHANDLER pPrev = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysToVirtHandlers, pPhys2Virt->Core.Key);
|
---|
1298 | #ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
|
---|
1299 | AssertReleaseMsg(pPrev != pPhys2Virt,
|
---|
1300 | ("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32} pPrev=%p\n",
|
---|
1301 | pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias, pPrev));
|
---|
1302 | #endif
|
---|
1303 | for (;;)
|
---|
1304 | {
|
---|
1305 | PPGMPHYS2VIRTHANDLER pNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPrev + (pPrev->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
|
---|
1306 | if (pNext == pPhys2Virt)
|
---|
1307 | {
|
---|
1308 | /* unlink. */
|
---|
1309 | LogFlow(("pgmHandlerVirtualClearPage: removed %p:{.offNextAlias=%#RX32} from alias chain. prev %p:{.offNextAlias=%#RX32} [%RGp-%RGp]\n",
|
---|
1310 | pPhys2Virt, pPhys2Virt->offNextAlias, pPrev, pPrev->offNextAlias, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast));
|
---|
1311 | if (!(pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK))
|
---|
1312 | pPrev->offNextAlias &= ~PGMPHYS2VIRTHANDLER_OFF_MASK;
|
---|
1313 | else
|
---|
1314 | {
|
---|
1315 | PPGMPHYS2VIRTHANDLER pNewNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPhys2Virt + (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
|
---|
1316 | pPrev->offNextAlias = ((intptr_t)pNewNext - (intptr_t)pPrev)
|
---|
1317 | | (pPrev->offNextAlias & ~PGMPHYS2VIRTHANDLER_OFF_MASK);
|
---|
1318 | }
|
---|
1319 | break;
|
---|
1320 | }
|
---|
1321 |
|
---|
1322 | /* next */
|
---|
1323 | if (pNext == pPrev)
|
---|
1324 | {
|
---|
1325 | #ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
|
---|
1326 | AssertReleaseMsg(pNext != pPrev,
|
---|
1327 | ("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32} pPrev=%p\n",
|
---|
1328 | pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias, pPrev));
|
---|
1329 | #endif
|
---|
1330 | break;
|
---|
1331 | }
|
---|
1332 | pPrev = pNext;
|
---|
1333 | }
|
---|
1334 | }
|
---|
1335 | Log2(("PHYS2VIRT: Removing %RGp-%RGp %#RX32 %s\n",
|
---|
1336 | pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offNextAlias, R3STRING(pCur->pszDesc)));
|
---|
1337 | pPhys2Virt->offNextAlias = 0;
|
---|
1338 | pPhys2Virt->Core.KeyLast = NIL_RTGCPHYS; /* require reinsert */
|
---|
1339 |
|
---|
1340 | /*
|
---|
1341 | * Clear the ram flags for this page.
|
---|
1342 | */
|
---|
1343 | PPGMPAGE pPage = pgmPhysGetPage(pVM, pPhys2Virt->Core.Key);
|
---|
1344 | AssertReturnVoid(pPage);
|
---|
1345 | PGM_PAGE_SET_HNDL_VIRT_STATE(pPage, PGM_PAGE_HNDL_VIRT_STATE_NONE);
|
---|
1346 | }
|
---|
1347 |
|
---|
1348 |
|
---|
1349 | /**
|
---|
1350 | * Internal worker for finding a 'in-use' shadow page give by it's physical address.
|
---|
1351 | *
|
---|
1352 | * @returns Pointer to the shadow page structure.
|
---|
1353 | * @param pPool The pool.
|
---|
1354 | * @param idx The pool page index.
|
---|
1355 | */
|
---|
1356 | DECLINLINE(PPGMPOOLPAGE) pgmPoolGetPageByIdx(PPGMPOOL pPool, unsigned idx)
|
---|
1357 | {
|
---|
1358 | AssertFatalMsg(idx >= PGMPOOL_IDX_FIRST && idx < pPool->cCurPages, ("idx=%d\n", idx));
|
---|
1359 | return &pPool->aPages[idx];
|
---|
1360 | }
|
---|
1361 |
|
---|
1362 |
|
---|
1363 | /**
|
---|
1364 | * Clear references to guest physical memory.
|
---|
1365 | *
|
---|
1366 | * @param pPool The pool.
|
---|
1367 | * @param pPoolPage The pool page.
|
---|
1368 | * @param pPhysPage The physical guest page tracking structure.
|
---|
1369 | * @param iPte Shadow PTE index
|
---|
1370 | */
|
---|
1371 | DECLINLINE(void) pgmTrackDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPoolPage, PPGMPAGE pPhysPage, uint16_t iPte)
|
---|
1372 | {
|
---|
1373 | /*
|
---|
1374 | * Just deal with the simple case here.
|
---|
1375 | */
|
---|
1376 | # ifdef VBOX_STRICT
|
---|
1377 | PVM pVM = pPool->CTX_SUFF(pVM); NOREF(pVM);
|
---|
1378 | # endif
|
---|
1379 | # ifdef LOG_ENABLED
|
---|
1380 | const unsigned uOrg = PGM_PAGE_GET_TRACKING(pPhysPage);
|
---|
1381 | # endif
|
---|
1382 | const unsigned cRefs = PGM_PAGE_GET_TD_CREFS(pPhysPage);
|
---|
1383 | if (cRefs == 1)
|
---|
1384 | {
|
---|
1385 | Assert(pPoolPage->idx == PGM_PAGE_GET_TD_IDX(pPhysPage));
|
---|
1386 | Assert(iPte == PGM_PAGE_GET_PTE_INDEX(pPhysPage));
|
---|
1387 | /* Invalidate the tracking data. */
|
---|
1388 | PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
|
---|
1389 | }
|
---|
1390 | else
|
---|
1391 | pgmPoolTrackPhysExtDerefGCPhys(pPool, pPoolPage, pPhysPage, iPte);
|
---|
1392 | Log2(("pgmTrackDerefGCPhys: %x -> %x pPhysPage=%R[pgmpage]\n", uOrg, PGM_PAGE_GET_TRACKING(pPhysPage), pPhysPage ));
|
---|
1393 | }
|
---|
1394 |
|
---|
1395 |
|
---|
1396 | /**
|
---|
1397 | * Moves the page to the head of the age list.
|
---|
1398 | *
|
---|
1399 | * This is done when the cached page is used in one way or another.
|
---|
1400 | *
|
---|
1401 | * @param pPool The pool.
|
---|
1402 | * @param pPage The cached page.
|
---|
1403 | */
|
---|
1404 | DECLINLINE(void) pgmPoolCacheUsed(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
|
---|
1405 | {
|
---|
1406 | PGM_LOCK_ASSERT_OWNER(pPool->CTX_SUFF(pVM));
|
---|
1407 |
|
---|
1408 | /*
|
---|
1409 | * Move to the head of the age list.
|
---|
1410 | */
|
---|
1411 | if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
|
---|
1412 | {
|
---|
1413 | /* unlink */
|
---|
1414 | pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
|
---|
1415 | if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
|
---|
1416 | pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
|
---|
1417 | else
|
---|
1418 | pPool->iAgeTail = pPage->iAgePrev;
|
---|
1419 |
|
---|
1420 | /* insert at head */
|
---|
1421 | pPage->iAgePrev = NIL_PGMPOOL_IDX;
|
---|
1422 | pPage->iAgeNext = pPool->iAgeHead;
|
---|
1423 | Assert(pPage->iAgeNext != NIL_PGMPOOL_IDX); /* we would've already been head then */
|
---|
1424 | pPool->iAgeHead = pPage->idx;
|
---|
1425 | pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->idx;
|
---|
1426 | }
|
---|
1427 | }
|
---|
1428 |
|
---|
1429 | /**
|
---|
1430 | * Locks a page to prevent flushing (important for cr3 root pages or shadow pae pd pages).
|
---|
1431 | *
|
---|
1432 | * @param pVM VM Handle.
|
---|
1433 | * @param pPage PGM pool page
|
---|
1434 | */
|
---|
1435 | DECLINLINE(void) pgmPoolLockPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
|
---|
1436 | {
|
---|
1437 | PGM_LOCK_ASSERT_OWNER(pPool->CTX_SUFF(pVM));
|
---|
1438 | ASMAtomicIncU32(&pPage->cLocked);
|
---|
1439 | }
|
---|
1440 |
|
---|
1441 |
|
---|
1442 | /**
|
---|
1443 | * Unlocks a page to allow flushing again
|
---|
1444 | *
|
---|
1445 | * @param pVM VM Handle.
|
---|
1446 | * @param pPage PGM pool page
|
---|
1447 | */
|
---|
1448 | DECLINLINE(void) pgmPoolUnlockPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
|
---|
1449 | {
|
---|
1450 | PGM_LOCK_ASSERT_OWNER(pPool->CTX_SUFF(pVM));
|
---|
1451 | Assert(pPage->cLocked);
|
---|
1452 | ASMAtomicDecU32(&pPage->cLocked);
|
---|
1453 | }
|
---|
1454 |
|
---|
1455 |
|
---|
1456 | /**
|
---|
1457 | * Checks if the page is locked (e.g. the active CR3 or one of the four PDs of a PAE PDPT)
|
---|
1458 | *
|
---|
1459 | * @returns VBox status code.
|
---|
1460 | * @param pPage PGM pool page
|
---|
1461 | */
|
---|
1462 | DECLINLINE(bool) pgmPoolIsPageLocked(PPGMPOOLPAGE pPage)
|
---|
1463 | {
|
---|
1464 | if (pPage->cLocked)
|
---|
1465 | {
|
---|
1466 | LogFlow(("pgmPoolIsPageLocked found root page %d\n", pPage->enmKind));
|
---|
1467 | if (pPage->cModifications)
|
---|
1468 | pPage->cModifications = 1; /* reset counter (can't use 0, or else it will be reinserted in the modified list) */
|
---|
1469 | return true;
|
---|
1470 | }
|
---|
1471 | return false;
|
---|
1472 | }
|
---|
1473 |
|
---|
1474 |
|
---|
1475 | /**
|
---|
1476 | * Tells if mappings are to be put into the shadow page table or not.
|
---|
1477 | *
|
---|
1478 | * @returns boolean result
|
---|
1479 | * @param pVM VM handle.
|
---|
1480 | */
|
---|
1481 | DECL_FORCE_INLINE(bool) pgmMapAreMappingsEnabled(PVM pVM)
|
---|
1482 | {
|
---|
1483 | #ifdef PGM_WITHOUT_MAPPINGS
|
---|
1484 | /* There are no mappings in VT-x and AMD-V mode. */
|
---|
1485 | Assert(pVM->pgm.s.fMappingsDisabled);
|
---|
1486 | return false;
|
---|
1487 | #else
|
---|
1488 | return !pVM->pgm.s.fMappingsDisabled;
|
---|
1489 | #endif
|
---|
1490 | }
|
---|
1491 |
|
---|
1492 |
|
---|
1493 | /**
|
---|
1494 | * Checks if the mappings are floating and enabled.
|
---|
1495 | *
|
---|
1496 | * @returns true / false.
|
---|
1497 | * @param pVM The VM handle.
|
---|
1498 | */
|
---|
1499 | DECL_FORCE_INLINE(bool) pgmMapAreMappingsFloating(PVM pVM)
|
---|
1500 | {
|
---|
1501 | #ifdef PGM_WITHOUT_MAPPINGS
|
---|
1502 | /* There are no mappings in VT-x and AMD-V mode. */
|
---|
1503 | Assert(pVM->pgm.s.fMappingsDisabled);
|
---|
1504 | return false;
|
---|
1505 | #else
|
---|
1506 | return !pVM->pgm.s.fMappingsDisabled
|
---|
1507 | && !pVM->pgm.s.fMappingsFixed;
|
---|
1508 | #endif
|
---|
1509 | }
|
---|
1510 |
|
---|
1511 | /** @} */
|
---|
1512 |
|
---|
1513 | #endif
|
---|
1514 |
|
---|