VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllPool.cpp@ 8170

Last change on this file since 8170 was 8155, checked in by vboxsync, 17 years ago

The Big Sun Rebranding Header Change
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 125.3 KB
Line 
1/* $Id: PGMAllPool.cpp 8155 2008-04-18 15:16:47Z vboxsync $ */
2/** @file
3 * PGM Shadow Page Pool.
4 */
5
6/*
7 * Copyright (C) 2006-2007 Sun Microsystems, Inc.
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18 * Clara, CA 95054 USA or visit http://www.sun.com if you need
19 * additional information or have any questions.
20 */
21
22
23/*******************************************************************************
24* Header Files *
25*******************************************************************************/
26#define LOG_GROUP LOG_GROUP_PGM_POOL
27#include <VBox/pgm.h>
28#include <VBox/mm.h>
29#include <VBox/em.h>
30#include <VBox/cpum.h>
31#ifdef IN_GC
32# include <VBox/patm.h>
33#endif
34#include "PGMInternal.h"
35#include <VBox/vm.h>
36#include <VBox/disopcode.h>
37
38#include <VBox/log.h>
39#include <VBox/err.h>
40#include <iprt/asm.h>
41
42
43/*******************************************************************************
44* Internal Functions *
45*******************************************************************************/
46__BEGIN_DECLS
47static void pgmPoolFlushAllInt(PPGMPOOL pPool);
48#ifdef PGMPOOL_WITH_USER_TRACKING
49DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind);
50DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind);
51static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
52#endif
53#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
54static void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint);
55#endif
56#ifdef PGMPOOL_WITH_CACHE
57static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint16_t iUserTable);
58#endif
59#ifdef PGMPOOL_WITH_MONITORING
60static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
61#endif
62#ifndef IN_RING3
63DECLEXPORT(int) pgmPoolAccessHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser);
64#endif
65__END_DECLS
66
67
68/**
69 * Checks if the specified page pool kind is for a 4MB or 2MB guest page.
70 *
71 * @returns true if it's the shadow of a 4MB or 2MB guest page, otherwise false.
72 * @param enmKind The page kind.
73 */
74DECLINLINE(bool) pgmPoolIsBigPage(PGMPOOLKIND enmKind)
75{
76 switch (enmKind)
77 {
78 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
79 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
80 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
81 return true;
82 default:
83 return false;
84 }
85}
86
87
88#ifdef IN_GC
89/**
90 * Maps a pool page into the current context.
91 *
92 * @returns Pointer to the mapping.
93 * @param pVM The VM handle.
94 * @param pPage The page to map.
95 */
96void *pgmGCPoolMapPage(PVM pVM, PPGMPOOLPAGE pPage)
97{
98 /* general pages. */
99 if (pPage->idx >= PGMPOOL_IDX_FIRST)
100 {
101 Assert(pPage->idx < pVM->pgm.s.pPoolGC->cCurPages);
102 void *pv;
103 int rc = PGMGCDynMapHCPage(pVM, pPage->Core.Key, &pv);
104 AssertReleaseRC(rc);
105 return pv;
106 }
107
108 /* special pages. */
109 switch (pPage->idx)
110 {
111 case PGMPOOL_IDX_PD:
112 return pVM->pgm.s.pGC32BitPD;
113 case PGMPOOL_IDX_PAE_PD:
114 case PGMPOOL_IDX_PAE_PD_0:
115 return pVM->pgm.s.apGCPaePDs[0];
116 case PGMPOOL_IDX_PAE_PD_1:
117 return pVM->pgm.s.apGCPaePDs[1];
118 case PGMPOOL_IDX_PAE_PD_2:
119 return pVM->pgm.s.apGCPaePDs[2];
120 case PGMPOOL_IDX_PAE_PD_3:
121 return pVM->pgm.s.apGCPaePDs[3];
122 case PGMPOOL_IDX_PDPT:
123 return pVM->pgm.s.pGCPaePDPT;
124 case PGMPOOL_IDX_PML4:
125 return pVM->pgm.s.pGCPaePML4;
126 default:
127 AssertReleaseMsgFailed(("Invalid index %d\n", pPage->idx));
128 return NULL;
129 }
130}
131#endif /* IN_GC */
132
133
134#ifdef PGMPOOL_WITH_MONITORING
135/**
136 * Determin the size of a write instruction.
137 * @returns number of bytes written.
138 * @param pDis The disassembler state.
139 */
140static unsigned pgmPoolDisasWriteSize(PDISCPUSTATE pDis)
141{
142 /*
143 * This is very crude and possibly wrong for some opcodes,
144 * but since it's not really supposed to be called we can
145 * probably live with that.
146 */
147 return DISGetParamSize(pDis, &pDis->param1);
148}
149
150
151/**
152 * Flushes a chain of pages sharing the same access monitor.
153 *
154 * @returns VBox status code suitable for scheduling.
155 * @param pPool The pool.
156 * @param pPage A page in the chain.
157 */
158int pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
159{
160 LogFlow(("pgmPoolMonitorChainFlush: Flush page %VGp type=%d\n", pPage->GCPhys, pPage->enmKind));
161
162 /*
163 * Find the list head.
164 */
165 uint16_t idx = pPage->idx;
166 if (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
167 {
168 while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
169 {
170 idx = pPage->iMonitoredPrev;
171 Assert(idx != pPage->idx);
172 pPage = &pPool->aPages[idx];
173 }
174 }
175
176 /*
177 * Itereate the list flushing each shadow page.
178 */
179 int rc = VINF_SUCCESS;
180 for (;;)
181 {
182 idx = pPage->iMonitoredNext;
183 Assert(idx != pPage->idx);
184 if (pPage->idx >= PGMPOOL_IDX_FIRST)
185 {
186 int rc2 = pgmPoolFlushPage(pPool, pPage);
187 if (rc2 == VERR_PGM_POOL_CLEARED && rc == VINF_SUCCESS)
188 rc = VINF_PGM_SYNC_CR3;
189 }
190 /* next */
191 if (idx == NIL_PGMPOOL_IDX)
192 break;
193 pPage = &pPool->aPages[idx];
194 }
195 return rc;
196}
197
198
199/**
200 * Wrapper for getting the current context pointer to the entry being modified.
201 *
202 * @returns Pointer to the current context mapping of the entry.
203 * @param pPool The pool.
204 * @param pvFault The fault virtual address.
205 * @param GCPhysFault The fault physical address.
206 * @param cbEntry The entry size.
207 */
208#ifdef IN_RING3
209DECLINLINE(const void *) pgmPoolMonitorGCPtr2CCPtr(PPGMPOOL pPool, RTHCPTR pvFault, RTGCPHYS GCPhysFault, const unsigned cbEntry)
210#else
211DECLINLINE(const void *) pgmPoolMonitorGCPtr2CCPtr(PPGMPOOL pPool, RTGCPTR pvFault, RTGCPHYS GCPhysFault, const unsigned cbEntry)
212#endif
213{
214#ifdef IN_GC
215 return (RTGCPTR)((RTGCUINTPTR)pvFault & ~(RTGCUINTPTR)(cbEntry - 1));
216
217#elif defined(IN_RING0)
218 void *pvRet;
219 int rc = pgmRamGCPhys2HCPtr(&pPool->pVMHC->pgm.s, GCPhysFault & ~(RTGCPHYS)(cbEntry - 1), &pvRet);
220 AssertFatalRCSuccess(rc);
221 return pvRet;
222
223#elif defined(IN_RING3)
224 return (RTHCPTR)((uintptr_t)pvFault & ~(RTHCUINTPTR)(cbEntry - 1));
225#else
226# error "huh?"
227#endif
228}
229
230
231/**
232 * Process shadow entries before they are changed by the guest.
233 *
234 * For PT entries we will clear them. For PD entries, we'll simply check
235 * for mapping conflicts and set the SyncCR3 FF if found.
236 *
237 * @param pPool The pool.
238 * @param pPage The head page.
239 * @param GCPhysFault The guest physical fault address.
240 * @param uAddress In R0 and GC this is the guest context fault address (flat).
241 * In R3 this is the host context 'fault' address.
242 * @param pCpu The disassembler state for figuring out the write size.
243 * This need not be specified if the caller knows we won't do cross entry accesses.
244 */
245#ifdef IN_RING3
246void pgmPoolMonitorChainChanging(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault, RTHCPTR pvAddress, PDISCPUSTATE pCpu)
247#else
248void pgmPoolMonitorChainChanging(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault, RTGCPTR pvAddress, PDISCPUSTATE pCpu)
249#endif
250{
251 Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
252 const unsigned off = GCPhysFault & PAGE_OFFSET_MASK;
253
254 LogFlow(("pgmPoolMonitorChainChanging: %VGv phys=%VGp kind=%d\n", pvAddress, GCPhysFault, pPage->enmKind));
255
256 for (;;)
257 {
258 union
259 {
260 void *pv;
261 PX86PT pPT;
262 PX86PTPAE pPTPae;
263 PX86PD pPD;
264 PX86PDPAE pPDPae;
265 PX86PDPT pPDPT;
266 } uShw;
267 uShw.pv = PGMPOOL_PAGE_2_PTR(pPool->CTXSUFF(pVM), pPage);
268
269 switch (pPage->enmKind)
270 {
271 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
272 {
273 const unsigned iShw = off / sizeof(X86PTE);
274 if (uShw.pPT->a[iShw].n.u1Present)
275 {
276# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
277 PCX86PTE pGstPte = (PCX86PTE)pgmPoolMonitorGCPtr2CCPtr(pPool, pvAddress, GCPhysFault, sizeof(*pGstPte));
278 Log4(("pgmPoolMonitorChainChanging 32_32: deref %VHp GCPhys %VGp\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, pGstPte->u & X86_PTE_PG_MASK));
279 pgmPoolTracDerefGCPhysHint(pPool, pPage,
280 uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK,
281 pGstPte->u & X86_PTE_PG_MASK);
282# endif
283 uShw.pPT->a[iShw].u = 0;
284 }
285 break;
286 }
287
288 /* page/2 sized */
289 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
290 if (!((off ^ pPage->GCPhys) & (PAGE_SIZE / 2)))
291 {
292 const unsigned iShw = (off / sizeof(X86PTE)) & (X86_PG_PAE_ENTRIES - 1);
293 if (uShw.pPTPae->a[iShw].n.u1Present)
294 {
295# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
296 PCX86PTE pGstPte = (PCX86PTE)pgmPoolMonitorGCPtr2CCPtr(pPool, pvAddress, GCPhysFault, sizeof(*pGstPte));
297 Log4(("pgmPoolMonitorChainChanging pae_32: deref %VHp GCPhys %VGp\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, pGstPte->u & X86_PTE_PG_MASK));
298 pgmPoolTracDerefGCPhysHint(pPool, pPage,
299 uShw.pPTPae->a[iShw].u & X86_PTE_PAE_PG_MASK,
300 pGstPte->u & X86_PTE_PG_MASK);
301# endif
302 uShw.pPTPae->a[iShw].u = 0;
303 }
304 }
305 break;
306
307 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
308 {
309 const unsigned iShw = off / sizeof(X86PTEPAE);
310 if (uShw.pPTPae->a[iShw].n.u1Present)
311 {
312# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
313 PCX86PTEPAE pGstPte = (PCX86PTEPAE)pgmPoolMonitorGCPtr2CCPtr(pPool, pvAddress, GCPhysFault, sizeof(*pGstPte));
314 Log4(("pgmPoolMonitorChainChanging pae_32: deref %VHp GCPhys %VGp\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, pGstPte->u & X86_PTE_PAE_PG_MASK));
315 pgmPoolTracDerefGCPhysHint(pPool, pPage,
316 uShw.pPTPae->a[iShw].u & X86_PTE_PAE_PG_MASK,
317 pGstPte->u & X86_PTE_PAE_PG_MASK);
318# endif
319 uShw.pPTPae->a[iShw].u = 0;
320 }
321 break;
322 }
323
324 case PGMPOOLKIND_ROOT_32BIT_PD:
325 {
326 const unsigned iShw = off / sizeof(X86PTE); // ASSUMING 32-bit guest paging!
327 if (uShw.pPD->a[iShw].u & PGM_PDFLAGS_MAPPING)
328 {
329 Assert(pgmMapAreMappingsEnabled(&pPool->CTXSUFF(pVM)->pgm.s));
330 VM_FF_SET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3);
331 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
332 }
333 /* paranoia / a bit assumptive. */
334 else if ( pCpu
335 && (off & 3)
336 && (off & 3) + pgmPoolDisasWriteSize(pCpu) > 4)
337 {
338 const unsigned iShw2 = (off + pgmPoolDisasWriteSize(pCpu) - 1) / sizeof(X86PTE);
339 if ( iShw2 != iShw
340 && iShw2 < ELEMENTS(uShw.pPD->a)
341 && uShw.pPD->a[iShw2].u & PGM_PDFLAGS_MAPPING)
342 {
343 Assert(pgmMapAreMappingsEnabled(&pPool->CTXSUFF(pVM)->pgm.s));
344 VM_FF_SET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3);
345 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
346 }
347 }
348#if 0 /* useful when running PGMAssertCR3(), a bit too troublesome for general use (TLBs). */
349 if ( uShw.pPD->a[iShw].n.u1Present
350 && !VM_FF_ISSET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3))
351 {
352 LogFlow(("pgmPoolMonitorChainChanging: iShw=%#x: %RX32 -> freeing it!\n", iShw, uShw.pPD->a[iShw].u));
353# ifdef IN_GC /* TLB load - we're pushing things a bit... */
354 ASMProbeReadByte(pvAddress);
355# endif
356 pgmPoolFree(pPool->CTXSUFF(pVM), uShw.pPD->a[iShw].u & X86_PDE_PG_MASK, pPage->idx, iShw);
357 uShw.pPD->a[iShw].u = 0;
358 }
359#endif
360 break;
361 }
362
363 case PGMPOOLKIND_ROOT_PAE_PD:
364 {
365 unsigned iShw = (off / sizeof(X86PTE)) * 2; // ASSUMING 32-bit guest paging!
366 for (unsigned i = 0; i < 2; i++, iShw++)
367 {
368 if ((uShw.pPDPae->a[iShw].u & (PGM_PDFLAGS_MAPPING | X86_PDE_P)) == (PGM_PDFLAGS_MAPPING | X86_PDE_P))
369 {
370 Assert(pgmMapAreMappingsEnabled(&pPool->CTXSUFF(pVM)->pgm.s));
371 VM_FF_SET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3);
372 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
373 }
374 /* paranoia / a bit assumptive. */
375 else if ( pCpu
376 && (off & 3)
377 && (off & 3) + pgmPoolDisasWriteSize(pCpu) > 4)
378 {
379 const unsigned iShw2 = iShw + 2;
380 if ( iShw2 < ELEMENTS(uShw.pPDPae->a)
381 && (uShw.pPDPae->a[iShw2].u & (PGM_PDFLAGS_MAPPING | X86_PDE_P)) == (PGM_PDFLAGS_MAPPING | X86_PDE_P))
382 {
383 Assert(pgmMapAreMappingsEnabled(&pPool->CTXSUFF(pVM)->pgm.s));
384 VM_FF_SET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3);
385 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
386 }
387 }
388#if 0 /* useful when running PGMAssertCR3(), a bit too troublesome for general use (TLBs). */
389 if ( uShw.pPDPae->a[iShw].n.u1Present
390 && !VM_FF_ISSET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3))
391 {
392 LogFlow(("pgmPoolMonitorChainChanging: iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPae->a[iShw].u));
393# ifdef IN_GC /* TLB load - we're pushing things a bit... */
394 ASMProbeReadByte(pvAddress);
395# endif
396 pgmPoolFree(pPool->CTXSUFF(pVM), uShw.pPDPae->a[iShw].u & X86_PDE_PAE_PG_MASK, pPage->idx, iShw);
397 uShw.pPDPae->a[iShw].u = 0;
398 }
399#endif
400 }
401 break;
402 }
403
404 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
405 {
406 const unsigned iShw = off / sizeof(X86PTEPAE);
407 if (uShw.pPDPae->a[iShw].u & PGM_PDFLAGS_MAPPING)
408 {
409 Assert(pgmMapAreMappingsEnabled(&pPool->CTXSUFF(pVM)->pgm.s));
410 VM_FF_SET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3);
411 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
412 }
413 /* paranoia / a bit assumptive. */
414 else if ( pCpu
415 && (off & 7)
416 && (off & 7) + pgmPoolDisasWriteSize(pCpu) > sizeof(X86PTEPAE))
417 {
418 const unsigned iShw2 = (off + pgmPoolDisasWriteSize(pCpu) - 1) / sizeof(X86PTEPAE);
419 if ( iShw2 != iShw
420 && iShw2 < ELEMENTS(uShw.pPDPae->a)
421 && uShw.pPDPae->a[iShw2].u & PGM_PDFLAGS_MAPPING)
422 {
423 Assert(pgmMapAreMappingsEnabled(&pPool->CTXSUFF(pVM)->pgm.s));
424 VM_FF_SET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3);
425 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
426 }
427 }
428#if 0 /* useful when running PGMAssertCR3(), a bit too troublesome for general use (TLBs). */
429 if ( uShw.pPDPae->a[iShw].n.u1Present
430 && !VM_FF_ISSET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3))
431 {
432 LogFlow(("pgmPoolMonitorChainChanging: iShw=%#x: %RX32 -> freeing it!\n", iShw, uShw.pPDPae->a[iShw].u));
433# ifdef IN_GC /* TLB load - we're pushing things a bit... */
434 ASMProbeReadByte(pvAddress);
435# endif
436 pgmPoolFree(pPool->CTXSUFF(pVM), uShwpPDPae->a[iShw].u & X86_PDE_PG_MASK, pPage->idx, iShw);
437 uShw.pPDPae->a[iShw].u = 0;
438 }
439#endif
440 break;
441 }
442
443 case PGMPOOLKIND_ROOT_PDPT:
444 {
445 /* Hopefully this doesn't happen very often:
446 * - touching unused parts of the page
447 * - messing with the bits of pd pointers without changing the physical address
448 */
449 const unsigned iShw = off / sizeof(X86PDPE);
450 if (iShw < X86_PG_PAE_PDPE_ENTRIES) /* don't use ELEMENTS(uShw.pPDPT->a), because that's for long mode only */
451 {
452 if (uShw.pPDPT->a[iShw].u & PGM_PLXFLAGS_MAPPING)
453 {
454 Assert(pgmMapAreMappingsEnabled(&pPool->CTXSUFF(pVM)->pgm.s));
455 VM_FF_SET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3);
456 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
457 }
458 /* paranoia / a bit assumptive. */
459 else if ( pCpu
460 && (off & 7)
461 && (off & 7) + pgmPoolDisasWriteSize(pCpu) > sizeof(X86PDPE))
462 {
463 const unsigned iShw2 = (off + pgmPoolDisasWriteSize(pCpu) - 1) / sizeof(X86PDPE);
464 if ( iShw2 != iShw
465 && iShw2 < X86_PG_PAE_PDPE_ENTRIES
466 && uShw.pPDPT->a[iShw2].u & PGM_PLXFLAGS_MAPPING)
467 {
468 Assert(pgmMapAreMappingsEnabled(&pPool->CTXSUFF(pVM)->pgm.s));
469 VM_FF_SET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3);
470 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
471 }
472 }
473 }
474 break;
475 }
476
477 default:
478 AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
479 }
480
481 /* next */
482 if (pPage->iMonitoredNext == NIL_PGMPOOL_IDX)
483 return;
484 pPage = &pPool->aPages[pPage->iMonitoredNext];
485 }
486}
487
488
489# ifndef IN_RING3
490/**
491 * Checks if a access could be a fork operation in progress.
492 *
493 * Meaning, that the guest is setuping up the parent process for Copy-On-Write.
494 *
495 * @returns true if it's likly that we're forking, otherwise false.
496 * @param pPool The pool.
497 * @param pCpu The disassembled instruction.
498 * @param offFault The access offset.
499 */
500DECLINLINE(bool) pgmPoolMonitorIsForking(PPGMPOOL pPool, PDISCPUSTATE pCpu, unsigned offFault)
501{
502 /*
503 * i386 linux is using btr to clear X86_PTE_RW.
504 * The functions involved are (2.6.16 source inspection):
505 * clear_bit
506 * ptep_set_wrprotect
507 * copy_one_pte
508 * copy_pte_range
509 * copy_pmd_range
510 * copy_pud_range
511 * copy_page_range
512 * dup_mmap
513 * dup_mm
514 * copy_mm
515 * copy_process
516 * do_fork
517 */
518 if ( pCpu->pCurInstr->opcode == OP_BTR
519 && !(offFault & 4)
520 /** @todo Validate that the bit index is X86_PTE_RW. */
521 )
522 {
523 STAM_COUNTER_INC(&pPool->CTXMID(StatMonitor,Fork));
524 return true;
525 }
526 return false;
527}
528
529
530/**
531 * Determin whether the page is likely to have been reused.
532 *
533 * @returns true if we consider the page as being reused for a different purpose.
534 * @returns false if we consider it to still be a paging page.
535 * @param pPage The page in question.
536 * @param pCpu The disassembly info for the faulting insturction.
537 * @param pvFault The fault address.
538 *
539 * @remark The REP prefix check is left to the caller because of STOSD/W.
540 */
541DECLINLINE(bool) pgmPoolMonitorIsReused(PPGMPOOLPAGE pPage, PDISCPUSTATE pCpu, RTGCPTR pvFault)
542{
543 switch (pCpu->pCurInstr->opcode)
544 {
545 case OP_PUSH:
546 Log4(("pgmPoolMonitorIsReused: PUSH\n"));
547 return true;
548 case OP_PUSHF:
549 Log4(("pgmPoolMonitorIsReused: PUSHF\n"));
550 return true;
551 case OP_PUSHA:
552 Log4(("pgmPoolMonitorIsReused: PUSHA\n"));
553 return true;
554 case OP_FXSAVE:
555 Log4(("pgmPoolMonitorIsReused: FXSAVE\n"));
556 return true;
557 case OP_MOVNTI: /* solaris - block_zero_no_xmm */
558 Log4(("pgmPoolMonitorIsReused: MOVNTI\n"));
559 return true;
560 case OP_MOVNTDQ: /* solaris - hwblkclr & hwblkpagecopy */
561 Log4(("pgmPoolMonitorIsReused: MOVNTDQ\n"));
562 return true;
563 }
564 if ( (pCpu->param1.flags & USE_REG_GEN32)
565 && (pCpu->param1.base.reg_gen32 == USE_REG_ESP))
566 {
567 Log4(("pgmPoolMonitorIsReused: ESP\n"));
568 return true;
569 }
570
571 //if (pPage->fCR3Mix)
572 // return false;
573 return false;
574}
575
576
577/**
578 * Flushes the page being accessed.
579 *
580 * @returns VBox status code suitable for scheduling.
581 * @param pVM The VM handle.
582 * @param pPool The pool.
583 * @param pPage The pool page (head).
584 * @param pCpu The disassembly of the write instruction.
585 * @param pRegFrame The trap register frame.
586 * @param GCPhysFault The fault address as guest physical address.
587 * @param pvFault The fault address.
588 */
589static int pgmPoolAccessHandlerFlush(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pCpu,
590 PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
591{
592 /*
593 * First, do the flushing.
594 */
595 int rc = pgmPoolMonitorChainFlush(pPool, pPage);
596
597 /*
598 * Emulate the instruction (xp/w2k problem, requires pc/cr2/sp detection).
599 */
600 uint32_t cbWritten;
601 int rc2 = EMInterpretInstructionCPU(pVM, pCpu, pRegFrame, pvFault, &cbWritten);
602 if (VBOX_SUCCESS(rc2))
603 pRegFrame->eip += pCpu->opsize;
604 else if (rc2 == VERR_EM_INTERPRETER)
605 {
606#ifdef IN_GC
607 if (PATMIsPatchGCAddr(pVM, (RTGCPTR)pRegFrame->eip))
608 {
609 LogFlow(("pgmPoolAccessHandlerPTWorker: Interpretation failed for patch code %04x:%RGv, ignoring.\n",
610 pRegFrame->cs, (RTGCPTR)pRegFrame->eip));
611 rc = VINF_SUCCESS;
612 STAM_COUNTER_INC(&pPool->StatMonitorGCIntrFailPatch2);
613 }
614 else
615#endif
616 {
617 rc = VINF_EM_RAW_EMULATE_INSTR;
618 STAM_COUNTER_INC(&pPool->CTXMID(StatMonitor,EmulateInstr));
619 }
620 }
621 else
622 rc = rc2;
623
624 /* See use in pgmPoolAccessHandlerSimple(). */
625 PGM_INVL_GUEST_TLBS();
626
627 LogFlow(("pgmPoolAccessHandlerPT: returns %Vrc (flushed)\n", rc));
628 return rc;
629
630}
631
632
633/**
634 * Handles the STOSD write accesses.
635 *
636 * @returns VBox status code suitable for scheduling.
637 * @param pVM The VM handle.
638 * @param pPool The pool.
639 * @param pPage The pool page (head).
640 * @param pCpu The disassembly of the write instruction.
641 * @param pRegFrame The trap register frame.
642 * @param GCPhysFault The fault address as guest physical address.
643 * @param pvFault The fault address.
644 */
645DECLINLINE(int) pgmPoolAccessHandlerSTOSD(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pCpu,
646 PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
647{
648 /*
649 * Increment the modification counter and insert it into the list
650 * of modified pages the first time.
651 */
652 if (!pPage->cModifications++)
653 pgmPoolMonitorModifiedInsert(pPool, pPage);
654
655 /*
656 * Execute REP STOSD.
657 *
658 * This ASSUMES that we're not invoked by Trap0e on in a out-of-sync
659 * write situation, meaning that it's safe to write here.
660 */
661#ifdef IN_GC
662 uint32_t *pu32 = (uint32_t *)pvFault;
663#else
664 RTGCPTR pu32 = pvFault;
665#endif
666 while (pRegFrame->ecx)
667 {
668 pgmPoolMonitorChainChanging(pPool, pPage, GCPhysFault, pu32, NULL);
669#ifdef IN_GC
670 *pu32++ = pRegFrame->eax;
671#else
672 PGMPhysWriteGCPhys(pVM, GCPhysFault, &pRegFrame->eax, 4);
673 pu32 += 4;
674#endif
675 GCPhysFault += 4;
676 pRegFrame->edi += 4;
677 pRegFrame->ecx--;
678 }
679 pRegFrame->eip += pCpu->opsize;
680
681 /* See use in pgmPoolAccessHandlerSimple(). */
682 PGM_INVL_GUEST_TLBS();
683
684 LogFlow(("pgmPoolAccessHandlerSTOSD: returns\n"));
685 return VINF_SUCCESS;
686}
687
688
689/**
690 * Handles the simple write accesses.
691 *
692 * @returns VBox status code suitable for scheduling.
693 * @param pVM The VM handle.
694 * @param pPool The pool.
695 * @param pPage The pool page (head).
696 * @param pCpu The disassembly of the write instruction.
697 * @param pRegFrame The trap register frame.
698 * @param GCPhysFault The fault address as guest physical address.
699 * @param pvFault The fault address.
700 */
701DECLINLINE(int) pgmPoolAccessHandlerSimple(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pCpu,
702 PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
703{
704 /*
705 * Increment the modification counter and insert it into the list
706 * of modified pages the first time.
707 */
708 if (!pPage->cModifications++)
709 pgmPoolMonitorModifiedInsert(pPool, pPage);
710
711 /*
712 * Clear all the pages. ASSUMES that pvFault is readable.
713 */
714 pgmPoolMonitorChainChanging(pPool, pPage, GCPhysFault, pvFault, pCpu);
715
716 /*
717 * Interpret the instruction.
718 */
719 uint32_t cb;
720 int rc = EMInterpretInstructionCPU(pVM, pCpu, pRegFrame, pvFault, &cb);
721 if (VBOX_SUCCESS(rc))
722 pRegFrame->eip += pCpu->opsize;
723 else if (rc == VERR_EM_INTERPRETER)
724 {
725 LogFlow(("pgmPoolAccessHandlerPTWorker: Interpretation failed for patch code %04x:%RGv - opcode=%d\n",
726 pRegFrame->cs, (RTGCPTR)pRegFrame->eip, pCpu->pCurInstr->opcode));
727 rc = VINF_EM_RAW_EMULATE_INSTR;
728 STAM_COUNTER_INC(&pPool->CTXMID(StatMonitor,EmulateInstr));
729 }
730
731 /*
732 * Quick hack, with logging enabled we're getting stale
733 * code TLBs but no data TLB for EIP and crash in EMInterpretDisasOne.
734 * Flushing here is BAD and expensive, I think EMInterpretDisasOne will
735 * have to be fixed to support this. But that'll have to wait till next week.
736 *
737 * An alternative is to keep track of the changed PTEs together with the
738 * GCPhys from the guest PT. This may proove expensive though.
739 *
740 * At the moment, it's VITAL that it's done AFTER the instruction interpreting
741 * because we need the stale TLBs in some cases (XP boot). This MUST be fixed properly!
742 */
743 PGM_INVL_GUEST_TLBS();
744
745 LogFlow(("pgmPoolAccessHandlerSimple: returns %Vrc cb=%d\n", rc, cb));
746 return rc;
747}
748
749
750/**
751 * \#PF Handler callback for PT write accesses.
752 *
753 * @returns VBox status code (appropriate for GC return).
754 * @param pVM VM Handle.
755 * @param uErrorCode CPU Error code.
756 * @param pRegFrame Trap register frame.
757 * NULL on DMA and other non CPU access.
758 * @param pvFault The fault address (cr2).
759 * @param GCPhysFault The GC physical address corresponding to pvFault.
760 * @param pvUser User argument.
761 */
762DECLEXPORT(int) pgmPoolAccessHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
763{
764 STAM_PROFILE_START(&pVM->pgm.s.CTXSUFF(pPool)->CTXSUFF(StatMonitor), a);
765 PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
766 PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)pvUser;
767 LogFlow(("pgmPoolAccessHandler: pvFault=%p pPage=%p:{.idx=%d} GCPhysFault=%VGp\n", pvFault, pPage, pPage->idx, GCPhysFault));
768
769 /*
770 * We should ALWAYS have the list head as user parameter. This
771 * is because we use that page to record the changes.
772 */
773 Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
774
775 /*
776 * Disassemble the faulting instruction.
777 */
778 DISCPUSTATE Cpu;
779 int rc = EMInterpretDisasOne(pVM, pRegFrame, &Cpu, NULL);
780 AssertRCReturn(rc, rc);
781
782 /*
783 * Check if it's worth dealing with.
784 */
785 bool fReused = false;
786 if ( ( pPage->cModifications < 48 /** @todo #define */ /** @todo need to check that it's not mapping EIP. */ /** @todo adjust this! */
787 || pPage->fCR3Mix)
788 && !(fReused = pgmPoolMonitorIsReused(pPage, &Cpu, pvFault))
789 && !pgmPoolMonitorIsForking(pPool, &Cpu, GCPhysFault & PAGE_OFFSET_MASK))
790 {
791 /*
792 * Simple instructions, no REP prefix.
793 */
794 if (!(Cpu.prefix & (PREFIX_REP | PREFIX_REPNE)))
795 {
796 rc = pgmPoolAccessHandlerSimple(pVM, pPool, pPage, &Cpu, pRegFrame, GCPhysFault, pvFault);
797 STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTXSUFF(pPool)->CTXSUFF(StatMonitor), &pPool->CTXMID(StatMonitor,Handled), a);
798 return rc;
799 }
800
801 /*
802 * Windows is frequently doing small memset() operations (netio test 4k+).
803 * We have to deal with these or we'll kill the cache and performance.
804 */
805 if ( Cpu.pCurInstr->opcode == OP_STOSWD
806 && CPUMGetGuestCPL(pVM, pRegFrame) == 0
807 && pRegFrame->ecx <= 0x20
808 && pRegFrame->ecx * 4 <= PAGE_SIZE - ((uintptr_t)pvFault & PAGE_OFFSET_MASK)
809 && !((uintptr_t)pvFault & 3)
810 && (pRegFrame->eax == 0 || pRegFrame->eax == 0x80) /* the two values observed. */
811 && Cpu.mode == CPUMODE_32BIT
812 && Cpu.opmode == CPUMODE_32BIT
813 && Cpu.addrmode == CPUMODE_32BIT
814 && Cpu.prefix == PREFIX_REP
815 && !pRegFrame->eflags.Bits.u1DF
816 )
817 {
818 rc = pgmPoolAccessHandlerSTOSD(pVM, pPool, pPage, &Cpu, pRegFrame, GCPhysFault, pvFault);
819 STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTXSUFF(pPool)->CTXSUFF(StatMonitor), &pPool->CTXMID(StatMonitor,RepStosd), a);
820 return rc;
821 }
822
823 /* REP prefix, don't bother. */
824 STAM_COUNTER_INC(&pPool->CTXMID(StatMonitor,RepPrefix));
825 Log4(("pgmPoolAccessHandler: eax=%#x ecx=%#x edi=%#x esi=%#x eip=%#x opcode=%d prefix=%#x\n",
826 pRegFrame->eax, pRegFrame->ecx, pRegFrame->edi, pRegFrame->esi, pRegFrame->eip, Cpu.pCurInstr->opcode, Cpu.prefix));
827 }
828
829 /*
830 * Not worth it, so flush it.
831 *
832 * If we considered it to be reused, don't to back to ring-3
833 * to emulate failed instructions since we usually cannot
834 * interpret then. This may be a bit risky, in which case
835 * the reuse detection must be fixed.
836 */
837 rc = pgmPoolAccessHandlerFlush(pVM, pPool, pPage, &Cpu, pRegFrame, GCPhysFault, pvFault);
838 if (rc == VINF_EM_RAW_EMULATE_INSTR && fReused)
839 rc = VINF_SUCCESS;
840 STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTXSUFF(pPool)->CTXSUFF(StatMonitor), &pPool->CTXMID(StatMonitor,FlushPage), a);
841 return rc;
842}
843
844# endif /* !IN_RING3 */
845#endif /* PGMPOOL_WITH_MONITORING */
846
847
848
849#ifdef PGMPOOL_WITH_CACHE
850/**
851 * Inserts a page into the GCPhys hash table.
852 *
853 * @param pPool The pool.
854 * @param pPage The page.
855 */
856DECLINLINE(void) pgmPoolHashInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
857{
858 Log3(("pgmPoolHashInsert: %VGp\n", pPage->GCPhys));
859 Assert(pPage->GCPhys != NIL_RTGCPHYS); Assert(pPage->iNext == NIL_PGMPOOL_IDX);
860 uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
861 pPage->iNext = pPool->aiHash[iHash];
862 pPool->aiHash[iHash] = pPage->idx;
863}
864
865
866/**
867 * Removes a page from the GCPhys hash table.
868 *
869 * @param pPool The pool.
870 * @param pPage The page.
871 */
872DECLINLINE(void) pgmPoolHashRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
873{
874 Log3(("pgmPoolHashRemove: %VGp\n", pPage->GCPhys));
875 uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
876 if (pPool->aiHash[iHash] == pPage->idx)
877 pPool->aiHash[iHash] = pPage->iNext;
878 else
879 {
880 uint16_t iPrev = pPool->aiHash[iHash];
881 for (;;)
882 {
883 const int16_t i = pPool->aPages[iPrev].iNext;
884 if (i == pPage->idx)
885 {
886 pPool->aPages[iPrev].iNext = pPage->iNext;
887 break;
888 }
889 if (i == NIL_PGMPOOL_IDX)
890 {
891 AssertReleaseMsgFailed(("GCPhys=%VGp idx=%#x\n", pPage->GCPhys, pPage->idx));
892 break;
893 }
894 iPrev = i;
895 }
896 }
897 pPage->iNext = NIL_PGMPOOL_IDX;
898}
899
900
901/**
902 * Frees up one cache page.
903 *
904 * @returns VBox status code.
905 * @retval VINF_SUCCESS on success.
906 * @retval VERR_PGM_POOL_CLEARED if the deregistration of a physical handler will cause a light weight pool flush.
907 * @param pPool The pool.
908 * @param iUser The user index.
909 */
910static int pgmPoolCacheFreeOne(PPGMPOOL pPool, uint16_t iUser)
911{
912 Assert(pPool->iAgeHead != pPool->iAgeTail); /* We shouldn't be here if there < 2 cached entries! */
913 STAM_COUNTER_INC(&pPool->StatCacheFreeUpOne);
914
915 /*
916 * Select one page from the tail of the age list.
917 */
918 uint16_t iToFree = pPool->iAgeTail;
919 if (iToFree == iUser)
920 iToFree = pPool->aPages[iToFree].iAgePrev;
921/* This is the alternative to the SyncCR3 pgmPoolCacheUsed calls.
922 if (pPool->aPages[iToFree].iUserHead != NIL_PGMPOOL_USER_INDEX)
923 {
924 uint16_t i = pPool->aPages[iToFree].iAgePrev;
925 for (unsigned j = 0; j < 10 && i != NIL_PGMPOOL_USER_INDEX; j++, i = pPool->aPages[i].iAgePrev)
926 {
927 if (pPool->aPages[iToFree].iUserHead == NIL_PGMPOOL_USER_INDEX)
928 continue;
929 iToFree = i;
930 break;
931 }
932 }
933*/
934 Assert(iToFree != iUser);
935 AssertRelease(iToFree != NIL_PGMPOOL_IDX);
936
937 int rc = pgmPoolFlushPage(pPool, &pPool->aPages[iToFree]);
938 if (rc == VINF_SUCCESS)
939 PGM_INVL_GUEST_TLBS(); /* see PT handler. */
940 return rc;
941}
942
943
944/**
945 * Checks if a kind mismatch is really a page being reused
946 * or if it's just normal remappings.
947 *
948 * @returns true if reused and the cached page (enmKind1) should be flushed
949 * @returns false if not reused.
950 * @param enmKind1 The kind of the cached page.
951 * @param enmKind2 The kind of the requested page.
952 */
953static bool pgmPoolCacheReusedByKind(PGMPOOLKIND enmKind1, PGMPOOLKIND enmKind2)
954{
955 switch (enmKind1)
956 {
957 /*
958 * Never reuse them. There is no remapping in non-paging mode.
959 */
960 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
961 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
962 return true;
963
964 /*
965 * It's perfectly fine to reuse these, except for PAE and non-paging stuff.
966 */
967 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
968 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
969 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
970 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
971 case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
972 switch (enmKind2)
973 {
974 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
975 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
976 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
977 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
978 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
979 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
980 return true;
981 default:
982 return false;
983 }
984
985 /*
986 * It's perfectly fine to reuse these, except for PAE and non-paging stuff.
987 */
988 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
989 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
990 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
991 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
992 switch (enmKind2)
993 {
994 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
995 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
996 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
997 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
998 case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
999 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1000 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1001 return true;
1002 default:
1003 return false;
1004 }
1005
1006 /*
1007 * These cannot be flushed, and it's common to reuse the PDs as PTs.
1008 */
1009 case PGMPOOLKIND_ROOT_32BIT_PD:
1010 case PGMPOOLKIND_ROOT_PAE_PD:
1011 case PGMPOOLKIND_ROOT_PDPT:
1012 case PGMPOOLKIND_ROOT_PML4:
1013 return false;
1014
1015 default:
1016 AssertFatalMsgFailed(("enmKind1=%d\n", enmKind1));
1017 }
1018}
1019
1020
1021/**
1022 * Attempts to satisfy a pgmPoolAlloc request from the cache.
1023 *
1024 * @returns VBox status code.
1025 * @retval VINF_PGM_CACHED_PAGE on success.
1026 * @retval VERR_FILE_NOT_FOUND if not found.
1027 * @param pPool The pool.
1028 * @param GCPhys The GC physical address of the page we're gonna shadow.
1029 * @param enmKind The kind of mapping.
1030 * @param iUser The shadow page pool index of the user table.
1031 * @param iUserTable The index into the user table (shadowed).
1032 * @param ppPage Where to store the pointer to the page.
1033 */
1034static int pgmPoolCacheAlloc(PPGMPOOL pPool, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, uint16_t iUser, uint16_t iUserTable, PPPGMPOOLPAGE ppPage)
1035{
1036 /*
1037 * Look up the GCPhys in the hash.
1038 */
1039 unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
1040 Log3(("pgmPoolCacheAlloc: %VGp kind %d iUser=%d iUserTable=%x SLOT=%d\n", GCPhys, enmKind, iUser, iUserTable, i));
1041 if (i != NIL_PGMPOOL_IDX)
1042 {
1043 do
1044 {
1045 PPGMPOOLPAGE pPage = &pPool->aPages[i];
1046 Log3(("pgmPoolCacheAlloc: slot %d found page %VGp\n", i, pPage->GCPhys));
1047 if (pPage->GCPhys == GCPhys)
1048 {
1049 if ((PGMPOOLKIND)pPage->enmKind == enmKind)
1050 {
1051 int rc = pgmPoolTrackAddUser(pPool, pPage, iUser, iUserTable);
1052 if (VBOX_SUCCESS(rc))
1053 {
1054 *ppPage = pPage;
1055 STAM_COUNTER_INC(&pPool->StatCacheHits);
1056 return VINF_PGM_CACHED_PAGE;
1057 }
1058 return rc;
1059 }
1060
1061 /*
1062 * The kind is different. In some cases we should now flush the page
1063 * as it has been reused, but in most cases this is normal remapping
1064 * of PDs as PT or big pages using the GCPhys field in a slightly
1065 * different way than the other kinds.
1066 */
1067 if (pgmPoolCacheReusedByKind((PGMPOOLKIND)pPage->enmKind, enmKind))
1068 {
1069 STAM_COUNTER_INC(&pPool->StatCacheKindMismatches);
1070 pgmPoolFlushPage(pPool, pPage); /* ASSUMES that VERR_PGM_POOL_CLEARED will be returned by pgmPoolTracInsert. */
1071 PGM_INVL_GUEST_TLBS(); /* see PT handler. */
1072 break;
1073 }
1074 }
1075
1076 /* next */
1077 i = pPage->iNext;
1078 } while (i != NIL_PGMPOOL_IDX);
1079 }
1080
1081 Log3(("pgmPoolCacheAlloc: Missed GCPhys=%RGp enmKind=%d\n", GCPhys, enmKind));
1082 STAM_COUNTER_INC(&pPool->StatCacheMisses);
1083 return VERR_FILE_NOT_FOUND;
1084}
1085
1086
1087/**
1088 * Inserts a page into the cache.
1089 *
1090 * @param pPool The pool.
1091 * @param pPage The cached page.
1092 * @param fCanBeCached Set if the page is fit for caching from the caller's point of view.
1093 */
1094static void pgmPoolCacheInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fCanBeCached)
1095{
1096 /*
1097 * Insert into the GCPhys hash if the page is fit for that.
1098 */
1099 Assert(!pPage->fCached);
1100 if (fCanBeCached)
1101 {
1102 pPage->fCached = true;
1103 pgmPoolHashInsert(pPool, pPage);
1104 Log3(("pgmPoolCacheInsert: Caching %p:{.Core=%RHp, .idx=%d, .enmKind=%d, GCPhys=%RGp}\n",
1105 pPage, pPage->Core.Key, pPage->idx, pPage->enmKind, pPage->GCPhys));
1106 STAM_COUNTER_INC(&pPool->StatCacheCacheable);
1107 }
1108 else
1109 {
1110 Log3(("pgmPoolCacheInsert: Not caching %p:{.Core=%RHp, .idx=%d, .enmKind=%d, GCPhys=%RGp}\n",
1111 pPage, pPage->Core.Key, pPage->idx, pPage->enmKind, pPage->GCPhys));
1112 STAM_COUNTER_INC(&pPool->StatCacheUncacheable);
1113 }
1114
1115 /*
1116 * Insert at the head of the age list.
1117 */
1118 pPage->iAgePrev = NIL_PGMPOOL_IDX;
1119 pPage->iAgeNext = pPool->iAgeHead;
1120 if (pPool->iAgeHead != NIL_PGMPOOL_IDX)
1121 pPool->aPages[pPool->iAgeHead].iAgePrev = pPage->idx;
1122 else
1123 pPool->iAgeTail = pPage->idx;
1124 pPool->iAgeHead = pPage->idx;
1125}
1126
1127
1128/**
1129 * Flushes a cached page.
1130 *
1131 * @param pPool The pool.
1132 * @param pPage The cached page.
1133 */
1134static void pgmPoolCacheFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1135{
1136 Log3(("pgmPoolCacheFlushPage: %VGp\n", pPage->GCPhys));
1137
1138 /*
1139 * Remove the page from the hash.
1140 */
1141 if (pPage->fCached)
1142 {
1143 pPage->fCached = false;
1144 pgmPoolHashRemove(pPool, pPage);
1145 }
1146 else
1147 Assert(pPage->iNext == NIL_PGMPOOL_IDX);
1148
1149 /*
1150 * Remove it from the age list.
1151 */
1152 if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
1153 pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
1154 else
1155 pPool->iAgeTail = pPage->iAgePrev;
1156 if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
1157 pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
1158 else
1159 pPool->iAgeHead = pPage->iAgeNext;
1160 pPage->iAgeNext = NIL_PGMPOOL_IDX;
1161 pPage->iAgePrev = NIL_PGMPOOL_IDX;
1162}
1163#endif /* PGMPOOL_WITH_CACHE */
1164
1165
1166#ifdef PGMPOOL_WITH_MONITORING
1167/**
1168 * Looks for pages sharing the monitor.
1169 *
1170 * @returns Pointer to the head page.
1171 * @returns NULL if not found.
1172 * @param pPool The Pool
1173 * @param pNewPage The page which is going to be monitored.
1174 */
1175static PPGMPOOLPAGE pgmPoolMonitorGetPageByGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pNewPage)
1176{
1177#ifdef PGMPOOL_WITH_CACHE
1178 /*
1179 * Look up the GCPhys in the hash.
1180 */
1181 RTGCPHYS GCPhys = pNewPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1);
1182 unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
1183 if (i == NIL_PGMPOOL_IDX)
1184 return NULL;
1185 do
1186 {
1187 PPGMPOOLPAGE pPage = &pPool->aPages[i];
1188 if ( pPage->GCPhys - GCPhys < PAGE_SIZE
1189 && pPage != pNewPage)
1190 {
1191 switch (pPage->enmKind)
1192 {
1193 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1194 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1195 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1196 case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
1197 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1198 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1199 case PGMPOOLKIND_ROOT_32BIT_PD:
1200 case PGMPOOLKIND_ROOT_PAE_PD:
1201 case PGMPOOLKIND_ROOT_PDPT:
1202 case PGMPOOLKIND_ROOT_PML4:
1203 {
1204 /* find the head */
1205 while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
1206 {
1207 Assert(pPage->iMonitoredPrev != pPage->idx);
1208 pPage = &pPool->aPages[pPage->iMonitoredPrev];
1209 }
1210 return pPage;
1211 }
1212
1213 /* ignore, no monitoring. */
1214 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1215 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1216 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1217 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1218 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1219 break;
1220 default:
1221 AssertFatalMsgFailed(("enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
1222 }
1223 }
1224
1225 /* next */
1226 i = pPage->iNext;
1227 } while (i != NIL_PGMPOOL_IDX);
1228#endif
1229 return NULL;
1230}
1231
1232/**
1233 * Enabled write monitoring of a guest page.
1234 *
1235 * @returns VBox status code.
1236 * @retval VINF_SUCCESS on success.
1237 * @retval VERR_PGM_POOL_CLEARED if the registration of the physical handler will cause a light weight pool flush.
1238 * @param pPool The pool.
1239 * @param pPage The cached page.
1240 */
1241static int pgmPoolMonitorInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1242{
1243 LogFlow(("pgmPoolMonitorInsert %VGp\n", pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1)));
1244
1245 /*
1246 * Filter out the relevant kinds.
1247 */
1248 switch (pPage->enmKind)
1249 {
1250 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1251 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1252 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1253 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1254 case PGMPOOLKIND_ROOT_PDPT:
1255 break;
1256
1257 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1258 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1259 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1260 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1261 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1262 /* Nothing to monitor here. */
1263 return VINF_SUCCESS;
1264
1265 case PGMPOOLKIND_ROOT_32BIT_PD:
1266 case PGMPOOLKIND_ROOT_PAE_PD:
1267#ifdef PGMPOOL_WITH_MIXED_PT_CR3
1268 break;
1269#endif
1270 case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
1271 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1272 case PGMPOOLKIND_ROOT_PML4:
1273 default:
1274 AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
1275 }
1276
1277 /*
1278 * Install handler.
1279 */
1280 int rc;
1281 PPGMPOOLPAGE pPageHead = pgmPoolMonitorGetPageByGCPhys(pPool, pPage);
1282 if (pPageHead)
1283 {
1284 Assert(pPageHead != pPage); Assert(pPageHead->iMonitoredNext != pPage->idx);
1285 Assert(pPageHead->iMonitoredPrev != pPage->idx);
1286 pPage->iMonitoredPrev = pPageHead->idx;
1287 pPage->iMonitoredNext = pPageHead->iMonitoredNext;
1288 if (pPageHead->iMonitoredNext != NIL_PGMPOOL_IDX)
1289 pPool->aPages[pPageHead->iMonitoredNext].iMonitoredPrev = pPage->idx;
1290 pPageHead->iMonitoredNext = pPage->idx;
1291 rc = VINF_SUCCESS;
1292 }
1293 else
1294 {
1295 Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX); Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1296 PVM pVM = pPool->CTXSUFF(pVM);
1297 const RTGCPHYS GCPhysPage = pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1);
1298 rc = PGMHandlerPhysicalRegisterEx(pVM, PGMPHYSHANDLERTYPE_PHYSICAL_WRITE,
1299 GCPhysPage, GCPhysPage + (PAGE_SIZE - 1),
1300 pPool->pfnAccessHandlerR3, MMHyperCCToR3(pVM, pPage),
1301 pPool->pfnAccessHandlerR0, MMHyperCCToR0(pVM, pPage),
1302 pPool->pfnAccessHandlerGC, MMHyperCCToGC(pVM, pPage),
1303 pPool->pszAccessHandler);
1304 /** @todo we should probably deal with out-of-memory conditions here, but for now increasing
1305 * the heap size should suffice. */
1306 AssertFatalRC(rc);
1307 if (pVM->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
1308 rc = VERR_PGM_POOL_CLEARED;
1309 }
1310 pPage->fMonitored = true;
1311 return rc;
1312}
1313
1314
1315/**
1316 * Disables write monitoring of a guest page.
1317 *
1318 * @returns VBox status code.
1319 * @retval VINF_SUCCESS on success.
1320 * @retval VERR_PGM_POOL_CLEARED if the deregistration of the physical handler will cause a light weight pool flush.
1321 * @param pPool The pool.
1322 * @param pPage The cached page.
1323 */
1324static int pgmPoolMonitorFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1325{
1326 /*
1327 * Filter out the relevant kinds.
1328 */
1329 switch (pPage->enmKind)
1330 {
1331 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1332 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1333 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1334 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1335 case PGMPOOLKIND_ROOT_PDPT:
1336 break;
1337
1338 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1339 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1340 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1341 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1342 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1343 /* Nothing to monitor here. */
1344 return VINF_SUCCESS;
1345
1346 case PGMPOOLKIND_ROOT_32BIT_PD:
1347 case PGMPOOLKIND_ROOT_PAE_PD:
1348#ifdef PGMPOOL_WITH_MIXED_PT_CR3
1349 break;
1350#endif
1351 case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
1352 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1353 case PGMPOOLKIND_ROOT_PML4:
1354 default:
1355 AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
1356 }
1357
1358 /*
1359 * Remove the page from the monitored list or uninstall it if last.
1360 */
1361 const PVM pVM = pPool->CTXSUFF(pVM);
1362 int rc;
1363 if ( pPage->iMonitoredNext != NIL_PGMPOOL_IDX
1364 || pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
1365 {
1366 if (pPage->iMonitoredPrev == NIL_PGMPOOL_IDX)
1367 {
1368 PPGMPOOLPAGE pNewHead = &pPool->aPages[pPage->iMonitoredNext];
1369 pNewHead->iMonitoredPrev = NIL_PGMPOOL_IDX;
1370 pNewHead->fCR3Mix = pPage->fCR3Mix;
1371 rc = PGMHandlerPhysicalChangeCallbacks(pVM, pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1),
1372 pPool->pfnAccessHandlerR3, MMHyperCCToR3(pVM, pNewHead),
1373 pPool->pfnAccessHandlerR0, MMHyperCCToR0(pVM, pNewHead),
1374 pPool->pfnAccessHandlerGC, MMHyperCCToGC(pVM, pNewHead),
1375 pPool->pszAccessHandler);
1376 AssertFatalRCSuccess(rc);
1377 pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
1378 }
1379 else
1380 {
1381 pPool->aPages[pPage->iMonitoredPrev].iMonitoredNext = pPage->iMonitoredNext;
1382 if (pPage->iMonitoredNext != NIL_PGMPOOL_IDX)
1383 {
1384 pPool->aPages[pPage->iMonitoredNext].iMonitoredPrev = pPage->iMonitoredPrev;
1385 pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
1386 }
1387 pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
1388 rc = VINF_SUCCESS;
1389 }
1390 }
1391 else
1392 {
1393 rc = PGMHandlerPhysicalDeregister(pVM, pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1));
1394 AssertFatalRC(rc);
1395 if (pVM->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
1396 rc = VERR_PGM_POOL_CLEARED;
1397 }
1398 pPage->fMonitored = false;
1399
1400 /*
1401 * Remove it from the list of modified pages (if in it).
1402 */
1403 pgmPoolMonitorModifiedRemove(pPool, pPage);
1404
1405 return rc;
1406}
1407
1408
1409#ifdef PGMPOOL_WITH_MIXED_PT_CR3
1410/**
1411 * Set or clear the fCR3Mix attribute in a chain of monitored pages.
1412 *
1413 * @param pPool The Pool.
1414 * @param pPage A page in the chain.
1415 * @param fCR3Mix The new fCR3Mix value.
1416 */
1417static void pgmPoolMonitorChainChangeCR3Mix(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fCR3Mix)
1418{
1419 /* current */
1420 pPage->fCR3Mix = fCR3Mix;
1421
1422 /* before */
1423 int16_t idx = pPage->iMonitoredPrev;
1424 while (idx != NIL_PGMPOOL_IDX)
1425 {
1426 pPool->aPages[idx].fCR3Mix = fCR3Mix;
1427 idx = pPool->aPages[idx].iMonitoredPrev;
1428 }
1429
1430 /* after */
1431 idx = pPage->iMonitoredNext;
1432 while (idx != NIL_PGMPOOL_IDX)
1433 {
1434 pPool->aPages[idx].fCR3Mix = fCR3Mix;
1435 idx = pPool->aPages[idx].iMonitoredNext;
1436 }
1437}
1438
1439
1440/**
1441 * Installs or modifies monitoring of a CR3 page (special).
1442 *
1443 * We're pretending the CR3 page is shadowed by the pool so we can use the
1444 * generic mechanisms in detecting chained monitoring. (This also gives us a
1445 * tast of what code changes are required to really pool CR3 shadow pages.)
1446 *
1447 * @returns VBox status code.
1448 * @param pPool The pool.
1449 * @param idxRoot The CR3 (root) page index.
1450 * @param GCPhysCR3 The (new) CR3 value.
1451 */
1452int pgmPoolMonitorMonitorCR3(PPGMPOOL pPool, uint16_t idxRoot, RTGCPHYS GCPhysCR3)
1453{
1454 Assert(idxRoot != NIL_PGMPOOL_IDX && idxRoot < PGMPOOL_IDX_FIRST);
1455 PPGMPOOLPAGE pPage = &pPool->aPages[idxRoot];
1456 LogFlow(("pgmPoolMonitorMonitorCR3: idxRoot=%d pPage=%p:{.GCPhys=%VGp, .fMonitored=%d} GCPhysCR3=%VGp\n",
1457 idxRoot, pPage, pPage->GCPhys, pPage->fMonitored, GCPhysCR3));
1458
1459 /*
1460 * The unlikely case where it already matches.
1461 */
1462 if (pPage->GCPhys == GCPhysCR3)
1463 {
1464 Assert(pPage->fMonitored);
1465 return VINF_SUCCESS;
1466 }
1467
1468 /*
1469 * Flush the current monitoring and remove it from the hash.
1470 */
1471 int rc = VINF_SUCCESS;
1472 if (pPage->fMonitored)
1473 {
1474 pgmPoolMonitorChainChangeCR3Mix(pPool, pPage, false);
1475 rc = pgmPoolMonitorFlush(pPool, pPage);
1476 if (rc == VERR_PGM_POOL_CLEARED)
1477 rc = VINF_SUCCESS;
1478 else
1479 AssertFatalRC(rc);
1480 pgmPoolHashRemove(pPool, pPage);
1481 }
1482
1483 /*
1484 * Monitor the page at the new location and insert it into the hash.
1485 */
1486 pPage->GCPhys = GCPhysCR3;
1487 int rc2 = pgmPoolMonitorInsert(pPool, pPage);
1488 if (rc2 != VERR_PGM_POOL_CLEARED)
1489 {
1490 AssertFatalRC(rc2);
1491 if (rc2 != VINF_SUCCESS && rc == VINF_SUCCESS)
1492 rc = rc2;
1493 }
1494 pgmPoolHashInsert(pPool, pPage);
1495 pgmPoolMonitorChainChangeCR3Mix(pPool, pPage, true);
1496 return rc;
1497}
1498
1499
1500/**
1501 * Removes the monitoring of a CR3 page (special).
1502 *
1503 * @returns VBox status code.
1504 * @param pPool The pool.
1505 * @param idxRoot The CR3 (root) page index.
1506 */
1507int pgmPoolMonitorUnmonitorCR3(PPGMPOOL pPool, uint16_t idxRoot)
1508{
1509 Assert(idxRoot != NIL_PGMPOOL_IDX && idxRoot < PGMPOOL_IDX_FIRST);
1510 PPGMPOOLPAGE pPage = &pPool->aPages[idxRoot];
1511 LogFlow(("pgmPoolMonitorUnmonitorCR3: idxRoot=%d pPage=%p:{.GCPhys=%VGp, .fMonitored=%d}\n",
1512 idxRoot, pPage, pPage->GCPhys, pPage->fMonitored));
1513
1514 if (!pPage->fMonitored)
1515 return VINF_SUCCESS;
1516
1517 pgmPoolMonitorChainChangeCR3Mix(pPool, pPage, false);
1518 int rc = pgmPoolMonitorFlush(pPool, pPage);
1519 if (rc != VERR_PGM_POOL_CLEARED)
1520 AssertFatalRC(rc);
1521 else
1522 rc = VINF_SUCCESS;
1523 pgmPoolHashRemove(pPool, pPage);
1524 Assert(!pPage->fMonitored);
1525 pPage->GCPhys = NIL_RTGCPHYS;
1526 return rc;
1527}
1528#endif /* PGMPOOL_WITH_MIXED_PT_CR3 */
1529
1530
1531/**
1532 * Inserts the page into the list of modified pages.
1533 *
1534 * @param pPool The pool.
1535 * @param pPage The page.
1536 */
1537void pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1538{
1539 Log3(("pgmPoolMonitorModifiedInsert: idx=%d\n", pPage->idx));
1540 AssertMsg( pPage->iModifiedNext == NIL_PGMPOOL_IDX
1541 && pPage->iModifiedPrev == NIL_PGMPOOL_IDX
1542 && pPool->iModifiedHead != pPage->idx,
1543 ("Next=%d Prev=%d idx=%d cModifications=%d Head=%d cModifiedPages=%d\n",
1544 pPage->iModifiedNext, pPage->iModifiedPrev, pPage->idx, pPage->cModifications,
1545 pPool->iModifiedHead, pPool->cModifiedPages));
1546
1547 pPage->iModifiedNext = pPool->iModifiedHead;
1548 if (pPool->iModifiedHead != NIL_PGMPOOL_IDX)
1549 pPool->aPages[pPool->iModifiedHead].iModifiedPrev = pPage->idx;
1550 pPool->iModifiedHead = pPage->idx;
1551 pPool->cModifiedPages++;
1552#ifdef VBOX_WITH_STATISTICS
1553 if (pPool->cModifiedPages > pPool->cModifiedPagesHigh)
1554 pPool->cModifiedPagesHigh = pPool->cModifiedPages;
1555#endif
1556}
1557
1558
1559/**
1560 * Removes the page from the list of modified pages and resets the
1561 * moficiation counter.
1562 *
1563 * @param pPool The pool.
1564 * @param pPage The page which is believed to be in the list of modified pages.
1565 */
1566static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1567{
1568 Log3(("pgmPoolMonitorModifiedRemove: idx=%d cModifications=%d\n", pPage->idx, pPage->cModifications));
1569 if (pPool->iModifiedHead == pPage->idx)
1570 {
1571 Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
1572 pPool->iModifiedHead = pPage->iModifiedNext;
1573 if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
1574 {
1575 pPool->aPages[pPage->iModifiedNext].iModifiedPrev = NIL_PGMPOOL_IDX;
1576 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
1577 }
1578 pPool->cModifiedPages--;
1579 }
1580 else if (pPage->iModifiedPrev != NIL_PGMPOOL_IDX)
1581 {
1582 pPool->aPages[pPage->iModifiedPrev].iModifiedNext = pPage->iModifiedNext;
1583 if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
1584 {
1585 pPool->aPages[pPage->iModifiedNext].iModifiedPrev = pPage->iModifiedPrev;
1586 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
1587 }
1588 pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
1589 pPool->cModifiedPages--;
1590 }
1591 else
1592 Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
1593 pPage->cModifications = 0;
1594}
1595
1596
1597/**
1598 * Zaps the list of modified pages, resetting their modification counters in the process.
1599 *
1600 * @param pVM The VM handle.
1601 */
1602void pgmPoolMonitorModifiedClearAll(PVM pVM)
1603{
1604 PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
1605 LogFlow(("pgmPoolMonitorModifiedClearAll: cModifiedPages=%d\n", pPool->cModifiedPages));
1606
1607 unsigned cPages = 0; NOREF(cPages);
1608 uint16_t idx = pPool->iModifiedHead;
1609 pPool->iModifiedHead = NIL_PGMPOOL_IDX;
1610 while (idx != NIL_PGMPOOL_IDX)
1611 {
1612 PPGMPOOLPAGE pPage = &pPool->aPages[idx];
1613 idx = pPage->iModifiedNext;
1614 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
1615 pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
1616 pPage->cModifications = 0;
1617 Assert(++cPages);
1618 }
1619 AssertMsg(cPages == pPool->cModifiedPages, ("%d != %d\n", cPages, pPool->cModifiedPages));
1620 pPool->cModifiedPages = 0;
1621}
1622
1623
1624/**
1625 * Clear all shadow pages and clear all modification counters.
1626 *
1627 * @param pVM The VM handle.
1628 * @remark Should only be used when monitoring is available, thus placed in
1629 * the PGMPOOL_WITH_MONITORING #ifdef.
1630 */
1631void pgmPoolClearAll(PVM pVM)
1632{
1633 PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
1634 STAM_PROFILE_START(&pPool->StatClearAll, c);
1635 LogFlow(("pgmPoolClearAll: cUsedPages=%d\n", pPool->cUsedPages));
1636
1637 /*
1638 * Iterate all the pages until we've encountered all that in use.
1639 * This is simple but not quite optimal solution.
1640 */
1641 unsigned cModifiedPages = 0; NOREF(cModifiedPages);
1642 unsigned cLeft = pPool->cUsedPages;
1643 unsigned iPage = pPool->cCurPages;
1644 while (--iPage >= PGMPOOL_IDX_FIRST)
1645 {
1646 PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
1647 if (pPage->GCPhys != NIL_RTGCPHYS)
1648 {
1649 switch (pPage->enmKind)
1650 {
1651 /*
1652 * We only care about shadow page tables.
1653 */
1654 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1655 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1656 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1657 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1658 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1659 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1660 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1661 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1662 {
1663#ifdef PGMPOOL_WITH_USER_TRACKING
1664 if (pPage->cPresent)
1665#endif
1666 {
1667 void *pvShw = PGMPOOL_PAGE_2_PTR(pPool->CTXSUFF(pVM), pPage);
1668 STAM_PROFILE_START(&pPool->StatZeroPage, z);
1669 ASMMemZeroPage(pvShw);
1670 STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
1671#ifdef PGMPOOL_WITH_USER_TRACKING
1672 pPage->cPresent = 0;
1673 pPage->iFirstPresent = ~0;
1674#endif
1675 }
1676 }
1677 /* fall thru */
1678
1679 default:
1680 Assert(!pPage->cModifications || ++cModifiedPages);
1681 Assert(pPage->iModifiedNext == NIL_PGMPOOL_IDX || pPage->cModifications);
1682 Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX || pPage->cModifications);
1683 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
1684 pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
1685 pPage->cModifications = 0;
1686 break;
1687
1688 }
1689 if (!--cLeft)
1690 break;
1691 }
1692 }
1693
1694 /* swipe the special pages too. */
1695 for (iPage = PGMPOOL_IDX_FIRST_SPECIAL; iPage < PGMPOOL_IDX_FIRST; iPage++)
1696 {
1697 PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
1698 if (pPage->GCPhys != NIL_RTGCPHYS)
1699 {
1700 Assert(!pPage->cModifications || ++cModifiedPages);
1701 Assert(pPage->iModifiedNext == NIL_PGMPOOL_IDX || pPage->cModifications);
1702 Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX || pPage->cModifications);
1703 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
1704 pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
1705 pPage->cModifications = 0;
1706 }
1707 }
1708
1709 AssertMsg(cModifiedPages == pPool->cModifiedPages, ("%d != %d\n", cModifiedPages, pPool->cModifiedPages));
1710 pPool->iModifiedHead = NIL_PGMPOOL_IDX;
1711 pPool->cModifiedPages = 0;
1712
1713#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1714 /*
1715 * Clear all the GCPhys links and rebuild the phys ext free list.
1716 */
1717 for (PPGMRAMRANGE pRam = pPool->CTXSUFF(pVM)->pgm.s.CTXALLSUFF(pRamRanges);
1718 pRam;
1719 pRam = CTXALLSUFF(pRam->pNext))
1720 {
1721 unsigned iPage = pRam->cb >> PAGE_SHIFT;
1722 while (iPage-- > 0)
1723 pRam->aPages[iPage].HCPhys &= MM_RAM_FLAGS_NO_REFS_MASK; /** @todo PAGE FLAGS */
1724 }
1725
1726 pPool->iPhysExtFreeHead = 0;
1727 PPGMPOOLPHYSEXT paPhysExts = pPool->CTXSUFF(paPhysExts);
1728 const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
1729 for (unsigned i = 0; i < cMaxPhysExts; i++)
1730 {
1731 paPhysExts[i].iNext = i + 1;
1732 paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
1733 paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
1734 paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
1735 }
1736 paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
1737#endif
1738
1739
1740 pPool->cPresent = 0;
1741 STAM_PROFILE_STOP(&pPool->StatClearAll, c);
1742}
1743#endif /* PGMPOOL_WITH_MONITORING */
1744
1745
1746#ifdef PGMPOOL_WITH_USER_TRACKING
1747/**
1748 * Frees up at least one user entry.
1749 *
1750 * @returns VBox status code.
1751 * @retval VINF_SUCCESS if successfully added.
1752 * @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
1753 * @param pPool The pool.
1754 * @param iUser The user index.
1755 */
1756static int pgmPoolTrackFreeOneUser(PPGMPOOL pPool, uint16_t iUser)
1757{
1758 STAM_COUNTER_INC(&pPool->StatTrackFreeUpOneUser);
1759#ifdef PGMPOOL_WITH_CACHE
1760 /*
1761 * Just free cached pages in a braindead fashion.
1762 */
1763 /** @todo walk the age list backwards and free the first with usage. */
1764 int rc = VINF_SUCCESS;
1765 do
1766 {
1767 int rc2 = pgmPoolCacheFreeOne(pPool, iUser);
1768 if (VBOX_FAILURE(rc2) && rc == VINF_SUCCESS)
1769 rc = rc2;
1770 } while (pPool->iUserFreeHead == NIL_PGMPOOL_USER_INDEX);
1771 return rc;
1772#else
1773 /*
1774 * Lazy approach.
1775 */
1776 pgmPoolFlushAllInt(pPool);
1777 return VERR_PGM_POOL_FLUSHED;
1778#endif
1779}
1780
1781
1782/**
1783 * Inserts a page into the cache.
1784 *
1785 * This will create user node for the page, insert it into the GCPhys
1786 * hash, and insert it into the age list.
1787 *
1788 * @returns VBox status code.
1789 * @retval VINF_SUCCESS if successfully added.
1790 * @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
1791 * @retval VERR_PGM_POOL_CLEARED if the deregistration of the physical handler will cause a light weight pool flush.
1792 * @param pPool The pool.
1793 * @param pPage The cached page.
1794 * @param GCPhys The GC physical address of the page we're gonna shadow.
1795 * @param iUser The user index.
1796 * @param iUserTable The user table index.
1797 */
1798DECLINLINE(int) pgmPoolTrackInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhys, uint16_t iUser, uint16_t iUserTable)
1799{
1800 int rc = VINF_SUCCESS;
1801 PPGMPOOLUSER pUser = pPool->CTXSUFF(paUsers);
1802
1803 LogFlow(("pgmPoolTrackInsert iUser %d iUserTable %d\n", iUser, iUserTable));
1804
1805 /*
1806 * Find free a user node.
1807 */
1808 uint16_t i = pPool->iUserFreeHead;
1809 if (i == NIL_PGMPOOL_USER_INDEX)
1810 {
1811 int rc = pgmPoolTrackFreeOneUser(pPool, iUser);
1812 if (VBOX_FAILURE(rc))
1813 return rc;
1814 i = pPool->iUserFreeHead;
1815 }
1816
1817 /*
1818 * Unlink the user node from the free list,
1819 * initialize and insert it into the user list.
1820 */
1821 pPool->iUserFreeHead = pUser[i].iNext;
1822 pUser[i].iNext = NIL_PGMPOOL_USER_INDEX;
1823 pUser[i].iUser = iUser;
1824 pUser[i].iUserTable = iUserTable;
1825 pPage->iUserHead = i;
1826
1827 /*
1828 * Insert into cache and enable monitoring of the guest page if enabled.
1829 *
1830 * Until we implement caching of all levels, including the CR3 one, we'll
1831 * have to make sure we don't try monitor & cache any recursive reuse of
1832 * a monitored CR3 page. Because all windows versions are doing this we'll
1833 * have to be able to do combined access monitoring, CR3 + PT and
1834 * PD + PT (guest PAE).
1835 *
1836 * Update:
1837 * We're now cooperating with the CR3 monitor if an uncachable page is found.
1838 */
1839#if defined(PGMPOOL_WITH_MONITORING) || defined(PGMPOOL_WITH_CACHE)
1840# ifdef PGMPOOL_WITH_MIXED_PT_CR3
1841 const bool fCanBeMonitored = true;
1842# else
1843 bool fCanBeMonitored = pPool->CTXSUFF(pVM)->pgm.s.GCPhysGstCR3Monitored == NIL_RTGCPHYS
1844 || (GCPhys & X86_PTE_PAE_PG_MASK) != (pPool->CTXSUFF(pVM)->pgm.s.GCPhysGstCR3Monitored & X86_PTE_PAE_PG_MASK)
1845 || pgmPoolIsBigPage((PGMPOOLKIND)pPage->enmKind);
1846# endif
1847# ifdef PGMPOOL_WITH_CACHE
1848 pgmPoolCacheInsert(pPool, pPage, fCanBeMonitored); /* This can be expanded. */
1849# endif
1850 if (fCanBeMonitored)
1851 {
1852# ifdef PGMPOOL_WITH_MONITORING
1853 rc = pgmPoolMonitorInsert(pPool, pPage);
1854 if (rc == VERR_PGM_POOL_CLEARED)
1855 {
1856 /* 'Failed' - free the usage, and keep it in the cache (if enabled). */
1857# ifndef PGMPOOL_WITH_CACHE
1858 pgmPoolMonitorFlush(pPool, pPage);
1859 rc = VERR_PGM_POOL_FLUSHED;
1860# endif
1861 pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
1862 pUser[i].iNext = pPool->iUserFreeHead;
1863 pUser[i].iUser = NIL_PGMPOOL_IDX;
1864 pPool->iUserFreeHead = i;
1865 }
1866 }
1867# endif
1868#endif /* PGMPOOL_WITH_MONITORING */
1869 return rc;
1870}
1871
1872
1873# ifdef PGMPOOL_WITH_CACHE /* (only used when the cache is enabled.) */
1874/**
1875 * Adds a user reference to a page.
1876 *
1877 * This will
1878 * This will move the page to the head of the
1879 *
1880 * @returns VBox status code.
1881 * @retval VINF_SUCCESS if successfully added.
1882 * @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
1883 * @param pPool The pool.
1884 * @param pPage The cached page.
1885 * @param iUser The user index.
1886 * @param iUserTable The user table.
1887 */
1888static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint16_t iUserTable)
1889{
1890 PPGMPOOLUSER paUsers = pPool->CTXSUFF(paUsers);
1891
1892 LogFlow(("pgmPoolTrackAddUser iUser %d iUserTable %d\n", iUser, iUserTable));
1893# ifdef VBOX_STRICT
1894 /*
1895 * Check that the entry doesn't already exists.
1896 */
1897 if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
1898 {
1899 uint16_t i = pPage->iUserHead;
1900 do
1901 {
1902 Assert(i < pPool->cMaxUsers);
1903 AssertMsg(paUsers[i].iUser != iUser || paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
1904 i = paUsers[i].iNext;
1905 } while (i != NIL_PGMPOOL_USER_INDEX);
1906 }
1907# endif
1908
1909 /*
1910 * Allocate a user node.
1911 */
1912 uint16_t i = pPool->iUserFreeHead;
1913 if (i == NIL_PGMPOOL_USER_INDEX)
1914 {
1915 int rc = pgmPoolTrackFreeOneUser(pPool, iUser);
1916 if (VBOX_FAILURE(rc))
1917 return rc;
1918 i = pPool->iUserFreeHead;
1919 }
1920 pPool->iUserFreeHead = paUsers[i].iNext;
1921
1922 /*
1923 * Initialize the user node and insert it.
1924 */
1925 paUsers[i].iNext = pPage->iUserHead;
1926 paUsers[i].iUser = iUser;
1927 paUsers[i].iUserTable = iUserTable;
1928 pPage->iUserHead = i;
1929
1930# ifdef PGMPOOL_WITH_CACHE
1931 /*
1932 * Tell the cache to update its replacement stats for this page.
1933 */
1934 pgmPoolCacheUsed(pPool, pPage);
1935# endif
1936 return VINF_SUCCESS;
1937}
1938# endif /* PGMPOOL_WITH_CACHE */
1939
1940
1941/**
1942 * Frees a user record associated with a page.
1943 *
1944 * This does not clear the entry in the user table, it simply replaces the
1945 * user record to the chain of free records.
1946 *
1947 * @param pPool The pool.
1948 * @param HCPhys The HC physical address of the shadow page.
1949 * @param iUser The shadow page pool index of the user table.
1950 * @param iUserTable The index into the user table (shadowed).
1951 */
1952static void pgmPoolTrackFreeUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint16_t iUserTable)
1953{
1954 /*
1955 * Unlink and free the specified user entry.
1956 */
1957 PPGMPOOLUSER paUsers = pPool->CTXSUFF(paUsers);
1958
1959 /* Special: For PAE and 32-bit paging, there are usually no more than one user. */
1960 uint16_t i = pPage->iUserHead;
1961 if ( i != NIL_PGMPOOL_USER_INDEX
1962 && paUsers[i].iUser == iUser
1963 && paUsers[i].iUserTable == iUserTable)
1964 {
1965 pPage->iUserHead = paUsers[i].iNext;
1966
1967 paUsers[i].iUser = NIL_PGMPOOL_IDX;
1968 paUsers[i].iNext = pPool->iUserFreeHead;
1969 pPool->iUserFreeHead = i;
1970 return;
1971 }
1972
1973 /* General: Linear search. */
1974 uint16_t iPrev = NIL_PGMPOOL_USER_INDEX;
1975 while (i != NIL_PGMPOOL_USER_INDEX)
1976 {
1977 if ( paUsers[i].iUser == iUser
1978 && paUsers[i].iUserTable == iUserTable)
1979 {
1980 if (iPrev != NIL_PGMPOOL_USER_INDEX)
1981 paUsers[iPrev].iNext = paUsers[i].iNext;
1982 else
1983 pPage->iUserHead = paUsers[i].iNext;
1984
1985 paUsers[i].iUser = NIL_PGMPOOL_IDX;
1986 paUsers[i].iNext = pPool->iUserFreeHead;
1987 pPool->iUserFreeHead = i;
1988 return;
1989 }
1990 iPrev = i;
1991 i = paUsers[i].iNext;
1992 }
1993
1994 /* Fatal: didn't find it */
1995 AssertFatalMsgFailed(("Didn't find the user entry! iUser=%#x iUserTable=%#x GCPhys=%VGp\n",
1996 iUser, iUserTable, pPage->GCPhys));
1997}
1998
1999
2000/**
2001 * Gets the entry size of a shadow table.
2002 *
2003 * @param enmKind The kind of page.
2004 *
2005 * @returns The size of the entry in bytes. That is, 4 or 8.
2006 * @returns If the kind is not for a table, an assertion is raised and 0 is
2007 * returned.
2008 */
2009DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind)
2010{
2011 switch (enmKind)
2012 {
2013 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2014 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2015 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2016 case PGMPOOLKIND_ROOT_32BIT_PD:
2017 return 4;
2018
2019 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2020 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2021 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2022 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2023 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2024 case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
2025 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2026 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2027 case PGMPOOLKIND_ROOT_PAE_PD:
2028 case PGMPOOLKIND_ROOT_PDPT:
2029 case PGMPOOLKIND_ROOT_PML4:
2030 return 8;
2031
2032 default:
2033 AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
2034 }
2035}
2036
2037
2038/**
2039 * Gets the entry size of a guest table.
2040 *
2041 * @param enmKind The kind of page.
2042 *
2043 * @returns The size of the entry in bytes. That is, 0, 4 or 8.
2044 * @returns If the kind is not for a table, an assertion is raised and 0 is
2045 * returned.
2046 */
2047DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind)
2048{
2049 switch (enmKind)
2050 {
2051 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2052 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2053 case PGMPOOLKIND_ROOT_32BIT_PD:
2054 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2055 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2056 case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
2057 return 4;
2058
2059 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2060 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2061 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2062 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2063 case PGMPOOLKIND_ROOT_PAE_PD:
2064 case PGMPOOLKIND_ROOT_PDPT:
2065 case PGMPOOLKIND_ROOT_PML4:
2066 return 8;
2067
2068 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2069 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2070 /** @todo can we return 0? (nobody is calling this...) */
2071 return 0;
2072
2073 default:
2074 AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
2075 }
2076}
2077
2078
2079#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2080/**
2081 * Scans one shadow page table for mappings of a physical page.
2082 *
2083 * @param pVM The VM handle.
2084 * @param pPhysPage The guest page in question.
2085 * @param iShw The shadow page table.
2086 * @param cRefs The number of references made in that PT.
2087 */
2088static void pgmPoolTrackFlushGCPhysPTInt(PVM pVM, PCPGMPAGE pPhysPage, uint16_t iShw, uint16_t cRefs)
2089{
2090 LogFlow(("pgmPoolTrackFlushGCPhysPT: HCPhys=%RHp iShw=%d cRefs=%d\n", pPhysPage->HCPhys, iShw, cRefs));
2091 PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2092
2093 /*
2094 * Assert sanity.
2095 */
2096 Assert(cRefs == 1);
2097 AssertFatalMsg(iShw < pPool->cCurPages && iShw != NIL_PGMPOOL_IDX, ("iShw=%d\n", iShw));
2098 PPGMPOOLPAGE pPage = &pPool->aPages[iShw];
2099
2100 /*
2101 * Then, clear the actual mappings to the page in the shadow PT.
2102 */
2103 switch (pPage->enmKind)
2104 {
2105 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2106 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2107 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2108 {
2109 const uint32_t u32 = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PTE_P;
2110 PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2111 for (unsigned i = pPage->iFirstPresent; i < ELEMENTS(pPT->a); i++)
2112 if ((pPT->a[i].u & (X86_PTE_PG_MASK | X86_PTE_P)) == u32)
2113 {
2114 Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX32 cRefs=%#x\n", i, pPT->a[i], cRefs));
2115 pPT->a[i].u = 0;
2116 cRefs--;
2117 if (!cRefs)
2118 return;
2119 }
2120#if defined(DEBUG) && !defined(IN_RING0) ///@todo RTLogPrintf is missing in R0.
2121 RTLogPrintf("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent);
2122 for (unsigned i = 0; i < ELEMENTS(pPT->a); i++)
2123 if ((pPT->a[i].u & (X86_PTE_PG_MASK | X86_PTE_P)) == u32)
2124 {
2125 RTLogPrintf("i=%d cRefs=%d\n", i, cRefs--);
2126 pPT->a[i].u = 0;
2127 }
2128#endif
2129 AssertFatalMsgFailed(("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent));
2130 break;
2131 }
2132
2133 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2134 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2135 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2136 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2137 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2138 {
2139 const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PTE_P;
2140 PX86PTPAE pPT = (PX86PTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2141 for (unsigned i = pPage->iFirstPresent; i < ELEMENTS(pPT->a); i++)
2142 if ((pPT->a[i].u & (X86_PTE_PAE_PG_MASK | X86_PTE_P)) == u64)
2143 {
2144 Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX64 cRefs=%#x\n", i, pPT->a[i], cRefs));
2145 pPT->a[i].u = 0;
2146 cRefs--;
2147 if (!cRefs)
2148 return;
2149 }
2150#if defined(DEBUG) && !defined(IN_RING0) ///@todo RTLogPrintf is missing in R0.
2151 RTLogPrintf("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent);
2152 for (unsigned i = 0; i < ELEMENTS(pPT->a); i++)
2153 if ((pPT->a[i].u & (X86_PTE_PAE_PG_MASK | X86_PTE_P)) == u64)
2154 {
2155 RTLogPrintf("i=%d cRefs=%d\n", i, cRefs--);
2156 pPT->a[i].u = 0;
2157 }
2158#endif
2159 AssertFatalMsgFailed(("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent));
2160 break;
2161 }
2162
2163 default:
2164 AssertFatalMsgFailed(("enmKind=%d iShw=%d\n", pPage->enmKind, iShw));
2165 }
2166}
2167
2168
2169/**
2170 * Scans one shadow page table for mappings of a physical page.
2171 *
2172 * @param pVM The VM handle.
2173 * @param pPhysPage The guest page in question.
2174 * @param iShw The shadow page table.
2175 * @param cRefs The number of references made in that PT.
2176 */
2177void pgmPoolTrackFlushGCPhysPT(PVM pVM, PPGMPAGE pPhysPage, uint16_t iShw, uint16_t cRefs)
2178{
2179 PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool); NOREF(pPool);
2180 LogFlow(("pgmPoolTrackFlushGCPhysPT: HCPhys=%RHp iShw=%d cRefs=%d\n", pPhysPage->HCPhys, iShw, cRefs));
2181 STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPT, f);
2182 pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, iShw, cRefs);
2183 pPhysPage->HCPhys &= MM_RAM_FLAGS_NO_REFS_MASK; /** @todo PAGE FLAGS */
2184 STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPT, f);
2185}
2186
2187
2188/**
2189 * Flushes a list of shadow page tables mapping the same physical page.
2190 *
2191 * @param pVM The VM handle.
2192 * @param pPhysPage The guest page in question.
2193 * @param iPhysExt The physical cross reference extent list to flush.
2194 */
2195void pgmPoolTrackFlushGCPhysPTs(PVM pVM, PPGMPAGE pPhysPage, uint16_t iPhysExt)
2196{
2197 PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2198 STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTs, f);
2199 LogFlow(("pgmPoolTrackFlushGCPhysPTs: HCPhys=%RHp iPhysExt\n", pPhysPage->HCPhys, iPhysExt));
2200
2201 const uint16_t iPhysExtStart = iPhysExt;
2202 PPGMPOOLPHYSEXT pPhysExt;
2203 do
2204 {
2205 Assert(iPhysExt < pPool->cMaxPhysExts);
2206 pPhysExt = &pPool->CTXSUFF(paPhysExts)[iPhysExt];
2207 for (unsigned i = 0; i < ELEMENTS(pPhysExt->aidx); i++)
2208 if (pPhysExt->aidx[i] != NIL_PGMPOOL_IDX)
2209 {
2210 pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, pPhysExt->aidx[i], 1);
2211 pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
2212 }
2213
2214 /* next */
2215 iPhysExt = pPhysExt->iNext;
2216 } while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
2217
2218 /* insert the list into the free list and clear the ram range entry. */
2219 pPhysExt->iNext = pPool->iPhysExtFreeHead;
2220 pPool->iPhysExtFreeHead = iPhysExtStart;
2221 pPhysPage->HCPhys &= MM_RAM_FLAGS_NO_REFS_MASK; /** @todo PAGE FLAGS */
2222
2223 STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTs, f);
2224}
2225#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
2226
2227
2228/**
2229 * Scans all shadow page tables for mappings of a physical page.
2230 *
2231 * This may be slow, but it's most likely more efficient than cleaning
2232 * out the entire page pool / cache.
2233 *
2234 * @returns VBox status code.
2235 * @retval VINF_SUCCESS if all references has been successfully cleared.
2236 * @retval VINF_PGM_GCPHYS_ALIASED if we're better off with a CR3 sync and
2237 * a page pool cleaning.
2238 *
2239 * @param pVM The VM handle.
2240 * @param pPhysPage The guest page in question.
2241 */
2242int pgmPoolTrackFlushGCPhysPTsSlow(PVM pVM, PPGMPAGE pPhysPage)
2243{
2244 PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2245 STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTsSlow, s);
2246 LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: cUsedPages=%d cPresent=%d HCPhys=%RHp\n",
2247 pPool->cUsedPages, pPool->cPresent, pPhysPage->HCPhys));
2248
2249#if 1
2250 /*
2251 * There is a limit to what makes sense.
2252 */
2253 if (pPool->cPresent > 1024)
2254 {
2255 LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: giving up... (cPresent=%d)\n", pPool->cPresent));
2256 STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
2257 return VINF_PGM_GCPHYS_ALIASED;
2258 }
2259#endif
2260
2261 /*
2262 * Iterate all the pages until we've encountered all that in use.
2263 * This is simple but not quite optimal solution.
2264 */
2265 const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PTE_P;
2266 const uint32_t u32 = u64;
2267 unsigned cLeft = pPool->cUsedPages;
2268 unsigned iPage = pPool->cCurPages;
2269 while (--iPage >= PGMPOOL_IDX_FIRST)
2270 {
2271 PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
2272 if (pPage->GCPhys != NIL_RTGCPHYS)
2273 {
2274 switch (pPage->enmKind)
2275 {
2276 /*
2277 * We only care about shadow page tables.
2278 */
2279 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2280 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2281 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2282 {
2283 unsigned cPresent = pPage->cPresent;
2284 PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2285 for (unsigned i = pPage->iFirstPresent; i < ELEMENTS(pPT->a); i++)
2286 if (pPT->a[i].n.u1Present)
2287 {
2288 if ((pPT->a[i].u & (X86_PTE_PG_MASK | X86_PTE_P)) == u32)
2289 {
2290 //Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX32\n", iPage, i, pPT->a[i]));
2291 pPT->a[i].u = 0;
2292 }
2293 if (!--cPresent)
2294 break;
2295 }
2296 break;
2297 }
2298
2299 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2300 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2301 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2302 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2303 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2304 {
2305 unsigned cPresent = pPage->cPresent;
2306 PX86PTPAE pPT = (PX86PTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2307 for (unsigned i = pPage->iFirstPresent; i < ELEMENTS(pPT->a); i++)
2308 if (pPT->a[i].n.u1Present)
2309 {
2310 if ((pPT->a[i].u & (X86_PTE_PAE_PG_MASK | X86_PTE_P)) == u64)
2311 {
2312 //Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX64\n", iPage, i, pPT->a[i]));
2313 pPT->a[i].u = 0;
2314 }
2315 if (!--cPresent)
2316 break;
2317 }
2318 break;
2319 }
2320 }
2321 if (!--cLeft)
2322 break;
2323 }
2324 }
2325
2326 pPhysPage->HCPhys &= MM_RAM_FLAGS_NO_REFS_MASK; /** @todo PAGE FLAGS */
2327 STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
2328 return VINF_SUCCESS;
2329}
2330
2331
2332/**
2333 * Clears the user entry in a user table.
2334 *
2335 * This is used to remove all references to a page when flushing it.
2336 */
2337static void pgmPoolTrackClearPageUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PCPGMPOOLUSER pUser)
2338{
2339 Assert(pUser->iUser != NIL_PGMPOOL_IDX);
2340 Assert(pUser->iUser < pPool->cCurPages);
2341
2342 /*
2343 * Map the user page.
2344 */
2345 PPGMPOOLPAGE pUserPage = &pPool->aPages[pUser->iUser];
2346 union
2347 {
2348 uint64_t *pau64;
2349 uint32_t *pau32;
2350 } u;
2351 u.pau64 = (uint64_t *)PGMPOOL_PAGE_2_PTR(pPool->CTXSUFF(pVM), pUserPage);
2352
2353#ifdef VBOX_STRICT
2354 /*
2355 * Some sanity checks.
2356 */
2357 switch (pUserPage->enmKind)
2358 {
2359 case PGMPOOLKIND_ROOT_32BIT_PD:
2360 Assert(!(u.pau32[pUser->iUser] & PGM_PDFLAGS_MAPPING));
2361 Assert(pUser->iUserTable < X86_PG_ENTRIES);
2362 break;
2363 case PGMPOOLKIND_ROOT_PAE_PD:
2364 Assert(!(u.pau64[pUser->iUser] & PGM_PDFLAGS_MAPPING));
2365 Assert(pUser->iUserTable < 2048 && pUser->iUser == PGMPOOL_IDX_PAE_PD);
2366 break;
2367 case PGMPOOLKIND_ROOT_PDPT:
2368 Assert(!(u.pau64[pUser->iUserTable] & PGM_PLXFLAGS_PERMANENT));
2369 Assert(pUser->iUserTable < 4);
2370 break;
2371 case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
2372 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2373 Assert(pUser->iUserTable < X86_PG_PAE_ENTRIES);
2374 break;
2375 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2376 case PGMPOOLKIND_ROOT_PML4:
2377 Assert(!(u.pau64[pUser->iUserTable] & PGM_PLXFLAGS_PERMANENT));
2378 Assert(pUser->iUserTable < X86_PG_PAE_ENTRIES);
2379 break;
2380 default:
2381 AssertMsgFailed(("enmKind=%d\n", pUserPage->enmKind));
2382 break;
2383 }
2384#endif /* VBOX_STRICT */
2385
2386 /*
2387 * Clear the entry in the user page.
2388 */
2389 switch (pUserPage->enmKind)
2390 {
2391 /* 32-bit entries */
2392 case PGMPOOLKIND_ROOT_32BIT_PD:
2393 u.pau32[pUser->iUserTable] = 0;
2394 break;
2395
2396 /* 64-bit entries */
2397 case PGMPOOLKIND_ROOT_PAE_PD:
2398 case PGMPOOLKIND_ROOT_PDPT:
2399 case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
2400 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2401 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2402 case PGMPOOLKIND_ROOT_PML4:
2403 u.pau64[pUser->iUserTable] = 0;
2404 break;
2405
2406 default:
2407 AssertFatalMsgFailed(("enmKind=%d iUser=%#x iUserTable=%#x\n", pUserPage->enmKind, pUser->iUser, pUser->iUserTable));
2408 }
2409}
2410
2411
2412/**
2413 * Clears all users of a page.
2414 */
2415static void pgmPoolTrackClearPageUsers(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2416{
2417 /*
2418 * Free all the user records.
2419 */
2420 PPGMPOOLUSER paUsers = pPool->CTXSUFF(paUsers);
2421 uint16_t i = pPage->iUserHead;
2422 while (i != NIL_PGMPOOL_USER_INDEX)
2423 {
2424 /* Clear enter in user table. */
2425 pgmPoolTrackClearPageUser(pPool, pPage, &paUsers[i]);
2426
2427 /* Free it. */
2428 const uint16_t iNext = paUsers[i].iNext;
2429 paUsers[i].iUser = NIL_PGMPOOL_IDX;
2430 paUsers[i].iNext = pPool->iUserFreeHead;
2431 pPool->iUserFreeHead = i;
2432
2433 /* Next. */
2434 i = iNext;
2435 }
2436 pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
2437}
2438
2439
2440#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2441/**
2442 * Allocates a new physical cross reference extent.
2443 *
2444 * @returns Pointer to the allocated extent on success. NULL if we're out of them.
2445 * @param pVM The VM handle.
2446 * @param piPhysExt Where to store the phys ext index.
2447 */
2448PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVM pVM, uint16_t *piPhysExt)
2449{
2450 PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2451 uint16_t iPhysExt = pPool->iPhysExtFreeHead;
2452 if (iPhysExt == NIL_PGMPOOL_PHYSEXT_INDEX)
2453 {
2454 STAM_COUNTER_INC(&pPool->StamTrackPhysExtAllocFailures);
2455 return NULL;
2456 }
2457 PPGMPOOLPHYSEXT pPhysExt = &pPool->CTXSUFF(paPhysExts)[iPhysExt];
2458 pPool->iPhysExtFreeHead = pPhysExt->iNext;
2459 pPhysExt->iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
2460 *piPhysExt = iPhysExt;
2461 return pPhysExt;
2462}
2463
2464
2465/**
2466 * Frees a physical cross reference extent.
2467 *
2468 * @param pVM The VM handle.
2469 * @param iPhysExt The extent to free.
2470 */
2471void pgmPoolTrackPhysExtFree(PVM pVM, uint16_t iPhysExt)
2472{
2473 PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2474 Assert(iPhysExt < pPool->cMaxPhysExts);
2475 PPGMPOOLPHYSEXT pPhysExt = &pPool->CTXSUFF(paPhysExts)[iPhysExt];
2476 for (unsigned i = 0; i < ELEMENTS(pPhysExt->aidx); i++)
2477 pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
2478 pPhysExt->iNext = pPool->iPhysExtFreeHead;
2479 pPool->iPhysExtFreeHead = iPhysExt;
2480}
2481
2482
2483/**
2484 * Frees a physical cross reference extent.
2485 *
2486 * @param pVM The VM handle.
2487 * @param iPhysExt The extent to free.
2488 */
2489void pgmPoolTrackPhysExtFreeList(PVM pVM, uint16_t iPhysExt)
2490{
2491 PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2492
2493 const uint16_t iPhysExtStart = iPhysExt;
2494 PPGMPOOLPHYSEXT pPhysExt;
2495 do
2496 {
2497 Assert(iPhysExt < pPool->cMaxPhysExts);
2498 pPhysExt = &pPool->CTXSUFF(paPhysExts)[iPhysExt];
2499 for (unsigned i = 0; i < ELEMENTS(pPhysExt->aidx); i++)
2500 pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
2501
2502 /* next */
2503 iPhysExt = pPhysExt->iNext;
2504 } while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
2505
2506 pPhysExt->iNext = pPool->iPhysExtFreeHead;
2507 pPool->iPhysExtFreeHead = iPhysExtStart;
2508}
2509
2510/**
2511 * Insert a reference into a list of physical cross reference extents.
2512 *
2513 * @returns The new ram range flags (top 16-bits).
2514 *
2515 * @param pVM The VM handle.
2516 * @param iPhysExt The physical extent index of the list head.
2517 * @param iShwPT The shadow page table index.
2518 *
2519 */
2520static uint16_t pgmPoolTrackPhysExtInsert(PVM pVM, uint16_t iPhysExt, uint16_t iShwPT)
2521{
2522 PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
2523 PPGMPOOLPHYSEXT paPhysExts = pPool->CTXSUFF(paPhysExts);
2524
2525 /* special common case. */
2526 if (paPhysExts[iPhysExt].aidx[2] == NIL_PGMPOOL_IDX)
2527 {
2528 paPhysExts[iPhysExt].aidx[2] = iShwPT;
2529 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliasedMany);
2530 LogFlow(("pgmPoolTrackPhysExtAddref: %d:{,,%d}\n", iPhysExt, iShwPT));
2531 return iPhysExt | (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT));
2532 }
2533
2534 /* general treatment. */
2535 const uint16_t iPhysExtStart = iPhysExt;
2536 unsigned cMax = 15;
2537 for (;;)
2538 {
2539 Assert(iPhysExt < pPool->cMaxPhysExts);
2540 for (unsigned i = 0; i < ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
2541 if (paPhysExts[iPhysExt].aidx[i] == NIL_PGMPOOL_IDX)
2542 {
2543 paPhysExts[iPhysExt].aidx[i] = iShwPT;
2544 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliasedMany);
2545 LogFlow(("pgmPoolTrackPhysExtAddref: %d:{%d} i=%d cMax=%d\n", iPhysExt, iShwPT, i, cMax));
2546 return iPhysExtStart | (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT));
2547 }
2548 if (!--cMax)
2549 {
2550 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackOverflows);
2551 pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
2552 LogFlow(("pgmPoolTrackPhysExtAddref: overflow (1) iShwPT=%d\n", iShwPT));
2553 return MM_RAM_FLAGS_IDX_OVERFLOWED | (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT));
2554 }
2555 }
2556
2557 /* add another extent to the list. */
2558 PPGMPOOLPHYSEXT pNew = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
2559 if (!pNew)
2560 {
2561 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackOverflows);
2562 pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
2563 return MM_RAM_FLAGS_IDX_OVERFLOWED | (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT));
2564 }
2565 pNew->iNext = iPhysExtStart;
2566 pNew->aidx[0] = iShwPT;
2567 LogFlow(("pgmPoolTrackPhysExtAddref: added new extent %d:{%d}->%d\n", iPhysExt, iShwPT, iPhysExtStart));
2568 return iPhysExt | (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT));
2569}
2570
2571
2572/**
2573 * Add a reference to guest physical page where extents are in use.
2574 *
2575 * @returns The new ram range flags (top 16-bits).
2576 *
2577 * @param pVM The VM handle.
2578 * @param u16 The ram range flags (top 16-bits).
2579 * @param iShwPT The shadow page table index.
2580 */
2581uint16_t pgmPoolTrackPhysExtAddref(PVM pVM, uint16_t u16, uint16_t iShwPT)
2582{
2583 if ((u16 >> (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT)) != MM_RAM_FLAGS_CREFS_PHYSEXT)
2584 {
2585 /*
2586 * Convert to extent list.
2587 */
2588 Assert((u16 >> (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT)) == 1);
2589 uint16_t iPhysExt;
2590 PPGMPOOLPHYSEXT pPhysExt = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
2591 if (pPhysExt)
2592 {
2593 LogFlow(("pgmPoolTrackPhysExtAddref: new extent: %d:{%d, %d}\n", iPhysExt, u16 & MM_RAM_FLAGS_IDX_MASK, iShwPT));
2594 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliased);
2595 pPhysExt->aidx[0] = u16 & MM_RAM_FLAGS_IDX_MASK;
2596 pPhysExt->aidx[1] = iShwPT;
2597 u16 = iPhysExt | (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT));
2598 }
2599 else
2600 u16 = MM_RAM_FLAGS_IDX_OVERFLOWED | (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT));
2601 }
2602 else if (u16 != (MM_RAM_FLAGS_IDX_OVERFLOWED | (MM_RAM_FLAGS_CREFS_PHYSEXT << (MM_RAM_FLAGS_CREFS_SHIFT - MM_RAM_FLAGS_IDX_SHIFT))))
2603 {
2604 /*
2605 * Insert into the extent list.
2606 */
2607 u16 = pgmPoolTrackPhysExtInsert(pVM, u16 & MM_RAM_FLAGS_IDX_MASK, iShwPT);
2608 }
2609 else
2610 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliasedLots);
2611 return u16;
2612}
2613
2614
2615/**
2616 * Clear references to guest physical memory.
2617 *
2618 * @param pPool The pool.
2619 * @param pPage The page.
2620 * @param pPhysPage Pointer to the aPages entry in the ram range.
2621 */
2622void pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMPAGE pPhysPage)
2623{
2624 const unsigned cRefs = pPhysPage->HCPhys >> MM_RAM_FLAGS_CREFS_SHIFT; /** @todo PAGE FLAGS */
2625 AssertFatalMsg(cRefs == MM_RAM_FLAGS_CREFS_PHYSEXT, ("cRefs=%d HCPhys=%RHp pPage=%p:{.idx=%d}\n", cRefs, pPhysPage->HCPhys, pPage, pPage->idx));
2626
2627 uint16_t iPhysExt = (pPhysPage->HCPhys >> MM_RAM_FLAGS_IDX_SHIFT) & MM_RAM_FLAGS_IDX_MASK;
2628 if (iPhysExt != MM_RAM_FLAGS_IDX_OVERFLOWED)
2629 {
2630 uint16_t iPhysExtPrev = NIL_PGMPOOL_PHYSEXT_INDEX;
2631 PPGMPOOLPHYSEXT paPhysExts = pPool->CTXSUFF(paPhysExts);
2632 do
2633 {
2634 Assert(iPhysExt < pPool->cMaxPhysExts);
2635
2636 /*
2637 * Look for the shadow page and check if it's all freed.
2638 */
2639 for (unsigned i = 0; i < ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
2640 {
2641 if (paPhysExts[iPhysExt].aidx[i] == pPage->idx)
2642 {
2643 paPhysExts[iPhysExt].aidx[i] = NIL_PGMPOOL_IDX;
2644
2645 for (i = 0; i < ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
2646 if (paPhysExts[iPhysExt].aidx[i] != NIL_PGMPOOL_IDX)
2647 {
2648 LogFlow(("pgmPoolTrackPhysExtDerefGCPhys: HCPhys=%RX64 idx=%d\n", pPhysPage->HCPhys, pPage->idx));
2649 return;
2650 }
2651
2652 /* we can free the node. */
2653 PVM pVM = pPool->CTXSUFF(pVM);
2654 const uint16_t iPhysExtNext = paPhysExts[iPhysExt].iNext;
2655 if ( iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX
2656 && iPhysExtNext == NIL_PGMPOOL_PHYSEXT_INDEX)
2657 {
2658 /* lonely node */
2659 pgmPoolTrackPhysExtFree(pVM, iPhysExt);
2660 LogFlow(("pgmPoolTrackPhysExtDerefGCPhys: HCPhys=%RX64 idx=%d lonely\n", pPhysPage->HCPhys, pPage->idx));
2661 pPhysPage->HCPhys &= MM_RAM_FLAGS_NO_REFS_MASK; /** @todo PAGE FLAGS */
2662 }
2663 else if (iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX)
2664 {
2665 /* head */
2666 LogFlow(("pgmPoolTrackPhysExtDerefGCPhys: HCPhys=%RX64 idx=%d head\n", pPhysPage->HCPhys, pPage->idx));
2667 pPhysPage->HCPhys = (pPhysPage->HCPhys & MM_RAM_FLAGS_NO_REFS_MASK) /** @todo PAGE FLAGS */
2668 | ((uint64_t)MM_RAM_FLAGS_CREFS_PHYSEXT << MM_RAM_FLAGS_CREFS_SHIFT)
2669 | ((uint64_t)iPhysExtNext << MM_RAM_FLAGS_IDX_SHIFT);
2670 pgmPoolTrackPhysExtFree(pVM, iPhysExt);
2671 }
2672 else
2673 {
2674 /* in list */
2675 LogFlow(("pgmPoolTrackPhysExtDerefGCPhys: HCPhys=%RX64 idx=%d\n", pPhysPage->HCPhys, pPage->idx));
2676 paPhysExts[iPhysExtPrev].iNext = iPhysExtNext;
2677 pgmPoolTrackPhysExtFree(pVM, iPhysExt);
2678 }
2679 iPhysExt = iPhysExtNext;
2680 return;
2681 }
2682 }
2683
2684 /* next */
2685 iPhysExtPrev = iPhysExt;
2686 iPhysExt = paPhysExts[iPhysExt].iNext;
2687 } while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
2688
2689 AssertFatalMsgFailed(("not-found! cRefs=%d HCPhys=%RHp pPage=%p:{.idx=%d}\n", cRefs, pPhysPage->HCPhys, pPage, pPage->idx));
2690 }
2691 else /* nothing to do */
2692 LogFlow(("pgmPoolTrackPhysExtDerefGCPhys: HCPhys=%RX64\n", pPhysPage->HCPhys));
2693}
2694
2695
2696
2697/**
2698 * Clear references to guest physical memory.
2699 *
2700 * This is the same as pgmPoolTracDerefGCPhys except that the guest physical address
2701 * is assumed to be correct, so the linear search can be skipped and we can assert
2702 * at an earlier point.
2703 *
2704 * @param pPool The pool.
2705 * @param pPage The page.
2706 * @param HCPhys The host physical address corresponding to the guest page.
2707 * @param GCPhys The guest physical address corresponding to HCPhys.
2708 */
2709static void pgmPoolTracDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhys)
2710{
2711 /*
2712 * Walk range list.
2713 */
2714 PPGMRAMRANGE pRam = pPool->CTXSUFF(pVM)->pgm.s.CTXALLSUFF(pRamRanges);
2715 while (pRam)
2716 {
2717 RTGCPHYS off = GCPhys - pRam->GCPhys;
2718 if (off < pRam->cb)
2719 {
2720 /* does it match? */
2721 const unsigned iPage = off >> PAGE_SHIFT;
2722 Assert(PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]));
2723 if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
2724 {
2725 pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage]);
2726 return;
2727 }
2728 break;
2729 }
2730 pRam = CTXALLSUFF(pRam->pNext);
2731 }
2732 AssertFatalMsgFailed(("HCPhys=%VHp GCPhys=%VGp\n", HCPhys, GCPhys));
2733}
2734
2735
2736/**
2737 * Clear references to guest physical memory.
2738 *
2739 * @param pPool The pool.
2740 * @param pPage The page.
2741 * @param HCPhys The host physical address corresponding to the guest page.
2742 * @param GCPhysHint The guest physical address which may corresponding to HCPhys.
2743 */
2744static void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint)
2745{
2746 /*
2747 * Walk range list.
2748 */
2749 PPGMRAMRANGE pRam = pPool->CTXSUFF(pVM)->pgm.s.CTXALLSUFF(pRamRanges);
2750 while (pRam)
2751 {
2752 RTGCPHYS off = GCPhysHint - pRam->GCPhys;
2753 if (off < pRam->cb)
2754 {
2755 /* does it match? */
2756 const unsigned iPage = off >> PAGE_SHIFT;
2757 Assert(PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]));
2758 if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
2759 {
2760 pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage]);
2761 return;
2762 }
2763 break;
2764 }
2765 pRam = CTXALLSUFF(pRam->pNext);
2766 }
2767
2768 /*
2769 * Damn, the hint didn't work. We'll have to do an expensive linear search.
2770 */
2771 STAM_COUNTER_INC(&pPool->StatTrackLinearRamSearches);
2772 pRam = pPool->CTXSUFF(pVM)->pgm.s.CTXALLSUFF(pRamRanges);
2773 while (pRam)
2774 {
2775 unsigned iPage = pRam->cb >> PAGE_SHIFT;
2776 while (iPage-- > 0)
2777 {
2778 if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
2779 {
2780 Log4(("pgmPoolTracDerefGCPhysHint: Linear HCPhys=%VHp GCPhysHint=%VGp GCPhysReal=%VGp\n",
2781 HCPhys, GCPhysHint, pRam->GCPhys + (iPage << PAGE_SHIFT)));
2782 pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage]);
2783 return;
2784 }
2785 }
2786 pRam = CTXALLSUFF(pRam->pNext);
2787 }
2788
2789 AssertFatalMsgFailed(("HCPhys=%VHp GCPhysHint=%VGp\n", HCPhys, GCPhysHint));
2790}
2791
2792
2793/**
2794 * Clear references to guest physical memory in a 32-bit / 32-bit page table.
2795 *
2796 * @param pPool The pool.
2797 * @param pPage The page.
2798 * @param pShwPT The shadow page table (mapping of the page).
2799 * @param pGstPT The guest page table.
2800 */
2801DECLINLINE(void) pgmPoolTrackDerefPT32Bit32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT, PCX86PT pGstPT)
2802{
2803 for (unsigned i = pPage->iFirstPresent; i < ELEMENTS(pShwPT->a); i++)
2804 if (pShwPT->a[i].n.u1Present)
2805 {
2806 Log4(("pgmPoolTrackDerefPT32Bit32Bit: i=%d pte=%RX32 hint=%RX32\n",
2807 i, pShwPT->a[i].u & X86_PTE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK));
2808 pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK);
2809 if (!--pPage->cPresent)
2810 break;
2811 }
2812}
2813
2814
2815/**
2816 * Clear references to guest physical memory in a PAE / 32-bit page table.
2817 *
2818 * @param pPool The pool.
2819 * @param pPage The page.
2820 * @param pShwPT The shadow page table (mapping of the page).
2821 * @param pGstPT The guest page table (just a half one).
2822 */
2823DECLINLINE(void) pgmPoolTrackDerefPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT, PCX86PT pGstPT)
2824{
2825 for (unsigned i = 0; i < ELEMENTS(pShwPT->a); i++)
2826 if (pShwPT->a[i].n.u1Present)
2827 {
2828 Log4(("pgmPoolTrackDerefPTPae32Bit: i=%d pte=%RX32 hint=%RX32\n",
2829 i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK));
2830 pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK);
2831 }
2832}
2833
2834
2835/**
2836 * Clear references to guest physical memory in a PAE / PAE page table.
2837 *
2838 * @param pPool The pool.
2839 * @param pPage The page.
2840 * @param pShwPT The shadow page table (mapping of the page).
2841 * @param pGstPT The guest page table.
2842 */
2843DECLINLINE(void) pgmPoolTrackDerefPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT, PCX86PTPAE pGstPT)
2844{
2845 for (unsigned i = 0; i < ELEMENTS(pShwPT->a); i++)
2846 if (pShwPT->a[i].n.u1Present)
2847 {
2848 Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX32 hint=%RX32\n",
2849 i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK));
2850 pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK);
2851 }
2852}
2853
2854
2855/**
2856 * Clear references to guest physical memory in a 32-bit / 4MB page table.
2857 *
2858 * @param pPool The pool.
2859 * @param pPage The page.
2860 * @param pShwPT The shadow page table (mapping of the page).
2861 */
2862DECLINLINE(void) pgmPoolTrackDerefPT32Bit4MB(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT)
2863{
2864 RTGCPHYS GCPhys = pPage->GCPhys;
2865 for (unsigned i = 0; i < ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
2866 if (pShwPT->a[i].n.u1Present)
2867 {
2868 Log4(("pgmPoolTrackDerefPT32Bit4MB: i=%d pte=%RX32 GCPhys=%RGp\n",
2869 i, pShwPT->a[i].u & X86_PTE_PG_MASK, GCPhys));
2870 pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & X86_PTE_PG_MASK, GCPhys);
2871 }
2872}
2873
2874
2875/**
2876 * Clear references to guest physical memory in a PAE / 2/4MB page table.
2877 *
2878 * @param pPool The pool.
2879 * @param pPage The page.
2880 * @param pShwPT The shadow page table (mapping of the page).
2881 */
2882DECLINLINE(void) pgmPoolTrackDerefPTPaeBig(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT)
2883{
2884 RTGCPHYS GCPhys = pPage->GCPhys;
2885 for (unsigned i = 0; i < ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
2886 if (pShwPT->a[i].n.u1Present)
2887 {
2888 Log4(("pgmPoolTrackDerefPTPae32Bit: i=%d pte=%RX32 hint=%RX32\n",
2889 i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, GCPhys));
2890 pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, GCPhys);
2891 }
2892}
2893#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
2894
2895
2896/**
2897 * Clear references to shadowed pages in a PAE page directory.
2898 *
2899 * @param pPool The pool.
2900 * @param pPage The page.
2901 * @param pShwPD The shadow page directory (mapping of the page).
2902 */
2903DECLINLINE(void) pgmPoolTrackDerefPDPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPAE pShwPD)
2904{
2905 for (unsigned i = 0; i < ELEMENTS(pShwPD->a); i++)
2906 {
2907 if (pShwPD->a[i].n.u1Present)
2908 {
2909 PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & X86_PDE_PAE_PG_MASK);
2910 if (pSubPage)
2911 pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
2912 else
2913 AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & X86_PDE_PAE_PG_MASK));
2914 /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
2915 }
2916 }
2917}
2918
2919
2920/**
2921 * Clear references to shadowed pages in a 64-bit page directory pointer table.
2922 *
2923 * @param pPool The pool.
2924 * @param pPage The page.
2925 * @param pShwPDPT The shadow page directory pointer table (mapping of the page).
2926 */
2927DECLINLINE(void) pgmPoolTrackDerefPDPT64Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
2928{
2929 for (unsigned i = 0; i < ELEMENTS(pShwPDPT->a); i++)
2930 {
2931 if (pShwPDPT->a[i].n.u1Present)
2932 {
2933 PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & X86_PDPE_PG_MASK);
2934 if (pSubPage)
2935 pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
2936 else
2937 AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & X86_PDPE_PG_MASK));
2938 /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
2939 }
2940 }
2941}
2942
2943
2944/**
2945 * Clears all references made by this page.
2946 *
2947 * This includes other shadow pages and GC physical addresses.
2948 *
2949 * @param pPool The pool.
2950 * @param pPage The page.
2951 */
2952static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2953{
2954 /*
2955 * Map the shadow page and take action according to the page kind.
2956 */
2957 void *pvShw = PGMPOOL_PAGE_2_PTR(pPool->CTXSUFF(pVM), pPage);
2958 switch (pPage->enmKind)
2959 {
2960#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2961 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2962 {
2963 STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
2964 void *pvGst;
2965 int rc = PGM_GCPHYS_2_PTR(pPool->CTXSUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
2966 pgmPoolTrackDerefPT32Bit32Bit(pPool, pPage, (PX86PT)pvShw, (PCX86PT)pvGst);
2967 STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
2968 break;
2969 }
2970
2971 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2972 {
2973 STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
2974 void *pvGst;
2975 int rc = PGM_GCPHYS_2_PTR_EX(pPool->CTXSUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
2976 pgmPoolTrackDerefPTPae32Bit(pPool, pPage, (PX86PTPAE)pvShw, (PCX86PT)pvGst);
2977 STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
2978 break;
2979 }
2980
2981 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2982 {
2983 STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
2984 void *pvGst;
2985 int rc = PGM_GCPHYS_2_PTR(pPool->CTXSUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
2986 pgmPoolTrackDerefPTPaePae(pPool, pPage, (PX86PTPAE)pvShw, (PCX86PTPAE)pvGst);
2987 STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
2988 break;
2989 }
2990
2991 case PGMPOOLKIND_32BIT_PT_FOR_PHYS: /* treat it like a 4 MB page */
2992 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2993 {
2994 STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
2995 pgmPoolTrackDerefPT32Bit4MB(pPool, pPage, (PX86PT)pvShw);
2996 STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
2997 break;
2998 }
2999
3000 case PGMPOOLKIND_PAE_PT_FOR_PHYS: /* treat it like a 4 MB page */
3001 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3002 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3003 {
3004 STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
3005 pgmPoolTrackDerefPTPaeBig(pPool, pPage, (PX86PTPAE)pvShw);
3006 STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
3007 break;
3008 }
3009
3010#else /* !PGMPOOL_WITH_GCPHYS_TRACKING */
3011 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3012 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3013 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3014 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3015 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3016 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3017 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3018 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3019 break;
3020#endif /* !PGMPOOL_WITH_GCPHYS_TRACKING */
3021
3022 case PGMPOOLKIND_PAE_PD_FOR_32BIT_PD:
3023 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3024 pgmPoolTrackDerefPDPae(pPool, pPage, (PX86PDPAE)pvShw);
3025 break;
3026
3027 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3028 pgmPoolTrackDerefPDPT64Bit(pPool, pPage, (PX86PDPT)pvShw);
3029 break;
3030
3031 default:
3032 AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
3033 }
3034
3035 /* paranoia, clear the shadow page. Remove this laser (i.e. let Alloc and ClearAll do it). */
3036 STAM_PROFILE_START(&pPool->StatZeroPage, z);
3037 ASMMemZeroPage(pvShw);
3038 STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
3039 pPage->fZeroed = true;
3040}
3041#endif /* PGMPOOL_WITH_USER_TRACKING */
3042
3043
3044/**
3045 * Flushes all the special root pages as part of a pgmPoolFlushAllInt operation.
3046 *
3047 * @param pPool The pool.
3048 */
3049static void pgmPoolFlushAllSpecialRoots(PPGMPOOL pPool)
3050{
3051 /*
3052 * These special pages are all mapped into the indexes 1..PGMPOOL_IDX_FIRST.
3053 */
3054 Assert(NIL_PGMPOOL_IDX == 0);
3055 for (unsigned i = 1; i < PGMPOOL_IDX_FIRST; i++)
3056 {
3057 /*
3058 * Get the page address.
3059 */
3060 PPGMPOOLPAGE pPage = &pPool->aPages[i];
3061 union
3062 {
3063 uint64_t *pau64;
3064 uint32_t *pau32;
3065 } u;
3066 u.pau64 = (uint64_t *)PGMPOOL_PAGE_2_PTR(pPool->CTXSUFF(pVM), pPage);
3067
3068 /*
3069 * Mark stuff not present.
3070 */
3071 switch (pPage->enmKind)
3072 {
3073 case PGMPOOLKIND_ROOT_32BIT_PD:
3074 for (unsigned iPage = 0; iPage < X86_PG_ENTRIES; iPage++)
3075 if ((u.pau32[iPage] & (PGM_PDFLAGS_MAPPING | X86_PDE_P)) == X86_PDE_P)
3076 u.pau32[iPage] = 0;
3077 break;
3078
3079 case PGMPOOLKIND_ROOT_PAE_PD:
3080 for (unsigned iPage = 0; iPage < X86_PG_PAE_ENTRIES * X86_PG_PAE_PDPE_ENTRIES; iPage++)
3081 if ((u.pau64[iPage] & (PGM_PDFLAGS_MAPPING | X86_PDE_P)) == X86_PDE_P)
3082 u.pau64[iPage] = 0;
3083 break;
3084
3085 case PGMPOOLKIND_ROOT_PML4:
3086 for (unsigned iPage = 0; iPage < X86_PG_PAE_ENTRIES; iPage++)
3087 if ((u.pau64[iPage] & (PGM_PLXFLAGS_PERMANENT | X86_PML4E_P)) == X86_PML4E_P)
3088 u.pau64[iPage] = 0;
3089 break;
3090
3091 case PGMPOOLKIND_ROOT_PDPT:
3092 /* Not root of shadowed pages currently, ignore it. */
3093 break;
3094 }
3095 }
3096
3097 /*
3098 * Paranoia (to be removed), flag a global CR3 sync.
3099 */
3100 VM_FF_SET(pPool->CTXSUFF(pVM), VM_FF_PGM_SYNC_CR3);
3101}
3102
3103
3104/**
3105 * Flushes the entire cache.
3106 *
3107 * It will assert a global CR3 flush (FF) and assumes the caller is aware of this
3108 * and execute this CR3 flush.
3109 *
3110 * @param pPool The pool.
3111 */
3112static void pgmPoolFlushAllInt(PPGMPOOL pPool)
3113{
3114 STAM_PROFILE_START(&pPool->StatFlushAllInt, a);
3115 LogFlow(("pgmPoolFlushAllInt:\n"));
3116
3117 /*
3118 * If there are no pages in the pool, there is nothing to do.
3119 */
3120 if (pPool->cCurPages <= PGMPOOL_IDX_FIRST)
3121 {
3122 STAM_PROFILE_STOP(&pPool->StatFlushAllInt, a);
3123 return;
3124 }
3125
3126 /*
3127 * Nuke the free list and reinsert all pages into it.
3128 */
3129 for (unsigned i = pPool->cCurPages - 1; i >= PGMPOOL_IDX_FIRST; i--)
3130 {
3131 PPGMPOOLPAGE pPage = &pPool->aPages[i];
3132
3133#ifdef IN_RING3
3134 Assert(pPage->Core.Key == MMPage2Phys(pPool->pVMHC, pPage->pvPageHC));
3135#endif
3136#ifdef PGMPOOL_WITH_MONITORING
3137 if (pPage->fMonitored)
3138 pgmPoolMonitorFlush(pPool, pPage);
3139 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
3140 pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
3141 pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
3142 pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
3143 pPage->cModifications = 0;
3144#endif
3145 pPage->GCPhys = NIL_RTGCPHYS;
3146 pPage->enmKind = PGMPOOLKIND_FREE;
3147 Assert(pPage->idx == i);
3148 pPage->iNext = i + 1;
3149 pPage->fZeroed = false; /* This could probably be optimized, but better safe than sorry. */
3150 pPage->fSeenNonGlobal = false;
3151 pPage->fMonitored= false;
3152 pPage->fCached = false;
3153 pPage->fReusedFlushPending = false;
3154 pPage->fCR3Mix = false;
3155#ifdef PGMPOOL_WITH_USER_TRACKING
3156 pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
3157#endif
3158#ifdef PGMPOOL_WITH_CACHE
3159 pPage->iAgeNext = NIL_PGMPOOL_IDX;
3160 pPage->iAgePrev = NIL_PGMPOOL_IDX;
3161#endif
3162 }
3163 pPool->aPages[pPool->cCurPages - 1].iNext = NIL_PGMPOOL_IDX;
3164 pPool->iFreeHead = PGMPOOL_IDX_FIRST;
3165 pPool->cUsedPages = 0;
3166
3167#ifdef PGMPOOL_WITH_USER_TRACKING
3168 /*
3169 * Zap and reinitialize the user records.
3170 */
3171 pPool->cPresent = 0;
3172 pPool->iUserFreeHead = 0;
3173 PPGMPOOLUSER paUsers = pPool->CTXSUFF(paUsers);
3174 const unsigned cMaxUsers = pPool->cMaxUsers;
3175 for (unsigned i = 0; i < cMaxUsers; i++)
3176 {
3177 paUsers[i].iNext = i + 1;
3178 paUsers[i].iUser = NIL_PGMPOOL_IDX;
3179 paUsers[i].iUserTable = 0xfffe;
3180 }
3181 paUsers[cMaxUsers - 1].iNext = NIL_PGMPOOL_USER_INDEX;
3182#endif
3183
3184#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
3185 /*
3186 * Clear all the GCPhys links and rebuild the phys ext free list.
3187 */
3188 for (PPGMRAMRANGE pRam = pPool->CTXSUFF(pVM)->pgm.s.CTXALLSUFF(pRamRanges);
3189 pRam;
3190 pRam = CTXALLSUFF(pRam->pNext))
3191 {
3192 unsigned iPage = pRam->cb >> PAGE_SHIFT;
3193 while (iPage-- > 0)
3194 pRam->aPages[iPage].HCPhys &= MM_RAM_FLAGS_NO_REFS_MASK; /** @todo PAGE FLAGS */
3195 }
3196
3197 pPool->iPhysExtFreeHead = 0;
3198 PPGMPOOLPHYSEXT paPhysExts = pPool->CTXSUFF(paPhysExts);
3199 const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
3200 for (unsigned i = 0; i < cMaxPhysExts; i++)
3201 {
3202 paPhysExts[i].iNext = i + 1;
3203 paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
3204 paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
3205 paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
3206 }
3207 paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
3208#endif
3209
3210#ifdef PGMPOOL_WITH_MONITORING
3211 /*
3212 * Just zap the modified list.
3213 */
3214 pPool->cModifiedPages = 0;
3215 pPool->iModifiedHead = NIL_PGMPOOL_IDX;
3216#endif
3217
3218#ifdef PGMPOOL_WITH_CACHE
3219 /*
3220 * Clear the GCPhys hash and the age list.
3221 */
3222 for (unsigned i = 0; i < ELEMENTS(pPool->aiHash); i++)
3223 pPool->aiHash[i] = NIL_PGMPOOL_IDX;
3224 pPool->iAgeHead = NIL_PGMPOOL_IDX;
3225 pPool->iAgeTail = NIL_PGMPOOL_IDX;
3226#endif
3227
3228 /*
3229 * Flush all the special root pages.
3230 * Reinsert active pages into the hash and ensure monitoring chains are correct.
3231 */
3232 pgmPoolFlushAllSpecialRoots(pPool);
3233 for (unsigned i = PGMPOOL_IDX_FIRST_SPECIAL; i < PGMPOOL_IDX_FIRST; i++)
3234 {
3235 PPGMPOOLPAGE pPage = &pPool->aPages[i];
3236 pPage->iNext = NIL_PGMPOOL_IDX;
3237#ifdef PGMPOOL_WITH_MONITORING
3238 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
3239 pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
3240 pPage->cModifications = 0;
3241 /* ASSUMES that we're not sharing with any of the other special pages (safe for now). */
3242 pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
3243 pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
3244 if (pPage->fMonitored)
3245 {
3246 PVM pVM = pPool->CTXSUFF(pVM);
3247 int rc = PGMHandlerPhysicalChangeCallbacks(pVM, pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1),
3248 pPool->pfnAccessHandlerR3, MMHyperCCToR3(pVM, pPage),
3249 pPool->pfnAccessHandlerR0, MMHyperCCToR0(pVM, pPage),
3250 pPool->pfnAccessHandlerGC, MMHyperCCToGC(pVM, pPage),
3251 pPool->pszAccessHandler);
3252 AssertFatalRCSuccess(rc);
3253# ifdef PGMPOOL_WITH_CACHE
3254 pgmPoolHashInsert(pPool, pPage);
3255# endif
3256 }
3257#endif
3258#ifdef PGMPOOL_WITH_USER_TRACKING
3259 Assert(pPage->iUserHead == NIL_PGMPOOL_USER_INDEX); /* for now */
3260#endif
3261#ifdef PGMPOOL_WITH_CACHE
3262 Assert(pPage->iAgeNext == NIL_PGMPOOL_IDX);
3263 Assert(pPage->iAgePrev == NIL_PGMPOOL_IDX);
3264#endif
3265 }
3266
3267 STAM_PROFILE_STOP(&pPool->StatFlushAllInt, a);
3268}
3269
3270
3271/**
3272 * Flushes a pool page.
3273 *
3274 * This moves the page to the free list after removing all user references to it.
3275 * In GC this will cause a CR3 reload if the page is traced back to an active root page.
3276 *
3277 * @returns VBox status code.
3278 * @retval VINF_SUCCESS on success.
3279 * @retval VERR_PGM_POOL_CLEARED if the deregistration of the physical handler will cause a light weight pool flush.
3280 * @param pPool The pool.
3281 * @param HCPhys The HC physical address of the shadow page.
3282 */
3283int pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
3284{
3285 int rc = VINF_SUCCESS;
3286 STAM_PROFILE_START(&pPool->StatFlushPage, f);
3287 LogFlow(("pgmPoolFlushPage: pPage=%p:{.Key=%VHp, .idx=%d, .enmKind=%d, .GCPhys=%VGp}\n",
3288 pPage, pPage->Core.Key, pPage->idx, pPage->enmKind, pPage->GCPhys));
3289
3290 /*
3291 * Quietly reject any attempts at flushing any of the special root pages.
3292 */
3293 if (pPage->idx < PGMPOOL_IDX_FIRST)
3294 {
3295 Log(("pgmPoolFlushPage: specaial root page, rejected. enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
3296 return VINF_SUCCESS;
3297 }
3298
3299 /*
3300 * Mark the page as being in need of a ASMMemZeroPage().
3301 */
3302 pPage->fZeroed = false;
3303
3304#ifdef PGMPOOL_WITH_USER_TRACKING
3305 /*
3306 * Clear the page.
3307 */
3308 pgmPoolTrackClearPageUsers(pPool, pPage);
3309 STAM_PROFILE_START(&pPool->StatTrackDeref,a);
3310 pgmPoolTrackDeref(pPool, pPage);
3311 STAM_PROFILE_STOP(&pPool->StatTrackDeref,a);
3312#endif
3313
3314#ifdef PGMPOOL_WITH_CACHE
3315 /*
3316 * Flush it from the cache.
3317 */
3318 pgmPoolCacheFlushPage(pPool, pPage);
3319#endif /* PGMPOOL_WITH_CACHE */
3320
3321#ifdef PGMPOOL_WITH_MONITORING
3322 /*
3323 * Deregistering the monitoring.
3324 */
3325 if (pPage->fMonitored)
3326 rc = pgmPoolMonitorFlush(pPool, pPage);
3327#endif
3328
3329 /*
3330 * Free the page.
3331 */
3332 Assert(pPage->iNext == NIL_PGMPOOL_IDX);
3333 pPage->iNext = pPool->iFreeHead;
3334 pPool->iFreeHead = pPage->idx;
3335 pPage->enmKind = PGMPOOLKIND_FREE;
3336 pPage->GCPhys = NIL_RTGCPHYS;
3337 pPage->fReusedFlushPending = false;
3338
3339 pPool->cUsedPages--;
3340 STAM_PROFILE_STOP(&pPool->StatFlushPage, f);
3341 return rc;
3342}
3343
3344
3345/**
3346 * Frees a usage of a pool page.
3347 *
3348 * The caller is responsible to updating the user table so that it no longer
3349 * references the shadow page.
3350 *
3351 * @param pPool The pool.
3352 * @param HCPhys The HC physical address of the shadow page.
3353 * @param iUser The shadow page pool index of the user table.
3354 * @param iUserTable The index into the user table (shadowed).
3355 */
3356void pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint16_t iUserTable)
3357{
3358 STAM_PROFILE_START(&pPool->StatFree, a);
3359 LogFlow(("pgmPoolFreeByPage: pPage=%p:{.Key=%VHp, .idx=%d, enmKind=%d} iUser=%#x iUserTable=%#x\n",
3360 pPage, pPage->Core.Key, pPage->idx, pPage->enmKind, iUser, iUserTable));
3361 Assert(pPage->idx >= PGMPOOL_IDX_FIRST);
3362#ifdef PGMPOOL_WITH_USER_TRACKING
3363 pgmPoolTrackFreeUser(pPool, pPage, iUser, iUserTable);
3364#endif
3365#ifdef PGMPOOL_WITH_CACHE
3366 if (!pPage->fCached)
3367#endif
3368 pgmPoolFlushPage(pPool, pPage); /* ASSUMES that VERR_PGM_POOL_CLEARED can be ignored here. */
3369 STAM_PROFILE_STOP(&pPool->StatFree, a);
3370}
3371
3372
3373/**
3374 * Makes one or more free page free.
3375 *
3376 * @returns VBox status code.
3377 * @retval VINF_SUCCESS on success.
3378 * @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
3379 *
3380 * @param pPool The pool.
3381 * @param iUser The user of the page.
3382 */
3383static int pgmPoolMakeMoreFreePages(PPGMPOOL pPool, uint16_t iUser)
3384{
3385 LogFlow(("pgmPoolMakeMoreFreePages: iUser=%#x\n", iUser));
3386
3387 /*
3388 * If the pool isn't full grown yet, expand it.
3389 */
3390 if (pPool->cCurPages < pPool->cMaxPages)
3391 {
3392 STAM_PROFILE_ADV_SUSPEND(&pPool->StatAlloc, a);
3393#ifdef IN_RING3
3394 int rc = PGMR3PoolGrow(pPool->pVMHC);
3395#else
3396 int rc = CTXALLMID(VMM, CallHost)(pPool->CTXSUFF(pVM), VMMCALLHOST_PGM_POOL_GROW, 0);
3397#endif
3398 if (VBOX_FAILURE(rc))
3399 return rc;
3400 STAM_PROFILE_ADV_RESUME(&pPool->StatAlloc, a);
3401 if (pPool->iFreeHead != NIL_PGMPOOL_IDX)
3402 return VINF_SUCCESS;
3403 }
3404
3405#ifdef PGMPOOL_WITH_CACHE
3406 /*
3407 * Free one cached page.
3408 */
3409 return pgmPoolCacheFreeOne(pPool, iUser);
3410#else
3411 /*
3412 * Flush the pool.
3413 * If we have tracking enabled, it should be possible to come up with
3414 * a cheap replacement strategy...
3415 */
3416 pgmPoolFlushAllInt(pPool);
3417 return VERR_PGM_POOL_FLUSHED;
3418#endif
3419}
3420
3421
3422/**
3423 * Allocates a page from the pool.
3424 *
3425 * This page may actually be a cached page and not in need of any processing
3426 * on the callers part.
3427 *
3428 * @returns VBox status code.
3429 * @retval VINF_SUCCESS if a NEW page was allocated.
3430 * @retval VINF_PGM_CACHED_PAGE if a CACHED page was returned.
3431 * @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
3432 * @param pVM The VM handle.
3433 * @param GCPhys The GC physical address of the page we're gonna shadow.
3434 * For 4MB and 2MB PD entries, it's the first address the
3435 * shadow PT is covering.
3436 * @param enmKind The kind of mapping.
3437 * @param iUser The shadow page pool index of the user table.
3438 * @param iUserTable The index into the user table (shadowed).
3439 * @param ppPage Where to store the pointer to the page. NULL is stored here on failure.
3440 */
3441int pgmPoolAlloc(PVM pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, uint16_t iUser, uint16_t iUserTable, PPPGMPOOLPAGE ppPage)
3442{
3443 PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
3444 STAM_PROFILE_ADV_START(&pPool->StatAlloc, a);
3445 LogFlow(("pgmPoolAlloc: GCPhys=%VGp enmKind=%d iUser=%#x iUserTable=%#x\n", GCPhys, enmKind, iUser, iUserTable));
3446 *ppPage = NULL;
3447
3448#ifdef PGMPOOL_WITH_CACHE
3449 if (pPool->fCacheEnabled)
3450 {
3451 int rc2 = pgmPoolCacheAlloc(pPool, GCPhys, enmKind, iUser, iUserTable, ppPage);
3452 if (VBOX_SUCCESS(rc2))
3453 {
3454 STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
3455 LogFlow(("pgmPoolAlloc: returns %Vrc *ppPage=%p:{.Key=%VHp, .idx=%d}\n", rc2, *ppPage, (*ppPage)->Core.Key, (*ppPage)->idx));
3456 return rc2;
3457 }
3458 }
3459#endif
3460
3461 /*
3462 * Allocate a new one.
3463 */
3464 int rc = VINF_SUCCESS;
3465 uint16_t iNew = pPool->iFreeHead;
3466 if (iNew == NIL_PGMPOOL_IDX)
3467 {
3468 rc = pgmPoolMakeMoreFreePages(pPool, iUser);
3469 if (VBOX_FAILURE(rc))
3470 {
3471 if (rc != VERR_PGM_POOL_CLEARED)
3472 {
3473 Log(("pgmPoolAlloc: returns %Vrc (Free)\n", rc));
3474 STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
3475 return rc;
3476 }
3477 rc = VERR_PGM_POOL_FLUSHED;
3478 }
3479 iNew = pPool->iFreeHead;
3480 AssertReleaseReturn(iNew != NIL_PGMPOOL_IDX, VERR_INTERNAL_ERROR);
3481 }
3482
3483 /* unlink the free head */
3484 PPGMPOOLPAGE pPage = &pPool->aPages[iNew];
3485 pPool->iFreeHead = pPage->iNext;
3486 pPage->iNext = NIL_PGMPOOL_IDX;
3487
3488 /*
3489 * Initialize it.
3490 */
3491 pPool->cUsedPages++; /* physical handler registration / pgmPoolTrackFlushGCPhysPTsSlow requirement. */
3492 pPage->enmKind = enmKind;
3493 pPage->GCPhys = GCPhys;
3494 pPage->fSeenNonGlobal = false; /* Set this to 'true' to disable this feature. */
3495 pPage->fMonitored = false;
3496 pPage->fCached = false;
3497 pPage->fReusedFlushPending = false;
3498 pPage->fCR3Mix = false;
3499#ifdef PGMPOOL_WITH_MONITORING
3500 pPage->cModifications = 0;
3501 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
3502 pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
3503#endif
3504#ifdef PGMPOOL_WITH_USER_TRACKING
3505 pPage->cPresent = 0;
3506 pPage->iFirstPresent = ~0;
3507
3508 /*
3509 * Insert into the tracking and cache. If this fails, free the page.
3510 */
3511 int rc3 = pgmPoolTrackInsert(pPool, pPage, GCPhys, iUser, iUserTable);
3512 if (VBOX_FAILURE(rc3))
3513 {
3514 if (rc3 != VERR_PGM_POOL_CLEARED)
3515 {
3516 pPool->cUsedPages--;
3517 pPage->enmKind = PGMPOOLKIND_FREE;
3518 pPage->GCPhys = NIL_RTGCPHYS;
3519 pPage->iNext = pPool->iFreeHead;
3520 pPool->iFreeHead = pPage->idx;
3521 STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
3522 Log(("pgmPoolAlloc: returns %Vrc (Insert)\n", rc3));
3523 return rc3;
3524 }
3525 rc = VERR_PGM_POOL_FLUSHED;
3526 }
3527#endif /* PGMPOOL_WITH_USER_TRACKING */
3528
3529 /*
3530 * Commit the allocation, clear the page and return.
3531 */
3532#ifdef VBOX_WITH_STATISTICS
3533 if (pPool->cUsedPages > pPool->cUsedPagesHigh)
3534 pPool->cUsedPagesHigh = pPool->cUsedPages;
3535#endif
3536
3537 if (!pPage->fZeroed)
3538 {
3539 STAM_PROFILE_START(&pPool->StatZeroPage, z);
3540 void *pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
3541 ASMMemZeroPage(pv);
3542 STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
3543 }
3544
3545 *ppPage = pPage;
3546 LogFlow(("pgmPoolAlloc: returns %Vrc *ppPage=%p:{.Key=%VHp, .idx=%d, .fCached=%RTbool, .fMonitored=%RTbool}\n",
3547 rc, pPage, pPage->Core.Key, pPage->idx, pPage->fCached, pPage->fMonitored));
3548 STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
3549 return rc;
3550}
3551
3552
3553/**
3554 * Frees a usage of a pool page.
3555 *
3556 * @param pVM The VM handle.
3557 * @param HCPhys The HC physical address of the shadow page.
3558 * @param iUser The shadow page pool index of the user table.
3559 * @param iUserTable The index into the user table (shadowed).
3560 */
3561void pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint16_t iUserTable)
3562{
3563 LogFlow(("pgmPoolFree: HCPhys=%VHp iUser=%#x iUserTable=%#x\n", HCPhys, iUser, iUserTable));
3564 PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
3565 pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, HCPhys), iUser, iUserTable);
3566}
3567
3568
3569/**
3570 * Gets a in-use page in the pool by it's physical address.
3571 *
3572 * @returns Pointer to the page.
3573 * @param pVM The VM handle.
3574 * @param HCPhys The HC physical address of the shadow page.
3575 * @remark This function will NEVER return NULL. It will assert if HCPhys is invalid.
3576 */
3577PPGMPOOLPAGE pgmPoolGetPageByHCPhys(PVM pVM, RTHCPHYS HCPhys)
3578{
3579 /** @todo profile this! */
3580 PPGMPOOL pPool = pVM->pgm.s.CTXSUFF(pPool);
3581 PPGMPOOLPAGE pPage = pgmPoolGetPage(pPool, HCPhys);
3582 Log3(("pgmPoolGetPageByHCPhys: HCPhys=%VHp -> %p:{.idx=%d .GCPhys=%VGp .enmKind=%d}\n",
3583 HCPhys, pPage, pPage->idx, pPage->GCPhys, pPage->enmKind));
3584 return pPage;
3585}
3586
3587
3588/**
3589 * Flushes the entire cache.
3590 *
3591 * It will assert a global CR3 flush (FF) and assumes the caller is aware of this
3592 * and execute this CR3 flush.
3593 *
3594 * @param pPool The pool.
3595 */
3596void pgmPoolFlushAll(PVM pVM)
3597{
3598 LogFlow(("pgmPoolFlushAll:\n"));
3599 pgmPoolFlushAllInt(pVM->pgm.s.CTXSUFF(pPool));
3600}
3601
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette