VirtualBox

source: vbox/trunk/src/VBox/VMM/PGMPhys.cpp@ 32131

Last change on this file since 32131 was 32121, checked in by vboxsync, 14 years ago

Wrong api

  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 150.2 KB
Line 
1/* $Id: PGMPhys.cpp 32121 2010-08-31 10:00:09Z vboxsync $ */
2/** @file
3 * PGM - Page Manager and Monitor, Physical Memory Addressing.
4 */
5
6/*
7 * Copyright (C) 2006-2007 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*******************************************************************************
20* Header Files *
21*******************************************************************************/
22#define LOG_GROUP LOG_GROUP_PGM_PHYS
23#include <VBox/pgm.h>
24#include <VBox/iom.h>
25#include <VBox/mm.h>
26#include <VBox/stam.h>
27#include <VBox/rem.h>
28#include <VBox/pdmdev.h>
29#include "PGMInternal.h"
30#include <VBox/vm.h>
31#include "PGMInline.h"
32#include <VBox/sup.h>
33#include <VBox/param.h>
34#include <VBox/err.h>
35#include <VBox/log.h>
36#include <iprt/assert.h>
37#include <iprt/alloc.h>
38#include <iprt/asm.h>
39#include <iprt/thread.h>
40#include <iprt/string.h>
41
42
43/*******************************************************************************
44* Defined Constants And Macros *
45*******************************************************************************/
46/** The number of pages to free in one batch. */
47#define PGMPHYS_FREE_PAGE_BATCH_SIZE 128
48
49
50/*******************************************************************************
51* Internal Functions *
52*******************************************************************************/
53static DECLCALLBACK(int) pgmR3PhysRomWriteHandler(PVM pVM, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser);
54
55
56/*
57 * PGMR3PhysReadU8-64
58 * PGMR3PhysWriteU8-64
59 */
60#define PGMPHYSFN_READNAME PGMR3PhysReadU8
61#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU8
62#define PGMPHYS_DATASIZE 1
63#define PGMPHYS_DATATYPE uint8_t
64#include "PGMPhysRWTmpl.h"
65
66#define PGMPHYSFN_READNAME PGMR3PhysReadU16
67#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU16
68#define PGMPHYS_DATASIZE 2
69#define PGMPHYS_DATATYPE uint16_t
70#include "PGMPhysRWTmpl.h"
71
72#define PGMPHYSFN_READNAME PGMR3PhysReadU32
73#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU32
74#define PGMPHYS_DATASIZE 4
75#define PGMPHYS_DATATYPE uint32_t
76#include "PGMPhysRWTmpl.h"
77
78#define PGMPHYSFN_READNAME PGMR3PhysReadU64
79#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU64
80#define PGMPHYS_DATASIZE 8
81#define PGMPHYS_DATATYPE uint64_t
82#include "PGMPhysRWTmpl.h"
83
84
85/**
86 * EMT worker for PGMR3PhysReadExternal.
87 */
88static DECLCALLBACK(int) pgmR3PhysReadExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, void *pvBuf, size_t cbRead)
89{
90 PGMPhysRead(pVM, *pGCPhys, pvBuf, cbRead);
91 return VINF_SUCCESS;
92}
93
94
95/**
96 * Write to physical memory, external users.
97 *
98 * @returns VBox status code.
99 * @retval VINF_SUCCESS.
100 *
101 * @param pVM VM Handle.
102 * @param GCPhys Physical address to write to.
103 * @param pvBuf What to write.
104 * @param cbWrite How many bytes to write.
105 *
106 * @thread Any but EMTs.
107 */
108VMMR3DECL(int) PGMR3PhysReadExternal(PVM pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead)
109{
110 VM_ASSERT_OTHER_THREAD(pVM);
111
112 AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
113 LogFlow(("PGMR3PhysReadExternal: %RGp %d\n", GCPhys, cbRead));
114
115 pgmLock(pVM);
116
117 /*
118 * Copy loop on ram ranges.
119 */
120 PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
121 for (;;)
122 {
123 /* Find range. */
124 while (pRam && GCPhys > pRam->GCPhysLast)
125 pRam = pRam->CTX_SUFF(pNext);
126 /* Inside range or not? */
127 if (pRam && GCPhys >= pRam->GCPhys)
128 {
129 /*
130 * Must work our way thru this page by page.
131 */
132 RTGCPHYS off = GCPhys - pRam->GCPhys;
133 while (off < pRam->cb)
134 {
135 unsigned iPage = off >> PAGE_SHIFT;
136 PPGMPAGE pPage = &pRam->aPages[iPage];
137
138 /*
139 * If the page has an ALL access handler, we'll have to
140 * delegate the job to EMT.
141 */
142 if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
143 {
144 pgmUnlock(pVM);
145
146 return VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysReadExternalEMT, 4,
147 pVM, &GCPhys, pvBuf, cbRead);
148 }
149 Assert(!PGM_PAGE_IS_MMIO(pPage));
150
151 /*
152 * Simple stuff, go ahead.
153 */
154 size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
155 if (cb > cbRead)
156 cb = cbRead;
157 const void *pvSrc;
158 int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc);
159 if (RT_SUCCESS(rc))
160 memcpy(pvBuf, pvSrc, cb);
161 else
162 {
163 AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
164 pRam->GCPhys + off, pPage, rc));
165 memset(pvBuf, 0xff, cb);
166 }
167
168 /* next page */
169 if (cb >= cbRead)
170 {
171 pgmUnlock(pVM);
172 return VINF_SUCCESS;
173 }
174 cbRead -= cb;
175 off += cb;
176 GCPhys += cb;
177 pvBuf = (char *)pvBuf + cb;
178 } /* walk pages in ram range. */
179 }
180 else
181 {
182 LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
183
184 /*
185 * Unassigned address space.
186 */
187 if (!pRam)
188 break;
189 size_t cb = pRam->GCPhys - GCPhys;
190 if (cb >= cbRead)
191 {
192 memset(pvBuf, 0xff, cbRead);
193 break;
194 }
195 memset(pvBuf, 0xff, cb);
196
197 cbRead -= cb;
198 pvBuf = (char *)pvBuf + cb;
199 GCPhys += cb;
200 }
201 } /* Ram range walk */
202
203 pgmUnlock(pVM);
204
205 return VINF_SUCCESS;
206}
207
208
209/**
210 * EMT worker for PGMR3PhysWriteExternal.
211 */
212static DECLCALLBACK(int) pgmR3PhysWriteExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, const void *pvBuf, size_t cbWrite)
213{
214 /** @todo VERR_EM_NO_MEMORY */
215 PGMPhysWrite(pVM, *pGCPhys, pvBuf, cbWrite);
216 return VINF_SUCCESS;
217}
218
219
220/**
221 * Write to physical memory, external users.
222 *
223 * @returns VBox status code.
224 * @retval VINF_SUCCESS.
225 * @retval VERR_EM_NO_MEMORY.
226 *
227 * @param pVM VM Handle.
228 * @param GCPhys Physical address to write to.
229 * @param pvBuf What to write.
230 * @param cbWrite How many bytes to write.
231 * @param pszWho Who is writing. For tracking down who is writing
232 * after we've saved the state.
233 *
234 * @thread Any but EMTs.
235 */
236VMMDECL(int) PGMR3PhysWriteExternal(PVM pVM, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite, const char *pszWho)
237{
238 VM_ASSERT_OTHER_THREAD(pVM);
239
240 AssertMsg(!pVM->pgm.s.fNoMorePhysWrites,
241 ("Calling PGMR3PhysWriteExternal after pgmR3Save()! GCPhys=%RGp cbWrite=%#x pszWho=%s\n",
242 GCPhys, cbWrite, pszWho));
243 AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
244 LogFlow(("PGMR3PhysWriteExternal: %RGp %d\n", GCPhys, cbWrite));
245
246 pgmLock(pVM);
247
248 /*
249 * Copy loop on ram ranges, stop when we hit something difficult.
250 */
251 PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
252 for (;;)
253 {
254 /* Find range. */
255 while (pRam && GCPhys > pRam->GCPhysLast)
256 pRam = pRam->CTX_SUFF(pNext);
257 /* Inside range or not? */
258 if (pRam && GCPhys >= pRam->GCPhys)
259 {
260 /*
261 * Must work our way thru this page by page.
262 */
263 RTGCPTR off = GCPhys - pRam->GCPhys;
264 while (off < pRam->cb)
265 {
266 RTGCPTR iPage = off >> PAGE_SHIFT;
267 PPGMPAGE pPage = &pRam->aPages[iPage];
268
269 /*
270 * Is the page problematic, we have to do the work on the EMT.
271 *
272 * Allocating writable pages and access handlers are
273 * problematic, write monitored pages are simple and can be
274 * dealt with here.
275 */
276 if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
277 || PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
278 {
279 if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
280 && !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
281 pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
282 else
283 {
284 pgmUnlock(pVM);
285
286 return VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysWriteExternalEMT, 4,
287 pVM, &GCPhys, pvBuf, cbWrite);
288 }
289 }
290 Assert(!PGM_PAGE_IS_MMIO(pPage));
291
292 /*
293 * Simple stuff, go ahead.
294 */
295 size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
296 if (cb > cbWrite)
297 cb = cbWrite;
298 void *pvDst;
299 int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst);
300 if (RT_SUCCESS(rc))
301 memcpy(pvDst, pvBuf, cb);
302 else
303 AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
304 pRam->GCPhys + off, pPage, rc));
305
306 /* next page */
307 if (cb >= cbWrite)
308 {
309 pgmUnlock(pVM);
310 return VINF_SUCCESS;
311 }
312
313 cbWrite -= cb;
314 off += cb;
315 GCPhys += cb;
316 pvBuf = (const char *)pvBuf + cb;
317 } /* walk pages in ram range */
318 }
319 else
320 {
321 /*
322 * Unassigned address space, skip it.
323 */
324 if (!pRam)
325 break;
326 size_t cb = pRam->GCPhys - GCPhys;
327 if (cb >= cbWrite)
328 break;
329 cbWrite -= cb;
330 pvBuf = (const char *)pvBuf + cb;
331 GCPhys += cb;
332 }
333 } /* Ram range walk */
334
335 pgmUnlock(pVM);
336 return VINF_SUCCESS;
337}
338
339
340/**
341 * VMR3ReqCall worker for PGMR3PhysGCPhys2CCPtrExternal to make pages writable.
342 *
343 * @returns see PGMR3PhysGCPhys2CCPtrExternal
344 * @param pVM The VM handle.
345 * @param pGCPhys Pointer to the guest physical address.
346 * @param ppv Where to store the mapping address.
347 * @param pLock Where to store the lock.
348 */
349static DECLCALLBACK(int) pgmR3PhysGCPhys2CCPtrDelegated(PVM pVM, PRTGCPHYS pGCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
350{
351 /*
352 * Just hand it to PGMPhysGCPhys2CCPtr and check that it's not a page with
353 * an access handler after it succeeds.
354 */
355 int rc = pgmLock(pVM);
356 AssertRCReturn(rc, rc);
357
358 rc = PGMPhysGCPhys2CCPtr(pVM, *pGCPhys, ppv, pLock);
359 if (RT_SUCCESS(rc))
360 {
361 PPGMPAGEMAPTLBE pTlbe;
362 int rc2 = pgmPhysPageQueryTlbe(&pVM->pgm.s, *pGCPhys, &pTlbe);
363 AssertFatalRC(rc2);
364 PPGMPAGE pPage = pTlbe->pPage;
365 if (PGM_PAGE_IS_MMIO(pPage))
366 {
367 PGMPhysReleasePageMappingLock(pVM, pLock);
368 rc = VERR_PGM_PHYS_PAGE_RESERVED;
369 }
370 else if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
371#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
372 || pgmPoolIsDirtyPage(pVM, *pGCPhys)
373#endif
374 )
375 {
376 /* We *must* flush any corresponding pgm pool page here, otherwise we'll
377 * not be informed about writes and keep bogus gst->shw mappings around.
378 */
379 pgmPoolFlushPageByGCPhys(pVM, *pGCPhys);
380 Assert(!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage));
381 /** @todo r=bird: return VERR_PGM_PHYS_PAGE_RESERVED here if it still has
382 * active handlers, see the PGMR3PhysGCPhys2CCPtrExternal docs. */
383 }
384 }
385
386 pgmUnlock(pVM);
387 return rc;
388}
389
390
391/**
392 * Requests the mapping of a guest page into ring-3, external threads.
393 *
394 * When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
395 * release it.
396 *
397 * This API will assume your intention is to write to the page, and will
398 * therefore replace shared and zero pages. If you do not intend to modify the
399 * page, use the PGMR3PhysGCPhys2CCPtrReadOnlyExternal() API.
400 *
401 * @returns VBox status code.
402 * @retval VINF_SUCCESS on success.
403 * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
404 * backing or if the page has any active access handlers. The caller
405 * must fall back on using PGMR3PhysWriteExternal.
406 * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
407 *
408 * @param pVM The VM handle.
409 * @param GCPhys The guest physical address of the page that should be mapped.
410 * @param ppv Where to store the address corresponding to GCPhys.
411 * @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
412 *
413 * @remark Avoid calling this API from within critical sections (other than the
414 * PGM one) because of the deadlock risk when we have to delegating the
415 * task to an EMT.
416 * @thread Any.
417 */
418VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrExternal(PVM pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
419{
420 AssertPtr(ppv);
421 AssertPtr(pLock);
422
423 Assert(VM_IS_EMT(pVM) || !PGMIsLockOwner(pVM));
424
425 int rc = pgmLock(pVM);
426 AssertRCReturn(rc, rc);
427
428 /*
429 * Query the Physical TLB entry for the page (may fail).
430 */
431 PPGMPAGEMAPTLBE pTlbe;
432 rc = pgmPhysPageQueryTlbe(&pVM->pgm.s, GCPhys, &pTlbe);
433 if (RT_SUCCESS(rc))
434 {
435 PPGMPAGE pPage = pTlbe->pPage;
436 if (PGM_PAGE_IS_MMIO(pPage))
437 rc = VERR_PGM_PHYS_PAGE_RESERVED;
438 else
439 {
440 /*
441 * If the page is shared, the zero page, or being write monitored
442 * it must be converted to an page that's writable if possible.
443 * We can only deal with write monitored pages here, the rest have
444 * to be on an EMT.
445 */
446 if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
447 || PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
448#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
449 || pgmPoolIsDirtyPage(pVM, GCPhys)
450#endif
451 )
452 {
453 if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
454 && !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
455#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
456 && !pgmPoolIsDirtyPage(pVM, GCPhys)
457#endif
458 )
459 pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
460 else
461 {
462 pgmUnlock(pVM);
463
464 return VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysGCPhys2CCPtrDelegated, 4,
465 pVM, &GCPhys, ppv, pLock);
466 }
467 }
468
469 /*
470 * Now, just perform the locking and calculate the return address.
471 */
472 PPGMPAGEMAP pMap = pTlbe->pMap;
473 if (pMap)
474 pMap->cRefs++;
475
476 unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
477 if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
478 {
479 if (cLocks == 0)
480 pVM->pgm.s.cWriteLockedPages++;
481 PGM_PAGE_INC_WRITE_LOCKS(pPage);
482 }
483 else if (cLocks != PGM_PAGE_GET_WRITE_LOCKS(pPage))
484 {
485 PGM_PAGE_INC_WRITE_LOCKS(pPage);
486 AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent write locked state!\n", GCPhys, pPage));
487 if (pMap)
488 pMap->cRefs++; /* Extra ref to prevent it from going away. */
489 }
490
491 *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
492 pLock->uPageAndType = (uintptr_t)pPage | PGMPAGEMAPLOCK_TYPE_WRITE;
493 pLock->pvMap = pMap;
494 }
495 }
496
497 pgmUnlock(pVM);
498 return rc;
499}
500
501
502/**
503 * Requests the mapping of a guest page into ring-3, external threads.
504 *
505 * When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
506 * release it.
507 *
508 * @returns VBox status code.
509 * @retval VINF_SUCCESS on success.
510 * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
511 * backing or if the page as an active ALL access handler. The caller
512 * must fall back on using PGMPhysRead.
513 * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
514 *
515 * @param pVM The VM handle.
516 * @param GCPhys The guest physical address of the page that should be mapped.
517 * @param ppv Where to store the address corresponding to GCPhys.
518 * @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
519 *
520 * @remark Avoid calling this API from within critical sections (other than
521 * the PGM one) because of the deadlock risk.
522 * @thread Any.
523 */
524VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrReadOnlyExternal(PVM pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
525{
526 int rc = pgmLock(pVM);
527 AssertRCReturn(rc, rc);
528
529 /*
530 * Query the Physical TLB entry for the page (may fail).
531 */
532 PPGMPAGEMAPTLBE pTlbe;
533 rc = pgmPhysPageQueryTlbe(&pVM->pgm.s, GCPhys, &pTlbe);
534 if (RT_SUCCESS(rc))
535 {
536 PPGMPAGE pPage = pTlbe->pPage;
537#if 1
538 /* MMIO pages doesn't have any readable backing. */
539 if (PGM_PAGE_IS_MMIO(pPage))
540 rc = VERR_PGM_PHYS_PAGE_RESERVED;
541#else
542 if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
543 rc = VERR_PGM_PHYS_PAGE_RESERVED;
544#endif
545 else
546 {
547 /*
548 * Now, just perform the locking and calculate the return address.
549 */
550 PPGMPAGEMAP pMap = pTlbe->pMap;
551 if (pMap)
552 pMap->cRefs++;
553
554 unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
555 if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
556 {
557 if (cLocks == 0)
558 pVM->pgm.s.cReadLockedPages++;
559 PGM_PAGE_INC_READ_LOCKS(pPage);
560 }
561 else if (cLocks != PGM_PAGE_GET_READ_LOCKS(pPage))
562 {
563 PGM_PAGE_INC_READ_LOCKS(pPage);
564 AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent readonly locked state!\n", GCPhys, pPage));
565 if (pMap)
566 pMap->cRefs++; /* Extra ref to prevent it from going away. */
567 }
568
569 *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
570 pLock->uPageAndType = (uintptr_t)pPage | PGMPAGEMAPLOCK_TYPE_READ;
571 pLock->pvMap = pMap;
572 }
573 }
574
575 pgmUnlock(pVM);
576 return rc;
577}
578
579
580/**
581 * Relinks the RAM ranges using the pSelfRC and pSelfR0 pointers.
582 *
583 * Called when anything was relocated.
584 *
585 * @param pVM Pointer to the shared VM structure.
586 */
587void pgmR3PhysRelinkRamRanges(PVM pVM)
588{
589 PPGMRAMRANGE pCur;
590
591#ifdef VBOX_STRICT
592 for (pCur = pVM->pgm.s.pRamRangesR3; pCur; pCur = pCur->pNextR3)
593 {
594 Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pCur->pSelfR0 == MMHyperCCToR0(pVM, pCur));
595 Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pCur->pSelfRC == MMHyperCCToRC(pVM, pCur));
596 Assert((pCur->GCPhys & PAGE_OFFSET_MASK) == 0);
597 Assert((pCur->GCPhysLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
598 Assert((pCur->cb & PAGE_OFFSET_MASK) == 0);
599 Assert(pCur->cb == pCur->GCPhysLast - pCur->GCPhys + 1);
600 for (PPGMRAMRANGE pCur2 = pVM->pgm.s.pRamRangesR3; pCur2; pCur2 = pCur2->pNextR3)
601 Assert( pCur2 == pCur
602 || strcmp(pCur2->pszDesc, pCur->pszDesc)); /** @todo fix MMIO ranges!! */
603 }
604#endif
605
606 pCur = pVM->pgm.s.pRamRangesR3;
607 if (pCur)
608 {
609 pVM->pgm.s.pRamRangesR0 = pCur->pSelfR0;
610 pVM->pgm.s.pRamRangesRC = pCur->pSelfRC;
611
612 for (; pCur->pNextR3; pCur = pCur->pNextR3)
613 {
614 pCur->pNextR0 = pCur->pNextR3->pSelfR0;
615 pCur->pNextRC = pCur->pNextR3->pSelfRC;
616 }
617
618 Assert(pCur->pNextR0 == NIL_RTR0PTR);
619 Assert(pCur->pNextRC == NIL_RTRCPTR);
620 }
621 else
622 {
623 Assert(pVM->pgm.s.pRamRangesR0 == NIL_RTR0PTR);
624 Assert(pVM->pgm.s.pRamRangesRC == NIL_RTRCPTR);
625 }
626 ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
627}
628
629
630/**
631 * Links a new RAM range into the list.
632 *
633 * @param pVM Pointer to the shared VM structure.
634 * @param pNew Pointer to the new list entry.
635 * @param pPrev Pointer to the previous list entry. If NULL, insert as head.
636 */
637static void pgmR3PhysLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, PPGMRAMRANGE pPrev)
638{
639 AssertMsg(pNew->pszDesc, ("%RGp-%RGp\n", pNew->GCPhys, pNew->GCPhysLast));
640 Assert((pNew->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pNew->pSelfR0 == MMHyperCCToR0(pVM, pNew));
641 Assert((pNew->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pNew->pSelfRC == MMHyperCCToRC(pVM, pNew));
642
643 pgmLock(pVM);
644
645 PPGMRAMRANGE pRam = pPrev ? pPrev->pNextR3 : pVM->pgm.s.pRamRangesR3;
646 pNew->pNextR3 = pRam;
647 pNew->pNextR0 = pRam ? pRam->pSelfR0 : NIL_RTR0PTR;
648 pNew->pNextRC = pRam ? pRam->pSelfRC : NIL_RTRCPTR;
649
650 if (pPrev)
651 {
652 pPrev->pNextR3 = pNew;
653 pPrev->pNextR0 = pNew->pSelfR0;
654 pPrev->pNextRC = pNew->pSelfRC;
655 }
656 else
657 {
658 pVM->pgm.s.pRamRangesR3 = pNew;
659 pVM->pgm.s.pRamRangesR0 = pNew->pSelfR0;
660 pVM->pgm.s.pRamRangesRC = pNew->pSelfRC;
661 }
662 ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
663 pgmUnlock(pVM);
664}
665
666
667/**
668 * Unlink an existing RAM range from the list.
669 *
670 * @param pVM Pointer to the shared VM structure.
671 * @param pRam Pointer to the new list entry.
672 * @param pPrev Pointer to the previous list entry. If NULL, insert as head.
673 */
674static void pgmR3PhysUnlinkRamRange2(PVM pVM, PPGMRAMRANGE pRam, PPGMRAMRANGE pPrev)
675{
676 Assert(pPrev ? pPrev->pNextR3 == pRam : pVM->pgm.s.pRamRangesR3 == pRam);
677 Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pRam->pSelfR0 == MMHyperCCToR0(pVM, pRam));
678 Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pRam->pSelfRC == MMHyperCCToRC(pVM, pRam));
679
680 pgmLock(pVM);
681
682 PPGMRAMRANGE pNext = pRam->pNextR3;
683 if (pPrev)
684 {
685 pPrev->pNextR3 = pNext;
686 pPrev->pNextR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
687 pPrev->pNextRC = pNext ? pNext->pSelfRC : NIL_RTRCPTR;
688 }
689 else
690 {
691 Assert(pVM->pgm.s.pRamRangesR3 == pRam);
692 pVM->pgm.s.pRamRangesR3 = pNext;
693 pVM->pgm.s.pRamRangesR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
694 pVM->pgm.s.pRamRangesRC = pNext ? pNext->pSelfRC : NIL_RTRCPTR;
695 }
696 ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
697 pgmUnlock(pVM);
698}
699
700
701/**
702 * Unlink an existing RAM range from the list.
703 *
704 * @param pVM Pointer to the shared VM structure.
705 * @param pRam Pointer to the new list entry.
706 */
707static void pgmR3PhysUnlinkRamRange(PVM pVM, PPGMRAMRANGE pRam)
708{
709 pgmLock(pVM);
710
711 /* find prev. */
712 PPGMRAMRANGE pPrev = NULL;
713 PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesR3;
714 while (pCur != pRam)
715 {
716 pPrev = pCur;
717 pCur = pCur->pNextR3;
718 }
719 AssertFatal(pCur);
720
721 pgmR3PhysUnlinkRamRange2(pVM, pRam, pPrev);
722 pgmUnlock(pVM);
723}
724
725
726/**
727 * Frees a range of pages, replacing them with ZERO pages of the specified type.
728 *
729 * @returns VBox status code.
730 * @param pVM The VM handle.
731 * @param pRam The RAM range in which the pages resides.
732 * @param GCPhys The address of the first page.
733 * @param GCPhysLast The address of the last page.
734 * @param uType The page type to replace then with.
735 */
736static int pgmR3PhysFreePageRange(PVM pVM, PPGMRAMRANGE pRam, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast, uint8_t uType)
737{
738 Assert(PGMIsLockOwner(pVM));
739 uint32_t cPendingPages = 0;
740 PGMMFREEPAGESREQ pReq;
741 int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
742 AssertLogRelRCReturn(rc, rc);
743
744 /* Iterate the pages. */
745 PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
746 uint32_t cPagesLeft = ((GCPhysLast - GCPhys) >> PAGE_SHIFT) + 1;
747 while (cPagesLeft-- > 0)
748 {
749 rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys);
750 AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
751
752 PGM_PAGE_SET_TYPE(pPageDst, uType);
753
754 GCPhys += PAGE_SIZE;
755 pPageDst++;
756 }
757
758 if (cPendingPages)
759 {
760 rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
761 AssertLogRelRCReturn(rc, rc);
762 }
763 GMMR3FreePagesCleanup(pReq);
764
765 return rc;
766}
767
768#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) || defined(RT_OS_SOLARIS) || defined(RT_OS_LINUX) || defined(RT_OS_FREEBSD))
769/**
770 * Rendezvous callback used by PGMR3ChangeMemBalloon that changes the memory balloon size
771 *
772 * This is only called on one of the EMTs while the other ones are waiting for
773 * it to complete this function.
774 *
775 * @returns VINF_SUCCESS (VBox strict status code).
776 * @param pVM The VM handle.
777 * @param pVCpu The VMCPU for the EMT we're being called on. Unused.
778 * @param pvUser User parameter
779 */
780static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysChangeMemBalloonRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
781{
782 uintptr_t *paUser = (uintptr_t *)pvUser;
783 bool fInflate = !!paUser[0];
784 unsigned cPages = paUser[1];
785 RTGCPHYS *paPhysPage = (RTGCPHYS *)paUser[2];
786 uint32_t cPendingPages = 0;
787 PGMMFREEPAGESREQ pReq;
788 int rc;
789
790 Log(("pgmR3PhysChangeMemBalloonRendezvous: %s %x pages\n", (fInflate) ? "inflate" : "deflate", cPages));
791 pgmLock(pVM);
792
793 if (fInflate)
794 {
795 /* Flush the PGM pool cache as we might have stale references to pages that we just freed. */
796 pgmR3PoolClearAllRendezvous(pVM, pVCpu, NULL);
797
798 /* Replace pages with ZERO pages. */
799 rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
800 if (RT_FAILURE(rc))
801 {
802 pgmUnlock(pVM);
803 AssertLogRelRC(rc);
804 return rc;
805 }
806
807 /* Iterate the pages. */
808 for (unsigned i = 0; i < cPages; i++)
809 {
810 PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, paPhysPage[i]);
811 if ( pPage == NULL
812 || pPage->uTypeY != PGMPAGETYPE_RAM)
813 {
814 Log(("pgmR3PhysChangeMemBalloonRendezvous: invalid physical page %RGp pPage->u3Type=%d\n", paPhysPage[i], (pPage) ? pPage->uTypeY : 0));
815 break;
816 }
817
818 LogFlow(("balloon page: %RGp\n", paPhysPage[i]));
819
820 /* Flush the shadow PT if this page was previously used as a guest page table. */
821 pgmPoolFlushPageByGCPhys(pVM, paPhysPage[i]);
822
823 rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, paPhysPage[i]);
824 if (RT_FAILURE(rc))
825 {
826 pgmUnlock(pVM);
827 AssertLogRelRC(rc);
828 return rc;
829 }
830 Assert(PGM_PAGE_IS_ZERO(pPage));
831 PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_BALLOONED);
832 }
833
834 if (cPendingPages)
835 {
836 rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
837 if (RT_FAILURE(rc))
838 {
839 pgmUnlock(pVM);
840 AssertLogRelRC(rc);
841 return rc;
842 }
843 }
844 GMMR3FreePagesCleanup(pReq);
845 }
846 else
847 {
848 /* Iterate the pages. */
849 for (unsigned i = 0; i < cPages; i++)
850 {
851 PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, paPhysPage[i]);
852 AssertBreak(pPage && pPage->uTypeY == PGMPAGETYPE_RAM);
853
854 LogFlow(("Free ballooned page: %RGp\n", paPhysPage[i]));
855
856 Assert(PGM_PAGE_IS_BALLOONED(pPage));
857
858 /* Change back to zero page. */
859 PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ZERO);
860 }
861
862 /* Note that we currently do not map any ballooned pages in our shadow page tables, so no need to flush the pgm pool. */
863 }
864
865 /* Notify GMM about the balloon change. */
866 rc = GMMR3BalloonedPages(pVM, (fInflate) ? GMMBALLOONACTION_INFLATE : GMMBALLOONACTION_DEFLATE, cPages);
867 if (RT_SUCCESS(rc))
868 {
869 if (!fInflate)
870 {
871 Assert(pVM->pgm.s.cBalloonedPages >= cPages);
872 pVM->pgm.s.cBalloonedPages -= cPages;
873 }
874 else
875 pVM->pgm.s.cBalloonedPages += cPages;
876 }
877
878 pgmUnlock(pVM);
879
880 /* Flush the recompiler's TLB as well. */
881 for (VMCPUID i = 0; i < pVM->cCpus; i++)
882 CPUMSetChangedFlags(&pVM->aCpus[i], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
883
884 AssertLogRelRC(rc);
885 return rc;
886}
887
888/**
889 * Frees a range of ram pages, replacing them with ZERO pages; helper for PGMR3PhysFreeRamPages
890 *
891 * @returns VBox status code.
892 * @param pVM The VM handle.
893 * @param fInflate Inflate or deflate memory balloon
894 * @param cPages Number of pages to free
895 * @param paPhysPage Array of guest physical addresses
896 */
897static DECLCALLBACK(void) pgmR3PhysChangeMemBalloonHelper(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
898{
899 uintptr_t paUser[3];
900
901 paUser[0] = fInflate;
902 paUser[1] = cPages;
903 paUser[2] = (uintptr_t)paPhysPage;
904 int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
905 AssertRC(rc);
906
907 /* Made a copy in PGMR3PhysFreeRamPages; free it here. */
908 RTMemFree(paPhysPage);
909}
910#endif
911
912/**
913 * Inflate or deflate a memory balloon
914 *
915 * @returns VBox status code.
916 * @param pVM The VM handle.
917 * @param fInflate Inflate or deflate memory balloon
918 * @param cPages Number of pages to free
919 * @param paPhysPage Array of guest physical addresses
920 */
921VMMR3DECL(int) PGMR3PhysChangeMemBalloon(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
922{
923 /* This must match GMMR0Init; currently we only support memory ballooning on all 64-bit hosts except Mac OS X */
924#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) || defined(RT_OS_SOLARIS) || defined(RT_OS_LINUX) || defined(RT_OS_FREEBSD))
925 int rc;
926
927 /* Older additions (ancient non-functioning balloon code) pass wrong physical addresses. */
928 AssertReturn(!(paPhysPage[0] & 0xfff), VERR_INVALID_PARAMETER);
929
930 /* We own the IOM lock here and could cause a deadlock by waiting for another VCPU that is blocking on the IOM lock.
931 * In the SMP case we post a request packet to postpone the job.
932 */
933 if (pVM->cCpus > 1)
934 {
935 unsigned cbPhysPage = cPages * sizeof(paPhysPage[0]);
936 RTGCPHYS *paPhysPageCopy = (RTGCPHYS *)RTMemAlloc(cbPhysPage);
937 AssertReturn(paPhysPageCopy, VERR_NO_MEMORY);
938
939 memcpy(paPhysPageCopy, paPhysPage, cbPhysPage);
940
941 rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysChangeMemBalloonHelper, 4, pVM, fInflate, cPages, paPhysPageCopy);
942 AssertRC(rc);
943 }
944 else
945 {
946 uintptr_t paUser[3];
947
948 paUser[0] = fInflate;
949 paUser[1] = cPages;
950 paUser[2] = (uintptr_t)paPhysPage;
951 rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
952 AssertRC(rc);
953 }
954 return rc;
955#else
956 return VERR_NOT_IMPLEMENTED;
957#endif
958}
959
960/**
961 * Rendezvous callback used by PGMR3WriteProtectRAM that write protects all physical RAM
962 *
963 * This is only called on one of the EMTs while the other ones are waiting for
964 * it to complete this function.
965 *
966 * @returns VINF_SUCCESS (VBox strict status code).
967 * @param pVM The VM handle.
968 * @param pVCpu The VMCPU for the EMT we're being called on. Unused.
969 * @param pvUser User parameter
970 */
971static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysWriteProtectRAMRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
972{
973 int rc = VINF_SUCCESS;
974
975 pgmLock(pVM);
976#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
977 pgmPoolResetDirtyPages(pVM);
978#endif
979
980 /** @todo pointless to write protect the physical page pointed to by RSP. */
981
982 for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
983 pRam;
984 pRam = pRam->CTX_SUFF(pNext))
985 {
986 if (!PGM_RAM_RANGE_IS_AD_HOC(pRam))
987 {
988 unsigned cPages = pRam->cb >> PAGE_SHIFT;
989 for (unsigned iPage = 0; iPage < cPages; iPage++)
990 {
991 PPGMPAGE pPage = &pRam->aPages[iPage];
992 if (RT_LIKELY(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM))
993 {
994 /*
995 * A RAM page.
996 */
997 switch (PGM_PAGE_GET_STATE(pPage))
998 {
999 case PGM_PAGE_STATE_ALLOCATED:
1000 /** @todo Optimize this: Don't always re-enable write
1001 * monitoring if the page is known to be very busy. */
1002 if (PGM_PAGE_IS_WRITTEN_TO(pPage))
1003 {
1004 PGM_PAGE_CLEAR_WRITTEN_TO(pPage);
1005 /* Remember this dirty page for the next (memory) sync. */
1006 PGM_PAGE_SET_FT_DIRTY(pPage);
1007 }
1008
1009 PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_WRITE_MONITORED);
1010 pVM->pgm.s.cMonitoredPages++;
1011 break;
1012
1013 case PGM_PAGE_STATE_SHARED:
1014 AssertFailed();
1015 break;
1016
1017 case PGM_PAGE_STATE_WRITE_MONITORED: /* nothing to change. */
1018 default:
1019 break;
1020 }
1021 }
1022 }
1023 }
1024 }
1025 pgmR3PoolWriteProtectPages(pVM);
1026 PGM_INVL_ALL_VCPU_TLBS(pVM);
1027 for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1028 CPUMSetChangedFlags(&pVM->aCpus[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
1029
1030 pgmUnlock(pVM);
1031 return rc;
1032}
1033
1034/**
1035 * Protect all physical RAM to monitor writes
1036 *
1037 * @returns VBox status code.
1038 * @param pVM The VM handle.
1039 */
1040VMMR3DECL(int) PGMR3PhysWriteProtectRAM(PVM pVM)
1041{
1042 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
1043
1044 int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysWriteProtectRAMRendezvous, NULL);
1045 AssertRC(rc);
1046 return rc;
1047}
1048
1049/**
1050 * Enumerate all dirty FT pages
1051 *
1052 * @returns VBox status code.
1053 * @param pVM The VM handle.
1054 * @param pfnEnum Enumerate callback handler
1055 * @param pvUser Enumerate callback handler parameter
1056 */
1057VMMR3DECL(int) PGMR3PhysEnumDirtyFTPages(PVM pVM, PFNPGMENUMDIRTYFTPAGES pfnEnum, void *pvUser)
1058{
1059 int rc = VINF_SUCCESS;
1060
1061 pgmLock(pVM);
1062 for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
1063 pRam;
1064 pRam = pRam->CTX_SUFF(pNext))
1065 {
1066 if (!PGM_RAM_RANGE_IS_AD_HOC(pRam))
1067 {
1068 unsigned cPages = pRam->cb >> PAGE_SHIFT;
1069 for (unsigned iPage = 0; iPage < cPages; iPage++)
1070 {
1071 PPGMPAGE pPage = &pRam->aPages[iPage];
1072 if (RT_LIKELY(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM))
1073 {
1074 /*
1075 * A RAM page.
1076 */
1077 switch (PGM_PAGE_GET_STATE(pPage))
1078 {
1079 case PGM_PAGE_STATE_ALLOCATED:
1080 case PGM_PAGE_STATE_WRITE_MONITORED:
1081 if ( !PGM_PAGE_IS_WRITTEN_TO(pPage) /* not very recently updated? */
1082 && PGM_PAGE_IS_FT_DIRTY(pPage))
1083 {
1084 unsigned cbPageRange = PAGE_SIZE;
1085 unsigned iPageClean = iPage + 1;
1086 RTGCPHYS GCPhysPage = pRam->GCPhys + iPage * PAGE_SIZE;
1087 uint8_t *pu8Page = NULL;
1088 PGMPAGEMAPLOCK Lock;
1089
1090 /* Find the next clean page, so we can merge adjacent dirty pages. */
1091 for (; iPageClean < cPages; iPageClean++)
1092 {
1093 PPGMPAGE pPageNext = &pRam->aPages[iPageClean];
1094 if ( RT_UNLIKELY(PGM_PAGE_GET_TYPE(pPageNext) != PGMPAGETYPE_RAM)
1095 || PGM_PAGE_GET_STATE(pPageNext) != PGM_PAGE_STATE_ALLOCATED
1096 || PGM_PAGE_IS_WRITTEN_TO(pPageNext)
1097 || !PGM_PAGE_IS_FT_DIRTY(pPageNext)
1098 /* Crossing a chunk boundary? */
1099 || (GCPhysPage & GMM_PAGEID_IDX_MASK) != ((GCPhysPage + cbPageRange) & GMM_PAGEID_IDX_MASK)
1100 )
1101 break;
1102
1103 cbPageRange += PAGE_SIZE;
1104 }
1105
1106 rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysPage, (const void **)&pu8Page, &Lock);
1107 if (RT_SUCCESS(rc))
1108 {
1109 /** @todo this is risky; the range might be changed, but little choice as the sync costs a lot of time */
1110 pgmUnlock(pVM);
1111 pfnEnum(pVM, GCPhysPage, pu8Page, cbPageRange, pvUser);
1112 pgmLock(pVM);
1113 PGMPhysReleasePageMappingLock(pVM, &Lock);
1114 }
1115
1116 for (iPage; iPage < iPageClean; iPage++)
1117 PGM_PAGE_CLEAR_FT_DIRTY(&pRam->aPages[iPage]);
1118
1119 iPage = iPageClean - 1;
1120 }
1121 break;
1122 }
1123 }
1124 }
1125 }
1126 }
1127 pgmUnlock(pVM);
1128 return rc;
1129}
1130
1131/**
1132 * Query the amount of free memory inside VMMR0
1133 *
1134 * @returns VBox status code.
1135 * @param pVM The VM handle.
1136 * @param puTotalAllocSize Pointer to total allocated memory inside VMMR0 (in bytes)
1137 * @param puTotalFreeSize Pointer to total free (allocated but not used yet) memory inside VMMR0 (in bytes)
1138 * @param puTotalBalloonSize Pointer to total ballooned memory inside VMMR0 (in bytes)
1139 * @param puTotalSharedSize Pointer to total shared memory inside VMMR0 (in bytes)
1140 */
1141VMMR3DECL(int) PGMR3QueryVMMMemoryStats(PVM pVM, uint64_t *puTotalAllocSize, uint64_t *puTotalFreeSize, uint64_t *puTotalBalloonSize, uint64_t *puTotalSharedSize)
1142{
1143 int rc;
1144
1145 uint64_t cAllocPages = 0, cFreePages = 0, cBalloonPages = 0, cSharedPages = 0;
1146 rc = GMMR3QueryHypervisorMemoryStats(pVM, &cAllocPages, &cFreePages, &cBalloonPages, &cSharedPages);
1147 AssertRCReturn(rc, rc);
1148
1149 if (puTotalAllocSize)
1150 *puTotalAllocSize = cAllocPages * _4K;
1151
1152 if (puTotalFreeSize)
1153 *puTotalFreeSize = cFreePages * _4K;
1154
1155 if (puTotalBalloonSize)
1156 *puTotalBalloonSize = cBalloonPages * _4K;
1157
1158 if (puTotalSharedSize)
1159 *puTotalSharedSize = cSharedPages * _4K;
1160
1161 Log(("PGMR3QueryVMMMemoryStats: all=%x free=%x ballooned=%x shared=%x\n", cAllocPages, cFreePages, cBalloonPages, cSharedPages));
1162 return VINF_SUCCESS;
1163}
1164
1165/**
1166 * Query memory stats for the VM
1167 *
1168 * @returns VBox status code.
1169 * @param pVM The VM handle.
1170 * @param puTotalAllocSize Pointer to total allocated memory inside the VM (in bytes)
1171 * @param puTotalFreeSize Pointer to total free (allocated but not used yet) memory inside the VM (in bytes)
1172 * @param puTotalBalloonSize Pointer to total ballooned memory inside the VM (in bytes)
1173 * @param puTotalSharedSize Pointer to total shared memory inside the VM (in bytes)
1174 */
1175VMMR3DECL(int) PGMR3QueryMemoryStats(PVM pVM, uint64_t *pulTotalMem, uint64_t *pulPrivateMem, uint64_t *puTotalSharedMem, uint64_t *puTotalZeroMem)
1176{
1177 if (pulTotalMem)
1178 *pulTotalMem = (uint64_t)pVM->pgm.s.cAllPages * _4K;
1179
1180 if (pulPrivateMem)
1181 *pulPrivateMem = (uint64_t)pVM->pgm.s.cPrivatePages * _4K;
1182
1183 if (puTotalSharedMem)
1184 *puTotalSharedMem = (uint64_t)pVM->pgm.s.cReusedSharedPages * _4K;
1185
1186 if (puTotalZeroMem)
1187 *puTotalZeroMem = (uint64_t)pVM->pgm.s.cZeroPages * _4K;
1188
1189 Log(("PGMR3QueryMemoryStats: all=%x private=%x reused=%x zero=%x\n", pVM->pgm.s.cAllPages, pVM->pgm.s.cPrivatePages, pVM->pgm.s.cReusedSharedPages, pVM->pgm.s.cZeroPages));
1190 return VINF_SUCCESS;
1191}
1192
1193/**
1194 * PGMR3PhysRegisterRam worker that initializes and links a RAM range.
1195 *
1196 * @param pVM The VM handle.
1197 * @param pNew The new RAM range.
1198 * @param GCPhys The address of the RAM range.
1199 * @param GCPhysLast The last address of the RAM range.
1200 * @param RCPtrNew The RC address if the range is floating. NIL_RTRCPTR
1201 * if in HMA.
1202 * @param R0PtrNew Ditto for R0.
1203 * @param pszDesc The description.
1204 * @param pPrev The previous RAM range (for linking).
1205 */
1206static void pgmR3PhysInitAndLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast,
1207 RTRCPTR RCPtrNew, RTR0PTR R0PtrNew, const char *pszDesc, PPGMRAMRANGE pPrev)
1208{
1209 /*
1210 * Initialize the range.
1211 */
1212 pNew->pSelfR0 = R0PtrNew != NIL_RTR0PTR ? R0PtrNew : MMHyperCCToR0(pVM, pNew);
1213 pNew->pSelfRC = RCPtrNew != NIL_RTRCPTR ? RCPtrNew : MMHyperCCToRC(pVM, pNew);
1214 pNew->GCPhys = GCPhys;
1215 pNew->GCPhysLast = GCPhysLast;
1216 pNew->cb = GCPhysLast - GCPhys + 1;
1217 pNew->pszDesc = pszDesc;
1218 pNew->fFlags = RCPtrNew != NIL_RTRCPTR ? PGM_RAM_RANGE_FLAGS_FLOATING : 0;
1219 pNew->pvR3 = NULL;
1220 pNew->paLSPages = NULL;
1221
1222 uint32_t const cPages = pNew->cb >> PAGE_SHIFT;
1223 RTGCPHYS iPage = cPages;
1224 while (iPage-- > 0)
1225 PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_RAM);
1226
1227 /* Update the page count stats. */
1228 pVM->pgm.s.cZeroPages += cPages;
1229 pVM->pgm.s.cAllPages += cPages;
1230
1231 /*
1232 * Link it.
1233 */
1234 pgmR3PhysLinkRamRange(pVM, pNew, pPrev);
1235}
1236
1237
1238/**
1239 * Relocate a floating RAM range.
1240 *
1241 * @copydoc FNPGMRELOCATE.
1242 */
1243static DECLCALLBACK(bool) pgmR3PhysRamRangeRelocate(PVM pVM, RTGCPTR GCPtrOld, RTGCPTR GCPtrNew, PGMRELOCATECALL enmMode, void *pvUser)
1244{
1245 PPGMRAMRANGE pRam = (PPGMRAMRANGE)pvUser;
1246 Assert(pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING);
1247 Assert(pRam->pSelfRC == GCPtrOld + PAGE_SIZE);
1248
1249 switch (enmMode)
1250 {
1251 case PGMRELOCATECALL_SUGGEST:
1252 return true;
1253 case PGMRELOCATECALL_RELOCATE:
1254 {
1255 /* Update myself and then relink all the ranges. */
1256 pgmLock(pVM);
1257 pRam->pSelfRC = (RTRCPTR)(GCPtrNew + PAGE_SIZE);
1258 pgmR3PhysRelinkRamRanges(pVM);
1259 pgmUnlock(pVM);
1260 return true;
1261 }
1262
1263 default:
1264 AssertFailedReturn(false);
1265 }
1266}
1267
1268
1269/**
1270 * PGMR3PhysRegisterRam worker that registers a high chunk.
1271 *
1272 * @returns VBox status code.
1273 * @param pVM The VM handle.
1274 * @param GCPhys The address of the RAM.
1275 * @param cRamPages The number of RAM pages to register.
1276 * @param cbChunk The size of the PGMRAMRANGE guest mapping.
1277 * @param iChunk The chunk number.
1278 * @param pszDesc The RAM range description.
1279 * @param ppPrev Previous RAM range pointer. In/Out.
1280 */
1281static int pgmR3PhysRegisterHighRamChunk(PVM pVM, RTGCPHYS GCPhys, uint32_t cRamPages,
1282 uint32_t cbChunk, uint32_t iChunk, const char *pszDesc,
1283 PPGMRAMRANGE *ppPrev)
1284{
1285 const char *pszDescChunk = iChunk == 0
1286 ? pszDesc
1287 : MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s (#%u)", pszDesc, iChunk + 1);
1288 AssertReturn(pszDescChunk, VERR_NO_MEMORY);
1289
1290 /*
1291 * Allocate memory for the new chunk.
1292 */
1293 size_t const cChunkPages = RT_ALIGN_Z(RT_UOFFSETOF(PGMRAMRANGE, aPages[cRamPages]), PAGE_SIZE) >> PAGE_SHIFT;
1294 PSUPPAGE paChunkPages = (PSUPPAGE)RTMemTmpAllocZ(sizeof(SUPPAGE) * cChunkPages);
1295 AssertReturn(paChunkPages, VERR_NO_TMP_MEMORY);
1296 RTR0PTR R0PtrChunk = NIL_RTR0PTR;
1297 void *pvChunk = NULL;
1298 int rc = SUPR3PageAllocEx(cChunkPages, 0 /*fFlags*/, &pvChunk,
1299#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1300 VMMIsHwVirtExtForced(pVM) ? &R0PtrChunk : NULL,
1301#else
1302 NULL,
1303#endif
1304 paChunkPages);
1305 if (RT_SUCCESS(rc))
1306 {
1307#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1308 if (!VMMIsHwVirtExtForced(pVM))
1309 R0PtrChunk = NIL_RTR0PTR;
1310#else
1311 R0PtrChunk = (uintptr_t)pvChunk;
1312#endif
1313 memset(pvChunk, 0, cChunkPages << PAGE_SHIFT);
1314
1315 PPGMRAMRANGE pNew = (PPGMRAMRANGE)pvChunk;
1316
1317 /*
1318 * Create a mapping and map the pages into it.
1319 * We push these in below the HMA.
1320 */
1321 RTGCPTR GCPtrChunkMap = pVM->pgm.s.GCPtrPrevRamRangeMapping - cbChunk;
1322 rc = PGMR3MapPT(pVM, GCPtrChunkMap, cbChunk, 0 /*fFlags*/, pgmR3PhysRamRangeRelocate, pNew, pszDescChunk);
1323 if (RT_SUCCESS(rc))
1324 {
1325 pVM->pgm.s.GCPtrPrevRamRangeMapping = GCPtrChunkMap;
1326
1327 RTGCPTR const GCPtrChunk = GCPtrChunkMap + PAGE_SIZE;
1328 RTGCPTR GCPtrPage = GCPtrChunk;
1329 for (uint32_t iPage = 0; iPage < cChunkPages && RT_SUCCESS(rc); iPage++, GCPtrPage += PAGE_SIZE)
1330 rc = PGMMap(pVM, GCPtrPage, paChunkPages[iPage].Phys, PAGE_SIZE, 0);
1331 if (RT_SUCCESS(rc))
1332 {
1333 /*
1334 * Ok, init and link the range.
1335 */
1336 pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhys + ((RTGCPHYS)cRamPages << PAGE_SHIFT) - 1,
1337 (RTRCPTR)GCPtrChunk, R0PtrChunk, pszDescChunk, *ppPrev);
1338 *ppPrev = pNew;
1339 }
1340 }
1341
1342 if (RT_FAILURE(rc))
1343 SUPR3PageFreeEx(pvChunk, cChunkPages);
1344 }
1345
1346 RTMemTmpFree(paChunkPages);
1347 return rc;
1348}
1349
1350
1351/**
1352 * Sets up a range RAM.
1353 *
1354 * This will check for conflicting registrations, make a resource
1355 * reservation for the memory (with GMM), and setup the per-page
1356 * tracking structures (PGMPAGE).
1357 *
1358 * @returns VBox stutus code.
1359 * @param pVM Pointer to the shared VM structure.
1360 * @param GCPhys The physical address of the RAM.
1361 * @param cb The size of the RAM.
1362 * @param pszDesc The description - not copied, so, don't free or change it.
1363 */
1364VMMR3DECL(int) PGMR3PhysRegisterRam(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, const char *pszDesc)
1365{
1366 /*
1367 * Validate input.
1368 */
1369 Log(("PGMR3PhysRegisterRam: GCPhys=%RGp cb=%RGp pszDesc=%s\n", GCPhys, cb, pszDesc));
1370 AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
1371 AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
1372 AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
1373 RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
1374 AssertMsgReturn(GCPhysLast > GCPhys, ("The range wraps! GCPhys=%RGp cb=%RGp\n", GCPhys, cb), VERR_INVALID_PARAMETER);
1375 AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
1376 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
1377
1378 pgmLock(pVM);
1379
1380 /*
1381 * Find range location and check for conflicts.
1382 * (We don't lock here because the locking by EMT is only required on update.)
1383 */
1384 PPGMRAMRANGE pPrev = NULL;
1385 PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
1386 while (pRam && GCPhysLast >= pRam->GCPhys)
1387 {
1388 if ( GCPhysLast >= pRam->GCPhys
1389 && GCPhys <= pRam->GCPhysLast)
1390 AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
1391 GCPhys, GCPhysLast, pszDesc,
1392 pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
1393 VERR_PGM_RAM_CONFLICT);
1394
1395 /* next */
1396 pPrev = pRam;
1397 pRam = pRam->pNextR3;
1398 }
1399
1400 /*
1401 * Register it with GMM (the API bitches).
1402 */
1403 const RTGCPHYS cPages = cb >> PAGE_SHIFT;
1404 int rc = MMR3IncreaseBaseReservation(pVM, cPages);
1405 if (RT_FAILURE(rc))
1406 {
1407 pgmUnlock(pVM);
1408 return rc;
1409 }
1410
1411 if ( GCPhys >= _4G
1412 && cPages > 256)
1413 {
1414 /*
1415 * The PGMRAMRANGE structures for the high memory can get very big.
1416 * In order to avoid SUPR3PageAllocEx allocation failures due to the
1417 * allocation size limit there and also to avoid being unable to find
1418 * guest mapping space for them, we split this memory up into 4MB in
1419 * (potential) raw-mode configs and 16MB chunks in forced AMD-V/VT-x
1420 * mode.
1421 *
1422 * The first and last page of each mapping are guard pages and marked
1423 * not-present. So, we've got 4186112 and 16769024 bytes available for
1424 * the PGMRAMRANGE structure.
1425 *
1426 * Note! The sizes used here will influence the saved state.
1427 */
1428 uint32_t cbChunk;
1429 uint32_t cPagesPerChunk;
1430 if (VMMIsHwVirtExtForced(pVM))
1431 {
1432 cbChunk = 16U*_1M;
1433 cPagesPerChunk = 1048048; /* max ~1048059 */
1434 AssertCompile(sizeof(PGMRAMRANGE) + sizeof(PGMPAGE) * 1048048 < 16U*_1M - PAGE_SIZE * 2);
1435 }
1436 else
1437 {
1438 cbChunk = 4U*_1M;
1439 cPagesPerChunk = 261616; /* max ~261627 */
1440 AssertCompile(sizeof(PGMRAMRANGE) + sizeof(PGMPAGE) * 261616 < 4U*_1M - PAGE_SIZE * 2);
1441 }
1442 AssertRelease(RT_UOFFSETOF(PGMRAMRANGE, aPages[cPagesPerChunk]) + PAGE_SIZE * 2 <= cbChunk);
1443
1444 RTGCPHYS cPagesLeft = cPages;
1445 RTGCPHYS GCPhysChunk = GCPhys;
1446 uint32_t iChunk = 0;
1447 while (cPagesLeft > 0)
1448 {
1449 uint32_t cPagesInChunk = cPagesLeft;
1450 if (cPagesInChunk > cPagesPerChunk)
1451 cPagesInChunk = cPagesPerChunk;
1452
1453 rc = pgmR3PhysRegisterHighRamChunk(pVM, GCPhysChunk, cPagesInChunk, cbChunk, iChunk, pszDesc, &pPrev);
1454 AssertRCReturn(rc, rc);
1455
1456 /* advance */
1457 GCPhysChunk += (RTGCPHYS)cPagesInChunk << PAGE_SHIFT;
1458 cPagesLeft -= cPagesInChunk;
1459 iChunk++;
1460 }
1461 }
1462 else
1463 {
1464 /*
1465 * Allocate, initialize and link the new RAM range.
1466 */
1467 const size_t cbRamRange = RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]);
1468 PPGMRAMRANGE pNew;
1469 rc = MMR3HyperAllocOnceNoRel(pVM, cbRamRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
1470 AssertLogRelMsgRCReturn(rc, ("cbRamRange=%zu\n", cbRamRange), rc);
1471
1472 pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhysLast, NIL_RTRCPTR, NIL_RTR0PTR, pszDesc, pPrev);
1473 }
1474 PGMPhysInvalidatePageMapTLB(pVM);
1475 pgmUnlock(pVM);
1476
1477 /*
1478 * Notify REM.
1479 */
1480 REMR3NotifyPhysRamRegister(pVM, GCPhys, cb, REM_NOTIFY_PHYS_RAM_FLAGS_RAM);
1481
1482 return VINF_SUCCESS;
1483}
1484
1485
1486/**
1487 * Worker called by PGMR3InitFinalize if we're configured to pre-allocate RAM.
1488 *
1489 * We do this late in the init process so that all the ROM and MMIO ranges have
1490 * been registered already and we don't go wasting memory on them.
1491 *
1492 * @returns VBox status code.
1493 *
1494 * @param pVM Pointer to the shared VM structure.
1495 */
1496int pgmR3PhysRamPreAllocate(PVM pVM)
1497{
1498 Assert(pVM->pgm.s.fRamPreAlloc);
1499 Log(("pgmR3PhysRamPreAllocate: enter\n"));
1500
1501 /*
1502 * Walk the RAM ranges and allocate all RAM pages, halt at
1503 * the first allocation error.
1504 */
1505 uint64_t cPages = 0;
1506 uint64_t NanoTS = RTTimeNanoTS();
1507 pgmLock(pVM);
1508 for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3; pRam; pRam = pRam->pNextR3)
1509 {
1510 PPGMPAGE pPage = &pRam->aPages[0];
1511 RTGCPHYS GCPhys = pRam->GCPhys;
1512 uint32_t cLeft = pRam->cb >> PAGE_SHIFT;
1513 while (cLeft-- > 0)
1514 {
1515 if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
1516 {
1517 switch (PGM_PAGE_GET_STATE(pPage))
1518 {
1519 case PGM_PAGE_STATE_ZERO:
1520 {
1521 int rc = pgmPhysAllocPage(pVM, pPage, GCPhys);
1522 if (RT_FAILURE(rc))
1523 {
1524 LogRel(("PGM: RAM Pre-allocation failed at %RGp (in %s) with rc=%Rrc\n", GCPhys, pRam->pszDesc, rc));
1525 pgmUnlock(pVM);
1526 return rc;
1527 }
1528 cPages++;
1529 break;
1530 }
1531
1532 case PGM_PAGE_STATE_BALLOONED:
1533 case PGM_PAGE_STATE_ALLOCATED:
1534 case PGM_PAGE_STATE_WRITE_MONITORED:
1535 case PGM_PAGE_STATE_SHARED:
1536 /* nothing to do here. */
1537 break;
1538 }
1539 }
1540
1541 /* next */
1542 pPage++;
1543 GCPhys += PAGE_SIZE;
1544 }
1545 }
1546 pgmUnlock(pVM);
1547 NanoTS = RTTimeNanoTS() - NanoTS;
1548
1549 LogRel(("PGM: Pre-allocated %llu pages in %llu ms\n", cPages, NanoTS / 1000000));
1550 Log(("pgmR3PhysRamPreAllocate: returns VINF_SUCCESS\n"));
1551 return VINF_SUCCESS;
1552}
1553
1554
1555/**
1556 * Resets (zeros) the RAM.
1557 *
1558 * ASSUMES that the caller owns the PGM lock.
1559 *
1560 * @returns VBox status code.
1561 * @param pVM Pointer to the shared VM structure.
1562 */
1563int pgmR3PhysRamReset(PVM pVM)
1564{
1565 Assert(PGMIsLockOwner(pVM));
1566
1567 /* Reset the memory balloon. */
1568 int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
1569 AssertRC(rc);
1570
1571#ifdef VBOX_WITH_PAGE_SHARING
1572 /* Clear all registered shared modules. */
1573 rc = GMMR3ResetSharedModules(pVM);
1574 AssertRC(rc);
1575#endif
1576 /* Reset counters. */
1577 pVM->pgm.s.cReusedSharedPages = 0;
1578 pVM->pgm.s.cBalloonedPages = 0;
1579
1580 /*
1581 * We batch up pages that should be freed instead of calling GMM for
1582 * each and every one of them.
1583 */
1584 uint32_t cPendingPages = 0;
1585 PGMMFREEPAGESREQ pReq;
1586 rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
1587 AssertLogRelRCReturn(rc, rc);
1588
1589 /*
1590 * Walk the ram ranges.
1591 */
1592 for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3; pRam; pRam = pRam->pNextR3)
1593 {
1594 uint32_t iPage = pRam->cb >> PAGE_SHIFT;
1595 AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
1596
1597 if (!pVM->pgm.s.fRamPreAlloc)
1598 {
1599 /* Replace all RAM pages by ZERO pages. */
1600 while (iPage-- > 0)
1601 {
1602 PPGMPAGE pPage = &pRam->aPages[iPage];
1603 switch (PGM_PAGE_GET_TYPE(pPage))
1604 {
1605 case PGMPAGETYPE_RAM:
1606 /* Do not replace pages part of a 2 MB continuous range with zero pages, but zero them instead. */
1607 if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
1608 {
1609 void *pvPage;
1610 rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
1611 AssertLogRelRCReturn(rc, rc);
1612 ASMMemZeroPage(pvPage);
1613 }
1614 else
1615 if (PGM_PAGE_IS_BALLOONED(pPage))
1616 {
1617 /* Turn into a zero page; the balloon status is lost when the VM reboots. */
1618 PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ZERO);
1619 }
1620 else
1621 if (!PGM_PAGE_IS_ZERO(pPage))
1622 {
1623 rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1624 AssertLogRelRCReturn(rc, rc);
1625 }
1626 break;
1627
1628 case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
1629 pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
1630 true /*fDoAccounting*/);
1631 break;
1632
1633 case PGMPAGETYPE_MMIO2:
1634 case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
1635 case PGMPAGETYPE_ROM:
1636 case PGMPAGETYPE_MMIO:
1637 break;
1638 default:
1639 AssertFailed();
1640 }
1641 } /* for each page */
1642 }
1643 else
1644 {
1645 /* Zero the memory. */
1646 while (iPage-- > 0)
1647 {
1648 PPGMPAGE pPage = &pRam->aPages[iPage];
1649 switch (PGM_PAGE_GET_TYPE(pPage))
1650 {
1651 case PGMPAGETYPE_RAM:
1652 switch (PGM_PAGE_GET_STATE(pPage))
1653 {
1654 case PGM_PAGE_STATE_ZERO:
1655 break;
1656
1657 case PGM_PAGE_STATE_BALLOONED:
1658 /* Turn into a zero page; the balloon status is lost when the VM reboots. */
1659 PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ZERO);
1660 break;
1661
1662 case PGM_PAGE_STATE_SHARED:
1663 case PGM_PAGE_STATE_WRITE_MONITORED:
1664 rc = pgmPhysPageMakeWritable(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1665 AssertLogRelRCReturn(rc, rc);
1666 /* no break */
1667
1668 case PGM_PAGE_STATE_ALLOCATED:
1669 {
1670 void *pvPage;
1671 rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
1672 AssertLogRelRCReturn(rc, rc);
1673 ASMMemZeroPage(pvPage);
1674 break;
1675 }
1676 }
1677 break;
1678
1679 case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
1680 pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
1681 true /*fDoAccounting*/);
1682 break;
1683
1684 case PGMPAGETYPE_MMIO2:
1685 case PGMPAGETYPE_ROM_SHADOW:
1686 case PGMPAGETYPE_ROM:
1687 case PGMPAGETYPE_MMIO:
1688 break;
1689 default:
1690 AssertFailed();
1691
1692 }
1693 } /* for each page */
1694 }
1695
1696 }
1697
1698 /*
1699 * Finish off any pages pending freeing.
1700 */
1701 if (cPendingPages)
1702 {
1703 rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
1704 AssertLogRelRCReturn(rc, rc);
1705 }
1706 GMMR3FreePagesCleanup(pReq);
1707
1708 return VINF_SUCCESS;
1709}
1710
1711/**
1712 * Frees all RAM during VM termination
1713 *
1714 * ASSUMES that the caller owns the PGM lock.
1715 *
1716 * @returns VBox status code.
1717 * @param pVM Pointer to the shared VM structure.
1718 */
1719int pgmR3PhysRamTerm(PVM pVM)
1720{
1721 Assert(PGMIsLockOwner(pVM));
1722
1723 /* Reset the memory balloon. */
1724 int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
1725 AssertRC(rc);
1726
1727#ifdef VBOX_WITH_PAGE_SHARING
1728 /* Clear all registered shared modules. */
1729 rc = GMMR3ResetSharedModules(pVM);
1730 AssertRC(rc);
1731#endif
1732
1733 /*
1734 * We batch up pages that should be freed instead of calling GMM for
1735 * each and every one of them.
1736 */
1737 uint32_t cPendingPages = 0;
1738 PGMMFREEPAGESREQ pReq;
1739 rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
1740 AssertLogRelRCReturn(rc, rc);
1741
1742 /*
1743 * Walk the ram ranges.
1744 */
1745 for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3; pRam; pRam = pRam->pNextR3)
1746 {
1747 uint32_t iPage = pRam->cb >> PAGE_SHIFT;
1748 AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
1749
1750 /* Replace all RAM pages by ZERO pages. */
1751 while (iPage-- > 0)
1752 {
1753 PPGMPAGE pPage = &pRam->aPages[iPage];
1754 switch (PGM_PAGE_GET_TYPE(pPage))
1755 {
1756 case PGMPAGETYPE_RAM:
1757 /* Free all shared pages. Private pages are automatically freed during GMM VM cleanup. */
1758 if (PGM_PAGE_IS_SHARED(pPage))
1759 {
1760 rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1761 AssertLogRelRCReturn(rc, rc);
1762 }
1763 break;
1764
1765 case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
1766 case PGMPAGETYPE_MMIO2:
1767 case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
1768 case PGMPAGETYPE_ROM:
1769 case PGMPAGETYPE_MMIO:
1770 break;
1771 default:
1772 AssertFailed();
1773 }
1774 } /* for each page */
1775 }
1776
1777 /*
1778 * Finish off any pages pending freeing.
1779 */
1780 if (cPendingPages)
1781 {
1782 rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
1783 AssertLogRelRCReturn(rc, rc);
1784 }
1785 GMMR3FreePagesCleanup(pReq);
1786 return VINF_SUCCESS;
1787}
1788
1789/**
1790 * This is the interface IOM is using to register an MMIO region.
1791 *
1792 * It will check for conflicts and ensure that a RAM range structure
1793 * is present before calling the PGMR3HandlerPhysicalRegister API to
1794 * register the callbacks.
1795 *
1796 * @returns VBox status code.
1797 *
1798 * @param pVM Pointer to the shared VM structure.
1799 * @param GCPhys The start of the MMIO region.
1800 * @param cb The size of the MMIO region.
1801 * @param pfnHandlerR3 The address of the ring-3 handler. (IOMR3MMIOHandler)
1802 * @param pvUserR3 The user argument for R3.
1803 * @param pfnHandlerR0 The address of the ring-0 handler. (IOMMMIOHandler)
1804 * @param pvUserR0 The user argument for R0.
1805 * @param pfnHandlerRC The address of the RC handler. (IOMMMIOHandler)
1806 * @param pvUserRC The user argument for RC.
1807 * @param pszDesc The description of the MMIO region.
1808 */
1809VMMR3DECL(int) PGMR3PhysMMIORegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb,
1810 R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnHandlerR3, RTR3PTR pvUserR3,
1811 R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnHandlerR0, RTR0PTR pvUserR0,
1812 RCPTRTYPE(PFNPGMRCPHYSHANDLER) pfnHandlerRC, RTRCPTR pvUserRC,
1813 R3PTRTYPE(const char *) pszDesc)
1814{
1815 /*
1816 * Assert on some assumption.
1817 */
1818 VM_ASSERT_EMT(pVM);
1819 AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
1820 AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
1821 AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
1822 AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
1823
1824 /*
1825 * Make sure there's a RAM range structure for the region.
1826 */
1827 int rc;
1828 RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
1829 bool fRamExists = false;
1830 PPGMRAMRANGE pRamPrev = NULL;
1831 PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
1832 while (pRam && GCPhysLast >= pRam->GCPhys)
1833 {
1834 if ( GCPhysLast >= pRam->GCPhys
1835 && GCPhys <= pRam->GCPhysLast)
1836 {
1837 /* Simplification: all within the same range. */
1838 AssertLogRelMsgReturn( GCPhys >= pRam->GCPhys
1839 && GCPhysLast <= pRam->GCPhysLast,
1840 ("%RGp-%RGp (MMIO/%s) falls partly outside %RGp-%RGp (%s)\n",
1841 GCPhys, GCPhysLast, pszDesc,
1842 pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
1843 VERR_PGM_RAM_CONFLICT);
1844
1845 /* Check that it's all RAM or MMIO pages. */
1846 PCPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
1847 uint32_t cLeft = cb >> PAGE_SHIFT;
1848 while (cLeft-- > 0)
1849 {
1850 AssertLogRelMsgReturn( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1851 || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO,
1852 ("%RGp-%RGp (MMIO/%s): %RGp is not a RAM or MMIO page - type=%d desc=%s\n",
1853 GCPhys, GCPhysLast, pszDesc, PGM_PAGE_GET_TYPE(pPage), pRam->pszDesc),
1854 VERR_PGM_RAM_CONFLICT);
1855 pPage++;
1856 }
1857
1858 /* Looks good. */
1859 fRamExists = true;
1860 break;
1861 }
1862
1863 /* next */
1864 pRamPrev = pRam;
1865 pRam = pRam->pNextR3;
1866 }
1867 PPGMRAMRANGE pNew;
1868 if (fRamExists)
1869 {
1870 pNew = NULL;
1871
1872 /*
1873 * Make all the pages in the range MMIO/ZERO pages, freeing any
1874 * RAM pages currently mapped here. This might not be 100% correct
1875 * for PCI memory, but we're doing the same thing for MMIO2 pages.
1876 */
1877 rc = pgmLock(pVM);
1878 if (RT_SUCCESS(rc))
1879 {
1880 rc = pgmR3PhysFreePageRange(pVM, pRam, GCPhys, GCPhysLast, PGMPAGETYPE_MMIO);
1881 pgmUnlock(pVM);
1882 }
1883 AssertRCReturn(rc, rc);
1884
1885 /* Force a PGM pool flush as guest ram references have been changed. */
1886 /** todo; not entirely SMP safe; assuming for now the guest takes care of this internally (not touch mapped mmio while changing the mapping). */
1887 PVMCPU pVCpu = VMMGetCpu(pVM);
1888 pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
1889 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
1890 }
1891 else
1892 {
1893 pgmLock(pVM);
1894
1895 /*
1896 * No RAM range, insert an ad hoc one.
1897 *
1898 * Note that we don't have to tell REM about this range because
1899 * PGMHandlerPhysicalRegisterEx will do that for us.
1900 */
1901 Log(("PGMR3PhysMMIORegister: Adding ad hoc MMIO range for %RGp-%RGp %s\n", GCPhys, GCPhysLast, pszDesc));
1902
1903 const uint32_t cPages = cb >> PAGE_SHIFT;
1904 const size_t cbRamRange = RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]);
1905 rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]), 16, MM_TAG_PGM_PHYS, (void **)&pNew);
1906 AssertLogRelMsgRCReturn(rc, ("cbRamRange=%zu\n", cbRamRange), rc);
1907
1908 /* Initialize the range. */
1909 pNew->pSelfR0 = MMHyperCCToR0(pVM, pNew);
1910 pNew->pSelfRC = MMHyperCCToRC(pVM, pNew);
1911 pNew->GCPhys = GCPhys;
1912 pNew->GCPhysLast = GCPhysLast;
1913 pNew->cb = cb;
1914 pNew->pszDesc = pszDesc;
1915 pNew->fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO;
1916 pNew->pvR3 = NULL;
1917 pNew->paLSPages = NULL;
1918
1919 uint32_t iPage = cPages;
1920 while (iPage-- > 0)
1921 PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_MMIO);
1922 Assert(PGM_PAGE_GET_TYPE(&pNew->aPages[0]) == PGMPAGETYPE_MMIO);
1923
1924 /* update the page count stats. */
1925 pVM->pgm.s.cPureMmioPages += cPages;
1926 pVM->pgm.s.cAllPages += cPages;
1927
1928 /* link it */
1929 pgmR3PhysLinkRamRange(pVM, pNew, pRamPrev);
1930
1931 pgmUnlock(pVM);
1932 }
1933
1934 /*
1935 * Register the access handler.
1936 */
1937 rc = PGMHandlerPhysicalRegisterEx(pVM, PGMPHYSHANDLERTYPE_MMIO, GCPhys, GCPhysLast,
1938 pfnHandlerR3, pvUserR3,
1939 pfnHandlerR0, pvUserR0,
1940 pfnHandlerRC, pvUserRC, pszDesc);
1941 if ( RT_FAILURE(rc)
1942 && !fRamExists)
1943 {
1944 pVM->pgm.s.cPureMmioPages -= cb >> PAGE_SHIFT;
1945 pVM->pgm.s.cAllPages -= cb >> PAGE_SHIFT;
1946
1947 /* remove the ad hoc range. */
1948 pgmR3PhysUnlinkRamRange2(pVM, pNew, pRamPrev);
1949 pNew->cb = pNew->GCPhys = pNew->GCPhysLast = NIL_RTGCPHYS;
1950 MMHyperFree(pVM, pRam);
1951 }
1952 PGMPhysInvalidatePageMapTLB(pVM);
1953
1954 return rc;
1955}
1956
1957
1958/**
1959 * This is the interface IOM is using to register an MMIO region.
1960 *
1961 * It will take care of calling PGMHandlerPhysicalDeregister and clean up
1962 * any ad hoc PGMRAMRANGE left behind.
1963 *
1964 * @returns VBox status code.
1965 * @param pVM Pointer to the shared VM structure.
1966 * @param GCPhys The start of the MMIO region.
1967 * @param cb The size of the MMIO region.
1968 */
1969VMMR3DECL(int) PGMR3PhysMMIODeregister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb)
1970{
1971 VM_ASSERT_EMT(pVM);
1972
1973 /*
1974 * First deregister the handler, then check if we should remove the ram range.
1975 */
1976 int rc = PGMHandlerPhysicalDeregister(pVM, GCPhys);
1977 if (RT_SUCCESS(rc))
1978 {
1979 RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
1980 PPGMRAMRANGE pRamPrev = NULL;
1981 PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
1982 while (pRam && GCPhysLast >= pRam->GCPhys)
1983 {
1984 /** @todo We're being a bit too careful here. rewrite. */
1985 if ( GCPhysLast == pRam->GCPhysLast
1986 && GCPhys == pRam->GCPhys)
1987 {
1988 Assert(pRam->cb == cb);
1989
1990 /*
1991 * See if all the pages are dead MMIO pages.
1992 */
1993 uint32_t const cPages = cb >> PAGE_SHIFT;
1994 bool fAllMMIO = true;
1995 uint32_t iPage = 0;
1996 uint32_t cLeft = cPages;
1997 while (cLeft-- > 0)
1998 {
1999 PPGMPAGE pPage = &pRam->aPages[iPage];
2000 if ( PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO
2001 /*|| not-out-of-action later */)
2002 {
2003 fAllMMIO = false;
2004 Assert(PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO2_ALIAS_MMIO);
2005 AssertMsgFailed(("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
2006 break;
2007 }
2008 Assert(PGM_PAGE_IS_ZERO(pPage));
2009 pPage++;
2010 }
2011 if (fAllMMIO)
2012 {
2013 /*
2014 * Ad-hoc range, unlink and free it.
2015 */
2016 Log(("PGMR3PhysMMIODeregister: Freeing ad hoc MMIO range for %RGp-%RGp %s\n",
2017 GCPhys, GCPhysLast, pRam->pszDesc));
2018
2019 pVM->pgm.s.cAllPages -= cPages;
2020 pVM->pgm.s.cPureMmioPages -= cPages;
2021
2022 pgmR3PhysUnlinkRamRange2(pVM, pRam, pRamPrev);
2023 pRam->cb = pRam->GCPhys = pRam->GCPhysLast = NIL_RTGCPHYS;
2024 MMHyperFree(pVM, pRam);
2025 break;
2026 }
2027 }
2028
2029 /*
2030 * Range match? It will all be within one range (see PGMAllHandler.cpp).
2031 */
2032 if ( GCPhysLast >= pRam->GCPhys
2033 && GCPhys <= pRam->GCPhysLast)
2034 {
2035 Assert(GCPhys >= pRam->GCPhys);
2036 Assert(GCPhysLast <= pRam->GCPhysLast);
2037
2038 /*
2039 * Turn the pages back into RAM pages.
2040 */
2041 uint32_t iPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2042 uint32_t cLeft = cb >> PAGE_SHIFT;
2043 while (cLeft--)
2044 {
2045 PPGMPAGE pPage = &pRam->aPages[iPage];
2046 AssertMsg(PGM_PAGE_IS_MMIO(pPage), ("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
2047 AssertMsg(PGM_PAGE_IS_ZERO(pPage), ("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
2048 if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO)
2049 PGM_PAGE_SET_TYPE(pPage, PGMPAGETYPE_RAM);
2050 }
2051 break;
2052 }
2053
2054 /* next */
2055 pRamPrev = pRam;
2056 pRam = pRam->pNextR3;
2057 }
2058 }
2059
2060 /* Force a PGM pool flush as guest ram references have been changed. */
2061 /** todo; not entirely SMP safe; assuming for now the guest takes care of this internally (not touch mapped mmio while changing the mapping). */
2062 PVMCPU pVCpu = VMMGetCpu(pVM);
2063 pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
2064 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2065
2066 PGMPhysInvalidatePageMapTLB(pVM);
2067 return rc;
2068}
2069
2070
2071/**
2072 * Locate a MMIO2 range.
2073 *
2074 * @returns Pointer to the MMIO2 range.
2075 * @param pVM Pointer to the shared VM structure.
2076 * @param pDevIns The device instance owning the region.
2077 * @param iRegion The region.
2078 */
2079DECLINLINE(PPGMMMIO2RANGE) pgmR3PhysMMIO2Find(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion)
2080{
2081 /*
2082 * Search the list.
2083 */
2084 for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
2085 if ( pCur->pDevInsR3 == pDevIns
2086 && pCur->iRegion == iRegion)
2087 return pCur;
2088 return NULL;
2089}
2090
2091
2092/**
2093 * Allocate and register an MMIO2 region.
2094 *
2095 * As mentioned elsewhere, MMIO2 is just RAM spelled differently. It's RAM
2096 * associated with a device. It is also non-shared memory with a permanent
2097 * ring-3 mapping and page backing (presently).
2098 *
2099 * A MMIO2 range may overlap with base memory if a lot of RAM is configured for
2100 * the VM, in which case we'll drop the base memory pages. Presently we will
2101 * make no attempt to preserve anything that happens to be present in the base
2102 * memory that is replaced, this is of course incorrectly but it's too much
2103 * effort.
2104 *
2105 * @returns VBox status code.
2106 * @retval VINF_SUCCESS on success, *ppv pointing to the R3 mapping of the
2107 * memory.
2108 * @retval VERR_ALREADY_EXISTS if the region already exists.
2109 *
2110 * @param pVM Pointer to the shared VM structure.
2111 * @param pDevIns The device instance owning the region.
2112 * @param iRegion The region number. If the MMIO2 memory is a PCI
2113 * I/O region this number has to be the number of that
2114 * region. Otherwise it can be any number safe
2115 * UINT8_MAX.
2116 * @param cb The size of the region. Must be page aligned.
2117 * @param fFlags Reserved for future use, must be zero.
2118 * @param ppv Where to store the pointer to the ring-3 mapping of
2119 * the memory.
2120 * @param pszDesc The description.
2121 */
2122VMMR3DECL(int) PGMR3PhysMMIO2Register(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS cb, uint32_t fFlags, void **ppv, const char *pszDesc)
2123{
2124 /*
2125 * Validate input.
2126 */
2127 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2128 AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2129 AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2130 AssertPtrReturn(ppv, VERR_INVALID_POINTER);
2131 AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
2132 AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
2133 AssertReturn(pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion) == NULL, VERR_ALREADY_EXISTS);
2134 AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2135 AssertReturn(cb, VERR_INVALID_PARAMETER);
2136 AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
2137
2138 const uint32_t cPages = cb >> PAGE_SHIFT;
2139 AssertLogRelReturn(((RTGCPHYS)cPages << PAGE_SHIFT) == cb, VERR_INVALID_PARAMETER);
2140 AssertLogRelReturn(cPages <= INT32_MAX / 2, VERR_NO_MEMORY);
2141
2142 /*
2143 * For the 2nd+ instance, mangle the description string so it's unique.
2144 */
2145 if (pDevIns->iInstance > 0) /** @todo Move to PDMDevHlp.cpp and use a real string cache. */
2146 {
2147 pszDesc = MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s [%u]", pszDesc, pDevIns->iInstance);
2148 if (!pszDesc)
2149 return VERR_NO_MEMORY;
2150 }
2151
2152 /*
2153 * Try reserve and allocate the backing memory first as this is what is
2154 * most likely to fail.
2155 */
2156 int rc = MMR3AdjustFixedReservation(pVM, cPages, pszDesc);
2157 if (RT_SUCCESS(rc))
2158 {
2159 void *pvPages;
2160 PSUPPAGE paPages = (PSUPPAGE)RTMemTmpAlloc(cPages * sizeof(SUPPAGE));
2161 if (RT_SUCCESS(rc))
2162 rc = SUPR3PageAllocEx(cPages, 0 /*fFlags*/, &pvPages, NULL /*pR0Ptr*/, paPages);
2163 if (RT_SUCCESS(rc))
2164 {
2165 memset(pvPages, 0, cPages * PAGE_SIZE);
2166
2167 /*
2168 * Create the MMIO2 range record for it.
2169 */
2170 const size_t cbRange = RT_OFFSETOF(PGMMMIO2RANGE, RamRange.aPages[cPages]);
2171 PPGMMMIO2RANGE pNew;
2172 rc = MMR3HyperAllocOnceNoRel(pVM, cbRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
2173 AssertLogRelMsgRC(rc, ("cbRamRange=%zu\n", cbRange));
2174 if (RT_SUCCESS(rc))
2175 {
2176 pNew->pDevInsR3 = pDevIns;
2177 pNew->pvR3 = pvPages;
2178 //pNew->pNext = NULL;
2179 //pNew->fMapped = false;
2180 //pNew->fOverlapping = false;
2181 pNew->iRegion = iRegion;
2182 pNew->idSavedState = UINT8_MAX;
2183 pNew->RamRange.pSelfR0 = MMHyperCCToR0(pVM, &pNew->RamRange);
2184 pNew->RamRange.pSelfRC = MMHyperCCToRC(pVM, &pNew->RamRange);
2185 pNew->RamRange.GCPhys = NIL_RTGCPHYS;
2186 pNew->RamRange.GCPhysLast = NIL_RTGCPHYS;
2187 pNew->RamRange.pszDesc = pszDesc;
2188 pNew->RamRange.cb = cb;
2189 pNew->RamRange.fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO2;
2190 pNew->RamRange.pvR3 = pvPages;
2191 //pNew->RamRange.paLSPages = NULL;
2192
2193 uint32_t iPage = cPages;
2194 while (iPage-- > 0)
2195 {
2196 PGM_PAGE_INIT(&pNew->RamRange.aPages[iPage],
2197 paPages[iPage].Phys, NIL_GMM_PAGEID,
2198 PGMPAGETYPE_MMIO2, PGM_PAGE_STATE_ALLOCATED);
2199 }
2200
2201 /* update page count stats */
2202 pVM->pgm.s.cAllPages += cPages;
2203 pVM->pgm.s.cPrivatePages += cPages;
2204
2205 /*
2206 * Link it into the list.
2207 * Since there is no particular order, just push it.
2208 */
2209 pgmLock(pVM);
2210 pNew->pNextR3 = pVM->pgm.s.pMmio2RangesR3;
2211 pVM->pgm.s.pMmio2RangesR3 = pNew;
2212 pgmUnlock(pVM);
2213
2214 *ppv = pvPages;
2215 RTMemTmpFree(paPages);
2216 PGMPhysInvalidatePageMapTLB(pVM);
2217 return VINF_SUCCESS;
2218 }
2219
2220 SUPR3PageFreeEx(pvPages, cPages);
2221 }
2222 RTMemTmpFree(paPages);
2223 MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pszDesc);
2224 }
2225 if (pDevIns->iInstance > 0)
2226 MMR3HeapFree((void *)pszDesc);
2227 return rc;
2228}
2229
2230
2231/**
2232 * Deregisters and frees an MMIO2 region.
2233 *
2234 * Any physical (and virtual) access handlers registered for the region must
2235 * be deregistered before calling this function.
2236 *
2237 * @returns VBox status code.
2238 * @param pVM Pointer to the shared VM structure.
2239 * @param pDevIns The device instance owning the region.
2240 * @param iRegion The region. If it's UINT32_MAX it'll be a wildcard match.
2241 */
2242VMMR3DECL(int) PGMR3PhysMMIO2Deregister(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion)
2243{
2244 /*
2245 * Validate input.
2246 */
2247 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2248 AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2249 AssertReturn(iRegion <= UINT8_MAX || iRegion == UINT32_MAX, VERR_INVALID_PARAMETER);
2250
2251 pgmLock(pVM);
2252 int rc = VINF_SUCCESS;
2253 unsigned cFound = 0;
2254 PPGMMMIO2RANGE pPrev = NULL;
2255 PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3;
2256 while (pCur)
2257 {
2258 if ( pCur->pDevInsR3 == pDevIns
2259 && ( iRegion == UINT32_MAX
2260 || pCur->iRegion == iRegion))
2261 {
2262 cFound++;
2263
2264 /*
2265 * Unmap it if it's mapped.
2266 */
2267 if (pCur->fMapped)
2268 {
2269 int rc2 = PGMR3PhysMMIO2Unmap(pVM, pCur->pDevInsR3, pCur->iRegion, pCur->RamRange.GCPhys);
2270 AssertRC(rc2);
2271 if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2272 rc = rc2;
2273 }
2274
2275 /*
2276 * Unlink it
2277 */
2278 PPGMMMIO2RANGE pNext = pCur->pNextR3;
2279 if (pPrev)
2280 pPrev->pNextR3 = pNext;
2281 else
2282 pVM->pgm.s.pMmio2RangesR3 = pNext;
2283 pCur->pNextR3 = NULL;
2284
2285 /*
2286 * Free the memory.
2287 */
2288 int rc2 = SUPR3PageFreeEx(pCur->pvR3, pCur->RamRange.cb >> PAGE_SHIFT);
2289 AssertRC(rc2);
2290 if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2291 rc = rc2;
2292
2293 uint32_t const cPages = pCur->RamRange.cb >> PAGE_SHIFT;
2294 rc2 = MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pCur->RamRange.pszDesc);
2295 AssertRC(rc2);
2296 if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2297 rc = rc2;
2298
2299 /* we're leaking hyper memory here if done at runtime. */
2300#ifdef VBOX_STRICT
2301 VMSTATE const enmState = VMR3GetState(pVM);
2302 AssertMsg( enmState == VMSTATE_POWERING_OFF
2303 || enmState == VMSTATE_POWERING_OFF_LS
2304 || enmState == VMSTATE_OFF
2305 || enmState == VMSTATE_OFF_LS
2306 || enmState == VMSTATE_DESTROYING
2307 || enmState == VMSTATE_TERMINATED
2308 || enmState == VMSTATE_CREATING
2309 , ("%s\n", VMR3GetStateName(enmState)));
2310#endif
2311 /*rc = MMHyperFree(pVM, pCur);
2312 AssertRCReturn(rc, rc); - not safe, see the alloc call. */
2313
2314
2315 /* update page count stats */
2316 pVM->pgm.s.cAllPages -= cPages;
2317 pVM->pgm.s.cPrivatePages -= cPages;
2318
2319 /* next */
2320 pCur = pNext;
2321 }
2322 else
2323 {
2324 pPrev = pCur;
2325 pCur = pCur->pNextR3;
2326 }
2327 }
2328 PGMPhysInvalidatePageMapTLB(pVM);
2329 pgmUnlock(pVM);
2330 return !cFound && iRegion != UINT32_MAX ? VERR_NOT_FOUND : rc;
2331}
2332
2333
2334/**
2335 * Maps a MMIO2 region.
2336 *
2337 * This is done when a guest / the bios / state loading changes the
2338 * PCI config. The replacing of base memory has the same restrictions
2339 * as during registration, of course.
2340 *
2341 * @returns VBox status code.
2342 *
2343 * @param pVM Pointer to the shared VM structure.
2344 * @param pDevIns The
2345 */
2346VMMR3DECL(int) PGMR3PhysMMIO2Map(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
2347{
2348 /*
2349 * Validate input
2350 */
2351 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2352 AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2353 AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2354 AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
2355 AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
2356 AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2357
2358 PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2359 AssertReturn(pCur, VERR_NOT_FOUND);
2360 AssertReturn(!pCur->fMapped, VERR_WRONG_ORDER);
2361 Assert(pCur->RamRange.GCPhys == NIL_RTGCPHYS);
2362 Assert(pCur->RamRange.GCPhysLast == NIL_RTGCPHYS);
2363
2364 const RTGCPHYS GCPhysLast = GCPhys + pCur->RamRange.cb - 1;
2365 AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
2366
2367 /*
2368 * Find our location in the ram range list, checking for
2369 * restriction we don't bother implementing yet (partially overlapping).
2370 */
2371 bool fRamExists = false;
2372 PPGMRAMRANGE pRamPrev = NULL;
2373 PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
2374 while (pRam && GCPhysLast >= pRam->GCPhys)
2375 {
2376 if ( GCPhys <= pRam->GCPhysLast
2377 && GCPhysLast >= pRam->GCPhys)
2378 {
2379 /* completely within? */
2380 AssertLogRelMsgReturn( GCPhys >= pRam->GCPhys
2381 && GCPhysLast <= pRam->GCPhysLast,
2382 ("%RGp-%RGp (MMIO2/%s) falls partly outside %RGp-%RGp (%s)\n",
2383 GCPhys, GCPhysLast, pCur->RamRange.pszDesc,
2384 pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
2385 VERR_PGM_RAM_CONFLICT);
2386 fRamExists = true;
2387 break;
2388 }
2389
2390 /* next */
2391 pRamPrev = pRam;
2392 pRam = pRam->pNextR3;
2393 }
2394 if (fRamExists)
2395 {
2396 PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2397 uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2398 while (cPagesLeft-- > 0)
2399 {
2400 AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
2401 ("%RGp isn't a RAM page (%d) - mapping %RGp-%RGp (MMIO2/%s).\n",
2402 GCPhys, PGM_PAGE_GET_TYPE(pPage), GCPhys, GCPhysLast, pCur->RamRange.pszDesc),
2403 VERR_PGM_RAM_CONFLICT);
2404 pPage++;
2405 }
2406 }
2407 Log(("PGMR3PhysMMIO2Map: %RGp-%RGp fRamExists=%RTbool %s\n",
2408 GCPhys, GCPhysLast, fRamExists, pCur->RamRange.pszDesc));
2409
2410 /*
2411 * Make the changes.
2412 */
2413 pgmLock(pVM);
2414
2415 pCur->RamRange.GCPhys = GCPhys;
2416 pCur->RamRange.GCPhysLast = GCPhysLast;
2417 pCur->fMapped = true;
2418 pCur->fOverlapping = fRamExists;
2419
2420 if (fRamExists)
2421 {
2422/** @todo use pgmR3PhysFreePageRange here. */
2423 uint32_t cPendingPages = 0;
2424 PGMMFREEPAGESREQ pReq;
2425 int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
2426 AssertLogRelRCReturn(rc, rc);
2427
2428 /* replace the pages, freeing all present RAM pages. */
2429 PPGMPAGE pPageSrc = &pCur->RamRange.aPages[0];
2430 PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2431 uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2432 while (cPagesLeft-- > 0)
2433 {
2434 rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys);
2435 AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
2436
2437 RTHCPHYS const HCPhys = PGM_PAGE_GET_HCPHYS(pPageSrc);
2438 PGM_PAGE_SET_HCPHYS(pPageDst, HCPhys);
2439 PGM_PAGE_SET_TYPE(pPageDst, PGMPAGETYPE_MMIO2);
2440 PGM_PAGE_SET_STATE(pPageDst, PGM_PAGE_STATE_ALLOCATED);
2441 PGM_PAGE_SET_PDE_TYPE(pPageDst, PGM_PAGE_PDE_TYPE_DONTCARE);
2442 PGM_PAGE_SET_PTE_INDEX(pPageDst, 0);
2443 PGM_PAGE_SET_TRACKING(pPageDst, 0);
2444
2445 pVM->pgm.s.cZeroPages--;
2446 GCPhys += PAGE_SIZE;
2447 pPageSrc++;
2448 pPageDst++;
2449 }
2450
2451 /* Flush physical page map TLB. */
2452 PGMPhysInvalidatePageMapTLB(pVM);
2453
2454 if (cPendingPages)
2455 {
2456 rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2457 AssertLogRelRCReturn(rc, rc);
2458 }
2459 GMMR3FreePagesCleanup(pReq);
2460
2461 /* Force a PGM pool flush as guest ram references have been changed. */
2462 /** todo; not entirely SMP safe; assuming for now the guest takes care of this internally (not touch mapped mmio while changing the mapping). */
2463 PVMCPU pVCpu = VMMGetCpu(pVM);
2464 pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
2465 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2466
2467 pgmUnlock(pVM);
2468 }
2469 else
2470 {
2471 RTGCPHYS cb = pCur->RamRange.cb;
2472
2473 /* Clear the tracking data of pages we're going to reactivate. */
2474 PPGMPAGE pPageSrc = &pCur->RamRange.aPages[0];
2475 uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2476 while (cPagesLeft-- > 0)
2477 {
2478 PGM_PAGE_SET_TRACKING(pPageSrc, 0);
2479 PGM_PAGE_SET_PTE_INDEX(pPageSrc, 0);
2480 pPageSrc++;
2481 }
2482
2483 /* link in the ram range */
2484 pgmR3PhysLinkRamRange(pVM, &pCur->RamRange, pRamPrev);
2485 pgmUnlock(pVM);
2486
2487 REMR3NotifyPhysRamRegister(pVM, GCPhys, cb, REM_NOTIFY_PHYS_RAM_FLAGS_MMIO2);
2488 }
2489
2490 PGMPhysInvalidatePageMapTLB(pVM);
2491 return VINF_SUCCESS;
2492}
2493
2494
2495/**
2496 * Unmaps a MMIO2 region.
2497 *
2498 * This is done when a guest / the bios / state loading changes the
2499 * PCI config. The replacing of base memory has the same restrictions
2500 * as during registration, of course.
2501 */
2502VMMR3DECL(int) PGMR3PhysMMIO2Unmap(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
2503{
2504 /*
2505 * Validate input
2506 */
2507 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2508 AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2509 AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2510 AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
2511 AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
2512 AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2513
2514 PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2515 AssertReturn(pCur, VERR_NOT_FOUND);
2516 AssertReturn(pCur->fMapped, VERR_WRONG_ORDER);
2517 AssertReturn(pCur->RamRange.GCPhys == GCPhys, VERR_INVALID_PARAMETER);
2518 Assert(pCur->RamRange.GCPhysLast != NIL_RTGCPHYS);
2519
2520 Log(("PGMR3PhysMMIO2Unmap: %RGp-%RGp %s\n",
2521 pCur->RamRange.GCPhys, pCur->RamRange.GCPhysLast, pCur->RamRange.pszDesc));
2522
2523 /*
2524 * Unmap it.
2525 */
2526 pgmLock(pVM);
2527
2528 RTGCPHYS GCPhysRangeREM;
2529 RTGCPHYS cbRangeREM;
2530 bool fInformREM;
2531 if (pCur->fOverlapping)
2532 {
2533 /* Restore the RAM pages we've replaced. */
2534 PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
2535 while (pRam->GCPhys > pCur->RamRange.GCPhysLast)
2536 pRam = pRam->pNextR3;
2537
2538 PPGMPAGE pPageDst = &pRam->aPages[(pCur->RamRange.GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2539 uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2540 while (cPagesLeft-- > 0)
2541 {
2542 PGM_PAGE_INIT_ZERO(pPageDst, pVM, PGMPAGETYPE_RAM);
2543 pVM->pgm.s.cZeroPages++;
2544 pPageDst++;
2545 }
2546
2547 /* Flush physical page map TLB. */
2548 PGMPhysInvalidatePageMapTLB(pVM);
2549
2550 GCPhysRangeREM = NIL_RTGCPHYS; /* shuts up gcc */
2551 cbRangeREM = RTGCPHYS_MAX; /* ditto */
2552 fInformREM = false;
2553 }
2554 else
2555 {
2556 GCPhysRangeREM = pCur->RamRange.GCPhys;
2557 cbRangeREM = pCur->RamRange.cb;
2558 fInformREM = true;
2559
2560 pgmR3PhysUnlinkRamRange(pVM, &pCur->RamRange);
2561 }
2562
2563 pCur->RamRange.GCPhys = NIL_RTGCPHYS;
2564 pCur->RamRange.GCPhysLast = NIL_RTGCPHYS;
2565 pCur->fOverlapping = false;
2566 pCur->fMapped = false;
2567
2568 /* Force a PGM pool flush as guest ram references have been changed. */
2569 /** todo; not entirely SMP safe; assuming for now the guest takes care of this internally (not touch mapped mmio while changing the mapping). */
2570 PVMCPU pVCpu = VMMGetCpu(pVM);
2571 pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
2572 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2573
2574 PGMPhysInvalidatePageMapTLB(pVM);
2575 pgmUnlock(pVM);
2576
2577 if (fInformREM)
2578 REMR3NotifyPhysRamDeregister(pVM, GCPhysRangeREM, cbRangeREM);
2579
2580 return VINF_SUCCESS;
2581}
2582
2583
2584/**
2585 * Checks if the given address is an MMIO2 base address or not.
2586 *
2587 * @returns true/false accordingly.
2588 * @param pVM Pointer to the shared VM structure.
2589 * @param pDevIns The owner of the memory, optional.
2590 * @param GCPhys The address to check.
2591 */
2592VMMR3DECL(bool) PGMR3PhysMMIO2IsBase(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys)
2593{
2594 /*
2595 * Validate input
2596 */
2597 VM_ASSERT_EMT_RETURN(pVM, false);
2598 AssertPtrReturn(pDevIns, false);
2599 AssertReturn(GCPhys != NIL_RTGCPHYS, false);
2600 AssertReturn(GCPhys != 0, false);
2601 AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), false);
2602
2603 /*
2604 * Search the list.
2605 */
2606 pgmLock(pVM);
2607 for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
2608 if (pCur->RamRange.GCPhys == GCPhys)
2609 {
2610 Assert(pCur->fMapped);
2611 pgmUnlock(pVM);
2612 return true;
2613 }
2614 pgmUnlock(pVM);
2615 return false;
2616}
2617
2618
2619/**
2620 * Gets the HC physical address of a page in the MMIO2 region.
2621 *
2622 * This is API is intended for MMHyper and shouldn't be called
2623 * by anyone else...
2624 *
2625 * @returns VBox status code.
2626 * @param pVM Pointer to the shared VM structure.
2627 * @param pDevIns The owner of the memory, optional.
2628 * @param iRegion The region.
2629 * @param off The page expressed an offset into the MMIO2 region.
2630 * @param pHCPhys Where to store the result.
2631 */
2632VMMR3DECL(int) PGMR3PhysMMIO2GetHCPhys(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, PRTHCPHYS pHCPhys)
2633{
2634 /*
2635 * Validate input
2636 */
2637 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2638 AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2639 AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2640
2641 pgmLock(pVM);
2642 PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2643 AssertReturn(pCur, VERR_NOT_FOUND);
2644 AssertReturn(off < pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2645
2646 PCPGMPAGE pPage = &pCur->RamRange.aPages[off >> PAGE_SHIFT];
2647 *pHCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2648 pgmUnlock(pVM);
2649 return VINF_SUCCESS;
2650}
2651
2652
2653/**
2654 * Maps a portion of an MMIO2 region into kernel space (host).
2655 *
2656 * The kernel mapping will become invalid when the MMIO2 memory is deregistered
2657 * or the VM is terminated.
2658 *
2659 * @return VBox status code.
2660 *
2661 * @param pVM Pointer to the shared VM structure.
2662 * @param pDevIns The device owning the MMIO2 memory.
2663 * @param iRegion The region.
2664 * @param off The offset into the region. Must be page aligned.
2665 * @param cb The number of bytes to map. Must be page aligned.
2666 * @param pszDesc Mapping description.
2667 * @param pR0Ptr Where to store the R0 address.
2668 */
2669VMMR3DECL(int) PGMR3PhysMMIO2MapKernel(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, RTGCPHYS cb,
2670 const char *pszDesc, PRTR0PTR pR0Ptr)
2671{
2672 /*
2673 * Validate input.
2674 */
2675 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2676 AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2677 AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2678
2679 PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2680 AssertReturn(pCur, VERR_NOT_FOUND);
2681 AssertReturn(off < pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2682 AssertReturn(cb <= pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2683 AssertReturn(off + cb <= pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2684
2685 /*
2686 * Pass the request on to the support library/driver.
2687 */
2688 int rc = SUPR3PageMapKernel(pCur->pvR3, off, cb, 0, pR0Ptr);
2689
2690 return rc;
2691}
2692
2693
2694/**
2695 * Registers a ROM image.
2696 *
2697 * Shadowed ROM images requires double the amount of backing memory, so,
2698 * don't use that unless you have to. Shadowing of ROM images is process
2699 * where we can select where the reads go and where the writes go. On real
2700 * hardware the chipset provides means to configure this. We provide
2701 * PGMR3PhysProtectROM() for this purpose.
2702 *
2703 * A read-only copy of the ROM image will always be kept around while we
2704 * will allocate RAM pages for the changes on demand (unless all memory
2705 * is configured to be preallocated).
2706 *
2707 * @returns VBox status.
2708 * @param pVM VM Handle.
2709 * @param pDevIns The device instance owning the ROM.
2710 * @param GCPhys First physical address in the range.
2711 * Must be page aligned!
2712 * @param cbRange The size of the range (in bytes).
2713 * Must be page aligned!
2714 * @param pvBinary Pointer to the binary data backing the ROM image.
2715 * This must be exactly \a cbRange in size.
2716 * @param fFlags Mask of flags. PGMPHYS_ROM_FLAGS_SHADOWED
2717 * and/or PGMPHYS_ROM_FLAGS_PERMANENT_BINARY.
2718 * @param pszDesc Pointer to description string. This must not be freed.
2719 *
2720 * @remark There is no way to remove the rom, automatically on device cleanup or
2721 * manually from the device yet. This isn't difficult in any way, it's
2722 * just not something we expect to be necessary for a while.
2723 */
2724VMMR3DECL(int) PGMR3PhysRomRegister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
2725 const void *pvBinary, uint32_t fFlags, const char *pszDesc)
2726{
2727 Log(("PGMR3PhysRomRegister: pDevIns=%p GCPhys=%RGp(-%RGp) cb=%RGp pvBinary=%p fFlags=%#x pszDesc=%s\n",
2728 pDevIns, GCPhys, GCPhys + cb, cb, pvBinary, fFlags, pszDesc));
2729
2730 /*
2731 * Validate input.
2732 */
2733 AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2734 AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
2735 AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
2736 RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
2737 AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
2738 AssertPtrReturn(pvBinary, VERR_INVALID_PARAMETER);
2739 AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
2740 AssertReturn(!(fFlags & ~(PGMPHYS_ROM_FLAGS_SHADOWED | PGMPHYS_ROM_FLAGS_PERMANENT_BINARY)), VERR_INVALID_PARAMETER);
2741 VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
2742
2743 const uint32_t cPages = cb >> PAGE_SHIFT;
2744
2745 /*
2746 * Find the ROM location in the ROM list first.
2747 */
2748 PPGMROMRANGE pRomPrev = NULL;
2749 PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3;
2750 while (pRom && GCPhysLast >= pRom->GCPhys)
2751 {
2752 if ( GCPhys <= pRom->GCPhysLast
2753 && GCPhysLast >= pRom->GCPhys)
2754 AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
2755 GCPhys, GCPhysLast, pszDesc,
2756 pRom->GCPhys, pRom->GCPhysLast, pRom->pszDesc),
2757 VERR_PGM_RAM_CONFLICT);
2758 /* next */
2759 pRomPrev = pRom;
2760 pRom = pRom->pNextR3;
2761 }
2762
2763 /*
2764 * Find the RAM location and check for conflicts.
2765 *
2766 * Conflict detection is a bit different than for RAM
2767 * registration since a ROM can be located within a RAM
2768 * range. So, what we have to check for is other memory
2769 * types (other than RAM that is) and that we don't span
2770 * more than one RAM range (layz).
2771 */
2772 bool fRamExists = false;
2773 PPGMRAMRANGE pRamPrev = NULL;
2774 PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
2775 while (pRam && GCPhysLast >= pRam->GCPhys)
2776 {
2777 if ( GCPhys <= pRam->GCPhysLast
2778 && GCPhysLast >= pRam->GCPhys)
2779 {
2780 /* completely within? */
2781 AssertLogRelMsgReturn( GCPhys >= pRam->GCPhys
2782 && GCPhysLast <= pRam->GCPhysLast,
2783 ("%RGp-%RGp (%s) falls partly outside %RGp-%RGp (%s)\n",
2784 GCPhys, GCPhysLast, pszDesc,
2785 pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
2786 VERR_PGM_RAM_CONFLICT);
2787 fRamExists = true;
2788 break;
2789 }
2790
2791 /* next */
2792 pRamPrev = pRam;
2793 pRam = pRam->pNextR3;
2794 }
2795 if (fRamExists)
2796 {
2797 PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2798 uint32_t cPagesLeft = cPages;
2799 while (cPagesLeft-- > 0)
2800 {
2801 AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
2802 ("%RGp (%R[pgmpage]) isn't a RAM page - registering %RGp-%RGp (%s).\n",
2803 pRam->GCPhys + ((RTGCPHYS)(uintptr_t)(pPage - &pRam->aPages[0]) << PAGE_SHIFT),
2804 pPage, GCPhys, GCPhysLast, pszDesc), VERR_PGM_RAM_CONFLICT);
2805 Assert(PGM_PAGE_IS_ZERO(pPage));
2806 pPage++;
2807 }
2808 }
2809
2810 /*
2811 * Update the base memory reservation if necessary.
2812 */
2813 uint32_t cExtraBaseCost = fRamExists ? 0 : cPages;
2814 if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
2815 cExtraBaseCost += cPages;
2816 if (cExtraBaseCost)
2817 {
2818 int rc = MMR3IncreaseBaseReservation(pVM, cExtraBaseCost);
2819 if (RT_FAILURE(rc))
2820 return rc;
2821 }
2822
2823 /*
2824 * Allocate memory for the virgin copy of the RAM.
2825 */
2826 PGMMALLOCATEPAGESREQ pReq;
2827 int rc = GMMR3AllocatePagesPrepare(pVM, &pReq, cPages, GMMACCOUNT_BASE);
2828 AssertRCReturn(rc, rc);
2829
2830 for (uint32_t iPage = 0; iPage < cPages; iPage++)
2831 {
2832 pReq->aPages[iPage].HCPhysGCPhys = GCPhys + (iPage << PAGE_SHIFT);
2833 pReq->aPages[iPage].idPage = NIL_GMM_PAGEID;
2834 pReq->aPages[iPage].idSharedPage = NIL_GMM_PAGEID;
2835 }
2836
2837 pgmLock(pVM);
2838 rc = GMMR3AllocatePagesPerform(pVM, pReq);
2839 pgmUnlock(pVM);
2840 if (RT_FAILURE(rc))
2841 {
2842 GMMR3AllocatePagesCleanup(pReq);
2843 return rc;
2844 }
2845
2846 /*
2847 * Allocate the new ROM range and RAM range (if necessary).
2848 */
2849 PPGMROMRANGE pRomNew;
2850 rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMROMRANGE, aPages[cPages]), 0, MM_TAG_PGM_PHYS, (void **)&pRomNew);
2851 if (RT_SUCCESS(rc))
2852 {
2853 PPGMRAMRANGE pRamNew = NULL;
2854 if (!fRamExists)
2855 rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]), sizeof(PGMPAGE), MM_TAG_PGM_PHYS, (void **)&pRamNew);
2856 if (RT_SUCCESS(rc))
2857 {
2858 pgmLock(pVM);
2859
2860 /*
2861 * Initialize and insert the RAM range (if required).
2862 */
2863 PPGMROMPAGE pRomPage = &pRomNew->aPages[0];
2864 if (!fRamExists)
2865 {
2866 pRamNew->pSelfR0 = MMHyperCCToR0(pVM, pRamNew);
2867 pRamNew->pSelfRC = MMHyperCCToRC(pVM, pRamNew);
2868 pRamNew->GCPhys = GCPhys;
2869 pRamNew->GCPhysLast = GCPhysLast;
2870 pRamNew->cb = cb;
2871 pRamNew->pszDesc = pszDesc;
2872 pRamNew->fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_ROM;
2873 pRamNew->pvR3 = NULL;
2874 pRamNew->paLSPages = NULL;
2875
2876 PPGMPAGE pPage = &pRamNew->aPages[0];
2877 for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
2878 {
2879 PGM_PAGE_INIT(pPage,
2880 pReq->aPages[iPage].HCPhysGCPhys,
2881 pReq->aPages[iPage].idPage,
2882 PGMPAGETYPE_ROM,
2883 PGM_PAGE_STATE_ALLOCATED);
2884
2885 pRomPage->Virgin = *pPage;
2886 }
2887
2888 pVM->pgm.s.cAllPages += cPages;
2889 pgmR3PhysLinkRamRange(pVM, pRamNew, pRamPrev);
2890 }
2891 else
2892 {
2893 PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2894 for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
2895 {
2896 PGM_PAGE_SET_TYPE(pPage, PGMPAGETYPE_ROM);
2897 PGM_PAGE_SET_HCPHYS(pPage, pReq->aPages[iPage].HCPhysGCPhys);
2898 PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ALLOCATED);
2899 PGM_PAGE_SET_PAGEID(pPage, pReq->aPages[iPage].idPage);
2900 PGM_PAGE_SET_PDE_TYPE(pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
2901 PGM_PAGE_SET_PTE_INDEX(pPage, 0);
2902 PGM_PAGE_SET_TRACKING(pPage, 0);
2903
2904 pRomPage->Virgin = *pPage;
2905 }
2906
2907 pRamNew = pRam;
2908
2909 pVM->pgm.s.cZeroPages -= cPages;
2910 }
2911 pVM->pgm.s.cPrivatePages += cPages;
2912
2913 /* Flush physical page map TLB. */
2914 PGMPhysInvalidatePageMapTLB(pVM);
2915
2916 pgmUnlock(pVM);
2917
2918
2919 /*
2920 * !HACK ALERT! REM + (Shadowed) ROM ==> mess.
2921 *
2922 * If it's shadowed we'll register the handler after the ROM notification
2923 * so we get the access handler callbacks that we should. If it isn't
2924 * shadowed we'll do it the other way around to make REM use the built-in
2925 * ROM behavior and not the handler behavior (which is to route all access
2926 * to PGM atm).
2927 */
2928 if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
2929 {
2930 REMR3NotifyPhysRomRegister(pVM, GCPhys, cb, NULL, true /* fShadowed */);
2931 rc = PGMR3HandlerPhysicalRegister(pVM,
2932 fFlags & PGMPHYS_ROM_FLAGS_SHADOWED
2933 ? PGMPHYSHANDLERTYPE_PHYSICAL_ALL
2934 : PGMPHYSHANDLERTYPE_PHYSICAL_WRITE,
2935 GCPhys, GCPhysLast,
2936 pgmR3PhysRomWriteHandler, pRomNew,
2937 NULL, "pgmPhysRomWriteHandler", MMHyperCCToR0(pVM, pRomNew),
2938 NULL, "pgmPhysRomWriteHandler", MMHyperCCToRC(pVM, pRomNew), pszDesc);
2939 }
2940 else
2941 {
2942 rc = PGMR3HandlerPhysicalRegister(pVM,
2943 fFlags & PGMPHYS_ROM_FLAGS_SHADOWED
2944 ? PGMPHYSHANDLERTYPE_PHYSICAL_ALL
2945 : PGMPHYSHANDLERTYPE_PHYSICAL_WRITE,
2946 GCPhys, GCPhysLast,
2947 pgmR3PhysRomWriteHandler, pRomNew,
2948 NULL, "pgmPhysRomWriteHandler", MMHyperCCToR0(pVM, pRomNew),
2949 NULL, "pgmPhysRomWriteHandler", MMHyperCCToRC(pVM, pRomNew), pszDesc);
2950 REMR3NotifyPhysRomRegister(pVM, GCPhys, cb, NULL, false /* fShadowed */);
2951 }
2952 if (RT_SUCCESS(rc))
2953 {
2954 pgmLock(pVM);
2955
2956 /*
2957 * Copy the image over to the virgin pages.
2958 * This must be done after linking in the RAM range.
2959 */
2960 PPGMPAGE pRamPage = &pRamNew->aPages[(GCPhys - pRamNew->GCPhys) >> PAGE_SHIFT];
2961 for (uint32_t iPage = 0; iPage < cPages; iPage++, pRamPage++)
2962 {
2963 void *pvDstPage;
2964 rc = pgmPhysPageMap(pVM, pRamPage, GCPhys + (iPage << PAGE_SHIFT), &pvDstPage);
2965 if (RT_FAILURE(rc))
2966 {
2967 VMSetError(pVM, rc, RT_SRC_POS, "Failed to map virgin ROM page at %RGp", GCPhys);
2968 break;
2969 }
2970 memcpy(pvDstPage, (const uint8_t *)pvBinary + (iPage << PAGE_SHIFT), PAGE_SIZE);
2971 }
2972 if (RT_SUCCESS(rc))
2973 {
2974 /*
2975 * Initialize the ROM range.
2976 * Note that the Virgin member of the pages has already been initialized above.
2977 */
2978 pRomNew->GCPhys = GCPhys;
2979 pRomNew->GCPhysLast = GCPhysLast;
2980 pRomNew->cb = cb;
2981 pRomNew->fFlags = fFlags;
2982 pRomNew->idSavedState = UINT8_MAX;
2983#ifdef VBOX_STRICT
2984 pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY
2985 ? pvBinary : RTMemDup(pvBinary, cPages * PAGE_SIZE);
2986#else
2987 pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY ? pvBinary : NULL;
2988#endif
2989 pRomNew->pszDesc = pszDesc;
2990
2991 for (unsigned iPage = 0; iPage < cPages; iPage++)
2992 {
2993 PPGMROMPAGE pPage = &pRomNew->aPages[iPage];
2994 pPage->enmProt = PGMROMPROT_READ_ROM_WRITE_IGNORE;
2995 PGM_PAGE_INIT_ZERO(&pPage->Shadow, pVM, PGMPAGETYPE_ROM_SHADOW);
2996 }
2997
2998 /* update the page count stats for the shadow pages. */
2999 if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3000 {
3001 pVM->pgm.s.cZeroPages += cPages;
3002 pVM->pgm.s.cAllPages += cPages;
3003 }
3004
3005 /*
3006 * Insert the ROM range, tell REM and return successfully.
3007 */
3008 pRomNew->pNextR3 = pRom;
3009 pRomNew->pNextR0 = pRom ? MMHyperCCToR0(pVM, pRom) : NIL_RTR0PTR;
3010 pRomNew->pNextRC = pRom ? MMHyperCCToRC(pVM, pRom) : NIL_RTRCPTR;
3011
3012 if (pRomPrev)
3013 {
3014 pRomPrev->pNextR3 = pRomNew;
3015 pRomPrev->pNextR0 = MMHyperCCToR0(pVM, pRomNew);
3016 pRomPrev->pNextRC = MMHyperCCToRC(pVM, pRomNew);
3017 }
3018 else
3019 {
3020 pVM->pgm.s.pRomRangesR3 = pRomNew;
3021 pVM->pgm.s.pRomRangesR0 = MMHyperCCToR0(pVM, pRomNew);
3022 pVM->pgm.s.pRomRangesRC = MMHyperCCToRC(pVM, pRomNew);
3023 }
3024
3025 PGMPhysInvalidatePageMapTLB(pVM);
3026 GMMR3AllocatePagesCleanup(pReq);
3027 pgmUnlock(pVM);
3028 return VINF_SUCCESS;
3029 }
3030
3031 /* bail out */
3032
3033 pgmUnlock(pVM);
3034 int rc2 = PGMHandlerPhysicalDeregister(pVM, GCPhys);
3035 AssertRC(rc2);
3036 pgmLock(pVM);
3037 }
3038
3039 if (!fRamExists)
3040 {
3041 pgmR3PhysUnlinkRamRange2(pVM, pRamNew, pRamPrev);
3042 MMHyperFree(pVM, pRamNew);
3043 }
3044 }
3045 MMHyperFree(pVM, pRomNew);
3046 }
3047
3048 /** @todo Purge the mapping cache or something... */
3049 GMMR3FreeAllocatedPages(pVM, pReq);
3050 GMMR3AllocatePagesCleanup(pReq);
3051 pgmUnlock(pVM);
3052 return rc;
3053}
3054
3055
3056/**
3057 * \#PF Handler callback for ROM write accesses.
3058 *
3059 * @returns VINF_SUCCESS if the handler have carried out the operation.
3060 * @returns VINF_PGM_HANDLER_DO_DEFAULT if the caller should carry out the access operation.
3061 * @param pVM VM Handle.
3062 * @param GCPhys The physical address the guest is writing to.
3063 * @param pvPhys The HC mapping of that address.
3064 * @param pvBuf What the guest is reading/writing.
3065 * @param cbBuf How much it's reading/writing.
3066 * @param enmAccessType The access type.
3067 * @param pvUser User argument.
3068 */
3069static DECLCALLBACK(int) pgmR3PhysRomWriteHandler(PVM pVM, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf,
3070 PGMACCESSTYPE enmAccessType, void *pvUser)
3071{
3072 PPGMROMRANGE pRom = (PPGMROMRANGE)pvUser;
3073 const uint32_t iPage = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
3074 Assert(iPage < (pRom->cb >> PAGE_SHIFT));
3075 PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
3076 Log5(("pgmR3PhysRomWriteHandler: %d %c %#08RGp %#04zx\n", pRomPage->enmProt, enmAccessType == PGMACCESSTYPE_READ ? 'R' : 'W', GCPhys, cbBuf));
3077
3078 if (enmAccessType == PGMACCESSTYPE_READ)
3079 {
3080 switch (pRomPage->enmProt)
3081 {
3082 /*
3083 * Take the default action.
3084 */
3085 case PGMROMPROT_READ_ROM_WRITE_IGNORE:
3086 case PGMROMPROT_READ_RAM_WRITE_IGNORE:
3087 case PGMROMPROT_READ_ROM_WRITE_RAM:
3088 case PGMROMPROT_READ_RAM_WRITE_RAM:
3089 return VINF_PGM_HANDLER_DO_DEFAULT;
3090
3091 default:
3092 AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
3093 pRom->aPages[iPage].enmProt, iPage, GCPhys),
3094 VERR_INTERNAL_ERROR);
3095 }
3096 }
3097 else
3098 {
3099 Assert(enmAccessType == PGMACCESSTYPE_WRITE);
3100 switch (pRomPage->enmProt)
3101 {
3102 /*
3103 * Ignore writes.
3104 */
3105 case PGMROMPROT_READ_ROM_WRITE_IGNORE:
3106 case PGMROMPROT_READ_RAM_WRITE_IGNORE:
3107 return VINF_SUCCESS;
3108
3109 /*
3110 * Write to the RAM page.
3111 */
3112 case PGMROMPROT_READ_ROM_WRITE_RAM:
3113 case PGMROMPROT_READ_RAM_WRITE_RAM: /* yes this will get here too, it's *way* simpler that way. */
3114 {
3115 /* This should be impossible now, pvPhys doesn't work cross page anylonger. */
3116 Assert(((GCPhys - pRom->GCPhys + cbBuf - 1) >> PAGE_SHIFT) == iPage);
3117
3118 /*
3119 * Take the lock, do lazy allocation, map the page and copy the data.
3120 *
3121 * Note that we have to bypass the mapping TLB since it works on
3122 * guest physical addresses and entering the shadow page would
3123 * kind of screw things up...
3124 */
3125 int rc = pgmLock(pVM);
3126 AssertRC(rc);
3127
3128 PPGMPAGE pShadowPage = &pRomPage->Shadow;
3129 if (!PGMROMPROT_IS_ROM(pRomPage->enmProt))
3130 {
3131 pShadowPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
3132 AssertLogRelReturn(pShadowPage, VERR_INTERNAL_ERROR);
3133 }
3134
3135 void *pvDstPage;
3136 rc = pgmPhysPageMakeWritableAndMap(pVM, pShadowPage, GCPhys & X86_PTE_PG_MASK, &pvDstPage);
3137 if (RT_SUCCESS(rc))
3138 {
3139 memcpy((uint8_t *)pvDstPage + (GCPhys & PAGE_OFFSET_MASK), pvBuf, cbBuf);
3140 pRomPage->LiveSave.fWrittenTo = true;
3141 }
3142
3143 pgmUnlock(pVM);
3144 return rc;
3145 }
3146
3147 default:
3148 AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
3149 pRom->aPages[iPage].enmProt, iPage, GCPhys),
3150 VERR_INTERNAL_ERROR);
3151 }
3152 }
3153}
3154
3155
3156/**
3157 * Called by PGMR3Reset to reset the shadow, switch to the virgin,
3158 * and verify that the virgin part is untouched.
3159 *
3160 * This is done after the normal memory has been cleared.
3161 *
3162 * ASSUMES that the caller owns the PGM lock.
3163 *
3164 * @param pVM The VM handle.
3165 */
3166int pgmR3PhysRomReset(PVM pVM)
3167{
3168 Assert(PGMIsLockOwner(pVM));
3169 for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3170 {
3171 const uint32_t cPages = pRom->cb >> PAGE_SHIFT;
3172
3173 if (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3174 {
3175 /*
3176 * Reset the physical handler.
3177 */
3178 int rc = PGMR3PhysRomProtect(pVM, pRom->GCPhys, pRom->cb, PGMROMPROT_READ_ROM_WRITE_IGNORE);
3179 AssertRCReturn(rc, rc);
3180
3181 /*
3182 * What we do with the shadow pages depends on the memory
3183 * preallocation option. If not enabled, we'll just throw
3184 * out all the dirty pages and replace them by the zero page.
3185 */
3186 if (!pVM->pgm.s.fRamPreAlloc)
3187 {
3188 /* Free the dirty pages. */
3189 uint32_t cPendingPages = 0;
3190 PGMMFREEPAGESREQ pReq;
3191 rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
3192 AssertRCReturn(rc, rc);
3193
3194 for (uint32_t iPage = 0; iPage < cPages; iPage++)
3195 if ( !PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow)
3196 && !PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow))
3197 {
3198 Assert(PGM_PAGE_GET_STATE(&pRom->aPages[iPage].Shadow) == PGM_PAGE_STATE_ALLOCATED);
3199 rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, &pRom->aPages[iPage].Shadow,
3200 pRom->GCPhys + (iPage << PAGE_SHIFT));
3201 AssertLogRelRCReturn(rc, rc);
3202 }
3203
3204 if (cPendingPages)
3205 {
3206 rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
3207 AssertLogRelRCReturn(rc, rc);
3208 }
3209 GMMR3FreePagesCleanup(pReq);
3210 }
3211 else
3212 {
3213 /* clear all the shadow pages. */
3214 for (uint32_t iPage = 0; iPage < cPages; iPage++)
3215 {
3216 Assert(!PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow));
3217 Assert(!PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow));
3218 void *pvDstPage;
3219 const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
3220 rc = pgmPhysPageMakeWritableAndMap(pVM, &pRom->aPages[iPage].Shadow, GCPhys, &pvDstPage);
3221 if (RT_FAILURE(rc))
3222 break;
3223 ASMMemZeroPage(pvDstPage);
3224 }
3225 AssertRCReturn(rc, rc);
3226 }
3227 }
3228
3229#ifdef VBOX_STRICT
3230 /*
3231 * Verify that the virgin page is unchanged if possible.
3232 */
3233 if (pRom->pvOriginal)
3234 {
3235 uint8_t const *pbSrcPage = (uint8_t const *)pRom->pvOriginal;
3236 for (uint32_t iPage = 0; iPage < cPages; iPage++, pbSrcPage += PAGE_SIZE)
3237 {
3238 const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
3239 void const *pvDstPage;
3240 int rc = pgmPhysPageMapReadOnly(pVM, &pRom->aPages[iPage].Virgin, GCPhys, &pvDstPage);
3241 if (RT_FAILURE(rc))
3242 break;
3243 if (memcmp(pvDstPage, pbSrcPage, PAGE_SIZE))
3244 LogRel(("pgmR3PhysRomReset: %RGp rom page changed (%s) - loaded saved state?\n",
3245 GCPhys, pRom->pszDesc));
3246 }
3247 }
3248#endif
3249 }
3250
3251 return VINF_SUCCESS;
3252}
3253
3254
3255/**
3256 * Called by PGMR3Term to free resources.
3257 *
3258 * ASSUMES that the caller owns the PGM lock.
3259 *
3260 * @param pVM The VM handle.
3261 */
3262void pgmR3PhysRomTerm(PVM pVM)
3263{
3264#ifdef RT_STRICT
3265 /*
3266 * Free the heap copy of the original bits.
3267 */
3268 for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3269 {
3270 if ( pRom->pvOriginal
3271 && !(pRom->fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY))
3272 {
3273 RTMemFree((void *)pRom->pvOriginal);
3274 pRom->pvOriginal = NULL;
3275 }
3276 }
3277#endif
3278}
3279
3280
3281/**
3282 * Change the shadowing of a range of ROM pages.
3283 *
3284 * This is intended for implementing chipset specific memory registers
3285 * and will not be very strict about the input. It will silently ignore
3286 * any pages that are not the part of a shadowed ROM.
3287 *
3288 * @returns VBox status code.
3289 * @retval VINF_PGM_SYNC_CR3
3290 *
3291 * @param pVM Pointer to the shared VM structure.
3292 * @param GCPhys Where to start. Page aligned.
3293 * @param cb How much to change. Page aligned.
3294 * @param enmProt The new ROM protection.
3295 */
3296VMMR3DECL(int) PGMR3PhysRomProtect(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, PGMROMPROT enmProt)
3297{
3298 /*
3299 * Check input
3300 */
3301 if (!cb)
3302 return VINF_SUCCESS;
3303 AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3304 AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3305 RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
3306 AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
3307 AssertReturn(enmProt >= PGMROMPROT_INVALID && enmProt <= PGMROMPROT_END, VERR_INVALID_PARAMETER);
3308
3309 /*
3310 * Process the request.
3311 */
3312 pgmLock(pVM);
3313 int rc = VINF_SUCCESS;
3314 bool fFlushTLB = false;
3315 for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3316 {
3317 if ( GCPhys <= pRom->GCPhysLast
3318 && GCPhysLast >= pRom->GCPhys
3319 && (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED))
3320 {
3321 /*
3322 * Iterate the relevant pages and make necessary the changes.
3323 */
3324 bool fChanges = false;
3325 uint32_t const cPages = pRom->GCPhysLast <= GCPhysLast
3326 ? pRom->cb >> PAGE_SHIFT
3327 : (GCPhysLast - pRom->GCPhys + 1) >> PAGE_SHIFT;
3328 for (uint32_t iPage = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
3329 iPage < cPages;
3330 iPage++)
3331 {
3332 PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
3333 if (PGMROMPROT_IS_ROM(pRomPage->enmProt) != PGMROMPROT_IS_ROM(enmProt))
3334 {
3335 fChanges = true;
3336
3337 /* flush references to the page. */
3338 PPGMPAGE pRamPage = pgmPhysGetPage(&pVM->pgm.s, pRom->GCPhys + (iPage << PAGE_SHIFT));
3339 int rc2 = pgmPoolTrackUpdateGCPhys(pVM, pRom->GCPhys + (iPage << PAGE_SHIFT), pRamPage,
3340 true /*fFlushPTEs*/, &fFlushTLB);
3341 if (rc2 != VINF_SUCCESS && (rc == VINF_SUCCESS || RT_FAILURE(rc2)))
3342 rc = rc2;
3343
3344 PPGMPAGE pOld = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Virgin : &pRomPage->Shadow;
3345 PPGMPAGE pNew = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Shadow : &pRomPage->Virgin;
3346
3347 *pOld = *pRamPage;
3348 *pRamPage = *pNew;
3349 /** @todo preserve the volatile flags (handlers) when these have been moved out of HCPhys! */
3350 }
3351 pRomPage->enmProt = enmProt;
3352 }
3353
3354 /*
3355 * Reset the access handler if we made changes, no need
3356 * to optimize this.
3357 */
3358 if (fChanges)
3359 {
3360 int rc2 = PGMHandlerPhysicalReset(pVM, pRom->GCPhys);
3361 if (RT_FAILURE(rc2))
3362 {
3363 pgmUnlock(pVM);
3364 AssertRC(rc);
3365 return rc2;
3366 }
3367 }
3368
3369 /* Advance - cb isn't updated. */
3370 GCPhys = pRom->GCPhys + (cPages << PAGE_SHIFT);
3371 }
3372 }
3373 pgmUnlock(pVM);
3374 if (fFlushTLB)
3375 PGM_INVL_ALL_VCPU_TLBS(pVM);
3376
3377 return rc;
3378}
3379
3380
3381/**
3382 * Sets the Address Gate 20 state.
3383 *
3384 * @param pVCpu The VCPU to operate on.
3385 * @param fEnable True if the gate should be enabled.
3386 * False if the gate should be disabled.
3387 */
3388VMMDECL(void) PGMR3PhysSetA20(PVMCPU pVCpu, bool fEnable)
3389{
3390 LogFlow(("PGMR3PhysSetA20 %d (was %d)\n", fEnable, pVCpu->pgm.s.fA20Enabled));
3391 if (pVCpu->pgm.s.fA20Enabled != fEnable)
3392 {
3393 pVCpu->pgm.s.fA20Enabled = fEnable;
3394 pVCpu->pgm.s.GCPhysA20Mask = ~(RTGCPHYS)(!fEnable << 20);
3395 REMR3A20Set(pVCpu->pVMR3, pVCpu, fEnable);
3396 /** @todo we're not handling this correctly for VT-x / AMD-V. See #2911 */
3397 }
3398}
3399
3400#ifdef PGM_WITH_LARGE_ADDRESS_SPACE_ON_32_BIT_HOST
3401/**
3402 * Tree enumeration callback for dealing with age rollover.
3403 * It will perform a simple compression of the current age.
3404 */
3405static DECLCALLBACK(int) pgmR3PhysChunkAgeingRolloverCallback(PAVLU32NODECORE pNode, void *pvUser)
3406{
3407 Assert(PGMIsLockOwner((PVM)pvUser));
3408 /* Age compression - ASSUMES iNow == 4. */
3409 PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
3410 if (pChunk->iAge >= UINT32_C(0xffffff00))
3411 pChunk->iAge = 3;
3412 else if (pChunk->iAge >= UINT32_C(0xfffff000))
3413 pChunk->iAge = 2;
3414 else if (pChunk->iAge)
3415 pChunk->iAge = 1;
3416 else /* iAge = 0 */
3417 pChunk->iAge = 4;
3418 return 0;
3419}
3420
3421
3422/**
3423 * Tree enumeration callback that updates the chunks that have
3424 * been used since the last
3425 */
3426static DECLCALLBACK(int) pgmR3PhysChunkAgeingCallback(PAVLU32NODECORE pNode, void *pvUser)
3427{
3428 PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
3429 if (!pChunk->iAge)
3430 {
3431 PVM pVM = (PVM)pvUser;
3432 pChunk->iAge = pVM->pgm.s.ChunkR3Map.iNow;
3433 }
3434 return 0;
3435}
3436
3437
3438/**
3439 * Performs ageing of the ring-3 chunk mappings.
3440 *
3441 * @param pVM The VM handle.
3442 */
3443VMMR3DECL(void) PGMR3PhysChunkAgeing(PVM pVM)
3444{
3445 pgmLock(pVM);
3446 pVM->pgm.s.ChunkR3Map.AgeingCountdown = RT_MIN(pVM->pgm.s.ChunkR3Map.cMax / 4, 1024);
3447 pVM->pgm.s.ChunkR3Map.iNow++;
3448 if (pVM->pgm.s.ChunkR3Map.iNow == 0)
3449 {
3450 pVM->pgm.s.ChunkR3Map.iNow = 4;
3451 RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /*fFromLeft*/, pgmR3PhysChunkAgeingRolloverCallback, pVM);
3452 }
3453 else
3454 RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /*fFromLeft*/, pgmR3PhysChunkAgeingCallback, pVM);
3455 pgmUnlock(pVM);
3456}
3457
3458
3459/**
3460 * The structure passed in the pvUser argument of pgmR3PhysChunkUnmapCandidateCallback().
3461 */
3462typedef struct PGMR3PHYSCHUNKUNMAPCB
3463{
3464 PVM pVM; /**< The VM handle. */
3465 PPGMCHUNKR3MAP pChunk; /**< The chunk to unmap. */
3466 uint32_t iLastAge; /**< Highest age found so far. */
3467} PGMR3PHYSCHUNKUNMAPCB, *PPGMR3PHYSCHUNKUNMAPCB;
3468
3469
3470/**
3471 * Callback used to find the mapping that's been unused for
3472 * the longest time.
3473 */
3474static DECLCALLBACK(int) pgmR3PhysChunkUnmapCandidateCallback(PAVLU32NODECORE pNode, void *pvUser)
3475{
3476 PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
3477 PPGMR3PHYSCHUNKUNMAPCB pArg = (PPGMR3PHYSCHUNKUNMAPCB)pvUser;
3478
3479 if ( pChunk->iAge
3480 && !pChunk->cRefs
3481 && pArg->iLastAge < pChunk->iAge)
3482 {
3483 /*
3484 * Check that it's not in any of the TLBs.
3485 */
3486 PVM pVM = pArg->pVM;
3487 for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
3488 if (pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk == pChunk)
3489 {
3490 pChunk = NULL;
3491 break;
3492 }
3493 if (pChunk)
3494 for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbHC.aEntries); i++)
3495 if (pVM->pgm.s.PhysTlbHC.aEntries[i].pMap == pChunk)
3496 {
3497 pChunk = NULL;
3498 break;
3499 }
3500 if (pChunk)
3501 {
3502 pArg->pChunk = pChunk;
3503 pArg->iLastAge = pChunk->iAge;
3504 }
3505 }
3506 return 0;
3507}
3508
3509
3510/**
3511 * Finds a good candidate for unmapping when the ring-3 mapping cache is full.
3512 *
3513 * The candidate will not be part of any TLBs, so no need to flush
3514 * anything afterwards.
3515 *
3516 * @returns Chunk id.
3517 * @param pVM The VM handle.
3518 */
3519static int32_t pgmR3PhysChunkFindUnmapCandidate(PVM pVM)
3520{
3521 Assert(PGMIsLockOwner(pVM));
3522
3523 /*
3524 * Do tree ageing first?
3525 */
3526 if (pVM->pgm.s.ChunkR3Map.AgeingCountdown-- == 0)
3527 {
3528 STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkAging, a);
3529 PGMR3PhysChunkAgeing(pVM);
3530 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkAging, a);
3531 }
3532
3533 /*
3534 * Enumerate the age tree starting with the left most node.
3535 */
3536 STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3537 PGMR3PHYSCHUNKUNMAPCB Args;
3538 Args.pVM = pVM;
3539 Args.pChunk = NULL;
3540 Args.iLastAge = 0;
3541 RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /*fFromLeft*/, pgmR3PhysChunkUnmapCandidateCallback, &Args);
3542 Assert(Args.pChunk);
3543 if (Args.pChunk)
3544 {
3545 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3546 return Args.pChunk->Core.Key;
3547 }
3548
3549 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3550 return INT32_MAX;
3551}
3552
3553/**
3554 * Rendezvous callback used by pgmR3PhysUnmapChunk that unmaps a chunk
3555 *
3556 * This is only called on one of the EMTs while the other ones are waiting for
3557 * it to complete this function.
3558 *
3559 * @returns VINF_SUCCESS (VBox strict status code).
3560 * @param pVM The VM handle.
3561 * @param pVCpu The VMCPU for the EMT we're being called on. Unused.
3562 * @param pvUser User pointer. Unused
3563 *
3564 */
3565DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysUnmapChunkRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
3566{
3567 int rc = VINF_SUCCESS;
3568 pgmLock(pVM);
3569
3570 if (pVM->pgm.s.ChunkR3Map.c >= pVM->pgm.s.ChunkR3Map.cMax)
3571 {
3572 /* Flush the pgm pool cache; call the internal rendezvous handler as we're already in a rendezvous handler here. */
3573 /* todo: also not really efficient to unmap a chunk that contains PD or PT pages. */
3574 pgmR3PoolClearAllRendezvous(pVM, &pVM->aCpus[0], NULL /* no need to flush the REM TLB as we already did that above */);
3575
3576 /*
3577 * Request the ring-0 part to unmap a chunk to make space in the mapping cache.
3578 */
3579 GMMMAPUNMAPCHUNKREQ Req;
3580 Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
3581 Req.Hdr.cbReq = sizeof(Req);
3582 Req.pvR3 = NULL;
3583 Req.idChunkMap = NIL_GMM_CHUNKID;
3584 Req.idChunkUnmap = pgmR3PhysChunkFindUnmapCandidate(pVM);
3585
3586 if (Req.idChunkUnmap != INT32_MAX)
3587 {
3588 STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkUnmap, a);
3589 rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
3590 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkUnmap, a);
3591 if (RT_SUCCESS(rc))
3592 {
3593 /* remove the unmapped one. */
3594 PPGMCHUNKR3MAP pUnmappedChunk = (PPGMCHUNKR3MAP)RTAvlU32Remove(&pVM->pgm.s.ChunkR3Map.pTree, Req.idChunkUnmap);
3595 AssertRelease(pUnmappedChunk);
3596 pUnmappedChunk->pv = NULL;
3597 pUnmappedChunk->Core.Key = UINT32_MAX;
3598#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
3599 MMR3HeapFree(pUnmappedChunk);
3600#else
3601 MMR3UkHeapFree(pVM, pUnmappedChunk, MM_TAG_PGM_CHUNK_MAPPING);
3602#endif
3603 pVM->pgm.s.ChunkR3Map.c--;
3604 pVM->pgm.s.cUnmappedChunks++;
3605
3606 /* Flush dangling PGM pointers (R3 & R0 ptrs to GC physical addresses) */
3607 /* todo: we should not flush chunks which include cr3 mappings. */
3608 for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3609 {
3610 PPGMCPU pPGM = &pVM->aCpus[idCpu].pgm.s;
3611
3612 pPGM->pGst32BitPdR3 = NULL;
3613 pPGM->pGstPaePdptR3 = NULL;
3614 pPGM->pGstAmd64Pml4R3 = NULL;
3615#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
3616 pPGM->pGst32BitPdR0 = NIL_RTR0PTR;
3617 pPGM->pGstPaePdptR0 = NIL_RTR0PTR;
3618 pPGM->pGstAmd64Pml4R0 = NIL_RTR0PTR;
3619#endif
3620 for (unsigned i = 0; i < RT_ELEMENTS(pPGM->apGstPaePDsR3); i++)
3621 {
3622 pPGM->apGstPaePDsR3[i] = NULL;
3623#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
3624 pPGM->apGstPaePDsR0[i] = NIL_RTR0PTR;
3625#endif
3626 }
3627
3628 /* Flush REM TLBs. */
3629 CPUMSetChangedFlags(&pVM->aCpus[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
3630 }
3631
3632 /* Flush REM translation blocks. */
3633 REMFlushTBs(pVM);
3634 }
3635 }
3636 }
3637 pgmUnlock(pVM);
3638 return rc;
3639}
3640
3641/**
3642 * Unmap a chunk to free up virtual address space (request packet handler for pgmR3PhysChunkMap)
3643 *
3644 * @returns VBox status code.
3645 * @param pVM The VM to operate on.
3646 */
3647void pgmR3PhysUnmapChunk(PVM pVM)
3648{
3649 int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysUnmapChunkRendezvous, NULL);
3650 AssertRC(rc);
3651}
3652#endif /* PGM_WITH_LARGE_ADDRESS_SPACE_ON_32_BIT_HOST */
3653
3654/**
3655 * Maps the given chunk into the ring-3 mapping cache.
3656 *
3657 * This will call ring-0.
3658 *
3659 * @returns VBox status code.
3660 * @param pVM The VM handle.
3661 * @param idChunk The chunk in question.
3662 * @param ppChunk Where to store the chunk tracking structure.
3663 *
3664 * @remarks Called from within the PGM critical section.
3665 * @remarks Can be called from any thread!
3666 */
3667int pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk)
3668{
3669 int rc;
3670
3671 Assert(PGMIsLockOwner(pVM));
3672 /*
3673 * Allocate a new tracking structure first.
3674 */
3675#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
3676 PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3HeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk));
3677#else
3678 PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3UkHeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk), NULL);
3679#endif
3680 AssertReturn(pChunk, VERR_NO_MEMORY);
3681 pChunk->Core.Key = idChunk;
3682
3683 /*
3684 * Request the ring-0 part to map the chunk in question.
3685 */
3686 GMMMAPUNMAPCHUNKREQ Req;
3687 Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
3688 Req.Hdr.cbReq = sizeof(Req);
3689 Req.pvR3 = NULL;
3690 Req.idChunkMap = idChunk;
3691 Req.idChunkUnmap = NIL_GMM_CHUNKID;
3692
3693 /* Must be callable from any thread, so can't use VMMR3CallR0. */
3694 STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkMap, a);
3695 rc = SUPR3CallVMMR0Ex(pVM->pVMR0, NIL_VMCPUID, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
3696 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkMap, a);
3697 if (RT_SUCCESS(rc))
3698 {
3699 /*
3700 * Update the tree.
3701 */
3702 /* insert the new one. */
3703 AssertPtr(Req.pvR3);
3704 pChunk->pv = Req.pvR3;
3705 bool fRc = RTAvlU32Insert(&pVM->pgm.s.ChunkR3Map.pTree, &pChunk->Core);
3706 AssertRelease(fRc);
3707 pVM->pgm.s.ChunkR3Map.c++;
3708 pVM->pgm.s.cMappedChunks++;
3709
3710 /* If we're running out of virtual address space, then we should unmap another chunk. */
3711 if (pVM->pgm.s.ChunkR3Map.c >= pVM->pgm.s.ChunkR3Map.cMax)
3712 {
3713#ifdef PGM_WITH_LARGE_ADDRESS_SPACE_ON_32_BIT_HOST
3714 /* Postpone the unmap operation (which requires a rendezvous operation) as we own the PGM lock here. */
3715 rc = VMR3ReqCallNoWaitU(pVM->pUVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysUnmapChunk, 1, pVM);
3716 AssertRC(rc);
3717#else
3718 AssertFatalFailed(); /* can't happen */
3719#endif
3720 }
3721 }
3722 else
3723 {
3724 AssertRC(rc);
3725#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
3726 MMR3HeapFree(pChunk);
3727#else
3728 MMR3UkHeapFree(pVM, pChunk, MM_TAG_PGM_CHUNK_MAPPING);
3729#endif
3730 pChunk = NULL;
3731 }
3732
3733 *ppChunk = pChunk;
3734 return rc;
3735}
3736
3737
3738/**
3739 * For VMMCALLRING3_PGM_MAP_CHUNK, considered internal.
3740 *
3741 * @returns see pgmR3PhysChunkMap.
3742 * @param pVM The VM handle.
3743 * @param idChunk The chunk to map.
3744 */
3745VMMR3DECL(int) PGMR3PhysChunkMap(PVM pVM, uint32_t idChunk)
3746{
3747 PPGMCHUNKR3MAP pChunk;
3748 int rc;
3749
3750 pgmLock(pVM);
3751 rc = pgmR3PhysChunkMap(pVM, idChunk, &pChunk);
3752 pgmUnlock(pVM);
3753 return rc;
3754}
3755
3756
3757/**
3758 * Invalidates the TLB for the ring-3 mapping cache.
3759 *
3760 * @param pVM The VM handle.
3761 */
3762VMMR3DECL(void) PGMR3PhysChunkInvalidateTLB(PVM pVM)
3763{
3764 pgmLock(pVM);
3765 for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
3766 {
3767 pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk = NIL_GMM_CHUNKID;
3768 pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk = NULL;
3769 }
3770 /* The page map TLB references chunks, so invalidate that one too. */
3771 PGMPhysInvalidatePageMapTLB(pVM);
3772 pgmUnlock(pVM);
3773}
3774
3775
3776/**
3777 * Response to VMMCALLRING3_PGM_ALLOCATE_LARGE_PAGE to allocate a large (2MB) page
3778 * for use with a nested paging PDE.
3779 *
3780 * @returns The following VBox status codes.
3781 * @retval VINF_SUCCESS on success.
3782 * @retval VINF_EM_NO_MEMORY if we're out of memory.
3783 *
3784 * @param pVM The VM handle.
3785 * @param GCPhys GC physical start address of the 2 MB range
3786 */
3787VMMR3DECL(int) PGMR3PhysAllocateLargeHandyPage(PVM pVM, RTGCPHYS GCPhys)
3788{
3789 pgmLock(pVM);
3790
3791 STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatAllocLargePage, a);
3792 int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_LARGE_HANDY_PAGE, 0, NULL);
3793 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatAllocLargePage, a);
3794 if (RT_SUCCESS(rc))
3795 {
3796 Assert(pVM->pgm.s.cLargeHandyPages == 1);
3797
3798 uint32_t idPage = pVM->pgm.s.aLargeHandyPage[0].idPage;
3799 RTHCPHYS HCPhys = pVM->pgm.s.aLargeHandyPage[0].HCPhysGCPhys;
3800
3801 void *pv;
3802
3803 /* Map the large page into our address space.
3804 *
3805 * Note: assuming that within the 2 MB range:
3806 * - GCPhys + PAGE_SIZE = HCPhys + PAGE_SIZE (whole point of this exercise)
3807 * - user space mapping is continuous as well
3808 * - page id (GCPhys) + 1 = page id (GCPhys + PAGE_SIZE)
3809 */
3810 rc = pgmPhysPageMapByPageID(pVM, idPage, HCPhys, &pv);
3811 AssertLogRelMsg(RT_SUCCESS(rc), ("idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc\n", idPage, HCPhys, rc));
3812
3813 if (RT_SUCCESS(rc))
3814 {
3815 /*
3816 * Clear the pages.
3817 */
3818 STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatClearLargePage, b);
3819 for (unsigned i = 0; i < _2M/PAGE_SIZE; i++)
3820 {
3821 ASMMemZeroPage(pv);
3822
3823 PPGMPAGE pPage;
3824 rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
3825 AssertRC(rc);
3826
3827 Assert(PGM_PAGE_IS_ZERO(pPage));
3828 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatRZPageReplaceZero);
3829 pVM->pgm.s.cZeroPages--;
3830
3831 /*
3832 * Do the PGMPAGE modifications.
3833 */
3834 pVM->pgm.s.cPrivatePages++;
3835 PGM_PAGE_SET_HCPHYS(pPage, HCPhys);
3836 PGM_PAGE_SET_PAGEID(pPage, idPage);
3837 PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ALLOCATED);
3838 PGM_PAGE_SET_PDE_TYPE(pPage, PGM_PAGE_PDE_TYPE_PDE);
3839 PGM_PAGE_SET_PTE_INDEX(pPage, 0);
3840 PGM_PAGE_SET_TRACKING(pPage, 0);
3841
3842 /* Somewhat dirty assumption that page ids are increasing. */
3843 idPage++;
3844
3845 HCPhys += PAGE_SIZE;
3846 GCPhys += PAGE_SIZE;
3847
3848 pv = (void *)((uintptr_t)pv + PAGE_SIZE);
3849
3850 Log3(("PGMR3PhysAllocateLargePage: idPage=%#x HCPhys=%RGp\n", idPage, HCPhys));
3851 }
3852 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatClearLargePage, b);
3853
3854 /* Flush all TLBs. */
3855 PGM_INVL_ALL_VCPU_TLBS(pVM);
3856 PGMPhysInvalidatePageMapTLB(pVM);
3857 }
3858 pVM->pgm.s.cLargeHandyPages = 0;
3859 }
3860
3861 pgmUnlock(pVM);
3862 return rc;
3863}
3864
3865
3866/**
3867 * Response to VM_FF_PGM_NEED_HANDY_PAGES and VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES.
3868 *
3869 * This function will also work the VM_FF_PGM_NO_MEMORY force action flag, to
3870 * signal and clear the out of memory condition. When contracted, this API is
3871 * used to try clear the condition when the user wants to resume.
3872 *
3873 * @returns The following VBox status codes.
3874 * @retval VINF_SUCCESS on success. FFs cleared.
3875 * @retval VINF_EM_NO_MEMORY if we're out of memory. The FF is not cleared in
3876 * this case and it gets accompanied by VM_FF_PGM_NO_MEMORY.
3877 *
3878 * @param pVM The VM handle.
3879 *
3880 * @remarks The VINF_EM_NO_MEMORY status is for the benefit of the FF processing
3881 * in EM.cpp and shouldn't be propagated outside TRPM, HWACCM, EM and
3882 * pgmPhysEnsureHandyPage. There is one exception to this in the \#PF
3883 * handler.
3884 */
3885VMMR3DECL(int) PGMR3PhysAllocateHandyPages(PVM pVM)
3886{
3887 pgmLock(pVM);
3888
3889 /*
3890 * Allocate more pages, noting down the index of the first new page.
3891 */
3892 uint32_t iClear = pVM->pgm.s.cHandyPages;
3893 AssertMsgReturn(iClear <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d", iClear), VERR_INTERNAL_ERROR);
3894 Log(("PGMR3PhysAllocateHandyPages: %d -> %d\n", iClear, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
3895 int rcAlloc = VINF_SUCCESS;
3896 int rcSeed = VINF_SUCCESS;
3897 int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
3898 while (rc == VERR_GMM_SEED_ME)
3899 {
3900 void *pvChunk;
3901 rcAlloc = rc = SUPR3PageAlloc(GMM_CHUNK_SIZE >> PAGE_SHIFT, &pvChunk);
3902 if (RT_SUCCESS(rc))
3903 {
3904 rcSeed = rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_SEED_CHUNK, (uintptr_t)pvChunk, NULL);
3905 if (RT_FAILURE(rc))
3906 SUPR3PageFree(pvChunk, GMM_CHUNK_SIZE >> PAGE_SHIFT);
3907 }
3908 if (RT_SUCCESS(rc))
3909 rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
3910 }
3911
3912 /* todo: we should split this up into an allocate and flush operation. sometimes you want to flush and not allocate more (which will trigger the vm account limit error) */
3913 if ( rc == VERR_GMM_HIT_VM_ACCOUNT_LIMIT
3914 && pVM->pgm.s.cHandyPages > 0)
3915 {
3916 /* Still handy pages left, so don't panic. */
3917 rc = VINF_SUCCESS;
3918 }
3919
3920 if (RT_SUCCESS(rc))
3921 {
3922 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
3923 Assert(pVM->pgm.s.cHandyPages > 0);
3924 VM_FF_CLEAR(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
3925 VM_FF_CLEAR(pVM, VM_FF_PGM_NO_MEMORY);
3926
3927 /*
3928 * Clear the pages.
3929 */
3930 while (iClear < pVM->pgm.s.cHandyPages)
3931 {
3932 PGMMPAGEDESC pPage = &pVM->pgm.s.aHandyPages[iClear];
3933 void *pv;
3934 rc = pgmPhysPageMapByPageID(pVM, pPage->idPage, pPage->HCPhysGCPhys, &pv);
3935 AssertLogRelMsgBreak(RT_SUCCESS(rc), ("idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc\n", pPage->idPage, pPage->HCPhysGCPhys, rc));
3936 ASMMemZeroPage(pv);
3937 iClear++;
3938 Log3(("PGMR3PhysAllocateHandyPages: idPage=%#x HCPhys=%RGp\n", pPage->idPage, pPage->HCPhysGCPhys));
3939 }
3940 }
3941 else
3942 {
3943 uint64_t cAllocPages, cMaxPages, cBalloonPages;
3944
3945 /*
3946 * We should never get here unless there is a genuine shortage of
3947 * memory (or some internal error). Flag the error so the VM can be
3948 * suspended ASAP and the user informed. If we're totally out of
3949 * handy pages we will return failure.
3950 */
3951 /* Report the failure. */
3952 LogRel(("PGM: Failed to procure handy pages; rc=%Rrc rcAlloc=%Rrc rcSeed=%Rrc cHandyPages=%#x\n"
3953 " cAllPages=%#x cPrivatePages=%#x cSharedPages=%#x cZeroPages=%#x\n",
3954 rc, rcAlloc, rcSeed,
3955 pVM->pgm.s.cHandyPages,
3956 pVM->pgm.s.cAllPages,
3957 pVM->pgm.s.cPrivatePages,
3958 pVM->pgm.s.cSharedPages,
3959 pVM->pgm.s.cZeroPages));
3960
3961 if (GMMR3QueryMemoryStats(pVM, &cAllocPages, &cMaxPages, &cBalloonPages) == VINF_SUCCESS)
3962 {
3963 LogRel(("GMM: Statistics:\n"
3964 " Allocated pages: %RX64\n"
3965 " Maximum pages: %RX64\n"
3966 " Ballooned pages: %RX64\n", cAllocPages, cMaxPages, cBalloonPages));
3967 }
3968
3969 if ( rc != VERR_NO_MEMORY
3970 && rc != VERR_LOCK_FAILED)
3971 {
3972 for (uint32_t i = 0; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
3973 {
3974 LogRel(("PGM: aHandyPages[#%#04x] = {.HCPhysGCPhys=%RHp, .idPage=%#08x, .idSharedPage=%#08x}\n",
3975 i, pVM->pgm.s.aHandyPages[i].HCPhysGCPhys, pVM->pgm.s.aHandyPages[i].idPage,
3976 pVM->pgm.s.aHandyPages[i].idSharedPage));
3977 uint32_t const idPage = pVM->pgm.s.aHandyPages[i].idPage;
3978 if (idPage != NIL_GMM_PAGEID)
3979 {
3980 for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
3981 pRam;
3982 pRam = pRam->pNextR3)
3983 {
3984 uint32_t const cPages = pRam->cb >> PAGE_SHIFT;
3985 for (uint32_t iPage = 0; iPage < cPages; iPage++)
3986 if (PGM_PAGE_GET_PAGEID(&pRam->aPages[iPage]) == idPage)
3987 LogRel(("PGM: Used by %RGp %R[pgmpage] (%s)\n",
3988 pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pRam->aPages[iPage], pRam->pszDesc));
3989 }
3990 }
3991 }
3992 }
3993
3994 /* Set the FFs and adjust rc. */
3995 VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
3996 VM_FF_SET(pVM, VM_FF_PGM_NO_MEMORY);
3997 if ( rc == VERR_NO_MEMORY
3998 || rc == VERR_LOCK_FAILED)
3999 rc = VINF_EM_NO_MEMORY;
4000 }
4001
4002 pgmUnlock(pVM);
4003 return rc;
4004}
4005
4006
4007/**
4008 * Frees the specified RAM page and replaces it with the ZERO page.
4009 *
4010 * This is used by ballooning, remapping MMIO2 and RAM reset.
4011 *
4012 * @param pVM Pointer to the shared VM structure.
4013 * @param pReq Pointer to the request.
4014 * @param pPage Pointer to the page structure.
4015 * @param GCPhys The guest physical address of the page, if applicable.
4016 *
4017 * @remarks The caller must own the PGM lock.
4018 */
4019int pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys)
4020{
4021 /*
4022 * Assert sanity.
4023 */
4024 Assert(PGMIsLockOwner(pVM));
4025 if (RT_UNLIKELY( PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
4026 && PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_ROM_SHADOW))
4027 {
4028 AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
4029 return VMSetError(pVM, VERR_PGM_PHYS_NOT_RAM, RT_SRC_POS, "GCPhys=%RGp type=%d", GCPhys, PGM_PAGE_GET_TYPE(pPage));
4030 }
4031
4032 if ( PGM_PAGE_IS_ZERO(pPage)
4033 || PGM_PAGE_IS_BALLOONED(pPage))
4034 return VINF_SUCCESS;
4035
4036 const uint32_t idPage = PGM_PAGE_GET_PAGEID(pPage);
4037 Log3(("pgmPhysFreePage: idPage=%#x GCPhys=%RGp pPage=%R[pgmpage]\n", idPage, GCPhys, pPage));
4038 if (RT_UNLIKELY( idPage == NIL_GMM_PAGEID
4039 || idPage > GMM_PAGEID_LAST
4040 || PGM_PAGE_GET_CHUNKID(pPage) == NIL_GMM_CHUNKID))
4041 {
4042 AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
4043 return VMSetError(pVM, VERR_PGM_PHYS_INVALID_PAGE_ID, RT_SRC_POS, "GCPhys=%RGp idPage=%#x", GCPhys, pPage);
4044 }
4045
4046 /* update page count stats. */
4047 if (PGM_PAGE_IS_SHARED(pPage))
4048 pVM->pgm.s.cSharedPages--;
4049 else
4050 pVM->pgm.s.cPrivatePages--;
4051 pVM->pgm.s.cZeroPages++;
4052
4053 /* Deal with write monitored pages. */
4054 if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
4055 {
4056 PGM_PAGE_SET_WRITTEN_TO(pPage);
4057 pVM->pgm.s.cWrittenToPages++;
4058 }
4059
4060 /*
4061 * pPage = ZERO page.
4062 */
4063 PGM_PAGE_SET_HCPHYS(pPage, pVM->pgm.s.HCPhysZeroPg);
4064 PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ZERO);
4065 PGM_PAGE_SET_PAGEID(pPage, NIL_GMM_PAGEID);
4066 PGM_PAGE_SET_PDE_TYPE(pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
4067 PGM_PAGE_SET_PTE_INDEX(pPage, 0);
4068 PGM_PAGE_SET_TRACKING(pPage, 0);
4069
4070 /* Flush physical page map TLB entry. */
4071 PGMPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
4072
4073 /*
4074 * Make sure it's not in the handy page array.
4075 */
4076 for (uint32_t i = pVM->pgm.s.cHandyPages; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
4077 {
4078 if (pVM->pgm.s.aHandyPages[i].idPage == idPage)
4079 {
4080 pVM->pgm.s.aHandyPages[i].idPage = NIL_GMM_PAGEID;
4081 break;
4082 }
4083 if (pVM->pgm.s.aHandyPages[i].idSharedPage == idPage)
4084 {
4085 pVM->pgm.s.aHandyPages[i].idSharedPage = NIL_GMM_PAGEID;
4086 break;
4087 }
4088 }
4089
4090 /*
4091 * Push it onto the page array.
4092 */
4093 uint32_t iPage = *pcPendingPages;
4094 Assert(iPage < PGMPHYS_FREE_PAGE_BATCH_SIZE);
4095 *pcPendingPages += 1;
4096
4097 pReq->aPages[iPage].idPage = idPage;
4098
4099 if (iPage + 1 < PGMPHYS_FREE_PAGE_BATCH_SIZE)
4100 return VINF_SUCCESS;
4101
4102 /*
4103 * Flush the pages.
4104 */
4105 int rc = GMMR3FreePagesPerform(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE);
4106 if (RT_SUCCESS(rc))
4107 {
4108 GMMR3FreePagesRePrep(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
4109 *pcPendingPages = 0;
4110 }
4111 return rc;
4112}
4113
4114
4115/**
4116 * Converts a GC physical address to a HC ring-3 pointer, with some
4117 * additional checks.
4118 *
4119 * @returns VBox status code.
4120 * @retval VINF_SUCCESS on success.
4121 * @retval VINF_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
4122 * access handler of some kind.
4123 * @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
4124 * accesses or is odd in any way.
4125 * @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
4126 *
4127 * @param pVM The VM handle.
4128 * @param GCPhys The GC physical address to convert.
4129 * @param fWritable Whether write access is required.
4130 * @param ppv Where to store the pointer corresponding to GCPhys on
4131 * success.
4132 */
4133VMMR3DECL(int) PGMR3PhysTlbGCPhys2Ptr(PVM pVM, RTGCPHYS GCPhys, bool fWritable, void **ppv)
4134{
4135 pgmLock(pVM);
4136
4137 PPGMRAMRANGE pRam;
4138 PPGMPAGE pPage;
4139 int rc = pgmPhysGetPageAndRangeEx(&pVM->pgm.s, GCPhys, &pPage, &pRam);
4140 if (RT_SUCCESS(rc))
4141 {
4142 if (PGM_PAGE_IS_BALLOONED(pPage))
4143 rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
4144 else if (!PGM_PAGE_HAS_ANY_HANDLERS(pPage))
4145 rc = VINF_SUCCESS;
4146 else
4147 {
4148 if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
4149 rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4150 else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
4151 {
4152 /** @todo Handle TLB loads of virtual handlers so ./test.sh can be made to work
4153 * in -norawr0 mode. */
4154 if (fWritable)
4155 rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
4156 }
4157 else
4158 {
4159 /* Temporarily disabled physical handler(s), since the recompiler
4160 doesn't get notified when it's reset we'll have to pretend it's
4161 operating normally. */
4162 if (pgmHandlerPhysicalIsAll(pVM, GCPhys))
4163 rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4164 else
4165 rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
4166 }
4167 }
4168 if (RT_SUCCESS(rc))
4169 {
4170 int rc2;
4171
4172 /* Make sure what we return is writable. */
4173 if (fWritable)
4174 switch (PGM_PAGE_GET_STATE(pPage))
4175 {
4176 case PGM_PAGE_STATE_ALLOCATED:
4177 break;
4178 case PGM_PAGE_STATE_BALLOONED:
4179 AssertFailed();
4180 break;
4181 case PGM_PAGE_STATE_ZERO:
4182 case PGM_PAGE_STATE_SHARED:
4183 if (rc == VINF_PGM_PHYS_TLB_CATCH_WRITE)
4184 break;
4185 case PGM_PAGE_STATE_WRITE_MONITORED:
4186 rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
4187 AssertLogRelRCReturn(rc2, rc2);
4188 break;
4189 }
4190
4191 /* Get a ring-3 mapping of the address. */
4192 PPGMPAGER3MAPTLBE pTlbe;
4193 rc2 = pgmPhysPageQueryTlbe(&pVM->pgm.s, GCPhys, &pTlbe);
4194 AssertLogRelRCReturn(rc2, rc2);
4195 *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
4196 /** @todo mapping/locking hell; this isn't horribly efficient since
4197 * pgmPhysPageLoadIntoTlb will repeat the lookup we've done here. */
4198
4199 Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage] *ppv=%p\n", GCPhys, rc, pPage, *ppv));
4200 }
4201 else
4202 Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage]\n", GCPhys, rc, pPage));
4203
4204 /* else: handler catching all access, no pointer returned. */
4205 }
4206 else
4207 rc = VERR_PGM_PHYS_TLB_UNASSIGNED;
4208
4209 pgmUnlock(pVM);
4210 return rc;
4211}
4212
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette