PGMPhys.cpp@ 37423

Last change on this file since 37423 was 37355, checked in by vboxsync, 14 years ago
build fix.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 161.6 KB

Line
1	/* $Id: PGMPhys.cpp 37355 2011-06-07 16:19:43Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor, Physical Memory Addressing.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_PGM_PHYS
23	#include <VBox/vmm/pgm.h>
24	#include <VBox/vmm/iom.h>
25	#include <VBox/vmm/mm.h>
26	#include <VBox/vmm/stam.h>
27	#include <VBox/vmm/rem.h>
28	#include <VBox/vmm/pdmdev.h>
29	#include "PGMInternal.h"
30	#include <VBox/vmm/vm.h>
31	#include "PGMInline.h"
32	#include <VBox/sup.h>
33	#include <VBox/param.h>
34	#include <VBox/err.h>
35	#include <VBox/log.h>
36	#include <iprt/assert.h>
37	#include <iprt/alloc.h>
38	#include <iprt/asm.h>
39	#include <iprt/thread.h>
40	#include <iprt/string.h>
41	#include <iprt/system.h>
42
43
44	/*******************************************************************************
45	* Defined Constants And Macros *
46	*******************************************************************************/
47	/** The number of pages to free in one batch. */
48	#define PGMPHYS_FREE_PAGE_BATCH_SIZE 128
49
50
51	/*******************************************************************************
52	* Internal Functions *
53	*******************************************************************************/
54	static DECLCALLBACK(int) pgmR3PhysRomWriteHandler(PVM pVM, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser);
55
56
57	/*
58	* PGMR3PhysReadU8-64
59	* PGMR3PhysWriteU8-64
60	*/
61	#define PGMPHYSFN_READNAME PGMR3PhysReadU8
62	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU8
63	#define PGMPHYS_DATASIZE 1
64	#define PGMPHYS_DATATYPE uint8_t
65	#include "PGMPhysRWTmpl.h"
66
67	#define PGMPHYSFN_READNAME PGMR3PhysReadU16
68	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU16
69	#define PGMPHYS_DATASIZE 2
70	#define PGMPHYS_DATATYPE uint16_t
71	#include "PGMPhysRWTmpl.h"
72
73	#define PGMPHYSFN_READNAME PGMR3PhysReadU32
74	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU32
75	#define PGMPHYS_DATASIZE 4
76	#define PGMPHYS_DATATYPE uint32_t
77	#include "PGMPhysRWTmpl.h"
78
79	#define PGMPHYSFN_READNAME PGMR3PhysReadU64
80	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU64
81	#define PGMPHYS_DATASIZE 8
82	#define PGMPHYS_DATATYPE uint64_t
83	#include "PGMPhysRWTmpl.h"
84
85
86	/**
87	* EMT worker for PGMR3PhysReadExternal.
88	*/
89	static DECLCALLBACK(int) pgmR3PhysReadExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, void *pvBuf, size_t cbRead)
90	{
91	PGMPhysRead(pVM, *pGCPhys, pvBuf, cbRead);
92	return VINF_SUCCESS;
93	}
94
95
96	/**
97	* Read from physical memory, external users.
98	*
99	* @returns VBox status code.
100	* @retval VINF_SUCCESS.
101	*
102	* @param pVM VM Handle.
103	* @param GCPhys Physical address to read from.
104	* @param pvBuf Where to read into.
105	* @param cbRead How many bytes to read.
106	*
107	* @thread Any but EMTs.
108	*/
109	VMMR3DECL(int) PGMR3PhysReadExternal(PVM pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead)
110	{
111	VM_ASSERT_OTHER_THREAD(pVM);
112
113	AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
114	LogFlow(("PGMR3PhysReadExternal: %RGp %d\n", GCPhys, cbRead));
115
116	pgmLock(pVM);
117
118	/*
119	* Copy loop on ram ranges.
120	*/
121	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
122	for (;;)
123	{
124	/* Inside range or not? */
125	if (pRam && GCPhys >= pRam->GCPhys)
126	{
127	/*
128	* Must work our way thru this page by page.
129	*/
130	RTGCPHYS off = GCPhys - pRam->GCPhys;
131	while (off < pRam->cb)
132	{
133	unsigned iPage = off >> PAGE_SHIFT;
134	PPGMPAGE pPage = &pRam->aPages[iPage];
135
136	/*
137	* If the page has an ALL access handler, we'll have to
138	* delegate the job to EMT.
139	*/
140	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
141	{
142	pgmUnlock(pVM);
143
144	return VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysReadExternalEMT, 4,
145	pVM, &GCPhys, pvBuf, cbRead);
146	}
147	Assert(!PGM_PAGE_IS_MMIO(pPage));
148
149	/*
150	* Simple stuff, go ahead.
151	*/
152	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
153	if (cb > cbRead)
154	cb = cbRead;
155	const void *pvSrc;
156	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc);
157	if (RT_SUCCESS(rc))
158	memcpy(pvBuf, pvSrc, cb);
159	else
160	{
161	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
162	pRam->GCPhys + off, pPage, rc));
163	memset(pvBuf, 0xff, cb);
164	}
165
166	/* next page */
167	if (cb >= cbRead)
168	{
169	pgmUnlock(pVM);
170	return VINF_SUCCESS;
171	}
172	cbRead -= cb;
173	off += cb;
174	GCPhys += cb;
175	pvBuf = (char *)pvBuf + cb;
176	} /* walk pages in ram range. */
177	}
178	else
179	{
180	LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
181
182	/*
183	* Unassigned address space.
184	*/
185	size_t cb = pRam ? pRam->GCPhys - GCPhys : ~(size_t)0;
186	if (cb >= cbRead)
187	{
188	memset(pvBuf, 0xff, cbRead);
189	break;
190	}
191	memset(pvBuf, 0xff, cb);
192
193	cbRead -= cb;
194	pvBuf = (char *)pvBuf + cb;
195	GCPhys += cb;
196	}
197
198	/* Advance range if necessary. */
199	while (pRam && GCPhys > pRam->GCPhysLast)
200	pRam = pRam->CTX_SUFF(pNext);
201	} /* Ram range walk */
202
203	pgmUnlock(pVM);
204
205	return VINF_SUCCESS;
206	}
207
208
209	/**
210	* EMT worker for PGMR3PhysWriteExternal.
211	*/
212	static DECLCALLBACK(int) pgmR3PhysWriteExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, const void *pvBuf, size_t cbWrite)
213	{
214	/** @todo VERR_EM_NO_MEMORY */
215	PGMPhysWrite(pVM, *pGCPhys, pvBuf, cbWrite);
216	return VINF_SUCCESS;
217	}
218
219
220	/**
221	* Write to physical memory, external users.
222	*
223	* @returns VBox status code.
224	* @retval VINF_SUCCESS.
225	* @retval VERR_EM_NO_MEMORY.
226	*
227	* @param pVM VM Handle.
228	* @param GCPhys Physical address to write to.
229	* @param pvBuf What to write.
230	* @param cbWrite How many bytes to write.
231	* @param pszWho Who is writing. For tracking down who is writing
232	* after we've saved the state.
233	*
234	* @thread Any but EMTs.
235	*/
236	VMMDECL(int) PGMR3PhysWriteExternal(PVM pVM, RTGCPHYS GCPhys, const void pvBuf, size_t cbWrite, const char pszWho)
237	{
238	VM_ASSERT_OTHER_THREAD(pVM);
239
240	AssertMsg(!pVM->pgm.s.fNoMorePhysWrites,
241	("Calling PGMR3PhysWriteExternal after pgmR3Save()! GCPhys=%RGp cbWrite=%#x pszWho=%s\n",
242	GCPhys, cbWrite, pszWho));
243	AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
244	LogFlow(("PGMR3PhysWriteExternal: %RGp %d\n", GCPhys, cbWrite));
245
246	pgmLock(pVM);
247
248	/*
249	* Copy loop on ram ranges, stop when we hit something difficult.
250	*/
251	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
252	for (;;)
253	{
254	/* Inside range or not? */
255	if (pRam && GCPhys >= pRam->GCPhys)
256	{
257	/*
258	* Must work our way thru this page by page.
259	*/
260	RTGCPTR off = GCPhys - pRam->GCPhys;
261	while (off < pRam->cb)
262	{
263	RTGCPTR iPage = off >> PAGE_SHIFT;
264	PPGMPAGE pPage = &pRam->aPages[iPage];
265
266	/*
267	* Is the page problematic, we have to do the work on the EMT.
268	*
269	* Allocating writable pages and access handlers are
270	* problematic, write monitored pages are simple and can be
271	* dealt with here.
272	*/
273	if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
274	\|\| PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
275	{
276	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
277	&& !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
278	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
279	else
280	{
281	pgmUnlock(pVM);
282
283	return VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysWriteExternalEMT, 4,
284	pVM, &GCPhys, pvBuf, cbWrite);
285	}
286	}
287	Assert(!PGM_PAGE_IS_MMIO(pPage));
288
289	/*
290	* Simple stuff, go ahead.
291	*/
292	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
293	if (cb > cbWrite)
294	cb = cbWrite;
295	void *pvDst;
296	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst);
297	if (RT_SUCCESS(rc))
298	memcpy(pvDst, pvBuf, cb);
299	else
300	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
301	pRam->GCPhys + off, pPage, rc));
302
303	/* next page */
304	if (cb >= cbWrite)
305	{
306	pgmUnlock(pVM);
307	return VINF_SUCCESS;
308	}
309
310	cbWrite -= cb;
311	off += cb;
312	GCPhys += cb;
313	pvBuf = (const char *)pvBuf + cb;
314	} /* walk pages in ram range */
315	}
316	else
317	{
318	/*
319	* Unassigned address space, skip it.
320	*/
321	if (!pRam)
322	break;
323	size_t cb = pRam->GCPhys - GCPhys;
324	if (cb >= cbWrite)
325	break;
326	cbWrite -= cb;
327	pvBuf = (const char *)pvBuf + cb;
328	GCPhys += cb;
329	}
330
331	/* Advance range if necessary. */
332	while (pRam && GCPhys > pRam->GCPhysLast)
333	pRam = pRam->CTX_SUFF(pNext);
334	} /* Ram range walk */
335
336	pgmUnlock(pVM);
337	return VINF_SUCCESS;
338	}
339
340
341	/**
342	* VMR3ReqCall worker for PGMR3PhysGCPhys2CCPtrExternal to make pages writable.
343	*
344	* @returns see PGMR3PhysGCPhys2CCPtrExternal
345	* @param pVM The VM handle.
346	* @param pGCPhys Pointer to the guest physical address.
347	* @param ppv Where to store the mapping address.
348	* @param pLock Where to store the lock.
349	*/
350	static DECLCALLBACK(int) pgmR3PhysGCPhys2CCPtrDelegated(PVM pVM, PRTGCPHYS pGCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
351	{
352	/*
353	* Just hand it to PGMPhysGCPhys2CCPtr and check that it's not a page with
354	* an access handler after it succeeds.
355	*/
356	int rc = pgmLock(pVM);
357	AssertRCReturn(rc, rc);
358
359	rc = PGMPhysGCPhys2CCPtr(pVM, *pGCPhys, ppv, pLock);
360	if (RT_SUCCESS(rc))
361	{
362	PPGMPAGEMAPTLBE pTlbe;
363	int rc2 = pgmPhysPageQueryTlbe(pVM, *pGCPhys, &pTlbe);
364	AssertFatalRC(rc2);
365	PPGMPAGE pPage = pTlbe->pPage;
366	if (PGM_PAGE_IS_MMIO(pPage))
367	{
368	PGMPhysReleasePageMappingLock(pVM, pLock);
369	rc = VERR_PGM_PHYS_PAGE_RESERVED;
370	}
371	else if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
372	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
373	\|\| pgmPoolIsDirtyPage(pVM, *pGCPhys)
374	#endif
375	)
376	{
377	/* We must flush any corresponding pgm pool page here, otherwise we'll
378	* not be informed about writes and keep bogus gst->shw mappings around.
379	*/
380	pgmPoolFlushPageByGCPhys(pVM, *pGCPhys);
381	Assert(!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage));
382	/** @todo r=bird: return VERR_PGM_PHYS_PAGE_RESERVED here if it still has
383	* active handlers, see the PGMR3PhysGCPhys2CCPtrExternal docs. */
384	}
385	}
386
387	pgmUnlock(pVM);
388	return rc;
389	}
390
391
392	/**
393	* Requests the mapping of a guest page into ring-3, external threads.
394	*
395	* When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
396	* release it.
397	*
398	* This API will assume your intention is to write to the page, and will
399	* therefore replace shared and zero pages. If you do not intend to modify the
400	* page, use the PGMR3PhysGCPhys2CCPtrReadOnlyExternal() API.
401	*
402	* @returns VBox status code.
403	* @retval VINF_SUCCESS on success.
404	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
405	* backing or if the page has any active access handlers. The caller
406	* must fall back on using PGMR3PhysWriteExternal.
407	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
408	*
409	* @param pVM The VM handle.
410	* @param GCPhys The guest physical address of the page that should be mapped.
411	* @param ppv Where to store the address corresponding to GCPhys.
412	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
413	*
414	* @remark Avoid calling this API from within critical sections (other than the
415	* PGM one) because of the deadlock risk when we have to delegating the
416	* task to an EMT.
417	* @thread Any.
418	*/
419	VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrExternal(PVM pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
420	{
421	AssertPtr(ppv);
422	AssertPtr(pLock);
423
424	Assert(VM_IS_EMT(pVM) \|\| !PGMIsLockOwner(pVM));
425
426	int rc = pgmLock(pVM);
427	AssertRCReturn(rc, rc);
428
429	/*
430	* Query the Physical TLB entry for the page (may fail).
431	*/
432	PPGMPAGEMAPTLBE pTlbe;
433	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
434	if (RT_SUCCESS(rc))
435	{
436	PPGMPAGE pPage = pTlbe->pPage;
437	if (PGM_PAGE_IS_MMIO(pPage))
438	rc = VERR_PGM_PHYS_PAGE_RESERVED;
439	else
440	{
441	/*
442	* If the page is shared, the zero page, or being write monitored
443	* it must be converted to an page that's writable if possible.
444	* We can only deal with write monitored pages here, the rest have
445	* to be on an EMT.
446	*/
447	if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
448	\|\| PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
449	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
450	\|\| pgmPoolIsDirtyPage(pVM, GCPhys)
451	#endif
452	)
453	{
454	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
455	&& !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
456	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
457	&& !pgmPoolIsDirtyPage(pVM, GCPhys)
458	#endif
459	)
460	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
461	else
462	{
463	pgmUnlock(pVM);
464
465	return VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysGCPhys2CCPtrDelegated, 4,
466	pVM, &GCPhys, ppv, pLock);
467	}
468	}
469
470	/*
471	* Now, just perform the locking and calculate the return address.
472	*/
473	PPGMPAGEMAP pMap = pTlbe->pMap;
474	if (pMap)
475	pMap->cRefs++;
476
477	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
478	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
479	{
480	if (cLocks == 0)
481	pVM->pgm.s.cWriteLockedPages++;
482	PGM_PAGE_INC_WRITE_LOCKS(pPage);
483	}
484	else if (cLocks != PGM_PAGE_GET_WRITE_LOCKS(pPage))
485	{
486	PGM_PAGE_INC_WRITE_LOCKS(pPage);
487	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent write locked state!\n", GCPhys, pPage));
488	if (pMap)
489	pMap->cRefs++; /* Extra ref to prevent it from going away. */
490	}
491
492	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
493	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_WRITE;
494	pLock->pvMap = pMap;
495	}
496	}
497
498	pgmUnlock(pVM);
499	return rc;
500	}
501
502
503	/**
504	* Requests the mapping of a guest page into ring-3, external threads.
505	*
506	* When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
507	* release it.
508	*
509	* @returns VBox status code.
510	* @retval VINF_SUCCESS on success.
511	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
512	* backing or if the page as an active ALL access handler. The caller
513	* must fall back on using PGMPhysRead.
514	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
515	*
516	* @param pVM The VM handle.
517	* @param GCPhys The guest physical address of the page that should be mapped.
518	* @param ppv Where to store the address corresponding to GCPhys.
519	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
520	*
521	* @remark Avoid calling this API from within critical sections (other than
522	* the PGM one) because of the deadlock risk.
523	* @thread Any.
524	*/
525	VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrReadOnlyExternal(PVM pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
526	{
527	int rc = pgmLock(pVM);
528	AssertRCReturn(rc, rc);
529
530	/*
531	* Query the Physical TLB entry for the page (may fail).
532	*/
533	PPGMPAGEMAPTLBE pTlbe;
534	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
535	if (RT_SUCCESS(rc))
536	{
537	PPGMPAGE pPage = pTlbe->pPage;
538	#if 1
539	/* MMIO pages doesn't have any readable backing. */
540	if (PGM_PAGE_IS_MMIO(pPage))
541	rc = VERR_PGM_PHYS_PAGE_RESERVED;
542	#else
543	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
544	rc = VERR_PGM_PHYS_PAGE_RESERVED;
545	#endif
546	else
547	{
548	/*
549	* Now, just perform the locking and calculate the return address.
550	*/
551	PPGMPAGEMAP pMap = pTlbe->pMap;
552	if (pMap)
553	pMap->cRefs++;
554
555	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
556	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
557	{
558	if (cLocks == 0)
559	pVM->pgm.s.cReadLockedPages++;
560	PGM_PAGE_INC_READ_LOCKS(pPage);
561	}
562	else if (cLocks != PGM_PAGE_GET_READ_LOCKS(pPage))
563	{
564	PGM_PAGE_INC_READ_LOCKS(pPage);
565	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent readonly locked state!\n", GCPhys, pPage));
566	if (pMap)
567	pMap->cRefs++; /* Extra ref to prevent it from going away. */
568	}
569
570	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
571	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_READ;
572	pLock->pvMap = pMap;
573	}
574	}
575
576	pgmUnlock(pVM);
577	return rc;
578	}
579
580
581	#define MAKE_LEAF(a_pNode) \
582	do { \
583	(a_pNode)->pLeftR3 = NIL_RTR3PTR; \
584	(a_pNode)->pRightR3 = NIL_RTR3PTR; \
585	(a_pNode)->pLeftR0 = NIL_RTR0PTR; \
586	(a_pNode)->pRightR0 = NIL_RTR0PTR; \
587	(a_pNode)->pLeftRC = NIL_RTRCPTR; \
588	(a_pNode)->pRightRC = NIL_RTRCPTR; \
589	} while (0)
590
591	#define INSERT_LEFT(a_pParent, a_pNode) \
592	do { \
593	(a_pParent)->pLeftR3 = (a_pNode); \
594	(a_pParent)->pLeftR0 = (a_pNode)->pSelfR0; \
595	(a_pParent)->pLeftRC = (a_pNode)->pSelfRC; \
596	} while (0)
597	#define INSERT_RIGHT(a_pParent, a_pNode) \
598	do { \
599	(a_pParent)->pRightR3 = (a_pNode); \
600	(a_pParent)->pRightR0 = (a_pNode)->pSelfR0; \
601	(a_pParent)->pRightRC = (a_pNode)->pSelfRC; \
602	} while (0)
603
604
605	/**
606	* Recursive tree builder.
607	*
608	* @param ppRam Pointer to the iterator variable.
609	* @param iHeight The hight about normal leaf nodes. Inserts a leaf
610	* node if 0.
611	*/
612	static PPGMRAMRANGE pgmR3PhysRebuildRamRangeSearchTreesRecursively(PPGMRAMRANGE *ppRam, int iDepth)
613	{
614	PPGMRAMRANGE pRam;
615	if (iDepth <= 0)
616	{
617	/*
618	* Leaf node.
619	*/
620	pRam = *ppRam;
621	if (pRam)
622	{
623	*ppRam = pRam->pNextR3;
624	MAKE_LEAF(pRam);
625	}
626	}
627	else
628	{
629
630	/*
631	* Intermediate node.
632	*/
633	PPGMRAMRANGE pLeft = pgmR3PhysRebuildRamRangeSearchTreesRecursively(ppRam, iDepth - 1);
634
635	pRam = *ppRam;
636	if (!pRam)
637	return pLeft;
638	*ppRam = pRam->pNextR3;
639	MAKE_LEAF(pRam);
640	INSERT_LEFT(pRam, pLeft);
641
642	PPGMRAMRANGE pRight = pgmR3PhysRebuildRamRangeSearchTreesRecursively(ppRam, iDepth - 1);
643	if (pRight)
644	INSERT_RIGHT(pRam, pRight);
645	}
646	return pRam;
647	}
648
649
650	/**
651	* Rebuilds the RAM range search trees.
652	*
653	* @param pVM The VM handle.
654	*/
655	static void pgmR3PhysRebuildRamRangeSearchTrees(PVM pVM)
656	{
657
658	/*
659	* Create the reasonably balanced tree in a sequential fashion.
660	* For simplicity (laziness) we use standard recursion here.
661	*/
662	int iDepth = 0;
663	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
664	PPGMRAMRANGE pRoot = pgmR3PhysRebuildRamRangeSearchTreesRecursively(&pRam, 0);
665	while (pRam)
666	{
667	PPGMRAMRANGE pLeft = pRoot;
668
669	pRoot = pRam;
670	pRam = pRam->pNextR3;
671	MAKE_LEAF(pRoot);
672	INSERT_LEFT(pRoot, pLeft);
673
674	PPGMRAMRANGE pRight = pgmR3PhysRebuildRamRangeSearchTreesRecursively(&pRam, iDepth);
675	if (pRight)
676	INSERT_RIGHT(pRoot, pRight);
677	/** @todo else: rotate the tree. */
678
679	iDepth++;
680	}
681
682	pVM->pgm.s.pRamRangeTreeR3 = pRoot;
683	pVM->pgm.s.pRamRangeTreeR0 = pRoot ? pRoot->pSelfR0 : NIL_RTR0PTR;
684	pVM->pgm.s.pRamRangeTreeRC = pRoot ? pRoot->pSelfRC : NIL_RTRCPTR;
685
686	#ifdef VBOX_STRICT
687	/*
688	* Verify that the above code works.
689	*/
690	unsigned cRanges = 0;
691	for (pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
692	cRanges++;
693	Assert(cRanges > 0);
694
695	unsigned cMaxDepth = ASMBitLastSetU32(cRanges);
696	if ((1U << cMaxDepth) < cRanges)
697	cMaxDepth++;
698
699	for (pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
700	{
701	unsigned cDepth = 0;
702	PPGMRAMRANGE pRam2 = pVM->pgm.s.pRamRangeTreeR3;
703	for (;;)
704	{
705	if (pRam == pRam2)
706	break;
707	Assert(pRam2);
708	if (pRam->GCPhys < pRam2->GCPhys)
709	pRam2 = pRam2->pLeftR3;
710	else
711	pRam2 = pRam2->pRightR3;
712	}
713	AssertMsg(cDepth <= cMaxDepth, ("cDepth=%d cMaxDepth=%d\n", cDepth, cMaxDepth));
714	}
715	#endif /* VBOX_STRICT */
716	}
717
718	#undef MAKE_LEAF
719	#undef INSERT_LEFT
720	#undef INSERT_RIGHT
721
722	/**
723	* Relinks the RAM ranges using the pSelfRC and pSelfR0 pointers.
724	*
725	* Called when anything was relocated.
726	*
727	* @param pVM Pointer to the shared VM structure.
728	*/
729	void pgmR3PhysRelinkRamRanges(PVM pVM)
730	{
731	PPGMRAMRANGE pCur;
732
733	#ifdef VBOX_STRICT
734	for (pCur = pVM->pgm.s.pRamRangesXR3; pCur; pCur = pCur->pNextR3)
735	{
736	Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pCur->pSelfR0 == MMHyperCCToR0(pVM, pCur));
737	Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pCur->pSelfRC == MMHyperCCToRC(pVM, pCur));
738	Assert((pCur->GCPhys & PAGE_OFFSET_MASK) == 0);
739	Assert((pCur->GCPhysLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
740	Assert((pCur->cb & PAGE_OFFSET_MASK) == 0);
741	Assert(pCur->cb == pCur->GCPhysLast - pCur->GCPhys + 1);
742	for (PPGMRAMRANGE pCur2 = pVM->pgm.s.pRamRangesXR3; pCur2; pCur2 = pCur2->pNextR3)
743	Assert( pCur2 == pCur
744	\|\| strcmp(pCur2->pszDesc, pCur->pszDesc)); /** @todo fix MMIO ranges!! */
745	}
746	#endif
747
748	pCur = pVM->pgm.s.pRamRangesXR3;
749	if (pCur)
750	{
751	pVM->pgm.s.pRamRangesXR0 = pCur->pSelfR0;
752	pVM->pgm.s.pRamRangesXRC = pCur->pSelfRC;
753
754	for (; pCur->pNextR3; pCur = pCur->pNextR3)
755	{
756	pCur->pNextR0 = pCur->pNextR3->pSelfR0;
757	pCur->pNextRC = pCur->pNextR3->pSelfRC;
758	}
759
760	Assert(pCur->pNextR0 == NIL_RTR0PTR);
761	Assert(pCur->pNextRC == NIL_RTRCPTR);
762	}
763	else
764	{
765	Assert(pVM->pgm.s.pRamRangesXR0 == NIL_RTR0PTR);
766	Assert(pVM->pgm.s.pRamRangesXRC == NIL_RTRCPTR);
767	}
768	ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
769
770	pgmR3PhysRebuildRamRangeSearchTrees(pVM);
771	}
772
773
774	/**
775	* Links a new RAM range into the list.
776	*
777	* @param pVM Pointer to the shared VM structure.
778	* @param pNew Pointer to the new list entry.
779	* @param pPrev Pointer to the previous list entry. If NULL, insert as head.
780	*/
781	static void pgmR3PhysLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, PPGMRAMRANGE pPrev)
782	{
783	AssertMsg(pNew->pszDesc, ("%RGp-%RGp\n", pNew->GCPhys, pNew->GCPhysLast));
784	Assert((pNew->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pNew->pSelfR0 == MMHyperCCToR0(pVM, pNew));
785	Assert((pNew->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pNew->pSelfRC == MMHyperCCToRC(pVM, pNew));
786
787	pgmLock(pVM);
788
789	PPGMRAMRANGE pRam = pPrev ? pPrev->pNextR3 : pVM->pgm.s.pRamRangesXR3;
790	pNew->pNextR3 = pRam;
791	pNew->pNextR0 = pRam ? pRam->pSelfR0 : NIL_RTR0PTR;
792	pNew->pNextRC = pRam ? pRam->pSelfRC : NIL_RTRCPTR;
793
794	if (pPrev)
795	{
796	pPrev->pNextR3 = pNew;
797	pPrev->pNextR0 = pNew->pSelfR0;
798	pPrev->pNextRC = pNew->pSelfRC;
799	}
800	else
801	{
802	pVM->pgm.s.pRamRangesXR3 = pNew;
803	pVM->pgm.s.pRamRangesXR0 = pNew->pSelfR0;
804	pVM->pgm.s.pRamRangesXRC = pNew->pSelfRC;
805	}
806	ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
807
808	pgmR3PhysRebuildRamRangeSearchTrees(pVM);
809	pgmUnlock(pVM);
810	}
811
812
813	/**
814	* Unlink an existing RAM range from the list.
815	*
816	* @param pVM Pointer to the shared VM structure.
817	* @param pRam Pointer to the new list entry.
818	* @param pPrev Pointer to the previous list entry. If NULL, insert as head.
819	*/
820	static void pgmR3PhysUnlinkRamRange2(PVM pVM, PPGMRAMRANGE pRam, PPGMRAMRANGE pPrev)
821	{
822	Assert(pPrev ? pPrev->pNextR3 == pRam : pVM->pgm.s.pRamRangesXR3 == pRam);
823	Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pRam->pSelfR0 == MMHyperCCToR0(pVM, pRam));
824	Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pRam->pSelfRC == MMHyperCCToRC(pVM, pRam));
825
826	pgmLock(pVM);
827
828	PPGMRAMRANGE pNext = pRam->pNextR3;
829	if (pPrev)
830	{
831	pPrev->pNextR3 = pNext;
832	pPrev->pNextR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
833	pPrev->pNextRC = pNext ? pNext->pSelfRC : NIL_RTRCPTR;
834	}
835	else
836	{
837	Assert(pVM->pgm.s.pRamRangesXR3 == pRam);
838	pVM->pgm.s.pRamRangesXR3 = pNext;
839	pVM->pgm.s.pRamRangesXR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
840	pVM->pgm.s.pRamRangesXRC = pNext ? pNext->pSelfRC : NIL_RTRCPTR;
841	}
842	ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
843
844	pgmR3PhysRebuildRamRangeSearchTrees(pVM);
845	pgmUnlock(pVM);
846	}
847
848
849	/**
850	* Unlink an existing RAM range from the list.
851	*
852	* @param pVM Pointer to the shared VM structure.
853	* @param pRam Pointer to the new list entry.
854	*/
855	static void pgmR3PhysUnlinkRamRange(PVM pVM, PPGMRAMRANGE pRam)
856	{
857	pgmLock(pVM);
858
859	/* find prev. */
860	PPGMRAMRANGE pPrev = NULL;
861	PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesXR3;
862	while (pCur != pRam)
863	{
864	pPrev = pCur;
865	pCur = pCur->pNextR3;
866	}
867	AssertFatal(pCur);
868
869	pgmR3PhysUnlinkRamRange2(pVM, pRam, pPrev);
870	pgmUnlock(pVM);
871	}
872
873
874	/**
875	* Frees a range of pages, replacing them with ZERO pages of the specified type.
876	*
877	* @returns VBox status code.
878	* @param pVM The VM handle.
879	* @param pRam The RAM range in which the pages resides.
880	* @param GCPhys The address of the first page.
881	* @param GCPhysLast The address of the last page.
882	* @param uType The page type to replace then with.
883	*/
884	static int pgmR3PhysFreePageRange(PVM pVM, PPGMRAMRANGE pRam, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast, uint8_t uType)
885	{
886	PGM_LOCK_ASSERT_OWNER(pVM);
887	uint32_t cPendingPages = 0;
888	PGMMFREEPAGESREQ pReq;
889	int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
890	AssertLogRelRCReturn(rc, rc);
891
892	/* Iterate the pages. */
893	PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
894	uint32_t cPagesLeft = ((GCPhysLast - GCPhys) >> PAGE_SHIFT) + 1;
895	while (cPagesLeft-- > 0)
896	{
897	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys);
898	AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
899
900	PGM_PAGE_SET_TYPE(pVM, pPageDst, uType);
901
902	GCPhys += PAGE_SIZE;
903	pPageDst++;
904	}
905
906	if (cPendingPages)
907	{
908	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
909	AssertLogRelRCReturn(rc, rc);
910	}
911	GMMR3FreePagesCleanup(pReq);
912
913	return rc;
914	}
915
916	#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) \|\| defined(RT_OS_SOLARIS) \|\| defined(RT_OS_LINUX) \|\| defined(RT_OS_FREEBSD))
917	/**
918	* Rendezvous callback used by PGMR3ChangeMemBalloon that changes the memory balloon size
919	*
920	* This is only called on one of the EMTs while the other ones are waiting for
921	* it to complete this function.
922	*
923	* @returns VINF_SUCCESS (VBox strict status code).
924	* @param pVM The VM handle.
925	* @param pVCpu The VMCPU for the EMT we're being called on. Unused.
926	* @param pvUser User parameter
927	*/
928	static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysChangeMemBalloonRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
929	{
930	uintptr_t paUser = (uintptr_t )pvUser;
931	bool fInflate = !!paUser[0];
932	unsigned cPages = paUser[1];
933	RTGCPHYS paPhysPage = (RTGCPHYS )paUser[2];
934	uint32_t cPendingPages = 0;
935	PGMMFREEPAGESREQ pReq;
936	int rc;
937
938	Log(("pgmR3PhysChangeMemBalloonRendezvous: %s %x pages\n", (fInflate) ? "inflate" : "deflate", cPages));
939	pgmLock(pVM);
940
941	if (fInflate)
942	{
943	/* Flush the PGM pool cache as we might have stale references to pages that we just freed. */
944	pgmR3PoolClearAllRendezvous(pVM, pVCpu, NULL);
945
946	/* Replace pages with ZERO pages. */
947	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
948	if (RT_FAILURE(rc))
949	{
950	pgmUnlock(pVM);
951	AssertLogRelRC(rc);
952	return rc;
953	}
954
955	/* Iterate the pages. */
956	for (unsigned i = 0; i < cPages; i++)
957	{
958	PPGMPAGE pPage = pgmPhysGetPage(pVM, paPhysPage[i]);
959	if ( pPage == NULL
960	\|\| PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM)
961	{
962	Log(("pgmR3PhysChangeMemBalloonRendezvous: invalid physical page %RGp pPage->u3Type=%d\n", paPhysPage[i], pPage ? PGM_PAGE_GET_TYPE(pPage) : 0));
963	break;
964	}
965
966	LogFlow(("balloon page: %RGp\n", paPhysPage[i]));
967
968	/* Flush the shadow PT if this page was previously used as a guest page table. */
969	pgmPoolFlushPageByGCPhys(pVM, paPhysPage[i]);
970
971	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, paPhysPage[i]);
972	if (RT_FAILURE(rc))
973	{
974	pgmUnlock(pVM);
975	AssertLogRelRC(rc);
976	return rc;
977	}
978	Assert(PGM_PAGE_IS_ZERO(pPage));
979	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_BALLOONED);
980	}
981
982	if (cPendingPages)
983	{
984	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
985	if (RT_FAILURE(rc))
986	{
987	pgmUnlock(pVM);
988	AssertLogRelRC(rc);
989	return rc;
990	}
991	}
992	GMMR3FreePagesCleanup(pReq);
993	}
994	else
995	{
996	/* Iterate the pages. */
997	for (unsigned i = 0; i < cPages; i++)
998	{
999	PPGMPAGE pPage = pgmPhysGetPage(pVM, paPhysPage[i]);
1000	AssertBreak(pPage && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM);
1001
1002	LogFlow(("Free ballooned page: %RGp\n", paPhysPage[i]));
1003
1004	Assert(PGM_PAGE_IS_BALLOONED(pPage));
1005
1006	/* Change back to zero page. */
1007	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
1008	}
1009
1010	/* Note that we currently do not map any ballooned pages in our shadow page tables, so no need to flush the pgm pool. */
1011	}
1012
1013	/* Notify GMM about the balloon change. */
1014	rc = GMMR3BalloonedPages(pVM, (fInflate) ? GMMBALLOONACTION_INFLATE : GMMBALLOONACTION_DEFLATE, cPages);
1015	if (RT_SUCCESS(rc))
1016	{
1017	if (!fInflate)
1018	{
1019	Assert(pVM->pgm.s.cBalloonedPages >= cPages);
1020	pVM->pgm.s.cBalloonedPages -= cPages;
1021	}
1022	else
1023	pVM->pgm.s.cBalloonedPages += cPages;
1024	}
1025
1026	pgmUnlock(pVM);
1027
1028	/* Flush the recompiler's TLB as well. */
1029	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1030	CPUMSetChangedFlags(&pVM->aCpus[i], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
1031
1032	AssertLogRelRC(rc);
1033	return rc;
1034	}
1035
1036	/**
1037	* Frees a range of ram pages, replacing them with ZERO pages; helper for PGMR3PhysFreeRamPages
1038	*
1039	* @returns VBox status code.
1040	* @param pVM The VM handle.
1041	* @param fInflate Inflate or deflate memory balloon
1042	* @param cPages Number of pages to free
1043	* @param paPhysPage Array of guest physical addresses
1044	*/
1045	static DECLCALLBACK(void) pgmR3PhysChangeMemBalloonHelper(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
1046	{
1047	uintptr_t paUser[3];
1048
1049	paUser[0] = fInflate;
1050	paUser[1] = cPages;
1051	paUser[2] = (uintptr_t)paPhysPage;
1052	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
1053	AssertRC(rc);
1054
1055	/* Made a copy in PGMR3PhysFreeRamPages; free it here. */
1056	RTMemFree(paPhysPage);
1057	}
1058	#endif
1059
1060	/**
1061	* Inflate or deflate a memory balloon
1062	*
1063	* @returns VBox status code.
1064	* @param pVM The VM handle.
1065	* @param fInflate Inflate or deflate memory balloon
1066	* @param cPages Number of pages to free
1067	* @param paPhysPage Array of guest physical addresses
1068	*/
1069	VMMR3DECL(int) PGMR3PhysChangeMemBalloon(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
1070	{
1071	/* This must match GMMR0Init; currently we only support memory ballooning on all 64-bit hosts except Mac OS X */
1072	#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) \|\| defined(RT_OS_SOLARIS) \|\| defined(RT_OS_LINUX) \|\| defined(RT_OS_FREEBSD))
1073	int rc;
1074
1075	/* Older additions (ancient non-functioning balloon code) pass wrong physical addresses. */
1076	AssertReturn(!(paPhysPage[0] & 0xfff), VERR_INVALID_PARAMETER);
1077
1078	/* We own the IOM lock here and could cause a deadlock by waiting for another VCPU that is blocking on the IOM lock.
1079	* In the SMP case we post a request packet to postpone the job.
1080	*/
1081	if (pVM->cCpus > 1)
1082	{
1083	unsigned cbPhysPage = cPages * sizeof(paPhysPage[0]);
1084	RTGCPHYS paPhysPageCopy = (RTGCPHYS )RTMemAlloc(cbPhysPage);
1085	AssertReturn(paPhysPageCopy, VERR_NO_MEMORY);
1086
1087	memcpy(paPhysPageCopy, paPhysPage, cbPhysPage);
1088
1089	rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysChangeMemBalloonHelper, 4, pVM, fInflate, cPages, paPhysPageCopy);
1090	AssertRC(rc);
1091	}
1092	else
1093	{
1094	uintptr_t paUser[3];
1095
1096	paUser[0] = fInflate;
1097	paUser[1] = cPages;
1098	paUser[2] = (uintptr_t)paPhysPage;
1099	rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
1100	AssertRC(rc);
1101	}
1102	return rc;
1103	#else
1104	return VERR_NOT_IMPLEMENTED;
1105	#endif
1106	}
1107
1108	/**
1109	* Rendezvous callback used by PGMR3WriteProtectRAM that write protects all
1110	* physical RAM.
1111	*
1112	* This is only called on one of the EMTs while the other ones are waiting for
1113	* it to complete this function.
1114	*
1115	* @returns VINF_SUCCESS (VBox strict status code).
1116	* @param pVM The VM handle.
1117	* @param pVCpu The VMCPU for the EMT we're being called on. Unused.
1118	* @param pvUser User parameter, unused.
1119	*/
1120	static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysWriteProtectRAMRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
1121	{
1122	int rc = VINF_SUCCESS;
1123	NOREF(pvUser);
1124
1125	pgmLock(pVM);
1126	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1127	pgmPoolResetDirtyPages(pVM);
1128	#endif
1129
1130	/** @todo pointless to write protect the physical page pointed to by RSP. */
1131
1132	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
1133	pRam;
1134	pRam = pRam->CTX_SUFF(pNext))
1135	{
1136	uint32_t cPages = pRam->cb >> PAGE_SHIFT;
1137	for (uint32_t iPage = 0; iPage < cPages; iPage++)
1138	{
1139	PPGMPAGE pPage = &pRam->aPages[iPage];
1140	PGMPAGETYPE enmPageType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
1141
1142	if ( RT_LIKELY(enmPageType == PGMPAGETYPE_RAM)
1143	\|\| enmPageType == PGMPAGETYPE_MMIO2)
1144	{
1145	/*
1146	* A RAM page.
1147	*/
1148	switch (PGM_PAGE_GET_STATE(pPage))
1149	{
1150	case PGM_PAGE_STATE_ALLOCATED:
1151	/** @todo Optimize this: Don't always re-enable write
1152	* monitoring if the page is known to be very busy. */
1153	if (PGM_PAGE_IS_WRITTEN_TO(pPage))
1154	{
1155	PGM_PAGE_CLEAR_WRITTEN_TO(pVM, pPage);
1156	/* Remember this dirty page for the next (memory) sync. */
1157	PGM_PAGE_SET_FT_DIRTY(pPage);
1158	}
1159
1160	pgmPhysPageWriteMonitor(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1161	break;
1162
1163	case PGM_PAGE_STATE_SHARED:
1164	AssertFailed();
1165	break;
1166
1167	case PGM_PAGE_STATE_WRITE_MONITORED: /* nothing to change. */
1168	default:
1169	break;
1170	}
1171	}
1172	}
1173	}
1174	pgmR3PoolWriteProtectPages(pVM);
1175	PGM_INVL_ALL_VCPU_TLBS(pVM);
1176	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1177	CPUMSetChangedFlags(&pVM->aCpus[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
1178
1179	pgmUnlock(pVM);
1180	return rc;
1181	}
1182
1183	/**
1184	* Protect all physical RAM to monitor writes
1185	*
1186	* @returns VBox status code.
1187	* @param pVM The VM handle.
1188	*/
1189	VMMR3DECL(int) PGMR3PhysWriteProtectRAM(PVM pVM)
1190	{
1191	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
1192
1193	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysWriteProtectRAMRendezvous, NULL);
1194	AssertRC(rc);
1195	return rc;
1196	}
1197
1198	/**
1199	* Enumerate all dirty FT pages.
1200	*
1201	* @returns VBox status code.
1202	* @param pVM The VM handle.
1203	* @param pfnEnum Enumerate callback handler.
1204	* @param pvUser Enumerate callback handler parameter.
1205	*/
1206	VMMR3DECL(int) PGMR3PhysEnumDirtyFTPages(PVM pVM, PFNPGMENUMDIRTYFTPAGES pfnEnum, void *pvUser)
1207	{
1208	int rc = VINF_SUCCESS;
1209
1210	pgmLock(pVM);
1211	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
1212	pRam;
1213	pRam = pRam->CTX_SUFF(pNext))
1214	{
1215	uint32_t cPages = pRam->cb >> PAGE_SHIFT;
1216	for (uint32_t iPage = 0; iPage < cPages; iPage++)
1217	{
1218	PPGMPAGE pPage = &pRam->aPages[iPage];
1219	PGMPAGETYPE enmPageType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
1220
1221	if ( RT_LIKELY(enmPageType == PGMPAGETYPE_RAM)
1222	\|\| enmPageType == PGMPAGETYPE_MMIO2)
1223	{
1224	/*
1225	* A RAM page.
1226	*/
1227	switch (PGM_PAGE_GET_STATE(pPage))
1228	{
1229	case PGM_PAGE_STATE_ALLOCATED:
1230	case PGM_PAGE_STATE_WRITE_MONITORED:
1231	if ( !PGM_PAGE_IS_WRITTEN_TO(pPage) /* not very recently updated? */
1232	&& PGM_PAGE_IS_FT_DIRTY(pPage))
1233	{
1234	unsigned cbPageRange = PAGE_SIZE;
1235	unsigned iPageClean = iPage + 1;
1236	RTGCPHYS GCPhysPage = pRam->GCPhys + iPage * PAGE_SIZE;
1237	uint8_t *pu8Page = NULL;
1238	PGMPAGEMAPLOCK Lock;
1239
1240	/* Find the next clean page, so we can merge adjacent dirty pages. */
1241	for (; iPageClean < cPages; iPageClean++)
1242	{
1243	PPGMPAGE pPageNext = &pRam->aPages[iPageClean];
1244	if ( RT_UNLIKELY(PGM_PAGE_GET_TYPE(pPageNext) != PGMPAGETYPE_RAM)
1245	\|\| PGM_PAGE_GET_STATE(pPageNext) != PGM_PAGE_STATE_ALLOCATED
1246	\|\| PGM_PAGE_IS_WRITTEN_TO(pPageNext)
1247	\|\| !PGM_PAGE_IS_FT_DIRTY(pPageNext)
1248	/* Crossing a chunk boundary? */
1249	\|\| (GCPhysPage & GMM_PAGEID_IDX_MASK) != ((GCPhysPage + cbPageRange) & GMM_PAGEID_IDX_MASK)
1250	)
1251	break;
1252
1253	cbPageRange += PAGE_SIZE;
1254	}
1255
1256	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysPage, (const void **)&pu8Page, &Lock);
1257	if (RT_SUCCESS(rc))
1258	{
1259	/** @todo this is risky; the range might be changed, but little choice as the sync
1260	* costs a lot of time. */
1261	pgmUnlock(pVM);
1262	pfnEnum(pVM, GCPhysPage, pu8Page, cbPageRange, pvUser);
1263	pgmLock(pVM);
1264	PGMPhysReleasePageMappingLock(pVM, &Lock);
1265	}
1266
1267	for (iPage; iPage < iPageClean; iPage++)
1268	PGM_PAGE_CLEAR_FT_DIRTY(&pRam->aPages[iPage]);
1269
1270	iPage = iPageClean - 1;
1271	}
1272	break;
1273	}
1274	}
1275	}
1276	}
1277	pgmUnlock(pVM);
1278	return rc;
1279	}
1280
1281
1282	/**
1283	* Gets the number of ram ranges.
1284	*
1285	* @returns Number of ram ranges. Returns UINT32_MAX if @a pVM is invalid.
1286	* @param pVM The VM handle.
1287	*/
1288	VMMR3DECL(uint32_t) PGMR3PhysGetRamRangeCount(PVM pVM)
1289	{
1290	VM_ASSERT_VALID_EXT_RETURN(pVM, UINT32_MAX);
1291
1292	pgmLock(pVM);
1293	uint32_t cRamRanges = 0;
1294	for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRangesX); pCur; pCur = pCur->CTX_SUFF(pNext))
1295	cRamRanges++;
1296	pgmUnlock(pVM);
1297	return cRamRanges;
1298	}
1299
1300
1301	/**
1302	* Get information about a range.
1303	*
1304	* @returns VINF_SUCCESS or VERR_OUT_OF_RANGE.
1305	* @param pVM The VM handle
1306	* @param iRange The ordinal of the range.
1307	* @param pGCPhysStart Where to return the start of the range. Optional.
1308	* @param pGCPhysLast Where to return the address of the last byte in the
1309	* range. Optional.
1310	* @param pfIsMmio Where to indicate that this is a pure MMIO range.
1311	* Optional.
1312	*/
1313	VMMR3DECL(int) PGMR3PhysGetRange(PVM pVM, uint32_t iRange, PRTGCPHYS pGCPhysStart, PRTGCPHYS pGCPhysLast,
1314	const char *ppszDesc, bool pfIsMmio)
1315	{
1316	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
1317
1318	pgmLock(pVM);
1319	uint32_t iCurRange = 0;
1320	for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRangesX); pCur; pCur = pCur->CTX_SUFF(pNext), iCurRange++)
1321	if (iCurRange == iRange)
1322	{
1323	if (pGCPhysStart)
1324	*pGCPhysStart = pCur->GCPhys;
1325	if (pGCPhysLast)
1326	*pGCPhysLast = pCur->GCPhysLast;
1327	if (pfIsMmio)
1328	*pfIsMmio = !!(pCur->fFlags & PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO);
1329
1330	pgmUnlock(pVM);
1331	return VINF_SUCCESS;
1332	}
1333	pgmUnlock(pVM);
1334	return VERR_OUT_OF_RANGE;
1335	}
1336
1337
1338	/**
1339	* Query the amount of free memory inside VMMR0
1340	*
1341	* @returns VBox status code.
1342	* @param pVM The VM handle.
1343	* @param pcbAllocMem Where to return the amount of memory allocated
1344	* by VMs.
1345	* @param pcbFreeMem Where to return the amount of memory that is
1346	* allocated from the host but not currently used
1347	* by any VMs.
1348	* @param pcbBallonedMem Where to return the sum of memory that is
1349	* currently ballooned by the VMs.
1350	* @param pcbSharedMem Where to return the amount of memory that is
1351	* currently shared.
1352	*/
1353	VMMR3DECL(int) PGMR3QueryGlobalMemoryStats(PVM pVM, uint64_t pcbAllocMem, uint64_t pcbFreeMem,
1354	uint64_t pcbBallonedMem, uint64_t pcbSharedMem)
1355	{
1356	uint64_t cAllocPages = 0;
1357	uint64_t cFreePages = 0;
1358	uint64_t cBalloonPages = 0;
1359	uint64_t cSharedPages = 0;
1360	int rc = GMMR3QueryHypervisorMemoryStats(pVM, &cAllocPages, &cFreePages, &cBalloonPages, &cSharedPages);
1361	AssertRCReturn(rc, rc);
1362
1363	if (pcbAllocMem)
1364	pcbAllocMem = cAllocPages _4K;
1365
1366	if (pcbFreeMem)
1367	pcbFreeMem = cFreePages _4K;
1368
1369	if (pcbBallonedMem)
1370	pcbBallonedMem = cBalloonPages _4K;
1371
1372	if (pcbSharedMem)
1373	pcbSharedMem = cSharedPages _4K;
1374
1375	Log(("PGMR3QueryVMMMemoryStats: all=%llx free=%llx ballooned=%llx shared=%llx\n",
1376	cAllocPages, cFreePages, cBalloonPages, cSharedPages));
1377	return VINF_SUCCESS;
1378	}
1379
1380
1381	/**
1382	* Query memory stats for the VM.
1383	*
1384	* @returns VBox status code.
1385	* @param pVM The VM handle.
1386	* @param pcbTotalMem Where to return total amount memory the VM may
1387	* possibly use.
1388	* @param pcbPrivateMem Where to return the amount of private memory
1389	* currently allocated.
1390	* @param pcbSharedMem Where to return the amount of actually shared
1391	* memory currently used by the VM.
1392	* @param pcbZeroMem Where to return the amount of memory backed by
1393	* zero pages.
1394	*
1395	* @remarks The total mem is normally larger than the sum of the three
1396	* components. There are two reasons for this, first the amount of
1397	* shared memory is what we're sure is shared instead of what could
1398	* possibly be shared with someone. Secondly, because the total may
1399	* include some pure MMIO pages that doesn't go into any of the three
1400	* sub-counts.
1401	*
1402	* @todo Why do we return reused shared pages instead of anything that could
1403	* potentially be shared? Doesn't this mean the first VM gets a much
1404	* lower number of shared pages?
1405	*/
1406	VMMR3DECL(int) PGMR3QueryMemoryStats(PVM pVM, uint64_t pcbTotalMem, uint64_t pcbPrivateMem,
1407	uint64_t pcbSharedMem, uint64_t pcbZeroMem)
1408	{
1409	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
1410
1411	if (pcbTotalMem)
1412	pcbTotalMem = (uint64_t)pVM->pgm.s.cAllPages PAGE_SIZE;
1413
1414	if (pcbPrivateMem)
1415	pcbPrivateMem = (uint64_t)pVM->pgm.s.cPrivatePages PAGE_SIZE;
1416
1417	if (pcbSharedMem)
1418	pcbSharedMem = (uint64_t)pVM->pgm.s.cReusedSharedPages PAGE_SIZE;
1419
1420	if (pcbZeroMem)
1421	pcbZeroMem = (uint64_t)pVM->pgm.s.cZeroPages PAGE_SIZE;
1422
1423	Log(("PGMR3QueryMemoryStats: all=%x private=%x reused=%x zero=%x\n", pVM->pgm.s.cAllPages, pVM->pgm.s.cPrivatePages, pVM->pgm.s.cReusedSharedPages, pVM->pgm.s.cZeroPages));
1424	return VINF_SUCCESS;
1425	}
1426
1427
1428	/**
1429	* PGMR3PhysRegisterRam worker that initializes and links a RAM range.
1430	*
1431	* @param pVM The VM handle.
1432	* @param pNew The new RAM range.
1433	* @param GCPhys The address of the RAM range.
1434	* @param GCPhysLast The last address of the RAM range.
1435	* @param RCPtrNew The RC address if the range is floating. NIL_RTRCPTR
1436	* if in HMA.
1437	* @param R0PtrNew Ditto for R0.
1438	* @param pszDesc The description.
1439	* @param pPrev The previous RAM range (for linking).
1440	*/
1441	static void pgmR3PhysInitAndLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast,
1442	RTRCPTR RCPtrNew, RTR0PTR R0PtrNew, const char *pszDesc, PPGMRAMRANGE pPrev)
1443	{
1444	/*
1445	* Initialize the range.
1446	*/
1447	pNew->pSelfR0 = R0PtrNew != NIL_RTR0PTR ? R0PtrNew : MMHyperCCToR0(pVM, pNew);
1448	pNew->pSelfRC = RCPtrNew != NIL_RTRCPTR ? RCPtrNew : MMHyperCCToRC(pVM, pNew);
1449	pNew->GCPhys = GCPhys;
1450	pNew->GCPhysLast = GCPhysLast;
1451	pNew->cb = GCPhysLast - GCPhys + 1;
1452	pNew->pszDesc = pszDesc;
1453	pNew->fFlags = RCPtrNew != NIL_RTRCPTR ? PGM_RAM_RANGE_FLAGS_FLOATING : 0;
1454	pNew->pvR3 = NULL;
1455	pNew->paLSPages = NULL;
1456
1457	uint32_t const cPages = pNew->cb >> PAGE_SHIFT;
1458	RTGCPHYS iPage = cPages;
1459	while (iPage-- > 0)
1460	PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_RAM);
1461
1462	/* Update the page count stats. */
1463	pVM->pgm.s.cZeroPages += cPages;
1464	pVM->pgm.s.cAllPages += cPages;
1465
1466	/*
1467	* Link it.
1468	*/
1469	pgmR3PhysLinkRamRange(pVM, pNew, pPrev);
1470	}
1471
1472
1473	/**
1474	* Relocate a floating RAM range.
1475	*
1476	* @copydoc FNPGMRELOCATE.
1477	*/
1478	static DECLCALLBACK(bool) pgmR3PhysRamRangeRelocate(PVM pVM, RTGCPTR GCPtrOld, RTGCPTR GCPtrNew, PGMRELOCATECALL enmMode, void *pvUser)
1479	{
1480	PPGMRAMRANGE pRam = (PPGMRAMRANGE)pvUser;
1481	Assert(pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING);
1482	Assert(pRam->pSelfRC == GCPtrOld + PAGE_SIZE);
1483
1484	switch (enmMode)
1485	{
1486	case PGMRELOCATECALL_SUGGEST:
1487	return true;
1488
1489	case PGMRELOCATECALL_RELOCATE:
1490	{
1491	/*
1492	* Update myself, then relink all the ranges and flush the RC TLB.
1493	*/
1494	pgmLock(pVM);
1495
1496	pRam->pSelfRC = (RTRCPTR)(GCPtrNew + PAGE_SIZE);
1497
1498	pgmR3PhysRelinkRamRanges(pVM);
1499	for (unsigned i = 0; i < PGM_RAMRANGE_TLB_ENTRIES; i++)
1500	pVM->pgm.s.apRamRangesTlbRC[i] = NIL_RTRCPTR;
1501
1502	pgmUnlock(pVM);
1503	return true;
1504	}
1505
1506	default:
1507	AssertFailedReturn(false);
1508	}
1509	}
1510
1511
1512	/**
1513	* PGMR3PhysRegisterRam worker that registers a high chunk.
1514	*
1515	* @returns VBox status code.
1516	* @param pVM The VM handle.
1517	* @param GCPhys The address of the RAM.
1518	* @param cRamPages The number of RAM pages to register.
1519	* @param cbChunk The size of the PGMRAMRANGE guest mapping.
1520	* @param iChunk The chunk number.
1521	* @param pszDesc The RAM range description.
1522	* @param ppPrev Previous RAM range pointer. In/Out.
1523	*/
1524	static int pgmR3PhysRegisterHighRamChunk(PVM pVM, RTGCPHYS GCPhys, uint32_t cRamPages,
1525	uint32_t cbChunk, uint32_t iChunk, const char *pszDesc,
1526	PPGMRAMRANGE *ppPrev)
1527	{
1528	const char *pszDescChunk = iChunk == 0
1529	? pszDesc
1530	: MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s (#%u)", pszDesc, iChunk + 1);
1531	AssertReturn(pszDescChunk, VERR_NO_MEMORY);
1532
1533	/*
1534	* Allocate memory for the new chunk.
1535	*/
1536	size_t const cChunkPages = RT_ALIGN_Z(RT_UOFFSETOF(PGMRAMRANGE, aPages[cRamPages]), PAGE_SIZE) >> PAGE_SHIFT;
1537	PSUPPAGE paChunkPages = (PSUPPAGE)RTMemTmpAllocZ(sizeof(SUPPAGE) * cChunkPages);
1538	AssertReturn(paChunkPages, VERR_NO_TMP_MEMORY);
1539	RTR0PTR R0PtrChunk = NIL_RTR0PTR;
1540	void *pvChunk = NULL;
1541	int rc = SUPR3PageAllocEx(cChunkPages, 0 /fFlags/, &pvChunk,
1542	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1543	VMMIsHwVirtExtForced(pVM) ? &R0PtrChunk : NULL,
1544	#else
1545	NULL,
1546	#endif
1547	paChunkPages);
1548	if (RT_SUCCESS(rc))
1549	{
1550	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1551	if (!VMMIsHwVirtExtForced(pVM))
1552	R0PtrChunk = NIL_RTR0PTR;
1553	#else
1554	R0PtrChunk = (uintptr_t)pvChunk;
1555	#endif
1556	memset(pvChunk, 0, cChunkPages << PAGE_SHIFT);
1557
1558	PPGMRAMRANGE pNew = (PPGMRAMRANGE)pvChunk;
1559
1560	/*
1561	* Create a mapping and map the pages into it.
1562	* We push these in below the HMA.
1563	*/
1564	RTGCPTR GCPtrChunkMap = pVM->pgm.s.GCPtrPrevRamRangeMapping - cbChunk;
1565	rc = PGMR3MapPT(pVM, GCPtrChunkMap, cbChunk, 0 /fFlags/, pgmR3PhysRamRangeRelocate, pNew, pszDescChunk);
1566	if (RT_SUCCESS(rc))
1567	{
1568	pVM->pgm.s.GCPtrPrevRamRangeMapping = GCPtrChunkMap;
1569
1570	RTGCPTR const GCPtrChunk = GCPtrChunkMap + PAGE_SIZE;
1571	RTGCPTR GCPtrPage = GCPtrChunk;
1572	for (uint32_t iPage = 0; iPage < cChunkPages && RT_SUCCESS(rc); iPage++, GCPtrPage += PAGE_SIZE)
1573	rc = PGMMap(pVM, GCPtrPage, paChunkPages[iPage].Phys, PAGE_SIZE, 0);
1574	if (RT_SUCCESS(rc))
1575	{
1576	/*
1577	* Ok, init and link the range.
1578	*/
1579	pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhys + ((RTGCPHYS)cRamPages << PAGE_SHIFT) - 1,
1580	(RTRCPTR)GCPtrChunk, R0PtrChunk, pszDescChunk, *ppPrev);
1581	*ppPrev = pNew;
1582	}
1583	}
1584
1585	if (RT_FAILURE(rc))
1586	SUPR3PageFreeEx(pvChunk, cChunkPages);
1587	}
1588
1589	RTMemTmpFree(paChunkPages);
1590	return rc;
1591	}
1592
1593
1594	/**
1595	* Sets up a range RAM.
1596	*
1597	* This will check for conflicting registrations, make a resource
1598	* reservation for the memory (with GMM), and setup the per-page
1599	* tracking structures (PGMPAGE).
1600	*
1601	* @returns VBox status code.
1602	* @param pVM Pointer to the shared VM structure.
1603	* @param GCPhys The physical address of the RAM.
1604	* @param cb The size of the RAM.
1605	* @param pszDesc The description - not copied, so, don't free or change it.
1606	*/
1607	VMMR3DECL(int) PGMR3PhysRegisterRam(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, const char *pszDesc)
1608	{
1609	/*
1610	* Validate input.
1611	*/
1612	Log(("PGMR3PhysRegisterRam: GCPhys=%RGp cb=%RGp pszDesc=%s\n", GCPhys, cb, pszDesc));
1613	AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
1614	AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
1615	AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
1616	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
1617	AssertMsgReturn(GCPhysLast > GCPhys, ("The range wraps! GCPhys=%RGp cb=%RGp\n", GCPhys, cb), VERR_INVALID_PARAMETER);
1618	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
1619	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
1620
1621	pgmLock(pVM);
1622
1623	/*
1624	* Find range location and check for conflicts.
1625	* (We don't lock here because the locking by EMT is only required on update.)
1626	*/
1627	PPGMRAMRANGE pPrev = NULL;
1628	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
1629	while (pRam && GCPhysLast >= pRam->GCPhys)
1630	{
1631	if ( GCPhysLast >= pRam->GCPhys
1632	&& GCPhys <= pRam->GCPhysLast)
1633	AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
1634	GCPhys, GCPhysLast, pszDesc,
1635	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
1636	VERR_PGM_RAM_CONFLICT);
1637
1638	/* next */
1639	pPrev = pRam;
1640	pRam = pRam->pNextR3;
1641	}
1642
1643	/*
1644	* Register it with GMM (the API bitches).
1645	*/
1646	const RTGCPHYS cPages = cb >> PAGE_SHIFT;
1647	int rc = MMR3IncreaseBaseReservation(pVM, cPages);
1648	if (RT_FAILURE(rc))
1649	{
1650	pgmUnlock(pVM);
1651	return rc;
1652	}
1653
1654	if ( GCPhys >= _4G
1655	&& cPages > 256)
1656	{
1657	/*
1658	* The PGMRAMRANGE structures for the high memory can get very big.
1659	* In order to avoid SUPR3PageAllocEx allocation failures due to the
1660	* allocation size limit there and also to avoid being unable to find
1661	* guest mapping space for them, we split this memory up into 4MB in
1662	* (potential) raw-mode configs and 16MB chunks in forced AMD-V/VT-x
1663	* mode.
1664	*
1665	* The first and last page of each mapping are guard pages and marked
1666	* not-present. So, we've got 4186112 and 16769024 bytes available for
1667	* the PGMRAMRANGE structure.
1668	*
1669	* Note! The sizes used here will influence the saved state.
1670	*/
1671	uint32_t cbChunk;
1672	uint32_t cPagesPerChunk;
1673	if (VMMIsHwVirtExtForced(pVM))
1674	{
1675	cbChunk = 16U*_1M;
1676	cPagesPerChunk = 1048048; /* max ~1048059 */
1677	AssertCompile(sizeof(PGMRAMRANGE) + sizeof(PGMPAGE) * 1048048 < 16U_1M - PAGE_SIZE 2);
1678	}
1679	else
1680	{
1681	cbChunk = 4U*_1M;
1682	cPagesPerChunk = 261616; /* max ~261627 */
1683	AssertCompile(sizeof(PGMRAMRANGE) + sizeof(PGMPAGE) * 261616 < 4U_1M - PAGE_SIZE 2);
1684	}
1685	AssertRelease(RT_UOFFSETOF(PGMRAMRANGE, aPages[cPagesPerChunk]) + PAGE_SIZE * 2 <= cbChunk);
1686
1687	RTGCPHYS cPagesLeft = cPages;
1688	RTGCPHYS GCPhysChunk = GCPhys;
1689	uint32_t iChunk = 0;
1690	while (cPagesLeft > 0)
1691	{
1692	uint32_t cPagesInChunk = cPagesLeft;
1693	if (cPagesInChunk > cPagesPerChunk)
1694	cPagesInChunk = cPagesPerChunk;
1695
1696	rc = pgmR3PhysRegisterHighRamChunk(pVM, GCPhysChunk, cPagesInChunk, cbChunk, iChunk, pszDesc, &pPrev);
1697	AssertRCReturn(rc, rc);
1698
1699	/* advance */
1700	GCPhysChunk += (RTGCPHYS)cPagesInChunk << PAGE_SHIFT;
1701	cPagesLeft -= cPagesInChunk;
1702	iChunk++;
1703	}
1704	}
1705	else
1706	{
1707	/*
1708	* Allocate, initialize and link the new RAM range.
1709	*/
1710	const size_t cbRamRange = RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]);
1711	PPGMRAMRANGE pNew;
1712	rc = MMR3HyperAllocOnceNoRel(pVM, cbRamRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
1713	AssertLogRelMsgRCReturn(rc, ("cbRamRange=%zu\n", cbRamRange), rc);
1714
1715	pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhysLast, NIL_RTRCPTR, NIL_RTR0PTR, pszDesc, pPrev);
1716	}
1717	pgmPhysInvalidatePageMapTLB(pVM);
1718	pgmUnlock(pVM);
1719
1720	/*
1721	* Notify REM.
1722	*/
1723	REMR3NotifyPhysRamRegister(pVM, GCPhys, cb, REM_NOTIFY_PHYS_RAM_FLAGS_RAM);
1724
1725	return VINF_SUCCESS;
1726	}
1727
1728
1729	/**
1730	* Worker called by PGMR3InitFinalize if we're configured to pre-allocate RAM.
1731	*
1732	* We do this late in the init process so that all the ROM and MMIO ranges have
1733	* been registered already and we don't go wasting memory on them.
1734	*
1735	* @returns VBox status code.
1736	*
1737	* @param pVM Pointer to the shared VM structure.
1738	*/
1739	int pgmR3PhysRamPreAllocate(PVM pVM)
1740	{
1741	Assert(pVM->pgm.s.fRamPreAlloc);
1742	Log(("pgmR3PhysRamPreAllocate: enter\n"));
1743
1744	/*
1745	* Walk the RAM ranges and allocate all RAM pages, halt at
1746	* the first allocation error.
1747	*/
1748	uint64_t cPages = 0;
1749	uint64_t NanoTS = RTTimeNanoTS();
1750	pgmLock(pVM);
1751	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
1752	{
1753	PPGMPAGE pPage = &pRam->aPages[0];
1754	RTGCPHYS GCPhys = pRam->GCPhys;
1755	uint32_t cLeft = pRam->cb >> PAGE_SHIFT;
1756	while (cLeft-- > 0)
1757	{
1758	if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
1759	{
1760	switch (PGM_PAGE_GET_STATE(pPage))
1761	{
1762	case PGM_PAGE_STATE_ZERO:
1763	{
1764	int rc = pgmPhysAllocPage(pVM, pPage, GCPhys);
1765	if (RT_FAILURE(rc))
1766	{
1767	LogRel(("PGM: RAM Pre-allocation failed at %RGp (in %s) with rc=%Rrc\n", GCPhys, pRam->pszDesc, rc));
1768	pgmUnlock(pVM);
1769	return rc;
1770	}
1771	cPages++;
1772	break;
1773	}
1774
1775	case PGM_PAGE_STATE_BALLOONED:
1776	case PGM_PAGE_STATE_ALLOCATED:
1777	case PGM_PAGE_STATE_WRITE_MONITORED:
1778	case PGM_PAGE_STATE_SHARED:
1779	/* nothing to do here. */
1780	break;
1781	}
1782	}
1783
1784	/* next */
1785	pPage++;
1786	GCPhys += PAGE_SIZE;
1787	}
1788	}
1789	pgmUnlock(pVM);
1790	NanoTS = RTTimeNanoTS() - NanoTS;
1791
1792	LogRel(("PGM: Pre-allocated %llu pages in %llu ms\n", cPages, NanoTS / 1000000));
1793	Log(("pgmR3PhysRamPreAllocate: returns VINF_SUCCESS\n"));
1794	return VINF_SUCCESS;
1795	}
1796
1797
1798	/**
1799	* Resets (zeros) the RAM.
1800	*
1801	* ASSUMES that the caller owns the PGM lock.
1802	*
1803	* @returns VBox status code.
1804	* @param pVM Pointer to the shared VM structure.
1805	*/
1806	int pgmR3PhysRamReset(PVM pVM)
1807	{
1808	PGM_LOCK_ASSERT_OWNER(pVM);
1809
1810	/* Reset the memory balloon. */
1811	int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
1812	AssertRC(rc);
1813
1814	#ifdef VBOX_WITH_PAGE_SHARING
1815	/* Clear all registered shared modules. */
1816	rc = GMMR3ResetSharedModules(pVM);
1817	AssertRC(rc);
1818	#endif
1819	/* Reset counters. */
1820	pVM->pgm.s.cReusedSharedPages = 0;
1821	pVM->pgm.s.cBalloonedPages = 0;
1822
1823	/*
1824	* We batch up pages that should be freed instead of calling GMM for
1825	* each and every one of them.
1826	*/
1827	uint32_t cPendingPages = 0;
1828	PGMMFREEPAGESREQ pReq;
1829	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
1830	AssertLogRelRCReturn(rc, rc);
1831
1832	/*
1833	* Walk the ram ranges.
1834	*/
1835	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
1836	{
1837	uint32_t iPage = pRam->cb >> PAGE_SHIFT;
1838	AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
1839
1840	if (!pVM->pgm.s.fRamPreAlloc)
1841	{
1842	/* Replace all RAM pages by ZERO pages. */
1843	while (iPage-- > 0)
1844	{
1845	PPGMPAGE pPage = &pRam->aPages[iPage];
1846	switch (PGM_PAGE_GET_TYPE(pPage))
1847	{
1848	case PGMPAGETYPE_RAM:
1849	/* Do not replace pages part of a 2 MB continuous range
1850	with zero pages, but zero them instead. */
1851	if ( PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE
1852	\|\| PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED)
1853	{
1854	void *pvPage;
1855	rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
1856	AssertLogRelRCReturn(rc, rc);
1857	ASMMemZeroPage(pvPage);
1858	}
1859	else if (PGM_PAGE_IS_BALLOONED(pPage))
1860	{
1861	/* Turn into a zero page; the balloon status is lost when the VM reboots. */
1862	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
1863	}
1864	else if (!PGM_PAGE_IS_ZERO(pPage))
1865	{
1866	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1867	AssertLogRelRCReturn(rc, rc);
1868	}
1869	break;
1870
1871	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
1872	pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
1873	true /fDoAccounting/);
1874	break;
1875
1876	case PGMPAGETYPE_MMIO2:
1877	case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
1878	case PGMPAGETYPE_ROM:
1879	case PGMPAGETYPE_MMIO:
1880	break;
1881	default:
1882	AssertFailed();
1883	}
1884	} /* for each page */
1885	}
1886	else
1887	{
1888	/* Zero the memory. */
1889	while (iPage-- > 0)
1890	{
1891	PPGMPAGE pPage = &pRam->aPages[iPage];
1892	switch (PGM_PAGE_GET_TYPE(pPage))
1893	{
1894	case PGMPAGETYPE_RAM:
1895	switch (PGM_PAGE_GET_STATE(pPage))
1896	{
1897	case PGM_PAGE_STATE_ZERO:
1898	break;
1899
1900	case PGM_PAGE_STATE_BALLOONED:
1901	/* Turn into a zero page; the balloon status is lost when the VM reboots. */
1902	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
1903	break;
1904
1905	case PGM_PAGE_STATE_SHARED:
1906	case PGM_PAGE_STATE_WRITE_MONITORED:
1907	rc = pgmPhysPageMakeWritable(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1908	AssertLogRelRCReturn(rc, rc);
1909	/* no break */
1910
1911	case PGM_PAGE_STATE_ALLOCATED:
1912	{
1913	void *pvPage;
1914	rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
1915	AssertLogRelRCReturn(rc, rc);
1916	ASMMemZeroPage(pvPage);
1917	break;
1918	}
1919	}
1920	break;
1921
1922	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
1923	pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
1924	true /fDoAccounting/);
1925	break;
1926
1927	case PGMPAGETYPE_MMIO2:
1928	case PGMPAGETYPE_ROM_SHADOW:
1929	case PGMPAGETYPE_ROM:
1930	case PGMPAGETYPE_MMIO:
1931	break;
1932	default:
1933	AssertFailed();
1934
1935	}
1936	} /* for each page */
1937	}
1938
1939	}
1940
1941	/*
1942	* Finish off any pages pending freeing.
1943	*/
1944	if (cPendingPages)
1945	{
1946	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
1947	AssertLogRelRCReturn(rc, rc);
1948	}
1949	GMMR3FreePagesCleanup(pReq);
1950
1951	return VINF_SUCCESS;
1952	}
1953
1954	/**
1955	* Frees all RAM during VM termination
1956	*
1957	* ASSUMES that the caller owns the PGM lock.
1958	*
1959	* @returns VBox status code.
1960	* @param pVM Pointer to the shared VM structure.
1961	*/
1962	int pgmR3PhysRamTerm(PVM pVM)
1963	{
1964	PGM_LOCK_ASSERT_OWNER(pVM);
1965
1966	/* Reset the memory balloon. */
1967	int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
1968	AssertRC(rc);
1969
1970	#ifdef VBOX_WITH_PAGE_SHARING
1971	/* Clear all registered shared modules. */
1972	rc = GMMR3ResetSharedModules(pVM);
1973	AssertRC(rc);
1974	#endif
1975
1976	/*
1977	* We batch up pages that should be freed instead of calling GMM for
1978	* each and every one of them.
1979	*/
1980	uint32_t cPendingPages = 0;
1981	PGMMFREEPAGESREQ pReq;
1982	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
1983	AssertLogRelRCReturn(rc, rc);
1984
1985	/*
1986	* Walk the ram ranges.
1987	*/
1988	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
1989	{
1990	uint32_t iPage = pRam->cb >> PAGE_SHIFT;
1991	AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
1992
1993	/* Replace all RAM pages by ZERO pages. */
1994	while (iPage-- > 0)
1995	{
1996	PPGMPAGE pPage = &pRam->aPages[iPage];
1997	switch (PGM_PAGE_GET_TYPE(pPage))
1998	{
1999	case PGMPAGETYPE_RAM:
2000	/* Free all shared pages. Private pages are automatically freed during GMM VM cleanup. */
2001	if (PGM_PAGE_IS_SHARED(pPage))
2002	{
2003	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
2004	AssertLogRelRCReturn(rc, rc);
2005	}
2006	break;
2007
2008	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
2009	case PGMPAGETYPE_MMIO2:
2010	case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
2011	case PGMPAGETYPE_ROM:
2012	case PGMPAGETYPE_MMIO:
2013	break;
2014	default:
2015	AssertFailed();
2016	}
2017	} /* for each page */
2018	}
2019
2020	/*
2021	* Finish off any pages pending freeing.
2022	*/
2023	if (cPendingPages)
2024	{
2025	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2026	AssertLogRelRCReturn(rc, rc);
2027	}
2028	GMMR3FreePagesCleanup(pReq);
2029	return VINF_SUCCESS;
2030	}
2031
2032	/**
2033	* This is the interface IOM is using to register an MMIO region.
2034	*
2035	* It will check for conflicts and ensure that a RAM range structure
2036	* is present before calling the PGMR3HandlerPhysicalRegister API to
2037	* register the callbacks.
2038	*
2039	* @returns VBox status code.
2040	*
2041	* @param pVM Pointer to the shared VM structure.
2042	* @param GCPhys The start of the MMIO region.
2043	* @param cb The size of the MMIO region.
2044	* @param pfnHandlerR3 The address of the ring-3 handler. (IOMR3MMIOHandler)
2045	* @param pvUserR3 The user argument for R3.
2046	* @param pfnHandlerR0 The address of the ring-0 handler. (IOMMMIOHandler)
2047	* @param pvUserR0 The user argument for R0.
2048	* @param pfnHandlerRC The address of the RC handler. (IOMMMIOHandler)
2049	* @param pvUserRC The user argument for RC.
2050	* @param pszDesc The description of the MMIO region.
2051	*/
2052	VMMR3DECL(int) PGMR3PhysMMIORegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb,
2053	R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnHandlerR3, RTR3PTR pvUserR3,
2054	R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnHandlerR0, RTR0PTR pvUserR0,
2055	RCPTRTYPE(PFNPGMRCPHYSHANDLER) pfnHandlerRC, RTRCPTR pvUserRC,
2056	R3PTRTYPE(const char *) pszDesc)
2057	{
2058	/*
2059	* Assert on some assumption.
2060	*/
2061	VM_ASSERT_EMT(pVM);
2062	AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2063	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2064	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
2065	AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
2066
2067	int rc = pgmLock(pVM);
2068	AssertRCReturn(rc, rc);
2069
2070	/*
2071	* Make sure there's a RAM range structure for the region.
2072	*/
2073	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
2074	bool fRamExists = false;
2075	PPGMRAMRANGE pRamPrev = NULL;
2076	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2077	while (pRam && GCPhysLast >= pRam->GCPhys)
2078	{
2079	if ( GCPhysLast >= pRam->GCPhys
2080	&& GCPhys <= pRam->GCPhysLast)
2081	{
2082	/* Simplification: all within the same range. */
2083	AssertLogRelMsgReturnStmt( GCPhys >= pRam->GCPhys
2084	&& GCPhysLast <= pRam->GCPhysLast,
2085	("%RGp-%RGp (MMIO/%s) falls partly outside %RGp-%RGp (%s)\n",
2086	GCPhys, GCPhysLast, pszDesc,
2087	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
2088	pgmUnlock(pVM),
2089	VERR_PGM_RAM_CONFLICT);
2090
2091	/* Check that it's all RAM or MMIO pages. */
2092	PCPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2093	uint32_t cLeft = cb >> PAGE_SHIFT;
2094	while (cLeft-- > 0)
2095	{
2096	AssertLogRelMsgReturnStmt( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2097	\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO,
2098	("%RGp-%RGp (MMIO/%s): %RGp is not a RAM or MMIO page - type=%d desc=%s\n",
2099	GCPhys, GCPhysLast, pszDesc, PGM_PAGE_GET_TYPE(pPage), pRam->pszDesc),
2100	pgmUnlock(pVM),
2101	VERR_PGM_RAM_CONFLICT);
2102	pPage++;
2103	}
2104
2105	/* Looks good. */
2106	fRamExists = true;
2107	break;
2108	}
2109
2110	/* next */
2111	pRamPrev = pRam;
2112	pRam = pRam->pNextR3;
2113	}
2114	PPGMRAMRANGE pNew;
2115	if (fRamExists)
2116	{
2117	pNew = NULL;
2118
2119	/*
2120	* Make all the pages in the range MMIO/ZERO pages, freeing any
2121	* RAM pages currently mapped here. This might not be 100% correct
2122	* for PCI memory, but we're doing the same thing for MMIO2 pages.
2123	*/
2124	rc = pgmR3PhysFreePageRange(pVM, pRam, GCPhys, GCPhysLast, PGMPAGETYPE_MMIO);
2125	AssertRCReturnStmt(rc, pgmUnlock(pVM), rc);
2126
2127	/* Force a PGM pool flush as guest ram references have been changed. */
2128	/** @todo not entirely SMP safe; assuming for now the guest takes
2129	* care of this internally (not touch mapped mmio while changing the
2130	* mapping). */
2131	PVMCPU pVCpu = VMMGetCpu(pVM);
2132	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2133	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2134	}
2135	else
2136	{
2137
2138	/*
2139	* No RAM range, insert an ad hoc one.
2140	*
2141	* Note that we don't have to tell REM about this range because
2142	* PGMHandlerPhysicalRegisterEx will do that for us.
2143	*/
2144	Log(("PGMR3PhysMMIORegister: Adding ad hoc MMIO range for %RGp-%RGp %s\n", GCPhys, GCPhysLast, pszDesc));
2145
2146	const uint32_t cPages = cb >> PAGE_SHIFT;
2147	const size_t cbRamRange = RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]);
2148	rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]), 16, MM_TAG_PGM_PHYS, (void **)&pNew);
2149	AssertLogRelMsgRCReturnStmt(rc, ("cbRamRange=%zu\n", cbRamRange), pgmUnlock(pVM), rc);
2150
2151	/* Initialize the range. */
2152	pNew->pSelfR0 = MMHyperCCToR0(pVM, pNew);
2153	pNew->pSelfRC = MMHyperCCToRC(pVM, pNew);
2154	pNew->GCPhys = GCPhys;
2155	pNew->GCPhysLast = GCPhysLast;
2156	pNew->cb = cb;
2157	pNew->pszDesc = pszDesc;
2158	pNew->fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO;
2159	pNew->pvR3 = NULL;
2160	pNew->paLSPages = NULL;
2161
2162	uint32_t iPage = cPages;
2163	while (iPage-- > 0)
2164	PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_MMIO);
2165	Assert(PGM_PAGE_GET_TYPE(&pNew->aPages[0]) == PGMPAGETYPE_MMIO);
2166
2167	/* update the page count stats. */
2168	pVM->pgm.s.cPureMmioPages += cPages;
2169	pVM->pgm.s.cAllPages += cPages;
2170
2171	/* link it */
2172	pgmR3PhysLinkRamRange(pVM, pNew, pRamPrev);
2173	}
2174
2175	/*
2176	* Register the access handler.
2177	*/
2178	rc = PGMHandlerPhysicalRegisterEx(pVM, PGMPHYSHANDLERTYPE_MMIO, GCPhys, GCPhysLast,
2179	pfnHandlerR3, pvUserR3,
2180	pfnHandlerR0, pvUserR0,
2181	pfnHandlerRC, pvUserRC, pszDesc);
2182	if ( RT_FAILURE(rc)
2183	&& !fRamExists)
2184	{
2185	pVM->pgm.s.cPureMmioPages -= cb >> PAGE_SHIFT;
2186	pVM->pgm.s.cAllPages -= cb >> PAGE_SHIFT;
2187
2188	/* remove the ad hoc range. */
2189	pgmR3PhysUnlinkRamRange2(pVM, pNew, pRamPrev);
2190	pNew->cb = pNew->GCPhys = pNew->GCPhysLast = NIL_RTGCPHYS;
2191	MMHyperFree(pVM, pRam);
2192	}
2193	pgmPhysInvalidatePageMapTLB(pVM);
2194
2195	pgmUnlock(pVM);
2196	return rc;
2197	}
2198
2199
2200	/**
2201	* This is the interface IOM is using to register an MMIO region.
2202	*
2203	* It will take care of calling PGMHandlerPhysicalDeregister and clean up
2204	* any ad hoc PGMRAMRANGE left behind.
2205	*
2206	* @returns VBox status code.
2207	* @param pVM Pointer to the shared VM structure.
2208	* @param GCPhys The start of the MMIO region.
2209	* @param cb The size of the MMIO region.
2210	*/
2211	VMMR3DECL(int) PGMR3PhysMMIODeregister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb)
2212	{
2213	VM_ASSERT_EMT(pVM);
2214
2215	int rc = pgmLock(pVM);
2216	AssertRCReturn(rc, rc);
2217
2218	/*
2219	* First deregister the handler, then check if we should remove the ram range.
2220	*/
2221	rc = PGMHandlerPhysicalDeregister(pVM, GCPhys);
2222	if (RT_SUCCESS(rc))
2223	{
2224	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
2225	PPGMRAMRANGE pRamPrev = NULL;
2226	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2227	while (pRam && GCPhysLast >= pRam->GCPhys)
2228	{
2229	/** @todo We're being a bit too careful here. rewrite. */
2230	if ( GCPhysLast == pRam->GCPhysLast
2231	&& GCPhys == pRam->GCPhys)
2232	{
2233	Assert(pRam->cb == cb);
2234
2235	/*
2236	* See if all the pages are dead MMIO pages.
2237	*/
2238	uint32_t const cPages = cb >> PAGE_SHIFT;
2239	bool fAllMMIO = true;
2240	uint32_t iPage = 0;
2241	uint32_t cLeft = cPages;
2242	while (cLeft-- > 0)
2243	{
2244	PPGMPAGE pPage = &pRam->aPages[iPage];
2245	if ( PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO
2246	/\|\| not-out-of-action later /)
2247	{
2248	fAllMMIO = false;
2249	Assert(PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO2_ALIAS_MMIO);
2250	AssertMsgFailed(("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
2251	break;
2252	}
2253	Assert(PGM_PAGE_IS_ZERO(pPage));
2254	pPage++;
2255	}
2256	if (fAllMMIO)
2257	{
2258	/*
2259	* Ad-hoc range, unlink and free it.
2260	*/
2261	Log(("PGMR3PhysMMIODeregister: Freeing ad hoc MMIO range for %RGp-%RGp %s\n",
2262	GCPhys, GCPhysLast, pRam->pszDesc));
2263
2264	pVM->pgm.s.cAllPages -= cPages;
2265	pVM->pgm.s.cPureMmioPages -= cPages;
2266
2267	pgmR3PhysUnlinkRamRange2(pVM, pRam, pRamPrev);
2268	pRam->cb = pRam->GCPhys = pRam->GCPhysLast = NIL_RTGCPHYS;
2269	MMHyperFree(pVM, pRam);
2270	break;
2271	}
2272	}
2273
2274	/*
2275	* Range match? It will all be within one range (see PGMAllHandler.cpp).
2276	*/
2277	if ( GCPhysLast >= pRam->GCPhys
2278	&& GCPhys <= pRam->GCPhysLast)
2279	{
2280	Assert(GCPhys >= pRam->GCPhys);
2281	Assert(GCPhysLast <= pRam->GCPhysLast);
2282
2283	/*
2284	* Turn the pages back into RAM pages.
2285	*/
2286	uint32_t iPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2287	uint32_t cLeft = cb >> PAGE_SHIFT;
2288	while (cLeft--)
2289	{
2290	PPGMPAGE pPage = &pRam->aPages[iPage];
2291	AssertMsg(PGM_PAGE_IS_MMIO(pPage), ("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
2292	AssertMsg(PGM_PAGE_IS_ZERO(pPage), ("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
2293	if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO)
2294	PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_RAM);
2295	}
2296	break;
2297	}
2298
2299	/* next */
2300	pRamPrev = pRam;
2301	pRam = pRam->pNextR3;
2302	}
2303	}
2304
2305	/* Force a PGM pool flush as guest ram references have been changed. */
2306	/** todo; not entirely SMP safe; assuming for now the guest takes care of this internally (not touch mapped mmio while changing the mapping). */
2307	PVMCPU pVCpu = VMMGetCpu(pVM);
2308	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2309	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2310
2311	pgmPhysInvalidatePageMapTLB(pVM);
2312	pgmPhysInvalidRamRangeTlbs(pVM);
2313	pgmUnlock(pVM);
2314	return rc;
2315	}
2316
2317
2318	/**
2319	* Locate a MMIO2 range.
2320	*
2321	* @returns Pointer to the MMIO2 range.
2322	* @param pVM Pointer to the shared VM structure.
2323	* @param pDevIns The device instance owning the region.
2324	* @param iRegion The region.
2325	*/
2326	DECLINLINE(PPGMMMIO2RANGE) pgmR3PhysMMIO2Find(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion)
2327	{
2328	/*
2329	* Search the list.
2330	*/
2331	for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
2332	if ( pCur->pDevInsR3 == pDevIns
2333	&& pCur->iRegion == iRegion)
2334	return pCur;
2335	return NULL;
2336	}
2337
2338
2339	/**
2340	* Allocate and register an MMIO2 region.
2341	*
2342	* As mentioned elsewhere, MMIO2 is just RAM spelled differently. It's RAM
2343	* associated with a device. It is also non-shared memory with a permanent
2344	* ring-3 mapping and page backing (presently).
2345	*
2346	* A MMIO2 range may overlap with base memory if a lot of RAM is configured for
2347	* the VM, in which case we'll drop the base memory pages. Presently we will
2348	* make no attempt to preserve anything that happens to be present in the base
2349	* memory that is replaced, this is of course incorrectly but it's too much
2350	* effort.
2351	*
2352	* @returns VBox status code.
2353	* @retval VINF_SUCCESS on success, *ppv pointing to the R3 mapping of the
2354	* memory.
2355	* @retval VERR_ALREADY_EXISTS if the region already exists.
2356	*
2357	* @param pVM Pointer to the shared VM structure.
2358	* @param pDevIns The device instance owning the region.
2359	* @param iRegion The region number. If the MMIO2 memory is a PCI
2360	* I/O region this number has to be the number of that
2361	* region. Otherwise it can be any number safe
2362	* UINT8_MAX.
2363	* @param cb The size of the region. Must be page aligned.
2364	* @param fFlags Reserved for future use, must be zero.
2365	* @param ppv Where to store the pointer to the ring-3 mapping of
2366	* the memory.
2367	* @param pszDesc The description.
2368	*/
2369	VMMR3DECL(int) PGMR3PhysMMIO2Register(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS cb, uint32_t fFlags, void *ppv, const char pszDesc)
2370	{
2371	/*
2372	* Validate input.
2373	*/
2374	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2375	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2376	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2377	AssertPtrReturn(ppv, VERR_INVALID_POINTER);
2378	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
2379	AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
2380	AssertReturn(pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion) == NULL, VERR_ALREADY_EXISTS);
2381	AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2382	AssertReturn(cb, VERR_INVALID_PARAMETER);
2383	AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
2384
2385	const uint32_t cPages = cb >> PAGE_SHIFT;
2386	AssertLogRelReturn(((RTGCPHYS)cPages << PAGE_SHIFT) == cb, VERR_INVALID_PARAMETER);
2387	AssertLogRelReturn(cPages <= INT32_MAX / 2, VERR_NO_MEMORY);
2388
2389	/*
2390	* For the 2nd+ instance, mangle the description string so it's unique.
2391	*/
2392	if (pDevIns->iInstance > 0) /** @todo Move to PDMDevHlp.cpp and use a real string cache. */
2393	{
2394	pszDesc = MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s [%u]", pszDesc, pDevIns->iInstance);
2395	if (!pszDesc)
2396	return VERR_NO_MEMORY;
2397	}
2398
2399	/*
2400	* Try reserve and allocate the backing memory first as this is what is
2401	* most likely to fail.
2402	*/
2403	int rc = MMR3AdjustFixedReservation(pVM, cPages, pszDesc);
2404	if (RT_SUCCESS(rc))
2405	{
2406	void *pvPages;
2407	PSUPPAGE paPages = (PSUPPAGE)RTMemTmpAlloc(cPages * sizeof(SUPPAGE));
2408	if (RT_SUCCESS(rc))
2409	rc = SUPR3PageAllocEx(cPages, 0 /fFlags/, &pvPages, NULL /pR0Ptr/, paPages);
2410	if (RT_SUCCESS(rc))
2411	{
2412	memset(pvPages, 0, cPages * PAGE_SIZE);
2413
2414	/*
2415	* Create the MMIO2 range record for it.
2416	*/
2417	const size_t cbRange = RT_OFFSETOF(PGMMMIO2RANGE, RamRange.aPages[cPages]);
2418	PPGMMMIO2RANGE pNew;
2419	rc = MMR3HyperAllocOnceNoRel(pVM, cbRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
2420	AssertLogRelMsgRC(rc, ("cbRamRange=%zu\n", cbRange));
2421	if (RT_SUCCESS(rc))
2422	{
2423	pNew->pDevInsR3 = pDevIns;
2424	pNew->pvR3 = pvPages;
2425	//pNew->pNext = NULL;
2426	//pNew->fMapped = false;
2427	//pNew->fOverlapping = false;
2428	pNew->iRegion = iRegion;
2429	pNew->idSavedState = UINT8_MAX;
2430	pNew->RamRange.pSelfR0 = MMHyperCCToR0(pVM, &pNew->RamRange);
2431	pNew->RamRange.pSelfRC = MMHyperCCToRC(pVM, &pNew->RamRange);
2432	pNew->RamRange.GCPhys = NIL_RTGCPHYS;
2433	pNew->RamRange.GCPhysLast = NIL_RTGCPHYS;
2434	pNew->RamRange.pszDesc = pszDesc;
2435	pNew->RamRange.cb = cb;
2436	pNew->RamRange.fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO2;
2437	pNew->RamRange.pvR3 = pvPages;
2438	//pNew->RamRange.paLSPages = NULL;
2439
2440	uint32_t iPage = cPages;
2441	while (iPage-- > 0)
2442	{
2443	PGM_PAGE_INIT(&pNew->RamRange.aPages[iPage],
2444	paPages[iPage].Phys, NIL_GMM_PAGEID,
2445	PGMPAGETYPE_MMIO2, PGM_PAGE_STATE_ALLOCATED);
2446	}
2447
2448	/* update page count stats */
2449	pVM->pgm.s.cAllPages += cPages;
2450	pVM->pgm.s.cPrivatePages += cPages;
2451
2452	/*
2453	* Link it into the list.
2454	* Since there is no particular order, just push it.
2455	*/
2456	pgmLock(pVM);
2457	pNew->pNextR3 = pVM->pgm.s.pMmio2RangesR3;
2458	pVM->pgm.s.pMmio2RangesR3 = pNew;
2459	pgmUnlock(pVM);
2460
2461	*ppv = pvPages;
2462	RTMemTmpFree(paPages);
2463	pgmPhysInvalidatePageMapTLB(pVM);
2464	return VINF_SUCCESS;
2465	}
2466
2467	SUPR3PageFreeEx(pvPages, cPages);
2468	}
2469	RTMemTmpFree(paPages);
2470	MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pszDesc);
2471	}
2472	if (pDevIns->iInstance > 0)
2473	MMR3HeapFree((void *)pszDesc);
2474	return rc;
2475	}
2476
2477
2478	/**
2479	* Deregisters and frees an MMIO2 region.
2480	*
2481	* Any physical (and virtual) access handlers registered for the region must
2482	* be deregistered before calling this function.
2483	*
2484	* @returns VBox status code.
2485	* @param pVM Pointer to the shared VM structure.
2486	* @param pDevIns The device instance owning the region.
2487	* @param iRegion The region. If it's UINT32_MAX it'll be a wildcard match.
2488	*/
2489	VMMR3DECL(int) PGMR3PhysMMIO2Deregister(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion)
2490	{
2491	/*
2492	* Validate input.
2493	*/
2494	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2495	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2496	AssertReturn(iRegion <= UINT8_MAX \|\| iRegion == UINT32_MAX, VERR_INVALID_PARAMETER);
2497
2498	pgmLock(pVM);
2499	int rc = VINF_SUCCESS;
2500	unsigned cFound = 0;
2501	PPGMMMIO2RANGE pPrev = NULL;
2502	PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3;
2503	while (pCur)
2504	{
2505	if ( pCur->pDevInsR3 == pDevIns
2506	&& ( iRegion == UINT32_MAX
2507	\|\| pCur->iRegion == iRegion))
2508	{
2509	cFound++;
2510
2511	/*
2512	* Unmap it if it's mapped.
2513	*/
2514	if (pCur->fMapped)
2515	{
2516	int rc2 = PGMR3PhysMMIO2Unmap(pVM, pCur->pDevInsR3, pCur->iRegion, pCur->RamRange.GCPhys);
2517	AssertRC(rc2);
2518	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2519	rc = rc2;
2520	}
2521
2522	/*
2523	* Unlink it
2524	*/
2525	PPGMMMIO2RANGE pNext = pCur->pNextR3;
2526	if (pPrev)
2527	pPrev->pNextR3 = pNext;
2528	else
2529	pVM->pgm.s.pMmio2RangesR3 = pNext;
2530	pCur->pNextR3 = NULL;
2531
2532	/*
2533	* Free the memory.
2534	*/
2535	int rc2 = SUPR3PageFreeEx(pCur->pvR3, pCur->RamRange.cb >> PAGE_SHIFT);
2536	AssertRC(rc2);
2537	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2538	rc = rc2;
2539
2540	uint32_t const cPages = pCur->RamRange.cb >> PAGE_SHIFT;
2541	rc2 = MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pCur->RamRange.pszDesc);
2542	AssertRC(rc2);
2543	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2544	rc = rc2;
2545
2546	/* we're leaking hyper memory here if done at runtime. */
2547	#ifdef VBOX_STRICT
2548	VMSTATE const enmState = VMR3GetState(pVM);
2549	AssertMsg( enmState == VMSTATE_POWERING_OFF
2550	\|\| enmState == VMSTATE_POWERING_OFF_LS
2551	\|\| enmState == VMSTATE_OFF
2552	\|\| enmState == VMSTATE_OFF_LS
2553	\|\| enmState == VMSTATE_DESTROYING
2554	\|\| enmState == VMSTATE_TERMINATED
2555	\|\| enmState == VMSTATE_CREATING
2556	, ("%s\n", VMR3GetStateName(enmState)));
2557	#endif
2558	/*rc = MMHyperFree(pVM, pCur);
2559	AssertRCReturn(rc, rc); - not safe, see the alloc call. */
2560
2561
2562	/* update page count stats */
2563	pVM->pgm.s.cAllPages -= cPages;
2564	pVM->pgm.s.cPrivatePages -= cPages;
2565
2566	/* next */
2567	pCur = pNext;
2568	}
2569	else
2570	{
2571	pPrev = pCur;
2572	pCur = pCur->pNextR3;
2573	}
2574	}
2575	pgmPhysInvalidatePageMapTLB(pVM);
2576	pgmUnlock(pVM);
2577	return !cFound && iRegion != UINT32_MAX ? VERR_NOT_FOUND : rc;
2578	}
2579
2580
2581	/**
2582	* Maps a MMIO2 region.
2583	*
2584	* This is done when a guest / the bios / state loading changes the
2585	* PCI config. The replacing of base memory has the same restrictions
2586	* as during registration, of course.
2587	*
2588	* @returns VBox status code.
2589	*
2590	* @param pVM Pointer to the shared VM structure.
2591	* @param pDevIns The
2592	*/
2593	VMMR3DECL(int) PGMR3PhysMMIO2Map(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
2594	{
2595	/*
2596	* Validate input
2597	*/
2598	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2599	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2600	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2601	AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
2602	AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
2603	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2604
2605	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2606	AssertReturn(pCur, VERR_NOT_FOUND);
2607	AssertReturn(!pCur->fMapped, VERR_WRONG_ORDER);
2608	Assert(pCur->RamRange.GCPhys == NIL_RTGCPHYS);
2609	Assert(pCur->RamRange.GCPhysLast == NIL_RTGCPHYS);
2610
2611	const RTGCPHYS GCPhysLast = GCPhys + pCur->RamRange.cb - 1;
2612	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
2613
2614	/*
2615	* Find our location in the ram range list, checking for
2616	* restriction we don't bother implementing yet (partially overlapping).
2617	*/
2618	bool fRamExists = false;
2619	PPGMRAMRANGE pRamPrev = NULL;
2620	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2621	while (pRam && GCPhysLast >= pRam->GCPhys)
2622	{
2623	if ( GCPhys <= pRam->GCPhysLast
2624	&& GCPhysLast >= pRam->GCPhys)
2625	{
2626	/* completely within? */
2627	AssertLogRelMsgReturn( GCPhys >= pRam->GCPhys
2628	&& GCPhysLast <= pRam->GCPhysLast,
2629	("%RGp-%RGp (MMIO2/%s) falls partly outside %RGp-%RGp (%s)\n",
2630	GCPhys, GCPhysLast, pCur->RamRange.pszDesc,
2631	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
2632	VERR_PGM_RAM_CONFLICT);
2633	fRamExists = true;
2634	break;
2635	}
2636
2637	/* next */
2638	pRamPrev = pRam;
2639	pRam = pRam->pNextR3;
2640	}
2641	if (fRamExists)
2642	{
2643	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2644	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2645	while (cPagesLeft-- > 0)
2646	{
2647	AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
2648	("%RGp isn't a RAM page (%d) - mapping %RGp-%RGp (MMIO2/%s).\n",
2649	GCPhys, PGM_PAGE_GET_TYPE(pPage), GCPhys, GCPhysLast, pCur->RamRange.pszDesc),
2650	VERR_PGM_RAM_CONFLICT);
2651	pPage++;
2652	}
2653	}
2654	Log(("PGMR3PhysMMIO2Map: %RGp-%RGp fRamExists=%RTbool %s\n",
2655	GCPhys, GCPhysLast, fRamExists, pCur->RamRange.pszDesc));
2656
2657	/*
2658	* Make the changes.
2659	*/
2660	pgmLock(pVM);
2661
2662	pCur->RamRange.GCPhys = GCPhys;
2663	pCur->RamRange.GCPhysLast = GCPhysLast;
2664	pCur->fMapped = true;
2665	pCur->fOverlapping = fRamExists;
2666
2667	if (fRamExists)
2668	{
2669	/** @todo use pgmR3PhysFreePageRange here. */
2670	uint32_t cPendingPages = 0;
2671	PGMMFREEPAGESREQ pReq;
2672	int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
2673	AssertLogRelRCReturn(rc, rc);
2674
2675	/* replace the pages, freeing all present RAM pages. */
2676	PPGMPAGE pPageSrc = &pCur->RamRange.aPages[0];
2677	PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2678	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2679	while (cPagesLeft-- > 0)
2680	{
2681	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys);
2682	AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
2683
2684	RTHCPHYS const HCPhys = PGM_PAGE_GET_HCPHYS(pPageSrc);
2685	PGM_PAGE_SET_HCPHYS(pVM, pPageDst, HCPhys);
2686	PGM_PAGE_SET_TYPE(pVM, pPageDst, PGMPAGETYPE_MMIO2);
2687	PGM_PAGE_SET_STATE(pVM, pPageDst, PGM_PAGE_STATE_ALLOCATED);
2688	PGM_PAGE_SET_PDE_TYPE(pVM, pPageDst, PGM_PAGE_PDE_TYPE_DONTCARE);
2689	PGM_PAGE_SET_PTE_INDEX(pVM, pPageDst, 0);
2690	PGM_PAGE_SET_TRACKING(pVM, pPageDst, 0);
2691
2692	pVM->pgm.s.cZeroPages--;
2693	GCPhys += PAGE_SIZE;
2694	pPageSrc++;
2695	pPageDst++;
2696	}
2697
2698	/* Flush physical page map TLB. */
2699	pgmPhysInvalidatePageMapTLB(pVM);
2700
2701	if (cPendingPages)
2702	{
2703	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2704	AssertLogRelRCReturn(rc, rc);
2705	}
2706	GMMR3FreePagesCleanup(pReq);
2707
2708	/* Force a PGM pool flush as guest ram references have been changed. */
2709	/** todo; not entirely SMP safe; assuming for now the guest takes care of this internally (not touch mapped mmio while changing the mapping). */
2710	PVMCPU pVCpu = VMMGetCpu(pVM);
2711	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2712	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2713
2714	pgmUnlock(pVM);
2715	}
2716	else
2717	{
2718	RTGCPHYS cb = pCur->RamRange.cb;
2719
2720	/* Clear the tracking data of pages we're going to reactivate. */
2721	PPGMPAGE pPageSrc = &pCur->RamRange.aPages[0];
2722	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2723	while (cPagesLeft-- > 0)
2724	{
2725	PGM_PAGE_SET_TRACKING(pVM, pPageSrc, 0);
2726	PGM_PAGE_SET_PTE_INDEX(pVM, pPageSrc, 0);
2727	pPageSrc++;
2728	}
2729
2730	/* link in the ram range */
2731	pgmR3PhysLinkRamRange(pVM, &pCur->RamRange, pRamPrev);
2732	pgmUnlock(pVM);
2733
2734	REMR3NotifyPhysRamRegister(pVM, GCPhys, cb, REM_NOTIFY_PHYS_RAM_FLAGS_MMIO2);
2735	}
2736
2737	pgmPhysInvalidatePageMapTLB(pVM);
2738	return VINF_SUCCESS;
2739	}
2740
2741
2742	/**
2743	* Unmaps a MMIO2 region.
2744	*
2745	* This is done when a guest / the bios / state loading changes the
2746	* PCI config. The replacing of base memory has the same restrictions
2747	* as during registration, of course.
2748	*/
2749	VMMR3DECL(int) PGMR3PhysMMIO2Unmap(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
2750	{
2751	/*
2752	* Validate input
2753	*/
2754	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2755	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2756	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2757	AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
2758	AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
2759	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2760
2761	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2762	AssertReturn(pCur, VERR_NOT_FOUND);
2763	AssertReturn(pCur->fMapped, VERR_WRONG_ORDER);
2764	AssertReturn(pCur->RamRange.GCPhys == GCPhys, VERR_INVALID_PARAMETER);
2765	Assert(pCur->RamRange.GCPhysLast != NIL_RTGCPHYS);
2766
2767	Log(("PGMR3PhysMMIO2Unmap: %RGp-%RGp %s\n",
2768	pCur->RamRange.GCPhys, pCur->RamRange.GCPhysLast, pCur->RamRange.pszDesc));
2769
2770	/*
2771	* Unmap it.
2772	*/
2773	pgmLock(pVM);
2774
2775	RTGCPHYS GCPhysRangeREM;
2776	RTGCPHYS cbRangeREM;
2777	bool fInformREM;
2778	if (pCur->fOverlapping)
2779	{
2780	/* Restore the RAM pages we've replaced. */
2781	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2782	while (pRam->GCPhys > pCur->RamRange.GCPhysLast)
2783	pRam = pRam->pNextR3;
2784
2785	PPGMPAGE pPageDst = &pRam->aPages[(pCur->RamRange.GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2786	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2787	while (cPagesLeft-- > 0)
2788	{
2789	PGM_PAGE_INIT_ZERO(pPageDst, pVM, PGMPAGETYPE_RAM);
2790	pVM->pgm.s.cZeroPages++;
2791	pPageDst++;
2792	}
2793
2794	/* Flush physical page map TLB. */
2795	pgmPhysInvalidatePageMapTLB(pVM);
2796
2797	GCPhysRangeREM = NIL_RTGCPHYS; /* shuts up gcc */
2798	cbRangeREM = RTGCPHYS_MAX; /* ditto */
2799	fInformREM = false;
2800	}
2801	else
2802	{
2803	GCPhysRangeREM = pCur->RamRange.GCPhys;
2804	cbRangeREM = pCur->RamRange.cb;
2805	fInformREM = true;
2806
2807	pgmR3PhysUnlinkRamRange(pVM, &pCur->RamRange);
2808	}
2809
2810	pCur->RamRange.GCPhys = NIL_RTGCPHYS;
2811	pCur->RamRange.GCPhysLast = NIL_RTGCPHYS;
2812	pCur->fOverlapping = false;
2813	pCur->fMapped = false;
2814
2815	/* Force a PGM pool flush as guest ram references have been changed. */
2816	/** @todo not entirely SMP safe; assuming for now the guest takes care
2817	* of this internally (not touch mapped mmio while changing the
2818	* mapping). */
2819	PVMCPU pVCpu = VMMGetCpu(pVM);
2820	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2821	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2822
2823	pgmPhysInvalidatePageMapTLB(pVM);
2824	pgmPhysInvalidRamRangeTlbs(pVM);
2825	pgmUnlock(pVM);
2826
2827	if (fInformREM)
2828	REMR3NotifyPhysRamDeregister(pVM, GCPhysRangeREM, cbRangeREM);
2829
2830	return VINF_SUCCESS;
2831	}
2832
2833
2834	/**
2835	* Checks if the given address is an MMIO2 base address or not.
2836	*
2837	* @returns true/false accordingly.
2838	* @param pVM Pointer to the shared VM structure.
2839	* @param pDevIns The owner of the memory, optional.
2840	* @param GCPhys The address to check.
2841	*/
2842	VMMR3DECL(bool) PGMR3PhysMMIO2IsBase(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys)
2843	{
2844	/*
2845	* Validate input
2846	*/
2847	VM_ASSERT_EMT_RETURN(pVM, false);
2848	AssertPtrReturn(pDevIns, false);
2849	AssertReturn(GCPhys != NIL_RTGCPHYS, false);
2850	AssertReturn(GCPhys != 0, false);
2851	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), false);
2852
2853	/*
2854	* Search the list.
2855	*/
2856	pgmLock(pVM);
2857	for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
2858	if (pCur->RamRange.GCPhys == GCPhys)
2859	{
2860	Assert(pCur->fMapped);
2861	pgmUnlock(pVM);
2862	return true;
2863	}
2864	pgmUnlock(pVM);
2865	return false;
2866	}
2867
2868
2869	/**
2870	* Gets the HC physical address of a page in the MMIO2 region.
2871	*
2872	* This is API is intended for MMHyper and shouldn't be called
2873	* by anyone else...
2874	*
2875	* @returns VBox status code.
2876	* @param pVM Pointer to the shared VM structure.
2877	* @param pDevIns The owner of the memory, optional.
2878	* @param iRegion The region.
2879	* @param off The page expressed an offset into the MMIO2 region.
2880	* @param pHCPhys Where to store the result.
2881	*/
2882	VMMR3DECL(int) PGMR3PhysMMIO2GetHCPhys(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, PRTHCPHYS pHCPhys)
2883	{
2884	/*
2885	* Validate input
2886	*/
2887	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2888	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2889	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2890
2891	pgmLock(pVM);
2892	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2893	AssertReturn(pCur, VERR_NOT_FOUND);
2894	AssertReturn(off < pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2895
2896	PCPGMPAGE pPage = &pCur->RamRange.aPages[off >> PAGE_SHIFT];
2897	*pHCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2898	pgmUnlock(pVM);
2899	return VINF_SUCCESS;
2900	}
2901
2902
2903	/**
2904	* Maps a portion of an MMIO2 region into kernel space (host).
2905	*
2906	* The kernel mapping will become invalid when the MMIO2 memory is deregistered
2907	* or the VM is terminated.
2908	*
2909	* @return VBox status code.
2910	*
2911	* @param pVM Pointer to the shared VM structure.
2912	* @param pDevIns The device owning the MMIO2 memory.
2913	* @param iRegion The region.
2914	* @param off The offset into the region. Must be page aligned.
2915	* @param cb The number of bytes to map. Must be page aligned.
2916	* @param pszDesc Mapping description.
2917	* @param pR0Ptr Where to store the R0 address.
2918	*/
2919	VMMR3DECL(int) PGMR3PhysMMIO2MapKernel(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, RTGCPHYS cb,
2920	const char *pszDesc, PRTR0PTR pR0Ptr)
2921	{
2922	/*
2923	* Validate input.
2924	*/
2925	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2926	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2927	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2928
2929	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2930	AssertReturn(pCur, VERR_NOT_FOUND);
2931	AssertReturn(off < pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2932	AssertReturn(cb <= pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2933	AssertReturn(off + cb <= pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2934
2935	/*
2936	* Pass the request on to the support library/driver.
2937	*/
2938	int rc = SUPR3PageMapKernel(pCur->pvR3, off, cb, 0, pR0Ptr);
2939
2940	return rc;
2941	}
2942
2943
2944	/**
2945	* Worker for PGMR3PhysRomRegister.
2946	*
2947	* This is here to simplify lock management, i.e. the caller does all the
2948	* locking and we can simply return without needing to remember to unlock
2949	* anything first.
2950	*
2951	* @returns VBox status.
2952	* @param pVM VM Handle.
2953	* @param pDevIns The device instance owning the ROM.
2954	* @param GCPhys First physical address in the range.
2955	* Must be page aligned!
2956	* @param cb The size of the range (in bytes).
2957	* Must be page aligned!
2958	* @param pvBinary Pointer to the binary data backing the ROM image.
2959	* @param cbBinary The size of the binary data pvBinary points to.
2960	* This must be less or equal to @a cb.
2961	* @param fFlags Mask of flags. PGMPHYS_ROM_FLAGS_SHADOWED
2962	* and/or PGMPHYS_ROM_FLAGS_PERMANENT_BINARY.
2963	* @param pszDesc Pointer to description string. This must not be freed.
2964	*/
2965	static int pgmR3PhysRomRegister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
2966	const void pvBinary, uint32_t cbBinary, uint32_t fFlags, const char pszDesc)
2967	{
2968	/*
2969	* Validate input.
2970	*/
2971	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2972	AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
2973	AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
2974	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
2975	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
2976	AssertPtrReturn(pvBinary, VERR_INVALID_PARAMETER);
2977	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
2978	AssertReturn(!(fFlags & ~(PGMPHYS_ROM_FLAGS_SHADOWED \| PGMPHYS_ROM_FLAGS_PERMANENT_BINARY)), VERR_INVALID_PARAMETER);
2979	VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
2980
2981	const uint32_t cPages = cb >> PAGE_SHIFT;
2982
2983	/*
2984	* Find the ROM location in the ROM list first.
2985	*/
2986	PPGMROMRANGE pRomPrev = NULL;
2987	PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3;
2988	while (pRom && GCPhysLast >= pRom->GCPhys)
2989	{
2990	if ( GCPhys <= pRom->GCPhysLast
2991	&& GCPhysLast >= pRom->GCPhys)
2992	AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
2993	GCPhys, GCPhysLast, pszDesc,
2994	pRom->GCPhys, pRom->GCPhysLast, pRom->pszDesc),
2995	VERR_PGM_RAM_CONFLICT);
2996	/* next */
2997	pRomPrev = pRom;
2998	pRom = pRom->pNextR3;
2999	}
3000
3001	/*
3002	* Find the RAM location and check for conflicts.
3003	*
3004	* Conflict detection is a bit different than for RAM
3005	* registration since a ROM can be located within a RAM
3006	* range. So, what we have to check for is other memory
3007	* types (other than RAM that is) and that we don't span
3008	* more than one RAM range (layz).
3009	*/
3010	bool fRamExists = false;
3011	PPGMRAMRANGE pRamPrev = NULL;
3012	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
3013	while (pRam && GCPhysLast >= pRam->GCPhys)
3014	{
3015	if ( GCPhys <= pRam->GCPhysLast
3016	&& GCPhysLast >= pRam->GCPhys)
3017	{
3018	/* completely within? */
3019	AssertLogRelMsgReturn( GCPhys >= pRam->GCPhys
3020	&& GCPhysLast <= pRam->GCPhysLast,
3021	("%RGp-%RGp (%s) falls partly outside %RGp-%RGp (%s)\n",
3022	GCPhys, GCPhysLast, pszDesc,
3023	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
3024	VERR_PGM_RAM_CONFLICT);
3025	fRamExists = true;
3026	break;
3027	}
3028
3029	/* next */
3030	pRamPrev = pRam;
3031	pRam = pRam->pNextR3;
3032	}
3033	if (fRamExists)
3034	{
3035	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
3036	uint32_t cPagesLeft = cPages;
3037	while (cPagesLeft-- > 0)
3038	{
3039	AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
3040	("%RGp (%R[pgmpage]) isn't a RAM page - registering %RGp-%RGp (%s).\n",
3041	pRam->GCPhys + ((RTGCPHYS)(uintptr_t)(pPage - &pRam->aPages[0]) << PAGE_SHIFT),
3042	pPage, GCPhys, GCPhysLast, pszDesc), VERR_PGM_RAM_CONFLICT);
3043	Assert(PGM_PAGE_IS_ZERO(pPage));
3044	pPage++;
3045	}
3046	}
3047
3048	/*
3049	* Update the base memory reservation if necessary.
3050	*/
3051	uint32_t cExtraBaseCost = fRamExists ? 0 : cPages;
3052	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3053	cExtraBaseCost += cPages;
3054	if (cExtraBaseCost)
3055	{
3056	int rc = MMR3IncreaseBaseReservation(pVM, cExtraBaseCost);
3057	if (RT_FAILURE(rc))
3058	return rc;
3059	}
3060
3061	/*
3062	* Allocate memory for the virgin copy of the RAM.
3063	*/
3064	PGMMALLOCATEPAGESREQ pReq;
3065	int rc = GMMR3AllocatePagesPrepare(pVM, &pReq, cPages, GMMACCOUNT_BASE);
3066	AssertRCReturn(rc, rc);
3067
3068	for (uint32_t iPage = 0; iPage < cPages; iPage++)
3069	{
3070	pReq->aPages[iPage].HCPhysGCPhys = GCPhys + (iPage << PAGE_SHIFT);
3071	pReq->aPages[iPage].idPage = NIL_GMM_PAGEID;
3072	pReq->aPages[iPage].idSharedPage = NIL_GMM_PAGEID;
3073	}
3074
3075	rc = GMMR3AllocatePagesPerform(pVM, pReq);
3076	if (RT_FAILURE(rc))
3077	{
3078	GMMR3AllocatePagesCleanup(pReq);
3079	return rc;
3080	}
3081
3082	/*
3083	* Allocate the new ROM range and RAM range (if necessary).
3084	*/
3085	PPGMROMRANGE pRomNew;
3086	rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMROMRANGE, aPages[cPages]), 0, MM_TAG_PGM_PHYS, (void **)&pRomNew);
3087	if (RT_SUCCESS(rc))
3088	{
3089	PPGMRAMRANGE pRamNew = NULL;
3090	if (!fRamExists)
3091	rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]), sizeof(PGMPAGE), MM_TAG_PGM_PHYS, (void **)&pRamNew);
3092	if (RT_SUCCESS(rc))
3093	{
3094	/*
3095	* Initialize and insert the RAM range (if required).
3096	*/
3097	PPGMROMPAGE pRomPage = &pRomNew->aPages[0];
3098	if (!fRamExists)
3099	{
3100	pRamNew->pSelfR0 = MMHyperCCToR0(pVM, pRamNew);
3101	pRamNew->pSelfRC = MMHyperCCToRC(pVM, pRamNew);
3102	pRamNew->GCPhys = GCPhys;
3103	pRamNew->GCPhysLast = GCPhysLast;
3104	pRamNew->cb = cb;
3105	pRamNew->pszDesc = pszDesc;
3106	pRamNew->fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_ROM;
3107	pRamNew->pvR3 = NULL;
3108	pRamNew->paLSPages = NULL;
3109
3110	PPGMPAGE pPage = &pRamNew->aPages[0];
3111	for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
3112	{
3113	PGM_PAGE_INIT(pPage,
3114	pReq->aPages[iPage].HCPhysGCPhys,
3115	pReq->aPages[iPage].idPage,
3116	PGMPAGETYPE_ROM,
3117	PGM_PAGE_STATE_ALLOCATED);
3118
3119	pRomPage->Virgin = *pPage;
3120	}
3121
3122	pVM->pgm.s.cAllPages += cPages;
3123	pgmR3PhysLinkRamRange(pVM, pRamNew, pRamPrev);
3124	}
3125	else
3126	{
3127	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
3128	for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
3129	{
3130	PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_ROM);
3131	PGM_PAGE_SET_HCPHYS(pVM, pPage, pReq->aPages[iPage].HCPhysGCPhys);
3132	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
3133	PGM_PAGE_SET_PAGEID(pVM, pPage, pReq->aPages[iPage].idPage);
3134	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
3135	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
3136	PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
3137
3138	pRomPage->Virgin = *pPage;
3139	}
3140
3141	pRamNew = pRam;
3142
3143	pVM->pgm.s.cZeroPages -= cPages;
3144	}
3145	pVM->pgm.s.cPrivatePages += cPages;
3146
3147	/* Flush physical page map TLB. */
3148	pgmPhysInvalidatePageMapTLB(pVM);
3149
3150
3151	/*
3152	* !HACK ALERT! REM + (Shadowed) ROM ==> mess.
3153	*
3154	* If it's shadowed we'll register the handler after the ROM notification
3155	* so we get the access handler callbacks that we should. If it isn't
3156	* shadowed we'll do it the other way around to make REM use the built-in
3157	* ROM behavior and not the handler behavior (which is to route all access
3158	* to PGM atm).
3159	*/
3160	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3161	{
3162	REMR3NotifyPhysRomRegister(pVM, GCPhys, cb, NULL, true /* fShadowed */);
3163	rc = PGMR3HandlerPhysicalRegister(pVM,
3164	fFlags & PGMPHYS_ROM_FLAGS_SHADOWED
3165	? PGMPHYSHANDLERTYPE_PHYSICAL_ALL
3166	: PGMPHYSHANDLERTYPE_PHYSICAL_WRITE,
3167	GCPhys, GCPhysLast,
3168	pgmR3PhysRomWriteHandler, pRomNew,
3169	NULL, "pgmPhysRomWriteHandler", MMHyperCCToR0(pVM, pRomNew),
3170	NULL, "pgmPhysRomWriteHandler", MMHyperCCToRC(pVM, pRomNew), pszDesc);
3171	}
3172	else
3173	{
3174	rc = PGMR3HandlerPhysicalRegister(pVM,
3175	fFlags & PGMPHYS_ROM_FLAGS_SHADOWED
3176	? PGMPHYSHANDLERTYPE_PHYSICAL_ALL
3177	: PGMPHYSHANDLERTYPE_PHYSICAL_WRITE,
3178	GCPhys, GCPhysLast,
3179	pgmR3PhysRomWriteHandler, pRomNew,
3180	NULL, "pgmPhysRomWriteHandler", MMHyperCCToR0(pVM, pRomNew),
3181	NULL, "pgmPhysRomWriteHandler", MMHyperCCToRC(pVM, pRomNew), pszDesc);
3182	REMR3NotifyPhysRomRegister(pVM, GCPhys, cb, NULL, false /* fShadowed */);
3183	}
3184	if (RT_SUCCESS(rc))
3185	{
3186	/*
3187	* Copy the image over to the virgin pages.
3188	* This must be done after linking in the RAM range.
3189	*/
3190	size_t cbBinaryLeft = cbBinary;
3191	PPGMPAGE pRamPage = &pRamNew->aPages[(GCPhys - pRamNew->GCPhys) >> PAGE_SHIFT];
3192	for (uint32_t iPage = 0; iPage < cPages; iPage++, pRamPage++)
3193	{
3194	void *pvDstPage;
3195	rc = pgmPhysPageMap(pVM, pRamPage, GCPhys + (iPage << PAGE_SHIFT), &pvDstPage);
3196	if (RT_FAILURE(rc))
3197	{
3198	VMSetError(pVM, rc, RT_SRC_POS, "Failed to map virgin ROM page at %RGp", GCPhys);
3199	break;
3200	}
3201	if (cbBinaryLeft >= PAGE_SIZE)
3202	{
3203	memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << PAGE_SHIFT), PAGE_SIZE);
3204	cbBinaryLeft -= PAGE_SIZE;
3205	}
3206	else
3207	{
3208	ASMMemZeroPage(pvDstPage); /* (shouldn't be necessary, but can't hurt either) */
3209	if (cbBinaryLeft > 0)
3210	{
3211	memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << PAGE_SHIFT), cbBinaryLeft);
3212	cbBinaryLeft = 0;
3213	}
3214	}
3215	}
3216	if (RT_SUCCESS(rc))
3217	{
3218	/*
3219	* Initialize the ROM range.
3220	* Note that the Virgin member of the pages has already been initialized above.
3221	*/
3222	pRomNew->GCPhys = GCPhys;
3223	pRomNew->GCPhysLast = GCPhysLast;
3224	pRomNew->cb = cb;
3225	pRomNew->fFlags = fFlags;
3226	pRomNew->idSavedState = UINT8_MAX;
3227	pRomNew->cbOriginal = cbBinary;
3228	#ifdef VBOX_STRICT
3229	pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY
3230	? pvBinary : RTMemDup(pvBinary, cbBinary);
3231	#else
3232	pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY ? pvBinary : NULL;
3233	#endif
3234	pRomNew->pszDesc = pszDesc;
3235
3236	for (unsigned iPage = 0; iPage < cPages; iPage++)
3237	{
3238	PPGMROMPAGE pPage = &pRomNew->aPages[iPage];
3239	pPage->enmProt = PGMROMPROT_READ_ROM_WRITE_IGNORE;
3240	PGM_PAGE_INIT_ZERO(&pPage->Shadow, pVM, PGMPAGETYPE_ROM_SHADOW);
3241	}
3242
3243	/* update the page count stats for the shadow pages. */
3244	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3245	{
3246	pVM->pgm.s.cZeroPages += cPages;
3247	pVM->pgm.s.cAllPages += cPages;
3248	}
3249
3250	/*
3251	* Insert the ROM range, tell REM and return successfully.
3252	*/
3253	pRomNew->pNextR3 = pRom;
3254	pRomNew->pNextR0 = pRom ? MMHyperCCToR0(pVM, pRom) : NIL_RTR0PTR;
3255	pRomNew->pNextRC = pRom ? MMHyperCCToRC(pVM, pRom) : NIL_RTRCPTR;
3256
3257	if (pRomPrev)
3258	{
3259	pRomPrev->pNextR3 = pRomNew;
3260	pRomPrev->pNextR0 = MMHyperCCToR0(pVM, pRomNew);
3261	pRomPrev->pNextRC = MMHyperCCToRC(pVM, pRomNew);
3262	}
3263	else
3264	{
3265	pVM->pgm.s.pRomRangesR3 = pRomNew;
3266	pVM->pgm.s.pRomRangesR0 = MMHyperCCToR0(pVM, pRomNew);
3267	pVM->pgm.s.pRomRangesRC = MMHyperCCToRC(pVM, pRomNew);
3268	}
3269
3270	pgmPhysInvalidatePageMapTLB(pVM);
3271	GMMR3AllocatePagesCleanup(pReq);
3272	return VINF_SUCCESS;
3273	}
3274
3275	/* bail out */
3276
3277	int rc2 = PGMHandlerPhysicalDeregister(pVM, GCPhys);
3278	AssertRC(rc2);
3279	}
3280
3281	if (!fRamExists)
3282	{
3283	pgmR3PhysUnlinkRamRange2(pVM, pRamNew, pRamPrev);
3284	MMHyperFree(pVM, pRamNew);
3285	}
3286	}
3287	MMHyperFree(pVM, pRomNew);
3288	}
3289
3290	/** @todo Purge the mapping cache or something... */
3291	GMMR3FreeAllocatedPages(pVM, pReq);
3292	GMMR3AllocatePagesCleanup(pReq);
3293	return rc;
3294	}
3295
3296
3297	/**
3298	* Registers a ROM image.
3299	*
3300	* Shadowed ROM images requires double the amount of backing memory, so,
3301	* don't use that unless you have to. Shadowing of ROM images is process
3302	* where we can select where the reads go and where the writes go. On real
3303	* hardware the chipset provides means to configure this. We provide
3304	* PGMR3PhysProtectROM() for this purpose.
3305	*
3306	* A read-only copy of the ROM image will always be kept around while we
3307	* will allocate RAM pages for the changes on demand (unless all memory
3308	* is configured to be preallocated).
3309	*
3310	* @returns VBox status.
3311	* @param pVM VM Handle.
3312	* @param pDevIns The device instance owning the ROM.
3313	* @param GCPhys First physical address in the range.
3314	* Must be page aligned!
3315	* @param cb The size of the range (in bytes).
3316	* Must be page aligned!
3317	* @param pvBinary Pointer to the binary data backing the ROM image.
3318	* @param cbBinary The size of the binary data pvBinary points to.
3319	* This must be less or equal to @a cb.
3320	* @param fFlags Mask of flags. PGMPHYS_ROM_FLAGS_SHADOWED
3321	* and/or PGMPHYS_ROM_FLAGS_PERMANENT_BINARY.
3322	* @param pszDesc Pointer to description string. This must not be freed.
3323	*
3324	* @remark There is no way to remove the rom, automatically on device cleanup or
3325	* manually from the device yet. This isn't difficult in any way, it's
3326	* just not something we expect to be necessary for a while.
3327	*/
3328	VMMR3DECL(int) PGMR3PhysRomRegister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
3329	const void pvBinary, uint32_t cbBinary, uint32_t fFlags, const char pszDesc)
3330	{
3331	Log(("PGMR3PhysRomRegister: pDevIns=%p GCPhys=%RGp(-%RGp) cb=%RGp pvBinary=%p cbBinary=%#x fFlags=%#x pszDesc=%s\n",
3332	pDevIns, GCPhys, GCPhys + cb, cb, pvBinary, cbBinary, fFlags, pszDesc));
3333	pgmLock(pVM);
3334	int rc = pgmR3PhysRomRegister(pVM, pDevIns, GCPhys, cb, pvBinary, cbBinary, fFlags, pszDesc);
3335	pgmUnlock(pVM);
3336	return rc;
3337	}
3338
3339
3340	/**
3341	* \#PF Handler callback for ROM write accesses.
3342	*
3343	* @returns VINF_SUCCESS if the handler have carried out the operation.
3344	* @returns VINF_PGM_HANDLER_DO_DEFAULT if the caller should carry out the access operation.
3345	* @param pVM VM Handle.
3346	* @param GCPhys The physical address the guest is writing to.
3347	* @param pvPhys The HC mapping of that address.
3348	* @param pvBuf What the guest is reading/writing.
3349	* @param cbBuf How much it's reading/writing.
3350	* @param enmAccessType The access type.
3351	* @param pvUser User argument.
3352	*/
3353	static DECLCALLBACK(int) pgmR3PhysRomWriteHandler(PVM pVM, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf,
3354	PGMACCESSTYPE enmAccessType, void *pvUser)
3355	{
3356	PPGMROMRANGE pRom = (PPGMROMRANGE)pvUser;
3357	const uint32_t iPage = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
3358	Assert(iPage < (pRom->cb >> PAGE_SHIFT));
3359	PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
3360	Log5(("pgmR3PhysRomWriteHandler: %d %c %#08RGp %#04zx\n", pRomPage->enmProt, enmAccessType == PGMACCESSTYPE_READ ? 'R' : 'W', GCPhys, cbBuf));
3361
3362	if (enmAccessType == PGMACCESSTYPE_READ)
3363	{
3364	switch (pRomPage->enmProt)
3365	{
3366	/*
3367	* Take the default action.
3368	*/
3369	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
3370	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
3371	case PGMROMPROT_READ_ROM_WRITE_RAM:
3372	case PGMROMPROT_READ_RAM_WRITE_RAM:
3373	return VINF_PGM_HANDLER_DO_DEFAULT;
3374
3375	default:
3376	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
3377	pRom->aPages[iPage].enmProt, iPage, GCPhys),
3378	VERR_INTERNAL_ERROR);
3379	}
3380	}
3381	else
3382	{
3383	Assert(enmAccessType == PGMACCESSTYPE_WRITE);
3384	switch (pRomPage->enmProt)
3385	{
3386	/*
3387	* Ignore writes.
3388	*/
3389	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
3390	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
3391	return VINF_SUCCESS;
3392
3393	/*
3394	* Write to the RAM page.
3395	*/
3396	case PGMROMPROT_READ_ROM_WRITE_RAM:
3397	case PGMROMPROT_READ_RAM_WRITE_RAM: /* yes this will get here too, it's way simpler that way. */
3398	{
3399	/* This should be impossible now, pvPhys doesn't work cross page anylonger. */
3400	Assert(((GCPhys - pRom->GCPhys + cbBuf - 1) >> PAGE_SHIFT) == iPage);
3401
3402	/*
3403	* Take the lock, do lazy allocation, map the page and copy the data.
3404	*
3405	* Note that we have to bypass the mapping TLB since it works on
3406	* guest physical addresses and entering the shadow page would
3407	* kind of screw things up...
3408	*/
3409	int rc = pgmLock(pVM);
3410	AssertRC(rc);
3411
3412	PPGMPAGE pShadowPage = &pRomPage->Shadow;
3413	if (!PGMROMPROT_IS_ROM(pRomPage->enmProt))
3414	{
3415	pShadowPage = pgmPhysGetPage(pVM, GCPhys);
3416	AssertLogRelReturn(pShadowPage, VERR_INTERNAL_ERROR);
3417	}
3418
3419	void *pvDstPage;
3420	rc = pgmPhysPageMakeWritableAndMap(pVM, pShadowPage, GCPhys & X86_PTE_PG_MASK, &pvDstPage);
3421	if (RT_SUCCESS(rc))
3422	{
3423	memcpy((uint8_t *)pvDstPage + (GCPhys & PAGE_OFFSET_MASK), pvBuf, cbBuf);
3424	pRomPage->LiveSave.fWrittenTo = true;
3425	}
3426
3427	pgmUnlock(pVM);
3428	return rc;
3429	}
3430
3431	default:
3432	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
3433	pRom->aPages[iPage].enmProt, iPage, GCPhys),
3434	VERR_INTERNAL_ERROR);
3435	}
3436	}
3437	}
3438
3439
3440	/**
3441	* Called by PGMR3Reset to reset the shadow, switch to the virgin,
3442	* and verify that the virgin part is untouched.
3443	*
3444	* This is done after the normal memory has been cleared.
3445	*
3446	* ASSUMES that the caller owns the PGM lock.
3447	*
3448	* @param pVM The VM handle.
3449	*/
3450	int pgmR3PhysRomReset(PVM pVM)
3451	{
3452	PGM_LOCK_ASSERT_OWNER(pVM);
3453	for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3454	{
3455	const uint32_t cPages = pRom->cb >> PAGE_SHIFT;
3456
3457	if (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3458	{
3459	/*
3460	* Reset the physical handler.
3461	*/
3462	int rc = PGMR3PhysRomProtect(pVM, pRom->GCPhys, pRom->cb, PGMROMPROT_READ_ROM_WRITE_IGNORE);
3463	AssertRCReturn(rc, rc);
3464
3465	/*
3466	* What we do with the shadow pages depends on the memory
3467	* preallocation option. If not enabled, we'll just throw
3468	* out all the dirty pages and replace them by the zero page.
3469	*/
3470	if (!pVM->pgm.s.fRamPreAlloc)
3471	{
3472	/* Free the dirty pages. */
3473	uint32_t cPendingPages = 0;
3474	PGMMFREEPAGESREQ pReq;
3475	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
3476	AssertRCReturn(rc, rc);
3477
3478	for (uint32_t iPage = 0; iPage < cPages; iPage++)
3479	if ( !PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow)
3480	&& !PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow))
3481	{
3482	Assert(PGM_PAGE_GET_STATE(&pRom->aPages[iPage].Shadow) == PGM_PAGE_STATE_ALLOCATED);
3483	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, &pRom->aPages[iPage].Shadow,
3484	pRom->GCPhys + (iPage << PAGE_SHIFT));
3485	AssertLogRelRCReturn(rc, rc);
3486	}
3487
3488	if (cPendingPages)
3489	{
3490	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
3491	AssertLogRelRCReturn(rc, rc);
3492	}
3493	GMMR3FreePagesCleanup(pReq);
3494	}
3495	else
3496	{
3497	/* clear all the shadow pages. */
3498	for (uint32_t iPage = 0; iPage < cPages; iPage++)
3499	{
3500	if (PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow))
3501	continue;
3502	Assert(!PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow));
3503	void *pvDstPage;
3504	const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
3505	rc = pgmPhysPageMakeWritableAndMap(pVM, &pRom->aPages[iPage].Shadow, GCPhys, &pvDstPage);
3506	if (RT_FAILURE(rc))
3507	break;
3508	ASMMemZeroPage(pvDstPage);
3509	}
3510	AssertRCReturn(rc, rc);
3511	}
3512	}
3513
3514	#ifdef VBOX_STRICT
3515	/*
3516	* Verify that the virgin page is unchanged if possible.
3517	*/
3518	if (pRom->pvOriginal)
3519	{
3520	size_t cbSrcLeft = pRom->cbOriginal;
3521	uint8_t const pbSrcPage = (uint8_t const )pRom->pvOriginal;
3522	for (uint32_t iPage = 0; iPage < cPages && cbSrcLeft > 0; iPage++, pbSrcPage += PAGE_SIZE)
3523	{
3524	const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
3525	void const *pvDstPage;
3526	int rc = pgmPhysPageMapReadOnly(pVM, &pRom->aPages[iPage].Virgin, GCPhys, &pvDstPage);
3527	if (RT_FAILURE(rc))
3528	break;
3529
3530	if (memcmp(pvDstPage, pbSrcPage, RT_MIN(cbSrcLeft, PAGE_SIZE)))
3531	LogRel(("pgmR3PhysRomReset: %RGp rom page changed (%s) - loaded saved state?\n",
3532	GCPhys, pRom->pszDesc));
3533	cbSrcLeft -= RT_MIN(cbSrcLeft, PAGE_SIZE);
3534	}
3535	}
3536	#endif
3537	}
3538
3539	return VINF_SUCCESS;
3540	}
3541
3542
3543	/**
3544	* Called by PGMR3Term to free resources.
3545	*
3546	* ASSUMES that the caller owns the PGM lock.
3547	*
3548	* @param pVM The VM handle.
3549	*/
3550	void pgmR3PhysRomTerm(PVM pVM)
3551	{
3552	#ifdef RT_STRICT
3553	/*
3554	* Free the heap copy of the original bits.
3555	*/
3556	for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3557	{
3558	if ( pRom->pvOriginal
3559	&& !(pRom->fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY))
3560	{
3561	RTMemFree((void *)pRom->pvOriginal);
3562	pRom->pvOriginal = NULL;
3563	}
3564	}
3565	#endif
3566	}
3567
3568
3569	/**
3570	* Change the shadowing of a range of ROM pages.
3571	*
3572	* This is intended for implementing chipset specific memory registers
3573	* and will not be very strict about the input. It will silently ignore
3574	* any pages that are not the part of a shadowed ROM.
3575	*
3576	* @returns VBox status code.
3577	* @retval VINF_PGM_SYNC_CR3
3578	*
3579	* @param pVM Pointer to the shared VM structure.
3580	* @param GCPhys Where to start. Page aligned.
3581	* @param cb How much to change. Page aligned.
3582	* @param enmProt The new ROM protection.
3583	*/
3584	VMMR3DECL(int) PGMR3PhysRomProtect(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, PGMROMPROT enmProt)
3585	{
3586	/*
3587	* Check input
3588	*/
3589	if (!cb)
3590	return VINF_SUCCESS;
3591	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3592	AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3593	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
3594	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
3595	AssertReturn(enmProt >= PGMROMPROT_INVALID && enmProt <= PGMROMPROT_END, VERR_INVALID_PARAMETER);
3596
3597	/*
3598	* Process the request.
3599	*/
3600	pgmLock(pVM);
3601	int rc = VINF_SUCCESS;
3602	bool fFlushTLB = false;
3603	for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3604	{
3605	if ( GCPhys <= pRom->GCPhysLast
3606	&& GCPhysLast >= pRom->GCPhys
3607	&& (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED))
3608	{
3609	/*
3610	* Iterate the relevant pages and make necessary the changes.
3611	*/
3612	bool fChanges = false;
3613	uint32_t const cPages = pRom->GCPhysLast <= GCPhysLast
3614	? pRom->cb >> PAGE_SHIFT
3615	: (GCPhysLast - pRom->GCPhys + 1) >> PAGE_SHIFT;
3616	for (uint32_t iPage = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
3617	iPage < cPages;
3618	iPage++)
3619	{
3620	PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
3621	if (PGMROMPROT_IS_ROM(pRomPage->enmProt) != PGMROMPROT_IS_ROM(enmProt))
3622	{
3623	fChanges = true;
3624
3625	/* flush references to the page. */
3626	PPGMPAGE pRamPage = pgmPhysGetPage(pVM, pRom->GCPhys + (iPage << PAGE_SHIFT));
3627	int rc2 = pgmPoolTrackUpdateGCPhys(pVM, pRom->GCPhys + (iPage << PAGE_SHIFT), pRamPage,
3628	true /fFlushPTEs/, &fFlushTLB);
3629	if (rc2 != VINF_SUCCESS && (rc == VINF_SUCCESS \|\| RT_FAILURE(rc2)))
3630	rc = rc2;
3631
3632	PPGMPAGE pOld = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Virgin : &pRomPage->Shadow;
3633	PPGMPAGE pNew = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Shadow : &pRomPage->Virgin;
3634
3635	pOld = pRamPage;
3636	pRamPage = pNew;
3637	/** @todo preserve the volatile flags (handlers) when these have been moved out of HCPhys! */
3638	}
3639	pRomPage->enmProt = enmProt;
3640	}
3641
3642	/*
3643	* Reset the access handler if we made changes, no need
3644	* to optimize this.
3645	*/
3646	if (fChanges)
3647	{
3648	int rc2 = PGMHandlerPhysicalReset(pVM, pRom->GCPhys);
3649	if (RT_FAILURE(rc2))
3650	{
3651	pgmUnlock(pVM);
3652	AssertRC(rc);
3653	return rc2;
3654	}
3655	}
3656
3657	/* Advance - cb isn't updated. */
3658	GCPhys = pRom->GCPhys + (cPages << PAGE_SHIFT);
3659	}
3660	}
3661	pgmUnlock(pVM);
3662	if (fFlushTLB)
3663	PGM_INVL_ALL_VCPU_TLBS(pVM);
3664
3665	return rc;
3666	}
3667
3668
3669	/**
3670	* Sets the Address Gate 20 state.
3671	*
3672	* @param pVCpu The VCPU to operate on.
3673	* @param fEnable True if the gate should be enabled.
3674	* False if the gate should be disabled.
3675	*/
3676	VMMDECL(void) PGMR3PhysSetA20(PVMCPU pVCpu, bool fEnable)
3677	{
3678	LogFlow(("PGMR3PhysSetA20 %d (was %d)\n", fEnable, pVCpu->pgm.s.fA20Enabled));
3679	if (pVCpu->pgm.s.fA20Enabled != fEnable)
3680	{
3681	pVCpu->pgm.s.fA20Enabled = fEnable;
3682	pVCpu->pgm.s.GCPhysA20Mask = ~(RTGCPHYS)(!fEnable << 20);
3683	REMR3A20Set(pVCpu->pVMR3, pVCpu, fEnable);
3684	/** @todo we're not handling this correctly for VT-x / AMD-V. See #2911 */
3685	}
3686	}
3687
3688	#ifdef PGM_WITH_LARGE_ADDRESS_SPACE_ON_32_BIT_HOST
3689
3690	/**
3691	* Tree enumeration callback for dealing with age rollover.
3692	* It will perform a simple compression of the current age.
3693	*/
3694	static DECLCALLBACK(int) pgmR3PhysChunkAgeingRolloverCallback(PAVLU32NODECORE pNode, void *pvUser)
3695	{
3696	/* Age compression - ASSUMES iNow == 4. */
3697	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
3698	if (pChunk->iAge >= UINT32_C(0xffffff00))
3699	pChunk->iAge = 3;
3700	else if (pChunk->iAge >= UINT32_C(0xfffff000))
3701	pChunk->iAge = 2;
3702	else if (pChunk->iAge)
3703	pChunk->iAge = 1;
3704	else /* iAge = 0 */
3705	pChunk->iAge = 4;
3706	return 0;
3707	}
3708
3709
3710	/**
3711	* Tree enumeration callback that updates the chunks that have
3712	* been used since the last
3713	*/
3714	static DECLCALLBACK(int) pgmR3PhysChunkAgeingCallback(PAVLU32NODECORE pNode, void *pvUser)
3715	{
3716	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
3717	if (!pChunk->iAge)
3718	{
3719	PVM pVM = (PVM)pvUser;
3720	pChunk->iAge = pVM->pgm.s.ChunkR3Map.iNow;
3721	}
3722	return 0;
3723	}
3724
3725
3726	/**
3727	* Performs ageing of the ring-3 chunk mappings.
3728	*
3729	* @param pVM The VM handle.
3730	*/
3731	VMMR3DECL(void) PGMR3PhysChunkAgeing(PVM pVM)
3732	{
3733	pgmLock(pVM);
3734	pVM->pgm.s.ChunkR3Map.AgeingCountdown = RT_MIN(pVM->pgm.s.ChunkR3Map.cMax / 4, 1024);
3735	pVM->pgm.s.ChunkR3Map.iNow++;
3736	if (pVM->pgm.s.ChunkR3Map.iNow == 0)
3737	{
3738	pVM->pgm.s.ChunkR3Map.iNow = 4;
3739	RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /fFromLeft/, pgmR3PhysChunkAgeingRolloverCallback, pVM);
3740	}
3741	else
3742	RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /fFromLeft/, pgmR3PhysChunkAgeingCallback, pVM);
3743	pgmUnlock(pVM);
3744	}
3745
3746
3747	/**
3748	* The structure passed in the pvUser argument of pgmR3PhysChunkUnmapCandidateCallback().
3749	*/
3750	typedef struct PGMR3PHYSCHUNKUNMAPCB
3751	{
3752	PVM pVM; /*< The VM handle. /
3753	PPGMCHUNKR3MAP pChunk; /*< The chunk to unmap. /
3754	uint32_t iLastAge; /*< Highest age found so far. /
3755	} PGMR3PHYSCHUNKUNMAPCB, *PPGMR3PHYSCHUNKUNMAPCB;
3756
3757
3758	/**
3759	* Callback used to find the mapping that's been unused for
3760	* the longest time.
3761	*/
3762	static DECLCALLBACK(int) pgmR3PhysChunkUnmapCandidateCallback(PAVLU32NODECORE pNode, void *pvUser)
3763	{
3764	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
3765	PPGMR3PHYSCHUNKUNMAPCB pArg = (PPGMR3PHYSCHUNKUNMAPCB)pvUser;
3766
3767	if ( pChunk->iAge
3768	&& !pChunk->cRefs
3769	&& pArg->iLastAge < pChunk->iAge)
3770	{
3771	/*
3772	* Check that it's not in any of the TLBs.
3773	*/
3774	PVM pVM = pArg->pVM;
3775	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
3776	if (pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk == pChunk)
3777	{
3778	pChunk = NULL;
3779	break;
3780	}
3781	if (pChunk)
3782	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbHC.aEntries); i++)
3783	if (pVM->pgm.s.PhysTlbHC.aEntries[i].pMap == pChunk)
3784	{
3785	pChunk = NULL;
3786	break;
3787	}
3788	if (pChunk)
3789	{
3790	pArg->pChunk = pChunk;
3791	pArg->iLastAge = pChunk->iAge;
3792	}
3793	}
3794	return 0;
3795	}
3796
3797
3798	/**
3799	* Finds a good candidate for unmapping when the ring-3 mapping cache is full.
3800	*
3801	* The candidate will not be part of any TLBs, so no need to flush
3802	* anything afterwards.
3803	*
3804	* @returns Chunk id.
3805	* @param pVM The VM handle.
3806	*/
3807	static int32_t pgmR3PhysChunkFindUnmapCandidate(PVM pVM)
3808	{
3809	PGM_LOCK_ASSERT_OWNER(pVM);
3810
3811	/*
3812	* Do tree ageing first?
3813	*/
3814	if (pVM->pgm.s.ChunkR3Map.AgeingCountdown-- == 0)
3815	{
3816	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkAging, a);
3817	PGMR3PhysChunkAgeing(pVM);
3818	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkAging, a);
3819	}
3820
3821	/*
3822	* Enumerate the age tree starting with the left most node.
3823	*/
3824	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3825	PGMR3PHYSCHUNKUNMAPCB Args;
3826	Args.pVM = pVM;
3827	Args.pChunk = NULL;
3828	Args.iLastAge = 0;
3829	RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /fFromLeft/, pgmR3PhysChunkUnmapCandidateCallback, &Args);
3830	Assert(Args.pChunk);
3831	if (Args.pChunk)
3832	{
3833	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3834	return Args.pChunk->Core.Key;
3835	}
3836
3837	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3838	return INT32_MAX;
3839	}
3840
3841	/**
3842	* Rendezvous callback used by pgmR3PhysUnmapChunk that unmaps a chunk
3843	*
3844	* This is only called on one of the EMTs while the other ones are waiting for
3845	* it to complete this function.
3846	*
3847	* @returns VINF_SUCCESS (VBox strict status code).
3848	* @param pVM The VM handle.
3849	* @param pVCpu The VMCPU for the EMT we're being called on. Unused.
3850	* @param pvUser User pointer. Unused
3851	*
3852	*/
3853	DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysUnmapChunkRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
3854	{
3855	int rc = VINF_SUCCESS;
3856	pgmLock(pVM);
3857
3858	if (pVM->pgm.s.ChunkR3Map.c >= pVM->pgm.s.ChunkR3Map.cMax)
3859	{
3860	/* Flush the pgm pool cache; call the internal rendezvous handler as we're already in a rendezvous handler here. */
3861	/* todo: also not really efficient to unmap a chunk that contains PD or PT pages. */
3862	pgmR3PoolClearAllRendezvous(pVM, &pVM->aCpus[0], NULL /* no need to flush the REM TLB as we already did that above */);
3863
3864	/*
3865	* Request the ring-0 part to unmap a chunk to make space in the mapping cache.
3866	*/
3867	GMMMAPUNMAPCHUNKREQ Req;
3868	Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
3869	Req.Hdr.cbReq = sizeof(Req);
3870	Req.pvR3 = NULL;
3871	Req.idChunkMap = NIL_GMM_CHUNKID;
3872	Req.idChunkUnmap = pgmR3PhysChunkFindUnmapCandidate(pVM);
3873
3874	if (Req.idChunkUnmap != INT32_MAX)
3875	{
3876	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkUnmap, a);
3877	rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
3878	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkUnmap, a);
3879	if (RT_SUCCESS(rc))
3880	{
3881	/* remove the unmapped one. */
3882	PPGMCHUNKR3MAP pUnmappedChunk = (PPGMCHUNKR3MAP)RTAvlU32Remove(&pVM->pgm.s.ChunkR3Map.pTree, Req.idChunkUnmap);
3883	AssertRelease(pUnmappedChunk);
3884	pUnmappedChunk->pv = NULL;
3885	pUnmappedChunk->Core.Key = UINT32_MAX;
3886	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
3887	MMR3HeapFree(pUnmappedChunk);
3888	#else
3889	MMR3UkHeapFree(pVM, pUnmappedChunk, MM_TAG_PGM_CHUNK_MAPPING);
3890	#endif
3891	pVM->pgm.s.ChunkR3Map.c--;
3892	pVM->pgm.s.cUnmappedChunks++;
3893
3894	/* Flush dangling PGM pointers (R3 & R0 ptrs to GC physical addresses) */
3895	/* todo: we should not flush chunks which include cr3 mappings. */
3896	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3897	{
3898	PPGMCPU pPGM = &pVM->aCpus[idCpu].pgm.s;
3899
3900	pPGM->pGst32BitPdR3 = NULL;
3901	pPGM->pGstPaePdptR3 = NULL;
3902	pPGM->pGstAmd64Pml4R3 = NULL;
3903	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
3904	pPGM->pGst32BitPdR0 = NIL_RTR0PTR;
3905	pPGM->pGstPaePdptR0 = NIL_RTR0PTR;
3906	pPGM->pGstAmd64Pml4R0 = NIL_RTR0PTR;
3907	#endif
3908	for (unsigned i = 0; i < RT_ELEMENTS(pPGM->apGstPaePDsR3); i++)
3909	{
3910	pPGM->apGstPaePDsR3[i] = NULL;
3911	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
3912	pPGM->apGstPaePDsR0[i] = NIL_RTR0PTR;
3913	#endif
3914	}
3915
3916	/* Flush REM TLBs. */
3917	CPUMSetChangedFlags(&pVM->aCpus[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
3918	}
3919
3920	/* Flush REM translation blocks. */
3921	REMFlushTBs(pVM);
3922	}
3923	}
3924	}
3925	pgmUnlock(pVM);
3926	return rc;
3927	}
3928
3929	/**
3930	* Unmap a chunk to free up virtual address space (request packet handler for pgmR3PhysChunkMap)
3931	*
3932	* @returns VBox status code.
3933	* @param pVM The VM to operate on.
3934	*/
3935	void pgmR3PhysUnmapChunk(PVM pVM)
3936	{
3937	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysUnmapChunkRendezvous, NULL);
3938	AssertRC(rc);
3939	}
3940
3941	#endif /* PGM_WITH_LARGE_ADDRESS_SPACE_ON_32_BIT_HOST */
3942
3943	/**
3944	* Maps the given chunk into the ring-3 mapping cache.
3945	*
3946	* This will call ring-0.
3947	*
3948	* @returns VBox status code.
3949	* @param pVM The VM handle.
3950	* @param idChunk The chunk in question.
3951	* @param ppChunk Where to store the chunk tracking structure.
3952	*
3953	* @remarks Called from within the PGM critical section.
3954	* @remarks Can be called from any thread!
3955	*/
3956	int pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk)
3957	{
3958	int rc;
3959
3960	PGM_LOCK_ASSERT_OWNER(pVM);
3961
3962	/*
3963	* Allocate a new tracking structure first.
3964	*/
3965	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
3966	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3HeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk));
3967	#else
3968	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3UkHeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk), NULL);
3969	#endif
3970	AssertReturn(pChunk, VERR_NO_MEMORY);
3971	pChunk->Core.Key = idChunk;
3972
3973	/*
3974	* Request the ring-0 part to map the chunk in question.
3975	*/
3976	GMMMAPUNMAPCHUNKREQ Req;
3977	Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
3978	Req.Hdr.cbReq = sizeof(Req);
3979	Req.pvR3 = NULL;
3980	Req.idChunkMap = idChunk;
3981	Req.idChunkUnmap = NIL_GMM_CHUNKID;
3982
3983	/* Must be callable from any thread, so can't use VMMR3CallR0. */
3984	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkMap, a);
3985	rc = SUPR3CallVMMR0Ex(pVM->pVMR0, NIL_VMCPUID, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
3986	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkMap, a);
3987	if (RT_SUCCESS(rc))
3988	{
3989	/*
3990	* Update the tree.
3991	*/
3992	/* insert the new one. */
3993	AssertPtr(Req.pvR3);
3994	pChunk->pv = Req.pvR3;
3995	bool fRc = RTAvlU32Insert(&pVM->pgm.s.ChunkR3Map.pTree, &pChunk->Core);
3996	AssertRelease(fRc);
3997	pVM->pgm.s.ChunkR3Map.c++;
3998	pVM->pgm.s.cMappedChunks++;
3999
4000	/* If we're running out of virtual address space, then we should unmap another chunk. */
4001	if (pVM->pgm.s.ChunkR3Map.c >= pVM->pgm.s.ChunkR3Map.cMax)
4002	{
4003	#ifdef PGM_WITH_LARGE_ADDRESS_SPACE_ON_32_BIT_HOST
4004	/* Postpone the unmap operation (which requires a rendezvous operation) as we own the PGM lock here. */
4005	rc = VMR3ReqCallNoWaitU(pVM->pUVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysUnmapChunk, 1, pVM);
4006	AssertRC(rc);
4007	#else
4008	AssertFatalFailed(); /* can't happen */
4009	#endif
4010	}
4011	}
4012	else
4013	{
4014	/** @todo this may fail because of /proc/sys/vm/max_map_count, so we
4015	* should probably restrict ourselves on linux. */
4016	AssertRC(rc);
4017	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
4018	MMR3HeapFree(pChunk);
4019	#else
4020	MMR3UkHeapFree(pVM, pChunk, MM_TAG_PGM_CHUNK_MAPPING);
4021	#endif
4022	pChunk = NULL;
4023	}
4024
4025	*ppChunk = pChunk;
4026	return rc;
4027	}
4028
4029
4030	/**
4031	* For VMMCALLRING3_PGM_MAP_CHUNK, considered internal.
4032	*
4033	* @returns see pgmR3PhysChunkMap.
4034	* @param pVM The VM handle.
4035	* @param idChunk The chunk to map.
4036	*/
4037	VMMR3DECL(int) PGMR3PhysChunkMap(PVM pVM, uint32_t idChunk)
4038	{
4039	PPGMCHUNKR3MAP pChunk;
4040	int rc;
4041
4042	pgmLock(pVM);
4043	rc = pgmR3PhysChunkMap(pVM, idChunk, &pChunk);
4044	pgmUnlock(pVM);
4045	return rc;
4046	}
4047
4048
4049	/**
4050	* Invalidates the TLB for the ring-3 mapping cache.
4051	*
4052	* @param pVM The VM handle.
4053	*/
4054	VMMR3DECL(void) PGMR3PhysChunkInvalidateTLB(PVM pVM)
4055	{
4056	pgmLock(pVM);
4057	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
4058	{
4059	pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk = NIL_GMM_CHUNKID;
4060	pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk = NULL;
4061	}
4062	/* The page map TLB references chunks, so invalidate that one too. */
4063	pgmPhysInvalidatePageMapTLB(pVM);
4064	pgmUnlock(pVM);
4065	}
4066
4067
4068	/**
4069	* Response to VMMCALLRING3_PGM_ALLOCATE_LARGE_PAGE to allocate a large (2MB) page
4070	* for use with a nested paging PDE.
4071	*
4072	* @returns The following VBox status codes.
4073	* @retval VINF_SUCCESS on success.
4074	* @retval VINF_EM_NO_MEMORY if we're out of memory.
4075	*
4076	* @param pVM The VM handle.
4077	* @param GCPhys GC physical start address of the 2 MB range
4078	*/
4079	VMMR3DECL(int) PGMR3PhysAllocateLargeHandyPage(PVM pVM, RTGCPHYS GCPhys)
4080	{
4081	#ifdef PGM_WITH_LARGE_PAGES
4082	uint64_t u64TimeStamp1, u64TimeStamp2;
4083
4084	pgmLock(pVM);
4085
4086	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatAllocLargePage, a);
4087	u64TimeStamp1 = RTTimeMilliTS();
4088	int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_LARGE_HANDY_PAGE, 0, NULL);
4089	u64TimeStamp2 = RTTimeMilliTS();
4090	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatAllocLargePage, a);
4091	if (RT_SUCCESS(rc))
4092	{
4093	Assert(pVM->pgm.s.cLargeHandyPages == 1);
4094
4095	uint32_t idPage = pVM->pgm.s.aLargeHandyPage[0].idPage;
4096	RTHCPHYS HCPhys = pVM->pgm.s.aLargeHandyPage[0].HCPhysGCPhys;
4097
4098	void *pv;
4099
4100	/* Map the large page into our address space.
4101	*
4102	* Note: assuming that within the 2 MB range:
4103	* - GCPhys + PAGE_SIZE = HCPhys + PAGE_SIZE (whole point of this exercise)
4104	* - user space mapping is continuous as well
4105	* - page id (GCPhys) + 1 = page id (GCPhys + PAGE_SIZE)
4106	*/
4107	rc = pgmPhysPageMapByPageID(pVM, idPage, HCPhys, &pv);
4108	AssertLogRelMsg(RT_SUCCESS(rc), ("idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc\n", idPage, HCPhys, rc));
4109
4110	if (RT_SUCCESS(rc))
4111	{
4112	/*
4113	* Clear the pages.
4114	*/
4115	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatClearLargePage, b);
4116	for (unsigned i = 0; i < _2M/PAGE_SIZE; i++)
4117	{
4118	ASMMemZeroPage(pv);
4119
4120	PPGMPAGE pPage;
4121	rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
4122	AssertRC(rc);
4123
4124	Assert(PGM_PAGE_IS_ZERO(pPage));
4125	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatRZPageReplaceZero);
4126	pVM->pgm.s.cZeroPages--;
4127
4128	/*
4129	* Do the PGMPAGE modifications.
4130	*/
4131	pVM->pgm.s.cPrivatePages++;
4132	PGM_PAGE_SET_HCPHYS(pVM, pPage, HCPhys);
4133	PGM_PAGE_SET_PAGEID(pVM, pPage, idPage);
4134	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
4135	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PDE);
4136	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
4137	PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
4138
4139	/* Somewhat dirty assumption that page ids are increasing. */
4140	idPage++;
4141
4142	HCPhys += PAGE_SIZE;
4143	GCPhys += PAGE_SIZE;
4144
4145	pv = (void *)((uintptr_t)pv + PAGE_SIZE);
4146
4147	Log3(("PGMR3PhysAllocateLargePage: idPage=%#x HCPhys=%RGp\n", idPage, HCPhys));
4148	}
4149	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatClearLargePage, b);
4150
4151	/* Flush all TLBs. */
4152	PGM_INVL_ALL_VCPU_TLBS(pVM);
4153	pgmPhysInvalidatePageMapTLB(pVM);
4154	}
4155	pVM->pgm.s.cLargeHandyPages = 0;
4156	}
4157
4158	if (RT_SUCCESS(rc))
4159	{
4160	static uint32_t cTimeOut = 0;
4161	uint64_t u64TimeStampDelta = u64TimeStamp2 - u64TimeStamp1;
4162
4163	if (u64TimeStampDelta > 100)
4164	{
4165	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatLargePageOverflow);
4166	if ( ++cTimeOut > 10
4167	\|\| u64TimeStampDelta > 1000 /* more than one second forces an early retirement from allocating large pages. */)
4168	{
4169	/* If repeated attempts to allocate a large page takes more than 100 ms, then we fall back to normal 4k pages.
4170	* E.g. Vista 64 tries to move memory around, which takes a huge amount of time.
4171	*/
4172	LogRel(("PGMR3PhysAllocateLargePage: allocating large pages takes too long (last attempt %d ms; nr of timeouts %d); DISABLE\n", u64TimeStampDelta, cTimeOut));
4173	PGMSetLargePageUsage(pVM, false);
4174	}
4175	}
4176	else
4177	if (cTimeOut > 0)
4178	cTimeOut--;
4179	}
4180
4181	pgmUnlock(pVM);
4182	return rc;
4183	#else
4184	return VERR_NOT_IMPLEMENTED;
4185	#endif /* PGM_WITH_LARGE_PAGES */
4186	}
4187
4188
4189	/**
4190	* Response to VM_FF_PGM_NEED_HANDY_PAGES and VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES.
4191	*
4192	* This function will also work the VM_FF_PGM_NO_MEMORY force action flag, to
4193	* signal and clear the out of memory condition. When contracted, this API is
4194	* used to try clear the condition when the user wants to resume.
4195	*
4196	* @returns The following VBox status codes.
4197	* @retval VINF_SUCCESS on success. FFs cleared.
4198	* @retval VINF_EM_NO_MEMORY if we're out of memory. The FF is not cleared in
4199	* this case and it gets accompanied by VM_FF_PGM_NO_MEMORY.
4200	*
4201	* @param pVM The VM handle.
4202	*
4203	* @remarks The VINF_EM_NO_MEMORY status is for the benefit of the FF processing
4204	* in EM.cpp and shouldn't be propagated outside TRPM, HWACCM, EM and
4205	* pgmPhysEnsureHandyPage. There is one exception to this in the \#PF
4206	* handler.
4207	*/
4208	VMMR3DECL(int) PGMR3PhysAllocateHandyPages(PVM pVM)
4209	{
4210	pgmLock(pVM);
4211
4212	/*
4213	* Allocate more pages, noting down the index of the first new page.
4214	*/
4215	uint32_t iClear = pVM->pgm.s.cHandyPages;
4216	AssertMsgReturn(iClear <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d", iClear), VERR_INTERNAL_ERROR);
4217	Log(("PGMR3PhysAllocateHandyPages: %d -> %d\n", iClear, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
4218	int rcAlloc = VINF_SUCCESS;
4219	int rcSeed = VINF_SUCCESS;
4220	int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
4221	while (rc == VERR_GMM_SEED_ME)
4222	{
4223	void *pvChunk;
4224	rcAlloc = rc = SUPR3PageAlloc(GMM_CHUNK_SIZE >> PAGE_SHIFT, &pvChunk);
4225	if (RT_SUCCESS(rc))
4226	{
4227	rcSeed = rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_SEED_CHUNK, (uintptr_t)pvChunk, NULL);
4228	if (RT_FAILURE(rc))
4229	SUPR3PageFree(pvChunk, GMM_CHUNK_SIZE >> PAGE_SHIFT);
4230	}
4231	if (RT_SUCCESS(rc))
4232	rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
4233	}
4234
4235	/* todo: we should split this up into an allocate and flush operation. sometimes you want to flush and not allocate more (which will trigger the vm account limit error) */
4236	if ( rc == VERR_GMM_HIT_VM_ACCOUNT_LIMIT
4237	&& pVM->pgm.s.cHandyPages > 0)
4238	{
4239	/* Still handy pages left, so don't panic. */
4240	rc = VINF_SUCCESS;
4241	}
4242
4243	if (RT_SUCCESS(rc))
4244	{
4245	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
4246	Assert(pVM->pgm.s.cHandyPages > 0);
4247	VM_FF_CLEAR(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
4248	VM_FF_CLEAR(pVM, VM_FF_PGM_NO_MEMORY);
4249
4250	/*
4251	* Clear the pages.
4252	*/
4253	while (iClear < pVM->pgm.s.cHandyPages)
4254	{
4255	PGMMPAGEDESC pPage = &pVM->pgm.s.aHandyPages[iClear];
4256	void *pv;
4257	rc = pgmPhysPageMapByPageID(pVM, pPage->idPage, pPage->HCPhysGCPhys, &pv);
4258	AssertLogRelMsgBreak(RT_SUCCESS(rc), ("idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc\n", pPage->idPage, pPage->HCPhysGCPhys, rc));
4259	ASMMemZeroPage(pv);
4260	iClear++;
4261	Log3(("PGMR3PhysAllocateHandyPages: idPage=%#x HCPhys=%RGp\n", pPage->idPage, pPage->HCPhysGCPhys));
4262	}
4263	}
4264	else
4265	{
4266	uint64_t cAllocPages, cMaxPages, cBalloonPages;
4267
4268	/*
4269	* We should never get here unless there is a genuine shortage of
4270	* memory (or some internal error). Flag the error so the VM can be
4271	* suspended ASAP and the user informed. If we're totally out of
4272	* handy pages we will return failure.
4273	*/
4274	/* Report the failure. */
4275	LogRel(("PGM: Failed to procure handy pages; rc=%Rrc rcAlloc=%Rrc rcSeed=%Rrc cHandyPages=%#x\n"
4276	" cAllPages=%#x cPrivatePages=%#x cSharedPages=%#x cZeroPages=%#x\n",
4277	rc, rcAlloc, rcSeed,
4278	pVM->pgm.s.cHandyPages,
4279	pVM->pgm.s.cAllPages,
4280	pVM->pgm.s.cPrivatePages,
4281	pVM->pgm.s.cSharedPages,
4282	pVM->pgm.s.cZeroPages));
4283
4284	if (GMMR3QueryMemoryStats(pVM, &cAllocPages, &cMaxPages, &cBalloonPages) == VINF_SUCCESS)
4285	{
4286	LogRel(("GMM: Statistics:\n"
4287	" Allocated pages: %RX64\n"
4288	" Maximum pages: %RX64\n"
4289	" Ballooned pages: %RX64\n", cAllocPages, cMaxPages, cBalloonPages));
4290	}
4291
4292	if ( rc != VERR_NO_MEMORY
4293	&& rc != VERR_LOCK_FAILED)
4294	{
4295	for (uint32_t i = 0; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
4296	{
4297	LogRel(("PGM: aHandyPages[#%#04x] = {.HCPhysGCPhys=%RHp, .idPage=%#08x, .idSharedPage=%#08x}\n",
4298	i, pVM->pgm.s.aHandyPages[i].HCPhysGCPhys, pVM->pgm.s.aHandyPages[i].idPage,
4299	pVM->pgm.s.aHandyPages[i].idSharedPage));
4300	uint32_t const idPage = pVM->pgm.s.aHandyPages[i].idPage;
4301	if (idPage != NIL_GMM_PAGEID)
4302	{
4303	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
4304	pRam;
4305	pRam = pRam->pNextR3)
4306	{
4307	uint32_t const cPages = pRam->cb >> PAGE_SHIFT;
4308	for (uint32_t iPage = 0; iPage < cPages; iPage++)
4309	if (PGM_PAGE_GET_PAGEID(&pRam->aPages[iPage]) == idPage)
4310	LogRel(("PGM: Used by %RGp %R[pgmpage] (%s)\n",
4311	pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pRam->aPages[iPage], pRam->pszDesc));
4312	}
4313	}
4314	}
4315	}
4316
4317	/* Set the FFs and adjust rc. */
4318	VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
4319	VM_FF_SET(pVM, VM_FF_PGM_NO_MEMORY);
4320	if ( rc == VERR_NO_MEMORY
4321	\|\| rc == VERR_LOCK_FAILED)
4322	rc = VINF_EM_NO_MEMORY;
4323	}
4324
4325	pgmUnlock(pVM);
4326	return rc;
4327	}
4328
4329
4330	/**
4331	* Frees the specified RAM page and replaces it with the ZERO page.
4332	*
4333	* This is used by ballooning, remapping MMIO2, RAM reset and state loading.
4334	*
4335	* @param pVM Pointer to the shared VM structure.
4336	* @param pReq Pointer to the request.
4337	* @param pcPendingPages Where the number of pages waiting to be freed are
4338	* kept. This will normally be incremented.
4339	* @param pPage Pointer to the page structure.
4340	* @param GCPhys The guest physical address of the page, if applicable.
4341	*
4342	* @remarks The caller must own the PGM lock.
4343	*/
4344	int pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys)
4345	{
4346	/*
4347	* Assert sanity.
4348	*/
4349	PGM_LOCK_ASSERT_OWNER(pVM);
4350	if (RT_UNLIKELY( PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
4351	&& PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_ROM_SHADOW))
4352	{
4353	AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
4354	return VMSetError(pVM, VERR_PGM_PHYS_NOT_RAM, RT_SRC_POS, "GCPhys=%RGp type=%d", GCPhys, PGM_PAGE_GET_TYPE(pPage));
4355	}
4356
4357	/** @todo What about ballooning of large pages??! */
4358	Assert( PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE
4359	&& PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE_DISABLED);
4360
4361	if ( PGM_PAGE_IS_ZERO(pPage)
4362	\|\| PGM_PAGE_IS_BALLOONED(pPage))
4363	return VINF_SUCCESS;
4364
4365	const uint32_t idPage = PGM_PAGE_GET_PAGEID(pPage);
4366	Log3(("pgmPhysFreePage: idPage=%#x GCPhys=%RGp pPage=%R[pgmpage]\n", idPage, GCPhys, pPage));
4367	if (RT_UNLIKELY( idPage == NIL_GMM_PAGEID
4368	\|\| idPage > GMM_PAGEID_LAST
4369	\|\| PGM_PAGE_GET_CHUNKID(pPage) == NIL_GMM_CHUNKID))
4370	{
4371	AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
4372	return VMSetError(pVM, VERR_PGM_PHYS_INVALID_PAGE_ID, RT_SRC_POS, "GCPhys=%RGp idPage=%#x", GCPhys, pPage);
4373	}
4374
4375	/* update page count stats. */
4376	if (PGM_PAGE_IS_SHARED(pPage))
4377	pVM->pgm.s.cSharedPages--;
4378	else
4379	pVM->pgm.s.cPrivatePages--;
4380	pVM->pgm.s.cZeroPages++;
4381
4382	/* Deal with write monitored pages. */
4383	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
4384	{
4385	PGM_PAGE_SET_WRITTEN_TO(pVM, pPage);
4386	pVM->pgm.s.cWrittenToPages++;
4387	}
4388
4389	/*
4390	* pPage = ZERO page.
4391	*/
4392	PGM_PAGE_SET_HCPHYS(pVM, pPage, pVM->pgm.s.HCPhysZeroPg);
4393	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
4394	PGM_PAGE_SET_PAGEID(pVM, pPage, NIL_GMM_PAGEID);
4395	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
4396	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
4397	PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
4398
4399	/* Flush physical page map TLB entry. */
4400	pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
4401
4402	/*
4403	* Make sure it's not in the handy page array.
4404	*/
4405	for (uint32_t i = pVM->pgm.s.cHandyPages; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
4406	{
4407	if (pVM->pgm.s.aHandyPages[i].idPage == idPage)
4408	{
4409	pVM->pgm.s.aHandyPages[i].idPage = NIL_GMM_PAGEID;
4410	break;
4411	}
4412	if (pVM->pgm.s.aHandyPages[i].idSharedPage == idPage)
4413	{
4414	pVM->pgm.s.aHandyPages[i].idSharedPage = NIL_GMM_PAGEID;
4415	break;
4416	}
4417	}
4418
4419	/*
4420	* Push it onto the page array.
4421	*/
4422	uint32_t iPage = *pcPendingPages;
4423	Assert(iPage < PGMPHYS_FREE_PAGE_BATCH_SIZE);
4424	*pcPendingPages += 1;
4425
4426	pReq->aPages[iPage].idPage = idPage;
4427
4428	if (iPage + 1 < PGMPHYS_FREE_PAGE_BATCH_SIZE)
4429	return VINF_SUCCESS;
4430
4431	/*
4432	* Flush the pages.
4433	*/
4434	int rc = GMMR3FreePagesPerform(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE);
4435	if (RT_SUCCESS(rc))
4436	{
4437	GMMR3FreePagesRePrep(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
4438	*pcPendingPages = 0;
4439	}
4440	return rc;
4441	}
4442
4443
4444	/**
4445	* Converts a GC physical address to a HC ring-3 pointer, with some
4446	* additional checks.
4447	*
4448	* @returns VBox status code.
4449	* @retval VINF_SUCCESS on success.
4450	* @retval VINF_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
4451	* access handler of some kind.
4452	* @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
4453	* accesses or is odd in any way.
4454	* @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
4455	*
4456	* @param pVM The VM handle.
4457	* @param GCPhys The GC physical address to convert.
4458	* @param fWritable Whether write access is required.
4459	* @param ppv Where to store the pointer corresponding to GCPhys on
4460	* success.
4461	*/
4462	VMMR3DECL(int) PGMR3PhysTlbGCPhys2Ptr(PVM pVM, RTGCPHYS GCPhys, bool fWritable, void **ppv)
4463	{
4464	pgmLock(pVM);
4465
4466	PPGMRAMRANGE pRam;
4467	PPGMPAGE pPage;
4468	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
4469	if (RT_SUCCESS(rc))
4470	{
4471	if (PGM_PAGE_IS_BALLOONED(pPage))
4472	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
4473	else if (!PGM_PAGE_HAS_ANY_HANDLERS(pPage))
4474	rc = VINF_SUCCESS;
4475	else
4476	{
4477	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
4478	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4479	else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
4480	{
4481	/** @todo Handle TLB loads of virtual handlers so ./test.sh can be made to work
4482	* in -norawr0 mode. */
4483	if (fWritable)
4484	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
4485	}
4486	else
4487	{
4488	/* Temporarily disabled physical handler(s), since the recompiler
4489	doesn't get notified when it's reset we'll have to pretend it's
4490	operating normally. */
4491	if (pgmHandlerPhysicalIsAll(pVM, GCPhys))
4492	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4493	else
4494	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
4495	}
4496	}
4497	if (RT_SUCCESS(rc))
4498	{
4499	int rc2;
4500
4501	/* Make sure what we return is writable. */
4502	if (fWritable)
4503	switch (PGM_PAGE_GET_STATE(pPage))
4504	{
4505	case PGM_PAGE_STATE_ALLOCATED:
4506	break;
4507	case PGM_PAGE_STATE_BALLOONED:
4508	AssertFailed();
4509	break;
4510	case PGM_PAGE_STATE_ZERO:
4511	case PGM_PAGE_STATE_SHARED:
4512	if (rc == VINF_PGM_PHYS_TLB_CATCH_WRITE)
4513	break;
4514	case PGM_PAGE_STATE_WRITE_MONITORED:
4515	rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
4516	AssertLogRelRCReturn(rc2, rc2);
4517	break;
4518	}
4519
4520	/* Get a ring-3 mapping of the address. */
4521	PPGMPAGER3MAPTLBE pTlbe;
4522	rc2 = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
4523	AssertLogRelRCReturn(rc2, rc2);
4524	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
4525	/** @todo mapping/locking hell; this isn't horribly efficient since
4526	* pgmPhysPageLoadIntoTlb will repeat the lookup we've done here. */
4527
4528	Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage] ppv=%p\n", GCPhys, rc, pPage, ppv));
4529	}
4530	else
4531	Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage]\n", GCPhys, rc, pPage));
4532
4533	/* else: handler catching all access, no pointer returned. */
4534	}
4535	else
4536	rc = VERR_PGM_PHYS_TLB_UNASSIGNED;
4537
4538	pgmUnlock(pVM);
4539	return rc;
4540	}
4541

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR3/PGMPhys.cpp@ 37423

Download in other formats: