PGMPhys.cpp@ 41836

Last change on this file since 41836 was 41836, checked in by vboxsync, 13 years ago
Doxygen.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 164.2 KB

Line
1	/* $Id: PGMPhys.cpp 41836 2012-06-19 16:20:52Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor, Physical Memory Addressing.
4	*/
5
6	/*
7	* Copyright (C) 2006-2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_PGM_PHYS
23	#include <VBox/vmm/pgm.h>
24	#include <VBox/vmm/iom.h>
25	#include <VBox/vmm/mm.h>
26	#include <VBox/vmm/stam.h>
27	#ifdef VBOX_WITH_REM
28	# include <VBox/vmm/rem.h>
29	#endif
30	#include <VBox/vmm/pdmdev.h>
31	#include "PGMInternal.h"
32	#include <VBox/vmm/vm.h>
33	#include "PGMInline.h"
34	#include <VBox/sup.h>
35	#include <VBox/param.h>
36	#include <VBox/err.h>
37	#include <VBox/log.h>
38	#include <iprt/assert.h>
39	#include <iprt/alloc.h>
40	#include <iprt/asm.h>
41	#include <iprt/thread.h>
42	#include <iprt/string.h>
43	#include <iprt/system.h>
44
45
46	/*******************************************************************************
47	* Defined Constants And Macros *
48	*******************************************************************************/
49	/** The number of pages to free in one batch. */
50	#define PGMPHYS_FREE_PAGE_BATCH_SIZE 128
51
52
53	/*******************************************************************************
54	* Internal Functions *
55	*******************************************************************************/
56	static DECLCALLBACK(int) pgmR3PhysRomWriteHandler(PVM pVM, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser);
57
58
59	/*
60	* PGMR3PhysReadU8-64
61	* PGMR3PhysWriteU8-64
62	*/
63	#define PGMPHYSFN_READNAME PGMR3PhysReadU8
64	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU8
65	#define PGMPHYS_DATASIZE 1
66	#define PGMPHYS_DATATYPE uint8_t
67	#include "PGMPhysRWTmpl.h"
68
69	#define PGMPHYSFN_READNAME PGMR3PhysReadU16
70	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU16
71	#define PGMPHYS_DATASIZE 2
72	#define PGMPHYS_DATATYPE uint16_t
73	#include "PGMPhysRWTmpl.h"
74
75	#define PGMPHYSFN_READNAME PGMR3PhysReadU32
76	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU32
77	#define PGMPHYS_DATASIZE 4
78	#define PGMPHYS_DATATYPE uint32_t
79	#include "PGMPhysRWTmpl.h"
80
81	#define PGMPHYSFN_READNAME PGMR3PhysReadU64
82	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU64
83	#define PGMPHYS_DATASIZE 8
84	#define PGMPHYS_DATATYPE uint64_t
85	#include "PGMPhysRWTmpl.h"
86
87
88	/**
89	* EMT worker for PGMR3PhysReadExternal.
90	*/
91	static DECLCALLBACK(int) pgmR3PhysReadExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, void *pvBuf, size_t cbRead)
92	{
93	PGMPhysRead(pVM, *pGCPhys, pvBuf, cbRead);
94	return VINF_SUCCESS;
95	}
96
97
98	/**
99	* Read from physical memory, external users.
100	*
101	* @returns VBox status code.
102	* @retval VINF_SUCCESS.
103	*
104	* @param pVM Pointer to the VM.
105	* @param GCPhys Physical address to read from.
106	* @param pvBuf Where to read into.
107	* @param cbRead How many bytes to read.
108	*
109	* @thread Any but EMTs.
110	*/
111	VMMR3DECL(int) PGMR3PhysReadExternal(PVM pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead)
112	{
113	VM_ASSERT_OTHER_THREAD(pVM);
114
115	AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
116	LogFlow(("PGMR3PhysReadExternal: %RGp %d\n", GCPhys, cbRead));
117
118	pgmLock(pVM);
119
120	/*
121	* Copy loop on ram ranges.
122	*/
123	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
124	for (;;)
125	{
126	/* Inside range or not? */
127	if (pRam && GCPhys >= pRam->GCPhys)
128	{
129	/*
130	* Must work our way thru this page by page.
131	*/
132	RTGCPHYS off = GCPhys - pRam->GCPhys;
133	while (off < pRam->cb)
134	{
135	unsigned iPage = off >> PAGE_SHIFT;
136	PPGMPAGE pPage = &pRam->aPages[iPage];
137
138	/*
139	* If the page has an ALL access handler, we'll have to
140	* delegate the job to EMT.
141	*/
142	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
143	{
144	pgmUnlock(pVM);
145
146	return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysReadExternalEMT, 4,
147	pVM, &GCPhys, pvBuf, cbRead);
148	}
149	Assert(!PGM_PAGE_IS_MMIO(pPage));
150
151	/*
152	* Simple stuff, go ahead.
153	*/
154	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
155	if (cb > cbRead)
156	cb = cbRead;
157	PGMPAGEMAPLOCK PgMpLck;
158	const void *pvSrc;
159	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc, &PgMpLck);
160	if (RT_SUCCESS(rc))
161	{
162	memcpy(pvBuf, pvSrc, cb);
163	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
164	}
165	else
166	{
167	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
168	pRam->GCPhys + off, pPage, rc));
169	memset(pvBuf, 0xff, cb);
170	}
171
172	/* next page */
173	if (cb >= cbRead)
174	{
175	pgmUnlock(pVM);
176	return VINF_SUCCESS;
177	}
178	cbRead -= cb;
179	off += cb;
180	GCPhys += cb;
181	pvBuf = (char *)pvBuf + cb;
182	} /* walk pages in ram range. */
183	}
184	else
185	{
186	LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
187
188	/*
189	* Unassigned address space.
190	*/
191	size_t cb = pRam ? pRam->GCPhys - GCPhys : ~(size_t)0;
192	if (cb >= cbRead)
193	{
194	memset(pvBuf, 0xff, cbRead);
195	break;
196	}
197	memset(pvBuf, 0xff, cb);
198
199	cbRead -= cb;
200	pvBuf = (char *)pvBuf + cb;
201	GCPhys += cb;
202	}
203
204	/* Advance range if necessary. */
205	while (pRam && GCPhys > pRam->GCPhysLast)
206	pRam = pRam->CTX_SUFF(pNext);
207	} /* Ram range walk */
208
209	pgmUnlock(pVM);
210
211	return VINF_SUCCESS;
212	}
213
214
215	/**
216	* EMT worker for PGMR3PhysWriteExternal.
217	*/
218	static DECLCALLBACK(int) pgmR3PhysWriteExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, const void *pvBuf, size_t cbWrite)
219	{
220	/** @todo VERR_EM_NO_MEMORY */
221	PGMPhysWrite(pVM, *pGCPhys, pvBuf, cbWrite);
222	return VINF_SUCCESS;
223	}
224
225
226	/**
227	* Write to physical memory, external users.
228	*
229	* @returns VBox status code.
230	* @retval VINF_SUCCESS.
231	* @retval VERR_EM_NO_MEMORY.
232	*
233	* @param pVM Pointer to the VM.
234	* @param GCPhys Physical address to write to.
235	* @param pvBuf What to write.
236	* @param cbWrite How many bytes to write.
237	* @param pszWho Who is writing. For tracking down who is writing
238	* after we've saved the state.
239	*
240	* @thread Any but EMTs.
241	*/
242	VMMDECL(int) PGMR3PhysWriteExternal(PVM pVM, RTGCPHYS GCPhys, const void pvBuf, size_t cbWrite, const char pszWho)
243	{
244	VM_ASSERT_OTHER_THREAD(pVM);
245
246	AssertMsg(!pVM->pgm.s.fNoMorePhysWrites,
247	("Calling PGMR3PhysWriteExternal after pgmR3Save()! GCPhys=%RGp cbWrite=%#x pszWho=%s\n",
248	GCPhys, cbWrite, pszWho));
249	AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
250	LogFlow(("PGMR3PhysWriteExternal: %RGp %d\n", GCPhys, cbWrite));
251
252	pgmLock(pVM);
253
254	/*
255	* Copy loop on ram ranges, stop when we hit something difficult.
256	*/
257	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
258	for (;;)
259	{
260	/* Inside range or not? */
261	if (pRam && GCPhys >= pRam->GCPhys)
262	{
263	/*
264	* Must work our way thru this page by page.
265	*/
266	RTGCPTR off = GCPhys - pRam->GCPhys;
267	while (off < pRam->cb)
268	{
269	RTGCPTR iPage = off >> PAGE_SHIFT;
270	PPGMPAGE pPage = &pRam->aPages[iPage];
271
272	/*
273	* Is the page problematic, we have to do the work on the EMT.
274	*
275	* Allocating writable pages and access handlers are
276	* problematic, write monitored pages are simple and can be
277	* dealt with here.
278	*/
279	if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
280	\|\| PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
281	{
282	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
283	&& !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
284	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
285	else
286	{
287	pgmUnlock(pVM);
288
289	return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysWriteExternalEMT, 4,
290	pVM, &GCPhys, pvBuf, cbWrite);
291	}
292	}
293	Assert(!PGM_PAGE_IS_MMIO(pPage));
294
295	/*
296	* Simple stuff, go ahead.
297	*/
298	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
299	if (cb > cbWrite)
300	cb = cbWrite;
301	PGMPAGEMAPLOCK PgMpLck;
302	void *pvDst;
303	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst, &PgMpLck);
304	if (RT_SUCCESS(rc))
305	{
306	memcpy(pvDst, pvBuf, cb);
307	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
308	}
309	else
310	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
311	pRam->GCPhys + off, pPage, rc));
312
313	/* next page */
314	if (cb >= cbWrite)
315	{
316	pgmUnlock(pVM);
317	return VINF_SUCCESS;
318	}
319
320	cbWrite -= cb;
321	off += cb;
322	GCPhys += cb;
323	pvBuf = (const char *)pvBuf + cb;
324	} /* walk pages in ram range */
325	}
326	else
327	{
328	/*
329	* Unassigned address space, skip it.
330	*/
331	if (!pRam)
332	break;
333	size_t cb = pRam->GCPhys - GCPhys;
334	if (cb >= cbWrite)
335	break;
336	cbWrite -= cb;
337	pvBuf = (const char *)pvBuf + cb;
338	GCPhys += cb;
339	}
340
341	/* Advance range if necessary. */
342	while (pRam && GCPhys > pRam->GCPhysLast)
343	pRam = pRam->CTX_SUFF(pNext);
344	} /* Ram range walk */
345
346	pgmUnlock(pVM);
347	return VINF_SUCCESS;
348	}
349
350
351	/**
352	* VMR3ReqCall worker for PGMR3PhysGCPhys2CCPtrExternal to make pages writable.
353	*
354	* @returns see PGMR3PhysGCPhys2CCPtrExternal
355	* @param pVM Pointer to the VM.
356	* @param pGCPhys Pointer to the guest physical address.
357	* @param ppv Where to store the mapping address.
358	* @param pLock Where to store the lock.
359	*/
360	static DECLCALLBACK(int) pgmR3PhysGCPhys2CCPtrDelegated(PVM pVM, PRTGCPHYS pGCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
361	{
362	/*
363	* Just hand it to PGMPhysGCPhys2CCPtr and check that it's not a page with
364	* an access handler after it succeeds.
365	*/
366	int rc = pgmLock(pVM);
367	AssertRCReturn(rc, rc);
368
369	rc = PGMPhysGCPhys2CCPtr(pVM, *pGCPhys, ppv, pLock);
370	if (RT_SUCCESS(rc))
371	{
372	PPGMPAGEMAPTLBE pTlbe;
373	int rc2 = pgmPhysPageQueryTlbe(pVM, *pGCPhys, &pTlbe);
374	AssertFatalRC(rc2);
375	PPGMPAGE pPage = pTlbe->pPage;
376	if (PGM_PAGE_IS_MMIO(pPage))
377	{
378	PGMPhysReleasePageMappingLock(pVM, pLock);
379	rc = VERR_PGM_PHYS_PAGE_RESERVED;
380	}
381	else if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
382	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
383	\|\| pgmPoolIsDirtyPage(pVM, *pGCPhys)
384	#endif
385	)
386	{
387	/* We must flush any corresponding pgm pool page here, otherwise we'll
388	* not be informed about writes and keep bogus gst->shw mappings around.
389	*/
390	pgmPoolFlushPageByGCPhys(pVM, *pGCPhys);
391	Assert(!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage));
392	/** @todo r=bird: return VERR_PGM_PHYS_PAGE_RESERVED here if it still has
393	* active handlers, see the PGMR3PhysGCPhys2CCPtrExternal docs. */
394	}
395	}
396
397	pgmUnlock(pVM);
398	return rc;
399	}
400
401
402	/**
403	* Requests the mapping of a guest page into ring-3, external threads.
404	*
405	* When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
406	* release it.
407	*
408	* This API will assume your intention is to write to the page, and will
409	* therefore replace shared and zero pages. If you do not intend to modify the
410	* page, use the PGMR3PhysGCPhys2CCPtrReadOnlyExternal() API.
411	*
412	* @returns VBox status code.
413	* @retval VINF_SUCCESS on success.
414	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
415	* backing or if the page has any active access handlers. The caller
416	* must fall back on using PGMR3PhysWriteExternal.
417	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
418	*
419	* @param pVM Pointer to the VM.
420	* @param GCPhys The guest physical address of the page that should be mapped.
421	* @param ppv Where to store the address corresponding to GCPhys.
422	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
423	*
424	* @remark Avoid calling this API from within critical sections (other than the
425	* PGM one) because of the deadlock risk when we have to delegating the
426	* task to an EMT.
427	* @thread Any.
428	*/
429	VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrExternal(PVM pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
430	{
431	AssertPtr(ppv);
432	AssertPtr(pLock);
433
434	Assert(VM_IS_EMT(pVM) \|\| !PGMIsLockOwner(pVM));
435
436	int rc = pgmLock(pVM);
437	AssertRCReturn(rc, rc);
438
439	/*
440	* Query the Physical TLB entry for the page (may fail).
441	*/
442	PPGMPAGEMAPTLBE pTlbe;
443	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
444	if (RT_SUCCESS(rc))
445	{
446	PPGMPAGE pPage = pTlbe->pPage;
447	if (PGM_PAGE_IS_MMIO(pPage))
448	rc = VERR_PGM_PHYS_PAGE_RESERVED;
449	else
450	{
451	/*
452	* If the page is shared, the zero page, or being write monitored
453	* it must be converted to an page that's writable if possible.
454	* We can only deal with write monitored pages here, the rest have
455	* to be on an EMT.
456	*/
457	if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
458	\|\| PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
459	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
460	\|\| pgmPoolIsDirtyPage(pVM, GCPhys)
461	#endif
462	)
463	{
464	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
465	&& !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
466	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
467	&& !pgmPoolIsDirtyPage(pVM, GCPhys)
468	#endif
469	)
470	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
471	else
472	{
473	pgmUnlock(pVM);
474
475	return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysGCPhys2CCPtrDelegated, 4,
476	pVM, &GCPhys, ppv, pLock);
477	}
478	}
479
480	/*
481	* Now, just perform the locking and calculate the return address.
482	*/
483	PPGMPAGEMAP pMap = pTlbe->pMap;
484	if (pMap)
485	pMap->cRefs++;
486
487	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
488	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
489	{
490	if (cLocks == 0)
491	pVM->pgm.s.cWriteLockedPages++;
492	PGM_PAGE_INC_WRITE_LOCKS(pPage);
493	}
494	else if (cLocks != PGM_PAGE_GET_WRITE_LOCKS(pPage))
495	{
496	PGM_PAGE_INC_WRITE_LOCKS(pPage);
497	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent write locked state!\n", GCPhys, pPage));
498	if (pMap)
499	pMap->cRefs++; /* Extra ref to prevent it from going away. */
500	}
501
502	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
503	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_WRITE;
504	pLock->pvMap = pMap;
505	}
506	}
507
508	pgmUnlock(pVM);
509	return rc;
510	}
511
512
513	/**
514	* Requests the mapping of a guest page into ring-3, external threads.
515	*
516	* When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
517	* release it.
518	*
519	* @returns VBox status code.
520	* @retval VINF_SUCCESS on success.
521	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
522	* backing or if the page as an active ALL access handler. The caller
523	* must fall back on using PGMPhysRead.
524	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
525	*
526	* @param pVM Pointer to the VM.
527	* @param GCPhys The guest physical address of the page that should be mapped.
528	* @param ppv Where to store the address corresponding to GCPhys.
529	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
530	*
531	* @remark Avoid calling this API from within critical sections (other than
532	* the PGM one) because of the deadlock risk.
533	* @thread Any.
534	*/
535	VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrReadOnlyExternal(PVM pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
536	{
537	int rc = pgmLock(pVM);
538	AssertRCReturn(rc, rc);
539
540	/*
541	* Query the Physical TLB entry for the page (may fail).
542	*/
543	PPGMPAGEMAPTLBE pTlbe;
544	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
545	if (RT_SUCCESS(rc))
546	{
547	PPGMPAGE pPage = pTlbe->pPage;
548	#if 1
549	/* MMIO pages doesn't have any readable backing. */
550	if (PGM_PAGE_IS_MMIO(pPage))
551	rc = VERR_PGM_PHYS_PAGE_RESERVED;
552	#else
553	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
554	rc = VERR_PGM_PHYS_PAGE_RESERVED;
555	#endif
556	else
557	{
558	/*
559	* Now, just perform the locking and calculate the return address.
560	*/
561	PPGMPAGEMAP pMap = pTlbe->pMap;
562	if (pMap)
563	pMap->cRefs++;
564
565	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
566	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
567	{
568	if (cLocks == 0)
569	pVM->pgm.s.cReadLockedPages++;
570	PGM_PAGE_INC_READ_LOCKS(pPage);
571	}
572	else if (cLocks != PGM_PAGE_GET_READ_LOCKS(pPage))
573	{
574	PGM_PAGE_INC_READ_LOCKS(pPage);
575	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent readonly locked state!\n", GCPhys, pPage));
576	if (pMap)
577	pMap->cRefs++; /* Extra ref to prevent it from going away. */
578	}
579
580	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
581	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_READ;
582	pLock->pvMap = pMap;
583	}
584	}
585
586	pgmUnlock(pVM);
587	return rc;
588	}
589
590
591	#define MAKE_LEAF(a_pNode) \
592	do { \
593	(a_pNode)->pLeftR3 = NIL_RTR3PTR; \
594	(a_pNode)->pRightR3 = NIL_RTR3PTR; \
595	(a_pNode)->pLeftR0 = NIL_RTR0PTR; \
596	(a_pNode)->pRightR0 = NIL_RTR0PTR; \
597	(a_pNode)->pLeftRC = NIL_RTRCPTR; \
598	(a_pNode)->pRightRC = NIL_RTRCPTR; \
599	} while (0)
600
601	#define INSERT_LEFT(a_pParent, a_pNode) \
602	do { \
603	(a_pParent)->pLeftR3 = (a_pNode); \
604	(a_pParent)->pLeftR0 = (a_pNode)->pSelfR0; \
605	(a_pParent)->pLeftRC = (a_pNode)->pSelfRC; \
606	} while (0)
607	#define INSERT_RIGHT(a_pParent, a_pNode) \
608	do { \
609	(a_pParent)->pRightR3 = (a_pNode); \
610	(a_pParent)->pRightR0 = (a_pNode)->pSelfR0; \
611	(a_pParent)->pRightRC = (a_pNode)->pSelfRC; \
612	} while (0)
613
614
615	/**
616	* Recursive tree builder.
617	*
618	* @param ppRam Pointer to the iterator variable.
619	* @param iHeight The hight about normal leaf nodes. Inserts a leaf
620	* node if 0.
621	*/
622	static PPGMRAMRANGE pgmR3PhysRebuildRamRangeSearchTreesRecursively(PPGMRAMRANGE *ppRam, int iDepth)
623	{
624	PPGMRAMRANGE pRam;
625	if (iDepth <= 0)
626	{
627	/*
628	* Leaf node.
629	*/
630	pRam = *ppRam;
631	if (pRam)
632	{
633	*ppRam = pRam->pNextR3;
634	MAKE_LEAF(pRam);
635	}
636	}
637	else
638	{
639
640	/*
641	* Intermediate node.
642	*/
643	PPGMRAMRANGE pLeft = pgmR3PhysRebuildRamRangeSearchTreesRecursively(ppRam, iDepth - 1);
644
645	pRam = *ppRam;
646	if (!pRam)
647	return pLeft;
648	*ppRam = pRam->pNextR3;
649	MAKE_LEAF(pRam);
650	INSERT_LEFT(pRam, pLeft);
651
652	PPGMRAMRANGE pRight = pgmR3PhysRebuildRamRangeSearchTreesRecursively(ppRam, iDepth - 1);
653	if (pRight)
654	INSERT_RIGHT(pRam, pRight);
655	}
656	return pRam;
657	}
658
659
660	/**
661	* Rebuilds the RAM range search trees.
662	*
663	* @param pVM Pointer to the VM.
664	*/
665	static void pgmR3PhysRebuildRamRangeSearchTrees(PVM pVM)
666	{
667
668	/*
669	* Create the reasonably balanced tree in a sequential fashion.
670	* For simplicity (laziness) we use standard recursion here.
671	*/
672	int iDepth = 0;
673	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
674	PPGMRAMRANGE pRoot = pgmR3PhysRebuildRamRangeSearchTreesRecursively(&pRam, 0);
675	while (pRam)
676	{
677	PPGMRAMRANGE pLeft = pRoot;
678
679	pRoot = pRam;
680	pRam = pRam->pNextR3;
681	MAKE_LEAF(pRoot);
682	INSERT_LEFT(pRoot, pLeft);
683
684	PPGMRAMRANGE pRight = pgmR3PhysRebuildRamRangeSearchTreesRecursively(&pRam, iDepth);
685	if (pRight)
686	INSERT_RIGHT(pRoot, pRight);
687	/** @todo else: rotate the tree. */
688
689	iDepth++;
690	}
691
692	pVM->pgm.s.pRamRangeTreeR3 = pRoot;
693	pVM->pgm.s.pRamRangeTreeR0 = pRoot ? pRoot->pSelfR0 : NIL_RTR0PTR;
694	pVM->pgm.s.pRamRangeTreeRC = pRoot ? pRoot->pSelfRC : NIL_RTRCPTR;
695
696	#ifdef VBOX_STRICT
697	/*
698	* Verify that the above code works.
699	*/
700	unsigned cRanges = 0;
701	for (pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
702	cRanges++;
703	Assert(cRanges > 0);
704
705	unsigned cMaxDepth = ASMBitLastSetU32(cRanges);
706	if ((1U << cMaxDepth) < cRanges)
707	cMaxDepth++;
708
709	for (pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
710	{
711	unsigned cDepth = 0;
712	PPGMRAMRANGE pRam2 = pVM->pgm.s.pRamRangeTreeR3;
713	for (;;)
714	{
715	if (pRam == pRam2)
716	break;
717	Assert(pRam2);
718	if (pRam->GCPhys < pRam2->GCPhys)
719	pRam2 = pRam2->pLeftR3;
720	else
721	pRam2 = pRam2->pRightR3;
722	}
723	AssertMsg(cDepth <= cMaxDepth, ("cDepth=%d cMaxDepth=%d\n", cDepth, cMaxDepth));
724	}
725	#endif /* VBOX_STRICT */
726	}
727
728	#undef MAKE_LEAF
729	#undef INSERT_LEFT
730	#undef INSERT_RIGHT
731
732	/**
733	* Relinks the RAM ranges using the pSelfRC and pSelfR0 pointers.
734	*
735	* Called when anything was relocated.
736	*
737	* @param pVM Pointer to the VM.
738	*/
739	void pgmR3PhysRelinkRamRanges(PVM pVM)
740	{
741	PPGMRAMRANGE pCur;
742
743	#ifdef VBOX_STRICT
744	for (pCur = pVM->pgm.s.pRamRangesXR3; pCur; pCur = pCur->pNextR3)
745	{
746	Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pCur->pSelfR0 == MMHyperCCToR0(pVM, pCur));
747	Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pCur->pSelfRC == MMHyperCCToRC(pVM, pCur));
748	Assert((pCur->GCPhys & PAGE_OFFSET_MASK) == 0);
749	Assert((pCur->GCPhysLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
750	Assert((pCur->cb & PAGE_OFFSET_MASK) == 0);
751	Assert(pCur->cb == pCur->GCPhysLast - pCur->GCPhys + 1);
752	for (PPGMRAMRANGE pCur2 = pVM->pgm.s.pRamRangesXR3; pCur2; pCur2 = pCur2->pNextR3)
753	Assert( pCur2 == pCur
754	\|\| strcmp(pCur2->pszDesc, pCur->pszDesc)); /** @todo fix MMIO ranges!! */
755	}
756	#endif
757
758	pCur = pVM->pgm.s.pRamRangesXR3;
759	if (pCur)
760	{
761	pVM->pgm.s.pRamRangesXR0 = pCur->pSelfR0;
762	pVM->pgm.s.pRamRangesXRC = pCur->pSelfRC;
763
764	for (; pCur->pNextR3; pCur = pCur->pNextR3)
765	{
766	pCur->pNextR0 = pCur->pNextR3->pSelfR0;
767	pCur->pNextRC = pCur->pNextR3->pSelfRC;
768	}
769
770	Assert(pCur->pNextR0 == NIL_RTR0PTR);
771	Assert(pCur->pNextRC == NIL_RTRCPTR);
772	}
773	else
774	{
775	Assert(pVM->pgm.s.pRamRangesXR0 == NIL_RTR0PTR);
776	Assert(pVM->pgm.s.pRamRangesXRC == NIL_RTRCPTR);
777	}
778	ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
779
780	pgmR3PhysRebuildRamRangeSearchTrees(pVM);
781	}
782
783
784	/**
785	* Links a new RAM range into the list.
786	*
787	* @param pVM Pointer to the VM.
788	* @param pNew Pointer to the new list entry.
789	* @param pPrev Pointer to the previous list entry. If NULL, insert as head.
790	*/
791	static void pgmR3PhysLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, PPGMRAMRANGE pPrev)
792	{
793	AssertMsg(pNew->pszDesc, ("%RGp-%RGp\n", pNew->GCPhys, pNew->GCPhysLast));
794	Assert((pNew->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pNew->pSelfR0 == MMHyperCCToR0(pVM, pNew));
795	Assert((pNew->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pNew->pSelfRC == MMHyperCCToRC(pVM, pNew));
796
797	pgmLock(pVM);
798
799	PPGMRAMRANGE pRam = pPrev ? pPrev->pNextR3 : pVM->pgm.s.pRamRangesXR3;
800	pNew->pNextR3 = pRam;
801	pNew->pNextR0 = pRam ? pRam->pSelfR0 : NIL_RTR0PTR;
802	pNew->pNextRC = pRam ? pRam->pSelfRC : NIL_RTRCPTR;
803
804	if (pPrev)
805	{
806	pPrev->pNextR3 = pNew;
807	pPrev->pNextR0 = pNew->pSelfR0;
808	pPrev->pNextRC = pNew->pSelfRC;
809	}
810	else
811	{
812	pVM->pgm.s.pRamRangesXR3 = pNew;
813	pVM->pgm.s.pRamRangesXR0 = pNew->pSelfR0;
814	pVM->pgm.s.pRamRangesXRC = pNew->pSelfRC;
815	}
816	ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
817
818	pgmR3PhysRebuildRamRangeSearchTrees(pVM);
819	pgmUnlock(pVM);
820	}
821
822
823	/**
824	* Unlink an existing RAM range from the list.
825	*
826	* @param pVM Pointer to the VM.
827	* @param pRam Pointer to the new list entry.
828	* @param pPrev Pointer to the previous list entry. If NULL, insert as head.
829	*/
830	static void pgmR3PhysUnlinkRamRange2(PVM pVM, PPGMRAMRANGE pRam, PPGMRAMRANGE pPrev)
831	{
832	Assert(pPrev ? pPrev->pNextR3 == pRam : pVM->pgm.s.pRamRangesXR3 == pRam);
833	Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pRam->pSelfR0 == MMHyperCCToR0(pVM, pRam));
834	Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pRam->pSelfRC == MMHyperCCToRC(pVM, pRam));
835
836	pgmLock(pVM);
837
838	PPGMRAMRANGE pNext = pRam->pNextR3;
839	if (pPrev)
840	{
841	pPrev->pNextR3 = pNext;
842	pPrev->pNextR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
843	pPrev->pNextRC = pNext ? pNext->pSelfRC : NIL_RTRCPTR;
844	}
845	else
846	{
847	Assert(pVM->pgm.s.pRamRangesXR3 == pRam);
848	pVM->pgm.s.pRamRangesXR3 = pNext;
849	pVM->pgm.s.pRamRangesXR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
850	pVM->pgm.s.pRamRangesXRC = pNext ? pNext->pSelfRC : NIL_RTRCPTR;
851	}
852	ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
853
854	pgmR3PhysRebuildRamRangeSearchTrees(pVM);
855	pgmUnlock(pVM);
856	}
857
858
859	/**
860	* Unlink an existing RAM range from the list.
861	*
862	* @param pVM Pointer to the VM.
863	* @param pRam Pointer to the new list entry.
864	*/
865	static void pgmR3PhysUnlinkRamRange(PVM pVM, PPGMRAMRANGE pRam)
866	{
867	pgmLock(pVM);
868
869	/* find prev. */
870	PPGMRAMRANGE pPrev = NULL;
871	PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesXR3;
872	while (pCur != pRam)
873	{
874	pPrev = pCur;
875	pCur = pCur->pNextR3;
876	}
877	AssertFatal(pCur);
878
879	pgmR3PhysUnlinkRamRange2(pVM, pRam, pPrev);
880	pgmUnlock(pVM);
881	}
882
883
884	/**
885	* Frees a range of pages, replacing them with ZERO pages of the specified type.
886	*
887	* @returns VBox status code.
888	* @param pVM Pointer to the VM.
889	* @param pRam The RAM range in which the pages resides.
890	* @param GCPhys The address of the first page.
891	* @param GCPhysLast The address of the last page.
892	* @param uType The page type to replace then with.
893	*/
894	static int pgmR3PhysFreePageRange(PVM pVM, PPGMRAMRANGE pRam, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast, uint8_t uType)
895	{
896	PGM_LOCK_ASSERT_OWNER(pVM);
897	uint32_t cPendingPages = 0;
898	PGMMFREEPAGESREQ pReq;
899	int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
900	AssertLogRelRCReturn(rc, rc);
901
902	/* Iterate the pages. */
903	PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
904	uint32_t cPagesLeft = ((GCPhysLast - GCPhys) >> PAGE_SHIFT) + 1;
905	while (cPagesLeft-- > 0)
906	{
907	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys);
908	AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
909
910	PGM_PAGE_SET_TYPE(pVM, pPageDst, uType);
911
912	GCPhys += PAGE_SIZE;
913	pPageDst++;
914	}
915
916	if (cPendingPages)
917	{
918	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
919	AssertLogRelRCReturn(rc, rc);
920	}
921	GMMR3FreePagesCleanup(pReq);
922
923	return rc;
924	}
925
926	#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) \|\| defined(RT_OS_SOLARIS) \|\| defined(RT_OS_LINUX) \|\| defined(RT_OS_FREEBSD))
927
928	/**
929	* Rendezvous callback used by PGMR3ChangeMemBalloon that changes the memory balloon size
930	*
931	* This is only called on one of the EMTs while the other ones are waiting for
932	* it to complete this function.
933	*
934	* @returns VINF_SUCCESS (VBox strict status code).
935	* @param pVM Pointer to the VM.
936	* @param pVCpu The VMCPU for the EMT we're being called on. Unused.
937	* @param pvUser User parameter
938	*/
939	static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysChangeMemBalloonRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
940	{
941	uintptr_t paUser = (uintptr_t )pvUser;
942	bool fInflate = !!paUser[0];
943	unsigned cPages = paUser[1];
944	RTGCPHYS paPhysPage = (RTGCPHYS )paUser[2];
945	uint32_t cPendingPages = 0;
946	PGMMFREEPAGESREQ pReq;
947	int rc;
948
949	Log(("pgmR3PhysChangeMemBalloonRendezvous: %s %x pages\n", (fInflate) ? "inflate" : "deflate", cPages));
950	pgmLock(pVM);
951
952	if (fInflate)
953	{
954	/* Flush the PGM pool cache as we might have stale references to pages that we just freed. */
955	pgmR3PoolClearAllRendezvous(pVM, pVCpu, NULL);
956
957	/* Replace pages with ZERO pages. */
958	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
959	if (RT_FAILURE(rc))
960	{
961	pgmUnlock(pVM);
962	AssertLogRelRC(rc);
963	return rc;
964	}
965
966	/* Iterate the pages. */
967	for (unsigned i = 0; i < cPages; i++)
968	{
969	PPGMPAGE pPage = pgmPhysGetPage(pVM, paPhysPage[i]);
970	if ( pPage == NULL
971	\|\| PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM)
972	{
973	Log(("pgmR3PhysChangeMemBalloonRendezvous: invalid physical page %RGp pPage->u3Type=%d\n", paPhysPage[i], pPage ? PGM_PAGE_GET_TYPE(pPage) : 0));
974	break;
975	}
976
977	LogFlow(("balloon page: %RGp\n", paPhysPage[i]));
978
979	/* Flush the shadow PT if this page was previously used as a guest page table. */
980	pgmPoolFlushPageByGCPhys(pVM, paPhysPage[i]);
981
982	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, paPhysPage[i]);
983	if (RT_FAILURE(rc))
984	{
985	pgmUnlock(pVM);
986	AssertLogRelRC(rc);
987	return rc;
988	}
989	Assert(PGM_PAGE_IS_ZERO(pPage));
990	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_BALLOONED);
991	}
992
993	if (cPendingPages)
994	{
995	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
996	if (RT_FAILURE(rc))
997	{
998	pgmUnlock(pVM);
999	AssertLogRelRC(rc);
1000	return rc;
1001	}
1002	}
1003	GMMR3FreePagesCleanup(pReq);
1004	}
1005	else
1006	{
1007	/* Iterate the pages. */
1008	for (unsigned i = 0; i < cPages; i++)
1009	{
1010	PPGMPAGE pPage = pgmPhysGetPage(pVM, paPhysPage[i]);
1011	AssertBreak(pPage && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM);
1012
1013	LogFlow(("Free ballooned page: %RGp\n", paPhysPage[i]));
1014
1015	Assert(PGM_PAGE_IS_BALLOONED(pPage));
1016
1017	/* Change back to zero page. */
1018	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
1019	}
1020
1021	/* Note that we currently do not map any ballooned pages in our shadow page tables, so no need to flush the pgm pool. */
1022	}
1023
1024	/* Notify GMM about the balloon change. */
1025	rc = GMMR3BalloonedPages(pVM, (fInflate) ? GMMBALLOONACTION_INFLATE : GMMBALLOONACTION_DEFLATE, cPages);
1026	if (RT_SUCCESS(rc))
1027	{
1028	if (!fInflate)
1029	{
1030	Assert(pVM->pgm.s.cBalloonedPages >= cPages);
1031	pVM->pgm.s.cBalloonedPages -= cPages;
1032	}
1033	else
1034	pVM->pgm.s.cBalloonedPages += cPages;
1035	}
1036
1037	pgmUnlock(pVM);
1038
1039	/* Flush the recompiler's TLB as well. */
1040	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1041	CPUMSetChangedFlags(&pVM->aCpus[i], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
1042
1043	AssertLogRelRC(rc);
1044	return rc;
1045	}
1046
1047
1048	/**
1049	* Frees a range of ram pages, replacing them with ZERO pages; helper for PGMR3PhysFreeRamPages
1050	*
1051	* @returns VBox status code.
1052	* @param pVM Pointer to the VM.
1053	* @param fInflate Inflate or deflate memory balloon
1054	* @param cPages Number of pages to free
1055	* @param paPhysPage Array of guest physical addresses
1056	*/
1057	static DECLCALLBACK(void) pgmR3PhysChangeMemBalloonHelper(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
1058	{
1059	uintptr_t paUser[3];
1060
1061	paUser[0] = fInflate;
1062	paUser[1] = cPages;
1063	paUser[2] = (uintptr_t)paPhysPage;
1064	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
1065	AssertRC(rc);
1066
1067	/* Made a copy in PGMR3PhysFreeRamPages; free it here. */
1068	RTMemFree(paPhysPage);
1069	}
1070
1071	#endif /* 64-bit host && (Windows \|\| Solaris \|\| Linux \|\| FreeBSD) */
1072
1073	/**
1074	* Inflate or deflate a memory balloon
1075	*
1076	* @returns VBox status code.
1077	* @param pVM Pointer to the VM.
1078	* @param fInflate Inflate or deflate memory balloon
1079	* @param cPages Number of pages to free
1080	* @param paPhysPage Array of guest physical addresses
1081	*/
1082	VMMR3DECL(int) PGMR3PhysChangeMemBalloon(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
1083	{
1084	/* This must match GMMR0Init; currently we only support memory ballooning on all 64-bit hosts except Mac OS X */
1085	#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) \|\| defined(RT_OS_SOLARIS) \|\| defined(RT_OS_LINUX) \|\| defined(RT_OS_FREEBSD))
1086	int rc;
1087
1088	/* Older additions (ancient non-functioning balloon code) pass wrong physical addresses. */
1089	AssertReturn(!(paPhysPage[0] & 0xfff), VERR_INVALID_PARAMETER);
1090
1091	/* We own the IOM lock here and could cause a deadlock by waiting for another VCPU that is blocking on the IOM lock.
1092	* In the SMP case we post a request packet to postpone the job.
1093	*/
1094	if (pVM->cCpus > 1)
1095	{
1096	unsigned cbPhysPage = cPages * sizeof(paPhysPage[0]);
1097	RTGCPHYS paPhysPageCopy = (RTGCPHYS )RTMemAlloc(cbPhysPage);
1098	AssertReturn(paPhysPageCopy, VERR_NO_MEMORY);
1099
1100	memcpy(paPhysPageCopy, paPhysPage, cbPhysPage);
1101
1102	rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysChangeMemBalloonHelper, 4, pVM, fInflate, cPages, paPhysPageCopy);
1103	AssertRC(rc);
1104	}
1105	else
1106	{
1107	uintptr_t paUser[3];
1108
1109	paUser[0] = fInflate;
1110	paUser[1] = cPages;
1111	paUser[2] = (uintptr_t)paPhysPage;
1112	rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
1113	AssertRC(rc);
1114	}
1115	return rc;
1116
1117	#else
1118	NOREF(pVM); NOREF(fInflate); NOREF(cPages); NOREF(paPhysPage);
1119	return VERR_NOT_IMPLEMENTED;
1120	#endif
1121	}
1122
1123
1124	/**
1125	* Rendezvous callback used by PGMR3WriteProtectRAM that write protects all
1126	* physical RAM.
1127	*
1128	* This is only called on one of the EMTs while the other ones are waiting for
1129	* it to complete this function.
1130	*
1131	* @returns VINF_SUCCESS (VBox strict status code).
1132	* @param pVM Pointer to the VM.
1133	* @param pVCpu The VMCPU for the EMT we're being called on. Unused.
1134	* @param pvUser User parameter, unused.
1135	*/
1136	static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysWriteProtectRAMRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
1137	{
1138	int rc = VINF_SUCCESS;
1139	NOREF(pvUser); NOREF(pVCpu);
1140
1141	pgmLock(pVM);
1142	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1143	pgmPoolResetDirtyPages(pVM);
1144	#endif
1145
1146	/** @todo pointless to write protect the physical page pointed to by RSP. */
1147
1148	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
1149	pRam;
1150	pRam = pRam->CTX_SUFF(pNext))
1151	{
1152	uint32_t cPages = pRam->cb >> PAGE_SHIFT;
1153	for (uint32_t iPage = 0; iPage < cPages; iPage++)
1154	{
1155	PPGMPAGE pPage = &pRam->aPages[iPage];
1156	PGMPAGETYPE enmPageType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
1157
1158	if ( RT_LIKELY(enmPageType == PGMPAGETYPE_RAM)
1159	\|\| enmPageType == PGMPAGETYPE_MMIO2)
1160	{
1161	/*
1162	* A RAM page.
1163	*/
1164	switch (PGM_PAGE_GET_STATE(pPage))
1165	{
1166	case PGM_PAGE_STATE_ALLOCATED:
1167	/** @todo Optimize this: Don't always re-enable write
1168	* monitoring if the page is known to be very busy. */
1169	if (PGM_PAGE_IS_WRITTEN_TO(pPage))
1170	{
1171	PGM_PAGE_CLEAR_WRITTEN_TO(pVM, pPage);
1172	/* Remember this dirty page for the next (memory) sync. */
1173	PGM_PAGE_SET_FT_DIRTY(pPage);
1174	}
1175
1176	pgmPhysPageWriteMonitor(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1177	break;
1178
1179	case PGM_PAGE_STATE_SHARED:
1180	AssertFailed();
1181	break;
1182
1183	case PGM_PAGE_STATE_WRITE_MONITORED: /* nothing to change. */
1184	default:
1185	break;
1186	}
1187	}
1188	}
1189	}
1190	pgmR3PoolWriteProtectPages(pVM);
1191	PGM_INVL_ALL_VCPU_TLBS(pVM);
1192	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1193	CPUMSetChangedFlags(&pVM->aCpus[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
1194
1195	pgmUnlock(pVM);
1196	return rc;
1197	}
1198
1199	/**
1200	* Protect all physical RAM to monitor writes
1201	*
1202	* @returns VBox status code.
1203	* @param pVM Pointer to the VM.
1204	*/
1205	VMMR3DECL(int) PGMR3PhysWriteProtectRAM(PVM pVM)
1206	{
1207	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
1208
1209	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysWriteProtectRAMRendezvous, NULL);
1210	AssertRC(rc);
1211	return rc;
1212	}
1213
1214	/**
1215	* Enumerate all dirty FT pages.
1216	*
1217	* @returns VBox status code.
1218	* @param pVM Pointer to the VM.
1219	* @param pfnEnum Enumerate callback handler.
1220	* @param pvUser Enumerate callback handler parameter.
1221	*/
1222	VMMR3DECL(int) PGMR3PhysEnumDirtyFTPages(PVM pVM, PFNPGMENUMDIRTYFTPAGES pfnEnum, void *pvUser)
1223	{
1224	int rc = VINF_SUCCESS;
1225
1226	pgmLock(pVM);
1227	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
1228	pRam;
1229	pRam = pRam->CTX_SUFF(pNext))
1230	{
1231	uint32_t cPages = pRam->cb >> PAGE_SHIFT;
1232	for (uint32_t iPage = 0; iPage < cPages; iPage++)
1233	{
1234	PPGMPAGE pPage = &pRam->aPages[iPage];
1235	PGMPAGETYPE enmPageType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
1236
1237	if ( RT_LIKELY(enmPageType == PGMPAGETYPE_RAM)
1238	\|\| enmPageType == PGMPAGETYPE_MMIO2)
1239	{
1240	/*
1241	* A RAM page.
1242	*/
1243	switch (PGM_PAGE_GET_STATE(pPage))
1244	{
1245	case PGM_PAGE_STATE_ALLOCATED:
1246	case PGM_PAGE_STATE_WRITE_MONITORED:
1247	if ( !PGM_PAGE_IS_WRITTEN_TO(pPage) /* not very recently updated? */
1248	&& PGM_PAGE_IS_FT_DIRTY(pPage))
1249	{
1250	unsigned cbPageRange = PAGE_SIZE;
1251	unsigned iPageClean = iPage + 1;
1252	RTGCPHYS GCPhysPage = pRam->GCPhys + iPage * PAGE_SIZE;
1253	uint8_t *pu8Page = NULL;
1254	PGMPAGEMAPLOCK Lock;
1255
1256	/* Find the next clean page, so we can merge adjacent dirty pages. */
1257	for (; iPageClean < cPages; iPageClean++)
1258	{
1259	PPGMPAGE pPageNext = &pRam->aPages[iPageClean];
1260	if ( RT_UNLIKELY(PGM_PAGE_GET_TYPE(pPageNext) != PGMPAGETYPE_RAM)
1261	\|\| PGM_PAGE_GET_STATE(pPageNext) != PGM_PAGE_STATE_ALLOCATED
1262	\|\| PGM_PAGE_IS_WRITTEN_TO(pPageNext)
1263	\|\| !PGM_PAGE_IS_FT_DIRTY(pPageNext)
1264	/* Crossing a chunk boundary? */
1265	\|\| (GCPhysPage & GMM_PAGEID_IDX_MASK) != ((GCPhysPage + cbPageRange) & GMM_PAGEID_IDX_MASK)
1266	)
1267	break;
1268
1269	cbPageRange += PAGE_SIZE;
1270	}
1271
1272	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysPage, (const void **)&pu8Page, &Lock);
1273	if (RT_SUCCESS(rc))
1274	{
1275	/** @todo this is risky; the range might be changed, but little choice as the sync
1276	* costs a lot of time. */
1277	pgmUnlock(pVM);
1278	pfnEnum(pVM, GCPhysPage, pu8Page, cbPageRange, pvUser);
1279	pgmLock(pVM);
1280	PGMPhysReleasePageMappingLock(pVM, &Lock);
1281	}
1282
1283	for (iPage; iPage < iPageClean; iPage++)
1284	PGM_PAGE_CLEAR_FT_DIRTY(&pRam->aPages[iPage]);
1285
1286	iPage = iPageClean - 1;
1287	}
1288	break;
1289	}
1290	}
1291	}
1292	}
1293	pgmUnlock(pVM);
1294	return rc;
1295	}
1296
1297
1298	/**
1299	* Gets the number of ram ranges.
1300	*
1301	* @returns Number of ram ranges. Returns UINT32_MAX if @a pVM is invalid.
1302	* @param pVM Pointer to the VM.
1303	*/
1304	VMMR3DECL(uint32_t) PGMR3PhysGetRamRangeCount(PVM pVM)
1305	{
1306	VM_ASSERT_VALID_EXT_RETURN(pVM, UINT32_MAX);
1307
1308	pgmLock(pVM);
1309	uint32_t cRamRanges = 0;
1310	for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRangesX); pCur; pCur = pCur->CTX_SUFF(pNext))
1311	cRamRanges++;
1312	pgmUnlock(pVM);
1313	return cRamRanges;
1314	}
1315
1316
1317	/**
1318	* Get information about a range.
1319	*
1320	* @returns VINF_SUCCESS or VERR_OUT_OF_RANGE.
1321	* @param pVM Pointer to the VM.
1322	* @param iRange The ordinal of the range.
1323	* @param pGCPhysStart Where to return the start of the range. Optional.
1324	* @param pGCPhysLast Where to return the address of the last byte in the
1325	* range. Optional.
1326	* @param pfIsMmio Where to indicate that this is a pure MMIO range.
1327	* Optional.
1328	*/
1329	VMMR3DECL(int) PGMR3PhysGetRange(PVM pVM, uint32_t iRange, PRTGCPHYS pGCPhysStart, PRTGCPHYS pGCPhysLast,
1330	const char *ppszDesc, bool pfIsMmio)
1331	{
1332	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
1333
1334	pgmLock(pVM);
1335	uint32_t iCurRange = 0;
1336	for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRangesX); pCur; pCur = pCur->CTX_SUFF(pNext), iCurRange++)
1337	if (iCurRange == iRange)
1338	{
1339	if (pGCPhysStart)
1340	*pGCPhysStart = pCur->GCPhys;
1341	if (pGCPhysLast)
1342	*pGCPhysLast = pCur->GCPhysLast;
1343	if (ppszDesc)
1344	*ppszDesc = pCur->pszDesc;
1345	if (pfIsMmio)
1346	*pfIsMmio = !!(pCur->fFlags & PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO);
1347
1348	pgmUnlock(pVM);
1349	return VINF_SUCCESS;
1350	}
1351	pgmUnlock(pVM);
1352	return VERR_OUT_OF_RANGE;
1353	}
1354
1355
1356	/**
1357	* Query the amount of free memory inside VMMR0
1358	*
1359	* @returns VBox status code.
1360	* @param pVM Pointer to the VM.
1361	* @param pcbAllocMem Where to return the amount of memory allocated
1362	* by VMs.
1363	* @param pcbFreeMem Where to return the amount of memory that is
1364	* allocated from the host but not currently used
1365	* by any VMs.
1366	* @param pcbBallonedMem Where to return the sum of memory that is
1367	* currently ballooned by the VMs.
1368	* @param pcbSharedMem Where to return the amount of memory that is
1369	* currently shared.
1370	*/
1371	VMMR3DECL(int) PGMR3QueryGlobalMemoryStats(PVM pVM, uint64_t pcbAllocMem, uint64_t pcbFreeMem,
1372	uint64_t pcbBallonedMem, uint64_t pcbSharedMem)
1373	{
1374	uint64_t cAllocPages = 0;
1375	uint64_t cFreePages = 0;
1376	uint64_t cBalloonPages = 0;
1377	uint64_t cSharedPages = 0;
1378	int rc = GMMR3QueryHypervisorMemoryStats(pVM, &cAllocPages, &cFreePages, &cBalloonPages, &cSharedPages);
1379	AssertRCReturn(rc, rc);
1380
1381	if (pcbAllocMem)
1382	pcbAllocMem = cAllocPages _4K;
1383
1384	if (pcbFreeMem)
1385	pcbFreeMem = cFreePages _4K;
1386
1387	if (pcbBallonedMem)
1388	pcbBallonedMem = cBalloonPages _4K;
1389
1390	if (pcbSharedMem)
1391	pcbSharedMem = cSharedPages _4K;
1392
1393	Log(("PGMR3QueryVMMMemoryStats: all=%llx free=%llx ballooned=%llx shared=%llx\n",
1394	cAllocPages, cFreePages, cBalloonPages, cSharedPages));
1395	return VINF_SUCCESS;
1396	}
1397
1398
1399	/**
1400	* Query memory stats for the VM.
1401	*
1402	* @returns VBox status code.
1403	* @param pVM Pointer to the VM.
1404	* @param pcbTotalMem Where to return total amount memory the VM may
1405	* possibly use.
1406	* @param pcbPrivateMem Where to return the amount of private memory
1407	* currently allocated.
1408	* @param pcbSharedMem Where to return the amount of actually shared
1409	* memory currently used by the VM.
1410	* @param pcbZeroMem Where to return the amount of memory backed by
1411	* zero pages.
1412	*
1413	* @remarks The total mem is normally larger than the sum of the three
1414	* components. There are two reasons for this, first the amount of
1415	* shared memory is what we're sure is shared instead of what could
1416	* possibly be shared with someone. Secondly, because the total may
1417	* include some pure MMIO pages that doesn't go into any of the three
1418	* sub-counts.
1419	*
1420	* @todo Why do we return reused shared pages instead of anything that could
1421	* potentially be shared? Doesn't this mean the first VM gets a much
1422	* lower number of shared pages?
1423	*/
1424	VMMR3DECL(int) PGMR3QueryMemoryStats(PVM pVM, uint64_t pcbTotalMem, uint64_t pcbPrivateMem,
1425	uint64_t pcbSharedMem, uint64_t pcbZeroMem)
1426	{
1427	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
1428
1429	if (pcbTotalMem)
1430	pcbTotalMem = (uint64_t)pVM->pgm.s.cAllPages PAGE_SIZE;
1431
1432	if (pcbPrivateMem)
1433	pcbPrivateMem = (uint64_t)pVM->pgm.s.cPrivatePages PAGE_SIZE;
1434
1435	if (pcbSharedMem)
1436	pcbSharedMem = (uint64_t)pVM->pgm.s.cReusedSharedPages PAGE_SIZE;
1437
1438	if (pcbZeroMem)
1439	pcbZeroMem = (uint64_t)pVM->pgm.s.cZeroPages PAGE_SIZE;
1440
1441	Log(("PGMR3QueryMemoryStats: all=%x private=%x reused=%x zero=%x\n", pVM->pgm.s.cAllPages, pVM->pgm.s.cPrivatePages, pVM->pgm.s.cReusedSharedPages, pVM->pgm.s.cZeroPages));
1442	return VINF_SUCCESS;
1443	}
1444
1445
1446	/**
1447	* PGMR3PhysRegisterRam worker that initializes and links a RAM range.
1448	*
1449	* @param pVM Pointer to the VM.
1450	* @param pNew The new RAM range.
1451	* @param GCPhys The address of the RAM range.
1452	* @param GCPhysLast The last address of the RAM range.
1453	* @param RCPtrNew The RC address if the range is floating. NIL_RTRCPTR
1454	* if in HMA.
1455	* @param R0PtrNew Ditto for R0.
1456	* @param pszDesc The description.
1457	* @param pPrev The previous RAM range (for linking).
1458	*/
1459	static void pgmR3PhysInitAndLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast,
1460	RTRCPTR RCPtrNew, RTR0PTR R0PtrNew, const char *pszDesc, PPGMRAMRANGE pPrev)
1461	{
1462	/*
1463	* Initialize the range.
1464	*/
1465	pNew->pSelfR0 = R0PtrNew != NIL_RTR0PTR ? R0PtrNew : MMHyperCCToR0(pVM, pNew);
1466	pNew->pSelfRC = RCPtrNew != NIL_RTRCPTR ? RCPtrNew : MMHyperCCToRC(pVM, pNew);
1467	pNew->GCPhys = GCPhys;
1468	pNew->GCPhysLast = GCPhysLast;
1469	pNew->cb = GCPhysLast - GCPhys + 1;
1470	pNew->pszDesc = pszDesc;
1471	pNew->fFlags = RCPtrNew != NIL_RTRCPTR ? PGM_RAM_RANGE_FLAGS_FLOATING : 0;
1472	pNew->pvR3 = NULL;
1473	pNew->paLSPages = NULL;
1474
1475	uint32_t const cPages = pNew->cb >> PAGE_SHIFT;
1476	RTGCPHYS iPage = cPages;
1477	while (iPage-- > 0)
1478	PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_RAM);
1479
1480	/* Update the page count stats. */
1481	pVM->pgm.s.cZeroPages += cPages;
1482	pVM->pgm.s.cAllPages += cPages;
1483
1484	/*
1485	* Link it.
1486	*/
1487	pgmR3PhysLinkRamRange(pVM, pNew, pPrev);
1488	}
1489
1490
1491	/**
1492	* Relocate a floating RAM range.
1493	*
1494	* @copydoc FNPGMRELOCATE.
1495	*/
1496	static DECLCALLBACK(bool) pgmR3PhysRamRangeRelocate(PVM pVM, RTGCPTR GCPtrOld, RTGCPTR GCPtrNew, PGMRELOCATECALL enmMode, void *pvUser)
1497	{
1498	PPGMRAMRANGE pRam = (PPGMRAMRANGE)pvUser;
1499	Assert(pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING);
1500	Assert(pRam->pSelfRC == GCPtrOld + PAGE_SIZE);
1501
1502	switch (enmMode)
1503	{
1504	case PGMRELOCATECALL_SUGGEST:
1505	return true;
1506
1507	case PGMRELOCATECALL_RELOCATE:
1508	{
1509	/*
1510	* Update myself, then relink all the ranges and flush the RC TLB.
1511	*/
1512	pgmLock(pVM);
1513
1514	pRam->pSelfRC = (RTRCPTR)(GCPtrNew + PAGE_SIZE);
1515
1516	pgmR3PhysRelinkRamRanges(pVM);
1517	for (unsigned i = 0; i < PGM_RAMRANGE_TLB_ENTRIES; i++)
1518	pVM->pgm.s.apRamRangesTlbRC[i] = NIL_RTRCPTR;
1519
1520	pgmUnlock(pVM);
1521	return true;
1522	}
1523
1524	default:
1525	AssertFailedReturn(false);
1526	}
1527	}
1528
1529
1530	/**
1531	* PGMR3PhysRegisterRam worker that registers a high chunk.
1532	*
1533	* @returns VBox status code.
1534	* @param pVM Pointer to the VM.
1535	* @param GCPhys The address of the RAM.
1536	* @param cRamPages The number of RAM pages to register.
1537	* @param cbChunk The size of the PGMRAMRANGE guest mapping.
1538	* @param iChunk The chunk number.
1539	* @param pszDesc The RAM range description.
1540	* @param ppPrev Previous RAM range pointer. In/Out.
1541	*/
1542	static int pgmR3PhysRegisterHighRamChunk(PVM pVM, RTGCPHYS GCPhys, uint32_t cRamPages,
1543	uint32_t cbChunk, uint32_t iChunk, const char *pszDesc,
1544	PPGMRAMRANGE *ppPrev)
1545	{
1546	const char *pszDescChunk = iChunk == 0
1547	? pszDesc
1548	: MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s (#%u)", pszDesc, iChunk + 1);
1549	AssertReturn(pszDescChunk, VERR_NO_MEMORY);
1550
1551	/*
1552	* Allocate memory for the new chunk.
1553	*/
1554	size_t const cChunkPages = RT_ALIGN_Z(RT_UOFFSETOF(PGMRAMRANGE, aPages[cRamPages]), PAGE_SIZE) >> PAGE_SHIFT;
1555	PSUPPAGE paChunkPages = (PSUPPAGE)RTMemTmpAllocZ(sizeof(SUPPAGE) * cChunkPages);
1556	AssertReturn(paChunkPages, VERR_NO_TMP_MEMORY);
1557	RTR0PTR R0PtrChunk = NIL_RTR0PTR;
1558	void *pvChunk = NULL;
1559	int rc = SUPR3PageAllocEx(cChunkPages, 0 /fFlags/, &pvChunk,
1560	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1561	VMMIsHwVirtExtForced(pVM) ? &R0PtrChunk : NULL,
1562	#else
1563	NULL,
1564	#endif
1565	paChunkPages);
1566	if (RT_SUCCESS(rc))
1567	{
1568	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1569	if (!VMMIsHwVirtExtForced(pVM))
1570	R0PtrChunk = NIL_RTR0PTR;
1571	#else
1572	R0PtrChunk = (uintptr_t)pvChunk;
1573	#endif
1574	memset(pvChunk, 0, cChunkPages << PAGE_SHIFT);
1575
1576	PPGMRAMRANGE pNew = (PPGMRAMRANGE)pvChunk;
1577
1578	/*
1579	* Create a mapping and map the pages into it.
1580	* We push these in below the HMA.
1581	*/
1582	RTGCPTR GCPtrChunkMap = pVM->pgm.s.GCPtrPrevRamRangeMapping - cbChunk;
1583	rc = PGMR3MapPT(pVM, GCPtrChunkMap, cbChunk, 0 /fFlags/, pgmR3PhysRamRangeRelocate, pNew, pszDescChunk);
1584	if (RT_SUCCESS(rc))
1585	{
1586	pVM->pgm.s.GCPtrPrevRamRangeMapping = GCPtrChunkMap;
1587
1588	RTGCPTR const GCPtrChunk = GCPtrChunkMap + PAGE_SIZE;
1589	RTGCPTR GCPtrPage = GCPtrChunk;
1590	for (uint32_t iPage = 0; iPage < cChunkPages && RT_SUCCESS(rc); iPage++, GCPtrPage += PAGE_SIZE)
1591	rc = PGMMap(pVM, GCPtrPage, paChunkPages[iPage].Phys, PAGE_SIZE, 0);
1592	if (RT_SUCCESS(rc))
1593	{
1594	/*
1595	* Ok, init and link the range.
1596	*/
1597	pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhys + ((RTGCPHYS)cRamPages << PAGE_SHIFT) - 1,
1598	(RTRCPTR)GCPtrChunk, R0PtrChunk, pszDescChunk, *ppPrev);
1599	*ppPrev = pNew;
1600	}
1601	}
1602
1603	if (RT_FAILURE(rc))
1604	SUPR3PageFreeEx(pvChunk, cChunkPages);
1605	}
1606
1607	RTMemTmpFree(paChunkPages);
1608	return rc;
1609	}
1610
1611
1612	/**
1613	* Sets up a range RAM.
1614	*
1615	* This will check for conflicting registrations, make a resource
1616	* reservation for the memory (with GMM), and setup the per-page
1617	* tracking structures (PGMPAGE).
1618	*
1619	* @returns VBox status code.
1620	* @param pVM Pointer to the VM.
1621	* @param GCPhys The physical address of the RAM.
1622	* @param cb The size of the RAM.
1623	* @param pszDesc The description - not copied, so, don't free or change it.
1624	*/
1625	VMMR3DECL(int) PGMR3PhysRegisterRam(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, const char *pszDesc)
1626	{
1627	/*
1628	* Validate input.
1629	*/
1630	Log(("PGMR3PhysRegisterRam: GCPhys=%RGp cb=%RGp pszDesc=%s\n", GCPhys, cb, pszDesc));
1631	AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
1632	AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
1633	AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
1634	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
1635	AssertMsgReturn(GCPhysLast > GCPhys, ("The range wraps! GCPhys=%RGp cb=%RGp\n", GCPhys, cb), VERR_INVALID_PARAMETER);
1636	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
1637	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
1638
1639	pgmLock(pVM);
1640
1641	/*
1642	* Find range location and check for conflicts.
1643	* (We don't lock here because the locking by EMT is only required on update.)
1644	*/
1645	PPGMRAMRANGE pPrev = NULL;
1646	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
1647	while (pRam && GCPhysLast >= pRam->GCPhys)
1648	{
1649	if ( GCPhysLast >= pRam->GCPhys
1650	&& GCPhys <= pRam->GCPhysLast)
1651	AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
1652	GCPhys, GCPhysLast, pszDesc,
1653	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
1654	VERR_PGM_RAM_CONFLICT);
1655
1656	/* next */
1657	pPrev = pRam;
1658	pRam = pRam->pNextR3;
1659	}
1660
1661	/*
1662	* Register it with GMM (the API bitches).
1663	*/
1664	const RTGCPHYS cPages = cb >> PAGE_SHIFT;
1665	int rc = MMR3IncreaseBaseReservation(pVM, cPages);
1666	if (RT_FAILURE(rc))
1667	{
1668	pgmUnlock(pVM);
1669	return rc;
1670	}
1671
1672	if ( GCPhys >= _4G
1673	&& cPages > 256)
1674	{
1675	/*
1676	* The PGMRAMRANGE structures for the high memory can get very big.
1677	* In order to avoid SUPR3PageAllocEx allocation failures due to the
1678	* allocation size limit there and also to avoid being unable to find
1679	* guest mapping space for them, we split this memory up into 4MB in
1680	* (potential) raw-mode configs and 16MB chunks in forced AMD-V/VT-x
1681	* mode.
1682	*
1683	* The first and last page of each mapping are guard pages and marked
1684	* not-present. So, we've got 4186112 and 16769024 bytes available for
1685	* the PGMRAMRANGE structure.
1686	*
1687	* Note! The sizes used here will influence the saved state.
1688	*/
1689	uint32_t cbChunk;
1690	uint32_t cPagesPerChunk;
1691	if (VMMIsHwVirtExtForced(pVM))
1692	{
1693	cbChunk = 16U*_1M;
1694	cPagesPerChunk = 1048048; /* max ~1048059 */
1695	AssertCompile(sizeof(PGMRAMRANGE) + sizeof(PGMPAGE) * 1048048 < 16U_1M - PAGE_SIZE 2);
1696	}
1697	else
1698	{
1699	cbChunk = 4U*_1M;
1700	cPagesPerChunk = 261616; /* max ~261627 */
1701	AssertCompile(sizeof(PGMRAMRANGE) + sizeof(PGMPAGE) * 261616 < 4U_1M - PAGE_SIZE 2);
1702	}
1703	AssertRelease(RT_UOFFSETOF(PGMRAMRANGE, aPages[cPagesPerChunk]) + PAGE_SIZE * 2 <= cbChunk);
1704
1705	RTGCPHYS cPagesLeft = cPages;
1706	RTGCPHYS GCPhysChunk = GCPhys;
1707	uint32_t iChunk = 0;
1708	while (cPagesLeft > 0)
1709	{
1710	uint32_t cPagesInChunk = cPagesLeft;
1711	if (cPagesInChunk > cPagesPerChunk)
1712	cPagesInChunk = cPagesPerChunk;
1713
1714	rc = pgmR3PhysRegisterHighRamChunk(pVM, GCPhysChunk, cPagesInChunk, cbChunk, iChunk, pszDesc, &pPrev);
1715	AssertRCReturn(rc, rc);
1716
1717	/* advance */
1718	GCPhysChunk += (RTGCPHYS)cPagesInChunk << PAGE_SHIFT;
1719	cPagesLeft -= cPagesInChunk;
1720	iChunk++;
1721	}
1722	}
1723	else
1724	{
1725	/*
1726	* Allocate, initialize and link the new RAM range.
1727	*/
1728	const size_t cbRamRange = RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]);
1729	PPGMRAMRANGE pNew;
1730	rc = MMR3HyperAllocOnceNoRel(pVM, cbRamRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
1731	AssertLogRelMsgRCReturn(rc, ("cbRamRange=%zu\n", cbRamRange), rc);
1732
1733	pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhysLast, NIL_RTRCPTR, NIL_RTR0PTR, pszDesc, pPrev);
1734	}
1735	pgmPhysInvalidatePageMapTLB(pVM);
1736	pgmUnlock(pVM);
1737
1738	#ifdef VBOX_WITH_REM
1739	/*
1740	* Notify REM.
1741	*/
1742	REMR3NotifyPhysRamRegister(pVM, GCPhys, cb, REM_NOTIFY_PHYS_RAM_FLAGS_RAM);
1743	#endif
1744
1745	return VINF_SUCCESS;
1746	}
1747
1748
1749	/**
1750	* Worker called by PGMR3InitFinalize if we're configured to pre-allocate RAM.
1751	*
1752	* We do this late in the init process so that all the ROM and MMIO ranges have
1753	* been registered already and we don't go wasting memory on them.
1754	*
1755	* @returns VBox status code.
1756	*
1757	* @param pVM Pointer to the VM.
1758	*/
1759	int pgmR3PhysRamPreAllocate(PVM pVM)
1760	{
1761	Assert(pVM->pgm.s.fRamPreAlloc);
1762	Log(("pgmR3PhysRamPreAllocate: enter\n"));
1763
1764	/*
1765	* Walk the RAM ranges and allocate all RAM pages, halt at
1766	* the first allocation error.
1767	*/
1768	uint64_t cPages = 0;
1769	uint64_t NanoTS = RTTimeNanoTS();
1770	pgmLock(pVM);
1771	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
1772	{
1773	PPGMPAGE pPage = &pRam->aPages[0];
1774	RTGCPHYS GCPhys = pRam->GCPhys;
1775	uint32_t cLeft = pRam->cb >> PAGE_SHIFT;
1776	while (cLeft-- > 0)
1777	{
1778	if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
1779	{
1780	switch (PGM_PAGE_GET_STATE(pPage))
1781	{
1782	case PGM_PAGE_STATE_ZERO:
1783	{
1784	int rc = pgmPhysAllocPage(pVM, pPage, GCPhys);
1785	if (RT_FAILURE(rc))
1786	{
1787	LogRel(("PGM: RAM Pre-allocation failed at %RGp (in %s) with rc=%Rrc\n", GCPhys, pRam->pszDesc, rc));
1788	pgmUnlock(pVM);
1789	return rc;
1790	}
1791	cPages++;
1792	break;
1793	}
1794
1795	case PGM_PAGE_STATE_BALLOONED:
1796	case PGM_PAGE_STATE_ALLOCATED:
1797	case PGM_PAGE_STATE_WRITE_MONITORED:
1798	case PGM_PAGE_STATE_SHARED:
1799	/* nothing to do here. */
1800	break;
1801	}
1802	}
1803
1804	/* next */
1805	pPage++;
1806	GCPhys += PAGE_SIZE;
1807	}
1808	}
1809	pgmUnlock(pVM);
1810	NanoTS = RTTimeNanoTS() - NanoTS;
1811
1812	LogRel(("PGM: Pre-allocated %llu pages in %llu ms\n", cPages, NanoTS / 1000000));
1813	Log(("pgmR3PhysRamPreAllocate: returns VINF_SUCCESS\n"));
1814	return VINF_SUCCESS;
1815	}
1816
1817
1818	/**
1819	* Resets (zeros) the RAM.
1820	*
1821	* ASSUMES that the caller owns the PGM lock.
1822	*
1823	* @returns VBox status code.
1824	* @param pVM Pointer to the VM.
1825	*/
1826	int pgmR3PhysRamReset(PVM pVM)
1827	{
1828	PGM_LOCK_ASSERT_OWNER(pVM);
1829
1830	/* Reset the memory balloon. */
1831	int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
1832	AssertRC(rc);
1833
1834	#ifdef VBOX_WITH_PAGE_SHARING
1835	/* Clear all registered shared modules. */
1836	rc = GMMR3ResetSharedModules(pVM);
1837	AssertRC(rc);
1838	#endif
1839	/* Reset counters. */
1840	pVM->pgm.s.cReusedSharedPages = 0;
1841	pVM->pgm.s.cBalloonedPages = 0;
1842
1843	/*
1844	* We batch up pages that should be freed instead of calling GMM for
1845	* each and every one of them.
1846	*/
1847	uint32_t cPendingPages = 0;
1848	PGMMFREEPAGESREQ pReq;
1849	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
1850	AssertLogRelRCReturn(rc, rc);
1851
1852	/*
1853	* Walk the ram ranges.
1854	*/
1855	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
1856	{
1857	uint32_t iPage = pRam->cb >> PAGE_SHIFT;
1858	AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
1859
1860	#ifndef NO_RAM_RESET
1861	if (!pVM->pgm.s.fRamPreAlloc)
1862	#else
1863	if (0)
1864	#endif
1865	{
1866	/* Replace all RAM pages by ZERO pages. */
1867	while (iPage-- > 0)
1868	{
1869	PPGMPAGE pPage = &pRam->aPages[iPage];
1870	switch (PGM_PAGE_GET_TYPE(pPage))
1871	{
1872	case PGMPAGETYPE_RAM:
1873	/* Do not replace pages part of a 2 MB continuous range
1874	with zero pages, but zero them instead. */
1875	if ( PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE
1876	\|\| PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED)
1877	{
1878	void *pvPage;
1879	rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
1880	AssertLogRelRCReturn(rc, rc);
1881	ASMMemZeroPage(pvPage);
1882	}
1883	else if (PGM_PAGE_IS_BALLOONED(pPage))
1884	{
1885	/* Turn into a zero page; the balloon status is lost when the VM reboots. */
1886	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
1887	}
1888	else if (!PGM_PAGE_IS_ZERO(pPage))
1889	{
1890	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1891	AssertLogRelRCReturn(rc, rc);
1892	}
1893	break;
1894
1895	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
1896	pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
1897	true /fDoAccounting/);
1898	break;
1899
1900	case PGMPAGETYPE_MMIO2:
1901	case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
1902	case PGMPAGETYPE_ROM:
1903	case PGMPAGETYPE_MMIO:
1904	break;
1905	default:
1906	AssertFailed();
1907	}
1908	} /* for each page */
1909	}
1910	else
1911	{
1912	/* Zero the memory. */
1913	while (iPage-- > 0)
1914	{
1915	PPGMPAGE pPage = &pRam->aPages[iPage];
1916	switch (PGM_PAGE_GET_TYPE(pPage))
1917	{
1918	case PGMPAGETYPE_RAM:
1919	switch (PGM_PAGE_GET_STATE(pPage))
1920	{
1921	case PGM_PAGE_STATE_ZERO:
1922	break;
1923
1924	case PGM_PAGE_STATE_BALLOONED:
1925	/* Turn into a zero page; the balloon status is lost when the VM reboots. */
1926	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
1927	break;
1928
1929	case PGM_PAGE_STATE_SHARED:
1930	case PGM_PAGE_STATE_WRITE_MONITORED:
1931	rc = pgmPhysPageMakeWritable(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1932	AssertLogRelRCReturn(rc, rc);
1933	/* no break */
1934
1935	case PGM_PAGE_STATE_ALLOCATED:
1936	{
1937	void *pvPage;
1938	rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
1939	AssertLogRelRCReturn(rc, rc);
1940	#ifndef NO_RAM_RESET
1941	ASMMemZeroPage(pvPage);
1942	#endif
1943	break;
1944	}
1945	}
1946	break;
1947
1948	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
1949	pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
1950	true /fDoAccounting/);
1951	break;
1952
1953	case PGMPAGETYPE_MMIO2:
1954	case PGMPAGETYPE_ROM_SHADOW:
1955	case PGMPAGETYPE_ROM:
1956	case PGMPAGETYPE_MMIO:
1957	break;
1958	default:
1959	AssertFailed();
1960
1961	}
1962	} /* for each page */
1963	}
1964
1965	}
1966
1967	/*
1968	* Finish off any pages pending freeing.
1969	*/
1970	if (cPendingPages)
1971	{
1972	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
1973	AssertLogRelRCReturn(rc, rc);
1974	}
1975	GMMR3FreePagesCleanup(pReq);
1976
1977	return VINF_SUCCESS;
1978	}
1979
1980	/**
1981	* Frees all RAM during VM termination
1982	*
1983	* ASSUMES that the caller owns the PGM lock.
1984	*
1985	* @returns VBox status code.
1986	* @param pVM Pointer to the VM.
1987	*/
1988	int pgmR3PhysRamTerm(PVM pVM)
1989	{
1990	PGM_LOCK_ASSERT_OWNER(pVM);
1991
1992	/* Reset the memory balloon. */
1993	int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
1994	AssertRC(rc);
1995
1996	#ifdef VBOX_WITH_PAGE_SHARING
1997	/* Clear all registered shared modules. */
1998	rc = GMMR3ResetSharedModules(pVM);
1999	AssertRC(rc);
2000	#endif
2001
2002	/*
2003	* We batch up pages that should be freed instead of calling GMM for
2004	* each and every one of them.
2005	*/
2006	uint32_t cPendingPages = 0;
2007	PGMMFREEPAGESREQ pReq;
2008	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
2009	AssertLogRelRCReturn(rc, rc);
2010
2011	/*
2012	* Walk the ram ranges.
2013	*/
2014	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
2015	{
2016	uint32_t iPage = pRam->cb >> PAGE_SHIFT;
2017	AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
2018
2019	/* Replace all RAM pages by ZERO pages. */
2020	while (iPage-- > 0)
2021	{
2022	PPGMPAGE pPage = &pRam->aPages[iPage];
2023	switch (PGM_PAGE_GET_TYPE(pPage))
2024	{
2025	case PGMPAGETYPE_RAM:
2026	/* Free all shared pages. Private pages are automatically freed during GMM VM cleanup. */
2027	if (PGM_PAGE_IS_SHARED(pPage))
2028	{
2029	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
2030	AssertLogRelRCReturn(rc, rc);
2031	}
2032	break;
2033
2034	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
2035	case PGMPAGETYPE_MMIO2:
2036	case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
2037	case PGMPAGETYPE_ROM:
2038	case PGMPAGETYPE_MMIO:
2039	break;
2040	default:
2041	AssertFailed();
2042	}
2043	} /* for each page */
2044	}
2045
2046	/*
2047	* Finish off any pages pending freeing.
2048	*/
2049	if (cPendingPages)
2050	{
2051	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2052	AssertLogRelRCReturn(rc, rc);
2053	}
2054	GMMR3FreePagesCleanup(pReq);
2055	return VINF_SUCCESS;
2056	}
2057
2058	/**
2059	* This is the interface IOM is using to register an MMIO region.
2060	*
2061	* It will check for conflicts and ensure that a RAM range structure
2062	* is present before calling the PGMR3HandlerPhysicalRegister API to
2063	* register the callbacks.
2064	*
2065	* @returns VBox status code.
2066	*
2067	* @param pVM Pointer to the VM.
2068	* @param GCPhys The start of the MMIO region.
2069	* @param cb The size of the MMIO region.
2070	* @param pfnHandlerR3 The address of the ring-3 handler. (IOMR3MMIOHandler)
2071	* @param pvUserR3 The user argument for R3.
2072	* @param pfnHandlerR0 The address of the ring-0 handler. (IOMMMIOHandler)
2073	* @param pvUserR0 The user argument for R0.
2074	* @param pfnHandlerRC The address of the RC handler. (IOMMMIOHandler)
2075	* @param pvUserRC The user argument for RC.
2076	* @param pszDesc The description of the MMIO region.
2077	*/
2078	VMMR3DECL(int) PGMR3PhysMMIORegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb,
2079	R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnHandlerR3, RTR3PTR pvUserR3,
2080	R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnHandlerR0, RTR0PTR pvUserR0,
2081	RCPTRTYPE(PFNPGMRCPHYSHANDLER) pfnHandlerRC, RTRCPTR pvUserRC,
2082	R3PTRTYPE(const char *) pszDesc)
2083	{
2084	/*
2085	* Assert on some assumption.
2086	*/
2087	VM_ASSERT_EMT(pVM);
2088	AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2089	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2090	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
2091	AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
2092
2093	int rc = pgmLock(pVM);
2094	AssertRCReturn(rc, rc);
2095
2096	/*
2097	* Make sure there's a RAM range structure for the region.
2098	*/
2099	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
2100	bool fRamExists = false;
2101	PPGMRAMRANGE pRamPrev = NULL;
2102	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2103	while (pRam && GCPhysLast >= pRam->GCPhys)
2104	{
2105	if ( GCPhysLast >= pRam->GCPhys
2106	&& GCPhys <= pRam->GCPhysLast)
2107	{
2108	/* Simplification: all within the same range. */
2109	AssertLogRelMsgReturnStmt( GCPhys >= pRam->GCPhys
2110	&& GCPhysLast <= pRam->GCPhysLast,
2111	("%RGp-%RGp (MMIO/%s) falls partly outside %RGp-%RGp (%s)\n",
2112	GCPhys, GCPhysLast, pszDesc,
2113	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
2114	pgmUnlock(pVM),
2115	VERR_PGM_RAM_CONFLICT);
2116
2117	/* Check that it's all RAM or MMIO pages. */
2118	PCPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2119	uint32_t cLeft = cb >> PAGE_SHIFT;
2120	while (cLeft-- > 0)
2121	{
2122	AssertLogRelMsgReturnStmt( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2123	\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO,
2124	("%RGp-%RGp (MMIO/%s): %RGp is not a RAM or MMIO page - type=%d desc=%s\n",
2125	GCPhys, GCPhysLast, pszDesc, PGM_PAGE_GET_TYPE(pPage), pRam->pszDesc),
2126	pgmUnlock(pVM),
2127	VERR_PGM_RAM_CONFLICT);
2128	pPage++;
2129	}
2130
2131	/* Looks good. */
2132	fRamExists = true;
2133	break;
2134	}
2135
2136	/* next */
2137	pRamPrev = pRam;
2138	pRam = pRam->pNextR3;
2139	}
2140	PPGMRAMRANGE pNew;
2141	if (fRamExists)
2142	{
2143	pNew = NULL;
2144
2145	/*
2146	* Make all the pages in the range MMIO/ZERO pages, freeing any
2147	* RAM pages currently mapped here. This might not be 100% correct
2148	* for PCI memory, but we're doing the same thing for MMIO2 pages.
2149	*/
2150	rc = pgmR3PhysFreePageRange(pVM, pRam, GCPhys, GCPhysLast, PGMPAGETYPE_MMIO);
2151	AssertRCReturnStmt(rc, pgmUnlock(pVM), rc);
2152
2153	/* Force a PGM pool flush as guest ram references have been changed. */
2154	/** @todo not entirely SMP safe; assuming for now the guest takes
2155	* care of this internally (not touch mapped mmio while changing the
2156	* mapping). */
2157	PVMCPU pVCpu = VMMGetCpu(pVM);
2158	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2159	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2160	}
2161	else
2162	{
2163
2164	/*
2165	* No RAM range, insert an ad hoc one.
2166	*
2167	* Note that we don't have to tell REM about this range because
2168	* PGMHandlerPhysicalRegisterEx will do that for us.
2169	*/
2170	Log(("PGMR3PhysMMIORegister: Adding ad hoc MMIO range for %RGp-%RGp %s\n", GCPhys, GCPhysLast, pszDesc));
2171
2172	const uint32_t cPages = cb >> PAGE_SHIFT;
2173	const size_t cbRamRange = RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]);
2174	rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]), 16, MM_TAG_PGM_PHYS, (void **)&pNew);
2175	AssertLogRelMsgRCReturnStmt(rc, ("cbRamRange=%zu\n", cbRamRange), pgmUnlock(pVM), rc);
2176
2177	/* Initialize the range. */
2178	pNew->pSelfR0 = MMHyperCCToR0(pVM, pNew);
2179	pNew->pSelfRC = MMHyperCCToRC(pVM, pNew);
2180	pNew->GCPhys = GCPhys;
2181	pNew->GCPhysLast = GCPhysLast;
2182	pNew->cb = cb;
2183	pNew->pszDesc = pszDesc;
2184	pNew->fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO;
2185	pNew->pvR3 = NULL;
2186	pNew->paLSPages = NULL;
2187
2188	uint32_t iPage = cPages;
2189	while (iPage-- > 0)
2190	PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_MMIO);
2191	Assert(PGM_PAGE_GET_TYPE(&pNew->aPages[0]) == PGMPAGETYPE_MMIO);
2192
2193	/* update the page count stats. */
2194	pVM->pgm.s.cPureMmioPages += cPages;
2195	pVM->pgm.s.cAllPages += cPages;
2196
2197	/* link it */
2198	pgmR3PhysLinkRamRange(pVM, pNew, pRamPrev);
2199	}
2200
2201	/*
2202	* Register the access handler.
2203	*/
2204	rc = PGMHandlerPhysicalRegisterEx(pVM, PGMPHYSHANDLERTYPE_MMIO, GCPhys, GCPhysLast,
2205	pfnHandlerR3, pvUserR3,
2206	pfnHandlerR0, pvUserR0,
2207	pfnHandlerRC, pvUserRC, pszDesc);
2208	if ( RT_FAILURE(rc)
2209	&& !fRamExists)
2210	{
2211	pVM->pgm.s.cPureMmioPages -= cb >> PAGE_SHIFT;
2212	pVM->pgm.s.cAllPages -= cb >> PAGE_SHIFT;
2213
2214	/* remove the ad hoc range. */
2215	pgmR3PhysUnlinkRamRange2(pVM, pNew, pRamPrev);
2216	pNew->cb = pNew->GCPhys = pNew->GCPhysLast = NIL_RTGCPHYS;
2217	MMHyperFree(pVM, pRam);
2218	}
2219	pgmPhysInvalidatePageMapTLB(pVM);
2220
2221	pgmUnlock(pVM);
2222	return rc;
2223	}
2224
2225
2226	/**
2227	* This is the interface IOM is using to register an MMIO region.
2228	*
2229	* It will take care of calling PGMHandlerPhysicalDeregister and clean up
2230	* any ad hoc PGMRAMRANGE left behind.
2231	*
2232	* @returns VBox status code.
2233	* @param pVM Pointer to the VM.
2234	* @param GCPhys The start of the MMIO region.
2235	* @param cb The size of the MMIO region.
2236	*/
2237	VMMR3DECL(int) PGMR3PhysMMIODeregister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb)
2238	{
2239	VM_ASSERT_EMT(pVM);
2240
2241	int rc = pgmLock(pVM);
2242	AssertRCReturn(rc, rc);
2243
2244	/*
2245	* First deregister the handler, then check if we should remove the ram range.
2246	*/
2247	rc = PGMHandlerPhysicalDeregister(pVM, GCPhys);
2248	if (RT_SUCCESS(rc))
2249	{
2250	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
2251	PPGMRAMRANGE pRamPrev = NULL;
2252	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2253	while (pRam && GCPhysLast >= pRam->GCPhys)
2254	{
2255	/** @todo We're being a bit too careful here. rewrite. */
2256	if ( GCPhysLast == pRam->GCPhysLast
2257	&& GCPhys == pRam->GCPhys)
2258	{
2259	Assert(pRam->cb == cb);
2260
2261	/*
2262	* See if all the pages are dead MMIO pages.
2263	*/
2264	uint32_t const cPages = cb >> PAGE_SHIFT;
2265	bool fAllMMIO = true;
2266	uint32_t iPage = 0;
2267	uint32_t cLeft = cPages;
2268	while (cLeft-- > 0)
2269	{
2270	PPGMPAGE pPage = &pRam->aPages[iPage];
2271	if ( PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO
2272	/\|\| not-out-of-action later /)
2273	{
2274	fAllMMIO = false;
2275	Assert(PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO2_ALIAS_MMIO);
2276	AssertMsgFailed(("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
2277	break;
2278	}
2279	Assert(PGM_PAGE_IS_ZERO(pPage));
2280	pPage++;
2281	}
2282	if (fAllMMIO)
2283	{
2284	/*
2285	* Ad-hoc range, unlink and free it.
2286	*/
2287	Log(("PGMR3PhysMMIODeregister: Freeing ad hoc MMIO range for %RGp-%RGp %s\n",
2288	GCPhys, GCPhysLast, pRam->pszDesc));
2289
2290	pVM->pgm.s.cAllPages -= cPages;
2291	pVM->pgm.s.cPureMmioPages -= cPages;
2292
2293	pgmR3PhysUnlinkRamRange2(pVM, pRam, pRamPrev);
2294	pRam->cb = pRam->GCPhys = pRam->GCPhysLast = NIL_RTGCPHYS;
2295	MMHyperFree(pVM, pRam);
2296	break;
2297	}
2298	}
2299
2300	/*
2301	* Range match? It will all be within one range (see PGMAllHandler.cpp).
2302	*/
2303	if ( GCPhysLast >= pRam->GCPhys
2304	&& GCPhys <= pRam->GCPhysLast)
2305	{
2306	Assert(GCPhys >= pRam->GCPhys);
2307	Assert(GCPhysLast <= pRam->GCPhysLast);
2308
2309	/*
2310	* Turn the pages back into RAM pages.
2311	*/
2312	uint32_t iPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2313	uint32_t cLeft = cb >> PAGE_SHIFT;
2314	while (cLeft--)
2315	{
2316	PPGMPAGE pPage = &pRam->aPages[iPage];
2317	AssertMsg(PGM_PAGE_IS_MMIO(pPage), ("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
2318	AssertMsg(PGM_PAGE_IS_ZERO(pPage), ("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
2319	if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO)
2320	PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_RAM);
2321	}
2322	break;
2323	}
2324
2325	/* next */
2326	pRamPrev = pRam;
2327	pRam = pRam->pNextR3;
2328	}
2329	}
2330
2331	/* Force a PGM pool flush as guest ram references have been changed. */
2332	/** @todo Not entirely SMP safe; assuming for now the guest takes care of
2333	* this internally (not touch mapped mmio while changing the mapping). */
2334	PVMCPU pVCpu = VMMGetCpu(pVM);
2335	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2336	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2337
2338	pgmPhysInvalidatePageMapTLB(pVM);
2339	pgmPhysInvalidRamRangeTlbs(pVM);
2340	pgmUnlock(pVM);
2341	return rc;
2342	}
2343
2344
2345	/**
2346	* Locate a MMIO2 range.
2347	*
2348	* @returns Pointer to the MMIO2 range.
2349	* @param pVM Pointer to the VM.
2350	* @param pDevIns The device instance owning the region.
2351	* @param iRegion The region.
2352	*/
2353	DECLINLINE(PPGMMMIO2RANGE) pgmR3PhysMMIO2Find(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion)
2354	{
2355	/*
2356	* Search the list.
2357	*/
2358	for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
2359	if ( pCur->pDevInsR3 == pDevIns
2360	&& pCur->iRegion == iRegion)
2361	return pCur;
2362	return NULL;
2363	}
2364
2365
2366	/**
2367	* Allocate and register an MMIO2 region.
2368	*
2369	* As mentioned elsewhere, MMIO2 is just RAM spelled differently. It's RAM
2370	* associated with a device. It is also non-shared memory with a permanent
2371	* ring-3 mapping and page backing (presently).
2372	*
2373	* A MMIO2 range may overlap with base memory if a lot of RAM is configured for
2374	* the VM, in which case we'll drop the base memory pages. Presently we will
2375	* make no attempt to preserve anything that happens to be present in the base
2376	* memory that is replaced, this is of course incorrectly but it's too much
2377	* effort.
2378	*
2379	* @returns VBox status code.
2380	* @retval VINF_SUCCESS on success, *ppv pointing to the R3 mapping of the
2381	* memory.
2382	* @retval VERR_ALREADY_EXISTS if the region already exists.
2383	*
2384	* @param pVM Pointer to the VM.
2385	* @param pDevIns The device instance owning the region.
2386	* @param iRegion The region number. If the MMIO2 memory is a PCI
2387	* I/O region this number has to be the number of that
2388	* region. Otherwise it can be any number safe
2389	* UINT8_MAX.
2390	* @param cb The size of the region. Must be page aligned.
2391	* @param fFlags Reserved for future use, must be zero.
2392	* @param ppv Where to store the pointer to the ring-3 mapping of
2393	* the memory.
2394	* @param pszDesc The description.
2395	*/
2396	VMMR3DECL(int) PGMR3PhysMMIO2Register(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS cb, uint32_t fFlags, void *ppv, const char pszDesc)
2397	{
2398	/*
2399	* Validate input.
2400	*/
2401	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2402	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2403	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2404	AssertPtrReturn(ppv, VERR_INVALID_POINTER);
2405	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
2406	AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
2407	AssertReturn(pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion) == NULL, VERR_ALREADY_EXISTS);
2408	AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2409	AssertReturn(cb, VERR_INVALID_PARAMETER);
2410	AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
2411
2412	const uint32_t cPages = cb >> PAGE_SHIFT;
2413	AssertLogRelReturn(((RTGCPHYS)cPages << PAGE_SHIFT) == cb, VERR_INVALID_PARAMETER);
2414	AssertLogRelReturn(cPages <= INT32_MAX / 2, VERR_NO_MEMORY);
2415
2416	/*
2417	* For the 2nd+ instance, mangle the description string so it's unique.
2418	*/
2419	if (pDevIns->iInstance > 0) /** @todo Move to PDMDevHlp.cpp and use a real string cache. */
2420	{
2421	pszDesc = MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s [%u]", pszDesc, pDevIns->iInstance);
2422	if (!pszDesc)
2423	return VERR_NO_MEMORY;
2424	}
2425
2426	/*
2427	* Try reserve and allocate the backing memory first as this is what is
2428	* most likely to fail.
2429	*/
2430	int rc = MMR3AdjustFixedReservation(pVM, cPages, pszDesc);
2431	if (RT_SUCCESS(rc))
2432	{
2433	void *pvPages;
2434	PSUPPAGE paPages = (PSUPPAGE)RTMemTmpAlloc(cPages * sizeof(SUPPAGE));
2435	if (RT_SUCCESS(rc))
2436	rc = SUPR3PageAllocEx(cPages, 0 /fFlags/, &pvPages, NULL /pR0Ptr/, paPages);
2437	if (RT_SUCCESS(rc))
2438	{
2439	memset(pvPages, 0, cPages * PAGE_SIZE);
2440
2441	/*
2442	* Create the MMIO2 range record for it.
2443	*/
2444	const size_t cbRange = RT_OFFSETOF(PGMMMIO2RANGE, RamRange.aPages[cPages]);
2445	PPGMMMIO2RANGE pNew;
2446	rc = MMR3HyperAllocOnceNoRel(pVM, cbRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
2447	AssertLogRelMsgRC(rc, ("cbRamRange=%zu\n", cbRange));
2448	if (RT_SUCCESS(rc))
2449	{
2450	pNew->pDevInsR3 = pDevIns;
2451	pNew->pvR3 = pvPages;
2452	//pNew->pNext = NULL;
2453	//pNew->fMapped = false;
2454	//pNew->fOverlapping = false;
2455	pNew->iRegion = iRegion;
2456	pNew->idSavedState = UINT8_MAX;
2457	pNew->RamRange.pSelfR0 = MMHyperCCToR0(pVM, &pNew->RamRange);
2458	pNew->RamRange.pSelfRC = MMHyperCCToRC(pVM, &pNew->RamRange);
2459	pNew->RamRange.GCPhys = NIL_RTGCPHYS;
2460	pNew->RamRange.GCPhysLast = NIL_RTGCPHYS;
2461	pNew->RamRange.pszDesc = pszDesc;
2462	pNew->RamRange.cb = cb;
2463	pNew->RamRange.fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO2;
2464	pNew->RamRange.pvR3 = pvPages;
2465	//pNew->RamRange.paLSPages = NULL;
2466
2467	uint32_t iPage = cPages;
2468	while (iPage-- > 0)
2469	{
2470	PGM_PAGE_INIT(&pNew->RamRange.aPages[iPage],
2471	paPages[iPage].Phys, NIL_GMM_PAGEID,
2472	PGMPAGETYPE_MMIO2, PGM_PAGE_STATE_ALLOCATED);
2473	}
2474
2475	/* update page count stats */
2476	pVM->pgm.s.cAllPages += cPages;
2477	pVM->pgm.s.cPrivatePages += cPages;
2478
2479	/*
2480	* Link it into the list.
2481	* Since there is no particular order, just push it.
2482	*/
2483	pgmLock(pVM);
2484	pNew->pNextR3 = pVM->pgm.s.pMmio2RangesR3;
2485	pVM->pgm.s.pMmio2RangesR3 = pNew;
2486	pgmUnlock(pVM);
2487
2488	*ppv = pvPages;
2489	RTMemTmpFree(paPages);
2490	pgmPhysInvalidatePageMapTLB(pVM);
2491	return VINF_SUCCESS;
2492	}
2493
2494	SUPR3PageFreeEx(pvPages, cPages);
2495	}
2496	RTMemTmpFree(paPages);
2497	MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pszDesc);
2498	}
2499	if (pDevIns->iInstance > 0)
2500	MMR3HeapFree((void *)pszDesc);
2501	return rc;
2502	}
2503
2504
2505	/**
2506	* Deregisters and frees an MMIO2 region.
2507	*
2508	* Any physical (and virtual) access handlers registered for the region must
2509	* be deregistered before calling this function.
2510	*
2511	* @returns VBox status code.
2512	* @param pVM Pointer to the VM.
2513	* @param pDevIns The device instance owning the region.
2514	* @param iRegion The region. If it's UINT32_MAX it'll be a wildcard match.
2515	*/
2516	VMMR3DECL(int) PGMR3PhysMMIO2Deregister(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion)
2517	{
2518	/*
2519	* Validate input.
2520	*/
2521	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2522	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2523	AssertReturn(iRegion <= UINT8_MAX \|\| iRegion == UINT32_MAX, VERR_INVALID_PARAMETER);
2524
2525	pgmLock(pVM);
2526	int rc = VINF_SUCCESS;
2527	unsigned cFound = 0;
2528	PPGMMMIO2RANGE pPrev = NULL;
2529	PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3;
2530	while (pCur)
2531	{
2532	if ( pCur->pDevInsR3 == pDevIns
2533	&& ( iRegion == UINT32_MAX
2534	\|\| pCur->iRegion == iRegion))
2535	{
2536	cFound++;
2537
2538	/*
2539	* Unmap it if it's mapped.
2540	*/
2541	if (pCur->fMapped)
2542	{
2543	int rc2 = PGMR3PhysMMIO2Unmap(pVM, pCur->pDevInsR3, pCur->iRegion, pCur->RamRange.GCPhys);
2544	AssertRC(rc2);
2545	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2546	rc = rc2;
2547	}
2548
2549	/*
2550	* Unlink it
2551	*/
2552	PPGMMMIO2RANGE pNext = pCur->pNextR3;
2553	if (pPrev)
2554	pPrev->pNextR3 = pNext;
2555	else
2556	pVM->pgm.s.pMmio2RangesR3 = pNext;
2557	pCur->pNextR3 = NULL;
2558
2559	/*
2560	* Free the memory.
2561	*/
2562	int rc2 = SUPR3PageFreeEx(pCur->pvR3, pCur->RamRange.cb >> PAGE_SHIFT);
2563	AssertRC(rc2);
2564	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2565	rc = rc2;
2566
2567	uint32_t const cPages = pCur->RamRange.cb >> PAGE_SHIFT;
2568	rc2 = MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pCur->RamRange.pszDesc);
2569	AssertRC(rc2);
2570	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2571	rc = rc2;
2572
2573	/* we're leaking hyper memory here if done at runtime. */
2574	#ifdef VBOX_STRICT
2575	VMSTATE const enmState = VMR3GetState(pVM);
2576	AssertMsg( enmState == VMSTATE_POWERING_OFF
2577	\|\| enmState == VMSTATE_POWERING_OFF_LS
2578	\|\| enmState == VMSTATE_OFF
2579	\|\| enmState == VMSTATE_OFF_LS
2580	\|\| enmState == VMSTATE_DESTROYING
2581	\|\| enmState == VMSTATE_TERMINATED
2582	\|\| enmState == VMSTATE_CREATING
2583	, ("%s\n", VMR3GetStateName(enmState)));
2584	#endif
2585	/*rc = MMHyperFree(pVM, pCur);
2586	AssertRCReturn(rc, rc); - not safe, see the alloc call. */
2587
2588
2589	/* update page count stats */
2590	pVM->pgm.s.cAllPages -= cPages;
2591	pVM->pgm.s.cPrivatePages -= cPages;
2592
2593	/* next */
2594	pCur = pNext;
2595	}
2596	else
2597	{
2598	pPrev = pCur;
2599	pCur = pCur->pNextR3;
2600	}
2601	}
2602	pgmPhysInvalidatePageMapTLB(pVM);
2603	pgmUnlock(pVM);
2604	return !cFound && iRegion != UINT32_MAX ? VERR_NOT_FOUND : rc;
2605	}
2606
2607
2608	/**
2609	* Maps a MMIO2 region.
2610	*
2611	* This is done when a guest / the bios / state loading changes the
2612	* PCI config. The replacing of base memory has the same restrictions
2613	* as during registration, of course.
2614	*
2615	* @returns VBox status code.
2616	*
2617	* @param pVM Pointer to the VM.
2618	* @param pDevIns The device instance owning the region.
2619	*/
2620	VMMR3DECL(int) PGMR3PhysMMIO2Map(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
2621	{
2622	/*
2623	* Validate input
2624	*/
2625	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2626	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2627	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2628	AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
2629	AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
2630	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2631
2632	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2633	AssertReturn(pCur, VERR_NOT_FOUND);
2634	AssertReturn(!pCur->fMapped, VERR_WRONG_ORDER);
2635	Assert(pCur->RamRange.GCPhys == NIL_RTGCPHYS);
2636	Assert(pCur->RamRange.GCPhysLast == NIL_RTGCPHYS);
2637
2638	const RTGCPHYS GCPhysLast = GCPhys + pCur->RamRange.cb - 1;
2639	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
2640
2641	/*
2642	* Find our location in the ram range list, checking for
2643	* restriction we don't bother implementing yet (partially overlapping).
2644	*/
2645	bool fRamExists = false;
2646	PPGMRAMRANGE pRamPrev = NULL;
2647	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2648	while (pRam && GCPhysLast >= pRam->GCPhys)
2649	{
2650	if ( GCPhys <= pRam->GCPhysLast
2651	&& GCPhysLast >= pRam->GCPhys)
2652	{
2653	/* completely within? */
2654	AssertLogRelMsgReturn( GCPhys >= pRam->GCPhys
2655	&& GCPhysLast <= pRam->GCPhysLast,
2656	("%RGp-%RGp (MMIO2/%s) falls partly outside %RGp-%RGp (%s)\n",
2657	GCPhys, GCPhysLast, pCur->RamRange.pszDesc,
2658	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
2659	VERR_PGM_RAM_CONFLICT);
2660	fRamExists = true;
2661	break;
2662	}
2663
2664	/* next */
2665	pRamPrev = pRam;
2666	pRam = pRam->pNextR3;
2667	}
2668	if (fRamExists)
2669	{
2670	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2671	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2672	while (cPagesLeft-- > 0)
2673	{
2674	AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
2675	("%RGp isn't a RAM page (%d) - mapping %RGp-%RGp (MMIO2/%s).\n",
2676	GCPhys, PGM_PAGE_GET_TYPE(pPage), GCPhys, GCPhysLast, pCur->RamRange.pszDesc),
2677	VERR_PGM_RAM_CONFLICT);
2678	pPage++;
2679	}
2680	}
2681	Log(("PGMR3PhysMMIO2Map: %RGp-%RGp fRamExists=%RTbool %s\n",
2682	GCPhys, GCPhysLast, fRamExists, pCur->RamRange.pszDesc));
2683
2684	/*
2685	* Make the changes.
2686	*/
2687	pgmLock(pVM);
2688
2689	pCur->RamRange.GCPhys = GCPhys;
2690	pCur->RamRange.GCPhysLast = GCPhysLast;
2691	pCur->fMapped = true;
2692	pCur->fOverlapping = fRamExists;
2693
2694	if (fRamExists)
2695	{
2696	/** @todo use pgmR3PhysFreePageRange here. */
2697	uint32_t cPendingPages = 0;
2698	PGMMFREEPAGESREQ pReq;
2699	int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
2700	AssertLogRelRCReturn(rc, rc);
2701
2702	/* replace the pages, freeing all present RAM pages. */
2703	PPGMPAGE pPageSrc = &pCur->RamRange.aPages[0];
2704	PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2705	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2706	while (cPagesLeft-- > 0)
2707	{
2708	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys);
2709	AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
2710
2711	RTHCPHYS const HCPhys = PGM_PAGE_GET_HCPHYS(pPageSrc);
2712	PGM_PAGE_SET_HCPHYS(pVM, pPageDst, HCPhys);
2713	PGM_PAGE_SET_TYPE(pVM, pPageDst, PGMPAGETYPE_MMIO2);
2714	PGM_PAGE_SET_STATE(pVM, pPageDst, PGM_PAGE_STATE_ALLOCATED);
2715	PGM_PAGE_SET_PDE_TYPE(pVM, pPageDst, PGM_PAGE_PDE_TYPE_DONTCARE);
2716	PGM_PAGE_SET_PTE_INDEX(pVM, pPageDst, 0);
2717	PGM_PAGE_SET_TRACKING(pVM, pPageDst, 0);
2718
2719	pVM->pgm.s.cZeroPages--;
2720	GCPhys += PAGE_SIZE;
2721	pPageSrc++;
2722	pPageDst++;
2723	}
2724
2725	/* Flush physical page map TLB. */
2726	pgmPhysInvalidatePageMapTLB(pVM);
2727
2728	if (cPendingPages)
2729	{
2730	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2731	AssertLogRelRCReturn(rc, rc);
2732	}
2733	GMMR3FreePagesCleanup(pReq);
2734
2735	/* Force a PGM pool flush as guest ram references have been changed. */
2736	/** @todo not entirely SMP safe; assuming for now the guest takes care of
2737	* this internally (not touch mapped mmio while changing the mapping). */
2738	PVMCPU pVCpu = VMMGetCpu(pVM);
2739	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2740	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2741
2742	pgmUnlock(pVM);
2743	}
2744	else
2745	{
2746	RTGCPHYS cb = pCur->RamRange.cb;
2747
2748	/* Clear the tracking data of pages we're going to reactivate. */
2749	PPGMPAGE pPageSrc = &pCur->RamRange.aPages[0];
2750	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2751	while (cPagesLeft-- > 0)
2752	{
2753	PGM_PAGE_SET_TRACKING(pVM, pPageSrc, 0);
2754	PGM_PAGE_SET_PTE_INDEX(pVM, pPageSrc, 0);
2755	pPageSrc++;
2756	}
2757
2758	/* link in the ram range */
2759	pgmR3PhysLinkRamRange(pVM, &pCur->RamRange, pRamPrev);
2760	pgmUnlock(pVM);
2761
2762	#ifdef VBOX_WITH_REM
2763	REMR3NotifyPhysRamRegister(pVM, GCPhys, cb, REM_NOTIFY_PHYS_RAM_FLAGS_MMIO2);
2764	#endif
2765	}
2766
2767	pgmPhysInvalidatePageMapTLB(pVM);
2768	return VINF_SUCCESS;
2769	}
2770
2771
2772	/**
2773	* Unmaps a MMIO2 region.
2774	*
2775	* This is done when a guest / the bios / state loading changes the
2776	* PCI config. The replacing of base memory has the same restrictions
2777	* as during registration, of course.
2778	*/
2779	VMMR3DECL(int) PGMR3PhysMMIO2Unmap(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
2780	{
2781	/*
2782	* Validate input
2783	*/
2784	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2785	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2786	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2787	AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
2788	AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
2789	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2790
2791	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2792	AssertReturn(pCur, VERR_NOT_FOUND);
2793	AssertReturn(pCur->fMapped, VERR_WRONG_ORDER);
2794	AssertReturn(pCur->RamRange.GCPhys == GCPhys, VERR_INVALID_PARAMETER);
2795	Assert(pCur->RamRange.GCPhysLast != NIL_RTGCPHYS);
2796
2797	Log(("PGMR3PhysMMIO2Unmap: %RGp-%RGp %s\n",
2798	pCur->RamRange.GCPhys, pCur->RamRange.GCPhysLast, pCur->RamRange.pszDesc));
2799
2800	/*
2801	* Unmap it.
2802	*/
2803	pgmLock(pVM);
2804
2805	#ifdef VBOX_WITH_REM
2806	RTGCPHYS GCPhysRangeREM;
2807	RTGCPHYS cbRangeREM;
2808	bool fInformREM;
2809	#endif
2810	if (pCur->fOverlapping)
2811	{
2812	/* Restore the RAM pages we've replaced. */
2813	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2814	while (pRam->GCPhys > pCur->RamRange.GCPhysLast)
2815	pRam = pRam->pNextR3;
2816
2817	PPGMPAGE pPageDst = &pRam->aPages[(pCur->RamRange.GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2818	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2819	while (cPagesLeft-- > 0)
2820	{
2821	PGM_PAGE_INIT_ZERO(pPageDst, pVM, PGMPAGETYPE_RAM);
2822	pVM->pgm.s.cZeroPages++;
2823	pPageDst++;
2824	}
2825
2826	/* Flush physical page map TLB. */
2827	pgmPhysInvalidatePageMapTLB(pVM);
2828	#ifdef VBOX_WITH_REM
2829	GCPhysRangeREM = NIL_RTGCPHYS; /* shuts up gcc */
2830	cbRangeREM = RTGCPHYS_MAX; /* ditto */
2831	fInformREM = false;
2832	#endif
2833	}
2834	else
2835	{
2836	#ifdef VBOX_WITH_REM
2837	GCPhysRangeREM = pCur->RamRange.GCPhys;
2838	cbRangeREM = pCur->RamRange.cb;
2839	fInformREM = true;
2840	#endif
2841	pgmR3PhysUnlinkRamRange(pVM, &pCur->RamRange);
2842	}
2843
2844	pCur->RamRange.GCPhys = NIL_RTGCPHYS;
2845	pCur->RamRange.GCPhysLast = NIL_RTGCPHYS;
2846	pCur->fOverlapping = false;
2847	pCur->fMapped = false;
2848
2849	/* Force a PGM pool flush as guest ram references have been changed. */
2850	/** @todo not entirely SMP safe; assuming for now the guest takes care
2851	* of this internally (not touch mapped mmio while changing the
2852	* mapping). */
2853	PVMCPU pVCpu = VMMGetCpu(pVM);
2854	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2855	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2856
2857	pgmPhysInvalidatePageMapTLB(pVM);
2858	pgmPhysInvalidRamRangeTlbs(pVM);
2859	pgmUnlock(pVM);
2860
2861	#ifdef VBOX_WITH_REM
2862	if (fInformREM)
2863	REMR3NotifyPhysRamDeregister(pVM, GCPhysRangeREM, cbRangeREM);
2864	#endif
2865
2866	return VINF_SUCCESS;
2867	}
2868
2869
2870	/**
2871	* Checks if the given address is an MMIO2 base address or not.
2872	*
2873	* @returns true/false accordingly.
2874	* @param pVM Pointer to the VM.
2875	* @param pDevIns The owner of the memory, optional.
2876	* @param GCPhys The address to check.
2877	*/
2878	VMMR3DECL(bool) PGMR3PhysMMIO2IsBase(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys)
2879	{
2880	/*
2881	* Validate input
2882	*/
2883	VM_ASSERT_EMT_RETURN(pVM, false);
2884	AssertPtrReturn(pDevIns, false);
2885	AssertReturn(GCPhys != NIL_RTGCPHYS, false);
2886	AssertReturn(GCPhys != 0, false);
2887	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), false);
2888
2889	/*
2890	* Search the list.
2891	*/
2892	pgmLock(pVM);
2893	for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
2894	if (pCur->RamRange.GCPhys == GCPhys)
2895	{
2896	Assert(pCur->fMapped);
2897	pgmUnlock(pVM);
2898	return true;
2899	}
2900	pgmUnlock(pVM);
2901	return false;
2902	}
2903
2904
2905	/**
2906	* Gets the HC physical address of a page in the MMIO2 region.
2907	*
2908	* This is API is intended for MMHyper and shouldn't be called
2909	* by anyone else...
2910	*
2911	* @returns VBox status code.
2912	* @param pVM Pointer to the VM.
2913	* @param pDevIns The owner of the memory, optional.
2914	* @param iRegion The region.
2915	* @param off The page expressed an offset into the MMIO2 region.
2916	* @param pHCPhys Where to store the result.
2917	*/
2918	VMMR3DECL(int) PGMR3PhysMMIO2GetHCPhys(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, PRTHCPHYS pHCPhys)
2919	{
2920	/*
2921	* Validate input
2922	*/
2923	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2924	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2925	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2926
2927	pgmLock(pVM);
2928	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2929	AssertReturn(pCur, VERR_NOT_FOUND);
2930	AssertReturn(off < pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2931
2932	PCPGMPAGE pPage = &pCur->RamRange.aPages[off >> PAGE_SHIFT];
2933	*pHCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2934	pgmUnlock(pVM);
2935	return VINF_SUCCESS;
2936	}
2937
2938
2939	/**
2940	* Maps a portion of an MMIO2 region into kernel space (host).
2941	*
2942	* The kernel mapping will become invalid when the MMIO2 memory is deregistered
2943	* or the VM is terminated.
2944	*
2945	* @return VBox status code.
2946	*
2947	* @param pVM Pointer to the VM.
2948	* @param pDevIns The device owning the MMIO2 memory.
2949	* @param iRegion The region.
2950	* @param off The offset into the region. Must be page aligned.
2951	* @param cb The number of bytes to map. Must be page aligned.
2952	* @param pszDesc Mapping description.
2953	* @param pR0Ptr Where to store the R0 address.
2954	*/
2955	VMMR3DECL(int) PGMR3PhysMMIO2MapKernel(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, RTGCPHYS cb,
2956	const char *pszDesc, PRTR0PTR pR0Ptr)
2957	{
2958	/*
2959	* Validate input.
2960	*/
2961	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2962	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2963	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2964
2965	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2966	AssertReturn(pCur, VERR_NOT_FOUND);
2967	AssertReturn(off < pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2968	AssertReturn(cb <= pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2969	AssertReturn(off + cb <= pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2970	NOREF(pszDesc);
2971
2972	/*
2973	* Pass the request on to the support library/driver.
2974	*/
2975	int rc = SUPR3PageMapKernel(pCur->pvR3, off, cb, 0, pR0Ptr);
2976
2977	return rc;
2978	}
2979
2980
2981	/**
2982	* Worker for PGMR3PhysRomRegister.
2983	*
2984	* This is here to simplify lock management, i.e. the caller does all the
2985	* locking and we can simply return without needing to remember to unlock
2986	* anything first.
2987	*
2988	* @returns VBox status.
2989	* @param pVM Pointer to the VM.
2990	* @param pDevIns The device instance owning the ROM.
2991	* @param GCPhys First physical address in the range.
2992	* Must be page aligned!
2993	* @param cb The size of the range (in bytes).
2994	* Must be page aligned!
2995	* @param pvBinary Pointer to the binary data backing the ROM image.
2996	* @param cbBinary The size of the binary data pvBinary points to.
2997	* This must be less or equal to @a cb.
2998	* @param fFlags Mask of flags. PGMPHYS_ROM_FLAGS_SHADOWED
2999	* and/or PGMPHYS_ROM_FLAGS_PERMANENT_BINARY.
3000	* @param pszDesc Pointer to description string. This must not be freed.
3001	*/
3002	static int pgmR3PhysRomRegister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
3003	const void pvBinary, uint32_t cbBinary, uint32_t fFlags, const char pszDesc)
3004	{
3005	/*
3006	* Validate input.
3007	*/
3008	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3009	AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
3010	AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
3011	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
3012	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
3013	AssertPtrReturn(pvBinary, VERR_INVALID_PARAMETER);
3014	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
3015	AssertReturn(!(fFlags & ~(PGMPHYS_ROM_FLAGS_SHADOWED \| PGMPHYS_ROM_FLAGS_PERMANENT_BINARY)), VERR_INVALID_PARAMETER);
3016	VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
3017
3018	const uint32_t cPages = cb >> PAGE_SHIFT;
3019
3020	/*
3021	* Find the ROM location in the ROM list first.
3022	*/
3023	PPGMROMRANGE pRomPrev = NULL;
3024	PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3;
3025	while (pRom && GCPhysLast >= pRom->GCPhys)
3026	{
3027	if ( GCPhys <= pRom->GCPhysLast
3028	&& GCPhysLast >= pRom->GCPhys)
3029	AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
3030	GCPhys, GCPhysLast, pszDesc,
3031	pRom->GCPhys, pRom->GCPhysLast, pRom->pszDesc),
3032	VERR_PGM_RAM_CONFLICT);
3033	/* next */
3034	pRomPrev = pRom;
3035	pRom = pRom->pNextR3;
3036	}
3037
3038	/*
3039	* Find the RAM location and check for conflicts.
3040	*
3041	* Conflict detection is a bit different than for RAM
3042	* registration since a ROM can be located within a RAM
3043	* range. So, what we have to check for is other memory
3044	* types (other than RAM that is) and that we don't span
3045	* more than one RAM range (layz).
3046	*/
3047	bool fRamExists = false;
3048	PPGMRAMRANGE pRamPrev = NULL;
3049	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
3050	while (pRam && GCPhysLast >= pRam->GCPhys)
3051	{
3052	if ( GCPhys <= pRam->GCPhysLast
3053	&& GCPhysLast >= pRam->GCPhys)
3054	{
3055	/* completely within? */
3056	AssertLogRelMsgReturn( GCPhys >= pRam->GCPhys
3057	&& GCPhysLast <= pRam->GCPhysLast,
3058	("%RGp-%RGp (%s) falls partly outside %RGp-%RGp (%s)\n",
3059	GCPhys, GCPhysLast, pszDesc,
3060	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
3061	VERR_PGM_RAM_CONFLICT);
3062	fRamExists = true;
3063	break;
3064	}
3065
3066	/* next */
3067	pRamPrev = pRam;
3068	pRam = pRam->pNextR3;
3069	}
3070	if (fRamExists)
3071	{
3072	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
3073	uint32_t cPagesLeft = cPages;
3074	while (cPagesLeft-- > 0)
3075	{
3076	AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
3077	("%RGp (%R[pgmpage]) isn't a RAM page - registering %RGp-%RGp (%s).\n",
3078	pRam->GCPhys + ((RTGCPHYS)(uintptr_t)(pPage - &pRam->aPages[0]) << PAGE_SHIFT),
3079	pPage, GCPhys, GCPhysLast, pszDesc), VERR_PGM_RAM_CONFLICT);
3080	Assert(PGM_PAGE_IS_ZERO(pPage));
3081	pPage++;
3082	}
3083	}
3084
3085	/*
3086	* Update the base memory reservation if necessary.
3087	*/
3088	uint32_t cExtraBaseCost = fRamExists ? 0 : cPages;
3089	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3090	cExtraBaseCost += cPages;
3091	if (cExtraBaseCost)
3092	{
3093	int rc = MMR3IncreaseBaseReservation(pVM, cExtraBaseCost);
3094	if (RT_FAILURE(rc))
3095	return rc;
3096	}
3097
3098	/*
3099	* Allocate memory for the virgin copy of the RAM.
3100	*/
3101	PGMMALLOCATEPAGESREQ pReq;
3102	int rc = GMMR3AllocatePagesPrepare(pVM, &pReq, cPages, GMMACCOUNT_BASE);
3103	AssertRCReturn(rc, rc);
3104
3105	for (uint32_t iPage = 0; iPage < cPages; iPage++)
3106	{
3107	pReq->aPages[iPage].HCPhysGCPhys = GCPhys + (iPage << PAGE_SHIFT);
3108	pReq->aPages[iPage].idPage = NIL_GMM_PAGEID;
3109	pReq->aPages[iPage].idSharedPage = NIL_GMM_PAGEID;
3110	}
3111
3112	rc = GMMR3AllocatePagesPerform(pVM, pReq);
3113	if (RT_FAILURE(rc))
3114	{
3115	GMMR3AllocatePagesCleanup(pReq);
3116	return rc;
3117	}
3118
3119	/*
3120	* Allocate the new ROM range and RAM range (if necessary).
3121	*/
3122	PPGMROMRANGE pRomNew;
3123	rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMROMRANGE, aPages[cPages]), 0, MM_TAG_PGM_PHYS, (void **)&pRomNew);
3124	if (RT_SUCCESS(rc))
3125	{
3126	PPGMRAMRANGE pRamNew = NULL;
3127	if (!fRamExists)
3128	rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]), sizeof(PGMPAGE), MM_TAG_PGM_PHYS, (void **)&pRamNew);
3129	if (RT_SUCCESS(rc))
3130	{
3131	/*
3132	* Initialize and insert the RAM range (if required).
3133	*/
3134	PPGMROMPAGE pRomPage = &pRomNew->aPages[0];
3135	if (!fRamExists)
3136	{
3137	pRamNew->pSelfR0 = MMHyperCCToR0(pVM, pRamNew);
3138	pRamNew->pSelfRC = MMHyperCCToRC(pVM, pRamNew);
3139	pRamNew->GCPhys = GCPhys;
3140	pRamNew->GCPhysLast = GCPhysLast;
3141	pRamNew->cb = cb;
3142	pRamNew->pszDesc = pszDesc;
3143	pRamNew->fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_ROM;
3144	pRamNew->pvR3 = NULL;
3145	pRamNew->paLSPages = NULL;
3146
3147	PPGMPAGE pPage = &pRamNew->aPages[0];
3148	for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
3149	{
3150	PGM_PAGE_INIT(pPage,
3151	pReq->aPages[iPage].HCPhysGCPhys,
3152	pReq->aPages[iPage].idPage,
3153	PGMPAGETYPE_ROM,
3154	PGM_PAGE_STATE_ALLOCATED);
3155
3156	pRomPage->Virgin = *pPage;
3157	}
3158
3159	pVM->pgm.s.cAllPages += cPages;
3160	pgmR3PhysLinkRamRange(pVM, pRamNew, pRamPrev);
3161	}
3162	else
3163	{
3164	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
3165	for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
3166	{
3167	PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_ROM);
3168	PGM_PAGE_SET_HCPHYS(pVM, pPage, pReq->aPages[iPage].HCPhysGCPhys);
3169	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
3170	PGM_PAGE_SET_PAGEID(pVM, pPage, pReq->aPages[iPage].idPage);
3171	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
3172	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
3173	PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
3174
3175	pRomPage->Virgin = *pPage;
3176	}
3177
3178	pRamNew = pRam;
3179
3180	pVM->pgm.s.cZeroPages -= cPages;
3181	}
3182	pVM->pgm.s.cPrivatePages += cPages;
3183
3184	/* Flush physical page map TLB. */
3185	pgmPhysInvalidatePageMapTLB(pVM);
3186
3187
3188	/*
3189	* !HACK ALERT! REM + (Shadowed) ROM ==> mess.
3190	*
3191	* If it's shadowed we'll register the handler after the ROM notification
3192	* so we get the access handler callbacks that we should. If it isn't
3193	* shadowed we'll do it the other way around to make REM use the built-in
3194	* ROM behavior and not the handler behavior (which is to route all access
3195	* to PGM atm).
3196	*/
3197	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3198	{
3199	#ifdef VBOX_WITH_REM
3200	REMR3NotifyPhysRomRegister(pVM, GCPhys, cb, NULL, true /* fShadowed */);
3201	#endif
3202	rc = PGMR3HandlerPhysicalRegister(pVM, PGMPHYSHANDLERTYPE_PHYSICAL_WRITE, GCPhys, GCPhysLast,
3203	pgmR3PhysRomWriteHandler, pRomNew,
3204	NULL, "pgmPhysRomWriteHandler", MMHyperCCToR0(pVM, pRomNew),
3205	NULL, "pgmPhysRomWriteHandler", MMHyperCCToRC(pVM, pRomNew), pszDesc);
3206	}
3207	else
3208	{
3209	rc = PGMR3HandlerPhysicalRegister(pVM, PGMPHYSHANDLERTYPE_PHYSICAL_WRITE, GCPhys, GCPhysLast,
3210	pgmR3PhysRomWriteHandler, pRomNew,
3211	NULL, "pgmPhysRomWriteHandler", MMHyperCCToR0(pVM, pRomNew),
3212	NULL, "pgmPhysRomWriteHandler", MMHyperCCToRC(pVM, pRomNew), pszDesc);
3213	#ifdef VBOX_WITH_REM
3214	REMR3NotifyPhysRomRegister(pVM, GCPhys, cb, NULL, false /* fShadowed */);
3215	#endif
3216	}
3217	if (RT_SUCCESS(rc))
3218	{
3219	/*
3220	* Copy the image over to the virgin pages.
3221	* This must be done after linking in the RAM range.
3222	*/
3223	size_t cbBinaryLeft = cbBinary;
3224	PPGMPAGE pRamPage = &pRamNew->aPages[(GCPhys - pRamNew->GCPhys) >> PAGE_SHIFT];
3225	for (uint32_t iPage = 0; iPage < cPages; iPage++, pRamPage++)
3226	{
3227	void *pvDstPage;
3228	rc = pgmPhysPageMap(pVM, pRamPage, GCPhys + (iPage << PAGE_SHIFT), &pvDstPage);
3229	if (RT_FAILURE(rc))
3230	{
3231	VMSetError(pVM, rc, RT_SRC_POS, "Failed to map virgin ROM page at %RGp", GCPhys);
3232	break;
3233	}
3234	if (cbBinaryLeft >= PAGE_SIZE)
3235	{
3236	memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << PAGE_SHIFT), PAGE_SIZE);
3237	cbBinaryLeft -= PAGE_SIZE;
3238	}
3239	else
3240	{
3241	ASMMemZeroPage(pvDstPage); /* (shouldn't be necessary, but can't hurt either) */
3242	if (cbBinaryLeft > 0)
3243	{
3244	memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << PAGE_SHIFT), cbBinaryLeft);
3245	cbBinaryLeft = 0;
3246	}
3247	}
3248	}
3249	if (RT_SUCCESS(rc))
3250	{
3251	/*
3252	* Initialize the ROM range.
3253	* Note that the Virgin member of the pages has already been initialized above.
3254	*/
3255	pRomNew->GCPhys = GCPhys;
3256	pRomNew->GCPhysLast = GCPhysLast;
3257	pRomNew->cb = cb;
3258	pRomNew->fFlags = fFlags;
3259	pRomNew->idSavedState = UINT8_MAX;
3260	pRomNew->cbOriginal = cbBinary;
3261	#ifdef VBOX_STRICT
3262	pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY
3263	? pvBinary : RTMemDup(pvBinary, cbBinary);
3264	#else
3265	pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY ? pvBinary : NULL;
3266	#endif
3267	pRomNew->pszDesc = pszDesc;
3268
3269	for (unsigned iPage = 0; iPage < cPages; iPage++)
3270	{
3271	PPGMROMPAGE pPage = &pRomNew->aPages[iPage];
3272	pPage->enmProt = PGMROMPROT_READ_ROM_WRITE_IGNORE;
3273	PGM_PAGE_INIT_ZERO(&pPage->Shadow, pVM, PGMPAGETYPE_ROM_SHADOW);
3274	}
3275
3276	/* update the page count stats for the shadow pages. */
3277	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3278	{
3279	pVM->pgm.s.cZeroPages += cPages;
3280	pVM->pgm.s.cAllPages += cPages;
3281	}
3282
3283	/*
3284	* Insert the ROM range, tell REM and return successfully.
3285	*/
3286	pRomNew->pNextR3 = pRom;
3287	pRomNew->pNextR0 = pRom ? MMHyperCCToR0(pVM, pRom) : NIL_RTR0PTR;
3288	pRomNew->pNextRC = pRom ? MMHyperCCToRC(pVM, pRom) : NIL_RTRCPTR;
3289
3290	if (pRomPrev)
3291	{
3292	pRomPrev->pNextR3 = pRomNew;
3293	pRomPrev->pNextR0 = MMHyperCCToR0(pVM, pRomNew);
3294	pRomPrev->pNextRC = MMHyperCCToRC(pVM, pRomNew);
3295	}
3296	else
3297	{
3298	pVM->pgm.s.pRomRangesR3 = pRomNew;
3299	pVM->pgm.s.pRomRangesR0 = MMHyperCCToR0(pVM, pRomNew);
3300	pVM->pgm.s.pRomRangesRC = MMHyperCCToRC(pVM, pRomNew);
3301	}
3302
3303	pgmPhysInvalidatePageMapTLB(pVM);
3304	GMMR3AllocatePagesCleanup(pReq);
3305	return VINF_SUCCESS;
3306	}
3307
3308	/* bail out */
3309
3310	int rc2 = PGMHandlerPhysicalDeregister(pVM, GCPhys);
3311	AssertRC(rc2);
3312	}
3313
3314	if (!fRamExists)
3315	{
3316	pgmR3PhysUnlinkRamRange2(pVM, pRamNew, pRamPrev);
3317	MMHyperFree(pVM, pRamNew);
3318	}
3319	}
3320	MMHyperFree(pVM, pRomNew);
3321	}
3322
3323	/** @todo Purge the mapping cache or something... */
3324	GMMR3FreeAllocatedPages(pVM, pReq);
3325	GMMR3AllocatePagesCleanup(pReq);
3326	return rc;
3327	}
3328
3329
3330	/**
3331	* Registers a ROM image.
3332	*
3333	* Shadowed ROM images requires double the amount of backing memory, so,
3334	* don't use that unless you have to. Shadowing of ROM images is process
3335	* where we can select where the reads go and where the writes go. On real
3336	* hardware the chipset provides means to configure this. We provide
3337	* PGMR3PhysProtectROM() for this purpose.
3338	*
3339	* A read-only copy of the ROM image will always be kept around while we
3340	* will allocate RAM pages for the changes on demand (unless all memory
3341	* is configured to be preallocated).
3342	*
3343	* @returns VBox status.
3344	* @param pVM Pointer to the VM.
3345	* @param pDevIns The device instance owning the ROM.
3346	* @param GCPhys First physical address in the range.
3347	* Must be page aligned!
3348	* @param cb The size of the range (in bytes).
3349	* Must be page aligned!
3350	* @param pvBinary Pointer to the binary data backing the ROM image.
3351	* @param cbBinary The size of the binary data pvBinary points to.
3352	* This must be less or equal to @a cb.
3353	* @param fFlags Mask of flags. PGMPHYS_ROM_FLAGS_SHADOWED
3354	* and/or PGMPHYS_ROM_FLAGS_PERMANENT_BINARY.
3355	* @param pszDesc Pointer to description string. This must not be freed.
3356	*
3357	* @remark There is no way to remove the rom, automatically on device cleanup or
3358	* manually from the device yet. This isn't difficult in any way, it's
3359	* just not something we expect to be necessary for a while.
3360	*/
3361	VMMR3DECL(int) PGMR3PhysRomRegister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
3362	const void pvBinary, uint32_t cbBinary, uint32_t fFlags, const char pszDesc)
3363	{
3364	Log(("PGMR3PhysRomRegister: pDevIns=%p GCPhys=%RGp(-%RGp) cb=%RGp pvBinary=%p cbBinary=%#x fFlags=%#x pszDesc=%s\n",
3365	pDevIns, GCPhys, GCPhys + cb, cb, pvBinary, cbBinary, fFlags, pszDesc));
3366	pgmLock(pVM);
3367	int rc = pgmR3PhysRomRegister(pVM, pDevIns, GCPhys, cb, pvBinary, cbBinary, fFlags, pszDesc);
3368	pgmUnlock(pVM);
3369	return rc;
3370	}
3371
3372
3373	/**
3374	* \#PF Handler callback for ROM write accesses.
3375	*
3376	* @returns VINF_SUCCESS if the handler have carried out the operation.
3377	* @returns VINF_PGM_HANDLER_DO_DEFAULT if the caller should carry out the access operation.
3378	* @param pVM Pointer to the VM.
3379	* @param GCPhys The physical address the guest is writing to.
3380	* @param pvPhys The HC mapping of that address.
3381	* @param pvBuf What the guest is reading/writing.
3382	* @param cbBuf How much it's reading/writing.
3383	* @param enmAccessType The access type.
3384	* @param pvUser User argument.
3385	*/
3386	static DECLCALLBACK(int) pgmR3PhysRomWriteHandler(PVM pVM, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf,
3387	PGMACCESSTYPE enmAccessType, void *pvUser)
3388	{
3389	PPGMROMRANGE pRom = (PPGMROMRANGE)pvUser;
3390	const uint32_t iPage = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
3391	Assert(iPage < (pRom->cb >> PAGE_SHIFT));
3392	PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
3393	Log5(("pgmR3PhysRomWriteHandler: %d %c %#08RGp %#04zx\n", pRomPage->enmProt, enmAccessType == PGMACCESSTYPE_READ ? 'R' : 'W', GCPhys, cbBuf));
3394	NOREF(pvPhys);
3395
3396	if (enmAccessType == PGMACCESSTYPE_READ)
3397	{
3398	switch (pRomPage->enmProt)
3399	{
3400	/*
3401	* Take the default action.
3402	*/
3403	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
3404	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
3405	case PGMROMPROT_READ_ROM_WRITE_RAM:
3406	case PGMROMPROT_READ_RAM_WRITE_RAM:
3407	return VINF_PGM_HANDLER_DO_DEFAULT;
3408
3409	default:
3410	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
3411	pRom->aPages[iPage].enmProt, iPage, GCPhys),
3412	VERR_IPE_NOT_REACHED_DEFAULT_CASE);
3413	}
3414	}
3415	else
3416	{
3417	Assert(enmAccessType == PGMACCESSTYPE_WRITE);
3418	switch (pRomPage->enmProt)
3419	{
3420	/*
3421	* Ignore writes.
3422	*/
3423	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
3424	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
3425	return VINF_SUCCESS;
3426
3427	/*
3428	* Write to the RAM page.
3429	*/
3430	case PGMROMPROT_READ_ROM_WRITE_RAM:
3431	case PGMROMPROT_READ_RAM_WRITE_RAM: /* yes this will get here too, it's way simpler that way. */
3432	{
3433	/* This should be impossible now, pvPhys doesn't work cross page anylonger. */
3434	Assert(((GCPhys - pRom->GCPhys + cbBuf - 1) >> PAGE_SHIFT) == iPage);
3435
3436	/*
3437	* Take the lock, do lazy allocation, map the page and copy the data.
3438	*
3439	* Note that we have to bypass the mapping TLB since it works on
3440	* guest physical addresses and entering the shadow page would
3441	* kind of screw things up...
3442	*/
3443	int rc = pgmLock(pVM);
3444	AssertRC(rc);
3445
3446	PPGMPAGE pShadowPage = &pRomPage->Shadow;
3447	if (!PGMROMPROT_IS_ROM(pRomPage->enmProt))
3448	{
3449	pShadowPage = pgmPhysGetPage(pVM, GCPhys);
3450	AssertLogRelReturn(pShadowPage, VERR_PGM_PHYS_PAGE_GET_IPE);
3451	}
3452
3453	void *pvDstPage;
3454	rc = pgmPhysPageMakeWritableAndMap(pVM, pShadowPage, GCPhys & X86_PTE_PG_MASK, &pvDstPage);
3455	if (RT_SUCCESS(rc))
3456	{
3457	memcpy((uint8_t *)pvDstPage + (GCPhys & PAGE_OFFSET_MASK), pvBuf, cbBuf);
3458	pRomPage->LiveSave.fWrittenTo = true;
3459	}
3460
3461	pgmUnlock(pVM);
3462	return rc;
3463	}
3464
3465	default:
3466	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
3467	pRom->aPages[iPage].enmProt, iPage, GCPhys),
3468	VERR_IPE_NOT_REACHED_DEFAULT_CASE);
3469	}
3470	}
3471	}
3472
3473
3474	/**
3475	* Called by PGMR3Reset to reset the shadow, switch to the virgin,
3476	* and verify that the virgin part is untouched.
3477	*
3478	* This is done after the normal memory has been cleared.
3479	*
3480	* ASSUMES that the caller owns the PGM lock.
3481	*
3482	* @param pVM Pointer to the VM.
3483	*/
3484	int pgmR3PhysRomReset(PVM pVM)
3485	{
3486	PGM_LOCK_ASSERT_OWNER(pVM);
3487	for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3488	{
3489	const uint32_t cPages = pRom->cb >> PAGE_SHIFT;
3490
3491	if (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3492	{
3493	/*
3494	* Reset the physical handler.
3495	*/
3496	int rc = PGMR3PhysRomProtect(pVM, pRom->GCPhys, pRom->cb, PGMROMPROT_READ_ROM_WRITE_IGNORE);
3497	AssertRCReturn(rc, rc);
3498
3499	/*
3500	* What we do with the shadow pages depends on the memory
3501	* preallocation option. If not enabled, we'll just throw
3502	* out all the dirty pages and replace them by the zero page.
3503	*/
3504	if (!pVM->pgm.s.fRamPreAlloc)
3505	{
3506	/* Free the dirty pages. */
3507	uint32_t cPendingPages = 0;
3508	PGMMFREEPAGESREQ pReq;
3509	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
3510	AssertRCReturn(rc, rc);
3511
3512	for (uint32_t iPage = 0; iPage < cPages; iPage++)
3513	if ( !PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow)
3514	&& !PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow))
3515	{
3516	Assert(PGM_PAGE_GET_STATE(&pRom->aPages[iPage].Shadow) == PGM_PAGE_STATE_ALLOCATED);
3517	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, &pRom->aPages[iPage].Shadow,
3518	pRom->GCPhys + (iPage << PAGE_SHIFT));
3519	AssertLogRelRCReturn(rc, rc);
3520	}
3521
3522	if (cPendingPages)
3523	{
3524	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
3525	AssertLogRelRCReturn(rc, rc);
3526	}
3527	GMMR3FreePagesCleanup(pReq);
3528	}
3529	else
3530	{
3531	/* clear all the shadow pages. */
3532	for (uint32_t iPage = 0; iPage < cPages; iPage++)
3533	{
3534	if (PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow))
3535	continue;
3536	Assert(!PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow));
3537	void *pvDstPage;
3538	const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
3539	rc = pgmPhysPageMakeWritableAndMap(pVM, &pRom->aPages[iPage].Shadow, GCPhys, &pvDstPage);
3540	if (RT_FAILURE(rc))
3541	break;
3542	ASMMemZeroPage(pvDstPage);
3543	}
3544	AssertRCReturn(rc, rc);
3545	}
3546	}
3547
3548	#ifdef VBOX_STRICT
3549	/*
3550	* Verify that the virgin page is unchanged if possible.
3551	*/
3552	if (pRom->pvOriginal)
3553	{
3554	size_t cbSrcLeft = pRom->cbOriginal;
3555	uint8_t const pbSrcPage = (uint8_t const )pRom->pvOriginal;
3556	for (uint32_t iPage = 0; iPage < cPages && cbSrcLeft > 0; iPage++, pbSrcPage += PAGE_SIZE)
3557	{
3558	const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
3559	void const *pvDstPage;
3560	int rc = pgmPhysPageMapReadOnly(pVM, &pRom->aPages[iPage].Virgin, GCPhys, &pvDstPage);
3561	if (RT_FAILURE(rc))
3562	break;
3563
3564	if (memcmp(pvDstPage, pbSrcPage, RT_MIN(cbSrcLeft, PAGE_SIZE)))
3565	LogRel(("pgmR3PhysRomReset: %RGp rom page changed (%s) - loaded saved state?\n",
3566	GCPhys, pRom->pszDesc));
3567	cbSrcLeft -= RT_MIN(cbSrcLeft, PAGE_SIZE);
3568	}
3569	}
3570	#endif
3571	}
3572
3573	return VINF_SUCCESS;
3574	}
3575
3576
3577	/**
3578	* Called by PGMR3Term to free resources.
3579	*
3580	* ASSUMES that the caller owns the PGM lock.
3581	*
3582	* @param pVM Pointer to the VM.
3583	*/
3584	void pgmR3PhysRomTerm(PVM pVM)
3585	{
3586	#ifdef RT_STRICT
3587	/*
3588	* Free the heap copy of the original bits.
3589	*/
3590	for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3591	{
3592	if ( pRom->pvOriginal
3593	&& !(pRom->fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY))
3594	{
3595	RTMemFree((void *)pRom->pvOriginal);
3596	pRom->pvOriginal = NULL;
3597	}
3598	}
3599	#endif
3600	}
3601
3602
3603	/**
3604	* Change the shadowing of a range of ROM pages.
3605	*
3606	* This is intended for implementing chipset specific memory registers
3607	* and will not be very strict about the input. It will silently ignore
3608	* any pages that are not the part of a shadowed ROM.
3609	*
3610	* @returns VBox status code.
3611	* @retval VINF_PGM_SYNC_CR3
3612	*
3613	* @param pVM Pointer to the VM.
3614	* @param GCPhys Where to start. Page aligned.
3615	* @param cb How much to change. Page aligned.
3616	* @param enmProt The new ROM protection.
3617	*/
3618	VMMR3DECL(int) PGMR3PhysRomProtect(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, PGMROMPROT enmProt)
3619	{
3620	/*
3621	* Check input
3622	*/
3623	if (!cb)
3624	return VINF_SUCCESS;
3625	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3626	AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3627	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
3628	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
3629	AssertReturn(enmProt >= PGMROMPROT_INVALID && enmProt <= PGMROMPROT_END, VERR_INVALID_PARAMETER);
3630
3631	/*
3632	* Process the request.
3633	*/
3634	pgmLock(pVM);
3635	int rc = VINF_SUCCESS;
3636	bool fFlushTLB = false;
3637	for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3638	{
3639	if ( GCPhys <= pRom->GCPhysLast
3640	&& GCPhysLast >= pRom->GCPhys
3641	&& (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED))
3642	{
3643	/*
3644	* Iterate the relevant pages and make necessary the changes.
3645	*/
3646	bool fChanges = false;
3647	uint32_t const cPages = pRom->GCPhysLast <= GCPhysLast
3648	? pRom->cb >> PAGE_SHIFT
3649	: (GCPhysLast - pRom->GCPhys + 1) >> PAGE_SHIFT;
3650	for (uint32_t iPage = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
3651	iPage < cPages;
3652	iPage++)
3653	{
3654	PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
3655	if (PGMROMPROT_IS_ROM(pRomPage->enmProt) != PGMROMPROT_IS_ROM(enmProt))
3656	{
3657	fChanges = true;
3658
3659	/* flush references to the page. */
3660	PPGMPAGE pRamPage = pgmPhysGetPage(pVM, pRom->GCPhys + (iPage << PAGE_SHIFT));
3661	int rc2 = pgmPoolTrackUpdateGCPhys(pVM, pRom->GCPhys + (iPage << PAGE_SHIFT), pRamPage,
3662	true /fFlushPTEs/, &fFlushTLB);
3663	if (rc2 != VINF_SUCCESS && (rc == VINF_SUCCESS \|\| RT_FAILURE(rc2)))
3664	rc = rc2;
3665
3666	PPGMPAGE pOld = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Virgin : &pRomPage->Shadow;
3667	PPGMPAGE pNew = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Shadow : &pRomPage->Virgin;
3668
3669	pOld = pRamPage;
3670	pRamPage = pNew;
3671	/** @todo preserve the volatile flags (handlers) when these have been moved out of HCPhys! */
3672	}
3673	pRomPage->enmProt = enmProt;
3674	}
3675
3676	/*
3677	* Reset the access handler if we made changes, no need
3678	* to optimize this.
3679	*/
3680	if (fChanges)
3681	{
3682	int rc2 = PGMHandlerPhysicalReset(pVM, pRom->GCPhys);
3683	if (RT_FAILURE(rc2))
3684	{
3685	pgmUnlock(pVM);
3686	AssertRC(rc);
3687	return rc2;
3688	}
3689	}
3690
3691	/* Advance - cb isn't updated. */
3692	GCPhys = pRom->GCPhys + (cPages << PAGE_SHIFT);
3693	}
3694	}
3695	pgmUnlock(pVM);
3696	if (fFlushTLB)
3697	PGM_INVL_ALL_VCPU_TLBS(pVM);
3698
3699	return rc;
3700	}
3701
3702
3703	/**
3704	* Sets the Address Gate 20 state.
3705	*
3706	* @param pVCpu Pointer to the VMCPU.
3707	* @param fEnable True if the gate should be enabled.
3708	* False if the gate should be disabled.
3709	*/
3710	VMMDECL(void) PGMR3PhysSetA20(PVMCPU pVCpu, bool fEnable)
3711	{
3712	LogFlow(("PGMR3PhysSetA20 %d (was %d)\n", fEnable, pVCpu->pgm.s.fA20Enabled));
3713	if (pVCpu->pgm.s.fA20Enabled != fEnable)
3714	{
3715	pVCpu->pgm.s.fA20Enabled = fEnable;
3716	pVCpu->pgm.s.GCPhysA20Mask = ~(RTGCPHYS)(!fEnable << 20);
3717	#ifdef VBOX_WITH_REM
3718	REMR3A20Set(pVCpu->pVMR3, pVCpu, fEnable);
3719	#endif
3720	#ifdef PGM_WITH_A20
3721	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL;
3722	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3723	pgmR3RefreshShadowModeAfterA20Change(pVCpu);
3724	HWACCMFlushTLB(pVCpu);
3725	#endif
3726	STAM_REL_COUNTER_INC(&pVCpu->pgm.s.cA20Changes);
3727	}
3728	}
3729
3730
3731	/**
3732	* Tree enumeration callback for dealing with age rollover.
3733	* It will perform a simple compression of the current age.
3734	*/
3735	static DECLCALLBACK(int) pgmR3PhysChunkAgeingRolloverCallback(PAVLU32NODECORE pNode, void *pvUser)
3736	{
3737	/* Age compression - ASSUMES iNow == 4. */
3738	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
3739	if (pChunk->iLastUsed >= UINT32_C(0xffffff00))
3740	pChunk->iLastUsed = 3;
3741	else if (pChunk->iLastUsed >= UINT32_C(0xfffff000))
3742	pChunk->iLastUsed = 2;
3743	else if (pChunk->iLastUsed)
3744	pChunk->iLastUsed = 1;
3745	else /* iLastUsed = 0 */
3746	pChunk->iLastUsed = 4;
3747
3748	NOREF(pvUser);
3749	return 0;
3750	}
3751
3752
3753	/**
3754	* The structure passed in the pvUser argument of pgmR3PhysChunkUnmapCandidateCallback().
3755	*/
3756	typedef struct PGMR3PHYSCHUNKUNMAPCB
3757	{
3758	PVM pVM; /*< Pointer to the VM. /
3759	PPGMCHUNKR3MAP pChunk; /*< The chunk to unmap. /
3760	} PGMR3PHYSCHUNKUNMAPCB, *PPGMR3PHYSCHUNKUNMAPCB;
3761
3762
3763	/**
3764	* Callback used to find the mapping that's been unused for
3765	* the longest time.
3766	*/
3767	static DECLCALLBACK(int) pgmR3PhysChunkUnmapCandidateCallback(PAVLU32NODECORE pNode, void *pvUser)
3768	{
3769	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
3770	PPGMR3PHYSCHUNKUNMAPCB pArg = (PPGMR3PHYSCHUNKUNMAPCB)pvUser;
3771
3772	/*
3773	* Check for locks and compare when last used.
3774	*/
3775	if (pChunk->cRefs)
3776	return 0;
3777	if (pChunk->cPermRefs)
3778	return 0;
3779	if ( pArg->pChunk
3780	&& pChunk->iLastUsed >= pArg->pChunk->iLastUsed)
3781	return 0;
3782
3783	/*
3784	* Check that it's not in any of the TLBs.
3785	*/
3786	PVM pVM = pArg->pVM;
3787	if ( pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(pChunk->Core.Key)].idChunk
3788	== pChunk->Core.Key)
3789	{
3790	pChunk = NULL;
3791	return 0;
3792	}
3793	#ifdef VBOX_STRICT
3794	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
3795	{
3796	Assert(pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk != pChunk);
3797	Assert(pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk != pChunk->Core.Key);
3798	}
3799	#endif
3800
3801	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbHC.aEntries); i++)
3802	if (pVM->pgm.s.PhysTlbHC.aEntries[i].pMap == pChunk)
3803	return 0;
3804
3805	pArg->pChunk = pChunk;
3806	return 0;
3807	}
3808
3809
3810	/**
3811	* Finds a good candidate for unmapping when the ring-3 mapping cache is full.
3812	*
3813	* The candidate will not be part of any TLBs, so no need to flush
3814	* anything afterwards.
3815	*
3816	* @returns Chunk id.
3817	* @param pVM Pointer to the VM.
3818	*/
3819	static int32_t pgmR3PhysChunkFindUnmapCandidate(PVM pVM)
3820	{
3821	PGM_LOCK_ASSERT_OWNER(pVM);
3822
3823	/*
3824	* Enumerate the age tree starting with the left most node.
3825	*/
3826	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3827	PGMR3PHYSCHUNKUNMAPCB Args;
3828	Args.pVM = pVM;
3829	Args.pChunk = NULL;
3830	RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /fFromLeft/, pgmR3PhysChunkUnmapCandidateCallback, &Args);
3831	Assert(Args.pChunk);
3832	if (Args.pChunk)
3833	{
3834	Assert(Args.pChunk->cRefs == 0);
3835	Assert(Args.pChunk->cPermRefs == 0);
3836	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3837	return Args.pChunk->Core.Key;
3838	}
3839
3840	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3841	return INT32_MAX;
3842	}
3843
3844
3845	/**
3846	* Rendezvous callback used by pgmR3PhysUnmapChunk that unmaps a chunk
3847	*
3848	* This is only called on one of the EMTs while the other ones are waiting for
3849	* it to complete this function.
3850	*
3851	* @returns VINF_SUCCESS (VBox strict status code).
3852	* @param pVM Pointer to the VM.
3853	* @param pVCpu The VMCPU for the EMT we're being called on. Unused.
3854	* @param pvUser User pointer. Unused
3855	*
3856	*/
3857	static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysUnmapChunkRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
3858	{
3859	int rc = VINF_SUCCESS;
3860	pgmLock(pVM);
3861	NOREF(pVCpu); NOREF(pvUser);
3862
3863	if (pVM->pgm.s.ChunkR3Map.c >= pVM->pgm.s.ChunkR3Map.cMax)
3864	{
3865	/* Flush the pgm pool cache; call the internal rendezvous handler as we're already in a rendezvous handler here. */
3866	/** @todo also not really efficient to unmap a chunk that contains PD
3867	* or PT pages. */
3868	pgmR3PoolClearAllRendezvous(pVM, &pVM->aCpus[0], NULL /* no need to flush the REM TLB as we already did that above */);
3869
3870	/*
3871	* Request the ring-0 part to unmap a chunk to make space in the mapping cache.
3872	*/
3873	GMMMAPUNMAPCHUNKREQ Req;
3874	Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
3875	Req.Hdr.cbReq = sizeof(Req);
3876	Req.pvR3 = NULL;
3877	Req.idChunkMap = NIL_GMM_CHUNKID;
3878	Req.idChunkUnmap = pgmR3PhysChunkFindUnmapCandidate(pVM);
3879	if (Req.idChunkUnmap != INT32_MAX)
3880	{
3881	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkUnmap, a);
3882	rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
3883	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkUnmap, a);
3884	if (RT_SUCCESS(rc))
3885	{
3886	/*
3887	* Remove the unmapped one.
3888	*/
3889	PPGMCHUNKR3MAP pUnmappedChunk = (PPGMCHUNKR3MAP)RTAvlU32Remove(&pVM->pgm.s.ChunkR3Map.pTree, Req.idChunkUnmap);
3890	AssertRelease(pUnmappedChunk);
3891	AssertRelease(!pUnmappedChunk->cRefs);
3892	AssertRelease(!pUnmappedChunk->cPermRefs);
3893	pUnmappedChunk->pv = NULL;
3894	pUnmappedChunk->Core.Key = UINT32_MAX;
3895	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
3896	MMR3HeapFree(pUnmappedChunk);
3897	#else
3898	MMR3UkHeapFree(pVM, pUnmappedChunk, MM_TAG_PGM_CHUNK_MAPPING);
3899	#endif
3900	pVM->pgm.s.ChunkR3Map.c--;
3901	pVM->pgm.s.cUnmappedChunks++;
3902
3903	/*
3904	* Flush dangling PGM pointers (R3 & R0 ptrs to GC physical addresses).
3905	*/
3906	/** todo: we should not flush chunks which include cr3 mappings. */
3907	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3908	{
3909	PPGMCPU pPGM = &pVM->aCpus[idCpu].pgm.s;
3910
3911	pPGM->pGst32BitPdR3 = NULL;
3912	pPGM->pGstPaePdptR3 = NULL;
3913	pPGM->pGstAmd64Pml4R3 = NULL;
3914	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
3915	pPGM->pGst32BitPdR0 = NIL_RTR0PTR;
3916	pPGM->pGstPaePdptR0 = NIL_RTR0PTR;
3917	pPGM->pGstAmd64Pml4R0 = NIL_RTR0PTR;
3918	#endif
3919	for (unsigned i = 0; i < RT_ELEMENTS(pPGM->apGstPaePDsR3); i++)
3920	{
3921	pPGM->apGstPaePDsR3[i] = NULL;
3922	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
3923	pPGM->apGstPaePDsR0[i] = NIL_RTR0PTR;
3924	#endif
3925	}
3926
3927	/* Flush REM TLBs. */
3928	CPUMSetChangedFlags(&pVM->aCpus[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
3929	}
3930	#ifdef VBOX_WITH_REM
3931	/* Flush REM translation blocks. */
3932	REMFlushTBs(pVM);
3933	#endif
3934	}
3935	}
3936	}
3937	pgmUnlock(pVM);
3938	return rc;
3939	}
3940
3941	/**
3942	* Unmap a chunk to free up virtual address space (request packet handler for pgmR3PhysChunkMap)
3943	*
3944	* @returns VBox status code.
3945	* @param pVM Pointer to the VM.
3946	*/
3947	void pgmR3PhysUnmapChunk(PVM pVM)
3948	{
3949	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysUnmapChunkRendezvous, NULL);
3950	AssertRC(rc);
3951	}
3952
3953
3954	/**
3955	* Maps the given chunk into the ring-3 mapping cache.
3956	*
3957	* This will call ring-0.
3958	*
3959	* @returns VBox status code.
3960	* @param pVM Pointer to the VM.
3961	* @param idChunk The chunk in question.
3962	* @param ppChunk Where to store the chunk tracking structure.
3963	*
3964	* @remarks Called from within the PGM critical section.
3965	* @remarks Can be called from any thread!
3966	*/
3967	int pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk)
3968	{
3969	int rc;
3970
3971	PGM_LOCK_ASSERT_OWNER(pVM);
3972
3973	/*
3974	* Move the chunk time forward.
3975	*/
3976	pVM->pgm.s.ChunkR3Map.iNow++;
3977	if (pVM->pgm.s.ChunkR3Map.iNow == 0)
3978	{
3979	pVM->pgm.s.ChunkR3Map.iNow = 4;
3980	RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /fFromLeft/, pgmR3PhysChunkAgeingRolloverCallback, NULL);
3981	}
3982
3983	/*
3984	* Allocate a new tracking structure first.
3985	*/
3986	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
3987	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3HeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk));
3988	#else
3989	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3UkHeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk), NULL);
3990	#endif
3991	AssertReturn(pChunk, VERR_NO_MEMORY);
3992	pChunk->Core.Key = idChunk;
3993	pChunk->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
3994
3995	/*
3996	* Request the ring-0 part to map the chunk in question.
3997	*/
3998	GMMMAPUNMAPCHUNKREQ Req;
3999	Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
4000	Req.Hdr.cbReq = sizeof(Req);
4001	Req.pvR3 = NULL;
4002	Req.idChunkMap = idChunk;
4003	Req.idChunkUnmap = NIL_GMM_CHUNKID;
4004
4005	/* Must be callable from any thread, so can't use VMMR3CallR0. */
4006	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkMap, a);
4007	rc = SUPR3CallVMMR0Ex(pVM->pVMR0, NIL_VMCPUID, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
4008	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkMap, a);
4009	if (RT_SUCCESS(rc))
4010	{
4011	pChunk->pv = Req.pvR3;
4012
4013	/*
4014	* If we're running out of virtual address space, then we should
4015	* unmap another chunk.
4016	*
4017	* Currently, an unmap operation requires that all other virtual CPUs
4018	* are idling and not by chance making use of the memory we're
4019	* unmapping. So, we create an async unmap operation here.
4020	*
4021	* Now, when creating or restoring a saved state this wont work very
4022	* well since we may want to restore all guest RAM + a little something.
4023	* So, we have to do the unmap synchronously. Fortunately for us
4024	* though, during these operations the other virtual CPUs are inactive
4025	* and it should be safe to do this.
4026	*/
4027	/** @todo Eventually we should lock all memory when used and do
4028	* map+unmap as one kernel call without any rendezvous or
4029	* other precautions. */
4030	if (pVM->pgm.s.ChunkR3Map.c + 1 >= pVM->pgm.s.ChunkR3Map.cMax)
4031	{
4032	switch (VMR3GetState(pVM))
4033	{
4034	case VMSTATE_LOADING:
4035	case VMSTATE_SAVING:
4036	{
4037	PVMCPU pVCpu = VMMGetCpu(pVM);
4038	if ( pVCpu
4039	&& pVM->pgm.s.cDeprecatedPageLocks == 0)
4040	{
4041	pgmR3PhysUnmapChunkRendezvous(pVM, pVCpu, NULL);
4042	break;
4043	}
4044	/* fall thru */
4045	}
4046	default:
4047	rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysUnmapChunk, 1, pVM);
4048	AssertRC(rc);
4049	break;
4050	}
4051	}
4052
4053	/*
4054	* Update the tree. We must do this after any unmapping to make sure
4055	* the chunk we're going to return isn't unmapped by accident.
4056	*/
4057	AssertPtr(Req.pvR3);
4058	bool fRc = RTAvlU32Insert(&pVM->pgm.s.ChunkR3Map.pTree, &pChunk->Core);
4059	AssertRelease(fRc);
4060	pVM->pgm.s.ChunkR3Map.c++;
4061	pVM->pgm.s.cMappedChunks++;
4062	}
4063	else
4064	{
4065	/** @todo this may fail because of /proc/sys/vm/max_map_count, so we
4066	* should probably restrict ourselves on linux. */
4067	AssertRC(rc);
4068	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
4069	MMR3HeapFree(pChunk);
4070	#else
4071	MMR3UkHeapFree(pVM, pChunk, MM_TAG_PGM_CHUNK_MAPPING);
4072	#endif
4073	pChunk = NULL;
4074	}
4075
4076	*ppChunk = pChunk;
4077	return rc;
4078	}
4079
4080
4081	/**
4082	* For VMMCALLRING3_PGM_MAP_CHUNK, considered internal.
4083	*
4084	* @returns see pgmR3PhysChunkMap.
4085	* @param pVM Pointer to the VM.
4086	* @param idChunk The chunk to map.
4087	*/
4088	VMMR3DECL(int) PGMR3PhysChunkMap(PVM pVM, uint32_t idChunk)
4089	{
4090	PPGMCHUNKR3MAP pChunk;
4091	int rc;
4092
4093	pgmLock(pVM);
4094	rc = pgmR3PhysChunkMap(pVM, idChunk, &pChunk);
4095	pgmUnlock(pVM);
4096	return rc;
4097	}
4098
4099
4100	/**
4101	* Invalidates the TLB for the ring-3 mapping cache.
4102	*
4103	* @param pVM Pointer to the VM.
4104	*/
4105	VMMR3DECL(void) PGMR3PhysChunkInvalidateTLB(PVM pVM)
4106	{
4107	pgmLock(pVM);
4108	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
4109	{
4110	pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk = NIL_GMM_CHUNKID;
4111	pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk = NULL;
4112	}
4113	/* The page map TLB references chunks, so invalidate that one too. */
4114	pgmPhysInvalidatePageMapTLB(pVM);
4115	pgmUnlock(pVM);
4116	}
4117
4118
4119	/**
4120	* Response to VMMCALLRING3_PGM_ALLOCATE_LARGE_HANDY_PAGE to allocate a large
4121	* (2MB) page for use with a nested paging PDE.
4122	*
4123	* @returns The following VBox status codes.
4124	* @retval VINF_SUCCESS on success.
4125	* @retval VINF_EM_NO_MEMORY if we're out of memory.
4126	*
4127	* @param pVM Pointer to the VM.
4128	* @param GCPhys GC physical start address of the 2 MB range
4129	*/
4130	VMMR3DECL(int) PGMR3PhysAllocateLargeHandyPage(PVM pVM, RTGCPHYS GCPhys)
4131	{
4132	#ifdef PGM_WITH_LARGE_PAGES
4133	uint64_t u64TimeStamp1, u64TimeStamp2;
4134
4135	pgmLock(pVM);
4136
4137	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatAllocLargePage, a);
4138	u64TimeStamp1 = RTTimeMilliTS();
4139	int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_LARGE_HANDY_PAGE, 0, NULL);
4140	u64TimeStamp2 = RTTimeMilliTS();
4141	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatAllocLargePage, a);
4142	if (RT_SUCCESS(rc))
4143	{
4144	Assert(pVM->pgm.s.cLargeHandyPages == 1);
4145
4146	uint32_t idPage = pVM->pgm.s.aLargeHandyPage[0].idPage;
4147	RTHCPHYS HCPhys = pVM->pgm.s.aLargeHandyPage[0].HCPhysGCPhys;
4148
4149	void *pv;
4150
4151	/* Map the large page into our address space.
4152	*
4153	* Note: assuming that within the 2 MB range:
4154	* - GCPhys + PAGE_SIZE = HCPhys + PAGE_SIZE (whole point of this exercise)
4155	* - user space mapping is continuous as well
4156	* - page id (GCPhys) + 1 = page id (GCPhys + PAGE_SIZE)
4157	*/
4158	rc = pgmPhysPageMapByPageID(pVM, idPage, HCPhys, &pv);
4159	AssertLogRelMsg(RT_SUCCESS(rc), ("idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc\n", idPage, HCPhys, rc));
4160
4161	if (RT_SUCCESS(rc))
4162	{
4163	/*
4164	* Clear the pages.
4165	*/
4166	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatClearLargePage, b);
4167	for (unsigned i = 0; i < _2M/PAGE_SIZE; i++)
4168	{
4169	ASMMemZeroPage(pv);
4170
4171	PPGMPAGE pPage;
4172	rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
4173	AssertRC(rc);
4174
4175	Assert(PGM_PAGE_IS_ZERO(pPage));
4176	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatRZPageReplaceZero);
4177	pVM->pgm.s.cZeroPages--;
4178
4179	/*
4180	* Do the PGMPAGE modifications.
4181	*/
4182	pVM->pgm.s.cPrivatePages++;
4183	PGM_PAGE_SET_HCPHYS(pVM, pPage, HCPhys);
4184	PGM_PAGE_SET_PAGEID(pVM, pPage, idPage);
4185	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
4186	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PDE);
4187	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
4188	PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
4189
4190	/* Somewhat dirty assumption that page ids are increasing. */
4191	idPage++;
4192
4193	HCPhys += PAGE_SIZE;
4194	GCPhys += PAGE_SIZE;
4195
4196	pv = (void *)((uintptr_t)pv + PAGE_SIZE);
4197
4198	Log3(("PGMR3PhysAllocateLargePage: idPage=%#x HCPhys=%RGp\n", idPage, HCPhys));
4199	}
4200	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatClearLargePage, b);
4201
4202	/* Flush all TLBs. */
4203	PGM_INVL_ALL_VCPU_TLBS(pVM);
4204	pgmPhysInvalidatePageMapTLB(pVM);
4205	}
4206	pVM->pgm.s.cLargeHandyPages = 0;
4207	}
4208
4209	if (RT_SUCCESS(rc))
4210	{
4211	static uint32_t cTimeOut = 0;
4212	uint64_t u64TimeStampDelta = u64TimeStamp2 - u64TimeStamp1;
4213
4214	if (u64TimeStampDelta > 100)
4215	{
4216	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatLargePageOverflow);
4217	if ( ++cTimeOut > 10
4218	\|\| u64TimeStampDelta > 1000 /* more than one second forces an early retirement from allocating large pages. */)
4219	{
4220	/* If repeated attempts to allocate a large page takes more than 100 ms, then we fall back to normal 4k pages.
4221	* E.g. Vista 64 tries to move memory around, which takes a huge amount of time.
4222	*/
4223	LogRel(("PGMR3PhysAllocateLargePage: allocating large pages takes too long (last attempt %d ms; nr of timeouts %d); DISABLE\n", u64TimeStampDelta, cTimeOut));
4224	PGMSetLargePageUsage(pVM, false);
4225	}
4226	}
4227	else
4228	if (cTimeOut > 0)
4229	cTimeOut--;
4230	}
4231
4232	pgmUnlock(pVM);
4233	return rc;
4234	#else
4235	return VERR_NOT_IMPLEMENTED;
4236	#endif /* PGM_WITH_LARGE_PAGES */
4237	}
4238
4239
4240	/**
4241	* Response to VM_FF_PGM_NEED_HANDY_PAGES and VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES.
4242	*
4243	* This function will also work the VM_FF_PGM_NO_MEMORY force action flag, to
4244	* signal and clear the out of memory condition. When contracted, this API is
4245	* used to try clear the condition when the user wants to resume.
4246	*
4247	* @returns The following VBox status codes.
4248	* @retval VINF_SUCCESS on success. FFs cleared.
4249	* @retval VINF_EM_NO_MEMORY if we're out of memory. The FF is not cleared in
4250	* this case and it gets accompanied by VM_FF_PGM_NO_MEMORY.
4251	*
4252	* @param pVM Pointer to the VM.
4253	*
4254	* @remarks The VINF_EM_NO_MEMORY status is for the benefit of the FF processing
4255	* in EM.cpp and shouldn't be propagated outside TRPM, HWACCM, EM and
4256	* pgmPhysEnsureHandyPage. There is one exception to this in the \#PF
4257	* handler.
4258	*/
4259	VMMR3DECL(int) PGMR3PhysAllocateHandyPages(PVM pVM)
4260	{
4261	pgmLock(pVM);
4262
4263	/*
4264	* Allocate more pages, noting down the index of the first new page.
4265	*/
4266	uint32_t iClear = pVM->pgm.s.cHandyPages;
4267	AssertMsgReturn(iClear <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d", iClear), VERR_PGM_HANDY_PAGE_IPE);
4268	Log(("PGMR3PhysAllocateHandyPages: %d -> %d\n", iClear, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
4269	int rcAlloc = VINF_SUCCESS;
4270	int rcSeed = VINF_SUCCESS;
4271	int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
4272	while (rc == VERR_GMM_SEED_ME)
4273	{
4274	void *pvChunk;
4275	rcAlloc = rc = SUPR3PageAlloc(GMM_CHUNK_SIZE >> PAGE_SHIFT, &pvChunk);
4276	if (RT_SUCCESS(rc))
4277	{
4278	rcSeed = rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_SEED_CHUNK, (uintptr_t)pvChunk, NULL);
4279	if (RT_FAILURE(rc))
4280	SUPR3PageFree(pvChunk, GMM_CHUNK_SIZE >> PAGE_SHIFT);
4281	}
4282	if (RT_SUCCESS(rc))
4283	rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
4284	}
4285
4286	/* todo: we should split this up into an allocate and flush operation. sometimes you want to flush and not allocate more (which will trigger the vm account limit error) */
4287	if ( rc == VERR_GMM_HIT_VM_ACCOUNT_LIMIT
4288	&& pVM->pgm.s.cHandyPages > 0)
4289	{
4290	/* Still handy pages left, so don't panic. */
4291	rc = VINF_SUCCESS;
4292	}
4293
4294	if (RT_SUCCESS(rc))
4295	{
4296	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
4297	Assert(pVM->pgm.s.cHandyPages > 0);
4298	VM_FF_CLEAR(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
4299	VM_FF_CLEAR(pVM, VM_FF_PGM_NO_MEMORY);
4300
4301	#ifdef VBOX_STRICT
4302	uint32_t i;
4303	for (i = iClear; i < pVM->pgm.s.cHandyPages; i++)
4304	if ( pVM->pgm.s.aHandyPages[i].idPage == NIL_GMM_PAGEID
4305	\|\| pVM->pgm.s.aHandyPages[i].idSharedPage != NIL_GMM_PAGEID
4306	\|\| (pVM->pgm.s.aHandyPages[i].HCPhysGCPhys & PAGE_OFFSET_MASK))
4307	break;
4308	if (i != pVM->pgm.s.cHandyPages)
4309	{
4310	RTAssertMsg1Weak(NULL, __LINE__, __FILE__, __FUNCTION__);
4311	RTAssertMsg2Weak("i=%d iClear=%d cHandyPages=%d\n", i, iClear, pVM->pgm.s.cHandyPages);
4312	for (uint32_t j = iClear; j < pVM->pgm.s.cHandyPages; j++)
4313	RTAssertMsg2Add(("%03d: idPage=%d HCPhysGCPhys=%RHp idSharedPage=%d%\n", j,
4314	pVM->pgm.s.aHandyPages[j].idPage,
4315	pVM->pgm.s.aHandyPages[j].HCPhysGCPhys,
4316	pVM->pgm.s.aHandyPages[j].idSharedPage,
4317	j == i ? " <---" : ""));
4318	RTAssertPanic();
4319	}
4320	#endif
4321	/*
4322	* Clear the pages.
4323	*/
4324	while (iClear < pVM->pgm.s.cHandyPages)
4325	{
4326	PGMMPAGEDESC pPage = &pVM->pgm.s.aHandyPages[iClear];
4327	void *pv;
4328	rc = pgmPhysPageMapByPageID(pVM, pPage->idPage, pPage->HCPhysGCPhys, &pv);
4329	AssertLogRelMsgBreak(RT_SUCCESS(rc),
4330	("%u/%u: idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc\n",
4331	iClear, pVM->pgm.s.cHandyPages, pPage->idPage, pPage->HCPhysGCPhys, rc));
4332	ASMMemZeroPage(pv);
4333	iClear++;
4334	Log3(("PGMR3PhysAllocateHandyPages: idPage=%#x HCPhys=%RGp\n", pPage->idPage, pPage->HCPhysGCPhys));
4335	}
4336	}
4337	else
4338	{
4339	uint64_t cAllocPages, cMaxPages, cBalloonPages;
4340
4341	/*
4342	* We should never get here unless there is a genuine shortage of
4343	* memory (or some internal error). Flag the error so the VM can be
4344	* suspended ASAP and the user informed. If we're totally out of
4345	* handy pages we will return failure.
4346	*/
4347	/* Report the failure. */
4348	LogRel(("PGM: Failed to procure handy pages; rc=%Rrc rcAlloc=%Rrc rcSeed=%Rrc cHandyPages=%#x\n"
4349	" cAllPages=%#x cPrivatePages=%#x cSharedPages=%#x cZeroPages=%#x\n",
4350	rc, rcAlloc, rcSeed,
4351	pVM->pgm.s.cHandyPages,
4352	pVM->pgm.s.cAllPages,
4353	pVM->pgm.s.cPrivatePages,
4354	pVM->pgm.s.cSharedPages,
4355	pVM->pgm.s.cZeroPages));
4356
4357	if (GMMR3QueryMemoryStats(pVM, &cAllocPages, &cMaxPages, &cBalloonPages) == VINF_SUCCESS)
4358	{
4359	LogRel(("GMM: Statistics:\n"
4360	" Allocated pages: %RX64\n"
4361	" Maximum pages: %RX64\n"
4362	" Ballooned pages: %RX64\n", cAllocPages, cMaxPages, cBalloonPages));
4363	}
4364
4365	if ( rc != VERR_NO_MEMORY
4366	&& rc != VERR_NO_PHYS_MEMORY
4367	&& rc != VERR_LOCK_FAILED)
4368	{
4369	for (uint32_t i = 0; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
4370	{
4371	LogRel(("PGM: aHandyPages[#%#04x] = {.HCPhysGCPhys=%RHp, .idPage=%#08x, .idSharedPage=%#08x}\n",
4372	i, pVM->pgm.s.aHandyPages[i].HCPhysGCPhys, pVM->pgm.s.aHandyPages[i].idPage,
4373	pVM->pgm.s.aHandyPages[i].idSharedPage));
4374	uint32_t const idPage = pVM->pgm.s.aHandyPages[i].idPage;
4375	if (idPage != NIL_GMM_PAGEID)
4376	{
4377	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
4378	pRam;
4379	pRam = pRam->pNextR3)
4380	{
4381	uint32_t const cPages = pRam->cb >> PAGE_SHIFT;
4382	for (uint32_t iPage = 0; iPage < cPages; iPage++)
4383	if (PGM_PAGE_GET_PAGEID(&pRam->aPages[iPage]) == idPage)
4384	LogRel(("PGM: Used by %RGp %R[pgmpage] (%s)\n",
4385	pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pRam->aPages[iPage], pRam->pszDesc));
4386	}
4387	}
4388	}
4389	}
4390
4391	/* Set the FFs and adjust rc. */
4392	VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
4393	VM_FF_SET(pVM, VM_FF_PGM_NO_MEMORY);
4394	if ( rc == VERR_NO_MEMORY
4395	\|\| rc == VERR_NO_PHYS_MEMORY
4396	\|\| rc == VERR_LOCK_FAILED)
4397	rc = VINF_EM_NO_MEMORY;
4398	}
4399
4400	pgmUnlock(pVM);
4401	return rc;
4402	}
4403
4404
4405	/**
4406	* Frees the specified RAM page and replaces it with the ZERO page.
4407	*
4408	* This is used by ballooning, remapping MMIO2, RAM reset and state loading.
4409	*
4410	* @param pVM Pointer to the VM.
4411	* @param pReq Pointer to the request.
4412	* @param pcPendingPages Where the number of pages waiting to be freed are
4413	* kept. This will normally be incremented.
4414	* @param pPage Pointer to the page structure.
4415	* @param GCPhys The guest physical address of the page, if applicable.
4416	*
4417	* @remarks The caller must own the PGM lock.
4418	*/
4419	int pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys)
4420	{
4421	/*
4422	* Assert sanity.
4423	*/
4424	PGM_LOCK_ASSERT_OWNER(pVM);
4425	if (RT_UNLIKELY( PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
4426	&& PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_ROM_SHADOW))
4427	{
4428	AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
4429	return VMSetError(pVM, VERR_PGM_PHYS_NOT_RAM, RT_SRC_POS, "GCPhys=%RGp type=%d", GCPhys, PGM_PAGE_GET_TYPE(pPage));
4430	}
4431
4432	/** @todo What about ballooning of large pages??! */
4433	Assert( PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE
4434	&& PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE_DISABLED);
4435
4436	if ( PGM_PAGE_IS_ZERO(pPage)
4437	\|\| PGM_PAGE_IS_BALLOONED(pPage))
4438	return VINF_SUCCESS;
4439
4440	const uint32_t idPage = PGM_PAGE_GET_PAGEID(pPage);
4441	Log3(("pgmPhysFreePage: idPage=%#x GCPhys=%RGp pPage=%R[pgmpage]\n", idPage, GCPhys, pPage));
4442	if (RT_UNLIKELY( idPage == NIL_GMM_PAGEID
4443	\|\| idPage > GMM_PAGEID_LAST
4444	\|\| PGM_PAGE_GET_CHUNKID(pPage) == NIL_GMM_CHUNKID))
4445	{
4446	AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
4447	return VMSetError(pVM, VERR_PGM_PHYS_INVALID_PAGE_ID, RT_SRC_POS, "GCPhys=%RGp idPage=%#x", GCPhys, pPage);
4448	}
4449
4450	/* update page count stats. */
4451	if (PGM_PAGE_IS_SHARED(pPage))
4452	pVM->pgm.s.cSharedPages--;
4453	else
4454	pVM->pgm.s.cPrivatePages--;
4455	pVM->pgm.s.cZeroPages++;
4456
4457	/* Deal with write monitored pages. */
4458	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
4459	{
4460	PGM_PAGE_SET_WRITTEN_TO(pVM, pPage);
4461	pVM->pgm.s.cWrittenToPages++;
4462	}
4463
4464	/*
4465	* pPage = ZERO page.
4466	*/
4467	PGM_PAGE_SET_HCPHYS(pVM, pPage, pVM->pgm.s.HCPhysZeroPg);
4468	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
4469	PGM_PAGE_SET_PAGEID(pVM, pPage, NIL_GMM_PAGEID);
4470	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
4471	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
4472	PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
4473
4474	/* Flush physical page map TLB entry. */
4475	pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
4476
4477	/*
4478	* Make sure it's not in the handy page array.
4479	*/
4480	for (uint32_t i = pVM->pgm.s.cHandyPages; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
4481	{
4482	if (pVM->pgm.s.aHandyPages[i].idPage == idPage)
4483	{
4484	pVM->pgm.s.aHandyPages[i].idPage = NIL_GMM_PAGEID;
4485	break;
4486	}
4487	if (pVM->pgm.s.aHandyPages[i].idSharedPage == idPage)
4488	{
4489	pVM->pgm.s.aHandyPages[i].idSharedPage = NIL_GMM_PAGEID;
4490	break;
4491	}
4492	}
4493
4494	/*
4495	* Push it onto the page array.
4496	*/
4497	uint32_t iPage = *pcPendingPages;
4498	Assert(iPage < PGMPHYS_FREE_PAGE_BATCH_SIZE);
4499	*pcPendingPages += 1;
4500
4501	pReq->aPages[iPage].idPage = idPage;
4502
4503	if (iPage + 1 < PGMPHYS_FREE_PAGE_BATCH_SIZE)
4504	return VINF_SUCCESS;
4505
4506	/*
4507	* Flush the pages.
4508	*/
4509	int rc = GMMR3FreePagesPerform(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE);
4510	if (RT_SUCCESS(rc))
4511	{
4512	GMMR3FreePagesRePrep(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
4513	*pcPendingPages = 0;
4514	}
4515	return rc;
4516	}
4517
4518
4519	/**
4520	* Converts a GC physical address to a HC ring-3 pointer, with some
4521	* additional checks.
4522	*
4523	* @returns VBox status code.
4524	* @retval VINF_SUCCESS on success.
4525	* @retval VINF_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
4526	* access handler of some kind.
4527	* @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
4528	* accesses or is odd in any way.
4529	* @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
4530	*
4531	* @param pVM Pointer to the VM.
4532	* @param GCPhys The GC physical address to convert. Since this is only
4533	* used for filling the REM TLB, the A20 mask must be
4534	* applied before calling this API.
4535	* @param fWritable Whether write access is required.
4536	* @param ppv Where to store the pointer corresponding to GCPhys on
4537	* success.
4538	*/
4539	VMMR3DECL(int) PGMR3PhysTlbGCPhys2Ptr(PVM pVM, RTGCPHYS GCPhys, bool fWritable, void **ppv)
4540	{
4541	pgmLock(pVM);
4542	PGM_A20_ASSERT_MASKED(VMMGetCpu(pVM), GCPhys);
4543
4544	PPGMRAMRANGE pRam;
4545	PPGMPAGE pPage;
4546	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
4547	if (RT_SUCCESS(rc))
4548	{
4549	if (PGM_PAGE_IS_BALLOONED(pPage))
4550	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
4551	else if (!PGM_PAGE_HAS_ANY_HANDLERS(pPage))
4552	rc = VINF_SUCCESS;
4553	else
4554	{
4555	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
4556	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4557	else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
4558	{
4559	/** @todo Handle TLB loads of virtual handlers so ./test.sh can be made to work
4560	* in -norawr0 mode. */
4561	if (fWritable)
4562	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
4563	}
4564	else
4565	{
4566	/* Temporarily disabled physical handler(s), since the recompiler
4567	doesn't get notified when it's reset we'll have to pretend it's
4568	operating normally. */
4569	if (pgmHandlerPhysicalIsAll(pVM, GCPhys))
4570	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4571	else
4572	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
4573	}
4574	}
4575	if (RT_SUCCESS(rc))
4576	{
4577	int rc2;
4578
4579	/* Make sure what we return is writable. */
4580	if (fWritable)
4581	switch (PGM_PAGE_GET_STATE(pPage))
4582	{
4583	case PGM_PAGE_STATE_ALLOCATED:
4584	break;
4585	case PGM_PAGE_STATE_BALLOONED:
4586	AssertFailed();
4587	break;
4588	case PGM_PAGE_STATE_ZERO:
4589	case PGM_PAGE_STATE_SHARED:
4590	if (rc == VINF_PGM_PHYS_TLB_CATCH_WRITE)
4591	break;
4592	case PGM_PAGE_STATE_WRITE_MONITORED:
4593	rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
4594	AssertLogRelRCReturn(rc2, rc2);
4595	break;
4596	}
4597
4598	/* Get a ring-3 mapping of the address. */
4599	PPGMPAGER3MAPTLBE pTlbe;
4600	rc2 = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
4601	AssertLogRelRCReturn(rc2, rc2);
4602	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
4603	/** @todo mapping/locking hell; this isn't horribly efficient since
4604	* pgmPhysPageLoadIntoTlb will repeat the lookup we've done here. */
4605
4606	Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage] ppv=%p\n", GCPhys, rc, pPage, ppv));
4607	}
4608	else
4609	Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage]\n", GCPhys, rc, pPage));
4610
4611	/* else: handler catching all access, no pointer returned. */
4612	}
4613	else
4614	rc = VERR_PGM_PHYS_TLB_UNASSIGNED;
4615
4616	pgmUnlock(pVM);
4617	return rc;
4618	}
4619

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR3/PGMPhys.cpp@ 41836

Download in other formats: