PGMPhys.cpp@ 43363

Last change on this file since 43363 was 43047, checked in by vboxsync, 12 years ago
VMM: Must flush changes pending in the handy page array before freeing memory. There may be requests to free shared pages in there, esp. in a guest shutdown scenario.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 166.7 KB

Line
1	/* $Id: PGMPhys.cpp 43047 2012-08-28 14:47:29Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor, Physical Memory Addressing.
4	*/
5
6	/*
7	* Copyright (C) 2006-2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_PGM_PHYS
23	#include <VBox/vmm/pgm.h>
24	#include <VBox/vmm/iom.h>
25	#include <VBox/vmm/mm.h>
26	#include <VBox/vmm/stam.h>
27	#ifdef VBOX_WITH_REM
28	# include <VBox/vmm/rem.h>
29	#endif
30	#include <VBox/vmm/pdmdev.h>
31	#include "PGMInternal.h"
32	#include <VBox/vmm/vm.h>
33	#include "PGMInline.h"
34	#include <VBox/sup.h>
35	#include <VBox/param.h>
36	#include <VBox/err.h>
37	#include <VBox/log.h>
38	#include <iprt/assert.h>
39	#include <iprt/alloc.h>
40	#include <iprt/asm.h>
41	#ifdef VBOX_STRICT
42	# include <iprt/crc.h>
43	#endif
44	#include <iprt/thread.h>
45	#include <iprt/string.h>
46	#include <iprt/system.h>
47
48
49	/*******************************************************************************
50	* Defined Constants And Macros *
51	*******************************************************************************/
52	/** The number of pages to free in one batch. */
53	#define PGMPHYS_FREE_PAGE_BATCH_SIZE 128
54
55
56	/*******************************************************************************
57	* Internal Functions *
58	*******************************************************************************/
59	static DECLCALLBACK(int) pgmR3PhysRomWriteHandler(PVM pVM, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser);
60
61
62	/*
63	* PGMR3PhysReadU8-64
64	* PGMR3PhysWriteU8-64
65	*/
66	#define PGMPHYSFN_READNAME PGMR3PhysReadU8
67	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU8
68	#define PGMPHYS_DATASIZE 1
69	#define PGMPHYS_DATATYPE uint8_t
70	#include "PGMPhysRWTmpl.h"
71
72	#define PGMPHYSFN_READNAME PGMR3PhysReadU16
73	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU16
74	#define PGMPHYS_DATASIZE 2
75	#define PGMPHYS_DATATYPE uint16_t
76	#include "PGMPhysRWTmpl.h"
77
78	#define PGMPHYSFN_READNAME PGMR3PhysReadU32
79	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU32
80	#define PGMPHYS_DATASIZE 4
81	#define PGMPHYS_DATATYPE uint32_t
82	#include "PGMPhysRWTmpl.h"
83
84	#define PGMPHYSFN_READNAME PGMR3PhysReadU64
85	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU64
86	#define PGMPHYS_DATASIZE 8
87	#define PGMPHYS_DATATYPE uint64_t
88	#include "PGMPhysRWTmpl.h"
89
90
91	/**
92	* EMT worker for PGMR3PhysReadExternal.
93	*/
94	static DECLCALLBACK(int) pgmR3PhysReadExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, void *pvBuf, size_t cbRead)
95	{
96	PGMPhysRead(pVM, *pGCPhys, pvBuf, cbRead);
97	return VINF_SUCCESS;
98	}
99
100
101	/**
102	* Read from physical memory, external users.
103	*
104	* @returns VBox status code.
105	* @retval VINF_SUCCESS.
106	*
107	* @param pVM Pointer to the VM.
108	* @param GCPhys Physical address to read from.
109	* @param pvBuf Where to read into.
110	* @param cbRead How many bytes to read.
111	*
112	* @thread Any but EMTs.
113	*/
114	VMMR3DECL(int) PGMR3PhysReadExternal(PVM pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead)
115	{
116	VM_ASSERT_OTHER_THREAD(pVM);
117
118	AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
119	LogFlow(("PGMR3PhysReadExternal: %RGp %d\n", GCPhys, cbRead));
120
121	pgmLock(pVM);
122
123	/*
124	* Copy loop on ram ranges.
125	*/
126	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
127	for (;;)
128	{
129	/* Inside range or not? */
130	if (pRam && GCPhys >= pRam->GCPhys)
131	{
132	/*
133	* Must work our way thru this page by page.
134	*/
135	RTGCPHYS off = GCPhys - pRam->GCPhys;
136	while (off < pRam->cb)
137	{
138	unsigned iPage = off >> PAGE_SHIFT;
139	PPGMPAGE pPage = &pRam->aPages[iPage];
140
141	/*
142	* If the page has an ALL access handler, we'll have to
143	* delegate the job to EMT.
144	*/
145	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
146	{
147	pgmUnlock(pVM);
148
149	return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysReadExternalEMT, 4,
150	pVM, &GCPhys, pvBuf, cbRead);
151	}
152	Assert(!PGM_PAGE_IS_MMIO(pPage));
153
154	/*
155	* Simple stuff, go ahead.
156	*/
157	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
158	if (cb > cbRead)
159	cb = cbRead;
160	PGMPAGEMAPLOCK PgMpLck;
161	const void *pvSrc;
162	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc, &PgMpLck);
163	if (RT_SUCCESS(rc))
164	{
165	memcpy(pvBuf, pvSrc, cb);
166	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
167	}
168	else
169	{
170	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
171	pRam->GCPhys + off, pPage, rc));
172	memset(pvBuf, 0xff, cb);
173	}
174
175	/* next page */
176	if (cb >= cbRead)
177	{
178	pgmUnlock(pVM);
179	return VINF_SUCCESS;
180	}
181	cbRead -= cb;
182	off += cb;
183	GCPhys += cb;
184	pvBuf = (char *)pvBuf + cb;
185	} /* walk pages in ram range. */
186	}
187	else
188	{
189	LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
190
191	/*
192	* Unassigned address space.
193	*/
194	size_t cb = pRam ? pRam->GCPhys - GCPhys : ~(size_t)0;
195	if (cb >= cbRead)
196	{
197	memset(pvBuf, 0xff, cbRead);
198	break;
199	}
200	memset(pvBuf, 0xff, cb);
201
202	cbRead -= cb;
203	pvBuf = (char *)pvBuf + cb;
204	GCPhys += cb;
205	}
206
207	/* Advance range if necessary. */
208	while (pRam && GCPhys > pRam->GCPhysLast)
209	pRam = pRam->CTX_SUFF(pNext);
210	} /* Ram range walk */
211
212	pgmUnlock(pVM);
213
214	return VINF_SUCCESS;
215	}
216
217
218	/**
219	* EMT worker for PGMR3PhysWriteExternal.
220	*/
221	static DECLCALLBACK(int) pgmR3PhysWriteExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, const void *pvBuf, size_t cbWrite)
222	{
223	/** @todo VERR_EM_NO_MEMORY */
224	PGMPhysWrite(pVM, *pGCPhys, pvBuf, cbWrite);
225	return VINF_SUCCESS;
226	}
227
228
229	/**
230	* Write to physical memory, external users.
231	*
232	* @returns VBox status code.
233	* @retval VINF_SUCCESS.
234	* @retval VERR_EM_NO_MEMORY.
235	*
236	* @param pVM Pointer to the VM.
237	* @param GCPhys Physical address to write to.
238	* @param pvBuf What to write.
239	* @param cbWrite How many bytes to write.
240	* @param pszWho Who is writing. For tracking down who is writing
241	* after we've saved the state.
242	*
243	* @thread Any but EMTs.
244	*/
245	VMMDECL(int) PGMR3PhysWriteExternal(PVM pVM, RTGCPHYS GCPhys, const void pvBuf, size_t cbWrite, const char pszWho)
246	{
247	VM_ASSERT_OTHER_THREAD(pVM);
248
249	AssertMsg(!pVM->pgm.s.fNoMorePhysWrites,
250	("Calling PGMR3PhysWriteExternal after pgmR3Save()! GCPhys=%RGp cbWrite=%#x pszWho=%s\n",
251	GCPhys, cbWrite, pszWho));
252	AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
253	LogFlow(("PGMR3PhysWriteExternal: %RGp %d\n", GCPhys, cbWrite));
254
255	pgmLock(pVM);
256
257	/*
258	* Copy loop on ram ranges, stop when we hit something difficult.
259	*/
260	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
261	for (;;)
262	{
263	/* Inside range or not? */
264	if (pRam && GCPhys >= pRam->GCPhys)
265	{
266	/*
267	* Must work our way thru this page by page.
268	*/
269	RTGCPTR off = GCPhys - pRam->GCPhys;
270	while (off < pRam->cb)
271	{
272	RTGCPTR iPage = off >> PAGE_SHIFT;
273	PPGMPAGE pPage = &pRam->aPages[iPage];
274
275	/*
276	* Is the page problematic, we have to do the work on the EMT.
277	*
278	* Allocating writable pages and access handlers are
279	* problematic, write monitored pages are simple and can be
280	* dealt with here.
281	*/
282	if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
283	\|\| PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
284	{
285	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
286	&& !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
287	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
288	else
289	{
290	pgmUnlock(pVM);
291
292	return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysWriteExternalEMT, 4,
293	pVM, &GCPhys, pvBuf, cbWrite);
294	}
295	}
296	Assert(!PGM_PAGE_IS_MMIO(pPage));
297
298	/*
299	* Simple stuff, go ahead.
300	*/
301	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
302	if (cb > cbWrite)
303	cb = cbWrite;
304	PGMPAGEMAPLOCK PgMpLck;
305	void *pvDst;
306	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst, &PgMpLck);
307	if (RT_SUCCESS(rc))
308	{
309	memcpy(pvDst, pvBuf, cb);
310	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
311	}
312	else
313	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
314	pRam->GCPhys + off, pPage, rc));
315
316	/* next page */
317	if (cb >= cbWrite)
318	{
319	pgmUnlock(pVM);
320	return VINF_SUCCESS;
321	}
322
323	cbWrite -= cb;
324	off += cb;
325	GCPhys += cb;
326	pvBuf = (const char *)pvBuf + cb;
327	} /* walk pages in ram range */
328	}
329	else
330	{
331	/*
332	* Unassigned address space, skip it.
333	*/
334	if (!pRam)
335	break;
336	size_t cb = pRam->GCPhys - GCPhys;
337	if (cb >= cbWrite)
338	break;
339	cbWrite -= cb;
340	pvBuf = (const char *)pvBuf + cb;
341	GCPhys += cb;
342	}
343
344	/* Advance range if necessary. */
345	while (pRam && GCPhys > pRam->GCPhysLast)
346	pRam = pRam->CTX_SUFF(pNext);
347	} /* Ram range walk */
348
349	pgmUnlock(pVM);
350	return VINF_SUCCESS;
351	}
352
353
354	/**
355	* VMR3ReqCall worker for PGMR3PhysGCPhys2CCPtrExternal to make pages writable.
356	*
357	* @returns see PGMR3PhysGCPhys2CCPtrExternal
358	* @param pVM Pointer to the VM.
359	* @param pGCPhys Pointer to the guest physical address.
360	* @param ppv Where to store the mapping address.
361	* @param pLock Where to store the lock.
362	*/
363	static DECLCALLBACK(int) pgmR3PhysGCPhys2CCPtrDelegated(PVM pVM, PRTGCPHYS pGCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
364	{
365	/*
366	* Just hand it to PGMPhysGCPhys2CCPtr and check that it's not a page with
367	* an access handler after it succeeds.
368	*/
369	int rc = pgmLock(pVM);
370	AssertRCReturn(rc, rc);
371
372	rc = PGMPhysGCPhys2CCPtr(pVM, *pGCPhys, ppv, pLock);
373	if (RT_SUCCESS(rc))
374	{
375	PPGMPAGEMAPTLBE pTlbe;
376	int rc2 = pgmPhysPageQueryTlbe(pVM, *pGCPhys, &pTlbe);
377	AssertFatalRC(rc2);
378	PPGMPAGE pPage = pTlbe->pPage;
379	if (PGM_PAGE_IS_MMIO(pPage))
380	{
381	PGMPhysReleasePageMappingLock(pVM, pLock);
382	rc = VERR_PGM_PHYS_PAGE_RESERVED;
383	}
384	else if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
385	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
386	\|\| pgmPoolIsDirtyPage(pVM, *pGCPhys)
387	#endif
388	)
389	{
390	/* We must flush any corresponding pgm pool page here, otherwise we'll
391	* not be informed about writes and keep bogus gst->shw mappings around.
392	*/
393	pgmPoolFlushPageByGCPhys(pVM, *pGCPhys);
394	Assert(!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage));
395	/** @todo r=bird: return VERR_PGM_PHYS_PAGE_RESERVED here if it still has
396	* active handlers, see the PGMR3PhysGCPhys2CCPtrExternal docs. */
397	}
398	}
399
400	pgmUnlock(pVM);
401	return rc;
402	}
403
404
405	/**
406	* Requests the mapping of a guest page into ring-3, external threads.
407	*
408	* When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
409	* release it.
410	*
411	* This API will assume your intention is to write to the page, and will
412	* therefore replace shared and zero pages. If you do not intend to modify the
413	* page, use the PGMR3PhysGCPhys2CCPtrReadOnlyExternal() API.
414	*
415	* @returns VBox status code.
416	* @retval VINF_SUCCESS on success.
417	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
418	* backing or if the page has any active access handlers. The caller
419	* must fall back on using PGMR3PhysWriteExternal.
420	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
421	*
422	* @param pVM Pointer to the VM.
423	* @param GCPhys The guest physical address of the page that should be mapped.
424	* @param ppv Where to store the address corresponding to GCPhys.
425	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
426	*
427	* @remark Avoid calling this API from within critical sections (other than the
428	* PGM one) because of the deadlock risk when we have to delegating the
429	* task to an EMT.
430	* @thread Any.
431	*/
432	VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrExternal(PVM pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
433	{
434	AssertPtr(ppv);
435	AssertPtr(pLock);
436
437	Assert(VM_IS_EMT(pVM) \|\| !PGMIsLockOwner(pVM));
438
439	int rc = pgmLock(pVM);
440	AssertRCReturn(rc, rc);
441
442	/*
443	* Query the Physical TLB entry for the page (may fail).
444	*/
445	PPGMPAGEMAPTLBE pTlbe;
446	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
447	if (RT_SUCCESS(rc))
448	{
449	PPGMPAGE pPage = pTlbe->pPage;
450	if (PGM_PAGE_IS_MMIO(pPage))
451	rc = VERR_PGM_PHYS_PAGE_RESERVED;
452	else
453	{
454	/*
455	* If the page is shared, the zero page, or being write monitored
456	* it must be converted to an page that's writable if possible.
457	* We can only deal with write monitored pages here, the rest have
458	* to be on an EMT.
459	*/
460	if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
461	\|\| PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
462	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
463	\|\| pgmPoolIsDirtyPage(pVM, GCPhys)
464	#endif
465	)
466	{
467	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
468	&& !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
469	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
470	&& !pgmPoolIsDirtyPage(pVM, GCPhys)
471	#endif
472	)
473	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
474	else
475	{
476	pgmUnlock(pVM);
477
478	return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysGCPhys2CCPtrDelegated, 4,
479	pVM, &GCPhys, ppv, pLock);
480	}
481	}
482
483	/*
484	* Now, just perform the locking and calculate the return address.
485	*/
486	PPGMPAGEMAP pMap = pTlbe->pMap;
487	if (pMap)
488	pMap->cRefs++;
489
490	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
491	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
492	{
493	if (cLocks == 0)
494	pVM->pgm.s.cWriteLockedPages++;
495	PGM_PAGE_INC_WRITE_LOCKS(pPage);
496	}
497	else if (cLocks != PGM_PAGE_GET_WRITE_LOCKS(pPage))
498	{
499	PGM_PAGE_INC_WRITE_LOCKS(pPage);
500	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent write locked state!\n", GCPhys, pPage));
501	if (pMap)
502	pMap->cRefs++; /* Extra ref to prevent it from going away. */
503	}
504
505	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
506	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_WRITE;
507	pLock->pvMap = pMap;
508	}
509	}
510
511	pgmUnlock(pVM);
512	return rc;
513	}
514
515
516	/**
517	* Requests the mapping of a guest page into ring-3, external threads.
518	*
519	* When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
520	* release it.
521	*
522	* @returns VBox status code.
523	* @retval VINF_SUCCESS on success.
524	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
525	* backing or if the page as an active ALL access handler. The caller
526	* must fall back on using PGMPhysRead.
527	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
528	*
529	* @param pVM Pointer to the VM.
530	* @param GCPhys The guest physical address of the page that should be mapped.
531	* @param ppv Where to store the address corresponding to GCPhys.
532	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
533	*
534	* @remark Avoid calling this API from within critical sections (other than
535	* the PGM one) because of the deadlock risk.
536	* @thread Any.
537	*/
538	VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrReadOnlyExternal(PVM pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
539	{
540	int rc = pgmLock(pVM);
541	AssertRCReturn(rc, rc);
542
543	/*
544	* Query the Physical TLB entry for the page (may fail).
545	*/
546	PPGMPAGEMAPTLBE pTlbe;
547	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
548	if (RT_SUCCESS(rc))
549	{
550	PPGMPAGE pPage = pTlbe->pPage;
551	#if 1
552	/* MMIO pages doesn't have any readable backing. */
553	if (PGM_PAGE_IS_MMIO(pPage))
554	rc = VERR_PGM_PHYS_PAGE_RESERVED;
555	#else
556	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
557	rc = VERR_PGM_PHYS_PAGE_RESERVED;
558	#endif
559	else
560	{
561	/*
562	* Now, just perform the locking and calculate the return address.
563	*/
564	PPGMPAGEMAP pMap = pTlbe->pMap;
565	if (pMap)
566	pMap->cRefs++;
567
568	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
569	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
570	{
571	if (cLocks == 0)
572	pVM->pgm.s.cReadLockedPages++;
573	PGM_PAGE_INC_READ_LOCKS(pPage);
574	}
575	else if (cLocks != PGM_PAGE_GET_READ_LOCKS(pPage))
576	{
577	PGM_PAGE_INC_READ_LOCKS(pPage);
578	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent readonly locked state!\n", GCPhys, pPage));
579	if (pMap)
580	pMap->cRefs++; /* Extra ref to prevent it from going away. */
581	}
582
583	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
584	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_READ;
585	pLock->pvMap = pMap;
586	}
587	}
588
589	pgmUnlock(pVM);
590	return rc;
591	}
592
593
594	#define MAKE_LEAF(a_pNode) \
595	do { \
596	(a_pNode)->pLeftR3 = NIL_RTR3PTR; \
597	(a_pNode)->pRightR3 = NIL_RTR3PTR; \
598	(a_pNode)->pLeftR0 = NIL_RTR0PTR; \
599	(a_pNode)->pRightR0 = NIL_RTR0PTR; \
600	(a_pNode)->pLeftRC = NIL_RTRCPTR; \
601	(a_pNode)->pRightRC = NIL_RTRCPTR; \
602	} while (0)
603
604	#define INSERT_LEFT(a_pParent, a_pNode) \
605	do { \
606	(a_pParent)->pLeftR3 = (a_pNode); \
607	(a_pParent)->pLeftR0 = (a_pNode)->pSelfR0; \
608	(a_pParent)->pLeftRC = (a_pNode)->pSelfRC; \
609	} while (0)
610	#define INSERT_RIGHT(a_pParent, a_pNode) \
611	do { \
612	(a_pParent)->pRightR3 = (a_pNode); \
613	(a_pParent)->pRightR0 = (a_pNode)->pSelfR0; \
614	(a_pParent)->pRightRC = (a_pNode)->pSelfRC; \
615	} while (0)
616
617
618	/**
619	* Recursive tree builder.
620	*
621	* @param ppRam Pointer to the iterator variable.
622	* @param iHeight The hight about normal leaf nodes. Inserts a leaf
623	* node if 0.
624	*/
625	static PPGMRAMRANGE pgmR3PhysRebuildRamRangeSearchTreesRecursively(PPGMRAMRANGE *ppRam, int iDepth)
626	{
627	PPGMRAMRANGE pRam;
628	if (iDepth <= 0)
629	{
630	/*
631	* Leaf node.
632	*/
633	pRam = *ppRam;
634	if (pRam)
635	{
636	*ppRam = pRam->pNextR3;
637	MAKE_LEAF(pRam);
638	}
639	}
640	else
641	{
642
643	/*
644	* Intermediate node.
645	*/
646	PPGMRAMRANGE pLeft = pgmR3PhysRebuildRamRangeSearchTreesRecursively(ppRam, iDepth - 1);
647
648	pRam = *ppRam;
649	if (!pRam)
650	return pLeft;
651	*ppRam = pRam->pNextR3;
652	MAKE_LEAF(pRam);
653	INSERT_LEFT(pRam, pLeft);
654
655	PPGMRAMRANGE pRight = pgmR3PhysRebuildRamRangeSearchTreesRecursively(ppRam, iDepth - 1);
656	if (pRight)
657	INSERT_RIGHT(pRam, pRight);
658	}
659	return pRam;
660	}
661
662
663	/**
664	* Rebuilds the RAM range search trees.
665	*
666	* @param pVM Pointer to the VM.
667	*/
668	static void pgmR3PhysRebuildRamRangeSearchTrees(PVM pVM)
669	{
670
671	/*
672	* Create the reasonably balanced tree in a sequential fashion.
673	* For simplicity (laziness) we use standard recursion here.
674	*/
675	int iDepth = 0;
676	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
677	PPGMRAMRANGE pRoot = pgmR3PhysRebuildRamRangeSearchTreesRecursively(&pRam, 0);
678	while (pRam)
679	{
680	PPGMRAMRANGE pLeft = pRoot;
681
682	pRoot = pRam;
683	pRam = pRam->pNextR3;
684	MAKE_LEAF(pRoot);
685	INSERT_LEFT(pRoot, pLeft);
686
687	PPGMRAMRANGE pRight = pgmR3PhysRebuildRamRangeSearchTreesRecursively(&pRam, iDepth);
688	if (pRight)
689	INSERT_RIGHT(pRoot, pRight);
690	/** @todo else: rotate the tree. */
691
692	iDepth++;
693	}
694
695	pVM->pgm.s.pRamRangeTreeR3 = pRoot;
696	pVM->pgm.s.pRamRangeTreeR0 = pRoot ? pRoot->pSelfR0 : NIL_RTR0PTR;
697	pVM->pgm.s.pRamRangeTreeRC = pRoot ? pRoot->pSelfRC : NIL_RTRCPTR;
698
699	#ifdef VBOX_STRICT
700	/*
701	* Verify that the above code works.
702	*/
703	unsigned cRanges = 0;
704	for (pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
705	cRanges++;
706	Assert(cRanges > 0);
707
708	unsigned cMaxDepth = ASMBitLastSetU32(cRanges);
709	if ((1U << cMaxDepth) < cRanges)
710	cMaxDepth++;
711
712	for (pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
713	{
714	unsigned cDepth = 0;
715	PPGMRAMRANGE pRam2 = pVM->pgm.s.pRamRangeTreeR3;
716	for (;;)
717	{
718	if (pRam == pRam2)
719	break;
720	Assert(pRam2);
721	if (pRam->GCPhys < pRam2->GCPhys)
722	pRam2 = pRam2->pLeftR3;
723	else
724	pRam2 = pRam2->pRightR3;
725	}
726	AssertMsg(cDepth <= cMaxDepth, ("cDepth=%d cMaxDepth=%d\n", cDepth, cMaxDepth));
727	}
728	#endif /* VBOX_STRICT */
729	}
730
731	#undef MAKE_LEAF
732	#undef INSERT_LEFT
733	#undef INSERT_RIGHT
734
735	/**
736	* Relinks the RAM ranges using the pSelfRC and pSelfR0 pointers.
737	*
738	* Called when anything was relocated.
739	*
740	* @param pVM Pointer to the VM.
741	*/
742	void pgmR3PhysRelinkRamRanges(PVM pVM)
743	{
744	PPGMRAMRANGE pCur;
745
746	#ifdef VBOX_STRICT
747	for (pCur = pVM->pgm.s.pRamRangesXR3; pCur; pCur = pCur->pNextR3)
748	{
749	Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pCur->pSelfR0 == MMHyperCCToR0(pVM, pCur));
750	Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pCur->pSelfRC == MMHyperCCToRC(pVM, pCur));
751	Assert((pCur->GCPhys & PAGE_OFFSET_MASK) == 0);
752	Assert((pCur->GCPhysLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
753	Assert((pCur->cb & PAGE_OFFSET_MASK) == 0);
754	Assert(pCur->cb == pCur->GCPhysLast - pCur->GCPhys + 1);
755	for (PPGMRAMRANGE pCur2 = pVM->pgm.s.pRamRangesXR3; pCur2; pCur2 = pCur2->pNextR3)
756	Assert( pCur2 == pCur
757	\|\| strcmp(pCur2->pszDesc, pCur->pszDesc)); /** @todo fix MMIO ranges!! */
758	}
759	#endif
760
761	pCur = pVM->pgm.s.pRamRangesXR3;
762	if (pCur)
763	{
764	pVM->pgm.s.pRamRangesXR0 = pCur->pSelfR0;
765	pVM->pgm.s.pRamRangesXRC = pCur->pSelfRC;
766
767	for (; pCur->pNextR3; pCur = pCur->pNextR3)
768	{
769	pCur->pNextR0 = pCur->pNextR3->pSelfR0;
770	pCur->pNextRC = pCur->pNextR3->pSelfRC;
771	}
772
773	Assert(pCur->pNextR0 == NIL_RTR0PTR);
774	Assert(pCur->pNextRC == NIL_RTRCPTR);
775	}
776	else
777	{
778	Assert(pVM->pgm.s.pRamRangesXR0 == NIL_RTR0PTR);
779	Assert(pVM->pgm.s.pRamRangesXRC == NIL_RTRCPTR);
780	}
781	ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
782
783	pgmR3PhysRebuildRamRangeSearchTrees(pVM);
784	}
785
786
787	/**
788	* Links a new RAM range into the list.
789	*
790	* @param pVM Pointer to the VM.
791	* @param pNew Pointer to the new list entry.
792	* @param pPrev Pointer to the previous list entry. If NULL, insert as head.
793	*/
794	static void pgmR3PhysLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, PPGMRAMRANGE pPrev)
795	{
796	AssertMsg(pNew->pszDesc, ("%RGp-%RGp\n", pNew->GCPhys, pNew->GCPhysLast));
797	Assert((pNew->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pNew->pSelfR0 == MMHyperCCToR0(pVM, pNew));
798	Assert((pNew->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pNew->pSelfRC == MMHyperCCToRC(pVM, pNew));
799
800	pgmLock(pVM);
801
802	PPGMRAMRANGE pRam = pPrev ? pPrev->pNextR3 : pVM->pgm.s.pRamRangesXR3;
803	pNew->pNextR3 = pRam;
804	pNew->pNextR0 = pRam ? pRam->pSelfR0 : NIL_RTR0PTR;
805	pNew->pNextRC = pRam ? pRam->pSelfRC : NIL_RTRCPTR;
806
807	if (pPrev)
808	{
809	pPrev->pNextR3 = pNew;
810	pPrev->pNextR0 = pNew->pSelfR0;
811	pPrev->pNextRC = pNew->pSelfRC;
812	}
813	else
814	{
815	pVM->pgm.s.pRamRangesXR3 = pNew;
816	pVM->pgm.s.pRamRangesXR0 = pNew->pSelfR0;
817	pVM->pgm.s.pRamRangesXRC = pNew->pSelfRC;
818	}
819	ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
820
821	pgmR3PhysRebuildRamRangeSearchTrees(pVM);
822	pgmUnlock(pVM);
823	}
824
825
826	/**
827	* Unlink an existing RAM range from the list.
828	*
829	* @param pVM Pointer to the VM.
830	* @param pRam Pointer to the new list entry.
831	* @param pPrev Pointer to the previous list entry. If NULL, insert as head.
832	*/
833	static void pgmR3PhysUnlinkRamRange2(PVM pVM, PPGMRAMRANGE pRam, PPGMRAMRANGE pPrev)
834	{
835	Assert(pPrev ? pPrev->pNextR3 == pRam : pVM->pgm.s.pRamRangesXR3 == pRam);
836	Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pRam->pSelfR0 == MMHyperCCToR0(pVM, pRam));
837	Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pRam->pSelfRC == MMHyperCCToRC(pVM, pRam));
838
839	pgmLock(pVM);
840
841	PPGMRAMRANGE pNext = pRam->pNextR3;
842	if (pPrev)
843	{
844	pPrev->pNextR3 = pNext;
845	pPrev->pNextR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
846	pPrev->pNextRC = pNext ? pNext->pSelfRC : NIL_RTRCPTR;
847	}
848	else
849	{
850	Assert(pVM->pgm.s.pRamRangesXR3 == pRam);
851	pVM->pgm.s.pRamRangesXR3 = pNext;
852	pVM->pgm.s.pRamRangesXR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
853	pVM->pgm.s.pRamRangesXRC = pNext ? pNext->pSelfRC : NIL_RTRCPTR;
854	}
855	ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
856
857	pgmR3PhysRebuildRamRangeSearchTrees(pVM);
858	pgmUnlock(pVM);
859	}
860
861
862	/**
863	* Unlink an existing RAM range from the list.
864	*
865	* @param pVM Pointer to the VM.
866	* @param pRam Pointer to the new list entry.
867	*/
868	static void pgmR3PhysUnlinkRamRange(PVM pVM, PPGMRAMRANGE pRam)
869	{
870	pgmLock(pVM);
871
872	/* find prev. */
873	PPGMRAMRANGE pPrev = NULL;
874	PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesXR3;
875	while (pCur != pRam)
876	{
877	pPrev = pCur;
878	pCur = pCur->pNextR3;
879	}
880	AssertFatal(pCur);
881
882	pgmR3PhysUnlinkRamRange2(pVM, pRam, pPrev);
883	pgmUnlock(pVM);
884	}
885
886
887	/**
888	* Frees a range of pages, replacing them with ZERO pages of the specified type.
889	*
890	* @returns VBox status code.
891	* @param pVM Pointer to the VM.
892	* @param pRam The RAM range in which the pages resides.
893	* @param GCPhys The address of the first page.
894	* @param GCPhysLast The address of the last page.
895	* @param uType The page type to replace then with.
896	*/
897	static int pgmR3PhysFreePageRange(PVM pVM, PPGMRAMRANGE pRam, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast, uint8_t uType)
898	{
899	PGM_LOCK_ASSERT_OWNER(pVM);
900	uint32_t cPendingPages = 0;
901	PGMMFREEPAGESREQ pReq;
902	int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
903	AssertLogRelRCReturn(rc, rc);
904
905	/* Iterate the pages. */
906	PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
907	uint32_t cPagesLeft = ((GCPhysLast - GCPhys) >> PAGE_SHIFT) + 1;
908	while (cPagesLeft-- > 0)
909	{
910	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys);
911	AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
912
913	PGM_PAGE_SET_TYPE(pVM, pPageDst, uType);
914
915	GCPhys += PAGE_SIZE;
916	pPageDst++;
917	}
918
919	if (cPendingPages)
920	{
921	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
922	AssertLogRelRCReturn(rc, rc);
923	}
924	GMMR3FreePagesCleanup(pReq);
925
926	return rc;
927	}
928
929	#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) \|\| defined(RT_OS_SOLARIS) \|\| defined(RT_OS_LINUX) \|\| defined(RT_OS_FREEBSD))
930
931	/**
932	* Rendezvous callback used by PGMR3ChangeMemBalloon that changes the memory balloon size
933	*
934	* This is only called on one of the EMTs while the other ones are waiting for
935	* it to complete this function.
936	*
937	* @returns VINF_SUCCESS (VBox strict status code).
938	* @param pVM Pointer to the VM.
939	* @param pVCpu The VMCPU for the EMT we're being called on. Unused.
940	* @param pvUser User parameter
941	*/
942	static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysChangeMemBalloonRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
943	{
944	uintptr_t paUser = (uintptr_t )pvUser;
945	bool fInflate = !!paUser[0];
946	unsigned cPages = paUser[1];
947	RTGCPHYS paPhysPage = (RTGCPHYS )paUser[2];
948	uint32_t cPendingPages = 0;
949	PGMMFREEPAGESREQ pReq;
950	int rc;
951
952	Log(("pgmR3PhysChangeMemBalloonRendezvous: %s %x pages\n", (fInflate) ? "inflate" : "deflate", cPages));
953	pgmLock(pVM);
954
955	if (fInflate)
956	{
957	/* Flush the PGM pool cache as we might have stale references to pages that we just freed. */
958	pgmR3PoolClearAllRendezvous(pVM, pVCpu, NULL);
959
960	/* Replace pages with ZERO pages. */
961	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
962	if (RT_FAILURE(rc))
963	{
964	pgmUnlock(pVM);
965	AssertLogRelRC(rc);
966	return rc;
967	}
968
969	/* Iterate the pages. */
970	for (unsigned i = 0; i < cPages; i++)
971	{
972	PPGMPAGE pPage = pgmPhysGetPage(pVM, paPhysPage[i]);
973	if ( pPage == NULL
974	\|\| PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM)
975	{
976	Log(("pgmR3PhysChangeMemBalloonRendezvous: invalid physical page %RGp pPage->u3Type=%d\n", paPhysPage[i], pPage ? PGM_PAGE_GET_TYPE(pPage) : 0));
977	break;
978	}
979
980	LogFlow(("balloon page: %RGp\n", paPhysPage[i]));
981
982	/* Flush the shadow PT if this page was previously used as a guest page table. */
983	pgmPoolFlushPageByGCPhys(pVM, paPhysPage[i]);
984
985	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, paPhysPage[i]);
986	if (RT_FAILURE(rc))
987	{
988	pgmUnlock(pVM);
989	AssertLogRelRC(rc);
990	return rc;
991	}
992	Assert(PGM_PAGE_IS_ZERO(pPage));
993	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_BALLOONED);
994	}
995
996	if (cPendingPages)
997	{
998	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
999	if (RT_FAILURE(rc))
1000	{
1001	pgmUnlock(pVM);
1002	AssertLogRelRC(rc);
1003	return rc;
1004	}
1005	}
1006	GMMR3FreePagesCleanup(pReq);
1007	}
1008	else
1009	{
1010	/* Iterate the pages. */
1011	for (unsigned i = 0; i < cPages; i++)
1012	{
1013	PPGMPAGE pPage = pgmPhysGetPage(pVM, paPhysPage[i]);
1014	AssertBreak(pPage && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM);
1015
1016	LogFlow(("Free ballooned page: %RGp\n", paPhysPage[i]));
1017
1018	Assert(PGM_PAGE_IS_BALLOONED(pPage));
1019
1020	/* Change back to zero page. */
1021	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
1022	}
1023
1024	/* Note that we currently do not map any ballooned pages in our shadow page tables, so no need to flush the pgm pool. */
1025	}
1026
1027	/* Notify GMM about the balloon change. */
1028	rc = GMMR3BalloonedPages(pVM, (fInflate) ? GMMBALLOONACTION_INFLATE : GMMBALLOONACTION_DEFLATE, cPages);
1029	if (RT_SUCCESS(rc))
1030	{
1031	if (!fInflate)
1032	{
1033	Assert(pVM->pgm.s.cBalloonedPages >= cPages);
1034	pVM->pgm.s.cBalloonedPages -= cPages;
1035	}
1036	else
1037	pVM->pgm.s.cBalloonedPages += cPages;
1038	}
1039
1040	pgmUnlock(pVM);
1041
1042	/* Flush the recompiler's TLB as well. */
1043	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1044	CPUMSetChangedFlags(&pVM->aCpus[i], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
1045
1046	AssertLogRelRC(rc);
1047	return rc;
1048	}
1049
1050
1051	/**
1052	* Frees a range of ram pages, replacing them with ZERO pages; helper for PGMR3PhysFreeRamPages
1053	*
1054	* @returns VBox status code.
1055	* @param pVM Pointer to the VM.
1056	* @param fInflate Inflate or deflate memory balloon
1057	* @param cPages Number of pages to free
1058	* @param paPhysPage Array of guest physical addresses
1059	*/
1060	static DECLCALLBACK(void) pgmR3PhysChangeMemBalloonHelper(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
1061	{
1062	uintptr_t paUser[3];
1063
1064	paUser[0] = fInflate;
1065	paUser[1] = cPages;
1066	paUser[2] = (uintptr_t)paPhysPage;
1067	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
1068	AssertRC(rc);
1069
1070	/* Made a copy in PGMR3PhysFreeRamPages; free it here. */
1071	RTMemFree(paPhysPage);
1072	}
1073
1074	#endif /* 64-bit host && (Windows \|\| Solaris \|\| Linux \|\| FreeBSD) */
1075
1076	/**
1077	* Inflate or deflate a memory balloon
1078	*
1079	* @returns VBox status code.
1080	* @param pVM Pointer to the VM.
1081	* @param fInflate Inflate or deflate memory balloon
1082	* @param cPages Number of pages to free
1083	* @param paPhysPage Array of guest physical addresses
1084	*/
1085	VMMR3DECL(int) PGMR3PhysChangeMemBalloon(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
1086	{
1087	/* This must match GMMR0Init; currently we only support memory ballooning on all 64-bit hosts except Mac OS X */
1088	#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) \|\| defined(RT_OS_SOLARIS) \|\| defined(RT_OS_LINUX) \|\| defined(RT_OS_FREEBSD))
1089	int rc;
1090
1091	/* Older additions (ancient non-functioning balloon code) pass wrong physical addresses. */
1092	AssertReturn(!(paPhysPage[0] & 0xfff), VERR_INVALID_PARAMETER);
1093
1094	/* We own the IOM lock here and could cause a deadlock by waiting for another VCPU that is blocking on the IOM lock.
1095	* In the SMP case we post a request packet to postpone the job.
1096	*/
1097	if (pVM->cCpus > 1)
1098	{
1099	unsigned cbPhysPage = cPages * sizeof(paPhysPage[0]);
1100	RTGCPHYS paPhysPageCopy = (RTGCPHYS )RTMemAlloc(cbPhysPage);
1101	AssertReturn(paPhysPageCopy, VERR_NO_MEMORY);
1102
1103	memcpy(paPhysPageCopy, paPhysPage, cbPhysPage);
1104
1105	rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysChangeMemBalloonHelper, 4, pVM, fInflate, cPages, paPhysPageCopy);
1106	AssertRC(rc);
1107	}
1108	else
1109	{
1110	uintptr_t paUser[3];
1111
1112	paUser[0] = fInflate;
1113	paUser[1] = cPages;
1114	paUser[2] = (uintptr_t)paPhysPage;
1115	rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
1116	AssertRC(rc);
1117	}
1118	return rc;
1119
1120	#else
1121	NOREF(pVM); NOREF(fInflate); NOREF(cPages); NOREF(paPhysPage);
1122	return VERR_NOT_IMPLEMENTED;
1123	#endif
1124	}
1125
1126
1127	/**
1128	* Rendezvous callback used by PGMR3WriteProtectRAM that write protects all
1129	* physical RAM.
1130	*
1131	* This is only called on one of the EMTs while the other ones are waiting for
1132	* it to complete this function.
1133	*
1134	* @returns VINF_SUCCESS (VBox strict status code).
1135	* @param pVM Pointer to the VM.
1136	* @param pVCpu The VMCPU for the EMT we're being called on. Unused.
1137	* @param pvUser User parameter, unused.
1138	*/
1139	static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysWriteProtectRAMRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
1140	{
1141	int rc = VINF_SUCCESS;
1142	NOREF(pvUser); NOREF(pVCpu);
1143
1144	pgmLock(pVM);
1145	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1146	pgmPoolResetDirtyPages(pVM);
1147	#endif
1148
1149	/** @todo pointless to write protect the physical page pointed to by RSP. */
1150
1151	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
1152	pRam;
1153	pRam = pRam->CTX_SUFF(pNext))
1154	{
1155	uint32_t cPages = pRam->cb >> PAGE_SHIFT;
1156	for (uint32_t iPage = 0; iPage < cPages; iPage++)
1157	{
1158	PPGMPAGE pPage = &pRam->aPages[iPage];
1159	PGMPAGETYPE enmPageType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
1160
1161	if ( RT_LIKELY(enmPageType == PGMPAGETYPE_RAM)
1162	\|\| enmPageType == PGMPAGETYPE_MMIO2)
1163	{
1164	/*
1165	* A RAM page.
1166	*/
1167	switch (PGM_PAGE_GET_STATE(pPage))
1168	{
1169	case PGM_PAGE_STATE_ALLOCATED:
1170	/** @todo Optimize this: Don't always re-enable write
1171	* monitoring if the page is known to be very busy. */
1172	if (PGM_PAGE_IS_WRITTEN_TO(pPage))
1173	{
1174	PGM_PAGE_CLEAR_WRITTEN_TO(pVM, pPage);
1175	/* Remember this dirty page for the next (memory) sync. */
1176	PGM_PAGE_SET_FT_DIRTY(pPage);
1177	}
1178
1179	pgmPhysPageWriteMonitor(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1180	break;
1181
1182	case PGM_PAGE_STATE_SHARED:
1183	AssertFailed();
1184	break;
1185
1186	case PGM_PAGE_STATE_WRITE_MONITORED: /* nothing to change. */
1187	default:
1188	break;
1189	}
1190	}
1191	}
1192	}
1193	pgmR3PoolWriteProtectPages(pVM);
1194	PGM_INVL_ALL_VCPU_TLBS(pVM);
1195	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1196	CPUMSetChangedFlags(&pVM->aCpus[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
1197
1198	pgmUnlock(pVM);
1199	return rc;
1200	}
1201
1202	/**
1203	* Protect all physical RAM to monitor writes
1204	*
1205	* @returns VBox status code.
1206	* @param pVM Pointer to the VM.
1207	*/
1208	VMMR3DECL(int) PGMR3PhysWriteProtectRAM(PVM pVM)
1209	{
1210	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
1211
1212	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysWriteProtectRAMRendezvous, NULL);
1213	AssertRC(rc);
1214	return rc;
1215	}
1216
1217	/**
1218	* Enumerate all dirty FT pages.
1219	*
1220	* @returns VBox status code.
1221	* @param pVM Pointer to the VM.
1222	* @param pfnEnum Enumerate callback handler.
1223	* @param pvUser Enumerate callback handler parameter.
1224	*/
1225	VMMR3DECL(int) PGMR3PhysEnumDirtyFTPages(PVM pVM, PFNPGMENUMDIRTYFTPAGES pfnEnum, void *pvUser)
1226	{
1227	int rc = VINF_SUCCESS;
1228
1229	pgmLock(pVM);
1230	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
1231	pRam;
1232	pRam = pRam->CTX_SUFF(pNext))
1233	{
1234	uint32_t cPages = pRam->cb >> PAGE_SHIFT;
1235	for (uint32_t iPage = 0; iPage < cPages; iPage++)
1236	{
1237	PPGMPAGE pPage = &pRam->aPages[iPage];
1238	PGMPAGETYPE enmPageType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
1239
1240	if ( RT_LIKELY(enmPageType == PGMPAGETYPE_RAM)
1241	\|\| enmPageType == PGMPAGETYPE_MMIO2)
1242	{
1243	/*
1244	* A RAM page.
1245	*/
1246	switch (PGM_PAGE_GET_STATE(pPage))
1247	{
1248	case PGM_PAGE_STATE_ALLOCATED:
1249	case PGM_PAGE_STATE_WRITE_MONITORED:
1250	if ( !PGM_PAGE_IS_WRITTEN_TO(pPage) /* not very recently updated? */
1251	&& PGM_PAGE_IS_FT_DIRTY(pPage))
1252	{
1253	unsigned cbPageRange = PAGE_SIZE;
1254	unsigned iPageClean = iPage + 1;
1255	RTGCPHYS GCPhysPage = pRam->GCPhys + iPage * PAGE_SIZE;
1256	uint8_t *pu8Page = NULL;
1257	PGMPAGEMAPLOCK Lock;
1258
1259	/* Find the next clean page, so we can merge adjacent dirty pages. */
1260	for (; iPageClean < cPages; iPageClean++)
1261	{
1262	PPGMPAGE pPageNext = &pRam->aPages[iPageClean];
1263	if ( RT_UNLIKELY(PGM_PAGE_GET_TYPE(pPageNext) != PGMPAGETYPE_RAM)
1264	\|\| PGM_PAGE_GET_STATE(pPageNext) != PGM_PAGE_STATE_ALLOCATED
1265	\|\| PGM_PAGE_IS_WRITTEN_TO(pPageNext)
1266	\|\| !PGM_PAGE_IS_FT_DIRTY(pPageNext)
1267	/* Crossing a chunk boundary? */
1268	\|\| (GCPhysPage & GMM_PAGEID_IDX_MASK) != ((GCPhysPage + cbPageRange) & GMM_PAGEID_IDX_MASK)
1269	)
1270	break;
1271
1272	cbPageRange += PAGE_SIZE;
1273	}
1274
1275	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysPage, (const void **)&pu8Page, &Lock);
1276	if (RT_SUCCESS(rc))
1277	{
1278	/** @todo this is risky; the range might be changed, but little choice as the sync
1279	* costs a lot of time. */
1280	pgmUnlock(pVM);
1281	pfnEnum(pVM, GCPhysPage, pu8Page, cbPageRange, pvUser);
1282	pgmLock(pVM);
1283	PGMPhysReleasePageMappingLock(pVM, &Lock);
1284	}
1285
1286	for (iPage; iPage < iPageClean; iPage++)
1287	PGM_PAGE_CLEAR_FT_DIRTY(&pRam->aPages[iPage]);
1288
1289	iPage = iPageClean - 1;
1290	}
1291	break;
1292	}
1293	}
1294	}
1295	}
1296	pgmUnlock(pVM);
1297	return rc;
1298	}
1299
1300
1301	/**
1302	* Gets the number of ram ranges.
1303	*
1304	* @returns Number of ram ranges. Returns UINT32_MAX if @a pVM is invalid.
1305	* @param pVM Pointer to the VM.
1306	*/
1307	VMMR3DECL(uint32_t) PGMR3PhysGetRamRangeCount(PVM pVM)
1308	{
1309	VM_ASSERT_VALID_EXT_RETURN(pVM, UINT32_MAX);
1310
1311	pgmLock(pVM);
1312	uint32_t cRamRanges = 0;
1313	for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRangesX); pCur; pCur = pCur->CTX_SUFF(pNext))
1314	cRamRanges++;
1315	pgmUnlock(pVM);
1316	return cRamRanges;
1317	}
1318
1319
1320	/**
1321	* Get information about a range.
1322	*
1323	* @returns VINF_SUCCESS or VERR_OUT_OF_RANGE.
1324	* @param pVM Pointer to the VM.
1325	* @param iRange The ordinal of the range.
1326	* @param pGCPhysStart Where to return the start of the range. Optional.
1327	* @param pGCPhysLast Where to return the address of the last byte in the
1328	* range. Optional.
1329	* @param pfIsMmio Where to indicate that this is a pure MMIO range.
1330	* Optional.
1331	*/
1332	VMMR3DECL(int) PGMR3PhysGetRange(PVM pVM, uint32_t iRange, PRTGCPHYS pGCPhysStart, PRTGCPHYS pGCPhysLast,
1333	const char *ppszDesc, bool pfIsMmio)
1334	{
1335	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
1336
1337	pgmLock(pVM);
1338	uint32_t iCurRange = 0;
1339	for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRangesX); pCur; pCur = pCur->CTX_SUFF(pNext), iCurRange++)
1340	if (iCurRange == iRange)
1341	{
1342	if (pGCPhysStart)
1343	*pGCPhysStart = pCur->GCPhys;
1344	if (pGCPhysLast)
1345	*pGCPhysLast = pCur->GCPhysLast;
1346	if (ppszDesc)
1347	*ppszDesc = pCur->pszDesc;
1348	if (pfIsMmio)
1349	*pfIsMmio = !!(pCur->fFlags & PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO);
1350
1351	pgmUnlock(pVM);
1352	return VINF_SUCCESS;
1353	}
1354	pgmUnlock(pVM);
1355	return VERR_OUT_OF_RANGE;
1356	}
1357
1358
1359	/**
1360	* Query the amount of free memory inside VMMR0
1361	*
1362	* @returns VBox status code.
1363	* @param pVM Pointer to the VM.
1364	* @param pcbAllocMem Where to return the amount of memory allocated
1365	* by VMs.
1366	* @param pcbFreeMem Where to return the amount of memory that is
1367	* allocated from the host but not currently used
1368	* by any VMs.
1369	* @param pcbBallonedMem Where to return the sum of memory that is
1370	* currently ballooned by the VMs.
1371	* @param pcbSharedMem Where to return the amount of memory that is
1372	* currently shared.
1373	*/
1374	VMMR3DECL(int) PGMR3QueryGlobalMemoryStats(PVM pVM, uint64_t pcbAllocMem, uint64_t pcbFreeMem,
1375	uint64_t pcbBallonedMem, uint64_t pcbSharedMem)
1376	{
1377	uint64_t cAllocPages = 0;
1378	uint64_t cFreePages = 0;
1379	uint64_t cBalloonPages = 0;
1380	uint64_t cSharedPages = 0;
1381	int rc = GMMR3QueryHypervisorMemoryStats(pVM, &cAllocPages, &cFreePages, &cBalloonPages, &cSharedPages);
1382	AssertRCReturn(rc, rc);
1383
1384	if (pcbAllocMem)
1385	pcbAllocMem = cAllocPages _4K;
1386
1387	if (pcbFreeMem)
1388	pcbFreeMem = cFreePages _4K;
1389
1390	if (pcbBallonedMem)
1391	pcbBallonedMem = cBalloonPages _4K;
1392
1393	if (pcbSharedMem)
1394	pcbSharedMem = cSharedPages _4K;
1395
1396	Log(("PGMR3QueryVMMMemoryStats: all=%llx free=%llx ballooned=%llx shared=%llx\n",
1397	cAllocPages, cFreePages, cBalloonPages, cSharedPages));
1398	return VINF_SUCCESS;
1399	}
1400
1401
1402	/**
1403	* Query memory stats for the VM.
1404	*
1405	* @returns VBox status code.
1406	* @param pVM Pointer to the VM.
1407	* @param pcbTotalMem Where to return total amount memory the VM may
1408	* possibly use.
1409	* @param pcbPrivateMem Where to return the amount of private memory
1410	* currently allocated.
1411	* @param pcbSharedMem Where to return the amount of actually shared
1412	* memory currently used by the VM.
1413	* @param pcbZeroMem Where to return the amount of memory backed by
1414	* zero pages.
1415	*
1416	* @remarks The total mem is normally larger than the sum of the three
1417	* components. There are two reasons for this, first the amount of
1418	* shared memory is what we're sure is shared instead of what could
1419	* possibly be shared with someone. Secondly, because the total may
1420	* include some pure MMIO pages that doesn't go into any of the three
1421	* sub-counts.
1422	*
1423	* @todo Why do we return reused shared pages instead of anything that could
1424	* potentially be shared? Doesn't this mean the first VM gets a much
1425	* lower number of shared pages?
1426	*/
1427	VMMR3DECL(int) PGMR3QueryMemoryStats(PVM pVM, uint64_t pcbTotalMem, uint64_t pcbPrivateMem,
1428	uint64_t pcbSharedMem, uint64_t pcbZeroMem)
1429	{
1430	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
1431
1432	if (pcbTotalMem)
1433	pcbTotalMem = (uint64_t)pVM->pgm.s.cAllPages PAGE_SIZE;
1434
1435	if (pcbPrivateMem)
1436	pcbPrivateMem = (uint64_t)pVM->pgm.s.cPrivatePages PAGE_SIZE;
1437
1438	if (pcbSharedMem)
1439	pcbSharedMem = (uint64_t)pVM->pgm.s.cReusedSharedPages PAGE_SIZE;
1440
1441	if (pcbZeroMem)
1442	pcbZeroMem = (uint64_t)pVM->pgm.s.cZeroPages PAGE_SIZE;
1443
1444	Log(("PGMR3QueryMemoryStats: all=%x private=%x reused=%x zero=%x\n", pVM->pgm.s.cAllPages, pVM->pgm.s.cPrivatePages, pVM->pgm.s.cReusedSharedPages, pVM->pgm.s.cZeroPages));
1445	return VINF_SUCCESS;
1446	}
1447
1448
1449	/**
1450	* PGMR3PhysRegisterRam worker that initializes and links a RAM range.
1451	*
1452	* @param pVM Pointer to the VM.
1453	* @param pNew The new RAM range.
1454	* @param GCPhys The address of the RAM range.
1455	* @param GCPhysLast The last address of the RAM range.
1456	* @param RCPtrNew The RC address if the range is floating. NIL_RTRCPTR
1457	* if in HMA.
1458	* @param R0PtrNew Ditto for R0.
1459	* @param pszDesc The description.
1460	* @param pPrev The previous RAM range (for linking).
1461	*/
1462	static void pgmR3PhysInitAndLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast,
1463	RTRCPTR RCPtrNew, RTR0PTR R0PtrNew, const char *pszDesc, PPGMRAMRANGE pPrev)
1464	{
1465	/*
1466	* Initialize the range.
1467	*/
1468	pNew->pSelfR0 = R0PtrNew != NIL_RTR0PTR ? R0PtrNew : MMHyperCCToR0(pVM, pNew);
1469	pNew->pSelfRC = RCPtrNew != NIL_RTRCPTR ? RCPtrNew : MMHyperCCToRC(pVM, pNew);
1470	pNew->GCPhys = GCPhys;
1471	pNew->GCPhysLast = GCPhysLast;
1472	pNew->cb = GCPhysLast - GCPhys + 1;
1473	pNew->pszDesc = pszDesc;
1474	pNew->fFlags = RCPtrNew != NIL_RTRCPTR ? PGM_RAM_RANGE_FLAGS_FLOATING : 0;
1475	pNew->pvR3 = NULL;
1476	pNew->paLSPages = NULL;
1477
1478	uint32_t const cPages = pNew->cb >> PAGE_SHIFT;
1479	RTGCPHYS iPage = cPages;
1480	while (iPage-- > 0)
1481	PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_RAM);
1482
1483	/* Update the page count stats. */
1484	pVM->pgm.s.cZeroPages += cPages;
1485	pVM->pgm.s.cAllPages += cPages;
1486
1487	/*
1488	* Link it.
1489	*/
1490	pgmR3PhysLinkRamRange(pVM, pNew, pPrev);
1491	}
1492
1493
1494	/**
1495	* Relocate a floating RAM range.
1496	*
1497	* @copydoc FNPGMRELOCATE.
1498	*/
1499	static DECLCALLBACK(bool) pgmR3PhysRamRangeRelocate(PVM pVM, RTGCPTR GCPtrOld, RTGCPTR GCPtrNew, PGMRELOCATECALL enmMode, void *pvUser)
1500	{
1501	PPGMRAMRANGE pRam = (PPGMRAMRANGE)pvUser;
1502	Assert(pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING);
1503	Assert(pRam->pSelfRC == GCPtrOld + PAGE_SIZE);
1504
1505	switch (enmMode)
1506	{
1507	case PGMRELOCATECALL_SUGGEST:
1508	return true;
1509
1510	case PGMRELOCATECALL_RELOCATE:
1511	{
1512	/*
1513	* Update myself, then relink all the ranges and flush the RC TLB.
1514	*/
1515	pgmLock(pVM);
1516
1517	pRam->pSelfRC = (RTRCPTR)(GCPtrNew + PAGE_SIZE);
1518
1519	pgmR3PhysRelinkRamRanges(pVM);
1520	for (unsigned i = 0; i < PGM_RAMRANGE_TLB_ENTRIES; i++)
1521	pVM->pgm.s.apRamRangesTlbRC[i] = NIL_RTRCPTR;
1522
1523	pgmUnlock(pVM);
1524	return true;
1525	}
1526
1527	default:
1528	AssertFailedReturn(false);
1529	}
1530	}
1531
1532
1533	/**
1534	* PGMR3PhysRegisterRam worker that registers a high chunk.
1535	*
1536	* @returns VBox status code.
1537	* @param pVM Pointer to the VM.
1538	* @param GCPhys The address of the RAM.
1539	* @param cRamPages The number of RAM pages to register.
1540	* @param cbChunk The size of the PGMRAMRANGE guest mapping.
1541	* @param iChunk The chunk number.
1542	* @param pszDesc The RAM range description.
1543	* @param ppPrev Previous RAM range pointer. In/Out.
1544	*/
1545	static int pgmR3PhysRegisterHighRamChunk(PVM pVM, RTGCPHYS GCPhys, uint32_t cRamPages,
1546	uint32_t cbChunk, uint32_t iChunk, const char *pszDesc,
1547	PPGMRAMRANGE *ppPrev)
1548	{
1549	const char *pszDescChunk = iChunk == 0
1550	? pszDesc
1551	: MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s (#%u)", pszDesc, iChunk + 1);
1552	AssertReturn(pszDescChunk, VERR_NO_MEMORY);
1553
1554	/*
1555	* Allocate memory for the new chunk.
1556	*/
1557	size_t const cChunkPages = RT_ALIGN_Z(RT_UOFFSETOF(PGMRAMRANGE, aPages[cRamPages]), PAGE_SIZE) >> PAGE_SHIFT;
1558	PSUPPAGE paChunkPages = (PSUPPAGE)RTMemTmpAllocZ(sizeof(SUPPAGE) * cChunkPages);
1559	AssertReturn(paChunkPages, VERR_NO_TMP_MEMORY);
1560	RTR0PTR R0PtrChunk = NIL_RTR0PTR;
1561	void *pvChunk = NULL;
1562	int rc = SUPR3PageAllocEx(cChunkPages, 0 /fFlags/, &pvChunk,
1563	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1564	VMMIsHwVirtExtForced(pVM) ? &R0PtrChunk : NULL,
1565	#else
1566	NULL,
1567	#endif
1568	paChunkPages);
1569	if (RT_SUCCESS(rc))
1570	{
1571	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1572	if (!VMMIsHwVirtExtForced(pVM))
1573	R0PtrChunk = NIL_RTR0PTR;
1574	#else
1575	R0PtrChunk = (uintptr_t)pvChunk;
1576	#endif
1577	memset(pvChunk, 0, cChunkPages << PAGE_SHIFT);
1578
1579	PPGMRAMRANGE pNew = (PPGMRAMRANGE)pvChunk;
1580
1581	/*
1582	* Create a mapping and map the pages into it.
1583	* We push these in below the HMA.
1584	*/
1585	RTGCPTR GCPtrChunkMap = pVM->pgm.s.GCPtrPrevRamRangeMapping - cbChunk;
1586	rc = PGMR3MapPT(pVM, GCPtrChunkMap, cbChunk, 0 /fFlags/, pgmR3PhysRamRangeRelocate, pNew, pszDescChunk);
1587	if (RT_SUCCESS(rc))
1588	{
1589	pVM->pgm.s.GCPtrPrevRamRangeMapping = GCPtrChunkMap;
1590
1591	RTGCPTR const GCPtrChunk = GCPtrChunkMap + PAGE_SIZE;
1592	RTGCPTR GCPtrPage = GCPtrChunk;
1593	for (uint32_t iPage = 0; iPage < cChunkPages && RT_SUCCESS(rc); iPage++, GCPtrPage += PAGE_SIZE)
1594	rc = PGMMap(pVM, GCPtrPage, paChunkPages[iPage].Phys, PAGE_SIZE, 0);
1595	if (RT_SUCCESS(rc))
1596	{
1597	/*
1598	* Ok, init and link the range.
1599	*/
1600	pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhys + ((RTGCPHYS)cRamPages << PAGE_SHIFT) - 1,
1601	(RTRCPTR)GCPtrChunk, R0PtrChunk, pszDescChunk, *ppPrev);
1602	*ppPrev = pNew;
1603	}
1604	}
1605
1606	if (RT_FAILURE(rc))
1607	SUPR3PageFreeEx(pvChunk, cChunkPages);
1608	}
1609
1610	RTMemTmpFree(paChunkPages);
1611	return rc;
1612	}
1613
1614
1615	/**
1616	* Sets up a range RAM.
1617	*
1618	* This will check for conflicting registrations, make a resource
1619	* reservation for the memory (with GMM), and setup the per-page
1620	* tracking structures (PGMPAGE).
1621	*
1622	* @returns VBox status code.
1623	* @param pVM Pointer to the VM.
1624	* @param GCPhys The physical address of the RAM.
1625	* @param cb The size of the RAM.
1626	* @param pszDesc The description - not copied, so, don't free or change it.
1627	*/
1628	VMMR3DECL(int) PGMR3PhysRegisterRam(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, const char *pszDesc)
1629	{
1630	/*
1631	* Validate input.
1632	*/
1633	Log(("PGMR3PhysRegisterRam: GCPhys=%RGp cb=%RGp pszDesc=%s\n", GCPhys, cb, pszDesc));
1634	AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
1635	AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
1636	AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
1637	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
1638	AssertMsgReturn(GCPhysLast > GCPhys, ("The range wraps! GCPhys=%RGp cb=%RGp\n", GCPhys, cb), VERR_INVALID_PARAMETER);
1639	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
1640	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
1641
1642	pgmLock(pVM);
1643
1644	/*
1645	* Find range location and check for conflicts.
1646	* (We don't lock here because the locking by EMT is only required on update.)
1647	*/
1648	PPGMRAMRANGE pPrev = NULL;
1649	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
1650	while (pRam && GCPhysLast >= pRam->GCPhys)
1651	{
1652	if ( GCPhysLast >= pRam->GCPhys
1653	&& GCPhys <= pRam->GCPhysLast)
1654	AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
1655	GCPhys, GCPhysLast, pszDesc,
1656	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
1657	VERR_PGM_RAM_CONFLICT);
1658
1659	/* next */
1660	pPrev = pRam;
1661	pRam = pRam->pNextR3;
1662	}
1663
1664	/*
1665	* Register it with GMM (the API bitches).
1666	*/
1667	const RTGCPHYS cPages = cb >> PAGE_SHIFT;
1668	int rc = MMR3IncreaseBaseReservation(pVM, cPages);
1669	if (RT_FAILURE(rc))
1670	{
1671	pgmUnlock(pVM);
1672	return rc;
1673	}
1674
1675	if ( GCPhys >= _4G
1676	&& cPages > 256)
1677	{
1678	/*
1679	* The PGMRAMRANGE structures for the high memory can get very big.
1680	* In order to avoid SUPR3PageAllocEx allocation failures due to the
1681	* allocation size limit there and also to avoid being unable to find
1682	* guest mapping space for them, we split this memory up into 4MB in
1683	* (potential) raw-mode configs and 16MB chunks in forced AMD-V/VT-x
1684	* mode.
1685	*
1686	* The first and last page of each mapping are guard pages and marked
1687	* not-present. So, we've got 4186112 and 16769024 bytes available for
1688	* the PGMRAMRANGE structure.
1689	*
1690	* Note! The sizes used here will influence the saved state.
1691	*/
1692	uint32_t cbChunk;
1693	uint32_t cPagesPerChunk;
1694	if (VMMIsHwVirtExtForced(pVM))
1695	{
1696	cbChunk = 16U*_1M;
1697	cPagesPerChunk = 1048048; /* max ~1048059 */
1698	AssertCompile(sizeof(PGMRAMRANGE) + sizeof(PGMPAGE) * 1048048 < 16U_1M - PAGE_SIZE 2);
1699	}
1700	else
1701	{
1702	cbChunk = 4U*_1M;
1703	cPagesPerChunk = 261616; /* max ~261627 */
1704	AssertCompile(sizeof(PGMRAMRANGE) + sizeof(PGMPAGE) * 261616 < 4U_1M - PAGE_SIZE 2);
1705	}
1706	AssertRelease(RT_UOFFSETOF(PGMRAMRANGE, aPages[cPagesPerChunk]) + PAGE_SIZE * 2 <= cbChunk);
1707
1708	RTGCPHYS cPagesLeft = cPages;
1709	RTGCPHYS GCPhysChunk = GCPhys;
1710	uint32_t iChunk = 0;
1711	while (cPagesLeft > 0)
1712	{
1713	uint32_t cPagesInChunk = cPagesLeft;
1714	if (cPagesInChunk > cPagesPerChunk)
1715	cPagesInChunk = cPagesPerChunk;
1716
1717	rc = pgmR3PhysRegisterHighRamChunk(pVM, GCPhysChunk, cPagesInChunk, cbChunk, iChunk, pszDesc, &pPrev);
1718	AssertRCReturn(rc, rc);
1719
1720	/* advance */
1721	GCPhysChunk += (RTGCPHYS)cPagesInChunk << PAGE_SHIFT;
1722	cPagesLeft -= cPagesInChunk;
1723	iChunk++;
1724	}
1725	}
1726	else
1727	{
1728	/*
1729	* Allocate, initialize and link the new RAM range.
1730	*/
1731	const size_t cbRamRange = RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]);
1732	PPGMRAMRANGE pNew;
1733	rc = MMR3HyperAllocOnceNoRel(pVM, cbRamRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
1734	AssertLogRelMsgRCReturn(rc, ("cbRamRange=%zu\n", cbRamRange), rc);
1735
1736	pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhysLast, NIL_RTRCPTR, NIL_RTR0PTR, pszDesc, pPrev);
1737	}
1738	pgmPhysInvalidatePageMapTLB(pVM);
1739	pgmUnlock(pVM);
1740
1741	#ifdef VBOX_WITH_REM
1742	/*
1743	* Notify REM.
1744	*/
1745	REMR3NotifyPhysRamRegister(pVM, GCPhys, cb, REM_NOTIFY_PHYS_RAM_FLAGS_RAM);
1746	#endif
1747
1748	return VINF_SUCCESS;
1749	}
1750
1751
1752	/**
1753	* Worker called by PGMR3InitFinalize if we're configured to pre-allocate RAM.
1754	*
1755	* We do this late in the init process so that all the ROM and MMIO ranges have
1756	* been registered already and we don't go wasting memory on them.
1757	*
1758	* @returns VBox status code.
1759	*
1760	* @param pVM Pointer to the VM.
1761	*/
1762	int pgmR3PhysRamPreAllocate(PVM pVM)
1763	{
1764	Assert(pVM->pgm.s.fRamPreAlloc);
1765	Log(("pgmR3PhysRamPreAllocate: enter\n"));
1766
1767	/*
1768	* Walk the RAM ranges and allocate all RAM pages, halt at
1769	* the first allocation error.
1770	*/
1771	uint64_t cPages = 0;
1772	uint64_t NanoTS = RTTimeNanoTS();
1773	pgmLock(pVM);
1774	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
1775	{
1776	PPGMPAGE pPage = &pRam->aPages[0];
1777	RTGCPHYS GCPhys = pRam->GCPhys;
1778	uint32_t cLeft = pRam->cb >> PAGE_SHIFT;
1779	while (cLeft-- > 0)
1780	{
1781	if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
1782	{
1783	switch (PGM_PAGE_GET_STATE(pPage))
1784	{
1785	case PGM_PAGE_STATE_ZERO:
1786	{
1787	int rc = pgmPhysAllocPage(pVM, pPage, GCPhys);
1788	if (RT_FAILURE(rc))
1789	{
1790	LogRel(("PGM: RAM Pre-allocation failed at %RGp (in %s) with rc=%Rrc\n", GCPhys, pRam->pszDesc, rc));
1791	pgmUnlock(pVM);
1792	return rc;
1793	}
1794	cPages++;
1795	break;
1796	}
1797
1798	case PGM_PAGE_STATE_BALLOONED:
1799	case PGM_PAGE_STATE_ALLOCATED:
1800	case PGM_PAGE_STATE_WRITE_MONITORED:
1801	case PGM_PAGE_STATE_SHARED:
1802	/* nothing to do here. */
1803	break;
1804	}
1805	}
1806
1807	/* next */
1808	pPage++;
1809	GCPhys += PAGE_SIZE;
1810	}
1811	}
1812	pgmUnlock(pVM);
1813	NanoTS = RTTimeNanoTS() - NanoTS;
1814
1815	LogRel(("PGM: Pre-allocated %llu pages in %llu ms\n", cPages, NanoTS / 1000000));
1816	Log(("pgmR3PhysRamPreAllocate: returns VINF_SUCCESS\n"));
1817	return VINF_SUCCESS;
1818	}
1819
1820
1821	/**
1822	* Checks shared page checksums.
1823	*
1824	* @param pVM Pointer to the shared VM structure.
1825	*/
1826	void pgmR3PhysAssertSharedPageChecksums(PVM pVM)
1827	{
1828	#ifdef VBOX_STRICT
1829	pgmLock(pVM);
1830
1831	if (pVM->pgm.s.cSharedPages > 0)
1832	{
1833	/*
1834	* Walk the ram ranges.
1835	*/
1836	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
1837	{
1838	uint32_t iPage = pRam->cb >> PAGE_SHIFT;
1839	AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
1840
1841	while (iPage-- > 0)
1842	{
1843	PPGMPAGE pPage = &pRam->aPages[iPage];
1844	if (PGM_PAGE_IS_SHARED(pPage))
1845	{
1846	uint32_t u32Checksum = pPage->s.u2Unused0 \| ((uint32_t)pPage->s.u2Unused1 << 8);
1847	if (!u32Checksum)
1848	{
1849	RTGCPHYS GCPhysPage = pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
1850	void const *pvPage;
1851	int rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhysPage, &pvPage);
1852	if (RT_SUCCESS(rc))
1853	{
1854	uint32_t u32Checksum2 = RTCrc32(pvPage, PAGE_SIZE);
1855	# if 0
1856	AssertMsg((u32Checksum2 & UINT32_C(0x00000303)) == u32Checksum, ("GCPhysPage=%RGp\n", GCPhysPage));
1857	# else
1858	if ((u32Checksum2 & UINT32_C(0x00000303)) == u32Checksum)
1859	LogFlow(("shpg %#x @ %RGp %#x [OK]\n", PGM_PAGE_GET_PAGEID(pPage), GCPhysPage, u32Checksum2));
1860	else
1861	AssertMsgFailed(("shpg %#x @ %RGp %#x\n", PGM_PAGE_GET_PAGEID(pPage), GCPhysPage, u32Checksum2));
1862	# endif
1863	}
1864	else
1865	AssertRC(rc);
1866	}
1867	}
1868
1869	} /* for each page */
1870
1871	} /* for each ram range */
1872	}
1873
1874	pgmUnlock(pVM);
1875	#endif /* VBOX_STRICT */
1876	NOREF(pVM);
1877	}
1878
1879
1880	/**
1881	* Resets (zeros) the RAM.
1882	*
1883	* ASSUMES that the caller owns the PGM lock.
1884	*
1885	* @returns VBox status code.
1886	* @param pVM Pointer to the VM.
1887	*/
1888	int pgmR3PhysRamReset(PVM pVM)
1889	{
1890	PGM_LOCK_ASSERT_OWNER(pVM);
1891
1892	/* Reset the memory balloon. */
1893	int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
1894	AssertRC(rc);
1895
1896	#ifdef VBOX_WITH_PAGE_SHARING
1897	/* Clear all registered shared modules. */
1898	pgmR3PhysAssertSharedPageChecksums(pVM);
1899	rc = GMMR3ResetSharedModules(pVM);
1900	AssertRC(rc);
1901	#endif
1902	/* Reset counters. */
1903	pVM->pgm.s.cReusedSharedPages = 0;
1904	pVM->pgm.s.cBalloonedPages = 0;
1905
1906	/*
1907	* We batch up pages that should be freed instead of calling GMM for
1908	* each and every one of them.
1909	*/
1910	uint32_t cPendingPages = 0;
1911	PGMMFREEPAGESREQ pReq;
1912	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
1913	AssertLogRelRCReturn(rc, rc);
1914
1915	/*
1916	* Walk the ram ranges.
1917	*/
1918	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
1919	{
1920	uint32_t iPage = pRam->cb >> PAGE_SHIFT;
1921	AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
1922
1923	#ifndef NO_RAM_RESET
1924	if (!pVM->pgm.s.fRamPreAlloc)
1925	#else
1926	if (0)
1927	#endif
1928	{
1929	/* Replace all RAM pages by ZERO pages. */
1930	while (iPage-- > 0)
1931	{
1932	PPGMPAGE pPage = &pRam->aPages[iPage];
1933	switch (PGM_PAGE_GET_TYPE(pPage))
1934	{
1935	case PGMPAGETYPE_RAM:
1936	/* Do not replace pages part of a 2 MB continuous range
1937	with zero pages, but zero them instead. */
1938	if ( PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE
1939	\|\| PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED)
1940	{
1941	void *pvPage;
1942	rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
1943	AssertLogRelRCReturn(rc, rc);
1944	ASMMemZeroPage(pvPage);
1945	}
1946	else if (PGM_PAGE_IS_BALLOONED(pPage))
1947	{
1948	/* Turn into a zero page; the balloon status is lost when the VM reboots. */
1949	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
1950	}
1951	else if (!PGM_PAGE_IS_ZERO(pPage))
1952	{
1953	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1954	AssertLogRelRCReturn(rc, rc);
1955	}
1956	break;
1957
1958	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
1959	pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
1960	true /fDoAccounting/);
1961	break;
1962
1963	case PGMPAGETYPE_MMIO2:
1964	case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
1965	case PGMPAGETYPE_ROM:
1966	case PGMPAGETYPE_MMIO:
1967	break;
1968	default:
1969	AssertFailed();
1970	}
1971	} /* for each page */
1972	}
1973	else
1974	{
1975	/* Zero the memory. */
1976	while (iPage-- > 0)
1977	{
1978	PPGMPAGE pPage = &pRam->aPages[iPage];
1979	switch (PGM_PAGE_GET_TYPE(pPage))
1980	{
1981	case PGMPAGETYPE_RAM:
1982	switch (PGM_PAGE_GET_STATE(pPage))
1983	{
1984	case PGM_PAGE_STATE_ZERO:
1985	break;
1986
1987	case PGM_PAGE_STATE_BALLOONED:
1988	/* Turn into a zero page; the balloon status is lost when the VM reboots. */
1989	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
1990	break;
1991
1992	case PGM_PAGE_STATE_SHARED:
1993	case PGM_PAGE_STATE_WRITE_MONITORED:
1994	rc = pgmPhysPageMakeWritable(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1995	AssertLogRelRCReturn(rc, rc);
1996	/* no break */
1997
1998	case PGM_PAGE_STATE_ALLOCATED:
1999	{
2000	void *pvPage;
2001	rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
2002	AssertLogRelRCReturn(rc, rc);
2003	#ifndef NO_RAM_RESET
2004	ASMMemZeroPage(pvPage);
2005	#endif
2006	break;
2007	}
2008	}
2009	break;
2010
2011	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
2012	pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
2013	true /fDoAccounting/);
2014	break;
2015
2016	case PGMPAGETYPE_MMIO2:
2017	case PGMPAGETYPE_ROM_SHADOW:
2018	case PGMPAGETYPE_ROM:
2019	case PGMPAGETYPE_MMIO:
2020	break;
2021	default:
2022	AssertFailed();
2023
2024	}
2025	} /* for each page */
2026	}
2027
2028	}
2029
2030	/*
2031	* Finish off any pages pending freeing.
2032	*/
2033	if (cPendingPages)
2034	{
2035	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2036	AssertLogRelRCReturn(rc, rc);
2037	}
2038	GMMR3FreePagesCleanup(pReq);
2039
2040	return VINF_SUCCESS;
2041	}
2042
2043
2044	/**
2045	* Frees all RAM during VM termination
2046	*
2047	* ASSUMES that the caller owns the PGM lock.
2048	*
2049	* @returns VBox status code.
2050	* @param pVM Pointer to the VM.
2051	*/
2052	int pgmR3PhysRamTerm(PVM pVM)
2053	{
2054	PGM_LOCK_ASSERT_OWNER(pVM);
2055
2056	/* Reset the memory balloon. */
2057	int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
2058	AssertRC(rc);
2059
2060	#ifdef VBOX_WITH_PAGE_SHARING
2061	/*
2062	* Clear all registered shared modules.
2063	*/
2064	pgmR3PhysAssertSharedPageChecksums(pVM);
2065	rc = GMMR3ResetSharedModules(pVM);
2066	AssertRC(rc);
2067
2068	/*
2069	* Flush the handy pages updates to make sure no shared pages are hiding
2070	* in there. (No unlikely if the VM shuts down, apparently.)
2071	*/
2072	rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_FLUSH_HANDY_PAGES, 0, NULL);
2073	#endif
2074
2075	/*
2076	* We batch up pages that should be freed instead of calling GMM for
2077	* each and every one of them.
2078	*/
2079	uint32_t cPendingPages = 0;
2080	PGMMFREEPAGESREQ pReq;
2081	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
2082	AssertLogRelRCReturn(rc, rc);
2083
2084	/*
2085	* Walk the ram ranges.
2086	*/
2087	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
2088	{
2089	uint32_t iPage = pRam->cb >> PAGE_SHIFT;
2090	AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
2091
2092	while (iPage-- > 0)
2093	{
2094	PPGMPAGE pPage = &pRam->aPages[iPage];
2095	switch (PGM_PAGE_GET_TYPE(pPage))
2096	{
2097	case PGMPAGETYPE_RAM:
2098	/* Free all shared pages. Private pages are automatically freed during GMM VM cleanup. */
2099	/** @todo change this to explicitly free private pages here. */
2100	if (PGM_PAGE_IS_SHARED(pPage))
2101	{
2102	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
2103	AssertLogRelRCReturn(rc, rc);
2104	}
2105	break;
2106
2107	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
2108	case PGMPAGETYPE_MMIO2:
2109	case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
2110	case PGMPAGETYPE_ROM:
2111	case PGMPAGETYPE_MMIO:
2112	break;
2113	default:
2114	AssertFailed();
2115	}
2116	} /* for each page */
2117	}
2118
2119	/*
2120	* Finish off any pages pending freeing.
2121	*/
2122	if (cPendingPages)
2123	{
2124	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2125	AssertLogRelRCReturn(rc, rc);
2126	}
2127	GMMR3FreePagesCleanup(pReq);
2128	return VINF_SUCCESS;
2129	}
2130
2131	/**
2132	* This is the interface IOM is using to register an MMIO region.
2133	*
2134	* It will check for conflicts and ensure that a RAM range structure
2135	* is present before calling the PGMR3HandlerPhysicalRegister API to
2136	* register the callbacks.
2137	*
2138	* @returns VBox status code.
2139	*
2140	* @param pVM Pointer to the VM.
2141	* @param GCPhys The start of the MMIO region.
2142	* @param cb The size of the MMIO region.
2143	* @param pfnHandlerR3 The address of the ring-3 handler. (IOMR3MMIOHandler)
2144	* @param pvUserR3 The user argument for R3.
2145	* @param pfnHandlerR0 The address of the ring-0 handler. (IOMMMIOHandler)
2146	* @param pvUserR0 The user argument for R0.
2147	* @param pfnHandlerRC The address of the RC handler. (IOMMMIOHandler)
2148	* @param pvUserRC The user argument for RC.
2149	* @param pszDesc The description of the MMIO region.
2150	*/
2151	VMMR3DECL(int) PGMR3PhysMMIORegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb,
2152	R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnHandlerR3, RTR3PTR pvUserR3,
2153	R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnHandlerR0, RTR0PTR pvUserR0,
2154	RCPTRTYPE(PFNPGMRCPHYSHANDLER) pfnHandlerRC, RTRCPTR pvUserRC,
2155	R3PTRTYPE(const char *) pszDesc)
2156	{
2157	/*
2158	* Assert on some assumption.
2159	*/
2160	VM_ASSERT_EMT(pVM);
2161	AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2162	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2163	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
2164	AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
2165
2166	int rc = pgmLock(pVM);
2167	AssertRCReturn(rc, rc);
2168
2169	/*
2170	* Make sure there's a RAM range structure for the region.
2171	*/
2172	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
2173	bool fRamExists = false;
2174	PPGMRAMRANGE pRamPrev = NULL;
2175	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2176	while (pRam && GCPhysLast >= pRam->GCPhys)
2177	{
2178	if ( GCPhysLast >= pRam->GCPhys
2179	&& GCPhys <= pRam->GCPhysLast)
2180	{
2181	/* Simplification: all within the same range. */
2182	AssertLogRelMsgReturnStmt( GCPhys >= pRam->GCPhys
2183	&& GCPhysLast <= pRam->GCPhysLast,
2184	("%RGp-%RGp (MMIO/%s) falls partly outside %RGp-%RGp (%s)\n",
2185	GCPhys, GCPhysLast, pszDesc,
2186	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
2187	pgmUnlock(pVM),
2188	VERR_PGM_RAM_CONFLICT);
2189
2190	/* Check that it's all RAM or MMIO pages. */
2191	PCPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2192	uint32_t cLeft = cb >> PAGE_SHIFT;
2193	while (cLeft-- > 0)
2194	{
2195	AssertLogRelMsgReturnStmt( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2196	\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO,
2197	("%RGp-%RGp (MMIO/%s): %RGp is not a RAM or MMIO page - type=%d desc=%s\n",
2198	GCPhys, GCPhysLast, pszDesc, PGM_PAGE_GET_TYPE(pPage), pRam->pszDesc),
2199	pgmUnlock(pVM),
2200	VERR_PGM_RAM_CONFLICT);
2201	pPage++;
2202	}
2203
2204	/* Looks good. */
2205	fRamExists = true;
2206	break;
2207	}
2208
2209	/* next */
2210	pRamPrev = pRam;
2211	pRam = pRam->pNextR3;
2212	}
2213	PPGMRAMRANGE pNew;
2214	if (fRamExists)
2215	{
2216	pNew = NULL;
2217
2218	/*
2219	* Make all the pages in the range MMIO/ZERO pages, freeing any
2220	* RAM pages currently mapped here. This might not be 100% correct
2221	* for PCI memory, but we're doing the same thing for MMIO2 pages.
2222	*/
2223	rc = pgmR3PhysFreePageRange(pVM, pRam, GCPhys, GCPhysLast, PGMPAGETYPE_MMIO);
2224	AssertRCReturnStmt(rc, pgmUnlock(pVM), rc);
2225
2226	/* Force a PGM pool flush as guest ram references have been changed. */
2227	/** @todo not entirely SMP safe; assuming for now the guest takes
2228	* care of this internally (not touch mapped mmio while changing the
2229	* mapping). */
2230	PVMCPU pVCpu = VMMGetCpu(pVM);
2231	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2232	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2233	}
2234	else
2235	{
2236
2237	/*
2238	* No RAM range, insert an ad hoc one.
2239	*
2240	* Note that we don't have to tell REM about this range because
2241	* PGMHandlerPhysicalRegisterEx will do that for us.
2242	*/
2243	Log(("PGMR3PhysMMIORegister: Adding ad hoc MMIO range for %RGp-%RGp %s\n", GCPhys, GCPhysLast, pszDesc));
2244
2245	const uint32_t cPages = cb >> PAGE_SHIFT;
2246	const size_t cbRamRange = RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]);
2247	rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]), 16, MM_TAG_PGM_PHYS, (void **)&pNew);
2248	AssertLogRelMsgRCReturnStmt(rc, ("cbRamRange=%zu\n", cbRamRange), pgmUnlock(pVM), rc);
2249
2250	/* Initialize the range. */
2251	pNew->pSelfR0 = MMHyperCCToR0(pVM, pNew);
2252	pNew->pSelfRC = MMHyperCCToRC(pVM, pNew);
2253	pNew->GCPhys = GCPhys;
2254	pNew->GCPhysLast = GCPhysLast;
2255	pNew->cb = cb;
2256	pNew->pszDesc = pszDesc;
2257	pNew->fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO;
2258	pNew->pvR3 = NULL;
2259	pNew->paLSPages = NULL;
2260
2261	uint32_t iPage = cPages;
2262	while (iPage-- > 0)
2263	PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_MMIO);
2264	Assert(PGM_PAGE_GET_TYPE(&pNew->aPages[0]) == PGMPAGETYPE_MMIO);
2265
2266	/* update the page count stats. */
2267	pVM->pgm.s.cPureMmioPages += cPages;
2268	pVM->pgm.s.cAllPages += cPages;
2269
2270	/* link it */
2271	pgmR3PhysLinkRamRange(pVM, pNew, pRamPrev);
2272	}
2273
2274	/*
2275	* Register the access handler.
2276	*/
2277	rc = PGMHandlerPhysicalRegisterEx(pVM, PGMPHYSHANDLERTYPE_MMIO, GCPhys, GCPhysLast,
2278	pfnHandlerR3, pvUserR3,
2279	pfnHandlerR0, pvUserR0,
2280	pfnHandlerRC, pvUserRC, pszDesc);
2281	if ( RT_FAILURE(rc)
2282	&& !fRamExists)
2283	{
2284	pVM->pgm.s.cPureMmioPages -= cb >> PAGE_SHIFT;
2285	pVM->pgm.s.cAllPages -= cb >> PAGE_SHIFT;
2286
2287	/* remove the ad hoc range. */
2288	pgmR3PhysUnlinkRamRange2(pVM, pNew, pRamPrev);
2289	pNew->cb = pNew->GCPhys = pNew->GCPhysLast = NIL_RTGCPHYS;
2290	MMHyperFree(pVM, pRam);
2291	}
2292	pgmPhysInvalidatePageMapTLB(pVM);
2293
2294	pgmUnlock(pVM);
2295	return rc;
2296	}
2297
2298
2299	/**
2300	* This is the interface IOM is using to register an MMIO region.
2301	*
2302	* It will take care of calling PGMHandlerPhysicalDeregister and clean up
2303	* any ad hoc PGMRAMRANGE left behind.
2304	*
2305	* @returns VBox status code.
2306	* @param pVM Pointer to the VM.
2307	* @param GCPhys The start of the MMIO region.
2308	* @param cb The size of the MMIO region.
2309	*/
2310	VMMR3DECL(int) PGMR3PhysMMIODeregister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb)
2311	{
2312	VM_ASSERT_EMT(pVM);
2313
2314	int rc = pgmLock(pVM);
2315	AssertRCReturn(rc, rc);
2316
2317	/*
2318	* First deregister the handler, then check if we should remove the ram range.
2319	*/
2320	rc = PGMHandlerPhysicalDeregister(pVM, GCPhys);
2321	if (RT_SUCCESS(rc))
2322	{
2323	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
2324	PPGMRAMRANGE pRamPrev = NULL;
2325	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2326	while (pRam && GCPhysLast >= pRam->GCPhys)
2327	{
2328	/** @todo We're being a bit too careful here. rewrite. */
2329	if ( GCPhysLast == pRam->GCPhysLast
2330	&& GCPhys == pRam->GCPhys)
2331	{
2332	Assert(pRam->cb == cb);
2333
2334	/*
2335	* See if all the pages are dead MMIO pages.
2336	*/
2337	uint32_t const cPages = cb >> PAGE_SHIFT;
2338	bool fAllMMIO = true;
2339	uint32_t iPage = 0;
2340	uint32_t cLeft = cPages;
2341	while (cLeft-- > 0)
2342	{
2343	PPGMPAGE pPage = &pRam->aPages[iPage];
2344	if ( PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO
2345	/\|\| not-out-of-action later /)
2346	{
2347	fAllMMIO = false;
2348	Assert(PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO2_ALIAS_MMIO);
2349	AssertMsgFailed(("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
2350	break;
2351	}
2352	Assert(PGM_PAGE_IS_ZERO(pPage));
2353	pPage++;
2354	}
2355	if (fAllMMIO)
2356	{
2357	/*
2358	* Ad-hoc range, unlink and free it.
2359	*/
2360	Log(("PGMR3PhysMMIODeregister: Freeing ad hoc MMIO range for %RGp-%RGp %s\n",
2361	GCPhys, GCPhysLast, pRam->pszDesc));
2362
2363	pVM->pgm.s.cAllPages -= cPages;
2364	pVM->pgm.s.cPureMmioPages -= cPages;
2365
2366	pgmR3PhysUnlinkRamRange2(pVM, pRam, pRamPrev);
2367	pRam->cb = pRam->GCPhys = pRam->GCPhysLast = NIL_RTGCPHYS;
2368	MMHyperFree(pVM, pRam);
2369	break;
2370	}
2371	}
2372
2373	/*
2374	* Range match? It will all be within one range (see PGMAllHandler.cpp).
2375	*/
2376	if ( GCPhysLast >= pRam->GCPhys
2377	&& GCPhys <= pRam->GCPhysLast)
2378	{
2379	Assert(GCPhys >= pRam->GCPhys);
2380	Assert(GCPhysLast <= pRam->GCPhysLast);
2381
2382	/*
2383	* Turn the pages back into RAM pages.
2384	*/
2385	uint32_t iPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2386	uint32_t cLeft = cb >> PAGE_SHIFT;
2387	while (cLeft--)
2388	{
2389	PPGMPAGE pPage = &pRam->aPages[iPage];
2390	AssertMsg(PGM_PAGE_IS_MMIO(pPage), ("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
2391	AssertMsg(PGM_PAGE_IS_ZERO(pPage), ("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
2392	if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO)
2393	PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_RAM);
2394	}
2395	break;
2396	}
2397
2398	/* next */
2399	pRamPrev = pRam;
2400	pRam = pRam->pNextR3;
2401	}
2402	}
2403
2404	/* Force a PGM pool flush as guest ram references have been changed. */
2405	/** @todo Not entirely SMP safe; assuming for now the guest takes care of
2406	* this internally (not touch mapped mmio while changing the mapping). */
2407	PVMCPU pVCpu = VMMGetCpu(pVM);
2408	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2409	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2410
2411	pgmPhysInvalidatePageMapTLB(pVM);
2412	pgmPhysInvalidRamRangeTlbs(pVM);
2413	pgmUnlock(pVM);
2414	return rc;
2415	}
2416
2417
2418	/**
2419	* Locate a MMIO2 range.
2420	*
2421	* @returns Pointer to the MMIO2 range.
2422	* @param pVM Pointer to the VM.
2423	* @param pDevIns The device instance owning the region.
2424	* @param iRegion The region.
2425	*/
2426	DECLINLINE(PPGMMMIO2RANGE) pgmR3PhysMMIO2Find(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion)
2427	{
2428	/*
2429	* Search the list.
2430	*/
2431	for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
2432	if ( pCur->pDevInsR3 == pDevIns
2433	&& pCur->iRegion == iRegion)
2434	return pCur;
2435	return NULL;
2436	}
2437
2438
2439	/**
2440	* Allocate and register an MMIO2 region.
2441	*
2442	* As mentioned elsewhere, MMIO2 is just RAM spelled differently. It's RAM
2443	* associated with a device. It is also non-shared memory with a permanent
2444	* ring-3 mapping and page backing (presently).
2445	*
2446	* A MMIO2 range may overlap with base memory if a lot of RAM is configured for
2447	* the VM, in which case we'll drop the base memory pages. Presently we will
2448	* make no attempt to preserve anything that happens to be present in the base
2449	* memory that is replaced, this is of course incorrectly but it's too much
2450	* effort.
2451	*
2452	* @returns VBox status code.
2453	* @retval VINF_SUCCESS on success, *ppv pointing to the R3 mapping of the
2454	* memory.
2455	* @retval VERR_ALREADY_EXISTS if the region already exists.
2456	*
2457	* @param pVM Pointer to the VM.
2458	* @param pDevIns The device instance owning the region.
2459	* @param iRegion The region number. If the MMIO2 memory is a PCI
2460	* I/O region this number has to be the number of that
2461	* region. Otherwise it can be any number safe
2462	* UINT8_MAX.
2463	* @param cb The size of the region. Must be page aligned.
2464	* @param fFlags Reserved for future use, must be zero.
2465	* @param ppv Where to store the pointer to the ring-3 mapping of
2466	* the memory.
2467	* @param pszDesc The description.
2468	*/
2469	VMMR3DECL(int) PGMR3PhysMMIO2Register(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS cb, uint32_t fFlags, void *ppv, const char pszDesc)
2470	{
2471	/*
2472	* Validate input.
2473	*/
2474	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2475	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2476	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2477	AssertPtrReturn(ppv, VERR_INVALID_POINTER);
2478	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
2479	AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
2480	AssertReturn(pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion) == NULL, VERR_ALREADY_EXISTS);
2481	AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2482	AssertReturn(cb, VERR_INVALID_PARAMETER);
2483	AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
2484
2485	const uint32_t cPages = cb >> PAGE_SHIFT;
2486	AssertLogRelReturn(((RTGCPHYS)cPages << PAGE_SHIFT) == cb, VERR_INVALID_PARAMETER);
2487	AssertLogRelReturn(cPages <= INT32_MAX / 2, VERR_NO_MEMORY);
2488
2489	/*
2490	* For the 2nd+ instance, mangle the description string so it's unique.
2491	*/
2492	if (pDevIns->iInstance > 0) /** @todo Move to PDMDevHlp.cpp and use a real string cache. */
2493	{
2494	pszDesc = MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s [%u]", pszDesc, pDevIns->iInstance);
2495	if (!pszDesc)
2496	return VERR_NO_MEMORY;
2497	}
2498
2499	/*
2500	* Try reserve and allocate the backing memory first as this is what is
2501	* most likely to fail.
2502	*/
2503	int rc = MMR3AdjustFixedReservation(pVM, cPages, pszDesc);
2504	if (RT_SUCCESS(rc))
2505	{
2506	void *pvPages;
2507	PSUPPAGE paPages = (PSUPPAGE)RTMemTmpAlloc(cPages * sizeof(SUPPAGE));
2508	if (RT_SUCCESS(rc))
2509	rc = SUPR3PageAllocEx(cPages, 0 /fFlags/, &pvPages, NULL /pR0Ptr/, paPages);
2510	if (RT_SUCCESS(rc))
2511	{
2512	memset(pvPages, 0, cPages * PAGE_SIZE);
2513
2514	/*
2515	* Create the MMIO2 range record for it.
2516	*/
2517	const size_t cbRange = RT_OFFSETOF(PGMMMIO2RANGE, RamRange.aPages[cPages]);
2518	PPGMMMIO2RANGE pNew;
2519	rc = MMR3HyperAllocOnceNoRel(pVM, cbRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
2520	AssertLogRelMsgRC(rc, ("cbRamRange=%zu\n", cbRange));
2521	if (RT_SUCCESS(rc))
2522	{
2523	pNew->pDevInsR3 = pDevIns;
2524	pNew->pvR3 = pvPages;
2525	//pNew->pNext = NULL;
2526	//pNew->fMapped = false;
2527	//pNew->fOverlapping = false;
2528	pNew->iRegion = iRegion;
2529	pNew->idSavedState = UINT8_MAX;
2530	pNew->RamRange.pSelfR0 = MMHyperCCToR0(pVM, &pNew->RamRange);
2531	pNew->RamRange.pSelfRC = MMHyperCCToRC(pVM, &pNew->RamRange);
2532	pNew->RamRange.GCPhys = NIL_RTGCPHYS;
2533	pNew->RamRange.GCPhysLast = NIL_RTGCPHYS;
2534	pNew->RamRange.pszDesc = pszDesc;
2535	pNew->RamRange.cb = cb;
2536	pNew->RamRange.fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO2;
2537	pNew->RamRange.pvR3 = pvPages;
2538	//pNew->RamRange.paLSPages = NULL;
2539
2540	uint32_t iPage = cPages;
2541	while (iPage-- > 0)
2542	{
2543	PGM_PAGE_INIT(&pNew->RamRange.aPages[iPage],
2544	paPages[iPage].Phys, NIL_GMM_PAGEID,
2545	PGMPAGETYPE_MMIO2, PGM_PAGE_STATE_ALLOCATED);
2546	}
2547
2548	/* update page count stats */
2549	pVM->pgm.s.cAllPages += cPages;
2550	pVM->pgm.s.cPrivatePages += cPages;
2551
2552	/*
2553	* Link it into the list.
2554	* Since there is no particular order, just push it.
2555	*/
2556	pgmLock(pVM);
2557	pNew->pNextR3 = pVM->pgm.s.pMmio2RangesR3;
2558	pVM->pgm.s.pMmio2RangesR3 = pNew;
2559	pgmUnlock(pVM);
2560
2561	*ppv = pvPages;
2562	RTMemTmpFree(paPages);
2563	pgmPhysInvalidatePageMapTLB(pVM);
2564	return VINF_SUCCESS;
2565	}
2566
2567	SUPR3PageFreeEx(pvPages, cPages);
2568	}
2569	RTMemTmpFree(paPages);
2570	MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pszDesc);
2571	}
2572	if (pDevIns->iInstance > 0)
2573	MMR3HeapFree((void *)pszDesc);
2574	return rc;
2575	}
2576
2577
2578	/**
2579	* Deregisters and frees an MMIO2 region.
2580	*
2581	* Any physical (and virtual) access handlers registered for the region must
2582	* be deregistered before calling this function.
2583	*
2584	* @returns VBox status code.
2585	* @param pVM Pointer to the VM.
2586	* @param pDevIns The device instance owning the region.
2587	* @param iRegion The region. If it's UINT32_MAX it'll be a wildcard match.
2588	*/
2589	VMMR3DECL(int) PGMR3PhysMMIO2Deregister(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion)
2590	{
2591	/*
2592	* Validate input.
2593	*/
2594	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2595	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2596	AssertReturn(iRegion <= UINT8_MAX \|\| iRegion == UINT32_MAX, VERR_INVALID_PARAMETER);
2597
2598	pgmLock(pVM);
2599	int rc = VINF_SUCCESS;
2600	unsigned cFound = 0;
2601	PPGMMMIO2RANGE pPrev = NULL;
2602	PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3;
2603	while (pCur)
2604	{
2605	if ( pCur->pDevInsR3 == pDevIns
2606	&& ( iRegion == UINT32_MAX
2607	\|\| pCur->iRegion == iRegion))
2608	{
2609	cFound++;
2610
2611	/*
2612	* Unmap it if it's mapped.
2613	*/
2614	if (pCur->fMapped)
2615	{
2616	int rc2 = PGMR3PhysMMIO2Unmap(pVM, pCur->pDevInsR3, pCur->iRegion, pCur->RamRange.GCPhys);
2617	AssertRC(rc2);
2618	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2619	rc = rc2;
2620	}
2621
2622	/*
2623	* Unlink it
2624	*/
2625	PPGMMMIO2RANGE pNext = pCur->pNextR3;
2626	if (pPrev)
2627	pPrev->pNextR3 = pNext;
2628	else
2629	pVM->pgm.s.pMmio2RangesR3 = pNext;
2630	pCur->pNextR3 = NULL;
2631
2632	/*
2633	* Free the memory.
2634	*/
2635	int rc2 = SUPR3PageFreeEx(pCur->pvR3, pCur->RamRange.cb >> PAGE_SHIFT);
2636	AssertRC(rc2);
2637	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2638	rc = rc2;
2639
2640	uint32_t const cPages = pCur->RamRange.cb >> PAGE_SHIFT;
2641	rc2 = MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pCur->RamRange.pszDesc);
2642	AssertRC(rc2);
2643	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2644	rc = rc2;
2645
2646	/* we're leaking hyper memory here if done at runtime. */
2647	#ifdef VBOX_STRICT
2648	VMSTATE const enmState = VMR3GetState(pVM);
2649	AssertMsg( enmState == VMSTATE_POWERING_OFF
2650	\|\| enmState == VMSTATE_POWERING_OFF_LS
2651	\|\| enmState == VMSTATE_OFF
2652	\|\| enmState == VMSTATE_OFF_LS
2653	\|\| enmState == VMSTATE_DESTROYING
2654	\|\| enmState == VMSTATE_TERMINATED
2655	\|\| enmState == VMSTATE_CREATING
2656	, ("%s\n", VMR3GetStateName(enmState)));
2657	#endif
2658	/*rc = MMHyperFree(pVM, pCur);
2659	AssertRCReturn(rc, rc); - not safe, see the alloc call. */
2660
2661
2662	/* update page count stats */
2663	pVM->pgm.s.cAllPages -= cPages;
2664	pVM->pgm.s.cPrivatePages -= cPages;
2665
2666	/* next */
2667	pCur = pNext;
2668	}
2669	else
2670	{
2671	pPrev = pCur;
2672	pCur = pCur->pNextR3;
2673	}
2674	}
2675	pgmPhysInvalidatePageMapTLB(pVM);
2676	pgmUnlock(pVM);
2677	return !cFound && iRegion != UINT32_MAX ? VERR_NOT_FOUND : rc;
2678	}
2679
2680
2681	/**
2682	* Maps a MMIO2 region.
2683	*
2684	* This is done when a guest / the bios / state loading changes the
2685	* PCI config. The replacing of base memory has the same restrictions
2686	* as during registration, of course.
2687	*
2688	* @returns VBox status code.
2689	*
2690	* @param pVM Pointer to the VM.
2691	* @param pDevIns The device instance owning the region.
2692	*/
2693	VMMR3DECL(int) PGMR3PhysMMIO2Map(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
2694	{
2695	/*
2696	* Validate input
2697	*/
2698	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2699	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2700	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2701	AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
2702	AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
2703	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2704
2705	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2706	AssertReturn(pCur, VERR_NOT_FOUND);
2707	AssertReturn(!pCur->fMapped, VERR_WRONG_ORDER);
2708	Assert(pCur->RamRange.GCPhys == NIL_RTGCPHYS);
2709	Assert(pCur->RamRange.GCPhysLast == NIL_RTGCPHYS);
2710
2711	const RTGCPHYS GCPhysLast = GCPhys + pCur->RamRange.cb - 1;
2712	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
2713
2714	/*
2715	* Find our location in the ram range list, checking for
2716	* restriction we don't bother implementing yet (partially overlapping).
2717	*/
2718	bool fRamExists = false;
2719	PPGMRAMRANGE pRamPrev = NULL;
2720	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2721	while (pRam && GCPhysLast >= pRam->GCPhys)
2722	{
2723	if ( GCPhys <= pRam->GCPhysLast
2724	&& GCPhysLast >= pRam->GCPhys)
2725	{
2726	/* completely within? */
2727	AssertLogRelMsgReturn( GCPhys >= pRam->GCPhys
2728	&& GCPhysLast <= pRam->GCPhysLast,
2729	("%RGp-%RGp (MMIO2/%s) falls partly outside %RGp-%RGp (%s)\n",
2730	GCPhys, GCPhysLast, pCur->RamRange.pszDesc,
2731	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
2732	VERR_PGM_RAM_CONFLICT);
2733	fRamExists = true;
2734	break;
2735	}
2736
2737	/* next */
2738	pRamPrev = pRam;
2739	pRam = pRam->pNextR3;
2740	}
2741	if (fRamExists)
2742	{
2743	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2744	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2745	while (cPagesLeft-- > 0)
2746	{
2747	AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
2748	("%RGp isn't a RAM page (%d) - mapping %RGp-%RGp (MMIO2/%s).\n",
2749	GCPhys, PGM_PAGE_GET_TYPE(pPage), GCPhys, GCPhysLast, pCur->RamRange.pszDesc),
2750	VERR_PGM_RAM_CONFLICT);
2751	pPage++;
2752	}
2753	}
2754	Log(("PGMR3PhysMMIO2Map: %RGp-%RGp fRamExists=%RTbool %s\n",
2755	GCPhys, GCPhysLast, fRamExists, pCur->RamRange.pszDesc));
2756
2757	/*
2758	* Make the changes.
2759	*/
2760	pgmLock(pVM);
2761
2762	pCur->RamRange.GCPhys = GCPhys;
2763	pCur->RamRange.GCPhysLast = GCPhysLast;
2764	pCur->fMapped = true;
2765	pCur->fOverlapping = fRamExists;
2766
2767	if (fRamExists)
2768	{
2769	/** @todo use pgmR3PhysFreePageRange here. */
2770	uint32_t cPendingPages = 0;
2771	PGMMFREEPAGESREQ pReq;
2772	int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
2773	AssertLogRelRCReturn(rc, rc);
2774
2775	/* replace the pages, freeing all present RAM pages. */
2776	PPGMPAGE pPageSrc = &pCur->RamRange.aPages[0];
2777	PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2778	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2779	while (cPagesLeft-- > 0)
2780	{
2781	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys);
2782	AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
2783
2784	RTHCPHYS const HCPhys = PGM_PAGE_GET_HCPHYS(pPageSrc);
2785	PGM_PAGE_SET_HCPHYS(pVM, pPageDst, HCPhys);
2786	PGM_PAGE_SET_TYPE(pVM, pPageDst, PGMPAGETYPE_MMIO2);
2787	PGM_PAGE_SET_STATE(pVM, pPageDst, PGM_PAGE_STATE_ALLOCATED);
2788	PGM_PAGE_SET_PDE_TYPE(pVM, pPageDst, PGM_PAGE_PDE_TYPE_DONTCARE);
2789	PGM_PAGE_SET_PTE_INDEX(pVM, pPageDst, 0);
2790	PGM_PAGE_SET_TRACKING(pVM, pPageDst, 0);
2791
2792	pVM->pgm.s.cZeroPages--;
2793	GCPhys += PAGE_SIZE;
2794	pPageSrc++;
2795	pPageDst++;
2796	}
2797
2798	/* Flush physical page map TLB. */
2799	pgmPhysInvalidatePageMapTLB(pVM);
2800
2801	if (cPendingPages)
2802	{
2803	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2804	AssertLogRelRCReturn(rc, rc);
2805	}
2806	GMMR3FreePagesCleanup(pReq);
2807
2808	/* Force a PGM pool flush as guest ram references have been changed. */
2809	/** @todo not entirely SMP safe; assuming for now the guest takes care of
2810	* this internally (not touch mapped mmio while changing the mapping). */
2811	PVMCPU pVCpu = VMMGetCpu(pVM);
2812	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2813	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2814
2815	pgmUnlock(pVM);
2816	}
2817	else
2818	{
2819	RTGCPHYS cb = pCur->RamRange.cb;
2820
2821	/* Clear the tracking data of pages we're going to reactivate. */
2822	PPGMPAGE pPageSrc = &pCur->RamRange.aPages[0];
2823	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2824	while (cPagesLeft-- > 0)
2825	{
2826	PGM_PAGE_SET_TRACKING(pVM, pPageSrc, 0);
2827	PGM_PAGE_SET_PTE_INDEX(pVM, pPageSrc, 0);
2828	pPageSrc++;
2829	}
2830
2831	/* link in the ram range */
2832	pgmR3PhysLinkRamRange(pVM, &pCur->RamRange, pRamPrev);
2833	pgmUnlock(pVM);
2834
2835	#ifdef VBOX_WITH_REM
2836	REMR3NotifyPhysRamRegister(pVM, GCPhys, cb, REM_NOTIFY_PHYS_RAM_FLAGS_MMIO2);
2837	#endif
2838	}
2839
2840	pgmPhysInvalidatePageMapTLB(pVM);
2841	return VINF_SUCCESS;
2842	}
2843
2844
2845	/**
2846	* Unmaps a MMIO2 region.
2847	*
2848	* This is done when a guest / the bios / state loading changes the
2849	* PCI config. The replacing of base memory has the same restrictions
2850	* as during registration, of course.
2851	*/
2852	VMMR3DECL(int) PGMR3PhysMMIO2Unmap(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
2853	{
2854	/*
2855	* Validate input
2856	*/
2857	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2858	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2859	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2860	AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
2861	AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
2862	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2863
2864	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2865	AssertReturn(pCur, VERR_NOT_FOUND);
2866	AssertReturn(pCur->fMapped, VERR_WRONG_ORDER);
2867	AssertReturn(pCur->RamRange.GCPhys == GCPhys, VERR_INVALID_PARAMETER);
2868	Assert(pCur->RamRange.GCPhysLast != NIL_RTGCPHYS);
2869
2870	Log(("PGMR3PhysMMIO2Unmap: %RGp-%RGp %s\n",
2871	pCur->RamRange.GCPhys, pCur->RamRange.GCPhysLast, pCur->RamRange.pszDesc));
2872
2873	/*
2874	* Unmap it.
2875	*/
2876	pgmLock(pVM);
2877
2878	#ifdef VBOX_WITH_REM
2879	RTGCPHYS GCPhysRangeREM;
2880	RTGCPHYS cbRangeREM;
2881	bool fInformREM;
2882	#endif
2883	if (pCur->fOverlapping)
2884	{
2885	/* Restore the RAM pages we've replaced. */
2886	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2887	while (pRam->GCPhys > pCur->RamRange.GCPhysLast)
2888	pRam = pRam->pNextR3;
2889
2890	PPGMPAGE pPageDst = &pRam->aPages[(pCur->RamRange.GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2891	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2892	while (cPagesLeft-- > 0)
2893	{
2894	PGM_PAGE_INIT_ZERO(pPageDst, pVM, PGMPAGETYPE_RAM);
2895	pVM->pgm.s.cZeroPages++;
2896	pPageDst++;
2897	}
2898
2899	/* Flush physical page map TLB. */
2900	pgmPhysInvalidatePageMapTLB(pVM);
2901	#ifdef VBOX_WITH_REM
2902	GCPhysRangeREM = NIL_RTGCPHYS; /* shuts up gcc */
2903	cbRangeREM = RTGCPHYS_MAX; /* ditto */
2904	fInformREM = false;
2905	#endif
2906	}
2907	else
2908	{
2909	#ifdef VBOX_WITH_REM
2910	GCPhysRangeREM = pCur->RamRange.GCPhys;
2911	cbRangeREM = pCur->RamRange.cb;
2912	fInformREM = true;
2913	#endif
2914	pgmR3PhysUnlinkRamRange(pVM, &pCur->RamRange);
2915	}
2916
2917	pCur->RamRange.GCPhys = NIL_RTGCPHYS;
2918	pCur->RamRange.GCPhysLast = NIL_RTGCPHYS;
2919	pCur->fOverlapping = false;
2920	pCur->fMapped = false;
2921
2922	/* Force a PGM pool flush as guest ram references have been changed. */
2923	/** @todo not entirely SMP safe; assuming for now the guest takes care
2924	* of this internally (not touch mapped mmio while changing the
2925	* mapping). */
2926	PVMCPU pVCpu = VMMGetCpu(pVM);
2927	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2928	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2929
2930	pgmPhysInvalidatePageMapTLB(pVM);
2931	pgmPhysInvalidRamRangeTlbs(pVM);
2932	pgmUnlock(pVM);
2933
2934	#ifdef VBOX_WITH_REM
2935	if (fInformREM)
2936	REMR3NotifyPhysRamDeregister(pVM, GCPhysRangeREM, cbRangeREM);
2937	#endif
2938
2939	return VINF_SUCCESS;
2940	}
2941
2942
2943	/**
2944	* Checks if the given address is an MMIO2 base address or not.
2945	*
2946	* @returns true/false accordingly.
2947	* @param pVM Pointer to the VM.
2948	* @param pDevIns The owner of the memory, optional.
2949	* @param GCPhys The address to check.
2950	*/
2951	VMMR3DECL(bool) PGMR3PhysMMIO2IsBase(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys)
2952	{
2953	/*
2954	* Validate input
2955	*/
2956	VM_ASSERT_EMT_RETURN(pVM, false);
2957	AssertPtrReturn(pDevIns, false);
2958	AssertReturn(GCPhys != NIL_RTGCPHYS, false);
2959	AssertReturn(GCPhys != 0, false);
2960	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), false);
2961
2962	/*
2963	* Search the list.
2964	*/
2965	pgmLock(pVM);
2966	for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
2967	if (pCur->RamRange.GCPhys == GCPhys)
2968	{
2969	Assert(pCur->fMapped);
2970	pgmUnlock(pVM);
2971	return true;
2972	}
2973	pgmUnlock(pVM);
2974	return false;
2975	}
2976
2977
2978	/**
2979	* Gets the HC physical address of a page in the MMIO2 region.
2980	*
2981	* This is API is intended for MMHyper and shouldn't be called
2982	* by anyone else...
2983	*
2984	* @returns VBox status code.
2985	* @param pVM Pointer to the VM.
2986	* @param pDevIns The owner of the memory, optional.
2987	* @param iRegion The region.
2988	* @param off The page expressed an offset into the MMIO2 region.
2989	* @param pHCPhys Where to store the result.
2990	*/
2991	VMMR3DECL(int) PGMR3PhysMMIO2GetHCPhys(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, PRTHCPHYS pHCPhys)
2992	{
2993	/*
2994	* Validate input
2995	*/
2996	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2997	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2998	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2999
3000	pgmLock(pVM);
3001	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
3002	AssertReturn(pCur, VERR_NOT_FOUND);
3003	AssertReturn(off < pCur->RamRange.cb, VERR_INVALID_PARAMETER);
3004
3005	PCPGMPAGE pPage = &pCur->RamRange.aPages[off >> PAGE_SHIFT];
3006	*pHCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3007	pgmUnlock(pVM);
3008	return VINF_SUCCESS;
3009	}
3010
3011
3012	/**
3013	* Maps a portion of an MMIO2 region into kernel space (host).
3014	*
3015	* The kernel mapping will become invalid when the MMIO2 memory is deregistered
3016	* or the VM is terminated.
3017	*
3018	* @return VBox status code.
3019	*
3020	* @param pVM Pointer to the VM.
3021	* @param pDevIns The device owning the MMIO2 memory.
3022	* @param iRegion The region.
3023	* @param off The offset into the region. Must be page aligned.
3024	* @param cb The number of bytes to map. Must be page aligned.
3025	* @param pszDesc Mapping description.
3026	* @param pR0Ptr Where to store the R0 address.
3027	*/
3028	VMMR3DECL(int) PGMR3PhysMMIO2MapKernel(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, RTGCPHYS cb,
3029	const char *pszDesc, PRTR0PTR pR0Ptr)
3030	{
3031	/*
3032	* Validate input.
3033	*/
3034	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
3035	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3036	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
3037
3038	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
3039	AssertReturn(pCur, VERR_NOT_FOUND);
3040	AssertReturn(off < pCur->RamRange.cb, VERR_INVALID_PARAMETER);
3041	AssertReturn(cb <= pCur->RamRange.cb, VERR_INVALID_PARAMETER);
3042	AssertReturn(off + cb <= pCur->RamRange.cb, VERR_INVALID_PARAMETER);
3043	NOREF(pszDesc);
3044
3045	/*
3046	* Pass the request on to the support library/driver.
3047	*/
3048	int rc = SUPR3PageMapKernel(pCur->pvR3, off, cb, 0, pR0Ptr);
3049
3050	return rc;
3051	}
3052
3053
3054	/**
3055	* Worker for PGMR3PhysRomRegister.
3056	*
3057	* This is here to simplify lock management, i.e. the caller does all the
3058	* locking and we can simply return without needing to remember to unlock
3059	* anything first.
3060	*
3061	* @returns VBox status.
3062	* @param pVM Pointer to the VM.
3063	* @param pDevIns The device instance owning the ROM.
3064	* @param GCPhys First physical address in the range.
3065	* Must be page aligned!
3066	* @param cb The size of the range (in bytes).
3067	* Must be page aligned!
3068	* @param pvBinary Pointer to the binary data backing the ROM image.
3069	* @param cbBinary The size of the binary data pvBinary points to.
3070	* This must be less or equal to @a cb.
3071	* @param fFlags Mask of flags. PGMPHYS_ROM_FLAGS_SHADOWED
3072	* and/or PGMPHYS_ROM_FLAGS_PERMANENT_BINARY.
3073	* @param pszDesc Pointer to description string. This must not be freed.
3074	*/
3075	static int pgmR3PhysRomRegister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
3076	const void pvBinary, uint32_t cbBinary, uint32_t fFlags, const char pszDesc)
3077	{
3078	/*
3079	* Validate input.
3080	*/
3081	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3082	AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
3083	AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
3084	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
3085	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
3086	AssertPtrReturn(pvBinary, VERR_INVALID_PARAMETER);
3087	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
3088	AssertReturn(!(fFlags & ~(PGMPHYS_ROM_FLAGS_SHADOWED \| PGMPHYS_ROM_FLAGS_PERMANENT_BINARY)), VERR_INVALID_PARAMETER);
3089	VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
3090
3091	const uint32_t cPages = cb >> PAGE_SHIFT;
3092
3093	/*
3094	* Find the ROM location in the ROM list first.
3095	*/
3096	PPGMROMRANGE pRomPrev = NULL;
3097	PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3;
3098	while (pRom && GCPhysLast >= pRom->GCPhys)
3099	{
3100	if ( GCPhys <= pRom->GCPhysLast
3101	&& GCPhysLast >= pRom->GCPhys)
3102	AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
3103	GCPhys, GCPhysLast, pszDesc,
3104	pRom->GCPhys, pRom->GCPhysLast, pRom->pszDesc),
3105	VERR_PGM_RAM_CONFLICT);
3106	/* next */
3107	pRomPrev = pRom;
3108	pRom = pRom->pNextR3;
3109	}
3110
3111	/*
3112	* Find the RAM location and check for conflicts.
3113	*
3114	* Conflict detection is a bit different than for RAM
3115	* registration since a ROM can be located within a RAM
3116	* range. So, what we have to check for is other memory
3117	* types (other than RAM that is) and that we don't span
3118	* more than one RAM range (layz).
3119	*/
3120	bool fRamExists = false;
3121	PPGMRAMRANGE pRamPrev = NULL;
3122	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
3123	while (pRam && GCPhysLast >= pRam->GCPhys)
3124	{
3125	if ( GCPhys <= pRam->GCPhysLast
3126	&& GCPhysLast >= pRam->GCPhys)
3127	{
3128	/* completely within? */
3129	AssertLogRelMsgReturn( GCPhys >= pRam->GCPhys
3130	&& GCPhysLast <= pRam->GCPhysLast,
3131	("%RGp-%RGp (%s) falls partly outside %RGp-%RGp (%s)\n",
3132	GCPhys, GCPhysLast, pszDesc,
3133	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
3134	VERR_PGM_RAM_CONFLICT);
3135	fRamExists = true;
3136	break;
3137	}
3138
3139	/* next */
3140	pRamPrev = pRam;
3141	pRam = pRam->pNextR3;
3142	}
3143	if (fRamExists)
3144	{
3145	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
3146	uint32_t cPagesLeft = cPages;
3147	while (cPagesLeft-- > 0)
3148	{
3149	AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
3150	("%RGp (%R[pgmpage]) isn't a RAM page - registering %RGp-%RGp (%s).\n",
3151	pRam->GCPhys + ((RTGCPHYS)(uintptr_t)(pPage - &pRam->aPages[0]) << PAGE_SHIFT),
3152	pPage, GCPhys, GCPhysLast, pszDesc), VERR_PGM_RAM_CONFLICT);
3153	Assert(PGM_PAGE_IS_ZERO(pPage));
3154	pPage++;
3155	}
3156	}
3157
3158	/*
3159	* Update the base memory reservation if necessary.
3160	*/
3161	uint32_t cExtraBaseCost = fRamExists ? 0 : cPages;
3162	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3163	cExtraBaseCost += cPages;
3164	if (cExtraBaseCost)
3165	{
3166	int rc = MMR3IncreaseBaseReservation(pVM, cExtraBaseCost);
3167	if (RT_FAILURE(rc))
3168	return rc;
3169	}
3170
3171	/*
3172	* Allocate memory for the virgin copy of the RAM.
3173	*/
3174	PGMMALLOCATEPAGESREQ pReq;
3175	int rc = GMMR3AllocatePagesPrepare(pVM, &pReq, cPages, GMMACCOUNT_BASE);
3176	AssertRCReturn(rc, rc);
3177
3178	for (uint32_t iPage = 0; iPage < cPages; iPage++)
3179	{
3180	pReq->aPages[iPage].HCPhysGCPhys = GCPhys + (iPage << PAGE_SHIFT);
3181	pReq->aPages[iPage].idPage = NIL_GMM_PAGEID;
3182	pReq->aPages[iPage].idSharedPage = NIL_GMM_PAGEID;
3183	}
3184
3185	rc = GMMR3AllocatePagesPerform(pVM, pReq);
3186	if (RT_FAILURE(rc))
3187	{
3188	GMMR3AllocatePagesCleanup(pReq);
3189	return rc;
3190	}
3191
3192	/*
3193	* Allocate the new ROM range and RAM range (if necessary).
3194	*/
3195	PPGMROMRANGE pRomNew;
3196	rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMROMRANGE, aPages[cPages]), 0, MM_TAG_PGM_PHYS, (void **)&pRomNew);
3197	if (RT_SUCCESS(rc))
3198	{
3199	PPGMRAMRANGE pRamNew = NULL;
3200	if (!fRamExists)
3201	rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]), sizeof(PGMPAGE), MM_TAG_PGM_PHYS, (void **)&pRamNew);
3202	if (RT_SUCCESS(rc))
3203	{
3204	/*
3205	* Initialize and insert the RAM range (if required).
3206	*/
3207	PPGMROMPAGE pRomPage = &pRomNew->aPages[0];
3208	if (!fRamExists)
3209	{
3210	pRamNew->pSelfR0 = MMHyperCCToR0(pVM, pRamNew);
3211	pRamNew->pSelfRC = MMHyperCCToRC(pVM, pRamNew);
3212	pRamNew->GCPhys = GCPhys;
3213	pRamNew->GCPhysLast = GCPhysLast;
3214	pRamNew->cb = cb;
3215	pRamNew->pszDesc = pszDesc;
3216	pRamNew->fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_ROM;
3217	pRamNew->pvR3 = NULL;
3218	pRamNew->paLSPages = NULL;
3219
3220	PPGMPAGE pPage = &pRamNew->aPages[0];
3221	for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
3222	{
3223	PGM_PAGE_INIT(pPage,
3224	pReq->aPages[iPage].HCPhysGCPhys,
3225	pReq->aPages[iPage].idPage,
3226	PGMPAGETYPE_ROM,
3227	PGM_PAGE_STATE_ALLOCATED);
3228
3229	pRomPage->Virgin = *pPage;
3230	}
3231
3232	pVM->pgm.s.cAllPages += cPages;
3233	pgmR3PhysLinkRamRange(pVM, pRamNew, pRamPrev);
3234	}
3235	else
3236	{
3237	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
3238	for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
3239	{
3240	PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_ROM);
3241	PGM_PAGE_SET_HCPHYS(pVM, pPage, pReq->aPages[iPage].HCPhysGCPhys);
3242	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
3243	PGM_PAGE_SET_PAGEID(pVM, pPage, pReq->aPages[iPage].idPage);
3244	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
3245	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
3246	PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
3247
3248	pRomPage->Virgin = *pPage;
3249	}
3250
3251	pRamNew = pRam;
3252
3253	pVM->pgm.s.cZeroPages -= cPages;
3254	}
3255	pVM->pgm.s.cPrivatePages += cPages;
3256
3257	/* Flush physical page map TLB. */
3258	pgmPhysInvalidatePageMapTLB(pVM);
3259
3260
3261	/*
3262	* !HACK ALERT! REM + (Shadowed) ROM ==> mess.
3263	*
3264	* If it's shadowed we'll register the handler after the ROM notification
3265	* so we get the access handler callbacks that we should. If it isn't
3266	* shadowed we'll do it the other way around to make REM use the built-in
3267	* ROM behavior and not the handler behavior (which is to route all access
3268	* to PGM atm).
3269	*/
3270	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3271	{
3272	#ifdef VBOX_WITH_REM
3273	REMR3NotifyPhysRomRegister(pVM, GCPhys, cb, NULL, true /* fShadowed */);
3274	#endif
3275	rc = PGMR3HandlerPhysicalRegister(pVM, PGMPHYSHANDLERTYPE_PHYSICAL_WRITE, GCPhys, GCPhysLast,
3276	pgmR3PhysRomWriteHandler, pRomNew,
3277	NULL, "pgmPhysRomWriteHandler", MMHyperCCToR0(pVM, pRomNew),
3278	NULL, "pgmPhysRomWriteHandler", MMHyperCCToRC(pVM, pRomNew), pszDesc);
3279	}
3280	else
3281	{
3282	rc = PGMR3HandlerPhysicalRegister(pVM, PGMPHYSHANDLERTYPE_PHYSICAL_WRITE, GCPhys, GCPhysLast,
3283	pgmR3PhysRomWriteHandler, pRomNew,
3284	NULL, "pgmPhysRomWriteHandler", MMHyperCCToR0(pVM, pRomNew),
3285	NULL, "pgmPhysRomWriteHandler", MMHyperCCToRC(pVM, pRomNew), pszDesc);
3286	#ifdef VBOX_WITH_REM
3287	REMR3NotifyPhysRomRegister(pVM, GCPhys, cb, NULL, false /* fShadowed */);
3288	#endif
3289	}
3290	if (RT_SUCCESS(rc))
3291	{
3292	/*
3293	* Copy the image over to the virgin pages.
3294	* This must be done after linking in the RAM range.
3295	*/
3296	size_t cbBinaryLeft = cbBinary;
3297	PPGMPAGE pRamPage = &pRamNew->aPages[(GCPhys - pRamNew->GCPhys) >> PAGE_SHIFT];
3298	for (uint32_t iPage = 0; iPage < cPages; iPage++, pRamPage++)
3299	{
3300	void *pvDstPage;
3301	rc = pgmPhysPageMap(pVM, pRamPage, GCPhys + (iPage << PAGE_SHIFT), &pvDstPage);
3302	if (RT_FAILURE(rc))
3303	{
3304	VMSetError(pVM, rc, RT_SRC_POS, "Failed to map virgin ROM page at %RGp", GCPhys);
3305	break;
3306	}
3307	if (cbBinaryLeft >= PAGE_SIZE)
3308	{
3309	memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << PAGE_SHIFT), PAGE_SIZE);
3310	cbBinaryLeft -= PAGE_SIZE;
3311	}
3312	else
3313	{
3314	ASMMemZeroPage(pvDstPage); /* (shouldn't be necessary, but can't hurt either) */
3315	if (cbBinaryLeft > 0)
3316	{
3317	memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << PAGE_SHIFT), cbBinaryLeft);
3318	cbBinaryLeft = 0;
3319	}
3320	}
3321	}
3322	if (RT_SUCCESS(rc))
3323	{
3324	/*
3325	* Initialize the ROM range.
3326	* Note that the Virgin member of the pages has already been initialized above.
3327	*/
3328	pRomNew->GCPhys = GCPhys;
3329	pRomNew->GCPhysLast = GCPhysLast;
3330	pRomNew->cb = cb;
3331	pRomNew->fFlags = fFlags;
3332	pRomNew->idSavedState = UINT8_MAX;
3333	pRomNew->cbOriginal = cbBinary;
3334	#ifdef VBOX_STRICT
3335	pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY
3336	? pvBinary : RTMemDup(pvBinary, cbBinary);
3337	#else
3338	pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY ? pvBinary : NULL;
3339	#endif
3340	pRomNew->pszDesc = pszDesc;
3341
3342	for (unsigned iPage = 0; iPage < cPages; iPage++)
3343	{
3344	PPGMROMPAGE pPage = &pRomNew->aPages[iPage];
3345	pPage->enmProt = PGMROMPROT_READ_ROM_WRITE_IGNORE;
3346	PGM_PAGE_INIT_ZERO(&pPage->Shadow, pVM, PGMPAGETYPE_ROM_SHADOW);
3347	}
3348
3349	/* update the page count stats for the shadow pages. */
3350	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3351	{
3352	pVM->pgm.s.cZeroPages += cPages;
3353	pVM->pgm.s.cAllPages += cPages;
3354	}
3355
3356	/*
3357	* Insert the ROM range, tell REM and return successfully.
3358	*/
3359	pRomNew->pNextR3 = pRom;
3360	pRomNew->pNextR0 = pRom ? MMHyperCCToR0(pVM, pRom) : NIL_RTR0PTR;
3361	pRomNew->pNextRC = pRom ? MMHyperCCToRC(pVM, pRom) : NIL_RTRCPTR;
3362
3363	if (pRomPrev)
3364	{
3365	pRomPrev->pNextR3 = pRomNew;
3366	pRomPrev->pNextR0 = MMHyperCCToR0(pVM, pRomNew);
3367	pRomPrev->pNextRC = MMHyperCCToRC(pVM, pRomNew);
3368	}
3369	else
3370	{
3371	pVM->pgm.s.pRomRangesR3 = pRomNew;
3372	pVM->pgm.s.pRomRangesR0 = MMHyperCCToR0(pVM, pRomNew);
3373	pVM->pgm.s.pRomRangesRC = MMHyperCCToRC(pVM, pRomNew);
3374	}
3375
3376	pgmPhysInvalidatePageMapTLB(pVM);
3377	GMMR3AllocatePagesCleanup(pReq);
3378	return VINF_SUCCESS;
3379	}
3380
3381	/* bail out */
3382
3383	int rc2 = PGMHandlerPhysicalDeregister(pVM, GCPhys);
3384	AssertRC(rc2);
3385	}
3386
3387	if (!fRamExists)
3388	{
3389	pgmR3PhysUnlinkRamRange2(pVM, pRamNew, pRamPrev);
3390	MMHyperFree(pVM, pRamNew);
3391	}
3392	}
3393	MMHyperFree(pVM, pRomNew);
3394	}
3395
3396	/** @todo Purge the mapping cache or something... */
3397	GMMR3FreeAllocatedPages(pVM, pReq);
3398	GMMR3AllocatePagesCleanup(pReq);
3399	return rc;
3400	}
3401
3402
3403	/**
3404	* Registers a ROM image.
3405	*
3406	* Shadowed ROM images requires double the amount of backing memory, so,
3407	* don't use that unless you have to. Shadowing of ROM images is process
3408	* where we can select where the reads go and where the writes go. On real
3409	* hardware the chipset provides means to configure this. We provide
3410	* PGMR3PhysProtectROM() for this purpose.
3411	*
3412	* A read-only copy of the ROM image will always be kept around while we
3413	* will allocate RAM pages for the changes on demand (unless all memory
3414	* is configured to be preallocated).
3415	*
3416	* @returns VBox status.
3417	* @param pVM Pointer to the VM.
3418	* @param pDevIns The device instance owning the ROM.
3419	* @param GCPhys First physical address in the range.
3420	* Must be page aligned!
3421	* @param cb The size of the range (in bytes).
3422	* Must be page aligned!
3423	* @param pvBinary Pointer to the binary data backing the ROM image.
3424	* @param cbBinary The size of the binary data pvBinary points to.
3425	* This must be less or equal to @a cb.
3426	* @param fFlags Mask of flags. PGMPHYS_ROM_FLAGS_SHADOWED
3427	* and/or PGMPHYS_ROM_FLAGS_PERMANENT_BINARY.
3428	* @param pszDesc Pointer to description string. This must not be freed.
3429	*
3430	* @remark There is no way to remove the rom, automatically on device cleanup or
3431	* manually from the device yet. This isn't difficult in any way, it's
3432	* just not something we expect to be necessary for a while.
3433	*/
3434	VMMR3DECL(int) PGMR3PhysRomRegister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
3435	const void pvBinary, uint32_t cbBinary, uint32_t fFlags, const char pszDesc)
3436	{
3437	Log(("PGMR3PhysRomRegister: pDevIns=%p GCPhys=%RGp(-%RGp) cb=%RGp pvBinary=%p cbBinary=%#x fFlags=%#x pszDesc=%s\n",
3438	pDevIns, GCPhys, GCPhys + cb, cb, pvBinary, cbBinary, fFlags, pszDesc));
3439	pgmLock(pVM);
3440	int rc = pgmR3PhysRomRegister(pVM, pDevIns, GCPhys, cb, pvBinary, cbBinary, fFlags, pszDesc);
3441	pgmUnlock(pVM);
3442	return rc;
3443	}
3444
3445
3446	/**
3447	* \#PF Handler callback for ROM write accesses.
3448	*
3449	* @returns VINF_SUCCESS if the handler have carried out the operation.
3450	* @returns VINF_PGM_HANDLER_DO_DEFAULT if the caller should carry out the access operation.
3451	* @param pVM Pointer to the VM.
3452	* @param GCPhys The physical address the guest is writing to.
3453	* @param pvPhys The HC mapping of that address.
3454	* @param pvBuf What the guest is reading/writing.
3455	* @param cbBuf How much it's reading/writing.
3456	* @param enmAccessType The access type.
3457	* @param pvUser User argument.
3458	*/
3459	static DECLCALLBACK(int) pgmR3PhysRomWriteHandler(PVM pVM, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf,
3460	PGMACCESSTYPE enmAccessType, void *pvUser)
3461	{
3462	PPGMROMRANGE pRom = (PPGMROMRANGE)pvUser;
3463	const uint32_t iPage = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
3464	Assert(iPage < (pRom->cb >> PAGE_SHIFT));
3465	PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
3466	Log5(("pgmR3PhysRomWriteHandler: %d %c %#08RGp %#04zx\n", pRomPage->enmProt, enmAccessType == PGMACCESSTYPE_READ ? 'R' : 'W', GCPhys, cbBuf));
3467	NOREF(pvPhys);
3468
3469	if (enmAccessType == PGMACCESSTYPE_READ)
3470	{
3471	switch (pRomPage->enmProt)
3472	{
3473	/*
3474	* Take the default action.
3475	*/
3476	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
3477	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
3478	case PGMROMPROT_READ_ROM_WRITE_RAM:
3479	case PGMROMPROT_READ_RAM_WRITE_RAM:
3480	return VINF_PGM_HANDLER_DO_DEFAULT;
3481
3482	default:
3483	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
3484	pRom->aPages[iPage].enmProt, iPage, GCPhys),
3485	VERR_IPE_NOT_REACHED_DEFAULT_CASE);
3486	}
3487	}
3488	else
3489	{
3490	Assert(enmAccessType == PGMACCESSTYPE_WRITE);
3491	switch (pRomPage->enmProt)
3492	{
3493	/*
3494	* Ignore writes.
3495	*/
3496	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
3497	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
3498	return VINF_SUCCESS;
3499
3500	/*
3501	* Write to the RAM page.
3502	*/
3503	case PGMROMPROT_READ_ROM_WRITE_RAM:
3504	case PGMROMPROT_READ_RAM_WRITE_RAM: /* yes this will get here too, it's way simpler that way. */
3505	{
3506	/* This should be impossible now, pvPhys doesn't work cross page anylonger. */
3507	Assert(((GCPhys - pRom->GCPhys + cbBuf - 1) >> PAGE_SHIFT) == iPage);
3508
3509	/*
3510	* Take the lock, do lazy allocation, map the page and copy the data.
3511	*
3512	* Note that we have to bypass the mapping TLB since it works on
3513	* guest physical addresses and entering the shadow page would
3514	* kind of screw things up...
3515	*/
3516	int rc = pgmLock(pVM);
3517	AssertRC(rc);
3518
3519	PPGMPAGE pShadowPage = &pRomPage->Shadow;
3520	if (!PGMROMPROT_IS_ROM(pRomPage->enmProt))
3521	{
3522	pShadowPage = pgmPhysGetPage(pVM, GCPhys);
3523	AssertLogRelReturn(pShadowPage, VERR_PGM_PHYS_PAGE_GET_IPE);
3524	}
3525
3526	void *pvDstPage;
3527	rc = pgmPhysPageMakeWritableAndMap(pVM, pShadowPage, GCPhys & X86_PTE_PG_MASK, &pvDstPage);
3528	if (RT_SUCCESS(rc))
3529	{
3530	memcpy((uint8_t *)pvDstPage + (GCPhys & PAGE_OFFSET_MASK), pvBuf, cbBuf);
3531	pRomPage->LiveSave.fWrittenTo = true;
3532	}
3533
3534	pgmUnlock(pVM);
3535	return rc;
3536	}
3537
3538	default:
3539	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
3540	pRom->aPages[iPage].enmProt, iPage, GCPhys),
3541	VERR_IPE_NOT_REACHED_DEFAULT_CASE);
3542	}
3543	}
3544	}
3545
3546
3547	/**
3548	* Called by PGMR3Reset to reset the shadow, switch to the virgin,
3549	* and verify that the virgin part is untouched.
3550	*
3551	* This is done after the normal memory has been cleared.
3552	*
3553	* ASSUMES that the caller owns the PGM lock.
3554	*
3555	* @param pVM Pointer to the VM.
3556	*/
3557	int pgmR3PhysRomReset(PVM pVM)
3558	{
3559	PGM_LOCK_ASSERT_OWNER(pVM);
3560	for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3561	{
3562	const uint32_t cPages = pRom->cb >> PAGE_SHIFT;
3563
3564	if (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3565	{
3566	/*
3567	* Reset the physical handler.
3568	*/
3569	int rc = PGMR3PhysRomProtect(pVM, pRom->GCPhys, pRom->cb, PGMROMPROT_READ_ROM_WRITE_IGNORE);
3570	AssertRCReturn(rc, rc);
3571
3572	/*
3573	* What we do with the shadow pages depends on the memory
3574	* preallocation option. If not enabled, we'll just throw
3575	* out all the dirty pages and replace them by the zero page.
3576	*/
3577	if (!pVM->pgm.s.fRamPreAlloc)
3578	{
3579	/* Free the dirty pages. */
3580	uint32_t cPendingPages = 0;
3581	PGMMFREEPAGESREQ pReq;
3582	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
3583	AssertRCReturn(rc, rc);
3584
3585	for (uint32_t iPage = 0; iPage < cPages; iPage++)
3586	if ( !PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow)
3587	&& !PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow))
3588	{
3589	Assert(PGM_PAGE_GET_STATE(&pRom->aPages[iPage].Shadow) == PGM_PAGE_STATE_ALLOCATED);
3590	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, &pRom->aPages[iPage].Shadow,
3591	pRom->GCPhys + (iPage << PAGE_SHIFT));
3592	AssertLogRelRCReturn(rc, rc);
3593	}
3594
3595	if (cPendingPages)
3596	{
3597	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
3598	AssertLogRelRCReturn(rc, rc);
3599	}
3600	GMMR3FreePagesCleanup(pReq);
3601	}
3602	else
3603	{
3604	/* clear all the shadow pages. */
3605	for (uint32_t iPage = 0; iPage < cPages; iPage++)
3606	{
3607	if (PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow))
3608	continue;
3609	Assert(!PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow));
3610	void *pvDstPage;
3611	const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
3612	rc = pgmPhysPageMakeWritableAndMap(pVM, &pRom->aPages[iPage].Shadow, GCPhys, &pvDstPage);
3613	if (RT_FAILURE(rc))
3614	break;
3615	ASMMemZeroPage(pvDstPage);
3616	}
3617	AssertRCReturn(rc, rc);
3618	}
3619	}
3620
3621	#ifdef VBOX_STRICT
3622	/*
3623	* Verify that the virgin page is unchanged if possible.
3624	*/
3625	if (pRom->pvOriginal)
3626	{
3627	size_t cbSrcLeft = pRom->cbOriginal;
3628	uint8_t const pbSrcPage = (uint8_t const )pRom->pvOriginal;
3629	for (uint32_t iPage = 0; iPage < cPages && cbSrcLeft > 0; iPage++, pbSrcPage += PAGE_SIZE)
3630	{
3631	const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
3632	void const *pvDstPage;
3633	int rc = pgmPhysPageMapReadOnly(pVM, &pRom->aPages[iPage].Virgin, GCPhys, &pvDstPage);
3634	if (RT_FAILURE(rc))
3635	break;
3636
3637	if (memcmp(pvDstPage, pbSrcPage, RT_MIN(cbSrcLeft, PAGE_SIZE)))
3638	LogRel(("pgmR3PhysRomReset: %RGp rom page changed (%s) - loaded saved state?\n",
3639	GCPhys, pRom->pszDesc));
3640	cbSrcLeft -= RT_MIN(cbSrcLeft, PAGE_SIZE);
3641	}
3642	}
3643	#endif
3644	}
3645
3646	return VINF_SUCCESS;
3647	}
3648
3649
3650	/**
3651	* Called by PGMR3Term to free resources.
3652	*
3653	* ASSUMES that the caller owns the PGM lock.
3654	*
3655	* @param pVM Pointer to the VM.
3656	*/
3657	void pgmR3PhysRomTerm(PVM pVM)
3658	{
3659	#ifdef RT_STRICT
3660	/*
3661	* Free the heap copy of the original bits.
3662	*/
3663	for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3664	{
3665	if ( pRom->pvOriginal
3666	&& !(pRom->fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY))
3667	{
3668	RTMemFree((void *)pRom->pvOriginal);
3669	pRom->pvOriginal = NULL;
3670	}
3671	}
3672	#endif
3673	}
3674
3675
3676	/**
3677	* Change the shadowing of a range of ROM pages.
3678	*
3679	* This is intended for implementing chipset specific memory registers
3680	* and will not be very strict about the input. It will silently ignore
3681	* any pages that are not the part of a shadowed ROM.
3682	*
3683	* @returns VBox status code.
3684	* @retval VINF_PGM_SYNC_CR3
3685	*
3686	* @param pVM Pointer to the VM.
3687	* @param GCPhys Where to start. Page aligned.
3688	* @param cb How much to change. Page aligned.
3689	* @param enmProt The new ROM protection.
3690	*/
3691	VMMR3DECL(int) PGMR3PhysRomProtect(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, PGMROMPROT enmProt)
3692	{
3693	/*
3694	* Check input
3695	*/
3696	if (!cb)
3697	return VINF_SUCCESS;
3698	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3699	AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3700	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
3701	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
3702	AssertReturn(enmProt >= PGMROMPROT_INVALID && enmProt <= PGMROMPROT_END, VERR_INVALID_PARAMETER);
3703
3704	/*
3705	* Process the request.
3706	*/
3707	pgmLock(pVM);
3708	int rc = VINF_SUCCESS;
3709	bool fFlushTLB = false;
3710	for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3711	{
3712	if ( GCPhys <= pRom->GCPhysLast
3713	&& GCPhysLast >= pRom->GCPhys
3714	&& (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED))
3715	{
3716	/*
3717	* Iterate the relevant pages and make necessary the changes.
3718	*/
3719	bool fChanges = false;
3720	uint32_t const cPages = pRom->GCPhysLast <= GCPhysLast
3721	? pRom->cb >> PAGE_SHIFT
3722	: (GCPhysLast - pRom->GCPhys + 1) >> PAGE_SHIFT;
3723	for (uint32_t iPage = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
3724	iPage < cPages;
3725	iPage++)
3726	{
3727	PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
3728	if (PGMROMPROT_IS_ROM(pRomPage->enmProt) != PGMROMPROT_IS_ROM(enmProt))
3729	{
3730	fChanges = true;
3731
3732	/* flush references to the page. */
3733	PPGMPAGE pRamPage = pgmPhysGetPage(pVM, pRom->GCPhys + (iPage << PAGE_SHIFT));
3734	int rc2 = pgmPoolTrackUpdateGCPhys(pVM, pRom->GCPhys + (iPage << PAGE_SHIFT), pRamPage,
3735	true /fFlushPTEs/, &fFlushTLB);
3736	if (rc2 != VINF_SUCCESS && (rc == VINF_SUCCESS \|\| RT_FAILURE(rc2)))
3737	rc = rc2;
3738
3739	PPGMPAGE pOld = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Virgin : &pRomPage->Shadow;
3740	PPGMPAGE pNew = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Shadow : &pRomPage->Virgin;
3741
3742	pOld = pRamPage;
3743	pRamPage = pNew;
3744	/** @todo preserve the volatile flags (handlers) when these have been moved out of HCPhys! */
3745	}
3746	pRomPage->enmProt = enmProt;
3747	}
3748
3749	/*
3750	* Reset the access handler if we made changes, no need
3751	* to optimize this.
3752	*/
3753	if (fChanges)
3754	{
3755	int rc2 = PGMHandlerPhysicalReset(pVM, pRom->GCPhys);
3756	if (RT_FAILURE(rc2))
3757	{
3758	pgmUnlock(pVM);
3759	AssertRC(rc);
3760	return rc2;
3761	}
3762	}
3763
3764	/* Advance - cb isn't updated. */
3765	GCPhys = pRom->GCPhys + (cPages << PAGE_SHIFT);
3766	}
3767	}
3768	pgmUnlock(pVM);
3769	if (fFlushTLB)
3770	PGM_INVL_ALL_VCPU_TLBS(pVM);
3771
3772	return rc;
3773	}
3774
3775
3776	/**
3777	* Sets the Address Gate 20 state.
3778	*
3779	* @param pVCpu Pointer to the VMCPU.
3780	* @param fEnable True if the gate should be enabled.
3781	* False if the gate should be disabled.
3782	*/
3783	VMMDECL(void) PGMR3PhysSetA20(PVMCPU pVCpu, bool fEnable)
3784	{
3785	LogFlow(("PGMR3PhysSetA20 %d (was %d)\n", fEnable, pVCpu->pgm.s.fA20Enabled));
3786	if (pVCpu->pgm.s.fA20Enabled != fEnable)
3787	{
3788	pVCpu->pgm.s.fA20Enabled = fEnable;
3789	pVCpu->pgm.s.GCPhysA20Mask = ~(RTGCPHYS)(!fEnable << 20);
3790	#ifdef VBOX_WITH_REM
3791	REMR3A20Set(pVCpu->pVMR3, pVCpu, fEnable);
3792	#endif
3793	#ifdef PGM_WITH_A20
3794	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL;
3795	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3796	pgmR3RefreshShadowModeAfterA20Change(pVCpu);
3797	HWACCMFlushTLB(pVCpu);
3798	#endif
3799	STAM_REL_COUNTER_INC(&pVCpu->pgm.s.cA20Changes);
3800	}
3801	}
3802
3803
3804	/**
3805	* Tree enumeration callback for dealing with age rollover.
3806	* It will perform a simple compression of the current age.
3807	*/
3808	static DECLCALLBACK(int) pgmR3PhysChunkAgeingRolloverCallback(PAVLU32NODECORE pNode, void *pvUser)
3809	{
3810	/* Age compression - ASSUMES iNow == 4. */
3811	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
3812	if (pChunk->iLastUsed >= UINT32_C(0xffffff00))
3813	pChunk->iLastUsed = 3;
3814	else if (pChunk->iLastUsed >= UINT32_C(0xfffff000))
3815	pChunk->iLastUsed = 2;
3816	else if (pChunk->iLastUsed)
3817	pChunk->iLastUsed = 1;
3818	else /* iLastUsed = 0 */
3819	pChunk->iLastUsed = 4;
3820
3821	NOREF(pvUser);
3822	return 0;
3823	}
3824
3825
3826	/**
3827	* The structure passed in the pvUser argument of pgmR3PhysChunkUnmapCandidateCallback().
3828	*/
3829	typedef struct PGMR3PHYSCHUNKUNMAPCB
3830	{
3831	PVM pVM; /*< Pointer to the VM. /
3832	PPGMCHUNKR3MAP pChunk; /*< The chunk to unmap. /
3833	} PGMR3PHYSCHUNKUNMAPCB, *PPGMR3PHYSCHUNKUNMAPCB;
3834
3835
3836	/**
3837	* Callback used to find the mapping that's been unused for
3838	* the longest time.
3839	*/
3840	static DECLCALLBACK(int) pgmR3PhysChunkUnmapCandidateCallback(PAVLU32NODECORE pNode, void *pvUser)
3841	{
3842	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
3843	PPGMR3PHYSCHUNKUNMAPCB pArg = (PPGMR3PHYSCHUNKUNMAPCB)pvUser;
3844
3845	/*
3846	* Check for locks and compare when last used.
3847	*/
3848	if (pChunk->cRefs)
3849	return 0;
3850	if (pChunk->cPermRefs)
3851	return 0;
3852	if ( pArg->pChunk
3853	&& pChunk->iLastUsed >= pArg->pChunk->iLastUsed)
3854	return 0;
3855
3856	/*
3857	* Check that it's not in any of the TLBs.
3858	*/
3859	PVM pVM = pArg->pVM;
3860	if ( pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(pChunk->Core.Key)].idChunk
3861	== pChunk->Core.Key)
3862	{
3863	pChunk = NULL;
3864	return 0;
3865	}
3866	#ifdef VBOX_STRICT
3867	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
3868	{
3869	Assert(pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk != pChunk);
3870	Assert(pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk != pChunk->Core.Key);
3871	}
3872	#endif
3873
3874	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbHC.aEntries); i++)
3875	if (pVM->pgm.s.PhysTlbHC.aEntries[i].pMap == pChunk)
3876	return 0;
3877
3878	pArg->pChunk = pChunk;
3879	return 0;
3880	}
3881
3882
3883	/**
3884	* Finds a good candidate for unmapping when the ring-3 mapping cache is full.
3885	*
3886	* The candidate will not be part of any TLBs, so no need to flush
3887	* anything afterwards.
3888	*
3889	* @returns Chunk id.
3890	* @param pVM Pointer to the VM.
3891	*/
3892	static int32_t pgmR3PhysChunkFindUnmapCandidate(PVM pVM)
3893	{
3894	PGM_LOCK_ASSERT_OWNER(pVM);
3895
3896	/*
3897	* Enumerate the age tree starting with the left most node.
3898	*/
3899	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3900	PGMR3PHYSCHUNKUNMAPCB Args;
3901	Args.pVM = pVM;
3902	Args.pChunk = NULL;
3903	RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /fFromLeft/, pgmR3PhysChunkUnmapCandidateCallback, &Args);
3904	Assert(Args.pChunk);
3905	if (Args.pChunk)
3906	{
3907	Assert(Args.pChunk->cRefs == 0);
3908	Assert(Args.pChunk->cPermRefs == 0);
3909	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3910	return Args.pChunk->Core.Key;
3911	}
3912
3913	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3914	return INT32_MAX;
3915	}
3916
3917
3918	/**
3919	* Rendezvous callback used by pgmR3PhysUnmapChunk that unmaps a chunk
3920	*
3921	* This is only called on one of the EMTs while the other ones are waiting for
3922	* it to complete this function.
3923	*
3924	* @returns VINF_SUCCESS (VBox strict status code).
3925	* @param pVM Pointer to the VM.
3926	* @param pVCpu The VMCPU for the EMT we're being called on. Unused.
3927	* @param pvUser User pointer. Unused
3928	*
3929	*/
3930	static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysUnmapChunkRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
3931	{
3932	int rc = VINF_SUCCESS;
3933	pgmLock(pVM);
3934	NOREF(pVCpu); NOREF(pvUser);
3935
3936	if (pVM->pgm.s.ChunkR3Map.c >= pVM->pgm.s.ChunkR3Map.cMax)
3937	{
3938	/* Flush the pgm pool cache; call the internal rendezvous handler as we're already in a rendezvous handler here. */
3939	/** @todo also not really efficient to unmap a chunk that contains PD
3940	* or PT pages. */
3941	pgmR3PoolClearAllRendezvous(pVM, &pVM->aCpus[0], NULL /* no need to flush the REM TLB as we already did that above */);
3942
3943	/*
3944	* Request the ring-0 part to unmap a chunk to make space in the mapping cache.
3945	*/
3946	GMMMAPUNMAPCHUNKREQ Req;
3947	Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
3948	Req.Hdr.cbReq = sizeof(Req);
3949	Req.pvR3 = NULL;
3950	Req.idChunkMap = NIL_GMM_CHUNKID;
3951	Req.idChunkUnmap = pgmR3PhysChunkFindUnmapCandidate(pVM);
3952	if (Req.idChunkUnmap != INT32_MAX)
3953	{
3954	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkUnmap, a);
3955	rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
3956	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkUnmap, a);
3957	if (RT_SUCCESS(rc))
3958	{
3959	/*
3960	* Remove the unmapped one.
3961	*/
3962	PPGMCHUNKR3MAP pUnmappedChunk = (PPGMCHUNKR3MAP)RTAvlU32Remove(&pVM->pgm.s.ChunkR3Map.pTree, Req.idChunkUnmap);
3963	AssertRelease(pUnmappedChunk);
3964	AssertRelease(!pUnmappedChunk->cRefs);
3965	AssertRelease(!pUnmappedChunk->cPermRefs);
3966	pUnmappedChunk->pv = NULL;
3967	pUnmappedChunk->Core.Key = UINT32_MAX;
3968	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
3969	MMR3HeapFree(pUnmappedChunk);
3970	#else
3971	MMR3UkHeapFree(pVM, pUnmappedChunk, MM_TAG_PGM_CHUNK_MAPPING);
3972	#endif
3973	pVM->pgm.s.ChunkR3Map.c--;
3974	pVM->pgm.s.cUnmappedChunks++;
3975
3976	/*
3977	* Flush dangling PGM pointers (R3 & R0 ptrs to GC physical addresses).
3978	*/
3979	/** todo: we should not flush chunks which include cr3 mappings. */
3980	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3981	{
3982	PPGMCPU pPGM = &pVM->aCpus[idCpu].pgm.s;
3983
3984	pPGM->pGst32BitPdR3 = NULL;
3985	pPGM->pGstPaePdptR3 = NULL;
3986	pPGM->pGstAmd64Pml4R3 = NULL;
3987	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
3988	pPGM->pGst32BitPdR0 = NIL_RTR0PTR;
3989	pPGM->pGstPaePdptR0 = NIL_RTR0PTR;
3990	pPGM->pGstAmd64Pml4R0 = NIL_RTR0PTR;
3991	#endif
3992	for (unsigned i = 0; i < RT_ELEMENTS(pPGM->apGstPaePDsR3); i++)
3993	{
3994	pPGM->apGstPaePDsR3[i] = NULL;
3995	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
3996	pPGM->apGstPaePDsR0[i] = NIL_RTR0PTR;
3997	#endif
3998	}
3999
4000	/* Flush REM TLBs. */
4001	CPUMSetChangedFlags(&pVM->aCpus[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
4002	}
4003	#ifdef VBOX_WITH_REM
4004	/* Flush REM translation blocks. */
4005	REMFlushTBs(pVM);
4006	#endif
4007	}
4008	}
4009	}
4010	pgmUnlock(pVM);
4011	return rc;
4012	}
4013
4014	/**
4015	* Unmap a chunk to free up virtual address space (request packet handler for pgmR3PhysChunkMap)
4016	*
4017	* @returns VBox status code.
4018	* @param pVM Pointer to the VM.
4019	*/
4020	void pgmR3PhysUnmapChunk(PVM pVM)
4021	{
4022	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysUnmapChunkRendezvous, NULL);
4023	AssertRC(rc);
4024	}
4025
4026
4027	/**
4028	* Maps the given chunk into the ring-3 mapping cache.
4029	*
4030	* This will call ring-0.
4031	*
4032	* @returns VBox status code.
4033	* @param pVM Pointer to the VM.
4034	* @param idChunk The chunk in question.
4035	* @param ppChunk Where to store the chunk tracking structure.
4036	*
4037	* @remarks Called from within the PGM critical section.
4038	* @remarks Can be called from any thread!
4039	*/
4040	int pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk)
4041	{
4042	int rc;
4043
4044	PGM_LOCK_ASSERT_OWNER(pVM);
4045
4046	/*
4047	* Move the chunk time forward.
4048	*/
4049	pVM->pgm.s.ChunkR3Map.iNow++;
4050	if (pVM->pgm.s.ChunkR3Map.iNow == 0)
4051	{
4052	pVM->pgm.s.ChunkR3Map.iNow = 4;
4053	RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /fFromLeft/, pgmR3PhysChunkAgeingRolloverCallback, NULL);
4054	}
4055
4056	/*
4057	* Allocate a new tracking structure first.
4058	*/
4059	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
4060	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3HeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk));
4061	#else
4062	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3UkHeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk), NULL);
4063	#endif
4064	AssertReturn(pChunk, VERR_NO_MEMORY);
4065	pChunk->Core.Key = idChunk;
4066	pChunk->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
4067
4068	/*
4069	* Request the ring-0 part to map the chunk in question.
4070	*/
4071	GMMMAPUNMAPCHUNKREQ Req;
4072	Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
4073	Req.Hdr.cbReq = sizeof(Req);
4074	Req.pvR3 = NULL;
4075	Req.idChunkMap = idChunk;
4076	Req.idChunkUnmap = NIL_GMM_CHUNKID;
4077
4078	/* Must be callable from any thread, so can't use VMMR3CallR0. */
4079	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkMap, a);
4080	rc = SUPR3CallVMMR0Ex(pVM->pVMR0, NIL_VMCPUID, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
4081	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkMap, a);
4082	if (RT_SUCCESS(rc))
4083	{
4084	pChunk->pv = Req.pvR3;
4085
4086	/*
4087	* If we're running out of virtual address space, then we should
4088	* unmap another chunk.
4089	*
4090	* Currently, an unmap operation requires that all other virtual CPUs
4091	* are idling and not by chance making use of the memory we're
4092	* unmapping. So, we create an async unmap operation here.
4093	*
4094	* Now, when creating or restoring a saved state this wont work very
4095	* well since we may want to restore all guest RAM + a little something.
4096	* So, we have to do the unmap synchronously. Fortunately for us
4097	* though, during these operations the other virtual CPUs are inactive
4098	* and it should be safe to do this.
4099	*/
4100	/** @todo Eventually we should lock all memory when used and do
4101	* map+unmap as one kernel call without any rendezvous or
4102	* other precautions. */
4103	if (pVM->pgm.s.ChunkR3Map.c + 1 >= pVM->pgm.s.ChunkR3Map.cMax)
4104	{
4105	switch (VMR3GetState(pVM))
4106	{
4107	case VMSTATE_LOADING:
4108	case VMSTATE_SAVING:
4109	{
4110	PVMCPU pVCpu = VMMGetCpu(pVM);
4111	if ( pVCpu
4112	&& pVM->pgm.s.cDeprecatedPageLocks == 0)
4113	{
4114	pgmR3PhysUnmapChunkRendezvous(pVM, pVCpu, NULL);
4115	break;
4116	}
4117	/* fall thru */
4118	}
4119	default:
4120	rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysUnmapChunk, 1, pVM);
4121	AssertRC(rc);
4122	break;
4123	}
4124	}
4125
4126	/*
4127	* Update the tree. We must do this after any unmapping to make sure
4128	* the chunk we're going to return isn't unmapped by accident.
4129	*/
4130	AssertPtr(Req.pvR3);
4131	bool fRc = RTAvlU32Insert(&pVM->pgm.s.ChunkR3Map.pTree, &pChunk->Core);
4132	AssertRelease(fRc);
4133	pVM->pgm.s.ChunkR3Map.c++;
4134	pVM->pgm.s.cMappedChunks++;
4135	}
4136	else
4137	{
4138	/** @todo this may fail because of /proc/sys/vm/max_map_count, so we
4139	* should probably restrict ourselves on linux. */
4140	AssertRC(rc);
4141	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
4142	MMR3HeapFree(pChunk);
4143	#else
4144	MMR3UkHeapFree(pVM, pChunk, MM_TAG_PGM_CHUNK_MAPPING);
4145	#endif
4146	pChunk = NULL;
4147	}
4148
4149	*ppChunk = pChunk;
4150	return rc;
4151	}
4152
4153
4154	/**
4155	* For VMMCALLRING3_PGM_MAP_CHUNK, considered internal.
4156	*
4157	* @returns see pgmR3PhysChunkMap.
4158	* @param pVM Pointer to the VM.
4159	* @param idChunk The chunk to map.
4160	*/
4161	VMMR3DECL(int) PGMR3PhysChunkMap(PVM pVM, uint32_t idChunk)
4162	{
4163	PPGMCHUNKR3MAP pChunk;
4164	int rc;
4165
4166	pgmLock(pVM);
4167	rc = pgmR3PhysChunkMap(pVM, idChunk, &pChunk);
4168	pgmUnlock(pVM);
4169	return rc;
4170	}
4171
4172
4173	/**
4174	* Invalidates the TLB for the ring-3 mapping cache.
4175	*
4176	* @param pVM Pointer to the VM.
4177	*/
4178	VMMR3DECL(void) PGMR3PhysChunkInvalidateTLB(PVM pVM)
4179	{
4180	pgmLock(pVM);
4181	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
4182	{
4183	pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk = NIL_GMM_CHUNKID;
4184	pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk = NULL;
4185	}
4186	/* The page map TLB references chunks, so invalidate that one too. */
4187	pgmPhysInvalidatePageMapTLB(pVM);
4188	pgmUnlock(pVM);
4189	}
4190
4191
4192	/**
4193	* Response to VMMCALLRING3_PGM_ALLOCATE_LARGE_HANDY_PAGE to allocate a large
4194	* (2MB) page for use with a nested paging PDE.
4195	*
4196	* @returns The following VBox status codes.
4197	* @retval VINF_SUCCESS on success.
4198	* @retval VINF_EM_NO_MEMORY if we're out of memory.
4199	*
4200	* @param pVM Pointer to the VM.
4201	* @param GCPhys GC physical start address of the 2 MB range
4202	*/
4203	VMMR3DECL(int) PGMR3PhysAllocateLargeHandyPage(PVM pVM, RTGCPHYS GCPhys)
4204	{
4205	#ifdef PGM_WITH_LARGE_PAGES
4206	uint64_t u64TimeStamp1, u64TimeStamp2;
4207
4208	pgmLock(pVM);
4209
4210	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatAllocLargePage, a);
4211	u64TimeStamp1 = RTTimeMilliTS();
4212	int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_LARGE_HANDY_PAGE, 0, NULL);
4213	u64TimeStamp2 = RTTimeMilliTS();
4214	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatAllocLargePage, a);
4215	if (RT_SUCCESS(rc))
4216	{
4217	Assert(pVM->pgm.s.cLargeHandyPages == 1);
4218
4219	uint32_t idPage = pVM->pgm.s.aLargeHandyPage[0].idPage;
4220	RTHCPHYS HCPhys = pVM->pgm.s.aLargeHandyPage[0].HCPhysGCPhys;
4221
4222	void *pv;
4223
4224	/* Map the large page into our address space.
4225	*
4226	* Note: assuming that within the 2 MB range:
4227	* - GCPhys + PAGE_SIZE = HCPhys + PAGE_SIZE (whole point of this exercise)
4228	* - user space mapping is continuous as well
4229	* - page id (GCPhys) + 1 = page id (GCPhys + PAGE_SIZE)
4230	*/
4231	rc = pgmPhysPageMapByPageID(pVM, idPage, HCPhys, &pv);
4232	AssertLogRelMsg(RT_SUCCESS(rc), ("idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc\n", idPage, HCPhys, rc));
4233
4234	if (RT_SUCCESS(rc))
4235	{
4236	/*
4237	* Clear the pages.
4238	*/
4239	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatClearLargePage, b);
4240	for (unsigned i = 0; i < _2M/PAGE_SIZE; i++)
4241	{
4242	ASMMemZeroPage(pv);
4243
4244	PPGMPAGE pPage;
4245	rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
4246	AssertRC(rc);
4247
4248	Assert(PGM_PAGE_IS_ZERO(pPage));
4249	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatRZPageReplaceZero);
4250	pVM->pgm.s.cZeroPages--;
4251
4252	/*
4253	* Do the PGMPAGE modifications.
4254	*/
4255	pVM->pgm.s.cPrivatePages++;
4256	PGM_PAGE_SET_HCPHYS(pVM, pPage, HCPhys);
4257	PGM_PAGE_SET_PAGEID(pVM, pPage, idPage);
4258	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
4259	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PDE);
4260	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
4261	PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
4262
4263	/* Somewhat dirty assumption that page ids are increasing. */
4264	idPage++;
4265
4266	HCPhys += PAGE_SIZE;
4267	GCPhys += PAGE_SIZE;
4268
4269	pv = (void *)((uintptr_t)pv + PAGE_SIZE);
4270
4271	Log3(("PGMR3PhysAllocateLargePage: idPage=%#x HCPhys=%RGp\n", idPage, HCPhys));
4272	}
4273	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatClearLargePage, b);
4274
4275	/* Flush all TLBs. */
4276	PGM_INVL_ALL_VCPU_TLBS(pVM);
4277	pgmPhysInvalidatePageMapTLB(pVM);
4278	}
4279	pVM->pgm.s.cLargeHandyPages = 0;
4280	}
4281
4282	if (RT_SUCCESS(rc))
4283	{
4284	static uint32_t cTimeOut = 0;
4285	uint64_t u64TimeStampDelta = u64TimeStamp2 - u64TimeStamp1;
4286
4287	if (u64TimeStampDelta > 100)
4288	{
4289	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatLargePageOverflow);
4290	if ( ++cTimeOut > 10
4291	\|\| u64TimeStampDelta > 1000 /* more than one second forces an early retirement from allocating large pages. */)
4292	{
4293	/* If repeated attempts to allocate a large page takes more than 100 ms, then we fall back to normal 4k pages.
4294	* E.g. Vista 64 tries to move memory around, which takes a huge amount of time.
4295	*/
4296	LogRel(("PGMR3PhysAllocateLargePage: allocating large pages takes too long (last attempt %d ms; nr of timeouts %d); DISABLE\n", u64TimeStampDelta, cTimeOut));
4297	PGMSetLargePageUsage(pVM, false);
4298	}
4299	}
4300	else
4301	if (cTimeOut > 0)
4302	cTimeOut--;
4303	}
4304
4305	pgmUnlock(pVM);
4306	return rc;
4307	#else
4308	return VERR_NOT_IMPLEMENTED;
4309	#endif /* PGM_WITH_LARGE_PAGES */
4310	}
4311
4312
4313	/**
4314	* Response to VM_FF_PGM_NEED_HANDY_PAGES and VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES.
4315	*
4316	* This function will also work the VM_FF_PGM_NO_MEMORY force action flag, to
4317	* signal and clear the out of memory condition. When contracted, this API is
4318	* used to try clear the condition when the user wants to resume.
4319	*
4320	* @returns The following VBox status codes.
4321	* @retval VINF_SUCCESS on success. FFs cleared.
4322	* @retval VINF_EM_NO_MEMORY if we're out of memory. The FF is not cleared in
4323	* this case and it gets accompanied by VM_FF_PGM_NO_MEMORY.
4324	*
4325	* @param pVM Pointer to the VM.
4326	*
4327	* @remarks The VINF_EM_NO_MEMORY status is for the benefit of the FF processing
4328	* in EM.cpp and shouldn't be propagated outside TRPM, HWACCM, EM and
4329	* pgmPhysEnsureHandyPage. There is one exception to this in the \#PF
4330	* handler.
4331	*/
4332	VMMR3DECL(int) PGMR3PhysAllocateHandyPages(PVM pVM)
4333	{
4334	pgmLock(pVM);
4335
4336	/*
4337	* Allocate more pages, noting down the index of the first new page.
4338	*/
4339	uint32_t iClear = pVM->pgm.s.cHandyPages;
4340	AssertMsgReturn(iClear <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d", iClear), VERR_PGM_HANDY_PAGE_IPE);
4341	Log(("PGMR3PhysAllocateHandyPages: %d -> %d\n", iClear, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
4342	int rcAlloc = VINF_SUCCESS;
4343	int rcSeed = VINF_SUCCESS;
4344	int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
4345	while (rc == VERR_GMM_SEED_ME)
4346	{
4347	void *pvChunk;
4348	rcAlloc = rc = SUPR3PageAlloc(GMM_CHUNK_SIZE >> PAGE_SHIFT, &pvChunk);
4349	if (RT_SUCCESS(rc))
4350	{
4351	rcSeed = rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_SEED_CHUNK, (uintptr_t)pvChunk, NULL);
4352	if (RT_FAILURE(rc))
4353	SUPR3PageFree(pvChunk, GMM_CHUNK_SIZE >> PAGE_SHIFT);
4354	}
4355	if (RT_SUCCESS(rc))
4356	rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
4357	}
4358
4359	/* todo: we should split this up into an allocate and flush operation. sometimes you want to flush and not allocate more (which will trigger the vm account limit error) */
4360	if ( rc == VERR_GMM_HIT_VM_ACCOUNT_LIMIT
4361	&& pVM->pgm.s.cHandyPages > 0)
4362	{
4363	/* Still handy pages left, so don't panic. */
4364	rc = VINF_SUCCESS;
4365	}
4366
4367	if (RT_SUCCESS(rc))
4368	{
4369	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
4370	Assert(pVM->pgm.s.cHandyPages > 0);
4371	VM_FF_CLEAR(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
4372	VM_FF_CLEAR(pVM, VM_FF_PGM_NO_MEMORY);
4373
4374	#ifdef VBOX_STRICT
4375	uint32_t i;
4376	for (i = iClear; i < pVM->pgm.s.cHandyPages; i++)
4377	if ( pVM->pgm.s.aHandyPages[i].idPage == NIL_GMM_PAGEID
4378	\|\| pVM->pgm.s.aHandyPages[i].idSharedPage != NIL_GMM_PAGEID
4379	\|\| (pVM->pgm.s.aHandyPages[i].HCPhysGCPhys & PAGE_OFFSET_MASK))
4380	break;
4381	if (i != pVM->pgm.s.cHandyPages)
4382	{
4383	RTAssertMsg1Weak(NULL, __LINE__, __FILE__, __FUNCTION__);
4384	RTAssertMsg2Weak("i=%d iClear=%d cHandyPages=%d\n", i, iClear, pVM->pgm.s.cHandyPages);
4385	for (uint32_t j = iClear; j < pVM->pgm.s.cHandyPages; j++)
4386	RTAssertMsg2Add(("%03d: idPage=%d HCPhysGCPhys=%RHp idSharedPage=%d%\n", j,
4387	pVM->pgm.s.aHandyPages[j].idPage,
4388	pVM->pgm.s.aHandyPages[j].HCPhysGCPhys,
4389	pVM->pgm.s.aHandyPages[j].idSharedPage,
4390	j == i ? " <---" : ""));
4391	RTAssertPanic();
4392	}
4393	#endif
4394	/*
4395	* Clear the pages.
4396	*/
4397	while (iClear < pVM->pgm.s.cHandyPages)
4398	{
4399	PGMMPAGEDESC pPage = &pVM->pgm.s.aHandyPages[iClear];
4400	void *pv;
4401	rc = pgmPhysPageMapByPageID(pVM, pPage->idPage, pPage->HCPhysGCPhys, &pv);
4402	AssertLogRelMsgBreak(RT_SUCCESS(rc),
4403	("%u/%u: idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc\n",
4404	iClear, pVM->pgm.s.cHandyPages, pPage->idPage, pPage->HCPhysGCPhys, rc));
4405	ASMMemZeroPage(pv);
4406	iClear++;
4407	Log3(("PGMR3PhysAllocateHandyPages: idPage=%#x HCPhys=%RGp\n", pPage->idPage, pPage->HCPhysGCPhys));
4408	}
4409	}
4410	else
4411	{
4412	uint64_t cAllocPages, cMaxPages, cBalloonPages;
4413
4414	/*
4415	* We should never get here unless there is a genuine shortage of
4416	* memory (or some internal error). Flag the error so the VM can be
4417	* suspended ASAP and the user informed. If we're totally out of
4418	* handy pages we will return failure.
4419	*/
4420	/* Report the failure. */
4421	LogRel(("PGM: Failed to procure handy pages; rc=%Rrc rcAlloc=%Rrc rcSeed=%Rrc cHandyPages=%#x\n"
4422	" cAllPages=%#x cPrivatePages=%#x cSharedPages=%#x cZeroPages=%#x\n",
4423	rc, rcAlloc, rcSeed,
4424	pVM->pgm.s.cHandyPages,
4425	pVM->pgm.s.cAllPages,
4426	pVM->pgm.s.cPrivatePages,
4427	pVM->pgm.s.cSharedPages,
4428	pVM->pgm.s.cZeroPages));
4429
4430	if (GMMR3QueryMemoryStats(pVM, &cAllocPages, &cMaxPages, &cBalloonPages) == VINF_SUCCESS)
4431	{
4432	LogRel(("GMM: Statistics:\n"
4433	" Allocated pages: %RX64\n"
4434	" Maximum pages: %RX64\n"
4435	" Ballooned pages: %RX64\n", cAllocPages, cMaxPages, cBalloonPages));
4436	}
4437
4438	if ( rc != VERR_NO_MEMORY
4439	&& rc != VERR_NO_PHYS_MEMORY
4440	&& rc != VERR_LOCK_FAILED)
4441	{
4442	for (uint32_t i = 0; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
4443	{
4444	LogRel(("PGM: aHandyPages[#%#04x] = {.HCPhysGCPhys=%RHp, .idPage=%#08x, .idSharedPage=%#08x}\n",
4445	i, pVM->pgm.s.aHandyPages[i].HCPhysGCPhys, pVM->pgm.s.aHandyPages[i].idPage,
4446	pVM->pgm.s.aHandyPages[i].idSharedPage));
4447	uint32_t const idPage = pVM->pgm.s.aHandyPages[i].idPage;
4448	if (idPage != NIL_GMM_PAGEID)
4449	{
4450	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
4451	pRam;
4452	pRam = pRam->pNextR3)
4453	{
4454	uint32_t const cPages = pRam->cb >> PAGE_SHIFT;
4455	for (uint32_t iPage = 0; iPage < cPages; iPage++)
4456	if (PGM_PAGE_GET_PAGEID(&pRam->aPages[iPage]) == idPage)
4457	LogRel(("PGM: Used by %RGp %R[pgmpage] (%s)\n",
4458	pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pRam->aPages[iPage], pRam->pszDesc));
4459	}
4460	}
4461	}
4462	}
4463
4464	/* Set the FFs and adjust rc. */
4465	VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
4466	VM_FF_SET(pVM, VM_FF_PGM_NO_MEMORY);
4467	if ( rc == VERR_NO_MEMORY
4468	\|\| rc == VERR_NO_PHYS_MEMORY
4469	\|\| rc == VERR_LOCK_FAILED)
4470	rc = VINF_EM_NO_MEMORY;
4471	}
4472
4473	pgmUnlock(pVM);
4474	return rc;
4475	}
4476
4477
4478	/**
4479	* Frees the specified RAM page and replaces it with the ZERO page.
4480	*
4481	* This is used by ballooning, remapping MMIO2, RAM reset and state loading.
4482	*
4483	* @param pVM Pointer to the VM.
4484	* @param pReq Pointer to the request.
4485	* @param pcPendingPages Where the number of pages waiting to be freed are
4486	* kept. This will normally be incremented.
4487	* @param pPage Pointer to the page structure.
4488	* @param GCPhys The guest physical address of the page, if applicable.
4489	*
4490	* @remarks The caller must own the PGM lock.
4491	*/
4492	int pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys)
4493	{
4494	/*
4495	* Assert sanity.
4496	*/
4497	PGM_LOCK_ASSERT_OWNER(pVM);
4498	if (RT_UNLIKELY( PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
4499	&& PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_ROM_SHADOW))
4500	{
4501	AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
4502	return VMSetError(pVM, VERR_PGM_PHYS_NOT_RAM, RT_SRC_POS, "GCPhys=%RGp type=%d", GCPhys, PGM_PAGE_GET_TYPE(pPage));
4503	}
4504
4505	/** @todo What about ballooning of large pages??! */
4506	Assert( PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE
4507	&& PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE_DISABLED);
4508
4509	if ( PGM_PAGE_IS_ZERO(pPage)
4510	\|\| PGM_PAGE_IS_BALLOONED(pPage))
4511	return VINF_SUCCESS;
4512
4513	const uint32_t idPage = PGM_PAGE_GET_PAGEID(pPage);
4514	Log3(("pgmPhysFreePage: idPage=%#x GCPhys=%RGp pPage=%R[pgmpage]\n", idPage, GCPhys, pPage));
4515	if (RT_UNLIKELY( idPage == NIL_GMM_PAGEID
4516	\|\| idPage > GMM_PAGEID_LAST
4517	\|\| PGM_PAGE_GET_CHUNKID(pPage) == NIL_GMM_CHUNKID))
4518	{
4519	AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
4520	return VMSetError(pVM, VERR_PGM_PHYS_INVALID_PAGE_ID, RT_SRC_POS, "GCPhys=%RGp idPage=%#x", GCPhys, pPage);
4521	}
4522
4523	/* update page count stats. */
4524	if (PGM_PAGE_IS_SHARED(pPage))
4525	pVM->pgm.s.cSharedPages--;
4526	else
4527	pVM->pgm.s.cPrivatePages--;
4528	pVM->pgm.s.cZeroPages++;
4529
4530	/* Deal with write monitored pages. */
4531	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
4532	{
4533	PGM_PAGE_SET_WRITTEN_TO(pVM, pPage);
4534	pVM->pgm.s.cWrittenToPages++;
4535	}
4536
4537	/*
4538	* pPage = ZERO page.
4539	*/
4540	PGM_PAGE_SET_HCPHYS(pVM, pPage, pVM->pgm.s.HCPhysZeroPg);
4541	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
4542	PGM_PAGE_SET_PAGEID(pVM, pPage, NIL_GMM_PAGEID);
4543	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
4544	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
4545	PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
4546
4547	/* Flush physical page map TLB entry. */
4548	pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
4549
4550	/*
4551	* Make sure it's not in the handy page array.
4552	*/
4553	for (uint32_t i = pVM->pgm.s.cHandyPages; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
4554	{
4555	if (pVM->pgm.s.aHandyPages[i].idPage == idPage)
4556	{
4557	pVM->pgm.s.aHandyPages[i].idPage = NIL_GMM_PAGEID;
4558	break;
4559	}
4560	if (pVM->pgm.s.aHandyPages[i].idSharedPage == idPage)
4561	{
4562	pVM->pgm.s.aHandyPages[i].idSharedPage = NIL_GMM_PAGEID;
4563	break;
4564	}
4565	}
4566
4567	/*
4568	* Push it onto the page array.
4569	*/
4570	uint32_t iPage = *pcPendingPages;
4571	Assert(iPage < PGMPHYS_FREE_PAGE_BATCH_SIZE);
4572	*pcPendingPages += 1;
4573
4574	pReq->aPages[iPage].idPage = idPage;
4575
4576	if (iPage + 1 < PGMPHYS_FREE_PAGE_BATCH_SIZE)
4577	return VINF_SUCCESS;
4578
4579	/*
4580	* Flush the pages.
4581	*/
4582	int rc = GMMR3FreePagesPerform(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE);
4583	if (RT_SUCCESS(rc))
4584	{
4585	GMMR3FreePagesRePrep(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
4586	*pcPendingPages = 0;
4587	}
4588	return rc;
4589	}
4590
4591
4592	/**
4593	* Converts a GC physical address to a HC ring-3 pointer, with some
4594	* additional checks.
4595	*
4596	* @returns VBox status code.
4597	* @retval VINF_SUCCESS on success.
4598	* @retval VINF_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
4599	* access handler of some kind.
4600	* @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
4601	* accesses or is odd in any way.
4602	* @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
4603	*
4604	* @param pVM Pointer to the VM.
4605	* @param GCPhys The GC physical address to convert. Since this is only
4606	* used for filling the REM TLB, the A20 mask must be
4607	* applied before calling this API.
4608	* @param fWritable Whether write access is required.
4609	* @param ppv Where to store the pointer corresponding to GCPhys on
4610	* success.
4611	*/
4612	VMMR3DECL(int) PGMR3PhysTlbGCPhys2Ptr(PVM pVM, RTGCPHYS GCPhys, bool fWritable, void **ppv)
4613	{
4614	pgmLock(pVM);
4615	PGM_A20_ASSERT_MASKED(VMMGetCpu(pVM), GCPhys);
4616
4617	PPGMRAMRANGE pRam;
4618	PPGMPAGE pPage;
4619	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
4620	if (RT_SUCCESS(rc))
4621	{
4622	if (PGM_PAGE_IS_BALLOONED(pPage))
4623	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
4624	else if (!PGM_PAGE_HAS_ANY_HANDLERS(pPage))
4625	rc = VINF_SUCCESS;
4626	else
4627	{
4628	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
4629	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4630	else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
4631	{
4632	/** @todo Handle TLB loads of virtual handlers so ./test.sh can be made to work
4633	* in -norawr0 mode. */
4634	if (fWritable)
4635	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
4636	}
4637	else
4638	{
4639	/* Temporarily disabled physical handler(s), since the recompiler
4640	doesn't get notified when it's reset we'll have to pretend it's
4641	operating normally. */
4642	if (pgmHandlerPhysicalIsAll(pVM, GCPhys))
4643	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4644	else
4645	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
4646	}
4647	}
4648	if (RT_SUCCESS(rc))
4649	{
4650	int rc2;
4651
4652	/* Make sure what we return is writable. */
4653	if (fWritable)
4654	switch (PGM_PAGE_GET_STATE(pPage))
4655	{
4656	case PGM_PAGE_STATE_ALLOCATED:
4657	break;
4658	case PGM_PAGE_STATE_BALLOONED:
4659	AssertFailed();
4660	break;
4661	case PGM_PAGE_STATE_ZERO:
4662	case PGM_PAGE_STATE_SHARED:
4663	if (rc == VINF_PGM_PHYS_TLB_CATCH_WRITE)
4664	break;
4665	case PGM_PAGE_STATE_WRITE_MONITORED:
4666	rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
4667	AssertLogRelRCReturn(rc2, rc2);
4668	break;
4669	}
4670
4671	/* Get a ring-3 mapping of the address. */
4672	PPGMPAGER3MAPTLBE pTlbe;
4673	rc2 = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
4674	AssertLogRelRCReturn(rc2, rc2);
4675	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
4676	/** @todo mapping/locking hell; this isn't horribly efficient since
4677	* pgmPhysPageLoadIntoTlb will repeat the lookup we've done here. */
4678
4679	Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage] ppv=%p\n", GCPhys, rc, pPage, ppv));
4680	}
4681	else
4682	Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage]\n", GCPhys, rc, pPage));
4683
4684	/* else: handler catching all access, no pointer returned. */
4685	}
4686	else
4687	rc = VERR_PGM_PHYS_TLB_UNASSIGNED;
4688
4689	pgmUnlock(pVM);
4690	return rc;
4691	}
4692

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR3/PGMPhys.cpp@ 43363

Download in other formats: