PGMPhys.cpp@ 43045

Last change on this file since 43045 was 43045, checked in by vboxsync, 13 years ago
PGM: Employ the 4 unused PGMPHYSPAGE bits for checksumming shared pages in strict builds.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 166.4 KB

Line
1	/* $Id: PGMPhys.cpp 43045 2012-08-28 14:21:43Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor, Physical Memory Addressing.
4	*/
5
6	/*
7	* Copyright (C) 2006-2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_PGM_PHYS
23	#include <VBox/vmm/pgm.h>
24	#include <VBox/vmm/iom.h>
25	#include <VBox/vmm/mm.h>
26	#include <VBox/vmm/stam.h>
27	#ifdef VBOX_WITH_REM
28	# include <VBox/vmm/rem.h>
29	#endif
30	#include <VBox/vmm/pdmdev.h>
31	#include "PGMInternal.h"
32	#include <VBox/vmm/vm.h>
33	#include "PGMInline.h"
34	#include <VBox/sup.h>
35	#include <VBox/param.h>
36	#include <VBox/err.h>
37	#include <VBox/log.h>
38	#include <iprt/assert.h>
39	#include <iprt/alloc.h>
40	#include <iprt/asm.h>
41	#ifdef VBOX_STRICT
42	# include <iprt/crc.h>
43	#endif
44	#include <iprt/thread.h>
45	#include <iprt/string.h>
46	#include <iprt/system.h>
47
48
49	/*******************************************************************************
50	* Defined Constants And Macros *
51	*******************************************************************************/
52	/** The number of pages to free in one batch. */
53	#define PGMPHYS_FREE_PAGE_BATCH_SIZE 128
54
55
56	/*******************************************************************************
57	* Internal Functions *
58	*******************************************************************************/
59	static DECLCALLBACK(int) pgmR3PhysRomWriteHandler(PVM pVM, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser);
60
61
62	/*
63	* PGMR3PhysReadU8-64
64	* PGMR3PhysWriteU8-64
65	*/
66	#define PGMPHYSFN_READNAME PGMR3PhysReadU8
67	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU8
68	#define PGMPHYS_DATASIZE 1
69	#define PGMPHYS_DATATYPE uint8_t
70	#include "PGMPhysRWTmpl.h"
71
72	#define PGMPHYSFN_READNAME PGMR3PhysReadU16
73	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU16
74	#define PGMPHYS_DATASIZE 2
75	#define PGMPHYS_DATATYPE uint16_t
76	#include "PGMPhysRWTmpl.h"
77
78	#define PGMPHYSFN_READNAME PGMR3PhysReadU32
79	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU32
80	#define PGMPHYS_DATASIZE 4
81	#define PGMPHYS_DATATYPE uint32_t
82	#include "PGMPhysRWTmpl.h"
83
84	#define PGMPHYSFN_READNAME PGMR3PhysReadU64
85	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU64
86	#define PGMPHYS_DATASIZE 8
87	#define PGMPHYS_DATATYPE uint64_t
88	#include "PGMPhysRWTmpl.h"
89
90
91	/**
92	* EMT worker for PGMR3PhysReadExternal.
93	*/
94	static DECLCALLBACK(int) pgmR3PhysReadExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, void *pvBuf, size_t cbRead)
95	{
96	PGMPhysRead(pVM, *pGCPhys, pvBuf, cbRead);
97	return VINF_SUCCESS;
98	}
99
100
101	/**
102	* Read from physical memory, external users.
103	*
104	* @returns VBox status code.
105	* @retval VINF_SUCCESS.
106	*
107	* @param pVM Pointer to the VM.
108	* @param GCPhys Physical address to read from.
109	* @param pvBuf Where to read into.
110	* @param cbRead How many bytes to read.
111	*
112	* @thread Any but EMTs.
113	*/
114	VMMR3DECL(int) PGMR3PhysReadExternal(PVM pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead)
115	{
116	VM_ASSERT_OTHER_THREAD(pVM);
117
118	AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
119	LogFlow(("PGMR3PhysReadExternal: %RGp %d\n", GCPhys, cbRead));
120
121	pgmLock(pVM);
122
123	/*
124	* Copy loop on ram ranges.
125	*/
126	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
127	for (;;)
128	{
129	/* Inside range or not? */
130	if (pRam && GCPhys >= pRam->GCPhys)
131	{
132	/*
133	* Must work our way thru this page by page.
134	*/
135	RTGCPHYS off = GCPhys - pRam->GCPhys;
136	while (off < pRam->cb)
137	{
138	unsigned iPage = off >> PAGE_SHIFT;
139	PPGMPAGE pPage = &pRam->aPages[iPage];
140
141	/*
142	* If the page has an ALL access handler, we'll have to
143	* delegate the job to EMT.
144	*/
145	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
146	{
147	pgmUnlock(pVM);
148
149	return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysReadExternalEMT, 4,
150	pVM, &GCPhys, pvBuf, cbRead);
151	}
152	Assert(!PGM_PAGE_IS_MMIO(pPage));
153
154	/*
155	* Simple stuff, go ahead.
156	*/
157	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
158	if (cb > cbRead)
159	cb = cbRead;
160	PGMPAGEMAPLOCK PgMpLck;
161	const void *pvSrc;
162	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc, &PgMpLck);
163	if (RT_SUCCESS(rc))
164	{
165	memcpy(pvBuf, pvSrc, cb);
166	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
167	}
168	else
169	{
170	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
171	pRam->GCPhys + off, pPage, rc));
172	memset(pvBuf, 0xff, cb);
173	}
174
175	/* next page */
176	if (cb >= cbRead)
177	{
178	pgmUnlock(pVM);
179	return VINF_SUCCESS;
180	}
181	cbRead -= cb;
182	off += cb;
183	GCPhys += cb;
184	pvBuf = (char *)pvBuf + cb;
185	} /* walk pages in ram range. */
186	}
187	else
188	{
189	LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
190
191	/*
192	* Unassigned address space.
193	*/
194	size_t cb = pRam ? pRam->GCPhys - GCPhys : ~(size_t)0;
195	if (cb >= cbRead)
196	{
197	memset(pvBuf, 0xff, cbRead);
198	break;
199	}
200	memset(pvBuf, 0xff, cb);
201
202	cbRead -= cb;
203	pvBuf = (char *)pvBuf + cb;
204	GCPhys += cb;
205	}
206
207	/* Advance range if necessary. */
208	while (pRam && GCPhys > pRam->GCPhysLast)
209	pRam = pRam->CTX_SUFF(pNext);
210	} /* Ram range walk */
211
212	pgmUnlock(pVM);
213
214	return VINF_SUCCESS;
215	}
216
217
218	/**
219	* EMT worker for PGMR3PhysWriteExternal.
220	*/
221	static DECLCALLBACK(int) pgmR3PhysWriteExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, const void *pvBuf, size_t cbWrite)
222	{
223	/** @todo VERR_EM_NO_MEMORY */
224	PGMPhysWrite(pVM, *pGCPhys, pvBuf, cbWrite);
225	return VINF_SUCCESS;
226	}
227
228
229	/**
230	* Write to physical memory, external users.
231	*
232	* @returns VBox status code.
233	* @retval VINF_SUCCESS.
234	* @retval VERR_EM_NO_MEMORY.
235	*
236	* @param pVM Pointer to the VM.
237	* @param GCPhys Physical address to write to.
238	* @param pvBuf What to write.
239	* @param cbWrite How many bytes to write.
240	* @param pszWho Who is writing. For tracking down who is writing
241	* after we've saved the state.
242	*
243	* @thread Any but EMTs.
244	*/
245	VMMDECL(int) PGMR3PhysWriteExternal(PVM pVM, RTGCPHYS GCPhys, const void pvBuf, size_t cbWrite, const char pszWho)
246	{
247	VM_ASSERT_OTHER_THREAD(pVM);
248
249	AssertMsg(!pVM->pgm.s.fNoMorePhysWrites,
250	("Calling PGMR3PhysWriteExternal after pgmR3Save()! GCPhys=%RGp cbWrite=%#x pszWho=%s\n",
251	GCPhys, cbWrite, pszWho));
252	AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
253	LogFlow(("PGMR3PhysWriteExternal: %RGp %d\n", GCPhys, cbWrite));
254
255	pgmLock(pVM);
256
257	/*
258	* Copy loop on ram ranges, stop when we hit something difficult.
259	*/
260	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
261	for (;;)
262	{
263	/* Inside range or not? */
264	if (pRam && GCPhys >= pRam->GCPhys)
265	{
266	/*
267	* Must work our way thru this page by page.
268	*/
269	RTGCPTR off = GCPhys - pRam->GCPhys;
270	while (off < pRam->cb)
271	{
272	RTGCPTR iPage = off >> PAGE_SHIFT;
273	PPGMPAGE pPage = &pRam->aPages[iPage];
274
275	/*
276	* Is the page problematic, we have to do the work on the EMT.
277	*
278	* Allocating writable pages and access handlers are
279	* problematic, write monitored pages are simple and can be
280	* dealt with here.
281	*/
282	if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
283	\|\| PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
284	{
285	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
286	&& !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
287	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
288	else
289	{
290	pgmUnlock(pVM);
291
292	return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysWriteExternalEMT, 4,
293	pVM, &GCPhys, pvBuf, cbWrite);
294	}
295	}
296	Assert(!PGM_PAGE_IS_MMIO(pPage));
297
298	/*
299	* Simple stuff, go ahead.
300	*/
301	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
302	if (cb > cbWrite)
303	cb = cbWrite;
304	PGMPAGEMAPLOCK PgMpLck;
305	void *pvDst;
306	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst, &PgMpLck);
307	if (RT_SUCCESS(rc))
308	{
309	memcpy(pvDst, pvBuf, cb);
310	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
311	}
312	else
313	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
314	pRam->GCPhys + off, pPage, rc));
315
316	/* next page */
317	if (cb >= cbWrite)
318	{
319	pgmUnlock(pVM);
320	return VINF_SUCCESS;
321	}
322
323	cbWrite -= cb;
324	off += cb;
325	GCPhys += cb;
326	pvBuf = (const char *)pvBuf + cb;
327	} /* walk pages in ram range */
328	}
329	else
330	{
331	/*
332	* Unassigned address space, skip it.
333	*/
334	if (!pRam)
335	break;
336	size_t cb = pRam->GCPhys - GCPhys;
337	if (cb >= cbWrite)
338	break;
339	cbWrite -= cb;
340	pvBuf = (const char *)pvBuf + cb;
341	GCPhys += cb;
342	}
343
344	/* Advance range if necessary. */
345	while (pRam && GCPhys > pRam->GCPhysLast)
346	pRam = pRam->CTX_SUFF(pNext);
347	} /* Ram range walk */
348
349	pgmUnlock(pVM);
350	return VINF_SUCCESS;
351	}
352
353
354	/**
355	* VMR3ReqCall worker for PGMR3PhysGCPhys2CCPtrExternal to make pages writable.
356	*
357	* @returns see PGMR3PhysGCPhys2CCPtrExternal
358	* @param pVM Pointer to the VM.
359	* @param pGCPhys Pointer to the guest physical address.
360	* @param ppv Where to store the mapping address.
361	* @param pLock Where to store the lock.
362	*/
363	static DECLCALLBACK(int) pgmR3PhysGCPhys2CCPtrDelegated(PVM pVM, PRTGCPHYS pGCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
364	{
365	/*
366	* Just hand it to PGMPhysGCPhys2CCPtr and check that it's not a page with
367	* an access handler after it succeeds.
368	*/
369	int rc = pgmLock(pVM);
370	AssertRCReturn(rc, rc);
371
372	rc = PGMPhysGCPhys2CCPtr(pVM, *pGCPhys, ppv, pLock);
373	if (RT_SUCCESS(rc))
374	{
375	PPGMPAGEMAPTLBE pTlbe;
376	int rc2 = pgmPhysPageQueryTlbe(pVM, *pGCPhys, &pTlbe);
377	AssertFatalRC(rc2);
378	PPGMPAGE pPage = pTlbe->pPage;
379	if (PGM_PAGE_IS_MMIO(pPage))
380	{
381	PGMPhysReleasePageMappingLock(pVM, pLock);
382	rc = VERR_PGM_PHYS_PAGE_RESERVED;
383	}
384	else if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
385	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
386	\|\| pgmPoolIsDirtyPage(pVM, *pGCPhys)
387	#endif
388	)
389	{
390	/* We must flush any corresponding pgm pool page here, otherwise we'll
391	* not be informed about writes and keep bogus gst->shw mappings around.
392	*/
393	pgmPoolFlushPageByGCPhys(pVM, *pGCPhys);
394	Assert(!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage));
395	/** @todo r=bird: return VERR_PGM_PHYS_PAGE_RESERVED here if it still has
396	* active handlers, see the PGMR3PhysGCPhys2CCPtrExternal docs. */
397	}
398	}
399
400	pgmUnlock(pVM);
401	return rc;
402	}
403
404
405	/**
406	* Requests the mapping of a guest page into ring-3, external threads.
407	*
408	* When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
409	* release it.
410	*
411	* This API will assume your intention is to write to the page, and will
412	* therefore replace shared and zero pages. If you do not intend to modify the
413	* page, use the PGMR3PhysGCPhys2CCPtrReadOnlyExternal() API.
414	*
415	* @returns VBox status code.
416	* @retval VINF_SUCCESS on success.
417	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
418	* backing or if the page has any active access handlers. The caller
419	* must fall back on using PGMR3PhysWriteExternal.
420	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
421	*
422	* @param pVM Pointer to the VM.
423	* @param GCPhys The guest physical address of the page that should be mapped.
424	* @param ppv Where to store the address corresponding to GCPhys.
425	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
426	*
427	* @remark Avoid calling this API from within critical sections (other than the
428	* PGM one) because of the deadlock risk when we have to delegating the
429	* task to an EMT.
430	* @thread Any.
431	*/
432	VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrExternal(PVM pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
433	{
434	AssertPtr(ppv);
435	AssertPtr(pLock);
436
437	Assert(VM_IS_EMT(pVM) \|\| !PGMIsLockOwner(pVM));
438
439	int rc = pgmLock(pVM);
440	AssertRCReturn(rc, rc);
441
442	/*
443	* Query the Physical TLB entry for the page (may fail).
444	*/
445	PPGMPAGEMAPTLBE pTlbe;
446	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
447	if (RT_SUCCESS(rc))
448	{
449	PPGMPAGE pPage = pTlbe->pPage;
450	if (PGM_PAGE_IS_MMIO(pPage))
451	rc = VERR_PGM_PHYS_PAGE_RESERVED;
452	else
453	{
454	/*
455	* If the page is shared, the zero page, or being write monitored
456	* it must be converted to an page that's writable if possible.
457	* We can only deal with write monitored pages here, the rest have
458	* to be on an EMT.
459	*/
460	if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
461	\|\| PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
462	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
463	\|\| pgmPoolIsDirtyPage(pVM, GCPhys)
464	#endif
465	)
466	{
467	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
468	&& !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
469	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
470	&& !pgmPoolIsDirtyPage(pVM, GCPhys)
471	#endif
472	)
473	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
474	else
475	{
476	pgmUnlock(pVM);
477
478	return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysGCPhys2CCPtrDelegated, 4,
479	pVM, &GCPhys, ppv, pLock);
480	}
481	}
482
483	/*
484	* Now, just perform the locking and calculate the return address.
485	*/
486	PPGMPAGEMAP pMap = pTlbe->pMap;
487	if (pMap)
488	pMap->cRefs++;
489
490	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
491	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
492	{
493	if (cLocks == 0)
494	pVM->pgm.s.cWriteLockedPages++;
495	PGM_PAGE_INC_WRITE_LOCKS(pPage);
496	}
497	else if (cLocks != PGM_PAGE_GET_WRITE_LOCKS(pPage))
498	{
499	PGM_PAGE_INC_WRITE_LOCKS(pPage);
500	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent write locked state!\n", GCPhys, pPage));
501	if (pMap)
502	pMap->cRefs++; /* Extra ref to prevent it from going away. */
503	}
504
505	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
506	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_WRITE;
507	pLock->pvMap = pMap;
508	}
509	}
510
511	pgmUnlock(pVM);
512	return rc;
513	}
514
515
516	/**
517	* Requests the mapping of a guest page into ring-3, external threads.
518	*
519	* When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
520	* release it.
521	*
522	* @returns VBox status code.
523	* @retval VINF_SUCCESS on success.
524	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
525	* backing or if the page as an active ALL access handler. The caller
526	* must fall back on using PGMPhysRead.
527	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
528	*
529	* @param pVM Pointer to the VM.
530	* @param GCPhys The guest physical address of the page that should be mapped.
531	* @param ppv Where to store the address corresponding to GCPhys.
532	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
533	*
534	* @remark Avoid calling this API from within critical sections (other than
535	* the PGM one) because of the deadlock risk.
536	* @thread Any.
537	*/
538	VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrReadOnlyExternal(PVM pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
539	{
540	int rc = pgmLock(pVM);
541	AssertRCReturn(rc, rc);
542
543	/*
544	* Query the Physical TLB entry for the page (may fail).
545	*/
546	PPGMPAGEMAPTLBE pTlbe;
547	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
548	if (RT_SUCCESS(rc))
549	{
550	PPGMPAGE pPage = pTlbe->pPage;
551	#if 1
552	/* MMIO pages doesn't have any readable backing. */
553	if (PGM_PAGE_IS_MMIO(pPage))
554	rc = VERR_PGM_PHYS_PAGE_RESERVED;
555	#else
556	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
557	rc = VERR_PGM_PHYS_PAGE_RESERVED;
558	#endif
559	else
560	{
561	/*
562	* Now, just perform the locking and calculate the return address.
563	*/
564	PPGMPAGEMAP pMap = pTlbe->pMap;
565	if (pMap)
566	pMap->cRefs++;
567
568	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
569	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
570	{
571	if (cLocks == 0)
572	pVM->pgm.s.cReadLockedPages++;
573	PGM_PAGE_INC_READ_LOCKS(pPage);
574	}
575	else if (cLocks != PGM_PAGE_GET_READ_LOCKS(pPage))
576	{
577	PGM_PAGE_INC_READ_LOCKS(pPage);
578	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent readonly locked state!\n", GCPhys, pPage));
579	if (pMap)
580	pMap->cRefs++; /* Extra ref to prevent it from going away. */
581	}
582
583	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
584	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_READ;
585	pLock->pvMap = pMap;
586	}
587	}
588
589	pgmUnlock(pVM);
590	return rc;
591	}
592
593
594	#define MAKE_LEAF(a_pNode) \
595	do { \
596	(a_pNode)->pLeftR3 = NIL_RTR3PTR; \
597	(a_pNode)->pRightR3 = NIL_RTR3PTR; \
598	(a_pNode)->pLeftR0 = NIL_RTR0PTR; \
599	(a_pNode)->pRightR0 = NIL_RTR0PTR; \
600	(a_pNode)->pLeftRC = NIL_RTRCPTR; \
601	(a_pNode)->pRightRC = NIL_RTRCPTR; \
602	} while (0)
603
604	#define INSERT_LEFT(a_pParent, a_pNode) \
605	do { \
606	(a_pParent)->pLeftR3 = (a_pNode); \
607	(a_pParent)->pLeftR0 = (a_pNode)->pSelfR0; \
608	(a_pParent)->pLeftRC = (a_pNode)->pSelfRC; \
609	} while (0)
610	#define INSERT_RIGHT(a_pParent, a_pNode) \
611	do { \
612	(a_pParent)->pRightR3 = (a_pNode); \
613	(a_pParent)->pRightR0 = (a_pNode)->pSelfR0; \
614	(a_pParent)->pRightRC = (a_pNode)->pSelfRC; \
615	} while (0)
616
617
618	/**
619	* Recursive tree builder.
620	*
621	* @param ppRam Pointer to the iterator variable.
622	* @param iHeight The hight about normal leaf nodes. Inserts a leaf
623	* node if 0.
624	*/
625	static PPGMRAMRANGE pgmR3PhysRebuildRamRangeSearchTreesRecursively(PPGMRAMRANGE *ppRam, int iDepth)
626	{
627	PPGMRAMRANGE pRam;
628	if (iDepth <= 0)
629	{
630	/*
631	* Leaf node.
632	*/
633	pRam = *ppRam;
634	if (pRam)
635	{
636	*ppRam = pRam->pNextR3;
637	MAKE_LEAF(pRam);
638	}
639	}
640	else
641	{
642
643	/*
644	* Intermediate node.
645	*/
646	PPGMRAMRANGE pLeft = pgmR3PhysRebuildRamRangeSearchTreesRecursively(ppRam, iDepth - 1);
647
648	pRam = *ppRam;
649	if (!pRam)
650	return pLeft;
651	*ppRam = pRam->pNextR3;
652	MAKE_LEAF(pRam);
653	INSERT_LEFT(pRam, pLeft);
654
655	PPGMRAMRANGE pRight = pgmR3PhysRebuildRamRangeSearchTreesRecursively(ppRam, iDepth - 1);
656	if (pRight)
657	INSERT_RIGHT(pRam, pRight);
658	}
659	return pRam;
660	}
661
662
663	/**
664	* Rebuilds the RAM range search trees.
665	*
666	* @param pVM Pointer to the VM.
667	*/
668	static void pgmR3PhysRebuildRamRangeSearchTrees(PVM pVM)
669	{
670
671	/*
672	* Create the reasonably balanced tree in a sequential fashion.
673	* For simplicity (laziness) we use standard recursion here.
674	*/
675	int iDepth = 0;
676	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
677	PPGMRAMRANGE pRoot = pgmR3PhysRebuildRamRangeSearchTreesRecursively(&pRam, 0);
678	while (pRam)
679	{
680	PPGMRAMRANGE pLeft = pRoot;
681
682	pRoot = pRam;
683	pRam = pRam->pNextR3;
684	MAKE_LEAF(pRoot);
685	INSERT_LEFT(pRoot, pLeft);
686
687	PPGMRAMRANGE pRight = pgmR3PhysRebuildRamRangeSearchTreesRecursively(&pRam, iDepth);
688	if (pRight)
689	INSERT_RIGHT(pRoot, pRight);
690	/** @todo else: rotate the tree. */
691
692	iDepth++;
693	}
694
695	pVM->pgm.s.pRamRangeTreeR3 = pRoot;
696	pVM->pgm.s.pRamRangeTreeR0 = pRoot ? pRoot->pSelfR0 : NIL_RTR0PTR;
697	pVM->pgm.s.pRamRangeTreeRC = pRoot ? pRoot->pSelfRC : NIL_RTRCPTR;
698
699	#ifdef VBOX_STRICT
700	/*
701	* Verify that the above code works.
702	*/
703	unsigned cRanges = 0;
704	for (pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
705	cRanges++;
706	Assert(cRanges > 0);
707
708	unsigned cMaxDepth = ASMBitLastSetU32(cRanges);
709	if ((1U << cMaxDepth) < cRanges)
710	cMaxDepth++;
711
712	for (pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
713	{
714	unsigned cDepth = 0;
715	PPGMRAMRANGE pRam2 = pVM->pgm.s.pRamRangeTreeR3;
716	for (;;)
717	{
718	if (pRam == pRam2)
719	break;
720	Assert(pRam2);
721	if (pRam->GCPhys < pRam2->GCPhys)
722	pRam2 = pRam2->pLeftR3;
723	else
724	pRam2 = pRam2->pRightR3;
725	}
726	AssertMsg(cDepth <= cMaxDepth, ("cDepth=%d cMaxDepth=%d\n", cDepth, cMaxDepth));
727	}
728	#endif /* VBOX_STRICT */
729	}
730
731	#undef MAKE_LEAF
732	#undef INSERT_LEFT
733	#undef INSERT_RIGHT
734
735	/**
736	* Relinks the RAM ranges using the pSelfRC and pSelfR0 pointers.
737	*
738	* Called when anything was relocated.
739	*
740	* @param pVM Pointer to the VM.
741	*/
742	void pgmR3PhysRelinkRamRanges(PVM pVM)
743	{
744	PPGMRAMRANGE pCur;
745
746	#ifdef VBOX_STRICT
747	for (pCur = pVM->pgm.s.pRamRangesXR3; pCur; pCur = pCur->pNextR3)
748	{
749	Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pCur->pSelfR0 == MMHyperCCToR0(pVM, pCur));
750	Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pCur->pSelfRC == MMHyperCCToRC(pVM, pCur));
751	Assert((pCur->GCPhys & PAGE_OFFSET_MASK) == 0);
752	Assert((pCur->GCPhysLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
753	Assert((pCur->cb & PAGE_OFFSET_MASK) == 0);
754	Assert(pCur->cb == pCur->GCPhysLast - pCur->GCPhys + 1);
755	for (PPGMRAMRANGE pCur2 = pVM->pgm.s.pRamRangesXR3; pCur2; pCur2 = pCur2->pNextR3)
756	Assert( pCur2 == pCur
757	\|\| strcmp(pCur2->pszDesc, pCur->pszDesc)); /** @todo fix MMIO ranges!! */
758	}
759	#endif
760
761	pCur = pVM->pgm.s.pRamRangesXR3;
762	if (pCur)
763	{
764	pVM->pgm.s.pRamRangesXR0 = pCur->pSelfR0;
765	pVM->pgm.s.pRamRangesXRC = pCur->pSelfRC;
766
767	for (; pCur->pNextR3; pCur = pCur->pNextR3)
768	{
769	pCur->pNextR0 = pCur->pNextR3->pSelfR0;
770	pCur->pNextRC = pCur->pNextR3->pSelfRC;
771	}
772
773	Assert(pCur->pNextR0 == NIL_RTR0PTR);
774	Assert(pCur->pNextRC == NIL_RTRCPTR);
775	}
776	else
777	{
778	Assert(pVM->pgm.s.pRamRangesXR0 == NIL_RTR0PTR);
779	Assert(pVM->pgm.s.pRamRangesXRC == NIL_RTRCPTR);
780	}
781	ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
782
783	pgmR3PhysRebuildRamRangeSearchTrees(pVM);
784	}
785
786
787	/**
788	* Links a new RAM range into the list.
789	*
790	* @param pVM Pointer to the VM.
791	* @param pNew Pointer to the new list entry.
792	* @param pPrev Pointer to the previous list entry. If NULL, insert as head.
793	*/
794	static void pgmR3PhysLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, PPGMRAMRANGE pPrev)
795	{
796	AssertMsg(pNew->pszDesc, ("%RGp-%RGp\n", pNew->GCPhys, pNew->GCPhysLast));
797	Assert((pNew->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pNew->pSelfR0 == MMHyperCCToR0(pVM, pNew));
798	Assert((pNew->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pNew->pSelfRC == MMHyperCCToRC(pVM, pNew));
799
800	pgmLock(pVM);
801
802	PPGMRAMRANGE pRam = pPrev ? pPrev->pNextR3 : pVM->pgm.s.pRamRangesXR3;
803	pNew->pNextR3 = pRam;
804	pNew->pNextR0 = pRam ? pRam->pSelfR0 : NIL_RTR0PTR;
805	pNew->pNextRC = pRam ? pRam->pSelfRC : NIL_RTRCPTR;
806
807	if (pPrev)
808	{
809	pPrev->pNextR3 = pNew;
810	pPrev->pNextR0 = pNew->pSelfR0;
811	pPrev->pNextRC = pNew->pSelfRC;
812	}
813	else
814	{
815	pVM->pgm.s.pRamRangesXR3 = pNew;
816	pVM->pgm.s.pRamRangesXR0 = pNew->pSelfR0;
817	pVM->pgm.s.pRamRangesXRC = pNew->pSelfRC;
818	}
819	ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
820
821	pgmR3PhysRebuildRamRangeSearchTrees(pVM);
822	pgmUnlock(pVM);
823	}
824
825
826	/**
827	* Unlink an existing RAM range from the list.
828	*
829	* @param pVM Pointer to the VM.
830	* @param pRam Pointer to the new list entry.
831	* @param pPrev Pointer to the previous list entry. If NULL, insert as head.
832	*/
833	static void pgmR3PhysUnlinkRamRange2(PVM pVM, PPGMRAMRANGE pRam, PPGMRAMRANGE pPrev)
834	{
835	Assert(pPrev ? pPrev->pNextR3 == pRam : pVM->pgm.s.pRamRangesXR3 == pRam);
836	Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pRam->pSelfR0 == MMHyperCCToR0(pVM, pRam));
837	Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pRam->pSelfRC == MMHyperCCToRC(pVM, pRam));
838
839	pgmLock(pVM);
840
841	PPGMRAMRANGE pNext = pRam->pNextR3;
842	if (pPrev)
843	{
844	pPrev->pNextR3 = pNext;
845	pPrev->pNextR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
846	pPrev->pNextRC = pNext ? pNext->pSelfRC : NIL_RTRCPTR;
847	}
848	else
849	{
850	Assert(pVM->pgm.s.pRamRangesXR3 == pRam);
851	pVM->pgm.s.pRamRangesXR3 = pNext;
852	pVM->pgm.s.pRamRangesXR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
853	pVM->pgm.s.pRamRangesXRC = pNext ? pNext->pSelfRC : NIL_RTRCPTR;
854	}
855	ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
856
857	pgmR3PhysRebuildRamRangeSearchTrees(pVM);
858	pgmUnlock(pVM);
859	}
860
861
862	/**
863	* Unlink an existing RAM range from the list.
864	*
865	* @param pVM Pointer to the VM.
866	* @param pRam Pointer to the new list entry.
867	*/
868	static void pgmR3PhysUnlinkRamRange(PVM pVM, PPGMRAMRANGE pRam)
869	{
870	pgmLock(pVM);
871
872	/* find prev. */
873	PPGMRAMRANGE pPrev = NULL;
874	PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesXR3;
875	while (pCur != pRam)
876	{
877	pPrev = pCur;
878	pCur = pCur->pNextR3;
879	}
880	AssertFatal(pCur);
881
882	pgmR3PhysUnlinkRamRange2(pVM, pRam, pPrev);
883	pgmUnlock(pVM);
884	}
885
886
887	/**
888	* Frees a range of pages, replacing them with ZERO pages of the specified type.
889	*
890	* @returns VBox status code.
891	* @param pVM Pointer to the VM.
892	* @param pRam The RAM range in which the pages resides.
893	* @param GCPhys The address of the first page.
894	* @param GCPhysLast The address of the last page.
895	* @param uType The page type to replace then with.
896	*/
897	static int pgmR3PhysFreePageRange(PVM pVM, PPGMRAMRANGE pRam, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast, uint8_t uType)
898	{
899	PGM_LOCK_ASSERT_OWNER(pVM);
900	uint32_t cPendingPages = 0;
901	PGMMFREEPAGESREQ pReq;
902	int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
903	AssertLogRelRCReturn(rc, rc);
904
905	/* Iterate the pages. */
906	PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
907	uint32_t cPagesLeft = ((GCPhysLast - GCPhys) >> PAGE_SHIFT) + 1;
908	while (cPagesLeft-- > 0)
909	{
910	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys);
911	AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
912
913	PGM_PAGE_SET_TYPE(pVM, pPageDst, uType);
914
915	GCPhys += PAGE_SIZE;
916	pPageDst++;
917	}
918
919	if (cPendingPages)
920	{
921	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
922	AssertLogRelRCReturn(rc, rc);
923	}
924	GMMR3FreePagesCleanup(pReq);
925
926	return rc;
927	}
928
929	#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) \|\| defined(RT_OS_SOLARIS) \|\| defined(RT_OS_LINUX) \|\| defined(RT_OS_FREEBSD))
930
931	/**
932	* Rendezvous callback used by PGMR3ChangeMemBalloon that changes the memory balloon size
933	*
934	* This is only called on one of the EMTs while the other ones are waiting for
935	* it to complete this function.
936	*
937	* @returns VINF_SUCCESS (VBox strict status code).
938	* @param pVM Pointer to the VM.
939	* @param pVCpu The VMCPU for the EMT we're being called on. Unused.
940	* @param pvUser User parameter
941	*/
942	static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysChangeMemBalloonRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
943	{
944	uintptr_t paUser = (uintptr_t )pvUser;
945	bool fInflate = !!paUser[0];
946	unsigned cPages = paUser[1];
947	RTGCPHYS paPhysPage = (RTGCPHYS )paUser[2];
948	uint32_t cPendingPages = 0;
949	PGMMFREEPAGESREQ pReq;
950	int rc;
951
952	Log(("pgmR3PhysChangeMemBalloonRendezvous: %s %x pages\n", (fInflate) ? "inflate" : "deflate", cPages));
953	pgmLock(pVM);
954
955	if (fInflate)
956	{
957	/* Flush the PGM pool cache as we might have stale references to pages that we just freed. */
958	pgmR3PoolClearAllRendezvous(pVM, pVCpu, NULL);
959
960	/* Replace pages with ZERO pages. */
961	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
962	if (RT_FAILURE(rc))
963	{
964	pgmUnlock(pVM);
965	AssertLogRelRC(rc);
966	return rc;
967	}
968
969	/* Iterate the pages. */
970	for (unsigned i = 0; i < cPages; i++)
971	{
972	PPGMPAGE pPage = pgmPhysGetPage(pVM, paPhysPage[i]);
973	if ( pPage == NULL
974	\|\| PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM)
975	{
976	Log(("pgmR3PhysChangeMemBalloonRendezvous: invalid physical page %RGp pPage->u3Type=%d\n", paPhysPage[i], pPage ? PGM_PAGE_GET_TYPE(pPage) : 0));
977	break;
978	}
979
980	LogFlow(("balloon page: %RGp\n", paPhysPage[i]));
981
982	/* Flush the shadow PT if this page was previously used as a guest page table. */
983	pgmPoolFlushPageByGCPhys(pVM, paPhysPage[i]);
984
985	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, paPhysPage[i]);
986	if (RT_FAILURE(rc))
987	{
988	pgmUnlock(pVM);
989	AssertLogRelRC(rc);
990	return rc;
991	}
992	Assert(PGM_PAGE_IS_ZERO(pPage));
993	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_BALLOONED);
994	}
995
996	if (cPendingPages)
997	{
998	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
999	if (RT_FAILURE(rc))
1000	{
1001	pgmUnlock(pVM);
1002	AssertLogRelRC(rc);
1003	return rc;
1004	}
1005	}
1006	GMMR3FreePagesCleanup(pReq);
1007	}
1008	else
1009	{
1010	/* Iterate the pages. */
1011	for (unsigned i = 0; i < cPages; i++)
1012	{
1013	PPGMPAGE pPage = pgmPhysGetPage(pVM, paPhysPage[i]);
1014	AssertBreak(pPage && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM);
1015
1016	LogFlow(("Free ballooned page: %RGp\n", paPhysPage[i]));
1017
1018	Assert(PGM_PAGE_IS_BALLOONED(pPage));
1019
1020	/* Change back to zero page. */
1021	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
1022	}
1023
1024	/* Note that we currently do not map any ballooned pages in our shadow page tables, so no need to flush the pgm pool. */
1025	}
1026
1027	/* Notify GMM about the balloon change. */
1028	rc = GMMR3BalloonedPages(pVM, (fInflate) ? GMMBALLOONACTION_INFLATE : GMMBALLOONACTION_DEFLATE, cPages);
1029	if (RT_SUCCESS(rc))
1030	{
1031	if (!fInflate)
1032	{
1033	Assert(pVM->pgm.s.cBalloonedPages >= cPages);
1034	pVM->pgm.s.cBalloonedPages -= cPages;
1035	}
1036	else
1037	pVM->pgm.s.cBalloonedPages += cPages;
1038	}
1039
1040	pgmUnlock(pVM);
1041
1042	/* Flush the recompiler's TLB as well. */
1043	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1044	CPUMSetChangedFlags(&pVM->aCpus[i], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
1045
1046	AssertLogRelRC(rc);
1047	return rc;
1048	}
1049
1050
1051	/**
1052	* Frees a range of ram pages, replacing them with ZERO pages; helper for PGMR3PhysFreeRamPages
1053	*
1054	* @returns VBox status code.
1055	* @param pVM Pointer to the VM.
1056	* @param fInflate Inflate or deflate memory balloon
1057	* @param cPages Number of pages to free
1058	* @param paPhysPage Array of guest physical addresses
1059	*/
1060	static DECLCALLBACK(void) pgmR3PhysChangeMemBalloonHelper(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
1061	{
1062	uintptr_t paUser[3];
1063
1064	paUser[0] = fInflate;
1065	paUser[1] = cPages;
1066	paUser[2] = (uintptr_t)paPhysPage;
1067	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
1068	AssertRC(rc);
1069
1070	/* Made a copy in PGMR3PhysFreeRamPages; free it here. */
1071	RTMemFree(paPhysPage);
1072	}
1073
1074	#endif /* 64-bit host && (Windows \|\| Solaris \|\| Linux \|\| FreeBSD) */
1075
1076	/**
1077	* Inflate or deflate a memory balloon
1078	*
1079	* @returns VBox status code.
1080	* @param pVM Pointer to the VM.
1081	* @param fInflate Inflate or deflate memory balloon
1082	* @param cPages Number of pages to free
1083	* @param paPhysPage Array of guest physical addresses
1084	*/
1085	VMMR3DECL(int) PGMR3PhysChangeMemBalloon(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
1086	{
1087	/* This must match GMMR0Init; currently we only support memory ballooning on all 64-bit hosts except Mac OS X */
1088	#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) \|\| defined(RT_OS_SOLARIS) \|\| defined(RT_OS_LINUX) \|\| defined(RT_OS_FREEBSD))
1089	int rc;
1090
1091	/* Older additions (ancient non-functioning balloon code) pass wrong physical addresses. */
1092	AssertReturn(!(paPhysPage[0] & 0xfff), VERR_INVALID_PARAMETER);
1093
1094	/* We own the IOM lock here and could cause a deadlock by waiting for another VCPU that is blocking on the IOM lock.
1095	* In the SMP case we post a request packet to postpone the job.
1096	*/
1097	if (pVM->cCpus > 1)
1098	{
1099	unsigned cbPhysPage = cPages * sizeof(paPhysPage[0]);
1100	RTGCPHYS paPhysPageCopy = (RTGCPHYS )RTMemAlloc(cbPhysPage);
1101	AssertReturn(paPhysPageCopy, VERR_NO_MEMORY);
1102
1103	memcpy(paPhysPageCopy, paPhysPage, cbPhysPage);
1104
1105	rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysChangeMemBalloonHelper, 4, pVM, fInflate, cPages, paPhysPageCopy);
1106	AssertRC(rc);
1107	}
1108	else
1109	{
1110	uintptr_t paUser[3];
1111
1112	paUser[0] = fInflate;
1113	paUser[1] = cPages;
1114	paUser[2] = (uintptr_t)paPhysPage;
1115	rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
1116	AssertRC(rc);
1117	}
1118	return rc;
1119
1120	#else
1121	NOREF(pVM); NOREF(fInflate); NOREF(cPages); NOREF(paPhysPage);
1122	return VERR_NOT_IMPLEMENTED;
1123	#endif
1124	}
1125
1126
1127	/**
1128	* Rendezvous callback used by PGMR3WriteProtectRAM that write protects all
1129	* physical RAM.
1130	*
1131	* This is only called on one of the EMTs while the other ones are waiting for
1132	* it to complete this function.
1133	*
1134	* @returns VINF_SUCCESS (VBox strict status code).
1135	* @param pVM Pointer to the VM.
1136	* @param pVCpu The VMCPU for the EMT we're being called on. Unused.
1137	* @param pvUser User parameter, unused.
1138	*/
1139	static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysWriteProtectRAMRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
1140	{
1141	int rc = VINF_SUCCESS;
1142	NOREF(pvUser); NOREF(pVCpu);
1143
1144	pgmLock(pVM);
1145	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1146	pgmPoolResetDirtyPages(pVM);
1147	#endif
1148
1149	/** @todo pointless to write protect the physical page pointed to by RSP. */
1150
1151	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
1152	pRam;
1153	pRam = pRam->CTX_SUFF(pNext))
1154	{
1155	uint32_t cPages = pRam->cb >> PAGE_SHIFT;
1156	for (uint32_t iPage = 0; iPage < cPages; iPage++)
1157	{
1158	PPGMPAGE pPage = &pRam->aPages[iPage];
1159	PGMPAGETYPE enmPageType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
1160
1161	if ( RT_LIKELY(enmPageType == PGMPAGETYPE_RAM)
1162	\|\| enmPageType == PGMPAGETYPE_MMIO2)
1163	{
1164	/*
1165	* A RAM page.
1166	*/
1167	switch (PGM_PAGE_GET_STATE(pPage))
1168	{
1169	case PGM_PAGE_STATE_ALLOCATED:
1170	/** @todo Optimize this: Don't always re-enable write
1171	* monitoring if the page is known to be very busy. */
1172	if (PGM_PAGE_IS_WRITTEN_TO(pPage))
1173	{
1174	PGM_PAGE_CLEAR_WRITTEN_TO(pVM, pPage);
1175	/* Remember this dirty page for the next (memory) sync. */
1176	PGM_PAGE_SET_FT_DIRTY(pPage);
1177	}
1178
1179	pgmPhysPageWriteMonitor(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1180	break;
1181
1182	case PGM_PAGE_STATE_SHARED:
1183	AssertFailed();
1184	break;
1185
1186	case PGM_PAGE_STATE_WRITE_MONITORED: /* nothing to change. */
1187	default:
1188	break;
1189	}
1190	}
1191	}
1192	}
1193	pgmR3PoolWriteProtectPages(pVM);
1194	PGM_INVL_ALL_VCPU_TLBS(pVM);
1195	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1196	CPUMSetChangedFlags(&pVM->aCpus[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
1197
1198	pgmUnlock(pVM);
1199	return rc;
1200	}
1201
1202	/**
1203	* Protect all physical RAM to monitor writes
1204	*
1205	* @returns VBox status code.
1206	* @param pVM Pointer to the VM.
1207	*/
1208	VMMR3DECL(int) PGMR3PhysWriteProtectRAM(PVM pVM)
1209	{
1210	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
1211
1212	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysWriteProtectRAMRendezvous, NULL);
1213	AssertRC(rc);
1214	return rc;
1215	}
1216
1217	/**
1218	* Enumerate all dirty FT pages.
1219	*
1220	* @returns VBox status code.
1221	* @param pVM Pointer to the VM.
1222	* @param pfnEnum Enumerate callback handler.
1223	* @param pvUser Enumerate callback handler parameter.
1224	*/
1225	VMMR3DECL(int) PGMR3PhysEnumDirtyFTPages(PVM pVM, PFNPGMENUMDIRTYFTPAGES pfnEnum, void *pvUser)
1226	{
1227	int rc = VINF_SUCCESS;
1228
1229	pgmLock(pVM);
1230	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
1231	pRam;
1232	pRam = pRam->CTX_SUFF(pNext))
1233	{
1234	uint32_t cPages = pRam->cb >> PAGE_SHIFT;
1235	for (uint32_t iPage = 0; iPage < cPages; iPage++)
1236	{
1237	PPGMPAGE pPage = &pRam->aPages[iPage];
1238	PGMPAGETYPE enmPageType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
1239
1240	if ( RT_LIKELY(enmPageType == PGMPAGETYPE_RAM)
1241	\|\| enmPageType == PGMPAGETYPE_MMIO2)
1242	{
1243	/*
1244	* A RAM page.
1245	*/
1246	switch (PGM_PAGE_GET_STATE(pPage))
1247	{
1248	case PGM_PAGE_STATE_ALLOCATED:
1249	case PGM_PAGE_STATE_WRITE_MONITORED:
1250	if ( !PGM_PAGE_IS_WRITTEN_TO(pPage) /* not very recently updated? */
1251	&& PGM_PAGE_IS_FT_DIRTY(pPage))
1252	{
1253	unsigned cbPageRange = PAGE_SIZE;
1254	unsigned iPageClean = iPage + 1;
1255	RTGCPHYS GCPhysPage = pRam->GCPhys + iPage * PAGE_SIZE;
1256	uint8_t *pu8Page = NULL;
1257	PGMPAGEMAPLOCK Lock;
1258
1259	/* Find the next clean page, so we can merge adjacent dirty pages. */
1260	for (; iPageClean < cPages; iPageClean++)
1261	{
1262	PPGMPAGE pPageNext = &pRam->aPages[iPageClean];
1263	if ( RT_UNLIKELY(PGM_PAGE_GET_TYPE(pPageNext) != PGMPAGETYPE_RAM)
1264	\|\| PGM_PAGE_GET_STATE(pPageNext) != PGM_PAGE_STATE_ALLOCATED
1265	\|\| PGM_PAGE_IS_WRITTEN_TO(pPageNext)
1266	\|\| !PGM_PAGE_IS_FT_DIRTY(pPageNext)
1267	/* Crossing a chunk boundary? */
1268	\|\| (GCPhysPage & GMM_PAGEID_IDX_MASK) != ((GCPhysPage + cbPageRange) & GMM_PAGEID_IDX_MASK)
1269	)
1270	break;
1271
1272	cbPageRange += PAGE_SIZE;
1273	}
1274
1275	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysPage, (const void **)&pu8Page, &Lock);
1276	if (RT_SUCCESS(rc))
1277	{
1278	/** @todo this is risky; the range might be changed, but little choice as the sync
1279	* costs a lot of time. */
1280	pgmUnlock(pVM);
1281	pfnEnum(pVM, GCPhysPage, pu8Page, cbPageRange, pvUser);
1282	pgmLock(pVM);
1283	PGMPhysReleasePageMappingLock(pVM, &Lock);
1284	}
1285
1286	for (iPage; iPage < iPageClean; iPage++)
1287	PGM_PAGE_CLEAR_FT_DIRTY(&pRam->aPages[iPage]);
1288
1289	iPage = iPageClean - 1;
1290	}
1291	break;
1292	}
1293	}
1294	}
1295	}
1296	pgmUnlock(pVM);
1297	return rc;
1298	}
1299
1300
1301	/**
1302	* Gets the number of ram ranges.
1303	*
1304	* @returns Number of ram ranges. Returns UINT32_MAX if @a pVM is invalid.
1305	* @param pVM Pointer to the VM.
1306	*/
1307	VMMR3DECL(uint32_t) PGMR3PhysGetRamRangeCount(PVM pVM)
1308	{
1309	VM_ASSERT_VALID_EXT_RETURN(pVM, UINT32_MAX);
1310
1311	pgmLock(pVM);
1312	uint32_t cRamRanges = 0;
1313	for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRangesX); pCur; pCur = pCur->CTX_SUFF(pNext))
1314	cRamRanges++;
1315	pgmUnlock(pVM);
1316	return cRamRanges;
1317	}
1318
1319
1320	/**
1321	* Get information about a range.
1322	*
1323	* @returns VINF_SUCCESS or VERR_OUT_OF_RANGE.
1324	* @param pVM Pointer to the VM.
1325	* @param iRange The ordinal of the range.
1326	* @param pGCPhysStart Where to return the start of the range. Optional.
1327	* @param pGCPhysLast Where to return the address of the last byte in the
1328	* range. Optional.
1329	* @param pfIsMmio Where to indicate that this is a pure MMIO range.
1330	* Optional.
1331	*/
1332	VMMR3DECL(int) PGMR3PhysGetRange(PVM pVM, uint32_t iRange, PRTGCPHYS pGCPhysStart, PRTGCPHYS pGCPhysLast,
1333	const char *ppszDesc, bool pfIsMmio)
1334	{
1335	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
1336
1337	pgmLock(pVM);
1338	uint32_t iCurRange = 0;
1339	for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRangesX); pCur; pCur = pCur->CTX_SUFF(pNext), iCurRange++)
1340	if (iCurRange == iRange)
1341	{
1342	if (pGCPhysStart)
1343	*pGCPhysStart = pCur->GCPhys;
1344	if (pGCPhysLast)
1345	*pGCPhysLast = pCur->GCPhysLast;
1346	if (ppszDesc)
1347	*ppszDesc = pCur->pszDesc;
1348	if (pfIsMmio)
1349	*pfIsMmio = !!(pCur->fFlags & PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO);
1350
1351	pgmUnlock(pVM);
1352	return VINF_SUCCESS;
1353	}
1354	pgmUnlock(pVM);
1355	return VERR_OUT_OF_RANGE;
1356	}
1357
1358
1359	/**
1360	* Query the amount of free memory inside VMMR0
1361	*
1362	* @returns VBox status code.
1363	* @param pVM Pointer to the VM.
1364	* @param pcbAllocMem Where to return the amount of memory allocated
1365	* by VMs.
1366	* @param pcbFreeMem Where to return the amount of memory that is
1367	* allocated from the host but not currently used
1368	* by any VMs.
1369	* @param pcbBallonedMem Where to return the sum of memory that is
1370	* currently ballooned by the VMs.
1371	* @param pcbSharedMem Where to return the amount of memory that is
1372	* currently shared.
1373	*/
1374	VMMR3DECL(int) PGMR3QueryGlobalMemoryStats(PVM pVM, uint64_t pcbAllocMem, uint64_t pcbFreeMem,
1375	uint64_t pcbBallonedMem, uint64_t pcbSharedMem)
1376	{
1377	uint64_t cAllocPages = 0;
1378	uint64_t cFreePages = 0;
1379	uint64_t cBalloonPages = 0;
1380	uint64_t cSharedPages = 0;
1381	int rc = GMMR3QueryHypervisorMemoryStats(pVM, &cAllocPages, &cFreePages, &cBalloonPages, &cSharedPages);
1382	AssertRCReturn(rc, rc);
1383
1384	if (pcbAllocMem)
1385	pcbAllocMem = cAllocPages _4K;
1386
1387	if (pcbFreeMem)
1388	pcbFreeMem = cFreePages _4K;
1389
1390	if (pcbBallonedMem)
1391	pcbBallonedMem = cBalloonPages _4K;
1392
1393	if (pcbSharedMem)
1394	pcbSharedMem = cSharedPages _4K;
1395
1396	Log(("PGMR3QueryVMMMemoryStats: all=%llx free=%llx ballooned=%llx shared=%llx\n",
1397	cAllocPages, cFreePages, cBalloonPages, cSharedPages));
1398	return VINF_SUCCESS;
1399	}
1400
1401
1402	/**
1403	* Query memory stats for the VM.
1404	*
1405	* @returns VBox status code.
1406	* @param pVM Pointer to the VM.
1407	* @param pcbTotalMem Where to return total amount memory the VM may
1408	* possibly use.
1409	* @param pcbPrivateMem Where to return the amount of private memory
1410	* currently allocated.
1411	* @param pcbSharedMem Where to return the amount of actually shared
1412	* memory currently used by the VM.
1413	* @param pcbZeroMem Where to return the amount of memory backed by
1414	* zero pages.
1415	*
1416	* @remarks The total mem is normally larger than the sum of the three
1417	* components. There are two reasons for this, first the amount of
1418	* shared memory is what we're sure is shared instead of what could
1419	* possibly be shared with someone. Secondly, because the total may
1420	* include some pure MMIO pages that doesn't go into any of the three
1421	* sub-counts.
1422	*
1423	* @todo Why do we return reused shared pages instead of anything that could
1424	* potentially be shared? Doesn't this mean the first VM gets a much
1425	* lower number of shared pages?
1426	*/
1427	VMMR3DECL(int) PGMR3QueryMemoryStats(PVM pVM, uint64_t pcbTotalMem, uint64_t pcbPrivateMem,
1428	uint64_t pcbSharedMem, uint64_t pcbZeroMem)
1429	{
1430	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
1431
1432	if (pcbTotalMem)
1433	pcbTotalMem = (uint64_t)pVM->pgm.s.cAllPages PAGE_SIZE;
1434
1435	if (pcbPrivateMem)
1436	pcbPrivateMem = (uint64_t)pVM->pgm.s.cPrivatePages PAGE_SIZE;
1437
1438	if (pcbSharedMem)
1439	pcbSharedMem = (uint64_t)pVM->pgm.s.cReusedSharedPages PAGE_SIZE;
1440
1441	if (pcbZeroMem)
1442	pcbZeroMem = (uint64_t)pVM->pgm.s.cZeroPages PAGE_SIZE;
1443
1444	Log(("PGMR3QueryMemoryStats: all=%x private=%x reused=%x zero=%x\n", pVM->pgm.s.cAllPages, pVM->pgm.s.cPrivatePages, pVM->pgm.s.cReusedSharedPages, pVM->pgm.s.cZeroPages));
1445	return VINF_SUCCESS;
1446	}
1447
1448
1449	/**
1450	* PGMR3PhysRegisterRam worker that initializes and links a RAM range.
1451	*
1452	* @param pVM Pointer to the VM.
1453	* @param pNew The new RAM range.
1454	* @param GCPhys The address of the RAM range.
1455	* @param GCPhysLast The last address of the RAM range.
1456	* @param RCPtrNew The RC address if the range is floating. NIL_RTRCPTR
1457	* if in HMA.
1458	* @param R0PtrNew Ditto for R0.
1459	* @param pszDesc The description.
1460	* @param pPrev The previous RAM range (for linking).
1461	*/
1462	static void pgmR3PhysInitAndLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast,
1463	RTRCPTR RCPtrNew, RTR0PTR R0PtrNew, const char *pszDesc, PPGMRAMRANGE pPrev)
1464	{
1465	/*
1466	* Initialize the range.
1467	*/
1468	pNew->pSelfR0 = R0PtrNew != NIL_RTR0PTR ? R0PtrNew : MMHyperCCToR0(pVM, pNew);
1469	pNew->pSelfRC = RCPtrNew != NIL_RTRCPTR ? RCPtrNew : MMHyperCCToRC(pVM, pNew);
1470	pNew->GCPhys = GCPhys;
1471	pNew->GCPhysLast = GCPhysLast;
1472	pNew->cb = GCPhysLast - GCPhys + 1;
1473	pNew->pszDesc = pszDesc;
1474	pNew->fFlags = RCPtrNew != NIL_RTRCPTR ? PGM_RAM_RANGE_FLAGS_FLOATING : 0;
1475	pNew->pvR3 = NULL;
1476	pNew->paLSPages = NULL;
1477
1478	uint32_t const cPages = pNew->cb >> PAGE_SHIFT;
1479	RTGCPHYS iPage = cPages;
1480	while (iPage-- > 0)
1481	PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_RAM);
1482
1483	/* Update the page count stats. */
1484	pVM->pgm.s.cZeroPages += cPages;
1485	pVM->pgm.s.cAllPages += cPages;
1486
1487	/*
1488	* Link it.
1489	*/
1490	pgmR3PhysLinkRamRange(pVM, pNew, pPrev);
1491	}
1492
1493
1494	/**
1495	* Relocate a floating RAM range.
1496	*
1497	* @copydoc FNPGMRELOCATE.
1498	*/
1499	static DECLCALLBACK(bool) pgmR3PhysRamRangeRelocate(PVM pVM, RTGCPTR GCPtrOld, RTGCPTR GCPtrNew, PGMRELOCATECALL enmMode, void *pvUser)
1500	{
1501	PPGMRAMRANGE pRam = (PPGMRAMRANGE)pvUser;
1502	Assert(pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING);
1503	Assert(pRam->pSelfRC == GCPtrOld + PAGE_SIZE);
1504
1505	switch (enmMode)
1506	{
1507	case PGMRELOCATECALL_SUGGEST:
1508	return true;
1509
1510	case PGMRELOCATECALL_RELOCATE:
1511	{
1512	/*
1513	* Update myself, then relink all the ranges and flush the RC TLB.
1514	*/
1515	pgmLock(pVM);
1516
1517	pRam->pSelfRC = (RTRCPTR)(GCPtrNew + PAGE_SIZE);
1518
1519	pgmR3PhysRelinkRamRanges(pVM);
1520	for (unsigned i = 0; i < PGM_RAMRANGE_TLB_ENTRIES; i++)
1521	pVM->pgm.s.apRamRangesTlbRC[i] = NIL_RTRCPTR;
1522
1523	pgmUnlock(pVM);
1524	return true;
1525	}
1526
1527	default:
1528	AssertFailedReturn(false);
1529	}
1530	}
1531
1532
1533	/**
1534	* PGMR3PhysRegisterRam worker that registers a high chunk.
1535	*
1536	* @returns VBox status code.
1537	* @param pVM Pointer to the VM.
1538	* @param GCPhys The address of the RAM.
1539	* @param cRamPages The number of RAM pages to register.
1540	* @param cbChunk The size of the PGMRAMRANGE guest mapping.
1541	* @param iChunk The chunk number.
1542	* @param pszDesc The RAM range description.
1543	* @param ppPrev Previous RAM range pointer. In/Out.
1544	*/
1545	static int pgmR3PhysRegisterHighRamChunk(PVM pVM, RTGCPHYS GCPhys, uint32_t cRamPages,
1546	uint32_t cbChunk, uint32_t iChunk, const char *pszDesc,
1547	PPGMRAMRANGE *ppPrev)
1548	{
1549	const char *pszDescChunk = iChunk == 0
1550	? pszDesc
1551	: MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s (#%u)", pszDesc, iChunk + 1);
1552	AssertReturn(pszDescChunk, VERR_NO_MEMORY);
1553
1554	/*
1555	* Allocate memory for the new chunk.
1556	*/
1557	size_t const cChunkPages = RT_ALIGN_Z(RT_UOFFSETOF(PGMRAMRANGE, aPages[cRamPages]), PAGE_SIZE) >> PAGE_SHIFT;
1558	PSUPPAGE paChunkPages = (PSUPPAGE)RTMemTmpAllocZ(sizeof(SUPPAGE) * cChunkPages);
1559	AssertReturn(paChunkPages, VERR_NO_TMP_MEMORY);
1560	RTR0PTR R0PtrChunk = NIL_RTR0PTR;
1561	void *pvChunk = NULL;
1562	int rc = SUPR3PageAllocEx(cChunkPages, 0 /fFlags/, &pvChunk,
1563	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1564	VMMIsHwVirtExtForced(pVM) ? &R0PtrChunk : NULL,
1565	#else
1566	NULL,
1567	#endif
1568	paChunkPages);
1569	if (RT_SUCCESS(rc))
1570	{
1571	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1572	if (!VMMIsHwVirtExtForced(pVM))
1573	R0PtrChunk = NIL_RTR0PTR;
1574	#else
1575	R0PtrChunk = (uintptr_t)pvChunk;
1576	#endif
1577	memset(pvChunk, 0, cChunkPages << PAGE_SHIFT);
1578
1579	PPGMRAMRANGE pNew = (PPGMRAMRANGE)pvChunk;
1580
1581	/*
1582	* Create a mapping and map the pages into it.
1583	* We push these in below the HMA.
1584	*/
1585	RTGCPTR GCPtrChunkMap = pVM->pgm.s.GCPtrPrevRamRangeMapping - cbChunk;
1586	rc = PGMR3MapPT(pVM, GCPtrChunkMap, cbChunk, 0 /fFlags/, pgmR3PhysRamRangeRelocate, pNew, pszDescChunk);
1587	if (RT_SUCCESS(rc))
1588	{
1589	pVM->pgm.s.GCPtrPrevRamRangeMapping = GCPtrChunkMap;
1590
1591	RTGCPTR const GCPtrChunk = GCPtrChunkMap + PAGE_SIZE;
1592	RTGCPTR GCPtrPage = GCPtrChunk;
1593	for (uint32_t iPage = 0; iPage < cChunkPages && RT_SUCCESS(rc); iPage++, GCPtrPage += PAGE_SIZE)
1594	rc = PGMMap(pVM, GCPtrPage, paChunkPages[iPage].Phys, PAGE_SIZE, 0);
1595	if (RT_SUCCESS(rc))
1596	{
1597	/*
1598	* Ok, init and link the range.
1599	*/
1600	pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhys + ((RTGCPHYS)cRamPages << PAGE_SHIFT) - 1,
1601	(RTRCPTR)GCPtrChunk, R0PtrChunk, pszDescChunk, *ppPrev);
1602	*ppPrev = pNew;
1603	}
1604	}
1605
1606	if (RT_FAILURE(rc))
1607	SUPR3PageFreeEx(pvChunk, cChunkPages);
1608	}
1609
1610	RTMemTmpFree(paChunkPages);
1611	return rc;
1612	}
1613
1614
1615	/**
1616	* Sets up a range RAM.
1617	*
1618	* This will check for conflicting registrations, make a resource
1619	* reservation for the memory (with GMM), and setup the per-page
1620	* tracking structures (PGMPAGE).
1621	*
1622	* @returns VBox status code.
1623	* @param pVM Pointer to the VM.
1624	* @param GCPhys The physical address of the RAM.
1625	* @param cb The size of the RAM.
1626	* @param pszDesc The description - not copied, so, don't free or change it.
1627	*/
1628	VMMR3DECL(int) PGMR3PhysRegisterRam(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, const char *pszDesc)
1629	{
1630	/*
1631	* Validate input.
1632	*/
1633	Log(("PGMR3PhysRegisterRam: GCPhys=%RGp cb=%RGp pszDesc=%s\n", GCPhys, cb, pszDesc));
1634	AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
1635	AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
1636	AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
1637	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
1638	AssertMsgReturn(GCPhysLast > GCPhys, ("The range wraps! GCPhys=%RGp cb=%RGp\n", GCPhys, cb), VERR_INVALID_PARAMETER);
1639	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
1640	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
1641
1642	pgmLock(pVM);
1643
1644	/*
1645	* Find range location and check for conflicts.
1646	* (We don't lock here because the locking by EMT is only required on update.)
1647	*/
1648	PPGMRAMRANGE pPrev = NULL;
1649	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
1650	while (pRam && GCPhysLast >= pRam->GCPhys)
1651	{
1652	if ( GCPhysLast >= pRam->GCPhys
1653	&& GCPhys <= pRam->GCPhysLast)
1654	AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
1655	GCPhys, GCPhysLast, pszDesc,
1656	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
1657	VERR_PGM_RAM_CONFLICT);
1658
1659	/* next */
1660	pPrev = pRam;
1661	pRam = pRam->pNextR3;
1662	}
1663
1664	/*
1665	* Register it with GMM (the API bitches).
1666	*/
1667	const RTGCPHYS cPages = cb >> PAGE_SHIFT;
1668	int rc = MMR3IncreaseBaseReservation(pVM, cPages);
1669	if (RT_FAILURE(rc))
1670	{
1671	pgmUnlock(pVM);
1672	return rc;
1673	}
1674
1675	if ( GCPhys >= _4G
1676	&& cPages > 256)
1677	{
1678	/*
1679	* The PGMRAMRANGE structures for the high memory can get very big.
1680	* In order to avoid SUPR3PageAllocEx allocation failures due to the
1681	* allocation size limit there and also to avoid being unable to find
1682	* guest mapping space for them, we split this memory up into 4MB in
1683	* (potential) raw-mode configs and 16MB chunks in forced AMD-V/VT-x
1684	* mode.
1685	*
1686	* The first and last page of each mapping are guard pages and marked
1687	* not-present. So, we've got 4186112 and 16769024 bytes available for
1688	* the PGMRAMRANGE structure.
1689	*
1690	* Note! The sizes used here will influence the saved state.
1691	*/
1692	uint32_t cbChunk;
1693	uint32_t cPagesPerChunk;
1694	if (VMMIsHwVirtExtForced(pVM))
1695	{
1696	cbChunk = 16U*_1M;
1697	cPagesPerChunk = 1048048; /* max ~1048059 */
1698	AssertCompile(sizeof(PGMRAMRANGE) + sizeof(PGMPAGE) * 1048048 < 16U_1M - PAGE_SIZE 2);
1699	}
1700	else
1701	{
1702	cbChunk = 4U*_1M;
1703	cPagesPerChunk = 261616; /* max ~261627 */
1704	AssertCompile(sizeof(PGMRAMRANGE) + sizeof(PGMPAGE) * 261616 < 4U_1M - PAGE_SIZE 2);
1705	}
1706	AssertRelease(RT_UOFFSETOF(PGMRAMRANGE, aPages[cPagesPerChunk]) + PAGE_SIZE * 2 <= cbChunk);
1707
1708	RTGCPHYS cPagesLeft = cPages;
1709	RTGCPHYS GCPhysChunk = GCPhys;
1710	uint32_t iChunk = 0;
1711	while (cPagesLeft > 0)
1712	{
1713	uint32_t cPagesInChunk = cPagesLeft;
1714	if (cPagesInChunk > cPagesPerChunk)
1715	cPagesInChunk = cPagesPerChunk;
1716
1717	rc = pgmR3PhysRegisterHighRamChunk(pVM, GCPhysChunk, cPagesInChunk, cbChunk, iChunk, pszDesc, &pPrev);
1718	AssertRCReturn(rc, rc);
1719
1720	/* advance */
1721	GCPhysChunk += (RTGCPHYS)cPagesInChunk << PAGE_SHIFT;
1722	cPagesLeft -= cPagesInChunk;
1723	iChunk++;
1724	}
1725	}
1726	else
1727	{
1728	/*
1729	* Allocate, initialize and link the new RAM range.
1730	*/
1731	const size_t cbRamRange = RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]);
1732	PPGMRAMRANGE pNew;
1733	rc = MMR3HyperAllocOnceNoRel(pVM, cbRamRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
1734	AssertLogRelMsgRCReturn(rc, ("cbRamRange=%zu\n", cbRamRange), rc);
1735
1736	pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhysLast, NIL_RTRCPTR, NIL_RTR0PTR, pszDesc, pPrev);
1737	}
1738	pgmPhysInvalidatePageMapTLB(pVM);
1739	pgmUnlock(pVM);
1740
1741	#ifdef VBOX_WITH_REM
1742	/*
1743	* Notify REM.
1744	*/
1745	REMR3NotifyPhysRamRegister(pVM, GCPhys, cb, REM_NOTIFY_PHYS_RAM_FLAGS_RAM);
1746	#endif
1747
1748	return VINF_SUCCESS;
1749	}
1750
1751
1752	/**
1753	* Worker called by PGMR3InitFinalize if we're configured to pre-allocate RAM.
1754	*
1755	* We do this late in the init process so that all the ROM and MMIO ranges have
1756	* been registered already and we don't go wasting memory on them.
1757	*
1758	* @returns VBox status code.
1759	*
1760	* @param pVM Pointer to the VM.
1761	*/
1762	int pgmR3PhysRamPreAllocate(PVM pVM)
1763	{
1764	Assert(pVM->pgm.s.fRamPreAlloc);
1765	Log(("pgmR3PhysRamPreAllocate: enter\n"));
1766
1767	/*
1768	* Walk the RAM ranges and allocate all RAM pages, halt at
1769	* the first allocation error.
1770	*/
1771	uint64_t cPages = 0;
1772	uint64_t NanoTS = RTTimeNanoTS();
1773	pgmLock(pVM);
1774	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
1775	{
1776	PPGMPAGE pPage = &pRam->aPages[0];
1777	RTGCPHYS GCPhys = pRam->GCPhys;
1778	uint32_t cLeft = pRam->cb >> PAGE_SHIFT;
1779	while (cLeft-- > 0)
1780	{
1781	if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
1782	{
1783	switch (PGM_PAGE_GET_STATE(pPage))
1784	{
1785	case PGM_PAGE_STATE_ZERO:
1786	{
1787	int rc = pgmPhysAllocPage(pVM, pPage, GCPhys);
1788	if (RT_FAILURE(rc))
1789	{
1790	LogRel(("PGM: RAM Pre-allocation failed at %RGp (in %s) with rc=%Rrc\n", GCPhys, pRam->pszDesc, rc));
1791	pgmUnlock(pVM);
1792	return rc;
1793	}
1794	cPages++;
1795	break;
1796	}
1797
1798	case PGM_PAGE_STATE_BALLOONED:
1799	case PGM_PAGE_STATE_ALLOCATED:
1800	case PGM_PAGE_STATE_WRITE_MONITORED:
1801	case PGM_PAGE_STATE_SHARED:
1802	/* nothing to do here. */
1803	break;
1804	}
1805	}
1806
1807	/* next */
1808	pPage++;
1809	GCPhys += PAGE_SIZE;
1810	}
1811	}
1812	pgmUnlock(pVM);
1813	NanoTS = RTTimeNanoTS() - NanoTS;
1814
1815	LogRel(("PGM: Pre-allocated %llu pages in %llu ms\n", cPages, NanoTS / 1000000));
1816	Log(("pgmR3PhysRamPreAllocate: returns VINF_SUCCESS\n"));
1817	return VINF_SUCCESS;
1818	}
1819
1820
1821	/**
1822	* Checks shared page checksums.
1823	*
1824	* @param pVM Pointer to the shared VM structure.
1825	*/
1826	void pgmR3PhysAssertSharedPageChecksums(PVM pVM)
1827	{
1828	#ifdef VBOX_STRICT
1829	pgmLock(pVM);
1830
1831	if (pVM->pgm.s.cSharedPages > 0)
1832	{
1833	/*
1834	* Walk the ram ranges.
1835	*/
1836	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
1837	{
1838	uint32_t iPage = pRam->cb >> PAGE_SHIFT;
1839	AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
1840
1841	while (iPage-- > 0)
1842	{
1843	PPGMPAGE pPage = &pRam->aPages[iPage];
1844	if (PGM_PAGE_IS_SHARED(pPage))
1845	{
1846	uint32_t u32Checksum = pPage->s.u2Unused0 \| ((uint32_t)pPage->s.u2Unused1 << 8);
1847	if (!u32Checksum)
1848	{
1849	RTGCPHYS GCPhysPage = pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
1850	void const *pvPage;
1851	int rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhysPage, &pvPage);
1852	if (RT_SUCCESS(rc))
1853	{
1854	uint32_t u32Checksum2 = RTCrc32(pvPage, PAGE_SIZE);
1855	# if 0
1856	AssertMsg((u32Checksum2 & UINT32_C(0x00000303)) == u32Checksum, ("GCPhysPage=%RGp\n", GCPhysPage));
1857	# else
1858	if ((u32Checksum2 & UINT32_C(0x00000303)) == u32Checksum)
1859	LogFlow(("shpg %#x @ %RGp %#x [OK]\n", PGM_PAGE_GET_PAGEID(pPage), GCPhysPage, u32Checksum2));
1860	else
1861	AssertMsgFailed(("shpg %#x @ %RGp %#x\n", PGM_PAGE_GET_PAGEID(pPage), GCPhysPage, u32Checksum2));
1862	# endif
1863	}
1864	else
1865	AssertRC(rc);
1866	}
1867	}
1868
1869	} /* for each page */
1870
1871	} /* for each ram range */
1872	}
1873
1874	pgmUnlock(pVM);
1875	#endif /* VBOX_STRICT */
1876	NOREF(pVM);
1877	}
1878
1879
1880	/**
1881	* Resets (zeros) the RAM.
1882	*
1883	* ASSUMES that the caller owns the PGM lock.
1884	*
1885	* @returns VBox status code.
1886	* @param pVM Pointer to the VM.
1887	*/
1888	int pgmR3PhysRamReset(PVM pVM)
1889	{
1890	PGM_LOCK_ASSERT_OWNER(pVM);
1891
1892	/* Reset the memory balloon. */
1893	int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
1894	AssertRC(rc);
1895
1896	#ifdef VBOX_WITH_PAGE_SHARING
1897	/* Clear all registered shared modules. */
1898	pgmR3PhysAssertSharedPageChecksums(pVM);
1899	rc = GMMR3ResetSharedModules(pVM);
1900	AssertRC(rc);
1901	#endif
1902	/* Reset counters. */
1903	pVM->pgm.s.cReusedSharedPages = 0;
1904	pVM->pgm.s.cBalloonedPages = 0;
1905
1906	/*
1907	* We batch up pages that should be freed instead of calling GMM for
1908	* each and every one of them.
1909	*/
1910	uint32_t cPendingPages = 0;
1911	PGMMFREEPAGESREQ pReq;
1912	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
1913	AssertLogRelRCReturn(rc, rc);
1914
1915	/*
1916	* Walk the ram ranges.
1917	*/
1918	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
1919	{
1920	uint32_t iPage = pRam->cb >> PAGE_SHIFT;
1921	AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
1922
1923	#ifndef NO_RAM_RESET
1924	if (!pVM->pgm.s.fRamPreAlloc)
1925	#else
1926	if (0)
1927	#endif
1928	{
1929	/* Replace all RAM pages by ZERO pages. */
1930	while (iPage-- > 0)
1931	{
1932	PPGMPAGE pPage = &pRam->aPages[iPage];
1933	switch (PGM_PAGE_GET_TYPE(pPage))
1934	{
1935	case PGMPAGETYPE_RAM:
1936	/* Do not replace pages part of a 2 MB continuous range
1937	with zero pages, but zero them instead. */
1938	if ( PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE
1939	\|\| PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED)
1940	{
1941	void *pvPage;
1942	rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
1943	AssertLogRelRCReturn(rc, rc);
1944	ASMMemZeroPage(pvPage);
1945	}
1946	else if (PGM_PAGE_IS_BALLOONED(pPage))
1947	{
1948	/* Turn into a zero page; the balloon status is lost when the VM reboots. */
1949	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
1950	}
1951	else if (!PGM_PAGE_IS_ZERO(pPage))
1952	{
1953	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1954	AssertLogRelRCReturn(rc, rc);
1955	}
1956	break;
1957
1958	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
1959	pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
1960	true /fDoAccounting/);
1961	break;
1962
1963	case PGMPAGETYPE_MMIO2:
1964	case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
1965	case PGMPAGETYPE_ROM:
1966	case PGMPAGETYPE_MMIO:
1967	break;
1968	default:
1969	AssertFailed();
1970	}
1971	} /* for each page */
1972	}
1973	else
1974	{
1975	/* Zero the memory. */
1976	while (iPage-- > 0)
1977	{
1978	PPGMPAGE pPage = &pRam->aPages[iPage];
1979	switch (PGM_PAGE_GET_TYPE(pPage))
1980	{
1981	case PGMPAGETYPE_RAM:
1982	switch (PGM_PAGE_GET_STATE(pPage))
1983	{
1984	case PGM_PAGE_STATE_ZERO:
1985	break;
1986
1987	case PGM_PAGE_STATE_BALLOONED:
1988	/* Turn into a zero page; the balloon status is lost when the VM reboots. */
1989	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
1990	break;
1991
1992	case PGM_PAGE_STATE_SHARED:
1993	case PGM_PAGE_STATE_WRITE_MONITORED:
1994	rc = pgmPhysPageMakeWritable(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1995	AssertLogRelRCReturn(rc, rc);
1996	/* no break */
1997
1998	case PGM_PAGE_STATE_ALLOCATED:
1999	{
2000	void *pvPage;
2001	rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
2002	AssertLogRelRCReturn(rc, rc);
2003	#ifndef NO_RAM_RESET
2004	ASMMemZeroPage(pvPage);
2005	#endif
2006	break;
2007	}
2008	}
2009	break;
2010
2011	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
2012	pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
2013	true /fDoAccounting/);
2014	break;
2015
2016	case PGMPAGETYPE_MMIO2:
2017	case PGMPAGETYPE_ROM_SHADOW:
2018	case PGMPAGETYPE_ROM:
2019	case PGMPAGETYPE_MMIO:
2020	break;
2021	default:
2022	AssertFailed();
2023
2024	}
2025	} /* for each page */
2026	}
2027
2028	}
2029
2030	/*
2031	* Finish off any pages pending freeing.
2032	*/
2033	if (cPendingPages)
2034	{
2035	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2036	AssertLogRelRCReturn(rc, rc);
2037	}
2038	GMMR3FreePagesCleanup(pReq);
2039
2040	return VINF_SUCCESS;
2041	}
2042
2043
2044	/**
2045	* Frees all RAM during VM termination
2046	*
2047	* ASSUMES that the caller owns the PGM lock.
2048	*
2049	* @returns VBox status code.
2050	* @param pVM Pointer to the VM.
2051	*/
2052	int pgmR3PhysRamTerm(PVM pVM)
2053	{
2054	PGM_LOCK_ASSERT_OWNER(pVM);
2055
2056	/* Reset the memory balloon. */
2057	int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
2058	AssertRC(rc);
2059
2060	#ifdef VBOX_WITH_PAGE_SHARING
2061	/* Clear all registered shared modules. */
2062	pgmR3PhysAssertSharedPageChecksums(pVM);
2063	rc = GMMR3ResetSharedModules(pVM);
2064	AssertRC(rc);
2065	#endif
2066
2067	/*
2068	* We batch up pages that should be freed instead of calling GMM for
2069	* each and every one of them.
2070	*/
2071	uint32_t cPendingPages = 0;
2072	PGMMFREEPAGESREQ pReq;
2073	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
2074	AssertLogRelRCReturn(rc, rc);
2075
2076	/*
2077	* Walk the ram ranges.
2078	*/
2079	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
2080	{
2081	uint32_t iPage = pRam->cb >> PAGE_SHIFT;
2082	AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
2083
2084	while (iPage-- > 0)
2085	{
2086	PPGMPAGE pPage = &pRam->aPages[iPage];
2087	switch (PGM_PAGE_GET_TYPE(pPage))
2088	{
2089	case PGMPAGETYPE_RAM:
2090	/* Free all shared pages. Private pages are automatically freed during GMM VM cleanup. */
2091	/** @todo change this to explicitly free private pages here. */
2092	if (PGM_PAGE_IS_SHARED(pPage))
2093	{
2094	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
2095	AssertLogRelRCReturn(rc, rc);
2096	}
2097	break;
2098
2099	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
2100	case PGMPAGETYPE_MMIO2:
2101	case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
2102	case PGMPAGETYPE_ROM:
2103	case PGMPAGETYPE_MMIO:
2104	break;
2105	default:
2106	AssertFailed();
2107	}
2108	} /* for each page */
2109	}
2110
2111	/*
2112	* Finish off any pages pending freeing.
2113	*/
2114	if (cPendingPages)
2115	{
2116	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2117	AssertLogRelRCReturn(rc, rc);
2118	}
2119	GMMR3FreePagesCleanup(pReq);
2120	return VINF_SUCCESS;
2121	}
2122
2123	/**
2124	* This is the interface IOM is using to register an MMIO region.
2125	*
2126	* It will check for conflicts and ensure that a RAM range structure
2127	* is present before calling the PGMR3HandlerPhysicalRegister API to
2128	* register the callbacks.
2129	*
2130	* @returns VBox status code.
2131	*
2132	* @param pVM Pointer to the VM.
2133	* @param GCPhys The start of the MMIO region.
2134	* @param cb The size of the MMIO region.
2135	* @param pfnHandlerR3 The address of the ring-3 handler. (IOMR3MMIOHandler)
2136	* @param pvUserR3 The user argument for R3.
2137	* @param pfnHandlerR0 The address of the ring-0 handler. (IOMMMIOHandler)
2138	* @param pvUserR0 The user argument for R0.
2139	* @param pfnHandlerRC The address of the RC handler. (IOMMMIOHandler)
2140	* @param pvUserRC The user argument for RC.
2141	* @param pszDesc The description of the MMIO region.
2142	*/
2143	VMMR3DECL(int) PGMR3PhysMMIORegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb,
2144	R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnHandlerR3, RTR3PTR pvUserR3,
2145	R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnHandlerR0, RTR0PTR pvUserR0,
2146	RCPTRTYPE(PFNPGMRCPHYSHANDLER) pfnHandlerRC, RTRCPTR pvUserRC,
2147	R3PTRTYPE(const char *) pszDesc)
2148	{
2149	/*
2150	* Assert on some assumption.
2151	*/
2152	VM_ASSERT_EMT(pVM);
2153	AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2154	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2155	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
2156	AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
2157
2158	int rc = pgmLock(pVM);
2159	AssertRCReturn(rc, rc);
2160
2161	/*
2162	* Make sure there's a RAM range structure for the region.
2163	*/
2164	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
2165	bool fRamExists = false;
2166	PPGMRAMRANGE pRamPrev = NULL;
2167	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2168	while (pRam && GCPhysLast >= pRam->GCPhys)
2169	{
2170	if ( GCPhysLast >= pRam->GCPhys
2171	&& GCPhys <= pRam->GCPhysLast)
2172	{
2173	/* Simplification: all within the same range. */
2174	AssertLogRelMsgReturnStmt( GCPhys >= pRam->GCPhys
2175	&& GCPhysLast <= pRam->GCPhysLast,
2176	("%RGp-%RGp (MMIO/%s) falls partly outside %RGp-%RGp (%s)\n",
2177	GCPhys, GCPhysLast, pszDesc,
2178	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
2179	pgmUnlock(pVM),
2180	VERR_PGM_RAM_CONFLICT);
2181
2182	/* Check that it's all RAM or MMIO pages. */
2183	PCPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2184	uint32_t cLeft = cb >> PAGE_SHIFT;
2185	while (cLeft-- > 0)
2186	{
2187	AssertLogRelMsgReturnStmt( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2188	\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO,
2189	("%RGp-%RGp (MMIO/%s): %RGp is not a RAM or MMIO page - type=%d desc=%s\n",
2190	GCPhys, GCPhysLast, pszDesc, PGM_PAGE_GET_TYPE(pPage), pRam->pszDesc),
2191	pgmUnlock(pVM),
2192	VERR_PGM_RAM_CONFLICT);
2193	pPage++;
2194	}
2195
2196	/* Looks good. */
2197	fRamExists = true;
2198	break;
2199	}
2200
2201	/* next */
2202	pRamPrev = pRam;
2203	pRam = pRam->pNextR3;
2204	}
2205	PPGMRAMRANGE pNew;
2206	if (fRamExists)
2207	{
2208	pNew = NULL;
2209
2210	/*
2211	* Make all the pages in the range MMIO/ZERO pages, freeing any
2212	* RAM pages currently mapped here. This might not be 100% correct
2213	* for PCI memory, but we're doing the same thing for MMIO2 pages.
2214	*/
2215	rc = pgmR3PhysFreePageRange(pVM, pRam, GCPhys, GCPhysLast, PGMPAGETYPE_MMIO);
2216	AssertRCReturnStmt(rc, pgmUnlock(pVM), rc);
2217
2218	/* Force a PGM pool flush as guest ram references have been changed. */
2219	/** @todo not entirely SMP safe; assuming for now the guest takes
2220	* care of this internally (not touch mapped mmio while changing the
2221	* mapping). */
2222	PVMCPU pVCpu = VMMGetCpu(pVM);
2223	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2224	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2225	}
2226	else
2227	{
2228
2229	/*
2230	* No RAM range, insert an ad hoc one.
2231	*
2232	* Note that we don't have to tell REM about this range because
2233	* PGMHandlerPhysicalRegisterEx will do that for us.
2234	*/
2235	Log(("PGMR3PhysMMIORegister: Adding ad hoc MMIO range for %RGp-%RGp %s\n", GCPhys, GCPhysLast, pszDesc));
2236
2237	const uint32_t cPages = cb >> PAGE_SHIFT;
2238	const size_t cbRamRange = RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]);
2239	rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]), 16, MM_TAG_PGM_PHYS, (void **)&pNew);
2240	AssertLogRelMsgRCReturnStmt(rc, ("cbRamRange=%zu\n", cbRamRange), pgmUnlock(pVM), rc);
2241
2242	/* Initialize the range. */
2243	pNew->pSelfR0 = MMHyperCCToR0(pVM, pNew);
2244	pNew->pSelfRC = MMHyperCCToRC(pVM, pNew);
2245	pNew->GCPhys = GCPhys;
2246	pNew->GCPhysLast = GCPhysLast;
2247	pNew->cb = cb;
2248	pNew->pszDesc = pszDesc;
2249	pNew->fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO;
2250	pNew->pvR3 = NULL;
2251	pNew->paLSPages = NULL;
2252
2253	uint32_t iPage = cPages;
2254	while (iPage-- > 0)
2255	PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_MMIO);
2256	Assert(PGM_PAGE_GET_TYPE(&pNew->aPages[0]) == PGMPAGETYPE_MMIO);
2257
2258	/* update the page count stats. */
2259	pVM->pgm.s.cPureMmioPages += cPages;
2260	pVM->pgm.s.cAllPages += cPages;
2261
2262	/* link it */
2263	pgmR3PhysLinkRamRange(pVM, pNew, pRamPrev);
2264	}
2265
2266	/*
2267	* Register the access handler.
2268	*/
2269	rc = PGMHandlerPhysicalRegisterEx(pVM, PGMPHYSHANDLERTYPE_MMIO, GCPhys, GCPhysLast,
2270	pfnHandlerR3, pvUserR3,
2271	pfnHandlerR0, pvUserR0,
2272	pfnHandlerRC, pvUserRC, pszDesc);
2273	if ( RT_FAILURE(rc)
2274	&& !fRamExists)
2275	{
2276	pVM->pgm.s.cPureMmioPages -= cb >> PAGE_SHIFT;
2277	pVM->pgm.s.cAllPages -= cb >> PAGE_SHIFT;
2278
2279	/* remove the ad hoc range. */
2280	pgmR3PhysUnlinkRamRange2(pVM, pNew, pRamPrev);
2281	pNew->cb = pNew->GCPhys = pNew->GCPhysLast = NIL_RTGCPHYS;
2282	MMHyperFree(pVM, pRam);
2283	}
2284	pgmPhysInvalidatePageMapTLB(pVM);
2285
2286	pgmUnlock(pVM);
2287	return rc;
2288	}
2289
2290
2291	/**
2292	* This is the interface IOM is using to register an MMIO region.
2293	*
2294	* It will take care of calling PGMHandlerPhysicalDeregister and clean up
2295	* any ad hoc PGMRAMRANGE left behind.
2296	*
2297	* @returns VBox status code.
2298	* @param pVM Pointer to the VM.
2299	* @param GCPhys The start of the MMIO region.
2300	* @param cb The size of the MMIO region.
2301	*/
2302	VMMR3DECL(int) PGMR3PhysMMIODeregister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb)
2303	{
2304	VM_ASSERT_EMT(pVM);
2305
2306	int rc = pgmLock(pVM);
2307	AssertRCReturn(rc, rc);
2308
2309	/*
2310	* First deregister the handler, then check if we should remove the ram range.
2311	*/
2312	rc = PGMHandlerPhysicalDeregister(pVM, GCPhys);
2313	if (RT_SUCCESS(rc))
2314	{
2315	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
2316	PPGMRAMRANGE pRamPrev = NULL;
2317	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2318	while (pRam && GCPhysLast >= pRam->GCPhys)
2319	{
2320	/** @todo We're being a bit too careful here. rewrite. */
2321	if ( GCPhysLast == pRam->GCPhysLast
2322	&& GCPhys == pRam->GCPhys)
2323	{
2324	Assert(pRam->cb == cb);
2325
2326	/*
2327	* See if all the pages are dead MMIO pages.
2328	*/
2329	uint32_t const cPages = cb >> PAGE_SHIFT;
2330	bool fAllMMIO = true;
2331	uint32_t iPage = 0;
2332	uint32_t cLeft = cPages;
2333	while (cLeft-- > 0)
2334	{
2335	PPGMPAGE pPage = &pRam->aPages[iPage];
2336	if ( PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO
2337	/\|\| not-out-of-action later /)
2338	{
2339	fAllMMIO = false;
2340	Assert(PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO2_ALIAS_MMIO);
2341	AssertMsgFailed(("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
2342	break;
2343	}
2344	Assert(PGM_PAGE_IS_ZERO(pPage));
2345	pPage++;
2346	}
2347	if (fAllMMIO)
2348	{
2349	/*
2350	* Ad-hoc range, unlink and free it.
2351	*/
2352	Log(("PGMR3PhysMMIODeregister: Freeing ad hoc MMIO range for %RGp-%RGp %s\n",
2353	GCPhys, GCPhysLast, pRam->pszDesc));
2354
2355	pVM->pgm.s.cAllPages -= cPages;
2356	pVM->pgm.s.cPureMmioPages -= cPages;
2357
2358	pgmR3PhysUnlinkRamRange2(pVM, pRam, pRamPrev);
2359	pRam->cb = pRam->GCPhys = pRam->GCPhysLast = NIL_RTGCPHYS;
2360	MMHyperFree(pVM, pRam);
2361	break;
2362	}
2363	}
2364
2365	/*
2366	* Range match? It will all be within one range (see PGMAllHandler.cpp).
2367	*/
2368	if ( GCPhysLast >= pRam->GCPhys
2369	&& GCPhys <= pRam->GCPhysLast)
2370	{
2371	Assert(GCPhys >= pRam->GCPhys);
2372	Assert(GCPhysLast <= pRam->GCPhysLast);
2373
2374	/*
2375	* Turn the pages back into RAM pages.
2376	*/
2377	uint32_t iPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2378	uint32_t cLeft = cb >> PAGE_SHIFT;
2379	while (cLeft--)
2380	{
2381	PPGMPAGE pPage = &pRam->aPages[iPage];
2382	AssertMsg(PGM_PAGE_IS_MMIO(pPage), ("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
2383	AssertMsg(PGM_PAGE_IS_ZERO(pPage), ("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
2384	if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO)
2385	PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_RAM);
2386	}
2387	break;
2388	}
2389
2390	/* next */
2391	pRamPrev = pRam;
2392	pRam = pRam->pNextR3;
2393	}
2394	}
2395
2396	/* Force a PGM pool flush as guest ram references have been changed. */
2397	/** @todo Not entirely SMP safe; assuming for now the guest takes care of
2398	* this internally (not touch mapped mmio while changing the mapping). */
2399	PVMCPU pVCpu = VMMGetCpu(pVM);
2400	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2401	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2402
2403	pgmPhysInvalidatePageMapTLB(pVM);
2404	pgmPhysInvalidRamRangeTlbs(pVM);
2405	pgmUnlock(pVM);
2406	return rc;
2407	}
2408
2409
2410	/**
2411	* Locate a MMIO2 range.
2412	*
2413	* @returns Pointer to the MMIO2 range.
2414	* @param pVM Pointer to the VM.
2415	* @param pDevIns The device instance owning the region.
2416	* @param iRegion The region.
2417	*/
2418	DECLINLINE(PPGMMMIO2RANGE) pgmR3PhysMMIO2Find(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion)
2419	{
2420	/*
2421	* Search the list.
2422	*/
2423	for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
2424	if ( pCur->pDevInsR3 == pDevIns
2425	&& pCur->iRegion == iRegion)
2426	return pCur;
2427	return NULL;
2428	}
2429
2430
2431	/**
2432	* Allocate and register an MMIO2 region.
2433	*
2434	* As mentioned elsewhere, MMIO2 is just RAM spelled differently. It's RAM
2435	* associated with a device. It is also non-shared memory with a permanent
2436	* ring-3 mapping and page backing (presently).
2437	*
2438	* A MMIO2 range may overlap with base memory if a lot of RAM is configured for
2439	* the VM, in which case we'll drop the base memory pages. Presently we will
2440	* make no attempt to preserve anything that happens to be present in the base
2441	* memory that is replaced, this is of course incorrectly but it's too much
2442	* effort.
2443	*
2444	* @returns VBox status code.
2445	* @retval VINF_SUCCESS on success, *ppv pointing to the R3 mapping of the
2446	* memory.
2447	* @retval VERR_ALREADY_EXISTS if the region already exists.
2448	*
2449	* @param pVM Pointer to the VM.
2450	* @param pDevIns The device instance owning the region.
2451	* @param iRegion The region number. If the MMIO2 memory is a PCI
2452	* I/O region this number has to be the number of that
2453	* region. Otherwise it can be any number safe
2454	* UINT8_MAX.
2455	* @param cb The size of the region. Must be page aligned.
2456	* @param fFlags Reserved for future use, must be zero.
2457	* @param ppv Where to store the pointer to the ring-3 mapping of
2458	* the memory.
2459	* @param pszDesc The description.
2460	*/
2461	VMMR3DECL(int) PGMR3PhysMMIO2Register(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS cb, uint32_t fFlags, void *ppv, const char pszDesc)
2462	{
2463	/*
2464	* Validate input.
2465	*/
2466	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2467	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2468	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2469	AssertPtrReturn(ppv, VERR_INVALID_POINTER);
2470	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
2471	AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
2472	AssertReturn(pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion) == NULL, VERR_ALREADY_EXISTS);
2473	AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2474	AssertReturn(cb, VERR_INVALID_PARAMETER);
2475	AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
2476
2477	const uint32_t cPages = cb >> PAGE_SHIFT;
2478	AssertLogRelReturn(((RTGCPHYS)cPages << PAGE_SHIFT) == cb, VERR_INVALID_PARAMETER);
2479	AssertLogRelReturn(cPages <= INT32_MAX / 2, VERR_NO_MEMORY);
2480
2481	/*
2482	* For the 2nd+ instance, mangle the description string so it's unique.
2483	*/
2484	if (pDevIns->iInstance > 0) /** @todo Move to PDMDevHlp.cpp and use a real string cache. */
2485	{
2486	pszDesc = MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s [%u]", pszDesc, pDevIns->iInstance);
2487	if (!pszDesc)
2488	return VERR_NO_MEMORY;
2489	}
2490
2491	/*
2492	* Try reserve and allocate the backing memory first as this is what is
2493	* most likely to fail.
2494	*/
2495	int rc = MMR3AdjustFixedReservation(pVM, cPages, pszDesc);
2496	if (RT_SUCCESS(rc))
2497	{
2498	void *pvPages;
2499	PSUPPAGE paPages = (PSUPPAGE)RTMemTmpAlloc(cPages * sizeof(SUPPAGE));
2500	if (RT_SUCCESS(rc))
2501	rc = SUPR3PageAllocEx(cPages, 0 /fFlags/, &pvPages, NULL /pR0Ptr/, paPages);
2502	if (RT_SUCCESS(rc))
2503	{
2504	memset(pvPages, 0, cPages * PAGE_SIZE);
2505
2506	/*
2507	* Create the MMIO2 range record for it.
2508	*/
2509	const size_t cbRange = RT_OFFSETOF(PGMMMIO2RANGE, RamRange.aPages[cPages]);
2510	PPGMMMIO2RANGE pNew;
2511	rc = MMR3HyperAllocOnceNoRel(pVM, cbRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
2512	AssertLogRelMsgRC(rc, ("cbRamRange=%zu\n", cbRange));
2513	if (RT_SUCCESS(rc))
2514	{
2515	pNew->pDevInsR3 = pDevIns;
2516	pNew->pvR3 = pvPages;
2517	//pNew->pNext = NULL;
2518	//pNew->fMapped = false;
2519	//pNew->fOverlapping = false;
2520	pNew->iRegion = iRegion;
2521	pNew->idSavedState = UINT8_MAX;
2522	pNew->RamRange.pSelfR0 = MMHyperCCToR0(pVM, &pNew->RamRange);
2523	pNew->RamRange.pSelfRC = MMHyperCCToRC(pVM, &pNew->RamRange);
2524	pNew->RamRange.GCPhys = NIL_RTGCPHYS;
2525	pNew->RamRange.GCPhysLast = NIL_RTGCPHYS;
2526	pNew->RamRange.pszDesc = pszDesc;
2527	pNew->RamRange.cb = cb;
2528	pNew->RamRange.fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO2;
2529	pNew->RamRange.pvR3 = pvPages;
2530	//pNew->RamRange.paLSPages = NULL;
2531
2532	uint32_t iPage = cPages;
2533	while (iPage-- > 0)
2534	{
2535	PGM_PAGE_INIT(&pNew->RamRange.aPages[iPage],
2536	paPages[iPage].Phys, NIL_GMM_PAGEID,
2537	PGMPAGETYPE_MMIO2, PGM_PAGE_STATE_ALLOCATED);
2538	}
2539
2540	/* update page count stats */
2541	pVM->pgm.s.cAllPages += cPages;
2542	pVM->pgm.s.cPrivatePages += cPages;
2543
2544	/*
2545	* Link it into the list.
2546	* Since there is no particular order, just push it.
2547	*/
2548	pgmLock(pVM);
2549	pNew->pNextR3 = pVM->pgm.s.pMmio2RangesR3;
2550	pVM->pgm.s.pMmio2RangesR3 = pNew;
2551	pgmUnlock(pVM);
2552
2553	*ppv = pvPages;
2554	RTMemTmpFree(paPages);
2555	pgmPhysInvalidatePageMapTLB(pVM);
2556	return VINF_SUCCESS;
2557	}
2558
2559	SUPR3PageFreeEx(pvPages, cPages);
2560	}
2561	RTMemTmpFree(paPages);
2562	MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pszDesc);
2563	}
2564	if (pDevIns->iInstance > 0)
2565	MMR3HeapFree((void *)pszDesc);
2566	return rc;
2567	}
2568
2569
2570	/**
2571	* Deregisters and frees an MMIO2 region.
2572	*
2573	* Any physical (and virtual) access handlers registered for the region must
2574	* be deregistered before calling this function.
2575	*
2576	* @returns VBox status code.
2577	* @param pVM Pointer to the VM.
2578	* @param pDevIns The device instance owning the region.
2579	* @param iRegion The region. If it's UINT32_MAX it'll be a wildcard match.
2580	*/
2581	VMMR3DECL(int) PGMR3PhysMMIO2Deregister(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion)
2582	{
2583	/*
2584	* Validate input.
2585	*/
2586	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2587	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2588	AssertReturn(iRegion <= UINT8_MAX \|\| iRegion == UINT32_MAX, VERR_INVALID_PARAMETER);
2589
2590	pgmLock(pVM);
2591	int rc = VINF_SUCCESS;
2592	unsigned cFound = 0;
2593	PPGMMMIO2RANGE pPrev = NULL;
2594	PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3;
2595	while (pCur)
2596	{
2597	if ( pCur->pDevInsR3 == pDevIns
2598	&& ( iRegion == UINT32_MAX
2599	\|\| pCur->iRegion == iRegion))
2600	{
2601	cFound++;
2602
2603	/*
2604	* Unmap it if it's mapped.
2605	*/
2606	if (pCur->fMapped)
2607	{
2608	int rc2 = PGMR3PhysMMIO2Unmap(pVM, pCur->pDevInsR3, pCur->iRegion, pCur->RamRange.GCPhys);
2609	AssertRC(rc2);
2610	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2611	rc = rc2;
2612	}
2613
2614	/*
2615	* Unlink it
2616	*/
2617	PPGMMMIO2RANGE pNext = pCur->pNextR3;
2618	if (pPrev)
2619	pPrev->pNextR3 = pNext;
2620	else
2621	pVM->pgm.s.pMmio2RangesR3 = pNext;
2622	pCur->pNextR3 = NULL;
2623
2624	/*
2625	* Free the memory.
2626	*/
2627	int rc2 = SUPR3PageFreeEx(pCur->pvR3, pCur->RamRange.cb >> PAGE_SHIFT);
2628	AssertRC(rc2);
2629	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2630	rc = rc2;
2631
2632	uint32_t const cPages = pCur->RamRange.cb >> PAGE_SHIFT;
2633	rc2 = MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pCur->RamRange.pszDesc);
2634	AssertRC(rc2);
2635	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2636	rc = rc2;
2637
2638	/* we're leaking hyper memory here if done at runtime. */
2639	#ifdef VBOX_STRICT
2640	VMSTATE const enmState = VMR3GetState(pVM);
2641	AssertMsg( enmState == VMSTATE_POWERING_OFF
2642	\|\| enmState == VMSTATE_POWERING_OFF_LS
2643	\|\| enmState == VMSTATE_OFF
2644	\|\| enmState == VMSTATE_OFF_LS
2645	\|\| enmState == VMSTATE_DESTROYING
2646	\|\| enmState == VMSTATE_TERMINATED
2647	\|\| enmState == VMSTATE_CREATING
2648	, ("%s\n", VMR3GetStateName(enmState)));
2649	#endif
2650	/*rc = MMHyperFree(pVM, pCur);
2651	AssertRCReturn(rc, rc); - not safe, see the alloc call. */
2652
2653
2654	/* update page count stats */
2655	pVM->pgm.s.cAllPages -= cPages;
2656	pVM->pgm.s.cPrivatePages -= cPages;
2657
2658	/* next */
2659	pCur = pNext;
2660	}
2661	else
2662	{
2663	pPrev = pCur;
2664	pCur = pCur->pNextR3;
2665	}
2666	}
2667	pgmPhysInvalidatePageMapTLB(pVM);
2668	pgmUnlock(pVM);
2669	return !cFound && iRegion != UINT32_MAX ? VERR_NOT_FOUND : rc;
2670	}
2671
2672
2673	/**
2674	* Maps a MMIO2 region.
2675	*
2676	* This is done when a guest / the bios / state loading changes the
2677	* PCI config. The replacing of base memory has the same restrictions
2678	* as during registration, of course.
2679	*
2680	* @returns VBox status code.
2681	*
2682	* @param pVM Pointer to the VM.
2683	* @param pDevIns The device instance owning the region.
2684	*/
2685	VMMR3DECL(int) PGMR3PhysMMIO2Map(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
2686	{
2687	/*
2688	* Validate input
2689	*/
2690	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2691	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2692	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2693	AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
2694	AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
2695	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2696
2697	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2698	AssertReturn(pCur, VERR_NOT_FOUND);
2699	AssertReturn(!pCur->fMapped, VERR_WRONG_ORDER);
2700	Assert(pCur->RamRange.GCPhys == NIL_RTGCPHYS);
2701	Assert(pCur->RamRange.GCPhysLast == NIL_RTGCPHYS);
2702
2703	const RTGCPHYS GCPhysLast = GCPhys + pCur->RamRange.cb - 1;
2704	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
2705
2706	/*
2707	* Find our location in the ram range list, checking for
2708	* restriction we don't bother implementing yet (partially overlapping).
2709	*/
2710	bool fRamExists = false;
2711	PPGMRAMRANGE pRamPrev = NULL;
2712	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2713	while (pRam && GCPhysLast >= pRam->GCPhys)
2714	{
2715	if ( GCPhys <= pRam->GCPhysLast
2716	&& GCPhysLast >= pRam->GCPhys)
2717	{
2718	/* completely within? */
2719	AssertLogRelMsgReturn( GCPhys >= pRam->GCPhys
2720	&& GCPhysLast <= pRam->GCPhysLast,
2721	("%RGp-%RGp (MMIO2/%s) falls partly outside %RGp-%RGp (%s)\n",
2722	GCPhys, GCPhysLast, pCur->RamRange.pszDesc,
2723	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
2724	VERR_PGM_RAM_CONFLICT);
2725	fRamExists = true;
2726	break;
2727	}
2728
2729	/* next */
2730	pRamPrev = pRam;
2731	pRam = pRam->pNextR3;
2732	}
2733	if (fRamExists)
2734	{
2735	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2736	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2737	while (cPagesLeft-- > 0)
2738	{
2739	AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
2740	("%RGp isn't a RAM page (%d) - mapping %RGp-%RGp (MMIO2/%s).\n",
2741	GCPhys, PGM_PAGE_GET_TYPE(pPage), GCPhys, GCPhysLast, pCur->RamRange.pszDesc),
2742	VERR_PGM_RAM_CONFLICT);
2743	pPage++;
2744	}
2745	}
2746	Log(("PGMR3PhysMMIO2Map: %RGp-%RGp fRamExists=%RTbool %s\n",
2747	GCPhys, GCPhysLast, fRamExists, pCur->RamRange.pszDesc));
2748
2749	/*
2750	* Make the changes.
2751	*/
2752	pgmLock(pVM);
2753
2754	pCur->RamRange.GCPhys = GCPhys;
2755	pCur->RamRange.GCPhysLast = GCPhysLast;
2756	pCur->fMapped = true;
2757	pCur->fOverlapping = fRamExists;
2758
2759	if (fRamExists)
2760	{
2761	/** @todo use pgmR3PhysFreePageRange here. */
2762	uint32_t cPendingPages = 0;
2763	PGMMFREEPAGESREQ pReq;
2764	int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
2765	AssertLogRelRCReturn(rc, rc);
2766
2767	/* replace the pages, freeing all present RAM pages. */
2768	PPGMPAGE pPageSrc = &pCur->RamRange.aPages[0];
2769	PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2770	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2771	while (cPagesLeft-- > 0)
2772	{
2773	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys);
2774	AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
2775
2776	RTHCPHYS const HCPhys = PGM_PAGE_GET_HCPHYS(pPageSrc);
2777	PGM_PAGE_SET_HCPHYS(pVM, pPageDst, HCPhys);
2778	PGM_PAGE_SET_TYPE(pVM, pPageDst, PGMPAGETYPE_MMIO2);
2779	PGM_PAGE_SET_STATE(pVM, pPageDst, PGM_PAGE_STATE_ALLOCATED);
2780	PGM_PAGE_SET_PDE_TYPE(pVM, pPageDst, PGM_PAGE_PDE_TYPE_DONTCARE);
2781	PGM_PAGE_SET_PTE_INDEX(pVM, pPageDst, 0);
2782	PGM_PAGE_SET_TRACKING(pVM, pPageDst, 0);
2783
2784	pVM->pgm.s.cZeroPages--;
2785	GCPhys += PAGE_SIZE;
2786	pPageSrc++;
2787	pPageDst++;
2788	}
2789
2790	/* Flush physical page map TLB. */
2791	pgmPhysInvalidatePageMapTLB(pVM);
2792
2793	if (cPendingPages)
2794	{
2795	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2796	AssertLogRelRCReturn(rc, rc);
2797	}
2798	GMMR3FreePagesCleanup(pReq);
2799
2800	/* Force a PGM pool flush as guest ram references have been changed. */
2801	/** @todo not entirely SMP safe; assuming for now the guest takes care of
2802	* this internally (not touch mapped mmio while changing the mapping). */
2803	PVMCPU pVCpu = VMMGetCpu(pVM);
2804	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2805	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2806
2807	pgmUnlock(pVM);
2808	}
2809	else
2810	{
2811	RTGCPHYS cb = pCur->RamRange.cb;
2812
2813	/* Clear the tracking data of pages we're going to reactivate. */
2814	PPGMPAGE pPageSrc = &pCur->RamRange.aPages[0];
2815	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2816	while (cPagesLeft-- > 0)
2817	{
2818	PGM_PAGE_SET_TRACKING(pVM, pPageSrc, 0);
2819	PGM_PAGE_SET_PTE_INDEX(pVM, pPageSrc, 0);
2820	pPageSrc++;
2821	}
2822
2823	/* link in the ram range */
2824	pgmR3PhysLinkRamRange(pVM, &pCur->RamRange, pRamPrev);
2825	pgmUnlock(pVM);
2826
2827	#ifdef VBOX_WITH_REM
2828	REMR3NotifyPhysRamRegister(pVM, GCPhys, cb, REM_NOTIFY_PHYS_RAM_FLAGS_MMIO2);
2829	#endif
2830	}
2831
2832	pgmPhysInvalidatePageMapTLB(pVM);
2833	return VINF_SUCCESS;
2834	}
2835
2836
2837	/**
2838	* Unmaps a MMIO2 region.
2839	*
2840	* This is done when a guest / the bios / state loading changes the
2841	* PCI config. The replacing of base memory has the same restrictions
2842	* as during registration, of course.
2843	*/
2844	VMMR3DECL(int) PGMR3PhysMMIO2Unmap(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
2845	{
2846	/*
2847	* Validate input
2848	*/
2849	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2850	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2851	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2852	AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
2853	AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
2854	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2855
2856	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2857	AssertReturn(pCur, VERR_NOT_FOUND);
2858	AssertReturn(pCur->fMapped, VERR_WRONG_ORDER);
2859	AssertReturn(pCur->RamRange.GCPhys == GCPhys, VERR_INVALID_PARAMETER);
2860	Assert(pCur->RamRange.GCPhysLast != NIL_RTGCPHYS);
2861
2862	Log(("PGMR3PhysMMIO2Unmap: %RGp-%RGp %s\n",
2863	pCur->RamRange.GCPhys, pCur->RamRange.GCPhysLast, pCur->RamRange.pszDesc));
2864
2865	/*
2866	* Unmap it.
2867	*/
2868	pgmLock(pVM);
2869
2870	#ifdef VBOX_WITH_REM
2871	RTGCPHYS GCPhysRangeREM;
2872	RTGCPHYS cbRangeREM;
2873	bool fInformREM;
2874	#endif
2875	if (pCur->fOverlapping)
2876	{
2877	/* Restore the RAM pages we've replaced. */
2878	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2879	while (pRam->GCPhys > pCur->RamRange.GCPhysLast)
2880	pRam = pRam->pNextR3;
2881
2882	PPGMPAGE pPageDst = &pRam->aPages[(pCur->RamRange.GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2883	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2884	while (cPagesLeft-- > 0)
2885	{
2886	PGM_PAGE_INIT_ZERO(pPageDst, pVM, PGMPAGETYPE_RAM);
2887	pVM->pgm.s.cZeroPages++;
2888	pPageDst++;
2889	}
2890
2891	/* Flush physical page map TLB. */
2892	pgmPhysInvalidatePageMapTLB(pVM);
2893	#ifdef VBOX_WITH_REM
2894	GCPhysRangeREM = NIL_RTGCPHYS; /* shuts up gcc */
2895	cbRangeREM = RTGCPHYS_MAX; /* ditto */
2896	fInformREM = false;
2897	#endif
2898	}
2899	else
2900	{
2901	#ifdef VBOX_WITH_REM
2902	GCPhysRangeREM = pCur->RamRange.GCPhys;
2903	cbRangeREM = pCur->RamRange.cb;
2904	fInformREM = true;
2905	#endif
2906	pgmR3PhysUnlinkRamRange(pVM, &pCur->RamRange);
2907	}
2908
2909	pCur->RamRange.GCPhys = NIL_RTGCPHYS;
2910	pCur->RamRange.GCPhysLast = NIL_RTGCPHYS;
2911	pCur->fOverlapping = false;
2912	pCur->fMapped = false;
2913
2914	/* Force a PGM pool flush as guest ram references have been changed. */
2915	/** @todo not entirely SMP safe; assuming for now the guest takes care
2916	* of this internally (not touch mapped mmio while changing the
2917	* mapping). */
2918	PVMCPU pVCpu = VMMGetCpu(pVM);
2919	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2920	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2921
2922	pgmPhysInvalidatePageMapTLB(pVM);
2923	pgmPhysInvalidRamRangeTlbs(pVM);
2924	pgmUnlock(pVM);
2925
2926	#ifdef VBOX_WITH_REM
2927	if (fInformREM)
2928	REMR3NotifyPhysRamDeregister(pVM, GCPhysRangeREM, cbRangeREM);
2929	#endif
2930
2931	return VINF_SUCCESS;
2932	}
2933
2934
2935	/**
2936	* Checks if the given address is an MMIO2 base address or not.
2937	*
2938	* @returns true/false accordingly.
2939	* @param pVM Pointer to the VM.
2940	* @param pDevIns The owner of the memory, optional.
2941	* @param GCPhys The address to check.
2942	*/
2943	VMMR3DECL(bool) PGMR3PhysMMIO2IsBase(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys)
2944	{
2945	/*
2946	* Validate input
2947	*/
2948	VM_ASSERT_EMT_RETURN(pVM, false);
2949	AssertPtrReturn(pDevIns, false);
2950	AssertReturn(GCPhys != NIL_RTGCPHYS, false);
2951	AssertReturn(GCPhys != 0, false);
2952	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), false);
2953
2954	/*
2955	* Search the list.
2956	*/
2957	pgmLock(pVM);
2958	for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
2959	if (pCur->RamRange.GCPhys == GCPhys)
2960	{
2961	Assert(pCur->fMapped);
2962	pgmUnlock(pVM);
2963	return true;
2964	}
2965	pgmUnlock(pVM);
2966	return false;
2967	}
2968
2969
2970	/**
2971	* Gets the HC physical address of a page in the MMIO2 region.
2972	*
2973	* This is API is intended for MMHyper and shouldn't be called
2974	* by anyone else...
2975	*
2976	* @returns VBox status code.
2977	* @param pVM Pointer to the VM.
2978	* @param pDevIns The owner of the memory, optional.
2979	* @param iRegion The region.
2980	* @param off The page expressed an offset into the MMIO2 region.
2981	* @param pHCPhys Where to store the result.
2982	*/
2983	VMMR3DECL(int) PGMR3PhysMMIO2GetHCPhys(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, PRTHCPHYS pHCPhys)
2984	{
2985	/*
2986	* Validate input
2987	*/
2988	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2989	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2990	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2991
2992	pgmLock(pVM);
2993	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2994	AssertReturn(pCur, VERR_NOT_FOUND);
2995	AssertReturn(off < pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2996
2997	PCPGMPAGE pPage = &pCur->RamRange.aPages[off >> PAGE_SHIFT];
2998	*pHCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2999	pgmUnlock(pVM);
3000	return VINF_SUCCESS;
3001	}
3002
3003
3004	/**
3005	* Maps a portion of an MMIO2 region into kernel space (host).
3006	*
3007	* The kernel mapping will become invalid when the MMIO2 memory is deregistered
3008	* or the VM is terminated.
3009	*
3010	* @return VBox status code.
3011	*
3012	* @param pVM Pointer to the VM.
3013	* @param pDevIns The device owning the MMIO2 memory.
3014	* @param iRegion The region.
3015	* @param off The offset into the region. Must be page aligned.
3016	* @param cb The number of bytes to map. Must be page aligned.
3017	* @param pszDesc Mapping description.
3018	* @param pR0Ptr Where to store the R0 address.
3019	*/
3020	VMMR3DECL(int) PGMR3PhysMMIO2MapKernel(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, RTGCPHYS cb,
3021	const char *pszDesc, PRTR0PTR pR0Ptr)
3022	{
3023	/*
3024	* Validate input.
3025	*/
3026	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
3027	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3028	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
3029
3030	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
3031	AssertReturn(pCur, VERR_NOT_FOUND);
3032	AssertReturn(off < pCur->RamRange.cb, VERR_INVALID_PARAMETER);
3033	AssertReturn(cb <= pCur->RamRange.cb, VERR_INVALID_PARAMETER);
3034	AssertReturn(off + cb <= pCur->RamRange.cb, VERR_INVALID_PARAMETER);
3035	NOREF(pszDesc);
3036
3037	/*
3038	* Pass the request on to the support library/driver.
3039	*/
3040	int rc = SUPR3PageMapKernel(pCur->pvR3, off, cb, 0, pR0Ptr);
3041
3042	return rc;
3043	}
3044
3045
3046	/**
3047	* Worker for PGMR3PhysRomRegister.
3048	*
3049	* This is here to simplify lock management, i.e. the caller does all the
3050	* locking and we can simply return without needing to remember to unlock
3051	* anything first.
3052	*
3053	* @returns VBox status.
3054	* @param pVM Pointer to the VM.
3055	* @param pDevIns The device instance owning the ROM.
3056	* @param GCPhys First physical address in the range.
3057	* Must be page aligned!
3058	* @param cb The size of the range (in bytes).
3059	* Must be page aligned!
3060	* @param pvBinary Pointer to the binary data backing the ROM image.
3061	* @param cbBinary The size of the binary data pvBinary points to.
3062	* This must be less or equal to @a cb.
3063	* @param fFlags Mask of flags. PGMPHYS_ROM_FLAGS_SHADOWED
3064	* and/or PGMPHYS_ROM_FLAGS_PERMANENT_BINARY.
3065	* @param pszDesc Pointer to description string. This must not be freed.
3066	*/
3067	static int pgmR3PhysRomRegister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
3068	const void pvBinary, uint32_t cbBinary, uint32_t fFlags, const char pszDesc)
3069	{
3070	/*
3071	* Validate input.
3072	*/
3073	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3074	AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
3075	AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
3076	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
3077	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
3078	AssertPtrReturn(pvBinary, VERR_INVALID_PARAMETER);
3079	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
3080	AssertReturn(!(fFlags & ~(PGMPHYS_ROM_FLAGS_SHADOWED \| PGMPHYS_ROM_FLAGS_PERMANENT_BINARY)), VERR_INVALID_PARAMETER);
3081	VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
3082
3083	const uint32_t cPages = cb >> PAGE_SHIFT;
3084
3085	/*
3086	* Find the ROM location in the ROM list first.
3087	*/
3088	PPGMROMRANGE pRomPrev = NULL;
3089	PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3;
3090	while (pRom && GCPhysLast >= pRom->GCPhys)
3091	{
3092	if ( GCPhys <= pRom->GCPhysLast
3093	&& GCPhysLast >= pRom->GCPhys)
3094	AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
3095	GCPhys, GCPhysLast, pszDesc,
3096	pRom->GCPhys, pRom->GCPhysLast, pRom->pszDesc),
3097	VERR_PGM_RAM_CONFLICT);
3098	/* next */
3099	pRomPrev = pRom;
3100	pRom = pRom->pNextR3;
3101	}
3102
3103	/*
3104	* Find the RAM location and check for conflicts.
3105	*
3106	* Conflict detection is a bit different than for RAM
3107	* registration since a ROM can be located within a RAM
3108	* range. So, what we have to check for is other memory
3109	* types (other than RAM that is) and that we don't span
3110	* more than one RAM range (layz).
3111	*/
3112	bool fRamExists = false;
3113	PPGMRAMRANGE pRamPrev = NULL;
3114	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
3115	while (pRam && GCPhysLast >= pRam->GCPhys)
3116	{
3117	if ( GCPhys <= pRam->GCPhysLast
3118	&& GCPhysLast >= pRam->GCPhys)
3119	{
3120	/* completely within? */
3121	AssertLogRelMsgReturn( GCPhys >= pRam->GCPhys
3122	&& GCPhysLast <= pRam->GCPhysLast,
3123	("%RGp-%RGp (%s) falls partly outside %RGp-%RGp (%s)\n",
3124	GCPhys, GCPhysLast, pszDesc,
3125	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
3126	VERR_PGM_RAM_CONFLICT);
3127	fRamExists = true;
3128	break;
3129	}
3130
3131	/* next */
3132	pRamPrev = pRam;
3133	pRam = pRam->pNextR3;
3134	}
3135	if (fRamExists)
3136	{
3137	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
3138	uint32_t cPagesLeft = cPages;
3139	while (cPagesLeft-- > 0)
3140	{
3141	AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
3142	("%RGp (%R[pgmpage]) isn't a RAM page - registering %RGp-%RGp (%s).\n",
3143	pRam->GCPhys + ((RTGCPHYS)(uintptr_t)(pPage - &pRam->aPages[0]) << PAGE_SHIFT),
3144	pPage, GCPhys, GCPhysLast, pszDesc), VERR_PGM_RAM_CONFLICT);
3145	Assert(PGM_PAGE_IS_ZERO(pPage));
3146	pPage++;
3147	}
3148	}
3149
3150	/*
3151	* Update the base memory reservation if necessary.
3152	*/
3153	uint32_t cExtraBaseCost = fRamExists ? 0 : cPages;
3154	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3155	cExtraBaseCost += cPages;
3156	if (cExtraBaseCost)
3157	{
3158	int rc = MMR3IncreaseBaseReservation(pVM, cExtraBaseCost);
3159	if (RT_FAILURE(rc))
3160	return rc;
3161	}
3162
3163	/*
3164	* Allocate memory for the virgin copy of the RAM.
3165	*/
3166	PGMMALLOCATEPAGESREQ pReq;
3167	int rc = GMMR3AllocatePagesPrepare(pVM, &pReq, cPages, GMMACCOUNT_BASE);
3168	AssertRCReturn(rc, rc);
3169
3170	for (uint32_t iPage = 0; iPage < cPages; iPage++)
3171	{
3172	pReq->aPages[iPage].HCPhysGCPhys = GCPhys + (iPage << PAGE_SHIFT);
3173	pReq->aPages[iPage].idPage = NIL_GMM_PAGEID;
3174	pReq->aPages[iPage].idSharedPage = NIL_GMM_PAGEID;
3175	}
3176
3177	rc = GMMR3AllocatePagesPerform(pVM, pReq);
3178	if (RT_FAILURE(rc))
3179	{
3180	GMMR3AllocatePagesCleanup(pReq);
3181	return rc;
3182	}
3183
3184	/*
3185	* Allocate the new ROM range and RAM range (if necessary).
3186	*/
3187	PPGMROMRANGE pRomNew;
3188	rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMROMRANGE, aPages[cPages]), 0, MM_TAG_PGM_PHYS, (void **)&pRomNew);
3189	if (RT_SUCCESS(rc))
3190	{
3191	PPGMRAMRANGE pRamNew = NULL;
3192	if (!fRamExists)
3193	rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]), sizeof(PGMPAGE), MM_TAG_PGM_PHYS, (void **)&pRamNew);
3194	if (RT_SUCCESS(rc))
3195	{
3196	/*
3197	* Initialize and insert the RAM range (if required).
3198	*/
3199	PPGMROMPAGE pRomPage = &pRomNew->aPages[0];
3200	if (!fRamExists)
3201	{
3202	pRamNew->pSelfR0 = MMHyperCCToR0(pVM, pRamNew);
3203	pRamNew->pSelfRC = MMHyperCCToRC(pVM, pRamNew);
3204	pRamNew->GCPhys = GCPhys;
3205	pRamNew->GCPhysLast = GCPhysLast;
3206	pRamNew->cb = cb;
3207	pRamNew->pszDesc = pszDesc;
3208	pRamNew->fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_ROM;
3209	pRamNew->pvR3 = NULL;
3210	pRamNew->paLSPages = NULL;
3211
3212	PPGMPAGE pPage = &pRamNew->aPages[0];
3213	for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
3214	{
3215	PGM_PAGE_INIT(pPage,
3216	pReq->aPages[iPage].HCPhysGCPhys,
3217	pReq->aPages[iPage].idPage,
3218	PGMPAGETYPE_ROM,
3219	PGM_PAGE_STATE_ALLOCATED);
3220
3221	pRomPage->Virgin = *pPage;
3222	}
3223
3224	pVM->pgm.s.cAllPages += cPages;
3225	pgmR3PhysLinkRamRange(pVM, pRamNew, pRamPrev);
3226	}
3227	else
3228	{
3229	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
3230	for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
3231	{
3232	PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_ROM);
3233	PGM_PAGE_SET_HCPHYS(pVM, pPage, pReq->aPages[iPage].HCPhysGCPhys);
3234	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
3235	PGM_PAGE_SET_PAGEID(pVM, pPage, pReq->aPages[iPage].idPage);
3236	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
3237	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
3238	PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
3239
3240	pRomPage->Virgin = *pPage;
3241	}
3242
3243	pRamNew = pRam;
3244
3245	pVM->pgm.s.cZeroPages -= cPages;
3246	}
3247	pVM->pgm.s.cPrivatePages += cPages;
3248
3249	/* Flush physical page map TLB. */
3250	pgmPhysInvalidatePageMapTLB(pVM);
3251
3252
3253	/*
3254	* !HACK ALERT! REM + (Shadowed) ROM ==> mess.
3255	*
3256	* If it's shadowed we'll register the handler after the ROM notification
3257	* so we get the access handler callbacks that we should. If it isn't
3258	* shadowed we'll do it the other way around to make REM use the built-in
3259	* ROM behavior and not the handler behavior (which is to route all access
3260	* to PGM atm).
3261	*/
3262	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3263	{
3264	#ifdef VBOX_WITH_REM
3265	REMR3NotifyPhysRomRegister(pVM, GCPhys, cb, NULL, true /* fShadowed */);
3266	#endif
3267	rc = PGMR3HandlerPhysicalRegister(pVM, PGMPHYSHANDLERTYPE_PHYSICAL_WRITE, GCPhys, GCPhysLast,
3268	pgmR3PhysRomWriteHandler, pRomNew,
3269	NULL, "pgmPhysRomWriteHandler", MMHyperCCToR0(pVM, pRomNew),
3270	NULL, "pgmPhysRomWriteHandler", MMHyperCCToRC(pVM, pRomNew), pszDesc);
3271	}
3272	else
3273	{
3274	rc = PGMR3HandlerPhysicalRegister(pVM, PGMPHYSHANDLERTYPE_PHYSICAL_WRITE, GCPhys, GCPhysLast,
3275	pgmR3PhysRomWriteHandler, pRomNew,
3276	NULL, "pgmPhysRomWriteHandler", MMHyperCCToR0(pVM, pRomNew),
3277	NULL, "pgmPhysRomWriteHandler", MMHyperCCToRC(pVM, pRomNew), pszDesc);
3278	#ifdef VBOX_WITH_REM
3279	REMR3NotifyPhysRomRegister(pVM, GCPhys, cb, NULL, false /* fShadowed */);
3280	#endif
3281	}
3282	if (RT_SUCCESS(rc))
3283	{
3284	/*
3285	* Copy the image over to the virgin pages.
3286	* This must be done after linking in the RAM range.
3287	*/
3288	size_t cbBinaryLeft = cbBinary;
3289	PPGMPAGE pRamPage = &pRamNew->aPages[(GCPhys - pRamNew->GCPhys) >> PAGE_SHIFT];
3290	for (uint32_t iPage = 0; iPage < cPages; iPage++, pRamPage++)
3291	{
3292	void *pvDstPage;
3293	rc = pgmPhysPageMap(pVM, pRamPage, GCPhys + (iPage << PAGE_SHIFT), &pvDstPage);
3294	if (RT_FAILURE(rc))
3295	{
3296	VMSetError(pVM, rc, RT_SRC_POS, "Failed to map virgin ROM page at %RGp", GCPhys);
3297	break;
3298	}
3299	if (cbBinaryLeft >= PAGE_SIZE)
3300	{
3301	memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << PAGE_SHIFT), PAGE_SIZE);
3302	cbBinaryLeft -= PAGE_SIZE;
3303	}
3304	else
3305	{
3306	ASMMemZeroPage(pvDstPage); /* (shouldn't be necessary, but can't hurt either) */
3307	if (cbBinaryLeft > 0)
3308	{
3309	memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << PAGE_SHIFT), cbBinaryLeft);
3310	cbBinaryLeft = 0;
3311	}
3312	}
3313	}
3314	if (RT_SUCCESS(rc))
3315	{
3316	/*
3317	* Initialize the ROM range.
3318	* Note that the Virgin member of the pages has already been initialized above.
3319	*/
3320	pRomNew->GCPhys = GCPhys;
3321	pRomNew->GCPhysLast = GCPhysLast;
3322	pRomNew->cb = cb;
3323	pRomNew->fFlags = fFlags;
3324	pRomNew->idSavedState = UINT8_MAX;
3325	pRomNew->cbOriginal = cbBinary;
3326	#ifdef VBOX_STRICT
3327	pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY
3328	? pvBinary : RTMemDup(pvBinary, cbBinary);
3329	#else
3330	pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY ? pvBinary : NULL;
3331	#endif
3332	pRomNew->pszDesc = pszDesc;
3333
3334	for (unsigned iPage = 0; iPage < cPages; iPage++)
3335	{
3336	PPGMROMPAGE pPage = &pRomNew->aPages[iPage];
3337	pPage->enmProt = PGMROMPROT_READ_ROM_WRITE_IGNORE;
3338	PGM_PAGE_INIT_ZERO(&pPage->Shadow, pVM, PGMPAGETYPE_ROM_SHADOW);
3339	}
3340
3341	/* update the page count stats for the shadow pages. */
3342	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3343	{
3344	pVM->pgm.s.cZeroPages += cPages;
3345	pVM->pgm.s.cAllPages += cPages;
3346	}
3347
3348	/*
3349	* Insert the ROM range, tell REM and return successfully.
3350	*/
3351	pRomNew->pNextR3 = pRom;
3352	pRomNew->pNextR0 = pRom ? MMHyperCCToR0(pVM, pRom) : NIL_RTR0PTR;
3353	pRomNew->pNextRC = pRom ? MMHyperCCToRC(pVM, pRom) : NIL_RTRCPTR;
3354
3355	if (pRomPrev)
3356	{
3357	pRomPrev->pNextR3 = pRomNew;
3358	pRomPrev->pNextR0 = MMHyperCCToR0(pVM, pRomNew);
3359	pRomPrev->pNextRC = MMHyperCCToRC(pVM, pRomNew);
3360	}
3361	else
3362	{
3363	pVM->pgm.s.pRomRangesR3 = pRomNew;
3364	pVM->pgm.s.pRomRangesR0 = MMHyperCCToR0(pVM, pRomNew);
3365	pVM->pgm.s.pRomRangesRC = MMHyperCCToRC(pVM, pRomNew);
3366	}
3367
3368	pgmPhysInvalidatePageMapTLB(pVM);
3369	GMMR3AllocatePagesCleanup(pReq);
3370	return VINF_SUCCESS;
3371	}
3372
3373	/* bail out */
3374
3375	int rc2 = PGMHandlerPhysicalDeregister(pVM, GCPhys);
3376	AssertRC(rc2);
3377	}
3378
3379	if (!fRamExists)
3380	{
3381	pgmR3PhysUnlinkRamRange2(pVM, pRamNew, pRamPrev);
3382	MMHyperFree(pVM, pRamNew);
3383	}
3384	}
3385	MMHyperFree(pVM, pRomNew);
3386	}
3387
3388	/** @todo Purge the mapping cache or something... */
3389	GMMR3FreeAllocatedPages(pVM, pReq);
3390	GMMR3AllocatePagesCleanup(pReq);
3391	return rc;
3392	}
3393
3394
3395	/**
3396	* Registers a ROM image.
3397	*
3398	* Shadowed ROM images requires double the amount of backing memory, so,
3399	* don't use that unless you have to. Shadowing of ROM images is process
3400	* where we can select where the reads go and where the writes go. On real
3401	* hardware the chipset provides means to configure this. We provide
3402	* PGMR3PhysProtectROM() for this purpose.
3403	*
3404	* A read-only copy of the ROM image will always be kept around while we
3405	* will allocate RAM pages for the changes on demand (unless all memory
3406	* is configured to be preallocated).
3407	*
3408	* @returns VBox status.
3409	* @param pVM Pointer to the VM.
3410	* @param pDevIns The device instance owning the ROM.
3411	* @param GCPhys First physical address in the range.
3412	* Must be page aligned!
3413	* @param cb The size of the range (in bytes).
3414	* Must be page aligned!
3415	* @param pvBinary Pointer to the binary data backing the ROM image.
3416	* @param cbBinary The size of the binary data pvBinary points to.
3417	* This must be less or equal to @a cb.
3418	* @param fFlags Mask of flags. PGMPHYS_ROM_FLAGS_SHADOWED
3419	* and/or PGMPHYS_ROM_FLAGS_PERMANENT_BINARY.
3420	* @param pszDesc Pointer to description string. This must not be freed.
3421	*
3422	* @remark There is no way to remove the rom, automatically on device cleanup or
3423	* manually from the device yet. This isn't difficult in any way, it's
3424	* just not something we expect to be necessary for a while.
3425	*/
3426	VMMR3DECL(int) PGMR3PhysRomRegister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
3427	const void pvBinary, uint32_t cbBinary, uint32_t fFlags, const char pszDesc)
3428	{
3429	Log(("PGMR3PhysRomRegister: pDevIns=%p GCPhys=%RGp(-%RGp) cb=%RGp pvBinary=%p cbBinary=%#x fFlags=%#x pszDesc=%s\n",
3430	pDevIns, GCPhys, GCPhys + cb, cb, pvBinary, cbBinary, fFlags, pszDesc));
3431	pgmLock(pVM);
3432	int rc = pgmR3PhysRomRegister(pVM, pDevIns, GCPhys, cb, pvBinary, cbBinary, fFlags, pszDesc);
3433	pgmUnlock(pVM);
3434	return rc;
3435	}
3436
3437
3438	/**
3439	* \#PF Handler callback for ROM write accesses.
3440	*
3441	* @returns VINF_SUCCESS if the handler have carried out the operation.
3442	* @returns VINF_PGM_HANDLER_DO_DEFAULT if the caller should carry out the access operation.
3443	* @param pVM Pointer to the VM.
3444	* @param GCPhys The physical address the guest is writing to.
3445	* @param pvPhys The HC mapping of that address.
3446	* @param pvBuf What the guest is reading/writing.
3447	* @param cbBuf How much it's reading/writing.
3448	* @param enmAccessType The access type.
3449	* @param pvUser User argument.
3450	*/
3451	static DECLCALLBACK(int) pgmR3PhysRomWriteHandler(PVM pVM, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf,
3452	PGMACCESSTYPE enmAccessType, void *pvUser)
3453	{
3454	PPGMROMRANGE pRom = (PPGMROMRANGE)pvUser;
3455	const uint32_t iPage = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
3456	Assert(iPage < (pRom->cb >> PAGE_SHIFT));
3457	PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
3458	Log5(("pgmR3PhysRomWriteHandler: %d %c %#08RGp %#04zx\n", pRomPage->enmProt, enmAccessType == PGMACCESSTYPE_READ ? 'R' : 'W', GCPhys, cbBuf));
3459	NOREF(pvPhys);
3460
3461	if (enmAccessType == PGMACCESSTYPE_READ)
3462	{
3463	switch (pRomPage->enmProt)
3464	{
3465	/*
3466	* Take the default action.
3467	*/
3468	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
3469	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
3470	case PGMROMPROT_READ_ROM_WRITE_RAM:
3471	case PGMROMPROT_READ_RAM_WRITE_RAM:
3472	return VINF_PGM_HANDLER_DO_DEFAULT;
3473
3474	default:
3475	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
3476	pRom->aPages[iPage].enmProt, iPage, GCPhys),
3477	VERR_IPE_NOT_REACHED_DEFAULT_CASE);
3478	}
3479	}
3480	else
3481	{
3482	Assert(enmAccessType == PGMACCESSTYPE_WRITE);
3483	switch (pRomPage->enmProt)
3484	{
3485	/*
3486	* Ignore writes.
3487	*/
3488	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
3489	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
3490	return VINF_SUCCESS;
3491
3492	/*
3493	* Write to the RAM page.
3494	*/
3495	case PGMROMPROT_READ_ROM_WRITE_RAM:
3496	case PGMROMPROT_READ_RAM_WRITE_RAM: /* yes this will get here too, it's way simpler that way. */
3497	{
3498	/* This should be impossible now, pvPhys doesn't work cross page anylonger. */
3499	Assert(((GCPhys - pRom->GCPhys + cbBuf - 1) >> PAGE_SHIFT) == iPage);
3500
3501	/*
3502	* Take the lock, do lazy allocation, map the page and copy the data.
3503	*
3504	* Note that we have to bypass the mapping TLB since it works on
3505	* guest physical addresses and entering the shadow page would
3506	* kind of screw things up...
3507	*/
3508	int rc = pgmLock(pVM);
3509	AssertRC(rc);
3510
3511	PPGMPAGE pShadowPage = &pRomPage->Shadow;
3512	if (!PGMROMPROT_IS_ROM(pRomPage->enmProt))
3513	{
3514	pShadowPage = pgmPhysGetPage(pVM, GCPhys);
3515	AssertLogRelReturn(pShadowPage, VERR_PGM_PHYS_PAGE_GET_IPE);
3516	}
3517
3518	void *pvDstPage;
3519	rc = pgmPhysPageMakeWritableAndMap(pVM, pShadowPage, GCPhys & X86_PTE_PG_MASK, &pvDstPage);
3520	if (RT_SUCCESS(rc))
3521	{
3522	memcpy((uint8_t *)pvDstPage + (GCPhys & PAGE_OFFSET_MASK), pvBuf, cbBuf);
3523	pRomPage->LiveSave.fWrittenTo = true;
3524	}
3525
3526	pgmUnlock(pVM);
3527	return rc;
3528	}
3529
3530	default:
3531	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
3532	pRom->aPages[iPage].enmProt, iPage, GCPhys),
3533	VERR_IPE_NOT_REACHED_DEFAULT_CASE);
3534	}
3535	}
3536	}
3537
3538
3539	/**
3540	* Called by PGMR3Reset to reset the shadow, switch to the virgin,
3541	* and verify that the virgin part is untouched.
3542	*
3543	* This is done after the normal memory has been cleared.
3544	*
3545	* ASSUMES that the caller owns the PGM lock.
3546	*
3547	* @param pVM Pointer to the VM.
3548	*/
3549	int pgmR3PhysRomReset(PVM pVM)
3550	{
3551	PGM_LOCK_ASSERT_OWNER(pVM);
3552	for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3553	{
3554	const uint32_t cPages = pRom->cb >> PAGE_SHIFT;
3555
3556	if (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3557	{
3558	/*
3559	* Reset the physical handler.
3560	*/
3561	int rc = PGMR3PhysRomProtect(pVM, pRom->GCPhys, pRom->cb, PGMROMPROT_READ_ROM_WRITE_IGNORE);
3562	AssertRCReturn(rc, rc);
3563
3564	/*
3565	* What we do with the shadow pages depends on the memory
3566	* preallocation option. If not enabled, we'll just throw
3567	* out all the dirty pages and replace them by the zero page.
3568	*/
3569	if (!pVM->pgm.s.fRamPreAlloc)
3570	{
3571	/* Free the dirty pages. */
3572	uint32_t cPendingPages = 0;
3573	PGMMFREEPAGESREQ pReq;
3574	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
3575	AssertRCReturn(rc, rc);
3576
3577	for (uint32_t iPage = 0; iPage < cPages; iPage++)
3578	if ( !PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow)
3579	&& !PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow))
3580	{
3581	Assert(PGM_PAGE_GET_STATE(&pRom->aPages[iPage].Shadow) == PGM_PAGE_STATE_ALLOCATED);
3582	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, &pRom->aPages[iPage].Shadow,
3583	pRom->GCPhys + (iPage << PAGE_SHIFT));
3584	AssertLogRelRCReturn(rc, rc);
3585	}
3586
3587	if (cPendingPages)
3588	{
3589	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
3590	AssertLogRelRCReturn(rc, rc);
3591	}
3592	GMMR3FreePagesCleanup(pReq);
3593	}
3594	else
3595	{
3596	/* clear all the shadow pages. */
3597	for (uint32_t iPage = 0; iPage < cPages; iPage++)
3598	{
3599	if (PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow))
3600	continue;
3601	Assert(!PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow));
3602	void *pvDstPage;
3603	const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
3604	rc = pgmPhysPageMakeWritableAndMap(pVM, &pRom->aPages[iPage].Shadow, GCPhys, &pvDstPage);
3605	if (RT_FAILURE(rc))
3606	break;
3607	ASMMemZeroPage(pvDstPage);
3608	}
3609	AssertRCReturn(rc, rc);
3610	}
3611	}
3612
3613	#ifdef VBOX_STRICT
3614	/*
3615	* Verify that the virgin page is unchanged if possible.
3616	*/
3617	if (pRom->pvOriginal)
3618	{
3619	size_t cbSrcLeft = pRom->cbOriginal;
3620	uint8_t const pbSrcPage = (uint8_t const )pRom->pvOriginal;
3621	for (uint32_t iPage = 0; iPage < cPages && cbSrcLeft > 0; iPage++, pbSrcPage += PAGE_SIZE)
3622	{
3623	const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
3624	void const *pvDstPage;
3625	int rc = pgmPhysPageMapReadOnly(pVM, &pRom->aPages[iPage].Virgin, GCPhys, &pvDstPage);
3626	if (RT_FAILURE(rc))
3627	break;
3628
3629	if (memcmp(pvDstPage, pbSrcPage, RT_MIN(cbSrcLeft, PAGE_SIZE)))
3630	LogRel(("pgmR3PhysRomReset: %RGp rom page changed (%s) - loaded saved state?\n",
3631	GCPhys, pRom->pszDesc));
3632	cbSrcLeft -= RT_MIN(cbSrcLeft, PAGE_SIZE);
3633	}
3634	}
3635	#endif
3636	}
3637
3638	return VINF_SUCCESS;
3639	}
3640
3641
3642	/**
3643	* Called by PGMR3Term to free resources.
3644	*
3645	* ASSUMES that the caller owns the PGM lock.
3646	*
3647	* @param pVM Pointer to the VM.
3648	*/
3649	void pgmR3PhysRomTerm(PVM pVM)
3650	{
3651	#ifdef RT_STRICT
3652	/*
3653	* Free the heap copy of the original bits.
3654	*/
3655	for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3656	{
3657	if ( pRom->pvOriginal
3658	&& !(pRom->fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY))
3659	{
3660	RTMemFree((void *)pRom->pvOriginal);
3661	pRom->pvOriginal = NULL;
3662	}
3663	}
3664	#endif
3665	}
3666
3667
3668	/**
3669	* Change the shadowing of a range of ROM pages.
3670	*
3671	* This is intended for implementing chipset specific memory registers
3672	* and will not be very strict about the input. It will silently ignore
3673	* any pages that are not the part of a shadowed ROM.
3674	*
3675	* @returns VBox status code.
3676	* @retval VINF_PGM_SYNC_CR3
3677	*
3678	* @param pVM Pointer to the VM.
3679	* @param GCPhys Where to start. Page aligned.
3680	* @param cb How much to change. Page aligned.
3681	* @param enmProt The new ROM protection.
3682	*/
3683	VMMR3DECL(int) PGMR3PhysRomProtect(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, PGMROMPROT enmProt)
3684	{
3685	/*
3686	* Check input
3687	*/
3688	if (!cb)
3689	return VINF_SUCCESS;
3690	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3691	AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3692	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
3693	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
3694	AssertReturn(enmProt >= PGMROMPROT_INVALID && enmProt <= PGMROMPROT_END, VERR_INVALID_PARAMETER);
3695
3696	/*
3697	* Process the request.
3698	*/
3699	pgmLock(pVM);
3700	int rc = VINF_SUCCESS;
3701	bool fFlushTLB = false;
3702	for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3703	{
3704	if ( GCPhys <= pRom->GCPhysLast
3705	&& GCPhysLast >= pRom->GCPhys
3706	&& (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED))
3707	{
3708	/*
3709	* Iterate the relevant pages and make necessary the changes.
3710	*/
3711	bool fChanges = false;
3712	uint32_t const cPages = pRom->GCPhysLast <= GCPhysLast
3713	? pRom->cb >> PAGE_SHIFT
3714	: (GCPhysLast - pRom->GCPhys + 1) >> PAGE_SHIFT;
3715	for (uint32_t iPage = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
3716	iPage < cPages;
3717	iPage++)
3718	{
3719	PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
3720	if (PGMROMPROT_IS_ROM(pRomPage->enmProt) != PGMROMPROT_IS_ROM(enmProt))
3721	{
3722	fChanges = true;
3723
3724	/* flush references to the page. */
3725	PPGMPAGE pRamPage = pgmPhysGetPage(pVM, pRom->GCPhys + (iPage << PAGE_SHIFT));
3726	int rc2 = pgmPoolTrackUpdateGCPhys(pVM, pRom->GCPhys + (iPage << PAGE_SHIFT), pRamPage,
3727	true /fFlushPTEs/, &fFlushTLB);
3728	if (rc2 != VINF_SUCCESS && (rc == VINF_SUCCESS \|\| RT_FAILURE(rc2)))
3729	rc = rc2;
3730
3731	PPGMPAGE pOld = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Virgin : &pRomPage->Shadow;
3732	PPGMPAGE pNew = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Shadow : &pRomPage->Virgin;
3733
3734	pOld = pRamPage;
3735	pRamPage = pNew;
3736	/** @todo preserve the volatile flags (handlers) when these have been moved out of HCPhys! */
3737	}
3738	pRomPage->enmProt = enmProt;
3739	}
3740
3741	/*
3742	* Reset the access handler if we made changes, no need
3743	* to optimize this.
3744	*/
3745	if (fChanges)
3746	{
3747	int rc2 = PGMHandlerPhysicalReset(pVM, pRom->GCPhys);
3748	if (RT_FAILURE(rc2))
3749	{
3750	pgmUnlock(pVM);
3751	AssertRC(rc);
3752	return rc2;
3753	}
3754	}
3755
3756	/* Advance - cb isn't updated. */
3757	GCPhys = pRom->GCPhys + (cPages << PAGE_SHIFT);
3758	}
3759	}
3760	pgmUnlock(pVM);
3761	if (fFlushTLB)
3762	PGM_INVL_ALL_VCPU_TLBS(pVM);
3763
3764	return rc;
3765	}
3766
3767
3768	/**
3769	* Sets the Address Gate 20 state.
3770	*
3771	* @param pVCpu Pointer to the VMCPU.
3772	* @param fEnable True if the gate should be enabled.
3773	* False if the gate should be disabled.
3774	*/
3775	VMMDECL(void) PGMR3PhysSetA20(PVMCPU pVCpu, bool fEnable)
3776	{
3777	LogFlow(("PGMR3PhysSetA20 %d (was %d)\n", fEnable, pVCpu->pgm.s.fA20Enabled));
3778	if (pVCpu->pgm.s.fA20Enabled != fEnable)
3779	{
3780	pVCpu->pgm.s.fA20Enabled = fEnable;
3781	pVCpu->pgm.s.GCPhysA20Mask = ~(RTGCPHYS)(!fEnable << 20);
3782	#ifdef VBOX_WITH_REM
3783	REMR3A20Set(pVCpu->pVMR3, pVCpu, fEnable);
3784	#endif
3785	#ifdef PGM_WITH_A20
3786	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL;
3787	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3788	pgmR3RefreshShadowModeAfterA20Change(pVCpu);
3789	HWACCMFlushTLB(pVCpu);
3790	#endif
3791	STAM_REL_COUNTER_INC(&pVCpu->pgm.s.cA20Changes);
3792	}
3793	}
3794
3795
3796	/**
3797	* Tree enumeration callback for dealing with age rollover.
3798	* It will perform a simple compression of the current age.
3799	*/
3800	static DECLCALLBACK(int) pgmR3PhysChunkAgeingRolloverCallback(PAVLU32NODECORE pNode, void *pvUser)
3801	{
3802	/* Age compression - ASSUMES iNow == 4. */
3803	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
3804	if (pChunk->iLastUsed >= UINT32_C(0xffffff00))
3805	pChunk->iLastUsed = 3;
3806	else if (pChunk->iLastUsed >= UINT32_C(0xfffff000))
3807	pChunk->iLastUsed = 2;
3808	else if (pChunk->iLastUsed)
3809	pChunk->iLastUsed = 1;
3810	else /* iLastUsed = 0 */
3811	pChunk->iLastUsed = 4;
3812
3813	NOREF(pvUser);
3814	return 0;
3815	}
3816
3817
3818	/**
3819	* The structure passed in the pvUser argument of pgmR3PhysChunkUnmapCandidateCallback().
3820	*/
3821	typedef struct PGMR3PHYSCHUNKUNMAPCB
3822	{
3823	PVM pVM; /*< Pointer to the VM. /
3824	PPGMCHUNKR3MAP pChunk; /*< The chunk to unmap. /
3825	} PGMR3PHYSCHUNKUNMAPCB, *PPGMR3PHYSCHUNKUNMAPCB;
3826
3827
3828	/**
3829	* Callback used to find the mapping that's been unused for
3830	* the longest time.
3831	*/
3832	static DECLCALLBACK(int) pgmR3PhysChunkUnmapCandidateCallback(PAVLU32NODECORE pNode, void *pvUser)
3833	{
3834	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
3835	PPGMR3PHYSCHUNKUNMAPCB pArg = (PPGMR3PHYSCHUNKUNMAPCB)pvUser;
3836
3837	/*
3838	* Check for locks and compare when last used.
3839	*/
3840	if (pChunk->cRefs)
3841	return 0;
3842	if (pChunk->cPermRefs)
3843	return 0;
3844	if ( pArg->pChunk
3845	&& pChunk->iLastUsed >= pArg->pChunk->iLastUsed)
3846	return 0;
3847
3848	/*
3849	* Check that it's not in any of the TLBs.
3850	*/
3851	PVM pVM = pArg->pVM;
3852	if ( pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(pChunk->Core.Key)].idChunk
3853	== pChunk->Core.Key)
3854	{
3855	pChunk = NULL;
3856	return 0;
3857	}
3858	#ifdef VBOX_STRICT
3859	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
3860	{
3861	Assert(pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk != pChunk);
3862	Assert(pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk != pChunk->Core.Key);
3863	}
3864	#endif
3865
3866	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbHC.aEntries); i++)
3867	if (pVM->pgm.s.PhysTlbHC.aEntries[i].pMap == pChunk)
3868	return 0;
3869
3870	pArg->pChunk = pChunk;
3871	return 0;
3872	}
3873
3874
3875	/**
3876	* Finds a good candidate for unmapping when the ring-3 mapping cache is full.
3877	*
3878	* The candidate will not be part of any TLBs, so no need to flush
3879	* anything afterwards.
3880	*
3881	* @returns Chunk id.
3882	* @param pVM Pointer to the VM.
3883	*/
3884	static int32_t pgmR3PhysChunkFindUnmapCandidate(PVM pVM)
3885	{
3886	PGM_LOCK_ASSERT_OWNER(pVM);
3887
3888	/*
3889	* Enumerate the age tree starting with the left most node.
3890	*/
3891	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3892	PGMR3PHYSCHUNKUNMAPCB Args;
3893	Args.pVM = pVM;
3894	Args.pChunk = NULL;
3895	RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /fFromLeft/, pgmR3PhysChunkUnmapCandidateCallback, &Args);
3896	Assert(Args.pChunk);
3897	if (Args.pChunk)
3898	{
3899	Assert(Args.pChunk->cRefs == 0);
3900	Assert(Args.pChunk->cPermRefs == 0);
3901	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3902	return Args.pChunk->Core.Key;
3903	}
3904
3905	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3906	return INT32_MAX;
3907	}
3908
3909
3910	/**
3911	* Rendezvous callback used by pgmR3PhysUnmapChunk that unmaps a chunk
3912	*
3913	* This is only called on one of the EMTs while the other ones are waiting for
3914	* it to complete this function.
3915	*
3916	* @returns VINF_SUCCESS (VBox strict status code).
3917	* @param pVM Pointer to the VM.
3918	* @param pVCpu The VMCPU for the EMT we're being called on. Unused.
3919	* @param pvUser User pointer. Unused
3920	*
3921	*/
3922	static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysUnmapChunkRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
3923	{
3924	int rc = VINF_SUCCESS;
3925	pgmLock(pVM);
3926	NOREF(pVCpu); NOREF(pvUser);
3927
3928	if (pVM->pgm.s.ChunkR3Map.c >= pVM->pgm.s.ChunkR3Map.cMax)
3929	{
3930	/* Flush the pgm pool cache; call the internal rendezvous handler as we're already in a rendezvous handler here. */
3931	/** @todo also not really efficient to unmap a chunk that contains PD
3932	* or PT pages. */
3933	pgmR3PoolClearAllRendezvous(pVM, &pVM->aCpus[0], NULL /* no need to flush the REM TLB as we already did that above */);
3934
3935	/*
3936	* Request the ring-0 part to unmap a chunk to make space in the mapping cache.
3937	*/
3938	GMMMAPUNMAPCHUNKREQ Req;
3939	Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
3940	Req.Hdr.cbReq = sizeof(Req);
3941	Req.pvR3 = NULL;
3942	Req.idChunkMap = NIL_GMM_CHUNKID;
3943	Req.idChunkUnmap = pgmR3PhysChunkFindUnmapCandidate(pVM);
3944	if (Req.idChunkUnmap != INT32_MAX)
3945	{
3946	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkUnmap, a);
3947	rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
3948	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkUnmap, a);
3949	if (RT_SUCCESS(rc))
3950	{
3951	/*
3952	* Remove the unmapped one.
3953	*/
3954	PPGMCHUNKR3MAP pUnmappedChunk = (PPGMCHUNKR3MAP)RTAvlU32Remove(&pVM->pgm.s.ChunkR3Map.pTree, Req.idChunkUnmap);
3955	AssertRelease(pUnmappedChunk);
3956	AssertRelease(!pUnmappedChunk->cRefs);
3957	AssertRelease(!pUnmappedChunk->cPermRefs);
3958	pUnmappedChunk->pv = NULL;
3959	pUnmappedChunk->Core.Key = UINT32_MAX;
3960	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
3961	MMR3HeapFree(pUnmappedChunk);
3962	#else
3963	MMR3UkHeapFree(pVM, pUnmappedChunk, MM_TAG_PGM_CHUNK_MAPPING);
3964	#endif
3965	pVM->pgm.s.ChunkR3Map.c--;
3966	pVM->pgm.s.cUnmappedChunks++;
3967
3968	/*
3969	* Flush dangling PGM pointers (R3 & R0 ptrs to GC physical addresses).
3970	*/
3971	/** todo: we should not flush chunks which include cr3 mappings. */
3972	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3973	{
3974	PPGMCPU pPGM = &pVM->aCpus[idCpu].pgm.s;
3975
3976	pPGM->pGst32BitPdR3 = NULL;
3977	pPGM->pGstPaePdptR3 = NULL;
3978	pPGM->pGstAmd64Pml4R3 = NULL;
3979	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
3980	pPGM->pGst32BitPdR0 = NIL_RTR0PTR;
3981	pPGM->pGstPaePdptR0 = NIL_RTR0PTR;
3982	pPGM->pGstAmd64Pml4R0 = NIL_RTR0PTR;
3983	#endif
3984	for (unsigned i = 0; i < RT_ELEMENTS(pPGM->apGstPaePDsR3); i++)
3985	{
3986	pPGM->apGstPaePDsR3[i] = NULL;
3987	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
3988	pPGM->apGstPaePDsR0[i] = NIL_RTR0PTR;
3989	#endif
3990	}
3991
3992	/* Flush REM TLBs. */
3993	CPUMSetChangedFlags(&pVM->aCpus[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
3994	}
3995	#ifdef VBOX_WITH_REM
3996	/* Flush REM translation blocks. */
3997	REMFlushTBs(pVM);
3998	#endif
3999	}
4000	}
4001	}
4002	pgmUnlock(pVM);
4003	return rc;
4004	}
4005
4006	/**
4007	* Unmap a chunk to free up virtual address space (request packet handler for pgmR3PhysChunkMap)
4008	*
4009	* @returns VBox status code.
4010	* @param pVM Pointer to the VM.
4011	*/
4012	void pgmR3PhysUnmapChunk(PVM pVM)
4013	{
4014	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysUnmapChunkRendezvous, NULL);
4015	AssertRC(rc);
4016	}
4017
4018
4019	/**
4020	* Maps the given chunk into the ring-3 mapping cache.
4021	*
4022	* This will call ring-0.
4023	*
4024	* @returns VBox status code.
4025	* @param pVM Pointer to the VM.
4026	* @param idChunk The chunk in question.
4027	* @param ppChunk Where to store the chunk tracking structure.
4028	*
4029	* @remarks Called from within the PGM critical section.
4030	* @remarks Can be called from any thread!
4031	*/
4032	int pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk)
4033	{
4034	int rc;
4035
4036	PGM_LOCK_ASSERT_OWNER(pVM);
4037
4038	/*
4039	* Move the chunk time forward.
4040	*/
4041	pVM->pgm.s.ChunkR3Map.iNow++;
4042	if (pVM->pgm.s.ChunkR3Map.iNow == 0)
4043	{
4044	pVM->pgm.s.ChunkR3Map.iNow = 4;
4045	RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /fFromLeft/, pgmR3PhysChunkAgeingRolloverCallback, NULL);
4046	}
4047
4048	/*
4049	* Allocate a new tracking structure first.
4050	*/
4051	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
4052	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3HeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk));
4053	#else
4054	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3UkHeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk), NULL);
4055	#endif
4056	AssertReturn(pChunk, VERR_NO_MEMORY);
4057	pChunk->Core.Key = idChunk;
4058	pChunk->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
4059
4060	/*
4061	* Request the ring-0 part to map the chunk in question.
4062	*/
4063	GMMMAPUNMAPCHUNKREQ Req;
4064	Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
4065	Req.Hdr.cbReq = sizeof(Req);
4066	Req.pvR3 = NULL;
4067	Req.idChunkMap = idChunk;
4068	Req.idChunkUnmap = NIL_GMM_CHUNKID;
4069
4070	/* Must be callable from any thread, so can't use VMMR3CallR0. */
4071	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkMap, a);
4072	rc = SUPR3CallVMMR0Ex(pVM->pVMR0, NIL_VMCPUID, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
4073	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkMap, a);
4074	if (RT_SUCCESS(rc))
4075	{
4076	pChunk->pv = Req.pvR3;
4077
4078	/*
4079	* If we're running out of virtual address space, then we should
4080	* unmap another chunk.
4081	*
4082	* Currently, an unmap operation requires that all other virtual CPUs
4083	* are idling and not by chance making use of the memory we're
4084	* unmapping. So, we create an async unmap operation here.
4085	*
4086	* Now, when creating or restoring a saved state this wont work very
4087	* well since we may want to restore all guest RAM + a little something.
4088	* So, we have to do the unmap synchronously. Fortunately for us
4089	* though, during these operations the other virtual CPUs are inactive
4090	* and it should be safe to do this.
4091	*/
4092	/** @todo Eventually we should lock all memory when used and do
4093	* map+unmap as one kernel call without any rendezvous or
4094	* other precautions. */
4095	if (pVM->pgm.s.ChunkR3Map.c + 1 >= pVM->pgm.s.ChunkR3Map.cMax)
4096	{
4097	switch (VMR3GetState(pVM))
4098	{
4099	case VMSTATE_LOADING:
4100	case VMSTATE_SAVING:
4101	{
4102	PVMCPU pVCpu = VMMGetCpu(pVM);
4103	if ( pVCpu
4104	&& pVM->pgm.s.cDeprecatedPageLocks == 0)
4105	{
4106	pgmR3PhysUnmapChunkRendezvous(pVM, pVCpu, NULL);
4107	break;
4108	}
4109	/* fall thru */
4110	}
4111	default:
4112	rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysUnmapChunk, 1, pVM);
4113	AssertRC(rc);
4114	break;
4115	}
4116	}
4117
4118	/*
4119	* Update the tree. We must do this after any unmapping to make sure
4120	* the chunk we're going to return isn't unmapped by accident.
4121	*/
4122	AssertPtr(Req.pvR3);
4123	bool fRc = RTAvlU32Insert(&pVM->pgm.s.ChunkR3Map.pTree, &pChunk->Core);
4124	AssertRelease(fRc);
4125	pVM->pgm.s.ChunkR3Map.c++;
4126	pVM->pgm.s.cMappedChunks++;
4127	}
4128	else
4129	{
4130	/** @todo this may fail because of /proc/sys/vm/max_map_count, so we
4131	* should probably restrict ourselves on linux. */
4132	AssertRC(rc);
4133	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
4134	MMR3HeapFree(pChunk);
4135	#else
4136	MMR3UkHeapFree(pVM, pChunk, MM_TAG_PGM_CHUNK_MAPPING);
4137	#endif
4138	pChunk = NULL;
4139	}
4140
4141	*ppChunk = pChunk;
4142	return rc;
4143	}
4144
4145
4146	/**
4147	* For VMMCALLRING3_PGM_MAP_CHUNK, considered internal.
4148	*
4149	* @returns see pgmR3PhysChunkMap.
4150	* @param pVM Pointer to the VM.
4151	* @param idChunk The chunk to map.
4152	*/
4153	VMMR3DECL(int) PGMR3PhysChunkMap(PVM pVM, uint32_t idChunk)
4154	{
4155	PPGMCHUNKR3MAP pChunk;
4156	int rc;
4157
4158	pgmLock(pVM);
4159	rc = pgmR3PhysChunkMap(pVM, idChunk, &pChunk);
4160	pgmUnlock(pVM);
4161	return rc;
4162	}
4163
4164
4165	/**
4166	* Invalidates the TLB for the ring-3 mapping cache.
4167	*
4168	* @param pVM Pointer to the VM.
4169	*/
4170	VMMR3DECL(void) PGMR3PhysChunkInvalidateTLB(PVM pVM)
4171	{
4172	pgmLock(pVM);
4173	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
4174	{
4175	pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk = NIL_GMM_CHUNKID;
4176	pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk = NULL;
4177	}
4178	/* The page map TLB references chunks, so invalidate that one too. */
4179	pgmPhysInvalidatePageMapTLB(pVM);
4180	pgmUnlock(pVM);
4181	}
4182
4183
4184	/**
4185	* Response to VMMCALLRING3_PGM_ALLOCATE_LARGE_HANDY_PAGE to allocate a large
4186	* (2MB) page for use with a nested paging PDE.
4187	*
4188	* @returns The following VBox status codes.
4189	* @retval VINF_SUCCESS on success.
4190	* @retval VINF_EM_NO_MEMORY if we're out of memory.
4191	*
4192	* @param pVM Pointer to the VM.
4193	* @param GCPhys GC physical start address of the 2 MB range
4194	*/
4195	VMMR3DECL(int) PGMR3PhysAllocateLargeHandyPage(PVM pVM, RTGCPHYS GCPhys)
4196	{
4197	#ifdef PGM_WITH_LARGE_PAGES
4198	uint64_t u64TimeStamp1, u64TimeStamp2;
4199
4200	pgmLock(pVM);
4201
4202	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatAllocLargePage, a);
4203	u64TimeStamp1 = RTTimeMilliTS();
4204	int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_LARGE_HANDY_PAGE, 0, NULL);
4205	u64TimeStamp2 = RTTimeMilliTS();
4206	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatAllocLargePage, a);
4207	if (RT_SUCCESS(rc))
4208	{
4209	Assert(pVM->pgm.s.cLargeHandyPages == 1);
4210
4211	uint32_t idPage = pVM->pgm.s.aLargeHandyPage[0].idPage;
4212	RTHCPHYS HCPhys = pVM->pgm.s.aLargeHandyPage[0].HCPhysGCPhys;
4213
4214	void *pv;
4215
4216	/* Map the large page into our address space.
4217	*
4218	* Note: assuming that within the 2 MB range:
4219	* - GCPhys + PAGE_SIZE = HCPhys + PAGE_SIZE (whole point of this exercise)
4220	* - user space mapping is continuous as well
4221	* - page id (GCPhys) + 1 = page id (GCPhys + PAGE_SIZE)
4222	*/
4223	rc = pgmPhysPageMapByPageID(pVM, idPage, HCPhys, &pv);
4224	AssertLogRelMsg(RT_SUCCESS(rc), ("idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc\n", idPage, HCPhys, rc));
4225
4226	if (RT_SUCCESS(rc))
4227	{
4228	/*
4229	* Clear the pages.
4230	*/
4231	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatClearLargePage, b);
4232	for (unsigned i = 0; i < _2M/PAGE_SIZE; i++)
4233	{
4234	ASMMemZeroPage(pv);
4235
4236	PPGMPAGE pPage;
4237	rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
4238	AssertRC(rc);
4239
4240	Assert(PGM_PAGE_IS_ZERO(pPage));
4241	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatRZPageReplaceZero);
4242	pVM->pgm.s.cZeroPages--;
4243
4244	/*
4245	* Do the PGMPAGE modifications.
4246	*/
4247	pVM->pgm.s.cPrivatePages++;
4248	PGM_PAGE_SET_HCPHYS(pVM, pPage, HCPhys);
4249	PGM_PAGE_SET_PAGEID(pVM, pPage, idPage);
4250	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
4251	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PDE);
4252	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
4253	PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
4254
4255	/* Somewhat dirty assumption that page ids are increasing. */
4256	idPage++;
4257
4258	HCPhys += PAGE_SIZE;
4259	GCPhys += PAGE_SIZE;
4260
4261	pv = (void *)((uintptr_t)pv + PAGE_SIZE);
4262
4263	Log3(("PGMR3PhysAllocateLargePage: idPage=%#x HCPhys=%RGp\n", idPage, HCPhys));
4264	}
4265	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatClearLargePage, b);
4266
4267	/* Flush all TLBs. */
4268	PGM_INVL_ALL_VCPU_TLBS(pVM);
4269	pgmPhysInvalidatePageMapTLB(pVM);
4270	}
4271	pVM->pgm.s.cLargeHandyPages = 0;
4272	}
4273
4274	if (RT_SUCCESS(rc))
4275	{
4276	static uint32_t cTimeOut = 0;
4277	uint64_t u64TimeStampDelta = u64TimeStamp2 - u64TimeStamp1;
4278
4279	if (u64TimeStampDelta > 100)
4280	{
4281	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatLargePageOverflow);
4282	if ( ++cTimeOut > 10
4283	\|\| u64TimeStampDelta > 1000 /* more than one second forces an early retirement from allocating large pages. */)
4284	{
4285	/* If repeated attempts to allocate a large page takes more than 100 ms, then we fall back to normal 4k pages.
4286	* E.g. Vista 64 tries to move memory around, which takes a huge amount of time.
4287	*/
4288	LogRel(("PGMR3PhysAllocateLargePage: allocating large pages takes too long (last attempt %d ms; nr of timeouts %d); DISABLE\n", u64TimeStampDelta, cTimeOut));
4289	PGMSetLargePageUsage(pVM, false);
4290	}
4291	}
4292	else
4293	if (cTimeOut > 0)
4294	cTimeOut--;
4295	}
4296
4297	pgmUnlock(pVM);
4298	return rc;
4299	#else
4300	return VERR_NOT_IMPLEMENTED;
4301	#endif /* PGM_WITH_LARGE_PAGES */
4302	}
4303
4304
4305	/**
4306	* Response to VM_FF_PGM_NEED_HANDY_PAGES and VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES.
4307	*
4308	* This function will also work the VM_FF_PGM_NO_MEMORY force action flag, to
4309	* signal and clear the out of memory condition. When contracted, this API is
4310	* used to try clear the condition when the user wants to resume.
4311	*
4312	* @returns The following VBox status codes.
4313	* @retval VINF_SUCCESS on success. FFs cleared.
4314	* @retval VINF_EM_NO_MEMORY if we're out of memory. The FF is not cleared in
4315	* this case and it gets accompanied by VM_FF_PGM_NO_MEMORY.
4316	*
4317	* @param pVM Pointer to the VM.
4318	*
4319	* @remarks The VINF_EM_NO_MEMORY status is for the benefit of the FF processing
4320	* in EM.cpp and shouldn't be propagated outside TRPM, HWACCM, EM and
4321	* pgmPhysEnsureHandyPage. There is one exception to this in the \#PF
4322	* handler.
4323	*/
4324	VMMR3DECL(int) PGMR3PhysAllocateHandyPages(PVM pVM)
4325	{
4326	pgmLock(pVM);
4327
4328	/*
4329	* Allocate more pages, noting down the index of the first new page.
4330	*/
4331	uint32_t iClear = pVM->pgm.s.cHandyPages;
4332	AssertMsgReturn(iClear <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d", iClear), VERR_PGM_HANDY_PAGE_IPE);
4333	Log(("PGMR3PhysAllocateHandyPages: %d -> %d\n", iClear, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
4334	int rcAlloc = VINF_SUCCESS;
4335	int rcSeed = VINF_SUCCESS;
4336	int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
4337	while (rc == VERR_GMM_SEED_ME)
4338	{
4339	void *pvChunk;
4340	rcAlloc = rc = SUPR3PageAlloc(GMM_CHUNK_SIZE >> PAGE_SHIFT, &pvChunk);
4341	if (RT_SUCCESS(rc))
4342	{
4343	rcSeed = rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_SEED_CHUNK, (uintptr_t)pvChunk, NULL);
4344	if (RT_FAILURE(rc))
4345	SUPR3PageFree(pvChunk, GMM_CHUNK_SIZE >> PAGE_SHIFT);
4346	}
4347	if (RT_SUCCESS(rc))
4348	rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
4349	}
4350
4351	/* todo: we should split this up into an allocate and flush operation. sometimes you want to flush and not allocate more (which will trigger the vm account limit error) */
4352	if ( rc == VERR_GMM_HIT_VM_ACCOUNT_LIMIT
4353	&& pVM->pgm.s.cHandyPages > 0)
4354	{
4355	/* Still handy pages left, so don't panic. */
4356	rc = VINF_SUCCESS;
4357	}
4358
4359	if (RT_SUCCESS(rc))
4360	{
4361	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
4362	Assert(pVM->pgm.s.cHandyPages > 0);
4363	VM_FF_CLEAR(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
4364	VM_FF_CLEAR(pVM, VM_FF_PGM_NO_MEMORY);
4365
4366	#ifdef VBOX_STRICT
4367	uint32_t i;
4368	for (i = iClear; i < pVM->pgm.s.cHandyPages; i++)
4369	if ( pVM->pgm.s.aHandyPages[i].idPage == NIL_GMM_PAGEID
4370	\|\| pVM->pgm.s.aHandyPages[i].idSharedPage != NIL_GMM_PAGEID
4371	\|\| (pVM->pgm.s.aHandyPages[i].HCPhysGCPhys & PAGE_OFFSET_MASK))
4372	break;
4373	if (i != pVM->pgm.s.cHandyPages)
4374	{
4375	RTAssertMsg1Weak(NULL, __LINE__, __FILE__, __FUNCTION__);
4376	RTAssertMsg2Weak("i=%d iClear=%d cHandyPages=%d\n", i, iClear, pVM->pgm.s.cHandyPages);
4377	for (uint32_t j = iClear; j < pVM->pgm.s.cHandyPages; j++)
4378	RTAssertMsg2Add(("%03d: idPage=%d HCPhysGCPhys=%RHp idSharedPage=%d%\n", j,
4379	pVM->pgm.s.aHandyPages[j].idPage,
4380	pVM->pgm.s.aHandyPages[j].HCPhysGCPhys,
4381	pVM->pgm.s.aHandyPages[j].idSharedPage,
4382	j == i ? " <---" : ""));
4383	RTAssertPanic();
4384	}
4385	#endif
4386	/*
4387	* Clear the pages.
4388	*/
4389	while (iClear < pVM->pgm.s.cHandyPages)
4390	{
4391	PGMMPAGEDESC pPage = &pVM->pgm.s.aHandyPages[iClear];
4392	void *pv;
4393	rc = pgmPhysPageMapByPageID(pVM, pPage->idPage, pPage->HCPhysGCPhys, &pv);
4394	AssertLogRelMsgBreak(RT_SUCCESS(rc),
4395	("%u/%u: idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc\n",
4396	iClear, pVM->pgm.s.cHandyPages, pPage->idPage, pPage->HCPhysGCPhys, rc));
4397	ASMMemZeroPage(pv);
4398	iClear++;
4399	Log3(("PGMR3PhysAllocateHandyPages: idPage=%#x HCPhys=%RGp\n", pPage->idPage, pPage->HCPhysGCPhys));
4400	}
4401	}
4402	else
4403	{
4404	uint64_t cAllocPages, cMaxPages, cBalloonPages;
4405
4406	/*
4407	* We should never get here unless there is a genuine shortage of
4408	* memory (or some internal error). Flag the error so the VM can be
4409	* suspended ASAP and the user informed. If we're totally out of
4410	* handy pages we will return failure.
4411	*/
4412	/* Report the failure. */
4413	LogRel(("PGM: Failed to procure handy pages; rc=%Rrc rcAlloc=%Rrc rcSeed=%Rrc cHandyPages=%#x\n"
4414	" cAllPages=%#x cPrivatePages=%#x cSharedPages=%#x cZeroPages=%#x\n",
4415	rc, rcAlloc, rcSeed,
4416	pVM->pgm.s.cHandyPages,
4417	pVM->pgm.s.cAllPages,
4418	pVM->pgm.s.cPrivatePages,
4419	pVM->pgm.s.cSharedPages,
4420	pVM->pgm.s.cZeroPages));
4421
4422	if (GMMR3QueryMemoryStats(pVM, &cAllocPages, &cMaxPages, &cBalloonPages) == VINF_SUCCESS)
4423	{
4424	LogRel(("GMM: Statistics:\n"
4425	" Allocated pages: %RX64\n"
4426	" Maximum pages: %RX64\n"
4427	" Ballooned pages: %RX64\n", cAllocPages, cMaxPages, cBalloonPages));
4428	}
4429
4430	if ( rc != VERR_NO_MEMORY
4431	&& rc != VERR_NO_PHYS_MEMORY
4432	&& rc != VERR_LOCK_FAILED)
4433	{
4434	for (uint32_t i = 0; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
4435	{
4436	LogRel(("PGM: aHandyPages[#%#04x] = {.HCPhysGCPhys=%RHp, .idPage=%#08x, .idSharedPage=%#08x}\n",
4437	i, pVM->pgm.s.aHandyPages[i].HCPhysGCPhys, pVM->pgm.s.aHandyPages[i].idPage,
4438	pVM->pgm.s.aHandyPages[i].idSharedPage));
4439	uint32_t const idPage = pVM->pgm.s.aHandyPages[i].idPage;
4440	if (idPage != NIL_GMM_PAGEID)
4441	{
4442	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
4443	pRam;
4444	pRam = pRam->pNextR3)
4445	{
4446	uint32_t const cPages = pRam->cb >> PAGE_SHIFT;
4447	for (uint32_t iPage = 0; iPage < cPages; iPage++)
4448	if (PGM_PAGE_GET_PAGEID(&pRam->aPages[iPage]) == idPage)
4449	LogRel(("PGM: Used by %RGp %R[pgmpage] (%s)\n",
4450	pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pRam->aPages[iPage], pRam->pszDesc));
4451	}
4452	}
4453	}
4454	}
4455
4456	/* Set the FFs and adjust rc. */
4457	VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
4458	VM_FF_SET(pVM, VM_FF_PGM_NO_MEMORY);
4459	if ( rc == VERR_NO_MEMORY
4460	\|\| rc == VERR_NO_PHYS_MEMORY
4461	\|\| rc == VERR_LOCK_FAILED)
4462	rc = VINF_EM_NO_MEMORY;
4463	}
4464
4465	pgmUnlock(pVM);
4466	return rc;
4467	}
4468
4469
4470	/**
4471	* Frees the specified RAM page and replaces it with the ZERO page.
4472	*
4473	* This is used by ballooning, remapping MMIO2, RAM reset and state loading.
4474	*
4475	* @param pVM Pointer to the VM.
4476	* @param pReq Pointer to the request.
4477	* @param pcPendingPages Where the number of pages waiting to be freed are
4478	* kept. This will normally be incremented.
4479	* @param pPage Pointer to the page structure.
4480	* @param GCPhys The guest physical address of the page, if applicable.
4481	*
4482	* @remarks The caller must own the PGM lock.
4483	*/
4484	int pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys)
4485	{
4486	/*
4487	* Assert sanity.
4488	*/
4489	PGM_LOCK_ASSERT_OWNER(pVM);
4490	if (RT_UNLIKELY( PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
4491	&& PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_ROM_SHADOW))
4492	{
4493	AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
4494	return VMSetError(pVM, VERR_PGM_PHYS_NOT_RAM, RT_SRC_POS, "GCPhys=%RGp type=%d", GCPhys, PGM_PAGE_GET_TYPE(pPage));
4495	}
4496
4497	/** @todo What about ballooning of large pages??! */
4498	Assert( PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE
4499	&& PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE_DISABLED);
4500
4501	if ( PGM_PAGE_IS_ZERO(pPage)
4502	\|\| PGM_PAGE_IS_BALLOONED(pPage))
4503	return VINF_SUCCESS;
4504
4505	const uint32_t idPage = PGM_PAGE_GET_PAGEID(pPage);
4506	Log3(("pgmPhysFreePage: idPage=%#x GCPhys=%RGp pPage=%R[pgmpage]\n", idPage, GCPhys, pPage));
4507	if (RT_UNLIKELY( idPage == NIL_GMM_PAGEID
4508	\|\| idPage > GMM_PAGEID_LAST
4509	\|\| PGM_PAGE_GET_CHUNKID(pPage) == NIL_GMM_CHUNKID))
4510	{
4511	AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
4512	return VMSetError(pVM, VERR_PGM_PHYS_INVALID_PAGE_ID, RT_SRC_POS, "GCPhys=%RGp idPage=%#x", GCPhys, pPage);
4513	}
4514
4515	/* update page count stats. */
4516	if (PGM_PAGE_IS_SHARED(pPage))
4517	pVM->pgm.s.cSharedPages--;
4518	else
4519	pVM->pgm.s.cPrivatePages--;
4520	pVM->pgm.s.cZeroPages++;
4521
4522	/* Deal with write monitored pages. */
4523	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
4524	{
4525	PGM_PAGE_SET_WRITTEN_TO(pVM, pPage);
4526	pVM->pgm.s.cWrittenToPages++;
4527	}
4528
4529	/*
4530	* pPage = ZERO page.
4531	*/
4532	PGM_PAGE_SET_HCPHYS(pVM, pPage, pVM->pgm.s.HCPhysZeroPg);
4533	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
4534	PGM_PAGE_SET_PAGEID(pVM, pPage, NIL_GMM_PAGEID);
4535	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
4536	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
4537	PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
4538
4539	/* Flush physical page map TLB entry. */
4540	pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
4541
4542	/*
4543	* Make sure it's not in the handy page array.
4544	*/
4545	for (uint32_t i = pVM->pgm.s.cHandyPages; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
4546	{
4547	if (pVM->pgm.s.aHandyPages[i].idPage == idPage)
4548	{
4549	pVM->pgm.s.aHandyPages[i].idPage = NIL_GMM_PAGEID;
4550	break;
4551	}
4552	if (pVM->pgm.s.aHandyPages[i].idSharedPage == idPage)
4553	{
4554	pVM->pgm.s.aHandyPages[i].idSharedPage = NIL_GMM_PAGEID;
4555	break;
4556	}
4557	}
4558
4559	/*
4560	* Push it onto the page array.
4561	*/
4562	uint32_t iPage = *pcPendingPages;
4563	Assert(iPage < PGMPHYS_FREE_PAGE_BATCH_SIZE);
4564	*pcPendingPages += 1;
4565
4566	pReq->aPages[iPage].idPage = idPage;
4567
4568	if (iPage + 1 < PGMPHYS_FREE_PAGE_BATCH_SIZE)
4569	return VINF_SUCCESS;
4570
4571	/*
4572	* Flush the pages.
4573	*/
4574	int rc = GMMR3FreePagesPerform(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE);
4575	if (RT_SUCCESS(rc))
4576	{
4577	GMMR3FreePagesRePrep(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
4578	*pcPendingPages = 0;
4579	}
4580	return rc;
4581	}
4582
4583
4584	/**
4585	* Converts a GC physical address to a HC ring-3 pointer, with some
4586	* additional checks.
4587	*
4588	* @returns VBox status code.
4589	* @retval VINF_SUCCESS on success.
4590	* @retval VINF_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
4591	* access handler of some kind.
4592	* @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
4593	* accesses or is odd in any way.
4594	* @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
4595	*
4596	* @param pVM Pointer to the VM.
4597	* @param GCPhys The GC physical address to convert. Since this is only
4598	* used for filling the REM TLB, the A20 mask must be
4599	* applied before calling this API.
4600	* @param fWritable Whether write access is required.
4601	* @param ppv Where to store the pointer corresponding to GCPhys on
4602	* success.
4603	*/
4604	VMMR3DECL(int) PGMR3PhysTlbGCPhys2Ptr(PVM pVM, RTGCPHYS GCPhys, bool fWritable, void **ppv)
4605	{
4606	pgmLock(pVM);
4607	PGM_A20_ASSERT_MASKED(VMMGetCpu(pVM), GCPhys);
4608
4609	PPGMRAMRANGE pRam;
4610	PPGMPAGE pPage;
4611	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
4612	if (RT_SUCCESS(rc))
4613	{
4614	if (PGM_PAGE_IS_BALLOONED(pPage))
4615	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
4616	else if (!PGM_PAGE_HAS_ANY_HANDLERS(pPage))
4617	rc = VINF_SUCCESS;
4618	else
4619	{
4620	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
4621	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4622	else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
4623	{
4624	/** @todo Handle TLB loads of virtual handlers so ./test.sh can be made to work
4625	* in -norawr0 mode. */
4626	if (fWritable)
4627	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
4628	}
4629	else
4630	{
4631	/* Temporarily disabled physical handler(s), since the recompiler
4632	doesn't get notified when it's reset we'll have to pretend it's
4633	operating normally. */
4634	if (pgmHandlerPhysicalIsAll(pVM, GCPhys))
4635	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4636	else
4637	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
4638	}
4639	}
4640	if (RT_SUCCESS(rc))
4641	{
4642	int rc2;
4643
4644	/* Make sure what we return is writable. */
4645	if (fWritable)
4646	switch (PGM_PAGE_GET_STATE(pPage))
4647	{
4648	case PGM_PAGE_STATE_ALLOCATED:
4649	break;
4650	case PGM_PAGE_STATE_BALLOONED:
4651	AssertFailed();
4652	break;
4653	case PGM_PAGE_STATE_ZERO:
4654	case PGM_PAGE_STATE_SHARED:
4655	if (rc == VINF_PGM_PHYS_TLB_CATCH_WRITE)
4656	break;
4657	case PGM_PAGE_STATE_WRITE_MONITORED:
4658	rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
4659	AssertLogRelRCReturn(rc2, rc2);
4660	break;
4661	}
4662
4663	/* Get a ring-3 mapping of the address. */
4664	PPGMPAGER3MAPTLBE pTlbe;
4665	rc2 = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
4666	AssertLogRelRCReturn(rc2, rc2);
4667	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
4668	/** @todo mapping/locking hell; this isn't horribly efficient since
4669	* pgmPhysPageLoadIntoTlb will repeat the lookup we've done here. */
4670
4671	Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage] ppv=%p\n", GCPhys, rc, pPage, ppv));
4672	}
4673	else
4674	Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage]\n", GCPhys, rc, pPage));
4675
4676	/* else: handler catching all access, no pointer returned. */
4677	}
4678	else
4679	rc = VERR_PGM_PHYS_TLB_UNASSIGNED;
4680
4681	pgmUnlock(pVM);
4682	return rc;
4683	}
4684

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR3/PGMPhys.cpp@ 43045

Download in other formats: