PGMPhys.cpp@ 105745

Last change on this file since 105745 was 105352, checked in by vboxsync, 4 months ago
VMM/VMR3Req,iprt/cdefs.h: Adjustments of VMR3ReqCallUV family to fit darwin/arm64 restrictions. bugref:10725
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 248.8 KB

Line
1	/* $Id: PGMPhys.cpp 105352 2024-07-16 11:21:19Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor, Physical Memory Addressing.
4	*/
5
6	/*
7	* Copyright (C) 2006-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_PGM_PHYS
33	#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */
34	#include <VBox/vmm/pgm.h>
35	#include <VBox/vmm/iem.h>
36	#include <VBox/vmm/iom.h>
37	#include <VBox/vmm/mm.h>
38	#include <VBox/vmm/nem.h>
39	#include <VBox/vmm/stam.h>
40	#include <VBox/vmm/pdmdev.h>
41	#include "PGMInternal.h"
42	#include <VBox/vmm/vmcc.h>
43
44	#include "PGMInline.h"
45
46	#include <VBox/sup.h>
47	#include <VBox/param.h>
48	#include <VBox/err.h>
49	#include <VBox/log.h>
50	#include <iprt/assert.h>
51	#include <iprt/alloc.h>
52	#include <iprt/asm.h>
53	#ifdef VBOX_STRICT
54	# include <iprt/crc.h>
55	#endif
56	#include <iprt/thread.h>
57	#include <iprt/string.h>
58	#include <iprt/system.h>
59
60
61	/*********************************************************************************************************************************
62	* Defined Constants And Macros *
63	*********************************************************************************************************************************/
64	/** The number of pages to free in one batch. */
65	#define PGMPHYS_FREE_PAGE_BATCH_SIZE 128
66
67
68
69	/*********************************************************************************************************************************
70	* Reading and Writing Guest Pysical Memory *
71	*********************************************************************************************************************************/
72
73	/*
74	* PGMR3PhysReadU8-64
75	* PGMR3PhysWriteU8-64
76	*/
77	#define PGMPHYSFN_READNAME PGMR3PhysReadU8
78	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU8
79	#define PGMPHYS_DATASIZE 1
80	#define PGMPHYS_DATATYPE uint8_t
81	#include "PGMPhysRWTmpl.h"
82
83	#define PGMPHYSFN_READNAME PGMR3PhysReadU16
84	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU16
85	#define PGMPHYS_DATASIZE 2
86	#define PGMPHYS_DATATYPE uint16_t
87	#include "PGMPhysRWTmpl.h"
88
89	#define PGMPHYSFN_READNAME PGMR3PhysReadU32
90	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU32
91	#define PGMPHYS_DATASIZE 4
92	#define PGMPHYS_DATATYPE uint32_t
93	#include "PGMPhysRWTmpl.h"
94
95	#define PGMPHYSFN_READNAME PGMR3PhysReadU64
96	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU64
97	#define PGMPHYS_DATASIZE 8
98	#define PGMPHYS_DATATYPE uint64_t
99	#include "PGMPhysRWTmpl.h"
100
101
102	/**
103	* EMT worker for PGMR3PhysReadExternal.
104	*/
105	static DECLCALLBACK(int) pgmR3PhysReadExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, void *pvBuf, size_t cbRead,
106	PGMACCESSORIGIN enmOrigin)
107	{
108	VBOXSTRICTRC rcStrict = PGMPhysRead(pVM, *pGCPhys, pvBuf, cbRead, enmOrigin);
109	AssertMsg(rcStrict == VINF_SUCCESS, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); NOREF(rcStrict);
110	return VINF_SUCCESS;
111	}
112
113
114	/**
115	* Read from physical memory, external users.
116	*
117	* @returns VBox status code.
118	* @retval VINF_SUCCESS.
119	*
120	* @param pVM The cross context VM structure.
121	* @param GCPhys Physical address to read from.
122	* @param pvBuf Where to read into.
123	* @param cbRead How many bytes to read.
124	* @param enmOrigin Who is calling.
125	*
126	* @thread Any but EMTs.
127	*/
128	VMMR3DECL(int) PGMR3PhysReadExternal(PVM pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead, PGMACCESSORIGIN enmOrigin)
129	{
130	VM_ASSERT_OTHER_THREAD(pVM);
131
132	AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
133	LogFlow(("PGMR3PhysReadExternal: %RGp %d\n", GCPhys, cbRead));
134
135	PGM_LOCK_VOID(pVM);
136
137	/*
138	* Copy loop on ram ranges.
139	*/
140	for (;;)
141	{
142	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
143
144	/* Inside range or not? */
145	if (pRam && GCPhys >= pRam->GCPhys)
146	{
147	/*
148	* Must work our way thru this page by page.
149	*/
150	RTGCPHYS off = GCPhys - pRam->GCPhys;
151	while (off < pRam->cb)
152	{
153	unsigned iPage = off >> GUEST_PAGE_SHIFT;
154	PPGMPAGE pPage = &pRam->aPages[iPage];
155
156	/*
157	* If the page has an ALL access handler, we'll have to
158	* delegate the job to EMT.
159	*/
160	if ( PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
161	\|\| PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
162	{
163	PGM_UNLOCK(pVM);
164
165	return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysReadExternalEMT, 5,
166	pVM, &GCPhys, pvBuf, cbRead, enmOrigin);
167	}
168	Assert(!PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage));
169
170	/*
171	* Simple stuff, go ahead.
172	*/
173	size_t cb = GUEST_PAGE_SIZE - (off & GUEST_PAGE_OFFSET_MASK);
174	if (cb > cbRead)
175	cb = cbRead;
176	PGMPAGEMAPLOCK PgMpLck;
177	const void *pvSrc;
178	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc, &PgMpLck);
179	if (RT_SUCCESS(rc))
180	{
181	memcpy(pvBuf, pvSrc, cb);
182	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
183	}
184	else
185	{
186	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
187	pRam->GCPhys + off, pPage, rc));
188	memset(pvBuf, 0xff, cb);
189	}
190
191	/* next page */
192	if (cb >= cbRead)
193	{
194	PGM_UNLOCK(pVM);
195	return VINF_SUCCESS;
196	}
197	cbRead -= cb;
198	off += cb;
199	GCPhys += cb;
200	pvBuf = (char *)pvBuf + cb;
201	} /* walk pages in ram range. */
202	}
203	else
204	{
205	LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
206
207	/*
208	* Unassigned address space.
209	*/
210	size_t cb = pRam ? pRam->GCPhys - GCPhys : ~(size_t)0;
211	if (cb >= cbRead)
212	{
213	memset(pvBuf, 0xff, cbRead);
214	break;
215	}
216	memset(pvBuf, 0xff, cb);
217
218	cbRead -= cb;
219	pvBuf = (char *)pvBuf + cb;
220	GCPhys += cb;
221	}
222	} /* Ram range walk */
223
224	PGM_UNLOCK(pVM);
225
226	return VINF_SUCCESS;
227	}
228
229
230	/**
231	* EMT worker for PGMR3PhysWriteExternal.
232	*/
233	static DECLCALLBACK(int) pgmR3PhysWriteExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, const void *pvBuf, size_t cbWrite,
234	PGMACCESSORIGIN enmOrigin)
235	{
236	/** @todo VERR_EM_NO_MEMORY */
237	VBOXSTRICTRC rcStrict = PGMPhysWrite(pVM, *pGCPhys, pvBuf, cbWrite, enmOrigin);
238	AssertMsg(rcStrict == VINF_SUCCESS, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); NOREF(rcStrict);
239	return VINF_SUCCESS;
240	}
241
242
243	/**
244	* Write to physical memory, external users.
245	*
246	* @returns VBox status code.
247	* @retval VINF_SUCCESS.
248	* @retval VERR_EM_NO_MEMORY.
249	*
250	* @param pVM The cross context VM structure.
251	* @param GCPhys Physical address to write to.
252	* @param pvBuf What to write.
253	* @param cbWrite How many bytes to write.
254	* @param enmOrigin Who is calling.
255	*
256	* @thread Any but EMTs.
257	*/
258	VMMDECL(int) PGMR3PhysWriteExternal(PVM pVM, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite, PGMACCESSORIGIN enmOrigin)
259	{
260	VM_ASSERT_OTHER_THREAD(pVM);
261
262	AssertMsg(!pVM->pgm.s.fNoMorePhysWrites,
263	("Calling PGMR3PhysWriteExternal after pgmR3Save()! GCPhys=%RGp cbWrite=%#x enmOrigin=%d\n",
264	GCPhys, cbWrite, enmOrigin));
265	AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
266	LogFlow(("PGMR3PhysWriteExternal: %RGp %d\n", GCPhys, cbWrite));
267
268	PGM_LOCK_VOID(pVM);
269
270	/*
271	* Copy loop on ram ranges, stop when we hit something difficult.
272	*/
273	for (;;)
274	{
275	PPGMRAMRANGE const pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
276
277	/* Inside range or not? */
278	if (pRam && GCPhys >= pRam->GCPhys)
279	{
280	/*
281	* Must work our way thru this page by page.
282	*/
283	RTGCPTR off = GCPhys - pRam->GCPhys;
284	while (off < pRam->cb)
285	{
286	RTGCPTR iPage = off >> GUEST_PAGE_SHIFT;
287	PPGMPAGE pPage = &pRam->aPages[iPage];
288
289	/*
290	* Is the page problematic, we have to do the work on the EMT.
291	*
292	* Allocating writable pages and access handlers are
293	* problematic, write monitored pages are simple and can be
294	* dealt with here.
295	*/
296	if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
297	\|\| PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
298	\|\| PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
299	{
300	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
301	&& !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
302	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, GCPhys);
303	else
304	{
305	PGM_UNLOCK(pVM);
306
307	return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysWriteExternalEMT, 5,
308	pVM, &GCPhys, pvBuf, cbWrite, enmOrigin);
309	}
310	}
311	Assert(!PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage));
312
313	/*
314	* Simple stuff, go ahead.
315	*/
316	size_t cb = GUEST_PAGE_SIZE - (off & GUEST_PAGE_OFFSET_MASK);
317	if (cb > cbWrite)
318	cb = cbWrite;
319	PGMPAGEMAPLOCK PgMpLck;
320	void *pvDst;
321	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst, &PgMpLck);
322	if (RT_SUCCESS(rc))
323	{
324	memcpy(pvDst, pvBuf, cb);
325	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
326	}
327	else
328	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
329	pRam->GCPhys + off, pPage, rc));
330
331	/* next page */
332	if (cb >= cbWrite)
333	{
334	PGM_UNLOCK(pVM);
335	return VINF_SUCCESS;
336	}
337
338	cbWrite -= cb;
339	off += cb;
340	GCPhys += cb;
341	pvBuf = (const char *)pvBuf + cb;
342	} /* walk pages in ram range */
343	}
344	else
345	{
346	/*
347	* Unassigned address space, skip it.
348	*/
349	if (!pRam)
350	break;
351	size_t cb = pRam->GCPhys - GCPhys;
352	if (cb >= cbWrite)
353	break;
354	cbWrite -= cb;
355	pvBuf = (const char *)pvBuf + cb;
356	GCPhys += cb;
357	}
358	} /* Ram range walk */
359
360	PGM_UNLOCK(pVM);
361	return VINF_SUCCESS;
362	}
363
364
365	/*********************************************************************************************************************************
366	* Mapping Guest Physical Memory *
367	*********************************************************************************************************************************/
368
369	/**
370	* VMR3ReqCall worker for PGMR3PhysGCPhys2CCPtrExternal to make pages writable.
371	*
372	* @returns see PGMR3PhysGCPhys2CCPtrExternal
373	* @param pVM The cross context VM structure.
374	* @param pGCPhys Pointer to the guest physical address.
375	* @param ppv Where to store the mapping address.
376	* @param pLock Where to store the lock.
377	*/
378	static DECLCALLBACK(int) pgmR3PhysGCPhys2CCPtrDelegated(PVM pVM, PRTGCPHYS pGCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
379	{
380	/*
381	* Just hand it to PGMPhysGCPhys2CCPtr and check that it's not a page with
382	* an access handler after it succeeds.
383	*/
384	int rc = PGM_LOCK(pVM);
385	AssertRCReturn(rc, rc);
386
387	rc = PGMPhysGCPhys2CCPtr(pVM, *pGCPhys, ppv, pLock);
388	if (RT_SUCCESS(rc))
389	{
390	PPGMPAGEMAPTLBE pTlbe;
391	int rc2 = pgmPhysPageQueryTlbe(pVM, *pGCPhys, &pTlbe);
392	AssertFatalRC(rc2);
393	PPGMPAGE pPage = pTlbe->pPage;
394	if (PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
395	{
396	PGMPhysReleasePageMappingLock(pVM, pLock);
397	rc = VERR_PGM_PHYS_PAGE_RESERVED;
398	}
399	else if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
400	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
401	\|\| pgmPoolIsDirtyPage(pVM, *pGCPhys)
402	#endif
403	)
404	{
405	/* We must flush any corresponding pgm pool page here, otherwise we'll
406	* not be informed about writes and keep bogus gst->shw mappings around.
407	*/
408	pgmPoolFlushPageByGCPhys(pVM, *pGCPhys);
409	Assert(!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage));
410	/** @todo r=bird: return VERR_PGM_PHYS_PAGE_RESERVED here if it still has
411	* active handlers, see the PGMR3PhysGCPhys2CCPtrExternal docs. */
412	}
413	}
414
415	PGM_UNLOCK(pVM);
416	return rc;
417	}
418
419
420	/**
421	* Requests the mapping of a guest page into ring-3, external threads.
422	*
423	* When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
424	* release it.
425	*
426	* This API will assume your intention is to write to the page, and will
427	* therefore replace shared and zero pages. If you do not intend to modify the
428	* page, use the PGMR3PhysGCPhys2CCPtrReadOnlyExternal() API.
429	*
430	* @returns VBox status code.
431	* @retval VINF_SUCCESS on success.
432	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
433	* backing or if the page has any active access handlers. The caller
434	* must fall back on using PGMR3PhysWriteExternal.
435	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
436	*
437	* @param pVM The cross context VM structure.
438	* @param GCPhys The guest physical address of the page that should be mapped.
439	* @param ppv Where to store the address corresponding to GCPhys.
440	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
441	*
442	* @remark Avoid calling this API from within critical sections (other than the
443	* PGM one) because of the deadlock risk when we have to delegating the
444	* task to an EMT.
445	* @thread Any.
446	*/
447	VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrExternal(PVM pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
448	{
449	AssertPtr(ppv);
450	AssertPtr(pLock);
451
452	Assert(VM_IS_EMT(pVM) \|\| !PGMIsLockOwner(pVM));
453
454	int rc = PGM_LOCK(pVM);
455	AssertRCReturn(rc, rc);
456
457	/*
458	* Query the Physical TLB entry for the page (may fail).
459	*/
460	PPGMPAGEMAPTLBE pTlbe;
461	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
462	if (RT_SUCCESS(rc))
463	{
464	PPGMPAGE pPage = pTlbe->pPage;
465	if (PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
466	rc = VERR_PGM_PHYS_PAGE_RESERVED;
467	else
468	{
469	/*
470	* If the page is shared, the zero page, or being write monitored
471	* it must be converted to an page that's writable if possible.
472	* We can only deal with write monitored pages here, the rest have
473	* to be on an EMT.
474	*/
475	if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
476	\|\| PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
477	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
478	\|\| pgmPoolIsDirtyPage(pVM, GCPhys)
479	#endif
480	)
481	{
482	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
483	&& !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
484	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
485	&& !pgmPoolIsDirtyPage(pVM, GCPhys) /** @todo we're very likely doing this twice. */
486	#endif
487	)
488	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, GCPhys);
489	else
490	{
491	PGM_UNLOCK(pVM);
492
493	return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysGCPhys2CCPtrDelegated, 4,
494	pVM, &GCPhys, ppv, pLock);
495	}
496	}
497
498	/*
499	* Now, just perform the locking and calculate the return address.
500	*/
501	PPGMPAGEMAP pMap = pTlbe->pMap;
502	if (pMap)
503	pMap->cRefs++;
504
505	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
506	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
507	{
508	if (cLocks == 0)
509	pVM->pgm.s.cWriteLockedPages++;
510	PGM_PAGE_INC_WRITE_LOCKS(pPage);
511	}
512	else if (cLocks != PGM_PAGE_GET_WRITE_LOCKS(pPage))
513	{
514	PGM_PAGE_INC_WRITE_LOCKS(pPage);
515	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent write locked state!\n", GCPhys, pPage));
516	if (pMap)
517	pMap->cRefs++; /* Extra ref to prevent it from going away. */
518	}
519
520	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
521	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_WRITE;
522	pLock->pvMap = pMap;
523	}
524	}
525
526	PGM_UNLOCK(pVM);
527	return rc;
528	}
529
530
531	/**
532	* Requests the mapping of a guest page into ring-3, external threads.
533	*
534	* When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
535	* release it.
536	*
537	* @returns VBox status code.
538	* @retval VINF_SUCCESS on success.
539	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
540	* backing or if the page as an active ALL access handler. The caller
541	* must fall back on using PGMPhysRead.
542	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
543	*
544	* @param pVM The cross context VM structure.
545	* @param GCPhys The guest physical address of the page that should be mapped.
546	* @param ppv Where to store the address corresponding to GCPhys.
547	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
548	*
549	* @remark Avoid calling this API from within critical sections (other than
550	* the PGM one) because of the deadlock risk.
551	* @thread Any.
552	*/
553	VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrReadOnlyExternal(PVM pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
554	{
555	int rc = PGM_LOCK(pVM);
556	AssertRCReturn(rc, rc);
557
558	/*
559	* Query the Physical TLB entry for the page (may fail).
560	*/
561	PPGMPAGEMAPTLBE pTlbe;
562	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
563	if (RT_SUCCESS(rc))
564	{
565	PPGMPAGE pPage = pTlbe->pPage;
566	#if 1
567	/* MMIO pages doesn't have any readable backing. */
568	if (PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
569	rc = VERR_PGM_PHYS_PAGE_RESERVED;
570	#else
571	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
572	rc = VERR_PGM_PHYS_PAGE_RESERVED;
573	#endif
574	else
575	{
576	/*
577	* Now, just perform the locking and calculate the return address.
578	*/
579	PPGMPAGEMAP pMap = pTlbe->pMap;
580	if (pMap)
581	pMap->cRefs++;
582
583	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
584	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
585	{
586	if (cLocks == 0)
587	pVM->pgm.s.cReadLockedPages++;
588	PGM_PAGE_INC_READ_LOCKS(pPage);
589	}
590	else if (cLocks != PGM_PAGE_GET_READ_LOCKS(pPage))
591	{
592	PGM_PAGE_INC_READ_LOCKS(pPage);
593	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent readonly locked state!\n", GCPhys, pPage));
594	if (pMap)
595	pMap->cRefs++; /* Extra ref to prevent it from going away. */
596	}
597
598	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
599	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_READ;
600	pLock->pvMap = pMap;
601	}
602	}
603
604	PGM_UNLOCK(pVM);
605	return rc;
606	}
607
608
609	/**
610	* Requests the mapping of multiple guest page into ring-3, external threads.
611	*
612	* When you're done with the pages, call PGMPhysBulkReleasePageMappingLock()
613	* ASAP to release them.
614	*
615	* This API will assume your intention is to write to the pages, and will
616	* therefore replace shared and zero pages. If you do not intend to modify the
617	* pages, use the PGMR3PhysBulkGCPhys2CCPtrReadOnlyExternal() API.
618	*
619	* @returns VBox status code.
620	* @retval VINF_SUCCESS on success.
621	* @retval VERR_PGM_PHYS_PAGE_RESERVED if any of the pages has no physical
622	* backing or if any of the pages the page has any active access
623	* handlers. The caller must fall back on using PGMR3PhysWriteExternal.
624	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if @a paGCPhysPages contains
625	* an invalid physical address.
626	*
627	* @param pVM The cross context VM structure.
628	* @param cPages Number of pages to lock.
629	* @param paGCPhysPages The guest physical address of the pages that
630	* should be mapped (@a cPages entries).
631	* @param papvPages Where to store the ring-3 mapping addresses
632	* corresponding to @a paGCPhysPages.
633	* @param paLocks Where to store the locking information that
634	* pfnPhysBulkReleasePageMappingLock needs (@a cPages
635	* in length).
636	*
637	* @remark Avoid calling this API from within critical sections (other than the
638	* PGM one) because of the deadlock risk when we have to delegating the
639	* task to an EMT.
640	* @thread Any.
641	*/
642	VMMR3DECL(int) PGMR3PhysBulkGCPhys2CCPtrExternal(PVM pVM, uint32_t cPages, PCRTGCPHYS paGCPhysPages,
643	void **papvPages, PPGMPAGEMAPLOCK paLocks)
644	{
645	Assert(cPages > 0);
646	AssertPtr(papvPages);
647	AssertPtr(paLocks);
648
649	Assert(VM_IS_EMT(pVM) \|\| !PGMIsLockOwner(pVM));
650
651	int rc = PGM_LOCK(pVM);
652	AssertRCReturn(rc, rc);
653
654	/*
655	* Lock the pages one by one.
656	* The loop body is similar to PGMR3PhysGCPhys2CCPtrExternal.
657	*/
658	int32_t cNextYield = 128;
659	uint32_t iPage;
660	for (iPage = 0; iPage < cPages; iPage++)
661	{
662	if (--cNextYield > 0)
663	{ /* likely */ }
664	else
665	{
666	PGM_UNLOCK(pVM);
667	ASMNopPause();
668	PGM_LOCK_VOID(pVM);
669	cNextYield = 128;
670	}
671
672	/*
673	* Query the Physical TLB entry for the page (may fail).
674	*/
675	PPGMPAGEMAPTLBE pTlbe;
676	rc = pgmPhysPageQueryTlbe(pVM, paGCPhysPages[iPage], &pTlbe);
677	if (RT_SUCCESS(rc))
678	{ }
679	else
680	break;
681	PPGMPAGE pPage = pTlbe->pPage;
682
683	/*
684	* No MMIO or active access handlers.
685	*/
686	if ( !PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage)
687	&& !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
688	{ }
689	else
690	{
691	rc = VERR_PGM_PHYS_PAGE_RESERVED;
692	break;
693	}
694
695	/*
696	* The page must be in the allocated state and not be a dirty pool page.
697	* We can handle converting a write monitored page to an allocated one, but
698	* anything more complicated must be delegated to an EMT.
699	*/
700	bool fDelegateToEmt = false;
701	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED)
702	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
703	fDelegateToEmt = pgmPoolIsDirtyPage(pVM, paGCPhysPages[iPage]);
704	#else
705	fDelegateToEmt = false;
706	#endif
707	else if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
708	{
709	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
710	if (!pgmPoolIsDirtyPage(pVM, paGCPhysPages[iPage]))
711	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, paGCPhysPages[iPage]);
712	else
713	fDelegateToEmt = true;
714	#endif
715	}
716	else
717	fDelegateToEmt = true;
718	if (!fDelegateToEmt)
719	{ }
720	else
721	{
722	/* We could do this delegation in bulk, but considered too much work vs gain. */
723	PGM_UNLOCK(pVM);
724	rc = VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysGCPhys2CCPtrDelegated, 4,
725	pVM, &paGCPhysPages[iPage], &papvPages[iPage], &paLocks[iPage]);
726	PGM_LOCK_VOID(pVM);
727	if (RT_FAILURE(rc))
728	break;
729	cNextYield = 128;
730	}
731
732	/*
733	* Now, just perform the locking and address calculation.
734	*/
735	PPGMPAGEMAP pMap = pTlbe->pMap;
736	if (pMap)
737	pMap->cRefs++;
738
739	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
740	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
741	{
742	if (cLocks == 0)
743	pVM->pgm.s.cWriteLockedPages++;
744	PGM_PAGE_INC_WRITE_LOCKS(pPage);
745	}
746	else if (cLocks != PGM_PAGE_GET_WRITE_LOCKS(pPage))
747	{
748	PGM_PAGE_INC_WRITE_LOCKS(pPage);
749	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent write locked state!\n", paGCPhysPages[iPage], pPage));
750	if (pMap)
751	pMap->cRefs++; /* Extra ref to prevent it from going away. */
752	}
753
754	papvPages[iPage] = (void *)((uintptr_t)pTlbe->pv \| (uintptr_t)(paGCPhysPages[iPage] & GUEST_PAGE_OFFSET_MASK));
755	paLocks[iPage].uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_WRITE;
756	paLocks[iPage].pvMap = pMap;
757	}
758
759	PGM_UNLOCK(pVM);
760
761	/*
762	* On failure we must unlock any pages we managed to get already.
763	*/
764	if (RT_FAILURE(rc) && iPage > 0)
765	PGMPhysBulkReleasePageMappingLocks(pVM, iPage, paLocks);
766
767	return rc;
768	}
769
770
771	/**
772	* Requests the mapping of multiple guest page into ring-3, for reading only,
773	* external threads.
774	*
775	* When you're done with the pages, call PGMPhysReleasePageMappingLock() ASAP
776	* to release them.
777	*
778	* @returns VBox status code.
779	* @retval VINF_SUCCESS on success.
780	* @retval VERR_PGM_PHYS_PAGE_RESERVED if any of the pages has no physical
781	* backing or if any of the pages the page has an active ALL access
782	* handler. The caller must fall back on using PGMR3PhysWriteExternal.
783	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if @a paGCPhysPages contains
784	* an invalid physical address.
785	*
786	* @param pVM The cross context VM structure.
787	* @param cPages Number of pages to lock.
788	* @param paGCPhysPages The guest physical address of the pages that
789	* should be mapped (@a cPages entries).
790	* @param papvPages Where to store the ring-3 mapping addresses
791	* corresponding to @a paGCPhysPages.
792	* @param paLocks Where to store the lock information that
793	* pfnPhysReleasePageMappingLock needs (@a cPages
794	* in length).
795	*
796	* @remark Avoid calling this API from within critical sections (other than
797	* the PGM one) because of the deadlock risk.
798	* @thread Any.
799	*/
800	VMMR3DECL(int) PGMR3PhysBulkGCPhys2CCPtrReadOnlyExternal(PVM pVM, uint32_t cPages, PCRTGCPHYS paGCPhysPages,
801	void const **papvPages, PPGMPAGEMAPLOCK paLocks)
802	{
803	Assert(cPages > 0);
804	AssertPtr(papvPages);
805	AssertPtr(paLocks);
806
807	Assert(VM_IS_EMT(pVM) \|\| !PGMIsLockOwner(pVM));
808
809	int rc = PGM_LOCK(pVM);
810	AssertRCReturn(rc, rc);
811
812	/*
813	* Lock the pages one by one.
814	* The loop body is similar to PGMR3PhysGCPhys2CCPtrReadOnlyExternal.
815	*/
816	int32_t cNextYield = 256;
817	uint32_t iPage;
818	for (iPage = 0; iPage < cPages; iPage++)
819	{
820	if (--cNextYield > 0)
821	{ /* likely */ }
822	else
823	{
824	PGM_UNLOCK(pVM);
825	ASMNopPause();
826	PGM_LOCK_VOID(pVM);
827	cNextYield = 256;
828	}
829
830	/*
831	* Query the Physical TLB entry for the page (may fail).
832	*/
833	PPGMPAGEMAPTLBE pTlbe;
834	rc = pgmPhysPageQueryTlbe(pVM, paGCPhysPages[iPage], &pTlbe);
835	if (RT_SUCCESS(rc))
836	{ }
837	else
838	break;
839	PPGMPAGE pPage = pTlbe->pPage;
840
841	/*
842	* No MMIO or active all access handlers, everything else can be accessed.
843	*/
844	if ( !PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage)
845	&& !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
846	{ }
847	else
848	{
849	rc = VERR_PGM_PHYS_PAGE_RESERVED;
850	break;
851	}
852
853	/*
854	* Now, just perform the locking and address calculation.
855	*/
856	PPGMPAGEMAP pMap = pTlbe->pMap;
857	if (pMap)
858	pMap->cRefs++;
859
860	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
861	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
862	{
863	if (cLocks == 0)
864	pVM->pgm.s.cReadLockedPages++;
865	PGM_PAGE_INC_READ_LOCKS(pPage);
866	}
867	else if (cLocks != PGM_PAGE_GET_READ_LOCKS(pPage))
868	{
869	PGM_PAGE_INC_READ_LOCKS(pPage);
870	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent readonly locked state!\n", paGCPhysPages[iPage], pPage));
871	if (pMap)
872	pMap->cRefs++; /* Extra ref to prevent it from going away. */
873	}
874
875	papvPages[iPage] = (void *)((uintptr_t)pTlbe->pv \| (uintptr_t)(paGCPhysPages[iPage] & GUEST_PAGE_OFFSET_MASK));
876	paLocks[iPage].uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_READ;
877	paLocks[iPage].pvMap = pMap;
878	}
879
880	PGM_UNLOCK(pVM);
881
882	/*
883	* On failure we must unlock any pages we managed to get already.
884	*/
885	if (RT_FAILURE(rc) && iPage > 0)
886	PGMPhysBulkReleasePageMappingLocks(pVM, iPage, paLocks);
887
888	return rc;
889	}
890
891
892	/**
893	* Converts a GC physical address to a HC ring-3 pointer, with some
894	* additional checks.
895	*
896	* @returns VBox status code.
897	* @retval VINF_SUCCESS on success.
898	* @retval VINF_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
899	* access handler of some kind.
900	* @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
901	* accesses or is odd in any way.
902	* @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
903	*
904	* @param pVM The cross context VM structure.
905	* @param GCPhys The GC physical address to convert. Since this is only
906	* used for filling the REM TLB, the A20 mask must be
907	* applied before calling this API.
908	* @param fWritable Whether write access is required.
909	* @param ppv Where to store the pointer corresponding to GCPhys on
910	* success.
911	*/
912	VMMR3DECL(int) PGMR3PhysTlbGCPhys2Ptr(PVM pVM, RTGCPHYS GCPhys, bool fWritable, void **ppv)
913	{
914	PGM_LOCK_VOID(pVM);
915	PGM_A20_ASSERT_MASKED(VMMGetCpu(pVM), GCPhys);
916
917	PPGMRAMRANGE pRam;
918	PPGMPAGE pPage;
919	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
920	if (RT_SUCCESS(rc))
921	{
922	if (PGM_PAGE_IS_BALLOONED(pPage))
923	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
924	else if (!PGM_PAGE_HAS_ANY_HANDLERS(pPage))
925	rc = VINF_SUCCESS;
926	else
927	{
928	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
929	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
930	else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
931	{
932	/** @todo Handle TLB loads of virtual handlers so ./test.sh can be made to work
933	* in -norawr0 mode. */
934	if (fWritable)
935	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
936	}
937	else
938	{
939	/* Temporarily disabled physical handler(s), since the recompiler
940	doesn't get notified when it's reset we'll have to pretend it's
941	operating normally. */
942	if (pgmHandlerPhysicalIsAll(pVM, GCPhys))
943	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
944	else
945	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
946	}
947	}
948	if (RT_SUCCESS(rc))
949	{
950	int rc2;
951
952	/* Make sure what we return is writable. */
953	if (fWritable)
954	switch (PGM_PAGE_GET_STATE(pPage))
955	{
956	case PGM_PAGE_STATE_ALLOCATED:
957	break;
958	case PGM_PAGE_STATE_BALLOONED:
959	AssertFailed();
960	break;
961	case PGM_PAGE_STATE_ZERO:
962	case PGM_PAGE_STATE_SHARED:
963	if (rc == VINF_PGM_PHYS_TLB_CATCH_WRITE)
964	break;
965	RT_FALL_THRU();
966	case PGM_PAGE_STATE_WRITE_MONITORED:
967	rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK);
968	AssertLogRelRCReturn(rc2, rc2);
969	break;
970	}
971
972	/* Get a ring-3 mapping of the address. */
973	PPGMPAGER3MAPTLBE pTlbe;
974	rc2 = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
975	AssertLogRelRCReturn(rc2, rc2);
976	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
977	/** @todo mapping/locking hell; this isn't horribly efficient since
978	* pgmPhysPageLoadIntoTlb will repeat the lookup we've done here. */
979
980	Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage] ppv=%p\n", GCPhys, rc, pPage, ppv));
981	}
982	else
983	Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage]\n", GCPhys, rc, pPage));
984
985	/* else: handler catching all access, no pointer returned. */
986	}
987	else
988	rc = VERR_PGM_PHYS_TLB_UNASSIGNED;
989
990	PGM_UNLOCK(pVM);
991	return rc;
992	}
993
994
995
996	/*********************************************************************************************************************************
997	* RAM Range Management *
998	*********************************************************************************************************************************/
999
1000	/**
1001	* Given the range @a GCPhys thru @a GCPhysLast, find overlapping RAM range or
1002	* the correct insertion point.
1003	*
1004	* @returns Pointer to overlapping RAM range if found, NULL if not.
1005	* @param pVM The cross context VM structure.
1006	* @param GCPhys The address of the first byte in the range.
1007	* @param GCPhysLast The address of the last byte in the range.
1008	* @param pidxInsert Where to return the lookup table index to insert the
1009	* range at when returning NULL. Set to UINT32_MAX when
1010	* returning the pointer to an overlapping range.
1011	* @note Caller must own the PGM lock.
1012	*/
1013	static PPGMRAMRANGE pgmR3PhysRamRangeFindOverlapping(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast, uint32_t *pidxInsert)
1014	{
1015	PGM_LOCK_ASSERT_OWNER(pVM);
1016	uint32_t iStart = 0;
1017	uint32_t iEnd = pVM->pgm.s.RamRangeUnion.cLookupEntries;
1018	for (;;)
1019	{
1020	uint32_t idxLookup = iStart + (iEnd - iStart) / 2;
1021	RTGCPHYS const GCPhysEntryFirst = PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]);
1022	if (GCPhysLast < GCPhysEntryFirst)
1023	{
1024	if (idxLookup > iStart)
1025	iEnd = idxLookup;
1026	else
1027	{
1028	*pidxInsert = idxLookup;
1029	return NULL;
1030	}
1031	}
1032	else
1033	{
1034	RTGCPHYS const GCPhysEntryLast = pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast;
1035	if (GCPhys > GCPhysEntryLast)
1036	{
1037	idxLookup += 1;
1038	if (idxLookup < iEnd)
1039	iStart = idxLookup;
1040	else
1041	{
1042	*pidxInsert = idxLookup;
1043	return NULL;
1044	}
1045	}
1046	else
1047	{
1048	/* overlap */
1049	Assert(GCPhysEntryLast > GCPhys && GCPhysEntryFirst < GCPhysLast);
1050	*pidxInsert = UINT32_MAX;
1051	return pVM->pgm.s.apRamRanges[PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup])];
1052	}
1053	}
1054	}
1055	}
1056
1057
1058	/**
1059	* Given the range @a GCPhys thru @a GCPhysLast, find the lookup table entry
1060	* that's overlapping it.
1061	*
1062	* @returns The lookup table index of the overlapping entry, UINT32_MAX if not
1063	* found.
1064	* @param pVM The cross context VM structure.
1065	* @param GCPhys The address of the first byte in the range.
1066	* @param GCPhysLast The address of the last byte in the range.
1067	* @note Caller must own the PGM lock.
1068	*/
1069	static uint32_t pgmR3PhysRamRangeFindOverlappingIndex(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast)
1070	{
1071	PGM_LOCK_ASSERT_OWNER(pVM);
1072	uint32_t iStart = 0;
1073	uint32_t iEnd = pVM->pgm.s.RamRangeUnion.cLookupEntries;
1074	for (;;)
1075	{
1076	uint32_t idxLookup = iStart + (iEnd - iStart) / 2;
1077	RTGCPHYS const GCPhysEntryFirst = PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]);
1078	if (GCPhysLast < GCPhysEntryFirst)
1079	{
1080	if (idxLookup > iStart)
1081	iEnd = idxLookup;
1082	else
1083	return UINT32_MAX;
1084	}
1085	else
1086	{
1087	RTGCPHYS const GCPhysEntryLast = pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast;
1088	if (GCPhys > GCPhysEntryLast)
1089	{
1090	idxLookup += 1;
1091	if (idxLookup < iEnd)
1092	iStart = idxLookup;
1093	else
1094	return UINT32_MAX;
1095	}
1096	else
1097	{
1098	/* overlap */
1099	Assert(GCPhysEntryLast > GCPhys && GCPhysEntryFirst < GCPhysLast);
1100	return idxLookup;
1101	}
1102	}
1103	}
1104	}
1105
1106
1107	/**
1108	* Insert @a pRam into the lookup table.
1109	*
1110	* @returns VBox status code.
1111	* @param pVM The cross context VM structure.
1112	* @param pRam The RAM range to insert into the lookup table.
1113	* @param GCPhys The new mapping address to assign @a pRam on insertion.
1114	* @param pidxLookup Optional lookup table hint. This is updated.
1115	* @note Caller must own PGM lock.
1116	*/
1117	static int pgmR3PhysRamRangeInsertLookup(PVM pVM, PPGMRAMRANGE pRam, RTGCPHYS GCPhys, uint32_t *pidxLookup)
1118	{
1119	PGM_LOCK_ASSERT_OWNER(pVM);
1120	#ifdef DEBUG_bird
1121	pgmPhysAssertRamRangesLocked(pVM, false /fInUpdate/, true /fRamRelaxed/);
1122	#endif
1123	AssertMsg(pRam->pszDesc, ("%RGp-%RGp\n", pRam->GCPhys, pRam->GCPhysLast));
1124	AssertLogRelMsgReturn( pRam->GCPhys == NIL_RTGCPHYS
1125	&& pRam->GCPhysLast == NIL_RTGCPHYS,
1126	("GCPhys=%RGp; range: GCPhys=%RGp LB %RGp GCPhysLast=%RGp %s\n",
1127	GCPhys, pRam->GCPhys, pRam->cb, pRam->GCPhysLast, pRam->pszDesc),
1128	VERR_ALREADY_EXISTS);
1129	uint32_t const idRamRange = pRam->idRange;
1130	AssertReturn(pVM->pgm.s.apRamRanges[idRamRange] == pRam, VERR_INTERNAL_ERROR_2);
1131
1132	AssertReturn(!(GCPhys & GUEST_PAGE_OFFSET_MASK), VERR_INTERNAL_ERROR_3);
1133	RTGCPHYS const GCPhysLast = GCPhys + pRam->cb - 1U;
1134	AssertReturn(GCPhysLast > GCPhys, VERR_INTERNAL_ERROR_4);
1135	LogFlowFunc(("GCPhys=%RGp LB %RGp GCPhysLast=%RGp id=%#x %s\n", GCPhys, pRam->cb, GCPhysLast, idRamRange, pRam->pszDesc));
1136
1137	/*
1138	* Find the lookup table location if necessary.
1139	*/
1140	uint32_t const cLookupEntries = pVM->pgm.s.RamRangeUnion.cLookupEntries;
1141	AssertLogRelMsgReturn(cLookupEntries + 1 < RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup), /* id=0 is unused, so < is correct. */
1142	("%#x\n", cLookupEntries), VERR_INTERNAL_ERROR_3);
1143
1144	uint32_t idxLookup = pidxLookup ? *pidxLookup : UINT32_MAX;
1145	if (cLookupEntries == 0)
1146	idxLookup = 0; /* special case: empty table */
1147	else
1148	{
1149	if ( idxLookup > cLookupEntries
1150	\|\| ( idxLookup != 0
1151	&& pVM->pgm.s.aRamRangeLookup[idxLookup - 1].GCPhysLast >= GCPhys)
1152	\|\| ( idxLookup < cLookupEntries
1153	&& PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]) < GCPhysLast))
1154	{
1155	PPGMRAMRANGE pOverlapping = pgmR3PhysRamRangeFindOverlapping(pVM, GCPhys, GCPhysLast, &idxLookup);
1156	AssertLogRelMsgReturn(!pOverlapping,
1157	("GCPhys=%RGp; GCPhysLast=%RGp %s - overlaps %RGp...%RGp %s\n",
1158	GCPhys, GCPhysLast, pRam->pszDesc,
1159	pOverlapping->GCPhys, pOverlapping->GCPhysLast, pOverlapping->pszDesc),
1160	VERR_PGM_RAM_CONFLICT);
1161	AssertLogRelMsgReturn(idxLookup <= cLookupEntries, ("%#x vs %#x\n", idxLookup, cLookupEntries), VERR_INTERNAL_ERROR_5);
1162	}
1163	/* else we've got a good hint. */
1164	}
1165
1166	/*
1167	* Do the actual job.
1168	*
1169	* The moving of existing table entries is done in a way that allows other
1170	* EMTs to perform concurrent lookups with the updating.
1171	*/
1172	bool const fUseAtomic = pVM->enmVMState != VMSTATE_CREATING
1173	&& pVM->cCpus > 1
1174	#ifdef RT_ARCH_AMD64
1175	&& g_CpumHostFeatures.s.fCmpXchg16b
1176	#endif
1177	;
1178
1179	/* Signal that we're modifying the lookup table: */
1180	uint32_t const idGeneration = (pVM->pgm.s.RamRangeUnion.idGeneration + 1) \| 1; /* paranoia^3 */
1181	ASMAtomicWriteU32(&pVM->pgm.s.RamRangeUnion.idGeneration, idGeneration);
1182
1183	/* Update the RAM range entry. */
1184	pRam->GCPhys = GCPhys;
1185	pRam->GCPhysLast = GCPhysLast;
1186
1187	/* Do we need to shift any lookup table entries? */
1188	if (idxLookup != cLookupEntries)
1189	{
1190	/* We do. Make a copy of the final entry first. */
1191	uint32_t cToMove = cLookupEntries - idxLookup;
1192	PGMRAMRANGELOOKUPENTRY *pCur = &pVM->pgm.s.aRamRangeLookup[cLookupEntries];
1193	pCur->GCPhysFirstAndId = pCur[-1].GCPhysFirstAndId;
1194	pCur->GCPhysLast = pCur[-1].GCPhysLast;
1195
1196	/* Then increase the table size. This will ensure that anyone starting
1197	a search from here on should have consistent data. */
1198	ASMAtomicWriteU32(&pVM->pgm.s.RamRangeUnion.cLookupEntries, cLookupEntries + 1);
1199
1200	/* Transfer the rest of the entries. */
1201	cToMove -= 1;
1202	if (cToMove > 0)
1203	{
1204	if (!fUseAtomic)
1205	do
1206	{
1207	pCur -= 1;
1208	pCur->GCPhysFirstAndId = pCur[-1].GCPhysFirstAndId;
1209	pCur->GCPhysLast = pCur[-1].GCPhysLast;
1210	cToMove -= 1;
1211	} while (cToMove > 0);
1212	else
1213	{
1214	#if RTASM_HAVE_WRITE_U128 >= 2
1215	do
1216	{
1217	pCur -= 1;
1218	ASMAtomicWriteU128U(&pCur->u128Volatile, pCur[-1].u128Normal);
1219	cToMove -= 1;
1220	} while (cToMove > 0);
1221
1222	#else
1223	uint64_t u64PrevLo = pCur[-1].u128Normal.s.Lo;
1224	uint64_t u64PrevHi = pCur[-1].u128Normal.s.Hi;
1225	do
1226	{
1227	pCur -= 1;
1228	uint64_t const u64CurLo = pCur[-1].u128Normal.s.Lo;
1229	uint64_t const u64CurHi = pCur[-1].u128Normal.s.Hi;
1230	uint128_t uOldIgn;
1231	AssertStmt(ASMAtomicCmpXchgU128v2(&pCur->u128Volatile.u, u64CurHi, u64CurLo, u64PrevHi, u64PrevLo, &uOldIgn),
1232	(pCur->u128Volatile.s.Lo = u64CurLo, pCur->u128Volatile.s.Hi = u64CurHi));
1233	u64PrevLo = u64CurLo;
1234	u64PrevHi = u64CurHi;
1235	cToMove -= 1;
1236	} while (cToMove > 0);
1237	#endif
1238	}
1239	}
1240	}
1241
1242	/*
1243	* Write the new entry.
1244	*/
1245	PGMRAMRANGELOOKUPENTRY *pInsert = &pVM->pgm.s.aRamRangeLookup[idxLookup];
1246	if (!fUseAtomic)
1247	{
1248	pInsert->GCPhysFirstAndId = idRamRange \| GCPhys;
1249	pInsert->GCPhysLast = GCPhysLast;
1250	}
1251	else
1252	{
1253	PGMRAMRANGELOOKUPENTRY NewEntry;
1254	NewEntry.GCPhysFirstAndId = idRamRange \| GCPhys;
1255	NewEntry.GCPhysLast = GCPhysLast;
1256	ASMAtomicWriteU128v2(&pInsert->u128Volatile.u, NewEntry.u128Normal.s.Hi, NewEntry.u128Normal.s.Lo);
1257	}
1258
1259	/*
1260	* Update the generation and count in one go, signaling the end of the updating.
1261	*/
1262	PGM::PGMRAMRANGEGENANDLOOKUPCOUNT GenAndCount;
1263	GenAndCount.cLookupEntries = cLookupEntries + 1;
1264	GenAndCount.idGeneration = idGeneration + 1;
1265	ASMAtomicWriteU64(&pVM->pgm.s.RamRangeUnion.u64Combined, GenAndCount.u64Combined);
1266
1267	if (pidxLookup)
1268	*pidxLookup = idxLookup + 1;
1269
1270	#ifdef DEBUG_bird
1271	pgmPhysAssertRamRangesLocked(pVM, false /fInUpdate/, false /fRamRelaxed/);
1272	#endif
1273	return VINF_SUCCESS;
1274	}
1275
1276
1277	/**
1278	* Removes @a pRam from the lookup table.
1279	*
1280	* @returns VBox status code.
1281	* @param pVM The cross context VM structure.
1282	* @param pRam The RAM range to insert into the lookup table.
1283	* @param pidxLookup Optional lookup table hint. This is updated.
1284	* @note Caller must own PGM lock.
1285	*/
1286	static int pgmR3PhysRamRangeRemoveLookup(PVM pVM, PPGMRAMRANGE pRam, uint32_t *pidxLookup)
1287	{
1288	PGM_LOCK_ASSERT_OWNER(pVM);
1289	AssertMsg(pRam->pszDesc, ("%RGp-%RGp\n", pRam->GCPhys, pRam->GCPhysLast));
1290
1291	RTGCPHYS const GCPhys = pRam->GCPhys;
1292	RTGCPHYS const GCPhysLast = pRam->GCPhysLast;
1293	AssertLogRelMsgReturn( GCPhys != NIL_RTGCPHYS
1294	\|\| GCPhysLast != NIL_RTGCPHYS,
1295	("range: GCPhys=%RGp LB %RGp GCPhysLast=%RGp %s\n", GCPhys, pRam->cb, GCPhysLast, pRam->pszDesc),
1296	VERR_NOT_FOUND);
1297	AssertLogRelMsgReturn( GCPhys != NIL_RTGCPHYS
1298	&& GCPhysLast == GCPhys + pRam->cb - 1U
1299	&& (GCPhys & GUEST_PAGE_OFFSET_MASK) == 0
1300	&& (GCPhysLast & GUEST_PAGE_OFFSET_MASK) == GUEST_PAGE_OFFSET_MASK
1301	&& GCPhysLast > GCPhys,
1302	("range: GCPhys=%RGp LB %RGp GCPhysLast=%RGp %s\n", GCPhys, pRam->cb, GCPhysLast, pRam->pszDesc),
1303	VERR_INTERNAL_ERROR_5);
1304	uint32_t const idRamRange = pRam->idRange;
1305	AssertReturn(pVM->pgm.s.apRamRanges[idRamRange] == pRam, VERR_INTERNAL_ERROR_4);
1306	LogFlowFunc(("GCPhys=%RGp LB %RGp GCPhysLast=%RGp id=%#x %s\n", GCPhys, pRam->cb, GCPhysLast, idRamRange, pRam->pszDesc));
1307
1308	/*
1309	* Find the lookup table location.
1310	*/
1311	uint32_t const cLookupEntries = pVM->pgm.s.RamRangeUnion.cLookupEntries;
1312	AssertLogRelMsgReturn( cLookupEntries > 0
1313	&& cLookupEntries < RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup), /* id=0 is unused, so < is correct. */
1314	("%#x\n", cLookupEntries), VERR_INTERNAL_ERROR_3);
1315
1316	uint32_t idxLookup = pidxLookup ? *pidxLookup : UINT32_MAX;
1317	if ( idxLookup >= cLookupEntries
1318	\|\| pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast != GCPhysLast
1319	\|\| pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysFirstAndId != (GCPhys \| idRamRange))
1320	{
1321	uint32_t iStart = 0;
1322	uint32_t iEnd = cLookupEntries;
1323	for (;;)
1324	{
1325	idxLookup = iStart + (iEnd - iStart) / 2;
1326	RTGCPHYS const GCPhysEntryFirst = PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]);
1327	if (GCPhysLast < GCPhysEntryFirst)
1328	{
1329	AssertLogRelMsgReturn(idxLookup > iStart,
1330	("range: GCPhys=%RGp LB %RGp GCPhysLast=%RGp %s\n",
1331	GCPhys, pRam->cb, GCPhysLast, pRam->pszDesc),
1332	VERR_NOT_FOUND);
1333	iEnd = idxLookup;
1334	}
1335	else
1336	{
1337	RTGCPHYS const GCPhysEntryLast = pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast;
1338	if (GCPhys > GCPhysEntryLast)
1339	{
1340	idxLookup += 1;
1341	AssertLogRelMsgReturn(idxLookup < iEnd,
1342	("range: GCPhys=%RGp LB %RGp GCPhysLast=%RGp %s\n",
1343	GCPhys, pRam->cb, GCPhysLast, pRam->pszDesc),
1344	VERR_NOT_FOUND);
1345	iStart = idxLookup;
1346	}
1347	else
1348	{
1349	uint32_t const idEntry = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
1350	AssertLogRelMsgReturn( GCPhysEntryFirst == GCPhys
1351	&& GCPhysEntryLast == GCPhysLast
1352	&& idEntry == idRamRange,
1353	("Found: %RGp..%RGp id=%#x; Wanted: GCPhys=%RGp LB %RGp GCPhysLast=%RGp id=%#x %s\n",
1354	GCPhysEntryFirst, GCPhysEntryLast, idEntry,
1355	GCPhys, pRam->cb, GCPhysLast, pRam->idRange, pRam->pszDesc),
1356	VERR_NOT_FOUND);
1357	break;
1358	}
1359	}
1360	}
1361	}
1362	/* else we've got a good hint. */
1363
1364	/*
1365	* Do the actual job.
1366	*
1367	* The moving of existing table entries is done in a way that allows other
1368	* EMTs to perform concurrent lookups with the updating.
1369	*/
1370	bool const fUseAtomic = pVM->enmVMState != VMSTATE_CREATING
1371	&& pVM->cCpus > 1
1372	#ifdef RT_ARCH_AMD64
1373	&& g_CpumHostFeatures.s.fCmpXchg16b
1374	#endif
1375	;
1376
1377	/* Signal that we're modifying the lookup table: */
1378	uint32_t const idGeneration = (pVM->pgm.s.RamRangeUnion.idGeneration + 1) \| 1; /* paranoia^3 */
1379	ASMAtomicWriteU32(&pVM->pgm.s.RamRangeUnion.idGeneration, idGeneration);
1380
1381	/* Do we need to shift any lookup table entries? (This is a lot simpler
1382	than insertion.) */
1383	if (idxLookup + 1U < cLookupEntries)
1384	{
1385	uint32_t cToMove = cLookupEntries - idxLookup - 1U;
1386	PGMRAMRANGELOOKUPENTRY *pCur = &pVM->pgm.s.aRamRangeLookup[idxLookup];
1387	if (!fUseAtomic)
1388	do
1389	{
1390	pCur->GCPhysFirstAndId = pCur[1].GCPhysFirstAndId;
1391	pCur->GCPhysLast = pCur[1].GCPhysLast;
1392	pCur += 1;
1393	cToMove -= 1;
1394	} while (cToMove > 0);
1395	else
1396	{
1397	#if RTASM_HAVE_WRITE_U128 >= 2
1398	do
1399	{
1400	ASMAtomicWriteU128U(&pCur->u128Volatile, pCur[1].u128Normal);
1401	pCur += 1;
1402	cToMove -= 1;
1403	} while (cToMove > 0);
1404
1405	#else
1406	uint64_t u64PrevLo = pCur->u128Normal.s.Lo;
1407	uint64_t u64PrevHi = pCur->u128Normal.s.Hi;
1408	do
1409	{
1410	uint64_t const u64CurLo = pCur[1].u128Normal.s.Lo;
1411	uint64_t const u64CurHi = pCur[1].u128Normal.s.Hi;
1412	uint128_t uOldIgn;
1413	AssertStmt(ASMAtomicCmpXchgU128v2(&pCur->u128Volatile.u, u64CurHi, u64CurLo, u64PrevHi, u64PrevLo, &uOldIgn),
1414	(pCur->u128Volatile.s.Lo = u64CurLo, pCur->u128Volatile.s.Hi = u64CurHi));
1415	u64PrevLo = u64CurLo;
1416	u64PrevHi = u64CurHi;
1417	pCur += 1;
1418	cToMove -= 1;
1419	} while (cToMove > 0);
1420	#endif
1421	}
1422	}
1423
1424	/* Update the RAM range entry to indicate that it is no longer mapped.
1425	The GCPhys member is accessed by the lockless TLB lookup code, so update
1426	it last and atomically to be on the safe side. */
1427	pRam->GCPhysLast = NIL_RTGCPHYS;
1428	ASMAtomicWriteU64(&pRam->GCPhys, NIL_RTGCPHYS);
1429
1430	/*
1431	* Update the generation and count in one go, signaling the end of the updating.
1432	*/
1433	PGM::PGMRAMRANGEGENANDLOOKUPCOUNT GenAndCount;
1434	GenAndCount.cLookupEntries = cLookupEntries - 1;
1435	GenAndCount.idGeneration = idGeneration + 1;
1436	ASMAtomicWriteU64(&pVM->pgm.s.RamRangeUnion.u64Combined, GenAndCount.u64Combined);
1437
1438	if (pidxLookup)
1439	*pidxLookup = idxLookup + 1;
1440
1441	return VINF_SUCCESS;
1442	}
1443
1444
1445	/**
1446	* Gets the number of ram ranges.
1447	*
1448	* @returns Number of ram ranges. Returns UINT32_MAX if @a pVM is invalid.
1449	* @param pVM The cross context VM structure.
1450	*/
1451	VMMR3DECL(uint32_t) PGMR3PhysGetRamRangeCount(PVM pVM)
1452	{
1453	VM_ASSERT_VALID_EXT_RETURN(pVM, UINT32_MAX);
1454
1455	PGM_LOCK_VOID(pVM);
1456	uint32_t const cRamRanges = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
1457	PGM_UNLOCK(pVM);
1458	return cRamRanges;
1459	}
1460
1461
1462	/**
1463	* Get information about a range.
1464	*
1465	* @returns VINF_SUCCESS or VERR_OUT_OF_RANGE.
1466	* @param pVM The cross context VM structure.
1467	* @param iRange The ordinal of the range.
1468	* @param pGCPhysStart Where to return the start of the range. Optional.
1469	* @param pGCPhysLast Where to return the address of the last byte in the
1470	* range. Optional.
1471	* @param ppszDesc Where to return the range description. Optional.
1472	* @param pfIsMmio Where to indicate that this is a pure MMIO range.
1473	* Optional.
1474	*/
1475	VMMR3DECL(int) PGMR3PhysGetRange(PVM pVM, uint32_t iRange, PRTGCPHYS pGCPhysStart, PRTGCPHYS pGCPhysLast,
1476	const char *ppszDesc, bool pfIsMmio)
1477	{
1478	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
1479
1480	PGM_LOCK_VOID(pVM);
1481	uint32_t const cLookupEntries = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
1482	if (iRange < cLookupEntries)
1483	{
1484	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[iRange]);
1485	Assert(idRamRange && idRamRange <= pVM->pgm.s.idRamRangeMax);
1486	PGMRAMRANGE const * const pRamRange = pVM->pgm.s.apRamRanges[idRamRange];
1487	AssertPtr(pRamRange);
1488
1489	if (pGCPhysStart)
1490	*pGCPhysStart = pRamRange->GCPhys;
1491	if (pGCPhysLast)
1492	*pGCPhysLast = pRamRange->GCPhysLast;
1493	if (ppszDesc)
1494	*ppszDesc = pRamRange->pszDesc;
1495	if (pfIsMmio)
1496	*pfIsMmio = !!(pRamRange->fFlags & PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO);
1497
1498	PGM_UNLOCK(pVM);
1499	return VINF_SUCCESS;
1500	}
1501	PGM_UNLOCK(pVM);
1502	return VERR_OUT_OF_RANGE;
1503	}
1504
1505
1506	/**
1507	* Gets RAM ranges that are supposed to be zero'ed at boot.
1508	*
1509	* This function gets all RAM ranges that are not ad hoc (ROM, MMIO, MMIO2) memory.
1510	* The RAM hole (if any) is -NOT- included because we don't return 0s when it is
1511	* read anyway.
1512	*
1513	* @returns VBox status code.
1514	* @param pVM The cross context VM structure.
1515	* @param pRanges Where to store the physical RAM ranges.
1516	* @param cMaxRanges The maximum ranges that can be stored.
1517	*/
1518	VMMR3_INT_DECL(int) PGMR3PhysGetRamBootZeroedRanges(PVM pVM, PPGMPHYSRANGES pRanges, uint32_t cMaxRanges)
1519	{
1520	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
1521	AssertPtrReturn(pRanges, VERR_INVALID_PARAMETER);
1522	AssertReturn(cMaxRanges > 0, VERR_INVALID_PARAMETER);
1523
1524	int rc = VINF_SUCCESS;
1525	uint32_t idxRange = 0;
1526	PGM_LOCK_VOID(pVM);
1527
1528	/*
1529	* The primary purpose of this API is the GIM Hyper-V hypercall which recommends (not
1530	* requires) that the largest ranges are reported earlier. Therefore, here we iterate
1531	* the ranges in reverse because in PGM the largest range is generally at the end.
1532	*/
1533	uint32_t const cLookupEntries = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
1534	for (int32_t idxLookup = cLookupEntries - 1; idxLookup >= 0; idxLookup--)
1535	{
1536	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
1537	Assert(idRamRange < RT_ELEMENTS(pVM->pgm.s.apRamRanges));
1538	PPGMRAMRANGE const pCur = pVM->pgm.s.apRamRanges[idRamRange];
1539	AssertContinue(pCur);
1540
1541	if (!PGM_RAM_RANGE_IS_AD_HOC(pCur))
1542	{
1543	if (idxRange < cMaxRanges)
1544	{
1545	/* Combine with previous range if it is contiguous, otherwise add it as a new range. */
1546	if ( idxRange > 0
1547	&& pRanges->aRanges[idxRange - 1].GCPhysStart == pCur->GCPhysLast + 1U)
1548	{
1549	pRanges->aRanges[idxRange - 1].GCPhysStart = pCur->GCPhys;
1550	pRanges->aRanges[idxRange - 1].cPages += (pCur->cb >> GUEST_PAGE_SHIFT);
1551	}
1552	else
1553	{
1554	pRanges->aRanges[idxRange].GCPhysStart = pCur->GCPhys;
1555	pRanges->aRanges[idxRange].cPages = pCur->cb >> GUEST_PAGE_SHIFT;
1556	++idxRange;
1557	}
1558	}
1559	else
1560	{
1561	rc = VERR_BUFFER_OVERFLOW;
1562	break;
1563	}
1564	}
1565	}
1566	pRanges->cRanges = idxRange;
1567	PGM_UNLOCK(pVM);
1568	return rc;
1569	}
1570
1571
1572	/*********************************************************************************************************************************
1573	* RAM *
1574	*********************************************************************************************************************************/
1575
1576	/**
1577	* Frees the specified RAM page and replaces it with the ZERO page.
1578	*
1579	* This is used by ballooning, remapping MMIO2, RAM reset and state loading.
1580	*
1581	* @param pVM The cross context VM structure.
1582	* @param pReq Pointer to the request. This is NULL when doing a
1583	* bulk free in NEM memory mode.
1584	* @param pcPendingPages Where the number of pages waiting to be freed are
1585	* kept. This will normally be incremented. This is
1586	* NULL when doing a bulk free in NEM memory mode.
1587	* @param pPage Pointer to the page structure.
1588	* @param GCPhys The guest physical address of the page, if applicable.
1589	* @param enmNewType New page type for NEM notification, since several
1590	* callers will change the type upon successful return.
1591	*
1592	* @remarks The caller must own the PGM lock.
1593	*/
1594	int pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys,
1595	PGMPAGETYPE enmNewType)
1596	{
1597	/*
1598	* Assert sanity.
1599	*/
1600	PGM_LOCK_ASSERT_OWNER(pVM);
1601	if (RT_UNLIKELY( PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
1602	&& PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_ROM_SHADOW))
1603	{
1604	AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
1605	return VMSetError(pVM, VERR_PGM_PHYS_NOT_RAM, RT_SRC_POS, "GCPhys=%RGp type=%d", GCPhys, PGM_PAGE_GET_TYPE(pPage));
1606	}
1607
1608	/** @todo What about ballooning of large pages??! */
1609	Assert( PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE
1610	&& PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE_DISABLED);
1611
1612	if ( PGM_PAGE_IS_ZERO(pPage)
1613	\|\| PGM_PAGE_IS_BALLOONED(pPage))
1614	return VINF_SUCCESS;
1615
1616	const uint32_t idPage = PGM_PAGE_GET_PAGEID(pPage);
1617	Log3(("pgmPhysFreePage: idPage=%#x GCPhys=%RGp pPage=%R[pgmpage]\n", idPage, GCPhys, pPage));
1618	if (RT_UNLIKELY(!PGM_IS_IN_NEM_MODE(pVM)
1619	? idPage == NIL_GMM_PAGEID
1620	\|\| idPage > GMM_PAGEID_LAST
1621	\|\| PGM_PAGE_GET_CHUNKID(pPage) == NIL_GMM_CHUNKID
1622	: idPage != NIL_GMM_PAGEID))
1623	{
1624	AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
1625	return VMSetError(pVM, VERR_PGM_PHYS_INVALID_PAGE_ID, RT_SRC_POS, "GCPhys=%RGp idPage=%#x", GCPhys, pPage);
1626	}
1627	#ifdef VBOX_WITH_NATIVE_NEM
1628	const RTHCPHYS HCPhysPrev = PGM_PAGE_GET_HCPHYS(pPage);
1629	#endif
1630
1631	/* update page count stats. */
1632	if (PGM_PAGE_IS_SHARED(pPage))
1633	pVM->pgm.s.cSharedPages--;
1634	else
1635	pVM->pgm.s.cPrivatePages--;
1636	pVM->pgm.s.cZeroPages++;
1637
1638	/* Deal with write monitored pages. */
1639	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
1640	{
1641	PGM_PAGE_SET_WRITTEN_TO(pVM, pPage);
1642	pVM->pgm.s.cWrittenToPages++;
1643	}
1644	PGM_PAGE_CLEAR_CODE_PAGE(pVM, pPage); /* No callback needed, IEMTlbInvalidateAllPhysicalAllCpus is called below. */
1645
1646	/*
1647	* pPage = ZERO page.
1648	*/
1649	PGM_PAGE_SET_HCPHYS(pVM, pPage, pVM->pgm.s.HCPhysZeroPg);
1650	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
1651	PGM_PAGE_SET_PAGEID(pVM, pPage, NIL_GMM_PAGEID);
1652	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
1653	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
1654	PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
1655
1656	/* Flush physical page map TLB entry. */
1657	pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
1658	IEMTlbInvalidateAllPhysicalAllCpus(pVM, NIL_VMCPUID, IEMTLBPHYSFLUSHREASON_FREED); /// @todo move to the perform step.
1659
1660	#ifdef VBOX_WITH_PGM_NEM_MODE
1661	/*
1662	* Skip the rest if we're doing a bulk free in NEM memory mode.
1663	*/
1664	if (!pReq)
1665	return VINF_SUCCESS;
1666	AssertLogRelReturn(!pVM->pgm.s.fNemMode, VERR_PGM_NOT_SUPPORTED_FOR_NEM_MODE);
1667	#endif
1668
1669	#ifdef VBOX_WITH_NATIVE_NEM
1670	/* Notify NEM. */
1671	/** @todo Remove this one? */
1672	if (VM_IS_NEM_ENABLED(pVM))
1673	{
1674	uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage);
1675	NEMHCNotifyPhysPageChanged(pVM, GCPhys, HCPhysPrev, pVM->pgm.s.HCPhysZeroPg, pVM->pgm.s.abZeroPg,
1676	pgmPhysPageCalcNemProtection(pPage, enmNewType), enmNewType, &u2State);
1677	PGM_PAGE_SET_NEM_STATE(pPage, u2State);
1678	}
1679	#else
1680	RT_NOREF(enmNewType);
1681	#endif
1682
1683	/*
1684	* Make sure it's not in the handy page array.
1685	*/
1686	for (uint32_t i = pVM->pgm.s.cHandyPages; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
1687	{
1688	if (pVM->pgm.s.aHandyPages[i].idPage == idPage)
1689	{
1690	pVM->pgm.s.aHandyPages[i].HCPhysGCPhys = NIL_GMMPAGEDESC_PHYS;
1691	pVM->pgm.s.aHandyPages[i].fZeroed = false;
1692	pVM->pgm.s.aHandyPages[i].idPage = NIL_GMM_PAGEID;
1693	break;
1694	}
1695	if (pVM->pgm.s.aHandyPages[i].idSharedPage == idPage)
1696	{
1697	pVM->pgm.s.aHandyPages[i].idSharedPage = NIL_GMM_PAGEID;
1698	break;
1699	}
1700	}
1701
1702	/*
1703	* Push it onto the page array.
1704	*/
1705	uint32_t iPage = *pcPendingPages;
1706	Assert(iPage < PGMPHYS_FREE_PAGE_BATCH_SIZE);
1707	*pcPendingPages += 1;
1708
1709	pReq->aPages[iPage].idPage = idPage;
1710
1711	if (iPage + 1 < PGMPHYS_FREE_PAGE_BATCH_SIZE)
1712	return VINF_SUCCESS;
1713
1714	/*
1715	* Flush the pages.
1716	*/
1717	int rc = GMMR3FreePagesPerform(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE);
1718	if (RT_SUCCESS(rc))
1719	{
1720	GMMR3FreePagesRePrep(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
1721	*pcPendingPages = 0;
1722	}
1723	return rc;
1724	}
1725
1726
1727	/**
1728	* Frees a range of pages, replacing them with ZERO pages of the specified type.
1729	*
1730	* @returns VBox status code.
1731	* @param pVM The cross context VM structure.
1732	* @param pRam The RAM range in which the pages resides.
1733	* @param GCPhys The address of the first page.
1734	* @param GCPhysLast The address of the last page.
1735	* @param pvMmio2 Pointer to the ring-3 mapping of any MMIO2 memory that
1736	* will replace the pages we're freeing up.
1737	*/
1738	static int pgmR3PhysFreePageRange(PVM pVM, PPGMRAMRANGE pRam, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast, void *pvMmio2)
1739	{
1740	PGM_LOCK_ASSERT_OWNER(pVM);
1741
1742	#ifdef VBOX_WITH_PGM_NEM_MODE
1743	/*
1744	* In simplified memory mode we don't actually free the memory,
1745	* we just unmap it and let NEM do any unlocking of it.
1746	*/
1747	if (pVM->pgm.s.fNemMode)
1748	{
1749	Assert(VM_IS_NEM_ENABLED(pVM) \|\| VM_IS_EXEC_ENGINE_IEM(pVM));
1750	uint8_t u2State = 0; /* (We don't support UINT8_MAX here.) */
1751	if (VM_IS_NEM_ENABLED(pVM))
1752	{
1753	uint32_t const fNemNotify = (pvMmio2 ? NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2 : 0) \| NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE;
1754	int rc = NEMR3NotifyPhysMmioExMapEarly(pVM, GCPhys, GCPhysLast - GCPhys + 1, fNemNotify,
1755	pRam->pbR3 ? pRam->pbR3 + GCPhys - pRam->GCPhys : NULL,
1756	pvMmio2, &u2State, NULL /puNemRange/);
1757	AssertLogRelRCReturn(rc, rc);
1758	}
1759
1760	/* Iterate the pages. */
1761	PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT];
1762	uint32_t cPagesLeft = ((GCPhysLast - GCPhys) >> GUEST_PAGE_SHIFT) + 1;
1763	while (cPagesLeft-- > 0)
1764	{
1765	int rc = pgmPhysFreePage(pVM, NULL, NULL, pPageDst, GCPhys, PGMPAGETYPE_MMIO);
1766	AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
1767
1768	PGM_PAGE_SET_TYPE(pVM, pPageDst, PGMPAGETYPE_MMIO);
1769	PGM_PAGE_SET_NEM_STATE(pPageDst, u2State);
1770
1771	GCPhys += GUEST_PAGE_SIZE;
1772	pPageDst++;
1773	}
1774	return VINF_SUCCESS;
1775	}
1776	#else /* !VBOX_WITH_PGM_NEM_MODE */
1777	RT_NOREF(pvMmio2);
1778	#endif /* !VBOX_WITH_PGM_NEM_MODE */
1779
1780	/*
1781	* Regular mode.
1782	*/
1783	/* Prepare. */
1784	uint32_t cPendingPages = 0;
1785	PGMMFREEPAGESREQ pReq;
1786	int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
1787	AssertLogRelRCReturn(rc, rc);
1788
1789	#ifdef VBOX_WITH_NATIVE_NEM
1790	/* Tell NEM up-front. */
1791	uint8_t u2State = UINT8_MAX;
1792	if (VM_IS_NEM_ENABLED(pVM))
1793	{
1794	uint32_t const fNemNotify = (pvMmio2 ? NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2 : 0) \| NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE;
1795	rc = NEMR3NotifyPhysMmioExMapEarly(pVM, GCPhys, GCPhysLast - GCPhys + 1, fNemNotify, NULL, pvMmio2,
1796	&u2State, NULL /puNemRange/);
1797	AssertLogRelRCReturnStmt(rc, GMMR3FreePagesCleanup(pReq), rc);
1798	}
1799	#endif
1800
1801	/* Iterate the pages. */
1802	PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT];
1803	uint32_t cPagesLeft = ((GCPhysLast - GCPhys) >> GUEST_PAGE_SHIFT) + 1;
1804	while (cPagesLeft-- > 0)
1805	{
1806	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys, PGMPAGETYPE_MMIO);
1807	AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
1808
1809	PGM_PAGE_SET_TYPE(pVM, pPageDst, PGMPAGETYPE_MMIO);
1810	#ifdef VBOX_WITH_NATIVE_NEM
1811	if (u2State != UINT8_MAX)
1812	PGM_PAGE_SET_NEM_STATE(pPageDst, u2State);
1813	#endif
1814
1815	GCPhys += GUEST_PAGE_SIZE;
1816	pPageDst++;
1817	}
1818
1819	/* Finish pending and cleanup. */
1820	if (cPendingPages)
1821	{
1822	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
1823	AssertLogRelRCReturn(rc, rc);
1824	}
1825	GMMR3FreePagesCleanup(pReq);
1826
1827	return rc;
1828	}
1829
1830
1831	/**
1832	* Wrapper around VMMR0_DO_PGM_PHYS_ALLOCATE_RAM_RANGE.
1833	*/
1834	static int pgmR3PhysAllocateRamRange(PVM pVM, PVMCPU pVCpu, uint32_t cGuestPages, uint32_t fFlags, PPGMRAMRANGE *ppRamRange)
1835	{
1836	int rc;
1837	PGMPHYSALLOCATERAMRANGEREQ AllocRangeReq;
1838	AllocRangeReq.idNewRange = UINT32_MAX / 4;
1839	if (SUPR3IsDriverless())
1840	rc = pgmPhysRamRangeAllocCommon(pVM, cGuestPages, fFlags, &AllocRangeReq.idNewRange);
1841	else
1842	{
1843	AllocRangeReq.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
1844	AllocRangeReq.Hdr.cbReq = sizeof(AllocRangeReq);
1845	AllocRangeReq.cbGuestPage = GUEST_PAGE_SIZE;
1846	AllocRangeReq.cGuestPages = cGuestPages;
1847	AllocRangeReq.fFlags = fFlags;
1848	rc = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_PGM_PHYS_ALLOCATE_RAM_RANGE, 0 /u64Arg/, &AllocRangeReq.Hdr);
1849	}
1850	if (RT_SUCCESS(rc))
1851	{
1852	Assert(AllocRangeReq.idNewRange != 0);
1853	Assert(AllocRangeReq.idNewRange < RT_ELEMENTS(pVM->pgm.s.apRamRanges));
1854	AssertPtr(pVM->pgm.s.apRamRanges[AllocRangeReq.idNewRange]);
1855	*ppRamRange = pVM->pgm.s.apRamRanges[AllocRangeReq.idNewRange];
1856	return VINF_SUCCESS;
1857	}
1858
1859	*ppRamRange = NULL;
1860	return rc;
1861	}
1862
1863
1864	/**
1865	* PGMR3PhysRegisterRam worker that initializes and links a RAM range.
1866	*
1867	* In NEM mode, this will allocate the pages backing the RAM range and this may
1868	* fail. NEM registration may also fail. (In regular HM mode it won't fail.)
1869	*
1870	* @returns VBox status code.
1871	* @param pVM The cross context VM structure.
1872	* @param pNew The new RAM range.
1873	* @param GCPhys The address of the RAM range.
1874	* @param GCPhysLast The last address of the RAM range.
1875	* @param pszDesc The description.
1876	* @param pidxLookup The lookup table insertion point.
1877	*/
1878	static int pgmR3PhysInitAndLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast,
1879	const char pszDesc, uint32_t pidxLookup)
1880	{
1881	/*
1882	* Initialize the range.
1883	*/
1884	Assert(pNew->cb == GCPhysLast - GCPhys + 1U); RT_NOREF(GCPhysLast);
1885	pNew->pszDesc = pszDesc;
1886	pNew->uNemRange = UINT32_MAX;
1887	pNew->pbR3 = NULL;
1888	pNew->paLSPages = NULL;
1889
1890	uint32_t const cPages = pNew->cb >> GUEST_PAGE_SHIFT;
1891	#ifdef VBOX_WITH_PGM_NEM_MODE
1892	if (!pVM->pgm.s.fNemMode)
1893	#endif
1894	{
1895	RTGCPHYS iPage = cPages;
1896	while (iPage-- > 0)
1897	PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_RAM);
1898
1899	/* Update the page count stats. */
1900	pVM->pgm.s.cZeroPages += cPages;
1901	pVM->pgm.s.cAllPages += cPages;
1902	}
1903	#ifdef VBOX_WITH_PGM_NEM_MODE
1904	else
1905	{
1906	int rc = SUPR3PageAlloc(RT_ALIGN_Z(pNew->cb, HOST_PAGE_SIZE) >> HOST_PAGE_SHIFT,
1907	pVM->pgm.s.fUseLargePages ? SUP_PAGE_ALLOC_F_LARGE_PAGES : 0, (void **)&pNew->pbR3);
1908	if (RT_FAILURE(rc))
1909	return rc;
1910
1911	RTGCPHYS iPage = cPages;
1912	while (iPage-- > 0)
1913	PGM_PAGE_INIT(&pNew->aPages[iPage], UINT64_C(0x0000fffffffff000), NIL_GMM_PAGEID,
1914	PGMPAGETYPE_RAM, PGM_PAGE_STATE_ALLOCATED);
1915
1916	/* Update the page count stats. */
1917	pVM->pgm.s.cPrivatePages += cPages;
1918	pVM->pgm.s.cAllPages += cPages;
1919	}
1920	#endif
1921
1922	/*
1923	* Insert it into the lookup table.
1924	*/
1925	int rc = pgmR3PhysRamRangeInsertLookup(pVM, pNew, GCPhys, pidxLookup);
1926	AssertRCReturn(rc, rc);
1927
1928	#ifdef VBOX_WITH_NATIVE_NEM
1929	/*
1930	* Notify NEM now that it has been linked.
1931	*
1932	* As above, it is assumed that on failure the VM creation will fail, so
1933	* no extra cleanup is needed here.
1934	*/
1935	if (VM_IS_NEM_ENABLED(pVM))
1936	{
1937	uint8_t u2State = UINT8_MAX;
1938	rc = NEMR3NotifyPhysRamRegister(pVM, GCPhys, pNew->cb, pNew->pbR3, &u2State, &pNew->uNemRange);
1939	if (RT_SUCCESS(rc) && u2State != UINT8_MAX)
1940	pgmPhysSetNemStateForPages(&pNew->aPages[0], cPages, u2State);
1941	return rc;
1942	}
1943	#endif
1944	return VINF_SUCCESS;
1945	}
1946
1947
1948	/**
1949	* Worker for PGMR3PhysRegisterRam called with the PGM lock.
1950	*
1951	* The caller releases the lock.
1952	*/
1953	static int pgmR3PhysRegisterRamWorker(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, RTGCPHYS cb, const char *pszDesc,
1954	uint32_t const cRamRanges, RTGCPHYS const GCPhysLast)
1955	{
1956	#ifdef VBOX_STRICT
1957	pgmPhysAssertRamRangesLocked(pVM, false /fInUpdate/, false /fRamRelaxed/);
1958	#endif
1959
1960	/*
1961	* Check that we've got enough free RAM ranges.
1962	*/
1963	AssertLogRelMsgReturn((uint64_t)pVM->pgm.s.idRamRangeMax + cRamRanges + 1 <= RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup),
1964	("idRamRangeMax=%#RX32 vs GCPhys=%RGp cb=%RGp / %#RX32 ranges (%s)\n",
1965	pVM->pgm.s.idRamRangeMax, GCPhys, cb, cRamRanges, pszDesc),
1966	VERR_PGM_TOO_MANY_RAM_RANGES);
1967
1968	/*
1969	* Check for conflicts via the lookup table. We search it backwards,
1970	* assuming that memory is added in ascending order by address.
1971	*/
1972	uint32_t idxLookup = pVM->pgm.s.RamRangeUnion.cLookupEntries;
1973	while (idxLookup)
1974	{
1975	if (GCPhys > pVM->pgm.s.aRamRangeLookup[idxLookup - 1].GCPhysLast)
1976	break;
1977	idxLookup--;
1978	RTGCPHYS const GCPhysCur = PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]);
1979	AssertLogRelMsgReturn( GCPhysLast < GCPhysCur
1980	\|\| GCPhys > pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast,
1981	("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
1982	GCPhys, GCPhysLast, pszDesc, GCPhysCur, pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast,
1983	pVM->pgm.s.apRamRanges[PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup])]->pszDesc),
1984	VERR_PGM_RAM_CONFLICT);
1985	}
1986
1987	/*
1988	* Register it with GMM (the API bitches).
1989	*/
1990	const RTGCPHYS cPages = cb >> GUEST_PAGE_SHIFT;
1991	int rc = MMR3IncreaseBaseReservation(pVM, cPages);
1992	if (RT_FAILURE(rc))
1993	return rc;
1994
1995	/*
1996	* Create the required chunks.
1997	*/
1998	RTGCPHYS cPagesLeft = cPages;
1999	RTGCPHYS GCPhysChunk = GCPhys;
2000	uint32_t idxChunk = 0;
2001	while (cPagesLeft > 0)
2002	{
2003	uint32_t cPagesInChunk = cPagesLeft;
2004	if (cPagesInChunk > PGM_MAX_PAGES_PER_RAM_RANGE)
2005	cPagesInChunk = PGM_MAX_PAGES_PER_RAM_RANGE;
2006
2007	const char *pszDescChunk = idxChunk == 0
2008	? pszDesc
2009	: MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s (#%u)", pszDesc, idxChunk + 1);
2010	AssertReturn(pszDescChunk, VERR_NO_MEMORY);
2011
2012	/*
2013	* Allocate a RAM range.
2014	*/
2015	PPGMRAMRANGE pNew = NULL;
2016	rc = pgmR3PhysAllocateRamRange(pVM, pVCpu, cPagesInChunk, 0 /fFlags/, &pNew);
2017	AssertLogRelMsgReturn(RT_SUCCESS(rc),
2018	("pgmR3PhysAllocateRamRange failed: GCPhysChunk=%RGp cPagesInChunk=%#RX32 (%s): %Rrc\n",
2019	GCPhysChunk, cPagesInChunk, pszDescChunk, rc),
2020	rc);
2021
2022	/*
2023	* Ok, init and link the range.
2024	*/
2025	rc = pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhysChunk,
2026	GCPhysChunk + ((RTGCPHYS)cPagesInChunk << GUEST_PAGE_SHIFT) - 1U,
2027	pszDescChunk, &idxLookup);
2028	AssertLogRelMsgReturn(RT_SUCCESS(rc),
2029	("pgmR3PhysInitAndLinkRamRange failed: GCPhysChunk=%RGp cPagesInChunk=%#RX32 (%s): %Rrc\n",
2030	GCPhysChunk, cPagesInChunk, pszDescChunk, rc),
2031	rc);
2032
2033	/* advance */
2034	GCPhysChunk += (RTGCPHYS)cPagesInChunk << GUEST_PAGE_SHIFT;
2035	cPagesLeft -= cPagesInChunk;
2036	idxChunk++;
2037	}
2038
2039	return rc;
2040	}
2041
2042
2043	/**
2044	* Sets up a range RAM.
2045	*
2046	* This will check for conflicting registrations, make a resource reservation
2047	* for the memory (with GMM), and setup the per-page tracking structures
2048	* (PGMPAGE).
2049	*
2050	* @returns VBox status code.
2051	* @param pVM The cross context VM structure.
2052	* @param GCPhys The physical address of the RAM.
2053	* @param cb The size of the RAM.
2054	* @param pszDesc The description - not copied, so, don't free or change it.
2055	*/
2056	VMMR3DECL(int) PGMR3PhysRegisterRam(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, const char *pszDesc)
2057	{
2058	/*
2059	* Validate input.
2060	*/
2061	Log(("PGMR3PhysRegisterRam: GCPhys=%RGp cb=%RGp pszDesc=%s\n", GCPhys, cb, pszDesc));
2062	AssertReturn(RT_ALIGN_T(GCPhys, GUEST_PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
2063	AssertReturn(RT_ALIGN_T(cb, GUEST_PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
2064	AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
2065	RTGCPHYS const GCPhysLast = GCPhys + (cb - 1);
2066	AssertMsgReturn(GCPhysLast > GCPhys, ("The range wraps! GCPhys=%RGp cb=%RGp\n", GCPhys, cb), VERR_INVALID_PARAMETER);
2067	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
2068	PVMCPU const pVCpu = VMMGetCpu(pVM);
2069	AssertReturn(pVCpu, VERR_VM_THREAD_NOT_EMT);
2070	AssertReturn(pVCpu->idCpu == 0, VERR_VM_THREAD_NOT_EMT);
2071
2072	/*
2073	* Calculate the number of RAM ranges required.
2074	* See also pgmPhysMmio2CalcChunkCount.
2075	*/
2076	uint32_t const cPagesPerChunk = PGM_MAX_PAGES_PER_RAM_RANGE;
2077	uint32_t const cRamRanges = (uint32_t)(((cb >> GUEST_PAGE_SHIFT) + cPagesPerChunk - 1) / cPagesPerChunk);
2078	AssertLogRelMsgReturn(cRamRanges * (RTGCPHYS)cPagesPerChunk * GUEST_PAGE_SIZE >= cb,
2079	("cb=%RGp cRamRanges=%#RX32 cPagesPerChunk=%#RX32\n", cb, cRamRanges, cPagesPerChunk),
2080	VERR_OUT_OF_RANGE);
2081
2082	PGM_LOCK_VOID(pVM);
2083
2084	int rc = pgmR3PhysRegisterRamWorker(pVM, pVCpu, GCPhys, cb, pszDesc, cRamRanges, GCPhysLast);
2085	#ifdef VBOX_STRICT
2086	pgmPhysAssertRamRangesLocked(pVM, false /fInUpdate/, false /fRamRelaxed/);
2087	#endif
2088
2089	PGM_UNLOCK(pVM);
2090	return rc;
2091	}
2092
2093
2094	/**
2095	* Worker called by PGMR3InitFinalize if we're configured to pre-allocate RAM.
2096	*
2097	* We do this late in the init process so that all the ROM and MMIO ranges have
2098	* been registered already and we don't go wasting memory on them.
2099	*
2100	* @returns VBox status code.
2101	*
2102	* @param pVM The cross context VM structure.
2103	*/
2104	int pgmR3PhysRamPreAllocate(PVM pVM)
2105	{
2106	Assert(pVM->pgm.s.fRamPreAlloc);
2107	Log(("pgmR3PhysRamPreAllocate: enter\n"));
2108	#ifdef VBOX_WITH_PGM_NEM_MODE
2109	AssertLogRelReturn(!pVM->pgm.s.fNemMode, VERR_PGM_NOT_SUPPORTED_FOR_NEM_MODE);
2110	#endif
2111
2112	/*
2113	* Walk the RAM ranges and allocate all RAM pages, halt at
2114	* the first allocation error.
2115	*/
2116	uint64_t cPages = 0;
2117	uint64_t NanoTS = RTTimeNanoTS();
2118	PGM_LOCK_VOID(pVM);
2119	uint32_t const cLookupEntries = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
2120	for (uint32_t idxLookup = 0; idxLookup < cLookupEntries; idxLookup++)
2121	{
2122	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
2123	AssertContinue(idRamRange < RT_ELEMENTS(pVM->pgm.s.apRamRanges));
2124	PPGMRAMRANGE const pRam = pVM->pgm.s.apRamRanges[idRamRange];
2125	AssertContinue(pRam);
2126
2127	PPGMPAGE pPage = &pRam->aPages[0];
2128	RTGCPHYS GCPhys = pRam->GCPhys;
2129	uint32_t cLeft = pRam->cb >> GUEST_PAGE_SHIFT;
2130	while (cLeft-- > 0)
2131	{
2132	if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
2133	{
2134	switch (PGM_PAGE_GET_STATE(pPage))
2135	{
2136	case PGM_PAGE_STATE_ZERO:
2137	{
2138	int rc = pgmPhysAllocPage(pVM, pPage, GCPhys);
2139	if (RT_FAILURE(rc))
2140	{
2141	LogRel(("PGM: RAM Pre-allocation failed at %RGp (in %s) with rc=%Rrc\n", GCPhys, pRam->pszDesc, rc));
2142	PGM_UNLOCK(pVM);
2143	return rc;
2144	}
2145	cPages++;
2146	break;
2147	}
2148
2149	case PGM_PAGE_STATE_BALLOONED:
2150	case PGM_PAGE_STATE_ALLOCATED:
2151	case PGM_PAGE_STATE_WRITE_MONITORED:
2152	case PGM_PAGE_STATE_SHARED:
2153	/* nothing to do here. */
2154	break;
2155	}
2156	}
2157
2158	/* next */
2159	pPage++;
2160	GCPhys += GUEST_PAGE_SIZE;
2161	}
2162	}
2163	PGM_UNLOCK(pVM);
2164	NanoTS = RTTimeNanoTS() - NanoTS;
2165
2166	LogRel(("PGM: Pre-allocated %llu pages in %llu ms\n", cPages, NanoTS / 1000000));
2167	Log(("pgmR3PhysRamPreAllocate: returns VINF_SUCCESS\n"));
2168	return VINF_SUCCESS;
2169	}
2170
2171
2172	/**
2173	* Checks shared page checksums.
2174	*
2175	* @param pVM The cross context VM structure.
2176	*/
2177	void pgmR3PhysAssertSharedPageChecksums(PVM pVM)
2178	{
2179	#ifdef VBOX_STRICT
2180	PGM_LOCK_VOID(pVM);
2181
2182	if (pVM->pgm.s.cSharedPages > 0)
2183	{
2184	/*
2185	* Walk the ram ranges.
2186	*/
2187	uint32_t const cLookupEntries = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
2188	for (uint32_t idxLookup = 0; idxLookup < cLookupEntries; idxLookup++)
2189	{
2190	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
2191	AssertContinue(idRamRange < RT_ELEMENTS(pVM->pgm.s.apRamRanges));
2192	PPGMRAMRANGE const pRam = pVM->pgm.s.apRamRanges[idRamRange];
2193	AssertContinue(pRam);
2194
2195	uint32_t iPage = pRam->cb >> GUEST_PAGE_SHIFT;
2196	AssertMsg(((RTGCPHYS)iPage << GUEST_PAGE_SHIFT) == pRam->cb,
2197	("%RGp %RGp\n", (RTGCPHYS)iPage << GUEST_PAGE_SHIFT, pRam->cb));
2198
2199	while (iPage-- > 0)
2200	{
2201	PPGMPAGE pPage = &pRam->aPages[iPage];
2202	if (PGM_PAGE_IS_SHARED(pPage))
2203	{
2204	uint32_t u32Checksum = pPage->s.u2Unused0/* \| ((uint32_t)pPage->s.u2Unused1 << 8)*/;
2205	if (!u32Checksum)
2206	{
2207	RTGCPHYS GCPhysPage = pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT);
2208	void const *pvPage;
2209	int rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhysPage, &pvPage);
2210	if (RT_SUCCESS(rc))
2211	{
2212	uint32_t u32Checksum2 = RTCrc32(pvPage, GUEST_PAGE_SIZE);
2213	# if 0
2214	AssertMsg((u32Checksum2 & /UINT32_C(0x00000303)/ 0x3) == u32Checksum, ("GCPhysPage=%RGp\n", GCPhysPage));
2215	# else
2216	if ((u32Checksum2 & /UINT32_C(0x00000303)/ 0x3) == u32Checksum)
2217	LogFlow(("shpg %#x @ %RGp %#x [OK]\n", PGM_PAGE_GET_PAGEID(pPage), GCPhysPage, u32Checksum2));
2218	else
2219	AssertMsgFailed(("shpg %#x @ %RGp %#x\n", PGM_PAGE_GET_PAGEID(pPage), GCPhysPage, u32Checksum2));
2220	# endif
2221	}
2222	else
2223	AssertRC(rc);
2224	}
2225	}
2226
2227	} /* for each page */
2228
2229	} /* for each ram range */
2230	}
2231
2232	PGM_UNLOCK(pVM);
2233	#endif /* VBOX_STRICT */
2234	NOREF(pVM);
2235	}
2236
2237
2238	/**
2239	* Resets the physical memory state.
2240	*
2241	* ASSUMES that the caller owns the PGM lock.
2242	*
2243	* @returns VBox status code.
2244	* @param pVM The cross context VM structure.
2245	*/
2246	int pgmR3PhysRamReset(PVM pVM)
2247	{
2248	PGM_LOCK_ASSERT_OWNER(pVM);
2249
2250	/* Reset the memory balloon. */
2251	int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
2252	AssertRC(rc);
2253
2254	#ifdef VBOX_WITH_PAGE_SHARING
2255	/* Clear all registered shared modules. */
2256	pgmR3PhysAssertSharedPageChecksums(pVM);
2257	rc = GMMR3ResetSharedModules(pVM);
2258	AssertRC(rc);
2259	#endif
2260	/* Reset counters. */
2261	pVM->pgm.s.cReusedSharedPages = 0;
2262	pVM->pgm.s.cBalloonedPages = 0;
2263
2264	return VINF_SUCCESS;
2265	}
2266
2267
2268	/**
2269	* Resets (zeros) the RAM after all devices and components have been reset.
2270	*
2271	* ASSUMES that the caller owns the PGM lock.
2272	*
2273	* @returns VBox status code.
2274	* @param pVM The cross context VM structure.
2275	*/
2276	int pgmR3PhysRamZeroAll(PVM pVM)
2277	{
2278	PGM_LOCK_ASSERT_OWNER(pVM);
2279
2280	/*
2281	* We batch up pages that should be freed instead of calling GMM for
2282	* each and every one of them.
2283	*/
2284	uint32_t cPendingPages = 0;
2285	PGMMFREEPAGESREQ pReq;
2286	int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
2287	AssertLogRelRCReturn(rc, rc);
2288
2289	/*
2290	* Walk the ram ranges.
2291	*/
2292	uint32_t const idRamRangeMax = RT_MIN(pVM->pgm.s.idRamRangeMax, RT_ELEMENTS(pVM->pgm.s.apRamRanges) - 1U);
2293	for (uint32_t idRamRange = 0; idRamRange <= idRamRangeMax; idRamRange++)
2294	{
2295	PPGMRAMRANGE const pRam = pVM->pgm.s.apRamRanges[idRamRange];
2296	Assert(pRam \|\| idRamRange == 0);
2297	if (!pRam) continue;
2298	Assert(pRam->idRange == idRamRange);
2299
2300	uint32_t iPage = pRam->cb >> GUEST_PAGE_SHIFT;
2301	AssertMsg(((RTGCPHYS)iPage << GUEST_PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << GUEST_PAGE_SHIFT, pRam->cb));
2302
2303	if ( !pVM->pgm.s.fRamPreAlloc
2304	#ifdef VBOX_WITH_PGM_NEM_MODE
2305	&& !pVM->pgm.s.fNemMode
2306	#endif
2307	&& pVM->pgm.s.fZeroRamPagesOnReset)
2308	{
2309	/* Replace all RAM pages by ZERO pages. */
2310	while (iPage-- > 0)
2311	{
2312	PPGMPAGE pPage = &pRam->aPages[iPage];
2313	switch (PGM_PAGE_GET_TYPE(pPage))
2314	{
2315	case PGMPAGETYPE_RAM:
2316	/* Do not replace pages part of a 2 MB continuous range
2317	with zero pages, but zero them instead. */
2318	if ( PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE
2319	\|\| PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED)
2320	{
2321	void *pvPage;
2322	rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT), &pvPage);
2323	AssertLogRelRCReturn(rc, rc);
2324	RT_BZERO(pvPage, GUEST_PAGE_SIZE);
2325	}
2326	else if (PGM_PAGE_IS_BALLOONED(pPage))
2327	{
2328	/* Turn into a zero page; the balloon status is lost when the VM reboots. */
2329	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
2330	}
2331	else if (!PGM_PAGE_IS_ZERO(pPage))
2332	{
2333	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage,
2334	pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT), PGMPAGETYPE_RAM);
2335	AssertLogRelRCReturn(rc, rc);
2336	}
2337	break;
2338
2339	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
2340	case PGMPAGETYPE_SPECIAL_ALIAS_MMIO: /** @todo perhaps leave the special page alone? I don't think VT-x copes with this code. */
2341	pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT),
2342	pRam, true /fDoAccounting/, false /fFlushIemTlbs/);
2343	break;
2344
2345	case PGMPAGETYPE_MMIO2:
2346	case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
2347	case PGMPAGETYPE_ROM:
2348	case PGMPAGETYPE_MMIO:
2349	break;
2350	default:
2351	AssertFailed();
2352	}
2353	} /* for each page */
2354	}
2355	else
2356	{
2357	/* Zero the memory. */
2358	while (iPage-- > 0)
2359	{
2360	PPGMPAGE pPage = &pRam->aPages[iPage];
2361	switch (PGM_PAGE_GET_TYPE(pPage))
2362	{
2363	case PGMPAGETYPE_RAM:
2364	switch (PGM_PAGE_GET_STATE(pPage))
2365	{
2366	case PGM_PAGE_STATE_ZERO:
2367	break;
2368
2369	case PGM_PAGE_STATE_BALLOONED:
2370	/* Turn into a zero page; the balloon status is lost when the VM reboots. */
2371	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
2372	break;
2373
2374	case PGM_PAGE_STATE_SHARED:
2375	case PGM_PAGE_STATE_WRITE_MONITORED:
2376	rc = pgmPhysPageMakeWritable(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT));
2377	AssertLogRelRCReturn(rc, rc);
2378	RT_FALL_THRU();
2379
2380	case PGM_PAGE_STATE_ALLOCATED:
2381	if (pVM->pgm.s.fZeroRamPagesOnReset)
2382	{
2383	void *pvPage;
2384	rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT), &pvPage);
2385	AssertLogRelRCReturn(rc, rc);
2386	RT_BZERO(pvPage, GUEST_PAGE_SIZE);
2387	}
2388	break;
2389	}
2390	break;
2391
2392	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
2393	case PGMPAGETYPE_SPECIAL_ALIAS_MMIO: /** @todo perhaps leave the special page alone? I don't think VT-x copes with this code. */
2394	pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT),
2395	pRam, true /fDoAccounting/, false /fFlushIemTlbs/);
2396	break;
2397
2398	case PGMPAGETYPE_MMIO2:
2399	case PGMPAGETYPE_ROM_SHADOW:
2400	case PGMPAGETYPE_ROM:
2401	case PGMPAGETYPE_MMIO:
2402	break;
2403	default:
2404	AssertFailed();
2405
2406	}
2407	} /* for each page */
2408	}
2409	}
2410
2411	/*
2412	* Finish off any pages pending freeing.
2413	*/
2414	if (cPendingPages)
2415	{
2416	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2417	AssertLogRelRCReturn(rc, rc);
2418	}
2419	GMMR3FreePagesCleanup(pReq);
2420
2421	/*
2422	* Flush the IEM TLB, just to be sure it really is done.
2423	*/
2424	IEMTlbInvalidateAllPhysicalAllCpus(pVM, NIL_VMCPUID, IEMTLBPHYSFLUSHREASON_ZERO_ALL);
2425
2426	return VINF_SUCCESS;
2427	}
2428
2429
2430	/**
2431	* Frees all RAM during VM termination
2432	*
2433	* ASSUMES that the caller owns the PGM lock.
2434	*
2435	* @returns VBox status code.
2436	* @param pVM The cross context VM structure.
2437	*/
2438	int pgmR3PhysRamTerm(PVM pVM)
2439	{
2440	PGM_LOCK_ASSERT_OWNER(pVM);
2441
2442	/* Reset the memory balloon. */
2443	int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
2444	AssertRC(rc);
2445
2446	#ifdef VBOX_WITH_PAGE_SHARING
2447	/*
2448	* Clear all registered shared modules.
2449	*/
2450	pgmR3PhysAssertSharedPageChecksums(pVM);
2451	rc = GMMR3ResetSharedModules(pVM);
2452	AssertRC(rc);
2453
2454	/*
2455	* Flush the handy pages updates to make sure no shared pages are hiding
2456	* in there. (Not unlikely if the VM shuts down, apparently.)
2457	*/
2458	# ifdef VBOX_WITH_PGM_NEM_MODE
2459	if (!pVM->pgm.s.fNemMode)
2460	# endif
2461	rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_FLUSH_HANDY_PAGES, 0, NULL);
2462	#endif
2463
2464	/*
2465	* We batch up pages that should be freed instead of calling GMM for
2466	* each and every one of them.
2467	*/
2468	uint32_t cPendingPages = 0;
2469	PGMMFREEPAGESREQ pReq;
2470	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
2471	AssertLogRelRCReturn(rc, rc);
2472
2473	/*
2474	* Walk the ram ranges.
2475	*/
2476	uint32_t const idRamRangeMax = RT_MIN(pVM->pgm.s.idRamRangeMax, RT_ELEMENTS(pVM->pgm.s.apRamRanges) - 1U);
2477	for (uint32_t idRamRange = 0; idRamRange <= idRamRangeMax; idRamRange++)
2478	{
2479	PPGMRAMRANGE const pRam = pVM->pgm.s.apRamRanges[idRamRange];
2480	Assert(pRam \|\| idRamRange == 0);
2481	if (!pRam) continue;
2482	Assert(pRam->idRange == idRamRange);
2483
2484	uint32_t iPage = pRam->cb >> GUEST_PAGE_SHIFT;
2485	AssertMsg(((RTGCPHYS)iPage << GUEST_PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << GUEST_PAGE_SHIFT, pRam->cb));
2486
2487	while (iPage-- > 0)
2488	{
2489	PPGMPAGE pPage = &pRam->aPages[iPage];
2490	switch (PGM_PAGE_GET_TYPE(pPage))
2491	{
2492	case PGMPAGETYPE_RAM:
2493	/* Free all shared pages. Private pages are automatically freed during GMM VM cleanup. */
2494	/** @todo change this to explicitly free private pages here. */
2495	if (PGM_PAGE_IS_SHARED(pPage))
2496	{
2497	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage,
2498	pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT), PGMPAGETYPE_RAM);
2499	AssertLogRelRCReturn(rc, rc);
2500	}
2501	break;
2502
2503	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
2504	case PGMPAGETYPE_SPECIAL_ALIAS_MMIO:
2505	case PGMPAGETYPE_MMIO2:
2506	case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
2507	case PGMPAGETYPE_ROM:
2508	case PGMPAGETYPE_MMIO:
2509	break;
2510	default:
2511	AssertFailed();
2512	}
2513	} /* for each page */
2514	}
2515
2516	/*
2517	* Finish off any pages pending freeing.
2518	*/
2519	if (cPendingPages)
2520	{
2521	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2522	AssertLogRelRCReturn(rc, rc);
2523	}
2524	GMMR3FreePagesCleanup(pReq);
2525	return VINF_SUCCESS;
2526	}
2527
2528
2529
2530	/*********************************************************************************************************************************
2531	* MMIO *
2532	*********************************************************************************************************************************/
2533
2534	/**
2535	* This is the interface IOM is using to register an MMIO region (unmapped).
2536	*
2537	*
2538	* @returns VBox status code.
2539	*
2540	* @param pVM The cross context VM structure.
2541	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2542	* @param cb The size of the MMIO region.
2543	* @param pszDesc The description of the MMIO region.
2544	* @param pidRamRange Where to return the RAM range ID for the MMIO region
2545	* on success.
2546	* @thread EMT(0)
2547	*/
2548	VMMR3_INT_DECL(int) PGMR3PhysMmioRegister(PVM pVM, PVMCPU pVCpu, RTGCPHYS cb, const char pszDesc, uint16_t pidRamRange)
2549	{
2550	/*
2551	* Assert assumptions.
2552	*/
2553	AssertPtrReturn(pidRamRange, VERR_INVALID_POINTER);
2554	*pidRamRange = UINT16_MAX;
2555	AssertReturn(pVCpu == VMMGetCpu(pVM) && pVCpu->idCpu == 0, VERR_VM_THREAD_NOT_EMT);
2556	VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
2557	/// @todo AssertReturn(!pVM->pgm.s.fRamRangesFrozen, VERR_WRONG_ORDER);
2558	AssertReturn(cb <= ((RTGCPHYS)PGM_MAX_PAGES_PER_RAM_RANGE << GUEST_PAGE_SHIFT), VERR_OUT_OF_RANGE);
2559	AssertReturn(!(cb & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2560	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
2561	AssertReturn(*pszDesc != '\0', VERR_INVALID_POINTER);
2562
2563	/*
2564	* Take the PGM lock and allocate an ad-hoc MMIO RAM range.
2565	*/
2566	int rc = PGM_LOCK(pVM);
2567	AssertRCReturn(rc, rc);
2568
2569	uint32_t const cPages = cb >> GUEST_PAGE_SHIFT;
2570	PPGMRAMRANGE pNew = NULL;
2571	rc = pgmR3PhysAllocateRamRange(pVM, pVCpu, cPages, PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO, &pNew);
2572	AssertLogRelMsg(RT_SUCCESS(rc), ("pgmR3PhysAllocateRamRange failed: cPages=%#RX32 (%s): %Rrc\n", cPages, pszDesc, rc));
2573	if (RT_SUCCESS(rc))
2574	{
2575	/* Initialize the range. */
2576	pNew->pszDesc = pszDesc;
2577	pNew->uNemRange = UINT32_MAX;
2578	pNew->pbR3 = NULL;
2579	pNew->paLSPages = NULL;
2580	Assert(pNew->fFlags == PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO && pNew->cb == cb);
2581
2582	uint32_t iPage = cPages;
2583	while (iPage-- > 0)
2584	PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_MMIO);
2585	Assert(PGM_PAGE_GET_TYPE(&pNew->aPages[0]) == PGMPAGETYPE_MMIO);
2586
2587	/* update the page count stats. */
2588	pVM->pgm.s.cPureMmioPages += cPages;
2589	pVM->pgm.s.cAllPages += cPages;
2590
2591	/*
2592	* Set the return value, release lock and return to IOM.
2593	*/
2594	*pidRamRange = pNew->idRange;
2595	}
2596
2597	PGM_UNLOCK(pVM);
2598	return rc;
2599	}
2600
2601
2602	/**
2603	* Worker for PGMR3PhysMmioMap that's called owning the lock.
2604	*/
2605	static int pgmR3PhysMmioMapLocked(PVM pVM, PVMCPU pVCpu, RTGCPHYS const GCPhys, RTGCPHYS const cb, RTGCPHYS const GCPhysLast,
2606	PPGMRAMRANGE const pMmioRamRange, PGMPHYSHANDLERTYPE const hType, uint64_t const uUser)
2607	{
2608	/* Check that the range isn't mapped already. */
2609	AssertLogRelMsgReturn(pMmioRamRange->GCPhys == NIL_RTGCPHYS,
2610	("desired %RGp mapping for '%s' - already mapped at %RGp!\n",
2611	GCPhys, pMmioRamRange->pszDesc, pMmioRamRange->GCPhys),
2612	VERR_ALREADY_EXISTS);
2613
2614	/*
2615	* Now, check if this falls into a regular RAM range or if we should use
2616	* the ad-hoc one (idRamRange).
2617	*/
2618	int rc;
2619	uint32_t idxInsert = UINT32_MAX;
2620	PPGMRAMRANGE const pOverlappingRange = pgmR3PhysRamRangeFindOverlapping(pVM, GCPhys, GCPhysLast, &idxInsert);
2621	if (pOverlappingRange)
2622	{
2623	/* Simplification: all within the same range. */
2624	AssertLogRelMsgReturn( GCPhys >= pOverlappingRange->GCPhys
2625	&& GCPhysLast <= pOverlappingRange->GCPhysLast,
2626	("%RGp-%RGp (MMIO/%s) falls partly outside %RGp-%RGp (%s)\n",
2627	GCPhys, GCPhysLast, pMmioRamRange->pszDesc,
2628	pOverlappingRange->GCPhys, pOverlappingRange->GCPhysLast, pOverlappingRange->pszDesc),
2629	VERR_PGM_RAM_CONFLICT);
2630
2631	/* Check that is isn't an ad hoc range, but a real RAM range. */
2632	AssertLogRelMsgReturn(!PGM_RAM_RANGE_IS_AD_HOC(pOverlappingRange),
2633	("%RGp-%RGp (MMIO/%s) mapping attempt in non-RAM range: %RGp-%RGp (%s)\n",
2634	GCPhys, GCPhysLast, pMmioRamRange->pszDesc,
2635	pOverlappingRange->GCPhys, pOverlappingRange->GCPhysLast, pOverlappingRange->pszDesc),
2636	VERR_PGM_RAM_CONFLICT);
2637
2638	/* Check that it's all RAM or MMIO pages. */
2639	PCPGMPAGE pPage = &pOverlappingRange->aPages[(GCPhys - pOverlappingRange->GCPhys) >> GUEST_PAGE_SHIFT];
2640	uint32_t cLeft = cb >> GUEST_PAGE_SHIFT;
2641	while (cLeft-- > 0)
2642	{
2643	AssertLogRelMsgReturn( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2644	\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO, /** @todo MMIO type isn't right */
2645	("%RGp-%RGp (MMIO/%s): %RGp is not a RAM or MMIO page - type=%d desc=%s\n",
2646	GCPhys, GCPhysLast, pMmioRamRange->pszDesc, pOverlappingRange->GCPhys,
2647	PGM_PAGE_GET_TYPE(pPage), pOverlappingRange->pszDesc),
2648	VERR_PGM_RAM_CONFLICT);
2649	pPage++;
2650	}
2651
2652	/*
2653	* Make all the pages in the range MMIO/ZERO pages, freeing any
2654	* RAM pages currently mapped here. This might not be 100% correct
2655	* for PCI memory, but we're doing the same thing for MMIO2 pages.
2656	*/
2657	rc = pgmR3PhysFreePageRange(pVM, pOverlappingRange, GCPhys, GCPhysLast, NULL);
2658	AssertRCReturn(rc, rc);
2659
2660	/* Force a PGM pool flush as guest ram references have been changed. */
2661	/** @todo not entirely SMP safe; assuming for now the guest takes
2662	* care of this internally (not touch mapped mmio while changing the
2663	* mapping). */
2664	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2665	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2666	}
2667	else
2668	{
2669	/*
2670	* No RAM range, use the ad hoc one (idRamRange).
2671	*
2672	* Note that we don't have to tell REM about this range because
2673	* PGMHandlerPhysicalRegisterEx will do that for us.
2674	*/
2675	AssertLogRelReturn(idxInsert <= pVM->pgm.s.RamRangeUnion.cLookupEntries, VERR_INTERNAL_ERROR_4);
2676	Log(("PGMR3PhysMmioMap: Inserting ad hoc MMIO range #%x for %RGp-%RGp %s\n",
2677	pMmioRamRange->idRange, GCPhys, GCPhysLast, pMmioRamRange->pszDesc));
2678
2679	Assert(PGM_PAGE_GET_TYPE(&pMmioRamRange->aPages[0]) == PGMPAGETYPE_MMIO);
2680
2681	/* We ASSUME that all the pages in the ad-hoc range are in the proper
2682	state and all that and that we don't need to re-initialize them here. */
2683
2684	#ifdef VBOX_WITH_NATIVE_NEM
2685	/* Notify NEM. */
2686	if (VM_IS_NEM_ENABLED(pVM))
2687	{
2688	uint8_t u2State = 0; /* (must have valid state as there can't be anything to preserve) */
2689	rc = NEMR3NotifyPhysMmioExMapEarly(pVM, GCPhys, cb, 0 /fFlags/, NULL, NULL, &u2State, &pMmioRamRange->uNemRange);
2690	AssertLogRelRCReturn(rc, rc);
2691
2692	uint32_t iPage = cb >> GUEST_PAGE_SHIFT;
2693	while (iPage-- > 0)
2694	PGM_PAGE_SET_NEM_STATE(&pMmioRamRange->aPages[iPage], u2State);
2695	}
2696	#endif
2697	/* Insert it into the lookup table (may in theory fail). */
2698	rc = pgmR3PhysRamRangeInsertLookup(pVM, pMmioRamRange, GCPhys, &idxInsert);
2699	}
2700	if (RT_SUCCESS(rc))
2701	{
2702	/*
2703	* Register the access handler.
2704	*/
2705	rc = PGMHandlerPhysicalRegister(pVM, GCPhys, GCPhysLast, hType, uUser, pMmioRamRange->pszDesc);
2706	if (RT_SUCCESS(rc))
2707	{
2708	#ifdef VBOX_WITH_NATIVE_NEM
2709	/* Late NEM notification (currently not used by anyone). */
2710	if (VM_IS_NEM_ENABLED(pVM))
2711	{
2712	if (pOverlappingRange)
2713	rc = NEMR3NotifyPhysMmioExMapLate(pVM, GCPhys, cb, NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE,
2714	pOverlappingRange->pbR3 + (uintptr_t)(GCPhys - pOverlappingRange->GCPhys),
2715	NULL /pvMmio2/, NULL /puNemRange/);
2716	else
2717	rc = NEMR3NotifyPhysMmioExMapLate(pVM, GCPhys, cb, 0 /fFlags/, NULL /pvRam/, NULL /pvMmio2/,
2718	&pMmioRamRange->uNemRange);
2719	AssertLogRelRC(rc);
2720	}
2721	if (RT_SUCCESS(rc))
2722	#endif
2723	{
2724	pgmPhysInvalidatePageMapTLB(pVM, false /fInRendezvous/);
2725	return VINF_SUCCESS;
2726	}
2727
2728	/*
2729	* Failed, so revert it all as best as we can (the memory content in
2730	* the overlapping case is gone).
2731	*/
2732	PGMHandlerPhysicalDeregister(pVM, GCPhys);
2733	}
2734	}
2735
2736	if (!pOverlappingRange)
2737	{
2738	#ifdef VBOX_WITH_NATIVE_NEM
2739	/* Notify NEM about the sudden removal of the RAM range we just told it about. */
2740	NEMR3NotifyPhysMmioExUnmap(pVM, GCPhys, cb, 0 /fFlags/, NULL /pvRam/, NULL /pvMmio2/,
2741	NULL /pu2State/, &pMmioRamRange->uNemRange);
2742	#endif
2743
2744	/* Remove the ad hoc range from the lookup table. */
2745	idxInsert -= 1;
2746	pgmR3PhysRamRangeRemoveLookup(pVM, pMmioRamRange, &idxInsert);
2747	}
2748
2749	pgmPhysInvalidatePageMapTLB(pVM, false /fInRendezvous/);
2750	return rc;
2751	}
2752
2753
2754	/**
2755	* This is the interface IOM is using to map an MMIO region.
2756	*
2757	* It will check for conflicts and ensure that a RAM range structure
2758	* is present before calling the PGMR3HandlerPhysicalRegister API to
2759	* register the callbacks.
2760	*
2761	* @returns VBox status code.
2762	*
2763	* @param pVM The cross context VM structure.
2764	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2765	* @param GCPhys The start of the MMIO region.
2766	* @param cb The size of the MMIO region.
2767	* @param idRamRange The RAM range ID for the MMIO region as returned by
2768	* PGMR3PhysMmioRegister().
2769	* @param hType The physical access handler type registration.
2770	* @param uUser The user argument.
2771	* @thread EMT(pVCpu)
2772	*/
2773	VMMR3_INT_DECL(int) PGMR3PhysMmioMap(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, RTGCPHYS cb, uint16_t idRamRange,
2774	PGMPHYSHANDLERTYPE hType, uint64_t uUser)
2775	{
2776	/*
2777	* Assert on some assumption.
2778	*/
2779	VMCPU_ASSERT_EMT(pVCpu);
2780	AssertReturn(!(cb & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2781	AssertReturn(!(GCPhys & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2782	RTGCPHYS const GCPhysLast = GCPhys + cb - 1U;
2783	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
2784	#ifdef VBOX_STRICT
2785	PCPGMPHYSHANDLERTYPEINT pType = pgmHandlerPhysicalTypeHandleToPtr(pVM, hType);
2786	Assert(pType);
2787	Assert(pType->enmKind == PGMPHYSHANDLERKIND_MMIO);
2788	#endif
2789	AssertReturn(idRamRange <= pVM->pgm.s.idRamRangeMax && idRamRange > 0, VERR_INVALID_HANDLE);
2790	PPGMRAMRANGE const pMmioRamRange = pVM->pgm.s.apRamRanges[idRamRange];
2791	AssertReturn(pMmioRamRange, VERR_INVALID_HANDLE);
2792	AssertReturn(pMmioRamRange->fFlags & PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO, VERR_INVALID_HANDLE);
2793	AssertReturn(pMmioRamRange->cb == cb, VERR_OUT_OF_RANGE);
2794
2795	/*
2796	* Take the PGM lock and do the work.
2797	*/
2798	int rc = PGM_LOCK(pVM);
2799	AssertRCReturn(rc, rc);
2800
2801	rc = pgmR3PhysMmioMapLocked(pVM, pVCpu, GCPhys, cb, GCPhysLast, pMmioRamRange, hType, uUser);
2802	#ifdef VBOX_STRICT
2803	pgmPhysAssertRamRangesLocked(pVM, false /fInUpdate/, false /fRamRelaxed/);
2804	#endif
2805
2806	PGM_UNLOCK(pVM);
2807	return rc;
2808	}
2809
2810
2811	/**
2812	* Worker for PGMR3PhysMmioUnmap that's called with the PGM lock held.
2813	*/
2814	static int pgmR3PhysMmioUnmapLocked(PVM pVM, PVMCPU pVCpu, RTGCPHYS const GCPhys, RTGCPHYS const cb,
2815	RTGCPHYS const GCPhysLast, PPGMRAMRANGE const pMmioRamRange)
2816	{
2817	/*
2818	* Lookup the RAM range containing the region to make sure it is actually mapped.
2819	*/
2820	uint32_t idxLookup = pgmR3PhysRamRangeFindOverlappingIndex(pVM, GCPhys, GCPhysLast);
2821	AssertLogRelMsgReturn(idxLookup < pVM->pgm.s.RamRangeUnion.cLookupEntries,
2822	("MMIO range not found at %RGp LB %RGp! (%s)\n", GCPhys, cb, pMmioRamRange->pszDesc),
2823	VERR_NOT_FOUND);
2824
2825	uint32_t const idLookupRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
2826	AssertLogRelReturn(idLookupRange != 0 && idLookupRange <= pVM->pgm.s.idRamRangeMax, VERR_INTERNAL_ERROR_5);
2827	PPGMRAMRANGE const pLookupRange = pVM->pgm.s.apRamRanges[idLookupRange];
2828	AssertLogRelReturn(pLookupRange, VERR_INTERNAL_ERROR_4);
2829
2830	AssertLogRelMsgReturn(pLookupRange == pMmioRamRange \|\| !PGM_RAM_RANGE_IS_AD_HOC(pLookupRange),
2831	("MMIO unmap mixup at %RGp LB %RGp (%s) vs %RGp LB %RGp (%s)\n",
2832	GCPhys, cb, pMmioRamRange->pszDesc, pLookupRange->GCPhys, pLookupRange->cb, pLookupRange->pszDesc),
2833	VERR_NOT_FOUND);
2834
2835	/*
2836	* Deregister the handler. This should reset any aliases, so an ad hoc
2837	* range will only contain MMIO type pages afterwards.
2838	*/
2839	int rc = PGMHandlerPhysicalDeregister(pVM, GCPhys);
2840	if (RT_SUCCESS(rc))
2841	{
2842	if (pLookupRange != pMmioRamRange)
2843	{
2844	/*
2845	* Turn the pages back into RAM pages.
2846	*/
2847	Log(("pgmR3PhysMmioUnmapLocked: Reverting MMIO range %RGp-%RGp (%s) in %RGp-%RGp (%s) to RAM.\n",
2848	GCPhys, GCPhysLast, pMmioRamRange->pszDesc,
2849	pLookupRange->GCPhys, pLookupRange->GCPhysLast, pLookupRange->pszDesc));
2850
2851	RTGCPHYS const offRange = GCPhys - pLookupRange->GCPhys;
2852	uint32_t iPage = offRange >> GUEST_PAGE_SHIFT;
2853	uint32_t cLeft = cb >> GUEST_PAGE_SHIFT;
2854	while (cLeft--)
2855	{
2856	PPGMPAGE pPage = &pLookupRange->aPages[iPage];
2857	AssertMsg( (PGM_PAGE_IS_MMIO(pPage) && PGM_PAGE_IS_ZERO(pPage))
2858	//\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO
2859	//\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_SPECIAL_ALIAS_MMIO
2860	, ("%RGp %R[pgmpage]\n", pLookupRange->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT), pPage));
2861	/** @todo this isn't entirely correct, is it now... aliases must be converted
2862	* to zero pages as they won't be. however, shouldn't
2863	* PGMHandlerPhysicalDeregister deal with this already? */
2864	if (PGM_PAGE_IS_MMIO_OR_ALIAS(pPage))
2865	PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_RAM);
2866	iPage++;
2867	}
2868
2869	#ifdef VBOX_WITH_NATIVE_NEM
2870	/* Notify REM (failure will probably leave things in a non-working state). */
2871	if (VM_IS_NEM_ENABLED(pVM))
2872	{
2873	uint8_t u2State = UINT8_MAX;
2874	rc = NEMR3NotifyPhysMmioExUnmap(pVM, GCPhys, GCPhysLast - GCPhys + 1, NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE,
2875	pLookupRange->pbR3 ? pLookupRange->pbR3 + GCPhys - pLookupRange->GCPhys : NULL,
2876	NULL, &u2State, &pLookupRange->uNemRange);
2877	AssertLogRelRC(rc);
2878	/** @todo status code propagation here... This is likely fatal, right? */
2879	if (u2State != UINT8_MAX)
2880	pgmPhysSetNemStateForPages(&pLookupRange->aPages[(GCPhys - pLookupRange->GCPhys) >> GUEST_PAGE_SHIFT],
2881	cb >> GUEST_PAGE_SHIFT, u2State);
2882	}
2883	#endif
2884	}
2885	else
2886	{
2887	/*
2888	* Unlink the ad hoc range.
2889	*/
2890	#ifdef VBOX_STRICT
2891	uint32_t iPage = cb >> GUEST_PAGE_SHIFT;
2892	while (iPage-- > 0)
2893	{
2894	PPGMPAGE const pPage = &pMmioRamRange->aPages[iPage];
2895	Assert(PGM_PAGE_IS_MMIO(pPage));
2896	}
2897	#endif
2898
2899	Log(("pgmR3PhysMmioUnmapLocked: Unmapping ad hoc MMIO range for %RGp-%RGp %s\n",
2900	GCPhys, GCPhysLast, pMmioRamRange->pszDesc));
2901
2902	#ifdef VBOX_WITH_NATIVE_NEM
2903	if (VM_IS_NEM_ENABLED(pVM)) /* Notify REM before we unlink the range. */
2904	{
2905	rc = NEMR3NotifyPhysMmioExUnmap(pVM, GCPhys, GCPhysLast - GCPhys + 1, 0 /fFlags/,
2906	NULL, NULL, NULL, &pMmioRamRange->uNemRange);
2907	AssertLogRelRCReturn(rc, rc); /* we're up the creek if this hits. */
2908	}
2909	#endif
2910
2911	pgmR3PhysRamRangeRemoveLookup(pVM, pMmioRamRange, &idxLookup);
2912	}
2913	}
2914
2915	/* Force a PGM pool flush as guest ram references have been changed. */
2916	/** @todo Not entirely SMP safe; assuming for now the guest takes care of
2917	* this internally (not touch mapped mmio while changing the mapping). */
2918	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2919	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2920
2921	pgmPhysInvalidatePageMapTLB(pVM, false /fInRendezvous/);
2922	/pgmPhysInvalidRamRangeTlbs(pVM); - not necessary /
2923
2924	return rc;
2925	}
2926
2927
2928	/**
2929	* This is the interface IOM is using to register an MMIO region.
2930	*
2931	* It will take care of calling PGMHandlerPhysicalDeregister and clean up
2932	* any ad hoc PGMRAMRANGE left behind.
2933	*
2934	* @returns VBox status code.
2935	* @param pVM The cross context VM structure.
2936	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2937	* @param GCPhys The start of the MMIO region.
2938	* @param cb The size of the MMIO region.
2939	* @param idRamRange The RAM range ID for the MMIO region as returned by
2940	* PGMR3PhysMmioRegister().
2941	* @thread EMT(pVCpu)
2942	*/
2943	VMMR3_INT_DECL(int) PGMR3PhysMmioUnmap(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, RTGCPHYS cb, uint16_t idRamRange)
2944	{
2945	/*
2946	* Input validation.
2947	*/
2948	VMCPU_ASSERT_EMT(pVCpu);
2949	AssertReturn(!(cb & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2950	AssertReturn(!(GCPhys & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2951	RTGCPHYS const GCPhysLast = GCPhys + cb - 1U;
2952	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
2953	AssertReturn(idRamRange <= pVM->pgm.s.idRamRangeMax && idRamRange > 0, VERR_INVALID_HANDLE);
2954	PPGMRAMRANGE const pMmioRamRange = pVM->pgm.s.apRamRanges[idRamRange];
2955	AssertReturn(pMmioRamRange, VERR_INVALID_HANDLE);
2956	AssertReturn(pMmioRamRange->fFlags & PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO, VERR_INVALID_HANDLE);
2957	AssertReturn(pMmioRamRange->cb == cb, VERR_OUT_OF_RANGE);
2958
2959	/*
2960	* Take the PGM lock and do what's asked.
2961	*/
2962	int rc = PGM_LOCK(pVM);
2963	AssertRCReturn(rc, rc);
2964
2965	rc = pgmR3PhysMmioUnmapLocked(pVM, pVCpu, GCPhys, cb, GCPhysLast, pMmioRamRange);
2966	#ifdef VBOX_STRICT
2967	pgmPhysAssertRamRangesLocked(pVM, false /fInUpdate/, false /fRamRelaxed/);
2968	#endif
2969
2970	PGM_UNLOCK(pVM);
2971	return rc;
2972	}
2973
2974
2975
2976	/*********************************************************************************************************************************
2977	* MMIO2 *
2978	*********************************************************************************************************************************/
2979
2980	/**
2981	* Validates the claim to an MMIO2 range and returns the pointer to it.
2982	*
2983	* @returns The MMIO2 entry index on success, negative error status on failure.
2984	* @param pVM The cross context VM structure.
2985	* @param pDevIns The device instance owning the region.
2986	* @param hMmio2 Handle to look up.
2987	* @param pcChunks Where to return the number of chunks associated with
2988	* this handle.
2989	*/
2990	static int32_t pgmR3PhysMmio2ResolveHandle(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, uint32_t *pcChunks)
2991	{
2992	*pcChunks = 0;
2993	uint32_t const idxFirst = hMmio2 - 1U;
2994	uint32_t const cMmio2Ranges = RT_MIN(pVM->pgm.s.cMmio2Ranges, RT_ELEMENTS(pVM->pgm.s.aMmio2Ranges));
2995	AssertReturn(idxFirst < cMmio2Ranges, VERR_INVALID_HANDLE);
2996
2997	PPGMREGMMIO2RANGE const pFirst = &pVM->pgm.s.aMmio2Ranges[idxFirst];
2998	AssertReturn(pFirst->idMmio2 == hMmio2, VERR_INVALID_HANDLE);
2999	AssertReturn((pFirst->fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK), VERR_INVALID_HANDLE);
3000	AssertReturn(pFirst->pDevInsR3 == pDevIns && RT_VALID_PTR(pDevIns), VERR_NOT_OWNER);
3001
3002	/* Figure out how many chunks this handle spans. */
3003	if (pFirst->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
3004	*pcChunks = 1;
3005	else
3006	{
3007	uint32_t cChunks = 1;
3008	for (uint32_t idx = idxFirst + 1;; idx++)
3009	{
3010	cChunks++;
3011	AssertReturn(idx < cMmio2Ranges, VERR_INTERNAL_ERROR_2);
3012	PPGMREGMMIO2RANGE const pCur = &pVM->pgm.s.aMmio2Ranges[idx];
3013	AssertLogRelMsgReturn( pCur->pDevInsR3 == pDevIns
3014	&& pCur->idMmio2 == idx + 1
3015	&& pCur->iSubDev == pFirst->iSubDev
3016	&& pCur->iRegion == pFirst->iRegion
3017	&& !(pCur->fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK),
3018	("cur: %p/%#x/%#x/%#x/%#x/%s; first: %p/%#x/%#x/%#x/%#x/%s\n",
3019	pCur->pDevInsR3, pCur->idMmio2, pCur->iSubDev, pCur->iRegion, pCur->fFlags,
3020	pVM->pgm.s.apMmio2RamRanges[idx]->pszDesc,
3021	pDevIns, idx + 1, pFirst->iSubDev, pFirst->iRegion, pFirst->fFlags,
3022	pVM->pgm.s.apMmio2RamRanges[idxFirst]->pszDesc),
3023	VERR_INTERNAL_ERROR_3);
3024	if (pCur->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
3025	break;
3026	}
3027	*pcChunks = cChunks;
3028	}
3029
3030	return (int32_t)idxFirst;
3031	}
3032
3033
3034	/**
3035	* Check if a device has already registered a MMIO2 region.
3036	*
3037	* @returns NULL if not registered, otherwise pointer to the MMIO2.
3038	* @param pVM The cross context VM structure.
3039	* @param pDevIns The device instance owning the region.
3040	* @param iSubDev The sub-device number.
3041	* @param iRegion The region.
3042	*/
3043	DECLINLINE(PPGMREGMMIO2RANGE) pgmR3PhysMmio2Find(PVM pVM, PPDMDEVINS pDevIns, uint32_t iSubDev, uint32_t iRegion)
3044	{
3045	/*
3046	* Search the array. There shouldn't be many entries.
3047	*/
3048	uint32_t idx = RT_MIN(pVM->pgm.s.cMmio2Ranges, RT_ELEMENTS(pVM->pgm.s.aMmio2Ranges));
3049	while (idx-- > 0)
3050	if (RT_LIKELY( pVM->pgm.s.aMmio2Ranges[idx].pDevInsR3 != pDevIns
3051	\|\| pVM->pgm.s.aMmio2Ranges[idx].iRegion != iRegion
3052	\|\| pVM->pgm.s.aMmio2Ranges[idx].iSubDev != iSubDev))
3053	{ /* likely */ }
3054	else
3055	return &pVM->pgm.s.aMmio2Ranges[idx];
3056	return NULL;
3057	}
3058
3059	/**
3060	* Worker for PGMR3PhysMmio2ControlDirtyPageTracking and PGMR3PhysMmio2Map.
3061	*/
3062	static int pgmR3PhysMmio2EnableDirtyPageTracing(PVM pVM, uint32_t idx, uint32_t cChunks)
3063	{
3064	int rc = VINF_SUCCESS;
3065	while (cChunks-- > 0)
3066	{
3067	PPGMREGMMIO2RANGE const pMmio2 = &pVM->pgm.s.aMmio2Ranges[idx];
3068	PPGMRAMRANGE const pRamRange = pVM->pgm.s.apMmio2RamRanges[idx];
3069
3070	Assert(!(pMmio2->fFlags & PGMREGMMIO2RANGE_F_IS_TRACKING));
3071	int rc2 = pgmHandlerPhysicalExRegister(pVM, pMmio2->pPhysHandlerR3, pRamRange->GCPhys, pRamRange->GCPhysLast);
3072	if (RT_SUCCESS(rc2))
3073	pMmio2->fFlags \|= PGMREGMMIO2RANGE_F_IS_TRACKING;
3074	else
3075	AssertLogRelMsgFailedStmt(("%#RGp-%#RGp %s failed -> %Rrc\n",
3076	pRamRange->GCPhys, pRamRange->GCPhysLast, pRamRange->pszDesc, rc2),
3077	rc = RT_SUCCESS(rc) ? rc2 : rc);
3078
3079	idx++;
3080	}
3081	return rc;
3082	}
3083
3084
3085	/**
3086	* Worker for PGMR3PhysMmio2ControlDirtyPageTracking and PGMR3PhysMmio2Unmap.
3087	*/
3088	static int pgmR3PhysMmio2DisableDirtyPageTracing(PVM pVM, uint32_t idx, uint32_t cChunks)
3089	{
3090	int rc = VINF_SUCCESS;
3091	while (cChunks-- > 0)
3092	{
3093	PPGMREGMMIO2RANGE const pMmio2 = &pVM->pgm.s.aMmio2Ranges[idx];
3094	PPGMRAMRANGE const pRamRange = pVM->pgm.s.apMmio2RamRanges[idx];
3095	if (pMmio2->fFlags & PGMREGMMIO2RANGE_F_IS_TRACKING)
3096	{
3097	int rc2 = pgmHandlerPhysicalExDeregister(pVM, pMmio2->pPhysHandlerR3);
3098	AssertLogRelMsgStmt(RT_SUCCESS(rc2),
3099	("%#RGp-%#RGp %s failed -> %Rrc\n",
3100	pRamRange->GCPhys, pRamRange->GCPhysLast, pRamRange->pszDesc, rc2),
3101	rc = RT_SUCCESS(rc) ? rc2 : rc);
3102	pMmio2->fFlags &= ~PGMREGMMIO2RANGE_F_IS_TRACKING;
3103	}
3104	idx++;
3105	}
3106	return rc;
3107	}
3108
3109	#if 0 // temp
3110
3111	/**
3112	* Common worker PGMR3PhysMmio2PreRegister & PGMR3PhysMMIO2Register that links a
3113	* complete registration entry into the lists and lookup tables.
3114	*
3115	* @param pVM The cross context VM structure.
3116	* @param pNew The new MMIO / MMIO2 registration to link.
3117	*/
3118	static void pgmR3PhysMmio2Link(PVM pVM, PPGMREGMMIO2RANGE pNew)
3119	{
3120	Assert(pNew->idMmio2 != UINT8_MAX);
3121
3122	/*
3123	* Link it into the list (order doesn't matter, so insert it at the head).
3124	*
3125	* Note! The range we're linking may consist of multiple chunks, so we
3126	* have to find the last one.
3127	*/
3128	PPGMREGMMIO2RANGE pLast = pNew;
3129	for (pLast = pNew; ; pLast = pLast->pNextR3)
3130	{
3131	if (pLast->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
3132	break;
3133	Assert(pLast->pNextR3);
3134	Assert(pLast->pNextR3->pDevInsR3 == pNew->pDevInsR3);
3135	Assert(pLast->pNextR3->iSubDev == pNew->iSubDev);
3136	Assert(pLast->pNextR3->iRegion == pNew->iRegion);
3137	Assert(pLast->pNextR3->idMmio2 == pLast->idMmio2 + 1);
3138	}
3139
3140	PGM_LOCK_VOID(pVM);
3141
3142	/* Link in the chain of ranges at the head of the list. */
3143	pLast->pNextR3 = pVM->pgm.s.pRegMmioRangesR3;
3144	pVM->pgm.s.pRegMmioRangesR3 = pNew;
3145
3146	/* Insert the MMIO2 range/page IDs. */
3147	uint8_t idMmio2 = pNew->idMmio2;
3148	for (;;)
3149	{
3150	Assert(pVM->pgm.s.apMmio2RangesR3[idMmio2 - 1] == NULL);
3151	Assert(pVM->pgm.s.apMmio2RangesR0[idMmio2 - 1] == NIL_RTR0PTR);
3152	pVM->pgm.s.apMmio2RangesR3[idMmio2 - 1] = pNew;
3153	pVM->pgm.s.apMmio2RangesR0[idMmio2 - 1] = pNew->RamRange.pSelfR0 - RT_UOFFSETOF(PGMREGMMIO2RANGE, RamRange);
3154	if (pNew->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
3155	break;
3156	pNew = pNew->pNextR3;
3157	idMmio2++;
3158	}
3159
3160	pgmPhysInvalidatePageMapTLB(pVM);
3161	PGM_UNLOCK(pVM);
3162	}
3163	#endif
3164
3165
3166	/**
3167	* Allocate and register an MMIO2 region.
3168	*
3169	* As mentioned elsewhere, MMIO2 is just RAM spelled differently. It's RAM
3170	* associated with a device. It is also non-shared memory with a permanent
3171	* ring-3 mapping and page backing (presently).
3172	*
3173	* A MMIO2 range may overlap with base memory if a lot of RAM is configured for
3174	* the VM, in which case we'll drop the base memory pages. Presently we will
3175	* make no attempt to preserve anything that happens to be present in the base
3176	* memory that is replaced, this is of course incorrect but it's too much
3177	* effort.
3178	*
3179	* @returns VBox status code.
3180	* @retval VINF_SUCCESS on success, *ppv pointing to the R3 mapping of the
3181	* memory.
3182	* @retval VERR_ALREADY_EXISTS if the region already exists.
3183	*
3184	* @param pVM The cross context VM structure.
3185	* @param pDevIns The device instance owning the region.
3186	* @param iSubDev The sub-device number.
3187	* @param iRegion The region number. If the MMIO2 memory is a PCI
3188	* I/O region this number has to be the number of that
3189	* region. Otherwise it can be any number save
3190	* UINT8_MAX.
3191	* @param cb The size of the region. Must be page aligned.
3192	* @param fFlags Reserved for future use, must be zero.
3193	* @param pszDesc The description.
3194	* @param ppv Where to store the pointer to the ring-3 mapping of
3195	* the memory.
3196	* @param phRegion Where to return the MMIO2 region handle. Optional.
3197	* @thread EMT(0)
3198	*
3199	* @note Only callable at VM creation time and during VM state loading.
3200	* The latter is for PCNet saved state compatibility with pre 4.3.6
3201	* state.
3202	*/
3203	VMMR3_INT_DECL(int) PGMR3PhysMmio2Register(PVM pVM, PPDMDEVINS pDevIns, uint32_t iSubDev, uint32_t iRegion, RTGCPHYS cb,
3204	uint32_t fFlags, const char pszDesc, void ppv, PGMMMIO2HANDLE phRegion)
3205	{
3206	/*
3207	* Validate input.
3208	*/
3209	AssertPtrReturn(ppv, VERR_INVALID_POINTER);
3210	*ppv = NULL;
3211	if (phRegion)
3212	{
3213	AssertPtrReturn(phRegion, VERR_INVALID_POINTER);
3214	*phRegion = NIL_PGMMMIO2HANDLE;
3215	}
3216	PVMCPU const pVCpu = VMMGetCpu(pVM);
3217	AssertReturn(pVCpu && pVCpu->idCpu == 0, VERR_VM_THREAD_NOT_EMT);
3218	VMSTATE const enmVMState = VMR3GetState(pVM);
3219	AssertMsgReturn(enmVMState == VMSTATE_CREATING \|\| enmVMState == VMSTATE_LOADING,
3220	("state %s, expected CREATING or LOADING\n", VMGetStateName(enmVMState)),
3221	VERR_VM_INVALID_VM_STATE);
3222
3223	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3224	AssertReturn(iSubDev <= UINT8_MAX, VERR_INVALID_PARAMETER);
3225	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
3226
3227	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
3228	AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
3229
3230	AssertReturn(!(cb & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3231	AssertReturn(cb, VERR_INVALID_PARAMETER);
3232	AssertReturn(!(fFlags & ~PGMPHYS_MMIO2_FLAGS_VALID_MASK), VERR_INVALID_FLAGS);
3233
3234	const uint32_t cGuestPages = cb >> GUEST_PAGE_SHIFT;
3235	AssertLogRelReturn(((RTGCPHYS)cGuestPages << GUEST_PAGE_SHIFT) == cb, VERR_INVALID_PARAMETER);
3236	AssertLogRelReturn(cGuestPages <= PGM_MAX_PAGES_PER_MMIO2_REGION, VERR_OUT_OF_RANGE);
3237	AssertLogRelReturn(cGuestPages <= (MM_MMIO_64_MAX >> GUEST_PAGE_SHIFT), VERR_OUT_OF_RANGE);
3238
3239	AssertReturn(pgmR3PhysMmio2Find(pVM, pDevIns, iSubDev, iRegion) == NULL, VERR_ALREADY_EXISTS);
3240
3241	/*
3242	* For the 2nd+ instance, mangle the description string so it's unique.
3243	*/
3244	if (pDevIns->iInstance > 0) /** @todo Move to PDMDevHlp.cpp and use a real string cache. */
3245	{
3246	pszDesc = MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s [%u]", pszDesc, pDevIns->iInstance);
3247	if (!pszDesc)
3248	return VERR_NO_MEMORY;
3249	}
3250
3251	/*
3252	* Check that we've got sufficient MMIO2 ID space for this request (the
3253	* allocation will be done later once we've got the backing memory secured,
3254	* but given the EMT0 restriction, that's not going to be a problem).
3255	*
3256	* The zero ID is not used as it could be confused with NIL_GMM_PAGEID, so
3257	* the IDs goes from 1 thru PGM_MAX_MMIO2_RANGES.
3258	*/
3259	unsigned const cChunks = pgmPhysMmio2CalcChunkCount(cb, NULL);
3260
3261	int rc = PGM_LOCK(pVM);
3262	AssertRCReturn(rc, rc);
3263
3264	AssertCompile(PGM_MAX_MMIO2_RANGES < 255);
3265	uint8_t const idMmio2 = pVM->pgm.s.cMmio2Ranges + 1;
3266	AssertLogRelReturnStmt(idMmio2 + cChunks <= PGM_MAX_MMIO2_RANGES, PGM_UNLOCK(pVM), VERR_PGM_TOO_MANY_MMIO2_RANGES);
3267
3268	/*
3269	* Try reserve and allocate the backing memory first as this is what is
3270	* most likely to fail.
3271	*/
3272	rc = MMR3AdjustFixedReservation(pVM, cGuestPages, pszDesc);
3273	if (RT_SUCCESS(rc))
3274	{
3275	/*
3276	* If we're in driverless we'll be doing the work here, otherwise we
3277	* must call ring-0 to do the job as we'll need physical addresses
3278	* and maybe a ring-0 mapping address for it all.
3279	*/
3280	if (SUPR3IsDriverless())
3281	rc = pgmPhysMmio2RegisterWorker(pVM, cGuestPages, idMmio2, cChunks, pDevIns, iSubDev, iRegion, fFlags);
3282	else
3283	{
3284	PGMPHYSMMIO2REGISTERREQ Mmio2RegReq;
3285	Mmio2RegReq.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
3286	Mmio2RegReq.Hdr.cbReq = sizeof(Mmio2RegReq);
3287	Mmio2RegReq.cbGuestPage = GUEST_PAGE_SIZE;
3288	Mmio2RegReq.cGuestPages = cGuestPages;
3289	Mmio2RegReq.idMmio2 = idMmio2;
3290	Mmio2RegReq.cChunks = cChunks;
3291	Mmio2RegReq.iSubDev = (uint8_t)iSubDev;
3292	Mmio2RegReq.iRegion = (uint8_t)iRegion;
3293	Mmio2RegReq.fFlags = fFlags;
3294	Mmio2RegReq.pDevIns = pDevIns;
3295	rc = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_PGM_PHYS_MMIO2_REGISTER, 0 /u64Arg/, &Mmio2RegReq.Hdr);
3296	}
3297	if (RT_SUCCESS(rc))
3298	{
3299	Assert(idMmio2 + cChunks - 1 == pVM->pgm.s.cMmio2Ranges);
3300
3301	/*
3302	* There are two things left to do:
3303	* 1. Add the description to the associated RAM ranges.
3304	* 2. Pre-allocate access handlers for dirty bit tracking if necessary.
3305	*/
3306	bool const fNeedHandler = (fFlags & PGMPHYS_MMIO2_FLAGS_TRACK_DIRTY_PAGES)
3307	#ifdef VBOX_WITH_PGM_NEM_MODE
3308	&& (!VM_IS_NEM_ENABLED(pVM) \|\| !NEMR3IsMmio2DirtyPageTrackingSupported(pVM))
3309	#endif
3310	;
3311	for (uint32_t idxChunk = 0; idxChunk < cChunks; idxChunk++)
3312	{
3313	PPGMREGMMIO2RANGE const pMmio2 = &pVM->pgm.s.aMmio2Ranges[idxChunk + idMmio2 - 1];
3314	Assert(pMmio2->idRamRange < RT_ELEMENTS(pVM->pgm.s.apRamRanges));
3315	PPGMRAMRANGE const pRamRange = pVM->pgm.s.apRamRanges[pMmio2->idRamRange];
3316	Assert(pRamRange->pbR3 == pMmio2->pbR3);
3317	Assert(pRamRange->cb == pMmio2->cbReal);
3318
3319	pRamRange->pszDesc = pszDesc; /** @todo mangle this if we got more than one chunk */
3320	if (fNeedHandler)
3321	{
3322	rc = pgmHandlerPhysicalExCreate(pVM, pVM->pgm.s.hMmio2DirtyPhysHandlerType, pMmio2->idMmio2,
3323	pszDesc, &pMmio2->pPhysHandlerR3);
3324	AssertLogRelMsgReturnStmt(RT_SUCCESS(rc),
3325	("idMmio2=%#x idxChunk=%#x rc=%Rc\n", idMmio2, idxChunk, rc),
3326	PGM_UNLOCK(pVM),
3327	rc); /* PGMR3Term will take care of it all. */
3328	}
3329	}
3330
3331	/*
3332	* Done!
3333	*/
3334	if (phRegion)
3335	*phRegion = idMmio2;
3336	*ppv = pVM->pgm.s.aMmio2Ranges[idMmio2 - 1].pbR3;
3337
3338	PGM_UNLOCK(pVM);
3339	return VINF_SUCCESS;
3340	}
3341
3342	MMR3AdjustFixedReservation(pVM, -(int32_t)cGuestPages, pszDesc);
3343	}
3344	if (pDevIns->iInstance > 0)
3345	MMR3HeapFree((void *)pszDesc);
3346	return rc;
3347	}
3348
3349	/**
3350	* Deregisters and frees an MMIO2 region.
3351	*
3352	* Any physical access handlers registered for the region must be deregistered
3353	* before calling this function.
3354	*
3355	* @returns VBox status code.
3356	* @param pVM The cross context VM structure.
3357	* @param pDevIns The device instance owning the region.
3358	* @param hMmio2 The MMIO2 handle to deregister, or NIL if all
3359	* regions for the given device is to be deregistered.
3360	* @thread EMT(0)
3361	*
3362	* @note Only callable during VM state loading. This is to jettison an unused
3363	* MMIO2 section present in PCNet saved state prior to VBox v4.3.6.
3364	*/
3365	VMMR3_INT_DECL(int) PGMR3PhysMmio2Deregister(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2)
3366	{
3367	/*
3368	* Validate input.
3369	*/
3370	PVMCPU const pVCpu = VMMGetCpu(pVM);
3371	AssertReturn(pVCpu && pVCpu->idCpu == 0, VERR_VM_THREAD_NOT_EMT);
3372	VMSTATE const enmVMState = VMR3GetState(pVM);
3373	AssertMsgReturn(enmVMState == VMSTATE_LOADING,
3374	("state %s, expected LOADING\n", VMGetStateName(enmVMState)),
3375	VERR_VM_INVALID_VM_STATE);
3376
3377	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3378
3379	/*
3380	* Take the PGM lock and scan for registrations matching the requirements.
3381	* We do this backwards to more easily reduce the cMmio2Ranges count when
3382	* stuff is removed.
3383	*/
3384	PGM_LOCK_VOID(pVM);
3385
3386	int rc = VINF_SUCCESS;
3387	unsigned cFound = 0;
3388	uint32_t const cMmio2Ranges = RT_MIN(pVM->pgm.s.cMmio2Ranges, RT_ELEMENTS(pVM->pgm.s.aMmio2Ranges));
3389	uint32_t idx = cMmio2Ranges;
3390	while (idx-- > 0)
3391	{
3392	PPGMREGMMIO2RANGE pCur = &pVM->pgm.s.aMmio2Ranges[idx];
3393	if ( pCur->pDevInsR3 == pDevIns
3394	&& ( hMmio2 == NIL_PGMMMIO2HANDLE
3395	\|\| pCur->idMmio2 == hMmio2))
3396	{
3397	cFound++;
3398
3399	/*
3400	* Wind back the first chunk for this registration.
3401	*/
3402	AssertLogRelMsgReturnStmt(pCur->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK, ("idx=%u fFlags=%#x\n", idx, pCur->fFlags),
3403	PGM_UNLOCK(pVM), VERR_INTERNAL_ERROR_3);
3404	uint32_t cGuestPages = pCur->cbReal >> GUEST_PAGE_SHIFT;
3405	uint32_t cChunks = 1;
3406	while ( idx > 0
3407	&& !(pCur->fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK))
3408	{
3409	AssertLogRelMsgReturnStmt( pCur[-1].pDevInsR3 == pDevIns
3410	&& pCur[-1].iRegion == pCur->iRegion
3411	&& pCur[-1].iSubDev == pCur->iSubDev,
3412	("[%u]: %p/%#x/%#x/fl=%#x; [%u]: %p/%#x/%#x/fl=%#x; cChunks=%#x\n",
3413	idx - 1, pCur[-1].pDevInsR3, pCur[-1].iRegion, pCur[-1].iSubDev, pCur[-1].fFlags,
3414	idx, pCur->pDevInsR3, pCur->iRegion, pCur->iSubDev, pCur->fFlags, cChunks),
3415	PGM_UNLOCK(pVM), VERR_INTERNAL_ERROR_3);
3416	cChunks++;
3417	pCur--;
3418	idx--;
3419	cGuestPages += pCur->cbReal >> GUEST_PAGE_SHIFT;
3420	}
3421	AssertLogRelMsgReturnStmt(pCur->fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK,
3422	("idx=%u fFlags=%#x cChunks=%#x\n", idx, pCur->fFlags, cChunks),
3423	PGM_UNLOCK(pVM), VERR_INTERNAL_ERROR_3);
3424
3425	/*
3426	* Unmap it if it's mapped.
3427	*/
3428	if (pCur->fFlags & PGMREGMMIO2RANGE_F_MAPPED)
3429	{
3430	int rc2 = PGMR3PhysMmio2Unmap(pVM, pCur->pDevInsR3, idx + 1, pCur->GCPhys);
3431	AssertRC(rc2);
3432	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
3433	rc = rc2;
3434	}
3435
3436	/*
3437	* Destroy access handlers.
3438	*/
3439	for (uint32_t iChunk = 0; iChunk < cChunks; iChunk++)
3440	if (pCur[iChunk].pPhysHandlerR3)
3441	{
3442	pgmHandlerPhysicalExDestroy(pVM, pCur[iChunk].pPhysHandlerR3);
3443	pCur[iChunk].pPhysHandlerR3 = NULL;
3444	}
3445
3446	/*
3447	* Call kernel mode / worker to do the actual deregistration.
3448	*/
3449	const char * const pszDesc = pVM->pgm.s.apMmio2RamRanges[idx] ? pVM->pgm.s.apMmio2RamRanges[idx]->pszDesc : NULL;
3450	int rc2;
3451	if (SUPR3IsDriverless())
3452	{
3453	Assert(PGM_IS_IN_NEM_MODE(pVM));
3454	rc2 = pgmPhysMmio2DeregisterWorker(pVM, idx, cChunks, pDevIns);
3455	AssertLogRelMsgStmt(RT_SUCCESS(rc2),
3456	("pgmPhysMmio2DeregisterWorker: rc=%Rrc idx=%#x cChunks=%#x %s\n",
3457	rc2, idx, cChunks, pszDesc),
3458	rc = RT_SUCCESS(rc) ? rc2 : rc);
3459	}
3460	else
3461	{
3462	PGMPHYSMMIO2DEREGISTERREQ Mmio2DeregReq;
3463	Mmio2DeregReq.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
3464	Mmio2DeregReq.Hdr.cbReq = sizeof(Mmio2DeregReq);
3465	Mmio2DeregReq.idMmio2 = idx + 1;
3466	Mmio2DeregReq.cChunks = cChunks;
3467	Mmio2DeregReq.pDevIns = pDevIns;
3468	rc2 = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_PGM_PHYS_MMIO2_DEREGISTER, 0 /u64Arg/, &Mmio2DeregReq.Hdr);
3469	AssertLogRelMsgStmt(RT_SUCCESS(rc2),
3470	("VMMR0_DO_PGM_PHYS_MMIO2_DEREGISTER: rc=%Rrc idx=%#x cChunks=%#x %s\n",
3471	rc2, idx, cChunks, pszDesc),
3472	rc = RT_SUCCESS(rc) ? rc2 : rc);
3473	pgmPhysInvalidRamRangeTlbs(pVM); /* Ensure no stale pointers in the ring-3 RAM range TLB. */
3474	}
3475	if (RT_FAILURE(rc2))
3476	{
3477	LogRel(("PGMR3PhysMmio2Deregister: Deregistration failed: %Rrc; cChunks=%u %s\n", rc, cChunks, pszDesc));
3478	if (RT_SUCCESS(rc))
3479	rc = rc2;
3480	}
3481
3482	/*
3483	* Adjust the memory reservation.
3484	*/
3485	if (!PGM_IS_IN_NEM_MODE(pVM) && RT_SUCCESS(rc2))
3486	{
3487	rc2 = MMR3AdjustFixedReservation(pVM, -(int32_t)cGuestPages, pszDesc);
3488	AssertLogRelMsgStmt(RT_SUCCESS(rc2), ("rc=%Rrc cGuestPages=%#x\n", rc, cGuestPages),
3489	rc = RT_SUCCESS(rc) ? rc2 : rc);
3490	}
3491
3492	/* Are we done? */
3493	if (hMmio2 != NIL_PGMMMIO2HANDLE)
3494	break;
3495	}
3496	}
3497	pgmPhysInvalidatePageMapTLB(pVM, false /fInRendezvous/);
3498	PGM_UNLOCK(pVM);
3499	return !cFound && hMmio2 != NIL_PGMMMIO2HANDLE ? VERR_NOT_FOUND : rc;
3500	}
3501
3502
3503	/**
3504	* Worker form PGMR3PhysMmio2Map.
3505	*/
3506	static int pgmR3PhysMmio2MapLocked(PVM pVM, uint32_t const idxFirst, uint32_t const cChunks,
3507	RTGCPHYS const GCPhys, RTGCPHYS const GCPhysLast)
3508	{
3509	/*
3510	* Validate the mapped status now that we've got the lock.
3511	*/
3512	for (uint32_t iChunk = 0, idx = idxFirst; iChunk < cChunks; iChunk++, idx++)
3513	{
3514	AssertReturn( pVM->pgm.s.aMmio2Ranges[idx].GCPhys == NIL_RTGCPHYS
3515	&& !(pVM->pgm.s.aMmio2Ranges[idx].fFlags & PGMREGMMIO2RANGE_F_MAPPED),
3516	VERR_WRONG_ORDER);
3517	PPGMRAMRANGE const pRamRange = pVM->pgm.s.apMmio2RamRanges[idx];
3518	AssertReturn(pRamRange->GCPhys == NIL_RTGCPHYS, VERR_INTERNAL_ERROR_3);
3519	AssertReturn(pRamRange->GCPhysLast == NIL_RTGCPHYS, VERR_INTERNAL_ERROR_3);
3520	Assert(pRamRange->pbR3 == pVM->pgm.s.aMmio2Ranges[idx].pbR3);
3521	Assert(pRamRange->idRange == pVM->pgm.s.aMmio2Ranges[idx].idRamRange);
3522	}
3523
3524	const char * const pszDesc = pVM->pgm.s.apMmio2RamRanges[idxFirst]->pszDesc;
3525	#ifdef VBOX_WITH_NATIVE_NEM
3526	uint32_t const fNemFlags = NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2
3527	\| (pVM->pgm.s.aMmio2Ranges[idxFirst].fFlags & PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES
3528	? NEM_NOTIFY_PHYS_MMIO_EX_F_TRACK_DIRTY_PAGES : 0);
3529	#endif
3530
3531	/*
3532	* Now, check if this falls into a regular RAM range or if we should use
3533	* the ad-hoc one.
3534	*
3535	* Note! For reasons of simplictly, we're considering the whole MMIO2 area
3536	* here rather than individual chunks.
3537	*/
3538	int rc = VINF_SUCCESS;
3539	uint32_t idxInsert = UINT32_MAX;
3540	PPGMRAMRANGE const pOverlappingRange = pgmR3PhysRamRangeFindOverlapping(pVM, GCPhys, GCPhysLast, &idxInsert);
3541	if (pOverlappingRange)
3542	{
3543	/* Simplification: all within the same range. */
3544	AssertLogRelMsgReturn( GCPhys >= pOverlappingRange->GCPhys
3545	&& GCPhysLast <= pOverlappingRange->GCPhysLast,
3546	("%RGp-%RGp (MMIO2/%s) falls partly outside %RGp-%RGp (%s)\n",
3547	GCPhys, GCPhysLast, pszDesc,
3548	pOverlappingRange->GCPhys, pOverlappingRange->GCPhysLast, pOverlappingRange->pszDesc),
3549	VERR_PGM_RAM_CONFLICT);
3550
3551	/* Check that is isn't an ad hoc range, but a real RAM range. */
3552	AssertLogRelMsgReturn(!PGM_RAM_RANGE_IS_AD_HOC(pOverlappingRange),
3553	("%RGp-%RGp (MMIO2/%s) mapping attempt in non-RAM range: %RGp-%RGp (%s)\n",
3554	GCPhys, GCPhysLast, pszDesc,
3555	pOverlappingRange->GCPhys, pOverlappingRange->GCPhysLast, pOverlappingRange->pszDesc),
3556	VERR_PGM_RAM_CONFLICT);
3557
3558	/* There can only be one MMIO2 chunk matching here! */
3559	AssertLogRelMsgReturn(cChunks == 1,
3560	("%RGp-%RGp (MMIO2/%s) consists of %u chunks whereas the RAM (%s) somehow doesn't!\n",
3561	GCPhys, GCPhysLast, pszDesc, cChunks, pOverlappingRange->pszDesc),
3562	VERR_PGM_PHYS_MMIO_EX_IPE);
3563
3564	/* Check that it's all RAM pages. */
3565	PCPGMPAGE pPage = &pOverlappingRange->aPages[(GCPhys - pOverlappingRange->GCPhys) >> GUEST_PAGE_SHIFT];
3566	uint32_t const cMmio2Pages = pVM->pgm.s.apMmio2RamRanges[idxFirst]->cb >> GUEST_PAGE_SHIFT;
3567	uint32_t cPagesLeft = cMmio2Pages;
3568	while (cPagesLeft-- > 0)
3569	{
3570	AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
3571	("%RGp-%RGp (MMIO2/%s): %RGp is not a RAM page - type=%d desc=%s\n", GCPhys, GCPhysLast,
3572	pszDesc, pOverlappingRange->GCPhys, PGM_PAGE_GET_TYPE(pPage), pOverlappingRange->pszDesc),
3573	VERR_PGM_RAM_CONFLICT);
3574	pPage++;
3575	}
3576
3577	#ifdef VBOX_WITH_PGM_NEM_MODE
3578	/* We cannot mix MMIO2 into a RAM range in simplified memory mode because pOverlappingRange->pbR3 can't point
3579	both at the RAM and MMIO2, so we won't ever write & read from the actual MMIO2 memory if we try. */
3580	AssertLogRelMsgReturn(!VM_IS_NEM_ENABLED(pVM),
3581	("Putting %s at %RGp-%RGp is not possible in NEM mode because existing %RGp-%RGp (%s) mapping\n",
3582	pszDesc, GCPhys, GCPhysLast,
3583	pOverlappingRange->GCPhys, pOverlappingRange->GCPhysLast, pOverlappingRange->pszDesc),
3584	VERR_PGM_NOT_SUPPORTED_FOR_NEM_MODE);
3585	#endif
3586
3587	/*
3588	* Make all the pages in the range MMIO/ZERO pages, freeing any
3589	* RAM pages currently mapped here. This might not be 100% correct,
3590	* but so what, we do the same from MMIO...
3591	*/
3592	rc = pgmR3PhysFreePageRange(pVM, pOverlappingRange, GCPhys, GCPhysLast, NULL);
3593	AssertRCReturn(rc, rc);
3594
3595	Log(("PGMR3PhysMmio2Map: %RGp-%RGp %s - inside %RGp-%RGp %s\n", GCPhys, GCPhysLast, pszDesc,
3596	pOverlappingRange->GCPhys, pOverlappingRange->GCPhysLast, pOverlappingRange->pszDesc));
3597
3598	/*
3599	* We're all in for mapping it now. Update the MMIO2 range to reflect it.
3600	*/
3601	pVM->pgm.s.aMmio2Ranges[idxFirst].GCPhys = GCPhys;
3602	pVM->pgm.s.aMmio2Ranges[idxFirst].fFlags \|= PGMREGMMIO2RANGE_F_OVERLAPPING \| PGMREGMMIO2RANGE_F_MAPPED;
3603
3604	/*
3605	* Replace the pages in the range.
3606	*/
3607	PPGMPAGE pPageSrc = &pVM->pgm.s.apMmio2RamRanges[idxFirst]->aPages[0];
3608	PPGMPAGE pPageDst = &pOverlappingRange->aPages[(GCPhys - pOverlappingRange->GCPhys) >> GUEST_PAGE_SHIFT];
3609	cPagesLeft = cMmio2Pages;
3610	while (cPagesLeft-- > 0)
3611	{
3612	Assert(PGM_PAGE_IS_MMIO(pPageDst));
3613
3614	RTHCPHYS const HCPhys = PGM_PAGE_GET_HCPHYS(pPageSrc);
3615	uint32_t const idPage = PGM_PAGE_GET_PAGEID(pPageSrc);
3616	PGM_PAGE_SET_PAGEID(pVM, pPageDst, idPage);
3617	PGM_PAGE_SET_HCPHYS(pVM, pPageDst, HCPhys);
3618	PGM_PAGE_SET_TYPE(pVM, pPageDst, PGMPAGETYPE_MMIO2);
3619	PGM_PAGE_SET_STATE(pVM, pPageDst, PGM_PAGE_STATE_ALLOCATED);
3620	PGM_PAGE_SET_PDE_TYPE(pVM, pPageDst, PGM_PAGE_PDE_TYPE_DONTCARE);
3621	PGM_PAGE_SET_PTE_INDEX(pVM, pPageDst, 0);
3622	PGM_PAGE_SET_TRACKING(pVM, pPageDst, 0);
3623	/* NEM state is not relevant, see VERR_PGM_NOT_SUPPORTED_FOR_NEM_MODE above. */
3624
3625	pVM->pgm.s.cZeroPages--;
3626	pPageSrc++;
3627	pPageDst++;
3628	}
3629
3630	/* Force a PGM pool flush as guest ram references have been changed. */
3631	/** @todo not entirely SMP safe; assuming for now the guest takes
3632	* care of this internally (not touch mapped mmio while changing the
3633	* mapping). */
3634	PVMCPU pVCpu = VMMGetCpu(pVM);
3635	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
3636	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3637	}
3638	else
3639	{
3640	/*
3641	* No RAM range, insert the ones prepared during registration.
3642	*/
3643	Log(("PGMR3PhysMmio2Map: %RGp-%RGp %s - no RAM overlap\n", GCPhys, GCPhysLast, pszDesc));
3644	RTGCPHYS GCPhysCur = GCPhys;
3645	uint32_t iChunk = 0;
3646	uint32_t idx = idxFirst;
3647	for (; iChunk < cChunks; iChunk++, idx++)
3648	{
3649	PPGMREGMMIO2RANGE const pMmio2 = &pVM->pgm.s.aMmio2Ranges[idx];
3650	PPGMRAMRANGE const pRamRange = pVM->pgm.s.apMmio2RamRanges[idx];
3651	Assert(pRamRange->idRange == pMmio2->idRamRange);
3652	Assert(pMmio2->GCPhys == NIL_RTGCPHYS);
3653
3654	#ifdef VBOX_WITH_NATIVE_NEM
3655	/* Tell NEM and get the new NEM state for the pages. */
3656	uint8_t u2NemState = 0;
3657	if (VM_IS_NEM_ENABLED(pVM))
3658	{
3659	rc = NEMR3NotifyPhysMmioExMapEarly(pVM, GCPhysCur, pRamRange->cb, fNemFlags, NULL /pvRam/, pRamRange->pbR3,
3660	&u2NemState, &pRamRange->uNemRange);
3661	AssertLogRelMsgBreak(RT_SUCCESS(rc),
3662	("%RGp LB %RGp fFlags=%#x (%s)\n",
3663	GCPhysCur, pRamRange->cb, pMmio2->fFlags, pRamRange->pszDesc));
3664	pMmio2->fFlags \|= PGMREGMMIO2RANGE_F_MAPPED; /* Set this early to indicate that NEM has been notified. */
3665	}
3666	#endif
3667
3668	/* Clear the tracking data of pages we're going to reactivate. */
3669	PPGMPAGE pPageSrc = &pRamRange->aPages[0];
3670	uint32_t cPagesLeft = pRamRange->cb >> GUEST_PAGE_SHIFT;
3671	while (cPagesLeft-- > 0)
3672	{
3673	PGM_PAGE_SET_TRACKING(pVM, pPageSrc, 0);
3674	PGM_PAGE_SET_PTE_INDEX(pVM, pPageSrc, 0);
3675	#ifdef VBOX_WITH_NATIVE_NEM
3676	PGM_PAGE_SET_NEM_STATE(pPageSrc, u2NemState);
3677	#endif
3678	pPageSrc++;
3679	}
3680
3681	/* Insert the RAM range into the lookup table. */
3682	rc = pgmR3PhysRamRangeInsertLookup(pVM, pRamRange, GCPhysCur, &idxInsert);
3683	AssertRCBreak(rc);
3684
3685	/* Mark the range as fully mapped. */
3686	pMmio2->fFlags &= ~PGMREGMMIO2RANGE_F_OVERLAPPING;
3687	pMmio2->fFlags \|= PGMREGMMIO2RANGE_F_MAPPED;
3688	pMmio2->GCPhys = GCPhysCur;
3689
3690	/* Advance. */
3691	GCPhysCur += pRamRange->cb;
3692	}
3693	if (RT_FAILURE(rc))
3694	{
3695	/*
3696	* Bail out anything we've done so far.
3697	*/
3698	idxInsert -= 1;
3699	do
3700	{
3701	PPGMREGMMIO2RANGE const pMmio2 = &pVM->pgm.s.aMmio2Ranges[idx];
3702	PPGMRAMRANGE const pRamRange = pVM->pgm.s.apMmio2RamRanges[idx];
3703
3704	#ifdef VBOX_WITH_NATIVE_NEM
3705	if ( VM_IS_NEM_ENABLED(pVM)
3706	&& (pVM->pgm.s.aMmio2Ranges[idx].fFlags & PGMREGMMIO2RANGE_F_MAPPED))
3707	{
3708	uint8_t u2NemState = UINT8_MAX;
3709	NEMR3NotifyPhysMmioExUnmap(pVM, GCPhysCur, pRamRange->cb, fNemFlags, NULL, pRamRange->pbR3,
3710	&u2NemState, &pRamRange->uNemRange);
3711	if (u2NemState != UINT8_MAX)
3712	pgmPhysSetNemStateForPages(pRamRange->aPages, pRamRange->cb >> GUEST_PAGE_SHIFT, u2NemState);
3713	}
3714	#endif
3715	if (pMmio2->GCPhys != NIL_RTGCPHYS)
3716	pgmR3PhysRamRangeRemoveLookup(pVM, pRamRange, &idxInsert);
3717
3718	pMmio2->GCPhys = NIL_RTGCPHYS;
3719	pMmio2->fFlags &= ~PGMREGMMIO2RANGE_F_MAPPED;
3720
3721	idx--;
3722	} while (iChunk-- > 0);
3723	return rc;
3724	}
3725	}
3726
3727	/*
3728	* If the range have dirty page monitoring enabled, enable that.
3729	*
3730	* We ignore failures here for now because if we fail, the whole mapping
3731	* will have to be reversed and we'll end up with nothing at all on the
3732	* screen and a grumpy guest, whereas if we just go on, we'll only have
3733	* visual distortions to gripe about. There will be something in the
3734	* release log.
3735	*/
3736	if ( pVM->pgm.s.aMmio2Ranges[idxFirst].pPhysHandlerR3
3737	&& (pVM->pgm.s.aMmio2Ranges[idxFirst].fFlags & PGMREGMMIO2RANGE_F_TRACKING_ENABLED))
3738	pgmR3PhysMmio2EnableDirtyPageTracing(pVM, idxFirst, cChunks);
3739
3740	/* Flush physical page map TLB. */
3741	pgmPhysInvalidatePageMapTLB(pVM, false /fInRendezvous/);
3742
3743	#ifdef VBOX_WITH_NATIVE_NEM
3744	/*
3745	* Late NEM notification (currently unused).
3746	*/
3747	if (VM_IS_NEM_ENABLED(pVM))
3748	{
3749	if (pOverlappingRange)
3750	{
3751	uint8_t * const pbRam = pOverlappingRange->pbR3 ? &pOverlappingRange->pbR3[GCPhys - pOverlappingRange->GCPhys] : NULL;
3752	rc = NEMR3NotifyPhysMmioExMapLate(pVM, GCPhys, GCPhysLast - GCPhys + 1U,
3753	fNemFlags \| NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE, pbRam,
3754	pVM->pgm.s.aMmio2Ranges[idxFirst].pbR3, NULL /puNemRange/);
3755	}
3756	else
3757	{
3758	for (uint32_t iChunk = 0, idx = idxFirst; iChunk < cChunks; iChunk++, idx++)
3759	{
3760	PPGMRAMRANGE const pRamRange = pVM->pgm.s.apMmio2RamRanges[idx];
3761	Assert(pVM->pgm.s.aMmio2Ranges[idx].GCPhys == pRamRange->GCPhys);
3762
3763	rc = NEMR3NotifyPhysMmioExMapLate(pVM, pRamRange->GCPhys, pRamRange->cb, fNemFlags, NULL /pvRam/,
3764	pRamRange->pbR3, &pRamRange->uNemRange);
3765	AssertRCBreak(rc);
3766	}
3767	}
3768	AssertLogRelRCReturnStmt(rc,
3769	PGMR3PhysMmio2Unmap(pVM, pVM->pgm.s.aMmio2Ranges[idxFirst].pDevInsR3, idxFirst + 1, GCPhys),
3770	rc);
3771	}
3772	#endif
3773
3774	return VINF_SUCCESS;
3775	}
3776
3777
3778	/**
3779	* Maps a MMIO2 region.
3780	*
3781	* This is typically done when a guest / the bios / state loading changes the
3782	* PCI config. The replacing of base memory has the same restrictions as during
3783	* registration, of course.
3784	*
3785	* @returns VBox status code.
3786	*
3787	* @param pVM The cross context VM structure.
3788	* @param pDevIns The device instance owning the region.
3789	* @param hMmio2 The handle of the region to map.
3790	* @param GCPhys The guest-physical address to be remapped.
3791	*/
3792	VMMR3_INT_DECL(int) PGMR3PhysMmio2Map(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, RTGCPHYS GCPhys)
3793	{
3794	/*
3795	* Validate input.
3796	*/
3797	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
3798	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3799	AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
3800	AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
3801	AssertReturn(!(GCPhys & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3802	AssertReturn(hMmio2 != NIL_PGMMMIO2HANDLE, VERR_INVALID_HANDLE);
3803
3804	uint32_t cChunks = 0;
3805	uint32_t const idxFirst = pgmR3PhysMmio2ResolveHandle(pVM, pDevIns, hMmio2, &cChunks);
3806	AssertReturn((int32_t)idxFirst >= 0, (int32_t)idxFirst);
3807
3808	/* Gather the full range size so we can validate the mapping address properly. */
3809	RTGCPHYS cbRange = 0;
3810	for (uint32_t iChunk = 0, idx = idxFirst; iChunk < cChunks; iChunk++, idx++)
3811	cbRange += pVM->pgm.s.apMmio2RamRanges[idx]->cb;
3812
3813	RTGCPHYS const GCPhysLast = GCPhys + cbRange - 1;
3814	AssertLogRelReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
3815
3816	/*
3817	* Take the PGM lock and call worker.
3818	*/
3819	int rc = PGM_LOCK(pVM);
3820	AssertRCReturn(rc, rc);
3821
3822	rc = pgmR3PhysMmio2MapLocked(pVM, idxFirst, cChunks, GCPhys, GCPhysLast);
3823	#ifdef VBOX_STRICT
3824	pgmPhysAssertRamRangesLocked(pVM, false /fInUpdate/, false /fRamRelaxed/);
3825	#endif
3826
3827	PGM_UNLOCK(pVM);
3828	return rc;
3829	}
3830
3831
3832	/**
3833	* Worker form PGMR3PhysMmio2Map.
3834	*/
3835	static int pgmR3PhysMmio2UnmapLocked(PVM pVM, uint32_t const idxFirst, uint32_t const cChunks, RTGCPHYS const GCPhysIn)
3836	{
3837	/*
3838	* Validate input.
3839	*/
3840	RTGCPHYS cbRange = 0;
3841	for (uint32_t iChunk = 0, idx = idxFirst; iChunk < cChunks; iChunk++, idx++)
3842	{
3843	PPGMREGMMIO2RANGE const pMmio2 = &pVM->pgm.s.aMmio2Ranges[idx];
3844	PPGMRAMRANGE const pRamRange = pVM->pgm.s.apMmio2RamRanges[idx];
3845	AssertReturn(pMmio2->idRamRange == pRamRange->idRange, VERR_INTERNAL_ERROR_3);
3846	AssertReturn(pMmio2->fFlags & PGMREGMMIO2RANGE_F_MAPPED, VERR_WRONG_ORDER);
3847	AssertReturn(pMmio2->GCPhys != NIL_RTGCPHYS, VERR_WRONG_ORDER);
3848	cbRange += pRamRange->cb;
3849	}
3850
3851	PPGMREGMMIO2RANGE const pFirstMmio2 = &pVM->pgm.s.aMmio2Ranges[idxFirst];
3852	PPGMRAMRANGE const pFirstRamRange = pVM->pgm.s.apMmio2RamRanges[idxFirst];
3853	const char * const pszDesc = pFirstRamRange->pszDesc;
3854	AssertLogRelMsgReturn(GCPhysIn == pFirstMmio2->GCPhys \|\| GCPhysIn == NIL_RTGCPHYS,
3855	("GCPhys=%RGp, actual address is %RGp\n", GCPhysIn, pFirstMmio2->GCPhys),
3856	VERR_MISMATCH);
3857	RTGCPHYS const GCPhys = pFirstMmio2->GCPhys; /* (it's always NIL_RTGCPHYS) */
3858	Log(("PGMR3PhysMmio2Unmap: %RGp-%RGp %s\n", GCPhys, GCPhys + cbRange - 1U, pszDesc));
3859
3860	uint16_t const fOldFlags = pFirstMmio2->fFlags;
3861	Assert(fOldFlags & PGMREGMMIO2RANGE_F_MAPPED);
3862
3863	/* Find the first entry in the lookup table and verify the overlapping flag. */
3864	uint32_t idxLookup = pgmR3PhysRamRangeFindOverlappingIndex(pVM, GCPhys, GCPhys + pFirstRamRange->cb - 1U);
3865	AssertLogRelMsgReturn(idxLookup < pVM->pgm.s.RamRangeUnion.cLookupEntries,
3866	("MMIO2 range not found at %RGp LB %RGp in the lookup table! (%s)\n",
3867	GCPhys, pFirstRamRange->cb, pszDesc),
3868	VERR_INTERNAL_ERROR_2);
3869
3870	uint32_t const idLookupRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
3871	AssertLogRelReturn(idLookupRange != 0 && idLookupRange <= pVM->pgm.s.idRamRangeMax, VERR_INTERNAL_ERROR_5);
3872	PPGMRAMRANGE const pLookupRange = pVM->pgm.s.apRamRanges[idLookupRange];
3873	AssertLogRelReturn(pLookupRange, VERR_INTERNAL_ERROR_3);
3874
3875	AssertLogRelMsgReturn(fOldFlags & PGMREGMMIO2RANGE_F_OVERLAPPING
3876	? pLookupRange != pFirstRamRange : pLookupRange == pFirstRamRange,
3877	("MMIO2 unmap mixup at %RGp LB %RGp fl=%#x (%s) vs %RGp LB %RGp (%s)\n",
3878	GCPhys, cbRange, fOldFlags, pszDesc, pLookupRange->GCPhys, pLookupRange->cb, pLookupRange->pszDesc),
3879	VERR_INTERNAL_ERROR_4);
3880
3881	/*
3882	* If monitoring dirty pages, we must deregister the handlers first.
3883	*/
3884	if ( pFirstMmio2->pPhysHandlerR3
3885	&& (fOldFlags & PGMREGMMIO2RANGE_F_TRACKING_ENABLED))
3886	pgmR3PhysMmio2DisableDirtyPageTracing(pVM, idxFirst, cChunks);
3887
3888	/*
3889	* Unmap it.
3890	*/
3891	int rcRet = VINF_SUCCESS;
3892	#ifdef VBOX_WITH_NATIVE_NEM
3893	uint32_t const fNemFlags = NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2
3894	\| (fOldFlags & PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES
3895	? NEM_NOTIFY_PHYS_MMIO_EX_F_TRACK_DIRTY_PAGES : 0);
3896	#endif
3897	if (fOldFlags & PGMREGMMIO2RANGE_F_OVERLAPPING)
3898	{
3899	/*
3900	* We've replaced RAM, replace with zero pages.
3901	*
3902	* Note! This is where we might differ a little from a real system, because
3903	* it's likely to just show the RAM pages as they were before the
3904	* MMIO2 region was mapped here.
3905	*/
3906	/* Only one chunk allowed when overlapping! */
3907	Assert(cChunks == 1);
3908	/* No NEM stuff should ever get here, see assertion in the mapping function. */
3909	AssertReturn(!VM_IS_NEM_ENABLED(pVM), VERR_INTERNAL_ERROR_4);
3910
3911	/* Restore the RAM pages we've replaced. */
3912	PPGMPAGE pPageDst = &pLookupRange->aPages[(pFirstRamRange->GCPhys - pLookupRange->GCPhys) >> GUEST_PAGE_SHIFT];
3913	uint32_t cPagesLeft = pFirstRamRange->cb >> GUEST_PAGE_SHIFT;
3914	pVM->pgm.s.cZeroPages += cPagesLeft;
3915	while (cPagesLeft-- > 0)
3916	{
3917	PGM_PAGE_INIT_ZERO(pPageDst, pVM, PGMPAGETYPE_RAM);
3918	pPageDst++;
3919	}
3920
3921	/* Update range state. */
3922	pFirstMmio2->fFlags &= ~(PGMREGMMIO2RANGE_F_OVERLAPPING \| PGMREGMMIO2RANGE_F_MAPPED);
3923	pFirstMmio2->GCPhys = NIL_RTGCPHYS;
3924	Assert(pFirstRamRange->GCPhys == NIL_RTGCPHYS);
3925	Assert(pFirstRamRange->GCPhysLast == NIL_RTGCPHYS);
3926	}
3927	else
3928	{
3929	/*
3930	* Unlink the chunks related to the MMIO/MMIO2 region.
3931	*/
3932	for (uint32_t iChunk = 0, idx = idxFirst; iChunk < cChunks; iChunk++, idx++)
3933	{
3934	PPGMREGMMIO2RANGE const pMmio2 = &pVM->pgm.s.aMmio2Ranges[idx];
3935	PPGMRAMRANGE const pRamRange = pVM->pgm.s.apMmio2RamRanges[idx];
3936	Assert(pMmio2->idRamRange == pRamRange->idRange);
3937	Assert(pMmio2->GCPhys == pRamRange->GCPhys);
3938
3939	#ifdef VBOX_WITH_NATIVE_NEM
3940	if (VM_IS_NEM_ENABLED(pVM)) /* Notify NEM. */
3941	{
3942	uint8_t u2State = UINT8_MAX;
3943	int rc = NEMR3NotifyPhysMmioExUnmap(pVM, pRamRange->GCPhys, pRamRange->cb, fNemFlags,
3944	NULL, pMmio2->pbR3, &u2State, &pRamRange->uNemRange);
3945	AssertLogRelMsgStmt(RT_SUCCESS(rc),
3946	("NEMR3NotifyPhysMmioExUnmap failed: %Rrc - GCPhys=RGp LB %RGp fNemFlags=%#x pbR3=%p %s\n",
3947	rc, pRamRange->GCPhys, pRamRange->cb, fNemFlags, pMmio2->pbR3, pRamRange->pszDesc),
3948	rcRet = rc);
3949	if (u2State != UINT8_MAX)
3950	pgmPhysSetNemStateForPages(pRamRange->aPages, pRamRange->cb >> GUEST_PAGE_SHIFT, u2State);
3951	}
3952	#endif
3953
3954	int rc = pgmR3PhysRamRangeRemoveLookup(pVM, pRamRange, &idxLookup);
3955	AssertLogRelMsgStmt(RT_SUCCESS(rc),
3956	("pgmR3PhysRamRangeRemoveLookup failed: %Rrc - GCPhys=%RGp LB %RGp %s\n",
3957	rc, pRamRange->GCPhys, pRamRange->cb, pRamRange->pszDesc),
3958	rcRet = rc);
3959
3960	pMmio2->GCPhys = NIL_RTGCPHYS;
3961	pMmio2->fFlags &= ~(PGMREGMMIO2RANGE_F_OVERLAPPING \| PGMREGMMIO2RANGE_F_MAPPED);
3962	Assert(pRamRange->GCPhys == NIL_RTGCPHYS);
3963	Assert(pRamRange->GCPhysLast == NIL_RTGCPHYS);
3964	}
3965	}
3966
3967	/* Force a PGM pool flush as guest ram references have been changed. */
3968	/** @todo not entirely SMP safe; assuming for now the guest takes care
3969	* of this internally (not touch mapped mmio while changing the
3970	* mapping). */
3971	PVMCPU pVCpu = VMMGetCpu(pVM);
3972	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
3973	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3974
3975	pgmPhysInvalidatePageMapTLB(pVM, false /fInRendezvous/);
3976	/* pgmPhysInvalidRamRangeTlbs(pVM); - not necessary */
3977
3978	return rcRet;
3979	}
3980
3981
3982	/**
3983	* Unmaps an MMIO2 region.
3984	*
3985	* This is typically done when a guest / the bios / state loading changes the
3986	* PCI config. The replacing of base memory has the same restrictions as during
3987	* registration, of course.
3988	*/
3989	VMMR3_INT_DECL(int) PGMR3PhysMmio2Unmap(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, RTGCPHYS GCPhys)
3990	{
3991	/*
3992	* Validate input
3993	*/
3994	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
3995	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3996	AssertReturn(hMmio2 != NIL_PGMMMIO2HANDLE, VERR_INVALID_HANDLE);
3997	if (GCPhys != NIL_RTGCPHYS)
3998	{
3999	AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
4000	AssertReturn(!(GCPhys & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
4001	}
4002
4003	uint32_t cChunks = 0;
4004	uint32_t const idxFirst = pgmR3PhysMmio2ResolveHandle(pVM, pDevIns, hMmio2, &cChunks);
4005	AssertReturn((int32_t)idxFirst >= 0, (int32_t)idxFirst);
4006
4007
4008	/*
4009	* Take the PGM lock and call worker.
4010	*/
4011	int rc = PGM_LOCK(pVM);
4012	AssertRCReturn(rc, rc);
4013
4014	rc = pgmR3PhysMmio2UnmapLocked(pVM, idxFirst, cChunks, GCPhys);
4015	#ifdef VBOX_STRICT
4016	pgmPhysAssertRamRangesLocked(pVM, false /fInUpdate/, false /fRamRelaxed/);
4017	#endif
4018
4019	PGM_UNLOCK(pVM);
4020	return rc;
4021	}
4022
4023
4024	/**
4025	* Reduces the mapping size of a MMIO2 region.
4026	*
4027	* This is mainly for dealing with old saved states after changing the default
4028	* size of a mapping region. See PDMDevHlpMmio2Reduce and
4029	* PDMPCIDEV::pfnRegionLoadChangeHookR3.
4030	*
4031	* The region must not currently be mapped when making this call. The VM state
4032	* must be state restore or VM construction.
4033	*
4034	* @returns VBox status code.
4035	* @param pVM The cross context VM structure.
4036	* @param pDevIns The device instance owning the region.
4037	* @param hMmio2 The handle of the region to reduce.
4038	* @param cbRegion The new mapping size.
4039	*/
4040	VMMR3_INT_DECL(int) PGMR3PhysMmio2Reduce(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, RTGCPHYS cbRegion)
4041	{
4042	/*
4043	* Validate input
4044	*/
4045	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
4046	AssertReturn(hMmio2 != NIL_PGMMMIO2HANDLE && hMmio2 != 0 && hMmio2 <= RT_ELEMENTS(pVM->pgm.s.aMmio2Ranges),
4047	VERR_INVALID_HANDLE);
4048	AssertReturn(cbRegion >= GUEST_PAGE_SIZE, VERR_INVALID_PARAMETER);
4049	AssertReturn(!(cbRegion & GUEST_PAGE_OFFSET_MASK), VERR_UNSUPPORTED_ALIGNMENT);
4050
4051	PVMCPU const pVCpu = VMMGetCpu(pVM);
4052	AssertReturn(pVCpu && pVCpu->idCpu == 0, VERR_VM_THREAD_NOT_EMT);
4053
4054	VMSTATE const enmVmState = VMR3GetState(pVM);
4055	AssertLogRelMsgReturn( enmVmState == VMSTATE_CREATING
4056	\|\| enmVmState == VMSTATE_LOADING,
4057	("enmVmState=%d (%s)\n", enmVmState, VMR3GetStateName(enmVmState)),
4058	VERR_VM_INVALID_VM_STATE);
4059
4060	/*
4061	* Grab the PGM lock and validate the request properly.
4062	*/
4063	int rc = PGM_LOCK(pVM);
4064	AssertRCReturn(rc, rc);
4065
4066	uint32_t cChunks = 0;
4067	uint32_t const idxFirst = pgmR3PhysMmio2ResolveHandle(pVM, pDevIns, hMmio2, &cChunks);
4068	if ((int32_t)idxFirst >= 0)
4069	{
4070	PPGMREGMMIO2RANGE const pFirstMmio2 = &pVM->pgm.s.aMmio2Ranges[idxFirst];
4071	PPGMRAMRANGE const pFirstRamRange = pVM->pgm.s.apMmio2RamRanges[idxFirst];
4072	if ( !(pFirstMmio2->fFlags & PGMREGMMIO2RANGE_F_MAPPED)
4073	&& pFirstMmio2->GCPhys == NIL_RTGCPHYS)
4074	{
4075	/*
4076	* NOTE! Current implementation does not support multiple ranges.
4077	* Implement when there is a real world need and thus a testcase.
4078	*/
4079	if (cChunks == 1)
4080	{
4081	/*
4082	* The request has to be within the initial size.
4083	*/
4084	if (cbRegion <= pFirstMmio2->cbReal)
4085	{
4086	/*
4087	* All we have to do is modify the size stored in the RAM range,
4088	* as it is the one used when mapping it and such.
4089	* The two page counts stored in PGMR0PERVM remain unchanged.
4090	*/
4091	Log(("PGMR3PhysMmio2Reduce: %s changes from %#RGp bytes (%#RGp) to %#RGp bytes.\n",
4092	pFirstRamRange->pszDesc, pFirstRamRange->cb, pFirstMmio2->cbReal, cbRegion));
4093	pFirstRamRange->cb = cbRegion;
4094	rc = VINF_SUCCESS;
4095	}
4096	else
4097	{
4098	AssertLogRelMsgFailed(("MMIO2/%s: cbRegion=%#RGp > cbReal=%#RGp\n",
4099	pFirstRamRange->pszDesc, cbRegion, pFirstMmio2->cbReal));
4100	rc = VERR_OUT_OF_RANGE;
4101	}
4102	}
4103	else
4104	{
4105	AssertLogRelMsgFailed(("MMIO2/%s: more than one chunk: %d (flags=%#x)\n",
4106	pFirstRamRange->pszDesc, cChunks, pFirstMmio2->fFlags));
4107	rc = VERR_NOT_SUPPORTED;
4108	}
4109	}
4110	else
4111	{
4112	AssertLogRelMsgFailed(("MMIO2/%s: cannot change size of mapped range: %RGp..%RGp\n", pFirstRamRange->pszDesc,
4113	pFirstMmio2->GCPhys, pFirstMmio2->GCPhys + pFirstRamRange->cb - 1U));
4114	rc = VERR_WRONG_ORDER;
4115	}
4116	}
4117	else
4118	rc = (int32_t)idxFirst;
4119
4120	PGM_UNLOCK(pVM);
4121	return rc;
4122	}
4123
4124
4125	/**
4126	* Validates @a hMmio2, making sure it belongs to @a pDevIns.
4127	*
4128	* @returns VBox status code.
4129	* @param pVM The cross context VM structure.
4130	* @param pDevIns The device which allegedly owns @a hMmio2.
4131	* @param hMmio2 The handle to validate.
4132	*/
4133	VMMR3_INT_DECL(int) PGMR3PhysMmio2ValidateHandle(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2)
4134	{
4135	/*
4136	* Validate input
4137	*/
4138	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
4139	AssertPtrReturn(pDevIns, VERR_INVALID_POINTER);
4140
4141	/*
4142	* Just do this the simple way.
4143	*/
4144	int rc = PGM_LOCK_VOID(pVM);
4145	AssertRCReturn(rc, rc);
4146	uint32_t cChunks;
4147	uint32_t const idxFirst = pgmR3PhysMmio2ResolveHandle(pVM, pDevIns, hMmio2, &cChunks);
4148	PGM_UNLOCK(pVM);
4149	AssertReturn((int32_t)idxFirst >= 0, (int32_t)idxFirst);
4150	return VINF_SUCCESS;
4151	}
4152
4153
4154	/**
4155	* Gets the mapping address of an MMIO2 region.
4156	*
4157	* @returns Mapping address, NIL_RTGCPHYS if not mapped or invalid handle.
4158	*
4159	* @param pVM The cross context VM structure.
4160	* @param pDevIns The device owning the MMIO2 handle.
4161	* @param hMmio2 The region handle.
4162	*/
4163	VMMR3_INT_DECL(RTGCPHYS) PGMR3PhysMmio2GetMappingAddress(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2)
4164	{
4165	RTGCPHYS GCPhysRet = NIL_RTGCPHYS;
4166
4167	int rc = PGM_LOCK_VOID(pVM);
4168	AssertRCReturn(rc, NIL_RTGCPHYS);
4169
4170	uint32_t cChunks;
4171	uint32_t const idxFirst = pgmR3PhysMmio2ResolveHandle(pVM, pDevIns, hMmio2, &cChunks);
4172	if ((int32_t)idxFirst >= 0)
4173	GCPhysRet = pVM->pgm.s.aMmio2Ranges[idxFirst].GCPhys;
4174
4175	PGM_UNLOCK(pVM);
4176	return NIL_RTGCPHYS;
4177	}
4178
4179
4180	/**
4181	* Worker for PGMR3PhysMmio2QueryAndResetDirtyBitmap.
4182	*
4183	* Called holding the PGM lock.
4184	*/
4185	static int pgmR3PhysMmio2QueryAndResetDirtyBitmapLocked(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2,
4186	void *pvBitmap, size_t cbBitmap)
4187	{
4188	/*
4189	* Continue validation.
4190	*/
4191	uint32_t cChunks;
4192	uint32_t const idxFirst = pgmR3PhysMmio2ResolveHandle(pVM, pDevIns, hMmio2, &cChunks);
4193	AssertReturn((int32_t)idxFirst >= 0, (int32_t)idxFirst);
4194	PPGMREGMMIO2RANGE const pFirstMmio2 = &pVM->pgm.s.aMmio2Ranges[idxFirst];
4195	AssertReturn(pFirstMmio2->fFlags & PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES, VERR_INVALID_FUNCTION);
4196
4197	int rc = VINF_SUCCESS;
4198	if (cbBitmap \|\| pvBitmap)
4199	{
4200	/*
4201	* Check the bitmap size and collect all the dirty flags.
4202	*/
4203	RTGCPHYS cbTotal = 0;
4204	uint16_t fTotalDirty = 0;
4205	for (uint32_t iChunk = 0, idx = idxFirst; iChunk < cChunks; iChunk++, idx++)
4206	{
4207	/* Not using cbReal here, because NEM is not in on the creating, only the mapping. */
4208	cbTotal += pVM->pgm.s.apMmio2RamRanges[idx]->cb;
4209	fTotalDirty \|= pVM->pgm.s.aMmio2Ranges[idx].fFlags;
4210	}
4211	size_t const cbTotalBitmap = RT_ALIGN_T(cbTotal, GUEST_PAGE_SIZE * 64, RTGCPHYS) / GUEST_PAGE_SIZE / 8;
4212
4213	AssertPtrReturn(pvBitmap, VERR_INVALID_POINTER);
4214	AssertReturn(RT_ALIGN_P(pvBitmap, sizeof(uint64_t)) == pvBitmap, VERR_INVALID_POINTER);
4215	AssertReturn(cbBitmap == cbTotalBitmap, VERR_INVALID_PARAMETER);
4216
4217	#ifdef VBOX_WITH_PGM_NEM_MODE
4218	/*
4219	* If there is no physical handler we must be in NEM mode and NEM
4220	* taking care of the dirty bit collecting.
4221	*/
4222	if (pFirstMmio2->pPhysHandlerR3 == NULL)
4223	{
4224	/** @todo This does not integrate at all with --execute-all-in-iem, leaving the
4225	* screen blank when using it together with --driverless. Fixing this won't be
4226	* entirely easy as we take the PGM_PAGE_HNDL_PHYS_STATE_DISABLED page status to
4227	* mean a dirty page. */
4228	AssertReturn(VM_IS_NEM_ENABLED(pVM), VERR_INTERNAL_ERROR_4);
4229	uint8_t pbBitmap = (uint8_t )pvBitmap;
4230	for (uint32_t iChunk = 0, idx = idxFirst; iChunk < cChunks; iChunk++, idx++)
4231	{
4232	PPGMRAMRANGE const pRamRange = pVM->pgm.s.apMmio2RamRanges[idx];
4233	size_t const cbBitmapChunk = (pRamRange->cb / GUEST_PAGE_SIZE + 7) / 8;
4234	Assert((RTGCPHYS)cbBitmapChunk * GUEST_PAGE_SIZE * 8 == pRamRange->cb);
4235	Assert(pRamRange->GCPhys == pVM->pgm.s.aMmio2Ranges[idx].GCPhys); /* (No MMIO2 inside RAM in NEM mode!)*/
4236	int rc2 = NEMR3PhysMmio2QueryAndResetDirtyBitmap(pVM, pRamRange->GCPhys, pRamRange->cb,
4237	pRamRange->uNemRange, pbBitmap, cbBitmapChunk);
4238	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
4239	rc = rc2;
4240	pbBitmap += pRamRange->cb / GUEST_PAGE_SIZE / 8;
4241	}
4242	}
4243	else
4244	#endif
4245	if (fTotalDirty & PGMREGMMIO2RANGE_F_IS_DIRTY)
4246	{
4247	if ( (pFirstMmio2->fFlags & (PGMREGMMIO2RANGE_F_MAPPED \| PGMREGMMIO2RANGE_F_TRACKING_ENABLED))
4248	== (PGMREGMMIO2RANGE_F_MAPPED \| PGMREGMMIO2RANGE_F_TRACKING_ENABLED))
4249	{
4250	/*
4251	* Reset each chunk, gathering dirty bits.
4252	*/
4253	RT_BZERO(pvBitmap, cbBitmap); /* simpler for now. */
4254	for (uint32_t iChunk = 0, idx = idxFirst, iPageNo = 0; iChunk < cChunks; iChunk++, idx++)
4255	{
4256	PPGMREGMMIO2RANGE const pMmio2 = &pVM->pgm.s.aMmio2Ranges[idx];
4257	if (pMmio2->fFlags & PGMREGMMIO2RANGE_F_IS_DIRTY)
4258	{
4259	int rc2 = pgmHandlerPhysicalResetMmio2WithBitmap(pVM, pMmio2->GCPhys, pvBitmap, iPageNo);
4260	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
4261	rc = rc2;
4262	pMmio2->fFlags &= ~PGMREGMMIO2RANGE_F_IS_DIRTY;
4263	}
4264	iPageNo += pVM->pgm.s.apMmio2RamRanges[idx]->cb >> GUEST_PAGE_SHIFT;
4265	}
4266	}
4267	else
4268	{
4269	/*
4270	* If not mapped or tracking is disabled, we return the
4271	* PGMREGMMIO2RANGE_F_IS_DIRTY status for all pages. We cannot
4272	* get more accurate data than that after unmapping or disabling.
4273	*/
4274	RT_BZERO(pvBitmap, cbBitmap);
4275	for (uint32_t iChunk = 0, idx = idxFirst, iPageNo = 0; iChunk < cChunks; iChunk++, idx++)
4276	{
4277	PPGMRAMRANGE const pRamRange = pVM->pgm.s.apMmio2RamRanges[idx];
4278	PPGMREGMMIO2RANGE const pMmio2 = &pVM->pgm.s.aMmio2Ranges[idx];
4279	if (pMmio2->fFlags & PGMREGMMIO2RANGE_F_IS_DIRTY)
4280	{
4281	ASMBitSetRange(pvBitmap, iPageNo, iPageNo + (pRamRange->cb >> GUEST_PAGE_SHIFT));
4282	pMmio2->fFlags &= ~PGMREGMMIO2RANGE_F_IS_DIRTY;
4283	}
4284	iPageNo += pRamRange->cb >> GUEST_PAGE_SHIFT;
4285	}
4286	}
4287	}
4288	/*
4289	* No dirty chunks.
4290	*/
4291	else
4292	RT_BZERO(pvBitmap, cbBitmap);
4293	}
4294	/*
4295	* No bitmap. Reset the region if tracking is currently enabled.
4296	*/
4297	else if ( (pFirstMmio2->fFlags & (PGMREGMMIO2RANGE_F_MAPPED \| PGMREGMMIO2RANGE_F_TRACKING_ENABLED))
4298	== (PGMREGMMIO2RANGE_F_MAPPED \| PGMREGMMIO2RANGE_F_TRACKING_ENABLED))
4299	{
4300	#ifdef VBOX_WITH_PGM_NEM_MODE
4301	if (pFirstMmio2->pPhysHandlerR3 == NULL)
4302	{
4303	AssertReturn(VM_IS_NEM_ENABLED(pVM), VERR_INTERNAL_ERROR_4);
4304	for (uint32_t iChunk = 0, idx = idxFirst; iChunk < cChunks; iChunk++, idx++)
4305	{
4306	PPGMRAMRANGE const pRamRange = pVM->pgm.s.apMmio2RamRanges[idx];
4307	Assert(pRamRange->GCPhys == pVM->pgm.s.aMmio2Ranges[idx].GCPhys); /* (No MMIO2 inside RAM in NEM mode!)*/
4308	int rc2 = NEMR3PhysMmio2QueryAndResetDirtyBitmap(pVM, pRamRange->GCPhys, pRamRange->cb,
4309	pRamRange->uNemRange, NULL, 0);
4310	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
4311	rc = rc2;
4312	}
4313	}
4314	else
4315	#endif
4316	{
4317	for (uint32_t iChunk = 0, idx = idxFirst; iChunk < cChunks; iChunk++, idx++)
4318	{
4319	pVM->pgm.s.aMmio2Ranges[idx].fFlags &= ~PGMREGMMIO2RANGE_F_IS_DIRTY;
4320	int rc2 = PGMHandlerPhysicalReset(pVM, pVM->pgm.s.aMmio2Ranges[idx].GCPhys);
4321	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
4322	rc = rc2;
4323	}
4324	}
4325	}
4326
4327	return rc;
4328	}
4329
4330
4331	/**
4332	* Queries the dirty page bitmap and resets the monitoring.
4333	*
4334	* The PGMPHYS_MMIO2_FLAGS_TRACK_DIRTY_PAGES flag must be specified when
4335	* creating the range for this to work.
4336	*
4337	* @returns VBox status code.
4338	* @retval VERR_INVALID_FUNCTION if not created using
4339	* PGMPHYS_MMIO2_FLAGS_TRACK_DIRTY_PAGES.
4340	* @param pVM The cross context VM structure.
4341	* @param pDevIns The device owning the MMIO2 handle.
4342	* @param hMmio2 The region handle.
4343	* @param pvBitmap The output bitmap. Must be 8-byte aligned. Ignored
4344	* when @a cbBitmap is zero.
4345	* @param cbBitmap The size of the bitmap. Must be the size of the whole
4346	* MMIO2 range, rounded up to the nearest 8 bytes.
4347	* When zero only a reset is done.
4348	*/
4349	VMMR3_INT_DECL(int) PGMR3PhysMmio2QueryAndResetDirtyBitmap(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2,
4350	void *pvBitmap, size_t cbBitmap)
4351	{
4352	/*
4353	* Do some basic validation before grapping the PGM lock and continuing.
4354	*/
4355	AssertPtrReturn(pDevIns, VERR_INVALID_POINTER);
4356	AssertReturn(RT_ALIGN_Z(cbBitmap, sizeof(uint64_t)) == cbBitmap, VERR_INVALID_PARAMETER);
4357	int rc = PGM_LOCK(pVM);
4358	if (RT_SUCCESS(rc))
4359	{
4360	STAM_PROFILE_START(&pVM->pgm.s.StatMmio2QueryAndResetDirtyBitmap, a);
4361	rc = pgmR3PhysMmio2QueryAndResetDirtyBitmapLocked(pVM, pDevIns, hMmio2, pvBitmap, cbBitmap);
4362	STAM_PROFILE_STOP(&pVM->pgm.s.StatMmio2QueryAndResetDirtyBitmap, a);
4363	PGM_UNLOCK(pVM);
4364	}
4365	return rc;
4366	}
4367
4368
4369	/**
4370	* Worker for PGMR3PhysMmio2ControlDirtyPageTracking
4371	*
4372	* Called owning the PGM lock.
4373	*/
4374	static int pgmR3PhysMmio2ControlDirtyPageTrackingLocked(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, bool fEnabled)
4375	{
4376	/*
4377	* Continue validation.
4378	*/
4379	uint32_t cChunks;
4380	uint32_t const idxFirst = pgmR3PhysMmio2ResolveHandle(pVM, pDevIns, hMmio2, &cChunks);
4381	AssertReturn((int32_t)idxFirst >= 0, (int32_t)idxFirst);
4382	PPGMREGMMIO2RANGE const pFirstMmio2 = &pVM->pgm.s.aMmio2Ranges[idxFirst];
4383	AssertReturn(pFirstMmio2->fFlags & PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES, VERR_INVALID_FUNCTION);
4384
4385	#ifdef VBOX_WITH_PGM_NEM_MODE
4386	/*
4387	* This is a nop if NEM is responsible for doing the tracking, we simply
4388	* leave the tracking on all the time there.
4389	*/
4390	if (pFirstMmio2->pPhysHandlerR3 == NULL)
4391	{
4392	AssertReturn(VM_IS_NEM_ENABLED(pVM), VERR_INTERNAL_ERROR_4);
4393	return VINF_SUCCESS;
4394	}
4395	#endif
4396
4397	/*
4398	* Anything needing doing?
4399	*/
4400	if (fEnabled != RT_BOOL(pFirstMmio2->fFlags & PGMREGMMIO2RANGE_F_TRACKING_ENABLED))
4401	{
4402	LogFlowFunc(("fEnabled=%RTbool %s\n", fEnabled, pVM->pgm.s.apMmio2RamRanges[idxFirst]->pszDesc));
4403
4404	/*
4405	* Update the PGMREGMMIO2RANGE_F_TRACKING_ENABLED flag.
4406	*/
4407	for (uint32_t iChunk = 0, idx = idxFirst; iChunk < cChunks; iChunk++, idx++)
4408	if (fEnabled)
4409	pVM->pgm.s.aMmio2Ranges[idx].fFlags \|= PGMREGMMIO2RANGE_F_TRACKING_ENABLED;
4410	else
4411	pVM->pgm.s.aMmio2Ranges[idx].fFlags &= ~PGMREGMMIO2RANGE_F_TRACKING_ENABLED;
4412
4413	/*
4414	* Enable/disable handlers if currently mapped.
4415	*
4416	* We ignore status codes here as we've already changed the flags and
4417	* returning a failure status now would be confusing. Besides, the two
4418	* functions will continue past failures. As argued in the mapping code,
4419	* it's in the release log.
4420	*/
4421	if (pFirstMmio2->fFlags & PGMREGMMIO2RANGE_F_MAPPED)
4422	{
4423	if (fEnabled)
4424	pgmR3PhysMmio2EnableDirtyPageTracing(pVM, idxFirst, cChunks);
4425	else
4426	pgmR3PhysMmio2DisableDirtyPageTracing(pVM, idxFirst, cChunks);
4427	}
4428	}
4429	else
4430	LogFlowFunc(("fEnabled=%RTbool %s - no change\n", fEnabled, pVM->pgm.s.apMmio2RamRanges[idxFirst]->pszDesc));
4431
4432	return VINF_SUCCESS;
4433	}
4434
4435
4436	/**
4437	* Controls the dirty page tracking for an MMIO2 range.
4438	*
4439	* @returns VBox status code.
4440	* @param pVM The cross context VM structure.
4441	* @param pDevIns The device owning the MMIO2 memory.
4442	* @param hMmio2 The handle of the region.
4443	* @param fEnabled The new tracking state.
4444	*/
4445	VMMR3_INT_DECL(int) PGMR3PhysMmio2ControlDirtyPageTracking(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, bool fEnabled)
4446	{
4447	/*
4448	* Do some basic validation before grapping the PGM lock and continuing.
4449	*/
4450	AssertPtrReturn(pDevIns, VERR_INVALID_POINTER);
4451	int rc = PGM_LOCK(pVM);
4452	if (RT_SUCCESS(rc))
4453	{
4454	rc = pgmR3PhysMmio2ControlDirtyPageTrackingLocked(pVM, pDevIns, hMmio2, fEnabled);
4455	PGM_UNLOCK(pVM);
4456	}
4457	return rc;
4458	}
4459
4460
4461	/**
4462	* Changes the region number of an MMIO2 region.
4463	*
4464	* This is only for dealing with save state issues, nothing else.
4465	*
4466	* @return VBox status code.
4467	*
4468	* @param pVM The cross context VM structure.
4469	* @param pDevIns The device owning the MMIO2 memory.
4470	* @param hMmio2 The handle of the region.
4471	* @param iNewRegion The new region index.
4472	*
4473	* @thread EMT(0)
4474	* @sa @bugref{9359}
4475	*/
4476	VMMR3_INT_DECL(int) PGMR3PhysMmio2ChangeRegionNo(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, uint32_t iNewRegion)
4477	{
4478	/*
4479	* Validate input.
4480	*/
4481	VM_ASSERT_EMT0_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
4482	VM_ASSERT_STATE_RETURN(pVM, VMSTATE_LOADING, VERR_VM_INVALID_VM_STATE);
4483	AssertReturn(iNewRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
4484
4485	int rc = PGM_LOCK(pVM);
4486	AssertRCReturn(rc, rc);
4487
4488	/* Validate and resolve the handle. */
4489	uint32_t cChunks;
4490	uint32_t const idxFirst = pgmR3PhysMmio2ResolveHandle(pVM, pDevIns, hMmio2, &cChunks);
4491	if ((int32_t)idxFirst >= 0)
4492	{
4493	/* Check that the new range number is unused. */
4494	PPGMREGMMIO2RANGE const pConflict = pgmR3PhysMmio2Find(pVM, pDevIns, pVM->pgm.s.aMmio2Ranges[idxFirst].iSubDev,
4495	iNewRegion);
4496	if (!pConflict)
4497	{
4498	/*
4499	* Make the change.
4500	*/
4501	for (uint32_t iChunk = 0, idx = idxFirst; iChunk < cChunks; iChunk++, idx++)
4502	pVM->pgm.s.aMmio2Ranges[idx].iRegion = (uint8_t)iNewRegion;
4503	rc = VINF_SUCCESS;
4504	}
4505	else
4506	{
4507	AssertLogRelMsgFailed(("MMIO2/%s: iNewRegion=%d conflicts with %s\n", pVM->pgm.s.apMmio2RamRanges[idxFirst]->pszDesc,
4508	iNewRegion, pVM->pgm.s.apMmio2RamRanges[pConflict->idRamRange]->pszDesc));
4509	rc = VERR_RESOURCE_IN_USE;
4510	}
4511	}
4512	else
4513	rc = (int32_t)idxFirst;
4514
4515	PGM_UNLOCK(pVM);
4516	return rc;
4517	}
4518
4519
4520
4521	/*********************************************************************************************************************************
4522	* ROM *
4523	*********************************************************************************************************************************/
4524
4525	/**
4526	* Worker for PGMR3PhysRomRegister.
4527	*
4528	* This is here to simplify lock management, i.e. the caller does all the
4529	* locking and we can simply return without needing to remember to unlock
4530	* anything first.
4531	*
4532	* @returns VBox status code.
4533	* @param pVM The cross context VM structure.
4534	* @param pDevIns The device instance owning the ROM.
4535	* @param GCPhys First physical address in the range.
4536	* Must be page aligned!
4537	* @param cb The size of the range (in bytes).
4538	* Must be page aligned!
4539	* @param pvBinary Pointer to the binary data backing the ROM image.
4540	* @param cbBinary The size of the binary data pvBinary points to.
4541	* This must be less or equal to @a cb.
4542	* @param fFlags Mask of flags. PGMPHYS_ROM_FLAGS_SHADOWED
4543	* and/or PGMPHYS_ROM_FLAGS_PERMANENT_BINARY.
4544	* @param pszDesc Pointer to description string. This must not be freed.
4545	*/
4546	static int pgmR3PhysRomRegisterLocked(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
4547	const void pvBinary, uint32_t cbBinary, uint8_t fFlags, const char pszDesc)
4548	{
4549	/*
4550	* Validate input.
4551	*/
4552	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
4553	AssertReturn(RT_ALIGN_T(GCPhys, GUEST_PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
4554	AssertReturn(RT_ALIGN_T(cb, GUEST_PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
4555	RTGCPHYS const GCPhysLast = GCPhys + (cb - 1);
4556	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
4557	AssertPtrReturn(pvBinary, VERR_INVALID_PARAMETER);
4558	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
4559	AssertReturn(!(fFlags & ~PGMPHYS_ROM_FLAGS_VALID_MASK), VERR_INVALID_PARAMETER);
4560
4561	PVMCPU const pVCpu = VMMGetCpu(pVM);
4562	AssertReturn(pVCpu && pVCpu->idCpu == 0, VERR_VM_THREAD_NOT_EMT);
4563	VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
4564
4565	const uint32_t cGuestPages = cb >> GUEST_PAGE_SHIFT;
4566	AssertReturn(cGuestPages <= PGM_MAX_PAGES_PER_ROM_RANGE, VERR_OUT_OF_RANGE);
4567
4568	#ifdef VBOX_WITH_PGM_NEM_MODE
4569	const uint32_t cHostPages = RT_ALIGN_T(cb, HOST_PAGE_SIZE, RTGCPHYS) >> HOST_PAGE_SHIFT;
4570	#endif
4571
4572	/*
4573	* Make sure we've got a free ROM range.
4574	*/
4575	uint8_t const idRomRange = pVM->pgm.s.cRomRanges;
4576	AssertLogRelReturn(idRomRange < RT_ELEMENTS(pVM->pgm.s.apRomRanges), VERR_PGM_TOO_MANY_ROM_RANGES);
4577
4578	/*
4579	* Look thru the existing ROM range and make sure there aren't any
4580	* overlapping registration.
4581	*/
4582	uint32_t const cRomRanges = RT_MIN(pVM->pgm.s.cRomRanges, RT_ELEMENTS(pVM->pgm.s.apRomRanges));
4583	for (uint32_t idx = 0; idx < cRomRanges; idx++)
4584	{
4585	PPGMROMRANGE const pRom = pVM->pgm.s.apRomRanges[idx];
4586	AssertLogRelMsgReturn( GCPhys > pRom->GCPhysLast
4587	\|\| GCPhysLast < pRom->GCPhys,
4588	("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
4589	GCPhys, GCPhysLast, pszDesc,
4590	pRom->GCPhys, pRom->GCPhysLast, pRom->pszDesc),
4591	VERR_PGM_RAM_CONFLICT);
4592	}
4593
4594	/*
4595	* Find the RAM location and check for conflicts.
4596	*
4597	* Conflict detection is a bit different than for RAM registration since a
4598	* ROM can be located within a RAM range. So, what we have to check for is
4599	* other memory types (other than RAM that is) and that we don't span more
4600	* than one RAM range (lazy).
4601	*/
4602	uint32_t idxInsert = UINT32_MAX;
4603	PPGMRAMRANGE const pOverlappingRange = pgmR3PhysRamRangeFindOverlapping(pVM, GCPhys, GCPhysLast, &idxInsert);
4604	if (pOverlappingRange)
4605	{
4606	/* completely within? */
4607	AssertLogRelMsgReturn( GCPhys >= pOverlappingRange->GCPhys
4608	&& GCPhysLast <= pOverlappingRange->GCPhysLast,
4609	("%RGp-%RGp (%s) falls partly outside %RGp-%RGp (%s)\n",
4610	GCPhys, GCPhysLast, pszDesc,
4611	pOverlappingRange->GCPhys, pOverlappingRange->GCPhysLast, pOverlappingRange->pszDesc),
4612	VERR_PGM_RAM_CONFLICT);
4613
4614	/* Check that is isn't an ad hoc range, but a real RAM range. */
4615	AssertLogRelMsgReturn(!PGM_RAM_RANGE_IS_AD_HOC(pOverlappingRange),
4616	("%RGp-%RGp (ROM/%s) mapping attempt in non-RAM range: %RGp-%RGp (%s)\n",
4617	GCPhys, GCPhysLast, pszDesc,
4618	pOverlappingRange->GCPhys, pOverlappingRange->GCPhysLast, pOverlappingRange->pszDesc),
4619	VERR_PGM_RAM_CONFLICT);
4620
4621	/* All the pages must be RAM pages. */
4622	PPGMPAGE pPage = &pOverlappingRange->aPages[(GCPhys - pOverlappingRange->GCPhys) >> GUEST_PAGE_SHIFT];
4623	uint32_t cPagesLeft = cGuestPages;
4624	while (cPagesLeft-- > 0)
4625	{
4626	AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
4627	("%RGp (%R[pgmpage]) isn't a RAM page - registering %RGp-%RGp (%s).\n",
4628	GCPhys + ((RTGCPHYS)cPagesLeft << GUEST_PAGE_SHIFT), pPage, GCPhys, GCPhysLast, pszDesc),
4629	VERR_PGM_RAM_CONFLICT);
4630	AssertLogRelMsgReturn(PGM_PAGE_IS_ZERO(pPage) \|\| PGM_IS_IN_NEM_MODE(pVM),
4631	("%RGp (%R[pgmpage]) is not a ZERO page - registering %RGp-%RGp (%s).\n",
4632	GCPhys + ((RTGCPHYS)cPagesLeft << GUEST_PAGE_SHIFT), pPage, GCPhys, GCPhysLast, pszDesc),
4633	VERR_PGM_UNEXPECTED_PAGE_STATE);
4634	pPage++;
4635	}
4636	}
4637
4638	/*
4639	* Update the base memory reservation if necessary.
4640	*/
4641	uint32_t const cExtraBaseCost = (pOverlappingRange ? 0 : cGuestPages)
4642	+ (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED ? cGuestPages : 0);
4643	if (cExtraBaseCost)
4644	{
4645	int rc = MMR3IncreaseBaseReservation(pVM, cExtraBaseCost);
4646	AssertRCReturn(rc, rc);
4647	}
4648
4649	#ifdef VBOX_WITH_NATIVE_NEM
4650	/*
4651	* Early NEM notification before we've made any changes or anything.
4652	*/
4653	uint32_t const fNemNotify = (pOverlappingRange ? NEM_NOTIFY_PHYS_ROM_F_REPLACE : 0)
4654	\| (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED ? NEM_NOTIFY_PHYS_ROM_F_SHADOW : 0);
4655	uint8_t u2NemState = UINT8_MAX;
4656	uint32_t uNemRange = 0;
4657	if (VM_IS_NEM_ENABLED(pVM))
4658	{
4659	int rc = NEMR3NotifyPhysRomRegisterEarly(pVM, GCPhys, cGuestPages << GUEST_PAGE_SHIFT,
4660	pOverlappingRange
4661	? PGM_RAMRANGE_CALC_PAGE_R3PTR(pOverlappingRange, GCPhys) : NULL,
4662	fNemNotify, &u2NemState,
4663	pOverlappingRange ? &pOverlappingRange->uNemRange : &uNemRange);
4664	AssertLogRelRCReturn(rc, rc);
4665	}
4666	#endif
4667
4668	/*
4669	* Allocate memory for the virgin copy of the RAM. In simplified memory
4670	* mode, we allocate memory for any ad-hoc RAM range and for shadow pages.
4671	*/
4672	int rc;
4673	PGMMALLOCATEPAGESREQ pReq = NULL;
4674	#ifdef VBOX_WITH_PGM_NEM_MODE
4675	void *pvRam = NULL;
4676	void *pvAlt = NULL;
4677	if (PGM_IS_IN_NEM_MODE(pVM))
4678	{
4679	if (!pOverlappingRange)
4680	{
4681	rc = SUPR3PageAlloc(cHostPages, 0, &pvRam);
4682	if (RT_FAILURE(rc))
4683	return rc;
4684	}
4685	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
4686	{
4687	rc = SUPR3PageAlloc(cHostPages, 0, &pvAlt);
4688	if (RT_FAILURE(rc))
4689	{
4690	if (pvRam)
4691	SUPR3PageFree(pvRam, cHostPages);
4692	return rc;
4693	}
4694	}
4695	}
4696	else
4697	#endif
4698	{
4699	rc = GMMR3AllocatePagesPrepare(pVM, &pReq, cGuestPages, GMMACCOUNT_BASE);
4700	AssertRCReturn(rc, rc);
4701
4702	for (uint32_t iPage = 0; iPage < cGuestPages; iPage++)
4703	{
4704	pReq->aPages[iPage].HCPhysGCPhys = GCPhys + (iPage << GUEST_PAGE_SHIFT);
4705	pReq->aPages[iPage].fZeroed = false;
4706	pReq->aPages[iPage].idPage = NIL_GMM_PAGEID;
4707	pReq->aPages[iPage].idSharedPage = NIL_GMM_PAGEID;
4708	}
4709
4710	rc = GMMR3AllocatePagesPerform(pVM, pReq);
4711	if (RT_FAILURE(rc))
4712	{
4713	GMMR3AllocatePagesCleanup(pReq);
4714	return rc;
4715	}
4716	}
4717
4718	/*
4719	* Allocate a RAM range if required.
4720	* Note! We don't clean up the RAM range here on failure, VM destruction does that.
4721	*/
4722	rc = VINF_SUCCESS;
4723	PPGMRAMRANGE pRamRange = NULL;
4724	if (!pOverlappingRange)
4725	rc = pgmR3PhysAllocateRamRange(pVM, pVCpu, cGuestPages, PGM_RAM_RANGE_FLAGS_AD_HOC_ROM, &pRamRange);
4726	if (RT_SUCCESS(rc))
4727	{
4728	/*
4729	* Allocate a ROM range.
4730	* Note! We don't clean up the ROM range here on failure, VM destruction does that.
4731	*/
4732	if (SUPR3IsDriverless())
4733	rc = pgmPhysRomRangeAllocCommon(pVM, cGuestPages, idRomRange, fFlags);
4734	else
4735	{
4736	PGMPHYSROMALLOCATERANGEREQ RomRangeReq;
4737	RomRangeReq.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
4738	RomRangeReq.Hdr.cbReq = sizeof(RomRangeReq);
4739	RomRangeReq.cbGuestPage = GUEST_PAGE_SIZE;
4740	RomRangeReq.cGuestPages = cGuestPages;
4741	RomRangeReq.idRomRange = idRomRange;
4742	RomRangeReq.fFlags = fFlags;
4743	rc = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_PGM_PHYS_ROM_ALLOCATE_RANGE, 0 /u64Arg/, &RomRangeReq.Hdr);
4744	}
4745	}
4746	if (RT_SUCCESS(rc))
4747	{
4748	/*
4749	* Initialize and map the RAM range (if required).
4750	*/
4751	PPGMROMRANGE const pRomRange = pVM->pgm.s.apRomRanges[idRomRange];
4752	AssertPtr(pRomRange);
4753	uint32_t const idxFirstRamPage = pOverlappingRange ? (GCPhys - pOverlappingRange->GCPhys) >> GUEST_PAGE_SHIFT : 0;
4754	PPGMROMPAGE pRomPage = &pRomRange->aPages[0];
4755	if (!pOverlappingRange)
4756	{
4757	/* Initialize the new RAM range and insert it into the lookup table. */
4758	pRamRange->pszDesc = pszDesc;
4759	#ifdef VBOX_WITH_NATIVE_NEM
4760	pRamRange->uNemRange = uNemRange;
4761	#endif
4762
4763	PPGMPAGE pRamPage = &pRamRange->aPages[idxFirstRamPage];
4764	#ifdef VBOX_WITH_PGM_NEM_MODE
4765	if (PGM_IS_IN_NEM_MODE(pVM))
4766	{
4767	AssertPtr(pvRam); Assert(pReq == NULL);
4768	pRamRange->pbR3 = (uint8_t *)pvRam;
4769	for (uint32_t iPage = 0; iPage < cGuestPages; iPage++, pRamPage++, pRomPage++)
4770	{
4771	PGM_PAGE_INIT(pRamPage, UINT64_C(0x0000fffffffff000), NIL_GMM_PAGEID,
4772	PGMPAGETYPE_ROM, PGM_PAGE_STATE_ALLOCATED);
4773	pRomPage->Virgin = *pRamPage;
4774	}
4775	}
4776	else
4777	#endif
4778	{
4779	Assert(!pRamRange->pbR3); Assert(!pvRam);
4780	for (uint32_t iPage = 0; iPage < cGuestPages; iPage++, pRamPage++, pRomPage++)
4781	{
4782	PGM_PAGE_INIT(pRamPage,
4783	pReq->aPages[iPage].HCPhysGCPhys,
4784	pReq->aPages[iPage].idPage,
4785	PGMPAGETYPE_ROM,
4786	PGM_PAGE_STATE_ALLOCATED);
4787
4788	pRomPage->Virgin = *pRamPage;
4789	}
4790	}
4791
4792	pVM->pgm.s.cAllPages += cGuestPages;
4793	pVM->pgm.s.cPrivatePages += cGuestPages;
4794
4795	rc = pgmR3PhysRamRangeInsertLookup(pVM, pRamRange, GCPhys, &idxInsert);
4796	}
4797	else
4798	{
4799	/* Insert the ROM into an existing RAM range. */
4800	PPGMPAGE pRamPage = &pOverlappingRange->aPages[idxFirstRamPage];
4801	#ifdef VBOX_WITH_PGM_NEM_MODE
4802	if (PGM_IS_IN_NEM_MODE(pVM))
4803	{
4804	Assert(pvRam == NULL); Assert(pReq == NULL);
4805	for (uint32_t iPage = 0; iPage < cGuestPages; iPage++, pRamPage++, pRomPage++)
4806	{
4807	Assert(PGM_PAGE_GET_HCPHYS(pRamPage) == UINT64_C(0x0000fffffffff000));
4808	Assert(PGM_PAGE_GET_PAGEID(pRamPage) == NIL_GMM_PAGEID);
4809	Assert(PGM_PAGE_GET_STATE(pRamPage) == PGM_PAGE_STATE_ALLOCATED);
4810	PGM_PAGE_SET_TYPE(pVM, pRamPage, PGMPAGETYPE_ROM);
4811	PGM_PAGE_SET_STATE(pVM, pRamPage, PGM_PAGE_STATE_ALLOCATED);
4812	PGM_PAGE_SET_PDE_TYPE(pVM, pRamPage, PGM_PAGE_PDE_TYPE_DONTCARE);
4813	PGM_PAGE_SET_PTE_INDEX(pVM, pRamPage, 0);
4814	PGM_PAGE_SET_TRACKING(pVM, pRamPage, 0);
4815
4816	pRomPage->Virgin = *pRamPage;
4817	}
4818	}
4819	else
4820	#endif
4821	{
4822	for (uint32_t iPage = 0; iPage < cGuestPages; iPage++, pRamPage++, pRomPage++)
4823	{
4824	PGM_PAGE_SET_TYPE(pVM, pRamPage, PGMPAGETYPE_ROM);
4825	PGM_PAGE_SET_HCPHYS(pVM, pRamPage, pReq->aPages[iPage].HCPhysGCPhys);
4826	PGM_PAGE_SET_STATE(pVM, pRamPage, PGM_PAGE_STATE_ALLOCATED);
4827	PGM_PAGE_SET_PAGEID(pVM, pRamPage, pReq->aPages[iPage].idPage);
4828	PGM_PAGE_SET_PDE_TYPE(pVM, pRamPage, PGM_PAGE_PDE_TYPE_DONTCARE);
4829	PGM_PAGE_SET_PTE_INDEX(pVM, pRamPage, 0);
4830	PGM_PAGE_SET_TRACKING(pVM, pRamPage, 0);
4831
4832	pRomPage->Virgin = *pRamPage;
4833	}
4834	pVM->pgm.s.cZeroPages -= cGuestPages;
4835	pVM->pgm.s.cPrivatePages += cGuestPages;
4836	}
4837	pRamRange = pOverlappingRange;
4838	}
4839
4840	if (RT_SUCCESS(rc))
4841	{
4842	#ifdef VBOX_WITH_NATIVE_NEM
4843	/* Set the NEM state of the pages if needed. */
4844	if (u2NemState != UINT8_MAX)
4845	pgmPhysSetNemStateForPages(&pRamRange->aPages[idxFirstRamPage], cGuestPages, u2NemState);
4846	#endif
4847
4848	/* Flush physical page map TLB. */
4849	pgmPhysInvalidatePageMapTLB(pVM, false /fInRendezvous/);
4850
4851	/*
4852	* Register the ROM access handler.
4853	*/
4854	rc = PGMHandlerPhysicalRegister(pVM, GCPhys, GCPhysLast, pVM->pgm.s.hRomPhysHandlerType, idRomRange, pszDesc);
4855	if (RT_SUCCESS(rc))
4856	{
4857	/*
4858	* Copy the image over to the virgin pages.
4859	* This must be done after linking in the RAM range.
4860	*/
4861	size_t cbBinaryLeft = cbBinary;
4862	PPGMPAGE pRamPage = &pRamRange->aPages[idxFirstRamPage];
4863	for (uint32_t iPage = 0; iPage < cGuestPages; iPage++, pRamPage++)
4864	{
4865	void *pvDstPage;
4866	rc = pgmPhysPageMap(pVM, pRamPage, GCPhys + (iPage << GUEST_PAGE_SHIFT), &pvDstPage);
4867	if (RT_FAILURE(rc))
4868	{
4869	VMSetError(pVM, rc, RT_SRC_POS, "Failed to map virgin ROM page at %RGp", GCPhys);
4870	break;
4871	}
4872	if (cbBinaryLeft >= GUEST_PAGE_SIZE)
4873	{
4874	memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << GUEST_PAGE_SHIFT), GUEST_PAGE_SIZE);
4875	cbBinaryLeft -= GUEST_PAGE_SIZE;
4876	}
4877	else
4878	{
4879	RT_BZERO(pvDstPage, GUEST_PAGE_SIZE); /* (shouldn't be necessary, but can't hurt either) */
4880	if (cbBinaryLeft > 0)
4881	{
4882	memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << GUEST_PAGE_SHIFT), cbBinaryLeft);
4883	cbBinaryLeft = 0;
4884	}
4885	}
4886	}
4887	if (RT_SUCCESS(rc))
4888	{
4889	/*
4890	* Initialize the ROM range.
4891	* Note that the Virgin member of the pages has already been initialized above.
4892	*/
4893	Assert(pRomRange->cb == cb);
4894	Assert(pRomRange->fFlags == fFlags);
4895	Assert(pRomRange->idSavedState == UINT8_MAX);
4896	pRomRange->GCPhys = GCPhys;
4897	pRomRange->GCPhysLast = GCPhysLast;
4898	pRomRange->cbOriginal = cbBinary;
4899	pRomRange->pszDesc = pszDesc;
4900	#ifdef VBOX_WITH_PGM_NEM_MODE
4901	pRomRange->pbR3Alternate = (uint8_t *)pvAlt;
4902	#endif
4903	pRomRange->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY
4904	? pvBinary : RTMemDup(pvBinary, cbBinary);
4905	if (pRomRange->pvOriginal)
4906	{
4907	for (unsigned iPage = 0; iPage < cGuestPages; iPage++)
4908	{
4909	PPGMROMPAGE const pPage = &pRomRange->aPages[iPage];
4910	pPage->enmProt = PGMROMPROT_READ_ROM_WRITE_IGNORE;
4911	#ifdef VBOX_WITH_PGM_NEM_MODE
4912	if (PGM_IS_IN_NEM_MODE(pVM))
4913	PGM_PAGE_INIT(&pPage->Shadow, UINT64_C(0x0000fffffffff000), NIL_GMM_PAGEID,
4914	PGMPAGETYPE_ROM_SHADOW, PGM_PAGE_STATE_ALLOCATED);
4915	else
4916	#endif
4917	PGM_PAGE_INIT_ZERO(&pPage->Shadow, pVM, PGMPAGETYPE_ROM_SHADOW);
4918	}
4919
4920	/* update the page count stats for the shadow pages. */
4921	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
4922	{
4923	if (PGM_IS_IN_NEM_MODE(pVM))
4924	pVM->pgm.s.cPrivatePages += cGuestPages;
4925	else
4926	pVM->pgm.s.cZeroPages += cGuestPages;
4927	pVM->pgm.s.cAllPages += cGuestPages;
4928	}
4929
4930	#ifdef VBOX_WITH_NATIVE_NEM
4931	/*
4932	* Notify NEM again.
4933	*/
4934	if (VM_IS_NEM_ENABLED(pVM))
4935	{
4936	u2NemState = UINT8_MAX;
4937	rc = NEMR3NotifyPhysRomRegisterLate(pVM, GCPhys, cb, PGM_RAMRANGE_CALC_PAGE_R3PTR(pRamRange, GCPhys),
4938	fNemNotify, &u2NemState, &pRamRange->uNemRange);
4939	if (u2NemState != UINT8_MAX)
4940	pgmPhysSetNemStateForPages(&pRamRange->aPages[idxFirstRamPage], cGuestPages, u2NemState);
4941	}
4942	else
4943	#endif
4944	GMMR3AllocatePagesCleanup(pReq);
4945	if (RT_SUCCESS(rc))
4946	{
4947	/*
4948	* Done!
4949	*/
4950	#ifdef VBOX_STRICT
4951	pgmPhysAssertRamRangesLocked(pVM, false /fInUpdate/, false /fRamRelaxed/);
4952	#endif
4953	return rc;
4954	}
4955
4956	/*
4957	* bail out
4958	*/
4959	#ifdef VBOX_WITH_NATIVE_NEM
4960	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
4961	{
4962	Assert(VM_IS_NEM_ENABLED(pVM));
4963	pVM->pgm.s.cPrivatePages -= cGuestPages;
4964	pVM->pgm.s.cAllPages -= cGuestPages;
4965	}
4966	#endif
4967	}
4968	else
4969	rc = VERR_NO_MEMORY;
4970	}
4971
4972	int rc2 = PGMHandlerPhysicalDeregister(pVM, GCPhys);
4973	AssertRC(rc2);
4974	}
4975
4976	idxInsert -= 1;
4977	if (!pOverlappingRange)
4978	pgmR3PhysRamRangeRemoveLookup(pVM, pRamRange, &idxInsert);
4979	}
4980	/* else: lookup insertion failed. */
4981
4982	if (pOverlappingRange)
4983	{
4984	PPGMPAGE pRamPage = &pOverlappingRange->aPages[idxFirstRamPage];
4985	#ifdef VBOX_WITH_PGM_NEM_MODE
4986	if (PGM_IS_IN_NEM_MODE(pVM))
4987	{
4988	Assert(pvRam == NULL); Assert(pReq == NULL);
4989	for (uint32_t iPage = 0; iPage < cGuestPages; iPage++, pRamPage++, pRomPage++)
4990	{
4991	Assert(PGM_PAGE_GET_HCPHYS(pRamPage) == UINT64_C(0x0000fffffffff000));
4992	Assert(PGM_PAGE_GET_PAGEID(pRamPage) == NIL_GMM_PAGEID);
4993	Assert(PGM_PAGE_GET_STATE(pRamPage) == PGM_PAGE_STATE_ALLOCATED);
4994	PGM_PAGE_SET_TYPE(pVM, pRamPage, PGMPAGETYPE_RAM);
4995	PGM_PAGE_SET_STATE(pVM, pRamPage, PGM_PAGE_STATE_ALLOCATED);
4996	}
4997	}
4998	else
4999	#endif
5000	{
5001	for (uint32_t iPage = 0; iPage < cGuestPages; iPage++, pRamPage++)
5002	PGM_PAGE_INIT_ZERO(pRamPage, pVM, PGMPAGETYPE_RAM);
5003	pVM->pgm.s.cZeroPages += cGuestPages;
5004	pVM->pgm.s.cPrivatePages -= cGuestPages;
5005	}
5006	}
5007	}
5008	pgmPhysInvalidatePageMapTLB(pVM, false /fInRendezvous/);
5009	pgmPhysInvalidRamRangeTlbs(pVM);
5010
5011	#ifdef VBOX_WITH_PGM_NEM_MODE
5012	if (PGM_IS_IN_NEM_MODE(pVM))
5013	{
5014	Assert(!pReq);
5015	if (pvRam)
5016	SUPR3PageFree(pvRam, cHostPages);
5017	if (pvAlt)
5018	SUPR3PageFree(pvAlt, cHostPages);
5019	}
5020	else
5021	#endif
5022	{
5023	GMMR3FreeAllocatedPages(pVM, pReq);
5024	GMMR3AllocatePagesCleanup(pReq);
5025	}
5026
5027	/* We don't bother to actually free either the ROM nor the RAM ranges
5028	themselves, as already mentioned above, we'll leave that to the VM
5029	termination cleanup code. */
5030	return rc;
5031	}
5032
5033
5034	/**
5035	* Registers a ROM image.
5036	*
5037	* Shadowed ROM images requires double the amount of backing memory, so,
5038	* don't use that unless you have to. Shadowing of ROM images is process
5039	* where we can select where the reads go and where the writes go. On real
5040	* hardware the chipset provides means to configure this. We provide
5041	* PGMR3PhysRomProtect() for this purpose.
5042	*
5043	* A read-only copy of the ROM image will always be kept around while we
5044	* will allocate RAM pages for the changes on demand (unless all memory
5045	* is configured to be preallocated).
5046	*
5047	* @returns VBox status code.
5048	* @param pVM The cross context VM structure.
5049	* @param pDevIns The device instance owning the ROM.
5050	* @param GCPhys First physical address in the range.
5051	* Must be page aligned!
5052	* @param cb The size of the range (in bytes).
5053	* Must be page aligned!
5054	* @param pvBinary Pointer to the binary data backing the ROM image.
5055	* @param cbBinary The size of the binary data pvBinary points to.
5056	* This must be less or equal to @a cb.
5057	* @param fFlags Mask of flags, PGMPHYS_ROM_FLAGS_XXX.
5058	* @param pszDesc Pointer to description string. This must not be freed.
5059	*
5060	* @remark There is no way to remove the rom, automatically on device cleanup or
5061	* manually from the device yet. This isn't difficult in any way, it's
5062	* just not something we expect to be necessary for a while.
5063	*/
5064	VMMR3DECL(int) PGMR3PhysRomRegister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
5065	const void pvBinary, uint32_t cbBinary, uint8_t fFlags, const char pszDesc)
5066	{
5067	Log(("PGMR3PhysRomRegister: pDevIns=%p GCPhys=%RGp(-%RGp) cb=%RGp pvBinary=%p cbBinary=%#x fFlags=%#x pszDesc=%s\n",
5068	pDevIns, GCPhys, GCPhys + cb, cb, pvBinary, cbBinary, fFlags, pszDesc));
5069	PGM_LOCK_VOID(pVM);
5070
5071	int rc = pgmR3PhysRomRegisterLocked(pVM, pDevIns, GCPhys, cb, pvBinary, cbBinary, fFlags, pszDesc);
5072
5073	PGM_UNLOCK(pVM);
5074	return rc;
5075	}
5076
5077
5078	/**
5079	* Called by PGMR3MemSetup to reset the shadow, switch to the virgin, and verify
5080	* that the virgin part is untouched.
5081	*
5082	* This is done after the normal memory has been cleared.
5083	*
5084	* ASSUMES that the caller owns the PGM lock.
5085	*
5086	* @param pVM The cross context VM structure.
5087	*/
5088	int pgmR3PhysRomReset(PVM pVM)
5089	{
5090	PGM_LOCK_ASSERT_OWNER(pVM);
5091	uint32_t const cRomRanges = RT_MIN(pVM->pgm.s.cRomRanges, RT_ELEMENTS(pVM->pgm.s.apRomRanges));
5092	for (uint32_t idx = 0; idx < cRomRanges; idx++)
5093	{
5094	PPGMROMRANGE const pRom = pVM->pgm.s.apRomRanges[idx];
5095	uint32_t const cGuestPages = pRom->cb >> GUEST_PAGE_SHIFT;
5096
5097	if (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
5098	{
5099	/*
5100	* Reset the physical handler.
5101	*/
5102	int rc = PGMR3PhysRomProtect(pVM, pRom->GCPhys, pRom->cb, PGMROMPROT_READ_ROM_WRITE_IGNORE);
5103	AssertRCReturn(rc, rc);
5104
5105	/*
5106	* What we do with the shadow pages depends on the memory
5107	* preallocation option. If not enabled, we'll just throw
5108	* out all the dirty pages and replace them by the zero page.
5109	*/
5110	#ifdef VBOX_WITH_PGM_NEM_MODE
5111	if (PGM_IS_IN_NEM_MODE(pVM))
5112	{
5113	/* Clear all the shadow pages (currently using alternate backing). */
5114	RT_BZERO(pRom->pbR3Alternate, pRom->cb);
5115	}
5116	else
5117	#endif
5118	if (!pVM->pgm.s.fRamPreAlloc)
5119	{
5120	/* Free the dirty pages. */
5121	uint32_t cPendingPages = 0;
5122	PGMMFREEPAGESREQ pReq;
5123	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
5124	AssertRCReturn(rc, rc);
5125
5126	for (uint32_t iPage = 0; iPage < cGuestPages; iPage++)
5127	if ( !PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow)
5128	&& !PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow))
5129	{
5130	Assert(PGM_PAGE_GET_STATE(&pRom->aPages[iPage].Shadow) == PGM_PAGE_STATE_ALLOCATED);
5131	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, &pRom->aPages[iPage].Shadow,
5132	pRom->GCPhys + (iPage << GUEST_PAGE_SHIFT),
5133	(PGMPAGETYPE)PGM_PAGE_GET_TYPE(&pRom->aPages[iPage].Shadow));
5134	AssertLogRelRCReturn(rc, rc);
5135	}
5136
5137	if (cPendingPages)
5138	{
5139	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
5140	AssertLogRelRCReturn(rc, rc);
5141	}
5142	GMMR3FreePagesCleanup(pReq);
5143	}
5144	else
5145	{
5146	/* clear all the shadow pages. */
5147	for (uint32_t iPage = 0; iPage < cGuestPages; iPage++)
5148	{
5149	if (PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow))
5150	continue;
5151	Assert(!PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow));
5152	void *pvDstPage;
5153	RTGCPHYS const GCPhys = pRom->GCPhys + (iPage << GUEST_PAGE_SHIFT);
5154	rc = pgmPhysPageMakeWritableAndMap(pVM, &pRom->aPages[iPage].Shadow, GCPhys, &pvDstPage);
5155	if (RT_FAILURE(rc))
5156	break;
5157	RT_BZERO(pvDstPage, GUEST_PAGE_SIZE);
5158	}
5159	AssertRCReturn(rc, rc);
5160	}
5161	}
5162
5163	/*
5164	* Restore the original ROM pages after a saved state load.
5165	* Also, in strict builds check that ROM pages remain unmodified.
5166	*/
5167	#ifndef VBOX_STRICT
5168	if (pVM->pgm.s.fRestoreRomPagesOnReset)
5169	#endif
5170	{
5171	size_t cbSrcLeft = pRom->cbOriginal;
5172	uint8_t const pbSrcPage = (uint8_t const )pRom->pvOriginal;
5173	uint32_t cRestored = 0;
5174	for (uint32_t iPage = 0; iPage < cGuestPages && cbSrcLeft > 0; iPage++, pbSrcPage += GUEST_PAGE_SIZE)
5175	{
5176	RTGCPHYS const GCPhys = pRom->GCPhys + (iPage << GUEST_PAGE_SHIFT);
5177	PPGMPAGE const pPage = pgmPhysGetPage(pVM, GCPhys);
5178	void const *pvDstPage = NULL;
5179	int rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhys, &pvDstPage);
5180	if (RT_FAILURE(rc))
5181	break;
5182
5183	if (memcmp(pvDstPage, pbSrcPage, RT_MIN(cbSrcLeft, GUEST_PAGE_SIZE)))
5184	{
5185	if (pVM->pgm.s.fRestoreRomPagesOnReset)
5186	{
5187	void *pvDstPageW = NULL;
5188	rc = pgmPhysPageMap(pVM, pPage, GCPhys, &pvDstPageW);
5189	AssertLogRelRCReturn(rc, rc);
5190	memcpy(pvDstPageW, pbSrcPage, RT_MIN(cbSrcLeft, GUEST_PAGE_SIZE));
5191	cRestored++;
5192	}
5193	else
5194	LogRel(("pgmR3PhysRomReset: %RGp: ROM page changed (%s)\n", GCPhys, pRom->pszDesc));
5195	}
5196	cbSrcLeft -= RT_MIN(cbSrcLeft, GUEST_PAGE_SIZE);
5197	}
5198	if (cRestored > 0)
5199	LogRel(("PGM: ROM \"%s\": Reloaded %u of %u pages.\n", pRom->pszDesc, cRestored, cGuestPages));
5200	}
5201	}
5202
5203	/* Clear the ROM restore flag now as we only need to do this once after
5204	loading saved state. */
5205	pVM->pgm.s.fRestoreRomPagesOnReset = false;
5206
5207	return VINF_SUCCESS;
5208	}
5209
5210
5211	/**
5212	* Called by PGMR3Term to free resources.
5213	*
5214	* ASSUMES that the caller owns the PGM lock.
5215	*
5216	* @param pVM The cross context VM structure.
5217	*/
5218	void pgmR3PhysRomTerm(PVM pVM)
5219	{
5220	/*
5221	* Free the heap copy of the original bits.
5222	*/
5223	uint32_t const cRomRanges = RT_MIN(pVM->pgm.s.cRomRanges, RT_ELEMENTS(pVM->pgm.s.apRomRanges));
5224	for (uint32_t idx = 0; idx < cRomRanges; idx++)
5225	{
5226	PPGMROMRANGE const pRom = pVM->pgm.s.apRomRanges[idx];
5227	if ( pRom->pvOriginal
5228	&& !(pRom->fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY))
5229	{
5230	RTMemFree((void *)pRom->pvOriginal);
5231	pRom->pvOriginal = NULL;
5232	}
5233	}
5234	}
5235
5236
5237	/**
5238	* Change the shadowing of a range of ROM pages.
5239	*
5240	* This is intended for implementing chipset specific memory registers
5241	* and will not be very strict about the input. It will silently ignore
5242	* any pages that are not the part of a shadowed ROM.
5243	*
5244	* @returns VBox status code.
5245	* @retval VINF_PGM_SYNC_CR3
5246	*
5247	* @param pVM The cross context VM structure.
5248	* @param GCPhys Where to start. Page aligned.
5249	* @param cb How much to change. Page aligned.
5250	* @param enmProt The new ROM protection.
5251	*/
5252	VMMR3DECL(int) PGMR3PhysRomProtect(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, PGMROMPROT enmProt)
5253	{
5254	LogFlow(("PGMR3PhysRomProtect: GCPhys=%RGp cb=%RGp enmProt=%d\n", GCPhys, cb, enmProt));
5255
5256	/*
5257	* Check input
5258	*/
5259	if (!cb)
5260	return VINF_SUCCESS;
5261	AssertReturn(!(GCPhys & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
5262	AssertReturn(!(cb & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
5263	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
5264	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
5265	AssertReturn(enmProt >= PGMROMPROT_INVALID && enmProt <= PGMROMPROT_END, VERR_INVALID_PARAMETER);
5266
5267	/*
5268	* Process the request.
5269	*/
5270	PGM_LOCK_VOID(pVM);
5271	int rc = VINF_SUCCESS;
5272	bool fFlushTLB = false;
5273	uint32_t const cRomRanges = RT_MIN(pVM->pgm.s.cRomRanges, RT_ELEMENTS(pVM->pgm.s.apRomRanges));
5274	for (uint32_t idx = 0; idx < cRomRanges; idx++)
5275	{
5276	PPGMROMRANGE const pRom = pVM->pgm.s.apRomRanges[idx];
5277	if ( GCPhys <= pRom->GCPhysLast
5278	&& GCPhysLast >= pRom->GCPhys
5279	&& (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED))
5280	{
5281	/*
5282	* Iterate the relevant pages and make necessary the changes.
5283	*/
5284	#ifdef VBOX_WITH_NATIVE_NEM
5285	PPGMRAMRANGE const pRam = pgmPhysGetRange(pVM, GCPhys);
5286	AssertPtrReturn(pRam, VERR_INTERNAL_ERROR_3);
5287	#endif
5288	bool fChanges = false;
5289	uint32_t const cPages = pRom->GCPhysLast <= GCPhysLast
5290	? pRom->cb >> GUEST_PAGE_SHIFT
5291	: (GCPhysLast - pRom->GCPhys + 1) >> GUEST_PAGE_SHIFT;
5292	for (uint32_t iPage = (GCPhys - pRom->GCPhys) >> GUEST_PAGE_SHIFT;
5293	iPage < cPages;
5294	iPage++)
5295	{
5296	PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
5297	if (PGMROMPROT_IS_ROM(pRomPage->enmProt) != PGMROMPROT_IS_ROM(enmProt))
5298	{
5299	fChanges = true;
5300
5301	/* flush references to the page. */
5302	RTGCPHYS const GCPhysPage = pRom->GCPhys + (iPage << GUEST_PAGE_SHIFT);
5303	PPGMPAGE pRamPage = pgmPhysGetPage(pVM, GCPhysPage);
5304	int rc2 = pgmPoolTrackUpdateGCPhys(pVM, GCPhysPage, pRamPage, true /fFlushPTEs/, &fFlushTLB);
5305	if (rc2 != VINF_SUCCESS && (rc == VINF_SUCCESS \|\| RT_FAILURE(rc2)))
5306	rc = rc2;
5307	#ifdef VBOX_WITH_NATIVE_NEM
5308	uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pRamPage);
5309	#endif
5310
5311	PPGMPAGE pOld = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Virgin : &pRomPage->Shadow;
5312	PPGMPAGE pNew = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Shadow : &pRomPage->Virgin;
5313
5314	pOld = pRamPage;
5315	pRamPage = pNew;
5316	/** @todo preserve the volatile flags (handlers) when these have been moved out of HCPhys! */
5317
5318	#ifdef VBOX_WITH_NATIVE_NEM
5319	# ifdef VBOX_WITH_PGM_NEM_MODE
5320	/* In simplified mode we have to switch the page data around too. */
5321	if (PGM_IS_IN_NEM_MODE(pVM))
5322	{
5323	uint8_t abPage[GUEST_PAGE_SIZE];
5324	uint8_t * const pbRamPage = PGM_RAMRANGE_CALC_PAGE_R3PTR(pRam, GCPhysPage);
5325	memcpy(abPage, &pRom->pbR3Alternate[(size_t)iPage << GUEST_PAGE_SHIFT], sizeof(abPage));
5326	memcpy(&pRom->pbR3Alternate[(size_t)iPage << GUEST_PAGE_SHIFT], pbRamPage, sizeof(abPage));
5327	memcpy(pbRamPage, abPage, sizeof(abPage));
5328	}
5329	# endif
5330	/* Tell NEM about the backing and protection change. */
5331	if (VM_IS_NEM_ENABLED(pVM))
5332	{
5333	PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pNew);
5334	NEMHCNotifyPhysPageChanged(pVM, GCPhys, PGM_PAGE_GET_HCPHYS(pOld), PGM_PAGE_GET_HCPHYS(pNew),
5335	PGM_RAMRANGE_CALC_PAGE_R3PTR(pRam, GCPhysPage),
5336	pgmPhysPageCalcNemProtection(pRamPage, enmType), enmType, &u2State);
5337	PGM_PAGE_SET_NEM_STATE(pRamPage, u2State);
5338	}
5339	#endif
5340	}
5341	pRomPage->enmProt = enmProt;
5342	}
5343
5344	/*
5345	* Reset the access handler if we made changes, no need to optimize this.
5346	*/
5347	if (fChanges)
5348	{
5349	int rc2 = PGMHandlerPhysicalReset(pVM, pRom->GCPhys);
5350	if (RT_FAILURE(rc2))
5351	{
5352	PGM_UNLOCK(pVM);
5353	AssertRC(rc);
5354	return rc2;
5355	}
5356
5357	/* Explicitly flush IEM. Not sure if this is really necessary, but better
5358	be on the safe side. This shouldn't be a high volume flush source. */
5359	IEMTlbInvalidateAllPhysicalAllCpus(pVM, NIL_VMCPUID, IEMTLBPHYSFLUSHREASON_ROM_PROTECT);
5360	}
5361
5362	/* Advance - cb isn't updated. */
5363	GCPhys = pRom->GCPhys + (cPages << GUEST_PAGE_SHIFT);
5364	}
5365	}
5366	PGM_UNLOCK(pVM);
5367	if (fFlushTLB)
5368	PGM_INVL_ALL_VCPU_TLBS(pVM);
5369
5370	return rc;
5371	}
5372
5373
5374
5375	/*********************************************************************************************************************************
5376	* Ballooning *
5377	*********************************************************************************************************************************/
5378
5379	#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) \|\| defined(RT_OS_SOLARIS) \|\| defined(RT_OS_LINUX) \|\| defined(RT_OS_FREEBSD))
5380
5381	/**
5382	* Rendezvous callback used by PGMR3ChangeMemBalloon that changes the memory balloon size
5383	*
5384	* This is only called on one of the EMTs while the other ones are waiting for
5385	* it to complete this function.
5386	*
5387	* @returns VINF_SUCCESS (VBox strict status code).
5388	* @param pVM The cross context VM structure.
5389	* @param pVCpu The cross context virtual CPU structure of the calling EMT. Unused.
5390	* @param pvUser User parameter
5391	*/
5392	static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysChangeMemBalloonRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
5393	{
5394	uintptr_t paUser = (uintptr_t )pvUser;
5395	bool fInflate = !!paUser[0];
5396	unsigned cPages = paUser[1];
5397	RTGCPHYS paPhysPage = (RTGCPHYS )paUser[2];
5398	uint32_t cPendingPages = 0;
5399	PGMMFREEPAGESREQ pReq;
5400	int rc;
5401
5402	Log(("pgmR3PhysChangeMemBalloonRendezvous: %s %x pages\n", (fInflate) ? "inflate" : "deflate", cPages));
5403	PGM_LOCK_VOID(pVM);
5404
5405	if (fInflate)
5406	{
5407	/* Flush the PGM pool cache as we might have stale references to pages that we just freed. */
5408	pgmR3PoolClearAllRendezvous(pVM, pVCpu, NULL);
5409
5410	/* Replace pages with ZERO pages. */
5411	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
5412	if (RT_FAILURE(rc))
5413	{
5414	PGM_UNLOCK(pVM);
5415	AssertLogRelRC(rc);
5416	return rc;
5417	}
5418
5419	/* Iterate the pages. */
5420	for (unsigned i = 0; i < cPages; i++)
5421	{
5422	PPGMPAGE pPage = pgmPhysGetPage(pVM, paPhysPage[i]);
5423	if ( pPage == NULL
5424	\|\| PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM)
5425	{
5426	Log(("pgmR3PhysChangeMemBalloonRendezvous: invalid physical page %RGp pPage->u3Type=%d\n", paPhysPage[i], pPage ? PGM_PAGE_GET_TYPE(pPage) : 0));
5427	break;
5428	}
5429
5430	LogFlow(("balloon page: %RGp\n", paPhysPage[i]));
5431
5432	/* Flush the shadow PT if this page was previously used as a guest page table. */
5433	pgmPoolFlushPageByGCPhys(pVM, paPhysPage[i]);
5434
5435	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, paPhysPage[i], (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage));
5436	if (RT_FAILURE(rc))
5437	{
5438	PGM_UNLOCK(pVM);
5439	AssertLogRelRC(rc);
5440	return rc;
5441	}
5442	Assert(PGM_PAGE_IS_ZERO(pPage));
5443	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_BALLOONED);
5444	}
5445
5446	if (cPendingPages)
5447	{
5448	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
5449	if (RT_FAILURE(rc))
5450	{
5451	PGM_UNLOCK(pVM);
5452	AssertLogRelRC(rc);
5453	return rc;
5454	}
5455	}
5456	GMMR3FreePagesCleanup(pReq);
5457	}
5458	else
5459	{
5460	/* Iterate the pages. */
5461	for (unsigned i = 0; i < cPages; i++)
5462	{
5463	PPGMPAGE pPage = pgmPhysGetPage(pVM, paPhysPage[i]);
5464	AssertBreak(pPage && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM);
5465
5466	LogFlow(("Free ballooned page: %RGp\n", paPhysPage[i]));
5467
5468	Assert(PGM_PAGE_IS_BALLOONED(pPage));
5469
5470	/* Change back to zero page. (NEM does not need to be informed.) */
5471	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
5472	}
5473
5474	/* Note that we currently do not map any ballooned pages in our shadow page tables, so no need to flush the pgm pool. */
5475	}
5476
5477	/* Notify GMM about the balloon change. */
5478	rc = GMMR3BalloonedPages(pVM, (fInflate) ? GMMBALLOONACTION_INFLATE : GMMBALLOONACTION_DEFLATE, cPages);
5479	if (RT_SUCCESS(rc))
5480	{
5481	if (!fInflate)
5482	{
5483	Assert(pVM->pgm.s.cBalloonedPages >= cPages);
5484	pVM->pgm.s.cBalloonedPages -= cPages;
5485	}
5486	else
5487	pVM->pgm.s.cBalloonedPages += cPages;
5488	}
5489
5490	PGM_UNLOCK(pVM);
5491
5492	/* Flush the recompiler's TLB as well. */
5493	for (VMCPUID i = 0; i < pVM->cCpus; i++)
5494	CPUMSetChangedFlags(pVM->apCpusR3[i], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
5495
5496	AssertLogRelRC(rc);
5497	return rc;
5498	}
5499
5500
5501	/**
5502	* Frees a range of ram pages, replacing them with ZERO pages; helper for PGMR3PhysFreeRamPages
5503	*
5504	* @param pVM The cross context VM structure.
5505	* @param fInflate Inflate or deflate memory balloon
5506	* @param cPages Number of pages to free
5507	* @param paPhysPage Array of guest physical addresses
5508	*/
5509	static DECLCALLBACK(void) pgmR3PhysChangeMemBalloonHelper(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
5510	{
5511	uintptr_t paUser[3];
5512
5513	paUser[0] = fInflate;
5514	paUser[1] = cPages;
5515	paUser[2] = (uintptr_t)paPhysPage;
5516	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
5517	AssertRC(rc);
5518
5519	/* Made a copy in PGMR3PhysFreeRamPages; free it here. */
5520	RTMemFree(paPhysPage);
5521	}
5522
5523	#endif /* 64-bit host && (Windows \|\| Solaris \|\| Linux \|\| FreeBSD) */
5524
5525	/**
5526	* Inflate or deflate a memory balloon
5527	*
5528	* @returns VBox status code.
5529	* @param pVM The cross context VM structure.
5530	* @param fInflate Inflate or deflate memory balloon
5531	* @param cPages Number of pages to free
5532	* @param paPhysPage Array of guest physical addresses
5533	*/
5534	VMMR3DECL(int) PGMR3PhysChangeMemBalloon(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
5535	{
5536	/* This must match GMMR0Init; currently we only support memory ballooning on all 64-bit hosts except Mac OS X */
5537	#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) \|\| defined(RT_OS_SOLARIS) \|\| defined(RT_OS_LINUX) \|\| defined(RT_OS_FREEBSD))
5538	int rc;
5539
5540	/* Older additions (ancient non-functioning balloon code) pass wrong physical addresses. */
5541	AssertReturn(!(paPhysPage[0] & 0xfff), VERR_INVALID_PARAMETER);
5542
5543	/* We own the IOM lock here and could cause a deadlock by waiting for another VCPU that is blocking on the IOM lock.
5544	* In the SMP case we post a request packet to postpone the job.
5545	*/
5546	if (pVM->cCpus > 1)
5547	{
5548	unsigned cbPhysPage = cPages * sizeof(paPhysPage[0]);
5549	RTGCPHYS paPhysPageCopy = (RTGCPHYS )RTMemAlloc(cbPhysPage);
5550	AssertReturn(paPhysPageCopy, VERR_NO_MEMORY);
5551
5552	memcpy(paPhysPageCopy, paPhysPage, cbPhysPage);
5553
5554	rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysChangeMemBalloonHelper, 4,
5555	pVM, fInflate, cPages, paPhysPageCopy);
5556	AssertRC(rc);
5557	}
5558	else
5559	{
5560	uintptr_t paUser[3];
5561
5562	paUser[0] = fInflate;
5563	paUser[1] = cPages;
5564	paUser[2] = (uintptr_t)paPhysPage;
5565	rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
5566	AssertRC(rc);
5567	}
5568	return rc;
5569
5570	#else
5571	NOREF(pVM); NOREF(fInflate); NOREF(cPages); NOREF(paPhysPage);
5572	return VERR_NOT_IMPLEMENTED;
5573	#endif
5574	}
5575
5576
5577
5578	/*********************************************************************************************************************************
5579	* Write Monitoring *
5580	*********************************************************************************************************************************/
5581
5582	/**
5583	* Rendezvous callback used by PGMR3WriteProtectRAM that write protects all
5584	* physical RAM.
5585	*
5586	* This is only called on one of the EMTs while the other ones are waiting for
5587	* it to complete this function.
5588	*
5589	* @returns VINF_SUCCESS (VBox strict status code).
5590	* @param pVM The cross context VM structure.
5591	* @param pVCpu The cross context virtual CPU structure of the calling EMT. Unused.
5592	* @param pvUser User parameter, unused.
5593	*/
5594	static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysWriteProtectRAMRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
5595	{
5596	int rc = VINF_SUCCESS;
5597	NOREF(pvUser); NOREF(pVCpu);
5598
5599	PGM_LOCK_VOID(pVM);
5600	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
5601	pgmPoolResetDirtyPages(pVM);
5602	#endif
5603
5604	uint32_t const cLookupEntries = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
5605	for (uint32_t idxLookup = 0; idxLookup < cLookupEntries; idxLookup++)
5606	{
5607	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
5608	AssertContinue(idRamRange < RT_ELEMENTS(pVM->pgm.s.apRamRanges));
5609	PPGMRAMRANGE const pRam = pVM->pgm.s.apRamRanges[idRamRange];
5610	AssertContinue(pRam);
5611
5612	uint32_t cPages = pRam->cb >> GUEST_PAGE_SHIFT;
5613	for (uint32_t iPage = 0; iPage < cPages; iPage++)
5614	{
5615	PPGMPAGE const pPage = &pRam->aPages[iPage];
5616	PGMPAGETYPE const enmPageType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
5617
5618	if ( RT_LIKELY(enmPageType == PGMPAGETYPE_RAM)
5619	\|\| enmPageType == PGMPAGETYPE_MMIO2)
5620	{
5621	/*
5622	* A RAM page.
5623	*/
5624	switch (PGM_PAGE_GET_STATE(pPage))
5625	{
5626	case PGM_PAGE_STATE_ALLOCATED:
5627	/** @todo Optimize this: Don't always re-enable write
5628	* monitoring if the page is known to be very busy. */
5629	if (PGM_PAGE_IS_WRITTEN_TO(pPage))
5630	PGM_PAGE_CLEAR_WRITTEN_TO(pVM, pPage);
5631
5632	pgmPhysPageWriteMonitor(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT));
5633	break;
5634
5635	case PGM_PAGE_STATE_SHARED:
5636	AssertFailed();
5637	break;
5638
5639	case PGM_PAGE_STATE_WRITE_MONITORED: /* nothing to change. */
5640	default:
5641	break;
5642	}
5643	}
5644	}
5645	}
5646	pgmR3PoolWriteProtectPages(pVM);
5647	PGM_INVL_ALL_VCPU_TLBS(pVM);
5648	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
5649	CPUMSetChangedFlags(pVM->apCpusR3[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
5650
5651	PGM_UNLOCK(pVM);
5652	return rc;
5653	}
5654
5655	/**
5656	* Protect all physical RAM to monitor writes
5657	*
5658	* @returns VBox status code.
5659	* @param pVM The cross context VM structure.
5660	*/
5661	VMMR3DECL(int) PGMR3PhysWriteProtectRAM(PVM pVM)
5662	{
5663	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
5664
5665	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysWriteProtectRAMRendezvous, NULL);
5666	AssertRC(rc);
5667	return rc;
5668	}
5669
5670
5671	/*********************************************************************************************************************************
5672	* Stats. *
5673	*********************************************************************************************************************************/
5674
5675	/**
5676	* Query the amount of free memory inside VMMR0
5677	*
5678	* @returns VBox status code.
5679	* @param pUVM The user mode VM handle.
5680	* @param pcbAllocMem Where to return the amount of memory allocated
5681	* by VMs.
5682	* @param pcbFreeMem Where to return the amount of memory that is
5683	* allocated from the host but not currently used
5684	* by any VMs.
5685	* @param pcbBallonedMem Where to return the sum of memory that is
5686	* currently ballooned by the VMs.
5687	* @param pcbSharedMem Where to return the amount of memory that is
5688	* currently shared.
5689	*/
5690	VMMR3DECL(int) PGMR3QueryGlobalMemoryStats(PUVM pUVM, uint64_t pcbAllocMem, uint64_t pcbFreeMem,
5691	uint64_t pcbBallonedMem, uint64_t pcbSharedMem)
5692	{
5693	UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
5694	VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
5695
5696	uint64_t cAllocPages = 0;
5697	uint64_t cFreePages = 0;
5698	uint64_t cBalloonPages = 0;
5699	uint64_t cSharedPages = 0;
5700	if (!SUPR3IsDriverless())
5701	{
5702	int rc = GMMR3QueryHypervisorMemoryStats(pUVM->pVM, &cAllocPages, &cFreePages, &cBalloonPages, &cSharedPages);
5703	AssertRCReturn(rc, rc);
5704	}
5705
5706	if (pcbAllocMem)
5707	pcbAllocMem = cAllocPages _4K;
5708
5709	if (pcbFreeMem)
5710	pcbFreeMem = cFreePages _4K;
5711
5712	if (pcbBallonedMem)
5713	pcbBallonedMem = cBalloonPages _4K;
5714
5715	if (pcbSharedMem)
5716	pcbSharedMem = cSharedPages _4K;
5717
5718	Log(("PGMR3QueryVMMMemoryStats: all=%llx free=%llx ballooned=%llx shared=%llx\n",
5719	cAllocPages, cFreePages, cBalloonPages, cSharedPages));
5720	return VINF_SUCCESS;
5721	}
5722
5723
5724	/**
5725	* Query memory stats for the VM.
5726	*
5727	* @returns VBox status code.
5728	* @param pUVM The user mode VM handle.
5729	* @param pcbTotalMem Where to return total amount memory the VM may
5730	* possibly use.
5731	* @param pcbPrivateMem Where to return the amount of private memory
5732	* currently allocated.
5733	* @param pcbSharedMem Where to return the amount of actually shared
5734	* memory currently used by the VM.
5735	* @param pcbZeroMem Where to return the amount of memory backed by
5736	* zero pages.
5737	*
5738	* @remarks The total mem is normally larger than the sum of the three
5739	* components. There are two reasons for this, first the amount of
5740	* shared memory is what we're sure is shared instead of what could
5741	* possibly be shared with someone. Secondly, because the total may
5742	* include some pure MMIO pages that doesn't go into any of the three
5743	* sub-counts.
5744	*
5745	* @todo Why do we return reused shared pages instead of anything that could
5746	* potentially be shared? Doesn't this mean the first VM gets a much
5747	* lower number of shared pages?
5748	*/
5749	VMMR3DECL(int) PGMR3QueryMemoryStats(PUVM pUVM, uint64_t pcbTotalMem, uint64_t pcbPrivateMem,
5750	uint64_t pcbSharedMem, uint64_t pcbZeroMem)
5751	{
5752	UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
5753	PVM pVM = pUVM->pVM;
5754	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
5755
5756	if (pcbTotalMem)
5757	pcbTotalMem = (uint64_t)pVM->pgm.s.cAllPages GUEST_PAGE_SIZE;
5758
5759	if (pcbPrivateMem)
5760	pcbPrivateMem = (uint64_t)pVM->pgm.s.cPrivatePages GUEST_PAGE_SIZE;
5761
5762	if (pcbSharedMem)
5763	pcbSharedMem = (uint64_t)pVM->pgm.s.cReusedSharedPages GUEST_PAGE_SIZE;
5764
5765	if (pcbZeroMem)
5766	pcbZeroMem = (uint64_t)pVM->pgm.s.cZeroPages GUEST_PAGE_SIZE;
5767
5768	Log(("PGMR3QueryMemoryStats: all=%x private=%x reused=%x zero=%x\n", pVM->pgm.s.cAllPages, pVM->pgm.s.cPrivatePages, pVM->pgm.s.cReusedSharedPages, pVM->pgm.s.cZeroPages));
5769	return VINF_SUCCESS;
5770	}
5771
5772
5773
5774	/*********************************************************************************************************************************
5775	* Chunk Mappings and Page Allocation *
5776	*********************************************************************************************************************************/
5777
5778	/**
5779	* Tree enumeration callback for dealing with age rollover.
5780	* It will perform a simple compression of the current age.
5781	*/
5782	static DECLCALLBACK(int) pgmR3PhysChunkAgeingRolloverCallback(PAVLU32NODECORE pNode, void *pvUser)
5783	{
5784	/* Age compression - ASSUMES iNow == 4. */
5785	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
5786	if (pChunk->iLastUsed >= UINT32_C(0xffffff00))
5787	pChunk->iLastUsed = 3;
5788	else if (pChunk->iLastUsed >= UINT32_C(0xfffff000))
5789	pChunk->iLastUsed = 2;
5790	else if (pChunk->iLastUsed)
5791	pChunk->iLastUsed = 1;
5792	else /* iLastUsed = 0 */
5793	pChunk->iLastUsed = 4;
5794
5795	NOREF(pvUser);
5796	return 0;
5797	}
5798
5799
5800	/**
5801	* The structure passed in the pvUser argument of pgmR3PhysChunkUnmapCandidateCallback().
5802	*/
5803	typedef struct PGMR3PHYSCHUNKUNMAPCB
5804	{
5805	PVM pVM; /*< Pointer to the VM. /
5806	PPGMCHUNKR3MAP pChunk; /*< The chunk to unmap. /
5807	} PGMR3PHYSCHUNKUNMAPCB, *PPGMR3PHYSCHUNKUNMAPCB;
5808
5809
5810	/**
5811	* Callback used to find the mapping that's been unused for
5812	* the longest time.
5813	*/
5814	static DECLCALLBACK(int) pgmR3PhysChunkUnmapCandidateCallback(PAVLU32NODECORE pNode, void *pvUser)
5815	{
5816	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
5817	PPGMR3PHYSCHUNKUNMAPCB pArg = (PPGMR3PHYSCHUNKUNMAPCB)pvUser;
5818
5819	/*
5820	* Check for locks and compare when last used.
5821	*/
5822	if (pChunk->cRefs)
5823	return 0;
5824	if (pChunk->cPermRefs)
5825	return 0;
5826	if ( pArg->pChunk
5827	&& pChunk->iLastUsed >= pArg->pChunk->iLastUsed)
5828	return 0;
5829
5830	/*
5831	* Check that it's not in any of the TLBs.
5832	*/
5833	PVM pVM = pArg->pVM;
5834	if ( pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(pChunk->Core.Key)].idChunk
5835	== pChunk->Core.Key)
5836	{
5837	pChunk = NULL;
5838	return 0;
5839	}
5840	#ifdef VBOX_STRICT
5841	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
5842	{
5843	Assert(pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk != pChunk);
5844	Assert(pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk != pChunk->Core.Key);
5845	}
5846	#endif
5847
5848	#if 0 /* This is too much work with the PGMCPU::PhysTlb as well. We flush them all instead. */
5849	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbR3.aEntries); i++)
5850	if (pVM->pgm.s.PhysTlbR3.aEntries[i].pMap == pChunk)
5851	return 0;
5852	#endif
5853
5854	pArg->pChunk = pChunk;
5855	return 0;
5856	}
5857
5858
5859	/**
5860	* Finds a good candidate for unmapping when the ring-3 mapping cache is full.
5861	*
5862	* The candidate will not be part of any TLBs, so no need to flush
5863	* anything afterwards.
5864	*
5865	* @returns Chunk id.
5866	* @param pVM The cross context VM structure.
5867	*/
5868	static int32_t pgmR3PhysChunkFindUnmapCandidate(PVM pVM)
5869	{
5870	PGM_LOCK_ASSERT_OWNER(pVM);
5871
5872	/*
5873	* Enumerate the age tree starting with the left most node.
5874	*/
5875	STAM_PROFILE_START(&pVM->pgm.s.Stats.StatChunkFindCandidate, a);
5876	PGMR3PHYSCHUNKUNMAPCB Args;
5877	Args.pVM = pVM;
5878	Args.pChunk = NULL;
5879	RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /fFromLeft/, pgmR3PhysChunkUnmapCandidateCallback, &Args);
5880	Assert(Args.pChunk);
5881	if (Args.pChunk)
5882	{
5883	Assert(Args.pChunk->cRefs == 0);
5884	Assert(Args.pChunk->cPermRefs == 0);
5885	STAM_PROFILE_STOP(&pVM->pgm.s.Stats.StatChunkFindCandidate, a);
5886	return Args.pChunk->Core.Key;
5887	}
5888
5889	STAM_PROFILE_STOP(&pVM->pgm.s.Stats.StatChunkFindCandidate, a);
5890	return INT32_MAX;
5891	}
5892
5893
5894	/**
5895	* Rendezvous callback used by pgmR3PhysUnmapChunk that unmaps a chunk
5896	*
5897	* This is only called on one of the EMTs while the other ones are waiting for
5898	* it to complete this function.
5899	*
5900	* @returns VINF_SUCCESS (VBox strict status code).
5901	* @param pVM The cross context VM structure.
5902	* @param pVCpu The cross context virtual CPU structure of the calling EMT. Unused.
5903	* @param pvUser User pointer. Unused
5904	*
5905	*/
5906	static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysUnmapChunkRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
5907	{
5908	int rc = VINF_SUCCESS;
5909	PGM_LOCK_VOID(pVM);
5910	NOREF(pVCpu); NOREF(pvUser);
5911
5912	if (pVM->pgm.s.ChunkR3Map.c >= pVM->pgm.s.ChunkR3Map.cMax)
5913	{
5914	/* Flush the pgm pool cache; call the internal rendezvous handler as we're already in a rendezvous handler here. */
5915	/** @todo also not really efficient to unmap a chunk that contains PD
5916	* or PT pages. */
5917	pgmR3PoolClearAllRendezvous(pVM, pVM->apCpusR3[0], NULL /* no need to flush the REM TLB as we already did that above */);
5918
5919	/*
5920	* Request the ring-0 part to unmap a chunk to make space in the mapping cache.
5921	*/
5922	GMMMAPUNMAPCHUNKREQ Req;
5923	Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
5924	Req.Hdr.cbReq = sizeof(Req);
5925	Req.pvR3 = NULL;
5926	Req.idChunkMap = NIL_GMM_CHUNKID;
5927	Req.idChunkUnmap = pgmR3PhysChunkFindUnmapCandidate(pVM);
5928	if (Req.idChunkUnmap != INT32_MAX)
5929	{
5930	STAM_PROFILE_START(&pVM->pgm.s.Stats.StatChunkUnmap, a);
5931	rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
5932	STAM_PROFILE_STOP(&pVM->pgm.s.Stats.StatChunkUnmap, a);
5933	if (RT_SUCCESS(rc))
5934	{
5935	/*
5936	* Remove the unmapped one.
5937	*/
5938	PPGMCHUNKR3MAP pUnmappedChunk = (PPGMCHUNKR3MAP)RTAvlU32Remove(&pVM->pgm.s.ChunkR3Map.pTree, Req.idChunkUnmap);
5939	AssertRelease(pUnmappedChunk);
5940	AssertRelease(!pUnmappedChunk->cRefs);
5941	AssertRelease(!pUnmappedChunk->cPermRefs);
5942	pUnmappedChunk->pv = NULL;
5943	pUnmappedChunk->Core.Key = UINT32_MAX;
5944	MMR3HeapFree(pUnmappedChunk);
5945	pVM->pgm.s.ChunkR3Map.c--;
5946	pVM->pgm.s.cUnmappedChunks++;
5947
5948	/*
5949	* Flush dangling PGM pointers (R3 & R0 ptrs to GC physical addresses).
5950	*/
5951	/** @todo We should not flush chunks which include cr3 mappings. */
5952	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
5953	{
5954	PPGMCPU pPGM = &pVM->apCpusR3[idCpu]->pgm.s;
5955
5956	pPGM->pGst32BitPdR3 = NULL;
5957	pPGM->pGstPaePdptR3 = NULL;
5958	pPGM->pGstAmd64Pml4R3 = NULL;
5959	pPGM->pGstEptPml4R3 = NULL;
5960	pPGM->pGst32BitPdR0 = NIL_RTR0PTR;
5961	pPGM->pGstPaePdptR0 = NIL_RTR0PTR;
5962	pPGM->pGstAmd64Pml4R0 = NIL_RTR0PTR;
5963	pPGM->pGstEptPml4R0 = NIL_RTR0PTR;
5964	for (unsigned i = 0; i < RT_ELEMENTS(pPGM->apGstPaePDsR3); i++)
5965	{
5966	pPGM->apGstPaePDsR3[i] = NULL;
5967	pPGM->apGstPaePDsR0[i] = NIL_RTR0PTR;
5968	}
5969
5970	/* Flush REM TLBs. */
5971	CPUMSetChangedFlags(pVM->apCpusR3[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
5972	}
5973
5974	pgmR3PhysChunkInvalidateTLB(pVM, true /fInRendezvous/); /* includes pgmPhysInvalidatePageMapTLB call */
5975	}
5976	}
5977	}
5978	PGM_UNLOCK(pVM);
5979	return rc;
5980	}
5981
5982	/**
5983	* Unmap a chunk to free up virtual address space (request packet handler for pgmR3PhysChunkMap)
5984	*
5985	* @param pVM The cross context VM structure.
5986	*/
5987	static DECLCALLBACK(void) pgmR3PhysUnmapChunk(PVM pVM)
5988	{
5989	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysUnmapChunkRendezvous, NULL);
5990	AssertRC(rc);
5991	}
5992
5993
5994	/**
5995	* Maps the given chunk into the ring-3 mapping cache.
5996	*
5997	* This will call ring-0.
5998	*
5999	* @returns VBox status code.
6000	* @param pVM The cross context VM structure.
6001	* @param idChunk The chunk in question.
6002	* @param ppChunk Where to store the chunk tracking structure.
6003	*
6004	* @remarks Called from within the PGM critical section.
6005	* @remarks Can be called from any thread!
6006	*/
6007	int pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk)
6008	{
6009	int rc;
6010
6011	PGM_LOCK_ASSERT_OWNER(pVM);
6012
6013	/*
6014	* Move the chunk time forward.
6015	*/
6016	pVM->pgm.s.ChunkR3Map.iNow++;
6017	if (pVM->pgm.s.ChunkR3Map.iNow == 0)
6018	{
6019	pVM->pgm.s.ChunkR3Map.iNow = 4;
6020	RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /fFromLeft/, pgmR3PhysChunkAgeingRolloverCallback, NULL);
6021	}
6022
6023	/*
6024	* Allocate a new tracking structure first.
6025	*/
6026	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3HeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk));
6027	AssertReturn(pChunk, VERR_NO_MEMORY);
6028	pChunk->Core.Key = idChunk;
6029	pChunk->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
6030
6031	/*
6032	* Request the ring-0 part to map the chunk in question.
6033	*/
6034	GMMMAPUNMAPCHUNKREQ Req;
6035	Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
6036	Req.Hdr.cbReq = sizeof(Req);
6037	Req.pvR3 = NULL;
6038	Req.idChunkMap = idChunk;
6039	Req.idChunkUnmap = NIL_GMM_CHUNKID;
6040
6041	/* Must be callable from any thread, so can't use VMMR3CallR0. */
6042	STAM_PROFILE_START(&pVM->pgm.s.Stats.StatChunkMap, a);
6043	rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), NIL_VMCPUID, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
6044	STAM_PROFILE_STOP(&pVM->pgm.s.Stats.StatChunkMap, a);
6045	if (RT_SUCCESS(rc))
6046	{
6047	pChunk->pv = Req.pvR3;
6048
6049	/*
6050	* If we're running out of virtual address space, then we should
6051	* unmap another chunk.
6052	*
6053	* Currently, an unmap operation requires that all other virtual CPUs
6054	* are idling and not by chance making use of the memory we're
6055	* unmapping. So, we create an async unmap operation here.
6056	*
6057	* Now, when creating or restoring a saved state this wont work very
6058	* well since we may want to restore all guest RAM + a little something.
6059	* So, we have to do the unmap synchronously. Fortunately for us
6060	* though, during these operations the other virtual CPUs are inactive
6061	* and it should be safe to do this.
6062	*/
6063	/** @todo Eventually we should lock all memory when used and do
6064	* map+unmap as one kernel call without any rendezvous or
6065	* other precautions. */
6066	if (pVM->pgm.s.ChunkR3Map.c + 1 >= pVM->pgm.s.ChunkR3Map.cMax)
6067	{
6068	switch (VMR3GetState(pVM))
6069	{
6070	case VMSTATE_LOADING:
6071	case VMSTATE_SAVING:
6072	{
6073	PVMCPU pVCpu = VMMGetCpu(pVM);
6074	if ( pVCpu
6075	&& pVM->pgm.s.cDeprecatedPageLocks == 0)
6076	{
6077	pgmR3PhysUnmapChunkRendezvous(pVM, pVCpu, NULL);
6078	break;
6079	}
6080	}
6081	RT_FALL_THRU();
6082	default:
6083	rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysUnmapChunk, 1, pVM);
6084	AssertRC(rc);
6085	break;
6086	}
6087	}
6088
6089	/*
6090	* Update the tree. We must do this after any unmapping to make sure
6091	* the chunk we're going to return isn't unmapped by accident.
6092	*/
6093	AssertPtr(Req.pvR3);
6094	bool fRc = RTAvlU32Insert(&pVM->pgm.s.ChunkR3Map.pTree, &pChunk->Core);
6095	AssertRelease(fRc);
6096	pVM->pgm.s.ChunkR3Map.c++;
6097	pVM->pgm.s.cMappedChunks++;
6098	}
6099	else
6100	{
6101	/** @todo this may fail because of /proc/sys/vm/max_map_count, so we
6102	* should probably restrict ourselves on linux. */
6103	AssertRC(rc);
6104	MMR3HeapFree(pChunk);
6105	pChunk = NULL;
6106	}
6107
6108	*ppChunk = pChunk;
6109	return rc;
6110	}
6111
6112
6113	/**
6114	* Invalidates the TLB for the ring-3 mapping cache.
6115	*
6116	* @param pVM The cross context VM structure.
6117	* @param fInRendezvous Set if we're in a rendezvous.
6118	*/
6119	DECLHIDDEN(void) pgmR3PhysChunkInvalidateTLB(PVM pVM, bool fInRendezvous)
6120	{
6121	PGM_LOCK_VOID(pVM);
6122	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
6123	{
6124	pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk = NIL_GMM_CHUNKID;
6125	pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk = NULL;
6126	}
6127	/* The page map TLB references chunks, so invalidate that one too. */
6128	pgmPhysInvalidatePageMapTLB(pVM, fInRendezvous);
6129	PGM_UNLOCK(pVM);
6130	}
6131
6132
6133	/**
6134	* Response to VM_FF_PGM_NEED_HANDY_PAGES and helper for pgmPhysEnsureHandyPage.
6135	*
6136	* This function will also work the VM_FF_PGM_NO_MEMORY force action flag, to
6137	* signal and clear the out of memory condition. When called, this API is used
6138	* to try clear the condition when the user wants to resume.
6139	*
6140	* @returns The following VBox status codes.
6141	* @retval VINF_SUCCESS on success. FFs cleared.
6142	* @retval VINF_EM_NO_MEMORY if we're out of memory. The FF is not cleared in
6143	* this case and it gets accompanied by VM_FF_PGM_NO_MEMORY.
6144	*
6145	* @param pVM The cross context VM structure.
6146	*
6147	* @remarks The VINF_EM_NO_MEMORY status is for the benefit of the FF processing
6148	* in EM.cpp and shouldn't be propagated outside TRPM, HM, EM and
6149	* pgmPhysEnsureHandyPage. There is one exception to this in the \#PF
6150	* handler.
6151	*/
6152	VMMR3DECL(int) PGMR3PhysAllocateHandyPages(PVM pVM)
6153	{
6154	PGM_LOCK_VOID(pVM);
6155
6156	/*
6157	* Allocate more pages, noting down the index of the first new page.
6158	*/
6159	uint32_t iClear = pVM->pgm.s.cHandyPages;
6160	AssertMsgReturn(iClear <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d", iClear), VERR_PGM_HANDY_PAGE_IPE);
6161	Log(("PGMR3PhysAllocateHandyPages: %d -> %d\n", iClear, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
6162	int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
6163	/** @todo we should split this up into an allocate and flush operation. sometimes you want to flush and not allocate more (which will trigger the vm account limit error) */
6164	if ( rc == VERR_GMM_HIT_VM_ACCOUNT_LIMIT
6165	&& pVM->pgm.s.cHandyPages > 0)
6166	{
6167	/* Still handy pages left, so don't panic. */
6168	rc = VINF_SUCCESS;
6169	}
6170
6171	if (RT_SUCCESS(rc))
6172	{
6173	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
6174	Assert(pVM->pgm.s.cHandyPages > 0);
6175	#ifdef VBOX_STRICT
6176	uint32_t i;
6177	for (i = iClear; i < pVM->pgm.s.cHandyPages; i++)
6178	if ( pVM->pgm.s.aHandyPages[i].idPage == NIL_GMM_PAGEID
6179	\|\| pVM->pgm.s.aHandyPages[i].idSharedPage != NIL_GMM_PAGEID
6180	\|\| (pVM->pgm.s.aHandyPages[i].HCPhysGCPhys & GUEST_PAGE_OFFSET_MASK))
6181	break;
6182	if (i != pVM->pgm.s.cHandyPages)
6183	{
6184	RTAssertMsg1Weak(NULL, __LINE__, __FILE__, __FUNCTION__);
6185	RTAssertMsg2Weak("i=%d iClear=%d cHandyPages=%d\n", i, iClear, pVM->pgm.s.cHandyPages);
6186	for (uint32_t j = iClear; j < pVM->pgm.s.cHandyPages; j++)
6187	RTAssertMsg2Add("%03d: idPage=%d HCPhysGCPhys=%RHp idSharedPage=%d%s\n", j,
6188	pVM->pgm.s.aHandyPages[j].idPage,
6189	pVM->pgm.s.aHandyPages[j].HCPhysGCPhys,
6190	pVM->pgm.s.aHandyPages[j].idSharedPage,
6191	j == i ? " <---" : "");
6192	RTAssertPanic();
6193	}
6194	#endif
6195	}
6196	else
6197	{
6198	/*
6199	* We should never get here unless there is a genuine shortage of
6200	* memory (or some internal error). Flag the error so the VM can be
6201	* suspended ASAP and the user informed. If we're totally out of
6202	* handy pages we will return failure.
6203	*/
6204	/* Report the failure. */
6205	LogRel(("PGM: Failed to procure handy pages; rc=%Rrc cHandyPages=%#x\n"
6206	" cAllPages=%#x cPrivatePages=%#x cSharedPages=%#x cZeroPages=%#x\n",
6207	rc, pVM->pgm.s.cHandyPages,
6208	pVM->pgm.s.cAllPages, pVM->pgm.s.cPrivatePages, pVM->pgm.s.cSharedPages, pVM->pgm.s.cZeroPages));
6209
6210	if ( rc != VERR_NO_MEMORY
6211	&& rc != VERR_NO_PHYS_MEMORY
6212	&& rc != VERR_LOCK_FAILED)
6213	for (uint32_t i = 0; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
6214	{
6215	LogRel(("PGM: aHandyPages[#%#04x] = {.HCPhysGCPhys=%RHp, .idPage=%#08x, .idSharedPage=%#08x}\n",
6216	i, pVM->pgm.s.aHandyPages[i].HCPhysGCPhys, pVM->pgm.s.aHandyPages[i].idPage,
6217	pVM->pgm.s.aHandyPages[i].idSharedPage));
6218	uint32_t const idPage = pVM->pgm.s.aHandyPages[i].idPage;
6219	if (idPage != NIL_GMM_PAGEID)
6220	{
6221	uint32_t const idRamRangeMax = RT_MIN(pVM->pgm.s.idRamRangeMax, RT_ELEMENTS(pVM->pgm.s.apRamRanges) - 1U);
6222	for (uint32_t idRamRange = 0; idRamRange <= idRamRangeMax; idRamRange++)
6223	{
6224	PPGMRAMRANGE const pRam = pVM->pgm.s.apRamRanges[idRamRange];
6225	Assert(pRam \|\| idRamRange == 0);
6226	if (!pRam) continue;
6227	Assert(pRam->idRange == idRamRange);
6228
6229	uint32_t const cPages = pRam->cb >> GUEST_PAGE_SHIFT;
6230	for (uint32_t iPage = 0; iPage < cPages; iPage++)
6231	if (PGM_PAGE_GET_PAGEID(&pRam->aPages[iPage]) == idPage)
6232	LogRel(("PGM: Used by %RGp %R[pgmpage] (%s)\n",
6233	pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT), &pRam->aPages[iPage], pRam->pszDesc));
6234	}
6235	}
6236	}
6237
6238	if (rc == VERR_NO_MEMORY)
6239	{
6240	uint64_t cbHostRamAvail = 0;
6241	int rc2 = RTSystemQueryAvailableRam(&cbHostRamAvail);
6242	if (RT_SUCCESS(rc2))
6243	LogRel(("Host RAM: %RU64MB available\n", cbHostRamAvail / _1M));
6244	else
6245	LogRel(("Cannot determine the amount of available host memory\n"));
6246	}
6247
6248	/* Set the FFs and adjust rc. */
6249	VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
6250	VM_FF_SET(pVM, VM_FF_PGM_NO_MEMORY);
6251	if ( rc == VERR_NO_MEMORY
6252	\|\| rc == VERR_NO_PHYS_MEMORY
6253	\|\| rc == VERR_LOCK_FAILED)
6254	rc = VINF_EM_NO_MEMORY;
6255	}
6256
6257	PGM_UNLOCK(pVM);
6258	return rc;
6259	}
6260
6261
6262	/*********************************************************************************************************************************
6263	* Other Stuff *
6264	*********************************************************************************************************************************/
6265
6266	#if !defined(VBOX_VMM_TARGET_ARMV8)
6267	/**
6268	* Sets the Address Gate 20 state.
6269	*
6270	* @param pVCpu The cross context virtual CPU structure.
6271	* @param fEnable True if the gate should be enabled.
6272	* False if the gate should be disabled.
6273	*/
6274	VMMDECL(void) PGMR3PhysSetA20(PVMCPU pVCpu, bool fEnable)
6275	{
6276	LogFlow(("PGMR3PhysSetA20 %d (was %d)\n", fEnable, pVCpu->pgm.s.fA20Enabled));
6277	if (pVCpu->pgm.s.fA20Enabled != fEnable)
6278	{
6279	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6280	PCCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
6281	if ( CPUMIsGuestInVmxRootMode(pCtx)
6282	&& !fEnable)
6283	{
6284	Log(("Cannot enter A20M mode while in VMX root mode\n"));
6285	return;
6286	}
6287	#endif
6288	pVCpu->pgm.s.fA20Enabled = fEnable;
6289	pVCpu->pgm.s.GCPhysA20Mask = ~((RTGCPHYS)!fEnable << 20);
6290	if (VM_IS_NEM_ENABLED(pVCpu->CTX_SUFF(pVM)))
6291	NEMR3NotifySetA20(pVCpu, fEnable);
6292	#ifdef PGM_WITH_A20
6293	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
6294	pgmR3RefreshShadowModeAfterA20Change(pVCpu);
6295	HMFlushTlb(pVCpu);
6296	#endif
6297	#if 0 /* PGMGetPage will apply the A20 mask to the GCPhys it returns, so we must invalid both sides of the TLB. */
6298	IEMTlbInvalidateAllPhysical(pVCpu);
6299	#else
6300	IEMTlbInvalidateAllGlobal(pVCpu);
6301	#endif
6302	STAM_REL_COUNTER_INC(&pVCpu->pgm.s.cA20Changes);
6303	}
6304	}
6305	#endif
6306

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR3/PGMPhys.cpp@ 105745

Download in other formats: