PGMAllPhys.cpp@ 31402

Last change on this file since 31402 was 31402, checked in by vboxsync, 14 years ago
PGM: Replaced the hazzardous raw-mode context dynamic mapping code with the PGMR0DynMap code used by darwin/x86. This is a risky change but it should pay off once stable by providing 100% certainty that dynamically mapped pages aren't resued behind our back (this has been observed in seemingly benign code paths recently).
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 127.3 KB

Line
1	/* $Id: PGMAllPhys.cpp 31402 2010-08-05 12:28:18Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor, Physical Memory Addressing.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_PGM_PHYS
22	#include <VBox/pgm.h>
23	#include <VBox/trpm.h>
24	#include <VBox/vmm.h>
25	#include <VBox/iom.h>
26	#include <VBox/em.h>
27	#include <VBox/rem.h>
28	#include "../PGMInternal.h"
29	#include <VBox/vm.h>
30	#include "../PGMInline.h"
31	#include <VBox/param.h>
32	#include <VBox/err.h>
33	#include <iprt/assert.h>
34	#include <iprt/string.h>
35	#include <iprt/asm-amd64-x86.h>
36	#include <VBox/log.h>
37	#ifdef IN_RING3
38	# include <iprt/thread.h>
39	#endif
40
41
42	/*******************************************************************************
43	* Defined Constants And Macros *
44	*******************************************************************************/
45	/** Enable the physical TLB. */
46	#define PGM_WITH_PHYS_TLB
47
48
49
50	#ifndef IN_RING3
51
52	/**
53	* \#PF Handler callback for physical memory accesses without a RC/R0 handler.
54	* This simply pushes everything to the HC handler.
55	*
56	* @returns VBox status code (appropritate for trap handling and GC return).
57	* @param pVM VM Handle.
58	* @param uErrorCode CPU Error code.
59	* @param pRegFrame Trap register frame.
60	* @param pvFault The fault address (cr2).
61	* @param GCPhysFault The GC physical address corresponding to pvFault.
62	* @param pvUser User argument.
63	*/
64	VMMDECL(int) pgmPhysHandlerRedirectToHC(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
65	{
66	return (uErrorCode & X86_TRAP_PF_RW) ? VINF_IOM_HC_MMIO_WRITE : VINF_IOM_HC_MMIO_READ;
67	}
68
69
70	/**
71	* \#PF Handler callback for Guest ROM range write access.
72	* We simply ignore the writes or fall back to the recompiler if we don't support the instruction.
73	*
74	* @returns VBox status code (appropritate for trap handling and GC return).
75	* @param pVM VM Handle.
76	* @param uErrorCode CPU Error code.
77	* @param pRegFrame Trap register frame.
78	* @param pvFault The fault address (cr2).
79	* @param GCPhysFault The GC physical address corresponding to pvFault.
80	* @param pvUser User argument. Pointer to the ROM range structure.
81	*/
82	VMMDECL(int) pgmPhysRomWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
83	{
84	int rc;
85	PPGMROMRANGE pRom = (PPGMROMRANGE)pvUser;
86	uint32_t iPage = (GCPhysFault - pRom->GCPhys) >> PAGE_SHIFT;
87	PVMCPU pVCpu = VMMGetCpu(pVM);
88
89	Assert(iPage < (pRom->cb >> PAGE_SHIFT));
90	switch (pRom->aPages[iPage].enmProt)
91	{
92	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
93	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
94	{
95	/*
96	* If it's a simple instruction which doesn't change the cpu state
97	* we will simply skip it. Otherwise we'll have to defer it to REM.
98	*/
99	uint32_t cbOp;
100	PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
101	rc = EMInterpretDisasOne(pVM, pVCpu, pRegFrame, pDis, &cbOp);
102	if ( RT_SUCCESS(rc)
103	&& pDis->mode == CPUMODE_32BIT /** @todo why does this matter? */
104	&& !(pDis->prefix & (PREFIX_REPNE \| PREFIX_REP \| PREFIX_SEG)))
105	{
106	switch (pDis->opcode)
107	{
108	/** @todo Find other instructions we can safely skip, possibly
109	* adding this kind of detection to DIS or EM. */
110	case OP_MOV:
111	pRegFrame->rip += cbOp;
112	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestROMWriteHandled);
113	return VINF_SUCCESS;
114	}
115	}
116	else if (RT_UNLIKELY(rc == VERR_INTERNAL_ERROR))
117	return rc;
118	break;
119	}
120
121	case PGMROMPROT_READ_RAM_WRITE_RAM:
122	pRom->aPages[iPage].LiveSave.fWrittenTo = true;
123	rc = PGMHandlerPhysicalPageTempOff(pVM, pRom->GCPhys, GCPhysFault & X86_PTE_PG_MASK);
124	AssertRC(rc);
125	break; /** @todo Must edit the shadow PT and restart the instruction, not use the interpreter! */
126
127	case PGMROMPROT_READ_ROM_WRITE_RAM:
128	/* Handle it in ring-3 because it's way easier there. */
129	pRom->aPages[iPage].LiveSave.fWrittenTo = true;
130	break;
131
132	default:
133	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhysFault=%RGp\n",
134	pRom->aPages[iPage].enmProt, iPage, GCPhysFault),
135	VERR_INTERNAL_ERROR);
136	}
137
138	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestROMWriteUnhandled);
139	return VINF_EM_RAW_EMULATE_INSTR;
140	}
141
142	#endif /* IN_RING3 */
143
144	/**
145	* Checks if Address Gate 20 is enabled or not.
146	*
147	* @returns true if enabled.
148	* @returns false if disabled.
149	* @param pVCpu VMCPU handle.
150	*/
151	VMMDECL(bool) PGMPhysIsA20Enabled(PVMCPU pVCpu)
152	{
153	LogFlow(("PGMPhysIsA20Enabled %d\n", pVCpu->pgm.s.fA20Enabled));
154	return pVCpu->pgm.s.fA20Enabled;
155	}
156
157
158	/**
159	* Validates a GC physical address.
160	*
161	* @returns true if valid.
162	* @returns false if invalid.
163	* @param pVM The VM handle.
164	* @param GCPhys The physical address to validate.
165	*/
166	VMMDECL(bool) PGMPhysIsGCPhysValid(PVM pVM, RTGCPHYS GCPhys)
167	{
168	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
169	return pPage != NULL;
170	}
171
172
173	/**
174	* Checks if a GC physical address is a normal page,
175	* i.e. not ROM, MMIO or reserved.
176	*
177	* @returns true if normal.
178	* @returns false if invalid, ROM, MMIO or reserved page.
179	* @param pVM The VM handle.
180	* @param GCPhys The physical address to check.
181	*/
182	VMMDECL(bool) PGMPhysIsGCPhysNormal(PVM pVM, RTGCPHYS GCPhys)
183	{
184	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
185	return pPage
186	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM;
187	}
188
189
190	/**
191	* Converts a GC physical address to a HC physical address.
192	*
193	* @returns VINF_SUCCESS on success.
194	* @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
195	* page but has no physical backing.
196	* @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
197	* GC physical address.
198	*
199	* @param pVM The VM handle.
200	* @param GCPhys The GC physical address to convert.
201	* @param pHCPhys Where to store the HC physical address on success.
202	*/
203	VMMDECL(int) PGMPhysGCPhys2HCPhys(PVM pVM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys)
204	{
205	pgmLock(pVM);
206	PPGMPAGE pPage;
207	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
208	if (RT_SUCCESS(rc))
209	*pHCPhys = PGM_PAGE_GET_HCPHYS(pPage) \| (GCPhys & PAGE_OFFSET_MASK);
210	pgmUnlock(pVM);
211	return rc;
212	}
213
214
215	/**
216	* Invalidates all page mapping TLBs.
217	*
218	* @param pVM The VM handle.
219	*/
220	VMMDECL(void) PGMPhysInvalidatePageMapTLB(PVM pVM)
221	{
222	pgmLock(pVM);
223	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatPageMapTlbFlushes);
224	/* Clear the shared R0/R3 TLB completely. */
225	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbHC.aEntries); i++)
226	{
227	pVM->pgm.s.PhysTlbHC.aEntries[i].GCPhys = NIL_RTGCPHYS;
228	pVM->pgm.s.PhysTlbHC.aEntries[i].pPage = 0;
229	pVM->pgm.s.PhysTlbHC.aEntries[i].pMap = 0;
230	pVM->pgm.s.PhysTlbHC.aEntries[i].pv = 0;
231	}
232	/** @todo clear the RC TLB whenever we add it. */
233	pgmUnlock(pVM);
234	}
235
236	/**
237	* Invalidates a page mapping TLB entry
238	*
239	* @param pVM The VM handle.
240	* @param GCPhys GCPhys entry to flush
241	*/
242	VMMDECL(void) PGMPhysInvalidatePageMapTLBEntry(PVM pVM, RTGCPHYS GCPhys)
243	{
244	Assert(PGMIsLocked(pVM));
245
246	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatPageMapTlbFlushEntry);
247	/* Clear the shared R0/R3 TLB entry. */
248	#ifdef IN_RC
249	unsigned idx = PGM_PAGER3MAPTLB_IDX(GCPhys);
250	pVM->pgm.s.PhysTlbHC.aEntries[idx].GCPhys = NIL_RTGCPHYS;
251	pVM->pgm.s.PhysTlbHC.aEntries[idx].pPage = 0;
252	pVM->pgm.s.PhysTlbHC.aEntries[idx].pMap = 0;
253	pVM->pgm.s.PhysTlbHC.aEntries[idx].pv = 0;
254	#else
255	PPGMPAGEMAPTLBE pTlbe = &pVM->pgm.s.CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
256	pTlbe->GCPhys = NIL_RTGCPHYS;
257	pTlbe->pPage = 0;
258	pTlbe->pMap = 0;
259	pTlbe->pv = 0;
260	#endif
261	/* @todo clear the RC TLB whenever we add it. */
262	}
263
264	/**
265	* Makes sure that there is at least one handy page ready for use.
266	*
267	* This will also take the appropriate actions when reaching water-marks.
268	*
269	* @returns VBox status code.
270	* @retval VINF_SUCCESS on success.
271	* @retval VERR_EM_NO_MEMORY if we're really out of memory.
272	*
273	* @param pVM The VM handle.
274	*
275	* @remarks Must be called from within the PGM critical section. It may
276	* nip back to ring-3/0 in some cases.
277	*/
278	static int pgmPhysEnsureHandyPage(PVM pVM)
279	{
280	AssertMsg(pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d\n", pVM->pgm.s.cHandyPages));
281
282	/*
283	* Do we need to do anything special?
284	*/
285	#ifdef IN_RING3
286	if (pVM->pgm.s.cHandyPages <= RT_MAX(PGM_HANDY_PAGES_SET_FF, PGM_HANDY_PAGES_R3_ALLOC))
287	#else
288	if (pVM->pgm.s.cHandyPages <= RT_MAX(PGM_HANDY_PAGES_SET_FF, PGM_HANDY_PAGES_RZ_TO_R3))
289	#endif
290	{
291	/*
292	* Allocate pages only if we're out of them, or in ring-3, almost out.
293	*/
294	#ifdef IN_RING3
295	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_R3_ALLOC)
296	#else
297	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_RZ_ALLOC)
298	#endif
299	{
300	Log(("PGM: cHandyPages=%u out of %u -> allocate more; VM_FF_PGM_NO_MEMORY=%RTbool\n",
301	pVM->pgm.s.cHandyPages, RT_ELEMENTS(pVM->pgm.s.aHandyPages), VM_FF_ISSET(pVM, VM_FF_PGM_NO_MEMORY) ));
302	#ifdef IN_RING3
303	int rc = PGMR3PhysAllocateHandyPages(pVM);
304	#else
305	int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES, 0);
306	#endif
307	if (RT_UNLIKELY(rc != VINF_SUCCESS))
308	{
309	if (RT_FAILURE(rc))
310	return rc;
311	AssertMsgReturn(rc == VINF_EM_NO_MEMORY, ("%Rrc\n", rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
312	if (!pVM->pgm.s.cHandyPages)
313	{
314	LogRel(("PGM: no more handy pages!\n"));
315	return VERR_EM_NO_MEMORY;
316	}
317	Assert(VM_FF_ISSET(pVM, VM_FF_PGM_NEED_HANDY_PAGES));
318	Assert(VM_FF_ISSET(pVM, VM_FF_PGM_NO_MEMORY));
319	#ifdef IN_RING3
320	REMR3NotifyFF(pVM);
321	#else
322	VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3); /* paranoia */
323	#endif
324	}
325	AssertMsgReturn( pVM->pgm.s.cHandyPages > 0
326	&& pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages),
327	("%u\n", pVM->pgm.s.cHandyPages),
328	VERR_INTERNAL_ERROR);
329	}
330	else
331	{
332	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_SET_FF)
333	VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
334	#ifndef IN_RING3
335	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_RZ_TO_R3)
336	{
337	Log(("PGM: VM_FF_TO_R3 - cHandyPages=%u out of %u\n", pVM->pgm.s.cHandyPages, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
338	VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3);
339	}
340	#endif
341	}
342	}
343
344	return VINF_SUCCESS;
345	}
346
347
348	/**
349	* Replace a zero or shared page with new page that we can write to.
350	*
351	* @returns The following VBox status codes.
352	* @retval VINF_SUCCESS on success, pPage is modified.
353	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
354	* @retval VERR_EM_NO_MEMORY if we're totally out of memory.
355	*
356	* @todo Propagate VERR_EM_NO_MEMORY up the call tree.
357	*
358	* @param pVM The VM address.
359	* @param pPage The physical page tracking structure. This will
360	* be modified on success.
361	* @param GCPhys The address of the page.
362	*
363	* @remarks Must be called from within the PGM critical section. It may
364	* nip back to ring-3/0 in some cases.
365	*
366	* @remarks This function shouldn't really fail, however if it does
367	* it probably means we've screwed up the size of handy pages and/or
368	* the low-water mark. Or, that some device I/O is causing a lot of
369	* pages to be allocated while while the host is in a low-memory
370	* condition. This latter should be handled elsewhere and in a more
371	* controlled manner, it's on the @bugref{3170} todo list...
372	*/
373	int pgmPhysAllocPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
374	{
375	LogFlow(("pgmPhysAllocPage: %R[pgmpage] %RGp\n", pPage, GCPhys));
376
377	/*
378	* Prereqs.
379	*/
380	Assert(PGMIsLocked(pVM));
381	AssertMsg(PGM_PAGE_IS_ZERO(pPage) \|\| PGM_PAGE_IS_SHARED(pPage), ("%R[pgmpage] %RGp\n", pPage, GCPhys));
382	Assert(!PGM_PAGE_IS_MMIO(pPage));
383
384	# ifdef PGM_WITH_LARGE_PAGES
385	if ( PGMIsUsingLargePages(pVM)
386	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
387	{
388	int rc = pgmPhysAllocLargePage(pVM, GCPhys);
389	if (rc == VINF_SUCCESS)
390	return rc;
391
392	/* fall back to 4KB pages. */
393	}
394	# endif
395
396	/*
397	* Flush any shadow page table mappings of the page.
398	* When VBOX_WITH_NEW_LAZY_PAGE_ALLOC isn't defined, there shouldn't be any.
399	*/
400	bool fFlushTLBs = false;
401	int rc = pgmPoolTrackFlushGCPhys(pVM, GCPhys, pPage, &fFlushTLBs);
402	AssertMsgReturn(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc), RT_FAILURE(rc) ? rc : VERR_IPE_UNEXPECTED_STATUS);
403
404	/*
405	* Ensure that we've got a page handy, take it and use it.
406	*/
407	int rc2 = pgmPhysEnsureHandyPage(pVM);
408	if (RT_FAILURE(rc2))
409	{
410	if (fFlushTLBs)
411	PGM_INVL_ALL_VCPU_TLBS(pVM);
412	Assert(rc2 == VERR_EM_NO_MEMORY);
413	return rc2;
414	}
415	/* re-assert preconditions since pgmPhysEnsureHandyPage may do a context switch. */
416	Assert(PGMIsLocked(pVM));
417	AssertMsg(PGM_PAGE_IS_ZERO(pPage) \|\| PGM_PAGE_IS_SHARED(pPage), ("%R[pgmpage] %RGp\n", pPage, GCPhys));
418	Assert(!PGM_PAGE_IS_MMIO(pPage));
419
420	uint32_t iHandyPage = --pVM->pgm.s.cHandyPages;
421	AssertMsg(iHandyPage < RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d\n", iHandyPage));
422	Assert(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys != NIL_RTHCPHYS);
423	Assert(!(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys & ~X86_PTE_PAE_PG_MASK));
424	Assert(pVM->pgm.s.aHandyPages[iHandyPage].idPage != NIL_GMM_PAGEID);
425	Assert(pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage == NIL_GMM_PAGEID);
426
427	/*
428	* There are one or two action to be taken the next time we allocate handy pages:
429	* - Tell the GMM (global memory manager) what the page is being used for.
430	* (Speeds up replacement operations - sharing and defragmenting.)
431	* - If the current backing is shared, it must be freed.
432	*/
433	const RTHCPHYS HCPhys = pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys;
434	pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys = GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
435
436	const void *pvSharedPage = NULL;
437
438	if (PGM_PAGE_IS_SHARED(pPage))
439	{
440	/* Mark this shared page for freeing/derefencing. */
441	pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage = PGM_PAGE_GET_PAGEID(pPage);
442	Assert(PGM_PAGE_GET_PAGEID(pPage) != NIL_GMM_PAGEID);
443
444	Log(("PGM: Replaced shared page %#x at %RGp with %#x / %RHp\n", PGM_PAGE_GET_PAGEID(pPage),
445	GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys));
446	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageReplaceShared));
447	pVM->pgm.s.cSharedPages--;
448
449	/* Grab the address of the page so we can make a copy later on. */
450	rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, GCPhys, &pvSharedPage);
451	AssertRC(rc);
452	}
453	else
454	{
455	Log2(("PGM: Replaced zero page %RGp with %#x / %RHp\n", GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys));
456	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatRZPageReplaceZero);
457	pVM->pgm.s.cZeroPages--;
458	}
459
460	/*
461	* Do the PGMPAGE modifications.
462	*/
463	pVM->pgm.s.cPrivatePages++;
464	PGM_PAGE_SET_HCPHYS(pPage, HCPhys);
465	PGM_PAGE_SET_PAGEID(pPage, pVM->pgm.s.aHandyPages[iHandyPage].idPage);
466	PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ALLOCATED);
467	PGM_PAGE_SET_PDE_TYPE(pPage, PGM_PAGE_PDE_TYPE_PT);
468	PGMPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
469
470	/* Copy the shared page contents to the replacement page. */
471	if (pvSharedPage)
472	{
473	/* Get the virtual address of the new page. */
474	void *pvNewPage;
475	rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvNewPage);
476	AssertRC(rc);
477	if (rc == VINF_SUCCESS)
478	{
479	/** @todo todo write ASMMemCopyPage */
480	memcpy(pvNewPage, pvSharedPage, PAGE_SIZE);
481	}
482	}
483
484	if ( fFlushTLBs
485	&& rc != VINF_PGM_GCPHYS_ALIASED)
486	PGM_INVL_ALL_VCPU_TLBS(pVM);
487	return rc;
488	}
489
490	#ifdef PGM_WITH_LARGE_PAGES
491	/**
492	* Replace a 2 MB range of zero pages with new pages that we can write to.
493	*
494	* @returns The following VBox status codes.
495	* @retval VINF_SUCCESS on success, pPage is modified.
496	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
497	* @retval VERR_EM_NO_MEMORY if we're totally out of memory.
498	*
499	* @todo Propagate VERR_EM_NO_MEMORY up the call tree.
500	*
501	* @param pVM The VM address.
502	* @param GCPhys The address of the page.
503	*
504	* @remarks Must be called from within the PGM critical section. It may
505	* nip back to ring-3/0 in some cases.
506	*/
507	int pgmPhysAllocLargePage(PVM pVM, RTGCPHYS GCPhys)
508	{
509	RTGCPHYS GCPhysBase = GCPhys & X86_PDE2M_PAE_PG_MASK;
510	LogFlow(("pgmPhysAllocLargePage: %RGp base %RGp\n", GCPhys, GCPhysBase));
511
512	/*
513	* Prereqs.
514	*/
515	Assert(PGMIsLocked(pVM));
516	Assert(PGMIsUsingLargePages(pVM));
517
518	PPGMPAGE pPage;
519	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhysBase, &pPage);
520	if ( RT_SUCCESS(rc)
521	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
522	{
523	unsigned uPDEType = PGM_PAGE_GET_PDE_TYPE(pPage);
524
525	/* Don't call this function for already allocated pages. */
526	Assert(uPDEType != PGM_PAGE_PDE_TYPE_PDE);
527
528	if ( uPDEType == PGM_PAGE_PDE_TYPE_DONTCARE
529	&& PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ZERO)
530	{
531	unsigned iPage;
532
533	GCPhys = GCPhysBase;
534
535	/* Lazy approach: check all pages in the 2 MB range.
536	* The whole range must be ram and unallocated
537	*/
538	for (iPage = 0; iPage < _2M/PAGE_SIZE; iPage++)
539	{
540	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
541	if ( RT_FAILURE(rc)
542	\|\| PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM /* Anything other than ram implies monitoring. */
543	\|\| PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ZERO) /* allocated, monitored or shared means we can't use a large page here */
544	{
545	LogFlow(("Found page %RGp with wrong attributes (type=%d; state=%d); cancel check. rc=%d\n", GCPhys, PGM_PAGE_GET_TYPE(pPage), PGM_PAGE_GET_STATE(pPage), rc));
546	break;
547	}
548	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_DONTCARE);
549	GCPhys += PAGE_SIZE;
550	}
551	/* Fetch the start page of the 2 MB range again. */
552	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhysBase, &pPage);
553	AssertRC(rc); /* can't fail */
554
555	if (iPage != _2M/PAGE_SIZE)
556	{
557	/* Failed. Mark as requiring a PT so we don't check the whole thing again in the future. */
558	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageRefused);
559	PGM_PAGE_SET_PDE_TYPE(pPage, PGM_PAGE_PDE_TYPE_PT);
560	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
561	}
562	else
563	{
564	# ifdef IN_RING3
565	rc = PGMR3PhysAllocateLargeHandyPage(pVM, GCPhysBase);
566	# else
567	rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_ALLOCATE_LARGE_HANDY_PAGE, GCPhysBase);
568	# endif
569	if (RT_SUCCESS(rc))
570	{
571	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
572	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageAlloc);
573	return VINF_SUCCESS;
574	}
575	LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
576
577	/* If we fail once, it most likely means the host's memory is too fragmented; don't bother trying again. */
578	PGMSetLargePageUsage(pVM, false);
579	return rc;
580	}
581	}
582	}
583	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
584	}
585
586	/**
587	* Recheck the entire 2 MB range to see if we can use it again as a large page.
588	*
589	* @returns The following VBox status codes.
590	* @retval VINF_SUCCESS on success, the large page can be used again
591	* @retval VERR_PGM_INVALID_LARGE_PAGE_RANGE if it can't be reused
592	*
593	* @param pVM The VM address.
594	* @param GCPhys The address of the page.
595	* @param pLargePage Page structure of the base page
596	*/
597	int pgmPhysIsValidLargePage(PVM pVM, RTGCPHYS GCPhys, PPGMPAGE pLargePage)
598	{
599	unsigned i;
600
601	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageRecheck);
602
603	GCPhys &= X86_PDE2M_PAE_PG_MASK;
604
605	/* Check the base page. */
606	Assert(PGM_PAGE_GET_PDE_TYPE(pLargePage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED);
607	if ( PGM_PAGE_GET_STATE(pLargePage) != PGM_PAGE_STATE_ALLOCATED
608	\|\| PGM_PAGE_GET_TYPE(pLargePage) != PGMPAGETYPE_RAM
609	\|\| PGM_PAGE_GET_HNDL_PHYS_STATE(pLargePage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
610	{
611	LogFlow(("pgmPhysIsValidLargePage: checks failed for base page %x %x %x\n", PGM_PAGE_GET_STATE(pLargePage), PGM_PAGE_GET_TYPE(pLargePage), PGM_PAGE_GET_HNDL_PHYS_STATE(pLargePage)));
612	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
613	}
614
615	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,IsValidLargePage), a);
616	/* Check all remaining pages in the 2 MB range. */
617	GCPhys += PAGE_SIZE;
618	for (i = 1; i < _2M/PAGE_SIZE; i++)
619	{
620	PPGMPAGE pPage;
621	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
622	AssertRCBreak(rc);
623
624	if ( PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
625	\|\| PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE
626	\|\| PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
627	\|\| PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
628	{
629	LogFlow(("pgmPhysIsValidLargePage: checks failed for page %d; %x %x %x\n", i, PGM_PAGE_GET_STATE(pPage), PGM_PAGE_GET_TYPE(pPage), PGM_PAGE_GET_HNDL_PHYS_STATE(pPage)));
630	break;
631	}
632
633	GCPhys += PAGE_SIZE;
634	}
635	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,IsValidLargePage), a);
636
637	if (i == _2M/PAGE_SIZE)
638	{
639	PGM_PAGE_SET_PDE_TYPE(pLargePage, PGM_PAGE_PDE_TYPE_PDE);
640	Log(("pgmPhysIsValidLargePage: page %RGp can be reused!\n", GCPhys - _2M));
641	return VINF_SUCCESS;
642	}
643
644	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
645	}
646
647	#endif /* PGM_WITH_LARGE_PAGES */
648
649	/**
650	* Deal with a write monitored page.
651	*
652	* @returns VBox strict status code.
653	*
654	* @param pVM The VM address.
655	* @param pPage The physical page tracking structure.
656	*
657	* @remarks Called from within the PGM critical section.
658	*/
659	void pgmPhysPageMakeWriteMonitoredWritable(PVM pVM, PPGMPAGE pPage)
660	{
661	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED);
662	PGM_PAGE_SET_WRITTEN_TO(pPage);
663	PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ALLOCATED);
664	Assert(pVM->pgm.s.cMonitoredPages > 0);
665	pVM->pgm.s.cMonitoredPages--;
666	pVM->pgm.s.cWrittenToPages++;
667	}
668
669
670	/**
671	* Deal with pages that are not writable, i.e. not in the ALLOCATED state.
672	*
673	* @returns VBox strict status code.
674	* @retval VINF_SUCCESS on success.
675	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
676	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
677	*
678	* @param pVM The VM address.
679	* @param pPage The physical page tracking structure.
680	* @param GCPhys The address of the page.
681	*
682	* @remarks Called from within the PGM critical section.
683	*/
684	int pgmPhysPageMakeWritable(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
685	{
686	Assert(PGMIsLockOwner(pVM));
687	switch (PGM_PAGE_GET_STATE(pPage))
688	{
689	case PGM_PAGE_STATE_WRITE_MONITORED:
690	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
691	/* fall thru */
692	default: /* to shut up GCC */
693	case PGM_PAGE_STATE_ALLOCATED:
694	return VINF_SUCCESS;
695
696	/*
697	* Zero pages can be dummy pages for MMIO or reserved memory,
698	* so we need to check the flags before joining cause with
699	* shared page replacement.
700	*/
701	case PGM_PAGE_STATE_ZERO:
702	if (PGM_PAGE_IS_MMIO(pPage))
703	return VERR_PGM_PHYS_PAGE_RESERVED;
704	/* fall thru */
705	case PGM_PAGE_STATE_SHARED:
706	return pgmPhysAllocPage(pVM, pPage, GCPhys);
707
708	/* Not allowed to write to ballooned pages. */
709	case PGM_PAGE_STATE_BALLOONED:
710	return VERR_PGM_PHYS_PAGE_BALLOONED;
711	}
712	}
713
714
715	/**
716	* Internal usage: Map the page specified by its GMM ID.
717	*
718	* This is similar to pgmPhysPageMap
719	*
720	* @returns VBox status code.
721	*
722	* @param pVM The VM handle.
723	* @param idPage The Page ID.
724	* @param HCPhys The physical address (for RC).
725	* @param ppv Where to store the mapping address.
726	*
727	* @remarks Called from within the PGM critical section. The mapping is only
728	* valid while your inside this section.
729	*/
730	int pgmPhysPageMapByPageID(PVM pVM, uint32_t idPage, RTHCPHYS HCPhys, void **ppv)
731	{
732	/*
733	* Validation.
734	*/
735	Assert(PGMIsLocked(pVM));
736	AssertReturn(HCPhys && !(HCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
737	const uint32_t idChunk = idPage >> GMM_CHUNKID_SHIFT;
738	AssertReturn(idChunk != NIL_GMM_CHUNKID, VERR_INVALID_PARAMETER);
739
740	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
741	/*
742	* Map it by HCPhys.
743	*/
744	return pgmRZDynMapHCPageInlined(VMMGetCpu(pVM), HCPhys, ppv RTLOG_COMMA_SRC_POS);
745
746	#else
747	/*
748	* Find/make Chunk TLB entry for the mapping chunk.
749	*/
750	PPGMCHUNKR3MAP pMap;
751	PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)];
752	if (pTlbe->idChunk == idChunk)
753	{
754	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,ChunkR3MapTlbHits));
755	pMap = pTlbe->pChunk;
756	}
757	else
758	{
759	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,ChunkR3MapTlbMisses));
760
761	/*
762	* Find the chunk, map it if necessary.
763	*/
764	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
765	if (!pMap)
766	{
767	# ifdef IN_RING0
768	int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_MAP_CHUNK, idChunk);
769	AssertRCReturn(rc, rc);
770	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
771	Assert(pMap);
772	# else
773	int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap);
774	if (RT_FAILURE(rc))
775	return rc;
776	# endif
777	}
778
779	/*
780	* Enter it into the Chunk TLB.
781	*/
782	pTlbe->idChunk = idChunk;
783	pTlbe->pChunk = pMap;
784	pMap->iAge = 0;
785	}
786
787	ppv = (uint8_t )pMap->pv + ((idPage &GMM_PAGEID_IDX_MASK) << PAGE_SHIFT);
788	return VINF_SUCCESS;
789	#endif
790	}
791
792
793	/**
794	* Maps a page into the current virtual address space so it can be accessed.
795	*
796	* @returns VBox status code.
797	* @retval VINF_SUCCESS on success.
798	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
799	*
800	* @param pVM The VM address.
801	* @param pPage The physical page tracking structure.
802	* @param GCPhys The address of the page.
803	* @param ppMap Where to store the address of the mapping tracking structure.
804	* @param ppv Where to store the mapping address of the page. The page
805	* offset is masked off!
806	*
807	* @remarks Called from within the PGM critical section.
808	*/
809	static int pgmPhysPageMapCommon(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAP ppMap, void **ppv)
810	{
811	Assert(PGMIsLocked(pVM));
812
813	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
814	/*
815	* Just some sketchy GC/R0-darwin code.
816	*/
817	*ppMap = NULL;
818	RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
819	Assert(HCPhys != pVM->pgm.s.HCPhysZeroPg);
820	pgmRZDynMapHCPageInlined(VMMGetCpu(pVM), HCPhys, ppv RTLOG_COMMA_SRC_POS);
821	return VINF_SUCCESS;
822
823	#else /* IN_RING3 \|\| IN_RING0 */
824
825
826	/*
827	* Special case: ZERO and MMIO2 pages.
828	*/
829	const uint32_t idChunk = PGM_PAGE_GET_CHUNKID(pPage);
830	if (idChunk == NIL_GMM_CHUNKID)
831	{
832	AssertMsgReturn(PGM_PAGE_GET_PAGEID(pPage) == NIL_GMM_PAGEID, ("pPage=%R[pgmpage]\n", pPage), VERR_INTERNAL_ERROR_2);
833	if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2)
834	{
835	/* Lookup the MMIO2 range and use pvR3 to calc the address. */
836	PPGMRAMRANGE pRam = pgmPhysGetRange(&pVM->pgm.s, GCPhys);
837	AssertMsgReturn(pRam \|\| !pRam->pvR3, ("pRam=%p pPage=%R[pgmpage]\n", pRam, pPage), VERR_INTERNAL_ERROR_2);
838	ppv = (void )((uintptr_t)pRam->pvR3 + (uintptr_t)((GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK) - pRam->GCPhys));
839	}
840	else if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO)
841	{
842	/** @todo deal with aliased MMIO2 pages somehow...
843	* One solution would be to seed MMIO2 pages to GMM and get unique Page IDs for
844	* them, that would also avoid this mess. It would actually be kind of
845	* elegant... */
846	AssertLogRelMsgFailedReturn(("%RGp\n", GCPhys), VERR_INTERNAL_ERROR_3);
847	}
848	else
849	{
850	/** @todo handle MMIO2 */
851	AssertMsgReturn(PGM_PAGE_IS_ZERO(pPage), ("pPage=%R[pgmpage]\n", pPage), VERR_INTERNAL_ERROR_2);
852	AssertMsgReturn(PGM_PAGE_GET_HCPHYS(pPage) == pVM->pgm.s.HCPhysZeroPg,
853	("pPage=%R[pgmpage]\n", pPage),
854	VERR_INTERNAL_ERROR_2);
855	*ppv = pVM->pgm.s.CTXALLSUFF(pvZeroPg);
856	}
857	*ppMap = NULL;
858	return VINF_SUCCESS;
859	}
860
861	/*
862	* Find/make Chunk TLB entry for the mapping chunk.
863	*/
864	PPGMCHUNKR3MAP pMap;
865	PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)];
866	if (pTlbe->idChunk == idChunk)
867	{
868	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,ChunkR3MapTlbHits));
869	pMap = pTlbe->pChunk;
870	}
871	else
872	{
873	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,ChunkR3MapTlbMisses));
874
875	/*
876	* Find the chunk, map it if necessary.
877	*/
878	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
879	if (!pMap)
880	{
881	#ifdef IN_RING0
882	int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_MAP_CHUNK, idChunk);
883	AssertRCReturn(rc, rc);
884	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
885	Assert(pMap);
886	#else
887	int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap);
888	if (RT_FAILURE(rc))
889	return rc;
890	#endif
891	}
892
893	/*
894	* Enter it into the Chunk TLB.
895	*/
896	pTlbe->idChunk = idChunk;
897	pTlbe->pChunk = pMap;
898	pMap->iAge = 0;
899	}
900
901	ppv = (uint8_t )pMap->pv + (PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) << PAGE_SHIFT);
902	*ppMap = pMap;
903	return VINF_SUCCESS;
904	#endif /* IN_RING3 */
905	}
906
907
908	/**
909	* Combination of pgmPhysPageMakeWritable and pgmPhysPageMapWritable.
910	*
911	* This is typically used is paths where we cannot use the TLB methods (like ROM
912	* pages) or where there is no point in using them since we won't get many hits.
913	*
914	* @returns VBox strict status code.
915	* @retval VINF_SUCCESS on success.
916	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
917	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
918	*
919	* @param pVM The VM address.
920	* @param pPage The physical page tracking structure.
921	* @param GCPhys The address of the page.
922	* @param ppv Where to store the mapping address of the page. The page
923	* offset is masked off!
924	*
925	* @remarks Called from within the PGM critical section. The mapping is only
926	* valid while your inside this section.
927	*/
928	int pgmPhysPageMakeWritableAndMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
929	{
930	int rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
931	if (RT_SUCCESS(rc))
932	{
933	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* returned */, ("%Rrc\n", rc));
934	PPGMPAGEMAP pMapIgnore;
935	int rc2 = pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, ppv);
936	if (RT_FAILURE(rc2)) /* preserve rc */
937	rc = rc2;
938	}
939	return rc;
940	}
941
942
943	/**
944	* Maps a page into the current virtual address space so it can be accessed for
945	* both writing and reading.
946	*
947	* This is typically used is paths where we cannot use the TLB methods (like ROM
948	* pages) or where there is no point in using them since we won't get many hits.
949	*
950	* @returns VBox status code.
951	* @retval VINF_SUCCESS on success.
952	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
953	*
954	* @param pVM The VM address.
955	* @param pPage The physical page tracking structure. Must be in the
956	* allocated state.
957	* @param GCPhys The address of the page.
958	* @param ppv Where to store the mapping address of the page. The page
959	* offset is masked off!
960	*
961	* @remarks Called from within the PGM critical section. The mapping is only
962	* valid while your inside this section.
963	*/
964	int pgmPhysPageMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
965	{
966	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
967	PPGMPAGEMAP pMapIgnore;
968	return pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, ppv);
969	}
970
971
972	/**
973	* Maps a page into the current virtual address space so it can be accessed for
974	* reading.
975	*
976	* This is typically used is paths where we cannot use the TLB methods (like ROM
977	* pages) or where there is no point in using them since we won't get many hits.
978	*
979	* @returns VBox status code.
980	* @retval VINF_SUCCESS on success.
981	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
982	*
983	* @param pVM The VM address.
984	* @param pPage The physical page tracking structure.
985	* @param GCPhys The address of the page.
986	* @param ppv Where to store the mapping address of the page. The page
987	* offset is masked off!
988	*
989	* @remarks Called from within the PGM critical section. The mapping is only
990	* valid while your inside this section.
991	*/
992	int pgmPhysPageMapReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const **ppv)
993	{
994	PPGMPAGEMAP pMapIgnore;
995	return pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, (void **)ppv);
996	}
997
998
999	#if !defined(IN_RC) && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1000	/**
1001	* Load a guest page into the ring-3 physical TLB.
1002	*
1003	* @returns VBox status code.
1004	* @retval VINF_SUCCESS on success
1005	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1006	* @param pPGM The PGM instance pointer.
1007	* @param GCPhys The guest physical address in question.
1008	*/
1009	int pgmPhysPageLoadIntoTlb(PPGM pPGM, RTGCPHYS GCPhys)
1010	{
1011	Assert(PGMIsLocked(PGM2VM(pPGM)));
1012
1013	/*
1014	* Find the ram range and page and hand it over to the with-page function.
1015	* 99.8% of requests are expected to be in the first range.
1016	*/
1017	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
1018	RTGCPHYS off = GCPhys - pRam->GCPhys;
1019	if (RT_UNLIKELY(off >= pRam->cb))
1020	{
1021	do
1022	{
1023	pRam = pRam->CTX_SUFF(pNext);
1024	if (!pRam)
1025	{
1026	STAM_COUNTER_INC(&pPGM->CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbMisses));
1027	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
1028	}
1029	off = GCPhys - pRam->GCPhys;
1030	} while (off >= pRam->cb);
1031	}
1032
1033	return pgmPhysPageLoadIntoTlbWithPage(pPGM, &pRam->aPages[off >> PAGE_SHIFT], GCPhys);
1034	}
1035
1036
1037	/**
1038	* Load a guest page into the ring-3 physical TLB.
1039	*
1040	* @returns VBox status code.
1041	* @retval VINF_SUCCESS on success
1042	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1043	*
1044	* @param pPGM The PGM instance pointer.
1045	* @param pPage Pointer to the PGMPAGE structure corresponding to
1046	* GCPhys.
1047	* @param GCPhys The guest physical address in question.
1048	*/
1049	int pgmPhysPageLoadIntoTlbWithPage(PPGM pPGM, PPGMPAGE pPage, RTGCPHYS GCPhys)
1050	{
1051	Assert(PGMIsLocked(PGM2VM(pPGM)));
1052	STAM_COUNTER_INC(&pPGM->CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbMisses));
1053
1054	/*
1055	* Map the page.
1056	* Make a special case for the zero page as it is kind of special.
1057	*/
1058	PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
1059	if ( !PGM_PAGE_IS_ZERO(pPage)
1060	&& !PGM_PAGE_IS_BALLOONED(pPage))
1061	{
1062	void *pv;
1063	PPGMPAGEMAP pMap;
1064	int rc = pgmPhysPageMapCommon(PGM2VM(pPGM), pPage, GCPhys, &pMap, &pv);
1065	if (RT_FAILURE(rc))
1066	return rc;
1067	pTlbe->pMap = pMap;
1068	pTlbe->pv = pv;
1069	Assert(!((uintptr_t)pTlbe->pv & PAGE_OFFSET_MASK));
1070	}
1071	else
1072	{
1073	Assert(PGM_PAGE_GET_HCPHYS(pPage) == pPGM->HCPhysZeroPg);
1074	pTlbe->pMap = NULL;
1075	pTlbe->pv = pPGM->CTXALLSUFF(pvZeroPg);
1076	}
1077	#ifdef PGM_WITH_PHYS_TLB
1078	if ( PGM_PAGE_GET_TYPE(pPage) < PGMPAGETYPE_ROM_SHADOW
1079	\|\| PGM_PAGE_GET_TYPE(pPage) > PGMPAGETYPE_ROM)
1080	pTlbe->GCPhys = GCPhys & X86_PTE_PAE_PG_MASK;
1081	else
1082	pTlbe->GCPhys = NIL_RTGCPHYS; /* ROM: Problematic because of the two pages. :-/ */
1083	#else
1084	pTlbe->GCPhys = NIL_RTGCPHYS;
1085	#endif
1086	pTlbe->pPage = pPage;
1087	return VINF_SUCCESS;
1088	}
1089	#endif /* !IN_RC && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
1090
1091
1092	/**
1093	* Internal version of PGMPhysGCPhys2CCPtr that expects the caller to
1094	* own the PGM lock and therefore not need to lock the mapped page.
1095	*
1096	* @returns VBox status code.
1097	* @retval VINF_SUCCESS on success.
1098	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1099	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1100	*
1101	* @param pVM The VM handle.
1102	* @param GCPhys The guest physical address of the page that should be mapped.
1103	* @param pPage Pointer to the PGMPAGE structure for the page.
1104	* @param ppv Where to store the address corresponding to GCPhys.
1105	*
1106	* @internal
1107	*/
1108	int pgmPhysGCPhys2CCPtrInternal(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
1109	{
1110	int rc;
1111	AssertReturn(pPage, VERR_INTERNAL_ERROR);
1112	Assert(PGMIsLocked(pVM));
1113
1114	/*
1115	* Make sure the page is writable.
1116	*/
1117	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
1118	{
1119	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1120	if (RT_FAILURE(rc))
1121	return rc;
1122	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
1123	}
1124	Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0);
1125
1126	/*
1127	* Get the mapping address.
1128	*/
1129	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1130	*ppv = pgmRZDynMapHCPageOff(pVM, PGM_PAGE_GET_HCPHYS(pPage) \| (GCPhys & PAGE_OFFSET_MASK) RTLOG_COMMA_SRC_POS);
1131	#else
1132	PPGMPAGEMAPTLBE pTlbe;
1133	rc = pgmPhysPageQueryTlbeWithPage(&pVM->pgm.s, pPage, GCPhys, &pTlbe);
1134	if (RT_FAILURE(rc))
1135	return rc;
1136	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1137	#endif
1138	return VINF_SUCCESS;
1139	}
1140
1141
1142	/**
1143	* Internal version of PGMPhysGCPhys2CCPtrReadOnly that expects the caller to
1144	* own the PGM lock and therefore not need to lock the mapped page.
1145	*
1146	* @returns VBox status code.
1147	* @retval VINF_SUCCESS on success.
1148	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1149	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1150	*
1151	* @param pVM The VM handle.
1152	* @param GCPhys The guest physical address of the page that should be mapped.
1153	* @param pPage Pointer to the PGMPAGE structure for the page.
1154	* @param ppv Where to store the address corresponding to GCPhys.
1155	*
1156	* @internal
1157	*/
1158	int pgmPhysGCPhys2CCPtrInternalReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, const void **ppv)
1159	{
1160	AssertReturn(pPage, VERR_INTERNAL_ERROR);
1161	Assert(PGMIsLocked(pVM));
1162	Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0);
1163
1164	/*
1165	* Get the mapping address.
1166	*/
1167	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1168	ppv = pgmRZDynMapHCPageOff(pVM, PGM_PAGE_GET_HCPHYS(pPage) \| (GCPhys & PAGE_OFFSET_MASK) RTLOG_COMMA_SRC_POS); /* @todo add a read only flag? */
1169	#else
1170	PPGMPAGEMAPTLBE pTlbe;
1171	int rc = pgmPhysPageQueryTlbeWithPage(&pVM->pgm.s, pPage, GCPhys, &pTlbe);
1172	if (RT_FAILURE(rc))
1173	return rc;
1174	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1175	#endif
1176	return VINF_SUCCESS;
1177	}
1178
1179
1180	/**
1181	* Requests the mapping of a guest page into the current context.
1182	*
1183	* This API should only be used for very short term, as it will consume
1184	* scarse resources (R0 and GC) in the mapping cache. When you're done
1185	* with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
1186	*
1187	* This API will assume your intention is to write to the page, and will
1188	* therefore replace shared and zero pages. If you do not intend to modify
1189	* the page, use the PGMPhysGCPhys2CCPtrReadOnly() API.
1190	*
1191	* @returns VBox status code.
1192	* @retval VINF_SUCCESS on success.
1193	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1194	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1195	*
1196	* @param pVM The VM handle.
1197	* @param GCPhys The guest physical address of the page that should be mapped.
1198	* @param ppv Where to store the address corresponding to GCPhys.
1199	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
1200	*
1201	* @remarks The caller is responsible for dealing with access handlers.
1202	* @todo Add an informational return code for pages with access handlers?
1203	*
1204	* @remark Avoid calling this API from within critical sections (other than the
1205	* PGM one) because of the deadlock risk. External threads may need to
1206	* delegate jobs to the EMTs.
1207	* @thread Any thread.
1208	*/
1209	VMMDECL(int) PGMPhysGCPhys2CCPtr(PVM pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
1210	{
1211	int rc = pgmLock(pVM);
1212	AssertRCReturn(rc, rc);
1213
1214	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1215	/*
1216	* Find the page and make sure it's writable.
1217	*/
1218	PPGMPAGE pPage;
1219	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
1220	if (RT_SUCCESS(rc))
1221	{
1222	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
1223	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1224	if (RT_SUCCESS(rc))
1225	{
1226	ppv = pgmRZDynMapHCPageOff(pVM, PGM_PAGE_GET_HCPHYS(pPage) \| (GCPhys & PAGE_OFFSET_MASK) RTLOG_COMMA_SRC_POS); /* @todo add a read only flag? */
1227	# if 0
1228	pLock->pvMap = 0;
1229	pLock->pvPage = pPage;
1230	# else
1231	pLock->u32Dummy = UINT32_MAX;
1232	# endif
1233	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
1234	rc = VINF_SUCCESS;
1235	}
1236	}
1237
1238	#else /* IN_RING3 \|\| IN_RING0 */
1239	/*
1240	* Query the Physical TLB entry for the page (may fail).
1241	*/
1242	PPGMPAGEMAPTLBE pTlbe;
1243	rc = pgmPhysPageQueryTlbe(&pVM->pgm.s, GCPhys, &pTlbe);
1244	if (RT_SUCCESS(rc))
1245	{
1246	/*
1247	* If the page is shared, the zero page, or being write monitored
1248	* it must be converted to a page that's writable if possible.
1249	*/
1250	PPGMPAGE pPage = pTlbe->pPage;
1251	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
1252	{
1253	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1254	if (RT_SUCCESS(rc))
1255	{
1256	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
1257	rc = pgmPhysPageQueryTlbeWithPage(&pVM->pgm.s, pPage, GCPhys, &pTlbe);
1258	}
1259	}
1260	if (RT_SUCCESS(rc))
1261	{
1262	/*
1263	* Now, just perform the locking and calculate the return address.
1264	*/
1265	PPGMPAGEMAP pMap = pTlbe->pMap;
1266	if (pMap)
1267	pMap->cRefs++;
1268
1269	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
1270	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
1271	{
1272	if (cLocks == 0)
1273	pVM->pgm.s.cWriteLockedPages++;
1274	PGM_PAGE_INC_WRITE_LOCKS(pPage);
1275	}
1276	else if (cLocks != PGM_PAGE_GET_WRITE_LOCKS(pPage))
1277	{
1278	PGM_PAGE_INC_WRITE_LOCKS(pPage);
1279	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent write locked state!\n", GCPhys, pPage));
1280	if (pMap)
1281	pMap->cRefs++; /* Extra ref to prevent it from going away. */
1282	}
1283
1284	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1285	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_WRITE;
1286	pLock->pvMap = pMap;
1287	}
1288	}
1289
1290	#endif /* IN_RING3 \|\| IN_RING0 */
1291	pgmUnlock(pVM);
1292	return rc;
1293	}
1294
1295
1296	/**
1297	* Requests the mapping of a guest page into the current context.
1298	*
1299	* This API should only be used for very short term, as it will consume
1300	* scarse resources (R0 and GC) in the mapping cache. When you're done
1301	* with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
1302	*
1303	* @returns VBox status code.
1304	* @retval VINF_SUCCESS on success.
1305	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1306	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1307	*
1308	* @param pVM The VM handle.
1309	* @param GCPhys The guest physical address of the page that should be mapped.
1310	* @param ppv Where to store the address corresponding to GCPhys.
1311	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
1312	*
1313	* @remarks The caller is responsible for dealing with access handlers.
1314	* @todo Add an informational return code for pages with access handlers?
1315	*
1316	* @remark Avoid calling this API from within critical sections (other than
1317	* the PGM one) because of the deadlock risk.
1318	* @thread Any thread.
1319	*/
1320	VMMDECL(int) PGMPhysGCPhys2CCPtrReadOnly(PVM pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
1321	{
1322	int rc = pgmLock(pVM);
1323	AssertRCReturn(rc, rc);
1324
1325	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1326	/*
1327	* Find the page and make sure it's readable.
1328	*/
1329	PPGMPAGE pPage;
1330	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
1331	if (RT_SUCCESS(rc))
1332	{
1333	if (RT_UNLIKELY(PGM_PAGE_IS_MMIO(pPage)))
1334	rc = VERR_PGM_PHYS_PAGE_RESERVED;
1335	else
1336	{
1337	ppv = pgmRZDynMapHCPageOff(pVM, PGM_PAGE_GET_HCPHYS(pPage) \| (GCPhys & PAGE_OFFSET_MASK) RTLOG_COMMA_SRC_POS); /* @todo add a read only flag? */
1338	# if 0
1339	pLock->pvMap = 0;
1340	pLock->pvPage = pPage;
1341	# else
1342	pLock->u32Dummy = UINT32_MAX;
1343	# endif
1344	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
1345	rc = VINF_SUCCESS;
1346	}
1347	}
1348
1349	#else /* IN_RING3 \|\| IN_RING0 */
1350	/*
1351	* Query the Physical TLB entry for the page (may fail).
1352	*/
1353	PPGMPAGEMAPTLBE pTlbe;
1354	rc = pgmPhysPageQueryTlbe(&pVM->pgm.s, GCPhys, &pTlbe);
1355	if (RT_SUCCESS(rc))
1356	{
1357	/* MMIO pages doesn't have any readable backing. */
1358	PPGMPAGE pPage = pTlbe->pPage;
1359	if (RT_UNLIKELY(PGM_PAGE_IS_MMIO(pPage)))
1360	rc = VERR_PGM_PHYS_PAGE_RESERVED;
1361	else
1362	{
1363	/*
1364	* Now, just perform the locking and calculate the return address.
1365	*/
1366	PPGMPAGEMAP pMap = pTlbe->pMap;
1367	if (pMap)
1368	pMap->cRefs++;
1369
1370	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
1371	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
1372	{
1373	if (cLocks == 0)
1374	pVM->pgm.s.cReadLockedPages++;
1375	PGM_PAGE_INC_READ_LOCKS(pPage);
1376	}
1377	else if (cLocks != PGM_PAGE_GET_READ_LOCKS(pPage))
1378	{
1379	PGM_PAGE_INC_READ_LOCKS(pPage);
1380	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent readonly locked state!\n", GCPhys, pPage));
1381	if (pMap)
1382	pMap->cRefs++; /* Extra ref to prevent it from going away. */
1383	}
1384
1385	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1386	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_READ;
1387	pLock->pvMap = pMap;
1388	}
1389	}
1390
1391	#endif /* IN_RING3 \|\| IN_RING0 */
1392	pgmUnlock(pVM);
1393	return rc;
1394	}
1395
1396
1397	/**
1398	* Requests the mapping of a guest page given by virtual address into the current context.
1399	*
1400	* This API should only be used for very short term, as it will consume
1401	* scarse resources (R0 and GC) in the mapping cache. When you're done
1402	* with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
1403	*
1404	* This API will assume your intention is to write to the page, and will
1405	* therefore replace shared and zero pages. If you do not intend to modify
1406	* the page, use the PGMPhysGCPtr2CCPtrReadOnly() API.
1407	*
1408	* @returns VBox status code.
1409	* @retval VINF_SUCCESS on success.
1410	* @retval VERR_PAGE_TABLE_NOT_PRESENT if the page directory for the virtual address isn't present.
1411	* @retval VERR_PAGE_NOT_PRESENT if the page at the virtual address isn't present.
1412	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1413	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1414	*
1415	* @param pVCpu VMCPU handle.
1416	* @param GCPhys The guest physical address of the page that should be mapped.
1417	* @param ppv Where to store the address corresponding to GCPhys.
1418	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
1419	*
1420	* @remark Avoid calling this API from within critical sections (other than
1421	* the PGM one) because of the deadlock risk.
1422	* @thread EMT
1423	*/
1424	VMMDECL(int) PGMPhysGCPtr2CCPtr(PVMCPU pVCpu, RTGCPTR GCPtr, void **ppv, PPGMPAGEMAPLOCK pLock)
1425	{
1426	VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
1427	RTGCPHYS GCPhys;
1428	int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, &GCPhys);
1429	if (RT_SUCCESS(rc))
1430	rc = PGMPhysGCPhys2CCPtr(pVCpu->CTX_SUFF(pVM), GCPhys, ppv, pLock);
1431	return rc;
1432	}
1433
1434
1435	/**
1436	* Requests the mapping of a guest page given by virtual address into the current context.
1437	*
1438	* This API should only be used for very short term, as it will consume
1439	* scarse resources (R0 and GC) in the mapping cache. When you're done
1440	* with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
1441	*
1442	* @returns VBox status code.
1443	* @retval VINF_SUCCESS on success.
1444	* @retval VERR_PAGE_TABLE_NOT_PRESENT if the page directory for the virtual address isn't present.
1445	* @retval VERR_PAGE_NOT_PRESENT if the page at the virtual address isn't present.
1446	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1447	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1448	*
1449	* @param pVCpu VMCPU handle.
1450	* @param GCPhys The guest physical address of the page that should be mapped.
1451	* @param ppv Where to store the address corresponding to GCPhys.
1452	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
1453	*
1454	* @remark Avoid calling this API from within critical sections (other than
1455	* the PGM one) because of the deadlock risk.
1456	* @thread EMT
1457	*/
1458	VMMDECL(int) PGMPhysGCPtr2CCPtrReadOnly(PVMCPU pVCpu, RTGCPTR GCPtr, void const **ppv, PPGMPAGEMAPLOCK pLock)
1459	{
1460	VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
1461	RTGCPHYS GCPhys;
1462	int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, &GCPhys);
1463	if (RT_SUCCESS(rc))
1464	rc = PGMPhysGCPhys2CCPtrReadOnly(pVCpu->CTX_SUFF(pVM), GCPhys, ppv, pLock);
1465	return rc;
1466	}
1467
1468
1469	/**
1470	* Release the mapping of a guest page.
1471	*
1472	* This is the counter part of PGMPhysGCPhys2CCPtr, PGMPhysGCPhys2CCPtrReadOnly
1473	* PGMPhysGCPtr2CCPtr and PGMPhysGCPtr2CCPtrReadOnly.
1474	*
1475	* @param pVM The VM handle.
1476	* @param pLock The lock structure initialized by the mapping function.
1477	*/
1478	VMMDECL(void) PGMPhysReleasePageMappingLock(PVM pVM, PPGMPAGEMAPLOCK pLock)
1479	{
1480	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1481	/* currently nothing to do here. */
1482	Assert(pLock->u32Dummy == UINT32_MAX);
1483	pLock->u32Dummy = 0;
1484
1485	#else
1486	PPGMPAGEMAP pMap = (PPGMPAGEMAP)pLock->pvMap;
1487	PPGMPAGE pPage = (PPGMPAGE)(pLock->uPageAndType & ~PGMPAGEMAPLOCK_TYPE_MASK);
1488	bool fWriteLock = (pLock->uPageAndType & PGMPAGEMAPLOCK_TYPE_MASK) == PGMPAGEMAPLOCK_TYPE_WRITE;
1489
1490	pLock->uPageAndType = 0;
1491	pLock->pvMap = NULL;
1492
1493	pgmLock(pVM);
1494	if (fWriteLock)
1495	{
1496	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
1497	Assert(cLocks > 0);
1498	if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
1499	{
1500	if (cLocks == 1)
1501	{
1502	Assert(pVM->pgm.s.cWriteLockedPages > 0);
1503	pVM->pgm.s.cWriteLockedPages--;
1504	}
1505	PGM_PAGE_DEC_WRITE_LOCKS(pPage);
1506	}
1507
1508	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
1509	{
1510	PGM_PAGE_SET_WRITTEN_TO(pPage);
1511	PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ALLOCATED);
1512	Assert(pVM->pgm.s.cMonitoredPages > 0);
1513	pVM->pgm.s.cMonitoredPages--;
1514	pVM->pgm.s.cWrittenToPages++;
1515	}
1516	}
1517	else
1518	{
1519	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
1520	Assert(cLocks > 0);
1521	if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
1522	{
1523	if (cLocks == 1)
1524	{
1525	Assert(pVM->pgm.s.cReadLockedPages > 0);
1526	pVM->pgm.s.cReadLockedPages--;
1527	}
1528	PGM_PAGE_DEC_READ_LOCKS(pPage);
1529	}
1530	}
1531
1532	if (pMap)
1533	{
1534	Assert(pMap->cRefs >= 1);
1535	pMap->cRefs--;
1536	pMap->iAge = 0;
1537	}
1538	pgmUnlock(pVM);
1539	#endif /* IN_RING3 */
1540	}
1541
1542
1543	/**
1544	* Converts a GC physical address to a HC ring-3 pointer.
1545	*
1546	* @returns VINF_SUCCESS on success.
1547	* @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
1548	* page but has no physical backing.
1549	* @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
1550	* GC physical address.
1551	* @returns VERR_PGM_GCPHYS_RANGE_CROSSES_BOUNDARY if the range crosses
1552	* a dynamic ram chunk boundary
1553	*
1554	* @param pVM The VM handle.
1555	* @param GCPhys The GC physical address to convert.
1556	* @param cbRange Physical range
1557	* @param pR3Ptr Where to store the R3 pointer on success.
1558	*
1559	* @deprecated Avoid when possible!
1560	*/
1561	VMMDECL(int) PGMPhysGCPhys2R3Ptr(PVM pVM, RTGCPHYS GCPhys, RTUINT cbRange, PRTR3PTR pR3Ptr)
1562	{
1563	/** @todo this is kind of hacky and needs some more work. */
1564	#ifndef DEBUG_sandervl
1565	VM_ASSERT_EMT(pVM); /* no longer safe for use outside the EMT thread! */
1566	#endif
1567
1568	Log(("PGMPhysGCPhys2R3Ptr(,%RGp,%#x,): dont use this API!\n", GCPhys, cbRange)); /** @todo eliminate this API! */
1569	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1570	AssertFailedReturn(VERR_NOT_IMPLEMENTED);
1571	#else
1572	pgmLock(pVM);
1573
1574	PPGMRAMRANGE pRam;
1575	PPGMPAGE pPage;
1576	int rc = pgmPhysGetPageAndRangeEx(&pVM->pgm.s, GCPhys, &pPage, &pRam);
1577	if (RT_SUCCESS(rc))
1578	rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, (void **)pR3Ptr);
1579
1580	pgmUnlock(pVM);
1581	Assert(rc <= VINF_SUCCESS);
1582	return rc;
1583	#endif
1584	}
1585
1586
1587	#ifdef VBOX_STRICT
1588	/**
1589	* PGMPhysGCPhys2R3Ptr convenience for use with assertions.
1590	*
1591	* @returns The R3Ptr, NIL_RTR3PTR on failure.
1592	* @param pVM The VM handle.
1593	* @param GCPhys The GC Physical addresss.
1594	* @param cbRange Physical range.
1595	*
1596	* @deprecated Avoid when possible.
1597	*/
1598	VMMDECL(RTR3PTR) PGMPhysGCPhys2R3PtrAssert(PVM pVM, RTGCPHYS GCPhys, RTUINT cbRange)
1599	{
1600	RTR3PTR R3Ptr;
1601	int rc = PGMPhysGCPhys2R3Ptr(pVM, GCPhys, cbRange, &R3Ptr);
1602	if (RT_SUCCESS(rc))
1603	return R3Ptr;
1604	return NIL_RTR3PTR;
1605	}
1606	#endif /* VBOX_STRICT */
1607
1608
1609	/**
1610	* Converts a guest pointer to a GC physical address.
1611	*
1612	* This uses the current CR3/CR0/CR4 of the guest.
1613	*
1614	* @returns VBox status code.
1615	* @param pVCpu The VMCPU Handle
1616	* @param GCPtr The guest pointer to convert.
1617	* @param pGCPhys Where to store the GC physical address.
1618	*/
1619	VMMDECL(int) PGMPhysGCPtr2GCPhys(PVMCPU pVCpu, RTGCPTR GCPtr, PRTGCPHYS pGCPhys)
1620	{
1621	int rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtr, NULL, pGCPhys);
1622	if (pGCPhys && RT_SUCCESS(rc))
1623	*pGCPhys \|= (RTGCUINTPTR)GCPtr & PAGE_OFFSET_MASK;
1624	return rc;
1625	}
1626
1627
1628	/**
1629	* Converts a guest pointer to a HC physical address.
1630	*
1631	* This uses the current CR3/CR0/CR4 of the guest.
1632	*
1633	* @returns VBox status code.
1634	* @param pVCpu The VMCPU Handle
1635	* @param GCPtr The guest pointer to convert.
1636	* @param pHCPhys Where to store the HC physical address.
1637	*/
1638	VMMDECL(int) PGMPhysGCPtr2HCPhys(PVMCPU pVCpu, RTGCPTR GCPtr, PRTHCPHYS pHCPhys)
1639	{
1640	PVM pVM = pVCpu->CTX_SUFF(pVM);
1641	RTGCPHYS GCPhys;
1642	int rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtr, NULL, &GCPhys);
1643	if (RT_SUCCESS(rc))
1644	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhys \| ((RTGCUINTPTR)GCPtr & PAGE_OFFSET_MASK), pHCPhys);
1645	return rc;
1646	}
1647
1648
1649
1650	#undef LOG_GROUP
1651	#define LOG_GROUP LOG_GROUP_PGM_PHYS_ACCESS
1652
1653
1654	#ifdef IN_RING3
1655	/**
1656	* Cache PGMPhys memory access
1657	*
1658	* @param pVM VM Handle.
1659	* @param pCache Cache structure pointer
1660	* @param GCPhys GC physical address
1661	* @param pbHC HC pointer corresponding to physical page
1662	*
1663	* @thread EMT.
1664	*/
1665	static void pgmPhysCacheAdd(PVM pVM, PGMPHYSCACHE pCache, RTGCPHYS GCPhys, uint8_t pbR3)
1666	{
1667	uint32_t iCacheIndex;
1668
1669	Assert(VM_IS_EMT(pVM));
1670
1671	GCPhys = PHYS_PAGE_ADDRESS(GCPhys);
1672	pbR3 = (uint8_t *)PAGE_ADDRESS(pbR3);
1673
1674	iCacheIndex = ((GCPhys >> PAGE_SHIFT) & PGM_MAX_PHYSCACHE_ENTRIES_MASK);
1675
1676	ASMBitSet(&pCache->aEntries, iCacheIndex);
1677
1678	pCache->Entry[iCacheIndex].GCPhys = GCPhys;
1679	pCache->Entry[iCacheIndex].pbR3 = pbR3;
1680	}
1681	#endif /* IN_RING3 */
1682
1683
1684	/**
1685	* Deals with reading from a page with one or more ALL access handlers.
1686	*
1687	* @returns VBox status code. Can be ignored in ring-3.
1688	* @retval VINF_SUCCESS.
1689	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in R0 and GC, NEVER in R3.
1690	*
1691	* @param pVM The VM handle.
1692	* @param pPage The page descriptor.
1693	* @param GCPhys The physical address to start reading at.
1694	* @param pvBuf Where to put the bits we read.
1695	* @param cb How much to read - less or equal to a page.
1696	*/
1697	static int pgmPhysReadHandler(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void *pvBuf, size_t cb)
1698	{
1699	/*
1700	* The most frequent access here is MMIO and shadowed ROM.
1701	* The current code ASSUMES all these access handlers covers full pages!
1702	*/
1703
1704	/*
1705	* Whatever we do we need the source page, map it first.
1706	*/
1707	const void *pvSrc = NULL;
1708	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, GCPhys, &pvSrc);
1709	if (RT_FAILURE(rc))
1710	{
1711	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
1712	GCPhys, pPage, rc));
1713	memset(pvBuf, 0xff, cb);
1714	return VINF_SUCCESS;
1715	}
1716	rc = VINF_PGM_HANDLER_DO_DEFAULT;
1717
1718	/*
1719	* Deal with any physical handlers.
1720	*/
1721	PPGMPHYSHANDLER pPhys = NULL;
1722	if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) == PGM_PAGE_HNDL_PHYS_STATE_ALL)
1723	{
1724	#ifdef IN_RING3
1725	pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
1726	AssertReleaseMsg(pPhys, ("GCPhys=%RGp cb=%#x\n", GCPhys, cb));
1727	Assert(GCPhys >= pPhys->Core.Key && GCPhys <= pPhys->Core.KeyLast);
1728	Assert((pPhys->Core.Key & PAGE_OFFSET_MASK) == 0);
1729	Assert((pPhys->Core.KeyLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
1730	Assert(pPhys->CTX_SUFF(pfnHandler));
1731
1732	PFNPGMR3PHYSHANDLER pfnHandler = pPhys->CTX_SUFF(pfnHandler);
1733	void *pvUser = pPhys->CTX_SUFF(pvUser);
1734
1735	Log5(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cb, pPage, R3STRING(pPhys->pszDesc) ));
1736	STAM_PROFILE_START(&pPhys->Stat, h);
1737	Assert(PGMIsLockOwner(pVM));
1738	/* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
1739	pgmUnlock(pVM);
1740	rc = pfnHandler(pVM, GCPhys, (void *)pvSrc, pvBuf, cb, PGMACCESSTYPE_READ, pvUser);
1741	pgmLock(pVM);
1742	# ifdef VBOX_WITH_STATISTICS
1743	pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
1744	if (pPhys)
1745	STAM_PROFILE_STOP(&pPhys->Stat, h);
1746	# else
1747	pPhys = NULL; /* might not be valid anymore. */
1748	# endif
1749	AssertLogRelMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp\n", rc, GCPhys));
1750	#else
1751	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
1752	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cb=%#x\n", GCPhys, cb));
1753	return VERR_PGM_PHYS_WR_HIT_HANDLER;
1754	#endif
1755	}
1756
1757	/*
1758	* Deal with any virtual handlers.
1759	*/
1760	if (PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) == PGM_PAGE_HNDL_VIRT_STATE_ALL)
1761	{
1762	unsigned iPage;
1763	PPGMVIRTHANDLER pVirt;
1764
1765	int rc2 = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys, &pVirt, &iPage);
1766	AssertReleaseMsg(RT_SUCCESS(rc2), ("GCPhys=%RGp cb=%#x rc2=%Rrc\n", GCPhys, cb, rc2));
1767	Assert((pVirt->Core.Key & PAGE_OFFSET_MASK) == 0);
1768	Assert((pVirt->Core.KeyLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
1769	Assert(GCPhys >= pVirt->aPhysToVirt[iPage].Core.Key && GCPhys <= pVirt->aPhysToVirt[iPage].Core.KeyLast);
1770
1771	#ifdef IN_RING3
1772	if (pVirt->pfnHandlerR3)
1773	{
1774	if (!pPhys)
1775	Log5(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] virt %s\n", GCPhys, cb, pPage, R3STRING(pVirt->pszDesc) ));
1776	else
1777	Log(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] phys/virt %s/%s\n", GCPhys, cb, pPage, R3STRING(pVirt->pszDesc), R3STRING(pPhys->pszDesc) ));
1778	RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pVirt->Core.Key & PAGE_BASE_GC_MASK)
1779	+ (iPage << PAGE_SHIFT)
1780	+ (GCPhys & PAGE_OFFSET_MASK);
1781
1782	STAM_PROFILE_START(&pVirt->Stat, h);
1783	rc2 = pVirt->CTX_SUFF(pfnHandler)(pVM, GCPtr, (void )pvSrc, pvBuf, cb, PGMACCESSTYPE_READ, /pVirt->CTX_SUFF(pvUser)*/ NULL);
1784	STAM_PROFILE_STOP(&pVirt->Stat, h);
1785	if (rc2 == VINF_SUCCESS)
1786	rc = VINF_SUCCESS;
1787	AssertLogRelMsg(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc2, GCPhys, pPage, pVirt->pszDesc));
1788	}
1789	else
1790	Log5(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] virt %s [no handler]\n", GCPhys, cb, pPage, R3STRING(pVirt->pszDesc) ));
1791	#else
1792	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
1793	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cb=%#x\n", GCPhys, cb));
1794	return VERR_PGM_PHYS_WR_HIT_HANDLER;
1795	#endif
1796	}
1797
1798	/*
1799	* Take the default action.
1800	*/
1801	if (rc == VINF_PGM_HANDLER_DO_DEFAULT)
1802	memcpy(pvBuf, pvSrc, cb);
1803	return rc;
1804	}
1805
1806
1807	/**
1808	* Read physical memory.
1809	*
1810	* This API respects access handlers and MMIO. Use PGMPhysSimpleReadGCPhys() if you
1811	* want to ignore those.
1812	*
1813	* @returns VBox status code. Can be ignored in ring-3.
1814	* @retval VINF_SUCCESS.
1815	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in R0 and GC, NEVER in R3.
1816	*
1817	* @param pVM VM Handle.
1818	* @param GCPhys Physical address start reading from.
1819	* @param pvBuf Where to put the read bits.
1820	* @param cbRead How many bytes to read.
1821	*/
1822	VMMDECL(int) PGMPhysRead(PVM pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead)
1823	{
1824	AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
1825	LogFlow(("PGMPhysRead: %RGp %d\n", GCPhys, cbRead));
1826
1827	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysRead));
1828	STAM_COUNTER_ADD(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysReadBytes), cbRead);
1829
1830	pgmLock(pVM);
1831
1832	/*
1833	* Copy loop on ram ranges.
1834	*/
1835	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
1836	for (;;)
1837	{
1838	/* Find range. */
1839	while (pRam && GCPhys > pRam->GCPhysLast)
1840	pRam = pRam->CTX_SUFF(pNext);
1841	/* Inside range or not? */
1842	if (pRam && GCPhys >= pRam->GCPhys)
1843	{
1844	/*
1845	* Must work our way thru this page by page.
1846	*/
1847	RTGCPHYS off = GCPhys - pRam->GCPhys;
1848	while (off < pRam->cb)
1849	{
1850	unsigned iPage = off >> PAGE_SHIFT;
1851	PPGMPAGE pPage = &pRam->aPages[iPage];
1852	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
1853	if (cb > cbRead)
1854	cb = cbRead;
1855
1856	/*
1857	* Any ALL access handlers?
1858	*/
1859	if (RT_UNLIKELY(PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)))
1860	{
1861	int rc = pgmPhysReadHandler(pVM, pPage, pRam->GCPhys + off, pvBuf, cb);
1862	if (RT_FAILURE(rc))
1863	{
1864	pgmUnlock(pVM);
1865	return rc;
1866	}
1867	}
1868	else
1869	{
1870	/*
1871	* Get the pointer to the page.
1872	*/
1873	const void *pvSrc;
1874	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc);
1875	if (RT_SUCCESS(rc))
1876	memcpy(pvBuf, pvSrc, cb);
1877	else
1878	{
1879	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
1880	pRam->GCPhys + off, pPage, rc));
1881	memset(pvBuf, 0xff, cb);
1882	}
1883	}
1884
1885	/* next page */
1886	if (cb >= cbRead)
1887	{
1888	pgmUnlock(pVM);
1889	return VINF_SUCCESS;
1890	}
1891	cbRead -= cb;
1892	off += cb;
1893	pvBuf = (char *)pvBuf + cb;
1894	} /* walk pages in ram range. */
1895
1896	GCPhys = pRam->GCPhysLast + 1;
1897	}
1898	else
1899	{
1900	LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
1901
1902	/*
1903	* Unassigned address space.
1904	*/
1905	if (!pRam)
1906	break;
1907	size_t cb = pRam->GCPhys - GCPhys;
1908	if (cb >= cbRead)
1909	{
1910	memset(pvBuf, 0xff, cbRead);
1911	break;
1912	}
1913	memset(pvBuf, 0xff, cb);
1914
1915	cbRead -= cb;
1916	pvBuf = (char *)pvBuf + cb;
1917	GCPhys += cb;
1918	}
1919	} /* Ram range walk */
1920
1921	pgmUnlock(pVM);
1922	return VINF_SUCCESS;
1923	}
1924
1925
1926	/**
1927	* Deals with writing to a page with one or more WRITE or ALL access handlers.
1928	*
1929	* @returns VBox status code. Can be ignored in ring-3.
1930	* @retval VINF_SUCCESS.
1931	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in R0 and GC, NEVER in R3.
1932	*
1933	* @param pVM The VM handle.
1934	* @param pPage The page descriptor.
1935	* @param GCPhys The physical address to start writing at.
1936	* @param pvBuf What to write.
1937	* @param cbWrite How much to write - less or equal to a page.
1938	*/
1939	static int pgmPhysWriteHandler(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const *pvBuf, size_t cbWrite)
1940	{
1941	void *pvDst = NULL;
1942	int rc;
1943
1944	/*
1945	* Give priority to physical handlers (like #PF does).
1946	*
1947	* Hope for a lonely physical handler first that covers the whole
1948	* write area. This should be a pretty frequent case with MMIO and
1949	* the heavy usage of full page handlers in the page pool.
1950	*/
1951	if ( !PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(pPage)
1952	\|\| PGM_PAGE_IS_MMIO(pPage) /* screw virtual handlers on MMIO pages */)
1953	{
1954	PPGMPHYSHANDLER pCur = pgmHandlerPhysicalLookup(pVM, GCPhys);
1955	if (pCur)
1956	{
1957	Assert(GCPhys >= pCur->Core.Key && GCPhys <= pCur->Core.KeyLast);
1958	Assert(pCur->CTX_SUFF(pfnHandler));
1959
1960	size_t cbRange = pCur->Core.KeyLast - GCPhys + 1;
1961	if (cbRange > cbWrite)
1962	cbRange = cbWrite;
1963
1964	#ifndef IN_RING3
1965	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
1966	NOREF(cbRange);
1967	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cbRange=%#x\n", GCPhys, cbRange));
1968	return VERR_PGM_PHYS_WR_HIT_HANDLER;
1969
1970	#else /* IN_RING3 */
1971	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cbRange, pPage, R3STRING(pCur->pszDesc) ));
1972	if (!PGM_PAGE_IS_MMIO(pPage))
1973	rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst);
1974	else
1975	rc = VINF_SUCCESS;
1976	if (RT_SUCCESS(rc))
1977	{
1978	PFNPGMR3PHYSHANDLER pfnHandler = pCur->CTX_SUFF(pfnHandler);
1979	void *pvUser = pCur->CTX_SUFF(pvUser);
1980
1981	STAM_PROFILE_START(&pCur->Stat, h);
1982	Assert(PGMIsLockOwner(pVM));
1983	/* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
1984	pgmUnlock(pVM);
1985	rc = pfnHandler(pVM, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, pvUser);
1986	pgmLock(pVM);
1987	# ifdef VBOX_WITH_STATISTICS
1988	pCur = pgmHandlerPhysicalLookup(pVM, GCPhys);
1989	if (pCur)
1990	STAM_PROFILE_STOP(&pCur->Stat, h);
1991	# else
1992	pCur = NULL; /* might not be valid anymore. */
1993	# endif
1994	if (rc == VINF_PGM_HANDLER_DO_DEFAULT)
1995	memcpy(pvDst, pvBuf, cbRange);
1996	else
1997	AssertLogRelMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, (pCur) ? pCur->pszDesc : ""));
1998	}
1999	else
2000	AssertLogRelMsgFailedReturn(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
2001	GCPhys, pPage, rc), rc);
2002	if (RT_LIKELY(cbRange == cbWrite))
2003	return VINF_SUCCESS;
2004
2005	/* more fun to be had below */
2006	cbWrite -= cbRange;
2007	GCPhys += cbRange;
2008	pvBuf = (uint8_t *)pvBuf + cbRange;
2009	pvDst = (uint8_t *)pvDst + cbRange;
2010	#endif /* IN_RING3 */
2011	}
2012	/* else: the handler is somewhere else in the page, deal with it below. */
2013	Assert(!PGM_PAGE_IS_MMIO(pPage)); /* MMIO handlers are all PAGE_SIZEed! */
2014	}
2015	/*
2016	* A virtual handler without any interfering physical handlers.
2017	* Hopefully it'll conver the whole write.
2018	*/
2019	else if (!PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage))
2020	{
2021	unsigned iPage;
2022	PPGMVIRTHANDLER pCur;
2023	rc = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys, &pCur, &iPage);
2024	if (RT_SUCCESS(rc))
2025	{
2026	size_t cbRange = (PAGE_OFFSET_MASK & pCur->Core.KeyLast) - (PAGE_OFFSET_MASK & GCPhys) + 1;
2027	if (cbRange > cbWrite)
2028	cbRange = cbWrite;
2029
2030	#ifndef IN_RING3
2031	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
2032	NOREF(cbRange);
2033	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cbRange=%#x\n", GCPhys, cbRange));
2034	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2035
2036	#else /* IN_RING3 */
2037
2038	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] virt %s\n", GCPhys, cbRange, pPage, R3STRING(pCur->pszDesc) ));
2039	rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst);
2040	if (RT_SUCCESS(rc))
2041	{
2042	rc = VINF_PGM_HANDLER_DO_DEFAULT;
2043	if (pCur->pfnHandlerR3)
2044	{
2045	RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pCur->Core.Key & PAGE_BASE_GC_MASK)
2046	+ (iPage << PAGE_SHIFT)
2047	+ (GCPhys & PAGE_OFFSET_MASK);
2048
2049	STAM_PROFILE_START(&pCur->Stat, h);
2050	rc = pCur->CTX_SUFF(pfnHandler)(pVM, GCPtr, pvDst, (void )pvBuf, cbRange, PGMACCESSTYPE_WRITE, /pCur->CTX_SUFF(pvUser)*/ NULL);
2051	STAM_PROFILE_STOP(&pCur->Stat, h);
2052	}
2053	if (rc == VINF_PGM_HANDLER_DO_DEFAULT)
2054	memcpy(pvDst, pvBuf, cbRange);
2055	else
2056	AssertLogRelMsg(rc == VINF_SUCCESS, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, pCur->pszDesc));
2057	}
2058	else
2059	AssertLogRelMsgFailedReturn(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
2060	GCPhys, pPage, rc), rc);
2061	if (RT_LIKELY(cbRange == cbWrite))
2062	return VINF_SUCCESS;
2063
2064	/* more fun to be had below */
2065	cbWrite -= cbRange;
2066	GCPhys += cbRange;
2067	pvBuf = (uint8_t *)pvBuf + cbRange;
2068	pvDst = (uint8_t *)pvDst + cbRange;
2069	#endif
2070	}
2071	/* else: the handler is somewhere else in the page, deal with it below. */
2072	}
2073
2074	/*
2075	* Deal with all the odd ends.
2076	*/
2077
2078	/* We need a writable destination page. */
2079	if (!pvDst)
2080	{
2081	rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst);
2082	AssertLogRelMsgReturn(RT_SUCCESS(rc),
2083	("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
2084	GCPhys, pPage, rc), rc);
2085	}
2086
2087	/* The loop state (big + ugly). */
2088	unsigned iVirtPage = 0;
2089	PPGMVIRTHANDLER pVirt = NULL;
2090	uint32_t offVirt = PAGE_SIZE;
2091	uint32_t offVirtLast = PAGE_SIZE;
2092	bool fMoreVirt = PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(pPage);
2093
2094	PPGMPHYSHANDLER pPhys = NULL;
2095	uint32_t offPhys = PAGE_SIZE;
2096	uint32_t offPhysLast = PAGE_SIZE;
2097	bool fMorePhys = PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage);
2098
2099	/* The loop. */
2100	for (;;)
2101	{
2102	/*
2103	* Find the closest handler at or above GCPhys.
2104	*/
2105	if (fMoreVirt && !pVirt)
2106	{
2107	rc = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys, &pVirt, &iVirtPage);
2108	if (RT_SUCCESS(rc))
2109	{
2110	offVirt = 0;
2111	offVirtLast = (pVirt->aPhysToVirt[iVirtPage].Core.KeyLast & PAGE_OFFSET_MASK) - (GCPhys & PAGE_OFFSET_MASK);
2112	}
2113	else
2114	{
2115	PPGMPHYS2VIRTHANDLER pVirtPhys;
2116	pVirtPhys = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysGetBestFit(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysToVirtHandlers,
2117	GCPhys, true /* fAbove */);
2118	if ( pVirtPhys
2119	&& (pVirtPhys->Core.Key >> PAGE_SHIFT) == (GCPhys >> PAGE_SHIFT))
2120	{
2121	/* ASSUME that pVirtPhys only covers one page. */
2122	Assert((pVirtPhys->Core.Key >> PAGE_SHIFT) == (pVirtPhys->Core.KeyLast >> PAGE_SHIFT));
2123	Assert(pVirtPhys->Core.Key > GCPhys);
2124
2125	pVirt = (PPGMVIRTHANDLER)((uintptr_t)pVirtPhys + pVirtPhys->offVirtHandler);
2126	iVirtPage = pVirtPhys - &pVirt->aPhysToVirt[0]; Assert(iVirtPage == 0);
2127	offVirt = (pVirtPhys->Core.Key & PAGE_OFFSET_MASK) - (GCPhys & PAGE_OFFSET_MASK);
2128	offVirtLast = (pVirtPhys->Core.KeyLast & PAGE_OFFSET_MASK) - (GCPhys & PAGE_OFFSET_MASK);
2129	}
2130	else
2131	{
2132	pVirt = NULL;
2133	fMoreVirt = false;
2134	offVirt = offVirtLast = PAGE_SIZE;
2135	}
2136	}
2137	}
2138
2139	if (fMorePhys && !pPhys)
2140	{
2141	pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
2142	if (pPhys)
2143	{
2144	offPhys = 0;
2145	offPhysLast = pPhys->Core.KeyLast - GCPhys; /* ASSUMES < 4GB handlers... */
2146	}
2147	else
2148	{
2149	pPhys = (PPGMPHYSHANDLER)RTAvlroGCPhysGetBestFit(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers,
2150	GCPhys, true /* fAbove */);
2151	if ( pPhys
2152	&& pPhys->Core.Key <= GCPhys + (cbWrite - 1))
2153	{
2154	offPhys = pPhys->Core.Key - GCPhys;
2155	offPhysLast = pPhys->Core.KeyLast - GCPhys; /* ASSUMES < 4GB handlers... */
2156	}
2157	else
2158	{
2159	pPhys = NULL;
2160	fMorePhys = false;
2161	offPhys = offPhysLast = PAGE_SIZE;
2162	}
2163	}
2164	}
2165
2166	/*
2167	* Handle access to space without handlers (that's easy).
2168	*/
2169	rc = VINF_PGM_HANDLER_DO_DEFAULT;
2170	uint32_t cbRange = (uint32_t)cbWrite;
2171	if (offPhys && offVirt)
2172	{
2173	if (cbRange > offPhys)
2174	cbRange = offPhys;
2175	if (cbRange > offVirt)
2176	cbRange = offVirt;
2177	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] miss\n", GCPhys, cbRange, pPage));
2178	}
2179	/*
2180	* Physical handler.
2181	*/
2182	else if (!offPhys && offVirt)
2183	{
2184	if (cbRange > offPhysLast + 1)
2185	cbRange = offPhysLast + 1;
2186	if (cbRange > offVirt)
2187	cbRange = offVirt;
2188	#ifdef IN_RING3
2189	PFNPGMR3PHYSHANDLER pfnHandler = pPhys->CTX_SUFF(pfnHandler);
2190	void *pvUser = pPhys->CTX_SUFF(pvUser);
2191
2192	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cbRange, pPage, R3STRING(pPhys->pszDesc) ));
2193	STAM_PROFILE_START(&pPhys->Stat, h);
2194	Assert(PGMIsLockOwner(pVM));
2195	/* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
2196	pgmUnlock(pVM);
2197	rc = pfnHandler(pVM, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, pvUser);
2198	pgmLock(pVM);
2199	# ifdef VBOX_WITH_STATISTICS
2200	pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
2201	if (pPhys)
2202	STAM_PROFILE_STOP(&pPhys->Stat, h);
2203	# else
2204	pPhys = NULL; /* might not be valid anymore. */
2205	# endif
2206	AssertLogRelMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, (pPhys) ? pPhys->pszDesc : ""));
2207	#else
2208	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
2209	NOREF(cbRange);
2210	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cbRange=%#x\n", GCPhys, cbRange));
2211	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2212	#endif
2213	}
2214	/*
2215	* Virtual handler.
2216	*/
2217	else if (offPhys && !offVirt)
2218	{
2219	if (cbRange > offVirtLast + 1)
2220	cbRange = offVirtLast + 1;
2221	if (cbRange > offPhys)
2222	cbRange = offPhys;
2223	#ifdef IN_RING3
2224	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cbRange, pPage, R3STRING(pVirt->pszDesc) ));
2225	if (pVirt->pfnHandlerR3)
2226	{
2227	RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pVirt->Core.Key & PAGE_BASE_GC_MASK)
2228	+ (iVirtPage << PAGE_SHIFT)
2229	+ (GCPhys & PAGE_OFFSET_MASK);
2230	STAM_PROFILE_START(&pVirt->Stat, h);
2231	rc = pVirt->CTX_SUFF(pfnHandler)(pVM, GCPtr, pvDst, (void )pvBuf, cbRange, PGMACCESSTYPE_WRITE, /pCur->CTX_SUFF(pvUser)*/ NULL);
2232	STAM_PROFILE_STOP(&pVirt->Stat, h);
2233	AssertLogRelMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, pVirt->pszDesc));
2234	}
2235	pVirt = NULL;
2236	#else
2237	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
2238	NOREF(cbRange);
2239	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cbRange=%#x\n", GCPhys, cbRange));
2240	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2241	#endif
2242	}
2243	/*
2244	* Both... give the physical one priority.
2245	*/
2246	else
2247	{
2248	Assert(!offPhys && !offVirt);
2249	if (cbRange > offVirtLast + 1)
2250	cbRange = offVirtLast + 1;
2251	if (cbRange > offPhysLast + 1)
2252	cbRange = offPhysLast + 1;
2253
2254	#ifdef IN_RING3
2255	if (pVirt->pfnHandlerR3)
2256	Log(("pgmPhysWriteHandler: overlapping phys and virt handlers at %RGp %R[pgmpage]; cbRange=%#x\n", GCPhys, pPage, cbRange));
2257	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys/virt %s/%s\n", GCPhys, cbRange, pPage, R3STRING(pPhys->pszDesc), R3STRING(pVirt->pszDesc) ));
2258
2259	PFNPGMR3PHYSHANDLER pfnHandler = pPhys->CTX_SUFF(pfnHandler);
2260	void *pvUser = pPhys->CTX_SUFF(pvUser);
2261
2262	STAM_PROFILE_START(&pPhys->Stat, h);
2263	Assert(PGMIsLockOwner(pVM));
2264	/* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
2265	pgmUnlock(pVM);
2266	rc = pfnHandler(pVM, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, pvUser);
2267	pgmLock(pVM);
2268	# ifdef VBOX_WITH_STATISTICS
2269	pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
2270	if (pPhys)
2271	STAM_PROFILE_STOP(&pPhys->Stat, h);
2272	# else
2273	pPhys = NULL; /* might not be valid anymore. */
2274	# endif
2275	AssertLogRelMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, (pPhys) ? pPhys->pszDesc : ""));
2276	if (pVirt->pfnHandlerR3)
2277	{
2278
2279	RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pVirt->Core.Key & PAGE_BASE_GC_MASK)
2280	+ (iVirtPage << PAGE_SHIFT)
2281	+ (GCPhys & PAGE_OFFSET_MASK);
2282	STAM_PROFILE_START(&pVirt->Stat, h2);
2283	int rc2 = pVirt->CTX_SUFF(pfnHandler)(pVM, GCPtr, pvDst, (void )pvBuf, cbRange, PGMACCESSTYPE_WRITE, /pCur->CTX_SUFF(pvUser)*/ NULL);
2284	STAM_PROFILE_STOP(&pVirt->Stat, h2);
2285	if (rc2 == VINF_SUCCESS && rc == VINF_PGM_HANDLER_DO_DEFAULT)
2286	rc = VINF_SUCCESS;
2287	else
2288	AssertLogRelMsg(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, pVirt->pszDesc));
2289	}
2290	pPhys = NULL;
2291	pVirt = NULL;
2292	#else
2293	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
2294	NOREF(cbRange);
2295	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cbRange=%#x\n", GCPhys, cbRange));
2296	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2297	#endif
2298	}
2299	if (rc == VINF_PGM_HANDLER_DO_DEFAULT)
2300	memcpy(pvDst, pvBuf, cbRange);
2301
2302	/*
2303	* Advance if we've got more stuff to do.
2304	*/
2305	if (cbRange >= cbWrite)
2306	return VINF_SUCCESS;
2307
2308	cbWrite -= cbRange;
2309	GCPhys += cbRange;
2310	pvBuf = (uint8_t *)pvBuf + cbRange;
2311	pvDst = (uint8_t *)pvDst + cbRange;
2312
2313	offPhys -= cbRange;
2314	offPhysLast -= cbRange;
2315	offVirt -= cbRange;
2316	offVirtLast -= cbRange;
2317	}
2318	}
2319
2320
2321	/**
2322	* Write to physical memory.
2323	*
2324	* This API respects access handlers and MMIO. Use PGMPhysSimpleWriteGCPhys() if you
2325	* want to ignore those.
2326	*
2327	* @returns VBox status code. Can be ignored in ring-3.
2328	* @retval VINF_SUCCESS.
2329	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in R0 and GC, NEVER in R3.
2330	*
2331	* @param pVM VM Handle.
2332	* @param GCPhys Physical address to write to.
2333	* @param pvBuf What to write.
2334	* @param cbWrite How many bytes to write.
2335	*/
2336	VMMDECL(int) PGMPhysWrite(PVM pVM, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite)
2337	{
2338	AssertMsg(!pVM->pgm.s.fNoMorePhysWrites, ("Calling PGMPhysWrite after pgmR3Save()!\n"));
2339	AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
2340	LogFlow(("PGMPhysWrite: %RGp %d\n", GCPhys, cbWrite));
2341
2342	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysWrite));
2343	STAM_COUNTER_ADD(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysWriteBytes), cbWrite);
2344
2345	pgmLock(pVM);
2346
2347	/*
2348	* Copy loop on ram ranges.
2349	*/
2350	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
2351	for (;;)
2352	{
2353	/* Find range. */
2354	while (pRam && GCPhys > pRam->GCPhysLast)
2355	pRam = pRam->CTX_SUFF(pNext);
2356	/* Inside range or not? */
2357	if (pRam && GCPhys >= pRam->GCPhys)
2358	{
2359	/*
2360	* Must work our way thru this page by page.
2361	*/
2362	RTGCPTR off = GCPhys - pRam->GCPhys;
2363	while (off < pRam->cb)
2364	{
2365	RTGCPTR iPage = off >> PAGE_SHIFT;
2366	PPGMPAGE pPage = &pRam->aPages[iPage];
2367	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
2368	if (cb > cbWrite)
2369	cb = cbWrite;
2370
2371	/*
2372	* Any active WRITE or ALL access handlers?
2373	*/
2374	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2375	{
2376	int rc = pgmPhysWriteHandler(pVM, pPage, pRam->GCPhys + off, pvBuf, cb);
2377	if (RT_FAILURE(rc))
2378	{
2379	pgmUnlock(pVM);
2380	return rc;
2381	}
2382	}
2383	else
2384	{
2385	/*
2386	* Get the pointer to the page.
2387	*/
2388	void *pvDst;
2389	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst);
2390	if (RT_SUCCESS(rc))
2391	{
2392	Assert(!PGM_PAGE_IS_BALLOONED(pPage));
2393	memcpy(pvDst, pvBuf, cb);
2394	}
2395	else
2396	/* Ignore writes to ballooned pages. */
2397	if (!PGM_PAGE_IS_BALLOONED(pPage))
2398	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
2399	pRam->GCPhys + off, pPage, rc));
2400	}
2401
2402	/* next page */
2403	if (cb >= cbWrite)
2404	{
2405	pgmUnlock(pVM);
2406	return VINF_SUCCESS;
2407	}
2408
2409	cbWrite -= cb;
2410	off += cb;
2411	pvBuf = (const char *)pvBuf + cb;
2412	} /* walk pages in ram range */
2413
2414	GCPhys = pRam->GCPhysLast + 1;
2415	}
2416	else
2417	{
2418	/*
2419	* Unassigned address space, skip it.
2420	*/
2421	if (!pRam)
2422	break;
2423	size_t cb = pRam->GCPhys - GCPhys;
2424	if (cb >= cbWrite)
2425	break;
2426	cbWrite -= cb;
2427	pvBuf = (const char *)pvBuf + cb;
2428	GCPhys += cb;
2429	}
2430	} /* Ram range walk */
2431
2432	pgmUnlock(pVM);
2433	return VINF_SUCCESS;
2434	}
2435
2436
2437	/**
2438	* Read from guest physical memory by GC physical address, bypassing
2439	* MMIO and access handlers.
2440	*
2441	* @returns VBox status.
2442	* @param pVM VM handle.
2443	* @param pvDst The destination address.
2444	* @param GCPhysSrc The source address (GC physical address).
2445	* @param cb The number of bytes to read.
2446	*/
2447	VMMDECL(int) PGMPhysSimpleReadGCPhys(PVM pVM, void *pvDst, RTGCPHYS GCPhysSrc, size_t cb)
2448	{
2449	/*
2450	* Treat the first page as a special case.
2451	*/
2452	if (!cb)
2453	return VINF_SUCCESS;
2454
2455	/* map the 1st page */
2456	void const *pvSrc;
2457	PGMPAGEMAPLOCK Lock;
2458	int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock);
2459	if (RT_FAILURE(rc))
2460	return rc;
2461
2462	/* optimize for the case where access is completely within the first page. */
2463	size_t cbPage = PAGE_SIZE - (GCPhysSrc & PAGE_OFFSET_MASK);
2464	if (RT_LIKELY(cb <= cbPage))
2465	{
2466	memcpy(pvDst, pvSrc, cb);
2467	PGMPhysReleasePageMappingLock(pVM, &Lock);
2468	return VINF_SUCCESS;
2469	}
2470
2471	/* copy to the end of the page. */
2472	memcpy(pvDst, pvSrc, cbPage);
2473	PGMPhysReleasePageMappingLock(pVM, &Lock);
2474	GCPhysSrc += cbPage;
2475	pvDst = (uint8_t *)pvDst + cbPage;
2476	cb -= cbPage;
2477
2478	/*
2479	* Page by page.
2480	*/
2481	for (;;)
2482	{
2483	/* map the page */
2484	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock);
2485	if (RT_FAILURE(rc))
2486	return rc;
2487
2488	/* last page? */
2489	if (cb <= PAGE_SIZE)
2490	{
2491	memcpy(pvDst, pvSrc, cb);
2492	PGMPhysReleasePageMappingLock(pVM, &Lock);
2493	return VINF_SUCCESS;
2494	}
2495
2496	/* copy the entire page and advance */
2497	memcpy(pvDst, pvSrc, PAGE_SIZE);
2498	PGMPhysReleasePageMappingLock(pVM, &Lock);
2499	GCPhysSrc += PAGE_SIZE;
2500	pvDst = (uint8_t *)pvDst + PAGE_SIZE;
2501	cb -= PAGE_SIZE;
2502	}
2503	/* won't ever get here. */
2504	}
2505
2506
2507	/**
2508	* Write to guest physical memory referenced by GC pointer.
2509	* Write memory to GC physical address in guest physical memory.
2510	*
2511	* This will bypass MMIO and access handlers.
2512	*
2513	* @returns VBox status.
2514	* @param pVM VM handle.
2515	* @param GCPhysDst The GC physical address of the destination.
2516	* @param pvSrc The source buffer.
2517	* @param cb The number of bytes to write.
2518	*/
2519	VMMDECL(int) PGMPhysSimpleWriteGCPhys(PVM pVM, RTGCPHYS GCPhysDst, const void *pvSrc, size_t cb)
2520	{
2521	LogFlow(("PGMPhysSimpleWriteGCPhys: %RGp %zu\n", GCPhysDst, cb));
2522
2523	/*
2524	* Treat the first page as a special case.
2525	*/
2526	if (!cb)
2527	return VINF_SUCCESS;
2528
2529	/* map the 1st page */
2530	void *pvDst;
2531	PGMPAGEMAPLOCK Lock;
2532	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock);
2533	if (RT_FAILURE(rc))
2534	return rc;
2535
2536	/* optimize for the case where access is completely within the first page. */
2537	size_t cbPage = PAGE_SIZE - (GCPhysDst & PAGE_OFFSET_MASK);
2538	if (RT_LIKELY(cb <= cbPage))
2539	{
2540	memcpy(pvDst, pvSrc, cb);
2541	PGMPhysReleasePageMappingLock(pVM, &Lock);
2542	return VINF_SUCCESS;
2543	}
2544
2545	/* copy to the end of the page. */
2546	memcpy(pvDst, pvSrc, cbPage);
2547	PGMPhysReleasePageMappingLock(pVM, &Lock);
2548	GCPhysDst += cbPage;
2549	pvSrc = (const uint8_t *)pvSrc + cbPage;
2550	cb -= cbPage;
2551
2552	/*
2553	* Page by page.
2554	*/
2555	for (;;)
2556	{
2557	/* map the page */
2558	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock);
2559	if (RT_FAILURE(rc))
2560	return rc;
2561
2562	/* last page? */
2563	if (cb <= PAGE_SIZE)
2564	{
2565	memcpy(pvDst, pvSrc, cb);
2566	PGMPhysReleasePageMappingLock(pVM, &Lock);
2567	return VINF_SUCCESS;
2568	}
2569
2570	/* copy the entire page and advance */
2571	memcpy(pvDst, pvSrc, PAGE_SIZE);
2572	PGMPhysReleasePageMappingLock(pVM, &Lock);
2573	GCPhysDst += PAGE_SIZE;
2574	pvSrc = (const uint8_t *)pvSrc + PAGE_SIZE;
2575	cb -= PAGE_SIZE;
2576	}
2577	/* won't ever get here. */
2578	}
2579
2580
2581	/**
2582	* Read from guest physical memory referenced by GC pointer.
2583	*
2584	* This function uses the current CR3/CR0/CR4 of the guest and will
2585	* bypass access handlers and not set any accessed bits.
2586	*
2587	* @returns VBox status.
2588	* @param pVCpu The VMCPU handle.
2589	* @param pvDst The destination address.
2590	* @param GCPtrSrc The source address (GC pointer).
2591	* @param cb The number of bytes to read.
2592	*/
2593	VMMDECL(int) PGMPhysSimpleReadGCPtr(PVMCPU pVCpu, void *pvDst, RTGCPTR GCPtrSrc, size_t cb)
2594	{
2595	PVM pVM = pVCpu->CTX_SUFF(pVM);
2596
2597	/*
2598	* Treat the first page as a special case.
2599	*/
2600	if (!cb)
2601	return VINF_SUCCESS;
2602
2603	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysSimpleRead));
2604	STAM_COUNTER_ADD(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysSimpleReadBytes), cb);
2605
2606	/* Take the PGM lock here, because many called functions take the lock for a very short period. That's counter-productive
2607	* when many VCPUs are fighting for the lock.
2608	*/
2609	pgmLock(pVM);
2610
2611	/* map the 1st page */
2612	void const *pvSrc;
2613	PGMPAGEMAPLOCK Lock;
2614	int rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrSrc, &pvSrc, &Lock);
2615	if (RT_FAILURE(rc))
2616	{
2617	pgmUnlock(pVM);
2618	return rc;
2619	}
2620
2621	/* optimize for the case where access is completely within the first page. */
2622	size_t cbPage = PAGE_SIZE - ((RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK);
2623	if (RT_LIKELY(cb <= cbPage))
2624	{
2625	memcpy(pvDst, pvSrc, cb);
2626	PGMPhysReleasePageMappingLock(pVM, &Lock);
2627	pgmUnlock(pVM);
2628	return VINF_SUCCESS;
2629	}
2630
2631	/* copy to the end of the page. */
2632	memcpy(pvDst, pvSrc, cbPage);
2633	PGMPhysReleasePageMappingLock(pVM, &Lock);
2634	GCPtrSrc = (RTGCPTR)((RTGCUINTPTR)GCPtrSrc + cbPage);
2635	pvDst = (uint8_t *)pvDst + cbPage;
2636	cb -= cbPage;
2637
2638	/*
2639	* Page by page.
2640	*/
2641	for (;;)
2642	{
2643	/* map the page */
2644	rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrSrc, &pvSrc, &Lock);
2645	if (RT_FAILURE(rc))
2646	{
2647	pgmUnlock(pVM);
2648	return rc;
2649	}
2650
2651	/* last page? */
2652	if (cb <= PAGE_SIZE)
2653	{
2654	memcpy(pvDst, pvSrc, cb);
2655	PGMPhysReleasePageMappingLock(pVM, &Lock);
2656	pgmUnlock(pVM);
2657	return VINF_SUCCESS;
2658	}
2659
2660	/* copy the entire page and advance */
2661	memcpy(pvDst, pvSrc, PAGE_SIZE);
2662	PGMPhysReleasePageMappingLock(pVM, &Lock);
2663	GCPtrSrc = (RTGCPTR)((RTGCUINTPTR)GCPtrSrc + PAGE_SIZE);
2664	pvDst = (uint8_t *)pvDst + PAGE_SIZE;
2665	cb -= PAGE_SIZE;
2666	}
2667	/* won't ever get here. */
2668	}
2669
2670
2671	/**
2672	* Write to guest physical memory referenced by GC pointer.
2673	*
2674	* This function uses the current CR3/CR0/CR4 of the guest and will
2675	* bypass access handlers and not set dirty or accessed bits.
2676	*
2677	* @returns VBox status.
2678	* @param pVCpu The VMCPU handle.
2679	* @param GCPtrDst The destination address (GC pointer).
2680	* @param pvSrc The source address.
2681	* @param cb The number of bytes to write.
2682	*/
2683	VMMDECL(int) PGMPhysSimpleWriteGCPtr(PVMCPU pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
2684	{
2685	PVM pVM = pVCpu->CTX_SUFF(pVM);
2686
2687	/*
2688	* Treat the first page as a special case.
2689	*/
2690	if (!cb)
2691	return VINF_SUCCESS;
2692
2693	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysSimpleWrite));
2694	STAM_COUNTER_ADD(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysSimpleWriteBytes), cb);
2695
2696	/* map the 1st page */
2697	void *pvDst;
2698	PGMPAGEMAPLOCK Lock;
2699	int rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
2700	if (RT_FAILURE(rc))
2701	return rc;
2702
2703	/* optimize for the case where access is completely within the first page. */
2704	size_t cbPage = PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK);
2705	if (RT_LIKELY(cb <= cbPage))
2706	{
2707	memcpy(pvDst, pvSrc, cb);
2708	PGMPhysReleasePageMappingLock(pVM, &Lock);
2709	return VINF_SUCCESS;
2710	}
2711
2712	/* copy to the end of the page. */
2713	memcpy(pvDst, pvSrc, cbPage);
2714	PGMPhysReleasePageMappingLock(pVM, &Lock);
2715	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + cbPage);
2716	pvSrc = (const uint8_t *)pvSrc + cbPage;
2717	cb -= cbPage;
2718
2719	/*
2720	* Page by page.
2721	*/
2722	for (;;)
2723	{
2724	/* map the page */
2725	rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
2726	if (RT_FAILURE(rc))
2727	return rc;
2728
2729	/* last page? */
2730	if (cb <= PAGE_SIZE)
2731	{
2732	memcpy(pvDst, pvSrc, cb);
2733	PGMPhysReleasePageMappingLock(pVM, &Lock);
2734	return VINF_SUCCESS;
2735	}
2736
2737	/* copy the entire page and advance */
2738	memcpy(pvDst, pvSrc, PAGE_SIZE);
2739	PGMPhysReleasePageMappingLock(pVM, &Lock);
2740	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + PAGE_SIZE);
2741	pvSrc = (const uint8_t *)pvSrc + PAGE_SIZE;
2742	cb -= PAGE_SIZE;
2743	}
2744	/* won't ever get here. */
2745	}
2746
2747
2748	/**
2749	* Write to guest physical memory referenced by GC pointer and update the PTE.
2750	*
2751	* This function uses the current CR3/CR0/CR4 of the guest and will
2752	* bypass access handlers but will set any dirty and accessed bits in the PTE.
2753	*
2754	* If you don't want to set the dirty bit, use PGMPhysSimpleWriteGCPtr().
2755	*
2756	* @returns VBox status.
2757	* @param pVCpu The VMCPU handle.
2758	* @param GCPtrDst The destination address (GC pointer).
2759	* @param pvSrc The source address.
2760	* @param cb The number of bytes to write.
2761	*/
2762	VMMDECL(int) PGMPhysSimpleDirtyWriteGCPtr(PVMCPU pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
2763	{
2764	PVM pVM = pVCpu->CTX_SUFF(pVM);
2765
2766	/*
2767	* Treat the first page as a special case.
2768	* Btw. this is the same code as in PGMPhyssimpleWriteGCPtr excep for the PGMGstModifyPage.
2769	*/
2770	if (!cb)
2771	return VINF_SUCCESS;
2772
2773	/* map the 1st page */
2774	void *pvDst;
2775	PGMPAGEMAPLOCK Lock;
2776	int rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
2777	if (RT_FAILURE(rc))
2778	return rc;
2779
2780	/* optimize for the case where access is completely within the first page. */
2781	size_t cbPage = PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK);
2782	if (RT_LIKELY(cb <= cbPage))
2783	{
2784	memcpy(pvDst, pvSrc, cb);
2785	PGMPhysReleasePageMappingLock(pVM, &Lock);
2786	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
2787	return VINF_SUCCESS;
2788	}
2789
2790	/* copy to the end of the page. */
2791	memcpy(pvDst, pvSrc, cbPage);
2792	PGMPhysReleasePageMappingLock(pVM, &Lock);
2793	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
2794	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + cbPage);
2795	pvSrc = (const uint8_t *)pvSrc + cbPage;
2796	cb -= cbPage;
2797
2798	/*
2799	* Page by page.
2800	*/
2801	for (;;)
2802	{
2803	/* map the page */
2804	rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
2805	if (RT_FAILURE(rc))
2806	return rc;
2807
2808	/* last page? */
2809	if (cb <= PAGE_SIZE)
2810	{
2811	memcpy(pvDst, pvSrc, cb);
2812	PGMPhysReleasePageMappingLock(pVM, &Lock);
2813	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
2814	return VINF_SUCCESS;
2815	}
2816
2817	/* copy the entire page and advance */
2818	memcpy(pvDst, pvSrc, PAGE_SIZE);
2819	PGMPhysReleasePageMappingLock(pVM, &Lock);
2820	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
2821	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + PAGE_SIZE);
2822	pvSrc = (const uint8_t *)pvSrc + PAGE_SIZE;
2823	cb -= PAGE_SIZE;
2824	}
2825	/* won't ever get here. */
2826	}
2827
2828
2829	/**
2830	* Read from guest physical memory referenced by GC pointer.
2831	*
2832	* This function uses the current CR3/CR0/CR4 of the guest and will
2833	* respect access handlers and set accessed bits.
2834	*
2835	* @returns VBox status.
2836	* @param pVCpu The VMCPU handle.
2837	* @param pvDst The destination address.
2838	* @param GCPtrSrc The source address (GC pointer).
2839	* @param cb The number of bytes to read.
2840	* @thread The vCPU EMT.
2841	*/
2842	VMMDECL(int) PGMPhysReadGCPtr(PVMCPU pVCpu, void *pvDst, RTGCPTR GCPtrSrc, size_t cb)
2843	{
2844	RTGCPHYS GCPhys;
2845	uint64_t fFlags;
2846	int rc;
2847	PVM pVM = pVCpu->CTX_SUFF(pVM);
2848
2849	/*
2850	* Anything to do?
2851	*/
2852	if (!cb)
2853	return VINF_SUCCESS;
2854
2855	LogFlow(("PGMPhysReadGCPtr: %RGv %zu\n", GCPtrSrc, cb));
2856
2857	/*
2858	* Optimize reads within a single page.
2859	*/
2860	if (((RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK) + cb <= PAGE_SIZE)
2861	{
2862	/* Convert virtual to physical address + flags */
2863	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtrSrc, &fFlags, &GCPhys);
2864	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrSrc), rc);
2865	GCPhys \|= (RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK;
2866
2867	/* mark the guest page as accessed. */
2868	if (!(fFlags & X86_PTE_A))
2869	{
2870	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)(X86_PTE_A));
2871	AssertRC(rc);
2872	}
2873
2874	return PGMPhysRead(pVM, GCPhys, pvDst, cb);
2875	}
2876
2877	/*
2878	* Page by page.
2879	*/
2880	for (;;)
2881	{
2882	/* Convert virtual to physical address + flags */
2883	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtrSrc, &fFlags, &GCPhys);
2884	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrSrc), rc);
2885	GCPhys \|= (RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK;
2886
2887	/* mark the guest page as accessed. */
2888	if (!(fFlags & X86_PTE_A))
2889	{
2890	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)(X86_PTE_A));
2891	AssertRC(rc);
2892	}
2893
2894	/* copy */
2895	size_t cbRead = PAGE_SIZE - ((RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK);
2896	rc = PGMPhysRead(pVM, GCPhys, pvDst, cbRead);
2897	if (cbRead >= cb \|\| RT_FAILURE(rc))
2898	return rc;
2899
2900	/* next */
2901	cb -= cbRead;
2902	pvDst = (uint8_t *)pvDst + cbRead;
2903	GCPtrSrc += cbRead;
2904	}
2905	}
2906
2907
2908	/**
2909	* Write to guest physical memory referenced by GC pointer.
2910	*
2911	* This function uses the current CR3/CR0/CR4 of the guest and will
2912	* respect access handlers and set dirty and accessed bits.
2913	*
2914	* @returns VBox status.
2915	* @retval VINF_SUCCESS.
2916	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in R0 and GC, NEVER in R3.
2917	*
2918	* @param pVCpu The VMCPU handle.
2919	* @param GCPtrDst The destination address (GC pointer).
2920	* @param pvSrc The source address.
2921	* @param cb The number of bytes to write.
2922	*/
2923	VMMDECL(int) PGMPhysWriteGCPtr(PVMCPU pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
2924	{
2925	RTGCPHYS GCPhys;
2926	uint64_t fFlags;
2927	int rc;
2928	PVM pVM = pVCpu->CTX_SUFF(pVM);
2929
2930	/*
2931	* Anything to do?
2932	*/
2933	if (!cb)
2934	return VINF_SUCCESS;
2935
2936	LogFlow(("PGMPhysWriteGCPtr: %RGv %zu\n", GCPtrDst, cb));
2937
2938	/*
2939	* Optimize writes within a single page.
2940	*/
2941	if (((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK) + cb <= PAGE_SIZE)
2942	{
2943	/* Convert virtual to physical address + flags */
2944	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtrDst, &fFlags, &GCPhys);
2945	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrDst), rc);
2946	GCPhys \|= (RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK;
2947
2948	/* Mention when we ignore X86_PTE_RW... */
2949	if (!(fFlags & X86_PTE_RW))
2950	Log(("PGMPhysGCPtr2GCPhys: Writing to RO page %RGv %#x\n", GCPtrDst, cb));
2951
2952	/* Mark the guest page as accessed and dirty if necessary. */
2953	if ((fFlags & (X86_PTE_A \| X86_PTE_D)) != (X86_PTE_A \| X86_PTE_D))
2954	{
2955	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D));
2956	AssertRC(rc);
2957	}
2958
2959	return PGMPhysWrite(pVM, GCPhys, pvSrc, cb);
2960	}
2961
2962	/*
2963	* Page by page.
2964	*/
2965	for (;;)
2966	{
2967	/* Convert virtual to physical address + flags */
2968	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtrDst, &fFlags, &GCPhys);
2969	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrDst), rc);
2970	GCPhys \|= (RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK;
2971
2972	/* Mention when we ignore X86_PTE_RW... */
2973	if (!(fFlags & X86_PTE_RW))
2974	Log(("PGMPhysGCPtr2GCPhys: Writing to RO page %RGv %#x\n", GCPtrDst, cb));
2975
2976	/* Mark the guest page as accessed and dirty if necessary. */
2977	if ((fFlags & (X86_PTE_A \| X86_PTE_D)) != (X86_PTE_A \| X86_PTE_D))
2978	{
2979	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D));
2980	AssertRC(rc);
2981	}
2982
2983	/* copy */
2984	size_t cbWrite = PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK);
2985	rc = PGMPhysWrite(pVM, GCPhys, pvSrc, cbWrite);
2986	if (cbWrite >= cb \|\| RT_FAILURE(rc))
2987	return rc;
2988
2989	/* next */
2990	cb -= cbWrite;
2991	pvSrc = (uint8_t *)pvSrc + cbWrite;
2992	GCPtrDst += cbWrite;
2993	}
2994	}
2995
2996
2997	/**
2998	* Performs a read of guest virtual memory for instruction emulation.
2999	*
3000	* This will check permissions, raise exceptions and update the access bits.
3001	*
3002	* The current implementation will bypass all access handlers. It may later be
3003	* changed to at least respect MMIO.
3004	*
3005	*
3006	* @returns VBox status code suitable to scheduling.
3007	* @retval VINF_SUCCESS if the read was performed successfully.
3008	* @retval VINF_EM_RAW_GUEST_TRAP if an exception was raised but not dispatched yet.
3009	* @retval VINF_TRPM_XCPT_DISPATCHED if an exception was raised and dispatched.
3010	*
3011	* @param pVCpu The VMCPU handle.
3012	* @param pCtxCore The context core.
3013	* @param pvDst Where to put the bytes we've read.
3014	* @param GCPtrSrc The source address.
3015	* @param cb The number of bytes to read. Not more than a page.
3016	*
3017	* @remark This function will dynamically map physical pages in GC. This may unmap
3018	* mappings done by the caller. Be careful!
3019	*/
3020	VMMDECL(int) PGMPhysInterpretedRead(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, void *pvDst, RTGCUINTPTR GCPtrSrc, size_t cb)
3021	{
3022	PVM pVM = pVCpu->CTX_SUFF(pVM);
3023	Assert(cb <= PAGE_SIZE);
3024
3025	/** @todo r=bird: This isn't perfect!
3026	* -# It's not checking for reserved bits being 1.
3027	* -# It's not correctly dealing with the access bit.
3028	* -# It's not respecting MMIO memory or any other access handlers.
3029	*/
3030	/*
3031	* 1. Translate virtual to physical. This may fault.
3032	* 2. Map the physical address.
3033	* 3. Do the read operation.
3034	* 4. Set access bits if required.
3035	*/
3036	int rc;
3037	unsigned cb1 = PAGE_SIZE - (GCPtrSrc & PAGE_OFFSET_MASK);
3038	if (cb <= cb1)
3039	{
3040	/*
3041	* Not crossing pages.
3042	*/
3043	RTGCPHYS GCPhys;
3044	uint64_t fFlags;
3045	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc, &fFlags, &GCPhys);
3046	if (RT_SUCCESS(rc))
3047	{
3048	/** @todo we should check reserved bits ... */
3049	void *pvSrc;
3050	rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GCPhys, &pvSrc);
3051	switch (rc)
3052	{
3053	case VINF_SUCCESS:
3054	Log(("PGMPhysInterpretedRead: pvDst=%p pvSrc=%p cb=%d\n", pvDst, (uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb));
3055	memcpy(pvDst, (uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb);
3056	break;
3057	case VERR_PGM_PHYS_PAGE_RESERVED:
3058	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3059	memset(pvDst, 0, cb); /** @todo this is wrong, it should be 0xff */
3060	break;
3061	default:
3062	return rc;
3063	}
3064
3065	/** @todo access bit emulation isn't 100% correct. */
3066	if (!(fFlags & X86_PTE_A))
3067	{
3068	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3069	AssertRC(rc);
3070	}
3071	return VINF_SUCCESS;
3072	}
3073	}
3074	else
3075	{
3076	/*
3077	* Crosses pages.
3078	*/
3079	size_t cb2 = cb - cb1;
3080	uint64_t fFlags1;
3081	RTGCPHYS GCPhys1;
3082	uint64_t fFlags2;
3083	RTGCPHYS GCPhys2;
3084	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc, &fFlags1, &GCPhys1);
3085	if (RT_SUCCESS(rc))
3086	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc + cb1, &fFlags2, &GCPhys2);
3087	if (RT_SUCCESS(rc))
3088	{
3089	/** @todo we should check reserved bits ... */
3090	AssertMsgFailed(("cb=%d cb1=%d cb2=%d GCPtrSrc=%RGv\n", cb, cb1, cb2, GCPtrSrc));
3091	void *pvSrc1;
3092	rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GCPhys1, &pvSrc1);
3093	switch (rc)
3094	{
3095	case VINF_SUCCESS:
3096	memcpy(pvDst, (uint8_t *)pvSrc1 + (GCPtrSrc & PAGE_OFFSET_MASK), cb1);
3097	break;
3098	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3099	memset(pvDst, 0, cb1); /** @todo this is wrong, it should be 0xff */
3100	break;
3101	default:
3102	return rc;
3103	}
3104
3105	void *pvSrc2;
3106	rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GCPhys2, &pvSrc2);
3107	switch (rc)
3108	{
3109	case VINF_SUCCESS:
3110	memcpy((uint8_t *)pvDst + cb1, pvSrc2, cb2);
3111	break;
3112	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3113	memset((uint8_t )pvDst + cb1, 0, cb2); /* @todo this is wrong, it should be 0xff */
3114	break;
3115	default:
3116	return rc;
3117	}
3118
3119	if (!(fFlags1 & X86_PTE_A))
3120	{
3121	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3122	AssertRC(rc);
3123	}
3124	if (!(fFlags2 & X86_PTE_A))
3125	{
3126	rc = PGMGstModifyPage(pVCpu, GCPtrSrc + cb1, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3127	AssertRC(rc);
3128	}
3129	return VINF_SUCCESS;
3130	}
3131	}
3132
3133	/*
3134	* Raise a #PF.
3135	*/
3136	uint32_t uErr;
3137
3138	/* Get the current privilege level. */
3139	uint32_t cpl = CPUMGetGuestCPL(pVCpu, pCtxCore);
3140	switch (rc)
3141	{
3142	case VINF_SUCCESS:
3143	uErr = (cpl >= 2) ? X86_TRAP_PF_RSVD \| X86_TRAP_PF_US : X86_TRAP_PF_RSVD;
3144	break;
3145
3146	case VERR_PAGE_NOT_PRESENT:
3147	case VERR_PAGE_TABLE_NOT_PRESENT:
3148	uErr = (cpl >= 2) ? X86_TRAP_PF_US : 0;
3149	break;
3150
3151	default:
3152	AssertMsgFailed(("rc=%Rrc GCPtrSrc=%RGv cb=%#x\n", rc, GCPtrSrc, cb));
3153	return rc;
3154	}
3155	Log(("PGMPhysInterpretedRead: GCPtrSrc=%RGv cb=%#x -> #PF(%#x)\n", GCPtrSrc, cb, uErr));
3156	return TRPMRaiseXcptErrCR2(pVCpu, pCtxCore, X86_XCPT_PF, uErr, GCPtrSrc);
3157	}
3158
3159
3160	/**
3161	* Performs a read of guest virtual memory for instruction emulation.
3162	*
3163	* This will check permissions, raise exceptions and update the access bits.
3164	*
3165	* The current implementation will bypass all access handlers. It may later be
3166	* changed to at least respect MMIO.
3167	*
3168	*
3169	* @returns VBox status code suitable to scheduling.
3170	* @retval VINF_SUCCESS if the read was performed successfully.
3171	* @retval VINF_EM_RAW_GUEST_TRAP if an exception was raised but not dispatched yet.
3172	* @retval VINF_TRPM_XCPT_DISPATCHED if an exception was raised and dispatched.
3173	*
3174	* @param pVCpu The VMCPU handle.
3175	* @param pCtxCore The context core.
3176	* @param pvDst Where to put the bytes we've read.
3177	* @param GCPtrSrc The source address.
3178	* @param cb The number of bytes to read. Not more than a page.
3179	* @param fRaiseTrap If set the trap will be raised on as per spec, if clear
3180	* an appropriate error status will be returned (no
3181	* informational at all).
3182	*
3183	*
3184	* @remarks Takes the PGM lock.
3185	* @remarks A page fault on the 2nd page of the access will be raised without
3186	* writing the bits on the first page since we're ASSUMING that the
3187	* caller is emulating an instruction access.
3188	* @remarks This function will dynamically map physical pages in GC. This may
3189	* unmap mappings done by the caller. Be careful!
3190	*/
3191	VMMDECL(int) PGMPhysInterpretedReadNoHandlers(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, void *pvDst, RTGCUINTPTR GCPtrSrc, size_t cb, bool fRaiseTrap)
3192	{
3193	PVM pVM = pVCpu->CTX_SUFF(pVM);
3194	Assert(cb <= PAGE_SIZE);
3195
3196	/*
3197	* 1. Translate virtual to physical. This may fault.
3198	* 2. Map the physical address.
3199	* 3. Do the read operation.
3200	* 4. Set access bits if required.
3201	*/
3202	int rc;
3203	unsigned cb1 = PAGE_SIZE - (GCPtrSrc & PAGE_OFFSET_MASK);
3204	if (cb <= cb1)
3205	{
3206	/*
3207	* Not crossing pages.
3208	*/
3209	RTGCPHYS GCPhys;
3210	uint64_t fFlags;
3211	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc, &fFlags, &GCPhys);
3212	if (RT_SUCCESS(rc))
3213	{
3214	if (1) /** @todo we should check reserved bits ... */
3215	{
3216	const void *pvSrc;
3217	PGMPAGEMAPLOCK Lock;
3218	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, &pvSrc, &Lock);
3219	switch (rc)
3220	{
3221	case VINF_SUCCESS:
3222	Log(("PGMPhysInterpretedReadNoHandlers: pvDst=%p pvSrc=%p (%RGv) cb=%d\n",
3223	pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), GCPtrSrc, cb));
3224	memcpy(pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb);
3225	break;
3226	case VERR_PGM_PHYS_PAGE_RESERVED:
3227	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3228	memset(pvDst, 0xff, cb);
3229	break;
3230	default:
3231	AssertMsgFailed(("%Rrc\n", rc));
3232	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3233	return rc;
3234	}
3235	PGMPhysReleasePageMappingLock(pVM, &Lock);
3236
3237	if (!(fFlags & X86_PTE_A))
3238	{
3239	/** @todo access bit emulation isn't 100% correct. */
3240	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3241	AssertRC(rc);
3242	}
3243	return VINF_SUCCESS;
3244	}
3245	}
3246	}
3247	else
3248	{
3249	/*
3250	* Crosses pages.
3251	*/
3252	size_t cb2 = cb - cb1;
3253	uint64_t fFlags1;
3254	RTGCPHYS GCPhys1;
3255	uint64_t fFlags2;
3256	RTGCPHYS GCPhys2;
3257	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc, &fFlags1, &GCPhys1);
3258	if (RT_SUCCESS(rc))
3259	{
3260	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc + cb1, &fFlags2, &GCPhys2);
3261	if (RT_SUCCESS(rc))
3262	{
3263	if (1) /** @todo we should check reserved bits ... */
3264	{
3265	const void *pvSrc;
3266	PGMPAGEMAPLOCK Lock;
3267	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys1, &pvSrc, &Lock);
3268	switch (rc)
3269	{
3270	case VINF_SUCCESS:
3271	Log(("PGMPhysInterpretedReadNoHandlers: pvDst=%p pvSrc=%p (%RGv) cb=%d [2]\n",
3272	pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), GCPtrSrc, cb1));
3273	memcpy(pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb1);
3274	PGMPhysReleasePageMappingLock(pVM, &Lock);
3275	break;
3276	case VERR_PGM_PHYS_PAGE_RESERVED:
3277	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3278	memset(pvDst, 0xff, cb1);
3279	break;
3280	default:
3281	AssertMsgFailed(("%Rrc\n", rc));
3282	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3283	return rc;
3284	}
3285
3286	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys2, &pvSrc, &Lock);
3287	switch (rc)
3288	{
3289	case VINF_SUCCESS:
3290	memcpy((uint8_t *)pvDst + cb1, pvSrc, cb2);
3291	PGMPhysReleasePageMappingLock(pVM, &Lock);
3292	break;
3293	case VERR_PGM_PHYS_PAGE_RESERVED:
3294	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3295	memset((uint8_t *)pvDst + cb1, 0xff, cb2);
3296	break;
3297	default:
3298	AssertMsgFailed(("%Rrc\n", rc));
3299	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3300	return rc;
3301	}
3302
3303	if (!(fFlags1 & X86_PTE_A))
3304	{
3305	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3306	AssertRC(rc);
3307	}
3308	if (!(fFlags2 & X86_PTE_A))
3309	{
3310	rc = PGMGstModifyPage(pVCpu, GCPtrSrc + cb1, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3311	AssertRC(rc);
3312	}
3313	return VINF_SUCCESS;
3314	}
3315	/* sort out which page */
3316	}
3317	else
3318	GCPtrSrc += cb1; /* fault on 2nd page */
3319	}
3320	}
3321
3322	/*
3323	* Raise a #PF if we're allowed to do that.
3324	*/
3325	/* Calc the error bits. */
3326	uint32_t cpl = CPUMGetGuestCPL(pVCpu, pCtxCore);
3327	uint32_t uErr;
3328	switch (rc)
3329	{
3330	case VINF_SUCCESS:
3331	uErr = (cpl >= 2) ? X86_TRAP_PF_RSVD \| X86_TRAP_PF_US : X86_TRAP_PF_RSVD;
3332	rc = VERR_ACCESS_DENIED;
3333	break;
3334
3335	case VERR_PAGE_NOT_PRESENT:
3336	case VERR_PAGE_TABLE_NOT_PRESENT:
3337	uErr = (cpl >= 2) ? X86_TRAP_PF_US : 0;
3338	break;
3339
3340	default:
3341	AssertMsgFailed(("rc=%Rrc GCPtrSrc=%RGv cb=%#x\n", rc, GCPtrSrc, cb));
3342	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3343	return rc;
3344	}
3345	if (fRaiseTrap)
3346	{
3347	Log(("PGMPhysInterpretedReadNoHandlers: GCPtrSrc=%RGv cb=%#x -> Raised #PF(%#x)\n", GCPtrSrc, cb, uErr));
3348	return TRPMRaiseXcptErrCR2(pVCpu, pCtxCore, X86_XCPT_PF, uErr, GCPtrSrc);
3349	}
3350	Log(("PGMPhysInterpretedReadNoHandlers: GCPtrSrc=%RGv cb=%#x -> #PF(%#x) [!raised]\n", GCPtrSrc, cb, uErr));
3351	return rc;
3352	}
3353
3354
3355	/**
3356	* Performs a write to guest virtual memory for instruction emulation.
3357	*
3358	* This will check permissions, raise exceptions and update the dirty and access
3359	* bits.
3360	*
3361	* @returns VBox status code suitable to scheduling.
3362	* @retval VINF_SUCCESS if the read was performed successfully.
3363	* @retval VINF_EM_RAW_GUEST_TRAP if an exception was raised but not dispatched yet.
3364	* @retval VINF_TRPM_XCPT_DISPATCHED if an exception was raised and dispatched.
3365	*
3366	* @param pVCpu The VMCPU handle.
3367	* @param pCtxCore The context core.
3368	* @param GCPtrDst The destination address.
3369	* @param pvSrc What to write.
3370	* @param cb The number of bytes to write. Not more than a page.
3371	* @param fRaiseTrap If set the trap will be raised on as per spec, if clear
3372	* an appropriate error status will be returned (no
3373	* informational at all).
3374	*
3375	* @remarks Takes the PGM lock.
3376	* @remarks A page fault on the 2nd page of the access will be raised without
3377	* writing the bits on the first page since we're ASSUMING that the
3378	* caller is emulating an instruction access.
3379	* @remarks This function will dynamically map physical pages in GC. This may
3380	* unmap mappings done by the caller. Be careful!
3381	*/
3382	VMMDECL(int) PGMPhysInterpretedWriteNoHandlers(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb, bool fRaiseTrap)
3383	{
3384	Assert(cb <= PAGE_SIZE);
3385	PVM pVM = pVCpu->CTX_SUFF(pVM);
3386
3387	/*
3388	* 1. Translate virtual to physical. This may fault.
3389	* 2. Map the physical address.
3390	* 3. Do the write operation.
3391	* 4. Set access bits if required.
3392	*/
3393	/** @todo Since this method is frequently used by EMInterpret or IOM
3394	* upon a write fault to an write access monitored page, we can
3395	* reuse the guest page table walking from the \#PF code. */
3396	int rc;
3397	unsigned cb1 = PAGE_SIZE - (GCPtrDst & PAGE_OFFSET_MASK);
3398	if (cb <= cb1)
3399	{
3400	/*
3401	* Not crossing pages.
3402	*/
3403	RTGCPHYS GCPhys;
3404	uint64_t fFlags;
3405	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrDst, &fFlags, &GCPhys);
3406	if (RT_SUCCESS(rc))
3407	{
3408	if ( (fFlags & X86_PTE_RW) /** @todo Also check reserved bits. */
3409	\|\| ( !(CPUMGetGuestCR0(pVCpu) & X86_CR0_WP)
3410	&& CPUMGetGuestCPL(pVCpu, pCtxCore) <= 2) ) /** @todo it's 2, right? Check cpl check below as well. */
3411	{
3412	void *pvDst;
3413	PGMPAGEMAPLOCK Lock;
3414	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys, &pvDst, &Lock);
3415	switch (rc)
3416	{
3417	case VINF_SUCCESS:
3418	Log(("PGMPhysInterpretedWriteNoHandlers: pvDst=%p (%RGv) pvSrc=%p cb=%d\n",
3419	(uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), GCPtrDst, pvSrc, cb));
3420	memcpy((uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), pvSrc, cb);
3421	PGMPhysReleasePageMappingLock(pVM, &Lock);
3422	break;
3423	case VERR_PGM_PHYS_PAGE_RESERVED:
3424	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3425	/* bit bucket */
3426	break;
3427	default:
3428	AssertMsgFailed(("%Rrc\n", rc));
3429	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3430	return rc;
3431	}
3432
3433	if (!(fFlags & (X86_PTE_A \| X86_PTE_D)))
3434	{
3435	/** @todo dirty & access bit emulation isn't 100% correct. */
3436	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D));
3437	AssertRC(rc);
3438	}
3439	return VINF_SUCCESS;
3440	}
3441	rc = VERR_ACCESS_DENIED;
3442	}
3443	}
3444	else
3445	{
3446	/*
3447	* Crosses pages.
3448	*/
3449	size_t cb2 = cb - cb1;
3450	uint64_t fFlags1;
3451	RTGCPHYS GCPhys1;
3452	uint64_t fFlags2;
3453	RTGCPHYS GCPhys2;
3454	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrDst, &fFlags1, &GCPhys1);
3455	if (RT_SUCCESS(rc))
3456	{
3457	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrDst + cb1, &fFlags2, &GCPhys2);
3458	if (RT_SUCCESS(rc))
3459	{
3460	if ( ( (fFlags1 & X86_PTE_RW) /** @todo Also check reserved bits. */
3461	&& (fFlags2 & X86_PTE_RW))
3462	\|\| ( !(CPUMGetGuestCR0(pVCpu) & X86_CR0_WP)
3463	&& CPUMGetGuestCPL(pVCpu, pCtxCore) <= 2) )
3464	{
3465	void *pvDst;
3466	PGMPAGEMAPLOCK Lock;
3467	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys1, &pvDst, &Lock);
3468	switch (rc)
3469	{
3470	case VINF_SUCCESS:
3471	Log(("PGMPhysInterpretedWriteNoHandlers: pvDst=%p (%RGv) pvSrc=%p cb=%d\n",
3472	(uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), GCPtrDst, pvSrc, cb1));
3473	memcpy((uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), pvSrc, cb1);
3474	PGMPhysReleasePageMappingLock(pVM, &Lock);
3475	break;
3476	case VERR_PGM_PHYS_PAGE_RESERVED:
3477	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3478	/* bit bucket */
3479	break;
3480	default:
3481	AssertMsgFailed(("%Rrc\n", rc));
3482	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3483	return rc;
3484	}
3485
3486	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys2, &pvDst, &Lock);
3487	switch (rc)
3488	{
3489	case VINF_SUCCESS:
3490	memcpy(pvDst, (const uint8_t *)pvSrc + cb1, cb2);
3491	PGMPhysReleasePageMappingLock(pVM, &Lock);
3492	break;
3493	case VERR_PGM_PHYS_PAGE_RESERVED:
3494	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3495	/* bit bucket */
3496	break;
3497	default:
3498	AssertMsgFailed(("%Rrc\n", rc));
3499	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3500	return rc;
3501	}
3502
3503	if (!(fFlags1 & (X86_PTE_A \| X86_PTE_RW)))
3504	{
3505	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, (X86_PTE_A \| X86_PTE_RW), ~(uint64_t)(X86_PTE_A \| X86_PTE_RW));
3506	AssertRC(rc);
3507	}
3508	if (!(fFlags2 & (X86_PTE_A \| X86_PTE_RW)))
3509	{
3510	rc = PGMGstModifyPage(pVCpu, GCPtrDst + cb1, 1, (X86_PTE_A \| X86_PTE_RW), ~(uint64_t)(X86_PTE_A \| X86_PTE_RW));
3511	AssertRC(rc);
3512	}
3513	return VINF_SUCCESS;
3514	}
3515	if ((fFlags1 & (X86_PTE_RW)) == X86_PTE_RW)
3516	GCPtrDst += cb1; /* fault on the 2nd page. */
3517	rc = VERR_ACCESS_DENIED;
3518	}
3519	else
3520	GCPtrDst += cb1; /* fault on the 2nd page. */
3521	}
3522	}
3523
3524	/*
3525	* Raise a #PF if we're allowed to do that.
3526	*/
3527	/* Calc the error bits. */
3528	uint32_t uErr;
3529	uint32_t cpl = CPUMGetGuestCPL(pVCpu, pCtxCore);
3530	switch (rc)
3531	{
3532	case VINF_SUCCESS:
3533	uErr = (cpl >= 2) ? X86_TRAP_PF_RSVD \| X86_TRAP_PF_US : X86_TRAP_PF_RSVD;
3534	rc = VERR_ACCESS_DENIED;
3535	break;
3536
3537	case VERR_ACCESS_DENIED:
3538	uErr = (cpl >= 2) ? X86_TRAP_PF_RW \| X86_TRAP_PF_US : X86_TRAP_PF_RW;
3539	break;
3540
3541	case VERR_PAGE_NOT_PRESENT:
3542	case VERR_PAGE_TABLE_NOT_PRESENT:
3543	uErr = (cpl >= 2) ? X86_TRAP_PF_US : 0;
3544	break;
3545
3546	default:
3547	AssertMsgFailed(("rc=%Rrc GCPtrDst=%RGv cb=%#x\n", rc, GCPtrDst, cb));
3548	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3549	return rc;
3550	}
3551	if (fRaiseTrap)
3552	{
3553	Log(("PGMPhysInterpretedWriteNoHandlers: GCPtrDst=%RGv cb=%#x -> Raised #PF(%#x)\n", GCPtrDst, cb, uErr));
3554	return TRPMRaiseXcptErrCR2(pVCpu, pCtxCore, X86_XCPT_PF, uErr, GCPtrDst);
3555	}
3556	Log(("PGMPhysInterpretedWriteNoHandlers: GCPtrDst=%RGv cb=%#x -> #PF(%#x) [!raised]\n", GCPtrDst, cb, uErr));
3557	return rc;
3558	}
3559

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllPhys.cpp@ 31402

Download in other formats: