PGMAllPhys.cpp@ 58437

Last change on this file since 58437 was 58170, checked in by vboxsync, 9 years ago
doxygen: fixes.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 165.0 KB

Line
1	/* $Id: PGMAllPhys.cpp 58170 2015-10-12 09:27:14Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor, Physical Memory Addressing.
4	*/
5
6	/*
7	* Copyright (C) 2006-2015 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_PGM_PHYS
23	#include <VBox/vmm/pgm.h>
24	#include <VBox/vmm/trpm.h>
25	#include <VBox/vmm/vmm.h>
26	#include <VBox/vmm/iom.h>
27	#include <VBox/vmm/em.h>
28	#ifdef VBOX_WITH_REM
29	# include <VBox/vmm/rem.h>
30	#endif
31	#include "PGMInternal.h"
32	#include <VBox/vmm/vm.h>
33	#include "PGMInline.h"
34	#include <VBox/param.h>
35	#include <VBox/err.h>
36	#include <iprt/assert.h>
37	#include <iprt/string.h>
38	#include <iprt/asm-amd64-x86.h>
39	#include <VBox/log.h>
40	#ifdef IN_RING3
41	# include <iprt/thread.h>
42	#endif
43
44
45	/*********************************************************************************************************************************
46	* Defined Constants And Macros *
47	*********************************************************************************************************************************/
48	/** Enable the physical TLB. */
49	#define PGM_WITH_PHYS_TLB
50
51	/** @def PGM_HANDLER_PHYS_IS_VALID_STATUS
52	* Checks if valid physical access handler return code (normal handler, not PF).
53	*
54	* Checks if the given strict status code is one of the expected ones for a
55	* physical access handler in the current context.
56	*
57	* @returns true or false.
58	* @param a_rcStrict The status code.
59	* @param a_fWrite Whether it is a write or read being serviced.
60	*
61	* @remarks We wish to keep the list of statuses here as short as possible.
62	* When changing, please make sure to update the PGMPhysRead,
63	* PGMPhysWrite, PGMPhysReadGCPtr and PGMPhysWriteGCPtr docs too.
64	*/
65	#ifdef IN_RING3
66	# define PGM_HANDLER_PHYS_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
67	( (a_rcStrict) == VINF_SUCCESS \
68	\|\| (a_rcStrict) == VINF_PGM_HANDLER_DO_DEFAULT)
69	#elif defined(IN_RING0) \|\| defined(IN_RC)
70	# define PGM_HANDLER_PHYS_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
71	( (a_rcStrict) == VINF_SUCCESS \
72	\|\| (a_rcStrict) == VINF_PGM_HANDLER_DO_DEFAULT \
73	\
74	\|\| (a_rcStrict) == ((a_fWrite) ? VINF_IOM_R3_MMIO_WRITE : VINF_IOM_R3_MMIO_READ) \
75	\|\| (a_rcStrict) == VINF_IOM_R3_MMIO_READ_WRITE \
76	\
77	\|\| ((a_fWrite) ? (a_rcStrict) == VINF_EM_RAW_EMULATE_IO_BLOCK : false) \
78	\
79	\|\| (a_rcStrict) == VINF_EM_RAW_EMULATE_INSTR \
80	\|\| (a_rcStrict) == VINF_EM_DBG_STOP \
81	\|\| (a_rcStrict) == VINF_EM_DBG_BREAKPOINT \
82	\|\| (a_rcStrict) == VINF_EM_OFF \
83	\|\| (a_rcStrict) == VINF_EM_SUSPEND \
84	\|\| (a_rcStrict) == VINF_EM_RESET \
85	)
86	#else
87	# error "Context?"
88	#endif
89
90	/** @def PGM_HANDLER_VIRT_IS_VALID_STATUS
91	* Checks if valid virtual access handler return code (normal handler, not PF).
92	*
93	* Checks if the given strict status code is one of the expected ones for a
94	* virtual access handler in the current context.
95	*
96	* @returns true or false.
97	* @param a_rcStrict The status code.
98	* @param a_fWrite Whether it is a write or read being serviced.
99	*
100	* @remarks We wish to keep the list of statuses here as short as possible.
101	* When changing, please make sure to update the PGMPhysRead,
102	* PGMPhysWrite, PGMPhysReadGCPtr and PGMPhysWriteGCPtr docs too.
103	*/
104	#ifdef IN_RING3
105	# define PGM_HANDLER_VIRT_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
106	( (a_rcStrict) == VINF_SUCCESS \
107	\|\| (a_rcStrict) == VINF_PGM_HANDLER_DO_DEFAULT)
108	#elif defined(IN_RING0)
109	# define PGM_HANDLER_VIRT_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
110	(false /* no virtual handlers in ring-0! */ )
111	#elif defined(IN_RC)
112	# define PGM_HANDLER_VIRT_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
113	( (a_rcStrict) == VINF_SUCCESS \
114	\|\| (a_rcStrict) == VINF_PGM_HANDLER_DO_DEFAULT \
115	\
116	\|\| ((a_fWrite) ? (a_rcStrict) == VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT : 0) \
117	\|\| ((a_fWrite) ? (a_rcStrict) == VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT : 0) \
118	\|\| ((a_fWrite) ? (a_rcStrict) == VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT : 0) \
119	\|\| ((a_fWrite) ? (a_rcStrict) == VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT : 0) \
120	\|\| ((a_fWrite) ? (a_rcStrict) == VINF_SELM_SYNC_GDT : 0) \
121	\|\| ((a_fWrite) ? (a_rcStrict) == VINF_CSAM_PENDING_ACTION : 0) \
122	\|\| (a_rcStrict) == VINF_PATM_CHECK_PATCH_PAGE \
123	\
124	\|\| (a_rcStrict) == VINF_EM_RAW_EMULATE_INSTR \
125	\|\| (a_rcStrict) == VINF_EM_DBG_STOP \
126	\|\| (a_rcStrict) == VINF_EM_DBG_BREAKPOINT \
127	)
128	#else
129	# error "Context?"
130	#endif
131
132
133
134	#ifndef IN_RING3
135
136	/**
137	* @callback_method_impl{FNPGMPHYSHANDLER,
138	* Dummy for forcing ring-3 handling of the access.}
139	*/
140	DECLEXPORT(VBOXSTRICTRC)
141	pgmPhysHandlerRedirectToHC(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf,
142	PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, void *pvUser)
143	{
144	NOREF(pVM); NOREF(pVCpu); NOREF(GCPhys); NOREF(pvPhys); NOREF(pvBuf); NOREF(cbBuf);
145	NOREF(enmAccessType); NOREF(enmOrigin); NOREF(pvUser);
146	return VINF_EM_RAW_EMULATE_INSTR;
147	}
148
149
150	/**
151	* @callback_method_impl{FNPGMRZPHYSPFHANDLER,
152	* Dummy for forcing ring-3 handling of the access.}
153	*/
154	VMMDECL(VBOXSTRICTRC) pgmPhysPfHandlerRedirectToHC(PVM pVM, PVMCPU pVCpu, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame,
155	RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
156	{
157	NOREF(pVM); NOREF(pVCpu); NOREF(uErrorCode); NOREF(pRegFrame); NOREF(pvFault); NOREF(GCPhysFault); NOREF(pvUser);
158	return VINF_EM_RAW_EMULATE_INSTR;
159	}
160
161
162	/**
163	* @callback_method_impl{FNPGMRZPHYSPFHANDLER,
164	* \#PF access handler callback for guest ROM range write access.}
165	*
166	* @remarks The @a pvUser argument points to the PGMROMRANGE.
167	*/
168	DECLEXPORT(VBOXSTRICTRC) pgmPhysRomWritePfHandler(PVM pVM, PVMCPU pVCpu, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame,
169	RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
170	{
171	int rc;
172	PPGMROMRANGE pRom = (PPGMROMRANGE)pvUser;
173	uint32_t iPage = (GCPhysFault - pRom->GCPhys) >> PAGE_SHIFT;
174	NOREF(uErrorCode); NOREF(pvFault);
175
176	Assert(uErrorCode & X86_TRAP_PF_RW); /* This shall not be used for read access! */
177
178	Assert(iPage < (pRom->cb >> PAGE_SHIFT));
179	switch (pRom->aPages[iPage].enmProt)
180	{
181	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
182	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
183	{
184	/*
185	* If it's a simple instruction which doesn't change the cpu state
186	* we will simply skip it. Otherwise we'll have to defer it to REM.
187	*/
188	uint32_t cbOp;
189	PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
190	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
191	if ( RT_SUCCESS(rc)
192	&& pDis->uCpuMode == DISCPUMODE_32BIT /** @todo why does this matter? */
193	&& !(pDis->fPrefix & (DISPREFIX_REPNE \| DISPREFIX_REP \| DISPREFIX_SEG)))
194	{
195	switch (pDis->bOpCode)
196	{
197	/** @todo Find other instructions we can safely skip, possibly
198	* adding this kind of detection to DIS or EM. */
199	case OP_MOV:
200	pRegFrame->rip += cbOp;
201	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestROMWriteHandled);
202	return VINF_SUCCESS;
203	}
204	}
205	break;
206	}
207
208	case PGMROMPROT_READ_RAM_WRITE_RAM:
209	pRom->aPages[iPage].LiveSave.fWrittenTo = true;
210	rc = PGMHandlerPhysicalPageTempOff(pVM, pRom->GCPhys, GCPhysFault & X86_PTE_PG_MASK);
211	AssertRC(rc);
212	break; /** @todo Must edit the shadow PT and restart the instruction, not use the interpreter! */
213
214	case PGMROMPROT_READ_ROM_WRITE_RAM:
215	/* Handle it in ring-3 because it's way easier there. */
216	pRom->aPages[iPage].LiveSave.fWrittenTo = true;
217	break;
218
219	default:
220	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhysFault=%RGp\n",
221	pRom->aPages[iPage].enmProt, iPage, GCPhysFault),
222	VERR_IPE_NOT_REACHED_DEFAULT_CASE);
223	}
224
225	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestROMWriteUnhandled);
226	return VINF_EM_RAW_EMULATE_INSTR;
227	}
228
229	#endif /* !IN_RING3 */
230
231
232	/**
233	* @callback_method_impl{FNPGMPHYSHANDLER,
234	* Access handler callback for ROM write accesses.}
235	*
236	* @remarks The @a pvUser argument points to the PGMROMRANGE.
237	*/
238	PGM_ALL_CB2_DECL(VBOXSTRICTRC)
239	pgmPhysRomWriteHandler(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf,
240	PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, void *pvUser)
241	{
242	PPGMROMRANGE pRom = (PPGMROMRANGE)pvUser;
243	const uint32_t iPage = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
244	Assert(iPage < (pRom->cb >> PAGE_SHIFT));
245	PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
246	Log5(("pgmPhysRomWriteHandler: %d %c %#08RGp %#04zx\n", pRomPage->enmProt, enmAccessType == PGMACCESSTYPE_READ ? 'R' : 'W', GCPhys, cbBuf));
247	NOREF(pVCpu); NOREF(pvPhys); NOREF(enmOrigin);
248
249	if (enmAccessType == PGMACCESSTYPE_READ)
250	{
251	switch (pRomPage->enmProt)
252	{
253	/*
254	* Take the default action.
255	*/
256	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
257	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
258	case PGMROMPROT_READ_ROM_WRITE_RAM:
259	case PGMROMPROT_READ_RAM_WRITE_RAM:
260	return VINF_PGM_HANDLER_DO_DEFAULT;
261
262	default:
263	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
264	pRom->aPages[iPage].enmProt, iPage, GCPhys),
265	VERR_IPE_NOT_REACHED_DEFAULT_CASE);
266	}
267	}
268	else
269	{
270	Assert(enmAccessType == PGMACCESSTYPE_WRITE);
271	switch (pRomPage->enmProt)
272	{
273	/*
274	* Ignore writes.
275	*/
276	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
277	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
278	return VINF_SUCCESS;
279
280	/*
281	* Write to the RAM page.
282	*/
283	case PGMROMPROT_READ_ROM_WRITE_RAM:
284	case PGMROMPROT_READ_RAM_WRITE_RAM: /* yes this will get here too, it's way simpler that way. */
285	{
286	/* This should be impossible now, pvPhys doesn't work cross page anylonger. */
287	Assert(((GCPhys - pRom->GCPhys + cbBuf - 1) >> PAGE_SHIFT) == iPage);
288
289	/*
290	* Take the lock, do lazy allocation, map the page and copy the data.
291	*
292	* Note that we have to bypass the mapping TLB since it works on
293	* guest physical addresses and entering the shadow page would
294	* kind of screw things up...
295	*/
296	int rc = pgmLock(pVM);
297	AssertRC(rc);
298
299	PPGMPAGE pShadowPage = &pRomPage->Shadow;
300	if (!PGMROMPROT_IS_ROM(pRomPage->enmProt))
301	{
302	pShadowPage = pgmPhysGetPage(pVM, GCPhys);
303	AssertLogRelReturn(pShadowPage, VERR_PGM_PHYS_PAGE_GET_IPE);
304	}
305
306	void *pvDstPage;
307	rc = pgmPhysPageMakeWritableAndMap(pVM, pShadowPage, GCPhys & X86_PTE_PG_MASK, &pvDstPage);
308	if (RT_SUCCESS(rc))
309	{
310	memcpy((uint8_t *)pvDstPage + (GCPhys & PAGE_OFFSET_MASK), pvBuf, cbBuf);
311	pRomPage->LiveSave.fWrittenTo = true;
312
313	AssertMsg( rc == VINF_SUCCESS
314	\|\| ( rc == VINF_PGM_SYNC_CR3
315	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
316	, ("%Rrc\n", rc));
317	rc = VINF_SUCCESS;
318	}
319
320	pgmUnlock(pVM);
321	return rc;
322	}
323
324	default:
325	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
326	pRom->aPages[iPage].enmProt, iPage, GCPhys),
327	VERR_IPE_NOT_REACHED_DEFAULT_CASE);
328	}
329	}
330	}
331
332
333	/**
334	* Invalidates the RAM range TLBs.
335	*
336	* @param pVM The cross context VM structure.
337	*/
338	void pgmPhysInvalidRamRangeTlbs(PVM pVM)
339	{
340	pgmLock(pVM);
341	for (uint32_t i = 0; i < PGM_RAMRANGE_TLB_ENTRIES; i++)
342	{
343	pVM->pgm.s.apRamRangesTlbR3[i] = NIL_RTR3PTR;
344	pVM->pgm.s.apRamRangesTlbR0[i] = NIL_RTR0PTR;
345	pVM->pgm.s.apRamRangesTlbRC[i] = NIL_RTRCPTR;
346	}
347	pgmUnlock(pVM);
348	}
349
350
351	/**
352	* Tests if a value of type RTGCPHYS is negative if the type had been signed
353	* instead of unsigned.
354	*
355	* @returns @c true if negative, @c false if positive or zero.
356	* @param a_GCPhys The value to test.
357	* @todo Move me to iprt/types.h.
358	*/
359	#define RTGCPHYS_IS_NEGATIVE(a_GCPhys) ((a_GCPhys) & ((RTGCPHYS)1 << (sizeof(RTGCPHYS)*8 - 1)))
360
361
362	/**
363	* Slow worker for pgmPhysGetRange.
364	*
365	* @copydoc pgmPhysGetRange
366	*/
367	PPGMRAMRANGE pgmPhysGetRangeSlow(PVM pVM, RTGCPHYS GCPhys)
368	{
369	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
370
371	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
372	while (pRam)
373	{
374	RTGCPHYS off = GCPhys - pRam->GCPhys;
375	if (off < pRam->cb)
376	{
377	pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
378	return pRam;
379	}
380	if (RTGCPHYS_IS_NEGATIVE(off))
381	pRam = pRam->CTX_SUFF(pLeft);
382	else
383	pRam = pRam->CTX_SUFF(pRight);
384	}
385	return NULL;
386	}
387
388
389	/**
390	* Slow worker for pgmPhysGetRangeAtOrAbove.
391	*
392	* @copydoc pgmPhysGetRangeAtOrAbove
393	*/
394	PPGMRAMRANGE pgmPhysGetRangeAtOrAboveSlow(PVM pVM, RTGCPHYS GCPhys)
395	{
396	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
397
398	PPGMRAMRANGE pLastLeft = NULL;
399	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
400	while (pRam)
401	{
402	RTGCPHYS off = GCPhys - pRam->GCPhys;
403	if (off < pRam->cb)
404	{
405	pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
406	return pRam;
407	}
408	if (RTGCPHYS_IS_NEGATIVE(off))
409	{
410	pLastLeft = pRam;
411	pRam = pRam->CTX_SUFF(pLeft);
412	}
413	else
414	pRam = pRam->CTX_SUFF(pRight);
415	}
416	return pLastLeft;
417	}
418
419
420	/**
421	* Slow worker for pgmPhysGetPage.
422	*
423	* @copydoc pgmPhysGetPage
424	*/
425	PPGMPAGE pgmPhysGetPageSlow(PVM pVM, RTGCPHYS GCPhys)
426	{
427	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
428
429	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
430	while (pRam)
431	{
432	RTGCPHYS off = GCPhys - pRam->GCPhys;
433	if (off < pRam->cb)
434	{
435	pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
436	return &pRam->aPages[off >> PAGE_SHIFT];
437	}
438
439	if (RTGCPHYS_IS_NEGATIVE(off))
440	pRam = pRam->CTX_SUFF(pLeft);
441	else
442	pRam = pRam->CTX_SUFF(pRight);
443	}
444	return NULL;
445	}
446
447
448	/**
449	* Slow worker for pgmPhysGetPageEx.
450	*
451	* @copydoc pgmPhysGetPageEx
452	*/
453	int pgmPhysGetPageExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage)
454	{
455	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
456
457	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
458	while (pRam)
459	{
460	RTGCPHYS off = GCPhys - pRam->GCPhys;
461	if (off < pRam->cb)
462	{
463	pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
464	*ppPage = &pRam->aPages[off >> PAGE_SHIFT];
465	return VINF_SUCCESS;
466	}
467
468	if (RTGCPHYS_IS_NEGATIVE(off))
469	pRam = pRam->CTX_SUFF(pLeft);
470	else
471	pRam = pRam->CTX_SUFF(pRight);
472	}
473
474	*ppPage = NULL;
475	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
476	}
477
478
479	/**
480	* Slow worker for pgmPhysGetPageAndRangeEx.
481	*
482	* @copydoc pgmPhysGetPageAndRangeEx
483	*/
484	int pgmPhysGetPageAndRangeExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam)
485	{
486	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
487
488	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
489	while (pRam)
490	{
491	RTGCPHYS off = GCPhys - pRam->GCPhys;
492	if (off < pRam->cb)
493	{
494	pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
495	*ppRam = pRam;
496	*ppPage = &pRam->aPages[off >> PAGE_SHIFT];
497	return VINF_SUCCESS;
498	}
499
500	if (RTGCPHYS_IS_NEGATIVE(off))
501	pRam = pRam->CTX_SUFF(pLeft);
502	else
503	pRam = pRam->CTX_SUFF(pRight);
504	}
505
506	*ppRam = NULL;
507	*ppPage = NULL;
508	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
509	}
510
511
512	/**
513	* Checks if Address Gate 20 is enabled or not.
514	*
515	* @returns true if enabled.
516	* @returns false if disabled.
517	* @param pVCpu The cross context virtual CPU structure.
518	*/
519	VMMDECL(bool) PGMPhysIsA20Enabled(PVMCPU pVCpu)
520	{
521	LogFlow(("PGMPhysIsA20Enabled %d\n", pVCpu->pgm.s.fA20Enabled));
522	return pVCpu->pgm.s.fA20Enabled;
523	}
524
525
526	/**
527	* Validates a GC physical address.
528	*
529	* @returns true if valid.
530	* @returns false if invalid.
531	* @param pVM The cross context VM structure.
532	* @param GCPhys The physical address to validate.
533	*/
534	VMMDECL(bool) PGMPhysIsGCPhysValid(PVM pVM, RTGCPHYS GCPhys)
535	{
536	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
537	return pPage != NULL;
538	}
539
540
541	/**
542	* Checks if a GC physical address is a normal page,
543	* i.e. not ROM, MMIO or reserved.
544	*
545	* @returns true if normal.
546	* @returns false if invalid, ROM, MMIO or reserved page.
547	* @param pVM The cross context VM structure.
548	* @param GCPhys The physical address to check.
549	*/
550	VMMDECL(bool) PGMPhysIsGCPhysNormal(PVM pVM, RTGCPHYS GCPhys)
551	{
552	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
553	return pPage
554	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM;
555	}
556
557
558	/**
559	* Converts a GC physical address to a HC physical address.
560	*
561	* @returns VINF_SUCCESS on success.
562	* @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
563	* page but has no physical backing.
564	* @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
565	* GC physical address.
566	*
567	* @param pVM The cross context VM structure.
568	* @param GCPhys The GC physical address to convert.
569	* @param pHCPhys Where to store the HC physical address on success.
570	*/
571	VMM_INT_DECL(int) PGMPhysGCPhys2HCPhys(PVM pVM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys)
572	{
573	pgmLock(pVM);
574	PPGMPAGE pPage;
575	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
576	if (RT_SUCCESS(rc))
577	*pHCPhys = PGM_PAGE_GET_HCPHYS(pPage) \| (GCPhys & PAGE_OFFSET_MASK);
578	pgmUnlock(pVM);
579	return rc;
580	}
581
582
583	/**
584	* Invalidates all page mapping TLBs.
585	*
586	* @param pVM The cross context VM structure.
587	*/
588	void pgmPhysInvalidatePageMapTLB(PVM pVM)
589	{
590	pgmLock(pVM);
591	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatPageMapTlbFlushes);
592
593	/* Clear the shared R0/R3 TLB completely. */
594	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbHC.aEntries); i++)
595	{
596	pVM->pgm.s.PhysTlbHC.aEntries[i].GCPhys = NIL_RTGCPHYS;
597	pVM->pgm.s.PhysTlbHC.aEntries[i].pPage = 0;
598	pVM->pgm.s.PhysTlbHC.aEntries[i].pMap = 0;
599	pVM->pgm.s.PhysTlbHC.aEntries[i].pv = 0;
600	}
601
602	/** @todo clear the RC TLB whenever we add it. */
603
604	pgmUnlock(pVM);
605	}
606
607
608	/**
609	* Invalidates a page mapping TLB entry
610	*
611	* @param pVM The cross context VM structure.
612	* @param GCPhys GCPhys entry to flush
613	*/
614	void pgmPhysInvalidatePageMapTLBEntry(PVM pVM, RTGCPHYS GCPhys)
615	{
616	PGM_LOCK_ASSERT_OWNER(pVM);
617
618	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatPageMapTlbFlushEntry);
619
620	#ifdef IN_RC
621	unsigned idx = PGM_PAGER3MAPTLB_IDX(GCPhys);
622	pVM->pgm.s.PhysTlbHC.aEntries[idx].GCPhys = NIL_RTGCPHYS;
623	pVM->pgm.s.PhysTlbHC.aEntries[idx].pPage = 0;
624	pVM->pgm.s.PhysTlbHC.aEntries[idx].pMap = 0;
625	pVM->pgm.s.PhysTlbHC.aEntries[idx].pv = 0;
626	#else
627	/* Clear the shared R0/R3 TLB entry. */
628	PPGMPAGEMAPTLBE pTlbe = &pVM->pgm.s.CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
629	pTlbe->GCPhys = NIL_RTGCPHYS;
630	pTlbe->pPage = 0;
631	pTlbe->pMap = 0;
632	pTlbe->pv = 0;
633	#endif
634
635	/** @todo clear the RC TLB whenever we add it. */
636	}
637
638	/**
639	* Makes sure that there is at least one handy page ready for use.
640	*
641	* This will also take the appropriate actions when reaching water-marks.
642	*
643	* @returns VBox status code.
644	* @retval VINF_SUCCESS on success.
645	* @retval VERR_EM_NO_MEMORY if we're really out of memory.
646	*
647	* @param pVM The cross context VM structure.
648	*
649	* @remarks Must be called from within the PGM critical section. It may
650	* nip back to ring-3/0 in some cases.
651	*/
652	static int pgmPhysEnsureHandyPage(PVM pVM)
653	{
654	AssertMsg(pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d\n", pVM->pgm.s.cHandyPages));
655
656	/*
657	* Do we need to do anything special?
658	*/
659	#ifdef IN_RING3
660	if (pVM->pgm.s.cHandyPages <= RT_MAX(PGM_HANDY_PAGES_SET_FF, PGM_HANDY_PAGES_R3_ALLOC))
661	#else
662	if (pVM->pgm.s.cHandyPages <= RT_MAX(PGM_HANDY_PAGES_SET_FF, PGM_HANDY_PAGES_RZ_TO_R3))
663	#endif
664	{
665	/*
666	* Allocate pages only if we're out of them, or in ring-3, almost out.
667	*/
668	#ifdef IN_RING3
669	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_R3_ALLOC)
670	#else
671	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_RZ_ALLOC)
672	#endif
673	{
674	Log(("PGM: cHandyPages=%u out of %u -> allocate more; VM_FF_PGM_NO_MEMORY=%RTbool\n",
675	pVM->pgm.s.cHandyPages, RT_ELEMENTS(pVM->pgm.s.aHandyPages), VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY) ));
676	#ifdef IN_RING3
677	int rc = PGMR3PhysAllocateHandyPages(pVM);
678	#else
679	int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES, 0);
680	#endif
681	if (RT_UNLIKELY(rc != VINF_SUCCESS))
682	{
683	if (RT_FAILURE(rc))
684	return rc;
685	AssertMsgReturn(rc == VINF_EM_NO_MEMORY, ("%Rrc\n", rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
686	if (!pVM->pgm.s.cHandyPages)
687	{
688	LogRel(("PGM: no more handy pages!\n"));
689	return VERR_EM_NO_MEMORY;
690	}
691	Assert(VM_FF_IS_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES));
692	Assert(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY));
693	#ifdef IN_RING3
694	# ifdef VBOX_WITH_REM
695	REMR3NotifyFF(pVM);
696	# endif
697	#else
698	VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3); /* paranoia */
699	#endif
700	}
701	AssertMsgReturn( pVM->pgm.s.cHandyPages > 0
702	&& pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages),
703	("%u\n", pVM->pgm.s.cHandyPages),
704	VERR_PGM_HANDY_PAGE_IPE);
705	}
706	else
707	{
708	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_SET_FF)
709	VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
710	#ifndef IN_RING3
711	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_RZ_TO_R3)
712	{
713	Log(("PGM: VM_FF_TO_R3 - cHandyPages=%u out of %u\n", pVM->pgm.s.cHandyPages, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
714	VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3);
715	}
716	#endif
717	}
718	}
719
720	return VINF_SUCCESS;
721	}
722
723
724	/**
725	* Replace a zero or shared page with new page that we can write to.
726	*
727	* @returns The following VBox status codes.
728	* @retval VINF_SUCCESS on success, pPage is modified.
729	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
730	* @retval VERR_EM_NO_MEMORY if we're totally out of memory.
731	*
732	* @todo Propagate VERR_EM_NO_MEMORY up the call tree.
733	*
734	* @param pVM The cross context VM structure.
735	* @param pPage The physical page tracking structure. This will
736	* be modified on success.
737	* @param GCPhys The address of the page.
738	*
739	* @remarks Must be called from within the PGM critical section. It may
740	* nip back to ring-3/0 in some cases.
741	*
742	* @remarks This function shouldn't really fail, however if it does
743	* it probably means we've screwed up the size of handy pages and/or
744	* the low-water mark. Or, that some device I/O is causing a lot of
745	* pages to be allocated while while the host is in a low-memory
746	* condition. This latter should be handled elsewhere and in a more
747	* controlled manner, it's on the @bugref{3170} todo list...
748	*/
749	int pgmPhysAllocPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
750	{
751	LogFlow(("pgmPhysAllocPage: %R[pgmpage] %RGp\n", pPage, GCPhys));
752
753	/*
754	* Prereqs.
755	*/
756	PGM_LOCK_ASSERT_OWNER(pVM);
757	AssertMsg(PGM_PAGE_IS_ZERO(pPage) \|\| PGM_PAGE_IS_SHARED(pPage), ("%R[pgmpage] %RGp\n", pPage, GCPhys));
758	Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage));
759
760	# ifdef PGM_WITH_LARGE_PAGES
761	/*
762	* Try allocate a large page if applicable.
763	*/
764	if ( PGMIsUsingLargePages(pVM)
765	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
766	{
767	RTGCPHYS GCPhysBase = GCPhys & X86_PDE2M_PAE_PG_MASK;
768	PPGMPAGE pBasePage;
769
770	int rc = pgmPhysGetPageEx(pVM, GCPhysBase, &pBasePage);
771	AssertRCReturn(rc, rc); /* paranoia; can't happen. */
772	if (PGM_PAGE_GET_PDE_TYPE(pBasePage) == PGM_PAGE_PDE_TYPE_DONTCARE)
773	{
774	rc = pgmPhysAllocLargePage(pVM, GCPhys);
775	if (rc == VINF_SUCCESS)
776	return rc;
777	}
778	/* Mark the base as type page table, so we don't check over and over again. */
779	PGM_PAGE_SET_PDE_TYPE(pVM, pBasePage, PGM_PAGE_PDE_TYPE_PT);
780
781	/* fall back to 4KB pages. */
782	}
783	# endif
784
785	/*
786	* Flush any shadow page table mappings of the page.
787	* When VBOX_WITH_NEW_LAZY_PAGE_ALLOC isn't defined, there shouldn't be any.
788	*/
789	bool fFlushTLBs = false;
790	int rc = pgmPoolTrackUpdateGCPhys(pVM, GCPhys, pPage, true /fFlushTLBs/, &fFlushTLBs);
791	AssertMsgReturn(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc), RT_FAILURE(rc) ? rc : VERR_IPE_UNEXPECTED_STATUS);
792
793	/*
794	* Ensure that we've got a page handy, take it and use it.
795	*/
796	int rc2 = pgmPhysEnsureHandyPage(pVM);
797	if (RT_FAILURE(rc2))
798	{
799	if (fFlushTLBs)
800	PGM_INVL_ALL_VCPU_TLBS(pVM);
801	Assert(rc2 == VERR_EM_NO_MEMORY);
802	return rc2;
803	}
804	/* re-assert preconditions since pgmPhysEnsureHandyPage may do a context switch. */
805	PGM_LOCK_ASSERT_OWNER(pVM);
806	AssertMsg(PGM_PAGE_IS_ZERO(pPage) \|\| PGM_PAGE_IS_SHARED(pPage), ("%R[pgmpage] %RGp\n", pPage, GCPhys));
807	Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage));
808
809	uint32_t iHandyPage = --pVM->pgm.s.cHandyPages;
810	AssertMsg(iHandyPage < RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d\n", iHandyPage));
811	Assert(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys != NIL_RTHCPHYS);
812	Assert(!(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys & ~X86_PTE_PAE_PG_MASK));
813	Assert(pVM->pgm.s.aHandyPages[iHandyPage].idPage != NIL_GMM_PAGEID);
814	Assert(pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage == NIL_GMM_PAGEID);
815
816	/*
817	* There are one or two action to be taken the next time we allocate handy pages:
818	* - Tell the GMM (global memory manager) what the page is being used for.
819	* (Speeds up replacement operations - sharing and defragmenting.)
820	* - If the current backing is shared, it must be freed.
821	*/
822	const RTHCPHYS HCPhys = pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys;
823	pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys = GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
824
825	void const *pvSharedPage = NULL;
826	if (PGM_PAGE_IS_SHARED(pPage))
827	{
828	/* Mark this shared page for freeing/dereferencing. */
829	pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage = PGM_PAGE_GET_PAGEID(pPage);
830	Assert(PGM_PAGE_GET_PAGEID(pPage) != NIL_GMM_PAGEID);
831
832	Log(("PGM: Replaced shared page %#x at %RGp with %#x / %RHp\n", PGM_PAGE_GET_PAGEID(pPage),
833	GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys));
834	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageReplaceShared));
835	pVM->pgm.s.cSharedPages--;
836
837	/* Grab the address of the page so we can make a copy later on. (safe) */
838	rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhys, &pvSharedPage);
839	AssertRC(rc);
840	}
841	else
842	{
843	Log2(("PGM: Replaced zero page %RGp with %#x / %RHp\n", GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys));
844	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatRZPageReplaceZero);
845	pVM->pgm.s.cZeroPages--;
846	}
847
848	/*
849	* Do the PGMPAGE modifications.
850	*/
851	pVM->pgm.s.cPrivatePages++;
852	PGM_PAGE_SET_HCPHYS(pVM, pPage, HCPhys);
853	PGM_PAGE_SET_PAGEID(pVM, pPage, pVM->pgm.s.aHandyPages[iHandyPage].idPage);
854	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
855	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PT);
856	pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
857
858	/* Copy the shared page contents to the replacement page. */
859	if (pvSharedPage)
860	{
861	/* Get the virtual address of the new page. */
862	PGMPAGEMAPLOCK PgMpLck;
863	void *pvNewPage;
864	rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvNewPage, &PgMpLck); AssertRC(rc);
865	if (RT_SUCCESS(rc))
866	{
867	memcpy(pvNewPage, pvSharedPage, PAGE_SIZE); /** @todo todo write ASMMemCopyPage */
868	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
869	}
870	}
871
872	if ( fFlushTLBs
873	&& rc != VINF_PGM_GCPHYS_ALIASED)
874	PGM_INVL_ALL_VCPU_TLBS(pVM);
875	return rc;
876	}
877
878	#ifdef PGM_WITH_LARGE_PAGES
879
880	/**
881	* Replace a 2 MB range of zero pages with new pages that we can write to.
882	*
883	* @returns The following VBox status codes.
884	* @retval VINF_SUCCESS on success, pPage is modified.
885	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
886	* @retval VERR_EM_NO_MEMORY if we're totally out of memory.
887	*
888	* @todo Propagate VERR_EM_NO_MEMORY up the call tree.
889	*
890	* @param pVM The cross context VM structure.
891	* @param GCPhys The address of the page.
892	*
893	* @remarks Must be called from within the PGM critical section. It may
894	* nip back to ring-3/0 in some cases.
895	*/
896	int pgmPhysAllocLargePage(PVM pVM, RTGCPHYS GCPhys)
897	{
898	RTGCPHYS GCPhysBase = GCPhys & X86_PDE2M_PAE_PG_MASK;
899	LogFlow(("pgmPhysAllocLargePage: %RGp base %RGp\n", GCPhys, GCPhysBase));
900
901	/*
902	* Prereqs.
903	*/
904	PGM_LOCK_ASSERT_OWNER(pVM);
905	Assert(PGMIsUsingLargePages(pVM));
906
907	PPGMPAGE pFirstPage;
908	int rc = pgmPhysGetPageEx(pVM, GCPhysBase, &pFirstPage);
909	if ( RT_SUCCESS(rc)
910	&& PGM_PAGE_GET_TYPE(pFirstPage) == PGMPAGETYPE_RAM)
911	{
912	unsigned uPDEType = PGM_PAGE_GET_PDE_TYPE(pFirstPage);
913
914	/* Don't call this function for already allocated pages. */
915	Assert(uPDEType != PGM_PAGE_PDE_TYPE_PDE);
916
917	if ( uPDEType == PGM_PAGE_PDE_TYPE_DONTCARE
918	&& PGM_PAGE_GET_STATE(pFirstPage) == PGM_PAGE_STATE_ZERO)
919	{
920	/* Lazy approach: check all pages in the 2 MB range.
921	* The whole range must be ram and unallocated. */
922	GCPhys = GCPhysBase;
923	unsigned iPage;
924	for (iPage = 0; iPage < _2M/PAGE_SIZE; iPage++)
925	{
926	PPGMPAGE pSubPage;
927	rc = pgmPhysGetPageEx(pVM, GCPhys, &pSubPage);
928	if ( RT_FAILURE(rc)
929	\|\| PGM_PAGE_GET_TYPE(pSubPage) != PGMPAGETYPE_RAM /* Anything other than ram implies monitoring. */
930	\|\| PGM_PAGE_GET_STATE(pSubPage) != PGM_PAGE_STATE_ZERO) /* Allocated, monitored or shared means we can't use a large page here */
931	{
932	LogFlow(("Found page %RGp with wrong attributes (type=%d; state=%d); cancel check. rc=%d\n", GCPhys, PGM_PAGE_GET_TYPE(pSubPage), PGM_PAGE_GET_STATE(pSubPage), rc));
933	break;
934	}
935	Assert(PGM_PAGE_GET_PDE_TYPE(pSubPage) == PGM_PAGE_PDE_TYPE_DONTCARE);
936	GCPhys += PAGE_SIZE;
937	}
938	if (iPage != _2M/PAGE_SIZE)
939	{
940	/* Failed. Mark as requiring a PT so we don't check the whole thing again in the future. */
941	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageRefused);
942	PGM_PAGE_SET_PDE_TYPE(pVM, pFirstPage, PGM_PAGE_PDE_TYPE_PT);
943	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
944	}
945
946	/*
947	* Do the allocation.
948	*/
949	# ifdef IN_RING3
950	rc = PGMR3PhysAllocateLargeHandyPage(pVM, GCPhysBase);
951	# else
952	rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_ALLOCATE_LARGE_HANDY_PAGE, GCPhysBase);
953	# endif
954	if (RT_SUCCESS(rc))
955	{
956	Assert(PGM_PAGE_GET_STATE(pFirstPage) == PGM_PAGE_STATE_ALLOCATED);
957	pVM->pgm.s.cLargePages++;
958	return VINF_SUCCESS;
959	}
960
961	/* If we fail once, it most likely means the host's memory is too
962	fragmented; don't bother trying again. */
963	LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
964	PGMSetLargePageUsage(pVM, false);
965	return rc;
966	}
967	}
968	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
969	}
970
971
972	/**
973	* Recheck the entire 2 MB range to see if we can use it again as a large page.
974	*
975	* @returns The following VBox status codes.
976	* @retval VINF_SUCCESS on success, the large page can be used again
977	* @retval VERR_PGM_INVALID_LARGE_PAGE_RANGE if it can't be reused
978	*
979	* @param pVM The cross context VM structure.
980	* @param GCPhys The address of the page.
981	* @param pLargePage Page structure of the base page
982	*/
983	int pgmPhysRecheckLargePage(PVM pVM, RTGCPHYS GCPhys, PPGMPAGE pLargePage)
984	{
985	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageRecheck);
986
987	GCPhys &= X86_PDE2M_PAE_PG_MASK;
988
989	/* Check the base page. */
990	Assert(PGM_PAGE_GET_PDE_TYPE(pLargePage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED);
991	if ( PGM_PAGE_GET_STATE(pLargePage) != PGM_PAGE_STATE_ALLOCATED
992	\|\| PGM_PAGE_GET_TYPE(pLargePage) != PGMPAGETYPE_RAM
993	\|\| PGM_PAGE_GET_HNDL_PHYS_STATE(pLargePage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
994	{
995	LogFlow(("pgmPhysRecheckLargePage: checks failed for base page %x %x %x\n", PGM_PAGE_GET_STATE(pLargePage), PGM_PAGE_GET_TYPE(pLargePage), PGM_PAGE_GET_HNDL_PHYS_STATE(pLargePage)));
996	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
997	}
998
999	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,IsValidLargePage), a);
1000	/* Check all remaining pages in the 2 MB range. */
1001	unsigned i;
1002	GCPhys += PAGE_SIZE;
1003	for (i = 1; i < _2M/PAGE_SIZE; i++)
1004	{
1005	PPGMPAGE pPage;
1006	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
1007	AssertRCBreak(rc);
1008
1009	if ( PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1010	\|\| PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE
1011	\|\| PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
1012	\|\| PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
1013	{
1014	LogFlow(("pgmPhysRecheckLargePage: checks failed for page %d; %x %x %x\n", i, PGM_PAGE_GET_STATE(pPage), PGM_PAGE_GET_TYPE(pPage), PGM_PAGE_GET_HNDL_PHYS_STATE(pPage)));
1015	break;
1016	}
1017
1018	GCPhys += PAGE_SIZE;
1019	}
1020	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,IsValidLargePage), a);
1021
1022	if (i == _2M/PAGE_SIZE)
1023	{
1024	PGM_PAGE_SET_PDE_TYPE(pVM, pLargePage, PGM_PAGE_PDE_TYPE_PDE);
1025	pVM->pgm.s.cLargePagesDisabled--;
1026	Log(("pgmPhysRecheckLargePage: page %RGp can be reused!\n", GCPhys - _2M));
1027	return VINF_SUCCESS;
1028	}
1029
1030	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
1031	}
1032
1033	#endif /* PGM_WITH_LARGE_PAGES */
1034
1035	/**
1036	* Deal with a write monitored page.
1037	*
1038	* @returns VBox strict status code.
1039	*
1040	* @param pVM The cross context VM structure.
1041	* @param pPage The physical page tracking structure.
1042	*
1043	* @remarks Called from within the PGM critical section.
1044	*/
1045	void pgmPhysPageMakeWriteMonitoredWritable(PVM pVM, PPGMPAGE pPage)
1046	{
1047	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED);
1048	PGM_PAGE_SET_WRITTEN_TO(pVM, pPage);
1049	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
1050	Assert(pVM->pgm.s.cMonitoredPages > 0);
1051	pVM->pgm.s.cMonitoredPages--;
1052	pVM->pgm.s.cWrittenToPages++;
1053	}
1054
1055
1056	/**
1057	* Deal with pages that are not writable, i.e. not in the ALLOCATED state.
1058	*
1059	* @returns VBox strict status code.
1060	* @retval VINF_SUCCESS on success.
1061	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
1062	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1063	*
1064	* @param pVM The cross context VM structure.
1065	* @param pPage The physical page tracking structure.
1066	* @param GCPhys The address of the page.
1067	*
1068	* @remarks Called from within the PGM critical section.
1069	*/
1070	int pgmPhysPageMakeWritable(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
1071	{
1072	PGM_LOCK_ASSERT_OWNER(pVM);
1073	switch (PGM_PAGE_GET_STATE(pPage))
1074	{
1075	case PGM_PAGE_STATE_WRITE_MONITORED:
1076	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
1077	/* fall thru */
1078	default: /* to shut up GCC */
1079	case PGM_PAGE_STATE_ALLOCATED:
1080	return VINF_SUCCESS;
1081
1082	/*
1083	* Zero pages can be dummy pages for MMIO or reserved memory,
1084	* so we need to check the flags before joining cause with
1085	* shared page replacement.
1086	*/
1087	case PGM_PAGE_STATE_ZERO:
1088	if (PGM_PAGE_IS_MMIO(pPage))
1089	return VERR_PGM_PHYS_PAGE_RESERVED;
1090	/* fall thru */
1091	case PGM_PAGE_STATE_SHARED:
1092	return pgmPhysAllocPage(pVM, pPage, GCPhys);
1093
1094	/* Not allowed to write to ballooned pages. */
1095	case PGM_PAGE_STATE_BALLOONED:
1096	return VERR_PGM_PHYS_PAGE_BALLOONED;
1097	}
1098	}
1099
1100
1101	/**
1102	* Internal usage: Map the page specified by its GMM ID.
1103	*
1104	* This is similar to pgmPhysPageMap
1105	*
1106	* @returns VBox status code.
1107	*
1108	* @param pVM The cross context VM structure.
1109	* @param idPage The Page ID.
1110	* @param HCPhys The physical address (for RC).
1111	* @param ppv Where to store the mapping address.
1112	*
1113	* @remarks Called from within the PGM critical section. The mapping is only
1114	* valid while you are inside this section.
1115	*/
1116	int pgmPhysPageMapByPageID(PVM pVM, uint32_t idPage, RTHCPHYS HCPhys, void **ppv)
1117	{
1118	/*
1119	* Validation.
1120	*/
1121	PGM_LOCK_ASSERT_OWNER(pVM);
1122	AssertReturn(HCPhys && !(HCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
1123	const uint32_t idChunk = idPage >> GMM_CHUNKID_SHIFT;
1124	AssertReturn(idChunk != NIL_GMM_CHUNKID, VERR_INVALID_PARAMETER);
1125
1126	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1127	/*
1128	* Map it by HCPhys.
1129	*/
1130	return pgmRZDynMapHCPageInlined(VMMGetCpu(pVM), HCPhys, ppv RTLOG_COMMA_SRC_POS);
1131
1132	#else
1133	/*
1134	* Find/make Chunk TLB entry for the mapping chunk.
1135	*/
1136	PPGMCHUNKR3MAP pMap;
1137	PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)];
1138	if (pTlbe->idChunk == idChunk)
1139	{
1140	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,ChunkR3MapTlbHits));
1141	pMap = pTlbe->pChunk;
1142	}
1143	else
1144	{
1145	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,ChunkR3MapTlbMisses));
1146
1147	/*
1148	* Find the chunk, map it if necessary.
1149	*/
1150	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
1151	if (pMap)
1152	pMap->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
1153	else
1154	{
1155	# ifdef IN_RING0
1156	int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_MAP_CHUNK, idChunk);
1157	AssertRCReturn(rc, rc);
1158	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
1159	Assert(pMap);
1160	# else
1161	int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap);
1162	if (RT_FAILURE(rc))
1163	return rc;
1164	# endif
1165	}
1166
1167	/*
1168	* Enter it into the Chunk TLB.
1169	*/
1170	pTlbe->idChunk = idChunk;
1171	pTlbe->pChunk = pMap;
1172	}
1173
1174	ppv = (uint8_t )pMap->pv + ((idPage &GMM_PAGEID_IDX_MASK) << PAGE_SHIFT);
1175	return VINF_SUCCESS;
1176	#endif
1177	}
1178
1179
1180	/**
1181	* Maps a page into the current virtual address space so it can be accessed.
1182	*
1183	* @returns VBox status code.
1184	* @retval VINF_SUCCESS on success.
1185	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1186	*
1187	* @param pVM The cross context VM structure.
1188	* @param pPage The physical page tracking structure.
1189	* @param GCPhys The address of the page.
1190	* @param ppMap Where to store the address of the mapping tracking structure.
1191	* @param ppv Where to store the mapping address of the page. The page
1192	* offset is masked off!
1193	*
1194	* @remarks Called from within the PGM critical section.
1195	*/
1196	static int pgmPhysPageMapCommon(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAP ppMap, void **ppv)
1197	{
1198	PGM_LOCK_ASSERT_OWNER(pVM);
1199	NOREF(GCPhys);
1200
1201	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1202	/*
1203	* Just some sketchy GC/R0-darwin code.
1204	*/
1205	*ppMap = NULL;
1206	RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
1207	Assert(HCPhys != pVM->pgm.s.HCPhysZeroPg);
1208	pgmRZDynMapHCPageInlined(VMMGetCpu(pVM), HCPhys, ppv RTLOG_COMMA_SRC_POS);
1209	return VINF_SUCCESS;
1210
1211	#else /* IN_RING3 \|\| IN_RING0 */
1212
1213
1214	/*
1215	* Special cases: MMIO2, ZERO and specially aliased MMIO pages.
1216	*/
1217	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2
1218	\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO)
1219	{
1220	/* Decode the page id to a page in a MMIO2 ram range. */
1221	uint8_t idMmio2 = PGM_MMIO2_PAGEID_GET_MMIO2_ID(PGM_PAGE_GET_PAGEID(pPage));
1222	uint32_t iPage = PGM_MMIO2_PAGEID_GET_IDX(PGM_PAGE_GET_PAGEID(pPage));
1223	AssertLogRelMsgReturn((uint8_t)(idMmio2 - 1U) < RT_ELEMENTS(pVM->pgm.s.CTX_SUFF(apMmio2Ranges)),
1224	("idMmio2=%u size=%u type=%u GCPHys=%#RGp Id=%u State=%u", idMmio2,
1225	RT_ELEMENTS(pVM->pgm.s.CTX_SUFF(apMmio2Ranges)), PGM_PAGE_GET_TYPE(pPage), GCPhys,
1226	pPage->s.idPage, pPage->s.uStateY),
1227	VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
1228	PPGMMMIO2RANGE pMmio2Range = pVM->pgm.s.CTX_SUFF(apMmio2Ranges)[idMmio2 - 1];
1229	AssertLogRelReturn(pMmio2Range, VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
1230	AssertLogRelReturn(pMmio2Range->idMmio2 == idMmio2, VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
1231	AssertLogRelReturn(iPage < (pMmio2Range->RamRange.cb >> PAGE_SHIFT), VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
1232	ppv = (uint8_t )pMmio2Range->RamRange.pvR3 + ((uintptr_t)iPage << PAGE_SHIFT);
1233	*ppMap = NULL;
1234	return VINF_SUCCESS;
1235	}
1236
1237	const uint32_t idChunk = PGM_PAGE_GET_CHUNKID(pPage);
1238	if (idChunk == NIL_GMM_CHUNKID)
1239	{
1240	AssertMsgReturn(PGM_PAGE_GET_PAGEID(pPage) == NIL_GMM_PAGEID, ("pPage=%R[pgmpage]\n", pPage),
1241	VERR_PGM_PHYS_PAGE_MAP_IPE_1);
1242	if (!PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
1243	{
1244	AssertMsgReturn(PGM_PAGE_IS_ZERO(pPage), ("pPage=%R[pgmpage]\n", pPage),
1245	VERR_PGM_PHYS_PAGE_MAP_IPE_3);
1246	AssertMsgReturn(PGM_PAGE_GET_HCPHYS(pPage)== pVM->pgm.s.HCPhysZeroPg, ("pPage=%R[pgmpage]\n", pPage),
1247	VERR_PGM_PHYS_PAGE_MAP_IPE_4);
1248	*ppv = pVM->pgm.s.CTXALLSUFF(pvZeroPg);
1249	}
1250	else
1251	# ifdef VBOX_WITH_2ND_IEM_STEP
1252	*ppv = pVM->pgm.s.CTXALLSUFF(pvZeroPg);
1253	# else
1254	{
1255	/* This kind of screws up the TLB entry if accessed from a different section afterwards. */
1256	static uint8_t s_abPlayItSafe[0x10002]; / I don't dare return the zero page at the moment. */
1257	ppv = (uint8_t )((uintptr_t)&s_abPlayItSafe[0x1000] & ~(uintptr_t)0xfff);
1258	}
1259	# endif
1260	*ppMap = NULL;
1261	return VINF_SUCCESS;
1262	}
1263
1264	/*
1265	* Find/make Chunk TLB entry for the mapping chunk.
1266	*/
1267	PPGMCHUNKR3MAP pMap;
1268	PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)];
1269	if (pTlbe->idChunk == idChunk)
1270	{
1271	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,ChunkR3MapTlbHits));
1272	pMap = pTlbe->pChunk;
1273	AssertPtr(pMap->pv);
1274	}
1275	else
1276	{
1277	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,ChunkR3MapTlbMisses));
1278
1279	/*
1280	* Find the chunk, map it if necessary.
1281	*/
1282	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
1283	if (pMap)
1284	{
1285	AssertPtr(pMap->pv);
1286	pMap->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
1287	}
1288	else
1289	{
1290	#ifdef IN_RING0
1291	int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_MAP_CHUNK, idChunk);
1292	AssertRCReturn(rc, rc);
1293	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
1294	Assert(pMap);
1295	#else
1296	int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap);
1297	if (RT_FAILURE(rc))
1298	return rc;
1299	#endif
1300	AssertPtr(pMap->pv);
1301	}
1302
1303	/*
1304	* Enter it into the Chunk TLB.
1305	*/
1306	pTlbe->idChunk = idChunk;
1307	pTlbe->pChunk = pMap;
1308	}
1309
1310	ppv = (uint8_t )pMap->pv + (PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) << PAGE_SHIFT);
1311	*ppMap = pMap;
1312	return VINF_SUCCESS;
1313	#endif /* IN_RING3 */
1314	}
1315
1316
1317	/**
1318	* Combination of pgmPhysPageMakeWritable and pgmPhysPageMapWritable.
1319	*
1320	* This is typically used is paths where we cannot use the TLB methods (like ROM
1321	* pages) or where there is no point in using them since we won't get many hits.
1322	*
1323	* @returns VBox strict status code.
1324	* @retval VINF_SUCCESS on success.
1325	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
1326	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1327	*
1328	* @param pVM The cross context VM structure.
1329	* @param pPage The physical page tracking structure.
1330	* @param GCPhys The address of the page.
1331	* @param ppv Where to store the mapping address of the page. The page
1332	* offset is masked off!
1333	*
1334	* @remarks Called from within the PGM critical section. The mapping is only
1335	* valid while you are inside section.
1336	*/
1337	int pgmPhysPageMakeWritableAndMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
1338	{
1339	int rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1340	if (RT_SUCCESS(rc))
1341	{
1342	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* returned */, ("%Rrc\n", rc));
1343	PPGMPAGEMAP pMapIgnore;
1344	int rc2 = pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, ppv);
1345	if (RT_FAILURE(rc2)) /* preserve rc */
1346	rc = rc2;
1347	}
1348	return rc;
1349	}
1350
1351
1352	/**
1353	* Maps a page into the current virtual address space so it can be accessed for
1354	* both writing and reading.
1355	*
1356	* This is typically used is paths where we cannot use the TLB methods (like ROM
1357	* pages) or where there is no point in using them since we won't get many hits.
1358	*
1359	* @returns VBox status code.
1360	* @retval VINF_SUCCESS on success.
1361	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1362	*
1363	* @param pVM The cross context VM structure.
1364	* @param pPage The physical page tracking structure. Must be in the
1365	* allocated state.
1366	* @param GCPhys The address of the page.
1367	* @param ppv Where to store the mapping address of the page. The page
1368	* offset is masked off!
1369	*
1370	* @remarks Called from within the PGM critical section. The mapping is only
1371	* valid while you are inside section.
1372	*/
1373	int pgmPhysPageMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
1374	{
1375	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
1376	PPGMPAGEMAP pMapIgnore;
1377	return pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, ppv);
1378	}
1379
1380
1381	/**
1382	* Maps a page into the current virtual address space so it can be accessed for
1383	* reading.
1384	*
1385	* This is typically used is paths where we cannot use the TLB methods (like ROM
1386	* pages) or where there is no point in using them since we won't get many hits.
1387	*
1388	* @returns VBox status code.
1389	* @retval VINF_SUCCESS on success.
1390	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1391	*
1392	* @param pVM The cross context VM structure.
1393	* @param pPage The physical page tracking structure.
1394	* @param GCPhys The address of the page.
1395	* @param ppv Where to store the mapping address of the page. The page
1396	* offset is masked off!
1397	*
1398	* @remarks Called from within the PGM critical section. The mapping is only
1399	* valid while you are inside this section.
1400	*/
1401	int pgmPhysPageMapReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const **ppv)
1402	{
1403	PPGMPAGEMAP pMapIgnore;
1404	return pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, (void **)ppv);
1405	}
1406
1407	#if !defined(IN_RC) && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1408
1409	/**
1410	* Load a guest page into the ring-3 physical TLB.
1411	*
1412	* @returns VBox status code.
1413	* @retval VINF_SUCCESS on success
1414	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1415	* @param pPGM The PGM instance pointer.
1416	* @param GCPhys The guest physical address in question.
1417	*/
1418	int pgmPhysPageLoadIntoTlb(PVM pVM, RTGCPHYS GCPhys)
1419	{
1420	PGM_LOCK_ASSERT_OWNER(pVM);
1421
1422	/*
1423	* Find the ram range and page and hand it over to the with-page function.
1424	* 99.8% of requests are expected to be in the first range.
1425	*/
1426	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
1427	if (!pPage)
1428	{
1429	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbMisses));
1430	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
1431	}
1432
1433	return pgmPhysPageLoadIntoTlbWithPage(pVM, pPage, GCPhys);
1434	}
1435
1436
1437	/**
1438	* Load a guest page into the ring-3 physical TLB.
1439	*
1440	* @returns VBox status code.
1441	* @retval VINF_SUCCESS on success
1442	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1443	*
1444	* @param pVM The cross context VM structure.
1445	* @param pPage Pointer to the PGMPAGE structure corresponding to
1446	* GCPhys.
1447	* @param GCPhys The guest physical address in question.
1448	*/
1449	int pgmPhysPageLoadIntoTlbWithPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
1450	{
1451	PGM_LOCK_ASSERT_OWNER(pVM);
1452	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbMisses));
1453
1454	/*
1455	* Map the page.
1456	* Make a special case for the zero page as it is kind of special.
1457	*/
1458	PPGMPAGEMAPTLBE pTlbe = &pVM->pgm.s.CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
1459	if ( !PGM_PAGE_IS_ZERO(pPage)
1460	&& !PGM_PAGE_IS_BALLOONED(pPage))
1461	{
1462	void *pv;
1463	PPGMPAGEMAP pMap;
1464	int rc = pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMap, &pv);
1465	if (RT_FAILURE(rc))
1466	return rc;
1467	pTlbe->pMap = pMap;
1468	pTlbe->pv = pv;
1469	Assert(!((uintptr_t)pTlbe->pv & PAGE_OFFSET_MASK));
1470	}
1471	else
1472	{
1473	AssertMsg(PGM_PAGE_GET_HCPHYS(pPage) == pVM->pgm.s.HCPhysZeroPg, ("%RGp/%R[pgmpage]\n", GCPhys, pPage));
1474	pTlbe->pMap = NULL;
1475	pTlbe->pv = pVM->pgm.s.CTXALLSUFF(pvZeroPg);
1476	}
1477	#ifdef PGM_WITH_PHYS_TLB
1478	if ( PGM_PAGE_GET_TYPE(pPage) < PGMPAGETYPE_ROM_SHADOW
1479	\|\| PGM_PAGE_GET_TYPE(pPage) > PGMPAGETYPE_ROM)
1480	pTlbe->GCPhys = GCPhys & X86_PTE_PAE_PG_MASK;
1481	else
1482	pTlbe->GCPhys = NIL_RTGCPHYS; /* ROM: Problematic because of the two pages. :-/ */
1483	#else
1484	pTlbe->GCPhys = NIL_RTGCPHYS;
1485	#endif
1486	pTlbe->pPage = pPage;
1487	return VINF_SUCCESS;
1488	}
1489
1490	#endif /* !IN_RC && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
1491
1492	/**
1493	* Internal version of PGMPhysGCPhys2CCPtr that expects the caller to
1494	* own the PGM lock and therefore not need to lock the mapped page.
1495	*
1496	* @returns VBox status code.
1497	* @retval VINF_SUCCESS on success.
1498	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1499	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1500	*
1501	* @param pVM The cross context VM structure.
1502	* @param GCPhys The guest physical address of the page that should be mapped.
1503	* @param pPage Pointer to the PGMPAGE structure for the page.
1504	* @param ppv Where to store the address corresponding to GCPhys.
1505	*
1506	* @internal
1507	* @deprecated Use pgmPhysGCPhys2CCPtrInternalEx.
1508	*/
1509	int pgmPhysGCPhys2CCPtrInternalDepr(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
1510	{
1511	int rc;
1512	AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM);
1513	PGM_LOCK_ASSERT_OWNER(pVM);
1514	pVM->pgm.s.cDeprecatedPageLocks++;
1515
1516	/*
1517	* Make sure the page is writable.
1518	*/
1519	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
1520	{
1521	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1522	if (RT_FAILURE(rc))
1523	return rc;
1524	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
1525	}
1526	Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0);
1527
1528	/*
1529	* Get the mapping address.
1530	*/
1531	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1532	void *pv;
1533	rc = pgmRZDynMapHCPageInlined(VMMGetCpu(pVM),
1534	PGM_PAGE_GET_HCPHYS(pPage),
1535	&pv
1536	RTLOG_COMMA_SRC_POS);
1537	if (RT_FAILURE(rc))
1538	return rc;
1539	ppv = (void )((uintptr_t)pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1540	#else
1541	PPGMPAGEMAPTLBE pTlbe;
1542	rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
1543	if (RT_FAILURE(rc))
1544	return rc;
1545	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1546	#endif
1547	return VINF_SUCCESS;
1548	}
1549
1550	#if !defined(IN_RC) && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1551
1552	/**
1553	* Locks a page mapping for writing.
1554	*
1555	* @param pVM The cross context VM structure.
1556	* @param pPage The page.
1557	* @param pTlbe The mapping TLB entry for the page.
1558	* @param pLock The lock structure (output).
1559	*/
1560	DECLINLINE(void) pgmPhysPageMapLockForWriting(PVM pVM, PPGMPAGE pPage, PPGMPAGEMAPTLBE pTlbe, PPGMPAGEMAPLOCK pLock)
1561	{
1562	PPGMPAGEMAP pMap = pTlbe->pMap;
1563	if (pMap)
1564	pMap->cRefs++;
1565
1566	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
1567	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
1568	{
1569	if (cLocks == 0)
1570	pVM->pgm.s.cWriteLockedPages++;
1571	PGM_PAGE_INC_WRITE_LOCKS(pPage);
1572	}
1573	else if (cLocks != PGM_PAGE_MAX_LOCKS)
1574	{
1575	PGM_PAGE_INC_WRITE_LOCKS(pPage);
1576	AssertMsgFailed(("%R[pgmpage] is entering permanent write locked state!\n", pPage));
1577	if (pMap)
1578	pMap->cRefs++; /* Extra ref to prevent it from going away. */
1579	}
1580
1581	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_WRITE;
1582	pLock->pvMap = pMap;
1583	}
1584
1585	/**
1586	* Locks a page mapping for reading.
1587	*
1588	* @param pVM The cross context VM structure.
1589	* @param pPage The page.
1590	* @param pTlbe The mapping TLB entry for the page.
1591	* @param pLock The lock structure (output).
1592	*/
1593	DECLINLINE(void) pgmPhysPageMapLockForReading(PVM pVM, PPGMPAGE pPage, PPGMPAGEMAPTLBE pTlbe, PPGMPAGEMAPLOCK pLock)
1594	{
1595	PPGMPAGEMAP pMap = pTlbe->pMap;
1596	if (pMap)
1597	pMap->cRefs++;
1598
1599	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
1600	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
1601	{
1602	if (cLocks == 0)
1603	pVM->pgm.s.cReadLockedPages++;
1604	PGM_PAGE_INC_READ_LOCKS(pPage);
1605	}
1606	else if (cLocks != PGM_PAGE_MAX_LOCKS)
1607	{
1608	PGM_PAGE_INC_READ_LOCKS(pPage);
1609	AssertMsgFailed(("%R[pgmpage] is entering permanent read locked state!\n", pPage));
1610	if (pMap)
1611	pMap->cRefs++; /* Extra ref to prevent it from going away. */
1612	}
1613
1614	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_READ;
1615	pLock->pvMap = pMap;
1616	}
1617
1618	#endif /* !IN_RC && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
1619
1620
1621	/**
1622	* Internal version of PGMPhysGCPhys2CCPtr that expects the caller to
1623	* own the PGM lock and have access to the page structure.
1624	*
1625	* @returns VBox status code.
1626	* @retval VINF_SUCCESS on success.
1627	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1628	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1629	*
1630	* @param pVM The cross context VM structure.
1631	* @param GCPhys The guest physical address of the page that should be mapped.
1632	* @param pPage Pointer to the PGMPAGE structure for the page.
1633	* @param ppv Where to store the address corresponding to GCPhys.
1634	* @param pLock Where to store the lock information that
1635	* pgmPhysReleaseInternalPageMappingLock needs.
1636	*
1637	* @internal
1638	*/
1639	int pgmPhysGCPhys2CCPtrInternal(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
1640	{
1641	int rc;
1642	AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM);
1643	PGM_LOCK_ASSERT_OWNER(pVM);
1644
1645	/*
1646	* Make sure the page is writable.
1647	*/
1648	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
1649	{
1650	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1651	if (RT_FAILURE(rc))
1652	return rc;
1653	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
1654	}
1655	Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0);
1656
1657	/*
1658	* Do the job.
1659	*/
1660	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1661	void *pv;
1662	PVMCPU pVCpu = VMMGetCpu(pVM);
1663	rc = pgmRZDynMapHCPageInlined(pVCpu,
1664	PGM_PAGE_GET_HCPHYS(pPage),
1665	&pv
1666	RTLOG_COMMA_SRC_POS);
1667	if (RT_FAILURE(rc))
1668	return rc;
1669	ppv = (void )((uintptr_t)pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1670	pLock->pvPage = pv;
1671	pLock->pVCpu = pVCpu;
1672
1673	#else
1674	PPGMPAGEMAPTLBE pTlbe;
1675	rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
1676	if (RT_FAILURE(rc))
1677	return rc;
1678	pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
1679	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1680	#endif
1681	return VINF_SUCCESS;
1682	}
1683
1684
1685	/**
1686	* Internal version of PGMPhysGCPhys2CCPtrReadOnly that expects the caller to
1687	* own the PGM lock and have access to the page structure.
1688	*
1689	* @returns VBox status code.
1690	* @retval VINF_SUCCESS on success.
1691	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1692	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1693	*
1694	* @param pVM The cross context VM structure.
1695	* @param GCPhys The guest physical address of the page that should be mapped.
1696	* @param pPage Pointer to the PGMPAGE structure for the page.
1697	* @param ppv Where to store the address corresponding to GCPhys.
1698	* @param pLock Where to store the lock information that
1699	* pgmPhysReleaseInternalPageMappingLock needs.
1700	*
1701	* @internal
1702	*/
1703	int pgmPhysGCPhys2CCPtrInternalReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, const void **ppv, PPGMPAGEMAPLOCK pLock)
1704	{
1705	AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM);
1706	PGM_LOCK_ASSERT_OWNER(pVM);
1707	Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0);
1708
1709	/*
1710	* Do the job.
1711	*/
1712	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1713	void *pv;
1714	PVMCPU pVCpu = VMMGetCpu(pVM);
1715	int rc = pgmRZDynMapHCPageInlined(pVCpu,
1716	PGM_PAGE_GET_HCPHYS(pPage),
1717	&pv
1718	RTLOG_COMMA_SRC_POS); /** @todo add a read only flag? */
1719	if (RT_FAILURE(rc))
1720	return rc;
1721	ppv = (void )((uintptr_t)pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1722	pLock->pvPage = pv;
1723	pLock->pVCpu = pVCpu;
1724
1725	#else
1726	PPGMPAGEMAPTLBE pTlbe;
1727	int rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
1728	if (RT_FAILURE(rc))
1729	return rc;
1730	pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
1731	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1732	#endif
1733	return VINF_SUCCESS;
1734	}
1735
1736
1737	/**
1738	* Requests the mapping of a guest page into the current context.
1739	*
1740	* This API should only be used for very short term, as it will consume scarse
1741	* resources (R0 and GC) in the mapping cache. When you're done with the page,
1742	* call PGMPhysReleasePageMappingLock() ASAP to release it.
1743	*
1744	* This API will assume your intention is to write to the page, and will
1745	* therefore replace shared and zero pages. If you do not intend to modify
1746	* the page, use the PGMPhysGCPhys2CCPtrReadOnly() API.
1747	*
1748	* @returns VBox status code.
1749	* @retval VINF_SUCCESS on success.
1750	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1751	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1752	*
1753	* @param pVM The cross context VM structure.
1754	* @param GCPhys The guest physical address of the page that should be
1755	* mapped.
1756	* @param ppv Where to store the address corresponding to GCPhys.
1757	* @param pLock Where to store the lock information that
1758	* PGMPhysReleasePageMappingLock needs.
1759	*
1760	* @remarks The caller is responsible for dealing with access handlers.
1761	* @todo Add an informational return code for pages with access handlers?
1762	*
1763	* @remark Avoid calling this API from within critical sections (other than
1764	* the PGM one) because of the deadlock risk. External threads may
1765	* need to delegate jobs to the EMTs.
1766	* @remarks Only one page is mapped! Make no assumption about what's after or
1767	* before the returned page!
1768	* @thread Any thread.
1769	*/
1770	VMM_INT_DECL(int) PGMPhysGCPhys2CCPtr(PVM pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
1771	{
1772	int rc = pgmLock(pVM);
1773	AssertRCReturn(rc, rc);
1774
1775	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1776	/*
1777	* Find the page and make sure it's writable.
1778	*/
1779	PPGMPAGE pPage;
1780	rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
1781	if (RT_SUCCESS(rc))
1782	{
1783	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
1784	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1785	if (RT_SUCCESS(rc))
1786	{
1787	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
1788
1789	PVMCPU pVCpu = VMMGetCpu(pVM);
1790	void *pv;
1791	rc = pgmRZDynMapHCPageInlined(pVCpu,
1792	PGM_PAGE_GET_HCPHYS(pPage),
1793	&pv
1794	RTLOG_COMMA_SRC_POS);
1795	if (RT_SUCCESS(rc))
1796	{
1797	AssertRCSuccess(rc);
1798
1799	pv = (void *)((uintptr_t)pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1800	*ppv = pv;
1801	pLock->pvPage = pv;
1802	pLock->pVCpu = pVCpu;
1803	}
1804	}
1805	}
1806
1807	#else /* IN_RING3 \|\| IN_RING0 */
1808	/*
1809	* Query the Physical TLB entry for the page (may fail).
1810	*/
1811	PPGMPAGEMAPTLBE pTlbe;
1812	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
1813	if (RT_SUCCESS(rc))
1814	{
1815	/*
1816	* If the page is shared, the zero page, or being write monitored
1817	* it must be converted to a page that's writable if possible.
1818	*/
1819	PPGMPAGE pPage = pTlbe->pPage;
1820	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
1821	{
1822	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1823	if (RT_SUCCESS(rc))
1824	{
1825	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
1826	rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
1827	}
1828	}
1829	if (RT_SUCCESS(rc))
1830	{
1831	/*
1832	* Now, just perform the locking and calculate the return address.
1833	*/
1834	pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
1835	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1836	}
1837	}
1838
1839	#endif /* IN_RING3 \|\| IN_RING0 */
1840	pgmUnlock(pVM);
1841	return rc;
1842	}
1843
1844
1845	/**
1846	* Requests the mapping of a guest page into the current context.
1847	*
1848	* This API should only be used for very short term, as it will consume scarse
1849	* resources (R0 and GC) in the mapping cache. When you're done with the page,
1850	* call PGMPhysReleasePageMappingLock() ASAP to release it.
1851	*
1852	* @returns VBox status code.
1853	* @retval VINF_SUCCESS on success.
1854	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1855	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1856	*
1857	* @param pVM The cross context VM structure.
1858	* @param GCPhys The guest physical address of the page that should be
1859	* mapped.
1860	* @param ppv Where to store the address corresponding to GCPhys.
1861	* @param pLock Where to store the lock information that
1862	* PGMPhysReleasePageMappingLock needs.
1863	*
1864	* @remarks The caller is responsible for dealing with access handlers.
1865	* @todo Add an informational return code for pages with access handlers?
1866	*
1867	* @remarks Avoid calling this API from within critical sections (other than
1868	* the PGM one) because of the deadlock risk.
1869	* @remarks Only one page is mapped! Make no assumption about what's after or
1870	* before the returned page!
1871	* @thread Any thread.
1872	*/
1873	VMM_INT_DECL(int) PGMPhysGCPhys2CCPtrReadOnly(PVM pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
1874	{
1875	int rc = pgmLock(pVM);
1876	AssertRCReturn(rc, rc);
1877
1878	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1879	/*
1880	* Find the page and make sure it's readable.
1881	*/
1882	PPGMPAGE pPage;
1883	rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
1884	if (RT_SUCCESS(rc))
1885	{
1886	if (RT_UNLIKELY(PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage)))
1887	rc = VERR_PGM_PHYS_PAGE_RESERVED;
1888	else
1889	{
1890	PVMCPU pVCpu = VMMGetCpu(pVM);
1891	void *pv;
1892	rc = pgmRZDynMapHCPageInlined(pVCpu,
1893	PGM_PAGE_GET_HCPHYS(pPage),
1894	&pv
1895	RTLOG_COMMA_SRC_POS); /** @todo add a read only flag? */
1896	if (RT_SUCCESS(rc))
1897	{
1898	AssertRCSuccess(rc);
1899
1900	pv = (void *)((uintptr_t)pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1901	*ppv = pv;
1902	pLock->pvPage = pv;
1903	pLock->pVCpu = pVCpu;
1904	}
1905	}
1906	}
1907
1908	#else /* IN_RING3 \|\| IN_RING0 */
1909	/*
1910	* Query the Physical TLB entry for the page (may fail).
1911	*/
1912	PPGMPAGEMAPTLBE pTlbe;
1913	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
1914	if (RT_SUCCESS(rc))
1915	{
1916	/* MMIO pages doesn't have any readable backing. */
1917	PPGMPAGE pPage = pTlbe->pPage;
1918	if (RT_UNLIKELY(PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage)))
1919	rc = VERR_PGM_PHYS_PAGE_RESERVED;
1920	else
1921	{
1922	/*
1923	* Now, just perform the locking and calculate the return address.
1924	*/
1925	pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
1926	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1927	}
1928	}
1929
1930	#endif /* IN_RING3 \|\| IN_RING0 */
1931	pgmUnlock(pVM);
1932	return rc;
1933	}
1934
1935
1936	/**
1937	* Requests the mapping of a guest page given by virtual address into the current context.
1938	*
1939	* This API should only be used for very short term, as it will consume
1940	* scarse resources (R0 and GC) in the mapping cache. When you're done
1941	* with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
1942	*
1943	* This API will assume your intention is to write to the page, and will
1944	* therefore replace shared and zero pages. If you do not intend to modify
1945	* the page, use the PGMPhysGCPtr2CCPtrReadOnly() API.
1946	*
1947	* @returns VBox status code.
1948	* @retval VINF_SUCCESS on success.
1949	* @retval VERR_PAGE_TABLE_NOT_PRESENT if the page directory for the virtual address isn't present.
1950	* @retval VERR_PAGE_NOT_PRESENT if the page at the virtual address isn't present.
1951	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1952	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1953	*
1954	* @param pVCpu The cross context virtual CPU structure.
1955	* @param GCPtr The guest physical address of the page that should be
1956	* mapped.
1957	* @param ppv Where to store the address corresponding to GCPhys.
1958	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
1959	*
1960	* @remark Avoid calling this API from within critical sections (other than
1961	* the PGM one) because of the deadlock risk.
1962	* @thread EMT
1963	*/
1964	VMM_INT_DECL(int) PGMPhysGCPtr2CCPtr(PVMCPU pVCpu, RTGCPTR GCPtr, void **ppv, PPGMPAGEMAPLOCK pLock)
1965	{
1966	VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
1967	RTGCPHYS GCPhys;
1968	int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, &GCPhys);
1969	if (RT_SUCCESS(rc))
1970	rc = PGMPhysGCPhys2CCPtr(pVCpu->CTX_SUFF(pVM), GCPhys, ppv, pLock);
1971	return rc;
1972	}
1973
1974
1975	/**
1976	* Requests the mapping of a guest page given by virtual address into the current context.
1977	*
1978	* This API should only be used for very short term, as it will consume
1979	* scarse resources (R0 and GC) in the mapping cache. When you're done
1980	* with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
1981	*
1982	* @returns VBox status code.
1983	* @retval VINF_SUCCESS on success.
1984	* @retval VERR_PAGE_TABLE_NOT_PRESENT if the page directory for the virtual address isn't present.
1985	* @retval VERR_PAGE_NOT_PRESENT if the page at the virtual address isn't present.
1986	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1987	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1988	*
1989	* @param pVCpu The cross context virtual CPU structure.
1990	* @param GCPtr The guest physical address of the page that should be
1991	* mapped.
1992	* @param ppv Where to store the address corresponding to GCPtr.
1993	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
1994	*
1995	* @remark Avoid calling this API from within critical sections (other than
1996	* the PGM one) because of the deadlock risk.
1997	* @thread EMT
1998	*/
1999	VMM_INT_DECL(int) PGMPhysGCPtr2CCPtrReadOnly(PVMCPU pVCpu, RTGCPTR GCPtr, void const **ppv, PPGMPAGEMAPLOCK pLock)
2000	{
2001	VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
2002	RTGCPHYS GCPhys;
2003	int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, &GCPhys);
2004	if (RT_SUCCESS(rc))
2005	rc = PGMPhysGCPhys2CCPtrReadOnly(pVCpu->CTX_SUFF(pVM), GCPhys, ppv, pLock);
2006	return rc;
2007	}
2008
2009
2010	/**
2011	* Release the mapping of a guest page.
2012	*
2013	* This is the counter part of PGMPhysGCPhys2CCPtr, PGMPhysGCPhys2CCPtrReadOnly
2014	* PGMPhysGCPtr2CCPtr and PGMPhysGCPtr2CCPtrReadOnly.
2015	*
2016	* @param pVM The cross context VM structure.
2017	* @param pLock The lock structure initialized by the mapping function.
2018	*/
2019	VMMDECL(void) PGMPhysReleasePageMappingLock(PVM pVM, PPGMPAGEMAPLOCK pLock)
2020	{
2021	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
2022	Assert(pLock->pvPage != NULL);
2023	Assert(pLock->pVCpu == VMMGetCpu(pVM));
2024	PGM_DYNMAP_UNUSED_HINT(pLock->pVCpu, pLock->pvPage);
2025	pLock->pVCpu = NULL;
2026	pLock->pvPage = NULL;
2027
2028	#else
2029	PPGMPAGEMAP pMap = (PPGMPAGEMAP)pLock->pvMap;
2030	PPGMPAGE pPage = (PPGMPAGE)(pLock->uPageAndType & ~PGMPAGEMAPLOCK_TYPE_MASK);
2031	bool fWriteLock = (pLock->uPageAndType & PGMPAGEMAPLOCK_TYPE_MASK) == PGMPAGEMAPLOCK_TYPE_WRITE;
2032
2033	pLock->uPageAndType = 0;
2034	pLock->pvMap = NULL;
2035
2036	pgmLock(pVM);
2037	if (fWriteLock)
2038	{
2039	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
2040	Assert(cLocks > 0);
2041	if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
2042	{
2043	if (cLocks == 1)
2044	{
2045	Assert(pVM->pgm.s.cWriteLockedPages > 0);
2046	pVM->pgm.s.cWriteLockedPages--;
2047	}
2048	PGM_PAGE_DEC_WRITE_LOCKS(pPage);
2049	}
2050
2051	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
2052	{
2053	PGM_PAGE_SET_WRITTEN_TO(pVM, pPage);
2054	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
2055	Assert(pVM->pgm.s.cMonitoredPages > 0);
2056	pVM->pgm.s.cMonitoredPages--;
2057	pVM->pgm.s.cWrittenToPages++;
2058	}
2059	}
2060	else
2061	{
2062	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
2063	Assert(cLocks > 0);
2064	if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
2065	{
2066	if (cLocks == 1)
2067	{
2068	Assert(pVM->pgm.s.cReadLockedPages > 0);
2069	pVM->pgm.s.cReadLockedPages--;
2070	}
2071	PGM_PAGE_DEC_READ_LOCKS(pPage);
2072	}
2073	}
2074
2075	if (pMap)
2076	{
2077	Assert(pMap->cRefs >= 1);
2078	pMap->cRefs--;
2079	}
2080	pgmUnlock(pVM);
2081	#endif /* IN_RING3 */
2082	}
2083
2084
2085	/**
2086	* Release the internal mapping of a guest page.
2087	*
2088	* This is the counter part of pgmPhysGCPhys2CCPtrInternalEx and
2089	* pgmPhysGCPhys2CCPtrInternalReadOnly.
2090	*
2091	* @param pVM The cross context VM structure.
2092	* @param pLock The lock structure initialized by the mapping function.
2093	*
2094	* @remarks Caller must hold the PGM lock.
2095	*/
2096	void pgmPhysReleaseInternalPageMappingLock(PVM pVM, PPGMPAGEMAPLOCK pLock)
2097	{
2098	PGM_LOCK_ASSERT_OWNER(pVM);
2099	PGMPhysReleasePageMappingLock(pVM, pLock); /* lazy for now */
2100	}
2101
2102
2103	/**
2104	* Converts a GC physical address to a HC ring-3 pointer.
2105	*
2106	* @returns VINF_SUCCESS on success.
2107	* @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
2108	* page but has no physical backing.
2109	* @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
2110	* GC physical address.
2111	* @returns VERR_PGM_GCPHYS_RANGE_CROSSES_BOUNDARY if the range crosses
2112	* a dynamic ram chunk boundary
2113	*
2114	* @param pVM The cross context VM structure.
2115	* @param GCPhys The GC physical address to convert.
2116	* @param pR3Ptr Where to store the R3 pointer on success.
2117	*
2118	* @deprecated Avoid when possible!
2119	*/
2120	int pgmPhysGCPhys2R3Ptr(PVM pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr)
2121	{
2122	/** @todo this is kind of hacky and needs some more work. */
2123	#ifndef DEBUG_sandervl
2124	VM_ASSERT_EMT(pVM); /* no longer safe for use outside the EMT thread! */
2125	#endif
2126
2127	Log(("pgmPhysGCPhys2R3Ptr(,%RGp,): dont use this API!\n", GCPhys)); /** @todo eliminate this API! */
2128	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
2129	NOREF(pVM); NOREF(pR3Ptr);
2130	AssertFailedReturn(VERR_NOT_IMPLEMENTED);
2131	#else
2132	pgmLock(pVM);
2133
2134	PPGMRAMRANGE pRam;
2135	PPGMPAGE pPage;
2136	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
2137	if (RT_SUCCESS(rc))
2138	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)pR3Ptr);
2139
2140	pgmUnlock(pVM);
2141	Assert(rc <= VINF_SUCCESS);
2142	return rc;
2143	#endif
2144	}
2145
2146	#if 0 /defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)/
2147
2148	/**
2149	* Maps and locks a guest CR3 or PD (PAE) page.
2150	*
2151	* @returns VINF_SUCCESS on success.
2152	* @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
2153	* page but has no physical backing.
2154	* @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
2155	* GC physical address.
2156	* @returns VERR_PGM_GCPHYS_RANGE_CROSSES_BOUNDARY if the range crosses
2157	* a dynamic ram chunk boundary
2158	*
2159	* @param pVM The cross context VM structure.
2160	* @param GCPhys The GC physical address to convert.
2161	* @param pR3Ptr Where to store the R3 pointer on success. This may or
2162	* may not be valid in ring-0 depending on the
2163	* VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 build option.
2164	*
2165	* @remarks The caller must own the PGM lock.
2166	*/
2167	int pgmPhysCr3ToHCPtr(PVM pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr)
2168	{
2169
2170	PPGMRAMRANGE pRam;
2171	PPGMPAGE pPage;
2172	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
2173	if (RT_SUCCESS(rc))
2174	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)pR3Ptr);
2175	Assert(rc <= VINF_SUCCESS);
2176	return rc;
2177	}
2178
2179
2180	int pgmPhysCr3ToHCPtr(PVM pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr)
2181	{
2182
2183	}
2184
2185	#endif
2186
2187	/**
2188	* Converts a guest pointer to a GC physical address.
2189	*
2190	* This uses the current CR3/CR0/CR4 of the guest.
2191	*
2192	* @returns VBox status code.
2193	* @param pVCpu The cross context virtual CPU structure.
2194	* @param GCPtr The guest pointer to convert.
2195	* @param pGCPhys Where to store the GC physical address.
2196	*/
2197	VMMDECL(int) PGMPhysGCPtr2GCPhys(PVMCPU pVCpu, RTGCPTR GCPtr, PRTGCPHYS pGCPhys)
2198	{
2199	int rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtr, NULL, pGCPhys);
2200	if (pGCPhys && RT_SUCCESS(rc))
2201	*pGCPhys \|= (RTGCUINTPTR)GCPtr & PAGE_OFFSET_MASK;
2202	return rc;
2203	}
2204
2205
2206	/**
2207	* Converts a guest pointer to a HC physical address.
2208	*
2209	* This uses the current CR3/CR0/CR4 of the guest.
2210	*
2211	* @returns VBox status code.
2212	* @param pVCpu The cross context virtual CPU structure.
2213	* @param GCPtr The guest pointer to convert.
2214	* @param pHCPhys Where to store the HC physical address.
2215	*/
2216	VMM_INT_DECL(int) PGMPhysGCPtr2HCPhys(PVMCPU pVCpu, RTGCPTR GCPtr, PRTHCPHYS pHCPhys)
2217	{
2218	PVM pVM = pVCpu->CTX_SUFF(pVM);
2219	RTGCPHYS GCPhys;
2220	int rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtr, NULL, &GCPhys);
2221	if (RT_SUCCESS(rc))
2222	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhys \| ((RTGCUINTPTR)GCPtr & PAGE_OFFSET_MASK), pHCPhys);
2223	return rc;
2224	}
2225
2226
2227
2228	#undef LOG_GROUP
2229	#define LOG_GROUP LOG_GROUP_PGM_PHYS_ACCESS
2230
2231
2232	#if defined(IN_RING3) && defined(SOME_UNUSED_FUNCTION)
2233	/**
2234	* Cache PGMPhys memory access
2235	*
2236	* @param pVM The cross context VM structure.
2237	* @param pCache Cache structure pointer
2238	* @param GCPhys GC physical address
2239	* @param pbHC HC pointer corresponding to physical page
2240	*
2241	* @thread EMT.
2242	*/
2243	static void pgmPhysCacheAdd(PVM pVM, PGMPHYSCACHE pCache, RTGCPHYS GCPhys, uint8_t pbR3)
2244	{
2245	uint32_t iCacheIndex;
2246
2247	Assert(VM_IS_EMT(pVM));
2248
2249	GCPhys = PHYS_PAGE_ADDRESS(GCPhys);
2250	pbR3 = (uint8_t *)PAGE_ADDRESS(pbR3);
2251
2252	iCacheIndex = ((GCPhys >> PAGE_SHIFT) & PGM_MAX_PHYSCACHE_ENTRIES_MASK);
2253
2254	ASMBitSet(&pCache->aEntries, iCacheIndex);
2255
2256	pCache->Entry[iCacheIndex].GCPhys = GCPhys;
2257	pCache->Entry[iCacheIndex].pbR3 = pbR3;
2258	}
2259	#endif /* IN_RING3 */
2260
2261
2262	/**
2263	* Deals with reading from a page with one or more ALL access handlers.
2264	*
2265	* @returns Strict VBox status code in ring-0 and raw-mode, ignorable in ring-3.
2266	* See PGM_HANDLER_PHYS_IS_VALID_STATUS and
2267	* PGM_HANDLER_VIRT_IS_VALID_STATUS for details.
2268	*
2269	* @param pVM The cross context VM structure.
2270	* @param pPage The page descriptor.
2271	* @param GCPhys The physical address to start reading at.
2272	* @param pvBuf Where to put the bits we read.
2273	* @param cb How much to read - less or equal to a page.
2274	* @param enmOrigin The origin of this call.
2275	*/
2276	static VBOXSTRICTRC pgmPhysReadHandler(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void *pvBuf, size_t cb,
2277	PGMACCESSORIGIN enmOrigin)
2278	{
2279	/*
2280	* The most frequent access here is MMIO and shadowed ROM.
2281	* The current code ASSUMES all these access handlers covers full pages!
2282	*/
2283
2284	/*
2285	* Whatever we do we need the source page, map it first.
2286	*/
2287	PGMPAGEMAPLOCK PgMpLck;
2288	const void *pvSrc = NULL;
2289	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, GCPhys, &pvSrc, &PgMpLck);
2290	/** @todo Check how this can work for MMIO pages? */
2291	if (RT_FAILURE(rc))
2292	{
2293	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
2294	GCPhys, pPage, rc));
2295	memset(pvBuf, 0xff, cb);
2296	return VINF_SUCCESS;
2297	}
2298
2299	VBOXSTRICTRC rcStrict = VINF_PGM_HANDLER_DO_DEFAULT;
2300
2301	/*
2302	* Deal with any physical handlers.
2303	*/
2304	PVMCPU pVCpu = VMMGetCpu(pVM);
2305	PPGMPHYSHANDLER pPhys = NULL;
2306	if ( PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) == PGM_PAGE_HNDL_PHYS_STATE_ALL
2307	\|\| PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
2308	{
2309	pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
2310	AssertReleaseMsg(pPhys, ("GCPhys=%RGp cb=%#x\n", GCPhys, cb));
2311	Assert(GCPhys >= pPhys->Core.Key && GCPhys <= pPhys->Core.KeyLast);
2312	Assert((pPhys->Core.Key & PAGE_OFFSET_MASK) == 0);
2313	Assert((pPhys->Core.KeyLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
2314	#ifndef IN_RING3
2315	if (enmOrigin != PGMACCESSORIGIN_IEM)
2316	{
2317	/* Cannot reliably handle informational status codes in this context */
2318	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2319	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2320	}
2321	#endif
2322	PFNPGMPHYSHANDLER pfnHandler = PGMPHYSHANDLER_GET_TYPE(pVM, pPhys)->CTX_SUFF(pfnHandler); Assert(pfnHandler);
2323	void *pvUser = pPhys->CTX_SUFF(pvUser);
2324
2325	Log5(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cb, pPage, R3STRING(pPhys->pszDesc) ));
2326	STAM_PROFILE_START(&pPhys->Stat, h);
2327	PGM_LOCK_ASSERT_OWNER(pVM);
2328
2329	/* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
2330	pgmUnlock(pVM);
2331	rcStrict = pfnHandler(pVM, pVCpu, GCPhys, (void *)pvSrc, pvBuf, cb, PGMACCESSTYPE_READ, enmOrigin, pvUser);
2332	pgmLock(pVM);
2333
2334	#ifdef VBOX_WITH_STATISTICS
2335	pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
2336	if (pPhys)
2337	STAM_PROFILE_STOP(&pPhys->Stat, h);
2338	#else
2339	pPhys = NULL; /* might not be valid anymore. */
2340	#endif
2341	AssertLogRelMsg(PGM_HANDLER_PHYS_IS_VALID_STATUS(rcStrict, false),
2342	("rcStrict=%Rrc GCPhys=%RGp\n", VBOXSTRICTRC_VAL(rcStrict), GCPhys));
2343	if ( rcStrict != VINF_PGM_HANDLER_DO_DEFAULT
2344	&& !PGM_PHYS_RW_IS_SUCCESS(rcStrict))
2345	{
2346	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2347	return rcStrict;
2348	}
2349	}
2350
2351	#if !defined(IN_RING0) && defined(VBOX_WITH_RAW_MODE)
2352	/*
2353	* Deal with any virtual handlers.
2354	*/
2355	if (PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) == PGM_PAGE_HNDL_VIRT_STATE_ALL)
2356	{
2357	unsigned iPage;
2358	PPGMVIRTHANDLER pVirt = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys, &iPage);
2359	AssertReleaseMsg(pVirt, ("GCPhys=%RGp cb=%#x\n", GCPhys, cb));
2360	Assert((pVirt->Core.Key & PAGE_OFFSET_MASK) == 0);
2361	Assert((pVirt->Core.KeyLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
2362	Assert(GCPhys >= pVirt->aPhysToVirt[iPage].Core.Key && GCPhys <= pVirt->aPhysToVirt[iPage].Core.KeyLast);
2363
2364	# ifndef IN_RING3
2365	if (enmOrigin != PGMACCESSORIGIN_IEM)
2366	{
2367	/* Cannot reliably handle informational status codes in this context */
2368	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2369	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2370	}
2371	# endif
2372	PPGMVIRTHANDLERTYPEINT pVirtType = PGMVIRTANDLER_GET_TYPE(pVM, pVirt);
2373	if (!pPhys)
2374	Log5(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] virt %s\n", GCPhys, cb, pPage, R3STRING(pVirt->pszDesc) ));
2375	else
2376	Log(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] phys/virt %s/%s\n", GCPhys, cb, pPage, R3STRING(pVirt->pszDesc), R3STRING(pPhys->pszDesc) ));
2377	RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pVirt->Core.Key & PAGE_BASE_GC_MASK)
2378	+ (iPage << PAGE_SHIFT)
2379	+ (GCPhys & PAGE_OFFSET_MASK);
2380
2381	STAM_PROFILE_START(&pVirt->Stat, h);
2382	VBOXSTRICTRC rcStrict2 = pVirtType->CTX_SUFF(pfnHandler)(pVM, pVCpu, GCPtr, (void *)pvSrc, pvBuf, cb,
2383	PGMACCESSTYPE_READ, enmOrigin, pVirt->CTX_SUFF(pvUser));
2384	STAM_PROFILE_STOP(&pVirt->Stat, h);
2385
2386	/* Merge status codes. */
2387	if (rcStrict2 == VINF_SUCCESS)
2388	{
2389	if (rcStrict == VINF_PGM_HANDLER_DO_DEFAULT)
2390	rcStrict = VINF_SUCCESS;
2391	}
2392	else if (rcStrict2 != VINF_PGM_HANDLER_DO_DEFAULT)
2393	{
2394	AssertLogRelMsg(PGM_HANDLER_VIRT_IS_VALID_STATUS(rcStrict2, false),
2395	("rcStrict2=%Rrc (rcStrict=%Rrc) GCPhys=%RGp pPage=%R[pgmpage] %s\n",
2396	VBOXSTRICTRC_VAL(rcStrict2), VBOXSTRICTRC_VAL(rcStrict), GCPhys, pPage, pVirt->pszDesc));
2397	if (!PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
2398	{
2399	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2400	return rcStrict2;
2401	}
2402	if (rcStrict == VINF_PGM_HANDLER_DO_DEFAULT)
2403	rcStrict = rcStrict2;
2404	else
2405	PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
2406	}
2407	}
2408	#endif /* !IN_RING0 && VBOX_WITH_RAW_MODE */
2409
2410	/*
2411	* Take the default action.
2412	*/
2413	if (rcStrict == VINF_PGM_HANDLER_DO_DEFAULT)
2414	{
2415	memcpy(pvBuf, pvSrc, cb);
2416	rcStrict = VINF_SUCCESS;
2417	}
2418	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2419	return rcStrict;
2420	}
2421
2422
2423	/**
2424	* Read physical memory.
2425	*
2426	* This API respects access handlers and MMIO. Use PGMPhysSimpleReadGCPhys() if you
2427	* want to ignore those.
2428	*
2429	* @returns Strict VBox status code in raw-mode and ring-0, normal VBox status
2430	* code in ring-3. Use PGM_PHYS_RW_IS_SUCCESS to check.
2431	* @retval VINF_SUCCESS in all context - read completed.
2432	*
2433	* @retval VINF_EM_OFF in RC and R0 - read completed.
2434	* @retval VINF_EM_SUSPEND in RC and R0 - read completed.
2435	* @retval VINF_EM_RESET in RC and R0 - read completed.
2436	* @retval VINF_EM_HALT in RC and R0 - read completed.
2437	* @retval VINF_SELM_SYNC_GDT in RC only - read completed.
2438	*
2439	* @retval VINF_EM_DBG_STOP in RC and R0 - read completed.
2440	* @retval VINF_EM_DBG_BREAKPOINT in RC and R0 - read completed.
2441	* @retval VINF_EM_RAW_EMULATE_INSTR in RC and R0 only.
2442	*
2443	* @retval VINF_IOM_R3_MMIO_READ in RC and R0.
2444	* @retval VINF_IOM_R3_MMIO_READ_WRITE in RC and R0.
2445	*
2446	* @retval VINF_PATM_CHECK_PATCH_PAGE in RC only.
2447	*
2448	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in RC and R0 for access origins that
2449	* haven't been cleared for strict status codes yet.
2450	*
2451	* @param pVM The cross context VM structure.
2452	* @param GCPhys Physical address start reading from.
2453	* @param pvBuf Where to put the read bits.
2454	* @param cbRead How many bytes to read.
2455	* @param enmOrigin The origin of this call.
2456	*/
2457	VMMDECL(VBOXSTRICTRC) PGMPhysRead(PVM pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead, PGMACCESSORIGIN enmOrigin)
2458	{
2459	AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
2460	LogFlow(("PGMPhysRead: %RGp %d\n", GCPhys, cbRead));
2461
2462	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysRead));
2463	STAM_COUNTER_ADD(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysReadBytes), cbRead);
2464
2465	pgmLock(pVM);
2466
2467	/*
2468	* Copy loop on ram ranges.
2469	*/
2470	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
2471	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
2472	for (;;)
2473	{
2474	/* Inside range or not? */
2475	if (pRam && GCPhys >= pRam->GCPhys)
2476	{
2477	/*
2478	* Must work our way thru this page by page.
2479	*/
2480	RTGCPHYS off = GCPhys - pRam->GCPhys;
2481	while (off < pRam->cb)
2482	{
2483	unsigned iPage = off >> PAGE_SHIFT;
2484	PPGMPAGE pPage = &pRam->aPages[iPage];
2485	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
2486	if (cb > cbRead)
2487	cb = cbRead;
2488
2489	/*
2490	* Normal page? Get the pointer to it.
2491	*/
2492	if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
2493	&& !PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
2494	{
2495	/*
2496	* Get the pointer to the page.
2497	*/
2498	PGMPAGEMAPLOCK PgMpLck;
2499	const void *pvSrc;
2500	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc, &PgMpLck);
2501	if (RT_SUCCESS(rc))
2502	{
2503	memcpy(pvBuf, pvSrc, cb);
2504	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2505	}
2506	else
2507	{
2508	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
2509	pRam->GCPhys + off, pPage, rc));
2510	memset(pvBuf, 0xff, cb);
2511	}
2512	}
2513	/*
2514	* Have ALL/MMIO access handlers.
2515	*/
2516	else
2517	{
2518	VBOXSTRICTRC rcStrict2 = pgmPhysReadHandler(pVM, pPage, pRam->GCPhys + off, pvBuf, cb, enmOrigin);
2519	if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
2520	PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
2521	else
2522	{
2523	pgmUnlock(pVM);
2524	return rcStrict2;
2525	}
2526	}
2527
2528	/* next page */
2529	if (cb >= cbRead)
2530	{
2531	pgmUnlock(pVM);
2532	return rcStrict;
2533	}
2534	cbRead -= cb;
2535	off += cb;
2536	pvBuf = (char *)pvBuf + cb;
2537	} /* walk pages in ram range. */
2538
2539	GCPhys = pRam->GCPhysLast + 1;
2540	}
2541	else
2542	{
2543	LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
2544
2545	/*
2546	* Unassigned address space.
2547	*/
2548	size_t cb = pRam ? pRam->GCPhys - GCPhys : ~(size_t)0;
2549	if (cb >= cbRead)
2550	{
2551	memset(pvBuf, 0xff, cbRead);
2552	break;
2553	}
2554	memset(pvBuf, 0xff, cb);
2555
2556	cbRead -= cb;
2557	pvBuf = (char *)pvBuf + cb;
2558	GCPhys += cb;
2559	}
2560
2561	/* Advance range if necessary. */
2562	while (pRam && GCPhys > pRam->GCPhysLast)
2563	pRam = pRam->CTX_SUFF(pNext);
2564	} /* Ram range walk */
2565
2566	pgmUnlock(pVM);
2567	return rcStrict;
2568	}
2569
2570
2571	/**
2572	* Deals with writing to a page with one or more WRITE or ALL access handlers.
2573	*
2574	* @returns Strict VBox status code in ring-0 and raw-mode, ignorable in ring-3.
2575	* See PGM_HANDLER_PHYS_IS_VALID_STATUS and
2576	* PGM_HANDLER_VIRT_IS_VALID_STATUS for details.
2577	*
2578	* @param pVM The cross context VM structure.
2579	* @param pPage The page descriptor.
2580	* @param GCPhys The physical address to start writing at.
2581	* @param pvBuf What to write.
2582	* @param cbWrite How much to write - less or equal to a page.
2583	* @param enmOrigin The origin of this call.
2584	*/
2585	static VBOXSTRICTRC pgmPhysWriteHandler(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const *pvBuf, size_t cbWrite,
2586	PGMACCESSORIGIN enmOrigin)
2587	{
2588	PGMPAGEMAPLOCK PgMpLck;
2589	void *pvDst = NULL;
2590	VBOXSTRICTRC rcStrict;
2591
2592	/*
2593	* Give priority to physical handlers (like #PF does).
2594	*
2595	* Hope for a lonely physical handler first that covers the whole
2596	* write area. This should be a pretty frequent case with MMIO and
2597	* the heavy usage of full page handlers in the page pool.
2598	*/
2599	PVMCPU pVCpu = VMMGetCpu(pVM);
2600	if ( !PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(pPage)
2601	\|\| PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage) /* screw virtual handlers on MMIO pages */)
2602	{
2603	PPGMPHYSHANDLER pCur = pgmHandlerPhysicalLookup(pVM, GCPhys);
2604	if (pCur)
2605	{
2606	Assert(GCPhys >= pCur->Core.Key && GCPhys <= pCur->Core.KeyLast);
2607	#ifndef IN_RING3
2608	if (enmOrigin != PGMACCESSORIGIN_IEM)
2609	/* Cannot reliably handle informational status codes in this context */
2610	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2611	#endif
2612	size_t cbRange = pCur->Core.KeyLast - GCPhys + 1;
2613	if (cbRange > cbWrite)
2614	cbRange = cbWrite;
2615
2616	Assert(PGMPHYSHANDLER_GET_TYPE(pVM, pCur)->CTX_SUFF(pfnHandler));
2617	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n",
2618	GCPhys, cbRange, pPage, R3STRING(pCur->pszDesc) ));
2619	if (!PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
2620	rcStrict = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst, &PgMpLck);
2621	else
2622	rcStrict = VINF_SUCCESS;
2623	if (RT_SUCCESS(rcStrict))
2624	{
2625	PFNPGMPHYSHANDLER pfnHandler = PGMPHYSHANDLER_GET_TYPE(pVM, pCur)->CTX_SUFF(pfnHandler);
2626	void *pvUser = pCur->CTX_SUFF(pvUser);
2627	STAM_PROFILE_START(&pCur->Stat, h);
2628
2629	/* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
2630	PGM_LOCK_ASSERT_OWNER(pVM);
2631	pgmUnlock(pVM);
2632	rcStrict = pfnHandler(pVM, pVCpu, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, enmOrigin, pvUser);
2633	pgmLock(pVM);
2634
2635	#ifdef VBOX_WITH_STATISTICS
2636	pCur = pgmHandlerPhysicalLookup(pVM, GCPhys);
2637	if (pCur)
2638	STAM_PROFILE_STOP(&pCur->Stat, h);
2639	#else
2640	pCur = NULL; /* might not be valid anymore. */
2641	#endif
2642	if (rcStrict == VINF_PGM_HANDLER_DO_DEFAULT)
2643	{
2644	if (pvDst)
2645	memcpy(pvDst, pvBuf, cbRange);
2646	rcStrict = VINF_SUCCESS;
2647	}
2648	else
2649	AssertLogRelMsg(PGM_HANDLER_PHYS_IS_VALID_STATUS(rcStrict, true),
2650	("rcStrict=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n",
2651	VBOXSTRICTRC_VAL(rcStrict), GCPhys, pPage, pCur ? R3STRING(pCur->pszDesc) : ""));
2652	}
2653	else
2654	AssertLogRelMsgFailedReturn(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
2655	GCPhys, pPage, VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
2656	if (RT_LIKELY(cbRange == cbWrite) \|\| !PGM_PHYS_RW_IS_SUCCESS(rcStrict))
2657	{
2658	if (pvDst)
2659	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2660	return rcStrict;
2661	}
2662
2663	/* more fun to be had below */
2664	cbWrite -= cbRange;
2665	GCPhys += cbRange;
2666	pvBuf = (uint8_t *)pvBuf + cbRange;
2667	pvDst = (uint8_t *)pvDst + cbRange;
2668	}
2669	else /* The handler is somewhere else in the page, deal with it below. */
2670	rcStrict = VINF_SUCCESS;
2671	Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage)); /* MMIO handlers are all PAGE_SIZEed! */
2672	}
2673	#if !defined(IN_RING0) && defined(VBOX_WITH_RAW_MODE)
2674	/*
2675	* A virtual handler without any interfering physical handlers.
2676	* Hopefully it'll cover the whole write.
2677	*/
2678	else if (!PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage))
2679	{
2680	unsigned iPage;
2681	PPGMVIRTHANDLER pVirt = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys, &iPage);
2682	if (pVirt)
2683	{
2684	# ifndef IN_RING3
2685	if (enmOrigin != PGMACCESSORIGIN_IEM)
2686	/* Cannot reliably handle informational status codes in this context */
2687	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2688	# endif
2689	PPGMVIRTHANDLERTYPEINT pVirtType = PGMVIRTANDLER_GET_TYPE(pVM, pVirt);
2690	size_t cbRange = (PAGE_OFFSET_MASK & pVirt->Core.KeyLast) - (PAGE_OFFSET_MASK & GCPhys) + 1;
2691	if (cbRange > cbWrite)
2692	cbRange = cbWrite;
2693
2694	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] virt %s\n",
2695	GCPhys, cbRange, pPage, R3STRING(pVirt->pszDesc) ));
2696	rcStrict = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst, &PgMpLck);
2697	if (RT_SUCCESS(rcStrict))
2698	{
2699	Assert(pVirtType->CTX_SUFF(pfnHandler));
2700	RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pVirt->Core.Key & PAGE_BASE_GC_MASK)
2701	+ (iPage << PAGE_SHIFT)
2702	+ (GCPhys & PAGE_OFFSET_MASK);
2703
2704	STAM_PROFILE_START(&pVirt->Stat, h);
2705	rcStrict = pVirtType->CTX_SUFF(pfnHandler)(pVM, pVCpu, GCPtr, pvDst, (void *)pvBuf, cbRange,
2706	PGMACCESSTYPE_WRITE, enmOrigin, pVirt->CTX_SUFF(pvUser));
2707	STAM_PROFILE_STOP(&pVirt->Stat, h);
2708	if (rcStrict == VINF_PGM_HANDLER_DO_DEFAULT)
2709	{
2710	memcpy(pvDst, pvBuf, cbRange);
2711	rcStrict = VINF_SUCCESS;
2712	}
2713	else
2714	AssertLogRelMsg(PGM_HANDLER_VIRT_IS_VALID_STATUS(rcStrict, true),
2715	("rcStrict=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n",
2716	VBOXSTRICTRC_VAL(rcStrict), GCPhys, pPage, R3STRING(pVirt->pszDesc)));
2717	}
2718	else
2719	AssertLogRelMsgFailedReturn(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
2720	GCPhys, pPage, VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
2721	if (RT_LIKELY(cbRange == cbWrite) \|\| !PGM_PHYS_RW_IS_SUCCESS(rcStrict))
2722	{
2723	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2724	return rcStrict;
2725	}
2726
2727	/* more fun to be had below */
2728	cbWrite -= cbRange;
2729	GCPhys += cbRange;
2730	pvBuf = (uint8_t *)pvBuf + cbRange;
2731	pvDst = (uint8_t *)pvDst + cbRange;
2732	}
2733	else /* The handler is somewhere else in the page, deal with it below. */
2734	rcStrict = VINF_SUCCESS;
2735	}
2736	#endif /* !IN_RING0 && VBOX_WITH_RAW_MODE */
2737	else
2738	rcStrict = VINF_SUCCESS;
2739
2740
2741	/*
2742	* Deal with all the odd ends.
2743	*/
2744	Assert(rcStrict != VINF_PGM_HANDLER_DO_DEFAULT);
2745
2746	/* We need a writable destination page. */
2747	if (!pvDst)
2748	{
2749	int rc2 = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst, &PgMpLck);
2750	AssertLogRelMsgReturn(RT_SUCCESS(rc2),
2751	("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n", GCPhys, pPage, rc2),
2752	rc2);
2753	}
2754
2755	/* The loop state (big + ugly). */
2756	#if !defined(IN_RING0) && defined(VBOX_WITH_RAW_MODE)
2757	unsigned iVirtPage = 0;
2758	PPGMVIRTHANDLER pVirt = NULL;
2759	uint32_t offVirt = PAGE_SIZE;
2760	uint32_t offVirtLast = PAGE_SIZE;
2761	bool fMoreVirt = PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(pPage);
2762	#else
2763	uint32_t const offVirt = UINT32_MAX;
2764	#endif
2765
2766	PPGMPHYSHANDLER pPhys = NULL;
2767	uint32_t offPhys = PAGE_SIZE;
2768	uint32_t offPhysLast = PAGE_SIZE;
2769	bool fMorePhys = PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage);
2770
2771	/* The loop. */
2772	for (;;)
2773	{
2774	#if !defined(IN_RING0) && defined(VBOX_WITH_RAW_MODE)
2775	/*
2776	* Find the closest handler at or above GCPhys.
2777	*/
2778	if (fMoreVirt && !pVirt)
2779	{
2780	pVirt = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys, &iVirtPage);
2781	if (pVirt)
2782	{
2783	offVirt = 0;
2784	offVirtLast = (pVirt->aPhysToVirt[iVirtPage].Core.KeyLast & PAGE_OFFSET_MASK) - (GCPhys & PAGE_OFFSET_MASK);
2785	}
2786	else
2787	{
2788	PPGMPHYS2VIRTHANDLER pVirtPhys;
2789	pVirtPhys = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysGetBestFit(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysToVirtHandlers,
2790	GCPhys, true /* fAbove */);
2791	if ( pVirtPhys
2792	&& (pVirtPhys->Core.Key >> PAGE_SHIFT) == (GCPhys >> PAGE_SHIFT))
2793	{
2794	/* ASSUME that pVirtPhys only covers one page. */
2795	Assert((pVirtPhys->Core.Key >> PAGE_SHIFT) == (pVirtPhys->Core.KeyLast >> PAGE_SHIFT));
2796	Assert(pVirtPhys->Core.Key > GCPhys);
2797
2798	pVirt = (PPGMVIRTHANDLER)((uintptr_t)pVirtPhys + pVirtPhys->offVirtHandler);
2799	iVirtPage = pVirtPhys - &pVirt->aPhysToVirt[0]; Assert(iVirtPage == 0);
2800	offVirt = (pVirtPhys->Core.Key & PAGE_OFFSET_MASK) - (GCPhys & PAGE_OFFSET_MASK);
2801	offVirtLast = (pVirtPhys->Core.KeyLast & PAGE_OFFSET_MASK) - (GCPhys & PAGE_OFFSET_MASK);
2802	}
2803	else
2804	{
2805	pVirt = NULL;
2806	fMoreVirt = false;
2807	offVirt = offVirtLast = PAGE_SIZE;
2808	}
2809	}
2810	}
2811	#endif
2812
2813	if (fMorePhys && !pPhys)
2814	{
2815	pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
2816	if (pPhys)
2817	{
2818	offPhys = 0;
2819	offPhysLast = pPhys->Core.KeyLast - GCPhys; /* ASSUMES < 4GB handlers... */
2820	}
2821	else
2822	{
2823	pPhys = (PPGMPHYSHANDLER)RTAvlroGCPhysGetBestFit(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers,
2824	GCPhys, true /* fAbove */);
2825	if ( pPhys
2826	&& pPhys->Core.Key <= GCPhys + (cbWrite - 1))
2827	{
2828	offPhys = pPhys->Core.Key - GCPhys;
2829	offPhysLast = pPhys->Core.KeyLast - GCPhys; /* ASSUMES < 4GB handlers... */
2830	}
2831	else
2832	{
2833	pPhys = NULL;
2834	fMorePhys = false;
2835	offPhys = offPhysLast = PAGE_SIZE;
2836	}
2837	}
2838	}
2839
2840	/*
2841	* Handle access to space without handlers (that's easy).
2842	*/
2843	VBOXSTRICTRC rcStrict2 = VINF_PGM_HANDLER_DO_DEFAULT;
2844	uint32_t cbRange = (uint32_t)cbWrite;
2845	if (offPhys && offVirt)
2846	{
2847	if (cbRange > offPhys)
2848	cbRange = offPhys;
2849	if (cbRange > offVirt)
2850	cbRange = offVirt;
2851	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] miss\n", GCPhys, cbRange, pPage));
2852	}
2853	/*
2854	* Physical handler.
2855	*/
2856	else if (!offPhys && offVirt)
2857	{
2858	#ifndef IN_RING3
2859	if (enmOrigin != PGMACCESSORIGIN_IEM)
2860	/* Cannot reliably handle informational status codes in this context */
2861	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2862	#endif
2863	if (cbRange > offPhysLast + 1)
2864	cbRange = offPhysLast + 1;
2865	if (cbRange > offVirt)
2866	cbRange = offVirt;
2867
2868	PFNPGMPHYSHANDLER pfnHandler = PGMPHYSHANDLER_GET_TYPE(pVM, pPhys)->CTX_SUFF(pfnHandler);
2869	void *pvUser = pPhys->CTX_SUFF(pvUser);
2870
2871	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cbRange, pPage, R3STRING(pPhys->pszDesc) ));
2872	STAM_PROFILE_START(&pPhys->Stat, h);
2873
2874	/* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
2875	PGM_LOCK_ASSERT_OWNER(pVM);
2876	pgmUnlock(pVM);
2877	rcStrict2 = pfnHandler(pVM, pVCpu, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, enmOrigin, pvUser);
2878	pgmLock(pVM);
2879
2880	#ifdef VBOX_WITH_STATISTICS
2881	pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
2882	if (pPhys)
2883	STAM_PROFILE_STOP(&pPhys->Stat, h);
2884	#else
2885	pPhys = NULL; /* might not be valid anymore. */
2886	#endif
2887	AssertLogRelMsg(PGM_HANDLER_PHYS_IS_VALID_STATUS(rcStrict2, true),
2888	("rcStrict2=%Rrc (rcStrict=%Rrc) GCPhys=%RGp pPage=%R[pgmpage] %s\n", VBOXSTRICTRC_VAL(rcStrict2),
2889	VBOXSTRICTRC_VAL(rcStrict), GCPhys, pPage, pPhys ? R3STRING(pPhys->pszDesc) : ""));
2890	}
2891	#if !defined(IN_RING0) && defined(VBOX_WITH_RAW_MODE)
2892	/*
2893	* Virtual handler.
2894	*/
2895	else if (offPhys && !offVirt)
2896	{
2897	# ifndef IN_RING3
2898	if (enmOrigin != PGMACCESSORIGIN_IEM)
2899	/* Cannot reliably handle informational status codes in this context */
2900	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2901	# endif
2902	if (cbRange > offVirtLast + 1)
2903	cbRange = offVirtLast + 1;
2904	if (cbRange > offPhys)
2905	cbRange = offPhys;
2906
2907	PPGMVIRTHANDLERTYPEINT pVirtType = PGMVIRTANDLER_GET_TYPE(pVM, pVirt);
2908	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cbRange, pPage, R3STRING(pVirt->pszDesc) ));
2909	Assert(pVirtType->CTX_SUFF(pfnHandler));
2910	RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pVirt->Core.Key & PAGE_BASE_GC_MASK)
2911	+ (iVirtPage << PAGE_SHIFT)
2912	+ (GCPhys & PAGE_OFFSET_MASK);
2913	STAM_PROFILE_START(&pVirt->Stat, h);
2914	rcStrict2 = pVirtType->CTX_SUFF(pfnHandler)(pVM, pVCpu, GCPtr, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE,
2915	enmOrigin, pVirt->CTX_SUFF(pvUser));
2916	STAM_PROFILE_STOP(&pVirt->Stat, h);
2917	AssertLogRelMsg(PGM_HANDLER_VIRT_IS_VALID_STATUS(rcStrict2, true),
2918	("rcStrict2=%Rrc (rcStrict=%Rrc) GCPhys=%RGp pPage=%R[pgmpage] %s\n", VBOXSTRICTRC_VAL(rcStrict2),
2919	VBOXSTRICTRC_VAL(rcStrict), GCPhys, pPage, pPhys ? R3STRING(pPhys->pszDesc) : ""));
2920	pVirt = NULL;
2921	}
2922	/*
2923	* Both... give the physical one priority.
2924	*/
2925	else
2926	{
2927	# ifndef IN_RING3
2928	if (enmOrigin != PGMACCESSORIGIN_IEM)
2929	/* Cannot reliably handle informational status codes in this context */
2930	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2931	# endif
2932	Assert(!offPhys && !offVirt);
2933	if (cbRange > offVirtLast + 1)
2934	cbRange = offVirtLast + 1;
2935	if (cbRange > offPhysLast + 1)
2936	cbRange = offPhysLast + 1;
2937
2938	PPGMVIRTHANDLERTYPEINT pVirtType = PGMVIRTANDLER_GET_TYPE(pVM, pVirt);
2939	if (pVirtType->pfnHandlerR3)
2940	Log(("pgmPhysWriteHandler: overlapping phys and virt handlers at %RGp %R[pgmpage]; cbRange=%#x\n", GCPhys, pPage, cbRange));
2941	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys/virt %s/%s\n", GCPhys, cbRange, pPage, R3STRING(pPhys->pszDesc), R3STRING(pVirt->pszDesc) ));
2942
2943	PFNPGMPHYSHANDLER pfnHandler = PGMPHYSHANDLER_GET_TYPE(pVM, pPhys)->CTX_SUFF(pfnHandler);
2944	void *pvUser = pPhys->CTX_SUFF(pvUser);
2945	STAM_PROFILE_START(&pPhys->Stat, h);
2946
2947	/* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
2948	PGM_LOCK_ASSERT_OWNER(pVM);
2949	pgmUnlock(pVM);
2950	rcStrict2 = pfnHandler(pVM, pVCpu, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, enmOrigin, pvUser);
2951	pgmLock(pVM);
2952
2953	# ifdef VBOX_WITH_STATISTICS
2954	pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
2955	if (pPhys)
2956	STAM_PROFILE_STOP(&pPhys->Stat, h);
2957	# else
2958	pPhys = NULL; /* might not be valid anymore. */
2959	# endif
2960	AssertLogRelMsg(PGM_HANDLER_PHYS_IS_VALID_STATUS(rcStrict2, true),
2961	("rcStrict2=%Rrc (rcStrict=%Rrc) GCPhys=%RGp pPage=%R[pgmpage] %s\n", VBOXSTRICTRC_VAL(rcStrict2),
2962	VBOXSTRICTRC_VAL(rcStrict), GCPhys, pPage, pPhys ? R3STRING(pPhys->pszDesc) : ""));
2963	if (rcStrict2 == VINF_PGM_HANDLER_DO_DEFAULT \|\| PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
2964	{
2965	Assert(pVirtType->CTX_SUFF(pfnHandler));
2966	RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pVirt->Core.Key & PAGE_BASE_GC_MASK)
2967	+ (iVirtPage << PAGE_SHIFT)
2968	+ (GCPhys & PAGE_OFFSET_MASK);
2969	pvUser = pVirt->CTX_SUFF(pvUser);
2970
2971	STAM_PROFILE_START(&pVirt->Stat, h2);
2972	VBOXSTRICTRC rcStrict3 = pVirtType->CTX_SUFF(pfnHandler)(pVM, pVCpu, GCPtr, pvDst, (void *)pvBuf, cbRange,
2973	PGMACCESSTYPE_WRITE, enmOrigin, pvUser);
2974	STAM_PROFILE_STOP(&pVirt->Stat, h2);
2975
2976	/* Merge the 3rd status into the 2nd. */
2977	if (rcStrict3 == VINF_SUCCESS)
2978	{
2979	if (rcStrict2 == VINF_PGM_HANDLER_DO_DEFAULT)
2980	rcStrict2 = VINF_SUCCESS;
2981	}
2982	else if (rcStrict3 != VINF_PGM_HANDLER_DO_DEFAULT)
2983	{
2984	AssertLogRelMsg(PGM_HANDLER_VIRT_IS_VALID_STATUS(rcStrict3, true),
2985	("rcStrict3=%Rrc (rcStrict2=%Rrc) (rcStrict=%Rrc) GCPhys=%RGp pPage=%R[pgmpage] %s\n",
2986	VBOXSTRICTRC_VAL(rcStrict3), VBOXSTRICTRC_VAL(rcStrict2), VBOXSTRICTRC_VAL(rcStrict),
2987	GCPhys, pPage, R3STRING(pVirt->pszDesc) ));
2988	if (rcStrict2 == VINF_PGM_HANDLER_DO_DEFAULT)
2989	rcStrict2 = rcStrict3;
2990	else if (!PGM_PHYS_RW_IS_SUCCESS(rcStrict3))
2991	rcStrict2 = rcStrict3;
2992	else
2993	PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict2, rcStrict3);
2994	}
2995	}
2996	pPhys = NULL;
2997	pVirt = NULL;
2998	}
2999	#endif /* !IN_RING0 && VBOX_WITH_RAW_MODE */
3000
3001
3002	/*
3003	* Execute the default action and merge the status codes.
3004	*/
3005	if (rcStrict2 == VINF_PGM_HANDLER_DO_DEFAULT)
3006	{
3007	memcpy(pvDst, pvBuf, cbRange);
3008	rcStrict2 = VINF_SUCCESS;
3009	}
3010	else if (!PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
3011	{
3012	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
3013	return rcStrict2;
3014	}
3015	else
3016	PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
3017
3018	/*
3019	* Advance if we've got more stuff to do.
3020	*/
3021	if (cbRange >= cbWrite)
3022	{
3023	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
3024	return rcStrict;
3025	}
3026
3027
3028	cbWrite -= cbRange;
3029	GCPhys += cbRange;
3030	pvBuf = (uint8_t *)pvBuf + cbRange;
3031	pvDst = (uint8_t *)pvDst + cbRange;
3032
3033	offPhys -= cbRange;
3034	offPhysLast -= cbRange;
3035	#if !defined(IN_RING0) && defined(VBOX_WITH_RAW_MODE)
3036	offVirt -= cbRange;
3037	offVirtLast -= cbRange;
3038	#endif
3039	}
3040	}
3041
3042
3043	/**
3044	* Write to physical memory.
3045	*
3046	* This API respects access handlers and MMIO. Use PGMPhysSimpleWriteGCPhys() if you
3047	* want to ignore those.
3048	*
3049	* @returns Strict VBox status code in raw-mode and ring-0, normal VBox status
3050	* code in ring-3. Use PGM_PHYS_RW_IS_SUCCESS to check.
3051	* @retval VINF_SUCCESS in all context - write completed.
3052	*
3053	* @retval VINF_EM_OFF in RC and R0 - write completed.
3054	* @retval VINF_EM_SUSPEND in RC and R0 - write completed.
3055	* @retval VINF_EM_RESET in RC and R0 - write completed.
3056	* @retval VINF_EM_HALT in RC and R0 - write completed.
3057	* @retval VINF_SELM_SYNC_GDT in RC only - write completed.
3058	*
3059	* @retval VINF_EM_DBG_STOP in RC and R0 - write completed.
3060	* @retval VINF_EM_DBG_BREAKPOINT in RC and R0 - write completed.
3061	* @retval VINF_EM_RAW_EMULATE_INSTR in RC and R0 only.
3062	*
3063	* @retval VINF_IOM_R3_MMIO_WRITE in RC and R0.
3064	* @retval VINF_IOM_R3_MMIO_READ_WRITE in RC and R0.
3065	*
3066	* @retval VINF_EM_RAW_EMULATE_IO_BLOCK in R0 only.
3067	*
3068	* @retval VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT in RC only - write completed.
3069	* @retval VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT in RC only.
3070	* @retval VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT in RC only.
3071	* @retval VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT in RC only.
3072	* @retval VINF_CSAM_PENDING_ACTION in RC only.
3073	* @retval VINF_PATM_CHECK_PATCH_PAGE in RC only.
3074	*
3075	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in RC and R0 for access origins that
3076	* haven't been cleared for strict status codes yet.
3077	*
3078	*
3079	* @param pVM The cross context VM structure.
3080	* @param GCPhys Physical address to write to.
3081	* @param pvBuf What to write.
3082	* @param cbWrite How many bytes to write.
3083	* @param enmOrigin Who is calling.
3084	*/
3085	VMMDECL(VBOXSTRICTRC) PGMPhysWrite(PVM pVM, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite, PGMACCESSORIGIN enmOrigin)
3086	{
3087	AssertMsg(!pVM->pgm.s.fNoMorePhysWrites, ("Calling PGMPhysWrite after pgmR3Save()! enmOrigin=%d\n", enmOrigin));
3088	AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
3089	LogFlow(("PGMPhysWrite: %RGp %d\n", GCPhys, cbWrite));
3090
3091	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysWrite));
3092	STAM_COUNTER_ADD(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysWriteBytes), cbWrite);
3093
3094	pgmLock(pVM);
3095
3096	/*
3097	* Copy loop on ram ranges.
3098	*/
3099	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
3100	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
3101	for (;;)
3102	{
3103	/* Inside range or not? */
3104	if (pRam && GCPhys >= pRam->GCPhys)
3105	{
3106	/*
3107	* Must work our way thru this page by page.
3108	*/
3109	RTGCPTR off = GCPhys - pRam->GCPhys;
3110	while (off < pRam->cb)
3111	{
3112	RTGCPTR iPage = off >> PAGE_SHIFT;
3113	PPGMPAGE pPage = &pRam->aPages[iPage];
3114	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
3115	if (cb > cbWrite)
3116	cb = cbWrite;
3117
3118	/*
3119	* Normal page? Get the pointer to it.
3120	*/
3121	if ( !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
3122	&& !PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
3123	{
3124	PGMPAGEMAPLOCK PgMpLck;
3125	void *pvDst;
3126	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst, &PgMpLck);
3127	if (RT_SUCCESS(rc))
3128	{
3129	Assert(!PGM_PAGE_IS_BALLOONED(pPage));
3130	memcpy(pvDst, pvBuf, cb);
3131	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
3132	}
3133	/* Ignore writes to ballooned pages. */
3134	else if (!PGM_PAGE_IS_BALLOONED(pPage))
3135	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
3136	pRam->GCPhys + off, pPage, rc));
3137	}
3138	/*
3139	* Active WRITE or ALL access handlers.
3140	*/
3141	else
3142	{
3143	VBOXSTRICTRC rcStrict2 = pgmPhysWriteHandler(pVM, pPage, pRam->GCPhys + off, pvBuf, cb, enmOrigin);
3144	if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
3145	PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
3146	else
3147	{
3148	pgmUnlock(pVM);
3149	return rcStrict2;
3150	}
3151	}
3152
3153	/* next page */
3154	if (cb >= cbWrite)
3155	{
3156	pgmUnlock(pVM);
3157	return rcStrict;
3158	}
3159
3160	cbWrite -= cb;
3161	off += cb;
3162	pvBuf = (const char *)pvBuf + cb;
3163	} /* walk pages in ram range */
3164
3165	GCPhys = pRam->GCPhysLast + 1;
3166	}
3167	else
3168	{
3169	/*
3170	* Unassigned address space, skip it.
3171	*/
3172	if (!pRam)
3173	break;
3174	size_t cb = pRam->GCPhys - GCPhys;
3175	if (cb >= cbWrite)
3176	break;
3177	cbWrite -= cb;
3178	pvBuf = (const char *)pvBuf + cb;
3179	GCPhys += cb;
3180	}
3181
3182	/* Advance range if necessary. */
3183	while (pRam && GCPhys > pRam->GCPhysLast)
3184	pRam = pRam->CTX_SUFF(pNext);
3185	} /* Ram range walk */
3186
3187	pgmUnlock(pVM);
3188	return rcStrict;
3189	}
3190
3191
3192	/**
3193	* Read from guest physical memory by GC physical address, bypassing
3194	* MMIO and access handlers.
3195	*
3196	* @returns VBox status code.
3197	* @param pVM The cross context VM structure.
3198	* @param pvDst The destination address.
3199	* @param GCPhysSrc The source address (GC physical address).
3200	* @param cb The number of bytes to read.
3201	*/
3202	VMMDECL(int) PGMPhysSimpleReadGCPhys(PVM pVM, void *pvDst, RTGCPHYS GCPhysSrc, size_t cb)
3203	{
3204	/*
3205	* Treat the first page as a special case.
3206	*/
3207	if (!cb)
3208	return VINF_SUCCESS;
3209
3210	/* map the 1st page */
3211	void const *pvSrc;
3212	PGMPAGEMAPLOCK Lock;
3213	int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock);
3214	if (RT_FAILURE(rc))
3215	return rc;
3216
3217	/* optimize for the case where access is completely within the first page. */
3218	size_t cbPage = PAGE_SIZE - (GCPhysSrc & PAGE_OFFSET_MASK);
3219	if (RT_LIKELY(cb <= cbPage))
3220	{
3221	memcpy(pvDst, pvSrc, cb);
3222	PGMPhysReleasePageMappingLock(pVM, &Lock);
3223	return VINF_SUCCESS;
3224	}
3225
3226	/* copy to the end of the page. */
3227	memcpy(pvDst, pvSrc, cbPage);
3228	PGMPhysReleasePageMappingLock(pVM, &Lock);
3229	GCPhysSrc += cbPage;
3230	pvDst = (uint8_t *)pvDst + cbPage;
3231	cb -= cbPage;
3232
3233	/*
3234	* Page by page.
3235	*/
3236	for (;;)
3237	{
3238	/* map the page */
3239	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock);
3240	if (RT_FAILURE(rc))
3241	return rc;
3242
3243	/* last page? */
3244	if (cb <= PAGE_SIZE)
3245	{
3246	memcpy(pvDst, pvSrc, cb);
3247	PGMPhysReleasePageMappingLock(pVM, &Lock);
3248	return VINF_SUCCESS;
3249	}
3250
3251	/* copy the entire page and advance */
3252	memcpy(pvDst, pvSrc, PAGE_SIZE);
3253	PGMPhysReleasePageMappingLock(pVM, &Lock);
3254	GCPhysSrc += PAGE_SIZE;
3255	pvDst = (uint8_t *)pvDst + PAGE_SIZE;
3256	cb -= PAGE_SIZE;
3257	}
3258	/* won't ever get here. */
3259	}
3260
3261
3262	/**
3263	* Write to guest physical memory referenced by GC pointer.
3264	* Write memory to GC physical address in guest physical memory.
3265	*
3266	* This will bypass MMIO and access handlers.
3267	*
3268	* @returns VBox status code.
3269	* @param pVM The cross context VM structure.
3270	* @param GCPhysDst The GC physical address of the destination.
3271	* @param pvSrc The source buffer.
3272	* @param cb The number of bytes to write.
3273	*/
3274	VMMDECL(int) PGMPhysSimpleWriteGCPhys(PVM pVM, RTGCPHYS GCPhysDst, const void *pvSrc, size_t cb)
3275	{
3276	LogFlow(("PGMPhysSimpleWriteGCPhys: %RGp %zu\n", GCPhysDst, cb));
3277
3278	/*
3279	* Treat the first page as a special case.
3280	*/
3281	if (!cb)
3282	return VINF_SUCCESS;
3283
3284	/* map the 1st page */
3285	void *pvDst;
3286	PGMPAGEMAPLOCK Lock;
3287	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock);
3288	if (RT_FAILURE(rc))
3289	return rc;
3290
3291	/* optimize for the case where access is completely within the first page. */
3292	size_t cbPage = PAGE_SIZE - (GCPhysDst & PAGE_OFFSET_MASK);
3293	if (RT_LIKELY(cb <= cbPage))
3294	{
3295	memcpy(pvDst, pvSrc, cb);
3296	PGMPhysReleasePageMappingLock(pVM, &Lock);
3297	return VINF_SUCCESS;
3298	}
3299
3300	/* copy to the end of the page. */
3301	memcpy(pvDst, pvSrc, cbPage);
3302	PGMPhysReleasePageMappingLock(pVM, &Lock);
3303	GCPhysDst += cbPage;
3304	pvSrc = (const uint8_t *)pvSrc + cbPage;
3305	cb -= cbPage;
3306
3307	/*
3308	* Page by page.
3309	*/
3310	for (;;)
3311	{
3312	/* map the page */
3313	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock);
3314	if (RT_FAILURE(rc))
3315	return rc;
3316
3317	/* last page? */
3318	if (cb <= PAGE_SIZE)
3319	{
3320	memcpy(pvDst, pvSrc, cb);
3321	PGMPhysReleasePageMappingLock(pVM, &Lock);
3322	return VINF_SUCCESS;
3323	}
3324
3325	/* copy the entire page and advance */
3326	memcpy(pvDst, pvSrc, PAGE_SIZE);
3327	PGMPhysReleasePageMappingLock(pVM, &Lock);
3328	GCPhysDst += PAGE_SIZE;
3329	pvSrc = (const uint8_t *)pvSrc + PAGE_SIZE;
3330	cb -= PAGE_SIZE;
3331	}
3332	/* won't ever get here. */
3333	}
3334
3335
3336	/**
3337	* Read from guest physical memory referenced by GC pointer.
3338	*
3339	* This function uses the current CR3/CR0/CR4 of the guest and will
3340	* bypass access handlers and not set any accessed bits.
3341	*
3342	* @returns VBox status code.
3343	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
3344	* @param pvDst The destination address.
3345	* @param GCPtrSrc The source address (GC pointer).
3346	* @param cb The number of bytes to read.
3347	*/
3348	VMMDECL(int) PGMPhysSimpleReadGCPtr(PVMCPU pVCpu, void *pvDst, RTGCPTR GCPtrSrc, size_t cb)
3349	{
3350	PVM pVM = pVCpu->CTX_SUFF(pVM);
3351	/** @todo fix the macro / state handling: VMCPU_ASSERT_EMT_OR_GURU(pVCpu); */
3352
3353	/*
3354	* Treat the first page as a special case.
3355	*/
3356	if (!cb)
3357	return VINF_SUCCESS;
3358
3359	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysSimpleRead));
3360	STAM_COUNTER_ADD(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysSimpleReadBytes), cb);
3361
3362	/* Take the PGM lock here, because many called functions take the lock for a very short period. That's counter-productive
3363	* when many VCPUs are fighting for the lock.
3364	*/
3365	pgmLock(pVM);
3366
3367	/* map the 1st page */
3368	void const *pvSrc;
3369	PGMPAGEMAPLOCK Lock;
3370	int rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrSrc, &pvSrc, &Lock);
3371	if (RT_FAILURE(rc))
3372	{
3373	pgmUnlock(pVM);
3374	return rc;
3375	}
3376
3377	/* optimize for the case where access is completely within the first page. */
3378	size_t cbPage = PAGE_SIZE - ((RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK);
3379	if (RT_LIKELY(cb <= cbPage))
3380	{
3381	memcpy(pvDst, pvSrc, cb);
3382	PGMPhysReleasePageMappingLock(pVM, &Lock);
3383	pgmUnlock(pVM);
3384	return VINF_SUCCESS;
3385	}
3386
3387	/* copy to the end of the page. */
3388	memcpy(pvDst, pvSrc, cbPage);
3389	PGMPhysReleasePageMappingLock(pVM, &Lock);
3390	GCPtrSrc = (RTGCPTR)((RTGCUINTPTR)GCPtrSrc + cbPage);
3391	pvDst = (uint8_t *)pvDst + cbPage;
3392	cb -= cbPage;
3393
3394	/*
3395	* Page by page.
3396	*/
3397	for (;;)
3398	{
3399	/* map the page */
3400	rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrSrc, &pvSrc, &Lock);
3401	if (RT_FAILURE(rc))
3402	{
3403	pgmUnlock(pVM);
3404	return rc;
3405	}
3406
3407	/* last page? */
3408	if (cb <= PAGE_SIZE)
3409	{
3410	memcpy(pvDst, pvSrc, cb);
3411	PGMPhysReleasePageMappingLock(pVM, &Lock);
3412	pgmUnlock(pVM);
3413	return VINF_SUCCESS;
3414	}
3415
3416	/* copy the entire page and advance */
3417	memcpy(pvDst, pvSrc, PAGE_SIZE);
3418	PGMPhysReleasePageMappingLock(pVM, &Lock);
3419	GCPtrSrc = (RTGCPTR)((RTGCUINTPTR)GCPtrSrc + PAGE_SIZE);
3420	pvDst = (uint8_t *)pvDst + PAGE_SIZE;
3421	cb -= PAGE_SIZE;
3422	}
3423	/* won't ever get here. */
3424	}
3425
3426
3427	/**
3428	* Write to guest physical memory referenced by GC pointer.
3429	*
3430	* This function uses the current CR3/CR0/CR4 of the guest and will
3431	* bypass access handlers and not set dirty or accessed bits.
3432	*
3433	* @returns VBox status code.
3434	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
3435	* @param GCPtrDst The destination address (GC pointer).
3436	* @param pvSrc The source address.
3437	* @param cb The number of bytes to write.
3438	*/
3439	VMMDECL(int) PGMPhysSimpleWriteGCPtr(PVMCPU pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
3440	{
3441	PVM pVM = pVCpu->CTX_SUFF(pVM);
3442	VMCPU_ASSERT_EMT(pVCpu);
3443
3444	/*
3445	* Treat the first page as a special case.
3446	*/
3447	if (!cb)
3448	return VINF_SUCCESS;
3449
3450	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysSimpleWrite));
3451	STAM_COUNTER_ADD(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysSimpleWriteBytes), cb);
3452
3453	/* map the 1st page */
3454	void *pvDst;
3455	PGMPAGEMAPLOCK Lock;
3456	int rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
3457	if (RT_FAILURE(rc))
3458	return rc;
3459
3460	/* optimize for the case where access is completely within the first page. */
3461	size_t cbPage = PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK);
3462	if (RT_LIKELY(cb <= cbPage))
3463	{
3464	memcpy(pvDst, pvSrc, cb);
3465	PGMPhysReleasePageMappingLock(pVM, &Lock);
3466	return VINF_SUCCESS;
3467	}
3468
3469	/* copy to the end of the page. */
3470	memcpy(pvDst, pvSrc, cbPage);
3471	PGMPhysReleasePageMappingLock(pVM, &Lock);
3472	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + cbPage);
3473	pvSrc = (const uint8_t *)pvSrc + cbPage;
3474	cb -= cbPage;
3475
3476	/*
3477	* Page by page.
3478	*/
3479	for (;;)
3480	{
3481	/* map the page */
3482	rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
3483	if (RT_FAILURE(rc))
3484	return rc;
3485
3486	/* last page? */
3487	if (cb <= PAGE_SIZE)
3488	{
3489	memcpy(pvDst, pvSrc, cb);
3490	PGMPhysReleasePageMappingLock(pVM, &Lock);
3491	return VINF_SUCCESS;
3492	}
3493
3494	/* copy the entire page and advance */
3495	memcpy(pvDst, pvSrc, PAGE_SIZE);
3496	PGMPhysReleasePageMappingLock(pVM, &Lock);
3497	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + PAGE_SIZE);
3498	pvSrc = (const uint8_t *)pvSrc + PAGE_SIZE;
3499	cb -= PAGE_SIZE;
3500	}
3501	/* won't ever get here. */
3502	}
3503
3504
3505	/**
3506	* Write to guest physical memory referenced by GC pointer and update the PTE.
3507	*
3508	* This function uses the current CR3/CR0/CR4 of the guest and will
3509	* bypass access handlers but will set any dirty and accessed bits in the PTE.
3510	*
3511	* If you don't want to set the dirty bit, use PGMPhysSimpleWriteGCPtr().
3512	*
3513	* @returns VBox status code.
3514	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
3515	* @param GCPtrDst The destination address (GC pointer).
3516	* @param pvSrc The source address.
3517	* @param cb The number of bytes to write.
3518	*/
3519	VMMDECL(int) PGMPhysSimpleDirtyWriteGCPtr(PVMCPU pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
3520	{
3521	PVM pVM = pVCpu->CTX_SUFF(pVM);
3522	VMCPU_ASSERT_EMT(pVCpu);
3523
3524	/*
3525	* Treat the first page as a special case.
3526	* Btw. this is the same code as in PGMPhyssimpleWriteGCPtr excep for the PGMGstModifyPage.
3527	*/
3528	if (!cb)
3529	return VINF_SUCCESS;
3530
3531	/* map the 1st page */
3532	void *pvDst;
3533	PGMPAGEMAPLOCK Lock;
3534	int rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
3535	if (RT_FAILURE(rc))
3536	return rc;
3537
3538	/* optimize for the case where access is completely within the first page. */
3539	size_t cbPage = PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK);
3540	if (RT_LIKELY(cb <= cbPage))
3541	{
3542	memcpy(pvDst, pvSrc, cb);
3543	PGMPhysReleasePageMappingLock(pVM, &Lock);
3544	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
3545	return VINF_SUCCESS;
3546	}
3547
3548	/* copy to the end of the page. */
3549	memcpy(pvDst, pvSrc, cbPage);
3550	PGMPhysReleasePageMappingLock(pVM, &Lock);
3551	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
3552	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + cbPage);
3553	pvSrc = (const uint8_t *)pvSrc + cbPage;
3554	cb -= cbPage;
3555
3556	/*
3557	* Page by page.
3558	*/
3559	for (;;)
3560	{
3561	/* map the page */
3562	rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
3563	if (RT_FAILURE(rc))
3564	return rc;
3565
3566	/* last page? */
3567	if (cb <= PAGE_SIZE)
3568	{
3569	memcpy(pvDst, pvSrc, cb);
3570	PGMPhysReleasePageMappingLock(pVM, &Lock);
3571	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
3572	return VINF_SUCCESS;
3573	}
3574
3575	/* copy the entire page and advance */
3576	memcpy(pvDst, pvSrc, PAGE_SIZE);
3577	PGMPhysReleasePageMappingLock(pVM, &Lock);
3578	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
3579	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + PAGE_SIZE);
3580	pvSrc = (const uint8_t *)pvSrc + PAGE_SIZE;
3581	cb -= PAGE_SIZE;
3582	}
3583	/* won't ever get here. */
3584	}
3585
3586
3587	/**
3588	* Read from guest physical memory referenced by GC pointer.
3589	*
3590	* This function uses the current CR3/CR0/CR4 of the guest and will
3591	* respect access handlers and set accessed bits.
3592	*
3593	* @returns Strict VBox status, see PGMPhysRead for details.
3594	* @retval VERR_PAGE_TABLE_NOT_PRESENT if there is no page mapped at the
3595	* specified virtual address.
3596	*
3597	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
3598	* @param pvDst The destination address.
3599	* @param GCPtrSrc The source address (GC pointer).
3600	* @param cb The number of bytes to read.
3601	* @param enmOrigin Who is calling.
3602	* @thread EMT(pVCpu)
3603	*/
3604	VMMDECL(VBOXSTRICTRC) PGMPhysReadGCPtr(PVMCPU pVCpu, void *pvDst, RTGCPTR GCPtrSrc, size_t cb, PGMACCESSORIGIN enmOrigin)
3605	{
3606	RTGCPHYS GCPhys;
3607	uint64_t fFlags;
3608	int rc;
3609	PVM pVM = pVCpu->CTX_SUFF(pVM);
3610	VMCPU_ASSERT_EMT(pVCpu);
3611
3612	/*
3613	* Anything to do?
3614	*/
3615	if (!cb)
3616	return VINF_SUCCESS;
3617
3618	LogFlow(("PGMPhysReadGCPtr: %RGv %zu\n", GCPtrSrc, cb));
3619
3620	/*
3621	* Optimize reads within a single page.
3622	*/
3623	if (((RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK) + cb <= PAGE_SIZE)
3624	{
3625	/* Convert virtual to physical address + flags */
3626	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtrSrc, &fFlags, &GCPhys);
3627	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrSrc), rc);
3628	GCPhys \|= (RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK;
3629
3630	/* mark the guest page as accessed. */
3631	if (!(fFlags & X86_PTE_A))
3632	{
3633	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)(X86_PTE_A));
3634	AssertRC(rc);
3635	}
3636
3637	return PGMPhysRead(pVM, GCPhys, pvDst, cb, enmOrigin);
3638	}
3639
3640	/*
3641	* Page by page.
3642	*/
3643	for (;;)
3644	{
3645	/* Convert virtual to physical address + flags */
3646	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtrSrc, &fFlags, &GCPhys);
3647	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrSrc), rc);
3648	GCPhys \|= (RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK;
3649
3650	/* mark the guest page as accessed. */
3651	if (!(fFlags & X86_PTE_A))
3652	{
3653	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)(X86_PTE_A));
3654	AssertRC(rc);
3655	}
3656
3657	/* copy */
3658	size_t cbRead = PAGE_SIZE - ((RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK);
3659	if (cbRead < cb)
3660	{
3661	VBOXSTRICTRC rcStrict = PGMPhysRead(pVM, GCPhys, pvDst, cbRead, enmOrigin);
3662	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
3663	{ /* likely */ }
3664	else
3665	return rcStrict;
3666	}
3667	else /* Last page (cbRead is PAGE_SIZE, we only need cb!) */
3668	return PGMPhysRead(pVM, GCPhys, pvDst, cb, enmOrigin);
3669
3670	/* next */
3671	Assert(cb > cbRead);
3672	cb -= cbRead;
3673	pvDst = (uint8_t *)pvDst + cbRead;
3674	GCPtrSrc += cbRead;
3675	}
3676	}
3677
3678
3679	/**
3680	* Write to guest physical memory referenced by GC pointer.
3681	*
3682	* This function uses the current CR3/CR0/CR4 of the guest and will
3683	* respect access handlers and set dirty and accessed bits.
3684	*
3685	* @returns Strict VBox status, see PGMPhysWrite for details.
3686	* @retval VERR_PAGE_TABLE_NOT_PRESENT if there is no page mapped at the
3687	* specified virtual address.
3688	*
3689	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
3690	* @param GCPtrDst The destination address (GC pointer).
3691	* @param pvSrc The source address.
3692	* @param cb The number of bytes to write.
3693	* @param enmOrigin Who is calling.
3694	*/
3695	VMMDECL(VBOXSTRICTRC) PGMPhysWriteGCPtr(PVMCPU pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb, PGMACCESSORIGIN enmOrigin)
3696	{
3697	RTGCPHYS GCPhys;
3698	uint64_t fFlags;
3699	int rc;
3700	PVM pVM = pVCpu->CTX_SUFF(pVM);
3701	VMCPU_ASSERT_EMT(pVCpu);
3702
3703	/*
3704	* Anything to do?
3705	*/
3706	if (!cb)
3707	return VINF_SUCCESS;
3708
3709	LogFlow(("PGMPhysWriteGCPtr: %RGv %zu\n", GCPtrDst, cb));
3710
3711	/*
3712	* Optimize writes within a single page.
3713	*/
3714	if (((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK) + cb <= PAGE_SIZE)
3715	{
3716	/* Convert virtual to physical address + flags */
3717	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtrDst, &fFlags, &GCPhys);
3718	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrDst), rc);
3719	GCPhys \|= (RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK;
3720
3721	/* Mention when we ignore X86_PTE_RW... */
3722	if (!(fFlags & X86_PTE_RW))
3723	Log(("PGMPhysWriteGCPtr: Writing to RO page %RGv %#x\n", GCPtrDst, cb));
3724
3725	/* Mark the guest page as accessed and dirty if necessary. */
3726	if ((fFlags & (X86_PTE_A \| X86_PTE_D)) != (X86_PTE_A \| X86_PTE_D))
3727	{
3728	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D));
3729	AssertRC(rc);
3730	}
3731
3732	return PGMPhysWrite(pVM, GCPhys, pvSrc, cb, enmOrigin);
3733	}
3734
3735	/*
3736	* Page by page.
3737	*/
3738	for (;;)
3739	{
3740	/* Convert virtual to physical address + flags */
3741	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtrDst, &fFlags, &GCPhys);
3742	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrDst), rc);
3743	GCPhys \|= (RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK;
3744
3745	/* Mention when we ignore X86_PTE_RW... */
3746	if (!(fFlags & X86_PTE_RW))
3747	Log(("PGMPhysWriteGCPtr: Writing to RO page %RGv %#x\n", GCPtrDst, cb));
3748
3749	/* Mark the guest page as accessed and dirty if necessary. */
3750	if ((fFlags & (X86_PTE_A \| X86_PTE_D)) != (X86_PTE_A \| X86_PTE_D))
3751	{
3752	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D));
3753	AssertRC(rc);
3754	}
3755
3756	/* copy */
3757	size_t cbWrite = PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK);
3758	if (cbWrite < cb)
3759	{
3760	VBOXSTRICTRC rcStrict = PGMPhysWrite(pVM, GCPhys, pvSrc, cbWrite, enmOrigin);
3761	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
3762	{ /* likely */ }
3763	else
3764	return rcStrict;
3765	}
3766	else /* Last page (cbWrite is PAGE_SIZE, we only need cb!) */
3767	return PGMPhysWrite(pVM, GCPhys, pvSrc, cb, enmOrigin);
3768
3769	/* next */
3770	Assert(cb > cbWrite);
3771	cb -= cbWrite;
3772	pvSrc = (uint8_t *)pvSrc + cbWrite;
3773	GCPtrDst += cbWrite;
3774	}
3775	}
3776
3777
3778	/**
3779	* Performs a read of guest virtual memory for instruction emulation.
3780	*
3781	* This will check permissions, raise exceptions and update the access bits.
3782	*
3783	* The current implementation will bypass all access handlers. It may later be
3784	* changed to at least respect MMIO.
3785	*
3786	*
3787	* @returns VBox status code suitable to scheduling.
3788	* @retval VINF_SUCCESS if the read was performed successfully.
3789	* @retval VINF_EM_RAW_GUEST_TRAP if an exception was raised but not dispatched yet.
3790	* @retval VINF_TRPM_XCPT_DISPATCHED if an exception was raised and dispatched.
3791	*
3792	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
3793	* @param pCtxCore The context core.
3794	* @param pvDst Where to put the bytes we've read.
3795	* @param GCPtrSrc The source address.
3796	* @param cb The number of bytes to read. Not more than a page.
3797	*
3798	* @remark This function will dynamically map physical pages in GC. This may unmap
3799	* mappings done by the caller. Be careful!
3800	*/
3801	VMMDECL(int) PGMPhysInterpretedRead(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, void *pvDst, RTGCUINTPTR GCPtrSrc, size_t cb)
3802	{
3803	PVM pVM = pVCpu->CTX_SUFF(pVM);
3804	Assert(cb <= PAGE_SIZE);
3805	VMCPU_ASSERT_EMT(pVCpu);
3806
3807	/** @todo r=bird: This isn't perfect!
3808	* -# It's not checking for reserved bits being 1.
3809	* -# It's not correctly dealing with the access bit.
3810	* -# It's not respecting MMIO memory or any other access handlers.
3811	*/
3812	/*
3813	* 1. Translate virtual to physical. This may fault.
3814	* 2. Map the physical address.
3815	* 3. Do the read operation.
3816	* 4. Set access bits if required.
3817	*/
3818	int rc;
3819	unsigned cb1 = PAGE_SIZE - (GCPtrSrc & PAGE_OFFSET_MASK);
3820	if (cb <= cb1)
3821	{
3822	/*
3823	* Not crossing pages.
3824	*/
3825	RTGCPHYS GCPhys;
3826	uint64_t fFlags;
3827	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc, &fFlags, &GCPhys);
3828	if (RT_SUCCESS(rc))
3829	{
3830	/** @todo we should check reserved bits ... */
3831	PGMPAGEMAPLOCK PgMpLck;
3832	void const *pvSrc;
3833	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, &pvSrc, &PgMpLck);
3834	switch (rc)
3835	{
3836	case VINF_SUCCESS:
3837	Log(("PGMPhysInterpretedRead: pvDst=%p pvSrc=%p cb=%d\n", pvDst, (uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb));
3838	memcpy(pvDst, (uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb);
3839	PGMPhysReleasePageMappingLock(pVM, &PgMpLck);
3840	break;
3841	case VERR_PGM_PHYS_PAGE_RESERVED:
3842	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3843	memset(pvDst, 0xff, cb);
3844	break;
3845	default:
3846	Assert(RT_FAILURE_NP(rc));
3847	return rc;
3848	}
3849
3850	/** @todo access bit emulation isn't 100% correct. */
3851	if (!(fFlags & X86_PTE_A))
3852	{
3853	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3854	AssertRC(rc);
3855	}
3856	return VINF_SUCCESS;
3857	}
3858	}
3859	else
3860	{
3861	/*
3862	* Crosses pages.
3863	*/
3864	size_t cb2 = cb - cb1;
3865	uint64_t fFlags1;
3866	RTGCPHYS GCPhys1;
3867	uint64_t fFlags2;
3868	RTGCPHYS GCPhys2;
3869	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc, &fFlags1, &GCPhys1);
3870	if (RT_SUCCESS(rc))
3871	{
3872	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc + cb1, &fFlags2, &GCPhys2);
3873	if (RT_SUCCESS(rc))
3874	{
3875	/** @todo we should check reserved bits ... */
3876	AssertMsgFailed(("cb=%d cb1=%d cb2=%d GCPtrSrc=%RGv\n", cb, cb1, cb2, GCPtrSrc));
3877	PGMPAGEMAPLOCK PgMpLck;
3878	void const *pvSrc1;
3879	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys1, &pvSrc1, &PgMpLck);
3880	switch (rc)
3881	{
3882	case VINF_SUCCESS:
3883	memcpy(pvDst, (uint8_t *)pvSrc1 + (GCPtrSrc & PAGE_OFFSET_MASK), cb1);
3884	PGMPhysReleasePageMappingLock(pVM, &PgMpLck);
3885	break;
3886	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3887	memset(pvDst, 0xff, cb1);
3888	break;
3889	default:
3890	Assert(RT_FAILURE_NP(rc));
3891	return rc;
3892	}
3893
3894	void const *pvSrc2;
3895	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys2, &pvSrc2, &PgMpLck);
3896	switch (rc)
3897	{
3898	case VINF_SUCCESS:
3899	memcpy((uint8_t *)pvDst + cb1, pvSrc2, cb2);
3900	PGMPhysReleasePageMappingLock(pVM, &PgMpLck);
3901	break;
3902	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3903	memset((uint8_t *)pvDst + cb1, 0xff, cb2);
3904	break;
3905	default:
3906	Assert(RT_FAILURE_NP(rc));
3907	return rc;
3908	}
3909
3910	if (!(fFlags1 & X86_PTE_A))
3911	{
3912	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3913	AssertRC(rc);
3914	}
3915	if (!(fFlags2 & X86_PTE_A))
3916	{
3917	rc = PGMGstModifyPage(pVCpu, GCPtrSrc + cb1, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3918	AssertRC(rc);
3919	}
3920	return VINF_SUCCESS;
3921	}
3922	}
3923	}
3924
3925	/*
3926	* Raise a #PF.
3927	*/
3928	uint32_t uErr;
3929
3930	/* Get the current privilege level. */
3931	uint32_t cpl = CPUMGetGuestCPL(pVCpu);
3932	switch (rc)
3933	{
3934	case VINF_SUCCESS:
3935	uErr = (cpl >= 2) ? X86_TRAP_PF_RSVD \| X86_TRAP_PF_US : X86_TRAP_PF_RSVD;
3936	break;
3937
3938	case VERR_PAGE_NOT_PRESENT:
3939	case VERR_PAGE_TABLE_NOT_PRESENT:
3940	uErr = (cpl >= 2) ? X86_TRAP_PF_US : 0;
3941	break;
3942
3943	default:
3944	AssertMsgFailed(("rc=%Rrc GCPtrSrc=%RGv cb=%#x\n", rc, GCPtrSrc, cb));
3945	return rc;
3946	}
3947	Log(("PGMPhysInterpretedRead: GCPtrSrc=%RGv cb=%#x -> #PF(%#x)\n", GCPtrSrc, cb, uErr));
3948	return TRPMRaiseXcptErrCR2(pVCpu, pCtxCore, X86_XCPT_PF, uErr, GCPtrSrc);
3949	}
3950
3951
3952	/**
3953	* Performs a read of guest virtual memory for instruction emulation.
3954	*
3955	* This will check permissions, raise exceptions and update the access bits.
3956	*
3957	* The current implementation will bypass all access handlers. It may later be
3958	* changed to at least respect MMIO.
3959	*
3960	*
3961	* @returns VBox status code suitable to scheduling.
3962	* @retval VINF_SUCCESS if the read was performed successfully.
3963	* @retval VINF_EM_RAW_GUEST_TRAP if an exception was raised but not dispatched yet.
3964	* @retval VINF_TRPM_XCPT_DISPATCHED if an exception was raised and dispatched.
3965	*
3966	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
3967	* @param pCtxCore The context core.
3968	* @param pvDst Where to put the bytes we've read.
3969	* @param GCPtrSrc The source address.
3970	* @param cb The number of bytes to read. Not more than a page.
3971	* @param fRaiseTrap If set the trap will be raised on as per spec, if clear
3972	* an appropriate error status will be returned (no
3973	* informational at all).
3974	*
3975	*
3976	* @remarks Takes the PGM lock.
3977	* @remarks A page fault on the 2nd page of the access will be raised without
3978	* writing the bits on the first page since we're ASSUMING that the
3979	* caller is emulating an instruction access.
3980	* @remarks This function will dynamically map physical pages in GC. This may
3981	* unmap mappings done by the caller. Be careful!
3982	*/
3983	VMMDECL(int) PGMPhysInterpretedReadNoHandlers(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, void *pvDst, RTGCUINTPTR GCPtrSrc, size_t cb,
3984	bool fRaiseTrap)
3985	{
3986	PVM pVM = pVCpu->CTX_SUFF(pVM);
3987	Assert(cb <= PAGE_SIZE);
3988	VMCPU_ASSERT_EMT(pVCpu);
3989
3990	/*
3991	* 1. Translate virtual to physical. This may fault.
3992	* 2. Map the physical address.
3993	* 3. Do the read operation.
3994	* 4. Set access bits if required.
3995	*/
3996	int rc;
3997	unsigned cb1 = PAGE_SIZE - (GCPtrSrc & PAGE_OFFSET_MASK);
3998	if (cb <= cb1)
3999	{
4000	/*
4001	* Not crossing pages.
4002	*/
4003	RTGCPHYS GCPhys;
4004	uint64_t fFlags;
4005	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc, &fFlags, &GCPhys);
4006	if (RT_SUCCESS(rc))
4007	{
4008	if (1) /** @todo we should check reserved bits ... */
4009	{
4010	const void *pvSrc;
4011	PGMPAGEMAPLOCK Lock;
4012	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, &pvSrc, &Lock);
4013	switch (rc)
4014	{
4015	case VINF_SUCCESS:
4016	Log(("PGMPhysInterpretedReadNoHandlers: pvDst=%p pvSrc=%p (%RGv) cb=%d\n",
4017	pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), GCPtrSrc, cb));
4018	memcpy(pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb);
4019	PGMPhysReleasePageMappingLock(pVM, &Lock);
4020	break;
4021	case VERR_PGM_PHYS_PAGE_RESERVED:
4022	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
4023	memset(pvDst, 0xff, cb);
4024	break;
4025	default:
4026	AssertMsgFailed(("%Rrc\n", rc));
4027	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
4028	return rc;
4029	}
4030
4031	if (!(fFlags & X86_PTE_A))
4032	{
4033	/** @todo access bit emulation isn't 100% correct. */
4034	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
4035	AssertRC(rc);
4036	}
4037	return VINF_SUCCESS;
4038	}
4039	}
4040	}
4041	else
4042	{
4043	/*
4044	* Crosses pages.
4045	*/
4046	size_t cb2 = cb - cb1;
4047	uint64_t fFlags1;
4048	RTGCPHYS GCPhys1;
4049	uint64_t fFlags2;
4050	RTGCPHYS GCPhys2;
4051	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc, &fFlags1, &GCPhys1);
4052	if (RT_SUCCESS(rc))
4053	{
4054	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc + cb1, &fFlags2, &GCPhys2);
4055	if (RT_SUCCESS(rc))
4056	{
4057	if (1) /** @todo we should check reserved bits ... */
4058	{
4059	const void *pvSrc;
4060	PGMPAGEMAPLOCK Lock;
4061	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys1, &pvSrc, &Lock);
4062	switch (rc)
4063	{
4064	case VINF_SUCCESS:
4065	Log(("PGMPhysInterpretedReadNoHandlers: pvDst=%p pvSrc=%p (%RGv) cb=%d [2]\n",
4066	pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), GCPtrSrc, cb1));
4067	memcpy(pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb1);
4068	PGMPhysReleasePageMappingLock(pVM, &Lock);
4069	break;
4070	case VERR_PGM_PHYS_PAGE_RESERVED:
4071	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
4072	memset(pvDst, 0xff, cb1);
4073	break;
4074	default:
4075	AssertMsgFailed(("%Rrc\n", rc));
4076	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
4077	return rc;
4078	}
4079
4080	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys2, &pvSrc, &Lock);
4081	switch (rc)
4082	{
4083	case VINF_SUCCESS:
4084	memcpy((uint8_t *)pvDst + cb1, pvSrc, cb2);
4085	PGMPhysReleasePageMappingLock(pVM, &Lock);
4086	break;
4087	case VERR_PGM_PHYS_PAGE_RESERVED:
4088	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
4089	memset((uint8_t *)pvDst + cb1, 0xff, cb2);
4090	break;
4091	default:
4092	AssertMsgFailed(("%Rrc\n", rc));
4093	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
4094	return rc;
4095	}
4096
4097	if (!(fFlags1 & X86_PTE_A))
4098	{
4099	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
4100	AssertRC(rc);
4101	}
4102	if (!(fFlags2 & X86_PTE_A))
4103	{
4104	rc = PGMGstModifyPage(pVCpu, GCPtrSrc + cb1, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
4105	AssertRC(rc);
4106	}
4107	return VINF_SUCCESS;
4108	}
4109	/* sort out which page */
4110	}
4111	else
4112	GCPtrSrc += cb1; /* fault on 2nd page */
4113	}
4114	}
4115
4116	/*
4117	* Raise a #PF if we're allowed to do that.
4118	*/
4119	/* Calc the error bits. */
4120	uint32_t cpl = CPUMGetGuestCPL(pVCpu);
4121	uint32_t uErr;
4122	switch (rc)
4123	{
4124	case VINF_SUCCESS:
4125	uErr = (cpl >= 2) ? X86_TRAP_PF_RSVD \| X86_TRAP_PF_US : X86_TRAP_PF_RSVD;
4126	rc = VERR_ACCESS_DENIED;
4127	break;
4128
4129	case VERR_PAGE_NOT_PRESENT:
4130	case VERR_PAGE_TABLE_NOT_PRESENT:
4131	uErr = (cpl >= 2) ? X86_TRAP_PF_US : 0;
4132	break;
4133
4134	default:
4135	AssertMsgFailed(("rc=%Rrc GCPtrSrc=%RGv cb=%#x\n", rc, GCPtrSrc, cb));
4136	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
4137	return rc;
4138	}
4139	if (fRaiseTrap)
4140	{
4141	Log(("PGMPhysInterpretedReadNoHandlers: GCPtrSrc=%RGv cb=%#x -> Raised #PF(%#x)\n", GCPtrSrc, cb, uErr));
4142	return TRPMRaiseXcptErrCR2(pVCpu, pCtxCore, X86_XCPT_PF, uErr, GCPtrSrc);
4143	}
4144	Log(("PGMPhysInterpretedReadNoHandlers: GCPtrSrc=%RGv cb=%#x -> #PF(%#x) [!raised]\n", GCPtrSrc, cb, uErr));
4145	return rc;
4146	}
4147
4148
4149	/**
4150	* Performs a write to guest virtual memory for instruction emulation.
4151	*
4152	* This will check permissions, raise exceptions and update the dirty and access
4153	* bits.
4154	*
4155	* @returns VBox status code suitable to scheduling.
4156	* @retval VINF_SUCCESS if the read was performed successfully.
4157	* @retval VINF_EM_RAW_GUEST_TRAP if an exception was raised but not dispatched yet.
4158	* @retval VINF_TRPM_XCPT_DISPATCHED if an exception was raised and dispatched.
4159	*
4160	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
4161	* @param pCtxCore The context core.
4162	* @param GCPtrDst The destination address.
4163	* @param pvSrc What to write.
4164	* @param cb The number of bytes to write. Not more than a page.
4165	* @param fRaiseTrap If set the trap will be raised on as per spec, if clear
4166	* an appropriate error status will be returned (no
4167	* informational at all).
4168	*
4169	* @remarks Takes the PGM lock.
4170	* @remarks A page fault on the 2nd page of the access will be raised without
4171	* writing the bits on the first page since we're ASSUMING that the
4172	* caller is emulating an instruction access.
4173	* @remarks This function will dynamically map physical pages in GC. This may
4174	* unmap mappings done by the caller. Be careful!
4175	*/
4176	VMMDECL(int) PGMPhysInterpretedWriteNoHandlers(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, RTGCPTR GCPtrDst, const void *pvSrc,
4177	size_t cb, bool fRaiseTrap)
4178	{
4179	Assert(cb <= PAGE_SIZE);
4180	PVM pVM = pVCpu->CTX_SUFF(pVM);
4181	VMCPU_ASSERT_EMT(pVCpu);
4182
4183	/*
4184	* 1. Translate virtual to physical. This may fault.
4185	* 2. Map the physical address.
4186	* 3. Do the write operation.
4187	* 4. Set access bits if required.
4188	*/
4189	/** @todo Since this method is frequently used by EMInterpret or IOM
4190	* upon a write fault to an write access monitored page, we can
4191	* reuse the guest page table walking from the \#PF code. */
4192	int rc;
4193	unsigned cb1 = PAGE_SIZE - (GCPtrDst & PAGE_OFFSET_MASK);
4194	if (cb <= cb1)
4195	{
4196	/*
4197	* Not crossing pages.
4198	*/
4199	RTGCPHYS GCPhys;
4200	uint64_t fFlags;
4201	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrDst, &fFlags, &GCPhys);
4202	if (RT_SUCCESS(rc))
4203	{
4204	if ( (fFlags & X86_PTE_RW) /** @todo Also check reserved bits. */
4205	\|\| ( !(CPUMGetGuestCR0(pVCpu) & X86_CR0_WP)
4206	&& CPUMGetGuestCPL(pVCpu) <= 2) ) /** @todo it's 2, right? Check cpl check below as well. */
4207	{
4208	void *pvDst;
4209	PGMPAGEMAPLOCK Lock;
4210	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys, &pvDst, &Lock);
4211	switch (rc)
4212	{
4213	case VINF_SUCCESS:
4214	Log(("PGMPhysInterpretedWriteNoHandlers: pvDst=%p (%RGv) pvSrc=%p cb=%d\n",
4215	(uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), GCPtrDst, pvSrc, cb));
4216	memcpy((uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), pvSrc, cb);
4217	PGMPhysReleasePageMappingLock(pVM, &Lock);
4218	break;
4219	case VERR_PGM_PHYS_PAGE_RESERVED:
4220	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
4221	/* bit bucket */
4222	break;
4223	default:
4224	AssertMsgFailed(("%Rrc\n", rc));
4225	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
4226	return rc;
4227	}
4228
4229	if (!(fFlags & (X86_PTE_A \| X86_PTE_D)))
4230	{
4231	/** @todo dirty & access bit emulation isn't 100% correct. */
4232	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D));
4233	AssertRC(rc);
4234	}
4235	return VINF_SUCCESS;
4236	}
4237	rc = VERR_ACCESS_DENIED;
4238	}
4239	}
4240	else
4241	{
4242	/*
4243	* Crosses pages.
4244	*/
4245	size_t cb2 = cb - cb1;
4246	uint64_t fFlags1;
4247	RTGCPHYS GCPhys1;
4248	uint64_t fFlags2;
4249	RTGCPHYS GCPhys2;
4250	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrDst, &fFlags1, &GCPhys1);
4251	if (RT_SUCCESS(rc))
4252	{
4253	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrDst + cb1, &fFlags2, &GCPhys2);
4254	if (RT_SUCCESS(rc))
4255	{
4256	if ( ( (fFlags1 & X86_PTE_RW) /** @todo Also check reserved bits. */
4257	&& (fFlags2 & X86_PTE_RW))
4258	\|\| ( !(CPUMGetGuestCR0(pVCpu) & X86_CR0_WP)
4259	&& CPUMGetGuestCPL(pVCpu) <= 2) )
4260	{
4261	void *pvDst;
4262	PGMPAGEMAPLOCK Lock;
4263	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys1, &pvDst, &Lock);
4264	switch (rc)
4265	{
4266	case VINF_SUCCESS:
4267	Log(("PGMPhysInterpretedWriteNoHandlers: pvDst=%p (%RGv) pvSrc=%p cb=%d\n",
4268	(uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), GCPtrDst, pvSrc, cb1));
4269	memcpy((uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), pvSrc, cb1);
4270	PGMPhysReleasePageMappingLock(pVM, &Lock);
4271	break;
4272	case VERR_PGM_PHYS_PAGE_RESERVED:
4273	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
4274	/* bit bucket */
4275	break;
4276	default:
4277	AssertMsgFailed(("%Rrc\n", rc));
4278	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
4279	return rc;
4280	}
4281
4282	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys2, &pvDst, &Lock);
4283	switch (rc)
4284	{
4285	case VINF_SUCCESS:
4286	memcpy(pvDst, (const uint8_t *)pvSrc + cb1, cb2);
4287	PGMPhysReleasePageMappingLock(pVM, &Lock);
4288	break;
4289	case VERR_PGM_PHYS_PAGE_RESERVED:
4290	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
4291	/* bit bucket */
4292	break;
4293	default:
4294	AssertMsgFailed(("%Rrc\n", rc));
4295	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
4296	return rc;
4297	}
4298
4299	if (!(fFlags1 & (X86_PTE_A \| X86_PTE_RW)))
4300	{
4301	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, (X86_PTE_A \| X86_PTE_RW), ~(uint64_t)(X86_PTE_A \| X86_PTE_RW));
4302	AssertRC(rc);
4303	}
4304	if (!(fFlags2 & (X86_PTE_A \| X86_PTE_RW)))
4305	{
4306	rc = PGMGstModifyPage(pVCpu, GCPtrDst + cb1, 1, (X86_PTE_A \| X86_PTE_RW), ~(uint64_t)(X86_PTE_A \| X86_PTE_RW));
4307	AssertRC(rc);
4308	}
4309	return VINF_SUCCESS;
4310	}
4311	if ((fFlags1 & (X86_PTE_RW)) == X86_PTE_RW)
4312	GCPtrDst += cb1; /* fault on the 2nd page. */
4313	rc = VERR_ACCESS_DENIED;
4314	}
4315	else
4316	GCPtrDst += cb1; /* fault on the 2nd page. */
4317	}
4318	}
4319
4320	/*
4321	* Raise a #PF if we're allowed to do that.
4322	*/
4323	/* Calc the error bits. */
4324	uint32_t uErr;
4325	uint32_t cpl = CPUMGetGuestCPL(pVCpu);
4326	switch (rc)
4327	{
4328	case VINF_SUCCESS:
4329	uErr = (cpl >= 2) ? X86_TRAP_PF_RSVD \| X86_TRAP_PF_US : X86_TRAP_PF_RSVD;
4330	rc = VERR_ACCESS_DENIED;
4331	break;
4332
4333	case VERR_ACCESS_DENIED:
4334	uErr = (cpl >= 2) ? X86_TRAP_PF_RW \| X86_TRAP_PF_US : X86_TRAP_PF_RW;
4335	break;
4336
4337	case VERR_PAGE_NOT_PRESENT:
4338	case VERR_PAGE_TABLE_NOT_PRESENT:
4339	uErr = (cpl >= 2) ? X86_TRAP_PF_US : 0;
4340	break;
4341
4342	default:
4343	AssertMsgFailed(("rc=%Rrc GCPtrDst=%RGv cb=%#x\n", rc, GCPtrDst, cb));
4344	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
4345	return rc;
4346	}
4347	if (fRaiseTrap)
4348	{
4349	Log(("PGMPhysInterpretedWriteNoHandlers: GCPtrDst=%RGv cb=%#x -> Raised #PF(%#x)\n", GCPtrDst, cb, uErr));
4350	return TRPMRaiseXcptErrCR2(pVCpu, pCtxCore, X86_XCPT_PF, uErr, GCPtrDst);
4351	}
4352	Log(("PGMPhysInterpretedWriteNoHandlers: GCPtrDst=%RGv cb=%#x -> #PF(%#x) [!raised]\n", GCPtrDst, cb, uErr));
4353	return rc;
4354	}
4355
4356
4357	/**
4358	* Return the page type of the specified physical address.
4359	*
4360	* @returns The page type.
4361	* @param pVM The cross context VM structure.
4362	* @param GCPhys Guest physical address
4363	*/
4364	VMM_INT_DECL(PGMPAGETYPE) PGMPhysGetPageType(PVM pVM, RTGCPHYS GCPhys)
4365	{
4366	pgmLock(pVM);
4367	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
4368	PGMPAGETYPE enmPgType = pPage ? (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage) : PGMPAGETYPE_INVALID;
4369	pgmUnlock(pVM);
4370
4371	return enmPgType;
4372	}
4373
4374
4375
4376
4377	/**
4378	* Converts a GC physical address to a HC ring-3 pointer, with some
4379	* additional checks.
4380	*
4381	* @returns VBox status code (no informational statuses).
4382	* @retval VINF_SUCCESS on success.
4383	* @retval VERR_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
4384	* access handler of some kind.
4385	* @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
4386	* accesses or is odd in any way.
4387	* @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
4388	*
4389	* @param pVM The cross context VM structure.
4390	* @param pVCpu The cross context virtual CPU structure of the
4391	* calling EMT.
4392	* @param GCPhys The GC physical address to convert. This API mask
4393	* the A20 line when necessary.
4394	* @param fWritable Whether write access is required.
4395	* @param fByPassHandlers Whether to bypass access handlers.
4396	* @param ppv Where to store the pointer corresponding to GCPhys
4397	* on success.
4398	* @param pLock
4399	*
4400	* @remarks This is more or a less a copy of PGMR3PhysTlbGCPhys2Ptr.
4401	* @thread EMT(pVCpu).
4402	*/
4403	VMM_INT_DECL(int) PGMPhysIemGCPhys2Ptr(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, bool fWritable, bool fByPassHandlers,
4404	void **ppv, PPGMPAGEMAPLOCK pLock)
4405	{
4406	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys);
4407
4408	pgmLock(pVM);
4409
4410	PPGMRAMRANGE pRam;
4411	PPGMPAGE pPage;
4412	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
4413	if (RT_SUCCESS(rc))
4414	{
4415	if (PGM_PAGE_IS_BALLOONED(pPage))
4416	rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
4417	else if (PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
4418	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4419	else if ( !PGM_PAGE_HAS_ANY_HANDLERS(pPage)
4420	\|\| (fByPassHandlers && !PGM_PAGE_IS_MMIO(pPage)) )
4421	rc = VINF_SUCCESS;
4422	else
4423	{
4424	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
4425	{
4426	Assert(!fByPassHandlers \|\| PGM_PAGE_IS_MMIO(pPage));
4427	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4428	}
4429	else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && fWritable)
4430	{
4431	Assert(!fByPassHandlers);
4432	rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
4433	}
4434	}
4435	if (RT_SUCCESS(rc))
4436	{
4437	int rc2;
4438
4439	/* Make sure what we return is writable. */
4440	if (fWritable)
4441	switch (PGM_PAGE_GET_STATE(pPage))
4442	{
4443	case PGM_PAGE_STATE_ALLOCATED:
4444	break;
4445	case PGM_PAGE_STATE_BALLOONED:
4446	AssertFailed();
4447	case PGM_PAGE_STATE_ZERO:
4448	case PGM_PAGE_STATE_SHARED:
4449	case PGM_PAGE_STATE_WRITE_MONITORED:
4450	rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
4451	AssertLogRelRCReturn(rc2, rc2);
4452	break;
4453	}
4454
4455	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4456	void *pv;
4457	rc = pgmRZDynMapHCPageInlined(pVCpu,
4458	PGM_PAGE_GET_HCPHYS(pPage),
4459	&pv
4460	RTLOG_COMMA_SRC_POS);
4461	if (RT_FAILURE(rc))
4462	return rc;
4463	ppv = (void )((uintptr_t)pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
4464	pLock->pvPage = pv;
4465	pLock->pVCpu = pVCpu;
4466
4467	#else
4468	/* Get a ring-3 mapping of the address. */
4469	PPGMPAGER3MAPTLBE pTlbe;
4470	rc2 = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
4471	AssertLogRelRCReturn(rc2, rc2);
4472
4473	/* Lock it and calculate the address. */
4474	if (fWritable)
4475	pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
4476	else
4477	pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
4478	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
4479	#endif
4480
4481	Log6(("PGMPhysIemGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage] ppv=%p\n", GCPhys, rc, pPage, ppv));
4482	}
4483	else
4484	Log6(("PGMPhysIemGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage]\n", GCPhys, rc, pPage));
4485
4486	/* else: handler catching all access, no pointer returned. */
4487	}
4488	else
4489	rc = VERR_PGM_PHYS_TLB_UNASSIGNED;
4490
4491	pgmUnlock(pVM);
4492	return rc;
4493	}
4494
4495
4496	/**
4497	* Checks if the give GCPhys page requires special handling for the given access
4498	* because it's MMIO or otherwise monitored.
4499	*
4500	* @returns VBox status code (no informational statuses).
4501	* @retval VINF_SUCCESS on success.
4502	* @retval VERR_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
4503	* access handler of some kind.
4504	* @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
4505	* accesses or is odd in any way.
4506	* @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
4507	*
4508	* @param pVM The cross context VM structure.
4509	* @param GCPhys The GC physical address to convert. Since this is
4510	* only used for filling the REM TLB, the A20 mask must
4511	* be applied before calling this API.
4512	* @param fWritable Whether write access is required.
4513	* @param fByPassHandlers Whether to bypass access handlers.
4514	*
4515	* @remarks This is a watered down version PGMPhysIemGCPhys2Ptr and really just
4516	* a stop gap thing that should be removed once there is a better TLB
4517	* for virtual address accesses.
4518	*/
4519	VMM_INT_DECL(int) PGMPhysIemQueryAccess(PVM pVM, RTGCPHYS GCPhys, bool fWritable, bool fByPassHandlers)
4520	{
4521	pgmLock(pVM);
4522	PGM_A20_ASSERT_MASKED(VMMGetCpu(pVM), GCPhys);
4523
4524	PPGMRAMRANGE pRam;
4525	PPGMPAGE pPage;
4526	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
4527	if (RT_SUCCESS(rc))
4528	{
4529	if (PGM_PAGE_IS_BALLOONED(pPage))
4530	rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
4531	else if (PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
4532	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4533	else if ( !PGM_PAGE_HAS_ANY_HANDLERS(pPage)
4534	\|\| (fByPassHandlers && !PGM_PAGE_IS_MMIO(pPage)) )
4535	rc = VINF_SUCCESS;
4536	else
4537	{
4538	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
4539	{
4540	Assert(!fByPassHandlers \|\| PGM_PAGE_IS_MMIO(pPage));
4541	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4542	}
4543	else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && fWritable)
4544	{
4545	Assert(!fByPassHandlers);
4546	rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
4547	}
4548	}
4549	}
4550
4551	pgmUnlock(pVM);
4552	return rc;
4553	}
4554

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllPhys.cpp@ 58437

Download in other formats: