IOMAllMmioNew.cpp@ 97145

Last change on this file since 97145 was 96407, checked in by vboxsync, 2 years ago
scm copyright and license note update
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 50.8 KB

Line
1	/* $Id: IOMAllMmioNew.cpp 96407 2022-08-22 17:43:14Z vboxsync $ */
2	/** @file
3	* IOM - Input / Output Monitor - Any Context, MMIO & String I/O.
4	*/
5
6	/*
7	* Copyright (C) 2006-2022 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_IOM_MMIO
33	#define VMCPU_INCL_CPUM_GST_CTX
34	#include <VBox/vmm/iom.h>
35	#include <VBox/vmm/cpum.h>
36	#include <VBox/vmm/pgm.h>
37	#include <VBox/vmm/selm.h>
38	#include <VBox/vmm/mm.h>
39	#include <VBox/vmm/em.h>
40	#include <VBox/vmm/pgm.h>
41	#include <VBox/vmm/trpm.h>
42	#include <VBox/vmm/iem.h>
43	#include "IOMInternal.h"
44	#include <VBox/vmm/vmcc.h>
45	#include <VBox/vmm/vmm.h>
46	#include <VBox/vmm/hm.h>
47	#include "IOMInline.h"
48
49	#include <VBox/dis.h>
50	#include <VBox/disopcode.h>
51	#include <VBox/vmm/pdmdev.h>
52	#include <VBox/param.h>
53	#include <VBox/err.h>
54	#include <iprt/assert.h>
55	#include <VBox/log.h>
56	#include <iprt/asm.h>
57	#include <iprt/string.h>
58
59
60	/*********************************************************************************************************************************
61	* Defined Constants And Macros *
62	*********************************************************************************************************************************/
63	/** @def IOM_MMIO_STATS_COMMA_DECL
64	* Parameter list declaration for statistics entry pointer. */
65	/** @def IOM_MMIO_STATS_COMMA_ARG
66	* Statistics entry pointer argument. */
67	#if defined(VBOX_WITH_STATISTICS) \|\| defined(DOXYGEN_RUNNING)
68	# define IOM_MMIO_STATS_COMMA_DECL , PIOMMMIOSTATSENTRY pStats
69	# define IOM_MMIO_STATS_COMMA_ARG , pStats
70	#else
71	# define IOM_MMIO_STATS_COMMA_DECL
72	# define IOM_MMIO_STATS_COMMA_ARG
73	#endif
74
75
76
77	#ifndef IN_RING3
78	/**
79	* Defers a pending MMIO write to ring-3.
80	*
81	* @returns VINF_IOM_R3_MMIO_COMMIT_WRITE
82	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
83	* @param GCPhys The write address.
84	* @param pvBuf The bytes being written.
85	* @param cbBuf How many bytes.
86	* @param idxRegEntry The MMIO registration index (handle) if available,
87	* otherwise UINT32_MAX.
88	*/
89	static VBOXSTRICTRC iomMmioRing3WritePending(PVMCPU pVCpu, RTGCPHYS GCPhys, void const *pvBuf, size_t cbBuf,
90	uint32_t idxRegEntry)
91	{
92	Log5(("iomMmioRing3WritePending: %RGp LB %#x (idx=%#x)\n", GCPhys, cbBuf, idxRegEntry));
93	if (pVCpu->iom.s.PendingMmioWrite.cbValue == 0)
94	{
95	pVCpu->iom.s.PendingMmioWrite.GCPhys = GCPhys;
96	AssertReturn(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue), VERR_IOM_MMIO_IPE_2);
97	pVCpu->iom.s.PendingMmioWrite.cbValue = (uint32_t)cbBuf;
98	pVCpu->iom.s.PendingMmioWrite.idxMmioRegionHint = idxRegEntry;
99	memcpy(pVCpu->iom.s.PendingMmioWrite.abValue, pvBuf, cbBuf);
100	}
101	else
102	{
103	/*
104	* Join with pending if adjecent.
105	*
106	* This may happen if the stack overflows into MMIO territory and RSP/ESP/SP
107	* isn't aligned. IEM will bounce buffer the access and do one write for each
108	* page. We get here when the 2nd page part is written.
109	*/
110	uint32_t const cbOldValue = pVCpu->iom.s.PendingMmioWrite.cbValue;
111	AssertMsgReturn(GCPhys == pVCpu->iom.s.PendingMmioWrite.GCPhys + cbOldValue,
112	("pending %RGp LB %#x; incoming %RGp LB %#x\n",
113	pVCpu->iom.s.PendingMmioWrite.GCPhys, cbOldValue, GCPhys, cbBuf),
114	VERR_IOM_MMIO_IPE_1);
115	AssertReturn(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue) - cbOldValue, VERR_IOM_MMIO_IPE_2);
116	pVCpu->iom.s.PendingMmioWrite.cbValue = cbOldValue + (uint32_t)cbBuf;
117	memcpy(&pVCpu->iom.s.PendingMmioWrite.abValue[cbOldValue], pvBuf, cbBuf);
118	}
119
120	VMCPU_FF_SET(pVCpu, VMCPU_FF_IOM);
121	return VINF_IOM_R3_MMIO_COMMIT_WRITE;
122	}
123	#endif
124
125
126	/**
127	* Deals with complicated MMIO writes.
128	*
129	* Complicated means unaligned or non-dword/qword sized accesses depending on
130	* the MMIO region's access mode flags.
131	*
132	* @returns Strict VBox status code. Any EM scheduling status code,
133	* VINF_IOM_R3_MMIO_WRITE, VINF_IOM_R3_MMIO_READ_WRITE or
134	* VINF_IOM_R3_MMIO_READ may be returned.
135	*
136	* @param pVM The cross context VM structure.
137	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
138	* @param pRegEntry The MMIO entry for the current context.
139	* @param GCPhys The physical address to start writing.
140	* @param offRegion MMIO region offset corresponding to @a GCPhys.
141	* @param pvValue Where to store the value.
142	* @param cbValue The size of the value to write.
143	* @param pStats Pointer to the statistics (never NULL).
144	*/
145	static VBOXSTRICTRC iomMmioDoComplicatedWrite(PVM pVM, PVMCPU pVCpu, CTX_SUFF(PIOMMMIOENTRY) pRegEntry,
146	RTGCPHYS GCPhys, RTGCPHYS offRegion,
147	void const *pvValue, unsigned cbValue IOM_MMIO_STATS_COMMA_DECL)
148	{
149	AssertReturn( (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) != IOMMMIO_FLAGS_WRITE_PASSTHRU
150	&& (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) <= IOMMMIO_FLAGS_WRITE_DWORD_QWORD_READ_MISSING,
151	VERR_IOM_MMIO_IPE_1);
152	AssertReturn(cbValue != 0 && cbValue <= 16, VERR_IOM_MMIO_IPE_2);
153	RTGCPHYS const GCPhysStart = GCPhys; NOREF(GCPhysStart);
154	bool const fReadMissing = (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_DWORD_READ_MISSING
155	\|\| (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_DWORD_QWORD_READ_MISSING;
156	RT_NOREF_PV(pVCpu); /* ring-3 */
157
158	/*
159	* Do debug stop if requested.
160	*/
161	VBOXSTRICTRC rc = VINF_SUCCESS; NOREF(pVM);
162	#ifdef VBOX_STRICT
163	if (!(pRegEntry->fFlags & IOMMMIO_FLAGS_DBGSTOP_ON_COMPLICATED_WRITE))
164	{ /* likely */ }
165	else
166	{
167	# ifdef IN_RING3
168	LogRel(("IOM: Complicated write %#x byte at %RGp to %s, initiating debugger intervention\n", cbValue, GCPhys,
169	R3STRING(pRegEntry->pszDesc)));
170	rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, RT_SRC_POS,
171	"Complicated write %#x byte at %RGp to %s\n", cbValue, GCPhys, pRegEntry->pszDesc);
172	if (rc == VERR_DBGF_NOT_ATTACHED)
173	rc = VINF_SUCCESS;
174	# else
175	return VINF_IOM_R3_MMIO_WRITE;
176	# endif
177	}
178	#endif
179
180	STAM_COUNTER_INC(&pStats->ComplicatedWrites);
181
182	/*
183	* Check if we should ignore the write.
184	*/
185	if ((pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_ONLY_DWORD)
186	{
187	Assert(cbValue != 4 \|\| (GCPhys & 3));
188	return VINF_SUCCESS;
189	}
190	if ((pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_ONLY_DWORD_QWORD)
191	{
192	Assert((cbValue != 4 && cbValue != 8) \|\| (GCPhys & (cbValue - 1)));
193	return VINF_SUCCESS;
194	}
195
196	/*
197	* Split and conquer.
198	*/
199	for (;;)
200	{
201	unsigned const offAccess = GCPhys & 3;
202	unsigned cbThisPart = 4 - offAccess;
203	if (cbThisPart > cbValue)
204	cbThisPart = cbValue;
205
206	/*
207	* Get the missing bits (if any).
208	*/
209	uint32_t u32MissingValue = 0;
210	if (fReadMissing && cbThisPart != 4)
211	{
212	VBOXSTRICTRC rc2 = pRegEntry->pfnReadCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
213	!(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS)
214	? offRegion & ~(RTGCPHYS)3 : (GCPhys & ~(RTGCPHYS)3),
215	&u32MissingValue, sizeof(u32MissingValue));
216	switch (VBOXSTRICTRC_VAL(rc2))
217	{
218	case VINF_SUCCESS:
219	break;
220	case VINF_IOM_MMIO_UNUSED_FF:
221	STAM_COUNTER_INC(&pStats->FFor00Reads);
222	u32MissingValue = UINT32_C(0xffffffff);
223	break;
224	case VINF_IOM_MMIO_UNUSED_00:
225	STAM_COUNTER_INC(&pStats->FFor00Reads);
226	u32MissingValue = 0;
227	break;
228	#ifndef IN_RING3
229	case VINF_IOM_R3_MMIO_READ:
230	case VINF_IOM_R3_MMIO_READ_WRITE:
231	case VINF_IOM_R3_MMIO_WRITE:
232	LogFlow(("iomMmioDoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [read]\n", GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rc2)));
233	rc2 = iomMmioRing3WritePending(pVCpu, GCPhys, pvValue, cbValue, pRegEntry->idxSelf);
234	if (rc == VINF_SUCCESS \|\| rc2 < rc)
235	rc = rc2;
236	return rc;
237	#endif
238	default:
239	if (RT_FAILURE(rc2))
240	{
241	Log(("iomMmioDoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [read]\n", GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rc2)));
242	return rc2;
243	}
244	AssertMsgReturn(rc2 >= VINF_EM_FIRST && rc2 <= VINF_EM_LAST, ("%Rrc\n", VBOXSTRICTRC_VAL(rc2)), VERR_IPE_UNEXPECTED_INFO_STATUS);
245	if (rc == VINF_SUCCESS \|\| rc2 < rc)
246	rc = rc2;
247	break;
248	}
249	}
250
251	/*
252	* Merge missing and given bits.
253	*/
254	uint32_t u32GivenMask;
255	uint32_t u32GivenValue;
256	switch (cbThisPart)
257	{
258	case 1:
259	u32GivenValue = (uint8_t const )pvValue;
260	u32GivenMask = UINT32_C(0x000000ff);
261	break;
262	case 2:
263	u32GivenValue = (uint16_t const )pvValue;
264	u32GivenMask = UINT32_C(0x0000ffff);
265	break;
266	case 3:
267	u32GivenValue = RT_MAKE_U32_FROM_U8(((uint8_t const )pvValue)[0], ((uint8_t const )pvValue)[1],
268	((uint8_t const *)pvValue)[2], 0);
269	u32GivenMask = UINT32_C(0x00ffffff);
270	break;
271	case 4:
272	u32GivenValue = (uint32_t const )pvValue;
273	u32GivenMask = UINT32_C(0xffffffff);
274	break;
275	default:
276	AssertFailedReturn(VERR_IOM_MMIO_IPE_3);
277	}
278	if (offAccess)
279	{
280	u32GivenValue <<= offAccess * 8;
281	u32GivenMask <<= offAccess * 8;
282	}
283
284	uint32_t u32Value = (u32MissingValue & ~u32GivenMask)
285	\| (u32GivenValue & u32GivenMask);
286
287	/*
288	* Do DWORD write to the device.
289	*/
290	VBOXSTRICTRC rc2 = pRegEntry->pfnWriteCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
291	!(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS)
292	? offRegion & ~(RTGCPHYS)3 : GCPhys & ~(RTGCPHYS)3,
293	&u32Value, sizeof(u32Value));
294	switch (VBOXSTRICTRC_VAL(rc2))
295	{
296	case VINF_SUCCESS:
297	break;
298	#ifndef IN_RING3
299	case VINF_IOM_R3_MMIO_READ:
300	case VINF_IOM_R3_MMIO_READ_WRITE:
301	case VINF_IOM_R3_MMIO_WRITE:
302	Log3(("iomMmioDoComplicatedWrite: deferring GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [write]\n", GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rc2)));
303	AssertReturn(pVCpu->iom.s.PendingMmioWrite.cbValue == 0, VERR_IOM_MMIO_IPE_1);
304	AssertReturn(cbValue + (GCPhys & 3) <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue), VERR_IOM_MMIO_IPE_2);
305	pVCpu->iom.s.PendingMmioWrite.GCPhys = GCPhys & ~(RTGCPHYS)3;
306	pVCpu->iom.s.PendingMmioWrite.cbValue = cbValue + (GCPhys & 3);
307	(uint32_t )pVCpu->iom.s.PendingMmioWrite.abValue = u32Value;
308	if (cbValue > cbThisPart)
309	memcpy(&pVCpu->iom.s.PendingMmioWrite.abValue[4],
310	(uint8_t const *)pvValue + cbThisPart, cbValue - cbThisPart);
311	VMCPU_FF_SET(pVCpu, VMCPU_FF_IOM);
312	if (rc == VINF_SUCCESS)
313	rc = VINF_IOM_R3_MMIO_COMMIT_WRITE;
314	return rc;
315	#endif
316	default:
317	if (RT_FAILURE(rc2))
318	{
319	Log(("iomMmioDoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [write]\n", GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rc2)));
320	return rc2;
321	}
322	AssertMsgReturn(rc2 >= VINF_EM_FIRST && rc2 <= VINF_EM_LAST, ("%Rrc\n", VBOXSTRICTRC_VAL(rc2)), VERR_IPE_UNEXPECTED_INFO_STATUS);
323	if (rc == VINF_SUCCESS \|\| rc2 < rc)
324	rc = rc2;
325	break;
326	}
327
328	/*
329	* Advance.
330	*/
331	cbValue -= cbThisPart;
332	if (!cbValue)
333	break;
334	GCPhys += cbThisPart;
335	offRegion += cbThisPart;
336	pvValue = (uint8_t const *)pvValue + cbThisPart;
337	}
338
339	return rc;
340	}
341
342
343
344
345	/**
346	* Wrapper which does the write.
347	*/
348	DECLINLINE(VBOXSTRICTRC) iomMmioDoWrite(PVMCC pVM, PVMCPU pVCpu, CTX_SUFF(PIOMMMIOENTRY) pRegEntry,
349	RTGCPHYS GCPhys, RTGCPHYS offRegion,
350	const void *pvData, uint32_t cb IOM_MMIO_STATS_COMMA_DECL)
351	{
352	VBOXSTRICTRC rcStrict;
353	if (RT_LIKELY(pRegEntry->pfnWriteCallback))
354	{
355	if ( (cb == 4 && !(GCPhys & 3))
356	\|\| (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_PASSTHRU
357	\|\| (cb == 8 && !(GCPhys & 7) && IOMMMIO_DOES_WRITE_MODE_ALLOW_QWORD(pRegEntry->fFlags)) )
358	rcStrict = pRegEntry->pfnWriteCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
359	!(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS) ? offRegion : GCPhys, pvData, cb);
360	else
361	rcStrict = iomMmioDoComplicatedWrite(pVM, pVCpu, pRegEntry, GCPhys, offRegion, pvData, cb IOM_MMIO_STATS_COMMA_ARG);
362	}
363	else
364	rcStrict = VINF_SUCCESS;
365	return rcStrict;
366	}
367
368
369	#ifdef IN_RING3
370	/**
371	* Helper for IOMR3ProcessForceFlag() that lives here to utilize iomMmioDoWrite et al.
372	*/
373	VBOXSTRICTRC iomR3MmioCommitWorker(PVM pVM, PVMCPU pVCpu, PIOMMMIOENTRYR3 pRegEntry, RTGCPHYS offRegion)
374	{
375	# ifdef VBOX_WITH_STATISTICS
376	STAM_PROFILE_START(UnusedMacroArg, Prf);
377	PIOMMMIOSTATSENTRY const pStats = iomMmioGetStats(pVM, pRegEntry);
378	# endif
379	PPDMDEVINS const pDevIns = pRegEntry->pDevIns;
380	int rc = PDMCritSectEnter(pVM, pDevIns->CTX_SUFF(pCritSectRo), VERR_IGNORED);
381	AssertRCReturn(rc, rc);
382
383	VBOXSTRICTRC rcStrict = iomMmioDoWrite(pVM, pVCpu, pRegEntry, pVCpu->iom.s.PendingMmioWrite.GCPhys, offRegion,
384	pVCpu->iom.s.PendingMmioWrite.abValue, pVCpu->iom.s.PendingMmioWrite.cbValue
385	IOM_MMIO_STATS_COMMA_ARG);
386
387	PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo));
388	STAM_PROFILE_STOP(&pStats->ProfWriteR3, Prf);
389	return rcStrict;
390	}
391	#endif /* IN_RING3 */
392
393
394	/**
395	* Deals with complicated MMIO reads.
396	*
397	* Complicated means unaligned or non-dword/qword sized accesses depending on
398	* the MMIO region's access mode flags.
399	*
400	* @returns Strict VBox status code. Any EM scheduling status code,
401	* VINF_IOM_R3_MMIO_READ, VINF_IOM_R3_MMIO_READ_WRITE or
402	* VINF_IOM_R3_MMIO_WRITE may be returned.
403	*
404	* @param pVM The cross context VM structure.
405	* @param pRegEntry The MMIO entry for the current context.
406	* @param GCPhys The physical address to start reading.
407	* @param offRegion MMIO region offset corresponding to @a GCPhys.
408	* @param pvValue Where to store the value.
409	* @param cbValue The size of the value to read.
410	* @param pStats Pointer to the statistics (never NULL).
411	*/
412	static VBOXSTRICTRC iomMMIODoComplicatedRead(PVM pVM, CTX_SUFF(PIOMMMIOENTRY) pRegEntry, RTGCPHYS GCPhys, RTGCPHYS offRegion,
413	void *pvValue, unsigned cbValue IOM_MMIO_STATS_COMMA_DECL)
414	{
415	AssertReturn( (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD
416	\|\| (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD_QWORD,
417	VERR_IOM_MMIO_IPE_1);
418	AssertReturn(cbValue != 0 && cbValue <= 16, VERR_IOM_MMIO_IPE_2);
419	#ifdef LOG_ENABLED
420	RTGCPHYS const GCPhysStart = GCPhys;
421	#endif
422
423	/*
424	* Do debug stop if requested.
425	*/
426	VBOXSTRICTRC rc = VINF_SUCCESS; NOREF(pVM);
427	#ifdef VBOX_STRICT
428	if (pRegEntry->fFlags & IOMMMIO_FLAGS_DBGSTOP_ON_COMPLICATED_READ)
429	{
430	# ifdef IN_RING3
431	rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, RT_SRC_POS,
432	"Complicated read %#x byte at %RGp to %s\n", cbValue, GCPhys, R3STRING(pRegEntry->pszDesc));
433	if (rc == VERR_DBGF_NOT_ATTACHED)
434	rc = VINF_SUCCESS;
435	# else
436	return VINF_IOM_R3_MMIO_READ;
437	# endif
438	}
439	#endif
440
441	STAM_COUNTER_INC(&pStats->ComplicatedReads);
442
443	/*
444	* Split and conquer.
445	*/
446	for (;;)
447	{
448	/*
449	* Do DWORD read from the device.
450	*/
451	uint32_t u32Value;
452	VBOXSTRICTRC rcStrict2 = pRegEntry->pfnReadCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
453	!(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS)
454	? offRegion & ~(RTGCPHYS)3 : GCPhys & ~(RTGCPHYS)3,
455	&u32Value, sizeof(u32Value));
456	switch (VBOXSTRICTRC_VAL(rcStrict2))
457	{
458	case VINF_SUCCESS:
459	break;
460	case VINF_IOM_MMIO_UNUSED_FF:
461	STAM_COUNTER_INC(&pStats->FFor00Reads);
462	u32Value = UINT32_C(0xffffffff);
463	break;
464	case VINF_IOM_MMIO_UNUSED_00:
465	STAM_COUNTER_INC(&pStats->FFor00Reads);
466	u32Value = 0;
467	break;
468	case VINF_IOM_R3_MMIO_READ:
469	case VINF_IOM_R3_MMIO_READ_WRITE:
470	case VINF_IOM_R3_MMIO_WRITE:
471	/** @todo What if we've split a transfer and already read
472	* something? Since reads can have sideeffects we could be
473	* kind of screwed here... */
474	LogFlow(("iomMMIODoComplicatedRead: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rcStrict2=%Rrc\n",
475	GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rcStrict2)));
476	return rcStrict2;
477	default:
478	if (RT_FAILURE(rcStrict2))
479	{
480	Log(("iomMMIODoComplicatedRead: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rcStrict2=%Rrc\n",
481	GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rcStrict2)));
482	return rcStrict2;
483	}
484	AssertMsgReturn(rcStrict2 >= VINF_EM_FIRST && rcStrict2 <= VINF_EM_LAST, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)),
485	VERR_IPE_UNEXPECTED_INFO_STATUS);
486	if (rc == VINF_SUCCESS \|\| rcStrict2 < rc)
487	rc = rcStrict2;
488	break;
489	}
490	u32Value >>= (GCPhys & 3) * 8;
491
492	/*
493	* Write what we've read.
494	*/
495	unsigned cbThisPart = 4 - (GCPhys & 3);
496	if (cbThisPart > cbValue)
497	cbThisPart = cbValue;
498
499	switch (cbThisPart)
500	{
501	case 1:
502	(uint8_t )pvValue = (uint8_t)u32Value;
503	break;
504	case 2:
505	(uint16_t )pvValue = (uint16_t)u32Value;
506	break;
507	case 3:
508	((uint8_t *)pvValue)[0] = RT_BYTE1(u32Value);
509	((uint8_t *)pvValue)[1] = RT_BYTE2(u32Value);
510	((uint8_t *)pvValue)[2] = RT_BYTE3(u32Value);
511	break;
512	case 4:
513	(uint32_t )pvValue = u32Value;
514	break;
515	}
516
517	/*
518	* Advance.
519	*/
520	cbValue -= cbThisPart;
521	if (!cbValue)
522	break;
523	GCPhys += cbThisPart;
524	offRegion += cbThisPart;
525	pvValue = (uint8_t *)pvValue + cbThisPart;
526	}
527
528	return rc;
529	}
530
531
532	/**
533	* Implements VINF_IOM_MMIO_UNUSED_FF.
534	*
535	* @returns VINF_SUCCESS.
536	* @param pvValue Where to store the zeros.
537	* @param cbValue How many bytes to read.
538	* @param pStats Pointer to the statistics (never NULL).
539	*/
540	static int iomMMIODoReadFFs(void *pvValue, size_t cbValue IOM_MMIO_STATS_COMMA_DECL)
541	{
542	switch (cbValue)
543	{
544	case 1: (uint8_t )pvValue = UINT8_C(0xff); break;
545	case 2: (uint16_t )pvValue = UINT16_C(0xffff); break;
546	case 4: (uint32_t )pvValue = UINT32_C(0xffffffff); break;
547	case 8: (uint64_t )pvValue = UINT64_C(0xffffffffffffffff); break;
548	default:
549	{
550	uint8_t pb = (uint8_t )pvValue;
551	while (cbValue--)
552	*pb++ = UINT8_C(0xff);
553	break;
554	}
555	}
556	STAM_COUNTER_INC(&pStats->FFor00Reads);
557	return VINF_SUCCESS;
558	}
559
560
561	/**
562	* Implements VINF_IOM_MMIO_UNUSED_00.
563	*
564	* @returns VINF_SUCCESS.
565	* @param pvValue Where to store the zeros.
566	* @param cbValue How many bytes to read.
567	* @param pStats Pointer to the statistics (never NULL).
568	*/
569	static int iomMMIODoRead00s(void *pvValue, size_t cbValue IOM_MMIO_STATS_COMMA_DECL)
570	{
571	switch (cbValue)
572	{
573	case 1: (uint8_t )pvValue = UINT8_C(0x00); break;
574	case 2: (uint16_t )pvValue = UINT16_C(0x0000); break;
575	case 4: (uint32_t )pvValue = UINT32_C(0x00000000); break;
576	case 8: (uint64_t )pvValue = UINT64_C(0x0000000000000000); break;
577	default:
578	{
579	uint8_t pb = (uint8_t )pvValue;
580	while (cbValue--)
581	*pb++ = UINT8_C(0x00);
582	break;
583	}
584	}
585	STAM_COUNTER_INC(&pStats->FFor00Reads);
586	return VINF_SUCCESS;
587	}
588
589
590	/**
591	* Wrapper which does the read.
592	*/
593	DECLINLINE(VBOXSTRICTRC) iomMmioDoRead(PVMCC pVM, CTX_SUFF(PIOMMMIOENTRY) pRegEntry, RTGCPHYS GCPhys, RTGCPHYS offRegion,
594	void *pvValue, uint32_t cbValue IOM_MMIO_STATS_COMMA_DECL)
595	{
596	VBOXSTRICTRC rcStrict;
597	if (RT_LIKELY(pRegEntry->pfnReadCallback))
598	{
599	if ( ( cbValue == 4
600	&& !(GCPhys & 3))
601	\|\| (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_PASSTHRU
602	\|\| ( cbValue == 8
603	&& !(GCPhys & 7)
604	&& (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD_QWORD ) )
605	rcStrict = pRegEntry->pfnReadCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
606	!(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS) ? offRegion : GCPhys, pvValue, cbValue);
607	else
608	rcStrict = iomMMIODoComplicatedRead(pVM, pRegEntry, GCPhys, offRegion, pvValue, cbValue IOM_MMIO_STATS_COMMA_ARG);
609	}
610	else
611	rcStrict = VINF_IOM_MMIO_UNUSED_FF;
612	if (rcStrict != VINF_SUCCESS)
613	{
614	switch (VBOXSTRICTRC_VAL(rcStrict))
615	{
616	case VINF_IOM_MMIO_UNUSED_FF: rcStrict = iomMMIODoReadFFs(pvValue, cbValue IOM_MMIO_STATS_COMMA_ARG); break;
617	case VINF_IOM_MMIO_UNUSED_00: rcStrict = iomMMIODoRead00s(pvValue, cbValue IOM_MMIO_STATS_COMMA_ARG); break;
618	}
619	}
620	return rcStrict;
621	}
622
623	#ifndef IN_RING3
624
625	/**
626	* Checks if we can handle an MMIO \#PF in R0/RC.
627	*/
628	DECLINLINE(bool) iomMmioCanHandlePfInRZ(PVMCC pVM, uint32_t uErrorCode, CTX_SUFF(PIOMMMIOENTRY) pRegEntry)
629	{
630	if (pRegEntry->cbRegion > 0)
631	{
632	if ( pRegEntry->pfnWriteCallback
633	&& pRegEntry->pfnReadCallback)
634	return true;
635
636	PIOMMMIOENTRYR3 const pRegEntryR3 = &pVM->iomr0.s.paMmioRing3Regs[pRegEntry->idxSelf];
637	if ( uErrorCode == UINT32_MAX
638	? pRegEntryR3->pfnWriteCallback \|\| pRegEntryR3->pfnReadCallback
639	: uErrorCode & X86_TRAP_PF_RW
640	? !pRegEntry->pfnWriteCallback && pRegEntryR3->pfnWriteCallback
641	: !pRegEntry->pfnReadCallback && pRegEntryR3->pfnReadCallback)
642	return false;
643
644	return true;
645	}
646	return false;
647	}
648
649
650	/**
651	* Common worker for the \#PF handler and IOMMMIOPhysHandler (APIC+VT-x).
652	*
653	* @returns VBox status code (appropriate for GC return).
654	* @param pVM The cross context VM structure.
655	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
656	* @param uErrorCode CPU Error code. This is UINT32_MAX when we don't have
657	* any error code (the EPT misconfig hack).
658	* @param GCPhysFault The GC physical address corresponding to pvFault.
659	* @param pRegEntry The MMIO entry for the current context.
660	*/
661	DECLINLINE(VBOXSTRICTRC) iomMmioCommonPfHandlerNew(PVMCC pVM, PVMCPUCC pVCpu, uint32_t uErrorCode,
662	RTGCPHYS GCPhysFault, CTX_SUFF(PIOMMMIOENTRY) pRegEntry)
663	{
664	Log(("iomMmioCommonPfHandler: GCPhysFault=%RGp uErr=%#x rip=%RGv\n", GCPhysFault, uErrorCode, CPUMGetGuestRIP(pVCpu) ));
665	RT_NOREF(GCPhysFault, uErrorCode);
666
667	VBOXSTRICTRC rcStrict;
668
669	#ifndef IN_RING3
670	/*
671	* Should we defer the request right away? This isn't usually the case, so
672	* do the simple test first and the try deal with uErrorCode being N/A.
673	*/
674	PPDMDEVINS const pDevIns = pRegEntry->pDevIns;
675	if (RT_LIKELY( pDevIns
676	&& iomMmioCanHandlePfInRZ(pVM, uErrorCode, pRegEntry)))
677	{
678	/*
679	* Enter the device critsect prior to engaging IOM in case of lock contention.
680	* Note! Perhaps not a good move?
681	*/
682	rcStrict = PDMCritSectEnter(pVM, pDevIns->CTX_SUFF(pCritSectRo), VINF_IOM_R3_MMIO_READ_WRITE);
683	if (rcStrict == VINF_SUCCESS)
684	{
685	#endif /* !IN_RING3 */
686
687	/*
688	* Let IEM call us back via iomMmioHandler.
689	*/
690	rcStrict = IEMExecOne(pVCpu);
691
692	#ifndef IN_RING3
693	PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo));
694	#endif
695	if (RT_SUCCESS(rcStrict))
696	{ /* likely */ }
697	else if ( rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED
698	\|\| rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
699	{
700	Log(("IOM: Hit unsupported IEM feature!\n"));
701	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
702	}
703	#ifndef IN_RING3
704	return rcStrict;
705	}
706	STAM_COUNTER_INC(&pVM->iom.s.StatMmioDevLockContentionR0);
707	}
708	else
709	rcStrict = VINF_IOM_R3_MMIO_READ_WRITE;
710
711	# ifdef VBOX_WITH_STATISTICS
712	if (rcStrict == VINF_IOM_R3_MMIO_READ_WRITE)
713	{
714	PIOMMMIOSTATSENTRY const pStats = iomMmioGetStats(pVM, pRegEntry);
715	if (uErrorCode & X86_TRAP_PF_RW)
716	{
717	STAM_COUNTER_INC(&pStats->WriteRZToR3);
718	STAM_COUNTER_INC(&pVM->iom.s.StatMmioWritesR0ToR3);
719	}
720	else
721	{
722	STAM_COUNTER_INC(&pStats->ReadRZToR3);
723	STAM_COUNTER_INC(&pVM->iom.s.StatMmioReadsR0ToR3);
724	}
725	}
726	# endif
727	#else /* IN_RING3 */
728	RT_NOREF(pVM, pRegEntry);
729	#endif /* IN_RING3 */
730	return rcStrict;
731	}
732
733
734	/**
735	* @callback_method_impl{FNPGMRZPHYSPFHANDLER,
736	* \#PF access handler callback for MMIO pages.}
737	*
738	* @remarks The @a uUser argument is the MMIO handle.
739	*/
740	DECLCALLBACK(VBOXSTRICTRC) iomMmioPfHandlerNew(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTXCORE pCtxCore,
741	RTGCPTR pvFault, RTGCPHYS GCPhysFault, uint64_t uUser)
742	{
743	STAM_PROFILE_START(&pVM->iom.s.StatMmioPfHandler, Prf);
744	LogFlow(("iomMmioPfHandlerNew: GCPhys=%RGp uErr=%#x pvFault=%RGv rip=%RGv\n",
745	GCPhysFault, (uint32_t)uErrorCode, pvFault, (RTGCPTR)pCtxCore->rip));
746	RT_NOREF(pvFault, pCtxCore);
747
748	/* Translate the MMIO handle to a registration entry for the current context. */
749	AssertReturn(uUser < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
750	# ifdef IN_RING0
751	AssertReturn(uUser < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
752	CTX_SUFF(PIOMMMIOENTRY) pRegEntry = &pVM->iomr0.s.paMmioRegs[uUser];
753	# else
754	CTX_SUFF(PIOMMMIOENTRY) pRegEntry = &pVM->iom.s.paMmioRegs[uUser];
755	# endif
756
757	VBOXSTRICTRC rcStrict = iomMmioCommonPfHandlerNew(pVM, pVCpu, (uint32_t)uErrorCode, GCPhysFault, pRegEntry);
758
759	STAM_PROFILE_STOP(&pVM->iom.s.StatMmioPfHandler, Prf);
760	return rcStrict;
761	}
762
763	#endif /* !IN_RING3 */
764
765	#ifdef IN_RING0
766	/**
767	* Physical access handler for MMIO ranges.
768	*
769	* This is actually only used by VT-x for APIC page accesses.
770	*
771	* @returns VBox status code (appropriate for GC return).
772	* @param pVM The cross context VM structure.
773	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
774	* @param uErrorCode CPU Error code.
775	* @param GCPhysFault The GC physical address.
776	*/
777	VMM_INT_DECL(VBOXSTRICTRC) IOMR0MmioPhysHandler(PVMCC pVM, PVMCPUCC pVCpu, uint32_t uErrorCode, RTGCPHYS GCPhysFault)
778	{
779	STAM_PROFILE_START(&pVM->iom.s.StatMmioPhysHandler, Prf);
780
781	/*
782	* We don't have a range here, so look it up before calling the common function.
783	*/
784	VBOXSTRICTRC rcStrict = IOM_LOCK_SHARED(pVM);
785	if (RT_SUCCESS(rcStrict))
786	{
787	RTGCPHYS offRegion;
788	CTX_SUFF(PIOMMMIOENTRY) pRegEntry = iomMmioGetEntry(pVM, GCPhysFault, &offRegion, &pVCpu->iom.s.idxMmioLastPhysHandler);
789	IOM_UNLOCK_SHARED(pVM);
790	if (RT_LIKELY(pRegEntry))
791	rcStrict = iomMmioCommonPfHandlerNew(pVM, pVCpu, (uint32_t)uErrorCode, GCPhysFault, pRegEntry);
792	else
793	rcStrict = VERR_IOM_MMIO_RANGE_NOT_FOUND;
794	}
795	else if (rcStrict == VERR_SEM_BUSY)
796	rcStrict = VINF_IOM_R3_MMIO_READ_WRITE;
797
798	STAM_PROFILE_STOP(&pVM->iom.s.StatMmioPhysHandler, Prf);
799	return rcStrict;
800	}
801	#endif /* IN_RING0 */
802
803
804	/**
805	* @callback_method_impl{FNPGMPHYSHANDLER, MMIO page accesses}
806	*
807	* @remarks The @a uUser argument is the MMIO handle.
808	*/
809	DECLCALLBACK(VBOXSTRICTRC) iomMmioHandlerNew(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysFault, void pvPhys, void pvBuf,
810	size_t cbBuf, PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, uint64_t uUser)
811	{
812	STAM_PROFILE_START(UnusedMacroArg, Prf);
813	STAM_COUNTER_INC(&pVM->iom.s.CTX_SUFF(StatMmioHandler));
814	Log4(("iomMmioHandlerNew: GCPhysFault=%RGp cbBuf=%#x enmAccessType=%d enmOrigin=%d uUser=%p\n", GCPhysFault, cbBuf, enmAccessType, enmOrigin, uUser));
815
816	Assert(enmAccessType == PGMACCESSTYPE_READ \|\| enmAccessType == PGMACCESSTYPE_WRITE);
817	AssertMsg(cbBuf >= 1, ("%zu\n", cbBuf));
818	NOREF(pvPhys); NOREF(enmOrigin);
819
820	#ifdef IN_RING3
821	int const rcToRing3 = VERR_IOM_MMIO_IPE_3;
822	#else
823	int const rcToRing3 = enmAccessType == PGMACCESSTYPE_READ ? VINF_IOM_R3_MMIO_READ : VINF_IOM_R3_MMIO_WRITE;
824	#endif
825
826	/*
827	* Translate uUser to an MMIO registration table entry. We can do this
828	* without any locking as the data is static after VM creation.
829	*/
830	AssertReturn(uUser < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
831	#ifdef IN_RING0
832	AssertReturn(uUser < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
833	CTX_SUFF(PIOMMMIOENTRY) const pRegEntry = &pVM->iomr0.s.paMmioRegs[uUser];
834	PIOMMMIOENTRYR3 const pRegEntryR3 = &pVM->iomr0.s.paMmioRing3Regs[uUser];
835	#else
836	CTX_SUFF(PIOMMMIOENTRY) const pRegEntry = &pVM->iom.s.paMmioRegs[uUser];
837	#endif
838	#ifdef VBOX_WITH_STATISTICS
839	PIOMMMIOSTATSENTRY const pStats = iomMmioGetStats(pVM, pRegEntry); /* (Works even without ring-0 device setup.) */
840	#endif
841	PPDMDEVINS const pDevIns = pRegEntry->pDevIns;
842
843	#ifdef VBOX_STRICT
844	/*
845	* Assert the right entry in strict builds. This may yield a false positive
846	* for SMP VMs if we're unlucky and the guest isn't well behaved.
847	*/
848	# ifdef IN_RING0
849	Assert(pRegEntry && (GCPhysFault - pRegEntryR3->GCPhysMapping < pRegEntryR3->cbRegion \|\| !pRegEntryR3->fMapped));
850	# else
851	Assert(pRegEntry && (GCPhysFault - pRegEntry->GCPhysMapping < pRegEntry->cbRegion \|\| !pRegEntry->fMapped));
852	# endif
853	#endif
854
855	#ifndef IN_RING3
856	/*
857	* If someone is doing FXSAVE, FXRSTOR, XSAVE, XRSTOR or other stuff dealing with
858	* large amounts of data, just go to ring-3 where we don't need to deal with partial
859	* successes. No chance any of these will be problematic read-modify-write stuff.
860	*
861	* Also drop back if the ring-0 registration entry isn't actually used.
862	*/
863	if ( RT_LIKELY(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue))
864	&& pRegEntry->cbRegion != 0
865	&& ( enmAccessType == PGMACCESSTYPE_READ
866	? pRegEntry->pfnReadCallback != NULL \|\| pVM->iomr0.s.paMmioRing3Regs[uUser].pfnReadCallback == NULL
867	: pRegEntry->pfnWriteCallback != NULL \|\| pVM->iomr0.s.paMmioRing3Regs[uUser].pfnWriteCallback == NULL)
868	&& pDevIns )
869	{ /* likely */ }
870	else
871	{
872	Log4(("iomMmioHandlerNew: to ring-3: to-big=%RTbool zero-size=%RTbool no-callback=%RTbool pDevIns=%p hRegion=%#RX64\n",
873	!(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue)), !(pRegEntry->cbRegion != 0),
874	!( enmAccessType == PGMACCESSTYPE_READ
875	? pRegEntry->pfnReadCallback != NULL \|\| pVM->iomr0.s.paMmioRing3Regs[uUser].pfnReadCallback == NULL
876	: pRegEntry->pfnWriteCallback != NULL \|\| pVM->iomr0.s.paMmioRing3Regs[uUser].pfnWriteCallback == NULL),
877	pDevIns, uUser));
878	STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pStats->ReadRZToR3 : &pStats->WriteRZToR3);
879	STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pVM->iom.s.StatMmioReadsR0ToR3 : &pVM->iom.s.StatMmioWritesR0ToR3);
880	return rcToRing3;
881	}
882	#endif /* !IN_RING3 */
883
884	/*
885	* If we've got an offset that's outside the region, defer to ring-3 if we
886	* can, or pretend there is nothing there. This shouldn't happen, but can
887	* if we're unlucky with an SMP VM and the guest isn't behaving very well.
888	*/
889	#ifdef IN_RING0
890	RTGCPHYS const GCPhysMapping = pRegEntryR3->GCPhysMapping;
891	#else
892	RTGCPHYS const GCPhysMapping = pRegEntry->GCPhysMapping;
893	#endif
894	RTGCPHYS const offRegion = GCPhysFault - GCPhysMapping;
895	if (RT_LIKELY(offRegion < pRegEntry->cbRegion && GCPhysMapping != NIL_RTGCPHYS))
896	{ /* likely */ }
897	else
898	{
899	STAM_REL_COUNTER_INC(&pVM->iom.s.StatMmioStaleMappings);
900	LogRelMax(64, ("iomMmioHandlerNew: Stale access at %#RGp to range #%#x currently residing at %RGp LB %RGp\n",
901	GCPhysFault, pRegEntry->idxSelf, GCPhysMapping, pRegEntry->cbRegion));
902	#ifdef IN_RING3
903	if (enmAccessType == PGMACCESSTYPE_READ)
904	iomMMIODoReadFFs(pvBuf, cbBuf IOM_MMIO_STATS_COMMA_ARG);
905	return VINF_SUCCESS;
906	#else
907	STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pStats->ReadRZToR3 : &pStats->WriteRZToR3);
908	STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pVM->iom.s.StatMmioReadsR0ToR3 : &pVM->iom.s.StatMmioWritesR0ToR3);
909	return rcToRing3;
910	#endif
911	}
912
913	/*
914	* Perform locking and the access.
915	*
916	* Writes requiring a return to ring-3 are buffered by IOM so IEM can
917	* commit the instruction.
918	*
919	* Note! We may end up locking the device even when the relevant callback is
920	* NULL. This is supposed to be an unlikely case, so not optimized yet.
921	*/
922	VBOXSTRICTRC rcStrict = PDMCritSectEnter(pVM, pDevIns->CTX_SUFF(pCritSectRo), rcToRing3);
923	if (rcStrict == VINF_SUCCESS)
924	{
925	if (enmAccessType == PGMACCESSTYPE_READ)
926	{
927	/*
928	* Read.
929	*/
930	rcStrict = iomMmioDoRead(pVM, pRegEntry, GCPhysFault, offRegion, pvBuf, (uint32_t)cbBuf IOM_MMIO_STATS_COMMA_ARG);
931
932	PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo));
933	#ifndef IN_RING3
934	if (rcStrict == VINF_IOM_R3_MMIO_READ)
935	{
936	STAM_COUNTER_INC(&pStats->ReadRZToR3);
937	STAM_COUNTER_INC(&pVM->iom.s.StatMmioReadsR0ToR3);
938	}
939	else
940	#endif
941	STAM_COUNTER_INC(&pStats->Reads);
942	STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfRead), Prf);
943	}
944	else
945	{
946	/*
947	* Write.
948	*/
949	rcStrict = iomMmioDoWrite(pVM, pVCpu, pRegEntry, GCPhysFault, offRegion, pvBuf, (uint32_t)cbBuf IOM_MMIO_STATS_COMMA_ARG);
950	PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo));
951	#ifndef IN_RING3
952	if (rcStrict == VINF_IOM_R3_MMIO_WRITE)
953	rcStrict = iomMmioRing3WritePending(pVCpu, GCPhysFault, pvBuf, cbBuf, pRegEntry->idxSelf);
954	if (rcStrict == VINF_IOM_R3_MMIO_WRITE)
955	{
956	STAM_COUNTER_INC(&pStats->WriteRZToR3);
957	STAM_COUNTER_INC(&pVM->iom.s.StatMmioWritesR0ToR3);
958	}
959	else if (rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE)
960	{
961	STAM_COUNTER_INC(&pStats->CommitRZToR3);
962	STAM_COUNTER_INC(&pVM->iom.s.StatMmioCommitsR0ToR3);
963	}
964	else
965	#endif
966	STAM_COUNTER_INC(&pStats->Writes);
967	STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfWrite), Prf);
968	}
969
970	/*
971	* Check the return code.
972	*/
973	#ifdef IN_RING3
974	AssertMsg(rcStrict == VINF_SUCCESS, ("%Rrc - Access type %d - %RGp - %s\n",
975	VBOXSTRICTRC_VAL(rcStrict), enmAccessType, GCPhysFault, pRegEntry->pszDesc));
976	#else
977	AssertMsg( rcStrict == VINF_SUCCESS
978	\|\| rcStrict == rcToRing3
979	\|\| (rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE && enmAccessType == PGMACCESSTYPE_WRITE)
980	\|\| rcStrict == VINF_EM_DBG_STOP
981	\|\| rcStrict == VINF_EM_DBG_EVENT
982	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
983	\|\| rcStrict == VINF_EM_OFF
984	\|\| rcStrict == VINF_EM_SUSPEND
985	\|\| rcStrict == VINF_EM_RESET
986	//\|\| rcStrict == VINF_EM_HALT /* ?? */
987	//\|\| rcStrict == VINF_EM_NO_MEMORY /* ?? */
988	, ("%Rrc - Access type %d - %RGp - %s #%u\n",
989	VBOXSTRICTRC_VAL(rcStrict), enmAccessType, GCPhysFault, pDevIns->pReg->szName, pDevIns->iInstance));
990	#endif
991	}
992	/*
993	* Deal with enter-critsect failures.
994	*/
995	#ifndef IN_RING3
996	else if (rcStrict == VINF_IOM_R3_MMIO_WRITE)
997	{
998	Assert(enmAccessType == PGMACCESSTYPE_WRITE);
999	rcStrict = iomMmioRing3WritePending(pVCpu, GCPhysFault, pvBuf, cbBuf, pRegEntry->idxSelf);
1000	if (rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE)
1001	{
1002	STAM_COUNTER_INC(&pStats->CommitRZToR3);
1003	STAM_COUNTER_INC(&pVM->iom.s.StatMmioCommitsR0ToR3);
1004	}
1005	else
1006	{
1007	STAM_COUNTER_INC(&pStats->WriteRZToR3);
1008	STAM_COUNTER_INC(&pVM->iom.s.StatMmioWritesR0ToR3);
1009	}
1010	STAM_COUNTER_INC(&pVM->iom.s.StatMmioDevLockContentionR0);
1011	}
1012	else if (rcStrict == VINF_IOM_R3_MMIO_READ)
1013	{
1014	Assert(enmAccessType == PGMACCESSTYPE_READ);
1015	STAM_COUNTER_INC(&pStats->ReadRZToR3);
1016	STAM_COUNTER_INC(&pVM->iom.s.StatMmioDevLockContentionR0);
1017	}
1018	#endif
1019	else
1020	AssertMsg(RT_FAILURE_NP(rcStrict), ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
1021	return rcStrict;
1022	}
1023
1024
1025	/**
1026	* Mapping an MMIO2 page in place of an MMIO page for direct access.
1027	*
1028	* This is a special optimization used by the VGA device. Call
1029	* IOMMmioResetRegion() to undo the mapping.
1030	*
1031	* @returns VBox status code. This API may return VINF_SUCCESS even if no
1032	* remapping is made.
1033	* @retval VERR_SEM_BUSY in ring-0 if we cannot get the IOM lock.
1034	*
1035	* @param pVM The cross context VM structure.
1036	* @param pDevIns The device instance @a hRegion and @a hMmio2 are
1037	* associated with.
1038	* @param hRegion The handle to the MMIO region.
1039	* @param offRegion The offset into @a hRegion of the page to be
1040	* remapped.
1041	* @param hMmio2 The MMIO2 handle.
1042	* @param offMmio2 Offset into @a hMmio2 of the page to be use for the
1043	* mapping.
1044	* @param fPageFlags Page flags to set. Must be (X86_PTE_RW \| X86_PTE_P)
1045	* for the time being.
1046	*/
1047	VMMDECL(int) IOMMmioMapMmio2Page(PVMCC pVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion, RTGCPHYS offRegion,
1048	uint64_t hMmio2, RTGCPHYS offMmio2, uint64_t fPageFlags)
1049	{
1050	/* Currently only called from the VGA device during MMIO. */
1051	Log(("IOMMmioMapMmio2Page %#RX64/%RGp -> %#RX64/%RGp flags=%RX64\n", hRegion, offRegion, hMmio2, offMmio2, fPageFlags));
1052	AssertReturn(fPageFlags == (X86_PTE_RW \| X86_PTE_P), VERR_INVALID_PARAMETER);
1053	AssertReturn(pDevIns, VERR_INVALID_POINTER);
1054
1055	/** @todo Why is this restricted to protected mode??? Try it in all modes! */
1056	PVMCPUCC pVCpu = VMMGetCpu(pVM);
1057
1058	/* This currently only works in real mode, protected mode without paging or with nested paging. */
1059	/** @todo NEM: MMIO page aliasing. */
1060	if ( !HMIsEnabled(pVM) /* useless without VT-x/AMD-V */
1061	\|\| ( CPUMIsGuestInPagedProtectedMode(pVCpu)
1062	&& !HMIsNestedPagingActive(pVM)))
1063	return VINF_SUCCESS; /* ignore / /* @todo return some indicator if we fail here */
1064
1065	/*
1066	* Translate the handle into an entry and check the region offset.
1067	*/
1068	AssertReturn(hRegion < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
1069	#ifdef IN_RING0
1070	AssertReturn(hRegion < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
1071	PIOMMMIOENTRYR3 const pRegEntry = &pVM->iomr0.s.paMmioRing3Regs[hRegion];
1072	AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
1073	AssertReturn(offRegion < pVM->iomr0.s.paMmioRegs[hRegion].cbRegion, VERR_OUT_OF_RANGE);
1074	AssertReturn( pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == pDevIns
1075	\|\| ( pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == NULL
1076	&& pRegEntry->pDevIns == pDevIns->pDevInsForR3), VERR_ACCESS_DENIED);
1077	#else
1078	PIOMMMIOENTRYR3 const pRegEntry = &pVM->iom.s.paMmioRegs[hRegion];
1079	AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
1080	AssertReturn(pRegEntry->pDevIns == pDevIns, VERR_ACCESS_DENIED);
1081	#endif
1082	AssertReturn(offRegion < pRegEntry->cbRegion, VERR_OUT_OF_RANGE);
1083	Assert((pRegEntry->cbRegion & GUEST_PAGE_OFFSET_MASK) == 0);
1084
1085	/*
1086	* When getting and using the mapping address, we must sit on the IOM lock
1087	* to prevent remapping. Shared suffices as we change nothing.
1088	*/
1089	int rc = IOM_LOCK_SHARED(pVM);
1090	if (rc == VINF_SUCCESS)
1091	{
1092	RTGCPHYS const GCPhys = pRegEntry->fMapped ? pRegEntry->GCPhysMapping : NIL_RTGCPHYS;
1093	if (GCPhys != NIL_RTGCPHYS)
1094	{
1095	Assert(!(GCPhys & GUEST_PAGE_OFFSET_MASK));
1096
1097	/*
1098	* Do the aliasing; page align the addresses since PGM is picky.
1099	*/
1100	rc = PGMHandlerPhysicalPageAliasMmio2(pVM, GCPhys, GCPhys + (offRegion & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK),
1101	pDevIns, hMmio2, offMmio2);
1102	}
1103	else
1104	AssertFailedStmt(rc = VERR_IOM_MMIO_REGION_NOT_MAPPED);
1105
1106	IOM_UNLOCK_SHARED(pVM);
1107	}
1108
1109	/** @todo either ditch this or replace it with something that works in the
1110	* nested case, since we really only care about nested paging! */
1111	#if 0
1112	/*
1113	* Modify the shadow page table. Since it's an MMIO page it won't be present and we
1114	* can simply prefetch it.
1115	*
1116	* Note: This is a NOP in the EPT case; we'll just let it fault again to resync the page.
1117	*/
1118	# if 0 /* The assertion is wrong for the PGM_SYNC_CLEAR_PGM_POOL and VINF_PGM_HANDLER_ALREADY_ALIASED cases. */
1119	# ifdef VBOX_STRICT
1120	uint64_t fFlags;
1121	RTHCPHYS HCPhys;
1122	rc = PGMShwGetPage(pVCpu, (RTGCPTR)GCPhys, &fFlags, &HCPhys);
1123	Assert(rc == VERR_PAGE_NOT_PRESENT \|\| rc == VERR_PAGE_TABLE_NOT_PRESENT);
1124	# endif
1125	# endif
1126	rc = PGMPrefetchPage(pVCpu, (RTGCPTR)GCPhys);
1127	Assert(rc == VINF_SUCCESS \|\| rc == VERR_PAGE_NOT_PRESENT \|\| rc == VERR_PAGE_TABLE_NOT_PRESENT);
1128	#endif
1129	return rc;
1130	}
1131
1132
1133	#ifdef IN_RING0 /* VT-x ring-0 only, move to IOMR0Mmio.cpp later. */
1134	/**
1135	* Mapping a HC page in place of an MMIO page for direct access.
1136	*
1137	* This is a special optimization used by the APIC in the VT-x case. This VT-x
1138	* code uses PGMHandlerPhysicalReset rather than IOMMmioResetRegion() to undo
1139	* the effects here.
1140	*
1141	* @todo Make VT-x usage more consistent.
1142	*
1143	* @returns VBox status code.
1144	*
1145	* @param pVM The cross context VM structure.
1146	* @param pVCpu The cross context virtual CPU structure.
1147	* @param GCPhys The address of the MMIO page to be changed.
1148	* @param HCPhys The address of the host physical page.
1149	* @param fPageFlags Page flags to set. Must be (X86_PTE_RW \| X86_PTE_P)
1150	* for the time being.
1151	*/
1152	VMMR0_INT_DECL(int) IOMR0MmioMapMmioHCPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint64_t fPageFlags)
1153	{
1154	/* Currently only called from VT-x code during a page fault. */
1155	Log(("IOMR0MmioMapMmioHCPage %RGp -> %RGp flags=%RX64\n", GCPhys, HCPhys, fPageFlags));
1156
1157	AssertReturn(fPageFlags == (X86_PTE_RW \| X86_PTE_P), VERR_INVALID_PARAMETER);
1158	/** @todo NEM: MMIO page aliasing?? */
1159	Assert(HMIsEnabled(pVM));
1160
1161	# ifdef VBOX_STRICT
1162	/*
1163	* Check input address (it's HM calling, not the device, so no region handle).
1164	*/
1165	int rcSem = IOM_LOCK_SHARED(pVM);
1166	if (rcSem == VINF_SUCCESS)
1167	{
1168	RTGCPHYS offIgn;
1169	uint16_t idxIgn = UINT16_MAX;
1170	PIOMMMIOENTRYR0 pRegEntry = iomMmioGetEntry(pVM, GCPhys, &offIgn, &idxIgn);
1171	IOM_UNLOCK_SHARED(pVM);
1172	Assert(pRegEntry);
1173	Assert(pRegEntry && !(pRegEntry->cbRegion & GUEST_PAGE_OFFSET_MASK));
1174	}
1175	# endif
1176
1177	/*
1178	* Do the aliasing; page align the addresses since PGM is picky.
1179	*/
1180	GCPhys &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1181	HCPhys &= ~(RTHCPHYS)GUEST_PAGE_OFFSET_MASK;
1182
1183	int rc = PGMHandlerPhysicalPageAliasHC(pVM, GCPhys, GCPhys, HCPhys);
1184	AssertRCReturn(rc, rc);
1185
1186	/** @todo either ditch this or replace it with something that works in the
1187	* nested case, since we really only care about nested paging! */
1188
1189	/*
1190	* Modify the shadow page table. Since it's an MMIO page it won't be present and we
1191	* can simply prefetch it.
1192	*
1193	* Note: This is a NOP in the EPT case; we'll just let it fault again to resync the page.
1194	*/
1195	rc = PGMPrefetchPage(pVCpu, (RTGCPTR)GCPhys);
1196	Assert(rc == VINF_SUCCESS \|\| rc == VERR_PAGE_NOT_PRESENT \|\| rc == VERR_PAGE_TABLE_NOT_PRESENT);
1197	return VINF_SUCCESS;
1198	}
1199	#endif
1200
1201
1202	/**
1203	* Reset a previously modified MMIO region; restore the access flags.
1204	*
1205	* This undoes the effects of IOMMmioMapMmio2Page() and is currently only
1206	* intended for some ancient VGA hack. However, it would be great to extend it
1207	* beyond VT-x and/or nested-paging.
1208	*
1209	* @returns VBox status code.
1210	*
1211	* @param pVM The cross context VM structure.
1212	* @param pDevIns The device instance @a hRegion is associated with.
1213	* @param hRegion The handle to the MMIO region.
1214	*/
1215	VMMDECL(int) IOMMmioResetRegion(PVMCC pVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion)
1216	{
1217	Log(("IOMMMIOResetRegion %#RX64\n", hRegion));
1218	AssertReturn(pDevIns, VERR_INVALID_POINTER);
1219
1220	/** @todo Get rid of this this real/protected or nested paging restriction,
1221	* it probably shouldn't be here and would be nasty when the CPU
1222	* changes mode while we have the hack enabled... */
1223	PVMCPUCC pVCpu = VMMGetCpu(pVM);
1224
1225	/* This currently only works in real mode, protected mode without paging or with nested paging. */
1226	/** @todo NEM: MMIO page aliasing. */
1227	if ( !HMIsEnabled(pVM) /* useless without VT-x/AMD-V */
1228	\|\| ( CPUMIsGuestInPagedProtectedMode(pVCpu)
1229	&& !HMIsNestedPagingActive(pVM)))
1230	return VINF_SUCCESS; /* ignore */
1231
1232	/*
1233	* Translate the handle into an entry and mapping address for PGM.
1234	* We have to take the lock to safely access the mapping address here.
1235	*/
1236	AssertReturn(hRegion < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
1237	#ifdef IN_RING0
1238	AssertReturn(hRegion < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
1239	PIOMMMIOENTRYR3 const pRegEntry = &pVM->iomr0.s.paMmioRing3Regs[hRegion];
1240	AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
1241	AssertReturn( pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == pDevIns
1242	\|\| ( pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == NULL
1243	&& pRegEntry->pDevIns == pDevIns->pDevInsForR3), VERR_ACCESS_DENIED);
1244	#else
1245	PIOMMMIOENTRYR3 const pRegEntry = &pVM->iom.s.paMmioRegs[hRegion];
1246	AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
1247	AssertReturn(pRegEntry->pDevIns == pDevIns, VERR_ACCESS_DENIED);
1248	#endif
1249	Assert((pRegEntry->cbRegion & GUEST_PAGE_OFFSET_MASK) == 0);
1250
1251	int rcSem = IOM_LOCK_SHARED(pVM);
1252	RTGCPHYS GCPhys = pRegEntry->fMapped ? pRegEntry->GCPhysMapping : NIL_RTGCPHYS;
1253	if (rcSem == VINF_SUCCESS)
1254	IOM_UNLOCK_SHARED(pVM);
1255
1256	Assert(!(GCPhys & GUEST_PAGE_OFFSET_MASK));
1257	Assert(!(pRegEntry->cbRegion & GUEST_PAGE_OFFSET_MASK));
1258
1259	/*
1260	* Call PGM to do the job work.
1261	*
1262	* After the call, all the pages should be non-present, unless there is
1263	* a page pool flush pending (unlikely).
1264	*/
1265	int rc = PGMHandlerPhysicalReset(pVM, GCPhys);
1266	AssertRC(rc);
1267
1268	# ifdef VBOX_STRICT
1269	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
1270	{
1271	RTGCPHYS cb = pRegEntry->cbRegion;
1272	while (cb)
1273	{
1274	uint64_t fFlags;
1275	RTHCPHYS HCPhys;
1276	rc = PGMShwGetPage(pVCpu, (RTGCPTR)GCPhys, &fFlags, &HCPhys);
1277	Assert(rc == VERR_PAGE_NOT_PRESENT \|\| rc == VERR_PAGE_TABLE_NOT_PRESENT);
1278	cb -= RT_MIN(GUEST_PAGE_SIZE, HOST_PAGE_SIZE);
1279	GCPhys += RT_MIN(GUEST_PAGE_SIZE, HOST_PAGE_SIZE);
1280	}
1281	}
1282	# endif
1283	return rc;
1284	}
1285

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/IOMAllMmioNew.cpp@ 97145

Download in other formats: