IOMAllMmioNew.cpp@ 98045

Last change on this file since 98045 was 98045, checked in by vboxsync, 23 months ago
IOM: Updates. bugref:10315
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 52.3 KB

Line
1	/* $Id: IOMAllMmioNew.cpp 98045 2023-01-10 21:31:53Z vboxsync $ */
2	/** @file
3	* IOM - Input / Output Monitor - Any Context, MMIO & String I/O.
4	*/
5
6	/*
7	* Copyright (C) 2006-2022 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_IOM_MMIO
33	#define VMCPU_INCL_CPUM_GST_CTX
34	#include <VBox/vmm/iom.h>
35	#include <VBox/vmm/cpum.h>
36	#include <VBox/vmm/pgm.h>
37	#include <VBox/vmm/selm.h>
38	#include <VBox/vmm/mm.h>
39	#include <VBox/vmm/em.h>
40	#include <VBox/vmm/pgm.h>
41	#include <VBox/vmm/trpm.h>
42	#include <VBox/vmm/iem.h>
43	#include "IOMInternal.h"
44	#include <VBox/vmm/vmcc.h>
45	#include <VBox/vmm/vmm.h>
46	#include <VBox/vmm/hm.h>
47	#include "IOMInline.h"
48
49	#include <VBox/dis.h>
50	#include <VBox/disopcode.h>
51	#include <VBox/vmm/pdmdev.h>
52	#include <VBox/param.h>
53	#include <VBox/err.h>
54	#include <iprt/assert.h>
55	#include <VBox/log.h>
56	#include <iprt/asm.h>
57	#include <iprt/string.h>
58
59
60	/*********************************************************************************************************************************
61	* Defined Constants And Macros *
62	*********************************************************************************************************************************/
63	/** @def IOM_MMIO_STATS_COMMA_DECL
64	* Parameter list declaration for statistics entry pointer. */
65	/** @def IOM_MMIO_STATS_COMMA_ARG
66	* Statistics entry pointer argument. */
67	#if defined(VBOX_WITH_STATISTICS) \|\| defined(DOXYGEN_RUNNING)
68	# define IOM_MMIO_STATS_COMMA_DECL , PIOMMMIOSTATSENTRY pStats
69	# define IOM_MMIO_STATS_COMMA_ARG , pStats
70	#else
71	# define IOM_MMIO_STATS_COMMA_DECL
72	# define IOM_MMIO_STATS_COMMA_ARG
73	#endif
74
75
76
77	#ifndef IN_RING3
78	/**
79	* Defers a pending MMIO write to ring-3.
80	*
81	* @returns VINF_IOM_R3_MMIO_COMMIT_WRITE
82	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
83	* @param GCPhys The write address.
84	* @param pvBuf The bytes being written.
85	* @param cbBuf How many bytes.
86	* @param idxRegEntry The MMIO registration index (handle) if available,
87	* otherwise UINT32_MAX.
88	*/
89	static VBOXSTRICTRC iomMmioRing3WritePending(PVMCPU pVCpu, RTGCPHYS GCPhys, void const *pvBuf, size_t cbBuf,
90	uint32_t idxRegEntry)
91	{
92	Log5(("iomMmioRing3WritePending: %RGp LB %#x (idx=%#x)\n", GCPhys, cbBuf, idxRegEntry));
93	if (pVCpu->iom.s.PendingMmioWrite.cbValue == 0)
94	{
95	pVCpu->iom.s.PendingMmioWrite.GCPhys = GCPhys;
96	AssertReturn(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue), VERR_IOM_MMIO_IPE_2);
97	pVCpu->iom.s.PendingMmioWrite.cbValue = (uint32_t)cbBuf;
98	pVCpu->iom.s.PendingMmioWrite.idxMmioRegionHint = idxRegEntry;
99	memcpy(pVCpu->iom.s.PendingMmioWrite.abValue, pvBuf, cbBuf);
100	}
101	else
102	{
103	/*
104	* Join with pending if adjecent.
105	*
106	* This may happen if the stack overflows into MMIO territory and RSP/ESP/SP
107	* isn't aligned. IEM will bounce buffer the access and do one write for each
108	* page. We get here when the 2nd page part is written.
109	*/
110	uint32_t const cbOldValue = pVCpu->iom.s.PendingMmioWrite.cbValue;
111	AssertMsgReturn(GCPhys == pVCpu->iom.s.PendingMmioWrite.GCPhys + cbOldValue,
112	("pending %RGp LB %#x; incoming %RGp LB %#x\n",
113	pVCpu->iom.s.PendingMmioWrite.GCPhys, cbOldValue, GCPhys, cbBuf),
114	VERR_IOM_MMIO_IPE_1);
115	AssertReturn(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue) - cbOldValue, VERR_IOM_MMIO_IPE_2);
116	pVCpu->iom.s.PendingMmioWrite.cbValue = cbOldValue + (uint32_t)cbBuf;
117	memcpy(&pVCpu->iom.s.PendingMmioWrite.abValue[cbOldValue], pvBuf, cbBuf);
118	}
119
120	VMCPU_FF_SET(pVCpu, VMCPU_FF_IOM);
121	return VINF_IOM_R3_MMIO_COMMIT_WRITE;
122	}
123	#endif
124
125
126	/**
127	* Deals with complicated MMIO writes.
128	*
129	* Complicated means unaligned or non-dword/qword sized accesses depending on
130	* the MMIO region's access mode flags.
131	*
132	* @returns Strict VBox status code. Any EM scheduling status code,
133	* VINF_IOM_R3_MMIO_WRITE, VINF_IOM_R3_MMIO_READ_WRITE or
134	* VINF_IOM_R3_MMIO_READ may be returned.
135	*
136	* @param pVM The cross context VM structure.
137	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
138	* @param pRegEntry The MMIO entry for the current context.
139	* @param GCPhys The physical address to start writing.
140	* @param offRegion MMIO region offset corresponding to @a GCPhys.
141	* @param pvValue Where to store the value.
142	* @param cbValue The size of the value to write.
143	* @param pStats Pointer to the statistics (never NULL).
144	*/
145	static VBOXSTRICTRC iomMmioDoComplicatedWrite(PVM pVM, PVMCPU pVCpu, CTX_SUFF(PIOMMMIOENTRY) pRegEntry,
146	RTGCPHYS GCPhys, RTGCPHYS offRegion,
147	void const *pvValue, unsigned cbValue IOM_MMIO_STATS_COMMA_DECL)
148	{
149	AssertReturn( (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) != IOMMMIO_FLAGS_WRITE_PASSTHRU
150	&& (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) <= IOMMMIO_FLAGS_WRITE_DWORD_QWORD_READ_MISSING,
151	VERR_IOM_MMIO_IPE_1);
152	AssertReturn(cbValue != 0 && cbValue <= 16, VERR_IOM_MMIO_IPE_2);
153	RTGCPHYS const GCPhysStart = GCPhys; NOREF(GCPhysStart);
154	bool const fReadMissing = (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_DWORD_READ_MISSING
155	\|\| (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_DWORD_QWORD_READ_MISSING;
156	RT_NOREF_PV(pVCpu); /* ring-3 */
157
158	/*
159	* Do debug stop if requested.
160	*/
161	VBOXSTRICTRC rc = VINF_SUCCESS; NOREF(pVM);
162	#ifdef VBOX_STRICT
163	if (!(pRegEntry->fFlags & IOMMMIO_FLAGS_DBGSTOP_ON_COMPLICATED_WRITE))
164	{ /* likely */ }
165	else
166	{
167	# ifdef IN_RING3
168	LogRel(("IOM: Complicated write %#x byte at %RGp to %s, initiating debugger intervention\n", cbValue, GCPhys,
169	R3STRING(pRegEntry->pszDesc)));
170	rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, RT_SRC_POS,
171	"Complicated write %#x byte at %RGp to %s\n", cbValue, GCPhys, pRegEntry->pszDesc);
172	if (rc == VERR_DBGF_NOT_ATTACHED)
173	rc = VINF_SUCCESS;
174	# else
175	return VINF_IOM_R3_MMIO_WRITE;
176	# endif
177	}
178	#endif
179
180	STAM_COUNTER_INC(&pStats->ComplicatedWrites);
181
182	/*
183	* Check if we should ignore the write.
184	*/
185	if ((pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_ONLY_DWORD)
186	{
187	Assert(cbValue != 4 \|\| (GCPhys & 3));
188	return VINF_SUCCESS;
189	}
190	if ((pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_ONLY_DWORD_QWORD)
191	{
192	Assert((cbValue != 4 && cbValue != 8) \|\| (GCPhys & (cbValue - 1)));
193	return VINF_SUCCESS;
194	}
195
196	/*
197	* Split and conquer.
198	*/
199	for (;;)
200	{
201	unsigned const offAccess = GCPhys & 3;
202	unsigned cbThisPart = 4 - offAccess;
203	if (cbThisPart > cbValue)
204	cbThisPart = cbValue;
205
206	/*
207	* Get the missing bits (if any).
208	*/
209	uint32_t u32MissingValue = 0;
210	if (fReadMissing && cbThisPart != 4)
211	{
212	VBOXSTRICTRC rc2 = pRegEntry->pfnReadCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
213	!(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS)
214	? offRegion & ~(RTGCPHYS)3 : (GCPhys & ~(RTGCPHYS)3),
215	&u32MissingValue, sizeof(u32MissingValue));
216	switch (VBOXSTRICTRC_VAL(rc2))
217	{
218	case VINF_SUCCESS:
219	break;
220	case VINF_IOM_MMIO_UNUSED_FF:
221	STAM_COUNTER_INC(&pStats->FFor00Reads);
222	u32MissingValue = UINT32_C(0xffffffff);
223	break;
224	case VINF_IOM_MMIO_UNUSED_00:
225	STAM_COUNTER_INC(&pStats->FFor00Reads);
226	u32MissingValue = 0;
227	break;
228	#ifndef IN_RING3
229	case VINF_IOM_R3_MMIO_READ:
230	case VINF_IOM_R3_MMIO_READ_WRITE:
231	case VINF_IOM_R3_MMIO_WRITE:
232	LogFlow(("iomMmioDoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [read]\n", GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rc2)));
233	rc2 = iomMmioRing3WritePending(pVCpu, GCPhys, pvValue, cbValue, pRegEntry->idxSelf);
234	if (rc == VINF_SUCCESS \|\| rc2 < rc)
235	rc = rc2;
236	return rc;
237	#endif
238	default:
239	if (RT_FAILURE(rc2))
240	{
241	Log(("iomMmioDoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [read]\n", GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rc2)));
242	return rc2;
243	}
244	AssertMsgReturn(rc2 >= VINF_EM_FIRST && rc2 <= VINF_EM_LAST, ("%Rrc\n", VBOXSTRICTRC_VAL(rc2)), VERR_IPE_UNEXPECTED_INFO_STATUS);
245	if (rc == VINF_SUCCESS \|\| rc2 < rc)
246	rc = rc2;
247	break;
248	}
249	}
250
251	/*
252	* Merge missing and given bits.
253	*/
254	uint32_t u32GivenMask;
255	uint32_t u32GivenValue;
256	switch (cbThisPart)
257	{
258	case 1:
259	u32GivenValue = (uint8_t const )pvValue;
260	u32GivenMask = UINT32_C(0x000000ff);
261	break;
262	case 2:
263	u32GivenValue = (uint16_t const )pvValue;
264	u32GivenMask = UINT32_C(0x0000ffff);
265	break;
266	case 3:
267	u32GivenValue = RT_MAKE_U32_FROM_U8(((uint8_t const )pvValue)[0], ((uint8_t const )pvValue)[1],
268	((uint8_t const *)pvValue)[2], 0);
269	u32GivenMask = UINT32_C(0x00ffffff);
270	break;
271	case 4:
272	u32GivenValue = (uint32_t const )pvValue;
273	u32GivenMask = UINT32_C(0xffffffff);
274	break;
275	default:
276	AssertFailedReturn(VERR_IOM_MMIO_IPE_3);
277	}
278	if (offAccess)
279	{
280	u32GivenValue <<= offAccess * 8;
281	u32GivenMask <<= offAccess * 8;
282	}
283
284	uint32_t u32Value = (u32MissingValue & ~u32GivenMask)
285	\| (u32GivenValue & u32GivenMask);
286
287	/*
288	* Do DWORD write to the device.
289	*/
290	VBOXSTRICTRC rc2 = pRegEntry->pfnWriteCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
291	!(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS)
292	? offRegion & ~(RTGCPHYS)3 : GCPhys & ~(RTGCPHYS)3,
293	&u32Value, sizeof(u32Value));
294	switch (VBOXSTRICTRC_VAL(rc2))
295	{
296	case VINF_SUCCESS:
297	break;
298	#ifndef IN_RING3
299	case VINF_IOM_R3_MMIO_READ:
300	case VINF_IOM_R3_MMIO_READ_WRITE:
301	case VINF_IOM_R3_MMIO_WRITE:
302	Log3(("iomMmioDoComplicatedWrite: deferring GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [write]\n", GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rc2)));
303	AssertReturn(pVCpu->iom.s.PendingMmioWrite.cbValue == 0, VERR_IOM_MMIO_IPE_1);
304	AssertReturn(cbValue + (GCPhys & 3) <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue), VERR_IOM_MMIO_IPE_2);
305	pVCpu->iom.s.PendingMmioWrite.GCPhys = GCPhys & ~(RTGCPHYS)3;
306	pVCpu->iom.s.PendingMmioWrite.cbValue = cbValue + (GCPhys & 3);
307	(uint32_t )pVCpu->iom.s.PendingMmioWrite.abValue = u32Value;
308	if (cbValue > cbThisPart)
309	memcpy(&pVCpu->iom.s.PendingMmioWrite.abValue[4],
310	(uint8_t const *)pvValue + cbThisPart, cbValue - cbThisPart);
311	VMCPU_FF_SET(pVCpu, VMCPU_FF_IOM);
312	if (rc == VINF_SUCCESS)
313	rc = VINF_IOM_R3_MMIO_COMMIT_WRITE;
314	return rc;
315	#endif
316	default:
317	if (RT_FAILURE(rc2))
318	{
319	Log(("iomMmioDoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [write]\n", GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rc2)));
320	return rc2;
321	}
322	AssertMsgReturn(rc2 >= VINF_EM_FIRST && rc2 <= VINF_EM_LAST, ("%Rrc\n", VBOXSTRICTRC_VAL(rc2)), VERR_IPE_UNEXPECTED_INFO_STATUS);
323	if (rc == VINF_SUCCESS \|\| rc2 < rc)
324	rc = rc2;
325	break;
326	}
327
328	/*
329	* Advance.
330	*/
331	cbValue -= cbThisPart;
332	if (!cbValue)
333	break;
334	GCPhys += cbThisPart;
335	offRegion += cbThisPart;
336	pvValue = (uint8_t const *)pvValue + cbThisPart;
337	}
338
339	return rc;
340	}
341
342
343
344
345	/**
346	* Wrapper which does the write.
347	*/
348	DECLINLINE(VBOXSTRICTRC) iomMmioDoWrite(PVMCC pVM, PVMCPU pVCpu, CTX_SUFF(PIOMMMIOENTRY) pRegEntry,
349	RTGCPHYS GCPhys, RTGCPHYS offRegion,
350	const void *pvData, uint32_t cb IOM_MMIO_STATS_COMMA_DECL)
351	{
352	VBOXSTRICTRC rcStrict;
353	if (RT_LIKELY(pRegEntry->pfnWriteCallback))
354	{
355	if ( (cb == 4 && !(GCPhys & 3))
356	\|\| (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_PASSTHRU
357	\|\| (cb == 8 && !(GCPhys & 7) && IOMMMIO_DOES_WRITE_MODE_ALLOW_QWORD(pRegEntry->fFlags)) )
358	rcStrict = pRegEntry->pfnWriteCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
359	!(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS) ? offRegion : GCPhys, pvData, cb);
360	else
361	rcStrict = iomMmioDoComplicatedWrite(pVM, pVCpu, pRegEntry, GCPhys, offRegion, pvData, cb IOM_MMIO_STATS_COMMA_ARG);
362	}
363	else
364	rcStrict = VINF_SUCCESS;
365	return rcStrict;
366	}
367
368
369	#ifdef IN_RING3
370	/**
371	* Helper for IOMR3ProcessForceFlag() that lives here to utilize iomMmioDoWrite et al.
372	*/
373	VBOXSTRICTRC iomR3MmioCommitWorker(PVM pVM, PVMCPU pVCpu, PIOMMMIOENTRYR3 pRegEntry, RTGCPHYS offRegion)
374	{
375	# ifdef VBOX_WITH_STATISTICS
376	STAM_PROFILE_START(UnusedMacroArg, Prf);
377	PIOMMMIOSTATSENTRY const pStats = iomMmioGetStats(pVM, pRegEntry);
378	# endif
379	PPDMDEVINS const pDevIns = pRegEntry->pDevIns;
380	int rc = PDMCritSectEnter(pVM, pDevIns->CTX_SUFF(pCritSectRo), VERR_IGNORED);
381	AssertRCReturn(rc, rc);
382
383	VBOXSTRICTRC rcStrict = iomMmioDoWrite(pVM, pVCpu, pRegEntry, pVCpu->iom.s.PendingMmioWrite.GCPhys, offRegion,
384	pVCpu->iom.s.PendingMmioWrite.abValue, pVCpu->iom.s.PendingMmioWrite.cbValue
385	IOM_MMIO_STATS_COMMA_ARG);
386
387	PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo));
388	STAM_PROFILE_STOP(&pStats->ProfWriteR3, Prf);
389	return rcStrict;
390	}
391	#endif /* IN_RING3 */
392
393
394	/**
395	* Deals with complicated MMIO reads.
396	*
397	* Complicated means unaligned or non-dword/qword sized accesses depending on
398	* the MMIO region's access mode flags.
399	*
400	* @returns Strict VBox status code. Any EM scheduling status code,
401	* VINF_IOM_R3_MMIO_READ, VINF_IOM_R3_MMIO_READ_WRITE or
402	* VINF_IOM_R3_MMIO_WRITE may be returned.
403	*
404	* @param pVM The cross context VM structure.
405	* @param pRegEntry The MMIO entry for the current context.
406	* @param GCPhys The physical address to start reading.
407	* @param offRegion MMIO region offset corresponding to @a GCPhys.
408	* @param pvValue Where to store the value.
409	* @param cbValue The size of the value to read.
410	* @param pStats Pointer to the statistics (never NULL).
411	*/
412	static VBOXSTRICTRC iomMMIODoComplicatedRead(PVM pVM, CTX_SUFF(PIOMMMIOENTRY) pRegEntry, RTGCPHYS GCPhys, RTGCPHYS offRegion,
413	void *pvValue, unsigned cbValue IOM_MMIO_STATS_COMMA_DECL)
414	{
415	AssertReturn( (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD
416	\|\| (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD_QWORD,
417	VERR_IOM_MMIO_IPE_1);
418	AssertReturn(cbValue != 0 && cbValue <= 16, VERR_IOM_MMIO_IPE_2);
419	#ifdef LOG_ENABLED
420	RTGCPHYS const GCPhysStart = GCPhys;
421	#endif
422
423	/*
424	* Do debug stop if requested.
425	*/
426	VBOXSTRICTRC rc = VINF_SUCCESS; NOREF(pVM);
427	#ifdef VBOX_STRICT
428	if (pRegEntry->fFlags & IOMMMIO_FLAGS_DBGSTOP_ON_COMPLICATED_READ)
429	{
430	# ifdef IN_RING3
431	rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, RT_SRC_POS,
432	"Complicated read %#x byte at %RGp to %s\n", cbValue, GCPhys, R3STRING(pRegEntry->pszDesc));
433	if (rc == VERR_DBGF_NOT_ATTACHED)
434	rc = VINF_SUCCESS;
435	# else
436	return VINF_IOM_R3_MMIO_READ;
437	# endif
438	}
439	#endif
440
441	STAM_COUNTER_INC(&pStats->ComplicatedReads);
442
443	/*
444	* Split and conquer.
445	*/
446	for (;;)
447	{
448	/*
449	* Do DWORD read from the device.
450	*/
451	uint32_t u32Value;
452	VBOXSTRICTRC rcStrict2 = pRegEntry->pfnReadCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
453	!(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS)
454	? offRegion & ~(RTGCPHYS)3 : GCPhys & ~(RTGCPHYS)3,
455	&u32Value, sizeof(u32Value));
456	switch (VBOXSTRICTRC_VAL(rcStrict2))
457	{
458	case VINF_SUCCESS:
459	break;
460	case VINF_IOM_MMIO_UNUSED_FF:
461	STAM_COUNTER_INC(&pStats->FFor00Reads);
462	u32Value = UINT32_C(0xffffffff);
463	break;
464	case VINF_IOM_MMIO_UNUSED_00:
465	STAM_COUNTER_INC(&pStats->FFor00Reads);
466	u32Value = 0;
467	break;
468	case VINF_IOM_R3_MMIO_READ:
469	case VINF_IOM_R3_MMIO_READ_WRITE:
470	case VINF_IOM_R3_MMIO_WRITE:
471	/** @todo What if we've split a transfer and already read
472	* something? Since reads can have sideeffects we could be
473	* kind of screwed here... */
474	LogFlow(("iomMMIODoComplicatedRead: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rcStrict2=%Rrc\n",
475	GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rcStrict2)));
476	return rcStrict2;
477	default:
478	if (RT_FAILURE(rcStrict2))
479	{
480	Log(("iomMMIODoComplicatedRead: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rcStrict2=%Rrc\n",
481	GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rcStrict2)));
482	return rcStrict2;
483	}
484	AssertMsgReturn(rcStrict2 >= VINF_EM_FIRST && rcStrict2 <= VINF_EM_LAST, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)),
485	VERR_IPE_UNEXPECTED_INFO_STATUS);
486	if (rc == VINF_SUCCESS \|\| rcStrict2 < rc)
487	rc = rcStrict2;
488	break;
489	}
490	u32Value >>= (GCPhys & 3) * 8;
491
492	/*
493	* Write what we've read.
494	*/
495	unsigned cbThisPart = 4 - (GCPhys & 3);
496	if (cbThisPart > cbValue)
497	cbThisPart = cbValue;
498
499	switch (cbThisPart)
500	{
501	case 1:
502	(uint8_t )pvValue = (uint8_t)u32Value;
503	break;
504	case 2:
505	(uint16_t )pvValue = (uint16_t)u32Value;
506	break;
507	case 3:
508	((uint8_t *)pvValue)[0] = RT_BYTE1(u32Value);
509	((uint8_t *)pvValue)[1] = RT_BYTE2(u32Value);
510	((uint8_t *)pvValue)[2] = RT_BYTE3(u32Value);
511	break;
512	case 4:
513	(uint32_t )pvValue = u32Value;
514	break;
515	}
516
517	/*
518	* Advance.
519	*/
520	cbValue -= cbThisPart;
521	if (!cbValue)
522	break;
523	GCPhys += cbThisPart;
524	offRegion += cbThisPart;
525	pvValue = (uint8_t *)pvValue + cbThisPart;
526	}
527
528	return rc;
529	}
530
531
532	/**
533	* Implements VINF_IOM_MMIO_UNUSED_FF.
534	*
535	* @returns VINF_SUCCESS.
536	* @param pvValue Where to store the zeros.
537	* @param cbValue How many bytes to read.
538	* @param pStats Pointer to the statistics (never NULL).
539	*/
540	static int iomMMIODoReadFFs(void *pvValue, size_t cbValue IOM_MMIO_STATS_COMMA_DECL)
541	{
542	switch (cbValue)
543	{
544	case 1: (uint8_t )pvValue = UINT8_C(0xff); break;
545	case 2: (uint16_t )pvValue = UINT16_C(0xffff); break;
546	case 4: (uint32_t )pvValue = UINT32_C(0xffffffff); break;
547	case 8: (uint64_t )pvValue = UINT64_C(0xffffffffffffffff); break;
548	default:
549	{
550	uint8_t pb = (uint8_t )pvValue;
551	while (cbValue--)
552	*pb++ = UINT8_C(0xff);
553	break;
554	}
555	}
556	STAM_COUNTER_INC(&pStats->FFor00Reads);
557	return VINF_SUCCESS;
558	}
559
560
561	/**
562	* Implements VINF_IOM_MMIO_UNUSED_00.
563	*
564	* @returns VINF_SUCCESS.
565	* @param pvValue Where to store the zeros.
566	* @param cbValue How many bytes to read.
567	* @param pStats Pointer to the statistics (never NULL).
568	*/
569	static int iomMMIODoRead00s(void *pvValue, size_t cbValue IOM_MMIO_STATS_COMMA_DECL)
570	{
571	switch (cbValue)
572	{
573	case 1: (uint8_t )pvValue = UINT8_C(0x00); break;
574	case 2: (uint16_t )pvValue = UINT16_C(0x0000); break;
575	case 4: (uint32_t )pvValue = UINT32_C(0x00000000); break;
576	case 8: (uint64_t )pvValue = UINT64_C(0x0000000000000000); break;
577	default:
578	{
579	uint8_t pb = (uint8_t )pvValue;
580	while (cbValue--)
581	*pb++ = UINT8_C(0x00);
582	break;
583	}
584	}
585	STAM_COUNTER_INC(&pStats->FFor00Reads);
586	return VINF_SUCCESS;
587	}
588
589
590	/**
591	* Wrapper which does the read.
592	*/
593	DECLINLINE(VBOXSTRICTRC) iomMmioDoRead(PVMCC pVM, CTX_SUFF(PIOMMMIOENTRY) pRegEntry, RTGCPHYS GCPhys, RTGCPHYS offRegion,
594	void *pvValue, uint32_t cbValue IOM_MMIO_STATS_COMMA_DECL)
595	{
596	VBOXSTRICTRC rcStrict;
597	if (RT_LIKELY(pRegEntry->pfnReadCallback))
598	{
599	if ( ( cbValue == 4
600	&& !(GCPhys & 3))
601	\|\| (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_PASSTHRU
602	\|\| ( cbValue == 8
603	&& !(GCPhys & 7)
604	&& (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD_QWORD ) )
605	rcStrict = pRegEntry->pfnReadCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
606	!(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS) ? offRegion : GCPhys, pvValue, cbValue);
607	else
608	rcStrict = iomMMIODoComplicatedRead(pVM, pRegEntry, GCPhys, offRegion, pvValue, cbValue IOM_MMIO_STATS_COMMA_ARG);
609	}
610	else
611	rcStrict = VINF_IOM_MMIO_UNUSED_FF;
612	if (rcStrict != VINF_SUCCESS)
613	{
614	switch (VBOXSTRICTRC_VAL(rcStrict))
615	{
616	case VINF_IOM_MMIO_UNUSED_FF: rcStrict = iomMMIODoReadFFs(pvValue, cbValue IOM_MMIO_STATS_COMMA_ARG); break;
617	case VINF_IOM_MMIO_UNUSED_00: rcStrict = iomMMIODoRead00s(pvValue, cbValue IOM_MMIO_STATS_COMMA_ARG); break;
618	}
619	}
620	return rcStrict;
621	}
622
623	#ifndef IN_RING3
624
625	/**
626	* Checks if we can handle an MMIO \#PF in R0/RC.
627	*/
628	DECLINLINE(bool) iomMmioCanHandlePfInRZ(PVMCC pVM, uint32_t uErrorCode, CTX_SUFF(PIOMMMIOENTRY) pRegEntry)
629	{
630	if (pRegEntry->cbRegion > 0)
631	{
632	if ( pRegEntry->pfnWriteCallback
633	&& pRegEntry->pfnReadCallback)
634	return true;
635
636	PIOMMMIOENTRYR3 const pRegEntryR3 = &pVM->iomr0.s.paMmioRing3Regs[pRegEntry->idxSelf];
637	if ( uErrorCode == UINT32_MAX
638	? pRegEntryR3->pfnWriteCallback \|\| pRegEntryR3->pfnReadCallback
639	: uErrorCode & X86_TRAP_PF_RW
640	? !pRegEntry->pfnWriteCallback && pRegEntryR3->pfnWriteCallback
641	: !pRegEntry->pfnReadCallback && pRegEntryR3->pfnReadCallback)
642	return false;
643
644	return true;
645	}
646	return false;
647	}
648
649
650	/**
651	* Common worker for the \#PF handler and IOMMMIOPhysHandler (APIC+VT-x).
652	*
653	* @returns VBox status code (appropriate for GC return).
654	* @param pVM The cross context VM structure.
655	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
656	* @param uErrorCode CPU Error code. This is UINT32_MAX when we don't have
657	* any error code (the EPT misconfig hack).
658	* @param GCPhysFault The GC physical address corresponding to pvFault.
659	* @param pRegEntry The MMIO entry for the current context.
660	*/
661	DECLINLINE(VBOXSTRICTRC) iomMmioCommonPfHandlerNew(PVMCC pVM, PVMCPUCC pVCpu, uint32_t uErrorCode,
662	RTGCPHYS GCPhysFault, CTX_SUFF(PIOMMMIOENTRY) pRegEntry)
663	{
664	Log(("iomMmioCommonPfHandler: GCPhysFault=%RGp uErr=%#x rip=%RGv\n", GCPhysFault, uErrorCode, CPUMGetGuestRIP(pVCpu) ));
665	RT_NOREF(GCPhysFault, uErrorCode);
666
667	VBOXSTRICTRC rcStrict;
668
669	#ifndef IN_RING3
670	/*
671	* Should we defer the request right away? This isn't usually the case, so
672	* do the simple test first and the try deal with uErrorCode being N/A.
673	*/
674	PPDMDEVINS const pDevIns = pRegEntry->pDevIns;
675	if (RT_LIKELY( pDevIns
676	&& iomMmioCanHandlePfInRZ(pVM, uErrorCode, pRegEntry)))
677	{
678	/*
679	* Enter the device critsect prior to engaging IOM in case of lock contention.
680	* Note! Perhaps not a good move?
681	*/
682	rcStrict = PDMCritSectEnter(pVM, pDevIns->CTX_SUFF(pCritSectRo), VINF_IOM_R3_MMIO_READ_WRITE);
683	if (rcStrict == VINF_SUCCESS)
684	{
685	#endif /* !IN_RING3 */
686
687	/*
688	* Let IEM call us back via iomMmioHandler.
689	*/
690	rcStrict = IEMExecOne(pVCpu);
691
692	#ifndef IN_RING3
693	PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo));
694	#endif
695	if (RT_SUCCESS(rcStrict))
696	{ /* likely */ }
697	else if ( rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED
698	\|\| rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
699	{
700	Log(("IOM: Hit unsupported IEM feature!\n"));
701	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
702	}
703	#ifndef IN_RING3
704	return rcStrict;
705	}
706	STAM_COUNTER_INC(&pVM->iom.s.StatMmioDevLockContentionR0);
707	}
708	else
709	rcStrict = VINF_IOM_R3_MMIO_READ_WRITE;
710
711	# ifdef VBOX_WITH_STATISTICS
712	if (rcStrict == VINF_IOM_R3_MMIO_READ_WRITE)
713	{
714	PIOMMMIOSTATSENTRY const pStats = iomMmioGetStats(pVM, pRegEntry);
715	if (uErrorCode & X86_TRAP_PF_RW)
716	{
717	STAM_COUNTER_INC(&pStats->WriteRZToR3);
718	STAM_COUNTER_INC(&pVM->iom.s.StatMmioWritesR0ToR3);
719	}
720	else
721	{
722	STAM_COUNTER_INC(&pStats->ReadRZToR3);
723	STAM_COUNTER_INC(&pVM->iom.s.StatMmioReadsR0ToR3);
724	}
725	}
726	# endif
727	#else /* IN_RING3 */
728	RT_NOREF(pVM, pRegEntry);
729	#endif /* IN_RING3 */
730	return rcStrict;
731	}
732
733
734	/**
735	* @callback_method_impl{FNPGMRZPHYSPFHANDLER,
736	* \#PF access handler callback for MMIO pages.}
737	*
738	* @remarks The @a uUser argument is the MMIO handle.
739	*/
740	DECLCALLBACK(VBOXSTRICTRC) iomMmioPfHandlerNew(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTX pCtx,
741	RTGCPTR pvFault, RTGCPHYS GCPhysFault, uint64_t uUser)
742	{
743	STAM_PROFILE_START(&pVM->iom.s.StatMmioPfHandler, Prf);
744	LogFlow(("iomMmioPfHandlerNew: GCPhys=%RGp uErr=%#x pvFault=%RGv rip=%RGv\n",
745	GCPhysFault, (uint32_t)uErrorCode, pvFault, (RTGCPTR)pCtx->rip));
746	RT_NOREF(pvFault, pCtx);
747
748	/* Translate the MMIO handle to a registration entry for the current context. */
749	AssertReturn(uUser < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
750	# ifdef IN_RING0
751	AssertReturn(uUser < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
752	CTX_SUFF(PIOMMMIOENTRY) pRegEntry = &pVM->iomr0.s.paMmioRegs[uUser];
753	# else
754	CTX_SUFF(PIOMMMIOENTRY) pRegEntry = &pVM->iom.s.paMmioRegs[uUser];
755	# endif
756
757	VBOXSTRICTRC rcStrict = iomMmioCommonPfHandlerNew(pVM, pVCpu, (uint32_t)uErrorCode, GCPhysFault, pRegEntry);
758
759	STAM_PROFILE_STOP(&pVM->iom.s.StatMmioPfHandler, Prf);
760	return rcStrict;
761	}
762
763	#endif /* !IN_RING3 */
764
765	#ifdef IN_RING0
766	/**
767	* Physical access handler for MMIO ranges.
768	*
769	* This is actually only used by VT-x for APIC page accesses.
770	*
771	* @returns VBox status code (appropriate for GC return).
772	* @param pVM The cross context VM structure.
773	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
774	* @param uErrorCode CPU Error code.
775	* @param GCPhysFault The GC physical address.
776	*/
777	VMM_INT_DECL(VBOXSTRICTRC) IOMR0MmioPhysHandler(PVMCC pVM, PVMCPUCC pVCpu, uint32_t uErrorCode, RTGCPHYS GCPhysFault)
778	{
779	STAM_PROFILE_START(&pVM->iom.s.StatMmioPhysHandler, Prf);
780
781	/*
782	* We don't have a range here, so look it up before calling the common function.
783	*/
784	VBOXSTRICTRC rcStrict = IOM_LOCK_SHARED(pVM);
785	if (RT_SUCCESS(rcStrict))
786	{
787	RTGCPHYS offRegion;
788	CTX_SUFF(PIOMMMIOENTRY) pRegEntry = iomMmioGetEntry(pVM, GCPhysFault, &offRegion, &pVCpu->iom.s.idxMmioLastPhysHandler);
789	IOM_UNLOCK_SHARED(pVM);
790	if (RT_LIKELY(pRegEntry))
791	rcStrict = iomMmioCommonPfHandlerNew(pVM, pVCpu, (uint32_t)uErrorCode, GCPhysFault, pRegEntry);
792	else
793	rcStrict = VERR_IOM_MMIO_RANGE_NOT_FOUND;
794	}
795	else if (rcStrict == VERR_SEM_BUSY)
796	rcStrict = VINF_IOM_R3_MMIO_READ_WRITE;
797
798	STAM_PROFILE_STOP(&pVM->iom.s.StatMmioPhysHandler, Prf);
799	return rcStrict;
800	}
801	#endif /* IN_RING0 */
802
803
804	/**
805	* @callback_method_impl{FNPGMPHYSHANDLER, MMIO page accesses}
806	*
807	* @remarks The @a uUser argument is the MMIO handle.
808	*/
809	DECLCALLBACK(VBOXSTRICTRC) iomMmioHandlerNew(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysFault, void pvPhys, void pvBuf,
810	size_t cbBuf, PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, uint64_t uUser)
811	{
812	STAM_PROFILE_START(UnusedMacroArg, Prf);
813	STAM_COUNTER_INC(&pVM->iom.s.CTX_SUFF(StatMmioHandler));
814	Log4(("iomMmioHandlerNew: GCPhysFault=%RGp cbBuf=%#x enmAccessType=%d enmOrigin=%d uUser=%p\n", GCPhysFault, cbBuf, enmAccessType, enmOrigin, uUser));
815
816	Assert(enmAccessType == PGMACCESSTYPE_READ \|\| enmAccessType == PGMACCESSTYPE_WRITE);
817	AssertMsg(cbBuf >= 1, ("%zu\n", cbBuf));
818	NOREF(pvPhys); NOREF(enmOrigin);
819
820	#ifdef IN_RING3
821	int const rcToRing3 = VERR_IOM_MMIO_IPE_3;
822	#else
823	int const rcToRing3 = enmAccessType == PGMACCESSTYPE_READ ? VINF_IOM_R3_MMIO_READ : VINF_IOM_R3_MMIO_WRITE;
824	#endif
825
826	/*
827	* Translate uUser to an MMIO registration table entry. We can do this
828	* without any locking as the data is static after VM creation.
829	*/
830	AssertReturn(uUser < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
831	#ifdef IN_RING0
832	AssertReturn(uUser < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
833	CTX_SUFF(PIOMMMIOENTRY) const pRegEntry = &pVM->iomr0.s.paMmioRegs[uUser];
834	PIOMMMIOENTRYR3 const pRegEntryR3 = &pVM->iomr0.s.paMmioRing3Regs[uUser];
835	#else
836	CTX_SUFF(PIOMMMIOENTRY) const pRegEntry = &pVM->iom.s.paMmioRegs[uUser];
837	#endif
838	#ifdef VBOX_WITH_STATISTICS
839	PIOMMMIOSTATSENTRY const pStats = iomMmioGetStats(pVM, pRegEntry); /* (Works even without ring-0 device setup.) */
840	#endif
841	PPDMDEVINS const pDevIns = pRegEntry->pDevIns;
842
843	#ifdef VBOX_STRICT
844	/*
845	* Assert the right entry in strict builds. This may yield a false positive
846	* for SMP VMs if we're unlucky and the guest isn't well behaved.
847	*/
848	# ifdef IN_RING0
849	Assert(pRegEntry && (GCPhysFault - pRegEntryR3->GCPhysMapping < pRegEntryR3->cbRegion \|\| !pRegEntryR3->fMapped));
850	# else
851	Assert(pRegEntry && (GCPhysFault - pRegEntry->GCPhysMapping < pRegEntry->cbRegion \|\| !pRegEntry->fMapped));
852	# endif
853	#endif
854
855	#ifndef IN_RING3
856	/*
857	* If someone is doing FXSAVE, FXRSTOR, XSAVE, XRSTOR or other stuff dealing with
858	* large amounts of data, just go to ring-3 where we don't need to deal with partial
859	* successes. No chance any of these will be problematic read-modify-write stuff.
860	*
861	* Also drop back if the ring-0 registration entry isn't actually used.
862	*/
863	if ( RT_LIKELY(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue))
864	&& pRegEntry->cbRegion != 0
865	&& ( enmAccessType == PGMACCESSTYPE_READ
866	? pRegEntry->pfnReadCallback != NULL \|\| pVM->iomr0.s.paMmioRing3Regs[uUser].pfnReadCallback == NULL
867	: pRegEntry->pfnWriteCallback != NULL \|\| pVM->iomr0.s.paMmioRing3Regs[uUser].pfnWriteCallback == NULL)
868	&& pDevIns )
869	{ /* likely */ }
870	else
871	{
872	Log4(("iomMmioHandlerNew: to ring-3: to-big=%RTbool zero-size=%RTbool no-callback=%RTbool pDevIns=%p hRegion=%#RX64\n",
873	!(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue)), !(pRegEntry->cbRegion != 0),
874	!( enmAccessType == PGMACCESSTYPE_READ
875	? pRegEntry->pfnReadCallback != NULL \|\| pVM->iomr0.s.paMmioRing3Regs[uUser].pfnReadCallback == NULL
876	: pRegEntry->pfnWriteCallback != NULL \|\| pVM->iomr0.s.paMmioRing3Regs[uUser].pfnWriteCallback == NULL),
877	pDevIns, uUser));
878	STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pStats->ReadRZToR3 : &pStats->WriteRZToR3);
879	STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pVM->iom.s.StatMmioReadsR0ToR3 : &pVM->iom.s.StatMmioWritesR0ToR3);
880	return rcToRing3;
881	}
882	#endif /* !IN_RING3 */
883
884	/*
885	* If we've got an offset that's outside the region, defer to ring-3 if we
886	* can, or pretend there is nothing there. This shouldn't happen, but can
887	* if we're unlucky with an SMP VM and the guest isn't behaving very well.
888	*/
889	#ifdef IN_RING0
890	RTGCPHYS const GCPhysMapping = pRegEntryR3->GCPhysMapping;
891	#else
892	RTGCPHYS const GCPhysMapping = pRegEntry->GCPhysMapping;
893	#endif
894	RTGCPHYS const offRegion = GCPhysFault - GCPhysMapping;
895	if (RT_LIKELY(offRegion < pRegEntry->cbRegion && GCPhysMapping != NIL_RTGCPHYS))
896	{ /* likely */ }
897	else
898	{
899	STAM_REL_COUNTER_INC(&pVM->iom.s.StatMmioStaleMappings);
900	LogRelMax(64, ("iomMmioHandlerNew: Stale access at %#RGp to range #%#x currently residing at %RGp LB %RGp\n",
901	GCPhysFault, pRegEntry->idxSelf, GCPhysMapping, pRegEntry->cbRegion));
902	#ifdef IN_RING3
903	if (enmAccessType == PGMACCESSTYPE_READ)
904	iomMMIODoReadFFs(pvBuf, cbBuf IOM_MMIO_STATS_COMMA_ARG);
905	return VINF_SUCCESS;
906	#else
907	STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pStats->ReadRZToR3 : &pStats->WriteRZToR3);
908	STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pVM->iom.s.StatMmioReadsR0ToR3 : &pVM->iom.s.StatMmioWritesR0ToR3);
909	return rcToRing3;
910	#endif
911	}
912
913	/*
914	* Guard against device configurations causing recursive MMIO accesses
915	* (see @bugref{10315}).
916	*/
917	uint8_t const idxDepth = pVCpu->iom.s.cMmioRecursionDepth;
918	if (RT_LIKELY(idxDepth < RT_ELEMENTS(pVCpu->iom.s.apMmioRecursionStack)))
919	{
920	pVCpu->iom.s.cMmioRecursionDepth = idxDepth + 1;
921	/** @todo Add iomr0 with a apMmioRecursionStack for ring-0. */
922	#ifdef IN_RING3
923	pVCpu->iom.s.apMmioRecursionStack[idxDepth] = pDevIns;
924	#endif
925	}
926	else
927	{
928	STAM_REL_COUNTER_INC(&pVM->iom.s.StatMmioTooDeepRecursion);
929	#ifdef IN_RING3
930	AssertCompile(RT_ELEMENTS(pVCpu->iom.s.apMmioRecursionStack) == 2);
931	LogRelMax(64, ("iomMmioHandlerNew: Too deep recursion %RGp LB %#zx: %p (%s); %p (%s); %p (%s)\n",
932	GCPhysFault, cbBuf, pDevIns, pDevIns->pReg->szName,
933	pVCpu->iom.s.apMmioRecursionStack[1], pVCpu->iom.s.apMmioRecursionStack[1]->pReg->szName,
934	pVCpu->iom.s.apMmioRecursionStack[0], pVCpu->iom.s.apMmioRecursionStack[0]->pReg->szName));
935	#else
936	LogRelMax(64, ("iomMmioHandlerNew: Too deep recursion %RGp LB %#zx!: %p (%s)\n",
937	GCPhysFault, cbBuf, pDevIns, pDevIns->pReg->szName));
938	#endif
939	return VINF_PGM_HANDLER_DO_DEFAULT;
940	}
941
942
943	/*
944	* Perform locking and the access.
945	*
946	* Writes requiring a return to ring-3 are buffered by IOM so IEM can
947	* commit the instruction.
948	*
949	* Note! We may end up locking the device even when the relevant callback is
950	* NULL. This is supposed to be an unlikely case, so not optimized yet.
951	*
952	* Note! All returns goes thru the one return statement at the end of the
953	* function in order to correctly maintaint the recursion counter.
954	*/
955	VBOXSTRICTRC rcStrict = PDMCritSectEnter(pVM, pDevIns->CTX_SUFF(pCritSectRo), rcToRing3);
956	if (rcStrict == VINF_SUCCESS)
957	{
958	if (enmAccessType == PGMACCESSTYPE_READ)
959	{
960	/*
961	* Read.
962	*/
963	rcStrict = iomMmioDoRead(pVM, pRegEntry, GCPhysFault, offRegion, pvBuf, (uint32_t)cbBuf IOM_MMIO_STATS_COMMA_ARG);
964
965	PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo));
966	#ifndef IN_RING3
967	if (rcStrict == VINF_IOM_R3_MMIO_READ)
968	{
969	STAM_COUNTER_INC(&pStats->ReadRZToR3);
970	STAM_COUNTER_INC(&pVM->iom.s.StatMmioReadsR0ToR3);
971	}
972	else
973	#endif
974	STAM_COUNTER_INC(&pStats->Reads);
975	STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfRead), Prf);
976	}
977	else
978	{
979	/*
980	* Write.
981	*/
982	rcStrict = iomMmioDoWrite(pVM, pVCpu, pRegEntry, GCPhysFault, offRegion, pvBuf, (uint32_t)cbBuf IOM_MMIO_STATS_COMMA_ARG);
983	PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo));
984	#ifndef IN_RING3
985	if (rcStrict == VINF_IOM_R3_MMIO_WRITE)
986	rcStrict = iomMmioRing3WritePending(pVCpu, GCPhysFault, pvBuf, cbBuf, pRegEntry->idxSelf);
987	if (rcStrict == VINF_IOM_R3_MMIO_WRITE)
988	{
989	STAM_COUNTER_INC(&pStats->WriteRZToR3);
990	STAM_COUNTER_INC(&pVM->iom.s.StatMmioWritesR0ToR3);
991	}
992	else if (rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE)
993	{
994	STAM_COUNTER_INC(&pStats->CommitRZToR3);
995	STAM_COUNTER_INC(&pVM->iom.s.StatMmioCommitsR0ToR3);
996	}
997	else
998	#endif
999	STAM_COUNTER_INC(&pStats->Writes);
1000	STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfWrite), Prf);
1001	}
1002
1003	/*
1004	* Check the return code.
1005	*/
1006	#ifdef IN_RING3
1007	AssertMsg(rcStrict == VINF_SUCCESS, ("%Rrc - Access type %d - %RGp - %s\n",
1008	VBOXSTRICTRC_VAL(rcStrict), enmAccessType, GCPhysFault, pRegEntry->pszDesc));
1009	#else
1010	AssertMsg( rcStrict == VINF_SUCCESS
1011	\|\| rcStrict == rcToRing3
1012	\|\| (rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE && enmAccessType == PGMACCESSTYPE_WRITE)
1013	\|\| rcStrict == VINF_EM_DBG_STOP
1014	\|\| rcStrict == VINF_EM_DBG_EVENT
1015	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
1016	\|\| rcStrict == VINF_EM_OFF
1017	\|\| rcStrict == VINF_EM_SUSPEND
1018	\|\| rcStrict == VINF_EM_RESET
1019	//\|\| rcStrict == VINF_EM_HALT /* ?? */
1020	//\|\| rcStrict == VINF_EM_NO_MEMORY /* ?? */
1021	, ("%Rrc - Access type %d - %RGp - %s #%u\n",
1022	VBOXSTRICTRC_VAL(rcStrict), enmAccessType, GCPhysFault, pDevIns->pReg->szName, pDevIns->iInstance));
1023	#endif
1024	}
1025	/*
1026	* Deal with enter-critsect failures.
1027	*/
1028	#ifndef IN_RING3
1029	else if (rcStrict == VINF_IOM_R3_MMIO_WRITE)
1030	{
1031	Assert(enmAccessType == PGMACCESSTYPE_WRITE);
1032	rcStrict = iomMmioRing3WritePending(pVCpu, GCPhysFault, pvBuf, cbBuf, pRegEntry->idxSelf);
1033	if (rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE)
1034	{
1035	STAM_COUNTER_INC(&pStats->CommitRZToR3);
1036	STAM_COUNTER_INC(&pVM->iom.s.StatMmioCommitsR0ToR3);
1037	}
1038	else
1039	{
1040	STAM_COUNTER_INC(&pStats->WriteRZToR3);
1041	STAM_COUNTER_INC(&pVM->iom.s.StatMmioWritesR0ToR3);
1042	}
1043	STAM_COUNTER_INC(&pVM->iom.s.StatMmioDevLockContentionR0);
1044	}
1045	else if (rcStrict == VINF_IOM_R3_MMIO_READ)
1046	{
1047	Assert(enmAccessType == PGMACCESSTYPE_READ);
1048	STAM_COUNTER_INC(&pStats->ReadRZToR3);
1049	STAM_COUNTER_INC(&pVM->iom.s.StatMmioDevLockContentionR0);
1050	}
1051	#endif
1052	else
1053	AssertMsg(RT_FAILURE_NP(rcStrict), ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
1054
1055	pVCpu->iom.s.cMmioRecursionDepth = idxDepth;
1056	return rcStrict;
1057	}
1058
1059
1060	/**
1061	* Mapping an MMIO2 page in place of an MMIO page for direct access.
1062	*
1063	* This is a special optimization used by the VGA device. Call
1064	* IOMMmioResetRegion() to undo the mapping.
1065	*
1066	* @returns VBox status code. This API may return VINF_SUCCESS even if no
1067	* remapping is made.
1068	* @retval VERR_SEM_BUSY in ring-0 if we cannot get the IOM lock.
1069	*
1070	* @param pVM The cross context VM structure.
1071	* @param pDevIns The device instance @a hRegion and @a hMmio2 are
1072	* associated with.
1073	* @param hRegion The handle to the MMIO region.
1074	* @param offRegion The offset into @a hRegion of the page to be
1075	* remapped.
1076	* @param hMmio2 The MMIO2 handle.
1077	* @param offMmio2 Offset into @a hMmio2 of the page to be use for the
1078	* mapping.
1079	* @param fPageFlags Page flags to set. Must be (X86_PTE_RW \| X86_PTE_P)
1080	* for the time being.
1081	*/
1082	VMMDECL(int) IOMMmioMapMmio2Page(PVMCC pVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion, RTGCPHYS offRegion,
1083	uint64_t hMmio2, RTGCPHYS offMmio2, uint64_t fPageFlags)
1084	{
1085	/* Currently only called from the VGA device during MMIO. */
1086	Log(("IOMMmioMapMmio2Page %#RX64/%RGp -> %#RX64/%RGp flags=%RX64\n", hRegion, offRegion, hMmio2, offMmio2, fPageFlags));
1087	AssertReturn(fPageFlags == (X86_PTE_RW \| X86_PTE_P), VERR_INVALID_PARAMETER);
1088	AssertReturn(pDevIns, VERR_INVALID_POINTER);
1089
1090	/** @todo Why is this restricted to protected mode??? Try it in all modes! */
1091	PVMCPUCC pVCpu = VMMGetCpu(pVM);
1092
1093	/* This currently only works in real mode, protected mode without paging or with nested paging. */
1094	/** @todo NEM: MMIO page aliasing. */
1095	if ( !HMIsEnabled(pVM) /* useless without VT-x/AMD-V */
1096	\|\| ( CPUMIsGuestInPagedProtectedMode(pVCpu)
1097	&& !HMIsNestedPagingActive(pVM)))
1098	return VINF_SUCCESS; /* ignore / /* @todo return some indicator if we fail here */
1099
1100	/*
1101	* Translate the handle into an entry and check the region offset.
1102	*/
1103	AssertReturn(hRegion < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
1104	#ifdef IN_RING0
1105	AssertReturn(hRegion < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
1106	PIOMMMIOENTRYR3 const pRegEntry = &pVM->iomr0.s.paMmioRing3Regs[hRegion];
1107	AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
1108	AssertReturn(offRegion < pVM->iomr0.s.paMmioRegs[hRegion].cbRegion, VERR_OUT_OF_RANGE);
1109	AssertReturn( pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == pDevIns
1110	\|\| ( pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == NULL
1111	&& pRegEntry->pDevIns == pDevIns->pDevInsForR3), VERR_ACCESS_DENIED);
1112	#else
1113	PIOMMMIOENTRYR3 const pRegEntry = &pVM->iom.s.paMmioRegs[hRegion];
1114	AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
1115	AssertReturn(pRegEntry->pDevIns == pDevIns, VERR_ACCESS_DENIED);
1116	#endif
1117	AssertReturn(offRegion < pRegEntry->cbRegion, VERR_OUT_OF_RANGE);
1118	Assert((pRegEntry->cbRegion & GUEST_PAGE_OFFSET_MASK) == 0);
1119
1120	/*
1121	* When getting and using the mapping address, we must sit on the IOM lock
1122	* to prevent remapping. Shared suffices as we change nothing.
1123	*/
1124	int rc = IOM_LOCK_SHARED(pVM);
1125	if (rc == VINF_SUCCESS)
1126	{
1127	RTGCPHYS const GCPhys = pRegEntry->fMapped ? pRegEntry->GCPhysMapping : NIL_RTGCPHYS;
1128	if (GCPhys != NIL_RTGCPHYS)
1129	{
1130	Assert(!(GCPhys & GUEST_PAGE_OFFSET_MASK));
1131
1132	/*
1133	* Do the aliasing; page align the addresses since PGM is picky.
1134	*/
1135	rc = PGMHandlerPhysicalPageAliasMmio2(pVM, GCPhys, GCPhys + (offRegion & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK),
1136	pDevIns, hMmio2, offMmio2);
1137	}
1138	else
1139	AssertFailedStmt(rc = VERR_IOM_MMIO_REGION_NOT_MAPPED);
1140
1141	IOM_UNLOCK_SHARED(pVM);
1142	}
1143
1144	/** @todo either ditch this or replace it with something that works in the
1145	* nested case, since we really only care about nested paging! */
1146	#if 0
1147	/*
1148	* Modify the shadow page table. Since it's an MMIO page it won't be present and we
1149	* can simply prefetch it.
1150	*
1151	* Note: This is a NOP in the EPT case; we'll just let it fault again to resync the page.
1152	*/
1153	# if 0 /* The assertion is wrong for the PGM_SYNC_CLEAR_PGM_POOL and VINF_PGM_HANDLER_ALREADY_ALIASED cases. */
1154	# ifdef VBOX_STRICT
1155	uint64_t fFlags;
1156	RTHCPHYS HCPhys;
1157	rc = PGMShwGetPage(pVCpu, (RTGCPTR)GCPhys, &fFlags, &HCPhys);
1158	Assert(rc == VERR_PAGE_NOT_PRESENT \|\| rc == VERR_PAGE_TABLE_NOT_PRESENT);
1159	# endif
1160	# endif
1161	rc = PGMPrefetchPage(pVCpu, (RTGCPTR)GCPhys);
1162	Assert(rc == VINF_SUCCESS \|\| rc == VERR_PAGE_NOT_PRESENT \|\| rc == VERR_PAGE_TABLE_NOT_PRESENT);
1163	#endif
1164	return rc;
1165	}
1166
1167
1168	#ifdef IN_RING0 /* VT-x ring-0 only, move to IOMR0Mmio.cpp later. */
1169	/**
1170	* Mapping a HC page in place of an MMIO page for direct access.
1171	*
1172	* This is a special optimization used by the APIC in the VT-x case. This VT-x
1173	* code uses PGMHandlerPhysicalReset rather than IOMMmioResetRegion() to undo
1174	* the effects here.
1175	*
1176	* @todo Make VT-x usage more consistent.
1177	*
1178	* @returns VBox status code.
1179	*
1180	* @param pVM The cross context VM structure.
1181	* @param pVCpu The cross context virtual CPU structure.
1182	* @param GCPhys The address of the MMIO page to be changed.
1183	* @param HCPhys The address of the host physical page.
1184	* @param fPageFlags Page flags to set. Must be (X86_PTE_RW \| X86_PTE_P)
1185	* for the time being.
1186	*/
1187	VMMR0_INT_DECL(int) IOMR0MmioMapMmioHCPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint64_t fPageFlags)
1188	{
1189	/* Currently only called from VT-x code during a page fault. */
1190	Log(("IOMR0MmioMapMmioHCPage %RGp -> %RGp flags=%RX64\n", GCPhys, HCPhys, fPageFlags));
1191
1192	AssertReturn(fPageFlags == (X86_PTE_RW \| X86_PTE_P), VERR_INVALID_PARAMETER);
1193	/** @todo NEM: MMIO page aliasing?? */
1194	Assert(HMIsEnabled(pVM));
1195
1196	# ifdef VBOX_STRICT
1197	/*
1198	* Check input address (it's HM calling, not the device, so no region handle).
1199	*/
1200	int rcSem = IOM_LOCK_SHARED(pVM);
1201	if (rcSem == VINF_SUCCESS)
1202	{
1203	RTGCPHYS offIgn;
1204	uint16_t idxIgn = UINT16_MAX;
1205	PIOMMMIOENTRYR0 pRegEntry = iomMmioGetEntry(pVM, GCPhys, &offIgn, &idxIgn);
1206	IOM_UNLOCK_SHARED(pVM);
1207	Assert(pRegEntry);
1208	Assert(pRegEntry && !(pRegEntry->cbRegion & GUEST_PAGE_OFFSET_MASK));
1209	}
1210	# endif
1211
1212	/*
1213	* Do the aliasing; page align the addresses since PGM is picky.
1214	*/
1215	GCPhys &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1216	HCPhys &= ~(RTHCPHYS)GUEST_PAGE_OFFSET_MASK;
1217
1218	int rc = PGMHandlerPhysicalPageAliasHC(pVM, GCPhys, GCPhys, HCPhys);
1219	AssertRCReturn(rc, rc);
1220
1221	/** @todo either ditch this or replace it with something that works in the
1222	* nested case, since we really only care about nested paging! */
1223
1224	/*
1225	* Modify the shadow page table. Since it's an MMIO page it won't be present and we
1226	* can simply prefetch it.
1227	*
1228	* Note: This is a NOP in the EPT case; we'll just let it fault again to resync the page.
1229	*/
1230	rc = PGMPrefetchPage(pVCpu, (RTGCPTR)GCPhys);
1231	Assert(rc == VINF_SUCCESS \|\| rc == VERR_PAGE_NOT_PRESENT \|\| rc == VERR_PAGE_TABLE_NOT_PRESENT);
1232	return VINF_SUCCESS;
1233	}
1234	#endif
1235
1236
1237	/**
1238	* Reset a previously modified MMIO region; restore the access flags.
1239	*
1240	* This undoes the effects of IOMMmioMapMmio2Page() and is currently only
1241	* intended for some ancient VGA hack. However, it would be great to extend it
1242	* beyond VT-x and/or nested-paging.
1243	*
1244	* @returns VBox status code.
1245	*
1246	* @param pVM The cross context VM structure.
1247	* @param pDevIns The device instance @a hRegion is associated with.
1248	* @param hRegion The handle to the MMIO region.
1249	*/
1250	VMMDECL(int) IOMMmioResetRegion(PVMCC pVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion)
1251	{
1252	Log(("IOMMMIOResetRegion %#RX64\n", hRegion));
1253	AssertReturn(pDevIns, VERR_INVALID_POINTER);
1254
1255	/** @todo Get rid of this this real/protected or nested paging restriction,
1256	* it probably shouldn't be here and would be nasty when the CPU
1257	* changes mode while we have the hack enabled... */
1258	PVMCPUCC pVCpu = VMMGetCpu(pVM);
1259
1260	/* This currently only works in real mode, protected mode without paging or with nested paging. */
1261	/** @todo NEM: MMIO page aliasing. */
1262	if ( !HMIsEnabled(pVM) /* useless without VT-x/AMD-V */
1263	\|\| ( CPUMIsGuestInPagedProtectedMode(pVCpu)
1264	&& !HMIsNestedPagingActive(pVM)))
1265	return VINF_SUCCESS; /* ignore */
1266
1267	/*
1268	* Translate the handle into an entry and mapping address for PGM.
1269	* We have to take the lock to safely access the mapping address here.
1270	*/
1271	AssertReturn(hRegion < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
1272	#ifdef IN_RING0
1273	AssertReturn(hRegion < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
1274	PIOMMMIOENTRYR3 const pRegEntry = &pVM->iomr0.s.paMmioRing3Regs[hRegion];
1275	AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
1276	AssertReturn( pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == pDevIns
1277	\|\| ( pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == NULL
1278	&& pRegEntry->pDevIns == pDevIns->pDevInsForR3), VERR_ACCESS_DENIED);
1279	#else
1280	PIOMMMIOENTRYR3 const pRegEntry = &pVM->iom.s.paMmioRegs[hRegion];
1281	AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
1282	AssertReturn(pRegEntry->pDevIns == pDevIns, VERR_ACCESS_DENIED);
1283	#endif
1284	Assert((pRegEntry->cbRegion & GUEST_PAGE_OFFSET_MASK) == 0);
1285
1286	int rcSem = IOM_LOCK_SHARED(pVM);
1287	RTGCPHYS GCPhys = pRegEntry->fMapped ? pRegEntry->GCPhysMapping : NIL_RTGCPHYS;
1288	if (rcSem == VINF_SUCCESS)
1289	IOM_UNLOCK_SHARED(pVM);
1290
1291	Assert(!(GCPhys & GUEST_PAGE_OFFSET_MASK));
1292	Assert(!(pRegEntry->cbRegion & GUEST_PAGE_OFFSET_MASK));
1293
1294	/*
1295	* Call PGM to do the job work.
1296	*
1297	* After the call, all the pages should be non-present, unless there is
1298	* a page pool flush pending (unlikely).
1299	*/
1300	int rc = PGMHandlerPhysicalReset(pVM, GCPhys);
1301	AssertRC(rc);
1302
1303	# ifdef VBOX_STRICT
1304	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
1305	{
1306	RTGCPHYS cb = pRegEntry->cbRegion;
1307	while (cb)
1308	{
1309	uint64_t fFlags;
1310	RTHCPHYS HCPhys;
1311	rc = PGMShwGetPage(pVCpu, (RTGCPTR)GCPhys, &fFlags, &HCPhys);
1312	Assert(rc == VERR_PAGE_NOT_PRESENT \|\| rc == VERR_PAGE_TABLE_NOT_PRESENT);
1313	cb -= RT_MIN(GUEST_PAGE_SIZE, HOST_PAGE_SIZE);
1314	GCPhys += RT_MIN(GUEST_PAGE_SIZE, HOST_PAGE_SIZE);
1315	}
1316	}
1317	# endif
1318	return rc;
1319	}
1320

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/IOMAllMmioNew.cpp@ 98045

Download in other formats: