IOMAllMmioNew.cpp@ 105834

Last change on this file since 105834 was 104255, checked in by vboxsync, 8 months ago
VMM/IOM: Initialize stack value before calling pfnReadCallback in iomMMIODoComplicatedRead. bugref:10651
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 52.7 KB

Line
1	/* $Id: IOMAllMmioNew.cpp 104255 2024-04-09 15:12:40Z vboxsync $ */
2	/** @file
3	* IOM - Input / Output Monitor - Any Context, MMIO & String I/O.
4	*/
5
6	/*
7	* Copyright (C) 2006-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_IOM_MMIO
33	#define VMCPU_INCL_CPUM_GST_CTX
34	#include <VBox/vmm/iom.h>
35	#include <VBox/vmm/cpum.h>
36	#include <VBox/vmm/pgm.h>
37	#include <VBox/vmm/selm.h>
38	#include <VBox/vmm/mm.h>
39	#include <VBox/vmm/em.h>
40	#include <VBox/vmm/pgm.h>
41	#include <VBox/vmm/trpm.h>
42	#include <VBox/vmm/iem.h>
43	#include "IOMInternal.h"
44	#include <VBox/vmm/vmcc.h>
45	#include <VBox/vmm/vmm.h>
46	#include <VBox/vmm/hm.h>
47	#include "IOMInline.h"
48
49	#include <VBox/vmm/pdmdev.h>
50	#include <VBox/param.h>
51	#include <VBox/err.h>
52	#include <iprt/assert.h>
53	#include <VBox/log.h>
54	#include <iprt/asm.h>
55	#include <iprt/string.h>
56
57
58	/*********************************************************************************************************************************
59	* Defined Constants And Macros *
60	*********************************************************************************************************************************/
61	/** @def IOM_MMIO_STATS_COMMA_DECL
62	* Parameter list declaration for statistics entry pointer. */
63	/** @def IOM_MMIO_STATS_COMMA_ARG
64	* Statistics entry pointer argument. */
65	#if defined(VBOX_WITH_STATISTICS) \|\| defined(DOXYGEN_RUNNING)
66	# define IOM_MMIO_STATS_COMMA_DECL , PIOMMMIOSTATSENTRY pStats
67	# define IOM_MMIO_STATS_COMMA_ARG , pStats
68	#else
69	# define IOM_MMIO_STATS_COMMA_DECL
70	# define IOM_MMIO_STATS_COMMA_ARG
71	#endif
72
73
74
75	#ifndef IN_RING3
76	/**
77	* Defers a pending MMIO write to ring-3.
78	*
79	* @returns VINF_IOM_R3_MMIO_COMMIT_WRITE
80	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
81	* @param GCPhys The write address.
82	* @param pvBuf The bytes being written.
83	* @param cbBuf How many bytes.
84	* @param idxRegEntry The MMIO registration index (handle) if available,
85	* otherwise UINT32_MAX.
86	*/
87	static VBOXSTRICTRC iomMmioRing3WritePending(PVMCPU pVCpu, RTGCPHYS GCPhys, void const *pvBuf, size_t cbBuf,
88	uint32_t idxRegEntry)
89	{
90	Log5(("iomMmioRing3WritePending: %RGp LB %#x (idx=%#x)\n", GCPhys, cbBuf, idxRegEntry));
91	if (pVCpu->iom.s.PendingMmioWrite.cbValue == 0)
92	{
93	pVCpu->iom.s.PendingMmioWrite.GCPhys = GCPhys;
94	AssertReturn(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue), VERR_IOM_MMIO_IPE_2);
95	pVCpu->iom.s.PendingMmioWrite.cbValue = (uint32_t)cbBuf;
96	pVCpu->iom.s.PendingMmioWrite.idxMmioRegionHint = idxRegEntry;
97	memcpy(pVCpu->iom.s.PendingMmioWrite.abValue, pvBuf, cbBuf);
98	}
99	else
100	{
101	/*
102	* Join with pending if adjecent.
103	*
104	* This may happen if the stack overflows into MMIO territory and RSP/ESP/SP
105	* isn't aligned. IEM will bounce buffer the access and do one write for each
106	* page. We get here when the 2nd page part is written.
107	*/
108	uint32_t const cbOldValue = pVCpu->iom.s.PendingMmioWrite.cbValue;
109	AssertMsgReturn(GCPhys == pVCpu->iom.s.PendingMmioWrite.GCPhys + cbOldValue,
110	("pending %RGp LB %#x; incoming %RGp LB %#x\n",
111	pVCpu->iom.s.PendingMmioWrite.GCPhys, cbOldValue, GCPhys, cbBuf),
112	VERR_IOM_MMIO_IPE_1);
113	AssertReturn(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue) - cbOldValue, VERR_IOM_MMIO_IPE_2);
114	pVCpu->iom.s.PendingMmioWrite.cbValue = cbOldValue + (uint32_t)cbBuf;
115	memcpy(&pVCpu->iom.s.PendingMmioWrite.abValue[cbOldValue], pvBuf, cbBuf);
116	}
117
118	VMCPU_FF_SET(pVCpu, VMCPU_FF_IOM);
119	return VINF_IOM_R3_MMIO_COMMIT_WRITE;
120	}
121	#endif
122
123
124	/**
125	* Deals with complicated MMIO writes.
126	*
127	* Complicated means unaligned or non-dword/qword sized accesses depending on
128	* the MMIO region's access mode flags.
129	*
130	* @returns Strict VBox status code. Any EM scheduling status code,
131	* VINF_IOM_R3_MMIO_WRITE, VINF_IOM_R3_MMIO_READ_WRITE or
132	* VINF_IOM_R3_MMIO_READ may be returned.
133	*
134	* @param pVM The cross context VM structure.
135	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
136	* @param pRegEntry The MMIO entry for the current context.
137	* @param GCPhys The physical address to start writing.
138	* @param offRegion MMIO region offset corresponding to @a GCPhys.
139	* @param pvValue Where to store the value.
140	* @param cbValue The size of the value to write.
141	* @param pStats Pointer to the statistics (never NULL).
142	*/
143	static VBOXSTRICTRC iomMmioDoComplicatedWrite(PVM pVM, PVMCPU pVCpu, CTX_SUFF(PIOMMMIOENTRY) pRegEntry,
144	RTGCPHYS GCPhys, RTGCPHYS offRegion,
145	void const *pvValue, unsigned cbValue IOM_MMIO_STATS_COMMA_DECL)
146	{
147	AssertReturn( (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) != IOMMMIO_FLAGS_WRITE_PASSTHRU
148	&& (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) <= IOMMMIO_FLAGS_WRITE_DWORD_QWORD_READ_MISSING,
149	VERR_IOM_MMIO_IPE_1);
150	AssertReturn(cbValue != 0 && cbValue <= 16, VERR_IOM_MMIO_IPE_2);
151	RTGCPHYS const GCPhysStart = GCPhys; NOREF(GCPhysStart);
152	bool const fReadMissing = (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_DWORD_READ_MISSING
153	\|\| (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_DWORD_QWORD_READ_MISSING;
154	RT_NOREF_PV(pVCpu); /* ring-3 */
155
156	/*
157	* Do debug stop if requested.
158	*/
159	VBOXSTRICTRC rc = VINF_SUCCESS; NOREF(pVM);
160	#ifdef VBOX_STRICT
161	if (!(pRegEntry->fFlags & IOMMMIO_FLAGS_DBGSTOP_ON_COMPLICATED_WRITE))
162	{ /* likely */ }
163	else
164	{
165	# ifdef IN_RING3
166	LogRel(("IOM: Complicated write %#x byte at %RGp to %s, initiating debugger intervention\n", cbValue, GCPhys,
167	R3STRING(pRegEntry->pszDesc)));
168	rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, RT_SRC_POS,
169	"Complicated write %#x byte at %RGp to %s\n", cbValue, GCPhys, pRegEntry->pszDesc);
170	if (rc == VERR_DBGF_NOT_ATTACHED)
171	rc = VINF_SUCCESS;
172	# else
173	return VINF_IOM_R3_MMIO_WRITE;
174	# endif
175	}
176	#endif
177
178	STAM_COUNTER_INC(&pStats->ComplicatedWrites);
179
180	/*
181	* Check if we should ignore the write.
182	*/
183	if ((pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_ONLY_DWORD)
184	{
185	Assert(cbValue != 4 \|\| (GCPhys & 3));
186	return VINF_SUCCESS;
187	}
188	if ((pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_ONLY_DWORD_QWORD)
189	{
190	Assert((cbValue != 4 && cbValue != 8) \|\| (GCPhys & (cbValue - 1)));
191	return VINF_SUCCESS;
192	}
193
194	/*
195	* Split and conquer.
196	*/
197	for (;;)
198	{
199	unsigned const offAccess = GCPhys & 3;
200	unsigned cbThisPart = 4 - offAccess;
201	if (cbThisPart > cbValue)
202	cbThisPart = cbValue;
203
204	/*
205	* Get the missing bits (if any).
206	*/
207	uint32_t u32MissingValue = 0;
208	if (fReadMissing && cbThisPart != 4)
209	{
210	VBOXSTRICTRC rc2 = pRegEntry->pfnReadCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
211	!(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS)
212	? offRegion & ~(RTGCPHYS)3 : (GCPhys & ~(RTGCPHYS)3),
213	&u32MissingValue, sizeof(u32MissingValue));
214	switch (VBOXSTRICTRC_VAL(rc2))
215	{
216	case VINF_SUCCESS:
217	break;
218	case VINF_IOM_MMIO_UNUSED_FF:
219	STAM_COUNTER_INC(&pStats->FFor00Reads);
220	u32MissingValue = UINT32_C(0xffffffff);
221	break;
222	case VINF_IOM_MMIO_UNUSED_00:
223	STAM_COUNTER_INC(&pStats->FFor00Reads);
224	u32MissingValue = 0;
225	break;
226	#ifndef IN_RING3
227	case VINF_IOM_R3_MMIO_READ:
228	case VINF_IOM_R3_MMIO_READ_WRITE:
229	case VINF_IOM_R3_MMIO_WRITE:
230	LogFlow(("iomMmioDoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [read]\n", GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rc2)));
231	rc2 = iomMmioRing3WritePending(pVCpu, GCPhys, pvValue, cbValue, pRegEntry->idxSelf);
232	if (rc == VINF_SUCCESS \|\| rc2 < rc)
233	rc = rc2;
234	return rc;
235	#endif
236	default:
237	if (RT_FAILURE(rc2))
238	{
239	Log(("iomMmioDoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [read]\n", GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rc2)));
240	return rc2;
241	}
242	AssertMsgReturn(rc2 >= VINF_EM_FIRST && rc2 <= VINF_EM_LAST, ("%Rrc\n", VBOXSTRICTRC_VAL(rc2)), VERR_IPE_UNEXPECTED_INFO_STATUS);
243	if (rc == VINF_SUCCESS \|\| rc2 < rc)
244	rc = rc2;
245	break;
246	}
247	}
248
249	/*
250	* Merge missing and given bits.
251	*/
252	uint32_t u32GivenMask;
253	uint32_t u32GivenValue;
254	switch (cbThisPart)
255	{
256	case 1:
257	u32GivenValue = (uint8_t const )pvValue;
258	u32GivenMask = UINT32_C(0x000000ff);
259	break;
260	case 2:
261	u32GivenValue = (uint16_t const )pvValue;
262	u32GivenMask = UINT32_C(0x0000ffff);
263	break;
264	case 3:
265	u32GivenValue = RT_MAKE_U32_FROM_U8(((uint8_t const )pvValue)[0], ((uint8_t const )pvValue)[1],
266	((uint8_t const *)pvValue)[2], 0);
267	u32GivenMask = UINT32_C(0x00ffffff);
268	break;
269	case 4:
270	u32GivenValue = (uint32_t const )pvValue;
271	u32GivenMask = UINT32_C(0xffffffff);
272	break;
273	default:
274	AssertFailedReturn(VERR_IOM_MMIO_IPE_3);
275	}
276	if (offAccess)
277	{
278	u32GivenValue <<= offAccess * 8;
279	u32GivenMask <<= offAccess * 8;
280	}
281
282	uint32_t u32Value = (u32MissingValue & ~u32GivenMask)
283	\| (u32GivenValue & u32GivenMask);
284
285	/*
286	* Do DWORD write to the device.
287	*/
288	VBOXSTRICTRC rc2 = pRegEntry->pfnWriteCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
289	!(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS)
290	? offRegion & ~(RTGCPHYS)3 : GCPhys & ~(RTGCPHYS)3,
291	&u32Value, sizeof(u32Value));
292	switch (VBOXSTRICTRC_VAL(rc2))
293	{
294	case VINF_SUCCESS:
295	break;
296	#ifndef IN_RING3
297	case VINF_IOM_R3_MMIO_READ:
298	case VINF_IOM_R3_MMIO_READ_WRITE:
299	case VINF_IOM_R3_MMIO_WRITE:
300	Log3(("iomMmioDoComplicatedWrite: deferring GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [write]\n", GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rc2)));
301	AssertReturn(pVCpu->iom.s.PendingMmioWrite.cbValue == 0, VERR_IOM_MMIO_IPE_1);
302	AssertReturn(cbValue + (GCPhys & 3) <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue), VERR_IOM_MMIO_IPE_2);
303	pVCpu->iom.s.PendingMmioWrite.GCPhys = GCPhys & ~(RTGCPHYS)3;
304	pVCpu->iom.s.PendingMmioWrite.cbValue = cbValue + (GCPhys & 3);
305	(uint32_t )pVCpu->iom.s.PendingMmioWrite.abValue = u32Value;
306	if (cbValue > cbThisPart)
307	memcpy(&pVCpu->iom.s.PendingMmioWrite.abValue[4],
308	(uint8_t const *)pvValue + cbThisPart, cbValue - cbThisPart);
309	VMCPU_FF_SET(pVCpu, VMCPU_FF_IOM);
310	if (rc == VINF_SUCCESS)
311	rc = VINF_IOM_R3_MMIO_COMMIT_WRITE;
312	return rc;
313	#endif
314	default:
315	if (RT_FAILURE(rc2))
316	{
317	Log(("iomMmioDoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [write]\n", GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rc2)));
318	return rc2;
319	}
320	AssertMsgReturn(rc2 >= VINF_EM_FIRST && rc2 <= VINF_EM_LAST, ("%Rrc\n", VBOXSTRICTRC_VAL(rc2)), VERR_IPE_UNEXPECTED_INFO_STATUS);
321	if (rc == VINF_SUCCESS \|\| rc2 < rc)
322	rc = rc2;
323	break;
324	}
325
326	/*
327	* Advance.
328	*/
329	cbValue -= cbThisPart;
330	if (!cbValue)
331	break;
332	GCPhys += cbThisPart;
333	offRegion += cbThisPart;
334	pvValue = (uint8_t const *)pvValue + cbThisPart;
335	}
336
337	return rc;
338	}
339
340
341
342
343	/**
344	* Wrapper which does the write.
345	*/
346	DECLINLINE(VBOXSTRICTRC) iomMmioDoWrite(PVMCC pVM, PVMCPU pVCpu, CTX_SUFF(PIOMMMIOENTRY) pRegEntry,
347	RTGCPHYS GCPhys, RTGCPHYS offRegion,
348	const void *pvData, uint32_t cb IOM_MMIO_STATS_COMMA_DECL)
349	{
350	VBOXSTRICTRC rcStrict;
351	if (RT_LIKELY(pRegEntry->pfnWriteCallback))
352	{
353	if ( (cb == 4 && !(GCPhys & 3))
354	\|\| (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_PASSTHRU
355	\|\| (cb == 8 && !(GCPhys & 7) && IOMMMIO_DOES_WRITE_MODE_ALLOW_QWORD(pRegEntry->fFlags)) )
356	rcStrict = pRegEntry->pfnWriteCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
357	!(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS) ? offRegion : GCPhys, pvData, cb);
358	else
359	rcStrict = iomMmioDoComplicatedWrite(pVM, pVCpu, pRegEntry, GCPhys, offRegion, pvData, cb IOM_MMIO_STATS_COMMA_ARG);
360	}
361	else
362	rcStrict = VINF_SUCCESS;
363	return rcStrict;
364	}
365
366
367	#ifdef IN_RING3
368	/**
369	* Helper for IOMR3ProcessForceFlag() that lives here to utilize iomMmioDoWrite et al.
370	*/
371	VBOXSTRICTRC iomR3MmioCommitWorker(PVM pVM, PVMCPU pVCpu, PIOMMMIOENTRYR3 pRegEntry, RTGCPHYS offRegion)
372	{
373	# ifdef VBOX_WITH_STATISTICS
374	STAM_PROFILE_START(UnusedMacroArg, Prf);
375	PIOMMMIOSTATSENTRY const pStats = iomMmioGetStats(pVM, pRegEntry);
376	# endif
377	PPDMDEVINS const pDevIns = pRegEntry->pDevIns;
378	int rc = PDMCritSectEnter(pVM, pDevIns->CTX_SUFF(pCritSectRo), VERR_IGNORED);
379	AssertRCReturn(rc, rc);
380
381	VBOXSTRICTRC rcStrict = iomMmioDoWrite(pVM, pVCpu, pRegEntry, pVCpu->iom.s.PendingMmioWrite.GCPhys, offRegion,
382	pVCpu->iom.s.PendingMmioWrite.abValue, pVCpu->iom.s.PendingMmioWrite.cbValue
383	IOM_MMIO_STATS_COMMA_ARG);
384
385	PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo));
386	STAM_PROFILE_STOP(&pStats->ProfWriteR3, Prf);
387	return rcStrict;
388	}
389	#endif /* IN_RING3 */
390
391
392	/**
393	* Deals with complicated MMIO reads.
394	*
395	* Complicated means unaligned or non-dword/qword sized accesses depending on
396	* the MMIO region's access mode flags.
397	*
398	* @returns Strict VBox status code. Any EM scheduling status code,
399	* VINF_IOM_R3_MMIO_READ, VINF_IOM_R3_MMIO_READ_WRITE or
400	* VINF_IOM_R3_MMIO_WRITE may be returned.
401	*
402	* @param pVM The cross context VM structure.
403	* @param pRegEntry The MMIO entry for the current context.
404	* @param GCPhys The physical address to start reading.
405	* @param offRegion MMIO region offset corresponding to @a GCPhys.
406	* @param pvValue Where to store the value.
407	* @param cbValue The size of the value to read.
408	* @param pStats Pointer to the statistics (never NULL).
409	*/
410	static VBOXSTRICTRC iomMMIODoComplicatedRead(PVM pVM, CTX_SUFF(PIOMMMIOENTRY) pRegEntry, RTGCPHYS GCPhys, RTGCPHYS offRegion,
411	void *pvValue, unsigned cbValue IOM_MMIO_STATS_COMMA_DECL)
412	{
413	AssertReturn( (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD
414	\|\| (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD_QWORD,
415	VERR_IOM_MMIO_IPE_1);
416	AssertReturn(cbValue != 0 && cbValue <= 16, VERR_IOM_MMIO_IPE_2);
417	#ifdef LOG_ENABLED
418	RTGCPHYS const GCPhysStart = GCPhys;
419	#endif
420
421	/*
422	* Do debug stop if requested.
423	*/
424	VBOXSTRICTRC rc = VINF_SUCCESS; NOREF(pVM);
425	#ifdef VBOX_STRICT
426	if (pRegEntry->fFlags & IOMMMIO_FLAGS_DBGSTOP_ON_COMPLICATED_READ)
427	{
428	# ifdef IN_RING3
429	rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, RT_SRC_POS,
430	"Complicated read %#x byte at %RGp to %s\n", cbValue, GCPhys, R3STRING(pRegEntry->pszDesc));
431	if (rc == VERR_DBGF_NOT_ATTACHED)
432	rc = VINF_SUCCESS;
433	# else
434	return VINF_IOM_R3_MMIO_READ;
435	# endif
436	}
437	#endif
438
439	STAM_COUNTER_INC(&pStats->ComplicatedReads);
440
441	/*
442	* Split and conquer.
443	*/
444	for (;;)
445	{
446	/*
447	* Do DWORD read from the device.
448	*/
449	uint32_t u32Value = 0;
450	VBOXSTRICTRC rcStrict2 = pRegEntry->pfnReadCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
451	!(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS)
452	? offRegion & ~(RTGCPHYS)3 : GCPhys & ~(RTGCPHYS)3,
453	&u32Value, sizeof(u32Value));
454	switch (VBOXSTRICTRC_VAL(rcStrict2))
455	{
456	case VINF_SUCCESS:
457	break;
458	case VINF_IOM_MMIO_UNUSED_FF:
459	STAM_COUNTER_INC(&pStats->FFor00Reads);
460	u32Value = UINT32_C(0xffffffff);
461	break;
462	case VINF_IOM_MMIO_UNUSED_00:
463	STAM_COUNTER_INC(&pStats->FFor00Reads);
464	u32Value = 0;
465	break;
466	case VINF_IOM_R3_MMIO_READ:
467	case VINF_IOM_R3_MMIO_READ_WRITE:
468	case VINF_IOM_R3_MMIO_WRITE:
469	/** @todo What if we've split a transfer and already read
470	* something? Since reads can have sideeffects we could be
471	* kind of screwed here... */
472	LogFlow(("iomMMIODoComplicatedRead: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rcStrict2=%Rrc\n",
473	GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rcStrict2)));
474	return rcStrict2;
475	default:
476	if (RT_FAILURE(rcStrict2))
477	{
478	Log(("iomMMIODoComplicatedRead: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rcStrict2=%Rrc\n",
479	GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rcStrict2)));
480	return rcStrict2;
481	}
482	AssertMsgReturn(rcStrict2 >= VINF_EM_FIRST && rcStrict2 <= VINF_EM_LAST, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)),
483	VERR_IPE_UNEXPECTED_INFO_STATUS);
484	if (rc == VINF_SUCCESS \|\| rcStrict2 < rc)
485	rc = rcStrict2;
486	break;
487	}
488	u32Value >>= (GCPhys & 3) * 8;
489
490	/*
491	* Write what we've read.
492	*/
493	unsigned cbThisPart = 4 - (GCPhys & 3);
494	if (cbThisPart > cbValue)
495	cbThisPart = cbValue;
496
497	switch (cbThisPart)
498	{
499	case 1:
500	(uint8_t )pvValue = (uint8_t)u32Value;
501	break;
502	case 2:
503	(uint16_t )pvValue = (uint16_t)u32Value;
504	break;
505	case 3:
506	((uint8_t *)pvValue)[0] = RT_BYTE1(u32Value);
507	((uint8_t *)pvValue)[1] = RT_BYTE2(u32Value);
508	((uint8_t *)pvValue)[2] = RT_BYTE3(u32Value);
509	break;
510	case 4:
511	(uint32_t )pvValue = u32Value;
512	break;
513	}
514
515	/*
516	* Advance.
517	*/
518	cbValue -= cbThisPart;
519	if (!cbValue)
520	break;
521	GCPhys += cbThisPart;
522	offRegion += cbThisPart;
523	pvValue = (uint8_t *)pvValue + cbThisPart;
524	}
525
526	return rc;
527	}
528
529
530	/**
531	* Implements VINF_IOM_MMIO_UNUSED_FF.
532	*
533	* @returns VINF_SUCCESS.
534	* @param pvValue Where to store the zeros.
535	* @param cbValue How many bytes to read.
536	* @param pStats Pointer to the statistics (never NULL).
537	*/
538	static int iomMMIODoReadFFs(void *pvValue, size_t cbValue IOM_MMIO_STATS_COMMA_DECL)
539	{
540	switch (cbValue)
541	{
542	case 1: (uint8_t )pvValue = UINT8_C(0xff); break;
543	case 2: (uint16_t )pvValue = UINT16_C(0xffff); break;
544	case 4: (uint32_t )pvValue = UINT32_C(0xffffffff); break;
545	case 8: (uint64_t )pvValue = UINT64_C(0xffffffffffffffff); break;
546	default:
547	{
548	uint8_t pb = (uint8_t )pvValue;
549	while (cbValue--)
550	*pb++ = UINT8_C(0xff);
551	break;
552	}
553	}
554	STAM_COUNTER_INC(&pStats->FFor00Reads);
555	return VINF_SUCCESS;
556	}
557
558
559	/**
560	* Implements VINF_IOM_MMIO_UNUSED_00.
561	*
562	* @returns VINF_SUCCESS.
563	* @param pvValue Where to store the zeros.
564	* @param cbValue How many bytes to read.
565	* @param pStats Pointer to the statistics (never NULL).
566	*/
567	static int iomMMIODoRead00s(void *pvValue, size_t cbValue IOM_MMIO_STATS_COMMA_DECL)
568	{
569	switch (cbValue)
570	{
571	case 1: (uint8_t )pvValue = UINT8_C(0x00); break;
572	case 2: (uint16_t )pvValue = UINT16_C(0x0000); break;
573	case 4: (uint32_t )pvValue = UINT32_C(0x00000000); break;
574	case 8: (uint64_t )pvValue = UINT64_C(0x0000000000000000); break;
575	default:
576	{
577	uint8_t pb = (uint8_t )pvValue;
578	while (cbValue--)
579	*pb++ = UINT8_C(0x00);
580	break;
581	}
582	}
583	STAM_COUNTER_INC(&pStats->FFor00Reads);
584	return VINF_SUCCESS;
585	}
586
587
588	/**
589	* Wrapper which does the read.
590	*/
591	DECLINLINE(VBOXSTRICTRC) iomMmioDoRead(PVMCC pVM, CTX_SUFF(PIOMMMIOENTRY) pRegEntry, RTGCPHYS GCPhys, RTGCPHYS offRegion,
592	void *pvValue, uint32_t cbValue IOM_MMIO_STATS_COMMA_DECL)
593	{
594	VBOXSTRICTRC rcStrict;
595	if (RT_LIKELY(pRegEntry->pfnReadCallback))
596	{
597	if ( ( cbValue == 4
598	&& !(GCPhys & 3))
599	\|\| (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_PASSTHRU
600	\|\| ( cbValue == 8
601	&& !(GCPhys & 7)
602	&& (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD_QWORD ) )
603	rcStrict = pRegEntry->pfnReadCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
604	!(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS) ? offRegion : GCPhys, pvValue, cbValue);
605	else
606	rcStrict = iomMMIODoComplicatedRead(pVM, pRegEntry, GCPhys, offRegion, pvValue, cbValue IOM_MMIO_STATS_COMMA_ARG);
607	}
608	else
609	rcStrict = VINF_IOM_MMIO_UNUSED_FF;
610	if (rcStrict != VINF_SUCCESS)
611	{
612	switch (VBOXSTRICTRC_VAL(rcStrict))
613	{
614	case VINF_IOM_MMIO_UNUSED_FF: rcStrict = iomMMIODoReadFFs(pvValue, cbValue IOM_MMIO_STATS_COMMA_ARG); break;
615	case VINF_IOM_MMIO_UNUSED_00: rcStrict = iomMMIODoRead00s(pvValue, cbValue IOM_MMIO_STATS_COMMA_ARG); break;
616	}
617	}
618	return rcStrict;
619	}
620
621	#ifndef IN_RING3
622
623	/**
624	* Checks if we can handle an MMIO \#PF in R0/RC.
625	*/
626	DECLINLINE(bool) iomMmioCanHandlePfInRZ(PVMCC pVM, uint32_t uErrorCode, CTX_SUFF(PIOMMMIOENTRY) pRegEntry)
627	{
628	if (pRegEntry->cbRegion > 0)
629	{
630	if ( pRegEntry->pfnWriteCallback
631	&& pRegEntry->pfnReadCallback)
632	return true;
633
634	PIOMMMIOENTRYR3 const pRegEntryR3 = &pVM->iomr0.s.paMmioRing3Regs[pRegEntry->idxSelf];
635	if ( uErrorCode == UINT32_MAX
636	? pRegEntryR3->pfnWriteCallback \|\| pRegEntryR3->pfnReadCallback
637	: uErrorCode & X86_TRAP_PF_RW
638	? !pRegEntry->pfnWriteCallback && pRegEntryR3->pfnWriteCallback
639	: !pRegEntry->pfnReadCallback && pRegEntryR3->pfnReadCallback)
640	return false;
641
642	return true;
643	}
644	return false;
645	}
646
647
648	/**
649	* Common worker for the \#PF handler and IOMMMIOPhysHandler (APIC+VT-x).
650	*
651	* @returns VBox status code (appropriate for GC return).
652	* @param pVM The cross context VM structure.
653	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
654	* @param uErrorCode CPU Error code. This is UINT32_MAX when we don't have
655	* any error code (the EPT misconfig hack).
656	* @param GCPhysFault The GC physical address corresponding to pvFault.
657	* @param pRegEntry The MMIO entry for the current context.
658	*/
659	DECLINLINE(VBOXSTRICTRC) iomMmioCommonPfHandlerNew(PVMCC pVM, PVMCPUCC pVCpu, uint32_t uErrorCode,
660	RTGCPHYS GCPhysFault, CTX_SUFF(PIOMMMIOENTRY) pRegEntry)
661	{
662	Log(("iomMmioCommonPfHandler: GCPhysFault=%RGp uErr=%#x rip=%RGv\n", GCPhysFault, uErrorCode, CPUMGetGuestRIP(pVCpu) ));
663	RT_NOREF(GCPhysFault, uErrorCode);
664
665	VBOXSTRICTRC rcStrict;
666
667	#ifndef IN_RING3
668	/*
669	* Should we defer the request right away? This isn't usually the case, so
670	* do the simple test first and the try deal with uErrorCode being N/A.
671	*/
672	PPDMDEVINS const pDevIns = pRegEntry->pDevIns;
673	if (RT_LIKELY( pDevIns
674	&& iomMmioCanHandlePfInRZ(pVM, uErrorCode, pRegEntry)))
675	{
676	/*
677	* Enter the device critsect prior to engaging IOM in case of lock contention.
678	* Note! Perhaps not a good move?
679	*/
680	rcStrict = PDMCritSectEnter(pVM, pDevIns->CTX_SUFF(pCritSectRo), VINF_IOM_R3_MMIO_READ_WRITE);
681	if (rcStrict == VINF_SUCCESS)
682	{
683	#endif /* !IN_RING3 */
684
685	/*
686	* Let IEM call us back via iomMmioHandler.
687	*/
688	rcStrict = IEMExecOne(pVCpu);
689
690	#ifndef IN_RING3
691	PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo));
692	#endif
693	if (RT_SUCCESS(rcStrict))
694	{ /* likely */ }
695	else if ( rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED
696	\|\| rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
697	{
698	Log(("IOM: Hit unsupported IEM feature!\n"));
699	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
700	}
701	#ifndef IN_RING3
702	return rcStrict;
703	}
704	STAM_COUNTER_INC(&pVM->iom.s.StatMmioDevLockContentionR0);
705	}
706	else
707	rcStrict = VINF_IOM_R3_MMIO_READ_WRITE;
708
709	# ifdef VBOX_WITH_STATISTICS
710	if (rcStrict == VINF_IOM_R3_MMIO_READ_WRITE)
711	{
712	PIOMMMIOSTATSENTRY const pStats = iomMmioGetStats(pVM, pRegEntry);
713	if (uErrorCode & X86_TRAP_PF_RW)
714	{
715	STAM_COUNTER_INC(&pStats->WriteRZToR3);
716	STAM_COUNTER_INC(&pVM->iom.s.StatMmioWritesR0ToR3);
717	}
718	else
719	{
720	STAM_COUNTER_INC(&pStats->ReadRZToR3);
721	STAM_COUNTER_INC(&pVM->iom.s.StatMmioReadsR0ToR3);
722	}
723	}
724	# endif
725	#else /* IN_RING3 */
726	RT_NOREF(pVM, pRegEntry);
727	#endif /* IN_RING3 */
728	return rcStrict;
729	}
730
731
732	/**
733	* @callback_method_impl{FNPGMRZPHYSPFHANDLER,
734	* \#PF access handler callback for MMIO pages.}
735	*
736	* @remarks The @a uUser argument is the MMIO handle.
737	*/
738	DECLCALLBACK(VBOXSTRICTRC) iomMmioPfHandlerNew(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTX pCtx,
739	RTGCPTR pvFault, RTGCPHYS GCPhysFault, uint64_t uUser)
740	{
741	STAM_PROFILE_START(&pVM->iom.s.StatMmioPfHandler, Prf);
742	LogFlow(("iomMmioPfHandlerNew: GCPhys=%RGp uErr=%#x pvFault=%RGv rip=%RGv\n",
743	GCPhysFault, (uint32_t)uErrorCode, pvFault, (RTGCPTR)pCtx->rip));
744	RT_NOREF(pvFault, pCtx);
745
746	/* Translate the MMIO handle to a registration entry for the current context. */
747	AssertReturn(uUser < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
748	# ifdef IN_RING0
749	AssertReturn(uUser < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
750	CTX_SUFF(PIOMMMIOENTRY) pRegEntry = &pVM->iomr0.s.paMmioRegs[uUser];
751	# else
752	CTX_SUFF(PIOMMMIOENTRY) pRegEntry = &pVM->iom.s.paMmioRegs[uUser];
753	# endif
754
755	VBOXSTRICTRC rcStrict = iomMmioCommonPfHandlerNew(pVM, pVCpu, (uint32_t)uErrorCode, GCPhysFault, pRegEntry);
756
757	STAM_PROFILE_STOP(&pVM->iom.s.StatMmioPfHandler, Prf);
758	return rcStrict;
759	}
760
761	#endif /* !IN_RING3 */
762
763	#ifdef IN_RING0
764	/**
765	* Physical access handler for MMIO ranges.
766	*
767	* This is actually only used by VT-x for APIC page accesses.
768	*
769	* @returns VBox status code (appropriate for GC return).
770	* @param pVM The cross context VM structure.
771	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
772	* @param uErrorCode CPU Error code.
773	* @param GCPhysFault The GC physical address.
774	*/
775	VMM_INT_DECL(VBOXSTRICTRC) IOMR0MmioPhysHandler(PVMCC pVM, PVMCPUCC pVCpu, uint32_t uErrorCode, RTGCPHYS GCPhysFault)
776	{
777	STAM_PROFILE_START(&pVM->iom.s.StatMmioPhysHandler, Prf);
778
779	/*
780	* We don't have a range here, so look it up before calling the common function.
781	*/
782	VBOXSTRICTRC rcStrict = IOM_LOCK_SHARED(pVM);
783	if (RT_SUCCESS(rcStrict))
784	{
785	RTGCPHYS offRegion;
786	CTX_SUFF(PIOMMMIOENTRY) pRegEntry = iomMmioGetEntry(pVM, GCPhysFault, &offRegion, &pVCpu->iom.s.idxMmioLastPhysHandler);
787	IOM_UNLOCK_SHARED(pVM);
788	if (RT_LIKELY(pRegEntry))
789	rcStrict = iomMmioCommonPfHandlerNew(pVM, pVCpu, (uint32_t)uErrorCode, GCPhysFault, pRegEntry);
790	else
791	rcStrict = VERR_IOM_MMIO_RANGE_NOT_FOUND;
792	}
793	else if (rcStrict == VERR_SEM_BUSY)
794	rcStrict = VINF_IOM_R3_MMIO_READ_WRITE;
795
796	STAM_PROFILE_STOP(&pVM->iom.s.StatMmioPhysHandler, Prf);
797	return rcStrict;
798	}
799	#endif /* IN_RING0 */
800
801
802	/**
803	* @callback_method_impl{FNPGMPHYSHANDLER, MMIO page accesses}
804	*
805	* @remarks The @a uUser argument is the MMIO handle.
806	*/
807	DECLCALLBACK(VBOXSTRICTRC) iomMmioHandlerNew(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysFault, void pvPhys, void pvBuf,
808	size_t cbBuf, PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, uint64_t uUser)
809	{
810	STAM_PROFILE_START(UnusedMacroArg, Prf);
811	STAM_COUNTER_INC(&pVM->iom.s.CTX_SUFF(StatMmioHandler));
812	Log4(("iomMmioHandlerNew: GCPhysFault=%RGp cbBuf=%#x enmAccessType=%d enmOrigin=%d uUser=%p\n", GCPhysFault, cbBuf, enmAccessType, enmOrigin, uUser));
813
814	Assert(enmAccessType == PGMACCESSTYPE_READ \|\| enmAccessType == PGMACCESSTYPE_WRITE);
815	AssertMsg(cbBuf >= 1, ("%zu\n", cbBuf));
816	NOREF(pvPhys); NOREF(enmOrigin);
817
818	#ifdef IN_RING3
819	int const rcToRing3 = VERR_IOM_MMIO_IPE_3;
820	#else
821	int const rcToRing3 = enmAccessType == PGMACCESSTYPE_READ ? VINF_IOM_R3_MMIO_READ : VINF_IOM_R3_MMIO_WRITE;
822	#endif
823
824	/*
825	* Translate uUser to an MMIO registration table entry. We can do this
826	* without any locking as the data is static after VM creation.
827	*/
828	AssertReturn(uUser < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
829	#ifdef IN_RING0
830	AssertReturn(uUser < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
831	CTX_SUFF(PIOMMMIOENTRY) const pRegEntry = &pVM->iomr0.s.paMmioRegs[uUser];
832	PIOMMMIOENTRYR3 const pRegEntryR3 = &pVM->iomr0.s.paMmioRing3Regs[uUser];
833	#else
834	CTX_SUFF(PIOMMMIOENTRY) const pRegEntry = &pVM->iom.s.paMmioRegs[uUser];
835	#endif
836	#ifdef VBOX_WITH_STATISTICS
837	PIOMMMIOSTATSENTRY const pStats = iomMmioGetStats(pVM, pRegEntry); /* (Works even without ring-0 device setup.) */
838	#endif
839	PPDMDEVINS const pDevIns = pRegEntry->pDevIns;
840
841	#ifdef VBOX_STRICT
842	/*
843	* Assert the right entry in strict builds. This may yield a false positive
844	* for SMP VMs if we're unlucky and the guest isn't well behaved.
845	*/
846	# ifdef IN_RING0
847	Assert(pRegEntry && (GCPhysFault - pRegEntryR3->GCPhysMapping < pRegEntryR3->cbRegion \|\| !pRegEntryR3->fMapped));
848	# else
849	Assert(pRegEntry && (GCPhysFault - pRegEntry->GCPhysMapping < pRegEntry->cbRegion \|\| !pRegEntry->fMapped));
850	# endif
851	#endif
852
853	#ifndef IN_RING3
854	/*
855	* If someone is doing FXSAVE, FXRSTOR, XSAVE, XRSTOR or other stuff dealing with
856	* large amounts of data, just go to ring-3 where we don't need to deal with partial
857	* successes. No chance any of these will be problematic read-modify-write stuff.
858	*
859	* Also drop back if the ring-0 registration entry isn't actually used.
860	*/
861	if ( RT_LIKELY(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue))
862	&& pRegEntry->cbRegion != 0
863	&& ( enmAccessType == PGMACCESSTYPE_READ
864	? pRegEntry->pfnReadCallback != NULL \|\| pVM->iomr0.s.paMmioRing3Regs[uUser].pfnReadCallback == NULL
865	: pRegEntry->pfnWriteCallback != NULL \|\| pVM->iomr0.s.paMmioRing3Regs[uUser].pfnWriteCallback == NULL)
866	&& pDevIns )
867	{ /* likely */ }
868	else
869	{
870	Log4(("iomMmioHandlerNew: to ring-3: to-big=%RTbool zero-size=%RTbool no-callback=%RTbool pDevIns=%p hRegion=%#RX64\n",
871	!(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue)), !(pRegEntry->cbRegion != 0),
872	!( enmAccessType == PGMACCESSTYPE_READ
873	? pRegEntry->pfnReadCallback != NULL \|\| pVM->iomr0.s.paMmioRing3Regs[uUser].pfnReadCallback == NULL
874	: pRegEntry->pfnWriteCallback != NULL \|\| pVM->iomr0.s.paMmioRing3Regs[uUser].pfnWriteCallback == NULL),
875	pDevIns, uUser));
876	STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pStats->ReadRZToR3 : &pStats->WriteRZToR3);
877	STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pVM->iom.s.StatMmioReadsR0ToR3 : &pVM->iom.s.StatMmioWritesR0ToR3);
878	return rcToRing3;
879	}
880	#endif /* !IN_RING3 */
881
882	/*
883	* If we've got an offset that's outside the region, defer to ring-3 if we
884	* can, or pretend there is nothing there. This shouldn't happen, but can
885	* if we're unlucky with an SMP VM and the guest isn't behaving very well.
886	*/
887	#ifdef IN_RING0
888	RTGCPHYS const GCPhysMapping = pRegEntryR3->GCPhysMapping;
889	#else
890	RTGCPHYS const GCPhysMapping = pRegEntry->GCPhysMapping;
891	#endif
892	RTGCPHYS const offRegion = GCPhysFault - GCPhysMapping;
893	if (RT_LIKELY(offRegion < pRegEntry->cbRegion && GCPhysMapping != NIL_RTGCPHYS))
894	{ /* likely */ }
895	else
896	{
897	STAM_REL_COUNTER_INC(&pVM->iom.s.StatMmioStaleMappings);
898	LogRelMax(64, ("iomMmioHandlerNew: Stale access at %#RGp to range #%#x currently residing at %RGp LB %RGp\n",
899	GCPhysFault, pRegEntry->idxSelf, GCPhysMapping, pRegEntry->cbRegion));
900	#ifdef IN_RING3
901	if (enmAccessType == PGMACCESSTYPE_READ)
902	iomMMIODoReadFFs(pvBuf, cbBuf IOM_MMIO_STATS_COMMA_ARG);
903	return VINF_SUCCESS;
904	#else
905	STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pStats->ReadRZToR3 : &pStats->WriteRZToR3);
906	STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pVM->iom.s.StatMmioReadsR0ToR3 : &pVM->iom.s.StatMmioWritesR0ToR3);
907	return rcToRing3;
908	#endif
909	}
910
911	/*
912	* Guard against device configurations causing recursive MMIO accesses
913	* (see @bugref{10315}).
914	*/
915	uint8_t const idxDepth = pVCpu->iom.s.cMmioRecursionDepth;
916	if (RT_LIKELY(idxDepth < RT_ELEMENTS(pVCpu->iom.s.apMmioRecursionStack)))
917	{
918	pVCpu->iom.s.cMmioRecursionDepth = idxDepth + 1;
919	/** @todo Add iomr0 with a apMmioRecursionStack for ring-0. */
920	#ifdef IN_RING3
921	pVCpu->iom.s.apMmioRecursionStack[idxDepth] = pDevIns;
922	#endif
923	}
924	else
925	{
926	STAM_REL_COUNTER_INC(&pVM->iom.s.StatMmioTooDeepRecursion);
927	#ifdef IN_RING3
928	AssertCompile(RT_ELEMENTS(pVCpu->iom.s.apMmioRecursionStack) == 2);
929	LogRelMax(64, ("iomMmioHandlerNew: Too deep recursion %RGp LB %#zx: %p (%s); %p (%s); %p (%s)\n",
930	GCPhysFault, cbBuf, pDevIns, pDevIns->pReg->szName,
931	pVCpu->iom.s.apMmioRecursionStack[1], pVCpu->iom.s.apMmioRecursionStack[1]->pReg->szName,
932	pVCpu->iom.s.apMmioRecursionStack[0], pVCpu->iom.s.apMmioRecursionStack[0]->pReg->szName));
933	#else
934	LogRelMax(64, ("iomMmioHandlerNew: Too deep recursion %RGp LB %#zx!: %p (%s)\n",
935	GCPhysFault, cbBuf, pDevIns, pDevIns->pReg->szName));
936	#endif
937	return VINF_PGM_HANDLER_DO_DEFAULT;
938	}
939
940
941	/*
942	* Perform locking and the access.
943	*
944	* Writes requiring a return to ring-3 are buffered by IOM so IEM can
945	* commit the instruction.
946	*
947	* Note! We may end up locking the device even when the relevant callback is
948	* NULL. This is supposed to be an unlikely case, so not optimized yet.
949	*
950	* Note! All returns goes thru the one return statement at the end of the
951	* function in order to correctly maintaint the recursion counter.
952	*/
953	VBOXSTRICTRC rcStrict = PDMCritSectEnter(pVM, pDevIns->CTX_SUFF(pCritSectRo), rcToRing3);
954	if (rcStrict == VINF_SUCCESS)
955	{
956	if (enmAccessType == PGMACCESSTYPE_READ)
957	{
958	/*
959	* Read.
960	*/
961	rcStrict = iomMmioDoRead(pVM, pRegEntry, GCPhysFault, offRegion, pvBuf, (uint32_t)cbBuf IOM_MMIO_STATS_COMMA_ARG);
962
963	PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo));
964	#ifndef IN_RING3
965	if (rcStrict == VINF_IOM_R3_MMIO_READ)
966	{
967	STAM_COUNTER_INC(&pStats->ReadRZToR3);
968	STAM_COUNTER_INC(&pVM->iom.s.StatMmioReadsR0ToR3);
969	}
970	else
971	#endif
972	STAM_COUNTER_INC(&pStats->Reads);
973	STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfRead), Prf);
974	}
975	else
976	{
977	/*
978	* Write.
979	*/
980	rcStrict = iomMmioDoWrite(pVM, pVCpu, pRegEntry, GCPhysFault, offRegion, pvBuf, (uint32_t)cbBuf IOM_MMIO_STATS_COMMA_ARG);
981	PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo));
982	#ifndef IN_RING3
983	if (rcStrict == VINF_IOM_R3_MMIO_WRITE)
984	rcStrict = iomMmioRing3WritePending(pVCpu, GCPhysFault, pvBuf, cbBuf, pRegEntry->idxSelf);
985	if (rcStrict == VINF_IOM_R3_MMIO_WRITE)
986	{
987	STAM_COUNTER_INC(&pStats->WriteRZToR3);
988	STAM_COUNTER_INC(&pVM->iom.s.StatMmioWritesR0ToR3);
989	}
990	else if (rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE)
991	{
992	STAM_COUNTER_INC(&pStats->CommitRZToR3);
993	STAM_COUNTER_INC(&pVM->iom.s.StatMmioCommitsR0ToR3);
994	}
995	else
996	#endif
997	STAM_COUNTER_INC(&pStats->Writes);
998	STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfWrite), Prf);
999	}
1000
1001	/*
1002	* Check the return code.
1003	*/
1004	#ifdef IN_RING3
1005	AssertMsg(rcStrict == VINF_SUCCESS, ("%Rrc - Access type %d - %RGp - %s\n",
1006	VBOXSTRICTRC_VAL(rcStrict), enmAccessType, GCPhysFault, pRegEntry->pszDesc));
1007	#else
1008	AssertMsg( rcStrict == VINF_SUCCESS
1009	\|\| rcStrict == rcToRing3
1010	\|\| (rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE && enmAccessType == PGMACCESSTYPE_WRITE)
1011	\|\| rcStrict == VINF_EM_DBG_STOP
1012	\|\| rcStrict == VINF_EM_DBG_EVENT
1013	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
1014	\|\| rcStrict == VINF_EM_OFF
1015	\|\| rcStrict == VINF_EM_SUSPEND
1016	\|\| rcStrict == VINF_EM_RESET
1017	//\|\| rcStrict == VINF_EM_HALT /* ?? */
1018	//\|\| rcStrict == VINF_EM_NO_MEMORY /* ?? */
1019	, ("%Rrc - Access type %d - %RGp - %s #%u\n",
1020	VBOXSTRICTRC_VAL(rcStrict), enmAccessType, GCPhysFault, pDevIns->pReg->szName, pDevIns->iInstance));
1021	#endif
1022	}
1023	/*
1024	* Deal with enter-critsect failures.
1025	*/
1026	#ifndef IN_RING3
1027	else if (rcStrict == VINF_IOM_R3_MMIO_WRITE)
1028	{
1029	Assert(enmAccessType == PGMACCESSTYPE_WRITE);
1030	rcStrict = iomMmioRing3WritePending(pVCpu, GCPhysFault, pvBuf, cbBuf, pRegEntry->idxSelf);
1031	if (rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE)
1032	{
1033	STAM_COUNTER_INC(&pStats->CommitRZToR3);
1034	STAM_COUNTER_INC(&pVM->iom.s.StatMmioCommitsR0ToR3);
1035	}
1036	else
1037	{
1038	STAM_COUNTER_INC(&pStats->WriteRZToR3);
1039	STAM_COUNTER_INC(&pVM->iom.s.StatMmioWritesR0ToR3);
1040	}
1041	STAM_COUNTER_INC(&pVM->iom.s.StatMmioDevLockContentionR0);
1042	}
1043	else if (rcStrict == VINF_IOM_R3_MMIO_READ)
1044	{
1045	Assert(enmAccessType == PGMACCESSTYPE_READ);
1046	STAM_COUNTER_INC(&pStats->ReadRZToR3);
1047	STAM_COUNTER_INC(&pVM->iom.s.StatMmioDevLockContentionR0);
1048	}
1049	#endif
1050	else
1051	AssertMsg(RT_FAILURE_NP(rcStrict), ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
1052
1053	pVCpu->iom.s.cMmioRecursionDepth = idxDepth;
1054	return rcStrict;
1055	}
1056
1057
1058	/**
1059	* Mapping an MMIO2 page in place of an MMIO page for direct access.
1060	*
1061	* This is a special optimization used by the VGA device. Call
1062	* IOMMmioResetRegion() to undo the mapping.
1063	*
1064	* @returns VBox status code. This API may return VINF_SUCCESS even if no
1065	* remapping is made.
1066	* @retval VERR_SEM_BUSY in ring-0 if we cannot get the IOM lock.
1067	*
1068	* @param pVM The cross context VM structure.
1069	* @param pDevIns The device instance @a hRegion and @a hMmio2 are
1070	* associated with.
1071	* @param hRegion The handle to the MMIO region.
1072	* @param offRegion The offset into @a hRegion of the page to be
1073	* remapped.
1074	* @param hMmio2 The MMIO2 handle.
1075	* @param offMmio2 Offset into @a hMmio2 of the page to be use for the
1076	* mapping.
1077	* @param fPageFlags Page flags to set. Must be (X86_PTE_RW \| X86_PTE_P)
1078	* for the time being.
1079	*/
1080	VMMDECL(int) IOMMmioMapMmio2Page(PVMCC pVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion, RTGCPHYS offRegion,
1081	uint64_t hMmio2, RTGCPHYS offMmio2, uint64_t fPageFlags)
1082	{
1083	/* Currently only called from the VGA device during MMIO. */
1084	Log(("IOMMmioMapMmio2Page %#RX64/%RGp -> %#RX64/%RGp flags=%RX64\n", hRegion, offRegion, hMmio2, offMmio2, fPageFlags));
1085	AssertReturn(fPageFlags == (X86_PTE_RW \| X86_PTE_P), VERR_INVALID_PARAMETER);
1086	AssertReturn(pDevIns, VERR_INVALID_POINTER);
1087
1088	#if defined(VBOX_VMM_TARGET_ARMV8)
1089	/** @todo NEM: MMIO page aliasing. */
1090	RT_NOREF(pVM, hRegion, offRegion, hMmio2, offMmio2);
1091	return VINF_SUCCESS; /* ignore / /* @todo return some indicator if we fail here */
1092	#else
1093	/** @todo Why is this restricted to protected mode??? Try it in all modes! */
1094	PVMCPUCC pVCpu = VMMGetCpu(pVM);
1095
1096	/* This currently only works in real mode, protected mode without paging or with nested paging. */
1097	/** @todo NEM: MMIO page aliasing. */
1098	if ( !HMIsEnabled(pVM) /* useless without VT-x/AMD-V */
1099	\|\| ( CPUMIsGuestInPagedProtectedMode(pVCpu)
1100	&& !HMIsNestedPagingActive(pVM)))
1101	return VINF_SUCCESS; /* ignore / /* @todo return some indicator if we fail here */
1102
1103	/*
1104	* Translate the handle into an entry and check the region offset.
1105	*/
1106	AssertReturn(hRegion < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
1107	# ifdef IN_RING0
1108	AssertReturn(hRegion < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
1109	PIOMMMIOENTRYR3 const pRegEntry = &pVM->iomr0.s.paMmioRing3Regs[hRegion];
1110	AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
1111	AssertReturn(offRegion < pVM->iomr0.s.paMmioRegs[hRegion].cbRegion, VERR_OUT_OF_RANGE);
1112	AssertReturn( pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == pDevIns
1113	\|\| ( pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == NULL
1114	&& pRegEntry->pDevIns == pDevIns->pDevInsForR3), VERR_ACCESS_DENIED);
1115	# else
1116	PIOMMMIOENTRYR3 const pRegEntry = &pVM->iom.s.paMmioRegs[hRegion];
1117	AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
1118	AssertReturn(pRegEntry->pDevIns == pDevIns, VERR_ACCESS_DENIED);
1119	# endif
1120	AssertReturn(offRegion < pRegEntry->cbRegion, VERR_OUT_OF_RANGE);
1121	Assert((pRegEntry->cbRegion & GUEST_PAGE_OFFSET_MASK) == 0);
1122
1123	/*
1124	* When getting and using the mapping address, we must sit on the IOM lock
1125	* to prevent remapping. Shared suffices as we change nothing.
1126	*/
1127	int rc = IOM_LOCK_SHARED(pVM);
1128	if (rc == VINF_SUCCESS)
1129	{
1130	RTGCPHYS const GCPhys = pRegEntry->fMapped ? pRegEntry->GCPhysMapping : NIL_RTGCPHYS;
1131	if (GCPhys != NIL_RTGCPHYS)
1132	{
1133	Assert(!(GCPhys & GUEST_PAGE_OFFSET_MASK));
1134
1135	/*
1136	* Do the aliasing; page align the addresses since PGM is picky.
1137	*/
1138	rc = PGMHandlerPhysicalPageAliasMmio2(pVM, GCPhys, GCPhys + (offRegion & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK),
1139	pDevIns, hMmio2, offMmio2);
1140	}
1141	else
1142	AssertFailedStmt(rc = VERR_IOM_MMIO_REGION_NOT_MAPPED);
1143
1144	IOM_UNLOCK_SHARED(pVM);
1145	}
1146
1147	/** @todo either ditch this or replace it with something that works in the
1148	* nested case, since we really only care about nested paging! */
1149	# if 0
1150	/*
1151	* Modify the shadow page table. Since it's an MMIO page it won't be present and we
1152	* can simply prefetch it.
1153	*
1154	* Note: This is a NOP in the EPT case; we'll just let it fault again to resync the page.
1155	*/
1156	# if 0 /* The assertion is wrong for the PGM_SYNC_CLEAR_PGM_POOL and VINF_PGM_HANDLER_ALREADY_ALIASED cases. */
1157	# ifdef VBOX_STRICT
1158	uint64_t fFlags;
1159	RTHCPHYS HCPhys;
1160	rc = PGMShwGetPage(pVCpu, (RTGCPTR)GCPhys, &fFlags, &HCPhys);
1161	Assert(rc == VERR_PAGE_NOT_PRESENT \|\| rc == VERR_PAGE_TABLE_NOT_PRESENT);
1162	# endif
1163	# endif
1164	rc = PGMPrefetchPage(pVCpu, (RTGCPTR)GCPhys);
1165	Assert(rc == VINF_SUCCESS \|\| rc == VERR_PAGE_NOT_PRESENT \|\| rc == VERR_PAGE_TABLE_NOT_PRESENT);
1166	# endif
1167	return rc;
1168	#endif
1169	}
1170
1171
1172	#ifdef IN_RING0 /* VT-x ring-0 only, move to IOMR0Mmio.cpp later. */
1173	/**
1174	* Mapping a HC page in place of an MMIO page for direct access.
1175	*
1176	* This is a special optimization used by the APIC in the VT-x case. This VT-x
1177	* code uses PGMHandlerPhysicalReset rather than IOMMmioResetRegion() to undo
1178	* the effects here.
1179	*
1180	* @todo Make VT-x usage more consistent.
1181	*
1182	* @returns VBox status code.
1183	*
1184	* @param pVM The cross context VM structure.
1185	* @param pVCpu The cross context virtual CPU structure.
1186	* @param GCPhys The address of the MMIO page to be changed.
1187	* @param HCPhys The address of the host physical page.
1188	* @param fPageFlags Page flags to set. Must be (X86_PTE_RW \| X86_PTE_P)
1189	* for the time being.
1190	*/
1191	VMMR0_INT_DECL(int) IOMR0MmioMapMmioHCPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint64_t fPageFlags)
1192	{
1193	/* Currently only called from VT-x code during a page fault. */
1194	Log(("IOMR0MmioMapMmioHCPage %RGp -> %RGp flags=%RX64\n", GCPhys, HCPhys, fPageFlags));
1195
1196	AssertReturn(fPageFlags == (X86_PTE_RW \| X86_PTE_P), VERR_INVALID_PARAMETER);
1197	/** @todo NEM: MMIO page aliasing?? */
1198	Assert(HMIsEnabled(pVM));
1199
1200	# ifdef VBOX_STRICT
1201	/*
1202	* Check input address (it's HM calling, not the device, so no region handle).
1203	*/
1204	int rcSem = IOM_LOCK_SHARED(pVM);
1205	if (rcSem == VINF_SUCCESS)
1206	{
1207	RTGCPHYS offIgn;
1208	uint16_t idxIgn = UINT16_MAX;
1209	PIOMMMIOENTRYR0 pRegEntry = iomMmioGetEntry(pVM, GCPhys, &offIgn, &idxIgn);
1210	IOM_UNLOCK_SHARED(pVM);
1211	Assert(pRegEntry);
1212	Assert(pRegEntry && !(pRegEntry->cbRegion & GUEST_PAGE_OFFSET_MASK));
1213	}
1214	# endif
1215
1216	/*
1217	* Do the aliasing; page align the addresses since PGM is picky.
1218	*/
1219	GCPhys &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1220	HCPhys &= ~(RTHCPHYS)GUEST_PAGE_OFFSET_MASK;
1221
1222	int rc = PGMHandlerPhysicalPageAliasHC(pVM, GCPhys, GCPhys, HCPhys);
1223	AssertRCReturn(rc, rc);
1224
1225	/** @todo either ditch this or replace it with something that works in the
1226	* nested case, since we really only care about nested paging! */
1227
1228	/*
1229	* Modify the shadow page table. Since it's an MMIO page it won't be present and we
1230	* can simply prefetch it.
1231	*
1232	* Note: This is a NOP in the EPT case; we'll just let it fault again to resync the page.
1233	*/
1234	rc = PGMPrefetchPage(pVCpu, (RTGCPTR)GCPhys);
1235	Assert(rc == VINF_SUCCESS \|\| rc == VERR_PAGE_NOT_PRESENT \|\| rc == VERR_PAGE_TABLE_NOT_PRESENT);
1236	return VINF_SUCCESS;
1237	}
1238	#endif
1239
1240
1241	/**
1242	* Reset a previously modified MMIO region; restore the access flags.
1243	*
1244	* This undoes the effects of IOMMmioMapMmio2Page() and is currently only
1245	* intended for some ancient VGA hack. However, it would be great to extend it
1246	* beyond VT-x and/or nested-paging.
1247	*
1248	* @returns VBox status code.
1249	*
1250	* @param pVM The cross context VM structure.
1251	* @param pDevIns The device instance @a hRegion is associated with.
1252	* @param hRegion The handle to the MMIO region.
1253	*/
1254	VMMDECL(int) IOMMmioResetRegion(PVMCC pVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion)
1255	{
1256	Log(("IOMMMIOResetRegion %#RX64\n", hRegion));
1257	AssertReturn(pDevIns, VERR_INVALID_POINTER);
1258
1259	#if defined(VBOX_VMM_TARGET_ARMV8)
1260	/** @todo NEM: MMIO page aliasing. */
1261	RT_NOREF(pVM, hRegion);
1262	return VINF_SUCCESS; /* ignore / /* @todo return some indicator if we fail here */
1263	#else
1264	/** @todo Get rid of this this real/protected or nested paging restriction,
1265	* it probably shouldn't be here and would be nasty when the CPU
1266	* changes mode while we have the hack enabled... */
1267	PVMCPUCC pVCpu = VMMGetCpu(pVM);
1268
1269	/* This currently only works in real mode, protected mode without paging or with nested paging. */
1270	/** @todo NEM: MMIO page aliasing. */
1271	if ( !HMIsEnabled(pVM) /* useless without VT-x/AMD-V */
1272	\|\| ( CPUMIsGuestInPagedProtectedMode(pVCpu)
1273	&& !HMIsNestedPagingActive(pVM)))
1274	return VINF_SUCCESS; /* ignore */
1275
1276	/*
1277	* Translate the handle into an entry and mapping address for PGM.
1278	* We have to take the lock to safely access the mapping address here.
1279	*/
1280	AssertReturn(hRegion < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
1281	# ifdef IN_RING0
1282	AssertReturn(hRegion < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
1283	PIOMMMIOENTRYR3 const pRegEntry = &pVM->iomr0.s.paMmioRing3Regs[hRegion];
1284	AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
1285	AssertReturn( pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == pDevIns
1286	\|\| ( pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == NULL
1287	&& pRegEntry->pDevIns == pDevIns->pDevInsForR3), VERR_ACCESS_DENIED);
1288	# else
1289	PIOMMMIOENTRYR3 const pRegEntry = &pVM->iom.s.paMmioRegs[hRegion];
1290	AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
1291	AssertReturn(pRegEntry->pDevIns == pDevIns, VERR_ACCESS_DENIED);
1292	# endif
1293	Assert((pRegEntry->cbRegion & GUEST_PAGE_OFFSET_MASK) == 0);
1294
1295	int rcSem = IOM_LOCK_SHARED(pVM);
1296	RTGCPHYS GCPhys = pRegEntry->fMapped ? pRegEntry->GCPhysMapping : NIL_RTGCPHYS;
1297	if (rcSem == VINF_SUCCESS)
1298	IOM_UNLOCK_SHARED(pVM);
1299
1300	Assert(!(GCPhys & GUEST_PAGE_OFFSET_MASK));
1301	Assert(!(pRegEntry->cbRegion & GUEST_PAGE_OFFSET_MASK));
1302
1303	/*
1304	* Call PGM to do the job work.
1305	*
1306	* After the call, all the pages should be non-present, unless there is
1307	* a page pool flush pending (unlikely).
1308	*/
1309	int rc = PGMHandlerPhysicalReset(pVM, GCPhys);
1310	AssertRC(rc);
1311
1312	# ifdef VBOX_STRICT
1313	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
1314	{
1315	RTGCPHYS cb = pRegEntry->cbRegion;
1316	while (cb)
1317	{
1318	uint64_t fFlags;
1319	RTHCPHYS HCPhys;
1320	rc = PGMShwGetPage(pVCpu, (RTGCPTR)GCPhys, &fFlags, &HCPhys);
1321	Assert(rc == VERR_PAGE_NOT_PRESENT \|\| rc == VERR_PAGE_TABLE_NOT_PRESENT);
1322	cb -= RT_MIN(GUEST_PAGE_SIZE, HOST_PAGE_SIZE);
1323	GCPhys += RT_MIN(GUEST_PAGE_SIZE, HOST_PAGE_SIZE);
1324	}
1325	}
1326	# endif
1327	return rc;
1328	#endif
1329	}
1330

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/IOMAllMmioNew.cpp@ 105834

Download in other formats: