VirtioCore.cpp@ 100400

Last change on this file since 100400 was 100400, checked in by vboxsync, 19 months ago
Devices/VirtIO: Add support for the VirtIO over MMIO transport mode useful for ARM, bugref:10459
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 131.8 KB

Line
1	/* $Id: VirtioCore.cpp 100400 2023-07-06 08:58:02Z vboxsync $ */
2
3	/** @file
4	* VirtioCore - Virtio Core (PCI, feature & config mgt, queue mgt & proxy, notification mgt)
5	*/
6
7	/*
8	* Copyright (C) 2009-2023 Oracle and/or its affiliates.
9	*
10	* This file is part of VirtualBox base platform packages, as
11	* available from https://www.virtualbox.org.
12	*
13	* This program is free software; you can redistribute it and/or
14	* modify it under the terms of the GNU General Public License
15	* as published by the Free Software Foundation, in version 3 of the
16	* License.
17	*
18	* This program is distributed in the hope that it will be useful, but
19	* WITHOUT ANY WARRANTY; without even the implied warranty of
20	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
21	* General Public License for more details.
22	*
23	* You should have received a copy of the GNU General Public License
24	* along with this program; if not, see <https://www.gnu.org/licenses>.
25	*
26	* SPDX-License-Identifier: GPL-3.0-only
27	*/
28
29
30	/*********************************************************************************************************************************
31	* Header Files *
32	*********************************************************************************************************************************/
33	#define LOG_GROUP LOG_GROUP_DEV_VIRTIO
34
35	#include <iprt/assert.h>
36	#include <iprt/uuid.h>
37	#include <iprt/mem.h>
38	#include <iprt/sg.h>
39	#include <iprt/assert.h>
40	#include <iprt/string.h>
41	#include <iprt/param.h>
42	#include <iprt/types.h>
43	#include <VBox/log.h>
44	#include <VBox/msi.h>
45	#include <iprt/types.h>
46	#include <VBox/AssertGuest.h>
47	#include <VBox/vmm/pdmdev.h>
48	#include "VirtioCore.h"
49
50
51	/*********************************************************************************************************************************
52	* Defined Constants And Macros *
53	*********************************************************************************************************************************/
54
55	#define INSTANCE(a_pVirtio) ((a_pVirtio)->szInstance)
56	#define VIRTQNAME(a_pVirtio, a_uVirtq) ((a_pVirtio)->aVirtqueues[(a_uVirtq)].szName)
57
58	#define IS_VIRTQ_EMPTY(pDevIns, pVirtio, pVirtq) \
59	(virtioCoreVirtqAvailCnt(pDevIns, pVirtio, pVirtq) == 0)
60
61	#define IS_DRIVER_OK(a_pVirtio) ((a_pVirtio)->fDeviceStatus & VIRTIO_STATUS_DRIVER_OK)
62	#define WAS_DRIVER_OK(a_pVirtio) ((a_pVirtio)->fPrevDeviceStatus & VIRTIO_STATUS_DRIVER_OK)
63
64	/**
65	* These defines are used to track guest virtio-net driver writing driver features accepted flags
66	* in two 32-bit operations (in arbitrary order), and one bit dedicated to ensured 'features complete'
67	* is handled once.
68	*/
69	#define DRIVER_FEATURES_0_WRITTEN 1 /*< fDriverFeatures[0] written by guest virtio-net /
70	#define DRIVER_FEATURES_1_WRITTEN 2 /*< fDriverFeatures[1] written by guest virtio-net /
71	#define DRIVER_FEATURES_0_AND_1_WRITTEN 3 /*< Both 32-bit parts of fDriverFeatures[] written /
72	#define DRIVER_FEATURES_COMPLETE_HANDLED 4 /*< Features negotiation complete handler called /
73
74	/**
75	* This macro returns true if the @a a_offAccess and access length (@a
76	* a_cbAccess) are within the range of the mapped capability struct described by
77	* @a a_LocCapData.
78	*
79	* @param[in] a_offAccess Input: The offset into the MMIO bar of the access.
80	* @param[in] a_cbAccess Input: The access size.
81	* @param[out] a_offsetIntoCap Output: uint32_t variable to return the intra-capability offset into.
82	* @param[in] a_LocCapData Input: The capability location info.
83	*/
84	#define MATCHES_VIRTIO_CAP_STRUCT(a_offAccess, a_cbAccess, a_offsetIntoCap, a_LocCapData) \
85	( ((a_offsetIntoCap) = (uint32_t)((a_offAccess) - (a_LocCapData).offMmio)) < (uint32_t)(a_LocCapData).cbMmio \
86	&& (a_offsetIntoCap) + (uint32_t)(a_cbAccess) <= (uint32_t)(a_LocCapData).cbMmio )
87
88
89	/*********************************************************************************************************************************
90	* Structures and Typedefs *
91	*********************************************************************************************************************************/
92
93	/** @name virtq related flags
94	* @{ */
95	#define VIRTQ_DESC_F_NEXT 1 /*< Indicates this descriptor chains to next /
96	#define VIRTQ_DESC_F_WRITE 2 /*< Marks buffer as write-only (default ro) /
97	#define VIRTQ_DESC_F_INDIRECT 4 /*< Buffer is list of buffer descriptors /
98
99	#define VIRTQ_USED_F_NO_NOTIFY 1 /*< Dev to Drv: Don't notify when buf added /
100	#define VIRTQ_AVAIL_F_NO_INTERRUPT 1 /*< Drv to Dev: Don't notify when buf eaten /
101	/** @} */
102
103	/**
104	* virtq-related structs
105	* (struct names follow VirtIO 1.0 spec, field names use VBox styled naming, w/respective spec'd name in comments)
106	*/
107	typedef struct virtq_desc
108	{
109	uint64_t GCPhysBuf; /*< addr GC Phys. address of buffer /
110	uint32_t cb; /*< len Buffer length /
111	uint16_t fFlags; /*< flags Buffer specific flags /
112	uint16_t uDescIdxNext; /*< next Idx set if VIRTIO_DESC_F_NEXT /
113	} VIRTQ_DESC_T, *PVIRTQ_DESC_T;
114
115	typedef struct virtq_avail
116	{
117	uint16_t fFlags; /*< flags avail ring guest-to-host flags /
118	uint16_t uIdx; /*< idx Index of next free ring slot /
119	RT_FLEXIBLE_ARRAY_EXTENSION
120	uint16_t auRing[RT_FLEXIBLE_ARRAY]; /*< ring Ring: avail drv to dev bufs /
121	//uint16_t uUsedEventIdx; /*< used_event (if VIRTQ_USED_F_EVENT_IDX) /
122	} VIRTQ_AVAIL_T, *PVIRTQ_AVAIL_T;
123
124	typedef struct virtq_used_elem
125	{
126	uint32_t uDescIdx; /*< idx Start of used desc chain /
127	uint32_t cbElem; /*< len Total len of used desc chain /
128	} VIRTQ_USED_ELEM_T;
129
130	typedef struct virt_used
131	{
132	uint16_t fFlags; /*< flags used ring host-to-guest flags /
133	uint16_t uIdx; /*< idx Index of next ring slot /
134	RT_FLEXIBLE_ARRAY_EXTENSION
135	VIRTQ_USED_ELEM_T aRing[RT_FLEXIBLE_ARRAY]; /*< ring Ring: used dev to drv bufs /
136	//uint16_t uAvailEventIdx; /*< avail_event if (VIRTQ_USED_F_EVENT_IDX) /
137	} VIRTQ_USED_T, *PVIRTQ_USED_T;
138
139	DECLHIDDEN(const char *) virtioCoreGetStateChangeText(VIRTIOVMSTATECHANGED enmState)
140	{
141	switch (enmState)
142	{
143	case kvirtIoVmStateChangedReset: return "VM RESET";
144	case kvirtIoVmStateChangedSuspend: return "VM SUSPEND";
145	case kvirtIoVmStateChangedPowerOff: return "VM POWER OFF";
146	case kvirtIoVmStateChangedResume: return "VM RESUME";
147	default: return "<BAD ENUM>";
148	}
149	}
150
151	/* Internal Functions */
152
153	static void virtioCoreNotifyGuestDriver(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq);
154	static int virtioNudgeGuest(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint8_t uCause, uint16_t uVec);
155
156	#ifdef IN_RING3
157	# ifdef LOG_ENABLED
158	DECLINLINE(uint16_t) virtioCoreR3CountPendingBufs(uint16_t uRingIdx, uint16_t uShadowIdx, uint16_t uQueueSize)
159	{
160	if (uShadowIdx == uRingIdx)
161	return 0;
162	else
163	if (uShadowIdx > uRingIdx)
164	return uShadowIdx - uRingIdx;
165	return uQueueSize - (uRingIdx - uShadowIdx);
166	}
167	# endif
168	#endif
169	/** @name Internal queue operations
170	* @{ */
171
172	/**
173	* Accessor for virtq descriptor
174	*/
175	#ifdef IN_RING3
176	DECLINLINE(void) virtioReadDesc(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq,
177	uint32_t idxDesc, PVIRTQ_DESC_T pDesc)
178	{
179	/*
180	* Shut up assertion for legacy virtio-net driver in FreeBSD up to 12.3 (see virtioCoreR3VirtqUsedBufPut()
181	* for more information).
182	*/
183	AssertMsg( IS_DRIVER_OK(pVirtio)
184	\|\| ( pVirtio->fLegacyDriver
185	&& pVirtq->GCPhysVirtqDesc),
186	("Called with guest driver not ready\n"));
187	uint16_t const cVirtqItems = RT_MAX(pVirtq->uQueueSize, 1); /* Make sure to avoid div-by-zero. */
188
189	virtioCoreGCPhysRead(pVirtio, pDevIns,
190	pVirtq->GCPhysVirtqDesc + sizeof(VIRTQ_DESC_T) * (idxDesc % cVirtqItems),
191	pDesc, sizeof(VIRTQ_DESC_T));
192	}
193	#endif
194
195	/**
196	* Accessors for virtq avail ring
197	*/
198	#ifdef IN_RING3
199	DECLINLINE(uint16_t) virtioReadAvailDescIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq, uint32_t availIdx)
200	{
201	uint16_t uDescIdx;
202
203	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
204	uint16_t const cVirtqItems = RT_MAX(pVirtq->uQueueSize, 1); /* Make sure to avoid div-by-zero. */
205	virtioCoreGCPhysRead(pVirtio, pDevIns,
206	pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[availIdx % cVirtqItems]),
207	&uDescIdx, sizeof(uDescIdx));
208	return uDescIdx;
209	}
210
211	DECLINLINE(uint16_t) virtioReadAvailUsedEvent(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
212	{
213	uint16_t uUsedEventIdx;
214	/* VirtIO 1.0 uUsedEventIdx (used_event) immediately follows ring */
215	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
216	virtioCoreGCPhysRead(pVirtio, pDevIns,
217	pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[pVirtq->uQueueSize]),
218	&uUsedEventIdx, sizeof(uUsedEventIdx));
219	return uUsedEventIdx;
220	}
221	#endif
222
223	DECLINLINE(uint16_t) virtioReadAvailRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
224	{
225	uint16_t uIdx = 0;
226	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
227	virtioCoreGCPhysRead(pVirtio, pDevIns,
228	pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF(VIRTQ_AVAIL_T, uIdx),
229	&uIdx, sizeof(uIdx));
230	return uIdx;
231	}
232
233	DECLINLINE(uint16_t) virtioReadAvailRingFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
234	{
235	uint16_t fFlags = 0;
236	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
237	virtioCoreGCPhysRead(pVirtio, pDevIns,
238	pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF(VIRTQ_AVAIL_T, fFlags),
239	&fFlags, sizeof(fFlags));
240	return fFlags;
241	}
242
243	/** @} */
244
245	/** @name Accessors for virtq used ring
246	* @{
247	*/
248
249	#ifdef IN_RING3
250	DECLINLINE(void) virtioWriteUsedElem(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq,
251	uint32_t usedIdx, uint32_t uDescIdx, uint32_t uLen)
252	{
253	VIRTQ_USED_ELEM_T elem = { uDescIdx, uLen };
254	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
255	uint16_t const cVirtqItems = RT_MAX(pVirtq->uQueueSize, 1); /* Make sure to avoid div-by-zero. */
256	virtioCoreGCPhysWrite(pVirtio, pDevIns,
257	pVirtq->GCPhysVirtqUsed
258	+ RT_UOFFSETOF_DYN(VIRTQ_USED_T, aRing[usedIdx % cVirtqItems]),
259	&elem, sizeof(elem));
260	}
261
262	DECLINLINE(void) virtioWriteUsedRingFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq, uint16_t fFlags)
263	{
264	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
265	RT_UNTRUSTED_VALIDATED_FENCE(); /* VirtIO 1.0, Section 3.2.1.4.1 */
266	virtioCoreGCPhysWrite(pVirtio, pDevIns,
267	pVirtq->GCPhysVirtqUsed + RT_UOFFSETOF(VIRTQ_USED_T, fFlags),
268	&fFlags, sizeof(fFlags));
269	}
270	#endif
271
272	DECLINLINE(void) virtioWriteUsedRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq, uint16_t uIdx)
273	{
274	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
275	RT_UNTRUSTED_VALIDATED_FENCE(); /* VirtIO 1.0, Section 3.2.1.4.1 */
276	virtioCoreGCPhysWrite(pVirtio, pDevIns,
277	pVirtq->GCPhysVirtqUsed + RT_UOFFSETOF(VIRTQ_USED_T, uIdx),
278	&uIdx, sizeof(uIdx));
279	}
280
281	#ifdef IN_RING3
282	DECLINLINE(uint16_t) virtioReadUsedRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
283	{
284	uint16_t uIdx = 0;
285	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
286	virtioCoreGCPhysRead(pVirtio, pDevIns,
287	pVirtq->GCPhysVirtqUsed + RT_UOFFSETOF(VIRTQ_USED_T, uIdx),
288	&uIdx, sizeof(uIdx));
289	return uIdx;
290	}
291
292	DECLINLINE(uint16_t) virtioReadUsedRingFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
293	{
294	uint16_t fFlags = 0;
295	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
296	virtioCoreGCPhysRead(pVirtio, pDevIns,
297	pVirtq->GCPhysVirtqUsed + RT_UOFFSETOF(VIRTQ_USED_T, fFlags),
298	&fFlags, sizeof(fFlags));
299	return fFlags;
300	}
301
302	DECLINLINE(void) virtioWriteUsedAvailEvent(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq, uint32_t uAvailEventIdx)
303	{
304	/** VirtIO 1.0 uAvailEventIdx (avail_event) immediately follows ring */
305	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
306	virtioCoreGCPhysWrite(pVirtio, pDevIns,
307	pVirtq->GCPhysVirtqUsed
308	+ RT_UOFFSETOF_DYN(VIRTQ_USED_T, aRing[pVirtq->uQueueSize]),
309	&uAvailEventIdx, sizeof(uAvailEventIdx));
310	}
311	#endif
312	/** @} */
313
314
315	DECLINLINE(uint16_t) virtioCoreVirtqAvailCnt(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
316	{
317	uint16_t uIdxActual = virtioReadAvailRingIdx(pDevIns, pVirtio, pVirtq);
318	uint16_t uIdxShadow = pVirtq->uAvailIdxShadow;
319	uint16_t uIdxDelta;
320
321	if (uIdxActual < uIdxShadow)
322	uIdxDelta = (uIdxActual + pVirtq->uQueueSize) - uIdxShadow;
323	else
324	uIdxDelta = uIdxActual - uIdxShadow;
325
326	return uIdxDelta;
327	}
328	/**
329	* Get count of new (e.g. pending) elements in available ring.
330	*
331	* @param pDevIns The device instance.
332	* @param pVirtio Pointer to the shared virtio state.
333	* @param uVirtq Virtq number
334	*
335	* @returns how many entries have been added to ring as a delta of the consumer's
336	* avail index and the queue's guest-side current avail index.
337	*/
338	DECLHIDDEN(uint16_t) virtioCoreVirtqAvailBufCount(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq)
339	{
340	AssertMsgReturn(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues), ("uVirtq out of range"), 0);
341	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
342
343	if (!IS_DRIVER_OK(pVirtio))
344	{
345	LogRelFunc(("Driver not ready\n"));
346	return 0;
347	}
348	if (!pVirtio->fLegacyDriver && !pVirtq->uEnable)
349	{
350	LogRelFunc(("virtq: %s not enabled\n", VIRTQNAME(pVirtio, uVirtq)));
351	return 0;
352	}
353	return virtioCoreVirtqAvailCnt(pDevIns, pVirtio, pVirtq);
354	}
355
356	#ifdef IN_RING3
357
358	static void virtioCoreR3FeatureDump(VIRTIOCORE pVirtio, PCDBGFINFOHLP pHlp, const VIRTIO_FEATURES_LIST s_aFeatures, int cFeatures, int fBanner)
359	{
360	#define MAXLINE 80
361	/* Display as a single buf to prevent interceding log messages */
362	uint16_t cbBuf = cFeatures * 132;
363	char pszBuf = (char )RTMemAllocZ(cbBuf);
364	Assert(pszBuf);
365	char *cp = pszBuf;
366	for (int i = 0; i < cFeatures; ++i)
367	{
368	bool isOffered = RT_BOOL(pVirtio->uDeviceFeatures & s_aFeatures[i].fFeatureBit);
369	bool isNegotiated = RT_BOOL(pVirtio->uDriverFeatures & s_aFeatures[i].fFeatureBit);
370	cp += RTStrPrintf(cp, cbBuf - (cp - pszBuf), " %s %s %s",
371	isOffered ? "+" : "-", isNegotiated ? "x" : " ", s_aFeatures[i].pcszDesc);
372	}
373	if (pHlp) {
374	if (fBanner)
375	pHlp->pfnPrintf(pHlp, "VirtIO Features Configuration\n\n"
376	" Offered Accepted Feature Description\n"
377	" ------- -------- ------- -----------\n");
378	pHlp->pfnPrintf(pHlp, "%s\n", pszBuf);
379	}
380	#ifdef LOG_ENABLED
381	else
382	{
383	if (fBanner)
384	Log(("VirtIO Features Configuration\n\n"
385	" Offered Accepted Feature Description\n"
386	" ------- -------- ------- -----------\n"));
387	Log(("%s\n", pszBuf));
388	}
389	#endif
390	RTMemFree(pszBuf);
391	}
392
393	/** API Function: See header file*/
394	DECLHIDDEN(void) virtioCorePrintDeviceFeatures(VIRTIOCORE *pVirtio, PCDBGFINFOHLP pHlp,
395	const VIRTIO_FEATURES_LIST *s_aDevSpecificFeatures, int cFeatures) {
396	virtioCoreR3FeatureDump(pVirtio, pHlp, s_aCoreFeatures, RT_ELEMENTS(s_aCoreFeatures), 1 /fBanner /);
397	virtioCoreR3FeatureDump(pVirtio, pHlp, s_aDevSpecificFeatures, cFeatures, 0 /fBanner /);
398	}
399
400	#endif
401
402	#ifdef LOG_ENABLED
403
404	/** API Function: See header file */
405	DECLHIDDEN(void) virtioCoreHexDump(uint8_t pv, uint32_t cb, uint32_t uBase, const char pszTitle)
406	{
407	#define ADJCURSOR(cb) pszOut += cb; cbRemain -= cb;
408	size_t cbPrint = 0, cbRemain = ((cb / 16) + 1) * 80;
409	char pszBuf = (char )RTMemAllocZ(cbRemain), *pszOut = pszBuf;
410	AssertMsgReturnVoid(pszBuf, ("Out of Memory"));
411	if (pszTitle)
412	{
413	cbPrint = RTStrPrintf(pszOut, cbRemain, "%s [%d bytes]:\n", pszTitle, cb);
414	ADJCURSOR(cbPrint);
415	}
416	for (uint32_t row = 0; row < RT_MAX(1, (cb / 16) + 1) && row * 16 < cb; row++)
417	{
418	cbPrint = RTStrPrintf(pszOut, cbRemain, "%04x: ", row * 16 + uBase); /* line address */
419	ADJCURSOR(cbPrint);
420	for (uint8_t col = 0; col < 16; col++)
421	{
422	uint32_t idx = row * 16 + col;
423	if (idx >= cb)
424	cbPrint = RTStrPrintf(pszOut, cbRemain, "-- %s", (col + 1) % 8 ? "" : " ");
425	else
426	cbPrint = RTStrPrintf(pszOut, cbRemain, "%02x %s", pv[idx], (col + 1) % 8 ? "" : " ");
427	ADJCURSOR(cbPrint);
428	}
429	for (uint32_t idx = row * 16; idx < row * 16 + 16; idx++)
430	{
431	cbPrint = RTStrPrintf(pszOut, cbRemain, "%c", (idx >= cb) ? ' ' : (pv[idx] >= 0x20 && pv[idx] <= 0x7e ? pv[idx] : '.'));
432	ADJCURSOR(cbPrint);
433	}
434	*pszOut++ = '\n';
435	--cbRemain;
436	}
437	Log(("%s\n", pszBuf));
438	RTMemFree(pszBuf);
439	RT_NOREF2(uBase, pv);
440	#undef ADJCURSOR
441	}
442
443	/* API FUnction: See header file */
444	DECLHIDDEN(void) virtioCoreGCPhysHexDump(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, uint16_t cb, uint32_t uBase, const char *pszTitle)
445	{
446	PVIRTIOCORE pVirtio = PDMDEVINS_2_DATA(pDevIns, PVIRTIOCORE);
447	#define ADJCURSOR(cb) pszOut += cb; cbRemain -= cb;
448	size_t cbPrint = 0, cbRemain = ((cb / 16) + 1) * 80;
449	char pszBuf = (char )RTMemAllocZ(cbRemain), *pszOut = pszBuf;
450	AssertMsgReturnVoid(pszBuf, ("Out of Memory"));
451	if (pszTitle)
452	{
453	cbPrint = RTStrPrintf(pszOut, cbRemain, "%s [%d bytes]:\n", pszTitle, cb);
454	ADJCURSOR(cbPrint);
455	}
456	for (uint16_t row = 0; row < (uint16_t)RT_MAX(1, (cb / 16) + 1) && row * 16 < cb; row++)
457	{
458	uint8_t c;
459	cbPrint = RTStrPrintf(pszOut, cbRemain, "%04x: ", row * 16 + uBase); /* line address */
460	ADJCURSOR(cbPrint);
461	for (uint8_t col = 0; col < 16; col++)
462	{
463	uint32_t idx = row * 16 + col;
464	virtioCoreGCPhysRead(pVirtio, pDevIns, GCPhys + idx, &c, 1);
465	if (idx >= cb)
466	cbPrint = RTStrPrintf(pszOut, cbRemain, "-- %s", (col + 1) % 8 ? "" : " ");
467	else
468	cbPrint = RTStrPrintf(pszOut, cbRemain, "%02x %s", c, (col + 1) % 8 ? "" : " ");
469	ADJCURSOR(cbPrint);
470	}
471	for (uint16_t idx = row * 16; idx < row * 16 + 16; idx++)
472	{
473	virtioCoreGCPhysRead(pVirtio, pDevIns, GCPhys + idx, &c, 1);
474	cbPrint = RTStrPrintf(pszOut, cbRemain, "%c", (idx >= cb) ? ' ' : (c >= 0x20 && c <= 0x7e ? c : '.'));
475	ADJCURSOR(cbPrint);
476	}
477	*pszOut++ = '\n';
478	--cbRemain;
479	}
480	Log(("%s\n", pszBuf));
481	RTMemFree(pszBuf);
482	RT_NOREF(uBase);
483	#undef ADJCURSOR
484	}
485
486
487	/** API function: See header file */
488	DECLHIDDEN(void) virtioCoreLogMappedIoValue(const char pszFunc, const char pszMember, uint32_t uMemberSize,
489	const void *pv, uint32_t cb, uint32_t uOffset, int fWrite,
490	int fHasIndex, uint32_t idx)
491	{
492	if (LogIs6Enabled())
493	{
494	char szIdx[16];
495	if (fHasIndex)
496	RTStrPrintf(szIdx, sizeof(szIdx), "[%d]", idx);
497	else
498	szIdx[0] = '\0';
499
500	if (cb == 1 \|\| cb == 2 \|\| cb == 4 \|\| cb == 8)
501	{
502	char szDepiction[64];
503	size_t cchDepiction;
504	if (uOffset != 0 \|\| cb != uMemberSize) /* display bounds if partial member access */
505	cchDepiction = RTStrPrintf(szDepiction, sizeof(szDepiction), "%s%s[%d:%d]",
506	pszMember, szIdx, uOffset, uOffset + cb - 1);
507	else
508	cchDepiction = RTStrPrintf(szDepiction, sizeof(szDepiction), "%s%s", pszMember, szIdx);
509
510	/* padding */
511	if (cchDepiction < 30)
512	szDepiction[cchDepiction++] = ' ';
513	while (cchDepiction < 30)
514	szDepiction[cchDepiction++] = '.';
515	szDepiction[cchDepiction] = '\0';
516
517	RTUINT64U uValue;
518	uValue.u = 0;
519	memcpy(uValue.au8, pv, cb);
520	Log6(("%-23s: Guest %s %s %#0*RX64\n",
521	pszFunc, fWrite ? "wrote" : "read ", szDepiction, 2 + cb * 2, uValue.u));
522	}
523	else /* odd number or oversized access, ... log inline hex-dump style */
524	{
525	Log6(("%-23s: Guest %s %s%s[%d:%d]: %.*Rhxs\n",
526	pszFunc, fWrite ? "wrote" : "read ", pszMember,
527	szIdx, uOffset, uOffset + cb, cb, pv));
528	}
529	}
530	RT_NOREF2(fWrite, pszFunc);
531	}
532
533	/**
534	* Log MMIO-mapped Virtio fDeviceStatus register bitmask, naming the bits
535	*/
536	DECLINLINE(void) virtioCoreFormatDeviceStatus(uint8_t bStatus, char *pszBuf, size_t uSize)
537	{
538	# define ADJCURSOR(len) { cp += len; uSize -= len; sep = (char *)" \| "; }
539	memset(pszBuf, 0, uSize);
540	char cp = pszBuf, sep = (char *)"";
541	size_t len;
542	if (bStatus == 0)
543	RTStrPrintf(cp, uSize, "RESET");
544	else
545	{
546	if (bStatus & VIRTIO_STATUS_ACKNOWLEDGE)
547	{
548	len = RTStrPrintf(cp, uSize, "ACKNOWLEDGE");
549	ADJCURSOR(len);
550	}
551	if (bStatus & VIRTIO_STATUS_DRIVER)
552	{
553	len = RTStrPrintf(cp, uSize, "%sDRIVER", sep);
554	ADJCURSOR(len);
555	}
556	if (bStatus & VIRTIO_STATUS_FEATURES_OK)
557	{
558	len = RTStrPrintf(cp, uSize, "%sFEATURES_OK", sep);
559	ADJCURSOR(len);
560	}
561	if (bStatus & VIRTIO_STATUS_DRIVER_OK)
562	{
563	len = RTStrPrintf(cp, uSize, "%sDRIVER_OK", sep);
564	ADJCURSOR(len);
565	}
566	if (bStatus & VIRTIO_STATUS_FAILED)
567	{
568	len = RTStrPrintf(cp, uSize, "%sFAILED", sep);
569	ADJCURSOR(len);
570	}
571	if (bStatus & VIRTIO_STATUS_DEVICE_NEEDS_RESET)
572	RTStrPrintf(cp, uSize, "%sNEEDS_RESET", sep);
573	}
574	# undef ADJCURSOR
575	}
576
577	#endif /* LOG_ENABLED */
578
579	/** API function: See header file */
580	DECLHIDDEN(int) virtioCoreIsLegacyMode(PVIRTIOCORE pVirtio)
581	{
582	return pVirtio->fLegacyDriver;
583	}
584
585	#ifdef IN_RING3
586
587	DECLHIDDEN(int) virtioCoreR3VirtqAttach(PVIRTIOCORE pVirtio, uint16_t uVirtq, const char *pcszName)
588	{
589	LogFunc(("Attaching %s to VirtIO core\n", pcszName));
590	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
591	pVirtq->uVirtq = uVirtq;
592	pVirtq->fUsedRingEvent = false;
593	pVirtq->fAttached = true;
594	RTStrCopy(pVirtq->szName, sizeof(pVirtq->szName), pcszName);
595	return VINF_SUCCESS;
596	}
597
598	DECLHIDDEN(int) virtioCoreR3VirtqDetach(PVIRTIOCORE pVirtio, uint16_t uVirtqNbr)
599	{
600	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtqNbr];
601	pVirtq->uVirtq = 0;
602	pVirtq->uAvailIdxShadow = 0;
603	pVirtq->uUsedIdxShadow = 0;
604	pVirtq->fUsedRingEvent = false;
605	pVirtq->fAttached = false;
606	memset(pVirtq->szName, 0, sizeof(pVirtq->szName));
607	return VINF_SUCCESS;
608	}
609
610	DECLHIDDEN(bool) virtioCoreR3VirtqIsAttached(PVIRTIOCORE pVirtio, uint16_t uVirtqNbr)
611	{
612	return pVirtio->aVirtqueues[uVirtqNbr].fAttached;
613	}
614
615	DECLHIDDEN(bool) virtioCoreR3VirtqIsEnabled(PVIRTIOCORE pVirtio, uint16_t uVirtqNbr)
616	{
617	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtqNbr];
618	return (bool)pVirtq->uEnable && pVirtq->GCPhysVirtqDesc;
619	}
620
621	/** API Fuunction: See header file */
622	DECLHIDDEN(void) virtioCoreR3VirtqInfo(PPDMDEVINS pDevIns, PCDBGFINFOHLP pHlp, const char *pszArgs, int uVirtq)
623	{
624	RT_NOREF(pszArgs);
625	PVIRTIOCORE pVirtio = PDMDEVINS_2_DATA(pDevIns, PVIRTIOCORE);
626	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
627
628	/** @todo add ability to dump physical contents described by any descriptor (using existing VirtIO core API function) */
629	// bool fDump = pszArgs && (pszArgs == 'd' \|\| pszArgs == 'D'); /* "dump" (avail phys descriptor)"
630
631	uint16_t uAvailIdx = virtioReadAvailRingIdx(pDevIns, pVirtio, pVirtq);
632	uint16_t uAvailIdxShadow = pVirtq->uAvailIdxShadow;
633
634	uint16_t uUsedIdx = virtioReadUsedRingIdx(pDevIns, pVirtio, pVirtq);
635	uint16_t uUsedIdxShadow = pVirtq->uUsedIdxShadow;
636
637	VIRTQBUF_T VirtqBuf;
638	PVIRTQBUF pVirtqBuf = &VirtqBuf;
639	bool fEmpty = IS_VIRTQ_EMPTY(pDevIns, pVirtio, pVirtq);
640
641	LogFunc(("%s, empty = %s\n", pVirtq->szName, fEmpty ? "true" : "false"));
642
643	int cSendSegs = 0, cReturnSegs = 0;
644	if (!fEmpty)
645	{
646	virtioCoreR3VirtqAvailBufPeek(pDevIns, pVirtio, uVirtq, pVirtqBuf);
647	cSendSegs = pVirtqBuf->pSgPhysSend ? pVirtqBuf->pSgPhysSend->cSegs : 0;
648	cReturnSegs = pVirtqBuf->pSgPhysReturn ? pVirtqBuf->pSgPhysReturn->cSegs : 0;
649	}
650
651	bool fAvailNoInterrupt = virtioReadAvailRingFlags(pDevIns, pVirtio, pVirtq) & VIRTQ_AVAIL_F_NO_INTERRUPT;
652	bool fUsedNoNotify = virtioReadUsedRingFlags(pDevIns, pVirtio, pVirtq) & VIRTQ_USED_F_NO_NOTIFY;
653
654	pHlp->pfnPrintf(pHlp, " queue enabled: ........... %s\n", pVirtq->uEnable ? "true" : "false");
655	pHlp->pfnPrintf(pHlp, " size: .................... %d\n", pVirtq->uQueueSize);
656	pHlp->pfnPrintf(pHlp, " notify offset: ........... %d\n", pVirtq->uNotifyOffset);
657	if (pVirtio->fMsiSupport)
658	pHlp->pfnPrintf(pHlp, " MSIX vector: ....... %4.4x\n", pVirtq->uMsixVector);
659	pHlp->pfnPrintf(pHlp, "\n");
660	pHlp->pfnPrintf(pHlp, " avail ring (%d entries):\n", uAvailIdx - uAvailIdxShadow);
661	pHlp->pfnPrintf(pHlp, " index: ................ %d\n", uAvailIdx);
662	pHlp->pfnPrintf(pHlp, " shadow: ............... %d\n", uAvailIdxShadow);
663	pHlp->pfnPrintf(pHlp, " flags: ................ %s\n", fAvailNoInterrupt ? "NO_INTERRUPT" : "");
664	pHlp->pfnPrintf(pHlp, "\n");
665	pHlp->pfnPrintf(pHlp, " used ring (%d entries):\n", uUsedIdx - uUsedIdxShadow);
666	pHlp->pfnPrintf(pHlp, " index: ................ %d\n", uUsedIdx);
667	pHlp->pfnPrintf(pHlp, " shadow: ............... %d\n", uUsedIdxShadow);
668	pHlp->pfnPrintf(pHlp, " flags: ................ %s\n", fUsedNoNotify ? "NO_NOTIFY" : "");
669	pHlp->pfnPrintf(pHlp, "\n");
670	if (!fEmpty)
671	{
672	pHlp->pfnPrintf(pHlp, " desc chain:\n");
673	pHlp->pfnPrintf(pHlp, " head idx: ............. %d\n", uUsedIdx);
674	pHlp->pfnPrintf(pHlp, " segs: ................. %d\n", cSendSegs + cReturnSegs);
675	pHlp->pfnPrintf(pHlp, " refCnt ................ %d\n", pVirtqBuf->cRefs);
676	pHlp->pfnPrintf(pHlp, "\n");
677	pHlp->pfnPrintf(pHlp, " host-to-guest (%d bytes):\n", pVirtqBuf->cbPhysSend);
678	pHlp->pfnPrintf(pHlp, " segs: .............. %d\n", cSendSegs);
679	if (cSendSegs)
680	{
681	pHlp->pfnPrintf(pHlp, " index: ............. %d\n", pVirtqBuf->pSgPhysSend->idxSeg);
682	pHlp->pfnPrintf(pHlp, " unsent ............. %d\n", pVirtqBuf->pSgPhysSend->cbSegLeft);
683	}
684	pHlp->pfnPrintf(pHlp, "\n");
685	pHlp->pfnPrintf(pHlp, " guest-to-host (%d bytes)\n", pVirtqBuf->cbPhysReturn);
686	pHlp->pfnPrintf(pHlp, " segs: .............. %d\n", cReturnSegs);
687	if (cReturnSegs)
688	{
689	pHlp->pfnPrintf(pHlp, " index: ............. %d\n", pVirtqBuf->pSgPhysReturn->idxSeg);
690	pHlp->pfnPrintf(pHlp, " unsent ............. %d\n", pVirtqBuf->pSgPhysReturn->cbSegLeft);
691	}
692	} else
693	pHlp->pfnPrintf(pHlp, " No desc chains available\n");
694	pHlp->pfnPrintf(pHlp, "\n");
695	}
696
697
698	/** API Function: See header file */
699	DECLHIDDEN(PVIRTQBUF) virtioCoreR3VirtqBufAlloc(void)
700	{
701	PVIRTQBUF pVirtqBuf = (PVIRTQBUF)RTMemAllocZ(sizeof(VIRTQBUF_T));
702	AssertReturn(pVirtqBuf, NULL);
703	pVirtqBuf->u32Magic = VIRTQBUF_MAGIC;
704	pVirtqBuf->cRefs = 1;
705	return pVirtqBuf;
706	}
707
708
709	/** API Function: See header file */
710	DECLHIDDEN(uint32_t) virtioCoreR3VirtqBufRetain(PVIRTQBUF pVirtqBuf)
711	{
712	AssertReturn(pVirtqBuf, UINT32_MAX);
713	AssertReturn(pVirtqBuf->u32Magic == VIRTQBUF_MAGIC, UINT32_MAX);
714	uint32_t cRefs = ASMAtomicIncU32(&pVirtqBuf->cRefs);
715	Assert(cRefs > 1);
716	Assert(cRefs < 16);
717	return cRefs;
718	}
719
720	/** API Function: See header file */
721	DECLHIDDEN(uint32_t) virtioCoreR3VirtqBufRelease(PVIRTIOCORE pVirtio, PVIRTQBUF pVirtqBuf)
722	{
723	if (!pVirtqBuf)
724	return 0;
725	AssertReturn(pVirtqBuf, 0);
726	AssertReturn(pVirtqBuf->u32Magic == VIRTQBUF_MAGIC, 0);
727	uint32_t cRefs = ASMAtomicDecU32(&pVirtqBuf->cRefs);
728	Assert(cRefs < 16);
729	if (cRefs == 0)
730	{
731	pVirtqBuf->u32Magic = ~VIRTQBUF_MAGIC;
732	RTMemFree(pVirtqBuf);
733	#ifdef VBOX_WITH_STATISTICS
734	STAM_REL_COUNTER_INC(&pVirtio->StatDescChainsFreed);
735	#endif
736	}
737	RT_NOREF(pVirtio);
738	return cRefs;
739	}
740
741	/** API Function: See header file */
742	DECLHIDDEN(void) virtioCoreNotifyConfigChanged(PVIRTIOCORE pVirtio)
743	{
744	virtioNudgeGuest(pVirtio->pDevInsR3, pVirtio, VIRTIO_ISR_DEVICE_CONFIG, pVirtio->uMsixConfig);
745	}
746
747
748	/** API Function: See header file */
749	DECLHIDDEN(void) virtioCoreVirtqEnableNotify(PVIRTIOCORE pVirtio, uint16_t uVirtq, bool fEnable)
750	{
751	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
752	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
753
754	if (IS_DRIVER_OK(pVirtio))
755	{
756	uint16_t fFlags = virtioReadUsedRingFlags(pVirtio->pDevInsR3, pVirtio, pVirtq);
757
758	if (fEnable)
759	fFlags &= ~VIRTQ_USED_F_NO_NOTIFY;
760	else
761	fFlags \|= VIRTQ_USED_F_NO_NOTIFY;
762
763	virtioWriteUsedRingFlags(pVirtio->pDevInsR3, pVirtio, pVirtq, fFlags);
764	}
765	}
766
767	/** API function: See Header file */
768	DECLHIDDEN(void) virtioCoreResetAll(PVIRTIOCORE pVirtio)
769	{
770	LogFunc(("\n"));
771	pVirtio->fDeviceStatus \|= VIRTIO_STATUS_DEVICE_NEEDS_RESET;
772	if (IS_DRIVER_OK(pVirtio))
773	{
774	if (!pVirtio->fLegacyDriver)
775	pVirtio->fGenUpdatePending = true;
776	virtioNudgeGuest(pVirtio->pDevInsR3, pVirtio, VIRTIO_ISR_DEVICE_CONFIG, pVirtio->uMsixConfig);
777	}
778	}
779
780	/** API function: See Header file */
781	DECLHIDDEN(int) virtioCoreR3VirtqAvailBufPeek(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq, PVIRTQBUF pVirtqBuf)
782	{
783	return virtioCoreR3VirtqAvailBufGet(pDevIns, pVirtio, uVirtq, pVirtqBuf, false);
784	}
785
786
787	/** API function: See Header file */
788	DECLHIDDEN(int) virtioCoreR3VirtqAvailBufNext(PVIRTIOCORE pVirtio, uint16_t uVirtq)
789	{
790	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
791	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
792
793	if (!pVirtio->fLegacyDriver)
794	AssertMsgReturn((pVirtio->fDeviceStatus & VIRTIO_STATUS_DRIVER_OK) && pVirtq->uEnable,
795	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
796
797	if (IS_VIRTQ_EMPTY(pVirtio->pDevInsR3, pVirtio, pVirtq))
798	return VERR_NOT_AVAILABLE;
799
800	Log6Func(("%s avail shadow idx: %u\n", pVirtq->szName, pVirtq->uAvailIdxShadow));
801	pVirtq->uAvailIdxShadow++;
802
803	return VINF_SUCCESS;
804	}
805
806	/** API Function: See header file */
807	DECLHIDDEN(int) virtioCoreR3VirtqAvailBufGet(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq,
808	uint16_t uHeadIdx, PVIRTQBUF pVirtqBuf)
809	{
810	AssertMsgReturn(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues),
811	("uVirtq out of range"), VERR_INVALID_PARAMETER);
812
813	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
814
815	if (!pVirtio->fLegacyDriver)
816	AssertMsgReturn((pVirtio->fDeviceStatus & VIRTIO_STATUS_DRIVER_OK) && pVirtq->uEnable,
817	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
818
819	uint16_t uDescIdx = uHeadIdx;
820
821	Log6Func(("%s DESC CHAIN: (head idx = %u)\n", pVirtio->aVirtqueues[uVirtq].szName, uHeadIdx));
822
823	/*
824	* Allocate and initialize the descriptor chain structure.
825	*/
826	pVirtqBuf->u32Magic = VIRTQBUF_MAGIC;
827	pVirtqBuf->cRefs = 1;
828	pVirtqBuf->uHeadIdx = uHeadIdx;
829	pVirtqBuf->uVirtq = uVirtq;
830
831	/*
832	* Gather segments.
833	*/
834	VIRTQ_DESC_T desc;
835
836	uint32_t cbIn = 0;
837	uint32_t cbOut = 0;
838	uint32_t cSegsIn = 0;
839	uint32_t cSegsOut = 0;
840
841	PVIRTIOSGSEG paSegsIn = pVirtqBuf->aSegsIn;
842	PVIRTIOSGSEG paSegsOut = pVirtqBuf->aSegsOut;
843
844	do
845	{
846	PVIRTIOSGSEG pSeg;
847	/*
848	* Malicious guests may go beyond paSegsIn or paSegsOut boundaries by linking
849	* several descriptors into a loop. Since there is no legitimate way to get a sequences of
850	* linked descriptors exceeding the total number of descriptors in the ring (see @bugref{8620}),
851	* the following aborts I/O if breach and employs a simple log throttling algorithm to notify.
852	*/
853	if (cSegsIn + cSegsOut >= pVirtq->uQueueSize)
854	{
855	static volatile uint32_t s_cMessages = 0;
856	static volatile uint32_t s_cThreshold = 1;
857	if (ASMAtomicIncU32(&s_cMessages) == ASMAtomicReadU32(&s_cThreshold))
858	{
859	LogRelMax(64, ("Too many linked descriptors; check if the guest arranges descriptors in a loop.\n"));
860	if (ASMAtomicReadU32(&s_cMessages) != 1)
861	LogRelMax(64, ("(the above error has occured %u times so far)\n", ASMAtomicReadU32(&s_cMessages)));
862	ASMAtomicWriteU32(&s_cThreshold, ASMAtomicReadU32(&s_cThreshold) * 10);
863	}
864	break;
865	}
866	RT_UNTRUSTED_VALIDATED_FENCE();
867
868	virtioReadDesc(pDevIns, pVirtio, pVirtq, uDescIdx, &desc);
869
870	if (desc.fFlags & VIRTQ_DESC_F_WRITE)
871	{
872	Log6Func(("%s IN idx=%-4u seg=%-3u addr=%RGp cb=%u\n", pVirtq->szName, uDescIdx, cSegsIn, desc.GCPhysBuf, desc.cb));
873	cbIn += desc.cb;
874	pSeg = &paSegsIn[cSegsIn++];
875	}
876	else
877	{
878	Log6Func(("%s OUT desc_idx=%-4u seg=%-3u addr=%RGp cb=%u\n", pVirtq->szName, uDescIdx, cSegsOut, desc.GCPhysBuf, desc.cb));
879	cbOut += desc.cb;
880	pSeg = &paSegsOut[cSegsOut++];
881	#ifdef DEEP_DEBUG
882	if (LogIs11Enabled())
883	{
884	virtioCoreGCPhysHexDump(pDevIns, desc.GCPhysBuf, desc.cb, 0, NULL);
885	Log(("\n"));
886	}
887	#endif
888	}
889	pSeg->GCPhys = desc.GCPhysBuf;
890	pSeg->cbSeg = desc.cb;
891	uDescIdx = desc.uDescIdxNext;
892	} while (desc.fFlags & VIRTQ_DESC_F_NEXT);
893
894	/*
895	* Add segments to the descriptor chain structure.
896	*/
897	if (cSegsIn)
898	{
899	virtioCoreGCPhysChainInit(&pVirtqBuf->SgBufIn, paSegsIn, cSegsIn);
900	pVirtqBuf->pSgPhysReturn = &pVirtqBuf->SgBufIn;
901	pVirtqBuf->cbPhysReturn = cbIn;
902	#ifdef VBOX_WITH_STATISTICS
903	STAM_REL_COUNTER_ADD(&pVirtio->StatDescChainsSegsIn, cSegsIn);
904	#endif
905	}
906
907	if (cSegsOut)
908	{
909	virtioCoreGCPhysChainInit(&pVirtqBuf->SgBufOut, paSegsOut, cSegsOut);
910	pVirtqBuf->pSgPhysSend = &pVirtqBuf->SgBufOut;
911	pVirtqBuf->cbPhysSend = cbOut;
912	#ifdef VBOX_WITH_STATISTICS
913	STAM_REL_COUNTER_ADD(&pVirtio->StatDescChainsSegsOut, cSegsOut);
914	#endif
915	}
916
917	#ifdef VBOX_WITH_STATISTICS
918	STAM_REL_COUNTER_INC(&pVirtio->StatDescChainsAllocated);
919	#endif
920	Log6Func(("%s -- segs OUT: %u (%u bytes) IN: %u (%u bytes) --\n",
921	pVirtq->szName, cSegsOut, cbOut, cSegsIn, cbIn));
922
923	return VINF_SUCCESS;
924	}
925
926	/** API function: See Header file */
927	DECLHIDDEN(int) virtioCoreR3VirtqAvailBufGet(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq,
928	PVIRTQBUF pVirtqBuf, bool fRemove)
929	{
930	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
931	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
932
933	if (IS_VIRTQ_EMPTY(pDevIns, pVirtio, pVirtq))
934	return VERR_NOT_AVAILABLE;
935
936	uint16_t uHeadIdx = virtioReadAvailDescIdx(pDevIns, pVirtio, pVirtq, pVirtq->uAvailIdxShadow);
937
938	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
939	virtioWriteUsedAvailEvent(pDevIns,pVirtio, pVirtq, pVirtq->uAvailIdxShadow + 1);
940
941	if (fRemove)
942	pVirtq->uAvailIdxShadow++;
943
944	return virtioCoreR3VirtqAvailBufGet(pDevIns, pVirtio, uVirtq, uHeadIdx, pVirtqBuf);
945	}
946
947	/** API function: See Header file */
948	DECLHIDDEN(int) virtioCoreR3VirtqUsedBufPut(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq, PRTSGBUF pSgVirtReturn,
949	PVIRTQBUF pVirtqBuf, bool fFence)
950	{
951	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
952	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
953
954	PVIRTIOSGBUF pSgPhysReturn = pVirtqBuf->pSgPhysReturn;
955
956	Assert(pVirtqBuf->u32Magic == VIRTQBUF_MAGIC);
957	Assert(pVirtqBuf->cRefs > 0);
958
959	/*
960	* Workaround for a bug in FreeBSD's virtio-net driver up until 12.3 which supports only the legacy style devive.
961	* When the device is re-initialized from the driver it violates the spec and posts commands to the control queue
962	* before setting the DRIVER_OK flag, breaking the following check and rendering the device non-functional.
963	* The queues are properly set up at this stage however so no real harm is done and we can safely continue here,
964	* for the legacy device only of course after making sure the queue is properly set up.
965	*/
966	AssertMsgReturn( IS_DRIVER_OK(pVirtio)
967	\|\| ( pVirtio->fLegacyDriver
968	&& pVirtq->GCPhysVirtqDesc),
969	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
970
971	Log6Func((" Copying device data to %s, [desc:%u -> used ring:%u]\n",
972	VIRTQNAME(pVirtio, uVirtq), pVirtqBuf->uHeadIdx, pVirtq->uUsedIdxShadow));
973
974	/* Copy s/g buf (virtual memory) to guest phys mem (VirtIO "IN" direction). */
975
976	size_t cbCopy = 0, cbTotal = 0, cbRemain = 0;
977
978	if (pSgVirtReturn)
979	{
980	size_t cbTarget = virtioCoreGCPhysChainCalcBufSize(pSgPhysReturn);
981	cbRemain = cbTotal = RTSgBufCalcTotalLength(pSgVirtReturn);
982	AssertMsgReturn(cbTarget >= cbRemain, ("No space to write data to phys memory"), VERR_BUFFER_OVERFLOW);
983	virtioCoreGCPhysChainReset(pSgPhysReturn);
984	while (cbRemain)
985	{
986	cbCopy = RT_MIN(pSgVirtReturn->cbSegLeft, pSgPhysReturn->cbSegLeft);
987	Assert(cbCopy > 0);
988	virtioCoreGCPhysWrite(pVirtio, pDevIns, (RTGCPHYS)pSgPhysReturn->GCPhysCur, pSgVirtReturn->pvSegCur, cbCopy);
989	RTSgBufAdvance(pSgVirtReturn, cbCopy);
990	virtioCoreGCPhysChainAdvance(pSgPhysReturn, cbCopy);
991	cbRemain -= cbCopy;
992	}
993
994	if (fFence)
995	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE(); /* needed? */
996
997	Assert(!(cbCopy >> 32));
998	}
999
1000	/* Flag if write-ahead crosses threshold where guest driver indicated it wants event notification */
1001	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
1002	if (pVirtq->uUsedIdxShadow == virtioReadAvailUsedEvent(pDevIns, pVirtio, pVirtq))
1003	pVirtq->fUsedRingEvent = true;
1004
1005	/*
1006	* Place used buffer's descriptor in used ring but don't update used ring's slot index.
1007	* That will be done with a subsequent client call to virtioCoreVirtqUsedRingSync()
1008	*
1009	* @todo r=aeichner: The increment of the shadow index is not atomic but this code can be called
1010	* concurrently!!
1011	*/
1012	virtioWriteUsedElem(pDevIns, pVirtio, pVirtq, pVirtq->uUsedIdxShadow++, pVirtqBuf->uHeadIdx, (uint32_t)cbTotal);
1013
1014	#ifdef LOG_ENABLED
1015	if (LogIs6Enabled() && pSgVirtReturn)
1016	{
1017
1018	LogFunc((" ... %d segs, %zu bytes, copied to %u byte buf@offset=%u. Residual: %zu bytes\n",
1019	pSgVirtReturn->cSegs, cbTotal - cbRemain, pVirtqBuf->cbPhysReturn,
1020	((virtioCoreGCPhysChainCalcBufSize(pVirtqBuf->pSgPhysReturn) -
1021	virtioCoreGCPhysChainCalcLengthLeft(pVirtqBuf->pSgPhysReturn)) - (cbTotal - cbRemain)),
1022	virtioCoreGCPhysChainCalcLengthLeft(pVirtqBuf->pSgPhysReturn) ));
1023
1024	uint16_t uPending = virtioCoreR3CountPendingBufs(
1025	virtioReadUsedRingIdx(pDevIns, pVirtio, pVirtq),
1026	pVirtq->uUsedIdxShadow, pVirtq->uQueueSize);
1027
1028	LogFunc((" %u used buf%s not synced in %s\n", uPending, uPending == 1 ? "" : "s ",
1029	VIRTQNAME(pVirtio, uVirtq)));
1030	}
1031	#endif
1032	return VINF_SUCCESS;
1033	}
1034
1035	/** API function: See Header file */
1036	DECLHIDDEN(int) virtioCoreR3VirtqUsedBufPut(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq,
1037	size_t cb, void const *pv, PVIRTQBUF pVirtqBuf, size_t cbEnqueue, bool fFence)
1038	{
1039	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1040	Assert(pv);
1041
1042	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1043	PVIRTIOSGBUF pSgPhysReturn = pVirtqBuf->pSgPhysReturn;
1044
1045	Assert(pVirtqBuf->u32Magic == VIRTQBUF_MAGIC);
1046	Assert(pVirtqBuf->cRefs > 0);
1047
1048	AssertMsgReturn(IS_DRIVER_OK(pVirtio), ("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
1049
1050	Log6Func((" Copying device data to %s, [desc chain head idx:%u]\n",
1051	VIRTQNAME(pVirtio, uVirtq), pVirtqBuf->uHeadIdx));
1052	/*
1053	* Convert virtual memory simple buffer to guest physical memory (VirtIO descriptor chain)
1054	*/
1055	uint8_t pvBuf = (uint8_t )pv;
1056	size_t cbRemain = cb, cbCopy = 0;
1057	while (cbRemain)
1058	{
1059	cbCopy = RT_MIN(pSgPhysReturn->cbSegLeft, cbRemain);
1060	Assert(cbCopy > 0);
1061	virtioCoreGCPhysWrite(pVirtio, pDevIns, (RTGCPHYS)pSgPhysReturn->GCPhysCur, pvBuf, cbCopy);
1062	virtioCoreGCPhysChainAdvance(pSgPhysReturn, cbCopy);
1063	pvBuf += cbCopy;
1064	cbRemain -= cbCopy;
1065	}
1066	LogFunc((" ...%zu bytes, copied to %u byte buf@offset=%u. Residual: %zu bytes\n",
1067	cb , pVirtqBuf->cbPhysReturn,
1068	((virtioCoreGCPhysChainCalcBufSize(pVirtqBuf->pSgPhysReturn) -
1069	virtioCoreGCPhysChainCalcLengthLeft(pVirtqBuf->pSgPhysReturn)) - cb),
1070	virtioCoreGCPhysChainCalcLengthLeft(pVirtqBuf->pSgPhysReturn)));
1071
1072	if (cbEnqueue)
1073	{
1074	if (fFence)
1075	{
1076	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE(); /* needed? */
1077	Assert(!(cbCopy >> 32));
1078	}
1079	/* Flag if write-ahead crosses threshold where guest driver indicated it wants event notification */
1080	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
1081	if (pVirtq->uUsedIdxShadow == virtioReadAvailUsedEvent(pDevIns, pVirtio, pVirtq))
1082	pVirtq->fUsedRingEvent = true;
1083	/*
1084	* Place used buffer's descriptor in used ring but don't update used ring's slot index.
1085	* That will be done with a subsequent client call to virtioCoreVirtqUsedRingSync()
1086	*/
1087	Log6Func((" Enqueue desc chain head idx %u to %s used ring @ %u\n", pVirtqBuf->uHeadIdx,
1088	VIRTQNAME(pVirtio, uVirtq), pVirtq->uUsedIdxShadow));
1089
1090	virtioWriteUsedElem(pDevIns, pVirtio, pVirtq, pVirtq->uUsedIdxShadow++, pVirtqBuf->uHeadIdx, (uint32_t)cbEnqueue);
1091
1092	#ifdef LOG_ENABLED
1093	if (LogIs6Enabled())
1094	{
1095	uint16_t uPending = virtioCoreR3CountPendingBufs(
1096	virtioReadUsedRingIdx(pDevIns, pVirtio, pVirtq),
1097	pVirtq->uUsedIdxShadow, pVirtq->uQueueSize);
1098
1099	LogFunc((" %u used buf%s not synced in %s\n",
1100	uPending, uPending == 1 ? "" : "s ", VIRTQNAME(pVirtio, uVirtq)));
1101	}
1102	#endif
1103	} /* fEnqueue */
1104
1105	return VINF_SUCCESS;
1106	}
1107
1108
1109	#endif /* IN_RING3 */
1110
1111	/** API function: See Header file */
1112	DECLHIDDEN(int) virtioCoreVirtqUsedRingSync(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq)
1113	{
1114	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1115	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1116
1117	if (!pVirtio->fLegacyDriver)
1118	AssertMsgReturn((pVirtio->fDeviceStatus & VIRTIO_STATUS_DRIVER_OK) && pVirtq->uEnable,
1119	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
1120
1121	Log6Func((" Sync %s used ring (%u -> idx)\n",
1122	pVirtq->szName, pVirtq->uUsedIdxShadow));
1123
1124	virtioWriteUsedRingIdx(pDevIns, pVirtio, pVirtq, pVirtq->uUsedIdxShadow);
1125	virtioCoreNotifyGuestDriver(pDevIns, pVirtio, uVirtq);
1126
1127	return VINF_SUCCESS;
1128	}
1129
1130	/**
1131	* This is called from the MMIO callback code when the guest does an MMIO access to the
1132	* mapped queue notification capability area corresponding to a particular queue, to notify
1133	* the queue handler of available data in the avail ring of the queue (VirtIO 1.0, 4.1.4.4.1)
1134	*
1135	* @param pDevIns The device instance.
1136	* @param pVirtio Pointer to the shared virtio state.
1137	* @param uVirtq Virtq to check for guest interrupt handling preference
1138	* @param uNotifyIdx Notification index
1139	*/
1140	static void virtioCoreVirtqNotified(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq, uint16_t uNotifyIdx)
1141	{
1142	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1143
1144	/* VirtIO 1.0, section 4.1.5.2 implies uVirtq and uNotifyIdx should match. Disregarding any of
1145	* these notifications (if those indicies disagree) may break device/driver synchronization,
1146	* causing eternal throughput starvation, yet there's no specified way to disambiguate
1147	* which queue to wake-up in any awkward situation where the two parameters differ.
1148	*/
1149	AssertMsg(uNotifyIdx == uVirtq,
1150	("Guest kicked virtq %d's notify addr w/non-corresponding virtq idx %d\n",
1151	uVirtq, uNotifyIdx));
1152	RT_NOREF(uNotifyIdx);
1153
1154	AssertReturnVoid(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1155	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1156
1157	Log6Func(("%s: (desc chains: %u)\n", *pVirtq->szName ? pVirtq->szName : "?UNAMED QUEUE?",
1158	virtioCoreVirtqAvailCnt(pDevIns, pVirtio, pVirtq)));
1159
1160	/* Inform client */
1161	pVirtioCC->pfnVirtqNotified(pDevIns, pVirtio, uVirtq);
1162	RT_NOREF2(pVirtio, pVirtq);
1163	}
1164
1165	/**
1166	* Trigger MSI-X or INT# interrupt to notify guest of data added to used ring of
1167	* the specified virtq, depending on the interrupt configuration of the device
1168	* and depending on negotiated and realtime constraints flagged by the guest driver.
1169	*
1170	* See VirtIO 1.0 specification (section 2.4.7).
1171	*
1172	* @param pDevIns The device instance.
1173	* @param pVirtio Pointer to the shared virtio state.
1174	* @param uVirtq Virtq to check for guest interrupt handling preference
1175	*/
1176	static void virtioCoreNotifyGuestDriver(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq)
1177	{
1178	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1179	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1180
1181	if (!IS_DRIVER_OK(pVirtio))
1182	{
1183	LogFunc(("Guest driver not in ready state.\n"));
1184	return;
1185	}
1186
1187	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
1188	{
1189	if (pVirtq->fUsedRingEvent)
1190	{
1191	#ifdef IN_RING3
1192	Log6Func(("...kicking guest %s, VIRTIO_F_EVENT_IDX set and threshold (%d) reached\n",
1193	pVirtq->szName, (uint16_t)virtioReadAvailUsedEvent(pDevIns, pVirtio, pVirtq)));
1194	#endif
1195	virtioNudgeGuest(pDevIns, pVirtio, VIRTIO_ISR_VIRTQ_INTERRUPT, pVirtq->uMsixVector);
1196	pVirtq->fUsedRingEvent = false;
1197	return;
1198	}
1199	#ifdef IN_RING3
1200	Log6Func(("...skip interrupt %s, VIRTIO_F_EVENT_IDX set but threshold (%d) not reached (%d)\n",
1201	pVirtq->szName,(uint16_t)virtioReadAvailUsedEvent(pDevIns, pVirtio, pVirtq), pVirtq->uUsedIdxShadow));
1202	#endif
1203	}
1204	else
1205	{
1206	/** If guest driver hasn't suppressed interrupts, interrupt */
1207	if (!(virtioReadAvailRingFlags(pDevIns, pVirtio, pVirtq) & VIRTQ_AVAIL_F_NO_INTERRUPT))
1208	{
1209	virtioNudgeGuest(pDevIns, pVirtio, VIRTIO_ISR_VIRTQ_INTERRUPT, pVirtq->uMsixVector);
1210	return;
1211	}
1212	Log6Func(("...skipping interrupt for %s (guest set VIRTQ_AVAIL_F_NO_INTERRUPT)\n", pVirtq->szName));
1213	}
1214	}
1215
1216	/**
1217	* Raise interrupt or MSI-X
1218	*
1219	* @param pDevIns The device instance.
1220	* @param pVirtio Pointer to the shared virtio state.
1221	* @param uCause Interrupt cause bit mask to set in PCI ISR port.
1222	* @param uVec MSI-X vector, if enabled
1223	*/
1224	static int virtioNudgeGuest(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint8_t uCause, uint16_t uMsixVector)
1225	{
1226	if (uCause == VIRTIO_ISR_VIRTQ_INTERRUPT)
1227	Log6Func(("Reason for interrupt - buffer added to 'used' ring.\n"));
1228	else
1229	if (uCause == VIRTIO_ISR_DEVICE_CONFIG)
1230	Log6Func(("Reason for interrupt - device config change\n"));
1231
1232	if (pVirtio->uIrqMmio)
1233	{
1234	pVirtio->uISR \|= uCause;
1235	PDMDevHlpISASetIrq(pDevIns, pVirtio->uIrqMmio, PDM_IRQ_LEVEL_HIGH);
1236	}
1237	else if (!pVirtio->fMsiSupport)
1238	{
1239	pVirtio->uISR \|= uCause;
1240	PDMDevHlpPCISetIrq(pDevIns, 0, PDM_IRQ_LEVEL_HIGH);
1241	}
1242	else if (uMsixVector != VIRTIO_MSI_NO_VECTOR)
1243	PDMDevHlpPCISetIrq(pDevIns, uMsixVector, 1);
1244	return VINF_SUCCESS;
1245	}
1246
1247	/**
1248	* Lower interrupt (Called when guest reads ISR and when resetting)
1249	*
1250	* @param pDevIns The device instance.
1251	*/
1252	static void virtioLowerInterrupt(PPDMDEVINS pDevIns, uint16_t uMsixVector)
1253	{
1254	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1255	if (pVirtio->uIrqMmio)
1256	PDMDevHlpISASetIrq(pDevIns, pVirtio->uIrqMmio, PDM_IRQ_LEVEL_LOW);
1257	else if (!pVirtio->fMsiSupport)
1258	PDMDevHlpPCISetIrq(pDevIns, 0, PDM_IRQ_LEVEL_LOW);
1259	else if (uMsixVector != VIRTIO_MSI_NO_VECTOR)
1260	PDMDevHlpPCISetIrq(pDevIns, pVirtio->uMsixConfig, PDM_IRQ_LEVEL_LOW);
1261	}
1262
1263	#ifdef IN_RING3
1264	static void virtioResetVirtq(PVIRTIOCORE pVirtio, uint16_t uVirtq)
1265	{
1266	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1267	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1268
1269	pVirtq->uQueueSize = VIRTQ_SIZE;
1270	pVirtq->uEnable = false;
1271	pVirtq->uNotifyOffset = uVirtq;
1272	pVirtq->fUsedRingEvent = false;
1273	pVirtq->uAvailIdxShadow = 0;
1274	pVirtq->uUsedIdxShadow = 0;
1275	pVirtq->uMsixVector = uVirtq + 2;
1276
1277	if (!pVirtio->fMsiSupport) /* VirtIO 1.0, 4.1.4.3 and 4.1.5.1.2 */
1278	pVirtq->uMsixVector = VIRTIO_MSI_NO_VECTOR;
1279
1280	virtioLowerInterrupt(pVirtio->pDevInsR3, pVirtq->uMsixVector);
1281	}
1282
1283	static void virtioResetDevice(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio)
1284	{
1285	LogFunc(("Resetting device VirtIO state\n"));
1286	pVirtio->fLegacyDriver = pVirtio->fOfferLegacy; /* Cleared if VIRTIO_F_VERSION_1 feature ack'd */
1287	pVirtio->uDeviceFeaturesSelect = 0;
1288	pVirtio->uDriverFeaturesSelect = 0;
1289	pVirtio->uConfigGeneration = 0;
1290	pVirtio->fDeviceStatus = 0;
1291	pVirtio->uISR = 0;
1292
1293	if (!pVirtio->fMsiSupport)
1294	virtioLowerInterrupt(pDevIns, 0);
1295	else
1296	{
1297	virtioLowerInterrupt(pDevIns, pVirtio->uMsixConfig);
1298	for (int i = 0; i < VIRTQ_MAX_COUNT; i++)
1299	virtioLowerInterrupt(pDevIns, pVirtio->aVirtqueues[i].uMsixVector);
1300	}
1301
1302	if (!pVirtio->fMsiSupport) /* VirtIO 1.0, 4.1.4.3 and 4.1.5.1.2 */
1303	pVirtio->uMsixConfig = VIRTIO_MSI_NO_VECTOR;
1304
1305	for (uint16_t uVirtq = 0; uVirtq < VIRTQ_MAX_COUNT; uVirtq++)
1306	virtioResetVirtq(pVirtio, uVirtq);
1307	}
1308
1309	/**
1310	* Invoked by this implementation when guest driver resets the device.
1311	* The driver itself will not until the device has read the status change.
1312	*/
1313	static void virtioGuestR3WasReset(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
1314	{
1315	Log(("%-23s: Guest reset the device\n", __FUNCTION__));
1316
1317	/* Let the client know */
1318	pVirtioCC->pfnStatusChanged(pVirtio, pVirtioCC, 0 /* fDriverOk */);
1319	virtioResetDevice(pDevIns, pVirtio);
1320	}
1321
1322	DECLHIDDEN(void) virtioCoreR3ResetDevice(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
1323	{
1324	virtioGuestR3WasReset(pDevIns, pVirtio, pVirtioCC);
1325	}
1326	#endif /* IN_RING3 */
1327
1328	/*
1329	* Determines whether guest virtio driver is modern or legacy and does callback
1330	* informing device-specific code that feature negotiation is complete.
1331	* Should be called only once (coordinated via the 'toggle' flag)
1332	*/
1333	#ifdef IN_RING3
1334	DECLINLINE(void) virtioR3DoFeaturesCompleteOnceOnly(PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
1335	{
1336	if (pVirtio->uDriverFeatures & VIRTIO_F_VERSION_1)
1337	{
1338	LogFunc(("VIRTIO_F_VERSION_1 feature ack'd by guest\n"));
1339	pVirtio->fLegacyDriver = 0;
1340	}
1341	else
1342	{
1343	if (pVirtio->fOfferLegacy)
1344	{
1345	pVirtio->fLegacyDriver = 1;
1346	LogFunc(("VIRTIO_F_VERSION_1 feature was NOT set by guest\n"));
1347	}
1348	else
1349	AssertMsgFailed(("Guest didn't accept VIRTIO_F_VERSION_1, but fLegacyOffered flag not set.\n"));
1350	}
1351	if (pVirtioCC->pfnFeatureNegotiationComplete)
1352	pVirtioCC->pfnFeatureNegotiationComplete(pVirtio, pVirtio->uDriverFeatures, pVirtio->fLegacyDriver);
1353	pVirtio->fDriverFeaturesWritten \|= DRIVER_FEATURES_COMPLETE_HANDLED;
1354	}
1355	#endif
1356
1357
1358	/**
1359	* Handles a write to the device status register from the driver.
1360	*
1361	* @returns VBox status code
1362	*
1363	* @param pDevIns The device instance.
1364	* @param pVirtio Pointer to the shared virtio state.
1365	* @param pVirtioCC Pointer to the current context virtio state.
1366	* @param fDeviceStatus The device status to be written.
1367	*/
1368	DECLINLINE(int) virtioDeviceStatusWrite(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC,
1369	uint8_t fDeviceStatus)
1370	{
1371	pVirtio->fDeviceStatus = fDeviceStatus;
1372	bool fDeviceReset = pVirtio->fDeviceStatus == 0;
1373	#ifdef LOG_ENABLED
1374	if (LogIs7Enabled())
1375	{
1376	char szOut[80] = { 0 };
1377	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
1378	Log(("%-23s: Guest wrote fDeviceStatus ................ (%s)\n", __FUNCTION__, szOut));
1379	}
1380	#endif
1381	bool const fStatusChanged = IS_DRIVER_OK(pVirtio) != WAS_DRIVER_OK(pVirtio);
1382
1383	if (fDeviceReset \|\| fStatusChanged)
1384	{
1385	#ifdef IN_RING0
1386	/* Since VirtIO status changes are cumbersome by nature, e.g. not a benchmark priority,
1387	* handle the rest in R3 to facilitate logging or whatever dev-specific client needs to do */
1388	Log6(("%-23s: RING0 => RING3 (demote)\n", __FUNCTION__));
1389	return VINF_IOM_R3_MMIO_WRITE;
1390	#endif
1391	}
1392
1393	#ifdef IN_RING3
1394	/*
1395	* Notify client only if status actually changed from last time and when we're reset.
1396	*/
1397	if (fDeviceReset)
1398	virtioGuestR3WasReset(pDevIns, pVirtio, pVirtioCC);
1399
1400	if (fStatusChanged)
1401	pVirtioCC->pfnStatusChanged(pVirtio, pVirtioCC, IS_DRIVER_OK(pVirtio));
1402	#endif
1403	/*
1404	* Save the current status for the next write so we can see what changed.
1405	*/
1406	pVirtio->fPrevDeviceStatus = pVirtio->fDeviceStatus;
1407	return VINF_SUCCESS;
1408	}
1409
1410
1411	/**
1412	* Handles a read from the device status register from the driver.
1413	*
1414	* @returns The device status register value.
1415	*
1416	* @param pVirtio Pointer to the shared virtio state.
1417	*/
1418	DECLINLINE(uint8_t) virtioDeviceStatusRead(PVIRTIOCORE pVirtio)
1419	{
1420	#ifdef LOG_ENABLED
1421	if (LogIs7Enabled())
1422	{
1423	char szOut[80] = { 0 };
1424	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
1425	LogFunc(("Guest read fDeviceStatus ................ (%s)\n", szOut));
1426	}
1427	#endif
1428	return pVirtio->fDeviceStatus;
1429	}
1430
1431
1432	/**
1433	* Handle accesses to Common Configuration capability
1434	*
1435	* @returns VBox status code
1436	*
1437	* @param pDevIns The device instance.
1438	* @param pVirtio Pointer to the shared virtio state.
1439	* @param pVirtioCC Pointer to the current context virtio state.
1440	* @param fWrite Set if write access, clear if read access.
1441	* @param uOffsetOfAccess The common configuration capability offset.
1442	* @param cb Number of bytes to read or write
1443	* @param pv Pointer to location to write to or read from
1444	*/
1445	static int virtioCommonCfgAccessed(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC,
1446	int fWrite, uint32_t uOffsetOfAccess, unsigned cb, void *pv)
1447	{
1448	uint16_t uVirtq = pVirtio->uVirtqSelect;
1449	int rc = VINF_SUCCESS;
1450	uint64_t val;
1451	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDeviceFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1452	{
1453	if (fWrite) /* Guest WRITE pCommonCfg>uDeviceFeatures */
1454	{
1455	/* VirtIO 1.0, 4.1.4.3 states device_feature is a (guest) driver readonly field,
1456	* yet the linux driver attempts to write/read it back twice */
1457	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1458	LogFunc(("... WARNING: Guest attempted to write readonly virtio_pci_common_cfg.device_feature (ignoring)\n"));
1459	return VINF_IOM_MMIO_UNUSED_00;
1460	}
1461	else /* Guest READ pCommonCfg->uDeviceFeatures */
1462	{
1463	switch (pVirtio->uDeviceFeaturesSelect)
1464	{
1465	case 0:
1466	val = pVirtio->uDeviceFeatures & UINT32_C(0xffffffff);
1467	memcpy(pv, &val, cb);
1468	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1469	break;
1470	case 1:
1471	val = pVirtio->uDeviceFeatures >> 32;
1472	memcpy(pv, &val, cb);
1473	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess + sizeof(uint32_t));
1474	break;
1475	default:
1476	LogFunc(("Guest read uDeviceFeatures with out of range selector (%#x), returning 0\n",
1477	pVirtio->uDeviceFeaturesSelect));
1478	return VINF_IOM_MMIO_UNUSED_00;
1479	}
1480	}
1481	}
1482	else
1483	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1484	{
1485	if (fWrite) /* Guest WRITE pCommonCfg->udriverFeatures */
1486	{
1487	switch (pVirtio->uDriverFeaturesSelect)
1488	{
1489	case 0:
1490	memcpy(&pVirtio->uDriverFeatures, pv, cb);
1491	pVirtio->fDriverFeaturesWritten \|= DRIVER_FEATURES_0_WRITTEN;
1492	LogFunc(("Set DRIVER_FEATURES_0_WRITTEN. pVirtio->fDriverFeaturesWritten=%d\n", pVirtio->fDriverFeaturesWritten));
1493	if ( (pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_0_AND_1_WRITTEN) == DRIVER_FEATURES_0_AND_1_WRITTEN
1494	&& !(pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_COMPLETE_HANDLED))
1495	#ifdef IN_RING0
1496	return VINF_IOM_R3_MMIO_WRITE;
1497	#endif
1498	#ifdef IN_RING3
1499	virtioR3DoFeaturesCompleteOnceOnly(pVirtio, pVirtioCC);
1500	#endif
1501	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1502	break;
1503	case 1:
1504	memcpy((char *)&pVirtio->uDriverFeatures + sizeof(uint32_t), pv, cb);
1505	pVirtio->fDriverFeaturesWritten \|= DRIVER_FEATURES_1_WRITTEN;
1506	LogFunc(("Set DRIVER_FEATURES_1_WRITTEN. pVirtio->fDriverFeaturesWritten=%d\n", pVirtio->fDriverFeaturesWritten));
1507	if ( (pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_0_AND_1_WRITTEN) == DRIVER_FEATURES_0_AND_1_WRITTEN
1508	&& !(pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_COMPLETE_HANDLED))
1509	#ifdef IN_RING0
1510	return VINF_IOM_R3_MMIO_WRITE;
1511	#endif
1512	#ifdef IN_RING3
1513	virtioR3DoFeaturesCompleteOnceOnly(pVirtio, pVirtioCC);
1514	#endif
1515	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess + sizeof(uint32_t));
1516	break;
1517	default:
1518	LogFunc(("Guest wrote uDriverFeatures with out of range selector (%#x), returning 0\n",
1519	pVirtio->uDriverFeaturesSelect));
1520	return VINF_SUCCESS;
1521	}
1522	}
1523	else /* Guest READ pCommonCfg->udriverFeatures */
1524	{
1525	switch (pVirtio->uDriverFeaturesSelect)
1526	{
1527	case 0:
1528	val = pVirtio->uDriverFeatures & 0xffffffff;
1529	memcpy(pv, &val, cb);
1530	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1531	break;
1532	case 1:
1533	val = (pVirtio->uDriverFeatures >> 32) & 0xffffffff;
1534	memcpy(pv, &val, cb);
1535	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess + 4);
1536	break;
1537	default:
1538	LogFunc(("Guest read uDriverFeatures with out of range selector (%#x), returning 0\n",
1539	pVirtio->uDriverFeaturesSelect));
1540	return VINF_IOM_MMIO_UNUSED_00;
1541	}
1542	}
1543	}
1544	else
1545	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uNumVirtqs, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1546	{
1547	if (fWrite)
1548	{
1549	Log2Func(("Guest attempted to write readonly virtio_pci_common_cfg.num_queues\n"));
1550	return VINF_SUCCESS;
1551	}
1552	(uint16_t )pv = VIRTQ_MAX_COUNT;
1553	VIRTIO_DEV_CONFIG_LOG_ACCESS(uNumVirtqs, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1554	}
1555	else
1556	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fDeviceStatus, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1557	{
1558	if (fWrite) /* Guest WRITE pCommonCfg->fDeviceStatus */
1559	rc = virtioDeviceStatusWrite(pDevIns, pVirtio, pVirtioCC, (uint8_t )pv);
1560	else /* Guest READ pCommonCfg->fDeviceStatus */
1561	(uint8_t )pv = virtioDeviceStatusRead(pVirtio);
1562	}
1563	else
1564	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixConfig, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1565	VIRTIO_DEV_CONFIG_ACCESS( uMsixConfig, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1566	else
1567	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uDeviceFeaturesSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1568	VIRTIO_DEV_CONFIG_ACCESS( uDeviceFeaturesSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1569	else
1570	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uDriverFeaturesSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1571	VIRTIO_DEV_CONFIG_ACCESS( uDriverFeaturesSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1572	else
1573	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uConfigGeneration, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1574	VIRTIO_DEV_CONFIG_ACCESS( uConfigGeneration, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1575	else
1576	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uVirtqSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1577	{
1578	if (fWrite) {
1579	uint16_t uVirtqNew = (uint16_t )pv;
1580
1581	if (uVirtqNew < RT_ELEMENTS(pVirtio->aVirtqueues))
1582	VIRTIO_DEV_CONFIG_ACCESS( uVirtqSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1583	else
1584	LogFunc(("... WARNING: Guest attempted to write invalid virtq selector (ignoring)\n"));
1585	}
1586	else
1587	VIRTIO_DEV_CONFIG_ACCESS( uVirtqSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1588	}
1589	else
1590	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( GCPhysVirtqDesc, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1591	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( GCPhysVirtqDesc, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1592	else
1593	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( GCPhysVirtqAvail, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1594	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( GCPhysVirtqAvail, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1595	else
1596	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( GCPhysVirtqUsed, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1597	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( GCPhysVirtqUsed, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1598	else
1599	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uQueueSize, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1600	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uQueueSize, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1601	else
1602	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uEnable, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1603	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uEnable, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1604	else
1605	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uNotifyOffset, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1606	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uNotifyOffset, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1607	else
1608	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixVector, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1609	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uMsixVector, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1610	else
1611	{
1612	Log2Func(("Bad guest %s access to virtio_pci_common_cfg: uOffsetOfAccess=%#x (%d), cb=%d\n",
1613	fWrite ? "write" : "read ", uOffsetOfAccess, uOffsetOfAccess, cb));
1614	return fWrite ? VINF_SUCCESS : VINF_IOM_MMIO_UNUSED_00;
1615	}
1616
1617	#ifndef IN_RING3
1618	RT_NOREF(pDevIns, pVirtioCC);
1619	#endif
1620	return rc;
1621	}
1622
1623	/**
1624	* @callback_method_impl{FNIOMIOPORTNEWIN)
1625	*
1626	* This I/O handler exists only to handle access from legacy drivers.
1627	*/
1628	static DECLCALLBACK(VBOXSTRICTRC) virtioLegacyIOPortIn(PPDMDEVINS pDevIns, void pvUser, RTIOPORT offPort, uint32_t pu32, unsigned cb)
1629	{
1630	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1631	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatRead), a);
1632
1633	RT_NOREF(pvUser);
1634	Log(("%-23s: Port read at offset=%RTiop, cb=%#x%s",
1635	__FUNCTION__, offPort, cb,
1636	VIRTIO_DEV_CONFIG_MATCH_MEMBER(fIsrStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort) ? "" : "\n"));
1637
1638	void pv = pu32; / To use existing macros */
1639	int fWrite = 0; /* To use existing macros */
1640
1641	uint16_t uVirtq = pVirtio->uVirtqSelect;
1642
1643	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDeviceFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1644	{
1645	uint32_t val = pVirtio->uDeviceFeatures & UINT32_C(0xffffffff);
1646	memcpy(pu32, &val, cb);
1647	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1648	}
1649	else
1650	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDriverFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1651	{
1652	uint32_t val = pVirtio->uDriverFeatures & UINT32_C(0xffffffff);
1653	memcpy(pu32, &val, cb);
1654	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1655	}
1656	else
1657	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fDeviceStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1658	{
1659	(uint8_t )pu32 = pVirtio->fDeviceStatus;
1660	#ifdef LOG_ENABLED
1661	if (LogIs7Enabled())
1662	{
1663	char szOut[80] = { 0 };
1664	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
1665	Log(("%-23s: Guest read fDeviceStatus ................ (%s)\n", __FUNCTION__, szOut));
1666	}
1667	#endif
1668	}
1669	else
1670	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fIsrStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1671	{
1672	ASSERT_GUEST_MSG(cb == 1, ("%d\n", cb));
1673	(uint8_t )pu32 = pVirtio->uISR;
1674	pVirtio->uISR = 0;
1675	virtioLowerInterrupt( pDevIns, 0);
1676	Log((" (ISR read and cleared)\n"));
1677	}
1678	else
1679	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uVirtqSelect, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1680	VIRTIO_DEV_CONFIG_ACCESS( uVirtqSelect, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1681	else
1682	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uVirtqPfn, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1683	{
1684	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[uVirtq];
1685	*pu32 = pVirtQueue->GCPhysVirtqDesc >> GUEST_PAGE_SHIFT;
1686	Log(("%-23s: Guest read uVirtqPfn .................... %#x\n", __FUNCTION__, *pu32));
1687	}
1688	else
1689	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uQueueSize, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1690	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uQueueSize, uVirtq, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio->aVirtqueues);
1691	else
1692	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uQueueNotify, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1693	VIRTIO_DEV_CONFIG_ACCESS( uQueueNotify, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1694	#ifdef LEGACY_MSIX_SUPPORTED
1695	else
1696	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixConfig, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1697	VIRTIO_DEV_CONFIG_ACCESS( uMsixConfig, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1698	else
1699	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixVector, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1700	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uMsixVector, uVirtq, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio->aVirtqueues);
1701	#endif
1702	else if (offPort >= sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T))
1703	{
1704	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
1705	#ifdef IN_RING3
1706	/* Access device-specific configuration */
1707	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1708	int rc = pVirtioCC->pfnDevCapRead(pDevIns, offPort - sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T), pv, cb);
1709	return rc;
1710	#else
1711	return VINF_IOM_R3_IOPORT_READ;
1712	#endif
1713	}
1714	else
1715	{
1716	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
1717	Log2Func(("Bad guest read access to virtio_legacy_pci_common_cfg: offset=%#x, cb=%x\n",
1718	offPort, cb));
1719	int rc = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
1720	"virtioLegacyIOPortIn: no valid port at offset offset=%RTiop cb=%#x\n", offPort, cb);
1721	return rc;
1722	}
1723	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
1724	return VINF_SUCCESS;
1725	}
1726
1727	/**
1728	* @callback_method_impl{ * @callback_method_impl{FNIOMIOPORTNEWOUT}
1729	*
1730	* This I/O Port interface exists only to handle access from legacy drivers.
1731	*/
1732	static DECLCALLBACK(VBOXSTRICTRC) virtioLegacyIOPortOut(PPDMDEVINS pDevIns, void *pvUser, RTIOPORT offPort, uint32_t u32, unsigned cb)
1733	{
1734	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1735	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatWrite), a);
1736	RT_NOREF(pvUser);
1737
1738	uint16_t uVirtq = pVirtio->uVirtqSelect;
1739	uint32_t u32OnStack = u32; /* allows us to use this impl's MMIO parsing macros */
1740	void pv = &u32OnStack; / To use existing macros */
1741	int fWrite = 1; /* To use existing macros */
1742
1743	Log(("%-23s: Port written at offset=%RTiop, cb=%#x, u32=%#x\n", __FUNCTION__, offPort, cb, u32));
1744
1745	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uVirtqSelect, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1746	{
1747	if (u32 < RT_ELEMENTS(pVirtio->aVirtqueues))
1748	VIRTIO_DEV_CONFIG_ACCESS( uVirtqSelect, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1749	else
1750	LogFunc(("... WARNING: Guest attempted to write invalid virtq selector (ignoring)\n"));
1751	}
1752	else
1753	#ifdef LEGACY_MSIX_SUPPORTED
1754	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixConfig, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1755	VIRTIO_DEV_CONFIG_ACCESS( uMsixConfig, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1756	else
1757	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixVector, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1758	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uMsixVector, uVirtq, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio->aVirtqueues);
1759	else
1760	#endif
1761	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDeviceFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1762	{
1763	/* Check to see if guest acknowledged unsupported features */
1764	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1765	LogFunc(("... WARNING: Guest attempted to write readonly virtio_pci_common_cfg.device_feature (ignoring)\n"));
1766	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1767	return VINF_SUCCESS;
1768	}
1769	else
1770	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDriverFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1771	{
1772	memcpy(&pVirtio->uDriverFeatures, pv, cb);
1773	if ((pVirtio->uDriverFeatures & ~VIRTIO_DEV_INDEPENDENT_LEGACY_FEATURES_OFFERED) == 0)
1774	{
1775	Log(("Guest asked for features host does not support! (host=%x guest=%x)\n",
1776	VIRTIO_DEV_INDEPENDENT_LEGACY_FEATURES_OFFERED, pVirtio->uDriverFeatures));
1777	pVirtio->uDriverFeatures &= VIRTIO_DEV_INDEPENDENT_LEGACY_FEATURES_OFFERED;
1778	}
1779	if (!(pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_COMPLETE_HANDLED))
1780	{
1781	#ifdef IN_RING0
1782	Log6(("%-23s: RING0 => RING3 (demote)\n", __FUNCTION__));
1783	return VINF_IOM_R3_IOPORT_WRITE;
1784	#endif
1785	#ifdef IN_RING3
1786	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1787	virtioR3DoFeaturesCompleteOnceOnly(pVirtio, pVirtioCC);
1788	#endif
1789	}
1790	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1791	}
1792	else
1793	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uQueueSize, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1794	{
1795	VIRTIO_DEV_CONFIG_LOG_ACCESS(uQueueSize, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1796	LogFunc(("... WARNING: Guest attempted to write readonly device_feature (queue size) (ignoring)\n"));
1797	return VINF_SUCCESS;
1798	}
1799	else
1800	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fDeviceStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1801	{
1802	bool const fDriverInitiatedReset = (pVirtio->fDeviceStatus = (uint8_t)u32) == 0;
1803	bool const fDriverStateImproved = IS_DRIVER_OK(pVirtio) && !WAS_DRIVER_OK(pVirtio);
1804	#ifdef LOG_ENABLED
1805	if (LogIs7Enabled())
1806	{
1807	char szOut[80] = { 0 };
1808	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
1809	Log(("%-23s: Guest wrote fDeviceStatus ................ (%s)\n", __FUNCTION__, szOut));
1810	}
1811	#endif
1812	if (fDriverStateImproved \|\| fDriverInitiatedReset)
1813	{
1814	#ifdef IN_RING0
1815	Log6(("%-23s: RING0 => RING3 (demote)\n", __FUNCTION__));
1816	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1817	return VINF_IOM_R3_IOPORT_WRITE;
1818	#endif
1819	}
1820
1821	#ifdef IN_RING3
1822	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1823	if (fDriverInitiatedReset)
1824	virtioGuestR3WasReset(pDevIns, pVirtio, pVirtioCC);
1825
1826	else if (fDriverStateImproved)
1827	pVirtioCC->pfnStatusChanged(pVirtio, pVirtioCC, 1 /* fDriverOk */);
1828
1829	#endif
1830	pVirtio->fPrevDeviceStatus = pVirtio->fDeviceStatus;
1831	}
1832	else
1833	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uVirtqPfn, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1834	{
1835	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1836	uint64_t uVirtqPfn = (uint64_t)u32;
1837
1838	if (uVirtqPfn)
1839	{
1840	/* Transitional devices calculate ring physical addresses using rigid spec-defined formulae,
1841	* instead of guest conveying respective address of each ring, as "modern" VirtIO drivers do,
1842	* thus there is no virtq PFN or single base queue address stored in instance data for
1843	* this transitional device, but rather it is derived, when read back, from GCPhysVirtqDesc */
1844
1845	pVirtq->GCPhysVirtqDesc = uVirtqPfn * VIRTIO_PAGE_SIZE;
1846	pVirtq->GCPhysVirtqAvail = pVirtq->GCPhysVirtqDesc + sizeof(VIRTQ_DESC_T) * pVirtq->uQueueSize;
1847	pVirtq->GCPhysVirtqUsed =
1848	RT_ALIGN(pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[pVirtq->uQueueSize]), VIRTIO_PAGE_SIZE);
1849	}
1850	else
1851	{
1852	/* Don't set ring addresses for queue (to meaningless values), when guest resets the virtq's PFN */
1853	pVirtq->GCPhysVirtqDesc = 0;
1854	pVirtq->GCPhysVirtqAvail = 0;
1855	pVirtq->GCPhysVirtqUsed = 0;
1856	}
1857	Log(("%-23s: Guest wrote uVirtqPfn .................... %#x:\n"
1858	"%68s... %p -> GCPhysVirtqDesc\n%68s... %p -> GCPhysVirtqAvail\n%68s... %p -> GCPhysVirtqUsed\n",
1859	__FUNCTION__, u32, " ", pVirtq->GCPhysVirtqDesc, " ", pVirtq->GCPhysVirtqAvail, " ", pVirtq->GCPhysVirtqUsed));
1860	}
1861	else
1862	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uQueueNotify, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1863	{
1864	#ifdef IN_RING3
1865	ASSERT_GUEST_MSG(cb == 2, ("cb=%u\n", cb));
1866	pVirtio->uQueueNotify = u32 & 0xFFFF;
1867	if (uVirtq < VIRTQ_MAX_COUNT)
1868	{
1869	RT_UNTRUSTED_VALIDATED_FENCE();
1870
1871	/* Need to check that queue is configured. Legacy spec didn't have a queue enabled flag */
1872	if (pVirtio->aVirtqueues[pVirtio->uQueueNotify].GCPhysVirtqDesc)
1873	virtioCoreVirtqNotified(pDevIns, pVirtio, pVirtio->uQueueNotify, pVirtio->uQueueNotify /* uNotifyIdx */);
1874	else
1875	Log(("The queue (#%d) being notified has not been initialized.\n", pVirtio->uQueueNotify));
1876	}
1877	else
1878	Log(("Invalid queue number (%d)\n", pVirtio->uQueueNotify));
1879	#else
1880	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1881	return VINF_IOM_R3_IOPORT_WRITE;
1882	#endif
1883	}
1884	else
1885	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fIsrStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1886	{
1887	VIRTIO_DEV_CONFIG_LOG_ACCESS( fIsrStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1888	LogFunc(("... WARNING: Guest attempted to write readonly device_feature (ISR status) (ignoring)\n"));
1889	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1890	return VINF_SUCCESS;
1891	}
1892	else if (offPort >= sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T))
1893	{
1894	#ifdef IN_RING3
1895
1896	/* Access device-specific configuration */
1897	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1898	return pVirtioCC->pfnDevCapWrite(pDevIns, offPort - sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T), pv, cb);
1899	#else
1900	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1901	return VINF_IOM_R3_IOPORT_WRITE;
1902	#endif
1903	}
1904	else
1905	{
1906	Log2Func(("Bad guest write access to virtio_legacy_pci_common_cfg: offset=%#x, cb=0x%x\n",
1907	offPort, cb));
1908	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1909	int rc = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
1910	"virtioLegacyIOPortOut: no valid port at offset offset=%RTiop cb=0x%#x\n", offPort, cb);
1911	return rc;
1912	}
1913
1914	RT_NOREF(uVirtq);
1915	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1916	return VINF_SUCCESS;
1917	}
1918
1919
1920	/**
1921	* Read from the device specific configuration at the given offset.
1922	*
1923	* @returns VBox status code.
1924	* @param pDevIns
1925	*/
1926	DECLINLINE(VBOXSTRICTRC) virtioDeviceCfgRead(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC,
1927	uint32_t offDevCfg, void *pv, unsigned cb)
1928	{
1929	#ifdef IN_RING3
1930	/*
1931	* Callback to client to manage device-specific configuration.
1932	*/
1933	VBOXSTRICTRC rcStrict = pVirtioCC->pfnDevCapRead(pDevIns, offDevCfg, pv, cb);
1934
1935	/*
1936	* Anytime any part of the dev-specific dev config (which this virtio core implementation sees
1937	* as a blob, and virtio dev-specific code separates into fields) is READ, it must be compared
1938	* for deltas from previous read to maintain a config gen. seq. counter (VirtIO 1.0, section 4.1.4.3.1)
1939	*/
1940	bool fDevSpecificFieldChanged = RT_BOOL(memcmp(pVirtioCC->pbDevSpecificCfg + offDevCfg,
1941	pVirtioCC->pbPrevDevSpecificCfg + offDevCfg,
1942	RT_MIN(cb, pVirtioCC->cbDevSpecificCfg - offDevCfg)));
1943
1944	memcpy(pVirtioCC->pbPrevDevSpecificCfg, pVirtioCC->pbDevSpecificCfg, pVirtioCC->cbDevSpecificCfg);
1945
1946	if (pVirtio->fGenUpdatePending \|\| fDevSpecificFieldChanged)
1947	{
1948	++pVirtio->uConfigGeneration;
1949	Log6Func(("Bumped cfg. generation to %d because %s%s\n", pVirtio->uConfigGeneration,
1950	fDevSpecificFieldChanged ? "<dev cfg changed> " : "",
1951	pVirtio->fGenUpdatePending ? "<update was pending>" : ""));
1952	pVirtio->fGenUpdatePending = false;
1953	}
1954
1955	virtioLowerInterrupt(pDevIns, 0);
1956	return rcStrict;
1957	#else
1958	return VINF_IOM_R3_MMIO_READ;
1959	#endif
1960	}
1961
1962
1963	/**
1964	* @callback_method_impl{FNIOMMMIONEWREAD,
1965	* Memory mapped I/O Handler for PCI Capabilities read operations.}
1966	*
1967	* This MMIO handler specifically supports the VIRTIO_PCI_CAP_PCI_CFG capability defined
1968	* in the VirtIO 1.0 specification, section 4.1.4.7, and as such is restricted to reads
1969	* of 1, 2 or 4 bytes, only.
1970	*
1971	*/
1972	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioRead(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void pv, unsigned cb)
1973	{
1974	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1975	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1976	AssertReturn(cb == 1 \|\| cb == 2 \|\| cb == 4, VERR_INVALID_PARAMETER);
1977	Assert(pVirtio == (PVIRTIOCORE)pvUser); RT_NOREF(pvUser);
1978
1979	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatRead), a);
1980
1981	VBOXSTRICTRC rcStrict;
1982	uint32_t uOffset;
1983	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocDeviceCap))
1984	rcStrict = virtioDeviceCfgRead(pDevIns, pVirtio, pVirtioCC, uOffset, pv, cb);
1985	else if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocCommonCfgCap))
1986	rcStrict = virtioCommonCfgAccessed(pDevIns, pVirtio, pVirtioCC, false /* fWrite */, uOffset, cb, pv);
1987	else if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocIsrCap))
1988	{
1989	(uint8_t )pv = pVirtio->uISR;
1990	Log6Func(("Read and clear ISR\n"));
1991	pVirtio->uISR = 0; /* VirtIO spec requires reads of ISR to clear it */
1992	virtioLowerInterrupt(pDevIns, 0);
1993	rcStrict = VINF_SUCCESS;
1994	}
1995	else
1996	{
1997	ASSERT_GUEST_MSG_FAILED(("Bad read access to mapped capabilities region: off=%RGp cb=%u\n", off, cb));
1998	rcStrict = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
1999	"virtioMmioRead: Bad MMIO access to capabilities, offset=%RTiop cb=%08x\n", off, cb);
2000	}
2001
2002	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
2003	return rcStrict;
2004	}
2005
2006	/**
2007	* @callback_method_impl{FNIOMMMIONEWREAD,
2008	* Memory mapped I/O Handler for PCI Capabilities write operations.}
2009	*
2010	* This MMIO handler specifically supports the VIRTIO_PCI_CAP_PCI_CFG capability defined
2011	* in the VirtIO 1.0 specification, section 4.1.4.7, and as such is restricted to writes
2012	* of 1, 2 or 4 bytes, only.
2013	*/
2014	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioWrite(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void const pv, unsigned cb)
2015	{
2016	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
2017	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2018	AssertReturn(cb == 1 \|\| cb == 2 \|\| cb == 4, VERR_INVALID_PARAMETER);
2019	Assert(pVirtio == (PVIRTIOCORE)pvUser); RT_NOREF(pvUser);
2020	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatWrite), a);
2021
2022	VBOXSTRICTRC rcStrict;
2023	uint32_t uOffset;
2024	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocDeviceCap))
2025	{
2026	#ifdef IN_RING3
2027	/*
2028	* Foreward this MMIO write access for client to deal with.
2029	*/
2030	rcStrict = pVirtioCC->pfnDevCapWrite(pDevIns, uOffset, pv, cb);
2031	#else
2032	Log6(("%-23s: RING0 => RING3 (demote)\n", __FUNCTION__));
2033	rcStrict = VINF_IOM_R3_MMIO_WRITE;
2034	#endif
2035	}
2036	else if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocCommonCfgCap))
2037	rcStrict = virtioCommonCfgAccessed(pDevIns, pVirtio, pVirtioCC, true /* fWrite /, uOffset, cb, (void )pv);
2038	else if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocIsrCap) && cb == sizeof(uint8_t))
2039	{
2040	pVirtio->uISR = (uint8_t )pv;
2041	Log6Func(("Setting uISR = 0x%02x (virtq interrupt: %d, dev confg interrupt: %d)\n",
2042	pVirtio->uISR & 0xff,
2043	pVirtio->uISR & VIRTIO_ISR_VIRTQ_INTERRUPT,
2044	RT_BOOL(pVirtio->uISR & VIRTIO_ISR_DEVICE_CONFIG)));
2045	rcStrict = VINF_SUCCESS;
2046	}
2047	else if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocNotifyCap) && cb == sizeof(uint16_t))
2048	{
2049	/* This should be guest driver dropping index of a new descriptor in avail ring */
2050	virtioCoreVirtqNotified(pDevIns, pVirtio, uOffset / VIRTIO_NOTIFY_OFFSET_MULTIPLIER, (uint16_t )pv);
2051	rcStrict = VINF_SUCCESS;
2052	}
2053	else
2054	{
2055	ASSERT_GUEST_MSG_FAILED(("Bad write access to mapped capabilities region: off=%RGp pv=%#p{%.*Rhxs} cb=%u\n", off, pv, cb, pv, cb));
2056	rcStrict = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
2057	"virtioMmioRead: Bad MMIO access to capabilities, offset=%RTiop cb=%08x\n", off, cb);
2058	}
2059
2060	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
2061	return rcStrict;
2062	}
2063
2064
2065	/**
2066	* @callback_method_impl{FNIOMMMIONEWREAD,
2067	* Memory mapped I/O Handler for Virtio over MMIO read operations.}
2068	*
2069	*/
2070	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioTransportRead(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void pv, unsigned cb)
2071	{
2072	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
2073	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2074	RT_NOREF(pvUser);
2075	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatRead), a);
2076
2077	if (off >= VIRTIO_MMIO_SIZE)
2078	{
2079	VBOXSTRICTRC rcStrict = virtioDeviceCfgRead(pDevIns, pVirtio, pVirtioCC, (uint32_t)off - VIRTIO_MMIO_SIZE, pv, cb);
2080	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
2081	return rcStrict;
2082	}
2083
2084	/* All accesses below need to be aligned on a 32-bit boundary and must be 32-bit in size. */
2085	ASSERT_GUEST_MSG_RETURN(!(off & 0x3) && cb == sizeof(uint32_t),
2086	("Bad read access: off=%RGp pv=%#p{%.*Rhxs} cb=%u\n", off, pv, cb, pv, cb),
2087	VINF_IOM_MMIO_UNUSED_FF);
2088
2089	int rc = VINF_SUCCESS;
2090	uint32_t pu32 = (uint32_t )pv;
2091	switch (off)
2092	{
2093	case VIRTIO_MMIO_REG_MAGIC_OFF:
2094	*pu32 = RT_H2LE_U32(VIRTIO_MMIO_REG_MAGIC_VALUE);
2095	break;
2096	case VIRTIO_MMIO_REG_VERSION_OFF:
2097	*pu32 = RT_H2LE_U32(VIRTIO_MMIO_REG_VERSION_VALUE);
2098	break;
2099	case VIRTIO_MMIO_REG_DEVICEID_OFF:
2100	*pu32 = pVirtio->uDeviceType;
2101	break;
2102	case VIRTIO_MMIO_REG_VENDORID_OFF:
2103	*pu32 = RT_H2LE_U32(DEVICE_PCI_VENDOR_ID_VIRTIO);
2104	break;
2105	case VIRTIO_MMIO_REG_DEVICEFEAT_OFF:
2106	{
2107	switch (pVirtio->uDeviceFeaturesSelect)
2108	{
2109	case 0:
2110	*pu32 = pVirtio->uDeviceFeatures & UINT32_C(0xffffffff);
2111	break;
2112	case 1:
2113	*pu32 = pVirtio->uDeviceFeatures >> 32;
2114	break;
2115	default:
2116	LogFunc(("Guest read uDeviceFeatures with out of range selector (%#x), returning 0\n",
2117	pVirtio->uDeviceFeaturesSelect));
2118	rc = VINF_IOM_MMIO_UNUSED_00;
2119	}
2120	break;
2121	}
2122	case VIRTIO_MMIO_REG_QUEUENUMMAX_OFF:
2123	pu32 = VIRTQ_SIZE; /* @todo */
2124	break;
2125	case VIRTIO_MMIO_REG_QUEUERDY_OFF:
2126	{
2127	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2128	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2129	*pu32 = pVirtQueue->uEnable;
2130	break;
2131	}
2132	case VIRTIO_MMIO_REG_INTRSTATUS_OFF:
2133	*pu32 = pVirtio->uISR;
2134	break;
2135	case VIRTIO_MMIO_REG_DEVSTATUS_OFF:
2136	*pu32 = virtioDeviceStatusRead(pVirtio);
2137	break;
2138	case VIRTIO_MMIO_REG_CFGGEN_OFF:
2139	*pu32 = pVirtio->uConfigGeneration;
2140	break;
2141	default:
2142	ASSERT_GUEST_MSG_FAILED(("Bad read access to mapped capabilities region: off=%RGp cb=%u\n", off, cb));
2143	rc = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
2144	"virtioMmioTransportRead: Bad MMIO access to capabilities, offset=%RTiop cb=%08x\n", off, cb);
2145	}
2146
2147	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
2148	return rc;
2149	}
2150
2151	/**
2152	* @callback_method_impl{FNIOMMMIONEWREAD,
2153	* Memory mapped I/O Handler for Virtio over MMIO write operations.}
2154	*/
2155	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioTransportWrite(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void const pv, unsigned cb)
2156	{
2157	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
2158	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2159	RT_NOREF(pvUser);
2160	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatWrite), a);
2161
2162	if (off >= VIRTIO_MMIO_SIZE)
2163	{
2164	#ifdef IN_RING3
2165	/*
2166	* Forward this MMIO write access for client to deal with.
2167	*/
2168	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
2169	return pVirtioCC->pfnDevCapWrite(pDevIns, (uint32_t)off - VIRTIO_MMIO_SIZE, pv, cb);
2170	#else
2171	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
2172	Log6(("%-23s: RING0 => RING3 (demote)\n", __FUNCTION__));
2173	return VINF_IOM_R3_MMIO_WRITE;
2174	#endif
2175	}
2176
2177	/* All accesses below need to be aligned on a 32-bit boundary and must be 32-bit in size. */
2178	ASSERT_GUEST_MSG_RETURN(!(off & 0x3) && cb == sizeof(uint32_t),
2179	("Bad write access: off=%RGp pv=%#p{%.*Rhxs} cb=%u\n", off, pv, cb, pv, cb),
2180	VINF_SUCCESS);
2181
2182	int rc = VINF_SUCCESS;
2183	uint32_t const u32Val = (const uint32_t )pv;
2184	switch (off)
2185	{
2186	case VIRTIO_MMIO_REG_DEVICEFEATSEL_OFF:
2187	{
2188	pVirtio->uDeviceFeaturesSelect = u32Val;
2189	break;
2190	}
2191	case VIRTIO_MMIO_REG_DRIVERFEAT_OFF:
2192	{
2193	switch (pVirtio->uDriverFeaturesSelect)
2194	{
2195	case 0:
2196	pVirtio->uDriverFeatures = (pVirtio->uDriverFeatures & UINT64_C(0xffffffff00000000)) \| u32Val;
2197	pVirtio->fDriverFeaturesWritten \|= DRIVER_FEATURES_0_WRITTEN;
2198	LogFunc(("Set DRIVER_FEATURES_0_WRITTEN. pVirtio->fDriverFeaturesWritten=%d\n", pVirtio->fDriverFeaturesWritten));
2199	if ( (pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_0_AND_1_WRITTEN) == DRIVER_FEATURES_0_AND_1_WRITTEN
2200	&& !(pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_COMPLETE_HANDLED))
2201	#ifdef IN_RING0
2202	return VINF_IOM_R3_MMIO_WRITE;
2203	#endif
2204	#ifdef IN_RING3
2205	virtioR3DoFeaturesCompleteOnceOnly(pVirtio, pVirtioCC);
2206	#endif
2207	break;
2208	case 1:
2209	pVirtio->uDriverFeatures = (pVirtio->uDriverFeatures & UINT64_C(0x00000000ffffffff)) \| ((uint64_t)u32Val << 32);
2210	pVirtio->fDriverFeaturesWritten \|= DRIVER_FEATURES_1_WRITTEN;
2211	LogFunc(("Set DRIVER_FEATURES_1_WRITTEN. pVirtio->fDriverFeaturesWritten=%d\n", pVirtio->fDriverFeaturesWritten));
2212	if ( (pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_0_AND_1_WRITTEN) == DRIVER_FEATURES_0_AND_1_WRITTEN
2213	&& !(pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_COMPLETE_HANDLED))
2214	#ifdef IN_RING0
2215	return VINF_IOM_R3_MMIO_WRITE;
2216	#endif
2217	#ifdef IN_RING3
2218	virtioR3DoFeaturesCompleteOnceOnly(pVirtio, pVirtioCC);
2219	#endif
2220	break;
2221	default:
2222	LogFunc(("Guest wrote uDriverFeatures with out of range selector (%#x), returning 0\n",
2223	pVirtio->uDriverFeaturesSelect));
2224	return VINF_SUCCESS;
2225	}
2226	break;
2227	}
2228	case VIRTIO_MMIO_REG_DRIVERFEATSEL_OFF:
2229	{
2230	pVirtio->uDriverFeaturesSelect = u32Val;
2231	break;
2232	}
2233	case VIRTIO_MMIO_REG_QUEUESEL_OFF:
2234	{
2235	if (u32Val < RT_ELEMENTS(pVirtio->aVirtqueues))
2236	pVirtio->uVirtqSelect = (uint16_t)u32Val;
2237	else
2238	LogFunc(("... WARNING: Guest attempted to write invalid virtq selector (ignoring)\n"));
2239	break;
2240	}
2241	case VIRTIO_MMIO_REG_QUEUENUM_OFF:
2242	{
2243	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2244	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2245	pVirtQueue->uQueueSize = (uint16_t)u32Val;
2246	break;
2247	}
2248	case VIRTIO_MMIO_REG_QUEUERDY_OFF:
2249	{
2250	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2251	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2252	pVirtQueue->uEnable = (uint16_t)u32Val;
2253	break;
2254	}
2255	case VIRTIO_MMIO_REG_QUEUENOTIFY_OFF:
2256	{
2257	virtioCoreVirtqNotified(pDevIns, pVirtio, u32Val, (uint16_t)u32Val);
2258	break;
2259	}
2260	case VIRTIO_MMIO_REG_INTRACK_OFF:
2261	{
2262	pVirtio->uISR &= ~u32Val;
2263	if (!pVirtio->uISR)
2264	virtioLowerInterrupt(pDevIns, 0);
2265	break;
2266	}
2267	case VIRTIO_MMIO_REG_DEVSTATUS_OFF:
2268	{
2269	rc = virtioDeviceStatusWrite(pDevIns, pVirtio, pVirtioCC, (uint8_t)u32Val);
2270	break;
2271	}
2272	case VIRTIO_MMIO_REG_QUEUEALIGN_LEGACY_OFF:
2273	{
2274	/* Written by edk2 even though we don't offer legacy mode, ignore. */
2275	break;
2276	}
2277	case VIRTIO_MMIO_REG_QUEUEDESCLOW_OFF:
2278	{
2279	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2280	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2281	pVirtQueue->GCPhysVirtqDesc = (pVirtQueue->GCPhysVirtqDesc & UINT64_C(0xffffffff00000000)) \| u32Val;
2282	break;
2283	}
2284	case VIRTIO_MMIO_REG_QUEUEDESCHIGH_OFF:
2285	{
2286	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2287	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2288	pVirtQueue->GCPhysVirtqDesc = (pVirtQueue->GCPhysVirtqDesc & UINT64_C(0x00000000ffffffff)) \| ((uint64_t)u32Val << 32);
2289	break;
2290	}
2291	case VIRTIO_MMIO_REG_QUEUEDRVLOW_OFF:
2292	{
2293	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2294	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2295	pVirtQueue->GCPhysVirtqAvail = (pVirtQueue->GCPhysVirtqAvail & UINT64_C(0xffffffff00000000)) \| u32Val;
2296	break;
2297	}
2298	case VIRTIO_MMIO_REG_QUEUEDRVHIGH_OFF:
2299	{
2300	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2301	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2302	pVirtQueue->GCPhysVirtqAvail = (pVirtQueue->GCPhysVirtqAvail & UINT64_C(0x00000000ffffffff)) \| ((uint64_t)u32Val << 32);
2303	break;
2304	}
2305	case VIRTIO_MMIO_REG_QUEUEDEVLOW_OFF:
2306	{
2307	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2308	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2309	pVirtQueue->GCPhysVirtqUsed = (pVirtQueue->GCPhysVirtqUsed & UINT64_C(0xffffffff00000000)) \| u32Val;
2310	break;
2311	}
2312	case VIRTIO_MMIO_REG_QUEUEDEVHIGH_OFF:
2313	{
2314	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2315	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2316	pVirtQueue->GCPhysVirtqUsed = (pVirtQueue->GCPhysVirtqUsed & UINT64_C(0x00000000ffffffff)) \| ((uint64_t)u32Val << 32);
2317	break;
2318	}
2319	default:
2320	ASSERT_GUEST_MSG_FAILED(("Bad write access to mapped capabilities region: off=%RGp pv=%#p{%.*Rhxs} cb=%u\n", off, pv, cb, pv, cb));
2321	rc = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
2322	"virtioMmioTransportWrite: Bad MMIO access to capabilities, offset=%RTiop cb=%08x\n", off, cb);
2323	}
2324
2325	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
2326	return rc;
2327	}
2328
2329
2330	#ifdef IN_RING3
2331
2332	/**
2333	* @callback_method_impl{FNPCICONFIGREAD}
2334	*/
2335	static DECLCALLBACK(VBOXSTRICTRC) virtioR3PciConfigRead(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev,
2336	uint32_t uAddress, unsigned cb, uint32_t *pu32Value)
2337	{
2338	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
2339	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2340	RT_NOREF(pPciDev);
2341
2342	if (uAddress == pVirtio->uPciCfgDataOff)
2343	{
2344	/* See comments in PCI Cfg capability initialization (in capabilities setup section of this code) */
2345	struct virtio_pci_cap *pPciCap = &pVirtioCC->pPciCfgCap->pciCap;
2346	uint32_t uLength = pPciCap->uLength;
2347
2348	Log7Func((" pDevIns=%p pPciDev=%p uAddress=%#x%s cb=%u uLength=%d, bar=%d\n",
2349	pDevIns, pPciDev, uAddress, uAddress < 0x10 ? " " : "", cb, uLength, pPciCap->uBar));
2350
2351	if ( (uLength != 1 && uLength != 2 && uLength != 4)
2352	\|\| pPciCap->uBar != VIRTIO_REGION_PCI_CAP)
2353	{
2354	ASSERT_GUEST_MSG_FAILED(("Guest read virtio_pci_cfg_cap.pci_cfg_data using mismatching config. "
2355	"Ignoring\n"));
2356	*pu32Value = UINT32_MAX;
2357	return VINF_SUCCESS;
2358	}
2359
2360	VBOXSTRICTRC rcStrict = virtioMmioRead(pDevIns, pVirtio, pPciCap->uOffset, pu32Value, cb);
2361	Log7Func((" Guest read virtio_pci_cfg_cap.pci_cfg_data, bar=%d, offset=%d, length=%d, result=0x%x -> %Rrc\n",
2362	pPciCap->uBar, pPciCap->uOffset, uLength, *pu32Value, VBOXSTRICTRC_VAL(rcStrict)));
2363	return rcStrict;
2364	}
2365	Log7Func((" pDevIns=%p pPciDev=%p uAddress=%#x%s cb=%u pu32Value=%p\n",
2366	pDevIns, pPciDev, uAddress, uAddress < 0x10 ? " " : "", cb, pu32Value));
2367	return VINF_PDM_PCI_DO_DEFAULT;
2368	}
2369
2370	/**
2371	* @callback_method_impl{FNPCICONFIGWRITE}
2372	*/
2373	static DECLCALLBACK(VBOXSTRICTRC) virtioR3PciConfigWrite(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev,
2374	uint32_t uAddress, unsigned cb, uint32_t u32Value)
2375	{
2376	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
2377	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2378	RT_NOREF(pPciDev);
2379
2380	Log7Func(("pDevIns=%p pPciDev=%p uAddress=%#x %scb=%u u32Value=%#x\n", pDevIns, pPciDev, uAddress, uAddress < 0xf ? " " : "", cb, u32Value));
2381	if (uAddress == pVirtio->uPciCfgDataOff)
2382	{
2383	/* See comments in PCI Cfg capability initialization (in capabilities setup section of this code) */
2384	struct virtio_pci_cap *pPciCap = &pVirtioCC->pPciCfgCap->pciCap;
2385	uint32_t uLength = pPciCap->uLength;
2386
2387	if ( (uLength != 1 && uLength != 2 && uLength != 4)
2388	\|\| cb != uLength
2389	\|\| pPciCap->uBar != VIRTIO_REGION_PCI_CAP)
2390	{
2391	ASSERT_GUEST_MSG_FAILED(("Guest write virtio_pci_cfg_cap.pci_cfg_data using mismatching config. Ignoring\n"));
2392	return VINF_SUCCESS;
2393	}
2394
2395	VBOXSTRICTRC rcStrict = virtioMmioWrite(pDevIns, pVirtio, pPciCap->uOffset, &u32Value, cb);
2396	Log2Func(("Guest wrote virtio_pci_cfg_cap.pci_cfg_data, bar=%d, offset=%x, length=%x, value=%d -> %Rrc\n",
2397	pPciCap->uBar, pPciCap->uOffset, uLength, u32Value, VBOXSTRICTRC_VAL(rcStrict)));
2398	return rcStrict;
2399	}
2400	return VINF_PDM_PCI_DO_DEFAULT;
2401	}
2402
2403
2404	/*********************************************************************************************************************************
2405	* Saved state (SSM) *
2406	*********************************************************************************************************************************/
2407
2408
2409	/**
2410	* Loads a saved device state (called from device-specific code on SSM final pass)
2411	*
2412	* @param pVirtio Pointer to the shared virtio state.
2413	* @param pHlp The ring-3 device helpers.
2414	* @param pSSM The saved state handle.
2415	* @returns VBox status code.
2416	*/
2417	DECLHIDDEN(int) virtioCoreR3LegacyDeviceLoadExec(PVIRTIOCORE pVirtio, PCPDMDEVHLPR3 pHlp, PSSMHANDLE pSSM,
2418	uint32_t uVersion, uint32_t uVirtioLegacy_3_1_Beta)
2419	{
2420	int rc;
2421	uint32_t uDriverFeaturesLegacy32bit;
2422
2423	rc = pHlp->pfnSSMGetU32( pSSM, &uDriverFeaturesLegacy32bit);
2424	AssertRCReturn(rc, rc);
2425	pVirtio->uDriverFeatures = (uint64_t)uDriverFeaturesLegacy32bit;
2426
2427	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtio->uVirtqSelect);
2428	AssertRCReturn(rc, rc);
2429
2430	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->fDeviceStatus);
2431	AssertRCReturn(rc, rc);
2432
2433	#ifdef LOG_ENABLED
2434	char szOut[80] = { 0 };
2435	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
2436	Log(("Loaded legacy device status = (%s)\n", szOut));
2437	#endif
2438
2439	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->uISR);
2440	AssertRCReturn(rc, rc);
2441
2442	uint32_t cQueues = 3; /* This constant default value copied from earliest v0.9 code */
2443	if (uVersion > uVirtioLegacy_3_1_Beta)
2444	{
2445	rc = pHlp->pfnSSMGetU32(pSSM, &cQueues);
2446	AssertRCReturn(rc, rc);
2447	}
2448
2449	AssertLogRelMsgReturn(cQueues <= VIRTQ_MAX_COUNT, ("%#x\n", cQueues), VERR_SSM_LOAD_CONFIG_MISMATCH);
2450	AssertLogRelMsgReturn(pVirtio->uVirtqSelect < cQueues \|\| (cQueues == 0 && pVirtio->uVirtqSelect),
2451	("uVirtqSelect=%u cQueues=%u\n", pVirtio->uVirtqSelect, cQueues),
2452	VERR_SSM_LOAD_CONFIG_MISMATCH);
2453
2454	Log(("\nRestoring %d legacy-only virtio-net device queues from saved state:\n", cQueues));
2455	for (unsigned uVirtq = 0; uVirtq < cQueues; uVirtq++)
2456	{
2457	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
2458
2459	if (uVirtq == cQueues - 1)
2460	RTStrPrintf(pVirtq->szName, sizeof(pVirtq->szName), "legacy-ctrlq");
2461	else if (uVirtq % 2)
2462	RTStrPrintf(pVirtq->szName, sizeof(pVirtq->szName), "legacy-xmitq<%d>", uVirtq / 2);
2463	else
2464	RTStrPrintf(pVirtq->szName, sizeof(pVirtq->szName), "legacy-recvq<%d>", uVirtq / 2);
2465
2466	rc = pHlp->pfnSSMGetU16(pSSM, &pVirtq->uQueueSize);
2467	AssertRCReturn(rc, rc);
2468
2469	uint32_t uVirtqPfn;
2470	rc = pHlp->pfnSSMGetU32(pSSM, &uVirtqPfn);
2471	AssertRCReturn(rc, rc);
2472
2473	rc = pHlp->pfnSSMGetU16(pSSM, &pVirtq->uAvailIdxShadow);
2474	AssertRCReturn(rc, rc);
2475
2476	rc = pHlp->pfnSSMGetU16(pSSM, &pVirtq->uUsedIdxShadow);
2477	AssertRCReturn(rc, rc);
2478
2479	if (uVirtqPfn)
2480	{
2481	pVirtq->GCPhysVirtqDesc = (uint64_t)uVirtqPfn * VIRTIO_PAGE_SIZE;
2482	pVirtq->GCPhysVirtqAvail = pVirtq->GCPhysVirtqDesc + sizeof(VIRTQ_DESC_T) * pVirtq->uQueueSize;
2483	pVirtq->GCPhysVirtqUsed =
2484	RT_ALIGN(pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[pVirtq->uQueueSize]), VIRTIO_PAGE_SIZE);
2485	pVirtq->uEnable = 1;
2486	}
2487	else
2488	{
2489	LogFunc(("WARNING: QUEUE \"%s\" PAGE NUMBER ZERO IN SAVED STATE\n", pVirtq->szName));
2490	pVirtq->uEnable = 0;
2491	}
2492	pVirtq->uNotifyOffset = 0; /* unused in legacy mode */
2493	pVirtq->uMsixVector = 0; /* unused in legacy mode */
2494	}
2495	pVirtio->fGenUpdatePending = 0; /* unused in legacy mode */
2496	pVirtio->uConfigGeneration = 0; /* unused in legacy mode */
2497	pVirtio->uPciCfgDataOff = 0; /* unused in legacy mode (port I/O used instead) */
2498
2499	return VINF_SUCCESS;
2500	}
2501
2502	/**
2503	* Loads a saved device state (called from device-specific code on SSM final pass)
2504	*
2505	* Note: This loads state saved by a Modern (VirtIO 1.0+) device, of which this transitional device is one,
2506	* and thus supports both legacy and modern guest virtio drivers.
2507	*
2508	* @param pVirtio Pointer to the shared virtio state.
2509	* @param pHlp The ring-3 device helpers.
2510	* @param pSSM The saved state handle.
2511	* @returns VBox status code.
2512	*/
2513	DECLHIDDEN(int) virtioCoreR3ModernDeviceLoadExec(PVIRTIOCORE pVirtio, PCPDMDEVHLPR3 pHlp, PSSMHANDLE pSSM,
2514	uint32_t uVersion, uint32_t uTestVersion, uint32_t cQueues)
2515	{
2516	RT_NOREF2(cQueues, uVersion);
2517	LogFunc(("\n"));
2518	/*
2519	* Check the marker and (embedded) version number.
2520	*/
2521	uint64_t uMarker = 0;
2522	int rc;
2523
2524	rc = pHlp->pfnSSMGetU64(pSSM, &uMarker);
2525	AssertRCReturn(rc, rc);
2526	if (uMarker != VIRTIO_SAVEDSTATE_MARKER)
2527	return pHlp->pfnSSMSetLoadError(pSSM, VERR_SSM_DATA_UNIT_FORMAT_CHANGED, RT_SRC_POS,
2528	N_("Expected marker value %#RX64 found %#RX64 instead"),
2529	VIRTIO_SAVEDSTATE_MARKER, uMarker);
2530	uint32_t uVersionSaved = 0;
2531	rc = pHlp->pfnSSMGetU32(pSSM, &uVersionSaved);
2532	AssertRCReturn(rc, rc);
2533	if (uVersionSaved != uTestVersion)
2534	return pHlp->pfnSSMSetLoadError(pSSM, VERR_SSM_DATA_UNIT_FORMAT_CHANGED, RT_SRC_POS,
2535	N_("Unsupported virtio version: %u"), uVersionSaved);
2536	/*
2537	* Load the state.
2538	*/
2539	rc = pHlp->pfnSSMGetU32( pSSM, &pVirtio->fLegacyDriver);
2540	AssertRCReturn(rc, rc);
2541	rc = pHlp->pfnSSMGetBool( pSSM, &pVirtio->fGenUpdatePending);
2542	AssertRCReturn(rc, rc);
2543	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->fDeviceStatus);
2544	AssertRCReturn(rc, rc);
2545	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->uConfigGeneration);
2546	AssertRCReturn(rc, rc);
2547	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->uPciCfgDataOff);
2548	AssertRCReturn(rc, rc);
2549	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->uISR);
2550	AssertRCReturn(rc, rc);
2551	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtio->uVirtqSelect);
2552	AssertRCReturn(rc, rc);
2553	rc = pHlp->pfnSSMGetU32( pSSM, &pVirtio->uDeviceFeaturesSelect);
2554	AssertRCReturn(rc, rc);
2555	rc = pHlp->pfnSSMGetU32( pSSM, &pVirtio->uDriverFeaturesSelect);
2556	AssertRCReturn(rc, rc);
2557	rc = pHlp->pfnSSMGetU64( pSSM, &pVirtio->uDriverFeatures);
2558	AssertRCReturn(rc, rc);
2559
2560	/** @todo Adapt this loop use cQueues argument instead of static queue count (safely with SSM versioning) */
2561	for (uint32_t i = 0; i < VIRTQ_MAX_COUNT; i++)
2562	{
2563	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[i];
2564	rc = pHlp->pfnSSMGetGCPhys64( pSSM, &pVirtq->GCPhysVirtqDesc);
2565	AssertRCReturn(rc, rc);
2566	rc = pHlp->pfnSSMGetGCPhys64( pSSM, &pVirtq->GCPhysVirtqAvail);
2567	AssertRCReturn(rc, rc);
2568	rc = pHlp->pfnSSMGetGCPhys64( pSSM, &pVirtq->GCPhysVirtqUsed);
2569	AssertRCReturn(rc, rc);
2570	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uNotifyOffset);
2571	AssertRCReturn(rc, rc);
2572	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uMsixVector);
2573	AssertRCReturn(rc, rc);
2574	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uEnable);
2575	AssertRCReturn(rc, rc);
2576	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uQueueSize);
2577	AssertRCReturn(rc, rc);
2578	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uAvailIdxShadow);
2579	AssertRCReturn(rc, rc);
2580	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uUsedIdxShadow);
2581	AssertRCReturn(rc, rc);
2582	rc = pHlp->pfnSSMGetMem( pSSM, pVirtq->szName, sizeof(pVirtq->szName));
2583	AssertRCReturn(rc, rc);
2584	}
2585	return VINF_SUCCESS;
2586	}
2587
2588	/**
2589	* Called from the FNSSMDEVSAVEEXEC function of the device.
2590	*
2591	* @param pVirtio Pointer to the shared virtio state.
2592	* @param pHlp The ring-3 device helpers.
2593	* @param pSSM The saved state handle.
2594	* @returns VBox status code.
2595	*/
2596	DECLHIDDEN(int) virtioCoreR3SaveExec(PVIRTIOCORE pVirtio, PCPDMDEVHLPR3 pHlp, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t cQueues)
2597	{
2598	RT_NOREF(cQueues);
2599	/** @todo figure out a way to save cQueues (with SSM versioning) */
2600
2601	LogFunc(("\n"));
2602	pHlp->pfnSSMPutU64(pSSM, VIRTIO_SAVEDSTATE_MARKER);
2603	pHlp->pfnSSMPutU32(pSSM, uVersion);
2604
2605	pHlp->pfnSSMPutU32( pSSM, pVirtio->fLegacyDriver);
2606	pHlp->pfnSSMPutBool(pSSM, pVirtio->fGenUpdatePending);
2607	pHlp->pfnSSMPutU8( pSSM, pVirtio->fDeviceStatus);
2608	pHlp->pfnSSMPutU8( pSSM, pVirtio->uConfigGeneration);
2609	pHlp->pfnSSMPutU8( pSSM, pVirtio->uPciCfgDataOff);
2610	pHlp->pfnSSMPutU8( pSSM, pVirtio->uISR);
2611	pHlp->pfnSSMPutU16( pSSM, pVirtio->uVirtqSelect);
2612	pHlp->pfnSSMPutU32( pSSM, pVirtio->uDeviceFeaturesSelect);
2613	pHlp->pfnSSMPutU32( pSSM, pVirtio->uDriverFeaturesSelect);
2614	pHlp->pfnSSMPutU64( pSSM, pVirtio->uDriverFeatures);
2615
2616	for (uint32_t i = 0; i < VIRTQ_MAX_COUNT; i++)
2617	{
2618	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[i];
2619
2620	pHlp->pfnSSMPutGCPhys64( pSSM, pVirtq->GCPhysVirtqDesc);
2621	pHlp->pfnSSMPutGCPhys64( pSSM, pVirtq->GCPhysVirtqAvail);
2622	pHlp->pfnSSMPutGCPhys64( pSSM, pVirtq->GCPhysVirtqUsed);
2623	pHlp->pfnSSMPutU16( pSSM, pVirtq->uNotifyOffset);
2624	pHlp->pfnSSMPutU16( pSSM, pVirtq->uMsixVector);
2625	pHlp->pfnSSMPutU16( pSSM, pVirtq->uEnable);
2626	pHlp->pfnSSMPutU16( pSSM, pVirtq->uQueueSize);
2627	pHlp->pfnSSMPutU16( pSSM, pVirtq->uAvailIdxShadow);
2628	pHlp->pfnSSMPutU16( pSSM, pVirtq->uUsedIdxShadow);
2629	int rc = pHlp->pfnSSMPutMem(pSSM, pVirtq->szName, 32);
2630	AssertRCReturn(rc, rc);
2631	}
2632	return VINF_SUCCESS;
2633	}
2634
2635
2636	/*********************************************************************************************************************************
2637	* Device Level *
2638	*********************************************************************************************************************************/
2639
2640	/**
2641	* This must be called by the client to handle VM state changes after the client takes care of its device-specific
2642	* tasks for the state change (i.e. reset, suspend, power-off, resume)
2643	*
2644	* @param pDevIns The device instance.
2645	* @param pVirtio Pointer to the shared virtio state.
2646	*/
2647	DECLHIDDEN(void) virtioCoreR3VmStateChanged(PVIRTIOCORE pVirtio, VIRTIOVMSTATECHANGED enmState)
2648	{
2649	LogFunc(("State changing to %s\n",
2650	virtioCoreGetStateChangeText(enmState)));
2651
2652	switch(enmState)
2653	{
2654	case kvirtIoVmStateChangedReset:
2655	virtioCoreResetAll(pVirtio);
2656	break;
2657	case kvirtIoVmStateChangedSuspend:
2658	break;
2659	case kvirtIoVmStateChangedPowerOff:
2660	break;
2661	case kvirtIoVmStateChangedResume:
2662	for (int uVirtq = 0; uVirtq < VIRTQ_MAX_COUNT; uVirtq++)
2663	{
2664	if ((!pVirtio->fLegacyDriver && pVirtio->aVirtqueues[uVirtq].uEnable)
2665	\| pVirtio->aVirtqueues[uVirtq].GCPhysVirtqDesc)
2666	virtioCoreNotifyGuestDriver(pVirtio->pDevInsR3, pVirtio, uVirtq);
2667	}
2668	break;
2669	default:
2670	LogRelFunc(("Bad enum value"));
2671	return;
2672	}
2673	}
2674
2675	/**
2676	* This should be called from PDMDEVREGR3::pfnDestruct.
2677	*
2678	* @param pDevIns The device instance.
2679	* @param pVirtio Pointer to the shared virtio state.
2680	* @param pVirtioCC Pointer to the ring-3 virtio state.
2681	*/
2682	DECLHIDDEN(void) virtioCoreR3Term(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
2683	{
2684	if (pVirtioCC->pbPrevDevSpecificCfg)
2685	{
2686	RTMemFree(pVirtioCC->pbPrevDevSpecificCfg);
2687	pVirtioCC->pbPrevDevSpecificCfg = NULL;
2688	}
2689
2690	RT_NOREF(pDevIns, pVirtio);
2691	}
2692
2693
2694	/**
2695	* Setup the Virtio device as a PCI device.
2696	*
2697	* @returns VBox status code.
2698	* @param pDevIns Device instance.
2699	* @param pVirtio Pointer to the shared virtio state. This
2700	* must be the first member in the shared
2701	* device instance data!
2702	* @param pVirtioCC Pointer to the ring-3 virtio state. This
2703	* must be the first member in the ring-3
2704	* device instance data!
2705	* @param pPciParams Values to populate industry standard PCI Configuration Space data structure
2706	* @param pcszInstance Device instance name (format-specifier)
2707	* @param cbDevSpecificCfg Size of virtio_pci_device_cap device-specific struct
2708	*/
2709	static int virtioR3PciTransportInit(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC, PVIRTIOPCIPARAMS pPciParams,
2710	const char *pcszInstance, uint16_t cbDevSpecificCfg)
2711	{
2712	/* Set PCI config registers (assume 32-bit mode) */
2713	PPDMPCIDEV pPciDev = pDevIns->apPciDevs[0];
2714	PDMPCIDEV_ASSERT_VALID(pDevIns, pPciDev);
2715
2716	PDMPciDevSetVendorId(pPciDev, DEVICE_PCI_VENDOR_ID_VIRTIO);
2717	PDMPciDevSetDeviceId(pPciDev, pPciParams->uDeviceId);
2718
2719	if (pPciParams->uDeviceId < DEVICE_PCI_DEVICE_ID_VIRTIO_BASE)
2720	/* Transitional devices MUST have a PCI Revision ID of 0. */
2721	PDMPciDevSetRevisionId(pPciDev, DEVICE_PCI_REVISION_ID_VIRTIO_TRANS);
2722	else
2723	/* Non-transitional devices SHOULD have a PCI Revision ID of 1 or higher. */
2724	PDMPciDevSetRevisionId(pPciDev, DEVICE_PCI_REVISION_ID_VIRTIO_V1);
2725
2726	PDMPciDevSetSubSystemId(pPciDev, pPciParams->uSubsystemId);
2727	PDMPciDevSetSubSystemVendorId(pPciDev, DEVICE_PCI_VENDOR_ID_VIRTIO);
2728	PDMPciDevSetClassBase(pPciDev, pPciParams->uClassBase);
2729	PDMPciDevSetClassSub(pPciDev, pPciParams->uClassSub);
2730	PDMPciDevSetClassProg(pPciDev, pPciParams->uClassProg);
2731	PDMPciDevSetInterruptLine(pPciDev, pPciParams->uInterruptLine);
2732	PDMPciDevSetInterruptPin(pPciDev, pPciParams->uInterruptPin);
2733
2734	/* Register PCI device */
2735	int rc = PDMDevHlpPCIRegister(pDevIns, pPciDev);
2736	if (RT_FAILURE(rc))
2737	return PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register PCI Device")); /* can we put params in this error? */
2738
2739	rc = PDMDevHlpPCIInterceptConfigAccesses(pDevIns, pPciDev, virtioR3PciConfigRead, virtioR3PciConfigWrite);
2740	AssertRCReturn(rc, rc);
2741
2742	/* Construct & map PCI vendor-specific capabilities for virtio host negotiation with guest driver */
2743
2744	#define CFG_ADDR_2_IDX(addr) ((uint8_t)(((uintptr_t)(addr) - (uintptr_t)&pPciDev->abConfig[0])))
2745	#define SET_PCI_CAP_LOC(a_pPciDev, a_pCfg, a_LocCap, a_uMmioLengthAlign) \
2746	do { \
2747	(a_LocCap).offMmio = (a_pCfg)->uOffset; \
2748	(a_LocCap).cbMmio = RT_ALIGN_T((a_pCfg)->uLength, a_uMmioLengthAlign, uint16_t); \
2749	(a_LocCap).offPci = (uint16_t)(uintptr_t)((uint8_t *)(a_pCfg) - &(a_pPciDev)->abConfig[0]); \
2750	(a_LocCap).cbPci = (a_pCfg)->uCapLen; \
2751	} while (0)
2752
2753	PVIRTIO_PCI_CAP_T pCfg;
2754	uint32_t cbRegion = 0;
2755
2756	/*
2757	* Common capability (VirtIO 1.0, section 4.1.4.3)
2758	*/
2759	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[0x40];
2760	pCfg->uCfgType = VIRTIO_PCI_CAP_COMMON_CFG;
2761	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2762	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
2763	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
2764	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2765	pCfg->uOffset = RT_ALIGN_32(0, 4); /* Currently 0, but reminder to 32-bit align if changing this */
2766	pCfg->uLength = sizeof(VIRTIO_PCI_COMMON_CFG_T);
2767	cbRegion += pCfg->uLength;
2768	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocCommonCfgCap, 2);
2769	pVirtioCC->pCommonCfgCap = pCfg;
2770
2771	/*
2772	* Notify capability (VirtIO 1.0, section 4.1.4.4).
2773	*
2774	* The size of the spec-defined subregion described by this VirtIO capability is
2775	* based-on the choice of this implementation to make the notification area of each
2776	* queue equal to queue's ordinal position (e.g. queue selector value). The VirtIO
2777	* specification leaves it up to implementation to define queue notification area layout.
2778	*/
2779	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2780	pCfg->uCfgType = VIRTIO_PCI_CAP_NOTIFY_CFG;
2781	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2782	pCfg->uCapLen = sizeof(VIRTIO_PCI_NOTIFY_CAP_T);
2783	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
2784	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2785	pCfg->uOffset = pVirtioCC->pCommonCfgCap->uOffset + pVirtioCC->pCommonCfgCap->uLength;
2786	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
2787	pCfg->uLength = VIRTQ_MAX_COUNT * VIRTIO_NOTIFY_OFFSET_MULTIPLIER + 2; /* will change in VirtIO 1.1 */
2788	cbRegion += pCfg->uLength;
2789	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocNotifyCap, 1);
2790	pVirtioCC->pNotifyCap = (PVIRTIO_PCI_NOTIFY_CAP_T)pCfg;
2791	pVirtioCC->pNotifyCap->uNotifyOffMultiplier = VIRTIO_NOTIFY_OFFSET_MULTIPLIER;
2792
2793	/* ISR capability (VirtIO 1.0, section 4.1.4.5)
2794	*
2795	* VirtIO 1.0 spec says 8-bit, unaligned in MMIO space. The specification example/diagram
2796	* illustrates this capability as 32-bit field with upper bits 'reserved'. Those depictions
2797	* differ. The spec's wording, not the diagram, is seen to work in practice.
2798	*/
2799	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2800	pCfg->uCfgType = VIRTIO_PCI_CAP_ISR_CFG;
2801	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2802	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
2803	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
2804	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2805	pCfg->uOffset = pVirtioCC->pNotifyCap->pciCap.uOffset + pVirtioCC->pNotifyCap->pciCap.uLength;
2806	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
2807	pCfg->uLength = sizeof(uint8_t);
2808	cbRegion += pCfg->uLength;
2809	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocIsrCap, 4);
2810	pVirtioCC->pIsrCap = pCfg;
2811
2812	/* PCI Cfg capability (VirtIO 1.0, section 4.1.4.7)
2813	*
2814	* This capability facilitates early-boot access to this device (BIOS).
2815	* This region isn't page-MMIO mapped. PCI configuration accesses are intercepted,
2816	* wherein uBar, uOffset and uLength are modulated by consumers to locate and read/write
2817	* values in any part of any region. (NOTE: Linux driver doesn't utilize this feature.
2818	* This capability only appears in lspci output on Linux if uLength is non-zero, 4-byte aligned,
2819	* during initialization of linux virtio driver).
2820	*/
2821	pVirtio->uPciCfgDataOff = pCfg->uCapNext + RT_OFFSETOF(VIRTIO_PCI_CFG_CAP_T, uPciCfgData);
2822	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2823	pCfg->uCfgType = VIRTIO_PCI_CAP_PCI_CFG;
2824	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2825	pCfg->uCapLen = sizeof(VIRTIO_PCI_CFG_CAP_T);
2826	pCfg->uCapNext = (pVirtio->fMsiSupport \|\| pVirtioCC->pbDevSpecificCfg) ? CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen : 0;
2827	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2828	pCfg->uOffset = 0;
2829	pCfg->uLength = 4;
2830	cbRegion += pCfg->uLength;
2831	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocPciCfgCap, 1);
2832	pVirtioCC->pPciCfgCap = (PVIRTIO_PCI_CFG_CAP_T)pCfg;
2833
2834	if (pVirtioCC->pbDevSpecificCfg)
2835	{
2836	/* Device-specific config capability (VirtIO 1.0, section 4.1.4.6).
2837	*
2838	* Client defines the device-specific config struct and passes size to virtioCoreR3Init()
2839	* to inform this.
2840	*/
2841	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2842	pCfg->uCfgType = VIRTIO_PCI_CAP_DEVICE_CFG;
2843	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2844	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
2845	pCfg->uCapNext = pVirtio->fMsiSupport ? CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen : 0;
2846	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2847	pCfg->uOffset = pVirtioCC->pIsrCap->uOffset + pVirtioCC->pIsrCap->uLength;
2848	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
2849	pCfg->uLength = cbDevSpecificCfg;
2850	cbRegion += pCfg->uLength;
2851	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocDeviceCap, 4);
2852	pVirtioCC->pDeviceCap = pCfg;
2853	}
2854	else
2855	Assert(pVirtio->LocDeviceCap.cbMmio == 0 && pVirtio->LocDeviceCap.cbPci == 0);
2856
2857	if (pVirtio->fMsiSupport)
2858	{
2859	PDMMSIREG aMsiReg;
2860	RT_ZERO(aMsiReg);
2861	aMsiReg.iMsixCapOffset = pCfg->uCapNext;
2862	aMsiReg.iMsixNextOffset = 0;
2863	aMsiReg.iMsixBar = VIRTIO_REGION_MSIX_CAP;
2864	aMsiReg.cMsixVectors = VBOX_MSIX_MAX_ENTRIES;
2865	rc = PDMDevHlpPCIRegisterMsi(pDevIns, &aMsiReg); /* see MsixR3init() */
2866	if (RT_FAILURE(rc))
2867	{
2868	/* See PDMDevHlp.cpp:pdmR3DevHlp_PCIRegisterMsi */
2869	LogFunc(("Failed to configure MSI-X (%Rrc). Reverting to INTx\n", rc));
2870	pVirtio->fMsiSupport = false;
2871	}
2872	else
2873	Log2Func(("Using MSI-X for guest driver notification\n"));
2874	}
2875	else
2876	LogFunc(("MSI-X not available for VBox, using INTx notification\n"));
2877
2878	/* Set offset to first capability and enable PCI dev capabilities */
2879	PDMPciDevSetCapabilityList(pPciDev, 0x40);
2880	PDMPciDevSetStatus(pPciDev, VBOX_PCI_STATUS_CAP_LIST);
2881
2882	size_t cbSize = RTStrPrintf(pVirtioCC->szMmioName, sizeof(pVirtioCC->szMmioName), "%s (modern)", pcszInstance);
2883	if (cbSize <= 0)
2884	return PDMDEV_SET_ERROR(pDevIns, VERR_BUFFER_OVERFLOW, N_("virtio: out of memory allocating string")); /* can we put params in this error? */
2885
2886	cbSize = RTStrPrintf(pVirtioCC->szPortIoName, sizeof(pVirtioCC->szPortIoName), "%s (legacy)", pcszInstance);
2887	if (cbSize <= 0)
2888	return PDMDEV_SET_ERROR(pDevIns, VERR_BUFFER_OVERFLOW, N_("virtio: out of memory allocating string")); /* can we put params in this error? */
2889
2890	if (pVirtio->fOfferLegacy)
2891	{
2892	/* As a transitional device that supports legacy VirtIO drivers, this VirtIO device generic implementation presents
2893	* legacy driver interface in I/O space at BAR0. The following maps the common (e.g. device independent)
2894	* dev config area as well as device-specific dev config area (whose size is passed to init function of this VirtIO
2895	* generic device code) for access via Port I/O, since legacy drivers (e.g. pre VirtIO 1.0) don't use MMIO callbacks.
2896	* (See VirtIO 1.1, Section 4.1.4.8).
2897	*/
2898	rc = PDMDevHlpPCIIORegionCreateIo(pDevIns, VIRTIO_REGION_LEGACY_IO, sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T) + cbDevSpecificCfg,
2899	virtioLegacyIOPortOut, virtioLegacyIOPortIn, NULL /pvUser/, pVirtioCC->szPortIoName,
2900	NULL /paExtDescs/, &pVirtio->hLegacyIoPorts);
2901	AssertLogRelRCReturn(rc, PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register legacy config in I/O space at BAR0 */")));
2902	}
2903
2904	/* Note: The Linux driver at drivers/virtio/virtio_pci_modern.c tries to map at least a page for the
2905	* 'unknown' device-specific capability without querying the capability to determine size, so pad w/extra page.
2906	*/
2907	rc = PDMDevHlpPCIIORegionCreateMmio(pDevIns, VIRTIO_REGION_PCI_CAP, RT_ALIGN_32(cbRegion + VIRTIO_PAGE_SIZE, VIRTIO_PAGE_SIZE),
2908	PCI_ADDRESS_SPACE_MEM, virtioMmioWrite, virtioMmioRead, pVirtio,
2909	IOMMMIO_FLAGS_READ_PASSTHRU \| IOMMMIO_FLAGS_WRITE_PASSTHRU,
2910	pVirtioCC->szMmioName,
2911	&pVirtio->hMmioPciCap);
2912	AssertLogRelRCReturn(rc, PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register PCI Capabilities address space")));
2913	return VINF_SUCCESS;
2914	}
2915
2916
2917	/**
2918	* Initializes the VirtIO device using the VirtIO over MMIO transport mode.
2919	*
2920	* @returns VBox status code.
2921	* @param pDevIns Device instance.
2922	* @param pVirtio Pointer to the shared virtio state. This
2923	* must be the first member in the shared
2924	* device instance data!
2925	* @param pVirtioCC Pointer to the ring-3 virtio state. This
2926	* must be the first member in the ring-3
2927	* device instance data!
2928	* @param pcszInstance Device instance name (format-specifier)
2929	* @param cbDevSpecificCfg Size of virtio_pci_device_cap device-specific struct
2930	* @param GCPhysMmioBase The physical guest address of the start of the MMIO area.
2931	* @param u16Irq The interrupt number to use for the virtio device.
2932	*/
2933	static int virtioR3MmioTransportInit(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC, const char *pcszInstance,
2934	uint16_t cbDevSpecificCfg, RTGCPHYS GCPhysMmioBase, uint16_t u16Irq)
2935	{
2936	pVirtio->uIrqMmio = u16Irq;
2937
2938	size_t cbSize = RTStrPrintf(pVirtioCC->szMmioName, sizeof(pVirtioCC->szMmioName), "%s (modern)", pcszInstance);
2939	if (cbSize <= 0)
2940	return PDMDEV_SET_ERROR(pDevIns, VERR_BUFFER_OVERFLOW, N_("virtio: out of memory allocating string")); /* can we put params in this error? */
2941
2942	/*
2943	* Register and map the MMIO region.
2944	*/
2945	int rc = PDMDevHlpMmioCreateAndMap(pDevIns, GCPhysMmioBase, RT_ALIGN_32(cbDevSpecificCfg + VIRTIO_MMIO_SIZE, 512),
2946	virtioMmioTransportWrite, virtioMmioTransportRead,
2947	IOMMMIO_FLAGS_READ_PASSTHRU \| IOMMMIO_FLAGS_WRITE_PASSTHRU,
2948	pVirtioCC->szMmioName, &pVirtio->hMmioPciCap);
2949	AssertLogRelRCReturn(rc, PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register PCI Capabilities address space")));
2950	return VINF_SUCCESS;
2951	}
2952
2953
2954	/** API Function: See header file */
2955	DECLHIDDEN(int) virtioCoreR3Init(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC, PVIRTIOPCIPARAMS pPciParams,
2956	const char *pcszInstance, uint64_t fDevSpecificFeatures, uint32_t fOfferLegacy,
2957	void *pvDevSpecificCfg, uint16_t cbDevSpecificCfg)
2958	{
2959	/*
2960	* Virtio state must be the first member of shared device instance data,
2961	* otherwise can't get our bearings in PCI config callbacks.
2962	*/
2963	AssertLogRelReturn(pVirtio == PDMINS_2_DATA(pDevIns, PVIRTIOCORE), VERR_STATE_CHANGED);
2964	AssertLogRelReturn(pVirtioCC == PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC), VERR_STATE_CHANGED);
2965
2966	pVirtio->pDevInsR3 = pDevIns;
2967
2968	/*
2969	* Caller must initialize these.
2970	*/
2971	AssertReturn(pVirtioCC->pfnStatusChanged, VERR_INVALID_POINTER);
2972	AssertReturn(pVirtioCC->pfnVirtqNotified, VERR_INVALID_POINTER);
2973	AssertReturn(VIRTQ_SIZE > 0 && VIRTQ_SIZE <= 32768, VERR_OUT_OF_RANGE); /* VirtIO specification-defined limit */
2974
2975	PCPDMDEVHLPR3 pHlp = pDevIns->pHlpR3;
2976
2977	uint16_t u16Irq = 0;
2978	int rc = pHlp->pfnCFGMQueryU16Def(pDevIns->pCfg, "Irq", &u16Irq, 0);
2979	if (RT_FAILURE(rc))
2980	return PDMDEV_SET_ERROR(pDevIns, rc, N_("Configuration error: Failed to get the \"Irq\" value"));
2981
2982	RTGCPHYS GCPhysMmioBase = 0;
2983	rc = pHlp->pfnCFGMQueryU64Def(pDevIns->pCfg, "MmioBase", &GCPhysMmioBase, NIL_RTGCPHYS);
2984	if (RT_FAILURE(rc))
2985	return PDMDEV_SET_ERROR(pDevIns, rc,
2986	N_("Configuration error: Failed to get the \"MmioBase\" value"));
2987
2988	#if 0 /* Until pdmR3DvHlp_PCISetIrq() impl is fixed and Assert that limits vec to 0 is removed
2989	* VBox legacy MSI support has not been implemented yet
2990	*/
2991	# ifdef VBOX_WITH_MSI_DEVICES
2992	pVirtio->fMsiSupport = true;
2993	# endif
2994	#endif
2995
2996	/*
2997	* Host features (presented as a buffet for guest to select from)
2998	* include both dev-specific features & reserved dev-independent features (bitmask).
2999	*/
3000	pVirtio->uDeviceType = pPciParams->uDeviceType;
3001	pVirtio->uDeviceFeatures = VIRTIO_F_VERSION_1
3002	\| VIRTIO_DEV_INDEPENDENT_FEATURES_OFFERED
3003	\| fDevSpecificFeatures;
3004
3005	pVirtio->fLegacyDriver = pVirtio->fOfferLegacy = fOfferLegacy;
3006
3007	RTStrCopy(pVirtio->szInstance, sizeof(pVirtio->szInstance), pcszInstance);
3008	pVirtioCC->cbDevSpecificCfg = cbDevSpecificCfg;
3009	pVirtioCC->pbDevSpecificCfg = (uint8_t *)pvDevSpecificCfg;
3010	pVirtioCC->pbPrevDevSpecificCfg = (uint8_t *)RTMemDup(pvDevSpecificCfg, cbDevSpecificCfg);
3011	AssertLogRelReturn(pVirtioCC->pbPrevDevSpecificCfg, VERR_NO_MEMORY);
3012
3013	if (GCPhysMmioBase != NIL_RTGCPHYS)
3014	rc = virtioR3MmioTransportInit(pDevIns, pVirtio, pVirtioCC, pcszInstance, cbDevSpecificCfg,
3015	GCPhysMmioBase, u16Irq);
3016	else
3017	rc = virtioR3PciTransportInit(pDevIns, pVirtio, pVirtioCC, pPciParams, pcszInstance, cbDevSpecificCfg);
3018	AssertLogRelRCReturn(rc, rc);
3019
3020	/*
3021	* Statistics.
3022	*/
3023	# ifdef VBOX_WITH_STATISTICS
3024	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsAllocated, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
3025	"Total number of allocated descriptor chains", "DescChainsAllocated");
3026	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsFreed, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
3027	"Total number of freed descriptor chains", "DescChainsFreed");
3028	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsSegsIn, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
3029	"Total number of inbound segments", "DescChainsSegsIn");
3030	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsSegsOut, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
3031	"Total number of outbound segments", "DescChainsSegsOut");
3032	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatReadR3, STAMTYPE_PROFILE, "IO/ReadR3", STAMUNIT_TICKS_PER_CALL, "Profiling IO reads in R3");
3033	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatReadR0, STAMTYPE_PROFILE, "IO/ReadR0", STAMUNIT_TICKS_PER_CALL, "Profiling IO reads in R0");
3034	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatReadRC, STAMTYPE_PROFILE, "IO/ReadRC", STAMUNIT_TICKS_PER_CALL, "Profiling IO reads in RC");
3035	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatWriteR3, STAMTYPE_PROFILE, "IO/WriteR3", STAMUNIT_TICKS_PER_CALL, "Profiling IO writes in R3");
3036	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatWriteR0, STAMTYPE_PROFILE, "IO/WriteR0", STAMUNIT_TICKS_PER_CALL, "Profiling IO writes in R0");
3037	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatWriteRC, STAMTYPE_PROFILE, "IO/WriteRC", STAMUNIT_TICKS_PER_CALL, "Profiling IO writes in RC");
3038	# endif /* VBOX_WITH_STATISTICS */
3039
3040	return VINF_SUCCESS;
3041	}
3042
3043	#else /* !IN_RING3 */
3044
3045	/**
3046	* Sets up the core ring-0/raw-mode virtio bits.
3047	*
3048	* @returns VBox status code.
3049	* @param pDevIns The device instance.
3050	* @param pVirtio Pointer to the shared virtio state. This must be the first
3051	* member in the shared device instance data!
3052	*/
3053	DECLHIDDEN(int) virtioCoreRZInit(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio)
3054	{
3055	AssertLogRelReturn(pVirtio == PDMINS_2_DATA(pDevIns, PVIRTIOCORE), VERR_STATE_CHANGED);
3056	int rc;
3057	#ifdef FUTURE_OPTIMIZATION
3058	rc = PDMDevHlpSetDeviceCritSect(pDevIns, PDMDevHlpCritSectGetNop(pDevIns));
3059	AssertRCReturn(rc, rc);
3060	#endif
3061
3062	if (pVirtio->uIrqMmio != 0)
3063	{
3064	rc = PDMDevHlpMmioSetUpContext(pDevIns, pVirtio->hMmioPciCap, virtioMmioTransportWrite, virtioMmioTransportRead, pVirtio);
3065	AssertRCReturn(rc, rc);
3066	}
3067	else
3068	{
3069	rc = PDMDevHlpMmioSetUpContext(pDevIns, pVirtio->hMmioPciCap, virtioMmioWrite, virtioMmioRead, pVirtio);
3070	AssertRCReturn(rc, rc);
3071
3072	if (pVirtio->fOfferLegacy)
3073	{
3074	rc = PDMDevHlpIoPortSetUpContext(pDevIns, pVirtio->hLegacyIoPorts, virtioLegacyIOPortOut, virtioLegacyIOPortIn, NULL /pvUser/);
3075	AssertRCReturn(rc, rc);
3076	}
3077	}
3078	return rc;
3079	}
3080
3081	#endif /* !IN_RING3 */
3082

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Devices/VirtIO/VirtioCore.cpp@ 100400

Download in other formats: