VirtioCore.cpp@ 102468

Last change on this file since 102468 was 100402, checked in by vboxsync, 18 months ago
Devices/VirtIO: Add support for the VirtIO over MMIO transport mode useful for ARM, bugref:10459 [build fix]
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 131.9 KB

Line
1	/* $Id: VirtioCore.cpp 100402 2023-07-06 09:06:38Z vboxsync $ */
2
3	/** @file
4	* VirtioCore - Virtio Core (PCI, feature & config mgt, queue mgt & proxy, notification mgt)
5	*/
6
7	/*
8	* Copyright (C) 2009-2023 Oracle and/or its affiliates.
9	*
10	* This file is part of VirtualBox base platform packages, as
11	* available from https://www.virtualbox.org.
12	*
13	* This program is free software; you can redistribute it and/or
14	* modify it under the terms of the GNU General Public License
15	* as published by the Free Software Foundation, in version 3 of the
16	* License.
17	*
18	* This program is distributed in the hope that it will be useful, but
19	* WITHOUT ANY WARRANTY; without even the implied warranty of
20	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
21	* General Public License for more details.
22	*
23	* You should have received a copy of the GNU General Public License
24	* along with this program; if not, see <https://www.gnu.org/licenses>.
25	*
26	* SPDX-License-Identifier: GPL-3.0-only
27	*/
28
29
30	/*********************************************************************************************************************************
31	* Header Files *
32	*********************************************************************************************************************************/
33	#define LOG_GROUP LOG_GROUP_DEV_VIRTIO
34
35	#include <iprt/assert.h>
36	#include <iprt/uuid.h>
37	#include <iprt/mem.h>
38	#include <iprt/sg.h>
39	#include <iprt/assert.h>
40	#include <iprt/string.h>
41	#include <iprt/param.h>
42	#include <iprt/types.h>
43	#include <VBox/log.h>
44	#include <VBox/msi.h>
45	#include <iprt/types.h>
46	#include <VBox/AssertGuest.h>
47	#include <VBox/vmm/pdmdev.h>
48	#include "VirtioCore.h"
49
50
51	/*********************************************************************************************************************************
52	* Defined Constants And Macros *
53	*********************************************************************************************************************************/
54
55	#define INSTANCE(a_pVirtio) ((a_pVirtio)->szInstance)
56	#define VIRTQNAME(a_pVirtio, a_uVirtq) ((a_pVirtio)->aVirtqueues[(a_uVirtq)].szName)
57
58	#define IS_VIRTQ_EMPTY(pDevIns, pVirtio, pVirtq) \
59	(virtioCoreVirtqAvailCnt(pDevIns, pVirtio, pVirtq) == 0)
60
61	#define IS_DRIVER_OK(a_pVirtio) ((a_pVirtio)->fDeviceStatus & VIRTIO_STATUS_DRIVER_OK)
62	#define WAS_DRIVER_OK(a_pVirtio) ((a_pVirtio)->fPrevDeviceStatus & VIRTIO_STATUS_DRIVER_OK)
63
64	/**
65	* These defines are used to track guest virtio-net driver writing driver features accepted flags
66	* in two 32-bit operations (in arbitrary order), and one bit dedicated to ensured 'features complete'
67	* is handled once.
68	*/
69	#define DRIVER_FEATURES_0_WRITTEN 1 /*< fDriverFeatures[0] written by guest virtio-net /
70	#define DRIVER_FEATURES_1_WRITTEN 2 /*< fDriverFeatures[1] written by guest virtio-net /
71	#define DRIVER_FEATURES_0_AND_1_WRITTEN 3 /*< Both 32-bit parts of fDriverFeatures[] written /
72	#define DRIVER_FEATURES_COMPLETE_HANDLED 4 /*< Features negotiation complete handler called /
73
74	/**
75	* This macro returns true if the @a a_offAccess and access length (@a
76	* a_cbAccess) are within the range of the mapped capability struct described by
77	* @a a_LocCapData.
78	*
79	* @param[in] a_offAccess Input: The offset into the MMIO bar of the access.
80	* @param[in] a_cbAccess Input: The access size.
81	* @param[out] a_offsetIntoCap Output: uint32_t variable to return the intra-capability offset into.
82	* @param[in] a_LocCapData Input: The capability location info.
83	*/
84	#define MATCHES_VIRTIO_CAP_STRUCT(a_offAccess, a_cbAccess, a_offsetIntoCap, a_LocCapData) \
85	( ((a_offsetIntoCap) = (uint32_t)((a_offAccess) - (a_LocCapData).offMmio)) < (uint32_t)(a_LocCapData).cbMmio \
86	&& (a_offsetIntoCap) + (uint32_t)(a_cbAccess) <= (uint32_t)(a_LocCapData).cbMmio )
87
88
89	/*********************************************************************************************************************************
90	* Structures and Typedefs *
91	*********************************************************************************************************************************/
92
93	/** @name virtq related flags
94	* @{ */
95	#define VIRTQ_DESC_F_NEXT 1 /*< Indicates this descriptor chains to next /
96	#define VIRTQ_DESC_F_WRITE 2 /*< Marks buffer as write-only (default ro) /
97	#define VIRTQ_DESC_F_INDIRECT 4 /*< Buffer is list of buffer descriptors /
98
99	#define VIRTQ_USED_F_NO_NOTIFY 1 /*< Dev to Drv: Don't notify when buf added /
100	#define VIRTQ_AVAIL_F_NO_INTERRUPT 1 /*< Drv to Dev: Don't notify when buf eaten /
101	/** @} */
102
103	/**
104	* virtq-related structs
105	* (struct names follow VirtIO 1.0 spec, field names use VBox styled naming, w/respective spec'd name in comments)
106	*/
107	typedef struct virtq_desc
108	{
109	uint64_t GCPhysBuf; /*< addr GC Phys. address of buffer /
110	uint32_t cb; /*< len Buffer length /
111	uint16_t fFlags; /*< flags Buffer specific flags /
112	uint16_t uDescIdxNext; /*< next Idx set if VIRTIO_DESC_F_NEXT /
113	} VIRTQ_DESC_T, *PVIRTQ_DESC_T;
114
115	typedef struct virtq_avail
116	{
117	uint16_t fFlags; /*< flags avail ring guest-to-host flags /
118	uint16_t uIdx; /*< idx Index of next free ring slot /
119	RT_FLEXIBLE_ARRAY_EXTENSION
120	uint16_t auRing[RT_FLEXIBLE_ARRAY]; /*< ring Ring: avail drv to dev bufs /
121	//uint16_t uUsedEventIdx; /*< used_event (if VIRTQ_USED_F_EVENT_IDX) /
122	} VIRTQ_AVAIL_T, *PVIRTQ_AVAIL_T;
123
124	typedef struct virtq_used_elem
125	{
126	uint32_t uDescIdx; /*< idx Start of used desc chain /
127	uint32_t cbElem; /*< len Total len of used desc chain /
128	} VIRTQ_USED_ELEM_T;
129
130	typedef struct virt_used
131	{
132	uint16_t fFlags; /*< flags used ring host-to-guest flags /
133	uint16_t uIdx; /*< idx Index of next ring slot /
134	RT_FLEXIBLE_ARRAY_EXTENSION
135	VIRTQ_USED_ELEM_T aRing[RT_FLEXIBLE_ARRAY]; /*< ring Ring: used dev to drv bufs /
136	//uint16_t uAvailEventIdx; /*< avail_event if (VIRTQ_USED_F_EVENT_IDX) /
137	} VIRTQ_USED_T, *PVIRTQ_USED_T;
138
139	DECLHIDDEN(const char *) virtioCoreGetStateChangeText(VIRTIOVMSTATECHANGED enmState)
140	{
141	switch (enmState)
142	{
143	case kvirtIoVmStateChangedReset: return "VM RESET";
144	case kvirtIoVmStateChangedSuspend: return "VM SUSPEND";
145	case kvirtIoVmStateChangedPowerOff: return "VM POWER OFF";
146	case kvirtIoVmStateChangedResume: return "VM RESUME";
147	default: return "<BAD ENUM>";
148	}
149	}
150
151	/* Internal Functions */
152
153	static void virtioCoreNotifyGuestDriver(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq);
154	static int virtioNudgeGuest(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint8_t uCause, uint16_t uVec);
155
156	#ifdef IN_RING3
157	# ifdef LOG_ENABLED
158	DECLINLINE(uint16_t) virtioCoreR3CountPendingBufs(uint16_t uRingIdx, uint16_t uShadowIdx, uint16_t uQueueSize)
159	{
160	if (uShadowIdx == uRingIdx)
161	return 0;
162	else
163	if (uShadowIdx > uRingIdx)
164	return uShadowIdx - uRingIdx;
165	return uQueueSize - (uRingIdx - uShadowIdx);
166	}
167	# endif
168	#endif
169	/** @name Internal queue operations
170	* @{ */
171
172	/**
173	* Accessor for virtq descriptor
174	*/
175	#ifdef IN_RING3
176	DECLINLINE(void) virtioReadDesc(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq,
177	uint32_t idxDesc, PVIRTQ_DESC_T pDesc)
178	{
179	/*
180	* Shut up assertion for legacy virtio-net driver in FreeBSD up to 12.3 (see virtioCoreR3VirtqUsedBufPut()
181	* for more information).
182	*/
183	AssertMsg( IS_DRIVER_OK(pVirtio)
184	\|\| ( pVirtio->fLegacyDriver
185	&& pVirtq->GCPhysVirtqDesc),
186	("Called with guest driver not ready\n"));
187	uint16_t const cVirtqItems = RT_MAX(pVirtq->uQueueSize, 1); /* Make sure to avoid div-by-zero. */
188
189	virtioCoreGCPhysRead(pVirtio, pDevIns,
190	pVirtq->GCPhysVirtqDesc + sizeof(VIRTQ_DESC_T) * (idxDesc % cVirtqItems),
191	pDesc, sizeof(VIRTQ_DESC_T));
192	}
193	#endif
194
195	/**
196	* Accessors for virtq avail ring
197	*/
198	#ifdef IN_RING3
199	DECLINLINE(uint16_t) virtioReadAvailDescIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq, uint32_t availIdx)
200	{
201	uint16_t uDescIdx;
202
203	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
204	uint16_t const cVirtqItems = RT_MAX(pVirtq->uQueueSize, 1); /* Make sure to avoid div-by-zero. */
205	virtioCoreGCPhysRead(pVirtio, pDevIns,
206	pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[availIdx % cVirtqItems]),
207	&uDescIdx, sizeof(uDescIdx));
208	return uDescIdx;
209	}
210
211	DECLINLINE(uint16_t) virtioReadAvailUsedEvent(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
212	{
213	uint16_t uUsedEventIdx;
214	/* VirtIO 1.0 uUsedEventIdx (used_event) immediately follows ring */
215	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
216	virtioCoreGCPhysRead(pVirtio, pDevIns,
217	pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[pVirtq->uQueueSize]),
218	&uUsedEventIdx, sizeof(uUsedEventIdx));
219	return uUsedEventIdx;
220	}
221	#endif
222
223	DECLINLINE(uint16_t) virtioReadAvailRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
224	{
225	uint16_t uIdx = 0;
226	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
227	virtioCoreGCPhysRead(pVirtio, pDevIns,
228	pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF(VIRTQ_AVAIL_T, uIdx),
229	&uIdx, sizeof(uIdx));
230	return uIdx;
231	}
232
233	DECLINLINE(uint16_t) virtioReadAvailRingFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
234	{
235	uint16_t fFlags = 0;
236	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
237	virtioCoreGCPhysRead(pVirtio, pDevIns,
238	pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF(VIRTQ_AVAIL_T, fFlags),
239	&fFlags, sizeof(fFlags));
240	return fFlags;
241	}
242
243	/** @} */
244
245	/** @name Accessors for virtq used ring
246	* @{
247	*/
248
249	#ifdef IN_RING3
250	DECLINLINE(void) virtioWriteUsedElem(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq,
251	uint32_t usedIdx, uint32_t uDescIdx, uint32_t uLen)
252	{
253	VIRTQ_USED_ELEM_T elem = { uDescIdx, uLen };
254	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
255	uint16_t const cVirtqItems = RT_MAX(pVirtq->uQueueSize, 1); /* Make sure to avoid div-by-zero. */
256	virtioCoreGCPhysWrite(pVirtio, pDevIns,
257	pVirtq->GCPhysVirtqUsed
258	+ RT_UOFFSETOF_DYN(VIRTQ_USED_T, aRing[usedIdx % cVirtqItems]),
259	&elem, sizeof(elem));
260	}
261
262	DECLINLINE(void) virtioWriteUsedRingFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq, uint16_t fFlags)
263	{
264	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
265	RT_UNTRUSTED_VALIDATED_FENCE(); /* VirtIO 1.0, Section 3.2.1.4.1 */
266	virtioCoreGCPhysWrite(pVirtio, pDevIns,
267	pVirtq->GCPhysVirtqUsed + RT_UOFFSETOF(VIRTQ_USED_T, fFlags),
268	&fFlags, sizeof(fFlags));
269	}
270	#endif
271
272	DECLINLINE(void) virtioWriteUsedRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq, uint16_t uIdx)
273	{
274	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
275	RT_UNTRUSTED_VALIDATED_FENCE(); /* VirtIO 1.0, Section 3.2.1.4.1 */
276	virtioCoreGCPhysWrite(pVirtio, pDevIns,
277	pVirtq->GCPhysVirtqUsed + RT_UOFFSETOF(VIRTQ_USED_T, uIdx),
278	&uIdx, sizeof(uIdx));
279	}
280
281	#ifdef IN_RING3
282	DECLINLINE(uint16_t) virtioReadUsedRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
283	{
284	uint16_t uIdx = 0;
285	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
286	virtioCoreGCPhysRead(pVirtio, pDevIns,
287	pVirtq->GCPhysVirtqUsed + RT_UOFFSETOF(VIRTQ_USED_T, uIdx),
288	&uIdx, sizeof(uIdx));
289	return uIdx;
290	}
291
292	DECLINLINE(uint16_t) virtioReadUsedRingFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
293	{
294	uint16_t fFlags = 0;
295	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
296	virtioCoreGCPhysRead(pVirtio, pDevIns,
297	pVirtq->GCPhysVirtqUsed + RT_UOFFSETOF(VIRTQ_USED_T, fFlags),
298	&fFlags, sizeof(fFlags));
299	return fFlags;
300	}
301
302	DECLINLINE(void) virtioWriteUsedAvailEvent(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq, uint32_t uAvailEventIdx)
303	{
304	/** VirtIO 1.0 uAvailEventIdx (avail_event) immediately follows ring */
305	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
306	virtioCoreGCPhysWrite(pVirtio, pDevIns,
307	pVirtq->GCPhysVirtqUsed
308	+ RT_UOFFSETOF_DYN(VIRTQ_USED_T, aRing[pVirtq->uQueueSize]),
309	&uAvailEventIdx, sizeof(uAvailEventIdx));
310	}
311	#endif
312	/** @} */
313
314
315	DECLINLINE(uint16_t) virtioCoreVirtqAvailCnt(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
316	{
317	uint16_t uIdxActual = virtioReadAvailRingIdx(pDevIns, pVirtio, pVirtq);
318	uint16_t uIdxShadow = pVirtq->uAvailIdxShadow;
319	uint16_t uIdxDelta;
320
321	if (uIdxActual < uIdxShadow)
322	uIdxDelta = (uIdxActual + pVirtq->uQueueSize) - uIdxShadow;
323	else
324	uIdxDelta = uIdxActual - uIdxShadow;
325
326	return uIdxDelta;
327	}
328	/**
329	* Get count of new (e.g. pending) elements in available ring.
330	*
331	* @param pDevIns The device instance.
332	* @param pVirtio Pointer to the shared virtio state.
333	* @param uVirtq Virtq number
334	*
335	* @returns how many entries have been added to ring as a delta of the consumer's
336	* avail index and the queue's guest-side current avail index.
337	*/
338	DECLHIDDEN(uint16_t) virtioCoreVirtqAvailBufCount(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq)
339	{
340	AssertMsgReturn(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues), ("uVirtq out of range"), 0);
341	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
342
343	if (!IS_DRIVER_OK(pVirtio))
344	{
345	LogRelFunc(("Driver not ready\n"));
346	return 0;
347	}
348	if (!pVirtio->fLegacyDriver && !pVirtq->uEnable)
349	{
350	LogRelFunc(("virtq: %s not enabled\n", VIRTQNAME(pVirtio, uVirtq)));
351	return 0;
352	}
353	return virtioCoreVirtqAvailCnt(pDevIns, pVirtio, pVirtq);
354	}
355
356	#ifdef IN_RING3
357
358	static void virtioCoreR3FeatureDump(VIRTIOCORE pVirtio, PCDBGFINFOHLP pHlp, const VIRTIO_FEATURES_LIST s_aFeatures, int cFeatures, int fBanner)
359	{
360	#define MAXLINE 80
361	/* Display as a single buf to prevent interceding log messages */
362	uint16_t cbBuf = cFeatures * 132;
363	char pszBuf = (char )RTMemAllocZ(cbBuf);
364	Assert(pszBuf);
365	char *cp = pszBuf;
366	for (int i = 0; i < cFeatures; ++i)
367	{
368	bool isOffered = RT_BOOL(pVirtio->uDeviceFeatures & s_aFeatures[i].fFeatureBit);
369	bool isNegotiated = RT_BOOL(pVirtio->uDriverFeatures & s_aFeatures[i].fFeatureBit);
370	cp += RTStrPrintf(cp, cbBuf - (cp - pszBuf), " %s %s %s",
371	isOffered ? "+" : "-", isNegotiated ? "x" : " ", s_aFeatures[i].pcszDesc);
372	}
373	if (pHlp) {
374	if (fBanner)
375	pHlp->pfnPrintf(pHlp, "VirtIO Features Configuration\n\n"
376	" Offered Accepted Feature Description\n"
377	" ------- -------- ------- -----------\n");
378	pHlp->pfnPrintf(pHlp, "%s\n", pszBuf);
379	}
380	#ifdef LOG_ENABLED
381	else
382	{
383	if (fBanner)
384	Log(("VirtIO Features Configuration\n\n"
385	" Offered Accepted Feature Description\n"
386	" ------- -------- ------- -----------\n"));
387	Log(("%s\n", pszBuf));
388	}
389	#endif
390	RTMemFree(pszBuf);
391	}
392
393	/** API Function: See header file*/
394	DECLHIDDEN(void) virtioCorePrintDeviceFeatures(VIRTIOCORE *pVirtio, PCDBGFINFOHLP pHlp,
395	const VIRTIO_FEATURES_LIST *s_aDevSpecificFeatures, int cFeatures) {
396	virtioCoreR3FeatureDump(pVirtio, pHlp, s_aCoreFeatures, RT_ELEMENTS(s_aCoreFeatures), 1 /fBanner /);
397	virtioCoreR3FeatureDump(pVirtio, pHlp, s_aDevSpecificFeatures, cFeatures, 0 /fBanner /);
398	}
399
400	#endif
401
402	#ifdef LOG_ENABLED
403
404	/** API Function: See header file */
405	DECLHIDDEN(void) virtioCoreHexDump(uint8_t pv, uint32_t cb, uint32_t uBase, const char pszTitle)
406	{
407	#define ADJCURSOR(cb) pszOut += cb; cbRemain -= cb;
408	size_t cbPrint = 0, cbRemain = ((cb / 16) + 1) * 80;
409	char pszBuf = (char )RTMemAllocZ(cbRemain), *pszOut = pszBuf;
410	AssertMsgReturnVoid(pszBuf, ("Out of Memory"));
411	if (pszTitle)
412	{
413	cbPrint = RTStrPrintf(pszOut, cbRemain, "%s [%d bytes]:\n", pszTitle, cb);
414	ADJCURSOR(cbPrint);
415	}
416	for (uint32_t row = 0; row < RT_MAX(1, (cb / 16) + 1) && row * 16 < cb; row++)
417	{
418	cbPrint = RTStrPrintf(pszOut, cbRemain, "%04x: ", row * 16 + uBase); /* line address */
419	ADJCURSOR(cbPrint);
420	for (uint8_t col = 0; col < 16; col++)
421	{
422	uint32_t idx = row * 16 + col;
423	if (idx >= cb)
424	cbPrint = RTStrPrintf(pszOut, cbRemain, "-- %s", (col + 1) % 8 ? "" : " ");
425	else
426	cbPrint = RTStrPrintf(pszOut, cbRemain, "%02x %s", pv[idx], (col + 1) % 8 ? "" : " ");
427	ADJCURSOR(cbPrint);
428	}
429	for (uint32_t idx = row * 16; idx < row * 16 + 16; idx++)
430	{
431	cbPrint = RTStrPrintf(pszOut, cbRemain, "%c", (idx >= cb) ? ' ' : (pv[idx] >= 0x20 && pv[idx] <= 0x7e ? pv[idx] : '.'));
432	ADJCURSOR(cbPrint);
433	}
434	*pszOut++ = '\n';
435	--cbRemain;
436	}
437	Log(("%s\n", pszBuf));
438	RTMemFree(pszBuf);
439	RT_NOREF2(uBase, pv);
440	#undef ADJCURSOR
441	}
442
443	/* API FUnction: See header file */
444	DECLHIDDEN(void) virtioCoreGCPhysHexDump(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, uint16_t cb, uint32_t uBase, const char *pszTitle)
445	{
446	PVIRTIOCORE pVirtio = PDMDEVINS_2_DATA(pDevIns, PVIRTIOCORE);
447	#define ADJCURSOR(cb) pszOut += cb; cbRemain -= cb;
448	size_t cbPrint = 0, cbRemain = ((cb / 16) + 1) * 80;
449	char pszBuf = (char )RTMemAllocZ(cbRemain), *pszOut = pszBuf;
450	AssertMsgReturnVoid(pszBuf, ("Out of Memory"));
451	if (pszTitle)
452	{
453	cbPrint = RTStrPrintf(pszOut, cbRemain, "%s [%d bytes]:\n", pszTitle, cb);
454	ADJCURSOR(cbPrint);
455	}
456	for (uint16_t row = 0; row < (uint16_t)RT_MAX(1, (cb / 16) + 1) && row * 16 < cb; row++)
457	{
458	uint8_t c;
459	cbPrint = RTStrPrintf(pszOut, cbRemain, "%04x: ", row * 16 + uBase); /* line address */
460	ADJCURSOR(cbPrint);
461	for (uint8_t col = 0; col < 16; col++)
462	{
463	uint32_t idx = row * 16 + col;
464	virtioCoreGCPhysRead(pVirtio, pDevIns, GCPhys + idx, &c, 1);
465	if (idx >= cb)
466	cbPrint = RTStrPrintf(pszOut, cbRemain, "-- %s", (col + 1) % 8 ? "" : " ");
467	else
468	cbPrint = RTStrPrintf(pszOut, cbRemain, "%02x %s", c, (col + 1) % 8 ? "" : " ");
469	ADJCURSOR(cbPrint);
470	}
471	for (uint16_t idx = row * 16; idx < row * 16 + 16; idx++)
472	{
473	virtioCoreGCPhysRead(pVirtio, pDevIns, GCPhys + idx, &c, 1);
474	cbPrint = RTStrPrintf(pszOut, cbRemain, "%c", (idx >= cb) ? ' ' : (c >= 0x20 && c <= 0x7e ? c : '.'));
475	ADJCURSOR(cbPrint);
476	}
477	*pszOut++ = '\n';
478	--cbRemain;
479	}
480	Log(("%s\n", pszBuf));
481	RTMemFree(pszBuf);
482	RT_NOREF(uBase);
483	#undef ADJCURSOR
484	}
485
486
487	/** API function: See header file */
488	DECLHIDDEN(void) virtioCoreLogMappedIoValue(const char pszFunc, const char pszMember, uint32_t uMemberSize,
489	const void *pv, uint32_t cb, uint32_t uOffset, int fWrite,
490	int fHasIndex, uint32_t idx)
491	{
492	if (LogIs6Enabled())
493	{
494	char szIdx[16];
495	if (fHasIndex)
496	RTStrPrintf(szIdx, sizeof(szIdx), "[%d]", idx);
497	else
498	szIdx[0] = '\0';
499
500	if (cb == 1 \|\| cb == 2 \|\| cb == 4 \|\| cb == 8)
501	{
502	char szDepiction[64];
503	size_t cchDepiction;
504	if (uOffset != 0 \|\| cb != uMemberSize) /* display bounds if partial member access */
505	cchDepiction = RTStrPrintf(szDepiction, sizeof(szDepiction), "%s%s[%d:%d]",
506	pszMember, szIdx, uOffset, uOffset + cb - 1);
507	else
508	cchDepiction = RTStrPrintf(szDepiction, sizeof(szDepiction), "%s%s", pszMember, szIdx);
509
510	/* padding */
511	if (cchDepiction < 30)
512	szDepiction[cchDepiction++] = ' ';
513	while (cchDepiction < 30)
514	szDepiction[cchDepiction++] = '.';
515	szDepiction[cchDepiction] = '\0';
516
517	RTUINT64U uValue;
518	uValue.u = 0;
519	memcpy(uValue.au8, pv, cb);
520	Log6(("%-23s: Guest %s %s %#0*RX64\n",
521	pszFunc, fWrite ? "wrote" : "read ", szDepiction, 2 + cb * 2, uValue.u));
522	}
523	else /* odd number or oversized access, ... log inline hex-dump style */
524	{
525	Log6(("%-23s: Guest %s %s%s[%d:%d]: %.*Rhxs\n",
526	pszFunc, fWrite ? "wrote" : "read ", pszMember,
527	szIdx, uOffset, uOffset + cb, cb, pv));
528	}
529	}
530	RT_NOREF2(fWrite, pszFunc);
531	}
532
533	/**
534	* Log MMIO-mapped Virtio fDeviceStatus register bitmask, naming the bits
535	*/
536	DECLINLINE(void) virtioCoreFormatDeviceStatus(uint8_t bStatus, char *pszBuf, size_t uSize)
537	{
538	# define ADJCURSOR(len) { cp += len; uSize -= len; sep = (char *)" \| "; }
539	memset(pszBuf, 0, uSize);
540	char cp = pszBuf, sep = (char *)"";
541	size_t len;
542	if (bStatus == 0)
543	RTStrPrintf(cp, uSize, "RESET");
544	else
545	{
546	if (bStatus & VIRTIO_STATUS_ACKNOWLEDGE)
547	{
548	len = RTStrPrintf(cp, uSize, "ACKNOWLEDGE");
549	ADJCURSOR(len);
550	}
551	if (bStatus & VIRTIO_STATUS_DRIVER)
552	{
553	len = RTStrPrintf(cp, uSize, "%sDRIVER", sep);
554	ADJCURSOR(len);
555	}
556	if (bStatus & VIRTIO_STATUS_FEATURES_OK)
557	{
558	len = RTStrPrintf(cp, uSize, "%sFEATURES_OK", sep);
559	ADJCURSOR(len);
560	}
561	if (bStatus & VIRTIO_STATUS_DRIVER_OK)
562	{
563	len = RTStrPrintf(cp, uSize, "%sDRIVER_OK", sep);
564	ADJCURSOR(len);
565	}
566	if (bStatus & VIRTIO_STATUS_FAILED)
567	{
568	len = RTStrPrintf(cp, uSize, "%sFAILED", sep);
569	ADJCURSOR(len);
570	}
571	if (bStatus & VIRTIO_STATUS_DEVICE_NEEDS_RESET)
572	RTStrPrintf(cp, uSize, "%sNEEDS_RESET", sep);
573	}
574	# undef ADJCURSOR
575	}
576
577	#endif /* LOG_ENABLED */
578
579	/** API function: See header file */
580	DECLHIDDEN(int) virtioCoreIsLegacyMode(PVIRTIOCORE pVirtio)
581	{
582	return pVirtio->fLegacyDriver;
583	}
584
585	#ifdef IN_RING3
586
587	DECLHIDDEN(int) virtioCoreR3VirtqAttach(PVIRTIOCORE pVirtio, uint16_t uVirtq, const char *pcszName)
588	{
589	LogFunc(("Attaching %s to VirtIO core\n", pcszName));
590	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
591	pVirtq->uVirtq = uVirtq;
592	pVirtq->fUsedRingEvent = false;
593	pVirtq->fAttached = true;
594	RTStrCopy(pVirtq->szName, sizeof(pVirtq->szName), pcszName);
595	return VINF_SUCCESS;
596	}
597
598	DECLHIDDEN(int) virtioCoreR3VirtqDetach(PVIRTIOCORE pVirtio, uint16_t uVirtqNbr)
599	{
600	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtqNbr];
601	pVirtq->uVirtq = 0;
602	pVirtq->uAvailIdxShadow = 0;
603	pVirtq->uUsedIdxShadow = 0;
604	pVirtq->fUsedRingEvent = false;
605	pVirtq->fAttached = false;
606	memset(pVirtq->szName, 0, sizeof(pVirtq->szName));
607	return VINF_SUCCESS;
608	}
609
610	DECLHIDDEN(bool) virtioCoreR3VirtqIsAttached(PVIRTIOCORE pVirtio, uint16_t uVirtqNbr)
611	{
612	return pVirtio->aVirtqueues[uVirtqNbr].fAttached;
613	}
614
615	DECLHIDDEN(bool) virtioCoreR3VirtqIsEnabled(PVIRTIOCORE pVirtio, uint16_t uVirtqNbr)
616	{
617	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtqNbr];
618	return (bool)pVirtq->uEnable && pVirtq->GCPhysVirtqDesc;
619	}
620
621	/** API Fuunction: See header file */
622	DECLHIDDEN(void) virtioCoreR3VirtqInfo(PPDMDEVINS pDevIns, PCDBGFINFOHLP pHlp, const char *pszArgs, int uVirtq)
623	{
624	RT_NOREF(pszArgs);
625	PVIRTIOCORE pVirtio = PDMDEVINS_2_DATA(pDevIns, PVIRTIOCORE);
626	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
627
628	/** @todo add ability to dump physical contents described by any descriptor (using existing VirtIO core API function) */
629	// bool fDump = pszArgs && (pszArgs == 'd' \|\| pszArgs == 'D'); /* "dump" (avail phys descriptor)"
630
631	uint16_t uAvailIdx = virtioReadAvailRingIdx(pDevIns, pVirtio, pVirtq);
632	uint16_t uAvailIdxShadow = pVirtq->uAvailIdxShadow;
633
634	uint16_t uUsedIdx = virtioReadUsedRingIdx(pDevIns, pVirtio, pVirtq);
635	uint16_t uUsedIdxShadow = pVirtq->uUsedIdxShadow;
636
637	VIRTQBUF_T VirtqBuf;
638	PVIRTQBUF pVirtqBuf = &VirtqBuf;
639	bool fEmpty = IS_VIRTQ_EMPTY(pDevIns, pVirtio, pVirtq);
640
641	LogFunc(("%s, empty = %s\n", pVirtq->szName, fEmpty ? "true" : "false"));
642
643	int cSendSegs = 0, cReturnSegs = 0;
644	if (!fEmpty)
645	{
646	virtioCoreR3VirtqAvailBufPeek(pDevIns, pVirtio, uVirtq, pVirtqBuf);
647	cSendSegs = pVirtqBuf->pSgPhysSend ? pVirtqBuf->pSgPhysSend->cSegs : 0;
648	cReturnSegs = pVirtqBuf->pSgPhysReturn ? pVirtqBuf->pSgPhysReturn->cSegs : 0;
649	}
650
651	bool fAvailNoInterrupt = virtioReadAvailRingFlags(pDevIns, pVirtio, pVirtq) & VIRTQ_AVAIL_F_NO_INTERRUPT;
652	bool fUsedNoNotify = virtioReadUsedRingFlags(pDevIns, pVirtio, pVirtq) & VIRTQ_USED_F_NO_NOTIFY;
653
654	pHlp->pfnPrintf(pHlp, " queue enabled: ........... %s\n", pVirtq->uEnable ? "true" : "false");
655	pHlp->pfnPrintf(pHlp, " size: .................... %d\n", pVirtq->uQueueSize);
656	pHlp->pfnPrintf(pHlp, " notify offset: ........... %d\n", pVirtq->uNotifyOffset);
657	if (pVirtio->fMsiSupport)
658	pHlp->pfnPrintf(pHlp, " MSIX vector: ....... %4.4x\n", pVirtq->uMsixVector);
659	pHlp->pfnPrintf(pHlp, "\n");
660	pHlp->pfnPrintf(pHlp, " avail ring (%d entries):\n", uAvailIdx - uAvailIdxShadow);
661	pHlp->pfnPrintf(pHlp, " index: ................ %d\n", uAvailIdx);
662	pHlp->pfnPrintf(pHlp, " shadow: ............... %d\n", uAvailIdxShadow);
663	pHlp->pfnPrintf(pHlp, " flags: ................ %s\n", fAvailNoInterrupt ? "NO_INTERRUPT" : "");
664	pHlp->pfnPrintf(pHlp, "\n");
665	pHlp->pfnPrintf(pHlp, " used ring (%d entries):\n", uUsedIdx - uUsedIdxShadow);
666	pHlp->pfnPrintf(pHlp, " index: ................ %d\n", uUsedIdx);
667	pHlp->pfnPrintf(pHlp, " shadow: ............... %d\n", uUsedIdxShadow);
668	pHlp->pfnPrintf(pHlp, " flags: ................ %s\n", fUsedNoNotify ? "NO_NOTIFY" : "");
669	pHlp->pfnPrintf(pHlp, "\n");
670	if (!fEmpty)
671	{
672	pHlp->pfnPrintf(pHlp, " desc chain:\n");
673	pHlp->pfnPrintf(pHlp, " head idx: ............. %d\n", uUsedIdx);
674	pHlp->pfnPrintf(pHlp, " segs: ................. %d\n", cSendSegs + cReturnSegs);
675	pHlp->pfnPrintf(pHlp, " refCnt ................ %d\n", pVirtqBuf->cRefs);
676	pHlp->pfnPrintf(pHlp, "\n");
677	pHlp->pfnPrintf(pHlp, " host-to-guest (%d bytes):\n", pVirtqBuf->cbPhysSend);
678	pHlp->pfnPrintf(pHlp, " segs: .............. %d\n", cSendSegs);
679	if (cSendSegs)
680	{
681	pHlp->pfnPrintf(pHlp, " index: ............. %d\n", pVirtqBuf->pSgPhysSend->idxSeg);
682	pHlp->pfnPrintf(pHlp, " unsent ............. %d\n", pVirtqBuf->pSgPhysSend->cbSegLeft);
683	}
684	pHlp->pfnPrintf(pHlp, "\n");
685	pHlp->pfnPrintf(pHlp, " guest-to-host (%d bytes)\n", pVirtqBuf->cbPhysReturn);
686	pHlp->pfnPrintf(pHlp, " segs: .............. %d\n", cReturnSegs);
687	if (cReturnSegs)
688	{
689	pHlp->pfnPrintf(pHlp, " index: ............. %d\n", pVirtqBuf->pSgPhysReturn->idxSeg);
690	pHlp->pfnPrintf(pHlp, " unsent ............. %d\n", pVirtqBuf->pSgPhysReturn->cbSegLeft);
691	}
692	} else
693	pHlp->pfnPrintf(pHlp, " No desc chains available\n");
694	pHlp->pfnPrintf(pHlp, "\n");
695	}
696
697
698	/** API Function: See header file */
699	DECLHIDDEN(PVIRTQBUF) virtioCoreR3VirtqBufAlloc(void)
700	{
701	PVIRTQBUF pVirtqBuf = (PVIRTQBUF)RTMemAllocZ(sizeof(VIRTQBUF_T));
702	AssertReturn(pVirtqBuf, NULL);
703	pVirtqBuf->u32Magic = VIRTQBUF_MAGIC;
704	pVirtqBuf->cRefs = 1;
705	return pVirtqBuf;
706	}
707
708
709	/** API Function: See header file */
710	DECLHIDDEN(uint32_t) virtioCoreR3VirtqBufRetain(PVIRTQBUF pVirtqBuf)
711	{
712	AssertReturn(pVirtqBuf, UINT32_MAX);
713	AssertReturn(pVirtqBuf->u32Magic == VIRTQBUF_MAGIC, UINT32_MAX);
714	uint32_t cRefs = ASMAtomicIncU32(&pVirtqBuf->cRefs);
715	Assert(cRefs > 1);
716	Assert(cRefs < 16);
717	return cRefs;
718	}
719
720	/** API Function: See header file */
721	DECLHIDDEN(uint32_t) virtioCoreR3VirtqBufRelease(PVIRTIOCORE pVirtio, PVIRTQBUF pVirtqBuf)
722	{
723	if (!pVirtqBuf)
724	return 0;
725	AssertReturn(pVirtqBuf, 0);
726	AssertReturn(pVirtqBuf->u32Magic == VIRTQBUF_MAGIC, 0);
727	uint32_t cRefs = ASMAtomicDecU32(&pVirtqBuf->cRefs);
728	Assert(cRefs < 16);
729	if (cRefs == 0)
730	{
731	pVirtqBuf->u32Magic = ~VIRTQBUF_MAGIC;
732	RTMemFree(pVirtqBuf);
733	#ifdef VBOX_WITH_STATISTICS
734	STAM_REL_COUNTER_INC(&pVirtio->StatDescChainsFreed);
735	#endif
736	}
737	RT_NOREF(pVirtio);
738	return cRefs;
739	}
740
741	/** API Function: See header file */
742	DECLHIDDEN(void) virtioCoreNotifyConfigChanged(PVIRTIOCORE pVirtio)
743	{
744	virtioNudgeGuest(pVirtio->pDevInsR3, pVirtio, VIRTIO_ISR_DEVICE_CONFIG, pVirtio->uMsixConfig);
745	}
746
747
748	/** API Function: See header file */
749	DECLHIDDEN(void) virtioCoreVirtqEnableNotify(PVIRTIOCORE pVirtio, uint16_t uVirtq, bool fEnable)
750	{
751	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
752	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
753
754	if (IS_DRIVER_OK(pVirtio))
755	{
756	uint16_t fFlags = virtioReadUsedRingFlags(pVirtio->pDevInsR3, pVirtio, pVirtq);
757
758	if (fEnable)
759	fFlags &= ~VIRTQ_USED_F_NO_NOTIFY;
760	else
761	fFlags \|= VIRTQ_USED_F_NO_NOTIFY;
762
763	virtioWriteUsedRingFlags(pVirtio->pDevInsR3, pVirtio, pVirtq, fFlags);
764	}
765	}
766
767	/** API function: See Header file */
768	DECLHIDDEN(void) virtioCoreResetAll(PVIRTIOCORE pVirtio)
769	{
770	LogFunc(("\n"));
771	pVirtio->fDeviceStatus \|= VIRTIO_STATUS_DEVICE_NEEDS_RESET;
772	if (IS_DRIVER_OK(pVirtio))
773	{
774	if (!pVirtio->fLegacyDriver)
775	pVirtio->fGenUpdatePending = true;
776	virtioNudgeGuest(pVirtio->pDevInsR3, pVirtio, VIRTIO_ISR_DEVICE_CONFIG, pVirtio->uMsixConfig);
777	}
778	}
779
780	/** API function: See Header file */
781	DECLHIDDEN(int) virtioCoreR3VirtqAvailBufPeek(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq, PVIRTQBUF pVirtqBuf)
782	{
783	return virtioCoreR3VirtqAvailBufGet(pDevIns, pVirtio, uVirtq, pVirtqBuf, false);
784	}
785
786
787	/** API function: See Header file */
788	DECLHIDDEN(int) virtioCoreR3VirtqAvailBufNext(PVIRTIOCORE pVirtio, uint16_t uVirtq)
789	{
790	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
791	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
792
793	if (!pVirtio->fLegacyDriver)
794	AssertMsgReturn((pVirtio->fDeviceStatus & VIRTIO_STATUS_DRIVER_OK) && pVirtq->uEnable,
795	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
796
797	if (IS_VIRTQ_EMPTY(pVirtio->pDevInsR3, pVirtio, pVirtq))
798	return VERR_NOT_AVAILABLE;
799
800	Log6Func(("%s avail shadow idx: %u\n", pVirtq->szName, pVirtq->uAvailIdxShadow));
801	pVirtq->uAvailIdxShadow++;
802
803	return VINF_SUCCESS;
804	}
805
806	/** API Function: See header file */
807	DECLHIDDEN(int) virtioCoreR3VirtqAvailBufGet(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq,
808	uint16_t uHeadIdx, PVIRTQBUF pVirtqBuf)
809	{
810	AssertMsgReturn(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues),
811	("uVirtq out of range"), VERR_INVALID_PARAMETER);
812
813	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
814
815	if (!pVirtio->fLegacyDriver)
816	AssertMsgReturn((pVirtio->fDeviceStatus & VIRTIO_STATUS_DRIVER_OK) && pVirtq->uEnable,
817	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
818
819	uint16_t uDescIdx = uHeadIdx;
820
821	Log6Func(("%s DESC CHAIN: (head idx = %u)\n", pVirtio->aVirtqueues[uVirtq].szName, uHeadIdx));
822
823	/*
824	* Allocate and initialize the descriptor chain structure.
825	*/
826	pVirtqBuf->u32Magic = VIRTQBUF_MAGIC;
827	pVirtqBuf->cRefs = 1;
828	pVirtqBuf->uHeadIdx = uHeadIdx;
829	pVirtqBuf->uVirtq = uVirtq;
830
831	/*
832	* Gather segments.
833	*/
834	VIRTQ_DESC_T desc;
835
836	uint32_t cbIn = 0;
837	uint32_t cbOut = 0;
838	uint32_t cSegsIn = 0;
839	uint32_t cSegsOut = 0;
840
841	PVIRTIOSGSEG paSegsIn = pVirtqBuf->aSegsIn;
842	PVIRTIOSGSEG paSegsOut = pVirtqBuf->aSegsOut;
843
844	do
845	{
846	PVIRTIOSGSEG pSeg;
847	/*
848	* Malicious guests may go beyond paSegsIn or paSegsOut boundaries by linking
849	* several descriptors into a loop. Since there is no legitimate way to get a sequences of
850	* linked descriptors exceeding the total number of descriptors in the ring (see @bugref{8620}),
851	* the following aborts I/O if breach and employs a simple log throttling algorithm to notify.
852	*/
853	if (cSegsIn + cSegsOut >= pVirtq->uQueueSize)
854	{
855	static volatile uint32_t s_cMessages = 0;
856	static volatile uint32_t s_cThreshold = 1;
857	if (ASMAtomicIncU32(&s_cMessages) == ASMAtomicReadU32(&s_cThreshold))
858	{
859	LogRelMax(64, ("Too many linked descriptors; check if the guest arranges descriptors in a loop.\n"));
860	if (ASMAtomicReadU32(&s_cMessages) != 1)
861	LogRelMax(64, ("(the above error has occured %u times so far)\n", ASMAtomicReadU32(&s_cMessages)));
862	ASMAtomicWriteU32(&s_cThreshold, ASMAtomicReadU32(&s_cThreshold) * 10);
863	}
864	break;
865	}
866	RT_UNTRUSTED_VALIDATED_FENCE();
867
868	virtioReadDesc(pDevIns, pVirtio, pVirtq, uDescIdx, &desc);
869
870	if (desc.fFlags & VIRTQ_DESC_F_WRITE)
871	{
872	Log6Func(("%s IN idx=%-4u seg=%-3u addr=%RGp cb=%u\n", pVirtq->szName, uDescIdx, cSegsIn, desc.GCPhysBuf, desc.cb));
873	cbIn += desc.cb;
874	pSeg = &paSegsIn[cSegsIn++];
875	}
876	else
877	{
878	Log6Func(("%s OUT desc_idx=%-4u seg=%-3u addr=%RGp cb=%u\n", pVirtq->szName, uDescIdx, cSegsOut, desc.GCPhysBuf, desc.cb));
879	cbOut += desc.cb;
880	pSeg = &paSegsOut[cSegsOut++];
881	#ifdef DEEP_DEBUG
882	if (LogIs11Enabled())
883	{
884	virtioCoreGCPhysHexDump(pDevIns, desc.GCPhysBuf, desc.cb, 0, NULL);
885	Log(("\n"));
886	}
887	#endif
888	}
889	pSeg->GCPhys = desc.GCPhysBuf;
890	pSeg->cbSeg = desc.cb;
891	uDescIdx = desc.uDescIdxNext;
892	} while (desc.fFlags & VIRTQ_DESC_F_NEXT);
893
894	/*
895	* Add segments to the descriptor chain structure.
896	*/
897	if (cSegsIn)
898	{
899	virtioCoreGCPhysChainInit(&pVirtqBuf->SgBufIn, paSegsIn, cSegsIn);
900	pVirtqBuf->pSgPhysReturn = &pVirtqBuf->SgBufIn;
901	pVirtqBuf->cbPhysReturn = cbIn;
902	#ifdef VBOX_WITH_STATISTICS
903	STAM_REL_COUNTER_ADD(&pVirtio->StatDescChainsSegsIn, cSegsIn);
904	#endif
905	}
906
907	if (cSegsOut)
908	{
909	virtioCoreGCPhysChainInit(&pVirtqBuf->SgBufOut, paSegsOut, cSegsOut);
910	pVirtqBuf->pSgPhysSend = &pVirtqBuf->SgBufOut;
911	pVirtqBuf->cbPhysSend = cbOut;
912	#ifdef VBOX_WITH_STATISTICS
913	STAM_REL_COUNTER_ADD(&pVirtio->StatDescChainsSegsOut, cSegsOut);
914	#endif
915	}
916
917	#ifdef VBOX_WITH_STATISTICS
918	STAM_REL_COUNTER_INC(&pVirtio->StatDescChainsAllocated);
919	#endif
920	Log6Func(("%s -- segs OUT: %u (%u bytes) IN: %u (%u bytes) --\n",
921	pVirtq->szName, cSegsOut, cbOut, cSegsIn, cbIn));
922
923	return VINF_SUCCESS;
924	}
925
926	/** API function: See Header file */
927	DECLHIDDEN(int) virtioCoreR3VirtqAvailBufGet(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq,
928	PVIRTQBUF pVirtqBuf, bool fRemove)
929	{
930	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
931	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
932
933	if (IS_VIRTQ_EMPTY(pDevIns, pVirtio, pVirtq))
934	return VERR_NOT_AVAILABLE;
935
936	uint16_t uHeadIdx = virtioReadAvailDescIdx(pDevIns, pVirtio, pVirtq, pVirtq->uAvailIdxShadow);
937
938	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
939	virtioWriteUsedAvailEvent(pDevIns,pVirtio, pVirtq, pVirtq->uAvailIdxShadow + 1);
940
941	if (fRemove)
942	pVirtq->uAvailIdxShadow++;
943
944	return virtioCoreR3VirtqAvailBufGet(pDevIns, pVirtio, uVirtq, uHeadIdx, pVirtqBuf);
945	}
946
947	/** API function: See Header file */
948	DECLHIDDEN(int) virtioCoreR3VirtqUsedBufPut(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq, PRTSGBUF pSgVirtReturn,
949	PVIRTQBUF pVirtqBuf, bool fFence)
950	{
951	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
952	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
953
954	PVIRTIOSGBUF pSgPhysReturn = pVirtqBuf->pSgPhysReturn;
955
956	Assert(pVirtqBuf->u32Magic == VIRTQBUF_MAGIC);
957	Assert(pVirtqBuf->cRefs > 0);
958
959	/*
960	* Workaround for a bug in FreeBSD's virtio-net driver up until 12.3 which supports only the legacy style devive.
961	* When the device is re-initialized from the driver it violates the spec and posts commands to the control queue
962	* before setting the DRIVER_OK flag, breaking the following check and rendering the device non-functional.
963	* The queues are properly set up at this stage however so no real harm is done and we can safely continue here,
964	* for the legacy device only of course after making sure the queue is properly set up.
965	*/
966	AssertMsgReturn( IS_DRIVER_OK(pVirtio)
967	\|\| ( pVirtio->fLegacyDriver
968	&& pVirtq->GCPhysVirtqDesc),
969	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
970
971	Log6Func((" Copying device data to %s, [desc:%u -> used ring:%u]\n",
972	VIRTQNAME(pVirtio, uVirtq), pVirtqBuf->uHeadIdx, pVirtq->uUsedIdxShadow));
973
974	/* Copy s/g buf (virtual memory) to guest phys mem (VirtIO "IN" direction). */
975
976	size_t cbCopy = 0, cbTotal = 0, cbRemain = 0;
977
978	if (pSgVirtReturn)
979	{
980	size_t cbTarget = virtioCoreGCPhysChainCalcBufSize(pSgPhysReturn);
981	cbRemain = cbTotal = RTSgBufCalcTotalLength(pSgVirtReturn);
982	AssertMsgReturn(cbTarget >= cbRemain, ("No space to write data to phys memory"), VERR_BUFFER_OVERFLOW);
983	virtioCoreGCPhysChainReset(pSgPhysReturn);
984	while (cbRemain)
985	{
986	cbCopy = RT_MIN(pSgVirtReturn->cbSegLeft, pSgPhysReturn->cbSegLeft);
987	Assert(cbCopy > 0);
988	virtioCoreGCPhysWrite(pVirtio, pDevIns, (RTGCPHYS)pSgPhysReturn->GCPhysCur, pSgVirtReturn->pvSegCur, cbCopy);
989	RTSgBufAdvance(pSgVirtReturn, cbCopy);
990	virtioCoreGCPhysChainAdvance(pSgPhysReturn, cbCopy);
991	cbRemain -= cbCopy;
992	}
993
994	if (fFence)
995	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE(); /* needed? */
996
997	Assert(!(cbCopy >> 32));
998	}
999
1000	/* Flag if write-ahead crosses threshold where guest driver indicated it wants event notification */
1001	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
1002	if (pVirtq->uUsedIdxShadow == virtioReadAvailUsedEvent(pDevIns, pVirtio, pVirtq))
1003	pVirtq->fUsedRingEvent = true;
1004
1005	/*
1006	* Place used buffer's descriptor in used ring but don't update used ring's slot index.
1007	* That will be done with a subsequent client call to virtioCoreVirtqUsedRingSync()
1008	*
1009	* @todo r=aeichner: The increment of the shadow index is not atomic but this code can be called
1010	* concurrently!!
1011	*/
1012	virtioWriteUsedElem(pDevIns, pVirtio, pVirtq, pVirtq->uUsedIdxShadow++, pVirtqBuf->uHeadIdx, (uint32_t)cbTotal);
1013
1014	#ifdef LOG_ENABLED
1015	if (LogIs6Enabled() && pSgVirtReturn)
1016	{
1017
1018	LogFunc((" ... %d segs, %zu bytes, copied to %u byte buf@offset=%u. Residual: %zu bytes\n",
1019	pSgVirtReturn->cSegs, cbTotal - cbRemain, pVirtqBuf->cbPhysReturn,
1020	((virtioCoreGCPhysChainCalcBufSize(pVirtqBuf->pSgPhysReturn) -
1021	virtioCoreGCPhysChainCalcLengthLeft(pVirtqBuf->pSgPhysReturn)) - (cbTotal - cbRemain)),
1022	virtioCoreGCPhysChainCalcLengthLeft(pVirtqBuf->pSgPhysReturn) ));
1023
1024	uint16_t uPending = virtioCoreR3CountPendingBufs(
1025	virtioReadUsedRingIdx(pDevIns, pVirtio, pVirtq),
1026	pVirtq->uUsedIdxShadow, pVirtq->uQueueSize);
1027
1028	LogFunc((" %u used buf%s not synced in %s\n", uPending, uPending == 1 ? "" : "s ",
1029	VIRTQNAME(pVirtio, uVirtq)));
1030	}
1031	#endif
1032	return VINF_SUCCESS;
1033	}
1034
1035	/** API function: See Header file */
1036	DECLHIDDEN(int) virtioCoreR3VirtqUsedBufPut(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq,
1037	size_t cb, void const *pv, PVIRTQBUF pVirtqBuf, size_t cbEnqueue, bool fFence)
1038	{
1039	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1040	Assert(pv);
1041
1042	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1043	PVIRTIOSGBUF pSgPhysReturn = pVirtqBuf->pSgPhysReturn;
1044
1045	Assert(pVirtqBuf->u32Magic == VIRTQBUF_MAGIC);
1046	Assert(pVirtqBuf->cRefs > 0);
1047
1048	AssertMsgReturn(IS_DRIVER_OK(pVirtio), ("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
1049
1050	Log6Func((" Copying device data to %s, [desc chain head idx:%u]\n",
1051	VIRTQNAME(pVirtio, uVirtq), pVirtqBuf->uHeadIdx));
1052	/*
1053	* Convert virtual memory simple buffer to guest physical memory (VirtIO descriptor chain)
1054	*/
1055	uint8_t pvBuf = (uint8_t )pv;
1056	size_t cbRemain = cb, cbCopy = 0;
1057	while (cbRemain)
1058	{
1059	cbCopy = RT_MIN(pSgPhysReturn->cbSegLeft, cbRemain);
1060	Assert(cbCopy > 0);
1061	virtioCoreGCPhysWrite(pVirtio, pDevIns, (RTGCPHYS)pSgPhysReturn->GCPhysCur, pvBuf, cbCopy);
1062	virtioCoreGCPhysChainAdvance(pSgPhysReturn, cbCopy);
1063	pvBuf += cbCopy;
1064	cbRemain -= cbCopy;
1065	}
1066	LogFunc((" ...%zu bytes, copied to %u byte buf@offset=%u. Residual: %zu bytes\n",
1067	cb , pVirtqBuf->cbPhysReturn,
1068	((virtioCoreGCPhysChainCalcBufSize(pVirtqBuf->pSgPhysReturn) -
1069	virtioCoreGCPhysChainCalcLengthLeft(pVirtqBuf->pSgPhysReturn)) - cb),
1070	virtioCoreGCPhysChainCalcLengthLeft(pVirtqBuf->pSgPhysReturn)));
1071
1072	if (cbEnqueue)
1073	{
1074	if (fFence)
1075	{
1076	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE(); /* needed? */
1077	Assert(!(cbCopy >> 32));
1078	}
1079	/* Flag if write-ahead crosses threshold where guest driver indicated it wants event notification */
1080	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
1081	if (pVirtq->uUsedIdxShadow == virtioReadAvailUsedEvent(pDevIns, pVirtio, pVirtq))
1082	pVirtq->fUsedRingEvent = true;
1083	/*
1084	* Place used buffer's descriptor in used ring but don't update used ring's slot index.
1085	* That will be done with a subsequent client call to virtioCoreVirtqUsedRingSync()
1086	*/
1087	Log6Func((" Enqueue desc chain head idx %u to %s used ring @ %u\n", pVirtqBuf->uHeadIdx,
1088	VIRTQNAME(pVirtio, uVirtq), pVirtq->uUsedIdxShadow));
1089
1090	virtioWriteUsedElem(pDevIns, pVirtio, pVirtq, pVirtq->uUsedIdxShadow++, pVirtqBuf->uHeadIdx, (uint32_t)cbEnqueue);
1091
1092	#ifdef LOG_ENABLED
1093	if (LogIs6Enabled())
1094	{
1095	uint16_t uPending = virtioCoreR3CountPendingBufs(
1096	virtioReadUsedRingIdx(pDevIns, pVirtio, pVirtq),
1097	pVirtq->uUsedIdxShadow, pVirtq->uQueueSize);
1098
1099	LogFunc((" %u used buf%s not synced in %s\n",
1100	uPending, uPending == 1 ? "" : "s ", VIRTQNAME(pVirtio, uVirtq)));
1101	}
1102	#endif
1103	} /* fEnqueue */
1104
1105	return VINF_SUCCESS;
1106	}
1107
1108
1109	#endif /* IN_RING3 */
1110
1111	/** API function: See Header file */
1112	DECLHIDDEN(int) virtioCoreVirtqUsedRingSync(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq)
1113	{
1114	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1115	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1116
1117	if (!pVirtio->fLegacyDriver)
1118	AssertMsgReturn((pVirtio->fDeviceStatus & VIRTIO_STATUS_DRIVER_OK) && pVirtq->uEnable,
1119	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
1120
1121	Log6Func((" Sync %s used ring (%u -> idx)\n",
1122	pVirtq->szName, pVirtq->uUsedIdxShadow));
1123
1124	virtioWriteUsedRingIdx(pDevIns, pVirtio, pVirtq, pVirtq->uUsedIdxShadow);
1125	virtioCoreNotifyGuestDriver(pDevIns, pVirtio, uVirtq);
1126
1127	return VINF_SUCCESS;
1128	}
1129
1130	/**
1131	* This is called from the MMIO callback code when the guest does an MMIO access to the
1132	* mapped queue notification capability area corresponding to a particular queue, to notify
1133	* the queue handler of available data in the avail ring of the queue (VirtIO 1.0, 4.1.4.4.1)
1134	*
1135	* @param pDevIns The device instance.
1136	* @param pVirtio Pointer to the shared virtio state.
1137	* @param uVirtq Virtq to check for guest interrupt handling preference
1138	* @param uNotifyIdx Notification index
1139	*/
1140	static void virtioCoreVirtqNotified(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq, uint16_t uNotifyIdx)
1141	{
1142	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1143
1144	/* VirtIO 1.0, section 4.1.5.2 implies uVirtq and uNotifyIdx should match. Disregarding any of
1145	* these notifications (if those indicies disagree) may break device/driver synchronization,
1146	* causing eternal throughput starvation, yet there's no specified way to disambiguate
1147	* which queue to wake-up in any awkward situation where the two parameters differ.
1148	*/
1149	AssertMsg(uNotifyIdx == uVirtq,
1150	("Guest kicked virtq %d's notify addr w/non-corresponding virtq idx %d\n",
1151	uVirtq, uNotifyIdx));
1152	RT_NOREF(uNotifyIdx);
1153
1154	AssertReturnVoid(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1155	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1156
1157	Log6Func(("%s: (desc chains: %u)\n", *pVirtq->szName ? pVirtq->szName : "?UNAMED QUEUE?",
1158	virtioCoreVirtqAvailCnt(pDevIns, pVirtio, pVirtq)));
1159
1160	/* Inform client */
1161	pVirtioCC->pfnVirtqNotified(pDevIns, pVirtio, uVirtq);
1162	RT_NOREF2(pVirtio, pVirtq);
1163	}
1164
1165	/**
1166	* Trigger MSI-X or INT# interrupt to notify guest of data added to used ring of
1167	* the specified virtq, depending on the interrupt configuration of the device
1168	* and depending on negotiated and realtime constraints flagged by the guest driver.
1169	*
1170	* See VirtIO 1.0 specification (section 2.4.7).
1171	*
1172	* @param pDevIns The device instance.
1173	* @param pVirtio Pointer to the shared virtio state.
1174	* @param uVirtq Virtq to check for guest interrupt handling preference
1175	*/
1176	static void virtioCoreNotifyGuestDriver(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq)
1177	{
1178	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1179	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1180
1181	if (!IS_DRIVER_OK(pVirtio))
1182	{
1183	LogFunc(("Guest driver not in ready state.\n"));
1184	return;
1185	}
1186
1187	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
1188	{
1189	if (pVirtq->fUsedRingEvent)
1190	{
1191	#ifdef IN_RING3
1192	Log6Func(("...kicking guest %s, VIRTIO_F_EVENT_IDX set and threshold (%d) reached\n",
1193	pVirtq->szName, (uint16_t)virtioReadAvailUsedEvent(pDevIns, pVirtio, pVirtq)));
1194	#endif
1195	virtioNudgeGuest(pDevIns, pVirtio, VIRTIO_ISR_VIRTQ_INTERRUPT, pVirtq->uMsixVector);
1196	pVirtq->fUsedRingEvent = false;
1197	return;
1198	}
1199	#ifdef IN_RING3
1200	Log6Func(("...skip interrupt %s, VIRTIO_F_EVENT_IDX set but threshold (%d) not reached (%d)\n",
1201	pVirtq->szName,(uint16_t)virtioReadAvailUsedEvent(pDevIns, pVirtio, pVirtq), pVirtq->uUsedIdxShadow));
1202	#endif
1203	}
1204	else
1205	{
1206	/** If guest driver hasn't suppressed interrupts, interrupt */
1207	if (!(virtioReadAvailRingFlags(pDevIns, pVirtio, pVirtq) & VIRTQ_AVAIL_F_NO_INTERRUPT))
1208	{
1209	virtioNudgeGuest(pDevIns, pVirtio, VIRTIO_ISR_VIRTQ_INTERRUPT, pVirtq->uMsixVector);
1210	return;
1211	}
1212	Log6Func(("...skipping interrupt for %s (guest set VIRTQ_AVAIL_F_NO_INTERRUPT)\n", pVirtq->szName));
1213	}
1214	}
1215
1216	/**
1217	* Raise interrupt or MSI-X
1218	*
1219	* @param pDevIns The device instance.
1220	* @param pVirtio Pointer to the shared virtio state.
1221	* @param uCause Interrupt cause bit mask to set in PCI ISR port.
1222	* @param uVec MSI-X vector, if enabled
1223	*/
1224	static int virtioNudgeGuest(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint8_t uCause, uint16_t uMsixVector)
1225	{
1226	if (uCause == VIRTIO_ISR_VIRTQ_INTERRUPT)
1227	Log6Func(("Reason for interrupt - buffer added to 'used' ring.\n"));
1228	else
1229	if (uCause == VIRTIO_ISR_DEVICE_CONFIG)
1230	Log6Func(("Reason for interrupt - device config change\n"));
1231
1232	if (pVirtio->uIrqMmio)
1233	{
1234	pVirtio->uISR \|= uCause;
1235	PDMDevHlpISASetIrq(pDevIns, pVirtio->uIrqMmio, PDM_IRQ_LEVEL_HIGH);
1236	}
1237	else if (!pVirtio->fMsiSupport)
1238	{
1239	pVirtio->uISR \|= uCause;
1240	PDMDevHlpPCISetIrq(pDevIns, 0, PDM_IRQ_LEVEL_HIGH);
1241	}
1242	else if (uMsixVector != VIRTIO_MSI_NO_VECTOR)
1243	PDMDevHlpPCISetIrq(pDevIns, uMsixVector, 1);
1244	return VINF_SUCCESS;
1245	}
1246
1247	/**
1248	* Lower interrupt (Called when guest reads ISR and when resetting)
1249	*
1250	* @param pDevIns The device instance.
1251	*/
1252	static void virtioLowerInterrupt(PPDMDEVINS pDevIns, uint16_t uMsixVector)
1253	{
1254	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1255	if (pVirtio->uIrqMmio)
1256	PDMDevHlpISASetIrq(pDevIns, pVirtio->uIrqMmio, PDM_IRQ_LEVEL_LOW);
1257	else if (!pVirtio->fMsiSupport)
1258	PDMDevHlpPCISetIrq(pDevIns, 0, PDM_IRQ_LEVEL_LOW);
1259	else if (uMsixVector != VIRTIO_MSI_NO_VECTOR)
1260	PDMDevHlpPCISetIrq(pDevIns, pVirtio->uMsixConfig, PDM_IRQ_LEVEL_LOW);
1261	}
1262
1263	#ifdef IN_RING3
1264	static void virtioResetVirtq(PVIRTIOCORE pVirtio, uint16_t uVirtq)
1265	{
1266	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1267	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1268
1269	pVirtq->uQueueSize = VIRTQ_SIZE;
1270	pVirtq->uEnable = false;
1271	pVirtq->uNotifyOffset = uVirtq;
1272	pVirtq->fUsedRingEvent = false;
1273	pVirtq->uAvailIdxShadow = 0;
1274	pVirtq->uUsedIdxShadow = 0;
1275	pVirtq->uMsixVector = uVirtq + 2;
1276
1277	if (!pVirtio->fMsiSupport) /* VirtIO 1.0, 4.1.4.3 and 4.1.5.1.2 */
1278	pVirtq->uMsixVector = VIRTIO_MSI_NO_VECTOR;
1279
1280	virtioLowerInterrupt(pVirtio->pDevInsR3, pVirtq->uMsixVector);
1281	}
1282
1283	static void virtioResetDevice(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio)
1284	{
1285	LogFunc(("Resetting device VirtIO state\n"));
1286	pVirtio->fLegacyDriver = pVirtio->fOfferLegacy; /* Cleared if VIRTIO_F_VERSION_1 feature ack'd */
1287	pVirtio->uDeviceFeaturesSelect = 0;
1288	pVirtio->uDriverFeaturesSelect = 0;
1289	pVirtio->uConfigGeneration = 0;
1290	pVirtio->fDeviceStatus = 0;
1291	pVirtio->uISR = 0;
1292
1293	if (!pVirtio->fMsiSupport)
1294	virtioLowerInterrupt(pDevIns, 0);
1295	else
1296	{
1297	virtioLowerInterrupt(pDevIns, pVirtio->uMsixConfig);
1298	for (int i = 0; i < VIRTQ_MAX_COUNT; i++)
1299	virtioLowerInterrupt(pDevIns, pVirtio->aVirtqueues[i].uMsixVector);
1300	}
1301
1302	if (!pVirtio->fMsiSupport) /* VirtIO 1.0, 4.1.4.3 and 4.1.5.1.2 */
1303	pVirtio->uMsixConfig = VIRTIO_MSI_NO_VECTOR;
1304
1305	for (uint16_t uVirtq = 0; uVirtq < VIRTQ_MAX_COUNT; uVirtq++)
1306	virtioResetVirtq(pVirtio, uVirtq);
1307	}
1308
1309	/**
1310	* Invoked by this implementation when guest driver resets the device.
1311	* The driver itself will not until the device has read the status change.
1312	*/
1313	static void virtioGuestR3WasReset(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
1314	{
1315	Log(("%-23s: Guest reset the device\n", __FUNCTION__));
1316
1317	/* Let the client know */
1318	pVirtioCC->pfnStatusChanged(pVirtio, pVirtioCC, 0 /* fDriverOk */);
1319	virtioResetDevice(pDevIns, pVirtio);
1320	}
1321
1322	DECLHIDDEN(void) virtioCoreR3ResetDevice(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
1323	{
1324	virtioGuestR3WasReset(pDevIns, pVirtio, pVirtioCC);
1325	}
1326	#endif /* IN_RING3 */
1327
1328	/*
1329	* Determines whether guest virtio driver is modern or legacy and does callback
1330	* informing device-specific code that feature negotiation is complete.
1331	* Should be called only once (coordinated via the 'toggle' flag)
1332	*/
1333	#ifdef IN_RING3
1334	DECLINLINE(void) virtioR3DoFeaturesCompleteOnceOnly(PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
1335	{
1336	if (pVirtio->uDriverFeatures & VIRTIO_F_VERSION_1)
1337	{
1338	LogFunc(("VIRTIO_F_VERSION_1 feature ack'd by guest\n"));
1339	pVirtio->fLegacyDriver = 0;
1340	}
1341	else
1342	{
1343	if (pVirtio->fOfferLegacy)
1344	{
1345	pVirtio->fLegacyDriver = 1;
1346	LogFunc(("VIRTIO_F_VERSION_1 feature was NOT set by guest\n"));
1347	}
1348	else
1349	AssertMsgFailed(("Guest didn't accept VIRTIO_F_VERSION_1, but fLegacyOffered flag not set.\n"));
1350	}
1351	if (pVirtioCC->pfnFeatureNegotiationComplete)
1352	pVirtioCC->pfnFeatureNegotiationComplete(pVirtio, pVirtio->uDriverFeatures, pVirtio->fLegacyDriver);
1353	pVirtio->fDriverFeaturesWritten \|= DRIVER_FEATURES_COMPLETE_HANDLED;
1354	}
1355	#endif
1356
1357
1358	/**
1359	* Handles a write to the device status register from the driver.
1360	*
1361	* @returns VBox status code
1362	*
1363	* @param pDevIns The device instance.
1364	* @param pVirtio Pointer to the shared virtio state.
1365	* @param pVirtioCC Pointer to the current context virtio state.
1366	* @param fDeviceStatus The device status to be written.
1367	*/
1368	DECLINLINE(int) virtioDeviceStatusWrite(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC,
1369	uint8_t fDeviceStatus)
1370	{
1371	pVirtio->fDeviceStatus = fDeviceStatus;
1372	bool fDeviceReset = pVirtio->fDeviceStatus == 0;
1373	#ifdef LOG_ENABLED
1374	if (LogIs7Enabled())
1375	{
1376	char szOut[80] = { 0 };
1377	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
1378	Log(("%-23s: Guest wrote fDeviceStatus ................ (%s)\n", __FUNCTION__, szOut));
1379	}
1380	#endif
1381	bool const fStatusChanged = IS_DRIVER_OK(pVirtio) != WAS_DRIVER_OK(pVirtio);
1382
1383	if (fDeviceReset \|\| fStatusChanged)
1384	{
1385	#ifdef IN_RING0
1386	/* Since VirtIO status changes are cumbersome by nature, e.g. not a benchmark priority,
1387	* handle the rest in R3 to facilitate logging or whatever dev-specific client needs to do */
1388	Log6(("%-23s: RING0 => RING3 (demote)\n", __FUNCTION__));
1389	return VINF_IOM_R3_MMIO_WRITE;
1390	#endif
1391	}
1392
1393	#ifdef IN_RING3
1394	/*
1395	* Notify client only if status actually changed from last time and when we're reset.
1396	*/
1397	if (fDeviceReset)
1398	virtioGuestR3WasReset(pDevIns, pVirtio, pVirtioCC);
1399
1400	if (fStatusChanged)
1401	pVirtioCC->pfnStatusChanged(pVirtio, pVirtioCC, IS_DRIVER_OK(pVirtio));
1402	#else
1403	RT_NOREF(pDevIns, pVirtioCC);
1404	#endif
1405	/*
1406	* Save the current status for the next write so we can see what changed.
1407	*/
1408	pVirtio->fPrevDeviceStatus = pVirtio->fDeviceStatus;
1409	return VINF_SUCCESS;
1410	}
1411
1412
1413	/**
1414	* Handles a read from the device status register from the driver.
1415	*
1416	* @returns The device status register value.
1417	*
1418	* @param pVirtio Pointer to the shared virtio state.
1419	*/
1420	DECLINLINE(uint8_t) virtioDeviceStatusRead(PVIRTIOCORE pVirtio)
1421	{
1422	#ifdef LOG_ENABLED
1423	if (LogIs7Enabled())
1424	{
1425	char szOut[80] = { 0 };
1426	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
1427	LogFunc(("Guest read fDeviceStatus ................ (%s)\n", szOut));
1428	}
1429	#endif
1430	return pVirtio->fDeviceStatus;
1431	}
1432
1433
1434	/**
1435	* Handle accesses to Common Configuration capability
1436	*
1437	* @returns VBox status code
1438	*
1439	* @param pDevIns The device instance.
1440	* @param pVirtio Pointer to the shared virtio state.
1441	* @param pVirtioCC Pointer to the current context virtio state.
1442	* @param fWrite Set if write access, clear if read access.
1443	* @param uOffsetOfAccess The common configuration capability offset.
1444	* @param cb Number of bytes to read or write
1445	* @param pv Pointer to location to write to or read from
1446	*/
1447	static int virtioCommonCfgAccessed(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC,
1448	int fWrite, uint32_t uOffsetOfAccess, unsigned cb, void *pv)
1449	{
1450	uint16_t uVirtq = pVirtio->uVirtqSelect;
1451	int rc = VINF_SUCCESS;
1452	uint64_t val;
1453	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDeviceFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1454	{
1455	if (fWrite) /* Guest WRITE pCommonCfg>uDeviceFeatures */
1456	{
1457	/* VirtIO 1.0, 4.1.4.3 states device_feature is a (guest) driver readonly field,
1458	* yet the linux driver attempts to write/read it back twice */
1459	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1460	LogFunc(("... WARNING: Guest attempted to write readonly virtio_pci_common_cfg.device_feature (ignoring)\n"));
1461	return VINF_IOM_MMIO_UNUSED_00;
1462	}
1463	else /* Guest READ pCommonCfg->uDeviceFeatures */
1464	{
1465	switch (pVirtio->uDeviceFeaturesSelect)
1466	{
1467	case 0:
1468	val = pVirtio->uDeviceFeatures & UINT32_C(0xffffffff);
1469	memcpy(pv, &val, cb);
1470	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1471	break;
1472	case 1:
1473	val = pVirtio->uDeviceFeatures >> 32;
1474	memcpy(pv, &val, cb);
1475	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess + sizeof(uint32_t));
1476	break;
1477	default:
1478	LogFunc(("Guest read uDeviceFeatures with out of range selector (%#x), returning 0\n",
1479	pVirtio->uDeviceFeaturesSelect));
1480	return VINF_IOM_MMIO_UNUSED_00;
1481	}
1482	}
1483	}
1484	else
1485	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1486	{
1487	if (fWrite) /* Guest WRITE pCommonCfg->udriverFeatures */
1488	{
1489	switch (pVirtio->uDriverFeaturesSelect)
1490	{
1491	case 0:
1492	memcpy(&pVirtio->uDriverFeatures, pv, cb);
1493	pVirtio->fDriverFeaturesWritten \|= DRIVER_FEATURES_0_WRITTEN;
1494	LogFunc(("Set DRIVER_FEATURES_0_WRITTEN. pVirtio->fDriverFeaturesWritten=%d\n", pVirtio->fDriverFeaturesWritten));
1495	if ( (pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_0_AND_1_WRITTEN) == DRIVER_FEATURES_0_AND_1_WRITTEN
1496	&& !(pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_COMPLETE_HANDLED))
1497	#ifdef IN_RING0
1498	return VINF_IOM_R3_MMIO_WRITE;
1499	#endif
1500	#ifdef IN_RING3
1501	virtioR3DoFeaturesCompleteOnceOnly(pVirtio, pVirtioCC);
1502	#endif
1503	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1504	break;
1505	case 1:
1506	memcpy((char *)&pVirtio->uDriverFeatures + sizeof(uint32_t), pv, cb);
1507	pVirtio->fDriverFeaturesWritten \|= DRIVER_FEATURES_1_WRITTEN;
1508	LogFunc(("Set DRIVER_FEATURES_1_WRITTEN. pVirtio->fDriverFeaturesWritten=%d\n", pVirtio->fDriverFeaturesWritten));
1509	if ( (pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_0_AND_1_WRITTEN) == DRIVER_FEATURES_0_AND_1_WRITTEN
1510	&& !(pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_COMPLETE_HANDLED))
1511	#ifdef IN_RING0
1512	return VINF_IOM_R3_MMIO_WRITE;
1513	#endif
1514	#ifdef IN_RING3
1515	virtioR3DoFeaturesCompleteOnceOnly(pVirtio, pVirtioCC);
1516	#endif
1517	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess + sizeof(uint32_t));
1518	break;
1519	default:
1520	LogFunc(("Guest wrote uDriverFeatures with out of range selector (%#x), returning 0\n",
1521	pVirtio->uDriverFeaturesSelect));
1522	return VINF_SUCCESS;
1523	}
1524	}
1525	else /* Guest READ pCommonCfg->udriverFeatures */
1526	{
1527	switch (pVirtio->uDriverFeaturesSelect)
1528	{
1529	case 0:
1530	val = pVirtio->uDriverFeatures & 0xffffffff;
1531	memcpy(pv, &val, cb);
1532	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1533	break;
1534	case 1:
1535	val = (pVirtio->uDriverFeatures >> 32) & 0xffffffff;
1536	memcpy(pv, &val, cb);
1537	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess + 4);
1538	break;
1539	default:
1540	LogFunc(("Guest read uDriverFeatures with out of range selector (%#x), returning 0\n",
1541	pVirtio->uDriverFeaturesSelect));
1542	return VINF_IOM_MMIO_UNUSED_00;
1543	}
1544	}
1545	}
1546	else
1547	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uNumVirtqs, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1548	{
1549	if (fWrite)
1550	{
1551	Log2Func(("Guest attempted to write readonly virtio_pci_common_cfg.num_queues\n"));
1552	return VINF_SUCCESS;
1553	}
1554	(uint16_t )pv = VIRTQ_MAX_COUNT;
1555	VIRTIO_DEV_CONFIG_LOG_ACCESS(uNumVirtqs, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1556	}
1557	else
1558	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fDeviceStatus, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1559	{
1560	if (fWrite) /* Guest WRITE pCommonCfg->fDeviceStatus */
1561	rc = virtioDeviceStatusWrite(pDevIns, pVirtio, pVirtioCC, (uint8_t )pv);
1562	else /* Guest READ pCommonCfg->fDeviceStatus */
1563	(uint8_t )pv = virtioDeviceStatusRead(pVirtio);
1564	}
1565	else
1566	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixConfig, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1567	VIRTIO_DEV_CONFIG_ACCESS( uMsixConfig, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1568	else
1569	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uDeviceFeaturesSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1570	VIRTIO_DEV_CONFIG_ACCESS( uDeviceFeaturesSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1571	else
1572	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uDriverFeaturesSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1573	VIRTIO_DEV_CONFIG_ACCESS( uDriverFeaturesSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1574	else
1575	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uConfigGeneration, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1576	VIRTIO_DEV_CONFIG_ACCESS( uConfigGeneration, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1577	else
1578	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uVirtqSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1579	{
1580	if (fWrite) {
1581	uint16_t uVirtqNew = (uint16_t )pv;
1582
1583	if (uVirtqNew < RT_ELEMENTS(pVirtio->aVirtqueues))
1584	VIRTIO_DEV_CONFIG_ACCESS( uVirtqSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1585	else
1586	LogFunc(("... WARNING: Guest attempted to write invalid virtq selector (ignoring)\n"));
1587	}
1588	else
1589	VIRTIO_DEV_CONFIG_ACCESS( uVirtqSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1590	}
1591	else
1592	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( GCPhysVirtqDesc, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1593	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( GCPhysVirtqDesc, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1594	else
1595	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( GCPhysVirtqAvail, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1596	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( GCPhysVirtqAvail, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1597	else
1598	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( GCPhysVirtqUsed, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1599	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( GCPhysVirtqUsed, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1600	else
1601	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uQueueSize, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1602	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uQueueSize, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1603	else
1604	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uEnable, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1605	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uEnable, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1606	else
1607	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uNotifyOffset, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1608	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uNotifyOffset, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1609	else
1610	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixVector, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1611	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uMsixVector, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1612	else
1613	{
1614	Log2Func(("Bad guest %s access to virtio_pci_common_cfg: uOffsetOfAccess=%#x (%d), cb=%d\n",
1615	fWrite ? "write" : "read ", uOffsetOfAccess, uOffsetOfAccess, cb));
1616	return fWrite ? VINF_SUCCESS : VINF_IOM_MMIO_UNUSED_00;
1617	}
1618
1619	#ifndef IN_RING3
1620	RT_NOREF(pDevIns, pVirtioCC);
1621	#endif
1622	return rc;
1623	}
1624
1625	/**
1626	* @callback_method_impl{FNIOMIOPORTNEWIN)
1627	*
1628	* This I/O handler exists only to handle access from legacy drivers.
1629	*/
1630	static DECLCALLBACK(VBOXSTRICTRC) virtioLegacyIOPortIn(PPDMDEVINS pDevIns, void pvUser, RTIOPORT offPort, uint32_t pu32, unsigned cb)
1631	{
1632	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1633	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatRead), a);
1634
1635	RT_NOREF(pvUser);
1636	Log(("%-23s: Port read at offset=%RTiop, cb=%#x%s",
1637	__FUNCTION__, offPort, cb,
1638	VIRTIO_DEV_CONFIG_MATCH_MEMBER(fIsrStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort) ? "" : "\n"));
1639
1640	void pv = pu32; / To use existing macros */
1641	int fWrite = 0; /* To use existing macros */
1642
1643	uint16_t uVirtq = pVirtio->uVirtqSelect;
1644
1645	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDeviceFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1646	{
1647	uint32_t val = pVirtio->uDeviceFeatures & UINT32_C(0xffffffff);
1648	memcpy(pu32, &val, cb);
1649	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1650	}
1651	else
1652	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDriverFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1653	{
1654	uint32_t val = pVirtio->uDriverFeatures & UINT32_C(0xffffffff);
1655	memcpy(pu32, &val, cb);
1656	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1657	}
1658	else
1659	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fDeviceStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1660	{
1661	(uint8_t )pu32 = pVirtio->fDeviceStatus;
1662	#ifdef LOG_ENABLED
1663	if (LogIs7Enabled())
1664	{
1665	char szOut[80] = { 0 };
1666	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
1667	Log(("%-23s: Guest read fDeviceStatus ................ (%s)\n", __FUNCTION__, szOut));
1668	}
1669	#endif
1670	}
1671	else
1672	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fIsrStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1673	{
1674	ASSERT_GUEST_MSG(cb == 1, ("%d\n", cb));
1675	(uint8_t )pu32 = pVirtio->uISR;
1676	pVirtio->uISR = 0;
1677	virtioLowerInterrupt( pDevIns, 0);
1678	Log((" (ISR read and cleared)\n"));
1679	}
1680	else
1681	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uVirtqSelect, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1682	VIRTIO_DEV_CONFIG_ACCESS( uVirtqSelect, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1683	else
1684	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uVirtqPfn, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1685	{
1686	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[uVirtq];
1687	*pu32 = pVirtQueue->GCPhysVirtqDesc >> GUEST_PAGE_SHIFT;
1688	Log(("%-23s: Guest read uVirtqPfn .................... %#x\n", __FUNCTION__, *pu32));
1689	}
1690	else
1691	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uQueueSize, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1692	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uQueueSize, uVirtq, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio->aVirtqueues);
1693	else
1694	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uQueueNotify, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1695	VIRTIO_DEV_CONFIG_ACCESS( uQueueNotify, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1696	#ifdef LEGACY_MSIX_SUPPORTED
1697	else
1698	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixConfig, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1699	VIRTIO_DEV_CONFIG_ACCESS( uMsixConfig, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1700	else
1701	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixVector, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1702	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uMsixVector, uVirtq, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio->aVirtqueues);
1703	#endif
1704	else if (offPort >= sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T))
1705	{
1706	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
1707	#ifdef IN_RING3
1708	/* Access device-specific configuration */
1709	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1710	int rc = pVirtioCC->pfnDevCapRead(pDevIns, offPort - sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T), pv, cb);
1711	return rc;
1712	#else
1713	return VINF_IOM_R3_IOPORT_READ;
1714	#endif
1715	}
1716	else
1717	{
1718	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
1719	Log2Func(("Bad guest read access to virtio_legacy_pci_common_cfg: offset=%#x, cb=%x\n",
1720	offPort, cb));
1721	int rc = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
1722	"virtioLegacyIOPortIn: no valid port at offset offset=%RTiop cb=%#x\n", offPort, cb);
1723	return rc;
1724	}
1725	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
1726	return VINF_SUCCESS;
1727	}
1728
1729	/**
1730	* @callback_method_impl{ * @callback_method_impl{FNIOMIOPORTNEWOUT}
1731	*
1732	* This I/O Port interface exists only to handle access from legacy drivers.
1733	*/
1734	static DECLCALLBACK(VBOXSTRICTRC) virtioLegacyIOPortOut(PPDMDEVINS pDevIns, void *pvUser, RTIOPORT offPort, uint32_t u32, unsigned cb)
1735	{
1736	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1737	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatWrite), a);
1738	RT_NOREF(pvUser);
1739
1740	uint16_t uVirtq = pVirtio->uVirtqSelect;
1741	uint32_t u32OnStack = u32; /* allows us to use this impl's MMIO parsing macros */
1742	void pv = &u32OnStack; / To use existing macros */
1743	int fWrite = 1; /* To use existing macros */
1744
1745	Log(("%-23s: Port written at offset=%RTiop, cb=%#x, u32=%#x\n", __FUNCTION__, offPort, cb, u32));
1746
1747	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uVirtqSelect, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1748	{
1749	if (u32 < RT_ELEMENTS(pVirtio->aVirtqueues))
1750	VIRTIO_DEV_CONFIG_ACCESS( uVirtqSelect, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1751	else
1752	LogFunc(("... WARNING: Guest attempted to write invalid virtq selector (ignoring)\n"));
1753	}
1754	else
1755	#ifdef LEGACY_MSIX_SUPPORTED
1756	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixConfig, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1757	VIRTIO_DEV_CONFIG_ACCESS( uMsixConfig, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1758	else
1759	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixVector, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1760	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uMsixVector, uVirtq, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio->aVirtqueues);
1761	else
1762	#endif
1763	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDeviceFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1764	{
1765	/* Check to see if guest acknowledged unsupported features */
1766	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1767	LogFunc(("... WARNING: Guest attempted to write readonly virtio_pci_common_cfg.device_feature (ignoring)\n"));
1768	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1769	return VINF_SUCCESS;
1770	}
1771	else
1772	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDriverFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1773	{
1774	memcpy(&pVirtio->uDriverFeatures, pv, cb);
1775	if ((pVirtio->uDriverFeatures & ~VIRTIO_DEV_INDEPENDENT_LEGACY_FEATURES_OFFERED) == 0)
1776	{
1777	Log(("Guest asked for features host does not support! (host=%x guest=%x)\n",
1778	VIRTIO_DEV_INDEPENDENT_LEGACY_FEATURES_OFFERED, pVirtio->uDriverFeatures));
1779	pVirtio->uDriverFeatures &= VIRTIO_DEV_INDEPENDENT_LEGACY_FEATURES_OFFERED;
1780	}
1781	if (!(pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_COMPLETE_HANDLED))
1782	{
1783	#ifdef IN_RING0
1784	Log6(("%-23s: RING0 => RING3 (demote)\n", __FUNCTION__));
1785	return VINF_IOM_R3_IOPORT_WRITE;
1786	#endif
1787	#ifdef IN_RING3
1788	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1789	virtioR3DoFeaturesCompleteOnceOnly(pVirtio, pVirtioCC);
1790	#endif
1791	}
1792	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1793	}
1794	else
1795	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uQueueSize, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1796	{
1797	VIRTIO_DEV_CONFIG_LOG_ACCESS(uQueueSize, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1798	LogFunc(("... WARNING: Guest attempted to write readonly device_feature (queue size) (ignoring)\n"));
1799	return VINF_SUCCESS;
1800	}
1801	else
1802	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fDeviceStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1803	{
1804	bool const fDriverInitiatedReset = (pVirtio->fDeviceStatus = (uint8_t)u32) == 0;
1805	bool const fDriverStateImproved = IS_DRIVER_OK(pVirtio) && !WAS_DRIVER_OK(pVirtio);
1806	#ifdef LOG_ENABLED
1807	if (LogIs7Enabled())
1808	{
1809	char szOut[80] = { 0 };
1810	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
1811	Log(("%-23s: Guest wrote fDeviceStatus ................ (%s)\n", __FUNCTION__, szOut));
1812	}
1813	#endif
1814	if (fDriverStateImproved \|\| fDriverInitiatedReset)
1815	{
1816	#ifdef IN_RING0
1817	Log6(("%-23s: RING0 => RING3 (demote)\n", __FUNCTION__));
1818	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1819	return VINF_IOM_R3_IOPORT_WRITE;
1820	#endif
1821	}
1822
1823	#ifdef IN_RING3
1824	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1825	if (fDriverInitiatedReset)
1826	virtioGuestR3WasReset(pDevIns, pVirtio, pVirtioCC);
1827
1828	else if (fDriverStateImproved)
1829	pVirtioCC->pfnStatusChanged(pVirtio, pVirtioCC, 1 /* fDriverOk */);
1830
1831	#endif
1832	pVirtio->fPrevDeviceStatus = pVirtio->fDeviceStatus;
1833	}
1834	else
1835	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uVirtqPfn, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1836	{
1837	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1838	uint64_t uVirtqPfn = (uint64_t)u32;
1839
1840	if (uVirtqPfn)
1841	{
1842	/* Transitional devices calculate ring physical addresses using rigid spec-defined formulae,
1843	* instead of guest conveying respective address of each ring, as "modern" VirtIO drivers do,
1844	* thus there is no virtq PFN or single base queue address stored in instance data for
1845	* this transitional device, but rather it is derived, when read back, from GCPhysVirtqDesc */
1846
1847	pVirtq->GCPhysVirtqDesc = uVirtqPfn * VIRTIO_PAGE_SIZE;
1848	pVirtq->GCPhysVirtqAvail = pVirtq->GCPhysVirtqDesc + sizeof(VIRTQ_DESC_T) * pVirtq->uQueueSize;
1849	pVirtq->GCPhysVirtqUsed =
1850	RT_ALIGN(pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[pVirtq->uQueueSize]), VIRTIO_PAGE_SIZE);
1851	}
1852	else
1853	{
1854	/* Don't set ring addresses for queue (to meaningless values), when guest resets the virtq's PFN */
1855	pVirtq->GCPhysVirtqDesc = 0;
1856	pVirtq->GCPhysVirtqAvail = 0;
1857	pVirtq->GCPhysVirtqUsed = 0;
1858	}
1859	Log(("%-23s: Guest wrote uVirtqPfn .................... %#x:\n"
1860	"%68s... %p -> GCPhysVirtqDesc\n%68s... %p -> GCPhysVirtqAvail\n%68s... %p -> GCPhysVirtqUsed\n",
1861	__FUNCTION__, u32, " ", pVirtq->GCPhysVirtqDesc, " ", pVirtq->GCPhysVirtqAvail, " ", pVirtq->GCPhysVirtqUsed));
1862	}
1863	else
1864	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uQueueNotify, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1865	{
1866	#ifdef IN_RING3
1867	ASSERT_GUEST_MSG(cb == 2, ("cb=%u\n", cb));
1868	pVirtio->uQueueNotify = u32 & 0xFFFF;
1869	if (uVirtq < VIRTQ_MAX_COUNT)
1870	{
1871	RT_UNTRUSTED_VALIDATED_FENCE();
1872
1873	/* Need to check that queue is configured. Legacy spec didn't have a queue enabled flag */
1874	if (pVirtio->aVirtqueues[pVirtio->uQueueNotify].GCPhysVirtqDesc)
1875	virtioCoreVirtqNotified(pDevIns, pVirtio, pVirtio->uQueueNotify, pVirtio->uQueueNotify /* uNotifyIdx */);
1876	else
1877	Log(("The queue (#%d) being notified has not been initialized.\n", pVirtio->uQueueNotify));
1878	}
1879	else
1880	Log(("Invalid queue number (%d)\n", pVirtio->uQueueNotify));
1881	#else
1882	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1883	return VINF_IOM_R3_IOPORT_WRITE;
1884	#endif
1885	}
1886	else
1887	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fIsrStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1888	{
1889	VIRTIO_DEV_CONFIG_LOG_ACCESS( fIsrStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1890	LogFunc(("... WARNING: Guest attempted to write readonly device_feature (ISR status) (ignoring)\n"));
1891	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1892	return VINF_SUCCESS;
1893	}
1894	else if (offPort >= sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T))
1895	{
1896	#ifdef IN_RING3
1897
1898	/* Access device-specific configuration */
1899	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1900	return pVirtioCC->pfnDevCapWrite(pDevIns, offPort - sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T), pv, cb);
1901	#else
1902	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1903	return VINF_IOM_R3_IOPORT_WRITE;
1904	#endif
1905	}
1906	else
1907	{
1908	Log2Func(("Bad guest write access to virtio_legacy_pci_common_cfg: offset=%#x, cb=0x%x\n",
1909	offPort, cb));
1910	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1911	int rc = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
1912	"virtioLegacyIOPortOut: no valid port at offset offset=%RTiop cb=0x%#x\n", offPort, cb);
1913	return rc;
1914	}
1915
1916	RT_NOREF(uVirtq);
1917	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1918	return VINF_SUCCESS;
1919	}
1920
1921
1922	/**
1923	* Read from the device specific configuration at the given offset.
1924	*
1925	* @returns VBox status code.
1926	* @param pDevIns
1927	*/
1928	DECLINLINE(VBOXSTRICTRC) virtioDeviceCfgRead(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC,
1929	uint32_t offDevCfg, void *pv, unsigned cb)
1930	{
1931	#ifdef IN_RING3
1932	/*
1933	* Callback to client to manage device-specific configuration.
1934	*/
1935	VBOXSTRICTRC rcStrict = pVirtioCC->pfnDevCapRead(pDevIns, offDevCfg, pv, cb);
1936
1937	/*
1938	* Anytime any part of the dev-specific dev config (which this virtio core implementation sees
1939	* as a blob, and virtio dev-specific code separates into fields) is READ, it must be compared
1940	* for deltas from previous read to maintain a config gen. seq. counter (VirtIO 1.0, section 4.1.4.3.1)
1941	*/
1942	bool fDevSpecificFieldChanged = RT_BOOL(memcmp(pVirtioCC->pbDevSpecificCfg + offDevCfg,
1943	pVirtioCC->pbPrevDevSpecificCfg + offDevCfg,
1944	RT_MIN(cb, pVirtioCC->cbDevSpecificCfg - offDevCfg)));
1945
1946	memcpy(pVirtioCC->pbPrevDevSpecificCfg, pVirtioCC->pbDevSpecificCfg, pVirtioCC->cbDevSpecificCfg);
1947
1948	if (pVirtio->fGenUpdatePending \|\| fDevSpecificFieldChanged)
1949	{
1950	++pVirtio->uConfigGeneration;
1951	Log6Func(("Bumped cfg. generation to %d because %s%s\n", pVirtio->uConfigGeneration,
1952	fDevSpecificFieldChanged ? "<dev cfg changed> " : "",
1953	pVirtio->fGenUpdatePending ? "<update was pending>" : ""));
1954	pVirtio->fGenUpdatePending = false;
1955	}
1956
1957	virtioLowerInterrupt(pDevIns, 0);
1958	return rcStrict;
1959	#else
1960	RT_NOREF(pDevIns, pVirtio, pVirtioCC, offDevCfg, pv, cb);
1961	return VINF_IOM_R3_MMIO_READ;
1962	#endif
1963	}
1964
1965
1966	/**
1967	* @callback_method_impl{FNIOMMMIONEWREAD,
1968	* Memory mapped I/O Handler for PCI Capabilities read operations.}
1969	*
1970	* This MMIO handler specifically supports the VIRTIO_PCI_CAP_PCI_CFG capability defined
1971	* in the VirtIO 1.0 specification, section 4.1.4.7, and as such is restricted to reads
1972	* of 1, 2 or 4 bytes, only.
1973	*
1974	*/
1975	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioRead(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void pv, unsigned cb)
1976	{
1977	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1978	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1979	AssertReturn(cb == 1 \|\| cb == 2 \|\| cb == 4, VERR_INVALID_PARAMETER);
1980	Assert(pVirtio == (PVIRTIOCORE)pvUser); RT_NOREF(pvUser);
1981
1982	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatRead), a);
1983
1984	VBOXSTRICTRC rcStrict;
1985	uint32_t uOffset;
1986	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocDeviceCap))
1987	rcStrict = virtioDeviceCfgRead(pDevIns, pVirtio, pVirtioCC, uOffset, pv, cb);
1988	else if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocCommonCfgCap))
1989	rcStrict = virtioCommonCfgAccessed(pDevIns, pVirtio, pVirtioCC, false /* fWrite */, uOffset, cb, pv);
1990	else if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocIsrCap))
1991	{
1992	(uint8_t )pv = pVirtio->uISR;
1993	Log6Func(("Read and clear ISR\n"));
1994	pVirtio->uISR = 0; /* VirtIO spec requires reads of ISR to clear it */
1995	virtioLowerInterrupt(pDevIns, 0);
1996	rcStrict = VINF_SUCCESS;
1997	}
1998	else
1999	{
2000	ASSERT_GUEST_MSG_FAILED(("Bad read access to mapped capabilities region: off=%RGp cb=%u\n", off, cb));
2001	rcStrict = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
2002	"virtioMmioRead: Bad MMIO access to capabilities, offset=%RTiop cb=%08x\n", off, cb);
2003	}
2004
2005	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
2006	return rcStrict;
2007	}
2008
2009	/**
2010	* @callback_method_impl{FNIOMMMIONEWREAD,
2011	* Memory mapped I/O Handler for PCI Capabilities write operations.}
2012	*
2013	* This MMIO handler specifically supports the VIRTIO_PCI_CAP_PCI_CFG capability defined
2014	* in the VirtIO 1.0 specification, section 4.1.4.7, and as such is restricted to writes
2015	* of 1, 2 or 4 bytes, only.
2016	*/
2017	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioWrite(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void const pv, unsigned cb)
2018	{
2019	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
2020	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2021	AssertReturn(cb == 1 \|\| cb == 2 \|\| cb == 4, VERR_INVALID_PARAMETER);
2022	Assert(pVirtio == (PVIRTIOCORE)pvUser); RT_NOREF(pvUser);
2023	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatWrite), a);
2024
2025	VBOXSTRICTRC rcStrict;
2026	uint32_t uOffset;
2027	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocDeviceCap))
2028	{
2029	#ifdef IN_RING3
2030	/*
2031	* Foreward this MMIO write access for client to deal with.
2032	*/
2033	rcStrict = pVirtioCC->pfnDevCapWrite(pDevIns, uOffset, pv, cb);
2034	#else
2035	Log6(("%-23s: RING0 => RING3 (demote)\n", __FUNCTION__));
2036	rcStrict = VINF_IOM_R3_MMIO_WRITE;
2037	#endif
2038	}
2039	else if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocCommonCfgCap))
2040	rcStrict = virtioCommonCfgAccessed(pDevIns, pVirtio, pVirtioCC, true /* fWrite /, uOffset, cb, (void )pv);
2041	else if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocIsrCap) && cb == sizeof(uint8_t))
2042	{
2043	pVirtio->uISR = (uint8_t )pv;
2044	Log6Func(("Setting uISR = 0x%02x (virtq interrupt: %d, dev confg interrupt: %d)\n",
2045	pVirtio->uISR & 0xff,
2046	pVirtio->uISR & VIRTIO_ISR_VIRTQ_INTERRUPT,
2047	RT_BOOL(pVirtio->uISR & VIRTIO_ISR_DEVICE_CONFIG)));
2048	rcStrict = VINF_SUCCESS;
2049	}
2050	else if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocNotifyCap) && cb == sizeof(uint16_t))
2051	{
2052	/* This should be guest driver dropping index of a new descriptor in avail ring */
2053	virtioCoreVirtqNotified(pDevIns, pVirtio, uOffset / VIRTIO_NOTIFY_OFFSET_MULTIPLIER, (uint16_t )pv);
2054	rcStrict = VINF_SUCCESS;
2055	}
2056	else
2057	{
2058	ASSERT_GUEST_MSG_FAILED(("Bad write access to mapped capabilities region: off=%RGp pv=%#p{%.*Rhxs} cb=%u\n", off, pv, cb, pv, cb));
2059	rcStrict = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
2060	"virtioMmioRead: Bad MMIO access to capabilities, offset=%RTiop cb=%08x\n", off, cb);
2061	}
2062
2063	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
2064	return rcStrict;
2065	}
2066
2067
2068	/**
2069	* @callback_method_impl{FNIOMMMIONEWREAD,
2070	* Memory mapped I/O Handler for Virtio over MMIO read operations.}
2071	*
2072	*/
2073	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioTransportRead(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void pv, unsigned cb)
2074	{
2075	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
2076	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2077	RT_NOREF(pvUser);
2078	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatRead), a);
2079
2080	if (off >= VIRTIO_MMIO_SIZE)
2081	{
2082	VBOXSTRICTRC rcStrict = virtioDeviceCfgRead(pDevIns, pVirtio, pVirtioCC, (uint32_t)off - VIRTIO_MMIO_SIZE, pv, cb);
2083	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
2084	return rcStrict;
2085	}
2086
2087	/* All accesses below need to be aligned on a 32-bit boundary and must be 32-bit in size. */
2088	ASSERT_GUEST_MSG_RETURN(!(off & 0x3) && cb == sizeof(uint32_t),
2089	("Bad read access: off=%RGp pv=%#p{%.*Rhxs} cb=%u\n", off, pv, cb, pv, cb),
2090	VINF_IOM_MMIO_UNUSED_FF);
2091
2092	int rc = VINF_SUCCESS;
2093	uint32_t pu32 = (uint32_t )pv;
2094	switch (off)
2095	{
2096	case VIRTIO_MMIO_REG_MAGIC_OFF:
2097	*pu32 = RT_H2LE_U32(VIRTIO_MMIO_REG_MAGIC_VALUE);
2098	break;
2099	case VIRTIO_MMIO_REG_VERSION_OFF:
2100	*pu32 = RT_H2LE_U32(VIRTIO_MMIO_REG_VERSION_VALUE);
2101	break;
2102	case VIRTIO_MMIO_REG_DEVICEID_OFF:
2103	*pu32 = pVirtio->uDeviceType;
2104	break;
2105	case VIRTIO_MMIO_REG_VENDORID_OFF:
2106	*pu32 = RT_H2LE_U32(DEVICE_PCI_VENDOR_ID_VIRTIO);
2107	break;
2108	case VIRTIO_MMIO_REG_DEVICEFEAT_OFF:
2109	{
2110	switch (pVirtio->uDeviceFeaturesSelect)
2111	{
2112	case 0:
2113	*pu32 = pVirtio->uDeviceFeatures & UINT32_C(0xffffffff);
2114	break;
2115	case 1:
2116	*pu32 = pVirtio->uDeviceFeatures >> 32;
2117	break;
2118	default:
2119	LogFunc(("Guest read uDeviceFeatures with out of range selector (%#x), returning 0\n",
2120	pVirtio->uDeviceFeaturesSelect));
2121	rc = VINF_IOM_MMIO_UNUSED_00;
2122	}
2123	break;
2124	}
2125	case VIRTIO_MMIO_REG_QUEUENUMMAX_OFF:
2126	pu32 = VIRTQ_SIZE; /* @todo */
2127	break;
2128	case VIRTIO_MMIO_REG_QUEUERDY_OFF:
2129	{
2130	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2131	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2132	*pu32 = pVirtQueue->uEnable;
2133	break;
2134	}
2135	case VIRTIO_MMIO_REG_INTRSTATUS_OFF:
2136	*pu32 = pVirtio->uISR;
2137	break;
2138	case VIRTIO_MMIO_REG_DEVSTATUS_OFF:
2139	*pu32 = virtioDeviceStatusRead(pVirtio);
2140	break;
2141	case VIRTIO_MMIO_REG_CFGGEN_OFF:
2142	*pu32 = pVirtio->uConfigGeneration;
2143	break;
2144	default:
2145	ASSERT_GUEST_MSG_FAILED(("Bad read access to mapped capabilities region: off=%RGp cb=%u\n", off, cb));
2146	rc = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
2147	"virtioMmioTransportRead: Bad MMIO access to capabilities, offset=%RTiop cb=%08x\n", off, cb);
2148	}
2149
2150	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
2151	return rc;
2152	}
2153
2154	/**
2155	* @callback_method_impl{FNIOMMMIONEWREAD,
2156	* Memory mapped I/O Handler for Virtio over MMIO write operations.}
2157	*/
2158	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioTransportWrite(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void const pv, unsigned cb)
2159	{
2160	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
2161	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2162	RT_NOREF(pvUser);
2163	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatWrite), a);
2164
2165	if (off >= VIRTIO_MMIO_SIZE)
2166	{
2167	#ifdef IN_RING3
2168	/*
2169	* Forward this MMIO write access for client to deal with.
2170	*/
2171	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
2172	return pVirtioCC->pfnDevCapWrite(pDevIns, (uint32_t)off - VIRTIO_MMIO_SIZE, pv, cb);
2173	#else
2174	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
2175	Log6(("%-23s: RING0 => RING3 (demote)\n", __FUNCTION__));
2176	return VINF_IOM_R3_MMIO_WRITE;
2177	#endif
2178	}
2179
2180	/* All accesses below need to be aligned on a 32-bit boundary and must be 32-bit in size. */
2181	ASSERT_GUEST_MSG_RETURN(!(off & 0x3) && cb == sizeof(uint32_t),
2182	("Bad write access: off=%RGp pv=%#p{%.*Rhxs} cb=%u\n", off, pv, cb, pv, cb),
2183	VINF_SUCCESS);
2184
2185	int rc = VINF_SUCCESS;
2186	uint32_t const u32Val = (const uint32_t )pv;
2187	switch (off)
2188	{
2189	case VIRTIO_MMIO_REG_DEVICEFEATSEL_OFF:
2190	{
2191	pVirtio->uDeviceFeaturesSelect = u32Val;
2192	break;
2193	}
2194	case VIRTIO_MMIO_REG_DRIVERFEAT_OFF:
2195	{
2196	switch (pVirtio->uDriverFeaturesSelect)
2197	{
2198	case 0:
2199	pVirtio->uDriverFeatures = (pVirtio->uDriverFeatures & UINT64_C(0xffffffff00000000)) \| u32Val;
2200	pVirtio->fDriverFeaturesWritten \|= DRIVER_FEATURES_0_WRITTEN;
2201	LogFunc(("Set DRIVER_FEATURES_0_WRITTEN. pVirtio->fDriverFeaturesWritten=%d\n", pVirtio->fDriverFeaturesWritten));
2202	if ( (pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_0_AND_1_WRITTEN) == DRIVER_FEATURES_0_AND_1_WRITTEN
2203	&& !(pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_COMPLETE_HANDLED))
2204	#ifdef IN_RING0
2205	return VINF_IOM_R3_MMIO_WRITE;
2206	#endif
2207	#ifdef IN_RING3
2208	virtioR3DoFeaturesCompleteOnceOnly(pVirtio, pVirtioCC);
2209	#endif
2210	break;
2211	case 1:
2212	pVirtio->uDriverFeatures = (pVirtio->uDriverFeatures & UINT64_C(0x00000000ffffffff)) \| ((uint64_t)u32Val << 32);
2213	pVirtio->fDriverFeaturesWritten \|= DRIVER_FEATURES_1_WRITTEN;
2214	LogFunc(("Set DRIVER_FEATURES_1_WRITTEN. pVirtio->fDriverFeaturesWritten=%d\n", pVirtio->fDriverFeaturesWritten));
2215	if ( (pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_0_AND_1_WRITTEN) == DRIVER_FEATURES_0_AND_1_WRITTEN
2216	&& !(pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_COMPLETE_HANDLED))
2217	#ifdef IN_RING0
2218	return VINF_IOM_R3_MMIO_WRITE;
2219	#endif
2220	#ifdef IN_RING3
2221	virtioR3DoFeaturesCompleteOnceOnly(pVirtio, pVirtioCC);
2222	#endif
2223	break;
2224	default:
2225	LogFunc(("Guest wrote uDriverFeatures with out of range selector (%#x), returning 0\n",
2226	pVirtio->uDriverFeaturesSelect));
2227	return VINF_SUCCESS;
2228	}
2229	break;
2230	}
2231	case VIRTIO_MMIO_REG_DRIVERFEATSEL_OFF:
2232	{
2233	pVirtio->uDriverFeaturesSelect = u32Val;
2234	break;
2235	}
2236	case VIRTIO_MMIO_REG_QUEUESEL_OFF:
2237	{
2238	if (u32Val < RT_ELEMENTS(pVirtio->aVirtqueues))
2239	pVirtio->uVirtqSelect = (uint16_t)u32Val;
2240	else
2241	LogFunc(("... WARNING: Guest attempted to write invalid virtq selector (ignoring)\n"));
2242	break;
2243	}
2244	case VIRTIO_MMIO_REG_QUEUENUM_OFF:
2245	{
2246	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2247	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2248	pVirtQueue->uQueueSize = (uint16_t)u32Val;
2249	break;
2250	}
2251	case VIRTIO_MMIO_REG_QUEUERDY_OFF:
2252	{
2253	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2254	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2255	pVirtQueue->uEnable = (uint16_t)u32Val;
2256	break;
2257	}
2258	case VIRTIO_MMIO_REG_QUEUENOTIFY_OFF:
2259	{
2260	virtioCoreVirtqNotified(pDevIns, pVirtio, u32Val, (uint16_t)u32Val);
2261	break;
2262	}
2263	case VIRTIO_MMIO_REG_INTRACK_OFF:
2264	{
2265	pVirtio->uISR &= ~u32Val;
2266	if (!pVirtio->uISR)
2267	virtioLowerInterrupt(pDevIns, 0);
2268	break;
2269	}
2270	case VIRTIO_MMIO_REG_DEVSTATUS_OFF:
2271	{
2272	rc = virtioDeviceStatusWrite(pDevIns, pVirtio, pVirtioCC, (uint8_t)u32Val);
2273	break;
2274	}
2275	case VIRTIO_MMIO_REG_QUEUEALIGN_LEGACY_OFF:
2276	{
2277	/* Written by edk2 even though we don't offer legacy mode, ignore. */
2278	break;
2279	}
2280	case VIRTIO_MMIO_REG_QUEUEDESCLOW_OFF:
2281	{
2282	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2283	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2284	pVirtQueue->GCPhysVirtqDesc = (pVirtQueue->GCPhysVirtqDesc & UINT64_C(0xffffffff00000000)) \| u32Val;
2285	break;
2286	}
2287	case VIRTIO_MMIO_REG_QUEUEDESCHIGH_OFF:
2288	{
2289	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2290	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2291	pVirtQueue->GCPhysVirtqDesc = (pVirtQueue->GCPhysVirtqDesc & UINT64_C(0x00000000ffffffff)) \| ((uint64_t)u32Val << 32);
2292	break;
2293	}
2294	case VIRTIO_MMIO_REG_QUEUEDRVLOW_OFF:
2295	{
2296	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2297	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2298	pVirtQueue->GCPhysVirtqAvail = (pVirtQueue->GCPhysVirtqAvail & UINT64_C(0xffffffff00000000)) \| u32Val;
2299	break;
2300	}
2301	case VIRTIO_MMIO_REG_QUEUEDRVHIGH_OFF:
2302	{
2303	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2304	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2305	pVirtQueue->GCPhysVirtqAvail = (pVirtQueue->GCPhysVirtqAvail & UINT64_C(0x00000000ffffffff)) \| ((uint64_t)u32Val << 32);
2306	break;
2307	}
2308	case VIRTIO_MMIO_REG_QUEUEDEVLOW_OFF:
2309	{
2310	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2311	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2312	pVirtQueue->GCPhysVirtqUsed = (pVirtQueue->GCPhysVirtqUsed & UINT64_C(0xffffffff00000000)) \| u32Val;
2313	break;
2314	}
2315	case VIRTIO_MMIO_REG_QUEUEDEVHIGH_OFF:
2316	{
2317	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2318	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2319	pVirtQueue->GCPhysVirtqUsed = (pVirtQueue->GCPhysVirtqUsed & UINT64_C(0x00000000ffffffff)) \| ((uint64_t)u32Val << 32);
2320	break;
2321	}
2322	default:
2323	ASSERT_GUEST_MSG_FAILED(("Bad write access to mapped capabilities region: off=%RGp pv=%#p{%.*Rhxs} cb=%u\n", off, pv, cb, pv, cb));
2324	rc = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
2325	"virtioMmioTransportWrite: Bad MMIO access to capabilities, offset=%RTiop cb=%08x\n", off, cb);
2326	}
2327
2328	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
2329	return rc;
2330	}
2331
2332
2333	#ifdef IN_RING3
2334
2335	/**
2336	* @callback_method_impl{FNPCICONFIGREAD}
2337	*/
2338	static DECLCALLBACK(VBOXSTRICTRC) virtioR3PciConfigRead(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev,
2339	uint32_t uAddress, unsigned cb, uint32_t *pu32Value)
2340	{
2341	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
2342	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2343	RT_NOREF(pPciDev);
2344
2345	if (uAddress == pVirtio->uPciCfgDataOff)
2346	{
2347	/* See comments in PCI Cfg capability initialization (in capabilities setup section of this code) */
2348	struct virtio_pci_cap *pPciCap = &pVirtioCC->pPciCfgCap->pciCap;
2349	uint32_t uLength = pPciCap->uLength;
2350
2351	Log7Func((" pDevIns=%p pPciDev=%p uAddress=%#x%s cb=%u uLength=%d, bar=%d\n",
2352	pDevIns, pPciDev, uAddress, uAddress < 0x10 ? " " : "", cb, uLength, pPciCap->uBar));
2353
2354	if ( (uLength != 1 && uLength != 2 && uLength != 4)
2355	\|\| pPciCap->uBar != VIRTIO_REGION_PCI_CAP)
2356	{
2357	ASSERT_GUEST_MSG_FAILED(("Guest read virtio_pci_cfg_cap.pci_cfg_data using mismatching config. "
2358	"Ignoring\n"));
2359	*pu32Value = UINT32_MAX;
2360	return VINF_SUCCESS;
2361	}
2362
2363	VBOXSTRICTRC rcStrict = virtioMmioRead(pDevIns, pVirtio, pPciCap->uOffset, pu32Value, cb);
2364	Log7Func((" Guest read virtio_pci_cfg_cap.pci_cfg_data, bar=%d, offset=%d, length=%d, result=0x%x -> %Rrc\n",
2365	pPciCap->uBar, pPciCap->uOffset, uLength, *pu32Value, VBOXSTRICTRC_VAL(rcStrict)));
2366	return rcStrict;
2367	}
2368	Log7Func((" pDevIns=%p pPciDev=%p uAddress=%#x%s cb=%u pu32Value=%p\n",
2369	pDevIns, pPciDev, uAddress, uAddress < 0x10 ? " " : "", cb, pu32Value));
2370	return VINF_PDM_PCI_DO_DEFAULT;
2371	}
2372
2373	/**
2374	* @callback_method_impl{FNPCICONFIGWRITE}
2375	*/
2376	static DECLCALLBACK(VBOXSTRICTRC) virtioR3PciConfigWrite(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev,
2377	uint32_t uAddress, unsigned cb, uint32_t u32Value)
2378	{
2379	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
2380	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2381	RT_NOREF(pPciDev);
2382
2383	Log7Func(("pDevIns=%p pPciDev=%p uAddress=%#x %scb=%u u32Value=%#x\n", pDevIns, pPciDev, uAddress, uAddress < 0xf ? " " : "", cb, u32Value));
2384	if (uAddress == pVirtio->uPciCfgDataOff)
2385	{
2386	/* See comments in PCI Cfg capability initialization (in capabilities setup section of this code) */
2387	struct virtio_pci_cap *pPciCap = &pVirtioCC->pPciCfgCap->pciCap;
2388	uint32_t uLength = pPciCap->uLength;
2389
2390	if ( (uLength != 1 && uLength != 2 && uLength != 4)
2391	\|\| cb != uLength
2392	\|\| pPciCap->uBar != VIRTIO_REGION_PCI_CAP)
2393	{
2394	ASSERT_GUEST_MSG_FAILED(("Guest write virtio_pci_cfg_cap.pci_cfg_data using mismatching config. Ignoring\n"));
2395	return VINF_SUCCESS;
2396	}
2397
2398	VBOXSTRICTRC rcStrict = virtioMmioWrite(pDevIns, pVirtio, pPciCap->uOffset, &u32Value, cb);
2399	Log2Func(("Guest wrote virtio_pci_cfg_cap.pci_cfg_data, bar=%d, offset=%x, length=%x, value=%d -> %Rrc\n",
2400	pPciCap->uBar, pPciCap->uOffset, uLength, u32Value, VBOXSTRICTRC_VAL(rcStrict)));
2401	return rcStrict;
2402	}
2403	return VINF_PDM_PCI_DO_DEFAULT;
2404	}
2405
2406
2407	/*********************************************************************************************************************************
2408	* Saved state (SSM) *
2409	*********************************************************************************************************************************/
2410
2411
2412	/**
2413	* Loads a saved device state (called from device-specific code on SSM final pass)
2414	*
2415	* @param pVirtio Pointer to the shared virtio state.
2416	* @param pHlp The ring-3 device helpers.
2417	* @param pSSM The saved state handle.
2418	* @returns VBox status code.
2419	*/
2420	DECLHIDDEN(int) virtioCoreR3LegacyDeviceLoadExec(PVIRTIOCORE pVirtio, PCPDMDEVHLPR3 pHlp, PSSMHANDLE pSSM,
2421	uint32_t uVersion, uint32_t uVirtioLegacy_3_1_Beta)
2422	{
2423	int rc;
2424	uint32_t uDriverFeaturesLegacy32bit;
2425
2426	rc = pHlp->pfnSSMGetU32( pSSM, &uDriverFeaturesLegacy32bit);
2427	AssertRCReturn(rc, rc);
2428	pVirtio->uDriverFeatures = (uint64_t)uDriverFeaturesLegacy32bit;
2429
2430	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtio->uVirtqSelect);
2431	AssertRCReturn(rc, rc);
2432
2433	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->fDeviceStatus);
2434	AssertRCReturn(rc, rc);
2435
2436	#ifdef LOG_ENABLED
2437	char szOut[80] = { 0 };
2438	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
2439	Log(("Loaded legacy device status = (%s)\n", szOut));
2440	#endif
2441
2442	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->uISR);
2443	AssertRCReturn(rc, rc);
2444
2445	uint32_t cQueues = 3; /* This constant default value copied from earliest v0.9 code */
2446	if (uVersion > uVirtioLegacy_3_1_Beta)
2447	{
2448	rc = pHlp->pfnSSMGetU32(pSSM, &cQueues);
2449	AssertRCReturn(rc, rc);
2450	}
2451
2452	AssertLogRelMsgReturn(cQueues <= VIRTQ_MAX_COUNT, ("%#x\n", cQueues), VERR_SSM_LOAD_CONFIG_MISMATCH);
2453	AssertLogRelMsgReturn(pVirtio->uVirtqSelect < cQueues \|\| (cQueues == 0 && pVirtio->uVirtqSelect),
2454	("uVirtqSelect=%u cQueues=%u\n", pVirtio->uVirtqSelect, cQueues),
2455	VERR_SSM_LOAD_CONFIG_MISMATCH);
2456
2457	Log(("\nRestoring %d legacy-only virtio-net device queues from saved state:\n", cQueues));
2458	for (unsigned uVirtq = 0; uVirtq < cQueues; uVirtq++)
2459	{
2460	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
2461
2462	if (uVirtq == cQueues - 1)
2463	RTStrPrintf(pVirtq->szName, sizeof(pVirtq->szName), "legacy-ctrlq");
2464	else if (uVirtq % 2)
2465	RTStrPrintf(pVirtq->szName, sizeof(pVirtq->szName), "legacy-xmitq<%d>", uVirtq / 2);
2466	else
2467	RTStrPrintf(pVirtq->szName, sizeof(pVirtq->szName), "legacy-recvq<%d>", uVirtq / 2);
2468
2469	rc = pHlp->pfnSSMGetU16(pSSM, &pVirtq->uQueueSize);
2470	AssertRCReturn(rc, rc);
2471
2472	uint32_t uVirtqPfn;
2473	rc = pHlp->pfnSSMGetU32(pSSM, &uVirtqPfn);
2474	AssertRCReturn(rc, rc);
2475
2476	rc = pHlp->pfnSSMGetU16(pSSM, &pVirtq->uAvailIdxShadow);
2477	AssertRCReturn(rc, rc);
2478
2479	rc = pHlp->pfnSSMGetU16(pSSM, &pVirtq->uUsedIdxShadow);
2480	AssertRCReturn(rc, rc);
2481
2482	if (uVirtqPfn)
2483	{
2484	pVirtq->GCPhysVirtqDesc = (uint64_t)uVirtqPfn * VIRTIO_PAGE_SIZE;
2485	pVirtq->GCPhysVirtqAvail = pVirtq->GCPhysVirtqDesc + sizeof(VIRTQ_DESC_T) * pVirtq->uQueueSize;
2486	pVirtq->GCPhysVirtqUsed =
2487	RT_ALIGN(pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[pVirtq->uQueueSize]), VIRTIO_PAGE_SIZE);
2488	pVirtq->uEnable = 1;
2489	}
2490	else
2491	{
2492	LogFunc(("WARNING: QUEUE \"%s\" PAGE NUMBER ZERO IN SAVED STATE\n", pVirtq->szName));
2493	pVirtq->uEnable = 0;
2494	}
2495	pVirtq->uNotifyOffset = 0; /* unused in legacy mode */
2496	pVirtq->uMsixVector = 0; /* unused in legacy mode */
2497	}
2498	pVirtio->fGenUpdatePending = 0; /* unused in legacy mode */
2499	pVirtio->uConfigGeneration = 0; /* unused in legacy mode */
2500	pVirtio->uPciCfgDataOff = 0; /* unused in legacy mode (port I/O used instead) */
2501
2502	return VINF_SUCCESS;
2503	}
2504
2505	/**
2506	* Loads a saved device state (called from device-specific code on SSM final pass)
2507	*
2508	* Note: This loads state saved by a Modern (VirtIO 1.0+) device, of which this transitional device is one,
2509	* and thus supports both legacy and modern guest virtio drivers.
2510	*
2511	* @param pVirtio Pointer to the shared virtio state.
2512	* @param pHlp The ring-3 device helpers.
2513	* @param pSSM The saved state handle.
2514	* @returns VBox status code.
2515	*/
2516	DECLHIDDEN(int) virtioCoreR3ModernDeviceLoadExec(PVIRTIOCORE pVirtio, PCPDMDEVHLPR3 pHlp, PSSMHANDLE pSSM,
2517	uint32_t uVersion, uint32_t uTestVersion, uint32_t cQueues)
2518	{
2519	RT_NOREF2(cQueues, uVersion);
2520	LogFunc(("\n"));
2521	/*
2522	* Check the marker and (embedded) version number.
2523	*/
2524	uint64_t uMarker = 0;
2525	int rc;
2526
2527	rc = pHlp->pfnSSMGetU64(pSSM, &uMarker);
2528	AssertRCReturn(rc, rc);
2529	if (uMarker != VIRTIO_SAVEDSTATE_MARKER)
2530	return pHlp->pfnSSMSetLoadError(pSSM, VERR_SSM_DATA_UNIT_FORMAT_CHANGED, RT_SRC_POS,
2531	N_("Expected marker value %#RX64 found %#RX64 instead"),
2532	VIRTIO_SAVEDSTATE_MARKER, uMarker);
2533	uint32_t uVersionSaved = 0;
2534	rc = pHlp->pfnSSMGetU32(pSSM, &uVersionSaved);
2535	AssertRCReturn(rc, rc);
2536	if (uVersionSaved != uTestVersion)
2537	return pHlp->pfnSSMSetLoadError(pSSM, VERR_SSM_DATA_UNIT_FORMAT_CHANGED, RT_SRC_POS,
2538	N_("Unsupported virtio version: %u"), uVersionSaved);
2539	/*
2540	* Load the state.
2541	*/
2542	rc = pHlp->pfnSSMGetU32( pSSM, &pVirtio->fLegacyDriver);
2543	AssertRCReturn(rc, rc);
2544	rc = pHlp->pfnSSMGetBool( pSSM, &pVirtio->fGenUpdatePending);
2545	AssertRCReturn(rc, rc);
2546	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->fDeviceStatus);
2547	AssertRCReturn(rc, rc);
2548	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->uConfigGeneration);
2549	AssertRCReturn(rc, rc);
2550	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->uPciCfgDataOff);
2551	AssertRCReturn(rc, rc);
2552	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->uISR);
2553	AssertRCReturn(rc, rc);
2554	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtio->uVirtqSelect);
2555	AssertRCReturn(rc, rc);
2556	rc = pHlp->pfnSSMGetU32( pSSM, &pVirtio->uDeviceFeaturesSelect);
2557	AssertRCReturn(rc, rc);
2558	rc = pHlp->pfnSSMGetU32( pSSM, &pVirtio->uDriverFeaturesSelect);
2559	AssertRCReturn(rc, rc);
2560	rc = pHlp->pfnSSMGetU64( pSSM, &pVirtio->uDriverFeatures);
2561	AssertRCReturn(rc, rc);
2562
2563	/** @todo Adapt this loop use cQueues argument instead of static queue count (safely with SSM versioning) */
2564	for (uint32_t i = 0; i < VIRTQ_MAX_COUNT; i++)
2565	{
2566	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[i];
2567	rc = pHlp->pfnSSMGetGCPhys64( pSSM, &pVirtq->GCPhysVirtqDesc);
2568	AssertRCReturn(rc, rc);
2569	rc = pHlp->pfnSSMGetGCPhys64( pSSM, &pVirtq->GCPhysVirtqAvail);
2570	AssertRCReturn(rc, rc);
2571	rc = pHlp->pfnSSMGetGCPhys64( pSSM, &pVirtq->GCPhysVirtqUsed);
2572	AssertRCReturn(rc, rc);
2573	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uNotifyOffset);
2574	AssertRCReturn(rc, rc);
2575	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uMsixVector);
2576	AssertRCReturn(rc, rc);
2577	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uEnable);
2578	AssertRCReturn(rc, rc);
2579	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uQueueSize);
2580	AssertRCReturn(rc, rc);
2581	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uAvailIdxShadow);
2582	AssertRCReturn(rc, rc);
2583	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uUsedIdxShadow);
2584	AssertRCReturn(rc, rc);
2585	rc = pHlp->pfnSSMGetMem( pSSM, pVirtq->szName, sizeof(pVirtq->szName));
2586	AssertRCReturn(rc, rc);
2587	}
2588	return VINF_SUCCESS;
2589	}
2590
2591	/**
2592	* Called from the FNSSMDEVSAVEEXEC function of the device.
2593	*
2594	* @param pVirtio Pointer to the shared virtio state.
2595	* @param pHlp The ring-3 device helpers.
2596	* @param pSSM The saved state handle.
2597	* @returns VBox status code.
2598	*/
2599	DECLHIDDEN(int) virtioCoreR3SaveExec(PVIRTIOCORE pVirtio, PCPDMDEVHLPR3 pHlp, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t cQueues)
2600	{
2601	RT_NOREF(cQueues);
2602	/** @todo figure out a way to save cQueues (with SSM versioning) */
2603
2604	LogFunc(("\n"));
2605	pHlp->pfnSSMPutU64(pSSM, VIRTIO_SAVEDSTATE_MARKER);
2606	pHlp->pfnSSMPutU32(pSSM, uVersion);
2607
2608	pHlp->pfnSSMPutU32( pSSM, pVirtio->fLegacyDriver);
2609	pHlp->pfnSSMPutBool(pSSM, pVirtio->fGenUpdatePending);
2610	pHlp->pfnSSMPutU8( pSSM, pVirtio->fDeviceStatus);
2611	pHlp->pfnSSMPutU8( pSSM, pVirtio->uConfigGeneration);
2612	pHlp->pfnSSMPutU8( pSSM, pVirtio->uPciCfgDataOff);
2613	pHlp->pfnSSMPutU8( pSSM, pVirtio->uISR);
2614	pHlp->pfnSSMPutU16( pSSM, pVirtio->uVirtqSelect);
2615	pHlp->pfnSSMPutU32( pSSM, pVirtio->uDeviceFeaturesSelect);
2616	pHlp->pfnSSMPutU32( pSSM, pVirtio->uDriverFeaturesSelect);
2617	pHlp->pfnSSMPutU64( pSSM, pVirtio->uDriverFeatures);
2618
2619	for (uint32_t i = 0; i < VIRTQ_MAX_COUNT; i++)
2620	{
2621	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[i];
2622
2623	pHlp->pfnSSMPutGCPhys64( pSSM, pVirtq->GCPhysVirtqDesc);
2624	pHlp->pfnSSMPutGCPhys64( pSSM, pVirtq->GCPhysVirtqAvail);
2625	pHlp->pfnSSMPutGCPhys64( pSSM, pVirtq->GCPhysVirtqUsed);
2626	pHlp->pfnSSMPutU16( pSSM, pVirtq->uNotifyOffset);
2627	pHlp->pfnSSMPutU16( pSSM, pVirtq->uMsixVector);
2628	pHlp->pfnSSMPutU16( pSSM, pVirtq->uEnable);
2629	pHlp->pfnSSMPutU16( pSSM, pVirtq->uQueueSize);
2630	pHlp->pfnSSMPutU16( pSSM, pVirtq->uAvailIdxShadow);
2631	pHlp->pfnSSMPutU16( pSSM, pVirtq->uUsedIdxShadow);
2632	int rc = pHlp->pfnSSMPutMem(pSSM, pVirtq->szName, 32);
2633	AssertRCReturn(rc, rc);
2634	}
2635	return VINF_SUCCESS;
2636	}
2637
2638
2639	/*********************************************************************************************************************************
2640	* Device Level *
2641	*********************************************************************************************************************************/
2642
2643	/**
2644	* This must be called by the client to handle VM state changes after the client takes care of its device-specific
2645	* tasks for the state change (i.e. reset, suspend, power-off, resume)
2646	*
2647	* @param pDevIns The device instance.
2648	* @param pVirtio Pointer to the shared virtio state.
2649	*/
2650	DECLHIDDEN(void) virtioCoreR3VmStateChanged(PVIRTIOCORE pVirtio, VIRTIOVMSTATECHANGED enmState)
2651	{
2652	LogFunc(("State changing to %s\n",
2653	virtioCoreGetStateChangeText(enmState)));
2654
2655	switch(enmState)
2656	{
2657	case kvirtIoVmStateChangedReset:
2658	virtioCoreResetAll(pVirtio);
2659	break;
2660	case kvirtIoVmStateChangedSuspend:
2661	break;
2662	case kvirtIoVmStateChangedPowerOff:
2663	break;
2664	case kvirtIoVmStateChangedResume:
2665	for (int uVirtq = 0; uVirtq < VIRTQ_MAX_COUNT; uVirtq++)
2666	{
2667	if ((!pVirtio->fLegacyDriver && pVirtio->aVirtqueues[uVirtq].uEnable)
2668	\| pVirtio->aVirtqueues[uVirtq].GCPhysVirtqDesc)
2669	virtioCoreNotifyGuestDriver(pVirtio->pDevInsR3, pVirtio, uVirtq);
2670	}
2671	break;
2672	default:
2673	LogRelFunc(("Bad enum value"));
2674	return;
2675	}
2676	}
2677
2678	/**
2679	* This should be called from PDMDEVREGR3::pfnDestruct.
2680	*
2681	* @param pDevIns The device instance.
2682	* @param pVirtio Pointer to the shared virtio state.
2683	* @param pVirtioCC Pointer to the ring-3 virtio state.
2684	*/
2685	DECLHIDDEN(void) virtioCoreR3Term(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
2686	{
2687	if (pVirtioCC->pbPrevDevSpecificCfg)
2688	{
2689	RTMemFree(pVirtioCC->pbPrevDevSpecificCfg);
2690	pVirtioCC->pbPrevDevSpecificCfg = NULL;
2691	}
2692
2693	RT_NOREF(pDevIns, pVirtio);
2694	}
2695
2696
2697	/**
2698	* Setup the Virtio device as a PCI device.
2699	*
2700	* @returns VBox status code.
2701	* @param pDevIns Device instance.
2702	* @param pVirtio Pointer to the shared virtio state. This
2703	* must be the first member in the shared
2704	* device instance data!
2705	* @param pVirtioCC Pointer to the ring-3 virtio state. This
2706	* must be the first member in the ring-3
2707	* device instance data!
2708	* @param pPciParams Values to populate industry standard PCI Configuration Space data structure
2709	* @param pcszInstance Device instance name (format-specifier)
2710	* @param cbDevSpecificCfg Size of virtio_pci_device_cap device-specific struct
2711	*/
2712	static int virtioR3PciTransportInit(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC, PVIRTIOPCIPARAMS pPciParams,
2713	const char *pcszInstance, uint16_t cbDevSpecificCfg)
2714	{
2715	/* Set PCI config registers (assume 32-bit mode) */
2716	PPDMPCIDEV pPciDev = pDevIns->apPciDevs[0];
2717	PDMPCIDEV_ASSERT_VALID(pDevIns, pPciDev);
2718
2719	PDMPciDevSetVendorId(pPciDev, DEVICE_PCI_VENDOR_ID_VIRTIO);
2720	PDMPciDevSetDeviceId(pPciDev, pPciParams->uDeviceId);
2721
2722	if (pPciParams->uDeviceId < DEVICE_PCI_DEVICE_ID_VIRTIO_BASE)
2723	/* Transitional devices MUST have a PCI Revision ID of 0. */
2724	PDMPciDevSetRevisionId(pPciDev, DEVICE_PCI_REVISION_ID_VIRTIO_TRANS);
2725	else
2726	/* Non-transitional devices SHOULD have a PCI Revision ID of 1 or higher. */
2727	PDMPciDevSetRevisionId(pPciDev, DEVICE_PCI_REVISION_ID_VIRTIO_V1);
2728
2729	PDMPciDevSetSubSystemId(pPciDev, pPciParams->uSubsystemId);
2730	PDMPciDevSetSubSystemVendorId(pPciDev, DEVICE_PCI_VENDOR_ID_VIRTIO);
2731	PDMPciDevSetClassBase(pPciDev, pPciParams->uClassBase);
2732	PDMPciDevSetClassSub(pPciDev, pPciParams->uClassSub);
2733	PDMPciDevSetClassProg(pPciDev, pPciParams->uClassProg);
2734	PDMPciDevSetInterruptLine(pPciDev, pPciParams->uInterruptLine);
2735	PDMPciDevSetInterruptPin(pPciDev, pPciParams->uInterruptPin);
2736
2737	/* Register PCI device */
2738	int rc = PDMDevHlpPCIRegister(pDevIns, pPciDev);
2739	if (RT_FAILURE(rc))
2740	return PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register PCI Device")); /* can we put params in this error? */
2741
2742	rc = PDMDevHlpPCIInterceptConfigAccesses(pDevIns, pPciDev, virtioR3PciConfigRead, virtioR3PciConfigWrite);
2743	AssertRCReturn(rc, rc);
2744
2745	/* Construct & map PCI vendor-specific capabilities for virtio host negotiation with guest driver */
2746
2747	#define CFG_ADDR_2_IDX(addr) ((uint8_t)(((uintptr_t)(addr) - (uintptr_t)&pPciDev->abConfig[0])))
2748	#define SET_PCI_CAP_LOC(a_pPciDev, a_pCfg, a_LocCap, a_uMmioLengthAlign) \
2749	do { \
2750	(a_LocCap).offMmio = (a_pCfg)->uOffset; \
2751	(a_LocCap).cbMmio = RT_ALIGN_T((a_pCfg)->uLength, a_uMmioLengthAlign, uint16_t); \
2752	(a_LocCap).offPci = (uint16_t)(uintptr_t)((uint8_t *)(a_pCfg) - &(a_pPciDev)->abConfig[0]); \
2753	(a_LocCap).cbPci = (a_pCfg)->uCapLen; \
2754	} while (0)
2755
2756	PVIRTIO_PCI_CAP_T pCfg;
2757	uint32_t cbRegion = 0;
2758
2759	/*
2760	* Common capability (VirtIO 1.0, section 4.1.4.3)
2761	*/
2762	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[0x40];
2763	pCfg->uCfgType = VIRTIO_PCI_CAP_COMMON_CFG;
2764	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2765	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
2766	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
2767	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2768	pCfg->uOffset = RT_ALIGN_32(0, 4); /* Currently 0, but reminder to 32-bit align if changing this */
2769	pCfg->uLength = sizeof(VIRTIO_PCI_COMMON_CFG_T);
2770	cbRegion += pCfg->uLength;
2771	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocCommonCfgCap, 2);
2772	pVirtioCC->pCommonCfgCap = pCfg;
2773
2774	/*
2775	* Notify capability (VirtIO 1.0, section 4.1.4.4).
2776	*
2777	* The size of the spec-defined subregion described by this VirtIO capability is
2778	* based-on the choice of this implementation to make the notification area of each
2779	* queue equal to queue's ordinal position (e.g. queue selector value). The VirtIO
2780	* specification leaves it up to implementation to define queue notification area layout.
2781	*/
2782	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2783	pCfg->uCfgType = VIRTIO_PCI_CAP_NOTIFY_CFG;
2784	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2785	pCfg->uCapLen = sizeof(VIRTIO_PCI_NOTIFY_CAP_T);
2786	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
2787	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2788	pCfg->uOffset = pVirtioCC->pCommonCfgCap->uOffset + pVirtioCC->pCommonCfgCap->uLength;
2789	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
2790	pCfg->uLength = VIRTQ_MAX_COUNT * VIRTIO_NOTIFY_OFFSET_MULTIPLIER + 2; /* will change in VirtIO 1.1 */
2791	cbRegion += pCfg->uLength;
2792	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocNotifyCap, 1);
2793	pVirtioCC->pNotifyCap = (PVIRTIO_PCI_NOTIFY_CAP_T)pCfg;
2794	pVirtioCC->pNotifyCap->uNotifyOffMultiplier = VIRTIO_NOTIFY_OFFSET_MULTIPLIER;
2795
2796	/* ISR capability (VirtIO 1.0, section 4.1.4.5)
2797	*
2798	* VirtIO 1.0 spec says 8-bit, unaligned in MMIO space. The specification example/diagram
2799	* illustrates this capability as 32-bit field with upper bits 'reserved'. Those depictions
2800	* differ. The spec's wording, not the diagram, is seen to work in practice.
2801	*/
2802	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2803	pCfg->uCfgType = VIRTIO_PCI_CAP_ISR_CFG;
2804	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2805	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
2806	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
2807	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2808	pCfg->uOffset = pVirtioCC->pNotifyCap->pciCap.uOffset + pVirtioCC->pNotifyCap->pciCap.uLength;
2809	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
2810	pCfg->uLength = sizeof(uint8_t);
2811	cbRegion += pCfg->uLength;
2812	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocIsrCap, 4);
2813	pVirtioCC->pIsrCap = pCfg;
2814
2815	/* PCI Cfg capability (VirtIO 1.0, section 4.1.4.7)
2816	*
2817	* This capability facilitates early-boot access to this device (BIOS).
2818	* This region isn't page-MMIO mapped. PCI configuration accesses are intercepted,
2819	* wherein uBar, uOffset and uLength are modulated by consumers to locate and read/write
2820	* values in any part of any region. (NOTE: Linux driver doesn't utilize this feature.
2821	* This capability only appears in lspci output on Linux if uLength is non-zero, 4-byte aligned,
2822	* during initialization of linux virtio driver).
2823	*/
2824	pVirtio->uPciCfgDataOff = pCfg->uCapNext + RT_OFFSETOF(VIRTIO_PCI_CFG_CAP_T, uPciCfgData);
2825	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2826	pCfg->uCfgType = VIRTIO_PCI_CAP_PCI_CFG;
2827	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2828	pCfg->uCapLen = sizeof(VIRTIO_PCI_CFG_CAP_T);
2829	pCfg->uCapNext = (pVirtio->fMsiSupport \|\| pVirtioCC->pbDevSpecificCfg) ? CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen : 0;
2830	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2831	pCfg->uOffset = 0;
2832	pCfg->uLength = 4;
2833	cbRegion += pCfg->uLength;
2834	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocPciCfgCap, 1);
2835	pVirtioCC->pPciCfgCap = (PVIRTIO_PCI_CFG_CAP_T)pCfg;
2836
2837	if (pVirtioCC->pbDevSpecificCfg)
2838	{
2839	/* Device-specific config capability (VirtIO 1.0, section 4.1.4.6).
2840	*
2841	* Client defines the device-specific config struct and passes size to virtioCoreR3Init()
2842	* to inform this.
2843	*/
2844	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2845	pCfg->uCfgType = VIRTIO_PCI_CAP_DEVICE_CFG;
2846	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2847	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
2848	pCfg->uCapNext = pVirtio->fMsiSupport ? CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen : 0;
2849	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2850	pCfg->uOffset = pVirtioCC->pIsrCap->uOffset + pVirtioCC->pIsrCap->uLength;
2851	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
2852	pCfg->uLength = cbDevSpecificCfg;
2853	cbRegion += pCfg->uLength;
2854	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocDeviceCap, 4);
2855	pVirtioCC->pDeviceCap = pCfg;
2856	}
2857	else
2858	Assert(pVirtio->LocDeviceCap.cbMmio == 0 && pVirtio->LocDeviceCap.cbPci == 0);
2859
2860	if (pVirtio->fMsiSupport)
2861	{
2862	PDMMSIREG aMsiReg;
2863	RT_ZERO(aMsiReg);
2864	aMsiReg.iMsixCapOffset = pCfg->uCapNext;
2865	aMsiReg.iMsixNextOffset = 0;
2866	aMsiReg.iMsixBar = VIRTIO_REGION_MSIX_CAP;
2867	aMsiReg.cMsixVectors = VBOX_MSIX_MAX_ENTRIES;
2868	rc = PDMDevHlpPCIRegisterMsi(pDevIns, &aMsiReg); /* see MsixR3init() */
2869	if (RT_FAILURE(rc))
2870	{
2871	/* See PDMDevHlp.cpp:pdmR3DevHlp_PCIRegisterMsi */
2872	LogFunc(("Failed to configure MSI-X (%Rrc). Reverting to INTx\n", rc));
2873	pVirtio->fMsiSupport = false;
2874	}
2875	else
2876	Log2Func(("Using MSI-X for guest driver notification\n"));
2877	}
2878	else
2879	LogFunc(("MSI-X not available for VBox, using INTx notification\n"));
2880
2881	/* Set offset to first capability and enable PCI dev capabilities */
2882	PDMPciDevSetCapabilityList(pPciDev, 0x40);
2883	PDMPciDevSetStatus(pPciDev, VBOX_PCI_STATUS_CAP_LIST);
2884
2885	size_t cbSize = RTStrPrintf(pVirtioCC->szMmioName, sizeof(pVirtioCC->szMmioName), "%s (modern)", pcszInstance);
2886	if (cbSize <= 0)
2887	return PDMDEV_SET_ERROR(pDevIns, VERR_BUFFER_OVERFLOW, N_("virtio: out of memory allocating string")); /* can we put params in this error? */
2888
2889	cbSize = RTStrPrintf(pVirtioCC->szPortIoName, sizeof(pVirtioCC->szPortIoName), "%s (legacy)", pcszInstance);
2890	if (cbSize <= 0)
2891	return PDMDEV_SET_ERROR(pDevIns, VERR_BUFFER_OVERFLOW, N_("virtio: out of memory allocating string")); /* can we put params in this error? */
2892
2893	if (pVirtio->fOfferLegacy)
2894	{
2895	/* As a transitional device that supports legacy VirtIO drivers, this VirtIO device generic implementation presents
2896	* legacy driver interface in I/O space at BAR0. The following maps the common (e.g. device independent)
2897	* dev config area as well as device-specific dev config area (whose size is passed to init function of this VirtIO
2898	* generic device code) for access via Port I/O, since legacy drivers (e.g. pre VirtIO 1.0) don't use MMIO callbacks.
2899	* (See VirtIO 1.1, Section 4.1.4.8).
2900	*/
2901	rc = PDMDevHlpPCIIORegionCreateIo(pDevIns, VIRTIO_REGION_LEGACY_IO, sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T) + cbDevSpecificCfg,
2902	virtioLegacyIOPortOut, virtioLegacyIOPortIn, NULL /pvUser/, pVirtioCC->szPortIoName,
2903	NULL /paExtDescs/, &pVirtio->hLegacyIoPorts);
2904	AssertLogRelRCReturn(rc, PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register legacy config in I/O space at BAR0 */")));
2905	}
2906
2907	/* Note: The Linux driver at drivers/virtio/virtio_pci_modern.c tries to map at least a page for the
2908	* 'unknown' device-specific capability without querying the capability to determine size, so pad w/extra page.
2909	*/
2910	rc = PDMDevHlpPCIIORegionCreateMmio(pDevIns, VIRTIO_REGION_PCI_CAP, RT_ALIGN_32(cbRegion + VIRTIO_PAGE_SIZE, VIRTIO_PAGE_SIZE),
2911	PCI_ADDRESS_SPACE_MEM, virtioMmioWrite, virtioMmioRead, pVirtio,
2912	IOMMMIO_FLAGS_READ_PASSTHRU \| IOMMMIO_FLAGS_WRITE_PASSTHRU,
2913	pVirtioCC->szMmioName,
2914	&pVirtio->hMmioPciCap);
2915	AssertLogRelRCReturn(rc, PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register PCI Capabilities address space")));
2916	return VINF_SUCCESS;
2917	}
2918
2919
2920	/**
2921	* Initializes the VirtIO device using the VirtIO over MMIO transport mode.
2922	*
2923	* @returns VBox status code.
2924	* @param pDevIns Device instance.
2925	* @param pVirtio Pointer to the shared virtio state. This
2926	* must be the first member in the shared
2927	* device instance data!
2928	* @param pVirtioCC Pointer to the ring-3 virtio state. This
2929	* must be the first member in the ring-3
2930	* device instance data!
2931	* @param pcszInstance Device instance name (format-specifier)
2932	* @param cbDevSpecificCfg Size of virtio_pci_device_cap device-specific struct
2933	* @param GCPhysMmioBase The physical guest address of the start of the MMIO area.
2934	* @param u16Irq The interrupt number to use for the virtio device.
2935	*/
2936	static int virtioR3MmioTransportInit(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC, const char *pcszInstance,
2937	uint16_t cbDevSpecificCfg, RTGCPHYS GCPhysMmioBase, uint16_t u16Irq)
2938	{
2939	pVirtio->uIrqMmio = u16Irq;
2940
2941	size_t cbSize = RTStrPrintf(pVirtioCC->szMmioName, sizeof(pVirtioCC->szMmioName), "%s (modern)", pcszInstance);
2942	if (cbSize <= 0)
2943	return PDMDEV_SET_ERROR(pDevIns, VERR_BUFFER_OVERFLOW, N_("virtio: out of memory allocating string")); /* can we put params in this error? */
2944
2945	/*
2946	* Register and map the MMIO region.
2947	*/
2948	int rc = PDMDevHlpMmioCreateAndMap(pDevIns, GCPhysMmioBase, RT_ALIGN_32(cbDevSpecificCfg + VIRTIO_MMIO_SIZE, 512),
2949	virtioMmioTransportWrite, virtioMmioTransportRead,
2950	IOMMMIO_FLAGS_READ_PASSTHRU \| IOMMMIO_FLAGS_WRITE_PASSTHRU,
2951	pVirtioCC->szMmioName, &pVirtio->hMmioPciCap);
2952	AssertLogRelRCReturn(rc, PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register PCI Capabilities address space")));
2953	return VINF_SUCCESS;
2954	}
2955
2956
2957	/** API Function: See header file */
2958	DECLHIDDEN(int) virtioCoreR3Init(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC, PVIRTIOPCIPARAMS pPciParams,
2959	const char *pcszInstance, uint64_t fDevSpecificFeatures, uint32_t fOfferLegacy,
2960	void *pvDevSpecificCfg, uint16_t cbDevSpecificCfg)
2961	{
2962	/*
2963	* Virtio state must be the first member of shared device instance data,
2964	* otherwise can't get our bearings in PCI config callbacks.
2965	*/
2966	AssertLogRelReturn(pVirtio == PDMINS_2_DATA(pDevIns, PVIRTIOCORE), VERR_STATE_CHANGED);
2967	AssertLogRelReturn(pVirtioCC == PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC), VERR_STATE_CHANGED);
2968
2969	pVirtio->pDevInsR3 = pDevIns;
2970
2971	/*
2972	* Caller must initialize these.
2973	*/
2974	AssertReturn(pVirtioCC->pfnStatusChanged, VERR_INVALID_POINTER);
2975	AssertReturn(pVirtioCC->pfnVirtqNotified, VERR_INVALID_POINTER);
2976	AssertReturn(VIRTQ_SIZE > 0 && VIRTQ_SIZE <= 32768, VERR_OUT_OF_RANGE); /* VirtIO specification-defined limit */
2977
2978	PCPDMDEVHLPR3 pHlp = pDevIns->pHlpR3;
2979
2980	uint16_t u16Irq = 0;
2981	int rc = pHlp->pfnCFGMQueryU16Def(pDevIns->pCfg, "Irq", &u16Irq, 0);
2982	if (RT_FAILURE(rc))
2983	return PDMDEV_SET_ERROR(pDevIns, rc, N_("Configuration error: Failed to get the \"Irq\" value"));
2984
2985	RTGCPHYS GCPhysMmioBase = 0;
2986	rc = pHlp->pfnCFGMQueryU64Def(pDevIns->pCfg, "MmioBase", &GCPhysMmioBase, NIL_RTGCPHYS);
2987	if (RT_FAILURE(rc))
2988	return PDMDEV_SET_ERROR(pDevIns, rc,
2989	N_("Configuration error: Failed to get the \"MmioBase\" value"));
2990
2991	#if 0 /* Until pdmR3DvHlp_PCISetIrq() impl is fixed and Assert that limits vec to 0 is removed
2992	* VBox legacy MSI support has not been implemented yet
2993	*/
2994	# ifdef VBOX_WITH_MSI_DEVICES
2995	pVirtio->fMsiSupport = true;
2996	# endif
2997	#endif
2998
2999	/*
3000	* Host features (presented as a buffet for guest to select from)
3001	* include both dev-specific features & reserved dev-independent features (bitmask).
3002	*/
3003	pVirtio->uDeviceType = pPciParams->uDeviceType;
3004	pVirtio->uDeviceFeatures = VIRTIO_F_VERSION_1
3005	\| VIRTIO_DEV_INDEPENDENT_FEATURES_OFFERED
3006	\| fDevSpecificFeatures;
3007
3008	pVirtio->fLegacyDriver = pVirtio->fOfferLegacy = fOfferLegacy;
3009
3010	RTStrCopy(pVirtio->szInstance, sizeof(pVirtio->szInstance), pcszInstance);
3011	pVirtioCC->cbDevSpecificCfg = cbDevSpecificCfg;
3012	pVirtioCC->pbDevSpecificCfg = (uint8_t *)pvDevSpecificCfg;
3013	pVirtioCC->pbPrevDevSpecificCfg = (uint8_t *)RTMemDup(pvDevSpecificCfg, cbDevSpecificCfg);
3014	AssertLogRelReturn(pVirtioCC->pbPrevDevSpecificCfg, VERR_NO_MEMORY);
3015
3016	if (GCPhysMmioBase != NIL_RTGCPHYS)
3017	rc = virtioR3MmioTransportInit(pDevIns, pVirtio, pVirtioCC, pcszInstance, cbDevSpecificCfg,
3018	GCPhysMmioBase, u16Irq);
3019	else
3020	rc = virtioR3PciTransportInit(pDevIns, pVirtio, pVirtioCC, pPciParams, pcszInstance, cbDevSpecificCfg);
3021	AssertLogRelRCReturn(rc, rc);
3022
3023	/*
3024	* Statistics.
3025	*/
3026	# ifdef VBOX_WITH_STATISTICS
3027	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsAllocated, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
3028	"Total number of allocated descriptor chains", "DescChainsAllocated");
3029	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsFreed, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
3030	"Total number of freed descriptor chains", "DescChainsFreed");
3031	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsSegsIn, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
3032	"Total number of inbound segments", "DescChainsSegsIn");
3033	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsSegsOut, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
3034	"Total number of outbound segments", "DescChainsSegsOut");
3035	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatReadR3, STAMTYPE_PROFILE, "IO/ReadR3", STAMUNIT_TICKS_PER_CALL, "Profiling IO reads in R3");
3036	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatReadR0, STAMTYPE_PROFILE, "IO/ReadR0", STAMUNIT_TICKS_PER_CALL, "Profiling IO reads in R0");
3037	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatReadRC, STAMTYPE_PROFILE, "IO/ReadRC", STAMUNIT_TICKS_PER_CALL, "Profiling IO reads in RC");
3038	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatWriteR3, STAMTYPE_PROFILE, "IO/WriteR3", STAMUNIT_TICKS_PER_CALL, "Profiling IO writes in R3");
3039	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatWriteR0, STAMTYPE_PROFILE, "IO/WriteR0", STAMUNIT_TICKS_PER_CALL, "Profiling IO writes in R0");
3040	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatWriteRC, STAMTYPE_PROFILE, "IO/WriteRC", STAMUNIT_TICKS_PER_CALL, "Profiling IO writes in RC");
3041	# endif /* VBOX_WITH_STATISTICS */
3042
3043	return VINF_SUCCESS;
3044	}
3045
3046	#else /* !IN_RING3 */
3047
3048	/**
3049	* Sets up the core ring-0/raw-mode virtio bits.
3050	*
3051	* @returns VBox status code.
3052	* @param pDevIns The device instance.
3053	* @param pVirtio Pointer to the shared virtio state. This must be the first
3054	* member in the shared device instance data!
3055	*/
3056	DECLHIDDEN(int) virtioCoreRZInit(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio)
3057	{
3058	AssertLogRelReturn(pVirtio == PDMINS_2_DATA(pDevIns, PVIRTIOCORE), VERR_STATE_CHANGED);
3059	int rc;
3060	#ifdef FUTURE_OPTIMIZATION
3061	rc = PDMDevHlpSetDeviceCritSect(pDevIns, PDMDevHlpCritSectGetNop(pDevIns));
3062	AssertRCReturn(rc, rc);
3063	#endif
3064
3065	if (pVirtio->uIrqMmio != 0)
3066	{
3067	rc = PDMDevHlpMmioSetUpContext(pDevIns, pVirtio->hMmioPciCap, virtioMmioTransportWrite, virtioMmioTransportRead, pVirtio);
3068	AssertRCReturn(rc, rc);
3069	}
3070	else
3071	{
3072	rc = PDMDevHlpMmioSetUpContext(pDevIns, pVirtio->hMmioPciCap, virtioMmioWrite, virtioMmioRead, pVirtio);
3073	AssertRCReturn(rc, rc);
3074
3075	if (pVirtio->fOfferLegacy)
3076	{
3077	rc = PDMDevHlpIoPortSetUpContext(pDevIns, pVirtio->hLegacyIoPorts, virtioLegacyIOPortOut, virtioLegacyIOPortIn, NULL /pvUser/);
3078	AssertRCReturn(rc, rc);
3079	}
3080	}
3081	return rc;
3082	}
3083
3084	#endif /* !IN_RING3 */
3085

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Devices/VirtIO/VirtioCore.cpp@ 102468

Download in other formats: