VirtioCore.cpp@ 105668

Last change on this file since 105668 was 105208, checked in by vboxsync, 5 months ago
Devices/VirtIO: Don't try copying any potential return data if the guest didn't set up an IN buffer, bugref:10712
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 138.0 KB

Line
1	/* $Id: VirtioCore.cpp 105208 2024-07-09 07:12:47Z vboxsync $ */
2
3	/** @file
4	* VirtioCore - Virtio Core (PCI, feature & config mgt, queue mgt & proxy, notification mgt)
5	*/
6
7	/*
8	* Copyright (C) 2009-2023 Oracle and/or its affiliates.
9	*
10	* This file is part of VirtualBox base platform packages, as
11	* available from https://www.virtualbox.org.
12	*
13	* This program is free software; you can redistribute it and/or
14	* modify it under the terms of the GNU General Public License
15	* as published by the Free Software Foundation, in version 3 of the
16	* License.
17	*
18	* This program is distributed in the hope that it will be useful, but
19	* WITHOUT ANY WARRANTY; without even the implied warranty of
20	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
21	* General Public License for more details.
22	*
23	* You should have received a copy of the GNU General Public License
24	* along with this program; if not, see <https://www.gnu.org/licenses>.
25	*
26	* SPDX-License-Identifier: GPL-3.0-only
27	*/
28
29
30	/*********************************************************************************************************************************
31	* Header Files *
32	*********************************************************************************************************************************/
33	#define LOG_GROUP LOG_GROUP_DEV_VIRTIO
34
35	#include <iprt/assert.h>
36	#include <iprt/uuid.h>
37	#include <iprt/mem.h>
38	#include <iprt/sg.h>
39	#include <iprt/assert.h>
40	#include <iprt/string.h>
41	#include <iprt/param.h>
42	#include <iprt/types.h>
43	#include <VBox/log.h>
44	#include <VBox/msi.h>
45	#include <iprt/types.h>
46	#include <VBox/AssertGuest.h>
47	#include <VBox/vmm/pdmdev.h>
48	#include "VirtioCore.h"
49
50
51	/*********************************************************************************************************************************
52	* Defined Constants And Macros *
53	*********************************************************************************************************************************/
54
55	#define INSTANCE(a_pVirtio) ((a_pVirtio)->szInstance)
56	#define VIRTQNAME(a_pVirtio, a_uVirtq) ((a_pVirtio)->aVirtqueues[(a_uVirtq)].szName)
57
58	#define IS_VIRTQ_EMPTY(pDevIns, pVirtio, pVirtq) \
59	(virtioCoreVirtqAvailCnt(pDevIns, pVirtio, pVirtq) == 0)
60
61	#define IS_DRIVER_OK(a_pVirtio) ((a_pVirtio)->fDeviceStatus & VIRTIO_STATUS_DRIVER_OK)
62	#define WAS_DRIVER_OK(a_pVirtio) ((a_pVirtio)->fPrevDeviceStatus & VIRTIO_STATUS_DRIVER_OK)
63
64	/**
65	* These defines are used to track guest virtio-net driver writing driver features accepted flags
66	* in two 32-bit operations (in arbitrary order), and one bit dedicated to ensured 'features complete'
67	* is handled once.
68	*/
69	#define DRIVER_FEATURES_0_WRITTEN 1 /*< fDriverFeatures[0] written by guest virtio-net /
70	#define DRIVER_FEATURES_1_WRITTEN 2 /*< fDriverFeatures[1] written by guest virtio-net /
71	#define DRIVER_FEATURES_0_AND_1_WRITTEN 3 /*< Both 32-bit parts of fDriverFeatures[] written /
72	#define DRIVER_FEATURES_COMPLETE_HANDLED 4 /*< Features negotiation complete handler called /
73
74	/**
75	* This macro returns true if the @a a_offAccess and access length (@a
76	* a_cbAccess) are within the range of the mapped capability struct described by
77	* @a a_LocCapData.
78	*
79	* @param[in] a_offAccess Input: The offset into the MMIO bar of the access.
80	* @param[in] a_cbAccess Input: The access size.
81	* @param[out] a_offsetIntoCap Output: uint32_t variable to return the intra-capability offset into.
82	* @param[in] a_LocCapData Input: The capability location info.
83	*/
84	#define MATCHES_VIRTIO_CAP_STRUCT(a_offAccess, a_cbAccess, a_offsetIntoCap, a_LocCapData) \
85	( ((a_offsetIntoCap) = (uint32_t)((a_offAccess) - (a_LocCapData).offMmio)) < (uint32_t)(a_LocCapData).cbMmio \
86	&& (a_offsetIntoCap) + (uint32_t)(a_cbAccess) <= (uint32_t)(a_LocCapData).cbMmio )
87
88
89	/*********************************************************************************************************************************
90	* Structures and Typedefs *
91	*********************************************************************************************************************************/
92
93	/** @name virtq related flags
94	* @{ */
95	#define VIRTQ_DESC_F_NEXT 1 /*< Indicates this descriptor chains to next /
96	#define VIRTQ_DESC_F_WRITE 2 /*< Marks buffer as write-only (default ro) /
97	#define VIRTQ_DESC_F_INDIRECT 4 /*< Buffer is list of buffer descriptors /
98
99	#define VIRTQ_USED_F_NO_NOTIFY 1 /*< Dev to Drv: Don't notify when buf added /
100	#define VIRTQ_AVAIL_F_NO_INTERRUPT 1 /*< Drv to Dev: Don't notify when buf eaten /
101	/** @} */
102
103	/**
104	* virtq-related structs
105	* (struct names follow VirtIO 1.0 spec, field names use VBox styled naming, w/respective spec'd name in comments)
106	*/
107	typedef struct virtq_desc
108	{
109	uint64_t GCPhysBuf; /*< addr GC Phys. address of buffer /
110	uint32_t cb; /*< len Buffer length /
111	uint16_t fFlags; /*< flags Buffer specific flags /
112	uint16_t uDescIdxNext; /*< next Idx set if VIRTIO_DESC_F_NEXT /
113	} VIRTQ_DESC_T, *PVIRTQ_DESC_T;
114
115	typedef struct virtq_avail
116	{
117	uint16_t fFlags; /*< flags avail ring guest-to-host flags /
118	uint16_t uIdx; /*< idx Index of next free ring slot /
119	RT_FLEXIBLE_ARRAY_EXTENSION
120	uint16_t auRing[RT_FLEXIBLE_ARRAY]; /*< ring Ring: avail drv to dev bufs /
121	//uint16_t uUsedEventIdx; /*< used_event (if VIRTQ_USED_F_EVENT_IDX) /
122	} VIRTQ_AVAIL_T, *PVIRTQ_AVAIL_T;
123
124	typedef struct virtq_used_elem
125	{
126	uint32_t uDescIdx; /*< idx Start of used desc chain /
127	uint32_t cbElem; /*< len Total len of used desc chain /
128	} VIRTQ_USED_ELEM_T;
129
130	typedef struct virt_used
131	{
132	uint16_t fFlags; /*< flags used ring host-to-guest flags /
133	uint16_t uIdx; /*< idx Index of next ring slot /
134	RT_FLEXIBLE_ARRAY_EXTENSION
135	VIRTQ_USED_ELEM_T aRing[RT_FLEXIBLE_ARRAY]; /*< ring Ring: used dev to drv bufs /
136	//uint16_t uAvailEventIdx; /*< avail_event if (VIRTQ_USED_F_EVENT_IDX) /
137	} VIRTQ_USED_T, *PVIRTQ_USED_T;
138
139	DECLHIDDEN(const char *) virtioCoreGetStateChangeText(VIRTIOVMSTATECHANGED enmState)
140	{
141	switch (enmState)
142	{
143	case kvirtIoVmStateChangedReset: return "VM RESET";
144	case kvirtIoVmStateChangedSuspend: return "VM SUSPEND";
145	case kvirtIoVmStateChangedPowerOff: return "VM POWER OFF";
146	case kvirtIoVmStateChangedResume: return "VM RESUME";
147	default: return "<BAD ENUM>";
148	}
149	}
150
151	/* Internal Functions */
152
153	static void virtioCoreNotifyGuestDriver(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq);
154	static int virtioNudgeGuest(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint8_t uCause, uint16_t uVec);
155
156	#ifdef IN_RING3
157	# ifdef LOG_ENABLED
158	DECLINLINE(uint16_t) virtioCoreR3CountPendingBufs(uint16_t uRingIdx, uint16_t uShadowIdx, uint16_t uQueueSize)
159	{
160	if (uShadowIdx == uRingIdx)
161	return 0;
162	else
163	if (uShadowIdx > uRingIdx)
164	return uShadowIdx - uRingIdx;
165	return uQueueSize - (uRingIdx - uShadowIdx);
166	}
167	# endif
168	#endif
169	/** @name Internal queue operations
170	* @{ */
171
172	/**
173	* Accessor for virtq descriptor
174	*/
175	#ifdef IN_RING3
176	DECLINLINE(void) virtioReadDesc(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq,
177	uint32_t idxDesc, PVIRTQ_DESC_T pDesc)
178	{
179	/*
180	* Shut up assertion for legacy virtio-net driver in FreeBSD up to 12.3 (see virtioCoreR3VirtqUsedBufPut()
181	* for more information).
182	*/
183	AssertMsg( IS_DRIVER_OK(pVirtio)
184	\|\| ( pVirtio->fLegacyDriver
185	&& pVirtq->GCPhysVirtqDesc),
186	("Called with guest driver not ready\n"));
187	uint16_t const cVirtqItems = RT_MAX(pVirtq->uQueueSize, 1); /* Make sure to avoid div-by-zero. */
188
189	virtioCoreGCPhysRead(pVirtio, pDevIns,
190	pVirtq->GCPhysVirtqDesc + sizeof(VIRTQ_DESC_T) * (idxDesc % cVirtqItems),
191	pDesc, sizeof(VIRTQ_DESC_T));
192	}
193	#endif
194
195	/**
196	* Accessors for virtq avail ring
197	*/
198	#ifdef IN_RING3
199	DECLINLINE(uint16_t) virtioReadAvailDescIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq, uint32_t availIdx)
200	{
201	uint16_t uDescIdx;
202
203	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
204	uint16_t const cVirtqItems = RT_MAX(pVirtq->uQueueSize, 1); /* Make sure to avoid div-by-zero. */
205	virtioCoreGCPhysRead(pVirtio, pDevIns,
206	pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[availIdx % cVirtqItems]),
207	&uDescIdx, sizeof(uDescIdx));
208	return uDescIdx;
209	}
210
211	DECLINLINE(uint16_t) virtioReadAvailUsedEvent(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
212	{
213	uint16_t uUsedEventIdx;
214	/* VirtIO 1.0 uUsedEventIdx (used_event) immediately follows ring */
215	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
216	virtioCoreGCPhysRead(pVirtio, pDevIns,
217	pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[pVirtq->uQueueSize]),
218	&uUsedEventIdx, sizeof(uUsedEventIdx));
219	return uUsedEventIdx;
220	}
221	#endif
222
223	DECLINLINE(uint16_t) virtioReadAvailRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
224	{
225	uint16_t uIdx = 0;
226	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
227	virtioCoreGCPhysRead(pVirtio, pDevIns,
228	pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF(VIRTQ_AVAIL_T, uIdx),
229	&uIdx, sizeof(uIdx));
230	return uIdx;
231	}
232
233	DECLINLINE(uint16_t) virtioReadAvailRingFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
234	{
235	uint16_t fFlags = 0;
236	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
237	virtioCoreGCPhysRead(pVirtio, pDevIns,
238	pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF(VIRTQ_AVAIL_T, fFlags),
239	&fFlags, sizeof(fFlags));
240	return fFlags;
241	}
242
243	/** @} */
244
245	/** @name Accessors for virtq used ring
246	* @{
247	*/
248
249	#ifdef IN_RING3
250	DECLINLINE(void) virtioWriteUsedElem(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq,
251	uint32_t usedIdx, uint32_t uDescIdx, uint32_t uLen)
252	{
253	VIRTQ_USED_ELEM_T elem = { uDescIdx, uLen };
254	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
255	uint16_t const cVirtqItems = RT_MAX(pVirtq->uQueueSize, 1); /* Make sure to avoid div-by-zero. */
256	virtioCoreGCPhysWrite(pVirtio, pDevIns,
257	pVirtq->GCPhysVirtqUsed
258	+ RT_UOFFSETOF_DYN(VIRTQ_USED_T, aRing[usedIdx % cVirtqItems]),
259	&elem, sizeof(elem));
260	}
261
262	DECLINLINE(void) virtioWriteUsedRingFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq, uint16_t fFlags)
263	{
264	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
265	RT_UNTRUSTED_VALIDATED_FENCE(); /* VirtIO 1.0, Section 3.2.1.4.1 */
266	virtioCoreGCPhysWrite(pVirtio, pDevIns,
267	pVirtq->GCPhysVirtqUsed + RT_UOFFSETOF(VIRTQ_USED_T, fFlags),
268	&fFlags, sizeof(fFlags));
269	}
270	#endif
271
272	DECLINLINE(void) virtioWriteUsedRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq, uint16_t uIdx)
273	{
274	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
275	RT_UNTRUSTED_VALIDATED_FENCE(); /* VirtIO 1.0, Section 3.2.1.4.1 */
276	virtioCoreGCPhysWrite(pVirtio, pDevIns,
277	pVirtq->GCPhysVirtqUsed + RT_UOFFSETOF(VIRTQ_USED_T, uIdx),
278	&uIdx, sizeof(uIdx));
279	}
280
281	#ifdef IN_RING3
282	DECLINLINE(uint16_t) virtioReadUsedRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
283	{
284	uint16_t uIdx = 0;
285	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
286	virtioCoreGCPhysRead(pVirtio, pDevIns,
287	pVirtq->GCPhysVirtqUsed + RT_UOFFSETOF(VIRTQ_USED_T, uIdx),
288	&uIdx, sizeof(uIdx));
289	return uIdx;
290	}
291
292	DECLINLINE(uint16_t) virtioReadUsedRingFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
293	{
294	uint16_t fFlags = 0;
295	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
296	virtioCoreGCPhysRead(pVirtio, pDevIns,
297	pVirtq->GCPhysVirtqUsed + RT_UOFFSETOF(VIRTQ_USED_T, fFlags),
298	&fFlags, sizeof(fFlags));
299	return fFlags;
300	}
301
302	DECLINLINE(void) virtioWriteUsedAvailEvent(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq, uint32_t uAvailEventIdx)
303	{
304	/** VirtIO 1.0 uAvailEventIdx (avail_event) immediately follows ring */
305	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
306	virtioCoreGCPhysWrite(pVirtio, pDevIns,
307	pVirtq->GCPhysVirtqUsed
308	+ RT_UOFFSETOF_DYN(VIRTQ_USED_T, aRing[pVirtq->uQueueSize]),
309	&uAvailEventIdx, sizeof(uAvailEventIdx));
310	}
311	#endif
312	/** @} */
313
314
315	DECLINLINE(uint16_t) virtioCoreVirtqAvailCnt(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
316	{
317	uint16_t uIdxActual = virtioReadAvailRingIdx(pDevIns, pVirtio, pVirtq);
318	uint16_t uIdxShadow = pVirtq->uAvailIdxShadow;
319	uint16_t uIdxDelta;
320
321	if (uIdxActual < uIdxShadow)
322	uIdxDelta = (uIdxActual + pVirtq->uQueueSize) - uIdxShadow;
323	else
324	uIdxDelta = uIdxActual - uIdxShadow;
325
326	return uIdxDelta;
327	}
328	/**
329	* Get count of new (e.g. pending) elements in available ring.
330	*
331	* @param pDevIns The device instance.
332	* @param pVirtio Pointer to the shared virtio state.
333	* @param uVirtq Virtq number
334	*
335	* @returns how many entries have been added to ring as a delta of the consumer's
336	* avail index and the queue's guest-side current avail index.
337	*/
338	DECLHIDDEN(uint16_t) virtioCoreVirtqAvailBufCount(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq)
339	{
340	AssertMsgReturn(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues), ("uVirtq out of range"), 0);
341	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
342
343	if (!IS_DRIVER_OK(pVirtio))
344	{
345	LogRelFunc(("Driver not ready\n"));
346	return 0;
347	}
348	if (!pVirtio->fLegacyDriver && !pVirtq->uEnable)
349	{
350	LogRelFunc(("virtq: %s not enabled\n", VIRTQNAME(pVirtio, uVirtq)));
351	return 0;
352	}
353	return virtioCoreVirtqAvailCnt(pDevIns, pVirtio, pVirtq);
354	}
355
356	#ifdef IN_RING3
357
358	static void virtioCoreR3FeatureDump(VIRTIOCORE pVirtio, PCDBGFINFOHLP pHlp, const VIRTIO_FEATURES_LIST s_aFeatures, int cFeatures, int fBanner)
359	{
360	#define MAXLINE 80
361	/* Display as a single buf to prevent interceding log messages */
362	uint16_t cbBuf = cFeatures * 132;
363	char pszBuf = (char )RTMemAllocZ(cbBuf);
364	Assert(pszBuf);
365	char *cp = pszBuf;
366	for (int i = 0; i < cFeatures; ++i)
367	{
368	bool isOffered = RT_BOOL(pVirtio->uDeviceFeatures & s_aFeatures[i].fFeatureBit);
369	bool isNegotiated = RT_BOOL(pVirtio->uDriverFeatures & s_aFeatures[i].fFeatureBit);
370	cp += RTStrPrintf(cp, cbBuf - (cp - pszBuf), " %s %s %s",
371	isOffered ? "+" : "-", isNegotiated ? "x" : " ", s_aFeatures[i].pcszDesc);
372	}
373	if (pHlp) {
374	if (fBanner)
375	pHlp->pfnPrintf(pHlp, "VirtIO Features Configuration\n\n"
376	" Offered Accepted Feature Description\n"
377	" ------- -------- ------- -----------\n");
378	pHlp->pfnPrintf(pHlp, "%s\n", pszBuf);
379	}
380	#ifdef LOG_ENABLED
381	else
382	{
383	if (fBanner)
384	Log(("VirtIO Features Configuration\n\n"
385	" Offered Accepted Feature Description\n"
386	" ------- -------- ------- -----------\n"));
387	Log(("%s\n", pszBuf));
388	}
389	#endif
390	RTMemFree(pszBuf);
391	}
392
393	/** API Function: See header file*/
394	DECLHIDDEN(void) virtioCorePrintDeviceFeatures(VIRTIOCORE *pVirtio, PCDBGFINFOHLP pHlp,
395	const VIRTIO_FEATURES_LIST *s_aDevSpecificFeatures, int cFeatures) {
396	virtioCoreR3FeatureDump(pVirtio, pHlp, s_aCoreFeatures, RT_ELEMENTS(s_aCoreFeatures), 1 /fBanner /);
397	virtioCoreR3FeatureDump(pVirtio, pHlp, s_aDevSpecificFeatures, cFeatures, 0 /fBanner /);
398	}
399
400	#endif
401
402	#ifdef LOG_ENABLED
403
404	/** API Function: See header file */
405	DECLHIDDEN(void) virtioCoreHexDump(uint8_t pv, uint32_t cb, uint32_t uBase, const char pszTitle)
406	{
407	#define ADJCURSOR(cb) pszOut += cb; cbRemain -= cb;
408	size_t cbPrint = 0, cbRemain = ((cb / 16) + 1) * 80;
409	char pszBuf = (char )RTMemAllocZ(cbRemain), *pszOut = pszBuf;
410	AssertMsgReturnVoid(pszBuf, ("Out of Memory"));
411	if (pszTitle)
412	{
413	cbPrint = RTStrPrintf(pszOut, cbRemain, "%s [%d bytes]:\n", pszTitle, cb);
414	ADJCURSOR(cbPrint);
415	}
416	for (uint32_t row = 0; row < RT_MAX(1, (cb / 16) + 1) && row * 16 < cb; row++)
417	{
418	cbPrint = RTStrPrintf(pszOut, cbRemain, "%04x: ", row * 16 + uBase); /* line address */
419	ADJCURSOR(cbPrint);
420	for (uint8_t col = 0; col < 16; col++)
421	{
422	uint32_t idx = row * 16 + col;
423	if (idx >= cb)
424	cbPrint = RTStrPrintf(pszOut, cbRemain, "-- %s", (col + 1) % 8 ? "" : " ");
425	else
426	cbPrint = RTStrPrintf(pszOut, cbRemain, "%02x %s", pv[idx], (col + 1) % 8 ? "" : " ");
427	ADJCURSOR(cbPrint);
428	}
429	for (uint32_t idx = row * 16; idx < row * 16 + 16; idx++)
430	{
431	cbPrint = RTStrPrintf(pszOut, cbRemain, "%c", (idx >= cb) ? ' ' : (pv[idx] >= 0x20 && pv[idx] <= 0x7e ? pv[idx] : '.'));
432	ADJCURSOR(cbPrint);
433	}
434	*pszOut++ = '\n';
435	--cbRemain;
436	}
437	Log(("%s\n", pszBuf));
438	RTMemFree(pszBuf);
439	RT_NOREF2(uBase, pv);
440	#undef ADJCURSOR
441	}
442
443	/* API FUnction: See header file */
444	DECLHIDDEN(void) virtioCoreGCPhysHexDump(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, uint16_t cb, uint32_t uBase, const char *pszTitle)
445	{
446	PVIRTIOCORE pVirtio = PDMDEVINS_2_DATA(pDevIns, PVIRTIOCORE);
447	#define ADJCURSOR(cb) pszOut += cb; cbRemain -= cb;
448	size_t cbPrint = 0, cbRemain = ((cb / 16) + 1) * 80;
449	char pszBuf = (char )RTMemAllocZ(cbRemain), *pszOut = pszBuf;
450	AssertMsgReturnVoid(pszBuf, ("Out of Memory"));
451	if (pszTitle)
452	{
453	cbPrint = RTStrPrintf(pszOut, cbRemain, "%s [%d bytes]:\n", pszTitle, cb);
454	ADJCURSOR(cbPrint);
455	}
456	for (uint16_t row = 0; row < (uint16_t)RT_MAX(1, (cb / 16) + 1) && row * 16 < cb; row++)
457	{
458	uint8_t c;
459	cbPrint = RTStrPrintf(pszOut, cbRemain, "%04x: ", row * 16 + uBase); /* line address */
460	ADJCURSOR(cbPrint);
461	for (uint8_t col = 0; col < 16; col++)
462	{
463	uint32_t idx = row * 16 + col;
464	virtioCoreGCPhysRead(pVirtio, pDevIns, GCPhys + idx, &c, 1);
465	if (idx >= cb)
466	cbPrint = RTStrPrintf(pszOut, cbRemain, "-- %s", (col + 1) % 8 ? "" : " ");
467	else
468	cbPrint = RTStrPrintf(pszOut, cbRemain, "%02x %s", c, (col + 1) % 8 ? "" : " ");
469	ADJCURSOR(cbPrint);
470	}
471	for (uint16_t idx = row * 16; idx < row * 16 + 16; idx++)
472	{
473	virtioCoreGCPhysRead(pVirtio, pDevIns, GCPhys + idx, &c, 1);
474	cbPrint = RTStrPrintf(pszOut, cbRemain, "%c", (idx >= cb) ? ' ' : (c >= 0x20 && c <= 0x7e ? c : '.'));
475	ADJCURSOR(cbPrint);
476	}
477	*pszOut++ = '\n';
478	--cbRemain;
479	}
480	Log(("%s\n", pszBuf));
481	RTMemFree(pszBuf);
482	RT_NOREF(uBase);
483	#undef ADJCURSOR
484	}
485
486
487	/** API function: See header file */
488	DECLHIDDEN(void) virtioCoreLogMappedIoValue(const char pszFunc, const char pszMember, uint32_t uMemberSize,
489	const void *pv, uint32_t cb, uint32_t uOffset, int fWrite,
490	int fHasIndex, uint32_t idx)
491	{
492	if (LogIs6Enabled())
493	{
494	char szIdx[16];
495	if (fHasIndex)
496	RTStrPrintf(szIdx, sizeof(szIdx), "[%d]", idx);
497	else
498	szIdx[0] = '\0';
499
500	if (cb == 1 \|\| cb == 2 \|\| cb == 4 \|\| cb == 8)
501	{
502	char szDepiction[64];
503	size_t cchDepiction;
504	if (uOffset != 0 \|\| cb != uMemberSize) /* display bounds if partial member access */
505	cchDepiction = RTStrPrintf(szDepiction, sizeof(szDepiction), "%s%s[%d:%d]",
506	pszMember, szIdx, uOffset, uOffset + cb - 1);
507	else
508	cchDepiction = RTStrPrintf(szDepiction, sizeof(szDepiction), "%s%s", pszMember, szIdx);
509
510	/* padding */
511	if (cchDepiction < 30)
512	szDepiction[cchDepiction++] = ' ';
513	while (cchDepiction < 30)
514	szDepiction[cchDepiction++] = '.';
515	szDepiction[cchDepiction] = '\0';
516
517	RTUINT64U uValue;
518	uValue.u = 0;
519	memcpy(uValue.au8, pv, cb);
520	Log6(("%-23s: Guest %s %s %#0*RX64\n",
521	pszFunc, fWrite ? "wrote" : "read ", szDepiction, 2 + cb * 2, uValue.u));
522	}
523	else /* odd number or oversized access, ... log inline hex-dump style */
524	{
525	Log6(("%-23s: Guest %s %s%s[%d:%d]: %.*Rhxs\n",
526	pszFunc, fWrite ? "wrote" : "read ", pszMember,
527	szIdx, uOffset, uOffset + cb, cb, pv));
528	}
529	}
530	RT_NOREF2(fWrite, pszFunc);
531	}
532
533	/**
534	* Log MMIO-mapped Virtio fDeviceStatus register bitmask, naming the bits
535	*/
536	DECLINLINE(void) virtioCoreFormatDeviceStatus(uint8_t bStatus, char *pszBuf, size_t uSize)
537	{
538	# define ADJCURSOR(len) { cp += len; uSize -= len; sep = (char *)" \| "; }
539	memset(pszBuf, 0, uSize);
540	char cp = pszBuf, sep = (char *)"";
541	size_t len;
542	if (bStatus == 0)
543	RTStrPrintf(cp, uSize, "RESET");
544	else
545	{
546	if (bStatus & VIRTIO_STATUS_ACKNOWLEDGE)
547	{
548	len = RTStrPrintf(cp, uSize, "ACKNOWLEDGE");
549	ADJCURSOR(len);
550	}
551	if (bStatus & VIRTIO_STATUS_DRIVER)
552	{
553	len = RTStrPrintf(cp, uSize, "%sDRIVER", sep);
554	ADJCURSOR(len);
555	}
556	if (bStatus & VIRTIO_STATUS_FEATURES_OK)
557	{
558	len = RTStrPrintf(cp, uSize, "%sFEATURES_OK", sep);
559	ADJCURSOR(len);
560	}
561	if (bStatus & VIRTIO_STATUS_DRIVER_OK)
562	{
563	len = RTStrPrintf(cp, uSize, "%sDRIVER_OK", sep);
564	ADJCURSOR(len);
565	}
566	if (bStatus & VIRTIO_STATUS_FAILED)
567	{
568	len = RTStrPrintf(cp, uSize, "%sFAILED", sep);
569	ADJCURSOR(len);
570	}
571	if (bStatus & VIRTIO_STATUS_DEVICE_NEEDS_RESET)
572	RTStrPrintf(cp, uSize, "%sNEEDS_RESET", sep);
573	}
574	# undef ADJCURSOR
575	}
576
577	#endif /* LOG_ENABLED */
578
579	/** API function: See header file */
580	DECLHIDDEN(int) virtioCoreIsLegacyMode(PVIRTIOCORE pVirtio)
581	{
582	return pVirtio->fLegacyDriver;
583	}
584
585	#ifdef IN_RING3
586
587	DECLHIDDEN(int) virtioCoreR3VirtqAttach(PVIRTIOCORE pVirtio, uint16_t uVirtq, const char *pcszName)
588	{
589	LogFunc(("Attaching %s to VirtIO core\n", pcszName));
590	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
591	pVirtq->uVirtq = uVirtq;
592	pVirtq->fUsedRingEvent = false;
593	pVirtq->fAttached = true;
594	RTStrCopy(pVirtq->szName, sizeof(pVirtq->szName), pcszName);
595	return VINF_SUCCESS;
596	}
597
598	DECLHIDDEN(int) virtioCoreR3VirtqDetach(PVIRTIOCORE pVirtio, uint16_t uVirtqNbr)
599	{
600	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtqNbr];
601	pVirtq->uVirtq = 0;
602	pVirtq->uAvailIdxShadow = 0;
603	pVirtq->uUsedIdxShadow = 0;
604	pVirtq->fUsedRingEvent = false;
605	pVirtq->fAttached = false;
606	memset(pVirtq->szName, 0, sizeof(pVirtq->szName));
607	return VINF_SUCCESS;
608	}
609
610	DECLHIDDEN(bool) virtioCoreR3VirtqIsAttached(PVIRTIOCORE pVirtio, uint16_t uVirtqNbr)
611	{
612	return pVirtio->aVirtqueues[uVirtqNbr].fAttached;
613	}
614
615	DECLHIDDEN(bool) virtioCoreR3VirtqIsEnabled(PVIRTIOCORE pVirtio, uint16_t uVirtqNbr)
616	{
617	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtqNbr];
618	return (bool)pVirtq->uEnable && pVirtq->GCPhysVirtqDesc;
619	}
620
621	DECLINLINE(void) virtioCoreR3DescInfo(PCDBGFINFOHLP pHlp, PVIRTQ_DESC_T pDesc, uint16_t iDesc, const char *cszTail)
622	{
623	if (pDesc->fFlags & VIRTQ_DESC_F_NEXT)
624	pHlp->pfnPrintf(pHlp, " [%4d]%c%c %5d bytes @ %p [%4d] %s\n",
625	iDesc, pDesc->fFlags & VIRTQ_DESC_F_INDIRECT ? 'I' : ' ',
626	pDesc->fFlags & VIRTQ_DESC_F_WRITE ? 'W' : 'R',
627	pDesc->cb, pDesc->GCPhysBuf, pDesc->uDescIdxNext, cszTail);
628	else
629	pHlp->pfnPrintf(pHlp, " [%4d]%c%c %5d bytes @ %p %s\n",
630	iDesc, pDesc->fFlags & VIRTQ_DESC_F_INDIRECT ? 'I' : ' ',
631	pDesc->fFlags & VIRTQ_DESC_F_WRITE ? 'W' : 'R',
632	pDesc->cb, pDesc->GCPhysBuf, cszTail);
633	}
634
635	/** API Fuunction: See header file */
636	DECLHIDDEN(void) virtioCoreR3VirtqInfo(PPDMDEVINS pDevIns, PCDBGFINFOHLP pHlp, const char *pszArgs, int uVirtq)
637	{
638	RT_NOREF(pszArgs);
639	PVIRTIOCORE pVirtio = PDMDEVINS_2_DATA(pDevIns, PVIRTIOCORE);
640	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
641
642	/** @todo add ability to dump physical contents described by any descriptor (using existing VirtIO core API function) */
643	// bool fDump = pszArgs && (pszArgs == 'd' \|\| pszArgs == 'D'); /* "dump" (avail phys descriptor)"
644
645	uint16_t uAvailIdx = virtioReadAvailRingIdx(pDevIns, pVirtio, pVirtq);
646	uint16_t uAvailIdxShadow = pVirtq->uAvailIdxShadow;
647
648	uint16_t uUsedIdx = virtioReadUsedRingIdx(pDevIns, pVirtio, pVirtq);
649	uint16_t uUsedIdxShadow = pVirtq->uUsedIdxShadow;
650
651	uint16_t uAvailEventIdx = 0;
652	uint16_t uUsedEventIdx = 0;
653	bool fNotify = !!(pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX);
654	if (fNotify)
655	{
656	uUsedEventIdx = virtioReadAvailUsedEvent(pDevIns, pVirtio, pVirtq);
657	/* There is no helper for reading AvailEvent since the device is not supposed to read it. */
658	virtioCoreGCPhysRead(pVirtio, pDevIns,
659	pVirtq->GCPhysVirtqUsed
660	+ RT_UOFFSETOF_DYN(VIRTQ_USED_T, aRing[pVirtq->uQueueSize]),
661	&uAvailEventIdx, sizeof(uAvailEventIdx));
662	}
663
664	VIRTQBUF_T VirtqBuf;
665	PVIRTQBUF pVirtqBuf = &VirtqBuf;
666	RT_ZERO(VirtqBuf); /* Make sure pSgPhysSend and pSgPhysReturn are initialized. */
667	bool fEmpty = IS_VIRTQ_EMPTY(pDevIns, pVirtio, pVirtq);
668
669	LogFunc(("%s, empty = %s\n", pVirtq->szName, fEmpty ? "true" : "false"));
670
671	int cSendSegs = 0, cReturnSegs = 0;
672	if (!fEmpty)
673	{
674	virtioCoreR3VirtqAvailBufPeek(pDevIns, pVirtio, uVirtq, pVirtqBuf);
675	cSendSegs = pVirtqBuf->pSgPhysSend ? pVirtqBuf->pSgPhysSend->cSegs : 0;
676	cReturnSegs = pVirtqBuf->pSgPhysReturn ? pVirtqBuf->pSgPhysReturn->cSegs : 0;
677	}
678
679	bool fAvailNoInterrupt = virtioReadAvailRingFlags(pDevIns, pVirtio, pVirtq) & VIRTQ_AVAIL_F_NO_INTERRUPT;
680	bool fUsedNoNotify = virtioReadUsedRingFlags(pDevIns, pVirtio, pVirtq) & VIRTQ_USED_F_NO_NOTIFY;
681
682	pHlp->pfnPrintf(pHlp, " queue enabled: ........... %s\n", pVirtq->uEnable ? "true" : "false");
683	pHlp->pfnPrintf(pHlp, " size: .................... %d\n", pVirtq->uQueueSize);
684	pHlp->pfnPrintf(pHlp, " notify offset: ........... %d\n", pVirtq->uNotifyOffset);
685	if (pVirtio->fMsiSupport)
686	pHlp->pfnPrintf(pHlp, " MSIX vector: ....... %4.4x\n", pVirtq->uMsixVector);
687	pHlp->pfnPrintf(pHlp, "\n");
688	pHlp->pfnPrintf(pHlp, " avail ring (%d entries):\n", uAvailIdx - uAvailIdxShadow);
689	pHlp->pfnPrintf(pHlp, " index: ................ %d (%d)\n", pVirtq->uQueueSize ? uAvailIdx % pVirtq->uQueueSize : uAvailIdx, uAvailIdx);
690	pHlp->pfnPrintf(pHlp, " shadow: ............... %d (%d)\n", pVirtq->uQueueSize ? uAvailIdxShadow % pVirtq->uQueueSize : uAvailIdxShadow, uAvailIdxShadow);
691	pHlp->pfnPrintf(pHlp, " flags: ................ %s\n", fAvailNoInterrupt ? "NO_INTERRUPT" : "");
692	pHlp->pfnPrintf(pHlp, "\n");
693	pHlp->pfnPrintf(pHlp, " used ring (%d entries):\n", uUsedIdxShadow - uUsedIdx);
694	pHlp->pfnPrintf(pHlp, " index: ................ %d (%d)\n", pVirtq->uQueueSize ? uUsedIdx % pVirtq->uQueueSize : uUsedIdx, uUsedIdx);
695	pHlp->pfnPrintf(pHlp, " shadow: ............... %d (%d)\n", pVirtq->uQueueSize ? uUsedIdxShadow % pVirtq->uQueueSize : uUsedIdxShadow, uUsedIdxShadow);
696	pHlp->pfnPrintf(pHlp, " flags: ................ %s\n", fUsedNoNotify ? "NO_NOTIFY" : "");
697	pHlp->pfnPrintf(pHlp, "\n");
698	if (!fEmpty)
699	{
700	pHlp->pfnPrintf(pHlp, " desc chain:\n");
701	pHlp->pfnPrintf(pHlp, " head idx: ............. %d (%d)\n", pVirtq->uQueueSize ? uUsedIdx % pVirtq->uQueueSize : uUsedIdx, uUsedIdx);
702	pHlp->pfnPrintf(pHlp, " segs: ................. %d\n", cSendSegs + cReturnSegs);
703	pHlp->pfnPrintf(pHlp, " refCnt ................ %d\n", pVirtqBuf->cRefs);
704	pHlp->pfnPrintf(pHlp, "\n");
705	pHlp->pfnPrintf(pHlp, " host-to-guest (%d bytes):\n", pVirtqBuf->cbPhysSend);
706	pHlp->pfnPrintf(pHlp, " segs: .............. %d\n", cSendSegs);
707	if (cSendSegs)
708	{
709	pHlp->pfnPrintf(pHlp, " index: ............. %d\n", pVirtqBuf->pSgPhysSend->idxSeg);
710	pHlp->pfnPrintf(pHlp, " unsent ............. %d\n", pVirtqBuf->pSgPhysSend->cbSegLeft);
711	}
712	pHlp->pfnPrintf(pHlp, "\n");
713	pHlp->pfnPrintf(pHlp, " guest-to-host (%d bytes):\n", pVirtqBuf->cbPhysReturn);
714	pHlp->pfnPrintf(pHlp, " segs: .............. %d\n", cReturnSegs);
715	if (cReturnSegs)
716	{
717	pHlp->pfnPrintf(pHlp, " index: ............. %d\n", pVirtqBuf->pSgPhysReturn->idxSeg);
718	pHlp->pfnPrintf(pHlp, " unsent ............. %d\n", pVirtqBuf->pSgPhysReturn->cbSegLeft);
719	}
720	} else
721	pHlp->pfnPrintf(pHlp, " no desc chains available\n");
722	pHlp->pfnPrintf(pHlp, "\n");
723
724	/* Avoid handling zero-sized queues, there is nothing to show anyway. */
725	if (pVirtq->uQueueSize == 0)
726	return;
727
728	pHlp->pfnPrintf(pHlp, " desc table:\n");
729	/*
730	* Each line in the descriptor table output consists of two parts: a fixed part and a variable "tail".
731	* The fixed part shows the descriptor index, its writability, size, physical address, and optionally
732	* which descriptor is next the chain. The tail shows which elements of avail/used rings point to
733	* this descriptor.
734	*/
735	VIRTQ_DESC_T descTable[VIRTQ_SIZE];
736	char aszTails[VIRTQ_SIZE][32];
737	virtioCoreGCPhysRead(pVirtio, pDevIns, pVirtq->GCPhysVirtqDesc,
738	&descTable, sizeof(VIRTQ_DESC_T) * pVirtq->uQueueSize);
739	RT_BZERO(aszTails, sizeof(aszTails)); /* No tails by default */
740
741	/* Fill avail tail fields. */
742
743	/* The first available descriptor gets outer reverse angle brackets. */
744	char chOuterLeft = '>', chOuterRight = '<';
745	char chLeft = '[', chRight = ']';
746	/* Use 'not-equal' instead of 'less' because of uint16_t wrapping! */
747	for (uint16_t i = uAvailIdxShadow; i != uAvailIdx; i++)
748	{
749	/* The last descriptor gets inner curly braces, inner square brackets for the rest. */
750	if (i + 1 == uAvailIdx) { chLeft = '{'; chRight = '}'; }
751	uint16_t uDescIdx = virtioReadAvailDescIdx(pDevIns, pVirtio, pVirtq, i);
752	/* Print an exclamation sign instead of outer right bracket if this descriptor triggers notification. */
753	RTStrPrintf(aszTails[uDescIdx], sizeof(aszTails[0]), "%c%c%4d%c%c ",
754	chOuterLeft, chLeft, i % pVirtq->uQueueSize, chRight,
755	fNotify ? ((i % pVirtq->uQueueSize) == (uAvailEventIdx % pVirtq->uQueueSize) ? '!' : chOuterRight) : chOuterRight);
756	chOuterLeft = chOuterRight = ' ';
757	}
758
759	/* Fill used tail fields, see comments in the similar loop above. */
760
761	chOuterLeft = '>'; chOuterRight = '<';
762	chLeft = '['; chRight = ']';
763	for (uint16_t i = uUsedIdx; i != uUsedIdxShadow; i++)
764	{
765	VIRTQ_USED_ELEM_T elem;
766	virtioCoreGCPhysRead(pVirtio, pDevIns,
767	pVirtq->GCPhysVirtqUsed
768	+ RT_UOFFSETOF_DYN(VIRTQ_USED_T, aRing[i % pVirtq->uQueueSize]),
769	&elem, sizeof(elem));
770	if (i + 1 == uUsedIdxShadow) { chLeft = '{'; chRight = '}'; }
771	char *szTail = aszTails[elem.uDescIdx % pVirtq->uQueueSize];
772	/* Add empty avail field if none is present, 9 spaces + terminating zero. */
773	if (*szTail == '\0')
774	RTStrCopy(szTail, 10, " ");
775	RTStrPrintf(szTail + 9, sizeof(aszTails[0]) - 9, " %c%c%4d%c%c %d bytes",
776	chOuterLeft, chLeft, i % pVirtq->uQueueSize, chRight,
777	fNotify ? ((i % pVirtq->uQueueSize) == (uUsedEventIdx % pVirtq->uQueueSize) ? '!' : chOuterRight) : chOuterRight,
778	elem.cbElem);
779	chOuterLeft = chOuterRight = ' ';
780	}
781
782	pHlp->pfnPrintf(pHlp, " index w/r size phys addr next @avail @used\n");
783	pHlp->pfnPrintf(pHlp, " ------ - ----------- ---------------- ------- -------- ------------------\n");
784	for (uint16_t i = 0; i < pVirtq->uQueueSize; i++)
785	virtioCoreR3DescInfo(pHlp, &descTable[i], i, aszTails[i]);
786	}
787
788
789	/** API Function: See header file */
790	DECLHIDDEN(PVIRTQBUF) virtioCoreR3VirtqBufAlloc(void)
791	{
792	PVIRTQBUF pVirtqBuf = (PVIRTQBUF)RTMemAllocZ(sizeof(VIRTQBUF_T));
793	AssertReturn(pVirtqBuf, NULL);
794	pVirtqBuf->u32Magic = VIRTQBUF_MAGIC;
795	pVirtqBuf->cRefs = 1;
796	return pVirtqBuf;
797	}
798
799
800	/** API Function: See header file */
801	DECLHIDDEN(uint32_t) virtioCoreR3VirtqBufRetain(PVIRTQBUF pVirtqBuf)
802	{
803	AssertReturn(pVirtqBuf, UINT32_MAX);
804	AssertReturn(pVirtqBuf->u32Magic == VIRTQBUF_MAGIC, UINT32_MAX);
805	uint32_t cRefs = ASMAtomicIncU32(&pVirtqBuf->cRefs);
806	Assert(cRefs > 1);
807	Assert(cRefs < 16);
808	return cRefs;
809	}
810
811	/** API Function: See header file */
812	DECLHIDDEN(uint32_t) virtioCoreR3VirtqBufRelease(PVIRTIOCORE pVirtio, PVIRTQBUF pVirtqBuf)
813	{
814	if (!pVirtqBuf)
815	return 0;
816	AssertReturn(pVirtqBuf, 0);
817	AssertReturn(pVirtqBuf->u32Magic == VIRTQBUF_MAGIC, 0);
818	uint32_t cRefs = ASMAtomicDecU32(&pVirtqBuf->cRefs);
819	Assert(cRefs < 16);
820	if (cRefs == 0)
821	{
822	pVirtqBuf->u32Magic = ~VIRTQBUF_MAGIC;
823	RTMemFree(pVirtqBuf);
824	#ifdef VBOX_WITH_STATISTICS
825	STAM_REL_COUNTER_INC(&pVirtio->StatDescChainsFreed);
826	#endif
827	}
828	RT_NOREF(pVirtio);
829	return cRefs;
830	}
831
832	/** API Function: See header file */
833	DECLHIDDEN(void) virtioCoreNotifyConfigChanged(PVIRTIOCORE pVirtio)
834	{
835	virtioNudgeGuest(pVirtio->pDevInsR3, pVirtio, VIRTIO_ISR_DEVICE_CONFIG, pVirtio->uMsixConfig);
836	}
837
838
839	/** API Function: See header file */
840	DECLHIDDEN(void) virtioCoreVirtqEnableNotify(PVIRTIOCORE pVirtio, uint16_t uVirtq, bool fEnable)
841	{
842	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
843	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
844
845	if (IS_DRIVER_OK(pVirtio))
846	{
847	uint16_t fFlags = virtioReadUsedRingFlags(pVirtio->pDevInsR3, pVirtio, pVirtq);
848
849	if (fEnable)
850	fFlags &= ~VIRTQ_USED_F_NO_NOTIFY;
851	else
852	fFlags \|= VIRTQ_USED_F_NO_NOTIFY;
853
854	virtioWriteUsedRingFlags(pVirtio->pDevInsR3, pVirtio, pVirtq, fFlags);
855	}
856	}
857
858	/** API function: See Header file */
859	DECLHIDDEN(void) virtioCoreResetAll(PVIRTIOCORE pVirtio)
860	{
861	LogFunc(("\n"));
862	pVirtio->fDeviceStatus \|= VIRTIO_STATUS_DEVICE_NEEDS_RESET;
863	if (IS_DRIVER_OK(pVirtio))
864	{
865	if (!pVirtio->fLegacyDriver)
866	pVirtio->fGenUpdatePending = true;
867	virtioNudgeGuest(pVirtio->pDevInsR3, pVirtio, VIRTIO_ISR_DEVICE_CONFIG, pVirtio->uMsixConfig);
868	}
869	}
870
871	/** API function: See Header file */
872	DECLHIDDEN(int) virtioCoreR3VirtqAvailBufPeek(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq, PVIRTQBUF pVirtqBuf)
873	{
874	return virtioCoreR3VirtqAvailBufGet(pDevIns, pVirtio, uVirtq, pVirtqBuf, false);
875	}
876
877
878	/** API function: See Header file */
879	DECLHIDDEN(int) virtioCoreR3VirtqAvailBufNext(PVIRTIOCORE pVirtio, uint16_t uVirtq)
880	{
881	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
882	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
883
884	if (!pVirtio->fLegacyDriver)
885	AssertMsgReturn((pVirtio->fDeviceStatus & VIRTIO_STATUS_DRIVER_OK) && pVirtq->uEnable,
886	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
887
888	if (IS_VIRTQ_EMPTY(pVirtio->pDevInsR3, pVirtio, pVirtq))
889	return VERR_NOT_AVAILABLE;
890
891	Log6Func(("%s avail shadow idx: %u\n", pVirtq->szName, pVirtq->uAvailIdxShadow));
892	pVirtq->uAvailIdxShadow++;
893
894	return VINF_SUCCESS;
895	}
896
897	/** API Function: See header file */
898	DECLHIDDEN(int) virtioCoreR3VirtqAvailBufGet(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq,
899	uint16_t uHeadIdx, PVIRTQBUF pVirtqBuf)
900	{
901	AssertMsgReturn(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues),
902	("uVirtq out of range"), VERR_INVALID_PARAMETER);
903
904	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
905
906	if (!pVirtio->fLegacyDriver)
907	AssertMsgReturn((pVirtio->fDeviceStatus & VIRTIO_STATUS_DRIVER_OK) && pVirtq->uEnable,
908	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
909
910	uint16_t uDescIdx = uHeadIdx;
911
912	Log6Func(("%s DESC CHAIN: (head idx = %u)\n", pVirtio->aVirtqueues[uVirtq].szName, uHeadIdx));
913
914	/*
915	* Allocate and initialize the descriptor chain structure.
916	*/
917	pVirtqBuf->u32Magic = VIRTQBUF_MAGIC;
918	pVirtqBuf->cRefs = 1;
919	pVirtqBuf->uHeadIdx = uHeadIdx;
920	pVirtqBuf->uVirtq = uVirtq;
921
922	/*
923	* Gather segments.
924	*/
925	VIRTQ_DESC_T desc;
926
927	uint32_t cbIn = 0;
928	uint32_t cbOut = 0;
929	uint32_t cSegsIn = 0;
930	uint32_t cSegsOut = 0;
931
932	PVIRTIOSGSEG paSegsIn = pVirtqBuf->aSegsIn;
933	PVIRTIOSGSEG paSegsOut = pVirtqBuf->aSegsOut;
934
935	do
936	{
937	PVIRTIOSGSEG pSeg;
938	/*
939	* Malicious guests may go beyond paSegsIn or paSegsOut boundaries by linking
940	* several descriptors into a loop. Since there is no legitimate way to get a sequences of
941	* linked descriptors exceeding the total number of descriptors in the ring (see @bugref{8620}),
942	* the following aborts I/O if breach and employs a simple log throttling algorithm to notify.
943	*/
944	if (cSegsIn + cSegsOut >= pVirtq->uQueueSize)
945	{
946	static volatile uint32_t s_cMessages = 0;
947	static volatile uint32_t s_cThreshold = 1;
948	if (ASMAtomicIncU32(&s_cMessages) == ASMAtomicReadU32(&s_cThreshold))
949	{
950	LogRelMax(64, ("Too many linked descriptors; check if the guest arranges descriptors in a loop (cSegsIn=%u cSegsOut=%u uQueueSize=%u uDescIdx=%u queue=%s).\n",
951	cSegsIn, cSegsOut, pVirtq->uQueueSize, uDescIdx, pVirtq->szName));
952	if (ASMAtomicReadU32(&s_cMessages) != 1)
953	LogRelMax(64, ("(the above error has occured %u times so far)\n", ASMAtomicReadU32(&s_cMessages)));
954	ASMAtomicWriteU32(&s_cThreshold, ASMAtomicReadU32(&s_cThreshold) * 10);
955	}
956	break;
957	}
958	/* Check if the limit has been reached for input chain (see section 2.4.4.1 of virtio 1.0 spec). */
959	if (cSegsIn >= RT_ELEMENTS(pVirtqBuf->aSegsIn))
960	{
961	LogRelMax(64, ("Too many input descriptors (cSegsIn=%u).\n", cSegsIn));
962	break;
963	}
964	/* Check if the limit has been reached for output chain (see section 2.4.4.1 of virtio 1.0 spec). */
965	if (cSegsOut >= RT_ELEMENTS(pVirtqBuf->aSegsOut))
966	{
967	LogRelMax(64, ("Too many output descriptors (cSegsOut=%u).\n", cSegsOut));
968	break;
969	}
970	RT_UNTRUSTED_VALIDATED_FENCE();
971
972	virtioReadDesc(pDevIns, pVirtio, pVirtq, uDescIdx, &desc);
973
974	if (desc.fFlags & VIRTQ_DESC_F_WRITE)
975	{
976	Log6Func(("%s IN idx=%-4u seg=%-3u addr=%RGp cb=%u\n", pVirtq->szName, uDescIdx, cSegsIn, desc.GCPhysBuf, desc.cb));
977	cbIn += desc.cb;
978	pSeg = &paSegsIn[cSegsIn++];
979	}
980	else
981	{
982	Log6Func(("%s OUT desc_idx=%-4u seg=%-3u addr=%RGp cb=%u\n", pVirtq->szName, uDescIdx, cSegsOut, desc.GCPhysBuf, desc.cb));
983	cbOut += desc.cb;
984	pSeg = &paSegsOut[cSegsOut++];
985	#ifdef DEEP_DEBUG
986	if (LogIs11Enabled())
987	{
988	virtioCoreGCPhysHexDump(pDevIns, desc.GCPhysBuf, desc.cb, 0, NULL);
989	Log(("\n"));
990	}
991	#endif
992	}
993	pSeg->GCPhys = desc.GCPhysBuf;
994	pSeg->cbSeg = desc.cb;
995	uDescIdx = desc.uDescIdxNext;
996	} while (desc.fFlags & VIRTQ_DESC_F_NEXT);
997
998	/*
999	* Add segments to the descriptor chain structure.
1000	*/
1001	if (cSegsIn)
1002	{
1003	virtioCoreGCPhysChainInit(&pVirtqBuf->SgBufIn, paSegsIn, cSegsIn);
1004	pVirtqBuf->pSgPhysReturn = &pVirtqBuf->SgBufIn;
1005	pVirtqBuf->cbPhysReturn = cbIn;
1006	#ifdef VBOX_WITH_STATISTICS
1007	STAM_REL_COUNTER_ADD(&pVirtio->StatDescChainsSegsIn, cSegsIn);
1008	#endif
1009	}
1010
1011	if (cSegsOut)
1012	{
1013	virtioCoreGCPhysChainInit(&pVirtqBuf->SgBufOut, paSegsOut, cSegsOut);
1014	pVirtqBuf->pSgPhysSend = &pVirtqBuf->SgBufOut;
1015	pVirtqBuf->cbPhysSend = cbOut;
1016	#ifdef VBOX_WITH_STATISTICS
1017	STAM_REL_COUNTER_ADD(&pVirtio->StatDescChainsSegsOut, cSegsOut);
1018	#endif
1019	}
1020
1021	#ifdef VBOX_WITH_STATISTICS
1022	STAM_REL_COUNTER_INC(&pVirtio->StatDescChainsAllocated);
1023	#endif
1024	Log6Func(("%s -- segs OUT: %u (%u bytes) IN: %u (%u bytes) --\n",
1025	pVirtq->szName, cSegsOut, cbOut, cSegsIn, cbIn));
1026
1027	return VINF_SUCCESS;
1028	}
1029
1030	/** API function: See Header file */
1031	DECLHIDDEN(int) virtioCoreR3VirtqAvailBufGet(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq,
1032	PVIRTQBUF pVirtqBuf, bool fRemove)
1033	{
1034	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1035	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1036
1037	if (IS_VIRTQ_EMPTY(pDevIns, pVirtio, pVirtq))
1038	return VERR_NOT_AVAILABLE;
1039
1040	uint16_t uHeadIdx = virtioReadAvailDescIdx(pDevIns, pVirtio, pVirtq, pVirtq->uAvailIdxShadow);
1041
1042	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
1043	virtioWriteUsedAvailEvent(pDevIns,pVirtio, pVirtq, pVirtq->uAvailIdxShadow + 1);
1044
1045	if (fRemove)
1046	pVirtq->uAvailIdxShadow++;
1047
1048	return virtioCoreR3VirtqAvailBufGet(pDevIns, pVirtio, uVirtq, uHeadIdx, pVirtqBuf);
1049	}
1050
1051	/** API function: See Header file */
1052	DECLHIDDEN(int) virtioCoreR3VirtqUsedBufPut(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq, PRTSGBUF pSgVirtReturn,
1053	PVIRTQBUF pVirtqBuf, bool fFence)
1054	{
1055	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1056	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1057
1058	PVIRTIOSGBUF pSgPhysReturn = pVirtqBuf->pSgPhysReturn;
1059
1060	Assert(pVirtqBuf->u32Magic == VIRTQBUF_MAGIC);
1061	Assert(pVirtqBuf->cRefs > 0);
1062
1063	/*
1064	* Workaround for a bug in FreeBSD's virtio-net driver up until 12.3 which supports only the legacy style devive.
1065	* When the device is re-initialized from the driver it violates the spec and posts commands to the control queue
1066	* before setting the DRIVER_OK flag, breaking the following check and rendering the device non-functional.
1067	* The queues are properly set up at this stage however so no real harm is done and we can safely continue here,
1068	* for the legacy device only of course after making sure the queue is properly set up.
1069	*/
1070	AssertMsgReturn( IS_DRIVER_OK(pVirtio)
1071	\|\| ( pVirtio->fLegacyDriver
1072	&& pVirtq->GCPhysVirtqDesc),
1073	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
1074
1075	Log6Func((" Copying device data to %s, [desc:%u -> used ring:%u]\n",
1076	VIRTQNAME(pVirtio, uVirtq), pVirtqBuf->uHeadIdx, pVirtq->uUsedIdxShadow));
1077
1078	/* Copy s/g buf (virtual memory) to guest phys mem (VirtIO "IN" direction). */
1079
1080	size_t cbCopy = 0, cbTotal = 0, cbRemain = 0;
1081
1082	/** @todo r=aeichner Check whether VirtIO should return an error if the device wants to return data but
1083	* the guest didn't set up an IN buffer. */
1084	if ( pSgVirtReturn
1085	&& pSgPhysReturn)
1086	{
1087	size_t cbTarget = virtioCoreGCPhysChainCalcBufSize(pSgPhysReturn);
1088	cbRemain = cbTotal = RTSgBufCalcTotalLength(pSgVirtReturn);
1089	AssertMsgReturn(cbTarget >= cbRemain, ("No space to write data to phys memory"), VERR_BUFFER_OVERFLOW);
1090	virtioCoreGCPhysChainReset(pSgPhysReturn);
1091	while (cbRemain)
1092	{
1093	cbCopy = RT_MIN(pSgVirtReturn->cbSegLeft, pSgPhysReturn->cbSegLeft);
1094	AssertReturn(cbCopy > 0, VERR_INVALID_PARAMETER);
1095	virtioCoreGCPhysWrite(pVirtio, pDevIns, (RTGCPHYS)pSgPhysReturn->GCPhysCur, pSgVirtReturn->pvSegCur, cbCopy);
1096	RTSgBufAdvance(pSgVirtReturn, cbCopy);
1097	virtioCoreGCPhysChainAdvance(pSgPhysReturn, cbCopy);
1098	cbRemain -= cbCopy;
1099	}
1100
1101	if (fFence)
1102	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE(); /* needed? */
1103
1104	Assert(!(cbCopy >> 32));
1105	}
1106
1107	/* Flag if write-ahead crosses threshold where guest driver indicated it wants event notification */
1108	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
1109	if (pVirtq->uUsedIdxShadow == virtioReadAvailUsedEvent(pDevIns, pVirtio, pVirtq))
1110	pVirtq->fUsedRingEvent = true;
1111
1112	/*
1113	* Place used buffer's descriptor in used ring but don't update used ring's slot index.
1114	* That will be done with a subsequent client call to virtioCoreVirtqUsedRingSync()
1115	*
1116	* @todo r=aeichner: The increment of the shadow index is not atomic but this code can be called
1117	* concurrently!!
1118	*/
1119	virtioWriteUsedElem(pDevIns, pVirtio, pVirtq, pVirtq->uUsedIdxShadow++, pVirtqBuf->uHeadIdx, (uint32_t)cbTotal);
1120
1121	#ifdef LOG_ENABLED
1122	if ( LogIs6Enabled()
1123	&& pSgVirtReturn
1124	&& pSgPhysReturn)
1125	{
1126
1127	LogFunc((" ... %d segs, %zu bytes, copied to %u byte buf@offset=%u. Residual: %zu bytes\n",
1128	pSgVirtReturn->cSegs, cbTotal - cbRemain, pVirtqBuf->cbPhysReturn,
1129	((virtioCoreGCPhysChainCalcBufSize(pVirtqBuf->pSgPhysReturn) -
1130	virtioCoreGCPhysChainCalcLengthLeft(pVirtqBuf->pSgPhysReturn)) - (cbTotal - cbRemain)),
1131	virtioCoreGCPhysChainCalcLengthLeft(pVirtqBuf->pSgPhysReturn) ));
1132
1133	uint16_t uPending = virtioCoreR3CountPendingBufs(
1134	virtioReadUsedRingIdx(pDevIns, pVirtio, pVirtq),
1135	pVirtq->uUsedIdxShadow, pVirtq->uQueueSize);
1136
1137	LogFunc((" %u used buf%s not synced in %s\n", uPending, uPending == 1 ? "" : "s ",
1138	VIRTQNAME(pVirtio, uVirtq)));
1139	}
1140	#endif
1141	return VINF_SUCCESS;
1142	}
1143
1144	/** API function: See Header file */
1145	DECLHIDDEN(int) virtioCoreR3VirtqUsedBufPut(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq,
1146	size_t cb, void const *pv, PVIRTQBUF pVirtqBuf, size_t cbEnqueue, bool fFence)
1147	{
1148	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1149	Assert(pv);
1150
1151	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1152	PVIRTIOSGBUF pSgPhysReturn = pVirtqBuf->pSgPhysReturn;
1153
1154	Assert(pVirtqBuf->u32Magic == VIRTQBUF_MAGIC);
1155	Assert(pVirtqBuf->cRefs > 0);
1156
1157	AssertMsgReturn(IS_DRIVER_OK(pVirtio), ("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
1158
1159	Log6Func((" Copying device data to %s, [desc chain head idx:%u]\n",
1160	VIRTQNAME(pVirtio, uVirtq), pVirtqBuf->uHeadIdx));
1161	/*
1162	* Convert virtual memory simple buffer to guest physical memory (VirtIO descriptor chain)
1163	*/
1164	uint8_t pvBuf = (uint8_t )pv;
1165	size_t cbRemain = cb, cbCopy = 0;
1166	while (cbRemain)
1167	{
1168	cbCopy = RT_MIN(pSgPhysReturn->cbSegLeft, cbRemain);
1169	Assert(cbCopy > 0);
1170	virtioCoreGCPhysWrite(pVirtio, pDevIns, (RTGCPHYS)pSgPhysReturn->GCPhysCur, pvBuf, cbCopy);
1171	virtioCoreGCPhysChainAdvance(pSgPhysReturn, cbCopy);
1172	pvBuf += cbCopy;
1173	cbRemain -= cbCopy;
1174	}
1175	LogFunc((" ...%zu bytes, copied to %u byte buf@offset=%u. Residual: %zu bytes\n",
1176	cb , pVirtqBuf->cbPhysReturn,
1177	((virtioCoreGCPhysChainCalcBufSize(pVirtqBuf->pSgPhysReturn) -
1178	virtioCoreGCPhysChainCalcLengthLeft(pVirtqBuf->pSgPhysReturn)) - cb),
1179	virtioCoreGCPhysChainCalcLengthLeft(pVirtqBuf->pSgPhysReturn)));
1180
1181	if (cbEnqueue)
1182	{
1183	if (fFence)
1184	{
1185	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE(); /* needed? */
1186	Assert(!(cbCopy >> 32));
1187	}
1188	/* Flag if write-ahead crosses threshold where guest driver indicated it wants event notification */
1189	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
1190	if (pVirtq->uUsedIdxShadow == virtioReadAvailUsedEvent(pDevIns, pVirtio, pVirtq))
1191	pVirtq->fUsedRingEvent = true;
1192	/*
1193	* Place used buffer's descriptor in used ring but don't update used ring's slot index.
1194	* That will be done with a subsequent client call to virtioCoreVirtqUsedRingSync()
1195	*/
1196	Log6Func((" Enqueue desc chain head idx %u to %s used ring @ %u\n", pVirtqBuf->uHeadIdx,
1197	VIRTQNAME(pVirtio, uVirtq), pVirtq->uUsedIdxShadow));
1198
1199	virtioWriteUsedElem(pDevIns, pVirtio, pVirtq, pVirtq->uUsedIdxShadow++, pVirtqBuf->uHeadIdx, (uint32_t)cbEnqueue);
1200
1201	#ifdef LOG_ENABLED
1202	if (LogIs6Enabled())
1203	{
1204	uint16_t uPending = virtioCoreR3CountPendingBufs(
1205	virtioReadUsedRingIdx(pDevIns, pVirtio, pVirtq),
1206	pVirtq->uUsedIdxShadow, pVirtq->uQueueSize);
1207
1208	LogFunc((" %u used buf%s not synced in %s\n",
1209	uPending, uPending == 1 ? "" : "s ", VIRTQNAME(pVirtio, uVirtq)));
1210	}
1211	#endif
1212	} /* fEnqueue */
1213
1214	return VINF_SUCCESS;
1215	}
1216
1217
1218	#endif /* IN_RING3 */
1219
1220	/** API function: See Header file */
1221	DECLHIDDEN(int) virtioCoreVirtqUsedRingSync(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq)
1222	{
1223	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1224	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1225
1226	if (!pVirtio->fLegacyDriver)
1227	AssertMsgReturn((pVirtio->fDeviceStatus & VIRTIO_STATUS_DRIVER_OK) && pVirtq->uEnable,
1228	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
1229
1230	Log6Func((" Sync %s used ring (%u -> idx)\n",
1231	pVirtq->szName, pVirtq->uUsedIdxShadow));
1232
1233	virtioWriteUsedRingIdx(pDevIns, pVirtio, pVirtq, pVirtq->uUsedIdxShadow);
1234	virtioCoreNotifyGuestDriver(pDevIns, pVirtio, uVirtq);
1235
1236	return VINF_SUCCESS;
1237	}
1238
1239	/**
1240	* This is called from the MMIO callback code when the guest does an MMIO access to the
1241	* mapped queue notification capability area corresponding to a particular queue, to notify
1242	* the queue handler of available data in the avail ring of the queue (VirtIO 1.0, 4.1.4.4.1)
1243	*
1244	* @param pDevIns The device instance.
1245	* @param pVirtio Pointer to the shared virtio state.
1246	* @param uVirtq Virtq to check for guest interrupt handling preference
1247	* @param uNotifyIdx Notification index
1248	*/
1249	static void virtioCoreVirtqNotified(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq, uint16_t uNotifyIdx)
1250	{
1251	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1252
1253	/* VirtIO 1.0, section 4.1.5.2 implies uVirtq and uNotifyIdx should match. Disregarding any of
1254	* these notifications (if those indicies disagree) may break device/driver synchronization,
1255	* causing eternal throughput starvation, yet there's no specified way to disambiguate
1256	* which queue to wake-up in any awkward situation where the two parameters differ.
1257	*/
1258	AssertMsg(uNotifyIdx == uVirtq,
1259	("Guest kicked virtq %d's notify addr w/non-corresponding virtq idx %d\n",
1260	uVirtq, uNotifyIdx));
1261	RT_NOREF(uNotifyIdx);
1262
1263	AssertReturnVoid(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1264	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1265
1266	Log6Func(("%s: (desc chains: %u)\n", *pVirtq->szName ? pVirtq->szName : "?UNAMED QUEUE?",
1267	virtioCoreVirtqAvailCnt(pDevIns, pVirtio, pVirtq)));
1268
1269	/* Inform client */
1270	pVirtioCC->pfnVirtqNotified(pDevIns, pVirtio, uVirtq);
1271	RT_NOREF2(pVirtio, pVirtq);
1272	}
1273
1274	/**
1275	* Trigger MSI-X or INT# interrupt to notify guest of data added to used ring of
1276	* the specified virtq, depending on the interrupt configuration of the device
1277	* and depending on negotiated and realtime constraints flagged by the guest driver.
1278	*
1279	* See VirtIO 1.0 specification (section 2.4.7).
1280	*
1281	* @param pDevIns The device instance.
1282	* @param pVirtio Pointer to the shared virtio state.
1283	* @param uVirtq Virtq to check for guest interrupt handling preference
1284	*/
1285	static void virtioCoreNotifyGuestDriver(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq)
1286	{
1287	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1288	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1289
1290	if (!IS_DRIVER_OK(pVirtio))
1291	{
1292	LogFunc(("Guest driver not in ready state.\n"));
1293	return;
1294	}
1295
1296	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
1297	{
1298	if (pVirtq->fUsedRingEvent)
1299	{
1300	#ifdef IN_RING3
1301	Log6Func(("...kicking guest %s, VIRTIO_F_EVENT_IDX set and threshold (%d) reached\n",
1302	pVirtq->szName, (uint16_t)virtioReadAvailUsedEvent(pDevIns, pVirtio, pVirtq)));
1303	#endif
1304	virtioNudgeGuest(pDevIns, pVirtio, VIRTIO_ISR_VIRTQ_INTERRUPT, pVirtq->uMsixVector);
1305	pVirtq->fUsedRingEvent = false;
1306	return;
1307	}
1308	#ifdef IN_RING3
1309	Log6Func(("...skip interrupt %s, VIRTIO_F_EVENT_IDX set but threshold (%d) not reached (%d)\n",
1310	pVirtq->szName,(uint16_t)virtioReadAvailUsedEvent(pDevIns, pVirtio, pVirtq), pVirtq->uUsedIdxShadow));
1311	#endif
1312	}
1313	else
1314	{
1315	/** If guest driver hasn't suppressed interrupts, interrupt */
1316	if (!(virtioReadAvailRingFlags(pDevIns, pVirtio, pVirtq) & VIRTQ_AVAIL_F_NO_INTERRUPT))
1317	{
1318	virtioNudgeGuest(pDevIns, pVirtio, VIRTIO_ISR_VIRTQ_INTERRUPT, pVirtq->uMsixVector);
1319	return;
1320	}
1321	Log6Func(("...skipping interrupt for %s (guest set VIRTQ_AVAIL_F_NO_INTERRUPT)\n", pVirtq->szName));
1322	}
1323	}
1324
1325	/**
1326	* Raise interrupt or MSI-X
1327	*
1328	* @param pDevIns The device instance.
1329	* @param pVirtio Pointer to the shared virtio state.
1330	* @param uCause Interrupt cause bit mask to set in PCI ISR port.
1331	* @param uVec MSI-X vector, if enabled
1332	*/
1333	static int virtioNudgeGuest(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint8_t uCause, uint16_t uMsixVector)
1334	{
1335	if (uCause == VIRTIO_ISR_VIRTQ_INTERRUPT)
1336	Log6Func(("Reason for interrupt - buffer added to 'used' ring.\n"));
1337	else
1338	if (uCause == VIRTIO_ISR_DEVICE_CONFIG)
1339	Log6Func(("Reason for interrupt - device config change\n"));
1340
1341	if (pVirtio->uIrqMmio)
1342	{
1343	pVirtio->uISR \|= uCause;
1344	PDMDevHlpISASetIrq(pDevIns, pVirtio->uIrqMmio, PDM_IRQ_LEVEL_HIGH);
1345	}
1346	else if (!pVirtio->fMsiSupport)
1347	{
1348	pVirtio->uISR \|= uCause;
1349	PDMDevHlpPCISetIrq(pDevIns, 0, PDM_IRQ_LEVEL_HIGH);
1350	}
1351	else if (uMsixVector != VIRTIO_MSI_NO_VECTOR)
1352	PDMDevHlpPCISetIrq(pDevIns, uMsixVector, 1);
1353	return VINF_SUCCESS;
1354	}
1355
1356	/**
1357	* Lower interrupt (Called when guest reads ISR and when resetting)
1358	*
1359	* @param pDevIns The device instance.
1360	*/
1361	static void virtioLowerInterrupt(PPDMDEVINS pDevIns, uint16_t uMsixVector)
1362	{
1363	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1364	if (pVirtio->uIrqMmio)
1365	PDMDevHlpISASetIrq(pDevIns, pVirtio->uIrqMmio, PDM_IRQ_LEVEL_LOW);
1366	else if (!pVirtio->fMsiSupport)
1367	PDMDevHlpPCISetIrq(pDevIns, 0, PDM_IRQ_LEVEL_LOW);
1368	else if (uMsixVector != VIRTIO_MSI_NO_VECTOR)
1369	PDMDevHlpPCISetIrq(pDevIns, pVirtio->uMsixConfig, PDM_IRQ_LEVEL_LOW);
1370	}
1371
1372	#ifdef IN_RING3
1373	static void virtioResetVirtq(PVIRTIOCORE pVirtio, uint16_t uVirtq)
1374	{
1375	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1376	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1377
1378	pVirtq->uQueueSize = VIRTQ_SIZE;
1379	pVirtq->uEnable = false;
1380	pVirtq->uNotifyOffset = uVirtq;
1381	pVirtq->fUsedRingEvent = false;
1382	pVirtq->uAvailIdxShadow = 0;
1383	pVirtq->uUsedIdxShadow = 0;
1384	pVirtq->uMsixVector = uVirtq + 2;
1385
1386	if (!pVirtio->fMsiSupport) /* VirtIO 1.0, 4.1.4.3 and 4.1.5.1.2 */
1387	pVirtq->uMsixVector = VIRTIO_MSI_NO_VECTOR;
1388
1389	virtioLowerInterrupt(pVirtio->pDevInsR3, pVirtq->uMsixVector);
1390	}
1391
1392	static void virtioResetDevice(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio)
1393	{
1394	LogFunc(("Resetting device VirtIO state\n"));
1395	pVirtio->fLegacyDriver = pVirtio->fOfferLegacy; /* Cleared if VIRTIO_F_VERSION_1 feature ack'd */
1396	pVirtio->uDeviceFeaturesSelect = 0;
1397	pVirtio->uDriverFeaturesSelect = 0;
1398	pVirtio->uConfigGeneration = 0;
1399	pVirtio->fDeviceStatus = 0;
1400	pVirtio->uISR = 0;
1401
1402	if (!pVirtio->fMsiSupport)
1403	virtioLowerInterrupt(pDevIns, 0);
1404	else
1405	{
1406	virtioLowerInterrupt(pDevIns, pVirtio->uMsixConfig);
1407	for (int i = 0; i < VIRTQ_MAX_COUNT; i++)
1408	virtioLowerInterrupt(pDevIns, pVirtio->aVirtqueues[i].uMsixVector);
1409	}
1410
1411	if (!pVirtio->fMsiSupport) /* VirtIO 1.0, 4.1.4.3 and 4.1.5.1.2 */
1412	pVirtio->uMsixConfig = VIRTIO_MSI_NO_VECTOR;
1413
1414	for (uint16_t uVirtq = 0; uVirtq < VIRTQ_MAX_COUNT; uVirtq++)
1415	virtioResetVirtq(pVirtio, uVirtq);
1416	}
1417
1418	/**
1419	* Invoked by this implementation when guest driver resets the device.
1420	* The driver itself will not until the device has read the status change.
1421	*/
1422	static void virtioGuestR3WasReset(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
1423	{
1424	Log(("%-23s: Guest reset the device\n", __FUNCTION__));
1425
1426	/* Let the client know */
1427	pVirtioCC->pfnStatusChanged(pVirtio, pVirtioCC, 0 /* fDriverOk */);
1428	virtioResetDevice(pDevIns, pVirtio);
1429	}
1430
1431	DECLHIDDEN(void) virtioCoreR3ResetDevice(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
1432	{
1433	virtioGuestR3WasReset(pDevIns, pVirtio, pVirtioCC);
1434	}
1435	#endif /* IN_RING3 */
1436
1437	/*
1438	* Determines whether guest virtio driver is modern or legacy and does callback
1439	* informing device-specific code that feature negotiation is complete.
1440	* Should be called only once (coordinated via the 'toggle' flag)
1441	*/
1442	#ifdef IN_RING3
1443	DECLINLINE(void) virtioR3DoFeaturesCompleteOnceOnly(PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
1444	{
1445	if (pVirtio->uDriverFeatures & VIRTIO_F_VERSION_1)
1446	{
1447	LogFunc(("VIRTIO_F_VERSION_1 feature ack'd by guest\n"));
1448	pVirtio->fLegacyDriver = 0;
1449	}
1450	else
1451	{
1452	if (pVirtio->fOfferLegacy)
1453	{
1454	pVirtio->fLegacyDriver = 1;
1455	LogFunc(("VIRTIO_F_VERSION_1 feature was NOT set by guest\n"));
1456	}
1457	else
1458	AssertMsgFailed(("Guest didn't accept VIRTIO_F_VERSION_1, but fLegacyOffered flag not set.\n"));
1459	}
1460	if (pVirtioCC->pfnFeatureNegotiationComplete)
1461	pVirtioCC->pfnFeatureNegotiationComplete(pVirtio, pVirtio->uDriverFeatures, pVirtio->fLegacyDriver);
1462	pVirtio->fDriverFeaturesWritten \|= DRIVER_FEATURES_COMPLETE_HANDLED;
1463	}
1464	#endif
1465
1466
1467	/**
1468	* Handles a write to the device status register from the driver.
1469	*
1470	* @returns VBox status code
1471	*
1472	* @param pDevIns The device instance.
1473	* @param pVirtio Pointer to the shared virtio state.
1474	* @param pVirtioCC Pointer to the current context virtio state.
1475	* @param fDeviceStatus The device status to be written.
1476	*/
1477	DECLINLINE(int) virtioDeviceStatusWrite(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC,
1478	uint8_t fDeviceStatus)
1479	{
1480	pVirtio->fDeviceStatus = fDeviceStatus;
1481	bool fDeviceReset = pVirtio->fDeviceStatus == 0;
1482	#ifdef LOG_ENABLED
1483	if (LogIs7Enabled())
1484	{
1485	char szOut[80] = { 0 };
1486	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
1487	Log(("%-23s: Guest wrote fDeviceStatus ................ (%s)\n", __FUNCTION__, szOut));
1488	}
1489	#endif
1490	bool const fStatusChanged = IS_DRIVER_OK(pVirtio) != WAS_DRIVER_OK(pVirtio);
1491
1492	if (fDeviceReset \|\| fStatusChanged)
1493	{
1494	#ifdef IN_RING0
1495	/* Since VirtIO status changes are cumbersome by nature, e.g. not a benchmark priority,
1496	* handle the rest in R3 to facilitate logging or whatever dev-specific client needs to do */
1497	Log6(("%-23s: RING0 => RING3 (demote)\n", __FUNCTION__));
1498	return VINF_IOM_R3_MMIO_WRITE;
1499	#endif
1500	}
1501
1502	#ifdef IN_RING3
1503	/*
1504	* Notify client only if status actually changed from last time and when we're reset.
1505	*/
1506	if (fDeviceReset)
1507	virtioGuestR3WasReset(pDevIns, pVirtio, pVirtioCC);
1508
1509	if (fStatusChanged)
1510	pVirtioCC->pfnStatusChanged(pVirtio, pVirtioCC, IS_DRIVER_OK(pVirtio));
1511	#else
1512	RT_NOREF(pDevIns, pVirtioCC);
1513	#endif
1514	/*
1515	* Save the current status for the next write so we can see what changed.
1516	*/
1517	pVirtio->fPrevDeviceStatus = pVirtio->fDeviceStatus;
1518	return VINF_SUCCESS;
1519	}
1520
1521
1522	/**
1523	* Handles a read from the device status register from the driver.
1524	*
1525	* @returns The device status register value.
1526	*
1527	* @param pVirtio Pointer to the shared virtio state.
1528	*/
1529	DECLINLINE(uint8_t) virtioDeviceStatusRead(PVIRTIOCORE pVirtio)
1530	{
1531	#ifdef LOG_ENABLED
1532	if (LogIs7Enabled())
1533	{
1534	char szOut[80] = { 0 };
1535	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
1536	LogFunc(("Guest read fDeviceStatus ................ (%s)\n", szOut));
1537	}
1538	#endif
1539	return pVirtio->fDeviceStatus;
1540	}
1541
1542
1543	/**
1544	* Handle accesses to Common Configuration capability
1545	*
1546	* @returns VBox status code
1547	*
1548	* @param pDevIns The device instance.
1549	* @param pVirtio Pointer to the shared virtio state.
1550	* @param pVirtioCC Pointer to the current context virtio state.
1551	* @param fWrite Set if write access, clear if read access.
1552	* @param uOffsetOfAccess The common configuration capability offset.
1553	* @param cb Number of bytes to read or write
1554	* @param pv Pointer to location to write to or read from
1555	*/
1556	static int virtioCommonCfgAccessed(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC,
1557	int fWrite, uint32_t uOffsetOfAccess, unsigned cb, void *pv)
1558	{
1559	uint16_t uVirtq = pVirtio->uVirtqSelect;
1560	int rc = VINF_SUCCESS;
1561	uint64_t val;
1562	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDeviceFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1563	{
1564	if (fWrite) /* Guest WRITE pCommonCfg>uDeviceFeatures */
1565	{
1566	/* VirtIO 1.0, 4.1.4.3 states device_feature is a (guest) driver readonly field,
1567	* yet the linux driver attempts to write/read it back twice */
1568	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1569	LogFunc(("... WARNING: Guest attempted to write readonly virtio_pci_common_cfg.device_feature (ignoring)\n"));
1570	return VINF_IOM_MMIO_UNUSED_00;
1571	}
1572	else /* Guest READ pCommonCfg->uDeviceFeatures */
1573	{
1574	switch (pVirtio->uDeviceFeaturesSelect)
1575	{
1576	case 0:
1577	val = pVirtio->uDeviceFeatures & UINT32_C(0xffffffff);
1578	memcpy(pv, &val, cb);
1579	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1580	break;
1581	case 1:
1582	val = pVirtio->uDeviceFeatures >> 32;
1583	memcpy(pv, &val, cb);
1584	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess + sizeof(uint32_t));
1585	break;
1586	default:
1587	LogFunc(("Guest read uDeviceFeatures with out of range selector (%#x), returning 0\n",
1588	pVirtio->uDeviceFeaturesSelect));
1589	return VINF_IOM_MMIO_UNUSED_00;
1590	}
1591	}
1592	}
1593	else
1594	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1595	{
1596	if (fWrite) /* Guest WRITE pCommonCfg->udriverFeatures */
1597	{
1598	switch (pVirtio->uDriverFeaturesSelect)
1599	{
1600	case 0:
1601	memcpy(&pVirtio->uDriverFeatures, pv, cb);
1602	pVirtio->fDriverFeaturesWritten \|= DRIVER_FEATURES_0_WRITTEN;
1603	LogFunc(("Set DRIVER_FEATURES_0_WRITTEN. pVirtio->fDriverFeaturesWritten=%d\n", pVirtio->fDriverFeaturesWritten));
1604	if ( (pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_0_AND_1_WRITTEN) == DRIVER_FEATURES_0_AND_1_WRITTEN
1605	&& !(pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_COMPLETE_HANDLED))
1606	#ifdef IN_RING0
1607	return VINF_IOM_R3_MMIO_WRITE;
1608	#endif
1609	#ifdef IN_RING3
1610	virtioR3DoFeaturesCompleteOnceOnly(pVirtio, pVirtioCC);
1611	#endif
1612	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1613	break;
1614	case 1:
1615	memcpy((char *)&pVirtio->uDriverFeatures + sizeof(uint32_t), pv, cb);
1616	pVirtio->fDriverFeaturesWritten \|= DRIVER_FEATURES_1_WRITTEN;
1617	LogFunc(("Set DRIVER_FEATURES_1_WRITTEN. pVirtio->fDriverFeaturesWritten=%d\n", pVirtio->fDriverFeaturesWritten));
1618	if ( (pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_0_AND_1_WRITTEN) == DRIVER_FEATURES_0_AND_1_WRITTEN
1619	&& !(pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_COMPLETE_HANDLED))
1620	#ifdef IN_RING0
1621	return VINF_IOM_R3_MMIO_WRITE;
1622	#endif
1623	#ifdef IN_RING3
1624	virtioR3DoFeaturesCompleteOnceOnly(pVirtio, pVirtioCC);
1625	#endif
1626	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess + sizeof(uint32_t));
1627	break;
1628	default:
1629	LogFunc(("Guest wrote uDriverFeatures with out of range selector (%#x), returning 0\n",
1630	pVirtio->uDriverFeaturesSelect));
1631	return VINF_SUCCESS;
1632	}
1633	}
1634	else /* Guest READ pCommonCfg->udriverFeatures */
1635	{
1636	switch (pVirtio->uDriverFeaturesSelect)
1637	{
1638	case 0:
1639	val = pVirtio->uDriverFeatures & 0xffffffff;
1640	memcpy(pv, &val, cb);
1641	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1642	break;
1643	case 1:
1644	val = (pVirtio->uDriverFeatures >> 32) & 0xffffffff;
1645	memcpy(pv, &val, cb);
1646	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess + 4);
1647	break;
1648	default:
1649	LogFunc(("Guest read uDriverFeatures with out of range selector (%#x), returning 0\n",
1650	pVirtio->uDriverFeaturesSelect));
1651	return VINF_IOM_MMIO_UNUSED_00;
1652	}
1653	}
1654	}
1655	else
1656	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uNumVirtqs, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1657	{
1658	if (fWrite)
1659	{
1660	Log2Func(("Guest attempted to write readonly virtio_pci_common_cfg.num_queues\n"));
1661	return VINF_SUCCESS;
1662	}
1663	(uint16_t )pv = VIRTQ_MAX_COUNT;
1664	VIRTIO_DEV_CONFIG_LOG_ACCESS(uNumVirtqs, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1665	}
1666	else
1667	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fDeviceStatus, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1668	{
1669	if (fWrite) /* Guest WRITE pCommonCfg->fDeviceStatus */
1670	rc = virtioDeviceStatusWrite(pDevIns, pVirtio, pVirtioCC, (uint8_t )pv);
1671	else /* Guest READ pCommonCfg->fDeviceStatus */
1672	(uint8_t )pv = virtioDeviceStatusRead(pVirtio);
1673	}
1674	else
1675	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixConfig, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1676	VIRTIO_DEV_CONFIG_ACCESS( uMsixConfig, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1677	else
1678	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uDeviceFeaturesSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1679	VIRTIO_DEV_CONFIG_ACCESS( uDeviceFeaturesSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1680	else
1681	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uDriverFeaturesSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1682	VIRTIO_DEV_CONFIG_ACCESS( uDriverFeaturesSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1683	else
1684	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uConfigGeneration, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1685	VIRTIO_DEV_CONFIG_ACCESS( uConfigGeneration, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1686	else
1687	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uVirtqSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1688	{
1689	if (fWrite) {
1690	uint16_t uVirtqNew = (uint16_t )pv;
1691
1692	if (uVirtqNew < RT_ELEMENTS(pVirtio->aVirtqueues))
1693	VIRTIO_DEV_CONFIG_ACCESS( uVirtqSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1694	else
1695	LogFunc(("... WARNING: Guest attempted to write invalid virtq selector (ignoring)\n"));
1696	}
1697	else
1698	VIRTIO_DEV_CONFIG_ACCESS( uVirtqSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1699	}
1700	else
1701	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( GCPhysVirtqDesc, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1702	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( GCPhysVirtqDesc, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1703	else
1704	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( GCPhysVirtqAvail, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1705	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( GCPhysVirtqAvail, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1706	else
1707	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( GCPhysVirtqUsed, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1708	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( GCPhysVirtqUsed, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1709	else
1710	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uQueueSize, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1711	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uQueueSize, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1712	else
1713	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uEnable, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1714	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uEnable, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1715	else
1716	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uNotifyOffset, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1717	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uNotifyOffset, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1718	else
1719	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixVector, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1720	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uMsixVector, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1721	else
1722	{
1723	Log2Func(("Bad guest %s access to virtio_pci_common_cfg: uOffsetOfAccess=%#x (%d), cb=%d\n",
1724	fWrite ? "write" : "read ", uOffsetOfAccess, uOffsetOfAccess, cb));
1725	return fWrite ? VINF_SUCCESS : VINF_IOM_MMIO_UNUSED_00;
1726	}
1727
1728	#ifndef IN_RING3
1729	RT_NOREF(pDevIns, pVirtioCC);
1730	#endif
1731	return rc;
1732	}
1733
1734	/**
1735	* @callback_method_impl{FNIOMIOPORTNEWIN)
1736	*
1737	* This I/O handler exists only to handle access from legacy drivers.
1738	*/
1739	static DECLCALLBACK(VBOXSTRICTRC) virtioLegacyIOPortIn(PPDMDEVINS pDevIns, void pvUser, RTIOPORT offPort, uint32_t pu32, unsigned cb)
1740	{
1741	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1742	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatRead), a);
1743
1744	RT_NOREF(pvUser);
1745	Log(("%-23s: Port read at offset=%RTiop, cb=%#x%s",
1746	__FUNCTION__, offPort, cb,
1747	VIRTIO_DEV_CONFIG_MATCH_MEMBER(fIsrStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort) ? "" : "\n"));
1748
1749	void pv = pu32; / To use existing macros */
1750	int fWrite = 0; /* To use existing macros */
1751
1752	uint16_t uVirtq = pVirtio->uVirtqSelect;
1753
1754	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDeviceFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1755	{
1756	uint32_t val = pVirtio->uDeviceFeatures & UINT32_C(0xffffffff);
1757	memcpy(pu32, &val, cb);
1758	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1759	}
1760	else
1761	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDriverFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1762	{
1763	uint32_t val = pVirtio->uDriverFeatures & UINT32_C(0xffffffff);
1764	memcpy(pu32, &val, cb);
1765	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1766	}
1767	else
1768	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fDeviceStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1769	{
1770	(uint8_t )pu32 = pVirtio->fDeviceStatus;
1771	#ifdef LOG_ENABLED
1772	if (LogIs7Enabled())
1773	{
1774	char szOut[80] = { 0 };
1775	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
1776	Log(("%-23s: Guest read fDeviceStatus ................ (%s)\n", __FUNCTION__, szOut));
1777	}
1778	#endif
1779	}
1780	else
1781	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fIsrStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1782	{
1783	ASSERT_GUEST_MSG(cb == 1, ("%d\n", cb));
1784	(uint8_t )pu32 = pVirtio->uISR;
1785	pVirtio->uISR = 0;
1786	virtioLowerInterrupt( pDevIns, 0);
1787	Log((" (ISR read and cleared)\n"));
1788	}
1789	else
1790	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uVirtqSelect, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1791	VIRTIO_DEV_CONFIG_ACCESS( uVirtqSelect, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1792	else
1793	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uVirtqPfn, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1794	{
1795	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[uVirtq];
1796	*pu32 = pVirtQueue->GCPhysVirtqDesc >> GUEST_PAGE_SHIFT;
1797	Log(("%-23s: Guest read uVirtqPfn .................... %#x\n", __FUNCTION__, *pu32));
1798	}
1799	else
1800	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uQueueSize, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1801	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uQueueSize, uVirtq, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio->aVirtqueues);
1802	else
1803	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uQueueNotify, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1804	VIRTIO_DEV_CONFIG_ACCESS( uQueueNotify, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1805	#ifdef LEGACY_MSIX_SUPPORTED
1806	else
1807	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixConfig, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1808	VIRTIO_DEV_CONFIG_ACCESS( uMsixConfig, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1809	else
1810	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixVector, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1811	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uMsixVector, uVirtq, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio->aVirtqueues);
1812	#endif
1813	else if (offPort >= sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T))
1814	{
1815	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
1816	#ifdef IN_RING3
1817	/* Access device-specific configuration */
1818	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1819	int rc = pVirtioCC->pfnDevCapRead(pDevIns, offPort - sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T), pv, cb);
1820	return rc;
1821	#else
1822	return VINF_IOM_R3_IOPORT_READ;
1823	#endif
1824	}
1825	else
1826	{
1827	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
1828	Log2Func(("Bad guest read access to virtio_legacy_pci_common_cfg: offset=%#x, cb=%x\n",
1829	offPort, cb));
1830	int rc = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
1831	"virtioLegacyIOPortIn: no valid port at offset offset=%RTiop cb=%#x\n", offPort, cb);
1832	return rc;
1833	}
1834	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
1835	return VINF_SUCCESS;
1836	}
1837
1838	/**
1839	* @callback_method_impl{ * @callback_method_impl{FNIOMIOPORTNEWOUT}
1840	*
1841	* This I/O Port interface exists only to handle access from legacy drivers.
1842	*/
1843	static DECLCALLBACK(VBOXSTRICTRC) virtioLegacyIOPortOut(PPDMDEVINS pDevIns, void *pvUser, RTIOPORT offPort, uint32_t u32, unsigned cb)
1844	{
1845	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1846	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatWrite), a);
1847	RT_NOREF(pvUser);
1848
1849	uint16_t uVirtq = pVirtio->uVirtqSelect;
1850	uint32_t u32OnStack = u32; /* allows us to use this impl's MMIO parsing macros */
1851	void pv = &u32OnStack; / To use existing macros */
1852	int fWrite = 1; /* To use existing macros */
1853
1854	Log(("%-23s: Port written at offset=%RTiop, cb=%#x, u32=%#x\n", __FUNCTION__, offPort, cb, u32));
1855
1856	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uVirtqSelect, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1857	{
1858	if (u32 < RT_ELEMENTS(pVirtio->aVirtqueues))
1859	VIRTIO_DEV_CONFIG_ACCESS( uVirtqSelect, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1860	else
1861	LogFunc(("... WARNING: Guest attempted to write invalid virtq selector (ignoring)\n"));
1862	}
1863	else
1864	#ifdef LEGACY_MSIX_SUPPORTED
1865	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixConfig, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1866	VIRTIO_DEV_CONFIG_ACCESS( uMsixConfig, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1867	else
1868	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixVector, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1869	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uMsixVector, uVirtq, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio->aVirtqueues);
1870	else
1871	#endif
1872	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDeviceFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1873	{
1874	/* Check to see if guest acknowledged unsupported features */
1875	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1876	LogFunc(("... WARNING: Guest attempted to write readonly virtio_pci_common_cfg.device_feature (ignoring)\n"));
1877	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1878	return VINF_SUCCESS;
1879	}
1880	else
1881	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDriverFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1882	{
1883	memcpy(&pVirtio->uDriverFeatures, pv, cb);
1884	if ((pVirtio->uDriverFeatures & ~VIRTIO_DEV_INDEPENDENT_LEGACY_FEATURES_OFFERED) == 0)
1885	{
1886	Log(("Guest asked for features host does not support! (host=%x guest=%x)\n",
1887	VIRTIO_DEV_INDEPENDENT_LEGACY_FEATURES_OFFERED, pVirtio->uDriverFeatures));
1888	pVirtio->uDriverFeatures &= VIRTIO_DEV_INDEPENDENT_LEGACY_FEATURES_OFFERED;
1889	}
1890	if (!(pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_COMPLETE_HANDLED))
1891	{
1892	#ifdef IN_RING0
1893	Log6(("%-23s: RING0 => RING3 (demote)\n", __FUNCTION__));
1894	return VINF_IOM_R3_IOPORT_WRITE;
1895	#endif
1896	#ifdef IN_RING3
1897	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1898	virtioR3DoFeaturesCompleteOnceOnly(pVirtio, pVirtioCC);
1899	#endif
1900	}
1901	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1902	}
1903	else
1904	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uQueueSize, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1905	{
1906	VIRTIO_DEV_CONFIG_LOG_ACCESS(uQueueSize, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1907	LogFunc(("... WARNING: Guest attempted to write readonly device_feature (queue size) (ignoring)\n"));
1908	return VINF_SUCCESS;
1909	}
1910	else
1911	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fDeviceStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1912	{
1913	bool const fDriverInitiatedReset = (pVirtio->fDeviceStatus = (uint8_t)u32) == 0;
1914	bool const fDriverStateImproved = IS_DRIVER_OK(pVirtio) && !WAS_DRIVER_OK(pVirtio);
1915	#ifdef LOG_ENABLED
1916	if (LogIs7Enabled())
1917	{
1918	char szOut[80] = { 0 };
1919	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
1920	Log(("%-23s: Guest wrote fDeviceStatus ................ (%s)\n", __FUNCTION__, szOut));
1921	}
1922	#endif
1923	if (fDriverStateImproved \|\| fDriverInitiatedReset)
1924	{
1925	#ifdef IN_RING0
1926	Log6(("%-23s: RING0 => RING3 (demote)\n", __FUNCTION__));
1927	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1928	return VINF_IOM_R3_IOPORT_WRITE;
1929	#endif
1930	}
1931
1932	#ifdef IN_RING3
1933	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1934	if (fDriverInitiatedReset)
1935	virtioGuestR3WasReset(pDevIns, pVirtio, pVirtioCC);
1936
1937	else if (fDriverStateImproved)
1938	pVirtioCC->pfnStatusChanged(pVirtio, pVirtioCC, 1 /* fDriverOk */);
1939
1940	#endif
1941	pVirtio->fPrevDeviceStatus = pVirtio->fDeviceStatus;
1942	}
1943	else
1944	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uVirtqPfn, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1945	{
1946	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1947	uint64_t uVirtqPfn = (uint64_t)u32;
1948
1949	if (uVirtqPfn)
1950	{
1951	/* Transitional devices calculate ring physical addresses using rigid spec-defined formulae,
1952	* instead of guest conveying respective address of each ring, as "modern" VirtIO drivers do,
1953	* thus there is no virtq PFN or single base queue address stored in instance data for
1954	* this transitional device, but rather it is derived, when read back, from GCPhysVirtqDesc */
1955
1956	pVirtq->GCPhysVirtqDesc = uVirtqPfn * VIRTIO_PAGE_SIZE;
1957	pVirtq->GCPhysVirtqAvail = pVirtq->GCPhysVirtqDesc + sizeof(VIRTQ_DESC_T) * pVirtq->uQueueSize;
1958	pVirtq->GCPhysVirtqUsed =
1959	RT_ALIGN(pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[pVirtq->uQueueSize]), VIRTIO_PAGE_SIZE);
1960	}
1961	else
1962	{
1963	/* Don't set ring addresses for queue (to meaningless values), when guest resets the virtq's PFN */
1964	pVirtq->GCPhysVirtqDesc = 0;
1965	pVirtq->GCPhysVirtqAvail = 0;
1966	pVirtq->GCPhysVirtqUsed = 0;
1967	}
1968	Log(("%-23s: Guest wrote uVirtqPfn .................... %#x:\n"
1969	"%68s... %p -> GCPhysVirtqDesc\n%68s... %p -> GCPhysVirtqAvail\n%68s... %p -> GCPhysVirtqUsed\n",
1970	__FUNCTION__, u32, " ", pVirtq->GCPhysVirtqDesc, " ", pVirtq->GCPhysVirtqAvail, " ", pVirtq->GCPhysVirtqUsed));
1971	}
1972	else
1973	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uQueueNotify, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1974	{
1975	#ifdef IN_RING3
1976	ASSERT_GUEST_MSG(cb == 2, ("cb=%u\n", cb));
1977	pVirtio->uQueueNotify = u32 & 0xFFFF;
1978	if (uVirtq < VIRTQ_MAX_COUNT)
1979	{
1980	RT_UNTRUSTED_VALIDATED_FENCE();
1981
1982	/* Need to check that queue is configured. Legacy spec didn't have a queue enabled flag */
1983	if (pVirtio->aVirtqueues[pVirtio->uQueueNotify].GCPhysVirtqDesc)
1984	virtioCoreVirtqNotified(pDevIns, pVirtio, pVirtio->uQueueNotify, pVirtio->uQueueNotify /* uNotifyIdx */);
1985	else
1986	Log(("The queue (#%d) being notified has not been initialized.\n", pVirtio->uQueueNotify));
1987	}
1988	else
1989	Log(("Invalid queue number (%d)\n", pVirtio->uQueueNotify));
1990	#else
1991	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1992	return VINF_IOM_R3_IOPORT_WRITE;
1993	#endif
1994	}
1995	else
1996	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fIsrStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1997	{
1998	VIRTIO_DEV_CONFIG_LOG_ACCESS( fIsrStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1999	LogFunc(("... WARNING: Guest attempted to write readonly device_feature (ISR status) (ignoring)\n"));
2000	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
2001	return VINF_SUCCESS;
2002	}
2003	else if (offPort >= sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T))
2004	{
2005	#ifdef IN_RING3
2006
2007	/* Access device-specific configuration */
2008	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2009	return pVirtioCC->pfnDevCapWrite(pDevIns, offPort - sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T), pv, cb);
2010	#else
2011	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
2012	return VINF_IOM_R3_IOPORT_WRITE;
2013	#endif
2014	}
2015	else
2016	{
2017	Log2Func(("Bad guest write access to virtio_legacy_pci_common_cfg: offset=%#x, cb=0x%x\n",
2018	offPort, cb));
2019	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
2020	int rc = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
2021	"virtioLegacyIOPortOut: no valid port at offset offset=%RTiop cb=0x%#x\n", offPort, cb);
2022	return rc;
2023	}
2024
2025	RT_NOREF(uVirtq);
2026	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
2027	return VINF_SUCCESS;
2028	}
2029
2030
2031	/**
2032	* Read from the device specific configuration at the given offset.
2033	*
2034	* @returns VBox status code.
2035	* @param pDevIns
2036	*/
2037	DECLINLINE(VBOXSTRICTRC) virtioDeviceCfgRead(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC,
2038	uint32_t offDevCfg, void *pv, unsigned cb)
2039	{
2040	#ifdef IN_RING3
2041	/*
2042	* Callback to client to manage device-specific configuration.
2043	*/
2044	VBOXSTRICTRC rcStrict = pVirtioCC->pfnDevCapRead(pDevIns, offDevCfg, pv, cb);
2045
2046	/*
2047	* Anytime any part of the dev-specific dev config (which this virtio core implementation sees
2048	* as a blob, and virtio dev-specific code separates into fields) is READ, it must be compared
2049	* for deltas from previous read to maintain a config gen. seq. counter (VirtIO 1.0, section 4.1.4.3.1)
2050	*/
2051	bool fDevSpecificFieldChanged = RT_BOOL(memcmp(pVirtioCC->pbDevSpecificCfg + offDevCfg,
2052	pVirtioCC->pbPrevDevSpecificCfg + offDevCfg,
2053	RT_MIN(cb, pVirtioCC->cbDevSpecificCfg - offDevCfg)));
2054
2055	memcpy(pVirtioCC->pbPrevDevSpecificCfg, pVirtioCC->pbDevSpecificCfg, pVirtioCC->cbDevSpecificCfg);
2056
2057	if (pVirtio->fGenUpdatePending \|\| fDevSpecificFieldChanged)
2058	{
2059	++pVirtio->uConfigGeneration;
2060	Log6Func(("Bumped cfg. generation to %d because %s%s\n", pVirtio->uConfigGeneration,
2061	fDevSpecificFieldChanged ? "<dev cfg changed> " : "",
2062	pVirtio->fGenUpdatePending ? "<update was pending>" : ""));
2063	pVirtio->fGenUpdatePending = false;
2064	}
2065
2066	virtioLowerInterrupt(pDevIns, 0);
2067	return rcStrict;
2068	#else
2069	RT_NOREF(pDevIns, pVirtio, pVirtioCC, offDevCfg, pv, cb);
2070	return VINF_IOM_R3_MMIO_READ;
2071	#endif
2072	}
2073
2074
2075	/**
2076	* @callback_method_impl{FNIOMMMIONEWREAD,
2077	* Memory mapped I/O Handler for PCI Capabilities read operations.}
2078	*
2079	* This MMIO handler specifically supports the VIRTIO_PCI_CAP_PCI_CFG capability defined
2080	* in the VirtIO 1.0 specification, section 4.1.4.7, and as such is restricted to reads
2081	* of 1, 2 or 4 bytes, only.
2082	*
2083	*/
2084	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioRead(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void pv, unsigned cb)
2085	{
2086	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
2087	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2088	AssertReturn(cb == 1 \|\| cb == 2 \|\| cb == 4, VINF_IOM_MMIO_UNUSED_FF);
2089	Assert(pVirtio == (PVIRTIOCORE)pvUser); RT_NOREF(pvUser);
2090
2091	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatRead), a);
2092
2093	VBOXSTRICTRC rcStrict;
2094	uint32_t uOffset;
2095	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocDeviceCap))
2096	rcStrict = virtioDeviceCfgRead(pDevIns, pVirtio, pVirtioCC, uOffset, pv, cb);
2097	else if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocCommonCfgCap))
2098	rcStrict = virtioCommonCfgAccessed(pDevIns, pVirtio, pVirtioCC, false /* fWrite */, uOffset, cb, pv);
2099	else if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocIsrCap))
2100	{
2101	(uint8_t )pv = pVirtio->uISR;
2102	Log6Func(("Read and clear ISR\n"));
2103	pVirtio->uISR = 0; /* VirtIO spec requires reads of ISR to clear it */
2104	virtioLowerInterrupt(pDevIns, 0);
2105	rcStrict = VINF_SUCCESS;
2106	}
2107	else
2108	{
2109	ASSERT_GUEST_MSG_FAILED(("Bad read access to mapped capabilities region: off=%RGp cb=%u\n", off, cb));
2110	memset(pv, 0xFF, cb);
2111	rcStrict = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
2112	"virtioMmioRead: Bad MMIO access to capabilities, offset=%RTiop cb=%08x\n", off, cb);
2113	}
2114
2115	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
2116	return rcStrict;
2117	}
2118
2119	/**
2120	* @callback_method_impl{FNIOMMMIONEWREAD,
2121	* Memory mapped I/O Handler for PCI Capabilities write operations.}
2122	*
2123	* This MMIO handler specifically supports the VIRTIO_PCI_CAP_PCI_CFG capability defined
2124	* in the VirtIO 1.0 specification, section 4.1.4.7, and as such is restricted to writes
2125	* of 1, 2 or 4 bytes, only.
2126	*/
2127	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioWrite(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void const pv, unsigned cb)
2128	{
2129	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
2130	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2131	AssertReturn(cb == 1 \|\| cb == 2 \|\| cb == 4, VERR_INVALID_PARAMETER);
2132	Assert(pVirtio == (PVIRTIOCORE)pvUser); RT_NOREF(pvUser);
2133	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatWrite), a);
2134
2135	VBOXSTRICTRC rcStrict;
2136	uint32_t uOffset;
2137	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocDeviceCap))
2138	{
2139	#ifdef IN_RING3
2140	/*
2141	* Foreward this MMIO write access for client to deal with.
2142	*/
2143	rcStrict = pVirtioCC->pfnDevCapWrite(pDevIns, uOffset, pv, cb);
2144	#else
2145	Log6(("%-23s: RING0 => RING3 (demote)\n", __FUNCTION__));
2146	rcStrict = VINF_IOM_R3_MMIO_WRITE;
2147	#endif
2148	}
2149	else if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocCommonCfgCap))
2150	rcStrict = virtioCommonCfgAccessed(pDevIns, pVirtio, pVirtioCC, true /* fWrite /, uOffset, cb, (void )pv);
2151	else if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocIsrCap) && cb == sizeof(uint8_t))
2152	{
2153	pVirtio->uISR = (uint8_t )pv;
2154	Log6Func(("Setting uISR = 0x%02x (virtq interrupt: %d, dev confg interrupt: %d)\n",
2155	pVirtio->uISR & 0xff,
2156	pVirtio->uISR & VIRTIO_ISR_VIRTQ_INTERRUPT,
2157	RT_BOOL(pVirtio->uISR & VIRTIO_ISR_DEVICE_CONFIG)));
2158	rcStrict = VINF_SUCCESS;
2159	}
2160	else if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocNotifyCap) && cb == sizeof(uint16_t))
2161	{
2162	/* This should be guest driver dropping index of a new descriptor in avail ring */
2163	virtioCoreVirtqNotified(pDevIns, pVirtio, uOffset / VIRTIO_NOTIFY_OFFSET_MULTIPLIER, (uint16_t )pv);
2164	rcStrict = VINF_SUCCESS;
2165	}
2166	else
2167	{
2168	ASSERT_GUEST_MSG_FAILED(("Bad write access to mapped capabilities region: off=%RGp pv=%#p{%.*Rhxs} cb=%u\n", off, pv, cb, pv, cb));
2169	rcStrict = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
2170	"virtioMmioRead: Bad MMIO access to capabilities, offset=%RTiop cb=%08x\n", off, cb);
2171	}
2172
2173	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
2174	return rcStrict;
2175	}
2176
2177
2178	/**
2179	* @callback_method_impl{FNIOMMMIONEWREAD,
2180	* Memory mapped I/O Handler for Virtio over MMIO read operations.}
2181	*
2182	*/
2183	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioTransportRead(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void pv, unsigned cb)
2184	{
2185	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
2186	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2187	RT_NOREF(pvUser);
2188	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatRead), a);
2189
2190	if (off >= VIRTIO_MMIO_SIZE)
2191	{
2192	VBOXSTRICTRC rcStrict = virtioDeviceCfgRead(pDevIns, pVirtio, pVirtioCC, (uint32_t)off - VIRTIO_MMIO_SIZE, pv, cb);
2193	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
2194	return rcStrict;
2195	}
2196
2197	/* All accesses below need to be aligned on a 32-bit boundary and must be 32-bit in size. */
2198	ASSERT_GUEST_MSG_RETURN(!(off & 0x3) && cb == sizeof(uint32_t),
2199	("Bad read access: off=%RGp pv=%#p{%.*Rhxs} cb=%u\n", off, pv, cb, pv, cb),
2200	VINF_IOM_MMIO_UNUSED_FF);
2201
2202	int rc = VINF_SUCCESS;
2203	uint32_t pu32 = (uint32_t )pv;
2204	switch (off)
2205	{
2206	case VIRTIO_MMIO_REG_MAGIC_OFF:
2207	*pu32 = RT_H2LE_U32(VIRTIO_MMIO_REG_MAGIC_VALUE);
2208	break;
2209	case VIRTIO_MMIO_REG_VERSION_OFF:
2210	*pu32 = RT_H2LE_U32(VIRTIO_MMIO_REG_VERSION_VALUE);
2211	break;
2212	case VIRTIO_MMIO_REG_DEVICEID_OFF:
2213	*pu32 = pVirtio->uDeviceType;
2214	break;
2215	case VIRTIO_MMIO_REG_VENDORID_OFF:
2216	*pu32 = RT_H2LE_U32(DEVICE_PCI_VENDOR_ID_VIRTIO);
2217	break;
2218	case VIRTIO_MMIO_REG_DEVICEFEAT_OFF:
2219	{
2220	switch (pVirtio->uDeviceFeaturesSelect)
2221	{
2222	case 0:
2223	*pu32 = pVirtio->uDeviceFeatures & UINT32_C(0xffffffff);
2224	break;
2225	case 1:
2226	*pu32 = pVirtio->uDeviceFeatures >> 32;
2227	break;
2228	default:
2229	LogFunc(("Guest read uDeviceFeatures with out of range selector (%#x), returning 0\n",
2230	pVirtio->uDeviceFeaturesSelect));
2231	rc = VINF_IOM_MMIO_UNUSED_00;
2232	}
2233	break;
2234	}
2235	case VIRTIO_MMIO_REG_QUEUENUMMAX_OFF:
2236	pu32 = VIRTQ_SIZE; /* @todo */
2237	break;
2238	case VIRTIO_MMIO_REG_QUEUERDY_OFF:
2239	{
2240	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2241	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2242	*pu32 = pVirtQueue->uEnable;
2243	break;
2244	}
2245	case VIRTIO_MMIO_REG_INTRSTATUS_OFF:
2246	*pu32 = pVirtio->uISR;
2247	break;
2248	case VIRTIO_MMIO_REG_DEVSTATUS_OFF:
2249	*pu32 = virtioDeviceStatusRead(pVirtio);
2250	break;
2251	case VIRTIO_MMIO_REG_CFGGEN_OFF:
2252	*pu32 = pVirtio->uConfigGeneration;
2253	break;
2254	default:
2255	ASSERT_GUEST_MSG_FAILED(("Bad read access to mapped capabilities region: off=%RGp cb=%u\n", off, cb));
2256	rc = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
2257	"virtioMmioTransportRead: Bad MMIO access to capabilities, offset=%RTiop cb=%08x\n", off, cb);
2258	}
2259
2260	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
2261	return rc;
2262	}
2263
2264	/**
2265	* @callback_method_impl{FNIOMMMIONEWREAD,
2266	* Memory mapped I/O Handler for Virtio over MMIO write operations.}
2267	*/
2268	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioTransportWrite(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void const pv, unsigned cb)
2269	{
2270	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
2271	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2272	RT_NOREF(pvUser);
2273	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatWrite), a);
2274
2275	if (off >= VIRTIO_MMIO_SIZE)
2276	{
2277	#ifdef IN_RING3
2278	/*
2279	* Forward this MMIO write access for client to deal with.
2280	*/
2281	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
2282	return pVirtioCC->pfnDevCapWrite(pDevIns, (uint32_t)off - VIRTIO_MMIO_SIZE, pv, cb);
2283	#else
2284	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
2285	Log6(("%-23s: RING0 => RING3 (demote)\n", __FUNCTION__));
2286	return VINF_IOM_R3_MMIO_WRITE;
2287	#endif
2288	}
2289
2290	/* All accesses below need to be aligned on a 32-bit boundary and must be 32-bit in size. */
2291	ASSERT_GUEST_MSG_RETURN(!(off & 0x3) && cb == sizeof(uint32_t),
2292	("Bad write access: off=%RGp pv=%#p{%.*Rhxs} cb=%u\n", off, pv, cb, pv, cb),
2293	VINF_SUCCESS);
2294
2295	int rc = VINF_SUCCESS;
2296	uint32_t const u32Val = (const uint32_t )pv;
2297	switch (off)
2298	{
2299	case VIRTIO_MMIO_REG_DEVICEFEATSEL_OFF:
2300	{
2301	pVirtio->uDeviceFeaturesSelect = u32Val;
2302	break;
2303	}
2304	case VIRTIO_MMIO_REG_DRIVERFEAT_OFF:
2305	{
2306	switch (pVirtio->uDriverFeaturesSelect)
2307	{
2308	case 0:
2309	pVirtio->uDriverFeatures = (pVirtio->uDriverFeatures & UINT64_C(0xffffffff00000000)) \| u32Val;
2310	pVirtio->fDriverFeaturesWritten \|= DRIVER_FEATURES_0_WRITTEN;
2311	LogFunc(("Set DRIVER_FEATURES_0_WRITTEN. pVirtio->fDriverFeaturesWritten=%d\n", pVirtio->fDriverFeaturesWritten));
2312	if ( (pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_0_AND_1_WRITTEN) == DRIVER_FEATURES_0_AND_1_WRITTEN
2313	&& !(pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_COMPLETE_HANDLED))
2314	#ifdef IN_RING0
2315	return VINF_IOM_R3_MMIO_WRITE;
2316	#endif
2317	#ifdef IN_RING3
2318	virtioR3DoFeaturesCompleteOnceOnly(pVirtio, pVirtioCC);
2319	#endif
2320	break;
2321	case 1:
2322	pVirtio->uDriverFeatures = (pVirtio->uDriverFeatures & UINT64_C(0x00000000ffffffff)) \| ((uint64_t)u32Val << 32);
2323	pVirtio->fDriverFeaturesWritten \|= DRIVER_FEATURES_1_WRITTEN;
2324	LogFunc(("Set DRIVER_FEATURES_1_WRITTEN. pVirtio->fDriverFeaturesWritten=%d\n", pVirtio->fDriverFeaturesWritten));
2325	if ( (pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_0_AND_1_WRITTEN) == DRIVER_FEATURES_0_AND_1_WRITTEN
2326	&& !(pVirtio->fDriverFeaturesWritten & DRIVER_FEATURES_COMPLETE_HANDLED))
2327	#ifdef IN_RING0
2328	return VINF_IOM_R3_MMIO_WRITE;
2329	#endif
2330	#ifdef IN_RING3
2331	virtioR3DoFeaturesCompleteOnceOnly(pVirtio, pVirtioCC);
2332	#endif
2333	break;
2334	default:
2335	LogFunc(("Guest wrote uDriverFeatures with out of range selector (%#x), returning 0\n",
2336	pVirtio->uDriverFeaturesSelect));
2337	return VINF_SUCCESS;
2338	}
2339	break;
2340	}
2341	case VIRTIO_MMIO_REG_DRIVERFEATSEL_OFF:
2342	{
2343	pVirtio->uDriverFeaturesSelect = u32Val;
2344	break;
2345	}
2346	case VIRTIO_MMIO_REG_QUEUESEL_OFF:
2347	{
2348	if (u32Val < RT_ELEMENTS(pVirtio->aVirtqueues))
2349	pVirtio->uVirtqSelect = (uint16_t)u32Val;
2350	else
2351	LogFunc(("... WARNING: Guest attempted to write invalid virtq selector (ignoring)\n"));
2352	break;
2353	}
2354	case VIRTIO_MMIO_REG_QUEUENUM_OFF:
2355	{
2356	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2357	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2358	pVirtQueue->uQueueSize = (uint16_t)u32Val;
2359	break;
2360	}
2361	case VIRTIO_MMIO_REG_QUEUERDY_OFF:
2362	{
2363	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2364	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2365	pVirtQueue->uEnable = (uint16_t)u32Val;
2366	break;
2367	}
2368	case VIRTIO_MMIO_REG_QUEUENOTIFY_OFF:
2369	{
2370	virtioCoreVirtqNotified(pDevIns, pVirtio, u32Val, (uint16_t)u32Val);
2371	break;
2372	}
2373	case VIRTIO_MMIO_REG_INTRACK_OFF:
2374	{
2375	pVirtio->uISR &= ~u32Val;
2376	if (!pVirtio->uISR)
2377	virtioLowerInterrupt(pDevIns, 0);
2378	break;
2379	}
2380	case VIRTIO_MMIO_REG_DEVSTATUS_OFF:
2381	{
2382	rc = virtioDeviceStatusWrite(pDevIns, pVirtio, pVirtioCC, (uint8_t)u32Val);
2383	break;
2384	}
2385	case VIRTIO_MMIO_REG_QUEUEALIGN_LEGACY_OFF:
2386	{
2387	/* Written by edk2 even though we don't offer legacy mode, ignore. */
2388	break;
2389	}
2390	case VIRTIO_MMIO_REG_QUEUEDESCLOW_OFF:
2391	{
2392	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2393	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2394	pVirtQueue->GCPhysVirtqDesc = (pVirtQueue->GCPhysVirtqDesc & UINT64_C(0xffffffff00000000)) \| u32Val;
2395	break;
2396	}
2397	case VIRTIO_MMIO_REG_QUEUEDESCHIGH_OFF:
2398	{
2399	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2400	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2401	pVirtQueue->GCPhysVirtqDesc = (pVirtQueue->GCPhysVirtqDesc & UINT64_C(0x00000000ffffffff)) \| ((uint64_t)u32Val << 32);
2402	break;
2403	}
2404	case VIRTIO_MMIO_REG_QUEUEDRVLOW_OFF:
2405	{
2406	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2407	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2408	pVirtQueue->GCPhysVirtqAvail = (pVirtQueue->GCPhysVirtqAvail & UINT64_C(0xffffffff00000000)) \| u32Val;
2409	break;
2410	}
2411	case VIRTIO_MMIO_REG_QUEUEDRVHIGH_OFF:
2412	{
2413	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2414	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2415	pVirtQueue->GCPhysVirtqAvail = (pVirtQueue->GCPhysVirtqAvail & UINT64_C(0x00000000ffffffff)) \| ((uint64_t)u32Val << 32);
2416	break;
2417	}
2418	case VIRTIO_MMIO_REG_QUEUEDEVLOW_OFF:
2419	{
2420	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2421	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2422	pVirtQueue->GCPhysVirtqUsed = (pVirtQueue->GCPhysVirtqUsed & UINT64_C(0xffffffff00000000)) \| u32Val;
2423	break;
2424	}
2425	case VIRTIO_MMIO_REG_QUEUEDEVHIGH_OFF:
2426	{
2427	Assert(pVirtio->uVirtqSelect < RT_ELEMENTS(pVirtio->aVirtqueues));
2428	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[pVirtio->uVirtqSelect];
2429	pVirtQueue->GCPhysVirtqUsed = (pVirtQueue->GCPhysVirtqUsed & UINT64_C(0x00000000ffffffff)) \| ((uint64_t)u32Val << 32);
2430	break;
2431	}
2432	default:
2433	ASSERT_GUEST_MSG_FAILED(("Bad write access to mapped capabilities region: off=%RGp pv=%#p{%.*Rhxs} cb=%u\n", off, pv, cb, pv, cb));
2434	rc = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
2435	"virtioMmioTransportWrite: Bad MMIO access to capabilities, offset=%RTiop cb=%08x\n", off, cb);
2436	}
2437
2438	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
2439	return rc;
2440	}
2441
2442
2443	#ifdef IN_RING3
2444
2445	/**
2446	* @callback_method_impl{FNPCICONFIGREAD}
2447	*/
2448	static DECLCALLBACK(VBOXSTRICTRC) virtioR3PciConfigRead(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev,
2449	uint32_t uAddress, unsigned cb, uint32_t *pu32Value)
2450	{
2451	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
2452	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2453	RT_NOREF(pPciDev);
2454
2455	if (uAddress == pVirtio->uPciCfgDataOff)
2456	{
2457	/* See comments in PCI Cfg capability initialization (in capabilities setup section of this code) */
2458	struct virtio_pci_cap *pPciCap = &pVirtioCC->pPciCfgCap->pciCap;
2459	uint32_t uLength = pPciCap->uLength;
2460
2461	Log7Func((" pDevIns=%p pPciDev=%p uAddress=%#x%s cb=%u uLength=%d, bar=%d\n",
2462	pDevIns, pPciDev, uAddress, uAddress < 0x10 ? " " : "", cb, uLength, pPciCap->uBar));
2463
2464	if ( (uLength != 1 && uLength != 2 && uLength != 4)
2465	\|\| pPciCap->uBar != VIRTIO_REGION_PCI_CAP)
2466	{
2467	ASSERT_GUEST_MSG_FAILED(("Guest read virtio_pci_cfg_cap.pci_cfg_data using mismatching config. "
2468	"Ignoring\n"));
2469	*pu32Value = UINT32_MAX;
2470	return VINF_SUCCESS;
2471	}
2472
2473	VBOXSTRICTRC rcStrict = virtioMmioRead(pDevIns, pVirtio, pPciCap->uOffset, pu32Value, cb);
2474	Log7Func((" Guest read virtio_pci_cfg_cap.pci_cfg_data, bar=%d, offset=%d, length=%d, result=0x%x -> %Rrc\n",
2475	pPciCap->uBar, pPciCap->uOffset, uLength, *pu32Value, VBOXSTRICTRC_VAL(rcStrict)));
2476	return rcStrict;
2477	}
2478	Log7Func((" pDevIns=%p pPciDev=%p uAddress=%#x%s cb=%u pu32Value=%p\n",
2479	pDevIns, pPciDev, uAddress, uAddress < 0x10 ? " " : "", cb, pu32Value));
2480	return VINF_PDM_PCI_DO_DEFAULT;
2481	}
2482
2483	/**
2484	* @callback_method_impl{FNPCICONFIGWRITE}
2485	*/
2486	static DECLCALLBACK(VBOXSTRICTRC) virtioR3PciConfigWrite(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev,
2487	uint32_t uAddress, unsigned cb, uint32_t u32Value)
2488	{
2489	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
2490	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
2491	RT_NOREF(pPciDev);
2492
2493	Log7Func(("pDevIns=%p pPciDev=%p uAddress=%#x %scb=%u u32Value=%#x\n", pDevIns, pPciDev, uAddress, uAddress < 0xf ? " " : "", cb, u32Value));
2494	if (uAddress == pVirtio->uPciCfgDataOff)
2495	{
2496	/* See comments in PCI Cfg capability initialization (in capabilities setup section of this code) */
2497	struct virtio_pci_cap *pPciCap = &pVirtioCC->pPciCfgCap->pciCap;
2498	uint32_t uLength = pPciCap->uLength;
2499
2500	if ( (uLength != 1 && uLength != 2 && uLength != 4)
2501	\|\| cb != uLength
2502	\|\| pPciCap->uBar != VIRTIO_REGION_PCI_CAP)
2503	{
2504	ASSERT_GUEST_MSG_FAILED(("Guest write virtio_pci_cfg_cap.pci_cfg_data using mismatching config. Ignoring\n"));
2505	return VINF_SUCCESS;
2506	}
2507
2508	VBOXSTRICTRC rcStrict = virtioMmioWrite(pDevIns, pVirtio, pPciCap->uOffset, &u32Value, cb);
2509	Log2Func(("Guest wrote virtio_pci_cfg_cap.pci_cfg_data, bar=%d, offset=%x, length=%x, value=%d -> %Rrc\n",
2510	pPciCap->uBar, pPciCap->uOffset, uLength, u32Value, VBOXSTRICTRC_VAL(rcStrict)));
2511	return rcStrict;
2512	}
2513	return VINF_PDM_PCI_DO_DEFAULT;
2514	}
2515
2516
2517	/*********************************************************************************************************************************
2518	* Saved state (SSM) *
2519	*********************************************************************************************************************************/
2520
2521
2522	/**
2523	* Loads a saved device state (called from device-specific code on SSM final pass)
2524	*
2525	* @param pVirtio Pointer to the shared virtio state.
2526	* @param pHlp The ring-3 device helpers.
2527	* @param pSSM The saved state handle.
2528	* @returns VBox status code.
2529	*/
2530	DECLHIDDEN(int) virtioCoreR3LegacyDeviceLoadExec(PVIRTIOCORE pVirtio, PCPDMDEVHLPR3 pHlp, PSSMHANDLE pSSM,
2531	uint32_t uVersion, uint32_t uVirtioLegacy_3_1_Beta)
2532	{
2533	int rc;
2534	uint32_t uDriverFeaturesLegacy32bit;
2535
2536	rc = pHlp->pfnSSMGetU32( pSSM, &uDriverFeaturesLegacy32bit);
2537	AssertRCReturn(rc, rc);
2538	pVirtio->uDriverFeatures = (uint64_t)uDriverFeaturesLegacy32bit;
2539
2540	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtio->uVirtqSelect);
2541	AssertRCReturn(rc, rc);
2542
2543	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->fDeviceStatus);
2544	AssertRCReturn(rc, rc);
2545
2546	#ifdef LOG_ENABLED
2547	char szOut[80] = { 0 };
2548	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
2549	Log(("Loaded legacy device status = (%s)\n", szOut));
2550	#endif
2551
2552	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->uISR);
2553	AssertRCReturn(rc, rc);
2554
2555	uint32_t cQueues = 3; /* This constant default value copied from earliest v0.9 code */
2556	if (uVersion > uVirtioLegacy_3_1_Beta)
2557	{
2558	rc = pHlp->pfnSSMGetU32(pSSM, &cQueues);
2559	AssertRCReturn(rc, rc);
2560	}
2561
2562	AssertLogRelMsgReturn(cQueues <= VIRTQ_MAX_COUNT, ("%#x\n", cQueues), VERR_SSM_LOAD_CONFIG_MISMATCH);
2563	AssertLogRelMsgReturn(pVirtio->uVirtqSelect < cQueues \|\| (cQueues == 0 && pVirtio->uVirtqSelect),
2564	("uVirtqSelect=%u cQueues=%u\n", pVirtio->uVirtqSelect, cQueues),
2565	VERR_SSM_LOAD_CONFIG_MISMATCH);
2566
2567	Log(("\nRestoring %d legacy-only virtio-net device queues from saved state:\n", cQueues));
2568	for (unsigned uVirtq = 0; uVirtq < cQueues; uVirtq++)
2569	{
2570	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
2571
2572	if (uVirtq == cQueues - 1)
2573	RTStrPrintf(pVirtq->szName, sizeof(pVirtq->szName), "legacy-ctrlq");
2574	else if (uVirtq % 2)
2575	RTStrPrintf(pVirtq->szName, sizeof(pVirtq->szName), "legacy-xmitq<%d>", uVirtq / 2);
2576	else
2577	RTStrPrintf(pVirtq->szName, sizeof(pVirtq->szName), "legacy-recvq<%d>", uVirtq / 2);
2578
2579	rc = pHlp->pfnSSMGetU16(pSSM, &pVirtq->uQueueSize);
2580	AssertRCReturn(rc, rc);
2581
2582	uint32_t uVirtqPfn;
2583	rc = pHlp->pfnSSMGetU32(pSSM, &uVirtqPfn);
2584	AssertRCReturn(rc, rc);
2585
2586	rc = pHlp->pfnSSMGetU16(pSSM, &pVirtq->uAvailIdxShadow);
2587	AssertRCReturn(rc, rc);
2588
2589	rc = pHlp->pfnSSMGetU16(pSSM, &pVirtq->uUsedIdxShadow);
2590	AssertRCReturn(rc, rc);
2591
2592	if (uVirtqPfn)
2593	{
2594	pVirtq->GCPhysVirtqDesc = (uint64_t)uVirtqPfn * VIRTIO_PAGE_SIZE;
2595	pVirtq->GCPhysVirtqAvail = pVirtq->GCPhysVirtqDesc + sizeof(VIRTQ_DESC_T) * pVirtq->uQueueSize;
2596	pVirtq->GCPhysVirtqUsed =
2597	RT_ALIGN(pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[pVirtq->uQueueSize]), VIRTIO_PAGE_SIZE);
2598	pVirtq->uEnable = 1;
2599	}
2600	else
2601	{
2602	LogFunc(("WARNING: QUEUE \"%s\" PAGE NUMBER ZERO IN SAVED STATE\n", pVirtq->szName));
2603	pVirtq->uEnable = 0;
2604	}
2605	pVirtq->uNotifyOffset = 0; /* unused in legacy mode */
2606	pVirtq->uMsixVector = 0; /* unused in legacy mode */
2607	}
2608	pVirtio->fGenUpdatePending = 0; /* unused in legacy mode */
2609	pVirtio->uConfigGeneration = 0; /* unused in legacy mode */
2610	pVirtio->uPciCfgDataOff = 0; /* unused in legacy mode (port I/O used instead) */
2611
2612	return VINF_SUCCESS;
2613	}
2614
2615	/**
2616	* Loads a saved device state (called from device-specific code on SSM final pass)
2617	*
2618	* Note: This loads state saved by a Modern (VirtIO 1.0+) device, of which this transitional device is one,
2619	* and thus supports both legacy and modern guest virtio drivers.
2620	*
2621	* @param pVirtio Pointer to the shared virtio state.
2622	* @param pHlp The ring-3 device helpers.
2623	* @param pSSM The saved state handle.
2624	* @returns VBox status code.
2625	*/
2626	DECLHIDDEN(int) virtioCoreR3ModernDeviceLoadExec(PVIRTIOCORE pVirtio, PCPDMDEVHLPR3 pHlp, PSSMHANDLE pSSM,
2627	uint32_t uVersion, uint32_t uTestVersion, uint32_t cQueues)
2628	{
2629	RT_NOREF2(cQueues, uVersion);
2630	LogFunc(("\n"));
2631	/*
2632	* Check the marker and (embedded) version number.
2633	*/
2634	uint64_t uMarker = 0;
2635	int rc;
2636
2637	rc = pHlp->pfnSSMGetU64(pSSM, &uMarker);
2638	AssertRCReturn(rc, rc);
2639	if (uMarker != VIRTIO_SAVEDSTATE_MARKER)
2640	return pHlp->pfnSSMSetLoadError(pSSM, VERR_SSM_DATA_UNIT_FORMAT_CHANGED, RT_SRC_POS,
2641	N_("Expected marker value %#RX64 found %#RX64 instead"),
2642	VIRTIO_SAVEDSTATE_MARKER, uMarker);
2643	uint32_t uVersionSaved = 0;
2644	rc = pHlp->pfnSSMGetU32(pSSM, &uVersionSaved);
2645	AssertRCReturn(rc, rc);
2646	if (uVersionSaved != uTestVersion)
2647	return pHlp->pfnSSMSetLoadError(pSSM, VERR_SSM_DATA_UNIT_FORMAT_CHANGED, RT_SRC_POS,
2648	N_("Unsupported virtio version: %u"), uVersionSaved);
2649	/*
2650	* Load the state.
2651	*/
2652	rc = pHlp->pfnSSMGetU32( pSSM, &pVirtio->fLegacyDriver);
2653	AssertRCReturn(rc, rc);
2654	rc = pHlp->pfnSSMGetBool( pSSM, &pVirtio->fGenUpdatePending);
2655	AssertRCReturn(rc, rc);
2656	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->fDeviceStatus);
2657	AssertRCReturn(rc, rc);
2658	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->uConfigGeneration);
2659	AssertRCReturn(rc, rc);
2660	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->uPciCfgDataOff);
2661	AssertRCReturn(rc, rc);
2662	rc = pHlp->pfnSSMGetU8( pSSM, &pVirtio->uISR);
2663	AssertRCReturn(rc, rc);
2664	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtio->uVirtqSelect);
2665	AssertRCReturn(rc, rc);
2666	rc = pHlp->pfnSSMGetU32( pSSM, &pVirtio->uDeviceFeaturesSelect);
2667	AssertRCReturn(rc, rc);
2668	rc = pHlp->pfnSSMGetU32( pSSM, &pVirtio->uDriverFeaturesSelect);
2669	AssertRCReturn(rc, rc);
2670	rc = pHlp->pfnSSMGetU64( pSSM, &pVirtio->uDriverFeatures);
2671	AssertRCReturn(rc, rc);
2672
2673	/** @todo Adapt this loop use cQueues argument instead of static queue count (safely with SSM versioning) */
2674	for (uint32_t i = 0; i < VIRTQ_MAX_COUNT; i++)
2675	{
2676	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[i];
2677	rc = pHlp->pfnSSMGetGCPhys64( pSSM, &pVirtq->GCPhysVirtqDesc);
2678	AssertRCReturn(rc, rc);
2679	rc = pHlp->pfnSSMGetGCPhys64( pSSM, &pVirtq->GCPhysVirtqAvail);
2680	AssertRCReturn(rc, rc);
2681	rc = pHlp->pfnSSMGetGCPhys64( pSSM, &pVirtq->GCPhysVirtqUsed);
2682	AssertRCReturn(rc, rc);
2683	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uNotifyOffset);
2684	AssertRCReturn(rc, rc);
2685	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uMsixVector);
2686	AssertRCReturn(rc, rc);
2687	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uEnable);
2688	AssertRCReturn(rc, rc);
2689	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uQueueSize);
2690	AssertRCReturn(rc, rc);
2691	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uAvailIdxShadow);
2692	AssertRCReturn(rc, rc);
2693	rc = pHlp->pfnSSMGetU16( pSSM, &pVirtq->uUsedIdxShadow);
2694	AssertRCReturn(rc, rc);
2695	rc = pHlp->pfnSSMGetMem( pSSM, pVirtq->szName, sizeof(pVirtq->szName));
2696	AssertRCReturn(rc, rc);
2697	}
2698	return VINF_SUCCESS;
2699	}
2700
2701	/**
2702	* Called from the FNSSMDEVSAVEEXEC function of the device.
2703	*
2704	* @param pVirtio Pointer to the shared virtio state.
2705	* @param pHlp The ring-3 device helpers.
2706	* @param pSSM The saved state handle.
2707	* @returns VBox status code.
2708	*/
2709	DECLHIDDEN(int) virtioCoreR3SaveExec(PVIRTIOCORE pVirtio, PCPDMDEVHLPR3 pHlp, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t cQueues)
2710	{
2711	RT_NOREF(cQueues);
2712	/** @todo figure out a way to save cQueues (with SSM versioning) */
2713
2714	LogFunc(("\n"));
2715	pHlp->pfnSSMPutU64(pSSM, VIRTIO_SAVEDSTATE_MARKER);
2716	pHlp->pfnSSMPutU32(pSSM, uVersion);
2717
2718	pHlp->pfnSSMPutU32( pSSM, pVirtio->fLegacyDriver);
2719	pHlp->pfnSSMPutBool(pSSM, pVirtio->fGenUpdatePending);
2720	pHlp->pfnSSMPutU8( pSSM, pVirtio->fDeviceStatus);
2721	pHlp->pfnSSMPutU8( pSSM, pVirtio->uConfigGeneration);
2722	pHlp->pfnSSMPutU8( pSSM, pVirtio->uPciCfgDataOff);
2723	pHlp->pfnSSMPutU8( pSSM, pVirtio->uISR);
2724	pHlp->pfnSSMPutU16( pSSM, pVirtio->uVirtqSelect);
2725	pHlp->pfnSSMPutU32( pSSM, pVirtio->uDeviceFeaturesSelect);
2726	pHlp->pfnSSMPutU32( pSSM, pVirtio->uDriverFeaturesSelect);
2727	pHlp->pfnSSMPutU64( pSSM, pVirtio->uDriverFeatures);
2728
2729	for (uint32_t i = 0; i < VIRTQ_MAX_COUNT; i++)
2730	{
2731	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[i];
2732
2733	pHlp->pfnSSMPutGCPhys64( pSSM, pVirtq->GCPhysVirtqDesc);
2734	pHlp->pfnSSMPutGCPhys64( pSSM, pVirtq->GCPhysVirtqAvail);
2735	pHlp->pfnSSMPutGCPhys64( pSSM, pVirtq->GCPhysVirtqUsed);
2736	pHlp->pfnSSMPutU16( pSSM, pVirtq->uNotifyOffset);
2737	pHlp->pfnSSMPutU16( pSSM, pVirtq->uMsixVector);
2738	pHlp->pfnSSMPutU16( pSSM, pVirtq->uEnable);
2739	pHlp->pfnSSMPutU16( pSSM, pVirtq->uQueueSize);
2740	pHlp->pfnSSMPutU16( pSSM, pVirtq->uAvailIdxShadow);
2741	pHlp->pfnSSMPutU16( pSSM, pVirtq->uUsedIdxShadow);
2742	int rc = pHlp->pfnSSMPutMem(pSSM, pVirtq->szName, 32);
2743	AssertRCReturn(rc, rc);
2744	}
2745	return VINF_SUCCESS;
2746	}
2747
2748
2749	/*********************************************************************************************************************************
2750	* Device Level *
2751	*********************************************************************************************************************************/
2752
2753	/**
2754	* This must be called by the client to handle VM state changes after the client takes care of its device-specific
2755	* tasks for the state change (i.e. reset, suspend, power-off, resume)
2756	*
2757	* @param pDevIns The device instance.
2758	* @param pVirtio Pointer to the shared virtio state.
2759	*/
2760	DECLHIDDEN(void) virtioCoreR3VmStateChanged(PVIRTIOCORE pVirtio, VIRTIOVMSTATECHANGED enmState)
2761	{
2762	LogFunc(("State changing to %s\n",
2763	virtioCoreGetStateChangeText(enmState)));
2764
2765	switch(enmState)
2766	{
2767	case kvirtIoVmStateChangedReset:
2768	virtioCoreResetAll(pVirtio);
2769	break;
2770	case kvirtIoVmStateChangedSuspend:
2771	break;
2772	case kvirtIoVmStateChangedPowerOff:
2773	break;
2774	case kvirtIoVmStateChangedResume:
2775	for (int uVirtq = 0; uVirtq < VIRTQ_MAX_COUNT; uVirtq++)
2776	{
2777	if ((!pVirtio->fLegacyDriver && pVirtio->aVirtqueues[uVirtq].uEnable)
2778	\| pVirtio->aVirtqueues[uVirtq].GCPhysVirtqDesc)
2779	virtioCoreNotifyGuestDriver(pVirtio->pDevInsR3, pVirtio, uVirtq);
2780	}
2781	break;
2782	default:
2783	LogRelFunc(("Bad enum value"));
2784	return;
2785	}
2786	}
2787
2788	/**
2789	* This should be called from PDMDEVREGR3::pfnDestruct.
2790	*
2791	* @param pDevIns The device instance.
2792	* @param pVirtio Pointer to the shared virtio state.
2793	* @param pVirtioCC Pointer to the ring-3 virtio state.
2794	*/
2795	DECLHIDDEN(void) virtioCoreR3Term(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
2796	{
2797	if (pVirtioCC->pbPrevDevSpecificCfg)
2798	{
2799	RTMemFree(pVirtioCC->pbPrevDevSpecificCfg);
2800	pVirtioCC->pbPrevDevSpecificCfg = NULL;
2801	}
2802
2803	RT_NOREF(pDevIns, pVirtio);
2804	}
2805
2806
2807	/**
2808	* Setup the Virtio device as a PCI device.
2809	*
2810	* @returns VBox status code.
2811	* @param pDevIns Device instance.
2812	* @param pVirtio Pointer to the shared virtio state. This
2813	* must be the first member in the shared
2814	* device instance data!
2815	* @param pVirtioCC Pointer to the ring-3 virtio state. This
2816	* must be the first member in the ring-3
2817	* device instance data!
2818	* @param pPciParams Values to populate industry standard PCI Configuration Space data structure
2819	* @param pcszInstance Device instance name (format-specifier)
2820	* @param cbDevSpecificCfg Size of virtio_pci_device_cap device-specific struct
2821	*/
2822	static int virtioR3PciTransportInit(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC, PVIRTIOPCIPARAMS pPciParams,
2823	const char *pcszInstance, uint16_t cbDevSpecificCfg)
2824	{
2825	/* Set PCI config registers (assume 32-bit mode) */
2826	PPDMPCIDEV pPciDev = pDevIns->apPciDevs[0];
2827	PDMPCIDEV_ASSERT_VALID(pDevIns, pPciDev);
2828
2829	PDMPciDevSetVendorId(pPciDev, DEVICE_PCI_VENDOR_ID_VIRTIO);
2830	PDMPciDevSetDeviceId(pPciDev, pPciParams->uDeviceId);
2831
2832	if (pPciParams->uDeviceId < DEVICE_PCI_DEVICE_ID_VIRTIO_BASE)
2833	/* Transitional devices MUST have a PCI Revision ID of 0. */
2834	PDMPciDevSetRevisionId(pPciDev, DEVICE_PCI_REVISION_ID_VIRTIO_TRANS);
2835	else
2836	/* Non-transitional devices SHOULD have a PCI Revision ID of 1 or higher. */
2837	PDMPciDevSetRevisionId(pPciDev, DEVICE_PCI_REVISION_ID_VIRTIO_V1);
2838
2839	PDMPciDevSetSubSystemId(pPciDev, pPciParams->uSubsystemId);
2840	PDMPciDevSetSubSystemVendorId(pPciDev, DEVICE_PCI_VENDOR_ID_VIRTIO);
2841	PDMPciDevSetClassBase(pPciDev, pPciParams->uClassBase);
2842	PDMPciDevSetClassSub(pPciDev, pPciParams->uClassSub);
2843	PDMPciDevSetClassProg(pPciDev, pPciParams->uClassProg);
2844	PDMPciDevSetInterruptLine(pPciDev, pPciParams->uInterruptLine);
2845	PDMPciDevSetInterruptPin(pPciDev, pPciParams->uInterruptPin);
2846
2847	/* Register PCI device */
2848	int rc = PDMDevHlpPCIRegister(pDevIns, pPciDev);
2849	if (RT_FAILURE(rc))
2850	return PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register PCI Device")); /* can we put params in this error? */
2851
2852	rc = PDMDevHlpPCIInterceptConfigAccesses(pDevIns, pPciDev, virtioR3PciConfigRead, virtioR3PciConfigWrite);
2853	AssertRCReturn(rc, rc);
2854
2855	/* Construct & map PCI vendor-specific capabilities for virtio host negotiation with guest driver */
2856
2857	#define CFG_ADDR_2_IDX(addr) ((uint8_t)(((uintptr_t)(addr) - (uintptr_t)&pPciDev->abConfig[0])))
2858	#define SET_PCI_CAP_LOC(a_pPciDev, a_pCfg, a_LocCap, a_uMmioLengthAlign) \
2859	do { \
2860	(a_LocCap).offMmio = (a_pCfg)->uOffset; \
2861	(a_LocCap).cbMmio = RT_ALIGN_T((a_pCfg)->uLength, a_uMmioLengthAlign, uint16_t); \
2862	(a_LocCap).offPci = (uint16_t)(uintptr_t)((uint8_t *)(a_pCfg) - &(a_pPciDev)->abConfig[0]); \
2863	(a_LocCap).cbPci = (a_pCfg)->uCapLen; \
2864	} while (0)
2865
2866	PVIRTIO_PCI_CAP_T pCfg;
2867	uint32_t cbRegion = 0;
2868
2869	/*
2870	* Common capability (VirtIO 1.0, section 4.1.4.3)
2871	*/
2872	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[0x40];
2873	pCfg->uCfgType = VIRTIO_PCI_CAP_COMMON_CFG;
2874	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2875	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
2876	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
2877	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2878	pCfg->uOffset = RT_ALIGN_32(0, 4); /* Currently 0, but reminder to 32-bit align if changing this */
2879	pCfg->uLength = sizeof(VIRTIO_PCI_COMMON_CFG_T);
2880	cbRegion += pCfg->uLength;
2881	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocCommonCfgCap, 2);
2882	pVirtioCC->pCommonCfgCap = pCfg;
2883
2884	/*
2885	* Notify capability (VirtIO 1.0, section 4.1.4.4).
2886	*
2887	* The size of the spec-defined subregion described by this VirtIO capability is
2888	* based-on the choice of this implementation to make the notification area of each
2889	* queue equal to queue's ordinal position (e.g. queue selector value). The VirtIO
2890	* specification leaves it up to implementation to define queue notification area layout.
2891	*/
2892	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2893	pCfg->uCfgType = VIRTIO_PCI_CAP_NOTIFY_CFG;
2894	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2895	pCfg->uCapLen = sizeof(VIRTIO_PCI_NOTIFY_CAP_T);
2896	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
2897	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2898	pCfg->uOffset = pVirtioCC->pCommonCfgCap->uOffset + pVirtioCC->pCommonCfgCap->uLength;
2899	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
2900	pCfg->uLength = VIRTQ_MAX_COUNT * VIRTIO_NOTIFY_OFFSET_MULTIPLIER + 2; /* will change in VirtIO 1.1 */
2901	cbRegion += pCfg->uLength;
2902	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocNotifyCap, 1);
2903	pVirtioCC->pNotifyCap = (PVIRTIO_PCI_NOTIFY_CAP_T)pCfg;
2904	pVirtioCC->pNotifyCap->uNotifyOffMultiplier = VIRTIO_NOTIFY_OFFSET_MULTIPLIER;
2905
2906	/* ISR capability (VirtIO 1.0, section 4.1.4.5)
2907	*
2908	* VirtIO 1.0 spec says 8-bit, unaligned in MMIO space. The specification example/diagram
2909	* illustrates this capability as 32-bit field with upper bits 'reserved'. Those depictions
2910	* differ. The spec's wording, not the diagram, is seen to work in practice.
2911	*/
2912	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2913	pCfg->uCfgType = VIRTIO_PCI_CAP_ISR_CFG;
2914	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2915	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
2916	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
2917	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2918	pCfg->uOffset = pVirtioCC->pNotifyCap->pciCap.uOffset + pVirtioCC->pNotifyCap->pciCap.uLength;
2919	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
2920	pCfg->uLength = sizeof(uint8_t);
2921	cbRegion += pCfg->uLength;
2922	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocIsrCap, 4);
2923	pVirtioCC->pIsrCap = pCfg;
2924
2925	/* PCI Cfg capability (VirtIO 1.0, section 4.1.4.7)
2926	*
2927	* This capability facilitates early-boot access to this device (BIOS).
2928	* This region isn't page-MMIO mapped. PCI configuration accesses are intercepted,
2929	* wherein uBar, uOffset and uLength are modulated by consumers to locate and read/write
2930	* values in any part of any region. (NOTE: Linux driver doesn't utilize this feature.
2931	* This capability only appears in lspci output on Linux if uLength is non-zero, 4-byte aligned,
2932	* during initialization of linux virtio driver).
2933	*/
2934	pVirtio->uPciCfgDataOff = pCfg->uCapNext + RT_OFFSETOF(VIRTIO_PCI_CFG_CAP_T, uPciCfgData);
2935	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2936	pCfg->uCfgType = VIRTIO_PCI_CAP_PCI_CFG;
2937	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2938	pCfg->uCapLen = sizeof(VIRTIO_PCI_CFG_CAP_T);
2939	pCfg->uCapNext = (pVirtio->fMsiSupport \|\| pVirtioCC->pbDevSpecificCfg) ? CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen : 0;
2940	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2941	pCfg->uOffset = 0;
2942	pCfg->uLength = 4;
2943	cbRegion += pCfg->uLength;
2944	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocPciCfgCap, 1);
2945	pVirtioCC->pPciCfgCap = (PVIRTIO_PCI_CFG_CAP_T)pCfg;
2946
2947	if (pVirtioCC->pbDevSpecificCfg)
2948	{
2949	/* Device-specific config capability (VirtIO 1.0, section 4.1.4.6).
2950	*
2951	* Client defines the device-specific config struct and passes size to virtioCoreR3Init()
2952	* to inform this.
2953	*/
2954	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2955	pCfg->uCfgType = VIRTIO_PCI_CAP_DEVICE_CFG;
2956	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2957	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
2958	pCfg->uCapNext = pVirtio->fMsiSupport ? CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen : 0;
2959	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2960	pCfg->uOffset = pVirtioCC->pIsrCap->uOffset + pVirtioCC->pIsrCap->uLength;
2961	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
2962	pCfg->uLength = cbDevSpecificCfg;
2963	cbRegion += pCfg->uLength;
2964	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocDeviceCap, 4);
2965	pVirtioCC->pDeviceCap = pCfg;
2966	}
2967	else
2968	Assert(pVirtio->LocDeviceCap.cbMmio == 0 && pVirtio->LocDeviceCap.cbPci == 0);
2969
2970	if (pVirtio->fMsiSupport)
2971	{
2972	PDMMSIREG aMsiReg;
2973	RT_ZERO(aMsiReg);
2974	aMsiReg.iMsixCapOffset = pCfg->uCapNext;
2975	aMsiReg.iMsixNextOffset = 0;
2976	aMsiReg.iMsixBar = VIRTIO_REGION_MSIX_CAP;
2977	aMsiReg.cMsixVectors = VBOX_MSIX_MAX_ENTRIES;
2978	rc = PDMDevHlpPCIRegisterMsi(pDevIns, &aMsiReg); /* see MsixR3init() */
2979	if (RT_FAILURE(rc))
2980	{
2981	/* See PDMDevHlp.cpp:pdmR3DevHlp_PCIRegisterMsi */
2982	LogFunc(("Failed to configure MSI-X (%Rrc). Reverting to INTx\n", rc));
2983	pVirtio->fMsiSupport = false;
2984	}
2985	else
2986	Log2Func(("Using MSI-X for guest driver notification\n"));
2987	}
2988	else
2989	LogFunc(("MSI-X not available for VBox, using INTx notification\n"));
2990
2991	/* Set offset to first capability and enable PCI dev capabilities */
2992	PDMPciDevSetCapabilityList(pPciDev, 0x40);
2993	PDMPciDevSetStatus(pPciDev, VBOX_PCI_STATUS_CAP_LIST);
2994
2995	size_t cbSize = RTStrPrintf(pVirtioCC->szMmioName, sizeof(pVirtioCC->szMmioName), "%s (modern)", pcszInstance);
2996	if (cbSize <= 0)
2997	return PDMDEV_SET_ERROR(pDevIns, VERR_BUFFER_OVERFLOW, N_("virtio: out of memory allocating string")); /* can we put params in this error? */
2998
2999	cbSize = RTStrPrintf(pVirtioCC->szPortIoName, sizeof(pVirtioCC->szPortIoName), "%s (legacy)", pcszInstance);
3000	if (cbSize <= 0)
3001	return PDMDEV_SET_ERROR(pDevIns, VERR_BUFFER_OVERFLOW, N_("virtio: out of memory allocating string")); /* can we put params in this error? */
3002
3003	if (pVirtio->fOfferLegacy)
3004	{
3005	/* As a transitional device that supports legacy VirtIO drivers, this VirtIO device generic implementation presents
3006	* legacy driver interface in I/O space at BAR0. The following maps the common (e.g. device independent)
3007	* dev config area as well as device-specific dev config area (whose size is passed to init function of this VirtIO
3008	* generic device code) for access via Port I/O, since legacy drivers (e.g. pre VirtIO 1.0) don't use MMIO callbacks.
3009	* (See VirtIO 1.1, Section 4.1.4.8).
3010	*/
3011	rc = PDMDevHlpPCIIORegionCreateIo(pDevIns, VIRTIO_REGION_LEGACY_IO, sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T) + cbDevSpecificCfg,
3012	virtioLegacyIOPortOut, virtioLegacyIOPortIn, NULL /pvUser/, pVirtioCC->szPortIoName,
3013	NULL /paExtDescs/, &pVirtio->hLegacyIoPorts);
3014	AssertLogRelRCReturn(rc, PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register legacy config in I/O space at BAR0 */")));
3015	}
3016
3017	/* Note: The Linux driver at drivers/virtio/virtio_pci_modern.c tries to map at least a page for the
3018	* 'unknown' device-specific capability without querying the capability to determine size, so pad w/extra page.
3019	*/
3020	rc = PDMDevHlpPCIIORegionCreateMmio(pDevIns, VIRTIO_REGION_PCI_CAP, RT_ALIGN_32(cbRegion + VIRTIO_PAGE_SIZE, VIRTIO_PAGE_SIZE),
3021	PCI_ADDRESS_SPACE_MEM, virtioMmioWrite, virtioMmioRead, pVirtio,
3022	IOMMMIO_FLAGS_READ_PASSTHRU \| IOMMMIO_FLAGS_WRITE_PASSTHRU,
3023	pVirtioCC->szMmioName,
3024	&pVirtio->hMmioPciCap);
3025	AssertLogRelRCReturn(rc, PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register PCI Capabilities address space")));
3026	return VINF_SUCCESS;
3027	}
3028
3029
3030	/**
3031	* Initializes the VirtIO device using the VirtIO over MMIO transport mode.
3032	*
3033	* @returns VBox status code.
3034	* @param pDevIns Device instance.
3035	* @param pVirtio Pointer to the shared virtio state. This
3036	* must be the first member in the shared
3037	* device instance data!
3038	* @param pVirtioCC Pointer to the ring-3 virtio state. This
3039	* must be the first member in the ring-3
3040	* device instance data!
3041	* @param pcszInstance Device instance name (format-specifier)
3042	* @param cbDevSpecificCfg Size of virtio_pci_device_cap device-specific struct
3043	* @param GCPhysMmioBase The physical guest address of the start of the MMIO area.
3044	* @param u16Irq The interrupt number to use for the virtio device.
3045	*/
3046	static int virtioR3MmioTransportInit(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC, const char *pcszInstance,
3047	uint16_t cbDevSpecificCfg, RTGCPHYS GCPhysMmioBase, uint16_t u16Irq)
3048	{
3049	pVirtio->uIrqMmio = u16Irq;
3050
3051	size_t cbSize = RTStrPrintf(pVirtioCC->szMmioName, sizeof(pVirtioCC->szMmioName), "%s (modern)", pcszInstance);
3052	if (cbSize <= 0)
3053	return PDMDEV_SET_ERROR(pDevIns, VERR_BUFFER_OVERFLOW, N_("virtio: out of memory allocating string")); /* can we put params in this error? */
3054
3055	/*
3056	* Register and map the MMIO region.
3057	*/
3058	int rc = PDMDevHlpMmioCreateAndMap(pDevIns, GCPhysMmioBase, RT_ALIGN_32(cbDevSpecificCfg + VIRTIO_MMIO_SIZE, 512),
3059	virtioMmioTransportWrite, virtioMmioTransportRead,
3060	IOMMMIO_FLAGS_READ_PASSTHRU \| IOMMMIO_FLAGS_WRITE_PASSTHRU,
3061	pVirtioCC->szMmioName, &pVirtio->hMmioPciCap);
3062	AssertLogRelRCReturn(rc, PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register PCI Capabilities address space")));
3063	return VINF_SUCCESS;
3064	}
3065
3066
3067	/** API Function: See header file */
3068	DECLHIDDEN(int) virtioCoreR3Init(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC, PVIRTIOPCIPARAMS pPciParams,
3069	const char *pcszInstance, uint64_t fDevSpecificFeatures, uint32_t fOfferLegacy,
3070	void *pvDevSpecificCfg, uint16_t cbDevSpecificCfg)
3071	{
3072	/*
3073	* Virtio state must be the first member of shared device instance data,
3074	* otherwise can't get our bearings in PCI config callbacks.
3075	*/
3076	AssertLogRelReturn(pVirtio == PDMINS_2_DATA(pDevIns, PVIRTIOCORE), VERR_STATE_CHANGED);
3077	AssertLogRelReturn(pVirtioCC == PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC), VERR_STATE_CHANGED);
3078
3079	pVirtio->pDevInsR3 = pDevIns;
3080
3081	/*
3082	* Caller must initialize these.
3083	*/
3084	AssertReturn(pVirtioCC->pfnStatusChanged, VERR_INVALID_POINTER);
3085	AssertReturn(pVirtioCC->pfnVirtqNotified, VERR_INVALID_POINTER);
3086	AssertReturn(VIRTQ_SIZE > 0 && VIRTQ_SIZE <= 32768, VERR_OUT_OF_RANGE); /* VirtIO specification-defined limit */
3087
3088	PCPDMDEVHLPR3 pHlp = pDevIns->pHlpR3;
3089
3090	uint16_t u16Irq = 0;
3091	int rc = pHlp->pfnCFGMQueryU16Def(pDevIns->pCfg, "Irq", &u16Irq, 0);
3092	if (RT_FAILURE(rc))
3093	return PDMDEV_SET_ERROR(pDevIns, rc, N_("Configuration error: Failed to get the \"Irq\" value"));
3094
3095	RTGCPHYS GCPhysMmioBase = 0;
3096	rc = pHlp->pfnCFGMQueryU64Def(pDevIns->pCfg, "MmioBase", &GCPhysMmioBase, NIL_RTGCPHYS);
3097	if (RT_FAILURE(rc))
3098	return PDMDEV_SET_ERROR(pDevIns, rc,
3099	N_("Configuration error: Failed to get the \"MmioBase\" value"));
3100
3101	#if 0 /* Until pdmR3DvHlp_PCISetIrq() impl is fixed and Assert that limits vec to 0 is removed
3102	* VBox legacy MSI support has not been implemented yet
3103	*/
3104	# ifdef VBOX_WITH_MSI_DEVICES
3105	pVirtio->fMsiSupport = true;
3106	# endif
3107	#endif
3108
3109	/*
3110	* Host features (presented as a buffet for guest to select from)
3111	* include both dev-specific features & reserved dev-independent features (bitmask).
3112	*/
3113	pVirtio->uDeviceType = pPciParams->uDeviceType;
3114	pVirtio->uDeviceFeatures = VIRTIO_F_VERSION_1
3115	\| VIRTIO_DEV_INDEPENDENT_FEATURES_OFFERED
3116	\| fDevSpecificFeatures;
3117
3118	pVirtio->fLegacyDriver = pVirtio->fOfferLegacy = fOfferLegacy;
3119
3120	RTStrCopy(pVirtio->szInstance, sizeof(pVirtio->szInstance), pcszInstance);
3121	pVirtioCC->cbDevSpecificCfg = cbDevSpecificCfg;
3122	pVirtioCC->pbDevSpecificCfg = (uint8_t *)pvDevSpecificCfg;
3123	pVirtioCC->pbPrevDevSpecificCfg = (uint8_t *)RTMemDup(pvDevSpecificCfg, cbDevSpecificCfg);
3124	AssertLogRelReturn(pVirtioCC->pbPrevDevSpecificCfg, VERR_NO_MEMORY);
3125
3126	if (GCPhysMmioBase != NIL_RTGCPHYS)
3127	rc = virtioR3MmioTransportInit(pDevIns, pVirtio, pVirtioCC, pcszInstance, cbDevSpecificCfg,
3128	GCPhysMmioBase, u16Irq);
3129	else
3130	rc = virtioR3PciTransportInit(pDevIns, pVirtio, pVirtioCC, pPciParams, pcszInstance, cbDevSpecificCfg);
3131	AssertLogRelRCReturn(rc, rc);
3132
3133	/*
3134	* Statistics.
3135	*/
3136	# ifdef VBOX_WITH_STATISTICS
3137	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsAllocated, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
3138	"Total number of allocated descriptor chains", "DescChainsAllocated");
3139	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsFreed, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
3140	"Total number of freed descriptor chains", "DescChainsFreed");
3141	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsSegsIn, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
3142	"Total number of inbound segments", "DescChainsSegsIn");
3143	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsSegsOut, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
3144	"Total number of outbound segments", "DescChainsSegsOut");
3145	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatReadR3, STAMTYPE_PROFILE, "IO/ReadR3", STAMUNIT_TICKS_PER_CALL, "Profiling IO reads in R3");
3146	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatReadR0, STAMTYPE_PROFILE, "IO/ReadR0", STAMUNIT_TICKS_PER_CALL, "Profiling IO reads in R0");
3147	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatReadRC, STAMTYPE_PROFILE, "IO/ReadRC", STAMUNIT_TICKS_PER_CALL, "Profiling IO reads in RC");
3148	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatWriteR3, STAMTYPE_PROFILE, "IO/WriteR3", STAMUNIT_TICKS_PER_CALL, "Profiling IO writes in R3");
3149	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatWriteR0, STAMTYPE_PROFILE, "IO/WriteR0", STAMUNIT_TICKS_PER_CALL, "Profiling IO writes in R0");
3150	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatWriteRC, STAMTYPE_PROFILE, "IO/WriteRC", STAMUNIT_TICKS_PER_CALL, "Profiling IO writes in RC");
3151	# endif /* VBOX_WITH_STATISTICS */
3152
3153	return VINF_SUCCESS;
3154	}
3155
3156	#else /* !IN_RING3 */
3157
3158	/**
3159	* Sets up the core ring-0/raw-mode virtio bits.
3160	*
3161	* @returns VBox status code.
3162	* @param pDevIns The device instance.
3163	* @param pVirtio Pointer to the shared virtio state. This must be the first
3164	* member in the shared device instance data!
3165	*/
3166	DECLHIDDEN(int) virtioCoreRZInit(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio)
3167	{
3168	AssertLogRelReturn(pVirtio == PDMINS_2_DATA(pDevIns, PVIRTIOCORE), VERR_STATE_CHANGED);
3169	int rc;
3170	#ifdef FUTURE_OPTIMIZATION
3171	rc = PDMDevHlpSetDeviceCritSect(pDevIns, PDMDevHlpCritSectGetNop(pDevIns));
3172	AssertRCReturn(rc, rc);
3173	#endif
3174
3175	if (pVirtio->uIrqMmio != 0)
3176	{
3177	rc = PDMDevHlpMmioSetUpContext(pDevIns, pVirtio->hMmioPciCap, virtioMmioTransportWrite, virtioMmioTransportRead, pVirtio);
3178	AssertRCReturn(rc, rc);
3179	}
3180	else
3181	{
3182	rc = PDMDevHlpMmioSetUpContext(pDevIns, pVirtio->hMmioPciCap, virtioMmioWrite, virtioMmioRead, pVirtio);
3183	AssertRCReturn(rc, rc);
3184
3185	if (pVirtio->fOfferLegacy)
3186	{
3187	rc = PDMDevHlpIoPortSetUpContext(pDevIns, pVirtio->hLegacyIoPorts, virtioLegacyIOPortOut, virtioLegacyIOPortIn, NULL /pvUser/);
3188	AssertRCReturn(rc, rc);
3189	}
3190	}
3191	return rc;
3192	}
3193
3194	#endif /* !IN_RING3 */
3195

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Devices/VirtIO/VirtioCore.cpp@ 105668

Download in other formats: