VirtioCore.cpp@ 91703

Last change on this file since 91703 was 91703, checked in by vboxsync, 3 years ago
DevVirtioNet_1_0.cpp: Convert VirtIO to be 'transitional' device, that handles both legacy (0.9) guests and modern (1.0+) guests. Various other small improvements and reduction/improved formatting of logging. See BugRef:8561, Comment #137
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 99.2 KB

Line
1	/* $Id: VirtioCore.cpp 91703 2021-10-13 02:24:30Z vboxsync $ */
2
3	/** @file
4	* VirtioCore - Virtio Core (PCI, feature & config mgt, queue mgt & proxy, notification mgt)
5	*/
6
7	/*
8	* Copyright (C) 2009-2021 Oracle Corporation
9	*
10	* This file is part of VirtualBox Open Source Edition (OSE), as
11	* available from http://www.virtualbox.org. This file is free software;
12	* you can redistribute it and/or modify it under the terms of the GNU
13	* General Public License (GPL) as published by the Free Software
14	* Foundation, in version 2 as it comes in the "COPYING" file of the
15	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
16	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
17	*/
18
19
20	/*********************************************************************************************************************************
21	* Header Files *
22	*********************************************************************************************************************************/
23	#define LOG_GROUP LOG_GROUP_DEV_VIRTIO
24
25	#include <iprt/assert.h>
26	#include <iprt/uuid.h>
27	#include <iprt/mem.h>
28	#include <iprt/sg.h>
29	#include <iprt/assert.h>
30	#include <iprt/string.h>
31	#include <iprt/param.h>
32	#include <iprt/types.h>
33	#include <VBox/log.h>
34	#include <VBox/msi.h>
35	#include <iprt/types.h>
36	#include <VBox/AssertGuest.h>
37	#include <VBox/vmm/pdmdev.h>
38	#include "VirtioCore.h"
39
40
41	/*********************************************************************************************************************************
42	* Defined Constants And Macros *
43	*********************************************************************************************************************************/
44	#define INSTANCE(a_pVirtio) ((a_pVirtio)->szInstance)
45	#define VIRTQNAME(a_pVirtio, a_uVirtq) ((a_pVirtio)->aVirtqueues[(a_uVirtq)].szName)
46
47
48	#define IS_VIRTQ_EMPTY(pDevIns, pVirtio, pVirtq) \
49	(virtioCoreVirtqAvailCnt(pDevIns, pVirtio, pVirtq) == 0)
50
51
52	#define IS_DRIVER_OK(a_pVirtio) ((a_pVirtio)->fDeviceStatus & VIRTIO_STATUS_DRIVER_OK)
53	#define WAS_DRIVER_OK(a_pVirtio) ((a_pVirtio)->fPrevDeviceStatus & VIRTIO_STATUS_DRIVER_OK)
54
55	/**
56	* This macro returns true if the @a a_offAccess and access length (@a
57	* a_cbAccess) are within the range of the mapped capability struct described by
58	* @a a_LocCapData.
59	*
60	* @param[in] a_offAccess Input: The offset into the MMIO bar of the access.
61	* @param[in] a_cbAccess Input: The access size.
62	* @param[out] a_offsetIntoCap Output: uint32_t variable to return the intra-capability offset into.
63	* @param[in] a_LocCapData Input: The capability location info.
64	*/
65	#define MATCHES_VIRTIO_CAP_STRUCT(a_offAccess, a_cbAccess, a_offsetIntoCap, a_LocCapData) \
66	( ((a_offsetIntoCap) = (uint32_t)((a_offAccess) - (a_LocCapData).offMmio)) < (uint32_t)(a_LocCapData).cbMmio \
67	&& (a_offsetIntoCap) + (uint32_t)(a_cbAccess) <= (uint32_t)(a_LocCapData).cbMmio )
68
69
70	/** Marks the start of the virtio saved state (just for sanity). */
71	#define VIRTIO_SAVEDSTATE_MARKER UINT64_C(0x1133557799bbddff)
72	/** The current saved state version for the virtio core. */
73	#define VIRTIO_SAVEDSTATE_VERSION UINT32_C(1)
74
75
76	/*********************************************************************************************************************************
77	* Structures and Typedefs *
78	*********************************************************************************************************************************/
79
80
81	/** @name virtq related flags
82	* @{ */
83	#define VIRTQ_DESC_F_NEXT 1 /*< Indicates this descriptor chains to next /
84	#define VIRTQ_DESC_F_WRITE 2 /*< Marks buffer as write-only (default ro) /
85	#define VIRTQ_DESC_F_INDIRECT 4 /*< Buffer is list of buffer descriptors /
86
87	#define VIRTQ_USED_F_NO_NOTIFY 1 /*< Dev to Drv: Don't notify when buf added /
88	#define VIRTQ_AVAIL_F_NO_INTERRUPT 1 /*< Drv to Dev: Don't notify when buf eaten /
89	/** @} */
90
91	/**
92	* virtq related structs
93	* (struct names follow VirtIO 1.0 spec, typedef use VBox style)
94	*/
95	typedef struct virtq_desc
96	{
97	uint64_t GCPhysBuf; /*< addr GC Phys. address of buffer /
98	uint32_t cb; /*< len Buffer length /
99	uint16_t fFlags; /*< flags Buffer specific flags /
100	uint16_t uDescIdxNext; /*< next Idx set if VIRTIO_DESC_F_NEXT /
101	} VIRTQ_DESC_T, *PVIRTQ_DESC_T;
102
103	typedef struct virtq_avail
104	{
105	uint16_t fFlags; /*< flags avail ring guest-to-host flags /
106	uint16_t uIdx; /*< idx Index of next free ring slot /
107	RT_FLEXIBLE_ARRAY_EXTENSION
108	uint16_t auRing[RT_FLEXIBLE_ARRAY]; /*< ring Ring: avail drv to dev bufs /
109	//uint16_t uUsedEventIdx; /*< used_event (if VIRTQ_USED_F_EVENT_IDX) /
110	} VIRTQ_AVAIL_T, *PVIRTQ_AVAIL_T;
111
112	typedef struct virtq_used_elem
113	{
114	uint32_t uDescIdx; /*< idx Start of used desc chain /
115	uint32_t cbElem; /*< len Total len of used desc chain /
116	} VIRTQ_USED_ELEM_T;
117
118	typedef struct virt_used
119	{
120	uint16_t fFlags; /*< flags used ring host-to-guest flags /
121	uint16_t uIdx; /*< idx Index of next ring slot /
122	RT_FLEXIBLE_ARRAY_EXTENSION
123	VIRTQ_USED_ELEM_T aRing[RT_FLEXIBLE_ARRAY]; /*< ring Ring: used dev to drv bufs /
124	//uint16_t uAvailEventIdx; /*< avail_event if (VIRTQ_USED_F_EVENT_IDX) /
125	} VIRTQ_USED_T, *PVIRTQ_USED_T;
126
127
128	const char *virtioCoreGetStateChangeText(VIRTIOVMSTATECHANGED enmState)
129	{
130	switch (enmState)
131	{
132	case kvirtIoVmStateChangedReset: return "VM RESET";
133	case kvirtIoVmStateChangedSuspend: return "VM SUSPEND";
134	case kvirtIoVmStateChangedPowerOff: return "VM POWER OFF";
135	case kvirtIoVmStateChangedResume: return "VM RESUME";
136	default: return "<BAD ENUM>";
137	}
138	}
139
140	/* Internal Functions */
141
142	static void virtioCoreNotifyGuestDriver(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq);
143	static int virtioKick(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint8_t uCause, uint16_t uVec);
144
145	/** @name Internal queue operations
146	* @{ */
147
148	/**
149	* Accessor for virtq descriptor
150	*/
151	#ifdef IN_RING3
152	DECLINLINE(void) virtioReadDesc(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq,
153	uint32_t idxDesc, PVIRTQ_DESC_T pDesc)
154	{
155	AssertMsg(IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
156	uint16_t const cVirtqItems = RT_MAX(pVirtq->uQueueSize, 1); /* Make sure to avoid div-by-zero. */
157
158	virtioCoreGCPhysRead(pVirtio, pDevIns,
159	pVirtq->GCPhysVirtqDesc + sizeof(VIRTQ_DESC_T) * (idxDesc % cVirtqItems),
160	pDesc, sizeof(VIRTQ_DESC_T));
161	}
162	#endif
163
164	/**
165	* Accessors for virtq avail ring
166	*/
167	#ifdef IN_RING3
168	DECLINLINE(uint16_t) virtioReadAvailDescIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq, uint32_t availIdx)
169	{
170	uint16_t uDescIdx;
171
172	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
173	uint16_t const cVirtqItems = RT_MAX(pVirtq->uQueueSize, 1); /* Make sure to avoid div-by-zero. */
174	virtioCoreGCPhysRead(pVirtio, pDevIns,
175	pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[availIdx % cVirtqItems]),
176	&uDescIdx, sizeof(uDescIdx));
177	return uDescIdx;
178	}
179
180	DECLINLINE(uint16_t) virtioReadAvailUsedEvent(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
181	{
182	uint16_t uUsedEventIdx;
183	/* VirtIO 1.0 uUsedEventIdx (used_event) immediately follows ring */
184	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
185	virtioCoreGCPhysRead(pVirtio, pDevIns,
186	pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[pVirtq->uQueueSize]),
187	&uUsedEventIdx, sizeof(uUsedEventIdx));
188	return uUsedEventIdx;
189	}
190	#endif
191
192	DECLINLINE(uint16_t) virtioReadAvailRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
193	{
194	uint16_t uIdx = 0;
195	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
196	virtioCoreGCPhysRead(pVirtio, pDevIns,
197	pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF(VIRTQ_AVAIL_T, uIdx),
198	&uIdx, sizeof(uIdx));
199	return uIdx;
200	}
201
202	DECLINLINE(uint16_t) virtioReadAvailRingFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
203	{
204	uint16_t fFlags = 0;
205	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
206	virtioCoreGCPhysRead(pVirtio, pDevIns,
207	pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF(VIRTQ_AVAIL_T, fFlags),
208	&fFlags, sizeof(fFlags));
209
210	return fFlags;
211	}
212
213	/** @} */
214
215	/** @name Accessors for virtq used ring
216	* @{
217	*/
218
219	#ifdef IN_RING3
220	DECLINLINE(void) virtioWriteUsedElem(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq,
221	uint32_t usedIdx, uint32_t uDescIdx, uint32_t uLen)
222	{
223	VIRTQ_USED_ELEM_T elem = { uDescIdx, uLen };
224	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
225	uint16_t const cVirtqItems = RT_MAX(pVirtq->uQueueSize, 1); /* Make sure to avoid div-by-zero. */
226	virtioCoreGCPhysWrite(pVirtio, pDevIns,
227	pVirtq->GCPhysVirtqUsed
228	+ RT_UOFFSETOF_DYN(VIRTQ_USED_T, aRing[usedIdx % cVirtqItems]),
229	&elem, sizeof(elem));
230	}
231
232	DECLINLINE(void) virtioWriteUsedRingFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq, uint16_t fFlags)
233	{
234	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
235	RT_UNTRUSTED_VALIDATED_FENCE(); /* VirtIO 1.0, Section 3.2.1.4.1 */
236	virtioCoreGCPhysWrite(pVirtio, pDevIns,
237	pVirtq->GCPhysVirtqUsed + RT_UOFFSETOF(VIRTQ_USED_T, fFlags),
238	&fFlags, sizeof(fFlags));
239	}
240	#endif
241
242	DECLINLINE(void) virtioWriteUsedRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq, uint16_t uIdx)
243	{
244	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
245	RT_UNTRUSTED_VALIDATED_FENCE(); /* VirtIO 1.0, Section 3.2.1.4.1 */
246	virtioCoreGCPhysWrite(pVirtio, pDevIns,
247	pVirtq->GCPhysVirtqUsed + RT_UOFFSETOF(VIRTQ_USED_T, uIdx),
248	&uIdx, sizeof(uIdx));
249	}
250
251
252	#ifdef IN_RING3
253	DECLINLINE(uint16_t) virtioReadUsedRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
254	{
255	uint16_t uIdx = 0;
256	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
257	virtioCoreGCPhysRead(pVirtio, pDevIns,
258	pVirtq->GCPhysVirtqUsed + RT_UOFFSETOF(VIRTQ_USED_T, uIdx),
259	&uIdx, sizeof(uIdx));
260	return uIdx;
261	}
262
263	DECLINLINE(uint16_t) virtioReadUsedRingFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
264	{
265	uint16_t fFlags = 0;
266	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
267	virtioCoreGCPhysRead(pVirtio, pDevIns,
268	pVirtq->GCPhysVirtqUsed + RT_UOFFSETOF(VIRTQ_USED_T, fFlags),
269	&fFlags, sizeof(fFlags));
270	return fFlags;
271	}
272
273	DECLINLINE(void) virtioWriteUsedAvailEvent(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq, uint32_t uAvailEventIdx)
274	{
275	/** VirtIO 1.0 uAvailEventIdx (avail_event) immediately follows ring */
276	AssertMsg(pVirtio->fLegacyDriver \|\| IS_DRIVER_OK(pVirtio), ("Called with guest driver not ready\n"));
277	virtioCoreGCPhysWrite(pVirtio, pDevIns,
278	pVirtq->GCPhysVirtqUsed
279	+ RT_UOFFSETOF_DYN(VIRTQ_USED_T, aRing[pVirtq->uQueueSize]),
280	&uAvailEventIdx, sizeof(uAvailEventIdx));
281	}
282	#endif
283
284	DECLINLINE(uint16_t) virtioCoreVirtqAvailCnt(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTQUEUE pVirtq)
285	{
286	uint16_t uIdxActual = virtioReadAvailRingIdx(pDevIns, pVirtio, pVirtq);
287	uint16_t uIdxShadow = pVirtq->uAvailIdxShadow;
288	uint16_t uIdxDelta;
289
290	if (uIdxActual < uIdxShadow)
291	uIdxDelta = (uIdxActual + VIRTQ_SIZE) - uIdxShadow;
292	else
293	uIdxDelta = uIdxActual - uIdxShadow;
294
295	LogFunc(("%s, %u %s\n",
296	pVirtq->szName, uIdxDelta, uIdxDelta == 1 ? "entry" : "entries"));
297
298	return uIdxDelta;
299	}
300	/**
301	* Get count of new (e.g. pending) elements in available ring.
302	*
303	* @param pDevIns The device instance.
304	* @param pVirtio Pointer to the shared virtio state.
305	* @param uVirtq Virtq number
306	*
307	* @returns how many entries have been added to ring as a delta of the consumer's
308	* avail index and the queue's guest-side current avail index.
309	*/
310	uint16_t virtioCoreVirtqAvailBufCount(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq)
311	{
312	AssertMsgReturn(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues), ("uVirtq out of range"), 0);
313	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
314
315	if (!IS_DRIVER_OK(pVirtio))
316	{
317	LogRelFunc(("Driver not ready\n"));
318	return 0;
319	}
320	if (!pVirtio->fLegacyDriver && !pVirtq->uEnable)
321	{
322	LogRelFunc(("virtq: %d (%s) not enabled\n", uVirtq, VIRTQNAME(pVirtio, uVirtq)));
323	return 0;
324	}
325
326	return virtioCoreVirtqAvailCnt(pDevIns, pVirtio, pVirtq);
327	}
328
329	#ifdef IN_RING3
330
331	/** API Function: See header file*/
332	void virtioCorePrintFeatures(VIRTIOCORE *pVirtio, PCDBGFINFOHLP pHlp)
333	{
334	static struct
335	{
336	uint64_t fFeatureBit;
337	const char *pcszDesc;
338	} const s_aFeatures[] =
339	{
340	{ VIRTIO_F_RING_INDIRECT_DESC, " RING_INDIRECT_DESC Driver can use descriptors with VIRTQ_DESC_F_INDIRECT flag set\n" },
341	{ VIRTIO_F_RING_EVENT_IDX, " RING_EVENT_IDX Enables use_event and avail_event fields described in 2.4.7, 2.4.8\n" },
342	{ VIRTIO_F_VERSION_1, " VERSION Used to detect legacy drivers.\n" },
343	};
344
345	#define MAXLINE 80
346	/* Display as a single buf to prevent interceding log messages */
347	uint16_t cbBuf = RT_ELEMENTS(s_aFeatures) * 132;
348	char pszBuf = (char )RTMemAllocZ(cbBuf);
349	Assert(pszBuf);
350	char *cp = pszBuf;
351	for (unsigned i = 0; i < RT_ELEMENTS(s_aFeatures); ++i)
352	{
353	bool isOffered = RT_BOOL(pVirtio->uDeviceFeatures & s_aFeatures[i].fFeatureBit);
354	bool isNegotiated = RT_BOOL(pVirtio->uDriverFeatures & s_aFeatures[i].fFeatureBit);
355	cp += RTStrPrintf(cp, cbBuf - (cp - pszBuf), " %s %s %s",
356	isOffered ? "+" : "-", isNegotiated ? "x" : " ", s_aFeatures[i].pcszDesc);
357	}
358	if (pHlp)
359	pHlp->pfnPrintf(pHlp, "VirtIO Core Features Configuration\n\n"
360	" Offered Accepted Feature Description\n"
361	" ------- -------- ------- -----------\n"
362	"%s\n", pszBuf);
363	#ifdef LOG_ENABLED
364	else
365	Log3(("VirtIO Core Features Configuration\n\n"
366	" Offered Accepted Feature Description\n"
367	" ------- -------- ------- -----------\n"
368	"%s\n", pszBuf));
369	#endif
370	RTMemFree(pszBuf);
371	}
372	#endif
373
374	#ifdef LOG_ENABLED
375
376	/** API Function: See header file */
377	void virtioCoreHexDump(uint8_t pv, uint32_t cb, uint32_t uBase, const char pszTitle)
378	{
379	#define ADJCURSOR(cb) pszOut += cb; cbRemain -= cb;
380	size_t cbPrint = 0, cbRemain = ((cb / 16) + 1) * 80;
381	char pszBuf = (char )RTMemAllocZ(cbRemain), *pszOut = pszBuf;
382	AssertMsgReturnVoid(pszBuf, ("Out of Memory"));
383	if (pszTitle)
384	{
385	cbPrint = RTStrPrintf(pszOut, cbRemain, "%s [%d bytes]:\n", pszTitle, cb);
386	ADJCURSOR(cbPrint);
387	}
388	for (uint32_t row = 0; row < RT_MAX(1, (cb / 16) + 1) && row * 16 < cb; row++)
389	{
390	cbPrint = RTStrPrintf(pszOut, cbRemain, "%04x: ", row * 16 + uBase); /* line address */
391	ADJCURSOR(cbPrint);
392	for (uint8_t col = 0; col < 16; col++)
393	{
394	uint32_t idx = row * 16 + col;
395	if (idx >= cb)
396	cbPrint = RTStrPrintf(pszOut, cbRemain, "-- %s", (col + 1) % 8 ? "" : " ");
397	else
398	cbPrint = RTStrPrintf(pszOut, cbRemain, "%02x %s", pv[idx], (col + 1) % 8 ? "" : " ");
399	ADJCURSOR(cbPrint);
400	}
401	for (uint32_t idx = row * 16; idx < row * 16 + 16; idx++)
402	{
403	cbPrint = RTStrPrintf(pszOut, cbRemain, "%c", (idx >= cb) ? ' ' : (pv[idx] >= 0x20 && pv[idx] <= 0x7e ? pv[idx] : '.'));
404	ADJCURSOR(cbPrint);
405	}
406	*pszOut++ = '\n';
407	--cbRemain;
408	}
409	Log(("%s\n", pszBuf));
410	RTMemFree(pszBuf);
411	RT_NOREF2(uBase, pv);
412	#undef ADJCURSOR
413	}
414
415	/* API FUnction: See header file */
416	void virtioCoreGCPhysHexDump(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, uint16_t cb, uint32_t uBase, const char *pszTitle)
417	{
418	PVIRTIOCORE pVirtio = PDMDEVINS_2_DATA(pDevIns, PVIRTIOCORE);
419	#define ADJCURSOR(cb) pszOut += cb; cbRemain -= cb;
420	size_t cbPrint = 0, cbRemain = ((cb / 16) + 1) * 80;
421	char pszBuf = (char )RTMemAllocZ(cbRemain), *pszOut = pszBuf;
422	AssertMsgReturnVoid(pszBuf, ("Out of Memory"));
423	if (pszTitle)
424	{
425	cbPrint = RTStrPrintf(pszOut, cbRemain, "%s [%d bytes]:\n", pszTitle, cb);
426	ADJCURSOR(cbPrint);
427	}
428	for (uint16_t row = 0; row < (uint16_t)RT_MAX(1, (cb / 16) + 1) && row * 16 < cb; row++)
429	{
430	uint8_t c;
431	cbPrint = RTStrPrintf(pszOut, cbRemain, "%04x: ", row * 16 + uBase); /* line address */
432	ADJCURSOR(cbPrint);
433	for (uint8_t col = 0; col < 16; col++)
434	{
435	uint32_t idx = row * 16 + col;
436	virtioCoreGCPhysRead(pVirtio, pDevIns, GCPhys + idx, &c, 1);
437	if (idx >= cb)
438	cbPrint = RTStrPrintf(pszOut, cbRemain, "-- %s", (col + 1) % 8 ? "" : " ");
439	else
440	cbPrint = RTStrPrintf(pszOut, cbRemain, "%02x %s", c, (col + 1) % 8 ? "" : " ");
441	ADJCURSOR(cbPrint);
442	}
443	for (uint16_t idx = row * 16; idx < row * 16 + 16; idx++)
444	{
445	virtioCoreGCPhysRead(pVirtio, pDevIns, GCPhys + idx, &c, 1);
446	cbPrint = RTStrPrintf(pszOut, cbRemain, "%c", (idx >= cb) ? ' ' : (c >= 0x20 && c <= 0x7e ? c : '.'));
447	ADJCURSOR(cbPrint);
448	}
449	*pszOut++ = '\n';
450	--cbRemain;
451	}
452	Log(("%s\n", pszBuf));
453	RTMemFree(pszBuf);
454	RT_NOREF(uBase);
455	#undef ADJCURSOR
456	}
457	#endif /* LOG_ENABLED */
458
459	/** API function: See header file */
460	int virtioCoreIsLegacyMode(PVIRTIOCORE pVirtio)
461	{
462	Log12Func(("%s", pVirtio->fLegacyDriver ? "Legacy Guest Driver handling mode\n" : ""));
463	return pVirtio->fLegacyDriver;
464	}
465
466	/** API function: See header file */
467	void virtioCoreLogMappedIoValue(const char pszFunc, const char pszMember, uint32_t uMemberSize,
468	const void *pv, uint32_t cb, uint32_t uOffset, int fWrite,
469	int fHasIndex, uint32_t idx)
470	{
471	if (!LogIs6Enabled())
472	return;
473
474	char szIdx[16];
475	if (fHasIndex)
476	RTStrPrintf(szIdx, sizeof(szIdx), "[%d]", idx);
477	else
478	szIdx[0] = '\0';
479
480	if (cb == 1 \|\| cb == 2 \|\| cb == 4 \|\| cb == 8)
481	{
482	char szDepiction[64];
483	size_t cchDepiction;
484	if (uOffset != 0 \|\| cb != uMemberSize) /* display bounds if partial member access */
485	cchDepiction = RTStrPrintf(szDepiction, sizeof(szDepiction), "%s%s[%d:%d]",
486	pszMember, szIdx, uOffset, uOffset + cb - 1);
487	else
488	cchDepiction = RTStrPrintf(szDepiction, sizeof(szDepiction), "%s%s", pszMember, szIdx);
489
490	/* padding */
491	if (cchDepiction < 30)
492	szDepiction[cchDepiction++] = ' ';
493	while (cchDepiction < 30)
494	szDepiction[cchDepiction++] = '.';
495	szDepiction[cchDepiction] = '\0';
496
497	RTUINT64U uValue;
498	uValue.u = 0;
499	memcpy(uValue.au8, pv, cb);
500	Log6(("%-23s: Guest %s %s %#0*RX64\n",
501	pszFunc, fWrite ? "wrote" : "read ", szDepiction, 2 + cb * 2, uValue.u));
502	}
503	else /* odd number or oversized access, ... log inline hex-dump style */
504	{
505	Log6(("%-23s: Guest %s %s%s[%d:%d]: %.*Rhxs\n",
506	pszFunc, fWrite ? "wrote" : "read ", pszMember,
507	szIdx, uOffset, uOffset + cb, cb, pv));
508	}
509	RT_NOREF2(fWrite, pszFunc);
510	}
511
512	/**
513	* Makes the MMIO-mapped Virtio fDeviceStatus registers non-cryptic (buffers to
514	* keep the output clean during multi-threaded activity)
515	*/
516	DECLINLINE(void) virtioCoreFormatDeviceStatus(uint8_t bStatus, char *pszBuf, size_t uSize)
517	{
518
519	#define ADJCURSOR(len) cp += len; uSize -= len; sep = (char *)" \| ";
520
521	memset(pszBuf, 0, uSize);
522	size_t len;
523	char *cp = pszBuf;
524	char sep = (char )"";
525
526	if (bStatus == 0) {
527	RTStrPrintf(cp, uSize, "RESET");
528	return;
529	}
530	if (bStatus & VIRTIO_STATUS_ACKNOWLEDGE)
531	{
532	len = RTStrPrintf(cp, uSize, "ACKNOWLEDGE");
533	ADJCURSOR(len);
534	}
535	if (bStatus & VIRTIO_STATUS_DRIVER)
536	{
537	len = RTStrPrintf(cp, uSize, "%sDRIVER", sep);
538	ADJCURSOR(len);
539	}
540	if (bStatus & VIRTIO_STATUS_FEATURES_OK)
541	{
542	len = RTStrPrintf(cp, uSize, "%sFEATURES_OK", sep);
543	ADJCURSOR(len);
544	}
545	if (bStatus & VIRTIO_STATUS_DRIVER_OK)
546	{
547	len = RTStrPrintf(cp, uSize, "%sDRIVER_OK", sep);
548	ADJCURSOR(len);
549	}
550	if (bStatus & VIRTIO_STATUS_FAILED)
551	{
552	len = RTStrPrintf(cp, uSize, "%sFAILED", sep);
553	ADJCURSOR(len);
554	}
555	if (bStatus & VIRTIO_STATUS_DEVICE_NEEDS_RESET)
556	RTStrPrintf(cp, uSize, "%sNEEDS_RESET", sep);
557
558	#undef ADJCURSOR
559	}
560
561	#ifdef IN_RING3
562
563	int virtioCoreR3VirtqAttach(PVIRTIOCORE pVirtio, uint16_t uVirtq, const char *pcszName)
564	{
565	LogFunc(("Attaching %s to VirtIO core\n", pcszName));
566	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
567	pVirtq->uVirtq = uVirtq;
568	pVirtq->uAvailIdxShadow = 0;
569	pVirtq->uUsedIdxShadow = 0;
570	pVirtq->fUsedRingEvent = false;
571	RTStrCopy(pVirtq->szName, sizeof(pVirtq->szName), pcszName);
572	return VINF_SUCCESS;
573	}
574
575	/** API Fuunction: See header file */
576	void virtioCoreR3VirtqInfo(PPDMDEVINS pDevIns, PCDBGFINFOHLP pHlp, const char *pszArgs, int uVirtq)
577	{
578	RT_NOREF(pszArgs);
579	PVIRTIOCORE pVirtio = PDMDEVINS_2_DATA(pDevIns, PVIRTIOCORE);
580	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
581
582	/** @todo add ability to dump physical contents described by any descriptor (using existing VirtIO core API function) */
583	// bool fDump = pszArgs && (pszArgs == 'd' \|\| pszArgs == 'D'); /* "dump" (avail phys descriptor)"
584
585	uint16_t uAvailIdx = virtioReadAvailRingIdx(pDevIns, pVirtio, pVirtq);
586	uint16_t uAvailIdxShadow = pVirtq->uAvailIdxShadow;
587
588	uint16_t uUsedIdx = virtioReadUsedRingIdx(pDevIns, pVirtio, pVirtq);
589	uint16_t uUsedIdxShadow = pVirtq->uUsedIdxShadow;
590
591	PVIRTQBUF pVirtqBuf = NULL;
592
593	bool fEmpty = IS_VIRTQ_EMPTY(pDevIns, pVirtio, pVirtq);
594
595	LogFunc(("%s, empty = %s\n", pVirtq->szName, fEmpty ? "true" : "false"));
596
597	int cSendSegs = 0, cReturnSegs = 0;
598	if (!fEmpty)
599	{
600	virtioCoreR3VirtqAvailBufPeek(pDevIns, pVirtio, uVirtq, &pVirtqBuf);
601	cSendSegs = pVirtqBuf->pSgPhysSend ? pVirtqBuf->pSgPhysSend->cSegs : 0;
602	cReturnSegs = pVirtqBuf->pSgPhysReturn ? pVirtqBuf->pSgPhysReturn->cSegs : 0;
603	}
604
605	bool fAvailNoInterrupt = virtioReadAvailRingFlags(pDevIns, pVirtio, pVirtq) & VIRTQ_AVAIL_F_NO_INTERRUPT;
606	bool fUsedNoNotify = virtioReadUsedRingFlags(pDevIns, pVirtio, pVirtq) & VIRTQ_USED_F_NO_NOTIFY;
607
608	pHlp->pfnPrintf(pHlp, " queue enabled: ........... %s\n", pVirtq->uEnable ? "true" : "false");
609	pHlp->pfnPrintf(pHlp, " size: .................... %d\n", pVirtq->uQueueSize);
610	pHlp->pfnPrintf(pHlp, " notify offset: ........... %d\n", pVirtq->uNotifyOffset);
611	if (pVirtio->fMsiSupport)
612	pHlp->pfnPrintf(pHlp, " MSIX vector: ....... %4.4x\n", pVirtq->uMsixVector);
613	pHlp->pfnPrintf(pHlp, "\n");
614	pHlp->pfnPrintf(pHlp, " avail ring (%d entries):\n", uAvailIdx - uAvailIdxShadow);
615	pHlp->pfnPrintf(pHlp, " index: ................ %d\n", uAvailIdx);
616	pHlp->pfnPrintf(pHlp, " shadow: ............... %d\n", uAvailIdxShadow);
617	pHlp->pfnPrintf(pHlp, " flags: ................ %s\n", fAvailNoInterrupt ? "NO_INTERRUPT" : "");
618	pHlp->pfnPrintf(pHlp, "\n");
619	pHlp->pfnPrintf(pHlp, " used ring (%d entries):\n", uUsedIdx - uUsedIdxShadow);
620	pHlp->pfnPrintf(pHlp, " index: ................ %d\n", uUsedIdx);
621	pHlp->pfnPrintf(pHlp, " shadow: ............... %d\n", uUsedIdxShadow);
622	pHlp->pfnPrintf(pHlp, " flags: ................ %s\n", fUsedNoNotify ? "NO_NOTIFY" : "");
623	pHlp->pfnPrintf(pHlp, "\n");
624	if (!fEmpty)
625	{
626	pHlp->pfnPrintf(pHlp, " desc chain:\n");
627	pHlp->pfnPrintf(pHlp, " head idx: ............. %d\n", uUsedIdx);
628	pHlp->pfnPrintf(pHlp, " segs: ................. %d\n", cSendSegs + cReturnSegs);
629	pHlp->pfnPrintf(pHlp, " refCnt ................ %d\n", pVirtqBuf->cRefs);
630	pHlp->pfnPrintf(pHlp, "\n");
631	pHlp->pfnPrintf(pHlp, " host-to-guest (%d bytes):\n", pVirtqBuf->cbPhysSend);
632	pHlp->pfnPrintf(pHlp, " segs: .............. %d\n", cSendSegs);
633	if (cSendSegs)
634	{
635	pHlp->pfnPrintf(pHlp, " index: ............. %d\n", pVirtqBuf->pSgPhysSend->idxSeg);
636	pHlp->pfnPrintf(pHlp, " unsent ............. %d\n", pVirtqBuf->pSgPhysSend->cbSegLeft);
637	}
638	pHlp->pfnPrintf(pHlp, "\n");
639	pHlp->pfnPrintf(pHlp, " guest-to-host (%d bytes)\n", pVirtqBuf->cbPhysReturn);
640	pHlp->pfnPrintf(pHlp, " segs: .............. %d\n", cReturnSegs);
641	if (cReturnSegs)
642	{
643	pHlp->pfnPrintf(pHlp, " index: ............. %d\n", pVirtqBuf->pSgPhysReturn->idxSeg);
644	pHlp->pfnPrintf(pHlp, " unsent ............. %d\n", pVirtqBuf->pSgPhysReturn->cbSegLeft);
645	}
646	} else
647	pHlp->pfnPrintf(pHlp, " No desc chains available\n");
648	pHlp->pfnPrintf(pHlp, "\n");
649
650	}
651
652	/** API Function: See header file */
653	uint32_t virtioCoreR3VirtqBufRetain(PVIRTQBUF pVirtqBuf)
654	{
655	AssertReturn(pVirtqBuf, UINT32_MAX);
656	AssertReturn(pVirtqBuf->u32Magic == VIRTQBUF_MAGIC, UINT32_MAX);
657	uint32_t cRefs = ASMAtomicIncU32(&pVirtqBuf->cRefs);
658	Assert(cRefs > 1);
659	Assert(cRefs < 16);
660	return cRefs;
661	}
662
663
664	/** API Function: See header file */
665	uint32_t virtioCoreR3VirtqBufRelease(PVIRTIOCORE pVirtio, PVIRTQBUF pVirtqBuf)
666	{
667	if (!pVirtqBuf)
668	return 0;
669	AssertReturn(pVirtqBuf, 0);
670	AssertReturn(pVirtqBuf->u32Magic == VIRTQBUF_MAGIC, 0);
671	uint32_t cRefs = ASMAtomicDecU32(&pVirtqBuf->cRefs);
672	Assert(cRefs < 16);
673	if (cRefs == 0)
674	{
675	pVirtqBuf->u32Magic = ~VIRTQBUF_MAGIC;
676	RTMemFree(pVirtqBuf);
677	STAM_REL_COUNTER_INC(&pVirtio->StatDescChainsFreed);
678	}
679	return cRefs;
680	}
681
682	/** API Function: See header file */
683	void virtioCoreNotifyConfigChanged(PVIRTIOCORE pVirtio)
684	{
685	virtioKick(pVirtio->pDevInsR3, pVirtio, VIRTIO_ISR_DEVICE_CONFIG, pVirtio->uMsixConfig);
686	}
687
688	/** API Function: See header file */
689	void virtioCoreVirtqEnableNotify(PVIRTIOCORE pVirtio, uint16_t uVirtq, bool fEnable)
690	{
691
692	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
693	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
694
695	if (IS_DRIVER_OK(pVirtio))
696	{
697	uint16_t fFlags = virtioReadUsedRingFlags(pVirtio->pDevInsR3, pVirtio, pVirtq);
698
699	if (fEnable)
700	fFlags &= ~VIRTQ_USED_F_NO_NOTIFY;
701	else
702	fFlags \|= VIRTQ_USED_F_NO_NOTIFY;
703
704	virtioWriteUsedRingFlags(pVirtio->pDevInsR3, pVirtio, pVirtq, fFlags);
705	}
706	}
707
708	/** API function: See Header file */
709	void virtioCoreResetAll(PVIRTIOCORE pVirtio)
710	{
711	LogFunc(("\n"));
712	pVirtio->fDeviceStatus \|= VIRTIO_STATUS_DEVICE_NEEDS_RESET;
713	if (IS_DRIVER_OK(pVirtio))
714	{
715	if (!pVirtio->fLegacyDriver)
716	pVirtio->fGenUpdatePending = true;
717	virtioKick(pVirtio->pDevInsR3, pVirtio, VIRTIO_ISR_DEVICE_CONFIG, pVirtio->uMsixConfig);
718	}
719	}
720
721	/** API function: See Header file */
722	int virtioCoreR3VirtqAvailBufPeek(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq,
723	PPVIRTQBUF ppVirtqBuf)
724	{
725	return virtioCoreR3VirtqAvailBufGet(pDevIns, pVirtio, uVirtq, ppVirtqBuf, false);
726	}
727
728	/** API function: See Header file */
729	int virtioCoreR3VirtqAvailBufNext(PVIRTIOCORE pVirtio, uint16_t uVirtq)
730	{
731	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
732	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
733
734	if (!pVirtio->fLegacyDriver)
735	AssertMsgReturn((pVirtio->fDeviceStatus & VIRTIO_STATUS_DRIVER_OK) && pVirtq->uEnable,
736	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
737
738	if (IS_VIRTQ_EMPTY(pVirtio->pDevInsR3, pVirtio, pVirtq))
739	return VERR_NOT_AVAILABLE;
740
741	Log6Func(("%s avail shadow idx: %u\n", pVirtq->szName, pVirtq->uAvailIdxShadow));
742	pVirtq->uAvailIdxShadow++;
743
744	return VINF_SUCCESS;
745	}
746
747
748	/** API Function: See header file */
749	int virtioCoreR3VirtqAvailBufGet(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq,
750	uint16_t uHeadIdx, PPVIRTQBUF ppVirtqBuf)
751	{
752	AssertReturn(ppVirtqBuf, VERR_INVALID_POINTER);
753	*ppVirtqBuf = NULL;
754
755	AssertMsgReturn(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues),
756	("uVirtq out of range"), VERR_INVALID_PARAMETER);
757
758	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
759
760	if (!pVirtio->fLegacyDriver)
761	AssertMsgReturn((pVirtio->fDeviceStatus & VIRTIO_STATUS_DRIVER_OK) && pVirtq->uEnable,
762	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
763
764	uint16_t uDescIdx = uHeadIdx;
765
766	Log6Func(("%s DESC CHAIN: (head idx = %u)\n", pVirtio->aVirtqueues[uVirtq].szName, uHeadIdx));
767
768	/*
769	* Allocate and initialize the descriptor chain structure.
770	*/
771	PVIRTQBUF pVirtqBuf = (PVIRTQBUF)RTMemAllocZ(sizeof(VIRTQBUF_T));
772	AssertReturn(pVirtqBuf, VERR_NO_MEMORY);
773	pVirtqBuf->u32Magic = VIRTQBUF_MAGIC;
774	pVirtqBuf->cRefs = 1;
775	pVirtqBuf->uHeadIdx = uHeadIdx;
776	pVirtqBuf->uVirtq = uVirtq;
777	*ppVirtqBuf = pVirtqBuf;
778
779	/*
780	* Gather segments.
781	*/
782	VIRTQ_DESC_T desc;
783
784	uint32_t cbIn = 0;
785	uint32_t cbOut = 0;
786	uint32_t cSegsIn = 0;
787	uint32_t cSegsOut = 0;
788
789	PVIRTIOSGSEG paSegsIn = pVirtqBuf->aSegsIn;
790	PVIRTIOSGSEG paSegsOut = pVirtqBuf->aSegsOut;
791
792	do
793	{
794	PVIRTIOSGSEG pSeg;
795
796	/*
797	* Malicious guests may go beyond paSegsIn or paSegsOut boundaries by linking
798	* several descriptors into a loop. Since there is no legitimate way to get a sequences of
799	* linked descriptors exceeding the total number of descriptors in the ring (see @bugref{8620}),
800	* the following aborts I/O if breach and employs a simple log throttling algorithm to notify.
801	*/
802	if (cSegsIn + cSegsOut >= VIRTQ_SIZE)
803	{
804	static volatile uint32_t s_cMessages = 0;
805	static volatile uint32_t s_cThreshold = 1;
806	if (ASMAtomicIncU32(&s_cMessages) == ASMAtomicReadU32(&s_cThreshold))
807	{
808	LogRelMax(64, ("Too many linked descriptors; check if the guest arranges descriptors in a loop.\n"));
809	if (ASMAtomicReadU32(&s_cMessages) != 1)
810	LogRelMax(64, ("(the above error has occured %u times so far)\n", ASMAtomicReadU32(&s_cMessages)));
811	ASMAtomicWriteU32(&s_cThreshold, ASMAtomicReadU32(&s_cThreshold) * 10);
812	}
813	break;
814	}
815	RT_UNTRUSTED_VALIDATED_FENCE();
816
817	virtioReadDesc(pDevIns, pVirtio, pVirtq, uDescIdx, &desc);
818
819	if (desc.fFlags & VIRTQ_DESC_F_WRITE)
820	{
821	Log6Func(("%s IN idx=%u seg=%u addr=%RGp cb=%u\n", pVirtq->szName, uDescIdx, cSegsIn, desc.GCPhysBuf, desc.cb));
822	cbIn += desc.cb;
823	pSeg = &paSegsIn[cSegsIn++];
824	}
825	else
826	{
827	Log6Func(("%s OUT desc_idx=%u seg=%u addr=%RGp cb=%u\n", pVirtq->szName, uDescIdx, cSegsOut, desc.GCPhysBuf, desc.cb));
828	cbOut += desc.cb;
829	pSeg = &paSegsOut[cSegsOut++];
830	#ifdef DEEP_DEBUG
831	if (LogIs11Enabled())
832	{
833	virtioCoreGCPhysHexDump(pDevIns, desc.GCPhysBuf, desc.cb, 0, NULL);
834	Log(("\n"));
835	}
836	#endif
837	}
838
839	pSeg->GCPhys = desc.GCPhysBuf;
840	pSeg->cbSeg = desc.cb;
841
842	uDescIdx = desc.uDescIdxNext;
843	} while (desc.fFlags & VIRTQ_DESC_F_NEXT);
844
845	/*
846	* Add segments to the descriptor chain structure.
847	*/
848	if (cSegsIn)
849	{
850	virtioCoreGCPhysChainInit(&pVirtqBuf->SgBufIn, paSegsIn, cSegsIn);
851	pVirtqBuf->pSgPhysReturn = &pVirtqBuf->SgBufIn;
852	pVirtqBuf->cbPhysReturn = cbIn;
853	STAM_REL_COUNTER_ADD(&pVirtio->StatDescChainsSegsIn, cSegsIn);
854	}
855
856	if (cSegsOut)
857	{
858	virtioCoreGCPhysChainInit(&pVirtqBuf->SgBufOut, paSegsOut, cSegsOut);
859	pVirtqBuf->pSgPhysSend = &pVirtqBuf->SgBufOut;
860	pVirtqBuf->cbPhysSend = cbOut;
861	STAM_REL_COUNTER_ADD(&pVirtio->StatDescChainsSegsOut, cSegsOut);
862	}
863
864	STAM_REL_COUNTER_INC(&pVirtio->StatDescChainsAllocated);
865	Log6Func(("%s -- segs OUT: %u (%u bytes) IN: %u (%u bytes) --\n",
866	pVirtq->szName, cSegsOut, cbOut, cSegsIn, cbIn));
867
868	return VINF_SUCCESS;
869	}
870
871	/** API function: See Header file */
872	int virtioCoreR3VirtqAvailBufGet(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq,
873	PPVIRTQBUF ppVirtqBuf, bool fRemove)
874	{
875	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
876	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
877
878	if (IS_VIRTQ_EMPTY(pDevIns, pVirtio, pVirtq))
879	return VERR_NOT_AVAILABLE;
880
881	uint16_t uHeadIdx = virtioReadAvailDescIdx(pDevIns, pVirtio, pVirtq, pVirtq->uAvailIdxShadow);
882
883	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
884	virtioWriteUsedAvailEvent(pDevIns,pVirtio, pVirtq, pVirtq->uAvailIdxShadow + 1);
885
886	if (fRemove)
887	pVirtq->uAvailIdxShadow++;
888
889	int rc = virtioCoreR3VirtqAvailBufGet(pDevIns, pVirtio, uVirtq, uHeadIdx, ppVirtqBuf);
890	return rc;
891	}
892
893	/** API function: See Header file */
894	int virtioCoreR3VirtqUsedBufPut(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq, PRTSGBUF pSgVirtReturn,
895	PVIRTQBUF pVirtqBuf, bool fFence)
896	{
897	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
898	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
899
900	PVIRTIOSGBUF pSgPhysReturn = pVirtqBuf->pSgPhysReturn;
901
902	Assert(pVirtqBuf->u32Magic == VIRTQBUF_MAGIC);
903	Assert(pVirtqBuf->cRefs > 0);
904
905	AssertMsgReturn(IS_DRIVER_OK(pVirtio), ("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
906
907	Log6Func((" Copying device data to %s (%s guest), desc chain idx %d\n",
908	VIRTQNAME(pVirtio, uVirtq), pVirtio->fLegacyDriver ? "legacy" : "modern", virtioReadUsedRingIdx(pDevIns, pVirtio, pVirtq)));
909
910	/* Copy s/g buf (virtual memory) to guest phys mem (IN direction). */
911
912	size_t cbCopy = 0, cbTotal = 0, cbRemain = 0;
913
914	if (pSgVirtReturn)
915	{
916	size_t cbTarget = virtioCoreGCPhysChainCalcBufSize(pSgPhysReturn);
917	cbRemain = cbTotal = RTSgBufCalcTotalLength(pSgVirtReturn);
918	AssertMsgReturn(cbTarget >= cbRemain, ("No space to write data to phys memory"), VERR_BUFFER_OVERFLOW);
919	virtioCoreGCPhysChainReset(pSgPhysReturn);
920	while (cbRemain)
921	{
922	cbCopy = RT_MIN(pSgVirtReturn->cbSegLeft, pSgPhysReturn->cbSegLeft);
923	Assert(cbCopy > 0);
924	virtioCoreGCPhysWrite(pVirtio, pDevIns, (RTGCPHYS)pSgPhysReturn->GCPhysCur, pSgVirtReturn->pvSegCur, cbCopy);
925	RTSgBufAdvance(pSgVirtReturn, cbCopy);
926	virtioCoreGCPhysChainAdvance(pSgPhysReturn, cbCopy);
927	cbRemain -= cbCopy;
928	}
929
930	if (fFence)
931	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE(); /* needed? */
932
933	Assert(!(cbCopy >> 32));
934	}
935
936	/* If this write-ahead crosses threshold where the driver wants to get an event, flag it */
937	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
938	if (pVirtq->uUsedIdxShadow == virtioReadAvailUsedEvent(pDevIns, pVirtio, pVirtq))
939	pVirtq->fUsedRingEvent = true;
940
941	/*
942	* Place used buffer's descriptor in used ring but don't update used ring's slot index.
943	* That will be done with a subsequent client call to virtioCoreVirtqUsedRingSync() */
944	virtioWriteUsedElem(pDevIns, pVirtio, pVirtq, pVirtq->uUsedIdxShadow++, pVirtqBuf->uHeadIdx, (uint32_t)cbTotal);
945
946	if (pSgVirtReturn)
947	Log6Func((" ... %d segs, %zu bytes, copied to %u byte buf. residual: %zu bytes\n",
948	pSgVirtReturn->cSegs, cbTotal - cbRemain, pVirtqBuf->cbPhysReturn, pVirtqBuf->cbPhysReturn - cbTotal));
949
950	Log6Func((" %s used_idx=%u\n", VIRTQNAME(pVirtio, uVirtq), virtioReadUsedRingIdx(pDevIns, pVirtio, pVirtq)));
951
952	return VINF_SUCCESS;
953	}
954
955
956	#endif /* IN_RING3 */
957
958	/** API function: See Header file */
959	int virtioCoreVirtqUsedRingSync(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq)
960	{
961	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
962	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
963
964	if (!pVirtio->fLegacyDriver)
965	AssertMsgReturn((pVirtio->fDeviceStatus & VIRTIO_STATUS_DRIVER_OK) && pVirtq->uEnable,
966	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
967
968	Log6Func((" %s ++used_idx=%u\n", pVirtq->szName, pVirtq->uUsedIdxShadow));
969
970	virtioWriteUsedRingIdx(pDevIns, pVirtio, pVirtq, pVirtq->uUsedIdxShadow);
971	virtioCoreNotifyGuestDriver(pDevIns, pVirtio, uVirtq);
972
973	return VINF_SUCCESS;
974	}
975
976	/**
977	* This is called from the MMIO callback code when the guest does an MMIO access to the
978	* mapped queue notification capability area corresponding to a particular queue, to notify
979	* the queue handler of available data in the avail ring of the queue (VirtIO 1.0, 4.1.4.4.1)
980	*
981	* @param pDevIns The device instance.
982	* @param pVirtio Pointer to the shared virtio state.
983	* @param uVirtq Virtq to check for guest interrupt handling preference
984	* @param uNotifyIdx Notification index
985	*/
986	static void virtioCoreVirtqNotified(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq, uint16_t uNotifyIdx)
987	{
988	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
989
990	/* See VirtIO 1.0, section 4.1.5.2 It implies that uVirtq and uNotifyIdx should match.
991	* Disregarding this notification may cause throughput to stop, however there's no way to know
992	* which was queue was intended for wake-up if the two parameters disagree. */
993
994	AssertMsg(uNotifyIdx == uVirtq,
995	("Guest kicked virtq %d's notify addr w/non-corresponding virtq idx %d\n",
996	uVirtq, uNotifyIdx));
997	RT_NOREF(uNotifyIdx);
998
999	AssertReturnVoid(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1000	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1001
1002	Log6Func(("%s (desc chains: %u)\n", pVirtq->szName,
1003	virtioCoreVirtqAvailCnt(pDevIns, pVirtio, pVirtq)));
1004
1005	/* Inform client */
1006	pVirtioCC->pfnVirtqNotified(pDevIns, pVirtio, uVirtq);
1007	RT_NOREF2(pVirtio, pVirtq);
1008	}
1009
1010	/**
1011	* Trigger MSI-X or INT# interrupt to notify guest of data added to used ring of
1012	* the specified virtq, depending on the interrupt configuration of the device
1013	* and depending on negotiated and realtime constraints flagged by the guest driver.
1014	*
1015	* See VirtIO 1.0 specification (section 2.4.7).
1016	*
1017	* @param pDevIns The device instance.
1018	* @param pVirtio Pointer to the shared virtio state.
1019	* @param uVirtq Virtq to check for guest interrupt handling preference
1020	*/
1021	static void virtioCoreNotifyGuestDriver(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t uVirtq)
1022	{
1023	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1024	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1025
1026	if (!IS_DRIVER_OK(pVirtio))
1027	{
1028	LogFunc(("Guest driver not in ready state.\n"));
1029	return;
1030	}
1031
1032	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
1033	{
1034	if (pVirtq->fUsedRingEvent)
1035	{
1036	#ifdef IN_RING3
1037	Log6Func(("...kicking guest %s, VIRTIO_F_EVENT_IDX set and threshold (%d) reached\n",
1038	pVirtq->szName, (uint16_t)virtioReadAvailUsedEvent(pDevIns, pVirtio, pVirtq)));
1039	#endif
1040	virtioKick(pDevIns, pVirtio, VIRTIO_ISR_VIRTQ_INTERRUPT, pVirtq->uMsixVector);
1041	pVirtq->fUsedRingEvent = false;
1042	return;
1043	}
1044	#ifdef IN_RING3
1045	Log6Func(("...skip interrupt %s, VIRTIO_F_EVENT_IDX set but threshold (%d) not reached (%d)\n",
1046	pVirtq->szName,(uint16_t)virtioReadAvailUsedEvent(pDevIns, pVirtio, pVirtq), pVirtq->uUsedIdxShadow));
1047	#endif
1048	}
1049	else
1050	{
1051	/** If guest driver hasn't suppressed interrupts, interrupt */
1052	if (!(virtioReadAvailRingFlags(pDevIns, pVirtio, pVirtq) & VIRTQ_AVAIL_F_NO_INTERRUPT))
1053	{
1054	virtioKick(pDevIns, pVirtio, VIRTIO_ISR_VIRTQ_INTERRUPT, pVirtq->uMsixVector);
1055	return;
1056	}
1057	Log6Func(("...skipping interrupt for %s (guest set VIRTQ_AVAIL_F_NO_INTERRUPT)\n", pVirtq->szName));
1058	}
1059	}
1060
1061	/**
1062	* Raise interrupt or MSI-X
1063	*
1064	* @param pDevIns The device instance.
1065	* @param pVirtio Pointer to the shared virtio state.
1066	* @param uCause Interrupt cause bit mask to set in PCI ISR port.
1067	* @param uVec MSI-X vector, if enabled
1068	*/
1069	static int virtioKick(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint8_t uCause, uint16_t uMsixVector)
1070	{
1071	if (uCause == VIRTIO_ISR_VIRTQ_INTERRUPT)
1072	Log6Func(("reason: buffer added to 'used' ring.\n"));
1073	else
1074	if (uCause == VIRTIO_ISR_DEVICE_CONFIG)
1075	Log6Func(("reason: device config change\n"));
1076
1077	if (!pVirtio->fMsiSupport)
1078	{
1079	pVirtio->uISR \|= uCause;
1080	PDMDevHlpPCISetIrq(pDevIns, 0, PDM_IRQ_LEVEL_HIGH);
1081	}
1082	else if (uMsixVector != VIRTIO_MSI_NO_VECTOR)
1083	PDMDevHlpPCISetIrq(pDevIns, uMsixVector, 1);
1084	return VINF_SUCCESS;
1085	}
1086
1087	/**
1088	* Lower interrupt (Called when guest reads ISR and when resetting)
1089	*
1090	* @param pDevIns The device instance.
1091	*/
1092	static void virtioLowerInterrupt(PPDMDEVINS pDevIns, uint16_t uMsixVector)
1093	{
1094	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1095	if (!pVirtio->fMsiSupport)
1096	PDMDevHlpPCISetIrq(pDevIns, 0, PDM_IRQ_LEVEL_LOW);
1097	else if (uMsixVector != VIRTIO_MSI_NO_VECTOR)
1098	PDMDevHlpPCISetIrq(pDevIns, pVirtio->uMsixConfig, PDM_IRQ_LEVEL_LOW);
1099	}
1100
1101	#ifdef IN_RING3
1102	static void virtioResetVirtq(PVIRTIOCORE pVirtio, uint16_t uVirtq)
1103	{
1104	Assert(uVirtq < RT_ELEMENTS(pVirtio->aVirtqueues));
1105	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1106
1107	pVirtq->uQueueSize = VIRTQ_SIZE;
1108	pVirtq->uEnable = false;
1109	pVirtq->uNotifyOffset = uVirtq;
1110	pVirtq->fUsedRingEvent = false;
1111	pVirtq->uAvailIdxShadow = 0;
1112	pVirtq->uUsedIdxShadow = 0;
1113	pVirtq->uMsixVector = uVirtq + 2;
1114
1115	if (!pVirtio->fMsiSupport) /* VirtIO 1.0, 4.1.4.3 and 4.1.5.1.2 */
1116	pVirtq->uMsixVector = VIRTIO_MSI_NO_VECTOR;
1117
1118	virtioLowerInterrupt(pVirtio->pDevInsR3, pVirtq->uMsixVector);
1119	}
1120
1121	static void virtioResetDevice(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio)
1122	{
1123	LogFunc(("Resetting device VirtIO state\n"));
1124	pVirtio->fLegacyDriver = 1; /* Assume this. Cleared if VIRTIO_F_VERSION_1 feature ack'd */
1125	pVirtio->uDeviceFeaturesSelect = 0;
1126	pVirtio->uDriverFeaturesSelect = 0;
1127	pVirtio->uConfigGeneration = 0;
1128	pVirtio->fDeviceStatus = 0;
1129	pVirtio->uISR = 0;
1130
1131	if (!pVirtio->fMsiSupport)
1132	virtioLowerInterrupt(pDevIns, 0);
1133	else
1134	{
1135	virtioLowerInterrupt(pDevIns, pVirtio->uMsixConfig);
1136	for (int i = 0; i < VIRTQ_MAX_COUNT; i++)
1137	virtioLowerInterrupt(pDevIns, pVirtio->aVirtqueues[i].uMsixVector);
1138	}
1139
1140	if (!pVirtio->fMsiSupport) /* VirtIO 1.0, 4.1.4.3 and 4.1.5.1.2 */
1141	pVirtio->uMsixConfig = VIRTIO_MSI_NO_VECTOR;
1142
1143	for (uint16_t uVirtq = 0; uVirtq < VIRTQ_MAX_COUNT; uVirtq++)
1144	virtioResetVirtq(pVirtio, uVirtq);
1145	}
1146
1147	/**
1148	* Invoked by this implementation when guest driver resets the device.
1149	* The driver itself will not until the device has read the status change.
1150	*/
1151	static void virtioGuestR3WasReset(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
1152	{
1153	Log(("%-23s: Guest reset the device\n", __FUNCTION__));
1154
1155	/* Let the client know */
1156	pVirtioCC->pfnStatusChanged(pVirtio, pVirtioCC, 0 /* fDriverOk */);
1157	virtioResetDevice(pDevIns, pVirtio);
1158	}
1159	#endif /* IN_RING3 */
1160
1161	/**
1162	* Handle accesses to Common Configuration capability
1163	*
1164	* @returns VBox status code
1165	*
1166	* @param pDevIns The device instance.
1167	* @param pVirtio Pointer to the shared virtio state.
1168	* @param pVirtioCC Pointer to the current context virtio state.
1169	* @param fWrite Set if write access, clear if read access.
1170	* @param uOffsetOfAccess The common configuration capability offset.
1171	* @param cb Number of bytes to read or write
1172	* @param pv Pointer to location to write to or read from
1173	*/
1174	static int virtioCommonCfgAccessed(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC,
1175	int fWrite, uint32_t uOffsetOfAccess, unsigned cb, void *pv)
1176	{
1177	uint16_t uVirtq = pVirtio->uVirtqSelect;
1178	int rc = VINF_SUCCESS;
1179	uint64_t val;
1180	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDeviceFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1181	{
1182	if (fWrite) /* Guest WRITE pCommonCfg>uDeviceFeatures */
1183	{
1184	/* VirtIO 1.0, 4.1.4.3 states device_feature is a (guest) driver readonly field,
1185	* yet the linux driver attempts to write/read it back twice */
1186	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1187	LogFunc(("... WARNING: Guest attempted to write readonly virtio_pci_common_cfg.device_feature (ignoring)\n"));
1188	return VINF_IOM_MMIO_UNUSED_00;
1189	}
1190	else /* Guest READ pCommonCfg->uDeviceFeatures */
1191	{
1192	switch (pVirtio->uDeviceFeaturesSelect)
1193	{
1194	case 0:
1195	val = pVirtio->uDeviceFeatures & UINT32_C(0xffffffff);
1196	memcpy(pv, &val, cb);
1197	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1198	break;
1199	case 1:
1200	val = pVirtio->uDeviceFeatures >> 32;
1201	memcpy(pv, &val, cb);
1202	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess + sizeof(uint32_t));
1203	break;
1204	default:
1205	LogFunc(("Guest read uDeviceFeatures with out of range selector (%#x), returning 0\n",
1206	pVirtio->uDeviceFeaturesSelect));
1207	return VINF_IOM_MMIO_UNUSED_00;
1208	}
1209	}
1210	}
1211	else
1212	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1213	{
1214	if (fWrite) /* Guest WRITE pCommonCfg->udriverFeatures */
1215	{
1216	switch (pVirtio->uDriverFeaturesSelect)
1217	{
1218	case 0:
1219	memcpy(&pVirtio->uDriverFeatures, pv, cb);
1220	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1221	break;
1222	case 1:
1223	memcpy((char *)&pVirtio->uDriverFeatures + sizeof(uint32_t), pv, cb);
1224	if (pVirtio->uDriverFeatures & VIRTIO_F_VERSION_1)
1225	pVirtio->fLegacyDriver = 0;
1226	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess + sizeof(uint32_t));
1227	break;
1228	default:
1229	LogFunc(("Guest wrote uDriverFeatures with out of range selector (%#x), returning 0\n",
1230	pVirtio->uDriverFeaturesSelect));
1231	return VINF_SUCCESS;
1232	}
1233	}
1234	else /* Guest READ pCommonCfg->udriverFeatures */
1235	{
1236	switch (pVirtio->uDriverFeaturesSelect)
1237	{
1238	case 0:
1239	val = pVirtio->uDriverFeatures & 0xffffffff;
1240	memcpy(pv, &val, cb);
1241	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1242	break;
1243	case 1:
1244	val = (pVirtio->uDriverFeatures >> 32) & 0xffffffff;
1245	memcpy(pv, &val, cb);
1246	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess + 4);
1247	break;
1248	default:
1249	LogFunc(("Guest read uDriverFeatures with out of range selector (%#x), returning 0\n",
1250	pVirtio->uDriverFeaturesSelect));
1251	return VINF_IOM_MMIO_UNUSED_00;
1252	}
1253	}
1254	}
1255	else
1256	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uNumVirtqs, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1257	{
1258	if (fWrite)
1259	{
1260	Log2Func(("Guest attempted to write readonly virtio_pci_common_cfg.num_queues\n"));
1261	return VINF_SUCCESS;
1262	}
1263	(uint16_t )pv = VIRTQ_MAX_COUNT;
1264	VIRTIO_DEV_CONFIG_LOG_ACCESS(uNumVirtqs, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess);
1265	}
1266	else
1267	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fDeviceStatus, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1268	{
1269	if (fWrite) /* Guest WRITE pCommonCfg->fDeviceStatus */
1270	{
1271	pVirtio->fDeviceStatus = (uint8_t )pv;
1272	bool fDeviceReset = pVirtio->fDeviceStatus == 0;
1273
1274	if (LogIs7Enabled())
1275	{
1276	char szOut[80] = { 0 };
1277	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
1278	Log(("%-23s: Guest wrote fDeviceStatus ................ (%s)\n", __FUNCTION__, szOut));
1279	}
1280	bool const fStatusChanged = IS_DRIVER_OK(pVirtio) != WAS_DRIVER_OK(pVirtio);
1281
1282	if (fDeviceReset \|\| fStatusChanged)
1283	{
1284	#ifdef IN_RING0
1285	/* Since VirtIO status changes are cumbersome by nature, e.g. not a benchmark priority,
1286	* handle the rest in R3 to facilitate logging or whatever dev-specific client needs to do */
1287	Log6(("%-23s: RING0 => RING3 (demote)\n", __FUNCTION__));
1288	return VINF_IOM_R3_MMIO_WRITE;
1289	#endif
1290	}
1291
1292	#ifdef IN_RING3
1293	/*
1294	* Notify client only if status actually changed from last time and when we're reset.
1295	*/
1296	if (fDeviceReset)
1297	virtioGuestR3WasReset(pDevIns, pVirtio, pVirtioCC);
1298
1299	if (fStatusChanged)
1300	pVirtioCC->pfnStatusChanged(pVirtio, pVirtioCC, IS_DRIVER_OK(pVirtio));
1301	#endif
1302	/*
1303	* Save the current status for the next write so we can see what changed.
1304	*/
1305	pVirtio->fPrevDeviceStatus = pVirtio->fDeviceStatus;
1306	}
1307	else /* Guest READ pCommonCfg->fDeviceStatus */
1308	{
1309	(uint8_t )pv = pVirtio->fDeviceStatus;
1310
1311	if (LogIs7Enabled())
1312	{
1313	char szOut[80] = { 0 };
1314	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
1315	LogFunc(("Guest read fDeviceStatus ................ (%s)\n", szOut));
1316	}
1317	}
1318	}
1319	else
1320	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixConfig, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1321	VIRTIO_DEV_CONFIG_ACCESS( uMsixConfig, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1322	else
1323	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uDeviceFeaturesSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1324	VIRTIO_DEV_CONFIG_ACCESS( uDeviceFeaturesSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1325	else
1326	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uDriverFeaturesSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1327	VIRTIO_DEV_CONFIG_ACCESS( uDriverFeaturesSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1328	else
1329	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uConfigGeneration, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1330	VIRTIO_DEV_CONFIG_ACCESS( uConfigGeneration, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1331	else
1332	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uVirtqSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1333	VIRTIO_DEV_CONFIG_ACCESS( uVirtqSelect, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio);
1334	else
1335	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( GCPhysVirtqDesc, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1336	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( GCPhysVirtqDesc, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1337	else
1338	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( GCPhysVirtqAvail, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1339	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( GCPhysVirtqAvail, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1340	else
1341	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( GCPhysVirtqUsed, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1342	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( GCPhysVirtqUsed, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1343	else
1344	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uQueueSize, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1345	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uQueueSize, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1346	else
1347	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uEnable, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1348	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uEnable, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1349	else
1350	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uNotifyOffset, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1351	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uNotifyOffset, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1352	else
1353	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixVector, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess))
1354	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uMsixVector, uVirtq, VIRTIO_PCI_COMMON_CFG_T, uOffsetOfAccess, pVirtio->aVirtqueues);
1355	else
1356	{
1357	Log2Func(("Bad guest %s access to virtio_pci_common_cfg: uOffsetOfAccess=%#x (%d), cb=%d\n",
1358	fWrite ? "write" : "read ", uOffsetOfAccess, uOffsetOfAccess, cb));
1359	return fWrite ? VINF_SUCCESS : VINF_IOM_MMIO_UNUSED_00;
1360	}
1361
1362	#ifndef IN_RING3
1363	RT_NOREF(pDevIns, pVirtioCC);
1364	#endif
1365	return rc;
1366	}
1367
1368	/**
1369	* @callback_method_impl{FNIOMIOPORTNEWIN)
1370	*
1371	* This I/O handler exists only to handle access from legacy drivers.
1372	*/
1373
1374	static DECLCALLBACK(VBOXSTRICTRC) virtioLegacyIOPortIn(PPDMDEVINS pDevIns, void pvUser, RTIOPORT offPort, uint32_t pu32, unsigned cb)
1375	{
1376
1377	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1378	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatRead), a);
1379
1380	RT_NOREF(pvUser);
1381	// LogFunc((" Read from port offset=%RTiop cb=%#x\n", offPort, cb));
1382
1383	void pv = pu32; / To use existing macros */
1384	int fWrite = 0; /* To use existing macros */
1385
1386	uint16_t uVirtq = pVirtio->uVirtqSelect;
1387
1388	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDeviceFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1389	{
1390	uint32_t val = pVirtio->uDeviceFeatures & UINT32_C(0xffffffff);
1391	memcpy(pu32, &val, cb);
1392	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1393	}
1394	else
1395	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDriverFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1396	{
1397	uint32_t val = pVirtio->uDriverFeatures & 0xffffffff;
1398	memcpy(pu32, &val, cb);
1399	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1400	}
1401	else
1402	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fDeviceStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1403	{
1404	(uint8_t )pu32 = pVirtio->fDeviceStatus;
1405
1406	if (LogIs7Enabled())
1407	{
1408	char szOut[80] = { 0 };
1409	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
1410	Log(("%-23s: Guest read fDeviceStatus ................ (%s)\n", __FUNCTION__, szOut));
1411	}
1412	}
1413	else
1414	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fIsrStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1415	{
1416	ASSERT_GUEST_MSG(cb == 1, ("%d\n", cb));
1417	(uint8_t )pu32 = pVirtio->uISR;
1418	pVirtio->uISR = 0;
1419	virtioLowerInterrupt( pDevIns, 0);
1420	Log((" ISR read and cleared\n"));
1421	}
1422	else
1423	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uVirtqSelect, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1424	VIRTIO_DEV_CONFIG_ACCESS( uVirtqSelect, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1425	else
1426	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uVirtqPfn, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1427	{
1428	PVIRTQUEUE pVirtQueue = &pVirtio->aVirtqueues[uVirtq];
1429	*pu32 = pVirtQueue->GCPhysVirtqDesc >> PAGE_SHIFT;
1430	Log(("%-23s: Guest read uVirtqPfn .................... %#x\n", __FUNCTION__, *pu32));
1431	}
1432	else
1433	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uQueueSize, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1434	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uQueueSize, uVirtq, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio->aVirtqueues);
1435	else
1436	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uQueueNotify, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1437	VIRTIO_DEV_CONFIG_ACCESS( uQueueNotify, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1438	#if LEGACY_MSIX_SUPPORTED
1439	else
1440	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixConfig, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1441	VIRTIO_DEV_CONFIG_ACCESS( uMsixConfig, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1442	else
1443	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixVector, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1444	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uMsixVector, uVirtq, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio->aVirtqueues);
1445	#endif
1446	else if (offPort >= sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T))
1447	{
1448	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
1449	#if IN_RING3
1450	/* Access device-specific configuration */
1451	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1452	int rc = pVirtioCC->pfnDevCapRead(pDevIns, offPort - sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T), pv, cb);
1453	return rc;
1454	#else
1455	return VINF_IOM_R3_IOPORT_READ;
1456	#endif
1457	}
1458	else
1459	{
1460	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
1461	Log2Func(("Bad guest read access to virtio_legacy_pci_common_cfg: offset=%#x, cb=%x\n",
1462	offPort, cb));
1463	int rc = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
1464	"virtioLegacyIOPortIn: no valid port at offset offset=%RTiop cb=%#x\n", offPort, cb);
1465	return rc;
1466	}
1467
1468	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
1469	return VINF_SUCCESS;
1470	}
1471
1472
1473	/**
1474	* @callback_method_impl{ * @callback_method_impl{FNIOMIOPORTNEWOUT}
1475	*
1476	* This I/O Port interface exists only to handle access from legacy drivers.
1477	*/
1478	static DECLCALLBACK(VBOXSTRICTRC) virtioLegacyIOPortOut(PPDMDEVINS pDevIns, void *pvUser, RTIOPORT offPort, uint32_t u32, unsigned cb)
1479	{
1480	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1481	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatWrite), a);
1482	RT_NOREF(pvUser);
1483
1484	uint16_t uVirtq = pVirtio->uVirtqSelect;
1485	uint32_t u32OnStack = u32; /* allows us to use this impl's MMIO parsing macros */
1486	void pv = &u32OnStack; / To use existing macros */
1487	int fWrite = 1; /* To use existing macros */
1488
1489	// LogFunc(("Write to port offset=%RTiop, cb=%#x, u32=%#x\n", offPort, cb, u32));
1490
1491	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uVirtqSelect, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1492	VIRTIO_DEV_CONFIG_ACCESS( uVirtqSelect, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1493	else
1494	#if LEGACY_MSIX_SUPPORTED
1495	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixConfig, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1496	VIRTIO_DEV_CONFIG_ACCESS( uMsixConfig, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio);
1497	else
1498	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER( uMsixVector, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1499	VIRTIO_DEV_CONFIG_ACCESS_INDEXED( uMsixVector, uVirtq, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort, pVirtio->aVirtqueues);
1500	else
1501	#endif
1502	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDeviceFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1503	{
1504	/* Check to see if guest acknowledged unsupported features */
1505	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDeviceFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1506	LogFunc(("... WARNING: Guest attempted to write readonly virtio_pci_common_cfg.device_feature (ignoring)\n"));
1507	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1508	return VINF_SUCCESS;
1509	}
1510	else
1511	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uDriverFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1512	{
1513	memcpy(&pVirtio->uDriverFeatures, pv, cb);
1514	if ((pVirtio->uDriverFeatures & ~VIRTIO_DEV_INDEPENDENT_LEGACY_FEATURES_OFFERED) == 0)
1515	{
1516	Log(("Guest asked for features host does not support! (host=%x guest=%x)\n",
1517	VIRTIO_DEV_INDEPENDENT_LEGACY_FEATURES_OFFERED, pVirtio->uDriverFeatures));
1518	pVirtio->uDriverFeatures &= VIRTIO_DEV_INDEPENDENT_LEGACY_FEATURES_OFFERED;
1519	}
1520	VIRTIO_DEV_CONFIG_LOG_ACCESS(uDriverFeatures, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1521	}
1522	else
1523	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uQueueSize, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1524	{
1525	VIRTIO_DEV_CONFIG_LOG_ACCESS(uQueueSize, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1526	LogFunc(("... WARNING: Guest attempted to write readonly device_feature (queue size) (ignoring)\n"));
1527	return VINF_SUCCESS;
1528	}
1529	else
1530	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fDeviceStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1531	{
1532	bool const fDriverInitiatedReset = (pVirtio->fDeviceStatus = (uint8_t)u32) == 0;
1533	bool const fDriverStateImproved = IS_DRIVER_OK(pVirtio) && !WAS_DRIVER_OK(pVirtio);
1534
1535	if (LogIs7Enabled())
1536	{
1537	char szOut[80] = { 0 };
1538	virtioCoreFormatDeviceStatus(pVirtio->fDeviceStatus, szOut, sizeof(szOut));
1539	Log(("%-23s: Guest wrote fDeviceStatus ................ (%s)\n", __FUNCTION__, szOut));
1540	}
1541
1542	if (fDriverStateImproved \|\| fDriverInitiatedReset)
1543	{
1544	#ifdef IN_RING0
1545	Log6(("%-23s: RING0 => RING3 (demote)\n", __FUNCTION__));
1546	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1547	return VINF_IOM_R3_IOPORT_WRITE;
1548	#endif
1549	}
1550
1551	#ifdef IN_RING3
1552	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1553	if (fDriverInitiatedReset)
1554	virtioGuestR3WasReset(pDevIns, pVirtio, pVirtioCC);
1555
1556	else if (fDriverStateImproved)
1557	pVirtioCC->pfnStatusChanged(pVirtio, pVirtioCC, 1 /* fDriverOk */);
1558
1559	#endif
1560	pVirtio->fPrevDeviceStatus = pVirtio->fDeviceStatus;
1561	}
1562	else
1563	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uVirtqPfn, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1564	{
1565	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[uVirtq];
1566	uint64_t uVirtqPfn = (uint64_t)u32;
1567
1568	if (uVirtqPfn)
1569	{
1570	/* Transitional devices calculate ring physical addresses using rigid spec-defined formulae,
1571	* instead of guest conveying respective address of each ring, as "modern" VirtIO drivers do,
1572	* thus there is no virtq PFN or single base queue address stored in instance data for
1573	* this transitional device, but rather it is derived, when read back, from GCPhysVirtqDesc */
1574
1575	pVirtq->GCPhysVirtqDesc = uVirtqPfn * VIRTIO_PAGE_SIZE;
1576	pVirtq->GCPhysVirtqAvail = pVirtq->GCPhysVirtqDesc + sizeof(VIRTQ_DESC_T) * pVirtq->uQueueSize;
1577	pVirtq->GCPhysVirtqUsed =
1578	RT_ALIGN(pVirtq->GCPhysVirtqAvail + RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[pVirtq->uQueueSize]), VIRTIO_PAGE_SIZE);
1579	}
1580	else
1581	{
1582	/* Don't set ring addresses for queue (to meaningless values), when guest resets the virtq's PFN */
1583	pVirtq->GCPhysVirtqDesc = 0;
1584	pVirtq->GCPhysVirtqAvail = 0;
1585	pVirtq->GCPhysVirtqUsed = 0;
1586	}
1587	Log(("%-23s: Guest wrote uVirtqPfn .................... %#x:\n"
1588	"%68s... %p -> GCPhysVirtqDesc\n%68s... %p -> GCPhysVirtqAvail\n%68s... %p -> GCPhysVirtqUsed\n",
1589	__FUNCTION__, u32, " ", pVirtq->GCPhysVirtqDesc, " ", pVirtq->GCPhysVirtqAvail, " ", pVirtq->GCPhysVirtqUsed));
1590	}
1591	else
1592	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(uQueueNotify, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1593	{
1594	#ifdef IN_RING3
1595	ASSERT_GUEST_MSG(cb == 2, ("cb=%u\n", cb));
1596	pVirtio->uQueueNotify = u32 & 0xFFFF;
1597	if (uVirtq < VIRTQ_MAX_COUNT)
1598	{
1599	RT_UNTRUSTED_VALIDATED_FENCE();
1600
1601	/* Need to check that queue is configured. Legacy spec didn't have a queue enabled flag */
1602	if (pVirtio->aVirtqueues[pVirtio->uQueueNotify].GCPhysVirtqDesc)
1603	virtioCoreVirtqNotified(pDevIns, pVirtio, pVirtio->uQueueNotify, pVirtio->uQueueNotify /* uNotifyIdx */);
1604	else
1605	Log(("The queue (#%d) being notified has not been initialized.\n", pVirtio->uQueueNotify));
1606	}
1607	else
1608	Log(("Invalid queue number (%d)\n", pVirtio->uQueueNotify));
1609	#else
1610	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1611	return VINF_IOM_R3_IOPORT_WRITE;
1612	#endif
1613	}
1614	else
1615	if (VIRTIO_DEV_CONFIG_MATCH_MEMBER(fIsrStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort))
1616	{
1617	VIRTIO_DEV_CONFIG_LOG_ACCESS( fIsrStatus, VIRTIO_LEGACY_PCI_COMMON_CFG_T, offPort);
1618	LogFunc(("... WARNING: Guest attempted to write readonly device_feature (ISR status) (ignoring)\n"));
1619	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1620	return VINF_SUCCESS;
1621	}
1622	else if (offPort >= sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T))
1623	{
1624	#if IN_RING3
1625
1626	/* Access device-specific configuration */
1627	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1628	return pVirtioCC->pfnDevCapWrite(pDevIns, offPort - sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T), pv, cb);
1629	#else
1630	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1631	return VINF_IOM_R3_IOPORT_WRITE;
1632	#endif
1633	}
1634	else
1635	{
1636	Log2Func(("Bad guest write access to virtio_legacy_pci_common_cfg: offset=%#x, cb=0x%x\n",
1637	offPort, cb));
1638	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1639	int rc = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
1640	"virtioLegacyIOPortOut: no valid port at offset offset=%RTiop cb=0x%#x\n", offPort, cb);
1641	return rc;
1642	}
1643
1644	RT_NOREF(uVirtq);
1645	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1646	return VINF_SUCCESS;
1647	}
1648
1649
1650	/**
1651	* @callback_method_impl{FNIOMMMIONEWREAD,
1652	* Memory mapped I/O Handler for PCI Capabilities read operations.}
1653	*
1654	* This MMIO handler specifically supports the VIRTIO_PCI_CAP_PCI_CFG capability defined
1655	* in the VirtIO 1.0 specification, section 4.1.4.7, and as such is restricted to reads
1656	* of 1, 2 or 4 bytes, only.
1657	*
1658	*/
1659	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioRead(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void pv, unsigned cb)
1660	{
1661	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1662	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1663	AssertReturn(cb == 1 \|\| cb == 2 \|\| cb == 4, VERR_INVALID_PARAMETER);
1664	Assert(pVirtio == (PVIRTIOCORE)pvUser); RT_NOREF(pvUser);
1665	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatRead), a);
1666
1667
1668	uint32_t uOffset;
1669	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocDeviceCap))
1670	{
1671	#ifdef IN_RING3
1672	/*
1673	* Callback to client to manage device-specific configuration.
1674	*/
1675	VBOXSTRICTRC rcStrict = pVirtioCC->pfnDevCapRead(pDevIns, uOffset, pv, cb);
1676
1677	/*
1678	* Additionally, anytime any part of the device-specific configuration (which our client maintains)
1679	* is READ it needs to be checked to see if it changed since the last time any part was read, in
1680	* order to maintain the config generation (see VirtIO 1.0 spec, section 4.1.4.3.1)
1681	*/
1682	bool fDevSpecificFieldChanged = RT_BOOL(memcmp(pVirtioCC->pbDevSpecificCfg + uOffset,
1683	pVirtioCC->pbPrevDevSpecificCfg + uOffset,
1684	RT_MIN(cb, pVirtioCC->cbDevSpecificCfg - uOffset)));
1685
1686	memcpy(pVirtioCC->pbPrevDevSpecificCfg, pVirtioCC->pbDevSpecificCfg, pVirtioCC->cbDevSpecificCfg);
1687
1688	if (pVirtio->fGenUpdatePending \|\| fDevSpecificFieldChanged)
1689	{
1690	++pVirtio->uConfigGeneration;
1691	Log6Func(("Bumped cfg. generation to %d because %s%s\n",
1692	pVirtio->uConfigGeneration,
1693	fDevSpecificFieldChanged ? "<dev cfg changed> " : "",
1694	pVirtio->fGenUpdatePending ? "<update was pending>" : ""));
1695	pVirtio->fGenUpdatePending = false;
1696	}
1697
1698	virtioLowerInterrupt(pDevIns, 0);
1699	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
1700	return rcStrict;
1701	#else
1702	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
1703	return VINF_IOM_R3_MMIO_READ;
1704	#endif
1705	}
1706
1707	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocCommonCfgCap))
1708	return virtioCommonCfgAccessed(pDevIns, pVirtio, pVirtioCC, false /* fWrite */, uOffset, cb, pv);
1709
1710	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocIsrCap))
1711	{
1712	(uint8_t )pv = pVirtio->uISR;
1713	Log6Func(("Read and clear ISR\n"));
1714	pVirtio->uISR = 0; /* VirtIO spec requires reads of ISR to clear it */
1715	virtioLowerInterrupt(pDevIns, 0);
1716	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
1717	return VINF_SUCCESS;
1718	}
1719
1720	ASSERT_GUEST_MSG_FAILED(("Bad read access to mapped capabilities region: off=%RGp cb=%u\n", off, cb));
1721	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatRead), a);
1722	int rc = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
1723	"virtioMmioRead: Bad MMIO access to capabilities, offset=%RTiop cb=%08x\n", off, cb);
1724	return rc;
1725	}
1726
1727	/**
1728	* @callback_method_impl{FNIOMMMIONEWREAD,
1729	* Memory mapped I/O Handler for PCI Capabilities write operations.}
1730	*
1731	* This MMIO handler specifically supports the VIRTIO_PCI_CAP_PCI_CFG capability defined
1732	* in the VirtIO 1.0 specification, section 4.1.4.7, and as such is restricted to writes
1733	* of 1, 2 or 4 bytes, only.
1734	*/
1735	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioWrite(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void const pv, unsigned cb)
1736	{
1737	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1738	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1739	AssertReturn(cb == 1 \|\| cb == 2 \|\| cb == 4, VERR_INVALID_PARAMETER);
1740	Assert(pVirtio == (PVIRTIOCORE)pvUser); RT_NOREF(pvUser);
1741	STAM_PROFILE_ADV_START(&pVirtio->CTX_SUFF(StatWrite), a);
1742
1743	uint32_t uOffset;
1744	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocDeviceCap))
1745	{
1746	#ifdef IN_RING3
1747	/*
1748	* Foreward this MMIO write access for client to deal with.
1749	*/
1750	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1751	return pVirtioCC->pfnDevCapWrite(pDevIns, uOffset, pv, cb);
1752	#else
1753	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1754	return VINF_IOM_R3_MMIO_WRITE;
1755	#endif
1756	}
1757
1758	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocCommonCfgCap))
1759	{
1760	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1761	return virtioCommonCfgAccessed(pDevIns, pVirtio, pVirtioCC, true /* fWrite /, uOffset, cb, (void )pv);
1762	}
1763
1764	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocIsrCap) && cb == sizeof(uint8_t))
1765	{
1766	pVirtio->uISR = (uint8_t )pv;
1767	Log6Func(("Setting uISR = 0x%02x (virtq interrupt: %d, dev confg interrupt: %d)\n",
1768	pVirtio->uISR & 0xff,
1769	pVirtio->uISR & VIRTIO_ISR_VIRTQ_INTERRUPT,
1770	RT_BOOL(pVirtio->uISR & VIRTIO_ISR_DEVICE_CONFIG)));
1771	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1772	return VINF_SUCCESS;
1773	}
1774
1775	/* This should be guest driver dropping index of a new descriptor in avail ring */
1776	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, uOffset, pVirtio->LocNotifyCap) && cb == sizeof(uint16_t))
1777	{
1778	virtioCoreVirtqNotified(pDevIns, pVirtio, uOffset / VIRTIO_NOTIFY_OFFSET_MULTIPLIER, (uint16_t )pv);
1779	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1780	return VINF_SUCCESS;
1781	}
1782
1783	ASSERT_GUEST_MSG_FAILED(("Bad write access to mapped capabilities region: off=%RGp pv=%#p{%.*Rhxs} cb=%u\n", off, pv, cb, pv, cb));
1784	STAM_PROFILE_ADV_STOP(&pVirtio->CTX_SUFF(StatWrite), a);
1785	int rc = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS,
1786	"virtioMmioRead: Bad MMIO access to capabilities, offset=%RTiop cb=%08x\n", off, cb);
1787	return rc;
1788	}
1789
1790	#ifdef IN_RING3
1791
1792	/**
1793	* @callback_method_impl{FNPCICONFIGREAD}
1794	*/
1795	static DECLCALLBACK(VBOXSTRICTRC) virtioR3PciConfigRead(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev,
1796	uint32_t uAddress, unsigned cb, uint32_t *pu32Value)
1797	{
1798	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1799	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1800	RT_NOREF(pPciDev);
1801
1802	if (uAddress == pVirtio->uPciCfgDataOff)
1803	{
1804	/*
1805	* VirtIO 1.0 spec section 4.1.4.7 describes a required alternative access capability
1806	* whereby the guest driver can specify a bar, offset, and length via the PCI configuration space
1807	* (the virtio_pci_cfg_cap capability), and access data items.
1808	* This is used by BIOS to gain early boot access to the the storage device.
1809	*/
1810	struct virtio_pci_cap *pPciCap = &pVirtioCC->pPciCfgCap->pciCap;
1811	uint32_t uLength = pPciCap->uLength;
1812
1813	Log7Func((" pDevIns=%p pPciDev=%p uAddress=%#x%s cb=%u uLength=%d, bar=%d\n",
1814	pDevIns, pPciDev, uAddress, uAddress < 0x10 ? " " : "", cb, uLength, pPciCap->uBar));
1815
1816	if ( (uLength != 1 && uLength != 2 && uLength != 4)
1817	\|\| pPciCap->uBar != VIRTIO_REGION_PCI_CAP)
1818	{
1819	ASSERT_GUEST_MSG_FAILED(("Guest read virtio_pci_cfg_cap.pci_cfg_data using mismatching config. "
1820	"Ignoring\n"));
1821	*pu32Value = UINT32_MAX;
1822	return VINF_SUCCESS;
1823	}
1824
1825	VBOXSTRICTRC rcStrict = virtioMmioRead(pDevIns, pVirtio, pPciCap->uOffset, pu32Value, cb);
1826	Log7Func((" Guest read virtio_pci_cfg_cap.pci_cfg_data, bar=%d, offset=%d, length=%d, result=0x%x -> %Rrc\n",
1827	pPciCap->uBar, pPciCap->uOffset, uLength, *pu32Value, VBOXSTRICTRC_VAL(rcStrict)));
1828	return rcStrict;
1829	}
1830	Log7Func((" pDevIns=%p pPciDev=%p uAddress=%#x%s cb=%u pu32Value=%p\n",
1831	pDevIns, pPciDev, uAddress, uAddress < 0x10 ? " " : "", cb, pu32Value));
1832	return VINF_PDM_PCI_DO_DEFAULT;
1833	}
1834
1835	/**
1836	* @callback_method_impl{FNPCICONFIGWRITE}
1837	*/
1838	static DECLCALLBACK(VBOXSTRICTRC) virtioR3PciConfigWrite(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev,
1839	uint32_t uAddress, unsigned cb, uint32_t u32Value)
1840	{
1841	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1842	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1843	RT_NOREF(pPciDev);
1844
1845	Log7Func(("pDevIns=%p pPciDev=%p uAddress=%#x %scb=%u u32Value=%#x\n", pDevIns, pPciDev, uAddress, uAddress < 0xf ? " " : "", cb, u32Value));
1846	if (uAddress == pVirtio->uPciCfgDataOff)
1847	{
1848	/* VirtIO 1.0 spec section 4.1.4.7 describes a required alternative access capability
1849	* whereby the guest driver can specify a bar, offset, and length via the PCI configuration space
1850	* (the virtio_pci_cfg_cap capability), and access data items.
1851	* This is used by BIOS to gain early boot access to the the storage device.*/
1852
1853	struct virtio_pci_cap *pPciCap = &pVirtioCC->pPciCfgCap->pciCap;
1854	uint32_t uLength = pPciCap->uLength;
1855
1856	if ( (uLength != 1 && uLength != 2 && uLength != 4)
1857	\|\| cb != uLength
1858	\|\| pPciCap->uBar != VIRTIO_REGION_PCI_CAP)
1859	{
1860	ASSERT_GUEST_MSG_FAILED(("Guest write virtio_pci_cfg_cap.pci_cfg_data using mismatching config. Ignoring\n"));
1861	return VINF_SUCCESS;
1862	}
1863
1864	VBOXSTRICTRC rcStrict = virtioMmioWrite(pDevIns, pVirtio, pPciCap->uOffset, &u32Value, cb);
1865	Log2Func(("Guest wrote virtio_pci_cfg_cap.pci_cfg_data, bar=%d, offset=%x, length=%x, value=%d -> %Rrc\n",
1866	pPciCap->uBar, pPciCap->uOffset, uLength, u32Value, VBOXSTRICTRC_VAL(rcStrict)));
1867	return rcStrict;
1868	}
1869	return VINF_PDM_PCI_DO_DEFAULT;
1870	}
1871
1872
1873	/*********************************************************************************************************************************
1874	* Saved state. *
1875	*********************************************************************************************************************************/
1876
1877	/**
1878	* Called from the FNSSMDEVSAVEEXEC function of the device.
1879	*
1880	* @param pVirtio Pointer to the shared virtio state.
1881	* @param pHlp The ring-3 device helpers.
1882	* @param pSSM The saved state handle.
1883	* @returns VBox status code.
1884	*/
1885	int virtioCoreR3SaveExec(PVIRTIOCORE pVirtio, PCPDMDEVHLPR3 pHlp, PSSMHANDLE pSSM)
1886	{
1887	LogFunc(("\n"));
1888	pHlp->pfnSSMPutU64(pSSM, VIRTIO_SAVEDSTATE_MARKER);
1889	pHlp->pfnSSMPutU32(pSSM, VIRTIO_SAVEDSTATE_VERSION);
1890
1891	pHlp->pfnSSMPutBool(pSSM, pVirtio->fGenUpdatePending);
1892	pHlp->pfnSSMPutU8( pSSM, pVirtio->fDeviceStatus);
1893	pHlp->pfnSSMPutU8( pSSM, pVirtio->uConfigGeneration);
1894	pHlp->pfnSSMPutU8( pSSM, pVirtio->uPciCfgDataOff);
1895	pHlp->pfnSSMPutU8( pSSM, pVirtio->uISR);
1896	pHlp->pfnSSMPutU16( pSSM, pVirtio->uVirtqSelect);
1897	pHlp->pfnSSMPutU32( pSSM, pVirtio->uDeviceFeaturesSelect);
1898	pHlp->pfnSSMPutU32( pSSM, pVirtio->uDriverFeaturesSelect);
1899	pHlp->pfnSSMPutU64( pSSM, pVirtio->uDriverFeatures);
1900
1901	for (uint32_t i = 0; i < VIRTQ_MAX_COUNT; i++)
1902	{
1903	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[i];
1904
1905	pHlp->pfnSSMPutGCPhys64( pSSM, pVirtq->GCPhysVirtqDesc);
1906	pHlp->pfnSSMPutGCPhys64( pSSM, pVirtq->GCPhysVirtqAvail);
1907	pHlp->pfnSSMPutGCPhys64( pSSM, pVirtq->GCPhysVirtqUsed);
1908	pHlp->pfnSSMPutU16( pSSM, pVirtq->uNotifyOffset);
1909	pHlp->pfnSSMPutU16( pSSM, pVirtq->uMsixVector);
1910	pHlp->pfnSSMPutU16( pSSM, pVirtq->uEnable);
1911	pHlp->pfnSSMPutU16( pSSM, pVirtq->uQueueSize);
1912	pHlp->pfnSSMPutU16( pSSM, pVirtq->uAvailIdxShadow);
1913	pHlp->pfnSSMPutU16( pSSM, pVirtq->uUsedIdxShadow);
1914	int rc = pHlp->pfnSSMPutMem(pSSM, pVirtq->szName, 32);
1915	AssertRCReturn(rc, rc);
1916	}
1917
1918	return VINF_SUCCESS;
1919	}
1920
1921	/**
1922	* Called from the FNSSMDEVLOADEXEC function of the device.
1923	*
1924	* @param pVirtio Pointer to the shared virtio state.
1925	* @param pHlp The ring-3 device helpers.
1926	* @param pSSM The saved state handle.
1927	* @returns VBox status code.
1928	*/
1929	int virtioCoreR3LoadExec(PVIRTIOCORE pVirtio, PCPDMDEVHLPR3 pHlp, PSSMHANDLE pSSM)
1930	{
1931	LogFunc(("\n"));
1932	/*
1933	* Check the marker and (embedded) version number.
1934	*/
1935	uint64_t uMarker = 0;
1936	int rc = pHlp->pfnSSMGetU64(pSSM, &uMarker);
1937	AssertRCReturn(rc, rc);
1938	if (uMarker != VIRTIO_SAVEDSTATE_MARKER)
1939	return pHlp->pfnSSMSetLoadError(pSSM, VERR_SSM_DATA_UNIT_FORMAT_CHANGED, RT_SRC_POS,
1940	N_("Expected marker value %#RX64 found %#RX64 instead"),
1941	VIRTIO_SAVEDSTATE_MARKER, uMarker);
1942	uint32_t uVersion = 0;
1943	rc = pHlp->pfnSSMGetU32(pSSM, &uVersion);
1944	AssertRCReturn(rc, rc);
1945	if (uVersion != VIRTIO_SAVEDSTATE_VERSION)
1946	return pHlp->pfnSSMSetLoadError(pSSM, VERR_SSM_DATA_UNIT_FORMAT_CHANGED, RT_SRC_POS,
1947	N_("Unsupported virtio version: %u"), uVersion);
1948	/*
1949	* Load the state.
1950	*/
1951	pHlp->pfnSSMGetBool( pSSM, &pVirtio->fGenUpdatePending);
1952	pHlp->pfnSSMGetU8( pSSM, &pVirtio->fDeviceStatus);
1953	pHlp->pfnSSMGetU8( pSSM, &pVirtio->uConfigGeneration);
1954	pHlp->pfnSSMGetU8( pSSM, &pVirtio->uPciCfgDataOff);
1955	pHlp->pfnSSMGetU8( pSSM, &pVirtio->uISR);
1956	pHlp->pfnSSMGetU16( pSSM, &pVirtio->uVirtqSelect);
1957	pHlp->pfnSSMGetU32( pSSM, &pVirtio->uDeviceFeaturesSelect);
1958	pHlp->pfnSSMGetU32( pSSM, &pVirtio->uDriverFeaturesSelect);
1959	pHlp->pfnSSMGetU64( pSSM, &pVirtio->uDriverFeatures);
1960
1961	for (uint32_t i = 0; i < VIRTQ_MAX_COUNT; i++)
1962	{
1963	PVIRTQUEUE pVirtq = &pVirtio->aVirtqueues[i];
1964
1965	pHlp->pfnSSMGetGCPhys64( pSSM, &pVirtq->GCPhysVirtqDesc);
1966	pHlp->pfnSSMGetGCPhys64( pSSM, &pVirtq->GCPhysVirtqAvail);
1967	pHlp->pfnSSMGetGCPhys64( pSSM, &pVirtq->GCPhysVirtqUsed);
1968	pHlp->pfnSSMGetU16( pSSM, &pVirtq->uNotifyOffset);
1969	pHlp->pfnSSMGetU16( pSSM, &pVirtq->uMsixVector);
1970	pHlp->pfnSSMGetU16( pSSM, &pVirtq->uEnable);
1971	pHlp->pfnSSMGetU16( pSSM, &pVirtq->uQueueSize);
1972	pHlp->pfnSSMGetU16( pSSM, &pVirtq->uAvailIdxShadow);
1973	pHlp->pfnSSMGetU16( pSSM, &pVirtq->uUsedIdxShadow);
1974	rc = pHlp->pfnSSMGetMem( pSSM, pVirtq->szName, sizeof(pVirtq->szName));
1975	AssertRCReturn(rc, rc);
1976	}
1977
1978	return VINF_SUCCESS;
1979	}
1980
1981
1982	/*********************************************************************************************************************************
1983	* Device Level *
1984	*********************************************************************************************************************************/
1985
1986	/**
1987	* This must be called by the client to handle VM state changes
1988	* after the client takes care of its device-specific tasks for the state change.
1989	* (i.e. Reset, suspend, power-off, resume)
1990	*
1991	* @param pDevIns The device instance.
1992	* @param pVirtio Pointer to the shared virtio state.
1993	*/
1994	void virtioCoreR3VmStateChanged(PVIRTIOCORE pVirtio, VIRTIOVMSTATECHANGED enmState)
1995	{
1996	LogFunc(("State changing to %s\n",
1997	virtioCoreGetStateChangeText(enmState)));
1998
1999	switch(enmState)
2000	{
2001	case kvirtIoVmStateChangedReset:
2002	virtioCoreResetAll(pVirtio);
2003	break;
2004	case kvirtIoVmStateChangedSuspend:
2005	break;
2006	case kvirtIoVmStateChangedPowerOff:
2007	break;
2008	case kvirtIoVmStateChangedResume:
2009	for (int uVirtq = 0; uVirtq < VIRTQ_MAX_COUNT; uVirtq++)
2010	{
2011	if (!pVirtio->fLegacyDriver \|\| pVirtio->aVirtqueues[uVirtq].uEnable)
2012	virtioCoreNotifyGuestDriver(pVirtio->pDevInsR3, pVirtio, uVirtq);
2013	}
2014	break;
2015	default:
2016	LogRelFunc(("Bad enum value"));
2017	return;
2018	}
2019	}
2020
2021	/**
2022	* This should be called from PDMDEVREGR3::pfnDestruct.
2023	*
2024	* @param pDevIns The device instance.
2025	* @param pVirtio Pointer to the shared virtio state.
2026	* @param pVirtioCC Pointer to the ring-3 virtio state.
2027	*/
2028	void virtioCoreR3Term(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
2029	{
2030	if (pVirtioCC->pbPrevDevSpecificCfg)
2031	{
2032	RTMemFree(pVirtioCC->pbPrevDevSpecificCfg);
2033	pVirtioCC->pbPrevDevSpecificCfg = NULL;
2034	}
2035
2036	RT_NOREF(pDevIns, pVirtio);
2037	}
2038
2039	/** API Function: See header file */
2040	int virtioCoreR3Init(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC, PVIRTIOPCIPARAMS pPciParams,
2041	const char pcszInstance, uint64_t fDevSpecificFeatures, void pvDevSpecificCfg, uint16_t cbDevSpecificCfg)
2042	{
2043
2044
2045	/*
2046	* The pVirtio state must be the first member of the shared device instance
2047	* data, otherwise we cannot get our bearings in the PCI configuration callbacks.
2048	*/
2049	AssertLogRelReturn(pVirtio == PDMINS_2_DATA(pDevIns, PVIRTIOCORE), VERR_STATE_CHANGED);
2050	AssertLogRelReturn(pVirtioCC == PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC), VERR_STATE_CHANGED);
2051
2052	pVirtio->pDevInsR3 = pDevIns;
2053
2054	/*
2055	* Caller must initialize these.
2056	*/
2057	AssertReturn(pVirtioCC->pfnStatusChanged, VERR_INVALID_POINTER);
2058	AssertReturn(pVirtioCC->pfnVirtqNotified, VERR_INVALID_POINTER);
2059	AssertReturn(VIRTQ_SIZE > 0 && VIRTQ_SIZE <= 32768, VERR_OUT_OF_RANGE); /* VirtIO specification-defined limit */
2060
2061	#if 0 /* Until pdmR3DvHlp_PCISetIrq() impl is fixed and Assert that limits vec to 0 is removed
2062	* The legacy MSI support has not been implemented yet
2063	*/
2064	# ifdef VBOX_WITH_MSI_DEVICES
2065	pVirtio->fMsiSupport = true;
2066	# endif
2067	#endif
2068
2069
2070	/*
2071	* The host features offered include both device-specific features
2072	* and reserved feature bits (device independent)
2073	*/
2074	pVirtio->uDeviceFeatures = VIRTIO_F_VERSION_1
2075	\| VIRTIO_DEV_INDEPENDENT_FEATURES_OFFERED
2076	\| fDevSpecificFeatures;
2077
2078	RTStrCopy(pVirtio->szInstance, sizeof(pVirtio->szInstance), pcszInstance);
2079	pVirtioCC->cbDevSpecificCfg = cbDevSpecificCfg;
2080	pVirtioCC->pbDevSpecificCfg = (uint8_t *)pvDevSpecificCfg;
2081	pVirtioCC->pbPrevDevSpecificCfg = (uint8_t *)RTMemDup(pvDevSpecificCfg, cbDevSpecificCfg);
2082	AssertLogRelReturn(pVirtioCC->pbPrevDevSpecificCfg, VERR_NO_MEMORY);
2083
2084	/* Set PCI config registers (assume 32-bit mode) */
2085	PPDMPCIDEV pPciDev = pDevIns->apPciDevs[0];
2086	PDMPCIDEV_ASSERT_VALID(pDevIns, pPciDev);
2087
2088	PDMPciDevSetRevisionId(pPciDev, DEVICE_PCI_REVISION_ID_VIRTIO);
2089	PDMPciDevSetVendorId(pPciDev, DEVICE_PCI_VENDOR_ID_VIRTIO);
2090	PDMPciDevSetDeviceId(pPciDev, pPciParams->uDeviceId);
2091	PDMPciDevSetSubSystemId(pPciDev, DEVICE_PCI_NETWORK_SUBSYSTEM);
2092	PDMPciDevSetSubSystemVendorId(pPciDev, DEVICE_PCI_VENDOR_ID_VIRTIO);
2093	PDMPciDevSetClassBase(pPciDev, pPciParams->uClassBase);
2094	PDMPciDevSetClassSub(pPciDev, pPciParams->uClassSub);
2095	PDMPciDevSetClassProg(pPciDev, pPciParams->uClassProg);
2096	PDMPciDevSetInterruptLine(pPciDev, pPciParams->uInterruptLine);
2097	PDMPciDevSetInterruptPin(pPciDev, pPciParams->uInterruptPin);
2098
2099	/* Register PCI device */
2100	int rc = PDMDevHlpPCIRegister(pDevIns, pPciDev);
2101	if (RT_FAILURE(rc))
2102	return PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register PCI Device")); /* can we put params in this error? */
2103
2104	rc = PDMDevHlpPCIInterceptConfigAccesses(pDevIns, pPciDev, virtioR3PciConfigRead, virtioR3PciConfigWrite);
2105	AssertRCReturn(rc, rc);
2106
2107	/* Construct & map PCI vendor-specific capabilities for virtio host negotiation with guest driver */
2108
2109	#define CFG_ADDR_2_IDX(addr) ((uint8_t)(((uintptr_t)(addr) - (uintptr_t)&pPciDev->abConfig[0])))
2110	#define SET_PCI_CAP_LOC(a_pPciDev, a_pCfg, a_LocCap, a_uMmioLengthAlign) \
2111	do { \
2112	(a_LocCap).offMmio = (a_pCfg)->uOffset; \
2113	(a_LocCap).cbMmio = RT_ALIGN_T((a_pCfg)->uLength, a_uMmioLengthAlign, uint16_t); \
2114	(a_LocCap).offPci = (uint16_t)(uintptr_t)((uint8_t *)(a_pCfg) - &(a_pPciDev)->abConfig[0]); \
2115	(a_LocCap).cbPci = (a_pCfg)->uCapLen; \
2116	} while (0)
2117
2118	PVIRTIO_PCI_CAP_T pCfg;
2119	uint32_t cbRegion = 0;
2120
2121	/* Common capability (VirtIO 1.0 spec, section 4.1.4.3) */
2122	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[0x40];
2123	pCfg->uCfgType = VIRTIO_PCI_CAP_COMMON_CFG;
2124	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2125	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
2126	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
2127	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2128	pCfg->uOffset = RT_ALIGN_32(0, 4); /* Currently 0, but reminder to 32-bit align if changing this */
2129	pCfg->uLength = sizeof(VIRTIO_PCI_COMMON_CFG_T);
2130	cbRegion += pCfg->uLength;
2131	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocCommonCfgCap, 2);
2132	pVirtioCC->pCommonCfgCap = pCfg;
2133
2134	/*
2135	* Notify capability (VirtIO 1.0 spec, section 4.1.4.4). Note: uLength is based on the choice
2136	* of this implementation to make each queue's uNotifyOffset equal to (VirtqSelect) ordinal
2137	* value of the queue (different strategies are possible according to spec).
2138	*/
2139	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2140	pCfg->uCfgType = VIRTIO_PCI_CAP_NOTIFY_CFG;
2141	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2142	pCfg->uCapLen = sizeof(VIRTIO_PCI_NOTIFY_CAP_T);
2143	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
2144	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2145	pCfg->uOffset = pVirtioCC->pCommonCfgCap->uOffset + pVirtioCC->pCommonCfgCap->uLength;
2146	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
2147	pCfg->uLength = VIRTQ_MAX_COUNT * VIRTIO_NOTIFY_OFFSET_MULTIPLIER + 2; /* will change in VirtIO 1.1 */
2148	cbRegion += pCfg->uLength;
2149	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocNotifyCap, 1);
2150	pVirtioCC->pNotifyCap = (PVIRTIO_PCI_NOTIFY_CAP_T)pCfg;
2151	pVirtioCC->pNotifyCap->uNotifyOffMultiplier = VIRTIO_NOTIFY_OFFSET_MULTIPLIER;
2152
2153	/* ISR capability (VirtIO 1.0 spec, section 4.1.4.5)
2154	*
2155	* VirtIO 1.0 spec says 8-bit, unaligned in MMIO space. Example/diagram
2156	* of spec shows it as a 32-bit field with upper bits 'reserved'
2157	* Will take spec's words more literally than the diagram for now.
2158	*/
2159	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2160	pCfg->uCfgType = VIRTIO_PCI_CAP_ISR_CFG;
2161	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2162	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
2163	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
2164	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2165	pCfg->uOffset = pVirtioCC->pNotifyCap->pciCap.uOffset + pVirtioCC->pNotifyCap->pciCap.uLength;
2166	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
2167	pCfg->uLength = sizeof(uint8_t);
2168	cbRegion += pCfg->uLength;
2169	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocIsrCap, 4);
2170	pVirtioCC->pIsrCap = pCfg;
2171
2172	/* PCI Cfg capability (VirtIO 1.0 spec, section 4.1.4.7)
2173	* This capability doesn't get page-MMIO mapped. Instead uBar, uOffset and uLength are intercepted
2174	* by trapping PCI configuration I/O and get modulated by consumers to locate fetch and read/write
2175	* values from any region. NOTE: The linux driver not only doesn't use this feature, it will not
2176	* even list it as present if uLength isn't non-zero and also 4-byte-aligned as the linux driver is
2177	* initializing.
2178	*/
2179	pVirtio->uPciCfgDataOff = pCfg->uCapNext + RT_OFFSETOF(VIRTIO_PCI_CFG_CAP_T, uPciCfgData);
2180	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2181	pCfg->uCfgType = VIRTIO_PCI_CAP_PCI_CFG;
2182	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2183	pCfg->uCapLen = sizeof(VIRTIO_PCI_CFG_CAP_T);
2184	pCfg->uCapNext = (pVirtio->fMsiSupport \|\| pVirtioCC->pbDevSpecificCfg) ? CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen : 0;
2185	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2186	pCfg->uOffset = 0;
2187	pCfg->uLength = 4;
2188	cbRegion += pCfg->uLength;
2189	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocPciCfgCap, 1);
2190	pVirtioCC->pPciCfgCap = (PVIRTIO_PCI_CFG_CAP_T)pCfg;
2191
2192	if (pVirtioCC->pbDevSpecificCfg)
2193	{
2194	/* Device specific config capability (via VirtIO 1.0, section 4.1.4.6).
2195	* Client defines the device-specific config struct and passes size to virtioCoreR3Init()
2196	* to inform this. */
2197	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2198	pCfg->uCfgType = VIRTIO_PCI_CAP_DEVICE_CFG;
2199	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2200	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
2201	pCfg->uCapNext = pVirtio->fMsiSupport ? CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen : 0;
2202	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2203	pCfg->uOffset = pVirtioCC->pIsrCap->uOffset + pVirtioCC->pIsrCap->uLength;
2204	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
2205	pCfg->uLength = cbDevSpecificCfg;
2206	cbRegion += pCfg->uLength;
2207	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocDeviceCap, 4);
2208	pVirtioCC->pDeviceCap = pCfg;
2209	}
2210	else
2211	Assert(pVirtio->LocDeviceCap.cbMmio == 0 && pVirtio->LocDeviceCap.cbPci == 0);
2212
2213	if (pVirtio->fMsiSupport)
2214	{
2215	PDMMSIREG aMsiReg;
2216	RT_ZERO(aMsiReg);
2217	aMsiReg.iMsixCapOffset = pCfg->uCapNext;
2218	aMsiReg.iMsixNextOffset = 0;
2219	aMsiReg.iMsixBar = VIRTIO_REGION_MSIX_CAP;
2220	aMsiReg.cMsixVectors = VBOX_MSIX_MAX_ENTRIES;
2221	rc = PDMDevHlpPCIRegisterMsi(pDevIns, &aMsiReg); /* see MsixR3init() */
2222	if (RT_FAILURE(rc))
2223	{
2224	/* See PDMDevHlp.cpp:pdmR3DevHlp_PCIRegisterMsi */
2225	LogFunc(("Failed to configure MSI-X (%Rrc). Reverting to INTx\n", rc));
2226	pVirtio->fMsiSupport = false;
2227	}
2228	else
2229	Log2Func(("Using MSI-X for guest driver notification\n"));
2230	}
2231	else
2232	LogFunc(("MSI-X not available for VBox, using INTx notification\n"));
2233
2234	/* Set offset to first capability and enable PCI dev capabilities */
2235	PDMPciDevSetCapabilityList(pPciDev, 0x40);
2236	PDMPciDevSetStatus(pPciDev, VBOX_PCI_STATUS_CAP_LIST);
2237
2238	size_t cbSize = RTStrPrintf(pVirtioCC->pcszMmioName, sizeof(pVirtioCC->pcszMmioName), "%s (modern)", pcszInstance);
2239	if (cbSize <= 0)
2240	return PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: out of memory allocating string")); /* can we put params in this error? */
2241
2242	cbSize = RTStrPrintf(pVirtioCC->pcszPortIoName, sizeof(pVirtioCC->pcszPortIoName), "%s (legacy)", pcszInstance);
2243	if (cbSize <= 0)
2244	return PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: out of memory allocating string")); /* can we put params in this error? */
2245
2246	/* As a transitional device that supports legacy VirtIO drivers, this VirtIO device generic implementation presents
2247	* legacy driver interface in I/O space at BAR0. The following maps the common (e.g. device independent)
2248	* dev config area as well as device-specific dev config area (whose size is passed to init function of this VirtIO
2249	* generic device code) for access via Port I/O, since legacy drivers (e.g. pre VirtIO 1.0) don't use MMIO callbacks.
2250	* (See VirtIO 1.1, Section 4.1.4.8).
2251	*/
2252	rc = PDMDevHlpPCIIORegionCreateIo(pDevIns, VIRTIO_REGION_LEGACY_IO, sizeof(VIRTIO_LEGACY_PCI_COMMON_CFG_T) + cbDevSpecificCfg,
2253	virtioLegacyIOPortOut, virtioLegacyIOPortIn, NULL /pvUser/, pVirtioCC->pcszPortIoName,
2254	NULL /paExtDescs/, &pVirtio->hLegacyIoPorts);
2255	AssertLogRelRCReturn(rc, PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register legacy config in I/O space at BAR0 */")));
2256
2257	/* Note: The Linux driver at drivers/virtio/virtio_pci_modern.c tries to map at least a page for the
2258	* 'unknown' device-specific capability without querying the capability to figure
2259	* out size, so pad with an extra page
2260	*/
2261	rc = PDMDevHlpPCIIORegionCreateMmio(pDevIns, VIRTIO_REGION_PCI_CAP, RT_ALIGN_32(cbRegion + VIRTIO_PAGE_SIZE, VIRTIO_PAGE_SIZE),
2262	PCI_ADDRESS_SPACE_MEM, virtioMmioWrite, virtioMmioRead, pVirtio,
2263	IOMMMIO_FLAGS_READ_PASSTHRU \| IOMMMIO_FLAGS_WRITE_PASSTHRU,
2264	pVirtioCC->pcszMmioName,
2265	&pVirtio->hMmioPciCap);
2266	AssertLogRelRCReturn(rc, PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register PCI Capabilities address space")));
2267	/*
2268	* Statistics.
2269	*/
2270	# ifdef VBOX_WITH_STATISTICS
2271	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsAllocated, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
2272	"Total number of allocated descriptor chains", "DescChainsAllocated");
2273	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsFreed, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
2274	"Total number of freed descriptor chains", "DescChainsFreed");
2275	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsSegsIn, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
2276	"Total number of inbound segments", "DescChainsSegsIn");
2277	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsSegsOut, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
2278	"Total number of outbound segments", "DescChainsSegsOut");
2279	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatReadR3, STAMTYPE_PROFILE, "IO/ReadR3", STAMUNIT_TICKS_PER_CALL, "Profiling IO reads in R3");
2280	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatReadR0, STAMTYPE_PROFILE, "IO/ReadR0", STAMUNIT_TICKS_PER_CALL, "Profiling IO reads in R0");
2281	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatReadRC, STAMTYPE_PROFILE, "IO/ReadRC", STAMUNIT_TICKS_PER_CALL, "Profiling IO reads in RC");
2282	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatWriteR3, STAMTYPE_PROFILE, "IO/WriteR3", STAMUNIT_TICKS_PER_CALL, "Profiling IO writes in R3");
2283	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatWriteR0, STAMTYPE_PROFILE, "IO/WriteR0", STAMUNIT_TICKS_PER_CALL, "Profiling IO writes in R0");
2284	PDMDevHlpSTAMRegister(pDevIns, &pVirtio->StatWriteRC, STAMTYPE_PROFILE, "IO/WriteRC", STAMUNIT_TICKS_PER_CALL, "Profiling IO writes in RC");
2285	# endif /* VBOX_WITH_STATISTICS */
2286
2287	virtioResetDevice(pDevIns, pVirtio); /* Reset VirtIO specific state of device */
2288
2289	return VINF_SUCCESS;
2290	}
2291
2292	#else /* !IN_RING3 */
2293
2294	/**
2295	* Sets up the core ring-0/raw-mode virtio bits.
2296	*
2297	* @returns VBox status code.
2298	* @param pDevIns The device instance.
2299	* @param pVirtio Pointer to the shared virtio state. This must be the first
2300	* member in the shared device instance data!
2301	*/
2302	int virtioCoreRZInit(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio)
2303	{
2304	AssertLogRelReturn(pVirtio == PDMINS_2_DATA(pDevIns, PVIRTIOCORE), VERR_STATE_CHANGED);
2305
2306	#ifdef FUTURE_OPTIMIZATION
2307	int rc = PDMDevHlpSetDeviceCritSect(pDevIns, PDMDevHlpCritSectGetNop(pDevIns));
2308	AssertRCReturn(rc, rc);
2309	#endif
2310	int rc = PDMDevHlpMmioSetUpContext(pDevIns, pVirtio->hMmioPciCap, virtioMmioWrite, virtioMmioRead, pVirtio);
2311	AssertRCReturn(rc, rc);
2312
2313	rc = PDMDevHlpIoPortSetUpContext(pDevIns, pVirtio->hLegacyIoPorts, virtioLegacyIOPortOut, virtioLegacyIOPortIn, NULL /pvUser/);
2314	AssertRCReturn(rc, rc);
2315
2316	return rc;
2317	}
2318
2319	#endif /* !IN_RING3 */
2320

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Devices/VirtIO/VirtioCore.cpp@ 91703

Download in other formats: