Virtio_1_0.cpp@ 84372

Last change on this file since 84372 was 84354, checked in by vboxsync, 5 years ago
Fix burns
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 89.7 KB

Line
1	/* $Id: Virtio_1_0.cpp 84354 2020-05-19 06:16:04Z vboxsync $ */
2	/** @file
3	* Virtio_1_0 - Virtio Common (PCI, feature & config mgt, queue mgt & proxy, notification mgt)
4	*/
5
6	/*
7	* Copyright (C) 2009-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_DEV_VIRTIO
23
24	#include <VBox/log.h>
25	#include <VBox/msi.h>
26	#include <VBox/AssertGuest.h>
27	#include <iprt/param.h>
28	#include <iprt/assert.h>
29	#include <iprt/uuid.h>
30	#include <iprt/mem.h>
31	#include <iprt/assert.h>
32	#include <iprt/sg.h>
33	#include <iprt/string.h>
34	#include <VBox/vmm/pdmdev.h>
35	#include "Virtio_1_0.h"
36
37
38	/*********************************************************************************************************************************
39	* Defined Constants And Macros *
40	*********************************************************************************************************************************/
41	#define INSTANCE(a_pVirtio) ((a_pVirtio)->szInstance)
42	#define VIRTQNAME(a_pVirtio, a_idxQueue) ((a_pVirtio)->virtqState[(a_idxQueue)].szVirtqName)
43	#define IS_DRIVER_OK(a_pVirtio) ((a_pVirtio)->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK)
44
45	/**
46	* This macro returns true if the @a a_offAccess and access length (@a
47	* a_cbAccess) are within the range of the mapped capability struct described by
48	* @a a_LocCapData.
49	*
50	* @param[in] a_offAccess The offset into the MMIO bar of the access.
51	* @param[in] a_cbAccess The access size.
52	* @param[out] a_offIntraVar The variable to return the intra-capability
53	* offset into. ASSUMES this is uint32_t.
54	* @param[in] a_LocCapData The capability location info.
55	*/
56	#define MATCHES_VIRTIO_CAP_STRUCT(a_offAccess, a_cbAccess, a_offIntraVar, a_LocCapData) \
57	( ((a_offIntraVar) = (uint32_t)((a_offAccess) - (a_LocCapData).offMmio)) < (uint32_t)(a_LocCapData).cbMmio \
58	&& (a_offIntraVar) + (uint32_t)(a_cbAccess) <= (uint32_t)(a_LocCapData).cbMmio )
59
60
61	/** Marks the start of the virtio saved state (just for sanity). */
62	#define VIRTIO_SAVEDSTATE_MARKER UINT64_C(0x1133557799bbddff)
63	/** The current saved state version for the virtio core. */
64	#define VIRTIO_SAVEDSTATE_VERSION UINT32_C(1)
65
66
67	/*********************************************************************************************************************************
68	* Structures and Typedefs *
69	*********************************************************************************************************************************/
70	/**
71	* virtq related structs
72	* (struct names follow VirtIO 1.0 spec, typedef use VBox style)
73	*/
74	typedef struct virtq_desc
75	{
76	uint64_t GCPhysBuf; /*< addr GC Phys. address of buffer /
77	uint32_t cb; /*< len Buffer length /
78	uint16_t fFlags; /*< flags Buffer specific flags /
79	uint16_t uDescIdxNext; /*< next Idx set if VIRTIO_DESC_F_NEXT /
80	} VIRTQ_DESC_T, *PVIRTQ_DESC_T;
81
82	typedef struct virtq_avail
83	{
84	uint16_t fFlags; /*< flags avail ring drv to dev flags /
85	uint16_t uIdx; /*< idx Index of next free ring slot /
86	uint16_t auRing[RT_FLEXIBLE_ARRAY]; /*< ring Ring: avail drv to dev bufs /
87	/* uint16_t uUsedEventIdx; - used_event (if VIRTQ_USED_F_EVENT_IDX) */
88	} VIRTQ_AVAIL_T, *PVIRTQ_AVAIL_T;
89
90	typedef struct virtq_used_elem
91	{
92	uint32_t uDescIdx; /*< idx Start of used desc chain /
93	uint32_t cbElem; /*< len Total len of used desc chain /
94	} VIRTQ_USED_ELEM_T;
95
96	typedef struct virt_used
97	{
98	uint16_t fFlags; /*< flags used ring host-to-guest flags /
99	uint16_t uIdx; /*< idx Index of next ring slot /
100	VIRTQ_USED_ELEM_T aRing[RT_FLEXIBLE_ARRAY]; /*< ring Ring: used dev to drv bufs /
101	/* uint16_t uAvailEventIdx; - avail_event if (VIRTQ_USED_F_EVENT_IDX) */
102	} VIRTQ_USED_T, *PVIRTQ_USED_T;
103
104
105	const char *virtioCoreGetStateChangeText(VIRTIOVMSTATECHANGED enmState)
106	{
107	switch (enmState)
108	{
109	case kvirtIoVmStateChangedReset: return "VM RESET";
110	case kvirtIoVmStateChangedSuspend: return "VM SUSPEND";
111	case kvirtIoVmStateChangedPowerOff: return "VM POWER OFF";
112	case kvirtIoVmStateChangedResume: return "VM RESUME";
113	default: return "<BAD ENUM>";
114	}
115	}
116
117	/* Internal Functions */
118
119	static void virtioNotifyGuestDriver(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, bool fForce);
120	static int virtioKick(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint8_t uCause, uint16_t uVec, bool fForce);
121
122	/** @name Internal queue operations
123	* @{ */
124
125	/**
126	* Accessor for virtq descriptor
127	*/
128	#ifdef IN_RING3
129	DECLINLINE(void) virtioReadDesc(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue,
130	uint32_t idxDesc, PVIRTQ_DESC_T pDesc)
131	{
132	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
133	uint16_t const cQueueItems = RT_MAX(pVirtio->uQueueSize[idxQueue], 1); /* Make sure to avoid div-by-zero. */
134	PDMDevHlpPCIPhysRead(pDevIns,
135	pVirtio->aGCPhysQueueDesc[idxQueue] + sizeof(VIRTQ_DESC_T) * (idxDesc % cQueueItems),
136	pDesc, sizeof(VIRTQ_DESC_T));
137	}
138	#endif
139
140	/**
141	* Accessors for virtq avail ring
142	*/
143	#ifdef IN_RING3
144	DECLINLINE(uint16_t) virtioReadAvailDescIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, uint32_t availIdx)
145	{
146	uint16_t uDescIdx;
147	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
148	uint16_t const cQueueItems = RT_MAX(pVirtio->uQueueSize[idxQueue], 1); /* Make sure to avoid div-by-zero. */
149	PDMDevHlpPCIPhysRead(pDevIns,
150	pVirtio->aGCPhysQueueAvail[idxQueue]
151	+ RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[availIdx % cQueueItems]),
152	&uDescIdx, sizeof(uDescIdx));
153	return uDescIdx;
154	}
155
156	DECLINLINE(uint16_t) virtioReadAvailUsedEvent(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
157	{
158	uint16_t uUsedEventIdx;
159	/* VirtIO 1.0 uUsedEventIdx (used_event) immediately follows ring */
160	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
161	PDMDevHlpPCIPhysRead(pDevIns,
162	pVirtio->aGCPhysQueueAvail[idxQueue] + RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[pVirtio->uQueueSize[idxQueue]]),
163	&uUsedEventIdx, sizeof(uUsedEventIdx));
164	return uUsedEventIdx;
165	}
166	#endif
167
168	DECLINLINE(uint16_t) virtioReadAvailRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
169	{
170	uint16_t uIdx = 0;
171	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
172	PDMDevHlpPCIPhysRead(pDevIns,
173	pVirtio->aGCPhysQueueAvail[idxQueue] + RT_UOFFSETOF(VIRTQ_AVAIL_T, uIdx),
174	&uIdx, sizeof(uIdx));
175	return uIdx;
176	}
177
178	DECLINLINE(bool) virtqIsEmpty(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
179	{
180	uint16_t uAvailGuestIdx = virtioReadAvailRingIdx(pDevIns, pVirtio, idxQueue);
181	bool fEmpty = uAvailGuestIdx == pVirtio->virtqState[idxQueue].uAvailIdx;
182
183	Log6Func(("%s idx=%u, shadow idx=%u (%s)\n",
184	VIRTQNAME(pVirtio, idxQueue), uAvailGuestIdx, pVirtio->virtqState[idxQueue].uAvailIdx,
185	fEmpty ? "Queue empty" : "Queue has available descriptors"));
186	return fEmpty;
187	}
188
189	DECLINLINE(uint16_t) virtioReadAvailRingFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
190	{
191	uint16_t fFlags;
192	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
193	PDMDevHlpPCIPhysRead(pDevIns,
194	pVirtio->aGCPhysQueueAvail[idxQueue] + RT_UOFFSETOF(VIRTQ_AVAIL_T, fFlags),
195	&fFlags, sizeof(fFlags));
196	return fFlags;
197	}
198
199	/** @} */
200
201	/** @name Accessors for virtq used ring
202	* @{
203	*/
204
205	#ifdef IN_RING3
206	DECLINLINE(void) virtioWriteUsedElem(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue,
207	uint32_t usedIdx, uint32_t uDescIdx, uint32_t uLen)
208	{
209	VIRTQ_USED_ELEM_T elem = { uDescIdx, uLen };
210	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
211	uint16_t const cQueueItems = RT_MAX(pVirtio->uQueueSize[idxQueue], 1); /* Make sure to avoid div-by-zero. */
212	PDMDevHlpPCIPhysWrite(pDevIns,
213	pVirtio->aGCPhysQueueUsed[idxQueue] + RT_UOFFSETOF_DYN(VIRTQ_USED_T, aRing[usedIdx % cQueueItems]),
214	&elem, sizeof(elem));
215	}
216
217	DECLINLINE(void) virtioWriteUsedRingFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, uint16_t fFlags)
218	{
219	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
220	RT_UNTRUSTED_VALIDATED_FENCE(); /* VirtIO 1.0, Section 3.2.1.4.1 */
221	PDMDevHlpPCIPhysWrite(pDevIns,
222	pVirtio->aGCPhysQueueUsed[idxQueue] + RT_UOFFSETOF(VIRTQ_USED_T, fFlags),
223	&fFlags, sizeof(fFlags));
224	}
225	#endif
226
227	DECLINLINE(void) virtioWriteUsedRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, uint16_t uIdx)
228	{
229	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
230	PDMDevHlpPCIPhysWrite(pDevIns,
231	pVirtio->aGCPhysQueueUsed[idxQueue] + RT_UOFFSETOF(VIRTQ_USED_T, uIdx),
232	&uIdx, sizeof(uIdx));
233	}
234
235	#ifdef LOG_ENABLED
236	DECLINLINE(uint16_t) virtioReadUsedRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
237	{
238	uint16_t uIdx = 0;
239	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
240	PDMDevHlpPCIPhysRead(pDevIns,
241	pVirtio->aGCPhysQueueUsed[idxQueue] + RT_UOFFSETOF(VIRTQ_USED_T, uIdx),
242	&uIdx, sizeof(uIdx));
243	return uIdx;
244	}
245	#endif
246
247
248	#ifdef IN_RING3
249	DECLINLINE(uint16_t) virtioReadUsedRingFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
250	{
251	uint16_t fFlags = 0;
252	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
253	PDMDevHlpPCIPhysRead(pDevIns,
254	pVirtio->aGCPhysQueueUsed[idxQueue] + RT_UOFFSETOF(VIRTQ_USED_T, fFlags),
255	&fFlags, sizeof(fFlags));
256	return fFlags;
257	}
258
259	DECLINLINE(void) virtioWriteUsedAvailEvent(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, uint32_t uAvailEventIdx)
260	{
261	/** VirtIO 1.0 uAvailEventIdx (avail_event) immediately follows ring */
262	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
263	PDMDevHlpPCIPhysWrite(pDevIns,
264	pVirtio->aGCPhysQueueUsed[idxQueue] + RT_UOFFSETOF_DYN(VIRTQ_USED_T, aRing[pVirtio->uQueueSize[idxQueue]]),
265	&uAvailEventIdx, sizeof(uAvailEventIdx));
266	}
267	#endif
268
269	/** @} */
270
271	void virtioCoreSgBufInit(PVIRTIOSGBUF pGcSgBuf, PVIRTIOSGSEG paSegs, size_t cSegs)
272	{
273	AssertPtr(pGcSgBuf);
274	Assert( (cSegs > 0 && VALID_PTR(paSegs)) \|\| (!cSegs && !paSegs));
275	Assert(cSegs < (~(unsigned)0 >> 1));
276
277	pGcSgBuf->paSegs = paSegs;
278	pGcSgBuf->cSegs = (unsigned)cSegs;
279	pGcSgBuf->idxSeg = 0;
280	if (cSegs && paSegs)
281	{
282	pGcSgBuf->gcPhysCur = paSegs[0].gcPhys;
283	pGcSgBuf->cbSegLeft = paSegs[0].cbSeg;
284	}
285	else
286	{
287	pGcSgBuf->gcPhysCur = 0;
288	pGcSgBuf->cbSegLeft = 0;
289	}
290	}
291
292	static RTGCPHYS virtioCoreSgBufGet(PVIRTIOSGBUF pGcSgBuf, size_t *pcbData)
293	{
294	size_t cbData;
295	RTGCPHYS pGcBuf;
296
297	/* Check that the S/G buffer has memory left. */
298	if (RT_LIKELY(pGcSgBuf->idxSeg < pGcSgBuf->cSegs && pGcSgBuf->cbSegLeft))
299	{ /* likely */ }
300	else
301	{
302	*pcbData = 0;
303	return 0;
304	}
305
306	AssertMsg( pGcSgBuf->cbSegLeft <= 128 * _1M
307	&& (RTGCPHYS)pGcSgBuf->gcPhysCur >= (RTGCPHYS)pGcSgBuf->paSegs[pGcSgBuf->idxSeg].gcPhys
308	&& (RTGCPHYS)pGcSgBuf->gcPhysCur + pGcSgBuf->cbSegLeft <=
309	(RTGCPHYS)pGcSgBuf->paSegs[pGcSgBuf->idxSeg].gcPhys + pGcSgBuf->paSegs[pGcSgBuf->idxSeg].cbSeg,
310	("pGcSgBuf->idxSeg=%d pGcSgBuf->cSegs=%d pGcSgBuf->gcPhysCur=%p pGcSgBuf->cbSegLeft=%zd "
311	"pGcSgBuf->paSegs[%d].gcPhys=%p pGcSgBuf->paSegs[%d].cbSeg=%zd\n",
312	pGcSgBuf->idxSeg, pGcSgBuf->cSegs, pGcSgBuf->gcPhysCur, pGcSgBuf->cbSegLeft,
313	pGcSgBuf->idxSeg, pGcSgBuf->paSegs[pGcSgBuf->idxSeg].gcPhys, pGcSgBuf->idxSeg,
314	pGcSgBuf->paSegs[pGcSgBuf->idxSeg].cbSeg));
315
316	cbData = RT_MIN(*pcbData, pGcSgBuf->cbSegLeft);
317	pGcBuf = pGcSgBuf->gcPhysCur;
318	pGcSgBuf->cbSegLeft -= cbData;
319	if (!pGcSgBuf->cbSegLeft)
320	{
321	pGcSgBuf->idxSeg++;
322
323	if (pGcSgBuf->idxSeg < pGcSgBuf->cSegs)
324	{
325	pGcSgBuf->gcPhysCur = pGcSgBuf->paSegs[pGcSgBuf->idxSeg].gcPhys;
326	pGcSgBuf->cbSegLeft = pGcSgBuf->paSegs[pGcSgBuf->idxSeg].cbSeg;
327	}
328	*pcbData = cbData;
329	}
330	else
331	pGcSgBuf->gcPhysCur = pGcSgBuf->gcPhysCur + cbData;
332
333	return pGcBuf;
334	}
335
336	void virtioCoreSgBufReset(PVIRTIOSGBUF pGcSgBuf)
337	{
338	AssertPtrReturnVoid(pGcSgBuf);
339
340	pGcSgBuf->idxSeg = 0;
341	if (pGcSgBuf->cSegs)
342	{
343	pGcSgBuf->gcPhysCur = pGcSgBuf->paSegs[0].gcPhys;
344	pGcSgBuf->cbSegLeft = pGcSgBuf->paSegs[0].cbSeg;
345	}
346	else
347	{
348	pGcSgBuf->gcPhysCur = 0;
349	pGcSgBuf->cbSegLeft = 0;
350	}
351	}
352
353	RTGCPHYS virtioCoreSgBufAdvance(PVIRTIOSGBUF pGcSgBuf, size_t cbAdvance)
354	{
355	AssertReturn(pGcSgBuf, 0);
356
357	size_t cbLeft = cbAdvance;
358	while (cbLeft)
359	{
360	size_t cbThisAdvance = cbLeft;
361	virtioCoreSgBufGet(pGcSgBuf, &cbThisAdvance);
362	if (!cbThisAdvance)
363	break;
364
365	cbLeft -= cbThisAdvance;
366	}
367	return cbAdvance - cbLeft;
368	}
369
370	RTGCPHYS virtioCoreSgBufGetNextSegment(PVIRTIOSGBUF pGcSgBuf, size_t *pcbSeg)
371	{
372	AssertReturn(pGcSgBuf, 0);
373	AssertPtrReturn(pcbSeg, 0);
374
375	if (!*pcbSeg)
376	*pcbSeg = pGcSgBuf->cbSegLeft;
377
378	return virtioCoreSgBufGet(pGcSgBuf, pcbSeg);
379	}
380
381	size_t virtioCoreSgBufCalcTotalLength(PVIRTIOSGBUF pGcSgBuf)
382	{
383	size_t cb = 0;
384	unsigned i = pGcSgBuf->cSegs;
385	while (i-- > 0)
386	cb += pGcSgBuf->paSegs[i].cbSeg;
387	return cb;
388	}
389
390	#ifdef LOG_ENABLED
391
392	void virtioPrintFeatures(VIRTIOCORE *pVirtio)
393	{
394	#ifdef LOG_ENABLED
395	static struct
396	{
397	uint64_t fFeatureBit;
398	const char *pcszDesc;
399	} const s_aFeatures[] =
400	{
401	{ VIRTIO_F_RING_INDIRECT_DESC, " RING_INDIRECT_DESC Driver can use descriptors with VIRTQ_DESC_F_INDIRECT flag set\n" },
402	{ VIRTIO_F_RING_EVENT_IDX, " RING_EVENT_IDX Enables use_event and avail_event fields described in 2.4.7, 2.4.8\n" },
403	{ VIRTIO_F_VERSION_1, " VERSION Used to detect legacy drivers.\n" },
404	};
405
406	#define MAXLINE 80
407	/* Display as a single buf to prevent interceding log messages */
408	uint16_t cbBuf = RT_ELEMENTS(s_aFeatures) * 132;
409	char pszBuf = (char )RTMemAllocZ(cbBuf);
410	Assert(pszBuf);
411	char *cp = pszBuf;
412	for (unsigned i = 0; i < RT_ELEMENTS(s_aFeatures); ++i)
413	{
414	bool isOffered = RT_BOOL(pVirtio->uDeviceFeatures & s_aFeatures[i].fFeatureBit);
415	bool isNegotiated = RT_BOOL(pVirtio->uDriverFeatures & s_aFeatures[i].fFeatureBit);
416	cp += RTStrPrintf(cp, cbBuf - (cp - pszBuf), " %s %s %s",
417	isOffered ? "+" : "-", isNegotiated ? "x" : " ", s_aFeatures[i].pcszDesc);
418	}
419	Log3(("VirtIO Features Configuration\n\n"
420	" Offered Accepted Feature Description\n"
421	" ------- -------- ------- -----------\n"
422	"%s\n", pszBuf));
423	RTMemFree(pszBuf);
424
425	#else /* !LOG_ENABLED */
426	RT_NOREF3(pThis, fFeatures, pcszText);
427	#endif /* !LOG_ENABLED */
428	}
429
430
431	/**
432	* Does a formatted hex dump using Log(()), recommend using VIRTIO_HEX_DUMP() macro to
433	* control enabling of logging efficiently.
434	*
435	* @param pv pointer to buffer to dump contents of
436	* @param cb count of characters to dump from buffer
437	* @param uBase base address of per-row address prefixing of hex output
438	* @param pszTitle Optional title. If present displays title that lists
439	* provided text with value of cb to indicate size next to it.
440	*/
441	void virtioCoreHexDump(uint8_t pv, uint32_t cb, uint32_t uBase, const char pszTitle)
442	{
443	if (pszTitle)
444	Log(("%s [%d bytes]:\n", pszTitle, cb));
445	for (uint32_t row = 0; row < RT_MAX(1, (cb / 16) + 1) && row * 16 < cb; row++)
446	{
447	Log(("%04x: ", row * 16 + uBase)); /* line address */
448	for (uint8_t col = 0; col < 16; col++)
449	{
450	uint32_t idx = row * 16 + col;
451	if (idx >= cb)
452	Log(("-- %s", (col + 1) % 8 ? "" : " "));
453	else
454	Log(("%02x %s", pv[idx], (col + 1) % 8 ? "" : " "));
455	}
456	for (uint32_t idx = row * 16; idx < row * 16 + 16; idx++)
457	Log(("%c", (idx >= cb) ? ' ' : (pv[idx] >= 0x20 && pv[idx] <= 0x7e ? pv[idx] : '.')));
458	Log(("\n"));
459	}
460	Log(("\n"));
461	RT_NOREF2(uBase, pv);
462	}
463
464	/**
465	* Do a hex dump of memory in guest physical context
466	*
467	* @param gcPhys pointer to buffer to dump contents of
468	* @param cb count of characters to dump from buffer
469	* @param uBase base address of per-row address prefixing of hex output
470	* @param pszTitle Optional title. If present displays title that lists
471	* provided text with value of cb to indicate size next to it.
472	*/
473	void virtioCoreGcPhysHexDump(PPDMDEVINS pDevIns, RTGCPHYS gcPhys, uint32_t cb, uint32_t uBase, const char *pszTitle)
474	{
475	if (pszTitle)
476	Log(("%s [%d bytes]:\n", pszTitle, cb));
477	for (uint32_t row = 0; row < RT_MAX(1, (cb / 16) + 1) && row * 16 < cb; row++)
478	{
479	uint8_t c;
480	Log(("%04x: ", row * 16 + uBase)); /* line address */
481	for (uint8_t col = 0; col < 16; col++)
482	{
483	uint32_t idx = row * 16 + col;
484	PDMDevHlpPCIPhysRead(pDevIns, gcPhys + idx, &c, 1);
485	if (idx >= cb)
486	Log(("-- %s", (col + 1) % 8 ? "" : " "));
487	else
488	Log(("%02x %s", c, (col + 1) % 8 ? "" : " "));
489	}
490	for (uint32_t idx = row * 16; idx < row * 16 + 16; idx++)
491	{
492	PDMDevHlpPCIPhysRead(pDevIns, gcPhys + idx, &c, 1);
493	Log(("%c", (idx >= cb) ? ' ' : (c >= 0x20 && c <= 0x7e ? c : '.')));
494	}
495	Log(("\n"));
496	}
497	Log(("\n"));
498	RT_NOREF(uBase);
499	}
500	#endif /* LOG_ENABLED */
501
502	/**
503	* Log memory-mapped I/O input or output value.
504	*
505	* This is designed to be invoked by macros that can make contextual assumptions
506	* (e.g. implicitly derive MACRO parameters from the invoking function). It is exposed
507	* for the VirtIO client doing the device-specific implementation in order to log in a
508	* similar fashion accesses to the device-specific MMIO configuration structure. Macros
509	* that leverage this function are found in virtioCommonCfgAccessed() and can be
510	* used as an example of how to use this effectively for the device-specific
511	* code.
512	*
513	* @param pszFunc To avoid displaying this function's name via __FUNCTION__ or LogFunc()
514	* @param pszMember Name of struct member
515	* @param pv pointer to value
516	* @param cb size of value
517	* @param uOffset offset into member where value starts
518	* @param fWrite True if write I/O
519	* @param fHasIndex True if the member is indexed
520	* @param idx The index if fHasIndex
521	*/
522	void virtioCoreLogMappedIoValue(const char pszFunc, const char pszMember, uint32_t uMemberSize,
523	const void *pv, uint32_t cb, uint32_t uOffset, int fWrite,
524	int fHasIndex, uint32_t idx)
525	{
526	if (!LogIs6Enabled())
527	return;
528
529	char szIdx[16];
530	if (fHasIndex)
531	RTStrPrintf(szIdx, sizeof(szIdx), "[%d]", idx);
532	else
533	szIdx[0] = '\0';
534
535	if (cb == 1 \|\| cb == 2 \|\| cb == 4 \|\| cb == 8)
536	{
537	char szDepiction[64];
538	size_t cchDepiction;
539	if (uOffset != 0 \|\| cb != uMemberSize) /* display bounds if partial member access */
540	cchDepiction = RTStrPrintf(szDepiction, sizeof(szDepiction), "%s%s[%d:%d]",
541	pszMember, szIdx, uOffset, uOffset + cb - 1);
542	else
543	cchDepiction = RTStrPrintf(szDepiction, sizeof(szDepiction), "%s%s", pszMember, szIdx);
544
545	/* padding */
546	if (cchDepiction < 30)
547	szDepiction[cchDepiction++] = ' ';
548	while (cchDepiction < 30)
549	szDepiction[cchDepiction++] = '.';
550	szDepiction[cchDepiction] = '\0';
551
552	RTUINT64U uValue;
553	uValue.u = 0;
554	memcpy(uValue.au8, pv, cb);
555	Log6(("%s: Guest %s %s %#0*RX64\n",
556	pszFunc, fWrite ? "wrote" : "read ", szDepiction, 2 + cb * 2, uValue.u));
557	}
558	else /* odd number or oversized access, ... log inline hex-dump style */
559	{
560	Log6(("%s: Guest %s %s%s[%d:%d]: %.*Rhxs\n",
561	pszFunc, fWrite ? "wrote" : "read ", pszMember,
562	szIdx, uOffset, uOffset + cb, cb, pv));
563	}
564	RT_NOREF2(fWrite, pszFunc);
565	}
566
567
568	/**
569	* Makes the MMIO-mapped Virtio uDeviceStatus registers non-cryptic
570	*/
571	DECLINLINE(void) virtioLogDeviceStatus(uint8_t bStatus)
572	{
573	if (bStatus == 0)
574	Log6(("RESET"));
575	else
576	{
577	int primed = 0;
578	if (bStatus & VIRTIO_STATUS_ACKNOWLEDGE)
579	Log6(("%sACKNOWLEDGE", primed++ ? "" : ""));
580	if (bStatus & VIRTIO_STATUS_DRIVER)
581	Log6(("%sDRIVER", primed++ ? " \| " : ""));
582	if (bStatus & VIRTIO_STATUS_FEATURES_OK)
583	Log6(("%sFEATURES_OK", primed++ ? " \| " : ""));
584	if (bStatus & VIRTIO_STATUS_DRIVER_OK)
585	Log6(("%sDRIVER_OK", primed++ ? " \| " : ""));
586	if (bStatus & VIRTIO_STATUS_FAILED)
587	Log6(("%sFAILED", primed++ ? " \| " : ""));
588	if (bStatus & VIRTIO_STATUS_DEVICE_NEEDS_RESET)
589	Log6(("%sNEEDS_RESET", primed++ ? " \| " : ""));
590	(void)primed;
591	}
592	}
593
594	#ifdef IN_RING3
595	/**
596	* Allocate client context for client to work with VirtIO-provided with queue
597	*
598	* @param pVirtio Pointer to the shared virtio state.
599	* @param idxQueue Queue number
600	* @param pcszName Name to give queue
601	*
602	* @returns VBox status code.
603	*/
604	int virtioCoreR3QueueAttach(PVIRTIOCORE pVirtio, uint16_t idxQueue, const char *pcszName)
605	{
606	LogFunc(("%s\n", pcszName));
607	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
608	pVirtq->uAvailIdx = 0;
609	pVirtq->uUsedIdx = 0;
610	pVirtq->fEventThresholdReached = false;
611	RTStrCopy(pVirtq->szVirtqName, sizeof(pVirtq->szVirtqName), pcszName);
612	return VINF_SUCCESS;
613	}
614	#endif /* IN_RING3 */
615
616
617	/**
618	* Check if the associated queue is empty
619	*
620	* @param pDevIns The device instance (for reading).
621	* @param pVirtio Pointer to the shared virtio state.
622	* @param idxQueue Queue number
623	*
624	* @retval true Queue is empty or unavailable.
625	* @retval false Queue is available and has entries
626	*/
627	bool virtioCoreQueueIsEmpty(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
628	{
629	if (pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK)
630	return virtqIsEmpty(pDevIns, pVirtio, idxQueue);
631	LogFunc(("VirtIO not ready: Returning 'true' for queue empty\n"));
632	return true;
633	}
634
635	#ifdef IN_RING3
636
637
638	int virtioCoreR3DescChainGet(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue,
639	uint16_t uHeadIdx, PPVIRTIO_DESC_CHAIN_T ppDescChain)
640	{
641	AssertReturn(ppDescChain, VERR_INVALID_POINTER);
642	*ppDescChain = NULL;
643
644	Assert(idxQueue < RT_ELEMENTS(pVirtio->virtqState));
645
646	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
647
648	AssertMsgReturn(IS_DRIVER_OK(pVirtio) && pVirtio->uQueueEnable[idxQueue],
649	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
650
651	uint16_t uDescIdx = uHeadIdx;
652
653	Log6Func(("%s DESC CHAIN: (head) desc_idx=%u\n", pVirtq->szVirtqName, uHeadIdx));
654	RT_NOREF(pVirtq);
655
656	/*
657	* Allocate and initialize the descriptor chain structure.
658	*/
659	PVIRTIO_DESC_CHAIN_T pDescChain = (PVIRTIO_DESC_CHAIN_T)RTMemAllocZ(sizeof(VIRTIO_DESC_CHAIN_T));
660	AssertReturn(pDescChain, VERR_NO_MEMORY);
661	pDescChain->u32Magic = VIRTIO_DESC_CHAIN_MAGIC;
662	pDescChain->cRefs = 1;
663	pDescChain->uHeadIdx = uHeadIdx;
664	*ppDescChain = pDescChain;
665
666	/*
667	* Gather segments.
668	*/
669	VIRTQ_DESC_T desc;
670
671	uint32_t cbIn = 0;
672	uint32_t cbOut = 0;
673	uint32_t cSegsIn = 0;
674	uint32_t cSegsOut = 0;
675	PVIRTIOSGSEG paSegsIn = pDescChain->aSegsIn;
676	PVIRTIOSGSEG paSegsOut = pDescChain->aSegsOut;
677
678	do
679	{
680	PVIRTIOSGSEG pSeg;
681
682	/*
683	* Malicious guests may go beyond paSegsIn or paSegsOut boundaries by linking
684	* several descriptors into a loop. Since there is no legitimate way to get a sequences of
685	* linked descriptors exceeding the total number of descriptors in the ring (see @bugref{8620}),
686	* the following aborts I/O if breach and employs a simple log throttling algorithm to notify.
687	*/
688	if (cSegsIn + cSegsOut >= VIRTQ_MAX_SIZE)
689	{
690	static volatile uint32_t s_cMessages = 0;
691	static volatile uint32_t s_cThreshold = 1;
692	if (ASMAtomicIncU32(&s_cMessages) == ASMAtomicReadU32(&s_cThreshold))
693	{
694	LogRelMax(64, ("Too many linked descriptors; check if the guest arranges descriptors in a loop.\n"));
695	if (ASMAtomicReadU32(&s_cMessages) != 1)
696	LogRelMax(64, ("(the above error has occured %u times so far)\n", ASMAtomicReadU32(&s_cMessages)));
697	ASMAtomicWriteU32(&s_cThreshold, ASMAtomicReadU32(&s_cThreshold) * 10);
698	}
699	break;
700	}
701	RT_UNTRUSTED_VALIDATED_FENCE();
702
703	virtioReadDesc(pDevIns, pVirtio, idxQueue, uDescIdx, &desc);
704
705	if (desc.fFlags & VIRTQ_DESC_F_WRITE)
706	{
707	Log6Func(("%s IN desc_idx=%u seg=%u addr=%RGp cb=%u\n", VIRTQNAME(pVirtio, idxQueue), uDescIdx, cSegsIn, desc.GCPhysBuf, desc.cb));
708	cbIn += desc.cb;
709	pSeg = &paSegsIn[cSegsIn++];
710	}
711	else
712	{
713	Log6Func(("%s OUT desc_idx=%u seg=%u addr=%RGp cb=%u\n", VIRTQNAME(pVirtio, idxQueue), uDescIdx, cSegsOut, desc.GCPhysBuf, desc.cb));
714	cbOut += desc.cb;
715	pSeg = &paSegsOut[cSegsOut++];
716	}
717
718	pSeg->gcPhys = desc.GCPhysBuf;
719	pSeg->cbSeg = desc.cb;
720
721	uDescIdx = desc.uDescIdxNext;
722	} while (desc.fFlags & VIRTQ_DESC_F_NEXT);
723
724	/*
725	* Add segments to the descriptor chain structure.
726	*/
727	if (cSegsIn)
728	{
729	virtioCoreSgBufInit(&pDescChain->SgBufIn, paSegsIn, cSegsIn);
730	pDescChain->pSgPhysReturn = &pDescChain->SgBufIn;
731	pDescChain->cbPhysReturn = cbIn;
732	STAM_REL_COUNTER_ADD(&pVirtio->StatDescChainsSegsIn, cSegsIn);
733	}
734
735	if (cSegsOut)
736	{
737	virtioCoreSgBufInit(&pDescChain->SgBufOut, paSegsOut, cSegsOut);
738	pDescChain->pSgPhysSend = &pDescChain->SgBufOut;
739	pDescChain->cbPhysSend = cbOut;
740	STAM_REL_COUNTER_ADD(&pVirtio->StatDescChainsSegsOut, cSegsOut);
741	}
742
743	STAM_REL_COUNTER_INC(&pVirtio->StatDescChainsAllocated);
744	Log6Func(("%s -- segs OUT: %u (%u bytes) IN: %u (%u bytes) --\n", pVirtq->szVirtqName, cSegsOut, cbOut, cSegsIn, cbIn));
745
746	return VINF_SUCCESS;
747	}
748
749
750	/**
751	* Retains a reference to the given descriptor chain.
752	*
753	* @returns New reference count.
754	* @retval UINT32_MAX on invalid parameter.
755	* @param pDescChain The descriptor chain to reference.
756	*/
757	uint32_t virtioCoreR3DescChainRetain(PVIRTIO_DESC_CHAIN_T pDescChain)
758	{
759	AssertReturn(pDescChain, UINT32_MAX);
760	AssertReturn(pDescChain->u32Magic == VIRTIO_DESC_CHAIN_MAGIC, UINT32_MAX);
761	uint32_t cRefs = ASMAtomicIncU32(&pDescChain->cRefs);
762	Assert(cRefs > 1);
763	Assert(cRefs < 16);
764	return cRefs;
765	}
766
767
768	/**
769	* Releases a reference to the given descriptor chain.
770	*
771	* @returns New reference count.
772	* @retval 0 if freed or invalid parameter.
773	* @param pVirtio Pointer to the shared virtio state.
774	* @param pDescChain The descriptor chain to reference. NULL is quietly
775	* ignored (returns 0).
776	*/
777	uint32_t virtioCoreR3DescChainRelease(PVIRTIOCORE pVirtio, PVIRTIO_DESC_CHAIN_T pDescChain)
778	{
779	if (!pDescChain)
780	return 0;
781	AssertReturn(pDescChain, 0);
782	AssertReturn(pDescChain->u32Magic == VIRTIO_DESC_CHAIN_MAGIC, 0);
783	uint32_t cRefs = ASMAtomicDecU32(&pDescChain->cRefs);
784	Assert(cRefs < 16);
785	if (cRefs == 0)
786	{
787	pDescChain->u32Magic = ~VIRTIO_DESC_CHAIN_MAGIC;
788	RTMemFree(pDescChain);
789	STAM_REL_COUNTER_INC(&pVirtio->StatDescChainsFreed);
790	}
791	return cRefs;
792	}
793
794
795	/*
796	* Notifies guest (via ISR or MSI-X) of device configuration change
797	*
798	* @param pVirtio Pointer to the shared virtio state.
799	*/
800	void virtioCoreNotifyConfigChanged(PVIRTIOCORE pVirtio)
801	{
802	virtioKick(pVirtio->pDevInsR3, pVirtio, VIRTIO_ISR_DEVICE_CONFIG, pVirtio->uMsixConfig, false);
803	}
804
805	/**
806	* Enable or Disable notification for the specified queue
807	*
808	* @param pVirtio Pointer to the shared virtio state.
809	* @param idxQueue Queue number
810	* @param fEnabled Selects notification mode (enabled or disabled)
811	*/
812	void virtioCoreQueueSetNotify(PVIRTIOCORE pVirtio, uint16_t idxQueue, bool fEnabled)
813	{
814	if (pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK)
815	{
816	uint16_t fFlags = virtioReadUsedRingFlags(pVirtio->pDevInsR3, pVirtio, idxQueue);
817
818	if (fEnabled)
819	fFlags &= ~ VIRTQ_USED_F_NO_NOTIFY;
820	else
821	fFlags \|= VIRTQ_USED_F_NO_NOTIFY;
822
823	virtioWriteUsedRingFlags(pVirtio->pDevInsR3, pVirtio, idxQueue, fFlags);
824	}
825	}
826
827	/**
828	* Initiate orderly reset procedure. This is an exposed API for clients that might need it.
829	* Invoked by client to reset the device and driver (see VirtIO 1.0 section 2.1.1/2.1.2)
830	*
831	* @param pVirtio Pointer to the virtio state.
832	*/
833	void virtioCoreResetAll(PVIRTIOCORE pVirtio)
834	{
835	LogFunc(("\n"));
836	pVirtio->uDeviceStatus \|= VIRTIO_STATUS_DEVICE_NEEDS_RESET;
837	if (pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK)
838	{
839	pVirtio->fGenUpdatePending = true;
840	virtioKick(pVirtio->pDevInsR3, pVirtio, VIRTIO_ISR_DEVICE_CONFIG, pVirtio->uMsixConfig, false /* fForce */);
841	}
842	}
843	/**
844	* Get count of new (e.g. pending) elements in available ring.
845	*
846	* @param pDevIns The device instance.
847	* @param pVirtio Pointer to the shared virtio state.
848	* @param idxQueue Queue number
849	*
850	* @returns how many entries have been added to ring as a delta of the consumer's
851	* avail index and the queue's guest-side current avail index.
852	*/
853	int virtioCoreR3QueuePendingCount(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
854	{
855	uint16_t uAvailRingIdx = virtioReadAvailRingIdx(pDevIns, pVirtio, idxQueue);
856	uint16_t uNextAvailIdx = pVirtio->virtqState[idxQueue].uAvailIdx;
857	uint16_t uDelta = uAvailRingIdx - uNextAvailIdx;
858	if (uAvailRingIdx > uNextAvailIdx)
859	return uDelta;
860	return VIRTQ_MAX_CNT + uDelta;
861	}
862	/**
863	* Fetches descriptor chain using avail ring of indicated queue and converts the descriptor
864	* chain into its OUT (to device) and IN to guest components, but does NOT remove it from
865	* the 'avail' queue. I.e. doesn't advance the index. This can be used with virtioQueueSkip(),
866	* which does advance the avail index. Together they facilitate a mechanism that allows
867	* work with a queue element (descriptor chain) to be aborted if necessary, by not advancing
868	* the pointer, or, upon success calling the skip function (above) to move to the next element.
869	*
870	* Additionally it converts the OUT desc chain data to a contiguous virtual
871	* memory buffer for easy consumption by the caller. The caller must return the
872	* descriptor chain pointer via virtioCoreR3QueuePut() and then call virtioCoreQueueSync()
873	* at some point to return the data to the guest and complete the transaction.
874	*
875	* @param pDevIns The device instance.
876	* @param pVirtio Pointer to the shared virtio state.
877	* @param idxQueue Queue number
878	* @param ppDescChain Address to store pointer to descriptor chain that contains the
879	* pre-processed transaction information pulled from the virtq.
880	*
881	* @returns VBox status code:
882	* @retval VINF_SUCCESS Success
883	* @retval VERR_INVALID_STATE VirtIO not in ready state (asserted).
884	* @retval VERR_NOT_AVAILABLE If the queue is empty.
885	*/
886
887	int virtioCoreR3QueuePeek(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue,
888	PPVIRTIO_DESC_CHAIN_T ppDescChain)
889	{
890	return virtioCoreR3QueueGet(pDevIns, pVirtio, idxQueue, ppDescChain, false);
891	}
892
893	/**
894	* Skip the next entry in the specified queue (typically used with virtioCoreR3QueuePeek())
895	*
896	* @param pVirtio Pointer to the virtio state.
897	* @param idxQueue Index of queue
898	*/
899	int virtioCoreR3QueueSkip(PVIRTIOCORE pVirtio, uint16_t idxQueue)
900	{
901	Assert(idxQueue < RT_ELEMENTS(pVirtio->virtqState));
902	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
903
904	AssertMsgReturn(IS_DRIVER_OK(pVirtio) && pVirtio->uQueueEnable[idxQueue],
905	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
906
907	if (virtioCoreQueueIsEmpty(pVirtio->pDevInsR3, pVirtio, idxQueue))
908	return VERR_NOT_AVAILABLE;
909
910	Log2Func(("%s avail_idx=%u\n", pVirtq->szVirtqName, pVirtq->uAvailIdx));
911	pVirtq->uAvailIdx++;
912
913	return VINF_SUCCESS;
914	}
915
916	/**
917	* Fetches descriptor chain using avail ring of indicated queue and converts the descriptor
918	* chain into its OUT (to device) and IN to guest components.
919	*
920	* Additionally it converts the OUT desc chain data to a contiguous virtual
921	* memory buffer for easy consumption by the caller. The caller must return the
922	* descriptor chain pointer via virtioCoreR3QueuePut() and then call virtioCoreQueueSync()
923	* at some point to return the data to the guest and complete the transaction.
924	*
925	* @param pDevIns The device instance.
926	* @param pVirtio Pointer to the shared virtio state.
927	* @param idxQueue Queue number
928	* @param ppDescChain Address to store pointer to descriptor chain that contains the
929	* pre-processed transaction information pulled from the virtq.
930	* Returned reference must be released by calling
931	* virtioCoreR3DescChainRelease().
932	* @param fRemove flags whether to remove desc chain from queue (false = peek)
933	*
934	* @returns VBox status code:
935	* @retval VINF_SUCCESS Success
936	* @retval VERR_INVALID_STATE VirtIO not in ready state (asserted).
937	* @retval VERR_NOT_AVAILABLE If the queue is empty.
938	*/
939	int virtioCoreR3QueueGet(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue,
940	PPVIRTIO_DESC_CHAIN_T ppDescChain, bool fRemove)
941	{
942	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
943
944	if (virtqIsEmpty(pDevIns, pVirtio, idxQueue))
945	return VERR_NOT_AVAILABLE;
946
947	uint16_t uHeadIdx = virtioReadAvailDescIdx(pDevIns, pVirtio, idxQueue, pVirtq->uAvailIdx);
948
949	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
950	virtioWriteUsedAvailEvent(pDevIns,pVirtio, idxQueue, pVirtq->uAvailIdx + 1);
951
952	if (fRemove)
953	pVirtq->uAvailIdx++;
954
955	int rc = virtioCoreR3DescChainGet(pDevIns, pVirtio, idxQueue, uHeadIdx, ppDescChain);
956	return rc;
957	}
958
959	/**
960	* Returns data to the guest to complete a transaction initiated by virtQueueGet().
961	*
962	* The caller passes in a pointer to a scatter-gather buffer of virtual memory segments
963	* and a pointer to the descriptor chain context originally derived from the pulled
964	* queue entry, and this function will write the virtual memory s/g buffer into the
965	* guest's physical memory free the descriptor chain. The caller handles the freeing
966	* (as needed) of the virtual memory buffer.
967	*
968	* @note This does a write-ahead to the used ring of the guest's queue. The data
969	* written won't be seen by the guest until the next call to virtioCoreQueueSync()
970	*
971	*
972	* @param pDevIns The device instance (for reading).
973	* @param pVirtio Pointer to the shared virtio state.
974	* @param idxQueue Queue number
975	*
976	* @param pSgVirtReturn Points to scatter-gather buffer of virtual memory
977	* segments the caller is returning to the guest.
978	*
979	* @param pDescChain This contains the context of the scatter-gather
980	* buffer originally pulled from the queue.
981	*
982	* @param fFence If true, put up copy fence (memory barrier) after
983	* copying to guest phys. mem.
984	*
985	* @returns VBox status code.
986	* @retval VINF_SUCCESS Success
987	* @retval VERR_INVALID_STATE VirtIO not in ready state
988	* @retval VERR_NOT_AVAILABLE Queue is empty
989	*
990	* @note This function will not release any reference to pDescChain. The
991	* caller must take care of that.
992	*/
993	int virtioCoreR3QueuePut(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, PRTSGBUF pSgVirtReturn,
994	PVIRTIO_DESC_CHAIN_T pDescChain, bool fFence)
995	{
996	Assert(idxQueue < RT_ELEMENTS(pVirtio->virtqState));
997	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
998	PVIRTIOSGBUF pSgPhysReturn = pDescChain->pSgPhysReturn;
999
1000	Assert(pDescChain->u32Magic == VIRTIO_DESC_CHAIN_MAGIC);
1001	Assert(pDescChain->cRefs > 0);
1002
1003	AssertMsgReturn(IS_DRIVER_OK(pVirtio), ("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
1004
1005	Log6Func(("Copying client data to %s, desc chain (head desc_idx %d)\n",
1006	VIRTQNAME(pVirtio, idxQueue), virtioReadUsedRingIdx(pDevIns, pVirtio, idxQueue)));
1007
1008	/* Copy s/g buf (virtual memory) to guest phys mem (IN direction). */
1009
1010	size_t cbCopy = 0, cbTotal = 0, cbRemain = 0;
1011
1012	if (pSgVirtReturn)
1013	{
1014	size_t cbTarget = virtioCoreSgBufCalcTotalLength(pSgPhysReturn);
1015	cbRemain = cbTotal = RTSgBufCalcTotalLength(pSgVirtReturn);
1016	AssertMsgReturn(cbTarget >= cbRemain, ("No space to write data to phys memory"), VERR_BUFFER_OVERFLOW);
1017	virtioCoreSgBufReset(pSgPhysReturn); /* Reset ptr because req data may have already been written */
1018	while (cbRemain)
1019	{
1020	cbCopy = RT_MIN(pSgVirtReturn->cbSegLeft, pSgPhysReturn->cbSegLeft);
1021	Assert(cbCopy > 0);
1022	PDMDevHlpPhysWrite(pDevIns, (RTGCPHYS)pSgPhysReturn->gcPhysCur, pSgVirtReturn->pvSegCur, cbCopy);
1023	RTSgBufAdvance(pSgVirtReturn, cbCopy);
1024	virtioCoreSgBufAdvance(pSgPhysReturn, cbCopy);
1025	cbRemain -= cbCopy;
1026	}
1027
1028	if (fFence)
1029	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE(); /* needed? */
1030
1031	Assert(!(cbCopy >> 32));
1032	}
1033
1034	/* If this write-ahead crosses threshold where the driver wants to get an event flag it */
1035	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
1036	if (pVirtq->uUsedIdx == virtioReadAvailUsedEvent(pDevIns, pVirtio, idxQueue))
1037	pVirtq->fEventThresholdReached = true;
1038
1039	/*
1040	* Place used buffer's descriptor in used ring but don't update used ring's slot index.
1041	* That will be done with a subsequent client call to virtioCoreQueueSync() */
1042	virtioWriteUsedElem(pDevIns, pVirtio, idxQueue, pVirtq->uUsedIdx++, pDescChain->uHeadIdx, (uint32_t)cbTotal);
1043
1044	if (pSgVirtReturn)
1045	Log6Func((".... Copied %zu bytes in %d segs to %u byte buffer, residual=%zu\n",
1046	cbTotal - cbRemain, pSgVirtReturn->cSegs, pDescChain->cbPhysReturn, pDescChain->cbPhysReturn - cbTotal));
1047
1048	Log6Func(("Write ahead used_idx=%u, %s used_idx=%u\n",
1049	pVirtq->uUsedIdx, VIRTQNAME(pVirtio, idxQueue), virtioReadUsedRingIdx(pDevIns, pVirtio, idxQueue)));
1050
1051	return VINF_SUCCESS;
1052	}
1053
1054	#endif /* IN_RING3 */
1055
1056	/**
1057	* Updates the indicated virtq's "used ring" descriptor index to match the
1058	* current write-head index, thus exposing the data added to the used ring by all
1059	* virtioCoreR3QueuePut() calls since the last sync. This should be called after one or
1060	* more virtioCoreR3QueuePut() calls to inform the guest driver there is data in the queue.
1061	* Explicit notifications (e.g. interrupt or MSI-X) will be sent to the guest,
1062	* depending on VirtIO features negotiated and conditions, otherwise the guest
1063	* will detect the update by polling. (see VirtIO 1.0
1064	* specification, Section 2.4 "Virtqueues").
1065	*
1066	* @param pDevIns The device instance.
1067	* @param pVirtio Pointer to the shared virtio state.
1068	* @param idxQueue Queue number
1069	* @param fForce Force guest notification even if VIRTQ_USED_F_NO_NOTIFY is set
1070	*
1071	* @returns VBox status code.
1072	* @retval VINF_SUCCESS Success
1073	* @retval VERR_INVALID_STATE VirtIO not in ready state
1074	*/
1075	int virtioCoreQueueSync(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, bool fForce)
1076	{
1077	Assert(idxQueue < RT_ELEMENTS(pVirtio->virtqState));
1078	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
1079
1080	AssertMsgReturn(IS_DRIVER_OK(pVirtio) && pVirtio->uQueueEnable[idxQueue],
1081	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
1082
1083	Log6Func(("Updating %s used_idx from %u to %u\n",
1084	VIRTQNAME(pVirtio, idxQueue), virtioReadUsedRingIdx(pDevIns, pVirtio, idxQueue), pVirtq->uUsedIdx));
1085
1086	virtioWriteUsedRingIdx(pDevIns, pVirtio, idxQueue, pVirtq->uUsedIdx);
1087	virtioNotifyGuestDriver(pDevIns, pVirtio, idxQueue, fForce);
1088
1089	return VINF_SUCCESS;
1090	}
1091
1092	/**
1093	*/
1094	static void virtioQueueNotified(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, uint16_t uNotifyIdx)
1095	{
1096
1097	PVIRTIOCORECC pVirtioCC = PDMDEVINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1098
1099	/* See VirtIO 1.0, section 4.1.5.2 It implies that idxQueue and uNotifyIdx should match.
1100	* Disregarding this notification may cause throughput to stop, however there's no way to know
1101	* which was queue was intended for wake-up if the two parameters disagree. */
1102
1103	AssertMsg(uNotifyIdx == idxQueue,
1104	("Guest kicked virtq %d's notify addr w/non-corresponding virtq idx %d\n",
1105	idxQueue, uNotifyIdx));
1106	RT_NOREF(uNotifyIdx);
1107
1108	AssertReturnVoid(idxQueue < RT_ELEMENTS(pVirtio->virtqState));
1109	Log6Func(("%s\n", pVirtio->virtqState[idxQueue].szVirtqName));
1110
1111	/* Inform client */
1112	pVirtioCC->pfnQueueNotified(pDevIns, pVirtio, idxQueue);
1113	}
1114
1115	/**
1116	* Trigger MSI-X or INT# interrupt to notify guest of data added to used ring of
1117	* the specified virtq, depending on the interrupt configuration of the device
1118	* and depending on negotiated and realtime constraints flagged by the guest driver.
1119	*
1120	* See VirtIO 1.0 specification (section 2.4.7).
1121	*
1122	* @param pDevIns The device instance.
1123	* @param pVirtio Pointer to the shared virtio state.
1124	* @param idxQueue Queue to check for guest interrupt handling preference
1125	* @param fForce Overrides idxQueue, forcing notification regardless of driver's
1126	* notification preferences. This is a safeguard to prevent
1127	* stalls upon resuming the VM. VirtIO 1.0 specification Section 4.1.5.5
1128	* indicates spurious interrupts are harmless to guest driver's state,
1129	* as they only cause the guest driver to [re]scan queues for work to do.
1130	*/
1131	static void virtioNotifyGuestDriver(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, bool fForce)
1132	{
1133
1134	Assert(idxQueue < RT_ELEMENTS(pVirtio->virtqState));
1135	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
1136
1137	AssertMsgReturnVoid(IS_DRIVER_OK(pVirtio), ("Guest driver not in ready state.\n"));
1138	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
1139	{
1140	if (pVirtq->fEventThresholdReached)
1141	{
1142	#ifdef IN_RING3
1143	Log6Func(("...kicking guest %s, VIRTIO_F_EVENT_IDX set and threshold (%d) reached\n",
1144	VIRTQNAME(pVirtio, idxQueue), (uint16_t)virtioReadAvailUsedEvent(pDevIns, pVirtio, idxQueue)));
1145	#endif
1146	virtioKick(pDevIns, pVirtio, VIRTIO_ISR_VIRTQ_INTERRUPT, pVirtio->uQueueMsixVector[idxQueue], fForce);
1147	pVirtq->fEventThresholdReached = false;
1148	return;
1149	}
1150	#ifdef IN_RING3
1151	Log6Func(("...skipping interrupt %s, VIRTIO_F_EVENT_IDX set but threshold (%d) not reached (%d)\n",
1152	VIRTQNAME(pVirtio, idxQueue),(uint16_t)virtioReadAvailUsedEvent(pDevIns, pVirtio, idxQueue), pVirtq->uUsedIdx));
1153	#endif
1154	}
1155	else
1156	{
1157	/** If guest driver hasn't suppressed interrupts, interrupt */
1158	if (fForce \|\| !(virtioReadAvailRingFlags(pDevIns, pVirtio, idxQueue) & VIRTQ_AVAIL_F_NO_INTERRUPT))
1159	{
1160	if (fForce)
1161	Log6Func(("... kicking guest, queue %s, because force flag set\n", VIRTQNAME(pVirtio, idxQueue)));
1162	virtioKick(pDevIns, pVirtio, VIRTIO_ISR_VIRTQ_INTERRUPT, pVirtio->uQueueMsixVector[idxQueue], fForce);
1163	return;
1164	}
1165	Log6Func(("...skipping interrupt, queue %s, Guest flagged VIRTQ_AVAIL_F_NO_INTERRUPT for queue\n",
1166	VIRTQNAME(pVirtio, idxQueue)));
1167	}
1168	}
1169
1170	/**
1171	* Raise interrupt or MSI-X
1172	*
1173	* @param pDevIns The device instance.
1174	* @param pVirtio Pointer to the shared virtio state.
1175	* @param uCause Interrupt cause bit mask to set in PCI ISR port.
1176	* @param uVec MSI-X vector, if enabled
1177	* @param uForce True of out-of-band
1178	*/
1179	static int virtioKick(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint8_t uCause, uint16_t uMsixVector, bool fForce)
1180	{
1181	if (fForce)
1182	Log6Func(("reason: forced\n"));
1183	else
1184	if (uCause == VIRTIO_ISR_VIRTQ_INTERRUPT)
1185	Log6Func(("reason: buffer added to 'used' ring.\n"));
1186	else
1187	if (uCause == VIRTIO_ISR_DEVICE_CONFIG)
1188	Log6Func(("reason: device config change\n"));
1189
1190	if (!pVirtio->fMsiSupport)
1191	{
1192	pVirtio->uISR \|= uCause;
1193	PDMDevHlpPCISetIrq(pDevIns, 0, PDM_IRQ_LEVEL_HIGH);
1194	}
1195	else if (uMsixVector != VIRTIO_MSI_NO_VECTOR)
1196	PDMDevHlpPCISetIrq(pDevIns, uMsixVector, 1);
1197	return VINF_SUCCESS;
1198	}
1199
1200	/**
1201	* Lower interrupt (Called when guest reads ISR and when resetting)
1202	*
1203	* @param pDevIns The device instance.
1204	*/
1205	static void virtioLowerInterrupt(PPDMDEVINS pDevIns, uint16_t uMsixVector)
1206	{
1207	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1208	if (!pVirtio->fMsiSupport)
1209	PDMDevHlpPCISetIrq(pDevIns, 0, PDM_IRQ_LEVEL_LOW);
1210	else if (uMsixVector != VIRTIO_MSI_NO_VECTOR)
1211	PDMDevHlpPCISetIrq(pDevIns, pVirtio->uMsixConfig, PDM_IRQ_LEVEL_LOW);
1212	}
1213
1214	#ifdef IN_RING3
1215	static void virtioResetQueue(PVIRTIOCORE pVirtio, uint16_t idxQueue)
1216	{
1217	Assert(idxQueue < RT_ELEMENTS(pVirtio->virtqState));
1218	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
1219	pVirtq->uAvailIdx = 0;
1220	pVirtq->uUsedIdx = 0;
1221	pVirtq->fEventThresholdReached = false;
1222	pVirtio->uQueueEnable[idxQueue] = false;
1223	pVirtio->uQueueSize[idxQueue] = VIRTQ_MAX_SIZE;
1224	pVirtio->uQueueNotifyOff[idxQueue] = idxQueue;
1225	pVirtio->uQueueMsixVector[idxQueue] = idxQueue + 2;
1226	if (!pVirtio->fMsiSupport) /* VirtIO 1.0, 4.1.4.3 and 4.1.5.1.2 */
1227	pVirtio->uQueueMsixVector[idxQueue] = VIRTIO_MSI_NO_VECTOR;
1228
1229	virtioLowerInterrupt(pVirtio->pDevInsR3, pVirtio->uQueueMsixVector[idxQueue]);
1230	}
1231
1232	static void virtioResetDevice(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio)
1233	{
1234	Log2Func(("\n"));
1235	pVirtio->uDeviceFeaturesSelect = 0;
1236	pVirtio->uDriverFeaturesSelect = 0;
1237	pVirtio->uConfigGeneration = 0;
1238	pVirtio->uDeviceStatus = 0;
1239	pVirtio->uISR = 0;
1240
1241	if (!pVirtio->fMsiSupport)
1242	virtioLowerInterrupt(pDevIns, 0);
1243	else
1244	{
1245	virtioLowerInterrupt(pDevIns, pVirtio->uMsixConfig);
1246	for (int i = 0; i < VIRTQ_MAX_CNT; i++)
1247	{
1248	virtioLowerInterrupt(pDevIns, pVirtio->uQueueMsixVector[i]);
1249	pVirtio->uQueueMsixVector[i];
1250	}
1251	}
1252
1253	if (!pVirtio->fMsiSupport) /* VirtIO 1.0, 4.1.4.3 and 4.1.5.1.2 */
1254	pVirtio->uMsixConfig = VIRTIO_MSI_NO_VECTOR;
1255
1256	for (uint16_t idxQueue = 0; idxQueue < VIRTQ_MAX_CNT; idxQueue++)
1257	virtioResetQueue(pVirtio, idxQueue);
1258	}
1259
1260	/**
1261	* Invoked by this implementation when guest driver resets the device.
1262	* The driver itself will not until the device has read the status change.
1263	*/
1264	static void virtioGuestR3WasReset(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
1265	{
1266	LogFunc(("Guest reset the device\n"));
1267
1268	/* Let the client know */
1269	pVirtioCC->pfnStatusChanged(pVirtio, pVirtioCC, 0);
1270	virtioResetDevice(pDevIns, pVirtio);
1271	}
1272	#endif /* IN_RING3 */
1273
1274	/**
1275	* Handle accesses to Common Configuration capability
1276	*
1277	* @returns VBox status code
1278	*
1279	* @param pDevIns The device instance.
1280	* @param pVirtio Pointer to the shared virtio state.
1281	* @param pVirtioCC Pointer to the current context virtio state.
1282	* @param fWrite Set if write access, clear if read access.
1283	* @param offCfg The common configuration capability offset.
1284	* @param cb Number of bytes to read or write
1285	* @param pv Pointer to location to write to or read from
1286	*/
1287	static int virtioCommonCfgAccessed(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC,
1288	int fWrite, uint32_t offCfg, unsigned cb, void *pv)
1289	{
1290	/**
1291	* This macro resolves to boolean true if the implied parameters, offCfg and cb,
1292	* match the field offset and size of a field in the Common Cfg struct, (or if
1293	* it is a 64-bit field, if it accesses either 32-bit part as a 32-bit access)
1294	* This is mandated by section 4.1.3.1 of the VirtIO 1.0 specification)
1295	*
1296	* @param member Member of VIRTIO_PCI_COMMON_CFG_T
1297	* @param offCfg Implied parameter: Offset into VIRTIO_PCI_COMMON_CFG_T
1298	* @param cb Implied parameter: Number of bytes to access
1299	* @result true or false
1300	*/
1301	#define MATCH_COMMON_CFG(member) \
1302	( ( RT_SIZEOFMEMB(VIRTIO_PCI_COMMON_CFG_T, member) == 8 \
1303	&& ( offCfg == RT_OFFSETOF(VIRTIO_PCI_COMMON_CFG_T, member) \
1304	\|\| offCfg == RT_OFFSETOF(VIRTIO_PCI_COMMON_CFG_T, member) + sizeof(uint32_t)) \
1305	&& cb == sizeof(uint32_t)) \
1306	\|\| ( offCfg == RT_OFFSETOF(VIRTIO_PCI_COMMON_CFG_T, member) \
1307	&& cb == RT_SIZEOFMEMB(VIRTIO_PCI_COMMON_CFG_T, member)) )
1308
1309	#ifdef LOG_ENABLED
1310	# define LOG_COMMON_CFG_ACCESS(member, a_offIntra) \
1311	if (LogIs7Enabled()) { \
1312	virtioCoreLogMappedIoValue(__FUNCTION__, #member, RT_SIZEOFMEMB(VIRTIO_PCI_COMMON_CFG_T, member), \
1313	pv, cb, a_offIntra, fWrite, false, 0); \
1314	}
1315	# define LOG_COMMON_CFG_ACCESS_INDEXED(member, idx, a_offIntra) \
1316	if (LogIs7Enabled()) { \
1317	virtioCoreLogMappedIoValue(__FUNCTION__, #member, RT_SIZEOFMEMB(VIRTIO_PCI_COMMON_CFG_T, member), \
1318	pv, cb, a_offIntra, fWrite, true, idx); \
1319	}
1320	#else
1321	# define LOG_COMMON_CFG_ACCESS(member, a_offIntra) do { } while (0)
1322	# define LOG_COMMON_CFG_ACCESS_INDEXED(member, idx, a_offIntra) do { } while (0)
1323	#endif
1324
1325	#define COMMON_CFG_ACCESSOR(member) \
1326	do \
1327	{ \
1328	uint32_t offIntra = offCfg - RT_OFFSETOF(VIRTIO_PCI_COMMON_CFG_T, member); \
1329	if (fWrite) \
1330	memcpy((char )&pVirtio->member + offIntra, (const char )pv, cb); \
1331	else \
1332	memcpy(pv, (const char *)&pVirtio->member + offIntra, cb); \
1333	LOG_COMMON_CFG_ACCESS(member, offIntra); \
1334	} while(0)
1335
1336	#define COMMON_CFG_ACCESSOR_INDEXED(member, idx) \
1337	do \
1338	{ \
1339	uint32_t offIntra = offCfg - RT_OFFSETOF(VIRTIO_PCI_COMMON_CFG_T, member); \
1340	if (fWrite) \
1341	memcpy((char *)&pVirtio->member[idx] + offIntra, pv, cb); \
1342	else \
1343	memcpy(pv, (const char *)&pVirtio->member[idx] + offIntra, cb); \
1344	LOG_COMMON_CFG_ACCESS_INDEXED(member, idx, offIntra); \
1345	} while(0)
1346
1347	#define COMMON_CFG_ACCESSOR_READONLY(member) \
1348	do \
1349	{ \
1350	uint32_t offIntra = offCfg - RT_OFFSETOF(VIRTIO_PCI_COMMON_CFG_T, member); \
1351	if (fWrite) \
1352	LogFunc(("Guest attempted to write readonly virtio_pci_common_cfg.%s\n", #member)); \
1353	else \
1354	{ \
1355	memcpy(pv, (const char *)&pVirtio->member + offIntra, cb); \
1356	LOG_COMMON_CFG_ACCESS(member, offIntra); \
1357	} \
1358	} while(0)
1359
1360	#define COMMON_CFG_ACCESSOR_INDEXED_READONLY(member, idx) \
1361	do \
1362	{ \
1363	uint32_t offIntra = offCfg - RT_OFFSETOF(VIRTIO_PCI_COMMON_CFG_T, member); \
1364	if (fWrite) \
1365	LogFunc(("Guest attempted to write readonly virtio_pci_common_cfg.%s[%d]\n", #member, idx)); \
1366	else \
1367	{ \
1368	memcpy(pv, (char const *)&pVirtio->member[idx] + offIntra, cb); \
1369	LOG_COMMON_CFG_ACCESS_INDEXED(member, idx, offIntra); \
1370	} \
1371	} while(0)
1372
1373
1374	int rc = VINF_SUCCESS;
1375	uint64_t val;
1376	if (MATCH_COMMON_CFG(uDeviceFeatures))
1377	{
1378	if (fWrite) /* Guest WRITE pCommonCfg>uDeviceFeatures */
1379	{
1380	LogFunc(("Guest attempted to write readonly virtio_pci_common_cfg.device_feature\n"));
1381	return VINF_SUCCESS;
1382	}
1383	else /* Guest READ pCommonCfg->uDeviceFeatures */
1384	{
1385	switch (pVirtio->uDeviceFeaturesSelect)
1386	{
1387	case 0:
1388	val = pVirtio->uDeviceFeatures & UINT32_C(0xffffffff);
1389	memcpy(pv, &val, cb);
1390	LOG_COMMON_CFG_ACCESS(uDeviceFeatures, offCfg - RT_UOFFSETOF(VIRTIO_PCI_COMMON_CFG_T, uDeviceFeatures));
1391	break;
1392	case 1:
1393	val = pVirtio->uDeviceFeatures >> 32;
1394	memcpy(pv, &val, cb);
1395	LOG_COMMON_CFG_ACCESS(uDeviceFeatures, offCfg - RT_UOFFSETOF(VIRTIO_PCI_COMMON_CFG_T, uDeviceFeatures) + 4);
1396	break;
1397	default:
1398	LogFunc(("Guest read uDeviceFeatures with out of range selector (%#x), returning 0\n",
1399	pVirtio->uDeviceFeaturesSelect));
1400	return VINF_IOM_MMIO_UNUSED_00;
1401	}
1402	}
1403	}
1404	else if (MATCH_COMMON_CFG(uDriverFeatures))
1405	{
1406	if (fWrite) /* Guest WRITE pCommonCfg->udriverFeatures */
1407	{
1408	switch (pVirtio->uDriverFeaturesSelect)
1409	{
1410	case 0:
1411	memcpy(&pVirtio->uDriverFeatures, pv, cb);
1412	LOG_COMMON_CFG_ACCESS(uDriverFeatures, offCfg - RT_UOFFSETOF(VIRTIO_PCI_COMMON_CFG_T, uDriverFeatures));
1413	break;
1414	case 1:
1415	memcpy((char *)&pVirtio->uDriverFeatures + sizeof(uint32_t), pv, cb);
1416	LOG_COMMON_CFG_ACCESS(uDriverFeatures, offCfg - RT_UOFFSETOF(VIRTIO_PCI_COMMON_CFG_T, uDriverFeatures) + 4);
1417	break;
1418	default:
1419	LogFunc(("Guest wrote uDriverFeatures with out of range selector (%#x), returning 0\n",
1420	pVirtio->uDriverFeaturesSelect));
1421	return VINF_SUCCESS;
1422	}
1423	}
1424	else /* Guest READ pCommonCfg->udriverFeatures */
1425	{
1426	switch (pVirtio->uDriverFeaturesSelect)
1427	{
1428	case 0:
1429	val = pVirtio->uDriverFeatures & 0xffffffff;
1430	memcpy(pv, &val, cb);
1431	LOG_COMMON_CFG_ACCESS(uDriverFeatures, offCfg - RT_UOFFSETOF(VIRTIO_PCI_COMMON_CFG_T, uDriverFeatures));
1432	break;
1433	case 1:
1434	val = (pVirtio->uDriverFeatures >> 32) & 0xffffffff;
1435	memcpy(pv, &val, cb);
1436	LOG_COMMON_CFG_ACCESS(uDriverFeatures, offCfg - RT_UOFFSETOF(VIRTIO_PCI_COMMON_CFG_T, uDriverFeatures) + 4);
1437	break;
1438	default:
1439	LogFunc(("Guest read uDriverFeatures with out of range selector (%#x), returning 0\n",
1440	pVirtio->uDriverFeaturesSelect));
1441	return VINF_IOM_MMIO_UNUSED_00;
1442	}
1443	}
1444	}
1445	else if (MATCH_COMMON_CFG(uNumQueues))
1446	{
1447	if (fWrite)
1448	{
1449	Log2Func(("Guest attempted to write readonly virtio_pci_common_cfg.num_queues\n"));
1450	return VINF_SUCCESS;
1451	}
1452	else
1453	{
1454	(uint16_t )pv = VIRTQ_MAX_CNT;
1455	LOG_COMMON_CFG_ACCESS(uNumQueues, 0);
1456	}
1457	}
1458	else if (MATCH_COMMON_CFG(uDeviceStatus))
1459	{
1460	if (fWrite) /* Guest WRITE pCommonCfg->uDeviceStatus */
1461	{
1462	uint8_t const fNewStatus = (uint8_t )pv;
1463	Log7Func(("Guest wrote uDeviceStatus ................ ("));
1464	if (LogIs7Enabled())
1465	virtioLogDeviceStatus(fNewStatus ^ pVirtio->uDeviceStatus);
1466	Log7((")\n"));
1467
1468	/* If the status changed or we were reset, we need to go to ring-3 as
1469	it requires notifying the parent device. */
1470	bool const fStatusChanged = (fNewStatus & VIRTIO_STATUS_DRIVER_OK)
1471	!= (pVirtio->uPrevDeviceStatus & VIRTIO_STATUS_DRIVER_OK);
1472	#ifndef IN_RING3
1473	if (fStatusChanged \|\| fNewStatus == 0)
1474	{
1475	Log6Func(("=>ring3\n"));
1476	return VINF_IOM_R3_MMIO_WRITE;
1477	}
1478	#endif
1479	pVirtio->uDeviceStatus = fNewStatus;
1480
1481	#ifdef IN_RING3
1482	/*
1483	* Notify client only if status actually changed from last time and when we're reset.
1484	*/
1485	if (pVirtio->uDeviceStatus == 0)
1486	virtioGuestR3WasReset(pDevIns, pVirtio, pVirtioCC);
1487	if (fStatusChanged)
1488	pVirtioCC->pfnStatusChanged(pVirtio, pVirtioCC, fNewStatus & VIRTIO_STATUS_DRIVER_OK);
1489	#endif
1490	/*
1491	* Save the current status for the next write so we can see what changed.
1492	*/
1493	pVirtio->uPrevDeviceStatus = pVirtio->uDeviceStatus;
1494	}
1495	else /* Guest READ pCommonCfg->uDeviceStatus */
1496	{
1497	Log7Func(("Guest read uDeviceStatus ................ ("));
1498	(uint8_t )pv = pVirtio->uDeviceStatus;
1499	if (LogIs7Enabled())
1500	virtioLogDeviceStatus(pVirtio->uDeviceStatus);
1501	Log7((")\n"));
1502	}
1503	}
1504	else
1505	if (MATCH_COMMON_CFG(uMsixConfig))
1506	COMMON_CFG_ACCESSOR(uMsixConfig);
1507	else
1508	if (MATCH_COMMON_CFG(uDeviceFeaturesSelect))
1509	COMMON_CFG_ACCESSOR(uDeviceFeaturesSelect);
1510	else
1511	if (MATCH_COMMON_CFG(uDriverFeaturesSelect))
1512	COMMON_CFG_ACCESSOR(uDriverFeaturesSelect);
1513	else
1514	if (MATCH_COMMON_CFG(uConfigGeneration))
1515	COMMON_CFG_ACCESSOR_READONLY(uConfigGeneration);
1516	else
1517	if (MATCH_COMMON_CFG(uQueueSelect))
1518	COMMON_CFG_ACCESSOR(uQueueSelect);
1519	else
1520	if (MATCH_COMMON_CFG(uQueueSize))
1521	COMMON_CFG_ACCESSOR_INDEXED(uQueueSize, pVirtio->uQueueSelect);
1522	else
1523	if (MATCH_COMMON_CFG(uQueueMsixVector))
1524	COMMON_CFG_ACCESSOR_INDEXED(uQueueMsixVector, pVirtio->uQueueSelect);
1525	else
1526	if (MATCH_COMMON_CFG(uQueueEnable))
1527	COMMON_CFG_ACCESSOR_INDEXED(uQueueEnable, pVirtio->uQueueSelect);
1528	else
1529	if (MATCH_COMMON_CFG(uQueueNotifyOff))
1530	COMMON_CFG_ACCESSOR_INDEXED_READONLY(uQueueNotifyOff, pVirtio->uQueueSelect);
1531	else
1532	if (MATCH_COMMON_CFG(aGCPhysQueueDesc))
1533	COMMON_CFG_ACCESSOR_INDEXED(aGCPhysQueueDesc, pVirtio->uQueueSelect);
1534	else
1535	if (MATCH_COMMON_CFG(aGCPhysQueueAvail))
1536	COMMON_CFG_ACCESSOR_INDEXED(aGCPhysQueueAvail, pVirtio->uQueueSelect);
1537	else
1538	if (MATCH_COMMON_CFG(aGCPhysQueueUsed))
1539	COMMON_CFG_ACCESSOR_INDEXED(aGCPhysQueueUsed, pVirtio->uQueueSelect);
1540	else
1541	{
1542	Log2Func(("Bad guest %s access to virtio_pci_common_cfg: offCfg=%#x (%d), cb=%d\n",
1543	fWrite ? "write" : "read ", offCfg, offCfg, cb));
1544	return fWrite ? VINF_SUCCESS : VINF_IOM_MMIO_UNUSED_00;
1545	}
1546
1547	#undef COMMON_CFG_ACCESSOR_READONLY
1548	#undef COMMON_CFG_ACCESSOR_INDEXED_READONLY
1549	#undef COMMON_CFG_ACCESSOR_INDEXED
1550	#undef COMMON_CFG_ACCESSOR
1551	#undef LOG_COMMON_CFG_ACCESS_INDEXED
1552	#undef LOG_COMMON_CFG_ACCESS
1553	#undef MATCH_COMMON_CFG
1554	#ifndef IN_RING3
1555	RT_NOREF(pDevIns, pVirtioCC);
1556	#endif
1557	return rc;
1558	}
1559
1560	/**
1561	* @callback_method_impl{FNIOMMMIONEWREAD,
1562	* Memory mapped I/O Handler for PCI Capabilities read operations.}
1563	*
1564	* This MMIO handler specifically supports the VIRTIO_PCI_CAP_PCI_CFG capability defined
1565	* in the VirtIO 1.0 specification, section 4.1.4.7, and as such is restricted to reads
1566	* of 1, 2 or 4 bytes, only.
1567	*
1568	*/
1569	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioRead(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void pv, unsigned cb)
1570	{
1571	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1572	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1573	AssertReturn(cb == 1 \|\| cb == 2 \|\| cb == 4, VERR_INVALID_PARAMETER);
1574	Assert(pVirtio == (PVIRTIOCORE)pvUser); RT_NOREF(pvUser);
1575
1576
1577	uint32_t offIntra;
1578	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, offIntra, pVirtio->LocDeviceCap))
1579	{
1580	#ifdef IN_RING3
1581	/*
1582	* Callback to client to manage device-specific configuration.
1583	*/
1584	VBOXSTRICTRC rcStrict = pVirtioCC->pfnDevCapRead(pDevIns, offIntra, pv, cb);
1585
1586	/*
1587	* Additionally, anytime any part of the device-specific configuration (which our client maintains)
1588	* is READ it needs to be checked to see if it changed since the last time any part was read, in
1589	* order to maintain the config generation (see VirtIO 1.0 spec, section 4.1.4.3.1)
1590	*/
1591	bool fDevSpecificFieldChanged = RT_BOOL(memcmp(pVirtioCC->pbDevSpecificCfg + offIntra,
1592	pVirtioCC->pbPrevDevSpecificCfg + offIntra,
1593	RT_MIN(cb, pVirtioCC->cbDevSpecificCfg - offIntra)));
1594
1595	memcpy(pVirtioCC->pbPrevDevSpecificCfg, pVirtioCC->pbDevSpecificCfg, pVirtioCC->cbDevSpecificCfg);
1596
1597	if (pVirtio->fGenUpdatePending \|\| fDevSpecificFieldChanged)
1598	{
1599	++pVirtio->uConfigGeneration;
1600	Log6Func(("Bumped cfg. generation to %d because %s%s\n",
1601	pVirtio->uConfigGeneration,
1602	fDevSpecificFieldChanged ? "<dev cfg changed> " : "",
1603	pVirtio->fGenUpdatePending ? "<update was pending>" : ""));
1604	pVirtio->fGenUpdatePending = false;
1605	}
1606
1607	virtioLowerInterrupt(pDevIns, 0);
1608	return rcStrict;
1609	#else
1610	return VINF_IOM_R3_MMIO_READ;
1611	#endif
1612	}
1613
1614	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, offIntra, pVirtio->LocCommonCfgCap))
1615	return virtioCommonCfgAccessed(pDevIns, pVirtio, pVirtioCC, false /* fWrite */, offIntra, cb, pv);
1616
1617	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, offIntra, pVirtio->LocIsrCap) && cb == sizeof(uint8_t))
1618	{
1619	(uint8_t )pv = pVirtio->uISR;
1620	Log6Func(("Read and clear ISR\n"));
1621	pVirtio->uISR = 0; /* VirtIO specification requires reads of ISR to clear it */
1622	virtioLowerInterrupt(pDevIns, 0);
1623	return VINF_SUCCESS;
1624	}
1625
1626	ASSERT_GUEST_MSG_FAILED(("Bad read access to mapped capabilities region: off=%RGp cb=%u\n", off, cb));
1627	return VINF_IOM_MMIO_UNUSED_00;
1628	}
1629
1630	/**
1631	* @callback_method_impl{FNIOMMMIONEWREAD,
1632	* Memory mapped I/O Handler for PCI Capabilities write operations.}
1633	*
1634	* This MMIO handler specifically supports the VIRTIO_PCI_CAP_PCI_CFG capability defined
1635	* in the VirtIO 1.0 specification, section 4.1.4.7, and as such is restricted to writes
1636	* of 1, 2 or 4 bytes, only.
1637	*/
1638	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioWrite(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void const pv, unsigned cb)
1639	{
1640	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1641	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1642
1643	AssertReturn(cb == 1 \|\| cb == 2 \|\| cb == 4, VERR_INVALID_PARAMETER);
1644
1645	Assert(pVirtio == (PVIRTIOCORE)pvUser); RT_NOREF(pvUser);
1646	uint32_t offIntra;
1647	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, offIntra, pVirtio->LocDeviceCap))
1648	{
1649	#ifdef IN_RING3
1650	/*
1651	* Pass this MMIO write access back to the client to handle
1652	*/
1653	return pVirtioCC->pfnDevCapWrite(pDevIns, offIntra, pv, cb);
1654	#else
1655	return VINF_IOM_R3_MMIO_WRITE;
1656	#endif
1657	}
1658
1659	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, offIntra, pVirtio->LocCommonCfgCap))
1660	return virtioCommonCfgAccessed(pDevIns, pVirtio, pVirtioCC, true /* fWrite /, offIntra, cb, (void )pv);
1661
1662	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, offIntra, pVirtio->LocIsrCap) && cb == sizeof(uint8_t))
1663	{
1664	pVirtio->uISR = (uint8_t )pv;
1665	Log6Func(("Setting uISR = 0x%02x (virtq interrupt: %d, dev confg interrupt: %d)\n",
1666	pVirtio->uISR & 0xff,
1667	pVirtio->uISR & VIRTIO_ISR_VIRTQ_INTERRUPT,
1668	RT_BOOL(pVirtio->uISR & VIRTIO_ISR_DEVICE_CONFIG)));
1669	return VINF_SUCCESS;
1670	}
1671
1672	/* This should be guest driver dropping index of a new descriptor in avail ring */
1673	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, offIntra, pVirtio->LocNotifyCap) && cb == sizeof(uint16_t))
1674	{
1675	virtioQueueNotified(pDevIns, pVirtio, offIntra / VIRTIO_NOTIFY_OFFSET_MULTIPLIER, (uint16_t )pv);
1676	return VINF_SUCCESS;
1677	}
1678
1679	ASSERT_GUEST_MSG_FAILED(("Bad write access to mapped capabilities region: off=%RGp pv=%#p{%.*Rhxs} cb=%u\n", off, pv, cb, pv, cb));
1680	return VINF_SUCCESS;
1681	}
1682
1683	#ifdef IN_RING3
1684
1685	/**
1686	* @callback_method_impl{FNPCICONFIGREAD}
1687	*/
1688	static DECLCALLBACK(VBOXSTRICTRC) virtioR3PciConfigRead(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev,
1689	uint32_t uAddress, unsigned cb, uint32_t *pu32Value)
1690	{
1691	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1692	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1693	RT_NOREF(pPciDev);
1694
1695	Log7Func(("pDevIns=%p pPciDev=%p uAddress=%#x cb=%u pu32Value=%p\n",
1696	pDevIns, pPciDev, uAddress, cb, pu32Value));
1697	if (uAddress == pVirtio->uPciCfgDataOff)
1698	{
1699	/*
1700	* VirtIO 1.0 spec section 4.1.4.7 describes a required alternative access capability
1701	* whereby the guest driver can specify a bar, offset, and length via the PCI configuration space
1702	* (the virtio_pci_cfg_cap capability), and access data items.
1703	*/
1704	struct virtio_pci_cap *pPciCap = &pVirtioCC->pPciCfgCap->pciCap;
1705	uint32_t uLength = pPciCap->uLength;
1706
1707	if ( (uLength != 1 && uLength != 2 && uLength != 4)
1708	\|\| cb != uLength
1709	\|\| pPciCap->uBar != VIRTIO_REGION_PCI_CAP)
1710	{
1711	ASSERT_GUEST_MSG_FAILED(("Guest read virtio_pci_cfg_cap.pci_cfg_data using mismatching config. Ignoring\n"));
1712	*pu32Value = UINT32_MAX;
1713	return VINF_SUCCESS;
1714	}
1715
1716	VBOXSTRICTRC rcStrict = virtioMmioRead(pDevIns, pVirtio, pPciCap->uOffset, pu32Value, cb);
1717	Log2Func(("virtio: Guest read virtio_pci_cfg_cap.pci_cfg_data, bar=%d, offset=%d, length=%d, result=%d -> %Rrc\n",
1718	pPciCap->uBar, pPciCap->uOffset, uLength, *pu32Value, VBOXSTRICTRC_VAL(rcStrict)));
1719	return rcStrict;
1720	}
1721	return VINF_PDM_PCI_DO_DEFAULT;
1722	}
1723
1724	/**
1725	* @callback_method_impl{FNPCICONFIGWRITE}
1726	*/
1727	static DECLCALLBACK(VBOXSTRICTRC) virtioR3PciConfigWrite(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev,
1728	uint32_t uAddress, unsigned cb, uint32_t u32Value)
1729	{
1730	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1731	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1732	RT_NOREF(pPciDev);
1733
1734	Log7Func(("pDevIns=%p pPciDev=%p uAddress=%#x cb=%u u32Value=%#x\n", pDevIns, pPciDev, uAddress, cb, u32Value));
1735	if (uAddress == pVirtio->uPciCfgDataOff)
1736	{
1737	/* VirtIO 1.0 spec section 4.1.4.7 describes a required alternative access capability
1738	* whereby the guest driver can specify a bar, offset, and length via the PCI configuration space
1739	* (the virtio_pci_cfg_cap capability), and access data items. */
1740
1741	struct virtio_pci_cap *pPciCap = &pVirtioCC->pPciCfgCap->pciCap;
1742	uint32_t uLength = pPciCap->uLength;
1743
1744	if ( (uLength != 1 && uLength != 2 && uLength != 4)
1745	\|\| cb != uLength
1746	\|\| pPciCap->uBar != VIRTIO_REGION_PCI_CAP)
1747	{
1748	ASSERT_GUEST_MSG_FAILED(("Guest write virtio_pci_cfg_cap.pci_cfg_data using mismatching config. Ignoring\n"));
1749	return VINF_SUCCESS;
1750	}
1751
1752	VBOXSTRICTRC rcStrict = virtioMmioWrite(pDevIns, pVirtio, pPciCap->uOffset, &u32Value, cb);
1753	Log2Func(("Guest wrote virtio_pci_cfg_cap.pci_cfg_data, bar=%d, offset=%x, length=%x, value=%d -> %Rrc\n",
1754	pPciCap->uBar, pPciCap->uOffset, uLength, u32Value, VBOXSTRICTRC_VAL(rcStrict)));
1755	return rcStrict;
1756	}
1757	return VINF_PDM_PCI_DO_DEFAULT;
1758	}
1759
1760
1761	/*********************************************************************************************************************************
1762	* Saved state. *
1763	*********************************************************************************************************************************/
1764
1765	/**
1766	* Called from the FNSSMDEVSAVEEXEC function of the device.
1767	*
1768	* @param pVirtio Pointer to the shared virtio state.
1769	* @param pHlp The ring-3 device helpers.
1770	* @param pSSM The saved state handle.
1771	* @returns VBox status code.
1772	*/
1773	int virtioCoreR3SaveExec(PVIRTIOCORE pVirtio, PCPDMDEVHLPR3 pHlp, PSSMHANDLE pSSM)
1774	{
1775	LogFunc(("\n"));
1776	pHlp->pfnSSMPutU64(pSSM, VIRTIO_SAVEDSTATE_MARKER);
1777	pHlp->pfnSSMPutU32(pSSM, VIRTIO_SAVEDSTATE_VERSION);
1778
1779	pHlp->pfnSSMPutBool(pSSM, pVirtio->fGenUpdatePending);
1780	pHlp->pfnSSMPutU8(pSSM, pVirtio->uDeviceStatus);
1781	pHlp->pfnSSMPutU8(pSSM, pVirtio->uConfigGeneration);
1782	pHlp->pfnSSMPutU8(pSSM, pVirtio->uPciCfgDataOff);
1783	pHlp->pfnSSMPutU8(pSSM, pVirtio->uISR);
1784	pHlp->pfnSSMPutU16(pSSM, pVirtio->uQueueSelect);
1785	pHlp->pfnSSMPutU32(pSSM, pVirtio->uDeviceFeaturesSelect);
1786	pHlp->pfnSSMPutU32(pSSM, pVirtio->uDriverFeaturesSelect);
1787	pHlp->pfnSSMPutU64(pSSM, pVirtio->uDriverFeatures);
1788
1789	for (uint32_t i = 0; i < VIRTQ_MAX_CNT; i++)
1790	{
1791	pHlp->pfnSSMPutGCPhys64(pSSM, pVirtio->aGCPhysQueueDesc[i]);
1792	pHlp->pfnSSMPutGCPhys64(pSSM, pVirtio->aGCPhysQueueAvail[i]);
1793	pHlp->pfnSSMPutGCPhys64(pSSM, pVirtio->aGCPhysQueueUsed[i]);
1794	pHlp->pfnSSMPutU16(pSSM, pVirtio->uQueueNotifyOff[i]);
1795	pHlp->pfnSSMPutU16(pSSM, pVirtio->uQueueMsixVector[i]);
1796	pHlp->pfnSSMPutU16(pSSM, pVirtio->uQueueEnable[i]);
1797	pHlp->pfnSSMPutU16(pSSM, pVirtio->uQueueSize[i]);
1798	pHlp->pfnSSMPutU16(pSSM, pVirtio->virtqState[i].uAvailIdx);
1799	pHlp->pfnSSMPutU16(pSSM, pVirtio->virtqState[i].uUsedIdx);
1800	int rc = pHlp->pfnSSMPutMem(pSSM, pVirtio->virtqState[i].szVirtqName, 32);
1801	AssertRCReturn(rc, rc);
1802	}
1803
1804	return VINF_SUCCESS;
1805	}
1806
1807	/**
1808	* Called from the FNSSMDEVLOADEXEC function of the device.
1809	*
1810	* @param pVirtio Pointer to the shared virtio state.
1811	* @param pHlp The ring-3 device helpers.
1812	* @param pSSM The saved state handle.
1813	* @returns VBox status code.
1814	*/
1815	int virtioCoreR3LoadExec(PVIRTIOCORE pVirtio, PCPDMDEVHLPR3 pHlp, PSSMHANDLE pSSM)
1816	{
1817	LogFunc(("\n"));
1818	/*
1819	* Check the marker and (embedded) version number.
1820	*/
1821	uint64_t uMarker = 0;
1822	int rc = pHlp->pfnSSMGetU64(pSSM, &uMarker);
1823	AssertRCReturn(rc, rc);
1824	if (uMarker != VIRTIO_SAVEDSTATE_MARKER)
1825	return pHlp->pfnSSMSetLoadError(pSSM, VERR_SSM_DATA_UNIT_FORMAT_CHANGED, RT_SRC_POS,
1826	N_("Expected marker value %#RX64 found %#RX64 instead"),
1827	VIRTIO_SAVEDSTATE_MARKER, uMarker);
1828	uint32_t uVersion = 0;
1829	rc = pHlp->pfnSSMGetU32(pSSM, &uVersion);
1830	AssertRCReturn(rc, rc);
1831	if (uVersion != VIRTIO_SAVEDSTATE_VERSION)
1832	return pHlp->pfnSSMSetLoadError(pSSM, VERR_SSM_DATA_UNIT_FORMAT_CHANGED, RT_SRC_POS,
1833	N_("Unsupported virtio version: %u"), uVersion);
1834	/*
1835	* Load the state.
1836	*/
1837	pHlp->pfnSSMGetBool(pSSM, &pVirtio->fGenUpdatePending);
1838	pHlp->pfnSSMGetU8(pSSM, &pVirtio->uDeviceStatus);
1839	pHlp->pfnSSMGetU8(pSSM, &pVirtio->uConfigGeneration);
1840	pHlp->pfnSSMGetU8(pSSM, &pVirtio->uPciCfgDataOff);
1841	pHlp->pfnSSMGetU8(pSSM, &pVirtio->uISR);
1842	pHlp->pfnSSMGetU16(pSSM, &pVirtio->uQueueSelect);
1843	pHlp->pfnSSMGetU32(pSSM, &pVirtio->uDeviceFeaturesSelect);
1844	pHlp->pfnSSMGetU32(pSSM, &pVirtio->uDriverFeaturesSelect);
1845	pHlp->pfnSSMGetU64(pSSM, &pVirtio->uDriverFeatures);
1846
1847	for (uint32_t i = 0; i < VIRTQ_MAX_CNT; i++)
1848	{
1849	pHlp->pfnSSMGetGCPhys64(pSSM, &pVirtio->aGCPhysQueueDesc[i]);
1850	pHlp->pfnSSMGetGCPhys64(pSSM, &pVirtio->aGCPhysQueueAvail[i]);
1851	pHlp->pfnSSMGetGCPhys64(pSSM, &pVirtio->aGCPhysQueueUsed[i]);
1852	pHlp->pfnSSMGetU16(pSSM, &pVirtio->uQueueNotifyOff[i]);
1853	pHlp->pfnSSMGetU16(pSSM, &pVirtio->uQueueMsixVector[i]);
1854	pHlp->pfnSSMGetU16(pSSM, &pVirtio->uQueueEnable[i]);
1855	pHlp->pfnSSMGetU16(pSSM, &pVirtio->uQueueSize[i]);
1856	pHlp->pfnSSMGetU16(pSSM, &pVirtio->virtqState[i].uAvailIdx);
1857	pHlp->pfnSSMGetU16(pSSM, &pVirtio->virtqState[i].uUsedIdx);
1858	rc = pHlp->pfnSSMGetMem(pSSM, pVirtio->virtqState[i].szVirtqName,
1859	sizeof(pVirtio->virtqState[i].szVirtqName));
1860	AssertRCReturn(rc, rc);
1861	}
1862
1863	return VINF_SUCCESS;
1864	}
1865
1866
1867	/*********************************************************************************************************************************
1868	* Device Level *
1869	*********************************************************************************************************************************/
1870
1871	/**
1872	* This must be called by the client to handle VM state changes
1873	* after the client takes care of its device-specific tasks for the state change.
1874	* (i.e. Reset, suspend, power-off, resume)
1875	*
1876	* @param pDevIns The device instance.
1877	* @param pVirtio Pointer to the shared virtio state.
1878	*/
1879	void virtioCoreR3VmStateChanged(PVIRTIOCORE pVirtio, VIRTIOVMSTATECHANGED enmState)
1880	{
1881	LogFunc(("State changing to %s\n",
1882	virtioCoreGetStateChangeText(enmState)));
1883
1884	switch(enmState)
1885	{
1886	case kvirtIoVmStateChangedReset:
1887	virtioCoreResetAll(pVirtio);
1888	break;
1889	case kvirtIoVmStateChangedSuspend:
1890	break;
1891	case kvirtIoVmStateChangedPowerOff:
1892	break;
1893	case kvirtIoVmStateChangedResume:
1894	virtioNotifyGuestDriver(pVirtio->pDevInsR3, pVirtio, 0 /* idxQueue /, true / fForce */);
1895	break;
1896	default:
1897	LogRelFunc(("Bad enum value"));
1898	return;
1899	}
1900	}
1901
1902	/**
1903	* This should be called from PDMDEVREGR3::pfnDestruct.
1904	*
1905	* @param pDevIns The device instance.
1906	* @param pVirtio Pointer to the shared virtio state.
1907	* @param pVirtioCC Pointer to the ring-3 virtio state.
1908	*/
1909	void virtioCoreR3Term(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
1910	{
1911	if (pVirtioCC->pbPrevDevSpecificCfg)
1912	{
1913	RTMemFree(pVirtioCC->pbPrevDevSpecificCfg);
1914	pVirtioCC->pbPrevDevSpecificCfg = NULL;
1915	}
1916	RT_NOREF(pDevIns, pVirtio);
1917	}
1918
1919
1920	/**rr
1921	* Setup PCI device controller and Virtio state
1922	*
1923	* This should be called from PDMDEVREGR3::pfnConstruct.
1924	*
1925	* @param pDevIns The device instance.
1926	* @param pVirtio Pointer to the shared virtio state. This
1927	* must be the first member in the shared
1928	* device instance data!
1929	* @param pVirtioCC Pointer to the ring-3 virtio state. This
1930	* must be the first member in the ring-3
1931	* device instance data!
1932	* @param pPciParams Values to populate industry standard PCI Configuration Space data structure
1933	* @param pcszInstance Device instance name (format-specifier)
1934	* @param fDevSpecificFeatures VirtIO device-specific features offered by
1935	* client
1936	* @param cbDevSpecificCfg Size of virtio_pci_device_cap device-specific struct
1937	* @param pvDevSpecificCfg Address of client's dev-specific
1938	* configuration struct.
1939	*/
1940	int virtioCoreR3Init(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC, PVIRTIOPCIPARAMS pPciParams,
1941	const char pcszInstance, uint64_t fDevSpecificFeatures, void pvDevSpecificCfg, uint16_t cbDevSpecificCfg)
1942	{
1943	/*
1944	* The pVirtio state must be the first member of the shared device instance
1945	* data, otherwise we cannot get our bearings in the PCI configuration callbacks.
1946	*/
1947	AssertLogRelReturn(pVirtio == PDMINS_2_DATA(pDevIns, PVIRTIOCORE), VERR_STATE_CHANGED);
1948	AssertLogRelReturn(pVirtioCC == PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC), VERR_STATE_CHANGED);
1949
1950	pVirtio->pDevInsR3 = pDevIns;
1951
1952	/*
1953	* Caller must initialize these.
1954	*/
1955	AssertReturn(pVirtioCC->pfnStatusChanged, VERR_INVALID_POINTER);
1956	AssertReturn(pVirtioCC->pfnQueueNotified, VERR_INVALID_POINTER);
1957
1958	#if 0 /* Until pdmR3DvHlp_PCISetIrq() impl is fixed and Assert that limits vec to 0 is removed */
1959	# ifdef VBOX_WITH_MSI_DEVICES
1960	pVirtio->fMsiSupport = true;
1961	# endif
1962	#endif
1963
1964	/*
1965	* The host features offered include both device-specific features
1966	* and reserved feature bits (device independent)
1967	*/
1968	pVirtio->uDeviceFeatures = VIRTIO_F_VERSION_1
1969	\| VIRTIO_DEV_INDEPENDENT_FEATURES_OFFERED
1970	\| fDevSpecificFeatures;
1971
1972	RTStrCopy(pVirtio->szInstance, sizeof(pVirtio->szInstance), pcszInstance);
1973
1974	pVirtio->uDeviceStatus = 0;
1975	pVirtioCC->cbDevSpecificCfg = cbDevSpecificCfg;
1976	pVirtioCC->pbDevSpecificCfg = (uint8_t *)pvDevSpecificCfg;
1977	pVirtioCC->pbPrevDevSpecificCfg = (uint8_t *)RTMemDup(pvDevSpecificCfg, cbDevSpecificCfg);
1978	AssertLogRelReturn(pVirtioCC->pbPrevDevSpecificCfg, VERR_NO_MEMORY);
1979
1980	/* Set PCI config registers (assume 32-bit mode) */
1981	PPDMPCIDEV pPciDev = pDevIns->apPciDevs[0];
1982	PDMPCIDEV_ASSERT_VALID(pDevIns, pPciDev);
1983
1984	PDMPciDevSetRevisionId(pPciDev, DEVICE_PCI_REVISION_ID_VIRTIO);
1985	PDMPciDevSetVendorId(pPciDev, DEVICE_PCI_VENDOR_ID_VIRTIO);
1986	PDMPciDevSetSubSystemVendorId(pPciDev, DEVICE_PCI_VENDOR_ID_VIRTIO);
1987	PDMPciDevSetDeviceId(pPciDev, pPciParams->uDeviceId);
1988	PDMPciDevSetClassBase(pPciDev, pPciParams->uClassBase);
1989	PDMPciDevSetClassSub(pPciDev, pPciParams->uClassSub);
1990	PDMPciDevSetClassProg(pPciDev, pPciParams->uClassProg);
1991	PDMPciDevSetSubSystemId(pPciDev, pPciParams->uSubsystemId);
1992	PDMPciDevSetInterruptLine(pPciDev, pPciParams->uInterruptLine);
1993	PDMPciDevSetInterruptPin(pPciDev, pPciParams->uInterruptPin);
1994
1995	/* Register PCI device */
1996	int rc = PDMDevHlpPCIRegister(pDevIns, pPciDev);
1997	if (RT_FAILURE(rc))
1998	return PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register PCI Device")); /* can we put params in this error? */
1999
2000	rc = PDMDevHlpPCIInterceptConfigAccesses(pDevIns, pPciDev, virtioR3PciConfigRead, virtioR3PciConfigWrite);
2001	AssertRCReturn(rc, rc);
2002
2003
2004	/* Construct & map PCI vendor-specific capabilities for virtio host negotiation with guest driver */
2005
2006	/* The following capability mapped via VirtIO 1.0: struct virtio_pci_cfg_cap (VIRTIO_PCI_CFG_CAP_T)
2007	* as a mandatory but suboptimal alternative interface to host device capabilities, facilitating
2008	* access the memory of any BAR. If the guest uses it (the VirtIO driver on Linux doesn't),
2009	* Unlike Common, Notify, ISR and Device capabilities, it is accessed directly via PCI Config region.
2010	* therefore does not contribute to the capabilities region (BAR) the other capabilities use.
2011	*/
2012	#define CFG_ADDR_2_IDX(addr) ((uint8_t)(((uintptr_t)(addr) - (uintptr_t)&pPciDev->abConfig[0])))
2013	#define SET_PCI_CAP_LOC(a_pPciDev, a_pCfg, a_LocCap, a_uMmioLengthAlign) \
2014	do { \
2015	(a_LocCap).offMmio = (a_pCfg)->uOffset; \
2016	(a_LocCap).cbMmio = RT_ALIGN_T((a_pCfg)->uLength, a_uMmioLengthAlign, uint16_t); \
2017	(a_LocCap).offPci = (uint16_t)(uintptr_t)((uint8_t *)(a_pCfg) - &(a_pPciDev)->abConfig[0]); \
2018	(a_LocCap).cbPci = (a_pCfg)->uCapLen; \
2019	} while (0)
2020
2021	PVIRTIO_PCI_CAP_T pCfg;
2022	uint32_t cbRegion = 0;
2023
2024	/* Common capability (VirtIO 1.0 spec, section 4.1.4.3) */
2025	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[0x40];
2026	pCfg->uCfgType = VIRTIO_PCI_CAP_COMMON_CFG;
2027	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2028	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
2029	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
2030	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2031	pCfg->uOffset = RT_ALIGN_32(0, 4); /* reminder, in case someone changes offset */
2032	pCfg->uLength = sizeof(VIRTIO_PCI_COMMON_CFG_T);
2033	cbRegion += pCfg->uLength;
2034	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocCommonCfgCap, 2);
2035	pVirtioCC->pCommonCfgCap = pCfg;
2036
2037	/*
2038	* Notify capability (VirtIO 1.0 spec, section 4.1.4.4). Note: uLength is based on the choice
2039	* of this implementation to make each queue's uQueueNotifyOff equal to (QueueSelect) ordinal
2040	* value of the queue (different strategies are possible according to spec).
2041	*/
2042	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2043	pCfg->uCfgType = VIRTIO_PCI_CAP_NOTIFY_CFG;
2044	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2045	pCfg->uCapLen = sizeof(VIRTIO_PCI_NOTIFY_CAP_T);
2046	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
2047	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2048	pCfg->uOffset = pVirtioCC->pCommonCfgCap->uOffset + pVirtioCC->pCommonCfgCap->uLength;
2049	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
2050
2051
2052	pCfg->uLength = VIRTQ_MAX_CNT * VIRTIO_NOTIFY_OFFSET_MULTIPLIER + 2; /* will change in VirtIO 1.1 */
2053	cbRegion += pCfg->uLength;
2054	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocNotifyCap, 1);
2055	pVirtioCC->pNotifyCap = (PVIRTIO_PCI_NOTIFY_CAP_T)pCfg;
2056	pVirtioCC->pNotifyCap->uNotifyOffMultiplier = VIRTIO_NOTIFY_OFFSET_MULTIPLIER;
2057
2058	/* ISR capability (VirtIO 1.0 spec, section 4.1.4.5)
2059	*
2060	* VirtIO 1.0 spec says 8-bit, unaligned in MMIO space. Example/diagram
2061	* of spec shows it as a 32-bit field with upper bits 'reserved'
2062	* Will take spec words more literally than the diagram for now.
2063	*/
2064	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2065	pCfg->uCfgType = VIRTIO_PCI_CAP_ISR_CFG;
2066	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2067	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
2068	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
2069	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2070	pCfg->uOffset = pVirtioCC->pNotifyCap->pciCap.uOffset + pVirtioCC->pNotifyCap->pciCap.uLength;
2071	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
2072	pCfg->uLength = sizeof(uint8_t);
2073	cbRegion += pCfg->uLength;
2074	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocIsrCap, 4);
2075	pVirtioCC->pIsrCap = pCfg;
2076
2077	/* PCI Cfg capability (VirtIO 1.0 spec, section 4.1.4.7)
2078	* This capability doesn't get page-MMIO mapped. Instead uBar, uOffset and uLength are intercepted
2079	* by trapping PCI configuration I/O and get modulated by consumers to locate fetch and read/write
2080	* values from any region. NOTE: The linux driver not only doesn't use this feature, it will not
2081	* even list it as present if uLength isn't non-zero and also 4-byte-aligned as the linux driver is
2082	* initializing.
2083	*/
2084	pVirtio->uPciCfgDataOff = pCfg->uCapNext + RT_OFFSETOF(VIRTIO_PCI_CFG_CAP_T, uPciCfgData);
2085	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2086	pCfg->uCfgType = VIRTIO_PCI_CAP_PCI_CFG;
2087	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2088	pCfg->uCapLen = sizeof(VIRTIO_PCI_CFG_CAP_T);
2089	pCfg->uCapNext = (pVirtio->fMsiSupport \|\| pVirtioCC->pbDevSpecificCfg) ? CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen : 0;
2090	pCfg->uBar = 0;
2091	pCfg->uOffset = 0;
2092	pCfg->uLength = 0;
2093	cbRegion += pCfg->uLength;
2094	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocPciCfgCap, 1);
2095	pVirtioCC->pPciCfgCap = (PVIRTIO_PCI_CFG_CAP_T)pCfg;
2096
2097	if (pVirtioCC->pbDevSpecificCfg)
2098	{
2099	/* Following capability (via VirtIO 1.0, section 4.1.4.6). Client defines the
2100	* device-specific config fields struct and passes size to this constructor */
2101	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2102	pCfg->uCfgType = VIRTIO_PCI_CAP_DEVICE_CFG;
2103	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2104	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
2105	pCfg->uCapNext = pVirtio->fMsiSupport ? CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen : 0;
2106	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2107	pCfg->uOffset = pVirtioCC->pIsrCap->uOffset + pVirtioCC->pIsrCap->uLength;
2108	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
2109	pCfg->uLength = cbDevSpecificCfg;
2110	cbRegion += pCfg->uLength;
2111	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocDeviceCap, 4);
2112	pVirtioCC->pDeviceCap = pCfg;
2113	}
2114	else
2115	Assert(pVirtio->LocDeviceCap.cbMmio == 0 && pVirtio->LocDeviceCap.cbPci == 0);
2116
2117	if (pVirtio->fMsiSupport)
2118	{
2119	PDMMSIREG aMsiReg;
2120	RT_ZERO(aMsiReg);
2121	aMsiReg.iMsixCapOffset = pCfg->uCapNext;
2122	aMsiReg.iMsixNextOffset = 0;
2123	aMsiReg.iMsixBar = VIRTIO_REGION_MSIX_CAP;
2124	aMsiReg.cMsixVectors = VBOX_MSIX_MAX_ENTRIES;
2125	rc = PDMDevHlpPCIRegisterMsi(pDevIns, &aMsiReg); /* see MsixR3init() */
2126	if (RT_FAILURE(rc))
2127	{
2128	/* See PDMDevHlp.cpp:pdmR3DevHlp_PCIRegisterMsi */
2129	LogFunc(("Failed to configure MSI-X (%Rrc). Reverting to INTx\n", rc));
2130	pVirtio->fMsiSupport = false;
2131	}
2132	else
2133	Log2Func(("Using MSI-X for guest driver notification\n"));
2134	}
2135	else
2136	LogFunc(("MSI-X not available for VBox, using INTx notification\n"));
2137
2138	/* Set offset to first capability and enable PCI dev capabilities */
2139	PDMPciDevSetCapabilityList(pPciDev, 0x40);
2140	PDMPciDevSetStatus(pPciDev, VBOX_PCI_STATUS_CAP_LIST);
2141
2142	/* Linux drivers/virtio/virtio_pci_modern.c tries to map at least a page for the
2143	* 'unknown' device-specific capability without querying the capability to figure
2144	* out size, so pad with an extra page
2145	*/
2146	size_t cbSize = RTStrPrintf(pVirtioCC->pcszMmioName, sizeof(pVirtioCC->pcszMmioName), "%s MMIO", pcszInstance);
2147	if (cbSize <= 0)
2148	return PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: out of memory allocating string")); /* can we put params in this error? */
2149
2150	rc = PDMDevHlpPCIIORegionCreateMmio(pDevIns, VIRTIO_REGION_PCI_CAP, RT_ALIGN_32(cbRegion + PAGE_SIZE, PAGE_SIZE),
2151	PCI_ADDRESS_SPACE_MEM, virtioMmioWrite, virtioMmioRead, pVirtio,
2152	IOMMMIO_FLAGS_READ_PASSTHRU \| IOMMMIO_FLAGS_WRITE_PASSTHRU,
2153	pVirtioCC->pcszMmioName,
2154	&pVirtio->hMmioPciCap);
2155	AssertLogRelRCReturn(rc, PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register PCI Capabilities address space")));
2156	/*
2157	* Statistics.
2158	*/
2159	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsAllocated, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
2160	"Total number of allocated descriptor chains", "DescChainsAllocated");
2161	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsFreed, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
2162	"Total number of freed descriptor chains", "DescChainsFreed");
2163	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsSegsIn, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
2164	"Total number of inbound segments", "DescChainsSegsIn");
2165	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsSegsOut, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
2166	"Total number of outbound segments", "DescChainsSegsOut");
2167
2168	return VINF_SUCCESS;
2169	}
2170
2171	#else /* !IN_RING3 */
2172
2173	/**
2174	* Sets up the core ring-0/raw-mode virtio bits.
2175	*
2176	* @returns VBox status code.
2177	* @param pDevIns The device instance.
2178	* @param pVirtio Pointer to the shared virtio state. This must be the first
2179	* member in the shared device instance data!
2180	*/
2181	int virtioCoreRZInit(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio)
2182	{
2183	AssertLogRelReturn(pVirtio == PDMINS_2_DATA(pDevIns, PVIRTIOCORE), VERR_STATE_CHANGED);
2184
2185	#ifdef FUTURE_OPTIMIZATION
2186	int rc = PDMDevHlpSetDeviceCritSect(pDevIns, PDMDevHlpCritSectGetNop(pDevIns));
2187	AssertRCReturn(rc, rc);
2188	#endif
2189	int rc = PDMDevHlpMmioSetUpContext(pDevIns, pVirtio->hMmioPciCap, virtioMmioWrite, virtioMmioRead, pVirtio);
2190	AssertRCReturn(rc, rc);
2191	return rc;
2192	}
2193
2194	#endif /* !IN_RING3 */
2195

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Devices/VirtIO/Virtio_1_0.cpp@ 84372

Download in other formats: