PDMAsyncCompletionFileNormal.cpp@ 100307

Last change on this file since 100307 was 99739, checked in by vboxsync, 19 months ago
*: doxygen corrections (mostly about removing @returns from functions returning void).
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 67.1 KB

Line
1	/* $Id: PDMAsyncCompletionFileNormal.cpp 99739 2023-05-11 01:01:08Z vboxsync $ */
2	/** @file
3	* PDM Async I/O - Async File I/O manager.
4	*/
5
6	/*
7	* Copyright (C) 2006-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_PDM_ASYNC_COMPLETION
33	#include <iprt/types.h>
34	#include <iprt/asm.h>
35	#include <iprt/file.h>
36	#include <iprt/mem.h>
37	#include <iprt/string.h>
38	#include <iprt/assert.h>
39	#include <VBox/log.h>
40
41	#include "PDMAsyncCompletionFileInternal.h"
42
43
44	/*********************************************************************************************************************************
45	* Defined Constants And Macros *
46	*********************************************************************************************************************************/
47	/** The update period for the I/O load statistics in ms. */
48	#define PDMACEPFILEMGR_LOAD_UPDATE_PERIOD 1000
49	/** Maximum number of requests a manager will handle. */
50	#define PDMACEPFILEMGR_REQS_STEP 64
51
52
53	/*********************************************************************************************************************************
54	* Internal functions *
55	*********************************************************************************************************************************/
56	static int pdmacFileAioMgrNormalProcessTaskList(PPDMACTASKFILE pTaskHead,
57	PPDMACEPFILEMGR pAioMgr,
58	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint);
59
60	static PPDMACTASKFILE pdmacFileAioMgrNormalRangeLockFree(PPDMACEPFILEMGR pAioMgr,
61	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
62	PPDMACFILERANGELOCK pRangeLock);
63
64	static void pdmacFileAioMgrNormalReqCompleteRc(PPDMACEPFILEMGR pAioMgr, RTFILEAIOREQ hReq,
65	int rc, size_t cbTransfered);
66
67
68	int pdmacFileAioMgrNormalInit(PPDMACEPFILEMGR pAioMgr)
69	{
70	pAioMgr->cRequestsActiveMax = PDMACEPFILEMGR_REQS_STEP;
71
72	int rc = RTFileAioCtxCreate(&pAioMgr->hAioCtx, RTFILEAIO_UNLIMITED_REQS, 0 /* fFlags */);
73	if (rc == VERR_OUT_OF_RANGE)
74	rc = RTFileAioCtxCreate(&pAioMgr->hAioCtx, pAioMgr->cRequestsActiveMax, 0 /* fFlags */);
75
76	if (RT_SUCCESS(rc))
77	{
78	/* Initialize request handle array. */
79	pAioMgr->iFreeEntry = 0;
80	pAioMgr->cReqEntries = pAioMgr->cRequestsActiveMax;
81	pAioMgr->pahReqsFree = (RTFILEAIOREQ )RTMemAllocZ(pAioMgr->cReqEntries sizeof(RTFILEAIOREQ));
82
83	if (pAioMgr->pahReqsFree)
84	{
85	/* Create the range lock memcache. */
86	rc = RTMemCacheCreate(&pAioMgr->hMemCacheRangeLocks, sizeof(PDMACFILERANGELOCK),
87	0, UINT32_MAX, NULL, NULL, NULL, 0);
88	if (RT_SUCCESS(rc))
89	return VINF_SUCCESS;
90
91	RTMemFree(pAioMgr->pahReqsFree);
92	}
93	else
94	{
95	RTFileAioCtxDestroy(pAioMgr->hAioCtx);
96	rc = VERR_NO_MEMORY;
97	}
98	}
99
100	return rc;
101	}
102
103	void pdmacFileAioMgrNormalDestroy(PPDMACEPFILEMGR pAioMgr)
104	{
105	RTFileAioCtxDestroy(pAioMgr->hAioCtx);
106
107	while (pAioMgr->iFreeEntry > 0)
108	{
109	pAioMgr->iFreeEntry--;
110	Assert(pAioMgr->pahReqsFree[pAioMgr->iFreeEntry] != NIL_RTFILEAIOREQ);
111	RTFileAioReqDestroy(pAioMgr->pahReqsFree[pAioMgr->iFreeEntry]);
112	}
113
114	RTMemFree(pAioMgr->pahReqsFree);
115	RTMemCacheDestroy(pAioMgr->hMemCacheRangeLocks);
116	}
117
118	#if 0 /* currently unused */
119	/**
120	* Sorts the endpoint list with insertion sort.
121	*/
122	static void pdmacFileAioMgrNormalEndpointsSortByLoad(PPDMACEPFILEMGR pAioMgr)
123	{
124	PPDMASYNCCOMPLETIONENDPOINTFILE pEpPrev, pEpCurr, pEpNextToSort;
125
126	pEpPrev = pAioMgr->pEndpointsHead;
127	pEpCurr = pEpPrev->AioMgr.pEndpointNext;
128
129	while (pEpCurr)
130	{
131	/* Remember the next element to sort because the list might change. */
132	pEpNextToSort = pEpCurr->AioMgr.pEndpointNext;
133
134	/* Unlink the current element from the list. */
135	PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEpCurr->AioMgr.pEndpointPrev;
136	PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEpCurr->AioMgr.pEndpointNext;
137
138	if (pPrev)
139	pPrev->AioMgr.pEndpointNext = pNext;
140	else
141	pAioMgr->pEndpointsHead = pNext;
142
143	if (pNext)
144	pNext->AioMgr.pEndpointPrev = pPrev;
145
146	/* Go back until we reached the place to insert the current endpoint into. */
147	while (pEpPrev && (pEpPrev->AioMgr.cReqsPerSec < pEpCurr->AioMgr.cReqsPerSec))
148	pEpPrev = pEpPrev->AioMgr.pEndpointPrev;
149
150	/* Link the endpoint into the list. */
151	if (pEpPrev)
152	pNext = pEpPrev->AioMgr.pEndpointNext;
153	else
154	pNext = pAioMgr->pEndpointsHead;
155
156	pEpCurr->AioMgr.pEndpointNext = pNext;
157	pEpCurr->AioMgr.pEndpointPrev = pEpPrev;
158
159	if (pNext)
160	pNext->AioMgr.pEndpointPrev = pEpCurr;
161
162	if (pEpPrev)
163	pEpPrev->AioMgr.pEndpointNext = pEpCurr;
164	else
165	pAioMgr->pEndpointsHead = pEpCurr;
166
167	pEpCurr = pEpNextToSort;
168	}
169
170	#ifdef DEBUG
171	/* Validate sorting algorithm */
172	unsigned cEndpoints = 0;
173	pEpCurr = pAioMgr->pEndpointsHead;
174
175	AssertMsg(pEpCurr, ("No endpoint in the list?\n"));
176	AssertMsg(!pEpCurr->AioMgr.pEndpointPrev, ("First element in the list points to previous element\n"));
177
178	while (pEpCurr)
179	{
180	cEndpoints++;
181
182	PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEpCurr->AioMgr.pEndpointNext;
183	PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEpCurr->AioMgr.pEndpointPrev;
184
185	Assert(!pNext \|\| pNext->AioMgr.cReqsPerSec <= pEpCurr->AioMgr.cReqsPerSec);
186	Assert(!pPrev \|\| pPrev->AioMgr.cReqsPerSec >= pEpCurr->AioMgr.cReqsPerSec);
187
188	pEpCurr = pNext;
189	}
190
191	AssertMsg(cEndpoints == pAioMgr->cEndpoints, ("Endpoints lost during sort!\n"));
192
193	#endif
194	}
195	#endif /* currently unused */
196
197	/**
198	* Removes an endpoint from the currently assigned manager.
199	*
200	* @returns TRUE if there are still requests pending on the current manager for this endpoint.
201	* FALSE otherwise.
202	* @param pEndpointRemove The endpoint to remove.
203	*/
204	static bool pdmacFileAioMgrNormalRemoveEndpoint(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointRemove)
205	{
206	PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEndpointRemove->AioMgr.pEndpointPrev;
207	PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEndpointRemove->AioMgr.pEndpointNext;
208	PPDMACEPFILEMGR pAioMgr = pEndpointRemove->pAioMgr;
209
210	pAioMgr->cEndpoints--;
211
212	if (pPrev)
213	pPrev->AioMgr.pEndpointNext = pNext;
214	else
215	pAioMgr->pEndpointsHead = pNext;
216
217	if (pNext)
218	pNext->AioMgr.pEndpointPrev = pPrev;
219
220	/* Make sure that there is no request pending on this manager for the endpoint. */
221	if (!pEndpointRemove->AioMgr.cRequestsActive)
222	{
223	Assert(!pEndpointRemove->pFlushReq);
224
225	/* Reopen the file so that the new endpoint can re-associate with the file */
226	RTFileClose(pEndpointRemove->hFile);
227	int rc = RTFileOpen(&pEndpointRemove->hFile, pEndpointRemove->Core.pszUri, pEndpointRemove->fFlags);
228	AssertRC(rc);
229	return false;
230	}
231
232	return true;
233	}
234
235	#if 0 /* currently unused */
236
237	static bool pdmacFileAioMgrNormalIsBalancePossible(PPDMACEPFILEMGR pAioMgr)
238	{
239	/* Balancing doesn't make sense with only one endpoint. */
240	if (pAioMgr->cEndpoints == 1)
241	return false;
242
243	/* Doesn't make sens to move endpoints if only one produces the whole load */
244	unsigned cEndpointsWithLoad = 0;
245
246	PPDMASYNCCOMPLETIONENDPOINTFILE pCurr = pAioMgr->pEndpointsHead;
247
248	while (pCurr)
249	{
250	if (pCurr->AioMgr.cReqsPerSec)
251	cEndpointsWithLoad++;
252
253	pCurr = pCurr->AioMgr.pEndpointNext;
254	}
255
256	return (cEndpointsWithLoad > 1);
257	}
258
259	/**
260	* Creates a new I/O manager and spreads the I/O load of the endpoints
261	* between the given I/O manager and the new one.
262	*
263	* @param pAioMgr The I/O manager with high I/O load.
264	*/
265	static void pdmacFileAioMgrNormalBalanceLoad(PPDMACEPFILEMGR pAioMgr)
266	{
267	/*
268	* Check if balancing would improve the situation.
269	*/
270	if (pdmacFileAioMgrNormalIsBalancePossible(pAioMgr))
271	{
272	PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pAioMgr->pEndpointsHead->Core.pEpClass;
273	PPDMACEPFILEMGR pAioMgrNew = NULL;
274
275	int rc = pdmacFileAioMgrCreate(pEpClassFile, &pAioMgrNew, PDMACEPFILEMGRTYPE_ASYNC);
276	if (RT_SUCCESS(rc))
277	{
278	/* We will sort the list by request count per second. */
279	pdmacFileAioMgrNormalEndpointsSortByLoad(pAioMgr);
280
281	/* Now move some endpoints to the new manager. */
282	unsigned cReqsHere = pAioMgr->pEndpointsHead->AioMgr.cReqsPerSec;
283	unsigned cReqsOther = 0;
284	PPDMASYNCCOMPLETIONENDPOINTFILE pCurr = pAioMgr->pEndpointsHead->AioMgr.pEndpointNext;
285
286	while (pCurr)
287	{
288	if (cReqsHere <= cReqsOther)
289	{
290	/*
291	* The other manager has more requests to handle now.
292	* We will keep the current endpoint.
293	*/
294	Log(("Keeping endpoint %#p{%s} with %u reqs/s\n", pCurr->Core.pszUri, pCurr->AioMgr.cReqsPerSec));
295	cReqsHere += pCurr->AioMgr.cReqsPerSec;
296	pCurr = pCurr->AioMgr.pEndpointNext;
297	}
298	else
299	{
300	/* Move to other endpoint. */
301	Log(("Moving endpoint %#p{%s} with %u reqs/s to other manager\n", pCurr, pCurr->Core.pszUri, pCurr->AioMgr.cReqsPerSec));
302	cReqsOther += pCurr->AioMgr.cReqsPerSec;
303
304	PPDMASYNCCOMPLETIONENDPOINTFILE pMove = pCurr;
305
306	pCurr = pCurr->AioMgr.pEndpointNext;
307
308	bool fReqsPending = pdmacFileAioMgrNormalRemoveEndpoint(pMove);
309
310	if (fReqsPending)
311	{
312	pMove->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_REMOVING;
313	pMove->AioMgr.fMoving = true;
314	pMove->AioMgr.pAioMgrDst = pAioMgrNew;
315	}
316	else
317	{
318	pMove->AioMgr.fMoving = false;
319	pMove->AioMgr.pAioMgrDst = NULL;
320	pdmacFileAioMgrAddEndpoint(pAioMgrNew, pMove);
321	}
322	}
323	}
324	}
325	else
326	{
327	/* Don't process further but leave a log entry about reduced performance. */
328	LogRel(("AIOMgr: Could not create new I/O manager (rc=%Rrc). Expect reduced performance\n", rc));
329	}
330	}
331	else
332	Log(("AIOMgr: Load balancing would not improve anything\n"));
333	}
334
335	#endif /* unused */
336
337	/**
338	* Increase the maximum number of active requests for the given I/O manager.
339	*
340	* @returns VBox status code.
341	* @param pAioMgr The I/O manager to grow.
342	*/
343	static int pdmacFileAioMgrNormalGrow(PPDMACEPFILEMGR pAioMgr)
344	{
345	LogFlowFunc(("pAioMgr=%#p\n", pAioMgr));
346
347	AssertMsg( pAioMgr->enmState == PDMACEPFILEMGRSTATE_GROWING
348	&& !pAioMgr->cRequestsActive,
349	("Invalid state of the I/O manager\n"));
350
351	#ifdef RT_OS_WINDOWS
352	/*
353	* Reopen the files of all assigned endpoints first so we can assign them to the new
354	* I/O context.
355	*/
356	PPDMASYNCCOMPLETIONENDPOINTFILE pCurr = pAioMgr->pEndpointsHead;
357
358	while (pCurr)
359	{
360	RTFileClose(pCurr->hFile);
361	int rc2 = RTFileOpen(&pCurr->hFile, pCurr->Core.pszUri, pCurr->fFlags); AssertRC(rc2);
362
363	pCurr = pCurr->AioMgr.pEndpointNext;
364	}
365	#endif
366
367	/* Create the new bigger context. */
368	pAioMgr->cRequestsActiveMax += PDMACEPFILEMGR_REQS_STEP;
369
370	RTFILEAIOCTX hAioCtxNew = NIL_RTFILEAIOCTX;
371	int rc = RTFileAioCtxCreate(&hAioCtxNew, RTFILEAIO_UNLIMITED_REQS, 0 /* fFlags */);
372	if (rc == VERR_OUT_OF_RANGE)
373	rc = RTFileAioCtxCreate(&hAioCtxNew, pAioMgr->cRequestsActiveMax, 0 /* fFlags */);
374
375	if (RT_SUCCESS(rc))
376	{
377	/* Close the old context. */
378	rc = RTFileAioCtxDestroy(pAioMgr->hAioCtx);
379	AssertRC(rc); /** @todo r=bird: Ignoring error code, will propagate. */
380
381	pAioMgr->hAioCtx = hAioCtxNew;
382
383	/* Create a new I/O task handle array */
384	uint32_t cReqEntriesNew = pAioMgr->cRequestsActiveMax + 1;
385	RTFILEAIOREQ pahReqNew = (RTFILEAIOREQ )RTMemAllocZ(cReqEntriesNew * sizeof(RTFILEAIOREQ));
386
387	if (pahReqNew)
388	{
389	/* Copy the cached request handles. */
390	for (uint32_t iReq = 0; iReq < pAioMgr->cReqEntries; iReq++)
391	pahReqNew[iReq] = pAioMgr->pahReqsFree[iReq];
392
393	RTMemFree(pAioMgr->pahReqsFree);
394	pAioMgr->pahReqsFree = pahReqNew;
395	pAioMgr->cReqEntries = cReqEntriesNew;
396	LogFlowFunc(("I/O manager increased to handle a maximum of %u requests\n",
397	pAioMgr->cRequestsActiveMax));
398	}
399	else
400	rc = VERR_NO_MEMORY;
401	}
402
403	#ifdef RT_OS_WINDOWS
404	/* Assign the file to the new context. */
405	pCurr = pAioMgr->pEndpointsHead;
406	while (pCurr)
407	{
408	rc = RTFileAioCtxAssociateWithFile(pAioMgr->hAioCtx, pCurr->hFile);
409	AssertRC(rc); /** @todo r=bird: Ignoring error code, will propagate. */
410
411	pCurr = pCurr->AioMgr.pEndpointNext;
412	}
413	#endif
414
415	if (RT_FAILURE(rc))
416	{
417	LogFlow(("Increasing size of the I/O manager failed with rc=%Rrc\n", rc));
418	pAioMgr->cRequestsActiveMax -= PDMACEPFILEMGR_REQS_STEP;
419	}
420
421	pAioMgr->enmState = PDMACEPFILEMGRSTATE_RUNNING;
422	LogFlowFunc(("returns rc=%Rrc\n", rc));
423
424	return rc;
425	}
426
427	/**
428	* Checks if a given status code is fatal.
429	* Non fatal errors can be fixed by migrating the endpoint to a
430	* failsafe manager.
431	*
432	* @returns true If the error is fatal and migrating to a failsafe manager doesn't help
433	* false If the error can be fixed by a migration. (image on NFS disk for example)
434	* @param rcReq The status code to check.
435	*/
436	DECLINLINE(bool) pdmacFileAioMgrNormalRcIsFatal(int rcReq)
437	{
438	return rcReq == VERR_DEV_IO_ERROR
439	\|\| rcReq == VERR_FILE_IO_ERROR
440	\|\| rcReq == VERR_DISK_IO_ERROR
441	\|\| rcReq == VERR_DISK_FULL
442	\|\| rcReq == VERR_FILE_TOO_BIG;
443	}
444
445	/**
446	* Error handler which will create the failsafe managers and destroy the failed I/O manager.
447	*
448	* @returns VBox status code
449	* @param pAioMgr The I/O manager the error occurred on.
450	* @param rc The error code.
451	* @param SRC_POS The source location of the error (use RT_SRC_POS).
452	*/
453	static int pdmacFileAioMgrNormalErrorHandler(PPDMACEPFILEMGR pAioMgr, int rc, RT_SRC_POS_DECL)
454	{
455	LogRel(("AIOMgr: I/O manager %#p encountered a critical error (rc=%Rrc) during operation. Falling back to failsafe mode. Expect reduced performance\n",
456	pAioMgr, rc));
457	LogRel(("AIOMgr: Error happened in %s:(%u){%s}\n", RT_SRC_POS_ARGS));
458	LogRel(("AIOMgr: Please contact the product vendor\n"));
459
460	PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pAioMgr->pEndpointsHead->Core.pEpClass;
461
462	pAioMgr->enmState = PDMACEPFILEMGRSTATE_FAULT;
463	ASMAtomicWriteU32((volatile uint32_t *)&pEpClassFile->enmMgrTypeOverride, PDMACEPFILEMGRTYPE_SIMPLE);
464
465	AssertMsgFailed(("Implement\n"));
466	return VINF_SUCCESS;
467	}
468
469	/**
470	* Put a list of tasks in the pending request list of an endpoint.
471	*/
472	DECLINLINE(void) pdmacFileAioMgrEpAddTaskList(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMACTASKFILE pTaskHead)
473	{
474	/* Add the rest of the tasks to the pending list */
475	if (!pEndpoint->AioMgr.pReqsPendingHead)
476	{
477	Assert(!pEndpoint->AioMgr.pReqsPendingTail);
478	pEndpoint->AioMgr.pReqsPendingHead = pTaskHead;
479	}
480	else
481	{
482	Assert(pEndpoint->AioMgr.pReqsPendingTail);
483	pEndpoint->AioMgr.pReqsPendingTail->pNext = pTaskHead;
484	}
485
486	/* Update the tail. */
487	while (pTaskHead->pNext)
488	pTaskHead = pTaskHead->pNext;
489
490	pEndpoint->AioMgr.pReqsPendingTail = pTaskHead;
491	pTaskHead->pNext = NULL;
492	}
493
494	/**
495	* Put one task in the pending request list of an endpoint.
496	*/
497	DECLINLINE(void) pdmacFileAioMgrEpAddTask(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMACTASKFILE pTask)
498	{
499	/* Add the rest of the tasks to the pending list */
500	if (!pEndpoint->AioMgr.pReqsPendingHead)
501	{
502	Assert(!pEndpoint->AioMgr.pReqsPendingTail);
503	pEndpoint->AioMgr.pReqsPendingHead = pTask;
504	}
505	else
506	{
507	Assert(pEndpoint->AioMgr.pReqsPendingTail);
508	pEndpoint->AioMgr.pReqsPendingTail->pNext = pTask;
509	}
510
511	pEndpoint->AioMgr.pReqsPendingTail = pTask;
512	pTask->pNext = NULL;
513	}
514
515	/**
516	* Allocates a async I/O request.
517	*
518	* @returns Handle to the request.
519	* @param pAioMgr The I/O manager.
520	*/
521	static RTFILEAIOREQ pdmacFileAioMgrNormalRequestAlloc(PPDMACEPFILEMGR pAioMgr)
522	{
523	/* Get a request handle. */
524	RTFILEAIOREQ hReq;
525	if (pAioMgr->iFreeEntry > 0)
526	{
527	pAioMgr->iFreeEntry--;
528	hReq = pAioMgr->pahReqsFree[pAioMgr->iFreeEntry];
529	pAioMgr->pahReqsFree[pAioMgr->iFreeEntry] = NIL_RTFILEAIOREQ;
530	Assert(hReq != NIL_RTFILEAIOREQ);
531	}
532	else
533	{
534	int rc = RTFileAioReqCreate(&hReq);
535	AssertRCReturn(rc, NIL_RTFILEAIOREQ);
536	}
537
538	return hReq;
539	}
540
541	/**
542	* Frees a async I/O request handle.
543	*
544	* @param pAioMgr The I/O manager.
545	* @param hReq The I/O request handle to free.
546	*/
547	static void pdmacFileAioMgrNormalRequestFree(PPDMACEPFILEMGR pAioMgr, RTFILEAIOREQ hReq)
548	{
549	Assert(pAioMgr->iFreeEntry < pAioMgr->cReqEntries);
550	Assert(pAioMgr->pahReqsFree[pAioMgr->iFreeEntry] == NIL_RTFILEAIOREQ);
551
552	pAioMgr->pahReqsFree[pAioMgr->iFreeEntry] = hReq;
553	pAioMgr->iFreeEntry++;
554	}
555
556	/**
557	* Wrapper around RTFIleAioCtxSubmit() which is also doing error handling.
558	*/
559	static int pdmacFileAioMgrNormalReqsEnqueue(PPDMACEPFILEMGR pAioMgr,
560	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
561	PRTFILEAIOREQ pahReqs, unsigned cReqs)
562	{
563	pAioMgr->cRequestsActive += cReqs;
564	pEndpoint->AioMgr.cRequestsActive += cReqs;
565
566	LogFlow(("Enqueuing %d requests. I/O manager has a total of %d active requests now\n", cReqs, pAioMgr->cRequestsActive));
567	LogFlow(("Endpoint has a total of %d active requests now\n", pEndpoint->AioMgr.cRequestsActive));
568
569	int rc = RTFileAioCtxSubmit(pAioMgr->hAioCtx, pahReqs, cReqs);
570	if (RT_FAILURE(rc))
571	{
572	if (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES)
573	{
574	PPDMASYNCCOMPLETIONEPCLASSFILE pEpClass = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass;
575
576	/* Append any not submitted task to the waiting list. */
577	for (size_t i = 0; i < cReqs; i++)
578	{
579	int rcReq = RTFileAioReqGetRC(pahReqs[i], NULL);
580
581	if (rcReq != VERR_FILE_AIO_IN_PROGRESS)
582	{
583	PPDMACTASKFILE pTask = (PPDMACTASKFILE)RTFileAioReqGetUser(pahReqs[i]);
584
585	Assert(pTask->hReq == pahReqs[i]);
586	pdmacFileAioMgrEpAddTask(pEndpoint, pTask);
587	pAioMgr->cRequestsActive--;
588	pEndpoint->AioMgr.cRequestsActive--;
589
590	if (pTask->enmTransferType == PDMACTASKFILETRANSFER_FLUSH)
591	{
592	/* Clear the pending flush */
593	Assert(pEndpoint->pFlushReq == pTask);
594	pEndpoint->pFlushReq = NULL;
595	}
596	}
597	}
598
599	pAioMgr->cRequestsActiveMax = pAioMgr->cRequestsActive;
600
601	/* Print an entry in the release log */
602	if (RT_UNLIKELY(!pEpClass->fOutOfResourcesWarningPrinted))
603	{
604	pEpClass->fOutOfResourcesWarningPrinted = true;
605	LogRel(("AIOMgr: Host limits number of active IO requests to %u. Expect a performance impact.\n",
606	pAioMgr->cRequestsActive));
607	}
608
609	LogFlow(("Removed requests. I/O manager has a total of %u active requests now\n", pAioMgr->cRequestsActive));
610	LogFlow(("Endpoint has a total of %u active requests now\n", pEndpoint->AioMgr.cRequestsActive));
611	rc = VINF_SUCCESS;
612	}
613	else /* Another kind of error happened (full disk, ...) */
614	{
615	/* An error happened. Find out which one caused the error and resubmit all other tasks. */
616	for (size_t i = 0; i < cReqs; i++)
617	{
618	int rcReq = RTFileAioReqGetRC(pahReqs[i], NULL);
619
620	if (rcReq == VERR_FILE_AIO_NOT_SUBMITTED)
621	{
622	/* We call ourself again to do any error handling which might come up now. */
623	rc = pdmacFileAioMgrNormalReqsEnqueue(pAioMgr, pEndpoint, &pahReqs[i], 1);
624	AssertRC(rc);
625	}
626	else if (rcReq != VERR_FILE_AIO_IN_PROGRESS)
627	pdmacFileAioMgrNormalReqCompleteRc(pAioMgr, pahReqs[i], rcReq, 0);
628	}
629
630
631	if ( pEndpoint->pFlushReq
632	&& !pAioMgr->cRequestsActive
633	&& !pEndpoint->fAsyncFlushSupported)
634	{
635	/*
636	* Complete a pending flush if we don't have requests enqueued and the host doesn't support
637	* the async flush API.
638	* Happens only if this we just noticed that this is not supported
639	* and the only active request was a flush.
640	*/
641	PPDMACTASKFILE pFlush = pEndpoint->pFlushReq;
642	pEndpoint->pFlushReq = NULL;
643	pFlush->pfnCompleted(pFlush, pFlush->pvUser, VINF_SUCCESS);
644	pdmacFileTaskFree(pEndpoint, pFlush);
645	}
646	}
647	}
648
649	return VINF_SUCCESS;
650	}
651
652	static bool pdmacFileAioMgrNormalIsRangeLocked(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
653	RTFOFF offStart, size_t cbRange,
654	PPDMACTASKFILE pTask, bool fAlignedReq)
655	{
656	AssertMsg( pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE
657	\|\| pTask->enmTransferType == PDMACTASKFILETRANSFER_READ,
658	("Invalid task type %d\n", pTask->enmTransferType));
659
660	/*
661	* If there is no unaligned request active and the current one is aligned
662	* just pass it through.
663	*/
664	if (!pEndpoint->AioMgr.cLockedReqsActive && fAlignedReq)
665	return false;
666
667	PPDMACFILERANGELOCK pRangeLock;
668	pRangeLock = (PPDMACFILERANGELOCK)RTAvlrFileOffsetRangeGet(pEndpoint->AioMgr.pTreeRangesLocked, offStart);
669	if (!pRangeLock)
670	{
671	pRangeLock = (PPDMACFILERANGELOCK)RTAvlrFileOffsetGetBestFit(pEndpoint->AioMgr.pTreeRangesLocked, offStart, true);
672	/* Check if we intersect with the range. */
673	if ( !pRangeLock
674	\|\| !( (pRangeLock->Core.Key) <= (offStart + (RTFOFF)cbRange - 1)
675	&& (pRangeLock->Core.KeyLast) >= offStart))
676	{
677	pRangeLock = NULL; /* False alarm */
678	}
679	}
680
681	/* Check whether we have one of the situations explained below */
682	if (pRangeLock)
683	{
684	/* Add to the list. */
685	pTask->pNext = NULL;
686
687	if (!pRangeLock->pWaitingTasksHead)
688	{
689	Assert(!pRangeLock->pWaitingTasksTail);
690	pRangeLock->pWaitingTasksHead = pTask;
691	pRangeLock->pWaitingTasksTail = pTask;
692	}
693	else
694	{
695	AssertPtr(pRangeLock->pWaitingTasksTail);
696	pRangeLock->pWaitingTasksTail->pNext = pTask;
697	pRangeLock->pWaitingTasksTail = pTask;
698	}
699	return true;
700	}
701
702	return false;
703	}
704
705	static int pdmacFileAioMgrNormalRangeLock(PPDMACEPFILEMGR pAioMgr,
706	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
707	RTFOFF offStart, size_t cbRange,
708	PPDMACTASKFILE pTask, bool fAlignedReq)
709	{
710	LogFlowFunc(("pAioMgr=%#p pEndpoint=%#p offStart=%RTfoff cbRange=%zu pTask=%#p\n",
711	pAioMgr, pEndpoint, offStart, cbRange, pTask));
712
713	AssertMsg(!pdmacFileAioMgrNormalIsRangeLocked(pEndpoint, offStart, cbRange, pTask, fAlignedReq),
714	("Range is already locked offStart=%RTfoff cbRange=%u\n",
715	offStart, cbRange));
716
717	/*
718	* If there is no unaligned request active and the current one is aligned
719	* just don't use the lock.
720	*/
721	if (!pEndpoint->AioMgr.cLockedReqsActive && fAlignedReq)
722	{
723	pTask->pRangeLock = NULL;
724	return VINF_SUCCESS;
725	}
726
727	PPDMACFILERANGELOCK pRangeLock = (PPDMACFILERANGELOCK)RTMemCacheAlloc(pAioMgr->hMemCacheRangeLocks);
728	if (!pRangeLock)
729	return VERR_NO_MEMORY;
730
731	/* Init the lock. */
732	pRangeLock->Core.Key = offStart;
733	pRangeLock->Core.KeyLast = offStart + cbRange - 1;
734	pRangeLock->cRefs = 1;
735	pRangeLock->fReadLock = pTask->enmTransferType == PDMACTASKFILETRANSFER_READ;
736	pRangeLock->pWaitingTasksHead = NULL;
737	pRangeLock->pWaitingTasksTail = NULL;
738
739	bool fInserted = RTAvlrFileOffsetInsert(pEndpoint->AioMgr.pTreeRangesLocked, &pRangeLock->Core);
740	AssertMsg(fInserted, ("Range lock was not inserted!\n")); NOREF(fInserted);
741
742	/* Let the task point to its lock. */
743	pTask->pRangeLock = pRangeLock;
744	pEndpoint->AioMgr.cLockedReqsActive++;
745
746	return VINF_SUCCESS;
747	}
748
749	static PPDMACTASKFILE pdmacFileAioMgrNormalRangeLockFree(PPDMACEPFILEMGR pAioMgr,
750	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
751	PPDMACFILERANGELOCK pRangeLock)
752	{
753	PPDMACTASKFILE pTasksWaitingHead;
754
755	LogFlowFunc(("pAioMgr=%#p pEndpoint=%#p pRangeLock=%#p\n",
756	pAioMgr, pEndpoint, pRangeLock));
757
758	/* pRangeLock can be NULL if there was no lock assigned with the task. */
759	if (!pRangeLock)
760	return NULL;
761
762	Assert(pRangeLock->cRefs == 1);
763
764	RTAvlrFileOffsetRemove(pEndpoint->AioMgr.pTreeRangesLocked, pRangeLock->Core.Key);
765	pTasksWaitingHead = pRangeLock->pWaitingTasksHead;
766	pRangeLock->pWaitingTasksHead = NULL;
767	pRangeLock->pWaitingTasksTail = NULL;
768	RTMemCacheFree(pAioMgr->hMemCacheRangeLocks, pRangeLock);
769	pEndpoint->AioMgr.cLockedReqsActive--;
770
771	return pTasksWaitingHead;
772	}
773
774	static int pdmacFileAioMgrNormalTaskPrepareBuffered(PPDMACEPFILEMGR pAioMgr,
775	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
776	PPDMACTASKFILE pTask, PRTFILEAIOREQ phReq)
777	{
778	AssertMsg( pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE
779	\|\| (uint64_t)(pTask->Off + pTask->DataSeg.cbSeg) <= pEndpoint->cbFile,
780	("Read exceeds file size offStart=%RTfoff cbToTransfer=%d cbFile=%llu\n",
781	pTask->Off, pTask->DataSeg.cbSeg, pEndpoint->cbFile));
782
783	pTask->fPrefetch = false;
784	pTask->cbBounceBuffer = 0;
785
786	/*
787	* Before we start to setup the request we have to check whether there is a task
788	* already active which range intersects with ours. We have to defer execution
789	* of this task in two cases:
790	* - The pending task is a write and the current is either read or write
791	* - The pending task is a read and the current task is a write task.
792	*
793	* To check whether a range is currently "locked" we use the AVL tree where every pending task
794	* is stored by its file offset range. The current task will be added to the active task
795	* and will be executed when the active one completes. (The method below
796	* which checks whether a range is already used will add the task)
797	*
798	* This is necessary because of the requirement to align all requests to a 512 boundary
799	* which is enforced by the host OS (Linux and Windows atm). It is possible that
800	* we have to process unaligned tasks and need to align them using bounce buffers.
801	* While the data is fetched from the file another request might arrive writing to
802	* the same range. This will result in data corruption if both are executed concurrently.
803	*/
804	int rc = VINF_SUCCESS;
805	bool fLocked = pdmacFileAioMgrNormalIsRangeLocked(pEndpoint, pTask->Off, pTask->DataSeg.cbSeg, pTask,
806	true /* fAlignedReq */);
807	if (!fLocked)
808	{
809	/* Get a request handle. */
810	RTFILEAIOREQ hReq = pdmacFileAioMgrNormalRequestAlloc(pAioMgr);
811	AssertMsg(hReq != NIL_RTFILEAIOREQ, ("Out of request handles\n"));
812
813	if (pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE)
814	{
815	/* Grow the file if needed. */
816	if (RT_UNLIKELY((uint64_t)(pTask->Off + pTask->DataSeg.cbSeg) > pEndpoint->cbFile))
817	{
818	ASMAtomicWriteU64(&pEndpoint->cbFile, pTask->Off + pTask->DataSeg.cbSeg);
819	RTFileSetSize(pEndpoint->hFile, pTask->Off + pTask->DataSeg.cbSeg);
820	}
821
822	rc = RTFileAioReqPrepareWrite(hReq, pEndpoint->hFile,
823	pTask->Off, pTask->DataSeg.pvSeg,
824	pTask->DataSeg.cbSeg, pTask);
825	}
826	else
827	rc = RTFileAioReqPrepareRead(hReq, pEndpoint->hFile,
828	pTask->Off, pTask->DataSeg.pvSeg,
829	pTask->DataSeg.cbSeg, pTask);
830	AssertRC(rc);
831
832	rc = pdmacFileAioMgrNormalRangeLock(pAioMgr, pEndpoint, pTask->Off,
833	pTask->DataSeg.cbSeg,
834	pTask, true /* fAlignedReq */);
835
836	if (RT_SUCCESS(rc))
837	{
838	pTask->hReq = hReq;
839	*phReq = hReq;
840	}
841	}
842	else
843	LogFlow(("Task %#p was deferred because the access range is locked\n", pTask));
844
845	return rc;
846	}
847
848	static int pdmacFileAioMgrNormalTaskPrepareNonBuffered(PPDMACEPFILEMGR pAioMgr,
849	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
850	PPDMACTASKFILE pTask, PRTFILEAIOREQ phReq)
851	{
852	/*
853	* Check if the alignment requirements are met.
854	* Offset, transfer size and buffer address
855	* need to be on a 512 boundary.
856	*/
857	RTFOFF offStart = pTask->Off & ~(RTFOFF)(512-1);
858	size_t cbToTransfer = RT_ALIGN_Z(pTask->DataSeg.cbSeg + (pTask->Off - offStart), 512);
859	PDMACTASKFILETRANSFER enmTransferType = pTask->enmTransferType;
860	bool fAlignedReq = cbToTransfer == pTask->DataSeg.cbSeg
861	&& offStart == pTask->Off;
862
863	AssertMsg( pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE
864	\|\| (uint64_t)(offStart + cbToTransfer) <= pEndpoint->cbFile,
865	("Read exceeds file size offStart=%RTfoff cbToTransfer=%d cbFile=%llu\n",
866	offStart, cbToTransfer, pEndpoint->cbFile));
867
868	pTask->fPrefetch = false;
869
870	/*
871	* Before we start to setup the request we have to check whether there is a task
872	* already active which range intersects with ours. We have to defer execution
873	* of this task in two cases:
874	* - The pending task is a write and the current is either read or write
875	* - The pending task is a read and the current task is a write task.
876	*
877	* To check whether a range is currently "locked" we use the AVL tree where every pending task
878	* is stored by its file offset range. The current task will be added to the active task
879	* and will be executed when the active one completes. (The method below
880	* which checks whether a range is already used will add the task)
881	*
882	* This is necessary because of the requirement to align all requests to a 512 boundary
883	* which is enforced by the host OS (Linux and Windows atm). It is possible that
884	* we have to process unaligned tasks and need to align them using bounce buffers.
885	* While the data is fetched from the file another request might arrive writing to
886	* the same range. This will result in data corruption if both are executed concurrently.
887	*/
888	int rc = VINF_SUCCESS;
889	bool fLocked = pdmacFileAioMgrNormalIsRangeLocked(pEndpoint, offStart, cbToTransfer, pTask, fAlignedReq);
890	if (!fLocked)
891	{
892	PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass;
893	void *pvBuf = pTask->DataSeg.pvSeg;
894
895	/* Get a request handle. */
896	RTFILEAIOREQ hReq = pdmacFileAioMgrNormalRequestAlloc(pAioMgr);
897	AssertMsg(hReq != NIL_RTFILEAIOREQ, ("Out of request handles\n"));
898
899	if ( !fAlignedReq
900	\|\| ((pEpClassFile->uBitmaskAlignment & (RTR3UINTPTR)pvBuf) != (RTR3UINTPTR)pvBuf))
901	{
902	LogFlow(("Using bounce buffer for task %#p cbToTransfer=%zd cbSeg=%zd offStart=%RTfoff off=%RTfoff\n",
903	pTask, cbToTransfer, pTask->DataSeg.cbSeg, offStart, pTask->Off));
904
905	/* Create bounce buffer. */
906	pTask->cbBounceBuffer = cbToTransfer;
907
908	AssertMsg(pTask->Off >= offStart, ("Overflow in calculation Off=%llu offStart=%llu\n",
909	pTask->Off, offStart));
910	pTask->offBounceBuffer = pTask->Off - offStart;
911
912	/** @todo I think we need something like a RTMemAllocAligned method here.
913	* Current assumption is that the maximum alignment is 4096byte
914	* (GPT disk on Windows)
915	* so we can use RTMemPageAlloc here.
916	*/
917	pTask->pvBounceBuffer = RTMemPageAlloc(cbToTransfer);
918	if (RT_LIKELY(pTask->pvBounceBuffer))
919	{
920	pvBuf = pTask->pvBounceBuffer;
921
922	if (pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE)
923	{
924	if ( RT_UNLIKELY(cbToTransfer != pTask->DataSeg.cbSeg)
925	\|\| RT_UNLIKELY(offStart != pTask->Off))
926	{
927	/* We have to fill the buffer first before we can update the data. */
928	LogFlow(("Prefetching data for task %#p\n", pTask));
929	pTask->fPrefetch = true;
930	enmTransferType = PDMACTASKFILETRANSFER_READ;
931	}
932	else
933	memcpy(pvBuf, pTask->DataSeg.pvSeg, pTask->DataSeg.cbSeg);
934	}
935	}
936	else
937	rc = VERR_NO_MEMORY;
938	}
939	else
940	pTask->cbBounceBuffer = 0;
941
942	if (RT_SUCCESS(rc))
943	{
944	AssertMsg((pEpClassFile->uBitmaskAlignment & (RTR3UINTPTR)pvBuf) == (RTR3UINTPTR)pvBuf,
945	("AIO: Alignment restrictions not met! pvBuf=%p uBitmaskAlignment=%p\n", pvBuf, pEpClassFile->uBitmaskAlignment));
946
947	if (enmTransferType == PDMACTASKFILETRANSFER_WRITE)
948	{
949	/* Grow the file if needed. */
950	if (RT_UNLIKELY((uint64_t)(pTask->Off + pTask->DataSeg.cbSeg) > pEndpoint->cbFile))
951	{
952	ASMAtomicWriteU64(&pEndpoint->cbFile, pTask->Off + pTask->DataSeg.cbSeg);
953	RTFileSetSize(pEndpoint->hFile, pTask->Off + pTask->DataSeg.cbSeg);
954	}
955
956	rc = RTFileAioReqPrepareWrite(hReq, pEndpoint->hFile,
957	offStart, pvBuf, cbToTransfer, pTask);
958	}
959	else
960	rc = RTFileAioReqPrepareRead(hReq, pEndpoint->hFile,
961	offStart, pvBuf, cbToTransfer, pTask);
962	AssertRC(rc);
963
964	rc = pdmacFileAioMgrNormalRangeLock(pAioMgr, pEndpoint, offStart, cbToTransfer, pTask, fAlignedReq);
965	if (RT_SUCCESS(rc))
966	{
967	pTask->hReq = hReq;
968	*phReq = hReq;
969	}
970	else
971	{
972	/* Cleanup */
973	if (pTask->cbBounceBuffer)
974	RTMemPageFree(pTask->pvBounceBuffer, pTask->cbBounceBuffer);
975	}
976	}
977	}
978	else
979	LogFlow(("Task %#p was deferred because the access range is locked\n", pTask));
980
981	return rc;
982	}
983
984	static int pdmacFileAioMgrNormalProcessTaskList(PPDMACTASKFILE pTaskHead,
985	PPDMACEPFILEMGR pAioMgr,
986	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
987	{
988	RTFILEAIOREQ apReqs[20];
989	unsigned cRequests = 0;
990	int rc = VINF_SUCCESS;
991
992	AssertMsg(pEndpoint->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE,
993	("Trying to process request lists of a non active endpoint!\n"));
994
995	/* Go through the list and queue the requests until we get a flush request */
996	while ( pTaskHead
997	&& !pEndpoint->pFlushReq
998	&& (pAioMgr->cRequestsActive + cRequests < pAioMgr->cRequestsActiveMax)
999	&& RT_SUCCESS(rc))
1000	{
1001	RTMSINTERVAL msWhenNext;
1002	PPDMACTASKFILE pCurr = pTaskHead;
1003
1004	if (!pdmacEpIsTransferAllowed(&pEndpoint->Core, (uint32_t)pCurr->DataSeg.cbSeg, &msWhenNext))
1005	{
1006	pAioMgr->msBwLimitExpired = RT_MIN(pAioMgr->msBwLimitExpired, msWhenNext);
1007	break;
1008	}
1009
1010	pTaskHead = pTaskHead->pNext;
1011
1012	pCurr->pNext = NULL;
1013
1014	AssertMsg(RT_VALID_PTR(pCurr->pEndpoint) && pCurr->pEndpoint == pEndpoint,
1015	("Endpoints do not match\n"));
1016
1017	switch (pCurr->enmTransferType)
1018	{
1019	case PDMACTASKFILETRANSFER_FLUSH:
1020	{
1021	/* If there is no data transfer request this flush request finished immediately. */
1022	if (pEndpoint->fAsyncFlushSupported)
1023	{
1024	/* Issue a flush to the host. */
1025	RTFILEAIOREQ hReq = pdmacFileAioMgrNormalRequestAlloc(pAioMgr);
1026	AssertMsg(hReq != NIL_RTFILEAIOREQ, ("Out of request handles\n"));
1027
1028	LogFlow(("Flush request %#p\n", hReq));
1029
1030	rc = RTFileAioReqPrepareFlush(hReq, pEndpoint->hFile, pCurr);
1031	if (RT_FAILURE(rc))
1032	{
1033	if (rc == VERR_NOT_SUPPORTED)
1034	LogRel(("AIOMgr: Async flushes not supported\n"));
1035	else
1036	LogRel(("AIOMgr: Preparing flush failed with %Rrc, disabling async flushes\n", rc));
1037	pEndpoint->fAsyncFlushSupported = false;
1038	pdmacFileAioMgrNormalRequestFree(pAioMgr, hReq);
1039	rc = VINF_SUCCESS; /* Fake success */
1040	}
1041	else
1042	{
1043	pCurr->hReq = hReq;
1044	apReqs[cRequests] = hReq;
1045	pEndpoint->AioMgr.cReqsProcessed++;
1046	cRequests++;
1047	}
1048	}
1049
1050	if ( !pEndpoint->AioMgr.cRequestsActive
1051	&& !pEndpoint->fAsyncFlushSupported)
1052	{
1053	pCurr->pfnCompleted(pCurr, pCurr->pvUser, VINF_SUCCESS);
1054	pdmacFileTaskFree(pEndpoint, pCurr);
1055	}
1056	else
1057	{
1058	Assert(!pEndpoint->pFlushReq);
1059	pEndpoint->pFlushReq = pCurr;
1060	}
1061	break;
1062	}
1063	case PDMACTASKFILETRANSFER_READ:
1064	case PDMACTASKFILETRANSFER_WRITE:
1065	{
1066	RTFILEAIOREQ hReq = NIL_RTFILEAIOREQ;
1067
1068	if (pCurr->hReq == NIL_RTFILEAIOREQ)
1069	{
1070	if (pEndpoint->enmBackendType == PDMACFILEEPBACKEND_BUFFERED)
1071	rc = pdmacFileAioMgrNormalTaskPrepareBuffered(pAioMgr, pEndpoint, pCurr, &hReq);
1072	else if (pEndpoint->enmBackendType == PDMACFILEEPBACKEND_NON_BUFFERED)
1073	rc = pdmacFileAioMgrNormalTaskPrepareNonBuffered(pAioMgr, pEndpoint, pCurr, &hReq);
1074	else
1075	AssertMsgFailed(("Invalid backend type %d\n", pEndpoint->enmBackendType));
1076
1077	AssertRC(rc);
1078	}
1079	else
1080	{
1081	LogFlow(("Task %#p has I/O request %#p already\n", pCurr, pCurr->hReq));
1082	hReq = pCurr->hReq;
1083	}
1084
1085	LogFlow(("Read/Write request %#p\n", hReq));
1086
1087	if (hReq != NIL_RTFILEAIOREQ)
1088	{
1089	apReqs[cRequests] = hReq;
1090	cRequests++;
1091	}
1092	break;
1093	}
1094	default:
1095	AssertMsgFailed(("Invalid transfer type %d\n", pCurr->enmTransferType));
1096	} /* switch transfer type */
1097
1098	/* Queue the requests if the array is full. */
1099	if (cRequests == RT_ELEMENTS(apReqs))
1100	{
1101	rc = pdmacFileAioMgrNormalReqsEnqueue(pAioMgr, pEndpoint, apReqs, cRequests);
1102	cRequests = 0;
1103	AssertMsg(RT_SUCCESS(rc) \|\| (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES),
1104	("Unexpected return code\n"));
1105	}
1106	}
1107
1108	if (cRequests)
1109	{
1110	rc = pdmacFileAioMgrNormalReqsEnqueue(pAioMgr, pEndpoint, apReqs, cRequests);
1111	AssertMsg(RT_SUCCESS(rc) \|\| (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES),
1112	("Unexpected return code rc=%Rrc\n", rc));
1113	}
1114
1115	if (pTaskHead)
1116	{
1117	/* Add the rest of the tasks to the pending list */
1118	pdmacFileAioMgrEpAddTaskList(pEndpoint, pTaskHead);
1119
1120	if (RT_UNLIKELY( pAioMgr->cRequestsActiveMax == pAioMgr->cRequestsActive
1121	&& !pEndpoint->pFlushReq))
1122	{
1123	#if 0
1124	/*
1125	* The I/O manager has no room left for more requests
1126	* but there are still requests to process.
1127	* Create a new I/O manager and let it handle some endpoints.
1128	*/
1129	pdmacFileAioMgrNormalBalanceLoad(pAioMgr);
1130	#else
1131	/* Grow the I/O manager */
1132	pAioMgr->enmState = PDMACEPFILEMGRSTATE_GROWING;
1133	#endif
1134	}
1135	}
1136
1137	/* Insufficient resources are not fatal. */
1138	if (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES)
1139	rc = VINF_SUCCESS;
1140
1141	return rc;
1142	}
1143
1144	/**
1145	* Adds all pending requests for the given endpoint
1146	* until a flush request is encountered or there is no
1147	* request anymore.
1148	*
1149	* @returns VBox status code.
1150	* @param pAioMgr The async I/O manager for the endpoint
1151	* @param pEndpoint The endpoint to get the requests from.
1152	*/
1153	static int pdmacFileAioMgrNormalQueueReqs(PPDMACEPFILEMGR pAioMgr,
1154	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
1155	{
1156	int rc = VINF_SUCCESS;
1157	PPDMACTASKFILE pTasksHead = NULL;
1158
1159	AssertMsg(pEndpoint->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE,
1160	("Trying to process request lists of a non active endpoint!\n"));
1161
1162	Assert(!pEndpoint->pFlushReq);
1163
1164	/* Check the pending list first */
1165	if (pEndpoint->AioMgr.pReqsPendingHead)
1166	{
1167	LogFlow(("Queuing pending requests first\n"));
1168
1169	pTasksHead = pEndpoint->AioMgr.pReqsPendingHead;
1170	/*
1171	* Clear the list as the processing routine will insert them into the list
1172	* again if it gets a flush request.
1173	*/
1174	pEndpoint->AioMgr.pReqsPendingHead = NULL;
1175	pEndpoint->AioMgr.pReqsPendingTail = NULL;
1176	rc = pdmacFileAioMgrNormalProcessTaskList(pTasksHead, pAioMgr, pEndpoint);
1177	AssertRC(rc); /** @todo r=bird: status code potentially overwritten. */
1178	}
1179
1180	if (!pEndpoint->pFlushReq && !pEndpoint->AioMgr.pReqsPendingHead)
1181	{
1182	/* Now the request queue. */
1183	pTasksHead = pdmacFileEpGetNewTasks(pEndpoint);
1184	if (pTasksHead)
1185	{
1186	rc = pdmacFileAioMgrNormalProcessTaskList(pTasksHead, pAioMgr, pEndpoint);
1187	AssertRC(rc);
1188	}
1189	}
1190
1191	return rc;
1192	}
1193
1194	static int pdmacFileAioMgrNormalProcessBlockingEvent(PPDMACEPFILEMGR pAioMgr)
1195	{
1196	int rc = VINF_SUCCESS;
1197	bool fNotifyWaiter = false;
1198
1199	LogFlowFunc((": Enter\n"));
1200
1201	Assert(pAioMgr->fBlockingEventPending);
1202
1203	switch (pAioMgr->enmBlockingEvent)
1204	{
1205	case PDMACEPFILEAIOMGRBLOCKINGEVENT_ADD_ENDPOINT:
1206	{
1207	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointNew = ASMAtomicReadPtrT(&pAioMgr->BlockingEventData.AddEndpoint.pEndpoint, PPDMASYNCCOMPLETIONENDPOINTFILE);
1208	AssertMsg(RT_VALID_PTR(pEndpointNew), ("Adding endpoint event without a endpoint to add\n"));
1209
1210	pEndpointNew->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE;
1211
1212	pEndpointNew->AioMgr.pEndpointNext = pAioMgr->pEndpointsHead;
1213	pEndpointNew->AioMgr.pEndpointPrev = NULL;
1214	if (pAioMgr->pEndpointsHead)
1215	pAioMgr->pEndpointsHead->AioMgr.pEndpointPrev = pEndpointNew;
1216	pAioMgr->pEndpointsHead = pEndpointNew;
1217
1218	/* Assign the completion point to this file. */
1219	rc = RTFileAioCtxAssociateWithFile(pAioMgr->hAioCtx, pEndpointNew->hFile);
1220	fNotifyWaiter = true;
1221	pAioMgr->cEndpoints++;
1222	break;
1223	}
1224	case PDMACEPFILEAIOMGRBLOCKINGEVENT_REMOVE_ENDPOINT:
1225	{
1226	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointRemove = ASMAtomicReadPtrT(&pAioMgr->BlockingEventData.RemoveEndpoint.pEndpoint, PPDMASYNCCOMPLETIONENDPOINTFILE);
1227	AssertMsg(RT_VALID_PTR(pEndpointRemove), ("Removing endpoint event without a endpoint to remove\n"));
1228
1229	pEndpointRemove->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_REMOVING;
1230	fNotifyWaiter = !pdmacFileAioMgrNormalRemoveEndpoint(pEndpointRemove);
1231	break;
1232	}
1233	case PDMACEPFILEAIOMGRBLOCKINGEVENT_CLOSE_ENDPOINT:
1234	{
1235	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointClose = ASMAtomicReadPtrT(&pAioMgr->BlockingEventData.CloseEndpoint.pEndpoint, PPDMASYNCCOMPLETIONENDPOINTFILE);
1236	AssertMsg(RT_VALID_PTR(pEndpointClose), ("Close endpoint event without a endpoint to close\n"));
1237
1238	if (pEndpointClose->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE)
1239	{
1240	LogFlowFunc((": Closing endpoint %#p{%s}\n", pEndpointClose, pEndpointClose->Core.pszUri));
1241
1242	/* Make sure all tasks finished. Process the queues a last time first. */
1243	rc = pdmacFileAioMgrNormalQueueReqs(pAioMgr, pEndpointClose);
1244	AssertRC(rc);
1245
1246	pEndpointClose->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_CLOSING;
1247	fNotifyWaiter = !pdmacFileAioMgrNormalRemoveEndpoint(pEndpointClose);
1248	}
1249	else if ( (pEndpointClose->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_CLOSING)
1250	&& (!pEndpointClose->AioMgr.cRequestsActive))
1251	fNotifyWaiter = true;
1252	break;
1253	}
1254	case PDMACEPFILEAIOMGRBLOCKINGEVENT_SHUTDOWN:
1255	{
1256	pAioMgr->enmState = PDMACEPFILEMGRSTATE_SHUTDOWN;
1257	if (!pAioMgr->cRequestsActive)
1258	fNotifyWaiter = true;
1259	break;
1260	}
1261	case PDMACEPFILEAIOMGRBLOCKINGEVENT_SUSPEND:
1262	{
1263	pAioMgr->enmState = PDMACEPFILEMGRSTATE_SUSPENDING;
1264	break;
1265	}
1266	case PDMACEPFILEAIOMGRBLOCKINGEVENT_RESUME:
1267	{
1268	pAioMgr->enmState = PDMACEPFILEMGRSTATE_RUNNING;
1269	fNotifyWaiter = true;
1270	break;
1271	}
1272	default:
1273	AssertReleaseMsgFailed(("Invalid event type %d\n", pAioMgr->enmBlockingEvent));
1274	}
1275
1276	if (fNotifyWaiter)
1277	{
1278	ASMAtomicWriteBool(&pAioMgr->fBlockingEventPending, false);
1279	pAioMgr->enmBlockingEvent = PDMACEPFILEAIOMGRBLOCKINGEVENT_INVALID;
1280
1281	/* Release the waiting thread. */
1282	LogFlow(("Signalling waiter\n"));
1283	rc = RTSemEventSignal(pAioMgr->EventSemBlock);
1284	AssertRC(rc);
1285	}
1286
1287	LogFlowFunc((": Leave\n"));
1288	return rc;
1289	}
1290
1291	/**
1292	* Checks all endpoints for pending events or new requests.
1293	*
1294	* @returns VBox status code.
1295	* @param pAioMgr The I/O manager handle.
1296	*/
1297	static int pdmacFileAioMgrNormalCheckEndpoints(PPDMACEPFILEMGR pAioMgr)
1298	{
1299	/* Check the assigned endpoints for new tasks if there isn't a flush request active at the moment. */
1300	int rc = VINF_SUCCESS;
1301	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint = pAioMgr->pEndpointsHead;
1302
1303	pAioMgr->msBwLimitExpired = RT_INDEFINITE_WAIT;
1304
1305	while (pEndpoint)
1306	{
1307	if (!pEndpoint->pFlushReq
1308	&& (pEndpoint->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE)
1309	&& !pEndpoint->AioMgr.fMoving)
1310	{
1311	rc = pdmacFileAioMgrNormalQueueReqs(pAioMgr, pEndpoint);
1312	if (RT_FAILURE(rc))
1313	return rc;
1314	}
1315	else if ( !pEndpoint->AioMgr.cRequestsActive
1316	&& pEndpoint->enmState != PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE)
1317	{
1318	/* Reopen the file so that the new endpoint can re-associate with the file */
1319	RTFileClose(pEndpoint->hFile);
1320	rc = RTFileOpen(&pEndpoint->hFile, pEndpoint->Core.pszUri, pEndpoint->fFlags);
1321	AssertRC(rc);
1322
1323	if (pEndpoint->AioMgr.fMoving)
1324	{
1325	pEndpoint->AioMgr.fMoving = false;
1326	pdmacFileAioMgrAddEndpoint(pEndpoint->AioMgr.pAioMgrDst, pEndpoint);
1327	}
1328	else
1329	{
1330	Assert(pAioMgr->fBlockingEventPending);
1331	ASMAtomicWriteBool(&pAioMgr->fBlockingEventPending, false);
1332
1333	/* Release the waiting thread. */
1334	LogFlow(("Signalling waiter\n"));
1335	rc = RTSemEventSignal(pAioMgr->EventSemBlock);
1336	AssertRC(rc);
1337	}
1338	}
1339
1340	pEndpoint = pEndpoint->AioMgr.pEndpointNext;
1341	}
1342
1343	return rc;
1344	}
1345
1346	/**
1347	* Wrapper around pdmacFileAioMgrNormalReqCompleteRc().
1348	*/
1349	static void pdmacFileAioMgrNormalReqComplete(PPDMACEPFILEMGR pAioMgr, RTFILEAIOREQ hReq)
1350	{
1351	size_t cbTransfered = 0;
1352	int rcReq = RTFileAioReqGetRC(hReq, &cbTransfered);
1353
1354	pdmacFileAioMgrNormalReqCompleteRc(pAioMgr, hReq, rcReq, cbTransfered);
1355	}
1356
1357	static void pdmacFileAioMgrNormalReqCompleteRc(PPDMACEPFILEMGR pAioMgr, RTFILEAIOREQ hReq,
1358	int rcReq, size_t cbTransfered)
1359	{
1360	int rc = VINF_SUCCESS;
1361	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint;
1362	PPDMACTASKFILE pTask = (PPDMACTASKFILE)RTFileAioReqGetUser(hReq);
1363	PPDMACTASKFILE pTasksWaiting;
1364
1365	LogFlowFunc(("pAioMgr=%#p hReq=%#p\n", pAioMgr, hReq));
1366
1367	pEndpoint = pTask->pEndpoint;
1368
1369	pTask->hReq = NIL_RTFILEAIOREQ;
1370
1371	pAioMgr->cRequestsActive--;
1372	pEndpoint->AioMgr.cRequestsActive--;
1373	pEndpoint->AioMgr.cReqsProcessed++;
1374
1375	/*
1376	* It is possible that the request failed on Linux with kernels < 2.6.23
1377	* if the passed buffer was allocated with remap_pfn_range or if the file
1378	* is on an NFS endpoint which does not support async and direct I/O at the same time.
1379	* The endpoint will be migrated to a failsafe manager in case a request fails.
1380	*/
1381	if (RT_FAILURE(rcReq))
1382	{
1383	/* Free bounce buffers and the IPRT request. */
1384	pdmacFileAioMgrNormalRequestFree(pAioMgr, hReq);
1385
1386	if (pTask->enmTransferType == PDMACTASKFILETRANSFER_FLUSH)
1387	{
1388	LogRel(("AIOMgr: Flush failed with %Rrc, disabling async flushes\n", rcReq));
1389	pEndpoint->fAsyncFlushSupported = false;
1390	AssertMsg(pEndpoint->pFlushReq == pTask, ("Failed flush request doesn't match active one\n"));
1391	/* The other method will take over now. */
1392
1393	pEndpoint->pFlushReq = NULL;
1394	/* Call completion callback */
1395	LogFlow(("Flush task=%#p completed with %Rrc\n", pTask, VINF_SUCCESS));
1396	pTask->pfnCompleted(pTask, pTask->pvUser, VINF_SUCCESS);
1397	pdmacFileTaskFree(pEndpoint, pTask);
1398	}
1399	else
1400	{
1401	/* Free the lock and process pending tasks if necessary */
1402	pTasksWaiting = pdmacFileAioMgrNormalRangeLockFree(pAioMgr, pEndpoint, pTask->pRangeLock);
1403	rc = pdmacFileAioMgrNormalProcessTaskList(pTasksWaiting, pAioMgr, pEndpoint);
1404	AssertRC(rc);
1405
1406	if (pTask->cbBounceBuffer)
1407	RTMemPageFree(pTask->pvBounceBuffer, pTask->cbBounceBuffer);
1408
1409	/*
1410	* Fatal errors are reported to the guest and non-fatal errors
1411	* will cause a migration to the failsafe manager in the hope
1412	* that the error disappears.
1413	*/
1414	if (!pdmacFileAioMgrNormalRcIsFatal(rcReq))
1415	{
1416	/* Queue the request on the pending list. */
1417	pTask->pNext = pEndpoint->AioMgr.pReqsPendingHead;
1418	pEndpoint->AioMgr.pReqsPendingHead = pTask;
1419
1420	/* Create a new failsafe manager if necessary. */
1421	if (!pEndpoint->AioMgr.fMoving)
1422	{
1423	PPDMACEPFILEMGR pAioMgrFailsafe;
1424
1425	LogRel(("%s: Request %#p failed with rc=%Rrc, migrating endpoint %s to failsafe manager.\n",
1426	RTThreadGetName(pAioMgr->Thread), pTask, rcReq, pEndpoint->Core.pszUri));
1427
1428	pEndpoint->AioMgr.fMoving = true;
1429
1430	rc = pdmacFileAioMgrCreate((PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass,
1431	&pAioMgrFailsafe, PDMACEPFILEMGRTYPE_SIMPLE);
1432	AssertRC(rc);
1433
1434	pEndpoint->AioMgr.pAioMgrDst = pAioMgrFailsafe;
1435
1436	/* Update the flags to open the file with. Disable async I/O and enable the host cache. */
1437	pEndpoint->fFlags &= ~(RTFILE_O_ASYNC_IO \| RTFILE_O_NO_CACHE);
1438	}
1439
1440	/* If this was the last request for the endpoint migrate it to the new manager. */
1441	if (!pEndpoint->AioMgr.cRequestsActive)
1442	{
1443	bool fReqsPending = pdmacFileAioMgrNormalRemoveEndpoint(pEndpoint);
1444	Assert(!fReqsPending); NOREF(fReqsPending);
1445
1446	rc = pdmacFileAioMgrAddEndpoint(pEndpoint->AioMgr.pAioMgrDst, pEndpoint);
1447	AssertRC(rc);
1448	}
1449	}
1450	else
1451	{
1452	pTask->pfnCompleted(pTask, pTask->pvUser, rcReq);
1453	pdmacFileTaskFree(pEndpoint, pTask);
1454	}
1455	}
1456	}
1457	else
1458	{
1459	if (pTask->enmTransferType == PDMACTASKFILETRANSFER_FLUSH)
1460	{
1461	/* Clear pending flush */
1462	AssertMsg(pEndpoint->pFlushReq == pTask, ("Completed flush request doesn't match active one\n"));
1463	pEndpoint->pFlushReq = NULL;
1464	pdmacFileAioMgrNormalRequestFree(pAioMgr, hReq);
1465
1466	/* Call completion callback */
1467	LogFlow(("Flush task=%#p completed with %Rrc\n", pTask, rcReq));
1468	pTask->pfnCompleted(pTask, pTask->pvUser, rcReq);
1469	pdmacFileTaskFree(pEndpoint, pTask);
1470	}
1471	else
1472	{
1473	/*
1474	* Restart an incomplete transfer.
1475	* This usually means that the request will return an error now
1476	* but to get the cause of the error (disk full, file too big, I/O error, ...)
1477	* the transfer needs to be continued.
1478	*/
1479	pTask->cbTransfered += cbTransfered;
1480
1481	if (RT_UNLIKELY( pTask->cbTransfered < pTask->DataSeg.cbSeg
1482	\|\| ( pTask->cbBounceBuffer
1483	&& pTask->cbTransfered < pTask->cbBounceBuffer)))
1484	{
1485	RTFOFF offStart;
1486	size_t cbToTransfer;
1487	uint8_t *pbBuf = NULL;
1488
1489	LogFlow(("Restarting incomplete transfer %#p (%zu bytes transferred)\n",
1490	pTask, cbTransfered));
1491	Assert(cbTransfered % 512 == 0);
1492
1493	if (pTask->cbBounceBuffer)
1494	{
1495	AssertPtr(pTask->pvBounceBuffer);
1496	offStart = (pTask->Off & ~((RTFOFF)512-1)) + pTask->cbTransfered;
1497	cbToTransfer = pTask->cbBounceBuffer - pTask->cbTransfered;
1498	pbBuf = (uint8_t *)pTask->pvBounceBuffer + pTask->cbTransfered;
1499	}
1500	else
1501	{
1502	Assert(!pTask->pvBounceBuffer);
1503	offStart = pTask->Off + pTask->cbTransfered;
1504	cbToTransfer = pTask->DataSeg.cbSeg - pTask->cbTransfered;
1505	pbBuf = (uint8_t *)pTask->DataSeg.pvSeg + pTask->cbTransfered;
1506	}
1507
1508	if (pTask->fPrefetch \|\| pTask->enmTransferType == PDMACTASKFILETRANSFER_READ)
1509	{
1510	rc = RTFileAioReqPrepareRead(hReq, pEndpoint->hFile, offStart,
1511	pbBuf, cbToTransfer, pTask);
1512	}
1513	else
1514	{
1515	AssertMsg(pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE,
1516	("Invalid transfer type\n"));
1517	rc = RTFileAioReqPrepareWrite(hReq, pEndpoint->hFile, offStart,
1518	pbBuf, cbToTransfer, pTask);
1519	}
1520	AssertRC(rc);
1521
1522	pTask->hReq = hReq;
1523	rc = pdmacFileAioMgrNormalReqsEnqueue(pAioMgr, pEndpoint, &hReq, 1);
1524	AssertMsg(RT_SUCCESS(rc) \|\| (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES),
1525	("Unexpected return code rc=%Rrc\n", rc));
1526	}
1527	else if (pTask->fPrefetch)
1528	{
1529	Assert(pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE);
1530	Assert(pTask->cbBounceBuffer);
1531
1532	memcpy(((uint8_t *)pTask->pvBounceBuffer) + pTask->offBounceBuffer,
1533	pTask->DataSeg.pvSeg,
1534	pTask->DataSeg.cbSeg);
1535
1536	/* Write it now. */
1537	pTask->fPrefetch = false;
1538	RTFOFF offStart = pTask->Off & ~(RTFOFF)(512-1);
1539	size_t cbToTransfer = RT_ALIGN_Z(pTask->DataSeg.cbSeg + (pTask->Off - offStart), 512);
1540
1541	pTask->cbTransfered = 0;
1542
1543	/* Grow the file if needed. */
1544	if (RT_UNLIKELY((uint64_t)(pTask->Off + pTask->DataSeg.cbSeg) > pEndpoint->cbFile))
1545	{
1546	ASMAtomicWriteU64(&pEndpoint->cbFile, pTask->Off + pTask->DataSeg.cbSeg);
1547	RTFileSetSize(pEndpoint->hFile, pTask->Off + pTask->DataSeg.cbSeg);
1548	}
1549
1550	rc = RTFileAioReqPrepareWrite(hReq, pEndpoint->hFile,
1551	offStart, pTask->pvBounceBuffer, cbToTransfer, pTask);
1552	AssertRC(rc);
1553	pTask->hReq = hReq;
1554	rc = pdmacFileAioMgrNormalReqsEnqueue(pAioMgr, pEndpoint, &hReq, 1);
1555	AssertMsg(RT_SUCCESS(rc) \|\| (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES),
1556	("Unexpected return code rc=%Rrc\n", rc));
1557	}
1558	else
1559	{
1560	if (RT_SUCCESS(rc) && pTask->cbBounceBuffer)
1561	{
1562	if (pTask->enmTransferType == PDMACTASKFILETRANSFER_READ)
1563	memcpy(pTask->DataSeg.pvSeg,
1564	((uint8_t *)pTask->pvBounceBuffer) + pTask->offBounceBuffer,
1565	pTask->DataSeg.cbSeg);
1566
1567	RTMemPageFree(pTask->pvBounceBuffer, pTask->cbBounceBuffer);
1568	}
1569
1570	pdmacFileAioMgrNormalRequestFree(pAioMgr, hReq);
1571
1572	/* Free the lock and process pending tasks if necessary */
1573	pTasksWaiting = pdmacFileAioMgrNormalRangeLockFree(pAioMgr, pEndpoint, pTask->pRangeLock);
1574	if (pTasksWaiting)
1575	{
1576	rc = pdmacFileAioMgrNormalProcessTaskList(pTasksWaiting, pAioMgr, pEndpoint);
1577	AssertRC(rc);
1578	}
1579
1580	/* Call completion callback */
1581	LogFlow(("Task=%#p completed with %Rrc\n", pTask, rcReq));
1582	pTask->pfnCompleted(pTask, pTask->pvUser, rcReq);
1583	pdmacFileTaskFree(pEndpoint, pTask);
1584
1585	/*
1586	* If there is no request left on the endpoint but a flush request is set
1587	* it completed now and we notify the owner.
1588	* Furthermore we look for new requests and continue.
1589	*/
1590	if (!pEndpoint->AioMgr.cRequestsActive && pEndpoint->pFlushReq)
1591	{
1592	/* Call completion callback */
1593	pTask = pEndpoint->pFlushReq;
1594	pEndpoint->pFlushReq = NULL;
1595
1596	AssertMsg(pTask->pEndpoint == pEndpoint, ("Endpoint of the flush request does not match assigned one\n"));
1597
1598	pTask->pfnCompleted(pTask, pTask->pvUser, VINF_SUCCESS);
1599	pdmacFileTaskFree(pEndpoint, pTask);
1600	}
1601	else if (RT_UNLIKELY(!pEndpoint->AioMgr.cRequestsActive && pEndpoint->AioMgr.fMoving))
1602	{
1603	/* If the endpoint is about to be migrated do it now. */
1604	bool fReqsPending = pdmacFileAioMgrNormalRemoveEndpoint(pEndpoint);
1605	Assert(!fReqsPending); NOREF(fReqsPending);
1606
1607	rc = pdmacFileAioMgrAddEndpoint(pEndpoint->AioMgr.pAioMgrDst, pEndpoint);
1608	AssertRC(rc);
1609	}
1610	}
1611	} /* Not a flush request */
1612	} /* request completed successfully */
1613	}
1614
1615	/** Helper macro for checking for error codes. */
1616	#define CHECK_RC(pAioMgr, rc) \
1617	if (RT_FAILURE(rc)) \
1618	{\
1619	int rc2 = pdmacFileAioMgrNormalErrorHandler(pAioMgr, rc, RT_SRC_POS);\
1620	return rc2;\
1621	}
1622
1623	/**
1624	* The normal I/O manager using the RTFileAio* API
1625	*
1626	* @returns VBox status code.
1627	* @param hThreadSelf Handle of the thread.
1628	* @param pvUser Opaque user data.
1629	*/
1630	DECLCALLBACK(int) pdmacFileAioMgrNormal(RTTHREAD hThreadSelf, void *pvUser)
1631	{
1632	int rc = VINF_SUCCESS;
1633	PPDMACEPFILEMGR pAioMgr = (PPDMACEPFILEMGR)pvUser;
1634	uint64_t uMillisEnd = RTTimeMilliTS() + PDMACEPFILEMGR_LOAD_UPDATE_PERIOD;
1635	NOREF(hThreadSelf);
1636
1637	while ( pAioMgr->enmState == PDMACEPFILEMGRSTATE_RUNNING
1638	\|\| pAioMgr->enmState == PDMACEPFILEMGRSTATE_SUSPENDING
1639	\|\| pAioMgr->enmState == PDMACEPFILEMGRSTATE_GROWING)
1640	{
1641	if (!pAioMgr->cRequestsActive)
1642	{
1643	ASMAtomicWriteBool(&pAioMgr->fWaitingEventSem, true);
1644	if (!ASMAtomicReadBool(&pAioMgr->fWokenUp))
1645	rc = RTSemEventWait(pAioMgr->EventSem, pAioMgr->msBwLimitExpired);
1646	ASMAtomicWriteBool(&pAioMgr->fWaitingEventSem, false);
1647	Assert(RT_SUCCESS(rc) \|\| rc == VERR_TIMEOUT);
1648
1649	LogFlow(("Got woken up\n"));
1650	ASMAtomicWriteBool(&pAioMgr->fWokenUp, false);
1651	}
1652
1653	/* Check for an external blocking event first. */
1654	if (pAioMgr->fBlockingEventPending)
1655	{
1656	rc = pdmacFileAioMgrNormalProcessBlockingEvent(pAioMgr);
1657	CHECK_RC(pAioMgr, rc);
1658	}
1659
1660	if (RT_LIKELY( pAioMgr->enmState == PDMACEPFILEMGRSTATE_RUNNING
1661	\|\| pAioMgr->enmState == PDMACEPFILEMGRSTATE_GROWING))
1662	{
1663	/* We got woken up because an endpoint issued new requests. Queue them. */
1664	rc = pdmacFileAioMgrNormalCheckEndpoints(pAioMgr);
1665	CHECK_RC(pAioMgr, rc);
1666
1667	while (pAioMgr->cRequestsActive)
1668	{
1669	RTFILEAIOREQ apReqs[20];
1670	uint32_t cReqsCompleted = 0;
1671	size_t cReqsWait;
1672
1673	if (pAioMgr->cRequestsActive > RT_ELEMENTS(apReqs))
1674	cReqsWait = RT_ELEMENTS(apReqs);
1675	else
1676	cReqsWait = pAioMgr->cRequestsActive;
1677
1678	LogFlow(("Waiting for %d of %d tasks to complete\n", 1, cReqsWait));
1679
1680	rc = RTFileAioCtxWait(pAioMgr->hAioCtx,
1681	1,
1682	RT_INDEFINITE_WAIT, apReqs,
1683	cReqsWait, &cReqsCompleted);
1684	if (RT_FAILURE(rc) && (rc != VERR_INTERRUPTED))
1685	CHECK_RC(pAioMgr, rc);
1686
1687	LogFlow(("%d tasks completed\n", cReqsCompleted));
1688
1689	for (uint32_t i = 0; i < cReqsCompleted; i++)
1690	pdmacFileAioMgrNormalReqComplete(pAioMgr, apReqs[i]);
1691
1692	/* Check for an external blocking event before we go to sleep again. */
1693	if (pAioMgr->fBlockingEventPending)
1694	{
1695	rc = pdmacFileAioMgrNormalProcessBlockingEvent(pAioMgr);
1696	CHECK_RC(pAioMgr, rc);
1697	}
1698
1699	/* Update load statistics. */
1700	uint64_t uMillisCurr = RTTimeMilliTS();
1701	if (uMillisCurr > uMillisEnd)
1702	{
1703	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointCurr = pAioMgr->pEndpointsHead;
1704
1705	/* Calculate timespan. */
1706	uMillisCurr -= uMillisEnd;
1707
1708	while (pEndpointCurr)
1709	{
1710	pEndpointCurr->AioMgr.cReqsPerSec = pEndpointCurr->AioMgr.cReqsProcessed / (uMillisCurr + PDMACEPFILEMGR_LOAD_UPDATE_PERIOD);
1711	pEndpointCurr->AioMgr.cReqsProcessed = 0;
1712	pEndpointCurr = pEndpointCurr->AioMgr.pEndpointNext;
1713	}
1714
1715	/* Set new update interval */
1716	uMillisEnd = RTTimeMilliTS() + PDMACEPFILEMGR_LOAD_UPDATE_PERIOD;
1717	}
1718
1719	/* Check endpoints for new requests. */
1720	if (pAioMgr->enmState != PDMACEPFILEMGRSTATE_GROWING)
1721	{
1722	rc = pdmacFileAioMgrNormalCheckEndpoints(pAioMgr);
1723	CHECK_RC(pAioMgr, rc);
1724	}
1725	} /* while requests are active. */
1726
1727	if (pAioMgr->enmState == PDMACEPFILEMGRSTATE_GROWING)
1728	{
1729	rc = pdmacFileAioMgrNormalGrow(pAioMgr);
1730	AssertRC(rc);
1731	Assert(pAioMgr->enmState == PDMACEPFILEMGRSTATE_RUNNING);
1732
1733	rc = pdmacFileAioMgrNormalCheckEndpoints(pAioMgr);
1734	CHECK_RC(pAioMgr, rc);
1735	}
1736	} /* if still running */
1737	} /* while running */
1738
1739	LogFlowFunc(("rc=%Rrc\n", rc));
1740	return rc;
1741	}
1742
1743	#undef CHECK_RC
1744

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR3/PDMAsyncCompletionFileNormal.cpp@ 100307

Download in other formats: