PDMAsyncCompletionFileNormal.cpp@ 26550

Last change on this file since 26550 was 26526, checked in by vboxsync, 15 years ago
VMM,asm.h,Config.kmk: win.amd64 warnings; fixed the ASMAtomicSubU32 signature.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 48.1 KB

Line
1	/* $Id: PDMAsyncCompletionFileNormal.cpp 26526 2010-02-15 03:36:01Z vboxsync $ */
2	/** @file
3	* PDM Async I/O - Transport data asynchronous in R3 using EMT.
4	* Async File I/O manager.
5	*/
6
7	/*
8	* Copyright (C) 2006-2008 Sun Microsystems, Inc.
9	*
10	* This file is part of VirtualBox Open Source Edition (OSE), as
11	* available from http://www.virtualbox.org. This file is free software;
12	* you can redistribute it and/or modify it under the terms of the GNU
13	* General Public License (GPL) as published by the Free Software
14	* Foundation, in version 2 as it comes in the "COPYING" file of the
15	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
16	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
17	*
18	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
19	* Clara, CA 95054 USA or visit http://www.sun.com if you need
20	* additional information or have any questions.
21	*/
22	#define LOG_GROUP LOG_GROUP_PDM_ASYNC_COMPLETION
23	#include <iprt/types.h>
24	#include <iprt/asm.h>
25	#include <iprt/file.h>
26	#include <iprt/mem.h>
27	#include <iprt/string.h>
28	#include <iprt/assert.h>
29	#include <VBox/log.h>
30
31	#include "PDMAsyncCompletionFileInternal.h"
32
33	/** The update period for the I/O load statistics in ms. */
34	#define PDMACEPFILEMGR_LOAD_UPDATE_PERIOD 1000
35	/** Maximum number of requests a manager will handle. */
36	#define PDMACEPFILEMGR_REQS_MAX 512 /* @todo: Find better solution wrt. the request number*/
37
38	/*******************************************************************************
39	* Internal functions *
40	*******************************************************************************/
41	static int pdmacFileAioMgrNormalProcessTaskList(PPDMACTASKFILE pTaskHead,
42	PPDMACEPFILEMGR pAioMgr,
43	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint);
44
45
46	int pdmacFileAioMgrNormalInit(PPDMACEPFILEMGR pAioMgr)
47	{
48	int rc = VINF_SUCCESS;
49
50	rc = RTFileAioCtxCreate(&pAioMgr->hAioCtx, RTFILEAIO_UNLIMITED_REQS);
51	if (rc == VERR_OUT_OF_RANGE)
52	rc = RTFileAioCtxCreate(&pAioMgr->hAioCtx, PDMACEPFILEMGR_REQS_MAX);
53
54	if (RT_SUCCESS(rc))
55	{
56	/* Initialize request handle array. */
57	pAioMgr->iFreeEntryNext = 0;
58	pAioMgr->iFreeReqNext = 0;
59	pAioMgr->cReqEntries = PDMACEPFILEMGR_REQS_MAX + 1;
60	pAioMgr->pahReqsFree = (RTFILEAIOREQ )RTMemAllocZ(pAioMgr->cReqEntries sizeof(RTFILEAIOREQ));
61
62	if (pAioMgr->pahReqsFree)
63	{
64	return VINF_SUCCESS;
65	}
66	else
67	{
68	RTFileAioCtxDestroy(pAioMgr->hAioCtx);
69	rc = VERR_NO_MEMORY;
70	}
71	}
72
73	return rc;
74	}
75
76	void pdmacFileAioMgrNormalDestroy(PPDMACEPFILEMGR pAioMgr)
77	{
78	RTFileAioCtxDestroy(pAioMgr->hAioCtx);
79
80	while (pAioMgr->iFreeReqNext != pAioMgr->iFreeEntryNext)
81	{
82	RTFileAioReqDestroy(pAioMgr->pahReqsFree[pAioMgr->iFreeReqNext]);
83	pAioMgr->iFreeReqNext = (pAioMgr->iFreeReqNext + 1) % pAioMgr->cReqEntries;
84	}
85
86	RTMemFree(pAioMgr->pahReqsFree);
87	}
88
89	/**
90	* Sorts the endpoint list with insertion sort.
91	*/
92	static void pdmacFileAioMgrNormalEndpointsSortByLoad(PPDMACEPFILEMGR pAioMgr)
93	{
94	PPDMASYNCCOMPLETIONENDPOINTFILE pEpPrev, pEpCurr, pEpNextToSort;
95
96	pEpPrev = pAioMgr->pEndpointsHead;
97	pEpCurr = pEpPrev->AioMgr.pEndpointNext;
98
99	while (pEpCurr)
100	{
101	/* Remember the next element to sort because the list might change. */
102	pEpNextToSort = pEpCurr->AioMgr.pEndpointNext;
103
104	/* Unlink the current element from the list. */
105	PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEpCurr->AioMgr.pEndpointPrev;
106	PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEpCurr->AioMgr.pEndpointNext;
107
108	if (pPrev)
109	pPrev->AioMgr.pEndpointNext = pNext;
110	else
111	pAioMgr->pEndpointsHead = pNext;
112
113	if (pNext)
114	pNext->AioMgr.pEndpointPrev = pPrev;
115
116	/* Go back until we reached the place to insert the current endpoint into. */
117	while (pEpPrev && (pEpPrev->AioMgr.cReqsPerSec < pEpCurr->AioMgr.cReqsPerSec))
118	pEpPrev = pEpPrev->AioMgr.pEndpointPrev;
119
120	/* Link the endpoint into the list. */
121	if (pEpPrev)
122	pNext = pEpPrev->AioMgr.pEndpointNext;
123	else
124	pNext = pAioMgr->pEndpointsHead;
125
126	pEpCurr->AioMgr.pEndpointNext = pNext;
127	pEpCurr->AioMgr.pEndpointPrev = pEpPrev;
128
129	if (pNext)
130	pNext->AioMgr.pEndpointPrev = pEpCurr;
131
132	if (pEpPrev)
133	pEpPrev->AioMgr.pEndpointNext = pEpCurr;
134	else
135	pAioMgr->pEndpointsHead = pEpCurr;
136
137	pEpCurr = pEpNextToSort;
138	}
139
140	#ifdef DEBUG
141	/* Validate sorting alogrithm */
142	unsigned cEndpoints = 0;
143	pEpCurr = pAioMgr->pEndpointsHead;
144
145	AssertMsg(pEpCurr, ("No endpoint in the list?\n"));
146	AssertMsg(!pEpCurr->AioMgr.pEndpointPrev, ("First element in the list points to previous element\n"));
147
148	while (pEpCurr)
149	{
150	cEndpoints++;
151
152	PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEpCurr->AioMgr.pEndpointNext;
153	PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEpCurr->AioMgr.pEndpointPrev;
154
155	Assert(!pNext \|\| pNext->AioMgr.cReqsPerSec <= pEpCurr->AioMgr.cReqsPerSec);
156	Assert(!pPrev \|\| pPrev->AioMgr.cReqsPerSec >= pEpCurr->AioMgr.cReqsPerSec);
157
158	pEpCurr = pNext;
159	}
160
161	AssertMsg(cEndpoints == pAioMgr->cEndpoints, ("Endpoints lost during sort!\n"));
162
163	#endif
164	}
165
166	/**
167	* Removes an endpoint from the currently assigned manager.
168	*
169	* @returns TRUE if there are still requests pending on the current manager for this endpoint.
170	* FALSE otherwise.
171	* @param pEndpointRemove The endpoint to remove.
172	*/
173	static bool pdmacFileAioMgrNormalRemoveEndpoint(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointRemove)
174	{
175	PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEndpointRemove->AioMgr.pEndpointPrev;
176	PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEndpointRemove->AioMgr.pEndpointNext;
177	PPDMACEPFILEMGR pAioMgr = pEndpointRemove->pAioMgr;
178
179	pAioMgr->cEndpoints--;
180
181	if (pPrev)
182	pPrev->AioMgr.pEndpointNext = pNext;
183	else
184	pAioMgr->pEndpointsHead = pNext;
185
186	if (pNext)
187	pNext->AioMgr.pEndpointPrev = pPrev;
188
189	/* Make sure that there is no request pending on this manager for the endpoint. */
190	if (!pEndpointRemove->AioMgr.cRequestsActive)
191	{
192	Assert(!pEndpointRemove->pFlushReq);
193
194	/* Reopen the file so that the new endpoint can reassociate with the file */
195	RTFileClose(pEndpointRemove->File);
196	int rc = RTFileOpen(&pEndpointRemove->File, pEndpointRemove->Core.pszUri, pEndpointRemove->fFlags);
197	AssertRC(rc);
198	return false;
199	}
200
201	return true;
202	}
203
204	static bool pdmacFileAioMgrNormalIsBalancePossible(PPDMACEPFILEMGR pAioMgr)
205	{
206	/* Balancing doesn't make sense with only one endpoint. */
207	if (pAioMgr->cEndpoints == 1)
208	return false;
209
210	/* Doesn't make sens to move endpoints if only one produces the whole load */
211	unsigned cEndpointsWithLoad = 0;
212
213	PPDMASYNCCOMPLETIONENDPOINTFILE pCurr = pAioMgr->pEndpointsHead;
214
215	while (pCurr)
216	{
217	if (pCurr->AioMgr.cReqsPerSec)
218	cEndpointsWithLoad++;
219
220	pCurr = pCurr->AioMgr.pEndpointNext;
221	}
222
223	return (cEndpointsWithLoad > 1);
224	}
225
226	/**
227	* Creates a new I/O manager and spreads the I/O load of the endpoints
228	* between the given I/O manager and the new one.
229	*
230	* @returns nothing.
231	* @param pAioMgr The I/O manager with high I/O load.
232	*/
233	static void pdmacFileAioMgrNormalBalanceLoad(PPDMACEPFILEMGR pAioMgr)
234	{
235	PPDMACEPFILEMGR pAioMgrNew = NULL;
236	int rc = VINF_SUCCESS;
237
238	/*
239	* Check if balancing would improve the situation.
240	*/
241	if (pdmacFileAioMgrNormalIsBalancePossible(pAioMgr))
242	{
243	rc = pdmacFileAioMgrCreate((PPDMASYNCCOMPLETIONEPCLASSFILE)pAioMgr->pEndpointsHead->Core.pEpClass,
244	&pAioMgrNew, false);
245	if (RT_SUCCESS(rc))
246	{
247	/* We will sort the list by request count per second. */
248	pdmacFileAioMgrNormalEndpointsSortByLoad(pAioMgr);
249
250	/* Now move some endpoints to the new manager. */
251	unsigned cReqsHere = pAioMgr->pEndpointsHead->AioMgr.cReqsPerSec;
252	unsigned cReqsOther = 0;
253	PPDMASYNCCOMPLETIONENDPOINTFILE pCurr = pAioMgr->pEndpointsHead->AioMgr.pEndpointNext;
254
255	while (pCurr)
256	{
257	if (cReqsHere <= cReqsOther)
258	{
259	/*
260	* The other manager has more requests to handle now.
261	* We will keep the current endpoint.
262	*/
263	Log(("Keeping endpoint %#p{%s} with %u reqs/s\n", pCurr->Core.pszUri, pCurr->AioMgr.cReqsPerSec));
264	cReqsHere += pCurr->AioMgr.cReqsPerSec;
265	pCurr = pCurr->AioMgr.pEndpointNext;
266	}
267	else
268	{
269	/* Move to other endpoint. */
270	Log(("Moving endpoint %#p{%s} with %u reqs/s to other manager\n", pCurr, pCurr->Core.pszUri, pCurr->AioMgr.cReqsPerSec));
271	cReqsOther += pCurr->AioMgr.cReqsPerSec;
272
273	PPDMASYNCCOMPLETIONENDPOINTFILE pMove = pCurr;
274
275	pCurr = pCurr->AioMgr.pEndpointNext;
276
277	bool fReqsPending = pdmacFileAioMgrNormalRemoveEndpoint(pMove);
278
279	if (fReqsPending)
280	{
281	pMove->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_REMOVING;
282	pMove->AioMgr.fMoving = true;
283	pMove->AioMgr.pAioMgrDst = pAioMgrNew;
284	}
285	else
286	{
287	pMove->AioMgr.fMoving = false;
288	pMove->AioMgr.pAioMgrDst = NULL;
289	pdmacFileAioMgrAddEndpoint(pAioMgrNew, pMove);
290	}
291	}
292	}
293	}
294	else
295	{
296	/* Don't process further but leave a log entry about reduced performance. */
297	LogRel(("AIOMgr: Could not create new I/O manager (rc=%Rrc). Expect reduced performance\n", rc));
298	}
299	}
300	else
301	Log(("AIOMgr: Load balancing would not improve anything\n"));
302	}
303
304	/**
305	* Error handler which will create the failsafe managers and destroy the failed I/O manager.
306	*
307	* @returns VBox status code
308	* @param pAioMgr The I/O manager the error ocurred on.
309	* @param rc The error code.
310	*/
311	static int pdmacFileAioMgrNormalErrorHandler(PPDMACEPFILEMGR pAioMgr, int rc, RT_SRC_POS_DECL)
312	{
313	LogRel(("AIOMgr: I/O manager %#p encountered a critical error (rc=%Rrc) during operation. Falling back to failsafe mode. Expect reduced performance\n",
314	pAioMgr, rc));
315	LogRel(("AIOMgr: Error happened in %s:(%u){%s}\n", RT_SRC_POS_ARGS));
316	LogRel(("AIOMgr: Please contact the product vendor\n"));
317
318	PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pAioMgr->pEndpointsHead->Core.pEpClass;
319
320	pAioMgr->enmState = PDMACEPFILEMGRSTATE_FAULT;
321	ASMAtomicWriteBool(&pEpClassFile->fFailsafe, true);
322
323	AssertMsgFailed(("Implement\n"));
324	return VINF_SUCCESS;
325	}
326
327	/**
328	* Put a list of tasks in the pending request list of an endpoint.
329	*/
330	DECLINLINE(void) pdmacFileAioMgrEpAddTaskList(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMACTASKFILE pTaskHead)
331	{
332	/* Add the rest of the tasks to the pending list */
333	if (!pEndpoint->AioMgr.pReqsPendingHead)
334	{
335	Assert(!pEndpoint->AioMgr.pReqsPendingTail);
336	pEndpoint->AioMgr.pReqsPendingHead = pTaskHead;
337	}
338	else
339	{
340	Assert(pEndpoint->AioMgr.pReqsPendingTail);
341	pEndpoint->AioMgr.pReqsPendingTail->pNext = pTaskHead;
342	}
343
344	/* Update the tail. */
345	while (pTaskHead->pNext)
346	pTaskHead = pTaskHead->pNext;
347
348	pEndpoint->AioMgr.pReqsPendingTail = pTaskHead;
349	}
350
351	/**
352	* Put one task in the pending request list of an endpoint.
353	*/
354	DECLINLINE(void) pdmacFileAioMgrEpAddTask(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMACTASKFILE pTask)
355	{
356	/* Add the rest of the tasks to the pending list */
357	if (!pEndpoint->AioMgr.pReqsPendingHead)
358	{
359	Assert(!pEndpoint->AioMgr.pReqsPendingTail);
360	pEndpoint->AioMgr.pReqsPendingHead = pTask;
361	}
362	else
363	{
364	Assert(pEndpoint->AioMgr.pReqsPendingTail);
365	pEndpoint->AioMgr.pReqsPendingTail->pNext = pTask;
366	}
367
368	pEndpoint->AioMgr.pReqsPendingTail = pTask;
369	}
370
371	/**
372	* Wrapper around RTFIleAioCtxSubmit() which is also doing error handling.
373	*/
374	static int pdmacFileAioMgrNormalReqsEnqueue(PPDMACEPFILEMGR pAioMgr,
375	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
376	PRTFILEAIOREQ pahReqs, unsigned cReqs)
377	{
378	int rc;
379
380	pAioMgr->cRequestsActive += cReqs;
381	pEndpoint->AioMgr.cRequestsActive += cReqs;
382
383	LogFlow(("Enqueuing %d requests. I/O manager has a total of %d active requests now\n", cReqs, pAioMgr->cRequestsActive));
384	LogFlow(("Endpoint has a total of %d active requests now\n", pEndpoint->AioMgr.cRequestsActive));
385
386	rc = RTFileAioCtxSubmit(pAioMgr->hAioCtx, pahReqs, cReqs);
387	if (RT_FAILURE(rc))
388	{
389	if (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES)
390	{
391	PPDMASYNCCOMPLETIONEPCLASSFILE pEpClass = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass;
392
393	/*
394	* We run out of resources.
395	* Need to check which requests got queued
396	* and put the rest on the pending list again.
397	*/
398	if (RT_UNLIKELY(!pEpClass->fOutOfResourcesWarningPrinted))
399	{
400	pEpClass->fOutOfResourcesWarningPrinted = true;
401	LogRel(("AIOMgr: The operating system doesn't have enough resources "
402	"to handle the I/O load of the VM. Expect reduced I/O performance\n"));
403	}
404
405	for (size_t i = 0; i < cReqs; i++)
406	{
407	int rcReq = RTFileAioReqGetRC(pahReqs[i], NULL);
408
409	if (rcReq != VERR_FILE_AIO_IN_PROGRESS)
410	{
411	AssertMsg(rcReq == VERR_FILE_AIO_NOT_SUBMITTED,
412	("Request returned unexpected return code: rc=%Rrc\n", rcReq));
413
414	PPDMACTASKFILE pTask = (PPDMACTASKFILE)RTFileAioReqGetUser(pahReqs[i]);
415
416	/* Put the entry on the free array */
417	pAioMgr->pahReqsFree[pAioMgr->iFreeEntryNext] = pahReqs[i];
418	pAioMgr->iFreeEntryNext = (pAioMgr->iFreeEntryNext + 1) % pAioMgr->cReqEntries;
419
420	pdmacFileAioMgrEpAddTask(pEndpoint, pTask);
421	pAioMgr->cRequestsActive--;
422	pEndpoint->AioMgr.cRequestsActive--;
423	}
424	}
425	LogFlow(("Removed requests. I/O manager has a total of %d active requests now\n", pAioMgr->cRequestsActive));
426	LogFlow(("Endpoint has a total of %d active requests now\n", pEndpoint->AioMgr.cRequestsActive));
427	}
428	else
429	AssertMsgFailed(("Unexpected return code rc=%Rrc\n", rc));
430	}
431
432	return rc;
433	}
434
435	/**
436	* Allocates a async I/O request.
437	*
438	* @returns Handle to the request.
439	* @param pAioMgr The I/O manager.
440	*/
441	static RTFILEAIOREQ pdmacFileAioMgrNormalRequestAlloc(PPDMACEPFILEMGR pAioMgr)
442	{
443	RTFILEAIOREQ hReq = NIL_RTFILEAIOREQ;
444
445	/* Get a request handle. */
446	if (pAioMgr->iFreeReqNext != pAioMgr->iFreeEntryNext)
447	{
448	hReq = pAioMgr->pahReqsFree[pAioMgr->iFreeReqNext];
449	pAioMgr->pahReqsFree[pAioMgr->iFreeReqNext] = NIL_RTFILEAIOREQ;
450	pAioMgr->iFreeReqNext = (pAioMgr->iFreeReqNext + 1) % pAioMgr->cReqEntries;
451	}
452	else
453	{
454	int rc = RTFileAioReqCreate(&hReq);
455	AssertRC(rc);
456	}
457
458	return hReq;
459	}
460
461	static bool pdmacFileAioMgrNormalIsRangeLocked(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
462	RTFOFF offStart, size_t cbRange,
463	PPDMACTASKFILE pTask)
464	{
465	PPDMACFILERANGELOCK pRangeLock = NULL; /** < Range lock */
466
467	AssertMsg( pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE
468	\|\| pTask->enmTransferType == PDMACTASKFILETRANSFER_READ,
469	("Invalid task type %d\n", pTask->enmTransferType));
470
471	pRangeLock = (PPDMACFILERANGELOCK)RTAvlrFileOffsetGet(pEndpoint->AioMgr.pTreeRangesLocked, offStart);
472	if (!pRangeLock)
473	{
474	pRangeLock = (PPDMACFILERANGELOCK)RTAvlrFileOffsetGetBestFit(pEndpoint->AioMgr.pTreeRangesLocked, offStart, true);
475	/* Check if we intersect with the range. */
476	if ( !pRangeLock
477	\|\| !( (pRangeLock->Core.Key) <= (offStart + (RTFOFF)cbRange - 1)
478	&& (pRangeLock->Core.KeyLast) >= offStart))
479	{
480	pRangeLock = NULL; /* False alarm */
481	}
482	}
483
484	/* Check whether we have one of the situations explained below */
485	if ( pRangeLock
486	#if 0 /** @todo: later. For now we will just block all requests if they interfere */
487	&& ( (pRangeLock->fReadLock && pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE)
488	\|\| (!pRangeLock->fReadLock)
489	#endif
490	)
491	{
492	/* Add to the list. */
493	pTask->pNext = NULL;
494
495	if (!pRangeLock->pWaitingTasksHead)
496	{
497	Assert(!pRangeLock->pWaitingTasksTail);
498	pRangeLock->pWaitingTasksHead = pTask;
499	pRangeLock->pWaitingTasksTail = pTask;
500	}
501	else
502	{
503	AssertPtr(pRangeLock->pWaitingTasksTail);
504	pRangeLock->pWaitingTasksTail->pNext = pTask;
505	pRangeLock->pWaitingTasksTail = pTask;
506	}
507	return true;
508	}
509
510	return false;
511	}
512
513	static int pdmacFileAioMgrNormalRangeLock(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
514	RTFOFF offStart, size_t cbRange,
515	PPDMACTASKFILE pTask)
516	{
517	AssertMsg(!pdmacFileAioMgrNormalIsRangeLocked(pEndpoint, offStart, cbRange, pTask),
518	("Range is already locked offStart=%RTfoff cbRange=%u\n",
519	offStart, cbRange));
520
521	PPDMACFILERANGELOCK pRangeLock = (PPDMACFILERANGELOCK)RTMemAllocZ(sizeof(PDMACFILERANGELOCK));
522	if (!pRangeLock)
523	return VERR_NO_MEMORY;
524
525	/* Init the lock. */
526	pRangeLock->Core.Key = offStart;
527	pRangeLock->Core.KeyLast = offStart + cbRange - 1;
528	pRangeLock->cRefs = 1;
529	pRangeLock->fReadLock = pTask->enmTransferType == PDMACTASKFILETRANSFER_READ;
530
531	bool fInserted = RTAvlrFileOffsetInsert(pEndpoint->AioMgr.pTreeRangesLocked, &pRangeLock->Core);
532	AssertMsg(fInserted, ("Range lock was not inserted!\n"));
533
534	/* Let the task point to its lock. */
535	pTask->pRangeLock = pRangeLock;
536
537	return VINF_SUCCESS;
538	}
539
540	static int pdmacFileAioMgrNormalRangeLockFree(PPDMACEPFILEMGR pAioMgr,
541	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
542	PPDMACFILERANGELOCK pRangeLock)
543	{
544	PPDMACTASKFILE pTasksWaitingHead;
545
546	AssertPtr(pRangeLock);
547	Assert(pRangeLock->cRefs == 1);
548
549	RTAvlrFileOffsetRemove(pEndpoint->AioMgr.pTreeRangesLocked, pRangeLock->Core.Key);
550	pTasksWaitingHead = pRangeLock->pWaitingTasksHead;
551	RTMemFree(pRangeLock);
552
553	return pdmacFileAioMgrNormalProcessTaskList(pTasksWaitingHead, pAioMgr, pEndpoint);
554	}
555
556	static int pdmacFileAioMgrNormalTaskPrepare(PPDMACEPFILEMGR pAioMgr,
557	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
558	PPDMACTASKFILE pTask, PRTFILEAIOREQ phReq)
559	{
560	int rc = VINF_SUCCESS;
561	RTFILEAIOREQ hReq = NIL_RTFILEAIOREQ;
562	PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass;
563	void *pvBuf = pTask->DataSeg.pvSeg;
564
565	/* Get a request handle. */
566	hReq = pdmacFileAioMgrNormalRequestAlloc(pAioMgr);
567	AssertMsg(hReq != NIL_RTFILEAIOREQ, ("Out of request handles\n"));
568
569	/*
570	* Check if the alignment requirements are met.
571	* Offset, transfer size and buffer address
572	* need to be on a 512 boundary.
573	*/
574	RTFOFF offStart = pTask->Off & ~(RTFOFF)(512-1);
575	size_t cbToTransfer = RT_ALIGN_Z(pTask->DataSeg.cbSeg + (pTask->Off - offStart), 512);
576	PDMACTASKFILETRANSFER enmTransferType = pTask->enmTransferType;
577
578	AssertMsg( pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE
579	\|\| (uint64_t)(offStart + cbToTransfer) <= pEndpoint->cbFile,
580	("Read exceeds file size offStart=%RTfoff cbToTransfer=%d cbFile=%llu\n",
581	offStart, cbToTransfer, pEndpoint->cbFile));
582
583	pTask->fPrefetch = false;
584
585	/*
586	* Before we start to setup the request we have to check whether there is a task
587	* already active which range intersects with ours. We have to defer execution
588	* of this task in two cases:
589	* - The pending task is a write and the current is either read or write
590	* - The pending task is a read and the current task is a write task.
591	*
592	* To check whether a range is currently "locked" we use the AVL tree where every pending task
593	* is stored by its file offset range. The current task will be added to the active task
594	* and will be executed when the active one completes. (The method below
595	* which checks whether a range is already used will add the task)
596	*
597	* This is neccessary because of the requirementto align all requests to a 512 boundary
598	* which is enforced by the host OS (Linux and Windows atm). It is possible that
599	* we have to process unaligned tasks and need to align them using bounce buffers.
600	* While the data is feteched from the file another request might arrive writing to
601	* the same range. This will result in data corruption if both are executed concurrently.
602	*/
603	bool fLocked = pdmacFileAioMgrNormalIsRangeLocked(pEndpoint, offStart, cbToTransfer, pTask);
604
605	if (!fLocked)
606	{
607	if ( RT_UNLIKELY(cbToTransfer != pTask->DataSeg.cbSeg)
608	\|\| RT_UNLIKELY(offStart != pTask->Off)
609	\|\| ((pEpClassFile->uBitmaskAlignment & (RTR3UINTPTR)pvBuf) != (RTR3UINTPTR)pvBuf))
610	{
611	LogFlow(("Using bounce buffer for task %#p cbToTransfer=%zd cbSeg=%zd offStart=%RTfoff off=%RTfoff\n",
612	pTask, cbToTransfer, pTask->DataSeg.cbSeg, offStart, pTask->Off));
613
614	/* Create bounce buffer. */
615	pTask->fBounceBuffer = true;
616
617	AssertMsg(pTask->Off >= offStart, ("Overflow in calculation Off=%llu offStart=%llu\n",
618	pTask->Off, offStart));
619	pTask->uBounceBufOffset = pTask->Off - offStart;
620
621	/** @todo: I think we need something like a RTMemAllocAligned method here.
622	* Current assumption is that the maximum alignment is 4096byte
623	* (GPT disk on Windows)
624	* so we can use RTMemPageAlloc here.
625	*/
626	pTask->pvBounceBuffer = RTMemPageAlloc(cbToTransfer);
627	if (RT_LIKELY(pTask->pvBounceBuffer))
628	{
629	pvBuf = pTask->pvBounceBuffer;
630
631	if (pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE)
632	{
633	if ( RT_UNLIKELY(cbToTransfer != pTask->DataSeg.cbSeg)
634	\|\| RT_UNLIKELY(offStart != pTask->Off))
635	{
636	/* We have to fill the buffer first before we can update the data. */
637	LogFlow(("Prefetching data for task %#p\n", pTask));
638	pTask->fPrefetch = true;
639	enmTransferType = PDMACTASKFILETRANSFER_READ;
640	}
641	else
642	memcpy(pvBuf, pTask->DataSeg.pvSeg, pTask->DataSeg.cbSeg);
643	}
644	}
645	else
646	rc = VERR_NO_MEMORY;
647	}
648	else
649	pTask->fBounceBuffer = false;
650
651	if (RT_SUCCESS(rc))
652	{
653	AssertMsg((pEpClassFile->uBitmaskAlignment & (RTR3UINTPTR)pvBuf) == (RTR3UINTPTR)pvBuf,
654	("AIO: Alignment restrictions not met! pvBuf=%p uBitmaskAlignment=%p\n", pvBuf, pEpClassFile->uBitmaskAlignment));
655
656	if (enmTransferType == PDMACTASKFILETRANSFER_WRITE)
657	{
658	/* Grow the file if needed. */
659	if (RT_UNLIKELY((uint64_t)(pTask->Off + pTask->DataSeg.cbSeg) > pEndpoint->cbFile))
660	{
661	ASMAtomicWriteU64(&pEndpoint->cbFile, pTask->Off + pTask->DataSeg.cbSeg);
662	RTFileSetSize(pEndpoint->File, pTask->Off + pTask->DataSeg.cbSeg);
663	}
664
665	rc = RTFileAioReqPrepareWrite(hReq, pEndpoint->File,
666	offStart, pvBuf, cbToTransfer, pTask);
667	}
668	else
669	rc = RTFileAioReqPrepareRead(hReq, pEndpoint->File,
670	offStart, pvBuf, cbToTransfer, pTask);
671	AssertRC(rc);
672
673	rc = pdmacFileAioMgrNormalRangeLock(pEndpoint, offStart, cbToTransfer, pTask);
674
675	if (RT_SUCCESS(rc))
676	*phReq = hReq;
677	else
678	{
679	/* Cleanup */
680	if (pTask->fBounceBuffer)
681	RTMemPageFree(pTask->pvBounceBuffer);
682	}
683	}
684	}
685	else
686	{
687	LogFlow(("Task %#p was deferred because the access range is locked\n", pTask));
688	rc = VINF_SUCCESS;
689	}
690
691	return rc;
692	}
693
694	static int pdmacFileAioMgrNormalProcessTaskList(PPDMACTASKFILE pTaskHead,
695	PPDMACEPFILEMGR pAioMgr,
696	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
697	{
698	RTFILEAIOREQ apReqs[20];
699	unsigned cRequests = 0;
700	unsigned cMaxRequests = PDMACEPFILEMGR_REQS_MAX - pAioMgr->cRequestsActive;
701	int rc = VINF_SUCCESS;
702
703	AssertMsg(pEndpoint->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE,
704	("Trying to process request lists of a non active endpoint!\n"));
705
706	/* Go through the list and queue the requests until we get a flush request */
707	while ( pTaskHead
708	&& !pEndpoint->pFlushReq
709	&& (cMaxRequests > 0)
710	&& RT_SUCCESS(rc))
711	{
712	PPDMACTASKFILE pCurr = pTaskHead;
713
714	pTaskHead = pTaskHead->pNext;
715
716	pCurr->pNext = NULL;
717
718	AssertMsg(VALID_PTR(pCurr->pEndpoint) && (pCurr->pEndpoint == pEndpoint),
719	("Endpoints do not match\n"));
720
721	switch (pCurr->enmTransferType)
722	{
723	case PDMACTASKFILETRANSFER_FLUSH:
724	{
725	/* If there is no data transfer request this flush request finished immediately. */
726	if (!pEndpoint->AioMgr.cRequestsActive)
727	{
728	pCurr->pfnCompleted(pCurr, pCurr->pvUser);
729	pdmacFileTaskFree(pEndpoint, pCurr);
730	}
731	else
732	{
733	Assert(!pEndpoint->pFlushReq);
734	pEndpoint->pFlushReq = pCurr;
735	}
736	break;
737	}
738	case PDMACTASKFILETRANSFER_READ:
739	case PDMACTASKFILETRANSFER_WRITE:
740	{
741	RTFILEAIOREQ hReq = NIL_RTFILEAIOREQ;
742
743	rc = pdmacFileAioMgrNormalTaskPrepare(pAioMgr, pEndpoint, pCurr, &hReq);
744	AssertRC(rc);
745
746	if (hReq != NIL_RTFILEAIOREQ)
747	{
748	apReqs[cRequests] = hReq;
749	pEndpoint->AioMgr.cReqsProcessed++;
750	cMaxRequests--;
751	cRequests++;
752	if (cRequests == RT_ELEMENTS(apReqs))
753	{
754	rc = pdmacFileAioMgrNormalReqsEnqueue(pAioMgr, pEndpoint, apReqs, cRequests);
755	cRequests = 0;
756	AssertMsg(RT_SUCCESS(rc) \|\| (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES),
757	("Unexpected return code\n"));
758	}
759	}
760	break;
761	}
762	default:
763	AssertMsgFailed(("Invalid transfer type %d\n", pCurr->enmTransferType));
764	}
765	}
766
767	if (cRequests)
768	{
769	rc = pdmacFileAioMgrNormalReqsEnqueue(pAioMgr, pEndpoint, apReqs, cRequests);
770	AssertMsg(RT_SUCCESS(rc) \|\| (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES),
771	("Unexpected return code rc=%Rrc\n", rc));
772	}
773
774	if (pTaskHead)
775	{
776	/* Add the rest of the tasks to the pending list */
777	pdmacFileAioMgrEpAddTaskList(pEndpoint, pTaskHead);
778
779	if (RT_UNLIKELY(!cMaxRequests && !pEndpoint->pFlushReq))
780	{
781	/*
782	* The I/O manager has no room left for more requests
783	* but there are still requests to process.
784	* Create a new I/O manager and let it handle some endpoints.
785	*/
786	pdmacFileAioMgrNormalBalanceLoad(pAioMgr);
787	}
788	}
789
790	/* Insufficient resources are not fatal. */
791	if (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES)
792	rc = VINF_SUCCESS;
793
794	return rc;
795	}
796
797	/**
798	* Adds all pending requests for the given endpoint
799	* until a flush request is encountered or there is no
800	* request anymore.
801	*
802	* @returns VBox status code.
803	* @param pAioMgr The async I/O manager for the endpoint
804	* @param pEndpoint The endpoint to get the requests from.
805	*/
806	static int pdmacFileAioMgrNormalQueueReqs(PPDMACEPFILEMGR pAioMgr,
807	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
808	{
809	int rc = VINF_SUCCESS;
810	PPDMACTASKFILE pTasksHead = NULL;
811
812	AssertMsg(pEndpoint->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE,
813	("Trying to process request lists of a non active endpoint!\n"));
814
815	Assert(!pEndpoint->pFlushReq);
816
817	/* Check the pending list first */
818	if (pEndpoint->AioMgr.pReqsPendingHead)
819	{
820	LogFlow(("Queuing pending requests first\n"));
821
822	pTasksHead = pEndpoint->AioMgr.pReqsPendingHead;
823	/*
824	* Clear the list as the processing routine will insert them into the list
825	* again if it gets a flush request.
826	*/
827	pEndpoint->AioMgr.pReqsPendingHead = NULL;
828	pEndpoint->AioMgr.pReqsPendingTail = NULL;
829	rc = pdmacFileAioMgrNormalProcessTaskList(pTasksHead, pAioMgr, pEndpoint);
830	AssertRC(rc);
831	}
832
833	if (!pEndpoint->pFlushReq && !pEndpoint->AioMgr.pReqsPendingHead)
834	{
835	/* Now the request queue. */
836	pTasksHead = pdmacFileEpGetNewTasks(pEndpoint);
837	if (pTasksHead)
838	{
839	rc = pdmacFileAioMgrNormalProcessTaskList(pTasksHead, pAioMgr, pEndpoint);
840	AssertRC(rc);
841	}
842	}
843
844	return rc;
845	}
846
847	static int pdmacFileAioMgrNormalProcessBlockingEvent(PPDMACEPFILEMGR pAioMgr)
848	{
849	int rc = VINF_SUCCESS;
850	bool fNotifyWaiter = false;
851
852	LogFlowFunc((": Enter\n"));
853
854	Assert(pAioMgr->fBlockingEventPending);
855
856	switch (pAioMgr->enmBlockingEvent)
857	{
858	case PDMACEPFILEAIOMGRBLOCKINGEVENT_ADD_ENDPOINT:
859	{
860	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointNew = (PPDMASYNCCOMPLETIONENDPOINTFILE)ASMAtomicReadPtr((void * volatile *)&pAioMgr->BlockingEventData.AddEndpoint.pEndpoint);
861	AssertMsg(VALID_PTR(pEndpointNew), ("Adding endpoint event without a endpoint to add\n"));
862
863	pEndpointNew->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE;
864
865	pEndpointNew->AioMgr.pEndpointNext = pAioMgr->pEndpointsHead;
866	pEndpointNew->AioMgr.pEndpointPrev = NULL;
867	if (pAioMgr->pEndpointsHead)
868	pAioMgr->pEndpointsHead->AioMgr.pEndpointPrev = pEndpointNew;
869	pAioMgr->pEndpointsHead = pEndpointNew;
870
871	/* Assign the completion point to this file. */
872	rc = RTFileAioCtxAssociateWithFile(pAioMgr->hAioCtx, pEndpointNew->File);
873	fNotifyWaiter = true;
874	pAioMgr->cEndpoints++;
875	break;
876	}
877	case PDMACEPFILEAIOMGRBLOCKINGEVENT_REMOVE_ENDPOINT:
878	{
879	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointRemove = (PPDMASYNCCOMPLETIONENDPOINTFILE)ASMAtomicReadPtr((void * volatile *)&pAioMgr->BlockingEventData.RemoveEndpoint.pEndpoint);
880	AssertMsg(VALID_PTR(pEndpointRemove), ("Removing endpoint event without a endpoint to remove\n"));
881
882	pEndpointRemove->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_REMOVING;
883	fNotifyWaiter = !pdmacFileAioMgrNormalRemoveEndpoint(pEndpointRemove);
884	break;
885	}
886	case PDMACEPFILEAIOMGRBLOCKINGEVENT_CLOSE_ENDPOINT:
887	{
888	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointClose = (PPDMASYNCCOMPLETIONENDPOINTFILE)ASMAtomicReadPtr((void * volatile *)&pAioMgr->BlockingEventData.CloseEndpoint.pEndpoint);
889	AssertMsg(VALID_PTR(pEndpointClose), ("Close endpoint event without a endpoint to close\n"));
890
891	if (pEndpointClose->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE)
892	{
893	LogFlowFunc((": Closing endpoint %#p{%s}\n", pEndpointClose, pEndpointClose->Core.pszUri));
894
895	/* Make sure all tasks finished. Process the queues a last time first. */
896	rc = pdmacFileAioMgrNormalQueueReqs(pAioMgr, pEndpointClose);
897	AssertRC(rc);
898
899	pEndpointClose->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_CLOSING;
900	fNotifyWaiter = !pdmacFileAioMgrNormalRemoveEndpoint(pEndpointClose);
901	}
902	else if ( (pEndpointClose->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_CLOSING)
903	&& (!pEndpointClose->AioMgr.cRequestsActive))
904	fNotifyWaiter = true;
905	break;
906	}
907	case PDMACEPFILEAIOMGRBLOCKINGEVENT_SHUTDOWN:
908	{
909	pAioMgr->enmState = PDMACEPFILEMGRSTATE_SHUTDOWN;
910	if (!pAioMgr->cRequestsActive)
911	fNotifyWaiter = true;
912	break;
913	}
914	case PDMACEPFILEAIOMGRBLOCKINGEVENT_SUSPEND:
915	{
916	pAioMgr->enmState = PDMACEPFILEMGRSTATE_SUSPENDING;
917	break;
918	}
919	case PDMACEPFILEAIOMGRBLOCKINGEVENT_RESUME:
920	{
921	pAioMgr->enmState = PDMACEPFILEMGRSTATE_RUNNING;
922	fNotifyWaiter = true;
923	break;
924	}
925	default:
926	AssertReleaseMsgFailed(("Invalid event type %d\n", pAioMgr->enmBlockingEvent));
927	}
928
929	if (fNotifyWaiter)
930	{
931	ASMAtomicWriteBool(&pAioMgr->fBlockingEventPending, false);
932	pAioMgr->enmBlockingEvent = PDMACEPFILEAIOMGRBLOCKINGEVENT_INVALID;
933
934	/* Release the waiting thread. */
935	LogFlow(("Signalling waiter\n"));
936	rc = RTSemEventSignal(pAioMgr->EventSemBlock);
937	AssertRC(rc);
938	}
939
940	LogFlowFunc((": Leave\n"));
941	return rc;
942	}
943
944	/**
945	* Checks all endpoints for pending events or new requests.
946	*
947	* @returns VBox status code.
948	* @param pAioMgr The I/O manager handle.
949	*/
950	static int pdmacFileAioMgrNormalCheckEndpoints(PPDMACEPFILEMGR pAioMgr)
951	{
952	/* Check the assigned endpoints for new tasks if there isn't a flush request active at the moment. */
953	int rc = VINF_SUCCESS;
954	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint = pAioMgr->pEndpointsHead;
955
956	while (pEndpoint)
957	{
958	if (!pEndpoint->pFlushReq
959	&& (pEndpoint->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE)
960	&& !pEndpoint->AioMgr.fMoving)
961	{
962	rc = pdmacFileAioMgrNormalQueueReqs(pAioMgr, pEndpoint);
963	if (RT_FAILURE(rc))
964	return rc;
965	}
966	else if (!pEndpoint->AioMgr.cRequestsActive)
967	{
968	/* Reopen the file so that the new endpoint can reassociate with the file */
969	RTFileClose(pEndpoint->File);
970	rc = RTFileOpen(&pEndpoint->File, pEndpoint->Core.pszUri, pEndpoint->fFlags);
971	AssertRC(rc);
972
973	if (pEndpoint->AioMgr.fMoving)
974	{
975	pEndpoint->AioMgr.fMoving = false;
976	pdmacFileAioMgrAddEndpoint(pEndpoint->AioMgr.pAioMgrDst, pEndpoint);
977	}
978	else
979	{
980	Assert(pAioMgr->fBlockingEventPending);
981	ASMAtomicWriteBool(&pAioMgr->fBlockingEventPending, false);
982
983	/* Release the waiting thread. */
984	LogFlow(("Signalling waiter\n"));
985	rc = RTSemEventSignal(pAioMgr->EventSemBlock);
986	AssertRC(rc);
987	}
988	}
989
990	pEndpoint = pEndpoint->AioMgr.pEndpointNext;
991	}
992
993	return rc;
994	}
995
996	/** Helper macro for checking for error codes. */
997	#define CHECK_RC(pAioMgr, rc) \
998	if (RT_FAILURE(rc)) \
999	{\
1000	int rc2 = pdmacFileAioMgrNormalErrorHandler(pAioMgr, rc, RT_SRC_POS);\
1001	return rc2;\
1002	}
1003
1004	/**
1005	* The normal I/O manager using the RTFileAio* API
1006	*
1007	* @returns VBox status code.
1008	* @param ThreadSelf Handle of the thread.
1009	* @param pvUser Opaque user data.
1010	*/
1011	int pdmacFileAioMgrNormal(RTTHREAD ThreadSelf, void *pvUser)
1012	{
1013	int rc = VINF_SUCCESS;
1014	PPDMACEPFILEMGR pAioMgr = (PPDMACEPFILEMGR)pvUser;
1015	uint64_t uMillisEnd = RTTimeMilliTS() + PDMACEPFILEMGR_LOAD_UPDATE_PERIOD;
1016
1017	while ( (pAioMgr->enmState == PDMACEPFILEMGRSTATE_RUNNING)
1018	\|\| (pAioMgr->enmState == PDMACEPFILEMGRSTATE_SUSPENDING))
1019	{
1020	ASMAtomicWriteBool(&pAioMgr->fWaitingEventSem, true);
1021	if (!ASMAtomicReadBool(&pAioMgr->fWokenUp))
1022	rc = RTSemEventWait(pAioMgr->EventSem, RT_INDEFINITE_WAIT);
1023	ASMAtomicWriteBool(&pAioMgr->fWaitingEventSem, false);
1024	AssertRC(rc);
1025
1026	LogFlow(("Got woken up\n"));
1027	ASMAtomicWriteBool(&pAioMgr->fWokenUp, false);
1028
1029	/* Check for an external blocking event first. */
1030	if (pAioMgr->fBlockingEventPending)
1031	{
1032	rc = pdmacFileAioMgrNormalProcessBlockingEvent(pAioMgr);
1033	CHECK_RC(pAioMgr, rc);
1034	}
1035
1036	if (RT_LIKELY(pAioMgr->enmState == PDMACEPFILEMGRSTATE_RUNNING))
1037	{
1038	/* We got woken up because an endpoint issued new requests. Queue them. */
1039	rc = pdmacFileAioMgrNormalCheckEndpoints(pAioMgr);
1040	CHECK_RC(pAioMgr, rc);
1041
1042	while (pAioMgr->cRequestsActive)
1043	{
1044	RTFILEAIOREQ apReqs[20];
1045	uint32_t cReqsCompleted = 0;
1046	size_t cReqsWait;
1047
1048	if (pAioMgr->cRequestsActive > RT_ELEMENTS(apReqs))
1049	cReqsWait = RT_ELEMENTS(apReqs);
1050	else
1051	cReqsWait = pAioMgr->cRequestsActive;
1052
1053	LogFlow(("Waiting for %d of %d tasks to complete\n", pAioMgr->cRequestsActive, cReqsWait));
1054
1055	rc = RTFileAioCtxWait(pAioMgr->hAioCtx,
1056	cReqsWait,
1057	RT_INDEFINITE_WAIT, apReqs,
1058	RT_ELEMENTS(apReqs), &cReqsCompleted);
1059	if (RT_FAILURE(rc) && (rc != VERR_INTERRUPTED))
1060	CHECK_RC(pAioMgr, rc);
1061
1062	LogFlow(("%d tasks completed\n", cReqsCompleted));
1063
1064	for (uint32_t i = 0; i < cReqsCompleted; i++)
1065	{
1066	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint;
1067	size_t cbTransfered = 0;
1068	int rcReq = RTFileAioReqGetRC(apReqs[i], &cbTransfered);
1069	PPDMACTASKFILE pTask = (PPDMACTASKFILE)RTFileAioReqGetUser(apReqs[i]);
1070
1071	pEndpoint = pTask->pEndpoint;
1072
1073	/*
1074	* It is possible that the request failed on Linux with kernels < 2.6.23
1075	* if the passed buffer was allocated with remap_pfn_range or if the file
1076	* is on an NFS endpoint which does not support async and direct I/O at the same time.
1077	* The endpoint will be migrated to a failsafe manager in case a request fails.
1078	*/
1079	if (RT_FAILURE(rcReq))
1080	{
1081	/* Free bounce buffers and the IPRT request. */
1082	pAioMgr->pahReqsFree[pAioMgr->iFreeEntryNext] = apReqs[i];
1083	pAioMgr->iFreeEntryNext = (pAioMgr->iFreeEntryNext + 1) % pAioMgr->cReqEntries;
1084
1085	pAioMgr->cRequestsActive--;
1086	pEndpoint->AioMgr.cRequestsActive--;
1087	pEndpoint->AioMgr.cReqsProcessed++;
1088
1089	if (pTask->fBounceBuffer)
1090	RTMemFree(pTask->pvBounceBuffer);
1091
1092	/* Queue the request on the pending list. */
1093	pTask->pNext = pEndpoint->AioMgr.pReqsPendingHead;
1094	pEndpoint->AioMgr.pReqsPendingHead = pTask;
1095
1096	/* Create a new failsafe manager if neccessary. */
1097	if (!pEndpoint->AioMgr.fMoving)
1098	{
1099	PPDMACEPFILEMGR pAioMgrFailsafe;
1100
1101	LogRel(("%s: Request %#p failed with rc=%Rrc, migrating endpoint %s to failsafe manager.\n",
1102	RTThreadGetName(pAioMgr->Thread), pTask, rcReq, pEndpoint->Core.pszUri));
1103
1104	pEndpoint->AioMgr.fMoving = true;
1105
1106	rc = pdmacFileAioMgrCreate((PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass,
1107	&pAioMgrFailsafe, true);
1108	AssertRC(rc);
1109
1110	pEndpoint->AioMgr.pAioMgrDst = pAioMgrFailsafe;
1111
1112	/* Update the flags to open the file with. Disable async I/O and enable the host cache. */
1113	pEndpoint->fFlags &= ~(RTFILE_O_ASYNC_IO \| RTFILE_O_NO_CACHE);
1114	}
1115
1116	/* If this was the last request for the endpoint migrate it to the new manager. */
1117	if (!pEndpoint->AioMgr.cRequestsActive)
1118	{
1119	bool fReqsPending = pdmacFileAioMgrNormalRemoveEndpoint(pEndpoint);
1120	Assert(!fReqsPending);
1121
1122	rc = pdmacFileAioMgrAddEndpoint(pEndpoint->AioMgr.pAioMgrDst, pEndpoint);
1123	AssertRC(rc);
1124	}
1125	}
1126	else
1127	{
1128	AssertMsg(( (cbTransfered == pTask->DataSeg.cbSeg)
1129	\|\| (pTask->fBounceBuffer && (cbTransfered >= pTask->DataSeg.cbSeg))),
1130	("Task didn't completed successfully (rc=%Rrc) or was incomplete (cbTransfered=%u)\n", rcReq, cbTransfered));
1131
1132	if (pTask->fPrefetch)
1133	{
1134	Assert(pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE);
1135	Assert(pTask->fBounceBuffer);
1136
1137	memcpy(((uint8_t *)pTask->pvBounceBuffer) + pTask->uBounceBufOffset,
1138	pTask->DataSeg.pvSeg,
1139	pTask->DataSeg.cbSeg);
1140
1141	/* Write it now. */
1142	pTask->fPrefetch = false;
1143	size_t cbToTransfer = RT_ALIGN_Z(pTask->DataSeg.cbSeg, 512);
1144	RTFOFF offStart = pTask->Off & ~(RTFOFF)(512-1);
1145
1146	/* Grow the file if needed. */
1147	if (RT_UNLIKELY((uint64_t)(pTask->Off + pTask->DataSeg.cbSeg) > pEndpoint->cbFile))
1148	{
1149	ASMAtomicWriteU64(&pEndpoint->cbFile, pTask->Off + pTask->DataSeg.cbSeg);
1150	RTFileSetSize(pEndpoint->File, pTask->Off + pTask->DataSeg.cbSeg);
1151	}
1152
1153	rc = RTFileAioReqPrepareWrite(apReqs[i], pEndpoint->File,
1154	offStart, pTask->pvBounceBuffer, cbToTransfer, pTask);
1155	AssertRC(rc);
1156	rc = RTFileAioCtxSubmit(pAioMgr->hAioCtx, &apReqs[i], 1);
1157	AssertRC(rc);
1158	}
1159	else
1160	{
1161	if (pTask->fBounceBuffer)
1162	{
1163	if (pTask->enmTransferType == PDMACTASKFILETRANSFER_READ)
1164	memcpy(pTask->DataSeg.pvSeg,
1165	((uint8_t *)pTask->pvBounceBuffer) + pTask->uBounceBufOffset,
1166	pTask->DataSeg.cbSeg);
1167
1168	RTMemPageFree(pTask->pvBounceBuffer);
1169	}
1170
1171	/* Put the entry on the free array */
1172	pAioMgr->pahReqsFree[pAioMgr->iFreeEntryNext] = apReqs[i];
1173	pAioMgr->iFreeEntryNext = (pAioMgr->iFreeEntryNext + 1) % pAioMgr->cReqEntries;
1174
1175	pAioMgr->cRequestsActive--;
1176	pEndpoint->AioMgr.cRequestsActive--;
1177	pEndpoint->AioMgr.cReqsProcessed++;
1178
1179	/* Free the lock and process pending tasks if neccessary */
1180	pdmacFileAioMgrNormalRangeLockFree(pAioMgr, pEndpoint, pTask->pRangeLock);
1181
1182	/* Call completion callback */
1183	pTask->pfnCompleted(pTask, pTask->pvUser);
1184	pdmacFileTaskFree(pEndpoint, pTask);
1185
1186	/*
1187	* If there is no request left on the endpoint but a flush request is set
1188	* it completed now and we notify the owner.
1189	* Furthermore we look for new requests and continue.
1190	*/
1191	if (!pEndpoint->AioMgr.cRequestsActive && pEndpoint->pFlushReq)
1192	{
1193	/* Call completion callback */
1194	pTask = pEndpoint->pFlushReq;
1195	pEndpoint->pFlushReq = NULL;
1196
1197	AssertMsg(pTask->pEndpoint == pEndpoint, ("Endpoint of the flush request does not match assigned one\n"));
1198
1199	pTask->pfnCompleted(pTask, pTask->pvUser);
1200	pdmacFileTaskFree(pEndpoint, pTask);
1201	}
1202	else if (RT_UNLIKELY(!pEndpoint->AioMgr.cRequestsActive && pEndpoint->AioMgr.fMoving))
1203	{
1204	/* If the endpoint is about to be migrated do it now. */
1205	bool fReqsPending = pdmacFileAioMgrNormalRemoveEndpoint(pEndpoint);
1206	Assert(!fReqsPending);
1207
1208	rc = pdmacFileAioMgrAddEndpoint(pEndpoint->AioMgr.pAioMgrDst, pEndpoint);
1209	AssertRC(rc);
1210	}
1211	}
1212	} /* request completed successfully */
1213	} /* for every completed request */
1214
1215	/* Check for an external blocking event before we go to sleep again. */
1216	if (pAioMgr->fBlockingEventPending)
1217	{
1218	rc = pdmacFileAioMgrNormalProcessBlockingEvent(pAioMgr);
1219	CHECK_RC(pAioMgr, rc);
1220	}
1221
1222	/* Update load statistics. */
1223	uint64_t uMillisCurr = RTTimeMilliTS();
1224	if (uMillisCurr > uMillisEnd)
1225	{
1226	PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointCurr = pAioMgr->pEndpointsHead;
1227
1228	/* Calculate timespan. */
1229	uMillisCurr -= uMillisEnd;
1230
1231	while (pEndpointCurr)
1232	{
1233	pEndpointCurr->AioMgr.cReqsPerSec = pEndpointCurr->AioMgr.cReqsProcessed / (uMillisCurr + PDMACEPFILEMGR_LOAD_UPDATE_PERIOD);
1234	pEndpointCurr->AioMgr.cReqsProcessed = 0;
1235	pEndpointCurr = pEndpointCurr->AioMgr.pEndpointNext;
1236	}
1237
1238	/* Set new update interval */
1239	uMillisEnd = RTTimeMilliTS() + PDMACEPFILEMGR_LOAD_UPDATE_PERIOD;
1240	}
1241
1242	/* Check endpoints for new requests. */
1243	rc = pdmacFileAioMgrNormalCheckEndpoints(pAioMgr);
1244	CHECK_RC(pAioMgr, rc);
1245	} /* while requests are active. */
1246	} /* if still running */
1247	} /* while running */
1248
1249	return rc;
1250	}
1251
1252	#undef CHECK_RC
1253

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/PDMAsyncCompletionFileNormal.cpp@ 26550

Download in other formats: