MMAllHyper.cpp@ 11311

Last change on this file since 11311 was 11311, checked in by vboxsync, 16 years ago
VMM: ELEMENTS -> RT_ELEMENTS.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 45.6 KB

Line
1	/* $Id: MMAllHyper.cpp 11311 2008-08-08 23:31:54Z vboxsync $ */
2	/** @file
3	* MM - Memory Monitor(/Manager) - Hypervisor Memory Area, All Contexts.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22
23	/*******************************************************************************
24	* Header Files *
25	*******************************************************************************/
26	#define LOG_GROUP LOG_GROUP_MM_HYPER_HEAP
27	#include <VBox/mm.h>
28	#include <VBox/stam.h>
29	#include "MMInternal.h"
30	#include <VBox/vm.h>
31
32	#include <VBox/err.h>
33	#include <VBox/param.h>
34	#include <iprt/assert.h>
35	#include <VBox/log.h>
36	#include <iprt/asm.h>
37	#include <iprt/string.h>
38
39
40	/*******************************************************************************
41	* Defined Constants And Macros *
42	*******************************************************************************/
43	#define ASSERT_L(u1, u2) AssertMsg((u1) < (u2), ("u1=%#x u2=%#x\n", u1, u2))
44	#define ASSERT_LE(u1, u2) AssertMsg((u1) <= (u2), ("u1=%#x u2=%#x\n", u1, u2))
45	#define ASSERT_GE(u1, u2) AssertMsg((u1) >= (u2), ("u1=%#x u2=%#x\n", u1, u2))
46	#define ASSERT_ALIGN(u1) AssertMsg(!((u1) & (MMHYPER_HEAP_ALIGN_MIN - 1)), ("u1=%#x (%d)\n", u1, u1))
47
48	#define ASSERT_OFFPREV(pHeap, pChunk) \
49	do { Assert(MMHYPERCHUNK_GET_OFFPREV(pChunk) <= 0); \
50	Assert(MMHYPERCHUNK_GET_OFFPREV(pChunk) >= (intptr_t)CTXSUFF((pHeap)->pbHeap) - (intptr_t)(pChunk)); \
51	AssertMsg( MMHYPERCHUNK_GET_OFFPREV(pChunk) != 0 \
52	\|\| (uint8_t *)(pChunk) == CTXSUFF((pHeap)->pbHeap), \
53	("pChunk=%p pvHyperHeap=%p\n", (pChunk), CTXSUFF((pHeap)->pbHeap))); \
54	} while (0)
55
56	#define ASSERT_OFFNEXT(pHeap, pChunk) \
57	do { ASSERT_ALIGN((pChunk)->offNext); \
58	ASSERT_L((pChunk)->offNext, (uintptr_t)CTXSUFF((pHeap)->pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
59	} while (0)
60
61	#define ASSERT_OFFHEAP(pHeap, pChunk) \
62	do { Assert((pChunk)->offHeap); \
63	AssertMsg((PMMHYPERHEAP)((pChunk)->offHeap + (uintptr_t)pChunk) == (pHeap), \
64	("offHeap=%RX32 pChunk=%p pHeap=%p\n", (pChunk)->offHeap, (pChunk), (pHeap))); \
65	Assert((pHeap)->u32Magic == MMHYPERHEAP_MAGIC); \
66	} while (0)
67
68	#ifdef VBOX_WITH_STATISTICS
69	#define ASSERT_OFFSTAT(pHeap, pChunk) \
70	do { if (MMHYPERCHUNK_ISFREE(pChunk)) \
71	Assert(!(pChunk)->offStat); \
72	else if ((pChunk)->offStat) \
73	{ \
74	Assert((pChunk)->offStat); \
75	AssertMsg(!((pChunk)->offStat & (MMHYPER_HEAP_ALIGN_MIN - 1)), ("offStat=%RX32\n", (pChunk)->offStat)); \
76	uintptr_t uPtr = (uintptr_t)(pChunk)->offStat + (uintptr_t)pChunk; NOREF(uPtr); \
77	AssertMsg(uPtr - (uintptr_t)CTXSUFF((pHeap)->pbHeap) < (pHeap)->offPageAligned, \
78	("%p - %p < %RX32\n", uPtr, CTXSUFF((pHeap)->pbHeap), (pHeap)->offPageAligned)); \
79	} \
80	} while (0)
81	#else
82	#define ASSERT_OFFSTAT(pHeap, pChunk) \
83	do { Assert(!(pChunk)->offStat); \
84	} while (0)
85	#endif
86
87	#define ASSERT_CHUNK(pHeap, pChunk) \
88	do { ASSERT_OFFNEXT(pHeap, pChunk); \
89	ASSERT_OFFPREV(pHeap, pChunk); \
90	ASSERT_OFFHEAP(pHeap, pChunk); \
91	ASSERT_OFFSTAT(pHeap, pChunk); \
92	} while (0)
93	#define ASSERT_CHUNK_USED(pHeap, pChunk) \
94	do { ASSERT_OFFNEXT(pHeap, pChunk); \
95	ASSERT_OFFPREV(pHeap, pChunk); \
96	Assert(MMHYPERCHUNK_ISUSED(pChunk)); \
97	} while (0)
98
99	#define ASSERT_FREE_OFFPREV(pHeap, pChunk) \
100	do { ASSERT_ALIGN((pChunk)->offPrev); \
101	ASSERT_GE(((pChunk)->offPrev & (MMHYPER_HEAP_ALIGN_MIN - 1)), (intptr_t)CTXSUFF((pHeap)->pbHeap) - (intptr_t)(pChunk)); \
102	Assert((pChunk)->offPrev != MMHYPERCHUNK_GET_OFFPREV(&(pChunk)->core) \|\| !(pChunk)->offPrev); \
103	AssertMsg( (pChunk)->offPrev \
104	\|\| (uintptr_t)(pChunk) - (uintptr_t)CTXSUFF((pHeap)->pbHeap) == (pHeap)->offFreeHead, \
105	("pChunk=%p offChunk=%#x offFreeHead=%#x\n", (pChunk), (uintptr_t)(pChunk) - (uintptr_t)CTXSUFF((pHeap)->pbHeap),\
106	(pHeap)->offFreeHead)); \
107	} while (0)
108
109	#define ASSERT_FREE_OFFNEXT(pHeap, pChunk) \
110	do { ASSERT_ALIGN((pChunk)->offNext); \
111	ASSERT_L((pChunk)->offNext, (uintptr_t)CTXSUFF((pHeap)->pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
112	Assert((pChunk)->offNext != (pChunk)->core.offNext \|\| !(pChunk)->offNext); \
113	AssertMsg( (pChunk)->offNext \
114	\|\| (uintptr_t)(pChunk) - (uintptr_t)CTXSUFF((pHeap)->pbHeap) == (pHeap)->offFreeTail, \
115	("pChunk=%p offChunk=%#x offFreeTail=%#x\n", (pChunk), (uintptr_t)(pChunk) - (uintptr_t)CTXSUFF((pHeap)->pbHeap), \
116	(pHeap)->offFreeTail)); \
117	} while (0)
118
119	#define ASSERT_FREE_CB(pHeap, pChunk) \
120	do { ASSERT_ALIGN((pChunk)->cb); \
121	Assert((pChunk)->cb > 0); \
122	if ((pChunk)->core.offNext) \
123	AssertMsg((pChunk)->cb == ((pChunk)->core.offNext - sizeof(MMHYPERCHUNK)), \
124	("cb=%d offNext=%d\n", (pChunk)->cb, (pChunk)->core.offNext)); \
125	else \
126	ASSERT_LE((pChunk)->cb, (uintptr_t)CTXSUFF((pHeap)->pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
127	} while (0)
128
129	#define ASSERT_CHUNK_FREE(pHeap, pChunk) \
130	do { ASSERT_CHUNK(pHeap, &(pChunk)->core); \
131	Assert(MMHYPERCHUNK_ISFREE(pChunk)); \
132	ASSERT_FREE_OFFNEXT(pHeap, pChunk); \
133	ASSERT_FREE_OFFPREV(pHeap, pChunk); \
134	ASSERT_FREE_CB(pHeap, pChunk); \
135	} while (0)
136
137
138	/*******************************************************************************
139	* Internal Functions *
140	*******************************************************************************/
141	static PMMHYPERCHUNK mmHyperAllocChunk(PMMHYPERHEAP pHeap, uint32_t cb, unsigned uAlignment);
142	static void *mmHyperAllocPages(PMMHYPERHEAP pHeap, uint32_t cb);
143	#ifdef VBOX_WITH_STATISTICS
144	static PMMHYPERSTAT mmHyperStat(PMMHYPERHEAP pHeap, MMTAG enmTag);
145	#ifdef IN_RING3
146	static void mmR3HyperStatRegisterOne(PVM pVM, PMMHYPERSTAT pStat);
147	#endif
148	#endif
149	static int mmHyperFree(PMMHYPERHEAP pHeap, PMMHYPERCHUNK pChunk);
150	#ifdef MMHYPER_HEAP_STRICT
151	static void mmHyperHeapCheck(PMMHYPERHEAP pHeap);
152	#endif
153
154
155	/**
156	* Allocates memory in the Hypervisor (GC VMM) area.
157	* The returned memory is of course zeroed.
158	*
159	* @returns VBox status code.
160	* @param pVM The VM to operate on.
161	* @param cb Number of bytes to allocate.
162	* @param uAlignment Required memory alignment in bytes.
163	* Values are 0,8,16,32 and PAGE_SIZE.
164	* 0 -> default alignment, i.e. 8 bytes.
165	* @param enmTag The statistics tag.
166	* @param ppv Where to store the address to the allocated
167	* memory.
168	* @remark This is assumed not to be used at times when serialization is required.
169	*/
170	MMDECL(int) MMHyperAlloc(PVM pVM, size_t cb, unsigned uAlignment, MMTAG enmTag, void **ppv)
171	{
172	AssertMsg(cb >= 8, ("Hey! Do you really mean to allocate less than 8 bytes?! cb=%d\n", cb));
173
174	/*
175	* Validate input and adjust it to reasonable values.
176	*/
177	if (!uAlignment \|\| uAlignment < MMHYPER_HEAP_ALIGN_MIN)
178	uAlignment = MMHYPER_HEAP_ALIGN_MIN;
179	uint32_t cbAligned;
180	switch (uAlignment)
181	{
182	case 8:
183	case 16:
184	case 32:
185	cbAligned = RT_ALIGN(cb, MMHYPER_HEAP_ALIGN_MIN);
186	if (!cbAligned \|\| cbAligned < cb)
187	{
188	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
189	AssertMsgFailed(("Nice try.\n"));
190	return VERR_INVALID_PARAMETER;
191	}
192	break;
193
194	case PAGE_SIZE:
195	AssertMsg(RT_ALIGN(cb, PAGE_SIZE) == cb, ("The size isn't page aligned. (cb=%#x)\n", cb));
196	cbAligned = RT_ALIGN(cb, PAGE_SIZE);
197	if (!cbAligned)
198	{
199	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
200	AssertMsgFailed(("Nice try.\n"));
201	return VERR_INVALID_PARAMETER;
202	}
203	break;
204
205	default:
206	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
207	AssertMsgFailed(("Invalid alignment %u\n", uAlignment));
208	return VERR_INVALID_PARAMETER;
209	}
210
211
212	/*
213	* Get heap and statisticsStatistics.
214	*/
215	PMMHYPERHEAP pHeap = CTXSUFF(pVM->mm.s.pHyperHeap);
216	#ifdef VBOX_WITH_STATISTICS
217	PMMHYPERSTAT pStat = mmHyperStat(pHeap, enmTag);
218	if (!pStat)
219	{
220	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_MM_HYPER_NO_MEMORY\n", cb, uAlignment));
221	AssertMsgFailed(("Failed to allocate statistics!\n"));
222	return VERR_MM_HYPER_NO_MEMORY;
223	}
224	#endif
225	if (uAlignment < PAGE_SIZE)
226	{
227	/*
228	* Allocate a chunk.
229	*/
230	PMMHYPERCHUNK pChunk = mmHyperAllocChunk(pHeap, cbAligned, uAlignment);
231	if (pChunk)
232	{
233	#ifdef VBOX_WITH_STATISTICS
234	const uint32_t cbChunk = pChunk->offNext
235	? pChunk->offNext
236	: CTXSUFF(pHeap->pbHeap) + pHeap->offPageAligned - (uint8_t *)pChunk;
237	pStat->cbAllocated += (uint32_t)cbChunk;
238	pStat->cbCurAllocated += (uint32_t)cbChunk;
239	if (pStat->cbCurAllocated > pStat->cbMaxAllocated)
240	pStat->cbMaxAllocated = pStat->cbCurAllocated;
241	pStat->cAllocations++;
242	pChunk->offStat = (uintptr_t)pStat - (uintptr_t)pChunk;
243	#else
244	pChunk->offStat = 0;
245	#endif
246	void *pv = pChunk + 1;
247	*ppv = pv;
248	ASMMemZero32(pv, cbAligned);
249	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VINF_SUCCESS and *ppv=%p\n", cb, uAlignment, pv));
250	return VINF_SUCCESS;
251	}
252	}
253	else
254	{
255	/*
256	* Allocate page aligned memory.
257	*/
258	void *pv = mmHyperAllocPages(pHeap, cbAligned);
259	if (pv)
260	{
261	#ifdef VBOX_WITH_STATISTICS
262	pStat->cbAllocated += cbAligned;
263	pStat->cbCurAllocated += cbAligned;
264	if (pStat->cbCurAllocated > pStat->cbMaxAllocated)
265	pStat->cbMaxAllocated = pStat->cbCurAllocated;
266	pStat->cAllocations++;
267	#endif
268	*ppv = pv;
269	/* ASMMemZero32(pv, cbAligned); - not required since memory is alloc-only and SUPPageAlloc zeros it. */
270	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VINF_SUCCESS and *ppv=%p\n", cb, uAlignment, ppv));
271	return VINF_SUCCESS;
272	}
273	}
274
275	#ifdef VBOX_WITH_STATISTICS
276	pStat->cAllocations++;
277	pStat->cFailures++;
278	#endif
279	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_MM_HYPER_NO_MEMORY\n", cb, uAlignment));
280	AssertMsgFailed(("Failed to allocate %d bytes!\n", cb));
281	return VERR_MM_HYPER_NO_MEMORY;
282	}
283
284
285
286	/**
287	* Allocates a chunk of memory from the specified heap.
288	* The caller validates the parameters of this request.
289	*
290	* @returns Pointer to the allocated chunk.
291	* @returns NULL on failure.
292	* @param pHeap The heap.
293	* @param cb Size of the memory block to allocate.
294	* @param uAlignment The alignment specifications for the allocated block.
295	* @internal
296	*/
297	static PMMHYPERCHUNK mmHyperAllocChunk(PMMHYPERHEAP pHeap, uint32_t cb, unsigned uAlignment)
298	{
299	Log3(("mmHyperAllocChunk: Enter cb=%#x uAlignment=%#x\n", cb, uAlignment));
300	#ifdef MMHYPER_HEAP_STRICT
301	mmHyperHeapCheck(pHeap);
302	#endif
303	#ifdef MMHYPER_HEAP_STRICT_FENCE
304	uint32_t cbFence = RT_MAX(MMHYPER_HEAP_STRICT_FENCE_SIZE, uAlignment);
305	cb += cbFence;
306	#endif
307
308	/*
309	* Check if there are any free chunks. (NIL_OFFSET use/not-use forces this check)
310	*/
311	if (pHeap->offFreeHead == NIL_OFFSET)
312	return NULL;
313
314	/*
315	* Small alignments - from the front of the heap.
316	*
317	* Must split off free chunks at the end to prevent messing up the
318	* last free node which we take the page aligned memory from the top of.
319	*/
320	PMMHYPERCHUNK pRet = NULL;
321	PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)((char *)CTXSUFF(pHeap->pbHeap) + pHeap->offFreeHead);
322	while (pFree)
323	{
324	ASSERT_CHUNK_FREE(pHeap, pFree);
325	if (pFree->cb >= cb)
326	{
327	unsigned offAlign = (uintptr_t)(&pFree->core + 1) & (uAlignment - 1);
328	if (offAlign)
329	offAlign = uAlignment - offAlign;
330	if (!offAlign \|\| pFree->cb - offAlign >= cb)
331	{
332	Log3(("mmHyperAllocChunk: Using pFree=%p pFree->cb=%d offAlign=%d\n", pFree, pFree->cb, offAlign));
333
334	/*
335	* Adjust the node in front.
336	* Because of multiple alignments we need to special case allocation of the first block.
337	*/
338	if (offAlign)
339	{
340	MMHYPERCHUNKFREE Free = *pFree;
341	if (MMHYPERCHUNK_GET_OFFPREV(&pFree->core))
342	{
343	/* just add a bit of memory to it. */
344	PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + MMHYPERCHUNK_GET_OFFPREV(&Free.core));
345	pPrev->core.offNext += offAlign;
346	AssertMsg(!MMHYPERCHUNK_ISFREE(&pPrev->core), ("Impossible!\n"));
347	Log3(("mmHyperAllocChunk: Added %d bytes to %p\n", offAlign, pPrev));
348	}
349	else
350	{
351	/* make new head node, mark it USED for simplisity. */
352	PMMHYPERCHUNK pPrev = (PMMHYPERCHUNK)CTXSUFF(pHeap->pbHeap);
353	Assert(pPrev == &pFree->core);
354	pPrev->offPrev = 0;
355	MMHYPERCHUNK_SET_TYPE(pPrev, MMHYPERCHUNK_FLAGS_USED);
356	pPrev->offNext = offAlign;
357	Log3(("mmHyperAllocChunk: Created new first node of %d bytes\n", offAlign));
358
359	}
360	Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - offAlign, -(int)offAlign));
361	pHeap->cbFree -= offAlign;
362
363	/* Recreate pFree node and adjusting everything... */
364	pFree = (PMMHYPERCHUNKFREE)((char *)pFree + offAlign);
365	*pFree = Free;
366
367	pFree->cb -= offAlign;
368	if (pFree->core.offNext)
369	{
370	pFree->core.offNext -= offAlign;
371	PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pFree + pFree->core.offNext);
372	MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pFree->core.offNext);
373	ASSERT_CHUNK(pHeap, pNext);
374	}
375	if (MMHYPERCHUNK_GET_OFFPREV(&pFree->core))
376	MMHYPERCHUNK_SET_OFFPREV(&pFree->core, MMHYPERCHUNK_GET_OFFPREV(&pFree->core) - offAlign);
377
378	if (pFree->offNext)
379	{
380	pFree->offNext -= offAlign;
381	PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext);
382	pNext->offPrev = -(int32_t)pFree->offNext;
383	ASSERT_CHUNK_FREE(pHeap, pNext);
384	}
385	else
386	pHeap->offFreeTail += offAlign;
387	if (pFree->offPrev)
388	{
389	pFree->offPrev -= offAlign;
390	PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
391	pPrev->offNext = -pFree->offPrev;
392	ASSERT_CHUNK_FREE(pHeap, pPrev);
393	}
394	else
395	pHeap->offFreeHead += offAlign;
396	pFree->core.offHeap = (uintptr_t)pHeap - (uintptr_t)pFree;
397	pFree->core.offStat = 0;
398	ASSERT_CHUNK_FREE(pHeap, pFree);
399	Log3(("mmHyperAllocChunk: Realigned pFree=%p\n", pFree));
400	}
401
402	/*
403	* Split off a new FREE chunk?
404	*/
405	if (pFree->cb >= cb + RT_ALIGN(sizeof(MMHYPERCHUNKFREE), MMHYPER_HEAP_ALIGN_MIN))
406	{
407	/*
408	* Move the FREE chunk up to make room for the new USED chunk.
409	*/
410	const int off = cb + sizeof(MMHYPERCHUNK);
411	PMMHYPERCHUNKFREE pNew = (PMMHYPERCHUNKFREE)((char *)&pFree->core + off);
412	pNew = pFree;
413	pNew->cb -= off;
414	if (pNew->core.offNext)
415	{
416	pNew->core.offNext -= off;
417	PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pNew + pNew->core.offNext);
418	MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pNew->core.offNext);
419	ASSERT_CHUNK(pHeap, pNext);
420	}
421	pNew->core.offPrev = -off;
422	MMHYPERCHUNK_SET_TYPE(pNew, MMHYPERCHUNK_FLAGS_FREE);
423
424	if (pNew->offNext)
425	{
426	pNew->offNext -= off;
427	PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pNew + pNew->offNext);
428	pNext->offPrev = -(int32_t)pNew->offNext;
429	ASSERT_CHUNK_FREE(pHeap, pNext);
430	}
431	else
432	pHeap->offFreeTail += off;
433	if (pNew->offPrev)
434	{
435	pNew->offPrev -= off;
436	PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pNew + pNew->offPrev);
437	pPrev->offNext = -pNew->offPrev;
438	ASSERT_CHUNK_FREE(pHeap, pPrev);
439	}
440	else
441	pHeap->offFreeHead += off;
442	pNew->core.offHeap = (uintptr_t)pHeap - (uintptr_t)pNew;
443	pNew->core.offStat = 0;
444	ASSERT_CHUNK_FREE(pHeap, pNew);
445
446	/*
447	* Update the old FREE node making it a USED node.
448	*/
449	pFree->core.offNext = off;
450	MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_USED);
451
452
453	Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree,
454	pHeap->cbFree - (cb + sizeof(MMHYPERCHUNK)), -(int)(cb + sizeof(MMHYPERCHUNK))));
455	pHeap->cbFree -= (uint32_t)(cb + sizeof(MMHYPERCHUNK));
456	pRet = &pFree->core;
457	ASSERT_CHUNK(pHeap, &pFree->core);
458	Log3(("mmHyperAllocChunk: Created free chunk pNew=%p cb=%d\n", pNew, pNew->cb));
459	}
460	else
461	{
462	/*
463	* Link out of free list.
464	*/
465	if (pFree->offNext)
466	{
467	PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext);
468	if (pFree->offPrev)
469	{
470	pNext->offPrev += pFree->offPrev;
471	PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
472	pPrev->offNext += pFree->offNext;
473	ASSERT_CHUNK_FREE(pHeap, pPrev);
474	}
475	else
476	{
477	pHeap->offFreeHead += pFree->offNext;
478	pNext->offPrev = 0;
479	}
480	ASSERT_CHUNK_FREE(pHeap, pNext);
481	}
482	else
483	{
484	if (pFree->offPrev)
485	{
486	pHeap->offFreeTail += pFree->offPrev;
487	PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
488	pPrev->offNext = 0;
489	ASSERT_CHUNK_FREE(pHeap, pPrev);
490	}
491	else
492	{
493	pHeap->offFreeHead = NIL_OFFSET;
494	pHeap->offFreeTail = NIL_OFFSET;
495	}
496	}
497
498	Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree,
499	pHeap->cbFree - pFree->cb, -(int32_t)pFree->cb));
500	pHeap->cbFree -= pFree->cb;
501	MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_USED);
502	pRet = &pFree->core;
503	ASSERT_CHUNK(pHeap, &pFree->core);
504	Log3(("mmHyperAllocChunk: Converted free chunk %p to used chunk.\n", pFree));
505	}
506	Log3(("mmHyperAllocChunk: Returning %p\n", pRet));
507	break;
508	}
509	}
510
511	/* next */
512	pFree = pFree->offNext ? (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext) : NULL;
513	}
514
515	#ifdef MMHYPER_HEAP_STRICT_FENCE
516	uint32_t pu32End = (uint32_t )((uint8_t *)(pRet + 1) + cb);
517	uint32_t *pu32EndReal = pRet->offNext
518	? (uint32_t )((uint8_t )pRet + pRet->offNext)
519	: (uint32_t *)(pHeap->CTXSUFF(pbHeap) + pHeap->cbHeap);
520	cbFence += (uintptr_t)pu32EndReal - (uintptr_t)pu32End; Assert(!(cbFence & 0x3));
521	ASMMemFill32((uint8_t *)pu32EndReal - cbFence, cbFence, MMHYPER_HEAP_STRICT_FENCE_U32);
522	pu32EndReal[-1] = cbFence;
523	#endif
524	#ifdef MMHYPER_HEAP_STRICT
525	mmHyperHeapCheck(pHeap);
526	#endif
527	return pRet;
528	}
529
530
531	/**
532	* Allocates one or more pages of memory from the specified heap.
533	* The caller validates the parameters of this request.
534	*
535	* @returns Pointer to the allocated chunk.
536	* @returns NULL on failure.
537	* @param pHeap The heap.
538	* @param cb Size of the memory block to allocate.
539	* @internal
540	*/
541	static void *mmHyperAllocPages(PMMHYPERHEAP pHeap, uint32_t cb)
542	{
543	Log3(("mmHyperAllocPages: Enter cb=%#x\n", cb));
544
545	#ifdef MMHYPER_HEAP_STRICT
546	mmHyperHeapCheck(pHeap);
547	#endif
548
549	/*
550	* Check if there are any free chunks. (NIL_OFFSET use/not-use forces this check)
551	*/
552	if (pHeap->offFreeHead == NIL_OFFSET)
553	return NULL;
554
555	/*
556	* Page aligned chunks.
557	*
558	* Page aligned chunks can only be allocated from the last FREE chunk.
559	* This is for reasons of simplicity and fragmentation. Page aligned memory
560	* must also be allocated in page aligned sizes. Page aligned memory cannot
561	* be freed either.
562	*
563	* So, for this to work, the last FREE chunk needs to end on a page aligned
564	* boundrary.
565	*/
566	PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)((char *)CTXSUFF(pHeap->pbHeap) + pHeap->offFreeTail);
567	ASSERT_CHUNK_FREE(pHeap, pFree);
568	if ( (((uintptr_t)(&pFree->core + 1) + pFree->cb) & (PAGE_OFFSET_MASK - 1))
569	\|\| pFree->cb + sizeof(MMHYPERCHUNK) < cb)
570	{
571	Log3(("mmHyperAllocPages: Not enough/no page aligned memory!\n"));
572	return NULL;
573	}
574
575	void *pvRet;
576	if (pFree->cb > cb)
577	{
578	/*
579	* Simple, just cut the top of the free node and return it.
580	*/
581	pFree->cb -= cb;
582	pvRet = (char *)(&pFree->core + 1) + pFree->cb;
583	AssertMsg(RT_ALIGN_P(pvRet, PAGE_SIZE) == pvRet, ("pvRet=%p cb=%#x pFree=%p pFree->cb=%#x\n", pvRet, cb, pFree, pFree->cb));
584	Log3(("mmHyperAllocPages: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - cb, -(int)cb));
585	pHeap->cbFree -= cb;
586	ASSERT_CHUNK_FREE(pHeap, pFree);
587	Log3(("mmHyperAllocPages: Allocated from pFree=%p new pFree->cb=%d\n", pFree, pFree->cb));
588	}
589	else
590	{
591	/*
592	* Unlink the FREE node.
593	*/
594	pvRet = (char *)(&pFree->core + 1) + pFree->cb - cb;
595	Log3(("mmHyperAllocPages: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pFree->cb, -(int32_t)pFree->cb));
596	pHeap->cbFree -= pFree->cb;
597
598	/* a scrap of spare memory (unlikely)? add it to the sprevious chunk. */
599	if (pvRet != (void *)pFree)
600	{
601	AssertMsg(MMHYPERCHUNK_GET_OFFPREV(&pFree->core), ("How the beep did someone manage to allocated up all the heap with page aligned memory?!?\n"));
602	PMMHYPERCHUNK pPrev = (PMMHYPERCHUNK)((char *)pFree + MMHYPERCHUNK_GET_OFFPREV(&pFree->core));
603	pPrev->offNext += (uintptr_t)pvRet - (uintptr_t)pFree;
604	AssertMsg(!MMHYPERCHUNK_ISFREE(pPrev), ("Free bug?\n"));
605	#ifdef VBOX_WITH_STATISTICS
606	PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pPrev + pPrev->offStat);
607	pStat->cbAllocated += (uintptr_t)pvRet - (uintptr_t)pFree;
608	pStat->cbCurAllocated += (uintptr_t)pvRet - (uintptr_t)pFree;
609	#endif
610	Log3(("mmHyperAllocPages: Added %d to %p (page align)\n", (uintptr_t)pvRet - (uintptr_t)pFree, pFree));
611	}
612
613	/* unlink from FREE chain. */
614	if (pFree->offPrev)
615	{
616	pHeap->offFreeTail += pFree->offPrev;
617	((PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev))->offNext = 0;
618	}
619	else
620	{
621	pHeap->offFreeTail = NIL_OFFSET;
622	pHeap->offFreeHead = NIL_OFFSET;
623	}
624	Log3(("mmHyperAllocPages: Unlinked pFree=%d\n", pFree));
625	}
626	pHeap->offPageAligned = (uintptr_t)pvRet - (uintptr_t)CTXSUFF(pHeap->pbHeap);
627	Log3(("mmHyperAllocPages: Returning %p (page aligned)\n", pvRet));
628
629	#ifdef MMHYPER_HEAP_STRICT
630	mmHyperHeapCheck(pHeap);
631	#endif
632	return pvRet;
633	}
634
635
636	#ifdef VBOX_WITH_STATISTICS
637	/**
638	* Get the statistic record for a tag.
639	*
640	* @returns Pointer to a stat record.
641	* @returns NULL on failure.
642	* @param pHeap The heap.
643	* @param enmTag The tag.
644	*/
645	static PMMHYPERSTAT mmHyperStat(PMMHYPERHEAP pHeap, MMTAG enmTag)
646	{
647	/* try look it up first. */
648	PMMHYPERSTAT pStat = (PMMHYPERSTAT)RTAvloGCPhysGet(&pHeap->HyperHeapStatTree, enmTag);
649	if (!pStat)
650	{
651	/* try allocate a new one */
652	PMMHYPERCHUNK pChunk = mmHyperAllocChunk(pHeap, RT_ALIGN(sizeof(*pStat), MMHYPER_HEAP_ALIGN_MIN), MMHYPER_HEAP_ALIGN_MIN);
653	if (!pChunk)
654	return NULL;
655	pStat = (PMMHYPERSTAT)(pChunk + 1);
656	pChunk->offStat = (uintptr_t)pStat - (uintptr_t)pChunk;
657
658	ASMMemZero32(pStat, sizeof(*pStat));
659	pStat->Core.Key = enmTag;
660	RTAvloGCPhysInsert(&pHeap->HyperHeapStatTree, &pStat->Core);
661	}
662	if (!pStat->fRegistered)
663	{
664	#ifdef IN_RING3
665	mmR3HyperStatRegisterOne(pHeap->pVMHC, pStat);
666	#else
667	/** @todo schedule a HC action. */
668	#endif
669	}
670	return pStat;
671	}
672
673	#ifdef IN_RING3
674	/**
675	* Registers statistics with STAM.
676	*
677	*/
678	static void mmR3HyperStatRegisterOne(PVM pVM, PMMHYPERSTAT pStat)
679	{
680	if (pStat->fRegistered)
681	return;
682	const char *pszTag = mmR3GetTagName((MMTAG)pStat->Core.Key);
683
684	char szName[128];
685	RTStrPrintf(szName, sizeof(szName), "/MM/HyperHeap/%s/cAllocations", pszTag);
686	STAMR3Register(pVM, &pStat->cAllocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_COUNT, "Number of alloc calls.");
687
688	RTStrPrintf(szName, sizeof(szName), "/MM/HyperHeap/%s/cFrees", pszTag);
689	STAMR3Register(pVM, &pStat->cFrees, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_COUNT, "Number of free calls.");
690
691	RTStrPrintf(szName, sizeof(szName), "/MM/HyperHeap/%s/cFailures", pszTag);
692	STAMR3Register(pVM, &pStat->cFailures, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_COUNT, "Number of failures.");
693
694	RTStrPrintf(szName, sizeof(szName), "/MM/HyperHeap/%s/cbAllocated", pszTag);
695	STAMR3Register(pVM, &pStat->cbAllocated, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_BYTES, "Total number of allocated bytes.");
696
697	RTStrPrintf(szName, sizeof(szName), "/MM/HyperHeap/%s/cbFreed", pszTag);
698	STAMR3Register(pVM, &pStat->cbFreed, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_BYTES, "Total number of freed bytes.");
699
700	RTStrPrintf(szName, sizeof(szName), "/MM/HyperHeap/%s/cbCurAllocated", pszTag);
701	STAMR3Register(pVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_BYTES, "Number of bytes currently allocated.");
702
703	RTStrPrintf(szName, sizeof(szName), "/MM/HyperHeap/%s/cbMaxAllocated", pszTag);
704	STAMR3Register(pVM, &pStat->cbMaxAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_BYTES, "Max number of bytes allocated at the same time.");
705
706	pStat->fRegistered = true;
707	}
708	#endif
709
710	#endif
711
712
713	/**
714	* Free memory allocated using MMHyperAlloc().
715	* The caller validates the parameters of this request.
716	*
717	* @returns VBox status code.
718	* @param pVM The VM to operate on.
719	* @param pv The memory to free.
720	* @remark Try avoid free hyper memory.
721	*/
722	MMDECL(int) MMHyperFree(PVM pVM, void *pv)
723	{
724	Log2(("MMHyperFree: pv=%p\n", pv));
725	if (!pv)
726	return VINF_SUCCESS;
727	AssertMsgReturn(RT_ALIGN_P(pv, MMHYPER_HEAP_ALIGN_MIN) == pv,
728	("Invalid pointer %p!\n", pv),
729	VERR_INVALID_POINTER);
730
731	/*
732	* Get the heap and stats.
733	* Validate the chunk at the same time.
734	*/
735	PMMHYPERCHUNK pChunk = (PMMHYPERCHUNK)((PMMHYPERCHUNK)pv - 1);
736
737	AssertMsgReturn( (uintptr_t)pChunk + pChunk->offNext >= (uintptr_t)pChunk
738	\|\| RT_ALIGN_32(pChunk->offNext, MMHYPER_HEAP_ALIGN_MIN) != pChunk->offNext,
739	("%p: offNext=%#RX32\n", pv, pChunk->offNext),
740	VERR_INVALID_POINTER);
741
742	AssertMsgReturn(MMHYPERCHUNK_ISUSED(pChunk),
743	("%p: Not used!\n", pv),
744	VERR_INVALID_POINTER);
745
746	int32_t offPrev = MMHYPERCHUNK_GET_OFFPREV(pChunk);
747	AssertMsgReturn( (uintptr_t)pChunk + offPrev <= (uintptr_t)pChunk
748	&& !((uint32_t)-offPrev & (MMHYPER_HEAP_ALIGN_MIN - 1)),
749	("%p: offPrev=%#RX32!\n", pv, offPrev),
750	VERR_INVALID_POINTER);
751
752	/* statistics */
753	#ifdef VBOX_WITH_STATISTICS
754	PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pChunk + pChunk->offStat);
755	AssertMsgReturn( RT_ALIGN_P(pStat, MMHYPER_HEAP_ALIGN_MIN) == (void *)pStat
756	&& pChunk->offStat,
757	("%p: offStat=%#RX32!\n", pv, pChunk->offStat),
758	VERR_INVALID_POINTER);
759	#else
760	AssertMsgReturn(!pChunk->offStat,
761	("%p: offStat=%#RX32!\n", pv, pChunk->offStat),
762	VERR_INVALID_POINTER);
763	#endif
764
765	/* The heap structure. */
766	PMMHYPERHEAP pHeap = (PMMHYPERHEAP)((uintptr_t)pChunk + pChunk->offHeap);
767	AssertMsgReturn( !((uintptr_t)pHeap & PAGE_OFFSET_MASK)
768	&& pChunk->offHeap,
769	("%p: pHeap=%#x offHeap=%RX32\n", pv, pHeap->u32Magic, pChunk->offHeap),
770	VERR_INVALID_POINTER);
771
772	AssertMsgReturn(pHeap->u32Magic == MMHYPERHEAP_MAGIC,
773	("%p: u32Magic=%#x\n", pv, pHeap->u32Magic),
774	VERR_INVALID_POINTER);
775	Assert(pHeap == CTXSUFF(pVM->mm.s.pHyperHeap));
776
777	/* Some more verifications using additional info from pHeap. */
778	AssertMsgReturn((uintptr_t)pChunk + offPrev >= (uintptr_t)CTXSUFF(pHeap->pbHeap),
779	("%p: offPrev=%#RX32!\n", pv, offPrev),
780	VERR_INVALID_POINTER);
781
782	AssertMsgReturn(pChunk->offNext < pHeap->cbHeap,
783	("%p: offNext=%#RX32!\n", pv, pChunk->offNext),
784	VERR_INVALID_POINTER);
785
786	AssertMsgReturn( (uintptr_t)pv - (uintptr_t)CTXSUFF(pHeap->pbHeap) <= pHeap->offPageAligned,
787	("Invalid pointer %p! (heap: %p-%p)\n", pv, CTXSUFF(pHeap->pbHeap),
788	(char *)CTXSUFF(pHeap->pbHeap) + pHeap->offPageAligned),
789	VERR_INVALID_POINTER);
790
791	#ifdef MMHYPER_HEAP_STRICT
792	mmHyperHeapCheck(pHeap);
793	#endif
794
795	#if defined(VBOX_WITH_STATISTICS) \|\| defined(MMHYPER_HEAP_FREE_POISON)
796	/* calc block size. */
797	const uint32_t cbChunk = pChunk->offNext
798	? pChunk->offNext
799	: CTXSUFF(pHeap->pbHeap) + pHeap->offPageAligned - (uint8_t *)pChunk;
800	#endif
801	#ifdef MMHYPER_HEAP_FREE_POISON
802	/* poison the block */
803	memset(pChunk + 1, MMHYPER_HEAP_FREE_POISON, cbChunk - sizeof(*pChunk));
804	#endif
805
806	#ifdef MMHYPER_HEAP_FREE_DELAY
807	# ifdef MMHYPER_HEAP_FREE_POISON
808	/*
809	* Check poison.
810	*/
811	unsigned i = RT_ELEMENTS(pHeap->aDelayedFrees);
812	while (i-- > 0)
813	if (pHeap->aDelayedFrees[i].offChunk)
814	{
815	PMMHYPERCHUNK pCur = (PMMHYPERCHUNK)((uintptr_t)pHeap + pHeap->aDelayedFrees[i].offChunk);
816	const size_t cb = pCur->offNext
817	? pCur->offNext - sizeof(*pCur)
818	: CTXSUFF(pHeap->pbHeap) + pHeap->offPageAligned - (uint8_t )pCur - sizeof(pCur);
819	uint8_t pab = (uint8_t )(pCur + 1);
820	for (unsigned off = 0; off < cb; off++)
821	AssertReleaseMsg(pab[off] == 0xCB,
822	("caller=%RTptr cb=%#zx off=%#x: %.*Rhxs\n",
823	pHeap->aDelayedFrees[i].uCaller, cb, off, RT_MIN(cb - off, 32), &pab[off]));
824	}
825	# endif /* MMHYPER_HEAP_FREE_POISON */
826
827	/*
828	* Delayed freeing.
829	*/
830	int rc = VINF_SUCCESS;
831	if (pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk)
832	{
833	PMMHYPERCHUNK pChunkFree = (PMMHYPERCHUNK)((uintptr_t)pHeap + pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk);
834	rc = mmHyperFree(pHeap, pChunkFree);
835	}
836	pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk = (uintptr_t)pChunk - (uintptr_t)pHeap;
837	pHeap->aDelayedFrees[pHeap->iDelayedFree].uCaller = (uintptr_t)ASMReturnAddress();
838	pHeap->iDelayedFree = (pHeap->iDelayedFree + 1) % RT_ELEMENTS(pHeap->aDelayedFrees);
839
840	#else /* !MMHYPER_HEAP_FREE_POISON */
841	/*
842	* Call the worker.
843	*/
844	int rc = mmHyperFree(pHeap, pChunk);
845	#endif /* !MMHYPER_HEAP_FREE_POISON */
846
847	/*
848	* Update statistics.
849	*/
850	#ifdef VBOX_WITH_STATISTICS
851	pStat->cFrees++;
852	if (VBOX_SUCCESS(rc))
853	{
854	pStat->cbFreed += cbChunk;
855	pStat->cbCurAllocated -= cbChunk;
856	}
857	else
858	pStat->cFailures++;
859	#endif
860
861	return rc;
862	}
863
864
865	/**
866	* Free memory a memory chunk.
867	*
868	* @returns VBox status code.
869	* @param pHeap The heap.
870	* @param pChunk The memory chunk to free.
871	*/
872	static int mmHyperFree(PMMHYPERHEAP pHeap, PMMHYPERCHUNK pChunk)
873	{
874	Log3(("mmHyperFree: Enter pHeap=%p pChunk=%p\n", pHeap, pChunk));
875	PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)pChunk;
876
877	/*
878	* Insert into the free list (which is sorted on address).
879	*
880	* We'll search towards the end of the heap to locate the
881	* closest FREE chunk.
882	*/
883	PMMHYPERCHUNKFREE pLeft = NULL;
884	PMMHYPERCHUNKFREE pRight = NULL;
885	if (pHeap->offFreeTail != NIL_OFFSET)
886	{
887	if (pFree->core.offNext)
888	{
889	pRight = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->core.offNext);
890	ASSERT_CHUNK(pHeap, &pRight->core);
891	while (!MMHYPERCHUNK_ISFREE(&pRight->core))
892	{
893	if (!pRight->core.offNext)
894	{
895	pRight = NULL;
896	break;
897	}
898	pRight = (PMMHYPERCHUNKFREE)((char *)pRight + pRight->core.offNext);
899	ASSERT_CHUNK(pHeap, &pRight->core);
900	}
901	}
902	if (!pRight)
903	pRight = (PMMHYPERCHUNKFREE)((char )CTXSUFF(pHeap->pbHeap) + pHeap->offFreeTail); /* @todo this can't be correct! 'pLeft = .. ; else' I think */
904	if (pRight)
905	{
906	ASSERT_CHUNK_FREE(pHeap, pRight);
907	if (pRight->offPrev)
908	{
909	pLeft = (PMMHYPERCHUNKFREE)((char *)pRight + pRight->offPrev);
910	ASSERT_CHUNK_FREE(pHeap, pLeft);
911	}
912	}
913	}
914	if (pLeft == pFree)
915	{
916	AssertMsgFailed(("Freed twice! pv=%p (pChunk=%p)\n", pChunk + 1, pChunk));
917	return VERR_INVALID_POINTER;
918	}
919	pChunk->offStat = 0;
920
921	/*
922	* Head free chunk list?
923	*/
924	if (!pLeft)
925	{
926	MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_FREE);
927	pFree->offPrev = 0;
928	pHeap->offFreeHead = (uintptr_t)pFree - (uintptr_t)CTXSUFF(pHeap->pbHeap);
929	if (pRight)
930	{
931	pFree->offNext = (uintptr_t)pRight - (uintptr_t)pFree;
932	pRight->offPrev = -(int32_t)pFree->offNext;
933	}
934	else
935	{
936	pFree->offNext = 0;
937	pHeap->offFreeTail = pHeap->offFreeHead;
938	}
939	Log3(("mmHyperFree: Inserted %p at head of free chain.\n", pFree));
940	}
941	else
942	{
943	/*
944	* Can we merge with left hand free chunk?
945	*/
946	if ((char )pLeft + pLeft->core.offNext == (char )pFree)
947	{
948	if (pFree->core.offNext)
949	{
950	pLeft->core.offNext = pLeft->core.offNext + pFree->core.offNext;
951	MMHYPERCHUNK_SET_OFFPREV(((PMMHYPERCHUNK)((char *)pLeft + pLeft->core.offNext)), -(int32_t)pLeft->core.offNext);
952	}
953	else
954	pLeft->core.offNext = 0;
955	pFree = pLeft;
956	Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pLeft->cb, -(int32_t)pLeft->cb));
957	pHeap->cbFree -= pLeft->cb;
958	Log3(("mmHyperFree: Merging %p into %p (cb=%d).\n", pFree, pLeft, pLeft->cb));
959	}
960	/*
961	* No, just link it into the free list then.
962	*/
963	else
964	{
965	MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_FREE);
966	pFree->offPrev = (uintptr_t)pLeft - (uintptr_t)pFree;
967	pLeft->offNext = -pFree->offPrev;
968	if (pRight)
969	{
970	pFree->offNext = (uintptr_t)pRight - (uintptr_t)pFree;
971	pRight->offPrev = -(int32_t)pFree->offNext;
972	}
973	else
974	{
975	pFree->offNext = 0;
976	pHeap->offFreeTail = (uintptr_t)pFree - (uintptr_t)CTXSUFF(pHeap->pbHeap);
977	}
978	Log3(("mmHyperFree: Inserted %p after %p in free list.\n", pFree, pLeft));
979	}
980	}
981
982	/*
983	* Can we merge with right hand free chunk?
984	*/
985	if (pRight && (char )pRight == (char )pFree + pFree->core.offNext)
986	{
987	/* core */
988	if (pRight->core.offNext)
989	{
990	pFree->core.offNext += pRight->core.offNext;
991	PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pFree + pFree->core.offNext);
992	MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pFree->core.offNext);
993	ASSERT_CHUNK(pHeap, pNext);
994	}
995	else
996	pFree->core.offNext = 0;
997
998	/* free */
999	if (pRight->offNext)
1000	{
1001	pFree->offNext += pRight->offNext;
1002	((PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext))->offPrev = -(int32_t)pFree->offNext;
1003	}
1004	else
1005	{
1006	pFree->offNext = 0;
1007	pHeap->offFreeTail = (uintptr_t)pFree - (uintptr_t)CTXSUFF(pHeap->pbHeap);
1008	}
1009	Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pRight->cb, -(int32_t)pRight->cb));
1010	pHeap->cbFree -= pRight->cb;
1011	Log3(("mmHyperFree: Merged %p (cb=%d) into %p.\n", pRight, pRight->cb, pFree));
1012	}
1013
1014	/* calculate the size. */
1015	if (pFree->core.offNext)
1016	pFree->cb = pFree->core.offNext - sizeof(MMHYPERCHUNK);
1017	else
1018	pFree->cb = pHeap->offPageAligned - ((uintptr_t)pFree - (uintptr_t)CTXSUFF(pHeap->pbHeap)) - sizeof(MMHYPERCHUNK);
1019	Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree + pFree->cb, pFree->cb));
1020	pHeap->cbFree += pFree->cb;
1021	ASSERT_CHUNK_FREE(pHeap, pFree);
1022
1023	#ifdef MMHYPER_HEAP_STRICT
1024	mmHyperHeapCheck(pHeap);
1025	#endif
1026	return VINF_SUCCESS;
1027	}
1028
1029
1030	#if defined(DEBUG) \|\| defined(MMHYPER_HEAP_STRICT)
1031	/**
1032	* Dumps a heap chunk to the log.
1033	*
1034	* @param pHeap Pointer to the heap.
1035	* @param pCur Pointer to the chunk.
1036	*/
1037	static void mmHyperHeapDumpOne(PMMHYPERHEAP pHeap, PMMHYPERCHUNKFREE pCur)
1038	{
1039	if (MMHYPERCHUNK_ISUSED(&pCur->core))
1040	{
1041	if (pCur->core.offStat)
1042	{
1043	PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pCur + pCur->core.offStat);
1044	const char *pszSelf = pCur->core.offStat == sizeof(MMHYPERCHUNK) ? " stat record" : "";
1045	#ifdef IN_RING3
1046	Log(("%p %06x USED offNext=%06x offPrev=-%06x %s%s\n",
1047	pCur, (uintptr_t)pCur - (uintptr_t)CTXSUFF(pHeap->pbHeap),
1048	pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1049	mmR3GetTagName((MMTAG)pStat->Core.Key), pszSelf));
1050	#else
1051	Log(("%p %06x USED offNext=%06x offPrev=-%06x %d%s\n",
1052	pCur, (uintptr_t)pCur - (uintptr_t)CTXSUFF(pHeap->pbHeap),
1053	pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1054	(MMTAG)pStat->Core.Key, pszSelf));
1055	#endif
1056	}
1057	else
1058	Log(("%p %06x USED offNext=%06x offPrev=-%06x\n",
1059	pCur, (uintptr_t)pCur - (uintptr_t)CTXSUFF(pHeap->pbHeap),
1060	pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core)));
1061	}
1062	else
1063	Log(("%p %06x FREE offNext=%06x offPrev=-%06x : cb=%06x offNext=%06x offPrev=-%06x\n",
1064	pCur, (uintptr_t)pCur - (uintptr_t)CTXSUFF(pHeap->pbHeap),
1065	pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core), pCur->cb, pCur->offNext, pCur->offPrev));
1066	}
1067	#endif /* DEBUG \|\| MMHYPER_HEAP_STRICT */
1068
1069
1070	#ifdef MMHYPER_HEAP_STRICT
1071	/**
1072	* Internal consitency check.
1073	*/
1074	static void mmHyperHeapCheck(PMMHYPERHEAP pHeap)
1075	{
1076	PMMHYPERCHUNKFREE pPrev = NULL;
1077	PMMHYPERCHUNKFREE pCur = (PMMHYPERCHUNKFREE)CTXSUFF(pHeap->pbHeap);
1078	for (;;)
1079	{
1080	if (MMHYPERCHUNK_ISUSED(&pCur->core))
1081	ASSERT_CHUNK_USED(pHeap, &pCur->core);
1082	else
1083	ASSERT_CHUNK_FREE(pHeap, pCur);
1084	if (pPrev)
1085	AssertMsg((int32_t)pPrev->core.offNext == -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1086	("pPrev->core.offNext=%d offPrev=%d\n", pPrev->core.offNext, MMHYPERCHUNK_GET_OFFPREV(&pCur->core)));
1087
1088	# ifdef MMHYPER_HEAP_STRICT_FENCE
1089	uint32_t off = (uint8_t *)pCur - CTXSUFF(pHeap->pbHeap);
1090	if ( MMHYPERCHUNK_ISUSED(&pCur->core)
1091	&& off < pHeap->offPageAligned)
1092	{
1093	uint32_t cbCur = pCur->core.offNext
1094	? pCur->core.offNext
1095	: pHeap->cbHeap - off;
1096	uint32_t pu32End = ((uint32_t )((uint8_t *)pCur + cbCur));
1097	uint32_t cbFence = pu32End[-1];
1098	if (RT_UNLIKELY( cbFence >= cbCur - sizeof(*pCur)
1099	\|\| cbFence < MMHYPER_HEAP_STRICT_FENCE_SIZE))
1100	{
1101	mmHyperHeapDumpOne(pHeap, pCur);
1102	Assert(cbFence < cbCur - sizeof(*pCur));
1103	Assert(cbFence >= MMHYPER_HEAP_STRICT_FENCE_SIZE);
1104	}
1105
1106	uint32_t pu32Bad = ASMMemIsAllU32((uint8_t )pu32End - cbFence, cbFence - sizeof(uint32_t), MMHYPER_HEAP_STRICT_FENCE_U32);
1107	if (RT_UNLIKELY(pu32Bad))
1108	{
1109	mmHyperHeapDumpOne(pHeap, pCur);
1110	Assert(!pu32Bad);
1111	}
1112	}
1113	# endif
1114
1115	/* next */
1116	if (!pCur->core.offNext)
1117	break;
1118	pPrev = pCur;
1119	pCur = (PMMHYPERCHUNKFREE)((char *)pCur + pCur->core.offNext);
1120	}
1121	}
1122	#endif
1123
1124
1125	/**
1126	* Performs consistency checks on the heap if MMHYPER_HEAP_STRICT was
1127	* defined at build time.
1128	*
1129	* @param pVM Pointer to the shared VM structure.
1130	*/
1131	MMDECL(void) MMHyperHeapCheck(PVM pVM)
1132	{
1133	#ifdef MMHYPER_HEAP_STRICT
1134	mmHyperHeapCheck(CTXSUFF(pVM->mm.s.pHyperHeap));
1135	#endif
1136	}
1137
1138
1139	#ifdef DEBUG
1140	/**
1141	* Dumps the hypervisor heap to Log.
1142	* @param pVM VM Handle.
1143	*/
1144	MMDECL(void) MMHyperHeapDump(PVM pVM)
1145	{
1146	Log(("MMHyperHeapDump: * heap dump - start *\n"));
1147	PMMHYPERHEAP pHeap = CTXSUFF(pVM->mm.s.pHyperHeap);
1148	PMMHYPERCHUNKFREE pCur = (PMMHYPERCHUNKFREE)CTXSUFF(pHeap->pbHeap);
1149	for (;;)
1150	{
1151	mmHyperHeapDumpOne(pHeap, pCur);
1152
1153	/* next */
1154	if (!pCur->core.offNext)
1155	break;
1156	pCur = (PMMHYPERCHUNKFREE)((char *)pCur + pCur->core.offNext);
1157	}
1158	Log(("MMHyperHeapDump: * heap dump - end *\n"));
1159	}
1160	#endif
1161
1162
1163	/**
1164	* Query the amount of free memory in the hypervisor heap.
1165	*
1166	* @returns Number of free bytes in the hypervisor heap.
1167	*/
1168	MMDECL(size_t) MMHyperHeapGetFreeSize(PVM pVM)
1169	{
1170	return CTXSUFF(pVM->mm.s.pHyperHeap)->cbFree;
1171	}
1172
1173	/**
1174	* Query the size the hypervisor heap.
1175	*
1176	* @returns The size of the hypervisor heap in bytes.
1177	*/
1178	MMDECL(size_t) MMHyperHeapGetSize(PVM pVM)
1179	{
1180	return CTXSUFF(pVM->mm.s.pHyperHeap)->cbHeap;
1181	}
1182
1183
1184	/**
1185	* Query the address and size the hypervisor memory area.
1186	*
1187	* @returns Base address of the hypervisor area.
1188	* @param pVM VM Handle.
1189	* @param pcb Where to store the size of the hypervisor area. (out)
1190	*/
1191	MMDECL(RTGCPTR) MMHyperGetArea(PVM pVM, size_t *pcb)
1192	{
1193	if (pcb)
1194	*pcb = pVM->mm.s.cbHyperArea;
1195	return pVM->mm.s.pvHyperAreaGC;
1196	}
1197
1198
1199	/**
1200	* Checks if an address is within the hypervisor memory area.
1201	*
1202	* @returns true if inside.
1203	* @returns false if outside.
1204	* @param pVM VM handle.
1205	* @param GCPtr The pointer to check.
1206	*/
1207	MMDECL(bool) MMHyperIsInsideArea(PVM pVM, RTGCPTR GCPtr)
1208	{
1209	return (RTGCUINTPTR)GCPtr - (RTGCUINTPTR)pVM->mm.s.pvHyperAreaGC < pVM->mm.s.cbHyperArea;
1210	}
1211

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/MMAllHyper.cpp@ 11311

Download in other formats: