MMAllHyper.cpp@ 26510

Last change on this file since 26510 was 26510, checked in by vboxsync, 15 years ago
VMM: tabs -> spaces.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 47.2 KB

Line
1	/* $Id: MMAllHyper.cpp 26510 2010-02-14 09:37:18Z vboxsync $ */
2	/** @file
3	* MM - Memory Manager - Hypervisor Memory Area, All Contexts.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22
23	/*******************************************************************************
24	* Header Files *
25	*******************************************************************************/
26	#define LOG_GROUP LOG_GROUP_MM_HYPER_HEAP
27	#include <VBox/mm.h>
28	#include <VBox/stam.h>
29	#include "MMInternal.h"
30	#include <VBox/vm.h>
31
32	#include <VBox/err.h>
33	#include <VBox/param.h>
34	#include <iprt/assert.h>
35	#include <VBox/log.h>
36	#include <iprt/asm.h>
37	#include <iprt/string.h>
38
39
40	/*******************************************************************************
41	* Defined Constants And Macros *
42	*******************************************************************************/
43	#define ASSERT_L(u1, u2) AssertMsg((u1) < (u2), ("u1=%#x u2=%#x\n", u1, u2))
44	#define ASSERT_LE(u1, u2) AssertMsg((u1) <= (u2), ("u1=%#x u2=%#x\n", u1, u2))
45	#define ASSERT_GE(u1, u2) AssertMsg((u1) >= (u2), ("u1=%#x u2=%#x\n", u1, u2))
46	#define ASSERT_ALIGN(u1) AssertMsg(!((u1) & (MMHYPER_HEAP_ALIGN_MIN - 1)), ("u1=%#x (%d)\n", u1, u1))
47
48	#define ASSERT_OFFPREV(pHeap, pChunk) \
49	do { Assert(MMHYPERCHUNK_GET_OFFPREV(pChunk) <= 0); \
50	Assert(MMHYPERCHUNK_GET_OFFPREV(pChunk) >= (intptr_t)(pHeap)->CTX_SUFF(pbHeap) - (intptr_t)(pChunk)); \
51	AssertMsg( MMHYPERCHUNK_GET_OFFPREV(pChunk) != 0 \
52	\|\| (uint8_t *)(pChunk) == (pHeap)->CTX_SUFF(pbHeap), \
53	("pChunk=%p pvHyperHeap=%p\n", (pChunk), (pHeap)->CTX_SUFF(pbHeap))); \
54	} while (0)
55
56	#define ASSERT_OFFNEXT(pHeap, pChunk) \
57	do { ASSERT_ALIGN((pChunk)->offNext); \
58	ASSERT_L((pChunk)->offNext, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
59	} while (0)
60
61	#define ASSERT_OFFHEAP(pHeap, pChunk) \
62	do { Assert((pChunk)->offHeap); \
63	AssertMsg((PMMHYPERHEAP)((pChunk)->offHeap + (uintptr_t)pChunk) == (pHeap), \
64	("offHeap=%RX32 pChunk=%p pHeap=%p\n", (pChunk)->offHeap, (pChunk), (pHeap))); \
65	Assert((pHeap)->u32Magic == MMHYPERHEAP_MAGIC); \
66	} while (0)
67
68	#ifdef VBOX_WITH_STATISTICS
69	#define ASSERT_OFFSTAT(pHeap, pChunk) \
70	do { if (MMHYPERCHUNK_ISFREE(pChunk)) \
71	Assert(!(pChunk)->offStat); \
72	else if ((pChunk)->offStat) \
73	{ \
74	Assert((pChunk)->offStat); \
75	AssertMsg(!((pChunk)->offStat & (MMHYPER_HEAP_ALIGN_MIN - 1)), ("offStat=%RX32\n", (pChunk)->offStat)); \
76	uintptr_t uPtr = (uintptr_t)(pChunk)->offStat + (uintptr_t)pChunk; NOREF(uPtr); \
77	AssertMsg(uPtr - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) < (pHeap)->offPageAligned, \
78	("%p - %p < %RX32\n", uPtr, (pHeap)->CTX_SUFF(pbHeap), (pHeap)->offPageAligned)); \
79	} \
80	} while (0)
81	#else
82	#define ASSERT_OFFSTAT(pHeap, pChunk) \
83	do { Assert(!(pChunk)->offStat); \
84	} while (0)
85	#endif
86
87	#define ASSERT_CHUNK(pHeap, pChunk) \
88	do { ASSERT_OFFNEXT(pHeap, pChunk); \
89	ASSERT_OFFPREV(pHeap, pChunk); \
90	ASSERT_OFFHEAP(pHeap, pChunk); \
91	ASSERT_OFFSTAT(pHeap, pChunk); \
92	} while (0)
93	#define ASSERT_CHUNK_USED(pHeap, pChunk) \
94	do { ASSERT_OFFNEXT(pHeap, pChunk); \
95	ASSERT_OFFPREV(pHeap, pChunk); \
96	Assert(MMHYPERCHUNK_ISUSED(pChunk)); \
97	} while (0)
98
99	#define ASSERT_FREE_OFFPREV(pHeap, pChunk) \
100	do { ASSERT_ALIGN((pChunk)->offPrev); \
101	ASSERT_GE(((pChunk)->offPrev & (MMHYPER_HEAP_ALIGN_MIN - 1)), (intptr_t)(pHeap)->CTX_SUFF(pbHeap) - (intptr_t)(pChunk)); \
102	Assert((pChunk)->offPrev != MMHYPERCHUNK_GET_OFFPREV(&(pChunk)->core) \|\| !(pChunk)->offPrev); \
103	AssertMsg( (pChunk)->offPrev \
104	\|\| (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) == (pHeap)->offFreeHead, \
105	("pChunk=%p offChunk=%#x offFreeHead=%#x\n", (pChunk), (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap),\
106	(pHeap)->offFreeHead)); \
107	} while (0)
108
109	#define ASSERT_FREE_OFFNEXT(pHeap, pChunk) \
110	do { ASSERT_ALIGN((pChunk)->offNext); \
111	ASSERT_L((pChunk)->offNext, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
112	Assert((pChunk)->offNext != (pChunk)->core.offNext \|\| !(pChunk)->offNext); \
113	AssertMsg( (pChunk)->offNext \
114	\|\| (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) == (pHeap)->offFreeTail, \
115	("pChunk=%p offChunk=%#x offFreeTail=%#x\n", (pChunk), (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap), \
116	(pHeap)->offFreeTail)); \
117	} while (0)
118
119	#define ASSERT_FREE_CB(pHeap, pChunk) \
120	do { ASSERT_ALIGN((pChunk)->cb); \
121	Assert((pChunk)->cb > 0); \
122	if ((pChunk)->core.offNext) \
123	AssertMsg((pChunk)->cb == ((pChunk)->core.offNext - sizeof(MMHYPERCHUNK)), \
124	("cb=%d offNext=%d\n", (pChunk)->cb, (pChunk)->core.offNext)); \
125	else \
126	ASSERT_LE((pChunk)->cb, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
127	} while (0)
128
129	#define ASSERT_CHUNK_FREE(pHeap, pChunk) \
130	do { ASSERT_CHUNK(pHeap, &(pChunk)->core); \
131	Assert(MMHYPERCHUNK_ISFREE(pChunk)); \
132	ASSERT_FREE_OFFNEXT(pHeap, pChunk); \
133	ASSERT_FREE_OFFPREV(pHeap, pChunk); \
134	ASSERT_FREE_CB(pHeap, pChunk); \
135	} while (0)
136
137
138	/*******************************************************************************
139	* Internal Functions *
140	*******************************************************************************/
141	static PMMHYPERCHUNK mmHyperAllocChunk(PMMHYPERHEAP pHeap, uint32_t cb, unsigned uAlignment);
142	static void *mmHyperAllocPages(PMMHYPERHEAP pHeap, uint32_t cb);
143	#ifdef VBOX_WITH_STATISTICS
144	static PMMHYPERSTAT mmHyperStat(PMMHYPERHEAP pHeap, MMTAG enmTag);
145	#ifdef IN_RING3
146	static void mmR3HyperStatRegisterOne(PVM pVM, PMMHYPERSTAT pStat);
147	#endif
148	#endif
149	static int mmHyperFree(PMMHYPERHEAP pHeap, PMMHYPERCHUNK pChunk);
150	#ifdef MMHYPER_HEAP_STRICT
151	static void mmHyperHeapCheck(PMMHYPERHEAP pHeap);
152	#endif
153
154
155
156	/**
157	* Locks the hypervisor heap.
158	* This might call back to Ring-3 in order to deal with lock contention in GC and R3.
159	*
160	* @param pVM The VM handle.
161	*/
162	static int mmHyperLock(PVM pVM)
163	{
164	PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
165
166	#ifdef IN_RING3
167	if (!PDMCritSectIsInitialized(&pHeap->Lock))
168	return VINF_SUCCESS; /* early init */
169	#else
170	Assert(PDMCritSectIsInitialized(&pHeap->Lock));
171	#endif
172	int rc = PDMCritSectEnter(&pHeap->Lock, VERR_SEM_BUSY);
173	#if defined(IN_RC) \|\| defined(IN_RING0)
174	if (rc == VERR_SEM_BUSY)
175	rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_MMHYPER_LOCK, 0);
176	#endif
177	AssertRC(rc);
178	return rc;
179	}
180
181
182	/**
183	* Unlocks the hypervisor heap.
184	*
185	* @param pVM The VM handle.
186	*/
187	static void mmHyperUnlock(PVM pVM)
188	{
189	PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
190
191	#ifdef IN_RING3
192	if (!PDMCritSectIsInitialized(&pHeap->Lock))
193	return; /* early init */
194	#endif
195	Assert(PDMCritSectIsInitialized(&pHeap->Lock));
196	PDMCritSectLeave(&pHeap->Lock);
197	}
198
199	/**
200	* Allocates memory in the Hypervisor (RC VMM) area.
201	* The returned memory is of course zeroed.
202	*
203	* @returns VBox status code.
204	* @param pVM The VM to operate on.
205	* @param cb Number of bytes to allocate.
206	* @param uAlignment Required memory alignment in bytes.
207	* Values are 0,8,16,32,64 and PAGE_SIZE.
208	* 0 -> default alignment, i.e. 8 bytes.
209	* @param enmTag The statistics tag.
210	* @param ppv Where to store the address to the allocated
211	* memory.
212	*/
213	static int mmHyperAllocInternal(PVM pVM, size_t cb, unsigned uAlignment, MMTAG enmTag, void **ppv)
214	{
215	AssertMsg(cb >= 8, ("Hey! Do you really mean to allocate less than 8 bytes?! cb=%d\n", cb));
216
217	/*
218	* Validate input and adjust it to reasonable values.
219	*/
220	if (!uAlignment \|\| uAlignment < MMHYPER_HEAP_ALIGN_MIN)
221	uAlignment = MMHYPER_HEAP_ALIGN_MIN;
222	uint32_t cbAligned;
223	switch (uAlignment)
224	{
225	case 8:
226	case 16:
227	case 32:
228	case 64:
229	cbAligned = RT_ALIGN_32(cb, MMHYPER_HEAP_ALIGN_MIN);
230	if (!cbAligned \|\| cbAligned < cb)
231	{
232	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
233	AssertMsgFailed(("Nice try.\n"));
234	return VERR_INVALID_PARAMETER;
235	}
236	break;
237
238	case PAGE_SIZE:
239	AssertMsg(RT_ALIGN_32(cb, PAGE_SIZE) == cb, ("The size isn't page aligned. (cb=%#x)\n", cb));
240	cbAligned = RT_ALIGN_32(cb, PAGE_SIZE);
241	if (!cbAligned)
242	{
243	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
244	AssertMsgFailed(("Nice try.\n"));
245	return VERR_INVALID_PARAMETER;
246	}
247	break;
248
249	default:
250	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
251	AssertMsgFailed(("Invalid alignment %u\n", uAlignment));
252	return VERR_INVALID_PARAMETER;
253	}
254
255
256	/*
257	* Get heap and statisticsStatistics.
258	*/
259	PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
260	#ifdef VBOX_WITH_STATISTICS
261	PMMHYPERSTAT pStat = mmHyperStat(pHeap, enmTag);
262	if (!pStat)
263	{
264	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_MM_HYPER_NO_MEMORY\n", cb, uAlignment));
265	AssertMsgFailed(("Failed to allocate statistics!\n"));
266	return VERR_MM_HYPER_NO_MEMORY;
267	}
268	#endif
269	if (uAlignment < PAGE_SIZE)
270	{
271	/*
272	* Allocate a chunk.
273	*/
274	PMMHYPERCHUNK pChunk = mmHyperAllocChunk(pHeap, cbAligned, uAlignment);
275	if (pChunk)
276	{
277	#ifdef VBOX_WITH_STATISTICS
278	const uint32_t cbChunk = pChunk->offNext
279	? pChunk->offNext
280	: pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t *)pChunk;
281	pStat->cbAllocated += (uint32_t)cbChunk;
282	pStat->cbCurAllocated += (uint32_t)cbChunk;
283	if (pStat->cbCurAllocated > pStat->cbMaxAllocated)
284	pStat->cbMaxAllocated = pStat->cbCurAllocated;
285	pStat->cAllocations++;
286	pChunk->offStat = (uintptr_t)pStat - (uintptr_t)pChunk;
287	#else
288	pChunk->offStat = 0;
289	#endif
290	void *pv = pChunk + 1;
291	*ppv = pv;
292	ASMMemZero32(pv, cbAligned);
293	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VINF_SUCCESS and *ppv=%p\n", cb, uAlignment, pv));
294	return VINF_SUCCESS;
295	}
296	}
297	else
298	{
299	/*
300	* Allocate page aligned memory.
301	*/
302	void *pv = mmHyperAllocPages(pHeap, cbAligned);
303	if (pv)
304	{
305	#ifdef VBOX_WITH_STATISTICS
306	pStat->cbAllocated += cbAligned;
307	pStat->cbCurAllocated += cbAligned;
308	if (pStat->cbCurAllocated > pStat->cbMaxAllocated)
309	pStat->cbMaxAllocated = pStat->cbCurAllocated;
310	pStat->cAllocations++;
311	#endif
312	*ppv = pv;
313	/* ASMMemZero32(pv, cbAligned); - not required since memory is alloc-only and SUPR3PageAlloc zeros it. */
314	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VINF_SUCCESS and *ppv=%p\n", cb, uAlignment, ppv));
315	return VINF_SUCCESS;
316	}
317	}
318
319	#ifdef VBOX_WITH_STATISTICS
320	pStat->cAllocations++;
321	pStat->cFailures++;
322	#endif
323	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_MM_HYPER_NO_MEMORY\n", cb, uAlignment));
324	AssertMsgFailed(("Failed to allocate %d bytes!\n", cb));
325	return VERR_MM_HYPER_NO_MEMORY;
326	}
327
328	/**
329	* Wrapper for mmHyperAllocInternal
330	*/
331	VMMDECL(int) MMHyperAlloc(PVM pVM, size_t cb, unsigned uAlignment, MMTAG enmTag, void **ppv)
332	{
333	int rc;
334
335	rc = mmHyperLock(pVM);
336	AssertRCReturn(rc, rc);
337
338	LogFlow(("MMHyperAlloc %x align=%x tag=%s\n", cb, uAlignment, mmGetTagName(enmTag)));
339
340	rc = mmHyperAllocInternal(pVM, cb, uAlignment, enmTag, ppv);
341
342	mmHyperUnlock(pVM);
343	return rc;
344	}
345
346	/**
347	* Allocates a chunk of memory from the specified heap.
348	* The caller validates the parameters of this request.
349	*
350	* @returns Pointer to the allocated chunk.
351	* @returns NULL on failure.
352	* @param pHeap The heap.
353	* @param cb Size of the memory block to allocate.
354	* @param uAlignment The alignment specifications for the allocated block.
355	* @internal
356	*/
357	static PMMHYPERCHUNK mmHyperAllocChunk(PMMHYPERHEAP pHeap, uint32_t cb, unsigned uAlignment)
358	{
359	Log3(("mmHyperAllocChunk: Enter cb=%#x uAlignment=%#x\n", cb, uAlignment));
360	#ifdef MMHYPER_HEAP_STRICT
361	mmHyperHeapCheck(pHeap);
362	#endif
363	#ifdef MMHYPER_HEAP_STRICT_FENCE
364	uint32_t cbFence = RT_MAX(MMHYPER_HEAP_STRICT_FENCE_SIZE, uAlignment);
365	cb += cbFence;
366	#endif
367
368	/*
369	* Check if there are any free chunks. (NIL_OFFSET use/not-use forces this check)
370	*/
371	if (pHeap->offFreeHead == NIL_OFFSET)
372	return NULL;
373
374	/*
375	* Small alignments - from the front of the heap.
376	*
377	* Must split off free chunks at the end to prevent messing up the
378	* last free node which we take the page aligned memory from the top of.
379	*/
380	PMMHYPERCHUNK pRet = NULL;
381	PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)((char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeHead);
382	while (pFree)
383	{
384	ASSERT_CHUNK_FREE(pHeap, pFree);
385	if (pFree->cb >= cb)
386	{
387	unsigned offAlign = (uintptr_t)(&pFree->core + 1) & (uAlignment - 1);
388	if (offAlign)
389	offAlign = uAlignment - offAlign;
390	if (!offAlign \|\| pFree->cb - offAlign >= cb)
391	{
392	Log3(("mmHyperAllocChunk: Using pFree=%p pFree->cb=%d offAlign=%d\n", pFree, pFree->cb, offAlign));
393
394	/*
395	* Adjust the node in front.
396	* Because of multiple alignments we need to special case allocation of the first block.
397	*/
398	if (offAlign)
399	{
400	MMHYPERCHUNKFREE Free = *pFree;
401	if (MMHYPERCHUNK_GET_OFFPREV(&pFree->core))
402	{
403	/* just add a bit of memory to it. */
404	PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + MMHYPERCHUNK_GET_OFFPREV(&Free.core));
405	pPrev->core.offNext += offAlign;
406	AssertMsg(!MMHYPERCHUNK_ISFREE(&pPrev->core), ("Impossible!\n"));
407	Log3(("mmHyperAllocChunk: Added %d bytes to %p\n", offAlign, pPrev));
408	}
409	else
410	{
411	/* make new head node, mark it USED for simplisity. */
412	PMMHYPERCHUNK pPrev = (PMMHYPERCHUNK)pHeap->CTX_SUFF(pbHeap);
413	Assert(pPrev == &pFree->core);
414	pPrev->offPrev = 0;
415	MMHYPERCHUNK_SET_TYPE(pPrev, MMHYPERCHUNK_FLAGS_USED);
416	pPrev->offNext = offAlign;
417	Log3(("mmHyperAllocChunk: Created new first node of %d bytes\n", offAlign));
418
419	}
420	Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - offAlign, -(int)offAlign));
421	pHeap->cbFree -= offAlign;
422
423	/* Recreate pFree node and adjusting everything... */
424	pFree = (PMMHYPERCHUNKFREE)((char *)pFree + offAlign);
425	*pFree = Free;
426
427	pFree->cb -= offAlign;
428	if (pFree->core.offNext)
429	{
430	pFree->core.offNext -= offAlign;
431	PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pFree + pFree->core.offNext);
432	MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pFree->core.offNext);
433	ASSERT_CHUNK(pHeap, pNext);
434	}
435	if (MMHYPERCHUNK_GET_OFFPREV(&pFree->core))
436	MMHYPERCHUNK_SET_OFFPREV(&pFree->core, MMHYPERCHUNK_GET_OFFPREV(&pFree->core) - offAlign);
437
438	if (pFree->offNext)
439	{
440	pFree->offNext -= offAlign;
441	PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext);
442	pNext->offPrev = -(int32_t)pFree->offNext;
443	ASSERT_CHUNK_FREE(pHeap, pNext);
444	}
445	else
446	pHeap->offFreeTail += offAlign;
447	if (pFree->offPrev)
448	{
449	pFree->offPrev -= offAlign;
450	PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
451	pPrev->offNext = -pFree->offPrev;
452	ASSERT_CHUNK_FREE(pHeap, pPrev);
453	}
454	else
455	pHeap->offFreeHead += offAlign;
456	pFree->core.offHeap = (uintptr_t)pHeap - (uintptr_t)pFree;
457	pFree->core.offStat = 0;
458	ASSERT_CHUNK_FREE(pHeap, pFree);
459	Log3(("mmHyperAllocChunk: Realigned pFree=%p\n", pFree));
460	}
461
462	/*
463	* Split off a new FREE chunk?
464	*/
465	if (pFree->cb >= cb + RT_ALIGN(sizeof(MMHYPERCHUNKFREE), MMHYPER_HEAP_ALIGN_MIN))
466	{
467	/*
468	* Move the FREE chunk up to make room for the new USED chunk.
469	*/
470	const int off = cb + sizeof(MMHYPERCHUNK);
471	PMMHYPERCHUNKFREE pNew = (PMMHYPERCHUNKFREE)((char *)&pFree->core + off);
472	pNew = pFree;
473	pNew->cb -= off;
474	if (pNew->core.offNext)
475	{
476	pNew->core.offNext -= off;
477	PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pNew + pNew->core.offNext);
478	MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pNew->core.offNext);
479	ASSERT_CHUNK(pHeap, pNext);
480	}
481	pNew->core.offPrev = -off;
482	MMHYPERCHUNK_SET_TYPE(pNew, MMHYPERCHUNK_FLAGS_FREE);
483
484	if (pNew->offNext)
485	{
486	pNew->offNext -= off;
487	PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pNew + pNew->offNext);
488	pNext->offPrev = -(int32_t)pNew->offNext;
489	ASSERT_CHUNK_FREE(pHeap, pNext);
490	}
491	else
492	pHeap->offFreeTail += off;
493	if (pNew->offPrev)
494	{
495	pNew->offPrev -= off;
496	PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pNew + pNew->offPrev);
497	pPrev->offNext = -pNew->offPrev;
498	ASSERT_CHUNK_FREE(pHeap, pPrev);
499	}
500	else
501	pHeap->offFreeHead += off;
502	pNew->core.offHeap = (uintptr_t)pHeap - (uintptr_t)pNew;
503	pNew->core.offStat = 0;
504	ASSERT_CHUNK_FREE(pHeap, pNew);
505
506	/*
507	* Update the old FREE node making it a USED node.
508	*/
509	pFree->core.offNext = off;
510	MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_USED);
511
512
513	Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree,
514	pHeap->cbFree - (cb + sizeof(MMHYPERCHUNK)), -(int)(cb + sizeof(MMHYPERCHUNK))));
515	pHeap->cbFree -= (uint32_t)(cb + sizeof(MMHYPERCHUNK));
516	pRet = &pFree->core;
517	ASSERT_CHUNK(pHeap, &pFree->core);
518	Log3(("mmHyperAllocChunk: Created free chunk pNew=%p cb=%d\n", pNew, pNew->cb));
519	}
520	else
521	{
522	/*
523	* Link out of free list.
524	*/
525	if (pFree->offNext)
526	{
527	PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext);
528	if (pFree->offPrev)
529	{
530	pNext->offPrev += pFree->offPrev;
531	PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
532	pPrev->offNext += pFree->offNext;
533	ASSERT_CHUNK_FREE(pHeap, pPrev);
534	}
535	else
536	{
537	pHeap->offFreeHead += pFree->offNext;
538	pNext->offPrev = 0;
539	}
540	ASSERT_CHUNK_FREE(pHeap, pNext);
541	}
542	else
543	{
544	if (pFree->offPrev)
545	{
546	pHeap->offFreeTail += pFree->offPrev;
547	PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
548	pPrev->offNext = 0;
549	ASSERT_CHUNK_FREE(pHeap, pPrev);
550	}
551	else
552	{
553	pHeap->offFreeHead = NIL_OFFSET;
554	pHeap->offFreeTail = NIL_OFFSET;
555	}
556	}
557
558	Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree,
559	pHeap->cbFree - pFree->cb, -(int32_t)pFree->cb));
560	pHeap->cbFree -= pFree->cb;
561	MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_USED);
562	pRet = &pFree->core;
563	ASSERT_CHUNK(pHeap, &pFree->core);
564	Log3(("mmHyperAllocChunk: Converted free chunk %p to used chunk.\n", pFree));
565	}
566	Log3(("mmHyperAllocChunk: Returning %p\n", pRet));
567	break;
568	}
569	}
570
571	/* next */
572	pFree = pFree->offNext ? (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext) : NULL;
573	}
574
575	#ifdef MMHYPER_HEAP_STRICT_FENCE
576	uint32_t pu32End = (uint32_t )((uint8_t *)(pRet + 1) + cb);
577	uint32_t *pu32EndReal = pRet->offNext
578	? (uint32_t )((uint8_t )pRet + pRet->offNext)
579	: (uint32_t *)(pHeap->CTX_SUFF(pbHeap) + pHeap->cbHeap);
580	cbFence += (uintptr_t)pu32EndReal - (uintptr_t)pu32End; Assert(!(cbFence & 0x3));
581	ASMMemFill32((uint8_t *)pu32EndReal - cbFence, cbFence, MMHYPER_HEAP_STRICT_FENCE_U32);
582	pu32EndReal[-1] = cbFence;
583	#endif
584	#ifdef MMHYPER_HEAP_STRICT
585	mmHyperHeapCheck(pHeap);
586	#endif
587	return pRet;
588	}
589
590
591	/**
592	* Allocates one or more pages of memory from the specified heap.
593	* The caller validates the parameters of this request.
594	*
595	* @returns Pointer to the allocated chunk.
596	* @returns NULL on failure.
597	* @param pHeap The heap.
598	* @param cb Size of the memory block to allocate.
599	* @internal
600	*/
601	static void *mmHyperAllocPages(PMMHYPERHEAP pHeap, uint32_t cb)
602	{
603	Log3(("mmHyperAllocPages: Enter cb=%#x\n", cb));
604
605	#ifdef MMHYPER_HEAP_STRICT
606	mmHyperHeapCheck(pHeap);
607	#endif
608
609	/*
610	* Check if there are any free chunks. (NIL_OFFSET use/not-use forces this check)
611	*/
612	if (pHeap->offFreeHead == NIL_OFFSET)
613	return NULL;
614
615	/*
616	* Page aligned chunks.
617	*
618	* Page aligned chunks can only be allocated from the last FREE chunk.
619	* This is for reasons of simplicity and fragmentation. Page aligned memory
620	* must also be allocated in page aligned sizes. Page aligned memory cannot
621	* be freed either.
622	*
623	* So, for this to work, the last FREE chunk needs to end on a page aligned
624	* boundrary.
625	*/
626	PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)((char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeTail);
627	ASSERT_CHUNK_FREE(pHeap, pFree);
628	if ( (((uintptr_t)(&pFree->core + 1) + pFree->cb) & (PAGE_OFFSET_MASK - 1))
629	\|\| pFree->cb + sizeof(MMHYPERCHUNK) < cb)
630	{
631	Log3(("mmHyperAllocPages: Not enough/no page aligned memory!\n"));
632	return NULL;
633	}
634
635	void *pvRet;
636	if (pFree->cb > cb)
637	{
638	/*
639	* Simple, just cut the top of the free node and return it.
640	*/
641	pFree->cb -= cb;
642	pvRet = (char *)(&pFree->core + 1) + pFree->cb;
643	AssertMsg(RT_ALIGN_P(pvRet, PAGE_SIZE) == pvRet, ("pvRet=%p cb=%#x pFree=%p pFree->cb=%#x\n", pvRet, cb, pFree, pFree->cb));
644	Log3(("mmHyperAllocPages: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - cb, -(int)cb));
645	pHeap->cbFree -= cb;
646	ASSERT_CHUNK_FREE(pHeap, pFree);
647	Log3(("mmHyperAllocPages: Allocated from pFree=%p new pFree->cb=%d\n", pFree, pFree->cb));
648	}
649	else
650	{
651	/*
652	* Unlink the FREE node.
653	*/
654	pvRet = (char *)(&pFree->core + 1) + pFree->cb - cb;
655	Log3(("mmHyperAllocPages: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pFree->cb, -(int32_t)pFree->cb));
656	pHeap->cbFree -= pFree->cb;
657
658	/* a scrap of spare memory (unlikely)? add it to the sprevious chunk. */
659	if (pvRet != (void *)pFree)
660	{
661	AssertMsg(MMHYPERCHUNK_GET_OFFPREV(&pFree->core), ("How the beep did someone manage to allocated up all the heap with page aligned memory?!?\n"));
662	PMMHYPERCHUNK pPrev = (PMMHYPERCHUNK)((char *)pFree + MMHYPERCHUNK_GET_OFFPREV(&pFree->core));
663	pPrev->offNext += (uintptr_t)pvRet - (uintptr_t)pFree;
664	AssertMsg(!MMHYPERCHUNK_ISFREE(pPrev), ("Free bug?\n"));
665	#ifdef VBOX_WITH_STATISTICS
666	PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pPrev + pPrev->offStat);
667	pStat->cbAllocated += (uintptr_t)pvRet - (uintptr_t)pFree;
668	pStat->cbCurAllocated += (uintptr_t)pvRet - (uintptr_t)pFree;
669	#endif
670	Log3(("mmHyperAllocPages: Added %d to %p (page align)\n", (uintptr_t)pvRet - (uintptr_t)pFree, pFree));
671	}
672
673	/* unlink from FREE chain. */
674	if (pFree->offPrev)
675	{
676	pHeap->offFreeTail += pFree->offPrev;
677	((PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev))->offNext = 0;
678	}
679	else
680	{
681	pHeap->offFreeTail = NIL_OFFSET;
682	pHeap->offFreeHead = NIL_OFFSET;
683	}
684	Log3(("mmHyperAllocPages: Unlinked pFree=%d\n", pFree));
685	}
686	pHeap->offPageAligned = (uintptr_t)pvRet - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
687	Log3(("mmHyperAllocPages: Returning %p (page aligned)\n", pvRet));
688
689	#ifdef MMHYPER_HEAP_STRICT
690	mmHyperHeapCheck(pHeap);
691	#endif
692	return pvRet;
693	}
694
695	#ifdef VBOX_WITH_STATISTICS
696
697	/**
698	* Get the statistic record for a tag.
699	*
700	* @returns Pointer to a stat record.
701	* @returns NULL on failure.
702	* @param pHeap The heap.
703	* @param enmTag The tag.
704	*/
705	static PMMHYPERSTAT mmHyperStat(PMMHYPERHEAP pHeap, MMTAG enmTag)
706	{
707	/* try look it up first. */
708	PMMHYPERSTAT pStat = (PMMHYPERSTAT)RTAvloGCPhysGet(&pHeap->HyperHeapStatTree, enmTag);
709	if (!pStat)
710	{
711	/* try allocate a new one */
712	PMMHYPERCHUNK pChunk = mmHyperAllocChunk(pHeap, RT_ALIGN(sizeof(*pStat), MMHYPER_HEAP_ALIGN_MIN), MMHYPER_HEAP_ALIGN_MIN);
713	if (!pChunk)
714	return NULL;
715	pStat = (PMMHYPERSTAT)(pChunk + 1);
716	pChunk->offStat = (uintptr_t)pStat - (uintptr_t)pChunk;
717
718	ASMMemZero32(pStat, sizeof(*pStat));
719	pStat->Core.Key = enmTag;
720	RTAvloGCPhysInsert(&pHeap->HyperHeapStatTree, &pStat->Core);
721	}
722	if (!pStat->fRegistered)
723	{
724	# ifdef IN_RING3
725	mmR3HyperStatRegisterOne(pHeap->pVMR3, pStat);
726	# else
727	/** @todo schedule a R3 action. */
728	# endif
729	}
730	return pStat;
731	}
732
733
734	# ifdef IN_RING3
735	/**
736	* Registers statistics with STAM.
737	*
738	*/
739	static void mmR3HyperStatRegisterOne(PVM pVM, PMMHYPERSTAT pStat)
740	{
741	if (pStat->fRegistered)
742	return;
743	const char *pszTag = mmGetTagName((MMTAG)pStat->Core.Key);
744	STAMR3RegisterF(pVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Number of bytes currently allocated.", "/MM/HyperHeap/%s", pszTag);
745	STAMR3RegisterF(pVM, &pStat->cAllocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of alloc calls.", "/MM/HyperHeap/%s/cAllocations", pszTag);
746	STAMR3RegisterF(pVM, &pStat->cFrees, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of free calls.", "/MM/HyperHeap/%s/cFrees", pszTag);
747	STAMR3RegisterF(pVM, &pStat->cFailures, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of failures.", "/MM/HyperHeap/%s/cFailures", pszTag);
748	STAMR3RegisterF(pVM, &pStat->cbAllocated, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of allocated bytes.", "/MM/HyperHeap/%s/cbAllocated", pszTag);
749	STAMR3RegisterF(pVM, &pStat->cbFreed, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of freed bytes.", "/MM/HyperHeap/%s/cbFreed", pszTag);
750	STAMR3RegisterF(pVM, &pStat->cbMaxAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Max number of bytes allocated at the same time.","/MM/HyperHeap/%s/cbMaxAllocated", pszTag);
751	pStat->fRegistered = true;
752	}
753	# endif /* IN_RING3 */
754
755	#endif /* VBOX_WITH_STATISTICS */
756
757
758	/**
759	* Free memory allocated using MMHyperAlloc().
760	* The caller validates the parameters of this request.
761	*
762	* @returns VBox status code.
763	* @param pVM The VM to operate on.
764	* @param pv The memory to free.
765	* @remark Try avoid free hyper memory.
766	*/
767	static int mmHyperFreeInternal(PVM pVM, void *pv)
768	{
769	Log2(("MMHyperFree: pv=%p\n", pv));
770	if (!pv)
771	return VINF_SUCCESS;
772	AssertMsgReturn(RT_ALIGN_P(pv, MMHYPER_HEAP_ALIGN_MIN) == pv,
773	("Invalid pointer %p!\n", pv),
774	VERR_INVALID_POINTER);
775
776	/*
777	* Get the heap and stats.
778	* Validate the chunk at the same time.
779	*/
780	PMMHYPERCHUNK pChunk = (PMMHYPERCHUNK)((PMMHYPERCHUNK)pv - 1);
781
782	AssertMsgReturn( (uintptr_t)pChunk + pChunk->offNext >= (uintptr_t)pChunk
783	\|\| RT_ALIGN_32(pChunk->offNext, MMHYPER_HEAP_ALIGN_MIN) != pChunk->offNext,
784	("%p: offNext=%#RX32\n", pv, pChunk->offNext),
785	VERR_INVALID_POINTER);
786
787	AssertMsgReturn(MMHYPERCHUNK_ISUSED(pChunk),
788	("%p: Not used!\n", pv),
789	VERR_INVALID_POINTER);
790
791	int32_t offPrev = MMHYPERCHUNK_GET_OFFPREV(pChunk);
792	AssertMsgReturn( (uintptr_t)pChunk + offPrev <= (uintptr_t)pChunk
793	&& !((uint32_t)-offPrev & (MMHYPER_HEAP_ALIGN_MIN - 1)),
794	("%p: offPrev=%#RX32!\n", pv, offPrev),
795	VERR_INVALID_POINTER);
796
797	/* statistics */
798	#ifdef VBOX_WITH_STATISTICS
799	PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pChunk + pChunk->offStat);
800	AssertMsgReturn( RT_ALIGN_P(pStat, MMHYPER_HEAP_ALIGN_MIN) == (void *)pStat
801	&& pChunk->offStat,
802	("%p: offStat=%#RX32!\n", pv, pChunk->offStat),
803	VERR_INVALID_POINTER);
804	#else
805	AssertMsgReturn(!pChunk->offStat,
806	("%p: offStat=%#RX32!\n", pv, pChunk->offStat),
807	VERR_INVALID_POINTER);
808	#endif
809
810	/* The heap structure. */
811	PMMHYPERHEAP pHeap = (PMMHYPERHEAP)((uintptr_t)pChunk + pChunk->offHeap);
812	AssertMsgReturn( !((uintptr_t)pHeap & PAGE_OFFSET_MASK)
813	&& pChunk->offHeap,
814	("%p: pHeap=%#x offHeap=%RX32\n", pv, pHeap->u32Magic, pChunk->offHeap),
815	VERR_INVALID_POINTER);
816
817	AssertMsgReturn(pHeap->u32Magic == MMHYPERHEAP_MAGIC,
818	("%p: u32Magic=%#x\n", pv, pHeap->u32Magic),
819	VERR_INVALID_POINTER);
820	Assert(pHeap == pVM->mm.s.CTX_SUFF(pHyperHeap));
821
822	/* Some more verifications using additional info from pHeap. */
823	AssertMsgReturn((uintptr_t)pChunk + offPrev >= (uintptr_t)pHeap->CTX_SUFF(pbHeap),
824	("%p: offPrev=%#RX32!\n", pv, offPrev),
825	VERR_INVALID_POINTER);
826
827	AssertMsgReturn(pChunk->offNext < pHeap->cbHeap,
828	("%p: offNext=%#RX32!\n", pv, pChunk->offNext),
829	VERR_INVALID_POINTER);
830
831	AssertMsgReturn( (uintptr_t)pv - (uintptr_t)pHeap->CTX_SUFF(pbHeap) <= pHeap->offPageAligned,
832	("Invalid pointer %p! (heap: %p-%p)\n", pv, pHeap->CTX_SUFF(pbHeap),
833	(char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned),
834	VERR_INVALID_POINTER);
835
836	#ifdef MMHYPER_HEAP_STRICT
837	mmHyperHeapCheck(pHeap);
838	#endif
839
840	#if defined(VBOX_WITH_STATISTICS) \|\| defined(MMHYPER_HEAP_FREE_POISON)
841	/* calc block size. */
842	const uint32_t cbChunk = pChunk->offNext
843	? pChunk->offNext
844	: pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t *)pChunk;
845	#endif
846	#ifdef MMHYPER_HEAP_FREE_POISON
847	/* poison the block */
848	memset(pChunk + 1, MMHYPER_HEAP_FREE_POISON, cbChunk - sizeof(*pChunk));
849	#endif
850
851	#ifdef MMHYPER_HEAP_FREE_DELAY
852	# ifdef MMHYPER_HEAP_FREE_POISON
853	/*
854	* Check poison.
855	*/
856	unsigned i = RT_ELEMENTS(pHeap->aDelayedFrees);
857	while (i-- > 0)
858	if (pHeap->aDelayedFrees[i].offChunk)
859	{
860	PMMHYPERCHUNK pCur = (PMMHYPERCHUNK)((uintptr_t)pHeap + pHeap->aDelayedFrees[i].offChunk);
861	const size_t cb = pCur->offNext
862	? pCur->offNext - sizeof(*pCur)
863	: pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t )pCur - sizeof(pCur);
864	uint8_t pab = (uint8_t )(pCur + 1);
865	for (unsigned off = 0; off < cb; off++)
866	AssertReleaseMsg(pab[off] == 0xCB,
867	("caller=%RTptr cb=%#zx off=%#x: %.*Rhxs\n",
868	pHeap->aDelayedFrees[i].uCaller, cb, off, RT_MIN(cb - off, 32), &pab[off]));
869	}
870	# endif /* MMHYPER_HEAP_FREE_POISON */
871
872	/*
873	* Delayed freeing.
874	*/
875	int rc = VINF_SUCCESS;
876	if (pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk)
877	{
878	PMMHYPERCHUNK pChunkFree = (PMMHYPERCHUNK)((uintptr_t)pHeap + pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk);
879	rc = mmHyperFree(pHeap, pChunkFree);
880	}
881	pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk = (uintptr_t)pChunk - (uintptr_t)pHeap;
882	pHeap->aDelayedFrees[pHeap->iDelayedFree].uCaller = (uintptr_t)ASMReturnAddress();
883	pHeap->iDelayedFree = (pHeap->iDelayedFree + 1) % RT_ELEMENTS(pHeap->aDelayedFrees);
884
885	#else /* !MMHYPER_HEAP_FREE_POISON */
886	/*
887	* Call the worker.
888	*/
889	int rc = mmHyperFree(pHeap, pChunk);
890	#endif /* !MMHYPER_HEAP_FREE_POISON */
891
892	/*
893	* Update statistics.
894	*/
895	#ifdef VBOX_WITH_STATISTICS
896	pStat->cFrees++;
897	if (RT_SUCCESS(rc))
898	{
899	pStat->cbFreed += cbChunk;
900	pStat->cbCurAllocated -= cbChunk;
901	}
902	else
903	pStat->cFailures++;
904	#endif
905
906	return rc;
907	}
908
909
910	/**
911	* Wrapper for mmHyperFreeInternal
912	*/
913	VMMDECL(int) MMHyperFree(PVM pVM, void *pv)
914	{
915	int rc;
916
917	rc = mmHyperLock(pVM);
918	AssertRCReturn(rc, rc);
919
920	LogFlow(("MMHyperFree %p\n", pv));
921
922	rc = mmHyperFreeInternal(pVM, pv);
923
924	mmHyperUnlock(pVM);
925	return rc;
926	}
927
928
929	/**
930	* Free memory a memory chunk.
931	*
932	* @returns VBox status code.
933	* @param pHeap The heap.
934	* @param pChunk The memory chunk to free.
935	*/
936	static int mmHyperFree(PMMHYPERHEAP pHeap, PMMHYPERCHUNK pChunk)
937	{
938	Log3(("mmHyperFree: Enter pHeap=%p pChunk=%p\n", pHeap, pChunk));
939	PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)pChunk;
940
941	/*
942	* Insert into the free list (which is sorted on address).
943	*
944	* We'll search towards the end of the heap to locate the
945	* closest FREE chunk.
946	*/
947	PMMHYPERCHUNKFREE pLeft = NULL;
948	PMMHYPERCHUNKFREE pRight = NULL;
949	if (pHeap->offFreeTail != NIL_OFFSET)
950	{
951	if (pFree->core.offNext)
952	{
953	pRight = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->core.offNext);
954	ASSERT_CHUNK(pHeap, &pRight->core);
955	while (!MMHYPERCHUNK_ISFREE(&pRight->core))
956	{
957	if (!pRight->core.offNext)
958	{
959	pRight = NULL;
960	break;
961	}
962	pRight = (PMMHYPERCHUNKFREE)((char *)pRight + pRight->core.offNext);
963	ASSERT_CHUNK(pHeap, &pRight->core);
964	}
965	}
966	if (!pRight)
967	pRight = (PMMHYPERCHUNKFREE)((char )pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeTail); /* @todo this can't be correct! 'pLeft = .. ; else' I think */
968	if (pRight)
969	{
970	ASSERT_CHUNK_FREE(pHeap, pRight);
971	if (pRight->offPrev)
972	{
973	pLeft = (PMMHYPERCHUNKFREE)((char *)pRight + pRight->offPrev);
974	ASSERT_CHUNK_FREE(pHeap, pLeft);
975	}
976	}
977	}
978	if (pLeft == pFree)
979	{
980	AssertMsgFailed(("Freed twice! pv=%p (pChunk=%p)\n", pChunk + 1, pChunk));
981	return VERR_INVALID_POINTER;
982	}
983	pChunk->offStat = 0;
984
985	/*
986	* Head free chunk list?
987	*/
988	if (!pLeft)
989	{
990	MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_FREE);
991	pFree->offPrev = 0;
992	pHeap->offFreeHead = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
993	if (pRight)
994	{
995	pFree->offNext = (uintptr_t)pRight - (uintptr_t)pFree;
996	pRight->offPrev = -(int32_t)pFree->offNext;
997	}
998	else
999	{
1000	pFree->offNext = 0;
1001	pHeap->offFreeTail = pHeap->offFreeHead;
1002	}
1003	Log3(("mmHyperFree: Inserted %p at head of free chain.\n", pFree));
1004	}
1005	else
1006	{
1007	/*
1008	* Can we merge with left hand free chunk?
1009	*/
1010	if ((char )pLeft + pLeft->core.offNext == (char )pFree)
1011	{
1012	if (pFree->core.offNext)
1013	{
1014	pLeft->core.offNext = pLeft->core.offNext + pFree->core.offNext;
1015	MMHYPERCHUNK_SET_OFFPREV(((PMMHYPERCHUNK)((char *)pLeft + pLeft->core.offNext)), -(int32_t)pLeft->core.offNext);
1016	}
1017	else
1018	pLeft->core.offNext = 0;
1019	pFree = pLeft;
1020	Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pLeft->cb, -(int32_t)pLeft->cb));
1021	pHeap->cbFree -= pLeft->cb;
1022	Log3(("mmHyperFree: Merging %p into %p (cb=%d).\n", pFree, pLeft, pLeft->cb));
1023	}
1024	/*
1025	* No, just link it into the free list then.
1026	*/
1027	else
1028	{
1029	MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_FREE);
1030	pFree->offPrev = (uintptr_t)pLeft - (uintptr_t)pFree;
1031	pLeft->offNext = -pFree->offPrev;
1032	if (pRight)
1033	{
1034	pFree->offNext = (uintptr_t)pRight - (uintptr_t)pFree;
1035	pRight->offPrev = -(int32_t)pFree->offNext;
1036	}
1037	else
1038	{
1039	pFree->offNext = 0;
1040	pHeap->offFreeTail = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
1041	}
1042	Log3(("mmHyperFree: Inserted %p after %p in free list.\n", pFree, pLeft));
1043	}
1044	}
1045
1046	/*
1047	* Can we merge with right hand free chunk?
1048	*/
1049	if (pRight && (char )pRight == (char )pFree + pFree->core.offNext)
1050	{
1051	/* core */
1052	if (pRight->core.offNext)
1053	{
1054	pFree->core.offNext += pRight->core.offNext;
1055	PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pFree + pFree->core.offNext);
1056	MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pFree->core.offNext);
1057	ASSERT_CHUNK(pHeap, pNext);
1058	}
1059	else
1060	pFree->core.offNext = 0;
1061
1062	/* free */
1063	if (pRight->offNext)
1064	{
1065	pFree->offNext += pRight->offNext;
1066	((PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext))->offPrev = -(int32_t)pFree->offNext;
1067	}
1068	else
1069	{
1070	pFree->offNext = 0;
1071	pHeap->offFreeTail = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
1072	}
1073	Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pRight->cb, -(int32_t)pRight->cb));
1074	pHeap->cbFree -= pRight->cb;
1075	Log3(("mmHyperFree: Merged %p (cb=%d) into %p.\n", pRight, pRight->cb, pFree));
1076	}
1077
1078	/* calculate the size. */
1079	if (pFree->core.offNext)
1080	pFree->cb = pFree->core.offNext - sizeof(MMHYPERCHUNK);
1081	else
1082	pFree->cb = pHeap->offPageAligned - ((uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap)) - sizeof(MMHYPERCHUNK);
1083	Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree + pFree->cb, pFree->cb));
1084	pHeap->cbFree += pFree->cb;
1085	ASSERT_CHUNK_FREE(pHeap, pFree);
1086
1087	#ifdef MMHYPER_HEAP_STRICT
1088	mmHyperHeapCheck(pHeap);
1089	#endif
1090	return VINF_SUCCESS;
1091	}
1092
1093
1094	#if defined(DEBUG) \|\| defined(MMHYPER_HEAP_STRICT)
1095	/**
1096	* Dumps a heap chunk to the log.
1097	*
1098	* @param pHeap Pointer to the heap.
1099	* @param pCur Pointer to the chunk.
1100	*/
1101	static void mmHyperHeapDumpOne(PMMHYPERHEAP pHeap, PMMHYPERCHUNKFREE pCur)
1102	{
1103	if (MMHYPERCHUNK_ISUSED(&pCur->core))
1104	{
1105	if (pCur->core.offStat)
1106	{
1107	PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pCur + pCur->core.offStat);
1108	const char *pszSelf = pCur->core.offStat == sizeof(MMHYPERCHUNK) ? " stat record" : "";
1109	#ifdef IN_RING3
1110	Log(("%p %06x USED offNext=%06x offPrev=-%06x %s%s\n",
1111	pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1112	pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1113	mmGetTagName((MMTAG)pStat->Core.Key), pszSelf));
1114	#else
1115	Log(("%p %06x USED offNext=%06x offPrev=-%06x %d%s\n",
1116	pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1117	pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1118	(MMTAG)pStat->Core.Key, pszSelf));
1119	#endif
1120	}
1121	else
1122	Log(("%p %06x USED offNext=%06x offPrev=-%06x\n",
1123	pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1124	pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core)));
1125	}
1126	else
1127	Log(("%p %06x FREE offNext=%06x offPrev=-%06x : cb=%06x offNext=%06x offPrev=-%06x\n",
1128	pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1129	pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core), pCur->cb, pCur->offNext, pCur->offPrev));
1130	}
1131	#endif /* DEBUG \|\| MMHYPER_HEAP_STRICT */
1132
1133
1134	#ifdef MMHYPER_HEAP_STRICT
1135	/**
1136	* Internal consitency check.
1137	*/
1138	static void mmHyperHeapCheck(PMMHYPERHEAP pHeap)
1139	{
1140	PMMHYPERCHUNKFREE pPrev = NULL;
1141	PMMHYPERCHUNKFREE pCur = (PMMHYPERCHUNKFREE)pHeap->CTX_SUFF(pbHeap);
1142	for (;;)
1143	{
1144	if (MMHYPERCHUNK_ISUSED(&pCur->core))
1145	ASSERT_CHUNK_USED(pHeap, &pCur->core);
1146	else
1147	ASSERT_CHUNK_FREE(pHeap, pCur);
1148	if (pPrev)
1149	AssertMsg((int32_t)pPrev->core.offNext == -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1150	("pPrev->core.offNext=%d offPrev=%d\n", pPrev->core.offNext, MMHYPERCHUNK_GET_OFFPREV(&pCur->core)));
1151
1152	# ifdef MMHYPER_HEAP_STRICT_FENCE
1153	uint32_t off = (uint8_t *)pCur - pHeap->CTX_SUFF(pbHeap);
1154	if ( MMHYPERCHUNK_ISUSED(&pCur->core)
1155	&& off < pHeap->offPageAligned)
1156	{
1157	uint32_t cbCur = pCur->core.offNext
1158	? pCur->core.offNext
1159	: pHeap->cbHeap - off;
1160	uint32_t pu32End = ((uint32_t )((uint8_t *)pCur + cbCur));
1161	uint32_t cbFence = pu32End[-1];
1162	if (RT_UNLIKELY( cbFence >= cbCur - sizeof(*pCur)
1163	\|\| cbFence < MMHYPER_HEAP_STRICT_FENCE_SIZE))
1164	{
1165	mmHyperHeapDumpOne(pHeap, pCur);
1166	Assert(cbFence < cbCur - sizeof(*pCur));
1167	Assert(cbFence >= MMHYPER_HEAP_STRICT_FENCE_SIZE);
1168	}
1169
1170	uint32_t pu32Bad = ASMMemIsAllU32((uint8_t )pu32End - cbFence, cbFence - sizeof(uint32_t), MMHYPER_HEAP_STRICT_FENCE_U32);
1171	if (RT_UNLIKELY(pu32Bad))
1172	{
1173	mmHyperHeapDumpOne(pHeap, pCur);
1174	Assert(!pu32Bad);
1175	}
1176	}
1177	# endif
1178
1179	/* next */
1180	if (!pCur->core.offNext)
1181	break;
1182	pPrev = pCur;
1183	pCur = (PMMHYPERCHUNKFREE)((char *)pCur + pCur->core.offNext);
1184	}
1185	}
1186	#endif
1187
1188
1189	/**
1190	* Performs consistency checks on the heap if MMHYPER_HEAP_STRICT was
1191	* defined at build time.
1192	*
1193	* @param pVM Pointer to the shared VM structure.
1194	*/
1195	VMMDECL(void) MMHyperHeapCheck(PVM pVM)
1196	{
1197	#ifdef MMHYPER_HEAP_STRICT
1198	int rc;
1199
1200	rc = mmHyperLock(pVM);
1201	AssertRC(rc);
1202	mmHyperHeapCheck(pVM->mm.s.CTX_SUFF(pHyperHeap));
1203	mmHyperUnlock(pVM);
1204	#endif
1205	}
1206
1207
1208	#ifdef DEBUG
1209	/**
1210	* Dumps the hypervisor heap to Log.
1211	* @param pVM VM Handle.
1212	*/
1213	VMMDECL(void) MMHyperHeapDump(PVM pVM)
1214	{
1215	Log(("MMHyperHeapDump: * heap dump - start *\n"));
1216	PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
1217	PMMHYPERCHUNKFREE pCur = (PMMHYPERCHUNKFREE)pHeap->CTX_SUFF(pbHeap);
1218	for (;;)
1219	{
1220	mmHyperHeapDumpOne(pHeap, pCur);
1221
1222	/* next */
1223	if (!pCur->core.offNext)
1224	break;
1225	pCur = (PMMHYPERCHUNKFREE)((char *)pCur + pCur->core.offNext);
1226	}
1227	Log(("MMHyperHeapDump: * heap dump - end *\n"));
1228	}
1229	#endif
1230
1231
1232	/**
1233	* Query the amount of free memory in the hypervisor heap.
1234	*
1235	* @returns Number of free bytes in the hypervisor heap.
1236	*/
1237	VMMDECL(size_t) MMHyperHeapGetFreeSize(PVM pVM)
1238	{
1239	return pVM->mm.s.CTX_SUFF(pHyperHeap)->cbFree;
1240	}
1241
1242	/**
1243	* Query the size the hypervisor heap.
1244	*
1245	* @returns The size of the hypervisor heap in bytes.
1246	*/
1247	VMMDECL(size_t) MMHyperHeapGetSize(PVM pVM)
1248	{
1249	return pVM->mm.s.CTX_SUFF(pHyperHeap)->cbHeap;
1250	}
1251
1252
1253	/**
1254	* Query the address and size the hypervisor memory area.
1255	*
1256	* @returns Base address of the hypervisor area.
1257	* @param pVM VM Handle.
1258	* @param pcb Where to store the size of the hypervisor area. (out)
1259	*/
1260	VMMDECL(RTGCPTR) MMHyperGetArea(PVM pVM, size_t *pcb)
1261	{
1262	if (pcb)
1263	*pcb = pVM->mm.s.cbHyperArea;
1264	return pVM->mm.s.pvHyperAreaGC;
1265	}
1266
1267
1268	/**
1269	* Checks if an address is within the hypervisor memory area.
1270	*
1271	* @returns true if inside.
1272	* @returns false if outside.
1273	* @param pVM VM handle.
1274	* @param GCPtr The pointer to check.
1275	*/
1276	VMMDECL(bool) MMHyperIsInsideArea(PVM pVM, RTGCPTR GCPtr)
1277	{
1278	return (RTGCUINTPTR)GCPtr - (RTGCUINTPTR)pVM->mm.s.pvHyperAreaGC < pVM->mm.s.cbHyperArea;
1279	}
1280

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/MMAllHyper.cpp@ 26510

Download in other formats: