MMAllHyper.cpp@ 49893

Last change on this file since 49893 was 49893, checked in by vboxsync, 11 years ago
MSR rewrite: initial hacking - half disabled.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 47.3 KB

Line
1	/* $Id: MMAllHyper.cpp 49893 2013-12-13 00:40:20Z vboxsync $ */
2	/** @file
3	* MM - Memory Manager - Hypervisor Memory Area, All Contexts.
4	*/
5
6	/*
7	* Copyright (C) 2006-2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_MM_HYPER_HEAP
23	#include <VBox/vmm/mm.h>
24	#include <VBox/vmm/stam.h>
25	#include "MMInternal.h"
26	#include <VBox/vmm/vm.h>
27
28	#include <VBox/err.h>
29	#include <VBox/param.h>
30	#include <iprt/assert.h>
31	#include <VBox/log.h>
32	#include <iprt/asm.h>
33	#include <iprt/string.h>
34
35
36	/*******************************************************************************
37	* Defined Constants And Macros *
38	*******************************************************************************/
39	#define ASSERT_L(u1, u2) AssertMsg((u1) < (u2), ("u1=%#x u2=%#x\n", u1, u2))
40	#define ASSERT_LE(u1, u2) AssertMsg((u1) <= (u2), ("u1=%#x u2=%#x\n", u1, u2))
41	#define ASSERT_GE(u1, u2) AssertMsg((u1) >= (u2), ("u1=%#x u2=%#x\n", u1, u2))
42	#define ASSERT_ALIGN(u1) AssertMsg(!((u1) & (MMHYPER_HEAP_ALIGN_MIN - 1)), ("u1=%#x (%d)\n", u1, u1))
43
44	#define ASSERT_OFFPREV(pHeap, pChunk) \
45	do { Assert(MMHYPERCHUNK_GET_OFFPREV(pChunk) <= 0); \
46	Assert(MMHYPERCHUNK_GET_OFFPREV(pChunk) >= (intptr_t)(pHeap)->CTX_SUFF(pbHeap) - (intptr_t)(pChunk)); \
47	AssertMsg( MMHYPERCHUNK_GET_OFFPREV(pChunk) != 0 \
48	\|\| (uint8_t *)(pChunk) == (pHeap)->CTX_SUFF(pbHeap), \
49	("pChunk=%p pvHyperHeap=%p\n", (pChunk), (pHeap)->CTX_SUFF(pbHeap))); \
50	} while (0)
51
52	#define ASSERT_OFFNEXT(pHeap, pChunk) \
53	do { ASSERT_ALIGN((pChunk)->offNext); \
54	ASSERT_L((pChunk)->offNext, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
55	} while (0)
56
57	#define ASSERT_OFFHEAP(pHeap, pChunk) \
58	do { Assert((pChunk)->offHeap); \
59	AssertMsg((PMMHYPERHEAP)((pChunk)->offHeap + (uintptr_t)pChunk) == (pHeap), \
60	("offHeap=%RX32 pChunk=%p pHeap=%p\n", (pChunk)->offHeap, (pChunk), (pHeap))); \
61	Assert((pHeap)->u32Magic == MMHYPERHEAP_MAGIC); \
62	} while (0)
63
64	#ifdef VBOX_WITH_STATISTICS
65	#define ASSERT_OFFSTAT(pHeap, pChunk) \
66	do { if (MMHYPERCHUNK_ISFREE(pChunk)) \
67	Assert(!(pChunk)->offStat); \
68	else if ((pChunk)->offStat) \
69	{ \
70	Assert((pChunk)->offStat); \
71	AssertMsg(!((pChunk)->offStat & (MMHYPER_HEAP_ALIGN_MIN - 1)), ("offStat=%RX32\n", (pChunk)->offStat)); \
72	uintptr_t uPtr = (uintptr_t)(pChunk)->offStat + (uintptr_t)pChunk; NOREF(uPtr); \
73	AssertMsg(uPtr - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) < (pHeap)->offPageAligned, \
74	("%p - %p < %RX32\n", uPtr, (pHeap)->CTX_SUFF(pbHeap), (pHeap)->offPageAligned)); \
75	} \
76	} while (0)
77	#else
78	#define ASSERT_OFFSTAT(pHeap, pChunk) \
79	do { Assert(!(pChunk)->offStat); \
80	} while (0)
81	#endif
82
83	#define ASSERT_CHUNK(pHeap, pChunk) \
84	do { ASSERT_OFFNEXT(pHeap, pChunk); \
85	ASSERT_OFFPREV(pHeap, pChunk); \
86	ASSERT_OFFHEAP(pHeap, pChunk); \
87	ASSERT_OFFSTAT(pHeap, pChunk); \
88	} while (0)
89	#define ASSERT_CHUNK_USED(pHeap, pChunk) \
90	do { ASSERT_OFFNEXT(pHeap, pChunk); \
91	ASSERT_OFFPREV(pHeap, pChunk); \
92	Assert(MMHYPERCHUNK_ISUSED(pChunk)); \
93	} while (0)
94
95	#define ASSERT_FREE_OFFPREV(pHeap, pChunk) \
96	do { ASSERT_ALIGN((pChunk)->offPrev); \
97	ASSERT_GE(((pChunk)->offPrev & (MMHYPER_HEAP_ALIGN_MIN - 1)), (intptr_t)(pHeap)->CTX_SUFF(pbHeap) - (intptr_t)(pChunk)); \
98	Assert((pChunk)->offPrev != MMHYPERCHUNK_GET_OFFPREV(&(pChunk)->core) \|\| !(pChunk)->offPrev); \
99	AssertMsg( (pChunk)->offPrev \
100	\|\| (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) == (pHeap)->offFreeHead, \
101	("pChunk=%p offChunk=%#x offFreeHead=%#x\n", (pChunk), (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap),\
102	(pHeap)->offFreeHead)); \
103	} while (0)
104
105	#define ASSERT_FREE_OFFNEXT(pHeap, pChunk) \
106	do { ASSERT_ALIGN((pChunk)->offNext); \
107	ASSERT_L((pChunk)->offNext, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
108	Assert((pChunk)->offNext != (pChunk)->core.offNext \|\| !(pChunk)->offNext); \
109	AssertMsg( (pChunk)->offNext \
110	\|\| (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) == (pHeap)->offFreeTail, \
111	("pChunk=%p offChunk=%#x offFreeTail=%#x\n", (pChunk), (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap), \
112	(pHeap)->offFreeTail)); \
113	} while (0)
114
115	#define ASSERT_FREE_CB(pHeap, pChunk) \
116	do { ASSERT_ALIGN((pChunk)->cb); \
117	Assert((pChunk)->cb > 0); \
118	if ((pChunk)->core.offNext) \
119	AssertMsg((pChunk)->cb == ((pChunk)->core.offNext - sizeof(MMHYPERCHUNK)), \
120	("cb=%d offNext=%d\n", (pChunk)->cb, (pChunk)->core.offNext)); \
121	else \
122	ASSERT_LE((pChunk)->cb, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
123	} while (0)
124
125	#define ASSERT_CHUNK_FREE(pHeap, pChunk) \
126	do { ASSERT_CHUNK(pHeap, &(pChunk)->core); \
127	Assert(MMHYPERCHUNK_ISFREE(pChunk)); \
128	ASSERT_FREE_OFFNEXT(pHeap, pChunk); \
129	ASSERT_FREE_OFFPREV(pHeap, pChunk); \
130	ASSERT_FREE_CB(pHeap, pChunk); \
131	} while (0)
132
133
134	/*******************************************************************************
135	* Internal Functions *
136	*******************************************************************************/
137	static PMMHYPERCHUNK mmHyperAllocChunk(PMMHYPERHEAP pHeap, uint32_t cb, unsigned uAlignment);
138	static void *mmHyperAllocPages(PMMHYPERHEAP pHeap, uint32_t cb);
139	#ifdef VBOX_WITH_STATISTICS
140	static PMMHYPERSTAT mmHyperStat(PMMHYPERHEAP pHeap, MMTAG enmTag);
141	#ifdef IN_RING3
142	static void mmR3HyperStatRegisterOne(PVM pVM, PMMHYPERSTAT pStat);
143	#endif
144	#endif
145	static int mmHyperFree(PMMHYPERHEAP pHeap, PMMHYPERCHUNK pChunk);
146	#ifdef MMHYPER_HEAP_STRICT
147	static void mmHyperHeapCheck(PMMHYPERHEAP pHeap);
148	#endif
149
150
151
152	/**
153	* Locks the hypervisor heap.
154	* This might call back to Ring-3 in order to deal with lock contention in GC and R3.
155	*
156	* @param pVM Pointer to the VM.
157	*/
158	static int mmHyperLock(PVM pVM)
159	{
160	PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
161
162	#ifdef IN_RING3
163	if (!PDMCritSectIsInitialized(&pHeap->Lock))
164	return VINF_SUCCESS; /* early init */
165	#else
166	Assert(PDMCritSectIsInitialized(&pHeap->Lock));
167	#endif
168	int rc = PDMCritSectEnter(&pHeap->Lock, VERR_SEM_BUSY);
169	#if defined(IN_RC) \|\| defined(IN_RING0)
170	if (rc == VERR_SEM_BUSY)
171	rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_MMHYPER_LOCK, 0);
172	#endif
173	AssertRC(rc);
174	return rc;
175	}
176
177
178	/**
179	* Unlocks the hypervisor heap.
180	*
181	* @param pVM Pointer to the VM.
182	*/
183	static void mmHyperUnlock(PVM pVM)
184	{
185	PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
186
187	#ifdef IN_RING3
188	if (!PDMCritSectIsInitialized(&pHeap->Lock))
189	return; /* early init */
190	#endif
191	Assert(PDMCritSectIsInitialized(&pHeap->Lock));
192	PDMCritSectLeave(&pHeap->Lock);
193	}
194
195	/**
196	* Allocates memory in the Hypervisor (RC VMM) area.
197	* The returned memory is of course zeroed.
198	*
199	* @returns VBox status code.
200	* @param pVM Pointer to the VM.
201	* @param cb Number of bytes to allocate.
202	* @param uAlignment Required memory alignment in bytes.
203	* Values are 0,8,16,32,64 and PAGE_SIZE.
204	* 0 -> default alignment, i.e. 8 bytes.
205	* @param enmTag The statistics tag.
206	* @param ppv Where to store the address to the allocated
207	* memory.
208	*/
209	static int mmHyperAllocInternal(PVM pVM, size_t cb, unsigned uAlignment, MMTAG enmTag, void **ppv)
210	{
211	AssertMsg(cb >= 8, ("Hey! Do you really mean to allocate less than 8 bytes?! cb=%d\n", cb));
212
213	/*
214	* Validate input and adjust it to reasonable values.
215	*/
216	if (!uAlignment \|\| uAlignment < MMHYPER_HEAP_ALIGN_MIN)
217	uAlignment = MMHYPER_HEAP_ALIGN_MIN;
218	uint32_t cbAligned;
219	switch (uAlignment)
220	{
221	case 8:
222	case 16:
223	case 32:
224	case 64:
225	cbAligned = RT_ALIGN_32(cb, MMHYPER_HEAP_ALIGN_MIN);
226	if (!cbAligned \|\| cbAligned < cb)
227	{
228	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
229	AssertMsgFailed(("Nice try.\n"));
230	return VERR_INVALID_PARAMETER;
231	}
232	break;
233
234	case PAGE_SIZE:
235	AssertMsg(RT_ALIGN_32(cb, PAGE_SIZE) == cb, ("The size isn't page aligned. (cb=%#x)\n", cb));
236	cbAligned = RT_ALIGN_32(cb, PAGE_SIZE);
237	if (!cbAligned)
238	{
239	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
240	AssertMsgFailed(("Nice try.\n"));
241	return VERR_INVALID_PARAMETER;
242	}
243	break;
244
245	default:
246	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
247	AssertMsgFailed(("Invalid alignment %u\n", uAlignment));
248	return VERR_INVALID_PARAMETER;
249	}
250
251
252	/*
253	* Get heap and statisticsStatistics.
254	*/
255	PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
256	#ifdef VBOX_WITH_STATISTICS
257	PMMHYPERSTAT pStat = mmHyperStat(pHeap, enmTag);
258	if (!pStat)
259	{
260	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_MM_HYPER_NO_MEMORY\n", cb, uAlignment));
261	AssertMsgFailed(("Failed to allocate statistics!\n"));
262	return VERR_MM_HYPER_NO_MEMORY;
263	}
264	#endif
265	if (uAlignment < PAGE_SIZE)
266	{
267	/*
268	* Allocate a chunk.
269	*/
270	PMMHYPERCHUNK pChunk = mmHyperAllocChunk(pHeap, cbAligned, uAlignment);
271	if (pChunk)
272	{
273	#ifdef VBOX_WITH_STATISTICS
274	const uint32_t cbChunk = pChunk->offNext
275	? pChunk->offNext
276	: pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t *)pChunk;
277	pStat->cbAllocated += (uint32_t)cbChunk;
278	pStat->cbCurAllocated += (uint32_t)cbChunk;
279	if (pStat->cbCurAllocated > pStat->cbMaxAllocated)
280	pStat->cbMaxAllocated = pStat->cbCurAllocated;
281	pStat->cAllocations++;
282	pChunk->offStat = (uintptr_t)pStat - (uintptr_t)pChunk;
283	#else
284	pChunk->offStat = 0;
285	#endif
286	void *pv = pChunk + 1;
287	*ppv = pv;
288	ASMMemZero32(pv, cbAligned);
289	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VINF_SUCCESS and *ppv=%p\n", cb, uAlignment, pv));
290	return VINF_SUCCESS;
291	}
292	}
293	else
294	{
295	/*
296	* Allocate page aligned memory.
297	*/
298	void *pv = mmHyperAllocPages(pHeap, cbAligned);
299	if (pv)
300	{
301	#ifdef VBOX_WITH_STATISTICS
302	pStat->cbAllocated += cbAligned;
303	pStat->cbCurAllocated += cbAligned;
304	if (pStat->cbCurAllocated > pStat->cbMaxAllocated)
305	pStat->cbMaxAllocated = pStat->cbCurAllocated;
306	pStat->cAllocations++;
307	#endif
308	*ppv = pv;
309	/* ASMMemZero32(pv, cbAligned); - not required since memory is alloc-only and SUPR3PageAlloc zeros it. */
310	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VINF_SUCCESS and *ppv=%p\n", cb, uAlignment, ppv));
311	return VINF_SUCCESS;
312	}
313	}
314
315	#ifdef VBOX_WITH_STATISTICS
316	pStat->cAllocations++;
317	pStat->cFailures++;
318	#endif
319	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_MM_HYPER_NO_MEMORY\n", cb, uAlignment));
320	AssertMsgFailed(("Failed to allocate %d bytes!\n", cb));
321	return VERR_MM_HYPER_NO_MEMORY;
322	}
323
324
325	/**
326	* Wrapper for mmHyperAllocInternal
327	*/
328	VMMDECL(int) MMHyperAlloc(PVM pVM, size_t cb, unsigned uAlignment, MMTAG enmTag, void **ppv)
329	{
330	int rc = mmHyperLock(pVM);
331	AssertRCReturn(rc, rc);
332
333	LogFlow(("MMHyperAlloc %x align=%x tag=%s\n", cb, uAlignment, mmGetTagName(enmTag)));
334
335	rc = mmHyperAllocInternal(pVM, cb, uAlignment, enmTag, ppv);
336
337	mmHyperUnlock(pVM);
338	return rc;
339	}
340
341
342	/**
343	* Duplicates a block of memory.
344	*/
345	VMMDECL(int) MMHyperDupMem(PVM pVM, const void pvSrc, size_t cb, unsigned uAlignment, MMTAG enmTag, void *ppv)
346	{
347	int rc = MMHyperAlloc(pVM, cb, uAlignment, enmTag, ppv);
348	if (RT_SUCCESS(rc))
349	memcpy(*ppv, pvSrc, cb);
350	return rc;
351	}
352
353
354	/**
355	* Allocates a chunk of memory from the specified heap.
356	* The caller validates the parameters of this request.
357	*
358	* @returns Pointer to the allocated chunk.
359	* @returns NULL on failure.
360	* @param pHeap The heap.
361	* @param cb Size of the memory block to allocate.
362	* @param uAlignment The alignment specifications for the allocated block.
363	* @internal
364	*/
365	static PMMHYPERCHUNK mmHyperAllocChunk(PMMHYPERHEAP pHeap, uint32_t cb, unsigned uAlignment)
366	{
367	Log3(("mmHyperAllocChunk: Enter cb=%#x uAlignment=%#x\n", cb, uAlignment));
368	#ifdef MMHYPER_HEAP_STRICT
369	mmHyperHeapCheck(pHeap);
370	#endif
371	#ifdef MMHYPER_HEAP_STRICT_FENCE
372	uint32_t cbFence = RT_MAX(MMHYPER_HEAP_STRICT_FENCE_SIZE, uAlignment);
373	cb += cbFence;
374	#endif
375
376	/*
377	* Check if there are any free chunks. (NIL_OFFSET use/not-use forces this check)
378	*/
379	if (pHeap->offFreeHead == NIL_OFFSET)
380	return NULL;
381
382	/*
383	* Small alignments - from the front of the heap.
384	*
385	* Must split off free chunks at the end to prevent messing up the
386	* last free node which we take the page aligned memory from the top of.
387	*/
388	PMMHYPERCHUNK pRet = NULL;
389	PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)((char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeHead);
390	while (pFree)
391	{
392	ASSERT_CHUNK_FREE(pHeap, pFree);
393	if (pFree->cb >= cb)
394	{
395	unsigned offAlign = (uintptr_t)(&pFree->core + 1) & (uAlignment - 1);
396	if (offAlign)
397	offAlign = uAlignment - offAlign;
398	if (!offAlign \|\| pFree->cb - offAlign >= cb)
399	{
400	Log3(("mmHyperAllocChunk: Using pFree=%p pFree->cb=%d offAlign=%d\n", pFree, pFree->cb, offAlign));
401
402	/*
403	* Adjust the node in front.
404	* Because of multiple alignments we need to special case allocation of the first block.
405	*/
406	if (offAlign)
407	{
408	MMHYPERCHUNKFREE Free = *pFree;
409	if (MMHYPERCHUNK_GET_OFFPREV(&pFree->core))
410	{
411	/* just add a bit of memory to it. */
412	PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + MMHYPERCHUNK_GET_OFFPREV(&Free.core));
413	pPrev->core.offNext += offAlign;
414	AssertMsg(!MMHYPERCHUNK_ISFREE(&pPrev->core), ("Impossible!\n"));
415	Log3(("mmHyperAllocChunk: Added %d bytes to %p\n", offAlign, pPrev));
416	}
417	else
418	{
419	/* make new head node, mark it USED for simplicity. */
420	PMMHYPERCHUNK pPrev = (PMMHYPERCHUNK)pHeap->CTX_SUFF(pbHeap);
421	Assert(pPrev == &pFree->core);
422	pPrev->offPrev = 0;
423	MMHYPERCHUNK_SET_TYPE(pPrev, MMHYPERCHUNK_FLAGS_USED);
424	pPrev->offNext = offAlign;
425	Log3(("mmHyperAllocChunk: Created new first node of %d bytes\n", offAlign));
426
427	}
428	Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - offAlign, -(int)offAlign));
429	pHeap->cbFree -= offAlign;
430
431	/* Recreate pFree node and adjusting everything... */
432	pFree = (PMMHYPERCHUNKFREE)((char *)pFree + offAlign);
433	*pFree = Free;
434
435	pFree->cb -= offAlign;
436	if (pFree->core.offNext)
437	{
438	pFree->core.offNext -= offAlign;
439	PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pFree + pFree->core.offNext);
440	MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pFree->core.offNext);
441	ASSERT_CHUNK(pHeap, pNext);
442	}
443	if (MMHYPERCHUNK_GET_OFFPREV(&pFree->core))
444	MMHYPERCHUNK_SET_OFFPREV(&pFree->core, MMHYPERCHUNK_GET_OFFPREV(&pFree->core) - offAlign);
445
446	if (pFree->offNext)
447	{
448	pFree->offNext -= offAlign;
449	PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext);
450	pNext->offPrev = -(int32_t)pFree->offNext;
451	ASSERT_CHUNK_FREE(pHeap, pNext);
452	}
453	else
454	pHeap->offFreeTail += offAlign;
455	if (pFree->offPrev)
456	{
457	pFree->offPrev -= offAlign;
458	PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
459	pPrev->offNext = -pFree->offPrev;
460	ASSERT_CHUNK_FREE(pHeap, pPrev);
461	}
462	else
463	pHeap->offFreeHead += offAlign;
464	pFree->core.offHeap = (uintptr_t)pHeap - (uintptr_t)pFree;
465	pFree->core.offStat = 0;
466	ASSERT_CHUNK_FREE(pHeap, pFree);
467	Log3(("mmHyperAllocChunk: Realigned pFree=%p\n", pFree));
468	}
469
470	/*
471	* Split off a new FREE chunk?
472	*/
473	if (pFree->cb >= cb + RT_ALIGN(sizeof(MMHYPERCHUNKFREE), MMHYPER_HEAP_ALIGN_MIN))
474	{
475	/*
476	* Move the FREE chunk up to make room for the new USED chunk.
477	*/
478	const int off = cb + sizeof(MMHYPERCHUNK);
479	PMMHYPERCHUNKFREE pNew = (PMMHYPERCHUNKFREE)((char *)&pFree->core + off);
480	pNew = pFree;
481	pNew->cb -= off;
482	if (pNew->core.offNext)
483	{
484	pNew->core.offNext -= off;
485	PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pNew + pNew->core.offNext);
486	MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pNew->core.offNext);
487	ASSERT_CHUNK(pHeap, pNext);
488	}
489	pNew->core.offPrev = -off;
490	MMHYPERCHUNK_SET_TYPE(pNew, MMHYPERCHUNK_FLAGS_FREE);
491
492	if (pNew->offNext)
493	{
494	pNew->offNext -= off;
495	PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pNew + pNew->offNext);
496	pNext->offPrev = -(int32_t)pNew->offNext;
497	ASSERT_CHUNK_FREE(pHeap, pNext);
498	}
499	else
500	pHeap->offFreeTail += off;
501	if (pNew->offPrev)
502	{
503	pNew->offPrev -= off;
504	PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pNew + pNew->offPrev);
505	pPrev->offNext = -pNew->offPrev;
506	ASSERT_CHUNK_FREE(pHeap, pPrev);
507	}
508	else
509	pHeap->offFreeHead += off;
510	pNew->core.offHeap = (uintptr_t)pHeap - (uintptr_t)pNew;
511	pNew->core.offStat = 0;
512	ASSERT_CHUNK_FREE(pHeap, pNew);
513
514	/*
515	* Update the old FREE node making it a USED node.
516	*/
517	pFree->core.offNext = off;
518	MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_USED);
519
520
521	Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree,
522	pHeap->cbFree - (cb + sizeof(MMHYPERCHUNK)), -(int)(cb + sizeof(MMHYPERCHUNK))));
523	pHeap->cbFree -= (uint32_t)(cb + sizeof(MMHYPERCHUNK));
524	pRet = &pFree->core;
525	ASSERT_CHUNK(pHeap, &pFree->core);
526	Log3(("mmHyperAllocChunk: Created free chunk pNew=%p cb=%d\n", pNew, pNew->cb));
527	}
528	else
529	{
530	/*
531	* Link out of free list.
532	*/
533	if (pFree->offNext)
534	{
535	PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext);
536	if (pFree->offPrev)
537	{
538	pNext->offPrev += pFree->offPrev;
539	PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
540	pPrev->offNext += pFree->offNext;
541	ASSERT_CHUNK_FREE(pHeap, pPrev);
542	}
543	else
544	{
545	pHeap->offFreeHead += pFree->offNext;
546	pNext->offPrev = 0;
547	}
548	ASSERT_CHUNK_FREE(pHeap, pNext);
549	}
550	else
551	{
552	if (pFree->offPrev)
553	{
554	pHeap->offFreeTail += pFree->offPrev;
555	PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
556	pPrev->offNext = 0;
557	ASSERT_CHUNK_FREE(pHeap, pPrev);
558	}
559	else
560	{
561	pHeap->offFreeHead = NIL_OFFSET;
562	pHeap->offFreeTail = NIL_OFFSET;
563	}
564	}
565
566	Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree,
567	pHeap->cbFree - pFree->cb, -(int32_t)pFree->cb));
568	pHeap->cbFree -= pFree->cb;
569	MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_USED);
570	pRet = &pFree->core;
571	ASSERT_CHUNK(pHeap, &pFree->core);
572	Log3(("mmHyperAllocChunk: Converted free chunk %p to used chunk.\n", pFree));
573	}
574	Log3(("mmHyperAllocChunk: Returning %p\n", pRet));
575	break;
576	}
577	}
578
579	/* next */
580	pFree = pFree->offNext ? (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext) : NULL;
581	}
582
583	#ifdef MMHYPER_HEAP_STRICT_FENCE
584	uint32_t pu32End = (uint32_t )((uint8_t *)(pRet + 1) + cb);
585	uint32_t *pu32EndReal = pRet->offNext
586	? (uint32_t )((uint8_t )pRet + pRet->offNext)
587	: (uint32_t *)(pHeap->CTX_SUFF(pbHeap) + pHeap->cbHeap);
588	cbFence += (uintptr_t)pu32EndReal - (uintptr_t)pu32End; Assert(!(cbFence & 0x3));
589	ASMMemFill32((uint8_t *)pu32EndReal - cbFence, cbFence, MMHYPER_HEAP_STRICT_FENCE_U32);
590	pu32EndReal[-1] = cbFence;
591	#endif
592	#ifdef MMHYPER_HEAP_STRICT
593	mmHyperHeapCheck(pHeap);
594	#endif
595	return pRet;
596	}
597
598
599	/**
600	* Allocates one or more pages of memory from the specified heap.
601	* The caller validates the parameters of this request.
602	*
603	* @returns Pointer to the allocated chunk.
604	* @returns NULL on failure.
605	* @param pHeap The heap.
606	* @param cb Size of the memory block to allocate.
607	* @internal
608	*/
609	static void *mmHyperAllocPages(PMMHYPERHEAP pHeap, uint32_t cb)
610	{
611	Log3(("mmHyperAllocPages: Enter cb=%#x\n", cb));
612
613	#ifdef MMHYPER_HEAP_STRICT
614	mmHyperHeapCheck(pHeap);
615	#endif
616
617	/*
618	* Check if there are any free chunks. (NIL_OFFSET use/not-use forces this check)
619	*/
620	if (pHeap->offFreeHead == NIL_OFFSET)
621	return NULL;
622
623	/*
624	* Page aligned chunks.
625	*
626	* Page aligned chunks can only be allocated from the last FREE chunk.
627	* This is for reasons of simplicity and fragmentation. Page aligned memory
628	* must also be allocated in page aligned sizes. Page aligned memory cannot
629	* be freed either.
630	*
631	* So, for this to work, the last FREE chunk needs to end on a page aligned
632	* boundary.
633	*/
634	PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)((char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeTail);
635	ASSERT_CHUNK_FREE(pHeap, pFree);
636	if ( (((uintptr_t)(&pFree->core + 1) + pFree->cb) & (PAGE_OFFSET_MASK - 1))
637	\|\| pFree->cb + sizeof(MMHYPERCHUNK) < cb)
638	{
639	Log3(("mmHyperAllocPages: Not enough/no page aligned memory!\n"));
640	return NULL;
641	}
642
643	void *pvRet;
644	if (pFree->cb > cb)
645	{
646	/*
647	* Simple, just cut the top of the free node and return it.
648	*/
649	pFree->cb -= cb;
650	pvRet = (char *)(&pFree->core + 1) + pFree->cb;
651	AssertMsg(RT_ALIGN_P(pvRet, PAGE_SIZE) == pvRet, ("pvRet=%p cb=%#x pFree=%p pFree->cb=%#x\n", pvRet, cb, pFree, pFree->cb));
652	Log3(("mmHyperAllocPages: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - cb, -(int)cb));
653	pHeap->cbFree -= cb;
654	ASSERT_CHUNK_FREE(pHeap, pFree);
655	Log3(("mmHyperAllocPages: Allocated from pFree=%p new pFree->cb=%d\n", pFree, pFree->cb));
656	}
657	else
658	{
659	/*
660	* Unlink the FREE node.
661	*/
662	pvRet = (char *)(&pFree->core + 1) + pFree->cb - cb;
663	Log3(("mmHyperAllocPages: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pFree->cb, -(int32_t)pFree->cb));
664	pHeap->cbFree -= pFree->cb;
665
666	/* a scrap of spare memory (unlikely)? add it to the sprevious chunk. */
667	if (pvRet != (void *)pFree)
668	{
669	AssertMsg(MMHYPERCHUNK_GET_OFFPREV(&pFree->core), ("How the beep did someone manage to allocated up all the heap with page aligned memory?!?\n"));
670	PMMHYPERCHUNK pPrev = (PMMHYPERCHUNK)((char *)pFree + MMHYPERCHUNK_GET_OFFPREV(&pFree->core));
671	pPrev->offNext += (uintptr_t)pvRet - (uintptr_t)pFree;
672	AssertMsg(!MMHYPERCHUNK_ISFREE(pPrev), ("Free bug?\n"));
673	#ifdef VBOX_WITH_STATISTICS
674	PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pPrev + pPrev->offStat);
675	pStat->cbAllocated += (uintptr_t)pvRet - (uintptr_t)pFree;
676	pStat->cbCurAllocated += (uintptr_t)pvRet - (uintptr_t)pFree;
677	#endif
678	Log3(("mmHyperAllocPages: Added %d to %p (page align)\n", (uintptr_t)pvRet - (uintptr_t)pFree, pFree));
679	}
680
681	/* unlink from FREE chain. */
682	if (pFree->offPrev)
683	{
684	pHeap->offFreeTail += pFree->offPrev;
685	((PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev))->offNext = 0;
686	}
687	else
688	{
689	pHeap->offFreeTail = NIL_OFFSET;
690	pHeap->offFreeHead = NIL_OFFSET;
691	}
692	Log3(("mmHyperAllocPages: Unlinked pFree=%d\n", pFree));
693	}
694	pHeap->offPageAligned = (uintptr_t)pvRet - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
695	Log3(("mmHyperAllocPages: Returning %p (page aligned)\n", pvRet));
696
697	#ifdef MMHYPER_HEAP_STRICT
698	mmHyperHeapCheck(pHeap);
699	#endif
700	return pvRet;
701	}
702
703	#ifdef VBOX_WITH_STATISTICS
704
705	/**
706	* Get the statistic record for a tag.
707	*
708	* @returns Pointer to a stat record.
709	* @returns NULL on failure.
710	* @param pHeap The heap.
711	* @param enmTag The tag.
712	*/
713	static PMMHYPERSTAT mmHyperStat(PMMHYPERHEAP pHeap, MMTAG enmTag)
714	{
715	/* try look it up first. */
716	PMMHYPERSTAT pStat = (PMMHYPERSTAT)RTAvloGCPhysGet(&pHeap->HyperHeapStatTree, enmTag);
717	if (!pStat)
718	{
719	/* try allocate a new one */
720	PMMHYPERCHUNK pChunk = mmHyperAllocChunk(pHeap, RT_ALIGN(sizeof(*pStat), MMHYPER_HEAP_ALIGN_MIN), MMHYPER_HEAP_ALIGN_MIN);
721	if (!pChunk)
722	return NULL;
723	pStat = (PMMHYPERSTAT)(pChunk + 1);
724	pChunk->offStat = (uintptr_t)pStat - (uintptr_t)pChunk;
725
726	ASMMemZero32(pStat, sizeof(*pStat));
727	pStat->Core.Key = enmTag;
728	RTAvloGCPhysInsert(&pHeap->HyperHeapStatTree, &pStat->Core);
729	}
730	if (!pStat->fRegistered)
731	{
732	# ifdef IN_RING3
733	mmR3HyperStatRegisterOne(pHeap->pVMR3, pStat);
734	# else
735	/** @todo schedule a R3 action. */
736	# endif
737	}
738	return pStat;
739	}
740
741
742	# ifdef IN_RING3
743	/**
744	* Registers statistics with STAM.
745	*
746	*/
747	static void mmR3HyperStatRegisterOne(PVM pVM, PMMHYPERSTAT pStat)
748	{
749	if (pStat->fRegistered)
750	return;
751	const char *pszTag = mmGetTagName((MMTAG)pStat->Core.Key);
752	STAMR3RegisterF(pVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Number of bytes currently allocated.", "/MM/HyperHeap/%s", pszTag);
753	STAMR3RegisterF(pVM, &pStat->cAllocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of alloc calls.", "/MM/HyperHeap/%s/cAllocations", pszTag);
754	STAMR3RegisterF(pVM, &pStat->cFrees, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of free calls.", "/MM/HyperHeap/%s/cFrees", pszTag);
755	STAMR3RegisterF(pVM, &pStat->cFailures, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of failures.", "/MM/HyperHeap/%s/cFailures", pszTag);
756	STAMR3RegisterF(pVM, &pStat->cbAllocated, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of allocated bytes.", "/MM/HyperHeap/%s/cbAllocated", pszTag);
757	STAMR3RegisterF(pVM, &pStat->cbFreed, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of freed bytes.", "/MM/HyperHeap/%s/cbFreed", pszTag);
758	STAMR3RegisterF(pVM, &pStat->cbMaxAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Max number of bytes allocated at the same time.","/MM/HyperHeap/%s/cbMaxAllocated", pszTag);
759	pStat->fRegistered = true;
760	}
761	# endif /* IN_RING3 */
762
763	#endif /* VBOX_WITH_STATISTICS */
764
765
766	/**
767	* Free memory allocated using MMHyperAlloc().
768	* The caller validates the parameters of this request.
769	*
770	* @returns VBox status code.
771	* @param pVM Pointer to the VM.
772	* @param pv The memory to free.
773	* @remark Try avoid free hyper memory.
774	*/
775	static int mmHyperFreeInternal(PVM pVM, void *pv)
776	{
777	Log2(("MMHyperFree: pv=%p\n", pv));
778	if (!pv)
779	return VINF_SUCCESS;
780	AssertMsgReturn(RT_ALIGN_P(pv, MMHYPER_HEAP_ALIGN_MIN) == pv,
781	("Invalid pointer %p!\n", pv),
782	VERR_INVALID_POINTER);
783
784	/*
785	* Get the heap and stats.
786	* Validate the chunk at the same time.
787	*/
788	PMMHYPERCHUNK pChunk = (PMMHYPERCHUNK)((PMMHYPERCHUNK)pv - 1);
789
790	AssertMsgReturn( (uintptr_t)pChunk + pChunk->offNext >= (uintptr_t)pChunk
791	\|\| RT_ALIGN_32(pChunk->offNext, MMHYPER_HEAP_ALIGN_MIN) != pChunk->offNext,
792	("%p: offNext=%#RX32\n", pv, pChunk->offNext),
793	VERR_INVALID_POINTER);
794
795	AssertMsgReturn(MMHYPERCHUNK_ISUSED(pChunk),
796	("%p: Not used!\n", pv),
797	VERR_INVALID_POINTER);
798
799	int32_t offPrev = MMHYPERCHUNK_GET_OFFPREV(pChunk);
800	AssertMsgReturn( (uintptr_t)pChunk + offPrev <= (uintptr_t)pChunk
801	&& !((uint32_t)-offPrev & (MMHYPER_HEAP_ALIGN_MIN - 1)),
802	("%p: offPrev=%#RX32!\n", pv, offPrev),
803	VERR_INVALID_POINTER);
804
805	/* statistics */
806	#ifdef VBOX_WITH_STATISTICS
807	PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pChunk + pChunk->offStat);
808	AssertMsgReturn( RT_ALIGN_P(pStat, MMHYPER_HEAP_ALIGN_MIN) == (void *)pStat
809	&& pChunk->offStat,
810	("%p: offStat=%#RX32!\n", pv, pChunk->offStat),
811	VERR_INVALID_POINTER);
812	#else
813	AssertMsgReturn(!pChunk->offStat,
814	("%p: offStat=%#RX32!\n", pv, pChunk->offStat),
815	VERR_INVALID_POINTER);
816	#endif
817
818	/* The heap structure. */
819	PMMHYPERHEAP pHeap = (PMMHYPERHEAP)((uintptr_t)pChunk + pChunk->offHeap);
820	AssertMsgReturn( !((uintptr_t)pHeap & PAGE_OFFSET_MASK)
821	&& pChunk->offHeap,
822	("%p: pHeap=%#x offHeap=%RX32\n", pv, pHeap->u32Magic, pChunk->offHeap),
823	VERR_INVALID_POINTER);
824
825	AssertMsgReturn(pHeap->u32Magic == MMHYPERHEAP_MAGIC,
826	("%p: u32Magic=%#x\n", pv, pHeap->u32Magic),
827	VERR_INVALID_POINTER);
828	Assert(pHeap == pVM->mm.s.CTX_SUFF(pHyperHeap));
829
830	/* Some more verifications using additional info from pHeap. */
831	AssertMsgReturn((uintptr_t)pChunk + offPrev >= (uintptr_t)pHeap->CTX_SUFF(pbHeap),
832	("%p: offPrev=%#RX32!\n", pv, offPrev),
833	VERR_INVALID_POINTER);
834
835	AssertMsgReturn(pChunk->offNext < pHeap->cbHeap,
836	("%p: offNext=%#RX32!\n", pv, pChunk->offNext),
837	VERR_INVALID_POINTER);
838
839	AssertMsgReturn( (uintptr_t)pv - (uintptr_t)pHeap->CTX_SUFF(pbHeap) <= pHeap->offPageAligned,
840	("Invalid pointer %p! (heap: %p-%p)\n", pv, pHeap->CTX_SUFF(pbHeap),
841	(char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned),
842	VERR_INVALID_POINTER);
843
844	#ifdef MMHYPER_HEAP_STRICT
845	mmHyperHeapCheck(pHeap);
846	#endif
847
848	#if defined(VBOX_WITH_STATISTICS) \|\| defined(MMHYPER_HEAP_FREE_POISON)
849	/* calc block size. */
850	const uint32_t cbChunk = pChunk->offNext
851	? pChunk->offNext
852	: pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t *)pChunk;
853	#endif
854	#ifdef MMHYPER_HEAP_FREE_POISON
855	/* poison the block */
856	memset(pChunk + 1, MMHYPER_HEAP_FREE_POISON, cbChunk - sizeof(*pChunk));
857	#endif
858
859	#ifdef MMHYPER_HEAP_FREE_DELAY
860	# ifdef MMHYPER_HEAP_FREE_POISON
861	/*
862	* Check poison.
863	*/
864	unsigned i = RT_ELEMENTS(pHeap->aDelayedFrees);
865	while (i-- > 0)
866	if (pHeap->aDelayedFrees[i].offChunk)
867	{
868	PMMHYPERCHUNK pCur = (PMMHYPERCHUNK)((uintptr_t)pHeap + pHeap->aDelayedFrees[i].offChunk);
869	const size_t cb = pCur->offNext
870	? pCur->offNext - sizeof(*pCur)
871	: pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t )pCur - sizeof(pCur);
872	uint8_t pab = (uint8_t )(pCur + 1);
873	for (unsigned off = 0; off < cb; off++)
874	AssertReleaseMsg(pab[off] == 0xCB,
875	("caller=%RTptr cb=%#zx off=%#x: %.*Rhxs\n",
876	pHeap->aDelayedFrees[i].uCaller, cb, off, RT_MIN(cb - off, 32), &pab[off]));
877	}
878	# endif /* MMHYPER_HEAP_FREE_POISON */
879
880	/*
881	* Delayed freeing.
882	*/
883	int rc = VINF_SUCCESS;
884	if (pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk)
885	{
886	PMMHYPERCHUNK pChunkFree = (PMMHYPERCHUNK)((uintptr_t)pHeap + pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk);
887	rc = mmHyperFree(pHeap, pChunkFree);
888	}
889	pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk = (uintptr_t)pChunk - (uintptr_t)pHeap;
890	pHeap->aDelayedFrees[pHeap->iDelayedFree].uCaller = (uintptr_t)ASMReturnAddress();
891	pHeap->iDelayedFree = (pHeap->iDelayedFree + 1) % RT_ELEMENTS(pHeap->aDelayedFrees);
892
893	#else /* !MMHYPER_HEAP_FREE_POISON */
894	/*
895	* Call the worker.
896	*/
897	int rc = mmHyperFree(pHeap, pChunk);
898	#endif /* !MMHYPER_HEAP_FREE_POISON */
899
900	/*
901	* Update statistics.
902	*/
903	#ifdef VBOX_WITH_STATISTICS
904	pStat->cFrees++;
905	if (RT_SUCCESS(rc))
906	{
907	pStat->cbFreed += cbChunk;
908	pStat->cbCurAllocated -= cbChunk;
909	}
910	else
911	pStat->cFailures++;
912	#endif
913
914	return rc;
915	}
916
917
918	/**
919	* Wrapper for mmHyperFreeInternal
920	*/
921	VMMDECL(int) MMHyperFree(PVM pVM, void *pv)
922	{
923	int rc;
924
925	rc = mmHyperLock(pVM);
926	AssertRCReturn(rc, rc);
927
928	LogFlow(("MMHyperFree %p\n", pv));
929
930	rc = mmHyperFreeInternal(pVM, pv);
931
932	mmHyperUnlock(pVM);
933	return rc;
934	}
935
936
937	/**
938	* Free memory a memory chunk.
939	*
940	* @returns VBox status code.
941	* @param pHeap The heap.
942	* @param pChunk The memory chunk to free.
943	*/
944	static int mmHyperFree(PMMHYPERHEAP pHeap, PMMHYPERCHUNK pChunk)
945	{
946	Log3(("mmHyperFree: Enter pHeap=%p pChunk=%p\n", pHeap, pChunk));
947	PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)pChunk;
948
949	/*
950	* Insert into the free list (which is sorted on address).
951	*
952	* We'll search towards the end of the heap to locate the
953	* closest FREE chunk.
954	*/
955	PMMHYPERCHUNKFREE pLeft = NULL;
956	PMMHYPERCHUNKFREE pRight = NULL;
957	if (pHeap->offFreeTail != NIL_OFFSET)
958	{
959	if (pFree->core.offNext)
960	{
961	pRight = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->core.offNext);
962	ASSERT_CHUNK(pHeap, &pRight->core);
963	while (!MMHYPERCHUNK_ISFREE(&pRight->core))
964	{
965	if (!pRight->core.offNext)
966	{
967	pRight = NULL;
968	break;
969	}
970	pRight = (PMMHYPERCHUNKFREE)((char *)pRight + pRight->core.offNext);
971	ASSERT_CHUNK(pHeap, &pRight->core);
972	}
973	}
974	if (!pRight)
975	pRight = (PMMHYPERCHUNKFREE)((char )pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeTail); /* @todo this can't be correct! 'pLeft = .. ; else' I think */
976	if (pRight)
977	{
978	ASSERT_CHUNK_FREE(pHeap, pRight);
979	if (pRight->offPrev)
980	{
981	pLeft = (PMMHYPERCHUNKFREE)((char *)pRight + pRight->offPrev);
982	ASSERT_CHUNK_FREE(pHeap, pLeft);
983	}
984	}
985	}
986	if (pLeft == pFree)
987	{
988	AssertMsgFailed(("Freed twice! pv=%p (pChunk=%p)\n", pChunk + 1, pChunk));
989	return VERR_INVALID_POINTER;
990	}
991	pChunk->offStat = 0;
992
993	/*
994	* Head free chunk list?
995	*/
996	if (!pLeft)
997	{
998	MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_FREE);
999	pFree->offPrev = 0;
1000	pHeap->offFreeHead = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
1001	if (pRight)
1002	{
1003	pFree->offNext = (uintptr_t)pRight - (uintptr_t)pFree;
1004	pRight->offPrev = -(int32_t)pFree->offNext;
1005	}
1006	else
1007	{
1008	pFree->offNext = 0;
1009	pHeap->offFreeTail = pHeap->offFreeHead;
1010	}
1011	Log3(("mmHyperFree: Inserted %p at head of free chain.\n", pFree));
1012	}
1013	else
1014	{
1015	/*
1016	* Can we merge with left hand free chunk?
1017	*/
1018	if ((char )pLeft + pLeft->core.offNext == (char )pFree)
1019	{
1020	if (pFree->core.offNext)
1021	{
1022	pLeft->core.offNext = pLeft->core.offNext + pFree->core.offNext;
1023	MMHYPERCHUNK_SET_OFFPREV(((PMMHYPERCHUNK)((char *)pLeft + pLeft->core.offNext)), -(int32_t)pLeft->core.offNext);
1024	}
1025	else
1026	pLeft->core.offNext = 0;
1027	pFree = pLeft;
1028	Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pLeft->cb, -(int32_t)pLeft->cb));
1029	pHeap->cbFree -= pLeft->cb;
1030	Log3(("mmHyperFree: Merging %p into %p (cb=%d).\n", pFree, pLeft, pLeft->cb));
1031	}
1032	/*
1033	* No, just link it into the free list then.
1034	*/
1035	else
1036	{
1037	MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_FREE);
1038	pFree->offPrev = (uintptr_t)pLeft - (uintptr_t)pFree;
1039	pLeft->offNext = -pFree->offPrev;
1040	if (pRight)
1041	{
1042	pFree->offNext = (uintptr_t)pRight - (uintptr_t)pFree;
1043	pRight->offPrev = -(int32_t)pFree->offNext;
1044	}
1045	else
1046	{
1047	pFree->offNext = 0;
1048	pHeap->offFreeTail = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
1049	}
1050	Log3(("mmHyperFree: Inserted %p after %p in free list.\n", pFree, pLeft));
1051	}
1052	}
1053
1054	/*
1055	* Can we merge with right hand free chunk?
1056	*/
1057	if (pRight && (char )pRight == (char )pFree + pFree->core.offNext)
1058	{
1059	/* core */
1060	if (pRight->core.offNext)
1061	{
1062	pFree->core.offNext += pRight->core.offNext;
1063	PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pFree + pFree->core.offNext);
1064	MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pFree->core.offNext);
1065	ASSERT_CHUNK(pHeap, pNext);
1066	}
1067	else
1068	pFree->core.offNext = 0;
1069
1070	/* free */
1071	if (pRight->offNext)
1072	{
1073	pFree->offNext += pRight->offNext;
1074	((PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext))->offPrev = -(int32_t)pFree->offNext;
1075	}
1076	else
1077	{
1078	pFree->offNext = 0;
1079	pHeap->offFreeTail = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
1080	}
1081	Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pRight->cb, -(int32_t)pRight->cb));
1082	pHeap->cbFree -= pRight->cb;
1083	Log3(("mmHyperFree: Merged %p (cb=%d) into %p.\n", pRight, pRight->cb, pFree));
1084	}
1085
1086	/* calculate the size. */
1087	if (pFree->core.offNext)
1088	pFree->cb = pFree->core.offNext - sizeof(MMHYPERCHUNK);
1089	else
1090	pFree->cb = pHeap->offPageAligned - ((uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap)) - sizeof(MMHYPERCHUNK);
1091	Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree + pFree->cb, pFree->cb));
1092	pHeap->cbFree += pFree->cb;
1093	ASSERT_CHUNK_FREE(pHeap, pFree);
1094
1095	#ifdef MMHYPER_HEAP_STRICT
1096	mmHyperHeapCheck(pHeap);
1097	#endif
1098	return VINF_SUCCESS;
1099	}
1100
1101
1102	#if defined(DEBUG) \|\| defined(MMHYPER_HEAP_STRICT)
1103	/**
1104	* Dumps a heap chunk to the log.
1105	*
1106	* @param pHeap Pointer to the heap.
1107	* @param pCur Pointer to the chunk.
1108	*/
1109	static void mmHyperHeapDumpOne(PMMHYPERHEAP pHeap, PMMHYPERCHUNKFREE pCur)
1110	{
1111	if (MMHYPERCHUNK_ISUSED(&pCur->core))
1112	{
1113	if (pCur->core.offStat)
1114	{
1115	PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pCur + pCur->core.offStat);
1116	const char *pszSelf = pCur->core.offStat == sizeof(MMHYPERCHUNK) ? " stat record" : "";
1117	#ifdef IN_RING3
1118	Log(("%p %06x USED offNext=%06x offPrev=-%06x %s%s\n",
1119	pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1120	pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1121	mmGetTagName((MMTAG)pStat->Core.Key), pszSelf));
1122	#else
1123	Log(("%p %06x USED offNext=%06x offPrev=-%06x %d%s\n",
1124	pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1125	pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1126	(MMTAG)pStat->Core.Key, pszSelf));
1127	#endif
1128	}
1129	else
1130	Log(("%p %06x USED offNext=%06x offPrev=-%06x\n",
1131	pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1132	pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core)));
1133	}
1134	else
1135	Log(("%p %06x FREE offNext=%06x offPrev=-%06x : cb=%06x offNext=%06x offPrev=-%06x\n",
1136	pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1137	pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core), pCur->cb, pCur->offNext, pCur->offPrev));
1138	}
1139	#endif /* DEBUG \|\| MMHYPER_HEAP_STRICT */
1140
1141
1142	#ifdef MMHYPER_HEAP_STRICT
1143	/**
1144	* Internal consistency check.
1145	*/
1146	static void mmHyperHeapCheck(PMMHYPERHEAP pHeap)
1147	{
1148	PMMHYPERCHUNKFREE pPrev = NULL;
1149	PMMHYPERCHUNKFREE pCur = (PMMHYPERCHUNKFREE)pHeap->CTX_SUFF(pbHeap);
1150	for (;;)
1151	{
1152	if (MMHYPERCHUNK_ISUSED(&pCur->core))
1153	ASSERT_CHUNK_USED(pHeap, &pCur->core);
1154	else
1155	ASSERT_CHUNK_FREE(pHeap, pCur);
1156	if (pPrev)
1157	AssertMsg((int32_t)pPrev->core.offNext == -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1158	("pPrev->core.offNext=%d offPrev=%d\n", pPrev->core.offNext, MMHYPERCHUNK_GET_OFFPREV(&pCur->core)));
1159
1160	# ifdef MMHYPER_HEAP_STRICT_FENCE
1161	uint32_t off = (uint8_t *)pCur - pHeap->CTX_SUFF(pbHeap);
1162	if ( MMHYPERCHUNK_ISUSED(&pCur->core)
1163	&& off < pHeap->offPageAligned)
1164	{
1165	uint32_t cbCur = pCur->core.offNext
1166	? pCur->core.offNext
1167	: pHeap->cbHeap - off;
1168	uint32_t pu32End = ((uint32_t )((uint8_t *)pCur + cbCur));
1169	uint32_t cbFence = pu32End[-1];
1170	if (RT_UNLIKELY( cbFence >= cbCur - sizeof(*pCur)
1171	\|\| cbFence < MMHYPER_HEAP_STRICT_FENCE_SIZE))
1172	{
1173	mmHyperHeapDumpOne(pHeap, pCur);
1174	Assert(cbFence < cbCur - sizeof(*pCur));
1175	Assert(cbFence >= MMHYPER_HEAP_STRICT_FENCE_SIZE);
1176	}
1177
1178	uint32_t pu32Bad = ASMMemIsAllU32((uint8_t )pu32End - cbFence, cbFence - sizeof(uint32_t), MMHYPER_HEAP_STRICT_FENCE_U32);
1179	if (RT_UNLIKELY(pu32Bad))
1180	{
1181	mmHyperHeapDumpOne(pHeap, pCur);
1182	Assert(!pu32Bad);
1183	}
1184	}
1185	# endif
1186
1187	/* next */
1188	if (!pCur->core.offNext)
1189	break;
1190	pPrev = pCur;
1191	pCur = (PMMHYPERCHUNKFREE)((char *)pCur + pCur->core.offNext);
1192	}
1193	}
1194	#endif
1195
1196
1197	/**
1198	* Performs consistency checks on the heap if MMHYPER_HEAP_STRICT was
1199	* defined at build time.
1200	*
1201	* @param pVM Pointer to the VM.
1202	*/
1203	VMMDECL(void) MMHyperHeapCheck(PVM pVM)
1204	{
1205	#ifdef MMHYPER_HEAP_STRICT
1206	int rc;
1207
1208	rc = mmHyperLock(pVM);
1209	AssertRC(rc);
1210	mmHyperHeapCheck(pVM->mm.s.CTX_SUFF(pHyperHeap));
1211	mmHyperUnlock(pVM);
1212	#endif
1213	}
1214
1215
1216	#ifdef DEBUG
1217	/**
1218	* Dumps the hypervisor heap to Log.
1219	* @param pVM Pointer to the VM.
1220	*/
1221	VMMDECL(void) MMHyperHeapDump(PVM pVM)
1222	{
1223	Log(("MMHyperHeapDump: * heap dump - start *\n"));
1224	PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
1225	PMMHYPERCHUNKFREE pCur = (PMMHYPERCHUNKFREE)pHeap->CTX_SUFF(pbHeap);
1226	for (;;)
1227	{
1228	mmHyperHeapDumpOne(pHeap, pCur);
1229
1230	/* next */
1231	if (!pCur->core.offNext)
1232	break;
1233	pCur = (PMMHYPERCHUNKFREE)((char *)pCur + pCur->core.offNext);
1234	}
1235	Log(("MMHyperHeapDump: * heap dump - end *\n"));
1236	}
1237	#endif
1238
1239
1240	/**
1241	* Query the amount of free memory in the hypervisor heap.
1242	*
1243	* @returns Number of free bytes in the hypervisor heap.
1244	*/
1245	VMMDECL(size_t) MMHyperHeapGetFreeSize(PVM pVM)
1246	{
1247	return pVM->mm.s.CTX_SUFF(pHyperHeap)->cbFree;
1248	}
1249
1250	/**
1251	* Query the size the hypervisor heap.
1252	*
1253	* @returns The size of the hypervisor heap in bytes.
1254	*/
1255	VMMDECL(size_t) MMHyperHeapGetSize(PVM pVM)
1256	{
1257	return pVM->mm.s.CTX_SUFF(pHyperHeap)->cbHeap;
1258	}
1259
1260
1261	/**
1262	* Query the address and size the hypervisor memory area.
1263	*
1264	* @returns Base address of the hypervisor area.
1265	* @param pVM Pointer to the VM.
1266	* @param pcb Where to store the size of the hypervisor area. (out)
1267	*/
1268	VMMDECL(RTGCPTR) MMHyperGetArea(PVM pVM, size_t *pcb)
1269	{
1270	if (pcb)
1271	*pcb = pVM->mm.s.cbHyperArea;
1272	return pVM->mm.s.pvHyperAreaGC;
1273	}
1274
1275
1276	/**
1277	* Checks if an address is within the hypervisor memory area.
1278	*
1279	* @returns true if inside.
1280	* @returns false if outside.
1281	* @param pVM Pointer to the VM.
1282	* @param GCPtr The pointer to check.
1283	*/
1284	VMMDECL(bool) MMHyperIsInsideArea(PVM pVM, RTGCPTR GCPtr)
1285	{
1286	return (RTGCUINTPTR)GCPtr - (RTGCUINTPTR)pVM->mm.s.pvHyperAreaGC < pVM->mm.s.cbHyperArea;
1287	}
1288

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/MMAllHyper.cpp@ 49893

Download in other formats: