MMAllHyper.cpp@ 51271

Last change on this file since 51271 was 51271, checked in by vboxsync, 10 years ago
VMM: Implemented hyper heap realloc. and adjusted CPUM CpuId arrays and MSR ranges handling to optionally work with the hyper heap.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 47.8 KB

Line
1	/* $Id: MMAllHyper.cpp 51271 2014-05-16 12:08:42Z vboxsync $ */
2	/** @file
3	* MM - Memory Manager - Hypervisor Memory Area, All Contexts.
4	*/
5
6	/*
7	* Copyright (C) 2006-2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_MM_HYPER_HEAP
23	#include <VBox/vmm/mm.h>
24	#include <VBox/vmm/stam.h>
25	#include "MMInternal.h"
26	#include <VBox/vmm/vm.h>
27
28	#include <VBox/err.h>
29	#include <VBox/param.h>
30	#include <iprt/assert.h>
31	#include <VBox/log.h>
32	#include <iprt/asm.h>
33	#include <iprt/string.h>
34
35
36	/*******************************************************************************
37	* Defined Constants And Macros *
38	*******************************************************************************/
39	#define ASSERT_L(u1, u2) AssertMsg((u1) < (u2), ("u1=%#x u2=%#x\n", u1, u2))
40	#define ASSERT_LE(u1, u2) AssertMsg((u1) <= (u2), ("u1=%#x u2=%#x\n", u1, u2))
41	#define ASSERT_GE(u1, u2) AssertMsg((u1) >= (u2), ("u1=%#x u2=%#x\n", u1, u2))
42	#define ASSERT_ALIGN(u1) AssertMsg(!((u1) & (MMHYPER_HEAP_ALIGN_MIN - 1)), ("u1=%#x (%d)\n", u1, u1))
43
44	#define ASSERT_OFFPREV(pHeap, pChunk) \
45	do { Assert(MMHYPERCHUNK_GET_OFFPREV(pChunk) <= 0); \
46	Assert(MMHYPERCHUNK_GET_OFFPREV(pChunk) >= (intptr_t)(pHeap)->CTX_SUFF(pbHeap) - (intptr_t)(pChunk)); \
47	AssertMsg( MMHYPERCHUNK_GET_OFFPREV(pChunk) != 0 \
48	\|\| (uint8_t *)(pChunk) == (pHeap)->CTX_SUFF(pbHeap), \
49	("pChunk=%p pvHyperHeap=%p\n", (pChunk), (pHeap)->CTX_SUFF(pbHeap))); \
50	} while (0)
51
52	#define ASSERT_OFFNEXT(pHeap, pChunk) \
53	do { ASSERT_ALIGN((pChunk)->offNext); \
54	ASSERT_L((pChunk)->offNext, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
55	} while (0)
56
57	#define ASSERT_OFFHEAP(pHeap, pChunk) \
58	do { Assert((pChunk)->offHeap); \
59	AssertMsg((PMMHYPERHEAP)((pChunk)->offHeap + (uintptr_t)pChunk) == (pHeap), \
60	("offHeap=%RX32 pChunk=%p pHeap=%p\n", (pChunk)->offHeap, (pChunk), (pHeap))); \
61	Assert((pHeap)->u32Magic == MMHYPERHEAP_MAGIC); \
62	} while (0)
63
64	#ifdef VBOX_WITH_STATISTICS
65	#define ASSERT_OFFSTAT(pHeap, pChunk) \
66	do { if (MMHYPERCHUNK_ISFREE(pChunk)) \
67	Assert(!(pChunk)->offStat); \
68	else if ((pChunk)->offStat) \
69	{ \
70	Assert((pChunk)->offStat); \
71	AssertMsg(!((pChunk)->offStat & (MMHYPER_HEAP_ALIGN_MIN - 1)), ("offStat=%RX32\n", (pChunk)->offStat)); \
72	uintptr_t uPtr = (uintptr_t)(pChunk)->offStat + (uintptr_t)pChunk; NOREF(uPtr); \
73	AssertMsg(uPtr - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) < (pHeap)->offPageAligned, \
74	("%p - %p < %RX32\n", uPtr, (pHeap)->CTX_SUFF(pbHeap), (pHeap)->offPageAligned)); \
75	} \
76	} while (0)
77	#else
78	#define ASSERT_OFFSTAT(pHeap, pChunk) \
79	do { Assert(!(pChunk)->offStat); \
80	} while (0)
81	#endif
82
83	#define ASSERT_CHUNK(pHeap, pChunk) \
84	do { ASSERT_OFFNEXT(pHeap, pChunk); \
85	ASSERT_OFFPREV(pHeap, pChunk); \
86	ASSERT_OFFHEAP(pHeap, pChunk); \
87	ASSERT_OFFSTAT(pHeap, pChunk); \
88	} while (0)
89	#define ASSERT_CHUNK_USED(pHeap, pChunk) \
90	do { ASSERT_OFFNEXT(pHeap, pChunk); \
91	ASSERT_OFFPREV(pHeap, pChunk); \
92	Assert(MMHYPERCHUNK_ISUSED(pChunk)); \
93	} while (0)
94
95	#define ASSERT_FREE_OFFPREV(pHeap, pChunk) \
96	do { ASSERT_ALIGN((pChunk)->offPrev); \
97	ASSERT_GE(((pChunk)->offPrev & (MMHYPER_HEAP_ALIGN_MIN - 1)), (intptr_t)(pHeap)->CTX_SUFF(pbHeap) - (intptr_t)(pChunk)); \
98	Assert((pChunk)->offPrev != MMHYPERCHUNK_GET_OFFPREV(&(pChunk)->core) \|\| !(pChunk)->offPrev); \
99	AssertMsg( (pChunk)->offPrev \
100	\|\| (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) == (pHeap)->offFreeHead, \
101	("pChunk=%p offChunk=%#x offFreeHead=%#x\n", (pChunk), (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap),\
102	(pHeap)->offFreeHead)); \
103	} while (0)
104
105	#define ASSERT_FREE_OFFNEXT(pHeap, pChunk) \
106	do { ASSERT_ALIGN((pChunk)->offNext); \
107	ASSERT_L((pChunk)->offNext, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
108	Assert((pChunk)->offNext != (pChunk)->core.offNext \|\| !(pChunk)->offNext); \
109	AssertMsg( (pChunk)->offNext \
110	\|\| (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) == (pHeap)->offFreeTail, \
111	("pChunk=%p offChunk=%#x offFreeTail=%#x\n", (pChunk), (uintptr_t)(pChunk) - (uintptr_t)(pHeap)->CTX_SUFF(pbHeap), \
112	(pHeap)->offFreeTail)); \
113	} while (0)
114
115	#define ASSERT_FREE_CB(pHeap, pChunk) \
116	do { ASSERT_ALIGN((pChunk)->cb); \
117	Assert((pChunk)->cb > 0); \
118	if ((pChunk)->core.offNext) \
119	AssertMsg((pChunk)->cb == ((pChunk)->core.offNext - sizeof(MMHYPERCHUNK)), \
120	("cb=%d offNext=%d\n", (pChunk)->cb, (pChunk)->core.offNext)); \
121	else \
122	ASSERT_LE((pChunk)->cb, (uintptr_t)(pHeap)->CTX_SUFF(pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
123	} while (0)
124
125	#define ASSERT_CHUNK_FREE(pHeap, pChunk) \
126	do { ASSERT_CHUNK(pHeap, &(pChunk)->core); \
127	Assert(MMHYPERCHUNK_ISFREE(pChunk)); \
128	ASSERT_FREE_OFFNEXT(pHeap, pChunk); \
129	ASSERT_FREE_OFFPREV(pHeap, pChunk); \
130	ASSERT_FREE_CB(pHeap, pChunk); \
131	} while (0)
132
133
134	/*******************************************************************************
135	* Internal Functions *
136	*******************************************************************************/
137	static PMMHYPERCHUNK mmHyperAllocChunk(PMMHYPERHEAP pHeap, uint32_t cb, unsigned uAlignment);
138	static void *mmHyperAllocPages(PMMHYPERHEAP pHeap, uint32_t cb);
139	#ifdef VBOX_WITH_STATISTICS
140	static PMMHYPERSTAT mmHyperStat(PMMHYPERHEAP pHeap, MMTAG enmTag);
141	#ifdef IN_RING3
142	static void mmR3HyperStatRegisterOne(PVM pVM, PMMHYPERSTAT pStat);
143	#endif
144	#endif
145	static int mmHyperFree(PMMHYPERHEAP pHeap, PMMHYPERCHUNK pChunk);
146	#ifdef MMHYPER_HEAP_STRICT
147	static void mmHyperHeapCheck(PMMHYPERHEAP pHeap);
148	#endif
149
150
151
152	/**
153	* Locks the hypervisor heap.
154	* This might call back to Ring-3 in order to deal with lock contention in GC and R3.
155	*
156	* @param pVM Pointer to the VM.
157	*/
158	static int mmHyperLock(PVM pVM)
159	{
160	PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
161
162	#ifdef IN_RING3
163	if (!PDMCritSectIsInitialized(&pHeap->Lock))
164	return VINF_SUCCESS; /* early init */
165	#else
166	Assert(PDMCritSectIsInitialized(&pHeap->Lock));
167	#endif
168	int rc = PDMCritSectEnter(&pHeap->Lock, VERR_SEM_BUSY);
169	#if defined(IN_RC) \|\| defined(IN_RING0)
170	if (rc == VERR_SEM_BUSY)
171	rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_MMHYPER_LOCK, 0);
172	#endif
173	AssertRC(rc);
174	return rc;
175	}
176
177
178	/**
179	* Unlocks the hypervisor heap.
180	*
181	* @param pVM Pointer to the VM.
182	*/
183	static void mmHyperUnlock(PVM pVM)
184	{
185	PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
186
187	#ifdef IN_RING3
188	if (!PDMCritSectIsInitialized(&pHeap->Lock))
189	return; /* early init */
190	#endif
191	Assert(PDMCritSectIsInitialized(&pHeap->Lock));
192	PDMCritSectLeave(&pHeap->Lock);
193	}
194
195	/**
196	* Allocates memory in the Hypervisor (RC VMM) area.
197	* The returned memory is of course zeroed.
198	*
199	* @returns VBox status code.
200	* @param pVM Pointer to the VM.
201	* @param cb Number of bytes to allocate.
202	* @param uAlignment Required memory alignment in bytes.
203	* Values are 0,8,16,32,64 and PAGE_SIZE.
204	* 0 -> default alignment, i.e. 8 bytes.
205	* @param enmTag The statistics tag.
206	* @param ppv Where to store the address to the allocated
207	* memory.
208	*/
209	static int mmHyperAllocInternal(PVM pVM, size_t cb, unsigned uAlignment, MMTAG enmTag, void **ppv)
210	{
211	AssertMsg(cb >= 8, ("Hey! Do you really mean to allocate less than 8 bytes?! cb=%d\n", cb));
212
213	/*
214	* Validate input and adjust it to reasonable values.
215	*/
216	if (!uAlignment \|\| uAlignment < MMHYPER_HEAP_ALIGN_MIN)
217	uAlignment = MMHYPER_HEAP_ALIGN_MIN;
218	uint32_t cbAligned;
219	switch (uAlignment)
220	{
221	case 8:
222	case 16:
223	case 32:
224	case 64:
225	cbAligned = RT_ALIGN_32(cb, MMHYPER_HEAP_ALIGN_MIN);
226	if (!cbAligned \|\| cbAligned < cb)
227	{
228	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
229	AssertMsgFailed(("Nice try.\n"));
230	return VERR_INVALID_PARAMETER;
231	}
232	break;
233
234	case PAGE_SIZE:
235	AssertMsg(RT_ALIGN_32(cb, PAGE_SIZE) == cb, ("The size isn't page aligned. (cb=%#x)\n", cb));
236	cbAligned = RT_ALIGN_32(cb, PAGE_SIZE);
237	if (!cbAligned)
238	{
239	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
240	AssertMsgFailed(("Nice try.\n"));
241	return VERR_INVALID_PARAMETER;
242	}
243	break;
244
245	default:
246	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
247	AssertMsgFailed(("Invalid alignment %u\n", uAlignment));
248	return VERR_INVALID_PARAMETER;
249	}
250
251
252	/*
253	* Get heap and statisticsStatistics.
254	*/
255	PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
256	#ifdef VBOX_WITH_STATISTICS
257	PMMHYPERSTAT pStat = mmHyperStat(pHeap, enmTag);
258	if (!pStat)
259	{
260	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_MM_HYPER_NO_MEMORY\n", cb, uAlignment));
261	AssertMsgFailed(("Failed to allocate statistics!\n"));
262	return VERR_MM_HYPER_NO_MEMORY;
263	}
264	#endif
265	if (uAlignment < PAGE_SIZE)
266	{
267	/*
268	* Allocate a chunk.
269	*/
270	PMMHYPERCHUNK pChunk = mmHyperAllocChunk(pHeap, cbAligned, uAlignment);
271	if (pChunk)
272	{
273	#ifdef VBOX_WITH_STATISTICS
274	const uint32_t cbChunk = pChunk->offNext
275	? pChunk->offNext
276	: pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t *)pChunk;
277	pStat->cbAllocated += (uint32_t)cbChunk;
278	pStat->cbCurAllocated += (uint32_t)cbChunk;
279	if (pStat->cbCurAllocated > pStat->cbMaxAllocated)
280	pStat->cbMaxAllocated = pStat->cbCurAllocated;
281	pStat->cAllocations++;
282	pChunk->offStat = (uintptr_t)pStat - (uintptr_t)pChunk;
283	#else
284	pChunk->offStat = 0;
285	#endif
286	void *pv = pChunk + 1;
287	*ppv = pv;
288	ASMMemZero32(pv, cbAligned);
289	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VINF_SUCCESS and *ppv=%p\n", cb, uAlignment, pv));
290	return VINF_SUCCESS;
291	}
292	}
293	else
294	{
295	/*
296	* Allocate page aligned memory.
297	*/
298	void *pv = mmHyperAllocPages(pHeap, cbAligned);
299	if (pv)
300	{
301	#ifdef VBOX_WITH_STATISTICS
302	pStat->cbAllocated += cbAligned;
303	pStat->cbCurAllocated += cbAligned;
304	if (pStat->cbCurAllocated > pStat->cbMaxAllocated)
305	pStat->cbMaxAllocated = pStat->cbCurAllocated;
306	pStat->cAllocations++;
307	#endif
308	*ppv = pv;
309	/* ASMMemZero32(pv, cbAligned); - not required since memory is alloc-only and SUPR3PageAlloc zeros it. */
310	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VINF_SUCCESS and *ppv=%p\n", cb, uAlignment, ppv));
311	return VINF_SUCCESS;
312	}
313	}
314
315	#ifdef VBOX_WITH_STATISTICS
316	pStat->cAllocations++;
317	pStat->cFailures++;
318	#endif
319	Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_MM_HYPER_NO_MEMORY\n", cb, uAlignment));
320	AssertMsgFailed(("Failed to allocate %d bytes!\n", cb));
321	return VERR_MM_HYPER_NO_MEMORY;
322	}
323
324
325	/**
326	* Wrapper for mmHyperAllocInternal
327	*/
328	VMMDECL(int) MMHyperAlloc(PVM pVM, size_t cb, unsigned uAlignment, MMTAG enmTag, void **ppv)
329	{
330	int rc = mmHyperLock(pVM);
331	AssertRCReturn(rc, rc);
332
333	LogFlow(("MMHyperAlloc %x align=%x tag=%s\n", cb, uAlignment, mmGetTagName(enmTag)));
334
335	rc = mmHyperAllocInternal(pVM, cb, uAlignment, enmTag, ppv);
336
337	mmHyperUnlock(pVM);
338	return rc;
339	}
340
341
342	/**
343	* Duplicates a block of memory.
344	*
345	* @returns VBox status code.
346	* @param pVM Pointer to the VM.
347	* @param pvSrc The source memory block to copy from.
348	* @param cb Size of the source memory block.
349	* @param uAlignment Required memory alignment in bytes.
350	* Values are 0,8,16,32,64 and PAGE_SIZE.
351	* 0 -> default alignment, i.e. 8 bytes.
352	* @param enmTag The statistics tag.
353	* @param ppv Where to store the address to the allocated
354	* memory.
355	*/
356	VMMDECL(int) MMHyperDupMem(PVM pVM, const void pvSrc, size_t cb, unsigned uAlignment, MMTAG enmTag, void *ppv)
357	{
358	int rc = MMHyperAlloc(pVM, cb, uAlignment, enmTag, ppv);
359	if (RT_SUCCESS(rc))
360	memcpy(*ppv, pvSrc, cb);
361	return rc;
362	}
363
364
365	/**
366	* Allocates a chunk of memory from the specified heap.
367	* The caller validates the parameters of this request.
368	*
369	* @returns Pointer to the allocated chunk.
370	* @returns NULL on failure.
371	* @param pHeap The heap.
372	* @param cb Size of the memory block to allocate.
373	* @param uAlignment The alignment specifications for the allocated block.
374	* @internal
375	*/
376	static PMMHYPERCHUNK mmHyperAllocChunk(PMMHYPERHEAP pHeap, uint32_t cb, unsigned uAlignment)
377	{
378	Log3(("mmHyperAllocChunk: Enter cb=%#x uAlignment=%#x\n", cb, uAlignment));
379	#ifdef MMHYPER_HEAP_STRICT
380	mmHyperHeapCheck(pHeap);
381	#endif
382	#ifdef MMHYPER_HEAP_STRICT_FENCE
383	uint32_t cbFence = RT_MAX(MMHYPER_HEAP_STRICT_FENCE_SIZE, uAlignment);
384	cb += cbFence;
385	#endif
386
387	/*
388	* Check if there are any free chunks. (NIL_OFFSET use/not-use forces this check)
389	*/
390	if (pHeap->offFreeHead == NIL_OFFSET)
391	return NULL;
392
393	/*
394	* Small alignments - from the front of the heap.
395	*
396	* Must split off free chunks at the end to prevent messing up the
397	* last free node which we take the page aligned memory from the top of.
398	*/
399	PMMHYPERCHUNK pRet = NULL;
400	PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)((char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeHead);
401	while (pFree)
402	{
403	ASSERT_CHUNK_FREE(pHeap, pFree);
404	if (pFree->cb >= cb)
405	{
406	unsigned offAlign = (uintptr_t)(&pFree->core + 1) & (uAlignment - 1);
407	if (offAlign)
408	offAlign = uAlignment - offAlign;
409	if (!offAlign \|\| pFree->cb - offAlign >= cb)
410	{
411	Log3(("mmHyperAllocChunk: Using pFree=%p pFree->cb=%d offAlign=%d\n", pFree, pFree->cb, offAlign));
412
413	/*
414	* Adjust the node in front.
415	* Because of multiple alignments we need to special case allocation of the first block.
416	*/
417	if (offAlign)
418	{
419	MMHYPERCHUNKFREE Free = *pFree;
420	if (MMHYPERCHUNK_GET_OFFPREV(&pFree->core))
421	{
422	/* just add a bit of memory to it. */
423	PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + MMHYPERCHUNK_GET_OFFPREV(&Free.core));
424	pPrev->core.offNext += offAlign;
425	AssertMsg(!MMHYPERCHUNK_ISFREE(&pPrev->core), ("Impossible!\n"));
426	Log3(("mmHyperAllocChunk: Added %d bytes to %p\n", offAlign, pPrev));
427	}
428	else
429	{
430	/* make new head node, mark it USED for simplicity. */
431	PMMHYPERCHUNK pPrev = (PMMHYPERCHUNK)pHeap->CTX_SUFF(pbHeap);
432	Assert(pPrev == &pFree->core);
433	pPrev->offPrev = 0;
434	MMHYPERCHUNK_SET_TYPE(pPrev, MMHYPERCHUNK_FLAGS_USED);
435	pPrev->offNext = offAlign;
436	Log3(("mmHyperAllocChunk: Created new first node of %d bytes\n", offAlign));
437
438	}
439	Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - offAlign, -(int)offAlign));
440	pHeap->cbFree -= offAlign;
441
442	/* Recreate pFree node and adjusting everything... */
443	pFree = (PMMHYPERCHUNKFREE)((char *)pFree + offAlign);
444	*pFree = Free;
445
446	pFree->cb -= offAlign;
447	if (pFree->core.offNext)
448	{
449	pFree->core.offNext -= offAlign;
450	PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pFree + pFree->core.offNext);
451	MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pFree->core.offNext);
452	ASSERT_CHUNK(pHeap, pNext);
453	}
454	if (MMHYPERCHUNK_GET_OFFPREV(&pFree->core))
455	MMHYPERCHUNK_SET_OFFPREV(&pFree->core, MMHYPERCHUNK_GET_OFFPREV(&pFree->core) - offAlign);
456
457	if (pFree->offNext)
458	{
459	pFree->offNext -= offAlign;
460	PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext);
461	pNext->offPrev = -(int32_t)pFree->offNext;
462	ASSERT_CHUNK_FREE(pHeap, pNext);
463	}
464	else
465	pHeap->offFreeTail += offAlign;
466	if (pFree->offPrev)
467	{
468	pFree->offPrev -= offAlign;
469	PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
470	pPrev->offNext = -pFree->offPrev;
471	ASSERT_CHUNK_FREE(pHeap, pPrev);
472	}
473	else
474	pHeap->offFreeHead += offAlign;
475	pFree->core.offHeap = (uintptr_t)pHeap - (uintptr_t)pFree;
476	pFree->core.offStat = 0;
477	ASSERT_CHUNK_FREE(pHeap, pFree);
478	Log3(("mmHyperAllocChunk: Realigned pFree=%p\n", pFree));
479	}
480
481	/*
482	* Split off a new FREE chunk?
483	*/
484	if (pFree->cb >= cb + RT_ALIGN(sizeof(MMHYPERCHUNKFREE), MMHYPER_HEAP_ALIGN_MIN))
485	{
486	/*
487	* Move the FREE chunk up to make room for the new USED chunk.
488	*/
489	const int off = cb + sizeof(MMHYPERCHUNK);
490	PMMHYPERCHUNKFREE pNew = (PMMHYPERCHUNKFREE)((char *)&pFree->core + off);
491	pNew = pFree;
492	pNew->cb -= off;
493	if (pNew->core.offNext)
494	{
495	pNew->core.offNext -= off;
496	PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pNew + pNew->core.offNext);
497	MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pNew->core.offNext);
498	ASSERT_CHUNK(pHeap, pNext);
499	}
500	pNew->core.offPrev = -off;
501	MMHYPERCHUNK_SET_TYPE(pNew, MMHYPERCHUNK_FLAGS_FREE);
502
503	if (pNew->offNext)
504	{
505	pNew->offNext -= off;
506	PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pNew + pNew->offNext);
507	pNext->offPrev = -(int32_t)pNew->offNext;
508	ASSERT_CHUNK_FREE(pHeap, pNext);
509	}
510	else
511	pHeap->offFreeTail += off;
512	if (pNew->offPrev)
513	{
514	pNew->offPrev -= off;
515	PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pNew + pNew->offPrev);
516	pPrev->offNext = -pNew->offPrev;
517	ASSERT_CHUNK_FREE(pHeap, pPrev);
518	}
519	else
520	pHeap->offFreeHead += off;
521	pNew->core.offHeap = (uintptr_t)pHeap - (uintptr_t)pNew;
522	pNew->core.offStat = 0;
523	ASSERT_CHUNK_FREE(pHeap, pNew);
524
525	/*
526	* Update the old FREE node making it a USED node.
527	*/
528	pFree->core.offNext = off;
529	MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_USED);
530
531
532	Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree,
533	pHeap->cbFree - (cb + sizeof(MMHYPERCHUNK)), -(int)(cb + sizeof(MMHYPERCHUNK))));
534	pHeap->cbFree -= (uint32_t)(cb + sizeof(MMHYPERCHUNK));
535	pRet = &pFree->core;
536	ASSERT_CHUNK(pHeap, &pFree->core);
537	Log3(("mmHyperAllocChunk: Created free chunk pNew=%p cb=%d\n", pNew, pNew->cb));
538	}
539	else
540	{
541	/*
542	* Link out of free list.
543	*/
544	if (pFree->offNext)
545	{
546	PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext);
547	if (pFree->offPrev)
548	{
549	pNext->offPrev += pFree->offPrev;
550	PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
551	pPrev->offNext += pFree->offNext;
552	ASSERT_CHUNK_FREE(pHeap, pPrev);
553	}
554	else
555	{
556	pHeap->offFreeHead += pFree->offNext;
557	pNext->offPrev = 0;
558	}
559	ASSERT_CHUNK_FREE(pHeap, pNext);
560	}
561	else
562	{
563	if (pFree->offPrev)
564	{
565	pHeap->offFreeTail += pFree->offPrev;
566	PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
567	pPrev->offNext = 0;
568	ASSERT_CHUNK_FREE(pHeap, pPrev);
569	}
570	else
571	{
572	pHeap->offFreeHead = NIL_OFFSET;
573	pHeap->offFreeTail = NIL_OFFSET;
574	}
575	}
576
577	Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree,
578	pHeap->cbFree - pFree->cb, -(int32_t)pFree->cb));
579	pHeap->cbFree -= pFree->cb;
580	MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_USED);
581	pRet = &pFree->core;
582	ASSERT_CHUNK(pHeap, &pFree->core);
583	Log3(("mmHyperAllocChunk: Converted free chunk %p to used chunk.\n", pFree));
584	}
585	Log3(("mmHyperAllocChunk: Returning %p\n", pRet));
586	break;
587	}
588	}
589
590	/* next */
591	pFree = pFree->offNext ? (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext) : NULL;
592	}
593
594	#ifdef MMHYPER_HEAP_STRICT_FENCE
595	uint32_t pu32End = (uint32_t )((uint8_t *)(pRet + 1) + cb);
596	uint32_t *pu32EndReal = pRet->offNext
597	? (uint32_t )((uint8_t )pRet + pRet->offNext)
598	: (uint32_t *)(pHeap->CTX_SUFF(pbHeap) + pHeap->cbHeap);
599	cbFence += (uintptr_t)pu32EndReal - (uintptr_t)pu32End; Assert(!(cbFence & 0x3));
600	ASMMemFill32((uint8_t *)pu32EndReal - cbFence, cbFence, MMHYPER_HEAP_STRICT_FENCE_U32);
601	pu32EndReal[-1] = cbFence;
602	#endif
603	#ifdef MMHYPER_HEAP_STRICT
604	mmHyperHeapCheck(pHeap);
605	#endif
606	return pRet;
607	}
608
609
610	/**
611	* Allocates one or more pages of memory from the specified heap.
612	* The caller validates the parameters of this request.
613	*
614	* @returns Pointer to the allocated chunk.
615	* @returns NULL on failure.
616	* @param pHeap The heap.
617	* @param cb Size of the memory block to allocate.
618	* @internal
619	*/
620	static void *mmHyperAllocPages(PMMHYPERHEAP pHeap, uint32_t cb)
621	{
622	Log3(("mmHyperAllocPages: Enter cb=%#x\n", cb));
623
624	#ifdef MMHYPER_HEAP_STRICT
625	mmHyperHeapCheck(pHeap);
626	#endif
627
628	/*
629	* Check if there are any free chunks. (NIL_OFFSET use/not-use forces this check)
630	*/
631	if (pHeap->offFreeHead == NIL_OFFSET)
632	return NULL;
633
634	/*
635	* Page aligned chunks.
636	*
637	* Page aligned chunks can only be allocated from the last FREE chunk.
638	* This is for reasons of simplicity and fragmentation. Page aligned memory
639	* must also be allocated in page aligned sizes. Page aligned memory cannot
640	* be freed either.
641	*
642	* So, for this to work, the last FREE chunk needs to end on a page aligned
643	* boundary.
644	*/
645	PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)((char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeTail);
646	ASSERT_CHUNK_FREE(pHeap, pFree);
647	if ( (((uintptr_t)(&pFree->core + 1) + pFree->cb) & (PAGE_OFFSET_MASK - 1))
648	\|\| pFree->cb + sizeof(MMHYPERCHUNK) < cb)
649	{
650	Log3(("mmHyperAllocPages: Not enough/no page aligned memory!\n"));
651	return NULL;
652	}
653
654	void *pvRet;
655	if (pFree->cb > cb)
656	{
657	/*
658	* Simple, just cut the top of the free node and return it.
659	*/
660	pFree->cb -= cb;
661	pvRet = (char *)(&pFree->core + 1) + pFree->cb;
662	AssertMsg(RT_ALIGN_P(pvRet, PAGE_SIZE) == pvRet, ("pvRet=%p cb=%#x pFree=%p pFree->cb=%#x\n", pvRet, cb, pFree, pFree->cb));
663	Log3(("mmHyperAllocPages: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - cb, -(int)cb));
664	pHeap->cbFree -= cb;
665	ASSERT_CHUNK_FREE(pHeap, pFree);
666	Log3(("mmHyperAllocPages: Allocated from pFree=%p new pFree->cb=%d\n", pFree, pFree->cb));
667	}
668	else
669	{
670	/*
671	* Unlink the FREE node.
672	*/
673	pvRet = (char *)(&pFree->core + 1) + pFree->cb - cb;
674	Log3(("mmHyperAllocPages: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pFree->cb, -(int32_t)pFree->cb));
675	pHeap->cbFree -= pFree->cb;
676
677	/* a scrap of spare memory (unlikely)? add it to the sprevious chunk. */
678	if (pvRet != (void *)pFree)
679	{
680	AssertMsg(MMHYPERCHUNK_GET_OFFPREV(&pFree->core), ("How the beep did someone manage to allocated up all the heap with page aligned memory?!?\n"));
681	PMMHYPERCHUNK pPrev = (PMMHYPERCHUNK)((char *)pFree + MMHYPERCHUNK_GET_OFFPREV(&pFree->core));
682	pPrev->offNext += (uintptr_t)pvRet - (uintptr_t)pFree;
683	AssertMsg(!MMHYPERCHUNK_ISFREE(pPrev), ("Free bug?\n"));
684	#ifdef VBOX_WITH_STATISTICS
685	PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pPrev + pPrev->offStat);
686	pStat->cbAllocated += (uintptr_t)pvRet - (uintptr_t)pFree;
687	pStat->cbCurAllocated += (uintptr_t)pvRet - (uintptr_t)pFree;
688	#endif
689	Log3(("mmHyperAllocPages: Added %d to %p (page align)\n", (uintptr_t)pvRet - (uintptr_t)pFree, pFree));
690	}
691
692	/* unlink from FREE chain. */
693	if (pFree->offPrev)
694	{
695	pHeap->offFreeTail += pFree->offPrev;
696	((PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev))->offNext = 0;
697	}
698	else
699	{
700	pHeap->offFreeTail = NIL_OFFSET;
701	pHeap->offFreeHead = NIL_OFFSET;
702	}
703	Log3(("mmHyperAllocPages: Unlinked pFree=%d\n", pFree));
704	}
705	pHeap->offPageAligned = (uintptr_t)pvRet - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
706	Log3(("mmHyperAllocPages: Returning %p (page aligned)\n", pvRet));
707
708	#ifdef MMHYPER_HEAP_STRICT
709	mmHyperHeapCheck(pHeap);
710	#endif
711	return pvRet;
712	}
713
714	#ifdef VBOX_WITH_STATISTICS
715
716	/**
717	* Get the statistic record for a tag.
718	*
719	* @returns Pointer to a stat record.
720	* @returns NULL on failure.
721	* @param pHeap The heap.
722	* @param enmTag The tag.
723	*/
724	static PMMHYPERSTAT mmHyperStat(PMMHYPERHEAP pHeap, MMTAG enmTag)
725	{
726	/* try look it up first. */
727	PMMHYPERSTAT pStat = (PMMHYPERSTAT)RTAvloGCPhysGet(&pHeap->HyperHeapStatTree, enmTag);
728	if (!pStat)
729	{
730	/* try allocate a new one */
731	PMMHYPERCHUNK pChunk = mmHyperAllocChunk(pHeap, RT_ALIGN(sizeof(*pStat), MMHYPER_HEAP_ALIGN_MIN), MMHYPER_HEAP_ALIGN_MIN);
732	if (!pChunk)
733	return NULL;
734	pStat = (PMMHYPERSTAT)(pChunk + 1);
735	pChunk->offStat = (uintptr_t)pStat - (uintptr_t)pChunk;
736
737	ASMMemZero32(pStat, sizeof(*pStat));
738	pStat->Core.Key = enmTag;
739	RTAvloGCPhysInsert(&pHeap->HyperHeapStatTree, &pStat->Core);
740	}
741	if (!pStat->fRegistered)
742	{
743	# ifdef IN_RING3
744	mmR3HyperStatRegisterOne(pHeap->pVMR3, pStat);
745	# else
746	/** @todo schedule a R3 action. */
747	# endif
748	}
749	return pStat;
750	}
751
752
753	# ifdef IN_RING3
754	/**
755	* Registers statistics with STAM.
756	*
757	*/
758	static void mmR3HyperStatRegisterOne(PVM pVM, PMMHYPERSTAT pStat)
759	{
760	if (pStat->fRegistered)
761	return;
762	const char *pszTag = mmGetTagName((MMTAG)pStat->Core.Key);
763	STAMR3RegisterF(pVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Number of bytes currently allocated.", "/MM/HyperHeap/%s", pszTag);
764	STAMR3RegisterF(pVM, &pStat->cAllocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of alloc calls.", "/MM/HyperHeap/%s/cAllocations", pszTag);
765	STAMR3RegisterF(pVM, &pStat->cFrees, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of free calls.", "/MM/HyperHeap/%s/cFrees", pszTag);
766	STAMR3RegisterF(pVM, &pStat->cFailures, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of failures.", "/MM/HyperHeap/%s/cFailures", pszTag);
767	STAMR3RegisterF(pVM, &pStat->cbAllocated, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of allocated bytes.", "/MM/HyperHeap/%s/cbAllocated", pszTag);
768	STAMR3RegisterF(pVM, &pStat->cbFreed, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of freed bytes.", "/MM/HyperHeap/%s/cbFreed", pszTag);
769	STAMR3RegisterF(pVM, &pStat->cbMaxAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Max number of bytes allocated at the same time.","/MM/HyperHeap/%s/cbMaxAllocated", pszTag);
770	pStat->fRegistered = true;
771	}
772	# endif /* IN_RING3 */
773
774	#endif /* VBOX_WITH_STATISTICS */
775
776
777	/**
778	* Free memory allocated using MMHyperAlloc().
779	* The caller validates the parameters of this request.
780	*
781	* @returns VBox status code.
782	* @param pVM Pointer to the VM.
783	* @param pv The memory to free.
784	* @remark Try avoid free hyper memory.
785	*/
786	static int mmHyperFreeInternal(PVM pVM, void *pv)
787	{
788	Log2(("MMHyperFree: pv=%p\n", pv));
789	if (!pv)
790	return VINF_SUCCESS;
791	AssertMsgReturn(RT_ALIGN_P(pv, MMHYPER_HEAP_ALIGN_MIN) == pv,
792	("Invalid pointer %p!\n", pv),
793	VERR_INVALID_POINTER);
794
795	/*
796	* Get the heap and stats.
797	* Validate the chunk at the same time.
798	*/
799	PMMHYPERCHUNK pChunk = (PMMHYPERCHUNK)((PMMHYPERCHUNK)pv - 1);
800
801	AssertMsgReturn( (uintptr_t)pChunk + pChunk->offNext >= (uintptr_t)pChunk
802	\|\| RT_ALIGN_32(pChunk->offNext, MMHYPER_HEAP_ALIGN_MIN) != pChunk->offNext,
803	("%p: offNext=%#RX32\n", pv, pChunk->offNext),
804	VERR_INVALID_POINTER);
805
806	AssertMsgReturn(MMHYPERCHUNK_ISUSED(pChunk),
807	("%p: Not used!\n", pv),
808	VERR_INVALID_POINTER);
809
810	int32_t offPrev = MMHYPERCHUNK_GET_OFFPREV(pChunk);
811	AssertMsgReturn( (uintptr_t)pChunk + offPrev <= (uintptr_t)pChunk
812	&& !((uint32_t)-offPrev & (MMHYPER_HEAP_ALIGN_MIN - 1)),
813	("%p: offPrev=%#RX32!\n", pv, offPrev),
814	VERR_INVALID_POINTER);
815
816	/* statistics */
817	#ifdef VBOX_WITH_STATISTICS
818	PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pChunk + pChunk->offStat);
819	AssertMsgReturn( RT_ALIGN_P(pStat, MMHYPER_HEAP_ALIGN_MIN) == (void *)pStat
820	&& pChunk->offStat,
821	("%p: offStat=%#RX32!\n", pv, pChunk->offStat),
822	VERR_INVALID_POINTER);
823	#else
824	AssertMsgReturn(!pChunk->offStat,
825	("%p: offStat=%#RX32!\n", pv, pChunk->offStat),
826	VERR_INVALID_POINTER);
827	#endif
828
829	/* The heap structure. */
830	PMMHYPERHEAP pHeap = (PMMHYPERHEAP)((uintptr_t)pChunk + pChunk->offHeap);
831	AssertMsgReturn( !((uintptr_t)pHeap & PAGE_OFFSET_MASK)
832	&& pChunk->offHeap,
833	("%p: pHeap=%#x offHeap=%RX32\n", pv, pHeap->u32Magic, pChunk->offHeap),
834	VERR_INVALID_POINTER);
835
836	AssertMsgReturn(pHeap->u32Magic == MMHYPERHEAP_MAGIC,
837	("%p: u32Magic=%#x\n", pv, pHeap->u32Magic),
838	VERR_INVALID_POINTER);
839	Assert(pHeap == pVM->mm.s.CTX_SUFF(pHyperHeap));
840
841	/* Some more verifications using additional info from pHeap. */
842	AssertMsgReturn((uintptr_t)pChunk + offPrev >= (uintptr_t)pHeap->CTX_SUFF(pbHeap),
843	("%p: offPrev=%#RX32!\n", pv, offPrev),
844	VERR_INVALID_POINTER);
845
846	AssertMsgReturn(pChunk->offNext < pHeap->cbHeap,
847	("%p: offNext=%#RX32!\n", pv, pChunk->offNext),
848	VERR_INVALID_POINTER);
849
850	AssertMsgReturn( (uintptr_t)pv - (uintptr_t)pHeap->CTX_SUFF(pbHeap) <= pHeap->offPageAligned,
851	("Invalid pointer %p! (heap: %p-%p)\n", pv, pHeap->CTX_SUFF(pbHeap),
852	(char *)pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned),
853	VERR_INVALID_POINTER);
854
855	#ifdef MMHYPER_HEAP_STRICT
856	mmHyperHeapCheck(pHeap);
857	#endif
858
859	#if defined(VBOX_WITH_STATISTICS) \|\| defined(MMHYPER_HEAP_FREE_POISON)
860	/* calc block size. */
861	const uint32_t cbChunk = pChunk->offNext
862	? pChunk->offNext
863	: pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t *)pChunk;
864	#endif
865	#ifdef MMHYPER_HEAP_FREE_POISON
866	/* poison the block */
867	memset(pChunk + 1, MMHYPER_HEAP_FREE_POISON, cbChunk - sizeof(*pChunk));
868	#endif
869
870	#ifdef MMHYPER_HEAP_FREE_DELAY
871	# ifdef MMHYPER_HEAP_FREE_POISON
872	/*
873	* Check poison.
874	*/
875	unsigned i = RT_ELEMENTS(pHeap->aDelayedFrees);
876	while (i-- > 0)
877	if (pHeap->aDelayedFrees[i].offChunk)
878	{
879	PMMHYPERCHUNK pCur = (PMMHYPERCHUNK)((uintptr_t)pHeap + pHeap->aDelayedFrees[i].offChunk);
880	const size_t cb = pCur->offNext
881	? pCur->offNext - sizeof(*pCur)
882	: pHeap->CTX_SUFF(pbHeap) + pHeap->offPageAligned - (uint8_t )pCur - sizeof(pCur);
883	uint8_t pab = (uint8_t )(pCur + 1);
884	for (unsigned off = 0; off < cb; off++)
885	AssertReleaseMsg(pab[off] == 0xCB,
886	("caller=%RTptr cb=%#zx off=%#x: %.*Rhxs\n",
887	pHeap->aDelayedFrees[i].uCaller, cb, off, RT_MIN(cb - off, 32), &pab[off]));
888	}
889	# endif /* MMHYPER_HEAP_FREE_POISON */
890
891	/*
892	* Delayed freeing.
893	*/
894	int rc = VINF_SUCCESS;
895	if (pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk)
896	{
897	PMMHYPERCHUNK pChunkFree = (PMMHYPERCHUNK)((uintptr_t)pHeap + pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk);
898	rc = mmHyperFree(pHeap, pChunkFree);
899	}
900	pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk = (uintptr_t)pChunk - (uintptr_t)pHeap;
901	pHeap->aDelayedFrees[pHeap->iDelayedFree].uCaller = (uintptr_t)ASMReturnAddress();
902	pHeap->iDelayedFree = (pHeap->iDelayedFree + 1) % RT_ELEMENTS(pHeap->aDelayedFrees);
903
904	#else /* !MMHYPER_HEAP_FREE_POISON */
905	/*
906	* Call the worker.
907	*/
908	int rc = mmHyperFree(pHeap, pChunk);
909	#endif /* !MMHYPER_HEAP_FREE_POISON */
910
911	/*
912	* Update statistics.
913	*/
914	#ifdef VBOX_WITH_STATISTICS
915	pStat->cFrees++;
916	if (RT_SUCCESS(rc))
917	{
918	pStat->cbFreed += cbChunk;
919	pStat->cbCurAllocated -= cbChunk;
920	}
921	else
922	pStat->cFailures++;
923	#endif
924
925	return rc;
926	}
927
928
929	/**
930	* Wrapper for mmHyperFreeInternal
931	*/
932	VMMDECL(int) MMHyperFree(PVM pVM, void *pv)
933	{
934	int rc;
935
936	rc = mmHyperLock(pVM);
937	AssertRCReturn(rc, rc);
938
939	LogFlow(("MMHyperFree %p\n", pv));
940
941	rc = mmHyperFreeInternal(pVM, pv);
942
943	mmHyperUnlock(pVM);
944	return rc;
945	}
946
947
948	/**
949	* Free memory a memory chunk.
950	*
951	* @returns VBox status code.
952	* @param pHeap The heap.
953	* @param pChunk The memory chunk to free.
954	*/
955	static int mmHyperFree(PMMHYPERHEAP pHeap, PMMHYPERCHUNK pChunk)
956	{
957	Log3(("mmHyperFree: Enter pHeap=%p pChunk=%p\n", pHeap, pChunk));
958	PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)pChunk;
959
960	/*
961	* Insert into the free list (which is sorted on address).
962	*
963	* We'll search towards the end of the heap to locate the
964	* closest FREE chunk.
965	*/
966	PMMHYPERCHUNKFREE pLeft = NULL;
967	PMMHYPERCHUNKFREE pRight = NULL;
968	if (pHeap->offFreeTail != NIL_OFFSET)
969	{
970	if (pFree->core.offNext)
971	{
972	pRight = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->core.offNext);
973	ASSERT_CHUNK(pHeap, &pRight->core);
974	while (!MMHYPERCHUNK_ISFREE(&pRight->core))
975	{
976	if (!pRight->core.offNext)
977	{
978	pRight = NULL;
979	break;
980	}
981	pRight = (PMMHYPERCHUNKFREE)((char *)pRight + pRight->core.offNext);
982	ASSERT_CHUNK(pHeap, &pRight->core);
983	}
984	}
985	if (!pRight)
986	pRight = (PMMHYPERCHUNKFREE)((char )pHeap->CTX_SUFF(pbHeap) + pHeap->offFreeTail); /* @todo this can't be correct! 'pLeft = .. ; else' I think */
987	if (pRight)
988	{
989	ASSERT_CHUNK_FREE(pHeap, pRight);
990	if (pRight->offPrev)
991	{
992	pLeft = (PMMHYPERCHUNKFREE)((char *)pRight + pRight->offPrev);
993	ASSERT_CHUNK_FREE(pHeap, pLeft);
994	}
995	}
996	}
997	if (pLeft == pFree)
998	{
999	AssertMsgFailed(("Freed twice! pv=%p (pChunk=%p)\n", pChunk + 1, pChunk));
1000	return VERR_INVALID_POINTER;
1001	}
1002	pChunk->offStat = 0;
1003
1004	/*
1005	* Head free chunk list?
1006	*/
1007	if (!pLeft)
1008	{
1009	MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_FREE);
1010	pFree->offPrev = 0;
1011	pHeap->offFreeHead = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
1012	if (pRight)
1013	{
1014	pFree->offNext = (uintptr_t)pRight - (uintptr_t)pFree;
1015	pRight->offPrev = -(int32_t)pFree->offNext;
1016	}
1017	else
1018	{
1019	pFree->offNext = 0;
1020	pHeap->offFreeTail = pHeap->offFreeHead;
1021	}
1022	Log3(("mmHyperFree: Inserted %p at head of free chain.\n", pFree));
1023	}
1024	else
1025	{
1026	/*
1027	* Can we merge with left hand free chunk?
1028	*/
1029	if ((char )pLeft + pLeft->core.offNext == (char )pFree)
1030	{
1031	if (pFree->core.offNext)
1032	{
1033	pLeft->core.offNext = pLeft->core.offNext + pFree->core.offNext;
1034	MMHYPERCHUNK_SET_OFFPREV(((PMMHYPERCHUNK)((char *)pLeft + pLeft->core.offNext)), -(int32_t)pLeft->core.offNext);
1035	}
1036	else
1037	pLeft->core.offNext = 0;
1038	pFree = pLeft;
1039	Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pLeft->cb, -(int32_t)pLeft->cb));
1040	pHeap->cbFree -= pLeft->cb;
1041	Log3(("mmHyperFree: Merging %p into %p (cb=%d).\n", pFree, pLeft, pLeft->cb));
1042	}
1043	/*
1044	* No, just link it into the free list then.
1045	*/
1046	else
1047	{
1048	MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_FREE);
1049	pFree->offPrev = (uintptr_t)pLeft - (uintptr_t)pFree;
1050	pLeft->offNext = -pFree->offPrev;
1051	if (pRight)
1052	{
1053	pFree->offNext = (uintptr_t)pRight - (uintptr_t)pFree;
1054	pRight->offPrev = -(int32_t)pFree->offNext;
1055	}
1056	else
1057	{
1058	pFree->offNext = 0;
1059	pHeap->offFreeTail = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
1060	}
1061	Log3(("mmHyperFree: Inserted %p after %p in free list.\n", pFree, pLeft));
1062	}
1063	}
1064
1065	/*
1066	* Can we merge with right hand free chunk?
1067	*/
1068	if (pRight && (char )pRight == (char )pFree + pFree->core.offNext)
1069	{
1070	/* core */
1071	if (pRight->core.offNext)
1072	{
1073	pFree->core.offNext += pRight->core.offNext;
1074	PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pFree + pFree->core.offNext);
1075	MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pFree->core.offNext);
1076	ASSERT_CHUNK(pHeap, pNext);
1077	}
1078	else
1079	pFree->core.offNext = 0;
1080
1081	/* free */
1082	if (pRight->offNext)
1083	{
1084	pFree->offNext += pRight->offNext;
1085	((PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext))->offPrev = -(int32_t)pFree->offNext;
1086	}
1087	else
1088	{
1089	pFree->offNext = 0;
1090	pHeap->offFreeTail = (uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap);
1091	}
1092	Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pRight->cb, -(int32_t)pRight->cb));
1093	pHeap->cbFree -= pRight->cb;
1094	Log3(("mmHyperFree: Merged %p (cb=%d) into %p.\n", pRight, pRight->cb, pFree));
1095	}
1096
1097	/* calculate the size. */
1098	if (pFree->core.offNext)
1099	pFree->cb = pFree->core.offNext - sizeof(MMHYPERCHUNK);
1100	else
1101	pFree->cb = pHeap->offPageAligned - ((uintptr_t)pFree - (uintptr_t)pHeap->CTX_SUFF(pbHeap)) - sizeof(MMHYPERCHUNK);
1102	Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree + pFree->cb, pFree->cb));
1103	pHeap->cbFree += pFree->cb;
1104	ASSERT_CHUNK_FREE(pHeap, pFree);
1105
1106	#ifdef MMHYPER_HEAP_STRICT
1107	mmHyperHeapCheck(pHeap);
1108	#endif
1109	return VINF_SUCCESS;
1110	}
1111
1112
1113	#if defined(DEBUG) \|\| defined(MMHYPER_HEAP_STRICT)
1114	/**
1115	* Dumps a heap chunk to the log.
1116	*
1117	* @param pHeap Pointer to the heap.
1118	* @param pCur Pointer to the chunk.
1119	*/
1120	static void mmHyperHeapDumpOne(PMMHYPERHEAP pHeap, PMMHYPERCHUNKFREE pCur)
1121	{
1122	if (MMHYPERCHUNK_ISUSED(&pCur->core))
1123	{
1124	if (pCur->core.offStat)
1125	{
1126	PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pCur + pCur->core.offStat);
1127	const char *pszSelf = pCur->core.offStat == sizeof(MMHYPERCHUNK) ? " stat record" : "";
1128	#ifdef IN_RING3
1129	Log(("%p %06x USED offNext=%06x offPrev=-%06x %s%s\n",
1130	pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1131	pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1132	mmGetTagName((MMTAG)pStat->Core.Key), pszSelf));
1133	#else
1134	Log(("%p %06x USED offNext=%06x offPrev=-%06x %d%s\n",
1135	pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1136	pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1137	(MMTAG)pStat->Core.Key, pszSelf));
1138	#endif
1139	}
1140	else
1141	Log(("%p %06x USED offNext=%06x offPrev=-%06x\n",
1142	pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1143	pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core)));
1144	}
1145	else
1146	Log(("%p %06x FREE offNext=%06x offPrev=-%06x : cb=%06x offNext=%06x offPrev=-%06x\n",
1147	pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
1148	pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core), pCur->cb, pCur->offNext, pCur->offPrev));
1149	}
1150	#endif /* DEBUG \|\| MMHYPER_HEAP_STRICT */
1151
1152
1153	#ifdef MMHYPER_HEAP_STRICT
1154	/**
1155	* Internal consistency check.
1156	*/
1157	static void mmHyperHeapCheck(PMMHYPERHEAP pHeap)
1158	{
1159	PMMHYPERCHUNKFREE pPrev = NULL;
1160	PMMHYPERCHUNKFREE pCur = (PMMHYPERCHUNKFREE)pHeap->CTX_SUFF(pbHeap);
1161	for (;;)
1162	{
1163	if (MMHYPERCHUNK_ISUSED(&pCur->core))
1164	ASSERT_CHUNK_USED(pHeap, &pCur->core);
1165	else
1166	ASSERT_CHUNK_FREE(pHeap, pCur);
1167	if (pPrev)
1168	AssertMsg((int32_t)pPrev->core.offNext == -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1169	("pPrev->core.offNext=%d offPrev=%d\n", pPrev->core.offNext, MMHYPERCHUNK_GET_OFFPREV(&pCur->core)));
1170
1171	# ifdef MMHYPER_HEAP_STRICT_FENCE
1172	uint32_t off = (uint8_t *)pCur - pHeap->CTX_SUFF(pbHeap);
1173	if ( MMHYPERCHUNK_ISUSED(&pCur->core)
1174	&& off < pHeap->offPageAligned)
1175	{
1176	uint32_t cbCur = pCur->core.offNext
1177	? pCur->core.offNext
1178	: pHeap->cbHeap - off;
1179	uint32_t pu32End = ((uint32_t )((uint8_t *)pCur + cbCur));
1180	uint32_t cbFence = pu32End[-1];
1181	if (RT_UNLIKELY( cbFence >= cbCur - sizeof(*pCur)
1182	\|\| cbFence < MMHYPER_HEAP_STRICT_FENCE_SIZE))
1183	{
1184	mmHyperHeapDumpOne(pHeap, pCur);
1185	Assert(cbFence < cbCur - sizeof(*pCur));
1186	Assert(cbFence >= MMHYPER_HEAP_STRICT_FENCE_SIZE);
1187	}
1188
1189	uint32_t pu32Bad = ASMMemIsAllU32((uint8_t )pu32End - cbFence, cbFence - sizeof(uint32_t), MMHYPER_HEAP_STRICT_FENCE_U32);
1190	if (RT_UNLIKELY(pu32Bad))
1191	{
1192	mmHyperHeapDumpOne(pHeap, pCur);
1193	Assert(!pu32Bad);
1194	}
1195	}
1196	# endif
1197
1198	/* next */
1199	if (!pCur->core.offNext)
1200	break;
1201	pPrev = pCur;
1202	pCur = (PMMHYPERCHUNKFREE)((char *)pCur + pCur->core.offNext);
1203	}
1204	}
1205	#endif
1206
1207
1208	/**
1209	* Performs consistency checks on the heap if MMHYPER_HEAP_STRICT was
1210	* defined at build time.
1211	*
1212	* @param pVM Pointer to the VM.
1213	*/
1214	VMMDECL(void) MMHyperHeapCheck(PVM pVM)
1215	{
1216	#ifdef MMHYPER_HEAP_STRICT
1217	int rc;
1218
1219	rc = mmHyperLock(pVM);
1220	AssertRC(rc);
1221	mmHyperHeapCheck(pVM->mm.s.CTX_SUFF(pHyperHeap));
1222	mmHyperUnlock(pVM);
1223	#endif
1224	}
1225
1226
1227	#ifdef DEBUG
1228	/**
1229	* Dumps the hypervisor heap to Log.
1230	* @param pVM Pointer to the VM.
1231	*/
1232	VMMDECL(void) MMHyperHeapDump(PVM pVM)
1233	{
1234	Log(("MMHyperHeapDump: * heap dump - start *\n"));
1235	PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
1236	PMMHYPERCHUNKFREE pCur = (PMMHYPERCHUNKFREE)pHeap->CTX_SUFF(pbHeap);
1237	for (;;)
1238	{
1239	mmHyperHeapDumpOne(pHeap, pCur);
1240
1241	/* next */
1242	if (!pCur->core.offNext)
1243	break;
1244	pCur = (PMMHYPERCHUNKFREE)((char *)pCur + pCur->core.offNext);
1245	}
1246	Log(("MMHyperHeapDump: * heap dump - end *\n"));
1247	}
1248	#endif
1249
1250
1251	/**
1252	* Query the amount of free memory in the hypervisor heap.
1253	*
1254	* @returns Number of free bytes in the hypervisor heap.
1255	*/
1256	VMMDECL(size_t) MMHyperHeapGetFreeSize(PVM pVM)
1257	{
1258	return pVM->mm.s.CTX_SUFF(pHyperHeap)->cbFree;
1259	}
1260
1261	/**
1262	* Query the size the hypervisor heap.
1263	*
1264	* @returns The size of the hypervisor heap in bytes.
1265	*/
1266	VMMDECL(size_t) MMHyperHeapGetSize(PVM pVM)
1267	{
1268	return pVM->mm.s.CTX_SUFF(pHyperHeap)->cbHeap;
1269	}
1270
1271
1272	/**
1273	* Query the address and size the hypervisor memory area.
1274	*
1275	* @returns Base address of the hypervisor area.
1276	* @param pVM Pointer to the VM.
1277	* @param pcb Where to store the size of the hypervisor area. (out)
1278	*/
1279	VMMDECL(RTGCPTR) MMHyperGetArea(PVM pVM, size_t *pcb)
1280	{
1281	if (pcb)
1282	*pcb = pVM->mm.s.cbHyperArea;
1283	return pVM->mm.s.pvHyperAreaGC;
1284	}
1285
1286
1287	/**
1288	* Checks if an address is within the hypervisor memory area.
1289	*
1290	* @returns true if inside.
1291	* @returns false if outside.
1292	* @param pVM Pointer to the VM.
1293	* @param GCPtr The pointer to check.
1294	*/
1295	VMMDECL(bool) MMHyperIsInsideArea(PVM pVM, RTGCPTR GCPtr)
1296	{
1297	return (RTGCUINTPTR)GCPtr - (RTGCUINTPTR)pVM->mm.s.pvHyperAreaGC < pVM->mm.s.cbHyperArea;
1298	}
1299

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/MMAllHyper.cpp@ 51271

Download in other formats: