VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/MMAllHyper.cpp@ 8098

Last change on this file since 8098 was 7632, checked in by vboxsync, 17 years ago

Added optional fences to the hyper heap.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 45.5 KB
Line 
1/* $Id: MMAllHyper.cpp 7632 2008-03-28 17:05:00Z vboxsync $ */
2/** @file
3 * MM - Memory Monitor(/Manager) - Hypervisor Memory Area, All Contexts.
4 */
5
6/*
7 * Copyright (C) 2006-2007 innotek GmbH
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*******************************************************************************
20* Header Files *
21*******************************************************************************/
22#define LOG_GROUP LOG_GROUP_MM_HYPER_HEAP
23#include <VBox/mm.h>
24#include <VBox/stam.h>
25#include "MMInternal.h"
26#include <VBox/vm.h>
27
28#include <VBox/err.h>
29#include <VBox/param.h>
30#include <iprt/assert.h>
31#include <VBox/log.h>
32#include <iprt/asm.h>
33#include <iprt/string.h>
34
35
36/*******************************************************************************
37* Defined Constants And Macros *
38*******************************************************************************/
39#define ASSERT_L(u1, u2) AssertMsg((u1) < (u2), ("u1=%#x u2=%#x\n", u1, u2))
40#define ASSERT_LE(u1, u2) AssertMsg((u1) <= (u2), ("u1=%#x u2=%#x\n", u1, u2))
41#define ASSERT_GE(u1, u2) AssertMsg((u1) >= (u2), ("u1=%#x u2=%#x\n", u1, u2))
42#define ASSERT_ALIGN(u1) AssertMsg(!((u1) & (MMHYPER_HEAP_ALIGN_MIN - 1)), ("u1=%#x (%d)\n", u1, u1))
43
44#define ASSERT_OFFPREV(pHeap, pChunk) \
45 do { Assert(MMHYPERCHUNK_GET_OFFPREV(pChunk) <= 0); \
46 Assert(MMHYPERCHUNK_GET_OFFPREV(pChunk) >= (intptr_t)CTXSUFF((pHeap)->pbHeap) - (intptr_t)(pChunk)); \
47 AssertMsg( MMHYPERCHUNK_GET_OFFPREV(pChunk) != 0 \
48 || (uint8_t *)(pChunk) == CTXSUFF((pHeap)->pbHeap), \
49 ("pChunk=%p pvHyperHeap=%p\n", (pChunk), CTXSUFF((pHeap)->pbHeap))); \
50 } while (0)
51
52#define ASSERT_OFFNEXT(pHeap, pChunk) \
53 do { ASSERT_ALIGN((pChunk)->offNext); \
54 ASSERT_L((pChunk)->offNext, (uintptr_t)CTXSUFF((pHeap)->pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
55 } while (0)
56
57#define ASSERT_OFFHEAP(pHeap, pChunk) \
58 do { Assert((pChunk)->offHeap); \
59 AssertMsg((PMMHYPERHEAP)((pChunk)->offHeap + (uintptr_t)pChunk) == (pHeap), \
60 ("offHeap=%RX32 pChunk=%p pHeap=%p\n", (pChunk)->offHeap, (pChunk), (pHeap))); \
61 Assert((pHeap)->u32Magic == MMHYPERHEAP_MAGIC); \
62 } while (0)
63
64#ifdef VBOX_WITH_STATISTICS
65#define ASSERT_OFFSTAT(pHeap, pChunk) \
66 do { if (MMHYPERCHUNK_ISFREE(pChunk)) \
67 Assert(!(pChunk)->offStat); \
68 else if ((pChunk)->offStat) \
69 { \
70 Assert((pChunk)->offStat); \
71 AssertMsg(!((pChunk)->offStat & (MMHYPER_HEAP_ALIGN_MIN - 1)), ("offStat=%RX32\n", (pChunk)->offStat)); \
72 uintptr_t uPtr = (uintptr_t)(pChunk)->offStat + (uintptr_t)pChunk; NOREF(uPtr); \
73 AssertMsg(uPtr - (uintptr_t)CTXSUFF((pHeap)->pbHeap) < (pHeap)->offPageAligned, \
74 ("%p - %p < %RX32\n", uPtr, CTXSUFF((pHeap)->pbHeap), (pHeap)->offPageAligned)); \
75 } \
76 } while (0)
77#else
78#define ASSERT_OFFSTAT(pHeap, pChunk) \
79 do { Assert(!(pChunk)->offStat); \
80 } while (0)
81#endif
82
83#define ASSERT_CHUNK(pHeap, pChunk) \
84 do { ASSERT_OFFNEXT(pHeap, pChunk); \
85 ASSERT_OFFPREV(pHeap, pChunk); \
86 ASSERT_OFFHEAP(pHeap, pChunk); \
87 ASSERT_OFFSTAT(pHeap, pChunk); \
88 } while (0)
89#define ASSERT_CHUNK_USED(pHeap, pChunk) \
90 do { ASSERT_OFFNEXT(pHeap, pChunk); \
91 ASSERT_OFFPREV(pHeap, pChunk); \
92 Assert(MMHYPERCHUNK_ISUSED(pChunk)); \
93 } while (0)
94
95#define ASSERT_FREE_OFFPREV(pHeap, pChunk) \
96 do { ASSERT_ALIGN((pChunk)->offPrev); \
97 ASSERT_GE(((pChunk)->offPrev & (MMHYPER_HEAP_ALIGN_MIN - 1)), (intptr_t)CTXSUFF((pHeap)->pbHeap) - (intptr_t)(pChunk)); \
98 Assert((pChunk)->offPrev != MMHYPERCHUNK_GET_OFFPREV(&(pChunk)->core) || !(pChunk)->offPrev); \
99 AssertMsg( (pChunk)->offPrev \
100 || (uintptr_t)(pChunk) - (uintptr_t)CTXSUFF((pHeap)->pbHeap) == (pHeap)->offFreeHead, \
101 ("pChunk=%p offChunk=%#x offFreeHead=%#x\n", (pChunk), (uintptr_t)(pChunk) - (uintptr_t)CTXSUFF((pHeap)->pbHeap),\
102 (pHeap)->offFreeHead)); \
103 } while (0)
104
105#define ASSERT_FREE_OFFNEXT(pHeap, pChunk) \
106 do { ASSERT_ALIGN((pChunk)->offNext); \
107 ASSERT_L((pChunk)->offNext, (uintptr_t)CTXSUFF((pHeap)->pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
108 Assert((pChunk)->offNext != (pChunk)->core.offNext || !(pChunk)->offNext); \
109 AssertMsg( (pChunk)->offNext \
110 || (uintptr_t)(pChunk) - (uintptr_t)CTXSUFF((pHeap)->pbHeap) == (pHeap)->offFreeTail, \
111 ("pChunk=%p offChunk=%#x offFreeTail=%#x\n", (pChunk), (uintptr_t)(pChunk) - (uintptr_t)CTXSUFF((pHeap)->pbHeap), \
112 (pHeap)->offFreeTail)); \
113 } while (0)
114
115#define ASSERT_FREE_CB(pHeap, pChunk) \
116 do { ASSERT_ALIGN((pChunk)->cb); \
117 Assert((pChunk)->cb > 0); \
118 if ((pChunk)->core.offNext) \
119 AssertMsg((pChunk)->cb == ((pChunk)->core.offNext - sizeof(MMHYPERCHUNK)), \
120 ("cb=%d offNext=%d\n", (pChunk)->cb, (pChunk)->core.offNext)); \
121 else \
122 ASSERT_LE((pChunk)->cb, (uintptr_t)CTXSUFF((pHeap)->pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
123 } while (0)
124
125#define ASSERT_CHUNK_FREE(pHeap, pChunk) \
126 do { ASSERT_CHUNK(pHeap, &(pChunk)->core); \
127 Assert(MMHYPERCHUNK_ISFREE(pChunk)); \
128 ASSERT_FREE_OFFNEXT(pHeap, pChunk); \
129 ASSERT_FREE_OFFPREV(pHeap, pChunk); \
130 ASSERT_FREE_CB(pHeap, pChunk); \
131 } while (0)
132
133
134/*******************************************************************************
135* Internal Functions *
136*******************************************************************************/
137static PMMHYPERCHUNK mmHyperAllocChunk(PMMHYPERHEAP pHeap, uint32_t cb, unsigned uAlignment);
138static void *mmHyperAllocPages(PMMHYPERHEAP pHeap, uint32_t cb);
139#ifdef VBOX_WITH_STATISTICS
140static PMMHYPERSTAT mmHyperStat(PMMHYPERHEAP pHeap, MMTAG enmTag);
141#ifdef IN_RING3
142static void mmR3HyperStatRegisterOne(PVM pVM, PMMHYPERSTAT pStat);
143#endif
144#endif
145static int mmHyperFree(PMMHYPERHEAP pHeap, PMMHYPERCHUNK pChunk);
146#ifdef MMHYPER_HEAP_STRICT
147static void mmHyperHeapCheck(PMMHYPERHEAP pHeap);
148#endif
149
150
151/**
152 * Allocates memory in the Hypervisor (GC VMM) area.
153 * The returned memory is of course zeroed.
154 *
155 * @returns VBox status code.
156 * @param pVM The VM to operate on.
157 * @param cb Number of bytes to allocate.
158 * @param uAlignment Required memory alignment in bytes.
159 * Values are 0,8,16,32 and PAGE_SIZE.
160 * 0 -> default alignment, i.e. 8 bytes.
161 * @param enmTag The statistics tag.
162 * @param ppv Where to store the address to the allocated
163 * memory.
164 * @remark This is assumed not to be used at times when serialization is required.
165 */
166MMDECL(int) MMHyperAlloc(PVM pVM, size_t cb, unsigned uAlignment, MMTAG enmTag, void **ppv)
167{
168 AssertMsg(cb >= 8, ("Hey! Do you really mean to allocate less than 8 bytes?! cb=%d\n", cb));
169
170 /*
171 * Validate input and adjust it to reasonable values.
172 */
173 if (!uAlignment || uAlignment < MMHYPER_HEAP_ALIGN_MIN)
174 uAlignment = MMHYPER_HEAP_ALIGN_MIN;
175 uint32_t cbAligned;
176 switch (uAlignment)
177 {
178 case 8:
179 case 16:
180 case 32:
181 cbAligned = RT_ALIGN(cb, MMHYPER_HEAP_ALIGN_MIN);
182 if (!cbAligned || cbAligned < cb)
183 {
184 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
185 AssertMsgFailed(("Nice try.\n"));
186 return VERR_INVALID_PARAMETER;
187 }
188 break;
189
190 case PAGE_SIZE:
191 AssertMsg(RT_ALIGN(cb, PAGE_SIZE) == cb, ("The size isn't page aligned. (cb=%#x)\n", cb));
192 cbAligned = RT_ALIGN(cb, PAGE_SIZE);
193 if (!cbAligned)
194 {
195 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
196 AssertMsgFailed(("Nice try.\n"));
197 return VERR_INVALID_PARAMETER;
198 }
199 break;
200
201 default:
202 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
203 AssertMsgFailed(("Invalid alignment %u\n", uAlignment));
204 return VERR_INVALID_PARAMETER;
205 }
206
207
208 /*
209 * Get heap and statisticsStatistics.
210 */
211 PMMHYPERHEAP pHeap = CTXSUFF(pVM->mm.s.pHyperHeap);
212#ifdef VBOX_WITH_STATISTICS
213 PMMHYPERSTAT pStat = mmHyperStat(pHeap, enmTag);
214 if (!pStat)
215 {
216 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_MM_HYPER_NO_MEMORY\n", cb, uAlignment));
217 AssertMsgFailed(("Failed to allocate statistics!\n"));
218 return VERR_MM_HYPER_NO_MEMORY;
219 }
220#endif
221 if (uAlignment < PAGE_SIZE)
222 {
223 /*
224 * Allocate a chunk.
225 */
226 PMMHYPERCHUNK pChunk = mmHyperAllocChunk(pHeap, cbAligned, uAlignment);
227 if (pChunk)
228 {
229#ifdef VBOX_WITH_STATISTICS
230 const uint32_t cbChunk = pChunk->offNext
231 ? pChunk->offNext
232 : CTXSUFF(pHeap->pbHeap) + pHeap->offPageAligned - (uint8_t *)pChunk;
233 pStat->cbAllocated += (uint32_t)cbChunk;
234 pStat->cbCurAllocated += (uint32_t)cbChunk;
235 if (pStat->cbCurAllocated > pStat->cbMaxAllocated)
236 pStat->cbMaxAllocated = pStat->cbCurAllocated;
237 pStat->cAllocations++;
238 pChunk->offStat = (uintptr_t)pStat - (uintptr_t)pChunk;
239#else
240 pChunk->offStat = 0;
241#endif
242 void *pv = pChunk + 1;
243 *ppv = pv;
244 ASMMemZero32(pv, cbAligned);
245 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VINF_SUCCESS and *ppv=%p\n", cb, uAlignment, pv));
246 return VINF_SUCCESS;
247 }
248 }
249 else
250 {
251 /*
252 * Allocate page aligned memory.
253 */
254 void *pv = mmHyperAllocPages(pHeap, cbAligned);
255 if (pv)
256 {
257#ifdef VBOX_WITH_STATISTICS
258 pStat->cbAllocated += cbAligned;
259 pStat->cbCurAllocated += cbAligned;
260 if (pStat->cbCurAllocated > pStat->cbMaxAllocated)
261 pStat->cbMaxAllocated = pStat->cbCurAllocated;
262 pStat->cAllocations++;
263#endif
264 *ppv = pv;
265 /* ASMMemZero32(pv, cbAligned); - not required since memory is alloc-only and SUPPageAlloc zeros it. */
266 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VINF_SUCCESS and *ppv=%p\n", cb, uAlignment, ppv));
267 return VINF_SUCCESS;
268 }
269 }
270
271#ifdef VBOX_WITH_STATISTICS
272 pStat->cAllocations++;
273 pStat->cFailures++;
274#endif
275 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_MM_HYPER_NO_MEMORY\n", cb, uAlignment));
276 AssertMsgFailed(("Failed to allocate %d bytes!\n", cb));
277 return VERR_MM_HYPER_NO_MEMORY;
278}
279
280
281
282/**
283 * Allocates a chunk of memory from the specified heap.
284 * The caller validates the parameters of this request.
285 *
286 * @returns Pointer to the allocated chunk.
287 * @returns NULL on failure.
288 * @param pHeap The heap.
289 * @param cb Size of the memory block to allocate.
290 * @param uAlignment The alignment specifications for the allocated block.
291 * @internal
292 */
293static PMMHYPERCHUNK mmHyperAllocChunk(PMMHYPERHEAP pHeap, uint32_t cb, unsigned uAlignment)
294{
295 Log3(("mmHyperAllocChunk: Enter cb=%#x uAlignment=%#x\n", cb, uAlignment));
296#ifdef MMHYPER_HEAP_STRICT
297 mmHyperHeapCheck(pHeap);
298#endif
299#ifdef MMHYPER_HEAP_STRICT_FENCE
300 uint32_t cbFence = RT_MAX(MMHYPER_HEAP_STRICT_FENCE_SIZE, uAlignment);
301 cb += cbFence;
302#endif
303
304 /*
305 * Check if there are any free chunks. (NIL_OFFSET use/not-use forces this check)
306 */
307 if (pHeap->offFreeHead == NIL_OFFSET)
308 return NULL;
309
310 /*
311 * Small alignments - from the front of the heap.
312 *
313 * Must split off free chunks at the end to prevent messing up the
314 * last free node which we take the page aligned memory from the top of.
315 */
316 PMMHYPERCHUNK pRet = NULL;
317 PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)((char *)CTXSUFF(pHeap->pbHeap) + pHeap->offFreeHead);
318 while (pFree)
319 {
320 ASSERT_CHUNK_FREE(pHeap, pFree);
321 if (pFree->cb >= cb)
322 {
323 unsigned offAlign = (uintptr_t)(&pFree->core + 1) & (uAlignment - 1);
324 if (offAlign)
325 offAlign = uAlignment - offAlign;
326 if (!offAlign || pFree->cb - offAlign >= cb)
327 {
328 Log3(("mmHyperAllocChunk: Using pFree=%p pFree->cb=%d offAlign=%d\n", pFree, pFree->cb, offAlign));
329
330 /*
331 * Adjust the node in front.
332 * Because of multiple alignments we need to special case allocation of the first block.
333 */
334 if (offAlign)
335 {
336 MMHYPERCHUNKFREE Free = *pFree;
337 if (MMHYPERCHUNK_GET_OFFPREV(&pFree->core))
338 {
339 /* just add a bit of memory to it. */
340 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + MMHYPERCHUNK_GET_OFFPREV(&Free.core));
341 pPrev->core.offNext += offAlign;
342 AssertMsg(!MMHYPERCHUNK_ISFREE(&pPrev->core), ("Impossible!\n"));
343 Log3(("mmHyperAllocChunk: Added %d bytes to %p\n", offAlign, pPrev));
344 }
345 else
346 {
347 /* make new head node, mark it USED for simplisity. */
348 PMMHYPERCHUNK pPrev = (PMMHYPERCHUNK)CTXSUFF(pHeap->pbHeap);
349 Assert(pPrev == &pFree->core);
350 pPrev->offPrev = 0;
351 MMHYPERCHUNK_SET_TYPE(pPrev, MMHYPERCHUNK_FLAGS_USED);
352 pPrev->offNext = offAlign;
353 Log3(("mmHyperAllocChunk: Created new first node of %d bytes\n", offAlign));
354
355 }
356 Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - offAlign, -(int)offAlign));
357 pHeap->cbFree -= offAlign;
358
359 /* Recreate pFree node and adjusting everything... */
360 pFree = (PMMHYPERCHUNKFREE)((char *)pFree + offAlign);
361 *pFree = Free;
362
363 pFree->cb -= offAlign;
364 if (pFree->core.offNext)
365 {
366 pFree->core.offNext -= offAlign;
367 PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pFree + pFree->core.offNext);
368 MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pFree->core.offNext);
369 ASSERT_CHUNK(pHeap, pNext);
370 }
371 if (MMHYPERCHUNK_GET_OFFPREV(&pFree->core))
372 MMHYPERCHUNK_SET_OFFPREV(&pFree->core, MMHYPERCHUNK_GET_OFFPREV(&pFree->core) - offAlign);
373
374 if (pFree->offNext)
375 {
376 pFree->offNext -= offAlign;
377 PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext);
378 pNext->offPrev = -(int32_t)pFree->offNext;
379 ASSERT_CHUNK_FREE(pHeap, pNext);
380 }
381 else
382 pHeap->offFreeTail += offAlign;
383 if (pFree->offPrev)
384 {
385 pFree->offPrev -= offAlign;
386 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
387 pPrev->offNext = -pFree->offPrev;
388 ASSERT_CHUNK_FREE(pHeap, pPrev);
389 }
390 else
391 pHeap->offFreeHead += offAlign;
392 pFree->core.offHeap = (uintptr_t)pHeap - (uintptr_t)pFree;
393 pFree->core.offStat = 0;
394 ASSERT_CHUNK_FREE(pHeap, pFree);
395 Log3(("mmHyperAllocChunk: Realigned pFree=%p\n", pFree));
396 }
397
398 /*
399 * Split off a new FREE chunk?
400 */
401 if (pFree->cb >= cb + RT_ALIGN(sizeof(MMHYPERCHUNKFREE), MMHYPER_HEAP_ALIGN_MIN))
402 {
403 /*
404 * Move the FREE chunk up to make room for the new USED chunk.
405 */
406 const int off = cb + sizeof(MMHYPERCHUNK);
407 PMMHYPERCHUNKFREE pNew = (PMMHYPERCHUNKFREE)((char *)&pFree->core + off);
408 *pNew = *pFree;
409 pNew->cb -= off;
410 if (pNew->core.offNext)
411 {
412 pNew->core.offNext -= off;
413 PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pNew + pNew->core.offNext);
414 MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pNew->core.offNext);
415 ASSERT_CHUNK(pHeap, pNext);
416 }
417 pNew->core.offPrev = -off;
418 MMHYPERCHUNK_SET_TYPE(pNew, MMHYPERCHUNK_FLAGS_FREE);
419
420 if (pNew->offNext)
421 {
422 pNew->offNext -= off;
423 PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pNew + pNew->offNext);
424 pNext->offPrev = -(int32_t)pNew->offNext;
425 ASSERT_CHUNK_FREE(pHeap, pNext);
426 }
427 else
428 pHeap->offFreeTail += off;
429 if (pNew->offPrev)
430 {
431 pNew->offPrev -= off;
432 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pNew + pNew->offPrev);
433 pPrev->offNext = -pNew->offPrev;
434 ASSERT_CHUNK_FREE(pHeap, pPrev);
435 }
436 else
437 pHeap->offFreeHead += off;
438 pNew->core.offHeap = (uintptr_t)pHeap - (uintptr_t)pNew;
439 pNew->core.offStat = 0;
440 ASSERT_CHUNK_FREE(pHeap, pNew);
441
442 /*
443 * Update the old FREE node making it a USED node.
444 */
445 pFree->core.offNext = off;
446 MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_USED);
447
448
449 Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree,
450 pHeap->cbFree - (cb + sizeof(MMHYPERCHUNK)), -(int)(cb + sizeof(MMHYPERCHUNK))));
451 pHeap->cbFree -= (uint32_t)(cb + sizeof(MMHYPERCHUNK));
452 pRet = &pFree->core;
453 ASSERT_CHUNK(pHeap, &pFree->core);
454 Log3(("mmHyperAllocChunk: Created free chunk pNew=%p cb=%d\n", pNew, pNew->cb));
455 }
456 else
457 {
458 /*
459 * Link out of free list.
460 */
461 if (pFree->offNext)
462 {
463 PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext);
464 if (pFree->offPrev)
465 {
466 pNext->offPrev += pFree->offPrev;
467 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
468 pPrev->offNext += pFree->offNext;
469 ASSERT_CHUNK_FREE(pHeap, pPrev);
470 }
471 else
472 {
473 pHeap->offFreeHead += pFree->offNext;
474 pNext->offPrev = 0;
475 }
476 ASSERT_CHUNK_FREE(pHeap, pNext);
477 }
478 else
479 {
480 if (pFree->offPrev)
481 {
482 pHeap->offFreeTail += pFree->offPrev;
483 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
484 pPrev->offNext = 0;
485 ASSERT_CHUNK_FREE(pHeap, pPrev);
486 }
487 else
488 {
489 pHeap->offFreeHead = NIL_OFFSET;
490 pHeap->offFreeTail = NIL_OFFSET;
491 }
492 }
493
494 Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree,
495 pHeap->cbFree - pFree->cb, -(int32_t)pFree->cb));
496 pHeap->cbFree -= pFree->cb;
497 MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_USED);
498 pRet = &pFree->core;
499 ASSERT_CHUNK(pHeap, &pFree->core);
500 Log3(("mmHyperAllocChunk: Converted free chunk %p to used chunk.\n", pFree));
501 }
502 Log3(("mmHyperAllocChunk: Returning %p\n", pRet));
503 break;
504 }
505 }
506
507 /* next */
508 pFree = pFree->offNext ? (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext) : NULL;
509 }
510
511#ifdef MMHYPER_HEAP_STRICT_FENCE
512 uint32_t *pu32End = (uint32_t *)((uint8_t *)(pRet + 1) + cb);
513 uint32_t *pu32EndReal = pRet->offNext
514 ? (uint32_t *)((uint8_t *)pRet + pRet->offNext)
515 : (uint32_t *)(pHeap->CTXSUFF(pbHeap) + pHeap->cbHeap);
516 cbFence += (uintptr_t)pu32EndReal - (uintptr_t)pu32End; Assert(!(cbFence & 0x3));
517 ASMMemFill32((uint8_t *)pu32EndReal - cbFence, cbFence, MMHYPER_HEAP_STRICT_FENCE_U32);
518 pu32EndReal[-1] = cbFence;
519#endif
520#ifdef MMHYPER_HEAP_STRICT
521 mmHyperHeapCheck(pHeap);
522#endif
523 return pRet;
524}
525
526
527/**
528 * Allocates one or more pages of memory from the specified heap.
529 * The caller validates the parameters of this request.
530 *
531 * @returns Pointer to the allocated chunk.
532 * @returns NULL on failure.
533 * @param pHeap The heap.
534 * @param cb Size of the memory block to allocate.
535 * @internal
536 */
537static void *mmHyperAllocPages(PMMHYPERHEAP pHeap, uint32_t cb)
538{
539 Log3(("mmHyperAllocPages: Enter cb=%#x\n", cb));
540
541#ifdef MMHYPER_HEAP_STRICT
542 mmHyperHeapCheck(pHeap);
543#endif
544
545 /*
546 * Check if there are any free chunks. (NIL_OFFSET use/not-use forces this check)
547 */
548 if (pHeap->offFreeHead == NIL_OFFSET)
549 return NULL;
550
551 /*
552 * Page aligned chunks.
553 *
554 * Page aligned chunks can only be allocated from the last FREE chunk.
555 * This is for reasons of simplicity and fragmentation. Page aligned memory
556 * must also be allocated in page aligned sizes. Page aligned memory cannot
557 * be freed either.
558 *
559 * So, for this to work, the last FREE chunk needs to end on a page aligned
560 * boundrary.
561 */
562 PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)((char *)CTXSUFF(pHeap->pbHeap) + pHeap->offFreeTail);
563 ASSERT_CHUNK_FREE(pHeap, pFree);
564 if ( (((uintptr_t)(&pFree->core + 1) + pFree->cb) & (PAGE_OFFSET_MASK - 1))
565 || pFree->cb + sizeof(MMHYPERCHUNK) < cb)
566 {
567 Log3(("mmHyperAllocPages: Not enough/no page aligned memory!\n"));
568 return NULL;
569 }
570
571 void *pvRet;
572 if (pFree->cb > cb)
573 {
574 /*
575 * Simple, just cut the top of the free node and return it.
576 */
577 pFree->cb -= cb;
578 pvRet = (char *)(&pFree->core + 1) + pFree->cb;
579 AssertMsg(RT_ALIGN_P(pvRet, PAGE_SIZE) == pvRet, ("pvRet=%p cb=%#x pFree=%p pFree->cb=%#x\n", pvRet, cb, pFree, pFree->cb));
580 Log3(("mmHyperAllocPages: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - cb, -(int)cb));
581 pHeap->cbFree -= cb;
582 ASSERT_CHUNK_FREE(pHeap, pFree);
583 Log3(("mmHyperAllocPages: Allocated from pFree=%p new pFree->cb=%d\n", pFree, pFree->cb));
584 }
585 else
586 {
587 /*
588 * Unlink the FREE node.
589 */
590 pvRet = (char *)(&pFree->core + 1) + pFree->cb - cb;
591 Log3(("mmHyperAllocPages: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pFree->cb, -(int32_t)pFree->cb));
592 pHeap->cbFree -= pFree->cb;
593
594 /* a scrap of spare memory (unlikely)? add it to the sprevious chunk. */
595 if (pvRet != (void *)pFree)
596 {
597 AssertMsg(MMHYPERCHUNK_GET_OFFPREV(&pFree->core), ("How the *beep* did someone manage to allocated up all the heap with page aligned memory?!?\n"));
598 PMMHYPERCHUNK pPrev = (PMMHYPERCHUNK)((char *)pFree + MMHYPERCHUNK_GET_OFFPREV(&pFree->core));
599 pPrev->offNext += (uintptr_t)pvRet - (uintptr_t)pFree;
600 AssertMsg(!MMHYPERCHUNK_ISFREE(pPrev), ("Free bug?\n"));
601#ifdef VBOX_WITH_STATISTICS
602 PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pPrev + pPrev->offStat);
603 pStat->cbAllocated += (uintptr_t)pvRet - (uintptr_t)pFree;
604 pStat->cbCurAllocated += (uintptr_t)pvRet - (uintptr_t)pFree;
605#endif
606 Log3(("mmHyperAllocPages: Added %d to %p (page align)\n", (uintptr_t)pvRet - (uintptr_t)pFree, pFree));
607 }
608
609 /* unlink from FREE chain. */
610 if (pFree->offPrev)
611 {
612 pHeap->offFreeTail += pFree->offPrev;
613 ((PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev))->offNext = 0;
614 }
615 else
616 {
617 pHeap->offFreeTail = NIL_OFFSET;
618 pHeap->offFreeHead = NIL_OFFSET;
619 }
620 Log3(("mmHyperAllocPages: Unlinked pFree=%d\n", pFree));
621 }
622 pHeap->offPageAligned = (uintptr_t)pvRet - (uintptr_t)CTXSUFF(pHeap->pbHeap);
623 Log3(("mmHyperAllocPages: Returning %p (page aligned)\n", pvRet));
624
625#ifdef MMHYPER_HEAP_STRICT
626 mmHyperHeapCheck(pHeap);
627#endif
628 return pvRet;
629}
630
631
632#ifdef VBOX_WITH_STATISTICS
633/**
634 * Get the statistic record for a tag.
635 *
636 * @returns Pointer to a stat record.
637 * @returns NULL on failure.
638 * @param pHeap The heap.
639 * @param enmTag The tag.
640 */
641static PMMHYPERSTAT mmHyperStat(PMMHYPERHEAP pHeap, MMTAG enmTag)
642{
643 /* try look it up first. */
644 PMMHYPERSTAT pStat = (PMMHYPERSTAT)RTAvloGCPhysGet(&pHeap->HyperHeapStatTree, enmTag);
645 if (!pStat)
646 {
647 /* try allocate a new one */
648 PMMHYPERCHUNK pChunk = mmHyperAllocChunk(pHeap, RT_ALIGN(sizeof(*pStat), MMHYPER_HEAP_ALIGN_MIN), MMHYPER_HEAP_ALIGN_MIN);
649 if (!pChunk)
650 return NULL;
651 pStat = (PMMHYPERSTAT)(pChunk + 1);
652 pChunk->offStat = (uintptr_t)pStat - (uintptr_t)pChunk;
653
654 ASMMemZero32(pStat, sizeof(*pStat));
655 pStat->Core.Key = enmTag;
656 RTAvloGCPhysInsert(&pHeap->HyperHeapStatTree, &pStat->Core);
657 }
658 if (!pStat->fRegistered)
659 {
660#ifdef IN_RING3
661 mmR3HyperStatRegisterOne(pHeap->pVMHC, pStat);
662#else
663 /** @todo schedule a HC action. */
664#endif
665 }
666 return pStat;
667}
668
669#ifdef IN_RING3
670/**
671 * Registers statistics with STAM.
672 *
673 */
674static void mmR3HyperStatRegisterOne(PVM pVM, PMMHYPERSTAT pStat)
675{
676 if (pStat->fRegistered)
677 return;
678 const char *pszTag = mmR3GetTagName((MMTAG)pStat->Core.Key);
679
680 char szName[128];
681 RTStrPrintf(szName, sizeof(szName), "/MM/HyperHeap/%s/cAllocations", pszTag);
682 STAMR3Register(pVM, &pStat->cAllocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_COUNT, "Number of alloc calls.");
683
684 RTStrPrintf(szName, sizeof(szName), "/MM/HyperHeap/%s/cFrees", pszTag);
685 STAMR3Register(pVM, &pStat->cFrees, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_COUNT, "Number of free calls.");
686
687 RTStrPrintf(szName, sizeof(szName), "/MM/HyperHeap/%s/cFailures", pszTag);
688 STAMR3Register(pVM, &pStat->cFailures, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_COUNT, "Number of failures.");
689
690 RTStrPrintf(szName, sizeof(szName), "/MM/HyperHeap/%s/cbAllocated", pszTag);
691 STAMR3Register(pVM, &pStat->cbAllocated, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_BYTES, "Total number of allocated bytes.");
692
693 RTStrPrintf(szName, sizeof(szName), "/MM/HyperHeap/%s/cbFreed", pszTag);
694 STAMR3Register(pVM, &pStat->cbFreed, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_BYTES, "Total number of freed bytes.");
695
696 RTStrPrintf(szName, sizeof(szName), "/MM/HyperHeap/%s/cbCurAllocated", pszTag);
697 STAMR3Register(pVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_BYTES, "Number of bytes currently allocated.");
698
699 RTStrPrintf(szName, sizeof(szName), "/MM/HyperHeap/%s/cbMaxAllocated", pszTag);
700 STAMR3Register(pVM, &pStat->cbMaxAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_BYTES, "Max number of bytes allocated at the same time.");
701
702 pStat->fRegistered = true;
703}
704#endif
705
706#endif
707
708
709/**
710 * Free memory allocated using MMHyperAlloc().
711 * The caller validates the parameters of this request.
712 *
713 * @returns VBox status code.
714 * @param pVM The VM to operate on.
715 * @param pv The memory to free.
716 * @remark Try avoid free hyper memory.
717 */
718MMDECL(int) MMHyperFree(PVM pVM, void *pv)
719{
720 Log2(("MMHyperFree: pv=%p\n", pv));
721 if (!pv)
722 return VINF_SUCCESS;
723 AssertMsgReturn(RT_ALIGN_P(pv, MMHYPER_HEAP_ALIGN_MIN) == pv,
724 ("Invalid pointer %p!\n", pv),
725 VERR_INVALID_POINTER);
726
727 /*
728 * Get the heap and stats.
729 * Validate the chunk at the same time.
730 */
731 PMMHYPERCHUNK pChunk = (PMMHYPERCHUNK)((PMMHYPERCHUNK)pv - 1);
732
733 AssertMsgReturn( (uintptr_t)pChunk + pChunk->offNext >= (uintptr_t)pChunk
734 || RT_ALIGN_32(pChunk->offNext, MMHYPER_HEAP_ALIGN_MIN) != pChunk->offNext,
735 ("%p: offNext=%#RX32\n", pv, pChunk->offNext),
736 VERR_INVALID_POINTER);
737
738 AssertMsgReturn(MMHYPERCHUNK_ISUSED(pChunk),
739 ("%p: Not used!\n", pv),
740 VERR_INVALID_POINTER);
741
742 int32_t offPrev = MMHYPERCHUNK_GET_OFFPREV(pChunk);
743 AssertMsgReturn( (uintptr_t)pChunk + offPrev <= (uintptr_t)pChunk
744 && !((uint32_t)-offPrev & (MMHYPER_HEAP_ALIGN_MIN - 1)),
745 ("%p: offPrev=%#RX32!\n", pv, offPrev),
746 VERR_INVALID_POINTER);
747
748 /* statistics */
749#ifdef VBOX_WITH_STATISTICS
750 PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pChunk + pChunk->offStat);
751 AssertMsgReturn( RT_ALIGN_P(pStat, MMHYPER_HEAP_ALIGN_MIN) == (void *)pStat
752 && pChunk->offStat,
753 ("%p: offStat=%#RX32!\n", pv, pChunk->offStat),
754 VERR_INVALID_POINTER);
755#else
756 AssertMsgReturn(!pChunk->offStat,
757 ("%p: offStat=%#RX32!\n", pv, pChunk->offStat),
758 VERR_INVALID_POINTER);
759#endif
760
761 /* The heap structure. */
762 PMMHYPERHEAP pHeap = (PMMHYPERHEAP)((uintptr_t)pChunk + pChunk->offHeap);
763 AssertMsgReturn( !((uintptr_t)pHeap & PAGE_OFFSET_MASK)
764 && pChunk->offHeap,
765 ("%p: pHeap=%#x offHeap=%RX32\n", pv, pHeap->u32Magic, pChunk->offHeap),
766 VERR_INVALID_POINTER);
767
768 AssertMsgReturn(pHeap->u32Magic == MMHYPERHEAP_MAGIC,
769 ("%p: u32Magic=%#x\n", pv, pHeap->u32Magic),
770 VERR_INVALID_POINTER);
771Assert(pHeap == CTXSUFF(pVM->mm.s.pHyperHeap));
772
773 /* Some more verifications using additional info from pHeap. */
774 AssertMsgReturn((uintptr_t)pChunk + offPrev >= (uintptr_t)CTXSUFF(pHeap->pbHeap),
775 ("%p: offPrev=%#RX32!\n", pv, offPrev),
776 VERR_INVALID_POINTER);
777
778 AssertMsgReturn(pChunk->offNext < pHeap->cbHeap,
779 ("%p: offNext=%#RX32!\n", pv, pChunk->offNext),
780 VERR_INVALID_POINTER);
781
782 AssertMsgReturn( (uintptr_t)pv - (uintptr_t)CTXSUFF(pHeap->pbHeap) <= pHeap->offPageAligned,
783 ("Invalid pointer %p! (heap: %p-%p)\n", pv, CTXSUFF(pHeap->pbHeap),
784 (char *)CTXSUFF(pHeap->pbHeap) + pHeap->offPageAligned),
785 VERR_INVALID_POINTER);
786
787#ifdef MMHYPER_HEAP_STRICT
788 mmHyperHeapCheck(pHeap);
789#endif
790
791#if defined(VBOX_WITH_STATISTICS) || defined(MMHYPER_HEAP_FREE_POISON)
792 /* calc block size. */
793 const uint32_t cbChunk = pChunk->offNext
794 ? pChunk->offNext
795 : CTXSUFF(pHeap->pbHeap) + pHeap->offPageAligned - (uint8_t *)pChunk;
796#endif
797#ifdef MMHYPER_HEAP_FREE_POISON
798 /* poison the block */
799 memset(pChunk + 1, MMHYPER_HEAP_FREE_POISON, cbChunk - sizeof(*pChunk));
800#endif
801
802#ifdef MMHYPER_HEAP_FREE_DELAY
803# ifdef MMHYPER_HEAP_FREE_POISON
804 /*
805 * Check poison.
806 */
807 unsigned i = ELEMENTS(pHeap->aDelayedFrees);
808 while (i-- > 0)
809 if (pHeap->aDelayedFrees[i].offChunk)
810 {
811 PMMHYPERCHUNK pCur = (PMMHYPERCHUNK)((uintptr_t)pHeap + pHeap->aDelayedFrees[i].offChunk);
812 const size_t cb = pCur->offNext
813 ? pCur->offNext - sizeof(*pCur)
814 : CTXSUFF(pHeap->pbHeap) + pHeap->offPageAligned - (uint8_t *)pCur - sizeof(*pCur);
815 uint8_t *pab = (uint8_t *)(pCur + 1);
816 for (unsigned off = 0; off < cb; off++)
817 AssertReleaseMsg(pab[off] == 0xCB,
818 ("caller=%RTptr cb=%#zx off=%#x: %.*Rhxs\n",
819 pHeap->aDelayedFrees[i].uCaller, cb, off, RT_MIN(cb - off, 32), &pab[off]));
820 }
821# endif /* MMHYPER_HEAP_FREE_POISON */
822
823 /*
824 * Delayed freeing.
825 */
826 int rc = VINF_SUCCESS;
827 if (pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk)
828 {
829 PMMHYPERCHUNK pChunkFree = (PMMHYPERCHUNK)((uintptr_t)pHeap + pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk);
830 rc = mmHyperFree(pHeap, pChunkFree);
831 }
832 pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk = (uintptr_t)pChunk - (uintptr_t)pHeap;
833 pHeap->aDelayedFrees[pHeap->iDelayedFree].uCaller = (uintptr_t)ASMReturnAddress();
834 pHeap->iDelayedFree = (pHeap->iDelayedFree + 1) % ELEMENTS(pHeap->aDelayedFrees);
835
836#else /* !MMHYPER_HEAP_FREE_POISON */
837 /*
838 * Call the worker.
839 */
840 int rc = mmHyperFree(pHeap, pChunk);
841#endif /* !MMHYPER_HEAP_FREE_POISON */
842
843 /*
844 * Update statistics.
845 */
846#ifdef VBOX_WITH_STATISTICS
847 pStat->cFrees++;
848 if (VBOX_SUCCESS(rc))
849 {
850 pStat->cbFreed += cbChunk;
851 pStat->cbCurAllocated -= cbChunk;
852 }
853 else
854 pStat->cFailures++;
855#endif
856
857 return rc;
858}
859
860
861/**
862 * Free memory a memory chunk.
863 *
864 * @returns VBox status code.
865 * @param pHeap The heap.
866 * @param pChunk The memory chunk to free.
867 */
868static int mmHyperFree(PMMHYPERHEAP pHeap, PMMHYPERCHUNK pChunk)
869{
870 Log3(("mmHyperFree: Enter pHeap=%p pChunk=%p\n", pHeap, pChunk));
871 PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)pChunk;
872
873 /*
874 * Insert into the free list (which is sorted on address).
875 *
876 * We'll search towards the end of the heap to locate the
877 * closest FREE chunk.
878 */
879 PMMHYPERCHUNKFREE pLeft = NULL;
880 PMMHYPERCHUNKFREE pRight = NULL;
881 if (pHeap->offFreeTail != NIL_OFFSET)
882 {
883 if (pFree->core.offNext)
884 {
885 pRight = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->core.offNext);
886 ASSERT_CHUNK(pHeap, &pRight->core);
887 while (!MMHYPERCHUNK_ISFREE(&pRight->core))
888 {
889 if (!pRight->core.offNext)
890 {
891 pRight = NULL;
892 break;
893 }
894 pRight = (PMMHYPERCHUNKFREE)((char *)pRight + pRight->core.offNext);
895 ASSERT_CHUNK(pHeap, &pRight->core);
896 }
897 }
898 if (!pRight)
899 pRight = (PMMHYPERCHUNKFREE)((char *)CTXSUFF(pHeap->pbHeap) + pHeap->offFreeTail); /** @todo this can't be correct! 'pLeft = .. ; else' I think */
900 if (pRight)
901 {
902 ASSERT_CHUNK_FREE(pHeap, pRight);
903 if (pRight->offPrev)
904 {
905 pLeft = (PMMHYPERCHUNKFREE)((char *)pRight + pRight->offPrev);
906 ASSERT_CHUNK_FREE(pHeap, pLeft);
907 }
908 }
909 }
910 if (pLeft == pFree)
911 {
912 AssertMsgFailed(("Freed twice! pv=%p (pChunk=%p)\n", pChunk + 1, pChunk));
913 return VERR_INVALID_POINTER;
914 }
915 pChunk->offStat = 0;
916
917 /*
918 * Head free chunk list?
919 */
920 if (!pLeft)
921 {
922 MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_FREE);
923 pFree->offPrev = 0;
924 pHeap->offFreeHead = (uintptr_t)pFree - (uintptr_t)CTXSUFF(pHeap->pbHeap);
925 if (pRight)
926 {
927 pFree->offNext = (uintptr_t)pRight - (uintptr_t)pFree;
928 pRight->offPrev = -(int32_t)pFree->offNext;
929 }
930 else
931 {
932 pFree->offNext = 0;
933 pHeap->offFreeTail = pHeap->offFreeHead;
934 }
935 Log3(("mmHyperFree: Inserted %p at head of free chain.\n", pFree));
936 }
937 else
938 {
939 /*
940 * Can we merge with left hand free chunk?
941 */
942 if ((char *)pLeft + pLeft->core.offNext == (char *)pFree)
943 {
944 if (pFree->core.offNext)
945 {
946 pLeft->core.offNext = pLeft->core.offNext + pFree->core.offNext;
947 MMHYPERCHUNK_SET_OFFPREV(((PMMHYPERCHUNK)((char *)pLeft + pLeft->core.offNext)), -(int32_t)pLeft->core.offNext);
948 }
949 else
950 pLeft->core.offNext = 0;
951 pFree = pLeft;
952 Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pLeft->cb, -(int32_t)pLeft->cb));
953 pHeap->cbFree -= pLeft->cb;
954 Log3(("mmHyperFree: Merging %p into %p (cb=%d).\n", pFree, pLeft, pLeft->cb));
955 }
956 /*
957 * No, just link it into the free list then.
958 */
959 else
960 {
961 MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_FREE);
962 pFree->offPrev = (uintptr_t)pLeft - (uintptr_t)pFree;
963 pLeft->offNext = -pFree->offPrev;
964 if (pRight)
965 {
966 pFree->offNext = (uintptr_t)pRight - (uintptr_t)pFree;
967 pRight->offPrev = -(int32_t)pFree->offNext;
968 }
969 else
970 {
971 pFree->offNext = 0;
972 pHeap->offFreeTail = (uintptr_t)pFree - (uintptr_t)CTXSUFF(pHeap->pbHeap);
973 }
974 Log3(("mmHyperFree: Inserted %p after %p in free list.\n", pFree, pLeft));
975 }
976 }
977
978 /*
979 * Can we merge with right hand free chunk?
980 */
981 if (pRight && (char *)pRight == (char *)pFree + pFree->core.offNext)
982 {
983 /* core */
984 if (pRight->core.offNext)
985 {
986 pFree->core.offNext += pRight->core.offNext;
987 PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pFree + pFree->core.offNext);
988 MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pFree->core.offNext);
989 ASSERT_CHUNK(pHeap, pNext);
990 }
991 else
992 pFree->core.offNext = 0;
993
994 /* free */
995 if (pRight->offNext)
996 {
997 pFree->offNext += pRight->offNext;
998 ((PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext))->offPrev = -(int32_t)pFree->offNext;
999 }
1000 else
1001 {
1002 pFree->offNext = 0;
1003 pHeap->offFreeTail = (uintptr_t)pFree - (uintptr_t)CTXSUFF(pHeap->pbHeap);
1004 }
1005 Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pRight->cb, -(int32_t)pRight->cb));
1006 pHeap->cbFree -= pRight->cb;
1007 Log3(("mmHyperFree: Merged %p (cb=%d) into %p.\n", pRight, pRight->cb, pFree));
1008 }
1009
1010 /* calculate the size. */
1011 if (pFree->core.offNext)
1012 pFree->cb = pFree->core.offNext - sizeof(MMHYPERCHUNK);
1013 else
1014 pFree->cb = pHeap->offPageAligned - ((uintptr_t)pFree - (uintptr_t)CTXSUFF(pHeap->pbHeap)) - sizeof(MMHYPERCHUNK);
1015 Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree + pFree->cb, pFree->cb));
1016 pHeap->cbFree += pFree->cb;
1017 ASSERT_CHUNK_FREE(pHeap, pFree);
1018
1019#ifdef MMHYPER_HEAP_STRICT
1020 mmHyperHeapCheck(pHeap);
1021#endif
1022 return VINF_SUCCESS;
1023}
1024
1025
1026#if defined(DEBUG) || defined(MMHYPER_HEAP_STRICT)
1027/**
1028 * Dumps a heap chunk to the log.
1029 *
1030 * @param pHeap Pointer to the heap.
1031 * @param pCur Pointer to the chunk.
1032 */
1033static void mmHyperHeapDumpOne(PMMHYPERHEAP pHeap, PMMHYPERCHUNKFREE pCur)
1034{
1035 if (MMHYPERCHUNK_ISUSED(&pCur->core))
1036 {
1037 if (pCur->core.offStat)
1038 {
1039 PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pCur + pCur->core.offStat);
1040 const char *pszSelf = pCur->core.offStat == sizeof(MMHYPERCHUNK) ? " stat record" : "";
1041#ifdef IN_RING3
1042 Log(("%p %06x USED offNext=%06x offPrev=-%06x %s%s\n",
1043 pCur, (uintptr_t)pCur - (uintptr_t)CTXSUFF(pHeap->pbHeap),
1044 pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1045 mmR3GetTagName((MMTAG)pStat->Core.Key), pszSelf));
1046#else
1047 Log(("%p %06x USED offNext=%06x offPrev=-%06x %d%s\n",
1048 pCur, (uintptr_t)pCur - (uintptr_t)CTXSUFF(pHeap->pbHeap),
1049 pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1050 (MMTAG)pStat->Core.Key, pszSelf));
1051#endif
1052 }
1053 else
1054 Log(("%p %06x USED offNext=%06x offPrev=-%06x\n",
1055 pCur, (uintptr_t)pCur - (uintptr_t)CTXSUFF(pHeap->pbHeap),
1056 pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core)));
1057 }
1058 else
1059 Log(("%p %06x FREE offNext=%06x offPrev=-%06x : cb=%06x offNext=%06x offPrev=-%06x\n",
1060 pCur, (uintptr_t)pCur - (uintptr_t)CTXSUFF(pHeap->pbHeap),
1061 pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core), pCur->cb, pCur->offNext, pCur->offPrev));
1062}
1063#endif /* DEBUG || MMHYPER_HEAP_STRICT */
1064
1065
1066#ifdef MMHYPER_HEAP_STRICT
1067/**
1068 * Internal consitency check.
1069 */
1070static void mmHyperHeapCheck(PMMHYPERHEAP pHeap)
1071{
1072 PMMHYPERCHUNKFREE pPrev = NULL;
1073 PMMHYPERCHUNKFREE pCur = (PMMHYPERCHUNKFREE)CTXSUFF(pHeap->pbHeap);
1074 for (;;)
1075 {
1076 if (MMHYPERCHUNK_ISUSED(&pCur->core))
1077 ASSERT_CHUNK_USED(pHeap, &pCur->core);
1078 else
1079 ASSERT_CHUNK_FREE(pHeap, pCur);
1080 if (pPrev)
1081 AssertMsg((int32_t)pPrev->core.offNext == -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1082 ("pPrev->core.offNext=%d offPrev=%d\n", pPrev->core.offNext, MMHYPERCHUNK_GET_OFFPREV(&pCur->core)));
1083
1084# ifdef MMHYPER_HEAP_STRICT_FENCE
1085 uint32_t off = (uint8_t *)pCur - CTXSUFF(pHeap->pbHeap);
1086 if ( MMHYPERCHUNK_ISUSED(&pCur->core)
1087 && off < pHeap->offPageAligned)
1088 {
1089 uint32_t cbCur = pCur->core.offNext
1090 ? pCur->core.offNext
1091 : pHeap->cbHeap - off;
1092 uint32_t *pu32End = ((uint32_t *)((uint8_t *)pCur + cbCur));
1093 uint32_t cbFence = pu32End[-1];
1094 if (RT_UNLIKELY( cbFence >= cbCur - sizeof(*pCur)
1095 || cbFence < MMHYPER_HEAP_STRICT_FENCE_SIZE))
1096 {
1097 mmHyperHeapDumpOne(pHeap, pCur);
1098 Assert(cbFence < cbCur - sizeof(*pCur));
1099 Assert(cbFence >= MMHYPER_HEAP_STRICT_FENCE_SIZE);
1100 }
1101
1102 uint32_t *pu32Bad = ASMMemIsAllU32((uint8_t *)pu32End - cbFence, cbFence - sizeof(uint32_t), MMHYPER_HEAP_STRICT_FENCE_U32);
1103 if (RT_UNLIKELY(pu32Bad))
1104 {
1105 mmHyperHeapDumpOne(pHeap, pCur);
1106 Assert(!pu32Bad);
1107 }
1108 }
1109# endif
1110
1111 /* next */
1112 if (!pCur->core.offNext)
1113 break;
1114 pPrev = pCur;
1115 pCur = (PMMHYPERCHUNKFREE)((char *)pCur + pCur->core.offNext);
1116 }
1117}
1118#endif
1119
1120
1121/**
1122 * Performs consistency checks on the heap if MMHYPER_HEAP_STRICT was
1123 * defined at build time.
1124 *
1125 * @param pVM Pointer to the shared VM structure.
1126 */
1127MMDECL(void) MMHyperHeapCheck(PVM pVM)
1128{
1129#ifdef MMHYPER_HEAP_STRICT
1130 mmHyperHeapCheck(CTXSUFF(pVM->mm.s.pHyperHeap));
1131#endif
1132}
1133
1134
1135#ifdef DEBUG
1136/**
1137 * Dumps the hypervisor heap to Log.
1138 * @param pVM VM Handle.
1139 */
1140MMDECL(void) MMHyperHeapDump(PVM pVM)
1141{
1142 Log(("MMHyperHeapDump: *** heap dump - start ***\n"));
1143 PMMHYPERHEAP pHeap = CTXSUFF(pVM->mm.s.pHyperHeap);
1144 PMMHYPERCHUNKFREE pCur = (PMMHYPERCHUNKFREE)CTXSUFF(pHeap->pbHeap);
1145 for (;;)
1146 {
1147 mmHyperHeapDumpOne(pHeap, pCur);
1148
1149 /* next */
1150 if (!pCur->core.offNext)
1151 break;
1152 pCur = (PMMHYPERCHUNKFREE)((char *)pCur + pCur->core.offNext);
1153 }
1154 Log(("MMHyperHeapDump: *** heap dump - end ***\n"));
1155}
1156#endif
1157
1158
1159/**
1160 * Query the amount of free memory in the hypervisor heap.
1161 *
1162 * @returns Number of free bytes in the hypervisor heap.
1163 */
1164MMDECL(size_t) MMHyperHeapGetFreeSize(PVM pVM)
1165{
1166 return CTXSUFF(pVM->mm.s.pHyperHeap)->cbFree;
1167}
1168
1169/**
1170 * Query the size the hypervisor heap.
1171 *
1172 * @returns The size of the hypervisor heap in bytes.
1173 */
1174MMDECL(size_t) MMHyperHeapGetSize(PVM pVM)
1175{
1176 return CTXSUFF(pVM->mm.s.pHyperHeap)->cbHeap;
1177}
1178
1179
1180/**
1181 * Query the address and size the hypervisor memory area.
1182 *
1183 * @returns Base address of the hypervisor area.
1184 * @param pVM VM Handle.
1185 * @param pcb Where to store the size of the hypervisor area. (out)
1186 */
1187MMDECL(RTGCPTR) MMHyperGetArea(PVM pVM, size_t *pcb)
1188{
1189 if (pcb)
1190 *pcb = pVM->mm.s.cbHyperArea;
1191 return pVM->mm.s.pvHyperAreaGC;
1192}
1193
1194
1195/**
1196 * Checks if an address is within the hypervisor memory area.
1197 *
1198 * @returns true if inside.
1199 * @returns false if outside.
1200 * @param pVM VM handle.
1201 * @param GCPtr The pointer to check.
1202 */
1203MMDECL(bool) MMHyperIsInsideArea(PVM pVM, RTGCPTR GCPtr)
1204{
1205 return (RTGCUINTPTR)GCPtr - (RTGCUINTPTR)pVM->mm.s.pvHyperAreaGC < pVM->mm.s.cbHyperArea;
1206}
1207
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette