VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/MMAllHyper.cpp@ 8863

Last change on this file since 8863 was 8155, checked in by vboxsync, 17 years ago

The Big Sun Rebranding Header Change
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 45.6 KB
Line 
1/* $Id: MMAllHyper.cpp 8155 2008-04-18 15:16:47Z vboxsync $ */
2/** @file
3 * MM - Memory Monitor(/Manager) - Hypervisor Memory Area, All Contexts.
4 */
5
6/*
7 * Copyright (C) 2006-2007 Sun Microsystems, Inc.
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18 * Clara, CA 95054 USA or visit http://www.sun.com if you need
19 * additional information or have any questions.
20 */
21
22
23/*******************************************************************************
24* Header Files *
25*******************************************************************************/
26#define LOG_GROUP LOG_GROUP_MM_HYPER_HEAP
27#include <VBox/mm.h>
28#include <VBox/stam.h>
29#include "MMInternal.h"
30#include <VBox/vm.h>
31
32#include <VBox/err.h>
33#include <VBox/param.h>
34#include <iprt/assert.h>
35#include <VBox/log.h>
36#include <iprt/asm.h>
37#include <iprt/string.h>
38
39
40/*******************************************************************************
41* Defined Constants And Macros *
42*******************************************************************************/
43#define ASSERT_L(u1, u2) AssertMsg((u1) < (u2), ("u1=%#x u2=%#x\n", u1, u2))
44#define ASSERT_LE(u1, u2) AssertMsg((u1) <= (u2), ("u1=%#x u2=%#x\n", u1, u2))
45#define ASSERT_GE(u1, u2) AssertMsg((u1) >= (u2), ("u1=%#x u2=%#x\n", u1, u2))
46#define ASSERT_ALIGN(u1) AssertMsg(!((u1) & (MMHYPER_HEAP_ALIGN_MIN - 1)), ("u1=%#x (%d)\n", u1, u1))
47
48#define ASSERT_OFFPREV(pHeap, pChunk) \
49 do { Assert(MMHYPERCHUNK_GET_OFFPREV(pChunk) <= 0); \
50 Assert(MMHYPERCHUNK_GET_OFFPREV(pChunk) >= (intptr_t)CTXSUFF((pHeap)->pbHeap) - (intptr_t)(pChunk)); \
51 AssertMsg( MMHYPERCHUNK_GET_OFFPREV(pChunk) != 0 \
52 || (uint8_t *)(pChunk) == CTXSUFF((pHeap)->pbHeap), \
53 ("pChunk=%p pvHyperHeap=%p\n", (pChunk), CTXSUFF((pHeap)->pbHeap))); \
54 } while (0)
55
56#define ASSERT_OFFNEXT(pHeap, pChunk) \
57 do { ASSERT_ALIGN((pChunk)->offNext); \
58 ASSERT_L((pChunk)->offNext, (uintptr_t)CTXSUFF((pHeap)->pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
59 } while (0)
60
61#define ASSERT_OFFHEAP(pHeap, pChunk) \
62 do { Assert((pChunk)->offHeap); \
63 AssertMsg((PMMHYPERHEAP)((pChunk)->offHeap + (uintptr_t)pChunk) == (pHeap), \
64 ("offHeap=%RX32 pChunk=%p pHeap=%p\n", (pChunk)->offHeap, (pChunk), (pHeap))); \
65 Assert((pHeap)->u32Magic == MMHYPERHEAP_MAGIC); \
66 } while (0)
67
68#ifdef VBOX_WITH_STATISTICS
69#define ASSERT_OFFSTAT(pHeap, pChunk) \
70 do { if (MMHYPERCHUNK_ISFREE(pChunk)) \
71 Assert(!(pChunk)->offStat); \
72 else if ((pChunk)->offStat) \
73 { \
74 Assert((pChunk)->offStat); \
75 AssertMsg(!((pChunk)->offStat & (MMHYPER_HEAP_ALIGN_MIN - 1)), ("offStat=%RX32\n", (pChunk)->offStat)); \
76 uintptr_t uPtr = (uintptr_t)(pChunk)->offStat + (uintptr_t)pChunk; NOREF(uPtr); \
77 AssertMsg(uPtr - (uintptr_t)CTXSUFF((pHeap)->pbHeap) < (pHeap)->offPageAligned, \
78 ("%p - %p < %RX32\n", uPtr, CTXSUFF((pHeap)->pbHeap), (pHeap)->offPageAligned)); \
79 } \
80 } while (0)
81#else
82#define ASSERT_OFFSTAT(pHeap, pChunk) \
83 do { Assert(!(pChunk)->offStat); \
84 } while (0)
85#endif
86
87#define ASSERT_CHUNK(pHeap, pChunk) \
88 do { ASSERT_OFFNEXT(pHeap, pChunk); \
89 ASSERT_OFFPREV(pHeap, pChunk); \
90 ASSERT_OFFHEAP(pHeap, pChunk); \
91 ASSERT_OFFSTAT(pHeap, pChunk); \
92 } while (0)
93#define ASSERT_CHUNK_USED(pHeap, pChunk) \
94 do { ASSERT_OFFNEXT(pHeap, pChunk); \
95 ASSERT_OFFPREV(pHeap, pChunk); \
96 Assert(MMHYPERCHUNK_ISUSED(pChunk)); \
97 } while (0)
98
99#define ASSERT_FREE_OFFPREV(pHeap, pChunk) \
100 do { ASSERT_ALIGN((pChunk)->offPrev); \
101 ASSERT_GE(((pChunk)->offPrev & (MMHYPER_HEAP_ALIGN_MIN - 1)), (intptr_t)CTXSUFF((pHeap)->pbHeap) - (intptr_t)(pChunk)); \
102 Assert((pChunk)->offPrev != MMHYPERCHUNK_GET_OFFPREV(&(pChunk)->core) || !(pChunk)->offPrev); \
103 AssertMsg( (pChunk)->offPrev \
104 || (uintptr_t)(pChunk) - (uintptr_t)CTXSUFF((pHeap)->pbHeap) == (pHeap)->offFreeHead, \
105 ("pChunk=%p offChunk=%#x offFreeHead=%#x\n", (pChunk), (uintptr_t)(pChunk) - (uintptr_t)CTXSUFF((pHeap)->pbHeap),\
106 (pHeap)->offFreeHead)); \
107 } while (0)
108
109#define ASSERT_FREE_OFFNEXT(pHeap, pChunk) \
110 do { ASSERT_ALIGN((pChunk)->offNext); \
111 ASSERT_L((pChunk)->offNext, (uintptr_t)CTXSUFF((pHeap)->pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
112 Assert((pChunk)->offNext != (pChunk)->core.offNext || !(pChunk)->offNext); \
113 AssertMsg( (pChunk)->offNext \
114 || (uintptr_t)(pChunk) - (uintptr_t)CTXSUFF((pHeap)->pbHeap) == (pHeap)->offFreeTail, \
115 ("pChunk=%p offChunk=%#x offFreeTail=%#x\n", (pChunk), (uintptr_t)(pChunk) - (uintptr_t)CTXSUFF((pHeap)->pbHeap), \
116 (pHeap)->offFreeTail)); \
117 } while (0)
118
119#define ASSERT_FREE_CB(pHeap, pChunk) \
120 do { ASSERT_ALIGN((pChunk)->cb); \
121 Assert((pChunk)->cb > 0); \
122 if ((pChunk)->core.offNext) \
123 AssertMsg((pChunk)->cb == ((pChunk)->core.offNext - sizeof(MMHYPERCHUNK)), \
124 ("cb=%d offNext=%d\n", (pChunk)->cb, (pChunk)->core.offNext)); \
125 else \
126 ASSERT_LE((pChunk)->cb, (uintptr_t)CTXSUFF((pHeap)->pbHeap) + (pHeap)->offPageAligned - (uintptr_t)(pChunk)); \
127 } while (0)
128
129#define ASSERT_CHUNK_FREE(pHeap, pChunk) \
130 do { ASSERT_CHUNK(pHeap, &(pChunk)->core); \
131 Assert(MMHYPERCHUNK_ISFREE(pChunk)); \
132 ASSERT_FREE_OFFNEXT(pHeap, pChunk); \
133 ASSERT_FREE_OFFPREV(pHeap, pChunk); \
134 ASSERT_FREE_CB(pHeap, pChunk); \
135 } while (0)
136
137
138/*******************************************************************************
139* Internal Functions *
140*******************************************************************************/
141static PMMHYPERCHUNK mmHyperAllocChunk(PMMHYPERHEAP pHeap, uint32_t cb, unsigned uAlignment);
142static void *mmHyperAllocPages(PMMHYPERHEAP pHeap, uint32_t cb);
143#ifdef VBOX_WITH_STATISTICS
144static PMMHYPERSTAT mmHyperStat(PMMHYPERHEAP pHeap, MMTAG enmTag);
145#ifdef IN_RING3
146static void mmR3HyperStatRegisterOne(PVM pVM, PMMHYPERSTAT pStat);
147#endif
148#endif
149static int mmHyperFree(PMMHYPERHEAP pHeap, PMMHYPERCHUNK pChunk);
150#ifdef MMHYPER_HEAP_STRICT
151static void mmHyperHeapCheck(PMMHYPERHEAP pHeap);
152#endif
153
154
155/**
156 * Allocates memory in the Hypervisor (GC VMM) area.
157 * The returned memory is of course zeroed.
158 *
159 * @returns VBox status code.
160 * @param pVM The VM to operate on.
161 * @param cb Number of bytes to allocate.
162 * @param uAlignment Required memory alignment in bytes.
163 * Values are 0,8,16,32 and PAGE_SIZE.
164 * 0 -> default alignment, i.e. 8 bytes.
165 * @param enmTag The statistics tag.
166 * @param ppv Where to store the address to the allocated
167 * memory.
168 * @remark This is assumed not to be used at times when serialization is required.
169 */
170MMDECL(int) MMHyperAlloc(PVM pVM, size_t cb, unsigned uAlignment, MMTAG enmTag, void **ppv)
171{
172 AssertMsg(cb >= 8, ("Hey! Do you really mean to allocate less than 8 bytes?! cb=%d\n", cb));
173
174 /*
175 * Validate input and adjust it to reasonable values.
176 */
177 if (!uAlignment || uAlignment < MMHYPER_HEAP_ALIGN_MIN)
178 uAlignment = MMHYPER_HEAP_ALIGN_MIN;
179 uint32_t cbAligned;
180 switch (uAlignment)
181 {
182 case 8:
183 case 16:
184 case 32:
185 cbAligned = RT_ALIGN(cb, MMHYPER_HEAP_ALIGN_MIN);
186 if (!cbAligned || cbAligned < cb)
187 {
188 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
189 AssertMsgFailed(("Nice try.\n"));
190 return VERR_INVALID_PARAMETER;
191 }
192 break;
193
194 case PAGE_SIZE:
195 AssertMsg(RT_ALIGN(cb, PAGE_SIZE) == cb, ("The size isn't page aligned. (cb=%#x)\n", cb));
196 cbAligned = RT_ALIGN(cb, PAGE_SIZE);
197 if (!cbAligned)
198 {
199 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
200 AssertMsgFailed(("Nice try.\n"));
201 return VERR_INVALID_PARAMETER;
202 }
203 break;
204
205 default:
206 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_INVALID_PARAMETER\n", cb, uAlignment));
207 AssertMsgFailed(("Invalid alignment %u\n", uAlignment));
208 return VERR_INVALID_PARAMETER;
209 }
210
211
212 /*
213 * Get heap and statisticsStatistics.
214 */
215 PMMHYPERHEAP pHeap = CTXSUFF(pVM->mm.s.pHyperHeap);
216#ifdef VBOX_WITH_STATISTICS
217 PMMHYPERSTAT pStat = mmHyperStat(pHeap, enmTag);
218 if (!pStat)
219 {
220 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_MM_HYPER_NO_MEMORY\n", cb, uAlignment));
221 AssertMsgFailed(("Failed to allocate statistics!\n"));
222 return VERR_MM_HYPER_NO_MEMORY;
223 }
224#endif
225 if (uAlignment < PAGE_SIZE)
226 {
227 /*
228 * Allocate a chunk.
229 */
230 PMMHYPERCHUNK pChunk = mmHyperAllocChunk(pHeap, cbAligned, uAlignment);
231 if (pChunk)
232 {
233#ifdef VBOX_WITH_STATISTICS
234 const uint32_t cbChunk = pChunk->offNext
235 ? pChunk->offNext
236 : CTXSUFF(pHeap->pbHeap) + pHeap->offPageAligned - (uint8_t *)pChunk;
237 pStat->cbAllocated += (uint32_t)cbChunk;
238 pStat->cbCurAllocated += (uint32_t)cbChunk;
239 if (pStat->cbCurAllocated > pStat->cbMaxAllocated)
240 pStat->cbMaxAllocated = pStat->cbCurAllocated;
241 pStat->cAllocations++;
242 pChunk->offStat = (uintptr_t)pStat - (uintptr_t)pChunk;
243#else
244 pChunk->offStat = 0;
245#endif
246 void *pv = pChunk + 1;
247 *ppv = pv;
248 ASMMemZero32(pv, cbAligned);
249 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VINF_SUCCESS and *ppv=%p\n", cb, uAlignment, pv));
250 return VINF_SUCCESS;
251 }
252 }
253 else
254 {
255 /*
256 * Allocate page aligned memory.
257 */
258 void *pv = mmHyperAllocPages(pHeap, cbAligned);
259 if (pv)
260 {
261#ifdef VBOX_WITH_STATISTICS
262 pStat->cbAllocated += cbAligned;
263 pStat->cbCurAllocated += cbAligned;
264 if (pStat->cbCurAllocated > pStat->cbMaxAllocated)
265 pStat->cbMaxAllocated = pStat->cbCurAllocated;
266 pStat->cAllocations++;
267#endif
268 *ppv = pv;
269 /* ASMMemZero32(pv, cbAligned); - not required since memory is alloc-only and SUPPageAlloc zeros it. */
270 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VINF_SUCCESS and *ppv=%p\n", cb, uAlignment, ppv));
271 return VINF_SUCCESS;
272 }
273 }
274
275#ifdef VBOX_WITH_STATISTICS
276 pStat->cAllocations++;
277 pStat->cFailures++;
278#endif
279 Log2(("MMHyperAlloc: cb=%#x uAlignment=%#x returns VERR_MM_HYPER_NO_MEMORY\n", cb, uAlignment));
280 AssertMsgFailed(("Failed to allocate %d bytes!\n", cb));
281 return VERR_MM_HYPER_NO_MEMORY;
282}
283
284
285
286/**
287 * Allocates a chunk of memory from the specified heap.
288 * The caller validates the parameters of this request.
289 *
290 * @returns Pointer to the allocated chunk.
291 * @returns NULL on failure.
292 * @param pHeap The heap.
293 * @param cb Size of the memory block to allocate.
294 * @param uAlignment The alignment specifications for the allocated block.
295 * @internal
296 */
297static PMMHYPERCHUNK mmHyperAllocChunk(PMMHYPERHEAP pHeap, uint32_t cb, unsigned uAlignment)
298{
299 Log3(("mmHyperAllocChunk: Enter cb=%#x uAlignment=%#x\n", cb, uAlignment));
300#ifdef MMHYPER_HEAP_STRICT
301 mmHyperHeapCheck(pHeap);
302#endif
303#ifdef MMHYPER_HEAP_STRICT_FENCE
304 uint32_t cbFence = RT_MAX(MMHYPER_HEAP_STRICT_FENCE_SIZE, uAlignment);
305 cb += cbFence;
306#endif
307
308 /*
309 * Check if there are any free chunks. (NIL_OFFSET use/not-use forces this check)
310 */
311 if (pHeap->offFreeHead == NIL_OFFSET)
312 return NULL;
313
314 /*
315 * Small alignments - from the front of the heap.
316 *
317 * Must split off free chunks at the end to prevent messing up the
318 * last free node which we take the page aligned memory from the top of.
319 */
320 PMMHYPERCHUNK pRet = NULL;
321 PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)((char *)CTXSUFF(pHeap->pbHeap) + pHeap->offFreeHead);
322 while (pFree)
323 {
324 ASSERT_CHUNK_FREE(pHeap, pFree);
325 if (pFree->cb >= cb)
326 {
327 unsigned offAlign = (uintptr_t)(&pFree->core + 1) & (uAlignment - 1);
328 if (offAlign)
329 offAlign = uAlignment - offAlign;
330 if (!offAlign || pFree->cb - offAlign >= cb)
331 {
332 Log3(("mmHyperAllocChunk: Using pFree=%p pFree->cb=%d offAlign=%d\n", pFree, pFree->cb, offAlign));
333
334 /*
335 * Adjust the node in front.
336 * Because of multiple alignments we need to special case allocation of the first block.
337 */
338 if (offAlign)
339 {
340 MMHYPERCHUNKFREE Free = *pFree;
341 if (MMHYPERCHUNK_GET_OFFPREV(&pFree->core))
342 {
343 /* just add a bit of memory to it. */
344 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + MMHYPERCHUNK_GET_OFFPREV(&Free.core));
345 pPrev->core.offNext += offAlign;
346 AssertMsg(!MMHYPERCHUNK_ISFREE(&pPrev->core), ("Impossible!\n"));
347 Log3(("mmHyperAllocChunk: Added %d bytes to %p\n", offAlign, pPrev));
348 }
349 else
350 {
351 /* make new head node, mark it USED for simplisity. */
352 PMMHYPERCHUNK pPrev = (PMMHYPERCHUNK)CTXSUFF(pHeap->pbHeap);
353 Assert(pPrev == &pFree->core);
354 pPrev->offPrev = 0;
355 MMHYPERCHUNK_SET_TYPE(pPrev, MMHYPERCHUNK_FLAGS_USED);
356 pPrev->offNext = offAlign;
357 Log3(("mmHyperAllocChunk: Created new first node of %d bytes\n", offAlign));
358
359 }
360 Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - offAlign, -(int)offAlign));
361 pHeap->cbFree -= offAlign;
362
363 /* Recreate pFree node and adjusting everything... */
364 pFree = (PMMHYPERCHUNKFREE)((char *)pFree + offAlign);
365 *pFree = Free;
366
367 pFree->cb -= offAlign;
368 if (pFree->core.offNext)
369 {
370 pFree->core.offNext -= offAlign;
371 PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pFree + pFree->core.offNext);
372 MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pFree->core.offNext);
373 ASSERT_CHUNK(pHeap, pNext);
374 }
375 if (MMHYPERCHUNK_GET_OFFPREV(&pFree->core))
376 MMHYPERCHUNK_SET_OFFPREV(&pFree->core, MMHYPERCHUNK_GET_OFFPREV(&pFree->core) - offAlign);
377
378 if (pFree->offNext)
379 {
380 pFree->offNext -= offAlign;
381 PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext);
382 pNext->offPrev = -(int32_t)pFree->offNext;
383 ASSERT_CHUNK_FREE(pHeap, pNext);
384 }
385 else
386 pHeap->offFreeTail += offAlign;
387 if (pFree->offPrev)
388 {
389 pFree->offPrev -= offAlign;
390 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
391 pPrev->offNext = -pFree->offPrev;
392 ASSERT_CHUNK_FREE(pHeap, pPrev);
393 }
394 else
395 pHeap->offFreeHead += offAlign;
396 pFree->core.offHeap = (uintptr_t)pHeap - (uintptr_t)pFree;
397 pFree->core.offStat = 0;
398 ASSERT_CHUNK_FREE(pHeap, pFree);
399 Log3(("mmHyperAllocChunk: Realigned pFree=%p\n", pFree));
400 }
401
402 /*
403 * Split off a new FREE chunk?
404 */
405 if (pFree->cb >= cb + RT_ALIGN(sizeof(MMHYPERCHUNKFREE), MMHYPER_HEAP_ALIGN_MIN))
406 {
407 /*
408 * Move the FREE chunk up to make room for the new USED chunk.
409 */
410 const int off = cb + sizeof(MMHYPERCHUNK);
411 PMMHYPERCHUNKFREE pNew = (PMMHYPERCHUNKFREE)((char *)&pFree->core + off);
412 *pNew = *pFree;
413 pNew->cb -= off;
414 if (pNew->core.offNext)
415 {
416 pNew->core.offNext -= off;
417 PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pNew + pNew->core.offNext);
418 MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pNew->core.offNext);
419 ASSERT_CHUNK(pHeap, pNext);
420 }
421 pNew->core.offPrev = -off;
422 MMHYPERCHUNK_SET_TYPE(pNew, MMHYPERCHUNK_FLAGS_FREE);
423
424 if (pNew->offNext)
425 {
426 pNew->offNext -= off;
427 PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pNew + pNew->offNext);
428 pNext->offPrev = -(int32_t)pNew->offNext;
429 ASSERT_CHUNK_FREE(pHeap, pNext);
430 }
431 else
432 pHeap->offFreeTail += off;
433 if (pNew->offPrev)
434 {
435 pNew->offPrev -= off;
436 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pNew + pNew->offPrev);
437 pPrev->offNext = -pNew->offPrev;
438 ASSERT_CHUNK_FREE(pHeap, pPrev);
439 }
440 else
441 pHeap->offFreeHead += off;
442 pNew->core.offHeap = (uintptr_t)pHeap - (uintptr_t)pNew;
443 pNew->core.offStat = 0;
444 ASSERT_CHUNK_FREE(pHeap, pNew);
445
446 /*
447 * Update the old FREE node making it a USED node.
448 */
449 pFree->core.offNext = off;
450 MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_USED);
451
452
453 Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree,
454 pHeap->cbFree - (cb + sizeof(MMHYPERCHUNK)), -(int)(cb + sizeof(MMHYPERCHUNK))));
455 pHeap->cbFree -= (uint32_t)(cb + sizeof(MMHYPERCHUNK));
456 pRet = &pFree->core;
457 ASSERT_CHUNK(pHeap, &pFree->core);
458 Log3(("mmHyperAllocChunk: Created free chunk pNew=%p cb=%d\n", pNew, pNew->cb));
459 }
460 else
461 {
462 /*
463 * Link out of free list.
464 */
465 if (pFree->offNext)
466 {
467 PMMHYPERCHUNKFREE pNext = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext);
468 if (pFree->offPrev)
469 {
470 pNext->offPrev += pFree->offPrev;
471 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
472 pPrev->offNext += pFree->offNext;
473 ASSERT_CHUNK_FREE(pHeap, pPrev);
474 }
475 else
476 {
477 pHeap->offFreeHead += pFree->offNext;
478 pNext->offPrev = 0;
479 }
480 ASSERT_CHUNK_FREE(pHeap, pNext);
481 }
482 else
483 {
484 if (pFree->offPrev)
485 {
486 pHeap->offFreeTail += pFree->offPrev;
487 PMMHYPERCHUNKFREE pPrev = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev);
488 pPrev->offNext = 0;
489 ASSERT_CHUNK_FREE(pHeap, pPrev);
490 }
491 else
492 {
493 pHeap->offFreeHead = NIL_OFFSET;
494 pHeap->offFreeTail = NIL_OFFSET;
495 }
496 }
497
498 Log3(("mmHyperAllocChunk: cbFree %d -> %d (%d)\n", pHeap->cbFree,
499 pHeap->cbFree - pFree->cb, -(int32_t)pFree->cb));
500 pHeap->cbFree -= pFree->cb;
501 MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_USED);
502 pRet = &pFree->core;
503 ASSERT_CHUNK(pHeap, &pFree->core);
504 Log3(("mmHyperAllocChunk: Converted free chunk %p to used chunk.\n", pFree));
505 }
506 Log3(("mmHyperAllocChunk: Returning %p\n", pRet));
507 break;
508 }
509 }
510
511 /* next */
512 pFree = pFree->offNext ? (PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext) : NULL;
513 }
514
515#ifdef MMHYPER_HEAP_STRICT_FENCE
516 uint32_t *pu32End = (uint32_t *)((uint8_t *)(pRet + 1) + cb);
517 uint32_t *pu32EndReal = pRet->offNext
518 ? (uint32_t *)((uint8_t *)pRet + pRet->offNext)
519 : (uint32_t *)(pHeap->CTXSUFF(pbHeap) + pHeap->cbHeap);
520 cbFence += (uintptr_t)pu32EndReal - (uintptr_t)pu32End; Assert(!(cbFence & 0x3));
521 ASMMemFill32((uint8_t *)pu32EndReal - cbFence, cbFence, MMHYPER_HEAP_STRICT_FENCE_U32);
522 pu32EndReal[-1] = cbFence;
523#endif
524#ifdef MMHYPER_HEAP_STRICT
525 mmHyperHeapCheck(pHeap);
526#endif
527 return pRet;
528}
529
530
531/**
532 * Allocates one or more pages of memory from the specified heap.
533 * The caller validates the parameters of this request.
534 *
535 * @returns Pointer to the allocated chunk.
536 * @returns NULL on failure.
537 * @param pHeap The heap.
538 * @param cb Size of the memory block to allocate.
539 * @internal
540 */
541static void *mmHyperAllocPages(PMMHYPERHEAP pHeap, uint32_t cb)
542{
543 Log3(("mmHyperAllocPages: Enter cb=%#x\n", cb));
544
545#ifdef MMHYPER_HEAP_STRICT
546 mmHyperHeapCheck(pHeap);
547#endif
548
549 /*
550 * Check if there are any free chunks. (NIL_OFFSET use/not-use forces this check)
551 */
552 if (pHeap->offFreeHead == NIL_OFFSET)
553 return NULL;
554
555 /*
556 * Page aligned chunks.
557 *
558 * Page aligned chunks can only be allocated from the last FREE chunk.
559 * This is for reasons of simplicity and fragmentation. Page aligned memory
560 * must also be allocated in page aligned sizes. Page aligned memory cannot
561 * be freed either.
562 *
563 * So, for this to work, the last FREE chunk needs to end on a page aligned
564 * boundrary.
565 */
566 PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)((char *)CTXSUFF(pHeap->pbHeap) + pHeap->offFreeTail);
567 ASSERT_CHUNK_FREE(pHeap, pFree);
568 if ( (((uintptr_t)(&pFree->core + 1) + pFree->cb) & (PAGE_OFFSET_MASK - 1))
569 || pFree->cb + sizeof(MMHYPERCHUNK) < cb)
570 {
571 Log3(("mmHyperAllocPages: Not enough/no page aligned memory!\n"));
572 return NULL;
573 }
574
575 void *pvRet;
576 if (pFree->cb > cb)
577 {
578 /*
579 * Simple, just cut the top of the free node and return it.
580 */
581 pFree->cb -= cb;
582 pvRet = (char *)(&pFree->core + 1) + pFree->cb;
583 AssertMsg(RT_ALIGN_P(pvRet, PAGE_SIZE) == pvRet, ("pvRet=%p cb=%#x pFree=%p pFree->cb=%#x\n", pvRet, cb, pFree, pFree->cb));
584 Log3(("mmHyperAllocPages: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - cb, -(int)cb));
585 pHeap->cbFree -= cb;
586 ASSERT_CHUNK_FREE(pHeap, pFree);
587 Log3(("mmHyperAllocPages: Allocated from pFree=%p new pFree->cb=%d\n", pFree, pFree->cb));
588 }
589 else
590 {
591 /*
592 * Unlink the FREE node.
593 */
594 pvRet = (char *)(&pFree->core + 1) + pFree->cb - cb;
595 Log3(("mmHyperAllocPages: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pFree->cb, -(int32_t)pFree->cb));
596 pHeap->cbFree -= pFree->cb;
597
598 /* a scrap of spare memory (unlikely)? add it to the sprevious chunk. */
599 if (pvRet != (void *)pFree)
600 {
601 AssertMsg(MMHYPERCHUNK_GET_OFFPREV(&pFree->core), ("How the *beep* did someone manage to allocated up all the heap with page aligned memory?!?\n"));
602 PMMHYPERCHUNK pPrev = (PMMHYPERCHUNK)((char *)pFree + MMHYPERCHUNK_GET_OFFPREV(&pFree->core));
603 pPrev->offNext += (uintptr_t)pvRet - (uintptr_t)pFree;
604 AssertMsg(!MMHYPERCHUNK_ISFREE(pPrev), ("Free bug?\n"));
605#ifdef VBOX_WITH_STATISTICS
606 PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pPrev + pPrev->offStat);
607 pStat->cbAllocated += (uintptr_t)pvRet - (uintptr_t)pFree;
608 pStat->cbCurAllocated += (uintptr_t)pvRet - (uintptr_t)pFree;
609#endif
610 Log3(("mmHyperAllocPages: Added %d to %p (page align)\n", (uintptr_t)pvRet - (uintptr_t)pFree, pFree));
611 }
612
613 /* unlink from FREE chain. */
614 if (pFree->offPrev)
615 {
616 pHeap->offFreeTail += pFree->offPrev;
617 ((PMMHYPERCHUNKFREE)((char *)pFree + pFree->offPrev))->offNext = 0;
618 }
619 else
620 {
621 pHeap->offFreeTail = NIL_OFFSET;
622 pHeap->offFreeHead = NIL_OFFSET;
623 }
624 Log3(("mmHyperAllocPages: Unlinked pFree=%d\n", pFree));
625 }
626 pHeap->offPageAligned = (uintptr_t)pvRet - (uintptr_t)CTXSUFF(pHeap->pbHeap);
627 Log3(("mmHyperAllocPages: Returning %p (page aligned)\n", pvRet));
628
629#ifdef MMHYPER_HEAP_STRICT
630 mmHyperHeapCheck(pHeap);
631#endif
632 return pvRet;
633}
634
635
636#ifdef VBOX_WITH_STATISTICS
637/**
638 * Get the statistic record for a tag.
639 *
640 * @returns Pointer to a stat record.
641 * @returns NULL on failure.
642 * @param pHeap The heap.
643 * @param enmTag The tag.
644 */
645static PMMHYPERSTAT mmHyperStat(PMMHYPERHEAP pHeap, MMTAG enmTag)
646{
647 /* try look it up first. */
648 PMMHYPERSTAT pStat = (PMMHYPERSTAT)RTAvloGCPhysGet(&pHeap->HyperHeapStatTree, enmTag);
649 if (!pStat)
650 {
651 /* try allocate a new one */
652 PMMHYPERCHUNK pChunk = mmHyperAllocChunk(pHeap, RT_ALIGN(sizeof(*pStat), MMHYPER_HEAP_ALIGN_MIN), MMHYPER_HEAP_ALIGN_MIN);
653 if (!pChunk)
654 return NULL;
655 pStat = (PMMHYPERSTAT)(pChunk + 1);
656 pChunk->offStat = (uintptr_t)pStat - (uintptr_t)pChunk;
657
658 ASMMemZero32(pStat, sizeof(*pStat));
659 pStat->Core.Key = enmTag;
660 RTAvloGCPhysInsert(&pHeap->HyperHeapStatTree, &pStat->Core);
661 }
662 if (!pStat->fRegistered)
663 {
664#ifdef IN_RING3
665 mmR3HyperStatRegisterOne(pHeap->pVMHC, pStat);
666#else
667 /** @todo schedule a HC action. */
668#endif
669 }
670 return pStat;
671}
672
673#ifdef IN_RING3
674/**
675 * Registers statistics with STAM.
676 *
677 */
678static void mmR3HyperStatRegisterOne(PVM pVM, PMMHYPERSTAT pStat)
679{
680 if (pStat->fRegistered)
681 return;
682 const char *pszTag = mmR3GetTagName((MMTAG)pStat->Core.Key);
683
684 char szName[128];
685 RTStrPrintf(szName, sizeof(szName), "/MM/HyperHeap/%s/cAllocations", pszTag);
686 STAMR3Register(pVM, &pStat->cAllocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_COUNT, "Number of alloc calls.");
687
688 RTStrPrintf(szName, sizeof(szName), "/MM/HyperHeap/%s/cFrees", pszTag);
689 STAMR3Register(pVM, &pStat->cFrees, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_COUNT, "Number of free calls.");
690
691 RTStrPrintf(szName, sizeof(szName), "/MM/HyperHeap/%s/cFailures", pszTag);
692 STAMR3Register(pVM, &pStat->cFailures, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_COUNT, "Number of failures.");
693
694 RTStrPrintf(szName, sizeof(szName), "/MM/HyperHeap/%s/cbAllocated", pszTag);
695 STAMR3Register(pVM, &pStat->cbAllocated, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_BYTES, "Total number of allocated bytes.");
696
697 RTStrPrintf(szName, sizeof(szName), "/MM/HyperHeap/%s/cbFreed", pszTag);
698 STAMR3Register(pVM, &pStat->cbFreed, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_BYTES, "Total number of freed bytes.");
699
700 RTStrPrintf(szName, sizeof(szName), "/MM/HyperHeap/%s/cbCurAllocated", pszTag);
701 STAMR3Register(pVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_BYTES, "Number of bytes currently allocated.");
702
703 RTStrPrintf(szName, sizeof(szName), "/MM/HyperHeap/%s/cbMaxAllocated", pszTag);
704 STAMR3Register(pVM, &pStat->cbMaxAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_BYTES, "Max number of bytes allocated at the same time.");
705
706 pStat->fRegistered = true;
707}
708#endif
709
710#endif
711
712
713/**
714 * Free memory allocated using MMHyperAlloc().
715 * The caller validates the parameters of this request.
716 *
717 * @returns VBox status code.
718 * @param pVM The VM to operate on.
719 * @param pv The memory to free.
720 * @remark Try avoid free hyper memory.
721 */
722MMDECL(int) MMHyperFree(PVM pVM, void *pv)
723{
724 Log2(("MMHyperFree: pv=%p\n", pv));
725 if (!pv)
726 return VINF_SUCCESS;
727 AssertMsgReturn(RT_ALIGN_P(pv, MMHYPER_HEAP_ALIGN_MIN) == pv,
728 ("Invalid pointer %p!\n", pv),
729 VERR_INVALID_POINTER);
730
731 /*
732 * Get the heap and stats.
733 * Validate the chunk at the same time.
734 */
735 PMMHYPERCHUNK pChunk = (PMMHYPERCHUNK)((PMMHYPERCHUNK)pv - 1);
736
737 AssertMsgReturn( (uintptr_t)pChunk + pChunk->offNext >= (uintptr_t)pChunk
738 || RT_ALIGN_32(pChunk->offNext, MMHYPER_HEAP_ALIGN_MIN) != pChunk->offNext,
739 ("%p: offNext=%#RX32\n", pv, pChunk->offNext),
740 VERR_INVALID_POINTER);
741
742 AssertMsgReturn(MMHYPERCHUNK_ISUSED(pChunk),
743 ("%p: Not used!\n", pv),
744 VERR_INVALID_POINTER);
745
746 int32_t offPrev = MMHYPERCHUNK_GET_OFFPREV(pChunk);
747 AssertMsgReturn( (uintptr_t)pChunk + offPrev <= (uintptr_t)pChunk
748 && !((uint32_t)-offPrev & (MMHYPER_HEAP_ALIGN_MIN - 1)),
749 ("%p: offPrev=%#RX32!\n", pv, offPrev),
750 VERR_INVALID_POINTER);
751
752 /* statistics */
753#ifdef VBOX_WITH_STATISTICS
754 PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pChunk + pChunk->offStat);
755 AssertMsgReturn( RT_ALIGN_P(pStat, MMHYPER_HEAP_ALIGN_MIN) == (void *)pStat
756 && pChunk->offStat,
757 ("%p: offStat=%#RX32!\n", pv, pChunk->offStat),
758 VERR_INVALID_POINTER);
759#else
760 AssertMsgReturn(!pChunk->offStat,
761 ("%p: offStat=%#RX32!\n", pv, pChunk->offStat),
762 VERR_INVALID_POINTER);
763#endif
764
765 /* The heap structure. */
766 PMMHYPERHEAP pHeap = (PMMHYPERHEAP)((uintptr_t)pChunk + pChunk->offHeap);
767 AssertMsgReturn( !((uintptr_t)pHeap & PAGE_OFFSET_MASK)
768 && pChunk->offHeap,
769 ("%p: pHeap=%#x offHeap=%RX32\n", pv, pHeap->u32Magic, pChunk->offHeap),
770 VERR_INVALID_POINTER);
771
772 AssertMsgReturn(pHeap->u32Magic == MMHYPERHEAP_MAGIC,
773 ("%p: u32Magic=%#x\n", pv, pHeap->u32Magic),
774 VERR_INVALID_POINTER);
775Assert(pHeap == CTXSUFF(pVM->mm.s.pHyperHeap));
776
777 /* Some more verifications using additional info from pHeap. */
778 AssertMsgReturn((uintptr_t)pChunk + offPrev >= (uintptr_t)CTXSUFF(pHeap->pbHeap),
779 ("%p: offPrev=%#RX32!\n", pv, offPrev),
780 VERR_INVALID_POINTER);
781
782 AssertMsgReturn(pChunk->offNext < pHeap->cbHeap,
783 ("%p: offNext=%#RX32!\n", pv, pChunk->offNext),
784 VERR_INVALID_POINTER);
785
786 AssertMsgReturn( (uintptr_t)pv - (uintptr_t)CTXSUFF(pHeap->pbHeap) <= pHeap->offPageAligned,
787 ("Invalid pointer %p! (heap: %p-%p)\n", pv, CTXSUFF(pHeap->pbHeap),
788 (char *)CTXSUFF(pHeap->pbHeap) + pHeap->offPageAligned),
789 VERR_INVALID_POINTER);
790
791#ifdef MMHYPER_HEAP_STRICT
792 mmHyperHeapCheck(pHeap);
793#endif
794
795#if defined(VBOX_WITH_STATISTICS) || defined(MMHYPER_HEAP_FREE_POISON)
796 /* calc block size. */
797 const uint32_t cbChunk = pChunk->offNext
798 ? pChunk->offNext
799 : CTXSUFF(pHeap->pbHeap) + pHeap->offPageAligned - (uint8_t *)pChunk;
800#endif
801#ifdef MMHYPER_HEAP_FREE_POISON
802 /* poison the block */
803 memset(pChunk + 1, MMHYPER_HEAP_FREE_POISON, cbChunk - sizeof(*pChunk));
804#endif
805
806#ifdef MMHYPER_HEAP_FREE_DELAY
807# ifdef MMHYPER_HEAP_FREE_POISON
808 /*
809 * Check poison.
810 */
811 unsigned i = ELEMENTS(pHeap->aDelayedFrees);
812 while (i-- > 0)
813 if (pHeap->aDelayedFrees[i].offChunk)
814 {
815 PMMHYPERCHUNK pCur = (PMMHYPERCHUNK)((uintptr_t)pHeap + pHeap->aDelayedFrees[i].offChunk);
816 const size_t cb = pCur->offNext
817 ? pCur->offNext - sizeof(*pCur)
818 : CTXSUFF(pHeap->pbHeap) + pHeap->offPageAligned - (uint8_t *)pCur - sizeof(*pCur);
819 uint8_t *pab = (uint8_t *)(pCur + 1);
820 for (unsigned off = 0; off < cb; off++)
821 AssertReleaseMsg(pab[off] == 0xCB,
822 ("caller=%RTptr cb=%#zx off=%#x: %.*Rhxs\n",
823 pHeap->aDelayedFrees[i].uCaller, cb, off, RT_MIN(cb - off, 32), &pab[off]));
824 }
825# endif /* MMHYPER_HEAP_FREE_POISON */
826
827 /*
828 * Delayed freeing.
829 */
830 int rc = VINF_SUCCESS;
831 if (pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk)
832 {
833 PMMHYPERCHUNK pChunkFree = (PMMHYPERCHUNK)((uintptr_t)pHeap + pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk);
834 rc = mmHyperFree(pHeap, pChunkFree);
835 }
836 pHeap->aDelayedFrees[pHeap->iDelayedFree].offChunk = (uintptr_t)pChunk - (uintptr_t)pHeap;
837 pHeap->aDelayedFrees[pHeap->iDelayedFree].uCaller = (uintptr_t)ASMReturnAddress();
838 pHeap->iDelayedFree = (pHeap->iDelayedFree + 1) % ELEMENTS(pHeap->aDelayedFrees);
839
840#else /* !MMHYPER_HEAP_FREE_POISON */
841 /*
842 * Call the worker.
843 */
844 int rc = mmHyperFree(pHeap, pChunk);
845#endif /* !MMHYPER_HEAP_FREE_POISON */
846
847 /*
848 * Update statistics.
849 */
850#ifdef VBOX_WITH_STATISTICS
851 pStat->cFrees++;
852 if (VBOX_SUCCESS(rc))
853 {
854 pStat->cbFreed += cbChunk;
855 pStat->cbCurAllocated -= cbChunk;
856 }
857 else
858 pStat->cFailures++;
859#endif
860
861 return rc;
862}
863
864
865/**
866 * Free memory a memory chunk.
867 *
868 * @returns VBox status code.
869 * @param pHeap The heap.
870 * @param pChunk The memory chunk to free.
871 */
872static int mmHyperFree(PMMHYPERHEAP pHeap, PMMHYPERCHUNK pChunk)
873{
874 Log3(("mmHyperFree: Enter pHeap=%p pChunk=%p\n", pHeap, pChunk));
875 PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)pChunk;
876
877 /*
878 * Insert into the free list (which is sorted on address).
879 *
880 * We'll search towards the end of the heap to locate the
881 * closest FREE chunk.
882 */
883 PMMHYPERCHUNKFREE pLeft = NULL;
884 PMMHYPERCHUNKFREE pRight = NULL;
885 if (pHeap->offFreeTail != NIL_OFFSET)
886 {
887 if (pFree->core.offNext)
888 {
889 pRight = (PMMHYPERCHUNKFREE)((char *)pFree + pFree->core.offNext);
890 ASSERT_CHUNK(pHeap, &pRight->core);
891 while (!MMHYPERCHUNK_ISFREE(&pRight->core))
892 {
893 if (!pRight->core.offNext)
894 {
895 pRight = NULL;
896 break;
897 }
898 pRight = (PMMHYPERCHUNKFREE)((char *)pRight + pRight->core.offNext);
899 ASSERT_CHUNK(pHeap, &pRight->core);
900 }
901 }
902 if (!pRight)
903 pRight = (PMMHYPERCHUNKFREE)((char *)CTXSUFF(pHeap->pbHeap) + pHeap->offFreeTail); /** @todo this can't be correct! 'pLeft = .. ; else' I think */
904 if (pRight)
905 {
906 ASSERT_CHUNK_FREE(pHeap, pRight);
907 if (pRight->offPrev)
908 {
909 pLeft = (PMMHYPERCHUNKFREE)((char *)pRight + pRight->offPrev);
910 ASSERT_CHUNK_FREE(pHeap, pLeft);
911 }
912 }
913 }
914 if (pLeft == pFree)
915 {
916 AssertMsgFailed(("Freed twice! pv=%p (pChunk=%p)\n", pChunk + 1, pChunk));
917 return VERR_INVALID_POINTER;
918 }
919 pChunk->offStat = 0;
920
921 /*
922 * Head free chunk list?
923 */
924 if (!pLeft)
925 {
926 MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_FREE);
927 pFree->offPrev = 0;
928 pHeap->offFreeHead = (uintptr_t)pFree - (uintptr_t)CTXSUFF(pHeap->pbHeap);
929 if (pRight)
930 {
931 pFree->offNext = (uintptr_t)pRight - (uintptr_t)pFree;
932 pRight->offPrev = -(int32_t)pFree->offNext;
933 }
934 else
935 {
936 pFree->offNext = 0;
937 pHeap->offFreeTail = pHeap->offFreeHead;
938 }
939 Log3(("mmHyperFree: Inserted %p at head of free chain.\n", pFree));
940 }
941 else
942 {
943 /*
944 * Can we merge with left hand free chunk?
945 */
946 if ((char *)pLeft + pLeft->core.offNext == (char *)pFree)
947 {
948 if (pFree->core.offNext)
949 {
950 pLeft->core.offNext = pLeft->core.offNext + pFree->core.offNext;
951 MMHYPERCHUNK_SET_OFFPREV(((PMMHYPERCHUNK)((char *)pLeft + pLeft->core.offNext)), -(int32_t)pLeft->core.offNext);
952 }
953 else
954 pLeft->core.offNext = 0;
955 pFree = pLeft;
956 Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pLeft->cb, -(int32_t)pLeft->cb));
957 pHeap->cbFree -= pLeft->cb;
958 Log3(("mmHyperFree: Merging %p into %p (cb=%d).\n", pFree, pLeft, pLeft->cb));
959 }
960 /*
961 * No, just link it into the free list then.
962 */
963 else
964 {
965 MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_FREE);
966 pFree->offPrev = (uintptr_t)pLeft - (uintptr_t)pFree;
967 pLeft->offNext = -pFree->offPrev;
968 if (pRight)
969 {
970 pFree->offNext = (uintptr_t)pRight - (uintptr_t)pFree;
971 pRight->offPrev = -(int32_t)pFree->offNext;
972 }
973 else
974 {
975 pFree->offNext = 0;
976 pHeap->offFreeTail = (uintptr_t)pFree - (uintptr_t)CTXSUFF(pHeap->pbHeap);
977 }
978 Log3(("mmHyperFree: Inserted %p after %p in free list.\n", pFree, pLeft));
979 }
980 }
981
982 /*
983 * Can we merge with right hand free chunk?
984 */
985 if (pRight && (char *)pRight == (char *)pFree + pFree->core.offNext)
986 {
987 /* core */
988 if (pRight->core.offNext)
989 {
990 pFree->core.offNext += pRight->core.offNext;
991 PMMHYPERCHUNK pNext = (PMMHYPERCHUNK)((char *)pFree + pFree->core.offNext);
992 MMHYPERCHUNK_SET_OFFPREV(pNext, -(int32_t)pFree->core.offNext);
993 ASSERT_CHUNK(pHeap, pNext);
994 }
995 else
996 pFree->core.offNext = 0;
997
998 /* free */
999 if (pRight->offNext)
1000 {
1001 pFree->offNext += pRight->offNext;
1002 ((PMMHYPERCHUNKFREE)((char *)pFree + pFree->offNext))->offPrev = -(int32_t)pFree->offNext;
1003 }
1004 else
1005 {
1006 pFree->offNext = 0;
1007 pHeap->offFreeTail = (uintptr_t)pFree - (uintptr_t)CTXSUFF(pHeap->pbHeap);
1008 }
1009 Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree - pRight->cb, -(int32_t)pRight->cb));
1010 pHeap->cbFree -= pRight->cb;
1011 Log3(("mmHyperFree: Merged %p (cb=%d) into %p.\n", pRight, pRight->cb, pFree));
1012 }
1013
1014 /* calculate the size. */
1015 if (pFree->core.offNext)
1016 pFree->cb = pFree->core.offNext - sizeof(MMHYPERCHUNK);
1017 else
1018 pFree->cb = pHeap->offPageAligned - ((uintptr_t)pFree - (uintptr_t)CTXSUFF(pHeap->pbHeap)) - sizeof(MMHYPERCHUNK);
1019 Log3(("mmHyperFree: cbFree %d -> %d (%d)\n", pHeap->cbFree, pHeap->cbFree + pFree->cb, pFree->cb));
1020 pHeap->cbFree += pFree->cb;
1021 ASSERT_CHUNK_FREE(pHeap, pFree);
1022
1023#ifdef MMHYPER_HEAP_STRICT
1024 mmHyperHeapCheck(pHeap);
1025#endif
1026 return VINF_SUCCESS;
1027}
1028
1029
1030#if defined(DEBUG) || defined(MMHYPER_HEAP_STRICT)
1031/**
1032 * Dumps a heap chunk to the log.
1033 *
1034 * @param pHeap Pointer to the heap.
1035 * @param pCur Pointer to the chunk.
1036 */
1037static void mmHyperHeapDumpOne(PMMHYPERHEAP pHeap, PMMHYPERCHUNKFREE pCur)
1038{
1039 if (MMHYPERCHUNK_ISUSED(&pCur->core))
1040 {
1041 if (pCur->core.offStat)
1042 {
1043 PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pCur + pCur->core.offStat);
1044 const char *pszSelf = pCur->core.offStat == sizeof(MMHYPERCHUNK) ? " stat record" : "";
1045#ifdef IN_RING3
1046 Log(("%p %06x USED offNext=%06x offPrev=-%06x %s%s\n",
1047 pCur, (uintptr_t)pCur - (uintptr_t)CTXSUFF(pHeap->pbHeap),
1048 pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1049 mmR3GetTagName((MMTAG)pStat->Core.Key), pszSelf));
1050#else
1051 Log(("%p %06x USED offNext=%06x offPrev=-%06x %d%s\n",
1052 pCur, (uintptr_t)pCur - (uintptr_t)CTXSUFF(pHeap->pbHeap),
1053 pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1054 (MMTAG)pStat->Core.Key, pszSelf));
1055#endif
1056 }
1057 else
1058 Log(("%p %06x USED offNext=%06x offPrev=-%06x\n",
1059 pCur, (uintptr_t)pCur - (uintptr_t)CTXSUFF(pHeap->pbHeap),
1060 pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core)));
1061 }
1062 else
1063 Log(("%p %06x FREE offNext=%06x offPrev=-%06x : cb=%06x offNext=%06x offPrev=-%06x\n",
1064 pCur, (uintptr_t)pCur - (uintptr_t)CTXSUFF(pHeap->pbHeap),
1065 pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core), pCur->cb, pCur->offNext, pCur->offPrev));
1066}
1067#endif /* DEBUG || MMHYPER_HEAP_STRICT */
1068
1069
1070#ifdef MMHYPER_HEAP_STRICT
1071/**
1072 * Internal consitency check.
1073 */
1074static void mmHyperHeapCheck(PMMHYPERHEAP pHeap)
1075{
1076 PMMHYPERCHUNKFREE pPrev = NULL;
1077 PMMHYPERCHUNKFREE pCur = (PMMHYPERCHUNKFREE)CTXSUFF(pHeap->pbHeap);
1078 for (;;)
1079 {
1080 if (MMHYPERCHUNK_ISUSED(&pCur->core))
1081 ASSERT_CHUNK_USED(pHeap, &pCur->core);
1082 else
1083 ASSERT_CHUNK_FREE(pHeap, pCur);
1084 if (pPrev)
1085 AssertMsg((int32_t)pPrev->core.offNext == -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
1086 ("pPrev->core.offNext=%d offPrev=%d\n", pPrev->core.offNext, MMHYPERCHUNK_GET_OFFPREV(&pCur->core)));
1087
1088# ifdef MMHYPER_HEAP_STRICT_FENCE
1089 uint32_t off = (uint8_t *)pCur - CTXSUFF(pHeap->pbHeap);
1090 if ( MMHYPERCHUNK_ISUSED(&pCur->core)
1091 && off < pHeap->offPageAligned)
1092 {
1093 uint32_t cbCur = pCur->core.offNext
1094 ? pCur->core.offNext
1095 : pHeap->cbHeap - off;
1096 uint32_t *pu32End = ((uint32_t *)((uint8_t *)pCur + cbCur));
1097 uint32_t cbFence = pu32End[-1];
1098 if (RT_UNLIKELY( cbFence >= cbCur - sizeof(*pCur)
1099 || cbFence < MMHYPER_HEAP_STRICT_FENCE_SIZE))
1100 {
1101 mmHyperHeapDumpOne(pHeap, pCur);
1102 Assert(cbFence < cbCur - sizeof(*pCur));
1103 Assert(cbFence >= MMHYPER_HEAP_STRICT_FENCE_SIZE);
1104 }
1105
1106 uint32_t *pu32Bad = ASMMemIsAllU32((uint8_t *)pu32End - cbFence, cbFence - sizeof(uint32_t), MMHYPER_HEAP_STRICT_FENCE_U32);
1107 if (RT_UNLIKELY(pu32Bad))
1108 {
1109 mmHyperHeapDumpOne(pHeap, pCur);
1110 Assert(!pu32Bad);
1111 }
1112 }
1113# endif
1114
1115 /* next */
1116 if (!pCur->core.offNext)
1117 break;
1118 pPrev = pCur;
1119 pCur = (PMMHYPERCHUNKFREE)((char *)pCur + pCur->core.offNext);
1120 }
1121}
1122#endif
1123
1124
1125/**
1126 * Performs consistency checks on the heap if MMHYPER_HEAP_STRICT was
1127 * defined at build time.
1128 *
1129 * @param pVM Pointer to the shared VM structure.
1130 */
1131MMDECL(void) MMHyperHeapCheck(PVM pVM)
1132{
1133#ifdef MMHYPER_HEAP_STRICT
1134 mmHyperHeapCheck(CTXSUFF(pVM->mm.s.pHyperHeap));
1135#endif
1136}
1137
1138
1139#ifdef DEBUG
1140/**
1141 * Dumps the hypervisor heap to Log.
1142 * @param pVM VM Handle.
1143 */
1144MMDECL(void) MMHyperHeapDump(PVM pVM)
1145{
1146 Log(("MMHyperHeapDump: *** heap dump - start ***\n"));
1147 PMMHYPERHEAP pHeap = CTXSUFF(pVM->mm.s.pHyperHeap);
1148 PMMHYPERCHUNKFREE pCur = (PMMHYPERCHUNKFREE)CTXSUFF(pHeap->pbHeap);
1149 for (;;)
1150 {
1151 mmHyperHeapDumpOne(pHeap, pCur);
1152
1153 /* next */
1154 if (!pCur->core.offNext)
1155 break;
1156 pCur = (PMMHYPERCHUNKFREE)((char *)pCur + pCur->core.offNext);
1157 }
1158 Log(("MMHyperHeapDump: *** heap dump - end ***\n"));
1159}
1160#endif
1161
1162
1163/**
1164 * Query the amount of free memory in the hypervisor heap.
1165 *
1166 * @returns Number of free bytes in the hypervisor heap.
1167 */
1168MMDECL(size_t) MMHyperHeapGetFreeSize(PVM pVM)
1169{
1170 return CTXSUFF(pVM->mm.s.pHyperHeap)->cbFree;
1171}
1172
1173/**
1174 * Query the size the hypervisor heap.
1175 *
1176 * @returns The size of the hypervisor heap in bytes.
1177 */
1178MMDECL(size_t) MMHyperHeapGetSize(PVM pVM)
1179{
1180 return CTXSUFF(pVM->mm.s.pHyperHeap)->cbHeap;
1181}
1182
1183
1184/**
1185 * Query the address and size the hypervisor memory area.
1186 *
1187 * @returns Base address of the hypervisor area.
1188 * @param pVM VM Handle.
1189 * @param pcb Where to store the size of the hypervisor area. (out)
1190 */
1191MMDECL(RTGCPTR) MMHyperGetArea(PVM pVM, size_t *pcb)
1192{
1193 if (pcb)
1194 *pcb = pVM->mm.s.cbHyperArea;
1195 return pVM->mm.s.pvHyperAreaGC;
1196}
1197
1198
1199/**
1200 * Checks if an address is within the hypervisor memory area.
1201 *
1202 * @returns true if inside.
1203 * @returns false if outside.
1204 * @param pVM VM handle.
1205 * @param GCPtr The pointer to check.
1206 */
1207MMDECL(bool) MMHyperIsInsideArea(PVM pVM, RTGCPTR GCPtr)
1208{
1209 return (RTGCUINTPTR)GCPtr - (RTGCUINTPTR)pVM->mm.s.pvHyperAreaGC < pVM->mm.s.cbHyperArea;
1210}
1211
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette