alloc-ef.cpp@ 61009

Last change on this file since 61009 was 59747, checked in by vboxsync, 9 years ago
iprt/asm.h: Cleaned up the ASMMemIsAll8/U32 mess and implmeneted the former in assembly. (Found inverted usage due to bad naming in copyUtf8Block, but it is fortunately an unused method.) Replaces the complicated ASMBitFirstSet based scanning in RTSgBufIsZero with a simple call to the new ASMMemIsZero function.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 31.2 KB

Line
1	/* $Id: alloc-ef.cpp 59747 2016-02-19 23:18:18Z vboxsync $ */
2	/** @file
3	* IPRT - Memory Allocation, electric fence.
4	*/
5
6	/*
7	* Copyright (C) 2006-2015 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*********************************************************************************************************************************
29	* Header Files *
30	*********************************************************************************************************************************/
31	#include "alloc-ef.h"
32	#include <iprt/mem.h>
33	#include <iprt/log.h>
34	#include <iprt/asm.h>
35	#include <iprt/thread.h>
36	#include <VBox/sup.h>
37	#include <iprt/err.h>
38	#include <errno.h>
39	#include <stdio.h>
40	#include <stdlib.h>
41
42	#include <iprt/alloc.h>
43	#include <iprt/assert.h>
44	#include <iprt/param.h>
45	#include <iprt/string.h>
46
47	#ifdef RTALLOC_REPLACE_MALLOC
48	# include <VBox/dis.h>
49	# include <VBox/disopcode.h>
50	# include <dlfcn.h>
51	# ifdef RT_OS_DARWIN
52	# include <malloc/malloc.h>
53	# endif
54	#endif
55
56
57	/*********************************************************************************************************************************
58	* Defined Constants And Macros *
59	*********************************************************************************************************************************/
60	#ifdef RTALLOC_REPLACE_MALLOC
61	# define RTMEM_REPLACMENT_ALIGN(a_cb) ((a_cb) >= 16 ? RT_ALIGN_Z(a_cb, 16) \
62	: (a_cb) >= sizeof(uintptr_t) ? RT_ALIGN_Z(a_cb, sizeof(uintptr_t)) : (a_cb))
63	#endif
64
65
66	/*********************************************************************************************************************************
67	* Global Variables *
68	*********************************************************************************************************************************/
69	#ifdef RTALLOC_EFENCE_TRACE
70	/** Spinlock protecting the all the block's globals. */
71	static volatile uint32_t g_BlocksLock;
72	/** Tree tracking the allocations. */
73	static AVLPVTREE g_BlocksTree;
74	# ifdef RTALLOC_EFENCE_FREE_DELAYED
75	/** Tail of the delayed blocks. */
76	static volatile PRTMEMBLOCK g_pBlocksDelayHead;
77	/** Tail of the delayed blocks. */
78	static volatile PRTMEMBLOCK g_pBlocksDelayTail;
79	/** Number of bytes in the delay list (includes fences). */
80	static volatile size_t g_cbBlocksDelay;
81	# endif /* RTALLOC_EFENCE_FREE_DELAYED */
82	# ifdef RTALLOC_REPLACE_MALLOC
83	/** @name For calling the real allocation API we've replaced.
84	* @{ */
85	void * (*g_pfnOrgMalloc)(size_t);
86	void * (*g_pfnOrgCalloc)(size_t, size_t);
87	void * (g_pfnOrgRealloc)(void , size_t);
88	void (g_pfnOrgFree)(void );
89	size_t (g_pfnOrgMallocSize)(void );
90	/** @} */
91	# endif
92	#endif /* RTALLOC_EFENCE_TRACE */
93	/** Array of pointers free watches for. */
94	void *gapvRTMemFreeWatch[4] = {NULL, NULL, NULL, NULL};
95	/** Enable logging of all freed memory. */
96	bool gfRTMemFreeLog = false;
97
98
99	/*********************************************************************************************************************************
100	* Internal Functions *
101	*********************************************************************************************************************************/
102	#ifdef RTALLOC_REPLACE_MALLOC
103	static void rtMemReplaceMallocAndFriends(void);
104	#endif
105
106
107	/**
108	* Complains about something.
109	*/
110	static void rtmemComplain(const char pszOp, const char pszFormat, ...)
111	{
112	va_list args;
113	fprintf(stderr, "RTMem error: %s: ", pszOp);
114	va_start(args, pszFormat);
115	vfprintf(stderr, pszFormat, args);
116	va_end(args);
117	RTAssertDoPanic();
118	}
119
120	/**
121	* Log an event.
122	*/
123	DECLINLINE(void) rtmemLog(const char pszOp, const char pszFormat, ...)
124	{
125	#if 0
126	va_list args;
127	fprintf(stderr, "RTMem info: %s: ", pszOp);
128	va_start(args, pszFormat);
129	vfprintf(stderr, pszFormat, args);
130	va_end(args);
131	#else
132	NOREF(pszOp); NOREF(pszFormat);
133	#endif
134	}
135
136
137	#ifdef RTALLOC_EFENCE_TRACE
138
139	/**
140	* Acquires the lock.
141	*/
142	DECLINLINE(void) rtmemBlockLock(void)
143	{
144	unsigned c = 0;
145	while (!ASMAtomicCmpXchgU32(&g_BlocksLock, 1, 0))
146	RTThreadSleepNoLog(((++c) >> 2) & 31);
147	}
148
149
150	/**
151	* Releases the lock.
152	*/
153	DECLINLINE(void) rtmemBlockUnlock(void)
154	{
155	Assert(g_BlocksLock == 1);
156	ASMAtomicXchgU32(&g_BlocksLock, 0);
157	}
158
159
160	/**
161	* Creates a block.
162	*/
163	DECLINLINE(PRTMEMBLOCK) rtmemBlockCreate(RTMEMTYPE enmType, size_t cbUnaligned, size_t cbAligned,
164	const char pszTag, void pvCaller, RT_SRC_POS_DECL)
165	{
166	# ifdef RTALLOC_REPLACE_MALLOC
167	if (!g_pfnOrgMalloc)
168	rtMemReplaceMallocAndFriends();
169	PRTMEMBLOCK pBlock = (PRTMEMBLOCK)g_pfnOrgMalloc(sizeof(*pBlock));
170	# else
171	PRTMEMBLOCK pBlock = (PRTMEMBLOCK)malloc(sizeof(*pBlock));
172	# endif
173	if (pBlock)
174	{
175	pBlock->enmType = enmType;
176	pBlock->cbUnaligned = cbUnaligned;
177	pBlock->cbAligned = cbAligned;
178	pBlock->pszTag = pszTag;
179	pBlock->pvCaller = pvCaller;
180	pBlock->iLine = iLine;
181	pBlock->pszFile = pszFile;
182	pBlock->pszFunction = pszFunction;
183	}
184	return pBlock;
185	}
186
187
188	/**
189	* Frees a block.
190	*/
191	DECLINLINE(void) rtmemBlockFree(PRTMEMBLOCK pBlock)
192	{
193	# ifdef RTALLOC_REPLACE_MALLOC
194	g_pfnOrgFree(pBlock);
195	# else
196	free(pBlock);
197	# endif
198	}
199
200
201	/**
202	* Insert a block from the tree.
203	*/
204	DECLINLINE(void) rtmemBlockInsert(PRTMEMBLOCK pBlock, void *pv)
205	{
206	pBlock->Core.Key = pv;
207	rtmemBlockLock();
208	bool fRc = RTAvlPVInsert(&g_BlocksTree, &pBlock->Core);
209	rtmemBlockUnlock();
210	AssertRelease(fRc);
211	}
212
213
214	/**
215	* Remove a block from the tree and returns it to the caller.
216	*/
217	DECLINLINE(PRTMEMBLOCK) rtmemBlockRemove(void *pv)
218	{
219	rtmemBlockLock();
220	PRTMEMBLOCK pBlock = (PRTMEMBLOCK)RTAvlPVRemove(&g_BlocksTree, pv);
221	rtmemBlockUnlock();
222	return pBlock;
223	}
224
225	/**
226	* Gets a block.
227	*/
228	DECLINLINE(PRTMEMBLOCK) rtmemBlockGet(void *pv)
229	{
230	rtmemBlockLock();
231	PRTMEMBLOCK pBlock = (PRTMEMBLOCK)RTAvlPVGet(&g_BlocksTree, pv);
232	rtmemBlockUnlock();
233	return pBlock;
234	}
235
236	/**
237	* Dumps one allocation.
238	*/
239	static DECLCALLBACK(int) RTMemDumpOne(PAVLPVNODECORE pNode, void *pvUser)
240	{
241	PRTMEMBLOCK pBlock = (PRTMEMBLOCK)pNode;
242	fprintf(stderr, "%p %08lx(+%02lx) %p\n",
243	pBlock->Core.Key,
244	(unsigned long)pBlock->cbUnaligned,
245	(unsigned long)(pBlock->cbAligned - pBlock->cbUnaligned),
246	pBlock->pvCaller);
247	NOREF(pvUser);
248	return 0;
249	}
250
251	/**
252	* Dumps the allocated blocks.
253	* This is something which you should call from gdb.
254	*/
255	extern "C" void RTMemDump(void);
256	void RTMemDump(void)
257	{
258	fprintf(stderr, "address size(alg) caller\n");
259	RTAvlPVDoWithAll(&g_BlocksTree, true, RTMemDumpOne, NULL);
260	}
261
262	# ifdef RTALLOC_EFENCE_FREE_DELAYED
263
264	/**
265	* Insert a delayed block.
266	*/
267	DECLINLINE(void) rtmemBlockDelayInsert(PRTMEMBLOCK pBlock)
268	{
269	size_t cbBlock = RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTALLOC_EFENCE_SIZE;
270	pBlock->Core.pRight = NULL;
271	pBlock->Core.pLeft = NULL;
272	rtmemBlockLock();
273	if (g_pBlocksDelayHead)
274	{
275	g_pBlocksDelayHead->Core.pLeft = (PAVLPVNODECORE)pBlock;
276	pBlock->Core.pRight = (PAVLPVNODECORE)g_pBlocksDelayHead;
277	g_pBlocksDelayHead = pBlock;
278	}
279	else
280	{
281	g_pBlocksDelayTail = pBlock;
282	g_pBlocksDelayHead = pBlock;
283	}
284	g_cbBlocksDelay += cbBlock;
285	rtmemBlockUnlock();
286	}
287
288	/**
289	* Removes a delayed block.
290	*/
291	DECLINLINE(PRTMEMBLOCK) rtmemBlockDelayRemove(void)
292	{
293	PRTMEMBLOCK pBlock = NULL;
294	rtmemBlockLock();
295	if (g_cbBlocksDelay > RTALLOC_EFENCE_FREE_DELAYED)
296	{
297	pBlock = g_pBlocksDelayTail;
298	if (pBlock)
299	{
300	g_pBlocksDelayTail = (PRTMEMBLOCK)pBlock->Core.pLeft;
301	if (pBlock->Core.pLeft)
302	pBlock->Core.pLeft->pRight = NULL;
303	else
304	g_pBlocksDelayHead = NULL;
305	g_cbBlocksDelay -= RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTALLOC_EFENCE_SIZE;
306	}
307	}
308	rtmemBlockUnlock();
309	return pBlock;
310	}
311
312	# endif /* RTALLOC_EFENCE_FREE_DELAYED */
313
314	#endif /* RTALLOC_EFENCE_TRACE */
315
316
317	#if defined(RTALLOC_REPLACE_MALLOC) && defined(RTALLOC_EFENCE_TRACE)
318	/*
319	*
320	* Replacing malloc, calloc, realloc, & free.
321	*
322	*/
323
324	/** Replacement for malloc. */
325	static void *rtMemReplacementMalloc(size_t cb)
326	{
327	size_t cbAligned = RTMEM_REPLACMENT_ALIGN(cb);
328	void *pv = rtR3MemAlloc("r-malloc", RTMEMTYPE_RTMEMALLOC, cb, cbAligned, "heap", ASMReturnAddress(), RT_SRC_POS);
329	if (!pv)
330	pv = g_pfnOrgMalloc(cb);
331	return pv;
332	}
333
334	/** Replacement for calloc. */
335	static void *rtMemReplacementCalloc(size_t cbItem, size_t cItems)
336	{
337	size_t cb = cbItem * cItems;
338	size_t cbAligned = RTMEM_REPLACMENT_ALIGN(cb);
339	void *pv = rtR3MemAlloc("r-calloc", RTMEMTYPE_RTMEMALLOCZ, cb, cbAligned, "heap", ASMReturnAddress(), RT_SRC_POS);
340	if (!pv)
341	pv = g_pfnOrgCalloc(cbItem, cItems);
342	return pv;
343	}
344
345	/** Replacement for realloc. */
346	static void rtMemReplacementRealloc(void pvOld, size_t cbNew)
347	{
348	if (pvOld)
349	{
350	/* We're not strict about where the memory was allocated. */
351	PRTMEMBLOCK pBlock = rtmemBlockGet(pvOld);
352	if (pBlock)
353	{
354	size_t cbAligned = RTMEM_REPLACMENT_ALIGN(cbNew);
355	return rtR3MemRealloc("r-realloc", RTMEMTYPE_RTMEMREALLOC, pvOld, cbAligned, "heap", ASMReturnAddress(), RT_SRC_POS);
356	}
357	return g_pfnOrgRealloc(pvOld, cbNew);
358	}
359	return rtMemReplacementMalloc(cbNew);
360	}
361
362	/** Replacement for free(). */
363	static void rtMemReplacementFree(void *pv)
364	{
365	if (pv)
366	{
367	/* We're not strict about where the memory was allocated. */
368	PRTMEMBLOCK pBlock = rtmemBlockGet(pv);
369	if (pBlock)
370	rtR3MemFree("r-free", RTMEMTYPE_RTMEMFREE, pv, ASMReturnAddress(), RT_SRC_POS);
371	else
372	g_pfnOrgFree(pv);
373	}
374	}
375
376	# ifdef RT_OS_DARWIN
377	/** Replacement for malloc. */
378	static size_t rtMemReplacementMallocSize(void *pv)
379	{
380	size_t cb;
381	if (pv)
382	{
383	/* We're not strict about where the memory was allocated. */
384	PRTMEMBLOCK pBlock = rtmemBlockGet(pv);
385	if (pBlock)
386	cb = pBlock->cbUnaligned;
387	else
388	cb = g_pfnOrgMallocSize(pv);
389	}
390	else
391	cb = 0;
392	return cb;
393	}
394	# endif
395
396
397	static void rtMemReplaceMallocAndFriends(void)
398	{
399	struct
400	{
401	const char *pszName;
402	PFNRT pfnReplacement;
403	PFNRT pfnOrg;
404	PFNRT *ppfnJumpBack;
405	} aApis[] =
406	{
407	{ "free", (PFNRT)rtMemReplacementFree, (PFNRT)free, (PFNRT *)&g_pfnOrgFree },
408	{ "realloc", (PFNRT)rtMemReplacementRealloc, (PFNRT)realloc, (PFNRT *)&g_pfnOrgRealloc },
409	{ "calloc", (PFNRT)rtMemReplacementCalloc, (PFNRT)calloc, (PFNRT *)&g_pfnOrgCalloc },
410	{ "malloc", (PFNRT)rtMemReplacementMalloc, (PFNRT)malloc, (PFNRT *)&g_pfnOrgMalloc },
411	#ifdef RT_OS_DARWIN
412	{ "malloc_size", (PFNRT)rtMemReplacementMallocSize, (PFNRT)malloc_size, (PFNRT *)&g_pfnOrgMallocSize },
413	#endif
414	};
415
416	/*
417	* Initialize the jump backs to avoid recursivly entering this function.
418	*/
419	for (unsigned i = 0; i < RT_ELEMENTS(aApis); i++)
420	*aApis[i].ppfnJumpBack = aApis[i].pfnOrg;
421
422	/*
423	* Give the user an option to skip replacing malloc.
424	*/
425	if (getenv("IPRT_DONT_REPLACE_MALLOC"))
426	return;
427
428	/*
429	* Allocate a page for jump back code (we leak it).
430	*/
431	uint8_t pbExecPage = (uint8_t )RTMemPageAlloc(PAGE_SIZE); AssertFatal(pbExecPage);
432	int rc = RTMemProtect(pbExecPage, PAGE_SIZE, RTMEM_PROT_READ \| RTMEM_PROT_WRITE \| RTMEM_PROT_EXEC); AssertFatalRC(rc);
433
434	/*
435	* Do the ground work.
436	*/
437	uint8_t *pb = pbExecPage;
438	for (unsigned i = 0; i < RT_ELEMENTS(aApis); i++)
439	{
440	/* Resolve it. */
441	PFNRT pfnOrg = (PFNRT)(uintptr_t)dlsym(RTLD_DEFAULT, aApis[i].pszName);
442	if (pfnOrg)
443	aApis[i].pfnOrg = pfnOrg;
444	else
445	pfnOrg = aApis[i].pfnOrg;
446
447	/* Figure what we can replace and how much to duplicate in the jump back code. */
448	# ifdef RT_ARCH_AMD64
449	uint32_t cbNeeded = 12;
450	DISCPUMODE const enmCpuMode = DISCPUMODE_64BIT;
451	# elif defined(RT_ARCH_X86)
452	uint32_t const cbNeeded = 5;
453	DISCPUMODE const enmCpuMode = DISCPUMODE_32BIT;
454	# else
455	# error "Port me"
456	# endif
457	uint32_t offJmpBack = 0;
458	uint32_t cbCopy = 0;
459	while (offJmpBack < cbNeeded)
460	{
461	DISCPUSTATE Dis;
462	uint32_t cbInstr = 1;
463	rc = DISInstr((void *)((uintptr_t)pfnOrg + offJmpBack), enmCpuMode, &Dis, &cbInstr); AssertFatalRC(rc);
464	AssertFatal(!(Dis.pCurInstr->fOpType & (DISOPTYPE_CONTROLFLOW)));
465	# ifdef RT_ARCH_AMD64
466	# ifdef RT_OS_DARWIN
467	/* Kludge for: cmp [malloc_def_zone_state], 1; jg 2; call _malloc_initialize; 2: */
468	DISQPVPARAMVAL Parm;
469	if ( Dis.ModRM.Bits.Mod == 0
470	&& Dis.ModRM.Bits.Rm == 5 /* wrt RIP */
471	&& (Dis.Param2.fUse & (DISUSE_IMMEDIATE16_SX8 \| DISUSE_IMMEDIATE32_SX8 \| DISUSE_IMMEDIATE64_SX8))
472	&& Dis.Param2.uValue == 1
473	&& Dis.pCurInstr->uOpcode == OP_CMP)
474	{
475	cbCopy = offJmpBack;
476
477	offJmpBack += cbInstr;
478	rc = DISInstr((void *)((uintptr_t)pfnOrg + offJmpBack), enmCpuMode, &Dis, &cbInstr); AssertFatalRC(rc);
479	if ( Dis.pCurInstr->uOpcode == OP_JNBE
480	&& Dis.Param1.uDisp.i8 == 5)
481	{
482	offJmpBack += cbInstr + 5;
483	AssertFatal(offJmpBack >= cbNeeded);
484	break;
485	}
486	}
487	# endif
488	AssertFatal(!(Dis.ModRM.Bits.Mod == 0 && Dis.ModRM.Bits.Rm == 5 /* wrt RIP */));
489	# endif
490	offJmpBack += cbInstr;
491	}
492	if (!cbCopy)
493	cbCopy = offJmpBack;
494
495	/* Assemble the jump back. */
496	memcpy(pb, (void *)(uintptr_t)pfnOrg, cbCopy);
497	uint32_t off = cbCopy;
498	# ifdef RT_ARCH_AMD64
499	pb[off++] = 0xff; /* jmp qword [$+8 wrt RIP] */
500	pb[off++] = 0x25;
501	(uint32_t )&pb[off] = 0;
502	off += 4;
503	(uint64_t )&pb[off] = (uintptr_t)pfnOrg + offJmpBack;
504	off += 8;
505	off = RT_ALIGN_32(off, 16);
506	# elif defined(RT_ARCH_X86)
507	pb[off++] = 0xe9; /* jmp rel32 */
508	(uint32_t )&pb[off] = (uintptr_t)pfnOrg + offJmpBack - (uintptr_t)&pb[4];
509	off += 4;
510	off = RT_ALIGN_32(off, 8);
511	# else
512	# error "Port me"
513	# endif
514	*aApis[i].ppfnJumpBack = (PFNRT)(uintptr_t)pb;
515	pb += off;
516	}
517
518	/*
519	* Modify the APIs.
520	*/
521	for (unsigned i = 0; i < RT_ELEMENTS(aApis); i++)
522	{
523	pb = (uint8_t *)(uintptr_t)aApis[i].pfnOrg;
524	rc = RTMemProtect(pb, 16, RTMEM_PROT_READ \| RTMEM_PROT_WRITE \| RTMEM_PROT_EXEC); AssertFatalRC(rc);
525
526	# ifdef RT_ARCH_AMD64
527	/* Assemble the LdrLoadDll patch. */
528	pb++ = 0x48; / mov rax, qword */
529	*pb++ = 0xb8;
530	(uint64_t )pb = (uintptr_t)aApis[i].pfnReplacement;
531	pb += 8;
532	pb++ = 0xff; / jmp rax */
533	*pb++ = 0xe0;
534	# elif defined(RT_ARCH_X86)
535	pb++ = 0xe9; / jmp rel32 */
536	(uint32_t )pb = (uintptr_t)aApis[i].pfnReplacement - (uintptr_t)&pb[4];
537	# else
538	# error "Port me"
539	# endif
540	}
541	}
542
543	#endif /* RTALLOC_REPLACE_MALLOC && RTALLOC_EFENCE_TRACE */
544
545
546	/**
547	* Internal allocator.
548	*/
549	RTDECL(void ) rtR3MemAlloc(const char pszOp, RTMEMTYPE enmType, size_t cbUnaligned, size_t cbAligned,
550	const char pszTag, void pvCaller, RT_SRC_POS_DECL)
551	{
552	/*
553	* Sanity.
554	*/
555	if ( RT_ALIGN_Z(RTALLOC_EFENCE_SIZE, PAGE_SIZE) != RTALLOC_EFENCE_SIZE
556	&& RTALLOC_EFENCE_SIZE <= 0)
557	{
558	rtmemComplain(pszOp, "Invalid E-fence size! %#x\n", RTALLOC_EFENCE_SIZE);
559	return NULL;
560	}
561	if (!cbUnaligned)
562	{
563	#if 0
564	rtmemComplain(pszOp, "Request of ZERO bytes allocation!\n");
565	return NULL;
566	#else
567	cbAligned = cbUnaligned = 1;
568	#endif
569	}
570
571	#ifndef RTALLOC_EFENCE_IN_FRONT
572	/* Alignment decreases fence accuracy, but this is at least partially
573	* counteracted by filling and checking the alignment padding. When the
574	* fence is in front then then no extra alignment is needed. */
575	cbAligned = RT_ALIGN_Z(cbAligned, RTALLOC_EFENCE_ALIGNMENT);
576	#endif
577
578	#ifdef RTALLOC_EFENCE_TRACE
579	/*
580	* Allocate the trace block.
581	*/
582	PRTMEMBLOCK pBlock = rtmemBlockCreate(enmType, cbUnaligned, cbAligned, pszTag, pvCaller, RT_SRC_POS_ARGS);
583	if (!pBlock)
584	{
585	rtmemComplain(pszOp, "Failed to allocate trace block!\n");
586	return NULL;
587	}
588	#endif
589
590	/*
591	* Allocate a block with page alignment space + the size of the E-fence.
592	*/
593	size_t cbBlock = RT_ALIGN_Z(cbAligned, PAGE_SIZE) + RTALLOC_EFENCE_SIZE;
594	void *pvBlock = RTMemPageAlloc(cbBlock);
595	if (pvBlock)
596	{
597	/*
598	* Calc the start of the fence and the user block
599	* and then change the page protection of the fence.
600	*/
601	#ifdef RTALLOC_EFENCE_IN_FRONT
602	void *pvEFence = pvBlock;
603	void pv = (char )pvEFence + RTALLOC_EFENCE_SIZE;
604	# ifdef RTALLOC_EFENCE_NOMAN_FILLER
605	memset((char *)pv + cbUnaligned, RTALLOC_EFENCE_NOMAN_FILLER, cbBlock - RTALLOC_EFENCE_SIZE - cbUnaligned);
606	# endif
607	#else
608	void pvEFence = (char )pvBlock + (cbBlock - RTALLOC_EFENCE_SIZE);
609	void pv = (char )pvEFence - cbAligned;
610	# ifdef RTALLOC_EFENCE_NOMAN_FILLER
611	memset(pvBlock, RTALLOC_EFENCE_NOMAN_FILLER, cbBlock - RTALLOC_EFENCE_SIZE - cbAligned);
612	memset((char *)pv + cbUnaligned, RTALLOC_EFENCE_NOMAN_FILLER, cbAligned - cbUnaligned);
613	# endif
614	#endif
615
616	#ifdef RTALLOC_EFENCE_FENCE_FILLER
617	memset(pvEFence, RTALLOC_EFENCE_FENCE_FILLER, RTALLOC_EFENCE_SIZE);
618	#endif
619	int rc = RTMemProtect(pvEFence, RTALLOC_EFENCE_SIZE, RTMEM_PROT_NONE);
620	if (!rc)
621	{
622	#ifdef RTALLOC_EFENCE_TRACE
623	rtmemBlockInsert(pBlock, pv);
624	#endif
625	if (enmType == RTMEMTYPE_RTMEMALLOCZ)
626	memset(pv, 0, cbUnaligned);
627	#ifdef RTALLOC_EFENCE_FILLER
628	else
629	memset(pv, RTALLOC_EFENCE_FILLER, cbUnaligned);
630	#endif
631
632	rtmemLog(pszOp, "returns %p (pvBlock=%p cbBlock=%#x pvEFence=%p cbUnaligned=%#x)\n", pv, pvBlock, cbBlock, pvEFence, cbUnaligned);
633	return pv;
634	}
635	rtmemComplain(pszOp, "RTMemProtect failed, pvEFence=%p size %d, rc=%d\n", pvEFence, RTALLOC_EFENCE_SIZE, rc);
636	RTMemPageFree(pvBlock, cbBlock);
637	}
638	else
639	rtmemComplain(pszOp, "Failed to allocated %lu (%lu) bytes.\n", (unsigned long)cbBlock, (unsigned long)cbUnaligned);
640
641	#ifdef RTALLOC_EFENCE_TRACE
642	rtmemBlockFree(pBlock);
643	#endif
644	return NULL;
645	}
646
647
648	/**
649	* Internal free.
650	*/
651	RTDECL(void) rtR3MemFree(const char pszOp, RTMEMTYPE enmType, void pv, void *pvCaller, RT_SRC_POS_DECL)
652	{
653	NOREF(enmType); RT_SRC_POS_NOREF();
654
655	/*
656	* Simple case.
657	*/
658	if (!pv)
659	return;
660
661	/*
662	* Check watch points.
663	*/
664	for (unsigned i = 0; i < RT_ELEMENTS(gapvRTMemFreeWatch); i++)
665	if (gapvRTMemFreeWatch[i] == pv)
666	RTAssertDoPanic();
667
668	#ifdef RTALLOC_EFENCE_TRACE
669	/*
670	* Find the block.
671	*/
672	PRTMEMBLOCK pBlock = rtmemBlockRemove(pv);
673	if (pBlock)
674	{
675	if (gfRTMemFreeLog)
676	RTLogPrintf("RTMem %s: pv=%p pvCaller=%p cbUnaligned=%#x\n", pszOp, pv, pvCaller, pBlock->cbUnaligned);
677
678	# ifdef RTALLOC_EFENCE_NOMAN_FILLER
679	/*
680	* Check whether the no man's land is untouched.
681	*/
682	# ifdef RTALLOC_EFENCE_IN_FRONT
683	void pvWrong = ASMMemFirstMismatchingU8((char )pv + pBlock->cbUnaligned,
684	RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) - pBlock->cbUnaligned,
685	RTALLOC_EFENCE_NOMAN_FILLER);
686	# else
687	/* Alignment must match allocation alignment in rtMemAlloc(). */
688	void pvWrong = ASMMemFirstMismatchingU8((char )pv + pBlock->cbUnaligned,
689	pBlock->cbAligned - pBlock->cbUnaligned,
690	RTALLOC_EFENCE_NOMAN_FILLER);
691	if (pvWrong)
692	RTAssertDoPanic();
693	pvWrong = ASMMemFirstMismatchingU8((void *)((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK),
694	RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) - pBlock->cbAligned,
695	RTALLOC_EFENCE_NOMAN_FILLER);
696	# endif
697	if (pvWrong)
698	RTAssertDoPanic();
699	# endif
700
701	# ifdef RTALLOC_EFENCE_FREE_FILL
702	/*
703	* Fill the user part of the block.
704	*/
705	memset(pv, RTALLOC_EFENCE_FREE_FILL, pBlock->cbUnaligned);
706	# endif
707
708	# if defined(RTALLOC_EFENCE_FREE_DELAYED) && RTALLOC_EFENCE_FREE_DELAYED > 0
709	/*
710	* We're doing delayed freeing.
711	* That means we'll expand the E-fence to cover the entire block.
712	*/
713	int rc = RTMemProtect(pv, pBlock->cbAligned, RTMEM_PROT_NONE);
714	if (RT_SUCCESS(rc))
715	{
716	/*
717	* Insert it into the free list and process pending frees.
718	*/
719	rtmemBlockDelayInsert(pBlock);
720	while ((pBlock = rtmemBlockDelayRemove()) != NULL)
721	{
722	pv = pBlock->Core.Key;
723	# ifdef RTALLOC_EFENCE_IN_FRONT
724	void pvBlock = (char )pv - RTALLOC_EFENCE_SIZE;
725	# else
726	void pvBlock = (void )((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK);
727	# endif
728	size_t cbBlock = RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTALLOC_EFENCE_SIZE;
729	rc = RTMemProtect(pvBlock, cbBlock, RTMEM_PROT_READ \| RTMEM_PROT_WRITE);
730	if (RT_SUCCESS(rc))
731	RTMemPageFree(pvBlock, RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTALLOC_EFENCE_SIZE);
732	else
733	rtmemComplain(pszOp, "RTMemProtect(%p, %#x, RTMEM_PROT_READ \| RTMEM_PROT_WRITE) -> %d\n", pvBlock, cbBlock, rc);
734	rtmemBlockFree(pBlock);
735	}
736	}
737	else
738	rtmemComplain(pszOp, "Failed to expand the efence of pv=%p cb=%d, rc=%d.\n", pv, pBlock, rc);
739
740	# else /* !RTALLOC_EFENCE_FREE_DELAYED */
741
742	/*
743	* Turn of the E-fence and free it.
744	*/
745	# ifdef RTALLOC_EFENCE_IN_FRONT
746	void pvBlock = (char )pv - RTALLOC_EFENCE_SIZE;
747	void *pvEFence = pvBlock;
748	# else
749	void pvBlock = (void )((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK);
750	void pvEFence = (char )pv + pBlock->cb;
751	# endif
752	int rc = RTMemProtect(pvEFence, RTALLOC_EFENCE_SIZE, RTMEM_PROT_READ \| RTMEM_PROT_WRITE);
753	if (RT_SUCCESS(rc))
754	RTMemPageFree(pvBlock, RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTALLOC_EFENCE_SIZE);
755	else
756	rtmemComplain(pszOp, "RTMemProtect(%p, %#x, RTMEM_PROT_READ \| RTMEM_PROT_WRITE) -> %d\n", pvEFence, RTALLOC_EFENCE_SIZE, rc);
757	rtmemBlockFree(pBlock);
758
759	# endif /* !RTALLOC_EFENCE_FREE_DELAYED */
760	}
761	else
762	rtmemComplain(pszOp, "pv=%p not found! Incorrect free!\n", pv);
763
764	#else /* !RTALLOC_EFENCE_TRACE */
765
766	/*
767	* We have no size tracking, so we're not doing any freeing because
768	* we cannot if the E-fence is after the block.
769	* Let's just expand the E-fence to the first page of the user bit
770	* since we know that it's around.
771	*/
772	int rc = RTMemProtect((void *)((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK), PAGE_SIZE, RTMEM_PROT_NONE);
773	if (RT_FAILURE(rc))
774	rtmemComplain(pszOp, "RTMemProtect(%p, PAGE_SIZE, RTMEM_PROT_NONE) -> %d\n", (void *)((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK), rc);
775	#endif /* !RTALLOC_EFENCE_TRACE */
776	}
777
778
779	/**
780	* Internal realloc.
781	*/
782	RTDECL(void ) rtR3MemRealloc(const char pszOp, RTMEMTYPE enmType, void *pvOld, size_t cbNew,
783	const char pszTag, void pvCaller, RT_SRC_POS_DECL)
784	{
785	/*
786	* Allocate new and copy.
787	*/
788	if (!pvOld)
789	return rtR3MemAlloc(pszOp, enmType, cbNew, cbNew, pszTag, pvCaller, RT_SRC_POS_ARGS);
790	if (!cbNew)
791	{
792	rtR3MemFree(pszOp, RTMEMTYPE_RTMEMREALLOC, pvOld, pvCaller, RT_SRC_POS_ARGS);
793	return NULL;
794	}
795
796	#ifdef RTALLOC_EFENCE_TRACE
797
798	/*
799	* Get the block, allocate the new, copy the data, free the old one.
800	*/
801	PRTMEMBLOCK pBlock = rtmemBlockGet(pvOld);
802	if (pBlock)
803	{
804	void *pvRet = rtR3MemAlloc(pszOp, enmType, cbNew, cbNew, pszTag, pvCaller, RT_SRC_POS_ARGS);
805	if (pvRet)
806	{
807	memcpy(pvRet, pvOld, RT_MIN(cbNew, pBlock->cbUnaligned));
808	rtR3MemFree(pszOp, RTMEMTYPE_RTMEMREALLOC, pvOld, pvCaller, RT_SRC_POS_ARGS);
809	}
810	return pvRet;
811	}
812	else
813	rtmemComplain(pszOp, "pvOld=%p was not found!\n", pvOld);
814	return NULL;
815
816	#else /* !RTALLOC_EFENCE_TRACE */
817
818	rtmemComplain(pszOp, "Not supported if RTALLOC_EFENCE_TRACE isn't defined!\n");
819	return NULL;
820
821	#endif /* !RTALLOC_EFENCE_TRACE */
822	}
823
824
825
826
827	RTDECL(void ) RTMemEfTmpAlloc(size_t cb, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
828	{
829	return rtR3MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
830	}
831
832
833	RTDECL(void ) RTMemEfTmpAllocZ(size_t cb, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
834	{
835	return rtR3MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
836	}
837
838
839	RTDECL(void) RTMemEfTmpFree(void *pv, RT_SRC_POS_DECL) RT_NO_THROW_DEF
840	{
841	if (pv)
842	rtR3MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, ASMReturnAddress(), RT_SRC_POS_ARGS);
843	}
844
845
846	RTDECL(void ) RTMemEfAlloc(size_t cb, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
847	{
848	return rtR3MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
849	}
850
851
852	RTDECL(void ) RTMemEfAllocZ(size_t cb, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
853	{
854	return rtR3MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
855	}
856
857
858	RTDECL(void ) RTMemEfAllocVar(size_t cbUnaligned, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
859	{
860	size_t cbAligned;
861	if (cbUnaligned >= 16)
862	cbAligned = RT_ALIGN_Z(cbUnaligned, 16);
863	else
864	cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
865	return rtR3MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
866	}
867
868
869	RTDECL(void ) RTMemEfAllocZVar(size_t cbUnaligned, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
870	{
871	size_t cbAligned;
872	if (cbUnaligned >= 16)
873	cbAligned = RT_ALIGN_Z(cbUnaligned, 16);
874	else
875	cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
876	return rtR3MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
877	}
878
879
880	RTDECL(void ) RTMemEfRealloc(void pvOld, size_t cbNew, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
881	{
882	return rtR3MemRealloc("Realloc", RTMEMTYPE_RTMEMREALLOC, pvOld, cbNew, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
883	}
884
885
886	RTDECL(void) RTMemEfFree(void *pv, RT_SRC_POS_DECL) RT_NO_THROW_DEF
887	{
888	if (pv)
889	rtR3MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, ASMReturnAddress(), RT_SRC_POS_ARGS);
890	}
891
892
893	RTDECL(void ) RTMemEfDup(const void pvSrc, size_t cb, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
894	{
895	void *pvDst = RTMemEfAlloc(cb, pszTag, RT_SRC_POS_ARGS);
896	if (pvDst)
897	memcpy(pvDst, pvSrc, cb);
898	return pvDst;
899	}
900
901
902	RTDECL(void ) RTMemEfDupEx(const void pvSrc, size_t cbSrc, size_t cbExtra, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
903	{
904	void *pvDst = RTMemEfAlloc(cbSrc + cbExtra, pszTag, RT_SRC_POS_ARGS);
905	if (pvDst)
906	{
907	memcpy(pvDst, pvSrc, cbSrc);
908	memset((uint8_t *)pvDst + cbSrc, 0, cbExtra);
909	}
910	return pvDst;
911	}
912
913
914
915
916	/*
917	*
918	* The NP (no position) versions.
919	*
920	*/
921
922
923
924	RTDECL(void ) RTMemEfTmpAllocNP(size_t cb, const char pszTag) RT_NO_THROW_DEF
925	{
926	return rtR3MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
927	}
928
929
930	RTDECL(void ) RTMemEfTmpAllocZNP(size_t cb, const char pszTag) RT_NO_THROW_DEF
931	{
932	return rtR3MemAlloc("TmpAllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
933	}
934
935
936	RTDECL(void) RTMemEfTmpFreeNP(void *pv) RT_NO_THROW_DEF
937	{
938	if (pv)
939	rtR3MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, ASMReturnAddress(), NULL, 0, NULL);
940	}
941
942
943	RTDECL(void ) RTMemEfAllocNP(size_t cb, const char pszTag) RT_NO_THROW_DEF
944	{
945	return rtR3MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
946	}
947
948
949	RTDECL(void ) RTMemEfAllocZNP(size_t cb, const char pszTag) RT_NO_THROW_DEF
950	{
951	return rtR3MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
952	}
953
954
955	RTDECL(void ) RTMemEfAllocVarNP(size_t cbUnaligned, const char pszTag) RT_NO_THROW_DEF
956	{
957	size_t cbAligned;
958	if (cbUnaligned >= 16)
959	cbAligned = RT_ALIGN_Z(cbUnaligned, 16);
960	else
961	cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
962	return rtR3MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), NULL, 0, NULL);
963	}
964
965
966	RTDECL(void ) RTMemEfAllocZVarNP(size_t cbUnaligned, const char pszTag) RT_NO_THROW_DEF
967	{
968	size_t cbAligned;
969	if (cbUnaligned >= 16)
970	cbAligned = RT_ALIGN_Z(cbUnaligned, 16);
971	else
972	cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
973	return rtR3MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), NULL, 0, NULL);
974	}
975
976
977	RTDECL(void ) RTMemEfReallocNP(void pvOld, size_t cbNew, const char *pszTag) RT_NO_THROW_DEF
978	{
979	return rtR3MemRealloc("Realloc", RTMEMTYPE_RTMEMREALLOC, pvOld, cbNew, pszTag, ASMReturnAddress(), NULL, 0, NULL);
980	}
981
982
983	RTDECL(void) RTMemEfFreeNP(void *pv) RT_NO_THROW_DEF
984	{
985	if (pv)
986	rtR3MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, ASMReturnAddress(), NULL, 0, NULL);
987	}
988
989
990	RTDECL(void ) RTMemEfDupNP(const void pvSrc, size_t cb, const char *pszTag) RT_NO_THROW_DEF
991	{
992	void *pvDst = RTMemEfAlloc(cb, pszTag, NULL, 0, NULL);
993	if (pvDst)
994	memcpy(pvDst, pvSrc, cb);
995	return pvDst;
996	}
997
998
999	RTDECL(void ) RTMemEfDupExNP(const void pvSrc, size_t cbSrc, size_t cbExtra, const char *pszTag) RT_NO_THROW_DEF
1000	{
1001	void *pvDst = RTMemEfAlloc(cbSrc + cbExtra, pszTag, NULL, 0, NULL);
1002	if (pvDst)
1003	{
1004	memcpy(pvDst, pvSrc, cbSrc);
1005	memset((uint8_t *)pvDst + cbSrc, 0, cbExtra);
1006	}
1007	return pvDst;
1008	}
1009

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r3/alloc-ef.cpp@ 61009

Download in other formats: