alloc-r0drv-linux.c@ 36555

Last change on this file since 36555 was 36555, checked in by vboxsync, 14 years ago
Use DECLHIDDEN, especially in IPRT.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 12.4 KB

Line
1	/* $Id: alloc-r0drv-linux.c 36555 2011-04-05 12:34:09Z vboxsync $ */
2	/** @file
3	* IPRT - Memory Allocation, Ring-0 Driver, Linux.
4	*/
5
6	/*
7	* Copyright (C) 2006-2010 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*******************************************************************************
29	* Header Files *
30	*******************************************************************************/
31	#include "the-linux-kernel.h"
32	#include "internal/iprt.h"
33	#include <iprt/mem.h>
34
35	#include <iprt/assert.h>
36	#include <iprt/err.h>
37	#include "r0drv/alloc-r0drv.h"
38
39	#if defined(RT_ARCH_AMD64) \|\| defined(DOXYGEN_RUNNING)
40	/**
41	* We need memory in the module range (~2GB to ~0) this can only be obtained
42	* thru APIs that are not exported (see module_alloc()).
43	*
44	* So, we'll have to create a quick and dirty heap here using BSS memory.
45	* Very annoying and it's going to restrict us!
46	*/
47	# define RTMEMALLOC_EXEC_HEAP
48	#endif
49	#ifdef RTMEMALLOC_EXEC_HEAP
50	# include <iprt/heap.h>
51	# include <iprt/spinlock.h>
52	# include <iprt/err.h>
53	#endif
54
55
56	/*******************************************************************************
57	* Global Variables *
58	*******************************************************************************/
59	#ifdef RTMEMALLOC_EXEC_HEAP
60
61	# ifdef CONFIG_DEBUG_SET_MODULE_RONX
62	# define RTMEMALLOC_EXEC_HEAP_VM_AREA 1
63	# endif
64	/** The heap. */
65	static RTHEAPSIMPLE g_HeapExec = NIL_RTHEAPSIMPLE;
66	/** Spinlock protecting the heap. */
67	static RTSPINLOCK g_HeapExecSpinlock = NIL_RTSPINLOCK;
68	# ifdef RTMEMALLOC_EXEC_HEAP_VM_AREA
69	static struct page **g_apPages;
70	static void *g_pvHeap;
71	static size_t g_cPages;
72	# endif
73
74
75	/**
76	* API for cleaning up the heap spinlock on IPRT termination.
77	* This is as RTMemExecDonate specific to AMD64 Linux/GNU.
78	*/
79	DECLHIDDEN(void) rtR0MemExecCleanup(void)
80	{
81	# ifdef RTMEMALLOC_EXEC_HEAP_VM_AREA
82	unsigned i;
83
84	/* according to linux/drivers/lguest/core.c this function undoes
85	* map_vm_area() as well as __get_vm_area(). */
86	if (g_pvHeap)
87	vunmap(g_pvHeap);
88	for (i = 0; i < g_cPages; i++)
89	__free_page(g_apPages[i]);
90	kfree(g_apPages);
91	# endif
92
93	RTSpinlockDestroy(g_HeapExecSpinlock);
94	g_HeapExecSpinlock = NIL_RTSPINLOCK;
95	}
96
97
98	# ifndef RTMEMALLOC_EXEC_HEAP_VM_AREA
99	/**
100	* Donate read+write+execute memory to the exec heap.
101	*
102	* This API is specific to AMD64 and Linux/GNU. A kernel module that desires to
103	* use RTMemExecAlloc on AMD64 Linux/GNU will have to donate some statically
104	* allocated memory in the module if it wishes for GCC generated code to work.
105	* GCC can only generate modules that work in the address range ~2GB to ~0
106	* currently.
107	*
108	* The API only accept one single donation.
109	*
110	* @returns IPRT status code.
111	* @param pvMemory Pointer to the memory block.
112	* @param cb The size of the memory block.
113	*/
114	RTR0DECL(int) RTR0MemExecDonate(void *pvMemory, size_t cb)
115	{
116	int rc;
117	AssertReturn(g_HeapExec == NIL_RTHEAPSIMPLE, VERR_WRONG_ORDER);
118
119	rc = RTSpinlockCreate(&g_HeapExecSpinlock);
120	if (RT_SUCCESS(rc))
121	{
122	rc = RTHeapSimpleInit(&g_HeapExec, pvMemory, cb);
123	if (RT_FAILURE(rc))
124	rtR0MemExecCleanup();
125	}
126	return rc;
127	}
128	RT_EXPORT_SYMBOL(RTR0MemExecDonate);
129
130	# else /* !RTMEMALLOC_EXEC_HEAP_VM_AREA */
131
132	/**
133	* RTR0MemExecDonate() does not work if CONFIG_DEBUG_SET_MODULE_RONX is enabled.
134	* In that case, allocate a VM area in the modules range and back it with kernel
135	* memory. Unfortunately __vmalloc_area() is not exported so we have to emulate
136	* it.
137	*/
138	RTR0DECL(int) RTR0MemExecInit(size_t cb)
139	{
140	int rc;
141	struct vm_struct *area;
142	size_t cPages;
143	size_t cbPages;
144	unsigned i;
145	struct page **ppPages;
146
147	AssertReturn(g_HeapExec == NIL_RTHEAPSIMPLE, VERR_WRONG_ORDER);
148
149	rc = RTSpinlockCreate(&g_HeapExecSpinlock);
150	if (RT_SUCCESS(rc))
151	{
152	cb = RT_ALIGN(cb, PAGE_SIZE);
153	area = __get_vm_area(cb, VM_ALLOC, MODULES_VADDR, MODULES_END);
154	if (!area)
155	{
156	rtR0MemExecCleanup();
157	return VERR_NO_MEMORY;
158	}
159	g_pvHeap = area->addr;
160	cPages = cb >> PAGE_SHIFT;
161	area->nr_pages = 0;
162	cbPages = cPages * sizeof(struct page *);
163	g_apPages = kmalloc(cbPages, GFP_KERNEL);
164	area->pages = g_apPages;
165	if (!g_apPages)
166	{
167	rtR0MemExecCleanup();
168	return VERR_NO_MEMORY;
169	}
170	memset(area->pages, 0, cbPages);
171	for (i = 0; i < cPages; i++)
172	{
173	g_apPages[i] = alloc_page(GFP_KERNEL \| __GFP_HIGHMEM);
174	if (!g_apPages[i])
175	{
176	area->nr_pages = i;
177	g_cPages = i;
178	rtR0MemExecCleanup();
179	return VERR_NO_MEMORY;
180	}
181	}
182	area->nr_pages = cPages;
183	g_cPages = i;
184	ppPages = g_apPages;
185	if (map_vm_area(area, PAGE_KERNEL_EXEC, &ppPages))
186	{
187	rtR0MemExecCleanup();
188	return VERR_NO_MEMORY;
189	}
190
191	rc = RTHeapSimpleInit(&g_HeapExec, g_pvHeap, cb);
192	if (RT_FAILURE(rc))
193	rtR0MemExecCleanup();
194	}
195	return rc;
196	}
197	RT_EXPORT_SYMBOL(RTR0MemExecInit);
198	# endif /* RTMEMALLOC_EXEC_HEAP_VM_AREA */
199	#endif /* RTMEMALLOC_EXEC_HEAP */
200
201
202
203	/**
204	* OS specific allocation function.
205	*/
206	DECLHIDDEN(int) rtR0MemAllocEx(size_t cb, uint32_t fFlags, PRTMEMHDR *ppHdr)
207	{
208	PRTMEMHDR pHdr;
209
210	/*
211	* Allocate.
212	*/
213	if (fFlags & RTMEMHDR_FLAG_EXEC)
214	{
215	if (fFlags & RTMEMHDR_FLAG_ANY_CTX)
216	return VERR_NOT_SUPPORTED;
217
218	#if defined(RT_ARCH_AMD64)
219	# ifdef RTMEMALLOC_EXEC_HEAP
220	if (g_HeapExec != NIL_RTHEAPSIMPLE)
221	{
222	RTSPINLOCKTMP SpinlockTmp = RTSPINLOCKTMP_INITIALIZER;
223	RTSpinlockAcquireNoInts(g_HeapExecSpinlock, &SpinlockTmp);
224	pHdr = (PRTMEMHDR)RTHeapSimpleAlloc(g_HeapExec, cb + sizeof(*pHdr), 0);
225	RTSpinlockReleaseNoInts(g_HeapExecSpinlock, &SpinlockTmp);
226	fFlags \|= RTMEMHDR_FLAG_EXEC_HEAP;
227	}
228	else
229	pHdr = NULL;
230	# else /* !RTMEMALLOC_EXEC_HEAP */
231	pHdr = (PRTMEMHDR)__vmalloc(cb + sizeof(*pHdr), GFP_KERNEL \| __GFP_HIGHMEM, MY_PAGE_KERNEL_EXEC);
232	# endif /* !RTMEMALLOC_EXEC_HEAP */
233
234	#elif defined(PAGE_KERNEL_EXEC) && defined(CONFIG_X86_PAE)
235	pHdr = (PRTMEMHDR)__vmalloc(cb + sizeof(*pHdr), GFP_KERNEL \| __GFP_HIGHMEM, MY_PAGE_KERNEL_EXEC);
236	#else
237	pHdr = (PRTMEMHDR)vmalloc(cb + sizeof(*pHdr));
238	#endif
239	}
240	else
241	{
242	if (cb <= PAGE_SIZE \|\| (fFlags & RTMEMHDR_FLAG_ANY_CTX))
243	{
244	fFlags \|= RTMEMHDR_FLAG_KMALLOC;
245	pHdr = kmalloc(cb + sizeof(*pHdr),
246	(fFlags & RTMEMHDR_FLAG_ANY_CTX_ALLOC) ? GFP_ATOMIC : GFP_KERNEL);
247	}
248	else
249	pHdr = vmalloc(cb + sizeof(*pHdr));
250	}
251	if (RT_UNLIKELY(!pHdr))
252	return VERR_NO_MEMORY;
253
254	/*
255	* Initialize.
256	*/
257	pHdr->u32Magic = RTMEMHDR_MAGIC;
258	pHdr->fFlags = fFlags;
259	pHdr->cb = cb;
260	pHdr->cbReq = cb;
261
262	*ppHdr = pHdr;
263	return VINF_SUCCESS;
264	}
265
266
267	/**
268	* OS specific free function.
269	*/
270	DECLHIDDEN(void) rtR0MemFree(PRTMEMHDR pHdr)
271	{
272	pHdr->u32Magic += 1;
273	if (pHdr->fFlags & RTMEMHDR_FLAG_KMALLOC)
274	kfree(pHdr);
275	#ifdef RTMEMALLOC_EXEC_HEAP
276	else if (pHdr->fFlags & RTMEMHDR_FLAG_EXEC_HEAP)
277	{
278	RTSPINLOCKTMP SpinlockTmp = RTSPINLOCKTMP_INITIALIZER;
279	RTSpinlockAcquireNoInts(g_HeapExecSpinlock, &SpinlockTmp);
280	RTHeapSimpleFree(g_HeapExec, pHdr);
281	RTSpinlockReleaseNoInts(g_HeapExecSpinlock, &SpinlockTmp);
282	}
283	#endif
284	else
285	vfree(pHdr);
286	}
287
288
289	/**
290	* Compute order. Some functions allocate 2^order pages.
291	*
292	* @returns order.
293	* @param cPages Number of pages.
294	*/
295	static int CalcPowerOf2Order(unsigned long cPages)
296	{
297	int iOrder;
298	unsigned long cTmp;
299
300	for (iOrder = 0, cTmp = cPages; cTmp >>= 1; ++iOrder)
301	;
302	if (cPages & ~(1 << iOrder))
303	++iOrder;
304
305	return iOrder;
306	}
307
308
309	/**
310	* Allocates physical contiguous memory (below 4GB).
311	* The allocation is page aligned and the content is undefined.
312	*
313	* @returns Pointer to the memory block. This is page aligned.
314	* @param pPhys Where to store the physical address.
315	* @param cb The allocation size in bytes. This is always
316	* rounded up to PAGE_SIZE.
317	*/
318	RTR0DECL(void *) RTMemContAlloc(PRTCCPHYS pPhys, size_t cb)
319	{
320	int cOrder;
321	unsigned cPages;
322	struct page *paPages;
323
324	/*
325	* validate input.
326	*/
327	Assert(VALID_PTR(pPhys));
328	Assert(cb > 0);
329
330	/*
331	* Allocate page pointer array.
332	*/
333	cb = RT_ALIGN_Z(cb, PAGE_SIZE);
334	cPages = cb >> PAGE_SHIFT;
335	cOrder = CalcPowerOf2Order(cPages);
336	#if (defined(RT_ARCH_AMD64) \|\| defined(CONFIG_X86_PAE)) && defined(GFP_DMA32)
337	/* ZONE_DMA32: 0-4GB */
338	paPages = alloc_pages(GFP_DMA32, cOrder);
339	if (!paPages)
340	#endif
341	#ifdef RT_ARCH_AMD64
342	/* ZONE_DMA; 0-16MB */
343	paPages = alloc_pages(GFP_DMA, cOrder);
344	#else
345	/* ZONE_NORMAL: 0-896MB */
346	paPages = alloc_pages(GFP_USER, cOrder);
347	#endif
348	if (paPages)
349	{
350	/*
351	* Reserve the pages and mark them executable.
352	*/
353	unsigned iPage;
354	for (iPage = 0; iPage < cPages; iPage++)
355	{
356	Assert(!PageHighMem(&paPages[iPage]));
357	if (iPage + 1 < cPages)
358	{
359	AssertMsg( (uintptr_t)phys_to_virt(page_to_phys(&paPages[iPage])) + PAGE_SIZE
360	== (uintptr_t)phys_to_virt(page_to_phys(&paPages[iPage + 1]))
361	&& page_to_phys(&paPages[iPage]) + PAGE_SIZE
362	== page_to_phys(&paPages[iPage + 1]),
363	("iPage=%i cPages=%u [0]=%#llx,%p [1]=%#llx,%p\n", iPage, cPages,
364	(long long)page_to_phys(&paPages[iPage]), phys_to_virt(page_to_phys(&paPages[iPage])),
365	(long long)page_to_phys(&paPages[iPage + 1]), phys_to_virt(page_to_phys(&paPages[iPage + 1])) ));
366	}
367
368	SetPageReserved(&paPages[iPage]);
369	#if LINUX_VERSION_CODE > KERNEL_VERSION(2, 4, 20) /** @todo find the exact kernel where change_page_attr was introduced. */
370	MY_SET_PAGES_EXEC(&paPages[iPage], 1);
371	#endif
372	}
373	*pPhys = page_to_phys(paPages);
374	return phys_to_virt(page_to_phys(paPages));
375	}
376
377	return NULL;
378	}
379	RT_EXPORT_SYMBOL(RTMemContAlloc);
380
381
382	/**
383	* Frees memory allocated ysing RTMemContAlloc().
384	*
385	* @param pv Pointer to return from RTMemContAlloc().
386	* @param cb The cb parameter passed to RTMemContAlloc().
387	*/
388	RTR0DECL(void) RTMemContFree(void *pv, size_t cb)
389	{
390	if (pv)
391	{
392	int cOrder;
393	unsigned cPages;
394	unsigned iPage;
395	struct page *paPages;
396
397	/* validate */
398	AssertMsg(!((uintptr_t)pv & PAGE_OFFSET_MASK), ("pv=%p\n", pv));
399	Assert(cb > 0);
400
401	/* calc order and get pages */
402	cb = RT_ALIGN_Z(cb, PAGE_SIZE);
403	cPages = cb >> PAGE_SHIFT;
404	cOrder = CalcPowerOf2Order(cPages);
405	paPages = virt_to_page(pv);
406
407	/*
408	* Restore page attributes freeing the pages.
409	*/
410	for (iPage = 0; iPage < cPages; iPage++)
411	{
412	ClearPageReserved(&paPages[iPage]);
413	#if LINUX_VERSION_CODE > KERNEL_VERSION(2, 4, 20) /** @todo find the exact kernel where change_page_attr was introduced. */
414	MY_SET_PAGES_NOEXEC(&paPages[iPage], 1);
415	#endif
416	}
417	__free_pages(paPages, cOrder);
418	}
419	}
420	RT_EXPORT_SYMBOL(RTMemContFree);
421

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/linux/alloc-r0drv-linux.c@ 36555

Download in other formats: