alloc-r0drv-linux.c@ 96763

Last change on this file since 96763 was 96717, checked in by vboxsync, 2 years ago

iprt: Linux: r0drv: Prevent build failure on 5.8+ 32-bit kernels, bugref:4567:65.

Replace vmalloc call with AssertMsgFailed. This code branch should not
be reached by 32-bit 5.8+ kernel. If it is a case, we should be noticed
about that.

Property svn:eol-style set to native
Property svn:keywords set to Id Revision

File size: 15.4 KB

Line
1	/* $Id: alloc-r0drv-linux.c 96717 2022-09-13 10:16:54Z vboxsync $ */
2	/** @file
3	* IPRT - Memory Allocation, Ring-0 Driver, Linux.
4	*/
5
6	/*
7	* Copyright (C) 2006-2022 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* The contents of this file may alternatively be used under the terms
26	* of the Common Development and Distribution License Version 1.0
27	* (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
28	* in the VirtualBox distribution, in which case the provisions of the
29	* CDDL are applicable instead of those of the GPL.
30	*
31	* You may elect to license modified versions of this file under the
32	* terms and conditions of either the GPL or the CDDL or both.
33	*
34	* SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
35	*/
36
37
38	/*********************************************************************************************************************************
39	* Header Files *
40	*********************************************************************************************************************************/
41	#include "the-linux-kernel.h"
42	#include "internal/iprt.h"
43	#include <iprt/mem.h>
44
45	#include <iprt/assert.h>
46	#include <iprt/errcore.h>
47	#include "r0drv/alloc-r0drv.h"
48
49
50	#if (defined(RT_ARCH_AMD64) \|\| defined(DOXYGEN_RUNNING)) && !defined(RTMEMALLOC_EXEC_HEAP)
51	# if RTLNX_VER_MIN(2,6,23) && RTLNX_VER_MAX(5,8,0) && !RTLNX_RHEL_MAJ_PREREQ(8,5)
52	/**
53	* Starting with 2.6.23 we can use __get_vm_area and map_vm_area to allocate
54	* memory in the moduel range. This is preferrable to the exec heap below.
55	*/
56	# define RTMEMALLOC_EXEC_VM_AREA
57	# else
58	/**
59	* We need memory in the module range (~2GB to ~0) this can only be obtained
60	* thru APIs that are not exported (see module_alloc()).
61	*
62	* So, we'll have to create a quick and dirty heap here using BSS memory.
63	* Very annoying and it's going to restrict us!
64	*/
65	# define RTMEMALLOC_EXEC_HEAP
66	# endif
67	#endif
68
69	#ifdef RTMEMALLOC_EXEC_HEAP
70	# include <iprt/heap.h>
71	# include <iprt/spinlock.h>
72	# include <iprt/errcore.h>
73	#endif
74
75	#include "internal/initterm.h"
76
77
78	/*********************************************************************************************************************************
79	* Structures and Typedefs *
80	*********************************************************************************************************************************/
81	#ifdef RTMEMALLOC_EXEC_VM_AREA
82	/**
83	* Extended header used for headers marked with RTMEMHDR_FLAG_EXEC_VM_AREA.
84	*
85	* This is used with allocating executable memory, for things like generated
86	* code and loaded modules.
87	*/
88	typedef struct RTMEMLNXHDREX
89	{
90	/** The VM area for this allocation. */
91	struct vm_struct *pVmArea;
92	void *pvDummy;
93	/** The header we present to the generic API. */
94	RTMEMHDR Hdr;
95	} RTMEMLNXHDREX;
96	AssertCompileSize(RTMEMLNXHDREX, 32);
97	/** Pointer to an extended memory header. */
98	typedef RTMEMLNXHDREX *PRTMEMLNXHDREX;
99	#endif
100
101
102	/*********************************************************************************************************************************
103	* Global Variables *
104	*********************************************************************************************************************************/
105	#ifdef RTMEMALLOC_EXEC_HEAP
106	/** The heap. */
107	static RTHEAPSIMPLE g_HeapExec = NIL_RTHEAPSIMPLE;
108	/** Spinlock protecting the heap. */
109	static RTSPINLOCK g_HeapExecSpinlock = NIL_RTSPINLOCK;
110	#endif
111
112
113	/**
114	* API for cleaning up the heap spinlock on IPRT termination.
115	* This is as RTMemExecDonate specific to AMD64 Linux/GNU.
116	*/
117	DECLHIDDEN(void) rtR0MemExecCleanup(void)
118	{
119	#ifdef RTMEMALLOC_EXEC_HEAP
120	RTSpinlockDestroy(g_HeapExecSpinlock);
121	g_HeapExecSpinlock = NIL_RTSPINLOCK;
122	#endif
123	}
124
125
126	/**
127	* Donate read+write+execute memory to the exec heap.
128	*
129	* This API is specific to AMD64 and Linux/GNU. A kernel module that desires to
130	* use RTMemExecAlloc on AMD64 Linux/GNU will have to donate some statically
131	* allocated memory in the module if it wishes for GCC generated code to work.
132	* GCC can only generate modules that work in the address range ~2GB to ~0
133	* currently.
134	*
135	* The API only accept one single donation.
136	*
137	* @returns IPRT status code.
138	* @retval VERR_NOT_SUPPORTED if the code isn't enabled.
139	* @param pvMemory Pointer to the memory block.
140	* @param cb The size of the memory block.
141	*/
142	RTR0DECL(int) RTR0MemExecDonate(void *pvMemory, size_t cb)
143	{
144	#ifdef RTMEMALLOC_EXEC_HEAP
145	int rc;
146	AssertReturn(g_HeapExec == NIL_RTHEAPSIMPLE, VERR_WRONG_ORDER);
147
148	rc = RTSpinlockCreate(&g_HeapExecSpinlock, RTSPINLOCK_FLAGS_INTERRUPT_SAFE, "RTR0MemExecDonate");
149	if (RT_SUCCESS(rc))
150	{
151	rc = RTHeapSimpleInit(&g_HeapExec, pvMemory, cb);
152	if (RT_FAILURE(rc))
153	rtR0MemExecCleanup();
154	}
155	return rc;
156	#else
157	RT_NOREF_PV(pvMemory); RT_NOREF_PV(cb);
158	return VERR_NOT_SUPPORTED;
159	#endif
160	}
161	RT_EXPORT_SYMBOL(RTR0MemExecDonate);
162
163
164
165	#ifdef RTMEMALLOC_EXEC_VM_AREA
166	/**
167	* Allocate executable kernel memory in the module range.
168	*
169	* @returns Pointer to a allocation header success. NULL on failure.
170	*
171	* @param cb The size the user requested.
172	*/
173	static PRTMEMHDR rtR0MemAllocExecVmArea(size_t cb)
174	{
175	size_t const cbAlloc = RT_ALIGN_Z(sizeof(RTMEMLNXHDREX) + cb, PAGE_SIZE);
176	size_t const cPages = cbAlloc >> PAGE_SHIFT;
177	struct page **papPages;
178	struct vm_struct *pVmArea;
179	size_t iPage;
180
181	pVmArea = __get_vm_area(cbAlloc, VM_ALLOC, MODULES_VADDR, MODULES_END);
182	if (!pVmArea)
183	return NULL;
184	pVmArea->nr_pages = 0; /* paranoia? */
185	pVmArea->pages = NULL; /* paranoia? */
186
187	papPages = (struct page *)kmalloc(cPages sizeof(papPages[0]), GFP_KERNEL \| __GFP_NOWARN);
188	if (!papPages)
189	{
190	vunmap(pVmArea->addr);
191	return NULL;
192	}
193
194	for (iPage = 0; iPage < cPages; iPage++)
195	{
196	papPages[iPage] = alloc_page(GFP_KERNEL \| __GFP_HIGHMEM \| __GFP_NOWARN);
197	if (!papPages[iPage])
198	break;
199	}
200	if (iPage == cPages)
201	{
202	/*
203	* Map the pages.
204	*
205	* Not entirely sure we really need to set nr_pages and pages here, but
206	* they provide a very convenient place for storing something we need
207	* in the free function, if nothing else...
208	*/
209	# if RTLNX_VER_MAX(3,17,0)
210	struct page **papPagesIterator = papPages;
211	# endif
212	pVmArea->nr_pages = cPages;
213	pVmArea->pages = papPages;
214	if (!map_vm_area(pVmArea, PAGE_KERNEL_EXEC,
215	# if RTLNX_VER_MAX(3,17,0)
216	&papPagesIterator
217	# else
218	papPages
219	# endif
220	))
221	{
222	PRTMEMLNXHDREX pHdrEx = (PRTMEMLNXHDREX)pVmArea->addr;
223	pHdrEx->pVmArea = pVmArea;
224	pHdrEx->pvDummy = NULL;
225	return &pHdrEx->Hdr;
226	}
227	/* bail out */
228	# if RTLNX_VER_MAX(3,17,0)
229	pVmArea->nr_pages = papPagesIterator - papPages;
230	# endif
231	}
232
233	vunmap(pVmArea->addr);
234
235	while (iPage-- > 0)
236	__free_page(papPages[iPage]);
237	kfree(papPages);
238
239	return NULL;
240	}
241	#endif /* RTMEMALLOC_EXEC_VM_AREA */
242
243
244	/**
245	* OS specific allocation function.
246	*/
247	DECLHIDDEN(int) rtR0MemAllocEx(size_t cb, uint32_t fFlags, PRTMEMHDR *ppHdr)
248	{
249	PRTMEMHDR pHdr;
250	IPRT_LINUX_SAVE_EFL_AC();
251
252	/*
253	* Allocate.
254	*/
255	if (fFlags & RTMEMHDR_FLAG_EXEC)
256	{
257	if (fFlags & RTMEMHDR_FLAG_ANY_CTX)
258	return VERR_NOT_SUPPORTED;
259
260	#if defined(RT_ARCH_AMD64)
261	# ifdef RTMEMALLOC_EXEC_HEAP
262	if (g_HeapExec != NIL_RTHEAPSIMPLE)
263	{
264	RTSpinlockAcquire(g_HeapExecSpinlock);
265	pHdr = (PRTMEMHDR)RTHeapSimpleAlloc(g_HeapExec, cb + sizeof(*pHdr), 0);
266	RTSpinlockRelease(g_HeapExecSpinlock);
267	fFlags \|= RTMEMHDR_FLAG_EXEC_HEAP;
268	}
269	else
270	pHdr = NULL;
271
272	# elif defined(RTMEMALLOC_EXEC_VM_AREA)
273	pHdr = rtR0MemAllocExecVmArea(cb);
274	fFlags \|= RTMEMHDR_FLAG_EXEC_VM_AREA;
275
276	# else /* !RTMEMALLOC_EXEC_HEAP */
277	# error "you don not want to go here..."
278	pHdr = (PRTMEMHDR)__vmalloc(cb + sizeof(*pHdr), GFP_KERNEL \| __GFP_HIGHMEM \| __GFP_NOWARN, MY_PAGE_KERNEL_EXEC);
279	# endif /* !RTMEMALLOC_EXEC_HEAP */
280
281	#elif defined(PAGE_KERNEL_EXEC) && defined(CONFIG_X86_PAE)
282	# if RTLNX_VER_MIN(5,8,0)
283	AssertMsgFailed(("This point should not be reached, please file a bug\n"));
284	pHdr = NULL;
285	# else
286	pHdr = (PRTMEMHDR)__vmalloc(cb + sizeof(*pHdr), GFP_KERNEL \| __GFP_HIGHMEM \| __GFP_NOWARN, MY_PAGE_KERNEL_EXEC);
287	# endif
288	#else
289	pHdr = (PRTMEMHDR)vmalloc(cb + sizeof(*pHdr));
290	#endif
291	}
292	else
293	{
294	if (
295	#if 1 /* vmalloc has serious performance issues, avoid it. */
296	cb <= PAGE_SIZE16 - sizeof(pHdr)
297	#else
298	cb <= PAGE_SIZE
299	#endif
300	\|\| (fFlags & RTMEMHDR_FLAG_ANY_CTX)
301	)
302	{
303	fFlags \|= RTMEMHDR_FLAG_KMALLOC;
304	pHdr = kmalloc(cb + sizeof(*pHdr),
305	(fFlags & RTMEMHDR_FLAG_ANY_CTX_ALLOC) ? (GFP_ATOMIC \| __GFP_NOWARN)
306	: (GFP_KERNEL \| __GFP_NOWARN));
307	if (RT_UNLIKELY( !pHdr
308	&& cb > PAGE_SIZE
309	&& !(fFlags & RTMEMHDR_FLAG_ANY_CTX) ))
310	{
311	fFlags &= ~RTMEMHDR_FLAG_KMALLOC;
312	pHdr = vmalloc(cb + sizeof(*pHdr));
313	}
314	}
315	else
316	pHdr = vmalloc(cb + sizeof(*pHdr));
317	}
318	if (RT_UNLIKELY(!pHdr))
319	{
320	IPRT_LINUX_RESTORE_EFL_AC();
321	return VERR_NO_MEMORY;
322	}
323
324	/*
325	* Initialize.
326	*/
327	pHdr->u32Magic = RTMEMHDR_MAGIC;
328	pHdr->fFlags = fFlags;
329	pHdr->cb = cb;
330	pHdr->cbReq = cb;
331
332	*ppHdr = pHdr;
333	IPRT_LINUX_RESTORE_EFL_AC();
334	return VINF_SUCCESS;
335	}
336
337
338	/**
339	* OS specific free function.
340	*/
341	DECLHIDDEN(void) rtR0MemFree(PRTMEMHDR pHdr)
342	{
343	IPRT_LINUX_SAVE_EFL_AC();
344
345	pHdr->u32Magic += 1;
346	if (pHdr->fFlags & RTMEMHDR_FLAG_KMALLOC)
347	kfree(pHdr);
348	#ifdef RTMEMALLOC_EXEC_HEAP
349	else if (pHdr->fFlags & RTMEMHDR_FLAG_EXEC_HEAP)
350	{
351	RTSpinlockAcquire(g_HeapExecSpinlock);
352	RTHeapSimpleFree(g_HeapExec, pHdr);
353	RTSpinlockRelease(g_HeapExecSpinlock);
354	}
355	#endif
356	#ifdef RTMEMALLOC_EXEC_VM_AREA
357	else if (pHdr->fFlags & RTMEMHDR_FLAG_EXEC_VM_AREA)
358	{
359	PRTMEMLNXHDREX pHdrEx = RT_FROM_MEMBER(pHdr, RTMEMLNXHDREX, Hdr);
360	size_t iPage = pHdrEx->pVmArea->nr_pages;
361	struct page **papPages = pHdrEx->pVmArea->pages;
362	void *pvMapping = pHdrEx->pVmArea->addr;
363
364	vunmap(pvMapping);
365
366	while (iPage-- > 0)
367	__free_page(papPages[iPage]);
368	kfree(papPages);
369	}
370	#endif
371	else
372	vfree(pHdr);
373
374	IPRT_LINUX_RESTORE_EFL_AC();
375	}
376
377
378
379	/**
380	* Compute order. Some functions allocate 2^order pages.
381	*
382	* @returns order.
383	* @param cPages Number of pages.
384	*/
385	static int CalcPowerOf2Order(unsigned long cPages)
386	{
387	int iOrder;
388	unsigned long cTmp;
389
390	for (iOrder = 0, cTmp = cPages; cTmp >>= 1; ++iOrder)
391	;
392	if (cPages & ~(1 << iOrder))
393	++iOrder;
394
395	return iOrder;
396	}
397
398
399	/**
400	* Allocates physical contiguous memory (below 4GB).
401	* The allocation is page aligned and the content is undefined.
402	*
403	* @returns Pointer to the memory block. This is page aligned.
404	* @param pPhys Where to store the physical address.
405	* @param cb The allocation size in bytes. This is always
406	* rounded up to PAGE_SIZE.
407	*/
408	RTR0DECL(void *) RTMemContAlloc(PRTCCPHYS pPhys, size_t cb)
409	{
410	int cOrder;
411	unsigned cPages;
412	struct page *paPages;
413	void *pvRet;
414	IPRT_LINUX_SAVE_EFL_AC();
415
416	/*
417	* validate input.
418	*/
419	AssertPtr(pPhys);
420	Assert(cb > 0);
421
422	/*
423	* Allocate page pointer array.
424	*/
425	cb = RT_ALIGN_Z(cb, PAGE_SIZE);
426	cPages = cb >> PAGE_SHIFT;
427	cOrder = CalcPowerOf2Order(cPages);
428	#if (defined(RT_ARCH_AMD64) \|\| defined(CONFIG_X86_PAE)) && defined(GFP_DMA32)
429	/* ZONE_DMA32: 0-4GB */
430	paPages = alloc_pages(GFP_DMA32 \| __GFP_NOWARN, cOrder);
431	if (!paPages)
432	#endif
433	#ifdef RT_ARCH_AMD64
434	/* ZONE_DMA; 0-16MB */
435	paPages = alloc_pages(GFP_DMA \| __GFP_NOWARN, cOrder);
436	#else
437	/* ZONE_NORMAL: 0-896MB */
438	paPages = alloc_pages(GFP_USER \| __GFP_NOWARN, cOrder);
439	#endif
440	if (paPages)
441	{
442	/*
443	* Reserve the pages and mark them executable.
444	*/
445	unsigned iPage;
446	for (iPage = 0; iPage < cPages; iPage++)
447	{
448	Assert(!PageHighMem(&paPages[iPage]));
449	if (iPage + 1 < cPages)
450	{
451	AssertMsg( (uintptr_t)phys_to_virt(page_to_phys(&paPages[iPage])) + PAGE_SIZE
452	== (uintptr_t)phys_to_virt(page_to_phys(&paPages[iPage + 1]))
453	&& page_to_phys(&paPages[iPage]) + PAGE_SIZE
454	== page_to_phys(&paPages[iPage + 1]),
455	("iPage=%i cPages=%u [0]=%#llx,%p [1]=%#llx,%p\n", iPage, cPages,
456	(long long)page_to_phys(&paPages[iPage]), phys_to_virt(page_to_phys(&paPages[iPage])),
457	(long long)page_to_phys(&paPages[iPage + 1]), phys_to_virt(page_to_phys(&paPages[iPage + 1])) ));
458	}
459
460	SetPageReserved(&paPages[iPage]);
461	}
462	*pPhys = page_to_phys(paPages);
463	pvRet = phys_to_virt(page_to_phys(paPages));
464	}
465	else
466	pvRet = NULL;
467
468	IPRT_LINUX_RESTORE_EFL_AC();
469	return pvRet;
470	}
471	RT_EXPORT_SYMBOL(RTMemContAlloc);
472
473
474	/**
475	* Frees memory allocated using RTMemContAlloc().
476	*
477	* @param pv Pointer to return from RTMemContAlloc().
478	* @param cb The cb parameter passed to RTMemContAlloc().
479	*/
480	RTR0DECL(void) RTMemContFree(void *pv, size_t cb)
481	{
482	if (pv)
483	{
484	int cOrder;
485	unsigned cPages;
486	unsigned iPage;
487	struct page *paPages;
488	IPRT_LINUX_SAVE_EFL_AC();
489
490	/* validate */
491	AssertMsg(!((uintptr_t)pv & PAGE_OFFSET_MASK), ("pv=%p\n", pv));
492	Assert(cb > 0);
493
494	/* calc order and get pages */
495	cb = RT_ALIGN_Z(cb, PAGE_SIZE);
496	cPages = cb >> PAGE_SHIFT;
497	cOrder = CalcPowerOf2Order(cPages);
498	paPages = virt_to_page(pv);
499
500	/*
501	* Restore page attributes freeing the pages.
502	*/
503	for (iPage = 0; iPage < cPages; iPage++)
504	{
505	ClearPageReserved(&paPages[iPage]);
506	}
507	__free_pages(paPages, cOrder);
508	IPRT_LINUX_RESTORE_EFL_AC();
509	}
510	}
511	RT_EXPORT_SYMBOL(RTMemContFree);
512

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/linux/alloc-r0drv-linux.c@ 96763

Download in other formats: