VirtualBox

source: vbox/trunk/src/VBox/Runtime/r0drv/linux/memobj-r0drv-linux.c@ 43637

Last change on this file since 43637 was 43637, checked in by vboxsync, 12 years ago

Linux 3.7 fixes

  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Rev
File size: 52.1 KB
Line 
1/* $Revision: 43637 $ */
2/** @file
3 * IPRT - Ring-0 Memory Objects, Linux.
4 */
5
6/*
7 * Copyright (C) 2006-2007 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * The contents of this file may alternatively be used under the terms
18 * of the Common Development and Distribution License Version 1.0
19 * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20 * VirtualBox OSE distribution, in which case the provisions of the
21 * CDDL are applicable instead of those of the GPL.
22 *
23 * You may elect to license modified versions of this file under the
24 * terms and conditions of either the GPL or the CDDL or both.
25 */
26
27
28/*******************************************************************************
29* Header Files *
30*******************************************************************************/
31#include "the-linux-kernel.h"
32
33#include <iprt/memobj.h>
34#include <iprt/alloc.h>
35#include <iprt/assert.h>
36#include <iprt/log.h>
37#include <iprt/process.h>
38#include <iprt/string.h>
39#include "internal/memobj.h"
40
41
42/*******************************************************************************
43* Defined Constants And Macros *
44*******************************************************************************/
45/* early 2.6 kernels */
46#ifndef PAGE_SHARED_EXEC
47# define PAGE_SHARED_EXEC PAGE_SHARED
48#endif
49#ifndef PAGE_READONLY_EXEC
50# define PAGE_READONLY_EXEC PAGE_READONLY
51#endif
52
53/*
54 * 2.6.29+ kernels don't work with remap_pfn_range() anymore because
55 * track_pfn_vma_new() is apparently not defined for non-RAM pages.
56 * It should be safe to use vm_insert_page() older kernels as well.
57 */
58#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 23)
59# define VBOX_USE_INSERT_PAGE
60#endif
61#if defined(CONFIG_X86_PAE) \
62 && ( defined(HAVE_26_STYLE_REMAP_PAGE_RANGE) \
63 || ( LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0) \
64 && LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 11)))
65# define VBOX_USE_PAE_HACK
66#endif
67
68
69/*******************************************************************************
70* Structures and Typedefs *
71*******************************************************************************/
72/**
73 * The Darwin version of the memory object structure.
74 */
75typedef struct RTR0MEMOBJLNX
76{
77 /** The core structure. */
78 RTR0MEMOBJINTERNAL Core;
79 /** Set if the allocation is contiguous.
80 * This means it has to be given back as one chunk. */
81 bool fContiguous;
82 /** Set if we've vmap'ed the memory into ring-0. */
83 bool fMappedToRing0;
84 /** The pages in the apPages array. */
85 size_t cPages;
86 /** Array of struct page pointers. (variable size) */
87 struct page *apPages[1];
88} RTR0MEMOBJLNX, *PRTR0MEMOBJLNX;
89
90
91static void rtR0MemObjLinuxFreePages(PRTR0MEMOBJLNX pMemLnx);
92
93
94/**
95 * Helper that converts from a RTR0PROCESS handle to a linux task.
96 *
97 * @returns The corresponding Linux task.
98 * @param R0Process IPRT ring-0 process handle.
99 */
100static struct task_struct *rtR0ProcessToLinuxTask(RTR0PROCESS R0Process)
101{
102 /** @todo fix rtR0ProcessToLinuxTask!! */
103 /** @todo many (all?) callers currently assume that we return 'current'! */
104 return R0Process == RTR0ProcHandleSelf() ? current : NULL;
105}
106
107
108/**
109 * Compute order. Some functions allocate 2^order pages.
110 *
111 * @returns order.
112 * @param cPages Number of pages.
113 */
114static int rtR0MemObjLinuxOrder(size_t cPages)
115{
116 int iOrder;
117 size_t cTmp;
118
119 for (iOrder = 0, cTmp = cPages; cTmp >>= 1; ++iOrder)
120 ;
121 if (cPages & ~((size_t)1 << iOrder))
122 ++iOrder;
123
124 return iOrder;
125}
126
127
128/**
129 * Converts from RTMEM_PROT_* to Linux PAGE_*.
130 *
131 * @returns Linux page protection constant.
132 * @param fProt The IPRT protection mask.
133 * @param fKernel Whether it applies to kernel or user space.
134 */
135static pgprot_t rtR0MemObjLinuxConvertProt(unsigned fProt, bool fKernel)
136{
137 switch (fProt)
138 {
139 default:
140 AssertMsgFailed(("%#x %d\n", fProt, fKernel));
141 case RTMEM_PROT_NONE:
142 return PAGE_NONE;
143
144 case RTMEM_PROT_READ:
145 return fKernel ? PAGE_KERNEL_RO : PAGE_READONLY;
146
147 case RTMEM_PROT_WRITE:
148 case RTMEM_PROT_WRITE | RTMEM_PROT_READ:
149 return fKernel ? PAGE_KERNEL : PAGE_SHARED;
150
151 case RTMEM_PROT_EXEC:
152 case RTMEM_PROT_EXEC | RTMEM_PROT_READ:
153#if defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64)
154 if (fKernel)
155 {
156 pgprot_t fPg = MY_PAGE_KERNEL_EXEC;
157 pgprot_val(fPg) &= ~_PAGE_RW;
158 return fPg;
159 }
160 return PAGE_READONLY_EXEC;
161#else
162 return fKernel ? MY_PAGE_KERNEL_EXEC : PAGE_READONLY_EXEC;
163#endif
164
165 case RTMEM_PROT_WRITE | RTMEM_PROT_EXEC:
166 case RTMEM_PROT_WRITE | RTMEM_PROT_EXEC | RTMEM_PROT_READ:
167 return fKernel ? MY_PAGE_KERNEL_EXEC : PAGE_SHARED_EXEC;
168 }
169}
170
171
172/**
173 * Worker for rtR0MemObjNativeReserveUser and rtR0MemObjNativerMapUser that creates
174 * an empty user space mapping.
175 *
176 * We acquire the mmap_sem of the task!
177 *
178 * @returns Pointer to the mapping.
179 * (void *)-1 on failure.
180 * @param R3PtrFixed (RTR3PTR)-1 if anywhere, otherwise a specific location.
181 * @param cb The size of the mapping.
182 * @param uAlignment The alignment of the mapping.
183 * @param pTask The Linux task to create this mapping in.
184 * @param fProt The RTMEM_PROT_* mask.
185 */
186static void *rtR0MemObjLinuxDoMmap(RTR3PTR R3PtrFixed, size_t cb, size_t uAlignment, struct task_struct *pTask, unsigned fProt)
187{
188 unsigned fLnxProt;
189 unsigned long ulAddr;
190
191 Assert((pTask == current)); /* do_mmap */
192
193 /*
194 * Convert from IPRT protection to mman.h PROT_ and call do_mmap.
195 */
196 fProt &= (RTMEM_PROT_NONE | RTMEM_PROT_READ | RTMEM_PROT_WRITE | RTMEM_PROT_EXEC);
197 if (fProt == RTMEM_PROT_NONE)
198 fLnxProt = PROT_NONE;
199 else
200 {
201 fLnxProt = 0;
202 if (fProt & RTMEM_PROT_READ)
203 fLnxProt |= PROT_READ;
204 if (fProt & RTMEM_PROT_WRITE)
205 fLnxProt |= PROT_WRITE;
206 if (fProt & RTMEM_PROT_EXEC)
207 fLnxProt |= PROT_EXEC;
208 }
209
210 if (R3PtrFixed != (RTR3PTR)-1)
211 {
212#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)
213 ulAddr = vm_mmap(NULL, R3PtrFixed, cb, fLnxProt, MAP_SHARED | MAP_ANONYMOUS | MAP_FIXED, 0);
214#else
215 down_write(&pTask->mm->mmap_sem);
216 ulAddr = do_mmap(NULL, R3PtrFixed, cb, fLnxProt, MAP_SHARED | MAP_ANONYMOUS | MAP_FIXED, 0);
217 up_write(&pTask->mm->mmap_sem);
218#endif
219 }
220 else
221 {
222#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)
223 ulAddr = vm_mmap(NULL, 0, cb, fLnxProt, MAP_SHARED | MAP_ANONYMOUS, 0);
224#else
225 down_write(&pTask->mm->mmap_sem);
226 ulAddr = do_mmap(NULL, 0, cb, fLnxProt, MAP_SHARED | MAP_ANONYMOUS, 0);
227 up_write(&pTask->mm->mmap_sem);
228#endif
229 if ( !(ulAddr & ~PAGE_MASK)
230 && (ulAddr & (uAlignment - 1)))
231 {
232 /** @todo implement uAlignment properly... We'll probably need to make some dummy mappings to fill
233 * up alignment gaps. This is of course complicated by fragmentation (which we might have cause
234 * ourselves) and further by there begin two mmap strategies (top / bottom). */
235 /* For now, just ignore uAlignment requirements... */
236 }
237 }
238
239
240 if (ulAddr & ~PAGE_MASK) /* ~PAGE_MASK == PAGE_OFFSET_MASK */
241 return (void *)-1;
242 return (void *)ulAddr;
243}
244
245
246/**
247 * Worker that destroys a user space mapping.
248 * Undoes what rtR0MemObjLinuxDoMmap did.
249 *
250 * We acquire the mmap_sem of the task!
251 *
252 * @param pv The ring-3 mapping.
253 * @param cb The size of the mapping.
254 * @param pTask The Linux task to destroy this mapping in.
255 */
256static void rtR0MemObjLinuxDoMunmap(void *pv, size_t cb, struct task_struct *pTask)
257{
258#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)
259 Assert(pTask == current);
260 vm_munmap((unsigned long)pv, cb);
261#elif defined(USE_RHEL4_MUNMAP)
262 down_write(&pTask->mm->mmap_sem);
263 do_munmap(pTask->mm, (unsigned long)pv, cb, 0); /* should it be 1 or 0? */
264 up_write(&pTask->mm->mmap_sem);
265#else
266 down_write(&pTask->mm->mmap_sem);
267 do_munmap(pTask->mm, (unsigned long)pv, cb);
268 up_write(&pTask->mm->mmap_sem);
269#endif
270}
271
272
273/**
274 * Internal worker that allocates physical pages and creates the memory object for them.
275 *
276 * @returns IPRT status code.
277 * @param ppMemLnx Where to store the memory object pointer.
278 * @param enmType The object type.
279 * @param cb The number of bytes to allocate.
280 * @param uAlignment The alignment of the physical memory.
281 * Only valid if fContiguous == true, ignored otherwise.
282 * @param fFlagsLnx The page allocation flags (GPFs).
283 * @param fContiguous Whether the allocation must be contiguous.
284 * @param rcNoMem What to return when we're out of pages.
285 */
286static int rtR0MemObjLinuxAllocPages(PRTR0MEMOBJLNX *ppMemLnx, RTR0MEMOBJTYPE enmType, size_t cb,
287 size_t uAlignment, unsigned fFlagsLnx, bool fContiguous, int rcNoMem)
288{
289 size_t iPage;
290 size_t const cPages = cb >> PAGE_SHIFT;
291 struct page *paPages;
292
293 /*
294 * Allocate a memory object structure that's large enough to contain
295 * the page pointer array.
296 */
297 PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_OFFSETOF(RTR0MEMOBJLNX, apPages[cPages]), enmType, NULL, cb);
298 if (!pMemLnx)
299 return VERR_NO_MEMORY;
300 pMemLnx->cPages = cPages;
301
302 if (cPages > 255)
303 {
304# ifdef __GFP_REPEAT
305 /* Try hard to allocate the memory, but the allocation attempt might fail. */
306 fFlagsLnx |= __GFP_REPEAT;
307# endif
308# ifdef __GFP_NOMEMALLOC
309 /* Introduced with Linux 2.6.12: Don't use emergency reserves */
310 fFlagsLnx |= __GFP_NOMEMALLOC;
311# endif
312 }
313
314 /*
315 * Allocate the pages.
316 * For small allocations we'll try contiguous first and then fall back on page by page.
317 */
318#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
319 if ( fContiguous
320 || cb <= PAGE_SIZE * 2)
321 {
322# ifdef VBOX_USE_INSERT_PAGE
323 paPages = alloc_pages(fFlagsLnx | __GFP_COMP, rtR0MemObjLinuxOrder(cPages));
324# else
325 paPages = alloc_pages(fFlagsLnx, rtR0MemObjLinuxOrder(cPages));
326# endif
327 if (paPages)
328 {
329 fContiguous = true;
330 for (iPage = 0; iPage < cPages; iPage++)
331 pMemLnx->apPages[iPage] = &paPages[iPage];
332 }
333 else if (fContiguous)
334 {
335 rtR0MemObjDelete(&pMemLnx->Core);
336 return rcNoMem;
337 }
338 }
339
340 if (!fContiguous)
341 {
342 for (iPage = 0; iPage < cPages; iPage++)
343 {
344 pMemLnx->apPages[iPage] = alloc_page(fFlagsLnx);
345 if (RT_UNLIKELY(!pMemLnx->apPages[iPage]))
346 {
347 while (iPage-- > 0)
348 __free_page(pMemLnx->apPages[iPage]);
349 rtR0MemObjDelete(&pMemLnx->Core);
350 return rcNoMem;
351 }
352 }
353 }
354
355#else /* < 2.4.22 */
356 /** @todo figure out why we didn't allocate page-by-page on 2.4.21 and older... */
357 paPages = alloc_pages(fFlagsLnx, rtR0MemObjLinuxOrder(cPages));
358 if (!paPages)
359 {
360 rtR0MemObjDelete(&pMemLnx->Core);
361 return rcNoMem;
362 }
363 for (iPage = 0; iPage < cPages; iPage++)
364 {
365 pMemLnx->apPages[iPage] = &paPages[iPage];
366 MY_SET_PAGES_EXEC(pMemLnx->apPages[iPage], 1);
367 if (PageHighMem(pMemLnx->apPages[iPage]))
368 BUG();
369 }
370
371 fContiguous = true;
372#endif /* < 2.4.22 */
373 pMemLnx->fContiguous = fContiguous;
374
375 /*
376 * Reserve the pages.
377 */
378 for (iPage = 0; iPage < cPages; iPage++)
379 SetPageReserved(pMemLnx->apPages[iPage]);
380
381 /*
382 * Note that the physical address of memory allocated with alloc_pages(flags, order)
383 * is always 2^(PAGE_SHIFT+order)-aligned.
384 */
385 if ( fContiguous
386 && uAlignment > PAGE_SIZE)
387 {
388 /*
389 * Check for alignment constraints. The physical address of memory allocated with
390 * alloc_pages(flags, order) is always 2^(PAGE_SHIFT+order)-aligned.
391 */
392 if (RT_UNLIKELY(page_to_phys(pMemLnx->apPages[0]) & (uAlignment - 1)))
393 {
394 /*
395 * This should never happen!
396 */
397 printk("rtR0MemObjLinuxAllocPages(cb=0x%lx, uAlignment=0x%lx): alloc_pages(..., %d) returned physical memory at 0x%lx!\n",
398 (unsigned long)cb, (unsigned long)uAlignment, rtR0MemObjLinuxOrder(cPages), (unsigned long)page_to_phys(pMemLnx->apPages[0]));
399 rtR0MemObjLinuxFreePages(pMemLnx);
400 return rcNoMem;
401 }
402 }
403
404 *ppMemLnx = pMemLnx;
405 return VINF_SUCCESS;
406}
407
408
409/**
410 * Frees the physical pages allocated by the rtR0MemObjLinuxAllocPages() call.
411 *
412 * This method does NOT free the object.
413 *
414 * @param pMemLnx The object which physical pages should be freed.
415 */
416static void rtR0MemObjLinuxFreePages(PRTR0MEMOBJLNX pMemLnx)
417{
418 size_t iPage = pMemLnx->cPages;
419 if (iPage > 0)
420 {
421 /*
422 * Restore the page flags.
423 */
424 while (iPage-- > 0)
425 {
426 ClearPageReserved(pMemLnx->apPages[iPage]);
427#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
428#else
429 MY_SET_PAGES_NOEXEC(pMemLnx->apPages[iPage], 1);
430#endif
431 }
432
433 /*
434 * Free the pages.
435 */
436#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
437 if (!pMemLnx->fContiguous)
438 {
439 iPage = pMemLnx->cPages;
440 while (iPage-- > 0)
441 __free_page(pMemLnx->apPages[iPage]);
442 }
443 else
444#endif
445 __free_pages(pMemLnx->apPages[0], rtR0MemObjLinuxOrder(pMemLnx->cPages));
446
447 pMemLnx->cPages = 0;
448 }
449}
450
451
452/**
453 * Maps the allocation into ring-0.
454 *
455 * This will update the RTR0MEMOBJLNX::Core.pv and RTR0MEMOBJ::fMappedToRing0 members.
456 *
457 * Contiguous mappings that isn't in 'high' memory will already be mapped into kernel
458 * space, so we'll use that mapping if possible. If execute access is required, we'll
459 * play safe and do our own mapping.
460 *
461 * @returns IPRT status code.
462 * @param pMemLnx The linux memory object to map.
463 * @param fExecutable Whether execute access is required.
464 */
465static int rtR0MemObjLinuxVMap(PRTR0MEMOBJLNX pMemLnx, bool fExecutable)
466{
467 int rc = VINF_SUCCESS;
468
469 /*
470 * Choose mapping strategy.
471 */
472 bool fMustMap = fExecutable
473 || !pMemLnx->fContiguous;
474 if (!fMustMap)
475 {
476 size_t iPage = pMemLnx->cPages;
477 while (iPage-- > 0)
478 if (PageHighMem(pMemLnx->apPages[iPage]))
479 {
480 fMustMap = true;
481 break;
482 }
483 }
484
485 Assert(!pMemLnx->Core.pv);
486 Assert(!pMemLnx->fMappedToRing0);
487
488 if (fMustMap)
489 {
490 /*
491 * Use vmap - 2.4.22 and later.
492 */
493#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
494 pgprot_t fPg;
495 pgprot_val(fPg) = _PAGE_PRESENT | _PAGE_RW;
496# ifdef _PAGE_NX
497 if (!fExecutable)
498 pgprot_val(fPg) |= _PAGE_NX;
499# endif
500
501# ifdef VM_MAP
502 pMemLnx->Core.pv = vmap(&pMemLnx->apPages[0], pMemLnx->cPages, VM_MAP, fPg);
503# else
504 pMemLnx->Core.pv = vmap(&pMemLnx->apPages[0], pMemLnx->cPages, VM_ALLOC, fPg);
505# endif
506 if (pMemLnx->Core.pv)
507 pMemLnx->fMappedToRing0 = true;
508 else
509 rc = VERR_MAP_FAILED;
510#else /* < 2.4.22 */
511 rc = VERR_NOT_SUPPORTED;
512#endif
513 }
514 else
515 {
516 /*
517 * Use the kernel RAM mapping.
518 */
519 pMemLnx->Core.pv = phys_to_virt(page_to_phys(pMemLnx->apPages[0]));
520 Assert(pMemLnx->Core.pv);
521 }
522
523 return rc;
524}
525
526
527/**
528 * Undoes what rtR0MemObjLinuxVMap() did.
529 *
530 * @param pMemLnx The linux memory object.
531 */
532static void rtR0MemObjLinuxVUnmap(PRTR0MEMOBJLNX pMemLnx)
533{
534#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
535 if (pMemLnx->fMappedToRing0)
536 {
537 Assert(pMemLnx->Core.pv);
538 vunmap(pMemLnx->Core.pv);
539 pMemLnx->fMappedToRing0 = false;
540 }
541#else /* < 2.4.22 */
542 Assert(!pMemLnx->fMappedToRing0);
543#endif
544 pMemLnx->Core.pv = NULL;
545}
546
547
548DECLHIDDEN(int) rtR0MemObjNativeFree(RTR0MEMOBJ pMem)
549{
550 PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
551
552 /*
553 * Release any memory that we've allocated or locked.
554 */
555 switch (pMemLnx->Core.enmType)
556 {
557 case RTR0MEMOBJTYPE_LOW:
558 case RTR0MEMOBJTYPE_PAGE:
559 case RTR0MEMOBJTYPE_CONT:
560 case RTR0MEMOBJTYPE_PHYS:
561 case RTR0MEMOBJTYPE_PHYS_NC:
562 rtR0MemObjLinuxVUnmap(pMemLnx);
563 rtR0MemObjLinuxFreePages(pMemLnx);
564 break;
565
566 case RTR0MEMOBJTYPE_LOCK:
567 if (pMemLnx->Core.u.Lock.R0Process != NIL_RTR0PROCESS)
568 {
569 struct task_struct *pTask = rtR0ProcessToLinuxTask(pMemLnx->Core.u.Lock.R0Process);
570 size_t iPage;
571 Assert(pTask);
572 if (pTask && pTask->mm)
573 down_read(&pTask->mm->mmap_sem);
574
575 iPage = pMemLnx->cPages;
576 while (iPage-- > 0)
577 {
578 if (!PageReserved(pMemLnx->apPages[iPage]))
579 SetPageDirty(pMemLnx->apPages[iPage]);
580 page_cache_release(pMemLnx->apPages[iPage]);
581 }
582
583 if (pTask && pTask->mm)
584 up_read(&pTask->mm->mmap_sem);
585 }
586 /* else: kernel memory - nothing to do here. */
587 break;
588
589 case RTR0MEMOBJTYPE_RES_VIRT:
590 Assert(pMemLnx->Core.pv);
591 if (pMemLnx->Core.u.ResVirt.R0Process != NIL_RTR0PROCESS)
592 {
593 struct task_struct *pTask = rtR0ProcessToLinuxTask(pMemLnx->Core.u.Lock.R0Process);
594 Assert(pTask);
595 if (pTask && pTask->mm)
596 rtR0MemObjLinuxDoMunmap(pMemLnx->Core.pv, pMemLnx->Core.cb, pTask);
597 }
598 else
599 {
600 vunmap(pMemLnx->Core.pv);
601
602 Assert(pMemLnx->cPages == 1 && pMemLnx->apPages[0] != NULL);
603 __free_page(pMemLnx->apPages[0]);
604 pMemLnx->apPages[0] = NULL;
605 pMemLnx->cPages = 0;
606 }
607 pMemLnx->Core.pv = NULL;
608 break;
609
610 case RTR0MEMOBJTYPE_MAPPING:
611 Assert(pMemLnx->cPages == 0); Assert(pMemLnx->Core.pv);
612 if (pMemLnx->Core.u.ResVirt.R0Process != NIL_RTR0PROCESS)
613 {
614 struct task_struct *pTask = rtR0ProcessToLinuxTask(pMemLnx->Core.u.Lock.R0Process);
615 Assert(pTask);
616 if (pTask && pTask->mm)
617 rtR0MemObjLinuxDoMunmap(pMemLnx->Core.pv, pMemLnx->Core.cb, pTask);
618 }
619 else
620 vunmap(pMemLnx->Core.pv);
621 pMemLnx->Core.pv = NULL;
622 break;
623
624 default:
625 AssertMsgFailed(("enmType=%d\n", pMemLnx->Core.enmType));
626 return VERR_INTERNAL_ERROR;
627 }
628 return VINF_SUCCESS;
629}
630
631
632DECLHIDDEN(int) rtR0MemObjNativeAllocPage(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable)
633{
634 PRTR0MEMOBJLNX pMemLnx;
635 int rc;
636
637#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
638 rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_PAGE, cb, PAGE_SIZE, GFP_HIGHUSER,
639 false /* non-contiguous */, VERR_NO_MEMORY);
640#else
641 rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_PAGE, cb, PAGE_SIZE, GFP_USER,
642 false /* non-contiguous */, VERR_NO_MEMORY);
643#endif
644 if (RT_SUCCESS(rc))
645 {
646 rc = rtR0MemObjLinuxVMap(pMemLnx, fExecutable);
647 if (RT_SUCCESS(rc))
648 {
649 *ppMem = &pMemLnx->Core;
650 return rc;
651 }
652
653 rtR0MemObjLinuxFreePages(pMemLnx);
654 rtR0MemObjDelete(&pMemLnx->Core);
655 }
656
657 return rc;
658}
659
660
661DECLHIDDEN(int) rtR0MemObjNativeAllocLow(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable)
662{
663 PRTR0MEMOBJLNX pMemLnx;
664 int rc;
665
666 /* Try to avoid GFP_DMA. GFM_DMA32 was introduced with Linux 2.6.15. */
667#if (defined(RT_ARCH_AMD64) || defined(CONFIG_X86_PAE)) && defined(GFP_DMA32)
668 /* ZONE_DMA32: 0-4GB */
669 rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_LOW, cb, PAGE_SIZE, GFP_DMA32,
670 false /* non-contiguous */, VERR_NO_LOW_MEMORY);
671 if (RT_FAILURE(rc))
672#endif
673#ifdef RT_ARCH_AMD64
674 /* ZONE_DMA: 0-16MB */
675 rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_LOW, cb, PAGE_SIZE, GFP_DMA,
676 false /* non-contiguous */, VERR_NO_LOW_MEMORY);
677#else
678# ifdef CONFIG_X86_PAE
679# endif
680 /* ZONE_NORMAL: 0-896MB */
681 rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_LOW, cb, PAGE_SIZE, GFP_USER,
682 false /* non-contiguous */, VERR_NO_LOW_MEMORY);
683#endif
684 if (RT_SUCCESS(rc))
685 {
686 rc = rtR0MemObjLinuxVMap(pMemLnx, fExecutable);
687 if (RT_SUCCESS(rc))
688 {
689 *ppMem = &pMemLnx->Core;
690 return rc;
691 }
692
693 rtR0MemObjLinuxFreePages(pMemLnx);
694 rtR0MemObjDelete(&pMemLnx->Core);
695 }
696
697 return rc;
698}
699
700
701DECLHIDDEN(int) rtR0MemObjNativeAllocCont(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable)
702{
703 PRTR0MEMOBJLNX pMemLnx;
704 int rc;
705
706#if (defined(RT_ARCH_AMD64) || defined(CONFIG_X86_PAE)) && defined(GFP_DMA32)
707 /* ZONE_DMA32: 0-4GB */
708 rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_CONT, cb, PAGE_SIZE, GFP_DMA32,
709 true /* contiguous */, VERR_NO_CONT_MEMORY);
710 if (RT_FAILURE(rc))
711#endif
712#ifdef RT_ARCH_AMD64
713 /* ZONE_DMA: 0-16MB */
714 rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_CONT, cb, PAGE_SIZE, GFP_DMA,
715 true /* contiguous */, VERR_NO_CONT_MEMORY);
716#else
717 /* ZONE_NORMAL (32-bit hosts): 0-896MB */
718 rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_CONT, cb, PAGE_SIZE, GFP_USER,
719 true /* contiguous */, VERR_NO_CONT_MEMORY);
720#endif
721 if (RT_SUCCESS(rc))
722 {
723 rc = rtR0MemObjLinuxVMap(pMemLnx, fExecutable);
724 if (RT_SUCCESS(rc))
725 {
726#if defined(RT_STRICT) && (defined(RT_ARCH_AMD64) || defined(CONFIG_HIGHMEM64G))
727 size_t iPage = pMemLnx->cPages;
728 while (iPage-- > 0)
729 Assert(page_to_phys(pMemLnx->apPages[iPage]) < _4G);
730#endif
731 pMemLnx->Core.u.Cont.Phys = page_to_phys(pMemLnx->apPages[0]);
732 *ppMem = &pMemLnx->Core;
733 return rc;
734 }
735
736 rtR0MemObjLinuxFreePages(pMemLnx);
737 rtR0MemObjDelete(&pMemLnx->Core);
738 }
739
740 return rc;
741}
742
743
744/**
745 * Worker for rtR0MemObjLinuxAllocPhysSub that tries one allocation strategy.
746 *
747 * @returns IPRT status.
748 * @param ppMemLnx Where to
749 * @param enmType The object type.
750 * @param cb The size of the allocation.
751 * @param uAlignment The alignment of the physical memory.
752 * Only valid for fContiguous == true, ignored otherwise.
753 * @param PhysHighest See rtR0MemObjNativeAllocPhys.
754 * @param fGfp The Linux GFP flags to use for the allocation.
755 */
756static int rtR0MemObjLinuxAllocPhysSub2(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJTYPE enmType,
757 size_t cb, size_t uAlignment, RTHCPHYS PhysHighest, unsigned fGfp)
758{
759 PRTR0MEMOBJLNX pMemLnx;
760 int rc;
761
762 rc = rtR0MemObjLinuxAllocPages(&pMemLnx, enmType, cb, uAlignment, fGfp,
763 enmType == RTR0MEMOBJTYPE_PHYS /* contiguous / non-contiguous */,
764 VERR_NO_PHYS_MEMORY);
765 if (RT_FAILURE(rc))
766 return rc;
767
768 /*
769 * Check the addresses if necessary. (Can be optimized a bit for PHYS.)
770 */
771 if (PhysHighest != NIL_RTHCPHYS)
772 {
773 size_t iPage = pMemLnx->cPages;
774 while (iPage-- > 0)
775 if (page_to_phys(pMemLnx->apPages[iPage]) > PhysHighest)
776 {
777 rtR0MemObjLinuxFreePages(pMemLnx);
778 rtR0MemObjDelete(&pMemLnx->Core);
779 return VERR_NO_MEMORY;
780 }
781 }
782
783 /*
784 * Complete the object.
785 */
786 if (enmType == RTR0MEMOBJTYPE_PHYS)
787 {
788 pMemLnx->Core.u.Phys.PhysBase = page_to_phys(pMemLnx->apPages[0]);
789 pMemLnx->Core.u.Phys.fAllocated = true;
790 }
791 *ppMem = &pMemLnx->Core;
792 return rc;
793}
794
795
796/**
797 * Worker for rtR0MemObjNativeAllocPhys and rtR0MemObjNativeAllocPhysNC.
798 *
799 * @returns IPRT status.
800 * @param ppMem Where to store the memory object pointer on success.
801 * @param enmType The object type.
802 * @param cb The size of the allocation.
803 * @param uAlignment The alignment of the physical memory.
804 * Only valid for enmType == RTR0MEMOBJTYPE_PHYS, ignored otherwise.
805 * @param PhysHighest See rtR0MemObjNativeAllocPhys.
806 */
807static int rtR0MemObjLinuxAllocPhysSub(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJTYPE enmType,
808 size_t cb, size_t uAlignment, RTHCPHYS PhysHighest)
809{
810 int rc;
811
812 /*
813 * There are two clear cases and that's the <=16MB and anything-goes ones.
814 * When the physical address limit is somewhere in-between those two we'll
815 * just have to try, starting with HIGHUSER and working our way thru the
816 * different types, hoping we'll get lucky.
817 *
818 * We should probably move this physical address restriction logic up to
819 * the page alloc function as it would be more efficient there. But since
820 * we don't expect this to be a performance issue just yet it can wait.
821 */
822 if (PhysHighest == NIL_RTHCPHYS)
823 /* ZONE_HIGHMEM: the whole physical memory */
824 rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_HIGHUSER);
825 else if (PhysHighest <= _1M * 16)
826 /* ZONE_DMA: 0-16MB */
827 rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_DMA);
828 else
829 {
830 rc = VERR_NO_MEMORY;
831 if (RT_FAILURE(rc))
832 /* ZONE_HIGHMEM: the whole physical memory */
833 rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_HIGHUSER);
834 if (RT_FAILURE(rc))
835 /* ZONE_NORMAL: 0-896MB */
836 rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_USER);
837#ifdef GFP_DMA32
838 if (RT_FAILURE(rc))
839 /* ZONE_DMA32: 0-4GB */
840 rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_DMA32);
841#endif
842 if (RT_FAILURE(rc))
843 /* ZONE_DMA: 0-16MB */
844 rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, GFP_DMA);
845 }
846 return rc;
847}
848
849
850/**
851 * Translates a kernel virtual address to a linux page structure by walking the
852 * page tables.
853 *
854 * @note We do assume that the page tables will not change as we are walking
855 * them. This assumption is rather forced by the fact that I could not
856 * immediately see any way of preventing this from happening. So, we
857 * take some extra care when accessing them.
858 *
859 * Because of this, we don't want to use this function on memory where
860 * attribute changes to nearby pages is likely to cause large pages to
861 * be used or split up. So, don't use this for the linear mapping of
862 * physical memory.
863 *
864 * @returns Pointer to the page structur or NULL if it could not be found.
865 * @param pv The kernel virtual address.
866 */
867static struct page *rtR0MemObjLinuxVirtToPage(void *pv)
868{
869 unsigned long ulAddr = (unsigned long)pv;
870 unsigned long pfn;
871 struct page *pPage;
872 pte_t *pEntry;
873 union
874 {
875 pgd_t Global;
876#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 11)
877 pud_t Upper;
878#endif
879 pmd_t Middle;
880 pte_t Entry;
881 } u;
882
883 /* Should this happen in a situation this code will be called in? And if
884 * so, can it change under our feet? See also
885 * "Documentation/vm/active_mm.txt" in the kernel sources. */
886 if (RT_UNLIKELY(!current->active_mm))
887 return NULL;
888 u.Global = *pgd_offset(current->active_mm, ulAddr);
889 if (RT_UNLIKELY(pgd_none(u.Global)))
890 return NULL;
891
892#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 11)
893 u.Upper = *pud_offset(&u.Global, ulAddr);
894 if (RT_UNLIKELY(pud_none(u.Upper)))
895 return NULL;
896# if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 25)
897 if (pud_large(u.Upper))
898 {
899 pPage = pud_page(u.Upper);
900 AssertReturn(pPage, NULL);
901 pfn = page_to_pfn(pPage); /* doing the safe way... */
902 pfn += (ulAddr >> PAGE_SHIFT) & ((UINT32_C(1) << (PUD_SHIFT - PAGE_SHIFT)) - 1);
903 return pfn_to_page(pfn);
904 }
905# endif
906
907 u.Middle = *pmd_offset(&u.Upper, ulAddr);
908#else /* < 2.6.11 */
909 u.Middle = *pmd_offset(&u.Global, ulAddr);
910#endif /* < 2.6.11 */
911 if (RT_UNLIKELY(pmd_none(u.Middle)))
912 return NULL;
913#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0)
914 if (pmd_large(u.Middle))
915 {
916 pPage = pmd_page(u.Middle);
917 AssertReturn(pPage, NULL);
918 pfn = page_to_pfn(pPage); /* doing the safe way... */
919 pfn += (ulAddr >> PAGE_SHIFT) & ((UINT32_C(1) << (PMD_SHIFT - PAGE_SHIFT)) - 1);
920 return pfn_to_page(pfn);
921 }
922#endif
923
924#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 5) || defined(pte_offset_map) /* As usual, RHEL 3 had pte_offset_map earlier. */
925 pEntry = pte_offset_map(&u.Middle, ulAddr);
926#else
927 pEntry = pte_offset(&u.Middle, ulAddr);
928#endif
929 if (RT_UNLIKELY(!pEntry))
930 return NULL;
931 u.Entry = *pEntry;
932#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 5) || defined(pte_offset_map)
933 pte_unmap(pEntry);
934#endif
935
936 if (RT_UNLIKELY(!pte_present(u.Entry)))
937 return NULL;
938 return pte_page(u.Entry);
939}
940
941
942DECLHIDDEN(int) rtR0MemObjNativeAllocPhys(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest, size_t uAlignment)
943{
944 return rtR0MemObjLinuxAllocPhysSub(ppMem, RTR0MEMOBJTYPE_PHYS, cb, uAlignment, PhysHighest);
945}
946
947
948DECLHIDDEN(int) rtR0MemObjNativeAllocPhysNC(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest)
949{
950 return rtR0MemObjLinuxAllocPhysSub(ppMem, RTR0MEMOBJTYPE_PHYS_NC, cb, PAGE_SIZE, PhysHighest);
951}
952
953
954DECLHIDDEN(int) rtR0MemObjNativeEnterPhys(PPRTR0MEMOBJINTERNAL ppMem, RTHCPHYS Phys, size_t cb, uint32_t uCachePolicy)
955{
956 /*
957 * All we need to do here is to validate that we can use
958 * ioremap on the specified address (32/64-bit dma_addr_t).
959 */
960 PRTR0MEMOBJLNX pMemLnx;
961 dma_addr_t PhysAddr = Phys;
962 AssertMsgReturn(PhysAddr == Phys, ("%#llx\n", (unsigned long long)Phys), VERR_ADDRESS_TOO_BIG);
963
964 pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_PHYS, NULL, cb);
965 if (!pMemLnx)
966 return VERR_NO_MEMORY;
967
968 pMemLnx->Core.u.Phys.PhysBase = PhysAddr;
969 pMemLnx->Core.u.Phys.fAllocated = false;
970 pMemLnx->Core.u.Phys.uCachePolicy = uCachePolicy;
971 Assert(!pMemLnx->cPages);
972 *ppMem = &pMemLnx->Core;
973 return VINF_SUCCESS;
974}
975
976
977DECLHIDDEN(int) rtR0MemObjNativeLockUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3Ptr, size_t cb, uint32_t fAccess, RTR0PROCESS R0Process)
978{
979 const int cPages = cb >> PAGE_SHIFT;
980 struct task_struct *pTask = rtR0ProcessToLinuxTask(R0Process);
981 struct vm_area_struct **papVMAs;
982 PRTR0MEMOBJLNX pMemLnx;
983 int rc = VERR_NO_MEMORY;
984 int const fWrite = fAccess & RTMEM_PROT_WRITE ? 1 : 0;
985
986 /*
987 * Check for valid task and size overflows.
988 */
989 if (!pTask)
990 return VERR_NOT_SUPPORTED;
991 if (((size_t)cPages << PAGE_SHIFT) != cb)
992 return VERR_OUT_OF_RANGE;
993
994 /*
995 * Allocate the memory object and a temporary buffer for the VMAs.
996 */
997 pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_OFFSETOF(RTR0MEMOBJLNX, apPages[cPages]), RTR0MEMOBJTYPE_LOCK, (void *)R3Ptr, cb);
998 if (!pMemLnx)
999 return VERR_NO_MEMORY;
1000
1001 papVMAs = (struct vm_area_struct **)RTMemAlloc(sizeof(*papVMAs) * cPages);
1002 if (papVMAs)
1003 {
1004 down_read(&pTask->mm->mmap_sem);
1005
1006 /*
1007 * Get user pages.
1008 */
1009 rc = get_user_pages(pTask, /* Task for fault accounting. */
1010 pTask->mm, /* Whose pages. */
1011 R3Ptr, /* Where from. */
1012 cPages, /* How many pages. */
1013 fWrite, /* Write to memory. */
1014 fWrite, /* force write access. */
1015 &pMemLnx->apPages[0], /* Page array. */
1016 papVMAs); /* vmas */
1017 if (rc == cPages)
1018 {
1019 /*
1020 * Flush dcache (required?), protect against fork and _really_ pin the page
1021 * table entries. get_user_pages() will protect against swapping out the
1022 * pages but it will NOT protect against removing page table entries. This
1023 * can be achieved with
1024 * - using mlock / mmap(..., MAP_LOCKED, ...) from userland. This requires
1025 * an appropriate limit set up with setrlimit(..., RLIMIT_MEMLOCK, ...).
1026 * Usual Linux distributions support only a limited size of locked pages
1027 * (e.g. 32KB).
1028 * - setting the PageReserved bit (as we do in rtR0MemObjLinuxAllocPages()
1029 * or by
1030 * - setting the VM_LOCKED flag. This is the same as doing mlock() without
1031 * a range check.
1032 */
1033 /** @todo The Linux fork() protection will require more work if this API
1034 * is to be used for anything but locking VM pages. */
1035 while (rc-- > 0)
1036 {
1037 flush_dcache_page(pMemLnx->apPages[rc]);
1038 papVMAs[rc]->vm_flags |= (VM_DONTCOPY | VM_LOCKED);
1039 }
1040
1041 up_read(&pTask->mm->mmap_sem);
1042
1043 RTMemFree(papVMAs);
1044
1045 pMemLnx->Core.u.Lock.R0Process = R0Process;
1046 pMemLnx->cPages = cPages;
1047 Assert(!pMemLnx->fMappedToRing0);
1048 *ppMem = &pMemLnx->Core;
1049
1050 return VINF_SUCCESS;
1051 }
1052
1053 /*
1054 * Failed - we need to unlock any pages that we succeeded to lock.
1055 */
1056 while (rc-- > 0)
1057 {
1058 if (!PageReserved(pMemLnx->apPages[rc]))
1059 SetPageDirty(pMemLnx->apPages[rc]);
1060 page_cache_release(pMemLnx->apPages[rc]);
1061 }
1062
1063 up_read(&pTask->mm->mmap_sem);
1064
1065 RTMemFree(papVMAs);
1066 rc = VERR_LOCK_FAILED;
1067 }
1068
1069 rtR0MemObjDelete(&pMemLnx->Core);
1070 return rc;
1071}
1072
1073
1074DECLHIDDEN(int) rtR0MemObjNativeLockKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pv, size_t cb, uint32_t fAccess)
1075{
1076 void *pvLast = (uint8_t *)pv + cb - 1;
1077 size_t const cPages = cb >> PAGE_SHIFT;
1078 PRTR0MEMOBJLNX pMemLnx;
1079 bool fLinearMapping;
1080 int rc;
1081 uint8_t *pbPage;
1082 size_t iPage;
1083 NOREF(fAccess);
1084
1085 if ( !RTR0MemKernelIsValidAddr(pv)
1086 || !RTR0MemKernelIsValidAddr(pv + cb))
1087 return VERR_INVALID_PARAMETER;
1088
1089 /*
1090 * The lower part of the kernel memory has a linear mapping between
1091 * physical and virtual addresses. So we take a short cut here. This is
1092 * assumed to be the cleanest way to handle those addresses (and the code
1093 * is well tested, though the test for determining it is not very nice).
1094 * If we ever decide it isn't we can still remove it.
1095 */
1096#if 0
1097 fLinearMapping = (unsigned long)pvLast < VMALLOC_START;
1098#else
1099 fLinearMapping = (unsigned long)pv >= (unsigned long)__va(0)
1100 && (unsigned long)pvLast < (unsigned long)high_memory;
1101#endif
1102
1103 /*
1104 * Allocate the memory object.
1105 */
1106 pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_OFFSETOF(RTR0MEMOBJLNX, apPages[cPages]), RTR0MEMOBJTYPE_LOCK, pv, cb);
1107 if (!pMemLnx)
1108 return VERR_NO_MEMORY;
1109
1110 /*
1111 * Gather the pages.
1112 * We ASSUME all kernel pages are non-swappable and non-movable.
1113 */
1114 rc = VINF_SUCCESS;
1115 pbPage = (uint8_t *)pvLast;
1116 iPage = cPages;
1117 if (!fLinearMapping)
1118 {
1119 while (iPage-- > 0)
1120 {
1121 struct page *pPage = rtR0MemObjLinuxVirtToPage(pbPage);
1122 if (RT_UNLIKELY(!pPage))
1123 {
1124 rc = VERR_LOCK_FAILED;
1125 break;
1126 }
1127 pMemLnx->apPages[iPage] = pPage;
1128 pbPage -= PAGE_SIZE;
1129 }
1130 }
1131 else
1132 {
1133 while (iPage-- > 0)
1134 {
1135 pMemLnx->apPages[iPage] = virt_to_page(pbPage);
1136 pbPage -= PAGE_SIZE;
1137 }
1138 }
1139 if (RT_SUCCESS(rc))
1140 {
1141 /*
1142 * Complete the memory object and return.
1143 */
1144 pMemLnx->Core.u.Lock.R0Process = NIL_RTR0PROCESS;
1145 pMemLnx->cPages = cPages;
1146 Assert(!pMemLnx->fMappedToRing0);
1147 *ppMem = &pMemLnx->Core;
1148
1149 return VINF_SUCCESS;
1150 }
1151
1152 rtR0MemObjDelete(&pMemLnx->Core);
1153 return rc;
1154}
1155
1156
1157DECLHIDDEN(int) rtR0MemObjNativeReserveKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pvFixed, size_t cb, size_t uAlignment)
1158{
1159#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
1160 const size_t cPages = cb >> PAGE_SHIFT;
1161 struct page *pDummyPage;
1162 struct page **papPages;
1163
1164 /* check for unsupported stuff. */
1165 AssertMsgReturn(pvFixed == (void *)-1, ("%p\n", pvFixed), VERR_NOT_SUPPORTED);
1166 if (uAlignment > PAGE_SIZE)
1167 return VERR_NOT_SUPPORTED;
1168
1169 /*
1170 * Allocate a dummy page and create a page pointer array for vmap such that
1171 * the dummy page is mapped all over the reserved area.
1172 */
1173 pDummyPage = alloc_page(GFP_HIGHUSER);
1174 if (!pDummyPage)
1175 return VERR_NO_MEMORY;
1176 papPages = RTMemAlloc(sizeof(*papPages) * cPages);
1177 if (papPages)
1178 {
1179 void *pv;
1180 size_t iPage = cPages;
1181 while (iPage-- > 0)
1182 papPages[iPage] = pDummyPage;
1183# ifdef VM_MAP
1184 pv = vmap(papPages, cPages, VM_MAP, PAGE_KERNEL_RO);
1185# else
1186 pv = vmap(papPages, cPages, VM_ALLOC, PAGE_KERNEL_RO);
1187# endif
1188 RTMemFree(papPages);
1189 if (pv)
1190 {
1191 PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_RES_VIRT, pv, cb);
1192 if (pMemLnx)
1193 {
1194 pMemLnx->Core.u.ResVirt.R0Process = NIL_RTR0PROCESS;
1195 pMemLnx->cPages = 1;
1196 pMemLnx->apPages[0] = pDummyPage;
1197 *ppMem = &pMemLnx->Core;
1198 return VINF_SUCCESS;
1199 }
1200 vunmap(pv);
1201 }
1202 }
1203 __free_page(pDummyPage);
1204 return VERR_NO_MEMORY;
1205
1206#else /* < 2.4.22 */
1207 /*
1208 * Could probably use ioremap here, but the caller is in a better position than us
1209 * to select some safe physical memory.
1210 */
1211 return VERR_NOT_SUPPORTED;
1212#endif
1213}
1214
1215
1216DECLHIDDEN(int) rtR0MemObjNativeReserveUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3PtrFixed, size_t cb, size_t uAlignment, RTR0PROCESS R0Process)
1217{
1218 PRTR0MEMOBJLNX pMemLnx;
1219 void *pv;
1220 struct task_struct *pTask = rtR0ProcessToLinuxTask(R0Process);
1221 if (!pTask)
1222 return VERR_NOT_SUPPORTED;
1223
1224 /*
1225 * Check that the specified alignment is supported.
1226 */
1227 if (uAlignment > PAGE_SIZE)
1228 return VERR_NOT_SUPPORTED;
1229
1230 /*
1231 * Let rtR0MemObjLinuxDoMmap do the difficult bits.
1232 */
1233 pv = rtR0MemObjLinuxDoMmap(R3PtrFixed, cb, uAlignment, pTask, RTMEM_PROT_NONE);
1234 if (pv == (void *)-1)
1235 return VERR_NO_MEMORY;
1236
1237 pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_RES_VIRT, pv, cb);
1238 if (!pMemLnx)
1239 {
1240 rtR0MemObjLinuxDoMunmap(pv, cb, pTask);
1241 return VERR_NO_MEMORY;
1242 }
1243
1244 pMemLnx->Core.u.ResVirt.R0Process = R0Process;
1245 *ppMem = &pMemLnx->Core;
1246 return VINF_SUCCESS;
1247}
1248
1249
1250DECLHIDDEN(int) rtR0MemObjNativeMapKernel(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap,
1251 void *pvFixed, size_t uAlignment,
1252 unsigned fProt, size_t offSub, size_t cbSub)
1253{
1254 int rc = VERR_NO_MEMORY;
1255 PRTR0MEMOBJLNX pMemLnxToMap = (PRTR0MEMOBJLNX)pMemToMap;
1256 PRTR0MEMOBJLNX pMemLnx;
1257
1258 /* Fail if requested to do something we can't. */
1259 AssertMsgReturn(!offSub && !cbSub, ("%#x %#x\n", offSub, cbSub), VERR_NOT_SUPPORTED);
1260 AssertMsgReturn(pvFixed == (void *)-1, ("%p\n", pvFixed), VERR_NOT_SUPPORTED);
1261 if (uAlignment > PAGE_SIZE)
1262 return VERR_NOT_SUPPORTED;
1263
1264 /*
1265 * Create the IPRT memory object.
1266 */
1267 pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_MAPPING, NULL, pMemLnxToMap->Core.cb);
1268 if (pMemLnx)
1269 {
1270 if (pMemLnxToMap->cPages)
1271 {
1272#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
1273 /*
1274 * Use vmap - 2.4.22 and later.
1275 */
1276 pgprot_t fPg = rtR0MemObjLinuxConvertProt(fProt, true /* kernel */);
1277# ifdef VM_MAP
1278 pMemLnx->Core.pv = vmap(&pMemLnxToMap->apPages[0], pMemLnxToMap->cPages, VM_MAP, fPg);
1279# else
1280 pMemLnx->Core.pv = vmap(&pMemLnxToMap->apPages[0], pMemLnxToMap->cPages, VM_ALLOC, fPg);
1281# endif
1282 if (pMemLnx->Core.pv)
1283 {
1284 pMemLnx->fMappedToRing0 = true;
1285 rc = VINF_SUCCESS;
1286 }
1287 else
1288 rc = VERR_MAP_FAILED;
1289
1290#else /* < 2.4.22 */
1291 /*
1292 * Only option here is to share mappings if possible and forget about fProt.
1293 */
1294 if (rtR0MemObjIsRing3(pMemToMap))
1295 rc = VERR_NOT_SUPPORTED;
1296 else
1297 {
1298 rc = VINF_SUCCESS;
1299 if (!pMemLnxToMap->Core.pv)
1300 rc = rtR0MemObjLinuxVMap(pMemLnxToMap, !!(fProt & RTMEM_PROT_EXEC));
1301 if (RT_SUCCESS(rc))
1302 {
1303 Assert(pMemLnxToMap->Core.pv);
1304 pMemLnx->Core.pv = pMemLnxToMap->Core.pv;
1305 }
1306 }
1307#endif
1308 }
1309 else
1310 {
1311 /*
1312 * MMIO / physical memory.
1313 */
1314 Assert(pMemLnxToMap->Core.enmType == RTR0MEMOBJTYPE_PHYS && !pMemLnxToMap->Core.u.Phys.fAllocated);
1315 pMemLnx->Core.pv = pMemLnxToMap->Core.u.Phys.uCachePolicy == RTMEM_CACHE_POLICY_MMIO
1316 ? ioremap_nocache(pMemLnxToMap->Core.u.Phys.PhysBase, pMemLnxToMap->Core.cb)
1317 : ioremap(pMemLnxToMap->Core.u.Phys.PhysBase, pMemLnxToMap->Core.cb);
1318 if (pMemLnx->Core.pv)
1319 {
1320 /** @todo fix protection. */
1321 rc = VINF_SUCCESS;
1322 }
1323 }
1324 if (RT_SUCCESS(rc))
1325 {
1326 pMemLnx->Core.u.Mapping.R0Process = NIL_RTR0PROCESS;
1327 *ppMem = &pMemLnx->Core;
1328 return VINF_SUCCESS;
1329 }
1330 rtR0MemObjDelete(&pMemLnx->Core);
1331 }
1332
1333 return rc;
1334}
1335
1336
1337#ifdef VBOX_USE_PAE_HACK
1338/**
1339 * Replace the PFN of a PTE with the address of the actual page.
1340 *
1341 * The caller maps a reserved dummy page at the address with the desired access
1342 * and flags.
1343 *
1344 * This hack is required for older Linux kernels which don't provide
1345 * remap_pfn_range().
1346 *
1347 * @returns 0 on success, -ENOMEM on failure.
1348 * @param mm The memory context.
1349 * @param ulAddr The mapping address.
1350 * @param Phys The physical address of the page to map.
1351 */
1352static int rtR0MemObjLinuxFixPte(struct mm_struct *mm, unsigned long ulAddr, RTHCPHYS Phys)
1353{
1354 int rc = -ENOMEM;
1355 pgd_t *pgd;
1356
1357 spin_lock(&mm->page_table_lock);
1358
1359 pgd = pgd_offset(mm, ulAddr);
1360 if (!pgd_none(*pgd) && !pgd_bad(*pgd))
1361 {
1362 pmd_t *pmd = pmd_offset(pgd, ulAddr);
1363 if (!pmd_none(*pmd))
1364 {
1365 pte_t *ptep = pte_offset_map(pmd, ulAddr);
1366 if (ptep)
1367 {
1368 pte_t pte = *ptep;
1369 pte.pte_high &= 0xfff00000;
1370 pte.pte_high |= ((Phys >> 32) & 0x000fffff);
1371 pte.pte_low &= 0x00000fff;
1372 pte.pte_low |= (Phys & 0xfffff000);
1373 set_pte(ptep, pte);
1374 pte_unmap(ptep);
1375 rc = 0;
1376 }
1377 }
1378 }
1379
1380 spin_unlock(&mm->page_table_lock);
1381 return rc;
1382}
1383#endif /* VBOX_USE_PAE_HACK */
1384
1385
1386DECLHIDDEN(int) rtR0MemObjNativeMapUser(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap, RTR3PTR R3PtrFixed,
1387 size_t uAlignment, unsigned fProt, RTR0PROCESS R0Process)
1388{
1389 struct task_struct *pTask = rtR0ProcessToLinuxTask(R0Process);
1390 PRTR0MEMOBJLNX pMemLnxToMap = (PRTR0MEMOBJLNX)pMemToMap;
1391 int rc = VERR_NO_MEMORY;
1392 PRTR0MEMOBJLNX pMemLnx;
1393#ifdef VBOX_USE_PAE_HACK
1394 struct page *pDummyPage;
1395 RTHCPHYS DummyPhys;
1396#endif
1397
1398 /*
1399 * Check for restrictions.
1400 */
1401 if (!pTask)
1402 return VERR_NOT_SUPPORTED;
1403 if (uAlignment > PAGE_SIZE)
1404 return VERR_NOT_SUPPORTED;
1405
1406#ifdef VBOX_USE_PAE_HACK
1407 /*
1408 * Allocate a dummy page for use when mapping the memory.
1409 */
1410 pDummyPage = alloc_page(GFP_USER);
1411 if (!pDummyPage)
1412 return VERR_NO_MEMORY;
1413 SetPageReserved(pDummyPage);
1414 DummyPhys = page_to_phys(pDummyPage);
1415#endif
1416
1417 /*
1418 * Create the IPRT memory object.
1419 */
1420 pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_MAPPING, NULL, pMemLnxToMap->Core.cb);
1421 if (pMemLnx)
1422 {
1423 /*
1424 * Allocate user space mapping.
1425 */
1426 void *pv;
1427 pv = rtR0MemObjLinuxDoMmap(R3PtrFixed, pMemLnxToMap->Core.cb, uAlignment, pTask, fProt);
1428 if (pv != (void *)-1)
1429 {
1430 /*
1431 * Map page by page into the mmap area.
1432 * This is generic, paranoid and not very efficient.
1433 */
1434 pgprot_t fPg = rtR0MemObjLinuxConvertProt(fProt, false /* user */);
1435 unsigned long ulAddrCur = (unsigned long)pv;
1436 const size_t cPages = pMemLnxToMap->Core.cb >> PAGE_SHIFT;
1437 size_t iPage;
1438
1439 down_write(&pTask->mm->mmap_sem);
1440
1441 rc = VINF_SUCCESS;
1442 if (pMemLnxToMap->cPages)
1443 {
1444 for (iPage = 0; iPage < cPages; iPage++, ulAddrCur += PAGE_SIZE)
1445 {
1446#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 11)
1447 RTHCPHYS Phys = page_to_phys(pMemLnxToMap->apPages[iPage]);
1448#endif
1449#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0) || defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1450 struct vm_area_struct *vma = find_vma(pTask->mm, ulAddrCur); /* this is probably the same for all the pages... */
1451 AssertBreakStmt(vma, rc = VERR_INTERNAL_ERROR);
1452#endif
1453#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 0) && defined(RT_ARCH_X86)
1454 /* remap_page_range() limitation on x86 */
1455 AssertBreakStmt(Phys < _4G, rc = VERR_NO_MEMORY);
1456#endif
1457
1458#if defined(VBOX_USE_INSERT_PAGE) && LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 22)
1459 rc = vm_insert_page(vma, ulAddrCur, pMemLnxToMap->apPages[iPage]);
1460 /* Thes flags help making 100% sure some bad stuff wont happen (swap, core, ++).
1461 * See remap_pfn_range() in mm/memory.c */
1462#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)
1463 vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
1464#else
1465 vma->vm_flags |= VM_RESERVED;
1466#endif
1467#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 11)
1468 rc = remap_pfn_range(vma, ulAddrCur, page_to_pfn(pMemLnxToMap->apPages[iPage]), PAGE_SIZE, fPg);
1469#elif defined(VBOX_USE_PAE_HACK)
1470 rc = remap_page_range(vma, ulAddrCur, DummyPhys, PAGE_SIZE, fPg);
1471 if (!rc)
1472 rc = rtR0MemObjLinuxFixPte(pTask->mm, ulAddrCur, Phys);
1473#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0) || defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1474 rc = remap_page_range(vma, ulAddrCur, Phys, PAGE_SIZE, fPg);
1475#else /* 2.4 */
1476 rc = remap_page_range(ulAddrCur, Phys, PAGE_SIZE, fPg);
1477#endif
1478 if (rc)
1479 {
1480 rc = VERR_NO_MEMORY;
1481 break;
1482 }
1483 }
1484 }
1485 else
1486 {
1487 RTHCPHYS Phys;
1488 if (pMemLnxToMap->Core.enmType == RTR0MEMOBJTYPE_PHYS)
1489 Phys = pMemLnxToMap->Core.u.Phys.PhysBase;
1490 else if (pMemLnxToMap->Core.enmType == RTR0MEMOBJTYPE_CONT)
1491 Phys = pMemLnxToMap->Core.u.Cont.Phys;
1492 else
1493 {
1494 AssertMsgFailed(("%d\n", pMemLnxToMap->Core.enmType));
1495 Phys = NIL_RTHCPHYS;
1496 }
1497 if (Phys != NIL_RTHCPHYS)
1498 {
1499 for (iPage = 0; iPage < cPages; iPage++, ulAddrCur += PAGE_SIZE, Phys += PAGE_SIZE)
1500 {
1501#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0) || defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1502 struct vm_area_struct *vma = find_vma(pTask->mm, ulAddrCur); /* this is probably the same for all the pages... */
1503 AssertBreakStmt(vma, rc = VERR_INTERNAL_ERROR);
1504#endif
1505#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 0) && defined(RT_ARCH_X86)
1506 /* remap_page_range() limitation on x86 */
1507 AssertBreakStmt(Phys < _4G, rc = VERR_NO_MEMORY);
1508#endif
1509
1510#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 11)
1511 rc = remap_pfn_range(vma, ulAddrCur, Phys, PAGE_SIZE, fPg);
1512#elif defined(VBOX_USE_PAE_HACK)
1513 rc = remap_page_range(vma, ulAddrCur, DummyPhys, PAGE_SIZE, fPg);
1514 if (!rc)
1515 rc = rtR0MemObjLinuxFixPte(pTask->mm, ulAddrCur, Phys);
1516#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0) || defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1517 rc = remap_page_range(vma, ulAddrCur, Phys, PAGE_SIZE, fPg);
1518#else /* 2.4 */
1519 rc = remap_page_range(ulAddrCur, Phys, PAGE_SIZE, fPg);
1520#endif
1521 if (rc)
1522 {
1523 rc = VERR_NO_MEMORY;
1524 break;
1525 }
1526 }
1527 }
1528 }
1529
1530 up_write(&pTask->mm->mmap_sem);
1531
1532 if (RT_SUCCESS(rc))
1533 {
1534#ifdef VBOX_USE_PAE_HACK
1535 __free_page(pDummyPage);
1536#endif
1537 pMemLnx->Core.pv = pv;
1538 pMemLnx->Core.u.Mapping.R0Process = R0Process;
1539 *ppMem = &pMemLnx->Core;
1540 return VINF_SUCCESS;
1541 }
1542
1543 /*
1544 * Bail out.
1545 */
1546 rtR0MemObjLinuxDoMunmap(pv, pMemLnxToMap->Core.cb, pTask);
1547 }
1548 rtR0MemObjDelete(&pMemLnx->Core);
1549 }
1550#ifdef VBOX_USE_PAE_HACK
1551 __free_page(pDummyPage);
1552#endif
1553
1554 return rc;
1555}
1556
1557
1558DECLHIDDEN(int) rtR0MemObjNativeProtect(PRTR0MEMOBJINTERNAL pMem, size_t offSub, size_t cbSub, uint32_t fProt)
1559{
1560 NOREF(pMem);
1561 NOREF(offSub);
1562 NOREF(cbSub);
1563 NOREF(fProt);
1564 return VERR_NOT_SUPPORTED;
1565}
1566
1567
1568DECLHIDDEN(RTHCPHYS) rtR0MemObjNativeGetPagePhysAddr(PRTR0MEMOBJINTERNAL pMem, size_t iPage)
1569{
1570 PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
1571
1572 if (pMemLnx->cPages)
1573 return page_to_phys(pMemLnx->apPages[iPage]);
1574
1575 switch (pMemLnx->Core.enmType)
1576 {
1577 case RTR0MEMOBJTYPE_CONT:
1578 return pMemLnx->Core.u.Cont.Phys + (iPage << PAGE_SHIFT);
1579
1580 case RTR0MEMOBJTYPE_PHYS:
1581 return pMemLnx->Core.u.Phys.PhysBase + (iPage << PAGE_SHIFT);
1582
1583 /* the parent knows */
1584 case RTR0MEMOBJTYPE_MAPPING:
1585 return rtR0MemObjNativeGetPagePhysAddr(pMemLnx->Core.uRel.Child.pParent, iPage);
1586
1587 /* cPages > 0 */
1588 case RTR0MEMOBJTYPE_LOW:
1589 case RTR0MEMOBJTYPE_LOCK:
1590 case RTR0MEMOBJTYPE_PHYS_NC:
1591 case RTR0MEMOBJTYPE_PAGE:
1592 default:
1593 AssertMsgFailed(("%d\n", pMemLnx->Core.enmType));
1594 /* fall thru */
1595
1596 case RTR0MEMOBJTYPE_RES_VIRT:
1597 return NIL_RTHCPHYS;
1598 }
1599}
1600
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette