VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAllN8veExecMem.cpp@ 106405

Last change on this file since 106405 was 106405, checked in by vboxsync, 6 weeks ago

VMM/IEM: Corrected the idxFreeHint setting at the end of iemExecMemAllocatorPrune, it was set to the end of the purned area rather than the start. bugref:10720

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 88.0 KB
Line 
1/* $Id: IEMAllN8veExecMem.cpp 106405 2024-10-16 21:24:15Z vboxsync $ */
2/** @file
3 * IEM - Native Recompiler, Executable Memory Allocator.
4 */
5
6/*
7 * Copyright (C) 2023-2024 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28
29/*********************************************************************************************************************************
30* Header Files *
31*********************************************************************************************************************************/
32#define LOG_GROUP LOG_GROUP_IEM_RE_NATIVE
33#define IEM_WITH_OPAQUE_DECODER_STATE
34#define VMM_INCLUDED_SRC_include_IEMMc_h /* block IEMMc.h inclusion. */
35#include <VBox/vmm/iem.h>
36#include <VBox/vmm/cpum.h>
37#include "IEMInternal.h"
38#include <VBox/vmm/vmcc.h>
39#include <VBox/log.h>
40#include <VBox/err.h>
41#include <VBox/param.h>
42#include <iprt/assert.h>
43#include <iprt/mem.h>
44#include <iprt/string.h>
45#if defined(RT_ARCH_AMD64)
46# include <iprt/x86.h>
47#elif defined(RT_ARCH_ARM64)
48# include <iprt/armv8.h>
49#endif
50
51#ifdef RT_OS_WINDOWS
52# include <iprt/formats/pecoff.h> /* this is incomaptible with windows.h, thus: */
53extern "C" DECLIMPORT(uint8_t) __cdecl RtlAddFunctionTable(void *pvFunctionTable, uint32_t cEntries, uintptr_t uBaseAddress);
54extern "C" DECLIMPORT(uint8_t) __cdecl RtlDelFunctionTable(void *pvFunctionTable);
55#else
56# include <iprt/formats/dwarf.h>
57# if defined(RT_OS_DARWIN)
58# include <libkern/OSCacheControl.h>
59# include <mach/mach.h>
60# include <mach/mach_vm.h>
61# define IEMNATIVE_USE_LIBUNWIND
62extern "C" void __register_frame(const void *pvFde);
63extern "C" void __deregister_frame(const void *pvFde);
64# else
65# ifdef DEBUG_bird /** @todo not thread safe yet */
66# define IEMNATIVE_USE_GDB_JIT
67# endif
68# ifdef IEMNATIVE_USE_GDB_JIT
69# include <iprt/critsect.h>
70# include <iprt/once.h>
71# include <iprt/formats/elf64.h>
72# endif
73extern "C" void __register_frame_info(void *pvBegin, void *pvObj); /* found no header for these two */
74extern "C" void *__deregister_frame_info(void *pvBegin); /* (returns pvObj from __register_frame_info call) */
75# endif
76#endif
77
78#include "IEMN8veRecompiler.h"
79
80
81/*********************************************************************************************************************************
82* Executable Memory Allocator *
83*********************************************************************************************************************************/
84/** The chunk sub-allocation unit size in bytes. */
85#define IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SIZE 256
86/** The chunk sub-allocation unit size as a shift factor. */
87#define IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SHIFT 8
88/** Enables adding a header to the sub-allocator allocations.
89 * This is useful for freeing up executable memory among other things. */
90#define IEMEXECMEM_ALT_SUB_WITH_ALLOC_HEADER
91/** Use alternative pruning. */
92#define IEMEXECMEM_ALT_SUB_WITH_ALT_PRUNING
93
94
95#if defined(IN_RING3) && !defined(RT_OS_WINDOWS)
96# ifdef IEMNATIVE_USE_GDB_JIT
97# define IEMNATIVE_USE_GDB_JIT_ET_DYN
98
99/** GDB JIT: Code entry. */
100typedef struct GDBJITCODEENTRY
101{
102 struct GDBJITCODEENTRY *pNext;
103 struct GDBJITCODEENTRY *pPrev;
104 uint8_t *pbSymFile;
105 uint64_t cbSymFile;
106} GDBJITCODEENTRY;
107
108/** GDB JIT: Actions. */
109typedef enum GDBJITACTIONS : uint32_t
110{
111 kGdbJitaction_NoAction = 0, kGdbJitaction_Register, kGdbJitaction_Unregister
112} GDBJITACTIONS;
113
114/** GDB JIT: Descriptor. */
115typedef struct GDBJITDESCRIPTOR
116{
117 uint32_t uVersion;
118 GDBJITACTIONS enmAction;
119 GDBJITCODEENTRY *pRelevant;
120 GDBJITCODEENTRY *pHead;
121 /** Our addition: */
122 GDBJITCODEENTRY *pTail;
123} GDBJITDESCRIPTOR;
124
125/** GDB JIT: Our simple symbol file data. */
126typedef struct GDBJITSYMFILE
127{
128 Elf64_Ehdr EHdr;
129# ifndef IEMNATIVE_USE_GDB_JIT_ET_DYN
130 Elf64_Shdr aShdrs[5];
131# else
132 Elf64_Shdr aShdrs[7];
133 Elf64_Phdr aPhdrs[2];
134# endif
135 /** The dwarf ehframe data for the chunk. */
136 uint8_t abEhFrame[512];
137 char szzStrTab[128];
138 Elf64_Sym aSymbols[3];
139# ifdef IEMNATIVE_USE_GDB_JIT_ET_DYN
140 Elf64_Sym aDynSyms[2];
141 Elf64_Dyn aDyn[6];
142# endif
143} GDBJITSYMFILE;
144
145extern "C" GDBJITDESCRIPTOR __jit_debug_descriptor;
146extern "C" DECLEXPORT(void) __jit_debug_register_code(void);
147
148/** Init once for g_IemNativeGdbJitLock. */
149static RTONCE g_IemNativeGdbJitOnce = RTONCE_INITIALIZER;
150/** Init once for the critical section. */
151static RTCRITSECT g_IemNativeGdbJitLock;
152
153/** GDB reads the info here. */
154GDBJITDESCRIPTOR __jit_debug_descriptor = { 1, kGdbJitaction_NoAction, NULL, NULL };
155
156/** GDB sets a breakpoint on this and checks __jit_debug_descriptor when hit. */
157DECL_NO_INLINE(RT_NOTHING, DECLEXPORT(void)) __jit_debug_register_code(void)
158{
159 ASMNopPause();
160}
161
162/** @callback_method_impl{FNRTONCE} */
163static DECLCALLBACK(int32_t) iemNativeGdbJitInitOnce(void *pvUser)
164{
165 RT_NOREF(pvUser);
166 return RTCritSectInit(&g_IemNativeGdbJitLock);
167}
168
169
170# endif /* IEMNATIVE_USE_GDB_JIT */
171
172/**
173 * Per-chunk unwind info for non-windows hosts.
174 */
175typedef struct IEMEXECMEMCHUNKEHFRAME
176{
177# ifdef IEMNATIVE_USE_LIBUNWIND
178 /** The offset of the FDA into abEhFrame. */
179 uintptr_t offFda;
180# else
181 /** 'struct object' storage area. */
182 uint8_t abObject[1024];
183# endif
184# ifdef IEMNATIVE_USE_GDB_JIT
185# if 0
186 /** The GDB JIT 'symbol file' data. */
187 GDBJITSYMFILE GdbJitSymFile;
188# endif
189 /** The GDB JIT list entry. */
190 GDBJITCODEENTRY GdbJitEntry;
191# endif
192 /** The dwarf ehframe data for the chunk. */
193 uint8_t abEhFrame[512];
194} IEMEXECMEMCHUNKEHFRAME;
195/** Pointer to per-chunk info info for non-windows hosts. */
196typedef IEMEXECMEMCHUNKEHFRAME *PIEMEXECMEMCHUNKEHFRAME;
197#endif
198
199
200/**
201 * An chunk of executable memory.
202 */
203typedef struct IEMEXECMEMCHUNK
204{
205 /** Number of free items in this chunk. */
206 uint32_t cFreeUnits;
207 /** Hint were to start searching for free space in the allocation bitmap. */
208 uint32_t idxFreeHint;
209 /** Pointer to the readable/writeable view of the memory chunk. */
210 void *pvChunkRw;
211 /** Pointer to the readable/executable view of the memory chunk. */
212 void *pvChunkRx;
213 /** Pointer to the context structure detailing the per chunk common code. */
214 PCIEMNATIVEPERCHUNKCTX pCtx;
215#ifdef IN_RING3
216 /**
217 * Pointer to the unwind information.
218 *
219 * This is used during C++ throw and longjmp (windows and probably most other
220 * platforms). Some debuggers (windbg) makes use of it as well.
221 *
222 * Windows: This is allocated from hHeap on windows because (at least for
223 * AMD64) the UNWIND_INFO structure address in the
224 * RUNTIME_FUNCTION entry is an RVA and the chunk is the "image".
225 *
226 * Others: Allocated from the regular heap to avoid unnecessary executable data
227 * structures. This points to an IEMEXECMEMCHUNKEHFRAME structure. */
228 void *pvUnwindInfo;
229#elif defined(IN_RING0)
230 /** Allocation handle. */
231 RTR0MEMOBJ hMemObj;
232#endif
233} IEMEXECMEMCHUNK;
234/** Pointer to a memory chunk. */
235typedef IEMEXECMEMCHUNK *PIEMEXECMEMCHUNK;
236
237
238/**
239 * Executable memory allocator for the native recompiler.
240 */
241typedef struct IEMEXECMEMALLOCATOR
242{
243 /** Magic value (IEMEXECMEMALLOCATOR_MAGIC). */
244 uint32_t uMagic;
245
246 /** The chunk size. */
247 uint32_t cbChunk;
248 /** The maximum number of chunks. */
249 uint32_t cMaxChunks;
250 /** The current number of chunks. */
251 uint32_t cChunks;
252 /** Hint where to start looking for available memory. */
253 uint32_t idxChunkHint;
254 /** Statistics: Current number of allocations. */
255 uint32_t cAllocations;
256
257 /** The total amount of memory available. */
258 uint64_t cbTotal;
259 /** Total amount of free memory. */
260 uint64_t cbFree;
261 /** Total amount of memory allocated. */
262 uint64_t cbAllocated;
263
264 /** Pointer to the allocation bitmaps for all the chunks (follows aChunks).
265 *
266 * Since the chunk size is a power of two and the minimum chunk size is a lot
267 * higher than the IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SIZE, each chunk will always
268 * require a whole number of uint64_t elements in the allocation bitmap. So,
269 * for sake of simplicity, they are allocated as one continous chunk for
270 * simplicity/laziness. */
271 uint64_t *pbmAlloc;
272 /** Number of units (IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SIZE) per chunk. */
273 uint32_t cUnitsPerChunk;
274 /** Number of bitmap elements per chunk (for quickly locating the bitmap
275 * portion corresponding to an chunk). */
276 uint32_t cBitmapElementsPerChunk;
277
278 /** Number of times we fruitlessly scanned a chunk for free space. */
279 uint64_t cFruitlessChunkScans;
280
281#ifdef IEMEXECMEM_ALT_SUB_WITH_ALT_PRUNING
282 /** The next chunk to prune in. */
283 uint32_t idxChunkPrune;
284 /** Where in chunk offset to start pruning at. */
285 uint32_t offChunkPrune;
286 /** Profiling the pruning code. */
287 STAMPROFILE StatPruneProf;
288 /** Number of bytes recovered by the pruning. */
289 STAMPROFILE StatPruneRecovered;
290#endif
291
292#ifdef VBOX_WITH_STATISTICS
293 STAMPROFILE StatAlloc;
294 /** Total amount of memory not being usable currently due to IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SIZE. */
295 uint64_t cbUnusable;
296 /** Allocation size distribution (in alloc units; 0 is the slop bucket). */
297 STAMCOUNTER aStatSizes[16];
298#endif
299
300#if defined(IN_RING3) && !defined(RT_OS_WINDOWS)
301 /** Pointer to the array of unwind info running parallel to aChunks (same
302 * allocation as this structure, located after the bitmaps).
303 * (For Windows, the structures must reside in 32-bit RVA distance to the
304 * actual chunk, so they are allocated off the chunk.) */
305 PIEMEXECMEMCHUNKEHFRAME paEhFrames;
306#endif
307
308 /** The allocation chunks. */
309 RT_FLEXIBLE_ARRAY_EXTENSION
310 IEMEXECMEMCHUNK aChunks[RT_FLEXIBLE_ARRAY];
311} IEMEXECMEMALLOCATOR;
312/** Pointer to an executable memory allocator. */
313typedef IEMEXECMEMALLOCATOR *PIEMEXECMEMALLOCATOR;
314
315/** Magic value for IEMEXECMEMALLOCATOR::uMagic (Scott Frederick Turow). */
316#define IEMEXECMEMALLOCATOR_MAGIC UINT32_C(0x19490412)
317
318
319#ifdef IEMEXECMEM_ALT_SUB_WITH_ALLOC_HEADER
320/**
321 * Allocation header.
322 */
323typedef struct IEMEXECMEMALLOCHDR
324{
325 union
326 {
327 struct
328 {
329 /** Magic value / eyecatcher (IEMEXECMEMALLOCHDR_MAGIC). */
330 uint32_t uMagic;
331 /** The allocation chunk (for speeding up freeing). */
332 uint32_t idxChunk;
333 };
334 /** Combined magic and chunk index, for the pruning scanner code. */
335 uint64_t u64MagicAndChunkIdx;
336 };
337 /** Pointer to the translation block the allocation belongs to.
338 * This is the whole point of the header. */
339 PIEMTB pTb;
340} IEMEXECMEMALLOCHDR;
341/** Pointer to an allocation header. */
342typedef IEMEXECMEMALLOCHDR *PIEMEXECMEMALLOCHDR;
343/** Magic value for IEMEXECMEMALLOCHDR ('ExeM'). */
344# define IEMEXECMEMALLOCHDR_MAGIC UINT32_C(0x4d657845)
345#endif
346
347
348static int iemExecMemAllocatorGrow(PVMCPUCC pVCpu, PIEMEXECMEMALLOCATOR pExecMemAllocator);
349
350
351#ifdef IEMEXECMEM_ALT_SUB_WITH_ALT_PRUNING
352/**
353 * Frees up executable memory when we're out space.
354 *
355 * This is an alternative to iemTbAllocatorFreeupNativeSpace() that frees up
356 * space in a more linear fashion from the allocator's point of view. It may
357 * also defragment if implemented & enabled
358 */
359static void iemExecMemAllocatorPrune(PVMCPU pVCpu, PIEMEXECMEMALLOCATOR pExecMemAllocator)
360{
361# ifndef IEMEXECMEM_ALT_SUB_WITH_ALLOC_HEADER
362# error "IEMEXECMEM_ALT_SUB_WITH_ALT_PRUNING requires IEMEXECMEM_ALT_SUB_WITH_ALLOC_HEADER"
363# endif
364 STAM_REL_PROFILE_START(&pExecMemAllocator->StatPruneProf, a);
365
366 /*
367 * Before we can start, we must process delayed frees.
368 */
369#if 1
370 PIEMTBALLOCATOR const pTbAllocator = iemTbAllocatorFreeBulkStart(pVCpu);
371#else
372 iemTbAllocatorProcessDelayedFrees(pVCpu, pVCpu->iem.s.pTbAllocatorR3);
373#endif
374
375 AssertCompile(RT_IS_POWER_OF_TWO(IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SIZE));
376
377 uint32_t const cbChunk = pExecMemAllocator->cbChunk;
378 AssertReturnVoid(RT_IS_POWER_OF_TWO(cbChunk));
379 AssertReturnVoid(cbChunk >= _1M && cbChunk <= _256M); /* see iemExecMemAllocatorInit */
380
381 uint32_t const cChunks = pExecMemAllocator->cChunks;
382 AssertReturnVoid(cChunks == pExecMemAllocator->cMaxChunks);
383 AssertReturnVoid(cChunks >= 1);
384
385 Assert(!pVCpu->iem.s.pCurTbR3);
386
387 /*
388 * Decide how much to prune. The chunk is is a multiple of two, so we'll be
389 * scanning a multiple of two here as well.
390 */
391 uint32_t cbToPrune = cbChunk;
392
393 /* Never more than 25%. */
394 if (cChunks < 4)
395 cbToPrune /= cChunks == 1 ? 4 : 2;
396
397 /* Upper limit. In a debug build a 4MB limit averages out at ~0.6ms per call. */
398 if (cbToPrune > _4M)
399 cbToPrune = _4M;
400
401 /*
402 * Adjust the pruning chunk and offset accordingly.
403 */
404 uint32_t idxChunk = pExecMemAllocator->idxChunkPrune;
405 uint32_t offChunk = pExecMemAllocator->offChunkPrune;
406 offChunk &= ~(uint32_t)(IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SIZE - 1U);
407 if (offChunk >= cbChunk)
408 {
409 offChunk = 0;
410 idxChunk += 1;
411 }
412 if (idxChunk >= cChunks)
413 {
414 offChunk = 0;
415 idxChunk = 0;
416 }
417
418 uint32_t const offPruneStart = offChunk;
419 uint32_t const offPruneEnd = RT_MIN(offChunk + cbToPrune, cbChunk);
420
421 /*
422 * Do the pruning. The current approach is the sever kind.
423 *
424 * This is memory bound, as we must load both the allocation header and the
425 * associated TB and then modify them. So, the CPU isn't all that unitilized
426 * here. Try apply some prefetching to speed it up a tiny bit.
427 */
428 uint64_t cbPruned = 0;
429 uint64_t const u64MagicAndChunkIdx = RT_MAKE_U64(IEMEXECMEMALLOCHDR_MAGIC, idxChunk);
430 uint8_t * const pbChunk = (uint8_t *)pExecMemAllocator->aChunks[idxChunk].pvChunkRx;
431 while (offChunk < offPruneEnd)
432 {
433 PIEMEXECMEMALLOCHDR pHdr = (PIEMEXECMEMALLOCHDR)&pbChunk[offChunk];
434
435 /* Is this the start of an allocation block for a TB? (We typically
436 have one allocation at the start of each chunk for the unwind info
437 where pTb is NULL.) */
438 PIEMTB pTb;
439 if ( pHdr->u64MagicAndChunkIdx == u64MagicAndChunkIdx
440 && RT_LIKELY((pTb = pHdr->pTb) != NULL))
441 {
442 AssertPtr(pTb);
443
444 uint32_t const cbBlock = RT_ALIGN_32(pTb->Native.cInstructions * sizeof(IEMNATIVEINSTR) + sizeof(*pHdr),
445 IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SIZE);
446
447 /* Prefetch the next header before freeing the current one and its TB. */
448 /** @todo Iff the block size was part of the header in some way, this could be
449 * a tiny bit faster. */
450 offChunk += cbBlock;
451#if defined(_MSC_VER) && defined(RT_ARCH_AMD64)
452 _mm_prefetch((char *)&pbChunk[offChunk], _MM_HINT_T0);
453#elif defined(_MSC_VER) && defined(RT_ARCH_ARM64)
454 __prefetch(&pbChunk[offChunk]);
455#else
456 __builtin_prefetch(&pbChunk[offChunk], 1 /*rw*/);
457#endif
458 /* Some paranoia first, though. */
459 AssertBreakStmt(offChunk <= cbChunk, offChunk -= cbBlock - IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SIZE);
460 cbPruned += cbBlock;
461
462#if 1
463 iemTbAllocatorFreeBulk(pVCpu, pTbAllocator, pTb);
464#else
465 iemTbAllocatorFree(pVCpu, pTb);
466#endif
467 }
468 else
469 offChunk += IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SIZE;
470 }
471 STAM_REL_PROFILE_ADD_PERIOD(&pExecMemAllocator->StatPruneRecovered, cbPruned);
472
473 pVCpu->iem.s.ppTbLookupEntryR3 = &pVCpu->iem.s.pTbLookupEntryDummyR3;
474
475 /*
476 * Save the current pruning point.
477 */
478 pExecMemAllocator->offChunkPrune = offChunk;
479 pExecMemAllocator->idxChunkPrune = idxChunk;
480
481 /* Set the hint to the start of the pruned region. */
482 pExecMemAllocator->idxChunkHint = idxChunk;
483 pExecMemAllocator->aChunks[idxChunk].idxFreeHint = offPruneStart / IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SIZE;
484
485 STAM_REL_PROFILE_STOP(&pExecMemAllocator->StatPruneProf, a);
486}
487#endif /* IEMEXECMEM_ALT_SUB_WITH_ALT_PRUNING */
488
489
490#if defined(VBOX_STRICT) || 0
491/**
492 * The old bitmap scanner code, for comparison and assertions.
493 */
494static uint32_t iemExecMemAllocatorFindReqFreeUnitsOld(uint64_t *pbmAlloc, uint32_t cToScan, uint32_t cReqUnits)
495{
496 /** @todo This can probably be done more efficiently for non-x86 systems. */
497 int iBit = ASMBitFirstClear(pbmAlloc, cToScan);
498 while (iBit >= 0 && (uint32_t)iBit <= cToScan - cReqUnits)
499 {
500 uint32_t idxAddBit = 1;
501 while (idxAddBit < cReqUnits && !ASMBitTest(pbmAlloc, (uint32_t)iBit + idxAddBit))
502 idxAddBit++;
503 if (idxAddBit >= cReqUnits)
504 return (uint32_t)iBit;
505 iBit = ASMBitNextClear(pbmAlloc, cToScan, iBit + idxAddBit - 1);
506 }
507 return UINT32_MAX;
508}
509#endif
510
511
512/**
513 * Bitmap scanner code that looks for a bunch of @a cReqUnits zero bits.
514 *
515 * Booting win11 with a r165098 release build the average native TB size is
516 * around 9 units (of 256 bytes). So, it is unlikely we need to scan any
517 * subsequent words once we hit a patch of zeros, thus @a a_fBig.
518 *
519 * @todo This needs more tweaking. While it *is* faster the the old code,
520 * it doens't seem like it's all that much. :/
521 */
522template<const bool a_fBig>
523static uint32_t iemExecMemAllocatorFindReqFreeUnits(uint64_t *pbmAlloc, uint32_t c64WordsToScan, uint32_t cReqUnits)
524{
525 /*
526 * Scan the (section of the) allocation bitmap in 64-bit words.
527 */
528 unsigned cPrevLeadingZeros = 0;
529 for (uint32_t off = 0; off < c64WordsToScan; off++)
530 {
531 uint64_t uWord = pbmAlloc[off];
532 if (uWord == UINT64_MAX)
533 {
534 /*
535 * Getting thru patches of UINT64_MAX is a frequent problem when the allocator
536 * fills up, so it's definitely worth optimizing.
537 *
538 * The complicated code below is a bit faster on arm. Reducing the per TB cost
539 * from 4255ns to 4106ns (best run out of 10). On win/amd64 there isn't an
540 * obvious gain here, at least not with the data currently being profiled.
541 */
542#if 1
543 off++;
544 uint32_t cQuads = (c64WordsToScan - off) / 4;
545
546 /* Align. */
547 if (cQuads > 1)
548 switch (((uintptr_t)&pbmAlloc[off] / sizeof(uint64_t)) & 3)
549 {
550 case 0:
551 break;
552 case 1:
553 {
554 uWord = pbmAlloc[off];
555 uint64_t uWord1 = pbmAlloc[off + 1];
556 uint64_t uWord2 = pbmAlloc[off + 2];
557 if ((uWord & uWord1 & uWord2) == UINT64_MAX)
558 {
559 off += 3;
560 cQuads = (c64WordsToScan - off) / 4;
561 }
562 else if (uWord == UINT64_MAX)
563 {
564 if (uWord1 != UINT64_MAX)
565 {
566 uWord = uWord1;
567 off += 1;
568 }
569 else
570 {
571 uWord = uWord2;
572 off += 2;
573 }
574 }
575 break;
576 }
577 case 2:
578 {
579 uWord = pbmAlloc[off];
580 uint64_t uWord1 = pbmAlloc[off + 1];
581 if ((uWord & uWord1) == UINT64_MAX)
582 {
583 off += 2;
584 cQuads = (c64WordsToScan - off) / 4;
585 }
586 else if (uWord == UINT64_MAX)
587 {
588 uWord = uWord1;
589 off += 1;
590 }
591 break;
592 }
593 case 3:
594 uWord = pbmAlloc[off];
595 if (uWord == UINT64_MAX)
596 {
597 off++;
598 cQuads = (c64WordsToScan - off) / 4;
599 }
600 break;
601 }
602 if (uWord == UINT64_MAX)
603 {
604 /* Looping over 32 bytes at a time. */
605 for (;;)
606 {
607 if (cQuads-- > 0)
608 {
609 uWord = pbmAlloc[off + 0];
610 uint64_t uWord1 = pbmAlloc[off + 1];
611 uint64_t uWord2 = pbmAlloc[off + 2];
612 uint64_t uWord3 = pbmAlloc[off + 3];
613 if ((uWord & uWord1 & uWord2 & uWord3) == UINT64_MAX)
614 off += 4;
615 else
616 {
617 if (uWord != UINT64_MAX)
618 { }
619 else if (uWord1 != UINT64_MAX)
620 {
621 uWord = uWord1;
622 off += 1;
623 }
624 else if (uWord2 != UINT64_MAX)
625 {
626 uWord = uWord2;
627 off += 2;
628 }
629 else
630 {
631 uWord = uWord3;
632 off += 3;
633 }
634 break;
635 }
636 }
637 else
638 {
639 if (off < c64WordsToScan)
640 {
641 uWord = pbmAlloc[off];
642 if (uWord != UINT64_MAX)
643 break;
644 off++;
645 if (off < c64WordsToScan)
646 {
647 uWord = pbmAlloc[off];
648 if (uWord != UINT64_MAX)
649 break;
650 off++;
651 if (off < c64WordsToScan)
652 {
653 uWord = pbmAlloc[off];
654 if (uWord != UINT64_MAX)
655 break;
656 Assert(off + 1 == c64WordsToScan);
657 }
658 }
659 }
660 return UINT32_MAX;
661 }
662 }
663 }
664#else
665 do
666 {
667 off++;
668 if (off < c64WordsToScan)
669 uWord = pbmAlloc[off];
670 else
671 return UINT32_MAX;
672 } while (uWord == UINT64_MAX);
673#endif
674 cPrevLeadingZeros = 0;
675 }
676
677 /*
678 * If we get down here, we have a word that isn't UINT64_MAX.
679 */
680 if (uWord != 0)
681 {
682 /*
683 * Fend of large request we cannot satisfy before the first set bit.
684 */
685 if (!a_fBig || cReqUnits < 64 + cPrevLeadingZeros)
686 {
687#ifdef __GNUC__
688 unsigned cZerosInWord = __builtin_popcountl(~uWord);
689#elif defined(_MSC_VER) && defined(RT_ARCH_AMD64)
690 unsigned cZerosInWord = __popcnt64(~uWord);
691#elif defined(_MSC_VER) && defined(RT_ARCH_ARM64)
692 unsigned cZerosInWord = _CountOneBits64(~uWord);
693#else
694# pragma message("need popcount intrinsic or something...")
695 unsigned cZerosInWord = 0;
696 for (uint64_t uTmp = ~uWords; uTmp; cZerosInWord++)
697 uTmp &= uTmp - 1; /* Clears the least significant bit set. */
698#endif
699 if (cZerosInWord + cPrevLeadingZeros >= cReqUnits)
700 {
701 /* Check if we've got a patch of zeros at the trailing end
702 when joined with the previous word: */
703#ifdef __GNUC__
704 unsigned cTrailingZeros = __builtin_ctzl(uWord);
705#else
706 unsigned cTrailingZeros = ASMBitFirstSetU64(uWord) - 1;
707#endif
708 if (cPrevLeadingZeros + cTrailingZeros >= cReqUnits)
709 return off * 64 - cPrevLeadingZeros;
710
711 /*
712 * Try leading zeros before we get on with the tedious stuff.
713 */
714#ifdef __GNUC__
715 cPrevLeadingZeros = __builtin_clzl(uWord);
716#else
717 cPrevLeadingZeros = 64 - ASMBitLastSetU64(uWord);
718#endif
719 if (cPrevLeadingZeros >= cReqUnits)
720 return (off + 1) * 64 - cPrevLeadingZeros;
721
722 /*
723 * Check the popcount again sans leading & trailing before looking
724 * inside the word.
725 */
726 cZerosInWord -= cPrevLeadingZeros + cTrailingZeros;
727 if (cZerosInWord >= cReqUnits)
728 {
729 /* 1; 64 - 0 - 1 = 63; */
730 unsigned const iBitLast = 64 - cPrevLeadingZeros - cReqUnits; /** @todo boundrary */
731 unsigned iBit = cTrailingZeros;
732 uWord >>= cTrailingZeros;
733 do
734 {
735 Assert(uWord & 1);
736#ifdef __GNUC__
737 unsigned iZeroBit = __builtin_ctzl(~uWord);
738#else
739 unsigned iZeroBit = ASMBitFirstSetU64(~uWord) - 1;
740#endif
741 iBit += iZeroBit;
742 uWord >>= iZeroBit;
743 Assert(iBit <= iBitLast);
744 Assert((uWord & 1) == 0);
745#ifdef __GNUC__
746 unsigned cZeros = __builtin_ctzl(uWord);
747#else
748 unsigned cZeros = ASMBitFirstSetU64(uWord) - 1;
749#endif
750 if (cZeros >= cReqUnits)
751 return off * 64 + iBit;
752
753 cZerosInWord -= cZeros; /* (may underflow as we will count shifted in zeros) */
754 iBit += cZeros;
755 uWord >>= cZeros;
756 } while ((int)cZerosInWord >= (int)cReqUnits && iBit < iBitLast);
757 }
758 continue; /* we've already calculated cPrevLeadingZeros */
759 }
760 }
761
762 /* Update the leading (MSB) zero count. */
763#ifdef __GNUC__
764 cPrevLeadingZeros = __builtin_clzl(uWord);
765#else
766 cPrevLeadingZeros = 64 - ASMBitLastSetU64(uWord);
767#endif
768 }
769 /*
770 * uWord == 0
771 */
772 else
773 {
774 if RT_CONSTEXPR_IF(!a_fBig)
775 return off * 64 - cPrevLeadingZeros;
776 else /* keep else */
777 {
778 if (cPrevLeadingZeros + 64 >= cReqUnits)
779 return off * 64 - cPrevLeadingZeros;
780 for (uint32_t off2 = off + 1;; off2++)
781 {
782 if (off2 < c64WordsToScan)
783 {
784 uWord = pbmAlloc[off2];
785 if (uWord == UINT64_MAX)
786 {
787 cPrevLeadingZeros = 0;
788 break;
789 }
790 if (uWord == 0)
791 {
792 if (cPrevLeadingZeros + (off2 - off + 1) * 64 >= cReqUnits)
793 return off * 64 - cPrevLeadingZeros;
794 }
795 else
796 {
797#ifdef __GNUC__
798 unsigned cTrailingZeros = __builtin_ctzl(uWord);
799#else
800 unsigned cTrailingZeros = ASMBitFirstSetU64(uWord) - 1;
801#endif
802 if (cPrevLeadingZeros + (off2 - off) * 64 + cTrailingZeros >= cReqUnits)
803 return off * 64 - cPrevLeadingZeros;
804#ifdef __GNUC__
805 cPrevLeadingZeros = __builtin_clzl(uWord);
806#else
807 cPrevLeadingZeros = 64 - ASMBitLastSetU64(uWord);
808#endif
809 break;
810 }
811 }
812 else
813 return UINT32_MAX;
814 }
815 }
816 }
817 }
818 return UINT32_MAX;
819}
820
821
822/**
823 * Try allocate a block of @a cReqUnits in the chunk @a idxChunk.
824 */
825static void *
826iemExecMemAllocatorAllocInChunkInt(PIEMEXECMEMALLOCATOR pExecMemAllocator, uint64_t *pbmAlloc, uint32_t idxFirst,
827 uint32_t cToScan, uint32_t cReqUnits, uint32_t idxChunk, PIEMTB pTb,
828 void **ppvExec, PCIEMNATIVEPERCHUNKCTX *ppChunkCtx)
829{
830 /*
831 * Shift the bitmap to the idxFirst bit so we can use ASMBitFirstClear.
832 */
833 Assert(!(cToScan & 63));
834 Assert(!(idxFirst & 63));
835 Assert(cToScan + idxFirst <= pExecMemAllocator->cUnitsPerChunk);
836 pbmAlloc += idxFirst / 64;
837 cToScan += idxFirst & 63;
838 Assert(!(cToScan & 63));
839
840#if 1
841 uint32_t const iBit = cReqUnits < 64
842 ? iemExecMemAllocatorFindReqFreeUnits<false>(pbmAlloc, cToScan / 64, cReqUnits)
843 : iemExecMemAllocatorFindReqFreeUnits<true>( pbmAlloc, cToScan / 64, cReqUnits);
844# ifdef VBOX_STRICT
845 uint32_t const iBitOld = iemExecMemAllocatorFindReqFreeUnitsOld(pbmAlloc, cToScan, cReqUnits);
846 AssertMsg( iBit == iBitOld
847 || (iBit / 64) == (iBitOld / 64), /* New algorithm will return trailing hit before middle. */
848 ("iBit=%#x (%#018RX64); iBitOld=%#x (%#018RX64); cReqUnits=%#x\n",
849 iBit, iBit != UINT32_MAX ? pbmAlloc[iBit / 64] : 0,
850 iBitOld, iBitOld != UINT32_MAX ? pbmAlloc[iBitOld / 64] : 0, cReqUnits));
851# endif
852#else
853 uint32_t const iBit = iemExecMemAllocatorFindReqFreeUnitsOld(pbmAlloc, cToScan, cReqUnits);
854#endif
855 if (iBit != UINT32_MAX)
856 {
857 ASMBitSetRange(pbmAlloc, (uint32_t)iBit, (uint32_t)iBit + cReqUnits);
858
859 PIEMEXECMEMCHUNK const pChunk = &pExecMemAllocator->aChunks[idxChunk];
860 pChunk->cFreeUnits -= cReqUnits;
861 pChunk->idxFreeHint = (uint32_t)iBit + cReqUnits;
862
863 pExecMemAllocator->cAllocations += 1;
864 uint32_t const cbReq = cReqUnits << IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SHIFT;
865 pExecMemAllocator->cbAllocated += cbReq;
866 pExecMemAllocator->cbFree -= cbReq;
867 pExecMemAllocator->idxChunkHint = idxChunk;
868
869 void * const pvMemRw = (uint8_t *)pChunk->pvChunkRw
870 + ((idxFirst + (uint32_t)iBit) << IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SHIFT);
871
872 if (ppChunkCtx)
873 *ppChunkCtx = pChunk->pCtx;
874
875 /*
876 * Initialize the header and return.
877 */
878# ifdef IEMEXECMEM_ALT_SUB_WITH_ALLOC_HEADER
879 PIEMEXECMEMALLOCHDR const pHdr = (PIEMEXECMEMALLOCHDR)pvMemRw;
880 pHdr->uMagic = IEMEXECMEMALLOCHDR_MAGIC;
881 pHdr->idxChunk = idxChunk;
882 pHdr->pTb = pTb;
883
884 if (ppvExec)
885 *ppvExec = (uint8_t *)pChunk->pvChunkRx
886 + ((idxFirst + (uint32_t)iBit) << IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SHIFT)
887 + sizeof(*pHdr);
888
889 return pHdr + 1;
890#else
891 if (ppvExec)
892 *ppvExec = (uint8_t *)pChunk->pvChunkRx
893 + ((idxFirst + (uint32_t)iBit) << IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SHIFT);
894
895 RT_NOREF(pTb);
896 return pvMem;
897#endif
898 }
899
900 return NULL;
901}
902
903
904/**
905 * Converts requested number of bytes into a unit count.
906 */
907DECL_FORCE_INLINE(uint32_t) iemExecMemAllocBytesToUnits(uint32_t cbReq)
908{
909#ifdef IEMEXECMEM_ALT_SUB_WITH_ALLOC_HEADER
910 return (cbReq + sizeof(IEMEXECMEMALLOCHDR) + IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SIZE - 1)
911#else
912 return (cbReq + IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SIZE - 1)
913#endif
914 >> IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SHIFT;
915}
916
917
918DECL_FORCE_INLINE(PIEMNATIVEINSTR)
919iemExecMemAllocatorAllocUnitsInChunkInner(PIEMEXECMEMALLOCATOR pExecMemAllocator, uint32_t idxChunk, uint32_t cReqUnits,
920 PIEMTB pTb, PIEMNATIVEINSTR *ppaExec, PCIEMNATIVEPERCHUNKCTX *ppChunkCtx)
921{
922 uint64_t * const pbmAlloc = &pExecMemAllocator->pbmAlloc[pExecMemAllocator->cBitmapElementsPerChunk * idxChunk];
923 uint32_t const idxHint = pExecMemAllocator->aChunks[idxChunk].idxFreeHint & ~(uint32_t)63;
924 if (idxHint + cReqUnits <= pExecMemAllocator->cUnitsPerChunk)
925 {
926 void *pvRet = iemExecMemAllocatorAllocInChunkInt(pExecMemAllocator, pbmAlloc, idxHint,
927 pExecMemAllocator->cUnitsPerChunk - idxHint,
928 cReqUnits, idxChunk, pTb, (void **)ppaExec, ppChunkCtx);
929 if (pvRet)
930 return (PIEMNATIVEINSTR)pvRet;
931 }
932 void *pvRet = iemExecMemAllocatorAllocInChunkInt(pExecMemAllocator, pbmAlloc, 0,
933 RT_MIN(pExecMemAllocator->cUnitsPerChunk,
934 RT_ALIGN_32(idxHint + cReqUnits, 64*4)),
935 cReqUnits, idxChunk, pTb, (void **)ppaExec, ppChunkCtx);
936 if (pvRet)
937 return (PIEMNATIVEINSTR)pvRet;
938
939 pExecMemAllocator->cFruitlessChunkScans += 1;
940 return NULL;
941}
942
943
944DECLINLINE(PIEMNATIVEINSTR)
945iemExecMemAllocatorAllocBytesInChunk(PIEMEXECMEMALLOCATOR pExecMemAllocator, uint32_t idxChunk, uint32_t cbReq,
946 PIEMNATIVEINSTR *ppaExec)
947{
948 uint32_t const cReqUnits = iemExecMemAllocBytesToUnits(cbReq);
949 if (cReqUnits <= pExecMemAllocator->aChunks[idxChunk].cFreeUnits)
950 return iemExecMemAllocatorAllocUnitsInChunkInner(pExecMemAllocator, idxChunk, cReqUnits, NULL /*pTb*/,
951 ppaExec, NULL /*ppChunkCtx*/);
952 return NULL;
953}
954
955
956/**
957 * Allocates @a cbReq bytes of executable memory.
958 *
959 * @returns Pointer to the readable/writeable memory, NULL if out of memory or other problem
960 * encountered.
961 * @param pVCpu The cross context virtual CPU structure of the
962 * calling thread.
963 * @param cbReq How many bytes are required.
964 * @param pTb The translation block that will be using the allocation.
965 * @param ppaExec Where to return the pointer to executable view of
966 * the allocated memory, optional.
967 * @param ppChunkCtx Where to return the per chunk attached context
968 * if available, optional.
969 */
970DECLHIDDEN(PIEMNATIVEINSTR) iemExecMemAllocatorAlloc(PVMCPU pVCpu, uint32_t cbReq, PIEMTB pTb,
971 PIEMNATIVEINSTR *ppaExec, PCIEMNATIVEPERCHUNKCTX *ppChunkCtx) RT_NOEXCEPT
972{
973 PIEMEXECMEMALLOCATOR pExecMemAllocator = pVCpu->iem.s.pExecMemAllocatorR3;
974 AssertReturn(pExecMemAllocator && pExecMemAllocator->uMagic == IEMEXECMEMALLOCATOR_MAGIC, NULL);
975 AssertMsgReturn(cbReq > 32 && cbReq < _512K, ("%#x\n", cbReq), NULL);
976 STAM_PROFILE_START(&pExecMemAllocator->StatAlloc, a);
977
978 uint32_t const cReqUnits = iemExecMemAllocBytesToUnits(cbReq);
979 STAM_COUNTER_INC(&pExecMemAllocator->aStatSizes[cReqUnits < RT_ELEMENTS(pExecMemAllocator->aStatSizes) ? cReqUnits : 0]);
980 for (unsigned iIteration = 0;; iIteration++)
981 {
982 if ( cbReq * 2 <= pExecMemAllocator->cbFree
983 || (cReqUnits == 1 || pExecMemAllocator->cbFree >= IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SIZE) )
984 {
985 uint32_t const cChunks = pExecMemAllocator->cChunks;
986 uint32_t const idxChunkHint = pExecMemAllocator->idxChunkHint < cChunks ? pExecMemAllocator->idxChunkHint : 0;
987
988 /*
989 * We do two passes here, the first pass we skip chunks with fewer than cReqUnits * 16,
990 * the 2nd pass we skip chunks. The second pass checks the one skipped in the first pass.
991 */
992 for (uint32_t cMinFreePass = cReqUnits == 1 ? cReqUnits : cReqUnits * 16, cMaxFreePass = UINT32_MAX;;)
993 {
994 for (uint32_t idxChunk = idxChunkHint; idxChunk < cChunks; idxChunk++)
995 if ( pExecMemAllocator->aChunks[idxChunk].cFreeUnits >= cMinFreePass
996 && pExecMemAllocator->aChunks[idxChunk].cFreeUnits <= cMaxFreePass)
997 {
998 PIEMNATIVEINSTR const pRet = iemExecMemAllocatorAllocUnitsInChunkInner(pExecMemAllocator, idxChunk,
999 cReqUnits, pTb, ppaExec, ppChunkCtx);
1000 if (pRet)
1001 {
1002 STAM_PROFILE_STOP(&pExecMemAllocator->StatAlloc, a);
1003#ifdef VBOX_WITH_STATISTICS
1004 pExecMemAllocator->cbUnusable += (cReqUnits << IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SHIFT) - cbReq;
1005#endif
1006 return pRet;
1007 }
1008 }
1009 for (uint32_t idxChunk = 0; idxChunk < idxChunkHint; idxChunk++)
1010 if ( pExecMemAllocator->aChunks[idxChunk].cFreeUnits >= cMinFreePass
1011 && pExecMemAllocator->aChunks[idxChunk].cFreeUnits <= cMaxFreePass)
1012 {
1013 PIEMNATIVEINSTR const pRet = iemExecMemAllocatorAllocUnitsInChunkInner(pExecMemAllocator, idxChunk,
1014 cReqUnits, pTb, ppaExec, ppChunkCtx);
1015 if (pRet)
1016 {
1017 STAM_PROFILE_STOP(&pExecMemAllocator->StatAlloc, a);
1018#ifdef VBOX_WITH_STATISTICS
1019 pExecMemAllocator->cbUnusable += (cReqUnits << IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SHIFT) - cbReq;
1020#endif
1021 return pRet;
1022 }
1023 }
1024 if (cMinFreePass <= cReqUnits * 2)
1025 break;
1026 cMaxFreePass = cMinFreePass - 1;
1027 cMinFreePass = cReqUnits * 2;
1028 }
1029 }
1030
1031 /*
1032 * Can we grow it with another chunk?
1033 */
1034 if (pExecMemAllocator->cChunks < pExecMemAllocator->cMaxChunks)
1035 {
1036 int rc = iemExecMemAllocatorGrow(pVCpu, pExecMemAllocator);
1037 AssertLogRelRCReturn(rc, NULL);
1038
1039 uint32_t const idxChunk = pExecMemAllocator->cChunks - 1;
1040 PIEMNATIVEINSTR const pRet = iemExecMemAllocatorAllocUnitsInChunkInner(pExecMemAllocator, idxChunk, cReqUnits, pTb,
1041 ppaExec, ppChunkCtx);
1042 if (pRet)
1043 {
1044 STAM_PROFILE_STOP(&pExecMemAllocator->StatAlloc, a);
1045#ifdef VBOX_WITH_STATISTICS
1046 pExecMemAllocator->cbUnusable += (cReqUnits << IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SHIFT) - cbReq;
1047#endif
1048 return pRet;
1049 }
1050 AssertFailed();
1051 }
1052
1053 /*
1054 * Try prune native TBs once.
1055 */
1056 if (iIteration == 0)
1057 {
1058#ifdef IEMEXECMEM_ALT_SUB_WITH_ALT_PRUNING
1059 iemExecMemAllocatorPrune(pVCpu, pExecMemAllocator);
1060#else
1061 /* No header included in the instruction count here. */
1062 uint32_t const cNeededInstrs = RT_ALIGN_32(cbReq, IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SIZE) / sizeof(IEMNATIVEINSTR);
1063 iemTbAllocatorFreeupNativeSpace(pVCpu, cNeededInstrs);
1064#endif
1065 }
1066 else
1067 {
1068 STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatNativeExecMemInstrBufAllocFailed);
1069 STAM_PROFILE_STOP(&pExecMemAllocator->StatAlloc, a);
1070 return NULL;
1071 }
1072 }
1073}
1074
1075
1076/** This is a hook to ensure the instruction cache is properly flushed before the code in the memory
1077 * given by @a pv and @a cb is executed */
1078DECLHIDDEN(void) iemExecMemAllocatorReadyForUse(PVMCPUCC pVCpu, void *pv, size_t cb) RT_NOEXCEPT
1079{
1080#ifdef RT_OS_DARWIN
1081 /*
1082 * We need to synchronize the stuff we wrote to the data cache with the
1083 * instruction cache, since these aren't coherent on arm (or at least not
1084 * on Apple Mn CPUs).
1085 *
1086 * Note! Since we don't any share JIT'ed code with the other CPUs, we don't
1087 * really care whether the dcache is fully flushed back to memory. It
1088 * only needs to hit the level 2 cache, which the level 1 instruction
1089 * and data caches seems to be sharing. In ARM terms, we need to reach
1090 * a point of unification (PoU), rather than a point of coherhency (PoC).
1091 *
1092 * https://developer.apple.com/documentation/apple-silicon/porting-just-in-time-compilers-to-apple-silicon
1093 *
1094 * https://developer.arm.com/documentation/den0013/d/Caches/Point-of-coherency-and-unification
1095 *
1096 * Experimenting with the approach used by sys_icache_invalidate() and
1097 * tweaking it a little, could let us shave off a bit of effort. The thing
1098 * that slows the apple code down on an M2 (runing Sonoma 13.4), seems to
1099 * the 'DSB ISH' instructions performed every 20 icache line flushes.
1100 * Skipping these saves ~100ns or more per TB when profiling the native
1101 * recompiler on the TBs from a win11 full boot-desktop-shutdow sequence.
1102 * Thus we will leave DCACHE_ICACHE_SYNC_WITH_WITH_IVAU_DSB undefined if we
1103 * can.
1104 *
1105 * There appears not to be much difference between DSB options 'ISH',
1106 * 'ISHST', 'NSH' and 'NSHST'. The latter is theoretically all we need, so
1107 * we'll use that one.
1108 *
1109 * See https://developer.arm.com/documentation/100941/0101/Barriers for
1110 * details on the barrier options.
1111 *
1112 * Note! The CFG value "/IEM/HostICacheInvalidationViaHostAPI" can be used
1113 * to disabling the experimental code should it misbehave.
1114 */
1115 uint8_t const fHostICacheInvalidation = pVCpu->iem.s.fHostICacheInvalidation;
1116 if (!(fHostICacheInvalidation & IEMNATIVE_ICACHE_F_USE_HOST_API))
1117 {
1118# define DCACHE_ICACHE_SYNC_DSB_OPTION "nshst"
1119/*# define DCACHE_ICACHE_SYNC_WITH_WITH_IVAU_DSB*/
1120
1121 /* Skipping this is fine, but doesn't impact perf much. */
1122 __asm__ __volatile__("dsb " DCACHE_ICACHE_SYNC_DSB_OPTION);
1123
1124 /* Invalidate the icache for the range [pv,pv+cb). */
1125# ifdef DCACHE_ICACHE_SYNC_WITH_WITH_IVAU_DSB
1126 size_t const cIvauDsbEvery= 20;
1127 unsigned cDsb = cIvauDsbEvery;
1128# endif
1129 size_t const cbCacheLine = 64;
1130 size_t cbInvalidate = cb + ((uintptr_t)pv & (cbCacheLine - 1)) ;
1131 size_t cCacheLines = RT_ALIGN_Z(cbInvalidate, cbCacheLine) / cbCacheLine;
1132 uintptr_t uPtr = (uintptr_t)pv & ~(uintptr_t)(cbCacheLine - 1);
1133 for (;; uPtr += cbCacheLine)
1134 {
1135 __asm__ /*__volatile__*/("ic ivau, %0" : : "r" (uPtr));
1136 cCacheLines -= 1;
1137 if (!cCacheLines)
1138 break;
1139# ifdef DCACHE_ICACHE_SYNC_WITH_WITH_IVAU_DSB
1140 cDsb -= 1;
1141 if (cDsb != 0)
1142 { /* likely */ }
1143 else
1144 {
1145 __asm__ __volatile__("dsb " DCACHE_ICACHE_SYNC_DSB_OPTION);
1146 cDsb = cIvauDsbEvery;
1147 }
1148# endif
1149 }
1150
1151 /*
1152 * The DSB here is non-optional it seems.
1153 *
1154 * The following ISB can be omitted on M2 without any obvious sideeffects,
1155 * it produces better number in the above mention profiling scenario.
1156 * This could be related to the kHasICDSB flag in cpu_capabilities.h,
1157 * but it doesn't look like that flag is set here (M2, Sonoma 13.4).
1158 *
1159 * I've made the inclusion of the ISH barrier as configurable and with
1160 * a default of skipping it.
1161 */
1162 if (!(fHostICacheInvalidation & IEMNATIVE_ICACHE_F_END_WITH_ISH))
1163 __asm__ __volatile__("dsb " DCACHE_ICACHE_SYNC_DSB_OPTION
1164 ::: "memory");
1165 else
1166 __asm__ __volatile__("dsb " DCACHE_ICACHE_SYNC_DSB_OPTION "\n\t"
1167 "isb"
1168 ::: "memory");
1169 }
1170 else
1171 sys_icache_invalidate(pv, cb);
1172
1173#elif defined(RT_OS_LINUX) && defined(RT_ARCH_ARM64)
1174 RT_NOREF(pVCpu);
1175
1176 /* There is __builtin___clear_cache() but it flushes both the instruction and data cache, so do it manually. */
1177 static uint32_t s_u32CtrEl0 = 0;
1178 if (!s_u32CtrEl0)
1179 asm volatile ("mrs %0, ctr_el0":"=r" (s_u32CtrEl0));
1180 uintptr_t cbICacheLine = (uintptr_t)4 << (s_u32CtrEl0 & 0xf);
1181
1182 uintptr_t pb = (uintptr_t)pv & ~(cbICacheLine - 1);
1183 for (; pb < (uintptr_t)pv + cb; pb += cbICacheLine)
1184 asm volatile ("ic ivau, %0" : : "r" (pb) : "memory");
1185
1186 asm volatile ("dsb ish\n\t isb\n\t" : : : "memory");
1187
1188#else
1189 RT_NOREF(pVCpu, pv, cb);
1190#endif
1191}
1192
1193
1194/**
1195 * Frees executable memory.
1196 */
1197DECLHIDDEN(void) iemExecMemAllocatorFree(PVMCPU pVCpu, void *pv, size_t cb) RT_NOEXCEPT
1198{
1199 PIEMEXECMEMALLOCATOR pExecMemAllocator = pVCpu->iem.s.pExecMemAllocatorR3;
1200 Assert(pExecMemAllocator && pExecMemAllocator->uMagic == IEMEXECMEMALLOCATOR_MAGIC);
1201 AssertPtr(pv);
1202#ifdef VBOX_WITH_STATISTICS
1203 size_t const cbOrig = cb;
1204#endif
1205#ifndef IEMEXECMEM_ALT_SUB_WITH_ALLOC_HEADER
1206 Assert(!((uintptr_t)pv & (IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SIZE - 1)));
1207
1208 /* Align the size as we did when allocating the block. */
1209 cb = RT_ALIGN_Z(cb, IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SIZE);
1210
1211#else
1212 PIEMEXECMEMALLOCHDR pHdr = (PIEMEXECMEMALLOCHDR)pv - 1;
1213 Assert(!((uintptr_t)pHdr & (IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SIZE - 1)));
1214 AssertReturnVoid(pHdr->uMagic == IEMEXECMEMALLOCHDR_MAGIC);
1215 uint32_t const idxChunk = pHdr->idxChunk;
1216 AssertReturnVoid(idxChunk < pExecMemAllocator->cChunks);
1217 pv = pHdr;
1218
1219 /* Adjust and align the size to cover the whole allocation area. */
1220 cb = RT_ALIGN_Z(cb + sizeof(*pHdr), IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SIZE);
1221#endif
1222
1223 /* Free it / assert sanity. */
1224 bool fFound = false;
1225 uint32_t const cbChunk = pExecMemAllocator->cbChunk;
1226#ifndef IEMEXECMEM_ALT_SUB_WITH_ALLOC_HEADER
1227 uint32_t const cChunks = pExecMemAllocator->cChunks;
1228 for (uint32_t idxChunk = 0; idxChunk < cChunks; idxChunk++)
1229#endif
1230 {
1231 uintptr_t const offChunk = (uintptr_t)pv - (uintptr_t)pExecMemAllocator->aChunks[idxChunk].pvChunkRx;
1232 fFound = offChunk < cbChunk;
1233 if (fFound)
1234 {
1235 uint32_t const idxFirst = (uint32_t)offChunk >> IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SHIFT;
1236 uint32_t const cReqUnits = (uint32_t)cb >> IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SHIFT;
1237
1238 /* Check that it's valid and free it. */
1239 uint64_t * const pbmAlloc = &pExecMemAllocator->pbmAlloc[pExecMemAllocator->cBitmapElementsPerChunk * idxChunk];
1240 AssertReturnVoid(ASMBitTest(pbmAlloc, idxFirst));
1241 for (uint32_t i = 1; i < cReqUnits; i++)
1242 AssertReturnVoid(ASMBitTest(pbmAlloc, idxFirst + i));
1243 ASMBitClearRange(pbmAlloc, idxFirst, idxFirst + cReqUnits);
1244
1245 /* Invalidate the header using the writeable memory view. */
1246 pHdr = (PIEMEXECMEMALLOCHDR)((uintptr_t)pExecMemAllocator->aChunks[idxChunk].pvChunkRw + offChunk);
1247#ifdef IEMEXECMEM_ALT_SUB_WITH_ALLOC_HEADER
1248 pHdr->uMagic = 0;
1249 pHdr->idxChunk = 0;
1250 pHdr->pTb = NULL;
1251#endif
1252 pExecMemAllocator->aChunks[idxChunk].cFreeUnits += cReqUnits;
1253 pExecMemAllocator->aChunks[idxChunk].idxFreeHint = idxFirst;
1254
1255 /* Update the stats. */
1256 pExecMemAllocator->cbAllocated -= cb;
1257 pExecMemAllocator->cbFree += cb;
1258 pExecMemAllocator->cAllocations -= 1;
1259#ifdef VBOX_WITH_STATISTICS
1260 pExecMemAllocator->cbUnusable -= (cReqUnits << IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SHIFT) - cbOrig;
1261#endif
1262 return;
1263 }
1264 }
1265 AssertFailed();
1266}
1267
1268
1269/**
1270 * Interface used by iemNativeRecompileAttachExecMemChunkCtx and unwind info
1271 * generators.
1272 */
1273DECLHIDDEN(PIEMNATIVEINSTR)
1274iemExecMemAllocatorAllocFromChunk(PVMCPU pVCpu, uint32_t idxChunk, uint32_t cbReq, PIEMNATIVEINSTR *ppaExec)
1275{
1276 PIEMEXECMEMALLOCATOR pExecMemAllocator = pVCpu->iem.s.pExecMemAllocatorR3;
1277 AssertReturn(idxChunk < pExecMemAllocator->cChunks, NULL);
1278 Assert(cbReq < _1M);
1279 return iemExecMemAllocatorAllocBytesInChunk(pExecMemAllocator, idxChunk, cbReq, ppaExec);
1280}
1281
1282
1283/**
1284 * For getting the per-chunk context detailing common code for a TB.
1285 *
1286 * This is for use by the disassembler.
1287 */
1288DECLHIDDEN(PCIEMNATIVEPERCHUNKCTX) iemExecMemGetTbChunkCtx(PVMCPU pVCpu, PCIEMTB pTb)
1289{
1290 PIEMEXECMEMALLOCATOR pExecMemAllocator = pVCpu->iem.s.pExecMemAllocatorR3;
1291 if ((pTb->fFlags & IEMTB_F_TYPE_MASK) == IEMTB_F_TYPE_NATIVE)
1292 {
1293 uintptr_t const uAddress = (uintptr_t)pTb->Native.paInstructions;
1294 uint32_t const cbChunk = pExecMemAllocator->cbChunk;
1295 uint32_t idxChunk = pExecMemAllocator->cChunks;
1296 while (idxChunk-- > 0)
1297 if (uAddress - (uintptr_t)pExecMemAllocator->aChunks[idxChunk].pvChunkRx < cbChunk)
1298 return pExecMemAllocator->aChunks[idxChunk].pCtx;
1299 }
1300 return NULL;
1301}
1302
1303
1304#ifdef IN_RING3
1305# ifdef RT_OS_WINDOWS
1306
1307/**
1308 * Initializes the unwind info structures for windows hosts.
1309 */
1310static int
1311iemExecMemAllocatorInitAndRegisterUnwindInfoForChunk(PVMCPUCC pVCpu, PIEMEXECMEMALLOCATOR pExecMemAllocator,
1312 void *pvChunk, uint32_t idxChunk)
1313{
1314 RT_NOREF(pVCpu);
1315
1316 /*
1317 * The AMD64 unwind opcodes.
1318 *
1319 * This is a program that starts with RSP after a RET instruction that
1320 * ends up in recompiled code, and the operations we describe here will
1321 * restore all non-volatile registers and bring RSP back to where our
1322 * RET address is. This means it's reverse order from what happens in
1323 * the prologue.
1324 *
1325 * Note! Using a frame register approach here both because we have one
1326 * and but mainly because the UWOP_ALLOC_LARGE argument values
1327 * would be a pain to write initializers for. On the positive
1328 * side, we're impervious to changes in the the stack variable
1329 * area can can deal with dynamic stack allocations if necessary.
1330 */
1331 static const IMAGE_UNWIND_CODE s_aOpcodes[] =
1332 {
1333 { { 16, IMAGE_AMD64_UWOP_SET_FPREG, 0 } }, /* RSP = RBP - FrameOffset * 10 (0x60) */
1334 { { 16, IMAGE_AMD64_UWOP_ALLOC_SMALL, 0 } }, /* RSP += 8; */
1335 { { 14, IMAGE_AMD64_UWOP_PUSH_NONVOL, X86_GREG_x15 } }, /* R15 = [RSP]; RSP += 8; */
1336 { { 12, IMAGE_AMD64_UWOP_PUSH_NONVOL, X86_GREG_x14 } }, /* R14 = [RSP]; RSP += 8; */
1337 { { 10, IMAGE_AMD64_UWOP_PUSH_NONVOL, X86_GREG_x13 } }, /* R13 = [RSP]; RSP += 8; */
1338 { { 8, IMAGE_AMD64_UWOP_PUSH_NONVOL, X86_GREG_x12 } }, /* R12 = [RSP]; RSP += 8; */
1339 { { 7, IMAGE_AMD64_UWOP_PUSH_NONVOL, X86_GREG_xDI } }, /* RDI = [RSP]; RSP += 8; */
1340 { { 6, IMAGE_AMD64_UWOP_PUSH_NONVOL, X86_GREG_xSI } }, /* RSI = [RSP]; RSP += 8; */
1341 { { 5, IMAGE_AMD64_UWOP_PUSH_NONVOL, X86_GREG_xBX } }, /* RBX = [RSP]; RSP += 8; */
1342 { { 4, IMAGE_AMD64_UWOP_PUSH_NONVOL, X86_GREG_xBP } }, /* RBP = [RSP]; RSP += 8; */
1343 };
1344 union
1345 {
1346 IMAGE_UNWIND_INFO Info;
1347 uint8_t abPadding[RT_UOFFSETOF(IMAGE_UNWIND_INFO, aOpcodes) + 16];
1348 } s_UnwindInfo =
1349 {
1350 {
1351 /* .Version = */ 1,
1352 /* .Flags = */ 0,
1353 /* .SizeOfProlog = */ 16, /* whatever */
1354 /* .CountOfCodes = */ RT_ELEMENTS(s_aOpcodes),
1355 /* .FrameRegister = */ X86_GREG_xBP,
1356 /* .FrameOffset = */ (-IEMNATIVE_FP_OFF_LAST_PUSH + 8) / 16 /* we're off by one slot. sigh. */,
1357 }
1358 };
1359 AssertCompile(-IEMNATIVE_FP_OFF_LAST_PUSH < 240 && -IEMNATIVE_FP_OFF_LAST_PUSH > 0);
1360 AssertCompile((-IEMNATIVE_FP_OFF_LAST_PUSH & 0xf) == 8);
1361
1362 /*
1363 * Calc how much space we need and allocate it off the exec heap.
1364 */
1365 unsigned const cFunctionEntries = 1;
1366 unsigned const cbUnwindInfo = sizeof(s_aOpcodes) + RT_UOFFSETOF(IMAGE_UNWIND_INFO, aOpcodes);
1367 unsigned const cbNeeded = sizeof(IMAGE_RUNTIME_FUNCTION_ENTRY) * cFunctionEntries + cbUnwindInfo;
1368 PIMAGE_RUNTIME_FUNCTION_ENTRY const paFunctions
1369 = (PIMAGE_RUNTIME_FUNCTION_ENTRY)iemExecMemAllocatorAllocBytesInChunk(pExecMemAllocator, idxChunk, cbNeeded, NULL);
1370 AssertReturn(paFunctions, VERR_INTERNAL_ERROR_5);
1371 pExecMemAllocator->aChunks[idxChunk].pvUnwindInfo = paFunctions;
1372
1373 /*
1374 * Initialize the structures.
1375 */
1376 PIMAGE_UNWIND_INFO const pInfo = (PIMAGE_UNWIND_INFO)&paFunctions[cFunctionEntries];
1377
1378 paFunctions[0].BeginAddress = 0;
1379 paFunctions[0].EndAddress = pExecMemAllocator->cbChunk;
1380 paFunctions[0].UnwindInfoAddress = (uint32_t)((uintptr_t)pInfo - (uintptr_t)pvChunk);
1381
1382 memcpy(pInfo, &s_UnwindInfo, RT_UOFFSETOF(IMAGE_UNWIND_INFO, aOpcodes));
1383 memcpy(&pInfo->aOpcodes[0], s_aOpcodes, sizeof(s_aOpcodes));
1384
1385 /*
1386 * Register it.
1387 */
1388 uint8_t fRet = RtlAddFunctionTable(paFunctions, cFunctionEntries, (uintptr_t)pvChunk);
1389 AssertReturn(fRet, VERR_INTERNAL_ERROR_3); /* Nothing to clean up on failure, since its within the chunk itself. */
1390
1391 return VINF_SUCCESS;
1392}
1393
1394
1395# else /* !RT_OS_WINDOWS */
1396
1397/**
1398 * Emits a LEB128 encoded value between -0x2000 and 0x2000 (both exclusive).
1399 */
1400DECLINLINE(RTPTRUNION) iemDwarfPutLeb128(RTPTRUNION Ptr, int32_t iValue)
1401{
1402 if (iValue >= 64)
1403 {
1404 Assert(iValue < 0x2000);
1405 *Ptr.pb++ = ((uint8_t)iValue & 0x7f) | 0x80;
1406 *Ptr.pb++ = (uint8_t)(iValue >> 7) & 0x3f;
1407 }
1408 else if (iValue >= 0)
1409 *Ptr.pb++ = (uint8_t)iValue;
1410 else if (iValue > -64)
1411 *Ptr.pb++ = ((uint8_t)iValue & 0x3f) | 0x40;
1412 else
1413 {
1414 Assert(iValue > -0x2000);
1415 *Ptr.pb++ = ((uint8_t)iValue & 0x7f) | 0x80;
1416 *Ptr.pb++ = ((uint8_t)(iValue >> 7) & 0x3f) | 0x40;
1417 }
1418 return Ptr;
1419}
1420
1421
1422/**
1423 * Emits an ULEB128 encoded value (up to 64-bit wide).
1424 */
1425DECLINLINE(RTPTRUNION) iemDwarfPutUleb128(RTPTRUNION Ptr, uint64_t uValue)
1426{
1427 while (uValue >= 0x80)
1428 {
1429 *Ptr.pb++ = ((uint8_t)uValue & 0x7f) | 0x80;
1430 uValue >>= 7;
1431 }
1432 *Ptr.pb++ = (uint8_t)uValue;
1433 return Ptr;
1434}
1435
1436
1437/**
1438 * Emits a CFA rule as register @a uReg + offset @a off.
1439 */
1440DECLINLINE(RTPTRUNION) iemDwarfPutCfaDefCfa(RTPTRUNION Ptr, uint32_t uReg, uint32_t off)
1441{
1442 *Ptr.pb++ = DW_CFA_def_cfa;
1443 Ptr = iemDwarfPutUleb128(Ptr, uReg);
1444 Ptr = iemDwarfPutUleb128(Ptr, off);
1445 return Ptr;
1446}
1447
1448
1449/**
1450 * Emits a register (@a uReg) save location:
1451 * CFA + @a off * data_alignment_factor
1452 */
1453DECLINLINE(RTPTRUNION) iemDwarfPutCfaOffset(RTPTRUNION Ptr, uint32_t uReg, uint32_t off)
1454{
1455 if (uReg < 0x40)
1456 *Ptr.pb++ = DW_CFA_offset | uReg;
1457 else
1458 {
1459 *Ptr.pb++ = DW_CFA_offset_extended;
1460 Ptr = iemDwarfPutUleb128(Ptr, uReg);
1461 }
1462 Ptr = iemDwarfPutUleb128(Ptr, off);
1463 return Ptr;
1464}
1465
1466
1467# if 0 /* unused */
1468/**
1469 * Emits a register (@a uReg) save location, using signed offset:
1470 * CFA + @a offSigned * data_alignment_factor
1471 */
1472DECLINLINE(RTPTRUNION) iemDwarfPutCfaSignedOffset(RTPTRUNION Ptr, uint32_t uReg, int32_t offSigned)
1473{
1474 *Ptr.pb++ = DW_CFA_offset_extended_sf;
1475 Ptr = iemDwarfPutUleb128(Ptr, uReg);
1476 Ptr = iemDwarfPutLeb128(Ptr, offSigned);
1477 return Ptr;
1478}
1479# endif
1480
1481
1482/**
1483 * Initializes the unwind info section for non-windows hosts.
1484 */
1485static int
1486iemExecMemAllocatorInitAndRegisterUnwindInfoForChunk(PVMCPUCC pVCpu, PIEMEXECMEMALLOCATOR pExecMemAllocator,
1487 void *pvChunk, uint32_t idxChunk)
1488{
1489 PIEMEXECMEMCHUNKEHFRAME const pEhFrame = &pExecMemAllocator->paEhFrames[idxChunk];
1490 pExecMemAllocator->aChunks[idxChunk].pvUnwindInfo = pEhFrame; /* not necessary, but whatever */
1491
1492 RTPTRUNION Ptr = { pEhFrame->abEhFrame };
1493
1494 /*
1495 * Generate the CIE first.
1496 */
1497# ifdef IEMNATIVE_USE_LIBUNWIND /* libunwind (llvm, darwin) only supports v1 and v3. */
1498 uint8_t const iDwarfVer = 3;
1499# else
1500 uint8_t const iDwarfVer = 4;
1501# endif
1502 RTPTRUNION const PtrCie = Ptr;
1503 *Ptr.pu32++ = 123; /* The CIE length will be determined later. */
1504 *Ptr.pu32++ = 0 /*UINT32_MAX*/; /* I'm a CIE in .eh_frame speak. */
1505 *Ptr.pb++ = iDwarfVer; /* DwARF version */
1506 *Ptr.pb++ = 0; /* Augmentation. */
1507 if (iDwarfVer >= 4)
1508 {
1509 *Ptr.pb++ = sizeof(uintptr_t); /* Address size. */
1510 *Ptr.pb++ = 0; /* Segment selector size. */
1511 }
1512# ifdef RT_ARCH_AMD64
1513 Ptr = iemDwarfPutLeb128(Ptr, 1); /* Code alignment factor (LEB128 = 1). */
1514# else
1515 Ptr = iemDwarfPutLeb128(Ptr, 4); /* Code alignment factor (LEB128 = 4). */
1516# endif
1517 Ptr = iemDwarfPutLeb128(Ptr, -8); /* Data alignment factor (LEB128 = -8). */
1518# ifdef RT_ARCH_AMD64
1519 Ptr = iemDwarfPutUleb128(Ptr, DWREG_AMD64_RA); /* Return address column (ULEB128) */
1520# elif defined(RT_ARCH_ARM64)
1521 Ptr = iemDwarfPutUleb128(Ptr, DWREG_ARM64_LR); /* Return address column (ULEB128) */
1522# else
1523# error "port me"
1524# endif
1525 /* Initial instructions: */
1526# ifdef RT_ARCH_AMD64
1527 Ptr = iemDwarfPutCfaDefCfa(Ptr, DWREG_AMD64_RBP, 16); /* CFA = RBP + 0x10 - first stack parameter */
1528 Ptr = iemDwarfPutCfaOffset(Ptr, DWREG_AMD64_RA, 1); /* Ret RIP = [CFA + 1*-8] */
1529 Ptr = iemDwarfPutCfaOffset(Ptr, DWREG_AMD64_RBP, 2); /* RBP = [CFA + 2*-8] */
1530 Ptr = iemDwarfPutCfaOffset(Ptr, DWREG_AMD64_RBX, 3); /* RBX = [CFA + 3*-8] */
1531 Ptr = iemDwarfPutCfaOffset(Ptr, DWREG_AMD64_R12, 4); /* R12 = [CFA + 4*-8] */
1532 Ptr = iemDwarfPutCfaOffset(Ptr, DWREG_AMD64_R13, 5); /* R13 = [CFA + 5*-8] */
1533 Ptr = iemDwarfPutCfaOffset(Ptr, DWREG_AMD64_R14, 6); /* R14 = [CFA + 6*-8] */
1534 Ptr = iemDwarfPutCfaOffset(Ptr, DWREG_AMD64_R15, 7); /* R15 = [CFA + 7*-8] */
1535# elif defined(RT_ARCH_ARM64)
1536# if 1
1537 Ptr = iemDwarfPutCfaDefCfa(Ptr, DWREG_ARM64_BP, 16); /* CFA = BP + 0x10 - first stack parameter */
1538# else
1539 Ptr = iemDwarfPutCfaDefCfa(Ptr, DWREG_ARM64_SP, IEMNATIVE_FRAME_VAR_SIZE + IEMNATIVE_FRAME_SAVE_REG_SIZE);
1540# endif
1541 Ptr = iemDwarfPutCfaOffset(Ptr, DWREG_ARM64_LR, 1); /* Ret PC = [CFA + 1*-8] */
1542 Ptr = iemDwarfPutCfaOffset(Ptr, DWREG_ARM64_BP, 2); /* Ret BP = [CFA + 2*-8] */
1543 Ptr = iemDwarfPutCfaOffset(Ptr, DWREG_ARM64_X28, 3); /* X28 = [CFA + 3*-8] */
1544 Ptr = iemDwarfPutCfaOffset(Ptr, DWREG_ARM64_X27, 4); /* X27 = [CFA + 4*-8] */
1545 Ptr = iemDwarfPutCfaOffset(Ptr, DWREG_ARM64_X26, 5); /* X26 = [CFA + 5*-8] */
1546 Ptr = iemDwarfPutCfaOffset(Ptr, DWREG_ARM64_X25, 6); /* X25 = [CFA + 6*-8] */
1547 Ptr = iemDwarfPutCfaOffset(Ptr, DWREG_ARM64_X24, 7); /* X24 = [CFA + 7*-8] */
1548 Ptr = iemDwarfPutCfaOffset(Ptr, DWREG_ARM64_X23, 8); /* X23 = [CFA + 8*-8] */
1549 Ptr = iemDwarfPutCfaOffset(Ptr, DWREG_ARM64_X22, 9); /* X22 = [CFA + 9*-8] */
1550 Ptr = iemDwarfPutCfaOffset(Ptr, DWREG_ARM64_X21, 10); /* X21 = [CFA +10*-8] */
1551 Ptr = iemDwarfPutCfaOffset(Ptr, DWREG_ARM64_X20, 11); /* X20 = [CFA +11*-8] */
1552 Ptr = iemDwarfPutCfaOffset(Ptr, DWREG_ARM64_X19, 12); /* X19 = [CFA +12*-8] */
1553 AssertCompile(IEMNATIVE_FRAME_SAVE_REG_SIZE / 8 == 12);
1554 /** @todo we we need to do something about clearing DWREG_ARM64_RA_SIGN_STATE or something? */
1555# else
1556# error "port me"
1557# endif
1558 while ((Ptr.u - PtrCie.u) & 3)
1559 *Ptr.pb++ = DW_CFA_nop;
1560 /* Finalize the CIE size. */
1561 *PtrCie.pu32 = Ptr.u - PtrCie.u - sizeof(uint32_t);
1562
1563 /*
1564 * Generate an FDE for the whole chunk area.
1565 */
1566# ifdef IEMNATIVE_USE_LIBUNWIND
1567 pEhFrame->offFda = Ptr.u - (uintptr_t)&pEhFrame->abEhFrame[0];
1568# endif
1569 RTPTRUNION const PtrFde = Ptr;
1570 *Ptr.pu32++ = 123; /* The CIE length will be determined later. */
1571 *Ptr.pu32 = Ptr.u - PtrCie.u; /* Negated self relative CIE address. */
1572 Ptr.pu32++;
1573 *Ptr.pu64++ = (uintptr_t)pvChunk; /* Absolute start PC of this FDE. */
1574 *Ptr.pu64++ = pExecMemAllocator->cbChunk; /* PC range length for this PDE. */
1575# if 0 /* not requried for recent libunwind.dylib nor recent libgcc/glib. */
1576 *Ptr.pb++ = DW_CFA_nop;
1577# endif
1578 while ((Ptr.u - PtrFde.u) & 3)
1579 *Ptr.pb++ = DW_CFA_nop;
1580 /* Finalize the FDE size. */
1581 *PtrFde.pu32 = Ptr.u - PtrFde.u - sizeof(uint32_t);
1582
1583 /* Terminator entry. */
1584 *Ptr.pu32++ = 0;
1585 *Ptr.pu32++ = 0; /* just to be sure... */
1586 Assert(Ptr.u - (uintptr_t)&pEhFrame->abEhFrame[0] <= sizeof(pEhFrame->abEhFrame));
1587
1588 /*
1589 * Register it.
1590 */
1591# ifdef IEMNATIVE_USE_LIBUNWIND
1592 __register_frame(&pEhFrame->abEhFrame[pEhFrame->offFda]);
1593# else
1594 memset(pEhFrame->abObject, 0xf6, sizeof(pEhFrame->abObject)); /* color the memory to better spot usage */
1595 __register_frame_info(pEhFrame->abEhFrame, pEhFrame->abObject);
1596# endif
1597
1598# ifdef IEMNATIVE_USE_GDB_JIT
1599 /*
1600 * Now for telling GDB about this (experimental).
1601 *
1602 * This seems to work best with ET_DYN.
1603 */
1604 GDBJITSYMFILE * const pSymFile = (GDBJITSYMFILE *)iemExecMemAllocatorAllocBytesInChunk(pExecMemAllocator, idxChunk,
1605 sizeof(GDBJITSYMFILE), NULL);
1606 AssertReturn(pSymFile, VERR_INTERNAL_ERROR_5);
1607 unsigned const offSymFileInChunk = (uintptr_t)pSymFile - (uintptr_t)pvChunk;
1608
1609 RT_ZERO(*pSymFile);
1610
1611 /*
1612 * The ELF header:
1613 */
1614 pSymFile->EHdr.e_ident[0] = ELFMAG0;
1615 pSymFile->EHdr.e_ident[1] = ELFMAG1;
1616 pSymFile->EHdr.e_ident[2] = ELFMAG2;
1617 pSymFile->EHdr.e_ident[3] = ELFMAG3;
1618 pSymFile->EHdr.e_ident[EI_VERSION] = EV_CURRENT;
1619 pSymFile->EHdr.e_ident[EI_CLASS] = ELFCLASS64;
1620 pSymFile->EHdr.e_ident[EI_DATA] = ELFDATA2LSB;
1621 pSymFile->EHdr.e_ident[EI_OSABI] = ELFOSABI_NONE;
1622# ifdef IEMNATIVE_USE_GDB_JIT_ET_DYN
1623 pSymFile->EHdr.e_type = ET_DYN;
1624# else
1625 pSymFile->EHdr.e_type = ET_REL;
1626# endif
1627# ifdef RT_ARCH_AMD64
1628 pSymFile->EHdr.e_machine = EM_AMD64;
1629# elif defined(RT_ARCH_ARM64)
1630 pSymFile->EHdr.e_machine = EM_AARCH64;
1631# else
1632# error "port me"
1633# endif
1634 pSymFile->EHdr.e_version = 1; /*?*/
1635 pSymFile->EHdr.e_entry = 0;
1636# if defined(IEMNATIVE_USE_GDB_JIT_ET_DYN)
1637 pSymFile->EHdr.e_phoff = RT_UOFFSETOF(GDBJITSYMFILE, aPhdrs);
1638# else
1639 pSymFile->EHdr.e_phoff = 0;
1640# endif
1641 pSymFile->EHdr.e_shoff = sizeof(pSymFile->EHdr);
1642 pSymFile->EHdr.e_flags = 0;
1643 pSymFile->EHdr.e_ehsize = sizeof(pSymFile->EHdr);
1644# if defined(IEMNATIVE_USE_GDB_JIT_ET_DYN)
1645 pSymFile->EHdr.e_phentsize = sizeof(pSymFile->aPhdrs[0]);
1646 pSymFile->EHdr.e_phnum = RT_ELEMENTS(pSymFile->aPhdrs);
1647# else
1648 pSymFile->EHdr.e_phentsize = 0;
1649 pSymFile->EHdr.e_phnum = 0;
1650# endif
1651 pSymFile->EHdr.e_shentsize = sizeof(pSymFile->aShdrs[0]);
1652 pSymFile->EHdr.e_shnum = RT_ELEMENTS(pSymFile->aShdrs);
1653 pSymFile->EHdr.e_shstrndx = 0; /* set later */
1654
1655 uint32_t offStrTab = 0;
1656#define APPEND_STR(a_szStr) do { \
1657 memcpy(&pSymFile->szzStrTab[offStrTab], a_szStr, sizeof(a_szStr)); \
1658 offStrTab += sizeof(a_szStr); \
1659 Assert(offStrTab < sizeof(pSymFile->szzStrTab)); \
1660 } while (0)
1661#define APPEND_STR_FMT(a_szStr, ...) do { \
1662 offStrTab += RTStrPrintf(&pSymFile->szzStrTab[offStrTab], sizeof(pSymFile->szzStrTab) - offStrTab, a_szStr, __VA_ARGS__); \
1663 offStrTab++; \
1664 Assert(offStrTab < sizeof(pSymFile->szzStrTab)); \
1665 } while (0)
1666
1667 /*
1668 * Section headers.
1669 */
1670 /* Section header #0: NULL */
1671 unsigned i = 0;
1672 APPEND_STR("");
1673 RT_ZERO(pSymFile->aShdrs[i]);
1674 i++;
1675
1676 /* Section header: .eh_frame */
1677 pSymFile->aShdrs[i].sh_name = offStrTab;
1678 APPEND_STR(".eh_frame");
1679 pSymFile->aShdrs[i].sh_type = SHT_PROGBITS;
1680 pSymFile->aShdrs[i].sh_flags = SHF_ALLOC | SHF_EXECINSTR;
1681# if defined(IEMNATIVE_USE_GDB_JIT_ET_DYN) || defined(IEMNATIVE_USE_GDB_JIT_ELF_RVAS)
1682 pSymFile->aShdrs[i].sh_offset
1683 = pSymFile->aShdrs[i].sh_addr = RT_UOFFSETOF(GDBJITSYMFILE, abEhFrame);
1684# else
1685 pSymFile->aShdrs[i].sh_addr = (uintptr_t)&pSymFile->abEhFrame[0];
1686 pSymFile->aShdrs[i].sh_offset = 0;
1687# endif
1688
1689 pSymFile->aShdrs[i].sh_size = sizeof(pEhFrame->abEhFrame);
1690 pSymFile->aShdrs[i].sh_link = 0;
1691 pSymFile->aShdrs[i].sh_info = 0;
1692 pSymFile->aShdrs[i].sh_addralign = 1;
1693 pSymFile->aShdrs[i].sh_entsize = 0;
1694 memcpy(pSymFile->abEhFrame, pEhFrame->abEhFrame, sizeof(pEhFrame->abEhFrame));
1695 i++;
1696
1697 /* Section header: .shstrtab */
1698 unsigned const iShStrTab = i;
1699 pSymFile->EHdr.e_shstrndx = iShStrTab;
1700 pSymFile->aShdrs[i].sh_name = offStrTab;
1701 APPEND_STR(".shstrtab");
1702 pSymFile->aShdrs[i].sh_type = SHT_STRTAB;
1703 pSymFile->aShdrs[i].sh_flags = SHF_ALLOC;
1704# if defined(IEMNATIVE_USE_GDB_JIT_ET_DYN) || defined(IEMNATIVE_USE_GDB_JIT_ELF_RVAS)
1705 pSymFile->aShdrs[i].sh_offset
1706 = pSymFile->aShdrs[i].sh_addr = RT_UOFFSETOF(GDBJITSYMFILE, szzStrTab);
1707# else
1708 pSymFile->aShdrs[i].sh_addr = (uintptr_t)&pSymFile->szzStrTab[0];
1709 pSymFile->aShdrs[i].sh_offset = 0;
1710# endif
1711 pSymFile->aShdrs[i].sh_size = sizeof(pSymFile->szzStrTab);
1712 pSymFile->aShdrs[i].sh_link = 0;
1713 pSymFile->aShdrs[i].sh_info = 0;
1714 pSymFile->aShdrs[i].sh_addralign = 1;
1715 pSymFile->aShdrs[i].sh_entsize = 0;
1716 i++;
1717
1718 /* Section header: .symbols */
1719 pSymFile->aShdrs[i].sh_name = offStrTab;
1720 APPEND_STR(".symtab");
1721 pSymFile->aShdrs[i].sh_type = SHT_SYMTAB;
1722 pSymFile->aShdrs[i].sh_flags = SHF_ALLOC;
1723 pSymFile->aShdrs[i].sh_offset
1724 = pSymFile->aShdrs[i].sh_addr = RT_UOFFSETOF(GDBJITSYMFILE, aSymbols);
1725 pSymFile->aShdrs[i].sh_size = sizeof(pSymFile->aSymbols);
1726 pSymFile->aShdrs[i].sh_link = iShStrTab;
1727 pSymFile->aShdrs[i].sh_info = RT_ELEMENTS(pSymFile->aSymbols);
1728 pSymFile->aShdrs[i].sh_addralign = sizeof(pSymFile->aSymbols[0].st_value);
1729 pSymFile->aShdrs[i].sh_entsize = sizeof(pSymFile->aSymbols[0]);
1730 i++;
1731
1732# if defined(IEMNATIVE_USE_GDB_JIT_ET_DYN)
1733 /* Section header: .symbols */
1734 pSymFile->aShdrs[i].sh_name = offStrTab;
1735 APPEND_STR(".dynsym");
1736 pSymFile->aShdrs[i].sh_type = SHT_DYNSYM;
1737 pSymFile->aShdrs[i].sh_flags = SHF_ALLOC;
1738 pSymFile->aShdrs[i].sh_offset
1739 = pSymFile->aShdrs[i].sh_addr = RT_UOFFSETOF(GDBJITSYMFILE, aDynSyms);
1740 pSymFile->aShdrs[i].sh_size = sizeof(pSymFile->aDynSyms);
1741 pSymFile->aShdrs[i].sh_link = iShStrTab;
1742 pSymFile->aShdrs[i].sh_info = RT_ELEMENTS(pSymFile->aDynSyms);
1743 pSymFile->aShdrs[i].sh_addralign = sizeof(pSymFile->aDynSyms[0].st_value);
1744 pSymFile->aShdrs[i].sh_entsize = sizeof(pSymFile->aDynSyms[0]);
1745 i++;
1746# endif
1747
1748# if defined(IEMNATIVE_USE_GDB_JIT_ET_DYN)
1749 /* Section header: .dynamic */
1750 pSymFile->aShdrs[i].sh_name = offStrTab;
1751 APPEND_STR(".dynamic");
1752 pSymFile->aShdrs[i].sh_type = SHT_DYNAMIC;
1753 pSymFile->aShdrs[i].sh_flags = SHF_ALLOC;
1754 pSymFile->aShdrs[i].sh_offset
1755 = pSymFile->aShdrs[i].sh_addr = RT_UOFFSETOF(GDBJITSYMFILE, aDyn);
1756 pSymFile->aShdrs[i].sh_size = sizeof(pSymFile->aDyn);
1757 pSymFile->aShdrs[i].sh_link = iShStrTab;
1758 pSymFile->aShdrs[i].sh_info = 0;
1759 pSymFile->aShdrs[i].sh_addralign = 1;
1760 pSymFile->aShdrs[i].sh_entsize = sizeof(pSymFile->aDyn[0]);
1761 i++;
1762# endif
1763
1764 /* Section header: .text */
1765 unsigned const iShText = i;
1766 pSymFile->aShdrs[i].sh_name = offStrTab;
1767 APPEND_STR(".text");
1768 pSymFile->aShdrs[i].sh_type = SHT_PROGBITS;
1769 pSymFile->aShdrs[i].sh_flags = SHF_ALLOC | SHF_EXECINSTR;
1770# if defined(IEMNATIVE_USE_GDB_JIT_ET_DYN) || defined(IEMNATIVE_USE_GDB_JIT_ELF_RVAS)
1771 pSymFile->aShdrs[i].sh_offset
1772 = pSymFile->aShdrs[i].sh_addr = sizeof(GDBJITSYMFILE);
1773# else
1774 pSymFile->aShdrs[i].sh_addr = (uintptr_t)(pSymFile + 1);
1775 pSymFile->aShdrs[i].sh_offset = 0;
1776# endif
1777 pSymFile->aShdrs[i].sh_size = pExecMemAllocator->cbChunk - offSymFileInChunk - sizeof(GDBJITSYMFILE);
1778 pSymFile->aShdrs[i].sh_link = 0;
1779 pSymFile->aShdrs[i].sh_info = 0;
1780 pSymFile->aShdrs[i].sh_addralign = 1;
1781 pSymFile->aShdrs[i].sh_entsize = 0;
1782 i++;
1783
1784 Assert(i == RT_ELEMENTS(pSymFile->aShdrs));
1785
1786# if defined(IEMNATIVE_USE_GDB_JIT_ET_DYN)
1787 /*
1788 * The program headers:
1789 */
1790 /* Everything in a single LOAD segment: */
1791 i = 0;
1792 pSymFile->aPhdrs[i].p_type = PT_LOAD;
1793 pSymFile->aPhdrs[i].p_flags = PF_X | PF_R;
1794 pSymFile->aPhdrs[i].p_offset
1795 = pSymFile->aPhdrs[i].p_vaddr
1796 = pSymFile->aPhdrs[i].p_paddr = 0;
1797 pSymFile->aPhdrs[i].p_filesz /* Size of segment in file. */
1798 = pSymFile->aPhdrs[i].p_memsz = pExecMemAllocator->cbChunk - offSymFileInChunk;
1799 pSymFile->aPhdrs[i].p_align = HOST_PAGE_SIZE;
1800 i++;
1801 /* The .dynamic segment. */
1802 pSymFile->aPhdrs[i].p_type = PT_DYNAMIC;
1803 pSymFile->aPhdrs[i].p_flags = PF_R;
1804 pSymFile->aPhdrs[i].p_offset
1805 = pSymFile->aPhdrs[i].p_vaddr
1806 = pSymFile->aPhdrs[i].p_paddr = RT_UOFFSETOF(GDBJITSYMFILE, aDyn);
1807 pSymFile->aPhdrs[i].p_filesz /* Size of segment in file. */
1808 = pSymFile->aPhdrs[i].p_memsz = sizeof(pSymFile->aDyn);
1809 pSymFile->aPhdrs[i].p_align = sizeof(pSymFile->aDyn[0].d_tag);
1810 i++;
1811
1812 Assert(i == RT_ELEMENTS(pSymFile->aPhdrs));
1813
1814 /*
1815 * The dynamic section:
1816 */
1817 i = 0;
1818 pSymFile->aDyn[i].d_tag = DT_SONAME;
1819 pSymFile->aDyn[i].d_un.d_val = offStrTab;
1820 APPEND_STR_FMT("iem-exec-chunk-%u-%u", pVCpu->idCpu, idxChunk);
1821 i++;
1822 pSymFile->aDyn[i].d_tag = DT_STRTAB;
1823 pSymFile->aDyn[i].d_un.d_ptr = RT_UOFFSETOF(GDBJITSYMFILE, szzStrTab);
1824 i++;
1825 pSymFile->aDyn[i].d_tag = DT_STRSZ;
1826 pSymFile->aDyn[i].d_un.d_val = sizeof(pSymFile->szzStrTab);
1827 i++;
1828 pSymFile->aDyn[i].d_tag = DT_SYMTAB;
1829 pSymFile->aDyn[i].d_un.d_ptr = RT_UOFFSETOF(GDBJITSYMFILE, aDynSyms);
1830 i++;
1831 pSymFile->aDyn[i].d_tag = DT_SYMENT;
1832 pSymFile->aDyn[i].d_un.d_val = sizeof(pSymFile->aDynSyms[0]);
1833 i++;
1834 pSymFile->aDyn[i].d_tag = DT_NULL;
1835 i++;
1836 Assert(i == RT_ELEMENTS(pSymFile->aDyn));
1837# endif /* IEMNATIVE_USE_GDB_JIT_ET_DYN */
1838
1839 /*
1840 * Symbol tables:
1841 */
1842 /** @todo gdb doesn't seem to really like this ... */
1843 i = 0;
1844 pSymFile->aSymbols[i].st_name = 0;
1845 pSymFile->aSymbols[i].st_shndx = SHN_UNDEF;
1846 pSymFile->aSymbols[i].st_value = 0;
1847 pSymFile->aSymbols[i].st_size = 0;
1848 pSymFile->aSymbols[i].st_info = ELF64_ST_INFO(STB_LOCAL, STT_NOTYPE);
1849 pSymFile->aSymbols[i].st_other = 0 /* STV_DEFAULT */;
1850# ifdef IEMNATIVE_USE_GDB_JIT_ET_DYN
1851 pSymFile->aDynSyms[0] = pSymFile->aSymbols[i];
1852# endif
1853 i++;
1854
1855 pSymFile->aSymbols[i].st_name = 0;
1856 pSymFile->aSymbols[i].st_shndx = SHN_ABS;
1857 pSymFile->aSymbols[i].st_value = 0;
1858 pSymFile->aSymbols[i].st_size = 0;
1859 pSymFile->aSymbols[i].st_info = ELF64_ST_INFO(STB_LOCAL, STT_FILE);
1860 pSymFile->aSymbols[i].st_other = 0 /* STV_DEFAULT */;
1861 i++;
1862
1863 pSymFile->aSymbols[i].st_name = offStrTab;
1864 APPEND_STR_FMT("iem_exec_chunk_%u_%u", pVCpu->idCpu, idxChunk);
1865# if 0
1866 pSymFile->aSymbols[i].st_shndx = iShText;
1867 pSymFile->aSymbols[i].st_value = 0;
1868# else
1869 pSymFile->aSymbols[i].st_shndx = SHN_ABS;
1870 pSymFile->aSymbols[i].st_value = (uintptr_t)(pSymFile + 1);
1871# endif
1872 pSymFile->aSymbols[i].st_size = pSymFile->aShdrs[iShText].sh_size;
1873 pSymFile->aSymbols[i].st_info = ELF64_ST_INFO(STB_GLOBAL, STT_FUNC);
1874 pSymFile->aSymbols[i].st_other = 0 /* STV_DEFAULT */;
1875# ifdef IEMNATIVE_USE_GDB_JIT_ET_DYN
1876 pSymFile->aDynSyms[1] = pSymFile->aSymbols[i];
1877 pSymFile->aDynSyms[1].st_value = (uintptr_t)(pSymFile + 1);
1878# endif
1879 i++;
1880
1881 Assert(i == RT_ELEMENTS(pSymFile->aSymbols));
1882 Assert(offStrTab < sizeof(pSymFile->szzStrTab));
1883
1884 /*
1885 * The GDB JIT entry and informing GDB.
1886 */
1887 pEhFrame->GdbJitEntry.pbSymFile = (uint8_t *)pSymFile;
1888# if 1
1889 pEhFrame->GdbJitEntry.cbSymFile = pExecMemAllocator->cbChunk - ((uintptr_t)pSymFile - (uintptr_t)pvChunk);
1890# else
1891 pEhFrame->GdbJitEntry.cbSymFile = sizeof(GDBJITSYMFILE);
1892# endif
1893
1894 RTOnce(&g_IemNativeGdbJitOnce, iemNativeGdbJitInitOnce, NULL);
1895 RTCritSectEnter(&g_IemNativeGdbJitLock);
1896 pEhFrame->GdbJitEntry.pNext = NULL;
1897 pEhFrame->GdbJitEntry.pPrev = __jit_debug_descriptor.pTail;
1898 if (__jit_debug_descriptor.pTail)
1899 __jit_debug_descriptor.pTail->pNext = &pEhFrame->GdbJitEntry;
1900 else
1901 __jit_debug_descriptor.pHead = &pEhFrame->GdbJitEntry;
1902 __jit_debug_descriptor.pTail = &pEhFrame->GdbJitEntry;
1903 __jit_debug_descriptor.pRelevant = &pEhFrame->GdbJitEntry;
1904
1905 /* Notify GDB: */
1906 __jit_debug_descriptor.enmAction = kGdbJitaction_Register;
1907 __jit_debug_register_code();
1908 __jit_debug_descriptor.enmAction = kGdbJitaction_NoAction;
1909 RTCritSectLeave(&g_IemNativeGdbJitLock);
1910
1911# else /* !IEMNATIVE_USE_GDB_JIT */
1912 RT_NOREF(pVCpu);
1913# endif /* !IEMNATIVE_USE_GDB_JIT */
1914
1915 return VINF_SUCCESS;
1916}
1917
1918# endif /* !RT_OS_WINDOWS */
1919#endif /* IN_RING3 */
1920
1921
1922/**
1923 * Adds another chunk to the executable memory allocator.
1924 *
1925 * This is used by the init code for the initial allocation and later by the
1926 * regular allocator function when it's out of memory.
1927 */
1928static int iemExecMemAllocatorGrow(PVMCPUCC pVCpu, PIEMEXECMEMALLOCATOR pExecMemAllocator)
1929{
1930 /* Check that we've room for growth. */
1931 uint32_t const idxChunk = pExecMemAllocator->cChunks;
1932 AssertLogRelReturn(idxChunk < pExecMemAllocator->cMaxChunks, VERR_OUT_OF_RESOURCES);
1933
1934 /* Allocate a chunk. */
1935#ifdef RT_OS_DARWIN
1936 void *pvChunk = RTMemPageAllocEx(pExecMemAllocator->cbChunk, 0);
1937#else
1938 void *pvChunk = RTMemPageAllocEx(pExecMemAllocator->cbChunk, RTMEMPAGEALLOC_F_EXECUTABLE);
1939#endif
1940 AssertLogRelReturn(pvChunk, VERR_NO_EXEC_MEMORY);
1941
1942#ifdef RT_OS_DARWIN
1943 /*
1944 * Because it is impossible to have a RWX memory allocation on macOS try to remap the memory
1945 * chunk readable/executable somewhere else so we can save us the hassle of switching between
1946 * protections when exeuctable memory is allocated.
1947 */
1948 int rc = VERR_NO_EXEC_MEMORY;
1949 mach_port_t hPortTask = mach_task_self();
1950 mach_vm_address_t AddrChunk = (mach_vm_address_t)pvChunk;
1951 mach_vm_address_t AddrRemapped = 0;
1952 vm_prot_t ProtCur = 0;
1953 vm_prot_t ProtMax = 0;
1954 kern_return_t krc = mach_vm_remap(hPortTask, &AddrRemapped, pExecMemAllocator->cbChunk, 0,
1955 VM_FLAGS_ANYWHERE | VM_FLAGS_RETURN_DATA_ADDR,
1956 hPortTask, AddrChunk, FALSE, &ProtCur, &ProtMax,
1957 VM_INHERIT_NONE);
1958 if (krc == KERN_SUCCESS)
1959 {
1960 krc = mach_vm_protect(mach_task_self(), AddrRemapped, pExecMemAllocator->cbChunk, FALSE, VM_PROT_READ | VM_PROT_EXECUTE);
1961 if (krc == KERN_SUCCESS)
1962 rc = VINF_SUCCESS;
1963 else
1964 {
1965 AssertLogRelMsgFailed(("mach_vm_protect -> %d (%#x)\n", krc, krc));
1966 krc = mach_vm_deallocate(hPortTask, AddrRemapped, pExecMemAllocator->cbChunk);
1967 Assert(krc == KERN_SUCCESS);
1968 }
1969 }
1970 else
1971 AssertLogRelMsgFailed(("mach_vm_remap -> %d (%#x)\n", krc, krc));
1972 if (RT_FAILURE(rc))
1973 {
1974 RTMemPageFree(pvChunk, pExecMemAllocator->cbChunk);
1975 return rc;
1976 }
1977
1978 void *pvChunkRx = (void *)AddrRemapped;
1979#else
1980 int rc = VINF_SUCCESS;
1981 void *pvChunkRx = pvChunk;
1982#endif
1983
1984 /*
1985 * Add the chunk.
1986 *
1987 * This must be done before the unwind init so windows can allocate
1988 * memory from the chunk when using the alternative sub-allocator.
1989 */
1990 pExecMemAllocator->aChunks[idxChunk].pvChunkRw = pvChunk;
1991 pExecMemAllocator->aChunks[idxChunk].pvChunkRx = pvChunkRx;
1992#ifdef IN_RING3
1993 pExecMemAllocator->aChunks[idxChunk].pvUnwindInfo = NULL;
1994#endif
1995 pExecMemAllocator->aChunks[idxChunk].cFreeUnits = pExecMemAllocator->cUnitsPerChunk;
1996 pExecMemAllocator->aChunks[idxChunk].idxFreeHint = 0;
1997 memset(&pExecMemAllocator->pbmAlloc[pExecMemAllocator->cBitmapElementsPerChunk * idxChunk],
1998 0, sizeof(pExecMemAllocator->pbmAlloc[0]) * pExecMemAllocator->cBitmapElementsPerChunk);
1999
2000 pExecMemAllocator->cChunks = idxChunk + 1;
2001 pExecMemAllocator->idxChunkHint = idxChunk;
2002
2003 pExecMemAllocator->cbTotal += pExecMemAllocator->cbChunk;
2004 pExecMemAllocator->cbFree += pExecMemAllocator->cbChunk;
2005
2006 /* If there is a chunk context init callback call it. */
2007 rc = iemNativeRecompileAttachExecMemChunkCtx(pVCpu, idxChunk, &pExecMemAllocator->aChunks[idxChunk].pCtx);
2008#ifdef IN_RING3
2009 /*
2010 * Initialize the unwind information (this cannot really fail atm).
2011 * (This sets pvUnwindInfo.)
2012 */
2013 if (RT_SUCCESS(rc))
2014 rc = iemExecMemAllocatorInitAndRegisterUnwindInfoForChunk(pVCpu, pExecMemAllocator, pvChunkRx, idxChunk);
2015#endif
2016 if (RT_SUCCESS(rc))
2017 { /* likely */ }
2018 else
2019 {
2020 /* Just in case the impossible happens, undo the above up: */
2021 pExecMemAllocator->cbTotal -= pExecMemAllocator->cbChunk;
2022 pExecMemAllocator->cbFree -= pExecMemAllocator->aChunks[idxChunk].cFreeUnits << IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SHIFT;
2023 pExecMemAllocator->cChunks = idxChunk;
2024 memset(&pExecMemAllocator->pbmAlloc[pExecMemAllocator->cBitmapElementsPerChunk * idxChunk],
2025 0xff, sizeof(pExecMemAllocator->pbmAlloc[0]) * pExecMemAllocator->cBitmapElementsPerChunk);
2026 pExecMemAllocator->aChunks[idxChunk].pvChunkRw = NULL;
2027 pExecMemAllocator->aChunks[idxChunk].cFreeUnits = 0;
2028
2029# ifdef RT_OS_DARWIN
2030 krc = mach_vm_deallocate(mach_task_self(), (mach_vm_address_t)pExecMemAllocator->aChunks[idxChunk].pvChunkRx,
2031 pExecMemAllocator->cbChunk);
2032 Assert(krc == KERN_SUCCESS);
2033# endif
2034
2035 RTMemPageFree(pvChunk, pExecMemAllocator->cbChunk);
2036 return rc;
2037 }
2038
2039 return VINF_SUCCESS;
2040}
2041
2042
2043/**
2044 * Initializes the executable memory allocator for native recompilation on the
2045 * calling EMT.
2046 *
2047 * @returns VBox status code.
2048 * @param pVCpu The cross context virtual CPU structure of the calling
2049 * thread.
2050 * @param cbMax The max size of the allocator.
2051 * @param cbInitial The initial allocator size.
2052 * @param cbChunk The chunk size, 0 or UINT32_MAX for default (@a cbMax
2053 * dependent).
2054 */
2055int iemExecMemAllocatorInit(PVMCPU pVCpu, uint64_t cbMax, uint64_t cbInitial, uint32_t cbChunk) RT_NOEXCEPT
2056{
2057 /*
2058 * Validate input.
2059 */
2060 AssertLogRelMsgReturn(cbMax >= _1M && cbMax <= _4G+_4G, ("cbMax=%RU64 (%RX64)\n", cbMax, cbMax), VERR_OUT_OF_RANGE);
2061 AssertReturn(cbInitial <= cbMax, VERR_OUT_OF_RANGE);
2062 AssertLogRelMsgReturn( cbChunk != UINT32_MAX
2063 || cbChunk == 0
2064 || ( RT_IS_POWER_OF_TWO(cbChunk)
2065 && cbChunk >= _1M
2066 && cbChunk <= _256M
2067 && cbChunk <= cbMax),
2068 ("cbChunk=%RU32 (%RX32) cbMax=%RU64\n", cbChunk, cbChunk, cbMax),
2069 VERR_OUT_OF_RANGE);
2070
2071 /*
2072 * Adjust/figure out the chunk size.
2073 */
2074 if (cbChunk == 0 || cbChunk == UINT32_MAX)
2075 {
2076 if (cbMax >= _256M)
2077 cbChunk = _64M;
2078 else
2079 {
2080 if (cbMax < _16M)
2081 cbChunk = cbMax >= _4M ? _4M : (uint32_t)cbMax;
2082 else
2083 cbChunk = (uint32_t)cbMax / 4;
2084 if (!RT_IS_POWER_OF_TWO(cbChunk))
2085 cbChunk = RT_BIT_32(ASMBitLastSetU32(cbChunk));
2086 }
2087 }
2088#if defined(RT_OS_AMD64)
2089 Assert(cbChunk <= _2G);
2090#elif defined(RT_OS_ARM64)
2091 if (cbChunk > _128M)
2092 cbChunk = _128M; /* Max relative branch distance is +/-2^(25+2) = +/-0x8000000 (134 217 728). */
2093#endif
2094
2095 if (cbChunk > cbMax)
2096 cbMax = cbChunk;
2097 else
2098 cbMax = (cbMax - 1 + cbChunk) / cbChunk * cbChunk;
2099 uint32_t const cMaxChunks = (uint32_t)(cbMax / cbChunk);
2100 AssertLogRelReturn((uint64_t)cMaxChunks * cbChunk == cbMax, VERR_INTERNAL_ERROR_3);
2101
2102 /*
2103 * Allocate and initialize the allocatore instance.
2104 */
2105 size_t const offBitmaps = RT_ALIGN_Z(RT_UOFFSETOF_DYN(IEMEXECMEMALLOCATOR, aChunks[cMaxChunks]), RT_CACHELINE_SIZE);
2106 size_t const cbBitmaps = (size_t)(cbChunk >> (IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SHIFT + 3)) * cMaxChunks;
2107 size_t cbNeeded = offBitmaps + cbBitmaps;
2108 AssertCompile(IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SHIFT <= 10);
2109 Assert(cbChunk > RT_BIT_32(IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SHIFT + 3));
2110#if defined(IN_RING3) && !defined(RT_OS_WINDOWS)
2111 size_t const offEhFrames = RT_ALIGN_Z(cbNeeded, RT_CACHELINE_SIZE);
2112 cbNeeded += sizeof(IEMEXECMEMCHUNKEHFRAME) * cMaxChunks;
2113#endif
2114 PIEMEXECMEMALLOCATOR pExecMemAllocator = (PIEMEXECMEMALLOCATOR)RTMemAllocZ(cbNeeded);
2115 AssertLogRelMsgReturn(pExecMemAllocator, ("cbNeeded=%zx cMaxChunks=%#x cbChunk=%#x\n", cbNeeded, cMaxChunks, cbChunk),
2116 VERR_NO_MEMORY);
2117 pExecMemAllocator->uMagic = IEMEXECMEMALLOCATOR_MAGIC;
2118 pExecMemAllocator->cbChunk = cbChunk;
2119 pExecMemAllocator->cMaxChunks = cMaxChunks;
2120 pExecMemAllocator->cChunks = 0;
2121 pExecMemAllocator->idxChunkHint = 0;
2122 pExecMemAllocator->cAllocations = 0;
2123 pExecMemAllocator->cbTotal = 0;
2124 pExecMemAllocator->cbFree = 0;
2125 pExecMemAllocator->cbAllocated = 0;
2126#ifdef VBOX_WITH_STATISTICS
2127 pExecMemAllocator->cbUnusable = 0;
2128#endif
2129 pExecMemAllocator->pbmAlloc = (uint64_t *)((uintptr_t)pExecMemAllocator + offBitmaps);
2130 pExecMemAllocator->cUnitsPerChunk = cbChunk >> IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SHIFT;
2131 pExecMemAllocator->cBitmapElementsPerChunk = cbChunk >> (IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SHIFT + 6);
2132 memset(pExecMemAllocator->pbmAlloc, 0xff, cbBitmaps); /* Mark everything as allocated. Clear when chunks are added. */
2133#if defined(IN_RING3) && !defined(RT_OS_WINDOWS)
2134 pExecMemAllocator->paEhFrames = (PIEMEXECMEMCHUNKEHFRAME)((uintptr_t)pExecMemAllocator + offEhFrames);
2135#endif
2136 for (uint32_t i = 0; i < cMaxChunks; i++)
2137 {
2138 pExecMemAllocator->aChunks[i].cFreeUnits = 0;
2139 pExecMemAllocator->aChunks[i].idxFreeHint = 0;
2140 pExecMemAllocator->aChunks[i].pvChunkRw = NULL;
2141#ifdef IN_RING0
2142 pExecMemAllocator->aChunks[i].hMemObj = NIL_RTR0MEMOBJ;
2143#else
2144 pExecMemAllocator->aChunks[i].pvUnwindInfo = NULL;
2145#endif
2146 }
2147 pVCpu->iem.s.pExecMemAllocatorR3 = pExecMemAllocator;
2148
2149 /*
2150 * Do the initial allocations.
2151 */
2152 while (cbInitial < (uint64_t)pExecMemAllocator->cChunks * pExecMemAllocator->cbChunk)
2153 {
2154 int rc = iemExecMemAllocatorGrow(pVCpu, pExecMemAllocator);
2155 AssertLogRelRCReturn(rc, rc);
2156 }
2157
2158 pExecMemAllocator->idxChunkHint = 0;
2159
2160 /*
2161 * Register statistics.
2162 */
2163 PUVM const pUVM = pVCpu->pUVCpu->pUVM;
2164 STAMR3RegisterFU(pUVM, &pExecMemAllocator->cAllocations, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,
2165 "Current number of allocations", "/IEM/CPU%u/re/ExecMem/cAllocations", pVCpu->idCpu);
2166 STAMR3RegisterFU(pUVM, &pExecMemAllocator->cChunks, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
2167 "Currently allocated chunks", "/IEM/CPU%u/re/ExecMem/cChunks", pVCpu->idCpu);
2168 STAMR3RegisterFU(pUVM, &pExecMemAllocator->cMaxChunks, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
2169 "Maximum number of chunks", "/IEM/CPU%u/re/ExecMem/cMaxChunks", pVCpu->idCpu);
2170 STAMR3RegisterFU(pUVM, &pExecMemAllocator->cbChunk, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,
2171 "Allocation chunk size", "/IEM/CPU%u/re/ExecMem/cbChunk", pVCpu->idCpu);
2172 STAMR3RegisterFU(pUVM, &pExecMemAllocator->cbAllocated, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,
2173 "Number of bytes current allocated", "/IEM/CPU%u/re/ExecMem/cbAllocated", pVCpu->idCpu);
2174 STAMR3RegisterFU(pUVM, &pExecMemAllocator->cbFree, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,
2175 "Number of bytes current free", "/IEM/CPU%u/re/ExecMem/cbFree", pVCpu->idCpu);
2176 STAMR3RegisterFU(pUVM, &pExecMemAllocator->cbTotal, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,
2177 "Total number of byte", "/IEM/CPU%u/re/ExecMem/cbTotal", pVCpu->idCpu);
2178#ifdef VBOX_WITH_STATISTICS
2179 STAMR3RegisterFU(pUVM, &pExecMemAllocator->cbUnusable, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,
2180 "Total number of bytes being unusable", "/IEM/CPU%u/re/ExecMem/cbUnusable", pVCpu->idCpu);
2181 STAMR3RegisterFU(pUVM, &pExecMemAllocator->StatAlloc, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL,
2182 "Profiling the allocator", "/IEM/CPU%u/re/ExecMem/ProfAlloc", pVCpu->idCpu);
2183 for (unsigned i = 1; i < RT_ELEMENTS(pExecMemAllocator->aStatSizes); i++)
2184 STAMR3RegisterFU(pUVM, &pExecMemAllocator->aStatSizes[i], STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
2185 "Number of allocations of this number of allocation units",
2186 "/IEM/CPU%u/re/ExecMem/aSize%02u", pVCpu->idCpu, i);
2187 STAMR3RegisterFU(pUVM, &pExecMemAllocator->aStatSizes[0], STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
2188 "Number of allocations 16 units or larger", "/IEM/CPU%u/re/ExecMem/aSize16OrLarger", pVCpu->idCpu);
2189#endif
2190#ifdef IEMEXECMEM_ALT_SUB_WITH_ALT_PRUNING
2191 STAMR3RegisterFU(pUVM, &pExecMemAllocator->StatPruneProf, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL,
2192 "Pruning executable memory (alt)", "/IEM/CPU%u/re/ExecMem/Pruning", pVCpu->idCpu);
2193 STAMR3RegisterFU(pUVM, &pExecMemAllocator->StatPruneRecovered, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES_PER_CALL,
2194 "Bytes recovered while pruning", "/IEM/CPU%u/re/ExecMem/PruningRecovered", pVCpu->idCpu);
2195#endif
2196 STAMR3RegisterFU(pUVM, &pExecMemAllocator->cFruitlessChunkScans, STAMTYPE_U64_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
2197 "Chunks fruitlessly scanned for free space", "/IEM/CPU%u/re/ExecMem/FruitlessChunkScans", pVCpu->idCpu);
2198
2199 return VINF_SUCCESS;
2200}
2201
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette