Line data Source code
1 : #ifdef __PLUMED_HAS_ASMJIT
2 : // This file is part of AsmJit project <https://asmjit.com>
3 : //
4 : // See asmjit.h or LICENSE.md for license and copyright information
5 : // SPDX-License-Identifier: Zlib
6 :
7 : #include "api-build_p.h"
8 : #ifndef ASMJIT_NO_JIT
9 :
10 : #include "osutils.h"
11 : #include "moved_string.h"
12 : #include "support.h"
13 : #include "virtmem.h"
14 :
15 : #if !defined(_WIN32)
16 : #include <errno.h>
17 : #include <fcntl.h>
18 : #include <sys/mman.h>
19 : #include <sys/stat.h>
20 : #include <sys/types.h>
21 : #include <unistd.h>
22 :
23 : // Linux has a `memfd_create` syscall that we would like to use, if available.
24 : #if defined(__linux__)
25 : #include <sys/syscall.h>
26 : #endif
27 :
28 : // Apple recently introduced MAP_JIT flag, which we want to use.
29 : #if defined(__APPLE__)
30 : #include <pthread.h>
31 : #include <TargetConditionals.h>
32 : #if TARGET_OS_OSX
33 : #include <sys/utsname.h>
34 : #include <libkern/OSCacheControl.h> // sys_icache_invalidate().
35 : #endif
36 : // Older SDK doesn't define `MAP_JIT`.
37 : #ifndef MAP_JIT
38 : #define MAP_JIT 0x800
39 : #endif
40 : #endif
41 :
42 : // BSD/MAC: `MAP_ANONYMOUS` is not defined, `MAP_ANON` is.
43 : #if !defined(MAP_ANONYMOUS)
44 : #define MAP_ANONYMOUS MAP_ANON
45 : #endif
46 : #endif
47 :
48 : #include <atomic>
49 :
50 : #if defined(__APPLE__) || defined(__BIONIC__)
51 : #define ASMJIT_VM_SHM_DETECT 0
52 : #else
53 : #define ASMJIT_VM_SHM_DETECT 1
54 : #endif
55 :
56 : // Android NDK doesn't provide `shm_open()` and `shm_unlink()`.
57 : #if defined(__BIONIC__)
58 : #define ASMJIT_VM_SHM_AVAILABLE 0
59 : #else
60 : #define ASMJIT_VM_SHM_AVAILABLE 1
61 : #endif
62 :
63 : #if defined(__APPLE__) && ASMJIT_ARCH_ARM >= 64
64 : #define ASMJIT_HAS_PTHREAD_JIT_WRITE_PROTECT_NP
65 : #endif
66 :
67 : ASMJIT_BEGIN_SUB_NAMESPACE(VirtMem)
68 :
69 : // Virtual Memory Utilities
70 : // ========================
71 :
72 : static const MemoryFlags dualMappingFilter[2] = {
73 : MemoryFlags::kAccessWrite | MemoryFlags::kMMapMaxAccessWrite,
74 : MemoryFlags::kAccessExecute | MemoryFlags::kMMapMaxAccessExecute
75 : };
76 :
77 : // Virtual Memory [Windows]
78 : // ========================
79 :
80 : #if defined(_WIN32)
81 :
82 : struct ScopedHandle {
83 : inline ScopedHandle() noexcept
84 : : value(nullptr) {}
85 :
86 : inline ~ScopedHandle() noexcept {
87 : if (value != nullptr)
88 : ::CloseHandle(value);
89 : }
90 :
91 : HANDLE value;
92 : };
93 :
94 : static void getVMInfo(Info& vmInfo) noexcept {
95 : SYSTEM_INFO systemInfo;
96 :
97 : ::GetSystemInfo(&systemInfo);
98 : vmInfo.pageSize = Support::alignUpPowerOf2<uint32_t>(systemInfo.dwPageSize);
99 : vmInfo.pageGranularity = systemInfo.dwAllocationGranularity;
100 : }
101 :
102 : // Returns windows-specific protectFlags from \ref MemoryFlags.
103 : static DWORD protectFlagsFromMemoryFlags(MemoryFlags memoryFlags) noexcept {
104 : DWORD protectFlags;
105 :
106 : // READ|WRITE|EXECUTE.
107 : if (Support::test(memoryFlags, MemoryFlags::kAccessExecute))
108 : protectFlags = Support::test(memoryFlags, MemoryFlags::kAccessWrite) ? PAGE_EXECUTE_READWRITE : PAGE_EXECUTE_READ;
109 : else if (Support::test(memoryFlags, MemoryFlags::kAccessRW))
110 : protectFlags = Support::test(memoryFlags, MemoryFlags::kAccessWrite) ? PAGE_READWRITE : PAGE_READONLY;
111 : else
112 : protectFlags = PAGE_NOACCESS;
113 :
114 : // Any other flags to consider?
115 : return protectFlags;
116 : }
117 :
118 : static DWORD desiredAccessFromMemoryFlags(MemoryFlags memoryFlags) noexcept {
119 : DWORD access = Support::test(memoryFlags, MemoryFlags::kAccessWrite) ? FILE_MAP_WRITE : FILE_MAP_READ;
120 : if (Support::test(memoryFlags, MemoryFlags::kAccessExecute))
121 : access |= FILE_MAP_EXECUTE;
122 : return access;
123 : }
124 :
125 : static HardenedRuntimeFlags getHardenedRuntimeFlags() noexcept {
126 : return HardenedRuntimeFlags::kNone;
127 : }
128 :
129 : Error alloc(void** p, size_t size, MemoryFlags memoryFlags) noexcept {
130 : *p = nullptr;
131 : if (size == 0)
132 : return DebugUtils::errored(kErrorInvalidArgument);
133 :
134 : DWORD protectFlags = protectFlagsFromMemoryFlags(memoryFlags);
135 : void* result = ::VirtualAlloc(nullptr, size, MEM_COMMIT | MEM_RESERVE, protectFlags);
136 :
137 : if (!result)
138 : return DebugUtils::errored(kErrorOutOfMemory);
139 :
140 : *p = result;
141 : return kErrorOk;
142 : }
143 :
144 : Error release(void* p, size_t size) noexcept {
145 : DebugUtils::unused(size);
146 : if (ASMJIT_UNLIKELY(!::VirtualFree(p, 0, MEM_RELEASE)))
147 : return DebugUtils::errored(kErrorInvalidArgument);
148 : return kErrorOk;
149 : }
150 :
151 : Error protect(void* p, size_t size, MemoryFlags memoryFlags) noexcept {
152 : DWORD protectFlags = protectFlagsFromMemoryFlags(memoryFlags);
153 : DWORD oldFlags;
154 :
155 : if (::VirtualProtect(p, size, protectFlags, &oldFlags))
156 : return kErrorOk;
157 :
158 : return DebugUtils::errored(kErrorInvalidArgument);
159 : }
160 :
161 : Error allocDualMapping(DualMapping* dm, size_t size, MemoryFlags memoryFlags) noexcept {
162 : dm->rx = nullptr;
163 : dm->rw = nullptr;
164 :
165 : if (size == 0)
166 : return DebugUtils::errored(kErrorInvalidArgument);
167 :
168 : ScopedHandle handle;
169 : handle.value = ::CreateFileMappingW(
170 : INVALID_HANDLE_VALUE,
171 : nullptr,
172 : PAGE_EXECUTE_READWRITE,
173 : (DWORD)(uint64_t(size) >> 32),
174 : (DWORD)(size & 0xFFFFFFFFu),
175 : nullptr);
176 :
177 : if (ASMJIT_UNLIKELY(!handle.value))
178 : return DebugUtils::errored(kErrorOutOfMemory);
179 :
180 : void* ptr[2];
181 : for (uint32_t i = 0; i < 2; i++) {
182 : MemoryFlags accessFlags = memoryFlags & ~dualMappingFilter[i];
183 : DWORD desiredAccess = desiredAccessFromMemoryFlags(accessFlags);
184 : ptr[i] = ::MapViewOfFile(handle.value, desiredAccess, 0, 0, size);
185 :
186 : if (ptr[i] == nullptr) {
187 : if (i == 0)
188 : ::UnmapViewOfFile(ptr[0]);
189 : return DebugUtils::errored(kErrorOutOfMemory);
190 : }
191 : }
192 :
193 : dm->rx = ptr[0];
194 : dm->rw = ptr[1];
195 : return kErrorOk;
196 : }
197 :
198 : Error releaseDualMapping(DualMapping* dm, size_t size) noexcept {
199 : DebugUtils::unused(size);
200 : bool failed = false;
201 :
202 : if (!::UnmapViewOfFile(dm->rx))
203 : failed = true;
204 :
205 : if (dm->rx != dm->rw && !UnmapViewOfFile(dm->rw))
206 : failed = true;
207 :
208 : if (failed)
209 : return DebugUtils::errored(kErrorInvalidArgument);
210 :
211 : dm->rx = nullptr;
212 : dm->rw = nullptr;
213 : return kErrorOk;
214 : }
215 :
216 : #endif
217 :
218 : // Virtual Memory [Posix]
219 : // ======================
220 :
221 : #if !defined(_WIN32)
222 :
223 355 : static void getVMInfo(Info& vmInfo) noexcept {
224 355 : uint32_t pageSize = uint32_t(::getpagesize());
225 :
226 355 : vmInfo.pageSize = pageSize;
227 355 : vmInfo.pageGranularity = Support::max<uint32_t>(pageSize, 65536);
228 355 : }
229 :
230 : #if !defined(SHM_ANON)
231 0 : static const char* getTmpDir() noexcept {
232 0 : const char* tmpDir = getenv("TMPDIR");
233 0 : return tmpDir ? tmpDir : "/tmp";
234 : }
235 : #endif
236 :
237 : // Translates libc errors specific to VirtualMemory mapping to `asmjit::Error`.
238 0 : static Error asmjitErrorFromErrno(int e) noexcept {
239 0 : switch (e) {
240 : case EACCES:
241 : case EAGAIN:
242 : case ENODEV:
243 : case EPERM:
244 : return kErrorInvalidState;
245 :
246 0 : case EFBIG:
247 : case ENOMEM:
248 : case EOVERFLOW:
249 0 : return kErrorOutOfMemory;
250 :
251 0 : case EMFILE:
252 : case ENFILE:
253 0 : return kErrorTooManyHandles;
254 :
255 0 : default:
256 0 : return kErrorInvalidArgument;
257 : }
258 : }
259 :
260 : // Some operating systems don't allow /dev/shm to be executable. On Linux this happens when /dev/shm is mounted with
261 : // 'noexec', which is enforced by systemd. Other operating systems like MacOS also restrict executable permissions
262 : // regarding /dev/shm, so we use a runtime detection before attempting to allocate executable memory. Sometimes we
263 : // don't need the detection as we know it would always result in `ShmStrategy::kTmpDir`.
264 : enum class ShmStrategy : uint32_t {
265 : kUnknown = 0,
266 : kDevShm = 1,
267 : kTmpDir = 2
268 : };
269 :
270 : class AnonymousMemory {
271 : public:
272 : enum FileType : uint32_t {
273 : kFileTypeNone,
274 : kFileTypeShm,
275 : kFileTypeTmp
276 : };
277 :
278 : int _fd;
279 : FileType _fileType;
280 : StringTmp<128> _tmpName;
281 :
282 : inline AnonymousMemory() noexcept
283 0 : : _fd(-1),
284 0 : _fileType(kFileTypeNone),
285 : _tmpName() {}
286 :
287 0 : inline ~AnonymousMemory() noexcept {
288 0 : unlink();
289 : close();
290 0 : }
291 :
292 0 : inline int fd() const noexcept { return _fd; }
293 :
294 0 : Error open(bool preferTmpOverDevShm) noexcept {
295 : #if defined(__linux__) && defined(__NR_memfd_create)
296 : // Linux specific 'memfd_create' - if the syscall returns `ENOSYS` it means
297 : // it's not available and we will never call it again (would be pointless).
298 :
299 : // Zero initialized, if ever changed to '1' that would mean the syscall is not
300 : // available and we must use `shm_open()` and `shm_unlink()`.
301 : static volatile uint32_t memfd_create_not_supported;
302 :
303 0 : if (!memfd_create_not_supported) {
304 0 : _fd = (int)syscall(__NR_memfd_create, "vmem", 0);
305 0 : if (ASMJIT_LIKELY(_fd >= 0))
306 : return kErrorOk;
307 :
308 0 : int e = errno;
309 0 : if (e == ENOSYS)
310 0 : memfd_create_not_supported = 1;
311 : else
312 0 : return DebugUtils::errored(asmjitErrorFromErrno(e));
313 : }
314 : #endif
315 :
316 : #if defined(SHM_ANON)
317 : // Originally FreeBSD extension, apparently works in other BSDs too.
318 : DebugUtils::unused(preferTmpOverDevShm);
319 : _fd = ::shm_open(SHM_ANON, O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR);
320 :
321 : if (ASMJIT_LIKELY(_fd >= 0))
322 : return kErrorOk;
323 : else
324 : return DebugUtils::errored(asmjitErrorFromErrno(errno));
325 : #else
326 : // POSIX API. We have to generate somehow a unique name. This is nothing cryptographic, just using a bit from
327 : // the stack address to always have a different base for different threads (as threads have their own stack)
328 : // and retries for avoiding collisions. We use `shm_open()` with flags that require creation of the file so we
329 : // never open an existing shared memory.
330 : static std::atomic<uint32_t> internalCounter;
331 : const char* kShmFormat = "/shm-id-%016llX";
332 :
333 : uint32_t kRetryCount = 100;
334 0 : uint64_t bits = ((uintptr_t)(void*)this) & 0x55555555u;
335 :
336 0 : for (uint32_t i = 0; i < kRetryCount; i++) {
337 0 : bits -= uint64_t(OSUtils::getTickCount()) * 773703683;
338 0 : bits = ((bits >> 14) ^ (bits << 6)) + uint64_t(++internalCounter) * 10619863;
339 :
340 : bool useTmp = !ASMJIT_VM_SHM_DETECT || preferTmpOverDevShm;
341 :
342 0 : if (useTmp) {
343 0 : _tmpName.assign(getTmpDir());
344 : _tmpName.appendFormat(kShmFormat, (unsigned long long)bits);
345 0 : _fd = ::open(_tmpName.data(), O_RDWR | O_CREAT | O_EXCL, 0);
346 0 : if (ASMJIT_LIKELY(_fd >= 0)) {
347 0 : _fileType = kFileTypeTmp;
348 0 : return kErrorOk;
349 : }
350 : }
351 : #if ASMJIT_VM_SHM_AVAILABLE
352 : else {
353 0 : _tmpName.assignFormat(kShmFormat, (unsigned long long)bits);
354 0 : _fd = ::shm_open(_tmpName.data(), O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR);
355 0 : if (ASMJIT_LIKELY(_fd >= 0)) {
356 0 : _fileType = kFileTypeShm;
357 0 : return kErrorOk;
358 : }
359 : }
360 : #endif
361 :
362 0 : int e = errno;
363 0 : if (e != EEXIST)
364 0 : return DebugUtils::errored(asmjitErrorFromErrno(e));
365 : }
366 :
367 : return DebugUtils::errored(kErrorFailedToOpenAnonymousMemory);
368 : #endif
369 : }
370 :
371 0 : void unlink() noexcept {
372 0 : FileType type = _fileType;
373 0 : _fileType = kFileTypeNone;
374 :
375 : #if ASMJIT_VM_SHM_AVAILABLE
376 0 : if (type == kFileTypeShm) {
377 0 : ::shm_unlink(_tmpName.data());
378 0 : return;
379 : }
380 : #endif
381 :
382 0 : if (type == kFileTypeTmp) {
383 0 : ::unlink(_tmpName.data());
384 0 : return;
385 : }
386 : }
387 :
388 : void close() noexcept {
389 0 : if (_fd >= 0) {
390 0 : ::close(_fd);
391 0 : _fd = -1;
392 : }
393 : }
394 :
395 0 : Error allocate(size_t size) noexcept {
396 : // TODO: Improve this by using `posix_fallocate()` when available.
397 0 : if (ftruncate(_fd, off_t(size)) != 0)
398 0 : return DebugUtils::errored(asmjitErrorFromErrno(errno));
399 :
400 : return kErrorOk;
401 : }
402 : };
403 :
404 : // Returns `mmap()` protection flags from \ref MemoryFlags.
405 27392 : static int mmProtFromMemoryFlags(MemoryFlags memoryFlags) noexcept {
406 : int protection = 0;
407 27392 : if (Support::test(memoryFlags, MemoryFlags::kAccessRead)) protection |= PROT_READ;
408 27392 : if (Support::test(memoryFlags, MemoryFlags::kAccessWrite)) protection |= PROT_READ | PROT_WRITE;
409 27392 : if (Support::test(memoryFlags, MemoryFlags::kAccessExecute)) protection |= PROT_READ | PROT_EXEC;
410 27392 : return protection;
411 : }
412 :
413 : #if defined(__APPLE__)
414 : // Detects whether the current process is hardened, which means that pages that have WRITE and EXECUTABLE flags cannot
415 : // be allocated without MAP_JIT flag.
416 : static inline bool hasHardenedRuntimeMacOS() noexcept {
417 : #if TARGET_OS_OSX && ASMJIT_ARCH_ARM >= 64
418 : // MacOS on AArch64 has always hardened runtime enabled.
419 : return true;
420 : #else
421 : static std::atomic<uint32_t> globalHardenedFlag;
422 :
423 : enum HardenedFlag : uint32_t {
424 : kHardenedFlagUnknown = 0,
425 : kHardenedFlagDisabled = 1,
426 : kHardenedFlagEnabled = 2
427 : };
428 :
429 : uint32_t flag = globalHardenedFlag.load();
430 : if (flag == kHardenedFlagUnknown) {
431 : size_t pageSize = ::getpagesize();
432 :
433 : void* ptr = mmap(nullptr, pageSize, PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
434 : if (ptr == MAP_FAILED) {
435 : flag = kHardenedFlagEnabled;
436 : }
437 : else {
438 : flag = kHardenedFlagDisabled;
439 : munmap(ptr, pageSize);
440 : }
441 : globalHardenedFlag.store(flag);
442 : }
443 :
444 : return flag == kHardenedFlagEnabled;
445 : #endif
446 : }
447 :
448 : static inline bool hasMapJitSupportMacOS() noexcept {
449 : #if TARGET_OS_OSX && ASMJIT_ARCH_ARM >= 64
450 : // MacOS for 64-bit AArch architecture always uses hardened runtime. Some documentation can be found here:
451 : // - https://developer.apple.com/documentation/apple_silicon/porting_just-in-time_compilers_to_apple_silicon
452 : return true;
453 : #elif TARGET_OS_OSX
454 : // MAP_JIT flag required to run unsigned JIT code is only supported by kernel version 10.14+ (Mojave) and IOS.
455 : static std::atomic<uint32_t> globalVersion;
456 :
457 : int ver = globalVersion.load();
458 : if (!ver) {
459 : struct utsname osname {};
460 : uname(&osname);
461 : ver = atoi(osname.release);
462 : globalVersion.store(ver);
463 : }
464 : return ver >= 18;
465 : #else
466 : // Assume it's available.
467 : return true;
468 : #endif
469 : }
470 : #endif // __APPLE__
471 :
472 : // Detects whether the current process is hardened, which means that pages that have WRITE and EXECUTABLE flags
473 : // cannot be normally allocated. On MacOS such allocation requires MAP_JIT flag.
474 : static inline bool hasHardenedRuntime() noexcept {
475 : #if defined(__APPLE__)
476 : return hasHardenedRuntimeMacOS();
477 : #else
478 : return false;
479 : #endif
480 : }
481 :
482 : // Detects whether MAP_JIT is available.
483 : static inline bool hasMapJitSupport() noexcept {
484 : #if defined(__APPLE__)
485 : return hasMapJitSupportMacOS();
486 : #else
487 : return false;
488 : #endif
489 : }
490 :
491 : // Returns either MAP_JIT or 0 based on `flags` and the host operating system.
492 : static inline int mmMapJitFromMemoryFlags(MemoryFlags memoryFlags) noexcept {
493 : #if defined(__APPLE__)
494 : // Always use MAP_JIT flag if user asked for it (could be used for testing on non-hardened processes) and detect
495 : // whether it must be used when the process is actually hardened (in that case it doesn't make sense to rely on
496 : // user `memoryFlags`).
497 : bool useMapJit = Support::test(memoryFlags, MemoryFlags::kMMapEnableMapJit) || hasHardenedRuntime();
498 : if (useMapJit)
499 : return hasMapJitSupport() ? int(MAP_JIT) : 0;
500 : else
501 : return 0;
502 : #else
503 : DebugUtils::unused(memoryFlags);
504 : return 0;
505 : #endif
506 : }
507 :
508 : // Returns BSD-specific `PROT_MAX()` flags.
509 : static inline int mmMaxProtFromMemoryFlags(MemoryFlags memoryFlags) noexcept {
510 : #if defined(PROT_MAX)
511 : static constexpr uint32_t kMaxProtShift = Support::ConstCTZ<uint32_t(MemoryFlags::kMMapMaxAccessRead)>::value;
512 :
513 : if (Support::test(memoryFlags, MemoryFlags::kMMapMaxAccessReadWrite | MemoryFlags::kMMapMaxAccessExecute))
514 : return PROT_MAX(mmProtFromMemoryFlags((MemoryFlags)(uint32_t(memoryFlags) >> kMaxProtShift)));
515 : else
516 : return 0;
517 : #else
518 : DebugUtils::unused(memoryFlags);
519 : return 0;
520 : #endif
521 : }
522 :
523 : #if ASMJIT_VM_SHM_DETECT
524 0 : static Error detectShmStrategy(ShmStrategy* strategyOut) noexcept {
525 : AnonymousMemory anonMem;
526 0 : Info vmInfo = info();
527 :
528 0 : ASMJIT_PROPAGATE(anonMem.open(false));
529 0 : ASMJIT_PROPAGATE(anonMem.allocate(vmInfo.pageSize));
530 :
531 0 : void* ptr = mmap(nullptr, vmInfo.pageSize, PROT_READ | PROT_EXEC, MAP_SHARED, anonMem.fd(), 0);
532 0 : if (ptr == MAP_FAILED) {
533 0 : int e = errno;
534 0 : if (e == EINVAL) {
535 0 : *strategyOut = ShmStrategy::kTmpDir;
536 0 : return kErrorOk;
537 : }
538 0 : return DebugUtils::errored(asmjitErrorFromErrno(e));
539 : }
540 : else {
541 0 : munmap(ptr, vmInfo.pageSize);
542 0 : *strategyOut = ShmStrategy::kDevShm;
543 0 : return kErrorOk;
544 : }
545 0 : }
546 : #endif
547 :
548 0 : static Error getShmStrategy(ShmStrategy* strategyOut) noexcept {
549 : #if ASMJIT_VM_SHM_DETECT
550 : // Initially don't assume anything. It has to be tested whether '/dev/shm' was mounted with 'noexec' flag or not.
551 : static std::atomic<uint32_t> globalShmStrategy;
552 :
553 0 : ShmStrategy strategy = static_cast<ShmStrategy>(globalShmStrategy.load());
554 0 : if (strategy == ShmStrategy::kUnknown) {
555 0 : ASMJIT_PROPAGATE(detectShmStrategy(&strategy));
556 0 : globalShmStrategy.store(static_cast<uint32_t>(strategy));
557 : }
558 :
559 0 : *strategyOut = strategy;
560 0 : return kErrorOk;
561 : #else
562 : *strategyOut = ShmStrategy::kTmpDir;
563 : return kErrorOk;
564 : #endif
565 : }
566 :
567 0 : static HardenedRuntimeFlags getHardenedRuntimeFlags() noexcept {
568 : HardenedRuntimeFlags hrFlags = HardenedRuntimeFlags::kNone;
569 :
570 : if (hasHardenedRuntime())
571 : hrFlags |= HardenedRuntimeFlags::kEnabled;
572 :
573 : if (hasMapJitSupport())
574 : hrFlags |= HardenedRuntimeFlags::kMapJit;
575 :
576 0 : return hrFlags;
577 : }
578 :
579 27392 : Error alloc(void** p, size_t size, MemoryFlags memoryFlags) noexcept {
580 27392 : *p = nullptr;
581 27392 : if (size == 0)
582 : return DebugUtils::errored(kErrorInvalidArgument);
583 :
584 27392 : int protection = mmProtFromMemoryFlags(memoryFlags) | mmMaxProtFromMemoryFlags(memoryFlags);
585 : int mmFlags = MAP_PRIVATE | MAP_ANONYMOUS | mmMapJitFromMemoryFlags(memoryFlags);
586 :
587 27392 : void* ptr = mmap(nullptr, size, protection, mmFlags, -1, 0);
588 27392 : if (ptr == MAP_FAILED)
589 : return DebugUtils::errored(kErrorOutOfMemory);
590 :
591 27392 : *p = ptr;
592 27392 : return kErrorOk;
593 : }
594 :
595 27392 : Error release(void* p, size_t size) noexcept {
596 27392 : if (ASMJIT_UNLIKELY(munmap(p, size) != 0))
597 : return DebugUtils::errored(kErrorInvalidArgument);
598 :
599 27392 : return kErrorOk;
600 : }
601 :
602 :
603 0 : Error protect(void* p, size_t size, MemoryFlags memoryFlags) noexcept {
604 0 : int protection = mmProtFromMemoryFlags(memoryFlags);
605 0 : if (mprotect(p, size, protection) == 0)
606 0 : return kErrorOk;
607 :
608 : return DebugUtils::errored(kErrorInvalidArgument);
609 : }
610 :
611 0 : Error allocDualMapping(DualMapping* dm, size_t size, MemoryFlags memoryFlags) noexcept {
612 0 : dm->rx = nullptr;
613 0 : dm->rw = nullptr;
614 :
615 0 : if (off_t(size) <= 0)
616 0 : return DebugUtils::errored(size == 0 ? kErrorInvalidArgument : kErrorTooLarge);
617 :
618 : bool preferTmpOverDevShm = Support::test(memoryFlags, MemoryFlags::kMappingPreferTmp);
619 0 : if (!preferTmpOverDevShm) {
620 : ShmStrategy strategy;
621 0 : ASMJIT_PROPAGATE(getShmStrategy(&strategy));
622 0 : preferTmpOverDevShm = (strategy == ShmStrategy::kTmpDir);
623 : }
624 :
625 : AnonymousMemory anonMem;
626 0 : ASMJIT_PROPAGATE(anonMem.open(preferTmpOverDevShm));
627 0 : ASMJIT_PROPAGATE(anonMem.allocate(size));
628 :
629 : void* ptr[2];
630 0 : for (uint32_t i = 0; i < 2; i++) {
631 0 : MemoryFlags accessFlags = memoryFlags & ~dualMappingFilter[i];
632 0 : int protection = mmProtFromMemoryFlags(accessFlags) | mmMaxProtFromMemoryFlags(accessFlags);
633 :
634 0 : ptr[i] = mmap(nullptr, size, protection, MAP_SHARED, anonMem.fd(), 0);
635 0 : if (ptr[i] == MAP_FAILED) {
636 : // Get the error now before `munmap()` has a chance to clobber it.
637 0 : int e = errno;
638 0 : if (i == 1)
639 0 : munmap(ptr[0], size);
640 0 : return DebugUtils::errored(asmjitErrorFromErrno(e));
641 : }
642 : }
643 :
644 0 : dm->rx = ptr[0];
645 0 : dm->rw = ptr[1];
646 0 : return kErrorOk;
647 0 : }
648 :
649 0 : Error releaseDualMapping(DualMapping* dm, size_t size) noexcept {
650 0 : Error err = release(dm->rx, size);
651 0 : if (dm->rx != dm->rw)
652 0 : err |= release(dm->rw, size);
653 :
654 0 : if (err)
655 : return DebugUtils::errored(kErrorInvalidArgument);
656 :
657 0 : dm->rx = nullptr;
658 0 : dm->rw = nullptr;
659 0 : return kErrorOk;
660 : }
661 : #endif
662 :
663 : // Virtual Memory - Flush Instruction Cache
664 : // ========================================
665 :
666 27492 : void flushInstructionCache(void* p, size_t size) noexcept {
667 : #if ASMJIT_ARCH_X86
668 : // X86/X86_64 architecture doesn't require to do anything to flush instruction cache.
669 : DebugUtils::unused(p, size);
670 : #elif defined(__APPLE__)
671 : sys_icache_invalidate(p, size);
672 : #elif defined(_WIN32)
673 : // Windows has a built-in support in `kernel32.dll`.
674 : FlushInstructionCache(GetCurrentProcess(), p, size);
675 : #elif defined(__GNUC__)
676 : char* start = static_cast<char*>(p);
677 : char* end = start + size;
678 : __builtin___clear_cache(start, end);
679 : #else
680 : #pragma message("asmjit::VirtMem::flushInstructionCache() doesn't have implementation for the target OS and compiler")
681 : DebugUtils::unused(p, size);
682 : #endif
683 27492 : }
684 :
685 : // Virtual Memory - Memory Info
686 : // ============================
687 :
688 27422 : Info info() noexcept {
689 : static std::atomic<uint32_t> vmInfoInitialized;
690 : static Info vmInfo;
691 :
692 27422 : if (!vmInfoInitialized.load()) {
693 : Info localMemInfo;
694 355 : getVMInfo(localMemInfo);
695 :
696 355 : vmInfo = localMemInfo;
697 : vmInfoInitialized.store(1u);
698 : }
699 :
700 27422 : return vmInfo;
701 : }
702 :
703 : // Virtual Memory - Hardened Runtime Info
704 : // ======================================
705 :
706 0 : HardenedRuntimeInfo hardenedRuntimeInfo() noexcept {
707 0 : return HardenedRuntimeInfo { getHardenedRuntimeFlags() };
708 : }
709 :
710 : // Virtual Memory - Project JIT Memory
711 : // ===================================
712 :
713 54984 : void protectJitMemory(ProtectJitAccess access) noexcept {
714 : #if defined(ASMJIT_HAS_PTHREAD_JIT_WRITE_PROTECT_NP)
715 : pthread_jit_write_protect_np(static_cast<uint32_t>(access));
716 : #else
717 : DebugUtils::unused(access);
718 : #endif
719 54984 : }
720 :
721 : ASMJIT_END_SUB_NAMESPACE
722 :
723 : #endif
724 :
725 : #endif // __PLUMED_HAS_ASMJIT
|
