Line data Source code
1 : #ifdef __PLUMED_HAS_ASMJIT
2 : // This file is part of AsmJit project <https://asmjit.com>
3 : //
4 : // See asmjit.h or LICENSE.md for license and copyright information
5 : // SPDX-License-Identifier: Zlib
6 :
7 : #include "api-build_p.h"
8 : #ifndef ASMJIT_NO_JIT
9 :
10 : #include "archtraits.h"
11 : #include "jitallocator.h"
12 : #include "osutils_p.h"
13 : #include "support.h"
14 : #include "virtmem.h"
15 : #include "zone.h"
16 : #include "zonelist.h"
17 : #include "zonetree.h"
18 :
19 : ASMJIT_BEGIN_NAMESPACE
20 :
21 : // JitAllocator - Constants
22 : // ========================
23 :
24 : //! Number of pools to use when `JitAllocatorOptions::kUseMultiplePools` is set.
25 : //!
26 : //! Each pool increases granularity twice to make memory management more
27 : //! efficient. Ideal number of pools appears to be 3 to 4 as it distributes
28 : //! small and large functions properly.
29 : static constexpr uint32_t kJitAllocatorMultiPoolCount = 3;
30 :
31 : //! Minimum granularity (and the default granularity for pool #0).
32 : static constexpr uint32_t kJitAllocatorBaseGranularity = 64;
33 :
34 : //! Maximum block size (32MB).
35 : static constexpr uint32_t kJitAllocatorMaxBlockSize = 1024 * 1024 * 32;
36 :
37 : // JitAllocator - Fill Pattern
38 : // ===========================
39 :
40 : static inline uint32_t JitAllocator_defaultFillPattern() noexcept {
41 : // X86 and X86_64 - 4x 'int3' instruction.
42 : if (ASMJIT_ARCH_X86)
43 : return 0xCCCCCCCCu;
44 :
45 : // Unknown...
46 : return 0u;
47 : }
48 :
49 : // JitAllocator - BitVectorRangeIterator
50 : // =====================================
51 :
52 : template<typename T, uint32_t B>
53 : class BitVectorRangeIterator {
54 : public:
55 : const T* _ptr;
56 : size_t _idx;
57 : size_t _end;
58 : T _bitWord;
59 :
60 : enum : uint32_t { kBitWordSize = Support::bitSizeOf<T>() };
61 : enum : T { kXorMask = B == 0 ? Support::allOnes<T>() : T(0) };
62 :
63 : ASMJIT_FORCE_INLINE BitVectorRangeIterator(const T* data, size_t numBitWords) noexcept {
64 : init(data, numBitWords);
65 : }
66 :
67 : ASMJIT_FORCE_INLINE BitVectorRangeIterator(const T* data, size_t numBitWords, size_t start, size_t end) noexcept {
68 : init(data, numBitWords, start, end);
69 : }
70 :
71 : ASMJIT_FORCE_INLINE void init(const T* data, size_t numBitWords) noexcept {
72 0 : init(data, numBitWords, 0, numBitWords * kBitWordSize);
73 : }
74 :
75 : ASMJIT_FORCE_INLINE void init(const T* data, size_t numBitWords, size_t start, size_t end) noexcept {
76 : ASMJIT_ASSERT(numBitWords >= (end + kBitWordSize - 1) / kBitWordSize);
77 : DebugUtils::unused(numBitWords);
78 :
79 : size_t idx = Support::alignDown(start, kBitWordSize);
80 100 : const T* ptr = data + (idx / kBitWordSize);
81 :
82 : T bitWord = 0;
83 100 : if (idx < end)
84 100 : bitWord = (*ptr ^ kXorMask) & (Support::allOnes<T>() << (start % kBitWordSize));
85 :
86 : _ptr = ptr;
87 : _idx = idx;
88 : _end = end;
89 : _bitWord = bitWord;
90 : }
91 :
92 : ASMJIT_FORCE_INLINE bool nextRange(size_t* rangeStart, size_t* rangeEnd, size_t rangeHint = std::numeric_limits<size_t>::max()) noexcept {
93 : // Skip all empty BitWords.
94 100 : while (_bitWord == 0) {
95 0 : _idx += kBitWordSize;
96 0 : if (_idx >= _end)
97 : return false;
98 0 : _bitWord = (*++_ptr) ^ kXorMask;
99 : }
100 :
101 100 : size_t i = Support::ctz(_bitWord);
102 :
103 100 : *rangeStart = _idx + i;
104 100 : _bitWord = ~(_bitWord ^ ~(Support::allOnes<T>() << i));
105 :
106 100 : if (_bitWord == 0) {
107 100 : *rangeEnd = Support::min(_idx + kBitWordSize, _end);
108 100 : while (*rangeEnd - *rangeStart < rangeHint) {
109 0 : _idx += kBitWordSize;
110 0 : if (_idx >= _end)
111 : break;
112 :
113 0 : _bitWord = (*++_ptr) ^ kXorMask;
114 0 : if (_bitWord != Support::allOnes<T>()) {
115 0 : size_t j = Support::ctz(~_bitWord);
116 0 : *rangeEnd = Support::min(_idx + j, _end);
117 0 : _bitWord = _bitWord ^ ~(Support::allOnes<T>() << j);
118 0 : break;
119 : }
120 :
121 0 : *rangeEnd = Support::min(_idx + kBitWordSize, _end);
122 : _bitWord = 0;
123 0 : continue;
124 : }
125 :
126 : return true;
127 : }
128 : else {
129 0 : size_t j = Support::ctz(_bitWord);
130 0 : *rangeEnd = Support::min(_idx + j, _end);
131 :
132 0 : _bitWord = ~(_bitWord ^ ~(Support::allOnes<T>() << j));
133 0 : return true;
134 : }
135 : }
136 : };
137 :
138 : // JitAllocator - Pool
139 : // ===================
140 :
141 : class JitAllocatorBlock;
142 :
143 : class JitAllocatorPool {
144 : public:
145 : ASMJIT_NONCOPYABLE(JitAllocatorPool)
146 :
147 : //! Double linked list of blocks.
148 : ZoneList<JitAllocatorBlock> blocks;
149 : //! Where to start looking first.
150 : JitAllocatorBlock* cursor;
151 :
152 : //! Count of blocks.
153 : uint32_t blockCount;
154 : //! Allocation granularity.
155 : uint16_t granularity;
156 : //! Log2(granularity).
157 : uint8_t granularityLog2;
158 : //! Count of empty blocks (either 0 or 1 as we won't keep more blocks empty).
159 : uint8_t emptyBlockCount;
160 :
161 : //! Number of bits reserved across all blocks.
162 : size_t totalAreaSize;
163 : //! Number of bits used across all blocks.
164 : size_t totalAreaUsed;
165 : //! Overhead of all blocks (in bytes).
166 : size_t totalOverheadBytes;
167 :
168 : inline JitAllocatorPool(uint32_t granularity) noexcept
169 27422 : : blocks(),
170 27422 : cursor(nullptr),
171 27422 : blockCount(0),
172 27422 : granularity(uint16_t(granularity)),
173 27422 : granularityLog2(uint8_t(Support::ctz(granularity))),
174 27422 : emptyBlockCount(0),
175 27422 : totalAreaSize(0),
176 27422 : totalAreaUsed(0),
177 27422 : totalOverheadBytes(0) {}
178 :
179 : inline void reset() noexcept {
180 : blocks.reset();
181 27422 : cursor = nullptr;
182 27422 : blockCount = 0;
183 27422 : totalAreaSize = 0;
184 27422 : totalAreaUsed = 0;
185 27422 : totalOverheadBytes = 0;
186 : }
187 :
188 27492 : inline size_t byteSizeFromAreaSize(uint32_t areaSize) const noexcept { return size_t(areaSize) * granularity; }
189 27492 : inline uint32_t areaSizeFromByteSize(size_t size) const noexcept { return uint32_t((size + granularity - 1) >> granularityLog2); }
190 :
191 : inline size_t bitWordCountFromAreaSize(uint32_t areaSize) const noexcept {
192 : using namespace Support;
193 100 : return alignUp<size_t>(areaSize, kBitWordSizeInBits) / kBitWordSizeInBits;
194 : }
195 : };
196 :
197 : // JitAllocator - Block
198 : // ====================
199 :
200 : class JitAllocatorBlock : public ZoneTreeNodeT<JitAllocatorBlock>,
201 : public ZoneListNode<JitAllocatorBlock> {
202 : public:
203 : ASMJIT_NONCOPYABLE(JitAllocatorBlock)
204 :
205 : enum Flags : uint32_t {
206 : //! Block is empty.
207 : kFlagEmpty = 0x00000001u,
208 : //! Block is dirty (largestUnusedArea, searchStart, searchEnd).
209 : kFlagDirty = 0x00000002u,
210 : //! Block is dual-mapped.
211 : kFlagDualMapped = 0x00000004u
212 : };
213 :
214 : //! Link to the pool that owns this block.
215 : JitAllocatorPool* _pool;
216 : //! Virtual memory mapping - either single mapping (both pointers equal) or
217 : //! dual mapping, where one pointer is Read+Execute and the second Read+Write.
218 : VirtMem::DualMapping _mapping;
219 : //! Virtual memory size (block size) [bytes].
220 : size_t _blockSize;
221 :
222 : //! Block flags.
223 : uint32_t _flags;
224 : //! Size of the whole block area (bit-vector size).
225 : uint32_t _areaSize;
226 : //! Used area (number of bits in bit-vector used).
227 : uint32_t _areaUsed;
228 : //! The largest unused continuous area in the bit-vector (or `areaSize` to initiate rescan).
229 : uint32_t _largestUnusedArea;
230 : //! Start of a search range (for unused bits).
231 : uint32_t _searchStart;
232 : //! End of a search range (for unused bits).
233 : uint32_t _searchEnd;
234 :
235 : //! Used bit-vector (0 = unused, 1 = used).
236 : Support::BitWord* _usedBitVector;
237 : //! Stop bit-vector (0 = don't care, 1 = stop).
238 : Support::BitWord* _stopBitVector;
239 :
240 : inline JitAllocatorBlock(
241 : JitAllocatorPool* pool,
242 : VirtMem::DualMapping mapping,
243 : size_t blockSize,
244 : uint32_t blockFlags,
245 : Support::BitWord* usedBitVector,
246 : Support::BitWord* stopBitVector,
247 : uint32_t areaSize) noexcept
248 27392 : : ZoneTreeNodeT(),
249 27392 : _pool(pool),
250 27392 : _mapping(mapping),
251 27392 : _blockSize(blockSize),
252 27392 : _flags(blockFlags),
253 27392 : _areaSize(areaSize),
254 27392 : _areaUsed(0),
255 27392 : _largestUnusedArea(areaSize),
256 27392 : _searchStart(0),
257 27392 : _searchEnd(areaSize),
258 27392 : _usedBitVector(usedBitVector),
259 27392 : _stopBitVector(stopBitVector) {}
260 :
261 27492 : inline JitAllocatorPool* pool() const noexcept { return _pool; }
262 :
263 55084 : inline uint8_t* rxPtr() const noexcept { return static_cast<uint8_t*>(_mapping.rx); }
264 0 : inline uint8_t* rwPtr() const noexcept { return static_cast<uint8_t*>(_mapping.rw); }
265 :
266 27592 : inline bool hasFlag(uint32_t f) const noexcept { return (_flags & f) != 0; }
267 27492 : inline void addFlags(uint32_t f) noexcept { _flags |= f; }
268 200 : inline void clearFlags(uint32_t f) noexcept { _flags &= ~f; }
269 :
270 : inline bool isDirty() const noexcept { return hasFlag(kFlagDirty); }
271 : inline void makeDirty() noexcept { addFlags(kFlagDirty); }
272 :
273 0 : inline size_t blockSize() const noexcept { return _blockSize; }
274 :
275 0 : inline uint32_t areaSize() const noexcept { return _areaSize; }
276 200 : inline uint32_t areaUsed() const noexcept { return _areaUsed; }
277 27592 : inline uint32_t areaAvailable() const noexcept { return _areaSize - _areaUsed; }
278 100 : inline uint32_t largestUnusedArea() const noexcept { return _largestUnusedArea; }
279 :
280 : inline void decreaseUsedArea(uint32_t value) noexcept {
281 : _areaUsed -= value;
282 : _pool->totalAreaUsed -= value;
283 : }
284 :
285 27492 : inline void markAllocatedArea(uint32_t allocatedAreaStart, uint32_t allocatedAreaEnd) noexcept {
286 27492 : uint32_t allocatedAreaSize = allocatedAreaEnd - allocatedAreaStart;
287 :
288 : // Mark the newly allocated space as occupied and also the sentinel.
289 27492 : Support::bitVectorFill(_usedBitVector, allocatedAreaStart, allocatedAreaSize);
290 27492 : Support::bitVectorSetBit(_stopBitVector, allocatedAreaEnd - 1, true);
291 :
292 : // Update search region and statistics.
293 27492 : _pool->totalAreaUsed += allocatedAreaSize;
294 27492 : _areaUsed += allocatedAreaSize;
295 :
296 27492 : if (areaAvailable() == 0) {
297 0 : _searchStart = _areaSize;
298 0 : _searchEnd = 0;
299 0 : _largestUnusedArea = 0;
300 : clearFlags(kFlagDirty);
301 : }
302 : else {
303 27492 : if (_searchStart == allocatedAreaStart)
304 100 : _searchStart = allocatedAreaEnd;
305 27492 : if (_searchEnd == allocatedAreaEnd)
306 0 : _searchEnd = allocatedAreaStart;
307 : addFlags(kFlagDirty);
308 : }
309 27492 : }
310 :
311 100 : inline void markReleasedArea(uint32_t releasedAreaStart, uint32_t releasedAreaEnd) noexcept {
312 100 : uint32_t releasedAreaSize = releasedAreaEnd - releasedAreaStart;
313 :
314 : // Update the search region and statistics.
315 100 : _pool->totalAreaUsed -= releasedAreaSize;
316 100 : _areaUsed -= releasedAreaSize;
317 100 : _searchStart = Support::min(_searchStart, releasedAreaStart);
318 100 : _searchEnd = Support::max(_searchEnd, releasedAreaEnd);
319 :
320 : // Unmark occupied bits and also the sentinel.
321 100 : Support::bitVectorClear(_usedBitVector, releasedAreaStart, releasedAreaSize);
322 100 : Support::bitVectorSetBit(_stopBitVector, releasedAreaEnd - 1, false);
323 :
324 100 : if (areaUsed() == 0) {
325 100 : _searchStart = 0;
326 100 : _searchEnd = _areaSize;
327 100 : _largestUnusedArea = _areaSize;
328 : addFlags(kFlagEmpty);
329 : clearFlags(kFlagDirty);
330 : }
331 : else {
332 : addFlags(kFlagDirty);
333 : }
334 100 : }
335 :
336 0 : inline void markShrunkArea(uint32_t shrunkAreaStart, uint32_t shrunkAreaEnd) noexcept {
337 0 : uint32_t shrunkAreaSize = shrunkAreaEnd - shrunkAreaStart;
338 :
339 : // Shrunk area cannot start at zero as it would mean that we have shrunk the first
340 : // block to zero bytes, which is not allowed as such block must be released instead.
341 : ASMJIT_ASSERT(shrunkAreaStart != 0);
342 : ASMJIT_ASSERT(shrunkAreaSize != 0);
343 :
344 : // Update the search region and statistics.
345 0 : _pool->totalAreaUsed -= shrunkAreaSize;
346 0 : _areaUsed -= shrunkAreaSize;
347 0 : _searchStart = Support::min(_searchStart, shrunkAreaStart);
348 0 : _searchEnd = Support::max(_searchEnd, shrunkAreaEnd);
349 :
350 : // Unmark the released space and move the sentinel.
351 0 : Support::bitVectorClear(_usedBitVector, shrunkAreaStart, shrunkAreaSize);
352 0 : Support::bitVectorSetBit(_stopBitVector, shrunkAreaEnd - 1, false);
353 0 : Support::bitVectorSetBit(_stopBitVector, shrunkAreaStart - 1, true);
354 :
355 : addFlags(kFlagDirty);
356 0 : }
357 :
358 : // RBTree default CMP uses '<' and '>' operators.
359 0 : inline bool operator<(const JitAllocatorBlock& other) const noexcept { return rxPtr() < other.rxPtr(); }
360 0 : inline bool operator>(const JitAllocatorBlock& other) const noexcept { return rxPtr() > other.rxPtr(); }
361 :
362 : // Special implementation for querying blocks by `key`, which must be in `[BlockPtr, BlockPtr + BlockSize)` range.
363 100 : inline bool operator<(const uint8_t* key) const noexcept { return rxPtr() + _blockSize <= key; }
364 100 : inline bool operator>(const uint8_t* key) const noexcept { return rxPtr() > key; }
365 : };
366 :
367 : // JitAllocator - PrivateImpl
368 : // ==========================
369 :
370 : class JitAllocatorPrivateImpl : public JitAllocator::Impl {
371 : public:
372 : //! Lock for thread safety.
373 : mutable Lock lock;
374 : //! System page size (also a minimum block size).
375 : uint32_t pageSize;
376 : //! Number of active allocations.
377 : size_t allocationCount;
378 :
379 : //! Blocks from all pools in RBTree.
380 : ZoneTree<JitAllocatorBlock> tree;
381 : //! Allocator pools.
382 : JitAllocatorPool* pools;
383 : //! Number of allocator pools.
384 : size_t poolCount;
385 :
386 : inline JitAllocatorPrivateImpl(JitAllocatorPool* pools, size_t poolCount) noexcept
387 27422 : : JitAllocator::Impl {},
388 : pageSize(0),
389 27422 : allocationCount(0),
390 27422 : pools(pools),
391 27422 : poolCount(poolCount) {}
392 27422 : inline ~JitAllocatorPrivateImpl() noexcept {}
393 : };
394 :
395 : static const JitAllocator::Impl JitAllocatorImpl_none {};
396 : static const JitAllocator::CreateParams JitAllocatorParams_none {};
397 :
398 : // JitAllocator - Utilities
399 : // ========================
400 :
401 27422 : static inline JitAllocatorPrivateImpl* JitAllocatorImpl_new(const JitAllocator::CreateParams* params) noexcept {
402 27422 : VirtMem::Info vmInfo = VirtMem::info();
403 :
404 27422 : if (!params)
405 : params = &JitAllocatorParams_none;
406 :
407 27422 : JitAllocatorOptions options = params->options;
408 27422 : uint32_t blockSize = params->blockSize;
409 27422 : uint32_t granularity = params->granularity;
410 27422 : uint32_t fillPattern = params->fillPattern;
411 :
412 : // Setup pool count to [1..3].
413 : size_t poolCount = 1;
414 27422 : if (Support::test(options, JitAllocatorOptions::kUseMultiplePools))
415 : poolCount = kJitAllocatorMultiPoolCount;;
416 :
417 : // Setup block size [64kB..256MB].
418 27422 : if (blockSize < 64 * 1024 || blockSize > 256 * 1024 * 1024 || !Support::isPowerOf2(blockSize))
419 27422 : blockSize = vmInfo.pageGranularity;
420 :
421 : // Setup granularity [64..256].
422 27422 : if (granularity < 64 || granularity > 256 || !Support::isPowerOf2(granularity))
423 : granularity = kJitAllocatorBaseGranularity;
424 :
425 : // Setup fill-pattern.
426 27422 : if (uint32_t(options & JitAllocatorOptions::kCustomFillPattern) == 0)
427 : fillPattern = JitAllocator_defaultFillPattern();
428 :
429 27422 : size_t size = sizeof(JitAllocatorPrivateImpl) + sizeof(JitAllocatorPool) * poolCount;
430 27422 : void* p = ::malloc(size);
431 27422 : if (ASMJIT_UNLIKELY(!p))
432 : return nullptr;
433 :
434 27422 : JitAllocatorPool* pools = reinterpret_cast<JitAllocatorPool*>((uint8_t*)p + sizeof(JitAllocatorPrivateImpl));
435 : JitAllocatorPrivateImpl* impl = new(p) JitAllocatorPrivateImpl(pools, poolCount);
436 :
437 27422 : impl->options = options;
438 27422 : impl->blockSize = blockSize;
439 27422 : impl->granularity = granularity;
440 27422 : impl->fillPattern = fillPattern;
441 27422 : impl->pageSize = vmInfo.pageSize;
442 :
443 54844 : for (size_t poolId = 0; poolId < poolCount; poolId++)
444 27422 : new(&pools[poolId]) JitAllocatorPool(granularity << poolId);
445 :
446 : return impl;
447 : }
448 :
449 : static inline void JitAllocatorImpl_destroy(JitAllocatorPrivateImpl* impl) noexcept {
450 : impl->~JitAllocatorPrivateImpl();
451 27422 : ::free(impl);
452 : }
453 :
454 27492 : static inline size_t JitAllocatorImpl_sizeToPoolId(const JitAllocatorPrivateImpl* impl, size_t size) noexcept {
455 27492 : size_t poolId = impl->poolCount - 1;
456 27492 : size_t granularity = size_t(impl->granularity) << poolId;
457 :
458 27492 : while (poolId) {
459 0 : if (Support::alignUp(size, granularity) == size)
460 : break;
461 0 : poolId--;
462 0 : granularity >>= 1;
463 : }
464 :
465 27492 : return poolId;
466 : }
467 :
468 : static inline size_t JitAllocatorImpl_bitVectorSizeToByteSize(uint32_t areaSize) noexcept {
469 : using Support::kBitWordSizeInBits;
470 27392 : return ((areaSize + kBitWordSizeInBits - 1u) / kBitWordSizeInBits) * sizeof(Support::BitWord);
471 : }
472 :
473 27392 : static inline size_t JitAllocatorImpl_calculateIdealBlockSize(JitAllocatorPrivateImpl* impl, JitAllocatorPool* pool, size_t allocationSize) noexcept {
474 : JitAllocatorBlock* last = pool->blocks.last();
475 27392 : size_t blockSize = last ? last->blockSize() : size_t(impl->blockSize);
476 :
477 27392 : if (blockSize < kJitAllocatorMaxBlockSize)
478 27392 : blockSize *= 2u;
479 :
480 27392 : if (allocationSize > blockSize) {
481 0 : blockSize = Support::alignUp(allocationSize, impl->blockSize);
482 0 : if (ASMJIT_UNLIKELY(blockSize < allocationSize))
483 0 : return 0; // Overflown.
484 : }
485 :
486 : return blockSize;
487 : }
488 :
489 0 : ASMJIT_FAVOR_SPEED static void JitAllocatorImpl_fillPattern(void* mem, uint32_t pattern, size_t sizeInBytes) noexcept {
490 0 : size_t n = sizeInBytes / 4u;
491 : uint32_t* p = static_cast<uint32_t*>(mem);
492 :
493 0 : for (size_t i = 0; i < n; i++)
494 0 : p[i] = pattern;
495 0 : }
496 :
497 : // Allocate a new `JitAllocatorBlock` for the given `blockSize`.
498 : //
499 : // NOTE: The block doesn't have `kFlagEmpty` flag set, because the new block
500 : // is only allocated when it's actually needed, so it would be cleared anyway.
501 27392 : static JitAllocatorBlock* JitAllocatorImpl_newBlock(JitAllocatorPrivateImpl* impl, JitAllocatorPool* pool, size_t blockSize) noexcept {
502 : using Support::BitWord;
503 : using Support::kBitWordSizeInBits;
504 :
505 27392 : uint32_t areaSize = uint32_t((blockSize + pool->granularity - 1) >> pool->granularityLog2);
506 27392 : uint32_t numBitWords = (areaSize + kBitWordSizeInBits - 1u) / kBitWordSizeInBits;
507 :
508 27392 : JitAllocatorBlock* block = static_cast<JitAllocatorBlock*>(::malloc(sizeof(JitAllocatorBlock)));
509 : BitWord* bitWords = nullptr;
510 27392 : VirtMem::DualMapping virtMem {};
511 : Error err = kErrorOutOfMemory;
512 :
513 27392 : if (block != nullptr)
514 27392 : bitWords = static_cast<BitWord*>(::malloc(size_t(numBitWords) * 2 * sizeof(BitWord)));
515 :
516 : uint32_t blockFlags = 0;
517 27392 : if (bitWords != nullptr) {
518 27392 : if (Support::test(impl->options, JitAllocatorOptions::kUseDualMapping)) {
519 0 : err = VirtMem::allocDualMapping(&virtMem, blockSize, VirtMem::MemoryFlags::kAccessRWX);
520 : blockFlags |= JitAllocatorBlock::kFlagDualMapped;
521 : }
522 : else {
523 27392 : err = VirtMem::alloc(&virtMem.rx, blockSize, VirtMem::MemoryFlags::kAccessRWX);
524 27392 : virtMem.rw = virtMem.rx;
525 : }
526 : }
527 :
528 : // Out of memory.
529 27392 : if (ASMJIT_UNLIKELY(!block || !bitWords || err != kErrorOk)) {
530 0 : if (bitWords)
531 0 : ::free(bitWords);
532 :
533 0 : if (block)
534 0 : ::free(block);
535 :
536 0 : return nullptr;
537 : }
538 :
539 : // Fill the memory if the secure mode is enabled.
540 27392 : if (Support::test(impl->options, JitAllocatorOptions::kFillUnusedMemory)) {
541 0 : VirtMem::ProtectJitReadWriteScope scope(virtMem.rw, blockSize);
542 0 : JitAllocatorImpl_fillPattern(virtMem.rw, impl->fillPattern, blockSize);
543 : }
544 :
545 27392 : memset(bitWords, 0, size_t(numBitWords) * 2 * sizeof(BitWord));
546 27392 : return new(block) JitAllocatorBlock(pool, virtMem, blockSize, blockFlags, bitWords, bitWords + numBitWords, areaSize);
547 : }
548 :
549 27392 : static void JitAllocatorImpl_deleteBlock(JitAllocatorPrivateImpl* impl, JitAllocatorBlock* block) noexcept {
550 : DebugUtils::unused(impl);
551 :
552 27392 : if (block->hasFlag(JitAllocatorBlock::kFlagDualMapped))
553 0 : VirtMem::releaseDualMapping(&block->_mapping, block->blockSize());
554 : else
555 27392 : VirtMem::release(block->rxPtr(), block->blockSize());
556 :
557 27392 : ::free(block->_usedBitVector);
558 27392 : ::free(block);
559 27392 : }
560 :
561 27392 : static void JitAllocatorImpl_insertBlock(JitAllocatorPrivateImpl* impl, JitAllocatorBlock* block) noexcept {
562 : JitAllocatorPool* pool = block->pool();
563 :
564 27392 : if (!pool->cursor)
565 27392 : pool->cursor = block;
566 :
567 : // Add to RBTree and List.
568 27392 : impl->tree.insert(block);
569 : pool->blocks.append(block);
570 :
571 : // Update statistics.
572 27392 : pool->blockCount++;
573 27392 : pool->totalAreaSize += block->areaSize();
574 27392 : pool->totalOverheadBytes += sizeof(JitAllocatorBlock) + JitAllocatorImpl_bitVectorSizeToByteSize(block->areaSize()) * 2u;
575 27392 : }
576 :
577 0 : static void JitAllocatorImpl_removeBlock(JitAllocatorPrivateImpl* impl, JitAllocatorBlock* block) noexcept {
578 : JitAllocatorPool* pool = block->pool();
579 :
580 : // Remove from RBTree and List.
581 0 : if (pool->cursor == block)
582 0 : pool->cursor = block->hasPrev() ? block->prev() : block->next();
583 :
584 0 : impl->tree.remove(block);
585 0 : pool->blocks.unlink(block);
586 :
587 : // Update statistics.
588 0 : pool->blockCount--;
589 0 : pool->totalAreaSize -= block->areaSize();
590 0 : pool->totalOverheadBytes -= sizeof(JitAllocatorBlock) + JitAllocatorImpl_bitVectorSizeToByteSize(block->areaSize()) * 2u;
591 0 : }
592 :
593 0 : static void JitAllocatorImpl_wipeOutBlock(JitAllocatorPrivateImpl* impl, JitAllocatorBlock* block) noexcept {
594 0 : if (block->hasFlag(JitAllocatorBlock::kFlagEmpty))
595 : return;
596 :
597 : JitAllocatorPool* pool = block->pool();
598 : uint32_t areaSize = block->areaSize();
599 0 : uint32_t granularity = pool->granularity;
600 : size_t numBitWords = pool->bitWordCountFromAreaSize(areaSize);
601 :
602 0 : VirtMem::protectJitMemory(VirtMem::ProtectJitAccess::kReadWrite);
603 0 : if (Support::test(impl->options, JitAllocatorOptions::kFillUnusedMemory)) {
604 : uint8_t* rwPtr = block->rwPtr();
605 0 : BitVectorRangeIterator<Support::BitWord, 0> it(block->_usedBitVector, pool->bitWordCountFromAreaSize(block->areaSize()));
606 :
607 : size_t rangeStart;
608 : size_t rangeEnd;
609 :
610 0 : while (it.nextRange(&rangeStart, &rangeEnd)) {
611 0 : uint8_t* spanPtr = rwPtr + rangeStart * granularity;
612 0 : size_t spanSize = (rangeEnd - rangeStart) * granularity;
613 :
614 0 : JitAllocatorImpl_fillPattern(spanPtr, impl->fillPattern, spanSize);
615 0 : VirtMem::flushInstructionCache(spanPtr, spanSize);
616 : }
617 : }
618 0 : VirtMem::protectJitMemory(VirtMem::ProtectJitAccess::kReadExecute);
619 :
620 0 : memset(block->_usedBitVector, 0, size_t(numBitWords) * sizeof(Support::BitWord));
621 0 : memset(block->_stopBitVector, 0, size_t(numBitWords) * sizeof(Support::BitWord));
622 :
623 0 : block->_areaUsed = 0;
624 0 : block->_largestUnusedArea = areaSize;
625 0 : block->_searchStart = 0;
626 0 : block->_searchEnd = areaSize;
627 : block->addFlags(JitAllocatorBlock::kFlagEmpty);
628 : block->clearFlags(JitAllocatorBlock::kFlagDirty);
629 : }
630 :
631 : // JitAllocator - Construction & Destruction
632 : // =========================================
633 :
634 27422 : JitAllocator::JitAllocator(const CreateParams* params) noexcept {
635 27422 : _impl = JitAllocatorImpl_new(params);
636 27422 : if (ASMJIT_UNLIKELY(!_impl))
637 0 : _impl = const_cast<JitAllocator::Impl*>(&JitAllocatorImpl_none);
638 27422 : }
639 :
640 54844 : JitAllocator::~JitAllocator() noexcept {
641 27422 : if (_impl == &JitAllocatorImpl_none)
642 : return;
643 :
644 27422 : reset(ResetPolicy::kHard);
645 27422 : JitAllocatorImpl_destroy(static_cast<JitAllocatorPrivateImpl*>(_impl));
646 27422 : }
647 :
648 : // JitAllocator - Reset
649 : // ====================
650 :
651 27422 : void JitAllocator::reset(ResetPolicy resetPolicy) noexcept {
652 27422 : if (_impl == &JitAllocatorImpl_none)
653 : return;
654 :
655 : JitAllocatorPrivateImpl* impl = static_cast<JitAllocatorPrivateImpl*>(_impl);
656 : impl->tree.reset();
657 27422 : size_t poolCount = impl->poolCount;
658 :
659 54844 : for (size_t poolId = 0; poolId < poolCount; poolId++) {
660 27422 : JitAllocatorPool& pool = impl->pools[poolId];
661 : JitAllocatorBlock* block = pool.blocks.first();
662 :
663 : JitAllocatorBlock* blockToKeep = nullptr;
664 27422 : if (resetPolicy != ResetPolicy::kHard && uint32_t(impl->options & JitAllocatorOptions::kImmediateRelease) == 0) {
665 : blockToKeep = block;
666 : block = block->next();
667 : }
668 :
669 54814 : while (block) {
670 : JitAllocatorBlock* next = block->next();
671 27392 : JitAllocatorImpl_deleteBlock(impl, block);
672 : block = next;
673 : }
674 :
675 : pool.reset();
676 :
677 27422 : if (blockToKeep) {
678 0 : blockToKeep->_listNodes[0] = nullptr;
679 0 : blockToKeep->_listNodes[1] = nullptr;
680 0 : JitAllocatorImpl_wipeOutBlock(impl, blockToKeep);
681 0 : JitAllocatorImpl_insertBlock(impl, blockToKeep);
682 0 : pool.emptyBlockCount = 1;
683 : }
684 : }
685 : }
686 :
687 : // JitAllocator - Statistics
688 : // =========================
689 :
690 0 : JitAllocator::Statistics JitAllocator::statistics() const noexcept {
691 : Statistics statistics;
692 : statistics.reset();
693 :
694 0 : if (ASMJIT_LIKELY(_impl != &JitAllocatorImpl_none)) {
695 : JitAllocatorPrivateImpl* impl = static_cast<JitAllocatorPrivateImpl*>(_impl);
696 : LockGuard guard(impl->lock);
697 :
698 0 : size_t poolCount = impl->poolCount;
699 0 : for (size_t poolId = 0; poolId < poolCount; poolId++) {
700 0 : const JitAllocatorPool& pool = impl->pools[poolId];
701 0 : statistics._blockCount += size_t(pool.blockCount);
702 0 : statistics._reservedSize += size_t(pool.totalAreaSize) * pool.granularity;
703 0 : statistics._usedSize += size_t(pool.totalAreaUsed) * pool.granularity;
704 0 : statistics._overheadSize += size_t(pool.totalOverheadBytes);
705 : }
706 :
707 0 : statistics._allocationCount = impl->allocationCount;
708 : }
709 :
710 0 : return statistics;
711 : }
712 :
713 : // JitAllocator - Alloc & Release
714 : // ==============================
715 :
716 27492 : Error JitAllocator::alloc(void** rxPtrOut, void** rwPtrOut, size_t size) noexcept {
717 27492 : if (ASMJIT_UNLIKELY(_impl == &JitAllocatorImpl_none))
718 : return DebugUtils::errored(kErrorNotInitialized);
719 :
720 : JitAllocatorPrivateImpl* impl = static_cast<JitAllocatorPrivateImpl*>(_impl);
721 : constexpr uint32_t kNoIndex = std::numeric_limits<uint32_t>::max();
722 :
723 27492 : *rxPtrOut = nullptr;
724 27492 : *rwPtrOut = nullptr;
725 :
726 : // Align to the minimum granularity by default.
727 27492 : size = Support::alignUp<size_t>(size, impl->granularity);
728 27492 : if (ASMJIT_UNLIKELY(size == 0))
729 : return DebugUtils::errored(kErrorInvalidArgument);
730 :
731 27492 : if (ASMJIT_UNLIKELY(size > std::numeric_limits<uint32_t>::max() / 2))
732 : return DebugUtils::errored(kErrorTooLarge);
733 :
734 : LockGuard guard(impl->lock);
735 27492 : JitAllocatorPool* pool = &impl->pools[JitAllocatorImpl_sizeToPoolId(impl, size)];
736 :
737 : uint32_t areaIndex = kNoIndex;
738 : uint32_t areaSize = uint32_t(pool->areaSizeFromByteSize(size));
739 :
740 : // Try to find the requested memory area in existing blocks.
741 : JitAllocatorBlock* block = pool->blocks.first();
742 27492 : if (block) {
743 : JitAllocatorBlock* initial = block;
744 : do {
745 100 : JitAllocatorBlock* next = block->hasNext() ? block->next() : pool->blocks.first();
746 100 : if (block->areaAvailable() >= areaSize) {
747 100 : if (block->isDirty() || block->largestUnusedArea() >= areaSize) {
748 100 : BitVectorRangeIterator<Support::BitWord, 0> it(block->_usedBitVector, pool->bitWordCountFromAreaSize(block->areaSize()), block->_searchStart, block->_searchEnd);
749 :
750 : size_t rangeStart = 0;
751 : size_t rangeEnd = block->areaSize();
752 :
753 : size_t searchStart = SIZE_MAX;
754 : size_t largestArea = 0;
755 :
756 100 : while (it.nextRange(&rangeStart, &rangeEnd, areaSize)) {
757 100 : size_t rangeSize = rangeEnd - rangeStart;
758 100 : if (rangeSize >= areaSize) {
759 100 : areaIndex = uint32_t(rangeStart);
760 : break;
761 : }
762 :
763 : searchStart = Support::min(searchStart, rangeStart);
764 : largestArea = Support::max(largestArea, rangeSize);
765 : }
766 :
767 100 : if (areaIndex != kNoIndex)
768 : break;
769 :
770 0 : if (searchStart != SIZE_MAX) {
771 : // Because we have iterated over the entire block, we can now mark the
772 : // largest unused area that can be used to cache the next traversal.
773 : size_t searchEnd = rangeEnd;
774 :
775 0 : block->_searchStart = uint32_t(searchStart);
776 0 : block->_searchEnd = uint32_t(searchEnd);
777 0 : block->_largestUnusedArea = uint32_t(largestArea);
778 : block->clearFlags(JitAllocatorBlock::kFlagDirty);
779 : }
780 : }
781 : }
782 :
783 : block = next;
784 0 : } while (block != initial);
785 : }
786 :
787 : // Allocate a new block if there is no region of a required width.
788 100 : if (areaIndex == kNoIndex) {
789 27392 : size_t blockSize = JitAllocatorImpl_calculateIdealBlockSize(impl, pool, size);
790 27392 : if (ASMJIT_UNLIKELY(!blockSize))
791 : return DebugUtils::errored(kErrorOutOfMemory);
792 :
793 27392 : block = JitAllocatorImpl_newBlock(impl, pool, blockSize);
794 : areaIndex = 0;
795 :
796 27392 : if (ASMJIT_UNLIKELY(!block))
797 : return DebugUtils::errored(kErrorOutOfMemory);
798 :
799 27392 : JitAllocatorImpl_insertBlock(impl, block);
800 27392 : block->_searchStart = areaSize;
801 27392 : block->_largestUnusedArea = block->areaSize() - areaSize;
802 : }
803 100 : else if (block->hasFlag(JitAllocatorBlock::kFlagEmpty)) {
804 100 : pool->emptyBlockCount--;
805 : block->clearFlags(JitAllocatorBlock::kFlagEmpty);
806 : }
807 :
808 : // Update statistics.
809 27492 : impl->allocationCount++;
810 27492 : block->markAllocatedArea(areaIndex, areaIndex + areaSize);
811 :
812 : // Return a pointer to the allocated memory.
813 : size_t offset = pool->byteSizeFromAreaSize(areaIndex);
814 : ASMJIT_ASSERT(offset <= block->blockSize() - size);
815 :
816 27492 : *rxPtrOut = block->rxPtr() + offset;
817 27492 : *rwPtrOut = block->rwPtr() + offset;
818 27492 : return kErrorOk;
819 : }
820 :
821 100 : Error JitAllocator::release(void* rxPtr) noexcept {
822 100 : if (ASMJIT_UNLIKELY(_impl == &JitAllocatorImpl_none))
823 : return DebugUtils::errored(kErrorNotInitialized);
824 :
825 100 : if (ASMJIT_UNLIKELY(!rxPtr))
826 : return DebugUtils::errored(kErrorInvalidArgument);
827 :
828 : JitAllocatorPrivateImpl* impl = static_cast<JitAllocatorPrivateImpl*>(_impl);
829 : LockGuard guard(impl->lock);
830 :
831 100 : JitAllocatorBlock* block = impl->tree.get(static_cast<uint8_t*>(rxPtr));
832 100 : if (ASMJIT_UNLIKELY(!block))
833 : return DebugUtils::errored(kErrorInvalidState);
834 :
835 : // Offset relative to the start of the block.
836 : JitAllocatorPool* pool = block->pool();
837 100 : size_t offset = (size_t)((uint8_t*)rxPtr - block->rxPtr());
838 :
839 : // The first bit representing the allocated area and its size.
840 100 : uint32_t areaIndex = uint32_t(offset >> pool->granularityLog2);
841 100 : uint32_t areaEnd = uint32_t(Support::bitVectorIndexOf(block->_stopBitVector, areaIndex, true)) + 1;
842 100 : uint32_t areaSize = areaEnd - areaIndex;
843 :
844 100 : impl->allocationCount--;
845 100 : block->markReleasedArea(areaIndex, areaEnd);
846 :
847 : // Fill the released memory if the secure mode is enabled.
848 100 : if (Support::test(impl->options, JitAllocatorOptions::kFillUnusedMemory)) {
849 0 : uint8_t* spanPtr = block->rwPtr() + areaIndex * pool->granularity;
850 0 : size_t spanSize = areaSize * pool->granularity;
851 :
852 : VirtMem::ProtectJitReadWriteScope scope(spanPtr, spanSize);
853 0 : JitAllocatorImpl_fillPattern(spanPtr, impl->fillPattern, spanSize);
854 : }
855 :
856 : // Release the whole block if it became empty.
857 100 : if (block->areaUsed() == 0) {
858 100 : if (pool->emptyBlockCount || Support::test(impl->options, JitAllocatorOptions::kImmediateRelease)) {
859 0 : JitAllocatorImpl_removeBlock(impl, block);
860 0 : JitAllocatorImpl_deleteBlock(impl, block);
861 : }
862 : else {
863 100 : pool->emptyBlockCount++;
864 : }
865 : }
866 :
867 : return kErrorOk;
868 : }
869 :
870 0 : Error JitAllocator::shrink(void* rxPtr, size_t newSize) noexcept {
871 0 : if (ASMJIT_UNLIKELY(_impl == &JitAllocatorImpl_none))
872 : return DebugUtils::errored(kErrorNotInitialized);
873 :
874 0 : if (ASMJIT_UNLIKELY(!rxPtr))
875 : return DebugUtils::errored(kErrorInvalidArgument);
876 :
877 0 : if (ASMJIT_UNLIKELY(newSize == 0))
878 0 : return release(rxPtr);
879 :
880 : JitAllocatorPrivateImpl* impl = static_cast<JitAllocatorPrivateImpl*>(_impl);
881 : LockGuard guard(impl->lock);
882 0 : JitAllocatorBlock* block = impl->tree.get(static_cast<uint8_t*>(rxPtr));
883 :
884 0 : if (ASMJIT_UNLIKELY(!block))
885 : return DebugUtils::errored(kErrorInvalidArgument);
886 :
887 : // Offset relative to the start of the block.
888 : JitAllocatorPool* pool = block->pool();
889 0 : size_t offset = (size_t)((uint8_t*)rxPtr - block->rxPtr());
890 :
891 : // The first bit representing the allocated area and its size.
892 0 : uint32_t areaStart = uint32_t(offset >> pool->granularityLog2);
893 0 : uint32_t areaEnd = uint32_t(Support::bitVectorIndexOf(block->_stopBitVector, areaStart, true)) + 1;
894 :
895 0 : uint32_t areaPrevSize = areaEnd - areaStart;
896 : uint32_t areaShrunkSize = pool->areaSizeFromByteSize(newSize);
897 :
898 0 : if (ASMJIT_UNLIKELY(areaShrunkSize > areaPrevSize))
899 : return DebugUtils::errored(kErrorInvalidState);
900 :
901 0 : uint32_t areaDiff = areaPrevSize - areaShrunkSize;
902 0 : if (areaDiff) {
903 0 : block->markShrunkArea(areaStart + areaShrunkSize, areaEnd);
904 :
905 : // Fill released memory if the secure mode is enabled.
906 0 : if (Support::test(impl->options, JitAllocatorOptions::kFillUnusedMemory))
907 0 : JitAllocatorImpl_fillPattern(block->rwPtr() + (areaStart + areaShrunkSize) * pool->granularity, fillPattern(), areaDiff * pool->granularity);
908 : }
909 :
910 : return kErrorOk;
911 : }
912 :
913 : // JitAllocator - Tests
914 : // ====================
915 :
916 : #if defined(ASMJIT_TEST)
917 : // A pseudo random number generator based on a paper by Sebastiano Vigna:
918 : // http://vigna.di.unimi.it/ftp/papers/xorshiftplus.pdf
919 : class Random {
920 : public:
921 : // Constants suggested as `23/18/5`.
922 : enum Steps : uint32_t {
923 : kStep1_SHL = 23,
924 : kStep2_SHR = 18,
925 : kStep3_SHR = 5
926 : };
927 :
928 : inline explicit Random(uint64_t seed = 0) noexcept { reset(seed); }
929 : inline Random(const Random& other) noexcept = default;
930 :
931 : inline void reset(uint64_t seed = 0) noexcept {
932 : // The number is arbitrary, it means nothing.
933 : constexpr uint64_t kZeroSeed = 0x1F0A2BE71D163FA0u;
934 :
935 : // Generate the state data by using splitmix64.
936 : for (uint32_t i = 0; i < 2; i++) {
937 : seed += 0x9E3779B97F4A7C15u;
938 : uint64_t x = seed;
939 : x = (x ^ (x >> 30)) * 0xBF58476D1CE4E5B9u;
940 : x = (x ^ (x >> 27)) * 0x94D049BB133111EBu;
941 : x = (x ^ (x >> 31));
942 : _state[i] = x != 0 ? x : kZeroSeed;
943 : }
944 : }
945 :
946 : inline uint32_t nextUInt32() noexcept {
947 : return uint32_t(nextUInt64() >> 32);
948 : }
949 :
950 : inline uint64_t nextUInt64() noexcept {
951 : uint64_t x = _state[0];
952 : uint64_t y = _state[1];
953 :
954 : x ^= x << kStep1_SHL;
955 : y ^= y >> kStep3_SHR;
956 : x ^= x >> kStep2_SHR;
957 : x ^= y;
958 :
959 : _state[0] = y;
960 : _state[1] = x;
961 : return x + y;
962 : }
963 :
964 : uint64_t _state[2];
965 : };
966 :
967 : // Helper class to verify that JitAllocator doesn't return addresses that overlap.
968 : class JitAllocatorWrapper {
969 : public:
970 : // Address to a memory region of a given size.
971 : class Range {
972 : public:
973 : inline Range(uint8_t* addr, size_t size) noexcept
974 : : addr(addr),
975 : size(size) {}
976 : uint8_t* addr;
977 : size_t size;
978 : };
979 :
980 : // Based on JitAllocator::Block, serves our purpose well...
981 : class Record : public ZoneTreeNodeT<Record>,
982 : public Range {
983 : public:
984 : inline Record(uint8_t* addr, size_t size)
985 : : ZoneTreeNodeT<Record>(),
986 : Range(addr, size) {}
987 :
988 : inline bool operator<(const Record& other) const noexcept { return addr < other.addr; }
989 : inline bool operator>(const Record& other) const noexcept { return addr > other.addr; }
990 :
991 : inline bool operator<(const uint8_t* key) const noexcept { return addr + size <= key; }
992 : inline bool operator>(const uint8_t* key) const noexcept { return addr > key; }
993 : };
994 :
995 : Zone _zone;
996 : ZoneAllocator _heap;
997 : ZoneTree<Record> _records;
998 : JitAllocator _allocator;
999 :
1000 : explicit JitAllocatorWrapper(const JitAllocator::CreateParams* params) noexcept
1001 : : _zone(1024 * 1024),
1002 : _heap(&_zone),
1003 : _allocator(params) {}
1004 :
1005 : void _insert(void* p_, size_t size) noexcept {
1006 : uint8_t* p = static_cast<uint8_t*>(p_);
1007 : uint8_t* pEnd = p + size - 1;
1008 :
1009 : Record* record;
1010 :
1011 : record = _records.get(p);
1012 : if (record)
1013 : EXPECT(record == nullptr, "Address [%p:%p] collides with a newly allocated [%p:%p]\n", record->addr, record->addr + record->size, p, p + size);
1014 :
1015 : record = _records.get(pEnd);
1016 : if (record)
1017 : EXPECT(record == nullptr, "Address [%p:%p] collides with a newly allocated [%p:%p]\n", record->addr, record->addr + record->size, p, p + size);
1018 :
1019 : record = _heap.newT<Record>(p, size);
1020 : EXPECT(record != nullptr, "Out of memory, cannot allocate 'Record'");
1021 :
1022 : _records.insert(record);
1023 : }
1024 :
1025 : void _remove(void* p) noexcept {
1026 : Record* record = _records.get(static_cast<uint8_t*>(p));
1027 : EXPECT(record != nullptr, "Address [%p] doesn't exist\n", p);
1028 :
1029 : _records.remove(record);
1030 : _heap.release(record, sizeof(Record));
1031 : }
1032 :
1033 : void* alloc(size_t size) noexcept {
1034 : void* rxPtr;
1035 : void* rwPtr;
1036 :
1037 : Error err = _allocator.alloc(&rxPtr, &rwPtr, size);
1038 : EXPECT(err == kErrorOk, "JitAllocator failed to allocate %zu bytes\n", size);
1039 :
1040 : _insert(rxPtr, size);
1041 : return rxPtr;
1042 : }
1043 :
1044 : void release(void* p) noexcept {
1045 : _remove(p);
1046 : EXPECT(_allocator.release(p) == kErrorOk, "JitAllocator failed to release '%p'\n", p);
1047 : }
1048 :
1049 : void shrink(void* p, size_t newSize) noexcept {
1050 : Record* record = _records.get(static_cast<uint8_t*>(p));
1051 : EXPECT(record != nullptr, "Address [%p] doesn't exist\n", p);
1052 :
1053 : if (!newSize)
1054 : return release(p);
1055 :
1056 : Error err = _allocator.shrink(p, newSize);
1057 : EXPECT(err == kErrorOk, "JitAllocator failed to shrink %p to %zu bytes\n", p, newSize);
1058 :
1059 : record->size = newSize;
1060 : }
1061 : };
1062 :
1063 : static void JitAllocatorTest_shuffle(void** ptrArray, size_t count, Random& prng) noexcept {
1064 : for (size_t i = 0; i < count; ++i)
1065 : std::swap(ptrArray[i], ptrArray[size_t(prng.nextUInt32() % count)]);
1066 : }
1067 :
1068 : static void JitAllocatorTest_usage(JitAllocator& allocator) noexcept {
1069 : JitAllocator::Statistics stats = allocator.statistics();
1070 : INFO(" Block Count : %9llu [Blocks]" , (unsigned long long)(stats.blockCount()));
1071 : INFO(" Reserved (VirtMem): %9llu [Bytes]" , (unsigned long long)(stats.reservedSize()));
1072 : INFO(" Used (VirtMem): %9llu [Bytes] (%.1f%%)", (unsigned long long)(stats.usedSize()), stats.usedSizeAsPercent());
1073 : INFO(" Overhead (HeapMem): %9llu [Bytes] (%.1f%%)", (unsigned long long)(stats.overheadSize()), stats.overheadSizeAsPercent());
1074 : }
1075 :
1076 : template<typename T, size_t kPatternSize, bool Bit>
1077 : static void BitVectorRangeIterator_testRandom(Random& rnd, size_t count) noexcept {
1078 : for (size_t i = 0; i < count; i++) {
1079 : T in[kPatternSize];
1080 : T out[kPatternSize];
1081 :
1082 : for (size_t j = 0; j < kPatternSize; j++) {
1083 : in[j] = T(uint64_t(rnd.nextUInt32() & 0xFFu) * 0x0101010101010101);
1084 : out[j] = Bit == 0 ? Support::allOnes<T>() : T(0);
1085 : }
1086 :
1087 : {
1088 : BitVectorRangeIterator<T, Bit> it(in, kPatternSize);
1089 : size_t rangeStart, rangeEnd;
1090 : while (it.nextRange(&rangeStart, &rangeEnd)) {
1091 : if (Bit)
1092 : Support::bitVectorFill(out, rangeStart, rangeEnd - rangeStart);
1093 : else
1094 : Support::bitVectorClear(out, rangeStart, rangeEnd - rangeStart);
1095 : }
1096 : }
1097 :
1098 : for (size_t j = 0; j < kPatternSize; j++) {
1099 : EXPECT(in[j] == out[j], "Invalid pattern detected at [%zu] (%llX != %llX)", j, (unsigned long long)in[j], (unsigned long long)out[j]);
1100 : }
1101 : }
1102 : }
1103 :
1104 : UNIT(jit_allocator) {
1105 : size_t kCount = BrokenAPI::hasArg("--quick") ? 1000 : 100000;
1106 :
1107 : struct TestParams {
1108 : const char* name;
1109 : JitAllocatorOptions options;
1110 : uint32_t blockSize;
1111 : uint32_t granularity;
1112 : };
1113 :
1114 : static TestParams testParams[] = {
1115 : { "Default", JitAllocatorOptions::kNone, 0, 0 },
1116 : { "16MB blocks", JitAllocatorOptions::kNone, 16 * 1024 * 1024, 0 },
1117 : { "256B granularity", JitAllocatorOptions::kNone, 0, 256 },
1118 : { "kUseDualMapping", JitAllocatorOptions::kUseDualMapping, 0, 0 },
1119 : { "kUseMultiplePools", JitAllocatorOptions::kUseMultiplePools, 0, 0 },
1120 : { "kFillUnusedMemory", JitAllocatorOptions::kFillUnusedMemory, 0, 0 },
1121 : { "kImmediateRelease", JitAllocatorOptions::kImmediateRelease, 0, 0 },
1122 : { "kUseDualMapping | kFillUnusedMemory", JitAllocatorOptions::kUseDualMapping | JitAllocatorOptions::kFillUnusedMemory, 0, 0 }
1123 : };
1124 :
1125 : INFO("BitVectorRangeIterator<uint32_t>");
1126 : {
1127 : Random rnd;
1128 : BitVectorRangeIterator_testRandom<uint32_t, 64, 0>(rnd, kCount);
1129 : }
1130 :
1131 : INFO("BitVectorRangeIterator<uint64_t>");
1132 : {
1133 : Random rnd;
1134 : BitVectorRangeIterator_testRandom<uint64_t, 64, 0>(rnd, kCount);
1135 : }
1136 :
1137 : for (uint32_t testId = 0; testId < ASMJIT_ARRAY_SIZE(testParams); testId++) {
1138 : INFO("JitAllocator(%s)", testParams[testId].name);
1139 :
1140 : JitAllocator::CreateParams params {};
1141 : params.options = testParams[testId].options;
1142 : params.blockSize = testParams[testId].blockSize;
1143 : params.granularity = testParams[testId].granularity;
1144 :
1145 : size_t fixedBlockSize = 256;
1146 :
1147 : JitAllocatorWrapper wrapper(¶ms);
1148 : Random prng(100);
1149 :
1150 : size_t i;
1151 :
1152 : INFO(" Memory alloc/release test - %d allocations", kCount);
1153 :
1154 : void** ptrArray = (void**)::malloc(sizeof(void*) * size_t(kCount));
1155 : EXPECT(ptrArray != nullptr,
1156 : "Couldn't allocate '%u' bytes for pointer-array", unsigned(sizeof(void*) * size_t(kCount)));
1157 :
1158 : // Random blocks tests...
1159 : INFO(" Allocating random blocks...");
1160 : for (i = 0; i < kCount; i++)
1161 : ptrArray[i] = wrapper.alloc((prng.nextUInt32() % 1024) + 8);
1162 : JitAllocatorTest_usage(wrapper._allocator);
1163 :
1164 : INFO(" Releasing all allocated blocks from the beginning...");
1165 : for (i = 0; i < kCount; i++)
1166 : wrapper.release(ptrArray[i]);
1167 : JitAllocatorTest_usage(wrapper._allocator);
1168 :
1169 : INFO(" Allocating random blocks again...", kCount);
1170 : for (i = 0; i < kCount; i++)
1171 : ptrArray[i] = wrapper.alloc((prng.nextUInt32() % 1024) + 8);
1172 : JitAllocatorTest_usage(wrapper._allocator);
1173 :
1174 : INFO(" Shuffling allocated blocks...");
1175 : JitAllocatorTest_shuffle(ptrArray, unsigned(kCount), prng);
1176 :
1177 : INFO(" Releasing 50%% of allocated blocks...");
1178 : for (i = 0; i < kCount / 2; i++)
1179 : wrapper.release(ptrArray[i]);
1180 : JitAllocatorTest_usage(wrapper._allocator);
1181 :
1182 : INFO(" Allocating 50%% more blocks again...");
1183 : for (i = 0; i < kCount / 2; i++)
1184 : ptrArray[i] = wrapper.alloc((prng.nextUInt32() % 1024) + 8);
1185 : JitAllocatorTest_usage(wrapper._allocator);
1186 :
1187 : INFO(" Releasing all allocated blocks from the end...");
1188 : for (i = 0; i < kCount; i++)
1189 : wrapper.release(ptrArray[kCount - i - 1]);
1190 : JitAllocatorTest_usage(wrapper._allocator);
1191 :
1192 : // Fixed blocks tests...
1193 : INFO(" Allocating %zuB blocks...", fixedBlockSize);
1194 : for (i = 0; i < kCount / 2; i++)
1195 : ptrArray[i] = wrapper.alloc(fixedBlockSize);
1196 : JitAllocatorTest_usage(wrapper._allocator);
1197 :
1198 : INFO(" Shrinking each %zuB block to 1 byte", fixedBlockSize);
1199 : for (i = 0; i < kCount / 2; i++)
1200 : wrapper.shrink(ptrArray[i], 1);
1201 : JitAllocatorTest_usage(wrapper._allocator);
1202 :
1203 : INFO(" Allocating more 64B blocks...", 64);
1204 : for (i = kCount / 2; i < kCount; i++)
1205 : ptrArray[i] = wrapper.alloc(64);
1206 : JitAllocatorTest_usage(wrapper._allocator);
1207 :
1208 : INFO(" Releasing all blocks from the beginning...");
1209 : for (i = 0; i < kCount; i++)
1210 : wrapper.release(ptrArray[i]);
1211 : JitAllocatorTest_usage(wrapper._allocator);
1212 :
1213 : INFO(" Allocating %zuB blocks...", fixedBlockSize);
1214 : for (i = 0; i < kCount; i++)
1215 : ptrArray[i] = wrapper.alloc(fixedBlockSize);
1216 : JitAllocatorTest_usage(wrapper._allocator);
1217 :
1218 : INFO(" Shuffling allocated blocks...");
1219 : JitAllocatorTest_shuffle(ptrArray, unsigned(kCount), prng);
1220 :
1221 : INFO(" Releasing 50%% of allocated blocks...");
1222 : for (i = 0; i < kCount / 2; i++)
1223 : wrapper.release(ptrArray[i]);
1224 : JitAllocatorTest_usage(wrapper._allocator);
1225 :
1226 : INFO(" Allocating 50%% more %zuB blocks again...", fixedBlockSize);
1227 : for (i = 0; i < kCount / 2; i++)
1228 : ptrArray[i] = wrapper.alloc(fixedBlockSize);
1229 : JitAllocatorTest_usage(wrapper._allocator);
1230 :
1231 : INFO(" Releasing all allocated blocks from the end...");
1232 : for (i = 0; i < kCount; i++)
1233 : wrapper.release(ptrArray[kCount - i - 1]);
1234 : JitAllocatorTest_usage(wrapper._allocator);
1235 :
1236 : ::free(ptrArray);
1237 : }
1238 : }
1239 : #endif
1240 :
1241 : ASMJIT_END_NAMESPACE
1242 :
1243 : #endif
1244 :
1245 : #endif // __PLUMED_HAS_ASMJIT
|
