Line data Source code
1 : #ifdef __PLUMED_HAS_ASMJIT
2 : // This file is part of AsmJit project <https://asmjit.com>
3 : //
4 : // See asmjit.h or LICENSE.md for license and copyright information
5 : // SPDX-License-Identifier: Zlib
6 :
7 : #include "api-build_p.h"
8 : #include "support.h"
9 : #include "zone.h"
10 :
11 : ASMJIT_BEGIN_NAMESPACE
12 :
13 : // Zone - Globals
14 : // ==============
15 :
16 : // Zero size block used by `Zone` that doesn't have any memory allocated. Should be allocated in read-only memory
17 : // and should never be modified.
18 : const Zone::Block Zone::_zeroBlock = { nullptr, nullptr, 0 };
19 :
20 : // Zone - Init & Reset
21 : // ===================
22 :
23 137460 : void Zone::_init(size_t blockSize, size_t blockAlignment, const Support::Temporary* temporary) noexcept {
24 : ASMJIT_ASSERT(blockSize >= kMinBlockSize);
25 : ASMJIT_ASSERT(blockSize <= kMaxBlockSize);
26 : ASMJIT_ASSERT(blockAlignment <= 64);
27 :
28 : // Just to make the compiler happy...
29 : constexpr size_t kBlockSizeMask = (Support::allOnes<size_t>() >> 4);
30 : constexpr size_t kBlockAlignmentShiftMask = 0x7u;
31 :
32 : _assignZeroBlock();
33 137460 : _blockSize = blockSize & kBlockSizeMask;
34 137460 : _isTemporary = temporary != nullptr;
35 137460 : _blockAlignmentShift = Support::ctz(blockAlignment) & kBlockAlignmentShiftMask;
36 :
37 : // Setup the first [temporary] block, if necessary.
38 137460 : if (temporary) {
39 : Block* block = temporary->data<Block>();
40 0 : block->prev = nullptr;
41 0 : block->next = nullptr;
42 :
43 : ASMJIT_ASSERT(temporary->size() >= kBlockSize);
44 0 : block->size = temporary->size() - kBlockSize;
45 :
46 0 : _assignBlock(block);
47 : }
48 137460 : }
49 :
50 274920 : void Zone::reset(ResetPolicy resetPolicy) noexcept {
51 274920 : Block* cur = _block;
52 :
53 : // Can't be altered.
54 274920 : if (cur == &_zeroBlock)
55 : return;
56 :
57 164952 : if (resetPolicy == ResetPolicy::kHard) {
58 : Block* initial = const_cast<Zone::Block*>(&_zeroBlock);
59 109968 : _ptr = initial->data();
60 109968 : _end = initial->data();
61 109968 : _block = initial;
62 :
63 : // Since cur can be in the middle of the double-linked list, we have to traverse both directions (`prev` and
64 : // `next`) separately to visit all.
65 109968 : Block* next = cur->next;
66 : do {
67 110670 : Block* prev = cur->prev;
68 :
69 : // If this is the first block and this ZoneTmp is temporary then the first block is statically allocated.
70 : // We cannot free it and it makes sense to keep it even when this is hard reset.
71 110670 : if (prev == nullptr && _isTemporary) {
72 0 : cur->prev = nullptr;
73 0 : cur->next = nullptr;
74 0 : _assignBlock(cur);
75 0 : break;
76 : }
77 :
78 110670 : ::free(cur);
79 : cur = prev;
80 110670 : } while (cur);
81 :
82 : cur = next;
83 110220 : while (cur) {
84 252 : next = cur->next;
85 252 : ::free(cur);
86 : cur = next;
87 : }
88 : }
89 : else {
90 55236 : while (cur->prev)
91 : cur = cur->prev;
92 54984 : _assignBlock(cur);
93 : }
94 : }
95 :
96 : // Zone - Alloc
97 : // ============
98 :
99 110922 : void* Zone::_alloc(size_t size, size_t alignment) noexcept {
100 110922 : Block* curBlock = _block;
101 110922 : Block* next = curBlock->next;
102 :
103 : size_t rawBlockAlignment = blockAlignment();
104 : size_t minimumAlignment = Support::max<size_t>(alignment, rawBlockAlignment);
105 :
106 : // If the `Zone` has been cleared the current block doesn't have to be the last one. Check if there is a block
107 : // that can be used instead of allocating a new one. If there is a `next` block it's completely unused, we don't
108 : // have to check for remaining bytes in that case.
109 110922 : if (next) {
110 : uint8_t* ptr = Support::alignUp(next->data(), minimumAlignment);
111 0 : uint8_t* end = Support::alignDown(next->data() + next->size, rawBlockAlignment);
112 :
113 0 : if (size <= (size_t)(end - ptr)) {
114 0 : _block = next;
115 0 : _ptr = ptr + size;
116 0 : _end = Support::alignDown(next->data() + next->size, rawBlockAlignment);
117 0 : return static_cast<void*>(ptr);
118 : }
119 : }
120 :
121 110922 : size_t blockAlignmentOverhead = alignment - Support::min<size_t>(alignment, Globals::kAllocAlignment);
122 : size_t newSize = Support::max(blockSize(), size);
123 :
124 : // Prevent arithmetic overflow.
125 110922 : if (ASMJIT_UNLIKELY(newSize > SIZE_MAX - kBlockSize - blockAlignmentOverhead))
126 : return nullptr;
127 :
128 : // Allocate new block - we add alignment overhead to `newSize`, which becomes the new block size, and we also add
129 : // `kBlockOverhead` to the allocator as it includes members of `Zone::Block` structure.
130 110922 : newSize += blockAlignmentOverhead;
131 110922 : Block* newBlock = static_cast<Block*>(::malloc(newSize + kBlockSize));
132 :
133 110922 : if (ASMJIT_UNLIKELY(!newBlock))
134 : return nullptr;
135 :
136 : // Align the pointer to `minimumAlignment` and adjust the size of this block accordingly. It's the same as using
137 : // `minimumAlignment - Support::alignUpDiff()`, just written differently.
138 : {
139 110922 : newBlock->prev = nullptr;
140 110922 : newBlock->next = nullptr;
141 110922 : newBlock->size = newSize;
142 :
143 110922 : if (curBlock != &_zeroBlock) {
144 954 : newBlock->prev = curBlock;
145 954 : curBlock->next = newBlock;
146 :
147 : // Does only happen if there is a next block, but the requested memory can't fit into it. In this case a new
148 : // buffer is allocated and inserted between the current block and the next one.
149 954 : if (next) {
150 0 : newBlock->next = next;
151 0 : next->prev = newBlock;
152 : }
153 : }
154 :
155 : uint8_t* ptr = Support::alignUp(newBlock->data(), minimumAlignment);
156 110922 : uint8_t* end = Support::alignDown(newBlock->data() + newSize, rawBlockAlignment);
157 :
158 110922 : _ptr = ptr + size;
159 110922 : _end = end;
160 110922 : _block = newBlock;
161 :
162 : ASMJIT_ASSERT(_ptr <= _end);
163 110922 : return static_cast<void*>(ptr);
164 : }
165 : }
166 :
167 273777 : void* Zone::allocZeroed(size_t size, size_t alignment) noexcept {
168 273777 : void* p = alloc(size, alignment);
169 273777 : if (ASMJIT_UNLIKELY(!p))
170 : return p;
171 273777 : return memset(p, 0, size);
172 : }
173 :
174 54984 : void* Zone::dup(const void* data, size_t size, bool nullTerminate) noexcept {
175 54984 : if (ASMJIT_UNLIKELY(!data || !size))
176 : return nullptr;
177 :
178 : ASMJIT_ASSERT(size != SIZE_MAX);
179 54984 : uint8_t* m = allocT<uint8_t>(size + nullTerminate);
180 54984 : if (ASMJIT_UNLIKELY(!m)) return nullptr;
181 :
182 : memcpy(m, data, size);
183 54984 : if (nullTerminate) m[size] = '\0';
184 :
185 : return static_cast<void*>(m);
186 : }
187 :
188 0 : char* Zone::sformat(const char* fmt, ...) noexcept {
189 0 : if (ASMJIT_UNLIKELY(!fmt))
190 : return nullptr;
191 :
192 : char buf[512];
193 : size_t size;
194 : va_list ap;
195 :
196 0 : va_start(ap, fmt);
197 0 : size = unsigned(vsnprintf(buf, ASMJIT_ARRAY_SIZE(buf) - 1, fmt, ap));
198 0 : va_end(ap);
199 :
200 0 : buf[size++] = 0;
201 0 : return static_cast<char*>(dup(buf, size));
202 : }
203 :
204 : // ZoneAllocator - Utilities
205 : // =========================
206 :
207 : #if defined(ASMJIT_BUILD_DEBUG)
208 : static bool ZoneAllocator_hasDynamicBlock(ZoneAllocator* self, ZoneAllocator::DynamicBlock* block) noexcept {
209 : ZoneAllocator::DynamicBlock* cur = self->_dynamicBlocks;
210 : while (cur) {
211 : if (cur == block)
212 : return true;
213 : cur = cur->next;
214 : }
215 : return false;
216 : }
217 : #endif
218 :
219 : // ZoneAllocator - Init & Reset
220 : // ============================
221 :
222 164952 : void ZoneAllocator::reset(Zone* zone) noexcept {
223 : // Free dynamic blocks.
224 164952 : DynamicBlock* block = _dynamicBlocks;
225 209131 : while (block) {
226 44179 : DynamicBlock* next = block->next;
227 44179 : ::free(block);
228 : block = next;
229 : }
230 :
231 : // Zero the entire class and initialize to the given `zone`.
232 : memset(this, 0, sizeof(*this));
233 164952 : _zone = zone;
234 164952 : }
235 :
236 : // asmjit::ZoneAllocator - Alloc & Release
237 : // =======================================
238 :
239 2796496 : void* ZoneAllocator::_alloc(size_t size, size_t& allocatedSize) noexcept {
240 : ASMJIT_ASSERT(isInitialized());
241 :
242 : // Use the memory pool only if the requested block has a reasonable size.
243 : uint32_t slot;
244 2796496 : if (_getSlotIndex(size, slot, allocatedSize)) {
245 : // Slot reuse.
246 2738298 : uint8_t* p = reinterpret_cast<uint8_t*>(_slots[slot]);
247 2738298 : size = allocatedSize;
248 :
249 2738298 : if (p) {
250 250701 : _slots[slot] = reinterpret_cast<Slot*>(p)->next;
251 250701 : return p;
252 : }
253 :
254 2487597 : _zone->align(kBlockAlignment);
255 2487597 : p = _zone->ptr();
256 2487597 : size_t remain = (size_t)(_zone->end() - p);
257 :
258 2487597 : if (ASMJIT_LIKELY(remain >= size)) {
259 2432277 : _zone->setPtr(p + size);
260 2432277 : return p;
261 : }
262 : else {
263 : // Distribute the remaining memory to suitable slots, if possible.
264 55320 : if (remain >= kLoGranularity) {
265 : do {
266 : size_t distSize = Support::min<size_t>(remain, kLoMaxSize);
267 123 : uint32_t distSlot = uint32_t((distSize - kLoGranularity) / kLoGranularity);
268 : ASMJIT_ASSERT(distSlot < kLoCount);
269 :
270 123 : reinterpret_cast<Slot*>(p)->next = _slots[distSlot];
271 123 : _slots[distSlot] = reinterpret_cast<Slot*>(p);
272 :
273 123 : p += distSize;
274 123 : remain -= distSize;
275 123 : } while (remain >= kLoGranularity);
276 123 : _zone->setPtr(p);
277 : }
278 :
279 55320 : p = static_cast<uint8_t*>(_zone->_alloc(size, kBlockAlignment));
280 55320 : if (ASMJIT_UNLIKELY(!p)) {
281 0 : allocatedSize = 0;
282 0 : return nullptr;
283 : }
284 :
285 : return p;
286 : }
287 : }
288 : else {
289 : // Allocate a dynamic block.
290 : size_t kBlockOverhead = sizeof(DynamicBlock) + sizeof(DynamicBlock*) + kBlockAlignment;
291 :
292 : // Handle a possible overflow.
293 58198 : if (ASMJIT_UNLIKELY(kBlockOverhead >= SIZE_MAX - size))
294 : return nullptr;
295 :
296 58198 : void* p = ::malloc(size + kBlockOverhead);
297 58198 : if (ASMJIT_UNLIKELY(!p)) {
298 0 : allocatedSize = 0;
299 0 : return nullptr;
300 : }
301 :
302 : // Link as first in `_dynamicBlocks` double-linked list.
303 : DynamicBlock* block = static_cast<DynamicBlock*>(p);
304 58198 : DynamicBlock* next = _dynamicBlocks;
305 :
306 58198 : if (next)
307 30288 : next->prev = block;
308 :
309 58198 : block->prev = nullptr;
310 58198 : block->next = next;
311 58198 : _dynamicBlocks = block;
312 :
313 : // Align the pointer to the guaranteed alignment and store `DynamicBlock`
314 : // at the beginning of the memory block, so `_releaseDynamic()` can find it.
315 58198 : p = Support::alignUp(static_cast<uint8_t*>(p) + sizeof(DynamicBlock) + sizeof(DynamicBlock*), kBlockAlignment);
316 58198 : reinterpret_cast<DynamicBlock**>(p)[-1] = block;
317 :
318 58198 : allocatedSize = size;
319 58198 : return p;
320 : }
321 : }
322 :
323 82476 : void* ZoneAllocator::_allocZeroed(size_t size, size_t& allocatedSize) noexcept {
324 : ASMJIT_ASSERT(isInitialized());
325 :
326 82476 : void* p = _alloc(size, allocatedSize);
327 82476 : if (ASMJIT_UNLIKELY(!p)) return p;
328 82476 : return memset(p, 0, allocatedSize);
329 : }
330 :
331 14019 : void ZoneAllocator::_releaseDynamic(void* p, size_t size) noexcept {
332 : DebugUtils::unused(size);
333 : ASMJIT_ASSERT(isInitialized());
334 :
335 : // Pointer to `DynamicBlock` is stored at [-1].
336 14019 : DynamicBlock* block = reinterpret_cast<DynamicBlock**>(p)[-1];
337 : ASMJIT_ASSERT(ZoneAllocator_hasDynamicBlock(this, block));
338 :
339 : // Unlink and free.
340 14019 : DynamicBlock* prev = block->prev;
341 14019 : DynamicBlock* next = block->next;
342 :
343 14019 : if (prev)
344 13920 : prev->next = next;
345 : else
346 99 : _dynamicBlocks = next;
347 :
348 14019 : if (next)
349 13899 : next->prev = prev;
350 :
351 14019 : ::free(block);
352 14019 : }
353 :
354 : ASMJIT_END_NAMESPACE
355 :
356 : #endif // __PLUMED_HAS_ASMJIT
|
