Line data Source code
1 : #ifdef __PLUMED_HAS_ASMJIT
2 : // This file is part of AsmJit project <https://asmjit.com>
3 : //
4 : // See asmjit.h or LICENSE.md for license and copyright information
5 : // SPDX-License-Identifier: Zlib
6 :
7 : #include "api-build_p.h"
8 : #include "assembler.h"
9 : #include "codewriter_p.h"
10 : #include "logger.h"
11 : #include "support.h"
12 :
13 : #include <algorithm>
14 : #include <tuple>
15 :
16 : ASMJIT_BEGIN_NAMESPACE
17 :
18 : // Globals
19 : // =======
20 :
21 : static const char CodeHolder_addrTabName[] = ".addrtab";
22 :
23 : //! Encode MOD byte.
24 : static inline uint32_t x86EncodeMod(uint32_t m, uint32_t o, uint32_t rm) noexcept {
25 : return (m << 6) | (o << 3) | rm;
26 : }
27 :
28 : // LabelLinkIterator
29 : // =================
30 :
31 : class LabelLinkIterator {
32 : public:
33 54984 : inline LabelLinkIterator(LabelEntry* le) noexcept { reset(le); }
34 :
35 : inline explicit operator bool() const noexcept { return isValid(); }
36 : inline bool isValid() const noexcept { return _link != nullptr; }
37 :
38 : inline LabelLink* link() const noexcept { return _link; }
39 : inline LabelLink* operator->() const noexcept { return _link; }
40 :
41 : inline void reset(LabelEntry* le) noexcept {
42 54984 : _pPrev = &le->_links;
43 54984 : _link = *_pPrev;
44 : }
45 :
46 : inline void next() noexcept {
47 0 : _pPrev = &_link->next;
48 0 : _link = *_pPrev;
49 0 : }
50 :
51 : inline void resolveAndNext(CodeHolder* code) noexcept {
52 : LabelLink* linkToDelete = _link;
53 :
54 0 : _link = _link->next;
55 0 : *_pPrev = _link;
56 :
57 0 : code->_unresolvedLinkCount--;
58 : code->_allocator.release(linkToDelete, sizeof(LabelLink));
59 0 : }
60 :
61 : LabelLink** _pPrev;
62 : LabelLink* _link;
63 : };
64 :
65 : // CodeHolder - Utilities
66 : // ======================
67 :
68 27492 : static void CodeHolder_resetInternal(CodeHolder* self, ResetPolicy resetPolicy) noexcept {
69 : uint32_t i;
70 : const ZoneVector<BaseEmitter*>& emitters = self->emitters();
71 :
72 : i = emitters.size();
73 27492 : while (i)
74 0 : self->detach(emitters[--i]);
75 :
76 : // Reset everything into its construction state.
77 : self->_environment.reset();
78 27492 : self->_baseAddress = Globals::kNoBaseAddress;
79 27492 : self->_logger = nullptr;
80 27492 : self->_errorHandler = nullptr;
81 :
82 : // Reset all sections.
83 : uint32_t numSections = self->_sections.size();
84 54984 : for (i = 0; i < numSections; i++) {
85 27492 : Section* section = self->_sections[i];
86 27492 : if (section->_buffer.data() && !section->_buffer.isExternal())
87 27492 : ::free(section->_buffer._data);
88 27492 : section->_buffer._data = nullptr;
89 27492 : section->_buffer._capacity = 0;
90 : }
91 :
92 : // Reset zone allocator and all containers using it.
93 : ZoneAllocator* allocator = self->allocator();
94 :
95 : self->_emitters.reset();
96 : self->_namedLabels.reset();
97 : self->_relocations.reset();
98 : self->_labelEntries.reset();
99 : self->_sections.reset();
100 : self->_sectionsByOrder.reset();
101 :
102 27492 : self->_unresolvedLinkCount = 0;
103 27492 : self->_addressTableSection = nullptr;
104 : self->_addressTableEntries.reset();
105 :
106 27492 : allocator->reset(&self->_zone);
107 27492 : self->_zone.reset(resetPolicy);
108 27492 : }
109 :
110 0 : static void CodeHolder_onSettingsUpdated(CodeHolder* self) noexcept {
111 : // Notify all attached emitters about a settings update.
112 0 : for (BaseEmitter* emitter : self->emitters()) {
113 0 : emitter->onSettingsUpdated();
114 : }
115 0 : }
116 :
117 : // CodeHolder - Construction & Destruction
118 : // =======================================
119 :
120 27492 : CodeHolder::CodeHolder(const Support::Temporary* temporary) noexcept
121 : : _environment(),
122 27492 : _baseAddress(Globals::kNoBaseAddress),
123 27492 : _logger(nullptr),
124 27492 : _errorHandler(nullptr),
125 27492 : _zone(16384 - Zone::kBlockOverhead, 1, temporary),
126 27492 : _allocator(&_zone),
127 27492 : _unresolvedLinkCount(0),
128 27492 : _addressTableSection(nullptr) {}
129 :
130 27492 : CodeHolder::~CodeHolder() noexcept {
131 27492 : CodeHolder_resetInternal(this, ResetPolicy::kHard);
132 27492 : }
133 :
134 : // CodeHolder - Init & Reset
135 : // =========================
136 :
137 : inline void CodeHolder_setSectionDefaultName(
138 : Section* section,
139 : char c0 = 0, char c1 = 0, char c2 = 0, char c3 = 0,
140 : char c4 = 0, char c5 = 0, char c6 = 0, char c7 = 0) noexcept {
141 :
142 27492 : section->_name.u32[0] = Support::bytepack32_4x8(uint8_t(c0), uint8_t(c1), uint8_t(c2), uint8_t(c3));
143 27492 : section->_name.u32[1] = Support::bytepack32_4x8(uint8_t(c4), uint8_t(c5), uint8_t(c6), uint8_t(c7));
144 : }
145 :
146 27492 : Error CodeHolder::init(const Environment& environment, uint64_t baseAddress) noexcept {
147 : // Cannot reinitialize if it's locked or there is one or more emitter attached.
148 27492 : if (isInitialized())
149 : return DebugUtils::errored(kErrorAlreadyInitialized);
150 :
151 : // If we are just initializing there should be no emitters attached.
152 : ASMJIT_ASSERT(_emitters.empty());
153 :
154 : // Create a default section and insert it to the `_sections` array.
155 27492 : Error err = _sections.willGrow(&_allocator) |
156 27492 : _sectionsByOrder.willGrow(&_allocator);
157 27492 : if (err == kErrorOk) {
158 27492 : Section* section = _allocator.allocZeroedT<Section>();
159 27492 : if (ASMJIT_LIKELY(section)) {
160 27492 : section->_flags = SectionFlags::kExecutable | SectionFlags::kReadOnly;
161 : CodeHolder_setSectionDefaultName(section, '.', 't', 'e', 'x', 't');
162 : _sections.appendUnsafe(section);
163 : _sectionsByOrder.appendUnsafe(section);
164 : }
165 : else {
166 : err = DebugUtils::errored(kErrorOutOfMemory);
167 : }
168 : }
169 :
170 27492 : if (ASMJIT_UNLIKELY(err)) {
171 0 : _zone.reset();
172 0 : return err;
173 : }
174 : else {
175 27492 : _environment = environment;
176 27492 : _baseAddress = baseAddress;
177 27492 : return kErrorOk;
178 : }
179 : }
180 :
181 0 : void CodeHolder::reset(ResetPolicy resetPolicy) noexcept {
182 0 : CodeHolder_resetInternal(this, resetPolicy);
183 0 : }
184 :
185 : // CodeHolder - Attach / Detach
186 : // ============================
187 :
188 54984 : Error CodeHolder::attach(BaseEmitter* emitter) noexcept {
189 : // Catch a possible misuse of the API.
190 54984 : if (ASMJIT_UNLIKELY(!emitter))
191 : return DebugUtils::errored(kErrorInvalidArgument);
192 :
193 : // Invalid emitter, this should not be possible.
194 : EmitterType type = emitter->emitterType();
195 54984 : if (ASMJIT_UNLIKELY(type == EmitterType::kNone || uint32_t(type) > uint32_t(EmitterType::kMaxValue)))
196 : return DebugUtils::errored(kErrorInvalidState);
197 :
198 54984 : uint64_t archMask = emitter->_archMask;
199 54984 : if (ASMJIT_UNLIKELY(!(archMask & (uint64_t(1) << uint32_t(arch())))))
200 : return DebugUtils::errored(kErrorInvalidArch);
201 :
202 : // This is suspicious, but don't fail if `emitter` is already attached
203 : // to this code holder. This is not error, but it's not recommended.
204 54984 : if (emitter->_code != nullptr) {
205 0 : if (emitter->_code == this)
206 0 : return kErrorOk;
207 : return DebugUtils::errored(kErrorInvalidState);
208 : }
209 :
210 : // Reserve the space now as we cannot fail after `onAttach()` succeeded.
211 54984 : ASMJIT_PROPAGATE(_emitters.willGrow(&_allocator, 1));
212 54984 : ASMJIT_PROPAGATE(emitter->onAttach(this));
213 :
214 : // Connect CodeHolder <-> BaseEmitter.
215 : ASMJIT_ASSERT(emitter->_code == this);
216 : _emitters.appendUnsafe(emitter);
217 :
218 54984 : return kErrorOk;
219 : }
220 :
221 54984 : Error CodeHolder::detach(BaseEmitter* emitter) noexcept {
222 54984 : if (ASMJIT_UNLIKELY(!emitter))
223 : return DebugUtils::errored(kErrorInvalidArgument);
224 :
225 54984 : if (ASMJIT_UNLIKELY(emitter->_code != this))
226 : return DebugUtils::errored(kErrorInvalidState);
227 :
228 : // NOTE: We always detach if we were asked to, if error happens during
229 : // `emitter->onDetach()` we just propagate it, but the BaseEmitter will
230 : // be detached.
231 : Error err = kErrorOk;
232 54984 : if (!emitter->isDestroyed())
233 0 : err = emitter->onDetach(this);
234 :
235 : // Disconnect CodeHolder <-> BaseEmitter.
236 : uint32_t index = _emitters.indexOf(emitter);
237 : ASMJIT_ASSERT(index != Globals::kNotFound);
238 :
239 54984 : _emitters.removeAt(index);
240 54984 : emitter->_code = nullptr;
241 :
242 54984 : return err;
243 : }
244 :
245 : // CodeHolder - Logging
246 : // ====================
247 :
248 0 : void CodeHolder::setLogger(Logger* logger) noexcept {
249 : #ifndef ASMJIT_NO_LOGGING
250 0 : _logger = logger;
251 0 : CodeHolder_onSettingsUpdated(this);
252 : #else
253 : DebugUtils::unused(logger);
254 : #endif
255 0 : }
256 :
257 : // CodeHolder - Error Handling
258 : // ===========================
259 :
260 0 : void CodeHolder::setErrorHandler(ErrorHandler* errorHandler) noexcept {
261 0 : _errorHandler = errorHandler;
262 0 : CodeHolder_onSettingsUpdated(this);
263 0 : }
264 :
265 : // CodeHolder - Code Buffer
266 : // ========================
267 :
268 27492 : static Error CodeHolder_reserveInternal(CodeHolder* self, CodeBuffer* cb, size_t n) noexcept {
269 27492 : uint8_t* oldData = cb->_data;
270 : uint8_t* newData;
271 :
272 27492 : if (oldData && !cb->isExternal())
273 0 : newData = static_cast<uint8_t*>(::realloc(oldData, n));
274 : else
275 27492 : newData = static_cast<uint8_t*>(::malloc(n));
276 :
277 27492 : if (ASMJIT_UNLIKELY(!newData))
278 : return DebugUtils::errored(kErrorOutOfMemory);
279 :
280 27492 : cb->_data = newData;
281 27492 : cb->_capacity = n;
282 :
283 : // Update pointers used by assemblers, if attached.
284 82476 : for (BaseEmitter* emitter : self->emitters()) {
285 54984 : if (emitter->isAssembler()) {
286 : BaseAssembler* a = static_cast<BaseAssembler*>(emitter);
287 27492 : if (&a->_section->_buffer == cb) {
288 : size_t offset = a->offset();
289 :
290 27492 : a->_bufferData = newData;
291 27492 : a->_bufferEnd = newData + n;
292 27492 : a->_bufferPtr = newData + offset;
293 : }
294 : }
295 : }
296 :
297 : return kErrorOk;
298 : }
299 :
300 27492 : Error CodeHolder::growBuffer(CodeBuffer* cb, size_t n) noexcept {
301 : // The size of the section must be valid.
302 : size_t size = cb->size();
303 27492 : if (ASMJIT_UNLIKELY(n > std::numeric_limits<uintptr_t>::max() - size))
304 : return DebugUtils::errored(kErrorOutOfMemory);
305 :
306 : // We can now check if growing the buffer is really necessary. It's unlikely
307 : // that this function is called while there is still room for `n` bytes.
308 : size_t capacity = cb->capacity();
309 27492 : size_t required = cb->size() + n;
310 27492 : if (ASMJIT_UNLIKELY(required <= capacity))
311 : return kErrorOk;
312 :
313 27492 : if (cb->isFixed())
314 : return DebugUtils::errored(kErrorTooLarge);
315 :
316 : size_t kInitialCapacity = 8096;
317 27492 : if (capacity < kInitialCapacity)
318 : capacity = kInitialCapacity;
319 : else
320 0 : capacity += Globals::kAllocOverhead;
321 :
322 : do {
323 : size_t old = capacity;
324 27492 : if (capacity < Globals::kGrowThreshold)
325 27492 : capacity *= 2;
326 : else
327 0 : capacity += Globals::kGrowThreshold;
328 :
329 : // Overflow.
330 27492 : if (ASMJIT_UNLIKELY(old > capacity))
331 : return DebugUtils::errored(kErrorOutOfMemory);
332 27492 : } while (capacity - Globals::kAllocOverhead < required);
333 :
334 27492 : return CodeHolder_reserveInternal(this, cb, capacity - Globals::kAllocOverhead);
335 : }
336 :
337 0 : Error CodeHolder::reserveBuffer(CodeBuffer* cb, size_t n) noexcept {
338 : size_t capacity = cb->capacity();
339 :
340 0 : if (n <= capacity)
341 : return kErrorOk;
342 :
343 0 : if (cb->isFixed())
344 : return DebugUtils::errored(kErrorTooLarge);
345 :
346 0 : return CodeHolder_reserveInternal(this, cb, n);
347 : }
348 :
349 : // CodeHolder - Sections
350 : // =====================
351 :
352 0 : Error CodeHolder::newSection(Section** sectionOut, const char* name, size_t nameSize, SectionFlags flags, uint32_t alignment, int32_t order) noexcept {
353 0 : *sectionOut = nullptr;
354 :
355 0 : if (nameSize == SIZE_MAX)
356 0 : nameSize = strlen(name);
357 :
358 : if (alignment == 0)
359 : alignment = 1;
360 :
361 0 : if (ASMJIT_UNLIKELY(!Support::isPowerOf2(alignment)))
362 : return DebugUtils::errored(kErrorInvalidArgument);
363 :
364 0 : if (ASMJIT_UNLIKELY(nameSize > Globals::kMaxSectionNameSize))
365 : return DebugUtils::errored(kErrorInvalidSectionName);
366 :
367 : uint32_t sectionId = _sections.size();
368 0 : if (ASMJIT_UNLIKELY(sectionId == Globals::kInvalidId))
369 : return DebugUtils::errored(kErrorTooManySections);
370 :
371 0 : ASMJIT_PROPAGATE(_sections.willGrow(&_allocator));
372 0 : ASMJIT_PROPAGATE(_sectionsByOrder.willGrow(&_allocator));
373 :
374 0 : Section* section = _allocator.allocZeroedT<Section>();
375 0 : if (ASMJIT_UNLIKELY(!section))
376 : return DebugUtils::errored(kErrorOutOfMemory);
377 :
378 0 : section->_id = sectionId;
379 0 : section->_flags = flags;
380 0 : section->_alignment = alignment;
381 0 : section->_order = order;
382 0 : memcpy(section->_name.str, name, nameSize);
383 :
384 0 : Section** insertPosition = std::lower_bound(_sectionsByOrder.begin(), _sectionsByOrder.end(), section, [](const Section* a, const Section* b) {
385 0 : return std::make_tuple(a->order(), a->id()) < std::make_tuple(b->order(), b->id());
386 : });
387 :
388 : _sections.appendUnsafe(section);
389 0 : _sectionsByOrder.insertUnsafe((size_t)(insertPosition - _sectionsByOrder.data()), section);
390 :
391 0 : *sectionOut = section;
392 0 : return kErrorOk;
393 : }
394 :
395 0 : Section* CodeHolder::sectionByName(const char* name, size_t nameSize) const noexcept {
396 0 : if (nameSize == SIZE_MAX)
397 0 : nameSize = strlen(name);
398 :
399 : // This could be also put in a hash-table similarly like we do with labels,
400 : // however it's questionable as the number of sections should be pretty low
401 : // in general. Create an issue if this becomes a problem.
402 0 : if (nameSize <= Globals::kMaxSectionNameSize) {
403 0 : for (Section* section : _sections)
404 0 : if (memcmp(section->_name.str, name, nameSize) == 0 && section->_name.str[nameSize] == '\0')
405 : return section;
406 : }
407 :
408 : return nullptr;
409 : }
410 :
411 0 : Section* CodeHolder::ensureAddressTableSection() noexcept {
412 0 : if (_addressTableSection)
413 : return _addressTableSection;
414 :
415 0 : newSection(&_addressTableSection,
416 : CodeHolder_addrTabName,
417 : sizeof(CodeHolder_addrTabName) - 1,
418 : SectionFlags::kNone,
419 : _environment.registerSize(),
420 : std::numeric_limits<int32_t>::max());
421 0 : return _addressTableSection;
422 : }
423 :
424 0 : Error CodeHolder::addAddressToAddressTable(uint64_t address) noexcept {
425 0 : AddressTableEntry* entry = _addressTableEntries.get(address);
426 0 : if (entry)
427 : return kErrorOk;
428 :
429 0 : Section* section = ensureAddressTableSection();
430 0 : if (ASMJIT_UNLIKELY(!section))
431 : return DebugUtils::errored(kErrorOutOfMemory);
432 :
433 0 : entry = _zone.newT<AddressTableEntry>(address);
434 0 : if (ASMJIT_UNLIKELY(!entry))
435 : return DebugUtils::errored(kErrorOutOfMemory);
436 :
437 0 : _addressTableEntries.insert(entry);
438 0 : section->_virtualSize += _environment.registerSize();
439 :
440 0 : return kErrorOk;
441 : }
442 :
443 : // CodeHolder - Labels & Symbols
444 : // =============================
445 :
446 : //! Only used to lookup a label from `_namedLabels`.
447 : class LabelByName {
448 : public:
449 : inline LabelByName(const char* key, size_t keySize, uint32_t hashCode, uint32_t parentId) noexcept
450 0 : : _key(key),
451 0 : _keySize(uint32_t(keySize)),
452 0 : _hashCode(hashCode),
453 0 : _parentId(parentId) {}
454 :
455 0 : inline uint32_t hashCode() const noexcept { return _hashCode; }
456 :
457 0 : inline bool matches(const LabelEntry* entry) const noexcept {
458 0 : return entry->nameSize() == _keySize &&
459 0 : entry->parentId() == _parentId &&
460 0 : ::memcmp(entry->name(), _key, _keySize) == 0;
461 : }
462 :
463 : const char* _key;
464 : uint32_t _keySize;
465 : uint32_t _hashCode;
466 : uint32_t _parentId;
467 : };
468 :
469 : // Returns a hash of `name` and fixes `nameSize` if it's `SIZE_MAX`.
470 0 : static uint32_t CodeHolder_hashNameAndGetSize(const char* name, size_t& nameSize) noexcept {
471 : uint32_t hashCode = 0;
472 0 : if (nameSize == SIZE_MAX) {
473 : size_t i = 0;
474 : for (;;) {
475 0 : uint8_t c = uint8_t(name[i]);
476 0 : if (!c) break;
477 0 : hashCode = Support::hashRound(hashCode, c);
478 0 : i++;
479 0 : }
480 0 : nameSize = i;
481 : }
482 : else {
483 0 : for (size_t i = 0; i < nameSize; i++) {
484 0 : uint8_t c = uint8_t(name[i]);
485 0 : if (ASMJIT_UNLIKELY(!c)) return DebugUtils::errored(kErrorInvalidLabelName);
486 0 : hashCode = Support::hashRound(hashCode, c);
487 : }
488 : }
489 : return hashCode;
490 : }
491 :
492 0 : LabelLink* CodeHolder::newLabelLink(LabelEntry* le, uint32_t sectionId, size_t offset, intptr_t rel, const OffsetFormat& format) noexcept {
493 0 : LabelLink* link = _allocator.allocT<LabelLink>();
494 0 : if (ASMJIT_UNLIKELY(!link)) return nullptr;
495 :
496 0 : link->next = le->_links;
497 0 : le->_links = link;
498 :
499 0 : link->sectionId = sectionId;
500 0 : link->relocId = Globals::kInvalidId;
501 0 : link->offset = offset;
502 0 : link->rel = rel;
503 0 : link->format = format;
504 :
505 0 : _unresolvedLinkCount++;
506 0 : return link;
507 : }
508 :
509 54984 : Error CodeHolder::newLabelEntry(LabelEntry** entryOut) noexcept {
510 54984 : *entryOut = nullptr;
511 :
512 : uint32_t labelId = _labelEntries.size();
513 54984 : if (ASMJIT_UNLIKELY(labelId == Globals::kInvalidId))
514 : return DebugUtils::errored(kErrorTooManyLabels);
515 :
516 54984 : ASMJIT_PROPAGATE(_labelEntries.willGrow(&_allocator));
517 54984 : LabelEntry* le = _allocator.allocZeroedT<LabelEntry>();
518 :
519 54984 : if (ASMJIT_UNLIKELY(!le))
520 : return DebugUtils::errored(kErrorOutOfMemory);
521 :
522 : le->_setId(labelId);
523 54984 : le->_parentId = Globals::kInvalidId;
524 54984 : le->_offset = 0;
525 : _labelEntries.appendUnsafe(le);
526 :
527 54984 : *entryOut = le;
528 54984 : return kErrorOk;
529 : }
530 :
531 0 : Error CodeHolder::newNamedLabelEntry(LabelEntry** entryOut, const char* name, size_t nameSize, LabelType type, uint32_t parentId) noexcept {
532 0 : *entryOut = nullptr;
533 0 : uint32_t hashCode = CodeHolder_hashNameAndGetSize(name, nameSize);
534 :
535 0 : if (ASMJIT_UNLIKELY(nameSize == 0)) {
536 0 : if (type == LabelType::kAnonymous)
537 0 : return newLabelEntry(entryOut);
538 : else
539 : return DebugUtils::errored(kErrorInvalidLabelName);
540 : }
541 :
542 0 : if (ASMJIT_UNLIKELY(nameSize > Globals::kMaxLabelNameSize))
543 : return DebugUtils::errored(kErrorLabelNameTooLong);
544 :
545 0 : switch (type) {
546 0 : case LabelType::kAnonymous: {
547 : // Anonymous labels cannot have a parent (or more specifically, parent is useless here).
548 0 : if (ASMJIT_UNLIKELY(parentId != Globals::kInvalidId))
549 : return DebugUtils::errored(kErrorInvalidParentLabel);
550 :
551 : uint32_t labelId = _labelEntries.size();
552 0 : if (ASMJIT_UNLIKELY(labelId == Globals::kInvalidId))
553 : return DebugUtils::errored(kErrorTooManyLabels);
554 :
555 0 : ASMJIT_PROPAGATE(_labelEntries.willGrow(&_allocator));
556 0 : LabelEntry* le = _allocator.allocZeroedT<LabelEntry>();
557 :
558 0 : if (ASMJIT_UNLIKELY(!le))
559 : return DebugUtils::errored(kErrorOutOfMemory);
560 :
561 : // NOTE: This LabelEntry has a name, but we leave its hashCode as zero as it's anonymous.
562 : le->_setId(labelId);
563 0 : le->_parentId = Globals::kInvalidId;
564 0 : le->_offset = 0;
565 0 : ASMJIT_PROPAGATE(le->_name.setData(&_zone, name, nameSize));
566 :
567 : _labelEntries.appendUnsafe(le);
568 :
569 0 : *entryOut = le;
570 0 : return kErrorOk;
571 : }
572 :
573 0 : case LabelType::kLocal: {
574 0 : if (ASMJIT_UNLIKELY(parentId >= _labelEntries.size()))
575 : return DebugUtils::errored(kErrorInvalidParentLabel);
576 :
577 0 : hashCode ^= parentId;
578 0 : break;
579 : }
580 :
581 0 : case LabelType::kGlobal:
582 : case LabelType::kExternal: {
583 0 : if (ASMJIT_UNLIKELY(parentId != Globals::kInvalidId))
584 : return DebugUtils::errored(kErrorInvalidParentLabel);
585 : break;
586 : }
587 :
588 : default: {
589 : return DebugUtils::errored(kErrorInvalidArgument);
590 : }
591 : }
592 :
593 : // Don't allow to insert duplicates. Local labels allow duplicates that have
594 : // different id, this is already accomplished by having a different hashes
595 : // between the same label names having different parent labels.
596 0 : LabelEntry* le = _namedLabels.get(LabelByName(name, nameSize, hashCode, parentId));
597 0 : if (ASMJIT_UNLIKELY(le))
598 : return DebugUtils::errored(kErrorLabelAlreadyDefined);
599 :
600 : Error err = kErrorOk;
601 : uint32_t labelId = _labelEntries.size();
602 :
603 0 : if (ASMJIT_UNLIKELY(labelId == Globals::kInvalidId))
604 : return DebugUtils::errored(kErrorTooManyLabels);
605 :
606 0 : ASMJIT_PROPAGATE(_labelEntries.willGrow(&_allocator));
607 0 : le = _allocator.allocZeroedT<LabelEntry>();
608 :
609 0 : if (ASMJIT_UNLIKELY(!le))
610 : return DebugUtils::errored(kErrorOutOfMemory);
611 :
612 0 : le->_hashCode = hashCode;
613 : le->_setId(labelId);
614 0 : le->_type = type;
615 0 : le->_parentId = parentId;
616 0 : le->_offset = 0;
617 0 : ASMJIT_PROPAGATE(le->_name.setData(&_zone, name, nameSize));
618 :
619 : _labelEntries.appendUnsafe(le);
620 : _namedLabels.insert(allocator(), le);
621 :
622 0 : *entryOut = le;
623 0 : return err;
624 : }
625 :
626 0 : uint32_t CodeHolder::labelIdByName(const char* name, size_t nameSize, uint32_t parentId) noexcept {
627 0 : uint32_t hashCode = CodeHolder_hashNameAndGetSize(name, nameSize);
628 0 : if (ASMJIT_UNLIKELY(!nameSize))
629 : return 0;
630 :
631 0 : if (parentId != Globals::kInvalidId)
632 0 : hashCode ^= parentId;
633 :
634 0 : LabelEntry* le = _namedLabels.get(LabelByName(name, nameSize, hashCode, parentId));
635 0 : return le ? le->id() : uint32_t(Globals::kInvalidId);
636 : }
637 :
638 27492 : ASMJIT_API Error CodeHolder::resolveUnresolvedLinks() noexcept {
639 27492 : if (!hasUnresolvedLinks())
640 : return kErrorOk;
641 :
642 : Error err = kErrorOk;
643 0 : for (LabelEntry* le : labelEntries()) {
644 0 : if (!le->isBound())
645 : continue;
646 :
647 : LabelLinkIterator link(le);
648 0 : if (link) {
649 : Support::FastUInt8 of = 0;
650 : Section* toSection = le->section();
651 : uint64_t toOffset = Support::addOverflow(toSection->offset(), le->offset(), &of);
652 :
653 : do {
654 0 : uint32_t linkSectionId = link->sectionId;
655 0 : if (link->relocId == Globals::kInvalidId) {
656 : Section* fromSection = sectionById(linkSectionId);
657 0 : size_t linkOffset = link->offset;
658 :
659 : CodeBuffer& buf = _sections[linkSectionId]->buffer();
660 : ASMJIT_ASSERT(linkOffset < buf.size());
661 :
662 : // Calculate the offset relative to the start of the virtual base.
663 : Support::FastUInt8 localOF = of;
664 : uint64_t fromOffset = Support::addOverflow<uint64_t>(fromSection->offset(), linkOffset, &localOF);
665 0 : int64_t displacement = int64_t(toOffset - fromOffset + uint64_t(int64_t(link->rel)));
666 :
667 0 : if (!localOF) {
668 : ASMJIT_ASSERT(size_t(linkOffset) < buf.size());
669 : ASMJIT_ASSERT(buf.size() - size_t(linkOffset) >= link->format.valueSize());
670 :
671 : // Overwrite a real displacement in the CodeBuffer.
672 0 : if (CodeWriterUtils::writeOffset(buf._data + linkOffset, displacement, link->format)) {
673 : link.resolveAndNext(this);
674 : continue;
675 : }
676 : }
677 :
678 : err = DebugUtils::errored(kErrorInvalidDisplacement);
679 : // Falls through to `link.next()`.
680 : }
681 :
682 : link.next();
683 0 : } while (link);
684 : }
685 : }
686 :
687 : return err;
688 : }
689 :
690 54984 : ASMJIT_API Error CodeHolder::bindLabel(const Label& label, uint32_t toSectionId, uint64_t toOffset) noexcept {
691 : LabelEntry* le = labelEntry(label);
692 54984 : if (ASMJIT_UNLIKELY(!le))
693 : return DebugUtils::errored(kErrorInvalidLabel);
694 :
695 54984 : if (ASMJIT_UNLIKELY(toSectionId > _sections.size()))
696 : return DebugUtils::errored(kErrorInvalidSection);
697 :
698 : // Label can be bound only once.
699 54984 : if (ASMJIT_UNLIKELY(le->isBound()))
700 : return DebugUtils::errored(kErrorLabelAlreadyBound);
701 :
702 : // Bind the label.
703 54984 : Section* section = _sections[toSectionId];
704 54984 : le->_section = section;
705 54984 : le->_offset = toOffset;
706 :
707 : Error err = kErrorOk;
708 : CodeBuffer& buf = section->buffer();
709 :
710 : // Fix all links to this label we have collected so far if they are within
711 : // the same section. We ignore any inter-section links as these have to be
712 : // fixed later.
713 : LabelLinkIterator link(le);
714 54984 : while (link) {
715 0 : uint32_t linkSectionId = link->sectionId;
716 0 : size_t linkOffset = link->offset;
717 :
718 0 : uint32_t relocId = link->relocId;
719 0 : if (relocId != Globals::kInvalidId) {
720 : // Adjust relocation data only.
721 0 : RelocEntry* re = _relocations[relocId];
722 0 : re->_payload += toOffset;
723 0 : re->_targetSectionId = toSectionId;
724 : }
725 : else {
726 0 : if (linkSectionId != toSectionId) {
727 : link.next();
728 0 : continue;
729 : }
730 :
731 : ASMJIT_ASSERT(linkOffset < buf.size());
732 0 : int64_t displacement = int64_t(toOffset - uint64_t(linkOffset) + uint64_t(int64_t(link->rel)));
733 :
734 : // Size of the value we are going to patch. Only BYTE/DWORD is allowed.
735 : ASMJIT_ASSERT(buf.size() - size_t(linkOffset) >= link->format.regionSize());
736 :
737 : // Overwrite a real displacement in the CodeBuffer.
738 0 : if (!CodeWriterUtils::writeOffset(buf._data + linkOffset, displacement, link->format)) {
739 : err = DebugUtils::errored(kErrorInvalidDisplacement);
740 : link.next();
741 0 : continue;
742 : }
743 : }
744 :
745 : link.resolveAndNext(this);
746 : }
747 :
748 : return err;
749 : }
750 :
751 : // CodeHolder - Relocations
752 : // ========================
753 :
754 0 : Error CodeHolder::newRelocEntry(RelocEntry** dst, RelocType relocType) noexcept {
755 0 : ASMJIT_PROPAGATE(_relocations.willGrow(&_allocator));
756 :
757 : uint32_t relocId = _relocations.size();
758 0 : if (ASMJIT_UNLIKELY(relocId == Globals::kInvalidId))
759 : return DebugUtils::errored(kErrorTooManyRelocations);
760 :
761 0 : RelocEntry* re = _allocator.allocZeroedT<RelocEntry>();
762 0 : if (ASMJIT_UNLIKELY(!re))
763 : return DebugUtils::errored(kErrorOutOfMemory);
764 :
765 0 : re->_id = relocId;
766 0 : re->_relocType = relocType;
767 0 : re->_sourceSectionId = Globals::kInvalidId;
768 0 : re->_targetSectionId = Globals::kInvalidId;
769 : _relocations.appendUnsafe(re);
770 :
771 0 : *dst = re;
772 0 : return kErrorOk;
773 : }
774 :
775 : // CodeHolder - Expression Evaluation
776 : // ==================================
777 :
778 0 : static Error CodeHolder_evaluateExpression(CodeHolder* self, Expression* exp, uint64_t* out) noexcept {
779 : uint64_t value[2];
780 0 : for (size_t i = 0; i < 2; i++) {
781 : uint64_t v;
782 0 : switch (exp->valueType[i]) {
783 0 : case ExpressionValueType::kNone: {
784 0 : v = 0;
785 0 : break;
786 : }
787 :
788 0 : case ExpressionValueType::kConstant: {
789 0 : v = exp->value[i].constant;
790 0 : break;
791 : }
792 :
793 0 : case ExpressionValueType::kLabel: {
794 0 : LabelEntry* le = exp->value[i].label;
795 0 : if (!le->isBound())
796 0 : return DebugUtils::errored(kErrorExpressionLabelNotBound);
797 0 : v = le->section()->offset() + le->offset();
798 0 : break;
799 : }
800 :
801 0 : case ExpressionValueType::kExpression: {
802 0 : Expression* nested = exp->value[i].expression;
803 0 : ASMJIT_PROPAGATE(CodeHolder_evaluateExpression(self, nested, &v));
804 : break;
805 : }
806 :
807 : default:
808 : return DebugUtils::errored(kErrorInvalidState);
809 : }
810 :
811 0 : value[i] = v;
812 : }
813 :
814 : uint64_t result;
815 : uint64_t& a = value[0];
816 : uint64_t& b = value[1];
817 :
818 0 : switch (exp->opType) {
819 0 : case ExpressionOpType::kAdd:
820 0 : result = a + b;
821 0 : break;
822 :
823 0 : case ExpressionOpType::kSub:
824 0 : result = a - b;
825 0 : break;
826 :
827 0 : case ExpressionOpType::kMul:
828 0 : result = a * b;
829 0 : break;
830 :
831 0 : case ExpressionOpType::kSll:
832 0 : result = (b > 63) ? uint64_t(0) : uint64_t(a << b);
833 : break;
834 :
835 0 : case ExpressionOpType::kSrl:
836 0 : result = (b > 63) ? uint64_t(0) : uint64_t(a >> b);
837 : break;
838 :
839 0 : case ExpressionOpType::kSra:
840 : result = Support::sar(a, Support::min<uint64_t>(b, 63));
841 0 : break;
842 :
843 : default:
844 : return DebugUtils::errored(kErrorInvalidState);
845 : }
846 :
847 0 : *out = result;
848 0 : return kErrorOk;
849 : }
850 :
851 : // CodeHolder - Utilities
852 : // ======================
853 :
854 27492 : Error CodeHolder::flatten() noexcept {
855 : uint64_t offset = 0;
856 54984 : for (Section* section : _sectionsByOrder) {
857 : uint64_t realSize = section->realSize();
858 27492 : if (realSize) {
859 : uint64_t alignedOffset = Support::alignUp(offset, section->alignment());
860 27492 : if (ASMJIT_UNLIKELY(alignedOffset < offset))
861 : return DebugUtils::errored(kErrorTooLarge);
862 :
863 : Support::FastUInt8 of = 0;
864 : offset = Support::addOverflow(alignedOffset, realSize, &of);
865 :
866 27492 : if (ASMJIT_UNLIKELY(of))
867 : return DebugUtils::errored(kErrorTooLarge);
868 : }
869 : }
870 :
871 : // Now we know that we can assign offsets of all sections properly.
872 : Section* prev = nullptr;
873 : offset = 0;
874 54984 : for (Section* section : _sectionsByOrder) {
875 : uint64_t realSize = section->realSize();
876 27492 : if (realSize)
877 : offset = Support::alignUp(offset, section->alignment());
878 27492 : section->_offset = offset;
879 :
880 : // Make sure the previous section extends a bit to cover the alignment.
881 27492 : if (prev)
882 0 : prev->_virtualSize = offset - prev->_offset;
883 :
884 : prev = section;
885 27492 : offset += realSize;
886 : }
887 :
888 : return kErrorOk;
889 : }
890 :
891 54984 : size_t CodeHolder::codeSize() const noexcept {
892 : Support::FastUInt8 of = 0;
893 : uint64_t offset = 0;
894 :
895 109968 : for (Section* section : _sectionsByOrder) {
896 : uint64_t realSize = section->realSize();
897 :
898 54984 : if (realSize) {
899 : uint64_t alignedOffset = Support::alignUp(offset, section->alignment());
900 : ASMJIT_ASSERT(alignedOffset >= offset);
901 : offset = Support::addOverflow(alignedOffset, realSize, &of);
902 : }
903 : }
904 :
905 54984 : if ((sizeof(uint64_t) > sizeof(size_t) && offset > SIZE_MAX) || of)
906 0 : return SIZE_MAX;
907 :
908 : return size_t(offset);
909 : }
910 :
911 27492 : Error CodeHolder::relocateToBase(uint64_t baseAddress) noexcept {
912 : // Base address must be provided.
913 27492 : if (ASMJIT_UNLIKELY(baseAddress == Globals::kNoBaseAddress))
914 : return DebugUtils::errored(kErrorInvalidArgument);
915 :
916 27492 : _baseAddress = baseAddress;
917 : uint32_t addressSize = _environment.registerSize();
918 :
919 27492 : Section* addressTableSection = _addressTableSection;
920 : uint32_t addressTableEntryCount = 0;
921 : uint8_t* addressTableEntryData = nullptr;
922 :
923 27492 : if (addressTableSection) {
924 0 : ASMJIT_PROPAGATE(
925 : reserveBuffer(&addressTableSection->_buffer, size_t(addressTableSection->virtualSize())));
926 : addressTableEntryData = addressTableSection->_buffer.data();
927 : }
928 :
929 : // Relocate all recorded locations.
930 27492 : for (const RelocEntry* re : _relocations) {
931 : // Possibly deleted or optimized-out entry.
932 0 : if (re->relocType() == RelocType::kNone)
933 0 : continue;
934 :
935 : Section* sourceSection = sectionById(re->sourceSectionId());
936 : Section* targetSection = nullptr;
937 :
938 0 : if (re->targetSectionId() != Globals::kInvalidId)
939 : targetSection = sectionById(re->targetSectionId());
940 :
941 0 : uint64_t value = re->payload();
942 : uint64_t sectionOffset = sourceSection->offset();
943 : uint64_t sourceOffset = re->sourceOffset();
944 :
945 : // Make sure that the `RelocEntry` doesn't go out of bounds.
946 0 : size_t regionSize = re->format().regionSize();
947 0 : if (ASMJIT_UNLIKELY(re->sourceOffset() >= sourceSection->bufferSize() ||
948 : sourceSection->bufferSize() - size_t(re->sourceOffset()) < regionSize))
949 0 : return DebugUtils::errored(kErrorInvalidRelocEntry);
950 :
951 : uint8_t* buffer = sourceSection->data();
952 :
953 0 : switch (re->relocType()) {
954 0 : case RelocType::kExpression: {
955 0 : Expression* expression = (Expression*)(uintptr_t(value));
956 0 : ASMJIT_PROPAGATE(CodeHolder_evaluateExpression(this, expression, &value));
957 : break;
958 : }
959 :
960 : case RelocType::kAbsToAbs: {
961 : break;
962 : }
963 :
964 0 : case RelocType::kRelToAbs: {
965 : // Value is currently a relative offset from the start of its section.
966 : // We have to convert it to an absolute offset (including base address).
967 0 : if (ASMJIT_UNLIKELY(!targetSection))
968 : return DebugUtils::errored(kErrorInvalidRelocEntry);
969 :
970 : //value += baseAddress + sectionOffset + sourceOffset + regionSize;
971 0 : value += baseAddress + targetSection->offset();
972 0 : break;
973 : }
974 :
975 0 : case RelocType::kAbsToRel: {
976 0 : value -= baseAddress + sectionOffset + sourceOffset + regionSize;
977 :
978 : // Sign extend as we are not interested in the high 32-bit word in a 32-bit address space.
979 0 : if (addressSize <= 4)
980 0 : value = uint64_t(int64_t(int32_t(value & 0xFFFFFFFFu)));
981 0 : else if (!Support::isInt32(int64_t(value)))
982 : return DebugUtils::errored(kErrorRelocOffsetOutOfRange);
983 :
984 : break;
985 : }
986 :
987 : case RelocType::kX64AddressEntry: {
988 0 : size_t valueOffset = size_t(re->sourceOffset()) + re->format().valueOffset();
989 0 : if (re->format().valueSize() != 4 || valueOffset < 2)
990 : return DebugUtils::errored(kErrorInvalidRelocEntry);
991 :
992 : // First try whether a relative 32-bit displacement would work.
993 0 : value -= baseAddress + sectionOffset + sourceOffset + regionSize;
994 0 : if (!Support::isInt32(int64_t(value))) {
995 : // Relative 32-bit displacement is not possible, use '.addrtab' section.
996 0 : AddressTableEntry* atEntry = _addressTableEntries.get(re->payload());
997 0 : if (ASMJIT_UNLIKELY(!atEntry))
998 : return DebugUtils::errored(kErrorInvalidRelocEntry);
999 :
1000 : // Cannot be null as we have just matched the `AddressTableEntry`.
1001 : ASMJIT_ASSERT(addressTableSection != nullptr);
1002 :
1003 0 : if (!atEntry->hasAssignedSlot())
1004 0 : atEntry->_slot = addressTableEntryCount++;
1005 :
1006 0 : size_t atEntryIndex = size_t(atEntry->slot()) * addressSize;
1007 0 : uint64_t addrSrc = sectionOffset + sourceOffset + regionSize;
1008 0 : uint64_t addrDst = addressTableSection->offset() + uint64_t(atEntryIndex);
1009 :
1010 0 : value = addrDst - addrSrc;
1011 0 : if (!Support::isInt32(int64_t(value)))
1012 : return DebugUtils::errored(kErrorRelocOffsetOutOfRange);
1013 :
1014 : // Bytes that replace [REX, OPCODE] bytes.
1015 : uint32_t byte0 = 0xFF;
1016 0 : uint32_t byte1 = buffer[valueOffset - 1];
1017 :
1018 0 : if (byte1 == 0xE8) {
1019 : // Patch CALL/MOD byte to FF /2 (-> 0x15).
1020 : byte1 = x86EncodeMod(0, 2, 5);
1021 : }
1022 0 : else if (byte1 == 0xE9) {
1023 : // Patch JMP/MOD byte to FF /4 (-> 0x25).
1024 : byte1 = x86EncodeMod(0, 4, 5);
1025 : }
1026 : else {
1027 : return DebugUtils::errored(kErrorInvalidRelocEntry);
1028 : }
1029 :
1030 : // Patch `jmp/call` instruction.
1031 0 : buffer[valueOffset - 2] = uint8_t(byte0);
1032 0 : buffer[valueOffset - 1] = uint8_t(byte1);
1033 :
1034 0 : Support::writeU64uLE(addressTableEntryData + atEntryIndex, re->payload());
1035 : }
1036 : break;
1037 : }
1038 :
1039 : default:
1040 : return DebugUtils::errored(kErrorInvalidRelocEntry);
1041 : }
1042 :
1043 0 : if (!CodeWriterUtils::writeOffset(buffer + re->sourceOffset(), int64_t(value), re->format())) {
1044 : return DebugUtils::errored(kErrorInvalidRelocEntry);
1045 : }
1046 : }
1047 :
1048 : // Fixup the virtual size of the address table if it's the last section.
1049 27492 : if (_sectionsByOrder.last() == addressTableSection) {
1050 : ASMJIT_ASSERT(addressTableSection != nullptr);
1051 :
1052 0 : size_t addressTableSize = addressTableEntryCount * addressSize;
1053 0 : addressTableSection->_buffer._size = addressTableSize;
1054 0 : addressTableSection->_virtualSize = addressTableSize;
1055 : }
1056 :
1057 : return kErrorOk;
1058 : }
1059 :
1060 0 : Error CodeHolder::copySectionData(void* dst, size_t dstSize, uint32_t sectionId, CopySectionFlags copyFlags) noexcept {
1061 0 : if (ASMJIT_UNLIKELY(!isSectionValid(sectionId)))
1062 : return DebugUtils::errored(kErrorInvalidSection);
1063 :
1064 : Section* section = sectionById(sectionId);
1065 : size_t bufferSize = section->bufferSize();
1066 :
1067 0 : if (ASMJIT_UNLIKELY(dstSize < bufferSize))
1068 : return DebugUtils::errored(kErrorInvalidArgument);
1069 :
1070 : memcpy(dst, section->data(), bufferSize);
1071 :
1072 0 : if (bufferSize < dstSize && Support::test(copyFlags, CopySectionFlags::kPadSectionBuffer)) {
1073 0 : size_t paddingSize = dstSize - bufferSize;
1074 0 : memset(static_cast<uint8_t*>(dst) + bufferSize, 0, paddingSize);
1075 : }
1076 :
1077 : return kErrorOk;
1078 : }
1079 :
1080 0 : Error CodeHolder::copyFlattenedData(void* dst, size_t dstSize, CopySectionFlags copyFlags) noexcept {
1081 : size_t end = 0;
1082 0 : for (Section* section : _sectionsByOrder) {
1083 0 : if (section->offset() > dstSize)
1084 : return DebugUtils::errored(kErrorInvalidArgument);
1085 :
1086 : size_t bufferSize = section->bufferSize();
1087 : size_t offset = size_t(section->offset());
1088 :
1089 0 : if (ASMJIT_UNLIKELY(dstSize - offset < bufferSize))
1090 : return DebugUtils::errored(kErrorInvalidArgument);
1091 :
1092 0 : uint8_t* dstTarget = static_cast<uint8_t*>(dst) + offset;
1093 : size_t paddingSize = 0;
1094 : memcpy(dstTarget, section->data(), bufferSize);
1095 :
1096 0 : if (Support::test(copyFlags, CopySectionFlags::kPadSectionBuffer) && bufferSize < section->virtualSize()) {
1097 0 : paddingSize = Support::min<size_t>(dstSize - offset, size_t(section->virtualSize())) - bufferSize;
1098 0 : memset(dstTarget + bufferSize, 0, paddingSize);
1099 : }
1100 :
1101 0 : end = Support::max(end, offset + bufferSize + paddingSize);
1102 : }
1103 :
1104 0 : if (end < dstSize && Support::test(copyFlags, CopySectionFlags::kPadTargetBuffer)) {
1105 0 : memset(static_cast<uint8_t*>(dst) + end, 0, dstSize - end);
1106 : }
1107 :
1108 : return kErrorOk;
1109 : }
1110 :
1111 : // CodeHolder - Tests
1112 : // ==================
1113 :
1114 : #if defined(ASMJIT_TEST)
1115 : UNIT(code_holder) {
1116 : CodeHolder code;
1117 :
1118 : INFO("Verifying CodeHolder::init()");
1119 : Environment env;
1120 : env.init(Arch::kX86);
1121 :
1122 : code.init(env);
1123 : EXPECT(code.arch() == Arch::kX86);
1124 :
1125 : INFO("Verifying named labels");
1126 : LabelEntry* le;
1127 : EXPECT(code.newNamedLabelEntry(&le, "NamedLabel", SIZE_MAX, LabelType::kGlobal) == kErrorOk);
1128 : EXPECT(strcmp(le->name(), "NamedLabel") == 0);
1129 : EXPECT(code.labelIdByName("NamedLabel") == le->id());
1130 :
1131 : INFO("Verifying section ordering");
1132 : Section* section1;
1133 : EXPECT(code.newSection(§ion1, "high-priority", SIZE_MAX, SectionFlags::kNone, 1, -1) == kErrorOk);
1134 : EXPECT(code.sections()[1] == section1);
1135 : EXPECT(code.sectionsByOrder()[0] == section1);
1136 :
1137 : Section* section0;
1138 : EXPECT(code.newSection(§ion0, "higher-priority", SIZE_MAX, SectionFlags::kNone, 1, -2) == kErrorOk);
1139 : EXPECT(code.sections()[2] == section0);
1140 : EXPECT(code.sectionsByOrder()[0] == section0);
1141 : EXPECT(code.sectionsByOrder()[1] == section1);
1142 :
1143 : Section* section3;
1144 : EXPECT(code.newSection(§ion3, "low-priority", SIZE_MAX, SectionFlags::kNone, 1, 2) == kErrorOk);
1145 : EXPECT(code.sections()[3] == section3);
1146 : EXPECT(code.sectionsByOrder()[3] == section3);
1147 : }
1148 : #endif
1149 :
1150 : ASMJIT_END_NAMESPACE
1151 :
1152 : #endif // __PLUMED_HAS_ASMJIT
|
