Line data Source code
1 : #ifdef __PLUMED_HAS_ASMJIT
2 : // This file is part of AsmJit project <https://asmjit.com>
3 : //
4 : // See asmjit.h or LICENSE.md for license and copyright information
5 : // SPDX-License-Identifier: Zlib
6 :
7 : #include "api-build_p.h"
8 : #ifndef ASMJIT_NO_COMPILER
9 :
10 : #include "formatter.h"
11 : #include "ralocal_p.h"
12 : #include "rapass_p.h"
13 : #include "support.h"
14 : #include "type.h"
15 : #include "zonestack.h"
16 :
17 : ASMJIT_BEGIN_NAMESPACE
18 :
19 : // RABlock - Control Flow
20 : // ======================
21 :
22 27492 : Error RABlock::appendSuccessor(RABlock* successor) noexcept {
23 27492 : RABlock* predecessor = this;
24 :
25 27492 : if (predecessor->hasSuccessor(successor))
26 : return kErrorOk;
27 :
28 27492 : ASMJIT_PROPAGATE(successor->_predecessors.willGrow(allocator()));
29 27492 : ASMJIT_PROPAGATE(predecessor->_successors.willGrow(allocator()));
30 :
31 : predecessor->_successors.appendUnsafe(successor);
32 : successor->_predecessors.appendUnsafe(predecessor);
33 :
34 27492 : return kErrorOk;
35 : }
36 :
37 0 : Error RABlock::prependSuccessor(RABlock* successor) noexcept {
38 0 : RABlock* predecessor = this;
39 :
40 0 : if (predecessor->hasSuccessor(successor))
41 : return kErrorOk;
42 :
43 0 : ASMJIT_PROPAGATE(successor->_predecessors.willGrow(allocator()));
44 0 : ASMJIT_PROPAGATE(predecessor->_successors.willGrow(allocator()));
45 :
46 : predecessor->_successors.prependUnsafe(successor);
47 : successor->_predecessors.prependUnsafe(predecessor);
48 :
49 0 : return kErrorOk;
50 : }
51 :
52 : // BaseRAPass - Construction & Destruction
53 : // =======================================
54 :
55 632316 : BaseRAPass::BaseRAPass() noexcept : FuncPass("BaseRAPass") {}
56 27492 : BaseRAPass::~BaseRAPass() noexcept {}
57 :
58 : // BaseRAPass - RunOnFunction
59 : // ==========================
60 :
61 54984 : static void BaseRAPass_reset(BaseRAPass* self, FuncDetail* funcDetail) noexcept {
62 : ZoneAllocator* allocator = self->allocator();
63 :
64 : self->_blocks.reset();
65 : self->_exits.reset();
66 : self->_pov.reset();
67 : self->_workRegs.reset();
68 54984 : self->_instructionCount = 0;
69 54984 : self->_createdBlockCount = 0;
70 :
71 : self->_sharedAssignments.reset();
72 54984 : self->_lastTimestamp = 0;
73 :
74 54984 : self->_archTraits = nullptr;
75 : self->_physRegIndex.reset();
76 : self->_physRegCount.reset();
77 54984 : self->_physRegTotal = 0;
78 : self->_scratchRegIndexes.fill(BaseReg::kIdBad);
79 :
80 : self->_availableRegs.reset();
81 : self->_availableRegCount.reset();
82 : self->_clobberedRegs.reset();
83 :
84 : self->_workRegs.reset();
85 : self->_workRegsOfGroup.forEach([](RAWorkRegs& regs) { regs.reset(); });
86 : self->_strategy.forEach([](RAStrategy& strategy) { strategy.reset(); });
87 : self->_globalLiveSpans.fill(nullptr);
88 : self->_globalMaxLiveCount.reset();
89 : self->_temporaryMem.reset();
90 :
91 : self->_stackAllocator.reset(allocator);
92 54984 : self->_argsAssignment.reset(funcDetail);
93 54984 : self->_numStackArgsToStackSlots = 0;
94 54984 : self->_maxWorkRegNameSize = 0;
95 54984 : }
96 :
97 27492 : static void BaseRAPass_resetVirtRegData(BaseRAPass* self) noexcept {
98 298806 : for (RAWorkReg* wReg : self->_workRegs) {
99 : VirtReg* vReg = wReg->virtReg();
100 :
101 : // Update the information regarding the stack of the virtual register.
102 271314 : if (wReg->hasStackSlot()) {
103 : RAStackSlot* slot = wReg->stackSlot();
104 : vReg->assignStackSlot(slot->offset());
105 : }
106 :
107 : // Reset work reg association so it cannot be used by accident (RAWorkReg data will be destroyed).
108 271314 : vReg->_workReg = nullptr;
109 : }
110 27492 : }
111 :
112 27492 : Error BaseRAPass::runOnFunction(Zone* zone, Logger* logger, FuncNode* func) {
113 27492 : _allocator.reset(zone);
114 :
115 : #ifndef ASMJIT_NO_LOGGING
116 27492 : _logger = logger;
117 27492 : _formatOptions.reset();
118 27492 : _diagnosticOptions = DiagnosticOptions::kNone;
119 :
120 27492 : if (logger) {
121 0 : _formatOptions = logger->options();
122 0 : _diagnosticOptions = _cb->diagnosticOptions();
123 : }
124 : #else
125 : DebugUtils::unused(logger);
126 : #endif
127 :
128 : // Initialize all core structures to use `zone` and `func`.
129 : BaseNode* end = func->endNode();
130 27492 : _func = func;
131 27492 : _stop = end->next();
132 27492 : _extraBlock = end;
133 :
134 27492 : BaseRAPass_reset(this, &_func->_funcDetail);
135 :
136 : // Initialize architecture-specific members.
137 27492 : onInit();
138 :
139 : // Perform all allocation steps required.
140 27492 : Error err = onPerformAllSteps();
141 :
142 : // Must be called regardless of the allocation status.
143 27492 : onDone();
144 :
145 : // Reset possible connections introduced by the register allocator.
146 27492 : BaseRAPass_resetVirtRegData(this);
147 :
148 : // Reset all core structures and everything that depends on the passed `Zone`.
149 27492 : BaseRAPass_reset(this, nullptr);
150 27492 : _allocator.reset(nullptr);
151 :
152 : #ifndef ASMJIT_NO_LOGGING
153 27492 : _logger = nullptr;
154 27492 : _formatOptions.reset();
155 27492 : _diagnosticOptions = DiagnosticOptions::kNone;
156 : #endif
157 :
158 27492 : _func = nullptr;
159 27492 : _stop = nullptr;
160 27492 : _extraBlock = nullptr;
161 :
162 : // Reset `Zone` as nothing should persist between `runOnFunction()` calls.
163 27492 : zone->reset();
164 :
165 : // We alter the compiler cursor, because it doesn't make sense to reference it after the compilation - some
166 : // nodes may disappear and the old cursor can go out anyway.
167 : cc()->_setCursor(cc()->lastNode());
168 :
169 27492 : return err;
170 : }
171 :
172 27492 : Error BaseRAPass::onPerformAllSteps() noexcept {
173 27492 : ASMJIT_PROPAGATE(buildCFG());
174 27492 : ASMJIT_PROPAGATE(buildCFGViews());
175 27492 : ASMJIT_PROPAGATE(removeUnreachableCode());
176 :
177 27492 : ASMJIT_PROPAGATE(buildCFGDominators());
178 27492 : ASMJIT_PROPAGATE(buildLiveness());
179 27492 : ASMJIT_PROPAGATE(assignArgIndexToWorkRegs());
180 :
181 : #ifndef ASMJIT_NO_LOGGING
182 27492 : if (hasDiagnosticOption(DiagnosticOptions::kRAAnnotate))
183 0 : ASMJIT_PROPAGATE(annotateCode());
184 : #endif
185 :
186 27492 : ASMJIT_PROPAGATE(runGlobalAllocator());
187 27492 : ASMJIT_PROPAGATE(runLocalAllocator());
188 :
189 27492 : ASMJIT_PROPAGATE(updateStackFrame());
190 27492 : ASMJIT_PROPAGATE(insertPrologEpilog());
191 :
192 27492 : ASMJIT_PROPAGATE(rewrite());
193 :
194 : return kErrorOk;
195 : }
196 :
197 : // BaseRAPass - CFG - Basic Block Management
198 : // =========================================
199 :
200 54984 : RABlock* BaseRAPass::newBlock(BaseNode* initialNode) noexcept {
201 54984 : RABlock* block = zone()->newT<RABlock>(this);
202 54984 : if (ASMJIT_UNLIKELY(!block))
203 : return nullptr;
204 :
205 : block->setFirst(initialNode);
206 : block->setLast(initialNode);
207 :
208 54984 : _createdBlockCount++;
209 54984 : return block;
210 : }
211 :
212 27492 : RABlock* BaseRAPass::newBlockOrExistingAt(LabelNode* cbLabel, BaseNode** stoppedAt) noexcept {
213 27492 : if (cbLabel->hasPassData())
214 : return cbLabel->passData<RABlock>();
215 :
216 : FuncNode* func = this->func();
217 : BaseNode* node = cbLabel->prev();
218 : RABlock* block = nullptr;
219 :
220 : // Try to find some label, but terminate the loop on any code. We try hard to coalesce code that contains two
221 : // consecutive labels or a combination of non-code nodes between 2 or more labels.
222 : //
223 : // Possible cases that would share the same basic block:
224 : //
225 : // 1. Two or more consecutive labels:
226 : // Label1:
227 : // Label2:
228 : //
229 : // 2. Two or more labels separated by non-code nodes:
230 : // Label1:
231 : // ; Some comment...
232 : // .align 16
233 : // Label2:
234 : size_t nPendingLabels = 0;
235 :
236 27492 : while (node) {
237 27492 : if (node->type() == NodeType::kLabel) {
238 : // Function has a different NodeType, just make sure this was not messed up as we must never associate
239 : // BasicBlock with a `func` itself.
240 : ASMJIT_ASSERT(node != func);
241 :
242 : block = node->passData<RABlock>();
243 0 : if (block) {
244 : // Exit node has always a block associated with it. If we went here it means that `cbLabel` passed here
245 : // is after the end of the function and cannot be merged with the function exit block.
246 0 : if (node == func->exitNode())
247 : block = nullptr;
248 : break;
249 : }
250 :
251 0 : nPendingLabels++;
252 : }
253 27492 : else if (node->type() == NodeType::kAlign) {
254 : // Align node is fine.
255 : }
256 : else {
257 : break;
258 : }
259 :
260 : node = node->prev();
261 : }
262 :
263 27492 : if (stoppedAt)
264 27492 : *stoppedAt = node;
265 :
266 27492 : if (!block) {
267 27492 : block = newBlock();
268 27492 : if (ASMJIT_UNLIKELY(!block))
269 : return nullptr;
270 : }
271 :
272 : cbLabel->setPassData<RABlock>(block);
273 : node = cbLabel;
274 :
275 27492 : while (nPendingLabels) {
276 : node = node->prev();
277 : for (;;) {
278 0 : if (node->type() == NodeType::kLabel) {
279 : node->setPassData<RABlock>(block);
280 0 : nPendingLabels--;
281 0 : break;
282 : }
283 :
284 : node = node->prev();
285 : ASMJIT_ASSERT(node != nullptr);
286 : }
287 : }
288 :
289 27492 : if (!block->first()) {
290 : block->setFirst(node);
291 : block->setLast(cbLabel);
292 : }
293 :
294 : return block;
295 : }
296 :
297 54984 : Error BaseRAPass::addBlock(RABlock* block) noexcept {
298 54984 : ASMJIT_PROPAGATE(_blocks.willGrow(allocator()));
299 :
300 54984 : block->_blockId = blockCount();
301 : _blocks.appendUnsafe(block);
302 54984 : return kErrorOk;
303 : }
304 :
305 : // BaseRAPass - CFG - Build
306 : // ========================
307 :
308 27492 : Error BaseRAPass::initSharedAssignments(const ZoneVector<uint32_t>& sharedAssignmentsMap) noexcept {
309 27492 : if (sharedAssignmentsMap.empty())
310 : return kErrorOk;
311 :
312 : uint32_t count = 0;
313 0 : for (RABlock* block : _blocks) {
314 0 : if (block->hasSharedAssignmentId()) {
315 0 : uint32_t sharedAssignmentId = sharedAssignmentsMap[block->sharedAssignmentId()];
316 : block->setSharedAssignmentId(sharedAssignmentId);
317 0 : count = Support::max(count, sharedAssignmentId + 1);
318 : }
319 : }
320 :
321 0 : ASMJIT_PROPAGATE(_sharedAssignments.resize(allocator(), count));
322 :
323 : // Aggregate all entry scratch GP regs from blocks of the same assignment to the assignment itself. It will then be
324 : // used instead of RABlock's own scratch regs mask, as shared assignments have precedence.
325 0 : for (RABlock* block : _blocks) {
326 0 : if (block->hasJumpTable()) {
327 : const RABlocks& successors = block->successors();
328 0 : if (!successors.empty()) {
329 0 : RABlock* firstSuccessor = successors[0];
330 : // NOTE: Shared assignments connect all possible successors so we only need the first to propagate exit scratch
331 : // GP registers.
332 : ASMJIT_ASSERT(firstSuccessor->hasSharedAssignmentId());
333 0 : RASharedAssignment& sa = _sharedAssignments[firstSuccessor->sharedAssignmentId()];
334 : sa.addEntryScratchGpRegs(block->exitScratchGpRegs());
335 : }
336 : }
337 0 : if (block->hasSharedAssignmentId()) {
338 0 : RASharedAssignment& sa = _sharedAssignments[block->sharedAssignmentId()];
339 0 : sa.addEntryScratchGpRegs(block->_entryScratchGpRegs);
340 : }
341 : }
342 :
343 : return kErrorOk;
344 : }
345 :
346 : // BaseRAPass - CFG - Views Order
347 : // ==============================
348 :
349 : class RABlockVisitItem {
350 : public:
351 : inline RABlockVisitItem(RABlock* block, uint32_t index) noexcept
352 27492 : : _block(block),
353 27492 : _index(index) {}
354 :
355 : inline RABlockVisitItem(const RABlockVisitItem& other) noexcept
356 27492 : : _block(other._block),
357 27492 : _index(other._index) {}
358 :
359 : inline RABlockVisitItem& operator=(const RABlockVisitItem& other) noexcept = default;
360 :
361 27492 : inline RABlock* block() const noexcept { return _block; }
362 27492 : inline uint32_t index() const noexcept { return _index; }
363 :
364 : RABlock* _block;
365 : uint32_t _index;
366 : };
367 :
368 27492 : Error BaseRAPass::buildCFGViews() noexcept {
369 : #ifndef ASMJIT_NO_LOGGING
370 : Logger* logger = getLoggerIf(DiagnosticOptions::kRADebugCFG);
371 0 : ASMJIT_RA_LOG_FORMAT("[BuildCFGViews]\n");
372 : #endif
373 :
374 : uint32_t count = blockCount();
375 27492 : if (ASMJIT_UNLIKELY(!count)) return kErrorOk;
376 :
377 27492 : ASMJIT_PROPAGATE(_pov.reserve(allocator(), count));
378 :
379 : ZoneStack<RABlockVisitItem> stack;
380 27492 : ASMJIT_PROPAGATE(stack.init(allocator()));
381 :
382 : ZoneBitVector visited;
383 27492 : ASMJIT_PROPAGATE(visited.resize(allocator(), count));
384 :
385 27492 : RABlock* current = _blocks[0];
386 : uint32_t i = 0;
387 :
388 : for (;;) {
389 : for (;;) {
390 82476 : if (i >= current->successors().size())
391 : break;
392 :
393 : // Skip if already visited.
394 27492 : RABlock* child = current->successors()[i++];
395 27492 : if (visited.bitAt(child->blockId()))
396 0 : continue;
397 :
398 : // Mark as visited to prevent visiting the same block multiple times.
399 : visited.setBit(child->blockId(), true);
400 :
401 : // Add the current block on the stack, we will get back to it later.
402 27492 : ASMJIT_PROPAGATE(stack.append(RABlockVisitItem(current, i)));
403 27492 : current = child;
404 : i = 0;
405 : }
406 :
407 : current->makeReachable();
408 54984 : current->_povOrder = _pov.size();
409 : _pov.appendUnsafe(current);
410 :
411 54984 : if (stack.empty())
412 : break;
413 :
414 27492 : RABlockVisitItem top = stack.pop();
415 27492 : current = top.block();
416 : i = top.index();
417 27492 : }
418 :
419 27492 : ASMJIT_RA_LOG_COMPLEX({
420 : StringTmp<1024> sb;
421 : for (RABlock* block : blocks()) {
422 : sb.clear();
423 : if (block->hasSuccessors()) {
424 : sb.appendFormat(" #%u -> {", block->blockId());
425 : _dumpBlockIds(sb, block->successors());
426 : sb.append("}\n");
427 : }
428 : else {
429 : sb.appendFormat(" #%u -> {Exit}\n", block->blockId());
430 : }
431 : logger->log(sb);
432 : }
433 : });
434 :
435 27492 : visited.release(allocator());
436 27492 : return kErrorOk;
437 : }
438 :
439 : // BaseRAPass - CFG - Dominators
440 : // =============================
441 :
442 : static ASMJIT_FORCE_INLINE RABlock* intersectBlocks(RABlock* b1, RABlock* b2) noexcept {
443 0 : while (b1 != b2) {
444 0 : while (b2->povOrder() > b1->povOrder()) b1 = b1->iDom();
445 0 : while (b1->povOrder() > b2->povOrder()) b2 = b2->iDom();
446 : }
447 : return b1;
448 : }
449 :
450 : // Based on "A Simple, Fast Dominance Algorithm".
451 27492 : Error BaseRAPass::buildCFGDominators() noexcept {
452 : #ifndef ASMJIT_NO_LOGGING
453 : Logger* logger = getLoggerIf(DiagnosticOptions::kRADebugCFG);
454 0 : ASMJIT_RA_LOG_FORMAT("[BuildCFGDominators]\n");
455 : #endif
456 :
457 27492 : if (_blocks.empty())
458 : return kErrorOk;
459 :
460 : RABlock* entryBlock = this->entryBlock();
461 : entryBlock->setIDom(entryBlock);
462 :
463 : bool changed = true;
464 : uint32_t nIters = 0;
465 :
466 82476 : while (changed) {
467 54984 : nIters++;
468 : changed = false;
469 :
470 : uint32_t i = _pov.size();
471 164952 : while (i) {
472 109968 : RABlock* block = _pov[--i];
473 109968 : if (block == entryBlock)
474 54984 : continue;
475 :
476 : RABlock* iDom = nullptr;
477 : const RABlocks& preds = block->predecessors();
478 :
479 : uint32_t j = preds.size();
480 109968 : while (j) {
481 54984 : RABlock* p = preds[--j];
482 54984 : if (!p->iDom())
483 0 : continue;
484 54984 : iDom = !iDom ? p : intersectBlocks(iDom, p);
485 : }
486 :
487 54984 : if (block->iDom() != iDom) {
488 27492 : ASMJIT_ASSUME(iDom != nullptr);
489 27492 : ASMJIT_RA_LOG_FORMAT(" IDom of #%u -> #%u\n", block->blockId(), iDom->blockId());
490 : block->setIDom(iDom);
491 : changed = true;
492 : }
493 : }
494 : }
495 :
496 27492 : ASMJIT_RA_LOG_FORMAT(" Done (%u iterations)\n", nIters);
497 : return kErrorOk;
498 : }
499 :
500 0 : bool BaseRAPass::_strictlyDominates(const RABlock* a, const RABlock* b) const noexcept {
501 : ASMJIT_ASSERT(a != nullptr); // There must be at least one block if this function is
502 : ASMJIT_ASSERT(b != nullptr); // called, as both `a` and `b` must be valid blocks.
503 : ASMJIT_ASSERT(a != b); // Checked by `dominates()` and `strictlyDominates()`.
504 :
505 : // Nothing strictly dominates the entry block.
506 : const RABlock* entryBlock = this->entryBlock();
507 0 : if (a == entryBlock)
508 : return false;
509 :
510 : const RABlock* iDom = b->iDom();
511 0 : while (iDom != a && iDom != entryBlock)
512 : iDom = iDom->iDom();
513 :
514 : return iDom != entryBlock;
515 : }
516 :
517 0 : const RABlock* BaseRAPass::_nearestCommonDominator(const RABlock* a, const RABlock* b) const noexcept {
518 : ASMJIT_ASSERT(a != nullptr); // There must be at least one block if this function is
519 : ASMJIT_ASSERT(b != nullptr); // called, as both `a` and `b` must be valid blocks.
520 : ASMJIT_ASSERT(a != b); // Checked by `dominates()` and `properlyDominates()`.
521 :
522 0 : if (a == b)
523 : return a;
524 :
525 : // If `a` strictly dominates `b` then `a` is the nearest common dominator.
526 0 : if (_strictlyDominates(a, b))
527 : return a;
528 :
529 : // If `b` strictly dominates `a` then `b` is the nearest common dominator.
530 0 : if (_strictlyDominates(b, a))
531 : return b;
532 :
533 : const RABlock* entryBlock = this->entryBlock();
534 : uint64_t timestamp = nextTimestamp();
535 :
536 : // Mark all A's dominators.
537 : const RABlock* block = a->iDom();
538 0 : while (block != entryBlock) {
539 : block->setTimestamp(timestamp);
540 : block = block->iDom();
541 : }
542 :
543 : // Check all B's dominators against marked dominators of A.
544 : block = b->iDom();
545 0 : while (block != entryBlock) {
546 0 : if (block->hasTimestamp(timestamp))
547 : return block;
548 : block = block->iDom();
549 : }
550 :
551 : return entryBlock;
552 : }
553 :
554 : // BaseRAPass - CFG - Utilities
555 : // ============================
556 :
557 27492 : Error BaseRAPass::removeUnreachableCode() noexcept {
558 : uint32_t numAllBlocks = blockCount();
559 : uint32_t numReachableBlocks = reachableBlockCount();
560 :
561 : // All reachable -> nothing to do.
562 27492 : if (numAllBlocks == numReachableBlocks)
563 : return kErrorOk;
564 :
565 : #ifndef ASMJIT_NO_LOGGING
566 : StringTmp<256> sb;
567 : Logger* logger = getLoggerIf(DiagnosticOptions::kRADebugUnreachable);
568 0 : ASMJIT_RA_LOG_FORMAT("[RemoveUnreachableCode - detected %u of %u unreachable blocks]\n", numAllBlocks - numReachableBlocks, numAllBlocks);
569 : #endif
570 :
571 0 : for (uint32_t i = 0; i < numAllBlocks; i++) {
572 0 : RABlock* block = _blocks[i];
573 0 : if (block->isReachable())
574 0 : continue;
575 :
576 0 : ASMJIT_RA_LOG_FORMAT(" Removing code from unreachable block {%u}\n", i);
577 : BaseNode* first = block->first();
578 : BaseNode* last = block->last();
579 :
580 : BaseNode* beforeFirst = first->prev();
581 : BaseNode* afterLast = last->next();
582 :
583 : BaseNode* node = first;
584 0 : while (node != afterLast) {
585 : BaseNode* next = node->next();
586 :
587 0 : if (node->isCode() || node->isRemovable()) {
588 : #ifndef ASMJIT_NO_LOGGING
589 0 : if (logger) {
590 0 : sb.clear();
591 0 : Formatter::formatNode(sb, _formatOptions, cc(), node);
592 0 : logger->logf(" %s\n", sb.data());
593 : }
594 : #endif
595 0 : cc()->removeNode(node);
596 : }
597 : node = next;
598 : }
599 :
600 0 : if (beforeFirst->next() == afterLast) {
601 : block->setFirst(nullptr);
602 : block->setLast(nullptr);
603 : }
604 : else {
605 : block->setFirst(beforeFirst->next());
606 : block->setLast(afterLast->prev());
607 : }
608 : }
609 :
610 : return kErrorOk;
611 : }
612 :
613 0 : BaseNode* BaseRAPass::findSuccessorStartingAt(BaseNode* node) noexcept {
614 0 : while (node && (node->isInformative() || node->hasNoEffect()))
615 : node = node->next();
616 0 : return node;
617 : }
618 :
619 27492 : bool BaseRAPass::isNextTo(BaseNode* node, BaseNode* target) noexcept {
620 : for (;;) {
621 : node = node->next();
622 27492 : if (node == target)
623 : return true;
624 :
625 0 : if (!node)
626 : return false;
627 :
628 0 : if (node->isCode() || node->isData())
629 : return false;
630 : }
631 : }
632 :
633 : // BaseRAPass - Registers - VirtReg / WorkReg Mapping
634 : // ==================================================
635 :
636 271314 : Error BaseRAPass::_asWorkReg(VirtReg* vReg, RAWorkReg** out) noexcept {
637 : // Checked by `asWorkReg()` - must be true.
638 : ASMJIT_ASSERT(vReg->_workReg == nullptr);
639 :
640 271314 : RegGroup group = vReg->group();
641 : ASMJIT_ASSERT(group <= RegGroup::kMaxVirt);
642 :
643 : RAWorkRegs& wRegs = workRegs();
644 : RAWorkRegs& wRegsByGroup = workRegs(group);
645 :
646 271314 : ASMJIT_PROPAGATE(wRegs.willGrow(allocator()));
647 271314 : ASMJIT_PROPAGATE(wRegsByGroup.willGrow(allocator()));
648 :
649 271314 : RAWorkReg* wReg = zone()->newT<RAWorkReg>(vReg, wRegs.size());
650 271314 : if (ASMJIT_UNLIKELY(!wReg))
651 : return DebugUtils::errored(kErrorOutOfMemory);
652 :
653 : vReg->setWorkReg(wReg);
654 271314 : if (!vReg->isStack())
655 : wReg->setRegByteMask(Support::lsbMask<uint64_t>(vReg->virtSize()));
656 : wRegs.appendUnsafe(wReg);
657 : wRegsByGroup.appendUnsafe(wReg);
658 :
659 : // Only used by RA logging.
660 271314 : _maxWorkRegNameSize = Support::max(_maxWorkRegNameSize, vReg->nameSize());
661 :
662 271314 : *out = wReg;
663 271314 : return kErrorOk;
664 : }
665 :
666 82476 : RAAssignment::WorkToPhysMap* BaseRAPass::newWorkToPhysMap() noexcept {
667 : uint32_t count = workRegCount();
668 82476 : size_t size = WorkToPhysMap::sizeOf(count);
669 :
670 : // If no registers are used it could be zero, in that case return a dummy
671 : // map instead of NULL.
672 82476 : if (ASMJIT_UNLIKELY(!size)) {
673 : static const RAAssignment::WorkToPhysMap nullMap = {{ 0 }};
674 : return const_cast<RAAssignment::WorkToPhysMap*>(&nullMap);
675 : }
676 :
677 : WorkToPhysMap* map = zone()->allocT<WorkToPhysMap>(size);
678 82476 : if (ASMJIT_UNLIKELY(!map))
679 : return nullptr;
680 :
681 : map->reset(count);
682 : return map;
683 : }
684 :
685 54984 : RAAssignment::PhysToWorkMap* BaseRAPass::newPhysToWorkMap() noexcept {
686 : uint32_t count = physRegTotal();
687 54984 : size_t size = PhysToWorkMap::sizeOf(count);
688 :
689 : PhysToWorkMap* map = zone()->allocT<PhysToWorkMap>(size);
690 54984 : if (ASMJIT_UNLIKELY(!map))
691 : return nullptr;
692 :
693 54984 : map->reset(count);
694 54984 : return map;
695 : }
696 :
697 : // BaseRAPass - Registers - Liveness Analysis and Statistics
698 : // =========================================================
699 :
700 : namespace LiveOps {
701 : typedef ZoneBitVector::BitWord BitWord;
702 :
703 : struct In {
704 : static ASMJIT_FORCE_INLINE BitWord op(BitWord dst, BitWord out, BitWord gen, BitWord kill) noexcept {
705 : DebugUtils::unused(dst);
706 56570 : return (out | gen) & ~kill;
707 : }
708 : };
709 :
710 : template<typename Operator>
711 : static ASMJIT_FORCE_INLINE bool op(BitWord* dst, const BitWord* a, uint32_t n) noexcept {
712 : BitWord changed = 0;
713 :
714 111554 : for (uint32_t i = 0; i < n; i++) {
715 56570 : BitWord before = dst[i];
716 56570 : BitWord after = Operator::op(before, a[i]);
717 :
718 28285 : dst[i] = after;
719 28285 : changed |= (before ^ after);
720 : }
721 :
722 27492 : return changed != 0;
723 : }
724 :
725 : template<typename Operator>
726 : static ASMJIT_FORCE_INLINE bool op(BitWord* dst, const BitWord* a, const BitWord* b, uint32_t n) noexcept {
727 : BitWord changed = 0;
728 :
729 : for (uint32_t i = 0; i < n; i++) {
730 : BitWord before = dst[i];
731 : BitWord after = Operator::op(before, a[i], b[i]);
732 :
733 : dst[i] = after;
734 : changed |= (before ^ after);
735 : }
736 :
737 : return changed != 0;
738 : }
739 :
740 : template<typename Operator>
741 : static ASMJIT_FORCE_INLINE bool op(BitWord* dst, const BitWord* a, const BitWord* b, const BitWord* c, uint32_t n) noexcept {
742 : BitWord changed = 0;
743 :
744 111554 : for (uint32_t i = 0; i < n; i++) {
745 56570 : BitWord before = dst[i];
746 56570 : BitWord after = Operator::op(before, a[i], b[i], c[i]);
747 :
748 56570 : dst[i] = after;
749 0 : changed |= (before ^ after);
750 : }
751 :
752 : return changed != 0;
753 : }
754 :
755 : static ASMJIT_FORCE_INLINE bool recalcInOut(RABlock* block, uint32_t numBitWords, bool initial = false) noexcept {
756 : bool changed = initial;
757 :
758 : const RABlocks& successors = block->successors();
759 : uint32_t numSuccessors = successors.size();
760 :
761 : // Calculate `OUT` based on `IN` of all successors.
762 164952 : for (uint32_t i = 0; i < numSuccessors; i++)
763 82476 : changed |= op<Support::Or>(block->liveOut().data(), successors[i]->liveIn().data(), numBitWords);
764 :
765 : // Calculate `IN` based on `OUT`, `GEN`, and `KILL` bits.
766 54984 : if (changed)
767 : changed = op<In>(block->liveIn().data(), block->liveOut().data(), block->gen().data(), block->kill().data(), numBitWords);
768 :
769 : return changed;
770 : }
771 : }
772 :
773 27492 : ASMJIT_FAVOR_SPEED Error BaseRAPass::buildLiveness() noexcept {
774 : #ifndef ASMJIT_NO_LOGGING
775 : Logger* logger = getLoggerIf(DiagnosticOptions::kRADebugLiveness);
776 : StringTmp<512> sb;
777 : #endif
778 :
779 27492 : ASMJIT_RA_LOG_FORMAT("[BuildLiveness]\n");
780 :
781 : uint32_t i;
782 :
783 : uint32_t numAllBlocks = blockCount();
784 : uint32_t numReachableBlocks = reachableBlockCount();
785 :
786 : uint32_t numVisits = numReachableBlocks;
787 : uint32_t numWorkRegs = workRegCount();
788 : uint32_t numBitWords = ZoneBitVector::_wordsPerBits(numWorkRegs);
789 :
790 27492 : if (!numWorkRegs) {
791 0 : ASMJIT_RA_LOG_FORMAT(" Done (no virtual registers)\n");
792 0 : return kErrorOk;
793 : }
794 :
795 : ZoneVector<uint32_t> nUsesPerWorkReg; // Number of USEs of each RAWorkReg.
796 : ZoneVector<uint32_t> nOutsPerWorkReg; // Number of OUTs of each RAWorkReg.
797 : ZoneVector<uint32_t> nInstsPerBlock; // Number of instructions of each RABlock.
798 :
799 27492 : ASMJIT_PROPAGATE(nUsesPerWorkReg.resize(allocator(), numWorkRegs));
800 27492 : ASMJIT_PROPAGATE(nOutsPerWorkReg.resize(allocator(), numWorkRegs));
801 27492 : ASMJIT_PROPAGATE(nInstsPerBlock.resize(allocator(), numAllBlocks));
802 :
803 : // Calculate GEN/KILL of Each Block
804 : // --------------------------------
805 :
806 82476 : for (i = 0; i < numReachableBlocks; i++) {
807 54984 : RABlock* block = _pov[i];
808 54984 : ASMJIT_PROPAGATE(block->resizeLiveBits(numWorkRegs));
809 :
810 : BaseNode* node = block->last();
811 : BaseNode* stop = block->first();
812 :
813 : uint32_t nInsts = 0;
814 : for (;;) {
815 386792 : if (node->isInst()) {
816 : InstNode* inst = node->as<InstNode>();
817 : RAInst* raInst = inst->passData<RAInst>();
818 : ASMJIT_ASSERT(raInst != nullptr);
819 :
820 : RATiedReg* tiedRegs = raInst->tiedRegs();
821 : uint32_t count = raInst->tiedCount();
822 :
823 946262 : for (uint32_t j = 0; j < count; j++) {
824 614454 : RATiedReg* tiedReg = &tiedRegs[j];
825 : uint32_t workId = tiedReg->workId();
826 :
827 : // Update `nUses` and `nOuts`.
828 614454 : nUsesPerWorkReg[workId] += 1u;
829 614454 : nOutsPerWorkReg[workId] += uint32_t(tiedReg->isWrite());
830 :
831 : // Mark as:
832 : // KILL - if this VirtReg is killed afterwards.
833 : // LAST - if this VirtReg is last in this basic block.
834 614454 : if (block->kill().bitAt(workId))
835 : tiedReg->addFlags(RATiedFlags::kKill);
836 602594 : else if (!block->gen().bitAt(workId))
837 : tiedReg->addFlags(RATiedFlags::kLast);
838 :
839 614454 : if (tiedReg->isWriteOnly()) {
840 : // KILL.
841 : block->kill().setBit(workId, true);
842 : }
843 : else {
844 : // GEN.
845 : block->kill().setBit(workId, false);
846 : block->gen().setBit(workId, true);
847 : }
848 :
849 614454 : if (tiedReg->isLeadConsecutive()) {
850 : RAWorkReg* workReg = workRegById(workId);
851 : workReg->markLeadConsecutive();
852 : }
853 :
854 614454 : if (tiedReg->hasConsecutiveParent()) {
855 : RAWorkReg* consecutiveParentReg = workRegById(tiedReg->consecutiveParent());
856 0 : consecutiveParentReg->addImmediateConsecutive(allocator(), workId);
857 : }
858 : }
859 :
860 331808 : nInsts++;
861 : }
862 :
863 386792 : if (node == stop)
864 : break;
865 :
866 : node = node->prev();
867 : ASMJIT_ASSERT(node != nullptr);
868 331808 : }
869 :
870 54984 : nInstsPerBlock[block->blockId()] = nInsts;
871 : }
872 :
873 : // Calculate IN/OUT of Each Block
874 : // ------------------------------
875 :
876 : {
877 : ZoneStack<RABlock*> workList;
878 : ZoneBitVector workBits;
879 :
880 27492 : ASMJIT_PROPAGATE(workList.init(allocator()));
881 27492 : ASMJIT_PROPAGATE(workBits.resize(allocator(), blockCount(), true));
882 :
883 82476 : for (i = 0; i < numReachableBlocks; i++) {
884 54984 : RABlock* block = _pov[i];
885 : LiveOps::recalcInOut(block, numBitWords, true);
886 54984 : ASMJIT_PROPAGATE(workList.append(block));
887 : }
888 :
889 82476 : while (!workList.empty()) {
890 54984 : RABlock* block = workList.popFirst();
891 : uint32_t blockId = block->blockId();
892 :
893 : workBits.setBit(blockId, false);
894 0 : if (LiveOps::recalcInOut(block, numBitWords)) {
895 : const RABlocks& predecessors = block->predecessors();
896 : uint32_t numPredecessors = predecessors.size();
897 :
898 0 : for (uint32_t j = 0; j < numPredecessors; j++) {
899 0 : RABlock* pred = predecessors[j];
900 0 : if (!workBits.bitAt(pred->blockId())) {
901 : workBits.setBit(pred->blockId(), true);
902 0 : ASMJIT_PROPAGATE(workList.append(pred));
903 : }
904 : }
905 : }
906 54984 : numVisits++;
907 : }
908 :
909 : workList.reset();
910 : workBits.release(allocator());
911 : }
912 :
913 27492 : ASMJIT_RA_LOG_COMPLEX({
914 : logger->logf(" LiveIn/Out Done (%u visits)\n", numVisits);
915 : for (i = 0; i < numAllBlocks; i++) {
916 : RABlock* block = _blocks[i];
917 :
918 : ASMJIT_PROPAGATE(sb.assignFormat(" {#%u}\n", block->blockId()));
919 : ASMJIT_PROPAGATE(_dumpBlockLiveness(sb, block));
920 :
921 : logger->log(sb);
922 : }
923 : });
924 :
925 : // Reserve the space in each `RAWorkReg` for references
926 : // ----------------------------------------------------
927 :
928 298806 : for (i = 0; i < numWorkRegs; i++) {
929 : RAWorkReg* workReg = workRegById(i);
930 542628 : ASMJIT_PROPAGATE(workReg->_refs.reserve(allocator(), nUsesPerWorkReg[i]));
931 542628 : ASMJIT_PROPAGATE(workReg->_writes.reserve(allocator(), nOutsPerWorkReg[i]));
932 : }
933 :
934 : // Assign block and instruction positions, build LiveCount and LiveSpans
935 : // ---------------------------------------------------------------------
936 :
937 : uint32_t position = 2;
938 82476 : for (i = 0; i < numAllBlocks; i++) {
939 54984 : RABlock* block = _blocks[i];
940 54984 : if (!block->isReachable())
941 0 : continue;
942 :
943 54984 : BaseNode* node = block->first();
944 : BaseNode* stop = block->last();
945 :
946 54984 : uint32_t endPosition = position + nInstsPerBlock[i] * 2;
947 : block->setFirstPosition(position);
948 : block->setEndPosition(endPosition);
949 :
950 54984 : RALiveCount curLiveCount;
951 54984 : RALiveCount maxLiveCount;
952 :
953 : // Process LIVE-IN.
954 : ZoneBitVector::ForEachBitSet it(block->liveIn());
955 54984 : while (it.hasNext()) {
956 0 : RAWorkReg* workReg = _workRegs[uint32_t(it.next())];
957 0 : curLiveCount[workReg->group()]++;
958 0 : ASMJIT_PROPAGATE(workReg->liveSpans().openAt(allocator(), position, endPosition));
959 : }
960 :
961 : for (;;) {
962 386792 : if (node->isInst()) {
963 : InstNode* inst = node->as<InstNode>();
964 : RAInst* raInst = inst->passData<RAInst>();
965 : ASMJIT_ASSERT(raInst != nullptr);
966 :
967 : RATiedReg* tiedRegs = raInst->tiedRegs();
968 : uint32_t count = raInst->tiedCount();
969 :
970 : inst->setPosition(position);
971 331808 : raInst->_liveCount = curLiveCount;
972 :
973 946262 : for (uint32_t j = 0; j < count; j++) {
974 614454 : RATiedReg* tiedReg = &tiedRegs[j];
975 : uint32_t workId = tiedReg->workId();
976 :
977 : // Create refs and writes.
978 : RAWorkReg* workReg = workRegById(workId);
979 : workReg->_refs.appendUnsafe(node);
980 614454 : if (tiedReg->isWrite())
981 : workReg->_writes.appendUnsafe(node);
982 :
983 : // We couldn't calculate this in previous steps, but since we know all LIVE-OUT at this point it becomes
984 : // trivial. If this is the last instruction that uses this `workReg` and it's not LIVE-OUT then it is
985 : // KILLed here.
986 614454 : if (tiedReg->isLast() && !block->liveOut().bitAt(workId))
987 : tiedReg->addFlags(RATiedFlags::kKill);
988 :
989 : LiveRegSpans& liveSpans = workReg->liveSpans();
990 : bool wasOpen;
991 1228908 : ASMJIT_PROPAGATE(liveSpans.openAt(allocator(), position + !tiedReg->isRead(), endPosition, wasOpen));
992 :
993 : RegGroup group = workReg->group();
994 614454 : if (!wasOpen) {
995 283174 : curLiveCount[group]++;
996 283174 : raInst->_liveCount[group]++;
997 : }
998 :
999 614454 : if (tiedReg->isKill()) {
1000 283174 : liveSpans.closeAt(position + !tiedReg->isRead() + 1);
1001 283174 : curLiveCount[group]--;
1002 : }
1003 :
1004 : // Update `RAWorkReg::useIdMask` and `RAWorkReg::hintRegId`.
1005 614454 : if (tiedReg->hasUseId()) {
1006 : uint32_t useId = tiedReg->useId();
1007 : workReg->addUseIdMask(Support::bitMask(useId));
1008 37365 : if (!workReg->hasHintRegId() && !Support::bitTest(raInst->_clobberedRegs[group], useId))
1009 : workReg->setHintRegId(useId);
1010 : }
1011 :
1012 614454 : if (tiedReg->useRegMask()) {
1013 : workReg->restrictPreferredMask(tiedReg->useRegMask());
1014 604581 : if (workReg->isLeadConsecutive())
1015 : workReg->restrictConsecutiveMask(tiedReg->useRegMask());
1016 : }
1017 :
1018 614454 : if (tiedReg->outRegMask()) {
1019 : workReg->restrictPreferredMask(tiedReg->outRegMask());
1020 614454 : if (workReg->isLeadConsecutive())
1021 : workReg->restrictConsecutiveMask(tiedReg->outRegMask());
1022 : }
1023 :
1024 : // Update `RAWorkReg::clobberedSurvivalMask`.
1025 614454 : if (raInst->_clobberedRegs[group] && !tiedReg->isOutOrKill()) {
1026 : workReg->addClobberSurvivalMask(raInst->_clobberedRegs[group]);
1027 : }
1028 : }
1029 :
1030 331808 : position += 2;
1031 : maxLiveCount.op<Support::Max>(raInst->_liveCount);
1032 : }
1033 :
1034 386792 : if (node == stop)
1035 : break;
1036 :
1037 331808 : node = node->next();
1038 : ASMJIT_ASSERT(node != nullptr);
1039 331808 : }
1040 :
1041 54984 : block->_maxLiveCount = maxLiveCount;
1042 : _globalMaxLiveCount.op<Support::Max>(maxLiveCount);
1043 : ASMJIT_ASSERT(position == block->endPosition());
1044 : }
1045 :
1046 : // Calculate WorkReg statistics
1047 : // ----------------------------
1048 :
1049 298806 : for (i = 0; i < numWorkRegs; i++) {
1050 271314 : RAWorkReg* workReg = _workRegs[i];
1051 :
1052 : LiveRegSpans& spans = workReg->liveSpans();
1053 : uint32_t width = spans.width();
1054 271314 : float freq = width ? float(double(workReg->_refs.size()) / double(width)) : float(0);
1055 :
1056 : RALiveStats& stats = workReg->liveStats();
1057 271314 : stats._width = width;
1058 271314 : stats._freq = freq;
1059 271314 : stats._priority = freq + float(int(workReg->virtReg()->weight())) * 0.01f;
1060 : }
1061 :
1062 27492 : ASMJIT_RA_LOG_COMPLEX({
1063 : sb.clear();
1064 : _dumpLiveSpans(sb);
1065 : logger->log(sb);
1066 : });
1067 :
1068 : nUsesPerWorkReg.release(allocator());
1069 : nOutsPerWorkReg.release(allocator());
1070 : nInstsPerBlock.release(allocator());
1071 :
1072 27492 : return kErrorOk;
1073 : }
1074 :
1075 27492 : Error BaseRAPass::assignArgIndexToWorkRegs() noexcept {
1076 : ZoneBitVector& liveIn = entryBlock()->liveIn();
1077 : uint32_t argCount = func()->argCount();
1078 :
1079 27492 : for (uint32_t argIndex = 0; argIndex < argCount; argIndex++) {
1080 0 : for (uint32_t valueIndex = 0; valueIndex < Globals::kMaxValuePack; valueIndex++) {
1081 : // Unassigned argument.
1082 0 : const RegOnly& regArg = func()->argPack(argIndex)[valueIndex];
1083 0 : if (!regArg.isReg() || !cc()->isVirtIdValid(regArg.id()))
1084 0 : continue;
1085 :
1086 : VirtReg* virtReg = cc()->virtRegById(regArg.id());
1087 0 : if (!virtReg)
1088 0 : continue;
1089 :
1090 : // Unreferenced argument.
1091 : RAWorkReg* workReg = virtReg->workReg();
1092 0 : if (!workReg)
1093 0 : continue;
1094 :
1095 : // Overwritten argument.
1096 : uint32_t workId = workReg->workId();
1097 0 : if (!liveIn.bitAt(workId))
1098 0 : continue;
1099 :
1100 : workReg->setArgIndex(argIndex, valueIndex);
1101 : const FuncValue& arg = func()->detail().arg(argIndex, valueIndex);
1102 :
1103 0 : if (arg.isReg() && _archTraits->regTypeToGroup(arg.regType()) == workReg->group()) {
1104 : workReg->setHintRegId(arg.regId());
1105 : }
1106 : }
1107 : }
1108 :
1109 27492 : return kErrorOk;
1110 : }
1111 :
1112 : // BaseRAPass - Allocation - Global
1113 : // ================================
1114 :
1115 : #ifndef ASMJIT_NO_LOGGING
1116 0 : static void RAPass_dumpSpans(String& sb, uint32_t index, const LiveRegSpans& liveSpans) noexcept {
1117 : sb.appendFormat(" %02u: ", index);
1118 :
1119 0 : for (uint32_t i = 0; i < liveSpans.size(); i++) {
1120 : const LiveRegSpan& liveSpan = liveSpans[i];
1121 0 : if (i) sb.append(", ");
1122 0 : sb.appendFormat("[%u:%u@%u]", liveSpan.a, liveSpan.b, liveSpan.id);
1123 : }
1124 :
1125 : sb.append('\n');
1126 0 : }
1127 : #endif
1128 :
1129 27492 : Error BaseRAPass::runGlobalAllocator() noexcept {
1130 27492 : ASMJIT_PROPAGATE(initGlobalLiveSpans());
1131 :
1132 137460 : for (RegGroup group : RegGroupVirtValues{}) {
1133 109968 : ASMJIT_PROPAGATE(binPack(group));
1134 : }
1135 :
1136 : return kErrorOk;
1137 : }
1138 :
1139 27492 : ASMJIT_FAVOR_SPEED Error BaseRAPass::initGlobalLiveSpans() noexcept {
1140 137460 : for (RegGroup group : RegGroupVirtValues{}) {
1141 109968 : size_t physCount = _physRegCount[group];
1142 : LiveRegSpans* liveSpans = nullptr;
1143 :
1144 109968 : if (physCount) {
1145 109968 : liveSpans = allocator()->allocT<LiveRegSpans>(physCount * sizeof(LiveRegSpans));
1146 109968 : if (ASMJIT_UNLIKELY(!liveSpans))
1147 : return DebugUtils::errored(kErrorOutOfMemory);
1148 :
1149 1429584 : for (size_t physId = 0; physId < physCount; physId++)
1150 1319616 : new(&liveSpans[physId]) LiveRegSpans();
1151 : }
1152 :
1153 109968 : _globalLiveSpans[group] = liveSpans;
1154 : }
1155 :
1156 : return kErrorOk;
1157 : }
1158 :
1159 : struct RAConsecutiveReg {
1160 : RAWorkReg* workReg;
1161 : RAWorkReg* parentReg;
1162 : };
1163 :
1164 109968 : ASMJIT_FAVOR_SPEED Error BaseRAPass::binPack(RegGroup group) noexcept {
1165 109968 : if (workRegCount(group) == 0)
1166 : return kErrorOk;
1167 :
1168 : #ifndef ASMJIT_NO_LOGGING
1169 : Logger* logger = getLoggerIf(DiagnosticOptions::kRADebugAssignment);
1170 : StringTmp<512> sb;
1171 :
1172 54984 : ASMJIT_RA_LOG_FORMAT("[BinPack] Available=%u (0x%08X) Count=%u RegGroup=%u\n",
1173 : Support::popcnt(_availableRegs[group]),
1174 : _availableRegs[group],
1175 : workRegCount(group),
1176 : uint32_t(group));
1177 : #endif
1178 :
1179 : uint32_t i;
1180 54984 : uint32_t physCount = _physRegCount[group];
1181 :
1182 : RAWorkRegs workRegs;
1183 : ZoneVector<RAConsecutiveReg> consecutiveRegs;
1184 : LiveRegSpans tmpSpans;
1185 :
1186 54984 : ASMJIT_PROPAGATE(workRegs.concat(allocator(), this->workRegs(group)));
1187 54984 : workRegs.sort([](const RAWorkReg* a, const RAWorkReg* b) noexcept {
1188 669150 : return b->liveStats().priority() - a->liveStats().priority();
1189 : });
1190 :
1191 : uint32_t numWorkRegs = workRegs.size();
1192 54984 : RegMask availableRegs = _availableRegs[group];
1193 :
1194 : // First try to pack everything that provides register-id hint as these are most likely function arguments and fixed
1195 : // (precolored) virtual registers.
1196 54984 : if (!workRegs.empty()) {
1197 : uint32_t dstIndex = 0;
1198 :
1199 326298 : for (i = 0; i < numWorkRegs; i++) {
1200 271314 : RAWorkReg* workReg = workRegs[i];
1201 :
1202 271314 : if (workReg->isLeadConsecutive()) {
1203 0 : ASMJIT_PROPAGATE(consecutiveRegs.append(allocator(), RAConsecutiveReg{workReg, nullptr}));
1204 : workReg->markProcessedConsecutive();
1205 : }
1206 :
1207 271314 : if (workReg->hasHintRegId()) {
1208 : uint32_t physId = workReg->hintRegId();
1209 27492 : if (Support::bitTest(availableRegs, physId)) {
1210 27492 : LiveRegSpans& live = _globalLiveSpans[group][physId];
1211 : Error err = tmpSpans.nonOverlappingUnionOf(allocator(), live, workReg->liveSpans(), LiveRegData(workReg->virtId()));
1212 :
1213 27492 : if (err == kErrorOk) {
1214 : live.swap(tmpSpans);
1215 : workReg->setHomeRegId(physId);
1216 : workReg->markAllocated();
1217 27492 : continue;
1218 : }
1219 :
1220 0 : if (err != 0xFFFFFFFFu)
1221 0 : return err;
1222 : }
1223 : }
1224 :
1225 243822 : workRegs[dstIndex++] = workReg;
1226 : }
1227 :
1228 : workRegs._setSize(dstIndex);
1229 : numWorkRegs = dstIndex;
1230 : }
1231 :
1232 : // Allocate consecutive registers - both leads and all consecutives. This is important and prioritized over the rest,
1233 : // because once a lead is allocated we really need to allocate its consecutives, otherwise we may bin pack other
1234 : // registers into their places, which would result in wrong hints to the local allocator, and then into many moves
1235 : // or spills.
1236 54984 : if (!consecutiveRegs.empty()) {
1237 : // This loop appends all other consecutive registers into `consecutiveRegs` array. Leads are at the beginning,
1238 : // non-leads follow.
1239 : i = 0;
1240 : for (;;) {
1241 : uint32_t stop = consecutiveRegs.size();
1242 0 : if (i == stop)
1243 : break;
1244 :
1245 0 : while (i < stop) {
1246 0 : RAWorkReg* workReg = consecutiveRegs[i].workReg;
1247 0 : if (workReg->hasImmediateConsecutives()) {
1248 : ZoneBitVector::ForEachBitSet it(workReg->immediateConsecutives());
1249 0 : while (it.hasNext()) {
1250 : uint32_t consecutiveWorkId = uint32_t(it.next());
1251 : RAWorkReg* consecutiveReg = workRegById(consecutiveWorkId);
1252 0 : if (!consecutiveReg->isProcessedConsecutive()) {
1253 0 : ASMJIT_PROPAGATE(consecutiveRegs.append(allocator(), RAConsecutiveReg{consecutiveReg, workReg}));
1254 : consecutiveReg->markProcessedConsecutive();
1255 : }
1256 : }
1257 : }
1258 0 : i++;
1259 : }
1260 : }
1261 :
1262 : uint32_t numConsecutiveRegs = consecutiveRegs.size();
1263 0 : for (i = 0; i < numConsecutiveRegs; i++) {
1264 0 : RAWorkReg* workReg = consecutiveRegs[i].workReg;
1265 0 : if (workReg->isAllocated())
1266 0 : continue;
1267 :
1268 0 : RAWorkReg* parentReg = consecutiveRegs[i].parentReg;
1269 : RegMask physRegs = 0;
1270 :
1271 0 : if (!parentReg) {
1272 0 : physRegs = availableRegs & workReg->preferredMask();
1273 0 : if (!physRegs) {
1274 0 : physRegs = availableRegs & workReg->consecutiveMask();
1275 :
1276 : // NOTE: This should never be true as it would mean we would never allocate this virtual register
1277 : // (not here, and not later when local register allocator processes RATiedReg sets).
1278 0 : if (ASMJIT_UNLIKELY(!physRegs))
1279 : return DebugUtils::errored(kErrorConsecutiveRegsAllocation);
1280 : }
1281 : }
1282 0 : else if (parentReg->hasHomeRegId()) {
1283 0 : uint32_t consecutiveId = parentReg->homeRegId() + 1;
1284 :
1285 : // NOTE: We don't support wrapping. If this goes beyond all allocable registers there is something wrong.
1286 0 : if (consecutiveId > 31 || !Support::bitTest(availableRegs, consecutiveId))
1287 : return DebugUtils::errored(kErrorConsecutiveRegsAllocation);
1288 :
1289 : workReg->setHintRegId(consecutiveId);
1290 : physRegs = Support::bitMask(consecutiveId);
1291 : }
1292 :
1293 0 : while (physRegs) {
1294 0 : uint32_t physId = Support::bitSizeOf<RegMask>() - 1 - Support::clz(physRegs);
1295 :
1296 0 : LiveRegSpans& live = _globalLiveSpans[group][physId];
1297 : Error err = tmpSpans.nonOverlappingUnionOf(allocator(), live, workReg->liveSpans(), LiveRegData(workReg->virtId()));
1298 :
1299 : if (err == kErrorOk) {
1300 : workReg->setHomeRegId(physId);
1301 : workReg->markAllocated();
1302 : live.swap(tmpSpans);
1303 : break;
1304 : }
1305 :
1306 0 : if (ASMJIT_UNLIKELY(err != 0xFFFFFFFFu))
1307 : return err;
1308 :
1309 0 : physRegs ^= Support::bitMask(physId);
1310 : }
1311 : }
1312 : }
1313 :
1314 : // Try to pack the rest.
1315 54984 : if (!workRegs.empty()) {
1316 : uint32_t dstIndex = 0;
1317 :
1318 295549 : for (i = 0; i < numWorkRegs; i++) {
1319 243822 : RAWorkReg* workReg = workRegs[i];
1320 :
1321 243822 : if (workReg->isAllocated())
1322 0 : continue;
1323 :
1324 : RegMask physRegs = availableRegs;
1325 243822 : if (physRegs & workReg->preferredMask())
1326 : physRegs &= workReg->preferredMask();
1327 :
1328 555246 : while (physRegs) {
1329 : RegMask preferredMask = physRegs;
1330 : uint32_t physId = Support::ctz(preferredMask);
1331 :
1332 544801 : if (workReg->clobberSurvivalMask()) {
1333 1002 : preferredMask &= workReg->clobberSurvivalMask();
1334 1002 : if (preferredMask)
1335 : physId = Support::ctz(preferredMask);
1336 : }
1337 :
1338 544801 : LiveRegSpans& live = _globalLiveSpans[group][physId];
1339 : Error err = tmpSpans.nonOverlappingUnionOf(allocator(), live, workReg->liveSpans(), LiveRegData(workReg->virtId()));
1340 :
1341 : if (err == kErrorOk) {
1342 : workReg->setHomeRegId(physId);
1343 : workReg->markAllocated();
1344 : live.swap(tmpSpans);
1345 : break;
1346 : }
1347 :
1348 311424 : if (ASMJIT_UNLIKELY(err != 0xFFFFFFFFu))
1349 : return err;
1350 :
1351 311424 : physRegs ^= Support::bitMask(physId);
1352 : }
1353 :
1354 : // Keep it in `workRegs` if it was not allocated.
1355 243822 : if (!physRegs)
1356 10445 : workRegs[dstIndex++] = workReg;
1357 : }
1358 :
1359 : workRegs._setSize(dstIndex);
1360 : numWorkRegs = dstIndex;
1361 : }
1362 :
1363 54984 : ASMJIT_RA_LOG_COMPLEX({
1364 : for (uint32_t physId = 0; physId < physCount; physId++) {
1365 : LiveRegSpans& live = _globalLiveSpans[group][physId];
1366 : if (live.empty())
1367 : continue;
1368 :
1369 : sb.clear();
1370 : RAPass_dumpSpans(sb, physId, live);
1371 : logger->log(sb);
1372 : }
1373 : });
1374 :
1375 : // Maybe unused if logging is disabled.
1376 : DebugUtils::unused(physCount);
1377 :
1378 54984 : if (workRegs.empty()) {
1379 53029 : ASMJIT_RA_LOG_FORMAT(" Completed.\n");
1380 : }
1381 : else {
1382 : _strategy[group].setType(RAStrategyType::kComplex);
1383 12400 : for (RAWorkReg* workReg : workRegs)
1384 : workReg->markStackPreferred();
1385 :
1386 1955 : ASMJIT_RA_LOG_COMPLEX({
1387 : uint32_t count = workRegs.size();
1388 : sb.clear();
1389 : sb.appendFormat(" Unassigned (%u): ", count);
1390 : for (i = 0; i < numWorkRegs; i++) {
1391 : RAWorkReg* workReg = workRegs[i];
1392 : if (i) sb.append(", ");
1393 : sb.append(workReg->name());
1394 : }
1395 : sb.append('\n');
1396 : logger->log(sb);
1397 : });
1398 : }
1399 :
1400 : return kErrorOk;
1401 : }
1402 :
1403 : // BaseRAPass - Allocation - Local
1404 : // ===============================
1405 :
1406 27492 : Error BaseRAPass::runLocalAllocator() noexcept {
1407 27492 : RALocalAllocator lra(this);
1408 27492 : ASMJIT_PROPAGATE(lra.init());
1409 :
1410 27492 : if (!blockCount())
1411 : return kErrorOk;
1412 :
1413 : // The allocation is done when this reaches zero.
1414 : uint32_t blocksRemaining = reachableBlockCount();
1415 :
1416 : // Current block.
1417 : uint32_t blockId = 0;
1418 27492 : RABlock* block = _blocks[blockId];
1419 :
1420 : // The first block (entry) must always be reachable.
1421 : ASMJIT_ASSERT(block->isReachable());
1422 :
1423 : // Assign function arguments for the initial block. The `lra` is valid now.
1424 27492 : lra.makeInitialAssignment();
1425 27492 : ASMJIT_PROPAGATE(setBlockEntryAssignment(block, block, lra._curAssignment));
1426 :
1427 : // The loop starts from the first block and iterates blocks in order, however, the algorithm also allows to jump to
1428 : // any other block when finished if it's a jump target. In-order iteration just makes sure that all blocks are visited.
1429 : for (;;) {
1430 : BaseNode* first = block->first();
1431 : BaseNode* last = block->last();
1432 54984 : BaseNode* terminator = block->hasTerminator() ? last : nullptr;
1433 :
1434 : BaseNode* beforeFirst = first->prev();
1435 : BaseNode* afterLast = last->next();
1436 :
1437 : bool unconditionalJump = false;
1438 : RABlock* consecutive = nullptr;
1439 :
1440 54984 : if (block->hasSuccessors())
1441 27492 : consecutive = block->successors()[0];
1442 :
1443 : lra.setBlock(block);
1444 : block->makeAllocated();
1445 :
1446 : BaseNode* node = first;
1447 441776 : while (node != afterLast) {
1448 : BaseNode* next = node->next();
1449 386792 : if (node->isInst()) {
1450 : InstNode* inst = node->as<InstNode>();
1451 :
1452 331808 : if (ASMJIT_UNLIKELY(inst == terminator)) {
1453 : const RABlocks& successors = block->successors();
1454 0 : if (block->hasConsecutive()) {
1455 0 : ASMJIT_PROPAGATE(lra.allocBranch(inst, successors.last(), successors.first()));
1456 :
1457 : node = next;
1458 0 : continue;
1459 0 : }
1460 0 : else if (successors.size() > 1) {
1461 0 : RABlock* cont = block->hasConsecutive() ? successors.first() : nullptr;
1462 0 : ASMJIT_PROPAGATE(lra.allocJumpTable(inst, successors, cont));
1463 :
1464 : node = next;
1465 0 : continue;
1466 0 : }
1467 : else {
1468 : // Otherwise this is an unconditional jump, special handling isn't required.
1469 : unconditionalJump = true;
1470 : }
1471 : }
1472 :
1473 331808 : ASMJIT_PROPAGATE(lra.allocInst(inst));
1474 331808 : if (inst->type() == NodeType::kInvoke)
1475 9639 : ASMJIT_PROPAGATE(emitPreCall(inst->as<InvokeNode>()));
1476 : else
1477 322169 : ASMJIT_PROPAGATE(lra.spillAfterAllocation(inst));
1478 : }
1479 : node = next;
1480 : }
1481 :
1482 54984 : if (consecutive) {
1483 27492 : BaseNode* prev = afterLast ? afterLast->prev() : cc()->lastNode();
1484 27492 : cc()->_setCursor(unconditionalJump ? prev->prev() : prev);
1485 :
1486 27492 : if (consecutive->hasEntryAssignment()) {
1487 0 : ASMJIT_PROPAGATE(lra.switchToAssignment(consecutive->entryPhysToWorkMap(), consecutive->liveIn(), consecutive->isAllocated(), false));
1488 : }
1489 : else {
1490 27492 : ASMJIT_PROPAGATE(lra.spillRegsBeforeEntry(consecutive));
1491 27492 : ASMJIT_PROPAGATE(setBlockEntryAssignment(consecutive, block, lra._curAssignment));
1492 27492 : lra._curAssignment.copyFrom(consecutive->entryPhysToWorkMap());
1493 : }
1494 : }
1495 :
1496 : // Important as the local allocator can insert instructions before
1497 : // and after any instruction within the basic block.
1498 : block->setFirst(beforeFirst->next());
1499 54984 : block->setLast(afterLast ? afterLast->prev() : cc()->lastNode());
1500 :
1501 54984 : if (--blocksRemaining == 0)
1502 : break;
1503 :
1504 : // Switch to the next consecutive block, if any.
1505 27492 : if (consecutive) {
1506 : block = consecutive;
1507 27492 : if (!block->isAllocated())
1508 27492 : continue;
1509 : }
1510 :
1511 : // Get the next block.
1512 : for (;;) {
1513 0 : if (++blockId >= blockCount())
1514 : blockId = 0;
1515 :
1516 0 : block = _blocks[blockId];
1517 0 : if (!block->isReachable() || block->isAllocated() || !block->hasEntryAssignment())
1518 : continue;
1519 :
1520 : break;
1521 : }
1522 :
1523 : // If we switched to some block we have to update the local allocator.
1524 0 : lra.replaceAssignment(block->entryPhysToWorkMap());
1525 : }
1526 :
1527 : _clobberedRegs.op<Support::Or>(lra._clobberedRegs);
1528 : return kErrorOk;
1529 : }
1530 :
1531 54984 : Error BaseRAPass::setBlockEntryAssignment(RABlock* block, const RABlock* fromBlock, const RAAssignment& fromAssignment) noexcept {
1532 54984 : if (block->hasSharedAssignmentId()) {
1533 : uint32_t sharedAssignmentId = block->sharedAssignmentId();
1534 :
1535 : // Shouldn't happen. Entry assignment of a block that has a shared-state will assign to all blocks
1536 : // with the same sharedAssignmentId. It's a bug if the shared state has been already assigned.
1537 0 : if (!_sharedAssignments[sharedAssignmentId].empty())
1538 : return DebugUtils::errored(kErrorInvalidState);
1539 :
1540 0 : return setSharedAssignment(sharedAssignmentId, fromAssignment);
1541 : }
1542 :
1543 54984 : PhysToWorkMap* physToWorkMap = clonePhysToWorkMap(fromAssignment.physToWorkMap());
1544 54984 : if (ASMJIT_UNLIKELY(!physToWorkMap))
1545 : return DebugUtils::errored(kErrorOutOfMemory);
1546 :
1547 : block->setEntryAssignment(physToWorkMap);
1548 :
1549 : // True if this is the first (entry) block, nothing to do in this case.
1550 54984 : if (block == fromBlock) {
1551 : // Entry block should never have a shared state.
1552 27492 : if (block->hasSharedAssignmentId())
1553 : return DebugUtils::errored(kErrorInvalidState);
1554 :
1555 27492 : return kErrorOk;
1556 : }
1557 :
1558 : const ZoneBitVector& liveOut = fromBlock->liveOut();
1559 : const ZoneBitVector& liveIn = block->liveIn();
1560 :
1561 : // It's possible that `fromBlock` has LIVE-OUT regs that `block` doesn't
1562 : // have in LIVE-IN, these have to be unassigned.
1563 : {
1564 27492 : ZoneBitVector::ForEachBitOp<Support::AndNot> it(liveOut, liveIn);
1565 54984 : while (it.hasNext()) {
1566 0 : uint32_t workId = uint32_t(it.next());
1567 : RAWorkReg* workReg = workRegById(workId);
1568 :
1569 : RegGroup group = workReg->group();
1570 : uint32_t physId = fromAssignment.workToPhysId(group, workId);
1571 :
1572 0 : if (physId != RAAssignment::kPhysNone)
1573 0 : physToWorkMap->unassign(group, physId, _physRegIndex.get(group) + physId);
1574 : }
1575 : }
1576 :
1577 27492 : return blockEntryAssigned(physToWorkMap);
1578 : }
1579 :
1580 0 : Error BaseRAPass::setSharedAssignment(uint32_t sharedAssignmentId, const RAAssignment& fromAssignment) noexcept {
1581 : ASMJIT_ASSERT(_sharedAssignments[sharedAssignmentId].empty());
1582 :
1583 0 : PhysToWorkMap* physToWorkMap = clonePhysToWorkMap(fromAssignment.physToWorkMap());
1584 0 : if (ASMJIT_UNLIKELY(!physToWorkMap))
1585 : return DebugUtils::errored(kErrorOutOfMemory);
1586 :
1587 0 : _sharedAssignments[sharedAssignmentId].assignPhysToWorkMap(physToWorkMap);
1588 :
1589 0 : ZoneBitVector& sharedLiveIn = _sharedAssignments[sharedAssignmentId]._liveIn;
1590 0 : ASMJIT_PROPAGATE(sharedLiveIn.resize(allocator(), workRegCount()));
1591 :
1592 0 : Support::Array<uint32_t, Globals::kNumVirtGroups> sharedAssigned {};
1593 0 : for (RABlock* block : blocks()) {
1594 0 : if (block->sharedAssignmentId() == sharedAssignmentId) {
1595 : ASMJIT_ASSERT(!block->hasEntryAssignment());
1596 :
1597 0 : PhysToWorkMap* entryPhysToWorkMap = clonePhysToWorkMap(fromAssignment.physToWorkMap());
1598 0 : if (ASMJIT_UNLIKELY(!entryPhysToWorkMap))
1599 : return DebugUtils::errored(kErrorOutOfMemory);
1600 :
1601 : block->setEntryAssignment(entryPhysToWorkMap);
1602 :
1603 : const ZoneBitVector& liveIn = block->liveIn();
1604 : sharedLiveIn.or_(liveIn);
1605 :
1606 0 : for (RegGroup group : RegGroupVirtValues{}) {
1607 0 : sharedAssigned[group] |= entryPhysToWorkMap->assigned[group];
1608 :
1609 : uint32_t physBaseIndex = _physRegIndex.get(group);
1610 : Support::BitWordIterator<RegMask> it(entryPhysToWorkMap->assigned[group]);
1611 :
1612 0 : while (it.hasNext()) {
1613 : uint32_t physId = it.next();
1614 0 : uint32_t workId = entryPhysToWorkMap->workIds[physBaseIndex + physId];
1615 :
1616 0 : if (!liveIn.bitAt(workId))
1617 0 : entryPhysToWorkMap->unassign(group, physId, physBaseIndex + physId);
1618 : }
1619 : }
1620 : }
1621 : }
1622 :
1623 0 : for (RegGroup group : RegGroupVirtValues{}) {
1624 : uint32_t physBaseIndex = _physRegIndex.get(group);
1625 0 : Support::BitWordIterator<RegMask> it(_availableRegs[group] & ~sharedAssigned[group]);
1626 :
1627 0 : while (it.hasNext()) {
1628 : uint32_t physId = it.next();
1629 0 : if (Support::bitTest(physToWorkMap->assigned[group], physId))
1630 0 : physToWorkMap->unassign(group, physId, physBaseIndex + physId);
1631 : }
1632 : }
1633 :
1634 0 : return blockEntryAssigned(physToWorkMap);
1635 : }
1636 :
1637 27492 : Error BaseRAPass::blockEntryAssigned(const PhysToWorkMap* physToWorkMap) noexcept {
1638 : // Complex allocation strategy requires to record register assignments upon block entry (or per shared state).
1639 137460 : for (RegGroup group : RegGroupVirtValues{}) {
1640 109968 : if (!_strategy[group].isComplex())
1641 : continue;
1642 :
1643 1955 : uint32_t physBaseIndex = _physRegIndex[group];
1644 1955 : Support::BitWordIterator<RegMask> it(physToWorkMap->assigned[group]);
1645 :
1646 1955 : while (it.hasNext()) {
1647 : uint32_t physId = it.next();
1648 0 : uint32_t workId = physToWorkMap->workIds[physBaseIndex + physId];
1649 :
1650 : RAWorkReg* workReg = workRegById(workId);
1651 : workReg->addAllocatedMask(Support::bitMask(physId));
1652 : }
1653 : }
1654 :
1655 27492 : return kErrorOk;
1656 : }
1657 :
1658 : // BaseRAPass - Allocation - Utilities
1659 : // ===================================
1660 :
1661 0 : Error BaseRAPass::useTemporaryMem(BaseMem& out, uint32_t size, uint32_t alignment) noexcept {
1662 : ASMJIT_ASSERT(alignment <= 64);
1663 :
1664 0 : if (_temporaryMem.isNone()) {
1665 0 : ASMJIT_PROPAGATE(cc()->_newStack(&_temporaryMem.as<BaseMem>(), size, alignment));
1666 : }
1667 : else {
1668 : ASMJIT_ASSERT(_temporaryMem.as<BaseMem>().isRegHome());
1669 :
1670 : uint32_t virtId = _temporaryMem.as<BaseMem>().baseId();
1671 : VirtReg* virtReg = cc()->virtRegById(virtId);
1672 :
1673 0 : cc()->setStackSize(virtId, Support::max(virtReg->virtSize(), size),
1674 : Support::max(virtReg->alignment(), alignment));
1675 : }
1676 :
1677 : out = _temporaryMem.as<BaseMem>();
1678 0 : return kErrorOk;
1679 : }
1680 :
1681 : // BaseRAPass - Allocation - Prolog & Epilog
1682 : // =========================================
1683 :
1684 27492 : Error BaseRAPass::updateStackFrame() noexcept {
1685 : // Update some StackFrame information that we updated during allocation. The only information we don't have at the
1686 : // moment is final local stack size, which is calculated last.
1687 : FuncFrame& frame = func()->frame();
1688 137460 : for (RegGroup group : RegGroupVirtValues{})
1689 109968 : frame.addDirtyRegs(group, _clobberedRegs[group]);
1690 : frame.setLocalStackAlignment(_stackAllocator.alignment());
1691 :
1692 : // If there are stack arguments that are not assigned to registers upon entry and the function doesn't require
1693 : // dynamic stack alignment we keep these arguments where they are. This will also mark all stack slots that match
1694 : // these arguments as allocated.
1695 27492 : if (_numStackArgsToStackSlots)
1696 0 : ASMJIT_PROPAGATE(_markStackArgsToKeep());
1697 :
1698 : // Calculate offsets of all stack slots and update StackSize to reflect the calculated local stack size.
1699 27492 : ASMJIT_PROPAGATE(_stackAllocator.calculateStackFrame());
1700 : frame.setLocalStackSize(_stackAllocator.stackSize());
1701 :
1702 : // Update the stack frame based on `_argsAssignment` and finalize it. Finalization means to apply final calculation
1703 : // to the stack layout.
1704 27492 : ASMJIT_PROPAGATE(_argsAssignment.updateFuncFrame(frame));
1705 27492 : ASMJIT_PROPAGATE(frame.finalize());
1706 :
1707 : // StackAllocator allocates all stots starting from [0], adjust them when necessary.
1708 27492 : if (frame.localStackOffset() != 0)
1709 0 : ASMJIT_PROPAGATE(_stackAllocator.adjustSlotOffsets(int32_t(frame.localStackOffset())));
1710 :
1711 : // Again, if there are stack arguments allocated in function's stack we have to handle them. This handles all cases
1712 : // (either regular or dynamic stack alignment).
1713 27492 : if (_numStackArgsToStackSlots)
1714 0 : ASMJIT_PROPAGATE(_updateStackArgs());
1715 :
1716 : return kErrorOk;
1717 : }
1718 :
1719 0 : Error BaseRAPass::_markStackArgsToKeep() noexcept {
1720 : FuncFrame& frame = func()->frame();
1721 0 : bool hasSAReg = frame.hasPreservedFP() || !frame.hasDynamicAlignment();
1722 :
1723 : RAWorkRegs& workRegs = _workRegs;
1724 : uint32_t numWorkRegs = workRegCount();
1725 :
1726 0 : for (uint32_t workId = 0; workId < numWorkRegs; workId++) {
1727 0 : RAWorkReg* workReg = workRegs[workId];
1728 0 : if (workReg->hasFlag(RAWorkRegFlags::kStackArgToStack)) {
1729 : ASMJIT_ASSERT(workReg->hasArgIndex());
1730 0 : const FuncValue& srcArg = _func->detail().arg(workReg->argIndex());
1731 :
1732 : // If the register doesn't have stack slot then we failed. It doesn't make much sense as it was marked as
1733 : // `kFlagStackArgToStack`, which requires the WorkReg was live-in upon function entry.
1734 : RAStackSlot* slot = workReg->stackSlot();
1735 0 : if (ASMJIT_UNLIKELY(!slot))
1736 : return DebugUtils::errored(kErrorInvalidState);
1737 :
1738 0 : if (hasSAReg && srcArg.isStack() && !srcArg.isIndirect()) {
1739 : uint32_t typeSize = TypeUtils::sizeOf(srcArg.typeId());
1740 0 : if (typeSize == slot->size()) {
1741 : slot->addFlags(RAStackSlot::kFlagStackArg);
1742 0 : continue;
1743 : }
1744 : }
1745 :
1746 : // NOTE: Update StackOffset here so when `_argsAssignment.updateFuncFrame()` is called it will take into
1747 : // consideration moving to stack slots. Without this we may miss some scratch registers later.
1748 0 : FuncValue& dstArg = _argsAssignment.arg(workReg->argIndex(), workReg->argValueIndex());
1749 : dstArg.assignStackOffset(0);
1750 : }
1751 : }
1752 :
1753 : return kErrorOk;
1754 : }
1755 :
1756 0 : Error BaseRAPass::_updateStackArgs() noexcept {
1757 : FuncFrame& frame = func()->frame();
1758 : RAWorkRegs& workRegs = _workRegs;
1759 : uint32_t numWorkRegs = workRegCount();
1760 :
1761 0 : for (uint32_t workId = 0; workId < numWorkRegs; workId++) {
1762 0 : RAWorkReg* workReg = workRegs[workId];
1763 0 : if (workReg->hasFlag(RAWorkRegFlags::kStackArgToStack)) {
1764 : ASMJIT_ASSERT(workReg->hasArgIndex());
1765 : RAStackSlot* slot = workReg->stackSlot();
1766 :
1767 0 : if (ASMJIT_UNLIKELY(!slot))
1768 : return DebugUtils::errored(kErrorInvalidState);
1769 :
1770 0 : if (slot->isStackArg()) {
1771 0 : const FuncValue& srcArg = _func->detail().arg(workReg->argIndex());
1772 0 : if (frame.hasPreservedFP()) {
1773 : slot->setBaseRegId(_fp.id());
1774 0 : slot->setOffset(int32_t(frame.saOffsetFromSA()) + srcArg.stackOffset());
1775 : }
1776 : else {
1777 0 : slot->setOffset(int32_t(frame.saOffsetFromSP()) + srcArg.stackOffset());
1778 : }
1779 : }
1780 : else {
1781 0 : FuncValue& dstArg = _argsAssignment.arg(workReg->argIndex(), workReg->argValueIndex());
1782 : dstArg.setStackOffset(slot->offset());
1783 : }
1784 : }
1785 : }
1786 :
1787 : return kErrorOk;
1788 : }
1789 :
1790 27492 : Error BaseRAPass::insertPrologEpilog() noexcept {
1791 27492 : FuncFrame& frame = _func->frame();
1792 :
1793 : cc()->_setCursor(func());
1794 27492 : ASMJIT_PROPAGATE(cc()->emitProlog(frame));
1795 27492 : ASMJIT_PROPAGATE(_iEmitHelper->emitArgsAssignment(frame, _argsAssignment));
1796 :
1797 : cc()->_setCursor(func()->exitNode());
1798 27492 : ASMJIT_PROPAGATE(cc()->emitEpilog(frame));
1799 :
1800 : return kErrorOk;
1801 : }
1802 :
1803 : // BaseRAPass - Rewriter
1804 : // =====================
1805 :
1806 27492 : Error BaseRAPass::rewrite() noexcept {
1807 27492 : return _rewrite(_func, _stop);
1808 : }
1809 :
1810 : // BaseRAPass - Logging
1811 : // ====================
1812 :
1813 : #ifndef ASMJIT_NO_LOGGING
1814 0 : static void RAPass_formatLiveness(BaseRAPass* pass, String& sb, const RAInst* raInst) noexcept {
1815 : const RATiedReg* tiedRegs = raInst->tiedRegs();
1816 : uint32_t tiedCount = raInst->tiedCount();
1817 :
1818 0 : for (uint32_t i = 0; i < tiedCount; i++) {
1819 0 : const RATiedReg& tiedReg = tiedRegs[i];
1820 :
1821 0 : if (i != 0)
1822 : sb.append(' ');
1823 :
1824 : sb.appendFormat("%s{", pass->workRegById(tiedReg.workId())->name());
1825 0 : sb.append(tiedReg.isReadWrite() ? 'X' :
1826 : tiedReg.isRead() ? 'R' :
1827 : tiedReg.isWrite() ? 'W' : '?');
1828 :
1829 0 : if (tiedReg.isLeadConsecutive())
1830 0 : sb.appendFormat("|Lead[%u]", tiedReg.consecutiveData() + 1u);
1831 :
1832 0 : if (tiedReg.hasUseId())
1833 : sb.appendFormat("|Use=%u", tiedReg.useId());
1834 0 : else if (tiedReg.isUse())
1835 : sb.append("|Use");
1836 :
1837 0 : if (tiedReg.isUseConsecutive() && !tiedReg.isLeadConsecutive())
1838 : sb.appendFormat("+%u", tiedReg.consecutiveData());
1839 :
1840 0 : if (tiedReg.hasOutId())
1841 : sb.appendFormat("|Out=%u", tiedReg.outId());
1842 0 : else if (tiedReg.isOut())
1843 : sb.append("|Out");
1844 :
1845 0 : if (tiedReg.isOutConsecutive() && !tiedReg.isLeadConsecutive())
1846 : sb.appendFormat("+%u", tiedReg.consecutiveData());
1847 :
1848 0 : if (tiedReg.isLast())
1849 : sb.append("|Last");
1850 :
1851 0 : if (tiedReg.isKill())
1852 : sb.append("|Kill");
1853 :
1854 : sb.append("}");
1855 : }
1856 0 : }
1857 :
1858 0 : ASMJIT_FAVOR_SIZE Error BaseRAPass::annotateCode() noexcept {
1859 0 : StringTmp<1024> sb;
1860 :
1861 0 : for (const RABlock* block : _blocks) {
1862 : BaseNode* node = block->first();
1863 0 : if (!node) continue;
1864 :
1865 : BaseNode* last = block->last();
1866 : for (;;) {
1867 0 : sb.clear();
1868 0 : Formatter::formatNode(sb, _formatOptions, cc(), node);
1869 :
1870 0 : if (hasDiagnosticOption(DiagnosticOptions::kRADebugLiveness) && node->isInst() && node->hasPassData()) {
1871 : const RAInst* raInst = node->passData<RAInst>();
1872 0 : if (raInst->tiedCount() > 0) {
1873 0 : sb.padEnd(40);
1874 0 : sb.append(" | ");
1875 0 : RAPass_formatLiveness(this, sb, raInst);
1876 : }
1877 : }
1878 :
1879 0 : node->setInlineComment(static_cast<char*>(cc()->_dataZone.dup(sb.data(), sb.size(), true)));
1880 0 : if (node == last)
1881 : break;
1882 : node = node->next();
1883 0 : }
1884 : }
1885 :
1886 0 : return kErrorOk;
1887 : }
1888 :
1889 0 : ASMJIT_FAVOR_SIZE Error BaseRAPass::_dumpBlockIds(String& sb, const RABlocks& blocks) noexcept {
1890 0 : for (uint32_t i = 0, size = blocks.size(); i < size; i++) {
1891 0 : const RABlock* block = blocks[i];
1892 0 : if (i != 0)
1893 0 : ASMJIT_PROPAGATE(sb.appendFormat(", #%u", block->blockId()));
1894 : else
1895 0 : ASMJIT_PROPAGATE(sb.appendFormat("#%u", block->blockId()));
1896 : }
1897 : return kErrorOk;
1898 : }
1899 :
1900 0 : ASMJIT_FAVOR_SIZE Error BaseRAPass::_dumpBlockLiveness(String& sb, const RABlock* block) noexcept {
1901 0 : for (uint32_t liveType = 0; liveType < RABlock::kLiveCount; liveType++) {
1902 0 : const char* bitsName = liveType == RABlock::kLiveIn ? "IN " :
1903 : liveType == RABlock::kLiveOut ? "OUT " :
1904 : liveType == RABlock::kLiveGen ? "GEN " : "KILL";
1905 :
1906 0 : const ZoneBitVector& bits = block->_liveBits[liveType];
1907 : uint32_t size = bits.size();
1908 : ASMJIT_ASSERT(size <= workRegCount());
1909 :
1910 : uint32_t n = 0;
1911 0 : for (uint32_t workId = 0; workId < size; workId++) {
1912 0 : if (bits.bitAt(workId)) {
1913 : RAWorkReg* wReg = workRegById(workId);
1914 :
1915 0 : if (!n)
1916 0 : sb.appendFormat(" %s [", bitsName);
1917 : else
1918 0 : sb.append(", ");
1919 :
1920 0 : sb.append(wReg->name());
1921 0 : n++;
1922 : }
1923 : }
1924 :
1925 0 : if (n)
1926 0 : sb.append("]\n");
1927 : }
1928 :
1929 0 : return kErrorOk;
1930 : }
1931 :
1932 0 : ASMJIT_FAVOR_SIZE Error BaseRAPass::_dumpLiveSpans(String& sb) noexcept {
1933 : uint32_t numWorkRegs = _workRegs.size();
1934 0 : uint32_t maxSize = _maxWorkRegNameSize;
1935 :
1936 0 : for (uint32_t workId = 0; workId < numWorkRegs; workId++) {
1937 0 : RAWorkReg* workReg = _workRegs[workId];
1938 :
1939 0 : sb.append(" ");
1940 :
1941 : size_t oldSize = sb.size();
1942 0 : sb.append(workReg->name());
1943 0 : sb.padEnd(oldSize + maxSize);
1944 :
1945 : RALiveStats& stats = workReg->liveStats();
1946 0 : sb.appendFormat(" {id:%04u width: %-4u freq: %0.4f priority=%0.4f}",
1947 0 : workReg->virtId(),
1948 0 : stats.width(),
1949 0 : stats.freq(),
1950 0 : stats.priority());
1951 0 : sb.append(": ");
1952 :
1953 : LiveRegSpans& liveSpans = workReg->liveSpans();
1954 0 : for (uint32_t x = 0; x < liveSpans.size(); x++) {
1955 : const LiveRegSpan& liveSpan = liveSpans[x];
1956 0 : if (x)
1957 0 : sb.append(", ");
1958 0 : sb.appendFormat("[%u:%u]", liveSpan.a, liveSpan.b);
1959 : }
1960 :
1961 0 : sb.append('\n');
1962 : }
1963 :
1964 0 : return kErrorOk;
1965 : }
1966 : #endif
1967 :
1968 : ASMJIT_END_NAMESPACE
1969 :
1970 : #endif // !ASMJIT_NO_COMPILER
1971 :
1972 : #endif // __PLUMED_HAS_ASMJIT
|
