Line data Source code
1 : #ifdef __PLUMED_HAS_ASMJIT
2 : // This file is part of AsmJit project <https://asmjit.com>
3 : //
4 : // See asmjit.h or LICENSE.md for license and copyright information
5 : // SPDX-License-Identifier: Zlib
6 :
7 : #include "api-build_p.h"
8 : #ifndef ASMJIT_NO_COMPILER
9 :
10 : #include "ralocal_p.h"
11 : #include "support.h"
12 :
13 : ASMJIT_BEGIN_NAMESPACE
14 :
15 : // RALocalAllocator - Utilities
16 : // ============================
17 :
18 : static ASMJIT_FORCE_INLINE RATiedReg* RALocal_findTiedRegByWorkId(RATiedReg* tiedRegs, size_t count, uint32_t workId) noexcept {
19 0 : for (size_t i = 0; i < count; i++)
20 0 : if (tiedRegs[i].workId() == workId)
21 : return &tiedRegs[i];
22 : return nullptr;
23 : }
24 :
25 : // RALocalAllocator - Init & Reset
26 : // ===============================
27 :
28 27492 : Error RALocalAllocator::init() noexcept {
29 : PhysToWorkMap* physToWorkMap;
30 : WorkToPhysMap* workToPhysMap;
31 :
32 27492 : physToWorkMap = _pass->newPhysToWorkMap();
33 27492 : workToPhysMap = _pass->newWorkToPhysMap();
34 27492 : if (!physToWorkMap || !workToPhysMap)
35 : return DebugUtils::errored(kErrorOutOfMemory);
36 :
37 27492 : _curAssignment.initLayout(_pass->_physRegCount, _pass->workRegs());
38 : _curAssignment.initMaps(physToWorkMap, workToPhysMap);
39 :
40 27492 : physToWorkMap = _pass->newPhysToWorkMap();
41 27492 : workToPhysMap = _pass->newWorkToPhysMap();
42 27492 : _tmpWorkToPhysMap = _pass->newWorkToPhysMap();
43 :
44 27492 : if (!physToWorkMap || !workToPhysMap || !_tmpWorkToPhysMap)
45 : return DebugUtils::errored(kErrorOutOfMemory);
46 :
47 27492 : _tmpAssignment.initLayout(_pass->_physRegCount, _pass->workRegs());
48 : _tmpAssignment.initMaps(physToWorkMap, workToPhysMap);
49 :
50 27492 : return kErrorOk;
51 : }
52 :
53 : // RALocalAllocator - Assignment
54 : // =============================
55 :
56 27492 : Error RALocalAllocator::makeInitialAssignment() noexcept {
57 27492 : FuncNode* func = _pass->func();
58 : RABlock* entry = _pass->entryBlock();
59 :
60 : ZoneBitVector& liveIn = entry->liveIn();
61 : uint32_t argCount = func->argCount();
62 : uint32_t numIter = 1;
63 :
64 54984 : for (uint32_t iter = 0; iter < numIter; iter++) {
65 27492 : for (uint32_t argIndex = 0; argIndex < argCount; argIndex++) {
66 0 : for (uint32_t valueIndex = 0; valueIndex < Globals::kMaxValuePack; valueIndex++) {
67 : // Unassigned argument.
68 0 : const RegOnly& regArg = func->argPack(argIndex)[valueIndex];
69 0 : if (!regArg.isReg() || !_cc->isVirtIdValid(regArg.id()))
70 0 : continue;
71 :
72 : VirtReg* virtReg = _cc->virtRegById(regArg.id());
73 :
74 : // Unreferenced argument.
75 : RAWorkReg* workReg = virtReg->workReg();
76 0 : if (!workReg)
77 0 : continue;
78 :
79 : // Overwritten argument.
80 : uint32_t workId = workReg->workId();
81 0 : if (!liveIn.bitAt(workId))
82 0 : continue;
83 :
84 : RegGroup group = workReg->group();
85 0 : if (_curAssignment.workToPhysId(group, workId) != RAAssignment::kPhysNone)
86 0 : continue;
87 :
88 0 : RegMask allocableRegs = _availableRegs[group] & ~_curAssignment.assigned(group);
89 0 : if (iter == 0) {
90 : // First iteration: Try to allocate to home RegId.
91 0 : if (workReg->hasHomeRegId()) {
92 : uint32_t physId = workReg->homeRegId();
93 0 : if (Support::bitTest(allocableRegs, physId)) {
94 0 : _curAssignment.assign(group, workId, physId, true);
95 0 : _pass->_argsAssignment.assignRegInPack(argIndex, valueIndex, workReg->type(), physId, workReg->typeId());
96 0 : continue;
97 : }
98 : }
99 :
100 : numIter = 2;
101 : }
102 : else {
103 : // Second iteration: Pick any other register if the is an unassigned one or assign to stack.
104 0 : if (allocableRegs) {
105 : uint32_t physId = Support::ctz(allocableRegs);
106 0 : _curAssignment.assign(group, workId, physId, true);
107 0 : _pass->_argsAssignment.assignRegInPack(argIndex, valueIndex, workReg->type(), physId, workReg->typeId());
108 : }
109 : else {
110 : // This register will definitely need stack, create the slot now and assign also `argIndex`
111 : // to it. We will patch `_argsAssignment` later after RAStackAllocator finishes.
112 0 : RAStackSlot* slot = _pass->getOrCreateStackSlot(workReg);
113 0 : if (ASMJIT_UNLIKELY(!slot))
114 : return DebugUtils::errored(kErrorOutOfMemory);
115 :
116 : // This means STACK_ARG may be moved to STACK.
117 : workReg->addFlags(RAWorkRegFlags::kStackArgToStack);
118 0 : _pass->_numStackArgsToStackSlots++;
119 : }
120 : }
121 : }
122 : }
123 : }
124 :
125 : return kErrorOk;
126 : }
127 :
128 0 : Error RALocalAllocator::replaceAssignment(const PhysToWorkMap* physToWorkMap) noexcept {
129 0 : _curAssignment.copyFrom(physToWorkMap);
130 0 : return kErrorOk;
131 : }
132 :
133 0 : Error RALocalAllocator::switchToAssignment(PhysToWorkMap* dstPhysToWorkMap, const ZoneBitVector& liveIn, bool dstReadOnly, bool tryMode) noexcept {
134 0 : RAAssignment dst;
135 : RAAssignment& cur = _curAssignment;
136 :
137 0 : dst.initLayout(_pass->_physRegCount, _pass->workRegs());
138 0 : dst.initMaps(dstPhysToWorkMap, _tmpWorkToPhysMap);
139 0 : dst.assignWorkIdsFromPhysIds();
140 :
141 0 : if (tryMode)
142 : return kErrorOk;
143 :
144 0 : for (RegGroup group : RegGroupVirtValues{}) {
145 : // STEP 1
146 : // ------
147 : //
148 : // - KILL all registers that are not live at `dst`,
149 : // - SPILL all registers that are not assigned at `dst`.
150 :
151 : if (!tryMode) {
152 : Support::BitWordIterator<RegMask> it(cur.assigned(group));
153 0 : while (it.hasNext()) {
154 : uint32_t physId = it.next();
155 : uint32_t workId = cur.physToWorkId(group, physId);
156 :
157 : // Must be true as we iterate over assigned registers.
158 : ASMJIT_ASSERT(workId != RAAssignment::kWorkNone);
159 :
160 : // KILL if it's not live on entry.
161 0 : if (!liveIn.bitAt(workId)) {
162 : onKillReg(group, workId, physId);
163 0 : continue;
164 : }
165 :
166 : // SPILL if it's not assigned on entry.
167 : uint32_t altId = dst.workToPhysId(group, workId);
168 0 : if (altId == RAAssignment::kPhysNone) {
169 0 : ASMJIT_PROPAGATE(onSpillReg(group, workId, physId));
170 : }
171 : }
172 : }
173 :
174 : // STEP 2
175 : // ------
176 : //
177 : // - MOVE and SWAP registers from their current assignments into their DST assignments.
178 : // - Build `willLoadRegs` mask of registers scheduled for `onLoadReg()`.
179 :
180 : // Current run-id (1 means more aggressive decisions).
181 : int32_t runId = -1;
182 : // Remaining registers scheduled for `onLoadReg()`.
183 : RegMask willLoadRegs = 0;
184 : // Remaining registers to be allocated in this loop.
185 : RegMask affectedRegs = dst.assigned(group);
186 :
187 0 : while (affectedRegs) {
188 0 : if (++runId == 2) {
189 : if (!tryMode)
190 : return DebugUtils::errored(kErrorInvalidState);
191 :
192 : // Stop in `tryMode` if we haven't done anything in past two rounds.
193 : break;
194 : }
195 :
196 : Support::BitWordIterator<RegMask> it(affectedRegs);
197 0 : while (it.hasNext()) {
198 : uint32_t physId = it.next();
199 : RegMask physMask = Support::bitMask<RegMask>(physId);
200 :
201 : uint32_t curWorkId = cur.physToWorkId(group, physId);
202 : uint32_t dstWorkId = dst.physToWorkId(group, physId);
203 :
204 : // The register must have assigned `dstWorkId` as we only iterate over assigned regs.
205 : ASMJIT_ASSERT(dstWorkId != RAAssignment::kWorkNone);
206 :
207 0 : if (curWorkId != RAAssignment::kWorkNone) {
208 : // Both assigned.
209 0 : if (curWorkId != dstWorkId) {
210 : // Wait a bit if this is the first run, we may avoid this if `curWorkId` moves out.
211 0 : if (runId <= 0)
212 0 : continue;
213 :
214 : uint32_t altPhysId = cur.workToPhysId(group, dstWorkId);
215 0 : if (altPhysId == RAAssignment::kPhysNone)
216 0 : continue;
217 :
218 : // Reset as we will do some changes to the current assignment.
219 : runId = -1;
220 :
221 0 : if (_archTraits->hasInstRegSwap(group)) {
222 0 : ASMJIT_PROPAGATE(onSwapReg(group, curWorkId, physId, dstWorkId, altPhysId));
223 : }
224 : else {
225 : // SPILL the reg if it's not dirty in DST, otherwise try to MOVE.
226 0 : if (!cur.isPhysDirty(group, physId)) {
227 : ASMJIT_PROPAGATE(onKillReg(group, curWorkId, physId));
228 : }
229 : else {
230 0 : RegMask allocableRegs = _pass->_availableRegs[group] & ~cur.assigned(group);
231 :
232 : // If possible don't conflict with assigned regs at DST.
233 0 : if (allocableRegs & ~dst.assigned(group))
234 : allocableRegs &= ~dst.assigned(group);
235 :
236 0 : if (allocableRegs) {
237 : // MOVE is possible, thus preferred.
238 : uint32_t tmpPhysId = Support::ctz(allocableRegs);
239 :
240 0 : ASMJIT_PROPAGATE(onMoveReg(group, curWorkId, tmpPhysId, physId));
241 0 : _pass->_clobberedRegs[group] |= Support::bitMask(tmpPhysId);
242 : }
243 : else {
244 : // MOVE is impossible, must SPILL.
245 0 : ASMJIT_PROPAGATE(onSpillReg(group, curWorkId, physId));
246 : }
247 : }
248 :
249 0 : goto Cleared;
250 : }
251 : }
252 : }
253 : else {
254 0 : Cleared:
255 : // DST assigned, CUR unassigned.
256 : uint32_t altPhysId = cur.workToPhysId(group, dstWorkId);
257 0 : if (altPhysId == RAAssignment::kPhysNone) {
258 0 : if (liveIn.bitAt(dstWorkId))
259 0 : willLoadRegs |= physMask; // Scheduled for `onLoadReg()`.
260 0 : affectedRegs &= ~physMask; // Unaffected from now.
261 0 : continue;
262 : }
263 0 : ASMJIT_PROPAGATE(onMoveReg(group, dstWorkId, physId, altPhysId));
264 : }
265 :
266 : // Both DST and CUR assigned to the same reg or CUR just moved to DST.
267 0 : if ((dst.dirty(group) & physMask) != (cur.dirty(group) & physMask)) {
268 0 : if ((dst.dirty(group) & physMask) == 0) {
269 : // CUR dirty, DST not dirty (the assert is just to visualize the condition).
270 : ASMJIT_ASSERT(!dst.isPhysDirty(group, physId) && cur.isPhysDirty(group, physId));
271 :
272 : // If `dstReadOnly` is true it means that that block was already processed and we cannot change from
273 : // CLEAN to DIRTY. In that case the register has to be saved as it cannot enter the block DIRTY.
274 0 : if (dstReadOnly)
275 0 : ASMJIT_PROPAGATE(onSaveReg(group, dstWorkId, physId));
276 : else
277 : dst.makeDirty(group, dstWorkId, physId);
278 : }
279 : else {
280 : // DST dirty, CUR not dirty (the assert is just to visualize the condition).
281 : ASMJIT_ASSERT(dst.isPhysDirty(group, physId) && !cur.isPhysDirty(group, physId));
282 :
283 : cur.makeDirty(group, dstWorkId, physId);
284 : }
285 : }
286 :
287 : // Must match now...
288 : ASMJIT_ASSERT(dst.physToWorkId(group, physId) == cur.physToWorkId(group, physId));
289 : ASMJIT_ASSERT(dst.isPhysDirty(group, physId) == cur.isPhysDirty(group, physId));
290 :
291 : runId = -1;
292 0 : affectedRegs &= ~physMask;
293 : }
294 : }
295 :
296 : // STEP 3
297 : // ------
298 : //
299 : // - Load registers specified by `willLoadRegs`.
300 :
301 : {
302 : Support::BitWordIterator<RegMask> it(willLoadRegs);
303 0 : while (it.hasNext()) {
304 : uint32_t physId = it.next();
305 :
306 0 : if (!cur.isPhysAssigned(group, physId)) {
307 : uint32_t workId = dst.physToWorkId(group, physId);
308 :
309 : // The algorithm is broken if it tries to load a register that is not in LIVE-IN.
310 : ASMJIT_ASSERT(liveIn.bitAt(workId) == true);
311 :
312 0 : ASMJIT_PROPAGATE(onLoadReg(group, workId, physId));
313 0 : if (dst.isPhysDirty(group, physId))
314 : cur.makeDirty(group, workId, physId);
315 : ASMJIT_ASSERT(dst.isPhysDirty(group, physId) == cur.isPhysDirty(group, physId));
316 : }
317 : else {
318 : // Not possible otherwise.
319 : ASMJIT_ASSERT(tryMode == true);
320 : }
321 : }
322 : }
323 : }
324 :
325 : if (!tryMode) {
326 : // Here is a code that dumps the conflicting part if something fails here:
327 : // if (!dst.equals(cur)) {
328 : // uint32_t physTotal = dst._layout.physTotal;
329 : // uint32_t workCount = dst._layout.workCount;
330 : //
331 : // fprintf(stderr, "Dirty DST=0x%08X CUR=0x%08X\n", dst.dirty(RegGroup::kGp), cur.dirty(RegGroup::kGp));
332 : // fprintf(stderr, "Assigned DST=0x%08X CUR=0x%08X\n", dst.assigned(RegGroup::kGp), cur.assigned(RegGroup::kGp));
333 : //
334 : // for (uint32_t physId = 0; physId < physTotal; physId++) {
335 : // uint32_t dstWorkId = dst._physToWorkMap->workIds[physId];
336 : // uint32_t curWorkId = cur._physToWorkMap->workIds[physId];
337 : // if (dstWorkId != curWorkId)
338 : // fprintf(stderr, "[PhysIdWork] PhysId=%u WorkId[DST(%u) != CUR(%u)]\n", physId, dstWorkId, curWorkId);
339 : // }
340 : //
341 : // for (uint32_t workId = 0; workId < workCount; workId++) {
342 : // uint32_t dstPhysId = dst._workToPhysMap->physIds[workId];
343 : // uint32_t curPhysId = cur._workToPhysMap->physIds[workId];
344 : // if (dstPhysId != curPhysId)
345 : // fprintf(stderr, "[WorkToPhys] WorkId=%u PhysId[DST(%u) != CUR(%u)]\n", workId, dstPhysId, curPhysId);
346 : // }
347 : // }
348 : ASMJIT_ASSERT(dst.equals(cur));
349 : }
350 :
351 : return kErrorOk;
352 : }
353 :
354 27492 : Error RALocalAllocator::spillScratchGpRegsBeforeEntry(RegMask scratchRegs) noexcept {
355 : RegGroup group = RegGroup::kGp;
356 : Support::BitWordIterator<RegMask> it(scratchRegs);
357 :
358 27492 : while (it.hasNext()) {
359 : uint32_t physId = it.next();
360 0 : if (_curAssignment.isPhysAssigned(group, physId)) {
361 : uint32_t workId = _curAssignment.physToWorkId(group, physId);
362 0 : ASMJIT_PROPAGATE(onSpillReg(group, workId, physId));
363 : }
364 : }
365 :
366 : return kErrorOk;
367 : }
368 :
369 : // RALocalAllocator - Allocation
370 : // =============================
371 :
372 331808 : Error RALocalAllocator::allocInst(InstNode* node) noexcept {
373 : RAInst* raInst = node->passData<RAInst>();
374 :
375 : RATiedReg* outTiedRegs[Globals::kMaxPhysRegs];
376 : RATiedReg* dupTiedRegs[Globals::kMaxPhysRegs];
377 : RATiedReg* consecutiveRegs[kMaxConsecutiveRegs];
378 :
379 : // The cursor must point to the previous instruction for a possible instruction insertion.
380 331808 : _cc->_setCursor(node->prev());
381 :
382 331808 : _node = node;
383 331808 : _raInst = raInst;
384 331808 : _tiedTotal = raInst->_tiedTotal;
385 331808 : _tiedCount = raInst->_tiedCount;
386 :
387 : // Whether we already replaced register operand with memory operand.
388 : bool rmAllocated = false;
389 :
390 1659040 : for (RegGroup group : RegGroupVirtValues{}) {
391 : uint32_t i, count = this->tiedCount(group);
392 : RATiedReg* tiedRegs = this->tiedRegs(group);
393 :
394 1327232 : RegMask willUse = _raInst->_usedRegs[group];
395 1327232 : RegMask willOut = _raInst->_clobberedRegs[group];
396 : RegMask willFree = 0;
397 :
398 : uint32_t usePending = count;
399 : uint32_t outTiedCount = 0;
400 : uint32_t dupTiedCount = 0;
401 : uint32_t consecutiveMask = 0;
402 :
403 : // STEP 1
404 : // ------
405 : //
406 : // Calculate `willUse` and `willFree` masks based on tied registers we have. In addition, aggregate information
407 : // regarding consecutive registers used by this instruction. We need that to make USE/OUT assignments.
408 : //
409 : // We don't do any assignment decisions at this stage as we just need to collect some information first. Then,
410 : // after we populate all masks needed we can finally make some decisions in the second loop. The main reason
411 : // for this is that we really need `willFree` to make assignment decisions for `willUse`, because if we mark
412 : // some registers that will be freed, we can consider them in decision making afterwards.
413 :
414 1941686 : for (i = 0; i < count; i++) {
415 614454 : RATiedReg* tiedReg = &tiedRegs[i];
416 :
417 614454 : if (tiedReg->hasAnyConsecutiveFlag()) {
418 0 : uint32_t consecutiveOffset = tiedReg->isLeadConsecutive() ? uint32_t(0) : tiedReg->consecutiveData();
419 :
420 0 : if (ASMJIT_UNLIKELY(Support::bitTest(consecutiveMask, consecutiveOffset)))
421 : return DebugUtils::errored(kErrorInvalidState);
422 :
423 0 : consecutiveMask |= Support::bitMask(consecutiveOffset);
424 0 : consecutiveRegs[consecutiveOffset] = tiedReg;
425 : }
426 :
427 : // Add OUT and KILL to `outPending` for CLOBBERing and/or OUT assignment.
428 614454 : if (tiedReg->isOutOrKill())
429 559912 : outTiedRegs[outTiedCount++] = tiedReg;
430 :
431 614454 : if (tiedReg->isDuplicate())
432 0 : dupTiedRegs[dupTiedCount++] = tiedReg;
433 :
434 614454 : if (!tiedReg->isUse()) {
435 : tiedReg->markUseDone();
436 283174 : usePending--;
437 283174 : continue;
438 : }
439 :
440 : // Don't assign anything here if this is a consecutive USE - we will handle this in STEP 2 instead.
441 331280 : if (tiedReg->isUseConsecutive())
442 0 : continue;
443 :
444 : uint32_t workId = tiedReg->workId();
445 : uint32_t assignedId = _curAssignment.workToPhysId(group, workId);
446 :
447 331280 : if (tiedReg->hasUseId()) {
448 : // If the register has `useId` it means it can only be allocated in that register.
449 : RegMask useMask = Support::bitMask(tiedReg->useId());
450 :
451 : // RAInstBuilder must have collected `usedRegs` on-the-fly.
452 : ASMJIT_ASSERT((willUse & useMask) != 0);
453 :
454 37365 : if (assignedId == tiedReg->useId()) {
455 : // If the register is already allocated in this one, mark it done and continue.
456 : tiedReg->markUseDone();
457 35117 : if (tiedReg->isWrite())
458 : _curAssignment.makeDirty(group, workId, assignedId);
459 35117 : usePending--;
460 35117 : willUse |= useMask;
461 : }
462 : else {
463 2248 : willFree |= useMask & _curAssignment.assigned(group);
464 : }
465 : }
466 : else {
467 : // Check if the register must be moved to `allocableRegs`.
468 : RegMask allocableRegs = tiedReg->useRegMask();
469 293915 : if (assignedId != RAAssignment::kPhysNone) {
470 : RegMask assignedMask = Support::bitMask(assignedId);
471 254524 : if ((allocableRegs & ~willUse) & assignedMask) {
472 : tiedReg->setUseId(assignedId);
473 : tiedReg->markUseDone();
474 254524 : if (tiedReg->isWrite())
475 : _curAssignment.makeDirty(group, workId, assignedId);
476 254524 : usePending--;
477 254524 : willUse |= assignedMask;
478 : }
479 : else {
480 0 : willFree |= assignedMask;
481 : }
482 : }
483 : }
484 : }
485 :
486 : // STEP 2
487 : // ------
488 : //
489 : // Verify that all the consecutive registers are really consecutive. Terminate if there is a gap. In addition,
490 : // decide which USE ids will be used in case that this consecutive sequence is USE (OUT registers are allocated
491 : // in a different step).
492 : uint32_t consecutiveCount = 0;
493 :
494 1327232 : if (consecutiveMask) {
495 0 : if ((consecutiveMask & (consecutiveMask + 1u)) != 0)
496 : return DebugUtils::errored(kErrorInvalidState);
497 :
498 : // Count of trailing ones is the count of consecutive registers. There cannot be gap.
499 0 : consecutiveCount = Support::ctz(~consecutiveMask);
500 :
501 : // Prioritize allocation that would result in least moves even when moving registers away from their homes.
502 0 : RATiedReg* lead = consecutiveRegs[0];
503 :
504 : // Assign the best possible USE Ids to all consecutives.
505 0 : if (lead->isUseConsecutive()) {
506 : uint32_t bestScore = 0;
507 : uint32_t bestLeadReg = 0xFFFFFFFF;
508 0 : RegMask allocableRegs = (_availableRegs[group] | willFree) & ~willUse;
509 :
510 : uint32_t assignments[kMaxConsecutiveRegs];
511 :
512 0 : for (i = 0; i < consecutiveCount; i++)
513 0 : assignments[i] = _curAssignment.workToPhysId(group, consecutiveRegs[i]->workId());
514 :
515 : Support::BitWordIterator<uint32_t> it(lead->useRegMask());
516 0 : while (it.hasNext()) {
517 : uint32_t regIndex = it.next();
518 0 : if (Support::bitTest(lead->useRegMask(), regIndex)) {
519 : uint32_t score = 15;
520 :
521 0 : for (i = 0; i < consecutiveCount; i++) {
522 0 : uint32_t consecutiveIndex = regIndex + i;
523 0 : if (!Support::bitTest(allocableRegs, consecutiveIndex)) {
524 : score = 0;
525 : break;
526 : }
527 :
528 0 : RAWorkReg* workReg = workRegById(consecutiveRegs[i]->workId());
529 0 : score += uint32_t(workReg->homeRegId() == consecutiveIndex);
530 0 : score += uint32_t(assignments[i] == consecutiveIndex) * 2;
531 : }
532 :
533 0 : if (score > bestScore) {
534 : bestScore = score;
535 : bestLeadReg = regIndex;
536 : }
537 : }
538 : }
539 :
540 0 : if (bestLeadReg == 0xFFFFFFFF)
541 0 : return DebugUtils::errored(kErrorConsecutiveRegsAllocation);
542 :
543 0 : for (i = 0; i < consecutiveCount; i++) {
544 0 : uint32_t consecutiveIndex = bestLeadReg + i;
545 :
546 0 : RATiedReg* tiedReg = consecutiveRegs[i];
547 : RegMask useMask = Support::bitMask(consecutiveIndex);
548 :
549 : uint32_t workId = tiedReg->workId();
550 : uint32_t assignedId = _curAssignment.workToPhysId(group, workId);
551 :
552 : tiedReg->setUseId(consecutiveIndex);
553 :
554 0 : if (assignedId == consecutiveIndex) {
555 : // If the register is already allocated in this one, mark it done and continue.
556 : tiedReg->markUseDone();
557 0 : if (tiedReg->isWrite())
558 : _curAssignment.makeDirty(group, workId, assignedId);
559 0 : usePending--;
560 0 : willUse |= useMask;
561 : }
562 : else {
563 0 : willUse |= useMask;
564 0 : willFree |= useMask & _curAssignment.assigned(group);
565 : }
566 : }
567 : }
568 : }
569 :
570 : // STEP 3
571 : // ------
572 : //
573 : // Do some decision making to find the best candidates of registers that need to be assigned, moved, and/or
574 : // spilled. Only USE registers are considered here, OUT will be decided later after all CLOBBERed and OUT
575 : // registers are unassigned.
576 :
577 1327232 : if (usePending) {
578 : // TODO: Not sure `liveRegs` should be used, maybe willUse and willFree would be enough and much more clear.
579 :
580 : // All registers that are currently alive without registers that will be freed.
581 33640 : RegMask liveRegs = _curAssignment.assigned(group) & ~willFree;
582 :
583 130709 : for (i = 0; i < count; i++) {
584 97069 : RATiedReg* tiedReg = &tiedRegs[i];
585 97069 : if (tiedReg->isUseDone())
586 55430 : continue;
587 :
588 : uint32_t workId = tiedReg->workId();
589 : uint32_t assignedId = _curAssignment.workToPhysId(group, workId);
590 :
591 : // REG/MEM: Patch register operand to memory operand if not allocated.
592 41639 : if (!rmAllocated && tiedReg->hasUseRM()) {
593 23166 : if (assignedId == RAAssignment::kPhysNone && Support::isPowerOf2(tiedReg->useRewriteMask())) {
594 : RAWorkReg* workReg = workRegById(tiedReg->workId());
595 23106 : uint32_t opIndex = Support::ctz(tiedReg->useRewriteMask()) / uint32_t(sizeof(Operand) / sizeof(uint32_t));
596 : uint32_t rmSize = tiedReg->rmSize();
597 :
598 23106 : if (rmSize <= workReg->virtReg()->virtSize()) {
599 22695 : Operand& op = node->operands()[opIndex];
600 22695 : op = _pass->workRegAsMem(workReg);
601 : op.as<BaseMem>().setSize(rmSize);
602 22695 : tiedReg->_useRewriteMask = 0;
603 :
604 : tiedReg->markUseDone();
605 22695 : usePending--;
606 :
607 : rmAllocated = true;
608 22695 : continue;
609 22695 : }
610 : }
611 : }
612 :
613 18944 : if (!tiedReg->hasUseId()) {
614 : // DECIDE where to assign the USE register.
615 16696 : RegMask allocableRegs = tiedReg->useRegMask() & ~(willFree | willUse);
616 16696 : uint32_t useId = decideOnAssignment(group, workId, assignedId, allocableRegs);
617 :
618 : RegMask useMask = Support::bitMask(useId);
619 16696 : willUse |= useMask;
620 16696 : willFree |= useMask & liveRegs;
621 : tiedReg->setUseId(useId);
622 :
623 16696 : if (assignedId != RAAssignment::kPhysNone) {
624 : RegMask assignedMask = Support::bitMask(assignedId);
625 :
626 0 : willFree |= assignedMask;
627 0 : liveRegs &= ~assignedMask;
628 :
629 : // OPTIMIZATION: Assign the USE register here if it's possible.
630 0 : if (!(liveRegs & useMask)) {
631 0 : ASMJIT_PROPAGATE(onMoveReg(group, workId, useId, assignedId));
632 : tiedReg->markUseDone();
633 0 : if (tiedReg->isWrite())
634 : _curAssignment.makeDirty(group, workId, useId);
635 0 : usePending--;
636 : }
637 : }
638 : else {
639 : // OPTIMIZATION: Assign the USE register here if it's possible.
640 16696 : if (!(liveRegs & useMask)) {
641 15728 : ASMJIT_PROPAGATE(onLoadReg(group, workId, useId));
642 : tiedReg->markUseDone();
643 15728 : if (tiedReg->isWrite())
644 : _curAssignment.makeDirty(group, workId, useId);
645 15728 : usePending--;
646 : }
647 : }
648 :
649 16696 : liveRegs |= useMask;
650 : }
651 : }
652 : }
653 :
654 : // Initially all used regs will be marked as clobbered.
655 1327232 : RegMask clobberedByInst = willUse | willOut;
656 :
657 : // STEP 4
658 : // ------
659 : //
660 : // Free all registers that we marked as `willFree`. Only registers that are not USEd by the instruction are
661 : // considered as we don't want to free regs we need.
662 :
663 1327232 : if (willFree) {
664 2883 : RegMask allocableRegs = _availableRegs[group] & ~(_curAssignment.assigned(group) | willFree | willUse | willOut);
665 : Support::BitWordIterator<RegMask> it(willFree);
666 :
667 : do {
668 : uint32_t assignedId = it.next();
669 2892 : if (_curAssignment.isPhysAssigned(group, assignedId)) {
670 : uint32_t workId = _curAssignment.physToWorkId(group, assignedId);
671 :
672 : // DECIDE whether to MOVE or SPILL.
673 2892 : if (allocableRegs) {
674 968 : uint32_t reassignedId = decideOnReassignment(group, workId, assignedId, allocableRegs);
675 968 : if (reassignedId != RAAssignment::kPhysNone) {
676 479 : ASMJIT_PROPAGATE(onMoveReg(group, workId, reassignedId, assignedId));
677 479 : allocableRegs ^= Support::bitMask(reassignedId);
678 : continue;
679 479 : }
680 : }
681 :
682 2413 : ASMJIT_PROPAGATE(onSpillReg(group, workId, assignedId));
683 : }
684 2892 : } while (it.hasNext());
685 : }
686 :
687 : // STEP 5
688 : // ------
689 : //
690 : // ALLOCATE / SHUFFLE all registers that we marked as `willUse` and weren't allocated yet. This is a bit
691 : // complicated as the allocation is iterative. In some cases we have to wait before allocating a particual
692 : // physical register as it's still occupied by some other one, which we need to move before we can use it.
693 : // In this case we skip it and allocate another some other instead (making it free for another iteration).
694 : //
695 : // NOTE: Iterations are mostly important for complicated allocations like function calls, where there can
696 : // be up to N registers used at once. Asm instructions won't run the loop more than once in 99.9% of cases
697 : // as they use 2..3 registers in average.
698 :
699 1327232 : if (usePending) {
700 : bool mustSwap = false;
701 : do {
702 : uint32_t oldPending = usePending;
703 :
704 10428 : for (i = 0; i < count; i++) {
705 7227 : RATiedReg* thisTiedReg = &tiedRegs[i];
706 7227 : if (thisTiedReg->isUseDone())
707 4011 : continue;
708 :
709 : uint32_t thisWorkId = thisTiedReg->workId();
710 : uint32_t thisPhysId = _curAssignment.workToPhysId(group, thisWorkId);
711 :
712 : // This would be a bug, fatal one!
713 : uint32_t targetPhysId = thisTiedReg->useId();
714 : ASMJIT_ASSERT(targetPhysId != thisPhysId);
715 :
716 : uint32_t targetWorkId = _curAssignment.physToWorkId(group, targetPhysId);
717 3216 : if (targetWorkId != RAAssignment::kWorkNone) {
718 : RAWorkReg* targetWorkReg = workRegById(targetWorkId);
719 :
720 : // Swapping two registers can solve two allocation tasks by emitting just a single instruction. However,
721 : // swap is only available on few architectures and it's definitely not available for each register group.
722 : // Calling `onSwapReg()` before checking these would be fatal.
723 0 : if (_archTraits->hasInstRegSwap(group) && thisPhysId != RAAssignment::kPhysNone) {
724 0 : ASMJIT_PROPAGATE(onSwapReg(group, thisWorkId, thisPhysId, targetWorkId, targetPhysId));
725 :
726 : thisTiedReg->markUseDone();
727 0 : if (thisTiedReg->isWrite())
728 : _curAssignment.makeDirty(group, thisWorkId, targetPhysId);
729 0 : usePending--;
730 :
731 : // Double-hit.
732 0 : RATiedReg* targetTiedReg = RALocal_findTiedRegByWorkId(tiedRegs, count, targetWorkReg->workId());
733 0 : if (targetTiedReg && targetTiedReg->useId() == thisPhysId) {
734 : targetTiedReg->markUseDone();
735 0 : if (targetTiedReg->isWrite())
736 : _curAssignment.makeDirty(group, targetWorkId, thisPhysId);
737 0 : usePending--;
738 : }
739 0 : continue;
740 0 : }
741 :
742 0 : if (!mustSwap)
743 0 : continue;
744 :
745 : // Only branched here if the previous iteration did nothing. This is essentially a SWAP operation without
746 : // having a dedicated instruction for that purpose (vector registers, etc). The simplest way to handle
747 : // such case is to SPILL the target register.
748 0 : ASMJIT_PROPAGATE(onSpillReg(group, targetWorkId, targetPhysId));
749 : }
750 :
751 3216 : if (thisPhysId != RAAssignment::kPhysNone) {
752 1306 : ASMJIT_PROPAGATE(onMoveReg(group, thisWorkId, targetPhysId, thisPhysId));
753 :
754 : thisTiedReg->markUseDone();
755 1306 : if (thisTiedReg->isWrite())
756 : _curAssignment.makeDirty(group, thisWorkId, targetPhysId);
757 1306 : usePending--;
758 : }
759 : else {
760 1910 : ASMJIT_PROPAGATE(onLoadReg(group, thisWorkId, targetPhysId));
761 :
762 : thisTiedReg->markUseDone();
763 1910 : if (thisTiedReg->isWrite())
764 : _curAssignment.makeDirty(group, thisWorkId, targetPhysId);
765 1910 : usePending--;
766 : }
767 : }
768 :
769 3201 : mustSwap = (oldPending == usePending);
770 3201 : } while (usePending);
771 : }
772 :
773 : // STEP 6
774 : // ------
775 : //
776 : // KILL registers marked as KILL/OUT.
777 :
778 : uint32_t outPending = outTiedCount;
779 1327232 : if (outTiedCount) {
780 952355 : for (i = 0; i < outTiedCount; i++) {
781 559912 : RATiedReg* tiedReg = outTiedRegs[i];
782 :
783 : uint32_t workId = tiedReg->workId();
784 : uint32_t physId = _curAssignment.workToPhysId(group, workId);
785 :
786 : // Must check if it's allocated as KILL can be related to OUT (like KILL immediately after OUT, which could
787 : // mean the register is not assigned).
788 559912 : if (physId != RAAssignment::kPhysNone) {
789 : ASMJIT_PROPAGATE(onKillReg(group, workId, physId));
790 258530 : willOut &= ~Support::bitMask(physId);
791 : }
792 :
793 : // We still maintain number of pending registers for OUT assignment. So, if this is only KILL, not OUT, we
794 : // can safely decrement it.
795 559912 : outPending -= !tiedReg->isOut();
796 : }
797 : }
798 :
799 : // STEP 7
800 : // ------
801 : //
802 : // SPILL registers that will be CLOBBERed. Since OUT and KILL were already processed this is used mostly to
803 : // handle function CALLs.
804 :
805 1327232 : if (willOut) {
806 : Support::BitWordIterator<RegMask> it(willOut);
807 : do {
808 : uint32_t physId = it.next();
809 : uint32_t workId = _curAssignment.physToWorkId(group, physId);
810 :
811 376714 : if (workId == RAAssignment::kWorkNone)
812 350627 : continue;
813 :
814 26087 : ASMJIT_PROPAGATE(onSpillReg(group, workId, physId));
815 376714 : } while (it.hasNext());
816 : }
817 :
818 : // STEP 8
819 : // ------
820 : //
821 : // Duplication.
822 :
823 1327232 : for (i = 0; i < dupTiedCount; i++) {
824 0 : RATiedReg* tiedReg = dupTiedRegs[i];
825 : uint32_t workId = tiedReg->workId();
826 : uint32_t srcId = tiedReg->useId();
827 :
828 : Support::BitWordIterator<RegMask> it(tiedReg->useRegMask());
829 0 : while (it.hasNext()) {
830 : uint32_t dstId = it.next();
831 0 : if (dstId == srcId)
832 0 : continue;
833 0 : _pass->emitMove(workId, dstId, srcId);
834 : }
835 : }
836 :
837 : // STEP 9
838 : // ------
839 : //
840 : // Vector registers can be cloberred partially by invoke - find if that's the case and clobber when necessary.
841 :
842 1327232 : if (node->isInvoke() && group == RegGroup::kVec) {
843 : const InvokeNode* invokeNode = node->as<InvokeNode>();
844 :
845 9639 : RegMask maybeClobberedRegs = invokeNode->detail().callConv().preservedRegs(group) & _curAssignment.assigned(group);
846 9639 : if (maybeClobberedRegs) {
847 : uint32_t saveRestoreVecSize = invokeNode->detail().callConv().saveRestoreRegSize(group);
848 : Support::BitWordIterator<RegMask> it(maybeClobberedRegs);
849 :
850 : do {
851 : uint32_t physId = it.next();
852 : uint32_t workId = _curAssignment.physToWorkId(group, physId);
853 :
854 : RAWorkReg* workReg = workRegById(workId);
855 : uint32_t virtSize = workReg->virtReg()->virtSize();
856 :
857 0 : if (virtSize > saveRestoreVecSize) {
858 0 : ASMJIT_PROPAGATE(onSpillReg(group, workId, physId));
859 : }
860 :
861 0 : } while (it.hasNext());
862 : }
863 : }
864 :
865 : // STEP 10
866 : // -------
867 : //
868 : // Assign OUT registers.
869 :
870 1327232 : if (outPending) {
871 : // Live registers, we need a separate register (outside of `_curAssignment) to hold these because of KILLed
872 : // registers. If we KILL a register here it will go out from `_curAssignment`, but we cannot assign to it in
873 : // here.
874 : RegMask liveRegs = _curAssignment.assigned(group);
875 :
876 : // Must avoid as they have been already OUTed (added during the loop).
877 : RegMask outRegs = 0;
878 :
879 : // Must avoid as they collide with already allocated ones.
880 304222 : RegMask avoidRegs = willUse & ~clobberedByInst;
881 :
882 : // Assign the best possible OUT ids of all consecutives.
883 304222 : if (consecutiveCount) {
884 0 : RATiedReg* lead = consecutiveRegs[0];
885 0 : if (lead->isOutConsecutive()) {
886 : uint32_t bestScore = 0;
887 : uint32_t bestLeadReg = 0xFFFFFFFF;
888 0 : RegMask allocableRegs = _availableRegs[group] & ~(outRegs | avoidRegs);
889 :
890 : Support::BitWordIterator<uint32_t> it(lead->outRegMask());
891 0 : while (it.hasNext()) {
892 : uint32_t regIndex = it.next();
893 0 : if (Support::bitTest(lead->outRegMask(), regIndex)) {
894 : uint32_t score = 15;
895 :
896 0 : for (i = 0; i < consecutiveCount; i++) {
897 0 : uint32_t consecutiveIndex = regIndex + i;
898 0 : if (!Support::bitTest(allocableRegs, consecutiveIndex)) {
899 : score = 0;
900 : break;
901 : }
902 :
903 0 : RAWorkReg* workReg = workRegById(consecutiveRegs[i]->workId());
904 0 : score += uint32_t(workReg->homeRegId() == consecutiveIndex);
905 : }
906 :
907 0 : if (score > bestScore) {
908 : bestScore = score;
909 : bestLeadReg = regIndex;
910 : }
911 : }
912 : }
913 :
914 0 : if (bestLeadReg == 0xFFFFFFFF)
915 : return DebugUtils::errored(kErrorConsecutiveRegsAllocation);
916 :
917 0 : for (i = 0; i < consecutiveCount; i++) {
918 0 : uint32_t consecutiveIndex = bestLeadReg + i;
919 0 : RATiedReg* tiedReg = consecutiveRegs[i];
920 : tiedReg->setOutId(consecutiveIndex);
921 : }
922 : }
923 : }
924 :
925 : // Allocate OUT registers.
926 775725 : for (i = 0; i < outTiedCount; i++) {
927 471503 : RATiedReg* tiedReg = outTiedRegs[i];
928 471503 : if (!tiedReg->isOut())
929 : continue;
930 :
931 : uint32_t workId = tiedReg->workId();
932 : uint32_t assignedId = _curAssignment.workToPhysId(group, workId);
933 :
934 304222 : if (assignedId != RAAssignment::kPhysNone)
935 : ASMJIT_PROPAGATE(onKillReg(group, workId, assignedId));
936 :
937 : uint32_t physId = tiedReg->outId();
938 304222 : if (physId == RAAssignment::kPhysNone) {
939 294583 : RegMask allocableRegs = tiedReg->outRegMask() & ~(outRegs | avoidRegs);
940 :
941 294583 : if (!(allocableRegs & ~liveRegs)) {
942 : // There are no more registers, decide which one to spill.
943 : uint32_t spillWorkId;
944 7346 : physId = decideOnSpillFor(group, workId, allocableRegs & liveRegs, &spillWorkId);
945 7346 : ASMJIT_PROPAGATE(onSpillReg(group, spillWorkId, physId));
946 : }
947 : else {
948 287237 : physId = decideOnAssignment(group, workId, RAAssignment::kPhysNone, allocableRegs & ~liveRegs);
949 : }
950 : }
951 :
952 : // OUTs are CLOBBERed thus cannot be ASSIGNed right now.
953 : ASMJIT_ASSERT(!_curAssignment.isPhysAssigned(group, physId));
954 :
955 304222 : if (!tiedReg->isKill())
956 : ASMJIT_PROPAGATE(onAssignReg(group, workId, physId, true));
957 :
958 : tiedReg->setOutId(physId);
959 : tiedReg->markOutDone();
960 :
961 304222 : outRegs |= Support::bitMask(physId);
962 304222 : liveRegs &= ~Support::bitMask(physId);
963 : outPending--;
964 : }
965 :
966 304222 : clobberedByInst |= outRegs;
967 : ASMJIT_ASSERT(outPending == 0);
968 : }
969 :
970 1327232 : _clobberedRegs[group] |= clobberedByInst;
971 : }
972 :
973 : return kErrorOk;
974 : }
975 :
976 322169 : Error RALocalAllocator::spillAfterAllocation(InstNode* node) noexcept {
977 : // This is experimental feature that would spill registers that don't have home-id and are last in this basic block.
978 : // This prevents saving these regs in other basic blocks and then restoring them (mostly relevant for loops).
979 : RAInst* raInst = node->passData<RAInst>();
980 : uint32_t count = raInst->tiedCount();
981 :
982 907472 : for (uint32_t i = 0; i < count; i++) {
983 : RATiedReg* tiedReg = raInst->tiedAt(i);
984 585303 : if (tiedReg->isLast()) {
985 : uint32_t workId = tiedReg->workId();
986 : RAWorkReg* workReg = workRegById(workId);
987 252829 : if (!workReg->hasHomeRegId()) {
988 : RegGroup group = workReg->group();
989 : uint32_t assignedId = _curAssignment.workToPhysId(group, workId);
990 8983 : if (assignedId != RAAssignment::kPhysNone) {
991 0 : _cc->_setCursor(node);
992 0 : ASMJIT_PROPAGATE(onSpillReg(group, workId, assignedId));
993 : }
994 : }
995 : }
996 : }
997 :
998 : return kErrorOk;
999 : }
1000 :
1001 0 : Error RALocalAllocator::allocBranch(InstNode* node, RABlock* target, RABlock* cont) noexcept {
1002 : // TODO: This should be used to make the branch allocation better.
1003 : DebugUtils::unused(cont);
1004 :
1005 : // The cursor must point to the previous instruction for a possible instruction insertion.
1006 0 : _cc->_setCursor(node->prev());
1007 :
1008 : // Use TryMode of `switchToAssignment()` if possible.
1009 0 : if (target->hasEntryAssignment()) {
1010 0 : ASMJIT_PROPAGATE(switchToAssignment(target->entryPhysToWorkMap(), target->liveIn(), target->isAllocated(), true));
1011 : }
1012 :
1013 0 : ASMJIT_PROPAGATE(allocInst(node));
1014 0 : ASMJIT_PROPAGATE(spillRegsBeforeEntry(target));
1015 :
1016 0 : if (target->hasEntryAssignment()) {
1017 0 : BaseNode* injectionPoint = _pass->extraBlock()->prev();
1018 0 : BaseNode* prevCursor = _cc->setCursor(injectionPoint);
1019 :
1020 0 : _tmpAssignment.copyFrom(_curAssignment);
1021 0 : ASMJIT_PROPAGATE(switchToAssignment(target->entryPhysToWorkMap(), target->liveIn(), target->isAllocated(), false));
1022 :
1023 0 : BaseNode* curCursor = _cc->cursor();
1024 0 : if (curCursor != injectionPoint) {
1025 : // Additional instructions emitted to switch from the current state to the `target` state. This means
1026 : // that we have to move these instructions into an independent code block and patch the jump location.
1027 0 : Operand& targetOp = node->op(node->opCount() - 1);
1028 0 : if (ASMJIT_UNLIKELY(!targetOp.isLabel()))
1029 0 : return DebugUtils::errored(kErrorInvalidState);
1030 :
1031 0 : Label trampoline = _cc->newLabel();
1032 : Label savedTarget = targetOp.as<Label>();
1033 :
1034 : // Patch `target` to point to the `trampoline` we just created.
1035 0 : targetOp = trampoline;
1036 :
1037 : // Clear a possible SHORT form as we have no clue now if the SHORT form would be encodable after patching
1038 : // the target to `trampoline` (X86 specific).
1039 : node->clearOptions(InstOptions::kShortForm);
1040 :
1041 : // Finalize the switch assignment sequence.
1042 0 : ASMJIT_PROPAGATE(_pass->emitJump(savedTarget));
1043 0 : _cc->_setCursor(injectionPoint);
1044 0 : _cc->bind(trampoline);
1045 : }
1046 :
1047 0 : _cc->_setCursor(prevCursor);
1048 : _curAssignment.swap(_tmpAssignment);
1049 : }
1050 : else {
1051 0 : ASMJIT_PROPAGATE(_pass->setBlockEntryAssignment(target, block(), _curAssignment));
1052 : }
1053 :
1054 : return kErrorOk;
1055 : }
1056 :
1057 0 : Error RALocalAllocator::allocJumpTable(InstNode* node, const RABlocks& targets, RABlock* cont) noexcept {
1058 : // TODO: Do we really need to use `cont`?
1059 : DebugUtils::unused(cont);
1060 :
1061 0 : if (targets.empty())
1062 : return DebugUtils::errored(kErrorInvalidState);
1063 :
1064 : // The cursor must point to the previous instruction for a possible instruction insertion.
1065 0 : _cc->_setCursor(node->prev());
1066 :
1067 : // All `targets` should have the same sharedAssignmentId, we just read the first.
1068 0 : RABlock* anyTarget = targets[0];
1069 0 : if (!anyTarget->hasSharedAssignmentId())
1070 : return DebugUtils::errored(kErrorInvalidState);
1071 :
1072 0 : RASharedAssignment& sharedAssignment = _pass->_sharedAssignments[anyTarget->sharedAssignmentId()];
1073 :
1074 0 : ASMJIT_PROPAGATE(allocInst(node));
1075 :
1076 0 : if (!sharedAssignment.empty()) {
1077 0 : ASMJIT_PROPAGATE(switchToAssignment(
1078 : sharedAssignment.physToWorkMap(),
1079 : sharedAssignment.liveIn(),
1080 : true, // Read-only.
1081 : false // Try-mode.
1082 : ));
1083 : }
1084 :
1085 0 : ASMJIT_PROPAGATE(spillRegsBeforeEntry(anyTarget));
1086 :
1087 0 : if (sharedAssignment.empty()) {
1088 0 : ASMJIT_PROPAGATE(_pass->setBlockEntryAssignment(anyTarget, block(), _curAssignment));
1089 : }
1090 :
1091 : return kErrorOk;
1092 : }
1093 :
1094 : // RALocalAllocator - Decision Making
1095 : // ==================================
1096 :
1097 303933 : uint32_t RALocalAllocator::decideOnAssignment(RegGroup group, uint32_t workId, uint32_t physId, RegMask allocableRegs) const noexcept {
1098 : ASMJIT_ASSERT(allocableRegs != 0);
1099 : DebugUtils::unused(group, physId);
1100 :
1101 : RAWorkReg* workReg = workRegById(workId);
1102 :
1103 : // Prefer home register id, if possible.
1104 303933 : if (workReg->hasHomeRegId()) {
1105 : uint32_t homeId = workReg->homeRegId();
1106 299176 : if (Support::bitTest(allocableRegs, homeId))
1107 : return homeId;
1108 : }
1109 :
1110 : // Prefer registers used upon block entries.
1111 : RegMask previouslyAssignedRegs = workReg->allocatedMask();
1112 26179 : if (allocableRegs & previouslyAssignedRegs)
1113 : allocableRegs &= previouslyAssignedRegs;
1114 :
1115 26179 : return Support::ctz(allocableRegs);
1116 : }
1117 :
1118 968 : uint32_t RALocalAllocator::decideOnReassignment(RegGroup group, uint32_t workId, uint32_t physId, RegMask allocableRegs) const noexcept {
1119 : ASMJIT_ASSERT(allocableRegs != 0);
1120 : DebugUtils::unused(group, physId);
1121 :
1122 : RAWorkReg* workReg = workRegById(workId);
1123 :
1124 : // Prefer allocating back to HomeId, if possible.
1125 968 : if (workReg->hasHomeRegId()) {
1126 707 : if (Support::bitTest(allocableRegs, workReg->homeRegId()))
1127 : return workReg->homeRegId();
1128 : }
1129 :
1130 : // TODO: [Register Allocator] This could be improved.
1131 :
1132 : // Decided to SPILL.
1133 : return RAAssignment::kPhysNone;
1134 : }
1135 :
1136 7346 : uint32_t RALocalAllocator::decideOnSpillFor(RegGroup group, uint32_t workId, RegMask spillableRegs, uint32_t* spillWorkId) const noexcept {
1137 : // May be used in the future to decide which register would be best to spill so `workId` can be assigned.
1138 : DebugUtils::unused(workId);
1139 : ASMJIT_ASSERT(spillableRegs != 0);
1140 :
1141 : Support::BitWordIterator<RegMask> it(spillableRegs);
1142 : uint32_t bestPhysId = it.next();
1143 : uint32_t bestWorkId = _curAssignment.physToWorkId(group, bestPhysId);
1144 :
1145 : // Avoid calculating the cost model if there is only one spillable register.
1146 7346 : if (it.hasNext()) {
1147 7346 : uint32_t bestCost = calculateSpillCost(group, bestWorkId, bestPhysId);
1148 : do {
1149 : uint32_t localPhysId = it.next();
1150 : uint32_t localWorkId = _curAssignment.physToWorkId(group, localPhysId);
1151 110190 : uint32_t localCost = calculateSpillCost(group, localWorkId, localPhysId);
1152 :
1153 110190 : if (localCost < bestCost) {
1154 : bestCost = localCost;
1155 : bestPhysId = localPhysId;
1156 : bestWorkId = localWorkId;
1157 : }
1158 110190 : } while (it.hasNext());
1159 : }
1160 :
1161 7346 : *spillWorkId = bestWorkId;
1162 7346 : return bestPhysId;
1163 : }
1164 :
1165 : ASMJIT_END_NAMESPACE
1166 :
1167 : #endif // !ASMJIT_NO_COMPILER
1168 :
1169 : #endif // __PLUMED_HAS_ASMJIT
|
