Line data Source code
1 : #ifdef __PLUMED_HAS_ASMJIT
2 : // This file is part of AsmJit project <https://asmjit.com>
3 : //
4 : // See asmjit.h or LICENSE.md for license and copyright information
5 : // SPDX-License-Identifier: Zlib
6 :
7 : #ifndef ASMJIT_CORE_RAPASS_P_H_INCLUDED
8 : #define ASMJIT_CORE_RAPASS_P_H_INCLUDED
9 :
10 : #include "api-config.h"
11 : #ifndef ASMJIT_NO_COMPILER
12 :
13 : #include "compiler.h"
14 : #include "emithelper_p.h"
15 : #include "raassignment_p.h"
16 : #include "radefs_p.h"
17 : #include "rastack_p.h"
18 : #include "support.h"
19 :
20 : ASMJIT_BEGIN_NAMESPACE
21 :
22 : //! \cond INTERNAL
23 : //! \addtogroup asmjit_ra
24 : //! \{
25 :
26 : //! Flags used by \ref RABlock.
27 : enum class RABlockFlags : uint32_t {
28 : //! No flags.
29 : kNone = 0,
30 :
31 : //! Block has been constructed from nodes.
32 : kIsConstructed = 0x00000001u,
33 : //! Block is reachable (set by `buildCFGViews()`).
34 : kIsReachable = 0x00000002u,
35 : //! Block is a target (has an associated label or multiple labels).
36 : kIsTargetable = 0x00000004u,
37 : //! Block has been allocated.
38 : kIsAllocated = 0x00000008u,
39 : //! Block is a function-exit.
40 : kIsFuncExit = 0x00000010u,
41 :
42 : //! Block has a terminator (jump, conditional jump, ret).
43 : kHasTerminator = 0x00000100u,
44 : //! Block naturally flows to the next block.
45 : kHasConsecutive = 0x00000200u,
46 : //! Block has a jump to a jump-table at the end.
47 : kHasJumpTable = 0x00000400u,
48 : //! Block contains fixed registers (precolored).
49 : kHasFixedRegs = 0x00000800u,
50 : //! Block contains function calls.
51 : kHasFuncCalls = 0x00001000u
52 : };
53 266940 : ASMJIT_DEFINE_ENUM_FLAGS(RABlockFlags)
54 :
55 : //! Basic block used by register allocator pass.
56 : class RABlock {
57 : public:
58 : ASMJIT_NONCOPYABLE(RABlock)
59 :
60 : typedef RAAssignment::PhysToWorkMap PhysToWorkMap;
61 : typedef RAAssignment::WorkToPhysMap WorkToPhysMap;
62 :
63 : //! \name Constants
64 : //! \{
65 :
66 : enum : uint32_t {
67 : //! Unassigned block id.
68 : kUnassignedId = 0xFFFFFFFFu
69 : };
70 :
71 : enum LiveType : uint32_t {
72 : kLiveIn = 0,
73 : kLiveOut = 1,
74 : kLiveGen = 2,
75 : kLiveKill = 3,
76 : kLiveCount = 4
77 : };
78 :
79 : //! \}
80 :
81 : //! \name Members
82 : //! \{
83 :
84 : //! Register allocator pass.
85 : BaseRAPass* _ra;
86 :
87 : //! Block id (indexed from zero).
88 : uint32_t _blockId = kUnassignedId;
89 : //! Block flags, see `Flags`.
90 : RABlockFlags _flags = RABlockFlags::kNone;
91 :
92 : //! First `BaseNode` of this block (inclusive).
93 : BaseNode* _first = nullptr;
94 : //! Last `BaseNode` of this block (inclusive).
95 : BaseNode* _last = nullptr;
96 :
97 : //! Initial position of this block (inclusive).
98 : uint32_t _firstPosition = 0;
99 : //! End position of this block (exclusive).
100 : uint32_t _endPosition = 0;
101 :
102 : //! Weight of this block (default 0, each loop adds one).
103 : uint32_t _weight = 0;
104 : //! Post-order view order, used during POV construction.
105 : uint32_t _povOrder = 0;
106 :
107 : //! Basic statistics about registers.
108 : RARegsStats _regsStats = RARegsStats();
109 : //! Maximum live-count per register group.
110 : RALiveCount _maxLiveCount = RALiveCount();
111 :
112 : //! Timestamp (used by block visitors).
113 : mutable uint64_t _timestamp = 0;
114 : //! Immediate dominator of this block.
115 : RABlock* _idom = nullptr;
116 :
117 : //! Block predecessors.
118 : RABlocks _predecessors {};
119 : //! Block successors.
120 : RABlocks _successors {};
121 :
122 : //! Liveness in/out/use/kill.
123 : ZoneBitVector _liveBits[kLiveCount] {};
124 :
125 : //! Shared assignment it or `Globals::kInvalidId` if this block doesn't have shared assignment.
126 : //! See \ref RASharedAssignment for more details.
127 : uint32_t _sharedAssignmentId = Globals::kInvalidId;
128 : //! Scratch registers that cannot be allocated upon block entry.
129 : RegMask _entryScratchGpRegs = 0;
130 : //! Scratch registers used at exit, by a terminator instruction.
131 : RegMask _exitScratchGpRegs = 0;
132 :
133 : //! Register assignment on entry.
134 : PhysToWorkMap* _entryPhysToWorkMap = nullptr;
135 :
136 : //! \}
137 :
138 : //! \name Construction & Destruction
139 : //! \{
140 :
141 54984 : inline RABlock(BaseRAPass* ra) noexcept
142 439872 : : _ra(ra) {}
143 :
144 : //! \}
145 :
146 : //! \name Accessors
147 : //! \{
148 :
149 : inline BaseRAPass* pass() const noexcept { return _ra; }
150 : inline ZoneAllocator* allocator() const noexcept;
151 :
152 137460 : inline uint32_t blockId() const noexcept { return _blockId; }
153 : inline RABlockFlags flags() const noexcept { return _flags; }
154 :
155 109968 : inline bool hasFlag(RABlockFlags flag) const noexcept { return Support::test(_flags, flag); }
156 9639 : inline void addFlags(RABlockFlags flags) noexcept { _flags |= flags; }
157 :
158 : inline bool isAssigned() const noexcept { return _blockId != kUnassignedId; }
159 :
160 : inline bool isConstructed() const noexcept { return hasFlag(RABlockFlags::kIsConstructed); }
161 : inline bool isReachable() const noexcept { return hasFlag(RABlockFlags::kIsReachable); }
162 : inline bool isTargetable() const noexcept { return hasFlag(RABlockFlags::kIsTargetable); }
163 : inline bool isAllocated() const noexcept { return hasFlag(RABlockFlags::kIsAllocated); }
164 : inline bool isFuncExit() const noexcept { return hasFlag(RABlockFlags::kIsFuncExit); }
165 : inline bool hasTerminator() const noexcept { return hasFlag(RABlockFlags::kHasTerminator); }
166 : inline bool hasConsecutive() const noexcept { return hasFlag(RABlockFlags::kHasConsecutive); }
167 : inline bool hasJumpTable() const noexcept { return hasFlag(RABlockFlags::kHasJumpTable); }
168 :
169 : inline void makeConstructed(const RARegsStats& regStats) noexcept {
170 : _flags |= RABlockFlags::kIsConstructed;
171 : _regsStats.combineWith(regStats);
172 : }
173 :
174 : inline void makeReachable() noexcept { _flags |= RABlockFlags::kIsReachable; }
175 : inline void makeTargetable() noexcept { _flags |= RABlockFlags::kIsTargetable; }
176 : inline void makeAllocated() noexcept { _flags |= RABlockFlags::kIsAllocated; }
177 :
178 : inline const RARegsStats& regsStats() const noexcept { return _regsStats; }
179 :
180 : inline bool hasPredecessors() const noexcept { return !_predecessors.empty(); }
181 : inline bool hasSuccessors() const noexcept { return !_successors.empty(); }
182 :
183 27492 : inline bool hasSuccessor(RABlock* block) noexcept {
184 27492 : if (block->_predecessors.size() < _successors.size())
185 0 : return block->_predecessors.contains(this);
186 : else
187 27492 : return _successors.contains(block);
188 : }
189 :
190 : inline const RABlocks& predecessors() const noexcept { return _predecessors; }
191 0 : inline const RABlocks& successors() const noexcept { return _successors; }
192 :
193 82570 : inline BaseNode* first() const noexcept { return _first; }
194 164952 : inline BaseNode* last() const noexcept { return _last; }
195 :
196 82476 : inline void setFirst(BaseNode* node) noexcept { _first = node; }
197 219936 : inline void setLast(BaseNode* node) noexcept { _last = node; }
198 :
199 : inline uint32_t firstPosition() const noexcept { return _firstPosition; }
200 54984 : inline void setFirstPosition(uint32_t position) noexcept { _firstPosition = position; }
201 :
202 : inline uint32_t endPosition() const noexcept { return _endPosition; }
203 54984 : inline void setEndPosition(uint32_t position) noexcept { _endPosition = position; }
204 :
205 0 : inline uint32_t povOrder() const noexcept { return _povOrder; }
206 :
207 : inline RegMask entryScratchGpRegs() const noexcept;
208 0 : inline RegMask exitScratchGpRegs() const noexcept { return _exitScratchGpRegs; }
209 :
210 : inline void addEntryScratchGpRegs(RegMask regMask) noexcept { _entryScratchGpRegs |= regMask; }
211 0 : inline void addExitScratchGpRegs(RegMask regMask) noexcept { _exitScratchGpRegs |= regMask; }
212 :
213 82476 : inline bool hasSharedAssignmentId() const noexcept { return _sharedAssignmentId != Globals::kInvalidId; }
214 0 : inline uint32_t sharedAssignmentId() const noexcept { return _sharedAssignmentId; }
215 0 : inline void setSharedAssignmentId(uint32_t id) noexcept { _sharedAssignmentId = id; }
216 :
217 : inline uint64_t timestamp() const noexcept { return _timestamp; }
218 0 : inline bool hasTimestamp(uint64_t ts) const noexcept { return _timestamp == ts; }
219 0 : inline void setTimestamp(uint64_t ts) const noexcept { _timestamp = ts; }
220 : inline void resetTimestamp() const noexcept { _timestamp = 0; }
221 :
222 0 : inline RABlock* consecutive() const noexcept { return hasConsecutive() ? _successors[0] : nullptr; }
223 :
224 109968 : inline RABlock* iDom() noexcept { return _idom; }
225 0 : inline const RABlock* iDom() const noexcept { return _idom; }
226 27492 : inline void setIDom(RABlock* block) noexcept { _idom = block; }
227 :
228 0 : inline ZoneBitVector& liveIn() noexcept { return _liveBits[kLiveIn]; }
229 : inline const ZoneBitVector& liveIn() const noexcept { return _liveBits[kLiveIn]; }
230 :
231 : inline ZoneBitVector& liveOut() noexcept { return _liveBits[kLiveOut]; }
232 27492 : inline const ZoneBitVector& liveOut() const noexcept { return _liveBits[kLiveOut]; }
233 :
234 : inline ZoneBitVector& gen() noexcept { return _liveBits[kLiveGen]; }
235 : inline const ZoneBitVector& gen() const noexcept { return _liveBits[kLiveGen]; }
236 :
237 : inline ZoneBitVector& kill() noexcept { return _liveBits[kLiveKill]; }
238 : inline const ZoneBitVector& kill() const noexcept { return _liveBits[kLiveKill]; }
239 :
240 54984 : inline Error resizeLiveBits(uint32_t size) noexcept {
241 54984 : ASMJIT_PROPAGATE(_liveBits[kLiveIn ].resize(allocator(), size));
242 54984 : ASMJIT_PROPAGATE(_liveBits[kLiveOut ].resize(allocator(), size));
243 54984 : ASMJIT_PROPAGATE(_liveBits[kLiveGen ].resize(allocator(), size));
244 54984 : ASMJIT_PROPAGATE(_liveBits[kLiveKill].resize(allocator(), size));
245 : return kErrorOk;
246 : }
247 :
248 27492 : inline bool hasEntryAssignment() const noexcept { return _entryPhysToWorkMap != nullptr; }
249 27492 : inline PhysToWorkMap* entryPhysToWorkMap() const noexcept { return _entryPhysToWorkMap; }
250 54984 : inline void setEntryAssignment(PhysToWorkMap* physToWorkMap) noexcept { _entryPhysToWorkMap = physToWorkMap; }
251 :
252 : //! \}
253 :
254 : //! \name Utilities
255 : //! \{
256 :
257 : //! Adds a successor to this block, and predecessor to `successor`, making connection on both sides.
258 : //!
259 : //! This API must be used to manage successors and predecessors, never manage it manually.
260 : Error appendSuccessor(RABlock* successor) noexcept;
261 :
262 : //! Similar to `appendSuccessor()`, but does prepend instead append.
263 : //!
264 : //! This function is used to add a natural flow (always first) to the block.
265 : Error prependSuccessor(RABlock* successor) noexcept;
266 :
267 : //! \}
268 : };
269 :
270 : //! Register allocator's data associated with each `InstNode`.
271 : class RAInst {
272 : public:
273 : ASMJIT_NONCOPYABLE(RAInst)
274 :
275 : //! \name Members
276 : //! \{
277 :
278 : //! Parent block.
279 : RABlock* _block;
280 : //! Instruction RW flags.
281 : InstRWFlags _instRWFlags;
282 : //! Aggregated RATiedFlags from all operands & instruction specific flags.
283 : RATiedFlags _flags;
284 : //! Total count of RATiedReg's.
285 : uint32_t _tiedTotal;
286 : //! Index of RATiedReg's per register group.
287 : RARegIndex _tiedIndex;
288 : //! Count of RATiedReg's per register group.
289 : RARegCount _tiedCount;
290 : //! Number of live, and thus interfering VirtReg's at this point.
291 : RALiveCount _liveCount;
292 : //! Fixed physical registers used.
293 : RARegMask _usedRegs;
294 : //! Clobbered registers (by a function call).
295 : RARegMask _clobberedRegs;
296 : //! Tied registers.
297 : RATiedReg _tiedRegs[1];
298 :
299 : //! \}
300 :
301 : //! \name Construction & Destruction
302 : //! \{
303 :
304 331808 : inline RAInst(RABlock* block, InstRWFlags instRWFlags, RATiedFlags tiedFlags, uint32_t tiedTotal, const RARegMask& clobberedRegs) noexcept {
305 331808 : _block = block;
306 331808 : _instRWFlags = instRWFlags;
307 331808 : _flags = tiedFlags;
308 331808 : _tiedTotal = tiedTotal;
309 : _tiedIndex.reset();
310 : _tiedCount.reset();
311 : _liveCount.reset();
312 : _usedRegs.reset();
313 331808 : _clobberedRegs = clobberedRegs;
314 331808 : }
315 :
316 : //! \}
317 :
318 : //! \name Accessors
319 : //! \{
320 :
321 : //! Returns instruction RW flags.
322 : inline InstRWFlags instRWFlags() const noexcept { return _instRWFlags; };
323 : //! Tests whether the given `flag` is present in instruction RW flags.
324 331808 : inline bool hasInstRWFlag(InstRWFlags flag) const noexcept { return Support::test(_instRWFlags, flag); }
325 : //! Adds `flags` to instruction RW flags.
326 : inline void addInstRWFlags(InstRWFlags flags) noexcept { _instRWFlags |= flags; }
327 :
328 : //! Returns the instruction flags.
329 : inline RATiedFlags flags() const noexcept { return _flags; }
330 : //! Tests whether the instruction has flag `flag`.
331 331808 : inline bool hasFlag(RATiedFlags flag) const noexcept { return Support::test(_flags, flag); }
332 : //! Replaces the existing instruction flags with `flags`.
333 : inline void setFlags(RATiedFlags flags) noexcept { _flags = flags; }
334 : //! Adds instruction `flags` to this RAInst.
335 : inline void addFlags(RATiedFlags flags) noexcept { _flags |= flags; }
336 : //! Clears instruction `flags` from this RAInst.
337 : inline void clearFlags(RATiedFlags flags) noexcept { _flags &= ~flags; }
338 :
339 : //! Tests whether this instruction can be transformed to another instruction if necessary.
340 : inline bool isTransformable() const noexcept { return hasFlag(RATiedFlags::kInst_IsTransformable); }
341 :
342 : //! Returns the associated block with this RAInst.
343 27492 : inline RABlock* block() const noexcept { return _block; }
344 :
345 : //! Returns tied registers (all).
346 1580727 : inline RATiedReg* tiedRegs() const noexcept { return const_cast<RATiedReg*>(_tiedRegs); }
347 : //! Returns tied registers for a given `group`.
348 1327232 : inline RATiedReg* tiedRegs(RegGroup group) const noexcept { return const_cast<RATiedReg*>(_tiedRegs) + _tiedIndex.get(group); }
349 :
350 : //! Returns count of all tied registers.
351 1317593 : inline uint32_t tiedCount() const noexcept { return _tiedTotal; }
352 : //! Returns count of tied registers of a given `group`.
353 : inline uint32_t tiedCount(RegGroup group) const noexcept { return _tiedCount[group]; }
354 :
355 : //! Returns `RATiedReg` at the given `index`.
356 : inline RATiedReg* tiedAt(uint32_t index) const noexcept {
357 : ASMJIT_ASSERT(index < _tiedTotal);
358 585303 : return tiedRegs() + index;
359 : }
360 :
361 : //! Returns `RATiedReg` at the given `index` of the given register `group`.
362 : inline RATiedReg* tiedOf(RegGroup group, uint32_t index) const noexcept {
363 : ASMJIT_ASSERT(index < _tiedCount.get(group));
364 : return tiedRegs(group) + index;
365 : }
366 :
367 : inline void setTiedAt(uint32_t index, RATiedReg& tied) noexcept {
368 : ASMJIT_ASSERT(index < _tiedTotal);
369 : _tiedRegs[index] = tied;
370 : }
371 :
372 : //! \name Static Functions
373 : //! \{
374 :
375 : static inline size_t sizeOf(uint32_t tiedRegCount) noexcept {
376 331808 : return sizeof(RAInst) - sizeof(RATiedReg) + tiedRegCount * sizeof(RATiedReg);
377 : }
378 :
379 : //! \}
380 : };
381 :
382 : //! A helper class that is used to build an array of RATiedReg items that are then copied to `RAInst`.
383 : class RAInstBuilder {
384 : public:
385 : ASMJIT_NONCOPYABLE(RAInstBuilder)
386 :
387 : //! \name Members
388 : //! \{
389 :
390 : //! Instruction RW flags.
391 : InstRWFlags _instRWFlags;
392 :
393 : //! Flags combined from all RATiedReg's.
394 : RATiedFlags _aggregatedFlags;
395 : //! Flags that will be cleared before storing the aggregated flags to `RAInst`.
396 : RATiedFlags _forbiddenFlags;
397 : RARegCount _count;
398 : RARegsStats _stats;
399 :
400 : RARegMask _used;
401 : RARegMask _clobbered;
402 :
403 : //! Current tied register in `_tiedRegs`.
404 : RATiedReg* _cur;
405 : //! Array of temporary tied registers.
406 : RATiedReg _tiedRegs[128];
407 :
408 : //! \}
409 :
410 : //! \name Construction & Destruction
411 : //! \{
412 :
413 27492 : inline RAInstBuilder() noexcept { reset(); }
414 :
415 : inline void init() noexcept { reset(); }
416 359300 : inline void reset() noexcept {
417 359300 : _instRWFlags = InstRWFlags::kNone;
418 359300 : _aggregatedFlags = RATiedFlags::kNone;
419 359300 : _forbiddenFlags = RATiedFlags::kNone;
420 : _count.reset();
421 : _stats.reset();
422 : _used.reset();
423 : _clobbered.reset();
424 359300 : _cur = _tiedRegs;
425 359300 : }
426 :
427 : //! \}
428 :
429 : //! \name Accessors
430 : //! \{
431 :
432 331808 : inline InstRWFlags instRWFlags() const noexcept { return _instRWFlags; }
433 : inline bool hasInstRWFlag(InstRWFlags flag) const noexcept { return Support::test(_instRWFlags, flag); }
434 0 : inline void addInstRWFlags(InstRWFlags flags) noexcept { _instRWFlags |= flags; }
435 : inline void clearInstRWFlags(InstRWFlags flags) noexcept { _instRWFlags &= ~flags; }
436 :
437 331808 : inline RATiedFlags aggregatedFlags() const noexcept { return _aggregatedFlags; }
438 6 : inline void addAggregatedFlags(RATiedFlags flags) noexcept { _aggregatedFlags |= flags; }
439 :
440 331808 : inline RATiedFlags forbiddenFlags() const noexcept { return _forbiddenFlags; }
441 : inline void addForbiddenFlags(RATiedFlags flags) noexcept { _forbiddenFlags |= flags; }
442 :
443 : //! Returns the number of tied registers added to the builder.
444 636124 : inline uint32_t tiedRegCount() const noexcept { return uint32_t((size_t)(_cur - _tiedRegs)); }
445 :
446 0 : inline RATiedReg* begin() noexcept { return _tiedRegs; }
447 0 : inline RATiedReg* end() noexcept { return _cur; }
448 :
449 : inline const RATiedReg* begin() const noexcept { return _tiedRegs; }
450 : inline const RATiedReg* end() const noexcept { return _cur; }
451 :
452 : //! Returns `RATiedReg` at the given `index`.
453 : inline RATiedReg* operator[](uint32_t index) noexcept {
454 : ASMJIT_ASSERT(index < tiedRegCount());
455 : return &_tiedRegs[index];
456 : }
457 :
458 : //! Returns `RATiedReg` at the given `index`. (const).
459 : inline const RATiedReg* operator[](uint32_t index) const noexcept {
460 : ASMJIT_ASSERT(index < tiedRegCount());
461 : return &_tiedRegs[index];
462 : }
463 :
464 : //! \}
465 :
466 : //! \name Utilities
467 : //! \{
468 :
469 673873 : Error add(
470 : RAWorkReg* workReg,
471 : RATiedFlags flags,
472 : RegMask useRegMask, uint32_t useId, uint32_t useRewriteMask,
473 : RegMask outRegMask, uint32_t outId, uint32_t outRewriteMask,
474 : uint32_t rmSize = 0,
475 : uint32_t consecutiveParent = Globals::kInvalidId) noexcept {
476 :
477 : RegGroup group = workReg->group();
478 : RATiedReg* tiedReg = workReg->tiedReg();
479 :
480 673873 : if (useId != BaseReg::kIdBad) {
481 : _stats.makeFixed(group);
482 27492 : _used[group] |= Support::bitMask(useId);
483 : flags |= RATiedFlags::kUseFixed;
484 : }
485 :
486 673873 : if (outId != BaseReg::kIdBad) {
487 0 : _clobbered[group] |= Support::bitMask(outId);
488 : flags |= RATiedFlags::kOutFixed;
489 : }
490 :
491 : _aggregatedFlags |= flags;
492 : _stats.makeUsed(group);
493 :
494 673873 : if (!tiedReg) {
495 : // Could happen when the builder is not reset properly after each instruction.
496 : ASMJIT_ASSERT(tiedRegCount() < ASMJIT_ARRAY_SIZE(_tiedRegs));
497 :
498 594942 : tiedReg = _cur++;
499 : tiedReg->init(workReg->workId(), flags, useRegMask, useId, useRewriteMask, outRegMask, outId, outRewriteMask, rmSize, consecutiveParent);
500 : workReg->setTiedReg(tiedReg);
501 :
502 : _count.add(group);
503 594942 : return kErrorOk;
504 : }
505 : else {
506 78931 : if (consecutiveParent != tiedReg->consecutiveParent()) {
507 0 : if (tiedReg->consecutiveParent() != Globals::kInvalidId)
508 : return DebugUtils::errored(kErrorInvalidState);
509 0 : tiedReg->_consecutiveParent = consecutiveParent;
510 : }
511 :
512 78931 : if (useId != BaseReg::kIdBad) {
513 0 : if (ASMJIT_UNLIKELY(tiedReg->hasUseId()))
514 : return DebugUtils::errored(kErrorOverlappedRegs);
515 : tiedReg->setUseId(useId);
516 : }
517 :
518 78931 : if (outId != BaseReg::kIdBad) {
519 0 : if (ASMJIT_UNLIKELY(tiedReg->hasOutId()))
520 : return DebugUtils::errored(kErrorOverlappedRegs);
521 : tiedReg->setOutId(outId);
522 : }
523 :
524 : tiedReg->addRefCount();
525 : tiedReg->addFlags(flags);
526 78931 : tiedReg->_useRegMask &= useRegMask;
527 78931 : tiedReg->_useRewriteMask |= useRewriteMask;
528 78931 : tiedReg->_outRegMask &= outRegMask;
529 78931 : tiedReg->_outRewriteMask |= outRewriteMask;
530 78931 : tiedReg->_rmSize = uint8_t(Support::max<uint32_t>(tiedReg->rmSize(), rmSize));
531 78931 : return kErrorOk;
532 : }
533 : }
534 :
535 9873 : Error addCallArg(RAWorkReg* workReg, uint32_t useId) noexcept {
536 : ASMJIT_ASSERT(useId != BaseReg::kIdBad);
537 :
538 : RATiedFlags flags = RATiedFlags::kUse | RATiedFlags::kRead | RATiedFlags::kUseFixed;
539 : RegGroup group = workReg->group();
540 : RegMask allocable = Support::bitMask(useId);
541 :
542 : _aggregatedFlags |= flags;
543 9873 : _used[group] |= allocable;
544 : _stats.makeFixed(group);
545 : _stats.makeUsed(group);
546 :
547 : RATiedReg* tiedReg = workReg->tiedReg();
548 9873 : if (!tiedReg) {
549 : // Could happen when the builder is not reset properly after each instruction.
550 : ASMJIT_ASSERT(tiedRegCount() < ASMJIT_ARRAY_SIZE(_tiedRegs));
551 :
552 9873 : tiedReg = _cur++;
553 : tiedReg->init(workReg->workId(), flags, allocable, useId, 0, allocable, BaseReg::kIdBad, 0);
554 : workReg->setTiedReg(tiedReg);
555 :
556 : _count.add(group);
557 9873 : return kErrorOk;
558 : }
559 : else {
560 0 : if (tiedReg->hasUseId()) {
561 : flags |= RATiedFlags::kDuplicate;
562 0 : tiedReg->_useRegMask |= allocable;
563 : }
564 : else {
565 : tiedReg->setUseId(useId);
566 0 : tiedReg->_useRegMask &= allocable;
567 : }
568 :
569 : tiedReg->addRefCount();
570 : tiedReg->addFlags(flags);
571 0 : return kErrorOk;
572 : }
573 : }
574 :
575 9639 : Error addCallRet(RAWorkReg* workReg, uint32_t outId) noexcept {
576 : ASMJIT_ASSERT(outId != BaseReg::kIdBad);
577 :
578 : RATiedFlags flags = RATiedFlags::kOut | RATiedFlags::kWrite | RATiedFlags::kOutFixed;
579 : RegGroup group = workReg->group();
580 : RegMask outRegs = Support::bitMask(outId);
581 :
582 : _aggregatedFlags |= flags;
583 9639 : _used[group] |= outRegs;
584 : _stats.makeFixed(group);
585 : _stats.makeUsed(group);
586 :
587 : RATiedReg* tiedReg = workReg->tiedReg();
588 9639 : if (!tiedReg) {
589 : // Could happen when the builder is not reset properly after each instruction.
590 : ASMJIT_ASSERT(tiedRegCount() < ASMJIT_ARRAY_SIZE(_tiedRegs));
591 :
592 9639 : tiedReg = _cur++;
593 : tiedReg->init(workReg->workId(), flags, Support::allOnes<RegMask>(), BaseReg::kIdBad, 0, outRegs, outId, 0);
594 : workReg->setTiedReg(tiedReg);
595 :
596 : _count.add(group);
597 9639 : return kErrorOk;
598 : }
599 : else {
600 0 : if (tiedReg->hasOutId())
601 : return DebugUtils::errored(kErrorOverlappedRegs);
602 :
603 : tiedReg->addRefCount();
604 : tiedReg->addFlags(flags);
605 : tiedReg->setOutId(outId);
606 0 : return kErrorOk;
607 : }
608 : }
609 :
610 : //! \}
611 : };
612 :
613 : //! Intersection of multiple register assignments.
614 : //!
615 : //! See \ref RAAssignment for more information about register assignments.
616 : class RASharedAssignment {
617 : public:
618 : typedef RAAssignment::PhysToWorkMap PhysToWorkMap;
619 : typedef RAAssignment::WorkToPhysMap WorkToPhysMap;
620 :
621 : //! \name Members
622 : //! \{
623 :
624 : //! Bit-mask of registers that cannot be used upon a block entry, for each block that has this shared assignment.
625 : //! Scratch registers can come from ISA limits (like jecx/loop instructions on x86) or because the registers are
626 : //! used by jump/branch instruction that uses registers to perform an indirect jump.
627 : RegMask _entryScratchGpRegs = 0;
628 : //! Union of all live-in registers.
629 : ZoneBitVector _liveIn {};
630 : //! Register assignment (PhysToWork).
631 : PhysToWorkMap* _physToWorkMap = nullptr;
632 :
633 : //! \}
634 :
635 : //! \name Accessors
636 : //! \{
637 :
638 0 : inline bool empty() const noexcept { return _physToWorkMap == nullptr; }
639 :
640 0 : inline RegMask entryScratchGpRegs() const noexcept { return _entryScratchGpRegs; }
641 0 : inline void addEntryScratchGpRegs(RegMask mask) noexcept { _entryScratchGpRegs |= mask; }
642 :
643 0 : inline const ZoneBitVector& liveIn() const noexcept { return _liveIn; }
644 :
645 : inline PhysToWorkMap* physToWorkMap() const noexcept { return _physToWorkMap; }
646 0 : inline void assignPhysToWorkMap(PhysToWorkMap* physToWorkMap) noexcept { _physToWorkMap = physToWorkMap; }
647 :
648 : //! \}
649 : };
650 :
651 : //! Register allocation pass used by `BaseCompiler`.
652 : class BaseRAPass : public FuncPass {
653 : public:
654 : ASMJIT_NONCOPYABLE(BaseRAPass)
655 : typedef FuncPass Base;
656 :
657 : enum : uint32_t {
658 : kCallArgWeight = 80
659 : };
660 :
661 : typedef RAAssignment::PhysToWorkMap PhysToWorkMap;
662 : typedef RAAssignment::WorkToPhysMap WorkToPhysMap;
663 :
664 : //! \name Members
665 : //! \{
666 :
667 : //! Allocator that uses zone passed to `runOnFunction()`.
668 : ZoneAllocator _allocator {};
669 : //! Emit helper.
670 : BaseEmitHelper* _iEmitHelper = nullptr;
671 :
672 : //! Logger, disabled if null.
673 : Logger* _logger = nullptr;
674 : //! Format options, copied from Logger, or zeroed if there is no logger.
675 : FormatOptions _formatOptions {};
676 : //! Diagnostic options, copied from Emitter, or zeroed if there is no logger.
677 : DiagnosticOptions _diagnosticOptions {};
678 :
679 : //! Function being processed.
680 : FuncNode* _func = nullptr;
681 : //! Stop node.
682 : BaseNode* _stop = nullptr;
683 : //! Node that is used to insert extra code after the function body.
684 : BaseNode* _extraBlock = nullptr;
685 :
686 : //! Blocks (first block is the entry, always exists).
687 : RABlocks _blocks {};
688 : //! Function exit blocks (usually one, but can contain more).
689 : RABlocks _exits {};
690 : //! Post order view (POV).
691 : RABlocks _pov {};
692 :
693 : //! Number of instruction nodes.
694 : uint32_t _instructionCount = 0;
695 : //! Number of created blocks (internal).
696 : uint32_t _createdBlockCount = 0;
697 :
698 : //! Shared assignment blocks.
699 : ZoneVector<RASharedAssignment> _sharedAssignments {};
700 :
701 : //! Timestamp generator (incremental).
702 : mutable uint64_t _lastTimestamp = 0;
703 :
704 : //! Architecture traits.
705 : const ArchTraits* _archTraits = nullptr;
706 : //! Index to physical registers in `RAAssignment::PhysToWorkMap`.
707 : RARegIndex _physRegIndex = RARegIndex();
708 : //! Count of physical registers in `RAAssignment::PhysToWorkMap`.
709 : RARegCount _physRegCount = RARegCount();
710 : //! Total number of physical registers.
711 : uint32_t _physRegTotal = 0;
712 : //! Indexes of a possible scratch registers that can be selected if necessary.
713 : Support::Array<uint8_t, 2> _scratchRegIndexes {};
714 :
715 : //! Registers available for allocation.
716 : RARegMask _availableRegs = RARegMask();
717 : //! Count of physical registers per group.
718 : RARegCount _availableRegCount = RARegCount();
719 : //! Registers clobbered by the function.
720 : RARegMask _clobberedRegs = RARegMask();
721 :
722 : //! Work registers (registers used by the function).
723 : RAWorkRegs _workRegs;
724 : //! Work registers per register group.
725 : Support::Array<RAWorkRegs, Globals::kNumVirtGroups> _workRegsOfGroup;
726 :
727 : //! Register allocation strategy per register group.
728 : Support::Array<RAStrategy, Globals::kNumVirtGroups> _strategy;
729 : //! Global max live-count (from all blocks) per register group.
730 : RALiveCount _globalMaxLiveCount = RALiveCount();
731 : //! Global live spans per register group.
732 : Support::Array<LiveRegSpans*, Globals::kNumVirtGroups> _globalLiveSpans {};
733 : //! Temporary stack slot.
734 : Operand _temporaryMem = Operand();
735 :
736 : //! Stack pointer.
737 : BaseReg _sp = BaseReg();
738 : //! Frame pointer.
739 : BaseReg _fp = BaseReg();
740 : //! Stack manager.
741 : RAStackAllocator _stackAllocator {};
742 : //! Function arguments assignment.
743 : FuncArgsAssignment _argsAssignment {};
744 : //! Some StackArgs have to be assigned to StackSlots.
745 : uint32_t _numStackArgsToStackSlots = 0;
746 :
747 : //! Maximum name-size computed from all WorkRegs.
748 : uint32_t _maxWorkRegNameSize = 0;
749 : //! Temporary string builder used to format comments.
750 : StringTmp<80> _tmpString;
751 :
752 : //! \}
753 :
754 : //! \name Construction & Destruction
755 : //! \{
756 :
757 : BaseRAPass() noexcept;
758 : virtual ~BaseRAPass() noexcept;
759 :
760 : //! \}
761 :
762 : //! \name Accessors
763 : //! \{
764 :
765 : //! Returns \ref Logger passed to \ref runOnFunction().
766 0 : inline Logger* logger() const noexcept { return _logger; }
767 :
768 : //! Returns either a valid logger if the given `option` is set and logging is enabled, or nullptr.
769 137460 : inline Logger* getLoggerIf(DiagnosticOptions option) const noexcept { return Support::test(_diagnosticOptions, option) ? _logger : nullptr; }
770 :
771 : //! Returns whether the diagnostic `option` is enabled.
772 : //!
773 : //! \note Returns false if there is no logger (as diagnostics without logging make no sense).
774 105205 : inline bool hasDiagnosticOption(DiagnosticOptions option) const noexcept { return Support::test(_diagnosticOptions, option); }
775 :
776 : //! Returns \ref Zone passed to \ref runOnFunction().
777 : inline Zone* zone() const noexcept { return _allocator.zone(); }
778 : //! Returns \ref ZoneAllocator used by the register allocator.
779 601218 : inline ZoneAllocator* allocator() const noexcept { return const_cast<ZoneAllocator*>(&_allocator); }
780 :
781 : inline const ZoneVector<RASharedAssignment>& sharedAssignments() const { return _sharedAssignments; }
782 : inline uint32_t sharedAssignmentCount() const noexcept { return _sharedAssignments.size(); }
783 :
784 : //! Returns the current function node.
785 202083 : inline FuncNode* func() const noexcept { return _func; }
786 : //! Returns the stop of the current function.
787 : inline BaseNode* stop() const noexcept { return _stop; }
788 :
789 : //! Returns an extra block used by the current function being processed.
790 0 : inline BaseNode* extraBlock() const noexcept { return _extraBlock; }
791 : //! Sets an extra block, see `extraBlock()`.
792 : inline void setExtraBlock(BaseNode* node) noexcept { _extraBlock = node; }
793 :
794 : inline uint32_t endPosition() const noexcept { return _instructionCount * 2; }
795 :
796 : inline const RARegMask& availableRegs() const noexcept { return _availableRegs; }
797 : inline const RARegMask& cloberredRegs() const noexcept { return _clobberedRegs; }
798 :
799 : //! \}
800 :
801 : //! \name Utilities
802 : //! \{
803 :
804 : inline void makeUnavailable(RegGroup group, uint32_t regId) noexcept {
805 27492 : _availableRegs[group] &= ~Support::bitMask(regId);
806 27492 : _availableRegCount[group]--;
807 0 : }
808 :
809 : //! Runs the register allocator for the given `func`.
810 : Error runOnFunction(Zone* zone, Logger* logger, FuncNode* func) override;
811 :
812 : //! Performs all allocation steps sequentially, called by `runOnFunction()`.
813 : Error onPerformAllSteps() noexcept;
814 :
815 : //! \}
816 :
817 : //! \name Events
818 : //! \{
819 :
820 : //! Called by \ref runOnFunction() before the register allocation to initialize
821 : //! architecture-specific data and constraints.
822 : virtual void onInit() noexcept = 0;
823 :
824 : //! Called by \ref runOnFunction(` after register allocation to clean everything
825 : //! up. Called even if the register allocation failed.
826 : virtual void onDone() noexcept = 0;
827 :
828 : //! \}
829 :
830 : //! \name CFG - Basic-Block Management
831 : //! \{
832 :
833 : //! Returns the function's entry block.
834 : inline RABlock* entryBlock() noexcept {
835 : ASMJIT_ASSERT(!_blocks.empty());
836 82476 : return _blocks[0];
837 : }
838 :
839 : //! \overload
840 : inline const RABlock* entryBlock() const noexcept {
841 : ASMJIT_ASSERT(!_blocks.empty());
842 0 : return _blocks[0];
843 : }
844 :
845 : //! Returns all basic blocks of this function.
846 : inline RABlocks& blocks() noexcept { return _blocks; }
847 : //! \overload
848 : inline const RABlocks& blocks() const noexcept { return _blocks; }
849 :
850 : //! Returns the count of basic blocks (returns size of `_blocks` array).
851 : inline uint32_t blockCount() const noexcept { return _blocks.size(); }
852 : //! Returns the count of reachable basic blocks (returns size of `_pov` array).
853 : inline uint32_t reachableBlockCount() const noexcept { return _pov.size(); }
854 :
855 : //! Tests whether the CFG has dangling blocks - these were created by `newBlock()`, but not added to CFG through
856 : //! `addBlocks()`. If `true` is returned and the CFG is constructed it means that something is missing and it's
857 : //! incomplete.
858 : //!
859 : //! \note This is only used to check if the number of created blocks matches the number of added blocks.
860 27492 : inline bool hasDanglingBlocks() const noexcept { return _createdBlockCount != blockCount(); }
861 :
862 : //! Gest a next timestamp to be used to mark CFG blocks.
863 0 : inline uint64_t nextTimestamp() const noexcept { return ++_lastTimestamp; }
864 :
865 : //! Createss a new `RABlock` instance.
866 : //!
867 : //! \note New blocks don't have ID assigned until they are added to the block array by calling `addBlock()`.
868 : RABlock* newBlock(BaseNode* initialNode = nullptr) noexcept;
869 :
870 : //! Tries to find a neighboring LabelNode (without going through code) that is already connected with `RABlock`.
871 : //! If no label is found then a new RABlock is created and assigned to all possible labels in a backward direction.
872 : RABlock* newBlockOrExistingAt(LabelNode* cbLabel, BaseNode** stoppedAt = nullptr) noexcept;
873 :
874 : //! Adds the given `block` to the block list and assign it a unique block id.
875 : Error addBlock(RABlock* block) noexcept;
876 :
877 54984 : inline Error addExitBlock(RABlock* block) noexcept {
878 54984 : block->addFlags(RABlockFlags::kIsFuncExit);
879 54984 : return _exits.append(allocator(), block);
880 : }
881 :
882 : ASMJIT_FORCE_INLINE RAInst* newRAInst(RABlock* block, InstRWFlags instRWFlags, RATiedFlags flags, uint32_t tiedRegCount, const RARegMask& clobberedRegs) noexcept {
883 331808 : void* p = zone()->alloc(RAInst::sizeOf(tiedRegCount));
884 331808 : if (ASMJIT_UNLIKELY(!p))
885 : return nullptr;
886 331808 : return new(p) RAInst(block, instRWFlags, flags, tiedRegCount, clobberedRegs);
887 : }
888 :
889 : ASMJIT_FORCE_INLINE Error assignRAInst(BaseNode* node, RABlock* block, RAInstBuilder& ib) noexcept {
890 : uint32_t tiedRegCount = ib.tiedRegCount();
891 : RAInst* raInst = newRAInst(block, ib.instRWFlags(), ib.aggregatedFlags(), tiedRegCount, ib._clobbered);
892 :
893 331808 : if (ASMJIT_UNLIKELY(!raInst))
894 : return DebugUtils::errored(kErrorOutOfMemory);
895 :
896 : RARegIndex index;
897 : RATiedFlags flagsFilter = ~ib.forbiddenFlags();
898 :
899 : index.buildIndexes(ib._count);
900 331808 : raInst->_tiedIndex = index;
901 331808 : raInst->_tiedCount = ib._count;
902 :
903 946262 : for (uint32_t i = 0; i < tiedRegCount; i++) {
904 : RATiedReg* tiedReg = ib[i];
905 : RAWorkReg* workReg = workRegById(tiedReg->workId());
906 :
907 : workReg->resetTiedReg();
908 : RegGroup group = workReg->group();
909 :
910 614454 : if (tiedReg->hasUseId()) {
911 : block->addFlags(RABlockFlags::kHasFixedRegs);
912 37365 : raInst->_usedRegs[group] |= Support::bitMask(tiedReg->useId());
913 : }
914 :
915 614454 : if (tiedReg->hasOutId()) {
916 : block->addFlags(RABlockFlags::kHasFixedRegs);
917 : }
918 :
919 614454 : RATiedReg& dst = raInst->_tiedRegs[index[group]++];
920 614454 : dst = *tiedReg;
921 : dst._flags &= flagsFilter;
922 :
923 614454 : if (!tiedReg->isDuplicate())
924 614454 : dst._useRegMask &= ~ib._used[group];
925 : }
926 :
927 : node->setPassData<RAInst>(raInst);
928 : return kErrorOk;
929 : }
930 :
931 : //! \}
932 :
933 : //! \name CFG - Build CFG
934 : //! \{
935 :
936 : //! Traverse the whole function and do the following:
937 : //!
938 : //! 1. Construct CFG (represented by `RABlock`) by populating `_blocks` and `_exits`. Blocks describe the control
939 : //! flow of the function and contain some additional information that is used by the register allocator.
940 : //!
941 : //! 2. Remove unreachable code immediately. This is not strictly necessary for BaseCompiler itself as the register
942 : //! allocator cannot reach such nodes, but keeping instructions that use virtual registers would fail during
943 : //! instruction encoding phase (Assembler).
944 : //!
945 : //! 3. `RAInst` is created for each `InstNode` or compatible. It contains information that is essential for further
946 : //! analysis and register allocation.
947 : //!
948 : //! Use `RACFGBuilderT` template that provides the necessary boilerplate.
949 : virtual Error buildCFG() noexcept = 0;
950 :
951 : //! Called after the CFG is built.
952 : Error initSharedAssignments(const ZoneVector<uint32_t>& sharedAssignmentsMap) noexcept;
953 :
954 : //! \}
955 :
956 : //! \name CFG - Views Order
957 : //! \{
958 :
959 : //! Constructs CFG views (only POV at the moment).
960 : Error buildCFGViews() noexcept;
961 :
962 : //! \}
963 :
964 : //! \name CFG - Dominators
965 : //! \{
966 :
967 : // Terminology:
968 : // - A node `X` dominates a node `Z` if any path from the entry point to `Z` has to go through `X`.
969 : // - A node `Z` post-dominates a node `X` if any path from `X` to the end of the graph has to go through `Z`.
970 :
971 : //! Constructs a dominator-tree from CFG.
972 : Error buildCFGDominators() noexcept;
973 :
974 : bool _strictlyDominates(const RABlock* a, const RABlock* b) const noexcept;
975 : const RABlock* _nearestCommonDominator(const RABlock* a, const RABlock* b) const noexcept;
976 :
977 : //! Tests whether the basic block `a` dominates `b` - non-strict, returns true when `a == b`.
978 : inline bool dominates(const RABlock* a, const RABlock* b) const noexcept { return a == b ? true : _strictlyDominates(a, b); }
979 : //! Tests whether the basic block `a` dominates `b` - strict dominance check, returns false when `a == b`.
980 : inline bool strictlyDominates(const RABlock* a, const RABlock* b) const noexcept { return a == b ? false : _strictlyDominates(a, b); }
981 :
982 : //! Returns a nearest common dominator of `a` and `b`.
983 : inline RABlock* nearestCommonDominator(RABlock* a, RABlock* b) const noexcept { return const_cast<RABlock*>(_nearestCommonDominator(a, b)); }
984 : //! Returns a nearest common dominator of `a` and `b` (const).
985 : inline const RABlock* nearestCommonDominator(const RABlock* a, const RABlock* b) const noexcept { return _nearestCommonDominator(a, b); }
986 :
987 : //! \}
988 :
989 : //! \name CFG - Utilities
990 : //! \{
991 :
992 : Error removeUnreachableCode() noexcept;
993 :
994 : //! Returns `node` or some node after that is ideal for beginning a new block. This function is mostly used after
995 : //! a conditional or unconditional jump to select the successor node. In some cases the next node could be a label,
996 : //! which means it could have assigned some block already.
997 : BaseNode* findSuccessorStartingAt(BaseNode* node) noexcept;
998 :
999 : //! Returns `true` of the `node` can flow to `target` without reaching code nor data. It's used to eliminate jumps
1000 : //! to labels that are next right to them.
1001 : bool isNextTo(BaseNode* node, BaseNode* target) noexcept;
1002 :
1003 : //! \}
1004 :
1005 : //! \name Virtual Register Management
1006 : //! \{
1007 :
1008 : //! Returns a native size of the general-purpose register of the target architecture.
1009 : inline uint32_t registerSize() const noexcept { return _sp.size(); }
1010 : inline uint32_t availableRegCount(RegGroup group) const noexcept { return _availableRegCount[group]; }
1011 :
1012 2337051 : inline RAWorkReg* workRegById(uint32_t workId) const noexcept { return _workRegs[workId]; }
1013 :
1014 326298 : inline RAWorkRegs& workRegs() noexcept { return _workRegs; }
1015 : inline RAWorkRegs& workRegs(RegGroup group) noexcept { return _workRegsOfGroup[group]; }
1016 :
1017 : inline const RAWorkRegs& workRegs() const noexcept { return _workRegs; }
1018 : inline const RAWorkRegs& workRegs(RegGroup group) const noexcept { return _workRegsOfGroup[group]; }
1019 :
1020 : inline uint32_t workRegCount() const noexcept { return _workRegs.size(); }
1021 : inline uint32_t workRegCount(RegGroup group) const noexcept { return _workRegsOfGroup[group].size(); }
1022 :
1023 27492 : inline void _buildPhysIndex() noexcept {
1024 : _physRegIndex.buildIndexes(_physRegCount);
1025 27492 : _physRegTotal = uint32_t(_physRegIndex[RegGroup::kMaxVirt]) +
1026 27492 : uint32_t(_physRegCount[RegGroup::kMaxVirt]) ;
1027 27492 : }
1028 : inline uint32_t physRegIndex(RegGroup group) const noexcept { return _physRegIndex[group]; }
1029 54984 : inline uint32_t physRegTotal() const noexcept { return _physRegTotal; }
1030 :
1031 : Error _asWorkReg(VirtReg* vReg, RAWorkReg** out) noexcept;
1032 :
1033 : //! Creates `RAWorkReg` data for the given `vReg`. The function does nothing
1034 : //! if `vReg` already contains link to `RAWorkReg`. Called by `constructBlocks()`.
1035 : inline Error asWorkReg(VirtReg* vReg, RAWorkReg** out) noexcept {
1036 712709 : *out = vReg->workReg();
1037 712709 : return *out ? kErrorOk : _asWorkReg(vReg, out);
1038 : }
1039 :
1040 : ASMJIT_FORCE_INLINE Error virtIndexAsWorkReg(uint32_t vIndex, RAWorkReg** out) noexcept {
1041 : const ZoneVector<VirtReg*>& virtRegs = cc()->virtRegs();
1042 712709 : if (ASMJIT_UNLIKELY(vIndex >= virtRegs.size()))
1043 : return DebugUtils::errored(kErrorInvalidVirtId);
1044 712709 : return asWorkReg(virtRegs[vIndex], out);
1045 : }
1046 :
1047 71131 : inline RAStackSlot* getOrCreateStackSlot(RAWorkReg* workReg) noexcept {
1048 : RAStackSlot* slot = workReg->stackSlot();
1049 :
1050 71131 : if (slot)
1051 : return slot;
1052 :
1053 30798 : slot = _stackAllocator.newSlot(_sp.id(), workReg->virtReg()->virtSize(), workReg->virtReg()->alignment(), RAStackSlot::kFlagRegHome);
1054 30798 : workReg->_stackSlot = slot;
1055 : workReg->markStackUsed();
1056 30798 : return slot;
1057 : }
1058 :
1059 71131 : inline BaseMem workRegAsMem(RAWorkReg* workReg) noexcept {
1060 71131 : getOrCreateStackSlot(workReg);
1061 : return BaseMem(OperandSignature::fromOpType(OperandType::kMem) |
1062 : OperandSignature::fromMemBaseType(_sp.type()) |
1063 : OperandSignature::fromBits(OperandSignature::kMemRegHomeFlag),
1064 71131 : workReg->virtId(), 0, 0);
1065 : }
1066 :
1067 : WorkToPhysMap* newWorkToPhysMap() noexcept;
1068 : PhysToWorkMap* newPhysToWorkMap() noexcept;
1069 :
1070 54984 : inline PhysToWorkMap* clonePhysToWorkMap(const PhysToWorkMap* map) noexcept {
1071 54984 : size_t size = PhysToWorkMap::sizeOf(_physRegTotal);
1072 54984 : return static_cast<PhysToWorkMap*>(zone()->dupAligned(map, size, sizeof(uint32_t)));
1073 : }
1074 :
1075 : //! \name Liveness Analysis & Statistics
1076 : //! \{
1077 :
1078 : //! 1. Calculates GEN/KILL/IN/OUT of each block.
1079 : //! 2. Calculates live spans and basic statistics of each work register.
1080 : Error buildLiveness() noexcept;
1081 :
1082 : //! Assigns argIndex to WorkRegs. Must be called after the liveness analysis
1083 : //! finishes as it checks whether the argument is live upon entry.
1084 : Error assignArgIndexToWorkRegs() noexcept;
1085 :
1086 : //! \}
1087 :
1088 : //! \name Register Allocation - Global
1089 : //! \{
1090 :
1091 : //! Runs a global register allocator.
1092 : Error runGlobalAllocator() noexcept;
1093 :
1094 : //! Initializes data structures used for global live spans.
1095 : Error initGlobalLiveSpans() noexcept;
1096 :
1097 : Error binPack(RegGroup group) noexcept;
1098 :
1099 : //! \}
1100 :
1101 : //! \name Register Allocation - Local
1102 : //! \{
1103 :
1104 : //! Runs a local register allocator.
1105 : Error runLocalAllocator() noexcept;
1106 : Error setBlockEntryAssignment(RABlock* block, const RABlock* fromBlock, const RAAssignment& fromAssignment) noexcept;
1107 : Error setSharedAssignment(uint32_t sharedAssignmentId, const RAAssignment& fromAssignment) noexcept;
1108 :
1109 : //! Called after the RA assignment has been assigned to a block.
1110 : //!
1111 : //! This cannot change the assignment, but can examine it.
1112 : Error blockEntryAssigned(const PhysToWorkMap* physToWorkMap) noexcept;
1113 :
1114 : //! \}
1115 :
1116 : //! \name Register Allocation Utilities
1117 : //! \{
1118 :
1119 : Error useTemporaryMem(BaseMem& out, uint32_t size, uint32_t alignment) noexcept;
1120 :
1121 : //! \}
1122 :
1123 : //! \name Function Prolog & Epilog
1124 : //! \{
1125 :
1126 : virtual Error updateStackFrame() noexcept;
1127 : Error _markStackArgsToKeep() noexcept;
1128 : Error _updateStackArgs() noexcept;
1129 : Error insertPrologEpilog() noexcept;
1130 :
1131 : //! \}
1132 :
1133 : //! \name Instruction Rewriter
1134 : //! \{
1135 :
1136 : Error rewrite() noexcept;
1137 : virtual Error _rewrite(BaseNode* first, BaseNode* stop) noexcept = 0;
1138 :
1139 : //! \}
1140 :
1141 : #ifndef ASMJIT_NO_LOGGING
1142 : //! \name Logging
1143 : //! \{
1144 :
1145 : Error annotateCode() noexcept;
1146 :
1147 : Error _dumpBlockIds(String& sb, const RABlocks& blocks) noexcept;
1148 : Error _dumpBlockLiveness(String& sb, const RABlock* block) noexcept;
1149 : Error _dumpLiveSpans(String& sb) noexcept;
1150 :
1151 : //! \}
1152 : #endif
1153 :
1154 : //! \name Emit
1155 : //! \{
1156 :
1157 : virtual Error emitMove(uint32_t workId, uint32_t dstPhysId, uint32_t srcPhysId) noexcept = 0;
1158 : virtual Error emitSwap(uint32_t aWorkId, uint32_t aPhysId, uint32_t bWorkId, uint32_t bPhysId) noexcept = 0;
1159 :
1160 : virtual Error emitLoad(uint32_t workId, uint32_t dstPhysId) noexcept = 0;
1161 : virtual Error emitSave(uint32_t workId, uint32_t srcPhysId) noexcept = 0;
1162 :
1163 : virtual Error emitJump(const Label& label) noexcept = 0;
1164 : virtual Error emitPreCall(InvokeNode* invokeNode) noexcept = 0;
1165 :
1166 : //! \}
1167 : };
1168 :
1169 274920 : inline ZoneAllocator* RABlock::allocator() const noexcept { return _ra->allocator(); }
1170 :
1171 : inline RegMask RABlock::entryScratchGpRegs() const noexcept {
1172 27492 : RegMask regs = _entryScratchGpRegs;
1173 27492 : if (hasSharedAssignmentId())
1174 0 : regs = _ra->_sharedAssignments[_sharedAssignmentId].entryScratchGpRegs();
1175 : return regs;
1176 : }
1177 :
1178 : //! \}
1179 : //! \endcond
1180 :
1181 : ASMJIT_END_NAMESPACE
1182 :
1183 : #endif // !ASMJIT_NO_COMPILER
1184 : #endif // ASMJIT_CORE_RAPASS_P_H_INCLUDED
1185 :
1186 : #endif // __PLUMED_HAS_ASMJIT
|
