Line data Source code
1 : #ifdef __PLUMED_HAS_ASMJIT
2 : // This file is part of AsmJit project <https://asmjit.com>
3 : //
4 : // See asmjit.h or LICENSE.md for license and copyright information
5 : // SPDX-License-Identifier: Zlib
6 :
7 : #ifndef ASMJIT_CORE_RADEFS_P_H_INCLUDED
8 : #define ASMJIT_CORE_RADEFS_P_H_INCLUDED
9 :
10 : #include "api-config.h"
11 : #include "archtraits.h"
12 : #include "compilerdefs.h"
13 : #include "logger.h"
14 : #include "operand.h"
15 : #include "support.h"
16 : #include "type.h"
17 : #include "zone.h"
18 : #include "zonevector.h"
19 :
20 : ASMJIT_BEGIN_NAMESPACE
21 :
22 : //! \cond INTERNAL
23 : //! \addtogroup asmjit_ra
24 : //! \{
25 :
26 : #ifndef ASMJIT_NO_LOGGING
27 : # define ASMJIT_RA_LOG_FORMAT(...) \
28 : do { \
29 : if (logger) \
30 : logger->logf(__VA_ARGS__); \
31 : } while (0)
32 : # define ASMJIT_RA_LOG_COMPLEX(...) \
33 : do { \
34 : if (logger) { \
35 : __VA_ARGS__ \
36 : } \
37 : } while (0)
38 : #else
39 : # define ASMJIT_RA_LOG_FORMAT(...) ((void)0)
40 : # define ASMJIT_RA_LOG_COMPLEX(...) ((void)0)
41 : #endif
42 :
43 : class BaseRAPass;
44 : class RABlock;
45 : class BaseNode;
46 : struct RAStackSlot;
47 :
48 : typedef ZoneVector<RABlock*> RABlocks;
49 : typedef ZoneVector<RAWorkReg*> RAWorkRegs;
50 :
51 : //! Maximum number of consecutive registers aggregated from all supported backends.
52 : static constexpr uint32_t kMaxConsecutiveRegs = 4;
53 :
54 : //! Provides architecture constraints used by register allocator.
55 : class RAConstraints {
56 : public:
57 : //! \name Members
58 : //! \{
59 :
60 : Support::Array<RegMask, Globals::kNumVirtGroups> _availableRegs {};
61 :
62 : //! \}
63 :
64 54984 : ASMJIT_NOINLINE Error init(Arch arch) noexcept {
65 54984 : switch (arch) {
66 54984 : case Arch::kX86:
67 : case Arch::kX64: {
68 54984 : uint32_t registerCount = arch == Arch::kX86 ? 8 : 16;
69 54984 : _availableRegs[RegGroup::kGp] = Support::lsbMask<RegMask>(registerCount) & ~Support::bitMask(4u);
70 54984 : _availableRegs[RegGroup::kVec] = Support::lsbMask<RegMask>(registerCount);
71 54984 : _availableRegs[RegGroup::kExtraVirt2] = Support::lsbMask<RegMask>(8);
72 54984 : _availableRegs[RegGroup::kExtraVirt3] = Support::lsbMask<RegMask>(8);
73 : return kErrorOk;
74 : }
75 :
76 0 : case Arch::kAArch64: {
77 0 : _availableRegs[RegGroup::kGp] = 0xFFFFFFFFu & ~Support::bitMask(18, 31u);
78 0 : _availableRegs[RegGroup::kVec] = 0xFFFFFFFFu;
79 0 : _availableRegs[RegGroup::kExtraVirt2] = 0;
80 0 : _availableRegs[RegGroup::kExtraVirt3] = 0;
81 0 : return kErrorOk;
82 : }
83 :
84 : default:
85 : return DebugUtils::errored(kErrorInvalidArch);
86 : }
87 : }
88 :
89 219936 : inline RegMask availableRegs(RegGroup group) const noexcept { return _availableRegs[group]; }
90 : };
91 :
92 : enum class RAStrategyType : uint8_t {
93 : kSimple = 0,
94 : kComplex = 1
95 : };
96 : ASMJIT_DEFINE_ENUM_COMPARE(RAStrategyType)
97 :
98 : enum class RAStrategyFlags : uint8_t {
99 : kNone = 0
100 : };
101 : ASMJIT_DEFINE_ENUM_FLAGS(RAStrategyFlags)
102 :
103 : //! Register allocation strategy.
104 : //!
105 : //! The idea is to select the best register allocation strategy for each virtual register group based on the
106 : //! complexity of the code.
107 : struct RAStrategy {
108 : //! \name Members
109 : //! \{
110 :
111 : RAStrategyType _type = RAStrategyType::kSimple;
112 : RAStrategyFlags _flags = RAStrategyFlags::kNone;
113 :
114 : //! \}
115 :
116 : //! \name Accessors
117 : //! \{
118 :
119 : inline void reset() noexcept {
120 219936 : _type = RAStrategyType::kSimple;
121 219936 : _flags = RAStrategyFlags::kNone;
122 : }
123 :
124 : inline RAStrategyType type() const noexcept { return _type; }
125 1955 : inline void setType(RAStrategyType type) noexcept { _type = type; }
126 :
127 : inline bool isSimple() const noexcept { return _type == RAStrategyType::kSimple; }
128 109968 : inline bool isComplex() const noexcept { return _type >= RAStrategyType::kComplex; }
129 :
130 : inline RAStrategyFlags flags() const noexcept { return _flags; }
131 : inline bool hasFlag(RAStrategyFlags flag) const noexcept { return Support::test(_flags, flag); }
132 : inline void addFlags(RAStrategyFlags flags) noexcept { _flags |= flags; }
133 :
134 : //! \}
135 : };
136 :
137 : //! Count of virtual or physical registers per group.
138 : //!
139 : //! \note This class uses 8-bit integers to represent counters, it's only used in places where this is sufficient,
140 : //! for example total count of machine's physical registers, count of virtual registers per instruction, etc...
141 : //! There is also `RALiveCount`, which uses 32-bit integers and is indeed much safer.
142 : struct RARegCount {
143 : //! \name Members
144 : //! \{
145 :
146 : union {
147 : uint8_t _regs[4];
148 : uint32_t _packed;
149 : };
150 :
151 : //! \}
152 :
153 : //! \name Construction & Destruction
154 : //! \{
155 :
156 : //! Resets all counters to zero.
157 496760 : inline void reset() noexcept { _packed = 0; }
158 :
159 : //! \}
160 :
161 : //! \name Overloaded Operators
162 : //! \{
163 :
164 : inline uint8_t& operator[](RegGroup group) noexcept {
165 : ASMJIT_ASSERT(group <= RegGroup::kMaxVirt);
166 929885 : return _regs[size_t(group)];
167 : }
168 :
169 : inline const uint8_t& operator[](RegGroup group) const noexcept {
170 : ASMJIT_ASSERT(group <= RegGroup::kMaxVirt);
171 : return _regs[size_t(group)];
172 : }
173 :
174 : inline bool operator==(const RARegCount& other) const noexcept { return _packed == other._packed; }
175 : inline bool operator!=(const RARegCount& other) const noexcept { return _packed != other._packed; }
176 :
177 : //! \}
178 :
179 : //! \name Accessors
180 : //! \{
181 :
182 : //! Returns the count of registers by the given register `group`.
183 : inline uint32_t get(RegGroup group) const noexcept {
184 : ASMJIT_ASSERT(group <= RegGroup::kMaxVirt);
185 :
186 1547168 : uint32_t shift = Support::byteShiftOfDWordStruct(uint32_t(group));
187 1574660 : return (_packed >> shift) & uint32_t(0xFF);
188 : }
189 :
190 : //! Sets the register count by a register `group`.
191 : inline void set(RegGroup group, uint32_t n) noexcept {
192 : ASMJIT_ASSERT(group <= RegGroup::kMaxVirt);
193 : ASMJIT_ASSERT(n <= 0xFF);
194 :
195 : uint32_t shift = Support::byteShiftOfDWordStruct(uint32_t(group));
196 27492 : _packed = (_packed & ~uint32_t(0xFF << shift)) + (n << shift);
197 : }
198 :
199 : //! Adds the register count by a register `group`.
200 : inline void add(RegGroup group, uint32_t n = 1) noexcept {
201 : ASMJIT_ASSERT(group <= RegGroup::kMaxVirt);
202 : ASMJIT_ASSERT(0xFF - uint32_t(_regs[size_t(group)]) >= n);
203 :
204 : uint32_t shift = Support::byteShiftOfDWordStruct(uint32_t(group));
205 614454 : _packed += n << shift;
206 : }
207 :
208 : //! \}
209 : };
210 :
211 : //! Provides mapping that can be used to fast index architecture register groups.
212 : struct RARegIndex : public RARegCount {
213 : //! Build register indexes based on the given `count` of registers.
214 : ASMJIT_FORCE_INLINE void buildIndexes(const RARegCount& count) noexcept {
215 414284 : uint32_t x = uint32_t(count._regs[0]);
216 414284 : uint32_t y = uint32_t(count._regs[1]) + x;
217 414284 : uint32_t z = uint32_t(count._regs[2]) + y;
218 :
219 : ASMJIT_ASSERT(y <= 0xFF);
220 : ASMJIT_ASSERT(z <= 0xFF);
221 386792 : _packed = Support::bytepack32_4x8(0, x, y, z);
222 : }
223 : };
224 :
225 : //! Registers mask.
226 : struct RARegMask {
227 : //! \name Members
228 : //! \{
229 :
230 : Support::Array<RegMask, Globals::kNumVirtGroups> _masks;
231 :
232 : //! \}
233 :
234 : //! \name Construction & Destruction
235 : //! \{
236 :
237 : inline void init(const RARegMask& other) noexcept { _masks = other._masks; }
238 : //! Reset all register masks to zero.
239 : inline void reset() noexcept { _masks.fill(0); }
240 :
241 : //! \}
242 :
243 : //! \name Overloaded Operators
244 : //! \{
245 :
246 : inline bool operator==(const RARegMask& other) const noexcept { return _masks == other._masks; }
247 : inline bool operator!=(const RARegMask& other) const noexcept { return _masks != other._masks; }
248 :
249 : template<typename Index>
250 : inline uint32_t& operator[](const Index& index) noexcept { return _masks[index]; }
251 :
252 : template<typename Index>
253 : inline const uint32_t& operator[](const Index& index) const noexcept { return _masks[index]; }
254 :
255 : //! \}
256 :
257 : //! \name Utilities
258 : //! \{
259 :
260 : //! Tests whether all register masks are zero (empty).
261 : inline bool empty() const noexcept {
262 : return _masks.aggregate<Support::Or>() == 0;
263 : }
264 :
265 : inline bool has(RegGroup group, RegMask mask = 0xFFFFFFFFu) const noexcept {
266 : return (_masks[group] & mask) != 0;
267 : }
268 :
269 : template<class Operator>
270 : inline void op(const RARegMask& other) noexcept {
271 : _masks.combine<Operator>(other._masks);
272 : }
273 :
274 : template<class Operator>
275 : inline void op(RegGroup group, RegMask mask) noexcept {
276 : _masks[group] = Operator::op(_masks[group], mask);
277 : }
278 :
279 : inline void clear(RegGroup group, RegMask mask) noexcept {
280 0 : _masks[group] = _masks[group] & ~mask;
281 : }
282 :
283 : //! \}
284 : };
285 :
286 : //! Information associated with each instruction, propagated to blocks, loops, and the whole function. This
287 : //! information can be used to do minor decisions before the register allocator tries to do its job. For
288 : //! example to use fast register allocation inside a block or loop it cannot have clobbered and/or fixed
289 : //! registers, etc...
290 27492 : class RARegsStats {
291 : public:
292 : //! \name Constants
293 : //! \{
294 :
295 : enum Index : uint32_t {
296 : kIndexUsed = 0,
297 : kIndexFixed = 8,
298 : kIndexClobbered = 16
299 : };
300 :
301 : enum Mask : uint32_t {
302 : kMaskUsed = 0xFFu << kIndexUsed,
303 : kMaskFixed = 0xFFu << kIndexFixed,
304 : kMaskClobbered = 0xFFu << kIndexClobbered
305 : };
306 :
307 : //! \}
308 :
309 : //! \name Members
310 : //! \{
311 :
312 : uint32_t _packed = 0;
313 :
314 : //! \}
315 :
316 : //! \name Accessors
317 : //! \{
318 :
319 414284 : inline void reset() noexcept { _packed = 0; }
320 386792 : inline void combineWith(const RARegsStats& other) noexcept { _packed |= other._packed; }
321 :
322 : inline bool hasUsed() const noexcept { return (_packed & kMaskUsed) != 0u; }
323 : inline bool hasUsed(RegGroup group) const noexcept { return (_packed & Support::bitMask(kIndexUsed + uint32_t(group))) != 0u; }
324 673873 : inline void makeUsed(RegGroup group) noexcept { _packed |= Support::bitMask(kIndexUsed + uint32_t(group)); }
325 :
326 : inline bool hasFixed() const noexcept { return (_packed & kMaskFixed) != 0u; }
327 : inline bool hasFixed(RegGroup group) const noexcept { return (_packed & Support::bitMask(kIndexFixed + uint32_t(group))) != 0u; }
328 27492 : inline void makeFixed(RegGroup group) noexcept { _packed |= Support::bitMask(kIndexFixed + uint32_t(group)); }
329 :
330 : inline bool hasClobbered() const noexcept { return (_packed & kMaskClobbered) != 0u; }
331 : inline bool hasClobbered(RegGroup group) const noexcept { return (_packed & Support::bitMask(kIndexClobbered + uint32_t(group))) != 0u; }
332 : inline void makeClobbered(RegGroup group) noexcept { _packed |= Support::bitMask(kIndexClobbered + uint32_t(group)); }
333 :
334 : //! \}
335 : };
336 :
337 : //! Count of live registers, per group.
338 : class RALiveCount {
339 : public:
340 : //! \name Members
341 : //! \{
342 :
343 : Support::Array<uint32_t, Globals::kNumVirtGroups> n {};
344 :
345 : //! \}
346 :
347 : //! \name Construction & Destruction
348 : //! \{
349 :
350 414284 : inline RALiveCount() noexcept = default;
351 : inline RALiveCount(const RALiveCount& other) noexcept = default;
352 :
353 : inline void init(const RALiveCount& other) noexcept { n = other.n; }
354 : inline void reset() noexcept { n.fill(0); }
355 :
356 : //! \}
357 :
358 : //! \name Overloaded Operators
359 : //! \{
360 :
361 : inline RALiveCount& operator=(const RALiveCount& other) noexcept = default;
362 :
363 : inline uint32_t& operator[](RegGroup group) noexcept { return n[group]; }
364 : inline const uint32_t& operator[](RegGroup group) const noexcept { return n[group]; }
365 :
366 : //! \}
367 :
368 : //! \name Utilities
369 : //! \{
370 :
371 : template<class Operator>
372 : inline void op(const RALiveCount& other) noexcept { n.combine<Operator>(other.n); }
373 :
374 : //! \}
375 : };
376 :
377 : struct RALiveInterval {
378 : //! \name Constants
379 : //! \{
380 :
381 : enum : uint32_t {
382 : kNaN = 0,
383 : kInf = 0xFFFFFFFFu
384 : };
385 :
386 : //! \}
387 :
388 : //! \name Members
389 : //! \{
390 :
391 : uint32_t a, b;
392 :
393 : //! \}
394 :
395 : //! \name Construction & Destruction
396 : //! \{
397 :
398 : inline RALiveInterval() noexcept : a(0), b(0) {}
399 283174 : inline RALiveInterval(uint32_t a, uint32_t b) noexcept : a(a), b(b) {}
400 : inline RALiveInterval(const RALiveInterval& other) noexcept : a(other.a), b(other.b) {}
401 :
402 : inline void init(uint32_t aVal, uint32_t bVal) noexcept {
403 2556954 : a = aVal;
404 2556954 : b = bVal;
405 : }
406 2556954 : inline void init(const RALiveInterval& other) noexcept { init(other.a, other.b); }
407 : inline void reset() noexcept { init(0, 0); }
408 :
409 : //! \}
410 :
411 : //! \name Overloaded Operators
412 : //! \{
413 :
414 : inline RALiveInterval& operator=(const RALiveInterval& other) = default;
415 :
416 : //! \}
417 :
418 : //! \name Accessors
419 : //! \{
420 :
421 : inline bool isValid() const noexcept { return a < b; }
422 283174 : inline uint32_t width() const noexcept { return b - a; }
423 :
424 : //! \}
425 : };
426 :
427 : //! Live span with payload of type `T`.
428 : template<typename T>
429 : class RALiveSpan : public RALiveInterval, public T {
430 : public:
431 : //! \name Types
432 : //! \{
433 :
434 : typedef T DataType;
435 :
436 : //! \}
437 :
438 : //! \name Construction & Destruction
439 : //! \{
440 :
441 : inline RALiveSpan() noexcept : RALiveInterval(), T() {}
442 : inline RALiveSpan(const RALiveSpan<T>& other) noexcept : RALiveInterval(other), T() {}
443 : inline RALiveSpan(const RALiveInterval& interval, const T& data) noexcept : RALiveInterval(interval), T(data) {}
444 : inline RALiveSpan(uint32_t a, uint32_t b) noexcept : RALiveInterval(a, b), T() {}
445 : inline RALiveSpan(uint32_t a, uint32_t b, const T& data) noexcept : RALiveInterval(a, b), T(data) {}
446 :
447 : inline void init(const RALiveSpan<T>& other) noexcept {
448 : RALiveInterval::init(static_cast<const RALiveInterval&>(other));
449 : T::init(static_cast<const T&>(other));
450 : }
451 :
452 : inline void init(const RALiveSpan<T>& span, const T& data) noexcept {
453 : RALiveInterval::init(static_cast<const RALiveInterval&>(span));
454 : T::init(data);
455 : }
456 :
457 : inline void init(const RALiveInterval& interval, const T& data) noexcept {
458 : RALiveInterval::init(interval);
459 : T::init(data);
460 : }
461 :
462 : //! \}
463 :
464 : //! \name Overloaded Operators
465 : //! \{
466 :
467 : inline RALiveSpan& operator=(const RALiveSpan& other) {
468 : init(other);
469 846428 : return *this;
470 : }
471 :
472 : //! \}
473 : };
474 :
475 : //! Vector of `RALiveSpan<T>` with additional convenience API.
476 : template<typename T>
477 : class RALiveSpans {
478 : public:
479 : ASMJIT_NONCOPYABLE(RALiveSpans)
480 :
481 : typedef typename T::DataType DataType;
482 : ZoneVector<T> _data;
483 :
484 : //! \name Construction & Destruction
485 : //! \{
486 :
487 : inline RALiveSpans() noexcept : _data() {}
488 :
489 : inline void reset() noexcept { _data.reset(); }
490 : inline void release(ZoneAllocator* allocator) noexcept { _data.release(allocator); }
491 :
492 : //! \}
493 :
494 : //! \name Accessors
495 : //! \{
496 :
497 : inline bool empty() const noexcept { return _data.empty(); }
498 : inline uint32_t size() const noexcept { return _data.size(); }
499 :
500 : inline T* data() noexcept { return _data.data(); }
501 : inline const T* data() const noexcept { return _data.data(); }
502 :
503 : inline bool isOpen() const noexcept {
504 : uint32_t size = _data.size();
505 : return size > 0 && _data[size - 1].b == RALiveInterval::kInf;
506 : }
507 :
508 : //! \}
509 :
510 : //! \name Utilities
511 : //! \{
512 :
513 : inline void swap(RALiveSpans<T>& other) noexcept { _data.swap(other._data); }
514 :
515 : //! Open the current live span.
516 : ASMJIT_FORCE_INLINE Error openAt(ZoneAllocator* allocator, uint32_t start, uint32_t end) noexcept {
517 : bool wasOpen;
518 : return openAt(allocator, start, end, wasOpen);
519 : }
520 :
521 : ASMJIT_FORCE_INLINE Error openAt(ZoneAllocator* allocator, uint32_t start, uint32_t end, bool& wasOpen) noexcept {
522 : uint32_t size = _data.size();
523 : wasOpen = false;
524 :
525 614454 : if (size > 0) {
526 343140 : T& last = _data[size - 1];
527 343140 : if (last.b >= start) {
528 331280 : wasOpen = last.b > start;
529 331280 : last.b = end;
530 331280 : return kErrorOk;
531 : }
532 : }
533 :
534 283174 : return _data.append(allocator, T(start, end));
535 : }
536 :
537 : ASMJIT_FORCE_INLINE void closeAt(uint32_t end) noexcept {
538 : ASMJIT_ASSERT(!empty());
539 :
540 : uint32_t size = _data.size();
541 283174 : _data[size - 1].b = end;
542 : }
543 :
544 : //! Returns the sum of width of all spans.
545 : //!
546 : //! \note Don't overuse, this iterates over all spans so it's O(N). It should be only called once and then cached.
547 : inline uint32_t width() const noexcept {
548 : uint32_t width = 0;
549 554488 : for (const T& span : _data)
550 283174 : width += span.width();
551 : return width;
552 : }
553 :
554 0 : inline T& operator[](uint32_t index) noexcept { return _data[index]; }
555 0 : inline const T& operator[](uint32_t index) const noexcept { return _data[index]; }
556 :
557 : inline bool intersects(const RALiveSpans<T>& other) const noexcept {
558 : return intersects(*this, other);
559 : }
560 :
561 : ASMJIT_FORCE_INLINE Error nonOverlappingUnionOf(ZoneAllocator* allocator, const RALiveSpans<T>& x, const RALiveSpans<T>& y, const DataType& yData) noexcept {
562 572293 : uint32_t finalSize = x.size() + y.size();
563 572293 : ASMJIT_PROPAGATE(_data.reserve(allocator, finalSize));
564 :
565 : T* dstPtr = _data.data();
566 : const T* xSpan = x.data();
567 : const T* ySpan = y.data();
568 :
569 572293 : const T* xEnd = xSpan + x.size();
570 572293 : const T* yEnd = ySpan + y.size();
571 :
572 : // Loop until we have intersection or either `xSpan == xEnd` or `ySpan == yEnd`, which means that there is no
573 : // intersection. We advance either `xSpan` or `ySpan` depending on their ranges.
574 572293 : if (xSpan != xEnd && ySpan != yEnd) {
575 : uint32_t xa, ya;
576 472409 : xa = xSpan->a;
577 : for (;;) {
578 531798 : while (ySpan->b <= xa) {
579 : dstPtr->init(*ySpan, yData);
580 114557 : dstPtr++;
581 114557 : if (++ySpan == yEnd)
582 113981 : goto Done;
583 : }
584 :
585 417241 : ya = ySpan->a;
586 1808034 : while (xSpan->b <= ya) {
587 1437797 : *dstPtr++ = *xSpan;
588 1437797 : if (++xSpan == xEnd)
589 47004 : goto Done;
590 : }
591 :
592 : // We know that `xSpan->b > ySpan->a`, so check if `ySpan->b > xSpan->a`.
593 370237 : xa = xSpan->a;
594 370237 : if (ySpan->b > xa)
595 : return 0xFFFFFFFFu;
596 : }
597 : }
598 :
599 260869 : Done:
600 1107297 : while (xSpan != xEnd) {
601 846428 : *dstPtr++ = *xSpan++;
602 : }
603 :
604 419041 : while (ySpan != yEnd) {
605 : dstPtr->init(*ySpan, yData);
606 158172 : dstPtr++;
607 158172 : ySpan++;
608 : }
609 :
610 : _data._setEndPtr(dstPtr);
611 : return kErrorOk;
612 : }
613 :
614 : static ASMJIT_FORCE_INLINE bool intersects(const RALiveSpans<T>& x, const RALiveSpans<T>& y) noexcept {
615 : const T* xSpan = x.data();
616 : const T* ySpan = y.data();
617 :
618 : const T* xEnd = xSpan + x.size();
619 : const T* yEnd = ySpan + y.size();
620 :
621 : // Loop until we have intersection or either `xSpan == xEnd` or `ySpan == yEnd`, which means that there is no
622 : // intersection. We advance either `xSpan` or `ySpan` depending on their end positions.
623 : if (xSpan == xEnd || ySpan == yEnd)
624 : return false;
625 :
626 : uint32_t xa, ya;
627 : xa = xSpan->a;
628 :
629 : for (;;) {
630 : while (ySpan->b <= xa)
631 : if (++ySpan == yEnd)
632 : return false;
633 :
634 : ya = ySpan->a;
635 : while (xSpan->b <= ya)
636 : if (++xSpan == xEnd)
637 : return false;
638 :
639 : // We know that `xSpan->b > ySpan->a`, so check if `ySpan->b > xSpan->a`.
640 : xa = xSpan->a;
641 : if (ySpan->b > xa)
642 : return true;
643 : }
644 : }
645 :
646 : //! \}
647 : };
648 :
649 : //! Statistics about a register liveness.
650 : class RALiveStats {
651 : public:
652 : uint32_t _width = 0;
653 : float _freq = 0.0f;
654 : float _priority = 0.0f;
655 :
656 : //! \name Accessors
657 : //! \{
658 :
659 0 : inline uint32_t width() const noexcept { return _width; }
660 117536 : inline float freq() const noexcept { return _freq; }
661 669150 : inline float priority() const noexcept { return _priority; }
662 :
663 : //! \}
664 : };
665 :
666 : struct LiveRegData {
667 : uint32_t id;
668 :
669 283174 : inline explicit LiveRegData(uint32_t id = BaseReg::kIdBad) noexcept : id(id) {}
670 : inline LiveRegData(const LiveRegData& other) noexcept : id(other.id) {}
671 :
672 2556954 : inline void init(const LiveRegData& other) noexcept { id = other.id; }
673 :
674 : inline bool operator==(const LiveRegData& other) const noexcept { return id == other.id; }
675 : inline bool operator!=(const LiveRegData& other) const noexcept { return id != other.id; }
676 : };
677 :
678 : typedef RALiveSpan<LiveRegData> LiveRegSpan;
679 : typedef RALiveSpans<LiveRegSpan> LiveRegSpans;
680 :
681 : //! Flags used by \ref RATiedReg.
682 : //!
683 : //! Register access information is encoded in 4 flags in total:
684 : //!
685 : //! - `kRead` - Register is Read (ReadWrite if combined with `kWrite`).
686 : //! - `kWrite` - Register is Written (ReadWrite if combined with `kRead`).
687 : //! - `kUse` - Encoded as Read or ReadWrite.
688 : //! - `kOut` - Encoded as WriteOnly.
689 : //!
690 : //! Let's describe all of these on two X86 instructions:
691 : //!
692 : //! - ADD x{R|W|Use}, x{R|Use} -> {x:R|W|Use }
693 : //! - LEA x{ W|Out}, [x{R|Use} + x{R|Out}] -> {x:R|W|Use|Out }
694 : //! - ADD x{R|W|Use}, y{R|Use} -> {x:R|W|Use y:R|Use}
695 : //! - LEA x{ W|Out}, [x{R|Use} + y{R|Out}] -> {x:R|W|Use|Out y:R|Use}
696 : //!
697 : //! It should be obvious from the example above how these flags get created. Each operand contains READ/WRITE
698 : //! information, which is then merged to RATiedReg's flags. However, we also need to represent the possitility
699 : //! to view the operation as two independent operations - USE and OUT, because the register allocator first
700 : //! allocates USE registers, and then assigns OUT registers independently of USE registers.
701 : enum class RATiedFlags : uint32_t {
702 : //! No flags.
703 : kNone = 0,
704 :
705 : // Access Flags
706 : // ------------
707 :
708 : //! Register is read.
709 : kRead = uint32_t(OpRWFlags::kRead),
710 : //! Register is written.
711 : kWrite = uint32_t(OpRWFlags::kWrite),
712 : //! Register both read and written.
713 : kRW = uint32_t(OpRWFlags::kRW),
714 :
715 : // Use / Out Flags
716 : // ---------------
717 :
718 : //! Register has a USE slot (read/rw).
719 : kUse = 0x00000004u,
720 : //! Register has an OUT slot (write-only).
721 : kOut = 0x00000008u,
722 : //! Register in USE slot can be patched to memory.
723 : kUseRM = 0x00000010u,
724 : //! Register in OUT slot can be patched to memory.
725 : kOutRM = 0x00000020u,
726 :
727 : //! Register has a fixed USE slot.
728 : kUseFixed = 0x00000040u,
729 : //! Register has a fixed OUT slot.
730 : kOutFixed = 0x00000080u,
731 : //! Register USE slot has been allocated.
732 : kUseDone = 0x00000100u,
733 : //! Register OUT slot has been allocated.
734 : kOutDone = 0x00000200u,
735 :
736 : // Consecutive Flags / Data
737 : // ------------------------
738 :
739 : kUseConsecutive = 0x00000400u,
740 : kOutConsecutive = 0x00000800u,
741 : kLeadConsecutive = 0x00001000u,
742 : kConsecutiveData = 0x00006000u,
743 :
744 : // Liveness Flags
745 : // --------------
746 :
747 : //! Register must be duplicated (function call only).
748 : kDuplicate = 0x00010000u,
749 : //! Last occurrence of this VirtReg in basic block.
750 : kLast = 0x00020000u,
751 : //! Kill this VirtReg after use.
752 : kKill = 0x00040000u,
753 :
754 : // X86 Specific Flags
755 : // ------------------
756 :
757 : // Architecture specific flags are used during RATiedReg building to ensure that architecture-specific constraints
758 : // are handled properly. These flags are not really needed after RATiedReg[] is built and copied to `RAInst`.
759 :
760 : //! This RATiedReg references GPB-LO or GPB-HI.
761 : kX86_Gpb = 0x01000000u,
762 :
763 : // Instruction Flags (Never used by RATiedReg)
764 : // -------------------------------------------
765 :
766 : //! Instruction is transformable to another instruction if necessary.
767 : //!
768 : //! This is flag that is only used by \ref RAInst to inform register allocator that the instruction has some
769 : //! constraints that can only be solved by transforming the instruction into another instruction, most likely
770 : //! by changing its InstId.
771 : kInst_IsTransformable = 0x80000000u
772 : };
773 3227944 : ASMJIT_DEFINE_ENUM_FLAGS(RATiedFlags)
774 :
775 : static_assert(uint32_t(RATiedFlags::kRead ) == 0x1, "RATiedFlags::kRead must be 0x1");
776 : static_assert(uint32_t(RATiedFlags::kWrite) == 0x2, "RATiedFlags::kWrite must be 0x2");
777 : static_assert(uint32_t(RATiedFlags::kRW ) == 0x3, "RATiedFlags::kRW must be 0x3");
778 :
779 : //! Tied register merges one ore more register operand into a single entity. It contains information about its access
780 : //! (Read|Write) and allocation slots (Use|Out) that are used by the register allocator and liveness analysis.
781 : struct RATiedReg {
782 : //! \name Members
783 : //! \{
784 :
785 : //! WorkReg id.
786 : uint32_t _workId;
787 : //! WorkReg id that is an immediate consecutive parent of this register, or Globals::kInvalidId if it has no parent.
788 : uint32_t _consecutiveParent;
789 : //! Allocation flags.
790 : RATiedFlags _flags;
791 :
792 : union {
793 : struct {
794 : //! How many times the VirtReg is referenced in all operands.
795 : uint8_t _refCount;
796 : //! Size of a memory operand in case that it's use instead of the register.
797 : uint8_t _rmSize;
798 : //! Physical register for use operation (ReadOnly / ReadWrite).
799 : uint8_t _useId;
800 : //! Physical register for out operation (WriteOnly).
801 : uint8_t _outId;
802 : };
803 : //! Packed data.
804 : uint32_t _packed;
805 : };
806 :
807 : //! Registers where inputs {R|X} can be allocated to.
808 : RegMask _useRegMask;
809 : //! Registers where outputs {W} can be allocated to.
810 : RegMask _outRegMask;
811 : //! Indexes used to rewrite USE regs.
812 : uint32_t _useRewriteMask;
813 : //! Indexes used to rewrite OUT regs.
814 : uint32_t _outRewriteMask;
815 :
816 : //! \}
817 :
818 : //! \name Statics
819 : //! \{
820 :
821 : static inline RATiedFlags consecutiveDataToFlags(uint32_t offset) noexcept {
822 : ASMJIT_ASSERT(offset < 4);
823 : constexpr uint32_t kOffsetShift = Support::ConstCTZ<uint32_t(RATiedFlags::kConsecutiveData)>::value;
824 0 : return (RATiedFlags)(offset << kOffsetShift);
825 : }
826 :
827 : static inline uint32_t consecutiveDataFromFlags(RATiedFlags flags) noexcept {
828 : constexpr uint32_t kOffsetShift = Support::ConstCTZ<uint32_t(RATiedFlags::kConsecutiveData)>::value;
829 0 : return uint32_t(flags & RATiedFlags::kConsecutiveData) >> kOffsetShift;
830 : }
831 :
832 : //! \}
833 :
834 : //! \name Construction & Destruction
835 : //! \{
836 :
837 : inline void init(uint32_t workId, RATiedFlags flags, RegMask useRegMask, uint32_t useId, uint32_t useRewriteMask, RegMask outRegMask, uint32_t outId, uint32_t outRewriteMask, uint32_t rmSize = 0, uint32_t consecutiveParent = Globals::kInvalidId) noexcept {
838 614454 : _workId = workId;
839 614454 : _consecutiveParent = consecutiveParent;
840 614454 : _flags = flags;
841 614454 : _refCount = 1;
842 614454 : _rmSize = uint8_t(rmSize);
843 614454 : _useId = uint8_t(useId);
844 614454 : _outId = uint8_t(outId);
845 614454 : _useRegMask = useRegMask;
846 614454 : _outRegMask = outRegMask;
847 614454 : _useRewriteMask = useRewriteMask;
848 614454 : _outRewriteMask = outRewriteMask;
849 : }
850 :
851 : //! \}
852 :
853 : //! \name Accessors
854 : //! \{
855 :
856 : //! Returns the associated WorkReg id.
857 3424696 : inline uint32_t workId() const noexcept { return _workId; }
858 :
859 614454 : inline bool hasConsecutiveParent() const noexcept { return _consecutiveParent != Globals::kInvalidId; }
860 78931 : inline uint32_t consecutiveParent() const noexcept { return _consecutiveParent; }
861 : inline uint32_t consecutiveData() const noexcept { return consecutiveDataFromFlags(_flags); }
862 :
863 : //! Returns TiedReg flags.
864 : inline RATiedFlags flags() const noexcept { return _flags; }
865 : //! Checks if the given `flag` is set.
866 6316647 : inline bool hasFlag(RATiedFlags flag) const noexcept { return Support::test(_flags, flag); }
867 : //! Adds tied register flags.
868 554488 : inline void addFlags(RATiedFlags flags) noexcept { _flags |= flags; }
869 :
870 : //! Tests whether the register is read (writes `true` also if it's Read/Write).
871 : inline bool isRead() const noexcept { return hasFlag(RATiedFlags::kRead); }
872 : //! Tests whether the register is written (writes `true` also if it's Read/Write).
873 : inline bool isWrite() const noexcept { return hasFlag(RATiedFlags::kWrite); }
874 : //! Tests whether the register is read only.
875 : inline bool isReadOnly() const noexcept { return (_flags & RATiedFlags::kRW) == RATiedFlags::kRead; }
876 : //! Tests whether the register is write only.
877 614454 : inline bool isWriteOnly() const noexcept { return (_flags & RATiedFlags::kRW) == RATiedFlags::kWrite; }
878 : //! Tests whether the register is read and written.
879 0 : inline bool isReadWrite() const noexcept { return (_flags & RATiedFlags::kRW) == RATiedFlags::kRW; }
880 :
881 : //! Tests whether the tied register has use operand (Read/ReadWrite).
882 : inline bool isUse() const noexcept { return hasFlag(RATiedFlags::kUse); }
883 : //! Tests whether the tied register has out operand (Write).
884 : inline bool isOut() const noexcept { return hasFlag(RATiedFlags::kOut); }
885 :
886 : //! Tests whether the tied register has \ref RATiedFlags::kLeadConsecutive flag set.
887 : inline bool isLeadConsecutive() const noexcept { return hasFlag(RATiedFlags::kLeadConsecutive); }
888 : //! Tests whether the tied register has \ref RATiedFlags::kUseConsecutive flag set.
889 : inline bool isUseConsecutive() const noexcept { return hasFlag(RATiedFlags::kUseConsecutive); }
890 : //! Tests whether the tied register has \ref RATiedFlags::kOutConsecutive flag set.
891 : inline bool isOutConsecutive() const noexcept { return hasFlag(RATiedFlags::kOutConsecutive); }
892 :
893 : //! Tests whether the tied register has any consecutive flag.
894 : inline bool hasAnyConsecutiveFlag() const noexcept { return hasFlag(RATiedFlags::kLeadConsecutive | RATiedFlags::kUseConsecutive | RATiedFlags::kOutConsecutive); }
895 :
896 : //! Tests whether the USE slot can be patched to memory operand.
897 : inline bool hasUseRM() const noexcept { return hasFlag(RATiedFlags::kUseRM); }
898 : //! Tests whether the OUT slot can be patched to memory operand.
899 : inline bool hasOutRM() const noexcept { return hasFlag(RATiedFlags::kOutRM); }
900 :
901 102037 : inline uint32_t rmSize() const noexcept { return _rmSize; }
902 :
903 : inline void makeReadOnly() noexcept {
904 0 : _flags = (_flags & ~(RATiedFlags::kOut | RATiedFlags::kWrite)) | RATiedFlags::kUse;
905 0 : _useRewriteMask |= _outRewriteMask;
906 0 : _outRewriteMask = 0;
907 0 : }
908 :
909 : inline void makeWriteOnly() noexcept {
910 4680 : _flags = (_flags & ~(RATiedFlags::kUse | RATiedFlags::kRead)) | RATiedFlags::kOut;
911 4680 : _outRewriteMask |= _useRewriteMask;
912 4680 : _useRewriteMask = 0;
913 4680 : }
914 :
915 : //! Tests whether the register would duplicate.
916 : inline bool isDuplicate() const noexcept { return hasFlag(RATiedFlags::kDuplicate); }
917 :
918 : //! Tests whether the register (and the instruction it's part of) appears last in the basic block.
919 : inline bool isLast() const noexcept { return hasFlag(RATiedFlags::kLast); }
920 : //! Tests whether the register should be killed after USEd and/or OUTed.
921 : inline bool isKill() const noexcept { return hasFlag(RATiedFlags::kKill); }
922 :
923 : //! Tests whether the register is OUT or KILL (used internally by local register allocator).
924 : inline bool isOutOrKill() const noexcept { return hasFlag(RATiedFlags::kOut | RATiedFlags::kKill); }
925 :
926 : //! Returns a register mask that describes allocable USE registers (Read/ReadWrite access).
927 925065 : inline RegMask useRegMask() const noexcept { return _useRegMask; }
928 : //! Returns a register mask that describes allocable OUT registers (WriteOnly access).
929 909037 : inline RegMask outRegMask() const noexcept { return _outRegMask; }
930 :
931 : inline uint32_t refCount() const noexcept { return _refCount; }
932 78931 : inline void addRefCount(uint32_t n = 1) noexcept { _refCount = uint8_t(_refCount + n); }
933 :
934 : //! Tests whether the register must be allocated to a fixed physical register before it's used.
935 1579132 : inline bool hasUseId() const noexcept { return _useId != BaseReg::kIdBad; }
936 : //! Tests whether the register must be allocated to a fixed physical register before it's written.
937 614454 : inline bool hasOutId() const noexcept { return _outId != BaseReg::kIdBad; }
938 :
939 : //! Returns a physical register id used for 'use' operation.
940 115311 : inline uint32_t useId() const noexcept { return _useId; }
941 : //! Returns a physical register id used for 'out' operation.
942 918676 : inline uint32_t outId() const noexcept { return _outId; }
943 :
944 637620 : inline uint32_t useRewriteMask() const noexcept { return _useRewriteMask; }
945 614454 : inline uint32_t outRewriteMask() const noexcept { return _outRewriteMask; }
946 :
947 : //! Sets a physical register used for 'use' operation.
948 271220 : inline void setUseId(uint32_t index) noexcept { _useId = uint8_t(index); }
949 : //! Sets a physical register used for 'out' operation.
950 304222 : inline void setOutId(uint32_t index) noexcept { _outId = uint8_t(index); }
951 :
952 : inline bool isUseDone() const noexcept { return hasFlag(RATiedFlags::kUseDone); }
953 : inline bool isOutDone() const noexcept { return hasFlag(RATiedFlags::kUseDone); }
954 :
955 : inline void markUseDone() noexcept { addFlags(RATiedFlags::kUseDone); }
956 : inline void markOutDone() noexcept { addFlags(RATiedFlags::kUseDone); }
957 :
958 : //! \}
959 : };
960 :
961 : //! Flags used by \ref RAWorkReg.
962 : enum class RAWorkRegFlags : uint32_t {
963 : //! No flags.
964 : kNone = 0,
965 :
966 : //! This register has already been allocated.
967 : kAllocated = 0x00000001u,
968 : //! Has been coalesced to another WorkReg.
969 : kCoalesced = 0x00000002u,
970 :
971 : //! Set when this register is used as a LEAD consecutive register at least once.
972 : kLeadConsecutive = 0x00000004u,
973 : //! Used to mark consecutive registers during processing.
974 : kProcessedConsecutive = 0x00000008u,
975 :
976 : //! Stack slot has to be allocated.
977 : kStackUsed = 0x00000010u,
978 : //! Stack allocation is preferred.
979 : kStackPreferred = 0x00000020u,
980 : //! Marked for stack argument reassignment.
981 : kStackArgToStack = 0x00000040u
982 : };
983 41243 : ASMJIT_DEFINE_ENUM_FLAGS(RAWorkRegFlags)
984 :
985 : //! Work register provides additional data of \ref VirtReg that is used by register allocator.
986 : //!
987 : //! In general when a virtual register is found by register allocator it maps it to \ref RAWorkReg
988 : //! and then only works with it. The reason for such mapping is that users can create many virtual
989 : //! registers, which are not used inside a register allocation scope (which is currently always a
990 : //! function). So register allocator basically scans the function for virtual registers and maps
991 : //! them into WorkRegs, which receive a temporary ID (workId), which starts from zero. This WorkId
992 : //! is then used in bit-arrays and other mappings.
993 : class RAWorkReg {
994 : public:
995 : ASMJIT_NONCOPYABLE(RAWorkReg)
996 :
997 : //! \name Constants
998 : //! \{
999 :
1000 : enum : uint32_t {
1001 : kIdNone = 0xFFFFFFFFu
1002 : };
1003 :
1004 : enum : uint32_t {
1005 : kNoArgIndex = 0xFFu
1006 : };
1007 :
1008 : //! \}
1009 :
1010 : //! \name Members
1011 : //! \{
1012 :
1013 : //! RAPass specific ID used during analysis and allocation.
1014 : uint32_t _workId = 0;
1015 : //! Copy of ID used by \ref VirtReg.
1016 : uint32_t _virtId = 0;
1017 :
1018 : //! Permanent association with \ref VirtReg.
1019 : VirtReg* _virtReg = nullptr;
1020 : //! Temporary association with \ref RATiedReg.
1021 : RATiedReg* _tiedReg = nullptr;
1022 : //! Stack slot associated with the register.
1023 : RAStackSlot* _stackSlot = nullptr;
1024 :
1025 : //! Copy of a signature used by \ref VirtReg.
1026 : OperandSignature _signature {};
1027 : //! RAPass specific flags used during analysis and allocation.
1028 : RAWorkRegFlags _flags = RAWorkRegFlags::kNone;
1029 :
1030 : //! Constains all USE ids collected from all instructions.
1031 : //!
1032 : //! If this mask is non-zero and not a power of two, it means that the register is used multiple times in
1033 : //! instructions where it requires to have a different use ID. This means that in general it's not possible
1034 : //! to keep this register in a single home.
1035 : RegMask _useIdMask = 0;
1036 : //! Preferred mask of registers (if non-zero) to allocate this register to.
1037 : //!
1038 : //! If this mask is zero it means that either there is no intersection of preferred registers collected from all
1039 : //! TiedRegs or there is no preference at all (the register can be allocated to any register all the time).
1040 : RegMask _preferredMask = 0xFFFFFFFFu;
1041 : //! Consecutive mask, which was collected from all instructions where this register was used as a lead consecutive
1042 : //! register.
1043 : RegMask _consecutiveMask = 0xFFFFFFFFu;
1044 : //! IDs of all physical registers that are clobbered during the lifetime of this WorkReg.
1045 : //!
1046 : //! This mask should be updated by `RAPass::buildLiveness()`, because it's global and should
1047 : //! be updated after unreachable code has been removed.
1048 : RegMask _clobberSurvivalMask = 0;
1049 : //! IDs of all physical registers this WorkReg has been allocated to.
1050 : RegMask _allocatedMask = 0;
1051 :
1052 : //! A byte-mask where each bit represents one valid byte of the register.
1053 : uint64_t _regByteMask = 0;
1054 :
1055 : //! Argument index (or `kNoArgIndex` if none).
1056 : uint8_t _argIndex = kNoArgIndex;
1057 : //! Argument value index in the pack (0 by default).
1058 : uint8_t _argValueIndex = 0;
1059 : //! Global home register ID (if any, assigned by RA).
1060 : uint8_t _homeRegId = BaseReg::kIdBad;
1061 : //! Global hint register ID (provided by RA or user).
1062 : uint8_t _hintRegId = BaseReg::kIdBad;
1063 :
1064 : //! Live spans of the `VirtReg`.
1065 : LiveRegSpans _liveSpans {};
1066 : //! Live statistics.
1067 : RALiveStats _liveStats {};
1068 :
1069 : //! All nodes that read/write this VirtReg/WorkReg.
1070 : ZoneVector<BaseNode*> _refs {};
1071 : //! All nodes that write to this VirtReg/WorkReg.
1072 : ZoneVector<BaseNode*> _writes {};
1073 :
1074 : //! Contains work IDs of all immediate consecutive registers of this register.
1075 : //!
1076 : //! \note This bit array only contains immediate consecutives. This means that if this is a register that is
1077 : //! followed by 3 more registers, then it would still have only a single immediate. The rest registers would
1078 : //! have immediate consecutive registers as well, except the last one.
1079 : ZoneBitVector _immediateConsecutives {};
1080 :
1081 : //! \}
1082 :
1083 : //! \name Construction & Destruction
1084 : //! \{
1085 :
1086 271314 : inline RAWorkReg(VirtReg* vReg, uint32_t workId) noexcept
1087 271314 : : _workId(workId),
1088 271314 : _virtId(vReg->id()),
1089 271314 : _virtReg(vReg),
1090 271314 : _signature(vReg->signature()) {}
1091 :
1092 : //! \}
1093 :
1094 : //! \name Accessors
1095 : //! \{
1096 :
1097 0 : inline uint32_t workId() const noexcept { return _workId; }
1098 594988 : inline uint32_t virtId() const noexcept { return _virtId; }
1099 :
1100 0 : inline const char* name() const noexcept { return _virtReg->name(); }
1101 : inline uint32_t nameSize() const noexcept { return _virtReg->nameSize(); }
1102 :
1103 50221 : inline TypeId typeId() const noexcept { return _virtReg->typeId(); }
1104 :
1105 : inline RAWorkRegFlags flags() const noexcept { return _flags; }
1106 515136 : inline bool hasFlag(RAWorkRegFlags flag) const noexcept { return Support::test(_flags, flag); }
1107 : inline void addFlags(RAWorkRegFlags flags) noexcept { _flags |= flags; }
1108 :
1109 : inline bool isAllocated() const noexcept { return hasFlag(RAWorkRegFlags::kAllocated); }
1110 : inline void markAllocated() noexcept { addFlags(RAWorkRegFlags::kAllocated); }
1111 :
1112 : inline bool isLeadConsecutive() const noexcept { return hasFlag(RAWorkRegFlags::kLeadConsecutive); }
1113 0 : inline void markLeadConsecutive() noexcept { addFlags(RAWorkRegFlags::kLeadConsecutive); }
1114 :
1115 : inline bool isProcessedConsecutive() const noexcept { return hasFlag(RAWorkRegFlags::kProcessedConsecutive); }
1116 0 : inline void markProcessedConsecutive() noexcept { addFlags(RAWorkRegFlags::kProcessedConsecutive); }
1117 :
1118 : inline bool isStackUsed() const noexcept { return hasFlag(RAWorkRegFlags::kStackUsed); }
1119 : inline void markStackUsed() noexcept { addFlags(RAWorkRegFlags::kStackUsed); }
1120 :
1121 : inline bool isStackPreferred() const noexcept { return hasFlag(RAWorkRegFlags::kStackPreferred); }
1122 : inline void markStackPreferred() noexcept { addFlags(RAWorkRegFlags::kStackPreferred); }
1123 :
1124 : //! Tests whether this RAWorkReg has been coalesced with another one (cannot be used anymore).
1125 : inline bool isCoalesced() const noexcept { return hasFlag(RAWorkRegFlags::kCoalesced); }
1126 :
1127 138457 : inline OperandSignature signature() const noexcept { return _signature; }
1128 : inline RegType type() const noexcept { return _signature.regType(); }
1129 : inline RegGroup group() const noexcept { return _signature.regGroup(); }
1130 :
1131 325218 : inline VirtReg* virtReg() const noexcept { return _virtReg; }
1132 :
1133 : inline bool hasTiedReg() const noexcept { return _tiedReg != nullptr; }
1134 693385 : inline RATiedReg* tiedReg() const noexcept { return _tiedReg; }
1135 614454 : inline void setTiedReg(RATiedReg* tiedReg) noexcept { _tiedReg = tiedReg; }
1136 614454 : inline void resetTiedReg() noexcept { _tiedReg = nullptr; }
1137 :
1138 271314 : inline bool hasStackSlot() const noexcept { return _stackSlot != nullptr; }
1139 142262 : inline RAStackSlot* stackSlot() const noexcept { return _stackSlot; }
1140 :
1141 : inline LiveRegSpans& liveSpans() noexcept { return _liveSpans; }
1142 : inline const LiveRegSpans& liveSpans() const noexcept { return _liveSpans; }
1143 :
1144 : inline RALiveStats& liveStats() noexcept { return _liveStats; }
1145 : inline const RALiveStats& liveStats() const noexcept { return _liveStats; }
1146 :
1147 : inline bool hasArgIndex() const noexcept { return _argIndex != kNoArgIndex; }
1148 0 : inline uint32_t argIndex() const noexcept { return _argIndex; }
1149 0 : inline uint32_t argValueIndex() const noexcept { return _argValueIndex; }
1150 :
1151 : inline void setArgIndex(uint32_t argIndex, uint32_t valueIndex) noexcept {
1152 0 : _argIndex = uint8_t(argIndex);
1153 0 : _argValueIndex = uint8_t(valueIndex);
1154 : }
1155 :
1156 557730 : inline bool hasHomeRegId() const noexcept { return _homeRegId != BaseReg::kIdBad; }
1157 299883 : inline uint32_t homeRegId() const noexcept { return _homeRegId; }
1158 260869 : inline void setHomeRegId(uint32_t physId) noexcept { _homeRegId = uint8_t(physId); }
1159 :
1160 271314 : inline bool hasHintRegId() const noexcept { return _hintRegId != BaseReg::kIdBad; }
1161 27492 : inline uint32_t hintRegId() const noexcept { return _hintRegId; }
1162 27492 : inline void setHintRegId(uint32_t physId) noexcept { _hintRegId = uint8_t(physId); }
1163 :
1164 : inline RegMask useIdMask() const noexcept { return _useIdMask; }
1165 : inline bool hasUseIdMask() const noexcept { return _useIdMask != 0u; }
1166 : inline bool hasMultipleUseIds() const noexcept { return _useIdMask != 0u && !Support::isPowerOf2(_useIdMask); }
1167 37365 : inline void addUseIdMask(RegMask mask) noexcept { _useIdMask |= mask; }
1168 :
1169 243822 : inline RegMask preferredMask() const noexcept { return _preferredMask; }
1170 : inline bool hasPrereffedMask() const noexcept { return _preferredMask != 0xFFFFFFFFu; }
1171 1219035 : inline void restrictPreferredMask(RegMask mask) noexcept { _preferredMask &= mask; }
1172 :
1173 0 : inline RegMask consecutiveMask() const noexcept { return _consecutiveMask; }
1174 : inline bool hasConsecutiveMask() const noexcept { return _consecutiveMask != 0xFFFFFFFFu; }
1175 0 : inline void restrictConsecutiveMask(RegMask mask) noexcept { _consecutiveMask &= mask; }
1176 :
1177 544801 : inline RegMask clobberSurvivalMask() const noexcept { return _clobberSurvivalMask; }
1178 1027 : inline void addClobberSurvivalMask(RegMask mask) noexcept { _clobberSurvivalMask |= mask; }
1179 :
1180 26179 : inline RegMask allocatedMask() const noexcept { return _allocatedMask; }
1181 0 : inline void addAllocatedMask(RegMask mask) noexcept { _allocatedMask |= mask; }
1182 :
1183 294583 : inline uint64_t regByteMask() const noexcept { return _regByteMask; }
1184 271314 : inline void setRegByteMask(uint64_t mask) noexcept { _regByteMask = mask; }
1185 :
1186 : inline bool hasImmediateConsecutives() const noexcept { return !_immediateConsecutives.empty(); }
1187 : inline const ZoneBitVector& immediateConsecutives() const noexcept { return _immediateConsecutives; }
1188 :
1189 0 : inline Error addImmediateConsecutive(ZoneAllocator* allocator, uint32_t workId) noexcept {
1190 0 : if (_immediateConsecutives.size() <= workId)
1191 0 : ASMJIT_PROPAGATE(_immediateConsecutives.resize(allocator, workId + 1));
1192 :
1193 : _immediateConsecutives.setBit(workId, true);
1194 0 : return kErrorOk;
1195 : }
1196 :
1197 : //! \}
1198 : };
1199 :
1200 : //! \}
1201 : //! \endcond
1202 :
1203 : ASMJIT_END_NAMESPACE
1204 :
1205 : #endif // ASMJIT_CORE_RADEFS_P_H_INCLUDED
1206 :
1207 : #endif // __PLUMED_HAS_ASMJIT
|
