Line data Source code
1 : #ifdef __PLUMED_HAS_ASMJIT
2 : // This file is part of AsmJit project <https://asmjit.com>
3 : //
4 : // See asmjit.h or LICENSE.md for license and copyright information
5 : // SPDX-License-Identifier: Zlib
6 :
7 : #ifndef ASMJIT_CORE_RAASSIGNMENT_P_H_INCLUDED
8 : #define ASMJIT_CORE_RAASSIGNMENT_P_H_INCLUDED
9 :
10 : #include "api-config.h"
11 : #ifndef ASMJIT_NO_COMPILER
12 :
13 : #include "radefs_p.h"
14 :
15 : ASMJIT_BEGIN_NAMESPACE
16 :
17 : //! \cond INTERNAL
18 : //! \addtogroup asmjit_ra
19 : //! \{
20 :
21 : //! Holds the current register assignment.
22 : //!
23 : //! Has two purposes:
24 : //!
25 : //! 1. Holds register assignment of a local register allocator (see \ref RALocalAllocator).
26 : //! 2. Holds register assignment of the entry of basic blocks (see \ref RABlock).
27 : class RAAssignment {
28 : public:
29 : ASMJIT_NONCOPYABLE(RAAssignment)
30 :
31 : enum Ids : uint32_t {
32 : kPhysNone = 0xFF,
33 : kWorkNone = RAWorkReg::kIdNone
34 : };
35 :
36 : enum DirtyBit : uint32_t {
37 : kClean = 0,
38 : kDirty = 1
39 : };
40 :
41 : struct Layout {
42 : //! Index of architecture registers per group.
43 : RARegIndex physIndex;
44 : //! Count of architecture registers per group.
45 : RARegCount physCount;
46 : //! Count of physical registers of all groups.
47 : uint32_t physTotal;
48 : //! Count of work registers.
49 : uint32_t workCount;
50 : //! WorkRegs data (vector).
51 : const RAWorkRegs* workRegs;
52 :
53 : inline void reset() noexcept {
54 : physIndex.reset();
55 : physCount.reset();
56 54984 : physTotal = 0;
57 54984 : workCount = 0;
58 54984 : workRegs = nullptr;
59 : }
60 : };
61 :
62 : struct PhysToWorkMap {
63 : //! Assigned registers (each bit represents one physical reg).
64 : RARegMask assigned;
65 : //! Dirty registers (spill slot out of sync or no spill slot).
66 : RARegMask dirty;
67 : //! PhysReg to WorkReg mapping.
68 : uint32_t workIds[1 /* ... */];
69 :
70 : static inline size_t sizeOf(size_t count) noexcept {
71 109968 : return sizeof(PhysToWorkMap) - sizeof(uint32_t) + count * sizeof(uint32_t);
72 : }
73 :
74 54984 : inline void reset(size_t count) noexcept {
75 : assigned.reset();
76 : dirty.reset();
77 :
78 2694216 : for (size_t i = 0; i < count; i++)
79 2639232 : workIds[i] = kWorkNone;
80 54984 : }
81 :
82 : inline void copyFrom(const PhysToWorkMap* other, size_t count) noexcept {
83 : size_t size = sizeOf(count);
84 : memcpy(this, other, size);
85 : }
86 :
87 0 : inline void unassign(RegGroup group, uint32_t physId, uint32_t indexInWorkIds) noexcept {
88 : assigned.clear(group, Support::bitMask(physId));
89 : dirty.clear(group, Support::bitMask(physId));
90 0 : workIds[indexInWorkIds] = kWorkNone;
91 0 : }
92 : };
93 :
94 : struct WorkToPhysMap {
95 : //! WorkReg to PhysReg mapping
96 : uint8_t physIds[1 /* ... */];
97 :
98 : static inline size_t sizeOf(size_t count) noexcept {
99 : return size_t(count) * sizeof(uint8_t);
100 : }
101 :
102 : inline void reset(size_t count) noexcept {
103 896418 : for (size_t i = 0; i < count; i++)
104 813942 : physIds[i] = kPhysNone;
105 : }
106 :
107 : inline void copyFrom(const WorkToPhysMap* other, size_t count) noexcept {
108 : size_t size = sizeOf(count);
109 : if (ASMJIT_LIKELY(size))
110 : memcpy(this, other, size);
111 : }
112 : };
113 :
114 : //! \name Members
115 : //! \{
116 :
117 : //! Physical registers layout.
118 : Layout _layout;
119 : //! WorkReg to PhysReg mapping.
120 : WorkToPhysMap* _workToPhysMap;
121 : //! PhysReg to WorkReg mapping and assigned/dirty bits.
122 : PhysToWorkMap* _physToWorkMap;
123 : //! Optimization to translate PhysRegs to WorkRegs faster.
124 : Support::Array<uint32_t*, Globals::kNumVirtGroups> _physToWorkIds;
125 :
126 : //! \}
127 :
128 : //! \name Construction & Destruction
129 : //! \{
130 :
131 54984 : inline RAAssignment() noexcept {
132 : _layout.reset();
133 : resetMaps();
134 54984 : }
135 :
136 : ASMJIT_FORCE_INLINE void initLayout(const RARegCount& physCount, const RAWorkRegs& workRegs) noexcept {
137 : // Layout must be initialized before data.
138 : ASMJIT_ASSERT(_physToWorkMap == nullptr);
139 : ASMJIT_ASSERT(_workToPhysMap == nullptr);
140 :
141 : _layout.physIndex.buildIndexes(physCount);
142 54984 : _layout.physCount = physCount;
143 54984 : _layout.physTotal = uint32_t(_layout.physIndex[RegGroup::kMaxVirt]) +
144 54984 : uint32_t(_layout.physCount[RegGroup::kMaxVirt]) ;
145 54984 : _layout.workCount = workRegs.size();
146 0 : _layout.workRegs = &workRegs;
147 : }
148 :
149 : ASMJIT_FORCE_INLINE void initMaps(PhysToWorkMap* physToWorkMap, WorkToPhysMap* workToPhysMap) noexcept {
150 54984 : _physToWorkMap = physToWorkMap;
151 54984 : _workToPhysMap = workToPhysMap;
152 274920 : for (RegGroup group : RegGroupVirtValues{})
153 219936 : _physToWorkIds[group] = physToWorkMap->workIds + _layout.physIndex.get(group);
154 : }
155 :
156 : ASMJIT_FORCE_INLINE void resetMaps() noexcept {
157 54984 : _physToWorkMap = nullptr;
158 54984 : _workToPhysMap = nullptr;
159 54984 : _physToWorkIds.fill(nullptr);
160 : }
161 :
162 : //! \}
163 :
164 : //! \name Accessors
165 : //! \{
166 :
167 54984 : inline PhysToWorkMap* physToWorkMap() const noexcept { return _physToWorkMap; }
168 0 : inline WorkToPhysMap* workToPhysMap() const noexcept { return _workToPhysMap; }
169 :
170 : inline RARegMask& assigned() noexcept { return _physToWorkMap->assigned; }
171 : inline const RARegMask& assigned() const noexcept { return _physToWorkMap->assigned; }
172 352632 : inline uint32_t assigned(RegGroup group) const noexcept { return _physToWorkMap->assigned[group]; }
173 :
174 : inline RARegMask& dirty() noexcept { return _physToWorkMap->dirty; }
175 : inline const RARegMask& dirty() const noexcept { return _physToWorkMap->dirty; }
176 0 : inline RegMask dirty(RegGroup group) const noexcept { return _physToWorkMap->dirty[group]; }
177 :
178 : inline uint32_t workToPhysId(RegGroup group, uint32_t workId) const noexcept {
179 : DebugUtils::unused(group);
180 : ASMJIT_ASSERT(workId != kWorkNone);
181 : ASMJIT_ASSERT(workId < _layout.workCount);
182 1249252 : return _workToPhysMap->physIds[workId];
183 : }
184 :
185 : inline uint32_t physToWorkId(RegGroup group, uint32_t physId) const noexcept {
186 : ASMJIT_ASSERT(physId < Globals::kMaxPhysRegs);
187 500358 : return _physToWorkIds[group][physId];
188 : }
189 :
190 : inline bool isPhysAssigned(RegGroup group, uint32_t physId) const noexcept {
191 : ASMJIT_ASSERT(physId < Globals::kMaxPhysRegs);
192 2892 : return Support::bitTest(_physToWorkMap->assigned[group], physId);
193 : }
194 :
195 : inline bool isPhysDirty(RegGroup group, uint32_t physId) const noexcept {
196 : ASMJIT_ASSERT(physId < Globals::kMaxPhysRegs);
197 153382 : return Support::bitTest(_physToWorkMap->dirty[group], physId);
198 : }
199 :
200 : //! \}
201 :
202 : //! \name Assignment
203 : //!
204 : //! These are low-level allocation helpers that are used to update the current mappings between physical and
205 : //! virt/work registers and also to update masks that represent allocated and dirty registers. These functions
206 : //! don't emit any code; they are only used to update and keep all mappings in sync.
207 : //!
208 : //! \{
209 :
210 : //! Assign [VirtReg/WorkReg] to a physical register.
211 294376 : inline void assign(RegGroup group, uint32_t workId, uint32_t physId, bool dirty) noexcept {
212 : ASMJIT_ASSERT(workToPhysId(group, workId) == kPhysNone);
213 : ASMJIT_ASSERT(physToWorkId(group, physId) == kWorkNone);
214 : ASMJIT_ASSERT(!isPhysAssigned(group, physId));
215 : ASMJIT_ASSERT(!isPhysDirty(group, physId));
216 :
217 294376 : _workToPhysMap->physIds[workId] = uint8_t(physId);
218 294376 : _physToWorkIds[group][physId] = workId;
219 :
220 : RegMask regMask = Support::bitMask(physId);
221 294376 : _physToWorkMap->assigned[group] |= regMask;
222 294376 : _physToWorkMap->dirty[group] |= regMask & Support::bitMaskFromBool<RegMask>(dirty);
223 :
224 : verify();
225 294376 : }
226 :
227 : //! Reassign [VirtReg/WorkReg] to `dstPhysId` from `srcPhysId`.
228 1785 : inline void reassign(RegGroup group, uint32_t workId, uint32_t dstPhysId, uint32_t srcPhysId) noexcept {
229 : ASMJIT_ASSERT(dstPhysId != srcPhysId);
230 : ASMJIT_ASSERT(workToPhysId(group, workId) == srcPhysId);
231 : ASMJIT_ASSERT(physToWorkId(group, srcPhysId) == workId);
232 : ASMJIT_ASSERT(isPhysAssigned(group, srcPhysId) == true);
233 : ASMJIT_ASSERT(isPhysAssigned(group, dstPhysId) == false);
234 :
235 1785 : _workToPhysMap->physIds[workId] = uint8_t(dstPhysId);
236 1785 : _physToWorkIds[group][srcPhysId] = kWorkNone;
237 1785 : _physToWorkIds[group][dstPhysId] = workId;
238 :
239 : RegMask srcMask = Support::bitMask(srcPhysId);
240 : RegMask dstMask = Support::bitMask(dstPhysId);
241 :
242 1785 : bool dirty = (_physToWorkMap->dirty[group] & srcMask) != 0;
243 1785 : RegMask regMask = dstMask | srcMask;
244 :
245 1785 : _physToWorkMap->assigned[group] ^= regMask;
246 1785 : _physToWorkMap->dirty[group] ^= regMask & Support::bitMaskFromBool<RegMask>(dirty);
247 :
248 : verify();
249 1785 : }
250 :
251 0 : inline void swap(RegGroup group, uint32_t aWorkId, uint32_t aPhysId, uint32_t bWorkId, uint32_t bPhysId) noexcept {
252 : ASMJIT_ASSERT(aPhysId != bPhysId);
253 : ASMJIT_ASSERT(workToPhysId(group, aWorkId) == aPhysId);
254 : ASMJIT_ASSERT(workToPhysId(group, bWorkId) == bPhysId);
255 : ASMJIT_ASSERT(physToWorkId(group, aPhysId) == aWorkId);
256 : ASMJIT_ASSERT(physToWorkId(group, bPhysId) == bWorkId);
257 : ASMJIT_ASSERT(isPhysAssigned(group, aPhysId));
258 : ASMJIT_ASSERT(isPhysAssigned(group, bPhysId));
259 :
260 0 : _workToPhysMap->physIds[aWorkId] = uint8_t(bPhysId);
261 0 : _workToPhysMap->physIds[bWorkId] = uint8_t(aPhysId);
262 0 : _physToWorkIds[group][aPhysId] = bWorkId;
263 0 : _physToWorkIds[group][bPhysId] = aWorkId;
264 :
265 : RegMask aMask = Support::bitMask(aPhysId);
266 : RegMask bMask = Support::bitMask(bPhysId);
267 0 : RegMask flipMask = Support::bitMaskFromBool<RegMask>(((_physToWorkMap->dirty[group] & aMask) != 0) ^ ((_physToWorkMap->dirty[group] & bMask) != 0));
268 0 : RegMask regMask = aMask | bMask;
269 0 : _physToWorkMap->dirty[group] ^= regMask & flipMask;
270 :
271 : verify();
272 0 : }
273 :
274 : //! Unassign [VirtReg/WorkReg] from a physical register.
275 294376 : inline void unassign(RegGroup group, uint32_t workId, uint32_t physId) noexcept {
276 : ASMJIT_ASSERT(physId < Globals::kMaxPhysRegs);
277 : ASMJIT_ASSERT(workToPhysId(group, workId) == physId);
278 : ASMJIT_ASSERT(physToWorkId(group, physId) == workId);
279 : ASMJIT_ASSERT(isPhysAssigned(group, physId));
280 :
281 294376 : _workToPhysMap->physIds[workId] = kPhysNone;
282 294376 : _physToWorkIds[group][physId] = kWorkNone;
283 :
284 : RegMask regMask = Support::bitMask(physId);
285 294376 : _physToWorkMap->assigned[group] &= ~regMask;
286 294376 : _physToWorkMap->dirty[group] &= ~regMask;
287 :
288 : verify();
289 294376 : }
290 :
291 : inline void makeClean(RegGroup group, uint32_t workId, uint32_t physId) noexcept {
292 : DebugUtils::unused(workId);
293 : RegMask regMask = Support::bitMask(physId);
294 30798 : _physToWorkMap->dirty[group] &= ~regMask;
295 : }
296 :
297 : inline void makeDirty(RegGroup group, uint32_t workId, uint32_t physId) noexcept {
298 : DebugUtils::unused(workId);
299 : RegMask regMask = Support::bitMask(physId);
300 21048 : _physToWorkMap->dirty[group] |= regMask;
301 21048 : }
302 :
303 : //! \}
304 :
305 : //! \name Utilities
306 : //! \{
307 :
308 : ASMJIT_FORCE_INLINE void swap(RAAssignment& other) noexcept {
309 : std::swap(_workToPhysMap, other._workToPhysMap);
310 : std::swap(_physToWorkMap, other._physToWorkMap);
311 : _physToWorkIds.swap(other._physToWorkIds);
312 : }
313 :
314 27492 : inline void assignWorkIdsFromPhysIds() noexcept {
315 27492 : memset(_workToPhysMap, uint8_t(BaseReg::kIdBad), WorkToPhysMap::sizeOf(_layout.workCount));
316 :
317 137460 : for (RegGroup group : RegGroupVirtValues{}) {
318 109968 : uint32_t physBaseIndex = _layout.physIndex[group];
319 109968 : Support::BitWordIterator<RegMask> it(_physToWorkMap->assigned[group]);
320 :
321 109968 : while (it.hasNext()) {
322 : uint32_t physId = it.next();
323 0 : uint32_t workId = _physToWorkMap->workIds[physBaseIndex + physId];
324 :
325 : ASMJIT_ASSERT(workId != kWorkNone);
326 0 : _workToPhysMap->physIds[workId] = uint8_t(physId);
327 : }
328 : }
329 27492 : }
330 :
331 27492 : inline void copyFrom(const PhysToWorkMap* physToWorkMap) noexcept {
332 27492 : memcpy(_physToWorkMap, physToWorkMap, PhysToWorkMap::sizeOf(_layout.physTotal));
333 27492 : assignWorkIdsFromPhysIds();
334 27492 : }
335 :
336 0 : inline void copyFrom(const PhysToWorkMap* physToWorkMap, const WorkToPhysMap* workToPhysMap) noexcept {
337 0 : memcpy(_physToWorkMap, physToWorkMap, PhysToWorkMap::sizeOf(_layout.physTotal));
338 0 : memcpy(_workToPhysMap, workToPhysMap, WorkToPhysMap::sizeOf(_layout.workCount));
339 0 : }
340 :
341 : inline void copyFrom(const RAAssignment& other) noexcept {
342 0 : copyFrom(other.physToWorkMap(), other.workToPhysMap());
343 : }
344 :
345 : // Not really useful outside of debugging.
346 : bool equals(const RAAssignment& other) const noexcept {
347 : // Layout should always match.
348 : if (_layout.physIndex != other._layout.physIndex ||
349 : _layout.physCount != other._layout.physCount ||
350 : _layout.physTotal != other._layout.physTotal ||
351 : _layout.workCount != other._layout.workCount ||
352 : _layout.workRegs != other._layout.workRegs)
353 : return false;
354 :
355 : uint32_t physTotal = _layout.physTotal;
356 : uint32_t workCount = _layout.workCount;
357 :
358 : for (uint32_t physId = 0; physId < physTotal; physId++) {
359 : uint32_t thisWorkId = _physToWorkMap->workIds[physId];
360 : uint32_t otherWorkId = other._physToWorkMap->workIds[physId];
361 : if (thisWorkId != otherWorkId)
362 : return false;
363 : }
364 :
365 : for (uint32_t workId = 0; workId < workCount; workId++) {
366 : uint32_t thisPhysId = _workToPhysMap->physIds[workId];
367 : uint32_t otherPhysId = other._workToPhysMap->physIds[workId];
368 : if (thisPhysId != otherPhysId)
369 : return false;
370 : }
371 :
372 : if (_physToWorkMap->assigned != other._physToWorkMap->assigned ||
373 : _physToWorkMap->dirty != other._physToWorkMap->dirty )
374 : return false;
375 :
376 : return true;
377 : }
378 :
379 : #if defined(ASMJIT_BUILD_DEBUG)
380 : ASMJIT_NOINLINE void verify() noexcept {
381 : // Verify WorkToPhysMap.
382 : {
383 : for (uint32_t workId = 0; workId < _layout.workCount; workId++) {
384 : uint32_t physId = _workToPhysMap->physIds[workId];
385 : if (physId != kPhysNone) {
386 : const RAWorkReg* workReg = _layout.workRegs->at(workId);
387 : RegGroup group = workReg->group();
388 : ASMJIT_ASSERT(_physToWorkIds[group][physId] == workId);
389 : }
390 : }
391 : }
392 :
393 : // Verify PhysToWorkMap.
394 : {
395 : for (RegGroup group : RegGroupVirtValues{}) {
396 : uint32_t physCount = _layout.physCount[group];
397 : for (uint32_t physId = 0; physId < physCount; physId++) {
398 : uint32_t workId = _physToWorkIds[group][physId];
399 : if (workId != kWorkNone) {
400 : ASMJIT_ASSERT(_workToPhysMap->physIds[workId] == physId);
401 : }
402 : }
403 : }
404 : }
405 : }
406 : #else
407 : inline void verify() noexcept {}
408 : #endif
409 :
410 : //! \}
411 : };
412 :
413 : //! \}
414 : //! \endcond
415 :
416 : ASMJIT_END_NAMESPACE
417 :
418 : #endif // !ASMJIT_NO_COMPILER
419 : #endif // ASMJIT_CORE_RAASSIGNMENT_P_H_INCLUDED
420 :
421 : #endif // __PLUMED_HAS_ASMJIT
|
