Line data Source code
1 : #ifdef __PLUMED_HAS_ASMJIT
2 : // This file is part of AsmJit project <https://asmjit.com>
3 : //
4 : // See asmjit.h or LICENSE.md for license and copyright information
5 : // SPDX-License-Identifier: Zlib
6 :
7 : #include "api-build_p.h"
8 : #if !defined(ASMJIT_NO_AARCH64) && !defined(ASMJIT_NO_COMPILER)
9 :
10 : #include "cpuinfo.h"
11 : #include "support.h"
12 : #include "type.h"
13 : #include "a64assembler.h"
14 : #include "a64compiler.h"
15 : #include "a64emithelper_p.h"
16 : #include "a64instapi_p.h"
17 : #include "a64instdb_p.h"
18 : #include "a64rapass_p.h"
19 :
20 : ASMJIT_BEGIN_SUB_NAMESPACE(a64)
21 :
22 : // a64::ARMRAPass - Helpers
23 : // ========================
24 :
25 : // TODO: [ARM] These should be shared with all backends.
26 : ASMJIT_MAYBE_UNUSED
27 : static inline uint64_t raImmMaskFromSize(uint32_t size) noexcept {
28 : ASMJIT_ASSERT(size > 0 && size < 256);
29 : static const uint64_t masks[] = {
30 : 0x00000000000000FFu, // 1
31 : 0x000000000000FFFFu, // 2
32 : 0x00000000FFFFFFFFu, // 4
33 : 0xFFFFFFFFFFFFFFFFu, // 8
34 : 0x0000000000000000u, // 16
35 : 0x0000000000000000u, // 32
36 : 0x0000000000000000u, // 64
37 : 0x0000000000000000u, // 128
38 : 0x0000000000000000u // 256
39 : };
40 : return masks[Support::ctz(size)];
41 : }
42 :
43 : static const RegMask raConsecutiveLeadCountToRegMaskFilter[5] = {
44 : 0xFFFFFFFFu, // [0] No consecutive.
45 : 0x00000000u, // [1] Invalid, never used.
46 : 0x7FFFFFFFu, // [2] 2 consecutive registers.
47 : 0x3FFFFFFFu, // [3] 3 consecutive registers.
48 : 0x1FFFFFFFu // [4] 4 consecutive registers.
49 : };
50 :
51 : static inline RATiedFlags raUseOutFlagsFromRWFlags(OpRWFlags rwFlags) noexcept {
52 : static constexpr RATiedFlags map[] = {
53 : RATiedFlags::kNone,
54 : RATiedFlags::kRead | RATiedFlags::kUse, // kRead
55 : RATiedFlags::kWrite | RATiedFlags::kOut, // kWrite
56 : RATiedFlags::kRW | RATiedFlags::kUse, // kRW
57 : };
58 :
59 0 : return map[uint32_t(rwFlags & OpRWFlags::kRW)];
60 : }
61 :
62 : static inline RATiedFlags raRegRwFlags(OpRWFlags flags) noexcept {
63 : return raUseOutFlagsFromRWFlags(flags);
64 : }
65 :
66 : static inline RATiedFlags raMemBaseRwFlags(OpRWFlags flags) noexcept {
67 : constexpr uint32_t shift = Support::ConstCTZ<uint32_t(OpRWFlags::kMemBaseRW)>::value;
68 0 : return raUseOutFlagsFromRWFlags(OpRWFlags(uint32_t(flags) >> shift) & OpRWFlags::kRW);
69 : }
70 :
71 : static inline RATiedFlags raMemIndexRwFlags(OpRWFlags flags) noexcept {
72 : constexpr uint32_t shift = Support::ConstCTZ<uint32_t(OpRWFlags::kMemIndexRW)>::value;
73 0 : return raUseOutFlagsFromRWFlags(OpRWFlags(uint32_t(flags) >> shift) & OpRWFlags::kRW);
74 : }
75 : // a64::RACFGBuilder
76 : // =================
77 :
78 : class RACFGBuilder : public RACFGBuilderT<RACFGBuilder> {
79 : public:
80 : Arch _arch;
81 :
82 : inline RACFGBuilder(ARMRAPass* pass) noexcept
83 0 : : RACFGBuilderT<RACFGBuilder>(pass),
84 0 : _arch(pass->cc()->arch()) {}
85 :
86 0 : inline Compiler* cc() const noexcept { return static_cast<Compiler*>(_cc); }
87 :
88 : Error onInst(InstNode* inst, InstControlFlow& controlType, RAInstBuilder& ib) noexcept;
89 :
90 : Error onBeforeInvoke(InvokeNode* invokeNode) noexcept;
91 : Error onInvoke(InvokeNode* invokeNode, RAInstBuilder& ib) noexcept;
92 :
93 : Error moveImmToRegArg(InvokeNode* invokeNode, const FuncValue& arg, const Imm& imm_, BaseReg* out) noexcept;
94 : Error moveImmToStackArg(InvokeNode* invokeNode, const FuncValue& arg, const Imm& imm_) noexcept;
95 : Error moveRegToStackArg(InvokeNode* invokeNode, const FuncValue& arg, const BaseReg& reg) noexcept;
96 :
97 : Error onBeforeRet(FuncRetNode* funcRet) noexcept;
98 : Error onRet(FuncRetNode* funcRet, RAInstBuilder& ib) noexcept;
99 : };
100 :
101 : // a64::RACFGBuilder - OnInst
102 : // ==========================
103 :
104 : // TODO: [ARM] This is just a workaround...
105 0 : static InstControlFlow getControlFlowType(InstId instId) noexcept {
106 0 : switch (BaseInst::extractRealId(instId)) {
107 : case Inst::kIdB:
108 : case Inst::kIdBr:
109 0 : if (BaseInst::extractARMCondCode(instId) == CondCode::kAL)
110 : return InstControlFlow::kJump;
111 : else
112 0 : return InstControlFlow::kBranch;
113 : case Inst::kIdBl:
114 : case Inst::kIdBlr:
115 : return InstControlFlow::kCall;
116 0 : case Inst::kIdCbz:
117 : case Inst::kIdCbnz:
118 : case Inst::kIdTbz:
119 : case Inst::kIdTbnz:
120 0 : return InstControlFlow::kBranch;
121 0 : case Inst::kIdRet:
122 0 : return InstControlFlow::kReturn;
123 0 : default:
124 0 : return InstControlFlow::kRegular;
125 : }
126 : }
127 :
128 0 : Error RACFGBuilder::onInst(InstNode* inst, InstControlFlow& controlType, RAInstBuilder& ib) noexcept {
129 : InstRWInfo rwInfo;
130 :
131 0 : if (Inst::isDefinedId(inst->realId())) {
132 : InstId instId = inst->id();
133 : uint32_t opCount = inst->opCount();
134 : const Operand* opArray = inst->operands();
135 0 : ASMJIT_PROPAGATE(InstInternal::queryRWInfo(_arch, inst->baseInst(), opArray, opCount, &rwInfo));
136 :
137 : const InstDB::InstInfo& instInfo = InstDB::infoById(instId);
138 : uint32_t singleRegOps = 0;
139 :
140 : ib.addInstRWFlags(rwInfo.instFlags());
141 :
142 0 : if (opCount) {
143 : uint32_t consecutiveOffset = 0xFFFFFFFFu;
144 : uint32_t consecutiveParent = Globals::kInvalidId;
145 :
146 0 : for (uint32_t i = 0; i < opCount; i++) {
147 0 : const Operand& op = opArray[i];
148 : const OpRWInfo& opRwInfo = rwInfo.operand(i);
149 :
150 0 : if (op.isReg()) {
151 : // Register Operand
152 : // ----------------
153 : const Reg& reg = op.as<Reg>();
154 :
155 : RATiedFlags flags = raRegRwFlags(opRwInfo.opFlags());
156 : uint32_t vIndex = Operand::virtIdToIndex(reg.id());
157 :
158 0 : if (vIndex < Operand::kVirtIdCount) {
159 : RAWorkReg* workReg;
160 0 : ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(vIndex, &workReg));
161 :
162 : // Use RW instead of Write in case that not the whole register is overwritten. This is important for
163 : // liveness as we cannot kill a register that will be used.
164 0 : if ((flags & RATiedFlags::kRW) == RATiedFlags::kWrite) {
165 0 : if (workReg->regByteMask() & ~(opRwInfo.writeByteMask() | opRwInfo.extendByteMask())) {
166 : // Not write-only operation.
167 : flags = (flags & ~RATiedFlags::kOut) | (RATiedFlags::kRead | RATiedFlags::kUse);
168 : }
169 : }
170 :
171 0 : RegGroup group = workReg->group();
172 :
173 0 : RegMask useRegs = _pass->_availableRegs[group];
174 : RegMask outRegs = useRegs;
175 :
176 : uint32_t useId = BaseReg::kIdBad;
177 : uint32_t outId = BaseReg::kIdBad;
178 :
179 : uint32_t useRewriteMask = 0;
180 : uint32_t outRewriteMask = 0;
181 :
182 0 : if (opRwInfo.consecutiveLeadCount()) {
183 : // There must be a single consecutive register lead, otherwise the RW data is invalid.
184 0 : if (consecutiveOffset != 0xFFFFFFFFu)
185 : return DebugUtils::errored(kErrorInvalidState);
186 :
187 : // A consecutive lead register cannot be used as a consecutive +1/+2/+3 register, the registers must be distinct.
188 0 : if (RATiedReg::consecutiveDataFromFlags(flags) != 0)
189 : return DebugUtils::errored(kErrorNotConsecutiveRegs);
190 :
191 0 : flags |= RATiedFlags::kLeadConsecutive | RATiedReg::consecutiveDataToFlags(opRwInfo.consecutiveLeadCount() - 1);
192 : consecutiveOffset = 0;
193 :
194 0 : RegMask filter = raConsecutiveLeadCountToRegMaskFilter[opRwInfo.consecutiveLeadCount()];
195 0 : if (Support::test(flags, RATiedFlags::kUse)) {
196 : flags |= RATiedFlags::kUseConsecutive;
197 0 : useRegs &= filter;
198 : }
199 : else {
200 : flags |= RATiedFlags::kOutConsecutive;
201 0 : outRegs &= filter;
202 : }
203 : }
204 :
205 0 : if (Support::test(flags, RATiedFlags::kUse)) {
206 0 : useRewriteMask = Support::bitMask(inst->getRewriteIndex(®._baseId));
207 0 : if (opRwInfo.hasOpFlag(OpRWFlags::kRegPhysId)) {
208 : useId = opRwInfo.physId();
209 : flags |= RATiedFlags::kUseFixed;
210 : }
211 0 : else if (opRwInfo.hasOpFlag(OpRWFlags::kConsecutive)) {
212 0 : if (consecutiveOffset == 0xFFFFFFFFu)
213 : return DebugUtils::errored(kErrorInvalidState);
214 0 : flags |= RATiedFlags::kUseConsecutive | RATiedReg::consecutiveDataToFlags(++consecutiveOffset);
215 : }
216 : }
217 : else {
218 0 : outRewriteMask = Support::bitMask(inst->getRewriteIndex(®._baseId));
219 0 : if (opRwInfo.hasOpFlag(OpRWFlags::kRegPhysId)) {
220 : outId = opRwInfo.physId();
221 : flags |= RATiedFlags::kOutFixed;
222 : }
223 0 : else if (opRwInfo.hasOpFlag(OpRWFlags::kConsecutive)) {
224 0 : if (consecutiveOffset == 0xFFFFFFFFu)
225 : return DebugUtils::errored(kErrorInvalidState);
226 0 : flags |= RATiedFlags::kOutConsecutive | RATiedReg::consecutiveDataToFlags(++consecutiveOffset);
227 : }
228 : }
229 :
230 : // Special cases regarding element access.
231 0 : if (reg.as<Vec>().hasElementIndex()) {
232 : // Only the first 0..15 registers can be used if the register uses
233 : // element accessor that accesses half-words (h[0..7] elements).
234 0 : if (instInfo.hasFlag(InstDB::kInstFlagVH0_15) && reg.as<Vec>().elementType() == Vec::kElementTypeH) {
235 0 : if (Support::test(flags, RATiedFlags::kUse))
236 0 : useId &= 0x0000FFFFu;
237 : else
238 0 : outId &= 0x0000FFFFu;
239 : }
240 : }
241 :
242 0 : ASMJIT_PROPAGATE(ib.add(workReg, flags, useRegs, useId, useRewriteMask, outRegs, outId, outRewriteMask, opRwInfo.rmSize(), consecutiveParent));
243 0 : if (singleRegOps == i)
244 0 : singleRegOps++;
245 :
246 0 : if (Support::test(flags, RATiedFlags::kLeadConsecutive | RATiedFlags::kUseConsecutive | RATiedFlags::kOutConsecutive))
247 0 : consecutiveParent = workReg->workId();
248 : }
249 : }
250 0 : else if (op.isMem()) {
251 : // Memory Operand
252 : // --------------
253 : const Mem& mem = op.as<Mem>();
254 :
255 0 : if (mem.isRegHome()) {
256 : RAWorkReg* workReg;
257 0 : ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(Operand::virtIdToIndex(mem.baseId()), &workReg));
258 0 : _pass->getOrCreateStackSlot(workReg);
259 : }
260 0 : else if (mem.hasBaseReg()) {
261 : uint32_t vIndex = Operand::virtIdToIndex(mem.baseId());
262 0 : if (vIndex < Operand::kVirtIdCount) {
263 : RAWorkReg* workReg;
264 0 : ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(vIndex, &workReg));
265 :
266 : RATiedFlags flags = raMemBaseRwFlags(opRwInfo.opFlags());
267 0 : RegGroup group = workReg->group();
268 0 : RegMask allocable = _pass->_availableRegs[group];
269 :
270 : // Base registers have never fixed id on ARM.
271 : const uint32_t useId = BaseReg::kIdBad;
272 : const uint32_t outId = BaseReg::kIdBad;
273 :
274 : uint32_t useRewriteMask = 0;
275 : uint32_t outRewriteMask = 0;
276 :
277 0 : if (Support::test(flags, RATiedFlags::kUse))
278 0 : useRewriteMask = Support::bitMask(inst->getRewriteIndex(&mem._baseId));
279 : else
280 0 : outRewriteMask = Support::bitMask(inst->getRewriteIndex(&mem._baseId));
281 :
282 0 : ASMJIT_PROPAGATE(ib.add(workReg, flags, allocable, useId, useRewriteMask, allocable, outId, outRewriteMask));
283 : }
284 : }
285 :
286 0 : if (mem.hasIndexReg()) {
287 : uint32_t vIndex = Operand::virtIdToIndex(mem.indexId());
288 0 : if (vIndex < Operand::kVirtIdCount) {
289 : RAWorkReg* workReg;
290 0 : ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(vIndex, &workReg));
291 :
292 : RATiedFlags flags = raMemIndexRwFlags(opRwInfo.opFlags());
293 0 : RegGroup group = workReg->group();
294 0 : RegMask allocable = _pass->_availableRegs[group];
295 :
296 : // Index registers have never fixed id on ARM.
297 : const uint32_t useId = BaseReg::kIdBad;
298 : const uint32_t outId = BaseReg::kIdBad;
299 :
300 : uint32_t useRewriteMask = 0;
301 : uint32_t outRewriteMask = 0;
302 :
303 0 : if (Support::test(flags, RATiedFlags::kUse))
304 0 : useRewriteMask = Support::bitMask(inst->getRewriteIndex(&mem._data[Operand::kDataMemIndexId]));
305 : else
306 0 : outRewriteMask = Support::bitMask(inst->getRewriteIndex(&mem._data[Operand::kDataMemIndexId]));
307 :
308 0 : ASMJIT_PROPAGATE(ib.add(workReg, RATiedFlags::kUse | RATiedFlags::kRead, allocable, useId, useRewriteMask, allocable, outId, outRewriteMask));
309 : }
310 : }
311 : }
312 : }
313 : }
314 :
315 0 : controlType = getControlFlowType(instId);
316 : }
317 :
318 : return kErrorOk;
319 : }
320 :
321 : // a64::RACFGBuilder - OnInvoke
322 : // ============================
323 :
324 0 : Error RACFGBuilder::onBeforeInvoke(InvokeNode* invokeNode) noexcept {
325 : const FuncDetail& fd = invokeNode->detail();
326 : uint32_t argCount = invokeNode->argCount();
327 :
328 : cc()->_setCursor(invokeNode->prev());
329 :
330 0 : for (uint32_t argIndex = 0; argIndex < argCount; argIndex++) {
331 0 : const FuncValuePack& argPack = fd.argPack(argIndex);
332 0 : for (uint32_t valueIndex = 0; valueIndex < Globals::kMaxValuePack; valueIndex++) {
333 0 : if (!argPack[valueIndex])
334 : break;
335 :
336 : const FuncValue& arg = argPack[valueIndex];
337 : const Operand& op = invokeNode->arg(argIndex, valueIndex);
338 :
339 0 : if (op.isNone())
340 0 : continue;
341 :
342 0 : if (op.isReg()) {
343 : const Reg& reg = op.as<Reg>();
344 : RAWorkReg* workReg;
345 0 : ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(Operand::virtIdToIndex(reg.id()), &workReg));
346 :
347 0 : if (arg.isReg()) {
348 0 : RegGroup regGroup = workReg->group();
349 : RegGroup argGroup = Reg::groupOf(arg.regType());
350 :
351 0 : if (regGroup != argGroup) {
352 : // TODO: [ARM] Conversion is not supported.
353 : return DebugUtils::errored(kErrorInvalidAssignment);
354 : }
355 : }
356 : else {
357 0 : ASMJIT_PROPAGATE(moveRegToStackArg(invokeNode, arg, reg));
358 : }
359 : }
360 0 : else if (op.isImm()) {
361 0 : if (arg.isReg()) {
362 0 : BaseReg reg;
363 0 : ASMJIT_PROPAGATE(moveImmToRegArg(invokeNode, arg, op.as<Imm>(), ®));
364 0 : invokeNode->_args[argIndex][valueIndex] = reg;
365 : }
366 : else {
367 0 : ASMJIT_PROPAGATE(moveImmToStackArg(invokeNode, arg, op.as<Imm>()));
368 : }
369 : }
370 : }
371 : }
372 :
373 : cc()->_setCursor(invokeNode);
374 :
375 0 : if (fd.hasRet()) {
376 0 : for (uint32_t valueIndex = 0; valueIndex < Globals::kMaxValuePack; valueIndex++) {
377 0 : const FuncValue& ret = fd.ret(valueIndex);
378 0 : if (!ret)
379 : break;
380 :
381 : const Operand& op = invokeNode->ret(valueIndex);
382 0 : if (op.isReg()) {
383 : const Reg& reg = op.as<Reg>();
384 : RAWorkReg* workReg;
385 0 : ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(Operand::virtIdToIndex(reg.id()), &workReg));
386 :
387 0 : if (ret.isReg()) {
388 0 : RegGroup regGroup = workReg->group();
389 : RegGroup retGroup = Reg::groupOf(ret.regType());
390 :
391 0 : if (regGroup != retGroup) {
392 : // TODO: [ARM] Conversion is not supported.
393 : return DebugUtils::errored(kErrorInvalidAssignment);
394 : }
395 : }
396 : }
397 : }
398 : }
399 :
400 : // This block has function call(s).
401 0 : _curBlock->addFlags(RABlockFlags::kHasFuncCalls);
402 0 : _pass->func()->frame().addAttributes(FuncAttributes::kHasFuncCalls);
403 0 : _pass->func()->frame().updateCallStackSize(fd.argStackSize());
404 :
405 0 : return kErrorOk;
406 : }
407 :
408 0 : Error RACFGBuilder::onInvoke(InvokeNode* invokeNode, RAInstBuilder& ib) noexcept {
409 : uint32_t argCount = invokeNode->argCount();
410 : const FuncDetail& fd = invokeNode->detail();
411 :
412 0 : for (uint32_t argIndex = 0; argIndex < argCount; argIndex++) {
413 0 : const FuncValuePack& argPack = fd.argPack(argIndex);
414 0 : for (uint32_t valueIndex = 0; valueIndex < Globals::kMaxValuePack; valueIndex++) {
415 0 : if (!argPack[valueIndex])
416 0 : continue;
417 :
418 : const FuncValue& arg = argPack[valueIndex];
419 : const Operand& op = invokeNode->arg(argIndex, valueIndex);
420 :
421 0 : if (op.isNone())
422 0 : continue;
423 :
424 0 : if (op.isReg()) {
425 : const Reg& reg = op.as<Reg>();
426 : RAWorkReg* workReg;
427 0 : ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(Operand::virtIdToIndex(reg.id()), &workReg));
428 :
429 0 : if (arg.isIndirect()) {
430 0 : RegGroup regGroup = workReg->group();
431 0 : if (regGroup != RegGroup::kGp)
432 : return DebugUtils::errored(kErrorInvalidState);
433 0 : ASMJIT_PROPAGATE(ib.addCallArg(workReg, arg.regId()));
434 : }
435 0 : else if (arg.isReg()) {
436 0 : RegGroup regGroup = workReg->group();
437 : RegGroup argGroup = Reg::groupOf(arg.regType());
438 :
439 0 : if (regGroup == argGroup) {
440 0 : ASMJIT_PROPAGATE(ib.addCallArg(workReg, arg.regId()));
441 : }
442 : }
443 : }
444 : }
445 : }
446 :
447 0 : for (uint32_t retIndex = 0; retIndex < Globals::kMaxValuePack; retIndex++) {
448 0 : const FuncValue& ret = fd.ret(retIndex);
449 0 : if (!ret)
450 : break;
451 :
452 : const Operand& op = invokeNode->ret(retIndex);
453 0 : if (op.isReg()) {
454 : const Reg& reg = op.as<Reg>();
455 : RAWorkReg* workReg;
456 0 : ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(Operand::virtIdToIndex(reg.id()), &workReg));
457 :
458 0 : if (ret.isReg()) {
459 0 : RegGroup regGroup = workReg->group();
460 : RegGroup retGroup = Reg::groupOf(ret.regType());
461 :
462 0 : if (regGroup == retGroup) {
463 0 : ASMJIT_PROPAGATE(ib.addCallRet(workReg, ret.regId()));
464 : }
465 : }
466 : else {
467 : return DebugUtils::errored(kErrorInvalidAssignment);
468 : }
469 : }
470 : }
471 :
472 : // Setup clobbered registers.
473 0 : ib._clobbered[0] = Support::lsbMask<RegMask>(_pass->_physRegCount[RegGroup(0)]) & ~fd.preservedRegs(RegGroup(0));
474 0 : ib._clobbered[1] = Support::lsbMask<RegMask>(_pass->_physRegCount[RegGroup(1)]) & ~fd.preservedRegs(RegGroup(1));
475 0 : ib._clobbered[2] = Support::lsbMask<RegMask>(_pass->_physRegCount[RegGroup(2)]) & ~fd.preservedRegs(RegGroup(2));
476 0 : ib._clobbered[3] = Support::lsbMask<RegMask>(_pass->_physRegCount[RegGroup(3)]) & ~fd.preservedRegs(RegGroup(3));
477 :
478 0 : return kErrorOk;
479 : }
480 :
481 : // a64::RACFGBuilder - MoveImmToRegArg
482 : // ===================================
483 :
484 0 : Error RACFGBuilder::moveImmToRegArg(InvokeNode* invokeNode, const FuncValue& arg, const Imm& imm_, BaseReg* out) noexcept {
485 : DebugUtils::unused(invokeNode);
486 : ASMJIT_ASSERT(arg.isReg());
487 :
488 : Imm imm(imm_);
489 : TypeId typeId = TypeId::kVoid;
490 :
491 0 : switch (arg.typeId()) {
492 0 : case TypeId::kInt8 : typeId = TypeId::kUInt64; imm.signExtend8Bits(); break;
493 0 : case TypeId::kUInt8 : typeId = TypeId::kUInt64; imm.zeroExtend8Bits(); break;
494 0 : case TypeId::kInt16 : typeId = TypeId::kUInt64; imm.signExtend16Bits(); break;
495 0 : case TypeId::kUInt16: typeId = TypeId::kUInt64; imm.zeroExtend16Bits(); break;
496 0 : case TypeId::kInt32 : typeId = TypeId::kUInt64; imm.signExtend32Bits(); break;
497 0 : case TypeId::kUInt32: typeId = TypeId::kUInt64; imm.zeroExtend32Bits(); break;
498 : case TypeId::kInt64 : typeId = TypeId::kUInt64; break;
499 : case TypeId::kUInt64: typeId = TypeId::kUInt64; break;
500 :
501 : default:
502 : return DebugUtils::errored(kErrorInvalidAssignment);
503 : }
504 :
505 0 : ASMJIT_PROPAGATE(cc()->_newReg(out, typeId, nullptr));
506 : cc()->virtRegById(out->id())->setWeight(BaseRAPass::kCallArgWeight);
507 0 : return cc()->mov(out->as<Gp>(), imm);
508 : }
509 :
510 : // a64::RACFGBuilder - MoveImmToStackArg
511 : // =====================================
512 :
513 0 : Error RACFGBuilder::moveImmToStackArg(InvokeNode* invokeNode, const FuncValue& arg, const Imm& imm_) noexcept {
514 0 : BaseReg reg;
515 :
516 0 : ASMJIT_PROPAGATE(moveImmToRegArg(invokeNode, arg, imm_, ®));
517 0 : ASMJIT_PROPAGATE(moveRegToStackArg(invokeNode, arg, reg));
518 :
519 : return kErrorOk;
520 : }
521 :
522 : // a64::RACFGBuilder - MoveRegToStackArg
523 : // =====================================
524 :
525 0 : Error RACFGBuilder::moveRegToStackArg(InvokeNode* invokeNode, const FuncValue& arg, const BaseReg& reg) noexcept {
526 : DebugUtils::unused(invokeNode);
527 0 : Mem stackPtr = ptr(_pass->_sp.as<Gp>(), arg.stackOffset());
528 :
529 0 : if (reg.isGp())
530 0 : return cc()->str(reg.as<Gp>(), stackPtr);
531 :
532 0 : if (reg.isVec())
533 0 : return cc()->str(reg.as<Vec>(), stackPtr);
534 :
535 : return DebugUtils::errored(kErrorInvalidState);
536 : }
537 :
538 : // a64::RACFGBuilder - OnReg
539 : // =========================
540 :
541 0 : Error RACFGBuilder::onBeforeRet(FuncRetNode* funcRet) noexcept {
542 : DebugUtils::unused(funcRet);
543 0 : return kErrorOk;
544 : }
545 :
546 0 : Error RACFGBuilder::onRet(FuncRetNode* funcRet, RAInstBuilder& ib) noexcept {
547 0 : const FuncDetail& funcDetail = _pass->func()->detail();
548 : const Operand* opArray = funcRet->operands();
549 : uint32_t opCount = funcRet->opCount();
550 :
551 0 : for (uint32_t i = 0; i < opCount; i++) {
552 0 : const Operand& op = opArray[i];
553 0 : if (op.isNone()) continue;
554 :
555 : const FuncValue& ret = funcDetail.ret(i);
556 0 : if (ASMJIT_UNLIKELY(!ret.isReg()))
557 : return DebugUtils::errored(kErrorInvalidAssignment);
558 :
559 0 : if (op.isReg()) {
560 : // Register return value.
561 : const Reg& reg = op.as<Reg>();
562 : uint32_t vIndex = Operand::virtIdToIndex(reg.id());
563 :
564 0 : if (vIndex < Operand::kVirtIdCount) {
565 : RAWorkReg* workReg;
566 0 : ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(vIndex, &workReg));
567 :
568 0 : RegGroup group = workReg->group();
569 0 : RegMask allocable = _pass->_availableRegs[group];
570 0 : ASMJIT_PROPAGATE(ib.add(workReg, RATiedFlags::kUse | RATiedFlags::kRead, allocable, ret.regId(), 0, 0, BaseReg::kIdBad, 0));
571 : }
572 : }
573 : else {
574 : return DebugUtils::errored(kErrorInvalidAssignment);
575 : }
576 : }
577 :
578 : return kErrorOk;
579 : }
580 :
581 : // a64::ARMRAPass - Construction & Destruction
582 : // ===========================================
583 :
584 0 : ARMRAPass::ARMRAPass() noexcept
585 0 : : BaseRAPass() { _iEmitHelper = &_emitHelper; }
586 0 : ARMRAPass::~ARMRAPass() noexcept {}
587 :
588 : // a64::ARMRAPass - OnInit / OnDone
589 : // ================================
590 :
591 0 : void ARMRAPass::onInit() noexcept {
592 : Arch arch = cc()->arch();
593 :
594 0 : _emitHelper._emitter = _cb;
595 :
596 0 : _archTraits = &ArchTraits::byArch(arch);
597 : _physRegCount.set(RegGroup::kGp, 32);
598 : _physRegCount.set(RegGroup::kVec, 32);
599 : _physRegCount.set(RegGroup::kExtraVirt2, 0);
600 : _physRegCount.set(RegGroup::kExtraVirt3, 0);
601 0 : _buildPhysIndex();
602 :
603 0 : _availableRegCount = _physRegCount;
604 0 : _availableRegs[RegGroup::kGp] = Support::lsbMask<uint32_t>(_physRegCount.get(RegGroup::kGp));
605 0 : _availableRegs[RegGroup::kVec] = Support::lsbMask<uint32_t>(_physRegCount.get(RegGroup::kVec));
606 : _availableRegs[RegGroup::kExtraVirt3] = Support::lsbMask<uint32_t>(_physRegCount.get(RegGroup::kExtraVirt2));
607 0 : _availableRegs[RegGroup::kExtraVirt3] = Support::lsbMask<uint32_t>(_physRegCount.get(RegGroup::kExtraVirt3));
608 :
609 0 : _scratchRegIndexes[0] = uint8_t(27);
610 0 : _scratchRegIndexes[1] = uint8_t(28);
611 :
612 : // The architecture specific setup makes implicitly all registers available. So
613 : // make unavailable all registers that are special and cannot be used in general.
614 0 : bool hasFP = _func->frame().hasPreservedFP();
615 :
616 0 : if (hasFP)
617 : makeUnavailable(RegGroup::kGp, Gp::kIdFp);
618 :
619 : makeUnavailable(RegGroup::kGp, Gp::kIdSp);
620 : makeUnavailable(RegGroup::kGp, Gp::kIdOs); // OS-specific use, usually TLS.
621 :
622 0 : _sp = sp;
623 0 : _fp = x29;
624 0 : }
625 :
626 0 : void ARMRAPass::onDone() noexcept {}
627 :
628 : // a64::ARMRAPass - BuildCFG
629 : // =========================
630 :
631 0 : Error ARMRAPass::buildCFG() noexcept {
632 0 : return RACFGBuilder(this).run();
633 : }
634 :
635 : // a64::ARMRAPass - Rewrite
636 : // ========================
637 :
638 0 : ASMJIT_FAVOR_SPEED Error ARMRAPass::_rewrite(BaseNode* first, BaseNode* stop) noexcept {
639 : uint32_t virtCount = cc()->_vRegArray.size();
640 :
641 : BaseNode* node = first;
642 0 : while (node != stop) {
643 : BaseNode* next = node->next();
644 0 : if (node->isInst()) {
645 : InstNode* inst = node->as<InstNode>();
646 : RAInst* raInst = node->passData<RAInst>();
647 :
648 : Operand* operands = inst->operands();
649 : uint32_t opCount = inst->opCount();
650 :
651 : uint32_t i;
652 :
653 : // Rewrite virtual registers into physical registers.
654 0 : if (raInst) {
655 : // If the instruction contains pass data (raInst) then it was a subject
656 : // for register allocation and must be rewritten to use physical regs.
657 : RATiedReg* tiedRegs = raInst->tiedRegs();
658 : uint32_t tiedCount = raInst->tiedCount();
659 :
660 0 : for (i = 0; i < tiedCount; i++) {
661 0 : RATiedReg* tiedReg = &tiedRegs[i];
662 :
663 : Support::BitWordIterator<uint32_t> useIt(tiedReg->useRewriteMask());
664 : uint32_t useId = tiedReg->useId();
665 0 : while (useIt.hasNext())
666 : inst->rewriteIdAtIndex(useIt.next(), useId);
667 :
668 : Support::BitWordIterator<uint32_t> outIt(tiedReg->outRewriteMask());
669 : uint32_t outId = tiedReg->outId();
670 0 : while (outIt.hasNext())
671 : inst->rewriteIdAtIndex(outIt.next(), outId);
672 : }
673 :
674 : // This data is allocated by Zone passed to `runOnFunction()`, which
675 : // will be reset after the RA pass finishes. So reset this data to
676 : // prevent having a dead pointer after the RA pass is complete.
677 : node->resetPassData();
678 :
679 0 : if (ASMJIT_UNLIKELY(node->type() != NodeType::kInst)) {
680 : // FuncRet terminates the flow, it must either be removed if the exit
681 : // label is next to it (optimization) or patched to an architecture
682 : // dependent jump instruction that jumps to the function's exit before
683 : // the epilog.
684 0 : if (node->type() == NodeType::kFuncRet) {
685 : RABlock* block = raInst->block();
686 0 : if (!isNextTo(node, _func->exitNode())) {
687 : cc()->_setCursor(node->prev());
688 0 : ASMJIT_PROPAGATE(emitJump(_func->exitNode()->label()));
689 : }
690 :
691 : BaseNode* prev = node->prev();
692 0 : cc()->removeNode(node);
693 : block->setLast(prev);
694 : }
695 : }
696 : }
697 :
698 : // Rewrite stack slot addresses.
699 0 : for (i = 0; i < opCount; i++) {
700 0 : Operand& op = operands[i];
701 0 : if (op.isMem()) {
702 : BaseMem& mem = op.as<BaseMem>();
703 0 : if (mem.isRegHome()) {
704 : uint32_t virtIndex = Operand::virtIdToIndex(mem.baseId());
705 0 : if (ASMJIT_UNLIKELY(virtIndex >= virtCount))
706 : return DebugUtils::errored(kErrorInvalidVirtId);
707 :
708 : VirtReg* virtReg = cc()->virtRegByIndex(virtIndex);
709 : RAWorkReg* workReg = virtReg->workReg();
710 : ASMJIT_ASSERT(workReg != nullptr);
711 :
712 : RAStackSlot* slot = workReg->stackSlot();
713 : int32_t offset = slot->offset();
714 :
715 : mem._setBase(_sp.type(), slot->baseRegId());
716 : mem.clearRegHome();
717 : mem.addOffsetLo32(offset);
718 : }
719 : }
720 : }
721 :
722 : // Rewrite `loadAddressOf()` construct.
723 0 : if (inst->realId() == Inst::kIdAdr && inst->opCount() == 2 && inst->op(1).isMem()) {
724 : BaseMem mem = inst->op(1).as<BaseMem>();
725 : int64_t offset = mem.offset();
726 :
727 0 : if (!mem.hasBaseOrIndex()) {
728 : inst->setId(Inst::kIdMov);
729 0 : inst->setOp(1, Imm(offset));
730 : }
731 : else {
732 0 : if (mem.hasIndex())
733 0 : return DebugUtils::errored(kErrorInvalidAddressIndex);
734 :
735 : GpX dst(inst->op(0).as<Gp>().id());
736 : GpX base(mem.baseId());
737 :
738 0 : InstId arithInstId = offset < 0 ? Inst::kIdSub : Inst::kIdAdd;
739 0 : uint64_t absOffset = offset < 0 ? Support::neg(uint64_t(offset)) : uint64_t(offset);
740 :
741 : inst->setId(arithInstId);
742 : inst->setOpCount(3);
743 : inst->setOp(1, base);
744 0 : inst->setOp(2, Imm(absOffset));
745 :
746 : // Use two operations if the offset cannot be encoded with ADD/SUB.
747 0 : if (absOffset > 0xFFFu && (absOffset & ~uint64_t(0xFFF000u)) != 0) {
748 0 : if (absOffset <= 0xFFFFFFu) {
749 : cc()->_setCursor(inst->prev());
750 0 : ASMJIT_PROPAGATE(cc()->emit(arithInstId, dst, base, Imm(absOffset & 0xFFFu)));
751 :
752 : inst->setOp(1, dst);
753 0 : inst->setOp(2, Imm(absOffset & 0xFFF000u));
754 : }
755 : else {
756 : cc()->_setCursor(inst->prev());
757 0 : ASMJIT_PROPAGATE(cc()->emit(Inst::kIdMov, inst->op(0), Imm(absOffset)));
758 :
759 : inst->setOp(1, base);
760 : inst->setOp(2, dst);
761 : }
762 : }
763 : }
764 : }
765 : }
766 :
767 : node = next;
768 : }
769 :
770 : return kErrorOk;
771 : }
772 :
773 : // a64::ARMRAPass - Prolog & Epilog
774 : // ================================
775 :
776 0 : Error ARMRAPass::updateStackFrame() noexcept {
777 0 : if (_func->frame().hasFuncCalls())
778 : _func->frame().addDirtyRegs(RegGroup::kGp, Support::bitMask(Gp::kIdLr));
779 :
780 0 : return BaseRAPass::updateStackFrame();
781 : }
782 :
783 : // a64::ARMRAPass - OnEmit
784 : // =======================
785 :
786 0 : Error ARMRAPass::emitMove(uint32_t workId, uint32_t dstPhysId, uint32_t srcPhysId) noexcept {
787 : RAWorkReg* wReg = workRegById(workId);
788 : BaseReg dst(wReg->signature(), dstPhysId);
789 : BaseReg src(wReg->signature(), srcPhysId);
790 :
791 : const char* comment = nullptr;
792 :
793 : #ifndef ASMJIT_NO_LOGGING
794 0 : if (hasDiagnosticOption(DiagnosticOptions::kRAAnnotate)) {
795 0 : _tmpString.assignFormat("<MOVE> %s", workRegById(workId)->name());
796 : comment = _tmpString.data();
797 : }
798 : #endif
799 :
800 0 : return _emitHelper.emitRegMove(dst, src, wReg->typeId(), comment);
801 : }
802 :
803 0 : Error ARMRAPass::emitSwap(uint32_t aWorkId, uint32_t aPhysId, uint32_t bWorkId, uint32_t bPhysId) noexcept {
804 : DebugUtils::unused(aWorkId, aPhysId, bWorkId, bPhysId);
805 0 : return DebugUtils::errored(kErrorInvalidState);
806 : }
807 :
808 0 : Error ARMRAPass::emitLoad(uint32_t workId, uint32_t dstPhysId) noexcept {
809 : RAWorkReg* wReg = workRegById(workId);
810 : BaseReg dstReg(wReg->signature(), dstPhysId);
811 0 : BaseMem srcMem(workRegAsMem(wReg));
812 :
813 : const char* comment = nullptr;
814 :
815 : #ifndef ASMJIT_NO_LOGGING
816 0 : if (hasDiagnosticOption(DiagnosticOptions::kRAAnnotate)) {
817 0 : _tmpString.assignFormat("<LOAD> %s", workRegById(workId)->name());
818 : comment = _tmpString.data();
819 : }
820 : #endif
821 :
822 0 : return _emitHelper.emitRegMove(dstReg, srcMem, wReg->typeId(), comment);
823 : }
824 :
825 0 : Error ARMRAPass::emitSave(uint32_t workId, uint32_t srcPhysId) noexcept {
826 : RAWorkReg* wReg = workRegById(workId);
827 0 : BaseMem dstMem(workRegAsMem(wReg));
828 : BaseReg srcReg(wReg->signature(), srcPhysId);
829 :
830 : const char* comment = nullptr;
831 :
832 : #ifndef ASMJIT_NO_LOGGING
833 0 : if (hasDiagnosticOption(DiagnosticOptions::kRAAnnotate)) {
834 0 : _tmpString.assignFormat("<SAVE> %s", workRegById(workId)->name());
835 : comment = _tmpString.data();
836 : }
837 : #endif
838 :
839 0 : return _emitHelper.emitRegMove(dstMem, srcReg, wReg->typeId(), comment);
840 : }
841 :
842 0 : Error ARMRAPass::emitJump(const Label& label) noexcept {
843 0 : return cc()->b(label);
844 : }
845 :
846 0 : Error ARMRAPass::emitPreCall(InvokeNode* invokeNode) noexcept {
847 : DebugUtils::unused(invokeNode);
848 0 : return kErrorOk;
849 : }
850 :
851 : ASMJIT_END_SUB_NAMESPACE
852 :
853 : #endif // !ASMJIT_NO_AARCH64 && !ASMJIT_NO_COMPILER
854 :
855 : #endif // __PLUMED_HAS_ASMJIT
|
