Line data Source code
1 : #ifdef __PLUMED_HAS_ASMJIT
2 : // This file is part of AsmJit project <https://asmjit.com>
3 : //
4 : // See asmjit.h or LICENSE.md for license and copyright information
5 : // SPDX-License-Identifier: Zlib
6 :
7 : #include "api-build_p.h"
8 : #if !defined(ASMJIT_NO_X86) && !defined(ASMJIT_NO_COMPILER)
9 :
10 : #include "cpuinfo.h"
11 : #include "support.h"
12 : #include "type.h"
13 : #include "x86assembler.h"
14 : #include "x86compiler.h"
15 : #include "x86instapi_p.h"
16 : #include "x86instdb_p.h"
17 : #include "x86emithelper_p.h"
18 : #include "x86rapass_p.h"
19 :
20 : ASMJIT_BEGIN_SUB_NAMESPACE(x86)
21 :
22 : // x86::X86RAPass - Utilities
23 : // ==========================
24 :
25 : static ASMJIT_FORCE_INLINE uint64_t raImmMaskFromSize(uint32_t size) noexcept {
26 : ASMJIT_ASSERT(size > 0 && size < 256);
27 : static constexpr uint64_t masks[] = {
28 : 0x00000000000000FFu, // 1
29 : 0x000000000000FFFFu, // 2
30 : 0x00000000FFFFFFFFu, // 4
31 : 0xFFFFFFFFFFFFFFFFu, // 8
32 : 0x0000000000000000u, // 16
33 : 0x0000000000000000u, // 32
34 : 0x0000000000000000u, // 64
35 : 0x0000000000000000u, // 128
36 : 0x0000000000000000u // 256
37 : };
38 0 : return masks[Support::ctz(size)];
39 : }
40 :
41 : static const RegMask raConsecutiveLeadCountToRegMaskFilter[5] = {
42 : 0xFFFFFFFFu, // [0] No consecutive.
43 : 0x00000000u, // [1] Invalid, never used.
44 : 0x55555555u, // [2] Even registers.
45 : 0x00000000u, // [3] Invalid, never used.
46 : 0x11111111u // [4] Every fourth register.
47 : };
48 :
49 : static ASMJIT_FORCE_INLINE RATiedFlags raUseOutFlagsFromRWFlags(OpRWFlags rwFlags) noexcept {
50 : static constexpr RATiedFlags map[] = {
51 : RATiedFlags::kNone,
52 : RATiedFlags::kRead | RATiedFlags::kUse, // kRead
53 : RATiedFlags::kWrite | RATiedFlags::kOut, // kWrite
54 : RATiedFlags::kRW | RATiedFlags::kUse, // kRW
55 : RATiedFlags::kNone,
56 : RATiedFlags::kRead | RATiedFlags::kUse | RATiedFlags::kUseRM, // kRead | kRegMem
57 : RATiedFlags::kWrite | RATiedFlags::kOut | RATiedFlags::kOutRM, // kWrite | kRegMem
58 : RATiedFlags::kRW | RATiedFlags::kUse | RATiedFlags::kUseRM // kRW | kRegMem
59 : };
60 :
61 575407 : return map[uint32_t(rwFlags & (OpRWFlags::kRW | OpRWFlags::kRegMem))];
62 : }
63 :
64 : static ASMJIT_FORCE_INLINE RATiedFlags raRegRwFlags(OpRWFlags flags) noexcept {
65 : return (RATiedFlags)raUseOutFlagsFromRWFlags(flags);
66 : }
67 :
68 : static ASMJIT_FORCE_INLINE RATiedFlags raMemBaseRwFlags(OpRWFlags flags) noexcept {
69 : constexpr uint32_t kShift = Support::ConstCTZ<uint32_t(OpRWFlags::kMemBaseRW)>::value;
70 70974 : return (RATiedFlags)raUseOutFlagsFromRWFlags(OpRWFlags(uint32_t(flags) >> kShift) & OpRWFlags::kRW);
71 : }
72 :
73 : static ASMJIT_FORCE_INLINE RATiedFlags raMemIndexRwFlags(OpRWFlags flags) noexcept {
74 : constexpr uint32_t kShift = Support::ConstCTZ<uint32_t(OpRWFlags::kMemIndexRW)>::value;
75 0 : return (RATiedFlags)raUseOutFlagsFromRWFlags(OpRWFlags(uint32_t(flags) >> kShift) & OpRWFlags::kRW);
76 : }
77 :
78 : // x86::RACFGBuilder
79 : // =================
80 :
81 : class RACFGBuilder : public RACFGBuilderT<RACFGBuilder> {
82 : public:
83 : Arch _arch;
84 : bool _is64Bit;
85 : bool _avxEnabled;
86 :
87 27492 : inline RACFGBuilder(X86RAPass* pass) noexcept
88 27492 : : RACFGBuilderT<RACFGBuilder>(pass),
89 27492 : _arch(pass->cc()->arch()),
90 27492 : _is64Bit(pass->registerSize() == 8),
91 27492 : _avxEnabled(pass->avxEnabled()) {
92 27492 : }
93 :
94 46958 : inline Compiler* cc() const noexcept { return static_cast<Compiler*>(_cc); }
95 :
96 : inline uint32_t choose(uint32_t sseInst, uint32_t avxInst) const noexcept {
97 0 : return _avxEnabled ? avxInst : sseInst;
98 : }
99 :
100 : Error onInst(InstNode* inst, InstControlFlow& cf, RAInstBuilder& ib) noexcept;
101 :
102 : Error onBeforeInvoke(InvokeNode* invokeNode) noexcept;
103 : Error onInvoke(InvokeNode* invokeNode, RAInstBuilder& ib) noexcept;
104 :
105 : Error moveVecToPtr(InvokeNode* invokeNode, const FuncValue& arg, const Vec& src, BaseReg* out) noexcept;
106 : Error moveImmToRegArg(InvokeNode* invokeNode, const FuncValue& arg, const Imm& imm_, BaseReg* out) noexcept;
107 : Error moveImmToStackArg(InvokeNode* invokeNode, const FuncValue& arg, const Imm& imm_) noexcept;
108 : Error moveRegToStackArg(InvokeNode* invokeNode, const FuncValue& arg, const BaseReg& reg) noexcept;
109 :
110 : Error onBeforeRet(FuncRetNode* funcRet) noexcept;
111 : Error onRet(FuncRetNode* funcRet, RAInstBuilder& ib) noexcept;
112 : };
113 :
114 : // x86::RACFGBuilder - OnInst
115 : // ==========================
116 :
117 331808 : Error RACFGBuilder::onInst(InstNode* inst, InstControlFlow& cf, RAInstBuilder& ib) noexcept {
118 : InstRWInfo rwInfo;
119 :
120 : InstId instId = inst->id();
121 331808 : if (Inst::isDefinedId(instId)) {
122 : uint32_t opCount = inst->opCount();
123 : const Operand* opArray = inst->operands();
124 304316 : ASMJIT_PROPAGATE(InstInternal::queryRWInfo(_arch, inst->baseInst(), opArray, opCount, &rwInfo));
125 :
126 : const InstDB::InstInfo& instInfo = InstDB::infoById(instId);
127 : bool hasGpbHiConstraint = false;
128 : uint32_t singleRegOps = 0;
129 :
130 : // Copy instruction RW flags to instruction builder except kMovOp, which is propagated manually later.
131 : ib.addInstRWFlags(rwInfo.instFlags() & ~InstRWFlags::kMovOp);
132 :
133 : // Mask of all operand types used by the instruction - can be used as an optimization later.
134 : uint32_t opTypesMask = 0u;
135 :
136 304316 : if (opCount) {
137 : // The mask is for all registers, but we are mostly interested in AVX-512 registers at the moment. The mask
138 : // will be combined with all available registers of the Compiler at the end so we it never use more registers
139 : // than available.
140 : RegMask instructionAllowedRegs = 0xFFFFFFFFu;
141 :
142 : uint32_t consecutiveOffset = 0;
143 : uint32_t consecutiveLeadId = Globals::kInvalidId;
144 : uint32_t consecutiveParent = Globals::kInvalidId;
145 :
146 304316 : if (instInfo.isEvex()) {
147 : // EVEX instruction and VEX instructions that can be encoded with EVEX have the possibility to use 32 SIMD
148 : // registers (XMM/YMM/ZMM).
149 206278 : if (instInfo.isVex() && !instInfo.isEvexCompatible()) {
150 1970 : if (instInfo.isEvexKRegOnly()) {
151 : // EVEX encodable only if the first operand is K register (compare instructions).
152 1970 : if (!Reg::isKReg(opArray[0]))
153 : instructionAllowedRegs = 0xFFFFu;
154 : }
155 0 : else if (instInfo.isEvexTwoOpOnly()) {
156 : // EVEX encodable only if the instruction has two operands (gather instructions).
157 0 : if (opCount != 2)
158 : instructionAllowedRegs = 0xFFFFu;
159 : }
160 : else {
161 : instructionAllowedRegs = 0xFFFFu;
162 : }
163 : }
164 : }
165 98038 : else if (instInfo.isEvexTransformable()) {
166 : ib.addAggregatedFlags(RATiedFlags::kInst_IsTransformable);
167 : }
168 : else {
169 : // Not EVEX, restrict everything to [0-15] registers.
170 : instructionAllowedRegs = 0xFFFFu;
171 : }
172 :
173 1040909 : for (uint32_t i = 0; i < opCount; i++) {
174 736593 : const Operand& op = opArray[i];
175 : const OpRWInfo& opRwInfo = rwInfo.operand(i);
176 :
177 736593 : opTypesMask |= 1u << uint32_t(op.opType());
178 :
179 736593 : if (op.isReg()) {
180 : // Register Operand
181 : // ----------------
182 : const Reg& reg = op.as<Reg>();
183 :
184 : RATiedFlags flags = raRegRwFlags(opRwInfo.opFlags());
185 : RegMask allowedRegs = instructionAllowedRegs;
186 :
187 : // X86-specific constraints related to LO|HI general purpose registers. This is only required when the
188 : // register is part of the encoding. If the register is fixed we won't restrict anything as it doesn't
189 : // restrict encoding of other registers.
190 575407 : if (reg.isGpb() && !opRwInfo.hasOpFlag(OpRWFlags::kRegPhysId)) {
191 : flags |= RATiedFlags::kX86_Gpb;
192 0 : if (!_is64Bit) {
193 : // Restrict to first four - AL|AH|BL|BH|CL|CH|DL|DH. In 32-bit mode it's not possible to access
194 : // SIL|DIL, etc, so this is just enough.
195 : allowedRegs = 0x0Fu;
196 : }
197 : else {
198 : // If we encountered GPB-HI register the situation is much more complicated than in 32-bit mode.
199 : // We need to patch all registers to not use ID higher than 7 and all GPB-LO registers to not use
200 : // index higher than 3. Instead of doing the patching here we just set a flag and will do it later,
201 : // to not complicate this loop.
202 0 : if (reg.isGpbHi()) {
203 : hasGpbHiConstraint = true;
204 : allowedRegs = 0x0Fu;
205 : }
206 : }
207 : }
208 :
209 : uint32_t vIndex = Operand::virtIdToIndex(reg.id());
210 575407 : if (vIndex < Operand::kVirtIdCount) {
211 : RAWorkReg* workReg;
212 1150814 : ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(vIndex, &workReg));
213 :
214 : // Use RW instead of Write in case that not the whole register is overwritten. This is important
215 : // for liveness as we cannot kill a register that will be used. For example `mov al, 0xFF` is not
216 : // a write-only operation if user allocated the whole `rax` register.
217 575407 : if ((flags & RATiedFlags::kRW) == RATiedFlags::kWrite) {
218 294583 : if (workReg->regByteMask() & ~(opRwInfo.writeByteMask() | opRwInfo.extendByteMask())) {
219 : // Not write-only operation.
220 : flags = (flags & ~RATiedFlags::kOut) | (RATiedFlags::kRead | RATiedFlags::kUse);
221 : }
222 : }
223 :
224 : // Do not use RegMem flag if changing Reg to Mem requires additional CPU feature that may not be enabled.
225 575407 : if (rwInfo.rmFeature() && Support::test(flags, RATiedFlags::kUseRM | RATiedFlags::kOutRM)) {
226 : flags &= ~(RATiedFlags::kUseRM | RATiedFlags::kOutRM);
227 : }
228 :
229 575407 : RegGroup group = workReg->group();
230 575407 : RegMask useRegs = _pass->_availableRegs[group] & allowedRegs;
231 : RegMask outRegs = useRegs;
232 :
233 : uint32_t useId = BaseReg::kIdBad;
234 : uint32_t outId = BaseReg::kIdBad;
235 :
236 : uint32_t useRewriteMask = 0;
237 : uint32_t outRewriteMask = 0;
238 :
239 575407 : if (opRwInfo.consecutiveLeadCount()) {
240 : // There must be a single consecutive register lead, otherwise the RW data is invalid.
241 0 : if (consecutiveLeadId != Globals::kInvalidId)
242 : return DebugUtils::errored(kErrorInvalidState);
243 :
244 : // A consecutive lead register cannot be used as a consecutive +1/+2/+3 register, the registers must be distinct.
245 0 : if (RATiedReg::consecutiveDataFromFlags(flags) != 0)
246 : return DebugUtils::errored(kErrorNotConsecutiveRegs);
247 :
248 0 : flags |= RATiedFlags::kLeadConsecutive | RATiedReg::consecutiveDataToFlags(opRwInfo.consecutiveLeadCount() - 1);
249 : consecutiveLeadId = workReg->workId();
250 :
251 0 : RegMask filter = raConsecutiveLeadCountToRegMaskFilter[opRwInfo.consecutiveLeadCount()];
252 0 : if (Support::test(flags, RATiedFlags::kUse)) {
253 : flags |= RATiedFlags::kUseConsecutive;
254 0 : useRegs &= filter;
255 : }
256 : else {
257 : flags |= RATiedFlags::kOutConsecutive;
258 0 : outRegs &= filter;
259 : }
260 : }
261 :
262 : if (Support::test(flags, RATiedFlags::kUse)) {
263 280824 : useRewriteMask = Support::bitMask(inst->getRewriteIndex(®._baseId));
264 280824 : if (opRwInfo.hasOpFlag(OpRWFlags::kRegPhysId)) {
265 : useId = opRwInfo.physId();
266 : flags |= RATiedFlags::kUseFixed;
267 : }
268 280824 : else if (opRwInfo.hasOpFlag(OpRWFlags::kConsecutive)) {
269 0 : if (consecutiveLeadId == Globals::kInvalidId)
270 : return DebugUtils::errored(kErrorInvalidState);
271 :
272 0 : if (consecutiveLeadId == workReg->workId())
273 : return DebugUtils::errored(kErrorOverlappedRegs);
274 :
275 0 : flags |= RATiedFlags::kUseConsecutive | RATiedReg::consecutiveDataToFlags(++consecutiveOffset);
276 : }
277 : }
278 : else {
279 294583 : outRewriteMask = Support::bitMask(inst->getRewriteIndex(®._baseId));
280 294583 : if (opRwInfo.hasOpFlag(OpRWFlags::kRegPhysId)) {
281 : outId = opRwInfo.physId();
282 : flags |= RATiedFlags::kOutFixed;
283 : }
284 294583 : else if (opRwInfo.hasOpFlag(OpRWFlags::kConsecutive)) {
285 0 : if (consecutiveLeadId == Globals::kInvalidId)
286 : return DebugUtils::errored(kErrorInvalidState);
287 :
288 0 : if (consecutiveLeadId == workReg->workId())
289 : return DebugUtils::errored(kErrorOverlappedRegs);
290 :
291 0 : flags |= RATiedFlags::kOutConsecutive | RATiedReg::consecutiveDataToFlags(++consecutiveOffset);
292 : }
293 : }
294 :
295 575407 : ASMJIT_PROPAGATE(ib.add(workReg, flags, useRegs, useId, useRewriteMask, outRegs, outId, outRewriteMask, opRwInfo.rmSize(), consecutiveParent));
296 575407 : if (singleRegOps == i)
297 575313 : singleRegOps++;
298 :
299 575407 : if (Support::test(flags, RATiedFlags::kLeadConsecutive | RATiedFlags::kUseConsecutive | RATiedFlags::kOutConsecutive))
300 0 : consecutiveParent = workReg->workId();
301 : }
302 : }
303 161186 : else if (op.isMem()) {
304 : // Memory Operand
305 : // --------------
306 : const Mem& mem = op.as<Mem>();
307 : ib.addForbiddenFlags(RATiedFlags::kUseRM | RATiedFlags::kOutRM);
308 :
309 70974 : if (mem.isRegHome()) {
310 : RAWorkReg* workReg;
311 0 : ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(Operand::virtIdToIndex(mem.baseId()), &workReg));
312 0 : _pass->getOrCreateStackSlot(workReg);
313 : }
314 70974 : else if (mem.hasBaseReg()) {
315 : uint32_t vIndex = Operand::virtIdToIndex(mem.baseId());
316 70974 : if (vIndex < Operand::kVirtIdCount) {
317 : RAWorkReg* workReg;
318 141948 : ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(vIndex, &workReg));
319 :
320 : RATiedFlags flags = raMemBaseRwFlags(opRwInfo.opFlags());
321 70974 : RegGroup group = workReg->group();
322 70974 : RegMask inOutRegs = _pass->_availableRegs[group];
323 :
324 : uint32_t useId = BaseReg::kIdBad;
325 : uint32_t outId = BaseReg::kIdBad;
326 :
327 : uint32_t useRewriteMask = 0;
328 : uint32_t outRewriteMask = 0;
329 :
330 70974 : if (Support::test(flags, RATiedFlags::kUse)) {
331 70974 : useRewriteMask = Support::bitMask(inst->getRewriteIndex(&mem._baseId));
332 70974 : if (opRwInfo.hasOpFlag(OpRWFlags::kMemPhysId)) {
333 : useId = opRwInfo.physId();
334 : flags |= RATiedFlags::kUseFixed;
335 : }
336 : }
337 : else {
338 0 : outRewriteMask = Support::bitMask(inst->getRewriteIndex(&mem._baseId));
339 0 : if (opRwInfo.hasOpFlag(OpRWFlags::kMemPhysId)) {
340 : outId = opRwInfo.physId();
341 : flags |= RATiedFlags::kOutFixed;
342 : }
343 : }
344 :
345 70974 : ASMJIT_PROPAGATE(ib.add(workReg, flags, inOutRegs, useId, useRewriteMask, inOutRegs, outId, outRewriteMask));
346 : }
347 : }
348 :
349 70974 : if (mem.hasIndexReg()) {
350 : uint32_t vIndex = Operand::virtIdToIndex(mem.indexId());
351 0 : if (vIndex < Operand::kVirtIdCount) {
352 : RAWorkReg* workReg;
353 0 : ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(vIndex, &workReg));
354 :
355 : RATiedFlags flags = raMemIndexRwFlags(opRwInfo.opFlags());
356 0 : RegGroup group = workReg->group();
357 0 : RegMask inOutRegs = _pass->_availableRegs[group] & instructionAllowedRegs;
358 :
359 : // Index registers have never fixed id on X86/x64.
360 : const uint32_t useId = BaseReg::kIdBad;
361 : const uint32_t outId = BaseReg::kIdBad;
362 :
363 : uint32_t useRewriteMask = 0;
364 : uint32_t outRewriteMask = 0;
365 :
366 0 : if (Support::test(flags, RATiedFlags::kUse))
367 0 : useRewriteMask = Support::bitMask(inst->getRewriteIndex(&mem._data[Operand::kDataMemIndexId]));
368 : else
369 0 : outRewriteMask = Support::bitMask(inst->getRewriteIndex(&mem._data[Operand::kDataMemIndexId]));
370 :
371 0 : ASMJIT_PROPAGATE(ib.add(workReg, RATiedFlags::kUse | RATiedFlags::kRead, inOutRegs, useId, useRewriteMask, inOutRegs, outId, outRewriteMask));
372 : }
373 : }
374 : }
375 : }
376 : }
377 :
378 : // Handle extra operand (either REP {cx|ecx|rcx} or AVX-512 {k} selector).
379 304316 : if (inst->hasExtraReg()) {
380 : uint32_t vIndex = Operand::virtIdToIndex(inst->extraReg().id());
381 0 : if (vIndex < Operand::kVirtIdCount) {
382 : RAWorkReg* workReg;
383 0 : ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(vIndex, &workReg));
384 :
385 0 : RegGroup group = workReg->group();
386 0 : RegMask inOutRegs = _pass->_availableRegs[group];
387 : uint32_t rewriteMask = Support::bitMask(inst->getRewriteIndex(&inst->extraReg()._id));
388 :
389 0 : if (group == RegGroup::kX86_K) {
390 : // AVX-512 mask selector {k} register - read-only, allocable to any register except {k0}.
391 0 : ASMJIT_PROPAGATE(ib.add(workReg, RATiedFlags::kUse | RATiedFlags::kRead, inOutRegs, BaseReg::kIdBad, rewriteMask, inOutRegs, BaseReg::kIdBad, 0));
392 : singleRegOps = 0;
393 : }
394 : else {
395 : // REP {cx|ecx|rcx} register - read & write, allocable to {cx|ecx|rcx} only.
396 0 : ASMJIT_PROPAGATE(ib.add(workReg, RATiedFlags::kUse | RATiedFlags::kRW, inOutRegs, Gp::kIdCx, rewriteMask, inOutRegs, Gp::kIdBad, 0));
397 : }
398 : }
399 : else {
400 : RegGroup group = inst->extraReg().group();
401 0 : if (group == RegGroup::kX86_K && inst->extraReg().id() != 0)
402 : singleRegOps = 0;
403 : }
404 : }
405 :
406 : // If this instruction has move semantics then check whether it could be eliminated if all virtual registers
407 : // are allocated into the same register. Take into account the virtual size of the destination register as that's
408 : // more important than a physical register size in this case.
409 304316 : if (rwInfo.hasInstFlag(InstRWFlags::kMovOp) && !inst->hasExtraReg() && Support::bitTest(opTypesMask, uint32_t(OperandType::kReg))) {
410 : // AVX+ move instructions have 3 operand form - the first two operands must be the same to guarantee move semantics.
411 30807 : if (opCount == 2 || (opCount == 3 && opArray[0] == opArray[1])) {
412 : uint32_t vIndex = Operand::virtIdToIndex(opArray[0].as<Reg>().id());
413 0 : if (vIndex < Operand::kVirtIdCount) {
414 0 : const VirtReg* vReg = _cc->virtRegByIndex(vIndex);
415 : const OpRWInfo& opRwInfo = rwInfo.operand(0);
416 :
417 0 : uint64_t remainingByteMask = vReg->workReg()->regByteMask() & ~opRwInfo.writeByteMask();
418 0 : if (remainingByteMask == 0u || (remainingByteMask & opRwInfo.extendByteMask()) == 0)
419 : ib.addInstRWFlags(InstRWFlags::kMovOp);
420 : }
421 : }
422 : }
423 :
424 : // Handle X86 constraints.
425 304316 : if (hasGpbHiConstraint) {
426 0 : for (RATiedReg& tiedReg : ib) {
427 0 : RegMask filter = tiedReg.hasFlag(RATiedFlags::kX86_Gpb) ? 0x0Fu : 0xFFu;
428 0 : tiedReg._useRegMask &= filter;
429 0 : tiedReg._outRegMask &= filter;
430 : }
431 : }
432 :
433 304316 : if (ib.tiedRegCount() == 1) {
434 : // Handle special cases of some instructions where all operands share the same
435 : // register. In such case the single operand becomes read-only or write-only.
436 : InstSameRegHint sameRegHint = InstSameRegHint::kNone;
437 116405 : if (singleRegOps == opCount) {
438 : sameRegHint = instInfo.sameRegHint();
439 : }
440 88236 : else if (opCount == 2 && inst->op(1).isImm()) {
441 : // Handle some tricks used by X86 asm.
442 : const BaseReg& reg = inst->op(0).as<BaseReg>();
443 : const Imm& imm = inst->op(1).as<Imm>();
444 :
445 88236 : const RAWorkReg* workReg = _pass->workRegById(ib[0]->workId());
446 : uint32_t workRegSize = workReg->signature().size();
447 :
448 88236 : switch (inst->id()) {
449 0 : case Inst::kIdOr: {
450 : // Sets the value of the destination register to -1, previous content unused.
451 0 : if (reg.size() >= 4 || reg.size() >= workRegSize) {
452 0 : if (imm.value() == -1 || imm.valueAs<uint64_t>() == raImmMaskFromSize(reg.size()))
453 : sameRegHint = InstSameRegHint::kWO;
454 : }
455 : ASMJIT_FALLTHROUGH;
456 : }
457 :
458 : case Inst::kIdAdd:
459 : case Inst::kIdAnd:
460 : case Inst::kIdRol:
461 : case Inst::kIdRor:
462 : case Inst::kIdSar:
463 : case Inst::kIdShl:
464 : case Inst::kIdShr:
465 : case Inst::kIdSub:
466 : case Inst::kIdXor: {
467 : // Updates [E|R]FLAGS without changing the content.
468 0 : if (reg.size() != 4 || reg.size() >= workRegSize) {
469 0 : if (imm.value() == 0)
470 : sameRegHint = InstSameRegHint::kRO;
471 : }
472 : break;
473 : }
474 : }
475 : }
476 :
477 28169 : switch (sameRegHint) {
478 : case InstSameRegHint::kNone:
479 : break;
480 : case InstSameRegHint::kRO:
481 : ib[0]->makeReadOnly();
482 : break;
483 : case InstSameRegHint::kWO:
484 : ib[0]->makeWriteOnly();
485 : break;
486 : }
487 : }
488 :
489 304316 : cf = instInfo.controlFlow();
490 : }
491 :
492 : return kErrorOk;
493 : }
494 :
495 : // x86::RACFGBuilder - OnInvoke
496 : // ============================
497 :
498 9639 : Error RACFGBuilder::onBeforeInvoke(InvokeNode* invokeNode) noexcept {
499 : const FuncDetail& fd = invokeNode->detail();
500 : uint32_t argCount = invokeNode->argCount();
501 :
502 : cc()->_setCursor(invokeNode->prev());
503 : RegType nativeRegType = cc()->_gpSignature.regType();
504 :
505 19512 : for (uint32_t argIndex = 0; argIndex < argCount; argIndex++) {
506 9873 : const FuncValuePack& argPack = fd.argPack(argIndex);
507 19746 : for (uint32_t valueIndex = 0; valueIndex < Globals::kMaxValuePack; valueIndex++) {
508 19746 : if (!argPack[valueIndex])
509 : break;
510 :
511 : const FuncValue& arg = argPack[valueIndex];
512 : const Operand& op = invokeNode->arg(argIndex, valueIndex);
513 :
514 9873 : if (op.isNone())
515 0 : continue;
516 :
517 9873 : if (op.isReg()) {
518 : const Reg& reg = op.as<Reg>();
519 : RAWorkReg* workReg;
520 19370 : ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(Operand::virtIdToIndex(reg.id()), &workReg));
521 :
522 9685 : if (arg.isReg()) {
523 9685 : RegGroup regGroup = workReg->group();
524 : RegGroup argGroup = Reg::groupOf(arg.regType());
525 :
526 9685 : if (arg.isIndirect()) {
527 0 : if (reg.isGp()) {
528 0 : if (reg.type() != nativeRegType)
529 0 : return DebugUtils::errored(kErrorInvalidAssignment);
530 : // It's considered allocated if this is an indirect argument and the user used GP.
531 0 : continue;
532 : }
533 :
534 0 : BaseReg indirectReg;
535 0 : moveVecToPtr(invokeNode, arg, reg.as<Vec>(), &indirectReg);
536 0 : invokeNode->_args[argIndex][valueIndex] = indirectReg;
537 : }
538 : else {
539 9685 : if (regGroup != argGroup) {
540 : // TODO: Conversion is not supported.
541 : return DebugUtils::errored(kErrorInvalidAssignment);
542 : }
543 : }
544 : }
545 : else {
546 0 : if (arg.isIndirect()) {
547 0 : if (reg.isGp()) {
548 0 : if (reg.type() != nativeRegType)
549 0 : return DebugUtils::errored(kErrorInvalidAssignment);
550 :
551 0 : ASMJIT_PROPAGATE(moveRegToStackArg(invokeNode, arg, reg));
552 0 : continue;
553 0 : }
554 :
555 0 : BaseReg indirectReg;
556 0 : moveVecToPtr(invokeNode, arg, reg.as<Vec>(), &indirectReg);
557 0 : ASMJIT_PROPAGATE(moveRegToStackArg(invokeNode, arg, indirectReg));
558 : }
559 : else {
560 0 : ASMJIT_PROPAGATE(moveRegToStackArg(invokeNode, arg, reg));
561 : }
562 : }
563 : }
564 188 : else if (op.isImm()) {
565 188 : if (arg.isReg()) {
566 188 : BaseReg reg;
567 188 : ASMJIT_PROPAGATE(moveImmToRegArg(invokeNode, arg, op.as<Imm>(), ®));
568 188 : invokeNode->_args[argIndex][valueIndex] = reg;
569 : }
570 : else {
571 0 : ASMJIT_PROPAGATE(moveImmToStackArg(invokeNode, arg, op.as<Imm>()));
572 : }
573 : }
574 : }
575 : }
576 :
577 : cc()->_setCursor(invokeNode);
578 9639 : if (fd.hasFlag(CallConvFlags::kCalleePopsStack) && fd.argStackSize() != 0)
579 0 : ASMJIT_PROPAGATE(cc()->sub(cc()->zsp(), fd.argStackSize()));
580 :
581 9639 : if (fd.hasRet()) {
582 19278 : for (uint32_t valueIndex = 0; valueIndex < Globals::kMaxValuePack; valueIndex++) {
583 19278 : const FuncValue& ret = fd.ret(valueIndex);
584 19278 : if (!ret)
585 : break;
586 :
587 : const Operand& op = invokeNode->ret(valueIndex);
588 9639 : if (op.isReg()) {
589 : const Reg& reg = op.as<Reg>();
590 : RAWorkReg* workReg;
591 19278 : ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(Operand::virtIdToIndex(reg.id()), &workReg));
592 :
593 9639 : if (ret.isReg()) {
594 9639 : if (ret.regType() == RegType::kX86_St) {
595 0 : if (workReg->group() != RegGroup::kVec)
596 0 : return DebugUtils::errored(kErrorInvalidAssignment);
597 :
598 : Reg dst(workReg->signature(), workReg->virtId());
599 0 : Mem mem;
600 :
601 : TypeId typeId = TypeUtils::scalarOf(workReg->typeId());
602 0 : if (ret.hasTypeId())
603 : typeId = ret.typeId();
604 :
605 0 : switch (typeId) {
606 0 : case TypeId::kFloat32:
607 0 : ASMJIT_PROPAGATE(_pass->useTemporaryMem(mem, 4, 4));
608 : mem.setSize(4);
609 0 : ASMJIT_PROPAGATE(cc()->fstp(mem));
610 0 : ASMJIT_PROPAGATE(cc()->emit(choose(Inst::kIdMovss, Inst::kIdVmovss), dst.as<Xmm>(), mem));
611 : break;
612 :
613 0 : case TypeId::kFloat64:
614 0 : ASMJIT_PROPAGATE(_pass->useTemporaryMem(mem, 8, 4));
615 : mem.setSize(8);
616 0 : ASMJIT_PROPAGATE(cc()->fstp(mem));
617 0 : ASMJIT_PROPAGATE(cc()->emit(choose(Inst::kIdMovsd, Inst::kIdVmovsd), dst.as<Xmm>(), mem));
618 : break;
619 :
620 : default:
621 : return DebugUtils::errored(kErrorInvalidAssignment);
622 : }
623 : }
624 : else {
625 9639 : RegGroup regGroup = workReg->group();
626 : RegGroup retGroup = Reg::groupOf(ret.regType());
627 :
628 9639 : if (regGroup != retGroup) {
629 : // TODO: Conversion is not supported.
630 : return DebugUtils::errored(kErrorInvalidAssignment);
631 : }
632 : }
633 : }
634 : }
635 : }
636 : }
637 :
638 : // This block has function call(s).
639 9639 : _curBlock->addFlags(RABlockFlags::kHasFuncCalls);
640 9639 : _pass->func()->frame().addAttributes(FuncAttributes::kHasFuncCalls);
641 9639 : _pass->func()->frame().updateCallStackSize(fd.argStackSize());
642 :
643 9639 : return kErrorOk;
644 : }
645 :
646 9639 : Error RACFGBuilder::onInvoke(InvokeNode* invokeNode, RAInstBuilder& ib) noexcept {
647 : uint32_t argCount = invokeNode->argCount();
648 : const FuncDetail& fd = invokeNode->detail();
649 :
650 19512 : for (uint32_t argIndex = 0; argIndex < argCount; argIndex++) {
651 9873 : const FuncValuePack& argPack = fd.argPack(argIndex);
652 49365 : for (uint32_t valueIndex = 0; valueIndex < Globals::kMaxValuePack; valueIndex++) {
653 39492 : if (!argPack[valueIndex])
654 29619 : continue;
655 :
656 : const FuncValue& arg = argPack[valueIndex];
657 : const Operand& op = invokeNode->arg(argIndex, valueIndex);
658 :
659 9873 : if (op.isNone())
660 0 : continue;
661 :
662 9873 : if (op.isReg()) {
663 : const Reg& reg = op.as<Reg>();
664 : RAWorkReg* workReg;
665 19746 : ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(Operand::virtIdToIndex(reg.id()), &workReg));
666 :
667 9873 : if (arg.isIndirect()) {
668 0 : RegGroup regGroup = workReg->group();
669 0 : if (regGroup != RegGroup::kGp)
670 : return DebugUtils::errored(kErrorInvalidState);
671 0 : ASMJIT_PROPAGATE(ib.addCallArg(workReg, arg.regId()));
672 : }
673 9873 : else if (arg.isReg()) {
674 9873 : RegGroup regGroup = workReg->group();
675 : RegGroup argGroup = Reg::groupOf(arg.regType());
676 :
677 9873 : if (regGroup == argGroup) {
678 9873 : ASMJIT_PROPAGATE(ib.addCallArg(workReg, arg.regId()));
679 : }
680 : }
681 : }
682 : }
683 : }
684 :
685 19278 : for (uint32_t retIndex = 0; retIndex < Globals::kMaxValuePack; retIndex++) {
686 19278 : const FuncValue& ret = fd.ret(retIndex);
687 19278 : if (!ret)
688 : break;
689 :
690 : // Not handled here...
691 : const Operand& op = invokeNode->ret(retIndex);
692 9639 : if (ret.regType() == RegType::kX86_St)
693 0 : continue;
694 :
695 9639 : if (op.isReg()) {
696 : const Reg& reg = op.as<Reg>();
697 : RAWorkReg* workReg;
698 19278 : ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(Operand::virtIdToIndex(reg.id()), &workReg));
699 :
700 9639 : if (ret.isReg()) {
701 9639 : RegGroup regGroup = workReg->group();
702 : RegGroup retGroup = Reg::groupOf(ret.regType());
703 :
704 9639 : if (regGroup == retGroup) {
705 9639 : ASMJIT_PROPAGATE(ib.addCallRet(workReg, ret.regId()));
706 : }
707 : }
708 : else {
709 : return DebugUtils::errored(kErrorInvalidAssignment);
710 : }
711 : }
712 : }
713 :
714 : // Setup clobbered registers.
715 48195 : for (RegGroup group : RegGroupVirtValues{})
716 38556 : ib._clobbered[group] = Support::lsbMask<RegMask>(_pass->_physRegCount[group]) & ~fd.preservedRegs(group);
717 :
718 : return kErrorOk;
719 : }
720 :
721 : // x86::RACFGBuilder - MoveVecToPtr
722 : // ================================
723 :
724 : static inline OperandSignature x86VecRegSignatureBySize(uint32_t size) noexcept {
725 0 : return OperandSignature{size >= 64 ? uint32_t(Zmm::kSignature) :
726 0 : size >= 32 ? uint32_t(Ymm::kSignature) : uint32_t(Xmm::kSignature)};
727 : }
728 :
729 0 : Error RACFGBuilder::moveVecToPtr(InvokeNode* invokeNode, const FuncValue& arg, const Vec& src, BaseReg* out) noexcept {
730 : DebugUtils::unused(invokeNode);
731 : ASMJIT_ASSERT(arg.isReg());
732 :
733 : uint32_t argSize = TypeUtils::sizeOf(arg.typeId());
734 0 : if (argSize == 0)
735 : return DebugUtils::errored(kErrorInvalidState);
736 :
737 : if (argSize < 16)
738 : argSize = 16;
739 :
740 0 : uint32_t argStackOffset = Support::alignUp(invokeNode->detail()._argStackSize, argSize);
741 0 : _funcNode->frame().updateCallStackAlignment(argSize);
742 0 : invokeNode->detail()._argStackSize = argStackOffset + argSize;
743 :
744 : Vec vecReg(x86VecRegSignatureBySize(argSize), src.id());
745 0 : Mem vecPtr = ptr(_pass->_sp.as<Gp>(), int32_t(argStackOffset));
746 :
747 : uint32_t vMovInstId = choose(Inst::kIdMovaps, Inst::kIdVmovaps);
748 0 : if (argSize > 16)
749 : vMovInstId = Inst::kIdVmovaps;
750 :
751 0 : ASMJIT_PROPAGATE(cc()->_newReg(out, ArchTraits::byArch(cc()->arch()).regTypeToTypeId(cc()->_gpSignature.regType()), nullptr));
752 :
753 : VirtReg* vReg = cc()->virtRegById(out->id());
754 : vReg->setWeight(BaseRAPass::kCallArgWeight);
755 :
756 0 : ASMJIT_PROPAGATE(cc()->lea(out->as<Gp>(), vecPtr));
757 0 : ASMJIT_PROPAGATE(cc()->emit(vMovInstId, ptr(out->as<Gp>()), vecReg));
758 :
759 0 : if (arg.isStack()) {
760 0 : Mem stackPtr = ptr(_pass->_sp.as<Gp>(), arg.stackOffset());
761 0 : ASMJIT_PROPAGATE(cc()->mov(stackPtr, out->as<Gp>()));
762 : }
763 :
764 : return kErrorOk;
765 : }
766 :
767 : // x86::RACFGBuilder - MoveImmToRegArg
768 : // ===================================
769 :
770 188 : Error RACFGBuilder::moveImmToRegArg(InvokeNode* invokeNode, const FuncValue& arg, const Imm& imm_, BaseReg* out) noexcept {
771 : DebugUtils::unused(invokeNode);
772 : ASMJIT_ASSERT(arg.isReg());
773 :
774 : Imm imm(imm_);
775 : TypeId rTypeId = TypeId::kUInt32;
776 :
777 188 : switch (arg.typeId()) {
778 0 : case TypeId::kInt8: imm.signExtend8Bits(); goto MovU32;
779 0 : case TypeId::kUInt8: imm.zeroExtend8Bits(); goto MovU32;
780 0 : case TypeId::kInt16: imm.signExtend16Bits(); goto MovU32;
781 0 : case TypeId::kUInt16: imm.zeroExtend16Bits(); goto MovU32;
782 :
783 : case TypeId::kInt32:
784 : case TypeId::kUInt32:
785 0 : MovU32:
786 : imm.zeroExtend32Bits();
787 : break;
788 :
789 : case TypeId::kInt64:
790 : case TypeId::kUInt64:
791 : // Moving to GPD automatically zero extends in 64-bit mode.
792 : if (imm.isUInt32()) {
793 : imm.zeroExtend32Bits();
794 : break;
795 : }
796 :
797 : rTypeId = TypeId::kUInt64;
798 : break;
799 :
800 : default:
801 : return DebugUtils::errored(kErrorInvalidAssignment);
802 : }
803 :
804 188 : ASMJIT_PROPAGATE(cc()->_newReg(out, rTypeId, nullptr));
805 : cc()->virtRegById(out->id())->setWeight(BaseRAPass::kCallArgWeight);
806 :
807 188 : return cc()->mov(out->as<x86::Gp>(), imm);
808 : }
809 :
810 : // x86::RACFGBuilder - MoveImmToStackArg
811 : // =====================================
812 :
813 0 : Error RACFGBuilder::moveImmToStackArg(InvokeNode* invokeNode, const FuncValue& arg, const Imm& imm_) noexcept {
814 : DebugUtils::unused(invokeNode);
815 : ASMJIT_ASSERT(arg.isStack());
816 :
817 0 : Mem stackPtr = ptr(_pass->_sp.as<Gp>(), arg.stackOffset());
818 0 : Imm imm[2];
819 :
820 : stackPtr.setSize(4);
821 : imm[0] = imm_;
822 : uint32_t nMovs = 0;
823 :
824 : // One stack entry has the same size as the native register size. That means that if we want to move a 32-bit
825 : // integer on the stack in 64-bit mode, we need to extend it to a 64-bit integer first. In 32-bit mode, pushing
826 : // a 64-bit on stack is done in two steps by pushing low and high parts separately.
827 0 : switch (arg.typeId()) {
828 0 : case TypeId::kInt8: imm[0].signExtend8Bits(); goto MovU32;
829 0 : case TypeId::kUInt8: imm[0].zeroExtend8Bits(); goto MovU32;
830 0 : case TypeId::kInt16: imm[0].signExtend16Bits(); goto MovU32;
831 0 : case TypeId::kUInt16: imm[0].zeroExtend16Bits(); goto MovU32;
832 :
833 : case TypeId::kInt32:
834 : case TypeId::kUInt32:
835 : case TypeId::kFloat32:
836 0 : MovU32:
837 : imm[0].zeroExtend32Bits();
838 : nMovs = 1;
839 0 : break;
840 :
841 0 : case TypeId::kInt64:
842 : case TypeId::kUInt64:
843 : case TypeId::kFloat64:
844 : case TypeId::kMmx32:
845 : case TypeId::kMmx64:
846 0 : if (_is64Bit && imm[0].isInt32()) {
847 : stackPtr.setSize(8);
848 : nMovs = 1;
849 0 : break;
850 : }
851 :
852 : imm[1].setValue(imm[0].uint32Hi());
853 : imm[0].zeroExtend32Bits();
854 : nMovs = 2;
855 0 : break;
856 :
857 : default:
858 : return DebugUtils::errored(kErrorInvalidAssignment);
859 : }
860 :
861 0 : for (uint32_t i = 0; i < nMovs; i++) {
862 0 : ASMJIT_PROPAGATE(cc()->mov(stackPtr, imm[i]));
863 : stackPtr.addOffsetLo32(int32_t(stackPtr.size()));
864 : }
865 :
866 : return kErrorOk;
867 : }
868 :
869 : // x86::RACFGBuilder - MoveRegToStackArg
870 : // =====================================
871 :
872 0 : Error RACFGBuilder::moveRegToStackArg(InvokeNode* invokeNode, const FuncValue& arg, const BaseReg& reg) noexcept {
873 : DebugUtils::unused(invokeNode);
874 : ASMJIT_ASSERT(arg.isStack());
875 :
876 0 : Mem stackPtr = ptr(_pass->_sp.as<Gp>(), arg.stackOffset());
877 0 : Reg r0, r1;
878 :
879 : VirtReg* vr = cc()->virtRegById(reg.id());
880 : uint32_t registerSize = cc()->registerSize();
881 : InstId instId = 0;
882 :
883 : TypeId dstTypeId = arg.typeId();
884 : TypeId srcTypeId = vr->typeId();
885 :
886 0 : switch (dstTypeId) {
887 : case TypeId::kInt64:
888 : case TypeId::kUInt64:
889 : // Extend BYTE->QWORD (GP).
890 : if (TypeUtils::isGp8(srcTypeId)) {
891 : r1.setRegT<RegType::kX86_GpbLo>(reg.id());
892 :
893 0 : instId = (dstTypeId == TypeId::kInt64 && srcTypeId == TypeId::kInt8) ? Inst::kIdMovsx : Inst::kIdMovzx;
894 0 : goto ExtendMovGpXQ;
895 : }
896 :
897 : // Extend WORD->QWORD (GP).
898 : if (TypeUtils::isGp16(srcTypeId)) {
899 : r1.setRegT<RegType::kX86_Gpw>(reg.id());
900 :
901 0 : instId = (dstTypeId == TypeId::kInt64 && srcTypeId == TypeId::kInt16) ? Inst::kIdMovsx : Inst::kIdMovzx;
902 0 : goto ExtendMovGpXQ;
903 : }
904 :
905 : // Extend DWORD->QWORD (GP).
906 : if (TypeUtils::isGp32(srcTypeId)) {
907 : r1.setRegT<RegType::kX86_Gpd>(reg.id());
908 :
909 : instId = Inst::kIdMovsxd;
910 0 : if (dstTypeId == TypeId::kInt64 && srcTypeId == TypeId::kInt32)
911 0 : goto ExtendMovGpXQ;
912 : else
913 0 : goto ZeroExtendGpDQ;
914 : }
915 :
916 : // Move QWORD (GP).
917 0 : if (TypeUtils::isGp64(srcTypeId)) goto MovGpQ;
918 0 : if (TypeUtils::isMmx(srcTypeId)) goto MovMmQ;
919 0 : if (TypeUtils::isVec(srcTypeId)) goto MovXmmQ;
920 : break;
921 :
922 : case TypeId::kInt32:
923 : case TypeId::kUInt32:
924 : case TypeId::kInt16:
925 : case TypeId::kUInt16:
926 : // DWORD <- WORD (Zero|Sign Extend).
927 : if (TypeUtils::isGp16(srcTypeId)) {
928 0 : bool isDstSigned = dstTypeId == TypeId::kInt16 || dstTypeId == TypeId::kInt32;
929 0 : bool isSrcSigned = srcTypeId == TypeId::kInt8 || srcTypeId == TypeId::kInt16;
930 :
931 : r1.setRegT<RegType::kX86_Gpw>(reg.id());
932 0 : instId = isDstSigned && isSrcSigned ? Inst::kIdMovsx : Inst::kIdMovzx;
933 0 : goto ExtendMovGpD;
934 : }
935 :
936 : // DWORD <- BYTE (Zero|Sign Extend).
937 : if (TypeUtils::isGp8(srcTypeId)) {
938 0 : bool isDstSigned = dstTypeId == TypeId::kInt16 || dstTypeId == TypeId::kInt32;
939 0 : bool isSrcSigned = srcTypeId == TypeId::kInt8 || srcTypeId == TypeId::kInt16;
940 :
941 : r1.setRegT<RegType::kX86_GpbLo>(reg.id());
942 0 : instId = isDstSigned && isSrcSigned ? Inst::kIdMovsx : Inst::kIdMovzx;
943 0 : goto ExtendMovGpD;
944 : }
945 : ASMJIT_FALLTHROUGH;
946 :
947 : case TypeId::kInt8:
948 : case TypeId::kUInt8:
949 0 : if (TypeUtils::isInt(srcTypeId)) goto MovGpD;
950 0 : if (TypeUtils::isMmx(srcTypeId)) goto MovMmD;
951 0 : if (TypeUtils::isVec(srcTypeId)) goto MovXmmD;
952 : break;
953 :
954 : case TypeId::kMmx32:
955 : case TypeId::kMmx64:
956 : // Extend BYTE->QWORD (GP).
957 : if (TypeUtils::isGp8(srcTypeId)) {
958 : r1.setRegT<RegType::kX86_GpbLo>(reg.id());
959 :
960 : instId = Inst::kIdMovzx;
961 0 : goto ExtendMovGpXQ;
962 : }
963 :
964 : // Extend WORD->QWORD (GP).
965 : if (TypeUtils::isGp16(srcTypeId)) {
966 : r1.setRegT<RegType::kX86_Gpw>(reg.id());
967 :
968 : instId = Inst::kIdMovzx;
969 0 : goto ExtendMovGpXQ;
970 : }
971 :
972 0 : if (TypeUtils::isGp32(srcTypeId)) goto ExtendMovGpDQ;
973 0 : if (TypeUtils::isGp64(srcTypeId)) goto MovGpQ;
974 0 : if (TypeUtils::isMmx(srcTypeId)) goto MovMmQ;
975 0 : if (TypeUtils::isVec(srcTypeId)) goto MovXmmQ;
976 : break;
977 :
978 : case TypeId::kFloat32:
979 : case TypeId::kFloat32x1:
980 0 : if (TypeUtils::isVec(srcTypeId)) goto MovXmmD;
981 : break;
982 :
983 : case TypeId::kFloat64:
984 : case TypeId::kFloat64x1:
985 0 : if (TypeUtils::isVec(srcTypeId)) goto MovXmmQ;
986 : break;
987 :
988 : default:
989 0 : if (TypeUtils::isVec(dstTypeId) && reg.as<Reg>().isVec()) {
990 : stackPtr.setSize(TypeUtils::sizeOf(dstTypeId));
991 : uint32_t vMovInstId = choose(Inst::kIdMovaps, Inst::kIdVmovaps);
992 :
993 : if (TypeUtils::isVec128(dstTypeId))
994 : r0.setRegT<RegType::kX86_Xmm>(reg.id());
995 : else if (TypeUtils::isVec256(dstTypeId))
996 : r0.setRegT<RegType::kX86_Ymm>(reg.id());
997 : else if (TypeUtils::isVec512(dstTypeId))
998 : r0.setRegT<RegType::kX86_Zmm>(reg.id());
999 : else
1000 : break;
1001 :
1002 0 : return cc()->emit(vMovInstId, stackPtr, r0);
1003 : }
1004 : break;
1005 : }
1006 : return DebugUtils::errored(kErrorInvalidAssignment);
1007 :
1008 : // Extend+Move Gp.
1009 0 : ExtendMovGpD:
1010 : stackPtr.setSize(4);
1011 : r0.setRegT<RegType::kX86_Gpd>(reg.id());
1012 :
1013 0 : ASMJIT_PROPAGATE(cc()->emit(instId, r0, r1));
1014 0 : ASMJIT_PROPAGATE(cc()->emit(Inst::kIdMov, stackPtr, r0));
1015 : return kErrorOk;
1016 :
1017 0 : ExtendMovGpXQ:
1018 0 : if (registerSize == 8) {
1019 : stackPtr.setSize(8);
1020 : r0.setRegT<RegType::kX86_Gpq>(reg.id());
1021 :
1022 0 : ASMJIT_PROPAGATE(cc()->emit(instId, r0, r1));
1023 0 : ASMJIT_PROPAGATE(cc()->emit(Inst::kIdMov, stackPtr, r0));
1024 : }
1025 : else {
1026 : stackPtr.setSize(4);
1027 : r0.setRegT<RegType::kX86_Gpd>(reg.id());
1028 :
1029 0 : ASMJIT_PROPAGATE(cc()->emit(instId, r0, r1));
1030 :
1031 0 : ExtendMovGpDQ:
1032 0 : ASMJIT_PROPAGATE(cc()->emit(Inst::kIdMov, stackPtr, r0));
1033 : stackPtr.addOffsetLo32(4);
1034 0 : ASMJIT_PROPAGATE(cc()->emit(Inst::kIdAnd, stackPtr, 0));
1035 : }
1036 : return kErrorOk;
1037 :
1038 : ZeroExtendGpDQ:
1039 : stackPtr.setSize(4);
1040 : r0.setRegT<RegType::kX86_Gpd>(reg.id());
1041 0 : goto ExtendMovGpDQ;
1042 :
1043 : MovGpD:
1044 : stackPtr.setSize(4);
1045 : r0.setRegT<RegType::kX86_Gpd>(reg.id());
1046 0 : return cc()->emit(Inst::kIdMov, stackPtr, r0);
1047 :
1048 0 : MovGpQ:
1049 : stackPtr.setSize(8);
1050 : r0.setRegT<RegType::kX86_Gpq>(reg.id());
1051 0 : return cc()->emit(Inst::kIdMov, stackPtr, r0);
1052 :
1053 : MovMmD:
1054 : stackPtr.setSize(4);
1055 : r0.setRegT<RegType::kX86_Mm>(reg.id());
1056 0 : return cc()->emit(choose(Inst::kIdMovd, Inst::kIdVmovd), stackPtr, r0);
1057 :
1058 0 : MovMmQ:
1059 : stackPtr.setSize(8);
1060 : r0.setRegT<RegType::kX86_Mm>(reg.id());
1061 0 : return cc()->emit(choose(Inst::kIdMovq, Inst::kIdVmovq), stackPtr, r0);
1062 :
1063 0 : MovXmmD:
1064 : stackPtr.setSize(4);
1065 : r0.setRegT<RegType::kX86_Xmm>(reg.id());
1066 0 : return cc()->emit(choose(Inst::kIdMovss, Inst::kIdVmovss), stackPtr, r0);
1067 :
1068 0 : MovXmmQ:
1069 : stackPtr.setSize(8);
1070 : r0.setRegT<RegType::kX86_Xmm>(reg.id());
1071 0 : return cc()->emit(choose(Inst::kIdMovlps, Inst::kIdVmovlps), stackPtr, r0);
1072 : }
1073 :
1074 : // x86::RACFGBuilder - OnReg
1075 : // =========================
1076 :
1077 27492 : Error RACFGBuilder::onBeforeRet(FuncRetNode* funcRet) noexcept {
1078 27492 : const FuncDetail& funcDetail = _pass->func()->detail();
1079 : const Operand* opArray = funcRet->operands();
1080 : uint32_t opCount = funcRet->opCount();
1081 :
1082 : cc()->_setCursor(funcRet->prev());
1083 :
1084 54984 : for (uint32_t i = 0; i < opCount; i++) {
1085 27492 : const Operand& op = opArray[i];
1086 : const FuncValue& ret = funcDetail.ret(i);
1087 :
1088 27492 : if (!op.isReg())
1089 0 : continue;
1090 :
1091 27492 : if (ret.regType() == RegType::kX86_St) {
1092 : const Reg& reg = op.as<Reg>();
1093 : uint32_t vIndex = Operand::virtIdToIndex(reg.id());
1094 :
1095 0 : if (vIndex < Operand::kVirtIdCount) {
1096 : RAWorkReg* workReg;
1097 0 : ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(vIndex, &workReg));
1098 :
1099 0 : if (workReg->group() != RegGroup::kVec)
1100 : return DebugUtils::errored(kErrorInvalidAssignment);
1101 :
1102 : Reg src(workReg->signature(), workReg->virtId());
1103 0 : Mem mem;
1104 :
1105 : TypeId typeId = TypeUtils::scalarOf(workReg->typeId());
1106 0 : if (ret.hasTypeId())
1107 : typeId = ret.typeId();
1108 :
1109 0 : switch (typeId) {
1110 0 : case TypeId::kFloat32:
1111 0 : ASMJIT_PROPAGATE(_pass->useTemporaryMem(mem, 4, 4));
1112 : mem.setSize(4);
1113 0 : ASMJIT_PROPAGATE(cc()->emit(choose(Inst::kIdMovss, Inst::kIdVmovss), mem, src.as<Xmm>()));
1114 0 : ASMJIT_PROPAGATE(cc()->fld(mem));
1115 : break;
1116 :
1117 0 : case TypeId::kFloat64:
1118 0 : ASMJIT_PROPAGATE(_pass->useTemporaryMem(mem, 8, 4));
1119 : mem.setSize(8);
1120 0 : ASMJIT_PROPAGATE(cc()->emit(choose(Inst::kIdMovsd, Inst::kIdVmovsd), mem, src.as<Xmm>()));
1121 0 : ASMJIT_PROPAGATE(cc()->fld(mem));
1122 : break;
1123 :
1124 : default:
1125 : return DebugUtils::errored(kErrorInvalidAssignment);
1126 : }
1127 : }
1128 : }
1129 : }
1130 :
1131 : return kErrorOk;
1132 : }
1133 :
1134 27492 : Error RACFGBuilder::onRet(FuncRetNode* funcRet, RAInstBuilder& ib) noexcept {
1135 27492 : const FuncDetail& funcDetail = _pass->func()->detail();
1136 : const Operand* opArray = funcRet->operands();
1137 : uint32_t opCount = funcRet->opCount();
1138 :
1139 54984 : for (uint32_t i = 0; i < opCount; i++) {
1140 27492 : const Operand& op = opArray[i];
1141 27492 : if (op.isNone()) continue;
1142 :
1143 : const FuncValue& ret = funcDetail.ret(i);
1144 27492 : if (ASMJIT_UNLIKELY(!ret.isReg()))
1145 : return DebugUtils::errored(kErrorInvalidAssignment);
1146 :
1147 : // Not handled here...
1148 27492 : if (ret.regType() == RegType::kX86_St)
1149 0 : continue;
1150 :
1151 27492 : if (op.isReg()) {
1152 : // Register return value.
1153 : const Reg& reg = op.as<Reg>();
1154 : uint32_t vIndex = Operand::virtIdToIndex(reg.id());
1155 :
1156 27492 : if (vIndex < Operand::kVirtIdCount) {
1157 : RAWorkReg* workReg;
1158 54984 : ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(vIndex, &workReg));
1159 :
1160 27492 : RegGroup group = workReg->group();
1161 27492 : RegMask inOutRegs = _pass->_availableRegs[group];
1162 27492 : ASMJIT_PROPAGATE(ib.add(workReg, RATiedFlags::kUse | RATiedFlags::kRead, inOutRegs, ret.regId(), 0, inOutRegs, BaseReg::kIdBad, 0));
1163 : }
1164 : }
1165 : else {
1166 : return DebugUtils::errored(kErrorInvalidAssignment);
1167 : }
1168 : }
1169 :
1170 : return kErrorOk;
1171 : }
1172 :
1173 : // x86::X86RAPass - Construction & Destruction
1174 : // ===========================================
1175 :
1176 27492 : X86RAPass::X86RAPass() noexcept
1177 27492 : : BaseRAPass() { _iEmitHelper = &_emitHelper; }
1178 27492 : X86RAPass::~X86RAPass() noexcept {}
1179 :
1180 : // x86::X86RAPass - OnInit & OnDone
1181 : // ================================
1182 :
1183 27492 : void X86RAPass::onInit() noexcept {
1184 : Arch arch = cc()->arch();
1185 27492 : uint32_t baseRegCount = Environment::is32Bit(arch) ? 8u : 16u;
1186 : uint32_t simdRegCount = baseRegCount;
1187 :
1188 27492 : if (Environment::is64Bit(arch) && _func->frame().isAvx512Enabled())
1189 : simdRegCount = 32u;
1190 :
1191 27492 : bool avxEnabled = _func->frame().isAvxEnabled();
1192 : bool avx512Enabled = _func->frame().isAvx512Enabled();
1193 :
1194 27492 : _emitHelper._emitter = _cb;
1195 27492 : _emitHelper._avxEnabled = avxEnabled || avx512Enabled;
1196 27492 : _emitHelper._avx512Enabled = avx512Enabled;
1197 :
1198 27492 : _archTraits = &ArchTraits::byArch(arch);
1199 : _physRegCount.set(RegGroup::kGp, baseRegCount);
1200 : _physRegCount.set(RegGroup::kVec, simdRegCount);
1201 : _physRegCount.set(RegGroup::kX86_K, 8);
1202 : _physRegCount.set(RegGroup::kX86_MM, 8);
1203 27492 : _buildPhysIndex();
1204 :
1205 27492 : _availableRegCount = _physRegCount;
1206 27492 : _availableRegs[RegGroup::kGp] = Support::lsbMask<RegMask>(_physRegCount.get(RegGroup::kGp));
1207 27492 : _availableRegs[RegGroup::kVec] = Support::lsbMask<RegMask>(_physRegCount.get(RegGroup::kVec));
1208 27492 : _availableRegs[RegGroup::kX86_K] = Support::lsbMask<RegMask>(_physRegCount.get(RegGroup::kX86_K)) ^ 1u;
1209 27492 : _availableRegs[RegGroup::kX86_MM] = Support::lsbMask<RegMask>(_physRegCount.get(RegGroup::kX86_MM));
1210 :
1211 27492 : _scratchRegIndexes[0] = uint8_t(Gp::kIdCx);
1212 27492 : _scratchRegIndexes[1] = uint8_t(baseRegCount - 1);
1213 :
1214 : // The architecture specific setup makes implicitly all registers available. So
1215 : // make unavailable all registers that are special and cannot be used in general.
1216 : bool hasFP = _func->frame().hasPreservedFP();
1217 :
1218 : makeUnavailable(RegGroup::kGp, Gp::kIdSp); // ESP|RSP used as a stack-pointer (SP).
1219 27492 : if (hasFP) makeUnavailable(RegGroup::kGp, Gp::kIdBp); // EBP|RBP used as a frame-pointer (FP).
1220 :
1221 27492 : _sp = cc()->zsp();
1222 27492 : _fp = cc()->zbp();
1223 27492 : }
1224 :
1225 27492 : void X86RAPass::onDone() noexcept {}
1226 :
1227 : // x86::X86RAPass - BuildCFG
1228 : // =========================
1229 :
1230 27492 : Error X86RAPass::buildCFG() noexcept {
1231 27492 : return RACFGBuilder(this).run();
1232 : }
1233 :
1234 : // x86::X86RAPass - Rewrite
1235 : // ========================
1236 :
1237 0 : static InstId transformVexToEvex(InstId instId) {
1238 0 : switch (instId) {
1239 : case Inst::kIdVbroadcastf128: return Inst::kIdVbroadcastf32x4;
1240 0 : case Inst::kIdVbroadcasti128: return Inst::kIdVbroadcasti32x4;
1241 0 : case Inst::kIdVextractf128: return Inst::kIdVextractf32x4;
1242 0 : case Inst::kIdVextracti128: return Inst::kIdVextracti32x4;
1243 0 : case Inst::kIdVinsertf128: return Inst::kIdVinsertf32x4;
1244 0 : case Inst::kIdVinserti128: return Inst::kIdVinserti32x4;
1245 0 : case Inst::kIdVmovdqa: return Inst::kIdVmovdqa32;
1246 0 : case Inst::kIdVmovdqu: return Inst::kIdVmovdqu32;
1247 0 : case Inst::kIdVpand: return Inst::kIdVpandd;
1248 0 : case Inst::kIdVpandn: return Inst::kIdVpandnd;
1249 0 : case Inst::kIdVpor: return Inst::kIdVpord;
1250 0 : case Inst::kIdVpxor: return Inst::kIdVpxord;
1251 0 : case Inst::kIdVroundpd: return Inst::kIdVrndscalepd;
1252 0 : case Inst::kIdVroundps: return Inst::kIdVrndscaleps;
1253 0 : case Inst::kIdVroundsd: return Inst::kIdVrndscalesd;
1254 0 : case Inst::kIdVroundss: return Inst::kIdVrndscaless;
1255 :
1256 0 : default:
1257 : // This should never happen as only transformable instructions should go this path.
1258 : ASMJIT_ASSERT(false);
1259 0 : return 0;
1260 : }
1261 : }
1262 :
1263 27492 : ASMJIT_FAVOR_SPEED Error X86RAPass::_rewrite(BaseNode* first, BaseNode* stop) noexcept {
1264 : uint32_t virtCount = cc()->_vRegArray.size();
1265 :
1266 : BaseNode* node = first;
1267 529469 : while (node != stop) {
1268 : BaseNode* next = node->next();
1269 501977 : if (node->isInst()) {
1270 : InstNode* inst = node->as<InstNode>();
1271 : RAInst* raInst = node->passData<RAInst>();
1272 :
1273 : Operand* operands = inst->operands();
1274 : uint32_t opCount = inst->opCount();
1275 : uint32_t maxRegId = 0;
1276 :
1277 : uint32_t i;
1278 :
1279 : // Rewrite virtual registers into physical registers.
1280 419501 : if (raInst) {
1281 : // This data is allocated by Zone passed to `runOnFunction()`, which will be reset after the RA pass finishes.
1282 : // So reset this data to prevent having a dead pointer after the RA pass is complete.
1283 : node->resetPassData();
1284 :
1285 : // If the instruction contains pass data (raInst) then it was a subject for register allocation and must be
1286 : // rewritten to use physical regs.
1287 : RATiedReg* tiedRegs = raInst->tiedRegs();
1288 : uint32_t tiedCount = raInst->tiedCount();
1289 :
1290 946262 : for (i = 0; i < tiedCount; i++) {
1291 614454 : RATiedReg* tiedReg = &tiedRegs[i];
1292 :
1293 : Support::BitWordIterator<uint32_t> useIt(tiedReg->useRewriteMask());
1294 : uint32_t useId = tiedReg->useId();
1295 934197 : while (useIt.hasNext()) {
1296 : maxRegId = Support::max(maxRegId, useId);
1297 : inst->rewriteIdAtIndex(useIt.next(), useId);
1298 : }
1299 :
1300 : Support::BitWordIterator<uint32_t> outIt(tiedReg->outRewriteMask());
1301 : uint32_t outId = tiedReg->outId();
1302 918397 : while (outIt.hasNext()) {
1303 : maxRegId = Support::max(maxRegId, outId);
1304 : inst->rewriteIdAtIndex(outIt.next(), outId);
1305 : }
1306 : }
1307 :
1308 : // Transform VEX instruction to EVEX when necessary.
1309 331808 : if (raInst->isTransformable()) {
1310 6 : if (maxRegId > 15) {
1311 0 : inst->setId(transformVexToEvex(inst->id()));
1312 : }
1313 : }
1314 :
1315 : // Remove moves that do not do anything.
1316 : //
1317 : // Usually these moves are inserted during code generation and originally they used different registers. If RA
1318 : // allocated these into the same register such redundant mov would appear.
1319 331808 : if (raInst->hasInstRWFlag(InstRWFlags::kMovOp) && !inst->hasExtraReg()) {
1320 0 : if (inst->opCount() == 2) {
1321 0 : if (inst->op(0) == inst->op(1)) {
1322 0 : cc()->removeNode(node);
1323 0 : goto Next;
1324 : }
1325 : }
1326 : }
1327 :
1328 331808 : if (ASMJIT_UNLIKELY(node->type() != NodeType::kInst)) {
1329 : // FuncRet terminates the flow, it must either be removed if the exit label is next to it (optimization) or
1330 : // patched to an architecture dependent jump instruction that jumps to the function's exit before the epilog.
1331 37131 : if (node->type() == NodeType::kFuncRet) {
1332 : RABlock* block = raInst->block();
1333 27492 : if (!isNextTo(node, _func->exitNode())) {
1334 : cc()->_setCursor(node->prev());
1335 0 : ASMJIT_PROPAGATE(emitJump(_func->exitNode()->label()));
1336 : }
1337 :
1338 : BaseNode* prev = node->prev();
1339 27492 : cc()->removeNode(node);
1340 : block->setLast(prev);
1341 : }
1342 : }
1343 : }
1344 :
1345 : // Rewrite stack slot addresses.
1346 1303988 : for (i = 0; i < opCount; i++) {
1347 884487 : Operand& op = operands[i];
1348 884487 : if (op.isMem()) {
1349 : BaseMem& mem = op.as<BaseMem>();
1350 142105 : if (mem.isRegHome()) {
1351 : uint32_t virtIndex = Operand::virtIdToIndex(mem.baseId());
1352 71131 : if (ASMJIT_UNLIKELY(virtIndex >= virtCount))
1353 : return DebugUtils::errored(kErrorInvalidVirtId);
1354 :
1355 : VirtReg* virtReg = cc()->virtRegByIndex(virtIndex);
1356 : RAWorkReg* workReg = virtReg->workReg();
1357 : ASMJIT_ASSERT(workReg != nullptr);
1358 :
1359 : RAStackSlot* slot = workReg->stackSlot();
1360 : int32_t offset = slot->offset();
1361 :
1362 : mem._setBase(_sp.type(), slot->baseRegId());
1363 : mem.clearRegHome();
1364 : mem.addOffsetLo32(offset);
1365 : }
1366 : }
1367 : }
1368 : }
1369 :
1370 82476 : Next:
1371 : node = next;
1372 : }
1373 :
1374 : return kErrorOk;
1375 : }
1376 :
1377 : // x86::X86RAPass - OnEmit
1378 : // =======================
1379 :
1380 1785 : Error X86RAPass::emitMove(uint32_t workId, uint32_t dstPhysId, uint32_t srcPhysId) noexcept {
1381 : RAWorkReg* wReg = workRegById(workId);
1382 : BaseReg dst(wReg->signature(), dstPhysId);
1383 : BaseReg src(wReg->signature(), srcPhysId);
1384 :
1385 : const char* comment = nullptr;
1386 :
1387 : #ifndef ASMJIT_NO_LOGGING
1388 1785 : if (hasDiagnosticOption(DiagnosticOptions::kRAAnnotate)) {
1389 0 : _tmpString.assignFormat("<MOVE> %s", workRegById(workId)->name());
1390 : comment = _tmpString.data();
1391 : }
1392 : #endif
1393 :
1394 1785 : return _emitHelper.emitRegMove(dst, src, wReg->typeId(), comment);
1395 : }
1396 :
1397 0 : Error X86RAPass::emitSwap(uint32_t aWorkId, uint32_t aPhysId, uint32_t bWorkId, uint32_t bPhysId) noexcept {
1398 : RAWorkReg* waReg = workRegById(aWorkId);
1399 : RAWorkReg* wbReg = workRegById(bWorkId);
1400 :
1401 : bool is64Bit = Support::max(waReg->typeId(), wbReg->typeId()) >= TypeId::kInt64;
1402 0 : OperandSignature sign = is64Bit ? OperandSignature{RegTraits<RegType::kX86_Gpq>::kSignature}
1403 : : OperandSignature{RegTraits<RegType::kX86_Gpd>::kSignature};
1404 :
1405 : #ifndef ASMJIT_NO_LOGGING
1406 0 : if (hasDiagnosticOption(DiagnosticOptions::kRAAnnotate)) {
1407 0 : _tmpString.assignFormat("<SWAP> %s, %s", waReg->name(), wbReg->name());
1408 : cc()->setInlineComment(_tmpString.data());
1409 : }
1410 : #endif
1411 :
1412 0 : return cc()->emit(Inst::kIdXchg, Reg(sign, aPhysId), Reg(sign, bPhysId));
1413 : }
1414 :
1415 17638 : Error X86RAPass::emitLoad(uint32_t workId, uint32_t dstPhysId) noexcept {
1416 : RAWorkReg* wReg = workRegById(workId);
1417 : BaseReg dstReg(wReg->signature(), dstPhysId);
1418 17638 : BaseMem srcMem(workRegAsMem(wReg));
1419 :
1420 : const char* comment = nullptr;
1421 :
1422 : #ifndef ASMJIT_NO_LOGGING
1423 17638 : if (hasDiagnosticOption(DiagnosticOptions::kRAAnnotate)) {
1424 0 : _tmpString.assignFormat("<LOAD> %s", workRegById(workId)->name());
1425 : comment = _tmpString.data();
1426 : }
1427 : #endif
1428 :
1429 17638 : return _emitHelper.emitRegMove(dstReg, srcMem, wReg->typeId(), comment);
1430 : }
1431 :
1432 30798 : Error X86RAPass::emitSave(uint32_t workId, uint32_t srcPhysId) noexcept {
1433 : RAWorkReg* wReg = workRegById(workId);
1434 30798 : BaseMem dstMem(workRegAsMem(wReg));
1435 : BaseReg srcReg(wReg->signature(), srcPhysId);
1436 :
1437 : const char* comment = nullptr;
1438 :
1439 : #ifndef ASMJIT_NO_LOGGING
1440 30798 : if (hasDiagnosticOption(DiagnosticOptions::kRAAnnotate)) {
1441 0 : _tmpString.assignFormat("<SAVE> %s", workRegById(workId)->name());
1442 : comment = _tmpString.data();
1443 : }
1444 : #endif
1445 :
1446 30798 : return _emitHelper.emitRegMove(dstMem, srcReg, wReg->typeId(), comment);
1447 : }
1448 :
1449 0 : Error X86RAPass::emitJump(const Label& label) noexcept {
1450 0 : return cc()->jmp(label);
1451 : }
1452 :
1453 9639 : Error X86RAPass::emitPreCall(InvokeNode* invokeNode) noexcept {
1454 9639 : if (invokeNode->detail().hasVarArgs() && cc()->is64Bit()) {
1455 : const FuncDetail& fd = invokeNode->detail();
1456 : uint32_t argCount = invokeNode->argCount();
1457 :
1458 0 : switch (invokeNode->detail().callConv().id()) {
1459 : case CallConvId::kX64SystemV: {
1460 : // AL register contains the number of arguments passed in XMM register(s).
1461 : uint32_t n = 0;
1462 0 : for (uint32_t argIndex = 0; argIndex < argCount; argIndex++) {
1463 0 : const FuncValuePack& argPack = fd.argPack(argIndex);
1464 0 : for (uint32_t valueIndex = 0; valueIndex < Globals::kMaxValuePack; valueIndex++) {
1465 0 : const FuncValue& arg = argPack[valueIndex];
1466 0 : if (!arg)
1467 : break;
1468 :
1469 0 : if (arg.isReg() && Reg::groupOf(arg.regType()) == RegGroup::kVec)
1470 0 : n++;
1471 : }
1472 : }
1473 :
1474 0 : if (!n)
1475 0 : ASMJIT_PROPAGATE(cc()->xor_(eax, eax));
1476 : else
1477 0 : ASMJIT_PROPAGATE(cc()->mov(eax, n));
1478 : break;
1479 : }
1480 :
1481 : case CallConvId::kX64Windows: {
1482 : // Each double-precision argument passed in XMM must be also passed in GP.
1483 0 : for (uint32_t argIndex = 0; argIndex < argCount; argIndex++) {
1484 0 : const FuncValuePack& argPack = fd.argPack(argIndex);
1485 0 : for (uint32_t valueIndex = 0; valueIndex < Globals::kMaxValuePack; valueIndex++) {
1486 0 : const FuncValue& arg = argPack[valueIndex];
1487 0 : if (!arg)
1488 : break;
1489 :
1490 0 : if (arg.isReg() && Reg::groupOf(arg.regType()) == RegGroup::kVec) {
1491 0 : Gp dst = gpq(fd.callConv().passedOrder(RegGroup::kGp)[argIndex]);
1492 : Xmm src = xmm(arg.regId());
1493 0 : ASMJIT_PROPAGATE(cc()->emit(choose(Inst::kIdMovq, Inst::kIdVmovq), dst, src));
1494 : }
1495 : }
1496 : }
1497 : break;
1498 : }
1499 :
1500 : default:
1501 : return DebugUtils::errored(kErrorInvalidState);
1502 : }
1503 : }
1504 :
1505 : return kErrorOk;
1506 : }
1507 :
1508 : ASMJIT_END_SUB_NAMESPACE
1509 :
1510 : #endif // !ASMJIT_NO_X86 && !ASMJIT_NO_COMPILER
1511 :
1512 : #endif // __PLUMED_HAS_ASMJIT
|
