Line data Source code
1 : #ifdef __PLUMED_HAS_ASMJIT
2 : // This file is part of AsmJit project <https://asmjit.com>
3 : //
4 : // See asmjit.h or LICENSE.md for license and copyright information
5 : // SPDX-License-Identifier: Zlib
6 :
7 : #include "api-build_p.h"
8 : #if !defined(ASMJIT_NO_X86)
9 :
10 : #include "x86func_p.h"
11 : #include "x86emithelper_p.h"
12 : #include "x86operand.h"
13 :
14 : ASMJIT_BEGIN_SUB_NAMESPACE(x86)
15 :
16 : namespace FuncInternal {
17 :
18 37131 : static inline bool shouldThreatAsCDeclIn64BitMode(CallConvId ccId) noexcept {
19 37131 : return ccId == CallConvId::kCDecl ||
20 37131 : ccId == CallConvId::kStdCall ||
21 37131 : ccId == CallConvId::kThisCall ||
22 : ccId == CallConvId::kFastCall ||
23 : ccId == CallConvId::kRegParm1 ||
24 : ccId == CallConvId::kRegParm2 ||
25 37131 : ccId == CallConvId::kRegParm3;
26 : }
27 :
28 37131 : ASMJIT_FAVOR_SIZE Error initCallConv(CallConv& cc, CallConvId ccId, const Environment& environment) noexcept {
29 : constexpr uint32_t kZax = Gp::kIdAx;
30 : constexpr uint32_t kZbx = Gp::kIdBx;
31 : constexpr uint32_t kZcx = Gp::kIdCx;
32 : constexpr uint32_t kZdx = Gp::kIdDx;
33 : constexpr uint32_t kZsp = Gp::kIdSp;
34 : constexpr uint32_t kZbp = Gp::kIdBp;
35 : constexpr uint32_t kZsi = Gp::kIdSi;
36 : constexpr uint32_t kZdi = Gp::kIdDi;
37 :
38 37131 : bool winABI = environment.isPlatformWindows() || environment.isMSVC();
39 :
40 : cc.setArch(environment.arch());
41 : cc.setSaveRestoreRegSize(RegGroup::kVec, 16);
42 : cc.setSaveRestoreRegSize(RegGroup::kX86_MM, 8);
43 : cc.setSaveRestoreRegSize(RegGroup::kX86_K, 8);
44 : cc.setSaveRestoreAlignment(RegGroup::kVec, 16);
45 : cc.setSaveRestoreAlignment(RegGroup::kX86_MM, 8);
46 : cc.setSaveRestoreAlignment(RegGroup::kX86_K, 8);
47 :
48 37131 : if (environment.is32Bit()) {
49 : bool isStandardCallConv = true;
50 :
51 : cc.setSaveRestoreRegSize(RegGroup::kGp, 4);
52 : cc.setSaveRestoreAlignment(RegGroup::kGp, 4);
53 :
54 : cc.setPreservedRegs(RegGroup::kGp, Support::bitMask(Gp::kIdBx, Gp::kIdSp, Gp::kIdBp, Gp::kIdSi, Gp::kIdDi));
55 : cc.setNaturalStackAlignment(4);
56 :
57 0 : switch (ccId) {
58 : case CallConvId::kCDecl:
59 : break;
60 :
61 : case CallConvId::kStdCall:
62 : cc.setFlags(CallConvFlags::kCalleePopsStack);
63 : break;
64 :
65 : case CallConvId::kFastCall:
66 : cc.setFlags(CallConvFlags::kCalleePopsStack);
67 : cc.setPassedOrder(RegGroup::kGp, kZcx, kZdx);
68 : break;
69 :
70 : case CallConvId::kVectorCall:
71 : cc.setFlags(CallConvFlags::kCalleePopsStack);
72 : cc.setPassedOrder(RegGroup::kGp, kZcx, kZdx);
73 : cc.setPassedOrder(RegGroup::kVec, 0, 1, 2, 3, 4, 5);
74 : break;
75 :
76 0 : case CallConvId::kThisCall:
77 : // NOTE: Even MINGW (starting with GCC 4.7.0) now uses __thiscall on MS Windows, so we won't bail to any
78 : // other calling convention if __thiscall was specified.
79 0 : if (winABI) {
80 : cc.setFlags(CallConvFlags::kCalleePopsStack);
81 : cc.setPassedOrder(RegGroup::kGp, kZcx);
82 : }
83 : else {
84 : ccId = CallConvId::kCDecl;
85 : }
86 : break;
87 :
88 : case CallConvId::kRegParm1:
89 : cc.setPassedOrder(RegGroup::kGp, kZax);
90 : break;
91 :
92 : case CallConvId::kRegParm2:
93 : cc.setPassedOrder(RegGroup::kGp, kZax, kZdx);
94 : break;
95 :
96 : case CallConvId::kRegParm3:
97 : cc.setPassedOrder(RegGroup::kGp, kZax, kZdx, kZcx);
98 : break;
99 :
100 0 : case CallConvId::kLightCall2:
101 : case CallConvId::kLightCall3:
102 : case CallConvId::kLightCall4: {
103 0 : uint32_t n = uint32_t(ccId) - uint32_t(CallConvId::kLightCall2) + 2;
104 :
105 : cc.setFlags(CallConvFlags::kPassFloatsByVec);
106 : cc.setPassedOrder(RegGroup::kGp, kZax, kZdx, kZcx, kZsi, kZdi);
107 : cc.setPassedOrder(RegGroup::kVec, 0, 1, 2, 3, 4, 5, 6, 7);
108 : cc.setPassedOrder(RegGroup::kX86_K, 0, 1, 2, 3, 4, 5, 6, 7);
109 : cc.setPassedOrder(RegGroup::kX86_MM, 0, 1, 2, 3, 4, 5, 6, 7);
110 : cc.setPreservedRegs(RegGroup::kGp, Support::lsbMask<uint32_t>(8));
111 0 : cc.setPreservedRegs(RegGroup::kVec, Support::lsbMask<uint32_t>(8) & ~Support::lsbMask<uint32_t>(n));
112 :
113 : cc.setNaturalStackAlignment(16);
114 : isStandardCallConv = false;
115 : break;
116 : }
117 :
118 : default:
119 : return DebugUtils::errored(kErrorInvalidArgument);
120 : }
121 :
122 : if (isStandardCallConv) {
123 : // MMX arguments is something where compiler vendors disagree. For example GCC and MSVC would pass first three
124 : // via registers and the rest via stack, however Clang passes all via stack. Returning MMX registers is even
125 : // more fun, where GCC uses MM0, but Clang uses EAX:EDX pair. I'm not sure it's something we should be worried
126 : // about as MMX is deprecated anyway.
127 : cc.setPassedOrder(RegGroup::kX86_MM, 0, 1, 2);
128 :
129 : // Vector arguments (XMM|YMM|ZMM) are passed via registers. However, if the function is variadic then they have
130 : // to be passed via stack.
131 : cc.setPassedOrder(RegGroup::kVec, 0, 1, 2);
132 :
133 : // Functions with variable arguments always use stack for MM and vector arguments.
134 : cc.addFlags(CallConvFlags::kPassVecByStackIfVA);
135 : }
136 :
137 0 : if (ccId == CallConvId::kCDecl) {
138 : cc.addFlags(CallConvFlags::kVarArgCompatible);
139 : }
140 : }
141 : else {
142 : cc.setSaveRestoreRegSize(RegGroup::kGp, 8);
143 : cc.setSaveRestoreAlignment(RegGroup::kGp, 8);
144 :
145 : // Preprocess the calling convention into a common id as many conventions are normally ignored even by C/C++
146 : // compilers and treated as `__cdecl`.
147 37131 : if (shouldThreatAsCDeclIn64BitMode(ccId))
148 37131 : ccId = winABI ? CallConvId::kX64Windows : CallConvId::kX64SystemV;
149 :
150 0 : switch (ccId) {
151 : case CallConvId::kX64SystemV: {
152 : cc.setFlags(CallConvFlags::kPassFloatsByVec |
153 : CallConvFlags::kPassMmxByXmm |
154 : CallConvFlags::kVarArgCompatible);
155 : cc.setNaturalStackAlignment(16);
156 : cc.setRedZoneSize(128);
157 : cc.setPassedOrder(RegGroup::kGp, kZdi, kZsi, kZdx, kZcx, 8, 9);
158 : cc.setPassedOrder(RegGroup::kVec, 0, 1, 2, 3, 4, 5, 6, 7);
159 : cc.setPreservedRegs(RegGroup::kGp, Support::bitMask(kZbx, kZsp, kZbp, 12, 13, 14, 15));
160 : break;
161 : }
162 :
163 : case CallConvId::kX64Windows: {
164 : cc.setStrategy(CallConvStrategy::kX64Windows);
165 : cc.setFlags(CallConvFlags::kPassFloatsByVec |
166 : CallConvFlags::kIndirectVecArgs |
167 : CallConvFlags::kPassMmxByGp |
168 : CallConvFlags::kVarArgCompatible);
169 : cc.setNaturalStackAlignment(16);
170 : // Maximum 4 arguments in registers, each adds 8 bytes to the spill zone.
171 : cc.setSpillZoneSize(4 * 8);
172 : cc.setPassedOrder(RegGroup::kGp, kZcx, kZdx, 8, 9);
173 : cc.setPassedOrder(RegGroup::kVec, 0, 1, 2, 3);
174 : cc.setPreservedRegs(RegGroup::kGp, Support::bitMask(kZbx, kZsp, kZbp, kZsi, kZdi, 12, 13, 14, 15));
175 : cc.setPreservedRegs(RegGroup::kVec, Support::bitMask(6, 7, 8, 9, 10, 11, 12, 13, 14, 15));
176 : break;
177 : }
178 :
179 : case CallConvId::kVectorCall: {
180 : cc.setStrategy(CallConvStrategy::kX64VectorCall);
181 : cc.setFlags(CallConvFlags::kPassFloatsByVec |
182 : CallConvFlags::kPassMmxByGp );
183 : cc.setNaturalStackAlignment(16);
184 : // Maximum 6 arguments in registers, each adds 8 bytes to the spill zone.
185 : cc.setSpillZoneSize(6 * 8);
186 : cc.setPassedOrder(RegGroup::kGp, kZcx, kZdx, 8, 9);
187 : cc.setPassedOrder(RegGroup::kVec, 0, 1, 2, 3, 4, 5);
188 : cc.setPreservedRegs(RegGroup::kGp, Support::bitMask(kZbx, kZsp, kZbp, kZsi, kZdi, 12, 13, 14, 15));
189 : cc.setPreservedRegs(RegGroup::kVec, Support::bitMask(6, 7, 8, 9, 10, 11, 12, 13, 14, 15));
190 : break;
191 : }
192 :
193 0 : case CallConvId::kLightCall2:
194 : case CallConvId::kLightCall3:
195 : case CallConvId::kLightCall4: {
196 0 : uint32_t n = uint32_t(ccId) - uint32_t(CallConvId::kLightCall2) + 2;
197 :
198 : cc.setFlags(CallConvFlags::kPassFloatsByVec);
199 : cc.setNaturalStackAlignment(16);
200 : cc.setPassedOrder(RegGroup::kGp, kZax, kZdx, kZcx, kZsi, kZdi);
201 : cc.setPassedOrder(RegGroup::kVec, 0, 1, 2, 3, 4, 5, 6, 7);
202 : cc.setPassedOrder(RegGroup::kX86_K, 0, 1, 2, 3, 4, 5, 6, 7);
203 : cc.setPassedOrder(RegGroup::kX86_MM, 0, 1, 2, 3, 4, 5, 6, 7);
204 :
205 : cc.setPreservedRegs(RegGroup::kGp, Support::lsbMask<uint32_t>(16));
206 0 : cc.setPreservedRegs(RegGroup::kVec, ~Support::lsbMask<uint32_t>(n));
207 : break;
208 : }
209 :
210 : default:
211 : return DebugUtils::errored(kErrorInvalidArgument);
212 : }
213 : }
214 :
215 : cc.setId(ccId);
216 37131 : return kErrorOk;
217 : }
218 :
219 47004 : ASMJIT_FAVOR_SIZE void unpackValues(FuncDetail& func, FuncValuePack& pack) noexcept {
220 : TypeId typeId = pack[0].typeId();
221 47004 : switch (typeId) {
222 : case TypeId::kInt64:
223 : case TypeId::kUInt64: {
224 188 : if (Environment::is32Bit(func.callConv().arch())) {
225 : // Convert a 64-bit return value to two 32-bit return values.
226 : pack[0].initTypeId(TypeId::kUInt32);
227 0 : pack[1].initTypeId(TypeId(uint32_t(typeId) - 2));
228 : break;
229 : }
230 : break;
231 : }
232 :
233 : default: {
234 : break;
235 : }
236 : }
237 47004 : }
238 :
239 37131 : ASMJIT_FAVOR_SIZE Error initFuncDetail(FuncDetail& func, const FuncSignature& signature, uint32_t registerSize) noexcept {
240 : const CallConv& cc = func.callConv();
241 : Arch arch = cc.arch();
242 37131 : uint32_t stackOffset = cc._spillZoneSize;
243 : uint32_t argCount = func.argCount();
244 :
245 : // Up to two return values can be returned in GP registers.
246 : static const uint8_t gpReturnIndexes[4] = {
247 : uint8_t(Gp::kIdAx),
248 : uint8_t(Gp::kIdDx),
249 : uint8_t(BaseReg::kIdBad),
250 : uint8_t(BaseReg::kIdBad)
251 : };
252 :
253 37131 : if (func.hasRet()) {
254 37131 : unpackValues(func, func._rets);
255 74262 : for (uint32_t valueIndex = 0; valueIndex < Globals::kMaxValuePack; valueIndex++) {
256 74262 : TypeId typeId = func._rets[valueIndex].typeId();
257 :
258 : // Terminate at the first void type (end of the pack).
259 74262 : if (typeId == TypeId::kVoid)
260 : break;
261 :
262 37131 : switch (typeId) {
263 0 : case TypeId::kInt64:
264 : case TypeId::kUInt64: {
265 0 : if (gpReturnIndexes[valueIndex] != BaseReg::kIdBad)
266 0 : func._rets[valueIndex].initReg(RegType::kX86_Gpq, gpReturnIndexes[valueIndex], typeId);
267 : else
268 : return DebugUtils::errored(kErrorInvalidState);
269 : break;
270 : }
271 :
272 0 : case TypeId::kInt8:
273 : case TypeId::kInt16:
274 : case TypeId::kInt32: {
275 0 : if (gpReturnIndexes[valueIndex] != BaseReg::kIdBad)
276 0 : func._rets[valueIndex].initReg(RegType::kX86_Gpd, gpReturnIndexes[valueIndex], TypeId::kInt32);
277 : else
278 : return DebugUtils::errored(kErrorInvalidState);
279 : break;
280 : }
281 :
282 0 : case TypeId::kUInt8:
283 : case TypeId::kUInt16:
284 : case TypeId::kUInt32: {
285 0 : if (gpReturnIndexes[valueIndex] != BaseReg::kIdBad)
286 0 : func._rets[valueIndex].initReg(RegType::kX86_Gpd, gpReturnIndexes[valueIndex], TypeId::kUInt32);
287 : else
288 : return DebugUtils::errored(kErrorInvalidState);
289 : break;
290 : }
291 :
292 : case TypeId::kFloat32:
293 : case TypeId::kFloat64: {
294 37131 : RegType regType = Environment::is32Bit(arch) ? RegType::kX86_St : RegType::kX86_Xmm;
295 37131 : func._rets[valueIndex].initReg(regType, valueIndex, typeId);
296 37131 : break;
297 : }
298 :
299 0 : case TypeId::kFloat80: {
300 : // 80-bit floats are always returned by FP0.
301 : func._rets[valueIndex].initReg(RegType::kX86_St, valueIndex, typeId);
302 : break;
303 : }
304 :
305 0 : case TypeId::kMmx32:
306 : case TypeId::kMmx64: {
307 : // MM registers are returned through XMM (SystemV) or GPQ (Win64).
308 : RegType regType = RegType::kX86_Mm;
309 : uint32_t regIndex = valueIndex;
310 0 : if (Environment::is64Bit(arch)) {
311 0 : regType = cc.strategy() == CallConvStrategy::kDefault ? RegType::kX86_Xmm : RegType::kX86_Gpq;
312 0 : regIndex = cc.strategy() == CallConvStrategy::kDefault ? valueIndex : gpReturnIndexes[valueIndex];
313 :
314 0 : if (regIndex == BaseReg::kIdBad)
315 : return DebugUtils::errored(kErrorInvalidState);
316 : }
317 :
318 0 : func._rets[valueIndex].initReg(regType, regIndex, typeId);
319 0 : break;
320 : }
321 :
322 0 : default: {
323 0 : func._rets[valueIndex].initReg(vecTypeIdToRegType(typeId), valueIndex, typeId);
324 0 : break;
325 : }
326 : }
327 : }
328 : }
329 :
330 37131 : switch (cc.strategy()) {
331 : case CallConvStrategy::kDefault: {
332 : uint32_t gpzPos = 0;
333 : uint32_t vecPos = 0;
334 :
335 47004 : for (uint32_t argIndex = 0; argIndex < argCount; argIndex++) {
336 9873 : unpackValues(func, func._args[argIndex]);
337 :
338 19746 : for (uint32_t valueIndex = 0; valueIndex < Globals::kMaxValuePack; valueIndex++) {
339 19746 : FuncValue& arg = func._args[argIndex][valueIndex];
340 :
341 : // Terminate if there are no more arguments in the pack.
342 19746 : if (!arg)
343 : break;
344 :
345 : TypeId typeId = arg.typeId();
346 :
347 9873 : if (TypeUtils::isInt(typeId)) {
348 : uint32_t regId = BaseReg::kIdBad;
349 :
350 188 : if (gpzPos < CallConv::kMaxRegArgsPerGroup)
351 188 : regId = cc._passedOrder[RegGroup::kGp].id[gpzPos];
352 :
353 188 : if (regId != BaseReg::kIdBad) {
354 188 : RegType regType = typeId <= TypeId::kUInt32 ? RegType::kX86_Gpd : RegType::kX86_Gpq;
355 188 : arg.assignRegData(regType, regId);
356 : func.addUsedRegs(RegGroup::kGp, Support::bitMask(regId));
357 188 : gpzPos++;
358 : }
359 : else {
360 : uint32_t size = Support::max<uint32_t>(TypeUtils::sizeOf(typeId), registerSize);
361 0 : arg.assignStackOffset(int32_t(stackOffset));
362 0 : stackOffset += size;
363 : }
364 : continue;
365 188 : }
366 :
367 9685 : if (TypeUtils::isFloat(typeId) || TypeUtils::isVec(typeId)) {
368 : uint32_t regId = BaseReg::kIdBad;
369 :
370 9685 : if (vecPos < CallConv::kMaxRegArgsPerGroup)
371 9685 : regId = cc._passedOrder[RegGroup::kVec].id[vecPos];
372 :
373 9685 : if (TypeUtils::isFloat(typeId)) {
374 : // If this is a float, but `kFlagPassFloatsByVec` is false, we have to use stack instead. This should
375 : // be only used by 32-bit calling conventions.
376 9685 : if (!cc.hasFlag(CallConvFlags::kPassFloatsByVec))
377 : regId = BaseReg::kIdBad;
378 : }
379 : else {
380 : // Pass vector registers via stack if this is a variable arguments function. This should be only used
381 : // by 32-bit calling conventions.
382 0 : if (signature.hasVarArgs() && cc.hasFlag(CallConvFlags::kPassVecByStackIfVA))
383 : regId = BaseReg::kIdBad;
384 : }
385 :
386 9685 : if (regId != BaseReg::kIdBad) {
387 : arg.initTypeId(typeId);
388 9685 : arg.assignRegData(vecTypeIdToRegType(typeId), regId);
389 : func.addUsedRegs(RegGroup::kVec, Support::bitMask(regId));
390 9685 : vecPos++;
391 : }
392 : else {
393 : uint32_t size = TypeUtils::sizeOf(typeId);
394 0 : arg.assignStackOffset(int32_t(stackOffset));
395 0 : stackOffset += size;
396 : }
397 : continue;
398 9685 : }
399 : }
400 : }
401 : break;
402 : }
403 :
404 : case CallConvStrategy::kX64Windows:
405 : case CallConvStrategy::kX64VectorCall: {
406 : // Both X64 and VectorCall behave similarly - arguments are indexed from left to right. The position of the
407 : // argument determines in which register the argument is allocated, so it's either GP or one of XMM/YMM/ZMM
408 : // registers.
409 : //
410 : // [ X64 ] [VecCall]
411 : // Index: #0 #1 #2 #3 #4 #5
412 : //
413 : // GP : RCX RDX R8 R9
414 : // VEC : XMM0 XMM1 XMM2 XMM3 XMM4 XMM5
415 : //
416 : // For example function `f(int a, double b, int c, double d)` will be:
417 : //
418 : // (a) (b) (c) (d)
419 : // RCX XMM1 R8 XMM3
420 : //
421 : // Unused vector registers are used by HVA.
422 : bool isVectorCall = (cc.strategy() == CallConvStrategy::kX64VectorCall);
423 :
424 0 : for (uint32_t argIndex = 0; argIndex < argCount; argIndex++) {
425 0 : unpackValues(func, func._args[argIndex]);
426 :
427 0 : for (uint32_t valueIndex = 0; valueIndex < Globals::kMaxValuePack; valueIndex++) {
428 0 : FuncValue& arg = func._args[argIndex][valueIndex];
429 :
430 : // Terminate if there are no more arguments in the pack.
431 0 : if (!arg)
432 : break;
433 :
434 : TypeId typeId = arg.typeId();
435 : uint32_t size = TypeUtils::sizeOf(typeId);
436 :
437 0 : if (TypeUtils::isInt(typeId) || TypeUtils::isMmx(typeId)) {
438 : uint32_t regId = BaseReg::kIdBad;
439 :
440 0 : if (argIndex < CallConv::kMaxRegArgsPerGroup)
441 0 : regId = cc._passedOrder[RegGroup::kGp].id[argIndex];
442 :
443 0 : if (regId != BaseReg::kIdBad) {
444 0 : RegType regType = size <= 4 && !TypeUtils::isMmx(typeId) ? RegType::kX86_Gpd : RegType::kX86_Gpq;
445 0 : arg.assignRegData(regType, regId);
446 : func.addUsedRegs(RegGroup::kGp, Support::bitMask(regId));
447 : }
448 : else {
449 0 : arg.assignStackOffset(int32_t(stackOffset));
450 0 : stackOffset += 8;
451 : }
452 : continue;
453 0 : }
454 :
455 0 : if (TypeUtils::isFloat(typeId) || TypeUtils::isVec(typeId)) {
456 : uint32_t regId = BaseReg::kIdBad;
457 :
458 0 : if (argIndex < CallConv::kMaxRegArgsPerGroup)
459 0 : regId = cc._passedOrder[RegGroup::kVec].id[argIndex];
460 :
461 0 : if (regId != BaseReg::kIdBad) {
462 : // X64-ABI doesn't allow vector types (XMM|YMM|ZMM) to be passed via registers, however, VectorCall
463 : // was designed for that purpose.
464 0 : if (TypeUtils::isFloat(typeId) || isVectorCall) {
465 0 : RegType regType = vecTypeIdToRegType(typeId);
466 0 : arg.assignRegData(regType, regId);
467 : func.addUsedRegs(RegGroup::kVec, Support::bitMask(regId));
468 0 : continue;
469 0 : }
470 : }
471 :
472 : // Passed via stack if the argument is float/double or indirectly. The trap is - if the argument is
473 : // passed indirectly, the address can be passed via register, if the argument's index has GP one.
474 0 : if (TypeUtils::isFloat(typeId)) {
475 0 : arg.assignStackOffset(int32_t(stackOffset));
476 : }
477 : else {
478 0 : uint32_t gpRegId = cc._passedOrder[RegGroup::kGp].id[argIndex];
479 0 : if (gpRegId != BaseReg::kIdBad)
480 0 : arg.assignRegData(RegType::kX86_Gpq, gpRegId);
481 : else
482 0 : arg.assignStackOffset(int32_t(stackOffset));
483 : arg.addFlags(FuncValue::kFlagIsIndirect);
484 : }
485 :
486 : // Always 8 bytes (float/double/pointer).
487 0 : stackOffset += 8;
488 0 : continue;
489 0 : }
490 : }
491 : }
492 : break;
493 : }
494 : }
495 :
496 37131 : func._argStackSize = stackOffset;
497 37131 : return kErrorOk;
498 : }
499 :
500 : } // {FuncInternal}
501 :
502 : ASMJIT_END_SUB_NAMESPACE
503 :
504 : #endif // !ASMJIT_NO_X86
505 :
506 : #endif // __PLUMED_HAS_ASMJIT
|
