Line data Source code
1 : #ifdef __PLUMED_HAS_ASMJIT
2 : // This file is part of AsmJit project <https://asmjit.com>
3 : //
4 : // See asmjit.h or LICENSE.md for license and copyright information
5 : // SPDX-License-Identifier: Zlib
6 :
7 : // ----------------------------------------------------------------------------
8 : // IMPORTANT: AsmJit now uses an external instruction database to populate
9 : // static tables within this file. Perform the following steps to regenerate
10 : // all tables enclosed by ${...}:
11 : //
12 : // 1. Install node.js environment <https://nodejs.org>
13 : // 2. Go to asmjit/tools directory
14 : // 3. Get the latest asmdb from <https://github.com/asmjit/asmdb> and
15 : // copy/link the `asmdb` directory to `asmjit/tools/asmdb`.
16 : // 4. Execute `node tablegen-x86.js`
17 : //
18 : // Instruction encoding and opcodes were added to the `x86inst.cpp` database
19 : // manually in the past and they are not updated by the script as it became
20 : // tricky. However, everything else is updated including instruction operands
21 : // and tables required to validate them, instruction read/write information
22 : // (including registers and flags), and all indexes to all tables.
23 : // ----------------------------------------------------------------------------
24 :
25 : #include "api-build_p.h"
26 : #if !defined(ASMJIT_NO_X86)
27 :
28 : #include "cpuinfo.h"
29 : #include "misc_p.h"
30 : #include "support.h"
31 : #include "x86instapi_p.h"
32 : #include "x86instdb_p.h"
33 : #include "x86opcode_p.h"
34 : #include "x86operand.h"
35 :
36 : ASMJIT_BEGIN_SUB_NAMESPACE(x86)
37 :
38 : // x86::InstInternal - Text
39 : // ========================
40 :
41 : #ifndef ASMJIT_NO_TEXT
42 0 : Error InstInternal::instIdToString(Arch arch, InstId instId, String& output) noexcept {
43 : DebugUtils::unused(arch);
44 :
45 0 : if (ASMJIT_UNLIKELY(!Inst::isDefinedId(instId)))
46 : return DebugUtils::errored(kErrorInvalidInstruction);
47 :
48 : const InstDB::InstInfo& info = InstDB::infoById(instId);
49 0 : return output.append(InstDB::_nameData + info._nameDataIndex);
50 : }
51 :
52 0 : InstId InstInternal::stringToInstId(Arch arch, const char* s, size_t len) noexcept {
53 : DebugUtils::unused(arch);
54 :
55 0 : if (ASMJIT_UNLIKELY(!s))
56 : return Inst::kIdNone;
57 :
58 0 : if (len == SIZE_MAX)
59 0 : len = strlen(s);
60 :
61 0 : if (ASMJIT_UNLIKELY(len == 0 || len > InstDB::kMaxNameSize))
62 : return Inst::kIdNone;
63 :
64 0 : uint32_t prefix = uint32_t(s[0]) - 'a';
65 0 : if (ASMJIT_UNLIKELY(prefix > 'z' - 'a'))
66 : return Inst::kIdNone;
67 :
68 0 : uint32_t index = InstDB::instNameIndex[prefix].start;
69 0 : if (ASMJIT_UNLIKELY(!index))
70 : return Inst::kIdNone;
71 :
72 : const char* nameData = InstDB::_nameData;
73 : const InstDB::InstInfo* table = InstDB::_instInfoTable;
74 :
75 0 : const InstDB::InstInfo* base = table + index;
76 0 : const InstDB::InstInfo* end = table + InstDB::instNameIndex[prefix].end;
77 :
78 0 : for (size_t lim = (size_t)(end - base); lim != 0; lim >>= 1) {
79 0 : const InstDB::InstInfo* cur = base + (lim >> 1);
80 0 : int result = Support::cmpInstName(nameData + cur[0]._nameDataIndex, s, len);
81 :
82 0 : if (result < 0) {
83 0 : base = cur + 1;
84 0 : lim--;
85 0 : continue;
86 : }
87 :
88 0 : if (result > 0)
89 0 : continue;
90 :
91 0 : return InstId((size_t)(cur - table));
92 : }
93 :
94 : return Inst::kIdNone;
95 : }
96 : #endif // !ASMJIT_NO_TEXT
97 :
98 : // x86::InstInternal - Validate
99 : // ============================
100 :
101 : #ifndef ASMJIT_NO_VALIDATION
102 : struct X86ValidationData {
103 : //! Allowed registers by \ref RegType.
104 : RegMask allowedRegMask[uint32_t(RegType::kMaxValue) + 1];
105 : uint32_t allowedMemBaseRegs;
106 : uint32_t allowedMemIndexRegs;
107 : };
108 :
109 : #define VALUE(x) \
110 : (x == uint32_t(RegType::kX86_GpbLo)) ? InstDB::OpFlags::kRegGpbLo : \
111 : (x == uint32_t(RegType::kX86_GpbHi)) ? InstDB::OpFlags::kRegGpbHi : \
112 : (x == uint32_t(RegType::kX86_Gpw )) ? InstDB::OpFlags::kRegGpw : \
113 : (x == uint32_t(RegType::kX86_Gpd )) ? InstDB::OpFlags::kRegGpd : \
114 : (x == uint32_t(RegType::kX86_Gpq )) ? InstDB::OpFlags::kRegGpq : \
115 : (x == uint32_t(RegType::kX86_Xmm )) ? InstDB::OpFlags::kRegXmm : \
116 : (x == uint32_t(RegType::kX86_Ymm )) ? InstDB::OpFlags::kRegYmm : \
117 : (x == uint32_t(RegType::kX86_Zmm )) ? InstDB::OpFlags::kRegZmm : \
118 : (x == uint32_t(RegType::kX86_Mm )) ? InstDB::OpFlags::kRegMm : \
119 : (x == uint32_t(RegType::kX86_KReg )) ? InstDB::OpFlags::kRegKReg : \
120 : (x == uint32_t(RegType::kX86_SReg )) ? InstDB::OpFlags::kRegSReg : \
121 : (x == uint32_t(RegType::kX86_CReg )) ? InstDB::OpFlags::kRegCReg : \
122 : (x == uint32_t(RegType::kX86_DReg )) ? InstDB::OpFlags::kRegDReg : \
123 : (x == uint32_t(RegType::kX86_St )) ? InstDB::OpFlags::kRegSt : \
124 : (x == uint32_t(RegType::kX86_Bnd )) ? InstDB::OpFlags::kRegBnd : \
125 : (x == uint32_t(RegType::kX86_Tmm )) ? InstDB::OpFlags::kRegTmm : \
126 : (x == uint32_t(RegType::kX86_Rip )) ? InstDB::OpFlags::kNone : InstDB::OpFlags::kNone
127 : static const InstDB::OpFlags _x86OpFlagFromRegType[uint32_t(RegType::kMaxValue) + 1] = { ASMJIT_LOOKUP_TABLE_32(VALUE, 0) };
128 : #undef VALUE
129 :
130 : #define REG_MASK_FROM_REG_TYPE_X86(x) \
131 : (x == uint32_t(RegType::kX86_GpbLo)) ? 0x0000000Fu : \
132 : (x == uint32_t(RegType::kX86_GpbHi)) ? 0x0000000Fu : \
133 : (x == uint32_t(RegType::kX86_Gpw )) ? 0x000000FFu : \
134 : (x == uint32_t(RegType::kX86_Gpd )) ? 0x000000FFu : \
135 : (x == uint32_t(RegType::kX86_Gpq )) ? 0x000000FFu : \
136 : (x == uint32_t(RegType::kX86_Xmm )) ? 0x000000FFu : \
137 : (x == uint32_t(RegType::kX86_Ymm )) ? 0x000000FFu : \
138 : (x == uint32_t(RegType::kX86_Zmm )) ? 0x000000FFu : \
139 : (x == uint32_t(RegType::kX86_Mm )) ? 0x000000FFu : \
140 : (x == uint32_t(RegType::kX86_KReg )) ? 0x000000FFu : \
141 : (x == uint32_t(RegType::kX86_SReg )) ? 0x0000007Eu : \
142 : (x == uint32_t(RegType::kX86_CReg )) ? 0x0000FFFFu : \
143 : (x == uint32_t(RegType::kX86_DReg )) ? 0x000000FFu : \
144 : (x == uint32_t(RegType::kX86_St )) ? 0x000000FFu : \
145 : (x == uint32_t(RegType::kX86_Bnd )) ? 0x0000000Fu : \
146 : (x == uint32_t(RegType::kX86_Tmm )) ? 0x000000FFu : \
147 : (x == uint32_t(RegType::kX86_Rip )) ? 0x00000001u : 0u
148 :
149 : #define REG_MASK_FROM_REG_TYPE_X64(x) \
150 : (x == uint32_t(RegType::kX86_GpbLo)) ? 0x0000FFFFu : \
151 : (x == uint32_t(RegType::kX86_GpbHi)) ? 0x0000000Fu : \
152 : (x == uint32_t(RegType::kX86_Gpw )) ? 0x0000FFFFu : \
153 : (x == uint32_t(RegType::kX86_Gpd )) ? 0x0000FFFFu : \
154 : (x == uint32_t(RegType::kX86_Gpq )) ? 0x0000FFFFu : \
155 : (x == uint32_t(RegType::kX86_Xmm )) ? 0xFFFFFFFFu : \
156 : (x == uint32_t(RegType::kX86_Ymm )) ? 0xFFFFFFFFu : \
157 : (x == uint32_t(RegType::kX86_Zmm )) ? 0xFFFFFFFFu : \
158 : (x == uint32_t(RegType::kX86_Mm )) ? 0x000000FFu : \
159 : (x == uint32_t(RegType::kX86_KReg )) ? 0x000000FFu : \
160 : (x == uint32_t(RegType::kX86_SReg )) ? 0x0000007Eu : \
161 : (x == uint32_t(RegType::kX86_CReg )) ? 0x0000FFFFu : \
162 : (x == uint32_t(RegType::kX86_DReg )) ? 0x0000FFFFu : \
163 : (x == uint32_t(RegType::kX86_St )) ? 0x000000FFu : \
164 : (x == uint32_t(RegType::kX86_Bnd )) ? 0x0000000Fu : \
165 : (x == uint32_t(RegType::kX86_Tmm )) ? 0x000000FFu : \
166 : (x == uint32_t(RegType::kX86_Rip )) ? 0x00000001u : 0u
167 :
168 : #define B(RegType) (uint32_t(1) << uint32_t(RegType))
169 :
170 : static const X86ValidationData _x86ValidationData = {
171 : { ASMJIT_LOOKUP_TABLE_32(REG_MASK_FROM_REG_TYPE_X86, 0) },
172 : B(RegType::kX86_Gpw) | B(RegType::kX86_Gpd) | B(RegType::kX86_Rip) | B(RegType::kLabelTag),
173 : B(RegType::kX86_Gpw) | B(RegType::kX86_Gpd) | B(RegType::kX86_Xmm) | B(RegType::kX86_Ymm) | B(RegType::kX86_Zmm)
174 : };
175 :
176 : static const X86ValidationData _x64ValidationData = {
177 : { ASMJIT_LOOKUP_TABLE_32(REG_MASK_FROM_REG_TYPE_X64, 0) },
178 : B(RegType::kX86_Gpd) | B(RegType::kX86_Gpq) | B(RegType::kX86_Rip) | B(RegType::kLabelTag),
179 : B(RegType::kX86_Gpd) | B(RegType::kX86_Gpq) | B(RegType::kX86_Xmm) | B(RegType::kX86_Ymm) | B(RegType::kX86_Zmm)
180 : };
181 :
182 : #undef B
183 :
184 : #undef REG_MASK_FROM_REG_TYPE_X64
185 : #undef REG_MASK_FROM_REG_TYPE_X86
186 :
187 : static ASMJIT_FORCE_INLINE bool x86IsZmmOrM512(const Operand_& op) noexcept {
188 0 : return Reg::isZmm(op) || (op.isMem() && op.size() == 64);
189 : }
190 :
191 : static ASMJIT_FORCE_INLINE bool x86CheckOSig(const InstDB::OpSignature& op, const InstDB::OpSignature& ref, bool& immOutOfRange) noexcept {
192 : // Fail if operand types are incompatible.
193 : InstDB::OpFlags commonFlags = op.flags() & ref.flags();
194 :
195 0 : if (!Support::test(commonFlags, InstDB::OpFlags::kOpMask)) {
196 : // Mark temporarily `immOutOfRange` so we can return a more descriptive error later.
197 0 : if (op.hasImm() && ref.hasImm()) {
198 : immOutOfRange = true;
199 : return true;
200 : }
201 :
202 : return false;
203 : }
204 :
205 : // Fail if some memory specific flags do not match.
206 0 : if (Support::test(commonFlags, InstDB::OpFlags::kMemMask)) {
207 0 : if (ref.hasFlag(InstDB::OpFlags::kFlagMemBase) && !op.hasFlag(InstDB::OpFlags::kFlagMemBase))
208 : return false;
209 : }
210 :
211 : // Fail if register indexes do not match.
212 0 : if (Support::test(commonFlags, InstDB::OpFlags::kRegMask)) {
213 0 : if (ref.regMask() && !Support::test(op.regMask(), ref.regMask()))
214 : return false;
215 : }
216 :
217 : return true;
218 : }
219 :
220 0 : ASMJIT_FAVOR_SIZE Error InstInternal::validate(Arch arch, const BaseInst& inst, const Operand_* operands, size_t opCount, ValidationFlags validationFlags) noexcept {
221 : // Only called when `arch` matches X86 family.
222 : ASMJIT_ASSERT(Environment::isFamilyX86(arch));
223 :
224 : const X86ValidationData* vd;
225 0 : if (arch == Arch::kX86)
226 : vd = &_x86ValidationData;
227 : else
228 : vd = &_x64ValidationData;
229 :
230 : uint32_t i;
231 0 : InstDB::Mode mode = InstDB::modeFromArch(arch);
232 :
233 : // Get the instruction data.
234 : InstId instId = inst.id();
235 : InstOptions options = inst.options();
236 :
237 0 : if (ASMJIT_UNLIKELY(!Inst::isDefinedId(instId)))
238 : return DebugUtils::errored(kErrorInvalidInstruction);
239 :
240 : const InstDB::InstInfo& instInfo = InstDB::infoById(instId);
241 : const InstDB::CommonInfo& commonInfo = instInfo.commonInfo();
242 :
243 : InstDB::InstFlags iFlags = instInfo.flags();
244 :
245 : constexpr InstOptions kRepAny = InstOptions::kX86_Rep | InstOptions::kX86_Repne;
246 : constexpr InstOptions kXAcqXRel = InstOptions::kX86_XAcquire | InstOptions::kX86_XRelease;
247 : constexpr InstOptions kAvx512Options = InstOptions::kX86_ZMask | InstOptions::kX86_ER | InstOptions::kX86_SAE;
248 :
249 : // Validate LOCK|XACQUIRE|XRELEASE Prefixes
250 : // ----------------------------------------
251 :
252 0 : if (Support::test(options, InstOptions::kX86_Lock | kXAcqXRel)) {
253 0 : if (Support::test(options, InstOptions::kX86_Lock)) {
254 0 : if (ASMJIT_UNLIKELY(!Support::test(iFlags, InstDB::InstFlags::kLock) && !Support::test(options, kXAcqXRel)))
255 : return DebugUtils::errored(kErrorInvalidLockPrefix);
256 :
257 0 : if (ASMJIT_UNLIKELY(opCount < 1 || !operands[0].isMem()))
258 : return DebugUtils::errored(kErrorInvalidLockPrefix);
259 : }
260 :
261 0 : if (Support::test(options, kXAcqXRel)) {
262 0 : if (ASMJIT_UNLIKELY(!Support::test(options, InstOptions::kX86_Lock) || (options & kXAcqXRel) == kXAcqXRel))
263 : return DebugUtils::errored(kErrorInvalidPrefixCombination);
264 :
265 0 : if (ASMJIT_UNLIKELY(Support::test(options, InstOptions::kX86_XAcquire) && !Support::test(iFlags, InstDB::InstFlags::kXAcquire)))
266 : return DebugUtils::errored(kErrorInvalidXAcquirePrefix);
267 :
268 0 : if (ASMJIT_UNLIKELY(Support::test(options, InstOptions::kX86_XRelease) && !Support::test(iFlags, InstDB::InstFlags::kXRelease)))
269 : return DebugUtils::errored(kErrorInvalidXReleasePrefix);
270 : }
271 : }
272 :
273 : // Validate REP and REPNE Prefixes
274 : // -------------------------------
275 :
276 0 : if (Support::test(options, kRepAny)) {
277 0 : if (ASMJIT_UNLIKELY((options & kRepAny) == kRepAny))
278 : return DebugUtils::errored(kErrorInvalidPrefixCombination);
279 :
280 0 : if (ASMJIT_UNLIKELY(!Support::test(iFlags, InstDB::InstFlags::kRep)))
281 : return DebugUtils::errored(kErrorInvalidRepPrefix);
282 : }
283 :
284 : // Translate Each Operand to the Corresponding OpSignature
285 : // -------------------------------------------------------
286 :
287 : InstDB::OpSignature oSigTranslated[Globals::kMaxOpCount];
288 : InstDB::OpFlags combinedOpFlags = InstDB::OpFlags::kNone;
289 : uint32_t combinedRegMask = 0;
290 : const Mem* memOp = nullptr;
291 :
292 0 : for (i = 0; i < opCount; i++) {
293 0 : const Operand_& op = operands[i];
294 0 : if (op.opType() == OperandType::kNone)
295 : break;
296 :
297 : InstDB::OpFlags opFlags = InstDB::OpFlags::kNone;
298 : RegMask regMask = 0;
299 :
300 0 : switch (op.opType()) {
301 : case OperandType::kReg: {
302 : RegType regType = op.as<BaseReg>().type();
303 0 : opFlags = _x86OpFlagFromRegType[size_t(regType)];
304 :
305 0 : if (ASMJIT_UNLIKELY(opFlags == InstDB::OpFlags::kNone))
306 : return DebugUtils::errored(kErrorInvalidRegType);
307 :
308 : // If `regId` is equal or greater than Operand::kVirtIdMin it means that the register is virtual and its
309 : // index will be assigned later by the register allocator. We must pass unless asked to disallow virtual
310 : // registers.
311 : uint32_t regId = op.id();
312 0 : if (regId < Operand::kVirtIdMin) {
313 0 : if (ASMJIT_UNLIKELY(regId >= 32))
314 : return DebugUtils::errored(kErrorInvalidPhysId);
315 :
316 0 : if (ASMJIT_UNLIKELY(Support::bitTest(vd->allowedRegMask[size_t(regType)], regId) == 0))
317 : return DebugUtils::errored(kErrorInvalidPhysId);
318 :
319 : regMask = Support::bitMask(regId);
320 0 : combinedRegMask |= regMask;
321 : }
322 : else {
323 0 : if (uint32_t(validationFlags & ValidationFlags::kEnableVirtRegs) == 0)
324 : return DebugUtils::errored(kErrorIllegalVirtReg);
325 : regMask = 0xFFFFFFFFu;
326 : }
327 : break;
328 : }
329 :
330 : // TODO: Validate base and index and combine these with `combinedRegMask`.
331 : case OperandType::kMem: {
332 : const Mem& m = op.as<Mem>();
333 : memOp = &m;
334 :
335 : uint32_t memSize = m.size();
336 : RegType baseType = m.baseType();
337 : RegType indexType = m.indexType();
338 :
339 0 : if (m.segmentId() > 6)
340 : return DebugUtils::errored(kErrorInvalidSegment);
341 :
342 : // Validate AVX-512 broadcast {1tox}.
343 0 : if (m.hasBroadcast()) {
344 0 : if (memSize != 0) {
345 : // If the size is specified it has to match the broadcast size.
346 0 : if (ASMJIT_UNLIKELY(commonInfo.hasAvx512B32() && memSize != 4))
347 : return DebugUtils::errored(kErrorInvalidBroadcast);
348 :
349 0 : if (ASMJIT_UNLIKELY(commonInfo.hasAvx512B64() && memSize != 8))
350 : return DebugUtils::errored(kErrorInvalidBroadcast);
351 : }
352 : else {
353 : // If there is no size we implicitly calculate it so we can validate N in {1toN} properly.
354 0 : memSize = commonInfo.hasAvx512B64() ? 8 :
355 : commonInfo.hasAvx512B32() ? 4 : 2;
356 : }
357 :
358 0 : memSize <<= uint32_t(m.getBroadcast());
359 : }
360 :
361 0 : if (baseType != RegType::kNone && baseType > RegType::kLabelTag) {
362 : uint32_t baseId = m.baseId();
363 :
364 0 : if (m.isRegHome()) {
365 : // Home address of a virtual register. In such case we don't want to validate the type of the
366 : // base register as it will always be patched to ESP|RSP.
367 : }
368 : else {
369 0 : if (ASMJIT_UNLIKELY(!Support::bitTest(vd->allowedMemBaseRegs, baseType)))
370 : return DebugUtils::errored(kErrorInvalidAddress);
371 : }
372 :
373 : // Create information that will be validated only if this is an implicit memory operand. Basically
374 : // only usable for string instructions and other instructions where memory operand is implicit and
375 : // has 'seg:[reg]' form.
376 0 : if (baseId < Operand::kVirtIdMin) {
377 0 : if (ASMJIT_UNLIKELY(baseId >= 32))
378 : return DebugUtils::errored(kErrorInvalidPhysId);
379 :
380 : // Physical base id.
381 : regMask = Support::bitMask(baseId);
382 0 : combinedRegMask |= regMask;
383 : }
384 : else {
385 : // Virtual base id - fill the whole mask for implicit mem validation. The register is not assigned
386 : // yet, so we cannot predict the phys id.
387 0 : if (uint32_t(validationFlags & ValidationFlags::kEnableVirtRegs) == 0)
388 : return DebugUtils::errored(kErrorIllegalVirtReg);
389 : regMask = 0xFFFFFFFFu;
390 : }
391 :
392 0 : if (indexType == RegType::kNone && !m.offsetLo32())
393 : opFlags |= InstDB::OpFlags::kFlagMemBase;
394 : }
395 0 : else if (baseType == RegType::kLabelTag) {
396 : // [Label] - there is no need to validate the base as it's label.
397 : }
398 : else {
399 : // Base is a 64-bit address.
400 0 : int64_t offset = m.offset();
401 0 : if (!Support::isInt32(offset)) {
402 0 : if (mode == InstDB::Mode::kX86) {
403 : // 32-bit mode: Make sure that the address is either `int32_t` or `uint32_t`.
404 0 : if (!Support::isUInt32(offset))
405 : return DebugUtils::errored(kErrorInvalidAddress64Bit);
406 : }
407 : else {
408 : // 64-bit mode: Zero extension is allowed if the address has 32-bit index register or the address
409 : // has no index register (it's still encodable).
410 0 : if (indexType != RegType::kNone) {
411 0 : if (!Support::isUInt32(offset))
412 : return DebugUtils::errored(kErrorInvalidAddress64Bit);
413 :
414 0 : if (indexType != RegType::kX86_Gpd)
415 : return DebugUtils::errored(kErrorInvalidAddress64BitZeroExtension);
416 : }
417 : else {
418 : // We don't validate absolute 64-bit addresses without an index register as this also depends
419 : // on the target's base address. We don't have the information to do it at this moment.
420 : }
421 : }
422 : }
423 : }
424 :
425 0 : if (indexType != RegType::kNone) {
426 0 : if (ASMJIT_UNLIKELY(!Support::bitTest(vd->allowedMemIndexRegs, indexType)))
427 : return DebugUtils::errored(kErrorInvalidAddress);
428 :
429 0 : if (indexType == RegType::kX86_Xmm) {
430 : opFlags |= InstDB::OpFlags::kVm32x | InstDB::OpFlags::kVm64x;
431 : }
432 0 : else if (indexType == RegType::kX86_Ymm) {
433 : opFlags |= InstDB::OpFlags::kVm32y | InstDB::OpFlags::kVm64y;
434 : }
435 0 : else if (indexType == RegType::kX86_Zmm) {
436 : opFlags |= InstDB::OpFlags::kVm32z | InstDB::OpFlags::kVm64z;
437 : }
438 : else {
439 0 : if (baseType != RegType::kNone)
440 : opFlags |= InstDB::OpFlags::kFlagMib;
441 : }
442 :
443 : // [RIP + {XMM|YMM|ZMM}] is not allowed.
444 0 : if (baseType == RegType::kX86_Rip && Support::test(opFlags, InstDB::OpFlags::kVmMask))
445 : return DebugUtils::errored(kErrorInvalidAddress);
446 :
447 : uint32_t indexId = m.indexId();
448 0 : if (indexId < Operand::kVirtIdMin) {
449 0 : if (ASMJIT_UNLIKELY(indexId >= 32))
450 : return DebugUtils::errored(kErrorInvalidPhysId);
451 :
452 0 : combinedRegMask |= Support::bitMask(indexId);
453 : }
454 : else {
455 0 : if (uint32_t(validationFlags & ValidationFlags::kEnableVirtRegs) == 0)
456 : return DebugUtils::errored(kErrorIllegalVirtReg);
457 : }
458 :
459 : // Only used for implicit memory operands having 'seg:[reg]' form, so clear it.
460 : regMask = 0;
461 : }
462 :
463 0 : switch (memSize) {
464 : case 0: opFlags |= InstDB::OpFlags::kMemUnspecified; break;
465 : case 1: opFlags |= InstDB::OpFlags::kMem8; break;
466 : case 2: opFlags |= InstDB::OpFlags::kMem16; break;
467 : case 4: opFlags |= InstDB::OpFlags::kMem32; break;
468 : case 6: opFlags |= InstDB::OpFlags::kMem48; break;
469 : case 8: opFlags |= InstDB::OpFlags::kMem64; break;
470 : case 10: opFlags |= InstDB::OpFlags::kMem80; break;
471 : case 16: opFlags |= InstDB::OpFlags::kMem128; break;
472 : case 32: opFlags |= InstDB::OpFlags::kMem256; break;
473 : case 64: opFlags |= InstDB::OpFlags::kMem512; break;
474 :
475 : default:
476 : return DebugUtils::errored(kErrorInvalidOperandSize);
477 : }
478 :
479 : break;
480 : }
481 :
482 : case OperandType::kImm: {
483 : uint64_t immValue = op.as<Imm>().valueAs<uint64_t>();
484 :
485 0 : if (int64_t(immValue) >= 0) {
486 0 : if (immValue <= 0x7u)
487 : opFlags = InstDB::OpFlags::kImmI64 | InstDB::OpFlags::kImmU64 | InstDB::OpFlags::kImmI32 | InstDB::OpFlags::kImmU32 |
488 : InstDB::OpFlags::kImmI16 | InstDB::OpFlags::kImmU16 | InstDB::OpFlags::kImmI8 | InstDB::OpFlags::kImmU8 |
489 : InstDB::OpFlags::kImmI4 | InstDB::OpFlags::kImmU4 ;
490 0 : else if (immValue <= 0xFu)
491 : opFlags = InstDB::OpFlags::kImmI64 | InstDB::OpFlags::kImmU64 | InstDB::OpFlags::kImmI32 | InstDB::OpFlags::kImmU32 |
492 : InstDB::OpFlags::kImmI16 | InstDB::OpFlags::kImmU16 | InstDB::OpFlags::kImmI8 | InstDB::OpFlags::kImmU8 |
493 : InstDB::OpFlags::kImmU4 ;
494 0 : else if (immValue <= 0x7Fu)
495 : opFlags = InstDB::OpFlags::kImmI64 | InstDB::OpFlags::kImmU64 | InstDB::OpFlags::kImmI32 | InstDB::OpFlags::kImmU32 |
496 : InstDB::OpFlags::kImmI16 | InstDB::OpFlags::kImmU16 | InstDB::OpFlags::kImmI8 | InstDB::OpFlags::kImmU8 ;
497 0 : else if (immValue <= 0xFFu)
498 : opFlags = InstDB::OpFlags::kImmI64 | InstDB::OpFlags::kImmU64 | InstDB::OpFlags::kImmI32 | InstDB::OpFlags::kImmU32 |
499 : InstDB::OpFlags::kImmI16 | InstDB::OpFlags::kImmU16 | InstDB::OpFlags::kImmU8 ;
500 0 : else if (immValue <= 0x7FFFu)
501 : opFlags = InstDB::OpFlags::kImmI64 | InstDB::OpFlags::kImmU64 | InstDB::OpFlags::kImmI32 | InstDB::OpFlags::kImmU32 |
502 : InstDB::OpFlags::kImmI16 | InstDB::OpFlags::kImmU16 ;
503 0 : else if (immValue <= 0xFFFFu)
504 : opFlags = InstDB::OpFlags::kImmI64 | InstDB::OpFlags::kImmU64 | InstDB::OpFlags::kImmI32 | InstDB::OpFlags::kImmU32 |
505 : InstDB::OpFlags::kImmU16 ;
506 0 : else if (immValue <= 0x7FFFFFFFu)
507 : opFlags = InstDB::OpFlags::kImmI64 | InstDB::OpFlags::kImmU64 | InstDB::OpFlags::kImmI32 | InstDB::OpFlags::kImmU32;
508 0 : else if (immValue <= 0xFFFFFFFFu)
509 : opFlags = InstDB::OpFlags::kImmI64 | InstDB::OpFlags::kImmU64 | InstDB::OpFlags::kImmU32;
510 : else if (immValue <= 0x7FFFFFFFFFFFFFFFu)
511 : opFlags = InstDB::OpFlags::kImmI64 | InstDB::OpFlags::kImmU64;
512 : else
513 : opFlags = InstDB::OpFlags::kImmU64;
514 : }
515 : else {
516 : immValue = Support::neg(immValue);
517 0 : if (immValue <= 0x8u)
518 : opFlags = InstDB::OpFlags::kImmI64 | InstDB::OpFlags::kImmI32 | InstDB::OpFlags::kImmI16 | InstDB::OpFlags::kImmI8 | InstDB::OpFlags::kImmI4;
519 0 : else if (immValue <= 0x80u)
520 : opFlags = InstDB::OpFlags::kImmI64 | InstDB::OpFlags::kImmI32 | InstDB::OpFlags::kImmI16 | InstDB::OpFlags::kImmI8;
521 0 : else if (immValue <= 0x8000u)
522 : opFlags = InstDB::OpFlags::kImmI64 | InstDB::OpFlags::kImmI32 | InstDB::OpFlags::kImmI16;
523 0 : else if (immValue <= 0x80000000u)
524 : opFlags = InstDB::OpFlags::kImmI64 | InstDB::OpFlags::kImmI32;
525 : else
526 : opFlags = InstDB::OpFlags::kImmI64;
527 : }
528 : break;
529 : }
530 :
531 : case OperandType::kLabel: {
532 : opFlags |= InstDB::OpFlags::kRel8 | InstDB::OpFlags::kRel32;
533 : break;
534 : }
535 :
536 : default:
537 : return DebugUtils::errored(kErrorInvalidState);
538 : }
539 :
540 : InstDB::OpSignature& oSigDst = oSigTranslated[i];
541 0 : oSigDst._flags = uint64_t(opFlags) & 0x00FFFFFFFFFFFFFFu;
542 0 : oSigDst._regMask = uint8_t(regMask & 0xFFu);
543 : combinedOpFlags |= opFlags;
544 : }
545 :
546 : // Decrease the number of operands of those that are none. This is important as Assembler and Compiler may just pass
547 : // more operands padded with none (which means that no operand is given at that index). However, validate that there
548 : // are no gaps (like [reg, none, reg] or [none, reg]).
549 0 : if (i < opCount) {
550 0 : while (--opCount > i)
551 0 : if (ASMJIT_UNLIKELY(!operands[opCount].isNone()))
552 : return DebugUtils::errored(kErrorInvalidInstruction);
553 : }
554 :
555 : // Validate X86 and X64 specific cases.
556 0 : if (mode == InstDB::Mode::kX86) {
557 : // Illegal use of 64-bit register in 32-bit mode.
558 0 : if (ASMJIT_UNLIKELY(Support::test(combinedOpFlags, InstDB::OpFlags::kRegGpq)))
559 : return DebugUtils::errored(kErrorInvalidUseOfGpq);
560 : }
561 : else {
562 : // Illegal use of a high 8-bit register with REX prefix.
563 0 : bool hasREX = inst.hasOption(InstOptions::kX86_Rex) || (combinedRegMask & 0xFFFFFF00u) != 0;
564 0 : if (ASMJIT_UNLIKELY(hasREX && Support::test(combinedOpFlags, InstDB::OpFlags::kRegGpbHi)))
565 : return DebugUtils::errored(kErrorInvalidUseOfGpbHi);
566 : }
567 :
568 : // Validate Instruction Signature by Comparing Against All `iSig` Rows
569 : // -------------------------------------------------------------------
570 :
571 0 : const InstDB::InstSignature* iSig = InstDB::_instSignatureTable + commonInfo._iSignatureIndex;
572 0 : const InstDB::InstSignature* iEnd = iSig + commonInfo._iSignatureCount;
573 :
574 0 : if (iSig != iEnd) {
575 : const InstDB::OpSignature* opSignatureTable = InstDB::_opSignatureTable;
576 :
577 : // If set it means that we matched a signature where only immediate value
578 : // was out of bounds. We can return a more descriptive error if we know this.
579 : bool globalImmOutOfRange = false;
580 :
581 : do {
582 : // Check if the architecture is compatible.
583 0 : if (!iSig->supportsMode(mode))
584 : continue;
585 :
586 : // Compare the operands table with reference operands.
587 : uint32_t j = 0;
588 : uint32_t iSigCount = iSig->opCount();
589 : bool localImmOutOfRange = false;
590 :
591 0 : if (iSigCount == opCount) {
592 0 : for (j = 0; j < opCount; j++)
593 0 : if (!x86CheckOSig(oSigTranslated[j], iSig->opSignature(j), localImmOutOfRange))
594 : break;
595 : }
596 0 : else if (iSigCount - iSig->implicitOpCount() == opCount) {
597 : uint32_t r = 0;
598 0 : for (j = 0; j < opCount && r < iSigCount; j++, r++) {
599 0 : const InstDB::OpSignature* oChk = oSigTranslated + j;
600 : const InstDB::OpSignature* oRef;
601 0 : Next:
602 0 : oRef = opSignatureTable + iSig->opSignatureIndex(r);
603 : // Skip implicit operands.
604 0 : if (oRef->isImplicit()) {
605 0 : if (++r >= iSigCount)
606 : break;
607 : else
608 0 : goto Next;
609 : }
610 :
611 0 : if (!x86CheckOSig(*oChk, *oRef, localImmOutOfRange))
612 : break;
613 : }
614 : }
615 :
616 0 : if (j == opCount) {
617 0 : if (!localImmOutOfRange) {
618 : // Match, must clear possible `globalImmOutOfRange`.
619 : globalImmOutOfRange = false;
620 : break;
621 : }
622 : globalImmOutOfRange = localImmOutOfRange;
623 : }
624 0 : } while (++iSig != iEnd);
625 :
626 0 : if (iSig == iEnd) {
627 0 : if (globalImmOutOfRange)
628 : return DebugUtils::errored(kErrorInvalidImmediate);
629 : else
630 : return DebugUtils::errored(kErrorInvalidInstruction);
631 : }
632 : }
633 :
634 : // Validate AVX512 Options
635 : // -----------------------
636 :
637 : const RegOnly& extraReg = inst.extraReg();
638 :
639 0 : if (Support::test(options, kAvx512Options)) {
640 0 : if (commonInfo.hasFlag(InstDB::InstFlags::kEvex)) {
641 : // Validate AVX-512 {z}.
642 0 : if (Support::test(options, InstOptions::kX86_ZMask)) {
643 0 : if (ASMJIT_UNLIKELY(Support::test(options, InstOptions::kX86_ZMask) && !commonInfo.hasAvx512Z()))
644 : return DebugUtils::errored(kErrorInvalidKZeroUse);
645 : }
646 :
647 : // Validate AVX-512 {sae} and {er}.
648 0 : if (Support::test(options, InstOptions::kX86_SAE | InstOptions::kX86_ER)) {
649 : // Rounding control is impossible if the instruction is not reg-to-reg.
650 0 : if (ASMJIT_UNLIKELY(memOp))
651 : return DebugUtils::errored(kErrorInvalidEROrSAE);
652 :
653 : // Check if {sae} or {er} is supported by the instruction.
654 0 : if (Support::test(options, InstOptions::kX86_ER)) {
655 : // NOTE: if both {sae} and {er} are set, we don't care, as {sae} is implied.
656 0 : if (ASMJIT_UNLIKELY(!commonInfo.hasAvx512ER()))
657 : return DebugUtils::errored(kErrorInvalidEROrSAE);
658 : }
659 : else {
660 0 : if (ASMJIT_UNLIKELY(!commonInfo.hasAvx512SAE()))
661 : return DebugUtils::errored(kErrorInvalidEROrSAE);
662 : }
663 :
664 : // {sae} and {er} are defined for either scalar ops or vector ops that require LL to be 10 (512-bit vector
665 : // operations). We don't need any more bits in the instruction database to be able to validate this, as
666 : // each AVX512 instruction that has broadcast is vector instruction (in this case we require zmm registers),
667 : // otherwise it's a scalar instruction, which is valid.
668 0 : if (commonInfo.hasAvx512B()) {
669 : // Supports broadcast, thus we require LL to be '10', which means there have to be ZMM registers used. We
670 : // don't calculate LL here, but we know that it would be '10' if there is at least one ZMM register used.
671 :
672 : // There is no {er}/{sae}-enabled instruction with less than two operands.
673 : ASMJIT_ASSERT(opCount >= 2);
674 0 : if (ASMJIT_UNLIKELY(!x86IsZmmOrM512(operands[0]) && !x86IsZmmOrM512(operands[1])))
675 : return DebugUtils::errored(kErrorInvalidEROrSAE);
676 : }
677 : }
678 : }
679 : else {
680 : // Not an AVX512 instruction - maybe OpExtra is xCX register used by REP/REPNE prefix.
681 : if (Support::test(options, kAvx512Options) || !Support::test(options, kRepAny))
682 : return DebugUtils::errored(kErrorInvalidInstruction);
683 : }
684 : }
685 :
686 : // Validate {Extra} Register
687 : // -------------------------
688 :
689 0 : if (extraReg.isReg()) {
690 0 : if (Support::test(options, kRepAny)) {
691 : // Validate REP|REPNE {cx|ecx|rcx}.
692 0 : if (ASMJIT_UNLIKELY(Support::test(iFlags, InstDB::InstFlags::kRepIgnored)))
693 : return DebugUtils::errored(kErrorInvalidExtraReg);
694 :
695 0 : if (extraReg.isPhysReg()) {
696 0 : if (ASMJIT_UNLIKELY(extraReg.id() != Gp::kIdCx))
697 : return DebugUtils::errored(kErrorInvalidExtraReg);
698 : }
699 :
700 : // The type of the {...} register must match the type of the base register
701 : // of memory operand. So if the memory operand uses 32-bit register the
702 : // count register must also be 32-bit, etc...
703 0 : if (ASMJIT_UNLIKELY(!memOp || extraReg.type() != memOp->baseType()))
704 : return DebugUtils::errored(kErrorInvalidExtraReg);
705 : }
706 0 : else if (commonInfo.hasFlag(InstDB::InstFlags::kEvex)) {
707 : // Validate AVX-512 {k}.
708 0 : if (ASMJIT_UNLIKELY(extraReg.type() != RegType::kX86_KReg))
709 : return DebugUtils::errored(kErrorInvalidExtraReg);
710 :
711 0 : if (ASMJIT_UNLIKELY(extraReg.id() == 0 || !commonInfo.hasAvx512K()))
712 : return DebugUtils::errored(kErrorInvalidKMaskUse);
713 : }
714 : else {
715 : return DebugUtils::errored(kErrorInvalidExtraReg);
716 : }
717 : }
718 :
719 : return kErrorOk;
720 : }
721 : #endif // !ASMJIT_NO_VALIDATION
722 :
723 : // x86::InstInternal - QueryRWInfo
724 : // ===============================
725 :
726 : #ifndef ASMJIT_NO_INTROSPECTION
727 : static const Support::Array<uint64_t, uint32_t(RegGroup::kMaxValue) + 1> rwRegGroupByteMask = {{
728 : 0x00000000000000FFu, // GP.
729 : 0xFFFFFFFFFFFFFFFFu, // XMM|YMM|ZMM.
730 : 0x00000000000000FFu, // MM.
731 : 0x00000000000000FFu, // KReg.
732 : 0x0000000000000003u, // SReg.
733 : 0x00000000000000FFu, // CReg.
734 : 0x00000000000000FFu, // DReg.
735 : 0x00000000000003FFu, // St().
736 : 0x000000000000FFFFu, // BND.
737 : 0x00000000000000FFu // RIP.
738 : }};
739 :
740 : static ASMJIT_FORCE_INLINE void rwZeroExtendGp(OpRWInfo& opRwInfo, const Gp& reg, uint32_t nativeGpSize) noexcept {
741 : ASMJIT_ASSERT(BaseReg::isGp(reg.as<Operand>()));
742 88236 : if (reg.size() + 4 == nativeGpSize) {
743 : opRwInfo.addOpFlags(OpRWFlags::kZExt);
744 0 : opRwInfo.setExtendByteMask(~opRwInfo.writeByteMask() & 0xFFu);
745 : }
746 : }
747 :
748 : static ASMJIT_FORCE_INLINE void rwZeroExtendAvxVec(OpRWInfo& opRwInfo, const Vec& reg) noexcept {
749 : DebugUtils::unused(reg);
750 :
751 0 : uint64_t msk = ~Support::fillTrailingBits(opRwInfo.writeByteMask());
752 0 : if (msk) {
753 : opRwInfo.addOpFlags(OpRWFlags::kZExt);
754 : opRwInfo.setExtendByteMask(msk);
755 : }
756 : }
757 :
758 : static ASMJIT_FORCE_INLINE void rwZeroExtendNonVec(OpRWInfo& opRwInfo, const Reg& reg) noexcept {
759 206347 : uint64_t msk = ~Support::fillTrailingBits(opRwInfo.writeByteMask()) & rwRegGroupByteMask[reg.group()];
760 206347 : if (msk) {
761 : opRwInfo.addOpFlags(OpRWFlags::kZExt);
762 : opRwInfo.setExtendByteMask(msk);
763 : }
764 : }
765 :
766 : static ASMJIT_FORCE_INLINE Error rwHandleAVX512(const BaseInst& inst, const InstDB::CommonInfo& commonInfo, InstRWInfo* out) noexcept {
767 216080 : if (inst.hasExtraReg() && inst.extraReg().type() == RegType::kX86_KReg && out->opCount() > 0) {
768 : // AVX-512 instruction that uses a destination with {k} register (zeroing vs masking).
769 : out->_extraReg.addOpFlags(OpRWFlags::kRead);
770 : out->_extraReg.setReadByteMask(0xFF);
771 0 : if (!inst.hasOption(InstOptions::kX86_ZMask) && !commonInfo.hasAvx512Flag(InstDB::Avx512Flags::kImplicitZ)) {
772 : out->_operands[0].addOpFlags(OpRWFlags::kRead);
773 0 : out->_operands[0]._readByteMask |= out->_operands[0]._writeByteMask;
774 : }
775 : }
776 :
777 : return kErrorOk;
778 : }
779 :
780 : static ASMJIT_FORCE_INLINE bool hasSameRegType(const BaseReg* regs, size_t opCount) noexcept {
781 : ASMJIT_ASSERT(opCount > 0);
782 : RegType regType = regs[0].type();
783 92421 : for (size_t i = 1; i < opCount; i++)
784 61614 : if (regs[i].type() != regType)
785 : return false;
786 : return true;
787 : }
788 :
789 304316 : Error InstInternal::queryRWInfo(Arch arch, const BaseInst& inst, const Operand_* operands, size_t opCount, InstRWInfo* out) noexcept {
790 : // Only called when `arch` matches X86 family.
791 : ASMJIT_ASSERT(Environment::isFamilyX86(arch));
792 :
793 : // Get the instruction data.
794 : InstId instId = inst.id();
795 304316 : if (ASMJIT_UNLIKELY(!Inst::isDefinedId(instId)))
796 : return DebugUtils::errored(kErrorInvalidInstruction);
797 :
798 : // Read/Write flags.
799 : const InstDB::InstInfo& instInfo = InstDB::_instInfoTable[instId];
800 304316 : const InstDB::CommonInfo& commonInfo = InstDB::_commonInfoTable[instInfo._commonInfoIndex];
801 304316 : const InstDB::AdditionalInfo& additionalInfo = InstDB::_additionalInfoTable[instInfo._additionalInfoIndex];
802 304316 : const InstDB::RWFlagsInfoTable& rwFlags = InstDB::_rwFlagsInfoTable[additionalInfo._rwFlagsIndex];
803 :
804 : // There are two data tables, one for `opCount == 2` and the second for
805 : // `opCount != 2`. There are two reasons for that:
806 : // - There are instructions that share the same name that have both 2 or 3 operands, which have different
807 : // RW information / semantics.
808 : // - There must be 2 tables otherwise the lookup index won't fit into 8 bits (there is more than 256 records
809 : // of combined rwInfo A and B).
810 304316 : const InstDB::RWInfo& instRwInfo = opCount == 2 ? InstDB::rwInfoA[InstDB::rwInfoIndexA[instId]]
811 145106 : : InstDB::rwInfoB[InstDB::rwInfoIndexB[instId]];
812 304316 : const InstDB::RWInfoRm& instRmInfo = InstDB::rwInfoRm[instRwInfo.rmInfo];
813 :
814 304316 : out->_instFlags = InstDB::_instFlagsTable[additionalInfo._instFlagsIndex];
815 304316 : out->_opCount = uint8_t(opCount);
816 304316 : out->_rmFeature = instRmInfo.rmFeature;
817 304316 : out->_extraReg.reset();
818 304316 : out->_readFlags = CpuRWFlags(rwFlags.readFlags);
819 304316 : out->_writeFlags = CpuRWFlags(rwFlags.writeFlags);
820 :
821 : uint32_t opTypeMask = 0u;
822 : uint32_t nativeGpSize = Environment::registerSizeFromArch(arch);
823 :
824 : constexpr OpRWFlags R = OpRWFlags::kRead;
825 : constexpr OpRWFlags W = OpRWFlags::kWrite;
826 : constexpr OpRWFlags X = OpRWFlags::kRW;
827 : constexpr OpRWFlags RegM = OpRWFlags::kRegMem;
828 : constexpr OpRWFlags RegPhys = OpRWFlags::kRegPhysId;
829 : constexpr OpRWFlags MibRead = OpRWFlags::kMemBaseRead | OpRWFlags::kMemIndexRead;
830 :
831 304316 : if (instRwInfo.category == InstDB::RWInfo::kCategoryGeneric) {
832 : uint32_t i;
833 : uint32_t rmOpsMask = 0;
834 : uint32_t rmMaxSize = 0;
835 :
836 776201 : for (i = 0; i < opCount; i++) {
837 560121 : OpRWInfo& op = out->_operands[i];
838 560121 : const Operand_& srcOp = operands[i];
839 560121 : const InstDB::RWInfoOp& rwOpData = InstDB::rwInfoOp[instRwInfo.opInfoIndex[i]];
840 :
841 560121 : opTypeMask |= Support::bitMask(srcOp.opType());
842 :
843 1976 : if (!srcOp.isRegOrMem()) {
844 1976 : op.reset();
845 1976 : continue;
846 : }
847 :
848 558145 : op._opFlags = rwOpData.flags & ~OpRWFlags::kZExt;
849 558145 : op._physId = rwOpData.physId;
850 558145 : op._rmSize = 0;
851 : op._resetReserved();
852 :
853 558145 : uint64_t rByteMask = rwOpData.rByteMask;
854 558145 : uint64_t wByteMask = rwOpData.wByteMask;
855 :
856 558145 : if (op.isRead() && !rByteMask) rByteMask = Support::lsbMask<uint64_t>(srcOp.size());
857 558145 : if (op.isWrite() && !wByteMask) wByteMask = Support::lsbMask<uint64_t>(srcOp.size());
858 :
859 558145 : op._readByteMask = rByteMask;
860 558145 : op._writeByteMask = wByteMask;
861 558145 : op._extendByteMask = 0;
862 558145 : op._consecutiveLeadCount = rwOpData.consecutiveLeadCount;
863 :
864 558145 : if (srcOp.isReg()) {
865 : // Zero extension.
866 487171 : if (op.isWrite()) {
867 206347 : if (srcOp.as<Reg>().isGp()) {
868 : // GP registers on X64 are special:
869 : // - 8-bit and 16-bit writes aren't zero extended.
870 : // - 32-bit writes ARE zero extended.
871 : rwZeroExtendGp(op, srcOp.as<Gp>(), nativeGpSize);
872 : }
873 206347 : else if (Support::test(rwOpData.flags, OpRWFlags::kZExt)) {
874 : // Otherwise follow ZExt.
875 : rwZeroExtendNonVec(op, srcOp.as<Gp>());
876 : }
877 : }
878 :
879 : // Aggregate values required to calculate valid Reg/M info.
880 : rmMaxSize = Support::max(rmMaxSize, srcOp.size());
881 487171 : rmOpsMask |= Support::bitMask<uint32_t>(i);
882 : }
883 : else {
884 : const x86::Mem& memOp = srcOp.as<x86::Mem>();
885 : // The RW flags of BASE+INDEX are either provided by the data, which means
886 : // that the instruction is border-case, or they are deduced from the operand.
887 70974 : if (memOp.hasBaseReg() && !op.hasOpFlag(OpRWFlags::kMemBaseRW))
888 : op.addOpFlags(OpRWFlags::kMemBaseRead);
889 70974 : if (memOp.hasIndexReg() && !op.hasOpFlag(OpRWFlags::kMemIndexRW))
890 : op.addOpFlags(OpRWFlags::kMemIndexRead);
891 : }
892 : }
893 :
894 : // Only keep kMovOp if the instruction is actually register to register move of the same kind.
895 216080 : if (out->hasInstFlag(InstRWFlags::kMovOp)) {
896 132588 : if (!(opCount >= 2 && opTypeMask == Support::bitMask(OperandType::kReg) && hasSameRegType(reinterpret_cast<const BaseReg*>(operands), opCount)))
897 : out->_instFlags &= ~InstRWFlags::kMovOp;
898 : }
899 :
900 : // Special cases require more logic.
901 216080 : if (instRmInfo.flags & (InstDB::RWInfoRm::kFlagMovssMovsd | InstDB::RWInfoRm::kFlagPextrw | InstDB::RWInfoRm::kFlagFeatureIfRMI)) {
902 0 : if (instRmInfo.flags & InstDB::RWInfoRm::kFlagMovssMovsd) {
903 0 : if (opCount == 2) {
904 0 : if (operands[0].isReg() && operands[1].isReg()) {
905 : // Doesn't zero extend the destination.
906 0 : out->_operands[0]._extendByteMask = 0;
907 : }
908 : }
909 : }
910 0 : else if (instRmInfo.flags & InstDB::RWInfoRm::kFlagPextrw) {
911 0 : if (opCount == 3 && Reg::isMm(operands[1])) {
912 0 : out->_rmFeature = 0;
913 : rmOpsMask = 0;
914 : }
915 : }
916 0 : else if (instRmInfo.flags & InstDB::RWInfoRm::kFlagFeatureIfRMI) {
917 0 : if (opCount != 3 || !operands[2].isImm()) {
918 0 : out->_rmFeature = 0;
919 : }
920 : }
921 : }
922 :
923 216080 : rmOpsMask &= instRmInfo.rmOpsMask;
924 216080 : if (rmOpsMask) {
925 : Support::BitWordIterator<uint32_t> it(rmOpsMask);
926 : do {
927 : i = it.next();
928 :
929 114299 : OpRWInfo& op = out->_operands[i];
930 : op.addOpFlags(RegM);
931 :
932 114299 : switch (instRmInfo.category) {
933 97964 : case InstDB::RWInfoRm::kCategoryFixed:
934 97964 : op.setRmSize(instRmInfo.fixedSize);
935 : break;
936 16335 : case InstDB::RWInfoRm::kCategoryConsistent:
937 16335 : op.setRmSize(operands[i].size());
938 : break;
939 0 : case InstDB::RWInfoRm::kCategoryHalf:
940 0 : op.setRmSize(rmMaxSize / 2u);
941 : break;
942 0 : case InstDB::RWInfoRm::kCategoryQuarter:
943 0 : op.setRmSize(rmMaxSize / 4u);
944 : break;
945 0 : case InstDB::RWInfoRm::kCategoryEighth:
946 0 : op.setRmSize(rmMaxSize / 8u);
947 : break;
948 : }
949 114299 : } while (it.hasNext());
950 : }
951 :
952 : return rwHandleAVX512(inst, commonInfo, out);
953 : }
954 :
955 88236 : switch (instRwInfo.category) {
956 88236 : case InstDB::RWInfo::kCategoryMov: {
957 : // Special case for 'mov' instruction. Here there are some variants that we have to handle as 'mov' can be
958 : // used to move between GP, segment, control and debug registers. Moving between GP registers also allow to
959 : // use memory operand.
960 :
961 : // We will again set the flag if it's actually a move from GP to GP register, otherwise this flag cannot be set.
962 : out->_instFlags &= ~InstRWFlags::kMovOp;
963 :
964 88236 : if (opCount == 2) {
965 88236 : if (operands[0].isReg() && operands[1].isReg()) {
966 : const Reg& o0 = operands[0].as<Reg>();
967 : const Reg& o1 = operands[1].as<Reg>();
968 :
969 0 : if (o0.isGp() && o1.isGp()) {
970 : out->_operands[0].reset(W | RegM, operands[0].size());
971 : out->_operands[1].reset(R | RegM, operands[1].size());
972 :
973 : rwZeroExtendGp(out->_operands[0], operands[0].as<Gp>(), nativeGpSize);
974 : out->_instFlags |= InstRWFlags::kMovOp;
975 0 : return kErrorOk;
976 : }
977 :
978 0 : if (o0.isGp() && o1.isSReg()) {
979 : out->_operands[0].reset(W | RegM, nativeGpSize);
980 : out->_operands[0].setRmSize(2);
981 : out->_operands[1].reset(R, 2);
982 0 : return kErrorOk;
983 : }
984 :
985 0 : if (o0.isSReg() && o1.isGp()) {
986 : out->_operands[0].reset(W, 2);
987 : out->_operands[1].reset(R | RegM, 2);
988 : out->_operands[1].setRmSize(2);
989 0 : return kErrorOk;
990 : }
991 :
992 0 : if (o0.isGp() && (o1.isCReg() || o1.isDReg())) {
993 : out->_operands[0].reset(W, nativeGpSize);
994 : out->_operands[1].reset(R, nativeGpSize);
995 0 : out->_writeFlags = CpuRWFlags::kX86_OF |
996 : CpuRWFlags::kX86_SF |
997 : CpuRWFlags::kX86_ZF |
998 : CpuRWFlags::kX86_AF |
999 : CpuRWFlags::kX86_PF |
1000 : CpuRWFlags::kX86_CF;
1001 0 : return kErrorOk;
1002 : }
1003 :
1004 0 : if ((o0.isCReg() || o0.isDReg()) && o1.isGp()) {
1005 : out->_operands[0].reset(W, nativeGpSize);
1006 : out->_operands[1].reset(R, nativeGpSize);
1007 0 : out->_writeFlags = CpuRWFlags::kX86_OF |
1008 : CpuRWFlags::kX86_SF |
1009 : CpuRWFlags::kX86_ZF |
1010 : CpuRWFlags::kX86_AF |
1011 : CpuRWFlags::kX86_PF |
1012 : CpuRWFlags::kX86_CF;
1013 0 : return kErrorOk;
1014 : }
1015 : }
1016 :
1017 88236 : if (operands[0].isReg() && operands[1].isMem()) {
1018 : const Reg& o0 = operands[0].as<Reg>();
1019 : const Mem& o1 = operands[1].as<Mem>();
1020 :
1021 0 : if (o0.isGp()) {
1022 0 : if (!o1.isOffset64Bit())
1023 : out->_operands[0].reset(W, o0.size());
1024 : else
1025 : out->_operands[0].reset(W | RegPhys, o0.size(), Gp::kIdAx);
1026 :
1027 : out->_operands[1].reset(R | MibRead, o0.size());
1028 : rwZeroExtendGp(out->_operands[0], operands[0].as<Gp>(), nativeGpSize);
1029 0 : return kErrorOk;
1030 : }
1031 :
1032 0 : if (o0.isSReg()) {
1033 : out->_operands[0].reset(W, 2);
1034 : out->_operands[1].reset(R, 2);
1035 0 : return kErrorOk;
1036 : }
1037 : }
1038 :
1039 88236 : if (operands[0].isMem() && operands[1].isReg()) {
1040 : const Mem& o0 = operands[0].as<Mem>();
1041 : const Reg& o1 = operands[1].as<Reg>();
1042 :
1043 0 : if (o1.isGp()) {
1044 : out->_operands[0].reset(W | MibRead, o1.size());
1045 0 : if (!o0.isOffset64Bit())
1046 : out->_operands[1].reset(R, o1.size());
1047 : else
1048 : out->_operands[1].reset(R | RegPhys, o1.size(), Gp::kIdAx);
1049 0 : return kErrorOk;
1050 : }
1051 :
1052 0 : if (o1.isSReg()) {
1053 : out->_operands[0].reset(W | MibRead, 2);
1054 : out->_operands[1].reset(R, 2);
1055 0 : return kErrorOk;
1056 : }
1057 : }
1058 :
1059 88236 : if (Reg::isGp(operands[0]) && operands[1].isImm()) {
1060 : const Reg& o0 = operands[0].as<Reg>();
1061 : out->_operands[0].reset(W | RegM, o0.size());
1062 88236 : out->_operands[1].reset();
1063 :
1064 : rwZeroExtendGp(out->_operands[0], operands[0].as<Gp>(), nativeGpSize);
1065 88236 : return kErrorOk;
1066 : }
1067 :
1068 0 : if (operands[0].isMem() && operands[1].isImm()) {
1069 : const Reg& o0 = operands[0].as<Reg>();
1070 : out->_operands[0].reset(W | MibRead, o0.size());
1071 0 : out->_operands[1].reset();
1072 0 : return kErrorOk;
1073 : }
1074 : }
1075 : break;
1076 : }
1077 :
1078 0 : case InstDB::RWInfo::kCategoryMovabs: {
1079 0 : if (opCount == 2) {
1080 0 : if (Reg::isGp(operands[0]) && operands[1].isMem()) {
1081 : const Reg& o0 = operands[0].as<Reg>();
1082 : out->_operands[0].reset(W | RegPhys, o0.size(), Gp::kIdAx);
1083 : out->_operands[1].reset(R | MibRead, o0.size());
1084 : rwZeroExtendGp(out->_operands[0], operands[0].as<Gp>(), nativeGpSize);
1085 0 : return kErrorOk;
1086 : }
1087 :
1088 0 : if (operands[0].isMem() && Reg::isGp(operands[1])) {
1089 : const Reg& o1 = operands[1].as<Reg>();
1090 : out->_operands[0].reset(W | MibRead, o1.size());
1091 : out->_operands[1].reset(R | RegPhys, o1.size(), Gp::kIdAx);
1092 0 : return kErrorOk;
1093 : }
1094 :
1095 0 : if (Reg::isGp(operands[0]) && operands[1].isImm()) {
1096 : const Reg& o0 = operands[0].as<Reg>();
1097 : out->_operands[0].reset(W, o0.size());
1098 0 : out->_operands[1].reset();
1099 :
1100 : rwZeroExtendGp(out->_operands[0], operands[0].as<Gp>(), nativeGpSize);
1101 0 : return kErrorOk;
1102 : }
1103 : }
1104 : break;
1105 : }
1106 :
1107 0 : case InstDB::RWInfo::kCategoryImul: {
1108 : // Special case for 'imul' instruction.
1109 : //
1110 : // There are 3 variants in general:
1111 : //
1112 : // 1. Standard multiplication: 'A = A * B'.
1113 : // 2. Multiplication with imm: 'A = B * C'.
1114 : // 3. Extended multiplication: 'A:B = B * C'.
1115 :
1116 0 : if (opCount == 2) {
1117 0 : if (operands[0].isReg() && operands[1].isImm()) {
1118 : out->_operands[0].reset(X, operands[0].size());
1119 0 : out->_operands[1].reset();
1120 :
1121 : rwZeroExtendGp(out->_operands[0], operands[0].as<Gp>(), nativeGpSize);
1122 0 : return kErrorOk;
1123 : }
1124 :
1125 0 : if (Reg::isGpw(operands[0]) && operands[1].size() == 1) {
1126 : // imul ax, r8/m8 <- AX = AL * r8/m8
1127 : out->_operands[0].reset(X | RegPhys, 2, Gp::kIdAx);
1128 : out->_operands[0].setReadByteMask(Support::lsbMask<uint64_t>(1));
1129 : out->_operands[1].reset(R | RegM, 1);
1130 : }
1131 : else {
1132 : // imul r?, r?/m?
1133 : out->_operands[0].reset(X, operands[0].size());
1134 : out->_operands[1].reset(R | RegM, operands[0].size());
1135 : rwZeroExtendGp(out->_operands[0], operands[0].as<Gp>(), nativeGpSize);
1136 : }
1137 :
1138 0 : if (operands[1].isMem())
1139 : out->_operands[1].addOpFlags(MibRead);
1140 0 : return kErrorOk;
1141 : }
1142 :
1143 0 : if (opCount == 3) {
1144 0 : if (operands[2].isImm()) {
1145 : out->_operands[0].reset(W, operands[0].size());
1146 : out->_operands[1].reset(R | RegM, operands[1].size());
1147 0 : out->_operands[2].reset();
1148 :
1149 : rwZeroExtendGp(out->_operands[0], operands[0].as<Gp>(), nativeGpSize);
1150 0 : if (operands[1].isMem())
1151 : out->_operands[1].addOpFlags(MibRead);
1152 0 : return kErrorOk;
1153 : }
1154 : else {
1155 : out->_operands[0].reset(W | RegPhys, operands[0].size(), Gp::kIdDx);
1156 : out->_operands[1].reset(X | RegPhys, operands[1].size(), Gp::kIdAx);
1157 : out->_operands[2].reset(R | RegM, operands[2].size());
1158 :
1159 : rwZeroExtendGp(out->_operands[0], operands[0].as<Gp>(), nativeGpSize);
1160 : rwZeroExtendGp(out->_operands[1], operands[1].as<Gp>(), nativeGpSize);
1161 0 : if (operands[2].isMem())
1162 : out->_operands[2].addOpFlags(MibRead);
1163 0 : return kErrorOk;
1164 : }
1165 : }
1166 : break;
1167 : }
1168 :
1169 0 : case InstDB::RWInfo::kCategoryMovh64: {
1170 : // Special case for 'movhpd|movhps' instructions. Note that this is only required for legacy (non-AVX)
1171 : // variants as AVX instructions use either 2 or 3 operands that are in `kCategoryGeneric` category.
1172 0 : if (opCount == 2) {
1173 0 : if (BaseReg::isVec(operands[0]) && operands[1].isMem()) {
1174 : out->_operands[0].reset(W, 8);
1175 : out->_operands[0].setWriteByteMask(Support::lsbMask<uint64_t>(8) << 8);
1176 : out->_operands[1].reset(R | MibRead, 8);
1177 0 : return kErrorOk;
1178 : }
1179 :
1180 0 : if (operands[0].isMem() && BaseReg::isVec(operands[1])) {
1181 : out->_operands[0].reset(W | MibRead, 8);
1182 : out->_operands[1].reset(R, 8);
1183 : out->_operands[1].setReadByteMask(Support::lsbMask<uint64_t>(8) << 8);
1184 0 : return kErrorOk;
1185 : }
1186 : }
1187 : break;
1188 : }
1189 :
1190 0 : case InstDB::RWInfo::kCategoryPunpcklxx: {
1191 : // Special case for 'punpcklbw|punpckldq|punpcklwd' instructions.
1192 0 : if (opCount == 2) {
1193 0 : if (Reg::isXmm(operands[0])) {
1194 : out->_operands[0].reset(X, 16);
1195 : out->_operands[0].setReadByteMask(0x0F0Fu);
1196 : out->_operands[0].setWriteByteMask(0xFFFFu);
1197 : out->_operands[1].reset(R, 16);
1198 : out->_operands[1].setWriteByteMask(0x0F0Fu);
1199 :
1200 0 : if (Reg::isXmm(operands[1])) {
1201 : return kErrorOk;
1202 : }
1203 :
1204 0 : if (operands[1].isMem()) {
1205 : out->_operands[1].addOpFlags(MibRead);
1206 0 : return kErrorOk;
1207 : }
1208 : }
1209 :
1210 0 : if (Reg::isMm(operands[0])) {
1211 : out->_operands[0].reset(X, 8);
1212 : out->_operands[0].setReadByteMask(0x0Fu);
1213 : out->_operands[0].setWriteByteMask(0xFFu);
1214 : out->_operands[1].reset(R, 4);
1215 : out->_operands[1].setReadByteMask(0x0Fu);
1216 :
1217 0 : if (Reg::isMm(operands[1])) {
1218 : return kErrorOk;
1219 : }
1220 :
1221 0 : if (operands[1].isMem()) {
1222 : out->_operands[1].addOpFlags(MibRead);
1223 0 : return kErrorOk;
1224 : }
1225 : }
1226 : }
1227 : break;
1228 : }
1229 :
1230 0 : case InstDB::RWInfo::kCategoryVmaskmov: {
1231 : // Special case for 'vmaskmovpd|vmaskmovps|vpmaskmovd|vpmaskmovq' instructions.
1232 0 : if (opCount == 3) {
1233 0 : if (BaseReg::isVec(operands[0]) && BaseReg::isVec(operands[1]) && operands[2].isMem()) {
1234 : out->_operands[0].reset(W, operands[0].size());
1235 : out->_operands[1].reset(R, operands[1].size());
1236 : out->_operands[2].reset(R | MibRead, operands[1].size());
1237 :
1238 : rwZeroExtendAvxVec(out->_operands[0], operands[0].as<Vec>());
1239 0 : return kErrorOk;
1240 : }
1241 :
1242 0 : if (operands[0].isMem() && BaseReg::isVec(operands[1]) && BaseReg::isVec(operands[2])) {
1243 : out->_operands[0].reset(X | MibRead, operands[1].size());
1244 : out->_operands[1].reset(R, operands[1].size());
1245 : out->_operands[2].reset(R, operands[2].size());
1246 0 : return kErrorOk;
1247 : }
1248 : }
1249 : break;
1250 : }
1251 :
1252 0 : case InstDB::RWInfo::kCategoryVmovddup: {
1253 : // Special case for 'vmovddup' instruction. This instruction has an interesting semantic as 128-bit XMM
1254 : // version only uses 64-bit memory operand (m64), however, 256/512-bit versions use 256/512-bit memory
1255 : // operand, respectively.
1256 0 : if (opCount == 2) {
1257 0 : if (BaseReg::isVec(operands[0]) && BaseReg::isVec(operands[1])) {
1258 : uint32_t o0Size = operands[0].size();
1259 0 : uint32_t o1Size = o0Size == 16 ? 8 : o0Size;
1260 :
1261 : out->_operands[0].reset(W, o0Size);
1262 : out->_operands[1].reset(R | RegM, o1Size);
1263 0 : out->_operands[1]._readByteMask &= 0x00FF00FF00FF00FFu;
1264 :
1265 : rwZeroExtendAvxVec(out->_operands[0], operands[0].as<Vec>());
1266 : return rwHandleAVX512(inst, commonInfo, out);
1267 : }
1268 :
1269 0 : if (BaseReg::isVec(operands[0]) && operands[1].isMem()) {
1270 : uint32_t o0Size = operands[0].size();
1271 0 : uint32_t o1Size = o0Size == 16 ? 8 : o0Size;
1272 :
1273 : out->_operands[0].reset(W, o0Size);
1274 : out->_operands[1].reset(R | MibRead, o1Size);
1275 :
1276 : rwZeroExtendAvxVec(out->_operands[0], operands[0].as<Vec>());
1277 : return rwHandleAVX512(inst, commonInfo, out);
1278 : }
1279 : }
1280 : break;
1281 : }
1282 :
1283 0 : case InstDB::RWInfo::kCategoryVmovmskpd:
1284 : case InstDB::RWInfo::kCategoryVmovmskps: {
1285 : // Special case for 'vmovmskpd|vmovmskps' instructions.
1286 0 : if (opCount == 2) {
1287 0 : if (BaseReg::isGp(operands[0]) && BaseReg::isVec(operands[1])) {
1288 : out->_operands[0].reset(W, 1);
1289 0 : out->_operands[0].setExtendByteMask(Support::lsbMask<uint32_t>(nativeGpSize - 1) << 1);
1290 : out->_operands[1].reset(R, operands[1].size());
1291 0 : return kErrorOk;
1292 : }
1293 : }
1294 : break;
1295 : }
1296 :
1297 0 : case InstDB::RWInfo::kCategoryVmov1_2:
1298 : case InstDB::RWInfo::kCategoryVmov1_4:
1299 : case InstDB::RWInfo::kCategoryVmov1_8: {
1300 : // Special case for instructions where the destination is 1:N (narrowing).
1301 : //
1302 : // Vmov1_2:
1303 : // vcvtpd2dq|vcvttpd2dq
1304 : // vcvtpd2udq|vcvttpd2udq
1305 : // vcvtpd2ps|vcvtps2ph
1306 : // vcvtqq2ps|vcvtuqq2ps
1307 : // vpmovwb|vpmovswb|vpmovuswb
1308 : // vpmovdw|vpmovsdw|vpmovusdw
1309 : // vpmovqd|vpmovsqd|vpmovusqd
1310 : //
1311 : // Vmov1_4:
1312 : // vpmovdb|vpmovsdb|vpmovusdb
1313 : // vpmovqw|vpmovsqw|vpmovusqw
1314 : //
1315 : // Vmov1_8:
1316 : // pmovmskb|vpmovmskb
1317 : // vpmovqb|vpmovsqb|vpmovusqb
1318 0 : uint32_t shift = instRwInfo.category - InstDB::RWInfo::kCategoryVmov1_2 + 1;
1319 :
1320 0 : if (opCount >= 2) {
1321 0 : if (opCount >= 3) {
1322 0 : if (opCount > 3)
1323 : return DebugUtils::errored(kErrorInvalidInstruction);
1324 0 : out->_operands[2].reset();
1325 : }
1326 :
1327 0 : if (operands[0].isReg() && operands[1].isReg()) {
1328 : uint32_t size1 = operands[1].size();
1329 0 : uint32_t size0 = size1 >> shift;
1330 :
1331 : out->_operands[0].reset(W, size0);
1332 : out->_operands[1].reset(R, size1);
1333 :
1334 0 : if (instRmInfo.rmOpsMask & 0x1) {
1335 : out->_operands[0].addOpFlags(RegM);
1336 : out->_operands[0].setRmSize(size0);
1337 : }
1338 :
1339 0 : if (instRmInfo.rmOpsMask & 0x2) {
1340 : out->_operands[1].addOpFlags(RegM);
1341 : out->_operands[1].setRmSize(size1);
1342 : }
1343 :
1344 : // Handle 'pmovmskb|vpmovmskb'.
1345 0 : if (BaseReg::isGp(operands[0]))
1346 : rwZeroExtendGp(out->_operands[0], operands[0].as<Gp>(), nativeGpSize);
1347 :
1348 0 : if (BaseReg::isVec(operands[0]))
1349 : rwZeroExtendAvxVec(out->_operands[0], operands[0].as<Vec>());
1350 :
1351 : return rwHandleAVX512(inst, commonInfo, out);
1352 : }
1353 :
1354 0 : if (operands[0].isReg() && operands[1].isMem()) {
1355 0 : uint32_t size1 = operands[1].size() ? operands[1].size() : uint32_t(16);
1356 0 : uint32_t size0 = size1 >> shift;
1357 :
1358 : out->_operands[0].reset(W, size0);
1359 : out->_operands[1].reset(R | MibRead, size1);
1360 0 : return kErrorOk;
1361 : }
1362 :
1363 0 : if (operands[0].isMem() && operands[1].isReg()) {
1364 : uint32_t size1 = operands[1].size();
1365 0 : uint32_t size0 = size1 >> shift;
1366 :
1367 : out->_operands[0].reset(W | MibRead, size0);
1368 : out->_operands[1].reset(R, size1);
1369 :
1370 : return rwHandleAVX512(inst, commonInfo, out);
1371 : }
1372 : }
1373 : break;
1374 : }
1375 :
1376 0 : case InstDB::RWInfo::kCategoryVmov2_1:
1377 : case InstDB::RWInfo::kCategoryVmov4_1:
1378 : case InstDB::RWInfo::kCategoryVmov8_1: {
1379 : // Special case for instructions where the destination is N:1 (widening).
1380 : //
1381 : // Vmov2_1:
1382 : // vcvtdq2pd|vcvtudq2pd
1383 : // vcvtps2pd|vcvtph2ps
1384 : // vcvtps2qq|vcvtps2uqq
1385 : // vcvttps2qq|vcvttps2uqq
1386 : // vpmovsxbw|vpmovzxbw
1387 : // vpmovsxwd|vpmovzxwd
1388 : // vpmovsxdq|vpmovzxdq
1389 : //
1390 : // Vmov4_1:
1391 : // vpmovsxbd|vpmovzxbd
1392 : // vpmovsxwq|vpmovzxwq
1393 : //
1394 : // Vmov8_1:
1395 : // vpmovsxbq|vpmovzxbq
1396 0 : uint32_t shift = instRwInfo.category - InstDB::RWInfo::kCategoryVmov2_1 + 1;
1397 :
1398 0 : if (opCount >= 2) {
1399 0 : if (opCount >= 3) {
1400 0 : if (opCount > 3)
1401 : return DebugUtils::errored(kErrorInvalidInstruction);
1402 0 : out->_operands[2].reset();
1403 : }
1404 :
1405 : uint32_t size0 = operands[0].size();
1406 0 : uint32_t size1 = size0 >> shift;
1407 :
1408 : out->_operands[0].reset(W, size0);
1409 : out->_operands[1].reset(R, size1);
1410 :
1411 0 : if (operands[0].isReg() && operands[1].isReg()) {
1412 0 : if (instRmInfo.rmOpsMask & 0x1) {
1413 : out->_operands[0].addOpFlags(RegM);
1414 : out->_operands[0].setRmSize(size0);
1415 : }
1416 :
1417 0 : if (instRmInfo.rmOpsMask & 0x2) {
1418 : out->_operands[1].addOpFlags(RegM);
1419 : out->_operands[1].setRmSize(size1);
1420 : }
1421 :
1422 : return rwHandleAVX512(inst, commonInfo, out);
1423 : }
1424 :
1425 0 : if (operands[0].isReg() && operands[1].isMem()) {
1426 : out->_operands[1].addOpFlags(MibRead);
1427 :
1428 : return rwHandleAVX512(inst, commonInfo, out);
1429 : }
1430 : }
1431 : break;
1432 : }
1433 : }
1434 :
1435 : return DebugUtils::errored(kErrorInvalidInstruction);
1436 : }
1437 : #endif // !ASMJIT_NO_INTROSPECTION
1438 :
1439 : // x86::InstInternal - QueryFeatures
1440 : // =================================
1441 :
1442 : #ifndef ASMJIT_NO_INTROSPECTION
1443 : struct RegAnalysis {
1444 : uint32_t regTypeMask;
1445 : uint32_t highVecUsed;
1446 :
1447 : inline bool hasRegType(RegType regType) const noexcept {
1448 : return Support::bitTest(regTypeMask, regType);
1449 : }
1450 : };
1451 :
1452 0 : static RegAnalysis InstInternal_regAnalysis(const Operand_* operands, size_t opCount) noexcept {
1453 : uint32_t mask = 0;
1454 : uint32_t highVecUsed = 0;
1455 :
1456 0 : for (uint32_t i = 0; i < opCount; i++) {
1457 0 : const Operand_& op = operands[i];
1458 0 : if (op.isReg()) {
1459 : const BaseReg& reg = op.as<BaseReg>();
1460 0 : mask |= Support::bitMask(reg.type());
1461 0 : if (reg.isVec())
1462 0 : highVecUsed |= uint32_t(reg.id() >= 16 && reg.id() < 32);
1463 : }
1464 0 : else if (op.isMem()) {
1465 : const BaseMem& mem = op.as<BaseMem>();
1466 0 : if (mem.hasBaseReg()) mask |= Support::bitMask(mem.baseType());
1467 0 : if (mem.hasIndexReg()) {
1468 0 : mask |= Support::bitMask(mem.indexType());
1469 0 : highVecUsed |= uint32_t(mem.indexId() >= 16 && mem.indexId() < 32);
1470 : }
1471 : }
1472 : }
1473 :
1474 0 : return RegAnalysis { mask, highVecUsed };
1475 : }
1476 :
1477 : static inline uint32_t InstInternal_usesAvx512(InstOptions instOptions, const RegOnly& extraReg, const RegAnalysis& regAnalysis) noexcept {
1478 : uint32_t hasEvex = uint32_t(instOptions & (InstOptions::kX86_Evex | InstOptions::kX86_AVX512Mask));
1479 0 : uint32_t hasKMask = extraReg.type() == RegType::kX86_KReg;
1480 0 : uint32_t hasKOrZmm = regAnalysis.regTypeMask & Support::bitMask(RegType::kX86_Zmm, RegType::kX86_KReg);
1481 :
1482 0 : return hasEvex | hasKMask | hasKOrZmm;
1483 : }
1484 :
1485 0 : Error InstInternal::queryFeatures(Arch arch, const BaseInst& inst, const Operand_* operands, size_t opCount, CpuFeatures* out) noexcept {
1486 : // Only called when `arch` matches X86 family.
1487 : DebugUtils::unused(arch);
1488 : ASMJIT_ASSERT(Environment::isFamilyX86(arch));
1489 :
1490 : // Get the instruction data.
1491 : InstId instId = inst.id();
1492 : InstOptions options = inst.options();
1493 :
1494 0 : if (ASMJIT_UNLIKELY(!Inst::isDefinedId(instId)))
1495 : return DebugUtils::errored(kErrorInvalidInstruction);
1496 :
1497 : const InstDB::InstInfo& instInfo = InstDB::infoById(instId);
1498 0 : const InstDB::AdditionalInfo& additionalInfo = InstDB::_additionalInfoTable[instInfo._additionalInfoIndex];
1499 :
1500 : const uint8_t* fData = additionalInfo.featuresBegin();
1501 : const uint8_t* fEnd = additionalInfo.featuresEnd();
1502 :
1503 : // Copy all features to `out`.
1504 : out->reset();
1505 : do {
1506 0 : uint32_t feature = fData[0];
1507 0 : if (!feature)
1508 : break;
1509 0 : out->add(feature);
1510 0 : } while (++fData != fEnd);
1511 :
1512 : // Since AsmJit aggregates instructions that share the same name we have to
1513 : // deal with some special cases and also with MMX/SSE and AVX/AVX2 overlaps.
1514 0 : if (fData != additionalInfo.featuresBegin()) {
1515 0 : RegAnalysis regAnalysis = InstInternal_regAnalysis(operands, opCount);
1516 :
1517 : // Handle MMX vs SSE overlap.
1518 0 : if (out->has(CpuFeatures::X86::kMMX) || out->has(CpuFeatures::X86::kMMX2)) {
1519 : // Only instructions defined by SSE and SSE2 overlap. Instructions introduced by newer instruction sets like
1520 : // SSE3+ don't state MMX as they require SSE3+.
1521 0 : if (out->has(CpuFeatures::X86::kSSE) || out->has(CpuFeatures::X86::kSSE2)) {
1522 0 : if (!regAnalysis.hasRegType(RegType::kX86_Xmm)) {
1523 : // The instruction doesn't use XMM register(s), thus it's MMX/MMX2 only.
1524 0 : out->remove(CpuFeatures::X86::kSSE);
1525 0 : out->remove(CpuFeatures::X86::kSSE2);
1526 0 : out->remove(CpuFeatures::X86::kSSE4_1);
1527 : }
1528 : else {
1529 0 : out->remove(CpuFeatures::X86::kMMX);
1530 0 : out->remove(CpuFeatures::X86::kMMX2);
1531 : }
1532 :
1533 : // Special case: PEXTRW instruction is MMX/SSE2 instruction. However, MMX/SSE version cannot access memory
1534 : // (only register to register extract) so when SSE4.1 introduced the whole family of PEXTR/PINSR instructions
1535 : // they also introduced PEXTRW with a new opcode 0x15 that can extract directly to memory. This instruction
1536 : // is, of course, not compatible with MMX/SSE2 and would #UD if SSE4.1 is not supported.
1537 0 : if (instId == Inst::kIdPextrw) {
1538 0 : if (opCount >= 1 && operands[0].isMem())
1539 0 : out->remove(CpuFeatures::X86::kSSE2);
1540 : else
1541 0 : out->remove(CpuFeatures::X86::kSSE4_1);
1542 : }
1543 : }
1544 : }
1545 :
1546 : // Handle PCLMULQDQ vs VPCLMULQDQ.
1547 0 : if (out->has(CpuFeatures::X86::kVPCLMULQDQ)) {
1548 0 : if (regAnalysis.hasRegType(RegType::kX86_Zmm) || Support::test(options, InstOptions::kX86_Evex)) {
1549 : // AVX512_F & VPCLMULQDQ.
1550 0 : out->remove(CpuFeatures::X86::kAVX, CpuFeatures::X86::kPCLMULQDQ);
1551 : }
1552 0 : else if (regAnalysis.hasRegType(RegType::kX86_Ymm)) {
1553 0 : out->remove(CpuFeatures::X86::kAVX512_F, CpuFeatures::X86::kAVX512_VL);
1554 : }
1555 : else {
1556 : // AVX & PCLMULQDQ.
1557 0 : out->remove(CpuFeatures::X86::kAVX512_F, CpuFeatures::X86::kAVX512_VL, CpuFeatures::X86::kVPCLMULQDQ);
1558 : }
1559 : }
1560 :
1561 : // Handle AVX vs AVX2 overlap.
1562 0 : if (out->has(CpuFeatures::X86::kAVX) && out->has(CpuFeatures::X86::kAVX2)) {
1563 : bool isAVX2 = true;
1564 : // Special case: VBROADCASTSS and VBROADCASTSD were introduced in AVX, but only version that uses memory as a
1565 : // source operand. AVX2 then added support for register source operand.
1566 0 : if (instId == Inst::kIdVbroadcastss || instId == Inst::kIdVbroadcastsd) {
1567 0 : if (opCount > 1 && operands[1].isMem())
1568 : isAVX2 = false;
1569 : }
1570 : else {
1571 : // AVX instruction set doesn't support integer operations on YMM registers as these were later introcuced by
1572 : // AVX2. In our case we have to check if YMM register(s) are in use and if that is the case this is an AVX2
1573 : // instruction.
1574 0 : if (!(regAnalysis.regTypeMask & Support::bitMask(RegType::kX86_Ymm, RegType::kX86_Zmm)))
1575 : isAVX2 = false;
1576 : }
1577 :
1578 : if (isAVX2)
1579 0 : out->remove(CpuFeatures::X86::kAVX);
1580 : else
1581 0 : out->remove(CpuFeatures::X86::kAVX2);
1582 : }
1583 :
1584 : // Handle AVX|AVX2|FMA|F16C vs AVX512 overlap.
1585 0 : if (out->has(CpuFeatures::X86::kAVX) || out->has(CpuFeatures::X86::kAVX2) || out->has(CpuFeatures::X86::kFMA) || out->has(CpuFeatures::X86::kF16C)) {
1586 : // Only AVX512-F|BW|DQ allow to encode AVX/AVX2/FMA/F16C instructions
1587 0 : if (out->has(CpuFeatures::X86::kAVX512_F) || out->has(CpuFeatures::X86::kAVX512_BW) || out->has(CpuFeatures::X86::kAVX512_DQ)) {
1588 : uint32_t usesAvx512 = InstInternal_usesAvx512(options, inst.extraReg(), regAnalysis);
1589 : uint32_t mustUseEvex = 0;
1590 :
1591 0 : switch (instId) {
1592 : // Special case: VPSLLDQ and VPSRLDQ instructions only allow `reg, reg. imm` combination in AVX|AVX2 mode,
1593 : // then AVX-512 introduced `reg, reg/mem, imm` combination that uses EVEX prefix. This means that if the
1594 : // second operand is memory then this is AVX-512_BW instruction and not AVX/AVX2 instruction.
1595 0 : case Inst::kIdVpslldq:
1596 : case Inst::kIdVpsrldq:
1597 0 : mustUseEvex = opCount >= 2 && operands[1].isMem();
1598 0 : break;
1599 :
1600 : // Special case: VPBROADCAST[B|D|Q|W] only supports r32/r64 with EVEX prefix.
1601 0 : case Inst::kIdVpbroadcastb:
1602 : case Inst::kIdVpbroadcastd:
1603 : case Inst::kIdVpbroadcastq:
1604 : case Inst::kIdVpbroadcastw:
1605 0 : mustUseEvex = opCount >= 2 && x86::Reg::isGp(operands[1]);
1606 0 : break;
1607 :
1608 : // Special case: VPERMPD - AVX2 vs AVX512-F case.
1609 0 : case Inst::kIdVpermpd:
1610 0 : mustUseEvex = opCount >= 3 && !operands[2].isImm();
1611 0 : break;
1612 :
1613 : // Special case: VPERMQ - AVX2 vs AVX512-F case.
1614 0 : case Inst::kIdVpermq:
1615 0 : mustUseEvex = opCount >= 3 && (operands[1].isMem() || !operands[2].isImm());
1616 0 : break;
1617 : }
1618 :
1619 0 : if (!(usesAvx512 | mustUseEvex | regAnalysis.highVecUsed))
1620 0 : out->remove(CpuFeatures::X86::kAVX512_F, CpuFeatures::X86::kAVX512_BW, CpuFeatures::X86::kAVX512_DQ, CpuFeatures::X86::kAVX512_VL);
1621 : else
1622 0 : out->remove(CpuFeatures::X86::kAVX, CpuFeatures::X86::kAVX2, CpuFeatures::X86::kFMA, CpuFeatures::X86::kF16C);
1623 : }
1624 : }
1625 :
1626 : // Handle AVX_VNNI vs AVX512_VNNI overlap.
1627 0 : if (out->has(CpuFeatures::X86::kAVX512_VNNI)) {
1628 : // By default the AVX512_VNNI instruction should be used, because it was introduced first. However, VEX|VEX3
1629 : // prefix can be used to force AVX_VNNI instead.
1630 : uint32_t usesAvx512 = InstInternal_usesAvx512(options, inst.extraReg(), regAnalysis);
1631 :
1632 0 : if (!usesAvx512 && Support::test(options, InstOptions::kX86_Vex | InstOptions::kX86_Vex3))
1633 0 : out->remove(CpuFeatures::X86::kAVX512_VNNI, CpuFeatures::X86::kAVX512_VL);
1634 : else
1635 0 : out->remove(CpuFeatures::X86::kAVX_VNNI);
1636 : }
1637 :
1638 : // Clear AVX512_VL if ZMM register is used.
1639 0 : if (regAnalysis.hasRegType(RegType::kX86_Zmm))
1640 0 : out->remove(CpuFeatures::X86::kAVX512_VL);
1641 : }
1642 :
1643 : return kErrorOk;
1644 : }
1645 : #endif // !ASMJIT_NO_INTROSPECTION
1646 :
1647 : // x86::InstInternal - Tests
1648 : // =========================
1649 :
1650 : #if defined(ASMJIT_TEST)
1651 : UNIT(x86_inst_api_text) {
1652 : // All known instructions should be matched.
1653 : INFO("Matching all X86 instructions");
1654 : for (uint32_t a = 1; a < Inst::_kIdCount; a++) {
1655 : StringTmp<128> aName;
1656 : EXPECT(InstInternal::instIdToString(Arch::kX86, a, aName) == kErrorOk,
1657 : "Failed to get the name of instruction #%u", a);
1658 :
1659 : uint32_t b = InstInternal::stringToInstId(Arch::kX86, aName.data(), aName.size());
1660 : StringTmp<128> bName;
1661 : InstInternal::instIdToString(Arch::kX86, b, bName);
1662 :
1663 : EXPECT(a == b,
1664 : "Instructions do not match \"%s\" (#%u) != \"%s\" (#%u)", aName.data(), a, bName.data(), b);
1665 : }
1666 : }
1667 :
1668 : template<typename... Args>
1669 : static Error queryRWInfoSimple(InstRWInfo* out, Arch arch, InstId instId, InstOptions options, Args&&... args) {
1670 : BaseInst inst(instId);
1671 : inst.addOptions(options);
1672 : Operand_ opArray[] = { std::forward<Args>(args)... };
1673 : return InstInternal::queryRWInfo(arch, inst, opArray, sizeof...(args), out);
1674 : }
1675 :
1676 : UNIT(x86_inst_api_rm_feature) {
1677 : INFO("Verifying whether RM/feature is reported correctly for PEXTRW instruction");
1678 : {
1679 : InstRWInfo rwi;
1680 :
1681 : queryRWInfoSimple(&rwi, Arch::kX64, Inst::kIdPextrw, InstOptions::kNone, eax, mm1, imm(1));
1682 : EXPECT(rwi.rmFeature() == 0);
1683 :
1684 : queryRWInfoSimple(&rwi, Arch::kX64, Inst::kIdPextrw, InstOptions::kNone, eax, xmm1, imm(1));
1685 : EXPECT(rwi.rmFeature() == CpuFeatures::X86::kSSE4_1);
1686 : }
1687 :
1688 : INFO("Verifying whether RM/feature is reported correctly for AVX512 shift instructions");
1689 : {
1690 : InstRWInfo rwi;
1691 :
1692 : queryRWInfoSimple(&rwi, Arch::kX64, Inst::kIdVpslld, InstOptions::kNone, xmm1, xmm2, imm(8));
1693 : EXPECT(rwi.rmFeature() == CpuFeatures::X86::kAVX512_F);
1694 :
1695 : queryRWInfoSimple(&rwi, Arch::kX64, Inst::kIdVpsllq, InstOptions::kNone, ymm1, ymm2, imm(8));
1696 : EXPECT(rwi.rmFeature() == CpuFeatures::X86::kAVX512_F);
1697 :
1698 : queryRWInfoSimple(&rwi, Arch::kX64, Inst::kIdVpsrad, InstOptions::kNone, xmm1, xmm2, imm(8));
1699 : EXPECT(rwi.rmFeature() == CpuFeatures::X86::kAVX512_F);
1700 :
1701 : queryRWInfoSimple(&rwi, Arch::kX64, Inst::kIdVpsrld, InstOptions::kNone, ymm1, ymm2, imm(8));
1702 : EXPECT(rwi.rmFeature() == CpuFeatures::X86::kAVX512_F);
1703 :
1704 : queryRWInfoSimple(&rwi, Arch::kX64, Inst::kIdVpsrlq, InstOptions::kNone, xmm1, xmm2, imm(8));
1705 : EXPECT(rwi.rmFeature() == CpuFeatures::X86::kAVX512_F);
1706 :
1707 : queryRWInfoSimple(&rwi, Arch::kX64, Inst::kIdVpslldq, InstOptions::kNone, xmm1, xmm2, imm(8));
1708 : EXPECT(rwi.rmFeature() == CpuFeatures::X86::kAVX512_BW);
1709 :
1710 : queryRWInfoSimple(&rwi, Arch::kX64, Inst::kIdVpsllw, InstOptions::kNone, ymm1, ymm2, imm(8));
1711 : EXPECT(rwi.rmFeature() == CpuFeatures::X86::kAVX512_BW);
1712 :
1713 : queryRWInfoSimple(&rwi, Arch::kX64, Inst::kIdVpsraw, InstOptions::kNone, xmm1, xmm2, imm(8));
1714 : EXPECT(rwi.rmFeature() == CpuFeatures::X86::kAVX512_BW);
1715 :
1716 : queryRWInfoSimple(&rwi, Arch::kX64, Inst::kIdVpsrldq, InstOptions::kNone, ymm1, ymm2, imm(8));
1717 : EXPECT(rwi.rmFeature() == CpuFeatures::X86::kAVX512_BW);
1718 :
1719 : queryRWInfoSimple(&rwi, Arch::kX64, Inst::kIdVpsrlw, InstOptions::kNone, xmm1, xmm2, imm(8));
1720 : EXPECT(rwi.rmFeature() == CpuFeatures::X86::kAVX512_BW);
1721 :
1722 : queryRWInfoSimple(&rwi, Arch::kX64, Inst::kIdVpslld, InstOptions::kNone, xmm1, xmm2, xmm3);
1723 : EXPECT(rwi.rmFeature() == 0);
1724 :
1725 : queryRWInfoSimple(&rwi, Arch::kX64, Inst::kIdVpsllw, InstOptions::kNone, xmm1, xmm2, xmm3);
1726 : EXPECT(rwi.rmFeature() == 0);
1727 : }
1728 : }
1729 : #endif
1730 :
1731 : ASMJIT_END_SUB_NAMESPACE
1732 :
1733 : #endif // !ASMJIT_NO_X86
1734 :
1735 : #endif // __PLUMED_HAS_ASMJIT
|
