Line data Source code
1 : #ifdef __PLUMED_HAS_ASMJIT
2 : // This file is part of AsmJit project <https://asmjit.com>
3 : //
4 : // See asmjit.h or LICENSE.md for license and copyright information
5 : // SPDX-License-Identifier: Zlib
6 :
7 : #ifndef ASMJIT_CORE_SUPPORT_H_INCLUDED
8 : #define ASMJIT_CORE_SUPPORT_H_INCLUDED
9 :
10 : #include "globals.h"
11 :
12 : #if defined(_MSC_VER)
13 : #include <intrin.h>
14 : #endif
15 :
16 : ASMJIT_BEGIN_NAMESPACE
17 :
18 : //! \addtogroup asmjit_utilities
19 : //! \{
20 :
21 : //! Contains support classes and functions that may be used by AsmJit source and header files. Anything defined
22 : //! here is considered internal and should not be used outside of AsmJit and related projects like AsmTK.
23 : namespace Support {
24 :
25 : // Support - Basic Traits
26 : // ======================
27 :
28 : #if ASMJIT_ARCH_X86
29 : typedef uint8_t FastUInt8;
30 : #else
31 : typedef uint32_t FastUInt8;
32 : #endif
33 :
34 : //! \cond INTERNAL
35 : namespace Internal {
36 : template<typename T, size_t Alignment>
37 : struct AliasedUInt {};
38 :
39 : template<> struct AliasedUInt<uint16_t, 2> { typedef uint16_t ASMJIT_MAY_ALIAS T; };
40 : template<> struct AliasedUInt<uint32_t, 4> { typedef uint32_t ASMJIT_MAY_ALIAS T; };
41 : template<> struct AliasedUInt<uint64_t, 8> { typedef uint64_t ASMJIT_MAY_ALIAS T; };
42 :
43 : template<> struct AliasedUInt<uint16_t, 1> { typedef uint16_t ASMJIT_MAY_ALIAS ASMJIT_ALIGN_TYPE(T, 1); };
44 : template<> struct AliasedUInt<uint32_t, 1> { typedef uint32_t ASMJIT_MAY_ALIAS ASMJIT_ALIGN_TYPE(T, 1); };
45 : template<> struct AliasedUInt<uint32_t, 2> { typedef uint32_t ASMJIT_MAY_ALIAS ASMJIT_ALIGN_TYPE(T, 2); };
46 : template<> struct AliasedUInt<uint64_t, 1> { typedef uint64_t ASMJIT_MAY_ALIAS ASMJIT_ALIGN_TYPE(T, 1); };
47 : template<> struct AliasedUInt<uint64_t, 2> { typedef uint64_t ASMJIT_MAY_ALIAS ASMJIT_ALIGN_TYPE(T, 2); };
48 : template<> struct AliasedUInt<uint64_t, 4> { typedef uint64_t ASMJIT_MAY_ALIAS ASMJIT_ALIGN_TYPE(T, 4); };
49 :
50 : // StdInt - Make an int-type by size (signed or unsigned) that is the
51 : // same as types defined by <stdint.h>.
52 : // Int32Or64 - Make an int-type that has at least 32 bits: [u]int[32|64]_t.
53 :
54 : template<size_t Size, unsigned Unsigned>
55 : struct StdInt {}; // Fail if not specialized.
56 :
57 : template<> struct StdInt<1, 0> { typedef int8_t Type; };
58 : template<> struct StdInt<1, 1> { typedef uint8_t Type; };
59 : template<> struct StdInt<2, 0> { typedef int16_t Type; };
60 : template<> struct StdInt<2, 1> { typedef uint16_t Type; };
61 : template<> struct StdInt<4, 0> { typedef int32_t Type; };
62 : template<> struct StdInt<4, 1> { typedef uint32_t Type; };
63 : template<> struct StdInt<8, 0> { typedef int64_t Type; };
64 : template<> struct StdInt<8, 1> { typedef uint64_t Type; };
65 :
66 : template<typename T, int Unsigned = std::is_unsigned<T>::value>
67 : struct Int32Or64 : public StdInt<sizeof(T) <= 4 ? size_t(4) : sizeof(T), Unsigned> {};
68 : }
69 : //! \endcond
70 :
71 : template<typename T>
72 : static constexpr bool isUnsigned() noexcept { return std::is_unsigned<T>::value; }
73 :
74 : //! Casts an integer `x` to either `int32_t` or `int64_t` depending on `T`.
75 : template<typename T>
76 : static constexpr typename Internal::Int32Or64<T, 0>::Type asInt(const T& x) noexcept {
77 : return (typename Internal::Int32Or64<T, 0>::Type)x;
78 : }
79 :
80 : //! Casts an integer `x` to either `uint32_t` or `uint64_t` depending on `T`.
81 : template<typename T>
82 : static constexpr typename Internal::Int32Or64<T, 1>::Type asUInt(const T& x) noexcept {
83 0 : return (typename Internal::Int32Or64<T, 1>::Type)x;
84 : }
85 :
86 : //! Casts an integer `x` to either `int32_t`, uint32_t`, `int64_t`, or `uint64_t` depending on `T`.
87 : template<typename T>
88 : static constexpr typename Internal::Int32Or64<T>::Type asNormalized(const T& x) noexcept {
89 : return (typename Internal::Int32Or64<T>::Type)x;
90 : }
91 :
92 : //! Casts an integer `x` to the same type as defined by `<stdint.h>`.
93 : template<typename T>
94 : static constexpr typename Internal::StdInt<sizeof(T), isUnsigned<T>()>::Type asStdInt(const T& x) noexcept {
95 0 : return (typename Internal::StdInt<sizeof(T), isUnsigned<T>()>::Type)x;
96 : }
97 :
98 : //! A helper class that can be used to iterate over enum values.
99 : template<typename T, T from = (T)0, T to = T::kMaxValue>
100 : struct EnumValues {
101 : typedef typename std::underlying_type<T>::type ValueType;
102 :
103 : struct Iterator {
104 : ValueType value;
105 :
106 219936 : inline T operator*() const { return (T)value; }
107 3070292 : inline void operator++() { ++value; }
108 :
109 : inline bool operator==(const Iterator& other) const noexcept { return value == other.value; }
110 274920 : inline bool operator!=(const Iterator& other) const noexcept { return value != other.value; }
111 : };
112 :
113 54984 : inline Iterator begin() const noexcept { return Iterator{ValueType(from)}; }
114 54984 : inline Iterator end() const noexcept { return Iterator{ValueType(to) + 1}; }
115 : };
116 :
117 : // Support - BitCast
118 : // =================
119 :
120 : //! \cond
121 : namespace Internal {
122 : template<typename DstT, typename SrcT>
123 : union BitCastUnion {
124 0 : inline BitCastUnion(SrcT src) noexcept : src(src) {}
125 : SrcT src;
126 : DstT dst;
127 : };
128 : }
129 : //! \endcond
130 :
131 : //! Bit-casts from `Src` type to `Dst` type.
132 : //!
133 : //! Useful to bit-cast between integers and floating points.
134 : template<typename Dst, typename Src>
135 0 : static inline Dst bitCast(const Src& x) noexcept { return Internal::BitCastUnion<Dst, Src>(x).dst; }
136 :
137 : // Support - BitOps
138 : // ================
139 :
140 : //! Storage used to store a pack of bits (should by compatible with a machine word).
141 : typedef Internal::StdInt<sizeof(uintptr_t), 1>::Type BitWord;
142 :
143 : template<typename T>
144 : static constexpr uint32_t bitSizeOf() noexcept { return uint32_t(sizeof(T) * 8u); }
145 :
146 : //! Number of bits stored in a single `BitWord`.
147 : static constexpr uint32_t kBitWordSizeInBits = bitSizeOf<BitWord>();
148 :
149 : //! Returns `0 - x` in a safe way (no undefined behavior), works for unsigned numbers as well.
150 : template<typename T>
151 : static constexpr T neg(const T& x) noexcept {
152 : typedef typename std::make_unsigned<T>::type U;
153 0 : return T(U(0) - U(x));
154 : }
155 :
156 : template<typename T>
157 : static constexpr T allOnes() noexcept { return neg<T>(T(1)); }
158 :
159 : //! Returns `x << y` (shift left logical) by explicitly casting `x` to an unsigned type and back.
160 : template<typename X, typename Y>
161 : static constexpr X shl(const X& x, const Y& y) noexcept {
162 : typedef typename std::make_unsigned<X>::type U;
163 0 : return X(U(x) << y);
164 : }
165 :
166 : //! Returns `x >> y` (shift right logical) by explicitly casting `x` to an unsigned type and back.
167 : template<typename X, typename Y>
168 : static constexpr X shr(const X& x, const Y& y) noexcept {
169 : typedef typename std::make_unsigned<X>::type U;
170 597751 : return X(U(x) >> y);
171 : }
172 :
173 : //! Returns `x >> y` (shift right arithmetic) by explicitly casting `x` to a signed type and back.
174 : template<typename X, typename Y>
175 : static constexpr X sar(const X& x, const Y& y) noexcept {
176 : typedef typename std::make_signed<X>::type S;
177 0 : return X(S(x) >> y);
178 : }
179 :
180 : template<typename X, typename Y>
181 : static constexpr X ror(const X& x, const Y& y) noexcept {
182 : typedef typename std::make_unsigned<X>::type U;
183 : return X((U(x) >> y) | (U(x) << (bitSizeOf<U>() - y)));
184 : }
185 :
186 : //! Returns `x | (x >> y)` - helper used by some bit manipulation helpers.
187 : template<typename X, typename Y>
188 206347 : static constexpr X or_shr(const X& x, const Y& y) noexcept { return X(x | shr(x, y)); }
189 :
190 : //! Returns `x & -x` - extracts lowest set isolated bit (like BLSI instruction).
191 : template<typename T>
192 : static constexpr T blsi(T x) noexcept {
193 : typedef typename std::make_unsigned<T>::type U;
194 0 : return T(U(x) & neg(U(x)));
195 : }
196 :
197 : //! Tests whether the given value `x` has `n`th bit set.
198 : template<typename T, typename IndexT>
199 : static constexpr bool bitTest(T x, IndexT n) noexcept {
200 : typedef typename std::make_unsigned<T>::type U;
201 400449 : return (U(x) & (U(1) << asStdInt(n))) != 0;
202 : }
203 :
204 : // Tests whether the given `value` is a consecutive mask of bits that starts at
205 : // the least significant bit.
206 : template<typename T>
207 : static inline constexpr bool isLsbMask(const T& value) {
208 : typedef typename std::make_unsigned<T>::type U;
209 : return value && ((U(value) + 1u) & U(value)) == 0;
210 : }
211 :
212 : // Tests whether the given value contains at least one bit or whether it's a
213 : // bit-mask of consecutive bits.
214 : //
215 : // This function is similar to \ref isLsbMask(), but the mask doesn't have to
216 : // start at a least significant bit.
217 : template<typename T>
218 : static inline constexpr bool isConsecutiveMask(const T& value) {
219 : typedef typename std::make_unsigned<T>::type U;
220 : return value && isLsbMask((U(value) - 1u) | U(value));
221 : }
222 :
223 : //! Generates a trailing bit-mask that has `n` least significant (trailing) bits set.
224 : template<typename T, typename CountT>
225 : static constexpr T lsbMask(const CountT& n) noexcept {
226 : typedef typename std::make_unsigned<T>::type U;
227 : return (sizeof(U) < sizeof(uintptr_t))
228 : // Prevent undefined behavior by using a larger type than T.
229 512436 : ? T(U((uintptr_t(1) << n) - uintptr_t(1)))
230 : // Prevent undefined behavior by checking `n` before shift.
231 473880 : : n ? T(shr(allOnes<T>(), bitSizeOf<T>() - size_t(n))) : T(0);
232 : }
233 :
234 : //! Generats a leading bit-mask that has `n` most significant (leading) bits set.
235 : template<typename T, typename CountT>
236 : static constexpr T msbMask(const CountT& n) noexcept {
237 : typedef typename std::make_unsigned<T>::type U;
238 : return (sizeof(U) < sizeof(uintptr_t))
239 : // Prevent undefined behavior by using a larger type than T.
240 : ? T(allOnes<uintptr_t>() >> (bitSizeOf<uintptr_t>() - n))
241 : // Prevent undefined behavior by performing `n & (nBits - 1)` so it's always within the range.
242 : : T(sar(U(n != 0) << (bitSizeOf<U>() - 1), n ? uint32_t(n - 1) : uint32_t(0)));
243 : }
244 :
245 : //! Returns a bit-mask that has `x` bit set.
246 : template<typename Index>
247 3188655 : static constexpr uint32_t bitMask(const Index& x) noexcept { return (1u << asUInt(x)); }
248 :
249 : //! Returns a bit-mask that has `x` bit set (multiple arguments).
250 : template<typename Index, typename... Args>
251 : static constexpr uint32_t bitMask(const Index& x, Args... args) noexcept { return bitMask(x) | bitMask(args...); }
252 :
253 : //! Converts a boolean value `b` to zero or full mask (all bits set).
254 : template<typename DstT, typename SrcT>
255 : static constexpr DstT bitMaskFromBool(SrcT b) noexcept {
256 : typedef typename std::make_unsigned<DstT>::type U;
257 571081 : return DstT(U(0) - U(b));
258 : }
259 :
260 : //! Tests whether `a & b` is non-zero.
261 : template<typename A, typename B>
262 18344093 : static inline constexpr bool test(A a, B b) noexcept { return (asUInt(a) & asUInt(b)) != 0; }
263 :
264 : //! \cond
265 : namespace Internal {
266 : // Fills all trailing bits right from the first most significant bit set.
267 : static constexpr uint8_t fillTrailingBitsImpl(uint8_t x) noexcept { return or_shr(or_shr(or_shr(x, 1), 2), 4); }
268 : // Fills all trailing bits right from the first most significant bit set.
269 : static constexpr uint16_t fillTrailingBitsImpl(uint16_t x) noexcept { return or_shr(or_shr(or_shr(or_shr(x, 1), 2), 4), 8); }
270 : // Fills all trailing bits right from the first most significant bit set.
271 : static constexpr uint32_t fillTrailingBitsImpl(uint32_t x) noexcept { return or_shr(or_shr(or_shr(or_shr(or_shr(x, 1), 2), 4), 8), 16); }
272 : // Fills all trailing bits right from the first most significant bit set.
273 206347 : static constexpr uint64_t fillTrailingBitsImpl(uint64_t x) noexcept { return or_shr(or_shr(or_shr(or_shr(or_shr(or_shr(x, 1), 2), 4), 8), 16), 32); }
274 : }
275 : //! \endcond
276 :
277 : // Fills all trailing bits right from the first most significant bit set.
278 : template<typename T>
279 : static constexpr T fillTrailingBits(const T& x) noexcept {
280 : typedef typename std::make_unsigned<T>::type U;
281 206347 : return T(Internal::fillTrailingBitsImpl(U(x)));
282 : }
283 :
284 : // Support - Count Leading/Trailing Zeros
285 : // ======================================
286 :
287 : //! \cond
288 : namespace Internal {
289 : namespace {
290 :
291 : template<typename T>
292 : struct BitScanData { T x; uint32_t n; };
293 :
294 : template<typename T, uint32_t N>
295 : struct BitScanCalc {
296 : static constexpr BitScanData<T> advanceLeft(const BitScanData<T>& data, uint32_t n) noexcept {
297 : return BitScanData<T> { data.x << n, data.n + n };
298 : }
299 :
300 : static constexpr BitScanData<T> advanceRight(const BitScanData<T>& data, uint32_t n) noexcept {
301 : return BitScanData<T> { data.x >> n, data.n + n };
302 : }
303 :
304 : static constexpr BitScanData<T> clz(const BitScanData<T>& data) noexcept {
305 : return BitScanCalc<T, N / 2>::clz(advanceLeft(data, data.x & (allOnes<T>() << (bitSizeOf<T>() - N)) ? uint32_t(0) : N));
306 : }
307 :
308 : static constexpr BitScanData<T> ctz(const BitScanData<T>& data) noexcept {
309 : return BitScanCalc<T, N / 2>::ctz(advanceRight(data, data.x & (allOnes<T>() >> (bitSizeOf<T>() - N)) ? uint32_t(0) : N));
310 : }
311 : };
312 :
313 : template<typename T>
314 : struct BitScanCalc<T, 0> {
315 : static constexpr BitScanData<T> clz(const BitScanData<T>& ctx) noexcept {
316 : return BitScanData<T> { 0, ctx.n - uint32_t(ctx.x >> (bitSizeOf<T>() - 1)) };
317 : }
318 :
319 : static constexpr BitScanData<T> ctz(const BitScanData<T>& ctx) noexcept {
320 : return BitScanData<T> { 0, ctx.n - uint32_t(ctx.x & 0x1) };
321 : }
322 : };
323 :
324 : template<typename T>
325 : constexpr uint32_t clzFallback(const T& x) noexcept {
326 : return BitScanCalc<T, bitSizeOf<T>() / 2u>::clz(BitScanData<T>{x, 1}).n;
327 : }
328 :
329 : template<typename T>
330 : constexpr uint32_t ctzFallback(const T& x) noexcept {
331 : return BitScanCalc<T, bitSizeOf<T>() / 2u>::ctz(BitScanData<T>{x, 1}).n;
332 : }
333 :
334 : template<typename T> inline uint32_t clzImpl(const T& x) noexcept { return clzFallback(asUInt(x)); }
335 : template<typename T> inline uint32_t ctzImpl(const T& x) noexcept { return ctzFallback(asUInt(x)); }
336 :
337 : #if !defined(ASMJIT_NO_INTRINSICS)
338 : # if defined(__GNUC__)
339 0 : template<> inline uint32_t clzImpl(const uint32_t& x) noexcept { return uint32_t(__builtin_clz(x)); }
340 : template<> inline uint32_t clzImpl(const uint64_t& x) noexcept { return uint32_t(__builtin_clzll(x)); }
341 744501 : template<> inline uint32_t ctzImpl(const uint32_t& x) noexcept { return uint32_t(__builtin_ctz(x)); }
342 137560 : template<> inline uint32_t ctzImpl(const uint64_t& x) noexcept { return uint32_t(__builtin_ctzll(x)); }
343 : # elif defined(_MSC_VER)
344 : template<> inline uint32_t clzImpl(const uint32_t& x) noexcept { unsigned long i; _BitScanReverse(&i, x); return uint32_t(i ^ 31); }
345 : template<> inline uint32_t ctzImpl(const uint32_t& x) noexcept { unsigned long i; _BitScanForward(&i, x); return uint32_t(i); }
346 : # if ASMJIT_ARCH_X86 == 64 || ASMJIT_ARCH_ARM == 64
347 : template<> inline uint32_t clzImpl(const uint64_t& x) noexcept { unsigned long i; _BitScanReverse64(&i, x); return uint32_t(i ^ 63); }
348 : template<> inline uint32_t ctzImpl(const uint64_t& x) noexcept { unsigned long i; _BitScanForward64(&i, x); return uint32_t(i); }
349 : # endif
350 : # endif
351 : #endif
352 :
353 : } // {anonymous}
354 : } // {Internal}
355 : //! \endcond
356 :
357 : //! Count leading zeros in `x` (returns a position of a first bit set in `x`).
358 : //!
359 : //! \note The input MUST NOT be zero, otherwise the result is undefined.
360 : template<typename T>
361 : static inline uint32_t clz(T x) noexcept { return Internal::clzImpl(asUInt(x)); }
362 :
363 : //! Count trailing zeros in `x` (returns a position of a first bit set in `x`).
364 : //!
365 : //! \note The input MUST NOT be zero, otherwise the result is undefined.
366 : template<typename T>
367 : static inline uint32_t ctz(T x) noexcept { return Internal::ctzImpl(asUInt(x)); }
368 :
369 : template<uint64_t kInput>
370 : struct ConstCTZ {
371 : static constexpr uint32_t value =
372 : (kInput & (uint64_t(1) << 0)) ? 0 :
373 : (kInput & (uint64_t(1) << 1)) ? 1 :
374 : (kInput & (uint64_t(1) << 2)) ? 2 :
375 : (kInput & (uint64_t(1) << 3)) ? 3 :
376 : (kInput & (uint64_t(1) << 4)) ? 4 :
377 : (kInput & (uint64_t(1) << 5)) ? 5 :
378 : (kInput & (uint64_t(1) << 6)) ? 6 :
379 : (kInput & (uint64_t(1) << 7)) ? 7 :
380 : (kInput & (uint64_t(1) << 8)) ? 8 :
381 : (kInput & (uint64_t(1) << 9)) ? 9 :
382 : (kInput & (uint64_t(1) << 10)) ? 10 :
383 : (kInput & (uint64_t(1) << 11)) ? 11 :
384 : (kInput & (uint64_t(1) << 12)) ? 12 :
385 : (kInput & (uint64_t(1) << 13)) ? 13 :
386 : (kInput & (uint64_t(1) << 14)) ? 14 :
387 : (kInput & (uint64_t(1) << 15)) ? 15 :
388 : (kInput & (uint64_t(1) << 16)) ? 16 :
389 : (kInput & (uint64_t(1) << 17)) ? 17 :
390 : (kInput & (uint64_t(1) << 18)) ? 18 :
391 : (kInput & (uint64_t(1) << 19)) ? 19 :
392 : (kInput & (uint64_t(1) << 20)) ? 20 :
393 : (kInput & (uint64_t(1) << 21)) ? 21 :
394 : (kInput & (uint64_t(1) << 22)) ? 22 :
395 : (kInput & (uint64_t(1) << 23)) ? 23 :
396 : (kInput & (uint64_t(1) << 24)) ? 24 :
397 : (kInput & (uint64_t(1) << 25)) ? 25 :
398 : (kInput & (uint64_t(1) << 26)) ? 26 :
399 : (kInput & (uint64_t(1) << 27)) ? 27 :
400 : (kInput & (uint64_t(1) << 28)) ? 28 :
401 : (kInput & (uint64_t(1) << 29)) ? 29 :
402 : (kInput & (uint64_t(1) << 30)) ? 30 :
403 : (kInput & (uint64_t(1) << 31)) ? 31 :
404 : (kInput & (uint64_t(1) << 32)) ? 32 :
405 : (kInput & (uint64_t(1) << 33)) ? 33 :
406 : (kInput & (uint64_t(1) << 34)) ? 34 :
407 : (kInput & (uint64_t(1) << 35)) ? 35 :
408 : (kInput & (uint64_t(1) << 36)) ? 36 :
409 : (kInput & (uint64_t(1) << 37)) ? 37 :
410 : (kInput & (uint64_t(1) << 38)) ? 38 :
411 : (kInput & (uint64_t(1) << 39)) ? 39 :
412 : (kInput & (uint64_t(1) << 40)) ? 40 :
413 : (kInput & (uint64_t(1) << 41)) ? 41 :
414 : (kInput & (uint64_t(1) << 42)) ? 42 :
415 : (kInput & (uint64_t(1) << 43)) ? 43 :
416 : (kInput & (uint64_t(1) << 44)) ? 44 :
417 : (kInput & (uint64_t(1) << 45)) ? 45 :
418 : (kInput & (uint64_t(1) << 46)) ? 46 :
419 : (kInput & (uint64_t(1) << 47)) ? 47 :
420 : (kInput & (uint64_t(1) << 48)) ? 48 :
421 : (kInput & (uint64_t(1) << 49)) ? 49 :
422 : (kInput & (uint64_t(1) << 50)) ? 50 :
423 : (kInput & (uint64_t(1) << 51)) ? 51 :
424 : (kInput & (uint64_t(1) << 52)) ? 52 :
425 : (kInput & (uint64_t(1) << 53)) ? 53 :
426 : (kInput & (uint64_t(1) << 54)) ? 54 :
427 : (kInput & (uint64_t(1) << 55)) ? 55 :
428 : (kInput & (uint64_t(1) << 56)) ? 56 :
429 : (kInput & (uint64_t(1) << 57)) ? 57 :
430 : (kInput & (uint64_t(1) << 58)) ? 58 :
431 : (kInput & (uint64_t(1) << 59)) ? 59 :
432 : (kInput & (uint64_t(1) << 60)) ? 60 :
433 : (kInput & (uint64_t(1) << 61)) ? 61 :
434 : (kInput & (uint64_t(1) << 62)) ? 62 :
435 : (kInput & (uint64_t(1) << 63)) ? 63 : 64;
436 : };
437 :
438 : // Support - PopCnt
439 : // ================
440 :
441 : // Based on the following resource:
442 : // http://graphics.stanford.edu/~seander/bithacks.html
443 : //
444 : // Alternatively, for a very small number of bits in `x`:
445 : // uint32_t n = 0;
446 : // while (x) {
447 : // x &= x - 1;
448 : // n++;
449 : // }
450 : // return n;
451 :
452 : //! \cond
453 : namespace Internal {
454 : static inline uint32_t constPopcntImpl(uint32_t x) noexcept {
455 : x = x - ((x >> 1) & 0x55555555u);
456 : x = (x & 0x33333333u) + ((x >> 2) & 0x33333333u);
457 : return (((x + (x >> 4)) & 0x0F0F0F0Fu) * 0x01010101u) >> 24;
458 : }
459 :
460 : static inline uint32_t constPopcntImpl(uint64_t x) noexcept {
461 : if (ASMJIT_ARCH_BITS >= 64) {
462 : x = x - ((x >> 1) & 0x5555555555555555u);
463 : x = (x & 0x3333333333333333u) + ((x >> 2) & 0x3333333333333333u);
464 : return uint32_t((((x + (x >> 4)) & 0x0F0F0F0F0F0F0F0Fu) * 0x0101010101010101u) >> 56);
465 : }
466 : else {
467 : return constPopcntImpl(uint32_t(x >> 32)) +
468 : constPopcntImpl(uint32_t(x & 0xFFFFFFFFu));
469 : }
470 : }
471 :
472 : static inline uint32_t popcntImpl(uint32_t x) noexcept {
473 : #if defined(__GNUC__)
474 109968 : return uint32_t(__builtin_popcount(x));
475 : #else
476 : return constPopcntImpl(asUInt(x));
477 : #endif
478 : }
479 :
480 : static inline uint32_t popcntImpl(uint64_t x) noexcept {
481 : #if defined(__GNUC__)
482 : return uint32_t(__builtin_popcountll(x));
483 : #else
484 : return constPopcntImpl(asUInt(x));
485 : #endif
486 : }
487 : }
488 : //! \endcond
489 :
490 : //! Calculates count of bits in `x`.
491 : template<typename T>
492 : static inline uint32_t popcnt(T x) noexcept { return Internal::popcntImpl(asUInt(x)); }
493 :
494 : //! Calculates count of bits in `x` (useful in constant expressions).
495 : template<typename T>
496 : static inline uint32_t constPopcnt(T x) noexcept { return Internal::constPopcntImpl(asUInt(x)); }
497 :
498 : // Support - Min/Max
499 : // =================
500 :
501 : // NOTE: These are constexpr `min()` and `max()` implementations that are not
502 : // exactly the same as `std::min()` and `std::max()`. The return value is not
503 : // a reference to `a` or `b` but it's a new value instead.
504 :
505 : template<typename T>
506 152378 : static constexpr T min(const T& a, const T& b) noexcept { return b < a ? b : a; }
507 :
508 : template<typename T, typename... Args>
509 : static constexpr T min(const T& a, const T& b, Args&&... args) noexcept { return min(min(a, b), std::forward<Args>(args)...); }
510 :
511 : template<typename T>
512 1546191 : static constexpr T max(const T& a, const T& b) noexcept { return a < b ? b : a; }
513 :
514 : template<typename T, typename... Args>
515 : static constexpr T max(const T& a, const T& b, Args&&... args) noexcept { return max(max(a, b), std::forward<Args>(args)...); }
516 :
517 : // Support - Immediate Helpers
518 : // ===========================
519 :
520 : namespace Internal {
521 : template<typename T, bool IsFloat>
522 : struct ImmConv {
523 : static inline int64_t fromT(const T& x) noexcept { return int64_t(x); }
524 0 : static inline T toT(int64_t x) noexcept { return T(uint64_t(x) & Support::allOnes<typename std::make_unsigned<T>::type>()); }
525 : };
526 :
527 : template<typename T>
528 : struct ImmConv<T, true> {
529 : static inline int64_t fromT(const T& x) noexcept { return int64_t(bitCast<int64_t>(double(x))); }
530 : static inline T toT(int64_t x) noexcept { return T(bitCast<double>(x)); }
531 : };
532 : }
533 :
534 : template<typename T>
535 : static inline int64_t immediateFromT(const T& x) noexcept { return Internal::ImmConv<T, std::is_floating_point<T>::value>::fromT(x); }
536 :
537 : template<typename T>
538 : static inline T immediateToT(int64_t x) noexcept { return Internal::ImmConv<T, std::is_floating_point<T>::value>::toT(x); }
539 :
540 : // Support - Overflow Arithmetic
541 : // =============================
542 :
543 : //! \cond
544 : namespace Internal {
545 : template<typename T>
546 : inline T addOverflowFallback(T x, T y, FastUInt8* of) noexcept {
547 : typedef typename std::make_unsigned<T>::type U;
548 :
549 : U result = U(x) + U(y);
550 : *of = FastUInt8(*of | FastUInt8(isUnsigned<T>() ? result < U(x) : T((U(x) ^ ~U(y)) & (U(x) ^ result)) < 0));
551 : return T(result);
552 : }
553 :
554 : template<typename T>
555 : inline T subOverflowFallback(T x, T y, FastUInt8* of) noexcept {
556 : typedef typename std::make_unsigned<T>::type U;
557 :
558 : U result = U(x) - U(y);
559 : *of = FastUInt8(*of | FastUInt8(isUnsigned<T>() ? result > U(x) : T((U(x) ^ U(y)) & (U(x) ^ result)) < 0));
560 : return T(result);
561 : }
562 :
563 : template<typename T>
564 : inline T mulOverflowFallback(T x, T y, FastUInt8* of) noexcept {
565 : typedef typename Internal::StdInt<sizeof(T) * 2, isUnsigned<T>()>::Type I;
566 : typedef typename std::make_unsigned<I>::type U;
567 :
568 : U mask = allOnes<U>();
569 : if (std::is_signed<T>::value) {
570 : U prod = U(I(x)) * U(I(y));
571 : *of = FastUInt8(*of | FastUInt8(I(prod) < I(std::numeric_limits<T>::lowest()) || I(prod) > I(std::numeric_limits<T>::max())));
572 : return T(I(prod & mask));
573 : }
574 : else {
575 : U prod = U(x) * U(y);
576 : *of = FastUInt8(*of | FastUInt8((prod & ~mask) != 0));
577 : return T(prod & mask);
578 : }
579 : }
580 :
581 : template<>
582 : inline int64_t mulOverflowFallback(int64_t x, int64_t y, FastUInt8* of) noexcept {
583 : int64_t result = int64_t(uint64_t(x) * uint64_t(y));
584 : *of = FastUInt8(*of | FastUInt8(x && (result / x != y)));
585 : return result;
586 : }
587 :
588 : template<>
589 : inline uint64_t mulOverflowFallback(uint64_t x, uint64_t y, FastUInt8* of) noexcept {
590 : uint64_t result = x * y;
591 : *of = FastUInt8(*of | FastUInt8(y != 0 && allOnes<uint64_t>() / y < x));
592 : return result;
593 : }
594 :
595 : // These can be specialized.
596 : template<typename T> inline T addOverflowImpl(const T& x, const T& y, FastUInt8* of) noexcept { return addOverflowFallback(x, y, of); }
597 : template<typename T> inline T subOverflowImpl(const T& x, const T& y, FastUInt8* of) noexcept { return subOverflowFallback(x, y, of); }
598 : template<typename T> inline T mulOverflowImpl(const T& x, const T& y, FastUInt8* of) noexcept { return mulOverflowFallback(x, y, of); }
599 :
600 : #if defined(__GNUC__) && !defined(ASMJIT_NO_INTRINSICS)
601 : #if defined(__clang__) || __GNUC__ >= 5
602 : #define ASMJIT_ARITH_OVERFLOW_SPECIALIZE(FUNC, T, RESULT_T, BUILTIN) \
603 : template<> \
604 : inline T FUNC(const T& x, const T& y, FastUInt8* of) noexcept { \
605 : RESULT_T result; \
606 : *of = FastUInt8(*of | (BUILTIN((RESULT_T)x, (RESULT_T)y, &result))); \
607 : return T(result); \
608 : }
609 : ASMJIT_ARITH_OVERFLOW_SPECIALIZE(addOverflowImpl, int32_t , int , __builtin_sadd_overflow )
610 : ASMJIT_ARITH_OVERFLOW_SPECIALIZE(addOverflowImpl, uint32_t, unsigned int , __builtin_uadd_overflow )
611 : ASMJIT_ARITH_OVERFLOW_SPECIALIZE(addOverflowImpl, int64_t , long long , __builtin_saddll_overflow)
612 27492 : ASMJIT_ARITH_OVERFLOW_SPECIALIZE(addOverflowImpl, uint64_t, unsigned long long, __builtin_uaddll_overflow)
613 : ASMJIT_ARITH_OVERFLOW_SPECIALIZE(subOverflowImpl, int32_t , int , __builtin_ssub_overflow )
614 : ASMJIT_ARITH_OVERFLOW_SPECIALIZE(subOverflowImpl, uint32_t, unsigned int , __builtin_usub_overflow )
615 : ASMJIT_ARITH_OVERFLOW_SPECIALIZE(subOverflowImpl, int64_t , long long , __builtin_ssubll_overflow)
616 : ASMJIT_ARITH_OVERFLOW_SPECIALIZE(subOverflowImpl, uint64_t, unsigned long long, __builtin_usubll_overflow)
617 : ASMJIT_ARITH_OVERFLOW_SPECIALIZE(mulOverflowImpl, int32_t , int , __builtin_smul_overflow )
618 : ASMJIT_ARITH_OVERFLOW_SPECIALIZE(mulOverflowImpl, uint32_t, unsigned int , __builtin_umul_overflow )
619 : ASMJIT_ARITH_OVERFLOW_SPECIALIZE(mulOverflowImpl, int64_t , long long , __builtin_smulll_overflow)
620 0 : ASMJIT_ARITH_OVERFLOW_SPECIALIZE(mulOverflowImpl, uint64_t, unsigned long long, __builtin_umulll_overflow)
621 : #undef ASMJIT_ARITH_OVERFLOW_SPECIALIZE
622 : #endif
623 : #endif
624 :
625 : // There is a bug in MSVC that makes these specializations unusable, maybe in the future...
626 : #if defined(_MSC_VER) && 0
627 : #define ASMJIT_ARITH_OVERFLOW_SPECIALIZE(FUNC, T, ALT_T, BUILTIN) \
628 : template<> \
629 : inline T FUNC(T x, T y, FastUInt8* of) noexcept { \
630 : ALT_T result; \
631 : *of = FastUInt8(*of | BUILTIN(0, (ALT_T)x, (ALT_T)y, &result)); \
632 : return T(result); \
633 : }
634 : ASMJIT_ARITH_OVERFLOW_SPECIALIZE(addOverflowImpl, uint32_t, unsigned int , _addcarry_u32 )
635 : ASMJIT_ARITH_OVERFLOW_SPECIALIZE(subOverflowImpl, uint32_t, unsigned int , _subborrow_u32)
636 : #if ARCH_BITS >= 64
637 : ASMJIT_ARITH_OVERFLOW_SPECIALIZE(addOverflowImpl, uint64_t, unsigned __int64 , _addcarry_u64 )
638 : ASMJIT_ARITH_OVERFLOW_SPECIALIZE(subOverflowImpl, uint64_t, unsigned __int64 , _subborrow_u64)
639 : #endif
640 : #undef ASMJIT_ARITH_OVERFLOW_SPECIALIZE
641 : #endif
642 : } // {Internal}
643 : //! \endcond
644 :
645 : template<typename T>
646 : static inline T addOverflow(const T& x, const T& y, FastUInt8* of) noexcept { return T(Internal::addOverflowImpl(asStdInt(x), asStdInt(y), of)); }
647 :
648 : template<typename T>
649 : static inline T subOverflow(const T& x, const T& y, FastUInt8* of) noexcept { return T(Internal::subOverflowImpl(asStdInt(x), asStdInt(y), of)); }
650 :
651 : template<typename T>
652 : static inline T mulOverflow(const T& x, const T& y, FastUInt8* of) noexcept { return T(Internal::mulOverflowImpl(asStdInt(x), asStdInt(y), of)); }
653 :
654 : // Support - Alignment
655 : // ===================
656 :
657 : template<typename X, typename Y>
658 : static constexpr bool isAligned(X base, Y alignment) noexcept {
659 : typedef typename Internal::StdInt<sizeof(X), 1>::Type U;
660 0 : return ((U)base % (U)alignment) == 0;
661 : }
662 :
663 : //! Tests whether the `x` is a power of two (only one bit is set).
664 : template<typename T>
665 : static constexpr bool isPowerOf2(T x) noexcept {
666 : typedef typename std::make_unsigned<T>::type U;
667 23166 : return x && !(U(x) & (U(x) - U(1)));
668 : }
669 :
670 : template<typename X, typename Y>
671 : static constexpr X alignUp(X x, Y alignment) noexcept {
672 : typedef typename Internal::StdInt<sizeof(X), 1>::Type U;
673 6905632 : return (X)( ((U)x + ((U)(alignment) - 1u)) & ~((U)(alignment) - 1u) );
674 : }
675 :
676 : template<typename T>
677 : static constexpr T alignUpPowerOf2(T x) noexcept {
678 : typedef typename Internal::StdInt<sizeof(T), 1>::Type U;
679 0 : return (T)(fillTrailingBits(U(x) - 1u) + 1u);
680 : }
681 :
682 : //! Returns either zero or a positive difference between `base` and `base` when
683 : //! aligned to `alignment`.
684 : template<typename X, typename Y>
685 4990 : static constexpr typename Internal::StdInt<sizeof(X), 1>::Type alignUpDiff(X base, Y alignment) noexcept {
686 : typedef typename Internal::StdInt<sizeof(X), 1>::Type U;
687 4990 : return alignUp(U(base), alignment) - U(base);
688 : }
689 :
690 : template<typename X, typename Y>
691 : static constexpr X alignDown(X x, Y alignment) noexcept {
692 : typedef typename Internal::StdInt<sizeof(X), 1>::Type U;
693 111022 : return (X)( (U)x & ~((U)(alignment) - 1u) );
694 : }
695 :
696 : // Support - NumGranularized
697 : // =========================
698 :
699 : //! Calculates the number of elements that would be required if `base` is
700 : //! granularized by `granularity`. This function can be used to calculate
701 : //! the number of BitWords to represent N bits, for example.
702 : template<typename X, typename Y>
703 : static constexpr X numGranularized(X base, Y granularity) noexcept {
704 : typedef typename Internal::StdInt<sizeof(X), 1>::Type U;
705 : return X((U(base) + U(granularity) - 1) / U(granularity));
706 : }
707 :
708 : // Support - IsBetween
709 : // ===================
710 :
711 : //! Checks whether `x` is greater than or equal to `a` and lesser than or equal to `b`.
712 : template<typename T>
713 0 : static constexpr bool isBetween(const T& x, const T& a, const T& b) noexcept {
714 1739753 : return x >= a && x <= b;
715 : }
716 :
717 : // Support - IsInt & IsUInt
718 : // ========================
719 :
720 : //! Checks whether the given integer `x` can be casted to a 4-bit signed integer.
721 : template<typename T>
722 : static constexpr bool isInt4(T x) noexcept {
723 : typedef typename std::make_signed<T>::type S;
724 : typedef typename std::make_unsigned<T>::type U;
725 :
726 : return std::is_signed<T>::value ? isBetween<S>(S(x), -8, 7) : U(x) <= U(7u);
727 : }
728 :
729 : //! Checks whether the given integer `x` can be casted to a 7-bit signed integer.
730 : template<typename T>
731 : static constexpr bool isInt7(T x) noexcept {
732 : typedef typename std::make_signed<T>::type S;
733 : typedef typename std::make_unsigned<T>::type U;
734 :
735 : return std::is_signed<T>::value ? isBetween<S>(S(x), -64, 63) : U(x) <= U(63u);
736 : }
737 :
738 : //! Checks whether the given integer `x` can be casted to an 8-bit signed integer.
739 : template<typename T>
740 : static constexpr bool isInt8(T x) noexcept {
741 : typedef typename std::make_signed<T>::type S;
742 : typedef typename std::make_unsigned<T>::type U;
743 :
744 : return std::is_signed<T>::value ? sizeof(T) <= 1 || isBetween<S>(S(x), -128, 127) : U(x) <= U(127u);
745 : }
746 :
747 : //! Checks whether the given integer `x` can be casted to a 9-bit signed integer.
748 : template<typename T>
749 : static constexpr bool isInt9(T x) noexcept {
750 : typedef typename std::make_signed<T>::type S;
751 : typedef typename std::make_unsigned<T>::type U;
752 :
753 : return std::is_signed<T>::value ? sizeof(T) <= 1 || isBetween<S>(S(x), -256, 255)
754 : : sizeof(T) <= 1 || U(x) <= U(255u);
755 : }
756 :
757 : //! Checks whether the given integer `x` can be casted to a 10-bit signed integer.
758 : template<typename T>
759 : static constexpr bool isInt10(T x) noexcept {
760 : typedef typename std::make_signed<T>::type S;
761 : typedef typename std::make_unsigned<T>::type U;
762 :
763 : return std::is_signed<T>::value ? sizeof(T) <= 1 || isBetween<S>(S(x), -512, 511)
764 : : sizeof(T) <= 1 || U(x) <= U(511u);
765 : }
766 :
767 : //! Checks whether the given integer `x` can be casted to a 16-bit signed integer.
768 : template<typename T>
769 : static constexpr bool isInt16(T x) noexcept {
770 : typedef typename std::make_signed<T>::type S;
771 : typedef typename std::make_unsigned<T>::type U;
772 :
773 : return std::is_signed<T>::value ? sizeof(T) <= 2 || isBetween<S>(S(x), -32768, 32767)
774 : : sizeof(T) <= 1 || U(x) <= U(32767u);
775 : }
776 :
777 : //! Checks whether the given integer `x` can be casted to a 32-bit signed integer.
778 : template<typename T>
779 0 : static constexpr bool isInt32(T x) noexcept {
780 : typedef typename std::make_signed<T>::type S;
781 : typedef typename std::make_unsigned<T>::type U;
782 :
783 : return std::is_signed<T>::value ? sizeof(T) <= 4 || isBetween<S>(S(x), -2147483647 - 1, 2147483647)
784 0 : : sizeof(T) <= 2 || U(x) <= U(2147483647u);
785 : }
786 :
787 : //! Checks whether the given integer `x` can be casted to a 4-bit unsigned integer.
788 : template<typename T>
789 : static constexpr bool isUInt4(T x) noexcept {
790 : typedef typename std::make_unsigned<T>::type U;
791 :
792 : return std::is_signed<T>::value ? x >= T(0) && x <= T(15)
793 : : U(x) <= U(15u);
794 : }
795 :
796 : //! Checks whether the given integer `x` can be casted to an 8-bit unsigned integer.
797 : template<typename T>
798 : static constexpr bool isUInt8(T x) noexcept {
799 : typedef typename std::make_unsigned<T>::type U;
800 :
801 : return std::is_signed<T>::value ? (sizeof(T) <= 1 || T(x) <= T(255)) && x >= T(0)
802 : : (sizeof(T) <= 1 || U(x) <= U(255u));
803 : }
804 :
805 : //! Checks whether the given integer `x` can be casted to a 12-bit unsigned integer (ARM specific).
806 : template<typename T>
807 : static constexpr bool isUInt12(T x) noexcept {
808 : typedef typename std::make_unsigned<T>::type U;
809 :
810 : return std::is_signed<T>::value ? (sizeof(T) <= 1 || T(x) <= T(4095)) && x >= T(0)
811 : : (sizeof(T) <= 1 || U(x) <= U(4095u));
812 : }
813 :
814 : //! Checks whether the given integer `x` can be casted to a 16-bit unsigned integer.
815 : template<typename T>
816 : static constexpr bool isUInt16(T x) noexcept {
817 : typedef typename std::make_unsigned<T>::type U;
818 :
819 : return std::is_signed<T>::value ? (sizeof(T) <= 2 || T(x) <= T(65535)) && x >= T(0)
820 : : (sizeof(T) <= 2 || U(x) <= U(65535u));
821 : }
822 :
823 : //! Checks whether the given integer `x` can be casted to a 32-bit unsigned integer.
824 : template<typename T>
825 : static constexpr bool isUInt32(T x) noexcept {
826 : typedef typename std::make_unsigned<T>::type U;
827 :
828 0 : return std::is_signed<T>::value ? (sizeof(T) <= 4 || T(x) <= T(4294967295u)) && x >= T(0)
829 : : (sizeof(T) <= 4 || U(x) <= U(4294967295u));
830 : }
831 :
832 : //! Checks whether the given integer `x` can be casted to a 32-bit unsigned integer.
833 : template<typename T>
834 : static constexpr bool isIntOrUInt32(T x) noexcept {
835 : return sizeof(T) <= 4 ? true : (uint32_t(uint64_t(x) >> 32) + 1u) <= 1u;
836 : }
837 :
838 : static bool inline isEncodableOffset32(int32_t offset, uint32_t nBits) noexcept {
839 0 : uint32_t nRev = 32 - nBits;
840 : return Support::sar(Support::shl(offset, nRev), nRev) == offset;
841 : }
842 :
843 : static bool inline isEncodableOffset64(int64_t offset, uint32_t nBits) noexcept {
844 0 : uint32_t nRev = 64 - nBits;
845 : return Support::sar(Support::shl(offset, nRev), nRev) == offset;
846 : }
847 :
848 : // Support - ByteSwap
849 : // ==================
850 :
851 : static inline uint16_t byteswap16(uint16_t x) noexcept {
852 : return uint16_t(((x >> 8) & 0xFFu) | ((x & 0xFFu) << 8));
853 : }
854 :
855 : static inline uint32_t byteswap32(uint32_t x) noexcept {
856 : return (x << 24) | (x >> 24) | ((x << 8) & 0x00FF0000u) | ((x >> 8) & 0x0000FF00);
857 : }
858 :
859 : static inline uint64_t byteswap64(uint64_t x) noexcept {
860 : #if (defined(__GNUC__) || defined(__clang__)) && !defined(ASMJIT_NO_INTRINSICS)
861 : return uint64_t(__builtin_bswap64(uint64_t(x)));
862 : #elif defined(_MSC_VER) && !defined(ASMJIT_NO_INTRINSICS)
863 : return uint64_t(_byteswap_uint64(uint64_t(x)));
864 : #else
865 : return (uint64_t(byteswap32(uint32_t(uint64_t(x) >> 32 ))) ) |
866 : (uint64_t(byteswap32(uint32_t(uint64_t(x) & 0xFFFFFFFFu))) << 32) ;
867 : #endif
868 : }
869 :
870 : // Support - BytePack & Unpack
871 : // ===========================
872 :
873 : //! Pack four 8-bit integer into a 32-bit integer as it is an array of `{b0,b1,b2,b3}`.
874 : static constexpr uint32_t bytepack32_4x8(uint32_t a, uint32_t b, uint32_t c, uint32_t d) noexcept {
875 414284 : return ASMJIT_ARCH_LE ? (a | (b << 8) | (c << 16) | (d << 24))
876 386792 : : (d | (c << 8) | (b << 16) | (a << 24));
877 : }
878 :
879 : template<typename T>
880 88142 : static constexpr uint32_t unpackU32At0(T x) noexcept { return ASMJIT_ARCH_LE ? uint32_t(uint64_t(x) & 0xFFFFFFFFu) : uint32_t(uint64_t(x) >> 32); }
881 : template<typename T>
882 93132 : static constexpr uint32_t unpackU32At1(T x) noexcept { return ASMJIT_ARCH_BE ? uint32_t(uint64_t(x) & 0xFFFFFFFFu) : uint32_t(uint64_t(x) >> 32); }
883 :
884 : // Support - Position of byte (in bit-shift)
885 : // =========================================
886 :
887 : static inline uint32_t byteShiftOfDWordStruct(uint32_t index) noexcept {
888 2161622 : return ASMJIT_ARCH_LE ? index * 8 : (uint32_t(sizeof(uint32_t)) - 1u - index) * 8;
889 : }
890 :
891 : // Support - String Utilities
892 : // ==========================
893 :
894 : template<typename T>
895 : static constexpr T asciiToLower(T c) noexcept { return T(c ^ T(T(c >= T('A') && c <= T('Z')) << 5)); }
896 :
897 : template<typename T>
898 : static constexpr T asciiToUpper(T c) noexcept { return T(c ^ T(T(c >= T('a') && c <= T('z')) << 5)); }
899 :
900 : static ASMJIT_FORCE_INLINE size_t strLen(const char* s, size_t maxSize) noexcept {
901 : size_t i = 0;
902 0 : while (i < maxSize && s[i] != '\0')
903 0 : i++;
904 : return i;
905 : }
906 :
907 0 : static constexpr uint32_t hashRound(uint32_t hash, uint32_t c) noexcept { return hash * 65599 + c; }
908 :
909 : // Gets a hash of the given string `data` of size `size`. Size must be valid
910 : // as this function doesn't check for a null terminator and allows it in the
911 : // middle of the string.
912 : static inline uint32_t hashString(const char* data, size_t size) noexcept {
913 : uint32_t hashCode = 0;
914 : for (uint32_t i = 0; i < size; i++)
915 : hashCode = hashRound(hashCode, uint8_t(data[i]));
916 : return hashCode;
917 : }
918 :
919 : static ASMJIT_FORCE_INLINE const char* findPackedString(const char* p, uint32_t id) noexcept {
920 : uint32_t i = 0;
921 0 : while (i < id) {
922 0 : while (p[0])
923 0 : p++;
924 0 : p++;
925 0 : i++;
926 : }
927 : return p;
928 : }
929 :
930 : //! Compares two instruction names.
931 : //!
932 : //! `a` is a null terminated instruction name from arch-specific `nameData[]`
933 : //! table. `b` is a possibly non-null terminated instruction name passed to
934 : //! `InstAPI::stringToInstId()`.
935 : static ASMJIT_FORCE_INLINE int cmpInstName(const char* a, const char* b, size_t size) noexcept {
936 0 : for (size_t i = 0; i < size; i++) {
937 0 : int c = int(uint8_t(a[i])) - int(uint8_t(b[i]));
938 0 : if (c != 0) return c;
939 : }
940 0 : return int(uint8_t(a[size]));
941 : }
942 :
943 : // Support - Memory Read Access - 8 Bits
944 : // =====================================
945 :
946 0 : static inline uint8_t readU8(const void* p) noexcept { return static_cast<const uint8_t*>(p)[0]; }
947 : static inline int8_t readI8(const void* p) noexcept { return static_cast<const int8_t*>(p)[0]; }
948 :
949 : // Support - Memory Read Access - 16 Bits
950 : // ======================================
951 :
952 : template<ByteOrder BO, size_t Alignment>
953 : static inline uint16_t readU16x(const void* p) noexcept {
954 : typedef typename Internal::AliasedUInt<uint16_t, Alignment>::T U16AlignedToN;
955 0 : uint16_t x = static_cast<const U16AlignedToN*>(p)[0];
956 : return BO == ByteOrder::kNative ? x : byteswap16(x);
957 : }
958 :
959 : template<size_t Alignment = 1>
960 : static inline uint16_t readU16u(const void* p) noexcept { return readU16x<ByteOrder::kNative, Alignment>(p); }
961 : template<size_t Alignment = 1>
962 : static inline uint16_t readU16uLE(const void* p) noexcept { return readU16x<ByteOrder::kLE, Alignment>(p); }
963 : template<size_t Alignment = 1>
964 : static inline uint16_t readU16uBE(const void* p) noexcept { return readU16x<ByteOrder::kBE, Alignment>(p); }
965 :
966 : static inline uint16_t readU16a(const void* p) noexcept { return readU16x<ByteOrder::kNative, 2>(p); }
967 : static inline uint16_t readU16aLE(const void* p) noexcept { return readU16x<ByteOrder::kLE, 2>(p); }
968 : static inline uint16_t readU16aBE(const void* p) noexcept { return readU16x<ByteOrder::kBE, 2>(p); }
969 :
970 : template<ByteOrder BO, size_t Alignment>
971 : static inline int16_t readI16x(const void* p) noexcept { return int16_t(readU16x<BO, Alignment>(p)); }
972 :
973 : template<size_t Alignment = 1>
974 : static inline int16_t readI16u(const void* p) noexcept { return int16_t(readU16x<ByteOrder::kNative, Alignment>(p)); }
975 : template<size_t Alignment = 1>
976 : static inline int16_t readI16uLE(const void* p) noexcept { return int16_t(readU16x<ByteOrder::kLE, Alignment>(p)); }
977 : template<size_t Alignment = 1>
978 : static inline int16_t readI16uBE(const void* p) noexcept { return int16_t(readU16x<ByteOrder::kBE, Alignment>(p)); }
979 :
980 : static inline int16_t readI16a(const void* p) noexcept { return int16_t(readU16x<ByteOrder::kNative, 2>(p)); }
981 : static inline int16_t readI16aLE(const void* p) noexcept { return int16_t(readU16x<ByteOrder::kLE, 2>(p)); }
982 : static inline int16_t readI16aBE(const void* p) noexcept { return int16_t(readU16x<ByteOrder::kBE, 2>(p)); }
983 :
984 : // Support - Memory Read Access - 24 Bits
985 : // ======================================
986 :
987 : template<ByteOrder BO = ByteOrder::kNative>
988 : static inline uint32_t readU24u(const void* p) noexcept {
989 : uint32_t b0 = readU8(static_cast<const uint8_t*>(p) + (BO == ByteOrder::kLE ? 2 : 0));
990 : uint32_t b1 = readU8(static_cast<const uint8_t*>(p) + (BO == ByteOrder::kLE ? 1 : 1));
991 : uint32_t b2 = readU8(static_cast<const uint8_t*>(p) + (BO == ByteOrder::kLE ? 0 : 2));
992 : return (b0 << 16) | (b1 << 8) | b2;
993 : }
994 :
995 : static inline uint32_t readU24uLE(const void* p) noexcept { return readU24u<ByteOrder::kLE>(p); }
996 : static inline uint32_t readU24uBE(const void* p) noexcept { return readU24u<ByteOrder::kBE>(p); }
997 :
998 : // Support - Memory Read Access - 32 Bits
999 : // ======================================
1000 :
1001 : template<ByteOrder BO, size_t Alignment>
1002 : static inline uint32_t readU32x(const void* p) noexcept {
1003 : typedef typename Internal::AliasedUInt<uint32_t, Alignment>::T U32AlignedToN;
1004 0 : uint32_t x = static_cast<const U32AlignedToN*>(p)[0];
1005 : return BO == ByteOrder::kNative ? x : byteswap32(x);
1006 : }
1007 :
1008 : template<size_t Alignment = 1>
1009 : static inline uint32_t readU32u(const void* p) noexcept { return readU32x<ByteOrder::kNative, Alignment>(p); }
1010 : template<size_t Alignment = 1>
1011 : static inline uint32_t readU32uLE(const void* p) noexcept { return readU32x<ByteOrder::kLE, Alignment>(p); }
1012 : template<size_t Alignment = 1>
1013 : static inline uint32_t readU32uBE(const void* p) noexcept { return readU32x<ByteOrder::kBE, Alignment>(p); }
1014 :
1015 : static inline uint32_t readU32a(const void* p) noexcept { return readU32x<ByteOrder::kNative, 4>(p); }
1016 : static inline uint32_t readU32aLE(const void* p) noexcept { return readU32x<ByteOrder::kLE, 4>(p); }
1017 : static inline uint32_t readU32aBE(const void* p) noexcept { return readU32x<ByteOrder::kBE, 4>(p); }
1018 :
1019 : template<ByteOrder BO, size_t Alignment>
1020 : static inline uint32_t readI32x(const void* p) noexcept { return int32_t(readU32x<BO, Alignment>(p)); }
1021 :
1022 : template<size_t Alignment = 1>
1023 : static inline int32_t readI32u(const void* p) noexcept { return int32_t(readU32x<ByteOrder::kNative, Alignment>(p)); }
1024 : template<size_t Alignment = 1>
1025 : static inline int32_t readI32uLE(const void* p) noexcept { return int32_t(readU32x<ByteOrder::kLE, Alignment>(p)); }
1026 : template<size_t Alignment = 1>
1027 : static inline int32_t readI32uBE(const void* p) noexcept { return int32_t(readU32x<ByteOrder::kBE, Alignment>(p)); }
1028 :
1029 : static inline int32_t readI32a(const void* p) noexcept { return int32_t(readU32x<ByteOrder::kNative, 4>(p)); }
1030 : static inline int32_t readI32aLE(const void* p) noexcept { return int32_t(readU32x<ByteOrder::kLE, 4>(p)); }
1031 : static inline int32_t readI32aBE(const void* p) noexcept { return int32_t(readU32x<ByteOrder::kBE, 4>(p)); }
1032 :
1033 : // Support - Memory Read Access - 64 Bits
1034 : // ======================================
1035 :
1036 : template<ByteOrder BO, size_t Alignment>
1037 : static inline uint64_t readU64x(const void* p) noexcept {
1038 : typedef typename Internal::AliasedUInt<uint64_t, Alignment>::T U64AlignedToN;
1039 0 : uint64_t x = static_cast<const U64AlignedToN*>(p)[0];
1040 : return BO == ByteOrder::kNative ? x : byteswap64(x);
1041 : }
1042 :
1043 : template<size_t Alignment = 1>
1044 : static inline uint64_t readU64u(const void* p) noexcept { return readU64x<ByteOrder::kNative, Alignment>(p); }
1045 : template<size_t Alignment = 1>
1046 : static inline uint64_t readU64uLE(const void* p) noexcept { return readU64x<ByteOrder::kLE, Alignment>(p); }
1047 : template<size_t Alignment = 1>
1048 : static inline uint64_t readU64uBE(const void* p) noexcept { return readU64x<ByteOrder::kBE, Alignment>(p); }
1049 :
1050 : static inline uint64_t readU64a(const void* p) noexcept { return readU64x<ByteOrder::kNative, 8>(p); }
1051 : static inline uint64_t readU64aLE(const void* p) noexcept { return readU64x<ByteOrder::kLE, 8>(p); }
1052 : static inline uint64_t readU64aBE(const void* p) noexcept { return readU64x<ByteOrder::kBE, 8>(p); }
1053 :
1054 : template<ByteOrder BO, size_t Alignment>
1055 : static inline int64_t readI64x(const void* p) noexcept { return int64_t(readU64x<BO, Alignment>(p)); }
1056 :
1057 : template<size_t Alignment = 1>
1058 : static inline int64_t readI64u(const void* p) noexcept { return int64_t(readU64x<ByteOrder::kNative, Alignment>(p)); }
1059 : template<size_t Alignment = 1>
1060 : static inline int64_t readI64uLE(const void* p) noexcept { return int64_t(readU64x<ByteOrder::kLE, Alignment>(p)); }
1061 : template<size_t Alignment = 1>
1062 : static inline int64_t readI64uBE(const void* p) noexcept { return int64_t(readU64x<ByteOrder::kBE, Alignment>(p)); }
1063 :
1064 : static inline int64_t readI64a(const void* p) noexcept { return int64_t(readU64x<ByteOrder::kNative, 8>(p)); }
1065 : static inline int64_t readI64aLE(const void* p) noexcept { return int64_t(readU64x<ByteOrder::kLE, 8>(p)); }
1066 : static inline int64_t readI64aBE(const void* p) noexcept { return int64_t(readU64x<ByteOrder::kBE, 8>(p)); }
1067 :
1068 : // Support - Memory Write Access - 8 Bits
1069 : // ======================================
1070 :
1071 0 : static inline void writeU8(void* p, uint8_t x) noexcept { static_cast<uint8_t*>(p)[0] = x; }
1072 : static inline void writeI8(void* p, int8_t x) noexcept { static_cast<int8_t*>(p)[0] = x; }
1073 :
1074 : // Support - Memory Write Access - 16 Bits
1075 : // =======================================
1076 :
1077 : template<ByteOrder BO = ByteOrder::kNative, size_t Alignment = 1>
1078 : static inline void writeU16x(void* p, uint16_t x) noexcept {
1079 : typedef typename Internal::AliasedUInt<uint16_t, Alignment>::T U16AlignedToN;
1080 0 : static_cast<U16AlignedToN*>(p)[0] = BO == ByteOrder::kNative ? x : byteswap16(x);
1081 : }
1082 :
1083 : template<size_t Alignment = 1>
1084 : static inline void writeU16uLE(void* p, uint16_t x) noexcept { writeU16x<ByteOrder::kLE, Alignment>(p, x); }
1085 : template<size_t Alignment = 1>
1086 : static inline void writeU16uBE(void* p, uint16_t x) noexcept { writeU16x<ByteOrder::kBE, Alignment>(p, x); }
1087 :
1088 : static inline void writeU16a(void* p, uint16_t x) noexcept { writeU16x<ByteOrder::kNative, 2>(p, x); }
1089 : static inline void writeU16aLE(void* p, uint16_t x) noexcept { writeU16x<ByteOrder::kLE, 2>(p, x); }
1090 : static inline void writeU16aBE(void* p, uint16_t x) noexcept { writeU16x<ByteOrder::kBE, 2>(p, x); }
1091 :
1092 :
1093 : template<ByteOrder BO = ByteOrder::kNative, size_t Alignment = 1>
1094 : static inline void writeI16x(void* p, int16_t x) noexcept { writeU16x<BO, Alignment>(p, uint16_t(x)); }
1095 :
1096 : template<size_t Alignment = 1>
1097 : static inline void writeI16uLE(void* p, int16_t x) noexcept { writeU16x<ByteOrder::kLE, Alignment>(p, uint16_t(x)); }
1098 : template<size_t Alignment = 1>
1099 : static inline void writeI16uBE(void* p, int16_t x) noexcept { writeU16x<ByteOrder::kBE, Alignment>(p, uint16_t(x)); }
1100 :
1101 : static inline void writeI16a(void* p, int16_t x) noexcept { writeU16x<ByteOrder::kNative, 2>(p, uint16_t(x)); }
1102 : static inline void writeI16aLE(void* p, int16_t x) noexcept { writeU16x<ByteOrder::kLE, 2>(p, uint16_t(x)); }
1103 : static inline void writeI16aBE(void* p, int16_t x) noexcept { writeU16x<ByteOrder::kBE, 2>(p, uint16_t(x)); }
1104 :
1105 : // Support - Memory Write Access - 24 Bits
1106 : // =======================================
1107 :
1108 : template<ByteOrder BO = ByteOrder::kNative>
1109 : static inline void writeU24u(void* p, uint32_t v) noexcept {
1110 : static_cast<uint8_t*>(p)[0] = uint8_t((v >> (BO == ByteOrder::kLE ? 0 : 16)) & 0xFFu);
1111 : static_cast<uint8_t*>(p)[1] = uint8_t((v >> (BO == ByteOrder::kLE ? 8 : 8)) & 0xFFu);
1112 : static_cast<uint8_t*>(p)[2] = uint8_t((v >> (BO == ByteOrder::kLE ? 16 : 0)) & 0xFFu);
1113 : }
1114 :
1115 : static inline void writeU24uLE(void* p, uint32_t v) noexcept { writeU24u<ByteOrder::kLE>(p, v); }
1116 : static inline void writeU24uBE(void* p, uint32_t v) noexcept { writeU24u<ByteOrder::kBE>(p, v); }
1117 :
1118 : // Support - Memory Write Access - 32 Bits
1119 : // =======================================
1120 :
1121 : template<ByteOrder BO = ByteOrder::kNative, size_t Alignment = 1>
1122 : static inline void writeU32x(void* p, uint32_t x) noexcept {
1123 : typedef typename Internal::AliasedUInt<uint32_t, Alignment>::T U32AlignedToN;
1124 128301 : static_cast<U32AlignedToN*>(p)[0] = (BO == ByteOrder::kNative) ? x : Support::byteswap32(x);
1125 : }
1126 :
1127 : template<size_t Alignment = 1>
1128 : static inline void writeU32u(void* p, uint32_t x) noexcept { writeU32x<ByteOrder::kNative, Alignment>(p, x); }
1129 : template<size_t Alignment = 1>
1130 : static inline void writeU32uLE(void* p, uint32_t x) noexcept { writeU32x<ByteOrder::kLE, Alignment>(p, x); }
1131 : template<size_t Alignment = 1>
1132 : static inline void writeU32uBE(void* p, uint32_t x) noexcept { writeU32x<ByteOrder::kBE, Alignment>(p, x); }
1133 :
1134 : static inline void writeU32a(void* p, uint32_t x) noexcept { writeU32x<ByteOrder::kNative, 4>(p, x); }
1135 : static inline void writeU32aLE(void* p, uint32_t x) noexcept { writeU32x<ByteOrder::kLE, 4>(p, x); }
1136 : static inline void writeU32aBE(void* p, uint32_t x) noexcept { writeU32x<ByteOrder::kBE, 4>(p, x); }
1137 :
1138 : template<ByteOrder BO = ByteOrder::kNative, size_t Alignment = 1>
1139 : static inline void writeI32x(void* p, int32_t x) noexcept { writeU32x<BO, Alignment>(p, uint32_t(x)); }
1140 :
1141 : template<size_t Alignment = 1>
1142 : static inline void writeI32u(void* p, int32_t x) noexcept { writeU32x<ByteOrder::kNative, Alignment>(p, uint32_t(x)); }
1143 : template<size_t Alignment = 1>
1144 : static inline void writeI32uLE(void* p, int32_t x) noexcept { writeU32x<ByteOrder::kLE, Alignment>(p, uint32_t(x)); }
1145 : template<size_t Alignment = 1>
1146 : static inline void writeI32uBE(void* p, int32_t x) noexcept { writeU32x<ByteOrder::kBE, Alignment>(p, uint32_t(x)); }
1147 :
1148 : static inline void writeI32a(void* p, int32_t x) noexcept { writeU32x<ByteOrder::kNative, 4>(p, uint32_t(x)); }
1149 : static inline void writeI32aLE(void* p, int32_t x) noexcept { writeU32x<ByteOrder::kLE, 4>(p, uint32_t(x)); }
1150 : static inline void writeI32aBE(void* p, int32_t x) noexcept { writeU32x<ByteOrder::kBE, 4>(p, uint32_t(x)); }
1151 :
1152 : // Support - Memory Write Access - 64 Bits
1153 : // =======================================
1154 :
1155 : template<ByteOrder BO = ByteOrder::kNative, size_t Alignment = 1>
1156 : static inline void writeU64x(void* p, uint64_t x) noexcept {
1157 : typedef typename Internal::AliasedUInt<uint64_t, Alignment>::T U64AlignedToN;
1158 0 : static_cast<U64AlignedToN*>(p)[0] = BO == ByteOrder::kNative ? x : byteswap64(x);
1159 : }
1160 :
1161 : template<size_t Alignment = 1>
1162 : static inline void writeU64u(void* p, uint64_t x) noexcept { writeU64x<ByteOrder::kNative, Alignment>(p, x); }
1163 : template<size_t Alignment = 1>
1164 0 : static inline void writeU64uLE(void* p, uint64_t x) noexcept { writeU64x<ByteOrder::kLE, Alignment>(p, x); }
1165 : template<size_t Alignment = 1>
1166 : static inline void writeU64uBE(void* p, uint64_t x) noexcept { writeU64x<ByteOrder::kBE, Alignment>(p, x); }
1167 :
1168 : static inline void writeU64a(void* p, uint64_t x) noexcept { writeU64x<ByteOrder::kNative, 8>(p, x); }
1169 : static inline void writeU64aLE(void* p, uint64_t x) noexcept { writeU64x<ByteOrder::kLE, 8>(p, x); }
1170 : static inline void writeU64aBE(void* p, uint64_t x) noexcept { writeU64x<ByteOrder::kBE, 8>(p, x); }
1171 :
1172 : template<ByteOrder BO = ByteOrder::kNative, size_t Alignment = 1>
1173 : static inline void writeI64x(void* p, int64_t x) noexcept { writeU64x<BO, Alignment>(p, uint64_t(x)); }
1174 :
1175 : template<size_t Alignment = 1>
1176 : static inline void writeI64u(void* p, int64_t x) noexcept { writeU64x<ByteOrder::kNative, Alignment>(p, uint64_t(x)); }
1177 : template<size_t Alignment = 1>
1178 : static inline void writeI64uLE(void* p, int64_t x) noexcept { writeU64x<ByteOrder::kLE, Alignment>(p, uint64_t(x)); }
1179 : template<size_t Alignment = 1>
1180 : static inline void writeI64uBE(void* p, int64_t x) noexcept { writeU64x<ByteOrder::kBE, Alignment>(p, uint64_t(x)); }
1181 :
1182 : static inline void writeI64a(void* p, int64_t x) noexcept { writeU64x<ByteOrder::kNative, 8>(p, uint64_t(x)); }
1183 : static inline void writeI64aLE(void* p, int64_t x) noexcept { writeU64x<ByteOrder::kLE, 8>(p, uint64_t(x)); }
1184 : static inline void writeI64aBE(void* p, int64_t x) noexcept { writeU64x<ByteOrder::kBE, 8>(p, uint64_t(x)); }
1185 :
1186 : // Support - Operators
1187 : // ===================
1188 :
1189 : //! \cond INTERNAL
1190 : struct Set { template<typename T> static inline T op(T x, T y) noexcept { DebugUtils::unused(x); return y; } };
1191 : struct SetNot { template<typename T> static inline T op(T x, T y) noexcept { DebugUtils::unused(x); return ~y; } };
1192 : struct And { template<typename T> static inline T op(T x, T y) noexcept { return x & y; } };
1193 28385 : struct AndNot { template<typename T> static inline T op(T x, T y) noexcept { return x & ~y; } };
1194 : struct NotAnd { template<typename T> static inline T op(T x, T y) noexcept { return ~x & y; } };
1195 84062 : struct Or { template<typename T> static inline T op(T x, T y) noexcept { return x | y; } };
1196 : struct Xor { template<typename T> static inline T op(T x, T y) noexcept { return x ^ y; } };
1197 : struct Add { template<typename T> static inline T op(T x, T y) noexcept { return x + y; } };
1198 : struct Sub { template<typename T> static inline T op(T x, T y) noexcept { return x - y; } };
1199 : struct Min { template<typename T> static inline T op(T x, T y) noexcept { return min<T>(x, y); } };
1200 : struct Max { template<typename T> static inline T op(T x, T y) noexcept { return max<T>(x, y); } };
1201 : //! \endcond
1202 :
1203 : // Support - BitWordIterator
1204 : // =========================
1205 :
1206 : //! Iterates over each bit in a number which is set to 1.
1207 : //!
1208 : //! Example of use:
1209 : //!
1210 : //! ```
1211 : //! uint32_t bitsToIterate = 0x110F;
1212 : //! Support::BitWordIterator<uint32_t> it(bitsToIterate);
1213 : //!
1214 : //! while (it.hasNext()) {
1215 : //! uint32_t bitIndex = it.next();
1216 : //! std::printf("Bit at %u is set\n", unsigned(bitIndex));
1217 : //! }
1218 : //! ```
1219 : template<typename T>
1220 : class BitWordIterator {
1221 : public:
1222 : ASMJIT_FORCE_INLINE explicit BitWordIterator(T bitWord) noexcept
1223 32330 : : _bitWord(bitWord) {}
1224 :
1225 : ASMJIT_FORCE_INLINE void init(T bitWord) noexcept { _bitWord = bitWord; }
1226 : ASMJIT_FORCE_INLINE bool hasNext() const noexcept { return _bitWord != 0; }
1227 :
1228 : ASMJIT_FORCE_INLINE uint32_t next() noexcept {
1229 : ASMJIT_ASSERT(_bitWord != 0);
1230 : uint32_t index = ctz(_bitWord);
1231 1227781 : _bitWord ^= T(1u) << index;
1232 : return index;
1233 : }
1234 :
1235 : T _bitWord;
1236 : };
1237 :
1238 : // Support - BitWordFlipIterator
1239 : // =============================
1240 :
1241 : template<typename T>
1242 : class BitWordFlipIterator {
1243 : public:
1244 : ASMJIT_FORCE_INLINE explicit BitWordFlipIterator(T bitWord) noexcept
1245 : : _bitWord(bitWord) {}
1246 :
1247 : ASMJIT_FORCE_INLINE void init(T bitWord) noexcept { _bitWord = bitWord; }
1248 : ASMJIT_FORCE_INLINE bool hasNext() const noexcept { return _bitWord != 0; }
1249 :
1250 : ASMJIT_FORCE_INLINE uint32_t nextAndFlip() noexcept {
1251 : ASMJIT_ASSERT(_bitWord != 0);
1252 : uint32_t index = ctz(_bitWord);
1253 : _bitWord ^= T(1u) << index;
1254 : return index;
1255 : }
1256 :
1257 : T _bitWord;
1258 : T _xorMask;
1259 : };
1260 :
1261 : // Support - BitVectorOps
1262 : // ======================
1263 :
1264 : //! \cond
1265 : namespace Internal {
1266 : template<typename T, class OperatorT, class FullWordOpT>
1267 27592 : static inline void bitVectorOp(T* buf, size_t index, size_t count) noexcept {
1268 27592 : if (count == 0)
1269 : return;
1270 :
1271 : const size_t kTSizeInBits = bitSizeOf<T>();
1272 27592 : size_t vecIndex = index / kTSizeInBits; // T[]
1273 27592 : size_t bitIndex = index % kTSizeInBits; // T[][]
1274 :
1275 27592 : buf += vecIndex;
1276 :
1277 : // The first BitWord requires special handling to preserve bits outside the fill region.
1278 : const T kFillMask = allOnes<T>();
1279 27592 : size_t firstNBits = min<size_t>(kTSizeInBits - bitIndex, count);
1280 :
1281 27592 : buf[0] = OperatorT::op(buf[0], (kFillMask >> (kTSizeInBits - firstNBits)) << bitIndex);
1282 27592 : buf++;
1283 27592 : count -= firstNBits;
1284 :
1285 : // All bits between the first and last affected BitWords can be just filled.
1286 27592 : while (count >= kTSizeInBits) {
1287 0 : buf[0] = FullWordOpT::op(buf[0], kFillMask);
1288 0 : buf++;
1289 0 : count -= kTSizeInBits;
1290 : }
1291 :
1292 : // The last BitWord requires special handling as well
1293 27592 : if (count)
1294 0 : buf[0] = OperatorT::op(buf[0], kFillMask >> (kTSizeInBits - count));
1295 : }
1296 : }
1297 : //! \endcond
1298 :
1299 : //! Sets bit in a bit-vector `buf` at `index`.
1300 : template<typename T>
1301 : static inline bool bitVectorGetBit(T* buf, size_t index) noexcept {
1302 : const size_t kTSizeInBits = bitSizeOf<T>();
1303 :
1304 913260 : size_t vecIndex = index / kTSizeInBits;
1305 : size_t bitIndex = index % kTSizeInBits;
1306 :
1307 913260 : return bool((buf[vecIndex] >> bitIndex) & 0x1u);
1308 : }
1309 :
1310 : //! Sets bit in a bit-vector `buf` at `index` to `value`.
1311 : template<typename T>
1312 1055802 : static inline void bitVectorSetBit(T* buf, size_t index, bool value) noexcept {
1313 : const size_t kTSizeInBits = bitSizeOf<T>();
1314 :
1315 1055802 : size_t vecIndex = index / kTSizeInBits;
1316 1055802 : size_t bitIndex = index % kTSizeInBits;
1317 :
1318 1055802 : T bitMask = T(1u) << bitIndex;
1319 1055802 : if (value)
1320 669438 : buf[vecIndex] |= bitMask;
1321 : else
1322 386364 : buf[vecIndex] &= ~bitMask;
1323 1055802 : }
1324 :
1325 : //! Sets bit in a bit-vector `buf` at `index` to `value`.
1326 : template<typename T>
1327 : static inline void bitVectorFlipBit(T* buf, size_t index) noexcept {
1328 : const size_t kTSizeInBits = bitSizeOf<T>();
1329 :
1330 : size_t vecIndex = index / kTSizeInBits;
1331 : size_t bitIndex = index % kTSizeInBits;
1332 :
1333 : T bitMask = T(1u) << bitIndex;
1334 : buf[vecIndex] ^= bitMask;
1335 : }
1336 :
1337 : //! Fills `count` bits in bit-vector `buf` starting at bit-index `index`.
1338 : template<typename T>
1339 27492 : static inline void bitVectorFill(T* buf, size_t index, size_t count) noexcept { Internal::bitVectorOp<T, Or, Set>(buf, index, count); }
1340 :
1341 : //! Clears `count` bits in bit-vector `buf` starting at bit-index `index`.
1342 : template<typename T>
1343 100 : static inline void bitVectorClear(T* buf, size_t index, size_t count) noexcept { Internal::bitVectorOp<T, AndNot, SetNot>(buf, index, count); }
1344 :
1345 : template<typename T>
1346 100 : static inline size_t bitVectorIndexOf(T* buf, size_t start, bool value) noexcept {
1347 : const size_t kTSizeInBits = bitSizeOf<T>();
1348 100 : size_t vecIndex = start / kTSizeInBits; // T[]
1349 100 : size_t bitIndex = start % kTSizeInBits; // T[][]
1350 :
1351 100 : T* p = buf + vecIndex;
1352 :
1353 : // We always look for zeros, if value is `true` we have to flip all bits before the search.
1354 : const T kFillMask = allOnes<T>();
1355 100 : const T kFlipMask = value ? T(0) : kFillMask;
1356 :
1357 : // The first BitWord requires special handling as there are some bits we want to ignore.
1358 100 : T bits = (*p ^ kFlipMask) & (kFillMask << bitIndex);
1359 : for (;;) {
1360 100 : if (bits)
1361 100 : return (size_t)(p - buf) * kTSizeInBits + ctz(bits);
1362 0 : bits = *++p ^ kFlipMask;
1363 : }
1364 : }
1365 :
1366 : // Support - BitVectorIterator
1367 : // ===========================
1368 :
1369 : template<typename T>
1370 : class BitVectorIterator {
1371 : public:
1372 : const T* _ptr;
1373 : size_t _idx;
1374 : size_t _end;
1375 : T _current;
1376 :
1377 : ASMJIT_FORCE_INLINE BitVectorIterator(const BitVectorIterator& other) noexcept = default;
1378 :
1379 : ASMJIT_FORCE_INLINE BitVectorIterator(const T* data, size_t numBitWords, size_t start = 0) noexcept {
1380 : init(data, numBitWords, start);
1381 : }
1382 :
1383 : ASMJIT_FORCE_INLINE void init(const T* data, size_t numBitWords, size_t start = 0) noexcept {
1384 : const T* ptr = data + (start / bitSizeOf<T>());
1385 : size_t idx = alignDown(start, bitSizeOf<T>());
1386 54984 : size_t end = numBitWords * bitSizeOf<T>();
1387 :
1388 : T bitWord = T(0);
1389 54984 : if (idx < end) {
1390 54984 : bitWord = *ptr++ & (allOnes<T>() << (start % bitSizeOf<T>()));
1391 56570 : while (!bitWord && (idx += bitSizeOf<T>()) < end)
1392 1586 : bitWord = *ptr++;
1393 : }
1394 :
1395 : _ptr = ptr;
1396 : _idx = idx;
1397 : _end = end;
1398 : _current = bitWord;
1399 : }
1400 :
1401 : ASMJIT_FORCE_INLINE bool hasNext() const noexcept {
1402 : return _current != T(0);
1403 : }
1404 :
1405 : ASMJIT_FORCE_INLINE size_t next() noexcept {
1406 : T bitWord = _current;
1407 : ASMJIT_ASSERT(bitWord != T(0));
1408 :
1409 : uint32_t bit = ctz(bitWord);
1410 0 : bitWord ^= T(1u) << bit;
1411 :
1412 0 : size_t n = _idx + bit;
1413 0 : while (!bitWord && (_idx += bitSizeOf<T>()) < _end)
1414 0 : bitWord = *_ptr++;
1415 :
1416 : _current = bitWord;
1417 : return n;
1418 : }
1419 :
1420 : ASMJIT_FORCE_INLINE size_t peekNext() const noexcept {
1421 : ASMJIT_ASSERT(_current != T(0));
1422 : return _idx + ctz(_current);
1423 : }
1424 : };
1425 :
1426 : // Support - BitVectorOpIterator
1427 : // =============================
1428 :
1429 : template<typename T, class OperatorT>
1430 : class BitVectorOpIterator {
1431 : public:
1432 : enum : uint32_t {
1433 : kTSizeInBits = bitSizeOf<T>()
1434 : };
1435 :
1436 : const T* _aPtr;
1437 : const T* _bPtr;
1438 : size_t _idx;
1439 : size_t _end;
1440 : T _current;
1441 :
1442 27492 : ASMJIT_FORCE_INLINE BitVectorOpIterator(const T* aData, const T* bData, size_t numBitWords, size_t start = 0) noexcept {
1443 : init(aData, bData, numBitWords, start);
1444 : }
1445 :
1446 : ASMJIT_FORCE_INLINE void init(const T* aData, const T* bData, size_t numBitWords, size_t start = 0) noexcept {
1447 : const T* aPtr = aData + (start / bitSizeOf<T>());
1448 : const T* bPtr = bData + (start / bitSizeOf<T>());
1449 : size_t idx = alignDown(start, bitSizeOf<T>());
1450 27492 : size_t end = numBitWords * bitSizeOf<T>();
1451 :
1452 : T bitWord = T(0);
1453 27492 : if (idx < end) {
1454 27492 : bitWord = OperatorT::op(*aPtr++, *bPtr++) & (allOnes<T>() << (start % bitSizeOf<T>()));
1455 28285 : while (!bitWord && (idx += kTSizeInBits) < end)
1456 793 : bitWord = OperatorT::op(*aPtr++, *bPtr++);
1457 : }
1458 :
1459 27492 : _aPtr = aPtr;
1460 27492 : _bPtr = bPtr;
1461 27492 : _idx = idx;
1462 27492 : _end = end;
1463 27492 : _current = bitWord;
1464 : }
1465 :
1466 : ASMJIT_FORCE_INLINE bool hasNext() noexcept {
1467 27492 : return _current != T(0);
1468 : }
1469 :
1470 : ASMJIT_FORCE_INLINE size_t next() noexcept {
1471 : T bitWord = _current;
1472 : ASMJIT_ASSERT(bitWord != T(0));
1473 :
1474 : uint32_t bit = ctz(bitWord);
1475 0 : bitWord ^= T(1u) << bit;
1476 :
1477 0 : size_t n = _idx + bit;
1478 0 : while (!bitWord && (_idx += kTSizeInBits) < _end)
1479 0 : bitWord = OperatorT::op(*_aPtr++, *_bPtr++);
1480 :
1481 0 : _current = bitWord;
1482 : return n;
1483 : }
1484 : };
1485 :
1486 : // Support - Sorting
1487 : // =================
1488 :
1489 : //! Sort order.
1490 : enum class SortOrder : uint32_t {
1491 : //!< Ascending order.
1492 : kAscending = 0,
1493 : //!< Descending order.
1494 : kDescending = 1
1495 : };
1496 :
1497 : //! A helper class that provides comparison of any user-defined type that
1498 : //! implements `<` and `>` operators (primitive types are supported as well).
1499 : template<SortOrder kOrder = SortOrder::kAscending>
1500 : struct Compare {
1501 : template<typename A, typename B>
1502 : inline int operator()(const A& a, const B& b) const noexcept {
1503 100 : return kOrder == SortOrder::kAscending ? int(a > b) - int(a < b) : int(a < b) - int(a > b);
1504 : }
1505 : };
1506 :
1507 : //! Insertion sort.
1508 : template<typename T, typename CompareT = Compare<SortOrder::kAscending>>
1509 78797 : static inline void iSort(T* base, size_t size, const CompareT& cmp = CompareT()) noexcept {
1510 281944 : for (T* pm = base + 1; pm < base + size; pm++)
1511 338150 : for (T* pl = pm; pl > base && cmp(pl[-1], pl[0]) > 0; pl--)
1512 : std::swap(pl[-1], pl[0]);
1513 78797 : }
1514 :
1515 : //! \cond
1516 : namespace Internal {
1517 : //! Quick-sort implementation.
1518 : template<typename T, class CompareT>
1519 : struct QSortImpl {
1520 : enum : size_t {
1521 : kStackSize = 64 * 2,
1522 : kISortThreshold = 7
1523 : };
1524 :
1525 : // Based on "PDCLib - Public Domain C Library" and rewritten to C++.
1526 82476 : static void sort(T* base, size_t size, const CompareT& cmp) noexcept {
1527 82476 : T* end = base + size;
1528 : T* stack[kStackSize];
1529 : T** stackptr = stack;
1530 :
1531 : for (;;) {
1532 122812 : if ((size_t)(end - base) > kISortThreshold) {
1533 : // We work from second to last - first will be pivot element.
1534 20168 : T* pi = base + 1;
1535 20168 : T* pj = end - 1;
1536 20168 : std::swap(base[(size_t)(end - base) / 2], base[0]);
1537 :
1538 20168 : if (cmp(*pi , *pj ) > 0) std::swap(*pi , *pj );
1539 20168 : if (cmp(*base, *pj ) > 0) std::swap(*base, *pj );
1540 20168 : if (cmp(*pi , *base) > 0) std::swap(*pi , *base);
1541 :
1542 : // Now we have the median for pivot element, entering main loop.
1543 : for (;;) {
1544 206323 : while (pi < pj && cmp(*++pi, *base) < 0) continue; // Move `i` right until `*i >= pivot`.
1545 214096 : while (pj > base && cmp(*--pj, *base) > 0) continue; // Move `j` left until `*j <= pivot`.
1546 :
1547 148339 : if (pi > pj) break;
1548 : std::swap(*pi, *pj);
1549 : }
1550 :
1551 : // Move pivot into correct place.
1552 : std::swap(*base, *pj);
1553 :
1554 : // Larger subfile base / end to stack, sort smaller.
1555 20168 : if (pj - base > end - pi) {
1556 : // Left is larger.
1557 5219 : *stackptr++ = base;
1558 5219 : *stackptr++ = pj;
1559 : base = pi;
1560 : }
1561 : else {
1562 : // Right is larger.
1563 14949 : *stackptr++ = pi;
1564 14949 : *stackptr++ = end;
1565 : end = pj;
1566 : }
1567 : ASMJIT_ASSERT(stackptr <= stack + kStackSize);
1568 123741 : }
1569 : else {
1570 : // UB sanitizer doesn't like applying offset to a nullptr base.
1571 102644 : if (base != end)
1572 78797 : iSort(base, (size_t)(end - base), cmp);
1573 :
1574 102644 : if (stackptr == stack)
1575 : break;
1576 :
1577 20168 : end = *--stackptr;
1578 20168 : base = *--stackptr;
1579 : }
1580 : }
1581 82476 : }
1582 : };
1583 : }
1584 : //! \endcond
1585 :
1586 : //! Quick sort implementation.
1587 : //!
1588 : //! The main reason to provide a custom qsort implementation is that we needed something that will
1589 : //! never throw `bad_alloc` exception. This implementation doesn't use dynamic memory allocation.
1590 : template<typename T, class CompareT = Compare<SortOrder::kAscending>>
1591 : static inline void qSort(T* base, size_t size, const CompareT& cmp = CompareT()) noexcept {
1592 82476 : Internal::QSortImpl<T, CompareT>::sort(base, size, cmp);
1593 : }
1594 :
1595 : // Support - Array
1596 : // ===============
1597 :
1598 : //! Array type, similar to std::array<T, N>, with the possibility to use enums in operator[].
1599 : //!
1600 : //! \note The array has C semantics - the elements in the array are not initialized.
1601 : template<typename T, size_t N>
1602 137460 : struct Array {
1603 : //! \name Members
1604 : //! \{
1605 :
1606 : //! The underlying array data, use \ref data() to access it.
1607 : T _data[N];
1608 :
1609 : //! \}
1610 :
1611 : //! \cond
1612 : // std compatibility.
1613 : typedef T value_type;
1614 : typedef size_t size_type;
1615 : typedef ptrdiff_t difference_type;
1616 :
1617 : typedef value_type& reference;
1618 : typedef const value_type& const_reference;
1619 :
1620 : typedef value_type* pointer;
1621 : typedef const value_type* const_pointer;
1622 :
1623 : typedef pointer iterator;
1624 : typedef const_pointer const_iterator;
1625 : //! \endcond
1626 :
1627 : //! \name Overloaded Operators
1628 : //! \{
1629 :
1630 : template<typename Index>
1631 259696 : inline T& operator[](const Index& index) noexcept {
1632 : typedef typename Internal::StdInt<sizeof(Index), 1>::Type U;
1633 : ASMJIT_ASSERT(U(index) < N);
1634 5568013 : return _data[U(index)];
1635 : }
1636 :
1637 : template<typename Index>
1638 : inline const T& operator[](const Index& index) const noexcept {
1639 : typedef typename Internal::StdInt<sizeof(Index), 1>::Type U;
1640 : ASMJIT_ASSERT(U(index) < N);
1641 1252162 : return _data[U(index)];
1642 : }
1643 :
1644 : inline bool operator==(const Array& other) const noexcept {
1645 : for (size_t i = 0; i < N; i++)
1646 : if (_data[i] != other._data[i])
1647 : return false;
1648 : return true;
1649 : }
1650 :
1651 : inline bool operator!=(const Array& other) const noexcept {
1652 : return !operator==(other);
1653 : }
1654 :
1655 : //! \}
1656 :
1657 : //! \name Accessors
1658 : //! \{
1659 :
1660 : inline bool empty() const noexcept { return false; }
1661 : inline size_t size() const noexcept { return N; }
1662 :
1663 37131 : inline T* data() noexcept { return _data; }
1664 : inline const T* data() const noexcept { return _data; }
1665 :
1666 : inline T& front() noexcept { return _data[0]; }
1667 : inline const T& front() const noexcept { return _data[0]; }
1668 :
1669 : inline T& back() noexcept { return _data[N - 1]; }
1670 : inline const T& back() const noexcept { return _data[N - 1]; }
1671 :
1672 : inline T* begin() noexcept { return _data; }
1673 : inline T* end() noexcept { return _data + N; }
1674 :
1675 : inline const T* begin() const noexcept { return _data; }
1676 : inline const T* end() const noexcept { return _data + N; }
1677 :
1678 : inline const T* cbegin() const noexcept { return _data; }
1679 : inline const T* cend() const noexcept { return _data + N; }
1680 :
1681 : //! \}
1682 :
1683 : //! \name Utilities
1684 : //! \{
1685 :
1686 : inline void swap(Array& other) noexcept {
1687 0 : for (size_t i = 0; i < N; i++)
1688 : std::swap(_data[i], other._data[i]);
1689 : }
1690 :
1691 54984 : inline void fill(const T& value) noexcept {
1692 9440344 : for (size_t i = 0; i < N; i++)
1693 7508288 : _data[i] = value;
1694 54984 : }
1695 :
1696 : inline void copyFrom(const Array& other) noexcept {
1697 : for (size_t i = 0; i < N; i++)
1698 : _data[i] = other._data[i];
1699 : }
1700 :
1701 : template<typename Operator>
1702 : inline void combine(const Array& other) noexcept {
1703 2071420 : for (size_t i = 0; i < N; i++)
1704 1657136 : _data[i] = Operator::op(_data[i], other._data[i]);
1705 : }
1706 :
1707 : template<typename Operator>
1708 : inline T aggregate(T initialValue = T()) const noexcept {
1709 : T value = initialValue;
1710 : for (size_t i = 0; i < N; i++)
1711 : value = Operator::op(value, _data[i]);
1712 : return value;
1713 : }
1714 :
1715 : template<typename Fn>
1716 : inline void forEach(Fn&& fn) noexcept {
1717 549840 : for (size_t i = 0; i < N; i++)
1718 219936 : fn(_data[i]);
1719 : }
1720 : //! \}
1721 : };
1722 :
1723 : // Support::Temporary
1724 : // ==================
1725 :
1726 : //! Used to pass a temporary buffer to:
1727 : //!
1728 : //! - Containers that use user-passed buffer as an initial storage (still can grow).
1729 : //! - Zone allocator that would use the temporary buffer as a first block.
1730 : struct Temporary {
1731 : //! \name Members
1732 : //! \{
1733 :
1734 : void* _data;
1735 : size_t _size;
1736 :
1737 : //! \}
1738 :
1739 : //! \name Construction & Destruction
1740 : //! \{
1741 :
1742 : inline constexpr Temporary(const Temporary& other) noexcept = default;
1743 : inline constexpr Temporary(void* data, size_t size) noexcept
1744 : : _data(data),
1745 : _size(size) {}
1746 :
1747 : //! \}
1748 :
1749 : //! \name Overloaded Operators
1750 : //! \{
1751 :
1752 : inline Temporary& operator=(const Temporary& other) noexcept = default;
1753 :
1754 : //! \}
1755 :
1756 : //! \name Accessors
1757 : //! \{
1758 :
1759 : //! Returns the data storage.
1760 : template<typename T = void>
1761 0 : inline constexpr T* data() const noexcept { return static_cast<T*>(_data); }
1762 : //! Returns the data storage size in bytes.
1763 0 : inline constexpr size_t size() const noexcept { return _size; }
1764 :
1765 : //! \}
1766 : };
1767 :
1768 : } // {Support}
1769 :
1770 : //! \}
1771 :
1772 : ASMJIT_END_NAMESPACE
1773 :
1774 : #endif // ASMJIT_CORE_SUPPORT_H_INCLUDED
1775 :
1776 : #endif // __PLUMED_HAS_ASMJIT
|
