#pragma once #ifdef _MSC_VER #include #else #include #endif #include #include #include #include #include #include #include #include // Assume little-endian #define IS_LE_MACHINE 1 #define IS_BE_MACHINE 0 #ifdef _MSC_VER #define ASSUME(cond) __assume(cond) #define LIKELY #define UNLIKELY #define SAFE_BUFFERS __declspec(safebuffers) #define NEVER_INLINE __declspec(noinline) #define FORCE_INLINE __forceinline #else #define ASSUME(cond) do { if (!(cond)) __builtin_unreachable(); } while (0) #define LIKELY(cond) __builtin_expect(!!(cond), 1) #define UNLIKELY(cond) __builtin_expect(!!(cond), 0) #define SAFE_BUFFERS #define NEVER_INLINE __attribute__((noinline)) #define FORCE_INLINE __attribute__((always_inline)) inline #endif #define CHECK_SIZE(type, size) static_assert(sizeof(type) == size, "Invalid " #type " type size") #define CHECK_ALIGN(type, align) static_assert(alignof(type) == align, "Invalid " #type " type alignment") #define CHECK_MAX_SIZE(type, size) static_assert(sizeof(type) <= size, #type " type size is too big") #define CHECK_SIZE_ALIGN(type, size, align) CHECK_SIZE(type, size); CHECK_ALIGN(type, align) // Return 32 bit sizeof() to avoid widening/narrowing conversions with size_t #define SIZE_32(...) static_cast(sizeof(__VA_ARGS__)) // Variant pattern matching helper #define MATCH(arg, ...) constexpr(std::is_same_v, __VA_ARGS__>) #define CONCATENATE_DETAIL(x, y) x ## y #define CONCATENATE(x, y) CONCATENATE_DETAIL(x, y) #define STRINGIZE_DETAIL(x) #x "" #define STRINGIZE(x) STRINGIZE_DETAIL(x) #define HERE "\n(in file " __FILE__ ":" STRINGIZE(__LINE__) ")" // Ensure that the expression evaluates to true. Obsolete. //#define EXPECTS(...) do { if (!(__VA_ARGS__)) fmt::raw_error("Precondition failed: " #__VA_ARGS__ HERE); } while (0) //#define ENSURES(...) do { if (!(__VA_ARGS__)) fmt::raw_error("Postcondition failed: " #__VA_ARGS__ HERE); } while (0) #define DECLARE(...) decltype(__VA_ARGS__) __VA_ARGS__ #define STR_CASE(...) case __VA_ARGS__: return #__VA_ARGS__ using schar = signed char; using uchar = unsigned char; using ushort = unsigned short; using uint = unsigned int; using ulong = unsigned long; using ullong = unsigned long long; using llong = long long; #if __APPLE__ using uptr = std::uint64_t; #else using uptr = std::uintptr_t; #endif using u8 = std::uint8_t; using u16 = std::uint16_t; using u32 = std::uint32_t; using u64 = std::uint64_t; using s8 = std::int8_t; using s16 = std::int16_t; using s32 = std::int32_t; using s64 = std::int64_t; using steady_clock = std::conditional< std::chrono::high_resolution_clock::is_steady, std::chrono::high_resolution_clock, std::chrono::steady_clock>::type; namespace gsl { using std::byte; } // Formatting helper, type-specific preprocessing for improving safety and functionality template struct fmt_unveil; template using fmt_unveil_t = typename fmt_unveil::type; struct fmt_type_info; namespace fmt { template const fmt_type_info* get_type_info(); } template struct se_storage; template class se_t; template class atomic_t; #if defined(__INTELLISENSE__) && !defined(_MSC_VER) namespace std { template using void_t = void; } #endif // Extract T::simple_type if available, remove cv qualifiers template struct simple_type_helper { using type = typename std::remove_cv::type; }; template struct simple_type_helper> { using type = typename T::simple_type; }; template using simple_t = typename simple_type_helper::type; // Bool type equivalent class b8 { u8 m_value; public: b8() = default; constexpr b8(bool value) : m_value(value) { } constexpr operator bool() const { return m_value != 0; } }; // Bool wrapper for restricting bool result conversions struct explicit_bool_t { const bool value; constexpr explicit_bool_t(bool value) : value(value) { } explicit constexpr operator bool() const { return value; } }; #ifndef _MSC_VER using u128 = __uint128_t; using s128 = __int128_t; #else // Unsigned 128-bit integer implementation (TODO) struct alignas(16) u128 { u64 lo, hi; u128() = default; constexpr u128(u64 l) : lo(l) , hi(0) { } friend u128 operator+(const u128& l, const u128& r) { u128 value; _addcarry_u64(_addcarry_u64(0, r.lo, l.lo, &value.lo), r.hi, l.hi, &value.hi); return value; } friend u128 operator+(const u128& l, u64 r) { u128 value; _addcarry_u64(_addcarry_u64(0, r, l.lo, &value.lo), l.hi, 0, &value.hi); return value; } friend u128 operator+(u64 l, const u128& r) { u128 value; _addcarry_u64(_addcarry_u64(0, r.lo, l, &value.lo), 0, r.hi, &value.hi); return value; } friend u128 operator-(const u128& l, const u128& r) { u128 value; _subborrow_u64(_subborrow_u64(0, r.lo, l.lo, &value.lo), r.hi, l.hi, &value.hi); return value; } friend u128 operator-(const u128& l, u64 r) { u128 value; _subborrow_u64(_subborrow_u64(0, r, l.lo, &value.lo), 0, l.hi, &value.hi); return value; } friend u128 operator-(u64 l, const u128& r) { u128 value; _subborrow_u64(_subborrow_u64(0, r.lo, l, &value.lo), r.hi, 0, &value.hi); return value; } u128 operator+() const { return *this; } u128 operator-() const { u128 value; _subborrow_u64(_subborrow_u64(0, lo, 0, &value.lo), hi, 0, &value.hi); return value; } u128& operator++() { _addcarry_u64(_addcarry_u64(0, 1, lo, &lo), 0, hi, &hi); return *this; } u128 operator++(int) { u128 value = *this; _addcarry_u64(_addcarry_u64(0, 1, lo, &lo), 0, hi, &hi); return value; } u128& operator--() { _subborrow_u64(_subborrow_u64(0, 1, lo, &lo), 0, hi, &hi); return *this; } u128 operator--(int) { u128 value = *this; _subborrow_u64(_subborrow_u64(0, 1, lo, &lo), 0, hi, &hi); return value; } u128 operator~() const { u128 value; value.lo = ~lo; value.hi = ~hi; return value; } friend u128 operator&(const u128& l, const u128& r) { u128 value; value.lo = l.lo & r.lo; value.hi = l.hi & r.hi; return value; } friend u128 operator|(const u128& l, const u128& r) { u128 value; value.lo = l.lo | r.lo; value.hi = l.hi | r.hi; return value; } friend u128 operator^(const u128& l, const u128& r) { u128 value; value.lo = l.lo ^ r.lo; value.hi = l.hi ^ r.hi; return value; } u128& operator+=(const u128& r) { _addcarry_u64(_addcarry_u64(0, r.lo, lo, &lo), r.hi, hi, &hi); return *this; } u128& operator+=(uint64_t r) { _addcarry_u64(_addcarry_u64(0, r, lo, &lo), 0, hi, &hi); return *this; } u128& operator&=(const u128& r) { lo &= r.lo; hi &= r.hi; return *this; } u128& operator|=(const u128& r) { lo |= r.lo; hi |= r.hi; return *this; } u128& operator^=(const u128& r) { lo ^= r.lo; hi ^= r.hi; return *this; } }; // Signed 128-bit integer implementation (TODO) struct alignas(16) s128 { u64 lo; s64 hi; s128() = default; constexpr s128(s64 l) : hi(l >> 63) , lo(l) { } constexpr s128(u64 l) : hi(0) , lo(l) { } }; #endif CHECK_SIZE_ALIGN(u128, 16, 16); CHECK_SIZE_ALIGN(s128, 16, 16); union alignas(2) f16 { u16 _u16; u8 _u8[2]; explicit f16(u16 raw) { _u16 = raw; } explicit operator float() const { // See http://stackoverflow.com/a/26779139 // The conversion doesn't handle NaN/Inf u32 raw = ((_u16 & 0x8000) << 16) | // Sign (just moved) (((_u16 & 0x7c00) + 0x1C000) << 13) | // Exponent ( exp - 15 + 127) ((_u16 & 0x03FF) << 13); // Mantissa return (float&)raw; } }; CHECK_SIZE_ALIGN(f16, 2, 2); using f32 = float; using f64 = double; struct ignore { template ignore(T) { } }; template ::value>> constexpr T align(const T& value, ullong align) { return static_cast((value + (align - 1)) & ~(align - 1)); } template inline u32 offset32(T T2::*const mptr) { #ifdef _MSC_VER static_assert(sizeof(mptr) == sizeof(u32), "Invalid pointer-to-member size"); return reinterpret_cast(mptr); #elif __GNUG__ static_assert(sizeof(mptr) == sizeof(std::size_t), "Invalid pointer-to-member size"); return static_cast(reinterpret_cast(mptr)); #else static_assert(sizeof(mptr) == 0, "Invalid pointer-to-member size"); #endif } template struct offset32_array { static_assert(std::is_array::value, "Invalid pointer-to-member type (array expected)"); template static inline u32 index32(const Arg& arg) { return SIZE_32(std::remove_extent_t) * static_cast(arg); } }; template struct offset32_array> { template static inline u32 index32(const Arg& arg) { return SIZE_32(T) * static_cast(arg); } }; template struct offset32_detail; template inline u32 offset32(T T2::*const mptr, const Arg& arg, const Args&... args) { return offset32_detail::offset32(mptr, arg, args...); } template struct offset32_detail { template static inline u32 offset32(T T2::*const mptr, const Arg& arg, const Args&... args) { return ::offset32(mptr, args...) + offset32_array::index32(arg); } }; template struct offset32_detail { template static inline u32 offset32(T T2::*const mptr, T3 T4::*const mptr2, const Args&... args) { return ::offset32(mptr) + ::offset32(mptr2, args...); } }; inline u32 cntlz32(u32 arg, bool nonzero = false) { #ifdef _MSC_VER ulong res; return _BitScanReverse(&res, arg) || nonzero ? res ^ 31 : 32; #else return arg || nonzero ? __builtin_clzll(arg) - 32 : 32; #endif } inline u64 cntlz64(u64 arg, bool nonzero = false) { #ifdef _MSC_VER ulong res; return _BitScanReverse64(&res, arg) || nonzero ? res ^ 63 : 64; #else return arg || nonzero ? __builtin_clzll(arg) : 64; #endif } inline u32 cnttz32(u32 arg, bool nonzero = false) { #ifdef _MSC_VER ulong res; return _BitScanForward(&res, arg) || nonzero ? res : 32; #else return arg || nonzero ? __builtin_ctzll(arg) : 32; #endif } inline u64 cnttz64(u64 arg, bool nonzero = false) { #ifdef _MSC_VER ulong res; return _BitScanForward64(&res, arg) || nonzero ? res : 64; #else return arg || nonzero ? __builtin_ctzll(arg) : 64; #endif } // Helper function, used by ""_u16, ""_u32, ""_u64 constexpr u8 to_u8(char c) { return static_cast(c); } // Convert 2-byte string to u16 value like reinterpret_cast does constexpr u16 operator""_u16(const char* s, std::size_t length) { return length != 2 ? throw s : #if IS_LE_MACHINE == 1 to_u8(s[1]) << 8 | to_u8(s[0]); #endif } // Convert 4-byte string to u32 value like reinterpret_cast does constexpr u32 operator""_u32(const char* s, std::size_t length) { return length != 4 ? throw s : #if IS_LE_MACHINE == 1 to_u8(s[3]) << 24 | to_u8(s[2]) << 16 | to_u8(s[1]) << 8 | to_u8(s[0]); #endif } // Convert 8-byte string to u64 value like reinterpret_cast does constexpr u64 operator""_u64(const char* s, std::size_t length) { return length != 8 ? throw s : #if IS_LE_MACHINE == 1 static_cast(to_u8(s[7]) << 24 | to_u8(s[6]) << 16 | to_u8(s[5]) << 8 | to_u8(s[4])) << 32 | to_u8(s[3]) << 24 | to_u8(s[2]) << 16 | to_u8(s[1]) << 8 | to_u8(s[0]); #endif } namespace fmt { [[noreturn]] void raw_error(const char* msg); [[noreturn]] void raw_verify_error(const char* msg, const fmt_type_info* sup, u64 arg); [[noreturn]] void raw_narrow_error(const char* msg, const fmt_type_info* sup, u64 arg); } struct verify_func { template bool operator()(T&& value) const { if (std::forward(value)) { return true; } return false; } }; template struct verify_impl { const char* cause; template auto operator,(T&& value) const { // Verification (can be safely disabled) if (!verify_func()(std::forward(value))) { fmt::raw_verify_error(cause, nullptr, N); } return verify_impl{cause}; } }; // Verification helper, checks several conditions delimited with comma operator inline auto verify(const char* cause) { return verify_impl<0>{cause}; } // Verification helper (returns value or lvalue reference, may require to use verify_move instead) template inline T verify(const char* cause, T&& value, F&& pred = F()) { if (!pred(std::forward(value))) { using unref = std::remove_const_t>; fmt::raw_verify_error(cause, fmt::get_type_info>(), fmt_unveil::get(value)); } return std::forward(value); } // Verification helper (must be used in return expression or in place of std::move) template inline std::remove_reference_t&& verify_move(const char* cause, T&& value, F&& pred = F()) { if (!pred(std::forward(value))) { using unref = std::remove_const_t>; fmt::raw_verify_error(cause, fmt::get_type_info>(), fmt_unveil::get(value)); } return std::move(value); } // narrow() function details template struct narrow_impl { // Temporarily (diagnostic) static_assert(std::is_void::value, "narrow_impl<> specialization not found"); // Returns true if value cannot be represented in type To static constexpr bool test(const From& value) { // Unspecialized cases (including cast to void) always considered narrowing return true; } }; // Unsigned to unsigned narrowing template struct narrow_impl::value && std::is_unsigned::value>> { static constexpr bool test(const From& value) { return sizeof(To) < sizeof(From) && static_cast(value) != value; } }; // Signed to signed narrowing template struct narrow_impl::value && std::is_signed::value>> { static constexpr bool test(const From& value) { return sizeof(To) < sizeof(From) && static_cast(value) != value; } }; // Unsigned to signed narrowing template struct narrow_impl::value && std::is_signed::value>> { static constexpr bool test(const From& value) { return sizeof(To) <= sizeof(From) && value > (static_cast>(-1) >> 1); } }; // Signed to unsigned narrowing (I) template struct narrow_impl::value && std::is_unsigned::value && sizeof(To) >= sizeof(From)>> { static constexpr bool test(const From& value) { return value < static_cast(0); } }; // Signed to unsigned narrowing (II) template struct narrow_impl::value && std::is_unsigned::value && sizeof(To) < sizeof(From)>> { static constexpr bool test(const From& value) { return static_cast>(value) > static_cast(-1); } }; // Simple type enabled (TODO: allow for To as well) template struct narrow_impl> : narrow_impl, To> { }; template (std::declval()))> inline To narrow(const From& value, const char* msg = nullptr) { // Narrow check if (narrow_impl::test(value)) { // Pack value as formatting argument fmt::raw_narrow_error(msg, fmt::get_type_info>(), fmt_unveil::get(value)); } return static_cast(value); } // Returns u32 size() for container template (std::declval().size()))> inline u32 size32(const CT& container, const char* msg = nullptr) { return narrow(container.size(), msg); } // Returns u32 size for an array template constexpr u32 size32(const T (&)[Size], const char* msg = nullptr) { return static_cast(Size); } template ::value>> constexpr bool test(const T1& value) { return value != 0; } template ::value && std::is_integral::value>> constexpr bool test(const T1& lhs, const T2& rhs) { return (lhs & rhs) != 0; } template ::value && std::is_integral::value>> inline bool test_and_set(T& lhs, const T2& rhs) { const bool result = (lhs & rhs) != 0; lhs |= rhs; return result; } template ::value && std::is_integral::value>> inline bool test_and_reset(T& lhs, const T2& rhs) { const bool result = (lhs & rhs) != 0; lhs &= ~rhs; return result; } template ::value && std::is_integral::value>> inline bool test_and_complement(T& lhs, const T2& rhs) { const bool result = (lhs & rhs) != 0; lhs ^= rhs; return result; } // Simplified hash algorithm for pointers. May be used in std::unordered_(map|set). template struct pointer_hash { std::size_t operator()(T* ptr) const { return reinterpret_cast(ptr) / Align; } }; template struct value_hash { std::size_t operator()(T value) const { return static_cast(value) >> Shift; } }; // Contains value of any POD type with fixed size and alignment. TT<> is the type converter applied. // For example, `simple_t` may be used to remove endianness. template