#pragma once #ifdef _MSC_VER #include #else #include #endif #include #include #include #include #include #include #include #include #include #include // Assume little-endian #define IS_LE_MACHINE 1 #define IS_BE_MACHINE 0 #ifndef __has_builtin #define __has_builtin(x) 0 #endif #ifdef _MSC_VER #define ASSUME(...) __assume(__VA_ARGS__) // MSVC __assume ignores side-effects #define LIKELY #define UNLIKELY #define SAFE_BUFFERS __declspec(safebuffers) #define NEVER_INLINE __declspec(noinline) #define FORCE_INLINE __forceinline #define RESTRICT __restrict #else // not _MSC_VER #ifdef __clang__ #if defined(__has_builtin) && __has_builtin(__builtin_assume) #pragma clang diagnostic ignored "-Wassume" // ignore the clang "side-effects ignored" warning #define ASSUME(...) __builtin_assume(!!(__VA_ARGS__)) // __builtin_assume (supported by modern clang) ignores side-effects #endif #endif #ifndef ASSUME // gcc and old clang #define ASSUME(...) do { if (!(__VA_ARGS__)) __builtin_unreachable(); } while (0) // note: the compiler will generate code to evaluate "cond" if the expression is opaque #endif #define LIKELY(...) __builtin_expect(!!(__VA_ARGS__), 1) #define UNLIKELY(...) __builtin_expect(!!(__VA_ARGS__), 0) #define SAFE_BUFFERS #define NEVER_INLINE __attribute__((noinline)) #define FORCE_INLINE __attribute__((always_inline)) inline #define RESTRICT __restrict__ #endif // _MSC_VER #define CHECK_SIZE(type, size) static_assert(sizeof(type) == size, "Invalid " #type " type size") #define CHECK_ALIGN(type, align) static_assert(alignof(type) == align, "Invalid " #type " type alignment") #define CHECK_MAX_SIZE(type, size) static_assert(sizeof(type) <= size, #type " type size is too big") #define CHECK_SIZE_ALIGN(type, size, align) CHECK_SIZE(type, size); CHECK_ALIGN(type, align) // Variant pattern matching helper #define MATCH(arg, ...) constexpr(std::is_same_v, __VA_ARGS__>) #define CONCATENATE_DETAIL(x, y) x ## y #define CONCATENATE(x, y) CONCATENATE_DETAIL(x, y) #define STRINGIZE_DETAIL(x) #x "" #define STRINGIZE(x) STRINGIZE_DETAIL(x) #define HERE "\n(in file " __FILE__ ":" STRINGIZE(__LINE__) ")" #define DECLARE(...) decltype(__VA_ARGS__) __VA_ARGS__ #define STR_CASE(...) case __VA_ARGS__: return #__VA_ARGS__ #define ASSERT(...) do { if(!(__VA_ARGS__)) fmt::raw_error("Assertion failed: " STRINGIZE(__VA_ARGS__) HERE); } while(0) #if defined(_DEBUG) || defined(_AUDIT) #define AUDIT(...) ASSERT(__VA_ARGS__) #else #define AUDIT(...) ((void)0) #endif #if defined(__cpp_lib_bit_cast) && (__cpp_lib_bit_cast >= 201806L) #include #else namespace std { template > constexpr To bit_cast(const From& from) noexcept { static_assert(sizeof(To) == sizeof(From), "std::bit_cast<>: incompatible type size"); To result; std::memcpy(&result, &from, sizeof(From)); return result; } } #endif using schar = signed char; using uchar = unsigned char; using ushort = unsigned short; using uint = unsigned int; using ulong = unsigned long; using ullong = unsigned long long; using llong = long long; #if __APPLE__ using uptr = std::uint64_t; #else using uptr = std::uintptr_t; #endif using u8 = std::uint8_t; using u16 = std::uint16_t; using u32 = std::uint32_t; using u64 = std::uint64_t; using s8 = std::int8_t; using s16 = std::int16_t; using s32 = std::int32_t; using s64 = std::int64_t; using steady_clock = std::conditional< std::chrono::high_resolution_clock::is_steady, std::chrono::high_resolution_clock, std::chrono::steady_clock>::type; // Get unsigned integral type from type size template struct get_int_impl { }; template<> struct get_int_impl { using type = u8; }; template<> struct get_int_impl { using type = u16; }; template<> struct get_int_impl { using type = u32; }; template<> struct get_int_impl { using type = u64; }; template using get_int_t = typename get_int_impl::type; namespace gsl { using std::byte; } // Formatting helper, type-specific preprocessing for improving safety and functionality template struct fmt_unveil; template using fmt_unveil_t = typename fmt_unveil::type; struct fmt_type_info; namespace fmt { template const fmt_type_info* get_type_info(); } template struct se_storage; template class se_t; template class atomic_t; // Extract T::simple_type if available, remove cv qualifiers template struct simple_type_helper { using type = typename std::remove_cv::type; }; template struct simple_type_helper> { using type = typename T::simple_type; }; template using simple_t = typename simple_type_helper::type; // Bool type equivalent class b8 { u8 m_value; public: b8() = default; constexpr b8(bool value) : m_value(value) { } constexpr operator bool() const { return m_value != 0; } }; #ifndef _MSC_VER using u128 = __uint128_t; using s128 = __int128_t; #else // Unsigned 128-bit integer implementation (TODO) struct alignas(16) u128 { u64 lo, hi; u128() = default; constexpr u128(u64 l) : lo(l) , hi(0) { } friend u128 operator+(const u128& l, const u128& r) { u128 value; _addcarry_u64(_addcarry_u64(0, r.lo, l.lo, &value.lo), r.hi, l.hi, &value.hi); return value; } friend u128 operator+(const u128& l, u64 r) { u128 value; _addcarry_u64(_addcarry_u64(0, r, l.lo, &value.lo), l.hi, 0, &value.hi); return value; } friend u128 operator+(u64 l, const u128& r) { u128 value; _addcarry_u64(_addcarry_u64(0, r.lo, l, &value.lo), 0, r.hi, &value.hi); return value; } friend u128 operator-(const u128& l, const u128& r) { u128 value; _subborrow_u64(_subborrow_u64(0, r.lo, l.lo, &value.lo), r.hi, l.hi, &value.hi); return value; } friend u128 operator-(const u128& l, u64 r) { u128 value; _subborrow_u64(_subborrow_u64(0, r, l.lo, &value.lo), 0, l.hi, &value.hi); return value; } friend u128 operator-(u64 l, const u128& r) { u128 value; _subborrow_u64(_subborrow_u64(0, r.lo, l, &value.lo), r.hi, 0, &value.hi); return value; } u128 operator+() const { return *this; } u128 operator-() const { u128 value; _subborrow_u64(_subborrow_u64(0, lo, 0, &value.lo), hi, 0, &value.hi); return value; } u128& operator++() { _addcarry_u64(_addcarry_u64(0, 1, lo, &lo), 0, hi, &hi); return *this; } u128 operator++(int) { u128 value = *this; _addcarry_u64(_addcarry_u64(0, 1, lo, &lo), 0, hi, &hi); return value; } u128& operator--() { _subborrow_u64(_subborrow_u64(0, 1, lo, &lo), 0, hi, &hi); return *this; } u128 operator--(int) { u128 value = *this; _subborrow_u64(_subborrow_u64(0, 1, lo, &lo), 0, hi, &hi); return value; } u128 operator~() const { u128 value; value.lo = ~lo; value.hi = ~hi; return value; } friend u128 operator&(const u128& l, const u128& r) { u128 value; value.lo = l.lo & r.lo; value.hi = l.hi & r.hi; return value; } friend u128 operator|(const u128& l, const u128& r) { u128 value; value.lo = l.lo | r.lo; value.hi = l.hi | r.hi; return value; } friend u128 operator^(const u128& l, const u128& r) { u128 value; value.lo = l.lo ^ r.lo; value.hi = l.hi ^ r.hi; return value; } u128& operator+=(const u128& r) { _addcarry_u64(_addcarry_u64(0, r.lo, lo, &lo), r.hi, hi, &hi); return *this; } u128& operator+=(uint64_t r) { _addcarry_u64(_addcarry_u64(0, r, lo, &lo), 0, hi, &hi); return *this; } u128& operator&=(const u128& r) { lo &= r.lo; hi &= r.hi; return *this; } u128& operator|=(const u128& r) { lo |= r.lo; hi |= r.hi; return *this; } u128& operator^=(const u128& r) { lo ^= r.lo; hi ^= r.hi; return *this; } }; // Signed 128-bit integer implementation (TODO) struct alignas(16) s128 { u64 lo; s64 hi; s128() = default; constexpr s128(s64 l) : hi(l >> 63) , lo(l) { } constexpr s128(u64 l) : hi(0) , lo(l) { } }; #endif CHECK_SIZE_ALIGN(u128, 16, 16); CHECK_SIZE_ALIGN(s128, 16, 16); using f32 = float; using f64 = double; union alignas(2) f16 { u16 _u16; u8 _u8[2]; explicit f16(u16 raw) { _u16 = raw; } explicit operator f32() const { // See http://stackoverflow.com/a/26779139 // The conversion doesn't handle NaN/Inf u32 raw = ((_u16 & 0x8000) << 16) | // Sign (just moved) (((_u16 & 0x7c00) + 0x1C000) << 13) | // Exponent ( exp - 15 + 127) ((_u16 & 0x03FF) << 13); // Mantissa return std::bit_cast(raw); } }; CHECK_SIZE_ALIGN(f16, 2, 2); template ::value>> constexpr T align(const T& value, ullong align) { return static_cast((value + (align - 1)) & ~(align - 1)); } template inline u32 offset32(T T2::*const mptr) { #ifdef _MSC_VER return std::bit_cast(mptr); #elif __GNUG__ return std::bit_cast(mptr); #else static_assert(sizeof(mptr) == 0, "Unsupported pointer-to-member size"); #endif } template struct offset32_array { static_assert(std::is_array::value, "Invalid pointer-to-member type (array expected)"); template static inline u32 index32(const Arg& arg) { return u32{sizeof(std::remove_extent_t)} * static_cast(arg); } }; template struct offset32_array> { template static inline u32 index32(const Arg& arg) { return u32{sizeof(T)} * static_cast(arg); } }; template struct offset32_detail; template inline u32 offset32(T T2::*const mptr, const Arg& arg, const Args&... args) { return offset32_detail::offset32(mptr, arg, args...); } template struct offset32_detail { template static inline u32 offset32(T T2::*const mptr, const Arg& arg, const Args&... args) { return ::offset32(mptr, args...) + offset32_array::index32(arg); } }; template struct offset32_detail { template static inline u32 offset32(T T2::*const mptr, T3 T4::*const mptr2, const Args&... args) { return ::offset32(mptr) + ::offset32(mptr2, args...); } }; // Helper function, used by ""_u16, ""_u32, ""_u64 constexpr u8 to_u8(char c) { return static_cast(c); } // Convert 2-byte string to u16 value like reinterpret_cast does constexpr u16 operator""_u16(const char* s, std::size_t length) { return length != 2 ? throw s : #if IS_LE_MACHINE == 1 to_u8(s[1]) << 8 | to_u8(s[0]); #endif } // Convert 4-byte string to u32 value like reinterpret_cast does constexpr u32 operator""_u32(const char* s, std::size_t length) { return length != 4 ? throw s : #if IS_LE_MACHINE == 1 to_u8(s[3]) << 24 | to_u8(s[2]) << 16 | to_u8(s[1]) << 8 | to_u8(s[0]); #endif } // Convert 8-byte string to u64 value like reinterpret_cast does constexpr u64 operator""_u64(const char* s, std::size_t length) { return length != 8 ? throw s : #if IS_LE_MACHINE == 1 static_cast(to_u8(s[7]) << 24 | to_u8(s[6]) << 16 | to_u8(s[5]) << 8 | to_u8(s[4])) << 32 | to_u8(s[3]) << 24 | to_u8(s[2]) << 16 | to_u8(s[1]) << 8 | to_u8(s[0]); #endif } namespace fmt { [[noreturn]] void raw_error(const char* msg); [[noreturn]] void raw_verify_error(const char* msg, const fmt_type_info* sup, u64 arg); [[noreturn]] void raw_narrow_error(const char* msg, const fmt_type_info* sup, u64 arg); } struct verify_func { template bool operator()(T&& value) const { if (std::forward(value)) { return true; } return false; } }; template struct verify_impl { const char* cause; template auto operator,(T&& value) const { // Verification (can be safely disabled) if (!verify_func()(std::forward(value))) { fmt::raw_verify_error(cause, nullptr, N); } return verify_impl{cause}; } }; // Verification helper, checks several conditions delimited with comma operator inline auto verify(const char* cause) { return verify_impl<0>{cause}; } // Verification helper (returns value or lvalue reference, may require to use verify_move instead) template inline T verify(const char* cause, T&& value, F&& pred = F()) { if (!pred(std::forward(value))) { using unref = std::remove_const_t>; fmt::raw_verify_error(cause, fmt::get_type_info>(), fmt_unveil::get(value)); } return std::forward(value); } // Verification helper (must be used in return expression or in place of std::move) template inline std::remove_reference_t&& verify_move(const char* cause, T&& value, F&& pred = F()) { if (!pred(std::forward(value))) { using unref = std::remove_const_t>; fmt::raw_verify_error(cause, fmt::get_type_info>(), fmt_unveil::get(value)); } return std::move(value); } // narrow() function details template struct narrow_impl { // Temporarily (diagnostic) static_assert(std::is_void::value, "narrow_impl<> specialization not found"); // Returns true if value cannot be represented in type To static constexpr bool test(const From& value) { // Unspecialized cases (including cast to void) always considered narrowing return true; } }; // Unsigned to unsigned narrowing template struct narrow_impl::value && std::is_unsigned::value>> { static constexpr bool test(const From& value) { return sizeof(To) < sizeof(From) && static_cast(value) != value; } }; // Signed to signed narrowing template struct narrow_impl::value && std::is_signed::value>> { static constexpr bool test(const From& value) { return sizeof(To) < sizeof(From) && static_cast(value) != value; } }; // Unsigned to signed narrowing template struct narrow_impl::value && std::is_signed::value>> { static constexpr bool test(const From& value) { return sizeof(To) <= sizeof(From) && value > (static_cast>(-1) >> 1); } }; // Signed to unsigned narrowing (I) template struct narrow_impl::value && std::is_unsigned::value && sizeof(To) >= sizeof(From)>> { static constexpr bool test(const From& value) { return value < static_cast(0); } }; // Signed to unsigned narrowing (II) template struct narrow_impl::value && std::is_unsigned::value && sizeof(To) < sizeof(From)>> { static constexpr bool test(const From& value) { return static_cast>(value) > static_cast(-1); } }; // Simple type enabled (TODO: allow for To as well) template struct narrow_impl> : narrow_impl, To> { }; template (std::declval()))> inline To narrow(const From& value, const char* msg = nullptr) { // Narrow check if (narrow_impl::test(value)) { // Pack value as formatting argument fmt::raw_narrow_error(msg, fmt::get_type_info>(), fmt_unveil::get(value)); } return static_cast(value); } // Returns u32 size() for container template (std::declval().size()))> inline u32 size32(const CT& container, const char* msg = nullptr) { return narrow(container.size(), msg); } // Returns u32 size for an array template constexpr u32 size32(const T (&)[Size], const char* msg = nullptr) { return static_cast(Size); } // Simplified hash algorithm for pointers. May be used in std::unordered_(map|set). template struct pointer_hash { std::size_t operator()(T* ptr) const { return reinterpret_cast(ptr) / Align; } }; template struct value_hash { std::size_t operator()(T value) const { return static_cast(value) >> Shift; } }; // Contains value of any POD type with fixed size and alignment. TT<> is the type converter applied. // For example, `simple_t` may be used to remove endianness. template