From a58f1a94e3a960e39e4dd5227f7a26cf9b0ee02d Mon Sep 17 00:00:00 2001 From: elsid Date: Sat, 23 Oct 2021 00:06:10 +0200 Subject: [PATCH] Add helpers for binary serialization To construct serializer from given entities: * Data source/destination - any value that has to be serialized/deserialized, usually already existing type. * Format - functional object to define high level serialization logic to define specific format and data schema. Like order of fields, allocation. * Visitor - functional object to define low level serialization logic to operator on given data part. * BinaryWriter - copies given value into provided buffer. * BinaryReader - copies value into given destination from provided buffer. * SizeAccumulator - calculates required buffer size for given data. --- apps/openmw_test_suite/CMakeLists.txt | 4 + .../serialization/binaryreader.cpp | 67 ++++++++++++++++ .../serialization/binarywriter.cpp | 57 +++++++++++++ .../detournavigator/serialization/format.hpp | 75 +++++++++++++++++ .../serialization/integration.cpp | 56 +++++++++++++ .../serialization/sizeaccumulator.cpp | 43 ++++++++++ .../serialization/binaryreader.hpp | 62 ++++++++++++++ .../serialization/binarywriter.hpp | 62 ++++++++++++++ .../detournavigator/serialization/format.hpp | 80 +++++++++++++++++++ .../serialization/sizeaccumulator.hpp | 41 ++++++++++ 10 files changed, 547 insertions(+) create mode 100644 apps/openmw_test_suite/detournavigator/serialization/binaryreader.cpp create mode 100644 apps/openmw_test_suite/detournavigator/serialization/binarywriter.cpp create mode 100644 apps/openmw_test_suite/detournavigator/serialization/format.hpp create mode 100644 apps/openmw_test_suite/detournavigator/serialization/integration.cpp create mode 100644 apps/openmw_test_suite/detournavigator/serialization/sizeaccumulator.cpp create mode 100644 components/detournavigator/serialization/binaryreader.hpp create mode 100644 components/detournavigator/serialization/binarywriter.hpp create mode 100644 components/detournavigator/serialization/format.hpp create mode 100644 components/detournavigator/serialization/sizeaccumulator.hpp diff --git a/apps/openmw_test_suite/CMakeLists.txt b/apps/openmw_test_suite/CMakeLists.txt index 0390b0a772..a2a35e3aab 100644 --- a/apps/openmw_test_suite/CMakeLists.txt +++ b/apps/openmw_test_suite/CMakeLists.txt @@ -36,6 +36,10 @@ if (GTEST_FOUND AND GMOCK_FOUND) detournavigator/recastmeshobject.cpp detournavigator/navmeshtilescache.cpp detournavigator/tilecachedrecastmeshmanager.cpp + detournavigator/serialization/binaryreader.cpp + detournavigator/serialization/binarywriter.cpp + detournavigator/serialization/sizeaccumulator.cpp + detournavigator/serialization/integration.cpp settings/parser.cpp diff --git a/apps/openmw_test_suite/detournavigator/serialization/binaryreader.cpp b/apps/openmw_test_suite/detournavigator/serialization/binaryreader.cpp new file mode 100644 index 0000000000..d071326cf5 --- /dev/null +++ b/apps/openmw_test_suite/detournavigator/serialization/binaryreader.cpp @@ -0,0 +1,67 @@ +#include "format.hpp" + +#include + +#include +#include + +#include +#include +#include + +namespace +{ + using namespace testing; + using namespace DetourNavigator::Serialization; + using namespace DetourNavigator::SerializationTesting; + + TEST(DetourNavigatorSerializationBinaryReaderTest, shouldReadArithmeticTypeValue) + { + std::uint32_t value = 42; + std::vector data(sizeof(value)); + std::memcpy(data.data(), &value, sizeof(value)); + BinaryReader binaryReader(data.data(), data.data() + data.size()); + std::uint32_t result = 0; + const TestFormat format; + binaryReader(format, result); + EXPECT_EQ(result, 42); + } + + TEST(DetourNavigatorSerializationBinaryReaderTest, shouldReadArithmeticTypeRangeValue) + { + const std::size_t count = 3; + std::vector data(sizeof(std::size_t) + count * sizeof(std::uint32_t)); + std::memcpy(data.data(), &count, sizeof(count)); + const std::uint32_t value1 = 960900021; + std::memcpy(data.data() + sizeof(count), &value1, sizeof(std::uint32_t)); + const std::uint32_t value2 = 1235496234; + std::memcpy(data.data() + sizeof(count) + sizeof(std::uint32_t), &value2, sizeof(std::uint32_t)); + const std::uint32_t value3 = 2342038092; + std::memcpy(data.data() + sizeof(count) + 2 * sizeof(std::uint32_t), &value3, sizeof(std::uint32_t)); + BinaryReader binaryReader(data.data(), data.data() + data.size()); + std::size_t resultCount = 0; + const TestFormat format; + binaryReader(format, resultCount); + std::vector result(resultCount); + binaryReader(format, result.data(), result.size()); + EXPECT_THAT(result, ElementsAre(value1, value2, value3)); + } + + TEST(DetourNavigatorSerializationBinaryReaderTest, forNotEnoughDataForArithmeticTypeShouldThrowException) + { + std::vector data(3); + BinaryReader binaryReader(data.data(), data.data() + data.size()); + std::uint32_t result = 0; + const TestFormat format; + EXPECT_THROW(binaryReader(format, result), std::runtime_error); + } + + TEST(DetourNavigatorSerializationBinaryReaderTest, forNotEnoughDataForArithmeticTypeRangeShouldThrowException) + { + std::vector data(7); + BinaryReader binaryReader(data.data(), data.data() + data.size()); + std::vector values(2); + const TestFormat format; + EXPECT_THROW(binaryReader(format, values.data(), values.size()), std::runtime_error); + } +} diff --git a/apps/openmw_test_suite/detournavigator/serialization/binarywriter.cpp b/apps/openmw_test_suite/detournavigator/serialization/binarywriter.cpp new file mode 100644 index 0000000000..fccc2be3da --- /dev/null +++ b/apps/openmw_test_suite/detournavigator/serialization/binarywriter.cpp @@ -0,0 +1,57 @@ +#include "format.hpp" + +#include + +#include +#include + +#include +#include +#include + +namespace +{ + using namespace testing; + using namespace DetourNavigator::Serialization; + using namespace DetourNavigator::SerializationTesting; + + TEST(DetourNavigatorSerializationBinaryWriterTest, shouldWriteArithmeticTypeValue) + { + std::vector result(4); + BinaryWriter binaryWriter(result.data(), result.data() + result.size()); + const TestFormat format; + binaryWriter(format, std::uint32_t(42)); + EXPECT_THAT(result, ElementsAre(std::byte(42), std::byte(0), std::byte(0), std::byte(0))); + } + + TEST(DetourNavigatorSerializationBinaryWriterTest, shouldWriteArithmeticTypeRangeValue) + { + std::vector result(8); + BinaryWriter binaryWriter(result.data(), result.data() + result.size()); + std::vector values({42, 13}); + const TestFormat format; + binaryWriter(format, values.data(), values.size()); + constexpr std::array expected { + std::byte(42), std::byte(0), std::byte(0), std::byte(0), + std::byte(13), std::byte(0), std::byte(0), std::byte(0), + }; + EXPECT_THAT(result, ElementsAreArray(expected)); + } + + TEST(DetourNavigatorSerializationBinaryWriterTest, forNotEnoughSpaceForArithmeticTypeShouldThrowException) + { + std::vector result(3); + BinaryWriter binaryWriter(result.data(), result.data() + result.size()); + const TestFormat format; + EXPECT_THROW(binaryWriter(format, std::uint32_t(42)), std::runtime_error); + } + + TEST(DetourNavigatorSerializationBinaryWriterTest, forNotEnoughSpaceForArithmeticTypeRangeShouldThrowException) + { + std::vector result(7); + BinaryWriter binaryWriter(result.data(), result.data() + result.size()); + std::vector values({42, 13}); + const TestFormat format; + EXPECT_THROW(binaryWriter(format, values.data(), values.size()), std::runtime_error); + } +} diff --git a/apps/openmw_test_suite/detournavigator/serialization/format.hpp b/apps/openmw_test_suite/detournavigator/serialization/format.hpp new file mode 100644 index 0000000000..7c5e26a0be --- /dev/null +++ b/apps/openmw_test_suite/detournavigator/serialization/format.hpp @@ -0,0 +1,75 @@ +#ifndef OPENMW_TEST_SUITE_DETOURNAVIGATOR_SERIALIZATION_FORMAT_H +#define OPENMW_TEST_SUITE_DETOURNAVIGATOR_SERIALIZATION_FORMAT_H + +#include + +#include +#include + +namespace DetourNavigator::SerializationTesting +{ + struct Pod + { + int mInt = 42; + double mDouble = 3.14; + + friend bool operator==(const Pod& l, const Pod& r) + { + const auto tuple = [] (const Pod& v) { return std::tuple(v.mInt, v.mDouble); }; + return tuple(l) == tuple(r); + } + }; + + enum Enum + { + A, + B, + C, + }; + + struct Composite + { + short mFloatArray[3] = {0}; + std::vector mIntVector; + std::vector mEnumVector; + std::vector mPodVector; + std::size_t mPodDataSize = 0; + std::vector mPodBuffer; + std::size_t mCharDataSize = 0; + std::vector mCharBuffer; + }; + + template + struct TestFormat : Serialization::Format> + { + using Serialization::Format>::operator(); + + template + auto operator()(Visitor&& visitor, T& value) const + -> std::enable_if_t, Pod>> + { + visitor(*this, value.mInt); + visitor(*this, value.mDouble); + } + + template + auto operator()(Visitor&& visitor, T& value) const + -> std::enable_if_t, Composite>> + { + visitor(*this, value.mFloatArray); + visitor(*this, value.mIntVector); + visitor(*this, value.mEnumVector); + visitor(*this, value.mPodVector); + visitor(*this, value.mPodDataSize); + if constexpr (mode == Serialization::Mode::Read) + value.mPodBuffer.resize(value.mPodDataSize); + visitor(*this, value.mPodBuffer.data(), value.mPodDataSize); + visitor(*this, value.mCharDataSize); + if constexpr (mode == Serialization::Mode::Read) + value.mCharBuffer.resize(value.mCharDataSize); + visitor(*this, value.mCharBuffer.data(), value.mCharDataSize); + } + }; +} + +#endif diff --git a/apps/openmw_test_suite/detournavigator/serialization/integration.cpp b/apps/openmw_test_suite/detournavigator/serialization/integration.cpp new file mode 100644 index 0000000000..e7e8eacc20 --- /dev/null +++ b/apps/openmw_test_suite/detournavigator/serialization/integration.cpp @@ -0,0 +1,56 @@ +#include "format.hpp" + +#include +#include +#include + +#include +#include + +#include + +namespace +{ + using namespace testing; + using namespace DetourNavigator::Serialization; + using namespace DetourNavigator::SerializationTesting; + + struct DetourNavigatorSerializationIntegrationTest : Test + { + Composite mComposite; + + DetourNavigatorSerializationIntegrationTest() + { + mComposite.mIntVector = {4, 5, 6}; + mComposite.mEnumVector = {Enum::A, Enum::B, Enum::C}; + mComposite.mPodVector = {Pod {4, 23.87}, Pod {5, -31.76}, Pod {6, 65.12}}; + mComposite.mPodBuffer = {Pod {7, 456.123}, Pod {8, -628.346}}; + mComposite.mPodDataSize = mComposite.mPodBuffer.size(); + std::string charData = "serialization"; + mComposite.mCharBuffer = {charData.begin(), charData.end()}; + mComposite.mCharDataSize = charData.size(); + } + }; + + TEST_F(DetourNavigatorSerializationIntegrationTest, sizeAccumulatorShouldSupportCustomSerializer) + { + SizeAccumulator sizeAccumulator; + TestFormat{}(sizeAccumulator, mComposite); + EXPECT_EQ(sizeAccumulator.value(), 143); + } + + TEST_F(DetourNavigatorSerializationIntegrationTest, binaryReaderShouldDeserializeDataWrittenByBinaryWriter) + { + std::vector data(143); + TestFormat{}(BinaryWriter(data.data(), data.data() + data.size()), mComposite); + Composite result; + TestFormat{}(BinaryReader(data.data(), data.data() + data.size()), result); + EXPECT_EQ(result.mIntVector, mComposite.mIntVector); + EXPECT_EQ(result.mEnumVector, mComposite.mEnumVector); + EXPECT_EQ(result.mPodVector, mComposite.mPodVector); + EXPECT_EQ(result.mPodDataSize, mComposite.mPodDataSize); + EXPECT_EQ(result.mPodBuffer, mComposite.mPodBuffer); + EXPECT_EQ(result.mCharDataSize, mComposite.mCharDataSize); + EXPECT_EQ(result.mCharBuffer, mComposite.mCharBuffer); + } +} diff --git a/apps/openmw_test_suite/detournavigator/serialization/sizeaccumulator.cpp b/apps/openmw_test_suite/detournavigator/serialization/sizeaccumulator.cpp new file mode 100644 index 0000000000..39b7ea8646 --- /dev/null +++ b/apps/openmw_test_suite/detournavigator/serialization/sizeaccumulator.cpp @@ -0,0 +1,43 @@ +#include "format.hpp" + +#include + +#include + +#include +#include +#include +#include + +namespace +{ + using namespace testing; + using namespace DetourNavigator::Serialization; + using namespace DetourNavigator::SerializationTesting; + + TEST(DetourNavigatorSerializationSizeAccumulatorTest, shouldProvideSizeForArithmeticType) + { + SizeAccumulator sizeAccumulator; + constexpr std::monostate format; + sizeAccumulator(format, std::uint32_t()); + EXPECT_EQ(sizeAccumulator.value(), 4); + } + + TEST(DetourNavigatorSerializationSizeAccumulatorTest, shouldProvideSizeForArithmeticTypeRange) + { + SizeAccumulator sizeAccumulator; + const std::uint64_t* const data = nullptr; + const std::size_t count = 3; + const std::monostate format; + sizeAccumulator(format, data, count); + EXPECT_EQ(sizeAccumulator.value(), 24); + } + + TEST(DetourNavigatorSerializationSizeAccumulatorTest, shouldSupportCustomSerializer) + { + SizeAccumulator sizeAccumulator; + const TestFormat format; + sizeAccumulator(format, Pod {}); + EXPECT_EQ(sizeAccumulator.value(), 12); + } +} diff --git a/components/detournavigator/serialization/binaryreader.hpp b/components/detournavigator/serialization/binaryreader.hpp new file mode 100644 index 0000000000..0d75c3ac99 --- /dev/null +++ b/components/detournavigator/serialization/binaryreader.hpp @@ -0,0 +1,62 @@ +#ifndef OPENMW_COMPONENTS_DETOURNAVIGATOR_SERIALIZATION_BINARYREADER_H +#define OPENMW_COMPONENTS_DETOURNAVIGATOR_SERIALIZATION_BINARYREADER_H + +#include +#include +#include +#include +#include + +namespace DetourNavigator::Serialization +{ + class BinaryReader + { + public: + explicit BinaryReader(const std::byte* pos, const std::byte* end) + : mPos(pos), mEnd(end) + { + assert(mPos <= mEnd); + } + + BinaryReader(const BinaryReader&) = delete; + + template + void operator()(Format&& format, T& value) + { + if constexpr (std::is_arithmetic_v) + { + if (mEnd - mPos < static_cast(sizeof(value))) + throw std::runtime_error("Not enough data"); + std::memcpy(&value, mPos, sizeof(value)); + mPos += sizeof(value); + } + else + { + format(*this, value); + } + } + + template + auto operator()(Format&& format, T* data, std::size_t count) + { + if constexpr (std::is_arithmetic_v) + { + if (mEnd - mPos < static_cast(count * sizeof(T))) + throw std::runtime_error("Not enough data"); + const std::size_t size = sizeof(T) * count; + std::memcpy(data, mPos, size); + mPos += size; + } + else + { + format(*this, data, count); + } + } + + private: + const std::byte* mPos; + const std::byte* const mEnd; + }; +} + +#endif diff --git a/components/detournavigator/serialization/binarywriter.hpp b/components/detournavigator/serialization/binarywriter.hpp new file mode 100644 index 0000000000..5e710d85d5 --- /dev/null +++ b/components/detournavigator/serialization/binarywriter.hpp @@ -0,0 +1,62 @@ +#ifndef OPENMW_COMPONENTS_DETOURNAVIGATOR_SERIALIZATION_BINARYWRITER_H +#define OPENMW_COMPONENTS_DETOURNAVIGATOR_SERIALIZATION_BINARYWRITER_H + +#include +#include +#include +#include +#include + +namespace DetourNavigator::Serialization +{ + struct BinaryWriter + { + public: + explicit BinaryWriter(std::byte* dest, const std::byte* end) + : mDest(dest), mEnd(end) + { + assert(mDest <= mEnd); + } + + BinaryWriter(const BinaryWriter&) = delete; + + template + void operator()(Format&& format, const T& value) + { + if constexpr (std::is_arithmetic_v) + { + if (mEnd - mDest < static_cast(sizeof(value))) + throw std::runtime_error("Not enough space"); + std::memcpy(mDest, &value, sizeof(value)); + mDest += sizeof(value); + } + else + { + format(*this, value); + } + } + + template + auto operator()(Format&& format, const T* data, std::size_t count) + { + if constexpr (std::is_arithmetic_v) + { + const std::size_t size = sizeof(T) * count; + if (mEnd - mDest < static_cast(size)) + throw std::runtime_error("Not enough space"); + std::memcpy(mDest, data, size); + mDest += size; + } + else + { + format(*this, data, count); + } + } + + private: + std::byte* mDest; + const std::byte* const mEnd; + }; +} + +#endif diff --git a/components/detournavigator/serialization/format.hpp b/components/detournavigator/serialization/format.hpp new file mode 100644 index 0000000000..d07ab9da6f --- /dev/null +++ b/components/detournavigator/serialization/format.hpp @@ -0,0 +1,80 @@ +#ifndef OPENMW_COMPONENTS_DETOURNAVIGATOR_SERIALIZATION_FORMAT_H +#define OPENMW_COMPONENTS_DETOURNAVIGATOR_SERIALIZATION_FORMAT_H + +#include +#include +#include +#include +#include +#include + +namespace DetourNavigator::Serialization +{ + enum class Mode + { + Read, + Write, + }; + + template + struct IsContiguousContainer : std::false_type {}; + + template + struct IsContiguousContainer> : std::true_type {}; + + template + constexpr bool isContiguousContainer = IsContiguousContainer>::value; + + template + struct Format + { + template + void operator()(Visitor&& visitor, T* data, std::size_t size) const + { + if constexpr (std::is_arithmetic_v) + { + visitor(self(), data, size); + } + else if constexpr (std::is_enum_v) + { + if constexpr (mode == Mode::Write) + visitor(self(), reinterpret_cast*>(data), size); + else + { + static_assert(mode == Mode::Read); + visitor(self(), reinterpret_cast*>(data), size); + } + } + else + { + std::for_each(data, data + size, [&] (auto& v) { visitor(self(), v); }); + } + } + + template + void operator()(Visitor&& visitor, T(& data)[size]) const + { + self()(std::forward(visitor), data, size); + } + + template + auto operator()(Visitor&& visitor, T&& value) const + -> std::enable_if_t> + { + if constexpr (mode == Mode::Write) + visitor(self(), value.size()); + else + { + static_assert(mode == Mode::Read); + std::size_t size = 0; + visitor(self(), size); + value.resize(size); + } + self()(std::forward(visitor), value.data(), value.size()); + } + + const Derived& self() const { return static_cast(*this); } + }; +} + +#endif diff --git a/components/detournavigator/serialization/sizeaccumulator.hpp b/components/detournavigator/serialization/sizeaccumulator.hpp new file mode 100644 index 0000000000..28bdb5c1cb --- /dev/null +++ b/components/detournavigator/serialization/sizeaccumulator.hpp @@ -0,0 +1,41 @@ +#ifndef OPENMW_COMPONENTS_DETOURNAVIGATOR_SERIALIZATION_SIZEACCUMULATOR_H +#define OPENMW_COMPONENTS_DETOURNAVIGATOR_SERIALIZATION_SIZEACCUMULATOR_H + +#include +#include + +namespace DetourNavigator::Serialization +{ + class SizeAccumulator + { + public: + SizeAccumulator() = default; + + SizeAccumulator(const SizeAccumulator&) = delete; + + std::size_t value() const { return mValue; } + + template + void operator()(Format&& format, const T& value) + { + if constexpr (std::is_arithmetic_v) + mValue += sizeof(T); + else + format(*this, value); + } + + template + auto operator()(Format&& format, const T* data, std::size_t count) + { + if constexpr (std::is_arithmetic_v) + mValue += count * sizeof(T); + else + format(*this, data, count); + } + + private: + std::size_t mValue = 0; + }; +} + +#endif