#include "serialization.hpp" #include #include #include #include #include #include #include #include "luastate.hpp" #include "utilpackage.hpp" namespace LuaUtil { constexpr unsigned char FORMAT_VERSION = 0; enum class SerializedType : char { NUMBER = 0x0, LONG_STRING = 0x1, BOOLEAN = 0x2, TABLE_START = 0x3, TABLE_END = 0x4, VEC2 = 0x10, VEC3 = 0x11, TRANSFORM_M = 0x12, TRANSFORM_Q = 0x13, VEC4 = 0x14, COLOR = 0x15, // All values should be lesser than 0x20 (SHORT_STRING_FLAG). }; constexpr unsigned char SHORT_STRING_FLAG = 0x20; // 0b001SSSSS. SSSSS = string length constexpr unsigned char CUSTOM_FULL_FLAG = 0x40; // 0b01TTTTTT + 32bit dataSize constexpr unsigned char CUSTOM_COMPACT_FLAG = 0x80; // 0b1SSSSTTT. SSSS = dataSize, TTT = (typeName size - 1) static void appendType(BinaryData& out, SerializedType type) { out.push_back(static_cast(type)); } template static void appendValue(BinaryData& out, T v) { v = Misc::toLittleEndian(v); out.append(reinterpret_cast(&v), sizeof(v)); } template static T getValue(std::string_view& binaryData) { if (binaryData.size() < sizeof(T)) throw std::runtime_error("Unexpected end of serialized data."); T v; std::memcpy(&v, binaryData.data(), sizeof(T)); binaryData = binaryData.substr(sizeof(T)); return Misc::fromLittleEndian(v); } static void appendString(BinaryData& out, std::string_view str) { if (str.size() < 32) out.push_back(SHORT_STRING_FLAG | char(str.size())); else { appendType(out, SerializedType::LONG_STRING); appendValue(out, str.size()); } out.append(str.data(), str.size()); } static void appendData(BinaryData& out, const void* data, size_t dataSize) { out.append(reinterpret_cast(data), dataSize); } void UserdataSerializer::append(BinaryData& out, std::string_view typeName, const void* data, size_t dataSize) { assert(!typeName.empty() && typeName.size() <= 64); if (typeName.size() <= 8 && dataSize < 16) { // Compact form: 0b1SSSSTTT. SSSS = dataSize, TTT = (typeName size - 1). unsigned char t = CUSTOM_COMPACT_FLAG | (dataSize << 3) | (typeName.size() - 1); out.push_back(t); } else { // Full form: 0b01TTTTTT + 32bit dataSize. unsigned char t = CUSTOM_FULL_FLAG | (typeName.size() - 1); out.push_back(t); appendValue(out, dataSize); } out.append(typeName.data(), typeName.size()); appendData(out, data, dataSize); } void UserdataSerializer::appendRefNum(BinaryData& out, ESM::RefNum refnum) { static_assert(sizeof(ESM::RefNum) == 8); refnum.mIndex = Misc::toLittleEndian(refnum.mIndex); refnum.mContentFile = Misc::toLittleEndian(refnum.mContentFile); append(out, sRefNumTypeName, &refnum, sizeof(ESM::RefNum)); } bool BasicSerializer::serialize(BinaryData& out, const sol::userdata& data) const { appendRefNum(out, cast(data)); return true; } bool BasicSerializer::deserialize(std::string_view typeName, std::string_view binaryData, lua_State* lua) const { if (typeName != sRefNumTypeName) return false; ESM::RefNum refnum = loadRefNum(binaryData); if (mAdjustContentFilesIndexFn) refnum.mContentFile = mAdjustContentFilesIndexFn(refnum.mContentFile); sol::stack::push(lua, refnum); return true; } ESM::RefNum UserdataSerializer::loadRefNum(std::string_view data) { if (data.size() != sizeof(ESM::RefNum)) throw std::runtime_error("Incorrect serialization format. Size of RefNum doesn't match."); ESM::RefNum refnum; std::memcpy(&refnum, data.data(), sizeof(ESM::RefNum)); refnum.mIndex = Misc::fromLittleEndian(refnum.mIndex); refnum.mContentFile = Misc::fromLittleEndian(refnum.mContentFile); return refnum; } static void serializeUserdata( BinaryData& out, const sol::userdata& data, const UserdataSerializer* customSerializer) { if (data.is()) { appendType(out, SerializedType::VEC2); osg::Vec2f v = data.as(); appendValue(out, v.x()); appendValue(out, v.y()); return; } if (data.is()) { appendType(out, SerializedType::VEC3); osg::Vec3f v = data.as(); appendValue(out, v.x()); appendValue(out, v.y()); appendValue(out, v.z()); return; } if (data.is()) { appendType(out, SerializedType::TRANSFORM_M); osg::Matrixf matrix = data.as().mM; for (size_t i = 0; i < 4; i++) for (size_t j = 0; j < 4; j++) appendValue(out, matrix(i, j)); return; } if (data.is()) { appendType(out, SerializedType::TRANSFORM_Q); osg::Quat quat = data.as().mQ; for (size_t i = 0; i < 4; i++) appendValue(out, quat[i]); return; } if (data.is()) { appendType(out, SerializedType::VEC4); osg::Vec4f v = data.as(); appendValue(out, v.x()); appendValue(out, v.y()); appendValue(out, v.z()); appendValue(out, v.w()); return; } if (data.is()) { appendType(out, SerializedType::COLOR); Misc::Color v = data.as(); appendValue(out, v.r()); appendValue(out, v.g()); appendValue(out, v.b()); appendValue(out, v.a()); return; } if (customSerializer && customSerializer->serialize(out, data)) return; else throw std::runtime_error("Value is not serializable."); } static void serialize( BinaryData& out, const sol::object& obj, const UserdataSerializer* customSerializer, int recursionCounter) { if (obj.get_type() == sol::type::lightuserdata) throw std::runtime_error("Light userdata is not allowed to be serialized."); if (obj.is()) throw std::runtime_error("Functions are not allowed to be serialized."); else if (obj.is()) serializeUserdata(out, obj, customSerializer); else if (obj.is()) { if (recursionCounter >= 32) throw std::runtime_error( "Can not serialize more than 32 nested tables. Likely the table contains itself."); sol::table table = obj; appendType(out, SerializedType::TABLE_START); for (auto& [key, value] : table) { serialize(out, key, customSerializer, recursionCounter + 1); serialize(out, value, customSerializer, recursionCounter + 1); } appendType(out, SerializedType::TABLE_END); } else if (obj.is()) { appendType(out, SerializedType::NUMBER); appendValue(out, obj.as()); } else if (obj.is()) appendString(out, obj.as()); else if (obj.is()) { char v = obj.as() ? 1 : 0; appendType(out, SerializedType::BOOLEAN); out.push_back(v); } else throw std::runtime_error("Unknown Lua type."); } static void deserializeImpl( lua_State* lua, std::string_view& binaryData, const UserdataSerializer* customSerializer, bool readOnly) { if (binaryData.empty()) throw std::runtime_error("Unexpected end of serialized data."); unsigned char type = binaryData[0]; binaryData = binaryData.substr(1); if (type & (CUSTOM_COMPACT_FLAG | CUSTOM_FULL_FLAG)) { size_t typeNameSize, dataSize; if (type & CUSTOM_COMPACT_FLAG) { // Compact form: 0b1SSSSTTT. SSSS = dataSize, TTT = (typeName size - 1). typeNameSize = (type & 7) + 1; dataSize = (type >> 3) & 15; } else { // Full form: 0b01TTTTTT + 32bit dataSize. typeNameSize = (type & 63) + 1; dataSize = getValue(binaryData); } std::string_view typeName = binaryData.substr(0, typeNameSize); std::string_view data = binaryData.substr(typeNameSize, dataSize); binaryData = binaryData.substr(typeNameSize + dataSize); if (!customSerializer || !customSerializer->deserialize(typeName, data, lua)) throw std::runtime_error("Unknown type in serialized data: " + std::string(typeName)); return; } if (type & SHORT_STRING_FLAG) { size_t size = type & 0x1f; sol::stack::push(lua, binaryData.substr(0, size)); binaryData = binaryData.substr(size); return; } switch (static_cast(type)) { case SerializedType::NUMBER: sol::stack::push(lua, getValue(binaryData)); return; case SerializedType::BOOLEAN: sol::stack::push(lua, getValue(binaryData) != 0); return; case SerializedType::LONG_STRING: { uint32_t size = getValue(binaryData); sol::stack::push(lua, binaryData.substr(0, size)); binaryData = binaryData.substr(size); return; } case SerializedType::TABLE_START: { lua_createtable(lua, 0, 0); while (!binaryData.empty() && binaryData[0] != char(SerializedType::TABLE_END)) { deserializeImpl(lua, binaryData, customSerializer, readOnly); deserializeImpl(lua, binaryData, customSerializer, readOnly); lua_settable(lua, -3); } if (binaryData.empty()) throw std::runtime_error("Unexpected end of serialized data."); binaryData = binaryData.substr(1); if (readOnly) sol::stack::push(lua, makeReadOnly(sol::stack::pop(lua))); return; } case SerializedType::TABLE_END: throw std::runtime_error("Unexpected end of table during deserialization."); case SerializedType::VEC2: { float x = getValue(binaryData); float y = getValue(binaryData); sol::stack::push(lua, osg::Vec2f(x, y)); return; } case SerializedType::VEC3: { float x = getValue(binaryData); float y = getValue(binaryData); float z = getValue(binaryData); sol::stack::push(lua, osg::Vec3f(x, y, z)); return; } case SerializedType::TRANSFORM_M: { osg::Matrixf mat; for (int i = 0; i < 4; i++) for (int j = 0; j < 4; j++) mat(i, j) = getValue(binaryData); sol::stack::push(lua, asTransform(mat)); return; } case SerializedType::TRANSFORM_Q: { osg::Quat q; for (int i = 0; i < 4; i++) q[i] = getValue(binaryData); sol::stack::push(lua, asTransform(q)); return; } case SerializedType::VEC4: { float x = getValue(binaryData); float y = getValue(binaryData); float z = getValue(binaryData); float w = getValue(binaryData); sol::stack::push(lua, osg::Vec4f(x, y, z, w)); return; } case SerializedType::COLOR: { float r = getValue(binaryData); float g = getValue(binaryData); float b = getValue(binaryData); float a = getValue(binaryData); sol::stack::push(lua, Misc::Color(r, g, b, a)); return; } } throw std::runtime_error("Unknown type in serialized data: " + std::to_string(type)); } BinaryData serialize(const sol::object& obj, const UserdataSerializer* customSerializer) { if (obj == sol::nil) return ""; BinaryData res; res.push_back(FORMAT_VERSION); serialize(res, obj, customSerializer, 0); return res; } sol::object deserialize( lua_State* lua, std::string_view binaryData, const UserdataSerializer* customSerializer, bool readOnly) { if (binaryData.empty()) return sol::nil; if (binaryData[0] != FORMAT_VERSION) throw std::runtime_error("Incorrect version of Lua serialization format: " + std::to_string(static_cast(binaryData[0]))); binaryData = binaryData.substr(1); deserializeImpl(lua, binaryData, customSerializer, readOnly); if (!binaryData.empty()) throw std::runtime_error("Unexpected data after serialized object"); return sol::stack::pop(lua); } }