mirror/OpenMW
1
0
Fork 0
mirror of https://gitlab.com/OpenMW/openmw.git synced 2025-01-27 03:35:27 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'unloadedcontainers' into 'master'

Browse Source 
Fix Lua memory usage

See merge request OpenMW/openmw!4363
This commit is contained in:
Petr Mikheev 2024-10-15 21:18:13 +00:00
commit bac0018a09
10 changed files with 425 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
apps/components_tests/lua/test_scriptscontainer.cpp
View File

@ -6,6 +6,7 @@
#include <components/lua/asyncpackage.hpp>
#include <components/lua/luastate.hpp>
#include <components/lua/scriptscontainer.hpp>
#include <components/lua/scripttracker.hpp>
#include <components/testing/util.hpp>
@ -135,6 +136,31 @@ return {
end,
},
}
)X");
constexpr VFS::Path::NormalizedView unloadPath("unload.lua");
VFSTestFile unloadScript(R"X(
x = 0
y = 0
z = 0
return {
engineHandlers = {
onSave = function(state)
print('saving', x, y, z)
return {x = x, y = y}
end,
onLoad = function(state)
x, y = state.x, state.y
print('loaded', x, y, z)
end
},
eventHandlers = {
Set = function(eventData)
x, y, z = eventData.x, eventData.y, eventData.z
end
}
}
)X");
struct LuaScriptsContainerTest : Test
@ -151,6 +177,7 @@ return {
{ testInterfacePath, &interfaceScript },
{ overrideInterfacePath, &overrideInterfaceScript },
{ useInterfacePath, &useInterfaceScript },
{ unloadPath, &unloadScript },
});
LuaUtil::ScriptsConfiguration mCfg;
@ -171,6 +198,7 @@ CUSTOM, NPC: loadSave2.lua
CUSTOM, PLAYER: testInterface.lua
CUSTOM, PLAYER: overrideInterface.lua
CUSTOM, PLAYER: useInterface.lua
CUSTOM: unload.lua
)X");
mCfg.init(std::move(cfg));
}
@ -511,4 +539,35 @@ CUSTOM, PLAYER: useInterface.lua
Log::sMinDebugLevel = level;
}
TEST_F(LuaScriptsContainerTest, Unload)
{
LuaUtil::ScriptTracker tracker;
LuaUtil::ScriptsContainer scripts1(&mLua, "Test", &tracker, false);
EXPECT_TRUE(scripts1.addCustomScript(*mCfg.findId(unloadPath)));
EXPECT_EQ(tracker.size(), 1);
mLua.protectedCall([&](LuaUtil::LuaView& lua) {
scripts1.receiveEvent("Set", LuaUtil::serialize(lua.sol().create_table_with("x", 3, "y", 2, "z", 1)));
testing::internal::CaptureStdout();
for (int i = 0; i < 600; ++i)
tracker.unloadInactiveScripts(lua);
EXPECT_EQ(tracker.size(), 0);
scripts1.receiveEvent("Set", LuaUtil::serialize(lua.sol().create_table_with("x", 10, "y", 20, "z", 30)));
EXPECT_EQ(internal::GetCapturedStdout(),
"Test[unload.lua]:\tsaving\t3\t2\t1\n"
"Test[unload.lua]:\tloaded\t3\t2\t0\n");
});
EXPECT_EQ(tracker.size(), 1);
ESM::LuaScripts data;
scripts1.save(data);
EXPECT_EQ(tracker.size(), 1);
mLua.protectedCall([&](LuaUtil::LuaView& lua) {
for (int i = 0; i < 600; ++i)
tracker.unloadInactiveScripts(lua);
});
EXPECT_EQ(tracker.size(), 0);
scripts1.load(data);
EXPECT_EQ(tracker.size(), 0);
}
}

4
apps/openmw/mwlua/localscripts.cpp
View File

@ -224,8 +224,8 @@ namespace MWLua
};
}
LocalScripts::LocalScripts(LuaUtil::LuaState* lua, const LObject& obj)
: LuaUtil::ScriptsContainer(lua, "L" + obj.id().toString())
LocalScripts::LocalScripts(LuaUtil::LuaState* lua, const LObject& obj, LuaUtil::ScriptTracker* tracker)
: LuaUtil::ScriptsContainer(lua, "L" + obj.id().toString(), tracker, false)
, mData(obj)
{
lua->protectedCall(

3
apps/openmw/mwlua/localscripts.hpp
View File

@ -62,12 +62,13 @@ namespace MWLua
{
public:
static void initializeSelfPackage(const Context&);
LocalScripts(LuaUtil::LuaState* lua, const LObject& obj);
LocalScripts(LuaUtil::LuaState* lua, const LObject& obj, LuaUtil::ScriptTracker* tracker = nullptr);
MWBase::LuaManager::ActorControls* getActorControls() { return &mData.mControls; }
const MWWorld::Ptr& getPtrOrEmpty() const { return mData.ptrOrEmpty(); }
void setActive(bool active);
bool isActive() const override { return mData.mIsActive; }
void onConsume(const LObject& consumable) { callEngineHandlers(mOnConsumeHandlers, consumable); }
void onActivated(const LObject& actor) { callEngineHandlers(mOnActivatedHandlers, actor); }
void onTeleported() { callEngineHandlers(mOnTeleportedHandlers); }

4
apps/openmw/mwlua/luamanagerimp.cpp
View File

@ -210,6 +210,8 @@ namespace MWLua
scripts->update(frameDuration);
mGlobalScripts.update(frameDuration);
}
mLua.protectedCall([&](LuaUtil::LuaView& lua) { mScriptTracker.unloadInactiveScripts(lua); });
}
void LuaManager::objectTeleported(const MWWorld::Ptr& ptr)
@ -560,7 +562,7 @@ namespace MWLua
}
else
{
scripts = std::make_shared<LocalScripts>(&mLua, LObject(getId(ptr)));
scripts = std::make_shared<LocalScripts>(&mLua, LObject(getId(ptr)), &mScriptTracker);
if (!autoStartConf.has_value())
autoStartConf = mConfiguration.getLocalConf(type, ptr.getCellRef().getRefId(), getId(ptr));
scripts->setAutoStartConf(std::move(*autoStartConf));

3
apps/openmw/mwlua/luamanagerimp.hpp
View File

@ -8,6 +8,7 @@
#include <components/lua/inputactions.hpp>
#include <components/lua/luastate.hpp>
#include <components/lua/scripttracker.hpp>
#include <components/lua/storage.hpp>
#include <components/lua_ui/resources.hpp>
#include <components/misc/color.hpp>
@ -234,6 +235,8 @@ namespace MWLua
LuaUtil::InputAction::Registry mInputActions;
LuaUtil::InputTrigger::Registry mInputTriggers;
LuaUtil::ScriptTracker mScriptTracker;
};
}

2
components/CMakeLists.txt
View File

@ -59,7 +59,7 @@ list(APPEND COMPONENT_FILES "${OpenMW_BINARY_DIR}/${OSG_PLUGIN_CHECKER_CPP_FILE}
add_component_dir (lua
luastate scriptscontainer asyncpackage utilpackage serialization configuration l10n storage utf8
shapes/box inputactions yamlloader
shapes/box inputactions yamlloader scripttracker
)
add_component_dir (l10n

312
components/lua/scriptscontainer.cpp
View File

@ -1,7 +1,18 @@
#include "scriptscontainer.hpp"
#include "scripttracker.hpp"
#include <components/esm/luascripts.hpp>
namespace
{
struct ScriptInfo
{
std::string_view mInitData;
const ESM::LuaScript* mSavedData;
};
}
namespace LuaUtil
{
static constexpr std::string_view ENGINE_HANDLERS = "engineHandlers";
@ -17,17 +28,23 @@ namespace LuaUtil
int64_t ScriptsContainer::sInstanceCount = 0;
ScriptsContainer::ScriptsContainer(LuaUtil::LuaState* lua, std::string_view namePrefix)
ScriptsContainer::ScriptsContainer(
LuaUtil::LuaState* lua, std::string_view namePrefix, ScriptTracker* tracker, bool load)
: mNamePrefix(namePrefix)
, mLua(*lua)
, mThis(std::make_shared<ScriptsContainer*>(this))
, mTracker(tracker)
{
sInstanceCount++;
registerEngineHandlers({ &mUpdateHandlers });
lua->protectedCall([&](LuaView& view) {
mPublicInterfaces = sol::table(view.sol(), sol::create);
addPackage("openmw.interfaces", mPublicInterfaces);
});
if (load)
{
LoadedData& data = mData.emplace<LoadedData>();
mLua.protectedCall([&](LuaView& view) {
data.mPublicInterfaces = sol::table(view.sol(), sol::create);
addPackage("openmw.interfaces", makeReadOnly(data.mPublicInterfaces));
});
}
}
void ScriptsContainer::printError(int scriptId, std::string_view msg, const std::exception& e)
@ -37,7 +54,9 @@ namespace LuaUtil
void ScriptsContainer::addPackage(std::string packageName, sol::object package)
{
mAPI.insert_or_assign(std::move(packageName), makeReadOnly(std::move(package)));
if (!package.is<sol::userdata>())
throw std::logic_error("Expected package to be read-only: " + packageName);
mAPI.insert_or_assign(std::move(packageName), std::move(package));
}
bool ScriptsContainer::addCustomScript(int scriptId, std::string_view initData)
@ -70,7 +89,7 @@ namespace LuaUtil
LuaView& view, int scriptId, std::optional<sol::function>& onInit, std::optional<sol::function>& onLoad)
{
assert(scriptId >= 0 && scriptId < static_cast<int>(mLua.getConfiguration().size()));
if (mScripts.count(scriptId) != 0)
if (hasScript(scriptId))
return false; // already present
const VFS::Path::Normalized& path = scriptPath(scriptId);
@ -79,7 +98,8 @@ namespace LuaUtil
debugName.append(path);
debugName.push_back(']');
Script& script = mScripts[scriptId];
LoadedData& data = ensureLoaded();
Script& script = data.mScripts[scriptId];
script.mHiddenData = view.newTable();
script.mHiddenData[sScriptIdKey] = ScriptId{ this, scriptId };
script.mHiddenData[sScriptDebugNameKey] = debugName;
@ -144,9 +164,9 @@ namespace LuaUtil
for (const auto& [key, fn] : cast<sol::table>(eventHandlers))
{
std::string_view eventName = cast<std::string_view>(key);
auto it = mEventHandlers.find(eventName);
if (it == mEventHandlers.end())
it = mEventHandlers.emplace(std::string(eventName), EventHandlerList()).first;
auto it = data.mEventHandlers.find(eventName);
if (it == data.mEventHandlers.end())
it = data.mEventHandlers.emplace(std::string(eventName), EventHandlerList()).first;
insertHandler(it->second, scriptId, cast<sol::function>(fn));
}
}
@ -167,29 +187,55 @@ namespace LuaUtil
}
catch (std::exception& e)
{
auto iter = mScripts.find(scriptId);
iter->second.mHiddenData[sScriptIdKey] = sol::nil;
auto iter = data.mScripts.find(scriptId);
mRemovedScriptsMemoryUsage[scriptId] = iter->second.mStats.mMemoryUsage;
mScripts.erase(iter);
data.mScripts.erase(iter);
Log(Debug::Error) << "Can't start " << debugName << "; " << e.what();
return false;
}
}
bool ScriptsContainer::hasScript(int scriptId) const
{
return std::visit(
[&](auto&& variant) {
using T = std::decay_t<decltype(variant)>;
if constexpr (std::is_same_v<T, UnloadedData>)
{
const auto& conf = mLua.getConfiguration();
if (scriptId >= 0 && static_cast<size_t>(scriptId) < conf.size())
{
const auto& path = conf[scriptId].mScriptPath;
for (const ESM::LuaScript& script : variant.mScripts)
{
if (script.mScriptPath == path)
return true;
}
}
}
else if constexpr (std::is_same_v<T, LoadedData>)
{
return variant.mScripts.count(scriptId) != 0;
}
return false;
},
mData);
}
void ScriptsContainer::removeScript(int scriptId)
{
auto scriptIter = mScripts.find(scriptId);
if (scriptIter == mScripts.end())
LoadedData& data = ensureLoaded();
auto scriptIter = data.mScripts.find(scriptId);
if (scriptIter == data.mScripts.end())
return; // no such script
Script& script = scriptIter->second;
if (script.mInterface)
removeInterface(scriptId, script);
script.mHiddenData[sScriptIdKey] = sol::nil;
mRemovedScriptsMemoryUsage[scriptId] = script.mStats.mMemoryUsage;
mScripts.erase(scriptIter);
data.mScripts.erase(scriptIter);
for (auto& [_, handlers] : mEngineHandlers)
removeHandler(handlers->mList, scriptId);
for (auto& [_, handlers] : mEventHandlers)
for (auto& [_, handlers] : data.mEventHandlers)
removeHandler(handlers, scriptId);
}
@ -199,7 +245,8 @@ namespace LuaUtil
const Script* prev = nullptr;
const Script* next = nullptr;
int nextId = 0;
for (const auto& [otherId, otherScript] : mScripts)
LoadedData& data = ensureLoaded();
for (const auto& [otherId, otherScript] : data.mScripts)
{
if (scriptId == otherId || script.mInterfaceName != otherScript.mInterfaceName)
continue;
@ -235,7 +282,7 @@ namespace LuaUtil
}
}
if (next == nullptr)
mPublicInterfaces[script.mInterfaceName] = *script.mInterface;
data.mPublicInterfaces[script.mInterfaceName] = *script.mInterface;
}
void ScriptsContainer::removeInterface(int scriptId, const Script& script)
@ -244,7 +291,8 @@ namespace LuaUtil
const Script* prev = nullptr;
const Script* next = nullptr;
int nextId = 0;
for (const auto& [otherId, otherScript] : mScripts)
LoadedData& data = ensureLoaded();
for (const auto& [otherId, otherScript] : data.mScripts)
{
if (scriptId == otherId || script.mInterfaceName != otherScript.mInterfaceName)
continue;
@ -275,9 +323,9 @@ namespace LuaUtil
}
}
else if (prev)
mPublicInterfaces[script.mInterfaceName] = *prev->mInterface;
data.mPublicInterfaces[script.mInterfaceName] = *prev->mInterface;
else
mPublicInterfaces[script.mInterfaceName] = sol::nil;
data.mPublicInterfaces[script.mInterfaceName] = sol::nil;
}
void ScriptsContainer::insertHandler(std::vector<Handler>& list, int scriptId, sol::function fn)
@ -302,8 +350,9 @@ namespace LuaUtil
void ScriptsContainer::receiveEvent(std::string_view eventName, std::string_view eventData)
{
auto it = mEventHandlers.find(eventName);
if (it == mEventHandlers.end())
LoadedData& data = ensureLoaded();
auto it = data.mEventHandlers.find(eventName);
if (it == data.mEventHandlers.end())
return;
mLua.protectedCall([&](LuaView& view) {
sol::object data;
@ -355,6 +404,12 @@ namespace LuaUtil
void ScriptsContainer::save(ESM::LuaScripts& data)
{
if (UnloadedData* unloadedData = std::get_if<UnloadedData>(&mData))
{
data.mScripts = unloadedData->mScripts;
return;
}
const auto& loadedData = std::get<LoadedData>(mData);
std::map<int, std::vector<ESM::LuaTimer>> timers;
auto saveTimerFn = [&](const Timer& timer, TimerType timerType) {
if (!timer.mSerializable)
@ -366,12 +421,12 @@ namespace LuaUtil
savedTimer.mCallbackArgument = timer.mSerializedArg;
timers[timer.mScriptId].push_back(std::move(savedTimer));
};
for (const Timer& timer : mSimulationTimersQueue)
for (const Timer& timer : loadedData.mSimulationTimersQueue)
saveTimerFn(timer, TimerType::SIMULATION_TIME);
for (const Timer& timer : mGameTimersQueue)
for (const Timer& timer : loadedData.mGameTimersQueue)
saveTimerFn(timer, TimerType::GAME_TIME);
data.mScripts.clear();
for (auto& [scriptId, script] : mScripts)
for (auto& [scriptId, script] : loadedData.mScripts)
{
ESM::LuaScript savedScript;
// Note: We can not use `scriptPath(scriptId)` here because `save` can be called during
@ -401,11 +456,6 @@ namespace LuaUtil
removeAllScripts();
const ScriptsConfiguration& cfg = mLua.getConfiguration();
struct ScriptInfo
{
std::string_view mInitData;
const ESM::LuaScript* mSavedData;
};
std::map<int, ScriptInfo> scripts;
for (const auto& [scriptId, initData] : mAutoStartScripts)
scripts[scriptId] = { initData, nullptr };
@ -428,6 +478,60 @@ namespace LuaUtil
}
mLua.protectedCall([&](LuaView& view) {
UnloadedData& container = ensureUnloaded(view);
for (const auto& [scriptId, scriptInfo] : scripts)
{
if (scriptInfo.mSavedData == nullptr)
continue;
ESM::LuaScript& script = container.mScripts.emplace_back(*scriptInfo.mSavedData);
for (ESM::LuaTimer& savedTimer : script.mTimers)
{
try
{
sol::object arg = deserialize(view.sol(), savedTimer.mCallbackArgument, mSavedDataDeserializer);
// It is important if the order of content files was changed. The deserialize-serialize
// procedure updates refnums, so timer.mSerializedArg may be not equal to
// savedTimer.mCallbackArgument.
savedTimer.mCallbackArgument = serialize(arg, mSerializer);
}
catch (std::exception& e)
{
printError(scriptId, "can not load timer", e);
}
}
}
});
}
ScriptsContainer::LoadedData& ScriptsContainer::ensureLoaded()
{
mRequiredLoading = true;
if (LoadedData* data = std::get_if<LoadedData>(&mData))
return *data;
UnloadedData& unloadedData = std::get<UnloadedData>(mData);
std::vector<ESM::LuaScript> savedScripts = std::move(unloadedData.mScripts);
LoadedData& data = mData.emplace<LoadedData>();
const ScriptsConfiguration& cfg = mLua.getConfiguration();
std::map<int, ScriptInfo> scripts;
for (const auto& [scriptId, initData] : mAutoStartScripts)
scripts[scriptId] = { initData, nullptr };
for (const ESM::LuaScript& s : savedScripts)
{
std::optional<int> scriptId = cfg.findId(s.mScriptPath);
auto it = scripts.find(*scriptId);
if (it != scripts.end())
it->second.mSavedData = &s;
else if (cfg.isCustomScript(*scriptId))
scripts[*scriptId] = { cfg[*scriptId].mInitializationData, &s };
}
mLua.protectedCall([&](LuaView& view) {
data.mPublicInterfaces = sol::table(view.sol(), sol::create);
addPackage("openmw.interfaces", makeReadOnly(data.mPublicInterfaces));
for (const auto& [scriptId, scriptInfo] : scripts)
{
std::optional<sol::function> onInit, onLoad;
@ -471,9 +575,9 @@ namespace LuaUtil
timer.mSerializedArg = serialize(timer.mArg, mSerializer);
if (savedTimer.mType == TimerType::GAME_TIME)
mGameTimersQueue.push_back(std::move(timer));
data.mGameTimersQueue.push_back(std::move(timer));
else
mSimulationTimersQueue.push_back(std::move(timer));
data.mSimulationTimersQueue.push_back(std::move(timer));
}
catch (std::exception& e)
{
@ -483,15 +587,32 @@ namespace LuaUtil
}
});
std::make_heap(mSimulationTimersQueue.begin(), mSimulationTimersQueue.end());
std::make_heap(mGameTimersQueue.begin(), mGameTimersQueue.end());
std::make_heap(data.mSimulationTimersQueue.begin(), data.mSimulationTimersQueue.end());
std::make_heap(data.mGameTimersQueue.begin(), data.mGameTimersQueue.end());
if (mTracker)
mTracker->onLoad(*this);
return data;
}
ScriptsContainer::UnloadedData& ScriptsContainer::ensureUnloaded(LuaView&)
{
if (UnloadedData* data = std::get_if<UnloadedData>(&mData))
return *data;
UnloadedData data;
save(data);
mAPI.erase("openmw.interfaces");
UnloadedData& out = mData.emplace<UnloadedData>(std::move(data));
for (auto& [_, handlers] : mEngineHandlers)
handlers->mList.clear();
mRequiredLoading = false;
return out;
}
ScriptsContainer::~ScriptsContainer()
{
sInstanceCount--;
for (auto& [_, script] : mScripts)
script.mHiddenData[sScriptIdKey] = sol::nil;
*mThis = nullptr;
}
@ -499,24 +620,36 @@ namespace LuaUtil
// external objects that are already removed during child class destruction.
void ScriptsContainer::removeAllScripts()
{
for (auto& [id, script] : mScripts)
{
script.mHiddenData[sScriptIdKey] = sol::nil;
mRemovedScriptsMemoryUsage[id] = script.mStats.mMemoryUsage;
}
mScripts.clear();
for (auto& [_, handlers] : mEngineHandlers)
handlers->mList.clear();
mEventHandlers.clear();
mSimulationTimersQueue.clear();
mGameTimersQueue.clear();
mPublicInterfaces.clear();
std::visit(
[&](auto&& variant) {
using T = std::decay_t<decltype(variant)>;
if constexpr (std::is_same_v<T, UnloadedData>)
{
variant.mScripts.clear();
}
else if constexpr (std::is_same_v<T, LoadedData>)
{
for (auto& [id, script] : variant.mScripts)
{
mRemovedScriptsMemoryUsage[id] = script.mStats.mMemoryUsage;
}
variant.mScripts.clear();
for (auto& [_, handlers] : mEngineHandlers)
handlers->mList.clear();
variant.mEventHandlers.clear();
variant.mSimulationTimersQueue.clear();
variant.mGameTimersQueue.clear();
variant.mPublicInterfaces.clear();
}
},
mData);
}
ScriptsContainer::Script& ScriptsContainer::getScript(int scriptId)
{
auto it = mScripts.find(scriptId);
if (it == mScripts.end())
LoadedData& data = ensureLoaded();
auto it = data.mScripts.find(scriptId);
if (it == data.mScripts.end())
throw std::logic_error("Script doesn't exist");
return it->second;
}
@ -543,7 +676,8 @@ namespace LuaUtil
t.mTime = time;
t.mArg = std::move(callbackArg);
t.mSerializedArg = serialize(t.mArg, mSerializer);
insertTimer(type == TimerType::GAME_TIME ? mGameTimersQueue : mSimulationTimersQueue, std::move(t));
LoadedData& data = ensureLoaded();
insertTimer(type == TimerType::GAME_TIME ? data.mGameTimersQueue : data.mSimulationTimersQueue, std::move(t));
}
void ScriptsContainer::setupUnsavableTimer(
@ -557,8 +691,8 @@ namespace LuaUtil
t.mCallback = mTemporaryCallbackCounter;
getScript(t.mScriptId).mTemporaryCallbacks.emplace(mTemporaryCallbackCounter, std::move(callback));
mTemporaryCallbackCounter++;
insertTimer(type == TimerType::GAME_TIME ? mGameTimersQueue : mSimulationTimersQueue, std::move(t));
LoadedData& data = ensureLoaded();
insertTimer(type == TimerType::GAME_TIME ? data.mGameTimersQueue : data.mSimulationTimersQueue, std::move(t));
}
void ScriptsContainer::callTimer(const Timer& t)
@ -599,41 +733,52 @@ namespace LuaUtil
void ScriptsContainer::processTimers(double simulationTime, double gameTime)
{
updateTimerQueue(mSimulationTimersQueue, simulationTime);
updateTimerQueue(mGameTimersQueue, gameTime);
LoadedData& data = ensureLoaded();
updateTimerQueue(data.mSimulationTimersQueue, simulationTime);
updateTimerQueue(data.mGameTimersQueue, gameTime);
}
static constexpr float instructionCountAvgCoef = 1.0f / 30; // averaging over approximately 30 frames
void ScriptsContainer::statsNextFrame()
{
for (auto& [scriptId, script] : mScripts)
if (LoadedData* data = std::get_if<LoadedData>(&mData))
{
// The averaging formula is: averageValue = averageValue * (1-c) + newValue * c
script.mStats.mAvgInstructionCount *= 1 - instructionCountAvgCoef;
if (script.mStats.mAvgInstructionCount < 5)
script.mStats.mAvgInstructionCount = 0; // speeding up converge to zero if newValue is zero
for (auto& [scriptId, script] : data->mScripts)
{
// The averaging formula is: averageValue = averageValue * (1-c) + newValue * c
script.mStats.mAvgInstructionCount *= 1 - instructionCountAvgCoef;
if (script.mStats.mAvgInstructionCount < 5)
script.mStats.mAvgInstructionCount = 0; // speeding up converge to zero if newValue is zero
}
}
}
void ScriptsContainer::addInstructionCount(int scriptId, int64_t instructionCount)
{
auto it = mScripts.find(scriptId);
if (it != mScripts.end())
it->second.mStats.mAvgInstructionCount += instructionCount * instructionCountAvgCoef;
if (LoadedData* data = std::get_if<LoadedData>(&mData))
{
auto it = data->mScripts.find(scriptId);
if (it != data->mScripts.end())
it->second.mStats.mAvgInstructionCount += instructionCount * instructionCountAvgCoef;
}
}
void ScriptsContainer::addMemoryUsage(int scriptId, int64_t memoryDelta)
{
int64_t* usage;
auto it = mScripts.find(scriptId);
if (it != mScripts.end())
usage = &it->second.mStats.mMemoryUsage;
else
{
auto [rIt, _] = mRemovedScriptsMemoryUsage.emplace(scriptId, 0);
usage = &rIt->second;
}
int64_t* usage = std::visit(
[&](auto&& variant) {
using T = std::decay_t<decltype(variant)>;
if constexpr (std::is_same_v<T, LoadedData>)
{
auto it = variant.mScripts.find(scriptId);
if (it != variant.mScripts.end())
return &it->second.mStats.mMemoryUsage;
}
auto [rIt, _] = mRemovedScriptsMemoryUsage.emplace(scriptId, 0);
return &rIt->second;
},
mData);
*usage += memoryDelta;
if (mLua.getSettings().mLogMemoryUsage)
@ -651,12 +796,21 @@ namespace LuaUtil
void ScriptsContainer::collectStats(std::vector<ScriptStats>& stats) const
{
stats.resize(mLua.getConfiguration().size());
for (auto& [id, script] : mScripts)
if (const LoadedData* data = std::get_if<LoadedData>(&mData))
{
stats[id].mAvgInstructionCount += script.mStats.mAvgInstructionCount;
stats[id].mMemoryUsage += script.mStats.mMemoryUsage;
for (auto& [id, script] : data->mScripts)
{
stats[id].mAvgInstructionCount += script.mStats.mAvgInstructionCount;
stats[id].mMemoryUsage += script.mStats.mMemoryUsage;
}
}
for (auto& [id, mem] : mRemovedScriptsMemoryUsage)
stats[id].mMemoryUsage += mem;
}
ScriptsContainer::Script::~Script()
{
if (mHiddenData != sol::nil)
mHiddenData[sScriptIdKey] = sol::nil;
}
}

54
components/lua/scriptscontainer.hpp
View File

@ -4,6 +4,7 @@
#include <map>
#include <set>
#include <string>
#include <variant>
#include <components/debug/debuglog.hpp>
#include <components/esm/luascripts.hpp>
@ -13,6 +14,7 @@
namespace LuaUtil
{
class ScriptTracker;
// ScriptsContainer is a base class for all scripts containers (LocalScripts,
// GlobalScripts, PlayerScripts, etc). Each script runs in a separate sandbox.
@ -72,14 +74,17 @@ namespace LuaUtil
using TimerType = ESM::LuaTimer::Type;
// `namePrefix` is a common prefix for all scripts in the container. Used in logs for error messages and `print`
// output. `autoStartScripts` specifies the list of scripts that should be autostarted in this container;
// the script names themselves are stored in ScriptsConfiguration.
ScriptsContainer(LuaState* lua, std::string_view namePrefix);
// output. `tracker` is a tracker for managing the container's state. `load` specifies whether the container
// should be constructed in a loaded state.
ScriptsContainer(
LuaState* lua, std::string_view namePrefix, ScriptTracker* tracker = nullptr, bool load = true);
ScriptsContainer(const ScriptsContainer&) = delete;
ScriptsContainer(ScriptsContainer&&) = delete;
virtual ~ScriptsContainer();
// `conf` specifies the list of scripts that should be autostarted in this container; the script
// names themselves are stored in ScriptsConfiguration.
void setAutoStartConf(ScriptIdsWithInitializationData conf) { mAutoStartScripts = std::move(conf); }
const ScriptIdsWithInitializationData& getAutoStartConf() const { return mAutoStartScripts; }
@ -91,9 +96,9 @@ namespace LuaUtil
// new script, adds it to the container, and calls onInit for this script. Returns `true` if the script was
// successfully added. The script should have CUSTOM flag. If the flag is not set, or file not found, or has
// syntax errors, returns false. If such script already exists in the container, then also returns false.
bool addCustomScript(int scriptId, std::string_view initData = "");
bool addCustomScript(int scriptId, std::string_view initData = {});
bool hasScript(int scriptId) const { return mScripts.count(scriptId) != 0; }
bool hasScript(int scriptId) const;
void removeScript(int scriptId);
void processTimers(double simulationTime, double gameTime);
@ -157,6 +162,8 @@ namespace LuaUtil
void collectStats(std::vector<ScriptStats>& stats) const;
static int64_t getInstanceCount() { return sInstanceCount; }
virtual bool isActive() const { return false; }
protected:
struct Handler
{
@ -180,6 +187,7 @@ namespace LuaUtil
template <typename... Args>
void callEngineHandlers(EngineHandlerList& handlers, const Args&... args)
{
ensureLoaded();
for (Handler& handler : handlers.mList)
{
try
@ -213,6 +221,8 @@ namespace LuaUtil
std::map<int64_t, sol::main_protected_function> mTemporaryCallbacks;
VFS::Path::Normalized mPath;
ScriptStats mStats;
~Script();
};
struct Timer
{
@ -257,21 +267,39 @@ namespace LuaUtil
ScriptIdsWithInitializationData mAutoStartScripts;
const UserdataSerializer* mSerializer = nullptr;
const UserdataSerializer* mSavedDataDeserializer = nullptr;
std::map<std::string, sol::object> mAPI;
std::map<int, Script> mScripts;
sol::table mPublicInterfaces;
std::map<std::string, sol::object> mAPI;
struct LoadedData
{
std::map<int, Script> mScripts;
sol::table mPublicInterfaces;
std::map<std::string, EventHandlerList, std::less<>> mEventHandlers;
std::vector<Timer> mSimulationTimersQueue;
std::vector<Timer> mGameTimersQueue;
};
using UnloadedData = ESM::LuaScripts;
// Unloads the container to free resources held by the shared Lua state. This method serializes the container's
// state. The serialized data is automatically restored to the Lua state as required. Unloading and reloading
// the container is functionally equivalent to saving and loading the game, meaning the appropriate engine
// handlers are invoked.
UnloadedData& ensureUnloaded(LuaView& lua);
LoadedData& ensureLoaded();
EngineHandlerList mUpdateHandlers{ "onUpdate" };
std::map<std::string_view, EngineHandlerList*> mEngineHandlers;
std::map<std::string, EventHandlerList, std::less<>> mEventHandlers;
std::vector<Timer> mSimulationTimersQueue;
std::vector<Timer> mGameTimersQueue;
std::variant<UnloadedData, LoadedData> mData;
int64_t mTemporaryCallbackCounter = 0;
std::map<int, int64_t> mRemovedScriptsMemoryUsage;
std::shared_ptr<ScriptsContainer*> mThis; // used by LuaState to track ownership of memory allocations
using WeakPtr = std::shared_ptr<ScriptsContainer*>;
WeakPtr mThis; // used by LuaState to track ownership of memory allocations
ScriptTracker* mTracker;
bool mRequiredLoading = false;
friend class ScriptTracker;
static int64_t sInstanceCount; // debug information, shown in Lua profiler
};

53
components/lua/scripttracker.cpp Normal file
View File

@ -0,0 +1,53 @@
#include "scripttracker.hpp"
namespace LuaUtil
{
namespace
{
constexpr unsigned sMinLoadedFrames = 50;
constexpr unsigned sMaxLoadedFrames = 600;
constexpr unsigned sUsageFrameGrowth = 10;
constexpr std::size_t sMinToProcess = 1;
constexpr std::size_t sToProcessDiv = 20; // 5%
}
void ScriptTracker::onLoad(ScriptsContainer& container)
{
mLoadedScripts.emplace(container.mThis, sMinLoadedFrames + mFrame);
}
void ScriptTracker::unloadInactiveScripts(LuaView& lua)
{
// This code is technically incorrect if mFrame overflows... but at 300fps that takes about half a year
std::size_t toProcess = std::max(mLoadedScripts.size() / sToProcessDiv, sMinToProcess);
while (toProcess && !mLoadedScripts.empty())
{
--toProcess;
auto [ptr, ttl] = std::move(mLoadedScripts.front());
mLoadedScripts.pop();
ScriptsContainer* container = *ptr.get();
// Object no longer exists, cease tracking
if (!container)
continue;
// Ignore activity of local scripts in the active grid
if (container->isActive())
ttl = std::max(ttl, mFrame + sMinLoadedFrames);
else
{
bool activeSinceLastPop = container->mRequiredLoading;
if (activeSinceLastPop)
{
container->mRequiredLoading = false;
ttl = std::min(ttl + sUsageFrameGrowth, mFrame + sMaxLoadedFrames);
}
else if (ttl < mFrame)
{
container->ensureUnloaded(lua);
continue;
}
}
mLoadedScripts.emplace(std::move(ptr), ttl);
}
++mFrame;
}
}

28
components/lua/scripttracker.hpp Normal file
View File

@ -0,0 +1,28 @@
#ifndef COMPONENTS_LUA_SCRIPTTRACKER_H
#define COMPONENTS_LUA_SCRIPTTRACKER_H
#include <memory>
#include <queue>
#include <utility>
#include "scriptscontainer.hpp"
namespace LuaUtil
{
class ScriptTracker
{
using Frame = unsigned int;
using TrackedScriptContainer = std::pair<ScriptsContainer::WeakPtr, Frame>;
std::queue<TrackedScriptContainer> mLoadedScripts;
Frame mFrame = 0;
public:
void unloadInactiveScripts(LuaView& lua);
void onLoad(ScriptsContainer& container);
std::size_t size() const { return mLoadedScripts.size(); }
};
}
#endif // COMPONENTS_LUA_SCRIPTTRACKER_H