From 4d18200b0c6a1c557d4f05f757bc264338104aa9 Mon Sep 17 00:00:00 2001
From: David Capello <david@igara.com>
Date: Mon, 22 Apr 2024 11:13:10 -0300
Subject: [PATCH] [lua] Fix bugs handling errors inside app.transaction()

This errors was reported in #4431: The Tx wasn't rolled back correctly
in case of a Lua error inside the transaction because Lua needs to be
compiled as C++ to avoid longjmps and support stack
unwinding (i.e. calling destructors).
---
 src/app/script/engine.cpp  | 15 ++++++++-------
 src/app/script/luacpp.h    | 13 +++++++------
 third_party/CMakeLists.txt | 14 +++++++++++++-
 3 files changed, 28 insertions(+), 14 deletions(-)

diff --git a/src/app/script/engine.cpp b/src/app/script/engine.cpp
index 4909e06b4..abb002918 100644
--- a/src/app/script/engine.cpp
+++ b/src/app/script/engine.cpp
@@ -42,6 +42,14 @@
 #include <stack>
 #include <string>
 
+// We use our own fopen() that supports Unicode filename on Windows
+// extern "C"
+FILE* lua_user_fopen(const char* fname,
+                     const char* mode)
+{
+  return base::open_file_raw(fname, mode);
+}
+
 namespace app {
 namespace script {
 
@@ -210,13 +218,6 @@ void register_websocket_class(lua_State* L);
 
 void set_app_params(lua_State* L, const Params& params);
 
-// We use our own fopen() that supports Unicode filename on Windows
-extern "C" FILE* lua_user_fopen(const char* fname,
-                                const char* mode)
-{
-  return base::open_file_raw(fname, mode);
-}
-
 Engine::Engine()
   : L(luaL_newstate())
   , m_delegate(nullptr)
diff --git a/src/app/script/luacpp.h b/src/app/script/luacpp.h
index 8ead070e4..7c2f242aa 100644
--- a/src/app/script/luacpp.h
+++ b/src/app/script/luacpp.h
@@ -1,5 +1,5 @@
 // Aseprite
-// Copyright (C) 2018-2023  Igara Studio S.A.
+// Copyright (C) 2018-2024  Igara Studio S.A.
 // Copyright (C) 2018  David Capello
 //
 // This program is distributed under the terms of
@@ -9,11 +9,12 @@
 #define APP_SCRIPT_LUACPP_H_INCLUDED
 #pragma once
 
-extern "C" {
-  #include "lua.h"
-  #include "lualib.h"
-  #include "lauxlib.h"
-}
+// We're compiling Lua with C++ support to handle error with
+// exceptions, so there is no need of extern "C" { ... } these
+// includes.
+#include "lua.h"
+#include "lualib.h"
+#include "lauxlib.h"
 
 #include "base/debug.h"
 
diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt
index 9bc9eb008..11985a5c7 100644
--- a/third_party/CMakeLists.txt
+++ b/third_party/CMakeLists.txt
@@ -205,7 +205,19 @@ if(ENABLE_SCRIPTING)
     target_compile_definitions(lua PUBLIC LUA_USE_LINUX=1)
   endif()
 
-  target_compile_definitions(lua PUBLIC LUA_FLOORN2I=1)
+  # Compile Lua as C++ to control errors with exceptions and have
+  # stack unwinding (i.e. calling destructors correctly).
+  if(MSVC)
+    target_compile_options(lua PRIVATE -TP)
+    target_compile_options(lualib PRIVATE -TP)
+    target_compile_options(lauxlib PRIVATE -TP)
+  else()
+    target_compile_options(lua PRIVATE -xc++)
+    target_compile_options(lualib PRIVATE -xc++)
+    target_compile_options(lauxlib PRIVATE -xc++)
+  endif()
+
+  target_compile_definitions(lua PUBLIC LUA_FLOORN2I=F2Ifloor)
   target_compile_definitions(lualib PRIVATE HAVE_SYSTEM)
   target_include_directories(lua PUBLIC lua)
   target_include_directories(lauxlib PUBLIC lua)