From 15d9fab0bbe5cae41116c7eb24dc1797f05957c1 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sun, 28 Jun 2020 18:15:30 +0200 Subject: [PATCH 1/3] Common: Rename UTF16ToUTF8 This function does *not* always convert from UTF-16. It converts from UTF-16 on Windows and UTF-32 on other operating systems. Also renaming UTF8ToUTF16 for consistency, even though it technically doesn't have the same problem since it only was implemented on Windows. --- Source/Core/Common/FileUtil.cpp | 2 +- .../Core/Common/Logging/ConsoleListenerWin.cpp | 2 +- Source/Core/Common/StringUtil.cpp | 18 +++++++++--------- Source/Core/Common/StringUtil.h | 10 +++++----- Source/Core/Common/Timer.cpp | 4 ++-- Source/Core/Core/HW/WiimoteReal/IOWin.cpp | 6 +++--- Source/Core/Core/HW/WiimoteReal/IOWin.h | 2 +- Source/Core/Core/HW/WiimoteReal/IOhidapi.cpp | 2 +- Source/Core/DolphinNoGUI/PlatformWin32.cpp | 2 +- .../ControllerInterface/DInput/DInput.cpp | 2 +- Source/Core/UICommon/AutoUpdate.cpp | 2 +- Source/Core/VideoBackends/D3DCommon/Common.cpp | 2 +- Source/Core/WinUpdater/Main.cpp | 2 +- Source/Core/WinUpdater/WinUI.cpp | 6 +++--- 14 files changed, 31 insertions(+), 31 deletions(-) diff --git a/Source/Core/Common/FileUtil.cpp b/Source/Core/Common/FileUtil.cpp index 5fc633c5e9..d5ba4d03a0 100644 --- a/Source/Core/Common/FileUtil.cpp +++ b/Source/Core/Common/FileUtil.cpp @@ -114,7 +114,7 @@ bool Exists(const std::string& path) bool IsDirectory(const std::string& path) { #ifdef _WIN32 - return PathIsDirectory(UTF8ToUTF16(path).c_str()); + return PathIsDirectory(UTF8ToWString(path).c_str()); #else return FileInfo(path).IsDirectory(); #endif diff --git a/Source/Core/Common/Logging/ConsoleListenerWin.cpp b/Source/Core/Common/Logging/ConsoleListenerWin.cpp index 9dbef6f2e2..13a1981f0d 100644 --- a/Source/Core/Common/Logging/ConsoleListenerWin.cpp +++ b/Source/Core/Common/Logging/ConsoleListenerWin.cpp @@ -17,5 +17,5 @@ ConsoleListener::~ConsoleListener() void ConsoleListener::Log([[maybe_unused]] Common::Log::LOG_LEVELS level, const char* text) { - ::OutputDebugStringW(UTF8ToUTF16(text).c_str()); + ::OutputDebugStringW(UTF8ToWString(text).c_str()); } diff --git a/Source/Core/Common/StringUtil.cpp b/Source/Core/Common/StringUtil.cpp index c44f3fa7b1..a8880d371b 100644 --- a/Source/Core/Common/StringUtil.cpp +++ b/Source/Core/Common/StringUtil.cpp @@ -457,29 +457,29 @@ std::string UTF16ToCP(u32 code_page, std::wstring_view input) return output; } -std::wstring UTF8ToUTF16(std::string_view input) +std::wstring UTF8ToWString(std::string_view input) { return CPToUTF16(CP_UTF8, input); } -std::string UTF16ToUTF8(std::wstring_view input) +std::string WStringToUTF8(std::wstring_view input) { return UTF16ToCP(CP_UTF8, input); } std::string SHIFTJISToUTF8(std::string_view input) { - return UTF16ToUTF8(CPToUTF16(CODEPAGE_SHIFT_JIS, input)); + return WStringToUTF8(CPToUTF16(CODEPAGE_SHIFT_JIS, input)); } std::string UTF8ToSHIFTJIS(std::string_view input) { - return UTF16ToCP(CODEPAGE_SHIFT_JIS, UTF8ToUTF16(input)); + return UTF16ToCP(CODEPAGE_SHIFT_JIS, UTF8ToWString(input)); } std::string CP1252ToUTF8(std::string_view input) { - return UTF16ToUTF8(CPToUTF16(CODEPAGE_WINDOWS_1252, input)); + return WStringToUTF8(CPToUTF16(CODEPAGE_WINDOWS_1252, input)); } std::string UTF16BEToUTF8(const char16_t* str, size_t max_size) @@ -487,7 +487,7 @@ std::string UTF16BEToUTF8(const char16_t* str, size_t max_size) const char16_t* str_end = std::find(str, str + max_size, '\0'); std::wstring result(static_cast(str_end - str), '\0'); std::transform(str, str_end, result.begin(), static_cast(Common::swap16)); - return UTF16ToUTF8(result); + return WStringToUTF8(result); } #else @@ -572,7 +572,7 @@ std::string UTF8ToSHIFTJIS(std::string_view input) return CodeTo("SJIS", "UTF-8", input); } -std::string UTF16ToUTF8(std::wstring_view input) +std::string WStringToUTF8(std::wstring_view input) { std::wstring_convert, wchar_t> converter; return converter.to_bytes(input.data(), input.data() + input.size()); @@ -591,7 +591,7 @@ std::string UTF16BEToUTF8(const char16_t* str, size_t max_size) std::filesystem::path StringToPath(std::string_view path) { #ifdef _MSC_VER - return std::filesystem::path(UTF8ToUTF16(path)); + return std::filesystem::path(UTF8ToWString(path)); #else return std::filesystem::path(path); #endif @@ -602,7 +602,7 @@ std::filesystem::path StringToPath(std::string_view path) std::string PathToString(const std::filesystem::path& path) { #ifdef _MSC_VER - return UTF16ToUTF8(path.native()); + return WStringToUTF8(path.native()); #else return path.native(); #endif diff --git a/Source/Core/Common/StringUtil.h b/Source/Core/Common/StringUtil.h index 8c3db9ce34..70d240dbf9 100644 --- a/Source/Core/Common/StringUtil.h +++ b/Source/Core/Common/StringUtil.h @@ -171,22 +171,22 @@ void StringPopBackIf(std::string* s, char c); std::string CP1252ToUTF8(std::string_view str); std::string SHIFTJISToUTF8(std::string_view str); std::string UTF8ToSHIFTJIS(std::string_view str); -std::string UTF16ToUTF8(std::wstring_view str); +std::string WStringToUTF8(std::wstring_view str); std::string UTF16BEToUTF8(const char16_t* str, size_t max_size); // Stops at \0 #ifdef _WIN32 -std::wstring UTF8ToUTF16(std::string_view str); +std::wstring UTF8ToWString(std::string_view str); #ifdef _UNICODE inline std::string TStrToUTF8(std::wstring_view str) { - return UTF16ToUTF8(str); + return WStringToUTF8(str); } inline std::wstring UTF8ToTStr(std::string_view str) { - return UTF8ToUTF16(str); + return UTF8ToWString(str); } #else inline std::string TStrToUTF8(std::string_view str) @@ -220,7 +220,7 @@ std::string ThousandSeparate(I value, int spaces = 0) stream << std::setw(spaces) << value; #ifdef _WIN32 - return UTF16ToUTF8(stream.str()); + return WStringToUTF8(stream.str()); #else return stream.str(); #endif diff --git a/Source/Core/Common/Timer.cpp b/Source/Core/Common/Timer.cpp index 794f2dfd49..66c71db568 100644 --- a/Source/Core/Common/Timer.cpp +++ b/Source/Core/Common/Timer.cpp @@ -218,7 +218,7 @@ std::string Timer::GetTimeFormatted() #ifdef _WIN32 struct timeb tp; (void)::ftime(&tp); - return UTF16ToUTF8(tmp) + fmt::format(":{:03}", tp.millitm); + return WStringToUTF8(tmp) + fmt::format(":{:03}", tp.millitm); #elif defined __APPLE__ struct timeval t; (void)gettimeofday(&t, nullptr); @@ -276,7 +276,7 @@ std::string Timer::GetDateTimeFormatted(double time) #ifdef _WIN32 wchar_t tmp[32] = {}; wcsftime(tmp, sizeof(tmp), L"%x %X", localTime); - return UTF16ToUTF8(tmp); + return WStringToUTF8(tmp); #else char tmp[32] = {}; strftime(tmp, sizeof(tmp), "%x %X", localTime); diff --git a/Source/Core/Core/HW/WiimoteReal/IOWin.cpp b/Source/Core/Core/HW/WiimoteReal/IOWin.cpp index 8f6c69c448..f755eedb6b 100644 --- a/Source/Core/Core/HW/WiimoteReal/IOWin.cpp +++ b/Source/Core/Core/HW/WiimoteReal/IOWin.cpp @@ -556,7 +556,7 @@ void WiimoteScannerWindows::FindWiimotes(std::vector& found_wiimotes, WinWriteMethod write_method = GetInitialWriteMethod(IsUsingToshibaStack); - if (!IsNewWiimote(UTF16ToUTF8(device_path)) || !IsWiimote(device_path, write_method)) + if (!IsNewWiimote(WStringToUTF8(device_path)) || !IsWiimote(device_path, write_method)) { free(detail_data); continue; @@ -608,7 +608,7 @@ bool WiimoteWindows::ConnectInternal() if (IsConnected()) return true; - if (!IsNewWiimote(UTF16ToUTF8(m_devicepath))) + if (!IsNewWiimote(WStringToUTF8(m_devicepath))) return false; auto const open_flags = FILE_SHARE_READ | FILE_SHARE_WRITE; @@ -886,7 +886,7 @@ void ProcessWiimotes(bool new_scan, const T& callback) DEBUG_LOG(WIIMOTE, "Authenticated %i connected %i remembered %i ", btdi.fAuthenticated, btdi.fConnected, btdi.fRemembered); - if (IsValidDeviceName(UTF16ToUTF8(btdi.szName))) + if (IsValidDeviceName(WStringToUTF8(btdi.szName))) { callback(hRadio, radioInfo, btdi); } diff --git a/Source/Core/Core/HW/WiimoteReal/IOWin.h b/Source/Core/Core/HW/WiimoteReal/IOWin.h index 86623d3919..bdca060553 100644 --- a/Source/Core/Core/HW/WiimoteReal/IOWin.h +++ b/Source/Core/Core/HW/WiimoteReal/IOWin.h @@ -26,7 +26,7 @@ class WiimoteWindows final : public Wiimote public: WiimoteWindows(const std::basic_string& path, WinWriteMethod initial_write_method); ~WiimoteWindows() override; - std::string GetId() const override { return UTF16ToUTF8(m_devicepath); } + std::string GetId() const override { return WStringToUTF8(m_devicepath); } protected: bool ConnectInternal() override; diff --git a/Source/Core/Core/HW/WiimoteReal/IOhidapi.cpp b/Source/Core/Core/HW/WiimoteReal/IOhidapi.cpp index e2abdafa80..1c92f1b048 100644 --- a/Source/Core/Core/HW/WiimoteReal/IOhidapi.cpp +++ b/Source/Core/Core/HW/WiimoteReal/IOhidapi.cpp @@ -61,7 +61,7 @@ void WiimoteScannerHidapi::FindWiimotes(std::vector& wiimotes, Wiimote hid_device_info* list = hid_enumerate(0x0, 0x0); for (hid_device_info* device = list; device; device = device->next) { - const std::string name = device->product_string ? UTF16ToUTF8(device->product_string) : ""; + const std::string name = device->product_string ? WStringToUTF8(device->product_string) : ""; const bool is_wiimote = IsValidDeviceName(name) || (device->vendor_id == 0x057e && (device->product_id == 0x0306 || device->product_id == 0x0330)); diff --git a/Source/Core/DolphinNoGUI/PlatformWin32.cpp b/Source/Core/DolphinNoGUI/PlatformWin32.cpp index 7ad2f0322c..b9331f56ab 100644 --- a/Source/Core/DolphinNoGUI/PlatformWin32.cpp +++ b/Source/Core/DolphinNoGUI/PlatformWin32.cpp @@ -113,7 +113,7 @@ bool PlatformWin32::Init() void PlatformWin32::SetTitle(const std::string& string) { - SetWindowTextW(m_hwnd, UTF8ToUTF16(string).c_str()); + SetWindowTextW(m_hwnd, UTF8ToWString(string).c_str()); } void PlatformWin32::MainLoop() diff --git a/Source/Core/InputCommon/ControllerInterface/DInput/DInput.cpp b/Source/Core/InputCommon/ControllerInterface/DInput/DInput.cpp index 6afa98ac2d..2fc0e1b972 100644 --- a/Source/Core/InputCommon/ControllerInterface/DInput/DInput.cpp +++ b/Source/Core/InputCommon/ControllerInterface/DInput/DInput.cpp @@ -38,7 +38,7 @@ std::string GetDeviceName(const LPDIRECTINPUTDEVICE8 device) std::string result; if (SUCCEEDED(device->GetProperty(DIPROP_PRODUCTNAME, &str.diph))) { - result = StripSpaces(UTF16ToUTF8(str.wsz)); + result = StripSpaces(WStringToUTF8(str.wsz)); } else { diff --git a/Source/Core/UICommon/AutoUpdate.cpp b/Source/Core/UICommon/AutoUpdate.cpp index 9d4fffbe02..4b457c6aa4 100644 --- a/Source/Core/UICommon/AutoUpdate.cpp +++ b/Source/Core/UICommon/AutoUpdate.cpp @@ -237,7 +237,7 @@ void AutoUpdateChecker::TriggerUpdate(const AutoUpdateChecker::NewVersionInforma STARTUPINFO sinfo = {sizeof(sinfo)}; sinfo.dwFlags = STARTF_FORCEOFFFEEDBACK; // No hourglass cursor after starting the process. PROCESS_INFORMATION pinfo; - if (CreateProcessW(UTF8ToUTF16(reloc_updater_path).c_str(), UTF8ToUTF16(command_line).data(), + if (CreateProcessW(UTF8ToWString(reloc_updater_path).c_str(), UTF8ToWString(command_line).data(), nullptr, nullptr, FALSE, 0, nullptr, nullptr, &sinfo, &pinfo)) { CloseHandle(pinfo.hThread); diff --git a/Source/Core/VideoBackends/D3DCommon/Common.cpp b/Source/Core/VideoBackends/D3DCommon/Common.cpp index b84fc4a4df..a80a3d6821 100644 --- a/Source/Core/VideoBackends/D3DCommon/Common.cpp +++ b/Source/Core/VideoBackends/D3DCommon/Common.cpp @@ -110,7 +110,7 @@ std::vector GetAdapterNames() std::string name; DXGI_ADAPTER_DESC desc; if (SUCCEEDED(adapter->GetDesc(&desc))) - name = UTF16ToUTF8(desc.Description); + name = WStringToUTF8(desc.Description); adapters.push_back(std::move(name)); } diff --git a/Source/Core/WinUpdater/Main.cpp b/Source/Core/WinUpdater/Main.cpp index 2e1bd36ff8..6afb807941 100644 --- a/Source/Core/WinUpdater/Main.cpp +++ b/Source/Core/WinUpdater/Main.cpp @@ -28,7 +28,7 @@ std::vector CommandLineToUtf8Argv(PCWSTR command_line) std::vector argv(nargs); for (int i = 0; i < nargs; ++i) { - argv[i] = UTF16ToUTF8(tokenized[i]); + argv[i] = WStringToUTF8(tokenized[i]); } LocalFree(tokenized); diff --git a/Source/Core/WinUpdater/WinUI.cpp b/Source/Core/WinUpdater/WinUI.cpp index 403f539b18..b972114f72 100644 --- a/Source/Core/WinUpdater/WinUI.cpp +++ b/Source/Core/WinUpdater/WinUI.cpp @@ -180,7 +180,7 @@ void ResetCurrentProgress() void Error(const std::string& text) { - auto wide_text = UTF8ToUTF16(text); + auto wide_text = UTF8ToWString(text); MessageBox(nullptr, (L"A fatal error occured and the updater cannot continue:\n " + wide_text).c_str(), @@ -200,7 +200,7 @@ void SetCurrentProgress(int current, int total) void SetDescription(const std::string& text) { - SetWindowText(label_handle, UTF8ToUTF16(text).c_str()); + SetWindowText(label_handle, UTF8ToWString(text).c_str()); } void MessageLoop() @@ -256,7 +256,7 @@ void LaunchApplication(std::string path) { // Hack: Launching the updater over the explorer ensures that admin priviliges are dropped. Why? // Ask Microsoft. - ShellExecuteW(nullptr, nullptr, L"explorer.exe", UTF8ToUTF16(path).c_str(), nullptr, SW_SHOW); + ShellExecuteW(nullptr, nullptr, L"explorer.exe", UTF8ToWString(path).c_str(), nullptr, SW_SHOW); } void Sleep(int sleep) From 38791eec1879b0a66d6ae920cfd6b42d9ae66dee Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sun, 28 Jun 2020 18:22:18 +0200 Subject: [PATCH 2/3] Common: Never convert from UCS-2 in WStringToUTF8 Probably not something we would run into in practice since Windows uses a separate implementation, but let's do it for the sake of correctness. --- Source/Core/Common/StringUtil.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Source/Core/Common/StringUtil.cpp b/Source/Core/Common/StringUtil.cpp index a8880d371b..e3c6f692f6 100644 --- a/Source/Core/Common/StringUtil.cpp +++ b/Source/Core/Common/StringUtil.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -574,7 +575,10 @@ std::string UTF8ToSHIFTJIS(std::string_view input) std::string WStringToUTF8(std::wstring_view input) { - std::wstring_convert, wchar_t> converter; + using codecvt = std::conditional_t, + std::codecvt_utf8>; + + std::wstring_convert converter; return converter.to_bytes(input.data(), input.data() + input.size()); } From f5da6e07d7405d89cc550b8c02f39a432f7888a3 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sun, 28 Jun 2020 19:16:23 +0200 Subject: [PATCH 3/3] Android: Use correct encoding when converting strings The functions with "UTF" in the name use "modified UTF-8" rather than the standard UTF-8 which Dolphin uses, at least according to Oracle's documentation, so it is incorrect for us to use them. This change fixes the problem by converting between UTF-8 and UTF-16 manually instead of letting JNI do it for us. --- .../jni/AndroidCommon/AndroidCommon.cpp | 21 +++++++++++-------- Source/Core/Common/StringUtil.cpp | 14 ++++++++++++- Source/Core/Common/StringUtil.h | 2 ++ 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/Source/Android/jni/AndroidCommon/AndroidCommon.cpp b/Source/Android/jni/AndroidCommon/AndroidCommon.cpp index cb7b9ff073..1f9973d451 100644 --- a/Source/Android/jni/AndroidCommon/AndroidCommon.cpp +++ b/Source/Android/jni/AndroidCommon/AndroidCommon.cpp @@ -5,25 +5,28 @@ #include "jni/AndroidCommon/AndroidCommon.h" #include +#include #include #include +#include "Common/StringUtil.h" + std::string GetJString(JNIEnv* env, jstring jstr) { - std::string result = ""; - if (!jstr) - return result; - - const char* s = env->GetStringUTFChars(jstr, nullptr); - result = s; - env->ReleaseStringUTFChars(jstr, s); - return result; + const jchar* jchars = env->GetStringChars(jstr, nullptr); + const jsize length = env->GetStringLength(jstr); + const std::u16string_view string_view(reinterpret_cast(jchars), length); + const std::string converted_string = UTF16ToUTF8(string_view); + env->ReleaseStringChars(jstr, jchars); + return converted_string; } jstring ToJString(JNIEnv* env, const std::string& str) { - return env->NewStringUTF(str.c_str()); + const std::u16string converted_string = UTF8ToUTF16(str); + return env->NewString(reinterpret_cast(converted_string.data()), + converted_string.size()); } std::vector JStringArrayToVector(JNIEnv* env, jobjectArray array) diff --git a/Source/Core/Common/StringUtil.cpp b/Source/Core/Common/StringUtil.cpp index e3c6f692f6..6c236de587 100644 --- a/Source/Core/Common/StringUtil.cpp +++ b/Source/Core/Common/StringUtil.cpp @@ -5,6 +5,7 @@ #include "Common/StringUtil.h" #include +#include #include #include #include @@ -33,7 +34,6 @@ constexpr u32 CODEPAGE_SHIFT_JIS = 932; constexpr u32 CODEPAGE_WINDOWS_1252 = 1252; #else -#include #include #include #include @@ -590,6 +590,18 @@ std::string UTF16BEToUTF8(const char16_t* str, size_t max_size) #endif +std::string UTF16ToUTF8(std::u16string_view input) +{ + std::wstring_convert, char16_t> converter; + return converter.to_bytes(input.data(), input.data() + input.size()); +} + +std::u16string UTF8ToUTF16(std::string_view input) +{ + std::wstring_convert, char16_t> converter; + return converter.from_bytes(input.data(), input.data() + input.size()); +} + #ifdef HAS_STD_FILESYSTEM // This is a replacement for path::u8path, which is deprecated starting with C++20. std::filesystem::path StringToPath(std::string_view path) diff --git a/Source/Core/Common/StringUtil.h b/Source/Core/Common/StringUtil.h index 70d240dbf9..d4fc19d683 100644 --- a/Source/Core/Common/StringUtil.h +++ b/Source/Core/Common/StringUtil.h @@ -173,6 +173,8 @@ std::string SHIFTJISToUTF8(std::string_view str); std::string UTF8ToSHIFTJIS(std::string_view str); std::string WStringToUTF8(std::wstring_view str); std::string UTF16BEToUTF8(const char16_t* str, size_t max_size); // Stops at \0 +std::string UTF16ToUTF8(std::u16string_view str); +std::u16string UTF8ToUTF16(std::string_view str); #ifdef _WIN32