overlays/osk: keep dialog open in continuous mode

rpcs3/Emu/Cell/Modules/cellOskDialog.cpp

@@ -155,6 +155,67 @@ std::shared_ptr<OskDialogBase> _get_osk_dialog(bool create)
 	return osk.dlg;
 }
 
+extern bool close_osk_from_ps_button()
+{
+	const auto osk = _get_osk_dialog(false);
+	if (!osk)
+	{
+		// The OSK is not open
+		return true;
+	}
+
+	osk_info& info = g_fxo->get<osk_info>();
+
+	// We can only close the osk in separate window mode when it is hidden (continuous_mode is set to CELL_OSKDIALOG_CONTINUOUS_MODE_HIDE)
+	if (!info.use_separate_windows || osk->continuous_mode != CELL_OSKDIALOG_CONTINUOUS_MODE_HIDE)
+	{
+		cellOskDialog.warning("close_osk_from_ps_button: can't close OSK (use_separate_windows=%d, continuous_mode=%s)", info.use_separate_windows.load(), osk->continuous_mode.load());
+		return false;
+	}
+
+	std::lock_guard lock(info.text_mtx);
+
+	if (auto cb = info.osk_force_finish_callback.load())
+	{
+		bool done = false;
+		bool close_osk = false;
+
+		sysutil_register_cb([&](ppu_thread& cb_ppu) -> s32
+		{
+			cellOskDialog.notice("osk_force_finish_callback()");
+			close_osk = cb(cb_ppu);
+			cellOskDialog.notice("osk_force_finish_callback returned %d", close_osk);
+			done = true;
+			return 0;
+		});
+
+		// wait for check callback
+		while (!done && !Emu.IsStopped())
+		{
+			std::this_thread::yield();
+		}
+
+		if (!close_osk)
+		{
+			// We are not allowed to close the OSK
+			cellOskDialog.warning("close_osk_from_ps_button: can't close OSK (osk_force_finish_callback returned false)");
+			return false;
+		}
+	}
+
+	// Forcefully terminate the OSK
+	cellOskDialog.warning("close_osk_from_ps_button: Terminating the OSK ...");
+
+	osk->Close(FAKE_CELL_OSKDIALOG_CLOSE_TERMINATE);
+	osk->state = OskDialogState::Closed;
+
+	cellOskDialog.notice("close_osk_from_ps_button: sending CELL_SYSUTIL_OSKDIALOG_FINISHED");
+	sysutil_send_system_cmd(CELL_SYSUTIL_OSKDIALOG_FINISHED, 0);
+
+	return true;
+}
+
+
 error_code cellOskDialogLoadAsync(u32 container, vm::ptr<CellOskDialogParam> dialogParam, vm::ptr<CellOskDialogInputFieldInfo> inputFieldInfo)
 {
 	cellOskDialog.warning("cellOskDialogLoadAsync(container=0x%x, dialogParam=*0x%x, inputFieldInfo=*0x%x)", container, dialogParam, inputFieldInfo);
@@ -229,7 +290,7 @@ error_code cellOskDialogLoadAsync(u32 container, vm::ptr<CellOskDialogParam> dia
 		{
 		case CELL_OSKDIALOG_CLOSE_CONFIRM:
 		{
-			if (auto ccb = info.osk_confirm_callback.exchange({}))
+			if (auto ccb = info.osk_confirm_callback.load())
 			{
 				std::vector<u16> string_to_send(CELL_OSKDIALOG_STRING_SIZE);
 				atomic_t<bool> done = false;
@@ -266,7 +327,7 @@ error_code cellOskDialogLoadAsync(u32 container, vm::ptr<CellOskDialogParam> dia
 
 			if (info.use_separate_windows.load() && osk->osk_text[0] == 0)
 			{
-				cellOskDialog.warning("cellOskDialogLoadAsync: input result is CELL_OSKDIALOG_INPUT_FIELD_RESULT_NO_INPUT_TEXT");
+				cellOskDialog.warning("on_osk_close: input result is CELL_OSKDIALOG_INPUT_FIELD_RESULT_NO_INPUT_TEXT");
 				osk->osk_input_result = CELL_OSKDIALOG_INPUT_FIELD_RESULT_NO_INPUT_TEXT;
 			}
 			else
@@ -300,12 +361,16 @@ error_code cellOskDialogLoadAsync(u32 container, vm::ptr<CellOskDialogParam> dia
 			case CELL_OSKDIALOG_CLOSE_CONFIRM:
 			{
 				info.last_dialog_state = CELL_SYSUTIL_OSKDIALOG_INPUT_ENTERED;
+
+				cellOskDialog.notice("on_osk_close: sending CELL_SYSUTIL_OSKDIALOG_INPUT_ENTERED");
 				sysutil_send_system_cmd(CELL_SYSUTIL_OSKDIALOG_INPUT_ENTERED, 0);
 				break;
 			}
 			case CELL_OSKDIALOG_CLOSE_CANCEL:
 			{
 				info.last_dialog_state = CELL_SYSUTIL_OSKDIALOG_INPUT_CANCELED;
+
+				cellOskDialog.notice("on_osk_close: sending CELL_SYSUTIL_OSKDIALOG_INPUT_CANCELED");
 				sysutil_send_system_cmd(CELL_SYSUTIL_OSKDIALOG_INPUT_CANCELED, 0);
 				break;
 			}
@@ -320,19 +385,20 @@ error_code cellOskDialogLoadAsync(u32 container, vm::ptr<CellOskDialogParam> dia
 				break;
 			}
 			}
-
-			if (info.osk_continuous_mode.load() == CELL_OSKDIALOG_CONTINUOUS_MODE_HIDE)
-			{
-				sysutil_send_system_cmd(CELL_SYSUTIL_OSKDIALOG_DISPLAY_CHANGED, CELL_OSKDIALOG_DISPLAY_STATUS_HIDE);
-			}
 		}
 		else if (status != FAKE_CELL_OSKDIALOG_CLOSE_ABORT) // Handled in cellOskDialogAbort
 		{
 			info.last_dialog_state = CELL_SYSUTIL_OSKDIALOG_FINISHED;
+
+			cellOskDialog.notice("on_osk_close: sending CELL_SYSUTIL_OSKDIALOG_FINISHED");
 			sysutil_send_system_cmd(CELL_SYSUTIL_OSKDIALOG_FINISHED, 0);
 		}
 
+		// The interception status of the continuous separate window is handled differently
+		if (!keep_seperate_window_open)
+		{
 			input::SetIntercepted(false);
+		}
 	};
 
 	// Set key callback
@@ -514,6 +580,7 @@ error_code cellOskDialogLoadAsync(u32 container, vm::ptr<CellOskDialogParam> dia
 	// Set device mask and event lock
 	osk->ignore_input_events = info.lock_ext_input.load();
 	osk->input_device = info.initial_input_device.load();
+	osk->continuous_mode = info.osk_continuous_mode.load();
 
 	if (info.use_separate_windows)
 	{
@@ -521,15 +588,10 @@ error_code cellOskDialogLoadAsync(u32 container, vm::ptr<CellOskDialogParam> dia
 		osk->mouse_input_enabled = (info.device_mask != CELL_OSKDIALOG_DEVICE_MASK_PAD);
 	}
 
-	if (info.osk_continuous_mode == CELL_OSKDIALOG_CONTINUOUS_MODE_HIDE)
-	{
-		info.last_dialog_state = CELL_SYSUTIL_OSKDIALOG_LOADED;
-		sysutil_send_system_cmd(CELL_SYSUTIL_OSKDIALOG_LOADED, 0);
-		return CELL_OK;
-	}
-
 	input::SetIntercepted(osk->pad_input_enabled, osk->keyboard_input_enabled, osk->mouse_input_enabled);
 
+	cellOskDialog.notice("cellOskDialogLoadAsync: creating OSK dialog ...");
+
 	Emu.BlockingCallFromMainThread([=, &info]()
 	{
 		osk->Create({
@@ -554,14 +616,19 @@ error_code cellOskDialogLoadAsync(u32 container, vm::ptr<CellOskDialogParam> dia
 		});
 	});
 
-	if (info.osk_continuous_mode == CELL_OSKDIALOG_CONTINUOUS_MODE_HIDE)
+	g_fxo->get<osk_info>().last_dialog_state = CELL_SYSUTIL_OSKDIALOG_LOADED;
+
+	if (info.use_separate_windows)
 	{
-		sysutil_send_system_cmd(CELL_SYSUTIL_OSKDIALOG_DISPLAY_CHANGED, CELL_OSKDIALOG_DISPLAY_STATUS_SHOW);
+		const bool visible = osk->continuous_mode != CELL_OSKDIALOG_CONTINUOUS_MODE_HIDE;
+		cellOskDialog.notice("cellOskDialogLoadAsync: sending CELL_SYSUTIL_OSKDIALOG_DISPLAY_CHANGED with %s", visible ? "CELL_OSKDIALOG_DISPLAY_STATUS_SHOW" : "CELL_OSKDIALOG_DISPLAY_STATUS_HIDE");
+		sysutil_send_system_cmd(CELL_SYSUTIL_OSKDIALOG_DISPLAY_CHANGED, visible ? CELL_OSKDIALOG_DISPLAY_STATUS_SHOW : CELL_OSKDIALOG_DISPLAY_STATUS_HIDE);
 	}
 
-	g_fxo->get<osk_info>().last_dialog_state = CELL_SYSUTIL_OSKDIALOG_LOADED;
+	cellOskDialog.notice("cellOskDialogLoadAsync: sending CELL_SYSUTIL_OSKDIALOG_LOADED");
 	sysutil_send_system_cmd(CELL_SYSUTIL_OSKDIALOG_LOADED, 0);
 
+	cellOskDialog.notice("cellOskDialogLoadAsync: created OSK dialog");
 	return CELL_OK;
 }
 
@@ -646,7 +713,15 @@ error_code getText(vm::ptr<CellOskDialogCallbackReturnParam> OutputInfo, bool is
 			info.reset();
 		}
 
+		if (keep_seperate_window_open)
+		{
+			cellOskDialog.notice("cellOskDialogUnloadAsync: terminating continuous overlay");
+			osk->Close(FAKE_CELL_OSKDIALOG_CLOSE_TERMINATE);
+		}
+
 		osk->state = OskDialogState::Unloaded;
+
+		cellOskDialog.notice("cellOskDialogUnloadAsync: sending CELL_SYSUTIL_OSKDIALOG_UNLOADED");
 		sysutil_send_system_cmd(CELL_SYSUTIL_OSKDIALOG_UNLOADED, 0);
 	}
 	else if (keep_seperate_window_open)
@@ -728,6 +803,8 @@ error_code cellOskDialogAbort()
 	}
 
 	g_fxo->get<osk_info>().last_dialog_state = CELL_SYSUTIL_OSKDIALOG_FINISHED;
+
+	cellOskDialog.notice("cellOskDialogAbort: sending CELL_SYSUTIL_OSKDIALOG_FINISHED");
 	sysutil_send_system_cmd(CELL_SYSUTIL_OSKDIALOG_FINISHED, 0);
 
 	return CELL_OK;
@@ -762,7 +839,7 @@ error_code cellOskDialogSetDeviceMask(u32 deviceMask)
 
 error_code cellOskDialogSetSeparateWindowOption(vm::ptr<CellOskDialogSeparateWindowOption> windowOption)
 {
-	cellOskDialog.todo("cellOskDialogSetSeparateWindowOption(windowOption=*0x%x)", windowOption);
+	cellOskDialog.warning("cellOskDialogSetSeparateWindowOption(windowOption=*0x%x)", windowOption);
 
 	if (!windowOption ||
 		!windowOption->inputFieldLayoutInfo ||
@@ -984,7 +1061,6 @@ error_code cellOskDialogExtSendFinishMessage(u32 /*CellOskDialogFinishReason*/ f
 		return CELL_MSGDIALOG_ERROR_DIALOG_NOT_OPENED;
 	}
 
-	// TODO: only hide the dialog if we use separate windows
 	osk->Close(finishReason);
 
 	return CELL_OK;
@@ -1172,21 +1248,23 @@ error_code cellOskDialogExtDisableHalfByteKana()
 
 error_code cellOskDialogExtRegisterForceFinishCallback(vm::ptr<cellOskDialogForceFinishCallback> pCallback)
 {
-	cellOskDialog.todo("cellOskDialogExtRegisterForceFinishCallback(pCallback=*0x%x)", pCallback);
+	cellOskDialog.warning("cellOskDialogExtRegisterForceFinishCallback(pCallback=*0x%x)", pCallback);
 
 	if (!pCallback)
 	{
 		return CELL_OSKDIALOG_ERROR_PARAM;
 	}
 
-	g_fxo->get<osk_info>().osk_force_finish_callback = pCallback;
+	osk_info& info = g_fxo->get<osk_info>();
+	std::lock_guard lock(info.text_mtx);
+	info.osk_force_finish_callback = pCallback;
 
-	// TODO: use force finish callback when the PS-Button is pressed and a System dialog shall be spawned while the OSK is loaded
+	// We use the force finish callback when the PS-Button is pressed and a System dialog shall be spawned while the OSK is loaded
-	// 1. Check if current mode is CELL_OSKDIALOG_CONTINUOUS_MODE_HIDE or (CELL_OSKDIALOG_CONTINUOUS_MODE_SHOW && hidden)
+	// 1. Check if we are in any continuous mode and the dialog is hidden
-	// 2. If one of the above is true, call osk_force_finish_callback
+	// 2. If the above is true, call osk_force_finish_callback, deny the PS-Button press otherwise
-	// 3. Check the return value of osk_force_finish_callback. If false, ignore the PS-Button press, else go to the next step
+	// 3. Check the return value of osk_force_finish_callback.
-	// 4. Close dialog etc., send CELL_SYSUTIL_OSKDIALOG_FINISHED
+	//    if false, ignore the PS-Button press,
-	// 5. After we return from the System dialog, send CELL_SYSUTIL_SYSTEM_MENU_CLOSE to notify the game that it can load the OSK and resume.
+	//    else close the dialog etc., send CELL_SYSUTIL_OSKDIALOG_FINISHED
 
 	return CELL_OK;
 }

rpcs3/Emu/Cell/Modules/cellOskDialog.h

@@ -88,6 +88,7 @@ enum CellOskDialogFinishReason
 enum CellOskDialogFinishReasonFake // Helper. Must be negative values.
 {
 	FAKE_CELL_OSKDIALOG_CLOSE_ABORT = -1,
+	FAKE_CELL_OSKDIALOG_CLOSE_TERMINATE = -2,
 };
 
 enum CellOskDialogType
@@ -304,6 +305,7 @@ public:
 	// Closes the dialog.
 	// Set status to CELL_OSKDIALOG_CLOSE_CONFIRM or CELL_OSKDIALOG_CLOSE_CANCEL for user input.
 	// Set status to -1 if closed by the game or system.
+	// Set status to -2 if terminated by the system.
 	virtual void Close(s32 status) = 0;
 	virtual ~OskDialogBase() {};
 
@@ -311,6 +313,7 @@ public:
 	std::function<void(CellOskDialogKeyMessage key_message)> on_osk_key_input_entered;
 
 	atomic_t<OskDialogState> state{ OskDialogState::Unloaded };
+	atomic_t<CellOskDialogContinuousMode> continuous_mode{ CELL_OSKDIALOG_CONTINUOUS_MODE_NONE };
 	atomic_t<CellOskDialogInputDevice> input_device{ CELL_OSKDIALOG_INPUT_DEVICE_PAD }; // The current input device.
 	atomic_t<bool> pad_input_enabled{ true };      // Determines if the OSK consumes the device's events.
 	atomic_t<bool> mouse_input_enabled{ true };    // Determines if the OSK consumes the device's events.

rpcs3/Emu/RSX/Overlays/overlay_osk.cpp

@@ -24,6 +24,25 @@ namespace rsx
 
 		void osk_dialog::Close(s32 status)
 		{
+			if (status == FAKE_CELL_OSKDIALOG_CLOSE_TERMINATE)
+			{
+				close(false, true);
+				return;
+			}
+
+			if (status != FAKE_CELL_OSKDIALOG_CLOSE_ABORT && m_use_separate_windows && continuous_mode == CELL_OSKDIALOG_CONTINUOUS_MODE_REMAIN_OPEN)
+			{
+				// Just call the on_osk_close and don't actually close the dialog.
+				if (on_osk_close)
+				{
+					Emu.CallFromMainThread([this, status]()
+					{
+						on_osk_close(status);
+					});
+				}
+				return;
+			}
+
 			fade_animation.current = color4f(1.f);
 			fade_animation.end = color4f(0.f);
 			fade_animation.duration = 0.5f;
@@ -38,8 +57,13 @@ namespace rsx
 					});
 				}
 
-				visible = false;
+				set_visible(false);
+
+				// Only really close the dialog if we aren't in the continuous separate window mode
+				if (!m_use_separate_windows || continuous_mode == CELL_OSKDIALOG_CONTINUOUS_MODE_NONE)
+				{
 					close(true, true);
+				}
 			};
 
 			fade_animation.active = true;
@@ -413,7 +437,8 @@ namespace rsx
 			}
 
 			m_update = true;
-			visible = true;
+			set_visible(continuous_mode != CELL_OSKDIALOG_CONTINUOUS_MODE_HIDE);
+
 			m_stop_input_loop = false;
 
 			fade_animation.current = color4f(0.f);
@@ -502,6 +527,32 @@ namespace rsx
 			select_cell(index, true);
 		}
 
+		void osk_dialog::set_visible(bool visible)
+		{
+			if (m_use_separate_windows)
+			{
+				if (visible && continuous_mode == CELL_OSKDIALOG_CONTINUOUS_MODE_HIDE)
+				{
+					continuous_mode = CELL_OSKDIALOG_CONTINUOUS_MODE_SHOW;
+				}
+				else if (!visible && continuous_mode == CELL_OSKDIALOG_CONTINUOUS_MODE_SHOW)
+				{
+					continuous_mode = CELL_OSKDIALOG_CONTINUOUS_MODE_HIDE;
+				}
+			}
+
+			if (this->visible != visible)
+			{
+				this->visible = visible;
+
+				if (m_use_separate_windows)
+				{
+					osk.notice("set_visible: sending CELL_SYSUTIL_OSKDIALOG_DISPLAY_CHANGED with %s", visible ? "CELL_OSKDIALOG_DISPLAY_STATUS_SHOW" : "CELL_OSKDIALOG_DISPLAY_STATUS_HIDE");
+					sysutil_send_system_cmd(CELL_SYSUTIL_OSKDIALOG_DISPLAY_CHANGED, visible ? CELL_OSKDIALOG_DISPLAY_STATUS_SHOW : CELL_OSKDIALOG_DISPLAY_STATUS_HIDE);
+				}
+			}
+		}
+
 		void osk_dialog::on_button_pressed(pad_button button_press)
 		{
 			if (!pad_input_enabled || ignore_input_events)
@@ -509,6 +560,7 @@ namespace rsx
 
 			if (input_device.exchange(CELL_OSKDIALOG_INPUT_DEVICE_PAD) != CELL_OSKDIALOG_INPUT_DEVICE_PAD)
 			{
+				osk.notice("on_button_pressed: sending CELL_SYSUTIL_OSKDIALOG_INPUT_DEVICE_CHANGED with CELL_OSKDIALOG_INPUT_DEVICE_PAD");
 				sysutil_send_system_cmd(CELL_SYSUTIL_OSKDIALOG_INPUT_DEVICE_CHANGED, CELL_OSKDIALOG_INPUT_DEVICE_PAD);
 			}
 
@@ -753,6 +805,7 @@ namespace rsx
 
 			if (input_device.exchange(CELL_OSKDIALOG_INPUT_DEVICE_KEYBOARD) != CELL_OSKDIALOG_INPUT_DEVICE_KEYBOARD)
 			{
+				osk.notice("on_key_pressed: sending CELL_SYSUTIL_OSKDIALOG_INPUT_DEVICE_CHANGED with CELL_OSKDIALOG_INPUT_DEVICE_KEYBOARD");
 				sysutil_send_system_cmd(CELL_SYSUTIL_OSKDIALOG_INPUT_DEVICE_CHANGED, CELL_OSKDIALOG_INPUT_DEVICE_KEYBOARD);
 			}
 

rpcs3/Emu/RSX/Overlays/overlay_osk.h

@@ -115,6 +115,8 @@ namespace rsx
 			void update_controls();
 			void update_selection_by_index(u32 index);
 
+			void set_visible(bool visible);
+
 			void on_button_pressed(pad_button button_press) override;
 			void on_key_pressed(u32 led, u32 mkey, u32 key_code, u32 out_key_code, bool pressed, std::u32string key) override;
 			void on_text_changed();

rpcs3/Emu/localized_string_id.h

@@ -114,6 +114,7 @@ enum class localized_string_id
 	CELL_MSG_DIALOG_ERROR_8001003E,
 
 	CELL_OSK_DIALOG_TITLE,
+	CELL_OSK_DIALOG_BUSY,
 
 	CELL_SAVEDATA_CB_BROKEN,
 	CELL_SAVEDATA_CB_FAILURE,

rpcs3/Input/pad_thread.cpp

@@ -406,7 +406,9 @@ void pad_thread::operator()()
 		// Handle home menu if requested
 		if (!is_vsh && !m_home_menu_open && Emu.IsRunning())
 		{
-			for (usz i = 0; i < m_pads.size(); i++)
+			bool ps_button_pressed = false;
+
+			for (usz i = 0; i < m_pads.size() && !ps_button_pressed; i++)
 			{
 				const auto& pad = m_pads[i];
 
@@ -416,12 +418,19 @@ void pad_thread::operator()()
 				for (const auto& button : pad->m_buttons)
 				{
 					if (button.m_offset == CELL_PAD_BTN_OFFSET_DIGITAL2 && button.m_outKeyCode == CELL_PAD_CTRL_PS && button.m_pressed)
+					{
+						// Make sure we call this function only once per button press
+						if (!m_ps_button_pressed)
 						{
 							open_home_menu();
+						}
+						ps_button_pressed = true;
 						break;
 					}
 				}
 			}
+
+			m_ps_button_pressed = ps_button_pressed;
 		}
 
 		// Handle paused emulation (if triggered by home menu).
@@ -619,9 +628,17 @@ void pad_thread::InitPadConfig(cfg_pad& cfg, pad_handler type, std::shared_ptr<P
 
 extern bool send_open_home_menu_cmds();
 extern void send_close_home_menu_cmds();
+extern bool close_osk_from_ps_button();
 
 void pad_thread::open_home_menu()
 {
+	// Check if the OSK is open and can be closed
+	if (!close_osk_from_ps_button())
+	{
+		rsx::overlays::queue_message(get_localized_string(localized_string_id::CELL_OSK_DIALOG_BUSY));
+		return;
+	}
+
 	if (auto manager = g_fxo->try_get<rsx::overlays::display_manager>())
 	{
 		if (m_home_menu_open.exchange(true))

rpcs3/Input/pad_thread.h

@@ -59,6 +59,7 @@ private:
 	u32 m_mask_start_press_to_resume = 0;
 	u64 m_track_start_press_begin_timestamp = 0;
 	bool m_resume_emulation_flag = false;
+	bool m_ps_button_pressed = false;
 	atomic_t<bool> m_home_menu_open = false;
 };
 

rpcs3/rpcs3qt/localized_emu.h

@@ -139,6 +139,7 @@ private:
 		case localized_string_id::CELL_MSG_DIALOG_ERROR_8001003E: return tr("Pointer is null.\n(%0)", "Error code").arg(std::forward<Args>(args)...);
 		case localized_string_id::CELL_MSG_DIALOG_ERROR_DEFAULT: return tr("An error has occurred.\n(%0)", "Error code").arg(std::forward<Args>(args)...);
 		case localized_string_id::CELL_OSK_DIALOG_TITLE: return tr("On Screen Keyboard", "OSK Dialog");
+		case localized_string_id::CELL_OSK_DIALOG_BUSY: return tr("The Home Menu can't be opened while the On Screen Keyboard is busy!", "OSK Dialog");
 		case localized_string_id::CELL_SAVEDATA_CB_BROKEN: return tr("Error - Save data corrupted", "Savedata Error");
 		case localized_string_id::CELL_SAVEDATA_CB_FAILURE: return tr("Error - Failed to save or load", "Savedata Error");
 		case localized_string_id::CELL_SAVEDATA_CB_NO_DATA: return tr("Error - Save data cannot be found", "Savedata Error");