Merge pull request #10401 from libretro/accessibility_menu_fixes

Accessibility menu fixes

Makefile.common

@@ -19,6 +19,16 @@ ifeq ($(HAVE_CXX11), 1)
    CXXFLAGS += $(CXX11_CFLAGS)
 endif
 
+ifeq ($(HAVE_NVDA), 1)
+   LIBS += nvdaControllerClient64.dll
+endif
+
+ifeq ($(HAVE_SAPI), 1)
+   LIBS += sapi.dll
+endif
+
+
+
 ifeq ($(HAVE_GL_CONTEXT),)
    HAVE_GL_CONTEXT = 0
    HAVE_GL_MODERN  = 0
@@ -1765,7 +1775,13 @@ endif
 # Accessibility
 ifeq ($(HAVE_ACCESSIBILITY), 1)
    DEFINES += -DHAVE_ACCESSIBILITY
+
+ifneq ($(findstring Win32,$(OS)),)
+   LIBS += -lsapi
 endif
+endif
+
+
 
 # Things that depend on network availability
 

frontend/drivers/platform_win32.c

@@ -51,6 +51,10 @@
 #include "../../msg_hash.h"
 #include "platform_win32.h"
 
+#ifdef HAVE_NVDA
+#include "../../nvdaController.h"
+#endif
+
 #ifndef SM_SERVERR2
 #define SM_SERVERR2 89
 #endif
@@ -717,6 +721,74 @@ static bool frontend_win32_set_fork(enum frontend_fork fork_mode)
 #endif
 
 #if defined(_WIN32) && !defined(_XBOX)
+static const char *accessibility_win_language_id(const char* language)
+{
+   if (string_is_equal(language,"en"))
+      return "409";
+   else if (string_is_equal(language,"it"))
+      return "410";
+   else if (string_is_equal(language,"sv"))
+      return "041d";
+   else if (string_is_equal(language,"fr"))
+      return "040c";
+   else if (string_is_equal(language,"de"))
+      return "407";
+   else if (string_is_equal(language,"he"))
+      return "040d";
+   else if (string_is_equal(language,"id"))
+      return "421";
+   else if (string_is_equal(language,"es"))
+      return "040a";
+   else if (string_is_equal(language,"nl"))
+      return "413";
+   else if (string_is_equal(language,"ro"))
+      return "418";
+   else if (string_is_equal(language,"pt_pt"))
+      return "816";
+   else if (string_is_equal(language,"pt_bt") || string_is_equal(language,"pt"))
+      return "416";
+   else if (string_is_equal(language,"th"))
+      return "041e";
+   else if (string_is_equal(language,"ja"))
+      return "411";
+   else if (string_is_equal(language,"sk"))
+      return "041b";
+   else if (string_is_equal(language,"hi"))
+      return "439";
+   else if (string_is_equal(language,"ar"))
+      return "401";
+   else if (string_is_equal(language,"hu"))
+      return "040e";
+   else if (string_is_equal(language,"zh_tw") || string_is_equal(language,"zh"))
+      return "804";
+   else if (string_is_equal(language,"el"))
+      return "408";
+   else if (string_is_equal(language,"ru"))
+      return "419";
+   else if (string_is_equal(language,"nb"))
+      return "414";
+   else if (string_is_equal(language,"da"))
+      return "406";
+   else if (string_is_equal(language,"fi"))
+      return "040b";
+   else if (string_is_equal(language,"zh_hk"))
+      return "0c04";
+   else if (string_is_equal(language,"zh_cn"))
+      return "804";
+   else if (string_is_equal(language,"tr"))
+      return "041f";
+   else if (string_is_equal(language,"ko"))
+      return "412";
+   else if (string_is_equal(language,"pl"))
+      return "415";
+   else if (string_is_equal(language,"cs")) 
+      return "405";
+   else
+      return "";
+
+
+}
+
 static const char *accessibility_win_language_code(const char* language)
 {
    if (string_is_equal(language,"en"))
@@ -730,9 +802,9 @@ static const char *accessibility_win_language_code(const char* language)
    else if (string_is_equal(language,"de"))
       return "Microsoft Stefan Desktop";
    else if (string_is_equal(language,"he"))
-      return "Microsoft Hemant Desktop";
-   else if (string_is_equal(language,"id"))
       return "Microsoft Asaf Desktop";
+   else if (string_is_equal(language,"id"))
+      return "Microsoft Andika Desktop";
    else if (string_is_equal(language,"es"))
       return "Microsoft Pablo Desktop";
    else if (string_is_equal(language,"nl"))
@@ -806,13 +878,72 @@ static bool create_win32_process(char* cmd)
    return true;
 }
 
+#ifdef HAVE_SAPI
+#define COBJMACROS
+#include <sapi.h>
+#include <ole2.h>
+#endif
+
+static ISpVoice* pVoice = NULL;
+#ifdef HAVE_NVDA
+bool USE_POWERSHELL = false;
+bool USE_NVDA = true;
+#else
+bool USE_POWERSHELL = true;
+bool USE_NVDA = false;
+#endif
+bool USE_NVDA_BRAILLE = false;
+
 static bool is_narrator_running_windows(void)
 {
    DWORD status = 0;
-   if (pi_set == false)
+   bool res;
+   if (USE_POWERSHELL)
+   {
+      if (pi_set == false)
+         return false;
+      if (GetExitCodeProcess(g_pi.hProcess, &status))
+      {
+         if (status == STILL_ACTIVE)
+            return true;
+      }
       return false;
-   if (GetExitCodeProcess(&g_pi, &status) && status == STILL_ACTIVE)
+   }
-      return true;
+#ifdef HAVE_NVDA
+   else if (USE_NVDA)
+   {
+      long res=nvdaController_testIfRunning();
+      if(res!=0) 
+      {
+         /* The running nvda service wasn't found, so revert
+            back to the powershell method
+         */
+         RARCH_LOG("Error communicating with NVDA\n");
+         USE_POWERSHELL = true;
+         USE_NVDA = false;
+	 return false;
+      }
+      return false;
+      /*
+      nvdaController_speakText(L"This is a test speech message");
+      nvdaController_brailleMessage(L"This is a test braille message");
+      */
+   }
+#endif
+#ifdef HAVE_SAPI
+   else
+   {
+      SPVOICESTATUS pStatus;
+      if (pVoice != NULL)
+      {
+         ISpVoice_GetStatus(pVoice, &pStatus, NULL);
+         if (pStatus.dwRunningState == SPRS_IS_SPEAKING)
+            return true;
+         else
+            return false;
+      }
+   }
+#endif
    return false;
 }
 
@@ -822,9 +953,12 @@ static bool accessibility_speak_windows(int speed,
    char cmd[1200];
    const char *voice      = get_user_language_iso639_1(true);
    const char *language   = accessibility_win_language_code(voice);
+   const char *langid     = accessibility_win_language_id(voice);
    bool res               = false;
    const char* speeds[10] = {"-10", "-7.5", "-5", "-2.5", "0", "2", "4", "6", "8", "10"};
 
+   HRESULT hr;
+
    if (speed < 1)
       speed = 1;
    else if (speed > 10)
@@ -836,22 +970,80 @@ static bool accessibility_speak_windows(int speed,
          return true;
    }
 
-   if (strlen(language) > 0) 
+   if (USE_POWERSHELL)
-      snprintf(cmd, sizeof(cmd),
-               "powershell.exe -NoProfile -WindowStyle Hidden -Command \"Add-Type -AssemblyName System.Speech; $synth = New-Object System.Speech.Synthesis.SpeechSynthesizer; $synth.SelectVoice(\\\"%s\\\"); $synth.Rate = %s; $synth.Speak(\\\"%s\\\");\"", language, speeds[speed-1], (char*) speak_text); 
-   else
-      snprintf(cmd, sizeof(cmd),
-               "powershell.exe -NoProfile -WindowStyle Hidden -Command \"Add-Type -AssemblyName System.Speech; $synth = New-Object System.Speech.Synthesis.SpeechSynthesizer; $synth.Rate = %s; $synth.Speak(\\\"%s\\\");\"", speeds[speed-1], (char*) speak_text); 
-   if (pi_set)
-      terminate_win32_process(g_pi);
-   res = create_win32_process(cmd);
-   if (!res)
    {
-      pi_set = false;
+      if (strlen(language) > 0) 
+         snprintf(cmd, sizeof(cmd),
+                  "powershell.exe -NoProfile -WindowStyle Hidden -Command \"Add-Type -AssemblyName System.Speech; $synth = New-Object System.Speech.Synthesis.SpeechSynthesizer; $synth.SelectVoice(\\\"%s\\\"); $synth.Rate = %s; $synth.Speak(\\\"%s\\\");\"", language, speeds[speed-1], (char*) speak_text); 
+      else
+         snprintf(cmd, sizeof(cmd),
+               "powershell.exe -NoProfile -WindowStyle Hidden -Command \"Add-Type -AssemblyName System.Speech; $synth = New-Object System.Speech.Synthesis.SpeechSynthesizer; $synth.Rate = %s; $synth.Speak(\\\"%s\\\");\"", speeds[speed-1], (char*) speak_text); 
+      if (pi_set)
+         terminate_win32_process(g_pi);
+      res = create_win32_process(cmd);
+      if (!res)
+      {
+         pi_set = false;
+         return true;
+      }
+      pi_set = true;
       return true;
    }
-   pi_set = true;
+#ifdef HAVE_NVDA
-   return true;
+   else if (USE_NVDA)
+   {
+      long res=nvdaController_testIfRunning();
+      const size_t cSize = strlen(speak_text)+1;
+      wchar_t* wc = malloc(sizeof(wchar_t)*cSize);
+      mbstowcs(wc, speak_text, cSize);
+
+      if(res!=0) 
+      {
+         RARCH_LOG("Error communicating with NVDA\n");
+         return false;
+      }
+      else
+      {
+         nvdaController_cancelSpeech();
+      }
+
+      if (USE_NVDA_BRAILLE)
+         nvdaController_brailleMessage(wc);
+      else
+      {
+         nvdaController_speakText(wc);
+      }
+      return true;
+   }
+#endif
+#ifdef HAVE_SAPI
+   else
+   {
+      /* stop the old voice if running */
+      if (pVoice != NULL)
+      {
+         CoUninitialize();
+         ISpVoice_Release(pVoice);
+      }
+      pVoice = NULL;
+
+      /* Play the new voice */
+      if (FAILED(CoInitialize(NULL)))
+         return NULL;
+
+      hr = CoCreateInstance(&CLSID_SpVoice, NULL, CLSCTX_ALL, &IID_ISpVoice, (void **)&pVoice);
+      if (SUCCEEDED(hr))
+      {
+         wchar_t wtext[1200];
+         snprintf(cmd, sizeof(cmd),
+                  "<rate speed=\"%s\"/><volume level=\"80\"/><lang langid=\"%s\"/>%s", speeds[speed], langid, speak_text);
+         mbstowcs(wtext, speak_text, sizeof(wtext));
+ 
+         hr = ISpVoice_Speak(pVoice, wtext, SPF_ASYNC /*SVSFlagsAsync*/, NULL);
+      }
+      return true;
+   }
+#endif
 }
 #endif
 

input/input_mapper.c

@@ -40,6 +40,7 @@
 #endif
 
 #include "../configuration.h"
+#include "../retroarch.h"
 
 void input_mapper_poll(input_mapper_t *handle,
       void *ol_pointer,
@@ -124,6 +125,12 @@ void input_mapper_poll(input_mapper_t *handle,
                remap_valid                   = (current_button_value == 1) &&
                   (j != remap_button) && (remap_button != RARCH_UNMAPPED);
 
+               /* gamepad override */
+               if (i==0 && get_gamepad_input_override() & (1<<j))
+               {
+                  BIT256_SET(handle->buttons[i], j);
+               }
+
                if (remap_valid)
                {
                   if (remap_button < RARCH_FIRST_CUSTOM_BIND)

menu/drivers/materialui.c

@@ -6813,7 +6813,7 @@ static enum menu_action materialui_parse_menu_entry_action(
              (materialui_list_get_size(mui, MENU_LIST_PLAIN) == 1))
          {
             materialui_switch_tabs(mui, NULL, action);
-            new_action = MENU_ACTION_NOOP;
+            new_action = MENU_ACTION_ACCESSIBILITY_SPEAK_TITLE_LABEL;
          }
          else
          {

menu/drivers/ozone/ozone.c

@@ -3287,7 +3287,7 @@ static enum menu_action ozone_parse_menu_entry_action(
 
             ozone_sidebar_goto(ozone, new_selection);
 
-            new_action = MENU_ACTION_NOOP;
+            new_action = MENU_ACTION_ACCESSIBILITY_SPEAK_TITLE;
             ozone->cursor_mode = false;
             break;
          }
@@ -3314,7 +3314,7 @@ static enum menu_action ozone_parse_menu_entry_action(
 
             ozone_sidebar_goto(ozone, new_selection);
 
-            new_action = MENU_ACTION_NOOP;
+            new_action = MENU_ACTION_ACCESSIBILITY_SPEAK_TITLE;
             ozone->cursor_mode = false;
             break;
          }
@@ -3330,7 +3330,7 @@ static enum menu_action ozone_parse_menu_entry_action(
          ozone->cursor_mode = false;
          if (ozone->cursor_in_sidebar)
          {
-            new_action = MENU_ACTION_NOOP;
+            new_action = MENU_ACTION_ACCESSIBILITY_SPEAK_TITLE;
             break;
          }
          else if (ozone->depth > 1)
@@ -3338,7 +3338,7 @@ static enum menu_action ozone_parse_menu_entry_action(
 
          ozone_go_to_sidebar(ozone, tag);
 
-         new_action = MENU_ACTION_NOOP;
+         new_action = MENU_ACTION_ACCESSIBILITY_SPEAK_TITLE;
          break;
       case MENU_ACTION_RIGHT:
          ozone->cursor_mode = false;
@@ -3351,14 +3351,14 @@ static enum menu_action ozone_parse_menu_entry_action(
 
          ozone_leave_sidebar(ozone, tag);
 
-         new_action = MENU_ACTION_NOOP;
+         new_action = MENU_ACTION_ACCESSIBILITY_SPEAK_LABEL;
          break;
       case MENU_ACTION_OK:
          ozone->cursor_mode = false;
          if (ozone->cursor_in_sidebar)
          {
             ozone_leave_sidebar(ozone, tag);
-            new_action = MENU_ACTION_NOOP;
+            new_action = MENU_ACTION_ACCESSIBILITY_SPEAK_LABEL;
             break;
          }
          break;
@@ -3370,14 +3370,14 @@ static enum menu_action ozone_parse_menu_entry_action(
             if (ozone->categories_selection_ptr != 0)
                ozone_sidebar_goto(ozone, 0);
 
-            new_action = MENU_ACTION_NOOP;
+            new_action = MENU_ACTION_ACCESSIBILITY_SPEAK_TITLE;
             break;
          }
 
          if (menu_entries_get_stack_size(0) == 1)
          {
             ozone_go_to_sidebar(ozone, tag);
-            new_action = MENU_ACTION_NOOP;
+            new_action = MENU_ACTION_ACCESSIBILITY_SPEAK_TITLE;
          }
          break;
 
@@ -3387,7 +3387,7 @@ static enum menu_action ozone_parse_menu_entry_action(
          /* Ignore if cursor is in sidebar */
          if (ozone->cursor_in_sidebar)
          {
-            new_action = MENU_ACTION_NOOP;
+            new_action = MENU_ACTION_ACCESSIBILITY_SPEAK_TITLE;
             break;
          }
 
@@ -3404,7 +3404,7 @@ static enum menu_action ozone_parse_menu_entry_action(
          /* > Ignore if cursor is in sidebar */
          if (ozone->cursor_in_sidebar)
          {
-            new_action = MENU_ACTION_NOOP;
+            new_action = MENU_ACTION_ACCESSIBILITY_SPEAK_TITLE;
             break;
          }
 

menu/menu_defines.h

@@ -340,7 +340,10 @@ enum menu_action
    MENU_ACTION_SCROLL_UP,
    MENU_ACTION_TOGGLE,
    MENU_ACTION_POINTER_MOVED,
-   MENU_ACTION_POINTER_PRESSED
+   MENU_ACTION_POINTER_PRESSED,
+   MENU_ACTION_ACCESSIBILITY_SPEAK_TITLE,
+   MENU_ACTION_ACCESSIBILITY_SPEAK_LABEL,
+   MENU_ACTION_ACCESSIBILITY_SPEAK_TITLE_LABEL
 };
 
 enum playlist_inline_core_display_type

menu/menu_driver.c

@@ -459,11 +459,23 @@ int generic_menu_entry_action(
       {
          case MENU_ACTION_INFO:
             break;
+         case MENU_ACTION_ACCESSIBILITY_SPEAK_TITLE:
+            menu_entries_get_title(title_name, sizeof(title_name));
+            break;
+         case MENU_ACTION_ACCESSIBILITY_SPEAK_LABEL:
+            get_current_menu_label(current_label, sizeof(current_label));
+            break;
+         case MENU_ACTION_ACCESSIBILITY_SPEAK_TITLE_LABEL:
+            menu_entries_get_title(title_name, sizeof(title_name));
+            get_current_menu_label(current_label, sizeof(current_label));
+            break;
          case MENU_ACTION_OK:
          case MENU_ACTION_LEFT:
          case MENU_ACTION_RIGHT:
          case MENU_ACTION_CANCEL:
             menu_entries_get_title(title_name, sizeof(title_name));
+            get_current_menu_label(current_label, sizeof(current_label));
+            break;
          case MENU_ACTION_UP:
          case MENU_ACTION_DOWN:
          case MENU_ACTION_SCROLL_UP:
@@ -471,9 +483,16 @@ int generic_menu_entry_action(
             get_current_menu_label(current_label, sizeof(current_label));
             break;
          case MENU_ACTION_START:
+            if (!string_is_equal(current_value, "..."))
+            {
+               menu_entries_get_title(title_name, sizeof(title_name));
+               get_current_menu_label(current_label, sizeof(current_label));
+            }
+            break;
          case MENU_ACTION_SELECT:
          case MENU_ACTION_SEARCH:
             get_current_menu_label(current_label, sizeof(current_label));
+            break;
          case MENU_ACTION_SCAN:
          default:
             break;

nvdaController.h

@@ -0,0 +1,80 @@
+/* this ALWAYS GENERATED file contains the definitions for the interfaces */
+
+
+ /* File created by MIDL compiler version 7.00.0555 */
+/* at Fri Feb 19 11:21:40 2010
+ */
+/* Compiler settings for interfaces\nvdaController\nvdaController.idl, interfaces\nvdaController\nvdaController.acf:
+    Oicf, W1, Zp8, env=Win64 (32b run), target_arch=AMD64 7.00.0555 
+    protocol : dce , ms_ext, c_ext, robust
+    error checks: allocation ref bounds_check enum stub_data 
+    VC __declspec() decoration level: 
+         __declspec(uuid()), __declspec(selectany), __declspec(novtable)
+         DECLSPEC_UUID(), MIDL_INTERFACE()
+*/
+/* @@MIDL_FILE_HEADING(  ) */
+
+#pragma warning( disable: 4049 )  /* more than 64k source lines */
+
+
+/* verify that the <rpcndr.h> version is high enough to compile this file*/
+#ifndef __REQUIRED_RPCNDR_H_VERSION__
+#define __REQUIRED_RPCNDR_H_VERSION__ 475
+#endif
+
+#include "rpc.h"
+#include "rpcndr.h"
+
+#ifndef __RPCNDR_H_VERSION__
+#error this stub requires an updated version of <rpcndr.h>
+#endif // __RPCNDR_H_VERSION__
+
+
+#ifndef __nvdaController_h__
+#define __nvdaController_h__
+
+#if defined(_MSC_VER) && (_MSC_VER >= 1020)
+#pragma once
+#endif
+
+/* Forward Declarations */ 
+
+#ifdef __cplusplus
+extern "C"{
+#endif 
+
+
+#ifndef __NvdaController_INTERFACE_DEFINED__
+#define __NvdaController_INTERFACE_DEFINED__
+
+/* interface NvdaController */
+/* [implicit_handle][version][uuid] */ 
+
+/* [comm_status][fault_status] */ error_status_t __stdcall nvdaController_testIfRunning( void);
+
+/* [comm_status][fault_status] */ error_status_t __stdcall nvdaController_speakText( 
+    /* [string][in] */ const wchar_t *text);
+
+/* [comm_status][fault_status] */ error_status_t __stdcall nvdaController_cancelSpeech( void);
+
+/* [comm_status][fault_status] */ error_status_t __stdcall nvdaController_brailleMessage( 
+    /* [string][in] */ const wchar_t *message);
+
+
+extern handle_t nvdaControllerBindingHandle;
+
+
+extern RPC_IF_HANDLE nvdaController_NvdaController_v1_0_c_ifspec;
+extern RPC_IF_HANDLE NvdaController_v1_0_c_ifspec;
+extern RPC_IF_HANDLE nvdaController_NvdaController_v1_0_s_ifspec;
+#endif /* __NvdaController_INTERFACE_DEFINED__ */
+
+/* Additional Prototypes for ALL interfaces */
+
+/* end of Additional Prototypes */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

qb/config.params.sh

@@ -1,5 +1,7 @@
 HAVE_LIBRETRO=             # Libretro library used
 HAVE_ASSETS_DIR=           # Assets install directory
+HAVE_NVDA=no               # NVDA support
+HAVE_SAPI=no               # SAPI support
 HAVE_BLISSBOX=auto         # Blissbox support
 HAVE_ANGLE=no              # ANGLE support (OpenGL wrapper)
 HAVE_CONFIGFILE=yes        # Config file support

retroarch.c

@@ -1297,6 +1297,7 @@ static const void *hid_driver_find_handle(int idx);
 #ifdef HAVE_ACCESSIBILITY
 #ifdef HAVE_TRANSLATE
 static bool is_narrator_running(void);
+int ai_gamepad_state[16];
 #endif
 static bool accessibility_startup_message(void);
 #endif
@@ -3895,7 +3896,7 @@ static bool command_get_status(const char* arg)
 
        core_info_get_current_core(&core_info);
 
-	   if (runloop_paused)
+       if (runloop_paused)
           status                = "PAUSED";
        if (core_info)
           system_id             = core_info->system_id;
@@ -6013,6 +6014,122 @@ error:
 
 /* TRANSLATION */
 #ifdef HAVE_TRANSLATE
+static int g_ai_service_auto = 0;
+
+int get_ai_service_auto(void)
+{
+   return g_ai_service_auto;
+}
+
+bool set_ai_service_auto(int num)
+{
+   g_ai_service_auto = num;
+   return true;
+}
+
+bool task_auto_translate_callback()
+{
+   bool was_paused                   = runloop_paused;
+   command_event(CMD_EVENT_AI_SERVICE_CALL, &was_paused);
+   return true;
+}
+
+
+/* Doesn't currently work.  Fix this. */
+bool is_ai_service_speech_running(void)
+{
+#ifdef HAVE_AUDIOMIXER
+   enum audio_mixer_state res = audio_driver_mixer_get_stream_state(10);
+   if (res == AUDIO_STREAM_STATE_NONE || res == AUDIO_STREAM_STATE_STOPPED)
+      return false;
+   return true;
+#else
+   return false;
+#endif
+}
+
+bool ai_service_speech_stop(void)
+{
+#ifdef HAVE_AUDIOMIXER
+   audio_driver_mixer_stop_stream(10);
+   audio_driver_mixer_remove_stream(10);
+#endif
+   return false;
+}
+
+
+static void task_auto_translate_handler(retro_task_t *task)
+{
+   http_transfer_data_t *data = NULL;
+   int* mode_ptr = task->user_data;
+
+   if (task_get_cancelled(task))
+      goto task_finished;
+   /* Narrator Mode */
+   if (*mode_ptr == 2)
+   {
+#ifdef HAVE_ACCESSIBILITY
+      if (is_narrator_running() == false)
+      {
+         goto task_finished;
+      }
+#endif
+   }
+   /* Speech Mode */
+   else if (*mode_ptr == 1)
+   {
+#ifdef HAVE_AUDIOMIXER
+      if (is_ai_service_speech_running() == false)
+      {
+         goto task_finished;
+      }
+#endif
+   }
+   return;
+task_finished:
+   if (get_ai_service_auto() == 1)
+      set_ai_service_auto(2);
+
+   task_set_finished(task, true);
+   if (*mode_ptr == 1 || *mode_ptr == 2)
+       task_auto_translate_callback();
+   if (task->user_data)
+       free(task->user_data);
+}
+
+bool call_auto_translate_task(bool* was_paused)
+{
+   settings_t *settings                  = configuration_settings;
+   int ai_service_mode                   = settings->uints.ai_service_mode;
+
+   /*Image Mode*/
+   if (ai_service_mode == 0)
+   {
+      if (get_ai_service_auto() == 1)
+         set_ai_service_auto(2);
+
+      command_event(CMD_EVENT_AI_SERVICE_CALL, was_paused);
+      return true;
+   }
+   else /* Speech or Narrator Mode */
+   {
+      retro_task_t  *t                   = NULL;
+      int* mode                          = (int*) malloc(sizeof(int));
+      *mode = ai_service_mode;
+      t = task_init();
+      if (!t)
+         return false;
+
+      t->handler = task_auto_translate_handler;
+      t->user_data = mode;
+      t->mute = true;
+      task_queue_push(t);
+   }
+   return true;
+}
+
+
+
 static void handle_translation_cb(
       retro_task_t *task, void *task_data, void *user_data, const char *error)
 {
@@ -6039,14 +6156,31 @@ static void handle_translation_cb(
    char* found_string                = NULL;
    char* error_string                = NULL;
    char* text_string                 = NULL;
+   char* auto_string                 = NULL;
+   char* key_string                  = NULL;
    int curr_state                    = 0;
+   settings_t* settings              = configuration_settings;
+   bool was_paused                   = runloop_paused;
+
+
+#ifdef GFX_MENU_WIDGETS
+   if (gfx_widgets_ai_service_overlay_get_state() != 0 
+       && get_ai_service_auto() == 2)
+   {
+      /* When auto mode is on, we turn off the overlay
+       * once we have the result for the next call.*/
+      gfx_widgets_ai_service_overlay_unload();
+   }
+#endif
 
 #ifdef DEBUG
-   RARCH_LOG("RESULT FROM AI SERVICE...\n");
+   if (get_ai_service_auto() != 2)
+      RARCH_LOG("RESULT FROM AI SERVICE...\n");
 #endif
+
    if (!data || error)
       goto finish;
-
+   
    data->data = (char*)realloc(data->data, data->len + 1);
    if (!data->data)
       goto finish;
@@ -6096,6 +6230,18 @@ static void handle_translation_cb(
                strlcpy(error_string, body_copy+start+1, i-start);
                curr_state = 0;
             }
+            else if (curr_state == 5)
+            {
+               auto_string = (char*)malloc(i-start+1);
+               strlcpy(auto_string, body_copy+start+1, i-start);
+               curr_state = 0;
+            }
+            else if (curr_state == 6)
+            {
+               key_string = (char*)malloc(i-start+1);
+               strlcpy(key_string, body_copy+start+1, i-start);
+               curr_state = 0;
+            }
             else if (string_is_equal(found_string, "image"))
             {
                curr_state = 1;
@@ -6116,6 +6262,16 @@ static void handle_translation_cb(
                curr_state = 4;
                free(found_string);
             }
+            else if (string_is_equal(found_string, "auto"))
+            {
+               curr_state = 5;
+               free(found_string);
+            }
+            else if (string_is_equal(found_string, "press"))
+            {
+               curr_state = 6;
+               free(found_string);
+            }
             else
             {
               curr_state = 0;
@@ -6146,7 +6302,7 @@ static void handle_translation_cb(
 #endif
    }
 
-   if (!raw_image_file_data && !raw_sound_data && !text_string)
+   if (!raw_image_file_data && !raw_sound_data && !text_string && get_ai_service_auto() != 2 && !key_string)
    {
       error = "Invalid JSON body.";
       goto finish;
@@ -6379,6 +6535,76 @@ static void handle_translation_cb(
    }
 #endif
 
+   if (key_string)
+   {
+      int length = strlen(key_string);
+      int i = 0;
+      int start = 0;
+      char t = ' ';
+      char key[8];
+
+      for (i=1;i<length;i++)
+      {
+         t = key_string[i];
+         if (i == length-1 || t == ' ' || t == ',')
+         {
+            if (i == length-1 && t != ' ' && t!= ',')
+               i++;
+
+            if (i-start > 7)
+            {
+               start = i;
+               continue;
+            }
+            
+            strncpy(key, key_string+start, i-start);
+            key[i-start] = '\0';
+            if (string_is_equal(key, "b"))
+               ai_gamepad_state[0] = 2;
+            if (string_is_equal(key, "y"))
+               ai_gamepad_state[1] = 2;
+            if (string_is_equal(key, "select"))
+               ai_gamepad_state[2] = 2;
+            if (string_is_equal(key, "start"))
+               ai_gamepad_state[3] = 2;
+
+            if (string_is_equal(key, "up"))
+               ai_gamepad_state[4] = 2;
+            if (string_is_equal(key, "down"))
+               ai_gamepad_state[5] = 2;
+            if (string_is_equal(key, "left"))
+               ai_gamepad_state[6] = 2;
+            if (string_is_equal(key, "right"))
+               ai_gamepad_state[7] = 2;
+
+            if (string_is_equal(key, "a"))
+               ai_gamepad_state[8] = 2;
+            if (string_is_equal(key, "x"))
+               ai_gamepad_state[9] = 2;
+            if (string_is_equal(key, "l"))
+               ai_gamepad_state[10] = 2;
+            if (string_is_equal(key, "r"))
+               ai_gamepad_state[11] = 2;
+
+            if (string_is_equal(key, "l2"))
+               ai_gamepad_state[12] = 2;
+            if (string_is_equal(key, "r2"))
+               ai_gamepad_state[13] = 2;
+            if (string_is_equal(key, "l3"))
+               ai_gamepad_state[14] = 2;
+            if (string_is_equal(key, "r3"))
+               ai_gamepad_state[15] = 2;
+
+            if (string_is_equal(key, "pause"))
+               command_event(CMD_EVENT_PAUSE, NULL);
+            if (string_is_equal(key, "unpause"))
+               command_event(CMD_EVENT_UNPAUSE, NULL);
+
+            start = i+1;
+         }
+      }
+   }
+
 #ifdef HAVE_ACCESSIBILITY
    if (text_string && is_accessibility_enabled())
       accessibility_speak_priority(text_string, 10);
@@ -6413,25 +6639,18 @@ finish:
       free(text_string);
    if (raw_output_data)
       free(raw_output_data);
-}
 
-static bool is_ai_service_speech_running(void)
+   if (string_is_equal(auto_string, "auto"))
-{
+   {
-#ifdef HAVE_AUDIOMIXER
+      if (get_ai_service_auto() != 0 && settings->bools.ai_service_pause == false)
-   enum audio_mixer_state res = audio_driver_mixer_get_stream_state(10);
+      { 
-   if (res != AUDIO_STREAM_STATE_NONE && res != AUDIO_STREAM_STATE_STOPPED)
+         call_auto_translate_task(&was_paused);
-      return true;
+      }
-#endif
+   }
-   return false;
+   if (auto_string)
-}
+      free(auto_string);
-
+   if (key_string)
-static bool ai_service_speech_stop(void)
+      free(key_string);
-{
-#ifdef HAVE_AUDIOMIXER
-   audio_driver_mixer_stop_stream(10);
-   audio_driver_mixer_remove_stream(10);
-#endif
-   return false;
 }
 
 static const char *ai_service_get_str(enum translation_lang id)
@@ -6572,6 +6791,7 @@ static const char *ai_service_get_str(enum translation_lang id)
    return "";
 }
 
+
 /*
    This function does all the stuff needed to translate the game screen,
    using the URL given in the settings.  Once the image from the frame
@@ -6598,8 +6818,17 @@ static const char *ai_service_get_str(enum translation_lang id)
    The server must output the translated image in the form of a
    JSON body, with the "image" field also as a base64 encoded
    24bit-BMP, or as an alpha channel png.
-   */
+
-static bool run_translation_service(void)
+  "paused" boolean is passed in to indicate if the current call
+   was made during a paused frame.  Due to how the menu widgets work,
+   if the ai service is called in "auto" mode, then this call will
+   be made while the menu widgets unpause the core for a frame to update
+   the on-screen widgets.  To tell the ai service what the pause
+   mode is honestly, we store the runloop_paused variable from before
+   the handle_translation_cb wipes the widgets, and pass that in here.
+*/
+
+static bool run_translation_service(bool paused)
 {
    struct video_viewport vp;
    uint8_t header[54];
@@ -6625,6 +6854,9 @@ static bool run_translation_service(void)
    const char *rf1                       = "{\"image\": \"";
    const char *rf2                       = "\"}\0";
    char *rf3                             = NULL;
+   char *state_son                       = NULL;
+   int state_son_length                  = 0;
+   int curr_length                       = 0;
    bool TRANSLATE_USE_BMP                = false;
    bool use_overlay                      = false;
 
@@ -6633,7 +6865,7 @@ static bool run_translation_service(void)
    core_info_t *core_info                = NULL;
 
 #ifdef HAVE_GFX_WIDGETS
-   if (gfx_widgets_ai_service_overlay_get_state() != 0)
+   if (gfx_widgets_ai_service_overlay_get_state() != 0 && get_ai_service_auto() == 1)
    {
       /* For the case when ai service pause is disabled. */
       gfx_widgets_ai_service_overlay_unload();
@@ -6794,32 +7026,96 @@ static bool run_translation_service(void)
    {
       unsigned i;
       /* include game label if provided */
-      rf3 = (char *)malloc(16+strlen(system_label));
+      rf3 = (char *)malloc(15+strlen(system_label));
-      memcpy(rf3, "\", \"label\": \"", 13*sizeof(uint8_t));
+      memcpy(rf3, ", \"label\": \"", 12*sizeof(uint8_t));
-      memcpy(rf3+13, system_label, strlen(system_label));
+      memcpy(rf3+12, system_label, strlen(system_label));
-      memcpy(rf3+13+strlen(system_label), "\"}\0", 3*sizeof(uint8_t));
+      memcpy(rf3+12+strlen(system_label), "\"}\0", 3*sizeof(uint8_t));
-      for (i=13;i<strlen(system_label)+13;i++)
+      for (i=12;i<strlen(system_label)+12;i++)
       {
          if (rf3[i] == '\"')
             rf3[i] = ' ';
       }
-      json_length = 11+out_length+16+strlen(system_label);
+      json_length = 11+out_length+15+strlen(system_label);
    }
    else
-      json_length = 11+out_length+3;
+      json_length = 11+out_length+1;
+
+   {
+      state_son_length = 177;
+      state_son = (char *) malloc(state_son_length);
+
+      memcpy(state_son, ", \"state\": {\"paused\": 0, \"a\": 0, \"b\": 0, \"select\": 0, \"start\": 0, \"up\": 0, \"down\": 0, \"left\": 0, \"right\": 0, \"x\": 0, \"y\": 0, \"l\": 0, \"r\":0, \"l2\": 0, \"r2\": 0, \"l3\":0, \"r3\": 0}}\0", state_son_length*sizeof(uint8_t));
+
+      if (paused)
+         state_son[22] = '1';
+
+      if (ai_gamepad_state[8])//a
+         state_son[30] = '1';
+      if (ai_gamepad_state[0])//b
+         state_son[38] = '1';
+      if (ai_gamepad_state[2])//select
+         state_son[51] = '1';
+      if (ai_gamepad_state[3])//start
+         state_son[63] = '1';
+
+      if (ai_gamepad_state[4])//up
+         state_son[72] = '1';
+      if (ai_gamepad_state[5])//down
+         state_son[83] = '1';
+      if (ai_gamepad_state[6])//left
+         state_son[94] = '1';
+      if (ai_gamepad_state[7])//right
+         state_son[106] = '1';
+
+      if (ai_gamepad_state[9])//x
+         state_son[114] = '1';
+      if (ai_gamepad_state[1])//y
+         state_son[122] = '1';
+      if (ai_gamepad_state[10])//l
+         state_son[130] = '1';
+      if (ai_gamepad_state[11])//r
+         state_son[138] = '1';
+
+      if (ai_gamepad_state[12])//l2
+         state_son[147] = '1';
+      if (ai_gamepad_state[13])//r2
+         state_son[156] = '1';
+      if (ai_gamepad_state[14])//l3
+         state_son[165] = '1';
+      if (ai_gamepad_state[15])//r3
+         state_son[174] = '1';
+
+      json_length+=state_son_length;
+   }
 
    json_buffer = (char*)malloc(json_length);
    if (!json_buffer)
       goto finish;
-
+   /* Image data */
    memcpy(json_buffer, (const void*)rf1, 11*sizeof(uint8_t));
    memcpy(json_buffer+11, bmp64_buffer, (out_length)*sizeof(uint8_t));
+   memcpy(json_buffer+11+out_length, "\"", 1*sizeof(uint8_t));
+   curr_length = 11+out_length+1;
+
+   /* State data */
+   memcpy(json_buffer+curr_length, state_son, state_son_length*sizeof(uint8_t));
+   curr_length+= state_son_length;
+
+   /* System Label */
    if (rf3)
-      memcpy(json_buffer+11+out_length, (const void*)rf3, (16+strlen(system_label))*sizeof(uint8_t));
+   {
+      memcpy(json_buffer+curr_length, (const void*)rf3, (15+strlen(system_label))*sizeof(uint8_t));   
+      curr_length+=15+strlen(system_label);
+   }
    else
-      memcpy(json_buffer+11+out_length, (const void*)rf2, 3*sizeof(uint8_t));
+   {
+      memcpy(json_buffer+curr_length, (const void*)rf2, 3*sizeof(uint8_t));
+      curr_length+=3;
+   }
+
 #ifdef DEBUG
-   RARCH_LOG("Request size: %d\n", out_length);
+   if (get_ai_service_auto()!=2)
+      RARCH_LOG("Request size: %d\n", out_length);
 #endif
    {
       char separator  = '?';
@@ -6913,7 +7209,8 @@ static bool run_translation_service(void)
                  sizeof(new_ai_service_url));
       }
 #ifdef DEBUG
-      RARCH_LOG("SENDING... %s\n", new_ai_service_url);
+      if (get_ai_service_auto() != 2)
+         RARCH_LOG("SENDING... %s\n", new_ai_service_url);
 #endif
       task_push_http_post_transfer(new_ai_service_url,
             json_buffer, true, NULL, handle_translation_cb, NULL);
@@ -8165,7 +8462,11 @@ bool command_event(enum event_command cmd, void *data)
          settings_t *settings      = configuration_settings;
          bool ai_service_pause     = settings->bools.ai_service_pause;
 
-         if (ai_service_pause)
+         if (!settings->bools.ai_service_enable)
+         {
+            break;
+         }
+         else if (ai_service_pause)
          {
             /* pause on call, unpause on second press. */
             if (!runloop_paused)
@@ -8187,8 +8488,22 @@ bool command_event(enum event_command cmd, void *data)
            /* Don't pause - useful for Text-To-Speech since
             * the audio can't currently play while paused.
             * Also useful for cases when users don't want the
-            * core's sound to stop while translating. */
+            * core's sound to stop while translating. 
-            command_event(CMD_EVENT_AI_SERVICE_CALL, NULL);
+            *
+            * Also, this mode is required for "auto" translation
+            * packages, since you don't want to pause for that.   
+            */ 
+            if (get_ai_service_auto() == 2)
+            {
+               /* Auto mode was turned on, but we pressed the
+                * toggle button, so turn it off now. */
+               set_ai_service_auto(0);
+#ifdef HAVE_MENU_WIDGETS
+               gfx_widgets_ai_service_overlay_unload();
+#endif
+            }
+            else
+               command_event(CMD_EVENT_AI_SERVICE_CALL, NULL);
          }
 #endif
          break;
@@ -9504,8 +9819,15 @@ bool command_event(enum event_command cmd, void *data)
 #endif
          else
          {
-            RARCH_LOG("AI Service Called...\n");
+            bool paused = runloop_paused;
-            run_translation_service();
+            if (data!=NULL)
+               paused = *((bool*)data);
+
+            if (get_ai_service_auto() == 0 && settings->bools.ai_service_pause == false)
+               set_ai_service_auto(1);
+            if (get_ai_service_auto() != 2)
+               RARCH_LOG("AI Service Called...\n");
+            run_translation_service(paused);
          }
 #endif
          break;
@@ -25918,7 +26240,6 @@ static int16_t input_state_with_logging(unsigned port,
       int16_t last_input = input_state_get_last(port, device, index, id);
       if (result != last_input)
          input_is_dirty  = true;
-
       /*arbitrary limit of up to 65536 elements in state array*/
       if (id < 65536)
          input_state_set_last(port, device, index, id, result);
@@ -25944,7 +26265,7 @@ static bool unserialize_hook(const void *buf, size_t size)
 }
 
 static void add_input_state_hook(void)
-{
+{  
    if (!input_state_callback_original)
    {
       input_state_callback_original = retro_ctx.state_cb;
@@ -29160,7 +29481,22 @@ static enum runloop_state runloop_check_state(retro_time_t current_time)
    }
    else
 #endif
+   {
       input_keys_pressed(&current_bits, &joypad_info);
+#ifdef HAVE_TRANSLATE
+      if (settings->bools.ai_service_enable)
+      {
+         reset_gamepad_input_override();
+      
+         for (int i=0;i<16;i++)
+         {
+            if (ai_gamepad_state[i] == 2)
+               set_gamepad_input_override(i, true);
+            ai_gamepad_state[i] = 0;
+         }
+      }      
+#endif
+   }
 
 #ifdef HAVE_MENU
    last_input                       = current_bits;
@@ -29647,7 +29983,31 @@ static enum runloop_state runloop_check_state(retro_time_t current_time)
       }
    }
 
-   if (!focused)
+#ifdef HAVE_TRANSLATE
+   /* Copy over the retropad state to a buffer for the translate service
+      to send off if it's run. */
+   ai_gamepad_state[0] = BIT256_GET(current_bits, RETRO_DEVICE_ID_JOYPAD_B);
+   ai_gamepad_state[1] = BIT256_GET(current_bits, RETRO_DEVICE_ID_JOYPAD_Y);
+   ai_gamepad_state[2] = BIT256_GET(current_bits, RETRO_DEVICE_ID_JOYPAD_SELECT);
+   ai_gamepad_state[3] = BIT256_GET(current_bits, RETRO_DEVICE_ID_JOYPAD_START);
+
+   ai_gamepad_state[4] = BIT256_GET(current_bits, RETRO_DEVICE_ID_JOYPAD_UP);
+   ai_gamepad_state[5] = BIT256_GET(current_bits, RETRO_DEVICE_ID_JOYPAD_DOWN);
+   ai_gamepad_state[6] = BIT256_GET(current_bits, RETRO_DEVICE_ID_JOYPAD_LEFT);
+   ai_gamepad_state[7] = BIT256_GET(current_bits, RETRO_DEVICE_ID_JOYPAD_RIGHT);
+
+   ai_gamepad_state[8] = BIT256_GET(current_bits, RETRO_DEVICE_ID_JOYPAD_A);
+   ai_gamepad_state[9] = BIT256_GET(current_bits, RETRO_DEVICE_ID_JOYPAD_X);
+   ai_gamepad_state[10] = BIT256_GET(current_bits, RETRO_DEVICE_ID_JOYPAD_L);
+   ai_gamepad_state[11] = BIT256_GET(current_bits, RETRO_DEVICE_ID_JOYPAD_R);
+
+   ai_gamepad_state[12] = BIT256_GET(current_bits, RETRO_DEVICE_ID_JOYPAD_L2);
+   ai_gamepad_state[13] = BIT256_GET(current_bits, RETRO_DEVICE_ID_JOYPAD_R2);
+   ai_gamepad_state[14] = BIT256_GET(current_bits, RETRO_DEVICE_ID_JOYPAD_L3);
+   ai_gamepad_state[15] = BIT256_GET(current_bits, RETRO_DEVICE_ID_JOYPAD_R3);
+#endif
+
+  if (!focused)
    {
       retro_ctx.poll_cb();
       return RUNLOOP_STATE_POLLED_AND_SLEEP;
@@ -29846,6 +30206,7 @@ static enum runloop_state runloop_check_state(retro_time_t current_time)
          RARCH_CHEAT_INDEX_MINUS, CMD_EVENT_CHEAT_INDEX_MINUS,
          RARCH_CHEAT_TOGGLE,      CMD_EVENT_CHEAT_TOGGLE);
 
+
 #if defined(HAVE_CG) || defined(HAVE_GLSL) || defined(HAVE_SLANG) || defined(HAVE_HLSL)
    if (settings->bools.video_shader_watch_files)
    {
@@ -30324,8 +30685,8 @@ static int16_t core_input_state_poll_late(unsigned port,
 {
    if (!current_core.input_polled)
       input_driver_poll();
-
    current_core.input_polled = true;
+   
    return input_state(port, device, idx, id);
 }
 
@@ -30757,4 +31118,27 @@ static bool accessibility_startup_message(void)
          10);
    return true;
 }
+
+
+static unsigned gamepad_input_override = 0;
+
+unsigned get_gamepad_input_override(void)
+{
+   return gamepad_input_override;
+}
+
+void set_gamepad_input_override(unsigned i, bool val)
+{
+   if (val)
+      gamepad_input_override = gamepad_input_override | (1<<i);
+   else
+      gamepad_input_override = gamepad_input_override & ((1<<i) ^ 0);
+}
+
+void reset_gamepad_input_override(void)
+{
+    gamepad_input_override = 0;
+}
+
+
 #endif

retroarch.h

@@ -2010,6 +2010,14 @@ void retroarch_init_task_queue(void);
 
 bool is_input_keyboard_display_on(void);
 
+
+/* Input overrides  */
+
+static unsigned gamepad_input_override;
+extern unsigned get_gamepad_input_override(void);
+extern void set_gamepad_input_override(unsigned i, bool val);
+extern void reset_gamepad_input_override(void);
+
 RETRO_END_DECLS
 
 #endif