Improve haptic feedback for input overlays (#14517)

Repurpose vibrate_on_keypress to enable device's standard keypress feedback on overlay key/button state changes

- Add keypress_vibrate function ptr to input_driver_t (only implemented on Android for now)
- (Android) Remove APP_CMD_VIBRATE_KEYPRESS
- (Android) Add doHapticFeedback, called directly to avoid latency

frontend/drivers/platform_unix.c

@@ -2088,6 +2088,8 @@ static void frontend_unix_init(void *data)
          "setScreenOrientation", "(I)V");
    GET_METHOD_ID(env, android_app->doVibrate, class,
          "doVibrate", "(IIII)V");
+   GET_METHOD_ID(env, android_app->doHapticFeedback, class,
+         "doHapticFeedback", "(I)V");
    GET_METHOD_ID(env, android_app->getUserLanguageString, class,
          "getUserLanguageString", "()Ljava/lang/String;");
    GET_METHOD_ID(env, android_app->isPlayStoreBuild, class,

frontend/drivers/platform_unix.h

@@ -168,6 +168,7 @@ struct android_app
    jmethodID setScreenOrientation;
    jmethodID getUserLanguageString;
    jmethodID doVibrate;
+   jmethodID doHapticFeedback;
 
    jmethodID isPlayStoreBuild;
    jmethodID getAvailableCores;
@@ -290,9 +291,7 @@ enum
     */
    APP_CMD_DESTROY,
 
-   APP_CMD_REINIT_DONE,
+   APP_CMD_REINIT_DONE
-
-   APP_CMD_VIBRATE_KEYPRESS
 };
 
 #define JNI_EXCEPTION(env) \

input/drivers/android_input.c

@@ -168,9 +168,9 @@ bool (*engine_lookup_name)(char *buf,
       int *vendorId, int *productId, size_t size, int id);
 void (*engine_handle_dpad)(struct android_app *, AInputEvent*, int, int);
 
-static void android_input_poll_input_gingerbread(android_input_t *android, bool vibrate_on_keypress);
+static void android_input_poll_input_gingerbread(android_input_t *android);
-static void android_input_poll_input_default(android_input_t *android, bool vibrate_on_keypress);
+static void android_input_poll_input_default(android_input_t *android);
-static void (*android_input_poll_input)(android_input_t *android, bool vibrate_on_keypress);
+static void (*android_input_poll_input)(android_input_t *android);
 
 static bool android_input_set_sensor_state(void *data, unsigned port,
       enum retro_sensor_action action, unsigned event_rate);
@@ -436,17 +436,6 @@ static void android_input_poll_main_cmd(void)
       case APP_CMD_DESTROY:
          android_app->destroyRequested = 1;
          break;
-      case APP_CMD_VIBRATE_KEYPRESS:
-      {
-         JNIEnv *env = (JNIEnv*)jni_thread_getenv();
-
-         if (env && g_android && g_android->doVibrate)
-         {
-            CALL_VOID_METHOD_PARAM(env, g_android->activity->clazz,
-                  g_android->doVibrate, (jint)-1, (jint)RETRO_RUMBLE_STRONG, (jint)255, (jint)1);
-         }
-         break;
-      }
    }
 }
 
@@ -636,7 +625,7 @@ static INLINE void android_mouse_calculate_deltas(android_input_t *android,
 
 static INLINE void android_input_poll_event_type_motion(
       android_input_t *android, AInputEvent *event,
-      int port, int source, bool vibrate_on_keypress)
+      int port, int source)
 {
    int btn;
    int getaction     = AMotionEvent_getAction(event);
@@ -707,9 +696,6 @@ static INLINE void android_input_poll_event_type_motion(
       int      pointer_max     = MIN(
             AMotionEvent_getPointerCount(event), MAX_TOUCH);
 
-      if (vibrate_on_keypress && action != AMOTION_EVENT_ACTION_MOVE)
-         android_app_write_cmd(g_android, APP_CMD_VIBRATE_KEYPRESS);
-
       if (action == AMOTION_EVENT_ACTION_DOWN && ENABLE_TOUCH_SCREEN_MOUSE)
       {
          /* When touch screen is pressed, set mouse
@@ -1273,8 +1259,7 @@ static void engine_handle_touchpad(
 }
 
 static void android_input_poll_input_gingerbread(
-      android_input_t *android,
+      android_input_t *android)
-      bool vibrate_on_keypress)
 {
    AInputEvent              *event = NULL;
    struct android_app *android_app = (struct android_app*)g_android;
@@ -1307,7 +1292,7 @@ static void android_input_poll_input_gingerbread(
             else if ((source & (AINPUT_SOURCE_TOUCHSCREEN
                         | AINPUT_SOURCE_STYLUS | AINPUT_SOURCE_MOUSE)))
                android_input_poll_event_type_motion(android, event,
-                     port, source, vibrate_on_keypress);
+                     port, source);
             else
                engine_handle_dpad(android_app, event, port, source);
             handled = 1;
@@ -1335,8 +1320,7 @@ static void android_input_poll_input_gingerbread(
    }
 }
 
-static void android_input_poll_input_default(android_input_t *android,
+static void android_input_poll_input_default(android_input_t *android)
-      bool vibrate_on_keypress)
 {
    AInputEvent              *event = NULL;
    struct android_app *android_app = (struct android_app*)g_android;
@@ -1370,7 +1354,7 @@ static void android_input_poll_input_default(android_input_t *android,
                else if ((source & (AINPUT_SOURCE_TOUCHSCREEN 
                            | AINPUT_SOURCE_STYLUS | AINPUT_SOURCE_MOUSE)))
                   android_input_poll_event_type_motion(android, event,
-                        port, source, vibrate_on_keypress);
+                        port, source);
                else
                   engine_handle_dpad(android_app, event, port, source);
                break;
@@ -1466,15 +1450,10 @@ static void android_input_poll(void *data)
                ? -1 : settings->uints.input_block_timeout,
                NULL, NULL, NULL)) >= 0)
    {
-      bool vibrate_on_keypress     = settings 
-         ? settings->bools.vibrate_on_keypress 
-         : false;
-
       switch (ident)
       {
          case LOOPER_ID_INPUT:
-            android_input_poll_input(android,
+            android_input_poll_input(android);
-                  vibrate_on_keypress);
             break;
          case LOOPER_ID_USER:
             android_input_poll_user(android);
@@ -1848,6 +1827,18 @@ static float android_input_get_sensor_input(void *data,
    return 0.0f;
 }
 
+static void android_input_keypress_vibrate()
+{
+   static const int keyboard_press = 3;
+   JNIEnv *env = (JNIEnv*)jni_thread_getenv();
+
+   if (!env)
+      return;
+
+   CALL_VOID_METHOD_PARAM(env, g_android->activity->clazz,
+         g_android->doHapticFeedback, (jint)keyboard_press);
+}
+
 input_driver_t input_android = {
    android_input_init,
    android_input_poll,
@@ -1859,5 +1850,6 @@ input_driver_t input_android = {
    "android",
 
    NULL,                            /* grab_mouse */
-   NULL
+   NULL,
+   android_input_keypress_vibrate
 };

input/drivers/cocoa_input.c

@@ -561,5 +561,6 @@ input_driver_t input_cocoa = {
    cocoa_input_get_capabilities,
    "cocoa",
    NULL,                         /* grab_mouse */
+   NULL,
    NULL
 };

input/drivers/ctr_input.c

@@ -46,5 +46,6 @@ input_driver_t input_ctr = {
    ctr_input_get_capabilities,
    "ctr",
    NULL,                         /* grab_mouse */
+   NULL,
    NULL
 };

input/drivers/dinput.c

@@ -1045,5 +1045,6 @@ input_driver_t input_dinput = {
    dinput_get_capabilities,
    "dinput",
    dinput_grab_mouse,
+   NULL,
    NULL
 };

input/drivers/dos_input.c

@@ -133,5 +133,6 @@ input_driver_t input_dos = {
    dos_input_get_capabilities,
    "dos",
    NULL,                         /* grab_mouse */
+   NULL,
    NULL
 };

input/drivers/gx_input.c

@@ -298,5 +298,6 @@ input_driver_t input_gx = {
    "gx",
 
    NULL,                         /* grab_mouse */
+   NULL,
    NULL
 };

input/drivers/linuxraw_input.c

@@ -209,5 +209,6 @@ input_driver_t input_linuxraw = {
    linuxraw_get_capabilities,
    "linuxraw",
    NULL,                         /* grab_mouse */
-   linux_terminal_grab_stdin
+   linux_terminal_grab_stdin,
+   NULL
 };

input/drivers/ps2_input.c

@@ -44,5 +44,6 @@ input_driver_t input_ps2 = {
    ps2_input_get_capabilities,
    "ps2",
    NULL,                         /* grab_mouse */
-   NULL 
+   NULL,
+   NULL
 };

input/drivers/ps3_input.c

@@ -218,5 +218,6 @@ input_driver_t input_ps3 = {
    "ps3",
 
    NULL,                         /* grab_mouse */
+   NULL,
    NULL
 };

input/drivers/ps4_input.c

@@ -143,5 +143,6 @@ input_driver_t input_ps4 = {
    ps4_input_get_capabilities,
    "ps4",
    NULL,                         /* grab_mouse */
+   NULL,
    NULL
 };

input/drivers/psl1ght_input.c

@@ -951,5 +951,6 @@ input_driver_t input_ps3 = {
    "ps3",
 
    NULL,                         /* grab_mouse */
+   NULL,
    NULL
 };

input/drivers/psp_input.c

@@ -441,5 +441,6 @@ input_driver_t input_psp = {
 #endif
 
    NULL,                         /* grab_mouse */
+   NULL,
    NULL
 };

input/drivers/qnx_input.c

@@ -852,5 +852,6 @@ input_driver_t input_qnx = {
    qnx_input_get_capabilities,
    "qnx_input",
    NULL,
+   NULL,
    NULL
 };

input/drivers/rwebinput_input.c

@@ -686,5 +686,6 @@ input_driver_t input_rwebinput = {
    rwebinput_get_capabilities,
    "rwebinput",
    rwebinput_grab_mouse,
+   NULL,
    NULL
 };

input/drivers/sdl_dingux_input.c

@@ -50,5 +50,6 @@ input_driver_t input_sdl_dingux = {
    sdl_dingux_input_get_capabilities,
    "sdl_dingux",
    NULL,                              /* grab_mouse */
-   NULL                               /* grab_stdin */
+   NULL,                              /* grab_stdin */
+   NULL
 };

input/drivers/sdl_input.c

@@ -475,6 +475,7 @@ input_driver_t input_sdl = {
    "sdl",
    NULL,                   /* grab_mouse */
 #endif
+   NULL,
    NULL
 };
 

input/drivers/switch_input.c

@@ -948,5 +948,6 @@ input_driver_t input_switch = {
    switch_input_get_capabilities,
    "switch",
    NULL,                            /* grab_mouse */
+   NULL,
    NULL
 };

input/drivers/udev_input.c

@@ -1540,8 +1540,9 @@ input_driver_t input_udev = {
    "udev",
    udev_input_grab_mouse,
 #ifdef __linux__
-   linux_terminal_grab_stdin
+   linux_terminal_grab_stdin,
 #else
-   NULL
+   NULL,
 #endif
+   NULL
 };

input/drivers/uwp_input.c

@@ -168,5 +168,6 @@ input_driver_t input_uwp = {
    uwp_input_get_capabilities,
    "uwp",
    NULL,                         /* grab_mouse */
+   NULL,
    NULL
 };

input/drivers/wayland_input.c

@@ -426,5 +426,6 @@ input_driver_t input_wayland = {
    input_wl_get_capabilities,
    "wayland",
    input_wl_grab_mouse,          /* grab_mouse */
+   NULL,
    NULL
 };

input/drivers/wiiu_input.c

@@ -168,5 +168,6 @@ input_driver_t input_wiiu = {
    wiiu_input_get_capabilities,
    "wiiu",
    NULL,                            /* grab_mouse */
+   NULL,
    NULL
 };

input/drivers/winraw_input.c

@@ -1087,5 +1087,6 @@ input_driver_t input_winraw = {
    winraw_get_capabilities,
    "raw",
    winraw_grab_mouse,
+   NULL,
    NULL
 };

input/drivers/x11_input.c

@@ -615,5 +615,6 @@ input_driver_t input_x = {
    x_input_get_capabilities,
    "x",
    x_grab_mouse,
+   NULL,
    NULL
 };

input/drivers/xdk_xinput_input.c

@@ -50,5 +50,6 @@ input_driver_t input_xinput = {
    xdk_input_get_capabilities,
    "xinput",
    NULL,                         /* grab_mouse */
+   NULL,
    NULL
 };

input/drivers/xenon360_input.c

@@ -80,5 +80,6 @@ input_driver_t input_xenon360 = {
    xenon360_input_get_capabilities,
    "xenon360",
    NULL,                            /* grab_mouse */
+   NULL,
    NULL
 };

input/input_driver.c

@@ -109,6 +109,7 @@ static float input_null_get_sensor_input(void *data, unsigned port, unsigned id)
 static uint64_t input_null_get_capabilities(void *data) { return 0; }
 static void input_null_grab_mouse(void *data, bool state) { }
 static bool input_null_grab_stdin(void *data) { return false; }
+static void input_null_keypress_vibrate() { }
 
 static input_driver_t input_null = {
    input_null_init,
@@ -120,7 +121,8 @@ static input_driver_t input_null = {
    input_null_get_capabilities,
    "null",
    input_null_grab_mouse,
-   input_null_grab_stdin
+   input_null_grab_stdin,
+   input_null_keypress_vibrate
 };
 
 static input_device_driver_t null_joypad = {
@@ -1860,7 +1862,7 @@ void input_poll_overlay(
       unsigned analog_dpad_mode,
       float axis_threshold)
 {
-   input_overlay_state_t old_key_state;
+   input_overlay_state_t old_ol_state;
    int i, j;
    input_overlay_t *ol                     = (input_overlay_t*)ol_data;
    uint16_t key_mod                        = 0;
@@ -1875,12 +1877,15 @@ void input_poll_overlay(
                settings->uints.input_overlay_show_inputs;
    unsigned input_overlay_show_inputs_port = settings->uints.input_overlay_show_inputs_port;
    float touch_scale                       = (float)settings->uints.input_touch_scale;
+   bool osk_state_changed                  = false;
+   unsigned pointer_count                  = 0;
+   static unsigned old_pointer_count;
 
    if (!ol_state)
       return;
 
-   memcpy(old_key_state.keys, ol_state->keys,
+   memcpy(&old_ol_state, ol_state,
-         sizeof(ol_state->keys));
+         sizeof(old_ol_state));
    memset(ol_state, 0, sizeof(*ol_state));
 
    if (current_input->input_state)
@@ -1963,6 +1968,8 @@ void input_poll_overlay(
 
          polled = true;
       }
+
+      pointer_count = i;
    }
 
    if (  OVERLAY_GET_KEY(ol_state, RETROK_LSHIFT) ||
@@ -1984,10 +1991,11 @@ void input_poll_overlay(
    /* CAPSLOCK SCROLLOCK NUMLOCK */
    for (i = 0; i < ARRAY_SIZE(ol_state->keys); i++)
    {
-      if (ol_state->keys[i] != old_key_state.keys[i])
+      if (ol_state->keys[i] != old_ol_state.keys[i])
       {
-         uint32_t orig_bits = old_key_state.keys[i];
+         uint32_t orig_bits = old_ol_state.keys[i];
          uint32_t new_bits  = ol_state->keys[i];
+         osk_state_changed  = true;
 
          for (j = 0; j < 32; j++)
             if ((orig_bits & (1 << j)) != (new_bits & (1 << j)))
@@ -2059,6 +2067,24 @@ void input_poll_overlay(
             button_pressed, opacity);
    else
       input_overlay_poll_clear(overlay_visibility, ol, opacity);
+
+   /* Create haptic feedback for any change in button/key state,
+    * unless pointer_count decreased. */
+   if (     current_input->keypress_vibrate
+         && settings->bools.vibrate_on_keypress
+         && pointer_count && pointer_count >= old_pointer_count
+         && !ol->blocked)
+   {
+      if (     osk_state_changed
+            || bits_any_different(
+                     ol_state->buttons.data,
+                     old_ol_state.buttons.data,
+                     ARRAY_SIZE(old_ol_state.buttons.data))
+         )
+         current_input->keypress_vibrate();
+   }
+
+   old_pointer_count = pointer_count;
 }
 #endif
 

input/input_driver.h

@@ -340,6 +340,12 @@ struct input_driver
     * @return True if the input driver has claimed stdin.
     */
    bool (*grab_stdin)(void *data);
+
+   /**
+    * Haptic feedback for touchscreen key presses. This function pointer can be
+    * set to NULL if haptic feedback / vibration is not supported.
+    */
+   void (*keypress_vibrate)(void);
 };
 
 struct rarch_joypad_driver

libretro-common/include/retro_miscellaneous.h

@@ -74,6 +74,17 @@ static INLINE bool bits_any_set(uint32_t* ptr, uint32_t count)
    return false;
 }
 
+static INLINE bool bits_any_different(uint32_t *a, uint32_t *b, uint32_t count)
+{
+   uint32_t i;
+   for (i = 0; i < count; i++)
+   {
+      if (a[i] != b[i])
+         return true;
+   }
+   return false;
+}
+
 #ifndef PATH_MAX_LENGTH
 #if defined(_XBOX1) || defined(_3DS) || defined(PSP) || defined(PS2) || defined(GEKKO)|| defined(WIIU) || defined(__PSL1GHT__) || defined(__PS3__)
 #define PATH_MAX_LENGTH 512

menu/menu_displaylist.c

@@ -6249,23 +6249,25 @@ unsigned menu_displaylist_build_list(
          break;
       case DISPLAYLIST_INPUT_HAPTIC_FEEDBACK_SETTINGS_LIST:
          {
-            const char *input_driver_id  = settings->arrays.input_driver;
+            input_driver_t *current_input =
+                  input_state_get_ptr()->current_driver;
 
-            if (string_is_equal(input_driver_id, "android"))
+            if (current_input->keypress_vibrate)
-            {
                if (MENU_DISPLAYLIST_PARSE_SETTINGS_ENUM(list,
                         MENU_ENUM_LABEL_VIBRATE_ON_KEYPRESS,
                         PARSE_ONLY_BOOL, false) == 0)
                   count++;
+
+            if (string_is_equal(current_input->ident, "android"))
                if (MENU_DISPLAYLIST_PARSE_SETTINGS_ENUM(list,
                         MENU_ENUM_LABEL_ENABLE_DEVICE_VIBRATION,
                         PARSE_ONLY_BOOL, false) == 0)
                   count++;
-            }
+
             if (MENU_DISPLAYLIST_PARSE_SETTINGS_ENUM(list,
                      MENU_ENUM_LABEL_INPUT_RUMBLE_GAIN,
                      PARSE_ONLY_UINT, false) == 0)
-                  count++;
+               count++;
          }
          break;
       case DISPLAYLIST_INPUT_HOTKEY_BINDS_LIST:

pkg/android/phoenix-common/src/com/retroarch/browser/retroactivity/RetroActivityCommon.java

@@ -17,6 +17,7 @@ import android.os.Bundle;
 import android.os.storage.StorageManager;
 import android.os.storage.StorageVolume;
 import android.system.Os;
+import android.view.HapticFeedbackConstants;
 import android.view.InputDevice;
 import android.view.Surface;
 import android.view.WindowManager;
@@ -116,6 +117,13 @@ public class RetroActivityCommon extends NativeActivity
     }
   }
 
+  public void doHapticFeedback(int effect)
+  {
+    getWindow().getDecorView().performHapticFeedback(effect,
+        HapticFeedbackConstants.FLAG_IGNORE_GLOBAL_SETTING | HapticFeedbackConstants.FLAG_IGNORE_VIEW_SETTING);
+    Log.i("RetroActivity", "Haptic Feedback effect " + effect);
+  }
+
   // Exiting cleanly from NDK seems to be nearly impossible.
   // Have to use exit(0) to avoid weird things happening, even with runOnUiThread() approaches.
   // Use a separate JNI function to explicitly trigger the readback.