Win8/10: Add support to draw with one finger, pan/scroll with two

https://community.aseprite.org/t/using-touchscreen-computers-to-draw-in-aseprite/677

data/pref.xml

@@ -154,6 +154,7 @@
     <section id="experimental" text="Experimental">
       <option id="use_native_clipboard" type="bool" default="true" />
       <option id="use_native_file_dialog" type="bool" default="false" />
+      <option id="one_finger_as_mouse_movement" type="bool" default="true" />
       <option id="flash_layer" type="bool" default="false" migrate="Options.FlashLayer" />
       <option id="nonactive_layers_opacity" type="int" default="255" />
     </section>

data/strings/en.ini

@@ -1014,6 +1014,15 @@ open_extension_folder = Open &Folder
 user_interface = User Interface
 native_clipboard = Use native clipboard
 native_file_dialog = Use native file dialog
+one_finger_as_mouse_movement = Interpret one finger as mouse movement
+one_finger_as_mouse_movement_tooltip = <<<END
+Only for Windows 8/10 tablets: Interprets one finger as mouse movement
+and two fingers as pan/scroll. Uncheck this to use the old behavior:
+One finger pans/scrolls.
+
+This option is available just to get the old behavior but
+will be removed in future versions.
+END
 flash_selected_layer = Flash layer when it is selected
 non_active_layer_opacity = Opacity for non-active layers:
 ok = &OK

data/widgets/options.xml

@@ -363,8 +363,12 @@
           <separator text="@.user_interface" horizontal="true" />
           <check id="native_clipboard" text="@.native_clipboard" />
           <check id="native_file_dialog" text="@.native_file_dialog" />
+          <check id="one_finger_as_mouse_movement"
+                 text="@.one_finger_as_mouse_movement"
+                 tooltip="@.one_finger_as_mouse_movement_tooltip"
+                 pref="experimental.one_finger_as_mouse_movement" />
           <check id="flash_layer" text="@.flash_selected_layer" />
-	  <hbox>
+          <hbox>
             <label text="@.non_active_layer_opacity" />
             <slider id="nonactive_layers_opacity" min="0" max="255" width="128" />
           </hbox>

src/app/app.cpp

@@ -278,6 +278,13 @@ void App::run()
 #ifdef ENABLE_UI
   // Run the GUI
   if (isGui()) {
+#ifdef _WIN32
+    // How to interpret one finger on Windows tablets.
+    ui::Manager::getDefault()->getDisplay()
+      ->setInterpretOneFingerGestureAsMouseMovement(
+        Preferences::instance().experimental.oneFingerAsMouseMovement());
+#endif
+
 #if !defined(_WIN32) && !defined(__APPLE__)
     // Setup app icon for Linux window managers
     try {

src/app/commands/cmd_options.cpp

@@ -244,6 +244,10 @@ public:
     if (m_pref.experimental.useNativeFileDialog())
       nativeFileDialog()->setSelected(true);
 
+#ifndef _WIN32
+    oneFingerAsMouseMovement()->setVisible(false);
+#endif
+
     if (m_pref.experimental.flashLayer())
       flashLayer()->setSelected(true);
 
@@ -479,6 +483,12 @@ public:
     m_pref.experimental.flashLayer(flashLayer()->isSelected());
     m_pref.experimental.nonactiveLayersOpacity(nonactiveLayersOpacity()->getValue());
 
+#ifdef _WIN32
+    manager()->getDisplay()
+      ->setInterpretOneFingerGestureAsMouseMovement(
+        oneFingerAsMouseMovement()->isSelected());
+#endif
+
     ui::set_use_native_cursors(m_pref.cursor.useNativeCursor());
     ui::set_mouse_cursor_scale(m_pref.cursor.cursorScale());
 

src/she/alleg4/alleg_display.cpp

@@ -730,6 +730,11 @@ void Alleg4Display::setLayout(const std::string& layout)
 #endif
 }
 
+void Alleg4Display::setInterpretOneFingerGestureAsMouseMovement(bool state)
+{
+  // Do nothing
+}
+
 void* Alleg4Display::nativeHandle()
 {
 #ifdef _WIN32

src/she/alleg4/alleg_display.h

@@ -1,5 +1,5 @@
 // SHE library
-// Copyright (C) 2012-2017  David Capello
+// Copyright (C) 2012-2018  David Capello
 //
 // This file is released under the terms of the MIT license.
 // Read LICENSE.txt for more information.
@@ -44,6 +44,7 @@ namespace she {
     void releaseMouse() override;
     std::string getLayout() override;
     void setLayout(const std::string& layout) override;
+    void setInterpretOneFingerGestureAsMouseMovement(bool state) override;
     void* nativeHandle() override;
 
   private:

src/she/display.h

@@ -1,5 +1,5 @@
 // SHE library
-// Copyright (C) 2012-2017  David Capello
+// Copyright (C) 2012-2018  David Capello
 //
 // This file is released under the terms of the MIT license.
 // Read LICENSE.txt for more information.
@@ -71,6 +71,12 @@ namespace she {
     virtual std::string getLayout() = 0;
     virtual void setLayout(const std::string& layout) = 0;
 
+    // For Windows 8/10 only in tablet devices: Set to true if you
+    // want to interpret one finger as the mouse movement and two
+    // fingers as pan/scroll (true by default). If you want to pan
+    // with one finger, call this function with false.
+    virtual void setInterpretOneFingerGestureAsMouseMovement(bool state) = 0;
+
     // Returns the HWND on Windows.
     virtual DisplayHandle nativeHandle() = 0;
   };

src/she/skia/skia_display.cpp

@@ -1,5 +1,5 @@
 // SHE library
-// Copyright (C) 2012-2017  David Capello
+// Copyright (C) 2012-2018  David Capello
 //
 // This file is released under the terms of the MIT license.
 // Read LICENSE.txt for more information.
@@ -185,6 +185,11 @@ void SkiaDisplay::setLayout(const std::string& layout)
   m_window.setLayout(layout);
 }
 
+void SkiaDisplay::setInterpretOneFingerGestureAsMouseMovement(bool state)
+{
+  m_window.setInterpretOneFingerGestureAsMouseMovement(state);
+}
+
 void SkiaDisplay::setTranslateDeadKeys(bool state)
 {
   m_window.setTranslateDeadKeys(state);

src/she/skia/skia_display.h

@@ -1,5 +1,5 @@
 // SHE library
-// Copyright (C) 2012-2017  David Capello
+// Copyright (C) 2012-2018  David Capello
 //
 // This file is released under the terms of the MIT license.
 // Read LICENSE.txt for more information.
@@ -61,6 +61,8 @@ public:
   std::string getLayout() override;
   void setLayout(const std::string& layout) override;
 
+  void setInterpretOneFingerGestureAsMouseMovement(bool state) override;
+
   void setTranslateDeadKeys(bool state);
 
   // Returns the HWND on Windows.

src/she/win/window.cpp

@@ -29,6 +29,9 @@
 
 #define KEY_TRACE(...)
 #define MOUSE_TRACE(...)
+#define TOUCH_TRACE(...)
+
+#define kFingerAsMouseTimeout 50
 
 // Gets the window client are in absolute/screen coordinates
 #define ABS_CLIENT_RC(rc)                               \
@@ -59,6 +62,14 @@ static PointerType wt_packet_pkcursor_to_pointer_type(int pkCursor)
   return PointerType::Unknown;
 }
 
+WinWindow::Touch::Touch()
+  : fingers(0)
+  , canBeMouse(false)
+  , asMouse(false)
+  , timerID(0)
+{
+}
+
 WinWindow::WinWindow(int width, int height, int scale)
   : m_hwnd(nullptr)
   , m_hcursor(nullptr)
@@ -74,6 +85,8 @@ WinWindow::WinWindow(int width, int height, int scale)
   , m_lastPointerId(0)
   , m_ictx(nullptr)
   , m_ignoreRandomMouseEvents(0)
+  // True by default, we prefer to interpret one finger as mouse movement
+  , m_touch(new Touch)
 #if SHE_USE_POINTER_API_FOR_MOUSE
   , m_emulateDoubleClick(false)
   , m_doubleClickMsecs(GetDoubleClickTime())
@@ -472,6 +485,19 @@ void WinWindow::setTranslateDeadKeys(bool state)
   }
 }
 
+void WinWindow::setInterpretOneFingerGestureAsMouseMovement(bool state)
+{
+  if (state) {
+    if (!m_touch)
+      m_touch = new Touch;
+  }
+  else if (m_touch) {
+    killTouchTimer();
+    delete m_touch;
+    m_touch = nullptr;
+  }
+}
+
 bool WinWindow::setCursor(HCURSOR hcursor, bool custom)
 {
   SetCursor(hcursor);
@@ -795,8 +821,21 @@ LRESULT WinWindow::wndProc(UINT msg, WPARAM wparam, LPARAM lparam)
 
       if (pi.pointerType == PT_TOUCH || pi.pointerType == PT_PEN) {
         auto& winApi = system()->winApi();
-        if (m_ictx && winApi.AddPointerInteractionContext)
+        if (m_ictx && winApi.AddPointerInteractionContext) {
           winApi.AddPointerInteractionContext(m_ictx, pi.pointerId);
+
+          if (m_touch && pi.pointerType == PT_TOUCH &&
+              !m_touch->asMouse) {
+            ++m_touch->fingers;
+            TOUCH_TRACE("POINTERENTER fingers=%d\n", m_touch->fingers);
+            if (m_touch->fingers == 1) {
+              waitTimerToConvertFingerAsMouseMovement();
+            }
+            else if (m_touch->canBeMouse && m_touch->fingers >= 2) {
+              delegateFingerToInteractionContext();
+            }
+          }
+        }
       }
 
       if (!m_hasMouse) {
@@ -817,12 +856,33 @@ LRESULT WinWindow::wndProc(UINT msg, WPARAM wparam, LPARAM lparam)
         break;
 
       MOUSE_TRACE("POINTERLEAVE id=%d\n", pi.pointerId);
-      m_ignoreRandomMouseEvents = 0;
+
+      // After releasing a finger a WM_MOUSEMOVE event in the trackpad
+      // position is generated, we'll ignore that message.
+      if (m_touch)
+        m_ignoreRandomMouseEvents = 1;
+      else
+        m_ignoreRandomMouseEvents = 0;
 
       if (pi.pointerType == PT_TOUCH || pi.pointerType == PT_PEN) {
         auto& winApi = system()->winApi();
-        if (m_ictx && winApi.RemovePointerInteractionContext)
+        if (m_ictx && winApi.RemovePointerInteractionContext) {
           winApi.RemovePointerInteractionContext(m_ictx, pi.pointerId);
+
+          if (m_touch && pi.pointerType == PT_TOUCH) {
+            if (m_touch->fingers > 0)
+              --m_touch->fingers;
+            TOUCH_TRACE("POINTERLEAVE fingers=%d\n", m_touch->fingers);
+            if (m_touch->fingers == 0) {
+              if (m_touch->canBeMouse)
+                sendDelayedTouchEvents();
+              else
+                clearDelayedTouchEvents();
+              killTouchTimer();
+              m_touch->asMouse = false;
+            }
+          }
+        }
       }
 
 #if 0 // Don't generate MouseLeave from pen/touch messages
@@ -851,7 +911,7 @@ LRESULT WinWindow::wndProc(UINT msg, WPARAM wparam, LPARAM lparam)
         auto& winApi = system()->winApi();
         if (m_ictx && winApi.ProcessPointerFramesInteractionContext) {
           winApi.ProcessPointerFramesInteractionContext(m_ictx, 1, 1, &pi);
-          if (pi.pointerType == PT_TOUCH)
+          if (!m_touch && pi.pointerType == PT_TOUCH)
             return 0;
         }
       }
@@ -874,7 +934,7 @@ LRESULT WinWindow::wndProc(UINT msg, WPARAM wparam, LPARAM lparam)
         auto& winApi = system()->winApi();
         if (m_ictx && winApi.ProcessPointerFramesInteractionContext) {
           winApi.ProcessPointerFramesInteractionContext(m_ictx, 1, 1, &pi);
-          if (pi.pointerType == PT_TOUCH)
+          if (!m_touch && pi.pointerType == PT_TOUCH)
             return 0;
         }
       }
@@ -901,7 +961,7 @@ LRESULT WinWindow::wndProc(UINT msg, WPARAM wparam, LPARAM lparam)
         auto& winApi = system()->winApi();
         if (m_ictx && winApi.ProcessPointerFramesInteractionContext) {
           winApi.ProcessPointerFramesInteractionContext(m_ictx, 1, 1, &pi);
-          if (pi.pointerType == PT_TOUCH)
+          if (!m_touch && pi.pointerType == PT_TOUCH)
             return 0;
         }
       }
@@ -916,7 +976,22 @@ LRESULT WinWindow::wndProc(UINT msg, WPARAM wparam, LPARAM lparam)
       }
 
       ev.setType(Event::MouseMove);
-      queueEvent(ev);
+
+      if (m_touch && pi.pointerType == PT_TOUCH) {
+        TOUCH_TRACE("POINTERUPDATE canBeMouse=%d asMouse=%d\n",
+                    m_touch->canBeMouse,
+                    m_touch->asMouse);
+        if (!m_touch->asMouse) {
+          if (m_touch->canBeMouse)
+            m_touch->delayedEvents.push_back(ev);
+          else
+            return 0;
+        }
+        else
+          queueEvent(ev);
+      }
+      else
+        queueEvent(ev);
 
       handlePointerButtonChange(ev, pi);
 
@@ -956,6 +1031,25 @@ LRESULT WinWindow::wndProc(UINT msg, WPARAM wparam, LPARAM lparam)
       return 0;
     }
 
+    case WM_TIMER:
+      TOUCH_TRACE("TIMER %d\n", wparam);
+      if (m_touch && m_touch->timerID == wparam) {
+        killTouchTimer();
+
+        if (!m_touch->asMouse &&
+            m_touch->canBeMouse &&
+            m_touch->fingers == 1) {
+          TOUCH_TRACE("-> finger as mouse, sent %d events\n",
+                      m_touch->delayedEvents.size());
+
+          convertFingerAsMouseMovement();
+        }
+        else {
+          delegateFingerToInteractionContext();
+        }
+      }
+      break;
+
     // Keyboard Messages
 
     case WM_SYSKEYDOWN:
@@ -1272,6 +1366,20 @@ void WinWindow::handlePointerButtonChange(Event& ev, POINTER_INFO& pi)
   }
 #endif
 
+  if (m_touch && pi.pointerType == PT_TOUCH) {
+    if (!m_touch->asMouse) {
+      if (m_touch->canBeMouse) {
+        // TODO Review why the ui layer needs a Event::MouseMove event
+        //      before ButtonDown/Up events.
+        Event evMouseMove = ev;
+        evMouseMove.setType(Event::MouseMove);
+        m_touch->delayedEvents.push_back(evMouseMove);
+        m_touch->delayedEvents.push_back(ev);
+      }
+      return;
+    }
+  }
+
   queueEvent(ev);
 }
 
@@ -1288,11 +1396,15 @@ void WinWindow::handleInteractionContextOutput(
               output->interactionFlags,
               output->inputType);
 
-  // We use the InteractionContext to interpret touch gestures only and double tap with pen.
+  // We use the InteractionContext to interpret touch gestures only
-  if ((output->inputType == PT_TOUCH) ||
+  // and double tap with pen.
-      (output->inputType == PT_PEN &&
+  if ((output->inputType == PT_TOUCH
-       output->interactionId == INTERACTION_ID_TAP &&
+       && (!m_touch
-       output->arguments.tap.count == 2)) {
+           || (!m_touch->asMouse && !m_touch->canBeMouse)
+           || (output->arguments.tap.count == 2)))
+      || (output->inputType == PT_PEN &&
+          output->interactionId == INTERACTION_ID_TAP &&
+          output->arguments.tap.count == 2)) {
     ABS_CLIENT_RC(rc);
 
     gfx::Point pos(int((output->x - rc.left) / m_scale),
@@ -1362,6 +1474,57 @@ void WinWindow::handleInteractionContextOutput(
   }
 }
 
+void WinWindow::waitTimerToConvertFingerAsMouseMovement()
+{
+  ASSERT(m_touch);
+  m_touch->canBeMouse = true;
+  clearDelayedTouchEvents();
+  SetTimer(m_hwnd, m_touch->timerID = 1,
+           kFingerAsMouseTimeout, nullptr);
+  TOUCH_TRACE(" - Set timer\n");
+}
+
+void WinWindow::convertFingerAsMouseMovement()
+{
+  ASSERT(m_touch);
+  m_touch->asMouse = true;
+  sendDelayedTouchEvents();
+}
+
+void WinWindow::delegateFingerToInteractionContext()
+{
+  ASSERT(m_touch);
+  m_touch->canBeMouse = false;
+  m_touch->asMouse = false;
+  clearDelayedTouchEvents();
+  if (m_touch->timerID > 0)
+    killTouchTimer();
+}
+
+void WinWindow::sendDelayedTouchEvents()
+{
+  ASSERT(m_touch);
+  for (auto& ev : m_touch->delayedEvents)
+    queueEvent(ev);
+  clearDelayedTouchEvents();
+}
+
+void WinWindow::clearDelayedTouchEvents()
+{
+  ASSERT(m_touch);
+  m_touch->delayedEvents.clear();
+}
+
+void WinWindow::killTouchTimer()
+{
+  ASSERT(m_touch);
+  if (m_touch->timerID > 0) {
+    KillTimer(m_hwnd, m_touch->timerID);
+    m_touch->timerID = 0;
+    TOUCH_TRACE(" - Kill timer\n");
+  }
+}
+
 //static
 void WinWindow::registerClass()
 {

src/she/win/window.h

@@ -52,6 +52,7 @@ namespace she {
 
     void setLayout(const std::string& layout);
     void setTranslateDeadKeys(bool state);
+    void setInterpretOneFingerGestureAsMouseMovement(bool state);
 
     HWND handle() { return m_hwnd; }
 
@@ -64,6 +65,13 @@ namespace she {
     void handleInteractionContextOutput(
       const INTERACTION_CONTEXT_OUTPUT* output);
 
+    void waitTimerToConvertFingerAsMouseMovement();
+    void convertFingerAsMouseMovement();
+    void delegateFingerToInteractionContext();
+    void sendDelayedTouchEvents();
+    void clearDelayedTouchEvents();
+    void killTouchTimer();
+
     virtual void onQueueEvent(Event& ev) { }
     virtual void onResize(const gfx::Size& sz) { }
     virtual void onPaint(HDC hdc) { }
@@ -127,6 +135,36 @@ namespace she {
     //    messages again.
     int m_ignoreRandomMouseEvents;
 
+    // Variables used to convert one finger in mouse-like movement,
+    // and two/more fingers in scroll movement/pan/zoom. The idea
+    // is as follows:
+    // 1) When a PT_TOUCH is received, we count this event in
+    //    m_fingers and setup a timer (m_fingerTimerID) to wait for
+    //    another touch event.
+    // 2) If another touch event is received, we process the messages
+    //    with the interaction context (m_ictx) to handle special
+    //    gestures (pan, magnify, etc.).
+    // 3) If the timeout is reached, and we've received only one
+    //    finger on the windows, we can send all awaiting events
+    //    (m_fingerEvents) like mouse movement messages/button
+    //    presses/releases.
+    struct Touch {
+      int fingers;              // Number of fingers in the window
+      // True when the timeout wasn't reached yet and the finger can be
+      // converted to mouse events yet.
+      bool canBeMouse;
+      // True if we're already processing finger/touch events as mouse
+      // movement events.
+      bool asMouse;
+      // Timeout (WM_TIMER) when the finger is converted to mouse events.
+      UINT_PTR timerID;
+      // Queued events to be sent when the finger is converted to mouse
+      // events (these events are discarded if another finger is
+      // introduced in the gesture e.g. to pan)
+      std::vector<Event> delayedEvents;
+      Touch();
+    } *m_touch;
+
 #if SHE_USE_POINTER_API_FOR_MOUSE
     // Emulate double-click with pointer API. I guess that this should
     // be done by the Interaction Context API but it looks like