Rework camera interface.

Hook up a more proper interface for libretro. Still very experimental.

camera/video4linux2.c

@@ -323,7 +323,7 @@ static int init_device(void *data)
    return init_mmap(v4l);
 }
 
-static int v4l_stop(void *data)
+static void v4l_stop(void *data)
 {
    enum v4l2_buf_type type;
    video4linux_t *v4l = (video4linux_t*)data;
@@ -331,16 +331,12 @@ static int v4l_stop(void *data)
    type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
 
    if (xioctl(v4l->fd, VIDIOC_STREAMOFF, &type) == -1)
-   {
       RARCH_ERR("Error - VIDIOC_STREAMOFF.\n");
-      return -1;
-   }
 
    v4l->ready = false;
-   return 0;
 }
 
-static int v4l_start(void *data)
+static bool v4l_start(void *data)
 {
    video4linux_t *v4l = (video4linux_t*)data;
    unsigned i;
@@ -359,7 +355,7 @@ static int v4l_start(void *data)
       if (xioctl(v4l->fd, VIDIOC_QBUF, &buf) == -1)
       {
          RARCH_ERR("Error - VIDIOC_QBUF.\n");
-         return -1;
+         return false;
       }
    }
 
@@ -368,21 +364,25 @@ static int v4l_start(void *data)
    if (xioctl(v4l->fd, VIDIOC_STREAMON, &type) == -1)
    {
       RARCH_ERR("Error - VIDIOC_STREAMON.\n");
-      return -1;
+      return false;
    }
 
    generate_YCbCr_to_RGB_lookup();
    v4l->ready = true;
 
-   return 0;
+   return true;
 }
 
-static void *v4l_init(const char *device, unsigned width, unsigned height)
+static void *v4l_init(const char *device, uint64_t caps, unsigned width, unsigned height)
 {
-   (void)width;
-   (void)height;
-
    struct stat st;
+
+   if (!(caps & RETRO_CAMERA_BUFFER_RAW_FRAMEBUFFER))
+   {
+      RARCH_ERR("video4linux2 returns raw framebuffers.\n");
+      return NULL;
+   }
+
    video4linux_t *v4l = (video4linux_t*)calloc(1, sizeof(video4linux_t));
    if (!v4l)
       return NULL;
@@ -392,8 +392,8 @@ static void *v4l_init(const char *device, unsigned width, unsigned height)
    else
       strlcpy(v4l->dev_name, device, sizeof(v4l->dev_name));
 
-   v4l->width    = 640; //FIXME - use width param
+   v4l->width    = width;
-   v4l->height   = 480; //FIXME - use height param
+   v4l->height   = height;
    v4l->ready    = false;
 
    if (stat(v4l->dev_name, &st) == -1)
@@ -445,7 +445,7 @@ static void v4l_free(void *data)
    YCbCr_to_RGB = NULL;
 }
 
-static void preprocess_image(void *data)
+static bool preprocess_image(void *data)
 {
    video4linux_t *v4l = (video4linux_t*)data;
    struct v4l2_buffer buf;
@@ -461,7 +461,7 @@ static void preprocess_image(void *data)
       switch (errno)
       {
          case EAGAIN:
-            return;
+            return false;
          case EIO:
             /* Could ignore EIO, see spec. */
 
@@ -469,7 +469,7 @@ static void preprocess_image(void *data)
 
          default:
             RARCH_ERR("VIDIOC_DQBUF.\n");
-            return;
+            return false;
       }
    }
 
@@ -479,37 +479,27 @@ static void preprocess_image(void *data)
 
    if (xioctl(v4l->fd, VIDIOC_QBUF, &buf) == -1)
       RARCH_ERR("VIDIOC_QBUF\n");
+
+   return true;
 }
 
-static void v4l_texture_image_2d(void *data)
+static bool v4l_poll(void *data, retro_camera_frame_raw_framebuffer_t frame_raw_cb,
-{
+      retro_camera_frame_opengl_texture_t frame_gl_cb)
-   preprocess_image(data);
-}
-
-static void v4l_texture_subimage_2d(void *data)
-{
-   preprocess_image(data);
-}
-
-static bool v4l_ready(void *data, unsigned *width, unsigned *height)
 {
    video4linux_t *v4l = (video4linux_t*)data;
-   return v4l->ready;
+   if (!v4l->ready)
-}
+      return false;
 
-static uint64_t v4l_set_capabilities(void *data, uint64_t state)
+   (void)frame_raw_cb;
-{
+   (void)frame_gl_cb;
-   (void)data;
-   uint64_t ret = 0;
 
-   //FIXME - set when driver supports this
+   if (preprocess_image(data))
-   //if (state & (1 << RETRO_CAMERA_RECV_GL_TEXTURE))
+   {
-      //ret |= (1 << RETRO_CAMERA_RECV_GL_TEXTURE);
+      // TODO: Call frame_raw_cb() here with updated data if new data was processed.
-
+      return true;
-   if (state & (1 << RETRO_CAMERA_RECV_RAW_FRAMEBUFFER))
+   }
-      ret |= (1 << RETRO_CAMERA_RECV_RAW_FRAMEBUFFER);
+   else
-
+      return false;
-   return ret;
 }
 
 const camera_driver_t camera_v4l2 = {
@@ -517,9 +507,7 @@ const camera_driver_t camera_v4l2 = {
    v4l_free,
    v4l_start,
    v4l_stop,
-   v4l_ready,
+   v4l_poll,
-   v4l_texture_image_2d,
-   v4l_texture_subimage_2d,
-   v4l_set_capabilities,
    "video4linux2",
 };
+

driver.c

@@ -473,6 +473,32 @@ bool driver_set_sensor_state(unsigned port, enum retro_sensor_action action, uns
       return false;
 }
 
+#ifdef HAVE_CAMERA
+bool driver_camera_start(void)
+{
+   if (driver.camera && driver.camera_data)
+      return driver.camera->start(driver.camera_data);
+   else
+      return false;
+}
+
+void driver_camera_stop(void)
+{
+   if (driver.camera && driver.camera_data)
+      driver.camera->stop(driver.camera_data);
+}
+
+void driver_camera_poll(void)
+{
+   if (driver.camera && driver.camera_data)
+   {
+      driver.camera->poll(driver.camera_data,
+            g_extern.system.camera_callback.frame_raw_framebuffer,
+            g_extern.system.camera_callback.frame_opengl_texture);
+   }
+}
+#endif
+
 uintptr_t driver_get_current_framebuffer(void)
 {
 #ifdef HAVE_FBO
@@ -543,8 +569,11 @@ void init_camera(void)
 
    find_camera_driver();
 
-   driver.camera_data = camera_init_func(*g_settings.camera.device ? g_settings.camera.device : NULL,
+   driver.camera_data = camera_init_func(
-         g_settings.camera.width, g_settings.camera.height);
+         *g_settings.camera.device ? g_settings.camera.device : NULL,
+         g_extern.system.camera_callback.caps,
+         g_settings.camera.width ? g_settings.camera.width : g_extern.system.camera_callback.width,
+         g_settings.camera.height ? g_settings.camera.height : g_extern.system.camera_callback.height);
 
    if (!driver.camera_data)
    {
@@ -560,7 +589,7 @@ void init_drivers(void)
    driver.audio_data_own = !driver.audio_data;
    driver.input_data_own = !driver.input_data;
 #ifdef HAVE_CAMERA
-   driver.camera_data_own = !driver.camera_data_own;
+   driver.camera_data_own = !driver.camera_data;
 #endif
 
    adjust_system_rates();
@@ -575,7 +604,9 @@ void init_drivers(void)
    init_audio();
 
 #ifdef HAVE_CAMERA
-   init_camera();
+   // Only init camera driver if we're ever going to use it.
+   if (g_extern.system.camera_callback.caps)
+      init_camera();
 #endif
 
    // Keep non-throttled state as good as possible.

driver.h

@@ -333,16 +333,21 @@ typedef struct input_driver
 
 typedef struct camera_driver
 {
-   //FIXME - params for init - queries for resolution, framerate, color format
+   // FIXME: params for init - queries for resolution, framerate, color format
-   //which might or might not be honored
+   // which might or might not be honored
-   void *(*init)(const char *device, unsigned width, unsigned height);
+   void *(*init)(const char *device, uint64_t buffer_types, unsigned width, unsigned height);
    void (*free)(void *data);
-   int  (*start)(void *data);
+
-   int  (*stop)(void *data);
+   bool (*start)(void *data);
-   bool (*ready)(void *data, unsigned *width, unsigned *height);
+   void (*stop)(void *data);
-   void (*texture_image_2d)(void *data);
+
-   void (*texture_subimage_2d)(void *data);
+   // Polls the camera driver.
-   uint64_t (*set_capabilities)(void *data, uint64_t mask);
+   // Will call the appropriate callback if a new frame is ready.
+   // Returns true if a new frame was handled.
+   bool (*poll)(void *data,
+         retro_camera_frame_raw_framebuffer_t frame_raw_cb,
+         retro_camera_frame_opengl_texture_t frame_gl_cb);
+
    const char *ident;
 } camera_driver_t;
 
@@ -536,6 +541,13 @@ bool driver_set_rumble_state(unsigned port, enum retro_rumble_effect effect, uin
 // Used by RETRO_ENVIRONMENT_GET_SENSOR_INTERFACE
 bool driver_set_sensor_state(unsigned port, enum retro_sensor_action action, unsigned rate);
 
+// Used by RETRO_ENVIRONMENT_GET_CAMERA_INTERFACE
+#ifdef HAVE_CAMERA
+bool driver_camera_start(void);
+void driver_camera_stop(void);
+void driver_camera_poll(void);
+#endif
+
 extern driver_t driver;
 
 //////////////////////////////////////////////// Backends

driver_funcs.h

@@ -18,7 +18,7 @@
 #ifndef _RARCH_DRIVER_FUNCS_H
 #define _RARCH_DRIVER_FUNCS_H
 
-#define camera_init_func(device, width, height) driver.camera->init(device, width, height)
+#define camera_init_func(device, caps, width, height) driver.camera->init(device, caps, width, height)
 
 #define audio_init_func(device, rate, latency)  driver.audio->init(device, rate, latency)
 #define audio_write_func(buf, size)             driver.audio->write(driver.audio_data, buf, size)

dynamic.c

@@ -784,15 +784,13 @@ bool rarch_environment_cb(unsigned cmd, void *data)
       }
 
 #ifdef HAVE_CAMERA
-      case RETRO_ENVIRONMENT_SET_CAMERA_RETRIEVE:
+      case RETRO_ENVIRONMENT_GET_CAMERA_INTERFACE:
       {
-         RARCH_LOG("Environ SET_CAMERA_RETRIEVE.\n");
+         RARCH_LOG("Environ GET_CAMERA_INTERFACE.\n");
-         uint64_t *mask_ptr = (uint64_t*)data;
+         struct retro_camera_callback *cb = (struct retro_camera_callback*)data;
-         uint64_t mask = *mask_ptr;
+         cb->start = driver_camera_start;
-         if (driver.camera)
+         cb->stop = driver_camera_stop;
-            *mask_ptr = driver.camera->set_capabilities(driver.camera_data, mask);
+         g_extern.system.camera_callback = *cb;
-         else
-            *mask_ptr = 0;
          break;
       }
 #endif

general.h

@@ -405,6 +405,7 @@ struct global
 
       struct retro_disk_control_callback disk_control; 
       struct retro_hw_render_callback hw_render_callback;
+      struct retro_camera_callback camera_callback;
 
       struct retro_frame_time_callback frame_time;
       retro_usec_t frame_time_last;

libretro.h

@@ -446,8 +446,6 @@ enum retro_mod
                                            // swapped out by the user (e.g. PSX).
 #define RETRO_ENVIRONMENT_SET_HW_RENDER 14
                                            // struct retro_hw_render_callback * --
-                                           // NOTE: This call is currently very experimental, and should not be considered part of the public API.
-                                           // The interface could be changed or removed at any time.
                                            // Sets an interface to let a libretro core render with hardware acceleration.
                                            // Should be called in retro_load_game().
                                            // If successful, libretro cores will be able to render to a frontend-provided framebuffer.
@@ -541,12 +539,23 @@ enum retro_mod
                                            // The purpose of this interface is to allow
                                            // setting state related to sensors such as polling rate, enabling/disable it entirely, etc.
                                            // Reading sensor state is done via the normal input_state_callback API.
-#define RETRO_ENVIRONMENT_SET_CAMERA_RETRIEVE (26 | RETRO_ENVIRONMENT_EXPERIMENTAL)
+#define RETRO_ENVIRONMENT_GET_CAMERA_INTERFACE (26 | RETRO_ENVIRONMENT_EXPERIMENTAL)
-                                           // uint64_t * --
+                                           // struct retro_camera_interface * --
-                                           // Sends a bitmask value to the camera driver, telling it which receive modes are expected to be handled by the 
+                                           // Gets an interface to a video camera driver.
-                                           // camera interface._
+                                           // A libretro core can use this interface to get access to a video camera.
-                                           // Example bitmask: caps = (1 << RETRO_CAMERA_RECV_GL_TEXTURE) | (1 << RETRO_CAMERA_RECV_RAW_FRAMEBUFFER).
+                                           // New video frames are delivered in a callback in same thread as retro_run().
-                                           // Returns a bitmask value that tells which camera retrieval modes have been set by the driver.
+                                           //
+                                           // GET_CAMERA_INTERFACE should be called in retro_load_game().
+                                           //
+                                           // Depending on the camera implementation used, camera frames will be delivered as a raw framebuffer,
+                                           // or as an OpenGL texture directly.
+                                           //
+                                           // The core has to tell the frontend here which types of buffers can be handled properly.
+                                           // An OpenGL texture can only be handled when using a libretro GL core (SET_HW_RENDER).
+                                           // It is recommended to use a libretro GL core when using camera interface.
+                                           //
+                                           // The camera is not started automatically. The retrieved start/stop functions must be used to explicitly
+                                           // start and stop the camera driver.
 
 // FIXME: Document the sensor API and work out behavior.
 // It will be marked as experimental until then.
@@ -558,14 +567,6 @@ enum retro_sensor_action
    RETRO_SENSOR_DUMMY = INT_MAX
 };
 
-enum retro_camera_mode
-{
-   RETRO_CAMERA_RECV_GL_TEXTURE = 0,
-   RETRO_CAMERA_RECV_RAW_FRAMEBUFFER,
-
-   RETRO_CAMERA_DUMMY = INT_MAX
-};
-
 typedef bool (*retro_set_sensor_state_t)(unsigned port, enum retro_sensor_action action, unsigned rate);
 struct retro_sensor_interface
 {
@@ -573,6 +574,48 @@ struct retro_sensor_interface
 };
 ////
 
+enum retro_camera_buffer
+{
+   RETRO_CAMERA_BUFFER_OPENGL_TEXTURE = 0,
+   RETRO_CAMERA_BUFFER_RAW_FRAMEBUFFER,
+
+   RETRO_CAMERA_BUFFER_DUMMY = INT_MAX
+};
+
+// Starts the camera driver. Can only be called in retro_run().
+typedef bool (*retro_camera_start_t)(void);
+// Stops the camera driver. Can only be called in retro_run().
+typedef void (*retro_camera_stop_t)(void);
+// A callback for raw framebuffer data. buffer points to an XRGB8888 buffer.
+// Width, height and pitch are similar to retro_video_refresh_t.
+// First pixel is top-left origin.
+typedef void (*retro_camera_frame_raw_framebuffer_t)(const uint32_t *buffer, unsigned width, unsigned height, size_t pitch);
+// A callback for when OpenGL textures are used.
+//
+// texture_id is a texture owned by camera driver.
+// Its state or content should be considered immutable, except for things like texture filtering and clamping.
+//
+// texture_target is the texture target for the GL texture.
+// These can include e.g. GL_TEXTURE_2D, GL_TEXTURE_RECTANGLE, and possibly more depending on extensions.
+// 
+// affine points to a packed 3x3 column-major matrix used to apply an affine transform to texture coordinates. (affine_matrix * vec3(coord_x, coord_y, 1.0))
+// After transform, normalized texture coord (0, 0) should be bottom-left and (1, 1) should be top-right (or (width, height) for RECTANGLE).
+//
+// GL-specific typedefs are avoided here to avoid relying on gl.h in the API definition.
+typedef void (*retro_camera_frame_opengl_texture_t)(unsigned texture_id, unsigned texture_target, const float *affine);
+struct retro_camera_callback
+{
+   uint64_t caps; // Set by libretro core. Example bitmask: caps = (1 << RETRO_CAMERA_BUFFER_OPENGL_TEXTURE) | (1 << RETRO_CAMERA_BUFFER_RAW_FRAMEBUFFER).
+
+   unsigned width; // Desired resolution for camera. Is only used as a hint.
+   unsigned height;
+   retro_camera_start_t start; // Set by frontend.
+   retro_camera_stop_t stop; // Set by frontend.
+
+   retro_camera_frame_raw_framebuffer_t frame_raw_framebuffer; // Set by libretro core if raw framebuffer callbacks will be used.
+   retro_camera_frame_opengl_texture_t frame_opengl_texture; // Set by libretro core if OpenGL texture callbacks will be used.
+};
+
 enum retro_rumble_effect
 {
    RETRO_RUMBLE_STRONG = 0,

retroarch.c

@@ -3135,6 +3135,11 @@ bool rarch_main_iterate(void)
       bsv_movie_set_frame_start(g_extern.bsv.movie);
 #endif
 
+#ifdef HAVE_CAMERA
+   if (g_extern.system.camera_callback.caps)
+      driver_camera_poll();
+#endif
+
    update_frame_time();
    pretro_run();
    limit_frame_time();