Added in rpng string streaming to ai service.

libretro-common/formats/png/rpng_encode.c

@@ -24,8 +24,9 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include <libretro.h>
 #include <encodings/crc32.h>
-#include <streams/file_stream.h>
+#include <streams/interface_stream.h>
 #include <streams/trans_stream.h>
 
 #include "rpng_internal.h"
@@ -37,6 +38,9 @@
    goto end; \
 } while(0)
 
+double DEFLATE_PADDING = 1.1;
+int PNG_ROUGH_HEADER = 100;
+
 static void dword_write_be(uint8_t *buf, uint32_t val)
 {
    *buf++ = (uint8_t)(val >> 24);
@@ -45,16 +49,16 @@ static void dword_write_be(uint8_t *buf, uint32_t val)
    *buf++ = (uint8_t)(val >>  0);
 }
 
-static bool png_write_crc(RFILE *file, const uint8_t *data, size_t size)
+static bool png_write_crc_string(intfstream_t *intf_s, const uint8_t *data, size_t size)
 {
    uint8_t crc_raw[4] = {0};
    uint32_t crc       = encoding_crc32(0, data, size);
 
    dword_write_be(crc_raw, crc);
-   return filestream_write(file, crc_raw, sizeof(crc_raw)) == sizeof(crc_raw);
+   return intfstream_write(intf_s, crc_raw, sizeof(crc_raw)) == sizeof(crc_raw);
 }
 
-static bool png_write_ihdr(RFILE *file, const struct png_ihdr *ihdr)
+static bool png_write_ihdr_string(intfstream_t *intf_s, const struct png_ihdr *ihdr)
 {
    uint8_t ihdr_raw[21];
 
@@ -83,32 +87,32 @@ static bool png_write_ihdr(RFILE *file, const struct png_ihdr *ihdr)
    dword_write_be(ihdr_raw +  0, sizeof(ihdr_raw) - 8);
    dword_write_be(ihdr_raw +  8, ihdr->width);
    dword_write_be(ihdr_raw + 12, ihdr->height);
-   if (filestream_write(file, ihdr_raw, sizeof(ihdr_raw)) != sizeof(ihdr_raw))
+   if (intfstream_write(intf_s, ihdr_raw, sizeof(ihdr_raw)) != sizeof(ihdr_raw))
       return false;
 
-   return png_write_crc(file, ihdr_raw + sizeof(uint32_t),
+   return png_write_crc_string(intf_s, ihdr_raw + sizeof(uint32_t),
          sizeof(ihdr_raw) - sizeof(uint32_t));
 }
 
-static bool png_write_idat(RFILE *file, const uint8_t *data, size_t size)
+static bool png_write_idat_string(intfstream_t* intf_s, const uint8_t *data, size_t size)
 {
-   if (filestream_write(file, data, size) != (ssize_t)size)
+   if (intfstream_write(intf_s, data, size) != (ssize_t)size)
       return false;
 
-   return png_write_crc(file, data + sizeof(uint32_t), size - sizeof(uint32_t));
+   return png_write_crc_string(intf_s, data + sizeof(uint32_t), size - sizeof(uint32_t));
 }
 
-static bool png_write_iend(RFILE *file)
+static bool png_write_iend_string(intfstream_t* intf_s)
 {
    const uint8_t data[] = {
       0, 0, 0, 0,
       'I', 'E', 'N', 'D',
    };
 
-   if (filestream_write(file, data, sizeof(data)) != sizeof(data))
+   if (intfstream_write(intf_s, data, sizeof(data)) != sizeof(data))
       return false;
 
-   return png_write_crc(file, data + sizeof(uint32_t),
+   return png_write_crc_string(intf_s, data + sizeof(uint32_t),
          sizeof(data) - sizeof(uint32_t));
 }
 
@@ -199,14 +203,12 @@ static unsigned filter_paeth(uint8_t *target,
    return count_sad(target, width);
 }
 
-static bool rpng_save_image(const char *path,
+bool rpng_save_image_stream(const uint8_t *data, intfstream_t* intf_s,
-      const uint8_t *data,
+      unsigned width, unsigned height, signed pitch, unsigned bpp)
-      unsigned width, unsigned height, unsigned pitch, unsigned bpp)
 {
    unsigned h;
-   bool ret = true;
    struct png_ihdr ihdr = {0};
-
+   bool ret = true;
    const struct trans_stream_backend *stream_backend = NULL;
    size_t encode_buf_size  = 0;
    uint8_t *encode_buf     = NULL;
@@ -221,22 +223,20 @@ static bool rpng_save_image(const char *path,
    void *stream            = NULL;
    uint32_t total_in       = 0;
    uint32_t total_out      = 0;
-   RFILE *file             = filestream_open(path,
+   
-         RETRO_VFS_FILE_ACCESS_WRITE,
+   if (!intf_s)
-         RETRO_VFS_FILE_ACCESS_HINT_NONE);
-   if (!file)
       GOTO_END_ERROR();
 
    stream_backend = trans_stream_get_zlib_deflate_backend();
 
-   if (filestream_write(file, png_magic, sizeof(png_magic)) != sizeof(png_magic))
+   if (intfstream_write(intf_s, png_magic, sizeof(png_magic)) != sizeof(png_magic))
       GOTO_END_ERROR();
 
    ihdr.width = width;
    ihdr.height = height;
    ihdr.depth = 8;
    ihdr.color_type = bpp == sizeof(uint32_t) ? 6 : 2; /* RGBA or RGB */
-   if (!png_write_ihdr(file, &ihdr))
+   if (!png_write_ihdr_string(intf_s, &ihdr))
       GOTO_END_ERROR();
 
    encode_buf_size = (width * bpp + 1) * height;
@@ -339,15 +339,12 @@ static bool rpng_save_image(const char *path,
 
    memcpy(deflate_buf + 4, "IDAT", 4);
    dword_write_be(deflate_buf + 0,        ((uint32_t)total_out));
-   if (!png_write_idat(file, deflate_buf, ((size_t)total_out + 8)))
+   if (!png_write_idat_string(intf_s, deflate_buf, ((size_t)total_out + 8)))
       GOTO_END_ERROR();
 
-   if (!png_write_iend(file))
+   if (!png_write_iend_string(intf_s))
       GOTO_END_ERROR();
-
 end:
-   if (file)
-      filestream_close(file);
    free(encode_buf);
    free(deflate_buf);
    free(rgba_line);
@@ -371,13 +368,78 @@ end:
 bool rpng_save_image_argb(const char *path, const uint32_t *data,
       unsigned width, unsigned height, unsigned pitch)
 {
-   return rpng_save_image(path, (const uint8_t*)data,
+   bool ret                      = false;
-         width, height, pitch, sizeof(uint32_t));
+   intfstream_t* intf_s          = NULL;
+   
+   intf_s = intfstream_open_file(path, 
+         RETRO_VFS_FILE_ACCESS_WRITE,
+         RETRO_VFS_FILE_ACCESS_HINT_NONE);
+
+   ret = rpng_save_image_stream((const uint8_t*) data, intf_s,
+                                width, height,
+                                (signed) pitch, sizeof(uint32_t));
+   intfstream_close(intf_s);
+   free(intf_s);
+   return ret;
 }
 
 bool rpng_save_image_bgr24(const char *path, const uint8_t *data,
       unsigned width, unsigned height, unsigned pitch)
 {
-   return rpng_save_image(path, (const uint8_t*)data,
+   bool ret                      = false;
-         width, height, pitch, 3);
+   intfstream_t* intf_s          = NULL;
+   
+   intf_s = intfstream_open_file(path, 
+         RETRO_VFS_FILE_ACCESS_WRITE,
+         RETRO_VFS_FILE_ACCESS_HINT_NONE);
+   ret = rpng_save_image_stream(data, intf_s, width, height, 
+                                (signed) pitch, 3);
+   intfstream_close(intf_s);
+   free(intf_s);
+   return ret;
 }
+
+
+uint8_t* rpng_save_image_bgr24_string(const uint8_t *data,
+      unsigned width, unsigned height, signed pitch, uint64_t* bytes)
+{
+   bool ret                    = false;
+   uint8_t* buf                = NULL;
+   uint8_t* output             = NULL;
+   int buf_length              = 0;
+   intfstream_t* intf_s        = NULL;
+
+   buf_length = (int)(width*height*3*DEFLATE_PADDING)+PNG_ROUGH_HEADER;
+   buf = malloc(buf_length*sizeof(uint8_t));
+   if (!buf)
+      GOTO_END_ERROR(); 
+   
+   intf_s = intfstream_open_writable_memory(buf, 
+         RETRO_VFS_FILE_ACCESS_WRITE,
+         RETRO_VFS_FILE_ACCESS_HINT_NONE,
+         buf_length);
+
+   ret = rpng_save_image_stream((const uint8_t*)data, 
+            intf_s, width, height, pitch, 3);
+
+   *bytes = intfstream_get_ptr(intf_s);
+   intfstream_rewind(intf_s);
+   output = malloc((*bytes)*sizeof(uint8_t));
+   if (!output)
+      GOTO_END_ERROR();
+   intfstream_read(intf_s, output, *bytes);
+
+end:
+   if (buf)
+      free(buf);
+   if (intf_s)
+      free(intf_s);
+   if (ret == false)
+   {
+      if (output)
+         free(output);
+      return NULL;
+   }
+   return output;
+}
+

libretro-common/include/formats/rpng.h

@@ -56,6 +56,9 @@ bool rpng_save_image_argb(const char *path, const uint32_t *data,
 bool rpng_save_image_bgr24(const char *path, const uint8_t *data,
       unsigned width, unsigned height, unsigned pitch);
 
+uint8_t* rpng_save_image_bgr24_string(const uint8_t *data,
+      unsigned width, unsigned height, signed pitch, uint64_t *bytes);
+
 RETRO_END_DECLS
 
 #endif

libretro-common/include/streams/interface_stream.h

@@ -74,6 +74,8 @@ int64_t intfstream_read(intfstream_internal_t *intf,
 int64_t intfstream_write(intfstream_internal_t *intf,
       const void *s, uint64_t len);
 
+int64_t intfstream_get_ptr(intfstream_internal_t *intf);
+
 char *intfstream_gets(intfstream_internal_t *intf,
       char *buffer, uint64_t len);
 
@@ -100,6 +102,9 @@ intfstream_t* intfstream_open_file(const char *path,
 intfstream_t *intfstream_open_memory(void *data,
       unsigned mode, unsigned hints, uint64_t size);
 
+intfstream_t *intfstream_open_writable_memory(void *data,
+      unsigned mode, unsigned hints, uint64_t size);
+
 intfstream_t *intfstream_open_chd_track(const char *path,
       unsigned mode, unsigned hints, int32_t track);
 

libretro-common/include/streams/memory_stream.h

@@ -56,6 +56,8 @@ void memstream_set_buffer(uint8_t *buffer, uint64_t size);
 
 uint64_t memstream_get_last_size(void);
 
+uint64_t memstream_get_ptr(memstream_t *stream);
+
 RETRO_END_DECLS
 
 #endif

libretro-common/streams/interface_stream.c

@@ -298,6 +298,24 @@ int64_t intfstream_write(intfstream_internal_t *intf,
    return 0;
 }
 
+int64_t intfstream_get_ptr(intfstream_internal_t* intf)
+{
+   if (!intf)
+      return 0;
+
+   switch (intf->type)
+   {
+      case INTFSTREAM_FILE:
+         return -1;
+      case INTFSTREAM_MEMORY:
+         return memstream_get_ptr(intf->memory.fp);
+      case INTFSTREAM_CHD:
+         return -1;
+   }
+
+   return 0;
+}
+
 char *intfstream_gets(intfstream_internal_t *intf,
       char *buffer, uint64_t len)
 {
@@ -441,7 +459,6 @@ intfstream_t *intfstream_open_memory(void *data,
    info.memory.writable = false;
 
    fd                   = (intfstream_t*)intfstream_init(&info);
-
    if (!fd)
       return NULL;
 
@@ -459,6 +476,37 @@ error:
    return NULL;
 }
 
+intfstream_t *intfstream_open_writable_memory(void *data,
+      unsigned mode, unsigned hints, uint64_t size)
+{
+   intfstream_info_t info;
+   intfstream_t *fd     = NULL;
+
+   info.type            = INTFSTREAM_MEMORY;
+   info.memory.buf.data = (uint8_t*)data;
+   info.memory.buf.size = size;
+   info.memory.writable = true;
+
+   fd                   = (intfstream_t*)intfstream_init(&info);
+   if (!fd)
+      return NULL;
+
+   if (!intfstream_open(fd, NULL, mode, hints))
+      goto error;
+
+   return fd;
+
+error:
+   if (fd)
+   {
+      intfstream_close(fd);
+      free(fd);
+   }
+   return NULL;
+}
+
+
+
 intfstream_t *intfstream_open_chd_track(const char *path,
       unsigned mode, unsigned hints, int32_t track)
 {

libretro-common/streams/memory_stream.c

@@ -92,6 +92,11 @@ void memstream_close(memstream_t *stream)
    free(stream);
 }
 
+uint64_t memstream_get_ptr(memstream_t *stream)
+{
+   return stream->ptr;
+}
+
 uint64_t memstream_read(memstream_t *stream, void *data, uint64_t bytes)
 {
    uint64_t avail = 0;

retroarch.c

@@ -148,6 +148,7 @@
 #ifdef HAVE_TRANSLATE
 #include <encodings/base64.h>
 #include <formats/rbmp.h>
+#include <formats/rpng.h>
 #include "translation_defines.h"
 #endif
 
@@ -2707,7 +2708,7 @@ static void handle_translation_cb(
    unsigned image_width, image_height;
    char* body_copy                   = NULL;
    uint8_t* raw_output_data          = NULL;
-   char* raw_bmp_data                = NULL;
+   char* raw_image_file_data         = NULL;
    struct scaler_ctx* scaler         = NULL;
    bool is_paused                    = runloop_paused;
    http_transfer_data_t *data        = (http_transfer_data_t*)task_data;
@@ -2716,7 +2717,9 @@ static void handle_translation_cb(
    int new_sound_size                = 0;
    const void* dummy_data            = NULL;
    void* raw_image_data              = NULL;
+   void* raw_image_data_alpha        = NULL;
    void* raw_sound_data              = NULL;  
+   int retval                        = 0;
    settings_t *settings              = configuration_settings;
 
    int i                             = 0;
@@ -2737,7 +2740,6 @@ static void handle_translation_cb(
    data->data[data->len] = '\0'; 
 
    /* Parse JSON body for the image and sound data */
-
    body_copy = strdup(data->data);
 
    while (true)
@@ -2756,7 +2758,7 @@ static void handle_translation_cb(
             *(found_string+i-start-1) = '\0';
             if (curr_state == 1)/*image*/
             {
-              raw_bmp_data = (char*)unbase64(found_string, 
+              raw_image_file_data = (char*)unbase64(found_string, 
                     strlen(found_string),
                     &new_image_size);
               curr_state = 0;
@@ -2788,31 +2790,91 @@ static void handle_translation_cb(
    if (found_string)
        free(found_string);
 
-   if (!raw_bmp_data && !raw_sound_data)
+   if (!raw_image_file_data && !raw_sound_data)
    {
       error = "Invalid JSON body.";
       goto finish;
    }
 
-   if (raw_bmp_data)
+   if (raw_image_file_data)
    { 
       /* Get the video frame dimensions reference */
       video_driver_cached_frame_get(&dummy_data, &width, &height, &pitch);
 
+      if (raw_image_file_data[0] == 'B' && raw_image_file_data[1] == 'M')
+      {
+         /* This is a BMP file coming back. */
          /* Get image data (24 bit), and convert to the emulated pixel format */
          image_width    = 
-         ((uint32_t) ((uint8_t)raw_bmp_data[21]) << 24) +
+            ((uint32_t) ((uint8_t)raw_image_file_data[21]) << 24) +
-         ((uint32_t) ((uint8_t)raw_bmp_data[20]) << 16) +
+            ((uint32_t) ((uint8_t)raw_image_file_data[20]) << 16) +
-         ((uint32_t) ((uint8_t)raw_bmp_data[19]) << 8) +
+            ((uint32_t) ((uint8_t)raw_image_file_data[19]) << 8) +
-         ((uint32_t) ((uint8_t)raw_bmp_data[18]) << 0);
+            ((uint32_t) ((uint8_t)raw_image_file_data[18]) << 0);
 
          image_height   = 
-         ((uint32_t) ((uint8_t)raw_bmp_data[25]) << 24) +
+            ((uint32_t) ((uint8_t)raw_image_file_data[25]) << 24) +
-         ((uint32_t) ((uint8_t)raw_bmp_data[24]) << 16) +
+            ((uint32_t) ((uint8_t)raw_image_file_data[24]) << 16) +
-         ((uint32_t) ((uint8_t)raw_bmp_data[23]) << 8) +
+            ((uint32_t) ((uint8_t)raw_image_file_data[23]) << 8) +
-         ((uint32_t) ((uint8_t)raw_bmp_data[22]) << 0);
+            ((uint32_t) ((uint8_t)raw_image_file_data[22]) << 0);
-      raw_image_data = raw_bmp_data + 54 * sizeof(uint8_t);
+         raw_image_data = raw_image_file_data + 54 * sizeof(uint8_t);
+      }
+      else if (raw_image_file_data[1] == 'P' && raw_image_file_data[2] == 'N' &&
+               raw_image_file_data[3] == 'G')
+      {
+         /* PNG coming back from the url */
+         image_width = 
+             ((uint32_t) ((uint8_t)raw_image_file_data[16])<<24)+
+             ((uint32_t) ((uint8_t)raw_image_file_data[17])<<16)+
+             ((uint32_t) ((uint8_t)raw_image_file_data[18])<<8)+
+             ((uint32_t) ((uint8_t)raw_image_file_data[19])<<0);
+         image_height = 
+             ((uint32_t) ((uint8_t)raw_image_file_data[20])<<24)+
+             ((uint32_t) ((uint8_t)raw_image_file_data[21])<<16)+
+             ((uint32_t) ((uint8_t)raw_image_file_data[22])<<8)+
+             ((uint32_t) ((uint8_t)raw_image_file_data[23])<<0);
         
+         rpng_t *rpng = rpng_alloc();
+         if (!rpng)
+         {
+            error = "Can't allocate memory.";
+            goto finish;
+         }
+         rpng_set_buf_ptr(rpng, raw_image_file_data, new_image_size);
+         rpng_start(rpng);
+         while (rpng_iterate_image(rpng));
+
+         do
+         {
+            retval = rpng_process_image(rpng, &raw_image_data_alpha, new_image_size, &image_width, &image_height);
+         }
+         while(retval == IMAGE_PROCESS_NEXT);
+      
+         /* Returned output from the png processor is an upside down RGBA
+          * image, so we have to change that to RGB first.  This should 
+          * probably be replaced with a scaler call.*/
+         {
+            int d,tw,th,tc;
+            d=0;
+            raw_image_data = malloc(image_width*image_height*3*sizeof(uint8_t));
+            for (i=0;i<image_width*image_height*4;i++)
+            {
+               if (i%4 != 3)
+               {
+                  tc = d%3;
+                  th = image_height-d/(3*image_width)-1;
+                  tw = (d%(image_width*3))/3;
+                  ((uint8_t*) raw_image_data)[tw*3+th*3*image_width+tc] = ((uint8_t *)raw_image_data_alpha)[i];
+                  d+=1;
+               }
+            }
+         }
+         rpng_free(rpng);
+      }
+      else
+      {
+         RARCH_LOG("Output from URL not a valid file type, or is not supported.\n");
+         goto finish;
+      }
       scaler = (struct scaler_ctx*)calloc(1, sizeof(struct scaler_ctx));
       if (!scaler)
          goto finish;
@@ -2920,10 +2982,12 @@ finish:
 
    if (body_copy)
       free(body_copy);
-   
+   if (raw_image_file_data)
-   if (raw_bmp_data)
+      free(raw_image_file_data);
-      free(raw_bmp_data);
+   if (raw_image_data_alpha)
-
+       free(raw_image_data_alpha);
+   if (raw_image_data)
+      free(raw_image_data);
    if (scaler)
       free(scaler);
  
@@ -3093,7 +3157,8 @@ static const char *ai_service_get_str(enum translation_lang id)
    will consist of a JSON body, with the "image" field as a base64
    encoded string of a 24bit-BMP that the will be translated.  The server
    must output the translated image in the form of a JSON body, with
-   the "image" field also as a base64 encoded, 24bit-BMP.
+   the "image" field also as a base64 encoded 24bit-BMP, or
+   as an alpha channel png.
    */
 static bool run_translation_service(void)
 {
@@ -3111,13 +3176,14 @@ static bool run_translation_service(void)
    bool error                            = false;
 
    uint8_t *bmp_buffer                   = NULL;
+   uint64_t buffer_bytes                 = 0;
    char *bmp64_buffer                    = NULL;
    char *json_buffer                     = NULL;
 
    int out_length                        = 0;
    const char *rf1                       = "{\"image\": \"";
    const char *rf2                       = "\"}\0";
-   
+   bool TRANSLATE_USE_BMP                = false;
    if (!scaler)
       goto finish;
 
@@ -3156,8 +3222,9 @@ static bool run_translation_service(void)
       if (!video_driver_read_viewport(bit24_image_prev, false))
          goto finish;
 
-      /* Rescale down to regular resolution */
+      /* TODO: Rescale down to regular resolution */
-
+      width = vp.width;
+      height = vp.height;
       bit24_image      = bit24_image_prev;
       bit24_image_prev = NULL;
    }
@@ -3198,6 +3265,8 @@ static bool run_translation_service(void)
      an array to contain the BMP image along with the BMP header as bytes,
      and then covert that to a b64 encoded array for transport in JSON.
    */
+   if (TRANSLATE_USE_BMP)
+   {
       form_bmp_header(header, width, height, false);
       bmp_buffer = (uint8_t*)malloc(width * height * 3+54);
       if (!bmp_buffer)
@@ -3205,9 +3274,17 @@ static bool run_translation_service(void)
 
       memcpy(bmp_buffer, header, 54*sizeof(uint8_t));
       memcpy(bmp_buffer+54, bit24_image, width * height * 3 * sizeof(uint8_t));
+      buffer_bytes = sizeof(uint8_t)*(width*height*3+54);
+   }
+   else
+   {
+      pitch = width*3;
+      bmp_buffer = rpng_save_image_bgr24_string(bit24_image+width*(height-1)*3, width, height, -pitch, &buffer_bytes);
+   }
 
-   bmp64_buffer = base64((void *)bmp_buffer, (int)(width * height * 3 + 54),
+   bmp64_buffer = base64((void *)bmp_buffer, sizeof(uint8_t)*buffer_bytes,
          &out_length);
+
    if (!bmp64_buffer)
       goto finish;
 
@@ -3270,10 +3347,12 @@ static bool run_translation_service(void)
       /* mode */
       {
          char temp_string[PATH_MAX_LENGTH];         
-         char* mode_chr = "image";
 
+         /*"image" is included for backwards compatability with
+          * vgtranslate < 1.04 */
+         char* mode_chr = "image,png";
          if (settings->uints.ai_service_mode == 1)
-            mode_chr = "sound";
+            mode_chr = "sound,wav";
          
          snprintf(temp_string,
                sizeof(temp_string),
@@ -3284,8 +3363,6 @@ static bool run_translation_service(void)
                  sizeof(new_ai_service_url));
       }
 
-
-
       RARCH_LOG("Server URL:  %s\n", new_ai_service_url);
       task_push_http_post_transfer(new_ai_service_url, 
             json_buffer, true, NULL, handle_translation_cb, NULL);