Added in rpng string streaming to ai service.

This commit is contained in:
Barry Rowe 2019-09-21 19:18:00 -07:00
parent 9f0f53b1d7
commit 4e155dbc8e
7 changed files with 275 additions and 73 deletions

View File

@ -24,8 +24,9 @@
#include <stdlib.h>
#include <string.h>
#include <libretro.h>
#include <encodings/crc32.h>
#include <streams/file_stream.h>
#include <streams/interface_stream.h>
#include <streams/trans_stream.h>
#include "rpng_internal.h"
@ -37,6 +38,9 @@
goto end; \
} while(0)
double DEFLATE_PADDING = 1.1;
int PNG_ROUGH_HEADER = 100;
static void dword_write_be(uint8_t *buf, uint32_t val)
{
*buf++ = (uint8_t)(val >> 24);
@ -45,16 +49,16 @@ static void dword_write_be(uint8_t *buf, uint32_t val)
*buf++ = (uint8_t)(val >> 0);
}
static bool png_write_crc(RFILE *file, const uint8_t *data, size_t size)
static bool png_write_crc_string(intfstream_t *intf_s, const uint8_t *data, size_t size)
{
uint8_t crc_raw[4] = {0};
uint32_t crc = encoding_crc32(0, data, size);
dword_write_be(crc_raw, crc);
return filestream_write(file, crc_raw, sizeof(crc_raw)) == sizeof(crc_raw);
return intfstream_write(intf_s, crc_raw, sizeof(crc_raw)) == sizeof(crc_raw);
}
static bool png_write_ihdr(RFILE *file, const struct png_ihdr *ihdr)
static bool png_write_ihdr_string(intfstream_t *intf_s, const struct png_ihdr *ihdr)
{
uint8_t ihdr_raw[21];
@ -83,32 +87,32 @@ static bool png_write_ihdr(RFILE *file, const struct png_ihdr *ihdr)
dword_write_be(ihdr_raw + 0, sizeof(ihdr_raw) - 8);
dword_write_be(ihdr_raw + 8, ihdr->width);
dword_write_be(ihdr_raw + 12, ihdr->height);
if (filestream_write(file, ihdr_raw, sizeof(ihdr_raw)) != sizeof(ihdr_raw))
if (intfstream_write(intf_s, ihdr_raw, sizeof(ihdr_raw)) != sizeof(ihdr_raw))
return false;
return png_write_crc(file, ihdr_raw + sizeof(uint32_t),
return png_write_crc_string(intf_s, ihdr_raw + sizeof(uint32_t),
sizeof(ihdr_raw) - sizeof(uint32_t));
}
static bool png_write_idat(RFILE *file, const uint8_t *data, size_t size)
static bool png_write_idat_string(intfstream_t* intf_s, const uint8_t *data, size_t size)
{
if (filestream_write(file, data, size) != (ssize_t)size)
if (intfstream_write(intf_s, data, size) != (ssize_t)size)
return false;
return png_write_crc(file, data + sizeof(uint32_t), size - sizeof(uint32_t));
return png_write_crc_string(intf_s, data + sizeof(uint32_t), size - sizeof(uint32_t));
}
static bool png_write_iend(RFILE *file)
static bool png_write_iend_string(intfstream_t* intf_s)
{
const uint8_t data[] = {
0, 0, 0, 0,
'I', 'E', 'N', 'D',
};
if (filestream_write(file, data, sizeof(data)) != sizeof(data))
if (intfstream_write(intf_s, data, sizeof(data)) != sizeof(data))
return false;
return png_write_crc(file, data + sizeof(uint32_t),
return png_write_crc_string(intf_s, data + sizeof(uint32_t),
sizeof(data) - sizeof(uint32_t));
}
@ -199,14 +203,12 @@ static unsigned filter_paeth(uint8_t *target,
return count_sad(target, width);
}
static bool rpng_save_image(const char *path,
const uint8_t *data,
unsigned width, unsigned height, unsigned pitch, unsigned bpp)
bool rpng_save_image_stream(const uint8_t *data, intfstream_t* intf_s,
unsigned width, unsigned height, signed pitch, unsigned bpp)
{
unsigned h;
bool ret = true;
struct png_ihdr ihdr = {0};
bool ret = true;
const struct trans_stream_backend *stream_backend = NULL;
size_t encode_buf_size = 0;
uint8_t *encode_buf = NULL;
@ -221,22 +223,20 @@ static bool rpng_save_image(const char *path,
void *stream = NULL;
uint32_t total_in = 0;
uint32_t total_out = 0;
RFILE *file = filestream_open(path,
RETRO_VFS_FILE_ACCESS_WRITE,
RETRO_VFS_FILE_ACCESS_HINT_NONE);
if (!file)
if (!intf_s)
GOTO_END_ERROR();
stream_backend = trans_stream_get_zlib_deflate_backend();
if (filestream_write(file, png_magic, sizeof(png_magic)) != sizeof(png_magic))
if (intfstream_write(intf_s, png_magic, sizeof(png_magic)) != sizeof(png_magic))
GOTO_END_ERROR();
ihdr.width = width;
ihdr.height = height;
ihdr.depth = 8;
ihdr.color_type = bpp == sizeof(uint32_t) ? 6 : 2; /* RGBA or RGB */
if (!png_write_ihdr(file, &ihdr))
if (!png_write_ihdr_string(intf_s, &ihdr))
GOTO_END_ERROR();
encode_buf_size = (width * bpp + 1) * height;
@ -339,15 +339,12 @@ static bool rpng_save_image(const char *path,
memcpy(deflate_buf + 4, "IDAT", 4);
dword_write_be(deflate_buf + 0, ((uint32_t)total_out));
if (!png_write_idat(file, deflate_buf, ((size_t)total_out + 8)))
if (!png_write_idat_string(intf_s, deflate_buf, ((size_t)total_out + 8)))
GOTO_END_ERROR();
if (!png_write_iend(file))
if (!png_write_iend_string(intf_s))
GOTO_END_ERROR();
end:
if (file)
filestream_close(file);
free(encode_buf);
free(deflate_buf);
free(rgba_line);
@ -371,13 +368,78 @@ end:
bool rpng_save_image_argb(const char *path, const uint32_t *data,
unsigned width, unsigned height, unsigned pitch)
{
return rpng_save_image(path, (const uint8_t*)data,
width, height, pitch, sizeof(uint32_t));
bool ret = false;
intfstream_t* intf_s = NULL;
intf_s = intfstream_open_file(path,
RETRO_VFS_FILE_ACCESS_WRITE,
RETRO_VFS_FILE_ACCESS_HINT_NONE);
ret = rpng_save_image_stream((const uint8_t*) data, intf_s,
width, height,
(signed) pitch, sizeof(uint32_t));
intfstream_close(intf_s);
free(intf_s);
return ret;
}
bool rpng_save_image_bgr24(const char *path, const uint8_t *data,
unsigned width, unsigned height, unsigned pitch)
{
return rpng_save_image(path, (const uint8_t*)data,
width, height, pitch, 3);
bool ret = false;
intfstream_t* intf_s = NULL;
intf_s = intfstream_open_file(path,
RETRO_VFS_FILE_ACCESS_WRITE,
RETRO_VFS_FILE_ACCESS_HINT_NONE);
ret = rpng_save_image_stream(data, intf_s, width, height,
(signed) pitch, 3);
intfstream_close(intf_s);
free(intf_s);
return ret;
}
uint8_t* rpng_save_image_bgr24_string(const uint8_t *data,
unsigned width, unsigned height, signed pitch, uint64_t* bytes)
{
bool ret = false;
uint8_t* buf = NULL;
uint8_t* output = NULL;
int buf_length = 0;
intfstream_t* intf_s = NULL;
buf_length = (int)(width*height*3*DEFLATE_PADDING)+PNG_ROUGH_HEADER;
buf = malloc(buf_length*sizeof(uint8_t));
if (!buf)
GOTO_END_ERROR();
intf_s = intfstream_open_writable_memory(buf,
RETRO_VFS_FILE_ACCESS_WRITE,
RETRO_VFS_FILE_ACCESS_HINT_NONE,
buf_length);
ret = rpng_save_image_stream((const uint8_t*)data,
intf_s, width, height, pitch, 3);
*bytes = intfstream_get_ptr(intf_s);
intfstream_rewind(intf_s);
output = malloc((*bytes)*sizeof(uint8_t));
if (!output)
GOTO_END_ERROR();
intfstream_read(intf_s, output, *bytes);
end:
if (buf)
free(buf);
if (intf_s)
free(intf_s);
if (ret == false)
{
if (output)
free(output);
return NULL;
}
return output;
}

View File

@ -56,6 +56,9 @@ bool rpng_save_image_argb(const char *path, const uint32_t *data,
bool rpng_save_image_bgr24(const char *path, const uint8_t *data,
unsigned width, unsigned height, unsigned pitch);
uint8_t* rpng_save_image_bgr24_string(const uint8_t *data,
unsigned width, unsigned height, signed pitch, uint64_t *bytes);
RETRO_END_DECLS
#endif

View File

@ -74,6 +74,8 @@ int64_t intfstream_read(intfstream_internal_t *intf,
int64_t intfstream_write(intfstream_internal_t *intf,
const void *s, uint64_t len);
int64_t intfstream_get_ptr(intfstream_internal_t *intf);
char *intfstream_gets(intfstream_internal_t *intf,
char *buffer, uint64_t len);
@ -100,6 +102,9 @@ intfstream_t* intfstream_open_file(const char *path,
intfstream_t *intfstream_open_memory(void *data,
unsigned mode, unsigned hints, uint64_t size);
intfstream_t *intfstream_open_writable_memory(void *data,
unsigned mode, unsigned hints, uint64_t size);
intfstream_t *intfstream_open_chd_track(const char *path,
unsigned mode, unsigned hints, int32_t track);

View File

@ -56,6 +56,8 @@ void memstream_set_buffer(uint8_t *buffer, uint64_t size);
uint64_t memstream_get_last_size(void);
uint64_t memstream_get_ptr(memstream_t *stream);
RETRO_END_DECLS
#endif

View File

@ -298,6 +298,24 @@ int64_t intfstream_write(intfstream_internal_t *intf,
return 0;
}
int64_t intfstream_get_ptr(intfstream_internal_t* intf)
{
if (!intf)
return 0;
switch (intf->type)
{
case INTFSTREAM_FILE:
return -1;
case INTFSTREAM_MEMORY:
return memstream_get_ptr(intf->memory.fp);
case INTFSTREAM_CHD:
return -1;
}
return 0;
}
char *intfstream_gets(intfstream_internal_t *intf,
char *buffer, uint64_t len)
{
@ -441,7 +459,6 @@ intfstream_t *intfstream_open_memory(void *data,
info.memory.writable = false;
fd = (intfstream_t*)intfstream_init(&info);
if (!fd)
return NULL;
@ -459,6 +476,37 @@ error:
return NULL;
}
intfstream_t *intfstream_open_writable_memory(void *data,
unsigned mode, unsigned hints, uint64_t size)
{
intfstream_info_t info;
intfstream_t *fd = NULL;
info.type = INTFSTREAM_MEMORY;
info.memory.buf.data = (uint8_t*)data;
info.memory.buf.size = size;
info.memory.writable = true;
fd = (intfstream_t*)intfstream_init(&info);
if (!fd)
return NULL;
if (!intfstream_open(fd, NULL, mode, hints))
goto error;
return fd;
error:
if (fd)
{
intfstream_close(fd);
free(fd);
}
return NULL;
}
intfstream_t *intfstream_open_chd_track(const char *path,
unsigned mode, unsigned hints, int32_t track)
{

View File

@ -71,7 +71,7 @@ static void memstream_init(memstream_t *stream,
memstream_t *memstream_open(unsigned writing)
{
memstream_t *stream;
memstream_t *stream;
if (!g_buffer || !g_size)
return NULL;
@ -92,6 +92,11 @@ void memstream_close(memstream_t *stream)
free(stream);
}
uint64_t memstream_get_ptr(memstream_t *stream)
{
return stream->ptr;
}
uint64_t memstream_read(memstream_t *stream, void *data, uint64_t bytes)
{
uint64_t avail = 0;

View File

@ -148,6 +148,7 @@
#ifdef HAVE_TRANSLATE
#include <encodings/base64.h>
#include <formats/rbmp.h>
#include <formats/rpng.h>
#include "translation_defines.h"
#endif
@ -2707,7 +2708,7 @@ static void handle_translation_cb(
unsigned image_width, image_height;
char* body_copy = NULL;
uint8_t* raw_output_data = NULL;
char* raw_bmp_data = NULL;
char* raw_image_file_data = NULL;
struct scaler_ctx* scaler = NULL;
bool is_paused = runloop_paused;
http_transfer_data_t *data = (http_transfer_data_t*)task_data;
@ -2715,8 +2716,10 @@ static void handle_translation_cb(
int new_image_size = 0;
int new_sound_size = 0;
const void* dummy_data = NULL;
void* raw_image_data = NULL;
void* raw_image_data = NULL;
void* raw_image_data_alpha = NULL;
void* raw_sound_data = NULL;
int retval = 0;
settings_t *settings = configuration_settings;
int i = 0;
@ -2737,7 +2740,6 @@ static void handle_translation_cb(
data->data[data->len] = '\0';
/* Parse JSON body for the image and sound data */
body_copy = strdup(data->data);
while (true)
@ -2756,7 +2758,7 @@ static void handle_translation_cb(
*(found_string+i-start-1) = '\0';
if (curr_state == 1)/*image*/
{
raw_bmp_data = (char*)unbase64(found_string,
raw_image_file_data = (char*)unbase64(found_string,
strlen(found_string),
&new_image_size);
curr_state = 0;
@ -2788,31 +2790,91 @@ static void handle_translation_cb(
if (found_string)
free(found_string);
if (!raw_bmp_data && !raw_sound_data)
if (!raw_image_file_data && !raw_sound_data)
{
error = "Invalid JSON body.";
goto finish;
}
if (raw_bmp_data)
if (raw_image_file_data)
{
/* Get the video frame dimensions reference */
video_driver_cached_frame_get(&dummy_data, &width, &height, &pitch);
/* Get image data (24 bit), and convert to the emulated pixel format */
image_width =
((uint32_t) ((uint8_t)raw_bmp_data[21]) << 24) +
((uint32_t) ((uint8_t)raw_bmp_data[20]) << 16) +
((uint32_t) ((uint8_t)raw_bmp_data[19]) << 8) +
((uint32_t) ((uint8_t)raw_bmp_data[18]) << 0);
if (raw_image_file_data[0] == 'B' && raw_image_file_data[1] == 'M')
{
/* This is a BMP file coming back. */
/* Get image data (24 bit), and convert to the emulated pixel format */
image_width =
((uint32_t) ((uint8_t)raw_image_file_data[21]) << 24) +
((uint32_t) ((uint8_t)raw_image_file_data[20]) << 16) +
((uint32_t) ((uint8_t)raw_image_file_data[19]) << 8) +
((uint32_t) ((uint8_t)raw_image_file_data[18]) << 0);
image_height =
((uint32_t) ((uint8_t)raw_bmp_data[25]) << 24) +
((uint32_t) ((uint8_t)raw_bmp_data[24]) << 16) +
((uint32_t) ((uint8_t)raw_bmp_data[23]) << 8) +
((uint32_t) ((uint8_t)raw_bmp_data[22]) << 0);
raw_image_data = raw_bmp_data + 54 * sizeof(uint8_t);
image_height =
((uint32_t) ((uint8_t)raw_image_file_data[25]) << 24) +
((uint32_t) ((uint8_t)raw_image_file_data[24]) << 16) +
((uint32_t) ((uint8_t)raw_image_file_data[23]) << 8) +
((uint32_t) ((uint8_t)raw_image_file_data[22]) << 0);
raw_image_data = raw_image_file_data + 54 * sizeof(uint8_t);
}
else if (raw_image_file_data[1] == 'P' && raw_image_file_data[2] == 'N' &&
raw_image_file_data[3] == 'G')
{
/* PNG coming back from the url */
image_width =
((uint32_t) ((uint8_t)raw_image_file_data[16])<<24)+
((uint32_t) ((uint8_t)raw_image_file_data[17])<<16)+
((uint32_t) ((uint8_t)raw_image_file_data[18])<<8)+
((uint32_t) ((uint8_t)raw_image_file_data[19])<<0);
image_height =
((uint32_t) ((uint8_t)raw_image_file_data[20])<<24)+
((uint32_t) ((uint8_t)raw_image_file_data[21])<<16)+
((uint32_t) ((uint8_t)raw_image_file_data[22])<<8)+
((uint32_t) ((uint8_t)raw_image_file_data[23])<<0);
rpng_t *rpng = rpng_alloc();
if (!rpng)
{
error = "Can't allocate memory.";
goto finish;
}
rpng_set_buf_ptr(rpng, raw_image_file_data, new_image_size);
rpng_start(rpng);
while (rpng_iterate_image(rpng));
do
{
retval = rpng_process_image(rpng, &raw_image_data_alpha, new_image_size, &image_width, &image_height);
}
while(retval == IMAGE_PROCESS_NEXT);
/* Returned output from the png processor is an upside down RGBA
* image, so we have to change that to RGB first. This should
* probably be replaced with a scaler call.*/
{
int d,tw,th,tc;
d=0;
raw_image_data = malloc(image_width*image_height*3*sizeof(uint8_t));
for (i=0;i<image_width*image_height*4;i++)
{
if (i%4 != 3)
{
tc = d%3;
th = image_height-d/(3*image_width)-1;
tw = (d%(image_width*3))/3;
((uint8_t*) raw_image_data)[tw*3+th*3*image_width+tc] = ((uint8_t *)raw_image_data_alpha)[i];
d+=1;
}
}
}
rpng_free(rpng);
}
else
{
RARCH_LOG("Output from URL not a valid file type, or is not supported.\n");
goto finish;
}
scaler = (struct scaler_ctx*)calloc(1, sizeof(struct scaler_ctx));
if (!scaler)
goto finish;
@ -2823,7 +2885,7 @@ static void handle_translation_cb(
In this case, we used the viewport to grab the image
and translate it, and we have the translated image in
the raw_image_data buffer.
*/
*/
/* TODO: write to the viewport in this case */
RARCH_LOG("Hardware frame buffer... writing to viewport"
@ -2835,7 +2897,7 @@ static void handle_translation_cb(
the video frame can change during run-time, but the
pitch may not, so we just assign it as the width
times the byte depth.
*/
*/
if (video_driver_pix_fmt == RETRO_PIXEL_FORMAT_XRGB8888)
{
@ -2920,10 +2982,12 @@ finish:
if (body_copy)
free(body_copy);
if (raw_bmp_data)
free(raw_bmp_data);
if (raw_image_file_data)
free(raw_image_file_data);
if (raw_image_data_alpha)
free(raw_image_data_alpha);
if (raw_image_data)
free(raw_image_data);
if (scaler)
free(scaler);
@ -3093,7 +3157,8 @@ static const char *ai_service_get_str(enum translation_lang id)
will consist of a JSON body, with the "image" field as a base64
encoded string of a 24bit-BMP that the will be translated. The server
must output the translated image in the form of a JSON body, with
the "image" field also as a base64 encoded, 24bit-BMP.
the "image" field also as a base64 encoded 24bit-BMP, or
as an alpha channel png.
*/
static bool run_translation_service(void)
{
@ -3111,13 +3176,14 @@ static bool run_translation_service(void)
bool error = false;
uint8_t *bmp_buffer = NULL;
uint64_t buffer_bytes = 0;
char *bmp64_buffer = NULL;
char *json_buffer = NULL;
int out_length = 0;
const char *rf1 = "{\"image\": \"";
const char *rf2 = "\"}\0";
bool TRANSLATE_USE_BMP = false;
if (!scaler)
goto finish;
@ -3156,8 +3222,9 @@ static bool run_translation_service(void)
if (!video_driver_read_viewport(bit24_image_prev, false))
goto finish;
/* Rescale down to regular resolution */
/* TODO: Rescale down to regular resolution */
width = vp.width;
height = vp.height;
bit24_image = bit24_image_prev;
bit24_image_prev = NULL;
}
@ -3198,16 +3265,26 @@ static bool run_translation_service(void)
an array to contain the BMP image along with the BMP header as bytes,
and then covert that to a b64 encoded array for transport in JSON.
*/
form_bmp_header(header, width, height, false);
bmp_buffer = (uint8_t*)malloc(width * height * 3+54);
if (!bmp_buffer)
goto finish;
if (TRANSLATE_USE_BMP)
{
form_bmp_header(header, width, height, false);
bmp_buffer = (uint8_t*)malloc(width * height * 3+54);
if (!bmp_buffer)
goto finish;
memcpy(bmp_buffer, header, 54*sizeof(uint8_t));
memcpy(bmp_buffer+54, bit24_image, width * height * 3 * sizeof(uint8_t));
memcpy(bmp_buffer, header, 54*sizeof(uint8_t));
memcpy(bmp_buffer+54, bit24_image, width * height * 3 * sizeof(uint8_t));
buffer_bytes = sizeof(uint8_t)*(width*height*3+54);
}
else
{
pitch = width*3;
bmp_buffer = rpng_save_image_bgr24_string(bit24_image+width*(height-1)*3, width, height, -pitch, &buffer_bytes);
}
bmp64_buffer = base64((void *)bmp_buffer, (int)(width * height * 3 + 54),
bmp64_buffer = base64((void *)bmp_buffer, sizeof(uint8_t)*buffer_bytes,
&out_length);
if (!bmp64_buffer)
goto finish;
@ -3270,10 +3347,12 @@ static bool run_translation_service(void)
/* mode */
{
char temp_string[PATH_MAX_LENGTH];
char* mode_chr = "image";
/*"image" is included for backwards compatability with
* vgtranslate < 1.04 */
char* mode_chr = "image,png";
if (settings->uints.ai_service_mode == 1)
mode_chr = "sound";
mode_chr = "sound,wav";
snprintf(temp_string,
sizeof(temp_string),
@ -3284,8 +3363,6 @@ static bool run_translation_service(void)
sizeof(new_ai_service_url));
}
RARCH_LOG("Server URL: %s\n", new_ai_service_url);
task_push_http_post_transfer(new_ai_service_url,
json_buffer, true, NULL, handle_translation_cb, NULL);