Update rarchdb

rarchdb/Makefile

@@ -1,12 +1,13 @@
 CFLAGS   = -g
 INCFLAGS = -I. -I../libretro-sdk/include
-DAT_CONVERTER_OBJ = rmsgpack.o \
+
+LUA_CONVERTER_OBJ = rmsgpack.o \
 		    rmsgpack_dom.o \
 		    rarchdb.o \
 		    bintree.o \
-		    db_parser.o \
-		    dat_converter.o \
+		    lua_converter.o \
 		    $(NULL)
+
 RARCHDB_TOOL_OBJ = rmsgpack.o \
 		    rmsgpack_dom.o \
 		    db_parser.o \
@@ -15,13 +16,15 @@ RARCHDB_TOOL_OBJ = rmsgpack.o \
 		    rarchdb.o \
 		    $(NULL)
 
-all: dat_converter rmsgpack_test rarchdb_tool
+LUA_FLAGS = `pkg-config lua --libs`
+
+all: rmsgpack_test rarchdb_tool lua_converter
 
 %.o: %.c
 	${CC} $(INCFLAGS) $< -c ${CFLAGS} -o $@
 
-dat_converter: ${DAT_CONVERTER_OBJ}
-	${CC} $(INCFLAGS) ${DAT_CONVERTER_OBJ} -o $@
+lua_converter: ${LUA_CONVERTER_OBJ}
+	${CC} $(INCFLAGS) ${LUA_CONVERTER_OBJ} ${LUA_FLAGS} -o $@
 
 rarchdb_tool: ${RARCHDB_TOOL_OBJ}
 	${CC} $(INCFLAGS) ${RARCHDB_TOOL_OBJ} -o $@
@@ -29,4 +32,4 @@ rarchdb_tool: ${RARCHDB_TOOL_OBJ}
 rmsgpack_test:
 	${CC} $(INCFLAGS) rmsgpack.c rmsgpack_test.c -g -o $@
 clean:
-	rm -rf *.o rmsgpack_test dat_converter rarchdb_tool
+	rm -rf *.o rmsgpack_test lua_converter rarchdb_tool

rarchdb/README.md

@@ -11,3 +11,27 @@ To find an entry with an index `rarchdb_tool <db file> find <index name> <value>
 
 The util `mkdb.sh <dat file> <db file>` will create a db file with indexes for crc sha1 and md5
 
+# lua converters
+In order to write you own converter you must have a lua file that implements the following functions:
+
+~~~.lua
+-- this function gets called before the db is created and shuold validate the
+-- arguments and set up the ground for db insertion
+function init(...)
+	local args = {...}
+	local script_name = args[1]
+end
+
+-- this is in iterator function. It is called before each insert.
+-- the function should return a table for insertion or nil when there are no
+-- more records to insert.
+function get_value()
+	return {
+		key = "value", -- will be saved as string
+		num = 3, -- will be saved as int
+		bin = binary("some string"), -- will be saved as binary
+		unum = uint(3), -- will be saved as uint
+		some_bool = true, -- will be saved as bool
+	}
+end
+~~~

rarchdb/dat_converter

@@ -0,0 +1,2 @@
+#!/bin/sh
+./lua_converter "$2" dat_converter.lua "$1"

rarchdb/dat_converter.lua

@@ -0,0 +1,127 @@
+local dat_obj = nil
+
+local function dat_lexer(f)
+    local line, err = f:read("*l")
+    return function()
+        local tok = nil
+        while not tok do
+            if not line then
+                return nil
+            end
+            tok, line = string.match(line, "^%s*(..-)(%s.*)")
+            if tok and string.match(tok, "^\"") then
+                tok, line = string.match(tok..line, "^\"([^\"]-)\"(.*)")
+            end
+            if not line then
+                line = f:read("*l")
+            end
+        end
+        return tok
+    end
+end
+
+local function dat_parse_table(lexer)
+    local res = {}
+    local state = "key"
+    local key = nil
+    for tok in lexer do
+        if state == "key" then
+            if tok == ")" then
+                return res
+            else
+                key = tok
+                state = "value"
+            end
+        else
+            if tok == "(" then
+                res[key] = dat_parse_table(lexer)
+            else
+                res[key] = tok
+            end
+            state = "key"
+        end
+    end
+    return res
+end
+
+local function dat_parser(lexer)
+    local res = {}
+    local state = "key"
+    local key = nil
+    local skip = true
+    for tok in lexer do
+        if state == "key" then
+            if tok == "game" then
+                skip = false
+            end
+            state = "value"
+        else
+            if tok == "(" then
+                local v = dat_parse_table(lexer)
+                if not skip then
+                    table.insert(res, v)
+                    skip = true
+                end
+            else
+                error("Expected '(' found '"..tok.."'")
+            end
+            state = "key"
+        end
+    end
+    return res
+end
+
+local function unhex(s)
+    if not s then return nil end
+    return (s:gsub('.', function (c)
+        return string.format('%02X', string.byte(c))
+    end))
+end
+
+function init(...)
+    local args = {...}
+    local dat_path = args[2]
+    assert(dat_path, "dat file argument is missing")
+    local dat_file, err = io.open(dat_path, "r")
+    if err then
+        error("could not open dat file '" .. dat_path .. "':" .. err)
+    end
+
+    print("Parsing dat file '" .. dat_path .. "'...")
+    dat_obj = dat_parser(dat_lexer(dat_file))
+    dat_file:close()
+end
+
+function get_value()
+    local t = table.remove(dat_obj)
+    if not t then
+        return
+    else
+        return {
+            name = t.name,
+            description = t.description,
+            rom_name = t.rom.name,
+            size = uint(tonumber(t.rom.size)),
+            users = uint(tonumber(t.users)),
+            releasemonth = uint(tonumber(t.releasemonth)),
+            releaseyear = uint(tonumber(t.releaseyear)),
+            rumble = uint(tonumber(t.rumble)),
+            analog = uint(tonumber(t.analog)),
+
+            esrb_rating = t.esrb_rating,
+            elspa_rating = t.elspa_rating,
+            pegi_rating = t.pegi_rating,
+            cero_rating = t.cero_rating,
+
+            developers = t.developers,
+            publisher = t.publisher,
+            origin = t.origin,
+
+            crc = binary(unhex(t.rom.crc)),
+            md5 = binary(unhex(t.rom.md5)),
+            sha1 = binary(unhex(t.rom.sha1)),
+            serial = binary(t.rom.serial),
+        }
+    end
+end
+

rarchdb/lua_converter.c

@@ -0,0 +1,184 @@
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <errno.h>
+#include <string.h>
+#include <stdlib.h>
+
+#include <lua.h>
+#include <lualib.h>
+#include <lauxlib.h>
+
+#include "rarchdb.h"
+int master_key = 1;
+
+const char *LUA_COMMON = " \
+function binary(s) if s ~= nil then return {binary = s} else return nil end end \
+function uint(s) if s ~= nil then return {uint = s} else return nil end end \
+";
+
+static int call_init(lua_State *L, int argc, const char** argv) {
+	int rv = -1;
+	int i;
+
+	lua_getglobal(L, "init");
+	for (i = 0; i < argc; i++) {
+		lua_pushstring(L, argv[i]);
+	}
+
+	if (lua_pcall(L, argc, 0, 0) != 0) {
+		printf(
+			"error running function `init': %s\n",
+			lua_tostring(L, -1)
+		);
+	}
+
+	return rv;
+}
+
+
+static int value_provider(void *ctx, struct rmsgpack_dom_value *out) {
+	lua_State *L = ctx;
+
+	int rv = -1;
+	int i;
+	const char *tmp_string = NULL;
+	char *tmp_buff = NULL;
+	struct rmsgpack_dom_value *tmp_value;
+	const int key_idx = -2;
+	const int value_idx = -1;
+	const int MAX_FIELDS = 100;
+	size_t tmp_len;
+	lua_Number tmp_num;
+
+	lua_getglobal(L, "get_value");
+
+	if (lua_pcall(L, 0, 1, 0) != 0) {
+		printf(
+			"error running function `get_value': %s\n",
+			lua_tostring(L, -1)
+		);
+	}
+
+	if (lua_isnil(L, -1)) {
+		rv = 1;
+	} else if (lua_istable(L, -1)) {
+		out->type = RDT_MAP;
+		out->map.len = 0;
+		out->map.items = calloc(MAX_FIELDS, sizeof(struct rmsgpack_dom_pair));
+		lua_pushnil(L);
+		while (lua_next(L, -2) != 0) {
+			if (out->map.len > MAX_FIELDS) {
+				printf("skipping due to too many keys\n");
+			} else if (!lua_isstring(L, key_idx)) {
+				printf("skipping non string key\n");
+			} else if (lua_isnil(L, value_idx)) {
+				// Skipping nil value fields to save disk space
+			} else {
+				i = out->map.len;
+				tmp_buff = strdup(lua_tostring(L, key_idx));
+				out->map.items[i].key.type = RDT_STRING;
+				out->map.items[i].key.string.len = strlen(tmp_buff);
+				out->map.items[i].key.string.buff = tmp_buff;
+
+				tmp_value = &out->map.items[i].value;
+				switch(lua_type(L, value_idx)) {
+					case LUA_TNUMBER:
+						tmp_num = lua_tonumber(L, value_idx);
+						tmp_value->type = RDT_INT;
+						tmp_value->int_ = tmp_num;
+						break;
+					case LUA_TBOOLEAN:
+						tmp_value->type = RDT_BOOL;
+						tmp_value->bool_ = lua_toboolean(L, value_idx);
+						break;
+					case LUA_TSTRING:
+						tmp_buff = strdup(lua_tostring(L, key_idx));
+						tmp_value->type = RDT_STRING;
+						tmp_value->string.len = strlen(tmp_buff);
+						tmp_value->string.buff = tmp_buff;
+						break;
+					case LUA_TTABLE:
+						lua_getfield(L, value_idx, "binary");
+						if (!lua_isstring(L, -1)) {
+							lua_pop(L, 1);
+							lua_getfield(L, value_idx, "uint");
+							if (!lua_isnumber(L, -1)) {
+								lua_pop(L, 1);
+								goto set_nil;
+							} else {
+								tmp_num = lua_tonumber(L, -1);
+								tmp_value->type = RDT_UINT;
+								tmp_value->uint_ = tmp_num;
+								lua_pop(L, 1);
+							}
+						} else {
+							tmp_string = lua_tolstring(L, -1, &tmp_len);
+							tmp_buff = malloc(tmp_len);
+							memcpy(tmp_buff, tmp_string, tmp_len);
+							tmp_value->type = RDT_BINARY;
+							tmp_value->binary.len = tmp_len;
+							tmp_value->binary.buff = tmp_buff;
+							lua_pop(L, 1);
+						}
+						break;
+					default:
+set_nil:
+						tmp_value->type = RDT_NULL;
+				}
+				out->map.len++;
+			}
+			lua_pop(L, 1);
+		}
+		rv = 0;
+	} else {
+		printf("function `get_value' must return a table or nil\n");
+	}
+	lua_pop(L, 1);
+	return rv;
+}
+
+int main(int argc, char **argv)
+{
+	const char* db_file;
+	const char* lua_file;
+	int dst = -1;
+	int rv = 0;
+
+	if (argc < 3) {
+		printf("usage:\n%s <db file> <lua file> [args ...]\n", argv[0]);
+		return 1;
+	}
+
+	db_file = argv[1];
+	lua_file = argv[2];
+
+	lua_State *L = luaL_newstate();
+	luaL_openlibs(L);
+	luaL_dostring(L, LUA_COMMON);
+
+	if (luaL_dofile(L, lua_file) != 0) {
+		return 1;
+	}
+
+	call_init(L, argc - 2, (const char**) argv + 2);
+
+	dst = open(db_file, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
+	if (dst == -1)
+	{
+		printf("Could not open destination file '%s': %s\n", db_file, strerror(errno));
+		rv = errno;
+		goto clean;
+	}
+
+	rv = rarchdb_create(dst, &value_provider, L);
+
+clean:
+	lua_close(L);
+	if (dst != -1) {
+		close(dst);
+	}
+	return rv;
+}
+

rarchdb/retro_endianness.h

@@ -0,0 +1,93 @@
+/* Copyright  (C) 2010-2014 The RetroArch team
+ *
+ * ---------------------------------------------------------------------------------------
+ * The following license statement only applies to this file (retro_endianness.h).
+ * ---------------------------------------------------------------------------------------
+ *
+ * Permission is hereby granted, free of charge,
+ * to any person obtaining a copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation the rights to
+ * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+ * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __LIBRETRO_SDK_ENDIANNESS_H
+#define __LIBRETRO_SDK_ENDIANNESS_H
+
+#include <retro_inline.h>
+#include <stdint.h>
+
+#define SWAP16(x) ((uint16_t)(				      \
+         (((uint16_t)(x) & 0x00ff) << 8)      |	\
+         (((uint16_t)(x) & 0xff00) >> 8)        \
+          ))
+
+#define SWAP32(x) ((uint32_t)(           \
+         (((uint32_t)(x) & 0x000000ff) << 24) | \
+         (((uint32_t)(x) & 0x0000ff00) <<  8) | \
+         (((uint32_t)(x) & 0x00ff0000) >>  8) | \
+         (((uint32_t)(x) & 0xff000000) >> 24)   \
+         ))
+
+static INLINE uint8_t is_little_endian(void)
+{
+   union
+   {
+      uint16_t x;
+      uint8_t y[2];
+   } u;
+
+   u.x = 1;
+   return u.y[0];
+}
+
+static INLINE uint32_t swap_if_big32(uint32_t val)
+{
+   if (is_little_endian())
+      return val;
+   return (val >> 24) | ((val >> 8) & 0xFF00) |
+      ((val << 8) & 0xFF0000) | (val << 24);
+}
+
+static INLINE uint32_t swap_if_little32(uint32_t val)
+{
+   if (is_little_endian())
+      return (val >> 24) | ((val >> 8) & 0xFF00) |
+         ((val << 8) & 0xFF0000) | (val << 24);
+   return val;
+}
+
+static INLINE uint16_t swap_if_big16(uint16_t val)
+{
+   if (is_little_endian())
+      return val;
+   return (val >> 8) | (val << 8);
+}
+
+static INLINE uint16_t swap_if_little16(uint16_t val)
+{
+   if (is_little_endian())
+      return (val >> 8) | (val << 8);
+   return val;
+}
+
+static INLINE void store32be(uint32_t *addr, uint32_t data)
+{
+   *addr = is_little_endian() ? SWAP32(data) : data;
+}
+
+static INLINE uint32_t load32be(const uint32_t *addr)
+{
+   return is_little_endian() ? SWAP32(*addr) : *addr;
+}
+
+#endif

rarchdb/retro_inline.h

@@ -0,0 +1,40 @@
+/* Copyright  (C) 2010-2014 The RetroArch team
+ *
+ * ---------------------------------------------------------------------------------------
+ * The following license statement only applies to this file (retro_inline.h).
+ * ---------------------------------------------------------------------------------------
+ *
+ * Permission is hereby granted, free of charge,
+ * to any person obtaining a copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation the rights to
+ * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+ * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __LIBRETRO_SDK_INLINE_H
+#define __LIBRETRO_SDK_INLINE_H
+
+#if !defined(__cplusplus) && defined(_WIN32)
+
+#ifndef INLINE
+#define INLINE _inline
+#endif
+
+#else
+
+#ifndef INLINE
+#define INLINE inline
+#endif
+
+#endif
+
+#endif

rarchdb/rmsgpack.c

@@ -57,6 +57,8 @@ static const uint8_t MPF_NIL = 0xc0;
 
 int rmsgpack_write_array_header(int fd, uint32_t size)
 {
+   uint16_t tmp_i16;
+   uint32_t tmp_i32;
    if (size < 16)
    {
       size = (size | MPF_FIXARRAY);
@@ -68,7 +70,8 @@ int rmsgpack_write_array_header(int fd, uint32_t size)
    {
       if (write(fd, &MPF_ARRAY16, sizeof(MPF_ARRAY16)) == -1)
          return -errno;
-      if (write(fd, (void *)(&size), sizeof(uint16_t)) == -1)
+      tmp_i16 = httobe16(size);
+      if (write(fd, (void *)(&tmp_i16), sizeof(uint16_t)) == -1)
          return -errno;
       return sizeof(int8_t) + sizeof(uint16_t);
    }
@@ -76,7 +79,8 @@ int rmsgpack_write_array_header(int fd, uint32_t size)
    {
       if (write(fd, &MPF_ARRAY32, sizeof(MPF_ARRAY32)) == -1)
          return -errno;
-      if (write(fd, (void *)(&size), sizeof(uint32_t)) == -1)
+      tmp_i32 = httobe32(size);
+      if (write(fd, (void *)(&tmp_i32), sizeof(uint32_t)) == -1)
          return -errno;
       return sizeof(int8_t) + sizeof(uint32_t);
    }
@@ -84,6 +88,8 @@ int rmsgpack_write_array_header(int fd, uint32_t size)
 
 int rmsgpack_write_map_header(int fd, uint32_t size)
 {
+	uint16_t tmp_i16;
+	uint32_t tmp_i32;
    if (size < 16)
    {
       size = (size | MPF_FIXMAP);
@@ -95,15 +101,17 @@ int rmsgpack_write_map_header(int fd, uint32_t size)
    {
       if (write(fd, &MPF_MAP16, sizeof(MPF_MAP16)) == -1)
          return -errno;
-      if (write(fd, (void *)(&size), sizeof(uint16_t)) == -1)
+      tmp_i16 = httobe16(size);
+      if (write(fd, (void *)(&tmp_i16), sizeof(uint16_t)) == -1)
          return -errno;
-      return sizeof(int8_t) + sizeof(uint16_t);
+      return sizeof(uint8_t) + sizeof(uint16_t);
    }
    else
    {
+	tmp_i32 = httobe32(size);
       if (write(fd, &MPF_MAP32, sizeof(MPF_MAP32)) == -1)
          return -errno;
-      if (write(fd, (void *)(&size), sizeof(uint32_t)) == -1)
+      if (write(fd, (void *)(&tmp_i32), sizeof(uint32_t)) == -1)
          return -errno;
       return sizeof(int8_t) + sizeof(uint32_t);
    }
@@ -131,13 +139,13 @@ int rmsgpack_write_string(int fd, const char *s, uint32_t len)
    }
    else if (len < 1<<16)
    {
-		if (write(fd, &MPF_STR16, sizeof(MPF_STR16)) == -1)
-			return -errno;
-		tmp_i16 = httobe16(len);
-		if (write(fd, &tmp_i16, sizeof(uint16_t)) == -1)
-			return -errno;
-		written += sizeof(uint16_t);
-	}
+	   if (write(fd, &MPF_STR16, sizeof(MPF_STR16)) == -1)
+		   return -errno;
+	   tmp_i16 = httobe16(len);
+	   if (write(fd, &tmp_i16, sizeof(uint16_t)) == -1)
+		   return -errno;
+	   written += sizeof(uint16_t);
+   }
    else
    {
 		if (write(fd, &MPF_STR32, sizeof(MPF_STR32)) == -1)