Switch to libretrodb

This commit is contained in:
twinaphex 2015-01-19 22:47:09 +01:00
parent da88bcdd91
commit b9259598cd
50 changed files with 5518 additions and 2660 deletions

View File

@ -164,11 +164,11 @@ OBJ += frontend/frontend.o \
# RarchDB
OBJ += rarchdb/bintree.o \
rarchdb/db_parser.o \
rarchdb/rarchdb.o \
rarchdb/rmsgpack.o \
rarchdb/rmsgpack_dom.o
OBJ += libretrodb/bintree.o \
libretrodb/rarchdb.o \
libretrodb/query.o \
libretrodb/rmsgpack.o \
libretrodb/rmsgpack_dom.o
# Miscellaneous

View File

@ -802,13 +802,13 @@ XML
#include "../audio/audio_utils.c"
/*============================================================
RARCHDB
LIBRETRODB
============================================================ */
#include "../rarchdb/bintree.c"
#include "../rarchdb/db_parser.c"
#include "../rarchdb/rarchdb.c"
#include "../rarchdb/rmsgpack.c"
#include "../rarchdb/rmsgpack_dom.c"
#include "../libretrodb/bintree.c"
#include "../libretrodb/db_parser.c"
#include "../libretrodb/rarchdb.c"
#include "../libretrodb/rmsgpack.c"
#include "../libretrodb/rmsgpack_dom.c"
#ifdef __cplusplus
}

9
libretrodb/.gitignore vendored Normal file
View File

@ -0,0 +1,9 @@
*.swp
*~
*.o
*.pyc
rmsgpack_test
lua_converter
rarchdb_tool
testlib.so
*.uncrustify

View File

@ -3,20 +3,34 @@ INCFLAGS = -I. -I../libretro-sdk/include
LUA_CONVERTER_OBJ = rmsgpack.o \
rmsgpack_dom.o \
lua_common.o \
rarchdb.o \
bintree.o \
query.o \
lua_converter.o \
$(NULL)
RARCHDB_TOOL_OBJ = rmsgpack.o \
rmsgpack_dom.o \
db_parser.o \
rarchdb_tool.o \
bintree.o \
rarchdb.o \
$(NULL)
rmsgpack_dom.o \
rarchdb_tool.o \
bintree.o \
query.o \
rarchdb.o \
$(NULL)
TESTLIB_C = testlib.c \
lua_common.c \
query.c \
rarchdb.c \
bintree.c \
rmsgpack.c \
rmsgpack_dom.c \
$(NULL)
LUA_FLAGS = `pkg-config lua --libs`
TESTLIB_FLAGS = ${CFLAGS} ${LUA_FLAGS} -shared -fpic
.PHONY: all clean check
all: rmsgpack_test rarchdb_tool lua_converter
@ -31,5 +45,12 @@ rarchdb_tool: ${RARCHDB_TOOL_OBJ}
rmsgpack_test:
${CC} $(INCFLAGS) rmsgpack.c rmsgpack_test.c -g -o $@
testlib.so: ${TESTLIB_C}
${CC} ${INCFLAGS} ${TESTLIB_FLAGS} ${TESTLIB_C} -o $@
check: testlib.so tests.lua
lua ./tests.lua
clean:
rm -rf *.o rmsgpack_test lua_converter rarchdb_tool
rm -rf *.o rmsgpack_test lua_converter rarchdb_tool testlib.so

View File

@ -3,14 +3,12 @@ A small read only database
Mainly to be used by retroarch
# Usage
To convert a dat file use `dat_converter <dat file> <db file>`
Files specified later in the chain **will override** earlier ones if the same key exists multiple times.
To list out the content of a db `rarchdb_tool <db file> list`
To create an index `rarchdb_tool <db file> create-index <index name> <field name>`
To find an entry with an index `rarchdb_tool <db file> find <index name> <value>`
The util `mkdb.sh <dat file> <db file>` will create a db file with indexes for crc sha1 and md5
# lua converters
In order to write you own converter you must have a lua file that implements the following functions:
@ -35,3 +33,23 @@ function get_value()
}
end
~~~
# dat file converter
To convert a dat file use:
~~~
dat_converter <db file> <dat file>
~~~
If you want to merge multiple dat files you need to run:
~~~
dat_converter <db file> <match key> <dat file> ...
~~~
for example:
~~~
dat_converter snes.rdb rom.crc snes1.dat snes2.dat
~~~

123
libretrodb/bintree.c Normal file
View File

@ -0,0 +1,123 @@
#include <stdlib.h>
#include <errno.h>
#include <stdio.h>
#include "bintree.h"
#if 0
static int NIL_VALUE = 1;
#endif
static void * NIL_NODE = &NIL_NODE;
static struct bintree_node * new_nil_node(struct bintree_node * parent);
void bintree_new(
struct bintree * t,
bintree_cmp_func cmp,
void * ctx
){
t->root = new_nil_node(NULL);
t->cmp = cmp;
t->ctx = ctx;
}
static struct bintree_node * new_nil_node(struct bintree_node * parent){
struct bintree_node * node = (struct bintree_node *)calloc(1, sizeof(struct bintree_node));
if (!node)
return NULL;
node->value = NIL_NODE;
node->parent = parent;
node->left = NULL;
node->right = NULL;
return node;
}
static inline int is_nil(const struct bintree_node * node){
return node == NULL || node->value == NIL_NODE;
}
static int insert(
struct bintree * t,
struct bintree_node * root,
void * value
){
int cmp_res = 0;
if (is_nil(root)) {
root->left = new_nil_node(root);
root->right = new_nil_node(root);
if (!root->left || !root->right) {
if (root->left) {
free(root->left);
root->left = NULL;
}
if (root->right) {
free(root->right);
root->right = NULL;
}
return -ENOMEM;
}
root->value = value;
} else {
cmp_res = t->cmp(root->value, value, t->ctx);
if (cmp_res > 0)
return insert(t, root->left, value);
else if (cmp_res < 0)
return insert(t, root->right, value);
else
return -EINVAL;
}
return 0;
}
int bintree_insert(
struct bintree * t,
void * value
){
return insert(t, t->root, value);
}
static int _bintree_iterate(
struct bintree_node * n,
bintree_iter_cb cb,
void * ctx
){
int rv;
if (is_nil(n))
return 0;
else {
if ((rv = _bintree_iterate(n->left, cb, ctx)) != 0)
return rv;
if ((rv = cb(n->value, ctx)) != 0)
return rv;
if ((rv = _bintree_iterate(n->right, cb, ctx)) != 0)
return rv;
}
return 0;
}
int bintree_iterate(
const struct bintree * t,
bintree_iter_cb cb,
void * ctx
){
return _bintree_iterate(t->root, cb, ctx);
}
static void bintree_free_node(struct bintree_node * n){
if (n == NULL)
return;
if (n->value == NIL_NODE) {
free(n);
return;
}
n->value = NULL;
bintree_free_node(n->left);
bintree_free_node(n->right);
free(n);
}
void bintree_free(struct bintree * t){
bintree_free_node(t->root);
}

45
libretrodb/bintree.h Normal file
View File

@ -0,0 +1,45 @@
#ifndef __RARCHDB_BINTREE_H__
#define __RARCHDB_BINTREE_H__
typedef int (* bintree_cmp_func)(
const void * a,
const void * b,
void * ctx
);
typedef int (* bintree_iter_cb)(
void * value,
void * ctx
);
struct bintree_node {
void * value;
struct bintree_node * parent;
struct bintree_node * left;
struct bintree_node * right;
};
struct bintree {
struct bintree_node * root;
bintree_cmp_func cmp;
void * ctx;
};
void bintree_new(
struct bintree * t,
bintree_cmp_func cmp,
void * ctx
);
int bintree_insert(
struct bintree * t,
void * value
);
int bintree_iterate(
const struct bintree * t,
bintree_iter_cb cb,
void * ctx
);
void bintree_free(struct bintree * t);
#endif

7
libretrodb/check_style Executable file
View File

@ -0,0 +1,7 @@
#!/bin/bash
uncrustify -c uncrustify.cfg *.c *.h
for f in *.uncrustify; do
source_file=$(basename "$f" .uncrustify);
cmp "$source_file" "$f" || (read -p "Style violations in '$source_file'" && vimdiff "$source_file" "$f");
done;
rm -f *.uncrustify

2
libretrodb/crustydiff Executable file
View File

@ -0,0 +1,2 @@
#!/bin/sh
uncrustify -c uncrustify.cfg "$1" && vimdiff "$1" "$1.uncrustify"

4
libretrodb/dat_converter Executable file
View File

@ -0,0 +1,4 @@
#!/bin/sh
rdb_file="$1"
shift 1
./lua_converter "$rdb_file" dat_converter.lua "$@"

View File

@ -1,4 +1,5 @@
local dat_obj = nil
local dat_obj = {}
local match_key = nil
local function dat_lexer(f)
local line, err = f:read("*l")
@ -78,18 +79,59 @@ local function unhex(s)
end))
end
local function get_match_key(mk, t)
for p in string.gmatch(mk, "(%w+)[.]?") do
if p == nil or t == nil then
error("Invalid match key '"..mk.."'")
end
t = t[p]
end
return t
end
table.update = function(a, b)
for k,v in pairs(b) do
a[k] = v
end
end
function init(...)
local args = {...}
local dat_path = args[2]
assert(dat_path, "dat file argument is missing")
local dat_file, err = io.open(dat_path, "r")
if err then
error("could not open dat file '" .. dat_path .. "':" .. err)
table.remove(args, 1)
if #args == 0 then
assert(dat_path, "dat file argument is missing")
end
print("Parsing dat file '" .. dat_path .. "'...")
dat_obj = dat_parser(dat_lexer(dat_file))
dat_file:close()
if #args > 1 then
match_key = table.remove(args, 1)
end
local dat_hash = {}
for _, dat_path in ipairs(args) do
local dat_file, err = io.open(dat_path, "r")
if err then
error("could not open dat file '" .. dat_path .. "':" .. err)
end
print("Parsing dat file '" .. dat_path .. "'...")
local objs = dat_parser(dat_lexer(dat_file))
dat_file:close()
for _, obj in pairs(objs) do
if match_key then
local mk = get_match_key(match_key, obj)
if mk == nil then
error("missing match key '" .. match_key .. "' in one of the entries")
end
if dat_hash[mk] == nil then
dat_hash[mk] = {}
table.insert(dat_obj, dat_hash[mk])
end
table.update(dat_hash[mk], obj)
else
table.insert(dat_obj, obj)
end
end
end
end
function get_value()
@ -124,4 +166,3 @@ function get_value()
}
end
end

93
libretrodb/lua_common.c Normal file
View File

@ -0,0 +1,93 @@
#include "lua_common.h"
#include <stdlib.h>
#include <string.h>
int rarchdb_lua_to_rmsgpack_value(
lua_State * L,
int index,
struct rmsgpack_dom_value * out
) {
int rv = -1;
int i;
const char * tmp_string = NULL;
char * tmp_buff = NULL;
struct rmsgpack_dom_value * tmp_value;
const int key_idx = -2;
const int value_idx = -1;
const int MAX_FIELDS = 100;
size_t tmp_len;
lua_Number tmp_num;
out->type = RDT_MAP;
out->map.len = 0;
out->map.items = calloc(MAX_FIELDS, sizeof(struct rmsgpack_dom_pair));
lua_pushnil(L);
while (lua_next(L, index - 1) != 0) {
if (out->map.len > MAX_FIELDS) {
printf("skipping due to too many keys\n");
} else if (!lua_isstring(L, key_idx)) {
printf("skipping non string key\n");
} else if (lua_isnil(L, value_idx)) {
// Skipping nil value fields to save disk space
} else {
i = out->map.len;
tmp_buff = strdup(lua_tostring(L, key_idx));
out->map.items[i].key.type = RDT_STRING;
out->map.items[i].key.string.len = strlen(tmp_buff);
out->map.items[i].key.string.buff = tmp_buff;
tmp_value = &out->map.items[i].value;
switch (lua_type(L, value_idx)) {
case LUA_TNUMBER:
tmp_num = lua_tonumber(L, value_idx);
tmp_value->type = RDT_INT;
tmp_value->int_ = tmp_num;
break;
case LUA_TBOOLEAN:
tmp_value->type = RDT_BOOL;
tmp_value->bool_ = lua_toboolean(L, value_idx);
break;
case LUA_TSTRING:
tmp_buff = strdup(lua_tostring(L, value_idx));
tmp_value->type = RDT_STRING;
tmp_value->string.len = strlen(tmp_buff);
tmp_value->string.buff = tmp_buff;
break;
case LUA_TTABLE:
lua_getfield(L, value_idx, "binary");
if (!lua_isstring(L, -1)) {
lua_pop(L, 1);
lua_getfield(L, value_idx, "uint");
if (!lua_isnumber(L, -1)) {
lua_pop(L, 1);
goto set_nil;
} else {
tmp_num = lua_tonumber(L, -1);
tmp_value->type = RDT_UINT;
tmp_value->uint_ = tmp_num;
lua_pop(L, 1);
}
} else {
tmp_string = lua_tolstring(L, -1, &tmp_len);
tmp_buff = malloc(tmp_len);
memcpy(tmp_buff, tmp_string, tmp_len);
tmp_value->type = RDT_BINARY;
tmp_value->binary.len = tmp_len;
tmp_value->binary.buff = tmp_buff;
lua_pop(L, 1);
}
break;
default:
set_nil:
tmp_value->type = RDT_NULL;
}
out->map.len++;
}
lua_pop(L, 1);
}
rv = 0;
return rv;
}

16
libretrodb/lua_common.h Normal file
View File

@ -0,0 +1,16 @@
#ifndef __RARCHDB_LUA_COMMON_H__
#define __RARCHDB_LUA_COMMON_H__
#include <lua.h>
#include <lualib.h>
#include <lauxlib.h>
#include "rmsgpack_dom.h"
int rarchdb_lua_to_rmsgpack_value(
lua_State * L,
int index,
struct rmsgpack_dom_value * out
);
#endif

119
libretrodb/lua_converter.c Normal file
View File

@ -0,0 +1,119 @@
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include <stdlib.h>
#include <lua.h>
#include <lualib.h>
#include <lauxlib.h>
#include "rarchdb.h"
#include "lua_common.h"
int master_key = 1;
const char * LUA_COMMON = " \
function binary(s) if s ~= nil then return {binary = s} else return nil end end \
function uint(s) if s ~= nil then return {uint = s} else return nil end end \
";
static int call_init(
lua_State * L,
int argc,
const char ** argv
) {
int rv = -1;
int i;
lua_getglobal(L, "init");
for (i = 0; i < argc; i++) {
lua_pushstring(L, argv[i]);
}
if (lua_pcall(L, argc, 0, 0) != 0) {
printf(
"error running function `init': %s\n",
lua_tostring(L, -1)
);
}
return rv;
}
static int value_provider(
void * ctx,
struct rmsgpack_dom_value * out
) {
int rv;
lua_State * L = ctx;
lua_getglobal(L, "get_value");
if (lua_pcall(L, 0, 1, 0) != 0) {
printf(
"error running function `get_value': %s\n",
lua_tostring(L, -1)
);
}
if (lua_isnil(L, -1)) {
rv = 1;
} else if (lua_istable(L, -1)) {
rv = rarchdb_lua_to_rmsgpack_value(L, -1, out);
} else {
printf("function `get_value' must return a table or nil\n");
}
lua_pop(L, 1);
return rv;
}
int main(
int argc,
char ** argv
){
const char * db_file;
const char * lua_file;
int dst = -1;
int rv = 0;
if (argc < 3) {
printf("usage:\n%s <db file> <lua file> [args ...]\n", argv[0]);
return 1;
}
db_file = argv[1];
lua_file = argv[2];
lua_State * L = luaL_newstate();
luaL_openlibs(L);
luaL_dostring(L, LUA_COMMON);
if (luaL_dofile(L, lua_file) != 0) {
return 1;
}
call_init(L, argc - 2, (const char **) argv + 2);
dst = open(db_file, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
if (dst == -1) {
printf(
"Could not open destination file '%s': %s\n",
db_file,
strerror(errno)
);
rv = errno;
goto clean;
}
rv = rarchdb_create(dst, &value_provider, L);
clean:
lua_close(L);
if (dst != -1) {
close(dst);
}
return rv;
}

829
libretrodb/query.c Normal file
View File

@ -0,0 +1,829 @@
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <ctype.h>
#include <string.h>
#include "rarchdb.h"
#include "rmsgpack_dom.h"
#define MAX_ERROR_LEN 256
#define MAX_ARGS 50
static char tmp_error_buff [MAX_ERROR_LEN] = {};
struct buffer {
const char * data;
size_t len;
off_t offset;
};
/* Errors */
static void raise_too_many_arguments(const char ** error) {
snprintf(
tmp_error_buff,
MAX_ERROR_LEN,
"Too many arguments in function call"
);
*error = tmp_error_buff;
}
static void raise_expected_number(
off_t where,
const char ** error
) {
snprintf(
tmp_error_buff,
MAX_ERROR_LEN,
"%lu::Expected number",
where
);
*error = tmp_error_buff;
}
static void raise_expected_string(
off_t where,
const char ** error
) {
snprintf(
tmp_error_buff,
MAX_ERROR_LEN,
"%lu::Expected string",
where
);
*error = tmp_error_buff;
}
static void raise_unexpected_eof(
off_t where,
const char ** error
) {
snprintf(
tmp_error_buff,
MAX_ERROR_LEN,
"%lu::Unexpected EOF",
where
);
*error = tmp_error_buff;
}
static void raise_enomem(const char ** error) {
snprintf(
tmp_error_buff,
MAX_ERROR_LEN,
"Out of memory"
);
*error = tmp_error_buff;
}
static void raise_unknown_function(
off_t where,
const char * name,
size_t len,
const char ** error
) {
int n = snprintf(
tmp_error_buff,
MAX_ERROR_LEN,
"%lu::Unknown function '",
where
);
if (len < (MAX_ERROR_LEN - n - 3)) {
strncpy(tmp_error_buff + n, name, len);
}
strcpy(tmp_error_buff + n + len, "'");
*error = tmp_error_buff;
}
static void raise_expected_eof(
off_t where,
char found,
const char ** error
) {
snprintf(
tmp_error_buff,
MAX_ERROR_LEN,
"%lu::Expected EOF found '%c'",
where,
found
);
*error = tmp_error_buff;
}
static void raise_unexpected_char(
off_t where,
char expected,
char found,
const char ** error
) {
snprintf(
tmp_error_buff,
MAX_ERROR_LEN,
"%lu::Expected '%c' found '%c'",
where,
expected,
found
);
*error = tmp_error_buff;
}
enum argument_type {
AT_FUNCTION,
AT_VALUE
};
struct argument;
typedef struct rmsgpack_dom_value (* rarch_query_func)(
struct rmsgpack_dom_value input,
unsigned argc,
const struct argument * argv
);
struct invocation {
rarch_query_func func;
unsigned argc;
struct argument * argv;
};
struct argument {
enum argument_type type;
union {
struct rmsgpack_dom_value value;
struct invocation invocation;
};
};
static void argument_free(struct argument * arg) {
unsigned i;
if (arg->type == AT_FUNCTION) {
for (i = 0; i < arg->invocation.argc; i++) {
argument_free(&arg->invocation.argv[i]);
}
} else {
rmsgpack_dom_value_free(&arg->value);
}
}
struct query {
unsigned ref_count;
struct invocation root;
};
struct registered_func {
const char * name;
rarch_query_func func;
};
static struct buffer parse_argument(
struct buffer buff,
struct argument * arg,
const char ** error
);
static struct rmsgpack_dom_value is_true(
struct rmsgpack_dom_value input,
unsigned argc,
const struct argument * argv
) {
struct rmsgpack_dom_value res;
res.type = RDT_BOOL;
res.bool_ = 0;
if (argc > 0 || input.type != RDT_BOOL) {
res.bool_ = 0;
} else {
res.bool_ = input.bool_;
}
return res;
}
static struct rmsgpack_dom_value equals (
struct rmsgpack_dom_value input,
unsigned argc,
const struct argument * argv
) {
struct rmsgpack_dom_value res;
struct argument arg;
res.type = RDT_BOOL;
if (argc != 1) {
res.bool_ = 0;
} else {
arg = argv[0];
if (arg.type != AT_VALUE) {
res.bool_ = 0;
} else {
if (input.type == RDT_UINT && arg.value.type == RDT_INT) {
arg.value.type = RDT_UINT;
arg.value.uint_ = arg.value.int_;
}
res.bool_ = (rmsgpack_dom_value_cmp(&input, &arg.value) == 0);
}
}
return res;
}
static struct rmsgpack_dom_value operator_or (
struct rmsgpack_dom_value input,
unsigned argc,
const struct argument * argv
) {
struct rmsgpack_dom_value res;
unsigned i;
res.type = RDT_BOOL;
res.bool_ = 0;
for (i = 0; i < argc; i++) {
if (argv[i].type == AT_VALUE) {
res = equals(input, 1, &argv[i]);
} else {
res = is_true(
argv[i].invocation.func(input,
argv[i].invocation.argc,
argv[i].invocation.argv
),
0,
NULL
);
}
if (res.bool_) {
return res;
}
}
return res;
}
static struct rmsgpack_dom_value between (
struct rmsgpack_dom_value input,
unsigned argc,
const struct argument * argv
) {
struct rmsgpack_dom_value res;
unsigned i;
res.type = RDT_BOOL;
res.bool_ = 0;
if (argc != 2) {
return res;
}
if (argv[0].type != AT_VALUE || argv[1].type != AT_VALUE) {
return res;
}
if (argv[0].value.type != RDT_INT || argv[1].value.type != RDT_INT) {
return res;
}
switch (input.type) {
case RDT_INT:
res.bool_ = input.int_ >= argv[0].value.int_ && input.int_ <= argv[1].value.int_;
break;
case RDT_UINT:
res.bool_ = input.int_ >= argv[0].value.uint_ && input.int_ <= argv[1].value.int_;
break;
default:
return res;
}
return res;
}
static struct rmsgpack_dom_value operator_and (
struct rmsgpack_dom_value input,
unsigned argc,
const struct argument * argv
) {
struct rmsgpack_dom_value res;
unsigned i;
res.type = RDT_BOOL;
res.bool_ = 0;
for (i = 0; i < argc; i++) {
if (argv[i].type == AT_VALUE) {
res = equals(input, 1, &argv[i]);
} else {
res = is_true(
argv[i].invocation.func(input,
argv[i].invocation.argc,
argv[i].invocation.argv
),
0,
NULL
);
}
if (!res.bool_) {
return res;
}
}
return res;
}
static struct rmsgpack_dom_value all_map (
struct rmsgpack_dom_value input,
unsigned argc,
const struct argument * argv
) {
struct rmsgpack_dom_value res;
struct rmsgpack_dom_value key;
struct rmsgpack_dom_value * value = NULL;
struct argument arg;
struct rmsgpack_dom_value nil_value;
unsigned i;
nil_value.type = RDT_NULL;
res.type = RDT_BOOL;
res.bool_ = 1;
if (argc % 2 != 0) {
res.bool_ = 0;
return res;
}
if (input.type != RDT_MAP) {
return res;
}
for (i = 0; i < argc; i += 2) {
arg = argv[i];
if (arg.type != AT_VALUE) {
res.bool_ = 0;
goto clean;
}
value = rmsgpack_dom_value_map_value(&input, &arg.value);
if (!value) {
// All missing fields are nil
value = &nil_value;
}
arg = argv[i + 1];
if (arg.type == AT_VALUE) {
res = equals(*value, 1, &arg);
} else {
res = is_true(arg.invocation.func(
*value,
arg.invocation.argc,
arg.invocation.argv
), 0, NULL);
value = NULL;
}
if (!res.bool_) {
break;
}
}
clean:
return res;
}
struct registered_func registered_functions[100] = {
{"is_true", is_true},
{"or", operator_or},
{"and", operator_and},
{"between", between},
{NULL, NULL}
};
static struct buffer chomp(struct buffer buff) {
off_t i = 0;
for (; buff.offset < buff.len && isspace(buff.data[buff.offset]); buff.offset++) ;
return buff;
}
static struct buffer expect_char(
struct buffer buff,
char c,
const char ** error
) {
if (buff.offset >= buff.len) {
raise_unexpected_eof(buff.offset, error);
} else if (buff.data[buff.offset] != c) {
raise_unexpected_char(
buff.offset,
c,
buff.data[buff.offset],
error
);
} else {
buff.offset++;
}
return buff;
}
static struct buffer expect_eof(
struct buffer buff,
const char ** error
) {
buff = chomp(buff);
if (buff.offset < buff.len) {
raise_expected_eof(buff.offset, buff.data[buff.offset], error);
}
return buff;
}
static int peek(
struct buffer buff,
const char * data
) {
size_t remain = buff.len - buff.offset;
if (remain < strlen(data)) {
return 0;
}
return (strncmp(
buff.data + buff.offset,
data,
strlen(data)
) == 0);
}
static int is_eot(struct buffer buff) {
return (buff.offset >= buff.len);
}
static void peek_char(
struct buffer buff,
char * c,
const char ** error
) {
if (is_eot(buff)) {
raise_unexpected_eof(buff.offset, error);
return;
}
*c = buff.data[buff.offset];
}
static struct buffer get_char(
struct buffer buff,
char * c,
const char ** error
) {
if (is_eot(buff)) {
raise_unexpected_eof(buff.offset, error);
return buff;
}
*c = buff.data[buff.offset];
buff.offset++;
return buff;
}
static struct buffer parse_string(
struct buffer buff,
struct rmsgpack_dom_value * value,
const char ** error
) {
char terminator;
char c;
const char * str_start;
buff = get_char(buff, &terminator, error);
if (terminator != '"' && terminator != '\'') {
buff.offset--;
raise_expected_string(buff.offset, error);
}
str_start = buff.data + buff.offset;
buff = get_char(buff, &c, error);
while (!*error) {
if (c == terminator) {
break;
}
buff = get_char(buff, &c, error);
}
if (!*error) {
value->type = RDT_STRING;
value->string.len = (buff.data + buff.offset) - str_start - 1;
value->string.buff = calloc(
value->string.len + 1,
sizeof(char)
);
if (!value->string.buff) {
raise_enomem(error);
} else {
memcpy(
value->string.buff,
str_start,
value->string.len
);
}
}
return buff;
}
static struct buffer parse_integer(
struct buffer buff,
struct rmsgpack_dom_value * value,
const char ** error
) {
value->type = RDT_INT;
if (sscanf(buff.data + buff.offset, "%ld", &value->int_) == 0) {
raise_expected_number(buff.offset, error);
} else {
while (isdigit(buff.data[buff.offset])) {
buff.offset++;
}
}
return buff;
}
static struct buffer parse_value(
struct buffer buff,
struct rmsgpack_dom_value * value,
const char ** error
) {
buff = chomp(buff);
if (peek(buff, "nil")) {
buff.offset += strlen("nil");
value->type = RDT_NULL;
} else if (peek(buff, "true")) {
buff.offset += strlen("true");
value->type = RDT_BOOL;
value->bool_ = 1;
} else if (peek(buff, "false")) {
buff.offset += strlen("false");
value->type = RDT_BOOL;
value->bool_ = 0;
//} else if (peek(buff, "{")) {
//} else if (peek(buff, "[")) {
//} else if (peek(buff, "b\"") || peek(buff, "b'")) {
} else if (peek(buff, "\"") || peek(buff, "'")) {
buff = parse_string(buff, value, error);
} else if (isdigit(buff.data[buff.offset])) {
buff = parse_integer(buff, value, error);
}
return buff;
}
static struct buffer get_ident(
struct buffer buff,
const char ** ident,
size_t * len,
const char ** error
) {
char c;
if (is_eot(buff)) {
raise_unexpected_eof(buff.offset, error);
return buff;
}
*ident = buff.data + buff.offset;
*len = 0;
peek_char(buff, &c, error);
if (*error) {
goto clean;
}
if (!isalpha(c)) {
return buff;
}
buff.offset++;
*len = *len + 1;
peek_char(buff, &c, error);
while (!*error) {
if (!(isalpha(c) || isdigit(c) || c == '_')) {
break;
}
buff.offset++;
*len = *len + 1;
peek_char(buff, &c, error);
}
clean:
return buff;
}
static struct buffer parse_method_call(
struct buffer buff,
struct invocation * invocation,
const char ** error
) {
const char * func_name;
size_t func_name_len;
struct argument args[MAX_ARGS];
unsigned argi = 0;
unsigned i;
struct registered_func * rf = registered_functions;
invocation->func = NULL;
buff = get_ident(buff, &func_name, &func_name_len, error);
if (*error) {
goto clean;
}
buff = chomp(buff);
buff = expect_char(buff, '(', error);
if (*error) {
goto clean;
}
while (rf->name) {
if (strncmp(rf->name, func_name, func_name_len) == 0) {
invocation->func = rf->func;
break;
}
rf++;
}
if (!invocation->func) {
raise_unknown_function(
buff.offset,
func_name,
func_name_len,
error
);
goto clean;
}
buff = chomp(buff);
while (1) {
if (argi >= MAX_ARGS) {
raise_too_many_arguments(error);
goto clean;
}
buff = parse_argument(buff, &args[argi], error);
if (*error) {
goto clean;
}
argi++;
buff = chomp(buff);
buff = expect_char(buff, ',', error);
if (*error) {
*error = NULL;
break;
}
buff = chomp(buff);
}
buff = expect_char(buff, ')', error);
if (*error) {
goto clean;
}
invocation->argc = argi;
invocation->argv = malloc(sizeof(struct argument) * argi);
if (!invocation->argv) {
raise_enomem(error);
goto clean;
}
memcpy(invocation->argv, args, sizeof(struct argument) * argi);
goto success;
clean:
for (i = 0; i < argi; i++) {
argument_free(&args[i]);
}
success:
return buff;
}
static struct buffer parse_argument(
struct buffer buff,
struct argument * arg,
const char ** error
) {
buff = chomp(buff);
if (
isalpha(buff.data[buff.offset])
&& !(
peek(buff, "nil")
|| peek(buff, "true")
|| peek(buff, "false")
)
) {
arg->type = AT_FUNCTION;
buff = parse_method_call(buff, &arg->invocation, error);
} else {
arg->type = AT_VALUE;
buff = parse_value(buff, &arg->value, error);
}
return buff;
}
static struct buffer parse_table(
struct buffer buff,
struct invocation * invocation,
const char ** error
) {
struct argument args[MAX_ARGS];
unsigned argi = 0;
unsigned i;
memset(args, 0, sizeof(struct argument) * MAX_ARGS);
buff = chomp(buff);
buff = expect_char(buff, '{', error);
if (*error) {
goto clean;
}
buff = chomp(buff);
while (1) {
if (argi >= MAX_ARGS) {
raise_too_many_arguments(error);
goto clean;
}
buff = parse_string(buff, &args[argi].value, error);
if (*error) {
goto clean;
}
args[argi].type = AT_VALUE;
buff = chomp(buff);
argi++;
buff = expect_char(buff, ':', error);
if (*error) {
goto clean;
}
buff = chomp(buff);
if (argi >= MAX_ARGS) {
raise_too_many_arguments(error);
goto clean;
}
buff = parse_argument(buff, &args[argi], error);
if (*error) {
goto clean;
}
argi++;
buff = chomp(buff);
buff = expect_char(buff, ',', error);
if (*error) {
*error = NULL;
break;
}
buff = chomp(buff);
}
buff = expect_char(buff, '}', error);
if (*error) {
goto clean;
}
invocation->func = all_map;
invocation->argc = argi;
invocation->argv = malloc(sizeof(struct argument) * argi);
if (!invocation->argv) {
raise_enomem(error);
goto clean;
}
memcpy(invocation->argv, args, sizeof(struct argument) * argi);
goto success;
clean:
for (i = 0; i < argi; i++) {
argument_free(&args[i]);
}
success:
return buff;
}
void rarchdb_query_free(rarchdb_query * q) {
struct query * real_q = q;
unsigned i;
real_q->ref_count--;
if (real_q->ref_count > 0) {
return;
}
for (i = 0; i < real_q->root.argc; i++) {
argument_free(&real_q->root.argv[i]);
}
}
rarchdb_query * rarchdb_query_compile(
struct rarchdb * db,
const char * query,
size_t buff_len,
const char ** error
) {
struct buffer buff;
struct query * q;
q = malloc(sizeof(struct query));
if (!q) {
goto clean;
}
memset(q, 0, sizeof(struct query));
q->ref_count = 1;
buff.data = query;
buff.len = buff_len;
buff.offset = 0;
*error = NULL;
buff = chomp(buff);
if (peek(buff, "{")) {
buff = parse_table(buff, &q->root, error);
if (*error) {
goto clean;
}
}
buff = expect_eof(buff, error);
if (*error) {
goto clean;
}
goto success;
clean:
if (q) {
rarchdb_query_free(q);
}
success:
return q;
}
void rarchdb_query_inc_ref(rarchdb_query * q) {
struct query * rq = q;
rq->ref_count += 1;
}
int rarchdb_query_filter(
rarchdb_query * q,
struct rmsgpack_dom_value * v
) {
struct invocation inv = ((struct query *)q)->root;
struct rmsgpack_dom_value res = inv.func(*v, inv.argc, inv.argv);
return (res.type == RDT_BOOL && res.bool_);
}

13
libretrodb/query.h Normal file
View File

@ -0,0 +1,13 @@
#ifndef __RARCHDB_QUERY_H__
#define __RARCHDB_QUERY_H__
#include "rarchdb.h"
void rarchdb_query_inc_ref(rarchdb_query * q);
void rarchdb_query_dec_ref(rarchdb_query * q);
int rarchdb_query_filter(
rarchdb_query * q,
struct rmsgpack_dom_value * v
);
#endif

489
libretrodb/rarchdb.c Normal file
View File

@ -0,0 +1,489 @@
#include "rarchdb.h"
#include <sys/types.h>
#ifdef _WIN32
#include <direct.h>
#else
#include <unistd.h>
#endif
#include <string.h>
#include <errno.h>
#include <sys/stat.h>
#include <stdlib.h>
#include <fcntl.h>
#include <stdio.h>
#include "rmsgpack_dom.h"
#include "rmsgpack.h"
#include "bintree.h"
#include "rarchdb_endian.h"
#include "query.h"
#define MAGIC_NUMBER "RARCHDB"
struct rarchdb_header {
char magic_number[sizeof(MAGIC_NUMBER)-1];
uint64_t metadata_offset;
};
struct rarchdb_metadata {
uint64_t count;
};
struct rarchdb_index {
char name[50];
uint64_t key_size;
uint64_t next;
};
struct node_iter_ctx {
struct rarchdb * db;
struct rarchdb_index * idx;
};
static struct rmsgpack_dom_value sentinal;
static int rarchdb_read_metadata(
int fd,
struct rarchdb_metadata * md
){
return rmsgpack_dom_read_into(fd, "count", &md->count, NULL);
}
static int rarchdb_write_metadata(
int fd,
struct rarchdb_metadata * md
){
rmsgpack_write_map_header(fd, 1);
rmsgpack_write_string(fd, "count", strlen("count"));
return rmsgpack_write_uint(fd, md->count);
}
static int validate_document(const struct rmsgpack_dom_value * doc) {
int rv = 0;
unsigned i;
struct rmsgpack_dom_value key;
struct rmsgpack_dom_value value;
if (doc->type != RDT_MAP) {
return -EINVAL;
}
for (i = 0; i < doc->map.len; i++) {
key = doc->map.items[i].key;
value = doc->map.items[i].value;
if (key.type != RDT_STRING) {
return -EINVAL;
}
if (key.string.len <= 0) {
return -EINVAL;
}
if (key.string.buff[0] == '$') {
return -EINVAL;
}
if (value.type == RDT_MAP) {
if ((rv == validate_document(&value)) != 0) {
return rv;
}
}
}
return rv;
}
int rarchdb_create(
int fd,
rarchdb_value_provider value_provider,
void * ctx
){
int rv;
uint64_t item_count = 0;
struct rmsgpack_dom_value item = {};
struct rarchdb_header header = {};
struct rarchdb_metadata md;
memcpy(header.magic_number, MAGIC_NUMBER, sizeof(MAGIC_NUMBER)-1);
off_t root = lseek(fd, 0, SEEK_CUR);
// We write the header in the end because we need to know the size of
// the db first
lseek(fd, sizeof(struct rarchdb_header), SEEK_CUR);
while ((rv = value_provider(ctx, &item)) == 0) {
if ((rv = validate_document(&item)) < 0)
goto clean;
if ((rv = rmsgpack_dom_write(fd, &item)) < 0)
goto clean;
item_count++;
}
if (rv < 0)
goto clean;
if ((rv = rmsgpack_dom_write(fd, &sentinal)) < 0)
goto clean;
header.metadata_offset = httobe64(lseek(fd, 0, SEEK_CUR));
md.count = item_count;
rarchdb_write_metadata(fd, &md);
lseek(fd, root, SEEK_SET);
write(fd, &header, sizeof(header));
clean:
rmsgpack_dom_value_free(&item);
return rv;
}
static int rarchdb_read_index_header(
int fd,
struct rarchdb_index * idx
){
uint64_t name_len = 50;
return rmsgpack_dom_read_into(
fd,
"name", idx->name, &name_len,
"key_size", &idx->key_size,
"next", &idx->next,
NULL
);
}
static void rarchdb_write_index_header(
int fd,
struct rarchdb_index * idx
){
rmsgpack_write_map_header(fd, 3);
rmsgpack_write_string(fd, "name", strlen("name"));
rmsgpack_write_string(fd, idx->name, strlen(idx->name));
rmsgpack_write_string(fd, "key_size", strlen("key_size"));
rmsgpack_write_uint(fd, idx->key_size);
rmsgpack_write_string(fd, "next", strlen("next"));
rmsgpack_write_uint(fd, idx->next);
}
void rarchdb_close(struct rarchdb * db) {
close(db->fd);
db->fd = -1;
}
int rarchdb_open(
const char * path,
struct rarchdb * db
){
struct rarchdb_header header;
struct rarchdb_metadata md;
int rv;
int fd = open(path, O_RDWR);
if (fd == -1)
return -errno;
db->root = lseek(fd, 0, SEEK_CUR);
if ((rv = read(fd, &header, sizeof(header))) == -1) {
rv = -errno;
goto error;
}
if (strncmp(header.magic_number, MAGIC_NUMBER, sizeof(MAGIC_NUMBER)) != 0) {
rv = -EINVAL;
goto error;
}
header.metadata_offset = betoht64(header.metadata_offset);
lseek(fd, header.metadata_offset, SEEK_SET);
if (rarchdb_read_metadata(fd, &md) < 0) {
rv = -EINVAL;
goto error;
}
db->count = md.count;
db->first_index_offset = lseek(fd, 0, SEEK_CUR);
db->fd = fd;
return 0;
error:
close(fd);
return rv;
}
static int rarchdb_find_index(
struct rarchdb * db,
const char * index_name,
struct rarchdb_index * idx
){
off_t eof = lseek(db->fd, 0, SEEK_END);
off_t offset = lseek(db->fd, db->first_index_offset, SEEK_SET);
while (offset < eof) {
rarchdb_read_index_header(db->fd, idx);
if (strncmp(index_name, idx->name, strlen(idx->name)) == 0)
return 0;
offset = lseek(db->fd, idx->next, SEEK_CUR);
}
return -1;
}
static int node_compare(
const void * a,
const void * b,
void * ctx
){
return memcmp(a, b, *(uint8_t *)ctx);
}
static int binsearch(
const void * buff,
const void * item,
uint64_t count,
uint8_t field_size,
uint64_t * offset
){
int mid = count / 2;
int item_size = field_size + sizeof(uint64_t);
uint64_t * current = (uint64_t *)buff + (mid * item_size);
int rv = node_compare(current, item, &field_size);
if (rv == 0) {
*offset = *(uint64_t *)(current + field_size);
return 0;
} else if (count == 0)
return -1;
else if (rv > 0)
return binsearch(buff, item, mid, field_size, offset);
return binsearch(current + item_size, item, count - mid, field_size, offset);
}
int rarchdb_find_entry(
struct rarchdb * db,
const char * index_name,
const void * key,
struct rmsgpack_dom_value * out
) {
struct rarchdb_index idx;
int rv;
void * buff;
uint64_t offset;
ssize_t bufflen, nread = 0;
if (rarchdb_find_index(db, index_name, &idx) < 0) {
return -1;
}
bufflen = idx.next;
buff = malloc(bufflen);
if (!buff)
return -ENOMEM;
while (nread < bufflen) {
void * buff_ = (uint64_t *)buff + nread;
rv = read(db->fd, buff_, bufflen - nread);
if (rv <= 0) {
free(buff);
return -errno;
}
nread += rv;
}
rv = binsearch(buff, key, db->count, idx.key_size, &offset);
free(buff);
if (rv == 0)
lseek(db->fd, offset, SEEK_SET);
rv = rmsgpack_dom_read(db->fd, out);
if (rv < 0)
return rv;
return rv;
}
int rarchdb_cursor_reset(struct rarchdb_cursor * cursor) {
cursor->eof = 0;
return lseek(
cursor->fd,
cursor->db->root + sizeof(struct rarchdb_header),
SEEK_SET
);
}
int rarchdb_cursor_read_item(
struct rarchdb_cursor * cursor,
struct rmsgpack_dom_value * out
) {
int rv;
if (cursor->eof)
return EOF;
retry:
rv = rmsgpack_dom_read(cursor->fd, out);
if (rv < 0)
return rv;
if (out->type == RDT_NULL) {
cursor->eof = 1;
return EOF;
}
if (cursor->query) {
if (!rarchdb_query_filter(cursor->query, out)) {
goto retry;
}
}
return 0;
}
void rarchdb_cursor_close(struct rarchdb_cursor * cursor) {
close(cursor->fd);
cursor->is_valid = 0;
cursor->fd = -1;
cursor->eof = 1;
cursor->db = NULL;
if (cursor->query) {
rarchdb_query_free(cursor->query);
}
cursor->query = NULL;
}
int rarchdb_cursor_open(
struct rarchdb * db,
struct rarchdb_cursor * cursor,
rarchdb_query * q
) {
cursor->fd = dup(db->fd);
if (cursor->fd == -1) {
return -errno;
}
cursor->db = db;
cursor->is_valid = 1;
rarchdb_cursor_reset(cursor);
cursor->query = q;
if (q) {
rarchdb_query_inc_ref(q);
}
return 0;
}
static int node_iter(
void * value,
void * ctx
){
struct node_iter_ctx * nictx = (struct node_iter_ctx *)ctx;
if (write(nictx->db->fd, value, nictx->idx->key_size + sizeof(uint64_t)) > 0)
return 0;
return -1;
}
static uint64_t rarchdb_tell(struct rarchdb * db) {
return lseek(db->fd, 0, SEEK_CUR);
}
int rarchdb_create_index(
struct rarchdb * db,
const char * name,
const char * field_name
){
int rv;
struct node_iter_ctx nictx;
struct rmsgpack_dom_value key;
struct rarchdb_index idx;
struct rmsgpack_dom_value item;
struct rmsgpack_dom_value * field;
void * buff = NULL;
uint64_t * buff_u64 = NULL;
uint8_t field_size = 0;
struct bintree tree;
uint64_t item_loc = rarchdb_tell(db);
uint64_t idx_header_offset;
struct rarchdb_cursor cur;
bintree_new(&tree, node_compare, &field_size);
if (rarchdb_cursor_open(db, &cur, NULL) != 0) {
rv = -1;
goto clean;
}
key.type = RDT_STRING;
key.string.len = strlen(field_name);
// We know we aren't going to change it
key.string.buff = (char *) field_name;
while (rarchdb_cursor_read_item(&cur, &item) == 0) {
if (item.type != RDT_MAP) {
rv = -EINVAL;
printf("Only map keys are supported\n");
goto clean;
}
field = rmsgpack_dom_value_map_value(&item, &key);
if (!field) {
rv = -EINVAL;
printf("field not found in item\n");
goto clean;
}
if (field->type != RDT_BINARY) {
rv = -EINVAL;
printf("field is not binary\n");
goto clean;
}
if (field->binary.len == 0) {
rv = -EINVAL;
printf("field is empty\n");
goto clean;
}
if (field_size == 0)
field_size = field->binary.len;
else if (field->binary.len != field_size) {
rv = -EINVAL;
printf("field is not of correct size\n");
goto clean;
}
buff = malloc(field_size + sizeof(uint64_t));
if (!buff) {
rv = -ENOMEM;
goto clean;
}
memcpy(buff, field->binary.buff, field_size);
buff_u64 = (uint64_t *)buff + field_size;
memcpy(buff_u64, &item_loc, sizeof(uint64_t));
if (bintree_insert(&tree, buff) != 0) {
printf("Value is not unique: ");
rmsgpack_dom_value_print(field);
printf("\n");
rv = -EINVAL;
goto clean;
}
buff = NULL;
rmsgpack_dom_value_free(&item);
item_loc = rarchdb_tell(db);
}
(void)rv;
(void)idx_header_offset;
idx_header_offset = lseek(db->fd, 0, SEEK_END);
strncpy(idx.name, name, 50);
idx.name[49] = '\0';
idx.key_size = field_size;
idx.next = db->count * (field_size + sizeof(uint64_t));
rarchdb_write_index_header(db->fd, &idx);
nictx.db = db;
nictx.idx = &idx;
bintree_iterate(&tree, node_iter, &nictx);
bintree_free(&tree);
clean:
rmsgpack_dom_value_free(&item);
if (buff)
free(buff);
if (cur.is_valid) {
rarchdb_cursor_close(&cur);
}
return 0;
}

73
libretrodb/rarchdb.h Normal file
View File

@ -0,0 +1,73 @@
#ifndef __RARCHDB_H__
#define __RARCHDB_H__
#include <stdint.h>
#include <unistd.h>
#include "rmsgpack_dom.h"
typedef void rarchdb_query;
struct rarchdb {
int fd;
uint64_t root;
uint64_t count;
uint64_t first_index_offset;
};
struct rarchdb_cursor {
int is_valid;
int fd;
int eof;
rarchdb_query * query;
struct rarchdb * db;
};
typedef int (* rarchdb_value_provider)(
void * ctx,
struct rmsgpack_dom_value * out
);
int rarchdb_create(
int fd,
rarchdb_value_provider value_provider,
void * ctx
);
void rarchdb_close(struct rarchdb * db);
int rarchdb_open(
const char * path,
struct rarchdb * db
);
int rarchdb_create_index(
struct rarchdb * db,
const char * name,
const char * field_name
);
int rarchdb_find_entry(
struct rarchdb * db,
const char * index_name,
const void * key,
struct rmsgpack_dom_value * out
);
int rarchdb_cursor_open(
struct rarchdb * db,
struct rarchdb_cursor * cursor,
rarchdb_query * query
);
void rarchdb_cursor_close(struct rarchdb_cursor * cursor);
rarchdb_query * rarchdb_query_compile(
struct rarchdb * db,
const char * query,
size_t buff_len,
const char ** error
);
void rarchdb_query_free(rarchdb_query * q);
int rarchdb_cursor_read_item(
struct rarchdb_cursor * cursor,
struct rmsgpack_dom_value * out
);
#endif

View File

@ -0,0 +1,28 @@
#ifndef __RARCHDB_MSGPACK_ENDIAN_H
#define __RARCHDB_MSGPACK_ENDIAN_H
#include <stdint.h>
#include <retro_endianness.h>
#ifndef swap64
#define swap64(val) \
((((uint64_t)(val) & 0x00000000000000ffULL) << 56) \
| (((uint64_t)(val) & 0x000000000000ff00ULL) << 40) \
| (((uint64_t)(val) & 0x0000000000ff0000ULL) << 24) \
| (((uint64_t)(val) & 0x00000000ff000000ULL) << 8) \
| (((uint64_t)(val) & 0x000000ff00000000ULL) >> 8) \
| (((uint64_t)(val) & 0x0000ff0000000000ULL) >> 24) \
| (((uint64_t)(val) & 0x00ff000000000000ULL) >> 40) \
| (((uint64_t)(val) & 0xff00000000000000ULL) >> 56))
#endif
#define httobe64(x) (is_little_endian() ? swap64(x) : (x))
#define httobe32(x) (is_little_endian() ? SWAP32(x) : (x))
#define httobe16(x) (is_little_endian() ? SWAP16(x) : (x))
#define betoht16(x) httobe16(x)
#define betoht32(x) httobe32(x)
#define betoht64(x) httobe64(x)
#endif

87
libretrodb/rarchdb_tool.c Normal file
View File

@ -0,0 +1,87 @@
#include <stdio.h>
#include <string.h>
#include "rarchdb.h"
#include "rmsgpack_dom.h"
int main(
int argc,
char ** argv
){
int rv;
struct rarchdb db;
struct rarchdb_cursor cur;
struct rmsgpack_dom_value item;
if (argc < 3) {
printf("Usage: %s <db file> <command> [extra args...]\n", argv[0]);
printf("Available Commands:\n");
printf("\tlist\n");
printf("\tcreate-index <index name> <field name>\n");
printf("\tfind <query expression>\n");
return 1;
}
const char * command = argv[2];
const char * path = argv[1];
if ((rv = rarchdb_open(path, &db)) != 0) {
printf("Could not open db file '%s': %s\n", path, strerror(-rv));
return 1;
}
else if (strcmp(command, "list") == 0) {
if ((rv = rarchdb_cursor_open(&db, &cur, NULL)) != 0) {
printf("Could not open cursor: %s\n", strerror(-rv));
return 1;
}
if (argc != 3) {
printf("Usage: %s <db file> list\n", argv[0]);
return 1;
}
while (rarchdb_cursor_read_item(&cur, &item) == 0) {
rmsgpack_dom_value_print(&item);
printf("\n");
rmsgpack_dom_value_free(&item);
}
}
else if (strcmp(command, "find") == 0) {
if (argc != 4) {
printf("Usage: %s <db file> find <query expression>\n", argv[0]);
return 1;
}
const char * query_exp = argv[3];
const char * error = NULL;
rarchdb_query * q = rarchdb_query_compile(&db, query_exp, strlen(query_exp), &error);
if (error) {
printf("%s\n", error);
return 1;
}
if ((rv = rarchdb_cursor_open(&db, &cur, q)) != 0) {
printf("Could not open cursor: %s\n", strerror(-rv));
return 1;
}
while (rarchdb_cursor_read_item(&cur, &item) == 0) {
rmsgpack_dom_value_print(&item);
printf("\n");
rmsgpack_dom_value_free(&item);
}
} else if (strcmp(command, "create-index") == 0) {
const char * index_name, * field_name;
if (argc != 5) {
printf("Usage: %s <db file> create-index <index name> <field name>\n", argv[0]);
return 1;
}
index_name = argv[3];
field_name = argv[4];
rarchdb_create_index(&db, index_name, field_name);
} else {
printf("Unkown command %s\n", argv[1]);
return 1;
}
rarchdb_close(&db);
}

View File

@ -0,0 +1,88 @@
/* Copyright (C) 2010-2014 The RetroArch team
*
* ---------------------------------------------------------------------------------------
* The following license statement only applies to this file (retro_endianness.h).
* ---------------------------------------------------------------------------------------
*
* Permission is hereby granted, free of charge,
* to any person obtaining a copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation the rights to
* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
* and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef __LIBRETRO_SDK_ENDIANNESS_H
#define __LIBRETRO_SDK_ENDIANNESS_H
#include <retro_inline.h>
#include <stdint.h>
#define SWAP16(x) ((uint16_t)( \
(((uint16_t)(x) & 0x00ff) << 8) | \
(((uint16_t)(x) & 0xff00) >> 8) \
))
#define SWAP32(x) ((uint32_t)( \
(((uint32_t)(x) & 0x000000ff) << 24) | \
(((uint32_t)(x) & 0x0000ff00) << 8) | \
(((uint32_t)(x) & 0x00ff0000) >> 8) | \
(((uint32_t)(x) & 0xff000000) >> 24) \
))
static INLINE uint8_t is_little_endian(void){
union {
uint16_t x;
uint8_t y[2];
} u;
u.x = 1;
return u.y[0];
}
static INLINE uint32_t swap_if_big32(uint32_t val){
if (is_little_endian())
return val;
return (val >> 24) | ((val >> 8) & 0xFF00) |
((val << 8) & 0xFF0000) | (val << 24);
}
static INLINE uint32_t swap_if_little32(uint32_t val){
if (is_little_endian())
return (val >> 24) | ((val >> 8) & 0xFF00) |
((val << 8) & 0xFF0000) | (val << 24);
return val;
}
static INLINE uint16_t swap_if_big16(uint16_t val){
if (is_little_endian())
return val;
return (val >> 8) | (val << 8);
}
static INLINE uint16_t swap_if_little16(uint16_t val){
if (is_little_endian())
return (val >> 8) | (val << 8);
return val;
}
static INLINE void store32be(
uint32_t * addr,
uint32_t data
){
*addr = is_little_endian() ? SWAP32(data) : data;
}
static INLINE uint32_t load32be(const uint32_t * addr){
return is_little_endian() ? SWAP32(*addr) : *addr;
}
#endif

40
libretrodb/retro_inline.h Normal file
View File

@ -0,0 +1,40 @@
/* Copyright (C) 2010-2014 The RetroArch team
*
* ---------------------------------------------------------------------------------------
* The following license statement only applies to this file (retro_inline.h).
* ---------------------------------------------------------------------------------------
*
* Permission is hereby granted, free of charge,
* to any person obtaining a copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation the rights to
* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
* and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef __LIBRETRO_SDK_INLINE_H
#define __LIBRETRO_SDK_INLINE_H
#if !defined(__cplusplus) && defined(_WIN32)
#ifndef INLINE
#define INLINE _inline
#endif
#else
#ifndef INLINE
#define INLINE inline
#endif
#endif
#endif

542
libretrodb/rmsgpack.c Normal file
View File

@ -0,0 +1,542 @@
/*
* An almost complete implementation of msgpack by
* Saggi Mizrahi <ficoos@gmail.com>
*
* TODO:
* - float types
* - ext types
*
* For more information http://msgpack.org/
*/
#include "rmsgpack.h"
#include <stdlib.h>
#ifdef _WIN32
#include <direct.h>
#else
#include <unistd.h>
#endif
#include <errno.h>
#include <stdint.h>
#include <string.h>
#include "rarchdb_endian.h"
static const uint8_t MPF_FIXMAP = 0x80;
static const uint8_t MPF_MAP16 = 0xde;
static const uint8_t MPF_MAP32 = 0xdf;
static const uint8_t MPF_FIXARRAY = 0x90;
static const uint8_t MPF_ARRAY16 = 0xdc;
static const uint8_t MPF_ARRAY32 = 0xdd;
static const uint8_t MPF_FIXSTR = 0xa0;
static const uint8_t MPF_STR8 = 0xd9;
static const uint8_t MPF_STR16 = 0xda;
static const uint8_t MPF_STR32 = 0xdb;
static const uint8_t MPF_BIN8 = 0xc4;
static const uint8_t MPF_BIN16 = 0xc5;
static const uint8_t MPF_BIN32 = 0xc6;
static const uint8_t MPF_FALSE = 0xc2;
static const uint8_t MPF_TRUE = 0xc3;
static const uint8_t MPF_INT8 = 0xd0;
static const uint8_t MPF_INT16 = 0xd1;
static const uint8_t MPF_INT32 = 0xd2;
static const uint8_t MPF_INT64 = 0xd3;
static const uint8_t MPF_UINT8 = 0xcc;
static const uint8_t MPF_UINT16 = 0xcd;
static const uint8_t MPF_UINT32 = 0xce;
static const uint8_t MPF_UINT64 = 0xcf;
static const uint8_t MPF_NIL = 0xc0;
int rmsgpack_write_array_header(
int fd,
uint32_t size
){
uint16_t tmp_i16;
uint32_t tmp_i32;
if (size < 16) {
size = (size | MPF_FIXARRAY);
if (write(fd, &size, sizeof(int8_t)) == -1)
return -errno;
return sizeof(int8_t);
} else if (size == (uint16_t)size) {
if (write(fd, &MPF_ARRAY16, sizeof(MPF_ARRAY16)) == -1)
return -errno;
tmp_i16 = httobe16(size);
if (write(fd, (void *)(&tmp_i16), sizeof(uint16_t)) == -1)
return -errno;
return sizeof(int8_t) + sizeof(uint16_t);
} else {
if (write(fd, &MPF_ARRAY32, sizeof(MPF_ARRAY32)) == -1)
return -errno;
tmp_i32 = httobe32(size);
if (write(fd, (void *)(&tmp_i32), sizeof(uint32_t)) == -1)
return -errno;
return sizeof(int8_t) + sizeof(uint32_t);
}
}
int rmsgpack_write_map_header(
int fd,
uint32_t size
){
uint16_t tmp_i16;
uint32_t tmp_i32;
if (size < 16) {
size = (size | MPF_FIXMAP);
if (write(fd, &size, sizeof(int8_t)) == -1)
return -errno;
return sizeof(int8_t);
} else if (size < (uint16_t)size) {
if (write(fd, &MPF_MAP16, sizeof(MPF_MAP16)) == -1)
return -errno;
tmp_i16 = httobe16(size);
if (write(fd, (void *)(&tmp_i16), sizeof(uint16_t)) == -1)
return -errno;
return sizeof(uint8_t) + sizeof(uint16_t);
} else {
tmp_i32 = httobe32(size);
if (write(fd, &MPF_MAP32, sizeof(MPF_MAP32)) == -1)
return -errno;
if (write(fd, (void *)(&tmp_i32), sizeof(uint32_t)) == -1)
return -errno;
return sizeof(int8_t) + sizeof(uint32_t);
}
}
int rmsgpack_write_string(
int fd,
const char * s,
uint32_t len
){
int8_t fixlen = 0;
uint16_t tmp_i16;
uint32_t tmp_i32;
int written = sizeof(int8_t);
if (len < 32) {
fixlen = len | MPF_FIXSTR;
if (write(fd, &fixlen, sizeof(int8_t)) == -1)
return -errno;
} else if (len < 1<<8) {
if (write(fd, &MPF_STR8, sizeof(MPF_STR8)) == -1)
return -errno;
if (write(fd, &len, sizeof(uint8_t)) == -1)
return -errno;
written += sizeof(uint8_t);
} else if (len < 1<<16) {
if (write(fd, &MPF_STR16, sizeof(MPF_STR16)) == -1)
return -errno;
tmp_i16 = httobe16(len);
if (write(fd, &tmp_i16, sizeof(uint16_t)) == -1)
return -errno;
written += sizeof(uint16_t);
} else {
if (write(fd, &MPF_STR32, sizeof(MPF_STR32)) == -1)
return -errno;
tmp_i32 = httobe32(len);
if (write(fd, &tmp_i32, sizeof(uint32_t)) == -1)
return -errno;
written += sizeof(uint32_t);
}
if (write(fd, s, len) == -1)
return -errno;
written += len;
return written;
}
int rmsgpack_write_bin(
int fd,
const void * s,
uint32_t len
){
uint16_t tmp_i16;
uint32_t tmp_i32;
int written = sizeof(int8_t);
if (len == (uint8_t)len) {
if (write(fd, &MPF_BIN8, sizeof(MPF_BIN8)) == -1)
return -errno;
if (write(fd, &len, sizeof(uint8_t)) == -1)
return -errno;
written += sizeof(uint8_t);
} else if (len == (uint16_t)len) {
if (write(fd, &MPF_BIN16, sizeof(MPF_BIN16)) == -1)
return -errno;
tmp_i16 = httobe16(len);
if (write(fd, &tmp_i16, sizeof(uint16_t)) == -1)
return -errno;
written += sizeof(uint16_t);
} else {
if (write(fd, &MPF_BIN32, sizeof(MPF_BIN32)) == -1)
return -errno;
tmp_i32 = httobe32(len);
if (write(fd, &tmp_i32, sizeof(uint32_t)) == -1)
return -errno;
written += sizeof(uint32_t);
}
if (write(fd, s, len) == -1)
return -errno;
written += len;
return 0;
}
int rmsgpack_write_nil(int fd){
if (write(fd, &MPF_NIL, sizeof(MPF_NIL)) == -1)
return -errno;
return sizeof(uint8_t);
}
int rmsgpack_write_bool(
int fd,
int value
){
if (value) {
if (write(fd, &MPF_TRUE, sizeof(MPF_TRUE)) == -1)
return -errno;
} else {
if (write(fd, &MPF_FALSE, sizeof(MPF_FALSE)) == -1)
return -errno;
}
return sizeof(uint8_t);
}
int rmsgpack_write_int(
int fd,
int64_t value
){
uint8_t tmpval = 0;
int16_t tmp_i16;
int32_t tmp_i32;
int written = sizeof(uint8_t);
if (value >=0 && value < 128) {
if (write(fd, &value, sizeof(int8_t)) == -1)
return -errno;
} else if (value < 0 && value > -32) {
tmpval = (value) | 0xe0;
if (write(fd, &tmpval, sizeof(uint8_t)) == -1)
return -errno;
} else if (value == (int8_t)value) {
if (write(fd, &MPF_INT8, sizeof(MPF_INT8)) == -1)
return -errno;
if (write(fd, &value, sizeof(int8_t)) == -1)
return -errno;
written += sizeof(int8_t);
} else if (value == (int16_t)value) {
if (write(fd, &MPF_INT16, sizeof(MPF_INT16)) == -1)
return -errno;
tmp_i16 = httobe16(value);
if (write(fd, &tmp_i16, sizeof(int16_t)) == -1)
return -errno;
written += sizeof(int16_t);
} else if (value == (int32_t)value) {
if (write(fd, &MPF_INT32, sizeof(MPF_INT32)) == -1)
return -errno;
tmp_i32 = httobe32(value);
if (write(fd, &tmp_i32, sizeof(int32_t)) == -1)
return -errno;
written += sizeof(int32_t);
} else {
if (write(fd, &MPF_INT64, sizeof(MPF_INT64)) == -1)
return -errno;
value = httobe64(value);
if (write(fd, &value, sizeof(int64_t)) == -1)
return -errno;
written += sizeof(int64_t);
}
return written;
}
int rmsgpack_write_uint(
int fd,
uint64_t value
){
uint16_t tmp_i16;
uint32_t tmp_i32;
int written = sizeof(uint8_t);
if (value == (uint8_t)value) {
if (write(fd, &MPF_UINT8, sizeof(MPF_UINT8)) == -1)
return -errno;
if (write(fd, &value, sizeof(uint8_t)) == -1)
return -errno;
written += sizeof(uint8_t);
} else if (value == (uint16_t)value) {
if (write(fd, &MPF_UINT16, sizeof(MPF_UINT16)) == -1)
return -errno;
tmp_i16 = httobe16(value);
if (write(fd, &tmp_i16, sizeof(uint16_t)) == -1)
return -errno;
written += sizeof(uint16_t);
} else if (value == (uint32_t)value) {
if (write(fd, &MPF_UINT32, sizeof(MPF_UINT32)) == -1)
return -errno;
tmp_i32 = httobe32(value);
if (write(fd, &tmp_i32, sizeof(uint32_t)) == -1)
return -errno;
written += sizeof(uint32_t);
} else {
if (write(fd, &MPF_UINT64, sizeof(MPF_UINT64)) == -1)
return -errno;
value = httobe64(value);
if (write(fd, &value, sizeof(uint64_t)) == -1)
return -errno;
written += sizeof(uint64_t);
}
return written;
}
static int read_uint(
int fd,
uint64_t * out,
size_t size
){
uint64_t tmp;
if (read(fd, &tmp, size) == -1)
return -errno;
switch (size) {
case 1:
*out = *(uint8_t *)(&tmp);
break;
case 2:
*out = betoht16(tmp);
break;
case 4:
*out = betoht32(tmp);
break;
case 8:
*out = betoht64(tmp);
break;
}
return 0;
}
static int read_int(
int fd,
int64_t * out,
size_t size
){
uint8_t tmp8 = 0;
uint16_t tmp16;
uint32_t tmp32;
uint64_t tmp64;
if (read(fd, &tmp64, size) == -1)
return -errno;
(void)tmp8;
switch (size) {
case 1:
*out = *((int8_t *)(&tmp64));
break;
case 2:
tmp16 = betoht16(tmp64);
*out = *((int16_t *)(&tmp16));
break;
case 4:
tmp32 = betoht32(tmp64);
*out = *((int32_t *)(&tmp32));
break;
case 8:
tmp64 = betoht64(tmp64);
*out = *((int64_t *)(&tmp64));
break;
}
return 0;
}
static int read_buff(
int fd,
size_t size,
char ** pbuff,
uint64_t * len
){
uint64_t tmp_len = 0;
if (read_uint(fd, &tmp_len, size) == -1)
return -errno;
*pbuff = (char *)calloc(tmp_len + 1, sizeof(char));
if (read(fd, *pbuff, tmp_len) == -1) {
free(*pbuff);
return -errno;
}
*len = tmp_len;
return 0;
}
static int read_map(
int fd,
uint32_t len,
struct rmsgpack_read_callbacks * callbacks,
void * data
){
int rv;
unsigned i;
if (
callbacks->read_map_start &&
(rv = callbacks->read_map_start(len, data)) < 0)
return rv;
for (i = 0; i < len; i++) {
if ((rv = rmsgpack_read(fd, callbacks, data)) < 0)
return rv;
if ((rv = rmsgpack_read(fd, callbacks, data)) < 0)
return rv;
}
return 0;
}
static int read_array(
int fd,
uint32_t len,
struct rmsgpack_read_callbacks * callbacks,
void * data
){
int rv;
unsigned i;
if (
callbacks->read_array_start &&
(rv = callbacks->read_array_start(len, data)) < 0)
return rv;
for (i = 0; i < len; i++) {
if ((rv = rmsgpack_read(fd, callbacks, data)) < 0)
return rv;
}
return 0;
}
int rmsgpack_read(
int fd,
struct rmsgpack_read_callbacks * callbacks,
void * data
){
int rv;
uint64_t tmp_len = 0;
uint64_t tmp_uint = 0;
int64_t tmp_int = 0;
uint8_t type = 0;
char * buff = NULL;
if (read(fd, &type, sizeof(uint8_t)) == -1)
return -errno;
if (type < MPF_FIXMAP) {
if (!callbacks->read_int)
return 0;
return callbacks->read_int(type, data);
} else if (type < MPF_FIXARRAY) {
tmp_len = type - MPF_FIXMAP;
return read_map(fd, tmp_len, callbacks, data);
} else if (type < MPF_FIXSTR) {
tmp_len = type - MPF_FIXARRAY;
return read_array(fd, tmp_len, callbacks, data);
} else if (type < MPF_NIL) {
tmp_len = type - MPF_FIXSTR;
buff = (char *)calloc(tmp_len + 1, sizeof(char));
if (!buff)
return -ENOMEM;
if (read(fd, buff, tmp_len) == -1) {
free(buff);
return -errno;
}
buff[tmp_len] = '\0';
if (!callbacks->read_string) {
free(buff);
return 0;
}
return callbacks->read_string(buff, tmp_len, data);
} else if (type > MPF_MAP32) {
if (!callbacks->read_int)
return 0;
return callbacks->read_int(type - 0xff - 1, data);
}
switch (type) {
case 0xc0:
if (callbacks->read_nil)
return callbacks->read_nil(data);
break;
case 0xc2:
if (callbacks->read_bool)
return callbacks->read_bool(0, data);
break;
case 0xc3:
if (callbacks->read_bool)
return callbacks->read_bool(1, data);
break;
case 0xc4:
case 0xc5:
case 0xc6:
if ((rv = read_buff(fd, 1<<(type - 0xc4), &buff, &tmp_len)) < 0)
return rv;
if (callbacks->read_bin)
return callbacks->read_bin(buff, tmp_len, data);
break;
case 0xcc:
case 0xcd:
case 0xce:
case 0xcf:
tmp_len = 1ULL << (type - 0xcc);
tmp_uint = 0;
if (read_uint(fd, &tmp_uint, tmp_len) == -1)
return -errno;
if (callbacks->read_uint)
return callbacks->read_uint(tmp_uint, data);
break;
case 0xd0:
case 0xd1:
case 0xd2:
case 0xd3:
tmp_len = 1ULL << (type - 0xd0);
tmp_int = 0;
if (read_int(fd, &tmp_int, tmp_len) == -1)
return -errno;
if (callbacks->read_int)
return callbacks->read_int(tmp_int, data);
break;
case 0xd9:
case 0xda:
case 0xdb:
if ((rv = read_buff(fd, 1<<(type - 0xd9), &buff, &tmp_len)) < 0)
return rv;
if (callbacks->read_string)
return callbacks->read_string(buff, tmp_len, data);
break;
case 0xdc:
case 0xdd:
if (read_uint(fd, &tmp_len, 2<<(type - 0xdc)) == -1)
return -errno;
return read_array(fd, tmp_len, callbacks, data);
case 0xde:
case 0xdf:
if (read_uint(fd, &tmp_len, 2<<(type - 0xde)) == -1)
return -errno;
return read_map(fd, tmp_len, callbacks, data);
}
return 0;
}

80
libretrodb/rmsgpack.h Normal file
View File

@ -0,0 +1,80 @@
#ifndef __RARCHDB_MSGPACK_H__
#define __RARCHDB_MSGPACK_H__
#include <stdint.h>
struct rmsgpack_read_callbacks {
int (* read_nil)(void *);
int (* read_bool)(
int,
void *
);
int (* read_int)(
int64_t,
void *
);
int (* read_uint)(
uint64_t,
void *
);
int (* read_string)(
char *,
uint32_t,
void *
);
int (* read_bin)(
void *,
uint32_t,
void *
);
int (* read_map_start)(
uint32_t,
void *
);
int (* read_array_start)(
uint32_t,
void *
);
};
int rmsgpack_write_array_header(
int fd,
uint32_t size
);
int rmsgpack_write_map_header(
int fd,
uint32_t size
);
int rmsgpack_write_string(
int fd,
const char * s,
uint32_t len
);
int rmsgpack_write_bin(
int fd,
const void * s,
uint32_t len
);
int rmsgpack_write_nil(int fd);
int rmsgpack_write_bool(
int fd,
int value
);
int rmsgpack_write_int(
int fd,
int64_t value
);
int rmsgpack_write_uint(
int fd,
uint64_t value
);
int rmsgpack_read(
int fd,
struct rmsgpack_read_callbacks * callbacks,
void * data
);
#endif

485
libretrodb/rmsgpack_dom.c Normal file
View File

@ -0,0 +1,485 @@
#include "rmsgpack_dom.h"
#include <stdint.h>
#include <stdio.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include <stdarg.h>
#include "rmsgpack.h"
#define MAX_DEPTH 128
struct dom_reader_state {
int i;
struct rmsgpack_dom_value * stack[MAX_DEPTH];
};
static struct rmsgpack_dom_value * dom_reader_state_pop(struct dom_reader_state * s){
struct rmsgpack_dom_value * v = s->stack[s->i];
s->i--;
return v;
}
static int dom_reader_state_push(
struct dom_reader_state * s,
struct rmsgpack_dom_value * v
){
if ((s->i + 1) == MAX_DEPTH) {
return -ENOMEM;
}
s->i++;
s->stack[s->i] = v;
return 0;
}
static int dom_read_nil(void * data){
struct dom_reader_state * dom_state = (struct dom_reader_state *)data;
struct rmsgpack_dom_value * v =
(struct rmsgpack_dom_value *)dom_reader_state_pop(dom_state);
v->type = RDT_NULL;
return 0;
}
static int dom_read_bool(
int value,
void * data
){
struct dom_reader_state * dom_state = (struct dom_reader_state *)data;
struct rmsgpack_dom_value * v =
(struct rmsgpack_dom_value *)dom_reader_state_pop(dom_state);
v->type = RDT_BOOL;
v->bool_ = value;
return 0;
}
static int dom_read_int(
int64_t value,
void * data
){
struct dom_reader_state * dom_state = (struct dom_reader_state *)data;
struct rmsgpack_dom_value * v =
(struct rmsgpack_dom_value *)dom_reader_state_pop(dom_state);
v->type = RDT_INT;
v->int_ = value;
return 0;
}
static int dom_read_uint(
uint64_t value,
void * data
){
struct dom_reader_state * dom_state = (struct dom_reader_state *)data;
struct rmsgpack_dom_value * v =
(struct rmsgpack_dom_value *)dom_reader_state_pop(dom_state);
v->type = RDT_UINT;
v->uint_ = value;
return 0;
}
static int dom_read_string(
char * value,
uint32_t len,
void * data
){
struct dom_reader_state * dom_state = (struct dom_reader_state *)data;
struct rmsgpack_dom_value * v =
(struct rmsgpack_dom_value *)dom_reader_state_pop(dom_state);
v->type = RDT_STRING;
v->string.len = len;
v->string.buff = value;
return 0;
}
static int dom_read_bin(
void * value,
uint32_t len,
void * data
){
struct dom_reader_state * dom_state = (struct dom_reader_state *)data;
struct rmsgpack_dom_value * v =
(struct rmsgpack_dom_value *)dom_reader_state_pop(dom_state);
v->type = RDT_BINARY;
v->binary.len = len;
v->binary.buff = (char *)value;
return 0;
}
static int dom_read_map_start(
uint32_t len,
void * data
){
unsigned i;
struct rmsgpack_dom_pair * items = NULL;
struct dom_reader_state * dom_state = (struct dom_reader_state *)data;
struct rmsgpack_dom_value * v = dom_reader_state_pop(dom_state);
v->type = RDT_MAP;
v->map.len = len;
v->map.items = NULL;
items = (struct rmsgpack_dom_pair *)calloc(len, sizeof(struct rmsgpack_dom_pair));
if (!items)
return -ENOMEM;
v->map.items = items;
for (i = 0; i < len; i++) {
if (dom_reader_state_push(dom_state, &items[i].value) < 0)
return -ENOMEM;
if (dom_reader_state_push(dom_state, &items[i].key) < 0)
return -ENOMEM;
}
return 0;
}
static int dom_read_array_start(
uint32_t len,
void * data
){
unsigned i;
struct dom_reader_state * dom_state = (struct dom_reader_state *)data;
struct rmsgpack_dom_value * v = dom_reader_state_pop(dom_state);
struct rmsgpack_dom_value * items = NULL;
v->type = RDT_ARRAY;
v->array.len = len;
v->array.items = NULL;
items = (struct rmsgpack_dom_value *)calloc(len, sizeof(struct rmsgpack_dom_pair));
if (!items)
return -ENOMEM;
v->array.items = items;
for (i = 0; i < len; i++) {
if (dom_reader_state_push(dom_state, &items[i]) < 0)
return -ENOMEM;
}
return 0;
}
static struct rmsgpack_read_callbacks dom_reader_callbacks = {
dom_read_nil,
dom_read_bool,
dom_read_int,
dom_read_uint,
dom_read_string,
dom_read_bin,
dom_read_map_start,
dom_read_array_start
};
void rmsgpack_dom_value_free(struct rmsgpack_dom_value * v){
unsigned i;
switch (v->type) {
case RDT_STRING:
free(v->string.buff);
break;
case RDT_BINARY:
free(v->binary.buff);
break;
case RDT_MAP:
for (i = 0; i < v->map.len; i++) {
rmsgpack_dom_value_free(&v->map.items[i].key);
rmsgpack_dom_value_free(&v->map.items[i].value);
}
free(v->map.items);
break;
case RDT_ARRAY:
for (i = 0; i < v->array.len; i++)
rmsgpack_dom_value_free(&v->array.items[i]);
free(v->array.items);
break;
case RDT_NULL:
case RDT_INT:
case RDT_BOOL:
case RDT_UINT:
// Do nothing
break;
}
}
struct rmsgpack_dom_value * rmsgpack_dom_value_map_value(
const struct rmsgpack_dom_value * map,
const struct rmsgpack_dom_value * key
){
unsigned i;
if (map->type != RDT_MAP)
return NULL;
for (i = 0; i < map->map.len; i++) {
if (rmsgpack_dom_value_cmp(key, &map->map.items[i].key) == 0)
return &map->map.items[i].value;
}
return NULL;
}
int rmsgpack_dom_value_cmp(
const struct rmsgpack_dom_value * a,
const struct rmsgpack_dom_value * b
){
int rv;
unsigned i;
if (a == b)
return 1;
if (a->type != b->type)
return 1;
switch (a->type) {
case RDT_NULL:
return 0;
case RDT_BOOL:
return a->bool_ == b->bool_ ? 0 : 1;
case RDT_INT:
return a->int_ == b->int_ ? 0 : 1;
case RDT_UINT:
return a->uint_ == b->uint_ ? 0 : 1;
case RDT_STRING:
if (a->string.len != b->string.len)
return 1;
return strncmp(a->string.buff, b->string.buff, a->string.len);
case RDT_BINARY:
if (a->binary.len != b->binary.len)
return 1;
return memcmp(a->binary.buff, b->binary.buff, a->binary.len);
case RDT_MAP:
if (a->map.len != b->map.len)
return 1;
for (i = 0; i < a->map.len; i++) {
if ((rv = rmsgpack_dom_value_cmp(&a->map.items[i].key, &b->map.items[i].key)) != 0)
return rv;
if ((rv = rmsgpack_dom_value_cmp(&a->map.items[i].value, &b->map.items[i].value)) != 0)
return rv;
}
break;
case RDT_ARRAY:
if (a->array.len != b->array.len)
return 1;
for (i = 0; i < a->array.len; i++) {
if ((rv = rmsgpack_dom_value_cmp(&a->array.items[i], &b->array.items[i])) != 0)
return rv;
}
break;
}
return 1;
}
void rmsgpack_dom_value_print(struct rmsgpack_dom_value * obj){
unsigned i;
switch (obj->type) {
case RDT_NULL:
printf("nil");
break;
case RDT_BOOL:
if (obj->bool_)
printf("true");
else
printf("false");
break;
case RDT_INT:
#ifdef _WIN32
printf("%I64d", (signed long long)obj->int_);
#else
printf("%lld", (signed long long)obj->int_);
#endif
break;
case RDT_UINT:
#ifdef _WIN32
printf("%I64u", (unsigned long long)obj->uint_);
#else
printf("%llu", (unsigned long long)obj->uint_);
#endif
break;
case RDT_STRING:
printf("\"%s\"", obj->string.buff);
break;
case RDT_BINARY:
printf("\"");
for (i = 0; i < obj->binary.len; i++)
printf("%02X", (unsigned char) obj->binary.buff[i]);
printf("\"");
break;
case RDT_MAP:
printf("{");
for (i = 0; i < obj->map.len; i++) {
rmsgpack_dom_value_print(&obj->map.items[i].key);
printf(": ");
rmsgpack_dom_value_print(&obj->map.items[i].value);
if (i < (obj->map.len - 1))
printf(", ");
}
printf("}");
break;
case RDT_ARRAY:
printf("[");
for (i = 0; i < obj->array.len; i++) {
rmsgpack_dom_value_print(&obj->array.items[i]);
if (i < (obj->array.len - 1))
printf(", ");
}
printf("]");
}
}
int rmsgpack_dom_write(
int fd,
const struct rmsgpack_dom_value * obj
){
unsigned i;
int rv = 0;
int written = 0;
switch (obj->type) {
case RDT_NULL:
return rmsgpack_write_nil(fd);
case RDT_BOOL:
return rmsgpack_write_bool(fd, obj->bool_);
case RDT_INT:
return rmsgpack_write_int(fd, obj->int_);
case RDT_UINT:
return rmsgpack_write_uint(fd, obj->uint_);
case RDT_STRING:
return rmsgpack_write_string(fd, obj->string.buff, obj->string.len);
case RDT_BINARY:
return rmsgpack_write_bin(fd, obj->binary.buff, obj->binary.len);
case RDT_MAP:
if ((rv = rmsgpack_write_map_header(fd, obj->map.len)) < 0)
return rv;
written += rv;
for (i = 0; i < obj->map.len; i++) {
if ((rv = rmsgpack_dom_write(fd, &obj->map.items[i].key)) < 0)
return rv;
written += rv;
if ((rv = rmsgpack_dom_write(fd, &obj->map.items[i].value)) < 0)
return rv;
written += rv;
}
break;
case RDT_ARRAY:
if ((rv = rmsgpack_write_array_header(fd, obj->array.len)) < 0)
return rv;
written += rv;
for (i = 0; i < obj->array.len; i++) {
if ((rv = rmsgpack_dom_write(fd, &obj->array.items[i])) < 0)
return rv;
written += rv;
}
}
return written;
}
int rmsgpack_dom_read(
int fd,
struct rmsgpack_dom_value * out
){
int rv = 0;
struct dom_reader_state s;
s.i = 0;
s.stack[0] = out;
rv = rmsgpack_read(fd, &dom_reader_callbacks, &s);
if (rv < 0)
rmsgpack_dom_value_free(out);
return rv;
}
int rmsgpack_dom_read_into(
int fd,
...
){
va_list ap;
struct rmsgpack_dom_value map;
int rv;
const char * key_name;
struct rmsgpack_dom_value key;
struct rmsgpack_dom_value * value;
int64_t * int_value;
uint64_t * uint_value;
int * bool_value;
char * buff_value;
uint64_t min_len;
int value_type = 0;
va_start(ap, fd);
rv = rmsgpack_dom_read(fd, &map);
(void)value_type;
if (rv < 0)
return rv;
if (map.type != RDT_MAP) {
rv = -EINVAL;
goto clean;
}
while (1) {
key_name = va_arg(ap, const char *);
if (key_name == NULL) {
rv = 0;
goto clean;
}
key.type = RDT_STRING;
key.string.len = strlen(key_name);
key.string.buff = (char *) key_name;
value = rmsgpack_dom_value_map_value(&map, &key);
switch (value->type) {
case RDT_INT:
int_value = va_arg(ap, int64_t *);
*int_value = value->int_;
break;
case RDT_BOOL:
bool_value = va_arg(ap, int *);
*bool_value = value->bool_;
break;
case RDT_UINT:
uint_value = va_arg(ap, uint64_t *);
*uint_value = value->uint_;
break;
case RDT_BINARY:
buff_value = va_arg(ap, char *);
uint_value = va_arg(ap, uint64_t *);
*uint_value = value->binary.len;
min_len = value->binary.len > *uint_value ? *uint_value : value->binary.len;
memcpy(
buff_value,
value->binary.buff,
min_len
);
break;
case RDT_STRING:
buff_value = va_arg(ap, char *);
uint_value = va_arg(ap, uint64_t *);
min_len = value->string.len + 1 > *uint_value ? *uint_value : value->string.len + 1;
*uint_value = min_len;
memcpy(
buff_value,
value->string.buff,
min_len
);
break;
default:
rv = -1;
goto clean;
}
}
clean:
va_end(ap);
rmsgpack_dom_value_free(&map);
return 0;
}

71
libretrodb/rmsgpack_dom.h Normal file
View File

@ -0,0 +1,71 @@
#ifndef __RARCHDB_MSGPACK_DOM_H__
#define __RARCHDB_MSGPACK_DOM_H__
#include <stdint.h>
enum rmsgpack_dom_type {
RDT_NULL = 0,
RDT_BOOL,
RDT_UINT,
RDT_INT,
RDT_STRING,
RDT_BINARY,
RDT_MAP,
RDT_ARRAY
};
struct rmsgpack_dom_value {
enum rmsgpack_dom_type type;
union {
uint64_t uint_;
int64_t int_;
struct {
uint32_t len;
char * buff;
} string;
struct {
uint32_t len;
char * buff;
} binary;
int bool_;
struct {
uint32_t len;
struct rmsgpack_dom_pair * items;
} map;
struct {
uint32_t len;
struct rmsgpack_dom_value * items;
} array;
};
};
struct rmsgpack_dom_pair {
struct rmsgpack_dom_value key;
struct rmsgpack_dom_value value;
};
void rmsgpack_dom_value_print(struct rmsgpack_dom_value * obj);
void rmsgpack_dom_value_free(struct rmsgpack_dom_value * v);
int rmsgpack_dom_value_cmp(
const struct rmsgpack_dom_value * a,
const struct rmsgpack_dom_value * b
);
struct rmsgpack_dom_value * rmsgpack_dom_value_map_value(
const struct rmsgpack_dom_value * map,
const struct rmsgpack_dom_value * key
);
int rmsgpack_dom_read(
int fd,
struct rmsgpack_dom_value * out
);
int rmsgpack_dom_write(
int fd,
const struct rmsgpack_dom_value * obj
);
int rmsgpack_dom_read_into(
int fd,
...
);
#endif

197
libretrodb/rmsgpack_test.c Normal file
View File

@ -0,0 +1,197 @@
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdlib.h>
#include <unistd.h>
#include "rmsgpack.h"
struct stub_state {
int i;
uint64_t stack[256];
};
static void stub_state_push_map(
struct stub_state * s,
uint32_t size
){
s->i++;
s->stack[s->i] = 1;
s->i++;
s->stack[s->i] = size * 2;
printf("{");
}
static void stub_state_push_array(
struct stub_state * s,
uint32_t size
){
s->i++;
s->stack[s->i] = 2;
s->i++;
s->stack[s->i] = size;
printf("[");
}
static void stub_state_pre_print(struct stub_state * s)
{}
static void stub_state_post_print(struct stub_state * s){
switch (s->stack[s->i - 1]) {
case 1:
if (s->stack[s->i] % 2 == 0) {
printf(": ");
s->stack[s->i]--;
} else if (s->stack[s->i] == 1) {
printf("}");
s->i -= 2;
stub_state_post_print(s);
} else {
printf(", ");
s->stack[s->i]--;
}
break;
case 2:
if (s->stack[s->i] == 1) {
printf("]");
s->i -= 2;
stub_state_post_print(s);
} else {
printf(", ");
s->stack[s->i]--;
}
break;
}
}
static int stub_read_map_start(
uint32_t size,
void * data
){
stub_state_push_map(data, size);
return 0;
}
static int stub_read_array_start(
uint32_t size,
void * data
){
stub_state_push_array(data, size);
return 0;
}
static int stub_read_string(
char * s,
uint32_t len,
void * data
){
stub_state_pre_print(data);
printf("'%s'", s);
stub_state_post_print(data);
free(s);
return 0;
}
static int stub_read_bin(
void * s,
uint32_t len,
void * data
){
stub_state_pre_print(data);
printf("b'%s'", s);
stub_state_post_print(data);
free(s);
return 0;
}
static int stub_read_uint(
uint64_t value,
void * data
){
stub_state_pre_print(data);
printf("%lu", value);
stub_state_post_print(data);
return 0;
}
static int stub_read_nil(void * data){
stub_state_pre_print(data);
printf("nil");
stub_state_post_print(data);
return 0;
}
static int stub_read_int(
int64_t value,
void * data
){
stub_state_pre_print(data);
printf("%ld", value);
stub_state_post_print(data);
return 0;
}
static int stub_read_bool(
int value,
void * data
){
stub_state_pre_print(data);
if (value)
printf("true");
else
printf("false");
stub_state_post_print(data);
return 0;
}
static struct rmsgpack_read_callbacks stub_callbacks = {
stub_read_nil,
stub_read_bool,
stub_read_int,
stub_read_uint,
stub_read_string,
stub_read_bin,
stub_read_map_start,
stub_read_array_start
};
int main(void){
int fd;
/*
int fd = open("test.msgpack", O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
int rv = 0;
if (fd == -1)
{
printf("Could not open file: %s", strerror(errno));
return errno;
}
rmsgpack_write_map_header(fd, 2);
rmsgpack_write_string(fd, "compact", strlen("compact"));
rmsgpack_write_bool(fd, 1);
rmsgpack_write_string(fd, "schema", strlen("schema"));
rmsgpack_write_array_header(fd, 10);
rmsgpack_write_string(fd, "schema", strlen("schema"));
rmsgpack_write_uint(fd, 1<<17);
rmsgpack_write_int(fd, (1<<17) + 1);
rmsgpack_write_int(fd, 4);
rmsgpack_write_int(fd, -3);
rmsgpack_write_int(fd, -22);
rmsgpack_write_int(fd, -35);
rmsgpack_write_int(fd, -421421412);
rmsgpack_write_int(fd, 4214);
rmsgpack_write_int(fd, -4214);
rmsgpack_write_uint(fd, 1<<17);
close(fd);
*/
struct stub_state state;
state.i = 0;
state.stack[0] = 0;
fd = open("test.msgpack", O_RDONLY);
rmsgpack_read(fd, &stub_callbacks, &state);
printf("\n");
close(fd);
return 0;
}

290
libretrodb/testlib.c Normal file
View File

@ -0,0 +1,290 @@
#define LUA_LIB
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include "lua.h"
#include "lauxlib.h"
#include "rarchdb.h"
#include "lua_common.h"
static int create_db (lua_State * L);
static int db_new (lua_State * L);
static int db_close (lua_State * L);
static int db_cursor_open (lua_State * L);
static int db_query (lua_State * L);
static int cursor_close (lua_State * L);
static int cursor_read (lua_State * L);
static int cursor_iter (lua_State * L);
static const luaL_Reg testlib[] = {
{"create_db", create_db},
{"RarchDB", db_new},
{NULL, NULL}
};
static const struct luaL_Reg cursor_mt [] = {
{"__gc", cursor_close},
{"read", cursor_read},
{"iter", cursor_iter},
{NULL, NULL}
};
static const struct luaL_Reg rarchdb_mt [] = {
{"__gc", db_close},
{"list_all", db_cursor_open},
{"query", db_query},
{NULL, NULL}
};
LUALIB_API int luaopen_testlib (lua_State * L) {
luaL_newmetatable(L, "RarchDB.DB");
lua_pushstring(L, "__index");
lua_pushvalue(L, -2);
lua_settable(L, -3);
luaL_openlib(L, NULL, rarchdb_mt, 0);
luaL_newmetatable(L, "RarchDB.Cursor");
lua_pushstring(L, "__index");
lua_pushvalue(L, -2);
lua_settable(L, -3);
luaL_openlib(L, NULL, cursor_mt, 0);
luaL_register(L, "testlib", testlib);
return 1;
}
static struct rarchdb_cursor * checkcursor(lua_State * L) {
void * ud = luaL_checkudata(L, 1, "RarchDB.Cursor");
luaL_argcheck(L, ud != NULL, 1, "`RarchDB.Cursor' expected");
return ud;
}
static struct rarchdb * checkdb(lua_State * L) {
void * ud = luaL_checkudata(L, 1, "RarchDB.DB");
luaL_argcheck(L, ud != NULL, 1, "`RarchDB.DB' expected");
return ud;
}
static int value_provider(
void * ctx,
struct rmsgpack_dom_value * out
) {
int rv;
lua_State * L = ctx;
lua_getfield(L, LUA_REGISTRYINDEX, "testlib_get_value");
if (lua_pcall(L, 0, 1, 0) != 0) {
printf(
"error running function `get_value': %s\n",
lua_tostring(L, -1)
);
}
if (lua_isnil(L, -1)) {
rv = 1;
} else if (lua_istable(L, -1)) {
rv = rarchdb_lua_to_rmsgpack_value(L, -1, out);
} else {
printf("function `get_value' must return a table or nil\n");
}
lua_pop(L, 1);
return rv;
}
static int create_db (lua_State * L) {
int dst;
const char * db_file;
int rv;
db_file = luaL_checkstring(L, -2);
if (!lua_isfunction(L, -1)) {
lua_pushstring(L, "second argument must be a function");
lua_error(L);
}
lua_setfield(L, LUA_REGISTRYINDEX, "testlib_get_value");
dst = open(db_file, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
if (dst == -1) {
lua_pushstring(L, "Could not open destination file");
lua_error(L);
}
rv = rarchdb_create(dst, &value_provider, L);
close(dst);
return 0;
}
static int db_new (lua_State * L) {
struct rarchdb * db = NULL;
const char * db_file = NULL;
int rv;
db_file = luaL_checkstring(L, -1);
db = lua_newuserdata(L, sizeof(struct rarchdb));
if ((rv = rarchdb_open(db_file, db)) == 0) {
luaL_getmetatable(L, "RarchDB.DB");
lua_setmetatable(L, -2);
lua_pushnil(L);
} else {
lua_pop(L, 1);
lua_pushnil(L);
lua_pushstring(L, strerror(-rv));
}
return 2;
}
static int db_close (lua_State * L) {
struct rarchdb * db = checkdb(L);
rarchdb_close(db);
return 0;
}
static int db_query (lua_State * L) {
int rv;
struct rarchdb_cursor * cursor = NULL;
struct rarchdb * db = checkdb(L);
const char * query = luaL_checkstring(L, -1);
const char * error = NULL;
rarchdb_query * q = rarchdb_query_compile(
db,
query,
strlen(query),
&error
);
if (error) {
lua_pushnil(L);
lua_pushstring(L, error);
} else {
cursor = lua_newuserdata(L, sizeof(struct rarchdb));
if ((rv = rarchdb_cursor_open(db, cursor, q)) == 0) {
luaL_getmetatable(L, "RarchDB.Cursor");
lua_setmetatable(L, -2);
lua_pushnil(L);
} else {
lua_pop(L, 1);
lua_pushnil(L);
lua_pushstring(L, strerror(-rv));
}
rarchdb_query_free(q);
}
return 2;
}
static int db_cursor_open (lua_State * L) {
int rv;
struct rarchdb_cursor * cursor = NULL;
struct rarchdb * db = checkdb(L);
cursor = lua_newuserdata(L, sizeof(struct rarchdb));
if ((rv = rarchdb_cursor_open(db, cursor, NULL)) == 0) {
luaL_getmetatable(L, "RarchDB.Cursor");
lua_setmetatable(L, -2);
lua_pushnil(L);
} else {
lua_pop(L, 1);
lua_pushnil(L);
lua_pushstring(L, strerror(-rv));
}
return 2;
}
static int cursor_close (lua_State * L) {
struct rarchdb_cursor * cursor = checkcursor(L);
rarchdb_cursor_close(cursor);
return 0;
}
static void push_rmsgpack_value(
lua_State * L,
struct rmsgpack_dom_value * value
) {
uint32_t i;
switch (value->type) {
case RDT_INT:
lua_pushnumber(
L,
value->int_
);
break;
case RDT_UINT:
lua_pushnumber(
L,
value->uint_
);
break;
case RDT_BINARY:
lua_pushlstring(
L,
value->binary.buff,
value->binary.len
);
break;
case RDT_BOOL:
lua_pushboolean(
L,
value->bool_
);
break;
case RDT_NULL:
lua_pushnil(L);
break;
case RDT_STRING:
lua_pushlstring(
L,
value->string.buff,
value->binary.len
);
break;
case RDT_MAP:
lua_createtable(L, 0, value->map.len);
for (i = 0; i < value->map.len; i++) {
push_rmsgpack_value(
L,
&value->map.items[i].key
);
push_rmsgpack_value(
L,
&value->map.items[i].value
);
lua_settable(L, -3);
}
break;
case RDT_ARRAY:
lua_createtable(L, value->array.len, 0);
for (i = 0; i < value->array.len; i++) {
lua_pushnumber(
L,
i + 1
);
push_rmsgpack_value(
L,
&value->array.items[i]
);
lua_settable(L, -3);
}
break;
}
}
static int cursor_read (lua_State * L) {
struct rarchdb_cursor * cursor = checkcursor(L);
struct rmsgpack_dom_value value;
if (rarchdb_cursor_read_item(cursor, &value) == 0) {
push_rmsgpack_value(L, &value);
} else {
lua_pushnil(L);
}
return 1;
}
static int cursor_iter (lua_State * L) {
struct rarchdb_cursor * cursor = checkcursor(L);
luaL_getmetafield(L, -1, "read");
lua_pushvalue(L, -2);
return 2;
}

81
libretrodb/tests.lua Normal file
View File

@ -0,0 +1,81 @@
local testlib = require 'testlib'
local DB_FILE = "/tmp/tmp.rdb"
local function create_db(data)
local i = 1;
testlib.create_db(DB_FILE, function()
if i > #data then
return
end
res = data[i]
i = i + 1
return res
end)
local db, err = testlib.RarchDB(DB_FILE)
if err then
error(err)
end
return db
end
local function assert_equals(a, b)
if type(a) ~= type(b) then
return false
end
if type(a) == "table" then
for k, v in pairs(a) do
if not assert_equals(v, b[k]) then
return
end
end
return
else
return assert(a == b, tostring(a) .. " != " .. tostring(b))
end
end
function query_test(data, result, query)
return function()
local db = create_db(data)
local c, err = db:query(query)
if err then
error(err)
end
local i = 0
for item in c:iter() do
i = i + 1
assert_equals(item, data[i])
end
assert(i == #result, "expected " .. tostring(#result) .. " results got " .. tostring(i))
end
end
tests = {
test_list_all = function()
data = {
{field=true},
{field=false},
}
local db = create_db(data)
local c = db:list_all()
local i = 1
for item in c:iter() do
assert_equals(item, data[i])
i = i + 1
end
end,
test_boolean_field = query_test({{a=true},{a=false}}, {{a=true}}, "{'a':true}"),
test_number_field = query_test({{a=3}, {a=4}}, {{a=3}}, "{'a':3}"),
test_string_field = query_test({{a="test"}, {a=4}}, {{a="test"}}, "{'a':'test'}"),
test_or_operator = query_test({{a="test"}, {a=4}, {a=5}}, {{a="test"}, {a=4}}, "{'a':or('test', 4)}"),
test_or_between = query_test({{a="test"}, {a=4}, {a=5}, {}}, {{a="test"}, {a=4}, {a=5}}, "{'a':or('test', between(2, 7))}"),
}
for name, cb in pairs(tests) do
local ok, err = pcall(cb)
if ok then
print("V", name)
else
print("X", name, ":", err)
end
end

3
libretrodb/uncrustify Executable file
View File

@ -0,0 +1,3 @@
#!/bin/sh
uncrustify -c uncrustify.cfg --replace $*

1578
libretrodb/uncrustify.cfg Normal file

File diff suppressed because it is too large Load Diff

View File

@ -147,20 +147,6 @@ static void common_load_content(bool persist)
driver.menu->msg_force = true;
}
static int action_ok_push_content_list(const char *path,
const char *label, unsigned type, size_t idx)
{
if (!driver.menu)
return -1;
menu_list_push_stack_refresh(
driver.menu->menu_list,
g_settings.menu_content_directory,
label,
MENU_FILE_DIRECTORY,
driver.menu->selection_ptr);
return 0;
}
static int action_ok_load_state(const char *path,
const char *label, unsigned type, size_t idx)
@ -828,6 +814,21 @@ static int action_ok_push_default(const char *path,
return 0;
}
static int action_ok_push_content_list(const char *path,
const char *label, unsigned type, size_t idx)
{
if (!driver.menu)
return -1;
menu_list_push_stack_refresh(
driver.menu->menu_list,
g_settings.menu_content_directory,
label,
MENU_FILE_DIRECTORY,
driver.menu->selection_ptr);
return 0;
}
static int action_ok_disk_cycle_tray_status(const char *path,
const char *label, unsigned type, size_t idx)
{

View File

@ -1,121 +0,0 @@
#include <stdlib.h>
#include <errno.h>
#include <stdio.h>
#include "bintree.h"
#if 0
static int NIL_VALUE = 1;
#endif
static void* NIL_NODE = &NIL_NODE;
static struct bintree_node *new_nil_node(struct bintree_node *parent);
void bintree_new(struct bintree *t, bintree_cmp_func cmp, void *ctx)
{
t->root = new_nil_node(NULL);
t->cmp = cmp;
t->ctx = ctx;
}
static struct bintree_node *new_nil_node(struct bintree_node *parent)
{
struct bintree_node *node = (struct bintree_node*)calloc(1, sizeof(struct bintree_node));
if (!node)
return NULL;
node->value = NIL_NODE;
node->parent = parent;
node->left = NULL;
node->right = NULL;
return node;
}
static inline int is_nil(const struct bintree_node *node)
{
return node == NULL || node->value == NIL_NODE;
}
static int insert(struct bintree *t, struct bintree_node *root, void *value)
{
int cmp_res = 0;
if (is_nil(root))
{
root->left = new_nil_node(root);
root->right = new_nil_node(root);
if (!root->left || !root->right)
{
if (root->left)
{
free(root->left);
root->left = NULL;
}
if (root->right)
{
free(root->right);
root->right = NULL;
}
return -ENOMEM;
}
root->value = value;
}
else
{
cmp_res = t->cmp(root->value, value, t->ctx);
if (cmp_res > 0)
return insert(t, root->left, value);
else if (cmp_res < 0)
return insert(t, root->right, value);
else
return -EINVAL;
}
return 0;
}
int bintree_insert(struct bintree *t, void *value)
{
return insert(t, t->root, value);
}
static int _bintree_iterate(struct bintree_node *n, bintree_iter_cb cb, void *ctx)
{
int rv;
if (is_nil(n))
return 0;
else
{
if((rv = _bintree_iterate(n->left, cb, ctx)) != 0)
return rv;
if ((rv = cb(n->value, ctx)) != 0)
return rv;
if((rv = _bintree_iterate(n->right, cb, ctx)) != 0)
return rv;
}
return 0;
}
int bintree_iterate(const struct bintree *t, bintree_iter_cb cb, void *ctx)
{
return _bintree_iterate(t->root, cb, ctx);
}
static void bintree_free_node(struct bintree_node *n)
{
if (n == NULL)
return;
if (n->value == NIL_NODE)
{
free(n);
return;
}
n->value = NULL;
bintree_free_node(n->left);
bintree_free_node(n->right);
free(n);
}
void bintree_free(struct bintree *t)
{
bintree_free_node(t->root);
}

View File

@ -1,27 +0,0 @@
#ifndef __RARCHDB_BINTREE_H__
#define __RARCHDB_BINTREE_H__
typedef int(*bintree_cmp_func)(const void *a, const void *b, void* ctx);
typedef int(*bintree_iter_cb)(void *value, void* ctx);
struct bintree_node {
void* value;
struct bintree_node *parent;
struct bintree_node *left;
struct bintree_node *right;
};
struct bintree {
struct bintree_node *root;
bintree_cmp_func cmp;
void *ctx;
};
void bintree_new(struct bintree *t, bintree_cmp_func cmp, void *ctx);
int bintree_insert(struct bintree *t, void *value);
int bintree_iterate(const struct bintree *t, bintree_iter_cb cb, void* ctx);
void bintree_free(struct bintree *t);
#endif

View File

@ -1,2 +0,0 @@
#!/bin/sh
./lua_converter "$2" dat_converter.lua "$1"

View File

@ -1,291 +0,0 @@
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include "rarchdb.h"
#include "db_parser.h"
#define MAX_TOKEN 256
static char *strndup_(const char *s, size_t n)
{
char* buff = calloc(n, sizeof(char));
if (!buff)
return 0;
strncpy(buff, s, n);
return buff;
}
static struct rmsgpack_dom_value *get_map_value(const struct rmsgpack_dom_value *m, char* key)
{
struct rmsgpack_dom_value v;
v.type = RDT_STRING;
v.string.len = strlen(key);
v.string.buff = key;
return rmsgpack_dom_value_map_value(m, &v);
}
static int load_string(int fd, struct rmsgpack_dom_value *out)
{
char tok[MAX_TOKEN];
ssize_t tok_size;
if ((tok_size = get_token(fd, tok, MAX_TOKEN)) < 0)
return tok_size;
out->type = RDT_STRING;
out->string.len = tok_size;
out->string.buff = strndup_(tok, tok_size);
return 0;
}
static int load_uint(int fd, struct rmsgpack_dom_value *out)
{
char tok[MAX_TOKEN], *c;
ssize_t tok_size;
uint64_t value = 0;
if ((tok_size = get_token(fd, tok, MAX_TOKEN)) < 0)
return tok_size;
for (c = tok; c < tok + tok_size; c++)
{
value *= 10;
value += *c - '0';
}
out->type = RDT_UINT;
out->uint_ = value;
return 0;
}
static int load_bin(int fd, struct rmsgpack_dom_value *out)
{
char tok[MAX_TOKEN];
ssize_t tok_size;
uint8_t h;
uint8_t l;
int i;
if ((tok_size = get_token(fd, tok, MAX_TOKEN)) < 0)
return tok_size;
out->type = RDT_BINARY;
out->binary.len = tok_size / 2;
for (i = 0; i < tok_size; i += 2)
{
if (tok[i] <= '9')
h = tok[i] - '0';
else
h = (tok[i] - 'A') + 10;
if (tok[i+1] <= '9')
l = tok[i+1] - '0';
else
l = (tok[i+1] - 'A') + 10;
tok[i/2] = h * 16 + l;
}
out->binary.buff = malloc(out->binary.len);
memcpy(out->binary.buff, tok, out->binary.len);
return 0;
}
static int dat_value_provider(void *ctx, struct rmsgpack_dom_value *out)
{
int rv, i;
static const int field_count = 22;
int fd = *((int*)ctx);
char* key;
out->type = RDT_MAP;
out->map.len = field_count;
out->map.items = calloc(field_count, sizeof(struct rmsgpack_dom_pair));
if (find_token(fd, "game") < 0)
return 1;
for (i = 0; i < field_count; i++)
{
if ((rv = load_string(fd, &out->map.items[i].key)) < 0)
goto failed;
key = out->map.items[i].key.string.buff;
if (strncmp(key, "name", sizeof("name")) == 0)
{
if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "description", sizeof("description")) == 0)
{
if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "users", sizeof("users")) == 0)
{
if ((rv = load_uint(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "releasemonth", sizeof("releasemonth")) == 0)
{
if ((rv = load_uint(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "releaseyear", sizeof("releaseyear")) == 0)
{
if ((rv = load_uint(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "rumble", sizeof("rumble")) == 0)
{
if ((rv = load_uint(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "analog", sizeof("analog")) == 0)
{
if ((rv = load_uint(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "serial", sizeof("serial")) == 0)
{
if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "esrb_rating", sizeof("esrb_rating")) == 0)
{
if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "elspa_rating", sizeof("elspa_rating")) == 0)
{
if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "pegi_rating", sizeof("pegi_rating")) == 0)
{
if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "cero_rating", sizeof("cero_rating")) == 0)
{
if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "developers", sizeof("developers")) == 0)
{
if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "publisher", sizeof("publisher")) == 0)
{
if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "origin", sizeof("origin")) == 0)
{
if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "rom", sizeof("rom")) == 0)
{
if (find_token(fd, "name") < 0)
goto failed;
if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "size", sizeof("size")) == 0)
{
if ((rv = load_uint(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "sha1", sizeof("sha1")) == 0)
{
if ((rv = load_bin(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "crc", sizeof("crc")) == 0)
{
if ((rv = load_bin(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "md5", sizeof("md5")) == 0)
{
if ((rv = load_bin(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, "serial", sizeof("serial")) == 0)
{
if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
goto failed;
}
else if (strncmp(key, ")", sizeof(")")) == 0)
{
rmsgpack_dom_value_free(&out->map.items[i].key);
out->map.len = i;
printf("Couldn't find all fields for item\n");
break;
}
else
{
rmsgpack_dom_value_free(&out->map.items[i].key);
i--;
}
}
printf("Inserting '%s' (%02X%02X%02X%02X)...\n",
get_map_value(out, "name")->string.buff,
(unsigned char)get_map_value(out, "crc")->binary.buff[0],
(unsigned char)get_map_value(out, "crc")->binary.buff[1],
(unsigned char)get_map_value(out, "crc")->binary.buff[2],
(unsigned char)get_map_value(out, "crc")->binary.buff[3]
);
return 0;
failed:
rmsgpack_dom_value_free(out);
out->type = RDT_NULL;
return rv;
}
int main(int argc, char **argv)
{
int rv = 0;
int src = -1;
int dst = -1;
if (argc != 3)
printf("Usage: %s <dat file> <output file>\n", argv[0]);
src = open(argv[1], O_RDONLY);
if (src == -1)
{
printf("Could not open source file '%s': %s\n", argv[1], strerror(errno));
rv = errno;
goto clean;
}
dst = open(argv[2], O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
if (dst == -1)
{
printf("Could not open destination file '%s': %s\n", argv[1], strerror(errno));
rv = errno;
goto clean;
}
rv = rarchdb_create(dst, &dat_value_provider, &src);
clean:
if (src != -1)
close(src);
if (dst != -1)
close(dst);
return rv;
}

View File

@ -1,80 +0,0 @@
#include "db_parser.h"
#include <errno.h>
#include <string.h>
#include <stdlib.h>
ssize_t get_token(int fd, char *token, size_t max_len)
{
char *c = token;
int rv;
ssize_t len = 0;
int in_string = 0;
while (1)
{
rv = read(fd, c, 1);
if (rv == 0)
return 0;
else if (rv < 1)
{
switch (errno)
{
case EINTR:
case EAGAIN:
continue;
default:
return -errno;
}
}
switch (*c)
{
case ' ':
case '\t':
case '\r':
case '\n':
if (c == token)
continue;
if (!in_string)
{
*c = '\0';
return len;
}
break;
case '\"':
if (c == token)
{
in_string = 1;
continue;
}
*c = '\0';
return len;
}
len++;
c++;
if (len == (ssize_t)max_len)
{
*c = '\0';
return len;
}
}
}
int find_token(int fd, const char *token)
{
int tmp_len = strlen(token);
char *tmp_token = (char*)calloc(tmp_len, 1);
if (!tmp_token)
return -1;
while (strncmp(tmp_token, token, tmp_len) != 0)
{
if (get_token(fd, tmp_token, tmp_len) <= 0)
return -1;
}
return 0;
}

View File

@ -1,15 +0,0 @@
#ifndef __RARCHDB_PARSER_H
#define __RARCHDB_PARSER_H
#ifdef _WIN32
#include <io.h>
#else
#include <unistd.h>
#endif
#define MAX_TOKEN_LEN 255
ssize_t get_token(int fd, char* token, size_t max_len);
int find_token(int fd, const char* token);
#endif

View File

@ -1,184 +0,0 @@
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include <stdlib.h>
#include <lua.h>
#include <lualib.h>
#include <lauxlib.h>
#include "rarchdb.h"
int master_key = 1;
const char *LUA_COMMON = " \
function binary(s) if s ~= nil then return {binary = s} else return nil end end \
function uint(s) if s ~= nil then return {uint = s} else return nil end end \
";
static int call_init(lua_State *L, int argc, const char** argv) {
int rv = -1;
int i;
lua_getglobal(L, "init");
for (i = 0; i < argc; i++) {
lua_pushstring(L, argv[i]);
}
if (lua_pcall(L, argc, 0, 0) != 0) {
printf(
"error running function `init': %s\n",
lua_tostring(L, -1)
);
}
return rv;
}
static int value_provider(void *ctx, struct rmsgpack_dom_value *out) {
lua_State *L = ctx;
int rv = -1;
int i;
const char *tmp_string = NULL;
char *tmp_buff = NULL;
struct rmsgpack_dom_value *tmp_value;
const int key_idx = -2;
const int value_idx = -1;
const int MAX_FIELDS = 100;
size_t tmp_len;
lua_Number tmp_num;
lua_getglobal(L, "get_value");
if (lua_pcall(L, 0, 1, 0) != 0) {
printf(
"error running function `get_value': %s\n",
lua_tostring(L, -1)
);
}
if (lua_isnil(L, -1)) {
rv = 1;
} else if (lua_istable(L, -1)) {
out->type = RDT_MAP;
out->map.len = 0;
out->map.items = calloc(MAX_FIELDS, sizeof(struct rmsgpack_dom_pair));
lua_pushnil(L);
while (lua_next(L, -2) != 0) {
if (out->map.len > MAX_FIELDS) {
printf("skipping due to too many keys\n");
} else if (!lua_isstring(L, key_idx)) {
printf("skipping non string key\n");
} else if (lua_isnil(L, value_idx)) {
// Skipping nil value fields to save disk space
} else {
i = out->map.len;
tmp_buff = strdup(lua_tostring(L, key_idx));
out->map.items[i].key.type = RDT_STRING;
out->map.items[i].key.string.len = strlen(tmp_buff);
out->map.items[i].key.string.buff = tmp_buff;
tmp_value = &out->map.items[i].value;
switch(lua_type(L, value_idx)) {
case LUA_TNUMBER:
tmp_num = lua_tonumber(L, value_idx);
tmp_value->type = RDT_INT;
tmp_value->int_ = tmp_num;
break;
case LUA_TBOOLEAN:
tmp_value->type = RDT_BOOL;
tmp_value->bool_ = lua_toboolean(L, value_idx);
break;
case LUA_TSTRING:
tmp_buff = strdup(lua_tostring(L, value_idx));
tmp_value->type = RDT_STRING;
tmp_value->string.len = strlen(tmp_buff);
tmp_value->string.buff = tmp_buff;
break;
case LUA_TTABLE:
lua_getfield(L, value_idx, "binary");
if (!lua_isstring(L, -1)) {
lua_pop(L, 1);
lua_getfield(L, value_idx, "uint");
if (!lua_isnumber(L, -1)) {
lua_pop(L, 1);
goto set_nil;
} else {
tmp_num = lua_tonumber(L, -1);
tmp_value->type = RDT_UINT;
tmp_value->uint_ = tmp_num;
lua_pop(L, 1);
}
} else {
tmp_string = lua_tolstring(L, -1, &tmp_len);
tmp_buff = malloc(tmp_len);
memcpy(tmp_buff, tmp_string, tmp_len);
tmp_value->type = RDT_BINARY;
tmp_value->binary.len = tmp_len;
tmp_value->binary.buff = tmp_buff;
lua_pop(L, 1);
}
break;
default:
set_nil:
tmp_value->type = RDT_NULL;
}
out->map.len++;
}
lua_pop(L, 1);
}
rv = 0;
} else {
printf("function `get_value' must return a table or nil\n");
}
lua_pop(L, 1);
return rv;
}
int main(int argc, char **argv)
{
const char* db_file;
const char* lua_file;
int dst = -1;
int rv = 0;
if (argc < 3) {
printf("usage:\n%s <db file> <lua file> [args ...]\n", argv[0]);
return 1;
}
db_file = argv[1];
lua_file = argv[2];
lua_State *L = luaL_newstate();
luaL_openlibs(L);
luaL_dostring(L, LUA_COMMON);
if (luaL_dofile(L, lua_file) != 0) {
return 1;
}
call_init(L, argc - 2, (const char**) argv + 2);
dst = open(db_file, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
if (dst == -1)
{
printf("Could not open destination file '%s': %s\n", db_file, strerror(errno));
rv = errno;
goto clean;
}
rv = rarchdb_create(dst, &value_provider, L);
clean:
lua_close(L);
if (dst != -1) {
close(dst);
}
return rv;
}

View File

@ -1,3 +0,0 @@
#!/bin/sh
./dat_converter "$1" "$2" && ./rarchdb_tool "$2" create-index crc crc && ./rarchdb_tool "$2" create-index md5 md5 && ./rarchdb_tool "$2" create-index sha1 sha1

View File

@ -1,395 +0,0 @@
#include "rarchdb.h"
#include <sys/types.h>
#ifdef _WIN32
#include <direct.h>
#else
#include <unistd.h>
#endif
#include <string.h>
#include <errno.h>
#include <sys/stat.h>
#include <stdlib.h>
#include <fcntl.h>
#include <stdio.h>
#include "rmsgpack_dom.h"
#include "rmsgpack.h"
#include "bintree.h"
#include "rarchdb_endian.h"
#define MAGIC_NUMBER "RARCHDB"
struct rarchdb_header
{
char magic_number[sizeof(MAGIC_NUMBER)-1];
uint64_t metadata_offset;
};
struct rarchdb_metadata
{
uint64_t count;
};
struct rarchdb_index
{
char name[50];
uint64_t key_size;
uint64_t next;
};
struct node_iter_ctx
{
struct rarchdb *db;
struct rarchdb_index *idx;
};
static struct rmsgpack_dom_value sentinal;
static int rarchdb_read_metadata(int fd, struct rarchdb_metadata *md)
{
return rmsgpack_dom_read_into(fd, "count", &md->count, NULL);
}
static int rarchdb_write_metadata(int fd, struct rarchdb_metadata *md)
{
rmsgpack_write_map_header(fd, 1);
rmsgpack_write_string(fd, "count", strlen("count"));
return rmsgpack_write_uint(fd, md->count);
}
int rarchdb_create(int fd, rarchdb_value_provider value_provider, void *ctx)
{
int rv;
uint64_t item_count = 0;
struct rmsgpack_dom_value item = {};
struct rarchdb_header header = {};
struct rarchdb_metadata md;
memcpy(header.magic_number, MAGIC_NUMBER, sizeof(MAGIC_NUMBER)-1);
off_t root = lseek(fd, 0, SEEK_CUR);
// We write the header in the end because we need to know the size of
// the db first
lseek(fd, sizeof(struct rarchdb_header), SEEK_CUR);
while ((rv = value_provider(ctx, &item)) == 0)
{
if((rv = rmsgpack_dom_write(fd, &item)) < 0)
return rv;
rmsgpack_dom_value_free(&item);
item_count++;
}
if (rv < 0)
return rv;
if((rv = rmsgpack_dom_write(fd, &sentinal)) < 0)
return rv;
header.metadata_offset = httobe64(lseek(fd, 0, SEEK_CUR));
md.count = item_count;
rarchdb_write_metadata(fd, &md);
lseek(fd, root, SEEK_SET);
write(fd, &header, sizeof(header));
printf(
#ifdef _WIN32
"Created DB with %I64u entries\n"
#else
"Created DB with %llu entries\n"
#endif
,(unsigned long long)item_count);
return 0;
}
static int rarchdb_read_index_header(int fd, struct rarchdb_index *idx)
{
uint64_t name_len = 50;
return rmsgpack_dom_read_into(fd,
"name", idx->name, &name_len,
"key_size", &idx->key_size,
"next", &idx->next,
NULL);
}
static void rarchdb_write_index_header(int fd, struct rarchdb_index *idx)
{
rmsgpack_write_map_header(fd, 3);
rmsgpack_write_string(fd, "name", strlen("name"));
rmsgpack_write_string(fd, idx->name, strlen(idx->name));
rmsgpack_write_string(fd, "key_size", strlen("key_size"));
rmsgpack_write_uint(fd, idx->key_size);
rmsgpack_write_string(fd, "next", strlen("next"));
rmsgpack_write_uint(fd, idx->next);
}
void rarchdb_close(struct rarchdb *db)
{
close(db->fd);
db->fd = -1;
}
int rarchdb_open(const char *path, struct rarchdb *db)
{
struct rarchdb_header header;
struct rarchdb_metadata md;
int rv;
int fd = open(path, O_RDWR);
if (fd == -1)
return -errno;
db->root = lseek(fd, 0, SEEK_CUR);
if ((rv = read(fd, &header, sizeof(header))) == -1)
{
rv = -errno;
goto error;
}
if (strncmp(header.magic_number, MAGIC_NUMBER, sizeof(MAGIC_NUMBER)) != 0)
{
rv = -EINVAL;
goto error;
}
header.metadata_offset = betoht64(header.metadata_offset);
lseek(fd, header.metadata_offset, SEEK_SET);
if (rarchdb_read_metadata(fd, &md) < 0)
{
rv = -EINVAL;
goto error;
}
db->count = md.count;
db->first_index_offset = lseek(fd, 0, SEEK_CUR);
db->fd = fd;
rarchdb_read_reset(db);
return 0;
error:
close(fd);
return rv;
}
static int rarchdb_find_index(struct rarchdb *db, const char *index_name, struct rarchdb_index *idx)
{
off_t eof = lseek(db->fd, 0, SEEK_END);
off_t offset = lseek(db->fd, db->first_index_offset, SEEK_SET);
while (offset < eof)
{
rarchdb_read_index_header(db->fd, idx);
if (strncmp(index_name, idx->name, strlen(idx->name)) == 0)
return 0;
offset = lseek(db->fd, idx->next, SEEK_CUR);
}
return -1;
}
static int node_compare(const void *a, const void* b, void *ctx)
{
return memcmp(a, b, *(uint8_t*)ctx);
}
static int binsearch(const void *buff, const void *item, uint64_t count, uint8_t field_size, uint64_t *offset)
{
int mid = count / 2;
int item_size = field_size + sizeof(uint64_t);
uint64_t *current = (uint64_t*)buff + (mid * item_size);
int rv = node_compare(current, item, &field_size);
if (rv == 0)
{
*offset = *(uint64_t*)(current + field_size);
return 0;
}
else if (count == 0)
return -1;
else if (rv > 0)
return binsearch(buff, item, mid, field_size, offset);
return binsearch(current + item_size, item, count - mid, field_size, offset);
}
int rarchdb_find_entry(struct rarchdb *db, const char *index_name, const void *key)
{
struct rarchdb_index idx;
int rv;
void *buff;
uint64_t offset;
ssize_t bufflen, nread = 0;
if (rarchdb_find_index(db, index_name, &idx) < 0)
{
rarchdb_read_reset(db);
return -1;
}
bufflen = idx.next;
buff = malloc(bufflen);
if (!buff)
return -ENOMEM;
while (nread < bufflen)
{
void *buff_ = (uint64_t*)buff + nread;
rv = read(db->fd, buff_, bufflen - nread);
if (rv <= 0)
{
free(buff);
return -errno;
}
nread += rv;
}
rv = binsearch(buff, key, db->count, idx.key_size, &offset);
free(buff);
rarchdb_read_reset(db);
if (rv == 0)
lseek(db->fd, offset, SEEK_SET);
return rv;
}
int rarchdb_read_reset(struct rarchdb *db)
{
db->eof = 0;
return lseek(db->fd, db->root + sizeof(struct rarchdb_header), SEEK_SET);
}
int rarchdb_read_item(struct rarchdb *db, struct rmsgpack_dom_value *out)
{
int rv;
if (db->eof)
return EOF;
rv = rmsgpack_dom_read(db->fd, out);
if (rv < 0)
return rv;
if (out->type == RDT_NULL)
{
db->eof = 1;
return EOF;
}
return 0;
}
static int node_iter(void *value, void *ctx)
{
struct node_iter_ctx *nictx = (struct node_iter_ctx*)ctx;
if (write(nictx->db->fd, value, nictx->idx->key_size + sizeof(uint64_t)) > 0)
return 0;
return -1;
}
uint64_t rarchdb_tell(struct rarchdb *db)
{
return lseek(db->fd, 0, SEEK_CUR);
}
int rarchdb_create_index(struct rarchdb *db, const char* name, const char *field_name)
{
int rv;
struct node_iter_ctx nictx;
struct rmsgpack_dom_value key;
struct rarchdb_index idx;
struct rmsgpack_dom_value item;
struct rmsgpack_dom_value *field;
void* buff = NULL;
uint64_t *buff_u64 = NULL;
uint8_t field_size = 0;
struct bintree tree;
uint64_t item_loc = rarchdb_tell(db);
uint64_t idx_header_offset;
bintree_new(&tree, node_compare, &field_size);
rarchdb_read_reset(db);
key.type = RDT_STRING;
key.string.len = strlen(field_name);
// We know we aren't going to change it
key.string.buff = (char *) field_name;
while(rarchdb_read_item(db, &item) == 0)
{
if (item.type != RDT_MAP)
{
rv = -EINVAL;
printf("Only map keys are supported\n");
goto clean;
}
field = rmsgpack_dom_value_map_value(&item, &key);
if (!field)
{
rv = -EINVAL;
printf("field not found in item\n");
goto clean;
}
if (field->type != RDT_BINARY)
{
rv = -EINVAL;
printf("field is not binary\n");
goto clean;
}
if (field->binary.len == 0)
{
rv = -EINVAL;
printf("field is empty\n");
goto clean;
}
if (field_size == 0)
field_size = field->binary.len;
else if (field->binary.len != field_size)
{
rv = -EINVAL;
printf("field is not of correct size\n");
goto clean;
}
buff = malloc(field_size + sizeof(uint64_t));
if(!buff)
{
rv = -ENOMEM;
goto clean;
}
memcpy(buff, field->binary.buff, field_size);
buff_u64 = (uint64_t*)buff + field_size;
memcpy(buff_u64, &item_loc, sizeof(uint64_t));
if (bintree_insert(&tree, buff) != 0)
{
printf("Value is not unique: ");
rmsgpack_dom_value_print(field);
printf("\n");
rv = -EINVAL;
goto clean;
}
buff = NULL;
rmsgpack_dom_value_free(&item);
item_loc = rarchdb_tell(db);
}
(void)rv;
(void)idx_header_offset;
idx_header_offset = lseek(db->fd, 0, SEEK_END);
strncpy(idx.name, name, 50);
idx.name[49] = '\0';
idx.key_size = field_size;
idx.next = db->count * (field_size + sizeof(uint64_t));
rarchdb_write_index_header(db->fd, &idx);
nictx.db = db;
nictx.idx = &idx;
bintree_iterate(&tree, node_iter, &nictx);
bintree_free(&tree);
clean:
rmsgpack_dom_value_free(&item);
if (buff)
free(buff);
rarchdb_read_reset(db);
return 0;
}

View File

@ -1,30 +0,0 @@
#ifndef __RARCHDB_H__
#define __RARCHDB_H__
#include <stdint.h>
#include "rmsgpack_dom.h"
struct rarchdb {
int fd;
int eof;
uint64_t root;
uint64_t count;
uint64_t first_index_offset;
};
typedef int(*rarchdb_value_provider)(void *ctx, struct rmsgpack_dom_value *out);
int rarchdb_create(int fd, rarchdb_value_provider value_provider, void *ctx);
void rarchdb_close(struct rarchdb *db);
int rarchdb_open(const char *path, struct rarchdb *db);
int rarchdb_read_item(struct rarchdb *db, struct rmsgpack_dom_value *out);
int rarchdb_read_reset(struct rarchdb *db);
int rarchdb_create_index(struct rarchdb *db, const char* name, const char *field_name);
int rarchdb_find_entry(struct rarchdb *db, const char *index_name, const void *key);
uint64_t rarchdb_tell(struct rarchdb *db);
#endif

View File

@ -1,28 +0,0 @@
#ifndef __RARCHDB_MSGPACK_ENDIAN_H
#define __RARCHDB_MSGPACK_ENDIAN_H
#include <stdint.h>
#include <retro_endianness.h>
#ifndef swap64
#define swap64(val) \
((((uint64_t)(val) & 0x00000000000000ffULL) << 56) \
| (((uint64_t)(val) & 0x000000000000ff00ULL) << 40) \
| (((uint64_t)(val) & 0x0000000000ff0000ULL) << 24) \
| (((uint64_t)(val) & 0x00000000ff000000ULL) << 8) \
| (((uint64_t)(val) & 0x000000ff00000000ULL) >> 8) \
| (((uint64_t)(val) & 0x0000ff0000000000ULL) >> 24) \
| (((uint64_t)(val) & 0x00ff000000000000ULL) >> 40) \
| (((uint64_t)(val) & 0xff00000000000000ULL) >> 56))
#endif
#define httobe64(x) (is_little_endian() ? swap64(x) : (x))
#define httobe32(x) (is_little_endian() ? SWAP32(x) : (x))
#define httobe16(x) (is_little_endian() ? SWAP16(x) : (x))
#define betoht16(x) httobe16(x)
#define betoht32(x) httobe32(x)
#define betoht64(x) httobe64(x)
#endif

View File

@ -1,116 +0,0 @@
#include <stdio.h>
#include <string.h>
#include "rarchdb.h"
#include "rmsgpack_dom.h"
static int list_db(void)
{
return 0;
}
int main(int argc, char** argv)
{
int rv;
struct rarchdb db;
struct rmsgpack_dom_value item;
if (argc < 3)
{
printf("Usage: %s <db file> <command> [extra args...]\n", argv[0]);
printf("Available Commands:\n");
printf("\tlist\n");
printf("\tcreate-index <index name> <field name>\n");
printf("\tfind <index name> <value>\n");
return 1;
}
const char* command = argv[2];
const char* path = argv[1];
if ((rv = rarchdb_open(path, &db)) != 0)
{
printf("Could not open db file '%s': %s\n", path, strerror(-rv));
return 1;
}
if (strcmp(command, "list") == 0)
{
if (argc != 3)
{
printf("Usage: %s <db file> list\n", argv[0]);
return 1;
}
while(rarchdb_read_item(&db, &item) == 0)
{
rmsgpack_dom_value_print(&item);
printf("\n");
rmsgpack_dom_value_free(&item);
}
}
else if (strcmp(command, "create-index") == 0)
{
const char *index_name, *field_name;
if (argc != 5)
{
printf("Usage: %s <db file> create-index <index name> <field name>\n", argv[0]);
return 1;
}
index_name = argv[3];
field_name = argv[4];
rarchdb_create_index(&db, index_name, field_name);
}
else if (strcmp(command, "find") == 0)
{
int i;
const char *index_name, *value;
char bin_value[256];
uint8_t h;
uint8_t l;
if (argc != 5)
{
printf("Usage: %s <db file> create-index <index name> <field name>\n", argv[0]);
return 1;
}
index_name = argv[3];
value = argv[4];
for (i = 0; i < strlen(value); i += 2)
{
if (value[i] <= '9')
h = value[i] - '0';
else
h = (value[i] - 'A') + 10;
if (value[i+1] <= '9')
l = value[i+1] - '0';
else
l = (value[i+1] - 'A') + 10;
bin_value[i/2] = h * 16 + l;
}
if (rarchdb_find_entry(&db, index_name, bin_value) != 0)
{
printf("Could not find item\n");
return 1;
}
if (rarchdb_read_item(&db, &item) == 0)
{
rmsgpack_dom_value_print(&item);
printf("\n");
rmsgpack_dom_value_free(&item);
}
}
else
{
printf("Unkown command %s\n", argv[1]);
return 1;
}
rarchdb_close(&db);
}

View File

@ -1,566 +0,0 @@
/*
* An almost complete implementation of msgpack by
* Saggi Mizrahi <ficoos@gmail.com>
*
* TODO:
* - float types
* - ext types
*
* For more information http://msgpack.org/
*/
#include "rmsgpack.h"
#include <stdlib.h>
#ifdef _WIN32
#include <direct.h>
#else
#include <unistd.h>
#endif
#include <errno.h>
#include <stdint.h>
#include <string.h>
#include "rarchdb_endian.h"
static const uint8_t MPF_FIXMAP = 0x80;
static const uint8_t MPF_MAP16 = 0xde;
static const uint8_t MPF_MAP32 = 0xdf;
static const uint8_t MPF_FIXARRAY = 0x90;
static const uint8_t MPF_ARRAY16 = 0xdc;
static const uint8_t MPF_ARRAY32 = 0xdd;
static const uint8_t MPF_FIXSTR = 0xa0;
static const uint8_t MPF_STR8 = 0xd9;
static const uint8_t MPF_STR16 = 0xda;
static const uint8_t MPF_STR32 = 0xdb;
static const uint8_t MPF_BIN8 = 0xc4;
static const uint8_t MPF_BIN16 = 0xc5;
static const uint8_t MPF_BIN32 = 0xc6;
static const uint8_t MPF_FALSE = 0xc2;
static const uint8_t MPF_TRUE = 0xc3;
static const uint8_t MPF_INT8 = 0xd0;
static const uint8_t MPF_INT16 = 0xd1;
static const uint8_t MPF_INT32 = 0xd2;
static const uint8_t MPF_INT64 = 0xd3;
static const uint8_t MPF_UINT8 = 0xcc;
static const uint8_t MPF_UINT16 = 0xcd;
static const uint8_t MPF_UINT32 = 0xce;
static const uint8_t MPF_UINT64 = 0xcf;
static const uint8_t MPF_NIL = 0xc0;
int rmsgpack_write_array_header(int fd, uint32_t size)
{
uint16_t tmp_i16;
uint32_t tmp_i32;
if (size < 16)
{
size = (size | MPF_FIXARRAY);
if (write(fd, &size, sizeof(int8_t)) == -1)
return -errno;
return sizeof(int8_t);
}
else if (size == (uint16_t)size)
{
if (write(fd, &MPF_ARRAY16, sizeof(MPF_ARRAY16)) == -1)
return -errno;
tmp_i16 = httobe16(size);
if (write(fd, (void *)(&tmp_i16), sizeof(uint16_t)) == -1)
return -errno;
return sizeof(int8_t) + sizeof(uint16_t);
}
else
{
if (write(fd, &MPF_ARRAY32, sizeof(MPF_ARRAY32)) == -1)
return -errno;
tmp_i32 = httobe32(size);
if (write(fd, (void *)(&tmp_i32), sizeof(uint32_t)) == -1)
return -errno;
return sizeof(int8_t) + sizeof(uint32_t);
}
}
int rmsgpack_write_map_header(int fd, uint32_t size)
{
uint16_t tmp_i16;
uint32_t tmp_i32;
if (size < 16)
{
size = (size | MPF_FIXMAP);
if (write(fd, &size, sizeof(int8_t)) == -1)
return -errno;
return sizeof(int8_t);
}
else if (size < (uint16_t)size)
{
if (write(fd, &MPF_MAP16, sizeof(MPF_MAP16)) == -1)
return -errno;
tmp_i16 = httobe16(size);
if (write(fd, (void *)(&tmp_i16), sizeof(uint16_t)) == -1)
return -errno;
return sizeof(uint8_t) + sizeof(uint16_t);
}
else
{
tmp_i32 = httobe32(size);
if (write(fd, &MPF_MAP32, sizeof(MPF_MAP32)) == -1)
return -errno;
if (write(fd, (void *)(&tmp_i32), sizeof(uint32_t)) == -1)
return -errno;
return sizeof(int8_t) + sizeof(uint32_t);
}
}
int rmsgpack_write_string(int fd, const char *s, uint32_t len)
{
int8_t fixlen = 0;
uint16_t tmp_i16;
uint32_t tmp_i32;
int written = sizeof(int8_t);
if (len < 32)
{
fixlen = len | MPF_FIXSTR;
if (write(fd, &fixlen, sizeof(int8_t)) == -1)
return -errno;
}
else if (len < 1<<8)
{
if (write(fd, &MPF_STR8, sizeof(MPF_STR8)) == -1)
return -errno;
if (write(fd, &len, sizeof(uint8_t)) == -1)
return -errno;
written += sizeof(uint8_t);
}
else if (len < 1<<16)
{
if (write(fd, &MPF_STR16, sizeof(MPF_STR16)) == -1)
return -errno;
tmp_i16 = httobe16(len);
if (write(fd, &tmp_i16, sizeof(uint16_t)) == -1)
return -errno;
written += sizeof(uint16_t);
}
else
{
if (write(fd, &MPF_STR32, sizeof(MPF_STR32)) == -1)
return -errno;
tmp_i32 = httobe32(len);
if (write(fd, &tmp_i32, sizeof(uint32_t)) == -1)
return -errno;
written += sizeof(uint32_t);
}
if (write(fd, s, len) == -1)
return -errno;
written += len;
return written;
}
int rmsgpack_write_bin(int fd, const void *s, uint32_t len)
{
uint16_t tmp_i16;
uint32_t tmp_i32;
int written = sizeof(int8_t);
if (len == (uint8_t)len)
{
if (write(fd, &MPF_BIN8, sizeof(MPF_BIN8)) == -1)
return -errno;
if (write(fd, &len, sizeof(uint8_t)) == -1)
return -errno;
written += sizeof(uint8_t);
}
else if (len == (uint16_t)len)
{
if (write(fd, &MPF_BIN16, sizeof(MPF_BIN16)) == -1)
return -errno;
tmp_i16 = httobe16(len);
if (write(fd, &tmp_i16, sizeof(uint16_t)) == -1)
return -errno;
written += sizeof(uint16_t);
}
else
{
if (write(fd, &MPF_BIN32, sizeof(MPF_BIN32)) == -1)
return -errno;
tmp_i32 = httobe32(len);
if (write(fd, &tmp_i32, sizeof(uint32_t)) == -1)
return -errno;
written += sizeof(uint32_t);
}
if (write(fd, s, len) == -1)
return -errno;
written += len;
return 0;
}
int rmsgpack_write_nil(int fd)
{
if (write(fd, &MPF_NIL, sizeof(MPF_NIL)) == -1)
return -errno;
return sizeof(uint8_t);
}
int rmsgpack_write_bool(int fd, int value)
{
if (value)
{
if (write(fd, &MPF_TRUE, sizeof(MPF_TRUE)) == -1)
return -errno;
}
else
{
if (write(fd, &MPF_FALSE, sizeof(MPF_FALSE)) == -1)
return -errno;
}
return sizeof(uint8_t);
}
int rmsgpack_write_int(int fd, int64_t value)
{
uint8_t tmpval = 0;
int16_t tmp_i16;
int32_t tmp_i32;
int written = sizeof(uint8_t);
if (value >=0 && value < 128)
{
if (write(fd, &value, sizeof(int8_t)) == -1)
return -errno;
}
else if (value < 0 && value > -32)
{
tmpval = (value) | 0xe0;
if (write(fd, &tmpval, sizeof(uint8_t)) == -1)
return -errno;
}
else if (value == (int8_t)value)
{
if (write(fd, &MPF_INT8, sizeof(MPF_INT8)) == -1)
return -errno;
if (write(fd, &value, sizeof(int8_t)) == -1)
return -errno;
written += sizeof(int8_t);
}
else if (value == (int16_t)value)
{
if (write(fd, &MPF_INT16, sizeof(MPF_INT16)) == -1)
return -errno;
tmp_i16 = httobe16(value);
if (write(fd, &tmp_i16, sizeof(int16_t)) == -1)
return -errno;
written += sizeof(int16_t);
}
else if (value == (int32_t)value)
{
if (write(fd, &MPF_INT32, sizeof(MPF_INT32)) == -1)
return -errno;
tmp_i32 = httobe32(value);
if (write(fd, &tmp_i32, sizeof(int32_t)) == -1)
return -errno;
written += sizeof(int32_t);
}
else
{
if (write(fd, &MPF_INT64, sizeof(MPF_INT64)) == -1)
return -errno;
value = httobe64(value);
if (write(fd, &value, sizeof(int64_t)) == -1)
return -errno;
written += sizeof(int64_t);
}
return written;
}
int rmsgpack_write_uint(int fd, uint64_t value)
{
uint16_t tmp_i16;
uint32_t tmp_i32;
int written = sizeof(uint8_t);
if (value == (uint8_t)value)
{
if (write(fd, &MPF_UINT8, sizeof(MPF_UINT8)) == -1)
return -errno;
if (write(fd, &value, sizeof(uint8_t)) == -1)
return -errno;
written += sizeof(uint8_t);
}
else if (value == (uint16_t)value)
{
if (write(fd, &MPF_UINT16, sizeof(MPF_UINT16)) == -1)
return -errno;
tmp_i16 = httobe16(value);
if (write(fd, &tmp_i16, sizeof(uint16_t)) == -1)
return -errno;
written += sizeof(uint16_t);
}
else if (value == (uint32_t)value)
{
if (write(fd, &MPF_UINT32, sizeof(MPF_UINT32)) == -1)
return -errno;
tmp_i32 = httobe32(value);
if (write(fd, &tmp_i32, sizeof(uint32_t)) == -1)
return -errno;
written += sizeof(uint32_t);
}
else
{
if (write(fd, &MPF_UINT64, sizeof(MPF_UINT64)) == -1)
return -errno;
value = httobe64(value);
if (write(fd, &value, sizeof(uint64_t)) == -1)
return -errno;
written += sizeof(uint64_t);
}
return written;
}
static int read_uint(int fd, uint64_t *out, size_t size)
{
uint64_t tmp;
if(read(fd, &tmp, size) == -1)
return -errno;
switch(size)
{
case 1:
*out = *(uint8_t*)(&tmp);
break;
case 2:
*out = betoht16(tmp);
break;
case 4:
*out = betoht32(tmp);
break;
case 8:
*out = betoht64(tmp);
break;
}
return 0;
}
static int read_int(int fd, int64_t *out, size_t size)
{
uint8_t tmp8 = 0;
uint16_t tmp16;
uint32_t tmp32;
uint64_t tmp64;
if(read(fd, &tmp64, size) == -1)
return -errno;
(void)tmp8;
switch(size)
{
case 1:
*out = *((int8_t*)(&tmp64));
break;
case 2:
tmp16 = betoht16(tmp64);
*out = *((int16_t*)(&tmp16));
break;
case 4:
tmp32 = betoht32(tmp64);
*out = *((int32_t*)(&tmp32));
break;
case 8:
tmp64 = betoht64(tmp64);
*out = *((int64_t*)(&tmp64));
break;
}
return 0;
}
static int read_buff(int fd, size_t size, char** pbuff, uint64_t *len)
{
uint64_t tmp_len = 0;
if(read_uint(fd, &tmp_len, size) == -1)
return -errno;
*pbuff = (char*)calloc(tmp_len + 1, sizeof(char));
if (read(fd, *pbuff, tmp_len) == -1)
{
free(*pbuff);
return -errno;
}
*len = tmp_len;
return 0;
}
static int read_map(int fd, uint32_t len, struct rmsgpack_read_callbacks *callbacks, void *data)
{
int rv;
unsigned i;
if (
callbacks->read_map_start &&
(rv = callbacks->read_map_start(len, data)) < 0)
return rv;
for (i = 0; i < len; i++)
{
if((rv = rmsgpack_read(fd, callbacks, data)) < 0)
return rv;
if((rv = rmsgpack_read(fd, callbacks, data)) < 0)
return rv;
}
return 0;
}
static int read_array(int fd, uint32_t len, struct rmsgpack_read_callbacks *callbacks, void *data)
{
int rv;
unsigned i;
if (
callbacks->read_array_start &&
(rv = callbacks->read_array_start(len, data)) < 0)
return rv;
for (i = 0; i < len; i++)
{
if((rv = rmsgpack_read(fd, callbacks, data)) < 0)
return rv;
}
return 0;
}
int rmsgpack_read(int fd, struct rmsgpack_read_callbacks *callbacks, void *data)
{
int rv;
uint64_t tmp_len = 0;
uint64_t tmp_uint = 0;
int64_t tmp_int = 0;
uint8_t type = 0;
char* buff = NULL;
if (read(fd, &type, sizeof(uint8_t)) == -1)
return -errno;
if (type < MPF_FIXMAP)
{
if (!callbacks->read_int)
return 0;
return callbacks->read_int(type, data);
}
else if (type < MPF_FIXARRAY)
{
tmp_len = type - MPF_FIXMAP;
return read_map(fd, tmp_len, callbacks, data);
}
else if (type < MPF_FIXSTR)
{
tmp_len = type - MPF_FIXARRAY;
return read_array(fd, tmp_len, callbacks, data);
}
else if (type < MPF_NIL)
{
tmp_len = type - MPF_FIXSTR;
buff = (char*)calloc(tmp_len + 1, sizeof(char));
if (!buff)
return -ENOMEM;
if(read(fd, buff, tmp_len) == -1)
{
free(buff);
return -errno;
}
buff[tmp_len] = '\0';
if (!callbacks->read_string)
{
free(buff);
return 0;
}
return callbacks->read_string(buff, tmp_len, data);
}
else if (type > MPF_MAP32)
{
if (!callbacks->read_int)
return 0;
return callbacks->read_int(type - 0xff - 1, data);
}
switch(type)
{
case 0xc0:
if(callbacks->read_nil)
return callbacks->read_nil(data);
break;
case 0xc2:
if(callbacks->read_bool)
return callbacks->read_bool(0, data);
break;
case 0xc3:
if(callbacks->read_bool)
return callbacks->read_bool(1, data);
break;
case 0xc4:
case 0xc5:
case 0xc6:
if((rv = read_buff(fd, 1<<(type - 0xc4), &buff, &tmp_len)) < 0)
return rv;
if (callbacks->read_bin)
return callbacks->read_bin(buff, tmp_len, data);
break;
case 0xcc:
case 0xcd:
case 0xce:
case 0xcf:
tmp_len = 1ULL << (type - 0xcc);
tmp_uint = 0;
if(read_uint(fd, &tmp_uint, tmp_len) == -1)
return -errno;
if (callbacks->read_uint)
return callbacks->read_uint(tmp_uint, data);
break;
case 0xd0:
case 0xd1:
case 0xd2:
case 0xd3:
tmp_len = 1ULL << (type - 0xd0);
tmp_int = 0;
if(read_int(fd, &tmp_int, tmp_len) == -1)
return -errno;
if (callbacks->read_int)
return callbacks->read_int(tmp_int, data);
break;
case 0xd9:
case 0xda:
case 0xdb:
if((rv = read_buff(fd, 1<<(type - 0xd9), &buff, &tmp_len)) < 0)
return rv;
if (callbacks->read_string)
return callbacks->read_string(buff, tmp_len, data);
break;
case 0xdc:
case 0xdd:
if(read_uint(fd, &tmp_len, 2<<(type - 0xdc)) == -1)
return -errno;
return read_array(fd, tmp_len, callbacks, data);
case 0xde:
case 0xdf:
if(read_uint(fd, &tmp_len, 2<<(type - 0xde)) == -1)
return -errno;
return read_map(fd, tmp_len, callbacks, data);
}
return 0;
}

View File

@ -1,30 +0,0 @@
#ifndef __RARCHDB_MSGPACK_H__
#define __RARCHDB_MSGPACK_H__
#include <stdint.h>
struct rmsgpack_read_callbacks {
int(*read_nil)(void*);
int(*read_bool)(int, void*);
int(*read_int)(int64_t, void*);
int(*read_uint)(uint64_t, void*);
int(*read_string)(char *, uint32_t, void*);
int(*read_bin)(void *, uint32_t, void*);
int(*read_map_start)(uint32_t, void*);
int(*read_array_start)(uint32_t, void*);
};
int rmsgpack_write_array_header(int fd, uint32_t size);
int rmsgpack_write_map_header(int fd, uint32_t size);
int rmsgpack_write_string(int fd, const char *s, uint32_t len);
int rmsgpack_write_bin(int fd, const void *s, uint32_t len);
int rmsgpack_write_nil(int fd);
int rmsgpack_write_bool(int fd, int value);
int rmsgpack_write_int(int fd, int64_t value);
int rmsgpack_write_uint(int fd, uint64_t value);
int rmsgpack_read(int fd, struct rmsgpack_read_callbacks *callbacks, void *data);
#endif

View File

@ -1,480 +0,0 @@
#include "rmsgpack_dom.h"
#include <stdint.h>
#include <stdio.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include <stdarg.h>
#include "rmsgpack.h"
#define MAX_DEPTH 128
struct dom_reader_state
{
int i;
struct rmsgpack_dom_value* stack[MAX_DEPTH];
};
static struct rmsgpack_dom_value *dom_reader_state_pop(struct dom_reader_state *s)
{
struct rmsgpack_dom_value *v = s->stack[s->i];
s->i--;
return v;
}
static int dom_reader_state_push(struct dom_reader_state *s, struct rmsgpack_dom_value *v)
{
//TODO: Verify we don't overflow
s->i++;
s->stack[s->i] = v;
return 0;
}
static int dom_read_nil(void *data)
{
struct dom_reader_state *dom_state = (struct dom_reader_state*)data;
struct rmsgpack_dom_value *v =
(struct rmsgpack_dom_value*)dom_reader_state_pop(dom_state);
v->type = RDT_NULL;
return 0;
}
static int dom_read_bool(int value, void *data)
{
struct dom_reader_state *dom_state = (struct dom_reader_state*)data;
struct rmsgpack_dom_value *v =
(struct rmsgpack_dom_value*)dom_reader_state_pop(dom_state);
v->type = RDT_BOOL;
v->bool_ = value;
return 0;
}
static int dom_read_int(int64_t value, void *data)
{
struct dom_reader_state *dom_state = (struct dom_reader_state*)data;
struct rmsgpack_dom_value *v =
(struct rmsgpack_dom_value*)dom_reader_state_pop(dom_state);
v->type = RDT_INT;
v->int_ = value;
return 0;
}
static int dom_read_uint(uint64_t value, void *data)
{
struct dom_reader_state *dom_state = (struct dom_reader_state*)data;
struct rmsgpack_dom_value *v =
(struct rmsgpack_dom_value*)dom_reader_state_pop(dom_state);
v->type = RDT_UINT;
v->uint_ = value;
return 0;
}
static int dom_read_string(char* value, uint32_t len, void *data)
{
struct dom_reader_state *dom_state = (struct dom_reader_state*)data;
struct rmsgpack_dom_value *v =
(struct rmsgpack_dom_value*)dom_reader_state_pop(dom_state);
v->type = RDT_STRING;
v->string.len = len;
v->string.buff = value;
return 0;
}
static int dom_read_bin(void* value, uint32_t len, void *data)
{
struct dom_reader_state *dom_state = (struct dom_reader_state*)data;
struct rmsgpack_dom_value *v =
(struct rmsgpack_dom_value*)dom_reader_state_pop(dom_state);
v->type = RDT_BINARY;
v->binary.len = len;
v->binary.buff = (char*)value;
return 0;
}
static int dom_read_map_start(uint32_t len, void *data)
{
unsigned i;
struct rmsgpack_dom_pair *items = NULL;
struct dom_reader_state *dom_state = (struct dom_reader_state*)data;
struct rmsgpack_dom_value *v = dom_reader_state_pop(dom_state);
v->type = RDT_MAP;
v->map.len = len;
v->map.items = NULL;
items = (struct rmsgpack_dom_pair *)calloc(len, sizeof(struct rmsgpack_dom_pair));
if (!items)
return -ENOMEM;
v->map.items = items;
for (i = 0; i < len; i++)
{
if (dom_reader_state_push(dom_state, &items[i].value) < 0)
return -ENOMEM;
if (dom_reader_state_push(dom_state, &items[i].key) < 0)
return -ENOMEM;
}
return 0;
}
static int dom_read_array_start(uint32_t len, void *data)
{
unsigned i;
struct dom_reader_state *dom_state = (struct dom_reader_state*)data;
struct rmsgpack_dom_value *v = dom_reader_state_pop(dom_state);
struct rmsgpack_dom_value *items = NULL;
v->type = RDT_ARRAY;
v->array.len = len;
v->array.items = NULL;
items = (struct rmsgpack_dom_value*)calloc(len, sizeof(struct rmsgpack_dom_pair));
if (!items)
return -ENOMEM;
v->array.items = items;
for (i = 0; i < len; i++)
{
if (dom_reader_state_push(dom_state, &items[i]) < 0)
return -ENOMEM;
}
return 0;
}
static struct rmsgpack_read_callbacks dom_reader_callbacks = {
dom_read_nil,
dom_read_bool,
dom_read_int,
dom_read_uint,
dom_read_string,
dom_read_bin,
dom_read_map_start,
dom_read_array_start
};
void rmsgpack_dom_value_free(struct rmsgpack_dom_value *v)
{
unsigned i;
switch (v->type)
{
case RDT_STRING:
free(v->string.buff);
break;
case RDT_BINARY:
free(v->binary.buff);
break;
case RDT_MAP:
for (i = 0; i < v->map.len; i++)
{
rmsgpack_dom_value_free(&v->map.items[i].key);
rmsgpack_dom_value_free(&v->map.items[i].value);
}
free(v->map.items);
break;
case RDT_ARRAY:
for (i = 0; i < v->array.len; i++)
rmsgpack_dom_value_free(&v->array.items[i]);
free(v->array.items);
break;
case RDT_NULL:
case RDT_INT:
case RDT_BOOL:
case RDT_UINT:
// Do nothing
break;
}
}
struct rmsgpack_dom_value *rmsgpack_dom_value_map_value(
const struct rmsgpack_dom_value *map, const struct rmsgpack_dom_value *key)
{
unsigned i;
if (map->type != RDT_MAP)
return NULL;
for (i = 0; i < map->map.len; i++)
{
if (rmsgpack_dom_value_cmp(key, &map->map.items[i].key) == 0)
return &map->map.items[i].value;
}
return NULL;
}
int rmsgpack_dom_value_cmp(
const struct rmsgpack_dom_value *a, const struct rmsgpack_dom_value *b)
{
int rv;
unsigned i;
if (a == b)
return 1;
if (a->type != b->type)
return 1;
switch (a->type)
{
case RDT_NULL:
return 0;
case RDT_BOOL:
return a->bool_ == b->bool_ ? 1 : 0;
case RDT_INT:
return a->int_ == b->int_ ? 1 : 0;
case RDT_UINT:
return a->uint_ == b->uint_ ? 1 : 0;
case RDT_STRING:
if (a->string.len != b->string.len)
return 1;
return strncmp(a->string.buff, b->string.buff, a->string.len);
case RDT_BINARY:
if (a->binary.len != b->binary.len)
return 1;
return memcmp(a->binary.buff, b->binary.buff, a->binary.len);
case RDT_MAP:
if (a->map.len != b->map.len)
return 1;
for (i = 0; i < a->map.len; i++)
{
if((rv = rmsgpack_dom_value_cmp(&a->map.items[i].key, &b->map.items[i].key)) != 0)
return rv;
if((rv = rmsgpack_dom_value_cmp(&a->map.items[i].value, &b->map.items[i].value)) != 0)
return rv;
}
break;
case RDT_ARRAY:
if (a->array.len != b->array.len)
return 1;
for (i = 0; i < a->array.len; i++)
{
if((rv = rmsgpack_dom_value_cmp(&a->array.items[i], &b->array.items[i])) != 0)
return rv;
}
break;
}
return 1;
}
void rmsgpack_dom_value_print(struct rmsgpack_dom_value *obj)
{
unsigned i;
switch (obj->type)
{
case RDT_NULL:
printf("nil");
break;
case RDT_BOOL:
if (obj->bool_)
printf("true");
else
printf("false");
break;
case RDT_INT:
#ifdef _WIN32
printf("%I64d", (signed long long)obj->int_);
#else
printf("%lld", (signed long long)obj->int_);
#endif
break;
case RDT_UINT:
#ifdef _WIN32
printf("%I64u", (unsigned long long)obj->uint_);
#else
printf("%llu", (unsigned long long)obj->uint_);
#endif
break;
case RDT_STRING:
printf("\"%s\"", obj->string.buff);
break;
case RDT_BINARY:
printf("\"");
for (i = 0; i < obj->binary.len; i++)
printf("%02X", (unsigned char) obj->binary.buff[i]);
printf("\"");
break;
case RDT_MAP:
printf("{");
for (i = 0; i < obj->map.len; i++)
{
rmsgpack_dom_value_print(&obj->map.items[i].key);
printf(": ");
rmsgpack_dom_value_print(&obj->map.items[i].value);
if (i < (obj->map.len - 1))
printf(", ");
}
printf("}");
break;
case RDT_ARRAY:
printf("[");
for (i = 0; i < obj->array.len; i++)
{
rmsgpack_dom_value_print(&obj->array.items[i]);
if (i < (obj->array.len - 1))
printf(", ");
}
printf("]");
}
}
int rmsgpack_dom_write(int fd, const struct rmsgpack_dom_value *obj)
{
unsigned i;
int rv = 0;
int written = 0;
switch (obj->type)
{
case RDT_NULL:
return rmsgpack_write_nil(fd);
case RDT_BOOL:
return rmsgpack_write_bool(fd, obj->bool_);
case RDT_INT:
return rmsgpack_write_int(fd, obj->int_);
case RDT_UINT:
return rmsgpack_write_uint(fd, obj->uint_);
case RDT_STRING:
return rmsgpack_write_string(fd, obj->string.buff, obj->string.len);
case RDT_BINARY:
return rmsgpack_write_bin(fd, obj->binary.buff, obj->binary.len);
case RDT_MAP:
if ((rv = rmsgpack_write_map_header(fd, obj->map.len)) < 0)
return rv;
written += rv;
for (i = 0; i < obj->map.len; i++)
{
if((rv = rmsgpack_dom_write(fd, &obj->map.items[i].key)) < 0)
return rv;
written += rv;
if((rv = rmsgpack_dom_write(fd, &obj->map.items[i].value)) < 0)
return rv;
written += rv;
}
break;
case RDT_ARRAY:
if ((rv = rmsgpack_write_array_header(fd, obj->array.len)) < 0)
return rv;
written += rv;
for (i = 0; i < obj->array.len; i++)
{
if((rv = rmsgpack_dom_write(fd, &obj->array.items[i])) < 0)
return rv;
written += rv;
}
}
return written;
}
int rmsgpack_dom_read(int fd, struct rmsgpack_dom_value *out)
{
int rv = 0;
struct dom_reader_state s;
s.i = 0;
s.stack[0] = out;
rv = rmsgpack_read(fd, &dom_reader_callbacks, &s);
if (rv < 0)
rmsgpack_dom_value_free(out);
return rv;
}
int rmsgpack_dom_read_into(int fd, ...)
{
va_list ap;
struct rmsgpack_dom_value map;
int rv;
const char* key_name;
struct rmsgpack_dom_value key;
struct rmsgpack_dom_value *value;
int64_t *int_value;
uint64_t *uint_value;
int *bool_value;
char *buff_value;
uint64_t min_len;
int value_type = 0;
va_start(ap, fd);
rv = rmsgpack_dom_read(fd, &map);
(void)value_type;
if (rv < 0)
return rv;
if (map.type != RDT_MAP)
{
rv = -EINVAL;
goto clean;
}
while (1)
{
key_name = va_arg(ap, const char*);
if (key_name == NULL)
{
rv = 0;
goto clean;
}
key.type = RDT_STRING;
key.string.len = strlen(key_name);
key.string.buff = (char *) key_name;
value = rmsgpack_dom_value_map_value(&map, &key);
switch(value->type)
{
case RDT_INT:
int_value = va_arg(ap, int64_t*);
*int_value = value->int_;
break;
case RDT_BOOL:
bool_value = va_arg(ap, int*);
*bool_value = value->bool_;
break;
case RDT_UINT:
uint_value = va_arg(ap, uint64_t*);
*uint_value = value->uint_;
break;
case RDT_BINARY:
buff_value = va_arg(ap, char*);
uint_value = va_arg(ap, uint64_t*);
*uint_value = value->binary.len;
min_len = value->binary.len > *uint_value ? *uint_value : value->binary.len;
memcpy(
buff_value,
value->binary.buff,
min_len
);
break;
case RDT_STRING:
buff_value = va_arg(ap, char*);
uint_value = va_arg(ap, uint64_t*);
min_len = value->string.len + 1 > *uint_value ? *uint_value : value->string.len + 1;
*uint_value = min_len;
memcpy(
buff_value,
value->string.buff,
min_len
);
break;
default:
rv = -1;
goto clean;
}
}
clean:
va_end(ap);
rmsgpack_dom_value_free(&map);
return 0;
}

View File

@ -1,56 +0,0 @@
#ifndef __RARCHDB_MSGPACK_DOM_H__
#define __RARCHDB_MSGPACK_DOM_H__
#include <stdint.h>
enum rmsgpack_dom_type {
RDT_NULL = 0,
RDT_BOOL,
RDT_UINT,
RDT_INT,
RDT_STRING,
RDT_BINARY,
RDT_MAP,
RDT_ARRAY
};
struct rmsgpack_dom_value {
enum rmsgpack_dom_type type;
union {
uint64_t uint_;
int64_t int_;
struct {
uint32_t len;
char* buff;
} string;
struct {
uint32_t len;
char* buff;
} binary;
int bool_;
struct {
uint32_t len;
struct rmsgpack_dom_pair *items;
} map;
struct {
uint32_t len;
struct rmsgpack_dom_value *items;
} array;
};
};
struct rmsgpack_dom_pair {
struct rmsgpack_dom_value key;
struct rmsgpack_dom_value value;
};
void rmsgpack_dom_value_print(struct rmsgpack_dom_value *obj);
void rmsgpack_dom_value_free(struct rmsgpack_dom_value *v);
int rmsgpack_dom_value_cmp(const struct rmsgpack_dom_value *a, const struct rmsgpack_dom_value *b);
struct rmsgpack_dom_value *rmsgpack_dom_value_map_value(const struct rmsgpack_dom_value *map, const struct rmsgpack_dom_value *key);
int rmsgpack_dom_read(int fd, struct rmsgpack_dom_value *out);
int rmsgpack_dom_write(int fd, const struct rmsgpack_dom_value *obj);
int rmsgpack_dom_read_into(int fd, ...);
#endif

View File

@ -1,191 +0,0 @@
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdlib.h>
#include <unistd.h>
#include "rmsgpack.h"
struct stub_state
{
int i;
uint64_t stack[256];
};
static void stub_state_push_map(struct stub_state *s, uint32_t size)
{
s->i++;
s->stack[s->i] = 1;
s->i++;
s->stack[s->i] = size * 2;
printf("{");
}
static void stub_state_push_array(struct stub_state *s, uint32_t size)
{
s->i++;
s->stack[s->i] = 2;
s->i++;
s->stack[s->i] = size;
printf("[");
}
static void stub_state_pre_print(struct stub_state *s)
{
}
static void stub_state_post_print(struct stub_state *s)
{
switch(s->stack[s->i - 1])
{
case 1:
if (s->stack[s->i] % 2 == 0)
{
printf(": ");
s->stack[s->i]--;
}
else if (s->stack[s->i] == 1)
{
printf("}");
s->i -= 2;
stub_state_post_print(s);
}
else
{
printf(", ");
s->stack[s->i]--;
}
break;
case 2:
if (s->stack[s->i] == 1)
{
printf("]");
s->i -= 2;
stub_state_post_print(s);
}
else
{
printf(", ");
s->stack[s->i]--;
}
break;
}
}
static int stub_read_map_start(uint32_t size, void* data)
{
stub_state_push_map(data, size);
return 0;
}
static int stub_read_array_start(uint32_t size, void* data)
{
stub_state_push_array(data, size);
return 0;
}
static int stub_read_string(char *s, uint32_t len, void* data)
{
stub_state_pre_print(data);
printf("'%s'", s);
stub_state_post_print(data);
free(s);
return 0;
}
static int stub_read_bin(void *s, uint32_t len, void* data)
{
stub_state_pre_print(data);
printf("b'%s'", s);
stub_state_post_print(data);
free(s);
return 0;
}
static int stub_read_uint(uint64_t value, void* data)
{
stub_state_pre_print(data);
printf("%lu", value);
stub_state_post_print(data);
return 0;
}
static int stub_read_nil(void* data)
{
stub_state_pre_print(data);
printf("nil");
stub_state_post_print(data);
return 0;
}
static int stub_read_int(int64_t value, void* data)
{
stub_state_pre_print(data);
printf("%ld", value);
stub_state_post_print(data);
return 0;
}
static int stub_read_bool(int value, void* data)
{
stub_state_pre_print(data);
if (value)
printf("true");
else
printf("false");
stub_state_post_print(data);
return 0;
}
static struct rmsgpack_read_callbacks stub_callbacks = {
stub_read_nil,
stub_read_bool,
stub_read_int,
stub_read_uint,
stub_read_string,
stub_read_bin,
stub_read_map_start,
stub_read_array_start
};
int main(void)
{
int fd;
/*
int fd = open("test.msgpack", O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
int rv = 0;
if (fd == -1)
{
printf("Could not open file: %s", strerror(errno));
return errno;
}
rmsgpack_write_map_header(fd, 2);
rmsgpack_write_string(fd, "compact", strlen("compact"));
rmsgpack_write_bool(fd, 1);
rmsgpack_write_string(fd, "schema", strlen("schema"));
rmsgpack_write_array_header(fd, 10);
rmsgpack_write_string(fd, "schema", strlen("schema"));
rmsgpack_write_uint(fd, 1<<17);
rmsgpack_write_int(fd, (1<<17) + 1);
rmsgpack_write_int(fd, 4);
rmsgpack_write_int(fd, -3);
rmsgpack_write_int(fd, -22);
rmsgpack_write_int(fd, -35);
rmsgpack_write_int(fd, -421421412);
rmsgpack_write_int(fd, 4214);
rmsgpack_write_int(fd, -4214);
rmsgpack_write_uint(fd, 1<<17);
close(fd);
*/
struct stub_state state;
state.i = 0;
state.stack[0] = 0;
fd = open("test.msgpack", O_RDONLY);
rmsgpack_read(fd, &stub_callbacks, &state);
printf("\n");
close(fd);
return 0;
}