From c16278f027ff238bfcbb6ef25e33971fb46d5fe3 Mon Sep 17 00:00:00 2001
From: twinaphex <libretro@gmail.com>
Date: Tue, 30 Dec 2014 21:02:15 +0100
Subject: [PATCH] Add rarchdb
---
rarchdb/.ycm_extra_conf.py | 115 ++++++++
rarchdb/Makefile | 31 +++
rarchdb/README.md | 13 +
rarchdb/bintree.c | 119 ++++++++
rarchdb/bintree.h | 27 ++
rarchdb/dat_converter.c | 291 ++++++++++++++++++++
rarchdb/db_parser.c | 80 ++++++
rarchdb/db_parser.h | 15 +
rarchdb/mkdb.sh | 3 +
rarchdb/rarchdb.c | 372 +++++++++++++++++++++++++
rarchdb/rarchdb.h | 28 ++
rarchdb/rarchdb_endian.h | 51 ++++
rarchdb/rarchdb_tool.c | 116 ++++++++
rarchdb/rmsgpack.c | 548 +++++++++++++++++++++++++++++++++++++
rarchdb/rmsgpack.h | 30 ++
rarchdb/rmsgpack_dom.c | 462 +++++++++++++++++++++++++++++++
rarchdb/rmsgpack_dom.h | 56 ++++
rarchdb/rmsgpack_test.c | 191 +++++++++++++
18 files changed, 2548 insertions(+)
create mode 100644 rarchdb/.ycm_extra_conf.py
create mode 100644 rarchdb/Makefile
create mode 100644 rarchdb/README.md
create mode 100644 rarchdb/bintree.c
create mode 100644 rarchdb/bintree.h
create mode 100644 rarchdb/dat_converter.c
create mode 100644 rarchdb/db_parser.c
create mode 100644 rarchdb/db_parser.h
create mode 100755 rarchdb/mkdb.sh
create mode 100644 rarchdb/rarchdb.c
create mode 100644 rarchdb/rarchdb.h
create mode 100644 rarchdb/rarchdb_endian.h
create mode 100644 rarchdb/rarchdb_tool.c
create mode 100644 rarchdb/rmsgpack.c
create mode 100644 rarchdb/rmsgpack.h
create mode 100644 rarchdb/rmsgpack_dom.c
create mode 100644 rarchdb/rmsgpack_dom.h
create mode 100644 rarchdb/rmsgpack_test.c
diff --git a/rarchdb/.ycm_extra_conf.py b/rarchdb/.ycm_extra_conf.py
new file mode 100644
index 0000000000..587f7d6b3e
--- /dev/null
+++ b/rarchdb/.ycm_extra_conf.py
@@ -0,0 +1,115 @@
+import os
+import ycm_core
+# Defaults, if no database can be found.
+defaultsc = [
+ 'gcc',
+ '-Wno-long-long',
+ '-Wno-variadic-macros',
+ '-pthread'
+ '-std=c99',
+ ]
+defaultscpp = [
+ 'c'
+ '-Wno-long-long',
+ '-Wno-variadic-macros',
+ '-pthread'
+ '-std=c99',
+ ]
+
+# Things that must be included.
+entered_flags = ['-fdelayed-template-parsing']
+
+#
+def DirectoryOfThisScript():
+ return os.path.dirname( os.path.abspath( __file__ ) )
+
+# Find all compilation databases in the build directory and load them.
+
+def MakeRelativePathsInFlagsAbsolute( flags, working_directory ):
+ if not working_directory:
+ return list( flags )
+ new_flags = []
+ make_next_absolute = False
+ path_flags = [ '-isystem', '-I', '-iquote', '--sysroot=' ]
+ for flag in flags:
+ new_flag = flag
+
+ if make_next_absolute:
+ make_next_absolute = False
+ if not flag.startswith( '/' ):
+ new_flag = os.path.join( working_directory, flag )
+
+ for path_flag in path_flags:
+ if flag == path_flag:
+ make_next_absolute = True
+ break
+
+ if flag.startswith( path_flag ):
+ path = flag[ len( path_flag ): ]
+ new_flag = path_flag + os.path.join( working_directory, path )
+ break
+
+ if new_flag:
+ new_flags.append( new_flag )
+ return new_flags
+
+
+def FlagsForFile( filename ):
+ # Search through all parent directories for a directory named 'build'
+ databases = []
+ path_focus = os.path.dirname(filename)
+ while len(path_focus) > 1:
+ for f in os.listdir(path_focus):
+ compilation_database_folder = path_focus + "/" + f
+ for r,d,f in os.walk(compilation_database_folder):
+ for files in f:
+ if files == 'compile_commands.json':
+ databases += [ycm_core.CompilationDatabase( r )]
+ path_focus = os.path.dirname(os.path.dirname(path_focus))
+
+ # Use a header's source file database for completion.
+ filetype_flags = []
+ if filename.endswith(".h"):
+ for f in os.listdir(os.path.dirname(filename)):
+ if filename.replace(".h",".cpp").find(f) != -1:
+ filename = filename.replace(".h",".cpp")
+ break
+ if filename.replace(".h",".c").find(f) != -1:
+ filename = filename.replace(".h",".c")
+ break
+ elif filename.endswith(".hpp"):
+ for f in os.listdir(os.path.dirname(filename)):
+ if filename.replace(".hpp",".cpp").find(f) != -1:
+ filename = filename.replace(".hpp",".cpp")
+ break
+
+ # Get the compile commands
+ final_flags = []
+ # If possible, from the database.
+ if len(databases) > 0:
+ for database in databases:
+ compilation_info = database.GetCompilationInfoForFile( filename )
+ fromfile_flags = MakeRelativePathsInFlagsAbsolute(
+ compilation_info.compiler_flags_,
+ compilation_info.compiler_working_dir_ )
+ final_flags += fromfile_flags
+
+ # If not, set some sane defaults.
+ else:
+ relative_to = DirectoryOfThisScript()
+ final_flags += MakeRelativePathsInFlagsAbsolute( flags, relative_to )
+ if not final_flags:
+ if filename.endswith(".c"):
+ final_flags = defaultsc
+ elif filename.endswith(".cpp"):
+ final_flags = defaultscpp
+
+ # This allows header files to be parsed according to their parent source
+ final_flags = filetype_flags + final_flags
+
+ # For things that must be included regardless:
+ final_flags += entered_flags
+ return {
+ 'flags': final_flags,
+ 'do_cache': True
+ }
diff --git a/rarchdb/Makefile b/rarchdb/Makefile
new file mode 100644
index 0000000000..ee5cb93760
--- /dev/null
+++ b/rarchdb/Makefile
@@ -0,0 +1,31 @@
+CFLAGS=-std=c99 -g
+DAT_CONVERTER_OBJ = rmsgpack.o \
+ rmsgpack_dom.o \
+ rarchdb.o \
+ bintree.o \
+ db_parser.o \
+ dat_converter.o \
+ $(NULL)
+RARCHDB_TOOL_OBJ = rmsgpack.o \
+ rmsgpack_dom.o \
+ db_parser.o \
+ rarchdb_tool.o \
+ bintree.o \
+ rarchdb.o \
+ $(NULL)
+
+all: dat_converter rmsgpack_test rarchdb_tool
+
+%.o: %.c
+ ${CC} $< -c ${CFLAGS} -o $@
+
+dat_converter: ${DAT_CONVERTER_OBJ}
+ ${CC} ${DAT_CONVERTER_OBJ} -o $@
+
+rarchdb_tool: ${RARCHDB_TOOL_OBJ}
+ ${CC} ${RARCHDB_TOOL_OBJ} -o $@
+
+rmsgpack_test:
+ gcc rmsgpack.c rmsgpack_test.c -std=c99 -g -o $@
+clean:
+ rm -rf *.o rmsgpack_test dat_converter rarchdb_tool
diff --git a/rarchdb/README.md b/rarchdb/README.md
new file mode 100644
index 0000000000..464996dbd7
--- /dev/null
+++ b/rarchdb/README.md
@@ -0,0 +1,13 @@
+# rarchdb
+A small read only database
+Mainly to be used by retroarch
+
+# Usage
+To convert a dat file use `dat_converter <dat file> <db file>`
+
+To list out the content of a db `rarchdb_tool <db file> list`
+To create an index `rarchdb_tool <db file> create-index <index name> <field name>`
+To find an entry with an index `rarchdb_tool <db file> find <index name> <value>`
+
+The util `mkdb.sh <dat file> <db file>` will create a db file with indexes for crc sha1 and md5
+
diff --git a/rarchdb/bintree.c b/rarchdb/bintree.c
new file mode 100644
index 0000000000..653f48cd29
--- /dev/null
+++ b/rarchdb/bintree.c
@@ -0,0 +1,119 @@
+#include <stdlib.h>
+#include <errno.h>
+#include <stdio.h>
+
+#include "bintree.h"
+
+static int NIL_VALUE = 1;
+static void* NIL_NODE = &NIL_NODE;
+
+static struct bintree_node *new_nil_node(struct bintree_node *parent);
+
+void bintree_new(struct bintree *t, bintree_cmp_func cmp, void *ctx)
+{
+ t->root = new_nil_node(NULL);
+ t->cmp = cmp;
+ t->ctx = ctx;
+}
+
+static struct bintree_node *new_nil_node(struct bintree_node *parent)
+{
+ struct bintree_node *node = calloc(1, sizeof(struct bintree_node));
+ if (!node)
+ return NULL;
+ node->value = NIL_NODE;
+ node->parent = parent;
+ node->left = NULL;
+ node->right = NULL;
+ return node;
+}
+
+static inline int is_nil(const struct bintree_node *node)
+{
+ return node == NULL || node->value == NIL_NODE;
+}
+
+static int insert(struct bintree *t, struct bintree_node *root, void *value)
+{
+ int cmp_res = 0;
+ if (is_nil(root))
+ {
+ root->left = new_nil_node(root);
+ root->right = new_nil_node(root);
+ if (!root->left || !root->right)
+ {
+ if (root->left)
+ {
+ free(root->left);
+ root->left = NULL;
+ }
+ if (root->right)
+ {
+ free(root->right);
+ root->right = NULL;
+ }
+ return -ENOMEM;
+ }
+ root->value = value;
+ }
+ else
+ {
+ cmp_res = t->cmp(root->value, value, t->ctx);
+ if (cmp_res > 0)
+ return insert(t, root->left, value);
+ else if (cmp_res < 0)
+ return insert(t, root->right, value);
+ else
+ return -EINVAL;
+ }
+ return 0;
+}
+
+int bintree_insert(struct bintree *t, void *value)
+{
+ return insert(t, t->root, value);
+}
+
+static int _bintree_iterate(struct bintree_node *n, bintree_iter_cb cb, void *ctx)
+{
+ int rv;
+ if (is_nil(n))
+ return 0;
+ else
+ {
+ if((rv = _bintree_iterate(n->left, cb, ctx)) != 0)
+ return rv;
+ if ((rv = cb(n->value, ctx)) != 0)
+ return rv;
+ if((rv = _bintree_iterate(n->right, cb, ctx)) != 0)
+ return rv;
+ }
+
+ return 0;
+}
+
+int bintree_iterate(const struct bintree *t, bintree_iter_cb cb, void *ctx)
+{
+ return _bintree_iterate(t->root, cb, ctx);
+}
+
+static void bintree_free_node(struct bintree_node *n)
+{
+ if (n == NULL)
+ return;
+
+ if (n->value == NIL_NODE)
+ {
+ free(n);
+ return;
+ }
+ n->value = NULL;
+ bintree_free_node(n->left);
+ bintree_free_node(n->right);
+ free(n);
+}
+
+void bintree_free(struct bintree *t)
+{
+ bintree_free_node(t->root);
+}
diff --git a/rarchdb/bintree.h b/rarchdb/bintree.h
new file mode 100644
index 0000000000..969726b177
--- /dev/null
+++ b/rarchdb/bintree.h
@@ -0,0 +1,27 @@
+#ifndef __RARCHDB_BINTREE_H__
+#define __RARCHDB_BINTREE_H__
+
+typedef int(*bintree_cmp_func)(const void *a, const void *b, void* ctx);
+
+typedef int(*bintree_iter_cb)(void *value, void* ctx);
+
+
+struct bintree_node {
+ void* value;
+ struct bintree_node *parent;
+ struct bintree_node *left;
+ struct bintree_node *right;
+};
+
+struct bintree {
+ struct bintree_node *root;
+ bintree_cmp_func cmp;
+ void *ctx;
+};
+
+void bintree_new(struct bintree *t, bintree_cmp_func cmp, void *ctx);
+int bintree_insert(struct bintree *t, void *value);
+int bintree_iterate(const struct bintree *t, bintree_iter_cb cb, void* ctx);
+void bintree_free(struct bintree *t);
+
+#endif
diff --git a/rarchdb/dat_converter.c b/rarchdb/dat_converter.c
new file mode 100644
index 0000000000..b3b849c2f3
--- /dev/null
+++ b/rarchdb/dat_converter.c
@@ -0,0 +1,291 @@
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <string.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "rarchdb.h"
+#include "db_parser.h"
+
+#define MAX_TOKEN 256
+
+static char *strndup_(const char *s, size_t n)
+{
+ char* buff = calloc(n, sizeof(char));
+
+ if (!buff)
+ return 0;
+
+ strncpy(buff, s, n);
+ return buff;
+}
+
+static struct rmsgpack_dom_value *get_map_value(const struct rmsgpack_dom_value *m, char* key)
+{
+ struct rmsgpack_dom_value v;
+
+ v.type = RDT_STRING;
+ v.string.len = strlen(key);
+ v.string.buff = key;
+ return rmsgpack_dom_value_map_value(m, &v);
+}
+
+static int load_string(int fd, struct rmsgpack_dom_value *out)
+{
+ char tok[MAX_TOKEN];
+ ssize_t tok_size;
+
+ if ((tok_size = get_token(fd, tok, MAX_TOKEN)) < 0)
+ return tok_size;
+
+ out->type = RDT_STRING;
+ out->string.len = tok_size;
+ out->string.buff = strndup_(tok, tok_size);
+ return 0;
+}
+
+static int load_uint(int fd, struct rmsgpack_dom_value *out)
+{
+ char tok[MAX_TOKEN], *c;
+ ssize_t tok_size;
+ uint64_t value = 0;
+
+ if ((tok_size = get_token(fd, tok, MAX_TOKEN)) < 0)
+ return tok_size;
+
+ for (c = tok; c < tok + tok_size; c++)
+ {
+ value *= 10;
+ value += *c - '0';
+ }
+
+ out->type = RDT_UINT;
+ out->uint_ = value;
+ return 0;
+}
+
+static int load_bin(int fd, struct rmsgpack_dom_value *out)
+{
+ char tok[MAX_TOKEN];
+ ssize_t tok_size;
+ uint8_t h;
+ uint8_t l;
+ int i;
+
+ if ((tok_size = get_token(fd, tok, MAX_TOKEN)) < 0)
+ return tok_size;
+
+ out->type = RDT_BINARY;
+ out->binary.len = tok_size / 2;
+
+ for (i = 0; i < tok_size; i += 2)
+ {
+ if (tok[i] <= '9')
+ h = tok[i] - '0';
+ else
+ h = (tok[i] - 'A') + 10;
+ if (tok[i+1] <= '9')
+ l = tok[i+1] - '0';
+ else
+ l = (tok[i+1] - 'A') + 10;
+ tok[i/2] = h * 16 + l;
+ }
+
+ out->binary.buff = malloc(out->binary.len);
+ memcpy(out->binary.buff, tok, out->binary.len);
+ return 0;
+}
+
+static int dat_value_provider(void *ctx, struct rmsgpack_dom_value *out)
+{
+ int rv, i;
+ static const int field_count = 22;
+ int fd = *((int*)ctx);
+ char* key;
+
+ out->type = RDT_MAP;
+ out->map.len = field_count;
+ out->map.items = calloc(field_count, sizeof(struct rmsgpack_dom_pair));
+
+ if (find_token(fd, "game") < 0)
+ return 1;
+
+ for (i = 0; i < field_count; i++)
+ {
+ if ((rv = load_string(fd, &out->map.items[i].key)) < 0)
+ goto failed;
+
+ key = out->map.items[i].key.string.buff;
+
+ if (strncmp(key, "name", sizeof("name")) == 0)
+ {
+ if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
+ goto failed;
+ }
+ else if (strncmp(key, "description", sizeof("description")) == 0)
+ {
+ if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
+ goto failed;
+ }
+ else if (strncmp(key, "users", sizeof("users")) == 0)
+ {
+ if ((rv = load_uint(fd, &out->map.items[i].value)) < 0)
+ goto failed;
+ }
+ else if (strncmp(key, "releasemonth", sizeof("releasemonth")) == 0)
+ {
+ if ((rv = load_uint(fd, &out->map.items[i].value)) < 0)
+ goto failed;
+ }
+ else if (strncmp(key, "releaseyear", sizeof("releaseyear")) == 0)
+ {
+ if ((rv = load_uint(fd, &out->map.items[i].value)) < 0)
+ goto failed;
+ }
+ else if (strncmp(key, "rumble", sizeof("rumble")) == 0)
+ {
+ if ((rv = load_uint(fd, &out->map.items[i].value)) < 0)
+ goto failed;
+ }
+ else if (strncmp(key, "analog", sizeof("analog")) == 0)
+ {
+ if ((rv = load_uint(fd, &out->map.items[i].value)) < 0)
+ goto failed;
+ }
+ else if (strncmp(key, "serial", sizeof("serial")) == 0)
+ {
+ if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
+ goto failed;
+ }
+ else if (strncmp(key, "esrb_rating", sizeof("esrb_rating")) == 0)
+ {
+ if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
+ goto failed;
+ }
+ else if (strncmp(key, "elspa_rating", sizeof("elspa_rating")) == 0)
+ {
+ if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
+ goto failed;
+ }
+ else if (strncmp(key, "pegi_rating", sizeof("pegi_rating")) == 0)
+ {
+ if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
+ goto failed;
+ }
+ else if (strncmp(key, "cero_rating", sizeof("cero_rating")) == 0)
+ {
+ if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
+ goto failed;
+ }
+ else if (strncmp(key, "developers", sizeof("developers")) == 0)
+ {
+ if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
+ goto failed;
+ }
+ else if (strncmp(key, "publisher", sizeof("publisher")) == 0)
+ {
+ if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
+ goto failed;
+ }
+ else if (strncmp(key, "origin", sizeof("origin")) == 0)
+ {
+ if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
+ goto failed;
+ }
+ else if (strncmp(key, "rom", sizeof("rom")) == 0)
+ {
+ if (find_token(fd, "name") < 0)
+ goto failed;
+
+ if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
+ goto failed;
+ }
+ else if (strncmp(key, "size", sizeof("size")) == 0)
+ {
+ if ((rv = load_uint(fd, &out->map.items[i].value)) < 0)
+ goto failed;
+ }
+ else if (strncmp(key, "sha1", sizeof("sha1")) == 0)
+ {
+ if ((rv = load_bin(fd, &out->map.items[i].value)) < 0)
+ goto failed;
+ }
+ else if (strncmp(key, "crc", sizeof("crc")) == 0)
+ {
+ if ((rv = load_bin(fd, &out->map.items[i].value)) < 0)
+ goto failed;
+ }
+ else if (strncmp(key, "md5", sizeof("md5")) == 0)
+ {
+ if ((rv = load_bin(fd, &out->map.items[i].value)) < 0)
+ goto failed;
+ }
+ else if (strncmp(key, "serial", sizeof("serial")) == 0)
+ {
+ if ((rv = load_string(fd, &out->map.items[i].value)) < 0)
+ goto failed;
+ }
+ else if (strncmp(key, ")", sizeof(")")) == 0)
+ {
+ rmsgpack_dom_value_free(&out->map.items[i].key);
+ out->map.len = i;
+ printf("Couldn't find all fields for item\n");
+ break;
+ }
+ else
+ {
+ rmsgpack_dom_value_free(&out->map.items[i].key);
+ i--;
+ }
+ }
+ printf("Inserting '%s' (%02X%02X%02X%02X)...\n",
+ get_map_value(out, "name")->string.buff,
+ (unsigned char)get_map_value(out, "crc")->binary.buff[0],
+ (unsigned char)get_map_value(out, "crc")->binary.buff[1],
+ (unsigned char)get_map_value(out, "crc")->binary.buff[2],
+ (unsigned char)get_map_value(out, "crc")->binary.buff[3]
+ );
+ return 0;
+
+failed:
+ rmsgpack_dom_value_free(out);
+ out->type = RDT_NULL;
+ return rv;
+}
+
+int main(int argc, char **argv)
+{
+ int rv = 0;
+ int src = -1;
+ int dst = -1;
+ if (argc != 3)
+ printf("Usage: %s <dat file> <output file>\n", argv[0]);
+
+ src = open(argv[1], O_RDONLY);
+ if (src == -1)
+ {
+ printf("Could not open source file '%s': %s\n", argv[1], strerror(errno));
+ rv = errno;
+ goto clean;
+ }
+
+ dst = open(argv[2], O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
+ if (dst == -1)
+ {
+ printf("Could not open destination file '%s': %s\n", argv[1], strerror(errno));
+ rv = errno;
+ goto clean;
+ }
+
+ rv = rarchdb_create(dst, &dat_value_provider, &src);
+
+clean:
+ if (src != -1)
+ close(src);
+ if (dst != -1)
+ close(dst);
+ return rv;
+}
diff --git a/rarchdb/db_parser.c b/rarchdb/db_parser.c
new file mode 100644
index 0000000000..4da31a4426
--- /dev/null
+++ b/rarchdb/db_parser.c
@@ -0,0 +1,80 @@
+#include "db_parser.h"
+
+#include <errno.h>
+#include <string.h>
+#include <stdlib.h>
+
+ssize_t get_token(int fd, char *token, size_t max_len)
+{
+ char *c = token;
+ int rv;
+ ssize_t len = 0;
+ int in_string = 0;
+
+ while (1)
+ {
+ rv = read(fd, c, 1);
+ if (rv == 0)
+ return 0;
+ else if (rv < 1)
+ {
+ switch (errno)
+ {
+ case EINTR:
+ case EAGAIN:
+ continue;
+ default:
+ return -errno;
+ }
+ }
+
+ switch (*c)
+ {
+ case ' ':
+ case '\t':
+ case '\r':
+ case '\n':
+ if (c == token)
+ continue;
+
+ if (!in_string)
+ {
+ *c = '\0';
+ return len;
+ }
+ break;
+ case '\"':
+ if (c == token)
+ {
+ in_string = 1;
+ continue;
+ }
+
+ *c = '\0';
+ return len;
+ }
+
+ len++;
+ c++;
+ if (len == (ssize_t)max_len)
+ {
+ *c = '\0';
+ return len;
+ }
+ }
+}
+
+int find_token(int fd, const char *token)
+{
+ int tmp_len = strlen(token);
+ char *tmp_token = (char*)calloc(tmp_len, 1);
+ if (!tmp_token)
+ return -1;
+ while (strncmp(tmp_token, token, tmp_len) != 0)
+ {
+ if (get_token(fd, tmp_token, tmp_len) <= 0)
+ return -1;
+ }
+
+ return 0;
+}
diff --git a/rarchdb/db_parser.h b/rarchdb/db_parser.h
new file mode 100644
index 0000000000..eb87753a35
--- /dev/null
+++ b/rarchdb/db_parser.h
@@ -0,0 +1,15 @@
+#ifndef __RARCHDB_PARSER_H
+#define __RARCHDB_PARSER_H
+
+#ifdef _WIN32
+#include <io.h>
+#else
+#include <unistd.h>
+#endif
+
+#define MAX_TOKEN_LEN 255
+
+ssize_t get_token(int fd, char* token, size_t max_len);
+int find_token(int fd, const char* token);
+
+#endif
diff --git a/rarchdb/mkdb.sh b/rarchdb/mkdb.sh
new file mode 100755
index 0000000000..007b00f1d5
--- /dev/null
+++ b/rarchdb/mkdb.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+./dat_converter "$1" "$2" && ./rarchdb_tool "$2" create-index crc crc && ./rarchdb_tool "$2" create-index md5 md5 && ./rarchdb_tool "$2" create-index sha1 sha1
+
diff --git a/rarchdb/rarchdb.c b/rarchdb/rarchdb.c
new file mode 100644
index 0000000000..c8151db0c3
--- /dev/null
+++ b/rarchdb/rarchdb.c
@@ -0,0 +1,372 @@
+#include "rarchdb.h"
+
+#include <sys/types.h>
+#include <unistd.h>
+#include <string.h>
+#include <errno.h>
+#include <sys/stat.h>
+#include <stdlib.h>
+#include <fcntl.h>
+
+#include <stdio.h>
+
+#include "rmsgpack_dom.h"
+#include "rmsgpack.h"
+#include "bintree.h"
+#include "rarchdb_endian.h"
+
+#define MAGIC_NUMBER "RARCHDB"
+
+struct rarchdb_header
+{
+ char magic_number[7];
+ uint64_t metadata_offset;
+};
+
+struct rarchdb_metadata
+{
+ uint64_t count;
+};
+
+struct rarchdb_index
+{
+ char name[50];
+ uint64_t key_size;
+ uint64_t next;
+};
+
+struct node_iter_ctx
+{
+ struct rarchdb *db;
+ struct rarchdb_index *idx;
+};
+
+static struct rmsgpack_dom_value sentinal;
+
+static int rarchdb_read_metadata(int fd, struct rarchdb_metadata *md)
+{
+ return rmsgpack_dom_read_into(fd, "count", &md->count, NULL);
+}
+
+static int rarchdb_write_metadata(int fd, struct rarchdb_metadata *md)
+{
+ rmsgpack_write_map_header(fd, 1);
+ rmsgpack_write_string(fd, "count", strlen("count"));
+ return rmsgpack_write_uint(fd, md->count);
+}
+
+int rarchdb_create(int fd, rarchdb_value_provider value_provider, void *ctx)
+{
+ int rv;
+ uint64_t item_count = 0;
+ struct rmsgpack_dom_value item = {};
+ struct rarchdb_header header = {};
+ struct rarchdb_metadata md;
+ strcpy(header.magic_number, MAGIC_NUMBER);
+ off_t root = lseek(fd, 0, SEEK_CUR);
+ // We write the header in the end because we need to know the size of
+ // the db first
+ lseek(fd, sizeof(struct rarchdb_header), SEEK_CUR);
+ while ((rv = value_provider(ctx, &item)) == 0)
+ {
+ if((rv = rmsgpack_dom_write(fd, &item)) < 0)
+ return rv;
+ rmsgpack_dom_value_free(&item);
+ item_count++;
+ }
+
+ if (rv < 0)
+ return rv;
+
+ if((rv = rmsgpack_dom_write(fd, &sentinal)) < 0)
+ return rv;
+
+ header.metadata_offset = httobe64(lseek(fd, 0, SEEK_CUR));
+ md.count = item_count;
+ rarchdb_write_metadata(fd, &md);
+ lseek(fd, root, SEEK_SET);
+ write(fd, &header, sizeof(header));
+ printf("Created DB with %lu entries\n", item_count);
+ return 0;
+}
+
+static int rarchdb_read_index_header(int fd, struct rarchdb_index *idx)
+{
+ uint64_t name_len = 50;
+ return rmsgpack_dom_read_into(fd,
+ "name", idx->name, &name_len,
+ "key_size", &idx->key_size,
+ "next", &idx->next,
+ NULL);
+}
+
+static void rarchdb_write_index_header(int fd, struct rarchdb_index *idx)
+{
+ rmsgpack_write_map_header(fd, 3);
+ rmsgpack_write_string(fd, "name", strlen("name"));
+ rmsgpack_write_string(fd, idx->name, strlen(idx->name));
+ rmsgpack_write_string(fd, "key_size", strlen("key_size"));
+ rmsgpack_write_uint(fd, idx->key_size);
+ rmsgpack_write_string(fd, "next", strlen("next"));
+ rmsgpack_write_uint(fd, idx->next);
+}
+
+void rarchdb_close(struct rarchdb *db)
+{
+ close(db->fd);
+ db->fd = -1;
+}
+
+int rarchdb_open(const char *path, struct rarchdb *db)
+{
+ struct rarchdb_header header;
+ struct rarchdb_metadata md;
+ int rv;
+ int fd = open(path, O_RDWR);
+ if (fd == -1)
+ return -errno;
+
+ db->root = lseek(fd, 0, SEEK_CUR);
+ if ((rv = read(fd, &header, sizeof(header))) == -1)
+ {
+ rv = -errno;
+ goto error;
+ }
+
+ if (strncmp(header.magic_number, MAGIC_NUMBER, sizeof(MAGIC_NUMBER)) != 0)
+ {
+ rv = -EINVAL;
+ goto error;
+ }
+
+ header.metadata_offset = betoht64(header.metadata_offset);
+ lseek(fd, header.metadata_offset, SEEK_SET);
+ if (rarchdb_read_metadata(fd, &md) < 0)
+ {
+ rv = -EINVAL;
+ goto error;
+ }
+ db->count = md.count;
+ db->first_index_offset = lseek(fd, 0, SEEK_CUR);
+ db->fd = fd;
+ rarchdb_read_reset(db);
+ return 0;
+error:
+ close(fd);
+ return rv;
+}
+
+static int rarchdb_find_index(struct rarchdb *db, const char *index_name, struct rarchdb_index *idx)
+{
+ off_t eof = lseek(db->fd, 0, SEEK_END);
+ off_t offset = lseek(db->fd, db->first_index_offset, SEEK_SET);
+ while (offset < eof)
+ {
+ rarchdb_read_index_header(db->fd, idx);
+ if (strncmp(index_name, idx->name, strlen(idx->name)) == 0)
+ return 0;
+ offset = lseek(db->fd, idx->next, SEEK_CUR);
+ }
+ return -1;
+}
+
+static int node_compare(const void *a, const void* b, void *ctx)
+{
+ return memcmp(a, b, *(uint8_t*)ctx);
+}
+
+static int binsearch(const void *buff, const void *item, uint64_t count, uint8_t field_size, uint64_t *offset)
+{
+ int mid = count / 2;
+ int item_size = field_size + sizeof(uint64_t);
+ const void *current = buff + (mid * item_size);
+ int rv = node_compare(current, item, &field_size);
+
+ if (rv == 0)
+ {
+ *offset = *(uint64_t*)(current + field_size);
+ return 0;
+ }
+ else if (count == 0)
+ return -1;
+ else if (rv > 0)
+ return binsearch(buff, item, mid, field_size, offset);
+
+ return binsearch(current + item_size, item, count - mid, field_size, offset);
+}
+
+int rarchdb_find_entry(struct rarchdb *db, const char *index_name, const void *key)
+{
+ struct rarchdb_index idx;
+ uint64_t offset;
+ ssize_t nread = 0;
+ int rv;
+ if (rarchdb_find_index(db, index_name, &idx) < 0)
+ {
+ rarchdb_read_reset(db);
+ return -1;
+ }
+ size_t bufflen = idx.next;
+ void* buff = malloc(bufflen);
+ if (!buff)
+ return -ENOMEM;
+
+ while (nread < bufflen)
+ {
+ rv = read(db->fd, buff + nread, bufflen - nread);
+ if (rv <= 0)
+ {
+ free(buff);
+ return -errno;
+ }
+ nread += rv;
+ }
+
+ rv = binsearch(buff, key, db->count, idx.key_size, &offset);
+ free(buff);
+ rarchdb_read_reset(db);
+
+ if (rv == 0)
+ lseek(db->fd, offset, SEEK_SET);
+
+ return rv;
+}
+
+int rarchdb_read_reset(struct rarchdb *db)
+{
+ db->eof = 0;
+ return lseek(db->fd, db->root + sizeof(struct rarchdb_header), SEEK_SET);
+}
+
+int rarchdb_read_item(struct rarchdb *db, struct rmsgpack_dom_value *out)
+{
+ int rv;
+ if (db->eof)
+ return EOF;
+
+ rv = rmsgpack_dom_read(db->fd, out);
+ if (rv < 0)
+ return rv;
+
+ if (out->type == RDT_NULL)
+ {
+ db->eof = 1;
+ return EOF;
+ }
+
+ return 0;
+}
+
+static int node_iter(void *value, void *ctx)
+{
+ struct node_iter_ctx *nictx = ctx;
+
+ if (write(nictx->db->fd, value, nictx->idx->key_size + sizeof(uint64_t)) > 0)
+ return 0;
+
+ return -1;
+}
+
+uint64_t rarchdb_tell(struct rarchdb *db)
+{
+ return lseek(db->fd, 0, SEEK_CUR);
+}
+
+int rarchdb_create_index(struct rarchdb *db, const char* name, const char *field_name)
+{
+ int rv;
+ struct rmsgpack_dom_value key;
+ struct rarchdb_index idx;
+ struct rmsgpack_dom_value item;
+ struct rmsgpack_dom_value *field;
+ void* buff = NULL;
+ uint8_t field_size = 0;
+ struct bintree tree;
+ uint64_t item_loc = rarchdb_tell(db);
+ uint64_t idx_header_offset;
+
+ bintree_new(&tree, node_compare, &field_size);
+ rarchdb_read_reset(db);
+
+ key.type = RDT_STRING;
+ key.string.len = strlen(field_name);
+ // We know we aren't going to change it
+ key.string.buff = (char *) field_name;
+ while(rarchdb_read_item(db, &item) == 0)
+ {
+ if (item.type != RDT_MAP)
+ {
+ rv = -EINVAL;
+ printf("Only map keys are supported\n");
+ goto clean;
+ }
+ field = rmsgpack_dom_value_map_value(&item, &key);
+ if (!field)
+ {
+ rv = -EINVAL;
+ printf("field not found in item\n");
+ goto clean;
+ }
+ if (field->type != RDT_BINARY)
+ {
+ rv = -EINVAL;
+ printf("field is not binary\n");
+ goto clean;
+ }
+
+ if (field->binary.len == 0)
+ {
+ rv = -EINVAL;
+ printf("field is empty\n");
+ goto clean;
+ }
+
+ if (field_size == 0)
+ field_size = field->binary.len;
+ else if (field->binary.len != field_size)
+ {
+ rv = -EINVAL;
+ printf("field is not of correct size\n");
+ goto clean;
+ }
+
+ buff = malloc(field_size + sizeof(uint64_t));
+ if(!buff)
+ {
+ rv = -ENOMEM;
+ goto clean;
+ }
+
+ memcpy(buff, field->binary.buff, field_size);
+ memcpy(buff + field_size, &item_loc, sizeof(uint64_t));
+
+ if (bintree_insert(&tree, buff) != 0)
+ {
+ printf("Value is not unique: ");
+ rmsgpack_dom_value_print(field);
+ printf("\n");
+ rv = -EINVAL;
+ goto clean;
+ }
+ buff = NULL;
+ rmsgpack_dom_value_free(&item);
+ item_loc = rarchdb_tell(db);
+ }
+ idx_header_offset = lseek(db->fd, 0, SEEK_END);
+ strncpy(idx.name, name, 50);
+ idx.name[49] = '\0';
+ idx.key_size = field_size;
+ idx.next = db->count * (field_size + sizeof(uint64_t));
+ rarchdb_write_index_header(db->fd, &idx);
+ struct node_iter_ctx nictx;
+ nictx.db = db;
+ nictx.idx = &idx;
+ bintree_iterate(&tree, node_iter, &nictx);
+ bintree_free(&tree);
+clean:
+ rmsgpack_dom_value_free(&item);
+ if (buff)
+ free(buff);
+ rarchdb_read_reset(db);
+ return 0;
+}
diff --git a/rarchdb/rarchdb.h b/rarchdb/rarchdb.h
new file mode 100644
index 0000000000..74d01740b5
--- /dev/null
+++ b/rarchdb/rarchdb.h
@@ -0,0 +1,28 @@
+#ifndef __RARCHDB_H__
+#define __RARCHDB_H__
+
+#include <stdint.h>
+#include "rmsgpack_dom.h"
+
+struct rarchdb {
+ int fd;
+ int eof;
+ uint64_t root;
+ uint64_t count;
+ uint64_t first_index_offset;
+};
+
+
+typedef int(*rarchdb_value_provider)(void *ctx, struct rmsgpack_dom_value *out);
+
+int rarchdb_create(int fd, rarchdb_value_provider value_provider, void *ctx);
+
+void rarchdb_close(struct rarchdb *db);
+int rarchdb_open(const char *path, struct rarchdb *db);
+
+int rarchdb_read_item(struct rarchdb *db, struct rmsgpack_dom_value *out);
+int rarchdb_read_reset(struct rarchdb *db);
+int rarchdb_create_index(struct rarchdb *db, const char* name, const char *field_name);
+int rarchdb_find_entry(struct rarchdb *db, const char *index_name, const void *key);
+
+#endif
diff --git a/rarchdb/rarchdb_endian.h b/rarchdb/rarchdb_endian.h
new file mode 100644
index 0000000000..ddc313f5bd
--- /dev/null
+++ b/rarchdb/rarchdb_endian.h
@@ -0,0 +1,51 @@
+#ifndef __RARCHDB_MSGPACK_ENDIAN_H
+#define __RARCHDB_MSGPACK_ENDIAN_H
+
+#include <stdint.h>
+
+#ifndef INLINE
+#define INLINE inline
+#endif
+
+static INLINE uint8_t is_little_endian(void)
+{
+ union
+ {
+ uint16_t x;
+ uint8_t y[2];
+ } u;
+
+ u.x = 1;
+ return u.y[0];
+}
+
+#define swap16(val) \
+ ((((uint16_t)(val) & 0x00ff) << 8) \
+ | (((uint16_t)(val) & 0xff00) >> 8))
+
+#define swap32(val) \
+ ((((uint32_t)(val) & 0x000000ff) << 24) \
+ | (((uint32_t)(val) & 0x0000ff00) << 8) \
+ | (((uint32_t)(val) & 0x00ff0000) >> 8) \
+ | (((uint32_t)(val) & 0xff000000) >> 24))
+
+#define swap64(val) \
+ ((((uint64_t)(val) & 0x00000000000000ffULL) << 56) \
+ | (((uint64_t)(val) & 0x000000000000ff00ULL) << 40) \
+ | (((uint64_t)(val) & 0x0000000000ff0000ULL) << 24) \
+ | (((uint64_t)(val) & 0x00000000ff000000ULL) << 8) \
+ | (((uint64_t)(val) & 0x000000ff00000000ULL) >> 8) \
+ | (((uint64_t)(val) & 0x0000ff0000000000ULL) >> 24) \
+ | (((uint64_t)(val) & 0x00ff000000000000ULL) >> 40) \
+ | (((uint64_t)(val) & 0xff00000000000000ULL) >> 56))
+
+
+#define httobe64(x) (is_little_endian() ? swap64(x) : (x))
+#define httobe32(x) (is_little_endian() ? swap32(x) : (x))
+#define httobe16(x) (is_little_endian() ? swap16(x) : (x))
+
+#define betoht16(x) httobe16(x)
+#define betoht32(x) httobe32(x)
+#define betoht64(x) httobe64(x)
+
+#endif
diff --git a/rarchdb/rarchdb_tool.c b/rarchdb/rarchdb_tool.c
new file mode 100644
index 0000000000..fa60448778
--- /dev/null
+++ b/rarchdb/rarchdb_tool.c
@@ -0,0 +1,116 @@
+#include <stdio.h>
+#include <string.h>
+
+#include "rarchdb.h"
+#include "rmsgpack_dom.h"
+
+static int list_db(void)
+{
+ return 0;
+}
+
+int main(int argc, char** argv)
+{
+ int rv;
+ struct rarchdb db;
+ struct rmsgpack_dom_value item;
+ if (argc < 3)
+ {
+ printf("Usage: %s <db file> <command> [extra args...]\n", argv[0]);
+ printf("Available Commands:\n");
+ printf("\tlist\n");
+ printf("\tcreate-index <index name> <field name>\n");
+ printf("\tfind <index name> <value>\n");
+ return 1;
+ }
+
+ const char* command = argv[2];
+ const char* path = argv[1];
+
+ if ((rv = rarchdb_open(path, &db)) != 0)
+ {
+ printf("Could not open db file '%s': %s\n", path, strerror(-rv));
+ return 1;
+ }
+
+ if (strcmp(command, "list") == 0)
+ {
+ if (argc != 3)
+ {
+ printf("Usage: %s <db file> list\n", argv[0]);
+ return 1;
+ }
+ while(rarchdb_read_item(&db, &item) == 0)
+ {
+ rmsgpack_dom_value_print(&item);
+ printf("\n");
+ rmsgpack_dom_value_free(&item);
+ }
+ }
+ else if (strcmp(command, "create-index") == 0)
+ {
+ const char *index_name, *field_name;
+
+ if (argc != 5)
+ {
+ printf("Usage: %s <db file> create-index <index name> <field name>\n", argv[0]);
+ return 1;
+ }
+
+ index_name = argv[3];
+ field_name = argv[4];
+
+ rarchdb_create_index(&db, index_name, field_name);
+ }
+ else if (strcmp(command, "find") == 0)
+ {
+ int i;
+ const char *index_name, *value;
+ char bin_value[256];
+ uint8_t h;
+ uint8_t l;
+
+ if (argc != 5)
+ {
+ printf("Usage: %s <db file> create-index <index name> <field name>\n", argv[0]);
+ return 1;
+ }
+
+ index_name = argv[3];
+ value = argv[4];
+
+ for (i = 0; i < strlen(value); i += 2)
+ {
+ if (value[i] <= '9')
+ h = value[i] - '0';
+ else
+ h = (value[i] - 'A') + 10;
+
+ if (value[i+1] <= '9')
+ l = value[i+1] - '0';
+ else
+ l = (value[i+1] - 'A') + 10;
+
+ bin_value[i/2] = h * 16 + l;
+ }
+
+ if (rarchdb_find_entry(&db, index_name, bin_value) != 0)
+ {
+ printf("Could not find item\n");
+ return 1;
+ }
+
+ if (rarchdb_read_item(&db, &item) == 0)
+ {
+ rmsgpack_dom_value_print(&item);
+ printf("\n");
+ rmsgpack_dom_value_free(&item);
+ }
+ }
+ else
+ {
+ printf("Unkown command %s\n", argv[1]);
+ return 1;
+ }
+ rarchdb_close(&db);
+}
diff --git a/rarchdb/rmsgpack.c b/rarchdb/rmsgpack.c
new file mode 100644
index 0000000000..8960d0ec18
--- /dev/null
+++ b/rarchdb/rmsgpack.c
@@ -0,0 +1,548 @@
+/*
+ * An almost complete implementation of msgpack by
+ * Saggi Mizrahi <ficoos@gmail.com>
+ *
+ * TODO:
+ * - float types
+ * - ext types
+ *
+ * For more information http://msgpack.org/
+ */
+
+#include "rmsgpack.h"
+
+#include <stdlib.h>
+#include <unistd.h>
+#include <errno.h>
+#include <stdint.h>
+#include <string.h>
+
+#include "rarchdb_endian.h"
+
+static const uint8_t MPF_FIXMAP = 0x80;
+static const uint8_t MPF_MAP16 = 0xde;
+static const uint8_t MPF_MAP32 = 0xdf;
+
+static const uint8_t MPF_FIXARRAY = 0x90;
+static const uint8_t MPF_ARRAY16 = 0xdc;
+static const uint8_t MPF_ARRAY32 = 0xdd;
+
+static const uint8_t MPF_FIXSTR = 0xa0;
+static const uint8_t MPF_STR8 = 0xd9;
+static const uint8_t MPF_STR16 = 0xda;
+static const uint8_t MPF_STR32 = 0xdb;
+
+static const uint8_t MPF_BIN8 = 0xc4;
+static const uint8_t MPF_BIN16 = 0xc5;
+static const uint8_t MPF_BIN32 = 0xc6;
+
+static const uint8_t MPF_FALSE = 0xc2;
+static const uint8_t MPF_TRUE = 0xc3;
+
+static const uint8_t MPF_INT8 = 0xd0;
+static const uint8_t MPF_INT16 = 0xd1;
+static const uint8_t MPF_INT32 = 0xd2;
+static const uint8_t MPF_INT64 = 0xd3;
+
+static const uint8_t MPF_UINT8 = 0xcc;
+static const uint8_t MPF_UINT16 = 0xcd;
+static const uint8_t MPF_UINT32 = 0xce;
+static const uint8_t MPF_UINT64 = 0xcf;
+
+static const uint8_t MPF_NIL = 0xc0;
+
+int rmsgpack_write_array_header(int fd, uint32_t size)
+{
+ if (size < 16)
+ {
+ size = (size | MPF_FIXARRAY);
+ if (write(fd, &size, sizeof(int8_t)) == -1)
+ return -errno;
+ return sizeof(int8_t);
+ }
+ else if (size == (uint16_t)size)
+ {
+ if (write(fd, &MPF_ARRAY16, sizeof(MPF_ARRAY16)) == -1)
+ return -errno;
+ if (write(fd, (void *)(&size), sizeof(uint16_t)) == -1)
+ return -errno;
+ return sizeof(int8_t) + sizeof(uint16_t);
+ }
+ else
+ {
+ if (write(fd, &MPF_ARRAY32, sizeof(MPF_ARRAY32)) == -1)
+ return -errno;
+ if (write(fd, (void *)(&size), sizeof(uint32_t)) == -1)
+ return -errno;
+ return sizeof(int8_t) + sizeof(uint32_t);
+ }
+}
+
+int rmsgpack_write_map_header(int fd, uint32_t size)
+{
+ if (size < 16)
+ {
+ size = (size | MPF_FIXMAP);
+ if (write(fd, &size, sizeof(int8_t)) == -1)
+ return -errno;
+ return sizeof(int8_t);
+ }
+ else if (size < (uint16_t)size)
+ {
+ if (write(fd, &MPF_MAP16, sizeof(MPF_MAP16)) == -1)
+ return -errno;
+ if (write(fd, (void *)(&size), sizeof(uint16_t)) == -1)
+ return -errno;
+ return sizeof(int8_t) + sizeof(uint16_t);
+ }
+ else
+ {
+ if (write(fd, &MPF_MAP32, sizeof(MPF_MAP32)) == -1)
+ return -errno;
+ if (write(fd, (void *)(&size), sizeof(uint32_t)) == -1)
+ return -errno;
+ return sizeof(int8_t) + sizeof(uint32_t);
+ }
+}
+
+int rmsgpack_write_string(int fd, const char *s, uint32_t len)
+{
+ int8_t fixlen = 0;
+ uint16_t tmp_i16;
+ uint32_t tmp_i32;
+ int written = sizeof(int8_t);
+ if (len < 32)
+ {
+ fixlen = len | MPF_FIXSTR;
+ if (write(fd, &fixlen, sizeof(int8_t)) == -1)
+ return -errno;
+ }
+ else if (len < 1<<8)
+ {
+ if (write(fd, &MPF_STR8, sizeof(MPF_STR8)) == -1)
+ return -errno;
+ if (write(fd, &len, sizeof(uint8_t)) == -1)
+ return -errno;
+ written += sizeof(uint8_t);
+ }
+ else if (len < 1<<16)
+ {
+ if (write(fd, &MPF_STR16, sizeof(MPF_STR16)) == -1)
+ return -errno;
+ tmp_i16 = httobe16(len);
+ if (write(fd, &tmp_i16, sizeof(uint16_t)) == -1)
+ return -errno;
+ written += sizeof(uint16_t);
+ }
+ else
+ {
+ if (write(fd, &MPF_STR32, sizeof(MPF_STR32)) == -1)
+ return -errno;
+ tmp_i32 = httobe32(len);
+ if (write(fd, &tmp_i32, sizeof(uint32_t)) == -1)
+ return -errno;
+ written += sizeof(uint32_t);
+ }
+ if (write(fd, s, len) == -1)
+ return -errno;
+ written += len;
+ return written;
+}
+
+int rmsgpack_write_bin(int fd, const void *s, uint32_t len)
+{
+ uint16_t tmp_i16;
+ uint32_t tmp_i32;
+ int written = sizeof(int8_t);
+ if (len == (uint8_t)len)
+ {
+ if (write(fd, &MPF_BIN8, sizeof(MPF_BIN8)) == -1)
+ return -errno;
+ if (write(fd, &len, sizeof(uint8_t)) == -1)
+ return -errno;
+ written += sizeof(uint8_t);
+ }
+ else if (len == (uint16_t)len)
+ {
+ if (write(fd, &MPF_BIN16, sizeof(MPF_BIN16)) == -1)
+ return -errno;
+ tmp_i16 = httobe16(len);
+ if (write(fd, &tmp_i16, sizeof(uint16_t)) == -1)
+ return -errno;
+ written += sizeof(uint16_t);
+ }
+ else
+ {
+ if (write(fd, &MPF_BIN32, sizeof(MPF_BIN32)) == -1)
+ return -errno;
+ tmp_i32 = httobe32(len);
+ if (write(fd, &tmp_i32, sizeof(uint32_t)) == -1)
+ return -errno;
+ written += sizeof(uint32_t);
+ }
+ if (write(fd, s, len) == -1)
+ return -errno;
+ written += len;
+ return 0;
+}
+
+int rmsgpack_write_nil(int fd)
+{
+ if (write(fd, &MPF_NIL, sizeof(MPF_NIL)) == -1)
+ return -errno;
+ return sizeof(uint8_t);
+}
+
+int rmsgpack_write_bool(int fd, int value)
+{
+ if (value)
+ {
+ if (write(fd, &MPF_TRUE, sizeof(MPF_TRUE)) == -1)
+ return -errno;
+ }
+ else
+ {
+ if (write(fd, &MPF_FALSE, sizeof(MPF_FALSE)) == -1)
+ return -errno;
+ }
+ return sizeof(uint8_t);
+}
+
+int rmsgpack_write_int(int fd, int64_t value)
+{
+ uint8_t tmpval = 0;
+
+ int16_t tmp_i16;
+ int32_t tmp_i32;
+ int written = sizeof(uint8_t);
+ if (value >=0 && value < 128)
+ {
+ if (write(fd, &value, sizeof(int8_t)) == -1)
+ return -errno;
+ }
+ else if (value < 0 && value > -32)
+ {
+ tmpval = (value) | 0xe0;
+ if (write(fd, &tmpval, sizeof(uint8_t)) == -1)
+ return -errno;
+ }
+ else if (value == (int8_t)value)
+ {
+ if (write(fd, &MPF_INT8, sizeof(MPF_INT8)) == -1)
+ return -errno;
+
+ if (write(fd, &value, sizeof(int8_t)) == -1)
+ return -errno;
+ written += sizeof(int8_t);
+ }
+ else if (value == (int16_t)value)
+ {
+ if (write(fd, &MPF_INT16, sizeof(MPF_INT16)) == -1)
+ return -errno;
+
+ tmp_i16 = httobe16(value);
+ if (write(fd, &tmp_i16, sizeof(int16_t)) == -1)
+ return -errno;
+ written += sizeof(int16_t);
+ }
+ else if (value == (int32_t)value)
+ {
+ if (write(fd, &MPF_INT32, sizeof(MPF_INT32)) == -1)
+ return -errno;
+
+ tmp_i32 = httobe32(value);
+ if (write(fd, &tmp_i32, sizeof(int32_t)) == -1)
+ return -errno;
+ written += sizeof(int32_t);
+ }
+ else
+ {
+ if (write(fd, &MPF_INT64, sizeof(MPF_INT64)) == -1)
+ return -errno;
+
+ value = httobe64(value);
+ if (write(fd, &value, sizeof(int64_t)) == -1)
+ return -errno;
+ written += sizeof(int64_t);
+ }
+ return written;
+}
+
+int rmsgpack_write_uint(int fd, uint64_t value)
+{
+ uint16_t tmp_i16;
+ uint32_t tmp_i32;
+ int written = sizeof(uint8_t);
+
+ if (value == (uint8_t)value)
+ {
+ if (write(fd, &MPF_UINT8, sizeof(MPF_UINT8)) == -1)
+ return -errno;
+
+ if (write(fd, &value, sizeof(uint8_t)) == -1)
+ return -errno;
+ written += sizeof(uint8_t);
+ }
+ else if (value == (uint16_t)value)
+ {
+ if (write(fd, &MPF_UINT16, sizeof(MPF_UINT16)) == -1)
+ return -errno;
+
+ tmp_i16 = httobe16(value);
+ if (write(fd, &tmp_i16, sizeof(uint16_t)) == -1)
+ return -errno;
+ written += sizeof(uint16_t);
+ }
+ else if (value == (uint32_t)value)
+ {
+ if (write(fd, &MPF_UINT32, sizeof(MPF_UINT32)) == -1)
+ return -errno;
+
+ tmp_i32 = httobe32(value);
+ if (write(fd, &tmp_i32, sizeof(uint32_t)) == -1)
+ return -errno;
+ written += sizeof(uint32_t);
+ }
+ else
+ {
+ if (write(fd, &MPF_UINT64, sizeof(MPF_UINT64)) == -1)
+ return -errno;
+
+ value = httobe64(value);
+ if (write(fd, &value, sizeof(uint64_t)) == -1)
+ return -errno;
+ written += sizeof(uint64_t);
+ }
+ return written;
+}
+
+static int read_uint(int fd, uint64_t *out, size_t size)
+{
+ uint64_t tmp;
+ if(read(fd, &tmp, size) == -1)
+ return -errno;
+ switch(size)
+ {
+ case 1:
+ *out = *(uint8_t*)(&tmp);
+ break;
+ case 2:
+ *out = betoht16(tmp);
+ break;
+ case 4:
+ *out = betoht32(tmp);
+ break;
+ case 8:
+ *out = betoht64(tmp);
+ break;
+ }
+ return 0;
+}
+
+static int read_int(int fd, int64_t *out, size_t size)
+{
+ uint64_t tmp;
+ if(read(fd, &tmp, size) == -1)
+ return -errno;
+
+ switch(size)
+ {
+ case 1:
+ *out = *((int8_t*)(&tmp));
+ break;
+ case 2:
+ tmp = betoht16(tmp);
+ *out = *((int16_t*)(&tmp));
+ break;
+ case 4:
+ tmp = betoht32(tmp);
+ *out = *((int32_t*)(&tmp));
+ break;
+ case 8:
+ tmp = betoht64(tmp);
+ *out = *((int64_t*)(&tmp));
+ break;
+ }
+ return 0;
+}
+
+static int read_buff(int fd, size_t size, char** pbuff, uint64_t *len)
+{
+ uint64_t tmp_len;
+ if(read_uint(fd, &tmp_len, size) == -1)
+ return -errno;
+
+ *pbuff = calloc(tmp_len + 1, sizeof(char));
+ if (read(fd, *pbuff, tmp_len) == -1)
+ {
+ free(*pbuff);
+ return -errno;
+ }
+ *len = tmp_len;
+ return 0;
+}
+
+static int read_map(int fd, uint32_t len, struct rmsgpack_read_callbacks *callbacks, void *data)
+{
+ int rv, i;
+ if (
+ callbacks->read_map_start &&
+ (rv = callbacks->read_map_start(len, data)) < 0)
+ return rv;
+
+ for (i = 0; i < len; i++)
+ {
+ if((rv = rmsgpack_read(fd, callbacks, data)) < 0)
+ return rv;
+ if((rv = rmsgpack_read(fd, callbacks, data)) < 0)
+ return rv;
+ }
+
+ return 0;
+}
+
+static int read_array(int fd, uint32_t len, struct rmsgpack_read_callbacks *callbacks, void *data)
+{
+ int rv, i;
+
+ if (
+ callbacks->read_array_start &&
+ (rv = callbacks->read_array_start(len, data)) < 0)
+ return rv;
+
+ for (i = 0; i < len; i++)
+ {
+ if((rv = rmsgpack_read(fd, callbacks, data)) < 0)
+ return rv;
+ }
+
+ return 0;
+}
+
+int rmsgpack_read(int fd, struct rmsgpack_read_callbacks *callbacks, void *data)
+{
+ int rv;
+ uint64_t tmp_len = 0;
+ uint64_t tmp_uint = 0;
+ int64_t tmp_int = 0;
+ uint8_t type = 0;
+ char* buff;
+ if (read(fd, &type, sizeof(uint8_t)) == -1)
+ return -errno;
+
+ if (type < MPF_FIXMAP)
+ {
+ if (!callbacks->read_int)
+ return 0;
+ return callbacks->read_int(type, data);
+ }
+ else if (type < MPF_FIXARRAY)
+ {
+ tmp_len = type - MPF_FIXMAP;
+ return read_map(fd, tmp_len, callbacks, data);
+ }
+ else if (type < MPF_FIXSTR)
+ {
+ tmp_len = type - MPF_FIXARRAY;
+ return read_array(fd, tmp_len, callbacks, data);
+ }
+ else if (type < MPF_NIL)
+ {
+ tmp_len = type - MPF_FIXSTR;
+ buff = calloc(tmp_len + 1, sizeof(char));
+ if (!buff)
+ return -ENOMEM;
+ if(read(fd, buff, tmp_len) == -1)
+ {
+ free(buff);
+ return -errno;
+ }
+ buff[tmp_len] = '\0';
+ if (!callbacks->read_string)
+ {
+ free(buff);
+ return 0;
+ }
+ return callbacks->read_string(buff, tmp_len, data);
+ }
+ else if (type > MPF_MAP32)
+ {
+ if (!callbacks->read_int)
+ return 0;
+ return callbacks->read_int(type - 0xff - 1, data);
+ }
+
+ switch(type)
+ {
+ case 0xc0:
+ if(callbacks->read_nil)
+ return callbacks->read_nil(data);
+ break;
+ case 0xc2:
+ if(callbacks->read_bool)
+ return callbacks->read_bool(0, data);
+ break;
+ case 0xc3:
+ if(callbacks->read_bool)
+ return callbacks->read_bool(1, data);
+ break;
+ case 0xc4:
+ case 0xc5:
+ case 0xc6:
+ if((rv = read_buff(fd, 1<<(type - 0xc4), &buff, &tmp_len)) < 0)
+ return rv;
+
+ if (callbacks->read_bin)
+ return callbacks->read_bin(buff, tmp_len, data);
+ break;
+ case 0xcc:
+ case 0xcd:
+ case 0xce:
+ case 0xcf:
+ tmp_len = 1<<(type - 0xcc);
+ tmp_uint = 0;
+ if(read_uint(fd, &tmp_uint, tmp_len) == -1)
+ return -errno;
+
+ if (callbacks->read_uint)
+ return callbacks->read_uint(tmp_uint, data);
+ break;
+ case 0xd0:
+ case 0xd1:
+ case 0xd2:
+ case 0xd3:
+ tmp_len = 1<<(type - 0xd0);
+ tmp_int = 0;
+ if(read_int(fd, &tmp_int, tmp_len) == -1)
+ return -errno;
+
+ if (callbacks->read_int)
+ return callbacks->read_int(tmp_int, data);
+ break;
+ case 0xd9:
+ case 0xda:
+ case 0xdb:
+ if((rv = read_buff(fd, 1<<(type - 0xd9), &buff, &tmp_len)) < 0)
+ return rv;
+
+ if (callbacks->read_string)
+ return callbacks->read_string(buff, tmp_len, data);
+ break;
+ case 0xdc:
+ case 0xdd:
+ if(read_uint(fd, &tmp_len, 2<<(type - 0xdc)) == -1)
+ return -errno;
+
+ return read_array(fd, tmp_len, callbacks, data);
+ break;
+ case 0xde:
+ case 0xdf:
+ if(read_uint(fd, &tmp_len, 2<<(type - 0xde)) == -1)
+ return -errno;
+
+ return read_map(fd, tmp_len, callbacks, data);
+ break;
+ }
+
+ return 0;
+}
diff --git a/rarchdb/rmsgpack.h b/rarchdb/rmsgpack.h
new file mode 100644
index 0000000000..e2ada18d05
--- /dev/null
+++ b/rarchdb/rmsgpack.h
@@ -0,0 +1,30 @@
+#ifndef __RARCHDB_MSGPACK_H__
+#define __RARCHDB_MSGPACK_H__
+
+#include <stdint.h>
+
+struct rmsgpack_read_callbacks {
+ int(*read_nil)(void*);
+ int(*read_bool)(int, void*);
+ int(*read_int)(int64_t, void*);
+ int(*read_uint)(uint64_t, void*);
+ int(*read_string)(char *, uint32_t, void*);
+ int(*read_bin)(void *, uint32_t, void*);
+ int(*read_map_start)(uint32_t, void*);
+ int(*read_array_start)(uint32_t, void*);
+};
+
+
+int rmsgpack_write_array_header(int fd, uint32_t size);
+int rmsgpack_write_map_header(int fd, uint32_t size);
+int rmsgpack_write_string(int fd, const char *s, uint32_t len);
+int rmsgpack_write_bin(int fd, const void *s, uint32_t len);
+int rmsgpack_write_nil(int fd);
+int rmsgpack_write_bool(int fd, int value);
+int rmsgpack_write_int(int fd, int64_t value);
+int rmsgpack_write_uint(int fd, uint64_t value);
+
+int rmsgpack_read(int fd, struct rmsgpack_read_callbacks *callbacks, void *data);
+
+#endif
+
diff --git a/rarchdb/rmsgpack_dom.c b/rarchdb/rmsgpack_dom.c
new file mode 100644
index 0000000000..f33c286352
--- /dev/null
+++ b/rarchdb/rmsgpack_dom.c
@@ -0,0 +1,462 @@
+#include "rmsgpack_dom.h"
+
+#include <stdint.h>
+#include <stdio.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdarg.h>
+
+#include "rmsgpack.h"
+
+
+#define MAX_DEPTH 128
+struct dom_reader_state
+{
+ int i;
+ struct rmsgpack_dom_value* stack[MAX_DEPTH];
+};
+
+
+static struct rmsgpack_dom_value *dom_reader_state_pop(struct dom_reader_state *s)
+{
+ struct rmsgpack_dom_value *v = s->stack[s->i];
+ s->i--;
+ return v;
+}
+
+static int dom_reader_state_push(struct dom_reader_state *s, struct rmsgpack_dom_value *v)
+{
+ //TODO: Verify we don't overflow
+ s->i++;
+ s->stack[s->i] = v;
+ return 0;
+}
+
+static int dom_read_nil(void *data)
+{
+ struct rmsgpack_dom_value *v = dom_reader_state_pop(data);
+ v->type = RDT_NULL;
+ return 0;
+}
+
+static int dom_read_bool(int value, void *data)
+{
+ struct rmsgpack_dom_value *v = dom_reader_state_pop(data);
+ v->type = RDT_BOOL;
+ v->bool_ = value;
+ return 0;
+}
+
+static int dom_read_int(int64_t value, void *data)
+{
+ struct rmsgpack_dom_value *v = dom_reader_state_pop(data);
+ v->type = RDT_INT;
+ v->int_ = value;
+ return 0;
+}
+
+static int dom_read_uint(uint64_t value, void *data)
+{
+ struct rmsgpack_dom_value *v = dom_reader_state_pop(data);
+ v->type = RDT_UINT;
+ v->uint_ = value;
+ return 0;
+}
+
+static int dom_read_string(char* value, uint32_t len, void *data)
+{
+ struct rmsgpack_dom_value *v = dom_reader_state_pop(data);
+ v->type = RDT_STRING;
+ v->string.len = len;
+ v->string.buff = value;
+ return 0;
+}
+
+static int dom_read_bin(void* value, uint32_t len, void *data)
+{
+ struct rmsgpack_dom_value *v = dom_reader_state_pop(data);
+ v->type = RDT_BINARY;
+ v->binary.len = len;
+ v->binary.buff = value;
+ return 0;
+}
+
+static int dom_read_map_start(uint32_t len, void *data)
+{
+ int i;
+ struct rmsgpack_dom_pair *items;
+ struct rmsgpack_dom_value *v = dom_reader_state_pop(data);
+ v->type = RDT_MAP;
+ v->map.len = len;
+ v->map.items = NULL;
+
+ items = (struct rmsgpack_dom_pair *)calloc(len, sizeof(struct rmsgpack_dom_pair));
+
+ if (!items)
+ return -ENOMEM;
+
+ v->map.items = items;
+ for (i = 0; i < len; i++)
+ {
+ if (dom_reader_state_push(data, &items[i].value) < 0)
+ return -ENOMEM;
+ if (dom_reader_state_push(data, &items[i].key) < 0)
+ return -ENOMEM;
+ }
+ return 0;
+}
+
+static int dom_read_array_start(uint32_t len, void *data)
+{
+ int i;
+ struct rmsgpack_dom_value *v = dom_reader_state_pop(data);
+ struct rmsgpack_dom_value *items;
+ v->type = RDT_ARRAY;
+ v->array.len = len;
+ v->array.items = NULL;
+
+ items = (struct rmsgpack_dom_value*)calloc(len, sizeof(struct rmsgpack_dom_pair));
+
+ if (!items)
+ return -ENOMEM;
+
+ v->array.items = items;
+
+ for (i = 0; i < len; i++)
+ {
+ if (dom_reader_state_push(data, &items[i]) < 0)
+ return -ENOMEM;
+ }
+ return 0;
+}
+
+static struct rmsgpack_read_callbacks dom_reader_callbacks = {
+ dom_read_nil,
+ dom_read_bool,
+ dom_read_int,
+ dom_read_uint,
+ dom_read_string,
+ dom_read_bin,
+ dom_read_map_start,
+ dom_read_array_start
+};
+
+void rmsgpack_dom_value_free(struct rmsgpack_dom_value *v)
+{
+ int i;
+
+ switch (v->type)
+ {
+ case RDT_STRING:
+ free(v->string.buff);
+ break;
+ case RDT_BINARY:
+ free(v->binary.buff);
+ break;
+ case RDT_MAP:
+ for (i = 0; i < v->map.len; i++)
+ {
+ rmsgpack_dom_value_free(&v->map.items[i].key);
+ rmsgpack_dom_value_free(&v->map.items[i].value);
+ }
+ free(v->map.items);
+ break;
+ case RDT_ARRAY:
+ for (i = 0; i < v->array.len; i++)
+ rmsgpack_dom_value_free(&v->array.items[i]);
+ free(v->array.items);
+ break;
+ case RDT_NULL:
+ case RDT_INT:
+ case RDT_BOOL:
+ case RDT_UINT:
+ // Do nothing
+ break;
+ }
+}
+
+struct rmsgpack_dom_value *rmsgpack_dom_value_map_value(
+ const struct rmsgpack_dom_value *map, const struct rmsgpack_dom_value *key)
+{
+ int i;
+ if (map->type != RDT_MAP)
+ return NULL;
+
+ for (i = 0; i < map->map.len; i++)
+ {
+ if (rmsgpack_dom_value_cmp(key, &map->map.items[i].key) == 0)
+ return &map->map.items[i].value;
+ }
+ return NULL;
+}
+
+int rmsgpack_dom_value_cmp(
+ const struct rmsgpack_dom_value *a, const struct rmsgpack_dom_value *b)
+{
+ int rv, i;
+ if (a == b)
+ return 1;
+
+ if (a->type != b->type)
+ return 1;
+
+ switch (a->type)
+ {
+ case RDT_NULL:
+ return 0;
+ break;
+ case RDT_BOOL:
+ return a->bool_ == b->bool_ ? 1 : 0;
+ break;
+ case RDT_INT:
+ return a->int_ == b->int_ ? 1 : 0;
+ break;
+ case RDT_UINT:
+ return a->uint_ == b->uint_ ? 1 : 0;
+ break;
+ case RDT_STRING:
+ if (a->string.len != b->string.len)
+ return 1;
+ return strncmp(a->string.buff, b->string.buff, a->string.len);
+ break;
+ case RDT_BINARY:
+ if (a->binary.len != b->binary.len)
+ return 1;
+ return memcmp(a->binary.buff, b->binary.buff, a->binary.len);
+ break;
+ case RDT_MAP:
+ if (a->map.len != b->map.len)
+ return 1;
+ for (i = 0; i < a->map.len; i++)
+ {
+ if((rv = rmsgpack_dom_value_cmp(&a->map.items[i].key, &b->map.items[i].key)) != 0)
+ return rv;
+ if((rv = rmsgpack_dom_value_cmp(&a->map.items[i].value, &b->map.items[i].value)) != 0)
+ return rv;
+ }
+ break;
+ case RDT_ARRAY:
+ if (a->array.len != b->array.len)
+ return 1;
+ for (i = 0; i < a->array.len; i++)
+ {
+ if((rv = rmsgpack_dom_value_cmp(&a->array.items[i], &b->array.items[i])) != 0)
+ return rv;
+ }
+ break;
+ }
+ return 1;
+}
+
+void rmsgpack_dom_value_print(struct rmsgpack_dom_value *obj)
+{
+ int i;
+
+ switch (obj->type)
+ {
+ case RDT_NULL:
+ printf("nil");
+ break;
+ case RDT_BOOL:
+ if (obj->bool_)
+ printf("true");
+ else
+ printf("false");
+ break;
+ case RDT_INT:
+ printf("%ld", obj->int_);
+ break;
+ case RDT_UINT:
+ printf("%lu", obj->uint_);
+ break;
+ case RDT_STRING:
+ printf("\"%s\"", obj->string.buff);
+ break;
+ case RDT_BINARY:
+ printf("\"");
+ for (i = 0; i < obj->binary.len; i++)
+ printf("%02X", (unsigned char) obj->binary.buff[i]);
+ printf("\"");
+ break;
+ case RDT_MAP:
+ printf("{");
+ for (i = 0; i < obj->map.len; i++)
+ {
+ rmsgpack_dom_value_print(&obj->map.items[i].key);
+ printf(": ");
+ rmsgpack_dom_value_print(&obj->map.items[i].value);
+ if (i < (obj->map.len - 1))
+ printf(", ");
+ }
+ printf("}");
+ break;
+ case RDT_ARRAY:
+ printf("[");
+ for (i = 0; i < obj->array.len; i++)
+ {
+ rmsgpack_dom_value_print(&obj->array.items[i]);
+ if (i < (obj->array.len - 1))
+ printf(", ");
+ }
+ printf("]");
+ }
+}
+int rmsgpack_dom_write(int fd, const struct rmsgpack_dom_value *obj)
+{
+ int i;
+ int rv = 0;
+ int written = 0;
+
+ switch (obj->type)
+ {
+ case RDT_NULL:
+ return rmsgpack_write_nil(fd);
+ break;
+ case RDT_BOOL:
+ return rmsgpack_write_bool(fd, obj->bool_);
+ break;
+ case RDT_INT:
+ return rmsgpack_write_int(fd, obj->int_);
+ break;
+ case RDT_UINT:
+ return rmsgpack_write_uint(fd, obj->uint_);
+ break;
+ case RDT_STRING:
+ return rmsgpack_write_string(fd, obj->string.buff, obj->string.len);
+ break;
+ case RDT_BINARY:
+ return rmsgpack_write_bin(fd, obj->binary.buff, obj->binary.len);
+ break;
+ case RDT_MAP:
+ if ((rv = rmsgpack_write_map_header(fd, obj->map.len)) < 0)
+ return rv;
+ written += rv;
+ for (i = 0; i < obj->map.len; i++)
+ {
+ if((rv = rmsgpack_dom_write(fd, &obj->map.items[i].key)) < 0)
+ return rv;
+ written += rv;
+ if((rv = rmsgpack_dom_write(fd, &obj->map.items[i].value)) < 0)
+ return rv;
+ written += rv;
+ }
+ break;
+ case RDT_ARRAY:
+ if ((rv = rmsgpack_write_array_header(fd, obj->array.len)) < 0)
+ return rv;
+ written += rv;
+ for (i = 0; i < obj->array.len; i++)
+ {
+ if((rv = rmsgpack_dom_write(fd, &obj->array.items[i])) < 0)
+ return rv;
+ written += rv;
+ }
+ }
+ return written;
+}
+
+int rmsgpack_dom_read(int fd, struct rmsgpack_dom_value *out)
+{
+ int rv = 0;
+ struct dom_reader_state s;
+ s.i = 0;
+ s.stack[0] = out;
+ rv = rmsgpack_read(fd, &dom_reader_callbacks, &s);
+
+ if (rv < 0)
+ rmsgpack_dom_value_free(out);
+
+ return rv;
+}
+
+int rmsgpack_dom_read_into(int fd, ...)
+{
+ va_list ap;
+ struct rmsgpack_dom_value map;
+ int rv;
+ const char* key_name;
+ int value_type;
+ struct rmsgpack_dom_value key;
+ struct rmsgpack_dom_value *value;
+ int64_t *int_value;
+ uint64_t *uint_value;
+ int *bool_value;
+ char *buff_value;
+ uint64_t min_len;
+
+ va_start(ap, fd);
+ rv = rmsgpack_dom_read(fd, &map);
+ if (rv < 0)
+ return rv;
+
+ if (map.type != RDT_MAP)
+ {
+ rv = -EINVAL;
+ goto clean;
+ }
+
+ while (1)
+ {
+ key_name = va_arg(ap, const char*);
+ if (key_name == NULL)
+ {
+ rv = 0;
+ goto clean;
+ }
+
+ key.type = RDT_STRING;
+ key.string.len = strlen(key_name);
+ key.string.buff = (char *) key_name;
+
+ value = rmsgpack_dom_value_map_value(&map, &key);
+
+ switch(value->type)
+ {
+ case RDT_INT:
+ int_value = va_arg(ap, int64_t*);
+ *int_value = value->int_;
+ break;
+ case RDT_BOOL:
+ bool_value = va_arg(ap, int*);
+ *bool_value = value->bool_;
+ break;
+ case RDT_UINT:
+ uint_value = va_arg(ap, uint64_t*);
+ *uint_value = value->uint_;
+ break;
+ case RDT_BINARY:
+ buff_value = va_arg(ap, char*);
+ uint_value = va_arg(ap, uint64_t*);
+ *uint_value = value->binary.len;
+ min_len = value->binary.len > *uint_value ? *uint_value : value->binary.len;
+ memcpy(
+ buff_value,
+ value->binary.buff,
+ min_len
+ );
+
+ break;
+ case RDT_STRING:
+ buff_value = va_arg(ap, char*);
+ uint_value = va_arg(ap, uint64_t*);
+ min_len = value->string.len + 1 > *uint_value ? *uint_value : value->string.len + 1;
+ *uint_value = min_len;
+ memcpy(
+ buff_value,
+ value->string.buff,
+ min_len
+ );
+ break;
+ default:
+ rv = -1;
+ goto clean;
+ }
+ }
+
+clean:
+ va_end(ap);
+ rmsgpack_dom_value_free(&map);
+ return 0;
+}
+
diff --git a/rarchdb/rmsgpack_dom.h b/rarchdb/rmsgpack_dom.h
new file mode 100644
index 0000000000..b843326786
--- /dev/null
+++ b/rarchdb/rmsgpack_dom.h
@@ -0,0 +1,56 @@
+#ifndef __RARCHDB_MSGPACK_DOM_H__
+#define __RARCHDB_MSGPACK_DOM_H__
+
+#include <stdint.h>
+
+enum rmsgpack_dom_type {
+ RDT_NULL = 0,
+ RDT_BOOL,
+ RDT_UINT,
+ RDT_INT,
+ RDT_STRING,
+ RDT_BINARY,
+ RDT_MAP,
+ RDT_ARRAY
+};
+
+struct rmsgpack_dom_value {
+ enum rmsgpack_dom_type type;
+ union {
+ uint64_t uint_;
+ int64_t int_;
+ struct {
+ uint32_t len;
+ char* buff;
+ } string;
+ struct {
+ uint32_t len;
+ char* buff;
+ } binary;
+ int bool_;
+ struct {
+ uint32_t len;
+ struct rmsgpack_dom_pair *items;
+ } map;
+ struct {
+ uint32_t len;
+ struct rmsgpack_dom_value *items;
+ } array;
+ };
+};
+
+struct rmsgpack_dom_pair {
+ struct rmsgpack_dom_value key;
+ struct rmsgpack_dom_value value;
+};
+
+void rmsgpack_dom_value_print(struct rmsgpack_dom_value *obj);
+void rmsgpack_dom_value_free(struct rmsgpack_dom_value *v);
+int rmsgpack_dom_value_cmp(const struct rmsgpack_dom_value *a, const struct rmsgpack_dom_value *b);
+
+struct rmsgpack_dom_value *rmsgpack_dom_value_map_value(const struct rmsgpack_dom_value *map, const struct rmsgpack_dom_value *key);
+
+int rmsgpack_dom_read(int fd, struct rmsgpack_dom_value *out);
+int rmsgpack_dom_write(int fd, const struct rmsgpack_dom_value *obj);
+int rmsgpack_dom_read_into(int fd, ...);
+#endif
diff --git a/rarchdb/rmsgpack_test.c b/rarchdb/rmsgpack_test.c
new file mode 100644
index 0000000000..fab8e52498
--- /dev/null
+++ b/rarchdb/rmsgpack_test.c
@@ -0,0 +1,191 @@
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "rmsgpack.h"
+
+struct stub_state
+{
+ int i;
+ uint64_t stack[256];
+};
+
+static void stub_state_push_map(struct stub_state *s, uint32_t size)
+{
+ s->i++;
+ s->stack[s->i] = 1;
+ s->i++;
+ s->stack[s->i] = size * 2;
+ printf("{");
+}
+
+static void stub_state_push_array(struct stub_state *s, uint32_t size)
+{
+ s->i++;
+ s->stack[s->i] = 2;
+ s->i++;
+ s->stack[s->i] = size;
+ printf("[");
+}
+
+static void stub_state_pre_print(struct stub_state *s)
+{
+}
+
+static void stub_state_post_print(struct stub_state *s)
+{
+ switch(s->stack[s->i - 1])
+ {
+ case 1:
+ if (s->stack[s->i] % 2 == 0)
+ {
+ printf(": ");
+ s->stack[s->i]--;
+ }
+ else if (s->stack[s->i] == 1)
+ {
+ printf("}");
+ s->i -= 2;
+ stub_state_post_print(s);
+ }
+ else
+ {
+ printf(", ");
+ s->stack[s->i]--;
+ }
+ break;
+ case 2:
+ if (s->stack[s->i] == 1)
+ {
+ printf("]");
+ s->i -= 2;
+ stub_state_post_print(s);
+ }
+ else
+ {
+ printf(", ");
+ s->stack[s->i]--;
+ }
+ break;
+ }
+
+}
+
+static int stub_read_map_start(uint32_t size, void* data)
+{
+ stub_state_push_map(data, size);
+ return 0;
+}
+
+static int stub_read_array_start(uint32_t size, void* data)
+{
+ stub_state_push_array(data, size);
+ return 0;
+}
+
+static int stub_read_string(char *s, uint32_t len, void* data)
+{
+ stub_state_pre_print(data);
+ printf("'%s'", s);
+ stub_state_post_print(data);
+ free(s);
+ return 0;
+}
+
+static int stub_read_bin(void *s, uint32_t len, void* data)
+{
+ stub_state_pre_print(data);
+ printf("b'%s'", s);
+ stub_state_post_print(data);
+ free(s);
+ return 0;
+}
+
+static int stub_read_uint(uint64_t value, void* data)
+{
+ stub_state_pre_print(data);
+ printf("%lu", value);
+ stub_state_post_print(data);
+ return 0;
+}
+
+static int stub_read_nil(void* data)
+{
+ stub_state_pre_print(data);
+ printf("nil");
+ stub_state_post_print(data);
+ return 0;
+}
+static int stub_read_int(int64_t value, void* data)
+{
+ stub_state_pre_print(data);
+ printf("%ld", value);
+ stub_state_post_print(data);
+ return 0;
+}
+
+static int stub_read_bool(int value, void* data)
+{
+ stub_state_pre_print(data);
+ if (value)
+ printf("true");
+ else
+ printf("false");
+ stub_state_post_print(data);
+ return 0;
+}
+
+static struct rmsgpack_read_callbacks stub_callbacks = {
+ stub_read_nil,
+ stub_read_bool,
+ stub_read_int,
+ stub_read_uint,
+ stub_read_string,
+ stub_read_bin,
+ stub_read_map_start,
+ stub_read_array_start
+};
+
+int main(void)
+{
+ int fd;
+ /*
+ int fd = open("test.msgpack", O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
+ int rv = 0;
+ if (fd == -1)
+ {
+ printf("Could not open file: %s", strerror(errno));
+ return errno;
+ }
+ rmsgpack_write_map_header(fd, 2);
+ rmsgpack_write_string(fd, "compact", strlen("compact"));
+ rmsgpack_write_bool(fd, 1);
+ rmsgpack_write_string(fd, "schema", strlen("schema"));
+ rmsgpack_write_array_header(fd, 10);
+ rmsgpack_write_string(fd, "schema", strlen("schema"));
+ rmsgpack_write_uint(fd, 1<<17);
+ rmsgpack_write_int(fd, (1<<17) + 1);
+ rmsgpack_write_int(fd, 4);
+ rmsgpack_write_int(fd, -3);
+ rmsgpack_write_int(fd, -22);
+ rmsgpack_write_int(fd, -35);
+ rmsgpack_write_int(fd, -421421412);
+ rmsgpack_write_int(fd, 4214);
+ rmsgpack_write_int(fd, -4214);
+ rmsgpack_write_uint(fd, 1<<17);
+ close(fd);
+ */
+
+ struct stub_state state;
+ state.i = 0;
+ state.stack[0] = 0;
+ fd = open("test.msgpack", O_RDONLY);
+ rmsgpack_read(fd, &stub_callbacks, &state);
+ printf("\n");
+ close(fd);
+
+ return 0;
+}