From c16278f027ff238bfcbb6ef25e33971fb46d5fe3 Mon Sep 17 00:00:00 2001 From: twinaphex Date: Tue, 30 Dec 2014 21:02:15 +0100 Subject: [PATCH] Add rarchdb --- rarchdb/.ycm_extra_conf.py | 115 ++++++++ rarchdb/Makefile | 31 +++ rarchdb/README.md | 13 + rarchdb/bintree.c | 119 ++++++++ rarchdb/bintree.h | 27 ++ rarchdb/dat_converter.c | 291 ++++++++++++++++++++ rarchdb/db_parser.c | 80 ++++++ rarchdb/db_parser.h | 15 + rarchdb/mkdb.sh | 3 + rarchdb/rarchdb.c | 372 +++++++++++++++++++++++++ rarchdb/rarchdb.h | 28 ++ rarchdb/rarchdb_endian.h | 51 ++++ rarchdb/rarchdb_tool.c | 116 ++++++++ rarchdb/rmsgpack.c | 548 +++++++++++++++++++++++++++++++++++++ rarchdb/rmsgpack.h | 30 ++ rarchdb/rmsgpack_dom.c | 462 +++++++++++++++++++++++++++++++ rarchdb/rmsgpack_dom.h | 56 ++++ rarchdb/rmsgpack_test.c | 191 +++++++++++++ 18 files changed, 2548 insertions(+) create mode 100644 rarchdb/.ycm_extra_conf.py create mode 100644 rarchdb/Makefile create mode 100644 rarchdb/README.md create mode 100644 rarchdb/bintree.c create mode 100644 rarchdb/bintree.h create mode 100644 rarchdb/dat_converter.c create mode 100644 rarchdb/db_parser.c create mode 100644 rarchdb/db_parser.h create mode 100755 rarchdb/mkdb.sh create mode 100644 rarchdb/rarchdb.c create mode 100644 rarchdb/rarchdb.h create mode 100644 rarchdb/rarchdb_endian.h create mode 100644 rarchdb/rarchdb_tool.c create mode 100644 rarchdb/rmsgpack.c create mode 100644 rarchdb/rmsgpack.h create mode 100644 rarchdb/rmsgpack_dom.c create mode 100644 rarchdb/rmsgpack_dom.h create mode 100644 rarchdb/rmsgpack_test.c diff --git a/rarchdb/.ycm_extra_conf.py b/rarchdb/.ycm_extra_conf.py new file mode 100644 index 0000000000..587f7d6b3e --- /dev/null +++ b/rarchdb/.ycm_extra_conf.py @@ -0,0 +1,115 @@ +import os +import ycm_core +# Defaults, if no database can be found. +defaultsc = [ + 'gcc', + '-Wno-long-long', + '-Wno-variadic-macros', + '-pthread' + '-std=c99', + ] +defaultscpp = [ + 'c' + '-Wno-long-long', + '-Wno-variadic-macros', + '-pthread' + '-std=c99', + ] + +# Things that must be included. +entered_flags = ['-fdelayed-template-parsing'] + +# +def DirectoryOfThisScript(): + return os.path.dirname( os.path.abspath( __file__ ) ) + +# Find all compilation databases in the build directory and load them. + +def MakeRelativePathsInFlagsAbsolute( flags, working_directory ): + if not working_directory: + return list( flags ) + new_flags = [] + make_next_absolute = False + path_flags = [ '-isystem', '-I', '-iquote', '--sysroot=' ] + for flag in flags: + new_flag = flag + + if make_next_absolute: + make_next_absolute = False + if not flag.startswith( '/' ): + new_flag = os.path.join( working_directory, flag ) + + for path_flag in path_flags: + if flag == path_flag: + make_next_absolute = True + break + + if flag.startswith( path_flag ): + path = flag[ len( path_flag ): ] + new_flag = path_flag + os.path.join( working_directory, path ) + break + + if new_flag: + new_flags.append( new_flag ) + return new_flags + + +def FlagsForFile( filename ): + # Search through all parent directories for a directory named 'build' + databases = [] + path_focus = os.path.dirname(filename) + while len(path_focus) > 1: + for f in os.listdir(path_focus): + compilation_database_folder = path_focus + "/" + f + for r,d,f in os.walk(compilation_database_folder): + for files in f: + if files == 'compile_commands.json': + databases += [ycm_core.CompilationDatabase( r )] + path_focus = os.path.dirname(os.path.dirname(path_focus)) + + # Use a header's source file database for completion. + filetype_flags = [] + if filename.endswith(".h"): + for f in os.listdir(os.path.dirname(filename)): + if filename.replace(".h",".cpp").find(f) != -1: + filename = filename.replace(".h",".cpp") + break + if filename.replace(".h",".c").find(f) != -1: + filename = filename.replace(".h",".c") + break + elif filename.endswith(".hpp"): + for f in os.listdir(os.path.dirname(filename)): + if filename.replace(".hpp",".cpp").find(f) != -1: + filename = filename.replace(".hpp",".cpp") + break + + # Get the compile commands + final_flags = [] + # If possible, from the database. + if len(databases) > 0: + for database in databases: + compilation_info = database.GetCompilationInfoForFile( filename ) + fromfile_flags = MakeRelativePathsInFlagsAbsolute( + compilation_info.compiler_flags_, + compilation_info.compiler_working_dir_ ) + final_flags += fromfile_flags + + # If not, set some sane defaults. + else: + relative_to = DirectoryOfThisScript() + final_flags += MakeRelativePathsInFlagsAbsolute( flags, relative_to ) + if not final_flags: + if filename.endswith(".c"): + final_flags = defaultsc + elif filename.endswith(".cpp"): + final_flags = defaultscpp + + # This allows header files to be parsed according to their parent source + final_flags = filetype_flags + final_flags + + # For things that must be included regardless: + final_flags += entered_flags + return { + 'flags': final_flags, + 'do_cache': True + } diff --git a/rarchdb/Makefile b/rarchdb/Makefile new file mode 100644 index 0000000000..ee5cb93760 --- /dev/null +++ b/rarchdb/Makefile @@ -0,0 +1,31 @@ +CFLAGS=-std=c99 -g +DAT_CONVERTER_OBJ = rmsgpack.o \ + rmsgpack_dom.o \ + rarchdb.o \ + bintree.o \ + db_parser.o \ + dat_converter.o \ + $(NULL) +RARCHDB_TOOL_OBJ = rmsgpack.o \ + rmsgpack_dom.o \ + db_parser.o \ + rarchdb_tool.o \ + bintree.o \ + rarchdb.o \ + $(NULL) + +all: dat_converter rmsgpack_test rarchdb_tool + +%.o: %.c + ${CC} $< -c ${CFLAGS} -o $@ + +dat_converter: ${DAT_CONVERTER_OBJ} + ${CC} ${DAT_CONVERTER_OBJ} -o $@ + +rarchdb_tool: ${RARCHDB_TOOL_OBJ} + ${CC} ${RARCHDB_TOOL_OBJ} -o $@ + +rmsgpack_test: + gcc rmsgpack.c rmsgpack_test.c -std=c99 -g -o $@ +clean: + rm -rf *.o rmsgpack_test dat_converter rarchdb_tool diff --git a/rarchdb/README.md b/rarchdb/README.md new file mode 100644 index 0000000000..464996dbd7 --- /dev/null +++ b/rarchdb/README.md @@ -0,0 +1,13 @@ +# rarchdb +A small read only database +Mainly to be used by retroarch + +# Usage +To convert a dat file use `dat_converter ` + +To list out the content of a db `rarchdb_tool list` +To create an index `rarchdb_tool create-index ` +To find an entry with an index `rarchdb_tool find ` + +The util `mkdb.sh ` will create a db file with indexes for crc sha1 and md5 + diff --git a/rarchdb/bintree.c b/rarchdb/bintree.c new file mode 100644 index 0000000000..653f48cd29 --- /dev/null +++ b/rarchdb/bintree.c @@ -0,0 +1,119 @@ +#include +#include +#include + +#include "bintree.h" + +static int NIL_VALUE = 1; +static void* NIL_NODE = &NIL_NODE; + +static struct bintree_node *new_nil_node(struct bintree_node *parent); + +void bintree_new(struct bintree *t, bintree_cmp_func cmp, void *ctx) +{ + t->root = new_nil_node(NULL); + t->cmp = cmp; + t->ctx = ctx; +} + +static struct bintree_node *new_nil_node(struct bintree_node *parent) +{ + struct bintree_node *node = calloc(1, sizeof(struct bintree_node)); + if (!node) + return NULL; + node->value = NIL_NODE; + node->parent = parent; + node->left = NULL; + node->right = NULL; + return node; +} + +static inline int is_nil(const struct bintree_node *node) +{ + return node == NULL || node->value == NIL_NODE; +} + +static int insert(struct bintree *t, struct bintree_node *root, void *value) +{ + int cmp_res = 0; + if (is_nil(root)) + { + root->left = new_nil_node(root); + root->right = new_nil_node(root); + if (!root->left || !root->right) + { + if (root->left) + { + free(root->left); + root->left = NULL; + } + if (root->right) + { + free(root->right); + root->right = NULL; + } + return -ENOMEM; + } + root->value = value; + } + else + { + cmp_res = t->cmp(root->value, value, t->ctx); + if (cmp_res > 0) + return insert(t, root->left, value); + else if (cmp_res < 0) + return insert(t, root->right, value); + else + return -EINVAL; + } + return 0; +} + +int bintree_insert(struct bintree *t, void *value) +{ + return insert(t, t->root, value); +} + +static int _bintree_iterate(struct bintree_node *n, bintree_iter_cb cb, void *ctx) +{ + int rv; + if (is_nil(n)) + return 0; + else + { + if((rv = _bintree_iterate(n->left, cb, ctx)) != 0) + return rv; + if ((rv = cb(n->value, ctx)) != 0) + return rv; + if((rv = _bintree_iterate(n->right, cb, ctx)) != 0) + return rv; + } + + return 0; +} + +int bintree_iterate(const struct bintree *t, bintree_iter_cb cb, void *ctx) +{ + return _bintree_iterate(t->root, cb, ctx); +} + +static void bintree_free_node(struct bintree_node *n) +{ + if (n == NULL) + return; + + if (n->value == NIL_NODE) + { + free(n); + return; + } + n->value = NULL; + bintree_free_node(n->left); + bintree_free_node(n->right); + free(n); +} + +void bintree_free(struct bintree *t) +{ + bintree_free_node(t->root); +} diff --git a/rarchdb/bintree.h b/rarchdb/bintree.h new file mode 100644 index 0000000000..969726b177 --- /dev/null +++ b/rarchdb/bintree.h @@ -0,0 +1,27 @@ +#ifndef __RARCHDB_BINTREE_H__ +#define __RARCHDB_BINTREE_H__ + +typedef int(*bintree_cmp_func)(const void *a, const void *b, void* ctx); + +typedef int(*bintree_iter_cb)(void *value, void* ctx); + + +struct bintree_node { + void* value; + struct bintree_node *parent; + struct bintree_node *left; + struct bintree_node *right; +}; + +struct bintree { + struct bintree_node *root; + bintree_cmp_func cmp; + void *ctx; +}; + +void bintree_new(struct bintree *t, bintree_cmp_func cmp, void *ctx); +int bintree_insert(struct bintree *t, void *value); +int bintree_iterate(const struct bintree *t, bintree_iter_cb cb, void* ctx); +void bintree_free(struct bintree *t); + +#endif diff --git a/rarchdb/dat_converter.c b/rarchdb/dat_converter.c new file mode 100644 index 0000000000..b3b849c2f3 --- /dev/null +++ b/rarchdb/dat_converter.c @@ -0,0 +1,291 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rarchdb.h" +#include "db_parser.h" + +#define MAX_TOKEN 256 + +static char *strndup_(const char *s, size_t n) +{ + char* buff = calloc(n, sizeof(char)); + + if (!buff) + return 0; + + strncpy(buff, s, n); + return buff; +} + +static struct rmsgpack_dom_value *get_map_value(const struct rmsgpack_dom_value *m, char* key) +{ + struct rmsgpack_dom_value v; + + v.type = RDT_STRING; + v.string.len = strlen(key); + v.string.buff = key; + return rmsgpack_dom_value_map_value(m, &v); +} + +static int load_string(int fd, struct rmsgpack_dom_value *out) +{ + char tok[MAX_TOKEN]; + ssize_t tok_size; + + if ((tok_size = get_token(fd, tok, MAX_TOKEN)) < 0) + return tok_size; + + out->type = RDT_STRING; + out->string.len = tok_size; + out->string.buff = strndup_(tok, tok_size); + return 0; +} + +static int load_uint(int fd, struct rmsgpack_dom_value *out) +{ + char tok[MAX_TOKEN], *c; + ssize_t tok_size; + uint64_t value = 0; + + if ((tok_size = get_token(fd, tok, MAX_TOKEN)) < 0) + return tok_size; + + for (c = tok; c < tok + tok_size; c++) + { + value *= 10; + value += *c - '0'; + } + + out->type = RDT_UINT; + out->uint_ = value; + return 0; +} + +static int load_bin(int fd, struct rmsgpack_dom_value *out) +{ + char tok[MAX_TOKEN]; + ssize_t tok_size; + uint8_t h; + uint8_t l; + int i; + + if ((tok_size = get_token(fd, tok, MAX_TOKEN)) < 0) + return tok_size; + + out->type = RDT_BINARY; + out->binary.len = tok_size / 2; + + for (i = 0; i < tok_size; i += 2) + { + if (tok[i] <= '9') + h = tok[i] - '0'; + else + h = (tok[i] - 'A') + 10; + if (tok[i+1] <= '9') + l = tok[i+1] - '0'; + else + l = (tok[i+1] - 'A') + 10; + tok[i/2] = h * 16 + l; + } + + out->binary.buff = malloc(out->binary.len); + memcpy(out->binary.buff, tok, out->binary.len); + return 0; +} + +static int dat_value_provider(void *ctx, struct rmsgpack_dom_value *out) +{ + int rv, i; + static const int field_count = 22; + int fd = *((int*)ctx); + char* key; + + out->type = RDT_MAP; + out->map.len = field_count; + out->map.items = calloc(field_count, sizeof(struct rmsgpack_dom_pair)); + + if (find_token(fd, "game") < 0) + return 1; + + for (i = 0; i < field_count; i++) + { + if ((rv = load_string(fd, &out->map.items[i].key)) < 0) + goto failed; + + key = out->map.items[i].key.string.buff; + + if (strncmp(key, "name", sizeof("name")) == 0) + { + if ((rv = load_string(fd, &out->map.items[i].value)) < 0) + goto failed; + } + else if (strncmp(key, "description", sizeof("description")) == 0) + { + if ((rv = load_string(fd, &out->map.items[i].value)) < 0) + goto failed; + } + else if (strncmp(key, "users", sizeof("users")) == 0) + { + if ((rv = load_uint(fd, &out->map.items[i].value)) < 0) + goto failed; + } + else if (strncmp(key, "releasemonth", sizeof("releasemonth")) == 0) + { + if ((rv = load_uint(fd, &out->map.items[i].value)) < 0) + goto failed; + } + else if (strncmp(key, "releaseyear", sizeof("releaseyear")) == 0) + { + if ((rv = load_uint(fd, &out->map.items[i].value)) < 0) + goto failed; + } + else if (strncmp(key, "rumble", sizeof("rumble")) == 0) + { + if ((rv = load_uint(fd, &out->map.items[i].value)) < 0) + goto failed; + } + else if (strncmp(key, "analog", sizeof("analog")) == 0) + { + if ((rv = load_uint(fd, &out->map.items[i].value)) < 0) + goto failed; + } + else if (strncmp(key, "serial", sizeof("serial")) == 0) + { + if ((rv = load_string(fd, &out->map.items[i].value)) < 0) + goto failed; + } + else if (strncmp(key, "esrb_rating", sizeof("esrb_rating")) == 0) + { + if ((rv = load_string(fd, &out->map.items[i].value)) < 0) + goto failed; + } + else if (strncmp(key, "elspa_rating", sizeof("elspa_rating")) == 0) + { + if ((rv = load_string(fd, &out->map.items[i].value)) < 0) + goto failed; + } + else if (strncmp(key, "pegi_rating", sizeof("pegi_rating")) == 0) + { + if ((rv = load_string(fd, &out->map.items[i].value)) < 0) + goto failed; + } + else if (strncmp(key, "cero_rating", sizeof("cero_rating")) == 0) + { + if ((rv = load_string(fd, &out->map.items[i].value)) < 0) + goto failed; + } + else if (strncmp(key, "developers", sizeof("developers")) == 0) + { + if ((rv = load_string(fd, &out->map.items[i].value)) < 0) + goto failed; + } + else if (strncmp(key, "publisher", sizeof("publisher")) == 0) + { + if ((rv = load_string(fd, &out->map.items[i].value)) < 0) + goto failed; + } + else if (strncmp(key, "origin", sizeof("origin")) == 0) + { + if ((rv = load_string(fd, &out->map.items[i].value)) < 0) + goto failed; + } + else if (strncmp(key, "rom", sizeof("rom")) == 0) + { + if (find_token(fd, "name") < 0) + goto failed; + + if ((rv = load_string(fd, &out->map.items[i].value)) < 0) + goto failed; + } + else if (strncmp(key, "size", sizeof("size")) == 0) + { + if ((rv = load_uint(fd, &out->map.items[i].value)) < 0) + goto failed; + } + else if (strncmp(key, "sha1", sizeof("sha1")) == 0) + { + if ((rv = load_bin(fd, &out->map.items[i].value)) < 0) + goto failed; + } + else if (strncmp(key, "crc", sizeof("crc")) == 0) + { + if ((rv = load_bin(fd, &out->map.items[i].value)) < 0) + goto failed; + } + else if (strncmp(key, "md5", sizeof("md5")) == 0) + { + if ((rv = load_bin(fd, &out->map.items[i].value)) < 0) + goto failed; + } + else if (strncmp(key, "serial", sizeof("serial")) == 0) + { + if ((rv = load_string(fd, &out->map.items[i].value)) < 0) + goto failed; + } + else if (strncmp(key, ")", sizeof(")")) == 0) + { + rmsgpack_dom_value_free(&out->map.items[i].key); + out->map.len = i; + printf("Couldn't find all fields for item\n"); + break; + } + else + { + rmsgpack_dom_value_free(&out->map.items[i].key); + i--; + } + } + printf("Inserting '%s' (%02X%02X%02X%02X)...\n", + get_map_value(out, "name")->string.buff, + (unsigned char)get_map_value(out, "crc")->binary.buff[0], + (unsigned char)get_map_value(out, "crc")->binary.buff[1], + (unsigned char)get_map_value(out, "crc")->binary.buff[2], + (unsigned char)get_map_value(out, "crc")->binary.buff[3] + ); + return 0; + +failed: + rmsgpack_dom_value_free(out); + out->type = RDT_NULL; + return rv; +} + +int main(int argc, char **argv) +{ + int rv = 0; + int src = -1; + int dst = -1; + if (argc != 3) + printf("Usage: %s \n", argv[0]); + + src = open(argv[1], O_RDONLY); + if (src == -1) + { + printf("Could not open source file '%s': %s\n", argv[1], strerror(errno)); + rv = errno; + goto clean; + } + + dst = open(argv[2], O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + if (dst == -1) + { + printf("Could not open destination file '%s': %s\n", argv[1], strerror(errno)); + rv = errno; + goto clean; + } + + rv = rarchdb_create(dst, &dat_value_provider, &src); + +clean: + if (src != -1) + close(src); + if (dst != -1) + close(dst); + return rv; +} diff --git a/rarchdb/db_parser.c b/rarchdb/db_parser.c new file mode 100644 index 0000000000..4da31a4426 --- /dev/null +++ b/rarchdb/db_parser.c @@ -0,0 +1,80 @@ +#include "db_parser.h" + +#include +#include +#include + +ssize_t get_token(int fd, char *token, size_t max_len) +{ + char *c = token; + int rv; + ssize_t len = 0; + int in_string = 0; + + while (1) + { + rv = read(fd, c, 1); + if (rv == 0) + return 0; + else if (rv < 1) + { + switch (errno) + { + case EINTR: + case EAGAIN: + continue; + default: + return -errno; + } + } + + switch (*c) + { + case ' ': + case '\t': + case '\r': + case '\n': + if (c == token) + continue; + + if (!in_string) + { + *c = '\0'; + return len; + } + break; + case '\"': + if (c == token) + { + in_string = 1; + continue; + } + + *c = '\0'; + return len; + } + + len++; + c++; + if (len == (ssize_t)max_len) + { + *c = '\0'; + return len; + } + } +} + +int find_token(int fd, const char *token) +{ + int tmp_len = strlen(token); + char *tmp_token = (char*)calloc(tmp_len, 1); + if (!tmp_token) + return -1; + while (strncmp(tmp_token, token, tmp_len) != 0) + { + if (get_token(fd, tmp_token, tmp_len) <= 0) + return -1; + } + + return 0; +} diff --git a/rarchdb/db_parser.h b/rarchdb/db_parser.h new file mode 100644 index 0000000000..eb87753a35 --- /dev/null +++ b/rarchdb/db_parser.h @@ -0,0 +1,15 @@ +#ifndef __RARCHDB_PARSER_H +#define __RARCHDB_PARSER_H + +#ifdef _WIN32 +#include +#else +#include +#endif + +#define MAX_TOKEN_LEN 255 + +ssize_t get_token(int fd, char* token, size_t max_len); +int find_token(int fd, const char* token); + +#endif diff --git a/rarchdb/mkdb.sh b/rarchdb/mkdb.sh new file mode 100755 index 0000000000..007b00f1d5 --- /dev/null +++ b/rarchdb/mkdb.sh @@ -0,0 +1,3 @@ +#!/bin/sh +./dat_converter "$1" "$2" && ./rarchdb_tool "$2" create-index crc crc && ./rarchdb_tool "$2" create-index md5 md5 && ./rarchdb_tool "$2" create-index sha1 sha1 + diff --git a/rarchdb/rarchdb.c b/rarchdb/rarchdb.c new file mode 100644 index 0000000000..c8151db0c3 --- /dev/null +++ b/rarchdb/rarchdb.c @@ -0,0 +1,372 @@ +#include "rarchdb.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "rmsgpack_dom.h" +#include "rmsgpack.h" +#include "bintree.h" +#include "rarchdb_endian.h" + +#define MAGIC_NUMBER "RARCHDB" + +struct rarchdb_header +{ + char magic_number[7]; + uint64_t metadata_offset; +}; + +struct rarchdb_metadata +{ + uint64_t count; +}; + +struct rarchdb_index +{ + char name[50]; + uint64_t key_size; + uint64_t next; +}; + +struct node_iter_ctx +{ + struct rarchdb *db; + struct rarchdb_index *idx; +}; + +static struct rmsgpack_dom_value sentinal; + +static int rarchdb_read_metadata(int fd, struct rarchdb_metadata *md) +{ + return rmsgpack_dom_read_into(fd, "count", &md->count, NULL); +} + +static int rarchdb_write_metadata(int fd, struct rarchdb_metadata *md) +{ + rmsgpack_write_map_header(fd, 1); + rmsgpack_write_string(fd, "count", strlen("count")); + return rmsgpack_write_uint(fd, md->count); +} + +int rarchdb_create(int fd, rarchdb_value_provider value_provider, void *ctx) +{ + int rv; + uint64_t item_count = 0; + struct rmsgpack_dom_value item = {}; + struct rarchdb_header header = {}; + struct rarchdb_metadata md; + strcpy(header.magic_number, MAGIC_NUMBER); + off_t root = lseek(fd, 0, SEEK_CUR); + // We write the header in the end because we need to know the size of + // the db first + lseek(fd, sizeof(struct rarchdb_header), SEEK_CUR); + while ((rv = value_provider(ctx, &item)) == 0) + { + if((rv = rmsgpack_dom_write(fd, &item)) < 0) + return rv; + rmsgpack_dom_value_free(&item); + item_count++; + } + + if (rv < 0) + return rv; + + if((rv = rmsgpack_dom_write(fd, &sentinal)) < 0) + return rv; + + header.metadata_offset = httobe64(lseek(fd, 0, SEEK_CUR)); + md.count = item_count; + rarchdb_write_metadata(fd, &md); + lseek(fd, root, SEEK_SET); + write(fd, &header, sizeof(header)); + printf("Created DB with %lu entries\n", item_count); + return 0; +} + +static int rarchdb_read_index_header(int fd, struct rarchdb_index *idx) +{ + uint64_t name_len = 50; + return rmsgpack_dom_read_into(fd, + "name", idx->name, &name_len, + "key_size", &idx->key_size, + "next", &idx->next, + NULL); +} + +static void rarchdb_write_index_header(int fd, struct rarchdb_index *idx) +{ + rmsgpack_write_map_header(fd, 3); + rmsgpack_write_string(fd, "name", strlen("name")); + rmsgpack_write_string(fd, idx->name, strlen(idx->name)); + rmsgpack_write_string(fd, "key_size", strlen("key_size")); + rmsgpack_write_uint(fd, idx->key_size); + rmsgpack_write_string(fd, "next", strlen("next")); + rmsgpack_write_uint(fd, idx->next); +} + +void rarchdb_close(struct rarchdb *db) +{ + close(db->fd); + db->fd = -1; +} + +int rarchdb_open(const char *path, struct rarchdb *db) +{ + struct rarchdb_header header; + struct rarchdb_metadata md; + int rv; + int fd = open(path, O_RDWR); + if (fd == -1) + return -errno; + + db->root = lseek(fd, 0, SEEK_CUR); + if ((rv = read(fd, &header, sizeof(header))) == -1) + { + rv = -errno; + goto error; + } + + if (strncmp(header.magic_number, MAGIC_NUMBER, sizeof(MAGIC_NUMBER)) != 0) + { + rv = -EINVAL; + goto error; + } + + header.metadata_offset = betoht64(header.metadata_offset); + lseek(fd, header.metadata_offset, SEEK_SET); + if (rarchdb_read_metadata(fd, &md) < 0) + { + rv = -EINVAL; + goto error; + } + db->count = md.count; + db->first_index_offset = lseek(fd, 0, SEEK_CUR); + db->fd = fd; + rarchdb_read_reset(db); + return 0; +error: + close(fd); + return rv; +} + +static int rarchdb_find_index(struct rarchdb *db, const char *index_name, struct rarchdb_index *idx) +{ + off_t eof = lseek(db->fd, 0, SEEK_END); + off_t offset = lseek(db->fd, db->first_index_offset, SEEK_SET); + while (offset < eof) + { + rarchdb_read_index_header(db->fd, idx); + if (strncmp(index_name, idx->name, strlen(idx->name)) == 0) + return 0; + offset = lseek(db->fd, idx->next, SEEK_CUR); + } + return -1; +} + +static int node_compare(const void *a, const void* b, void *ctx) +{ + return memcmp(a, b, *(uint8_t*)ctx); +} + +static int binsearch(const void *buff, const void *item, uint64_t count, uint8_t field_size, uint64_t *offset) +{ + int mid = count / 2; + int item_size = field_size + sizeof(uint64_t); + const void *current = buff + (mid * item_size); + int rv = node_compare(current, item, &field_size); + + if (rv == 0) + { + *offset = *(uint64_t*)(current + field_size); + return 0; + } + else if (count == 0) + return -1; + else if (rv > 0) + return binsearch(buff, item, mid, field_size, offset); + + return binsearch(current + item_size, item, count - mid, field_size, offset); +} + +int rarchdb_find_entry(struct rarchdb *db, const char *index_name, const void *key) +{ + struct rarchdb_index idx; + uint64_t offset; + ssize_t nread = 0; + int rv; + if (rarchdb_find_index(db, index_name, &idx) < 0) + { + rarchdb_read_reset(db); + return -1; + } + size_t bufflen = idx.next; + void* buff = malloc(bufflen); + if (!buff) + return -ENOMEM; + + while (nread < bufflen) + { + rv = read(db->fd, buff + nread, bufflen - nread); + if (rv <= 0) + { + free(buff); + return -errno; + } + nread += rv; + } + + rv = binsearch(buff, key, db->count, idx.key_size, &offset); + free(buff); + rarchdb_read_reset(db); + + if (rv == 0) + lseek(db->fd, offset, SEEK_SET); + + return rv; +} + +int rarchdb_read_reset(struct rarchdb *db) +{ + db->eof = 0; + return lseek(db->fd, db->root + sizeof(struct rarchdb_header), SEEK_SET); +} + +int rarchdb_read_item(struct rarchdb *db, struct rmsgpack_dom_value *out) +{ + int rv; + if (db->eof) + return EOF; + + rv = rmsgpack_dom_read(db->fd, out); + if (rv < 0) + return rv; + + if (out->type == RDT_NULL) + { + db->eof = 1; + return EOF; + } + + return 0; +} + +static int node_iter(void *value, void *ctx) +{ + struct node_iter_ctx *nictx = ctx; + + if (write(nictx->db->fd, value, nictx->idx->key_size + sizeof(uint64_t)) > 0) + return 0; + + return -1; +} + +uint64_t rarchdb_tell(struct rarchdb *db) +{ + return lseek(db->fd, 0, SEEK_CUR); +} + +int rarchdb_create_index(struct rarchdb *db, const char* name, const char *field_name) +{ + int rv; + struct rmsgpack_dom_value key; + struct rarchdb_index idx; + struct rmsgpack_dom_value item; + struct rmsgpack_dom_value *field; + void* buff = NULL; + uint8_t field_size = 0; + struct bintree tree; + uint64_t item_loc = rarchdb_tell(db); + uint64_t idx_header_offset; + + bintree_new(&tree, node_compare, &field_size); + rarchdb_read_reset(db); + + key.type = RDT_STRING; + key.string.len = strlen(field_name); + // We know we aren't going to change it + key.string.buff = (char *) field_name; + while(rarchdb_read_item(db, &item) == 0) + { + if (item.type != RDT_MAP) + { + rv = -EINVAL; + printf("Only map keys are supported\n"); + goto clean; + } + field = rmsgpack_dom_value_map_value(&item, &key); + if (!field) + { + rv = -EINVAL; + printf("field not found in item\n"); + goto clean; + } + if (field->type != RDT_BINARY) + { + rv = -EINVAL; + printf("field is not binary\n"); + goto clean; + } + + if (field->binary.len == 0) + { + rv = -EINVAL; + printf("field is empty\n"); + goto clean; + } + + if (field_size == 0) + field_size = field->binary.len; + else if (field->binary.len != field_size) + { + rv = -EINVAL; + printf("field is not of correct size\n"); + goto clean; + } + + buff = malloc(field_size + sizeof(uint64_t)); + if(!buff) + { + rv = -ENOMEM; + goto clean; + } + + memcpy(buff, field->binary.buff, field_size); + memcpy(buff + field_size, &item_loc, sizeof(uint64_t)); + + if (bintree_insert(&tree, buff) != 0) + { + printf("Value is not unique: "); + rmsgpack_dom_value_print(field); + printf("\n"); + rv = -EINVAL; + goto clean; + } + buff = NULL; + rmsgpack_dom_value_free(&item); + item_loc = rarchdb_tell(db); + } + idx_header_offset = lseek(db->fd, 0, SEEK_END); + strncpy(idx.name, name, 50); + idx.name[49] = '\0'; + idx.key_size = field_size; + idx.next = db->count * (field_size + sizeof(uint64_t)); + rarchdb_write_index_header(db->fd, &idx); + struct node_iter_ctx nictx; + nictx.db = db; + nictx.idx = &idx; + bintree_iterate(&tree, node_iter, &nictx); + bintree_free(&tree); +clean: + rmsgpack_dom_value_free(&item); + if (buff) + free(buff); + rarchdb_read_reset(db); + return 0; +} diff --git a/rarchdb/rarchdb.h b/rarchdb/rarchdb.h new file mode 100644 index 0000000000..74d01740b5 --- /dev/null +++ b/rarchdb/rarchdb.h @@ -0,0 +1,28 @@ +#ifndef __RARCHDB_H__ +#define __RARCHDB_H__ + +#include +#include "rmsgpack_dom.h" + +struct rarchdb { + int fd; + int eof; + uint64_t root; + uint64_t count; + uint64_t first_index_offset; +}; + + +typedef int(*rarchdb_value_provider)(void *ctx, struct rmsgpack_dom_value *out); + +int rarchdb_create(int fd, rarchdb_value_provider value_provider, void *ctx); + +void rarchdb_close(struct rarchdb *db); +int rarchdb_open(const char *path, struct rarchdb *db); + +int rarchdb_read_item(struct rarchdb *db, struct rmsgpack_dom_value *out); +int rarchdb_read_reset(struct rarchdb *db); +int rarchdb_create_index(struct rarchdb *db, const char* name, const char *field_name); +int rarchdb_find_entry(struct rarchdb *db, const char *index_name, const void *key); + +#endif diff --git a/rarchdb/rarchdb_endian.h b/rarchdb/rarchdb_endian.h new file mode 100644 index 0000000000..ddc313f5bd --- /dev/null +++ b/rarchdb/rarchdb_endian.h @@ -0,0 +1,51 @@ +#ifndef __RARCHDB_MSGPACK_ENDIAN_H +#define __RARCHDB_MSGPACK_ENDIAN_H + +#include + +#ifndef INLINE +#define INLINE inline +#endif + +static INLINE uint8_t is_little_endian(void) +{ + union + { + uint16_t x; + uint8_t y[2]; + } u; + + u.x = 1; + return u.y[0]; +} + +#define swap16(val) \ + ((((uint16_t)(val) & 0x00ff) << 8) \ + | (((uint16_t)(val) & 0xff00) >> 8)) + +#define swap32(val) \ + ((((uint32_t)(val) & 0x000000ff) << 24) \ + | (((uint32_t)(val) & 0x0000ff00) << 8) \ + | (((uint32_t)(val) & 0x00ff0000) >> 8) \ + | (((uint32_t)(val) & 0xff000000) >> 24)) + +#define swap64(val) \ + ((((uint64_t)(val) & 0x00000000000000ffULL) << 56) \ + | (((uint64_t)(val) & 0x000000000000ff00ULL) << 40) \ + | (((uint64_t)(val) & 0x0000000000ff0000ULL) << 24) \ + | (((uint64_t)(val) & 0x00000000ff000000ULL) << 8) \ + | (((uint64_t)(val) & 0x000000ff00000000ULL) >> 8) \ + | (((uint64_t)(val) & 0x0000ff0000000000ULL) >> 24) \ + | (((uint64_t)(val) & 0x00ff000000000000ULL) >> 40) \ + | (((uint64_t)(val) & 0xff00000000000000ULL) >> 56)) + + +#define httobe64(x) (is_little_endian() ? swap64(x) : (x)) +#define httobe32(x) (is_little_endian() ? swap32(x) : (x)) +#define httobe16(x) (is_little_endian() ? swap16(x) : (x)) + +#define betoht16(x) httobe16(x) +#define betoht32(x) httobe32(x) +#define betoht64(x) httobe64(x) + +#endif diff --git a/rarchdb/rarchdb_tool.c b/rarchdb/rarchdb_tool.c new file mode 100644 index 0000000000..fa60448778 --- /dev/null +++ b/rarchdb/rarchdb_tool.c @@ -0,0 +1,116 @@ +#include +#include + +#include "rarchdb.h" +#include "rmsgpack_dom.h" + +static int list_db(void) +{ + return 0; +} + +int main(int argc, char** argv) +{ + int rv; + struct rarchdb db; + struct rmsgpack_dom_value item; + if (argc < 3) + { + printf("Usage: %s [extra args...]\n", argv[0]); + printf("Available Commands:\n"); + printf("\tlist\n"); + printf("\tcreate-index \n"); + printf("\tfind \n"); + return 1; + } + + const char* command = argv[2]; + const char* path = argv[1]; + + if ((rv = rarchdb_open(path, &db)) != 0) + { + printf("Could not open db file '%s': %s\n", path, strerror(-rv)); + return 1; + } + + if (strcmp(command, "list") == 0) + { + if (argc != 3) + { + printf("Usage: %s list\n", argv[0]); + return 1; + } + while(rarchdb_read_item(&db, &item) == 0) + { + rmsgpack_dom_value_print(&item); + printf("\n"); + rmsgpack_dom_value_free(&item); + } + } + else if (strcmp(command, "create-index") == 0) + { + const char *index_name, *field_name; + + if (argc != 5) + { + printf("Usage: %s create-index \n", argv[0]); + return 1; + } + + index_name = argv[3]; + field_name = argv[4]; + + rarchdb_create_index(&db, index_name, field_name); + } + else if (strcmp(command, "find") == 0) + { + int i; + const char *index_name, *value; + char bin_value[256]; + uint8_t h; + uint8_t l; + + if (argc != 5) + { + printf("Usage: %s create-index \n", argv[0]); + return 1; + } + + index_name = argv[3]; + value = argv[4]; + + for (i = 0; i < strlen(value); i += 2) + { + if (value[i] <= '9') + h = value[i] - '0'; + else + h = (value[i] - 'A') + 10; + + if (value[i+1] <= '9') + l = value[i+1] - '0'; + else + l = (value[i+1] - 'A') + 10; + + bin_value[i/2] = h * 16 + l; + } + + if (rarchdb_find_entry(&db, index_name, bin_value) != 0) + { + printf("Could not find item\n"); + return 1; + } + + if (rarchdb_read_item(&db, &item) == 0) + { + rmsgpack_dom_value_print(&item); + printf("\n"); + rmsgpack_dom_value_free(&item); + } + } + else + { + printf("Unkown command %s\n", argv[1]); + return 1; + } + rarchdb_close(&db); +} diff --git a/rarchdb/rmsgpack.c b/rarchdb/rmsgpack.c new file mode 100644 index 0000000000..8960d0ec18 --- /dev/null +++ b/rarchdb/rmsgpack.c @@ -0,0 +1,548 @@ +/* + * An almost complete implementation of msgpack by + * Saggi Mizrahi + * + * TODO: + * - float types + * - ext types + * + * For more information http://msgpack.org/ + */ + +#include "rmsgpack.h" + +#include +#include +#include +#include +#include + +#include "rarchdb_endian.h" + +static const uint8_t MPF_FIXMAP = 0x80; +static const uint8_t MPF_MAP16 = 0xde; +static const uint8_t MPF_MAP32 = 0xdf; + +static const uint8_t MPF_FIXARRAY = 0x90; +static const uint8_t MPF_ARRAY16 = 0xdc; +static const uint8_t MPF_ARRAY32 = 0xdd; + +static const uint8_t MPF_FIXSTR = 0xa0; +static const uint8_t MPF_STR8 = 0xd9; +static const uint8_t MPF_STR16 = 0xda; +static const uint8_t MPF_STR32 = 0xdb; + +static const uint8_t MPF_BIN8 = 0xc4; +static const uint8_t MPF_BIN16 = 0xc5; +static const uint8_t MPF_BIN32 = 0xc6; + +static const uint8_t MPF_FALSE = 0xc2; +static const uint8_t MPF_TRUE = 0xc3; + +static const uint8_t MPF_INT8 = 0xd0; +static const uint8_t MPF_INT16 = 0xd1; +static const uint8_t MPF_INT32 = 0xd2; +static const uint8_t MPF_INT64 = 0xd3; + +static const uint8_t MPF_UINT8 = 0xcc; +static const uint8_t MPF_UINT16 = 0xcd; +static const uint8_t MPF_UINT32 = 0xce; +static const uint8_t MPF_UINT64 = 0xcf; + +static const uint8_t MPF_NIL = 0xc0; + +int rmsgpack_write_array_header(int fd, uint32_t size) +{ + if (size < 16) + { + size = (size | MPF_FIXARRAY); + if (write(fd, &size, sizeof(int8_t)) == -1) + return -errno; + return sizeof(int8_t); + } + else if (size == (uint16_t)size) + { + if (write(fd, &MPF_ARRAY16, sizeof(MPF_ARRAY16)) == -1) + return -errno; + if (write(fd, (void *)(&size), sizeof(uint16_t)) == -1) + return -errno; + return sizeof(int8_t) + sizeof(uint16_t); + } + else + { + if (write(fd, &MPF_ARRAY32, sizeof(MPF_ARRAY32)) == -1) + return -errno; + if (write(fd, (void *)(&size), sizeof(uint32_t)) == -1) + return -errno; + return sizeof(int8_t) + sizeof(uint32_t); + } +} + +int rmsgpack_write_map_header(int fd, uint32_t size) +{ + if (size < 16) + { + size = (size | MPF_FIXMAP); + if (write(fd, &size, sizeof(int8_t)) == -1) + return -errno; + return sizeof(int8_t); + } + else if (size < (uint16_t)size) + { + if (write(fd, &MPF_MAP16, sizeof(MPF_MAP16)) == -1) + return -errno; + if (write(fd, (void *)(&size), sizeof(uint16_t)) == -1) + return -errno; + return sizeof(int8_t) + sizeof(uint16_t); + } + else + { + if (write(fd, &MPF_MAP32, sizeof(MPF_MAP32)) == -1) + return -errno; + if (write(fd, (void *)(&size), sizeof(uint32_t)) == -1) + return -errno; + return sizeof(int8_t) + sizeof(uint32_t); + } +} + +int rmsgpack_write_string(int fd, const char *s, uint32_t len) +{ + int8_t fixlen = 0; + uint16_t tmp_i16; + uint32_t tmp_i32; + int written = sizeof(int8_t); + if (len < 32) + { + fixlen = len | MPF_FIXSTR; + if (write(fd, &fixlen, sizeof(int8_t)) == -1) + return -errno; + } + else if (len < 1<<8) + { + if (write(fd, &MPF_STR8, sizeof(MPF_STR8)) == -1) + return -errno; + if (write(fd, &len, sizeof(uint8_t)) == -1) + return -errno; + written += sizeof(uint8_t); + } + else if (len < 1<<16) + { + if (write(fd, &MPF_STR16, sizeof(MPF_STR16)) == -1) + return -errno; + tmp_i16 = httobe16(len); + if (write(fd, &tmp_i16, sizeof(uint16_t)) == -1) + return -errno; + written += sizeof(uint16_t); + } + else + { + if (write(fd, &MPF_STR32, sizeof(MPF_STR32)) == -1) + return -errno; + tmp_i32 = httobe32(len); + if (write(fd, &tmp_i32, sizeof(uint32_t)) == -1) + return -errno; + written += sizeof(uint32_t); + } + if (write(fd, s, len) == -1) + return -errno; + written += len; + return written; +} + +int rmsgpack_write_bin(int fd, const void *s, uint32_t len) +{ + uint16_t tmp_i16; + uint32_t tmp_i32; + int written = sizeof(int8_t); + if (len == (uint8_t)len) + { + if (write(fd, &MPF_BIN8, sizeof(MPF_BIN8)) == -1) + return -errno; + if (write(fd, &len, sizeof(uint8_t)) == -1) + return -errno; + written += sizeof(uint8_t); + } + else if (len == (uint16_t)len) + { + if (write(fd, &MPF_BIN16, sizeof(MPF_BIN16)) == -1) + return -errno; + tmp_i16 = httobe16(len); + if (write(fd, &tmp_i16, sizeof(uint16_t)) == -1) + return -errno; + written += sizeof(uint16_t); + } + else + { + if (write(fd, &MPF_BIN32, sizeof(MPF_BIN32)) == -1) + return -errno; + tmp_i32 = httobe32(len); + if (write(fd, &tmp_i32, sizeof(uint32_t)) == -1) + return -errno; + written += sizeof(uint32_t); + } + if (write(fd, s, len) == -1) + return -errno; + written += len; + return 0; +} + +int rmsgpack_write_nil(int fd) +{ + if (write(fd, &MPF_NIL, sizeof(MPF_NIL)) == -1) + return -errno; + return sizeof(uint8_t); +} + +int rmsgpack_write_bool(int fd, int value) +{ + if (value) + { + if (write(fd, &MPF_TRUE, sizeof(MPF_TRUE)) == -1) + return -errno; + } + else + { + if (write(fd, &MPF_FALSE, sizeof(MPF_FALSE)) == -1) + return -errno; + } + return sizeof(uint8_t); +} + +int rmsgpack_write_int(int fd, int64_t value) +{ + uint8_t tmpval = 0; + + int16_t tmp_i16; + int32_t tmp_i32; + int written = sizeof(uint8_t); + if (value >=0 && value < 128) + { + if (write(fd, &value, sizeof(int8_t)) == -1) + return -errno; + } + else if (value < 0 && value > -32) + { + tmpval = (value) | 0xe0; + if (write(fd, &tmpval, sizeof(uint8_t)) == -1) + return -errno; + } + else if (value == (int8_t)value) + { + if (write(fd, &MPF_INT8, sizeof(MPF_INT8)) == -1) + return -errno; + + if (write(fd, &value, sizeof(int8_t)) == -1) + return -errno; + written += sizeof(int8_t); + } + else if (value == (int16_t)value) + { + if (write(fd, &MPF_INT16, sizeof(MPF_INT16)) == -1) + return -errno; + + tmp_i16 = httobe16(value); + if (write(fd, &tmp_i16, sizeof(int16_t)) == -1) + return -errno; + written += sizeof(int16_t); + } + else if (value == (int32_t)value) + { + if (write(fd, &MPF_INT32, sizeof(MPF_INT32)) == -1) + return -errno; + + tmp_i32 = httobe32(value); + if (write(fd, &tmp_i32, sizeof(int32_t)) == -1) + return -errno; + written += sizeof(int32_t); + } + else + { + if (write(fd, &MPF_INT64, sizeof(MPF_INT64)) == -1) + return -errno; + + value = httobe64(value); + if (write(fd, &value, sizeof(int64_t)) == -1) + return -errno; + written += sizeof(int64_t); + } + return written; +} + +int rmsgpack_write_uint(int fd, uint64_t value) +{ + uint16_t tmp_i16; + uint32_t tmp_i32; + int written = sizeof(uint8_t); + + if (value == (uint8_t)value) + { + if (write(fd, &MPF_UINT8, sizeof(MPF_UINT8)) == -1) + return -errno; + + if (write(fd, &value, sizeof(uint8_t)) == -1) + return -errno; + written += sizeof(uint8_t); + } + else if (value == (uint16_t)value) + { + if (write(fd, &MPF_UINT16, sizeof(MPF_UINT16)) == -1) + return -errno; + + tmp_i16 = httobe16(value); + if (write(fd, &tmp_i16, sizeof(uint16_t)) == -1) + return -errno; + written += sizeof(uint16_t); + } + else if (value == (uint32_t)value) + { + if (write(fd, &MPF_UINT32, sizeof(MPF_UINT32)) == -1) + return -errno; + + tmp_i32 = httobe32(value); + if (write(fd, &tmp_i32, sizeof(uint32_t)) == -1) + return -errno; + written += sizeof(uint32_t); + } + else + { + if (write(fd, &MPF_UINT64, sizeof(MPF_UINT64)) == -1) + return -errno; + + value = httobe64(value); + if (write(fd, &value, sizeof(uint64_t)) == -1) + return -errno; + written += sizeof(uint64_t); + } + return written; +} + +static int read_uint(int fd, uint64_t *out, size_t size) +{ + uint64_t tmp; + if(read(fd, &tmp, size) == -1) + return -errno; + switch(size) + { + case 1: + *out = *(uint8_t*)(&tmp); + break; + case 2: + *out = betoht16(tmp); + break; + case 4: + *out = betoht32(tmp); + break; + case 8: + *out = betoht64(tmp); + break; + } + return 0; +} + +static int read_int(int fd, int64_t *out, size_t size) +{ + uint64_t tmp; + if(read(fd, &tmp, size) == -1) + return -errno; + + switch(size) + { + case 1: + *out = *((int8_t*)(&tmp)); + break; + case 2: + tmp = betoht16(tmp); + *out = *((int16_t*)(&tmp)); + break; + case 4: + tmp = betoht32(tmp); + *out = *((int32_t*)(&tmp)); + break; + case 8: + tmp = betoht64(tmp); + *out = *((int64_t*)(&tmp)); + break; + } + return 0; +} + +static int read_buff(int fd, size_t size, char** pbuff, uint64_t *len) +{ + uint64_t tmp_len; + if(read_uint(fd, &tmp_len, size) == -1) + return -errno; + + *pbuff = calloc(tmp_len + 1, sizeof(char)); + if (read(fd, *pbuff, tmp_len) == -1) + { + free(*pbuff); + return -errno; + } + *len = tmp_len; + return 0; +} + +static int read_map(int fd, uint32_t len, struct rmsgpack_read_callbacks *callbacks, void *data) +{ + int rv, i; + if ( + callbacks->read_map_start && + (rv = callbacks->read_map_start(len, data)) < 0) + return rv; + + for (i = 0; i < len; i++) + { + if((rv = rmsgpack_read(fd, callbacks, data)) < 0) + return rv; + if((rv = rmsgpack_read(fd, callbacks, data)) < 0) + return rv; + } + + return 0; +} + +static int read_array(int fd, uint32_t len, struct rmsgpack_read_callbacks *callbacks, void *data) +{ + int rv, i; + + if ( + callbacks->read_array_start && + (rv = callbacks->read_array_start(len, data)) < 0) + return rv; + + for (i = 0; i < len; i++) + { + if((rv = rmsgpack_read(fd, callbacks, data)) < 0) + return rv; + } + + return 0; +} + +int rmsgpack_read(int fd, struct rmsgpack_read_callbacks *callbacks, void *data) +{ + int rv; + uint64_t tmp_len = 0; + uint64_t tmp_uint = 0; + int64_t tmp_int = 0; + uint8_t type = 0; + char* buff; + if (read(fd, &type, sizeof(uint8_t)) == -1) + return -errno; + + if (type < MPF_FIXMAP) + { + if (!callbacks->read_int) + return 0; + return callbacks->read_int(type, data); + } + else if (type < MPF_FIXARRAY) + { + tmp_len = type - MPF_FIXMAP; + return read_map(fd, tmp_len, callbacks, data); + } + else if (type < MPF_FIXSTR) + { + tmp_len = type - MPF_FIXARRAY; + return read_array(fd, tmp_len, callbacks, data); + } + else if (type < MPF_NIL) + { + tmp_len = type - MPF_FIXSTR; + buff = calloc(tmp_len + 1, sizeof(char)); + if (!buff) + return -ENOMEM; + if(read(fd, buff, tmp_len) == -1) + { + free(buff); + return -errno; + } + buff[tmp_len] = '\0'; + if (!callbacks->read_string) + { + free(buff); + return 0; + } + return callbacks->read_string(buff, tmp_len, data); + } + else if (type > MPF_MAP32) + { + if (!callbacks->read_int) + return 0; + return callbacks->read_int(type - 0xff - 1, data); + } + + switch(type) + { + case 0xc0: + if(callbacks->read_nil) + return callbacks->read_nil(data); + break; + case 0xc2: + if(callbacks->read_bool) + return callbacks->read_bool(0, data); + break; + case 0xc3: + if(callbacks->read_bool) + return callbacks->read_bool(1, data); + break; + case 0xc4: + case 0xc5: + case 0xc6: + if((rv = read_buff(fd, 1<<(type - 0xc4), &buff, &tmp_len)) < 0) + return rv; + + if (callbacks->read_bin) + return callbacks->read_bin(buff, tmp_len, data); + break; + case 0xcc: + case 0xcd: + case 0xce: + case 0xcf: + tmp_len = 1<<(type - 0xcc); + tmp_uint = 0; + if(read_uint(fd, &tmp_uint, tmp_len) == -1) + return -errno; + + if (callbacks->read_uint) + return callbacks->read_uint(tmp_uint, data); + break; + case 0xd0: + case 0xd1: + case 0xd2: + case 0xd3: + tmp_len = 1<<(type - 0xd0); + tmp_int = 0; + if(read_int(fd, &tmp_int, tmp_len) == -1) + return -errno; + + if (callbacks->read_int) + return callbacks->read_int(tmp_int, data); + break; + case 0xd9: + case 0xda: + case 0xdb: + if((rv = read_buff(fd, 1<<(type - 0xd9), &buff, &tmp_len)) < 0) + return rv; + + if (callbacks->read_string) + return callbacks->read_string(buff, tmp_len, data); + break; + case 0xdc: + case 0xdd: + if(read_uint(fd, &tmp_len, 2<<(type - 0xdc)) == -1) + return -errno; + + return read_array(fd, tmp_len, callbacks, data); + break; + case 0xde: + case 0xdf: + if(read_uint(fd, &tmp_len, 2<<(type - 0xde)) == -1) + return -errno; + + return read_map(fd, tmp_len, callbacks, data); + break; + } + + return 0; +} diff --git a/rarchdb/rmsgpack.h b/rarchdb/rmsgpack.h new file mode 100644 index 0000000000..e2ada18d05 --- /dev/null +++ b/rarchdb/rmsgpack.h @@ -0,0 +1,30 @@ +#ifndef __RARCHDB_MSGPACK_H__ +#define __RARCHDB_MSGPACK_H__ + +#include + +struct rmsgpack_read_callbacks { + int(*read_nil)(void*); + int(*read_bool)(int, void*); + int(*read_int)(int64_t, void*); + int(*read_uint)(uint64_t, void*); + int(*read_string)(char *, uint32_t, void*); + int(*read_bin)(void *, uint32_t, void*); + int(*read_map_start)(uint32_t, void*); + int(*read_array_start)(uint32_t, void*); +}; + + +int rmsgpack_write_array_header(int fd, uint32_t size); +int rmsgpack_write_map_header(int fd, uint32_t size); +int rmsgpack_write_string(int fd, const char *s, uint32_t len); +int rmsgpack_write_bin(int fd, const void *s, uint32_t len); +int rmsgpack_write_nil(int fd); +int rmsgpack_write_bool(int fd, int value); +int rmsgpack_write_int(int fd, int64_t value); +int rmsgpack_write_uint(int fd, uint64_t value); + +int rmsgpack_read(int fd, struct rmsgpack_read_callbacks *callbacks, void *data); + +#endif + diff --git a/rarchdb/rmsgpack_dom.c b/rarchdb/rmsgpack_dom.c new file mode 100644 index 0000000000..f33c286352 --- /dev/null +++ b/rarchdb/rmsgpack_dom.c @@ -0,0 +1,462 @@ +#include "rmsgpack_dom.h" + +#include +#include +#include +#include +#include +#include + +#include "rmsgpack.h" + + +#define MAX_DEPTH 128 +struct dom_reader_state +{ + int i; + struct rmsgpack_dom_value* stack[MAX_DEPTH]; +}; + + +static struct rmsgpack_dom_value *dom_reader_state_pop(struct dom_reader_state *s) +{ + struct rmsgpack_dom_value *v = s->stack[s->i]; + s->i--; + return v; +} + +static int dom_reader_state_push(struct dom_reader_state *s, struct rmsgpack_dom_value *v) +{ + //TODO: Verify we don't overflow + s->i++; + s->stack[s->i] = v; + return 0; +} + +static int dom_read_nil(void *data) +{ + struct rmsgpack_dom_value *v = dom_reader_state_pop(data); + v->type = RDT_NULL; + return 0; +} + +static int dom_read_bool(int value, void *data) +{ + struct rmsgpack_dom_value *v = dom_reader_state_pop(data); + v->type = RDT_BOOL; + v->bool_ = value; + return 0; +} + +static int dom_read_int(int64_t value, void *data) +{ + struct rmsgpack_dom_value *v = dom_reader_state_pop(data); + v->type = RDT_INT; + v->int_ = value; + return 0; +} + +static int dom_read_uint(uint64_t value, void *data) +{ + struct rmsgpack_dom_value *v = dom_reader_state_pop(data); + v->type = RDT_UINT; + v->uint_ = value; + return 0; +} + +static int dom_read_string(char* value, uint32_t len, void *data) +{ + struct rmsgpack_dom_value *v = dom_reader_state_pop(data); + v->type = RDT_STRING; + v->string.len = len; + v->string.buff = value; + return 0; +} + +static int dom_read_bin(void* value, uint32_t len, void *data) +{ + struct rmsgpack_dom_value *v = dom_reader_state_pop(data); + v->type = RDT_BINARY; + v->binary.len = len; + v->binary.buff = value; + return 0; +} + +static int dom_read_map_start(uint32_t len, void *data) +{ + int i; + struct rmsgpack_dom_pair *items; + struct rmsgpack_dom_value *v = dom_reader_state_pop(data); + v->type = RDT_MAP; + v->map.len = len; + v->map.items = NULL; + + items = (struct rmsgpack_dom_pair *)calloc(len, sizeof(struct rmsgpack_dom_pair)); + + if (!items) + return -ENOMEM; + + v->map.items = items; + for (i = 0; i < len; i++) + { + if (dom_reader_state_push(data, &items[i].value) < 0) + return -ENOMEM; + if (dom_reader_state_push(data, &items[i].key) < 0) + return -ENOMEM; + } + return 0; +} + +static int dom_read_array_start(uint32_t len, void *data) +{ + int i; + struct rmsgpack_dom_value *v = dom_reader_state_pop(data); + struct rmsgpack_dom_value *items; + v->type = RDT_ARRAY; + v->array.len = len; + v->array.items = NULL; + + items = (struct rmsgpack_dom_value*)calloc(len, sizeof(struct rmsgpack_dom_pair)); + + if (!items) + return -ENOMEM; + + v->array.items = items; + + for (i = 0; i < len; i++) + { + if (dom_reader_state_push(data, &items[i]) < 0) + return -ENOMEM; + } + return 0; +} + +static struct rmsgpack_read_callbacks dom_reader_callbacks = { + dom_read_nil, + dom_read_bool, + dom_read_int, + dom_read_uint, + dom_read_string, + dom_read_bin, + dom_read_map_start, + dom_read_array_start +}; + +void rmsgpack_dom_value_free(struct rmsgpack_dom_value *v) +{ + int i; + + switch (v->type) + { + case RDT_STRING: + free(v->string.buff); + break; + case RDT_BINARY: + free(v->binary.buff); + break; + case RDT_MAP: + for (i = 0; i < v->map.len; i++) + { + rmsgpack_dom_value_free(&v->map.items[i].key); + rmsgpack_dom_value_free(&v->map.items[i].value); + } + free(v->map.items); + break; + case RDT_ARRAY: + for (i = 0; i < v->array.len; i++) + rmsgpack_dom_value_free(&v->array.items[i]); + free(v->array.items); + break; + case RDT_NULL: + case RDT_INT: + case RDT_BOOL: + case RDT_UINT: + // Do nothing + break; + } +} + +struct rmsgpack_dom_value *rmsgpack_dom_value_map_value( + const struct rmsgpack_dom_value *map, const struct rmsgpack_dom_value *key) +{ + int i; + if (map->type != RDT_MAP) + return NULL; + + for (i = 0; i < map->map.len; i++) + { + if (rmsgpack_dom_value_cmp(key, &map->map.items[i].key) == 0) + return &map->map.items[i].value; + } + return NULL; +} + +int rmsgpack_dom_value_cmp( + const struct rmsgpack_dom_value *a, const struct rmsgpack_dom_value *b) +{ + int rv, i; + if (a == b) + return 1; + + if (a->type != b->type) + return 1; + + switch (a->type) + { + case RDT_NULL: + return 0; + break; + case RDT_BOOL: + return a->bool_ == b->bool_ ? 1 : 0; + break; + case RDT_INT: + return a->int_ == b->int_ ? 1 : 0; + break; + case RDT_UINT: + return a->uint_ == b->uint_ ? 1 : 0; + break; + case RDT_STRING: + if (a->string.len != b->string.len) + return 1; + return strncmp(a->string.buff, b->string.buff, a->string.len); + break; + case RDT_BINARY: + if (a->binary.len != b->binary.len) + return 1; + return memcmp(a->binary.buff, b->binary.buff, a->binary.len); + break; + case RDT_MAP: + if (a->map.len != b->map.len) + return 1; + for (i = 0; i < a->map.len; i++) + { + if((rv = rmsgpack_dom_value_cmp(&a->map.items[i].key, &b->map.items[i].key)) != 0) + return rv; + if((rv = rmsgpack_dom_value_cmp(&a->map.items[i].value, &b->map.items[i].value)) != 0) + return rv; + } + break; + case RDT_ARRAY: + if (a->array.len != b->array.len) + return 1; + for (i = 0; i < a->array.len; i++) + { + if((rv = rmsgpack_dom_value_cmp(&a->array.items[i], &b->array.items[i])) != 0) + return rv; + } + break; + } + return 1; +} + +void rmsgpack_dom_value_print(struct rmsgpack_dom_value *obj) +{ + int i; + + switch (obj->type) + { + case RDT_NULL: + printf("nil"); + break; + case RDT_BOOL: + if (obj->bool_) + printf("true"); + else + printf("false"); + break; + case RDT_INT: + printf("%ld", obj->int_); + break; + case RDT_UINT: + printf("%lu", obj->uint_); + break; + case RDT_STRING: + printf("\"%s\"", obj->string.buff); + break; + case RDT_BINARY: + printf("\""); + for (i = 0; i < obj->binary.len; i++) + printf("%02X", (unsigned char) obj->binary.buff[i]); + printf("\""); + break; + case RDT_MAP: + printf("{"); + for (i = 0; i < obj->map.len; i++) + { + rmsgpack_dom_value_print(&obj->map.items[i].key); + printf(": "); + rmsgpack_dom_value_print(&obj->map.items[i].value); + if (i < (obj->map.len - 1)) + printf(", "); + } + printf("}"); + break; + case RDT_ARRAY: + printf("["); + for (i = 0; i < obj->array.len; i++) + { + rmsgpack_dom_value_print(&obj->array.items[i]); + if (i < (obj->array.len - 1)) + printf(", "); + } + printf("]"); + } +} +int rmsgpack_dom_write(int fd, const struct rmsgpack_dom_value *obj) +{ + int i; + int rv = 0; + int written = 0; + + switch (obj->type) + { + case RDT_NULL: + return rmsgpack_write_nil(fd); + break; + case RDT_BOOL: + return rmsgpack_write_bool(fd, obj->bool_); + break; + case RDT_INT: + return rmsgpack_write_int(fd, obj->int_); + break; + case RDT_UINT: + return rmsgpack_write_uint(fd, obj->uint_); + break; + case RDT_STRING: + return rmsgpack_write_string(fd, obj->string.buff, obj->string.len); + break; + case RDT_BINARY: + return rmsgpack_write_bin(fd, obj->binary.buff, obj->binary.len); + break; + case RDT_MAP: + if ((rv = rmsgpack_write_map_header(fd, obj->map.len)) < 0) + return rv; + written += rv; + for (i = 0; i < obj->map.len; i++) + { + if((rv = rmsgpack_dom_write(fd, &obj->map.items[i].key)) < 0) + return rv; + written += rv; + if((rv = rmsgpack_dom_write(fd, &obj->map.items[i].value)) < 0) + return rv; + written += rv; + } + break; + case RDT_ARRAY: + if ((rv = rmsgpack_write_array_header(fd, obj->array.len)) < 0) + return rv; + written += rv; + for (i = 0; i < obj->array.len; i++) + { + if((rv = rmsgpack_dom_write(fd, &obj->array.items[i])) < 0) + return rv; + written += rv; + } + } + return written; +} + +int rmsgpack_dom_read(int fd, struct rmsgpack_dom_value *out) +{ + int rv = 0; + struct dom_reader_state s; + s.i = 0; + s.stack[0] = out; + rv = rmsgpack_read(fd, &dom_reader_callbacks, &s); + + if (rv < 0) + rmsgpack_dom_value_free(out); + + return rv; +} + +int rmsgpack_dom_read_into(int fd, ...) +{ + va_list ap; + struct rmsgpack_dom_value map; + int rv; + const char* key_name; + int value_type; + struct rmsgpack_dom_value key; + struct rmsgpack_dom_value *value; + int64_t *int_value; + uint64_t *uint_value; + int *bool_value; + char *buff_value; + uint64_t min_len; + + va_start(ap, fd); + rv = rmsgpack_dom_read(fd, &map); + if (rv < 0) + return rv; + + if (map.type != RDT_MAP) + { + rv = -EINVAL; + goto clean; + } + + while (1) + { + key_name = va_arg(ap, const char*); + if (key_name == NULL) + { + rv = 0; + goto clean; + } + + key.type = RDT_STRING; + key.string.len = strlen(key_name); + key.string.buff = (char *) key_name; + + value = rmsgpack_dom_value_map_value(&map, &key); + + switch(value->type) + { + case RDT_INT: + int_value = va_arg(ap, int64_t*); + *int_value = value->int_; + break; + case RDT_BOOL: + bool_value = va_arg(ap, int*); + *bool_value = value->bool_; + break; + case RDT_UINT: + uint_value = va_arg(ap, uint64_t*); + *uint_value = value->uint_; + break; + case RDT_BINARY: + buff_value = va_arg(ap, char*); + uint_value = va_arg(ap, uint64_t*); + *uint_value = value->binary.len; + min_len = value->binary.len > *uint_value ? *uint_value : value->binary.len; + memcpy( + buff_value, + value->binary.buff, + min_len + ); + + break; + case RDT_STRING: + buff_value = va_arg(ap, char*); + uint_value = va_arg(ap, uint64_t*); + min_len = value->string.len + 1 > *uint_value ? *uint_value : value->string.len + 1; + *uint_value = min_len; + memcpy( + buff_value, + value->string.buff, + min_len + ); + break; + default: + rv = -1; + goto clean; + } + } + +clean: + va_end(ap); + rmsgpack_dom_value_free(&map); + return 0; +} + diff --git a/rarchdb/rmsgpack_dom.h b/rarchdb/rmsgpack_dom.h new file mode 100644 index 0000000000..b843326786 --- /dev/null +++ b/rarchdb/rmsgpack_dom.h @@ -0,0 +1,56 @@ +#ifndef __RARCHDB_MSGPACK_DOM_H__ +#define __RARCHDB_MSGPACK_DOM_H__ + +#include + +enum rmsgpack_dom_type { + RDT_NULL = 0, + RDT_BOOL, + RDT_UINT, + RDT_INT, + RDT_STRING, + RDT_BINARY, + RDT_MAP, + RDT_ARRAY +}; + +struct rmsgpack_dom_value { + enum rmsgpack_dom_type type; + union { + uint64_t uint_; + int64_t int_; + struct { + uint32_t len; + char* buff; + } string; + struct { + uint32_t len; + char* buff; + } binary; + int bool_; + struct { + uint32_t len; + struct rmsgpack_dom_pair *items; + } map; + struct { + uint32_t len; + struct rmsgpack_dom_value *items; + } array; + }; +}; + +struct rmsgpack_dom_pair { + struct rmsgpack_dom_value key; + struct rmsgpack_dom_value value; +}; + +void rmsgpack_dom_value_print(struct rmsgpack_dom_value *obj); +void rmsgpack_dom_value_free(struct rmsgpack_dom_value *v); +int rmsgpack_dom_value_cmp(const struct rmsgpack_dom_value *a, const struct rmsgpack_dom_value *b); + +struct rmsgpack_dom_value *rmsgpack_dom_value_map_value(const struct rmsgpack_dom_value *map, const struct rmsgpack_dom_value *key); + +int rmsgpack_dom_read(int fd, struct rmsgpack_dom_value *out); +int rmsgpack_dom_write(int fd, const struct rmsgpack_dom_value *obj); +int rmsgpack_dom_read_into(int fd, ...); +#endif diff --git a/rarchdb/rmsgpack_test.c b/rarchdb/rmsgpack_test.c new file mode 100644 index 0000000000..fab8e52498 --- /dev/null +++ b/rarchdb/rmsgpack_test.c @@ -0,0 +1,191 @@ +#include +#include +#include +#include +#include +#include + +#include "rmsgpack.h" + +struct stub_state +{ + int i; + uint64_t stack[256]; +}; + +static void stub_state_push_map(struct stub_state *s, uint32_t size) +{ + s->i++; + s->stack[s->i] = 1; + s->i++; + s->stack[s->i] = size * 2; + printf("{"); +} + +static void stub_state_push_array(struct stub_state *s, uint32_t size) +{ + s->i++; + s->stack[s->i] = 2; + s->i++; + s->stack[s->i] = size; + printf("["); +} + +static void stub_state_pre_print(struct stub_state *s) +{ +} + +static void stub_state_post_print(struct stub_state *s) +{ + switch(s->stack[s->i - 1]) + { + case 1: + if (s->stack[s->i] % 2 == 0) + { + printf(": "); + s->stack[s->i]--; + } + else if (s->stack[s->i] == 1) + { + printf("}"); + s->i -= 2; + stub_state_post_print(s); + } + else + { + printf(", "); + s->stack[s->i]--; + } + break; + case 2: + if (s->stack[s->i] == 1) + { + printf("]"); + s->i -= 2; + stub_state_post_print(s); + } + else + { + printf(", "); + s->stack[s->i]--; + } + break; + } + +} + +static int stub_read_map_start(uint32_t size, void* data) +{ + stub_state_push_map(data, size); + return 0; +} + +static int stub_read_array_start(uint32_t size, void* data) +{ + stub_state_push_array(data, size); + return 0; +} + +static int stub_read_string(char *s, uint32_t len, void* data) +{ + stub_state_pre_print(data); + printf("'%s'", s); + stub_state_post_print(data); + free(s); + return 0; +} + +static int stub_read_bin(void *s, uint32_t len, void* data) +{ + stub_state_pre_print(data); + printf("b'%s'", s); + stub_state_post_print(data); + free(s); + return 0; +} + +static int stub_read_uint(uint64_t value, void* data) +{ + stub_state_pre_print(data); + printf("%lu", value); + stub_state_post_print(data); + return 0; +} + +static int stub_read_nil(void* data) +{ + stub_state_pre_print(data); + printf("nil"); + stub_state_post_print(data); + return 0; +} +static int stub_read_int(int64_t value, void* data) +{ + stub_state_pre_print(data); + printf("%ld", value); + stub_state_post_print(data); + return 0; +} + +static int stub_read_bool(int value, void* data) +{ + stub_state_pre_print(data); + if (value) + printf("true"); + else + printf("false"); + stub_state_post_print(data); + return 0; +} + +static struct rmsgpack_read_callbacks stub_callbacks = { + stub_read_nil, + stub_read_bool, + stub_read_int, + stub_read_uint, + stub_read_string, + stub_read_bin, + stub_read_map_start, + stub_read_array_start +}; + +int main(void) +{ + int fd; + /* + int fd = open("test.msgpack", O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + int rv = 0; + if (fd == -1) + { + printf("Could not open file: %s", strerror(errno)); + return errno; + } + rmsgpack_write_map_header(fd, 2); + rmsgpack_write_string(fd, "compact", strlen("compact")); + rmsgpack_write_bool(fd, 1); + rmsgpack_write_string(fd, "schema", strlen("schema")); + rmsgpack_write_array_header(fd, 10); + rmsgpack_write_string(fd, "schema", strlen("schema")); + rmsgpack_write_uint(fd, 1<<17); + rmsgpack_write_int(fd, (1<<17) + 1); + rmsgpack_write_int(fd, 4); + rmsgpack_write_int(fd, -3); + rmsgpack_write_int(fd, -22); + rmsgpack_write_int(fd, -35); + rmsgpack_write_int(fd, -421421412); + rmsgpack_write_int(fd, 4214); + rmsgpack_write_int(fd, -4214); + rmsgpack_write_uint(fd, 1<<17); + close(fd); + */ + + struct stub_state state; + state.i = 0; + state.stack[0] = 0; + fd = open("test.msgpack", O_RDONLY); + rmsgpack_read(fd, &stub_callbacks, &state); + printf("\n"); + close(fd); + + return 0; +}