From edacf67e7545af5e2edce63efc16e6cd8c4b10c2 Mon Sep 17 00:00:00 2001 From: Nathan Strong Date: Thu, 18 Oct 2018 11:07:47 -0700 Subject: [PATCH 1/3] Capture CRC content for deferred-loading cores == DETAILS Fixes a bug where content CRC32 is not calculated when content loading is done by the core instead of libretro. This impacts the ability to do accurate content matching on netplay. This notably affects MAME, but is by no means limited to MAME. Change summary: - adds a method to the crc32 implementation that calculates crc32 for a file (as opposed to an in-memory buffer) - fix a minor bug that would print the "core will load its own content" right before attempting to load compressed content - in the actual "core will load its own content" path, calculate the CRC32 and log it before returning == TESTING Tested locally on OSX: - loaded content - started netplay - confirmed CRC showing in netplay data - verified CRC32 against external crc32 tool --- libretro-common/encodings/encoding_crc32.c | 17 +++++++++++++++++ libretro-common/include/encodings/crc32.h | 1 + libretro-common/streams/file_stream.c | 3 ++- tasks/task_content.c | 6 +++--- 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/libretro-common/encodings/encoding_crc32.c b/libretro-common/encodings/encoding_crc32.c index 4775e76357..45800e82a6 100644 --- a/libretro-common/encodings/encoding_crc32.c +++ b/libretro-common/encodings/encoding_crc32.c @@ -23,6 +23,8 @@ #include #include #include +#include +#include static const uint32_t crc32_table[256] = { 0x00000000L, 0x77073096L, 0xee0e612cL, 0x990951baL, 0x076dc419L, @@ -88,3 +90,18 @@ uint32_t encoding_crc32(uint32_t crc, const uint8_t *buf, size_t len) return crc ^ 0xffffffff; } + +uint32_t file_crc32(uint32_t crc, const char *path) { + if(path == NULL) + return 0; + + void *file_bytes = NULL; + int64_t file_len = 0; + + if(filestream_read_file(path, &file_bytes, &file_len)) { + crc = encoding_crc32(crc, (uint8_t *)file_bytes, file_len); + free(file_bytes); + return crc; + } + return 0; +} diff --git a/libretro-common/include/encodings/crc32.h b/libretro-common/include/encodings/crc32.h index 76fbb5475e..25e926276f 100644 --- a/libretro-common/include/encodings/crc32.h +++ b/libretro-common/include/encodings/crc32.h @@ -31,6 +31,7 @@ RETRO_BEGIN_DECLS uint32_t encoding_crc32(uint32_t crc, const uint8_t *buf, size_t len); +uint32_t file_crc32(uint32_t crc, const char *path); RETRO_END_DECLS diff --git a/libretro-common/streams/file_stream.c b/libretro-common/streams/file_stream.c index 680a809ec5..b24102ff3b 100644 --- a/libretro-common/streams/file_stream.c +++ b/libretro-common/streams/file_stream.c @@ -482,7 +482,8 @@ int filestream_close(RFILE *stream) * * Read the contents of a file into @buf. * - * Returns: number of items read, -1 on error. + * Returns: 1 on success, 0 on failure + * In the error case, the dereferenced buf is set to NULL and the len is set to -1. */ int64_t filestream_read_file(const char *path, void **buf, int64_t *len) { diff --git a/tasks/task_content.c b/tasks/task_content.c index c2df099419..c366ab1673 100644 --- a/tasks/task_content.c +++ b/tasks/task_content.c @@ -603,9 +603,6 @@ static bool content_file_load( } else { - RARCH_LOG("%s\n", - msg_hash_to_str( - MSG_CONTENT_LOADING_SKIPPED_IMPLEMENTATION_WILL_DO_IT)); #ifdef HAVE_COMPRESSION if ( !content_ctx->block_extract @@ -619,6 +616,9 @@ static bool content_file_load( goto error; #endif } + RARCH_LOG("%s\n", msg_hash_to_str(MSG_CONTENT_LOADING_SKIPPED_IMPLEMENTATION_WILL_DO_IT)); + content_rom_crc = file_crc32(0, path); + RARCH_LOG("CRC32: 0x%x .\n", (unsigned)content_rom_crc); } load_info.content = content; From d03c0be71d4ec1920c9edb4f2f3b0fcab0fcc430 Mon Sep 17 00:00:00 2001 From: Nathan Strong Date: Thu, 18 Oct 2018 16:38:02 -0700 Subject: [PATCH 2/3] Rewrite file hasher to limit amount hashed == DETAILS Since the content file could potentially be huge, hashing the whole thing at runtime may take a really long time. Plus, it was loading the whole file into RAM at once. Now, we only load 1MB at a time and hash up to the first 64MB. == TESTING I don't have any large content files to test it with, but I tested it with a small one and confirmed that the hash was correct. --- libretro-common/encodings/encoding_crc32.c | 47 ++++++++++++++++++---- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/libretro-common/encodings/encoding_crc32.c b/libretro-common/encodings/encoding_crc32.c index 45800e82a6..4dd932db6a 100644 --- a/libretro-common/encodings/encoding_crc32.c +++ b/libretro-common/encodings/encoding_crc32.c @@ -91,17 +91,50 @@ uint32_t encoding_crc32(uint32_t crc, const uint8_t *buf, size_t len) return crc ^ 0xffffffff; } +#define CRC32_BUFFER_SIZE 1048576 +#define CRC32_MAX_MB 64 + +/** + * Calculate a CRC32 from the first part of the given file. + * "first part" being the first (CRC32_BUFFER_SIZE * CRC32_MAX_MB) + * bytes. + * TODO: maybe make these numbers configurable? + * + * Returns: the crc32, or 0 if there was an error. + */ uint32_t file_crc32(uint32_t crc, const char *path) { if(path == NULL) return 0; - void *file_bytes = NULL; - int64_t file_len = 0; + RFILE *file = NULL; + unsigned char *buf = NULL; + int i, nread; - if(filestream_read_file(path, &file_bytes, &file_len)) { - crc = encoding_crc32(crc, (uint8_t *)file_bytes, file_len); - free(file_bytes); - return crc; + file = filestream_open(path, RETRO_VFS_FILE_ACCESS_READ, 0); + if(file == NULL) + goto error; + + buf = (char *)malloc(CRC32_BUFFER_SIZE); + if(buf == NULL) + goto error; + + for(i = 0; i < CRC32_MAX_MB; i++) { + nread = filestream_read(file, buf, CRC32_BUFFER_SIZE); + if(nread < 0) + goto error; + + crc = encoding_crc32(crc, buf, nread); + if(filestream_eof(file)) + break; } - return 0; + free(buf); + filestream_close(file); + return crc; + + error: + if(buf) + free(buf); + if(file) + filestream_close(file); + return 0; } From dc181968442d03c403610fe6636981438016f31b Mon Sep 17 00:00:00 2001 From: Nathan Strong Date: Thu, 18 Oct 2018 16:44:30 -0700 Subject: [PATCH 3/3] get rid of TODO --- libretro-common/encodings/encoding_crc32.c | 1 - 1 file changed, 1 deletion(-) diff --git a/libretro-common/encodings/encoding_crc32.c b/libretro-common/encodings/encoding_crc32.c index 4dd932db6a..85881e521b 100644 --- a/libretro-common/encodings/encoding_crc32.c +++ b/libretro-common/encodings/encoding_crc32.c @@ -98,7 +98,6 @@ uint32_t encoding_crc32(uint32_t crc, const uint8_t *buf, size_t len) * Calculate a CRC32 from the first part of the given file. * "first part" being the first (CRC32_BUFFER_SIZE * CRC32_MAX_MB) * bytes. - * TODO: maybe make these numbers configurable? * * Returns: the crc32, or 0 if there was an error. */