Add CRC calculation function that works with archives with or without a path inside (first file is used if no path)

Add all archive's contents to database scan list when scanning files and directories

Allow scanning a single file that is an archive

Remove unnecessary prototypes from archive_file.h

Simplify retrieving of CRCs from archives when scanning

database_info.c

@@ -20,6 +20,8 @@
 
 #include <compat/strl.h>
 #include <retro_endianness.h>
+#include <file/file_path.h>
+#include <file/archive_file.h>
 
 #include "list_special.h"
 #include "database_info.h"
@@ -398,6 +400,7 @@ database_info_handle_t *database_info_dir_init(const char *dir,
 {
    database_info_handle_t     *db  = (database_info_handle_t*)
       calloc(1, sizeof(*db));
+   int i = 0;
 
    if (!db)
       return NULL;
@@ -411,6 +414,47 @@ database_info_handle_t *database_info_dir_init(const char *dir,
    db->status         = DATABASE_STATUS_ITERATE;
    db->type           = type;
 
+   if (db->list->size > 0)
+   {
+      for (i = 0; i < db->list->size; i++)
+      {
+         const char *path = db->list->elems[i].data;
+
+         if (path_is_compressed_file(path))
+         {
+            struct string_list *archive_list =
+                  file_archive_get_file_list(path, NULL);
+
+            if (archive_list && archive_list->size > 0)
+            {
+               int i = 0;
+
+               for (i = 0; i < archive_list->size; i++)
+               {
+                  char new_path[PATH_MAX_LENGTH] = {0};
+                  size_t path_len = strlen(path);
+
+                  strlcpy(new_path, path, sizeof(new_path));
+
+                  if (path_len + strlen(archive_list->elems[i].data)
+                         + 1 < PATH_MAX_LENGTH)
+                  {
+                     new_path[path_len] = '#';
+                     strlcpy(new_path + path_len + 1,
+                                archive_list->elems[i].data,
+                                sizeof(new_path) - path_len);
+                  }
+
+                  string_list_append(db->list, new_path,
+                                        archive_list->elems[i].attr);
+               }
+
+               string_list_free(archive_list);
+            }
+         }
+      }
+   }
+
    return db;
 
 error:
@@ -436,6 +480,39 @@ database_info_handle_t *database_info_file_init(const char *path,
 
    string_list_append(db->list, path, attr);
 
+   if (path_is_compressed_file(path))
+   {
+      struct string_list *archive_list =
+            file_archive_get_file_list(path, NULL);
+
+      if (archive_list && archive_list->size > 0)
+      {
+         int i = 0;
+
+         for (i = 0; i < archive_list->size; i++)
+         {
+            char new_path[PATH_MAX_LENGTH] = {0};
+            size_t path_len = strlen(path);
+
+            strlcpy(new_path, path, sizeof(new_path));
+
+            if (path_len + strlen(archive_list->elems[i].data)
+                   + 1 < PATH_MAX_LENGTH)
+            {
+               new_path[path_len] = '#';
+               strlcpy(new_path + path_len + 1,
+                          archive_list->elems[i].data,
+                          sizeof(new_path) - path_len);
+            }
+
+            string_list_append(db->list, new_path,
+                                  archive_list->elems[i].attr);
+         }
+
+         string_list_free(archive_list);
+      }
+   }
+
    db->list_ptr       = 0;
    db->status         = DATABASE_STATUS_ITERATE;
    db->type           = type;

libretro-common/file/archive_file.c

@@ -58,7 +58,7 @@ struct file_archive_file_data
 
 #ifdef HAVE_MMAP
 /* Closes, unmaps and frees. */
-void file_archive_free(file_archive_file_data_t *data)
+static void file_archive_free(file_archive_file_data_t *data)
 {
    if (!data)
       return;
@@ -70,7 +70,7 @@ void file_archive_free(file_archive_file_data_t *data)
    free(data);
 }
 
-const uint8_t *file_archive_data(file_archive_file_data_t *data)
+static const uint8_t *file_archive_data(file_archive_file_data_t *data)
 {
    if (!data)
       return NULL;
@@ -119,7 +119,7 @@ error:
 #else
 
 /* Closes, unmaps and frees. */
-void file_archive_free(file_archive_file_data_t *data)
+static void file_archive_free(file_archive_file_data_t *data)
 {
    if (!data)
       return;
@@ -128,7 +128,7 @@ void file_archive_free(file_archive_file_data_t *data)
    free(data);
 }
 
-const uint8_t *file_archive_data(file_archive_file_data_t *data)
+static const uint8_t *file_archive_data(file_archive_file_data_t *data)
 {
    if (!data)
       return NULL;
@@ -269,7 +269,7 @@ static int file_archive_extract_cb(const char *name, const char *valid_exts,
    return 1;
 }
 
-int file_archive_parse_file_init(file_archive_transfer_t *state,
+static int file_archive_parse_file_init(file_archive_transfer_t *state,
       const char *file)
 {
    char *last                 = NULL;
@@ -387,7 +387,10 @@ int file_archive_parse_file_iterate(
          if (file_archive_parse_file_init(state, file) == 0)
          {
             if (userdata)
+            {
                userdata->context = state->stream;
+               strlcpy(userdata->archive_path, file, sizeof(userdata->archive_path));
+            }
             state->type = ARCHIVE_TRANSFER_ITERATE;
          }
          else
@@ -440,7 +443,7 @@ int file_archive_parse_file_iterate(
 }
 
 /**
- * file_archive_parse_file:
+ * file_archive_walk:
  * @file                        : filename path of archive
  * @valid_exts                  : Valid extensions of archive to be parsed.
  *                                If NULL, allow all.
@@ -452,7 +455,7 @@ int file_archive_parse_file_iterate(
  *
  * Returns: true (1) on success, otherwise false (0).
  **/
-static bool file_archive_parse_file(const char *file, const char *valid_exts,
+static bool file_archive_walk(const char *file, const char *valid_exts,
       file_archive_file_cb file_cb, struct archive_extract_userdata *userdata)
 {
    file_archive_transfer_t state = {0};
@@ -505,7 +508,7 @@ bool file_archive_extract_file(
 {
    struct string_list *list             = NULL;
    bool ret                             = true;
-   struct archive_extract_userdata userdata = {0};
+   struct archive_extract_userdata userdata = {{0}};
 
    /* We cannot extract if the libretro
     * implementation does not have any valid extensions. */
@@ -519,7 +522,6 @@ bool file_archive_extract_file(
       goto end;
    }
 
-   userdata.archive_path         = archive_path;
    userdata.archive_path_size    = archive_path_size;
    userdata.extraction_directory = extraction_directory;
    userdata.ext                  = list;
@@ -527,7 +529,7 @@ bool file_archive_extract_file(
    userdata.context              = NULL;
    userdata.list_only            = false;
 
-   if (!file_archive_parse_file(archive_path, valid_exts,
+   if (!file_archive_walk(archive_path, valid_exts,
             file_archive_extract_cb, &userdata))
    {
       /* Parsing file archive failed. */
@@ -563,27 +565,30 @@ end:
 struct string_list *file_archive_get_file_list(const char *path,
       const char *valid_exts)
 {
-   struct archive_extract_userdata userdata = {0};
+   struct archive_extract_userdata userdata = {{0}};
+
+#ifdef HAVE_COMPRESSION
+   if (!path_is_compressed_file(path))
+      return NULL;
+#else
+   return NULL;
+#endif
+
    userdata.list_only = true;
-   userdata.archive_path = strdup(path);
+   strlcpy(userdata.archive_path, path, sizeof(userdata.archive_path));
 
    userdata.list = string_list_new();
 
    if (!userdata.list)
       goto error;
 
-   if (!file_archive_parse_file(path, valid_exts,
+   if (!file_archive_walk(path, valid_exts,
             file_archive_get_file_list_cb, &userdata))
       goto error;
 
-   if (userdata.archive_path)
-      free(userdata.archive_path);
-
    return userdata.list;
 
 error:
-   if (userdata.archive_path)
-      free(userdata.archive_path);
    if (userdata.list)
       string_list_free(userdata.list);
    return NULL;
@@ -608,7 +613,7 @@ bool file_archive_perform_mode(const char *path, const char *valid_exts,
             handle.backend = file_archive_get_file_backend(userdata->archive_path);
             handle.stream = userdata->context;
 
-            if (!handle.backend->stream_decompress_data_to_file_init(&handle,
+            if (!handle.backend || !handle.backend->stream_decompress_data_to_file_init(&handle,
                      cdata, csize, size))
                goto error;
 
@@ -633,6 +638,50 @@ error:
    return false;
 }
 
+/**
+ * file_archive_filename_split:
+ * @str              : filename to turn into a string list
+ *
+ * Creates a new string list based on filename @path, delimited by a hash (#).
+ *
+ * Returns: new string list if successful, otherwise NULL.
+ */
+static struct string_list *file_archive_filename_split(const char *path)
+{
+   union string_list_elem_attr attr;
+   struct string_list *list = string_list_new();
+   const char *delim        = NULL;
+
+   memset(&attr, 0, sizeof(attr));
+
+   delim = path_get_archive_delim(path);
+
+   if (delim)
+   {
+      /* add archive path to list first */
+      if (!string_list_append_n(list, path, delim - path, attr))
+         goto error;
+
+      /* now add the path within the archive */
+      delim++;
+
+      if (*delim)
+      {
+         if (!string_list_append(list, delim, attr))
+            goto error;
+      }
+   }
+   else
+      if (!string_list_append(list, path, attr))
+         goto error;
+
+   return list;
+
+error:
+   string_list_free(list);
+   return NULL;
+}
+
 /* Generic compressed file loader.
  * Extracts to buf, unless optional_filename != 0
  * Then extracts to optional_filename and leaves buf alone.
@@ -685,60 +734,6 @@ error:
    return 0;
 }
 
-struct string_list *file_archive_file_list_new(const char *path,
-      const char* ext)
-{
-#ifdef HAVE_COMPRESSION
-   if (path_is_compressed_file(path))
-      return file_archive_get_file_list(path, ext);
-#endif
-   return NULL;
-}
-
-/**
- * file_archive_filename_split:
- * @str              : filename to turn into a string list
- *
- * Creates a new string list based on filename @path, delimited by a hash (#).
- *
- * Returns: new string list if successful, otherwise NULL.
- */
-struct string_list *file_archive_filename_split(const char *path)
-{
-   union string_list_elem_attr attr;
-   struct string_list *list = string_list_new();
-   const char *delim        = NULL;
-
-   memset(&attr, 0, sizeof(attr));
-
-   delim = path_get_archive_delim(path);
-
-   if (delim)
-   {
-      /* add archive path to list first */
-      if (!string_list_append_n(list, path, delim - path, attr))
-         goto error;
-
-      /* now add the path within the archive */
-      delim++;
-
-      if (*delim)
-      {
-         if (!string_list_append(list, delim, attr))
-            goto error;
-      }
-   }
-   else
-      if (!string_list_append(list, path, attr))
-         goto error;
-
-   return list;
-
-error:
-   string_list_free(list);
-   return NULL;
-}
-
 const struct file_archive_file_backend *file_archive_get_zlib_file_backend(void)
 {
 #ifdef HAVE_ZLIB
@@ -785,3 +780,74 @@ const struct file_archive_file_backend* file_archive_get_file_backend(const char
 
    return NULL;
 }
+
+/**
+ * file_archive_get_file_crc32:
+ * @path                         : filename path of archive
+ *
+ * Returns: CRC32 of the specified file in the archive, otherwise 0.
+ * If no path within the archive is specified, the first
+ * file found inside is used.
+ **/
+uint32_t file_archive_get_file_crc32(const char *path)
+{
+   const struct file_archive_file_backend *backend = file_archive_get_file_backend(path);
+   file_archive_transfer_t state = {0};
+   struct archive_extract_userdata userdata = {{0}};
+   bool returnerr = false;
+   bool contains_compressed = false;
+   const char *archive_path = NULL;
+
+   if (!backend)
+      return 0;
+
+   contains_compressed = path_contains_compressed_file(path);
+
+   if (contains_compressed)
+   {
+      archive_path = path_get_archive_delim(path);
+
+      /* move pointer right after the delimiter to give us the path */
+      if (archive_path)
+         archive_path += 1;
+   }
+
+   state.type = ARCHIVE_TRANSFER_INIT;
+
+   /* Initialize and open archive first.
+      Sets next state type to ITERATE. */
+   file_archive_parse_file_iterate(&state,
+            &returnerr, path, NULL, NULL,
+            &userdata);
+
+   for (;;)
+   {
+      /* Now find the first file in the archive. */
+      if (state.type == ARCHIVE_TRANSFER_ITERATE)
+         file_archive_parse_file_iterate(&state,
+                  &returnerr, path, NULL, NULL,
+                  &userdata);
+
+      /* If no path specified within archive, stop after
+       * finding the first file.
+       */
+      if (!contains_compressed)
+         break;
+
+      /* Stop when the right file in the archive is found. */
+      if (archive_path)
+      {
+         if (string_is_equal(userdata.extracted_file_path, archive_path))
+            break;
+      }
+      else
+         break;
+   }
+
+   file_archive_parse_file_iterate_stop(&state);
+
+   if (userdata.crc)
+      return userdata.crc;
+
+   return 0;
+}

libretro-common/file/archive_file_7z.c

@@ -393,8 +393,9 @@ static int sevenzip_parse_file_iterate_step(file_archive_transfer_t *state,
       return ret;
 
    userdata->extracted_file_path = filename;
+   userdata->crc = checksum;
 
-   if (!file_cb(filename, valid_exts, cdata, cmode,
+   if (file_cb && !file_cb(filename, valid_exts, cdata, cmode,
             csize, size, checksum, userdata))
       return 0;
 

libretro-common/file/archive_file_zlib.c

@@ -331,7 +331,7 @@ static int zip_file_read(
    file_archive_transfer_t zlib;
    bool returnerr = true;
    int ret        = 0;
-   struct archive_extract_userdata userdata = {0};
+   struct archive_extract_userdata userdata = {{0}};
 
    zlib.type      = ARCHIVE_TRANSFER_INIT;
 
@@ -448,8 +448,9 @@ static int zip_parse_file_iterate_step(file_archive_transfer_t *state,
       return ret;
 
    userdata->extracted_file_path = filename;
+   userdata->crc = checksum;
 
-   if (!file_cb(filename, valid_exts, cdata, cmode,
+   if (file_cb && !file_cb(filename, valid_exts, cdata, cmode,
             csize, size, checksum, userdata))
       return 0;
 

libretro-common/file/file_path.c

@@ -94,7 +94,7 @@ end:
  * after a compression extension is considered.
  *
  * Returns: pointer to the delimiter in the path if it contains
- * a compressed file, otherwise NULL.
+ * a path inside a compressed file, otherwise NULL.
  */
 const char *path_get_archive_delim(const char *path)
 {

libretro-common/include/file/archive_file.h

@@ -96,7 +96,7 @@ typedef struct
 
 struct archive_extract_userdata
 {
-   char *archive_path;
+   char archive_path[PATH_MAX_LENGTH];
    char *first_extracted_file_path;
    char *extracted_file_path;
    const char *extraction_directory;
@@ -191,29 +191,25 @@ bool file_archive_perform_mode(const char *name, const char *valid_exts,
       const uint8_t *cdata, unsigned cmode, uint32_t csize, uint32_t size,
       uint32_t crc32, struct archive_extract_userdata *userdata);
 
-void file_archive_deflate_init(void *data, int level);
-
 int file_archive_compressed_read(
       const char* path, void **buf,
       const char* optional_filename, ssize_t *length);
 
-struct string_list* file_archive_file_list_new(const char *path,
-      const char *ext);
-
-struct string_list* file_archive_filename_split(const char *path);
-
-const uint8_t* file_archive_data(file_archive_file_data_t *data);
-
-int file_archive_parse_file_init(file_archive_transfer_t *state,
-      const char *file);
-
-void file_archive_free(file_archive_file_data_t *data);
-
 const struct file_archive_file_backend* file_archive_get_zlib_file_backend(void);
 const struct file_archive_file_backend* file_archive_get_7z_file_backend(void);
 
 const struct file_archive_file_backend* file_archive_get_file_backend(const char *path);
 
+/**
+ * file_archive_get_file_crc32:
+ * @path                         : filename path of archive
+ *
+ * Returns: CRC32 of the specified file in the archive, otherwise 0.
+ * If no path within the archive is specified, the first
+ * file found inside is used.
+ **/
+uint32_t file_archive_get_file_crc32(const char *path);
+
 extern const struct file_archive_file_backend zlib_backend;
 extern const struct file_archive_file_backend sevenzip_backend;
 

menu/cbs/menu_cbs_ok.c

@@ -3843,6 +3843,9 @@ static int menu_cbs_init_bind_ok_compare_type(menu_file_list_cbs_t *cbs,
                   BIND_ACTION_OK(cbs, action_ok_compressed_archive_push_detect_core);
                   break;
                case MENU_LABEL_SCAN_FILE:
+#ifdef HAVE_LIBRETRODB
+                  BIND_ACTION_OK(cbs, action_ok_scan_file);
+#endif
                   break;
                default:
                   BIND_ACTION_OK(cbs, action_ok_compressed_archive_push);

menu/menu_displaylist.c

@@ -3378,9 +3378,7 @@ static int menu_displaylist_parse_generic(
       filter_ext = true;
 
    if (path_is_compressed)
-   {
+      str_list = file_archive_get_file_list(info->path, info->exts);
-      str_list = file_archive_file_list_new(info->path, info->exts);
-   }
    else
       str_list = dir_list_new(info->path,
             filter_ext ? info->exts : NULL,

tasks/task_database.c

@@ -60,24 +60,6 @@ typedef struct db_handle
    unsigned status;
 } db_handle_t;
 
-
-#ifdef HAVE_COMPRESSION
-static int archive_compare_crc32(const char *name, const char *valid_exts,
-      const uint8_t *cdata, unsigned cmode, uint32_t csize, uint32_t size,
-      uint32_t crc32, struct archive_extract_userdata *userdata)
-{
-   userdata->crc = crc32;
-
-   strlcpy(userdata->archive_name, userdata->extracted_file_path, sizeof(userdata->archive_name));
-
-#if 0
-   RARCH_LOG("Going to compare CRC 0x%x for %s\n", crc32, name);
-#endif
-
-   return 1;
-}
-#endif
-
 static int task_database_iterate_start(database_info_handle_t *db,
       const char *name)
 {
@@ -182,10 +164,8 @@ static int task_database_iterate_playlist(
    {
       case FILE_TYPE_COMPRESSED:
 #ifdef HAVE_COMPRESSION
-         db->type = DATABASE_TYPE_ITERATE_ARCHIVE;
+         db->type = DATABASE_TYPE_CRC_LOOKUP;
-         memset(&db->state, 0, sizeof(file_archive_transfer_t));
+         /* first check crc of archive itself */
-         db_state->archive_name[0] = '\0';
-         db->state.type = ARCHIVE_TRANSFER_INIT;
          return file_get_crc(db_state, name, &db_state->archive_crc);
 #else
          break;
@@ -404,37 +384,11 @@ static int task_database_iterate_playlist_archive(
 {
    bool returnerr = true;
 #ifdef HAVE_COMPRESSION
-   struct archive_extract_userdata userdata = {0};
-
    if (db_state->crc != 0)
       return task_database_iterate_crc_lookup(
             db_state, db, db_state->archive_name);
 
-   userdata.crc = db_state->crc;
+   db_state->crc = file_archive_get_file_crc32(name);
-   userdata.archive_path = strdup(name);
-
-   if (db->state.type == ARCHIVE_TRANSFER_INIT)
-      file_archive_parse_file_iterate(&db->state,
-               &returnerr, name, NULL, NULL,
-               &userdata);
-
-   if (file_archive_parse_file_iterate(&db->state,
-            &returnerr, name, NULL, archive_compare_crc32,
-            &userdata))
-   {
-      if (userdata.archive_path)
-         free(userdata.archive_path);
-      return 0;
-   }
-
-   if (userdata.crc)
-   {
-      db_state->crc = userdata.crc;
-      file_archive_parse_file_iterate_stop(&db->state);
-   }
-
-   if (userdata.archive_path)
-      free(userdata.archive_path);
 #endif
 
    return 1;
@@ -547,6 +501,10 @@ static int task_database_iterate(database_state_handle_t *db_state,
    if (!name)
       return 0;
 
+   if (db->type == DATABASE_TYPE_ITERATE)
+      if (path_contains_compressed_file(name))
+         db->type = DATABASE_TYPE_ITERATE_ARCHIVE;
+
    switch (db->type)
    {
       case DATABASE_TYPE_ITERATE:

tasks/task_decompress.c

@@ -20,6 +20,7 @@
 #include <file/archive_file.h>
 #include <retro_miscellaneous.h>
 #include <retro_stat.h>
+#include <compat/strl.h>
 
 #include "tasks_internal.h"
 #include "../verbosity.h"
@@ -146,10 +147,10 @@ static void task_decompress_handler(retro_task_t *task)
    int ret;
    bool retdec             = false;
    decompress_state_t *dec = (decompress_state_t*)task->state;
-   struct archive_extract_userdata userdata = {0};
+   struct archive_extract_userdata userdata = {{0}};
 
    userdata.dec            = dec;
-   userdata.archive_path   = dec->source_file;
+   strlcpy(userdata.archive_path, dec->source_file, sizeof(userdata.archive_path));
 
    ret                     = file_archive_parse_file_iterate(&dec->archive,
          &retdec, dec->source_file,
@@ -171,9 +172,9 @@ static void task_decompress_handler_target_file(retro_task_t *task)
    bool retdec;
    int ret;
    decompress_state_t *dec = (decompress_state_t*)task->state;
-   struct archive_extract_userdata userdata = {0};
+   struct archive_extract_userdata userdata = {{0}};
 
-   userdata.archive_path = dec->source_file;
+   strlcpy(userdata.archive_path, dec->source_file, sizeof(userdata.archive_path));
 
    ret = file_archive_parse_file_iterate(&dec->archive,
          &retdec, dec->source_file,
@@ -195,10 +196,10 @@ static void task_decompress_handler_subdir(retro_task_t *task)
    int ret;
    bool retdec;
    decompress_state_t *dec = (decompress_state_t*)task->state;
-   struct archive_extract_userdata userdata = {0};
+   struct archive_extract_userdata userdata = {{0}};
 
    userdata.dec            = dec;
-   userdata.archive_path   = dec->source_file;
+   strlcpy(userdata.archive_path, dec->source_file, sizeof(userdata.archive_path));
 
    ret                     = file_archive_parse_file_iterate(&dec->archive,
          &retdec, dec->source_file,