Merge pull request #9808 from jdgleaver/manual-scan-archives

(Manual Content Scanner) Add option to scan inside archives

intl/msg_hash_lbl.h

@@ -2123,6 +2123,8 @@ MSG_HASH(MENU_ENUM_LABEL_DEFERRED_DROPDOWN_BOX_LIST_MANUAL_CONTENT_SCAN_CORE_NAM
       "deferred_dropdown_box_list_manual_content_scan_core_name")
 MSG_HASH(MENU_ENUM_LABEL_MANUAL_CONTENT_SCAN_FILE_EXTS,
       "manual_content_scan_file_exts")
+MSG_HASH(MENU_ENUM_LABEL_MANUAL_CONTENT_SCAN_SEARCH_ARCHIVES,
+      "manual_content_scan_search_archives")
 MSG_HASH(MENU_ENUM_LABEL_MANUAL_CONTENT_SCAN_OVERWRITE,
       "manual_content_scan_overwrite")
 MSG_HASH(MENU_ENUM_LABEL_MANUAL_CONTENT_SCAN_START,

intl/msg_hash_us.h

@@ -10088,6 +10088,14 @@ MSG_HASH(
     MENU_ENUM_SUBLABEL_MANUAL_CONTENT_SCAN_FILE_EXTS,
     "Space-delimited list of file types to include in the scan. If empty, includes all files - or if a core is specified, all files supported by the core."
     )
+MSG_HASH(
+    MENU_ENUM_LABEL_VALUE_MANUAL_CONTENT_SCAN_SEARCH_ARCHIVES,
+    "Scan Inside Archives"
+    )
+MSG_HASH(
+    MENU_ENUM_SUBLABEL_MANUAL_CONTENT_SCAN_SEARCH_ARCHIVES,
+    "When enabled, archive files (.zip, .7z, etc.) will be searched for valid/supported content. May have a significant impact on scan performance."
+    )
 MSG_HASH(
     MENU_ENUM_LABEL_VALUE_MANUAL_CONTENT_SCAN_OVERWRITE,
     "Overwrite Existing Playlist"

manual_content_scan.c

@@ -21,6 +21,7 @@
  */
 
 #include <file/file_path.h>
+#include <file/archive_file.h>
 #include <string/stdstring.h>
 #include <lists/dir_list.h>
 #include <retro_miscellaneous.h>
@@ -47,6 +48,7 @@ typedef struct
    char file_exts_custom[PATH_MAX_LENGTH];
    enum manual_content_scan_system_name_type system_name_type;
    enum manual_content_scan_core_type core_type;
+   bool search_archives;
    bool overwrite_playlist;
 } scan_settings_t;
 
@@ -70,6 +72,7 @@ static scan_settings_t scan_settings = {
    "",                                          /* file_exts_custom */
    MANUAL_CONTENT_SCAN_SYSTEM_NAME_CONTENT_DIR, /* system_name_type */
    MANUAL_CONTENT_SCAN_CORE_DETECT,             /* core_type */
+   false,                                       /* search_archives */
    false                                        /* overwrite_playlist */
 };
 
@@ -107,6 +110,13 @@ size_t manual_content_scan_get_file_exts_custom_size(void)
    return sizeof(scan_settings.file_exts_custom);
 }
 
+/* Returns a pointer to the internal
+ * 'search_archives' bool */
+bool *manual_content_scan_get_search_archives_ptr(void)
+{
+   return &scan_settings.search_archives;
+}
+
 /* Returns a pointer to the internal
  * 'overwrite_playlist' bool */
 bool *manual_content_scan_get_overwrite_playlist_ptr(void)
@@ -741,33 +751,18 @@ bool manual_content_scan_get_task_config(manual_content_scan_task_config_t *task
          return false;
    }
 
-   /* Get file extensions list
+   /* Get file extensions list */
-    * > Note that compressed files are included by
-    *   default, regardless of extension filter
-    *   (since these can always be handled by the
-    *   frontend) */
-   task_config->include_compressed_content = true;
-
    if (!string_is_empty(scan_settings.file_exts_custom))
-   {
       strlcpy(
             task_config->file_exts,
             scan_settings.file_exts_custom,
             sizeof(task_config->file_exts));
-
-      /* User has explicitly specified which file
-       * types are allowed - have to exclude compressed
-       * content when calling dir_list_new() */
-      task_config->include_compressed_content = false;
-   }
    else if (scan_settings.core_type == MANUAL_CONTENT_SCAN_CORE_SET)
-   {
       if (!string_is_empty(scan_settings.file_exts_core))
          strlcpy(
                task_config->file_exts,
                scan_settings.file_exts_core,
                sizeof(task_config->file_exts));
-   }
 
    /* Our extension lists are space delimited
     * > dir_list_new() expects vertical bar
@@ -775,6 +770,9 @@ bool manual_content_scan_get_task_config(manual_content_scan_task_config_t *task
    if (!string_is_empty(task_config->file_exts))
       string_replace_all_chars(task_config->file_exts, ' ', '|');
 
+   /* Copy 'search inside archives' setting */
+   task_config->search_archives = scan_settings.search_archives;
+
    /* Copy 'overwrite playlist' setting */
    task_config->overwrite_playlist = scan_settings.overwrite_playlist;
 
@@ -789,6 +787,7 @@ struct string_list *manual_content_scan_get_content_list(manual_content_scan_tas
 {
    struct string_list *dir_list = NULL;
    bool filter_exts;
+   bool include_compressed;
 
    /* Sanity check */
    if (!task_config)
@@ -797,15 +796,30 @@ struct string_list *manual_content_scan_get_content_list(manual_content_scan_tas
    if (string_is_empty(task_config->content_dir))
       goto error;
 
+   /* Check whether files should be filtered by
+    * extension */
+   filter_exts = !string_is_empty(task_config->file_exts);
+
+   /* Check whether compressed files should be
+    * included in the directory list
+    * > If compressed files are already listed in
+    *   the 'file_exts' string, they will be included
+    *   automatically
+    * > If we don't have a 'file_exts' list, then all
+    *   files must be included regardless of type
+    * > If user has enabled 'search inside archives',
+    *   then compressed files must of course be included */
+   include_compressed = (!filter_exts || task_config->search_archives);
+
    /* Get directory listing
     * > Exclude directories and hidden files
     * > Scan recursively */
    dir_list = dir_list_new(
          task_config->content_dir,
-         string_is_empty(task_config->file_exts) ? NULL : task_config->file_exts,
+         filter_exts ? task_config->file_exts : NULL,
          false, /* include_dirs */
          false, /* include_hidden */
-         task_config->include_compressed_content,
+         include_compressed,
          true   /* recursive */
    );
 
@@ -830,22 +844,118 @@ error:
    return NULL;
 }
 
+/* Converts specified content path string to 'real'
+ * file path for use in playlists - i.e. handles
+ * identification of content *inside* archive files.
+ * Returns false if specified content is invalid. */
+static bool manual_content_scan_get_playlist_content_path(
+      manual_content_scan_task_config_t *task_config,
+      const char *content_path, int content_type,
+      char *playlist_content_path, size_t len)
+{
+   struct string_list *archive_list = NULL;
+
+   /* Sanity check */
+   if (!task_config || string_is_empty(content_path))
+      return false;
+
+   if (!path_is_valid(content_path))
+      return false;
+
+   /* In all cases, base content path must be
+    * copied to playlist_content_path */
+   strlcpy(playlist_content_path, content_path, len);
+
+   /* Check whether this is an archive file
+    * requiring special attention... */
+   if ((content_type == RARCH_COMPRESSED_ARCHIVE) &&
+       task_config->search_archives)
+   {
+      bool filter_exts         = !string_is_empty(task_config->file_exts);
+      const char *archive_file = NULL;
+
+      /* Important note:
+       * > If an archive file of a particular type is
+       *   included in the task_config->file_exts list,
+       *   dir_list_new() will assign it a file type of
+       *   RARCH_PLAIN_FILE
+       * > Thus we will only reach this point if
+       *   (a) We are not filtering by extension
+       *   (b) This is an archive file type *not*
+       *       already included in the supported
+       *       extensions list
+       * > These guarantees substantially reduce the
+       *   complexity of the following code... */
+
+      /* Get archive file contents */
+      archive_list = file_archive_get_file_list(
+            content_path, filter_exts ? task_config->file_exts : NULL);
+
+      if (!archive_list)
+         goto error;
+
+      if (archive_list->size < 1)
+         goto error;
+
+      /* Get first file contained in archive */
+      dir_list_sort(archive_list, true);
+      archive_file = archive_list->elems[0].data;
+      if (string_is_empty(archive_file))
+         goto error;
+
+      /* Have to take care to ensure that we don't make
+       * a mess of arcade content...
+       * > If we are filtering by extension, then the
+       *   archive file itself is *not* valid content,
+       *   so link to the first file inside the archive
+       * > If we are not filtering by extension, then:
+       *   - If archive contains one valid file, assume
+       *     it is a compressed ROM
+       *   - If archive contains multiple files, have to
+       *     assume it is MAME/FBA-style content, where
+       *     only the archive itself is valid */
+      if (filter_exts || (archive_list->size == 1))
+      {
+         /* Build path to file inside archive */
+         strlcat(playlist_content_path, "#", len);
+         strlcat(playlist_content_path, archive_file, len);
+      }
+
+      string_list_free(archive_list);
+   }
+
+   return true;
+
+error:
+   if (archive_list)
+      string_list_free(archive_list);
+   return false;
+}
+
 /* Adds specified content to playlist, if not already
  * present */
 void manual_content_scan_add_content_to_playlist(
       manual_content_scan_task_config_t *task_config,
-      playlist_t *playlist, const char *content_path)
+      playlist_t *playlist, const char *content_path,
+      int content_type)
 {
+   char playlist_content_path[PATH_MAX_LENGTH];
+
+   playlist_content_path[0] = '\0';
+
    /* Sanity check */
-   if (!task_config || !playlist || string_is_empty(content_path))
+   if (!task_config || !playlist)
       return;
 
-   if (!path_is_valid(content_path))
+   /* Get 'actual' content path */
+   if (!manual_content_scan_get_playlist_content_path(
+         task_config, content_path, content_type,
+         playlist_content_path, sizeof(playlist_content_path)))
       return;
 
    /* Check whether content is already included
     * in playlist */
-   if (!playlist_entry_exists(playlist, content_path))
+   if (!playlist_entry_exists(playlist, playlist_content_path))
    {
       struct playlist_entry entry = {0};
       char label[PATH_MAX_LENGTH];
@@ -854,7 +964,7 @@ void manual_content_scan_add_content_to_playlist(
 
       /* Get entry label */
       fill_short_pathname_representation(
-            label, content_path, sizeof(label));
+            label, playlist_content_path, sizeof(label));
 
       if (string_is_empty(label))
          return;
@@ -862,7 +972,7 @@ void manual_content_scan_add_content_to_playlist(
       /* Configure playlist entry
        * > The push function reads our entry as const,
        *   so these casts are safe */
-      entry.path      = (char*)content_path;
+      entry.path      = (char*)playlist_content_path;
       entry.label     = label;
       entry.core_path = (char*)"DETECT";
       entry.core_name = (char*)"DETECT";

manual_content_scan.h

@@ -66,8 +66,8 @@ typedef struct
    char core_path[PATH_MAX_LENGTH];
    char file_exts[PATH_MAX_LENGTH];
    bool core_set;
+   bool search_archives;
    bool overwrite_playlist;
-   bool include_compressed_content;
 } manual_content_scan_task_config_t;
 
 /*****************/
@@ -96,6 +96,10 @@ char *manual_content_scan_get_file_exts_custom_ptr(void);
  * 'file_exts_custom' string */
 size_t manual_content_scan_get_file_exts_custom_size(void);
 
+/* Returns a pointer to the internal
+ * 'search_archives' bool */
+bool *manual_content_scan_get_search_archives_ptr(void);
+
 /* Returns a pointer to the internal
  * 'overwrite_playlist' bool */
 bool *manual_content_scan_get_overwrite_playlist_ptr(void);
@@ -194,7 +198,8 @@ struct string_list *manual_content_scan_get_content_list(manual_content_scan_tas
  * present */
 void manual_content_scan_add_content_to_playlist(
       manual_content_scan_task_config_t *task_config,
-      playlist_t *playlist, const char *content_path);
+      playlist_t *playlist, const char *content_path,
+      int content_type);
 
 RETRO_END_DECLS
 

menu/cbs/menu_cbs_sublabel.c

@@ -720,6 +720,7 @@ default_sublabel_macro(action_bind_sublabel_manual_content_scan_system_name,
 default_sublabel_macro(action_bind_sublabel_manual_content_scan_system_name_custom,        MENU_ENUM_SUBLABEL_MANUAL_CONTENT_SCAN_SYSTEM_NAME_CUSTOM)
 default_sublabel_macro(action_bind_sublabel_manual_content_scan_core_name,                 MENU_ENUM_SUBLABEL_MANUAL_CONTENT_SCAN_CORE_NAME)
 default_sublabel_macro(action_bind_sublabel_manual_content_scan_file_exts,                 MENU_ENUM_SUBLABEL_MANUAL_CONTENT_SCAN_FILE_EXTS)
+default_sublabel_macro(action_bind_sublabel_manual_content_scan_search_archives,           MENU_ENUM_SUBLABEL_MANUAL_CONTENT_SCAN_SEARCH_ARCHIVES)
 default_sublabel_macro(action_bind_sublabel_manual_content_scan_overwrite,                 MENU_ENUM_SUBLABEL_MANUAL_CONTENT_SCAN_OVERWRITE)
 default_sublabel_macro(action_bind_sublabel_manual_content_scan_start,                     MENU_ENUM_SUBLABEL_MANUAL_CONTENT_SCAN_START)
 
@@ -3079,6 +3080,9 @@ int menu_cbs_init_bind_sublabel(menu_file_list_cbs_t *cbs,
          case MENU_ENUM_LABEL_MANUAL_CONTENT_SCAN_FILE_EXTS:
             BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_manual_content_scan_file_exts);
             break;
+         case MENU_ENUM_LABEL_MANUAL_CONTENT_SCAN_SEARCH_ARCHIVES:
+            BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_manual_content_scan_search_archives);
+            break;
          case MENU_ENUM_LABEL_MANUAL_CONTENT_SCAN_OVERWRITE:
             BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_manual_content_scan_overwrite);
             break;

menu/menu_displaylist.c

@@ -3716,6 +3716,12 @@ static bool menu_displaylist_parse_manual_content_scan_list(
          false) == 0)
       count++;
 
+   /* Search inside archive files */
+   if (menu_displaylist_parse_settings_enum(info->list,
+         MENU_ENUM_LABEL_MANUAL_CONTENT_SCAN_SEARCH_ARCHIVES, PARSE_ONLY_BOOL,
+         false) == 0)
+      count++;
+
    /* Overwrite playlist */
    if (menu_displaylist_parse_settings_enum(info->list,
          MENU_ENUM_LABEL_MANUAL_CONTENT_SCAN_OVERWRITE, PARSE_ONLY_BOOL,

menu/menu_setting.c

@@ -16485,6 +16485,21 @@ static bool setting_append_list(
          SETTINGS_DATA_LIST_CURRENT_ADD_FLAGS(list, list_info, SD_FLAG_ALLOW_INPUT);
          (*list)[list_info->index - 1].ui_type       = ST_UI_TYPE_STRING_LINE_EDIT;
 
+         CONFIG_BOOL(
+               list, list_info,
+               manual_content_scan_get_search_archives_ptr(),
+               MENU_ENUM_LABEL_MANUAL_CONTENT_SCAN_SEARCH_ARCHIVES,
+               MENU_ENUM_LABEL_VALUE_MANUAL_CONTENT_SCAN_SEARCH_ARCHIVES,
+               false,
+               MENU_ENUM_LABEL_VALUE_OFF,
+               MENU_ENUM_LABEL_VALUE_ON,
+               &group_info,
+               &subgroup_info,
+               parent_group,
+               general_write_handler,
+               general_read_handler,
+               SD_FLAG_NONE);
+
          CONFIG_BOOL(
                list, list_info,
                manual_content_scan_get_overwrite_playlist_ptr(),

msg_hash.h

@@ -2670,6 +2670,7 @@ enum msg_hash_enums
    MENU_LABEL(MANUAL_CONTENT_SCAN_SYSTEM_NAME_CUSTOM),
    MENU_LABEL(MANUAL_CONTENT_SCAN_CORE_NAME),
    MENU_LABEL(MANUAL_CONTENT_SCAN_FILE_EXTS),
+   MENU_LABEL(MANUAL_CONTENT_SCAN_SEARCH_ARCHIVES),
    MENU_LABEL(MANUAL_CONTENT_SCAN_OVERWRITE),
    MENU_LABEL(MANUAL_CONTENT_SCAN_START),
 

tasks/task_manual_content_scan.c

@@ -2,6 +2,7 @@
  *  Copyright (C) 2011-2017 - Daniel De Matteis
  *  Copyright (C) 2014-2017 - Jean-André Santoni
  *  Copyright (C) 2016-2019 - Brad Parker
+ *  Copyright (C)      2019 - James Leaver
  *
  *  RetroArch is free software: you can redistribute it and/or modify it under the terms
  *  of the GNU General Public License as published by the Free Software Found-
@@ -145,6 +146,8 @@ static void task_manual_content_scan_handler(retro_task_t *task)
          {
             const char *content_path =
                   manual_scan->content_list->elems[manual_scan->list_index].data;
+            int content_type         =
+                  manual_scan->content_list->elems[manual_scan->list_index].attr.i;
 
             if (!string_is_empty(content_path))
             {
@@ -169,7 +172,7 @@ static void task_manual_content_scan_handler(retro_task_t *task)
                /* Add content to playlist */
                manual_content_scan_add_content_to_playlist(
                      manual_scan->task_config, manual_scan->playlist,
-                     content_path);
+                     content_path, content_type);
             }
 
             /* Increment content index */