From 4220e5683d93fb24c9a1d1bbeff0345c1292c0b9 Mon Sep 17 00:00:00 2001
From: Eric Warmenhoven <eric@warmenhoven.org>
Date: Tue, 2 Jan 2024 00:57:44 -0500
Subject: [PATCH] Do CRC checks on PSP/PSP(PSN) content scan (#15623)

---
 config.def.h                               |  2 +
 configuration.c                            |  1 +
 configuration.h                            |  1 +
 intl/msg_hash_lbl.h                        |  4 ++
 intl/msg_hash_us.h                         |  8 ++++
 libretro-common/encodings/encoding_crc32.c | 45 +++++++++++++++++++++-
 menu/cbs/menu_cbs_sublabel.c               |  4 ++
 menu/menu_displaylist.c                    |  1 +
 menu/menu_setting.c                        | 15 ++++++++
 msg_hash.h                                 |  1 +
 tasks/task_database.c                      | 35 ++++++++++++++++-
 11 files changed, 113 insertions(+), 4 deletions(-)

diff --git a/config.def.h b/config.def.h
index 701c5a0b01..2ddd4ed6b2 100644
--- a/config.def.h
+++ b/config.def.h
@@ -1435,6 +1435,8 @@
 
 #define DEFAULT_SCAN_WITHOUT_CORE_MATCH false
 
+#define DEFAULT_SCAN_SERIAL_AND_CRC false
+
 #ifdef __WINRT__
 /* Be paranoid about WinRT file I/O performance, and leave this disabled by
  * default */
diff --git a/configuration.c b/configuration.c
index 423399f7ad..25ed1671c5 100644
--- a/configuration.c
+++ b/configuration.c
@@ -1722,6 +1722,7 @@ static struct config_bool_setting *populate_settings_bool(
    SETTING_BOOL("global_core_options",           &settings->bools.global_core_options, true, DEFAULT_GLOBAL_CORE_OPTIONS, false);
    SETTING_BOOL("auto_shaders_enable",           &settings->bools.auto_shaders_enable, true, DEFAULT_AUTO_SHADERS_ENABLE, false);
    SETTING_BOOL("scan_without_core_match",       &settings->bools.scan_without_core_match, true, DEFAULT_SCAN_WITHOUT_CORE_MATCH, false);
+   SETTING_BOOL("scan_serial_and_crc",           &settings->bools.scan_serial_and_crc, true, DEFAULT_SCAN_SERIAL_AND_CRC, false);
    SETTING_BOOL("sort_savefiles_enable",              &settings->bools.sort_savefiles_enable, true, DEFAULT_SORT_SAVEFILES_ENABLE, false);
    SETTING_BOOL("sort_savestates_enable",             &settings->bools.sort_savestates_enable, true, DEFAULT_SORT_SAVESTATES_ENABLE, false);
    SETTING_BOOL("sort_savefiles_by_content_enable",   &settings->bools.sort_savefiles_by_content_enable, true, DEFAULT_SORT_SAVEFILES_BY_CONTENT_ENABLE, false);
diff --git a/configuration.h b/configuration.h
index 5b961c03bc..323cca1bdb 100644
--- a/configuration.h
+++ b/configuration.h
@@ -1027,6 +1027,7 @@ typedef struct settings
       bool log_to_file_timestamp;
 
       bool scan_without_core_match;
+      bool scan_serial_and_crc;
 
       bool ai_service_enable;
       bool ai_service_pause;
diff --git a/intl/msg_hash_lbl.h b/intl/msg_hash_lbl.h
index 38c9ccec2a..99925168b4 100644
--- a/intl/msg_hash_lbl.h
+++ b/intl/msg_hash_lbl.h
@@ -6066,6 +6066,10 @@ MSG_HASH(
    MENU_ENUM_LABEL_SCAN_WITHOUT_CORE_MATCH,
    "scan_without_core_match"
    )
+MSG_HASH(
+   MENU_ENUM_LABEL_SCAN_SERIAL_AND_CRC,
+   "scan_serial_and_crc"
+   )
 MSG_HASH(
    MENU_ENUM_LABEL_MENU_XMB_ANIMATION_HORIZONTAL_HIGHLIGHT,
    "xmb_menu_animation_horizontal_highlight"
diff --git a/intl/msg_hash_us.h b/intl/msg_hash_us.h
index 999390dab4..6ed12fe5d2 100644
--- a/intl/msg_hash_us.h
+++ b/intl/msg_hash_us.h
@@ -7469,6 +7469,14 @@ MSG_HASH(
    MENU_ENUM_SUBLABEL_SCAN_WITHOUT_CORE_MATCH,
    "Allow content to be scanned and added to a playlist without a core installed that supports it."
    )
+MSG_HASH(
+   MENU_ENUM_LABEL_VALUE_SCAN_SERIAL_AND_CRC,
+   "Scan checks CRC on possible duplicates"
+   )
+MSG_HASH(
+   MENU_ENUM_SUBLABEL_SCAN_SERIAL_AND_CRC,
+   "Sometimes ISOs duplicate serials, particularly with PSP/PSN titles. Relying solely on the serial can sometimes cause the scanner to put content in the wrong system. This adds a CRC check, which slows down scanning considerably, but may be more accurate."
+   )
 MSG_HASH(
    MENU_ENUM_LABEL_VALUE_PLAYLIST_MANAGER_LIST,
    "Manage Playlists"
diff --git a/libretro-common/encodings/encoding_crc32.c b/libretro-common/encodings/encoding_crc32.c
index ab8ce8267e..79e2b177fb 100644
--- a/libretro-common/encodings/encoding_crc32.c
+++ b/libretro-common/encodings/encoding_crc32.c
@@ -25,6 +25,45 @@
 #include <encodings/crc32.h>
 #include <stdlib.h>
 
+#if __ARM_FEATURE_CRC32
+
+#ifdef _M_ARM64
+# include <arm64_neon.h>
+#else
+# include <arm_acle.h>
+#endif
+
+uint32_t encoding_crc32(uint32_t crc, const uint8_t *data, size_t len)
+{
+   crc = ~crc;
+
+   // Align data if it is not aligned
+   while (((uintptr_t)data & 7) && len > 0)
+   {
+      crc = __crc32b(crc, *(uint8_t *)data);
+      data++;
+      len--;
+   }
+
+   while (len >= 8)
+   {
+      crc = __crc32d(crc, *(uint64_t *)data);
+      data += 8;
+      len -= 8;
+   }
+
+   while (len > 0)
+   {
+      crc = __crc32b(crc, *(uint8_t *)data);
+      data++;
+      len--;
+   }
+
+   return ~crc;
+}
+
+#else
+
 static const uint32_t crc32_table[256] = {
   0x00000000L, 0x77073096L, 0xee0e612cL, 0x990951baL, 0x076dc419L,
   0x706af48fL, 0xe963a535L, 0x9e6495a3L, 0x0edb8832L, 0x79dcb8a4L,
@@ -82,10 +121,12 @@ static const uint32_t crc32_table[256] = {
 
 uint32_t encoding_crc32(uint32_t crc, const uint8_t *buf, size_t len)
 {
-   crc = crc ^ 0xffffffff;
+   crc = ~crc;
 
    while (len--)
       crc = crc32_table[(crc ^ (*buf++)) & 0xff] ^ (crc >> 8);
 
-   return crc ^ 0xffffffff;
+   return ~crc;
 }
+
+#endif
diff --git a/menu/cbs/menu_cbs_sublabel.c b/menu/cbs/menu_cbs_sublabel.c
index 632c0ae7c4..c4855453a5 100644
--- a/menu/cbs/menu_cbs_sublabel.c
+++ b/menu/cbs/menu_cbs_sublabel.c
@@ -1290,6 +1290,7 @@ DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_menu_rgui_thumbnail_delay,
 DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_content_runtime_log,                           MENU_ENUM_SUBLABEL_CONTENT_RUNTIME_LOG)
 DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_content_runtime_log_aggregate,                 MENU_ENUM_SUBLABEL_CONTENT_RUNTIME_LOG_AGGREGATE)
 DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_scan_without_core_match,                       MENU_ENUM_SUBLABEL_SCAN_WITHOUT_CORE_MATCH)
+DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_scan_serial_and_crc,                           MENU_ENUM_SUBLABEL_SCAN_SERIAL_AND_CRC)
 DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_playlist_sublabel_runtime_type,                MENU_ENUM_SUBLABEL_PLAYLIST_SUBLABEL_RUNTIME_TYPE)
 DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_playlist_sublabel_last_played_style,           MENU_ENUM_SUBLABEL_PLAYLIST_SUBLABEL_LAST_PLAYED_STYLE)
 DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_menu_rgui_internal_upscale_level,              MENU_ENUM_SUBLABEL_MENU_RGUI_INTERNAL_UPSCALE_LEVEL)
@@ -5379,6 +5380,9 @@ int menu_cbs_init_bind_sublabel(menu_file_list_cbs_t *cbs,
          case MENU_ENUM_LABEL_SCAN_WITHOUT_CORE_MATCH:
             BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_scan_without_core_match);
             break;
+         case MENU_ENUM_LABEL_SCAN_SERIAL_AND_CRC:
+            BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_scan_serial_and_crc);
+            break;
          case MENU_ENUM_LABEL_CONTENT_RUNTIME_LOG_AGGREGATE:
             BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_content_runtime_log_aggregate);
             break;
diff --git a/menu/menu_displaylist.c b/menu/menu_displaylist.c
index fc503218b6..608fd4299c 100644
--- a/menu/menu_displaylist.c
+++ b/menu/menu_displaylist.c
@@ -6706,6 +6706,7 @@ unsigned menu_displaylist_build_list(
                {MENU_ENUM_LABEL_PLAYLIST_SUBLABEL_LAST_PLAYED_STYLE, PARSE_ONLY_UINT, false},
                {MENU_ENUM_LABEL_PLAYLIST_FUZZY_ARCHIVE_MATCH,        PARSE_ONLY_BOOL, true},
                {MENU_ENUM_LABEL_SCAN_WITHOUT_CORE_MATCH,             PARSE_ONLY_BOOL, true},
+               {MENU_ENUM_LABEL_SCAN_SERIAL_AND_CRC,                 PARSE_ONLY_BOOL, true},
                {MENU_ENUM_LABEL_OZONE_TRUNCATE_PLAYLIST_NAME,        PARSE_ONLY_BOOL, true},
                {MENU_ENUM_LABEL_OZONE_SORT_AFTER_TRUNCATE_PLAYLIST_NAME, PARSE_ONLY_BOOL, false},
                {MENU_ENUM_LABEL_CONTENT_RUNTIME_LOG,                 PARSE_ONLY_BOOL, true},
diff --git a/menu/menu_setting.c b/menu/menu_setting.c
index cc4ff76001..3cd43db994 100644
--- a/menu/menu_setting.c
+++ b/menu/menu_setting.c
@@ -11266,6 +11266,21 @@ static bool setting_append_list(
                   general_read_handler,
                   SD_FLAG_NONE);
 
+            CONFIG_BOOL(
+                  list, list_info,
+                  &settings->bools.scan_serial_and_crc,
+                  MENU_ENUM_LABEL_SCAN_SERIAL_AND_CRC,
+                  MENU_ENUM_LABEL_VALUE_SCAN_SERIAL_AND_CRC,
+                  DEFAULT_SCAN_SERIAL_AND_CRC,
+                  MENU_ENUM_LABEL_VALUE_OFF,
+                  MENU_ENUM_LABEL_VALUE_ON,
+                  &group_info,
+                  &subgroup_info,
+                  parent_group,
+                  general_write_handler,
+                  general_read_handler,
+                  SD_FLAG_NONE);
+
             CONFIG_ACTION(
                   list, list_info,
                   MENU_ENUM_LABEL_CLOUD_SYNC_SETTINGS,
diff --git a/msg_hash.h b/msg_hash.h
index c3ccb27837..1f15caa11a 100644
--- a/msg_hash.h
+++ b/msg_hash.h
@@ -583,6 +583,7 @@ enum msg_hash_enums
    MENU_LABEL(MENU_XMB_ANIMATION_MOVE_UP_DOWN),
    MENU_LABEL(MENU_XMB_ANIMATION_OPENING_MAIN_MENU),
    MENU_LABEL(SCAN_WITHOUT_CORE_MATCH),
+   MENU_LABEL(SCAN_SERIAL_AND_CRC),
    MENU_LABEL(STREAMING_TITLE),
    MENU_LABEL(STREAMING_MODE),
    MENU_ENUM_LABEL_VALUE_VIDEO_STREAMING_MODE_TWITCH,
diff --git a/tasks/task_database.c b/tasks/task_database.c
index c5060aeff5..98c44f090e 100644
--- a/tasks/task_database.c
+++ b/tasks/task_database.c
@@ -1058,6 +1058,26 @@ static int task_database_iterate_playlist_lutro(
    return 0;
 }
 
+static bool task_database_check_serial_and_crc(
+      database_state_handle_t *db_state)
+{
+#ifdef RARCH_INTERNAL
+   settings_t *settings                    = config_get_ptr();
+#endif
+   const char         *db_path    =
+      database_info_get_current_name(db_state);
+
+#ifdef RARCH_INTERNAL
+   if (!settings->bools.scan_serial_and_crc)
+       return false;
+#endif
+
+   /* the PSP shares serials for disc/download content */
+   return string_starts_with(
+         path_basename_nocompression(db_path),
+         "Sony - PlayStation Portable");
+}
+
 static int task_database_iterate_serial_lookup(
       db_handle_t *_db,
       database_state_handle_t *db_state,
@@ -1100,8 +1120,19 @@ static int task_database_iterate_serial_lookup(
       if (db_info_entry && db_info_entry->serial)
       {
          if (string_is_equal(db_state->serial, db_info_entry->serial))
-            return database_info_list_iterate_found_match(_db,
-                  db_state, db, NULL);
+         {
+            if (task_database_check_serial_and_crc(db_state))
+            {
+               if (db_state->crc == 0)
+                  intfstream_file_get_crc(name, 0, SIZE_MAX, &db_state->crc);
+               if (db_state->crc == db_info_entry->crc32)
+                  return database_info_list_iterate_found_match(_db,
+                        db_state, db, NULL);
+            }
+            else
+               return database_info_list_iterate_found_match(_db,
+                     db_state, db, NULL);
+         }
       }
    }