From 4220e5683d93fb24c9a1d1bbeff0345c1292c0b9 Mon Sep 17 00:00:00 2001 From: Eric Warmenhoven Date: Tue, 2 Jan 2024 00:57:44 -0500 Subject: [PATCH] Do CRC checks on PSP/PSP(PSN) content scan (#15623) --- config.def.h | 2 + configuration.c | 1 + configuration.h | 1 + intl/msg_hash_lbl.h | 4 ++ intl/msg_hash_us.h | 8 ++++ libretro-common/encodings/encoding_crc32.c | 45 +++++++++++++++++++++- menu/cbs/menu_cbs_sublabel.c | 4 ++ menu/menu_displaylist.c | 1 + menu/menu_setting.c | 15 ++++++++ msg_hash.h | 1 + tasks/task_database.c | 35 ++++++++++++++++- 11 files changed, 113 insertions(+), 4 deletions(-) diff --git a/config.def.h b/config.def.h index 701c5a0b01..2ddd4ed6b2 100644 --- a/config.def.h +++ b/config.def.h @@ -1435,6 +1435,8 @@ #define DEFAULT_SCAN_WITHOUT_CORE_MATCH false +#define DEFAULT_SCAN_SERIAL_AND_CRC false + #ifdef __WINRT__ /* Be paranoid about WinRT file I/O performance, and leave this disabled by * default */ diff --git a/configuration.c b/configuration.c index 423399f7ad..25ed1671c5 100644 --- a/configuration.c +++ b/configuration.c @@ -1722,6 +1722,7 @@ static struct config_bool_setting *populate_settings_bool( SETTING_BOOL("global_core_options", &settings->bools.global_core_options, true, DEFAULT_GLOBAL_CORE_OPTIONS, false); SETTING_BOOL("auto_shaders_enable", &settings->bools.auto_shaders_enable, true, DEFAULT_AUTO_SHADERS_ENABLE, false); SETTING_BOOL("scan_without_core_match", &settings->bools.scan_without_core_match, true, DEFAULT_SCAN_WITHOUT_CORE_MATCH, false); + SETTING_BOOL("scan_serial_and_crc", &settings->bools.scan_serial_and_crc, true, DEFAULT_SCAN_SERIAL_AND_CRC, false); SETTING_BOOL("sort_savefiles_enable", &settings->bools.sort_savefiles_enable, true, DEFAULT_SORT_SAVEFILES_ENABLE, false); SETTING_BOOL("sort_savestates_enable", &settings->bools.sort_savestates_enable, true, DEFAULT_SORT_SAVESTATES_ENABLE, false); SETTING_BOOL("sort_savefiles_by_content_enable", &settings->bools.sort_savefiles_by_content_enable, true, DEFAULT_SORT_SAVEFILES_BY_CONTENT_ENABLE, false); diff --git a/configuration.h b/configuration.h index 5b961c03bc..323cca1bdb 100644 --- a/configuration.h +++ b/configuration.h @@ -1027,6 +1027,7 @@ typedef struct settings bool log_to_file_timestamp; bool scan_without_core_match; + bool scan_serial_and_crc; bool ai_service_enable; bool ai_service_pause; diff --git a/intl/msg_hash_lbl.h b/intl/msg_hash_lbl.h index 38c9ccec2a..99925168b4 100644 --- a/intl/msg_hash_lbl.h +++ b/intl/msg_hash_lbl.h @@ -6066,6 +6066,10 @@ MSG_HASH( MENU_ENUM_LABEL_SCAN_WITHOUT_CORE_MATCH, "scan_without_core_match" ) +MSG_HASH( + MENU_ENUM_LABEL_SCAN_SERIAL_AND_CRC, + "scan_serial_and_crc" + ) MSG_HASH( MENU_ENUM_LABEL_MENU_XMB_ANIMATION_HORIZONTAL_HIGHLIGHT, "xmb_menu_animation_horizontal_highlight" diff --git a/intl/msg_hash_us.h b/intl/msg_hash_us.h index 999390dab4..6ed12fe5d2 100644 --- a/intl/msg_hash_us.h +++ b/intl/msg_hash_us.h @@ -7469,6 +7469,14 @@ MSG_HASH( MENU_ENUM_SUBLABEL_SCAN_WITHOUT_CORE_MATCH, "Allow content to be scanned and added to a playlist without a core installed that supports it." ) +MSG_HASH( + MENU_ENUM_LABEL_VALUE_SCAN_SERIAL_AND_CRC, + "Scan checks CRC on possible duplicates" + ) +MSG_HASH( + MENU_ENUM_SUBLABEL_SCAN_SERIAL_AND_CRC, + "Sometimes ISOs duplicate serials, particularly with PSP/PSN titles. Relying solely on the serial can sometimes cause the scanner to put content in the wrong system. This adds a CRC check, which slows down scanning considerably, but may be more accurate." + ) MSG_HASH( MENU_ENUM_LABEL_VALUE_PLAYLIST_MANAGER_LIST, "Manage Playlists" diff --git a/libretro-common/encodings/encoding_crc32.c b/libretro-common/encodings/encoding_crc32.c index ab8ce8267e..79e2b177fb 100644 --- a/libretro-common/encodings/encoding_crc32.c +++ b/libretro-common/encodings/encoding_crc32.c @@ -25,6 +25,45 @@ #include #include +#if __ARM_FEATURE_CRC32 + +#ifdef _M_ARM64 +# include +#else +# include +#endif + +uint32_t encoding_crc32(uint32_t crc, const uint8_t *data, size_t len) +{ + crc = ~crc; + + // Align data if it is not aligned + while (((uintptr_t)data & 7) && len > 0) + { + crc = __crc32b(crc, *(uint8_t *)data); + data++; + len--; + } + + while (len >= 8) + { + crc = __crc32d(crc, *(uint64_t *)data); + data += 8; + len -= 8; + } + + while (len > 0) + { + crc = __crc32b(crc, *(uint8_t *)data); + data++; + len--; + } + + return ~crc; +} + +#else + static const uint32_t crc32_table[256] = { 0x00000000L, 0x77073096L, 0xee0e612cL, 0x990951baL, 0x076dc419L, 0x706af48fL, 0xe963a535L, 0x9e6495a3L, 0x0edb8832L, 0x79dcb8a4L, @@ -82,10 +121,12 @@ static const uint32_t crc32_table[256] = { uint32_t encoding_crc32(uint32_t crc, const uint8_t *buf, size_t len) { - crc = crc ^ 0xffffffff; + crc = ~crc; while (len--) crc = crc32_table[(crc ^ (*buf++)) & 0xff] ^ (crc >> 8); - return crc ^ 0xffffffff; + return ~crc; } + +#endif diff --git a/menu/cbs/menu_cbs_sublabel.c b/menu/cbs/menu_cbs_sublabel.c index 632c0ae7c4..c4855453a5 100644 --- a/menu/cbs/menu_cbs_sublabel.c +++ b/menu/cbs/menu_cbs_sublabel.c @@ -1290,6 +1290,7 @@ DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_menu_rgui_thumbnail_delay, DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_content_runtime_log, MENU_ENUM_SUBLABEL_CONTENT_RUNTIME_LOG) DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_content_runtime_log_aggregate, MENU_ENUM_SUBLABEL_CONTENT_RUNTIME_LOG_AGGREGATE) DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_scan_without_core_match, MENU_ENUM_SUBLABEL_SCAN_WITHOUT_CORE_MATCH) +DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_scan_serial_and_crc, MENU_ENUM_SUBLABEL_SCAN_SERIAL_AND_CRC) DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_playlist_sublabel_runtime_type, MENU_ENUM_SUBLABEL_PLAYLIST_SUBLABEL_RUNTIME_TYPE) DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_playlist_sublabel_last_played_style, MENU_ENUM_SUBLABEL_PLAYLIST_SUBLABEL_LAST_PLAYED_STYLE) DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_menu_rgui_internal_upscale_level, MENU_ENUM_SUBLABEL_MENU_RGUI_INTERNAL_UPSCALE_LEVEL) @@ -5379,6 +5380,9 @@ int menu_cbs_init_bind_sublabel(menu_file_list_cbs_t *cbs, case MENU_ENUM_LABEL_SCAN_WITHOUT_CORE_MATCH: BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_scan_without_core_match); break; + case MENU_ENUM_LABEL_SCAN_SERIAL_AND_CRC: + BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_scan_serial_and_crc); + break; case MENU_ENUM_LABEL_CONTENT_RUNTIME_LOG_AGGREGATE: BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_content_runtime_log_aggregate); break; diff --git a/menu/menu_displaylist.c b/menu/menu_displaylist.c index fc503218b6..608fd4299c 100644 --- a/menu/menu_displaylist.c +++ b/menu/menu_displaylist.c @@ -6706,6 +6706,7 @@ unsigned menu_displaylist_build_list( {MENU_ENUM_LABEL_PLAYLIST_SUBLABEL_LAST_PLAYED_STYLE, PARSE_ONLY_UINT, false}, {MENU_ENUM_LABEL_PLAYLIST_FUZZY_ARCHIVE_MATCH, PARSE_ONLY_BOOL, true}, {MENU_ENUM_LABEL_SCAN_WITHOUT_CORE_MATCH, PARSE_ONLY_BOOL, true}, + {MENU_ENUM_LABEL_SCAN_SERIAL_AND_CRC, PARSE_ONLY_BOOL, true}, {MENU_ENUM_LABEL_OZONE_TRUNCATE_PLAYLIST_NAME, PARSE_ONLY_BOOL, true}, {MENU_ENUM_LABEL_OZONE_SORT_AFTER_TRUNCATE_PLAYLIST_NAME, PARSE_ONLY_BOOL, false}, {MENU_ENUM_LABEL_CONTENT_RUNTIME_LOG, PARSE_ONLY_BOOL, true}, diff --git a/menu/menu_setting.c b/menu/menu_setting.c index cc4ff76001..3cd43db994 100644 --- a/menu/menu_setting.c +++ b/menu/menu_setting.c @@ -11266,6 +11266,21 @@ static bool setting_append_list( general_read_handler, SD_FLAG_NONE); + CONFIG_BOOL( + list, list_info, + &settings->bools.scan_serial_and_crc, + MENU_ENUM_LABEL_SCAN_SERIAL_AND_CRC, + MENU_ENUM_LABEL_VALUE_SCAN_SERIAL_AND_CRC, + DEFAULT_SCAN_SERIAL_AND_CRC, + MENU_ENUM_LABEL_VALUE_OFF, + MENU_ENUM_LABEL_VALUE_ON, + &group_info, + &subgroup_info, + parent_group, + general_write_handler, + general_read_handler, + SD_FLAG_NONE); + CONFIG_ACTION( list, list_info, MENU_ENUM_LABEL_CLOUD_SYNC_SETTINGS, diff --git a/msg_hash.h b/msg_hash.h index c3ccb27837..1f15caa11a 100644 --- a/msg_hash.h +++ b/msg_hash.h @@ -583,6 +583,7 @@ enum msg_hash_enums MENU_LABEL(MENU_XMB_ANIMATION_MOVE_UP_DOWN), MENU_LABEL(MENU_XMB_ANIMATION_OPENING_MAIN_MENU), MENU_LABEL(SCAN_WITHOUT_CORE_MATCH), + MENU_LABEL(SCAN_SERIAL_AND_CRC), MENU_LABEL(STREAMING_TITLE), MENU_LABEL(STREAMING_MODE), MENU_ENUM_LABEL_VALUE_VIDEO_STREAMING_MODE_TWITCH, diff --git a/tasks/task_database.c b/tasks/task_database.c index c5060aeff5..98c44f090e 100644 --- a/tasks/task_database.c +++ b/tasks/task_database.c @@ -1058,6 +1058,26 @@ static int task_database_iterate_playlist_lutro( return 0; } +static bool task_database_check_serial_and_crc( + database_state_handle_t *db_state) +{ +#ifdef RARCH_INTERNAL + settings_t *settings = config_get_ptr(); +#endif + const char *db_path = + database_info_get_current_name(db_state); + +#ifdef RARCH_INTERNAL + if (!settings->bools.scan_serial_and_crc) + return false; +#endif + + /* the PSP shares serials for disc/download content */ + return string_starts_with( + path_basename_nocompression(db_path), + "Sony - PlayStation Portable"); +} + static int task_database_iterate_serial_lookup( db_handle_t *_db, database_state_handle_t *db_state, @@ -1100,8 +1120,19 @@ static int task_database_iterate_serial_lookup( if (db_info_entry && db_info_entry->serial) { if (string_is_equal(db_state->serial, db_info_entry->serial)) - return database_info_list_iterate_found_match(_db, - db_state, db, NULL); + { + if (task_database_check_serial_and_crc(db_state)) + { + if (db_state->crc == 0) + intfstream_file_get_crc(name, 0, SIZE_MAX, &db_state->crc); + if (db_state->crc == db_info_entry->crc32) + return database_info_list_iterate_found_match(_db, + db_state, db, NULL); + } + else + return database_info_list_iterate_found_match(_db, + db_state, db, NULL); + } } }