Use JSON as format to connect with translation tools

intl/.gitignore

@@ -0,0 +1,2 @@
+*.json
+

intl/README.md

@@ -0,0 +1,17 @@
+# Internationalization Workflow (Draft)
+
+## Steps
+
+- Developers update strings in `msg_hash_us.h`.
+- Developers (can set a cron job) run `./h2json.py msg_hash_us.h` to generate `msg_hash_us.json`. It is just a convenient format that is supported by Weblate/Crowdin/Transifex and doesn't need to be inversion control.
+- Developers (can set a cron job) upload `msg_hash_us.json` to Weblate/Crowdin/Transifex.
+- Translators translate strings on Weblate/Crowdin/Transifex.
+- Developers (can set a cron job) download `msg_hash_xx.json` files.
+- Developers (can set a cron job) run `./json2h.py msg_hash_xx.json` to generate `msg_hash_xx.h`.
+
+## Pros
+
+- No new dependencies.
+- No performance impact.
+- Don't require translators to know how to use Git, how to read C code and how to create Pull Request.
+- Translators will be informed whenever a source string changes.

intl/h2json.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+
+# Convert *.h to *.json
+# Usage: ./h2json.py msg_has_us.h
+
+import re
+import sys
+import json
+
+try:
+    h_filename = sys.argv[1]
+    json_filename = h_filename.replace('.h', '.json')
+except IndexError:
+    print("Usage: ./h2json.py msg_has_us.h")
+    sys.exit(1)
+
+p = re.compile('MSG_HASH\(\s*[A-Z0-9_]+,\s*\".*\"\s*\)')
+
+header = """#if defined(_MSC_VER) && !defined(_XBOX) && (_MSC_VER >= 1500 && _MSC_VER < 1900)
+#if (_MSC_VER >= 1700)
+/* https://support.microsoft.com/en-us/kb/980263 */
+#pragma execution_character_set("utf-8")
+#endif
+#pragma warning(disable:4566)
+#endif
+"""
+
+
+def parse_message(message):
+    key_start = message.find('(') + 1
+    key_end = message.find(',')
+    key = message[key_start:key_end].strip()
+    value_start = message.find('"') + 1
+    value_end = message.rfind('"')
+    value = message[value_start:value_end].strip()
+    return key, value
+
+
+try:
+    with open(h_filename, 'r+') as h_file:
+        text = h_file.read()
+        result = p.findall(text)
+        seen = set()
+        messages = {}
+        for msg in result:
+            key, val = parse_message(msg)
+            messages[key] = val
+            if key not in seen:
+                seen.add(key)
+            else:
+                print("Duplicate key: " + key)
+        with open(json_filename, 'w') as json_file:
+            json.dump(messages, json_file, indent=2)
+except EnvironmentError:
+    print('Cannot read/write ' + h_filename)

intl/json2h.py

@@ -1,14 +1,15 @@
 #!/usr/bin/env python3
 
-# Apply template (us) updates to translations (fr, ja, chs, etc.)
-# Usage: ./template.py xx
-# xx is the language code postfix of translation files
+# Convert *.json to *.h
+# Usage: ./json2h.py msg_hash_fr.json
 
 import re
 import sys
+import json
 
 try:
-    lc = sys.argv[1]
+    json_filename = sys.argv[1]
+    h_filename = json_filename.replace('.json', '.h')
 except IndexError:
     print("Usage: ./template.py <language_postfix>")
     sys.exit(1)
@@ -24,6 +25,7 @@ header = """#if defined(_MSC_VER) && !defined(_XBOX) && (_MSC_VER >= 1500 && _MS
 #endif
 """
 
+
 def parse_message(message):
     key_start = message.find('(') + 1
     key_end = message.find(',')
@@ -33,7 +35,8 @@ def parse_message(message):
     value = message[value_start:value_end].strip()
     return key, value
 
-def messages(text):
+
+def parse_messages(text):
     result = p.findall(text)
     seen = set()
     msg_list = []
@@ -48,24 +51,29 @@ def messages(text):
 
     return msg_list
 
-def update(translation, template):
+
+def update(messages, template):
     new_translation = header + template
-    template_messages = messages(template)
-    translation_messages = messages(translation)
+    template_messages = parse_messages(template)
     for tp_msg in template_messages:
-        for ts_msg in translation_messages:
-            if tp_msg['key'] == ts_msg['key']:
-                new_translation = new_translation.replace(tp_msg['msg'], ts_msg['msg'])
+        if tp_msg['key'] in messages:
+            tp_msg_val = tp_msg['val']
+            tl_msg_val = messages[tp_msg['key']]
+            old_msg = tp_msg['msg']
+            new_msg = old_msg.replace(tp_msg_val, tl_msg_val)
+            new_translation = new_translation.replace(old_msg, new_msg)
     return new_translation
 
+
 with open('msg_hash_us.h', 'r') as template_file:
     template = template_file.read()
     try:
-        with open('msg_hash_' + lc + '.h', 'r+') as translation_file:
-            translation = translation_file.read()
-            new_translation = update(translation, template)
-            translation_file.seek(0)
-            translation_file.write(new_translation)
-            translation_file.truncate()
+        with open(json_filename, 'r+') as json_file:
+            messages = json.load(json_file)
+            new_translation = update(messages, template)
+            with open(h_filename, 'w') as h_file:
+                h_file.seek(0)
+                h_file.write(new_translation)
+                h_file.truncate()
     except EnvironmentError:
-        print('Cannot read/write "msg_hash_' + lc + '.h"')
+        print('Cannot read/write ' + json_filename)

intl/msg_hash_us.h

@@ -2500,7 +2500,6 @@ MSG_HASH(
     MENU_ENUM_LABEL_VALUE_RESTART_RETROARCH,
     "Restart RetroArch"
     )
-#endif
 MSG_HASH(
     MENU_ENUM_LABEL_VALUE_RDB_ENTRY_DETAIL,
     "Database Entry"