Minor updates:
authorBastien Montagne <montagne29@wanadoo.fr>
Sun, 13 Jan 2013 17:43:04 +0000 (17:43 +0000)
committerBastien Montagne <montagne29@wanadoo.fr>
Sun, 13 Jan 2013 17:43:04 +0000 (17:43 +0000)
*Made branches' po update multi-process!
*0.8 looks a bit too much a "similarity cutoff" value for messages, changed to 0.75.

release/scripts/modules/bl_i18n_utils/update_po.py
release/scripts/modules/bl_i18n_utils/utils.py

index a1fe20f9fce2360869ca9e34b4e851c3907dc444..6e03226f6d3717e22143a06a6447c891bbf76c24 100755 (executable)
@@ -22,7 +22,7 @@
 
 # Update po’s in the branches from blender.pot in /trunk/po dir.
 
 
 # Update po’s in the branches from blender.pot in /trunk/po dir.
 
-import subprocess
+import concurrent.futures
 import os
 import sys
 from codecs import open
 import os
 import sys
 from codecs import open
@@ -41,69 +41,22 @@ TRUNK_PO_DIR = settings.TRUNK_PO_DIR
 FILE_NAME_POT = settings.FILE_NAME_POT
 
 
 FILE_NAME_POT = settings.FILE_NAME_POT
 
 
-# XXX Not updated, not sure it's that much useful...
-def pproc_newcontext_po(po, pot_messages, pot_stats):
-    print("Adding new contexts to {}...".format(po))
-    messages, state, stats = utils.parse_messages(po)
-    known_ctxt = stats["contexts"]
-    print("Already known (present) context(s): {}".format(str(known_ctxt)))
-
-    new_ctxt = set()
-    added = 0
-    # Only use valid already translated messages!
-    allowed_keys = state["trans_msg"] - state["fuzzy_msg"] - state["comm_msg"]
-    for key in pot_messages.keys():
-        ctxt, msgid = key
-        if ctxt in known_ctxt:
-            continue
-        new_ctxt.add(ctxt)
-        for t_ctxt in known_ctxt:
-            # XXX The first match will win, this might not be optimal...
-            t_key = (t_ctxt, msgid)
-            if t_key in allowed_keys:
-                # Wrong comments (sources) will be removed by msgmerge...
-                messages[key] = messages[t_key]
-                messages[key]["msgctxt_lines"] = [ctxt]
-                added += 1
-
-    utils.write_messages(po, messages, state["comm_msg"], state["fuzzy_msg"])
-    print("Finished!\n    {} new context(s) was/were added {}, adding {} new "
-          "messages.\n".format(len(new_ctxt), str(new_ctxt), added))
-    return 0
-
-
-def process_po(po, lang):
+def process_po(data):
+    po, lang, pot_msgs = data
     # update po file
     # update po file
-    #cmd = (GETTEXT_MSGMERGE_EXECUTABLE,
-           #"--update",
-           #"-w", "1",  # XXX Ugly hack to prevent msgmerge merging short source comments together!
-           #"--no-wrap",
-           #"--backup=none",
-           #"--lang={}".format(lang),
-           #po,
-           #FILE_NAME_POT,
-           #)
-
-    pot = utils.I18nMessages(kind='PO', src=FILE_NAME_POT)
     msg = utils.I18nMessages(iso=lang, kind='PO', src=po)
     print("Updating {}...".format(po))
     msg = utils.I18nMessages(iso=lang, kind='PO', src=po)
     print("Updating {}...".format(po))
-    msg.update(pot)
+    msg.update(pot_msgs)
     msg.write(kind='PO', dest=po)
     msg.write(kind='PO', dest=po)
-    #print("Running ", " ".join(cmd))
-    #ret = subprocess.call(cmd)
-    print("Finished!\n")
+    print("Finished updating {}!\n".format(po))
     return 0
 
 
 def main():
     import argparse
     return 0
 
 
 def main():
     import argparse
-    parser = argparse.ArgumentParser(description="Write out messages.txt "
-                                                 "from Blender.")
+    parser = argparse.ArgumentParser(description="Write out messages.txt from Blender.")
     parser.add_argument('-t', '--trunk', action="store_true", help="Update po’s in /trunk/po rather than /branches.")
     parser.add_argument('-i', '--input', metavar="File", help="Input pot file path.")
     parser.add_argument('-t', '--trunk', action="store_true", help="Update po’s in /trunk/po rather than /branches.")
     parser.add_argument('-i', '--input', metavar="File", help="Input pot file path.")
-    #parser.add_argument('--pproc-contexts', action="store_true",
-                        #help="Pre-process po’s to avoid having plenty of fuzzy msgids just because a context was "
-                             #"added/changed!")
     parser.add_argument('-a', '--add', action="store_true",
                         help="Add missing po’s (useful only when one or more languages are given!).")
     parser.add_argument('langs', metavar='ISO_code', nargs='*', help="Restrict processed languages to those.")
     parser.add_argument('-a', '--add', action="store_true",
                         help="Add missing po’s (useful only when one or more languages are given!).")
     parser.add_argument('langs', metavar='ISO_code', nargs='*', help="Restrict processed languages to those.")
@@ -114,14 +67,8 @@ def main():
         FILE_NAME_POT = args.input
     ret = 0
 
         FILE_NAME_POT = args.input
     ret = 0
 
-    #if args.pproc_contexts:
-        #_ctxt_proc = pproc_newcontext_po
-        #pot_messages, _a, pot_stats = utils.parse_messages(FILE_NAME_POT)
-    if 0:
-        pass
-    else:
-        _ctxt_proc = lambda a, b, c: 0
-        pot_messages, pot_stats = None, None
+    pot_msgs = utils.I18nMessages(kind='PO', src=FILE_NAME_POT)
+    pool_data = []
 
     if args.langs:
         for lang in args.langs:
 
     if args.langs:
         for lang in args.langs:
@@ -137,37 +84,27 @@ def main():
                 if not os.path.exists(po):
                     shutil.copy(FILE_NAME_POT, po)
             if args.add or os.path.exists(po):
                 if not os.path.exists(po):
                     shutil.copy(FILE_NAME_POT, po)
             if args.add or os.path.exists(po):
-                t = _ctxt_proc(po, pot_messages, pot_stats)
-                if t:
-                    ret = t
-                t = process_po(po, lang)
-                if t:
-                    ret = t
+                pool_data.append((po, lang, pot_msgs))
     elif args.trunk:
         for po in os.listdir(TRUNK_PO_DIR):
             if po.endswith(".po"):
                 lang = os.path.basename(po)[:-3]
                 po = os.path.join(TRUNK_PO_DIR, po)
     elif args.trunk:
         for po in os.listdir(TRUNK_PO_DIR):
             if po.endswith(".po"):
                 lang = os.path.basename(po)[:-3]
                 po = os.path.join(TRUNK_PO_DIR, po)
-                t = _ctxt_proc(po, pot_messages, pot_stats)
-                if t:
-                    ret = t
-                t = process_po(po, lang)
-                if t:
-                    ret = t
+                pool_data.append((po, lang, pot_msgs))
     else:
         for lang in os.listdir(BRANCHES_DIR):
             po = os.path.join(BRANCHES_DIR, lang, ".".join((lang, "po")))
             if os.path.exists(po):
     else:
         for lang in os.listdir(BRANCHES_DIR):
             po = os.path.join(BRANCHES_DIR, lang, ".".join((lang, "po")))
             if os.path.exists(po):
-                t = _ctxt_proc(po, pot_messages, pot_stats)
-                if t:
-                    ret = t
-                t = process_po(po, lang)
-                if t:
-                    ret = t
+                pool_data.append((po, lang, pot_msgs))
+
+    with concurrent.futures.ProcessPoolExecutor() as executor:
+        for r in executor.map(process_po, pool_data, timeout=600):
+            if r != 0:
+                ret = r
 
     return ret
 
 
 if __name__ == "__main__":
     print("\n\n *** Running {} *** \n".format(__file__))
 
     return ret
 
 
 if __name__ == "__main__":
     print("\n\n *** Running {} *** \n".format(__file__))
-    sys.exit(main())
+    sys.exit(main())
\ No newline at end of file
index ff5ed4f93d789e5cf4a52cf7e6ad2cdb5e47610c..891bbae3871d9d082ccb43de734698934fd52072 100644 (file)
@@ -226,7 +226,7 @@ class I18nMessages:
     def merge(self, replace=False, *args):
         pass
 
     def merge(self, replace=False, *args):
         pass
 
-    def update(self, ref, use_similar=0.8, keep_old_commented=True):
+    def update(self, ref, use_similar=0.75, keep_old_commented=True):
         """
         Update this I18nMessage with the ref one. Translations from ref are never used. Source comments from ref
         completely replace current ones. If use_similar is not 0.0, it will try to match new messages in ref with an
         """
         Update this I18nMessage with the ref one. Translations from ref are never used. Source comments from ref
         completely replace current ones. If use_similar is not 0.0, it will try to match new messages in ref with an
@@ -493,7 +493,6 @@ class I18nMessages:
                     msgstr_lines.append(line)
                 else:
                     self.parsing_errors.append((line_nr, "regular string outside msgctxt, msgid or msgstr scope"))
                     msgstr_lines.append(line)
                 else:
                     self.parsing_errors.append((line_nr, "regular string outside msgctxt, msgid or msgstr scope"))
-                    print(line)
 
         # If no final empty line, last message is not finalized!
         if reading_msgstr:
 
         # If no final empty line, last message is not finalized!
         if reading_msgstr:
@@ -534,7 +533,6 @@ class I18nMessages:
                     else:
                         chunks += ["\n" + _pmsgctxt + "\"" + msg.msgctxt + "\""]
                 if len(msg.msgid_lines) > 1:
                     else:
                         chunks += ["\n" + _pmsgctxt + "\"" + msg.msgctxt + "\""]
                 if len(msg.msgid_lines) > 1:
-                    print(msg.msgid_lines)
                     chunks += [
                         "\n" + _pmsgid + "\"\"\n" + _p + "\"",
                         ("\"\n" + _p + "\"").join(msg.msgid_lines),
                     chunks += [
                         "\n" + _pmsgid + "\"\"\n" + _p + "\"",
                         ("\"\n" + _p + "\"").join(msg.msgid_lines),