From 4857d62ac6fa0c11def1a16b3f13a5a505e23ca3 Mon Sep 17 00:00:00 2001 From: Bastien Montagne Date: Tue, 19 Mar 2013 10:11:41 +0000 Subject: [PATCH] More i18n tools cleanup. Also do not import nl in trunk currently (void translation)! --- .../scripts/modules/bl_i18n_utils/check_po.py | 178 --------- .../scripts/modules/bl_i18n_utils/clean_po.py | 96 ----- .../bl_i18n_utils/import_po_from_branches.py | 107 ------ .../scripts/modules/bl_i18n_utils/settings.py | 2 +- .../modules/bl_i18n_utils/update_branches.py | 96 ----- .../modules/bl_i18n_utils/update_mo.py | 106 ------ .../modules/bl_i18n_utils/update_msg.py | 72 ---- .../modules/bl_i18n_utils/update_po.py | 113 ------ .../modules/bl_i18n_utils/update_pot.py | 338 ------------------ .../modules/bl_i18n_utils/update_trunk.py | 144 -------- 10 files changed, 1 insertion(+), 1251 deletions(-) delete mode 100755 release/scripts/modules/bl_i18n_utils/check_po.py delete mode 100755 release/scripts/modules/bl_i18n_utils/clean_po.py delete mode 100755 release/scripts/modules/bl_i18n_utils/import_po_from_branches.py delete mode 100755 release/scripts/modules/bl_i18n_utils/update_branches.py delete mode 100755 release/scripts/modules/bl_i18n_utils/update_mo.py delete mode 100755 release/scripts/modules/bl_i18n_utils/update_msg.py delete mode 100755 release/scripts/modules/bl_i18n_utils/update_po.py delete mode 100755 release/scripts/modules/bl_i18n_utils/update_pot.py delete mode 100755 release/scripts/modules/bl_i18n_utils/update_trunk.py diff --git a/release/scripts/modules/bl_i18n_utils/check_po.py b/release/scripts/modules/bl_i18n_utils/check_po.py deleted file mode 100755 index 96f91ee0914..00000000000 --- a/release/scripts/modules/bl_i18n_utils/check_po.py +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/python3 - -# ***** BEGIN GPL LICENSE BLOCK ***** -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ***** END GPL LICENSE BLOCK ***** - -# - -# Check po’s in branches (or in trunk) for missing/unneeded messages. - -import os -import sys -from codecs import open - -try: - import settings - import utils -except: - from . import (settings, utils) - -TRUNK_PO_DIR = settings.TRUNK_PO_DIR -BRANCHES_DIR = settings.BRANCHES_DIR - -FILE_NAME_POT = settings.FILE_NAME_POT - - -def print_diff(ref_msgs, msgs): - # Remove comments from messages list! - messages = set(msgs.msgs.keys()) - msgs.comm_msgs - unneeded = (messages - ref_msgs.msgs.keys()) - for msgkey in unneeded: - print('\tUnneeded message context/id "{}"'.format(msgkey)) - - missing = (ref_msgs.msgs.keys() - messages) - for msgkey in missing: - print('\tMissing message context/id "{}"'.format(msgkey)) - - for msgid in msgs.comm_msgs: - print('\tCommented message context/id "{}"'.format(msgkey)) - - print("\t{} unneeded messages, {} missing messages, {} commented messages." - "".format(len(unneeded), len(missing), len(msgs.comm_msgs))) - return 0 - - -#def process_po(ref_messages, po, glob_stats, do_stats, do_messages): -def process_po(ref_messages, po, do_stats, do_messages): - print("Checking {}...".format(po)) - ret = 0 - - messages = utils.I18nMessages(kind='PO', src=po) - if do_messages: - t = print_diff(ref_messages, messages) - if t: - ret = t - if do_stats: - print("\tStats:") - t = messages.print_stats(prefix=" ") - if t: - ret = t - if messages.parsing_errors: - print("\tERROR! This .po is broken!") - ret = 1 - return ret - - -# XXX Quick update for new I18Nfoo objects, need rework! -def main(): - import argparse - parser = argparse.ArgumentParser(description="Check po’s in branches " \ - "(or in trunk) for missing" \ - "/unneeded messages.") - parser.add_argument('-s', '--stats', action="store_true", - help="Print po’s stats.") - parser.add_argument('-m', '--messages', action="store_true", - help="Print po’s missing/unneeded/commented messages.") - parser.add_argument('-t', '--trunk', action="store_true", - help="Check po’s in /trunk/po rather than /branches.") - parser.add_argument('-p', '--pot', - help="Specify the .pot file used as reference.") - parser.add_argument('langs', metavar='ISO_code', nargs='*', - help="Restrict processed languages to those.") - args = parser.parse_args() - - if args.pot: - global FILE_NAME_POT - FILE_NAME_POT = args.pot - #glob_stats = {"nbr" : 0.0, - #"lvl" : 0.0, - #"lvl_ttips" : 0.0, - #"lvl_trans_ttips" : 0.0, - #"lvl_ttips_in_trans": 0.0, - #"lvl_comm" : 0.0, - #"nbr_signs" : 0, - #"nbr_trans_signs" : 0, - #"contexts" : set()} - ret = 0 - - pot_messages = None - if args.messages: - pot_messages = utils.I18nMessages(kind='PO', src=FILE_NAME_POT) - #glob_stats["nbr_signs"] = pot_stats["nbr_signs"] - - if args.langs: - for lang in args.langs: - if args.trunk: - po = os.path.join(TRUNK_PO_DIR, ".".join((lang, "po"))) - else: - po = os.path.join(BRANCHES_DIR, lang, ".".join((lang, "po"))) - if os.path.exists(po): - #t = process_po(pot_messages, po, glob_stats, args.stats, args.messages) - t = process_po(pot_messages, po, args.stats, args.messages) - if t: - ret = t - elif args.trunk: - for po in os.listdir(TRUNK_PO_DIR): - if po.endswith(".po"): - po = os.path.join(TRUNK_PO_DIR, po) - #t = process_po(pot_messages, po, glob_stats, args.stats, args.messages) - t = process_po(pot_messages, po, args.stats, args.messages) - if t: - ret = t - else: - for lang in os.listdir(BRANCHES_DIR): - for po in os.listdir(os.path.join(BRANCHES_DIR, lang)): - if po.endswith(".po"): - po = os.path.join(BRANCHES_DIR, lang, po) - #t = process_po(pot_messages, po, glob_stats, args.stats, args.messages) - t = process_po(pot_messages, po, args.stats, args.messages) - if t: - ret = t - - #if args.stats and glob_stats["nbr"] != 0.0: - #nbr_contexts = len(glob_stats["contexts"] - {""}) - #if nbr_contexts != 1: - #if nbr_contexts == 0: - #nbr_contexts = "No" - #_ctx_txt = "s are" - #else: - #_ctx_txt = " is" - #print("\nAverage stats for all {:.0f} processed files:\n" - #" {:>6.1%} done!\n" - #" {:>6.1%} of messages are tooltips.\n" - #" {:>6.1%} of tooltips are translated.\n" - #" {:>6.1%} of translated messages are tooltips.\n" - #" {:>6.1%} of messages are commented.\n" - #" The org msgids are currently made of {} signs.\n" - #" All processed translations are currently made of {} signs.\n" - #" {} specific context{} present:\n {}\n" - #"".format(glob_stats["nbr"], glob_stats["lvl"] / glob_stats["nbr"], - #glob_stats["lvl_ttips"] / glob_stats["nbr"], - #glob_stats["lvl_trans_ttips"] / glob_stats["nbr"], - #glob_stats["lvl_ttips_in_trans"] / glob_stats["nbr"], - #glob_stats["lvl_comm"] / glob_stats["nbr"], glob_stats["nbr_signs"], - #glob_stats["nbr_trans_signs"], nbr_contexts, _ctx_txt, - #"\n ".join(glob_stats["contexts"]-{""}))) - - return ret - - -if __name__ == "__main__": - print("\n\n *** Running {} *** \n".format(__file__)) - print(" *** WARNING! Number of tooltips is only an estimation! ***\n") - sys.exit(main()) diff --git a/release/scripts/modules/bl_i18n_utils/clean_po.py b/release/scripts/modules/bl_i18n_utils/clean_po.py deleted file mode 100755 index da8d25cb9f4..00000000000 --- a/release/scripts/modules/bl_i18n_utils/clean_po.py +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/python3 - -# ***** BEGIN GPL LICENSE BLOCK ***** -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ***** END GPL LICENSE BLOCK ***** - -# - -# Clean (i.e. remove commented messages) po’s in branches or trunk. - -import os -import sys -import collections - -try: - import settings - import utils -except: - from . import (settings, utils) - - -TRUNK_PO_DIR = settings.TRUNK_PO_DIR -BRANCHES_DIR = settings.BRANCHES_DIR - - -def do_clean(po, strict): - print("Cleaning {}...".format(po)) - msgs = utils.I18nMessages(kind='PO', src=po) - - if strict and msgs.parsing_errors: - print("ERROR! This .po file is broken!") - return 1 - - nbr_rem = len(msgs.comm_msgs) - for msgkey in msgs.comm_msgs: - del msgs.msgs[msgkey] - msgs.write(kind='PO', dest=po) - print("Removed {} commented messages.".format(nbr_rem)) - return 0 - - -def main(): - import argparse - parser = argparse.ArgumentParser(description="Clean po’s in branches or trunk (i.e. remove all commented " - "messages).") - parser.add_argument('-t', '--trunk', action="store_true", help="Clean po’s in trunk rather than branches.") - parser.add_argument('-s', '--strict', action="store_true", help="Raise an error if a po is broken.") - parser.add_argument('langs', metavar='ISO_code', nargs='*', help="Restrict processed languages to those.") - args = parser.parse_args() - - ret = 0 - - if args.langs: - for lang in args.langs: - if args.trunk: - po = os.path.join(TRUNK_PO_DIR, ".".join((lang, "po"))) - else: - po = os.path.join(BRANCHES_DIR, lang, ".".join((lang, "po"))) - if os.path.exists(po): - t = do_clean(po, args.strict) - if t: - ret = t - elif args.trunk: - for po in os.listdir(TRUNK_PO_DIR): - if po.endswith(".po"): - po = os.path.join(TRUNK_PO_DIR, po) - t = do_clean(po, args.strict) - if t: - ret = t - else: - for lang in os.listdir(BRANCHES_DIR): - for po in os.listdir(os.path.join(BRANCHES_DIR, lang)): - if po.endswith(".po"): - po = os.path.join(BRANCHES_DIR, lang, po) - t = do_clean(po, args.strict) - if t: - ret = t - - -if __name__ == "__main__": - print("\n\n *** Running {} *** \n".format(__file__)) - sys.exit(main()) diff --git a/release/scripts/modules/bl_i18n_utils/import_po_from_branches.py b/release/scripts/modules/bl_i18n_utils/import_po_from_branches.py deleted file mode 100755 index 956d2e96154..00000000000 --- a/release/scripts/modules/bl_i18n_utils/import_po_from_branches.py +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/python3 - -# ***** BEGIN GPL LICENSE BLOCK ***** -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ***** END GPL LICENSE BLOCK ***** - -# - -# Import in trunk/po all po from branches translated above the given threshold. - -import os -import shutil -import sys -import subprocess - -try: - import settings - import utils - import rtl_preprocess -except: - from . import (settings, utils, rtl_preprocess) - - -TRUNK_PO_DIR = settings.TRUNK_PO_DIR -BRANCHES_DIR = settings.BRANCHES_DIR - -IMPORT_LANGUAGES_SKIP = settings.IMPORT_LANGUAGES_SKIP - -RTL_PREPROCESS_FILE = settings.RTL_PREPROCESS_FILE - -PY3 = settings.PYTHON3_EXEC - - -def main(): - import argparse - parser = argparse.ArgumentParser(description="Import advanced enough po’s from branches to trunk.") - parser.add_argument('-t', '--threshold', type=float, help="Import threshold, as a percentage.") - parser.add_argument('-s', '--strict', action="store_true", help="Raise an error if a po is broken.") - parser.add_argument('langs', metavar='ISO_code', nargs='*', help="Restrict processed languages to those.") - args = parser.parse_args() - - ret = 0 - - threshold = float(settings.IMPORT_MIN_LEVEL) / 100.0 - if args.threshold is not None: - threshold = float(args.threshold) / 100.0 - - for lang in os.listdir(BRANCHES_DIR): - if (args.langs and lang not in args.langs) or lang in IMPORT_LANGUAGES_SKIP: - continue - po = os.path.join(BRANCHES_DIR, lang, ".".join((lang, "po"))) - if os.path.exists(po): - po_is_rtl = os.path.join(BRANCHES_DIR, lang, RTL_PREPROCESS_FILE) - msgs = utils.I18nMessages(iso=lang, kind='PO', src=po) - lvl = 0.0 - if msgs.nbr_msgs: - lvl = msgs.nbr_trans_msgs / msgs.nbr_msgs - if lvl > threshold: - if msgs.parsing_errors and args.strict: - print("{:<10}: {:>6.1%} done, but BROKEN, skipped.".format(lang, lvl)) - ret = 1 - else: - if os.path.exists(po_is_rtl): - out_po = os.path.join(TRUNK_PO_DIR, ".".join((lang, "po"))) - out_raw_po = os.path.join(TRUNK_PO_DIR, "_".join((lang, "raw.po"))) - keys = [] - trans = [] - for k, m in msgs.msgs.items(): - keys.append(k) - trans.append(m.msgstr) - trans = rtl_preprocess.log2vis(trans) - for k, t in zip(keys, trans): - msgs.msgs[k].msgstr = t - msgs.write(kind='PO', dest=out_po) - # Also copies org po! - shutil.copy(po, out_raw_po) - print("{:<10}: {:>6.1%} done, enough translated messages, processed and copied to trunk." - "".format(lang, lvl)) - else: - shutil.copy(po, TRUNK_PO_DIR) - print("{:<10}: {:>6.1%} done, enough translated messages, copied to trunk.".format(lang, lvl)) - else: - if msgs.parsing_errors and args.strict: - print("{:<10}: {:>6.1%} done, BROKEN and not enough translated messages, skipped".format(lang, lvl)) - ret = 1 - else: - print("{:<10}: {:>6.1%} done, not enough translated messages, skipped.".format(lang, lvl)) - return ret - - -if __name__ == "__main__": - print("\n\n *** Running {} *** \n".format(__file__)) - sys.exit(main()) diff --git a/release/scripts/modules/bl_i18n_utils/settings.py b/release/scripts/modules/bl_i18n_utils/settings.py index f38635510b2..fbfca8f639d 100644 --- a/release/scripts/modules/bl_i18n_utils/settings.py +++ b/release/scripts/modules/bl_i18n_utils/settings.py @@ -101,7 +101,7 @@ IMPORT_MIN_LEVEL = 0.0 # Languages in /branches we do not want to import in /trunk currently... IMPORT_LANGUAGES_SKIP = { - 'am_ET', 'bg_BG', 'fi_FI', 'el_GR', 'et_EE', 'ne_NP', 'pl_PL', 'ro_RO', 'uz_UZ', 'uz_UZ@cyrillic', + 'am_ET', 'bg_BG', 'fi_FI', 'el_GR', 'et_EE', 'ne_NP', 'nl_NL', 'pl_PL', 'ro_RO', 'uz_UZ', 'uz_UZ@cyrillic', } # Languages that need RTL pre-processing. diff --git a/release/scripts/modules/bl_i18n_utils/update_branches.py b/release/scripts/modules/bl_i18n_utils/update_branches.py deleted file mode 100755 index 1a22a9835dd..00000000000 --- a/release/scripts/modules/bl_i18n_utils/update_branches.py +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/python3 - -# ***** BEGIN GPL LICENSE BLOCK ***** -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ***** END GPL LICENSE BLOCK ***** - -# - -# Update all branches: -# * Generate a temp messages.txt file. -# * Use it to generate a temp .pot file. -# * Use it to update all .po’s in /branches. - -import subprocess -import os -import sys -import tempfile - -try: - import settings -except: - from . import settings - - -PY3 = settings.PYTHON3_EXEC - -FILE_NAME_POT = settings.FILE_NAME_POT - - -def main(): - import argparse - parser = argparse.ArgumentParser(description="Update all branches:\n" - "* Generate a temp messages.txt file.\n" - "* Use it to generate a blender.pot file.\n" - "* Use it to update all .po’s in /branches.") - #parser.add_argument('--pproc-contexts', action="store_true", - #help="Pre-process po’s to avoid having plenty of fuzzy msgids just because a context was " - #"added/changed!") - parser.add_argument('-c', '--no_checks', default=True, action="store_false", help="No checks over UI messages.") - parser.add_argument('-a', '--add', action="store_true", - help="Add missing po’s (useful only when one or more languages are given!).") - parser.add_argument('langs', metavar='ISO_code', nargs='*', help="Restrict processed languages to those.") - args = parser.parse_args() - - ret = 0 - - # Generate a temp messages file. - dummy, msgfile = tempfile.mkstemp(suffix=".txt", prefix="blender_messages_") - os.close(dummy) - cmd = (PY3, "./update_msg.py", "-o", msgfile) - t = subprocess.call(cmd) - if t: - ret = t - - # Generate blender.pot file in trunk/po. It's quite useful for translators that want to start - # a new translation and not not want to bother generating their own po from scratch! - potfile = FILE_NAME_POT - cmd = [PY3, "./update_pot.py", "-i", msgfile, "-o", potfile] - if not args.no_checks: - cmd.append("-c") - t = subprocess.call(cmd) - if t: - ret = t - - # Update branches’ po files. - cmd = [PY3, "./update_po.py", "-i", potfile] - if args.langs: - if args.add: - cmd.append("-a") - cmd += args.langs - #if args.pproc_contexts: - #cmd.append("--pproc-contexts") - t = subprocess.call(cmd) - if t: - ret = t - - return ret - - -if __name__ == "__main__": - print("\n\n *** Running {} *** \n".format(__file__)) - sys.exit(main()) diff --git a/release/scripts/modules/bl_i18n_utils/update_mo.py b/release/scripts/modules/bl_i18n_utils/update_mo.py deleted file mode 100755 index ab6fedc87a7..00000000000 --- a/release/scripts/modules/bl_i18n_utils/update_mo.py +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/python3 - -# ***** BEGIN GPL LICENSE BLOCK ***** -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ***** END GPL LICENSE BLOCK ***** - -# - -# Create or update mo’s under /trunk/locale/… - -import subprocess -import os -import sys - -try: - import settings - import utils -except: - from . import (settings, utils) - - -GETTEXT_MSGFMT_EXECUTABLE = settings.GETTEXT_MSGFMT_EXECUTABLE - -SOURCE_DIR = settings.SOURCE_DIR -TRUNK_MO_DIR = settings.TRUNK_MO_DIR -TRUNK_PO_DIR = settings.TRUNK_PO_DIR - -DOMAIN = settings.DOMAIN - - -def process_po(po, lang, mo=None): - if not mo: - mo_dir = os.path.join(TRUNK_MO_DIR, lang, "LC_MESSAGES") - # Create dirs if not existing! - if not os.path.isdir(mo_dir): - os.makedirs(mo_dir, exist_ok=True) - - # show stats - cmd = (GETTEXT_MSGFMT_EXECUTABLE, - "--statistics", - po, - "-o", - mo or os.path.join(mo_dir, ".".join((DOMAIN, "mo"))), - ) - - print("Running ", " ".join(cmd)) - ret = subprocess.call(cmd) - print("Finished.") - return ret - - -def main(): - import argparse - parser = argparse.ArgumentParser(description="Create or update mo’s " \ - "under {}.".format(TRUNK_MO_DIR)) - parser.add_argument('langs', metavar='ISO_code', nargs='*', - help="Restrict processed languages to those.") - parser.add_argument('--po', help="Only process that po file (implies --mo).", - nargs='?') - parser.add_argument('--mo', help="Mo file to generate (implies --po).", - nargs='?') - args = parser.parse_args() - - ret = 0 - - if args.po and args.mo: - if os.path.exists(args.po): - t = process_po(args.po, None, args.mo) - if t: - ret = t - elif args.langs: - for lang in args.langs: - po = os.path.join(TRUNK_PO_DIR, ".".join((lang, "po"))) - if os.path.exists(po): - t = process_po(po, lang) - if t: - ret = t - else: - for po in os.listdir(TRUNK_PO_DIR): - if po.endswith(".po") and not po.endswith("_raw.po"): - lang = os.path.basename(po)[:-3] - po = os.path.join(TRUNK_PO_DIR, po) - t = process_po(po, lang) - if t: - ret = t - - return ret - - -if __name__ == "__main__": - print("\n\n *** Running {} *** \n".format(__file__)) - sys.exit(main()) diff --git a/release/scripts/modules/bl_i18n_utils/update_msg.py b/release/scripts/modules/bl_i18n_utils/update_msg.py deleted file mode 100755 index c1b0d636655..00000000000 --- a/release/scripts/modules/bl_i18n_utils/update_msg.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/python3 - -# ***** BEGIN GPL LICENSE BLOCK ***** -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ***** END GPL LICENSE BLOCK ***** - -# - -# Write out messages.txt from Blender. - -import os -import sys -import subprocess - -try: - import settings -except: - from . import settings - - -BLENDER_ARGS = [ - settings.BLENDER_EXEC, - "--background", - "--factory-startup", - "--python", - os.path.join(settings.TOOLS_DIR, "bl_process_msg.py"), - "--", - "-m", -] - - -def main(): - import argparse - parser = argparse.ArgumentParser(description="Write out messages.txt " \ - "from Blender.") - parser.add_argument('-c', '--no_checks', default=True, - action="store_false", - help="No checks over UI messages.") - parser.add_argument('-b', '--blender', help="Blender executable path.") - parser.add_argument('-o', '--output', help="Output messages file path.") - args = parser.parse_args() - if args.blender: - BLENDER_ARGS[0] = args.blender - if not args.no_checks: - BLENDER_ARGS.append("-c") - if args.output: - BLENDER_ARGS.append("-o") - BLENDER_ARGS.append(args.output) - ret = subprocess.call(BLENDER_ARGS) - - return ret - - -if __name__ == "__main__": - print("\n\n *** Running {} *** \n".format(__file__)) - ret = main() - if ret: - raise(Exception(ret)) diff --git a/release/scripts/modules/bl_i18n_utils/update_po.py b/release/scripts/modules/bl_i18n_utils/update_po.py deleted file mode 100755 index eb12782264a..00000000000 --- a/release/scripts/modules/bl_i18n_utils/update_po.py +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/python3 - -# ***** BEGIN GPL LICENSE BLOCK ***** -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ***** END GPL LICENSE BLOCK ***** - -# - -# Update po’s in the branches from blender.pot in /trunk/po dir. - -import concurrent.futures -import os -import sys -from codecs import open -import shutil - -try: - import settings - import utils -except: - from . import (settings, utils) - - -GETTEXT_MSGMERGE_EXECUTABLE = settings.GETTEXT_MSGMERGE_EXECUTABLE -BRANCHES_DIR = settings.BRANCHES_DIR -TRUNK_PO_DIR = settings.TRUNK_PO_DIR -FILE_NAME_POT = settings.FILE_NAME_POT - - -def process_po(data): - po, lang, pot_msgs = data - # update po file - msg = utils.I18nMessages(iso=lang, kind='PO', src=po) - print("Updating {}...".format(po)) - msg.update(pot_msgs) - msg.write(kind='PO', dest=po) - print("Finished updating {}!\n".format(po)) - return 0 - - -def main(): - import argparse - parser = argparse.ArgumentParser(description="Write out messages.txt from Blender.") - parser.add_argument('-t', '--trunk', action="store_true", help="Update po’s in /trunk/po rather than /branches.") - parser.add_argument('-i', '--input', metavar="File", help="Input pot file path.") - parser.add_argument('-a', '--add', action="store_true", - help="Add missing po’s (useful only when one or more languages are given!).") - parser.add_argument('langs', metavar='ISO_code', nargs='*', help="Restrict processed languages to those.") - args = parser.parse_args() - - if args.input: - global FILE_NAME_POT - FILE_NAME_POT = args.input - ret = 0 - - pot_msgs = utils.I18nMessages(kind='PO', src=FILE_NAME_POT) - pool_data = [] - - if args.langs: - for lang in args.langs: - if args.trunk: - dr = TRUNK_PO_DIR - po = os.path.join(dr, ".".join((lang, "po"))) - else: - dr = os.path.join(BRANCHES_DIR, lang) - po = os.path.join(dr, ".".join((lang, "po"))) - if args.add: - if not os.path.exists(dr): - os.makedirs(dr) - if not os.path.exists(po): - shutil.copy(FILE_NAME_POT, po) - if args.add or os.path.exists(po): - pool_data.append((po, lang, pot_msgs)) - elif args.trunk: - for po in os.listdir(TRUNK_PO_DIR): - if po.endswith(".po"): - lang = os.path.basename(po)[:-3] - po = os.path.join(TRUNK_PO_DIR, po) - pool_data.append((po, lang, pot_msgs)) - else: - for lang in os.listdir(BRANCHES_DIR): - po = os.path.join(BRANCHES_DIR, lang, ".".join((lang, "po"))) - if os.path.exists(po): - pool_data.append((po, lang, pot_msgs)) - - for r in map(process_po, pool_data): - if r != 0: - ret = r - #with concurrent.futures.ProcessPoolExecutor() as executor: - #for r in executor.map(process_po, pool_data, timeout=600): - #if r != 0: - #ret = r - - return ret - - -if __name__ == "__main__": - print("\n\n *** Running {} *** \n".format(__file__)) - sys.exit(main()) \ No newline at end of file diff --git a/release/scripts/modules/bl_i18n_utils/update_pot.py b/release/scripts/modules/bl_i18n_utils/update_pot.py deleted file mode 100755 index e05e60937a8..00000000000 --- a/release/scripts/modules/bl_i18n_utils/update_pot.py +++ /dev/null @@ -1,338 +0,0 @@ -#!/usr/bin/python3 - -# ***** BEGIN GPL LICENSE BLOCK ***** -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ***** END GPL LICENSE BLOCK ***** - -# - -# Update blender.pot file from messages.txt - -import subprocess -import collections -import os -import sys -import re -import tempfile -import argparse -import time -import pickle - -try: - import settings - import utils -except: - from . import (settings, utils) - - -LANGUAGES_CATEGORIES = settings.LANGUAGES_CATEGORIES -LANGUAGES = settings.LANGUAGES - -PO_COMMENT_PREFIX = settings.PO_COMMENT_PREFIX -PO_COMMENT_PREFIX_SOURCE = settings.PO_COMMENT_PREFIX_SOURCE -PO_COMMENT_PREFIX_SOURCE_CUSTOM = settings.PO_COMMENT_PREFIX_SOURCE_CUSTOM -MSG_COMMENT_PREFIX = settings.MSG_COMMENT_PREFIX -MSG_CONTEXT_PREFIX = settings.MSG_CONTEXT_PREFIX -FILE_NAME_MESSAGES = settings.FILE_NAME_MESSAGES -FILE_NAME_POT = settings.FILE_NAME_POT -SOURCE_DIR = settings.SOURCE_DIR -POTFILES_DIR = settings.POTFILES_SOURCE_DIR -SRC_POTFILES = settings.FILE_NAME_SRC_POTFILES - -CONTEXT_DEFAULT = settings.CONTEXT_DEFAULT -PYGETTEXT_ALLOWED_EXTS = settings.PYGETTEXT_ALLOWED_EXTS -PYGETTEXT_MAX_MULTI_CTXT = settings.PYGETTEXT_MAX_MULTI_CTXT - -SVN_EXECUTABLE = settings.SVN_EXECUTABLE - -WARN_NC = settings.WARN_MSGID_NOT_CAPITALIZED -NC_ALLOWED = settings.WARN_MSGID_NOT_CAPITALIZED_ALLOWED - -SPELL_CACHE = settings.SPELL_CACHE - - -# Do this only once! -# Get contexts defined in blf. -CONTEXTS = {} -with open(os.path.join(SOURCE_DIR, settings.PYGETTEXT_CONTEXTS_DEFSRC)) as f: - reg = re.compile(settings.PYGETTEXT_CONTEXTS) - f = f.read() - # This regex is supposed to yield tuples - # (key=C_macro_name, value=C_string). - CONTEXTS = dict(m.groups() for m in reg.finditer(f)) - -# Build regexes to extract messages (with optional contexts) from C source. -pygettexts = tuple(re.compile(r).search - for r in settings.PYGETTEXT_KEYWORDS) -_clean_str = re.compile(settings.str_clean_re).finditer -clean_str = lambda s: "".join(m.group("clean") for m in _clean_str(s)) - - -def _new_messages(): - return getattr(collections, "OrderedDict", dict)() - - -def check_file(path, rel_path, messages): - def process_entry(ctxt, msg): - # Context. - if ctxt: - if ctxt in CONTEXTS: - ctxt = CONTEXTS[ctxt] - elif '"' in ctxt or "'" in ctxt: - ctxt = clean_str(ctxt) - else: - print("WARNING: raw context “{}” couldn’t be resolved!" - "".format(ctxt)) - ctxt = CONTEXT_DEFAULT - else: - ctxt = CONTEXT_DEFAULT - # Message. - if msg: - if '"' in msg or "'" in msg: - msg = clean_str(msg) - else: - print("WARNING: raw message “{}” couldn’t be resolved!" - "".format(msg)) - msg = "" - else: - msg = "" - return (ctxt, msg) - - with open(path, encoding="utf-8") as f: - f = f.read() - for srch in pygettexts: - m = srch(f) - line = pos = 0 - while m: - d = m.groupdict() - # Line. - line += f[pos:m.start()].count('\n') - msg = d.get("msg_raw") - # First, try the "multi-contexts" stuff! - ctxts = tuple(d.get("ctxt_raw{}".format(i)) for i in range(PYGETTEXT_MAX_MULTI_CTXT)) - if ctxts[0]: - for ctxt in ctxts: - if not ctxt: - break - ctxt, _msg = process_entry(ctxt, msg) - # And we are done for this item! - messages.setdefault((ctxt, _msg), []).append(":".join((rel_path, str(line)))) - else: - ctxt = d.get("ctxt_raw") - ctxt, msg = process_entry(ctxt, msg) - # And we are done for this item! - messages.setdefault((ctxt, msg), []).append(":".join((rel_path, str(line)))) - pos = m.end() - line += f[m.start():pos].count('\n') - m = srch(f, pos) - - -def py_xgettext(messages): - forbidden = set() - forced = set() - with open(SRC_POTFILES) as src: - for l in src: - if l[0] == '-': - forbidden.add(l[1:].rstrip('\n')) - elif l[0] != '#': - forced.add(l.rstrip('\n')) - for root, dirs, files in os.walk(POTFILES_DIR): - if "/.svn" in root: - continue - for fname in files: - if os.path.splitext(fname)[1] not in PYGETTEXT_ALLOWED_EXTS: - continue - path = os.path.join(root, fname) - rel_path = os.path.relpath(path, SOURCE_DIR) - if rel_path in forbidden: - continue - elif rel_path not in forced: - forced.add(rel_path) - for rel_path in sorted(forced): - path = os.path.join(SOURCE_DIR, rel_path) - if os.path.exists(path): - check_file(path, rel_path, messages) - - -# Spell checking! -import enchant -dict_spelling = enchant.Dict("en_US") - -from spell_check_utils import (dict_uimsgs, - split_words, - ) - -_spell_checked = set() - - -def spell_check(txt, cache): - ret = [] - - if cache is not None and txt in cache: - return ret - - for w in split_words(txt): - w_lower = w.lower() - if w_lower in dict_uimsgs | _spell_checked: - continue - if not dict_spelling.check(w): - ret.append("{}: suggestions are ({})" - .format(w, "'" + "', '".join(dict_spelling.suggest(w)) - + "'")) - else: - _spell_checked.add(w_lower) - - if not ret: - if cache is not None: - cache.add(txt) - - return ret - - -def get_svnrev(): - cmd = [SVN_EXECUTABLE, - "info", - "--xml", - SOURCE_DIR, - ] - xml = subprocess.check_output(cmd) - return re.search(b'revision="(\d+)"', xml).group(1) - - -def gen_empty_pot(): - blender_ver = "" - blender_rev = get_svnrev().decode() - utctime = time.gmtime() - time_str = time.strftime("%Y-%m-%d %H:%M+0000", utctime) - year_str = time.strftime("%Y", utctime) - - return utils.I18nMessages.gen_empty_messages("__POT__", blender_ver, blender_rev, time_str, year_str) - - -escape_re = tuple(re.compile(r[0]) for r in settings.ESCAPE_RE) -escape = lambda s, n: escape_re[n].sub(settings.ESCAPE_RE[n][1], s) - - -def merge_messages(msgs, messages, do_checks, spell_cache): - num_added = 0 - num_present = msgs.nbr_msgs - for (context, msgid), srcs in messages.items(): - if do_checks: - err = spell_check(msgid, spell_cache) - if err: - print("WARNING: spell check failed on “" + msgid + "”:") - print("\t\t" + "\n\t\t".join(err)) - print("\tFrom:\n\t\t" + "\n\t\t".join(srcs)) - - # Escape some chars in msgid! - for n in range(len(escape_re)): - msgid = escape(msgid, n) - - key = (context, msgid) - if key not in msgs.msgs: - msg = utils.I18nMessage([context], [msgid], [""], []) - msg.sources = srcs - msgs.msgs[key] = msg - num_added += 1 - else: - # We need to merge sources! - msgs.msgs[key].sources += srcs - - return num_added, num_present - - -def main(): - parser = argparse.ArgumentParser(description="Update blender.pot file from messages.txt and source code parsing, " - "and performs some checks over msgids.") - parser.add_argument('-w', '--warning', action="store_true", - help="Show warnings.") - parser.add_argument('-i', '--input', metavar="File", - help="Input messages file path.") - parser.add_argument('-o', '--output', metavar="File", - help="Output pot file path.") - - args = parser.parse_args() - if args.input: - global FILE_NAME_MESSAGES - FILE_NAME_MESSAGES = args.input - if args.output: - global FILE_NAME_POT - FILE_NAME_POT = args.output - - print("Running fake py gettext…") - # Not using any more xgettext, simpler to do it ourself! - messages = _new_messages() - py_xgettext(messages) - print("Finished, found {} messages.".format(len(messages))) - - if SPELL_CACHE and os.path.exists(SPELL_CACHE): - with open(SPELL_CACHE, 'rb') as f: - spell_cache = pickle.load(f) - else: - spell_cache = set() - - print("Generating POT file {}…".format(FILE_NAME_POT)) - msgs = gen_empty_pot() - tot_messages, _a = merge_messages(msgs, messages, True, spell_cache) - - # add messages collected automatically from RNA - print("\tMerging RNA messages from {}…".format(FILE_NAME_MESSAGES)) - messages.clear() - with open(FILE_NAME_MESSAGES, encoding="utf-8") as f: - srcs = [] - context = "" - for line in f: - line = utils.stripeol(line) - - if line.startswith(MSG_COMMENT_PREFIX): - srcs.append(line[len(MSG_COMMENT_PREFIX):].strip()) - elif line.startswith(MSG_CONTEXT_PREFIX): - context = line[len(MSG_CONTEXT_PREFIX):].strip() - else: - key = (context, line) - messages[key] = srcs - srcs = [] - context = "" - num_added, num_present = merge_messages(msgs, messages, True, spell_cache) - tot_messages += num_added - print("\tMerged {} messages ({} were already present).".format(num_added, num_present)) - - print("\tAdding languages labels...") - messages.clear() - messages.update(((CONTEXT_DEFAULT, lng[1]), ("Languages’ labels from bl_i18n_utils/settings.py",)) - for lng in LANGUAGES) - messages.update(((CONTEXT_DEFAULT, cat[1]), ("Language categories’ labels from bl_i18n_utils/settings.py",)) - for cat in LANGUAGES_CATEGORIES) - num_added, num_present = merge_messages(msgs, messages, True, spell_cache) - tot_messages += num_added - print("\tAdded {} language messages.".format(num_added)) - - # Write back all messages into blender.pot. - msgs.write('PO', FILE_NAME_POT) - - if SPELL_CACHE and spell_cache: - with open(SPELL_CACHE, 'wb') as f: - pickle.dump(spell_cache, f) - - print("Finished, total: {} messages!".format(tot_messages)) - - return 0 - - -if __name__ == "__main__": - print("\n\n *** Running {} *** \n".format(__file__)) - sys.exit(main()) diff --git a/release/scripts/modules/bl_i18n_utils/update_trunk.py b/release/scripts/modules/bl_i18n_utils/update_trunk.py deleted file mode 100755 index d7d1a9741cb..00000000000 --- a/release/scripts/modules/bl_i18n_utils/update_trunk.py +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/python3 - -# ***** BEGIN GPL LICENSE BLOCK ***** -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ***** END GPL LICENSE BLOCK ***** - -# - -# Update trunk from branches: -# * Remove po’s in trunk. -# * Copy po’s from branches advanced enough. -# * Clean po’s in trunk. -# * Compile po’s in trunk in mo’s, keeping track of those failing. -# * Remove po’s, mo’s (and their dir’s) that failed to compile or are no more present in trunk. - -import subprocess -import os -import sys -import shutil - -try: - import settings - import utils -except: - from . import (settings, utils) - -BRANCHES_DIR = settings.BRANCHES_DIR -TRUNK_PO_DIR = settings.TRUNK_PO_DIR -TRUNK_MO_DIR = settings.TRUNK_MO_DIR - -LANGUAGES_CATEGORIES = settings.LANGUAGES_CATEGORIES -LANGUAGES = settings.LANGUAGES -LANGUAGES_FILE = settings.LANGUAGES_FILE - -PY3 = settings.PYTHON3_EXEC - - -def main(): - import argparse - parser = argparse.ArgumentParser(description="" - "Update trunk from branches:\n" - "* Remove po’s in trunk.\n" - "* Copy po’s from branches advanced enough.\n" - "* Clean po’s in trunk.\n" - "* Compile po’s in trunk in mo’s, keeping track of those failing.\n" - "* Remove po’s and mo’s (and their dir’s) that " - "failed to compile or are no more present in trunk." - "* Generate languages file used by Blender's i18n.") - parser.add_argument('-t', '--threshold', type=int, help="Import threshold, as a percentage.") - parser.add_argument('-p', '--po', action="store_true", help="Remove failing po’s.") - parser.add_argument('-m', '--mo', action="store_true", help="Remove failing mo’s.") - parser.add_argument('langs', metavar='ISO_code', nargs='*', help="Restrict processed languages to those.") - args = parser.parse_args() - - ret = 0 - failed = set() - - # Remove po’s in trunk. - for po in os.listdir(TRUNK_PO_DIR): - if po.endswith(".po"): - lang = os.path.basename(po)[:-3] - if args.langs and lang not in args.langs: - continue - po = os.path.join(TRUNK_PO_DIR, po) - os.remove(po) - - # Copy po’s from branches. - #cmd = [PY3, "./import_po_from_branches.py", "-s"] - cmd = [PY3, "./import_po_from_branches.py"] - if args.threshold is not None: - cmd += ["-t", str(args.threshold)] - if args.langs: - cmd += args.langs - t = subprocess.call(cmd) - if t: - ret = t - - # Add in failed all mo’s no more having relevant po’s in trunk. - for lang in os.listdir(TRUNK_MO_DIR): - if lang in {".svn", LANGUAGES_FILE}: - continue # !!! - if not os.path.exists(os.path.join(TRUNK_PO_DIR, ".".join((lang, "po")))): - failed.add(lang) - - # Check and compile each po separately, to keep track of those failing. - # XXX There should not be any failing at this stage, import step is - # supposed to have already filtered them out! - for po in os.listdir(TRUNK_PO_DIR): - if po.endswith(".po") and not po.endswith("_raw.po"): - lang = os.path.basename(po)[:-3] - if args.langs and lang not in args.langs: - continue - - #cmd = [PY3, "./clean_po.py", "-t", "-s", lang] - cmd = [PY3, "./clean_po.py", "-t", lang] - t = subprocess.call(cmd) - if t: - ret = t - failed.add(lang) - continue - - cmd = [PY3, "./update_mo.py", lang] - t = subprocess.call(cmd) - if t: - ret = t - failed.add(lang) - continue - - # Generate languages file used by Blender's i18n system. - cmd = [PY3, "./update_languages_menu.py"] - t = subprocess.call(cmd) - if t: - ret = t - - # Remove failing po’s, mo’s and related dir’s. - for lang in failed: - print("Lang “{}” failed, removing it...".format(lang)) - if args.po: - po = os.path.join(TRUNK_PO_DIR, ".".join((lang, "po"))) - if os.path.exists(po): - os.remove(po) - if args.mo: - mo = os.path.join(TRUNK_MO_DIR, lang) - if os.path.exists(mo): - shutil.rmtree(mo) - - -if __name__ == "__main__": - print("\n\n *** Running {} *** \n".format(__file__)) - sys.exit(main())