forked from bartvdbraak/blender
1528 lines
66 KiB
Python
1528 lines
66 KiB
Python
# ***** BEGIN GPL LICENSE BLOCK *****
|
|
#
|
|
# This program is free software; you can redistribute it and/or
|
|
# modify it under the terms of the GNU General Public License
|
|
# as published by the Free Software Foundation; either version 2
|
|
# of the License, or (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, write to the Free Software Foundation,
|
|
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
|
#
|
|
# ***** END GPL LICENSE BLOCK *****
|
|
|
|
# <pep8 compliant>
|
|
|
|
# Some misc utilities...
|
|
|
|
import collections
|
|
import concurrent.futures
|
|
import copy
|
|
import hashlib
|
|
import os
|
|
import re
|
|
import struct
|
|
import sys
|
|
import tempfile
|
|
#import time
|
|
|
|
from bl_i18n_utils import (settings,
|
|
utils_rtl,
|
|
)
|
|
|
|
import bpy
|
|
|
|
|
|
##### Misc Utils #####
|
|
from bpy.app.translations import locale_explode
|
|
|
|
|
|
_valid_po_path_re = re.compile(r"^\S+:[0-9]+$")
|
|
def is_valid_po_path(path):
|
|
return bool(_valid_po_path_re.match(path))
|
|
|
|
|
|
def get_best_similar(data):
|
|
import difflib
|
|
key, use_similar, similar_pool = data
|
|
|
|
# try to find some close key in existing messages...
|
|
# Optimized code inspired by difflib.get_close_matches (as we only need the best match).
|
|
# We also consider to never make a match when len differs more than -len_key / 2, +len_key * 2 (which is valid
|
|
# as long as use_similar is not below ~0.7).
|
|
# Gives an overall ~20% of improvement!
|
|
#tmp = difflib.get_close_matches(key[1], similar_pool, n=1, cutoff=use_similar)
|
|
#if tmp:
|
|
#tmp = tmp[0]
|
|
tmp = None
|
|
s = difflib.SequenceMatcher()
|
|
s.set_seq2(key[1])
|
|
len_key = len(key[1])
|
|
min_len = len_key // 2
|
|
max_len = len_key * 2
|
|
for x in similar_pool:
|
|
if min_len < len(x) < max_len:
|
|
s.set_seq1(x)
|
|
if s.real_quick_ratio() >= use_similar and s.quick_ratio() >= use_similar:
|
|
sratio = s.ratio()
|
|
if sratio >= use_similar:
|
|
tmp = x
|
|
use_similar = sratio
|
|
return key, tmp
|
|
|
|
|
|
def locale_match(loc1, loc2):
|
|
"""
|
|
Return:
|
|
-n if loc1 is a subtype of loc2 (e.g. 'fr_FR' is a subtype of 'fr').
|
|
+n if loc2 is a subtype of loc1.
|
|
n becomes smaller when both locales are more similar (e.g. (sr, sr_SR) are more similar than (sr, sr_SR@latin)).
|
|
0 if they are exactly the same.
|
|
... (Ellipsis) if they cannot match!
|
|
Note: We consider that 'sr_SR@latin' is a subtype of 'sr@latin', 'sr_SR' and 'sr', but 'sr_SR' and 'sr@latin' won't
|
|
match (will return ...)!
|
|
Note: About similarity, diff in variants are more important than diff in countries, currently here are the cases:
|
|
(sr, sr_SR) -> 1
|
|
(sr@latin, sr_SR@latin) -> 1
|
|
(sr, sr@latin) -> 2
|
|
(sr_SR, sr_SR@latin) -> 2
|
|
(sr, sr_SR@latin) -> 3
|
|
"""
|
|
if loc1 == loc2:
|
|
return 0
|
|
l1, c1, v1, *_1 = locale_explode(loc1)
|
|
l2, c2, v2, *_2 = locale_explode(loc2)
|
|
|
|
if l1 == l2:
|
|
if c1 == c2:
|
|
if v1 == v2:
|
|
return 0
|
|
elif v2 is None:
|
|
return -2
|
|
elif v1 is None:
|
|
return 2
|
|
return ...
|
|
elif c2 is None:
|
|
if v1 == v2:
|
|
return -1
|
|
elif v2 is None:
|
|
return -3
|
|
return ...
|
|
elif c1 is None:
|
|
if v1 == v2:
|
|
return 1
|
|
elif v1 is None:
|
|
return 3
|
|
return ...
|
|
return ...
|
|
|
|
|
|
def find_best_isocode_matches(uid, iso_codes):
|
|
"""
|
|
Return an ordered tuple of elements in iso_codes that can match the given uid, from most similar to lesser ones.
|
|
"""
|
|
tmp = ((e, locale_match(e, uid)) for e in iso_codes)
|
|
return tuple(e[0] for e in sorted((e for e in tmp if e[1] is not ... and e[1] >= 0), key=lambda e: e[1]))
|
|
|
|
|
|
def get_po_files_from_dir(root_dir, langs=set()):
|
|
"""
|
|
Yield tuples (uid, po_path) of translations for each po file found in the given dir, which should be either
|
|
a dir containing po files using language uid's as names (e.g. fr.po, es_ES.po, etc.), or
|
|
a dir containing dirs which names are language uids, and containing po files of the same names.
|
|
"""
|
|
found_uids = set()
|
|
for p in os.listdir(root_dir):
|
|
uid = None
|
|
po_file = os.path.join(root_dir, p)
|
|
print(p)
|
|
if p.endswith(".po") and os.path.isfile(po_file):
|
|
uid = p[:-3]
|
|
if langs and uid not in langs:
|
|
continue
|
|
elif os.path.isdir(p):
|
|
uid = p
|
|
if langs and uid not in langs:
|
|
continue
|
|
po_file = os.path.join(root_dir, p, p + ".po")
|
|
if not os.path.isfile(po_file):
|
|
continue
|
|
else:
|
|
continue
|
|
if uid in found_uids:
|
|
printf("WARNING! {} id has been found more than once! only first one has been loaded!".format(uid))
|
|
continue
|
|
found_uids.add(uid)
|
|
yield uid, po_file
|
|
|
|
|
|
def enable_addons(addons={}, support={}, disable=False, check_only=False):
|
|
"""
|
|
Enable (or disable) addons based either on a set of names, or a set of 'support' types.
|
|
Returns the list of all affected addons (as fake modules)!
|
|
If "check_only" is set, no addon will be enabled nor disabled.
|
|
"""
|
|
import addon_utils
|
|
|
|
userpref = bpy.context.user_preferences
|
|
used_ext = {ext.module for ext in userpref.addons}
|
|
|
|
ret = [mod for mod in addon_utils.modules(addon_utils.addons_fake_modules)
|
|
if ((addons and mod.__name__ in addons) or
|
|
(not addons and addon_utils.module_bl_info(mod)["support"] in support))]
|
|
|
|
if not check_only:
|
|
for mod in ret:
|
|
module_name = mod.__name__
|
|
if disable:
|
|
if module_name not in used_ext:
|
|
continue
|
|
print(" Disabling module ", module_name)
|
|
bpy.ops.wm.addon_disable(module=module_name)
|
|
else:
|
|
if module_name in used_ext:
|
|
continue
|
|
print(" Enabling module ", module_name)
|
|
bpy.ops.wm.addon_enable(module=module_name)
|
|
|
|
# XXX There are currently some problems with bpy/rna...
|
|
# *Very* tricky to solve!
|
|
# So this is a hack to make all newly added operator visible by
|
|
# bpy.types.OperatorProperties.__subclasses__()
|
|
for cat in dir(bpy.ops):
|
|
cat = getattr(bpy.ops, cat)
|
|
for op in dir(cat):
|
|
getattr(cat, op).get_rna()
|
|
|
|
return ret
|
|
|
|
|
|
##### Main Classes #####
|
|
|
|
class I18nMessage:
|
|
"""
|
|
Internal representation of a message.
|
|
"""
|
|
__slots__ = ("msgctxt_lines", "msgid_lines", "msgstr_lines", "comment_lines", "is_fuzzy", "is_commented",
|
|
"settings")
|
|
|
|
def __init__(self, msgctxt_lines=[], msgid_lines=[], msgstr_lines=[], comment_lines=[],
|
|
is_commented=False, is_fuzzy=False, settings=settings):
|
|
self.settings = settings
|
|
self.msgctxt_lines = msgctxt_lines
|
|
self.msgid_lines = msgid_lines
|
|
self.msgstr_lines = msgstr_lines
|
|
self.comment_lines = comment_lines
|
|
self.is_fuzzy = is_fuzzy
|
|
self.is_commented = is_commented
|
|
|
|
def _get_msgctxt(self):
|
|
return "".join(self.msgctxt_lines)
|
|
def _set_msgctxt(self, ctxt):
|
|
self.msgctxt_lines = [ctxt]
|
|
msgctxt = property(_get_msgctxt, _set_msgctxt)
|
|
|
|
def _get_msgid(self):
|
|
return "".join(self.msgid_lines)
|
|
def _set_msgid(self, msgid):
|
|
self.msgid_lines = [msgid]
|
|
msgid = property(_get_msgid, _set_msgid)
|
|
|
|
def _get_msgstr(self):
|
|
return "".join(self.msgstr_lines)
|
|
def _set_msgstr(self, msgstr):
|
|
self.msgstr_lines = [msgstr]
|
|
msgstr = property(_get_msgstr, _set_msgstr)
|
|
|
|
def _get_sources(self):
|
|
lstrip1 = len(self.settings.PO_COMMENT_PREFIX_SOURCE)
|
|
lstrip2 = len(self.settings.PO_COMMENT_PREFIX_SOURCE_CUSTOM)
|
|
return ([l[lstrip1:] for l in self.comment_lines if l.startswith(self.settings.PO_COMMENT_PREFIX_SOURCE)] +
|
|
[l[lstrip2:] for l in self.comment_lines
|
|
if l.startswith(self.settings.PO_COMMENT_PREFIX_SOURCE_CUSTOM)])
|
|
def _set_sources(self, sources):
|
|
cmmlines = self.comment_lines.copy()
|
|
for l in cmmlines:
|
|
if (l.startswith(self.settings.PO_COMMENT_PREFIX_SOURCE) or
|
|
l.startswith(self.settings.PO_COMMENT_PREFIX_SOURCE_CUSTOM)):
|
|
self.comment_lines.remove(l)
|
|
lines_src = []
|
|
lines_src_custom = []
|
|
for src in sources:
|
|
if is_valid_po_path(src):
|
|
lines_src.append(self.settings.PO_COMMENT_PREFIX_SOURCE + src)
|
|
else:
|
|
lines_src_custom.append(self.settings.PO_COMMENT_PREFIX_SOURCE_CUSTOM + src)
|
|
self.comment_lines += lines_src_custom + lines_src
|
|
sources = property(_get_sources, _set_sources)
|
|
|
|
def _get_is_tooltip(self):
|
|
# XXX For now, we assume that all messages > 30 chars are tooltips!
|
|
return len(self.msgid) > 30
|
|
is_tooltip = property(_get_is_tooltip)
|
|
|
|
def copy(self):
|
|
# Deepcopy everything but the settings!
|
|
return self.__class__(msgctxt_lines=self.msgctxt_lines[:], msgid_lines=self.msgid_lines[:],
|
|
msgstr_lines=self.msgstr_lines[:], comment_lines=self.comment_lines[:],
|
|
is_commented=self.is_commented, is_fuzzy=self.is_fuzzy, settings=self.settings)
|
|
|
|
def normalize(self, max_len=80):
|
|
"""
|
|
Normalize this message, call this before exporting it...
|
|
Currently normalize msgctxt, msgid and msgstr lines to given max_len (if below 1, make them single line).
|
|
"""
|
|
max_len -= 2 # The two quotes!
|
|
|
|
def _splitlines(text):
|
|
lns = text.splitlines()
|
|
return [l + "\n" for l in lns[:-1]] + lns[-1:]
|
|
|
|
# We do not need the full power of textwrap... We just split first at escaped new lines, then into each line
|
|
# if needed... No word splitting, nor fancy spaces handling!
|
|
def _wrap(text, max_len, init_len):
|
|
if len(text) + init_len < max_len:
|
|
return [text]
|
|
lines = _splitlines(text)
|
|
ret = []
|
|
for l in lines:
|
|
tmp = []
|
|
cur_len = 0
|
|
words = l.split(' ')
|
|
for w in words:
|
|
cur_len += len(w) + 1
|
|
if cur_len > (max_len - 1) and tmp:
|
|
ret.append(" ".join(tmp) + " ")
|
|
del tmp[:]
|
|
cur_len = len(w) + 1
|
|
tmp.append(w)
|
|
if tmp:
|
|
ret.append(" ".join(tmp))
|
|
return ret
|
|
|
|
if max_len < 1:
|
|
self.msgctxt_lines = _splitlines(self.msgctxt)
|
|
self.msgid_lines = _splitlines(self.msgid)
|
|
self.msgstr_lines = _splitlines(self.msgstr)
|
|
else:
|
|
init_len = len(self.settings.PO_MSGCTXT) + 1
|
|
if self.is_commented:
|
|
init_len += len(self.settings.PO_COMMENT_PREFIX_MSG)
|
|
self.msgctxt_lines = _wrap(self.msgctxt, max_len, init_len)
|
|
|
|
init_len = len(self.settings.PO_MSGID) + 1
|
|
if self.is_commented:
|
|
init_len += len(self.settings.PO_COMMENT_PREFIX_MSG)
|
|
self.msgid_lines = _wrap(self.msgid, max_len, init_len)
|
|
|
|
init_len = len(self.settings.PO_MSGSTR) + 1
|
|
if self.is_commented:
|
|
init_len += len(self.settings.PO_COMMENT_PREFIX_MSG)
|
|
self.msgstr_lines = _wrap(self.msgstr, max_len, init_len)
|
|
|
|
# Be sure comment lines are not duplicated (can happen with sources...).
|
|
tmp = []
|
|
for l in self.comment_lines:
|
|
if l not in tmp:
|
|
tmp.append(l)
|
|
self.comment_lines = tmp
|
|
|
|
_esc_quotes = re.compile(r'(?!<\\)((?:\\\\)*)"')
|
|
_unesc_quotes = re.compile(r'(?!<\\)((?:\\\\)*)\\"')
|
|
_esc_names = ("msgctxt_lines", "msgid_lines", "msgstr_lines")
|
|
_esc_names_all = _esc_names + ("comment_lines",)
|
|
|
|
@classmethod
|
|
def do_escape(cls, txt):
|
|
"""Replace some chars by their escaped versions!"""
|
|
if "\n" in txt:
|
|
txt = txt.replace("\n", r"\n")
|
|
if "\t" in txt:
|
|
txt.replace("\t", r"\t")
|
|
if '"' in txt:
|
|
txt = cls._esc_quotes.sub(r'\1\"', txt)
|
|
return txt
|
|
|
|
@classmethod
|
|
def do_unescape(cls, txt):
|
|
"""Replace escaped chars by real ones!"""
|
|
if r"\n" in txt:
|
|
txt = txt.replace(r"\n", "\n")
|
|
if r"\t" in txt:
|
|
txt = txt.replace(r"\t", "\t")
|
|
if r'\"' in txt:
|
|
txt = cls._unesc_quotes.sub(r'\1"', txt)
|
|
return txt
|
|
|
|
def escape(self, do_all=False):
|
|
names = self._esc_names_all if do_all else self._esc_names
|
|
for name in names:
|
|
setattr(self, name, [self.do_escape(l) for l in getattr(self, name)])
|
|
|
|
def unescape(self, do_all=True):
|
|
names = self._esc_names_all if do_all else self._esc_names
|
|
for name in names:
|
|
setattr(self, name, [self.do_unescape(l) for l in getattr(self, name)])
|
|
|
|
|
|
class I18nMessages:
|
|
"""
|
|
Internal representation of messages for one language (iso code), with additional stats info.
|
|
"""
|
|
|
|
# Avoid parsing again!
|
|
# Keys should be (pseudo) file-names, values are tuples (hash, I18nMessages)
|
|
# Note: only used by po parser currently!
|
|
#_parser_cache = {}
|
|
|
|
def __init__(self, uid=None, kind=None, key=None, src=None, settings=settings):
|
|
self.settings = settings
|
|
self.uid = uid if uid is not None else settings.PARSER_TEMPLATE_ID
|
|
self.msgs = self._new_messages()
|
|
self.trans_msgs = set()
|
|
self.fuzzy_msgs = set()
|
|
self.comm_msgs = set()
|
|
self.ttip_msgs = set()
|
|
self.contexts = set()
|
|
self.nbr_msgs = 0
|
|
self.nbr_trans_msgs = 0
|
|
self.nbr_ttips = 0
|
|
self.nbr_trans_ttips = 0
|
|
self.nbr_comm_msgs = 0
|
|
self.nbr_signs = 0
|
|
self.nbr_trans_signs = 0
|
|
self.parsing_errors = []
|
|
if kind and src:
|
|
self.parse(kind, key, src)
|
|
self.update_info()
|
|
|
|
self._reverse_cache = None
|
|
|
|
@staticmethod
|
|
def _new_messages():
|
|
return getattr(collections, 'OrderedDict', dict)()
|
|
|
|
@classmethod
|
|
def gen_empty_messages(cls, uid, blender_ver, blender_rev, time, year, default_copyright=True, settings=settings):
|
|
"""Generate an empty I18nMessages object (only header is present!)."""
|
|
fmt = settings.PO_HEADER_MSGSTR
|
|
msgstr = fmt.format(blender_ver=str(blender_ver), blender_rev=int(blender_rev), time=str(time), uid=str(uid))
|
|
comment = ""
|
|
if default_copyright:
|
|
comment = settings.PO_HEADER_COMMENT_COPYRIGHT.format(year=str(year))
|
|
comment = comment + settings.PO_HEADER_COMMENT
|
|
|
|
msgs = cls(uid=uid, settings=settings)
|
|
key = settings.PO_HEADER_KEY
|
|
msgs.msgs[key] = I18nMessage([key[0]], [key[1]], msgstr.split("\n"), comment.split("\n"),
|
|
False, False, settings=settings)
|
|
msgs.update_info()
|
|
|
|
return msgs
|
|
|
|
def normalize(self, max_len=80):
|
|
for msg in self.msgs.values():
|
|
msg.normalize(max_len)
|
|
|
|
def escape(self, do_all=False):
|
|
for msg in self.msgs.values():
|
|
msg.escape(do_all)
|
|
|
|
def unescape(self, do_all=True):
|
|
for msg in self.msgs.values():
|
|
msg.unescape(do_all)
|
|
|
|
def check(self, fix=False):
|
|
"""
|
|
Check consistency between messages and their keys!
|
|
Check messages using format stuff are consistant between msgid and msgstr!
|
|
If fix is True, tries to fix the issues.
|
|
Return a list of found errors (empty if everything went OK!).
|
|
"""
|
|
ret = []
|
|
default_context = self.settings.DEFAULT_CONTEXT
|
|
_format = re.compile("%[.0-9]*[tslfd]").findall
|
|
done_keys = set()
|
|
tmp = {}
|
|
rem = set()
|
|
for key, msg in self.msgs.items():
|
|
msgctxt, msgid, msgstr = msg.msgctxt, msg.msgid, msg.msgstr
|
|
real_key = (msgctxt or default_context, msgid)
|
|
if key != real_key:
|
|
ret.append("Error! msg's context/message do not match its key ({} / {})".format(real_key, key))
|
|
if real_key in self.msgs:
|
|
ret.append("Error! msg's real_key already used!")
|
|
if fix:
|
|
rem.add(real_key)
|
|
elif fix:
|
|
tmp[real_key] = msg
|
|
done_keys.add(key)
|
|
if '%' in msgid and msgstr and len(_format(msgid)) != len(_format(msgstr)):
|
|
if not msg.is_fuzzy:
|
|
ret.append("Error! msg's format entities are not matched in msgid and msgstr ({})".format(real_key))
|
|
if fix:
|
|
msg.msgstr = ""
|
|
for k in rem:
|
|
del self.msgs[k]
|
|
self.msgs.update(tmp)
|
|
return ret
|
|
|
|
def clean_commented(self):
|
|
self.update_info()
|
|
nbr = len(self.comm_msgs)
|
|
for k in self.comm_msgs:
|
|
del self.msgs[k]
|
|
return nbr
|
|
|
|
def rtl_process(self):
|
|
keys = []
|
|
trans = []
|
|
for k, m in self.msgs.items():
|
|
keys.append(k)
|
|
trans.append(m.msgstr)
|
|
trans = utils_rtl.log2vis(trans, self.settings)
|
|
for k, t in zip(keys, trans):
|
|
self.msgs[k].msgstr = t
|
|
|
|
def merge(self, msgs, replace=False):
|
|
"""
|
|
Merge translations from msgs into self, following those rules:
|
|
* If a msg is in self and not in msgs, keep self untouched.
|
|
* If a msg is in msgs and not in self, skip it.
|
|
* Else (msg both in self and msgs):
|
|
* If self is not translated and msgs is translated or fuzzy, replace by msgs.
|
|
* If self is fuzzy, and msgs is translated, replace by msgs.
|
|
* If self is fuzzy, and msgs is fuzzy, and replace is True, replace by msgs.
|
|
* If self is translated, and msgs is translated, and replace is True, replace by msgs.
|
|
* Else, skip it!
|
|
"""
|
|
for k, m in msgs.msgs.items():
|
|
if k not in self.msgs:
|
|
continue
|
|
sm = self.msgs[k]
|
|
if (sm.is_commented or m.is_commented or not m.msgstr):
|
|
continue
|
|
if (not sm.msgstr or replace or (sm.is_fuzzy and (not m.is_fuzzy or replace))):
|
|
sm.msgstr = m.msgstr
|
|
sm.is_fuzzy = m.is_fuzzy
|
|
|
|
def update(self, ref, use_similar=None, keep_old_commented=True):
|
|
"""
|
|
Update this I18nMessage with the ref one. Translations from ref are never used. Source comments from ref
|
|
completely replace current ones. If use_similar is not 0.0, it will try to match new messages in ref with an
|
|
existing one. Messages no more found in ref will be marked as commented if keep_old_commented is True,
|
|
or removed.
|
|
"""
|
|
if use_similar is None:
|
|
use_similar = self.settings.SIMILAR_MSGID_THRESHOLD
|
|
|
|
similar_pool = {}
|
|
if use_similar > 0.0:
|
|
for key, msg in self.msgs.items():
|
|
if msg.msgstr: # No need to waste time with void translations!
|
|
similar_pool.setdefault(key[1], set()).add(key)
|
|
|
|
msgs = self._new_messages().fromkeys(ref.msgs.keys())
|
|
ref_keys = set(ref.msgs.keys())
|
|
org_keys = set(self.msgs.keys())
|
|
new_keys = ref_keys - org_keys
|
|
removed_keys = org_keys - ref_keys
|
|
|
|
# First process keys present in both org and ref messages.
|
|
for key in ref_keys - new_keys:
|
|
msg, refmsg = self.msgs[key], ref.msgs[key]
|
|
msg.sources = refmsg.sources
|
|
msg.is_commented = refmsg.is_commented
|
|
msgs[key] = msg
|
|
|
|
# Next process new keys.
|
|
if use_similar > 0.0:
|
|
with concurrent.futures.ProcessPoolExecutor() as exctr:
|
|
for key, msgid in exctr.map(get_best_similar,
|
|
tuple((nk, use_similar, tuple(similar_pool.keys())) for nk in new_keys)):
|
|
if msgid:
|
|
# Try to get the same context, else just get one...
|
|
skey = (key[0], msgid)
|
|
if skey not in similar_pool[msgid]:
|
|
skey = tuple(similar_pool[msgid])[0]
|
|
# We keep org translation and comments, and mark message as fuzzy.
|
|
msg, refmsg = self.msgs[skey].copy(), ref.msgs[key]
|
|
msg.msgctxt = refmsg.msgctxt
|
|
msg.msgid = refmsg.msgid
|
|
msg.sources = refmsg.sources
|
|
msg.is_fuzzy = True
|
|
msg.is_commented = refmsg.is_commented
|
|
msgs[key] = msg
|
|
else:
|
|
msgs[key] = ref.msgs[key]
|
|
else:
|
|
for key in new_keys:
|
|
msgs[key] = ref.msgs[key]
|
|
|
|
# Add back all "old" and already commented messages as commented ones, if required
|
|
# (and translation was not void!).
|
|
if keep_old_commented:
|
|
for key in removed_keys:
|
|
msgs[key] = self.msgs[key]
|
|
msgs[key].is_commented = True
|
|
msgs[key].sources = []
|
|
|
|
# Special 'meta' message, change project ID version and pot creation date...
|
|
key = self.settings.PO_HEADER_KEY
|
|
rep = []
|
|
markers = ("Project-Id-Version:", "POT-Creation-Date:")
|
|
for mrk in markers:
|
|
for rl in ref.msgs[key].msgstr_lines:
|
|
if rl.startswith(mrk):
|
|
for idx, ml in enumerate(msgs[key].msgstr_lines):
|
|
if ml.startswith(mrk):
|
|
rep.append((idx, rl))
|
|
for idx, txt in rep:
|
|
msgs[key].msgstr_lines[idx] = txt
|
|
|
|
# And finalize the update!
|
|
self.msgs = msgs
|
|
|
|
def update_info(self):
|
|
self.trans_msgs.clear()
|
|
self.fuzzy_msgs.clear()
|
|
self.comm_msgs.clear()
|
|
self.ttip_msgs.clear()
|
|
self.contexts.clear()
|
|
self.nbr_signs = 0
|
|
self.nbr_trans_signs = 0
|
|
for key, msg in self.msgs.items():
|
|
if key == self.settings.PO_HEADER_KEY:
|
|
continue
|
|
if msg.is_commented:
|
|
self.comm_msgs.add(key)
|
|
else:
|
|
if msg.msgstr:
|
|
self.trans_msgs.add(key)
|
|
if msg.is_fuzzy:
|
|
self.fuzzy_msgs.add(key)
|
|
if msg.is_tooltip:
|
|
self.ttip_msgs.add(key)
|
|
self.contexts.add(key[0])
|
|
self.nbr_signs += len(msg.msgid)
|
|
self.nbr_trans_signs += len(msg.msgstr)
|
|
self.nbr_msgs = len(self.msgs)
|
|
self.nbr_trans_msgs = len(self.trans_msgs - self.fuzzy_msgs)
|
|
self.nbr_ttips = len(self.ttip_msgs)
|
|
self.nbr_trans_ttips = len(self.ttip_msgs & (self.trans_msgs - self.fuzzy_msgs))
|
|
self.nbr_comm_msgs = len(self.comm_msgs)
|
|
|
|
def print_info(self, prefix="", output=print, print_stats=True, print_errors=True):
|
|
"""
|
|
Print out some info about an I18nMessages object.
|
|
"""
|
|
lvl = 0.0
|
|
lvl_ttips = 0.0
|
|
lvl_comm = 0.0
|
|
lvl_trans_ttips = 0.0
|
|
lvl_ttips_in_trans = 0.0
|
|
if self.nbr_msgs > 0:
|
|
lvl = float(self.nbr_trans_msgs) / float(self.nbr_msgs)
|
|
lvl_ttips = float(self.nbr_ttips) / float(self.nbr_msgs)
|
|
lvl_comm = float(self.nbr_comm_msgs) / float(self.nbr_msgs + self.nbr_comm_msgs)
|
|
if self.nbr_ttips > 0:
|
|
lvl_trans_ttips = float(self.nbr_trans_ttips) / float(self.nbr_ttips)
|
|
if self.nbr_trans_msgs > 0:
|
|
lvl_ttips_in_trans = float(self.nbr_trans_ttips) / float(self.nbr_trans_msgs)
|
|
|
|
lines = []
|
|
if print_stats:
|
|
lines += [
|
|
"",
|
|
"{:>6.1%} done! ({} translated messages over {}).\n"
|
|
"".format(lvl, self.nbr_trans_msgs, self.nbr_msgs),
|
|
"{:>6.1%} of messages are tooltips ({} over {}).\n"
|
|
"".format(lvl_ttips, self.nbr_ttips, self.nbr_msgs),
|
|
"{:>6.1%} of tooltips are translated ({} over {}).\n"
|
|
"".format(lvl_trans_ttips, self.nbr_trans_ttips, self.nbr_ttips),
|
|
"{:>6.1%} of translated messages are tooltips ({} over {}).\n"
|
|
"".format(lvl_ttips_in_trans, self.nbr_trans_ttips, self.nbr_trans_msgs),
|
|
"{:>6.1%} of messages are commented ({} over {}).\n"
|
|
"".format(lvl_comm, self.nbr_comm_msgs, self.nbr_comm_msgs + self.nbr_msgs),
|
|
"This translation is currently made of {} signs.\n".format(self.nbr_trans_signs)
|
|
]
|
|
if print_errors and self.parsing_errors:
|
|
lines += ["WARNING! Errors during parsing:\n"]
|
|
lines += [" Around line {}: {}\n".format(line, error) for line, error in self.parsing_errors]
|
|
output(prefix.join(lines))
|
|
|
|
def invalidate_reverse_cache(self, rebuild_now=False):
|
|
"""
|
|
Invalidate the reverse cache used by find_best_messages_matches.
|
|
"""
|
|
self._reverse_cache = None
|
|
if rebuild_now:
|
|
src_to_msg, ctxt_to_msg, msgid_to_msg, msgstr_to_msg = {}, {}, {}, {}
|
|
for key, msg in self.msgs.items():
|
|
if msg.is_commented:
|
|
continue
|
|
ctxt, msgid = key
|
|
ctxt_to_msg.setdefault(ctxt, set()).add(key)
|
|
msgid_to_msg.setdefault(msgid, set()).add(key)
|
|
msgstr_to_msg.setdefault(msg.msgstr, set()).add(key)
|
|
for src in msg.sources:
|
|
src_to_msg.setdefault(src, set()).add(key)
|
|
self._reverse_cache = (src_to_msg, ctxt_to_msg, msgid_to_msg, msgstr_to_msg)
|
|
|
|
def find_best_messages_matches(self, msgs, msgmap, rna_ctxt, rna_struct_name, rna_prop_name, rna_enum_name):
|
|
"""
|
|
Try to find the best I18nMessages (i.e. context/msgid pairs) for the given UI messages:
|
|
msgs: an object containing properties listed in msgmap's values.
|
|
msgmap: a dict of various messages to use for search:
|
|
{"but_label": subdict, "rna_label": subdict, "enum_label": subdict,
|
|
"but_tip": subdict, "rna_tip": subdict, "enum_tip": subdict}
|
|
each subdict being like that:
|
|
{"msgstr": id, "msgid": id, "msg_flags": id, "key": set()}
|
|
where msgstr and msgid are identifiers of string props in msgs (resp. translated and org message),
|
|
msg_flags is not used here, and key is a set of matching (msgctxt, msgid) keys for the item.
|
|
The other parameters are about the RNA element from which the strings come from, if it could be determined:
|
|
rna_ctxt: the labels' i18n context.
|
|
rna_struct_name, rna_prop_name, rna_enum_name: should be self-explanatory!
|
|
"""
|
|
# Build helper mappings.
|
|
# Note it's user responsibility to know when to invalidate (and hence force rebuild) this cache!
|
|
if self._reverse_cache is None:
|
|
self.invalidate_reverse_cache(True)
|
|
src_to_msg, ctxt_to_msg, msgid_to_msg, msgstr_to_msg = self._reverse_cache
|
|
|
|
# print(len(src_to_msg), len(ctxt_to_msg), len(msgid_to_msg), len(msgstr_to_msg))
|
|
|
|
# Build RNA key.
|
|
src, src_rna, src_enum = bpy.utils.make_rna_paths(rna_struct_name, rna_prop_name, rna_enum_name)
|
|
print("src: ", src_rna, src_enum)
|
|
|
|
# Labels.
|
|
elbl = getattr(msgs, msgmap["enum_label"]["msgstr"])
|
|
if elbl:
|
|
# Enum items' labels have no i18n context...
|
|
k = ctxt_to_msg[self.settings.DEFAULT_CONTEXT].copy()
|
|
if elbl in msgid_to_msg:
|
|
k &= msgid_to_msg[elbl]
|
|
elif elbl in msgstr_to_msg:
|
|
k &= msgstr_to_msg[elbl]
|
|
else:
|
|
k = set()
|
|
# We assume if we already have only one key, it's the good one!
|
|
if len(k) > 1 and src_enum in src_to_msg:
|
|
k &= src_to_msg[src_enum]
|
|
msgmap["enum_label"]["key"] = k
|
|
rlbl = getattr(msgs, msgmap["rna_label"]["msgstr"])
|
|
#print("rna label: " + rlbl, rlbl in msgid_to_msg, rlbl in msgstr_to_msg)
|
|
if rlbl:
|
|
k = ctxt_to_msg[rna_ctxt].copy()
|
|
if k and rlbl in msgid_to_msg:
|
|
k &= msgid_to_msg[rlbl]
|
|
elif k and rlbl in msgstr_to_msg:
|
|
k &= msgstr_to_msg[rlbl]
|
|
else:
|
|
k = set()
|
|
# We assume if we already have only one key, it's the good one!
|
|
if len(k) > 1 and src_rna in src_to_msg:
|
|
k &= src_to_msg[src_rna]
|
|
msgmap["rna_label"]["key"] = k
|
|
blbl = getattr(msgs, msgmap["but_label"]["msgstr"])
|
|
blbls = [blbl]
|
|
if blbl.endswith(self.settings.NUM_BUTTON_SUFFIX):
|
|
# Num buttons report their label with a trailing ': '...
|
|
blbls.append(blbl[:-len(self.settings.NUM_BUTTON_SUFFIX)])
|
|
print("button label: " + blbl)
|
|
if blbl and elbl not in blbls and (rlbl not in blbls or rna_ctxt != self.settings.DEFAULT_CONTEXT):
|
|
# Always Default context for button label :/
|
|
k = ctxt_to_msg[self.settings.DEFAULT_CONTEXT].copy()
|
|
found = False
|
|
for bl in blbls:
|
|
if bl in msgid_to_msg:
|
|
k &= msgid_to_msg[bl]
|
|
found = True
|
|
break
|
|
elif bl in msgstr_to_msg:
|
|
k &= msgstr_to_msg[bl]
|
|
found = True
|
|
break
|
|
if not found:
|
|
k = set()
|
|
# XXX No need to check against RNA path here, if blabel is different
|
|
# from rlabel, should not match anyway!
|
|
msgmap["but_label"]["key"] = k
|
|
|
|
# Tips (they never have a specific context).
|
|
etip = getattr(msgs, msgmap["enum_tip"]["msgstr"])
|
|
#print("enum tip: " + etip)
|
|
if etip:
|
|
k = ctxt_to_msg[self.settings.DEFAULT_CONTEXT].copy()
|
|
if etip in msgid_to_msg:
|
|
k &= msgid_to_msg[etip]
|
|
elif etip in msgstr_to_msg:
|
|
k &= msgstr_to_msg[etip]
|
|
else:
|
|
k = set()
|
|
# We assume if we already have only one key, it's the good one!
|
|
if len(k) > 1 and src_enum in src_to_msg:
|
|
k &= src_to_msg[src_enum]
|
|
msgmap["enum_tip"]["key"] = k
|
|
rtip = getattr(msgs, msgmap["rna_tip"]["msgstr"])
|
|
#print("rna tip: " + rtip)
|
|
if rtip:
|
|
k = ctxt_to_msg[self.settings.DEFAULT_CONTEXT].copy()
|
|
if k and rtip in msgid_to_msg:
|
|
k &= msgid_to_msg[rtip]
|
|
elif k and rtip in msgstr_to_msg:
|
|
k &= msgstr_to_msg[rtip]
|
|
else:
|
|
k = set()
|
|
# We assume if we already have only one key, it's the good one!
|
|
if len(k) > 1 and src_rna in src_to_msg:
|
|
k &= src_to_msg[src_rna]
|
|
msgmap["rna_tip"]["key"] = k
|
|
#print(k)
|
|
btip = getattr(msgs, msgmap["but_tip"]["msgstr"])
|
|
#print("button tip: " + btip)
|
|
if btip and btip not in {rtip, etip}:
|
|
k = ctxt_to_msg[self.settings.DEFAULT_CONTEXT].copy()
|
|
if btip in msgid_to_msg:
|
|
k &= msgid_to_msg[btip]
|
|
elif btip in msgstr_to_msg:
|
|
k &= msgstr_to_msg[btip]
|
|
else:
|
|
k = set()
|
|
# XXX No need to check against RNA path here, if btip is different from rtip, should not match anyway!
|
|
msgmap["but_tip"]["key"] = k
|
|
|
|
def parse(self, kind, key, src):
|
|
del self.parsing_errors[:]
|
|
self.parsers[kind](self, src, key)
|
|
if self.parsing_errors:
|
|
print("{} ({}):".format(key, src))
|
|
self.print_info(print_stats=False)
|
|
print("The parser solved them as well as it could...")
|
|
self.update_info()
|
|
|
|
def parse_messages_from_po(self, src, key=None):
|
|
"""
|
|
Parse a po file.
|
|
Note: This function will silently "arrange" mis-formated entries, thus using afterward write_messages() should
|
|
always produce a po-valid file, though not correct!
|
|
"""
|
|
reading_msgid = False
|
|
reading_msgstr = False
|
|
reading_msgctxt = False
|
|
reading_comment = False
|
|
is_commented = False
|
|
is_fuzzy = False
|
|
msgctxt_lines = []
|
|
msgid_lines = []
|
|
msgstr_lines = []
|
|
comment_lines = []
|
|
|
|
default_context = self.settings.DEFAULT_CONTEXT
|
|
|
|
# Helper function
|
|
def finalize_message(self, line_nr):
|
|
nonlocal reading_msgid, reading_msgstr, reading_msgctxt, reading_comment
|
|
nonlocal is_commented, is_fuzzy, msgid_lines, msgstr_lines, msgctxt_lines, comment_lines
|
|
|
|
msgid = I18nMessage.do_unescape("".join(msgid_lines))
|
|
msgctxt = I18nMessage.do_unescape("".join(msgctxt_lines))
|
|
msgkey = (msgctxt or default_context, msgid)
|
|
|
|
# Never allow overriding existing msgid/msgctxt pairs!
|
|
if msgkey in self.msgs:
|
|
self.parsing_errors.append((line_nr, "{} context/msgid is already in current messages!".format(msgkey)))
|
|
return
|
|
|
|
self.msgs[msgkey] = I18nMessage(msgctxt_lines, msgid_lines, msgstr_lines, comment_lines,
|
|
is_commented, is_fuzzy, settings=self.settings)
|
|
|
|
# Let's clean up and get ready for next message!
|
|
reading_msgid = reading_msgstr = reading_msgctxt = reading_comment = False
|
|
is_commented = is_fuzzy = False
|
|
msgctxt_lines = []
|
|
msgid_lines = []
|
|
msgstr_lines = []
|
|
comment_lines = []
|
|
|
|
# try to use src as file name...
|
|
if os.path.isfile(src):
|
|
if os.stat(src).st_size > self.settings.PARSER_MAX_FILE_SIZE:
|
|
# Security, else we could read arbitrary huge files!
|
|
print("WARNING: skipping file {}, too huge!".format(src))
|
|
return
|
|
if not key:
|
|
key = src
|
|
with open(src, 'r', encoding="utf-8") as f:
|
|
src = f.read()
|
|
|
|
_msgctxt = self.settings.PO_MSGCTXT
|
|
_comm_msgctxt = self.settings.PO_COMMENT_PREFIX_MSG + _msgctxt
|
|
_len_msgctxt = len(_msgctxt + '"')
|
|
_len_comm_msgctxt = len(_comm_msgctxt + '"')
|
|
_msgid = self.settings.PO_MSGID
|
|
_comm_msgid = self.settings.PO_COMMENT_PREFIX_MSG + _msgid
|
|
_len_msgid = len(_msgid + '"')
|
|
_len_comm_msgid = len(_comm_msgid + '"')
|
|
_msgstr = self.settings.PO_MSGSTR
|
|
_comm_msgstr = self.settings.PO_COMMENT_PREFIX_MSG + _msgstr
|
|
_len_msgstr = len(_msgstr + '"')
|
|
_len_comm_msgstr = len(_comm_msgstr + '"')
|
|
_comm_str = self.settings.PO_COMMENT_PREFIX_MSG
|
|
_comm_fuzzy = self.settings.PO_COMMENT_FUZZY
|
|
_len_comm_str = len(_comm_str + '"')
|
|
|
|
# Main loop over all lines in src...
|
|
for line_nr, line in enumerate(src.splitlines()):
|
|
if line == "":
|
|
if reading_msgstr:
|
|
finalize_message(self, line_nr)
|
|
continue
|
|
|
|
elif line.startswith(_msgctxt) or line.startswith(_comm_msgctxt):
|
|
reading_comment = False
|
|
reading_ctxt = True
|
|
if line.startswith(_comm_str):
|
|
is_commented = True
|
|
line = line[_len_comm_msgctxt:-1]
|
|
else:
|
|
line = line[_len_msgctxt:-1]
|
|
msgctxt_lines.append(line)
|
|
|
|
elif line.startswith(_msgid) or line.startswith(_comm_msgid):
|
|
reading_comment = False
|
|
reading_msgid = True
|
|
if line.startswith(_comm_str):
|
|
if not is_commented and reading_ctxt:
|
|
self.parsing_errors.append((line_nr, "commented msgid following regular msgctxt"))
|
|
is_commented = True
|
|
line = line[_len_comm_msgid:-1]
|
|
else:
|
|
line = line[_len_msgid:-1]
|
|
reading_ctxt = False
|
|
msgid_lines.append(line)
|
|
|
|
elif line.startswith(_msgstr) or line.startswith(_comm_msgstr):
|
|
if not reading_msgid:
|
|
self.parsing_errors.append((line_nr, "msgstr without a prior msgid"))
|
|
else:
|
|
reading_msgid = False
|
|
reading_msgstr = True
|
|
if line.startswith(_comm_str):
|
|
line = line[_len_comm_msgstr:-1]
|
|
if not is_commented:
|
|
self.parsing_errors.append((line_nr, "commented msgstr following regular msgid"))
|
|
else:
|
|
line = line[_len_msgstr:-1]
|
|
if is_commented:
|
|
self.parsing_errors.append((line_nr, "regular msgstr following commented msgid"))
|
|
msgstr_lines.append(line)
|
|
|
|
elif line.startswith(_comm_str[0]):
|
|
if line.startswith(_comm_str):
|
|
if reading_msgctxt:
|
|
if is_commented:
|
|
msgctxt_lines.append(line[_len_comm_str:-1])
|
|
else:
|
|
msgctxt_lines.append(line)
|
|
self.parsing_errors.append((line_nr, "commented string while reading regular msgctxt"))
|
|
elif reading_msgid:
|
|
if is_commented:
|
|
msgid_lines.append(line[_len_comm_str:-1])
|
|
else:
|
|
msgid_lines.append(line)
|
|
self.parsing_errors.append((line_nr, "commented string while reading regular msgid"))
|
|
elif reading_msgstr:
|
|
if is_commented:
|
|
msgstr_lines.append(line[_len_comm_str:-1])
|
|
else:
|
|
msgstr_lines.append(line)
|
|
self.parsing_errors.append((line_nr, "commented string while reading regular msgstr"))
|
|
else:
|
|
if reading_msgctxt or reading_msgid or reading_msgstr:
|
|
self.parsing_errors.append((line_nr,
|
|
"commented string within msgctxt, msgid or msgstr scope, ignored"))
|
|
elif line.startswith(_comm_fuzzy):
|
|
is_fuzzy = True
|
|
else:
|
|
comment_lines.append(line)
|
|
reading_comment = True
|
|
|
|
else:
|
|
if reading_msgctxt:
|
|
msgctxt_lines.append(line[1:-1])
|
|
elif reading_msgid:
|
|
msgid_lines.append(line[1:-1])
|
|
elif reading_msgstr:
|
|
line = line[1:-1]
|
|
msgstr_lines.append(line)
|
|
else:
|
|
self.parsing_errors.append((line_nr, "regular string outside msgctxt, msgid or msgstr scope"))
|
|
#self.parsing_errors += (str(comment_lines), str(msgctxt_lines), str(msgid_lines), str(msgstr_lines))
|
|
|
|
# If no final empty line, last message is not finalized!
|
|
if reading_msgstr:
|
|
finalize_message(self, line_nr)
|
|
self.unescape()
|
|
|
|
def write(self, kind, dest):
|
|
self.writers[kind](self, dest)
|
|
|
|
def write_messages_to_po(self, fname):
|
|
"""
|
|
Write messages in fname po file.
|
|
"""
|
|
default_context = self.settings.DEFAULT_CONTEXT
|
|
|
|
def _write(self, f):
|
|
_msgctxt = self.settings.PO_MSGCTXT
|
|
_msgid = self.settings.PO_MSGID
|
|
_msgstr = self.settings.PO_MSGSTR
|
|
_comm = self.settings.PO_COMMENT_PREFIX_MSG
|
|
|
|
self.escape()
|
|
|
|
for num, msg in enumerate(self.msgs.values()):
|
|
f.write("\n".join(msg.comment_lines))
|
|
# Only mark as fuzzy if msgstr is not empty!
|
|
if msg.is_fuzzy and msg.msgstr:
|
|
f.write("\n" + self.settings.PO_COMMENT_FUZZY)
|
|
_p = _comm if msg.is_commented else ""
|
|
chunks = []
|
|
if msg.msgctxt and msg.msgctxt != default_context:
|
|
if len(msg.msgctxt_lines) > 1:
|
|
chunks += [
|
|
"\n" + _p + _msgctxt + "\"\"\n" + _p + "\"",
|
|
("\"\n" + _p + "\"").join(msg.msgctxt_lines),
|
|
"\"",
|
|
]
|
|
else:
|
|
chunks += ["\n" + _p + _msgctxt + "\"" + msg.msgctxt + "\""]
|
|
if len(msg.msgid_lines) > 1:
|
|
chunks += [
|
|
"\n" + _p + _msgid + "\"\"\n" + _p + "\"",
|
|
("\"\n" + _p + "\"").join(msg.msgid_lines),
|
|
"\"",
|
|
]
|
|
else:
|
|
chunks += ["\n" + _p + _msgid + "\"" + msg.msgid + "\""]
|
|
if len(msg.msgstr_lines) > 1:
|
|
chunks += [
|
|
"\n" + _p + _msgstr + "\"\"\n" + _p + "\"",
|
|
("\"\n" + _p + "\"").join(msg.msgstr_lines),
|
|
"\"",
|
|
]
|
|
else:
|
|
chunks += ["\n" + _p + _msgstr + "\"" + msg.msgstr + "\""]
|
|
chunks += ["\n\n"]
|
|
f.write("".join(chunks))
|
|
|
|
self.unescape()
|
|
|
|
self.normalize(max_len=0) # No wrapping for now...
|
|
if isinstance(fname, str):
|
|
with open(fname, 'w', encoding="utf-8") as f:
|
|
_write(self, f)
|
|
# Else assume fname is already a file(like) object!
|
|
else:
|
|
_write(self, fname)
|
|
|
|
def write_messages_to_mo(self, fname):
|
|
"""
|
|
Write messages in fname mo file.
|
|
"""
|
|
# XXX Temp solution, until I can make own mo generator working...
|
|
import subprocess
|
|
with tempfile.NamedTemporaryFile(mode='w+', encoding="utf-8") as tmp_po_f:
|
|
self.write_messages_to_po(tmp_po_f)
|
|
cmd = (self.settings.GETTEXT_MSGFMT_EXECUTABLE,
|
|
"--statistics", # show stats
|
|
tmp_po_f.name,
|
|
"-o",
|
|
fname,
|
|
)
|
|
print("Running ", " ".join(cmd))
|
|
ret = subprocess.call(cmd)
|
|
print("Finished.")
|
|
return
|
|
# XXX Code below is currently broken (generates corrupted mo files it seems :( )!
|
|
# Using http://www.gnu.org/software/gettext/manual/html_node/MO-Files.html notation.
|
|
# Not generating hash table!
|
|
# Only translated, unfuzzy messages are taken into account!
|
|
default_context = self.settings.DEFAULT_CONTEXT
|
|
msgs = tuple(v for v in self.msgs.values() if not (v.is_fuzzy or v.is_commented) and v.msgstr and v.msgid)
|
|
msgs = sorted(msgs[:2],
|
|
key=lambda e: (e.msgctxt + e.msgid) if (e.msgctxt and e.msgctxt != default_context) else e.msgid)
|
|
magic_nbr = 0x950412de
|
|
format_rev = 0
|
|
N = len(msgs)
|
|
O = 32
|
|
T = O + N * 8
|
|
S = 0
|
|
H = T + N * 8
|
|
# Prepare our data! we need key (optional context and msgid), translation, and offset and length of both.
|
|
# Offset are relative to start of their own list.
|
|
EOT = b"0x04" # Used to concatenate context and msgid
|
|
_msgid_offset = 0
|
|
_msgstr_offset = 0
|
|
def _gen(v):
|
|
nonlocal _msgid_offset, _msgstr_offset
|
|
msgid = v.msgid.encode("utf-8")
|
|
msgstr = v.msgstr.encode("utf-8")
|
|
if v.msgctxt and v.msgctxt != default_context:
|
|
msgctxt = v.msgctxt.encode("utf-8")
|
|
msgid = msgctxt + EOT + msgid
|
|
# Don't forget the final NULL char!
|
|
_msgid_len = len(msgid) + 1
|
|
_msgstr_len = len(msgstr) + 1
|
|
ret = ((msgid, _msgid_len, _msgid_offset), (msgstr, _msgstr_len, _msgstr_offset))
|
|
_msgid_offset += _msgid_len
|
|
_msgstr_offset += _msgstr_len
|
|
return ret
|
|
msgs = tuple(_gen(v) for v in msgs)
|
|
msgid_start = H
|
|
msgstr_start = msgid_start + _msgid_offset
|
|
print(N, msgstr_start + _msgstr_offset)
|
|
print(msgs)
|
|
|
|
with open(fname, 'wb') as f:
|
|
# Header...
|
|
f.write(struct.pack("=8I", magic_nbr, format_rev, N, O, T, S, H, 0))
|
|
# Msgid's length and offset.
|
|
f.write(b"".join(struct.pack("=2I", length, msgid_start + offset) for (_1, length, offset), _2 in msgs))
|
|
# Msgstr's length and offset.
|
|
f.write(b"".join(struct.pack("=2I", length, msgstr_start + offset) for _1, (_2, length, offset) in msgs))
|
|
# No hash table!
|
|
# Msgid's.
|
|
f.write(b"\0".join(msgid for (msgid, _1, _2), _3 in msgs) + b"\0")
|
|
# Msgstr's.
|
|
f.write(b"\0".join(msgstr for _1, (msgstr, _2, _3) in msgs) + b"\0")
|
|
|
|
parsers = {
|
|
"PO": parse_messages_from_po,
|
|
}
|
|
|
|
writers = {
|
|
"PO": write_messages_to_po,
|
|
"MO": write_messages_to_mo,
|
|
}
|
|
|
|
|
|
class I18n:
|
|
"""
|
|
Internal representation of a whole translation set.
|
|
"""
|
|
|
|
@staticmethod
|
|
def _parser_check_file(path, maxsize=settings.PARSER_MAX_FILE_SIZE,
|
|
_begin_marker=settings.PARSER_PY_MARKER_BEGIN,
|
|
_end_marker=settings.PARSER_PY_MARKER_END):
|
|
if os.stat(path).st_size > maxsize:
|
|
# Security, else we could read arbitrary huge files!
|
|
print("WARNING: skipping file {}, too huge!".format(path))
|
|
return None, None, None, False
|
|
txt = ""
|
|
with open(path) as f:
|
|
txt = f.read()
|
|
_in = 0
|
|
_out = len(txt)
|
|
if _begin_marker:
|
|
_in = None
|
|
if _begin_marker in txt:
|
|
_in = txt.index(_begin_marker) + len(_begin_marker)
|
|
if _end_marker:
|
|
_out = None
|
|
if _end_marker in txt:
|
|
_out = txt.index(_end_marker)
|
|
if _in is not None and _out is not None:
|
|
in_txt, txt, out_txt = txt[:_in], txt[_in:_out], txt[_out:]
|
|
elif _in is not None:
|
|
in_txt, txt, out_txt = txt[:_in], txt[_in:], None
|
|
elif _out is not None:
|
|
in_txt, txt, out_txt = None, txt[:_out], txt[_out:]
|
|
else:
|
|
in_txt, txt, out_txt = None, txt, None
|
|
return in_txt, txt, out_txt, (True if "translations_tuple" in txt else False)
|
|
|
|
@staticmethod
|
|
def _dst(self, path, uid, kind):
|
|
if isinstance(path, str):
|
|
if kind == 'PO':
|
|
if uid == self.settings.PARSER_TEMPLATE_ID:
|
|
if not path.endswith(".pot"):
|
|
return os.path.join(os.path.dirname(path), "blender.pot")
|
|
if not path.endswith(".po"):
|
|
return os.path.join(os.path.dirname(path), uid + ".po")
|
|
elif kind == 'PY':
|
|
if not path.endswith(".py"):
|
|
if self.src.get(self.settings.PARSER_PY_ID):
|
|
return self.src[self.settings.PARSER_PY_ID]
|
|
return os.path.join(os.path.dirname(path), "translations.py")
|
|
return path
|
|
|
|
def __init__(self, kind=None, src=None, langs=set(), settings=settings):
|
|
self.settings = settings
|
|
self.trans = {}
|
|
self.src = {} # Should have the same keys as self.trans (plus PARSER_PY_ID for py file)!
|
|
self.dst = self._dst # A callable that transforms src_path into dst_path!
|
|
if kind and src:
|
|
self.parse(kind, src, langs)
|
|
self.update_info()
|
|
|
|
def _py_file_get(self):
|
|
return self.src.get(self.settings.PARSER_PY_ID)
|
|
def _py_file_set(self, value):
|
|
self.src[self.settings.PARSER_PY_ID] = value
|
|
py_file = property(_py_file_get, _py_file_set)
|
|
|
|
def escape(self, do_all=False):
|
|
for trans in self.trans.values():
|
|
trans.escape(do_all)
|
|
|
|
def unescape(self, do_all=True):
|
|
for trans in self.trans.values():
|
|
trans.unescape(do_all)
|
|
|
|
def update_info(self):
|
|
self.nbr_trans = 0
|
|
self.lvl = 0.0
|
|
self.lvl_ttips = 0.0
|
|
self.lvl_trans_ttips = 0.0
|
|
self.lvl_ttips_in_trans = 0.0
|
|
self.lvl_comm = 0.0
|
|
self.nbr_signs = 0
|
|
self.nbr_trans_signs = 0
|
|
self.contexts = set()
|
|
|
|
if self.settings.PARSER_TEMPLATE_ID in self.trans:
|
|
self.nbr_trans = len(self.trans) - 1
|
|
self.nbr_signs = self.trans[self.settings.PARSER_TEMPLATE_ID].nbr_signs
|
|
else:
|
|
self.nbr_trans = len(self.trans)
|
|
for msgs in self.trans.values():
|
|
msgs.update_info()
|
|
if msgs.nbr_msgs > 0:
|
|
self.lvl += float(msgs.nbr_trans_msgs) / float(msgs.nbr_msgs)
|
|
self.lvl_ttips += float(msgs.nbr_ttips) / float(msgs.nbr_msgs)
|
|
self.lvl_comm += float(msgs.nbr_comm_msgs) / float(msgs.nbr_msgs + msgs.nbr_comm_msgs)
|
|
if msgs.nbr_ttips > 0:
|
|
self.lvl_trans_ttips = float(msgs.nbr_trans_ttips) / float(msgs.nbr_ttips)
|
|
if msgs.nbr_trans_msgs > 0:
|
|
self.lvl_ttips_in_trans = float(msgs.nbr_trans_ttips) / float(msgs.nbr_trans_msgs)
|
|
if self.nbr_signs == 0:
|
|
self.nbr_signs = msgs.nbr_signs
|
|
self.nbr_trans_signs += msgs.nbr_trans_signs
|
|
self.contexts |= msgs.contexts
|
|
|
|
def print_stats(self, prefix="", print_msgs=True):
|
|
"""
|
|
Print out some stats about an I18n object.
|
|
If print_msgs is True, it will also print all its translations' stats.
|
|
"""
|
|
if print_msgs:
|
|
msgs_prefix = prefix + " "
|
|
for key, msgs in self.trans.items():
|
|
if key == self.settings.PARSER_TEMPLATE_ID:
|
|
continue
|
|
print(prefix + key + ":")
|
|
msgs.print_stats(prefix=msgs_prefix)
|
|
print(prefix)
|
|
|
|
nbr_contexts = len(self.contexts - {bpy.app.translations.contexts.default})
|
|
if nbr_contexts != 1:
|
|
if nbr_contexts == 0:
|
|
nbr_contexts = "No"
|
|
_ctx_txt = "s are"
|
|
else:
|
|
_ctx_txt = " is"
|
|
lines = (("",
|
|
"Average stats for all {} translations:\n".format(self.nbr_trans),
|
|
" {:>6.1%} done!\n".format(self.lvl / self.nbr_trans),
|
|
" {:>6.1%} of messages are tooltips.\n".format(self.lvl_ttips / self.nbr_trans),
|
|
" {:>6.1%} of tooltips are translated.\n".format(self.lvl_trans_ttips / self.nbr_trans),
|
|
" {:>6.1%} of translated messages are tooltips.\n".format(self.lvl_ttips_in_trans / self.nbr_trans),
|
|
" {:>6.1%} of messages are commented.\n".format(self.lvl_comm / self.nbr_trans),
|
|
" The org msgids are currently made of {} signs.\n".format(self.nbr_signs),
|
|
" All processed translations are currently made of {} signs.\n".format(self.nbr_trans_signs),
|
|
" {} specific context{} present:\n".format(self.nbr_contexts, _ctx_txt)) +
|
|
tuple(" " + c + "\n" for c in self.contexts - {bpy.app.translations.contexts.default}) +
|
|
("\n",)
|
|
)
|
|
print(prefix.join(lines))
|
|
|
|
@classmethod
|
|
def check_py_module_has_translations(clss, src, settings=settings):
|
|
"""
|
|
Check whether a given src (a py module, either a directory or a py file) has some i18n translation data,
|
|
and returns a tuple (src_file, translations_tuple) if yes, else (None, None).
|
|
"""
|
|
txts = []
|
|
if os.path.isdir(src):
|
|
for root, dnames, fnames in os.walk(src):
|
|
for fname in fnames:
|
|
if not fname.endswith(".py"):
|
|
continue
|
|
path = os.path.join(root, fname)
|
|
_1, txt, _2, has_trans = clss._parser_check_file(path)
|
|
if has_trans:
|
|
txts.append((path, txt))
|
|
elif src.endswith(".py") and os.path.isfile(src):
|
|
_1, txt, _2, has_trans = clss._parser_check_file(src)
|
|
if has_trans:
|
|
txts.append((src, txt))
|
|
for path, txt in txts:
|
|
tuple_id = "translations_tuple"
|
|
env = globals().copy()
|
|
exec(txt, env)
|
|
if tuple_id in env:
|
|
return path, env[tuple_id]
|
|
return None, None # No data...
|
|
|
|
|
|
def parse(self, kind, src, langs=set()):
|
|
self.parsers[kind](self, src, langs)
|
|
|
|
def parse_from_po(self, src, langs=set()):
|
|
"""
|
|
src must be a tuple (dir_of_pos, pot_file), where:
|
|
* dir_of_pos may either contains iso_CODE.po files, and/or iso_CODE/iso_CODE.po files.
|
|
* pot_file may be None (in which case there will be no ref messages).
|
|
if langs set is void, all languages found are loaded.
|
|
"""
|
|
root_dir, pot_file = src
|
|
if pot_file and os.path.isfile(pot_file):
|
|
self.trans[self.settings.PARSER_TEMPLATE_ID] = I18nMessages(self.settings.PARSER_TEMPLATE_ID, 'PO',
|
|
pot_file, pot_file, settings=self.settings)
|
|
self.src_po[self.settings.PARSER_TEMPLATE_ID] = pot_file
|
|
|
|
for uid, po_file in get_po_files_from_dir(root_dir, langs):
|
|
self.trans[uid] = I18nMessages(uid, 'PO', po_file, po_file, settings=self.settings)
|
|
self.src_po[uid] = po_file
|
|
|
|
def parse_from_py(self, src, langs=set()):
|
|
"""
|
|
src must be a valid path, either a py file or a module directory (in which case all py files inside it
|
|
will be checked, first file macthing will win!).
|
|
if langs set is void, all languages found are loaded.
|
|
"""
|
|
default_context = self.settings.DEFAULT_CONTEXT
|
|
self.src[self.settings.PARSER_PY_ID], msgs = self.check_py_module_has_translations(src, self.settings)
|
|
if msgs is None:
|
|
self.src[self.settings.PARSER_PY_ID] = src
|
|
msgs = ()
|
|
for key, (sources, gen_comments), *translations in msgs:
|
|
if self.settings.PARSER_TEMPLATE_ID not in self.trans:
|
|
self.trans[self.settings.PARSER_TEMPLATE_ID] = I18nMessages(self.settings.PARSER_TEMPLATE_ID,
|
|
settings=self.settings)
|
|
self.src[self.settings.PARSER_TEMPLATE_ID] = self.src[self.settings.PARSER_PY_ID]
|
|
if key in self.trans[self.settings.PARSER_TEMPLATE_ID].msgs:
|
|
print("ERROR! key {} is defined more than once! Skipping re-definitions!")
|
|
continue
|
|
custom_src = [c for c in sources if c.startswith("bpy.")]
|
|
src = [c for c in sources if not c.startswith("bpy.")]
|
|
common_comment_lines = [self.settings.PO_COMMENT_PREFIX_GENERATED + c for c in gen_comments] + \
|
|
[self.settings.PO_COMMENT_PREFIX_SOURCE_CUSTOM + c for c in custom_src] + \
|
|
[self.settings.PO_COMMENT_PREFIX_SOURCE + c for c in src]
|
|
ctxt = [key[0]] if key[0] else [default_context]
|
|
self.trans[self.settings.PARSER_TEMPLATE_ID].msgs[key] = I18nMessage(ctxt, [key[1]], [""],
|
|
common_comment_lines, False, False,
|
|
settings=self.settings)
|
|
for uid, msgstr, (is_fuzzy, user_comments) in translations:
|
|
if uid not in self.trans:
|
|
self.trans[uid] = I18nMessages(uid, settings=self.settings)
|
|
self.src[uid] = self.src[self.settings.PARSER_PY_ID]
|
|
comment_lines = [self.settings.PO_COMMENT_PREFIX + c for c in user_comments] + common_comment_lines
|
|
self.trans[uid].msgs[key] = I18nMessage(ctxt, [key[1]], [msgstr], comment_lines, False, is_fuzzy,
|
|
settings=self.settings)
|
|
#key = self.settings.PO_HEADER_KEY
|
|
#for uid, trans in self.trans.items():
|
|
#if key not in trans.msgs:
|
|
#trans.msgs[key]
|
|
self.unescape()
|
|
|
|
def write(self, kind, langs=set()):
|
|
self.writers[kind](self, langs)
|
|
|
|
def write_to_po(self, langs=set()):
|
|
"""
|
|
Write all translations into po files. By default, write in the same files (or dir) as the source, specify
|
|
a custom self.dst function to write somewhere else!
|
|
Note: If langs is set and you want to export the pot template as well, langs must contain PARSER_TEMPLATE_ID
|
|
({} currently).
|
|
""".format(self.settings.PARSER_TEMPLATE_ID)
|
|
keys = self.trans.keys()
|
|
if langs:
|
|
keys &= langs
|
|
for uid in keys:
|
|
dst = self.dst(self, self.src.get(uid, ""), uid, 'PO')
|
|
self.trans[uid].write('PO', dst)
|
|
|
|
def write_to_py(self, langs=set()):
|
|
"""
|
|
Write all translations as python code, either in a "translations.py" file under same dir as source(s), or in
|
|
specified file if self.py_file is set (default, as usual can be customized with self.dst callable!).
|
|
Note: If langs is set and you want to export the pot template as well, langs must contain PARSER_TEMPLATE_ID
|
|
({} currently).
|
|
""".format(self.settings.PARSER_TEMPLATE_ID)
|
|
default_context = self.settings.DEFAULT_CONTEXT
|
|
def _gen_py(self, langs, tab=" "):
|
|
_lencomm = len(self.settings.PO_COMMENT_PREFIX)
|
|
_lengen = len(self.settings.PO_COMMENT_PREFIX_GENERATED)
|
|
_lensrc = len(self.settings.PO_COMMENT_PREFIX_SOURCE)
|
|
_lencsrc = len(self.settings.PO_COMMENT_PREFIX_SOURCE_CUSTOM)
|
|
ret = [
|
|
"# NOTE: You can safely move around this auto-generated block (with the begin/end markers!),",
|
|
"# and edit the translations by hand.",
|
|
"# Just carefully respect the format of the tuple!",
|
|
"",
|
|
"# Tuple of tuples "
|
|
"((msgctxt, msgid), (sources, gen_comments), (lang, translation, (is_fuzzy, comments)), ...)",
|
|
"translations_tuple = (",
|
|
]
|
|
# First gather all keys (msgctxt, msgid) - theoretically, all translations should share the same, but...
|
|
# Note: using an ordered dict if possible (stupid sets cannot be ordered :/ ).
|
|
keys = I18nMessages._new_messages()
|
|
for trans in self.trans.values():
|
|
keys.update(trans.msgs)
|
|
# Get the ref translation (ideally, PARSER_TEMPLATE_ID one, else the first one that pops up!
|
|
# Ref translation will be used to generate sources "comments"
|
|
ref = self.trans.get(self.settings.PARSER_TEMPLATE_ID) or self.trans[list(self.trans.keys())[0]]
|
|
# Get all languages (uids) and sort them (PARSER_TEMPLATE_ID and PARSER_PY_ID excluded!)
|
|
translations = self.trans.keys() - {self.settings.PARSER_TEMPLATE_ID, self.settings.PARSER_PY_ID}
|
|
if langs:
|
|
translations &= langs
|
|
translations = [('"' + lng + '"', " " * (len(lng) + 6), self.trans[lng]) for lng in sorted(translations)]
|
|
print(k for k in keys.keys())
|
|
for key in keys.keys():
|
|
if ref.msgs[key].is_commented:
|
|
continue
|
|
# Key (context + msgid).
|
|
msgctxt, msgid = ref.msgs[key].msgctxt, ref.msgs[key].msgid
|
|
if not msgctxt:
|
|
msgctxt = default_context
|
|
ret.append(tab + "(({}, \"{}\"),".format('"' + msgctxt + '"' if msgctxt else "None", msgid))
|
|
# Common comments (mostly sources!).
|
|
sources = []
|
|
gen_comments = []
|
|
for comment in ref.msgs[key].comment_lines:
|
|
if comment.startswith(self.settings.PO_COMMENT_PREFIX_SOURCE_CUSTOM):
|
|
sources.append(comment[_lencsrc:])
|
|
elif comment.startswith(self.settings.PO_COMMENT_PREFIX_SOURCE):
|
|
sources.append(comment[_lensrc:])
|
|
elif comment.startswith(self.settings.PO_COMMENT_PREFIX_GENERATED):
|
|
gen_comments.append(comment[_lengen:])
|
|
if not (sources or gen_comments):
|
|
ret.append(tab + " ((), ()),")
|
|
else:
|
|
if len(sources) > 1:
|
|
ret.append(tab + ' (("' + sources[0] + '",')
|
|
ret += [tab + ' "' + s + '",' for s in sources[1:-1]]
|
|
ret.append(tab + ' "' + sources[-1] + '"),')
|
|
else:
|
|
ret.append(tab + " ((" + ('"' + sources[0] + '",' if sources else "") + "),")
|
|
if len(gen_comments) > 1:
|
|
ret.append(tab + ' ("' + gen_comments[0] + '",')
|
|
ret += [tab + ' "' + s + '",' for s in gen_comments[1:-1]]
|
|
ret.append(tab + ' "' + gen_comments[-1] + '")),')
|
|
else:
|
|
ret.append(tab + " (" + ('"' + gen_comments[0] + '",' if gen_comments else "") + ")),")
|
|
# All languages
|
|
for lngstr, lngsp, trans in translations:
|
|
if trans.msgs[key].is_commented:
|
|
continue
|
|
# Language code and translation.
|
|
ret.append(tab + " (" + lngstr + ', "' + trans.msgs[key].msgstr + '",')
|
|
# User comments and fuzzy.
|
|
comments = []
|
|
for comment in trans.msgs[key].comment_lines:
|
|
if comment.startswith(self.settings.PO_COMMENT_PREFIX):
|
|
comments.append(comment[_lencomm:])
|
|
ret.append(tab + lngsp + "(" + ("True" if trans.msgs[key].is_fuzzy else "False") + ",")
|
|
if len(comments) > 1:
|
|
ret.append(tab + lngsp + ' ("' + comments[0] + '",')
|
|
ret += [tab + lngsp + ' "' + s + '",' for s in comments[1:-1]]
|
|
ret.append(tab + lngsp + ' "' + comments[-1] + '"))),')
|
|
else:
|
|
ret[-1] = ret[-1] + " (" + (('"' + comments[0] + '",') if comments else "") + "))),"
|
|
|
|
ret.append(tab + "),")
|
|
ret += [
|
|
")",
|
|
"",
|
|
"translations_dict = {}",
|
|
"for msg in translations_tuple:",
|
|
tab + "key = msg[0]",
|
|
tab + "for lang, trans, (is_fuzzy, comments) in msg[2:]:",
|
|
tab * 2 + "if trans and not is_fuzzy:",
|
|
tab * 3 + "translations_dict.setdefault(lang, {})[key] = trans",
|
|
"",
|
|
]
|
|
return ret
|
|
|
|
self.escape(True)
|
|
dst = self.dst(self, self.src.get(self.settings.PARSER_PY_ID, ""), self.settings.PARSER_PY_ID, 'PY')
|
|
print(dst)
|
|
prev = txt = nxt = ""
|
|
if os.path.exists(dst):
|
|
if not os.path.isfile(dst):
|
|
print("WARNING: trying to write as python code into {}, which is not a file! Aborting.".format(dst))
|
|
return
|
|
prev, txt, nxt, has_trans = self._parser_check_file(dst)
|
|
if prev is None and nxt is None:
|
|
print("WARNING: Looks like given python file {} has no auto-generated translations yet, will be added "
|
|
"at the end of the file, you can move that section later if needed...".format(dst))
|
|
txt = ([txt, "", self.settings.PARSER_PY_MARKER_BEGIN] +
|
|
_gen_py(self, langs) +
|
|
["", self.settings.PARSER_PY_MARKER_END])
|
|
else:
|
|
# We completely replace the text found between start and end markers...
|
|
txt = _gen_py(self, langs)
|
|
else:
|
|
printf("Creating python file {} containing translations.".format(dst))
|
|
txt = [
|
|
"# ***** BEGIN GPL LICENSE BLOCK *****",
|
|
"#",
|
|
"# This program is free software; you can redistribute it and/or",
|
|
"# modify it under the terms of the GNU General Public License",
|
|
"# as published by the Free Software Foundation; either version 2",
|
|
"# of the License, or (at your option) any later version.",
|
|
"#",
|
|
"# This program is distributed in the hope that it will be useful,",
|
|
"# but WITHOUT ANY WARRANTY; without even the implied warranty of",
|
|
"# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the",
|
|
"# GNU General Public License for more details.",
|
|
"#",
|
|
"# You should have received a copy of the GNU General Public License",
|
|
"# along with this program; if not, write to the Free Software Foundation,",
|
|
"# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.",
|
|
"#",
|
|
"# ***** END GPL LICENSE BLOCK *****",
|
|
"",
|
|
self.settings.PARSER_PY_MARKER_BEGIN,
|
|
"",
|
|
]
|
|
txt += _gen_py(self, langs)
|
|
txt += [
|
|
"",
|
|
self.settings.PARSER_PY_MARKER_END,
|
|
]
|
|
with open(dst, 'w') as f:
|
|
f.write((prev or "") + "\n".join(txt) + (nxt or ""))
|
|
self.unescape()
|
|
|
|
parsers = {
|
|
"PO": parse_from_po,
|
|
"PY": parse_from_py,
|
|
}
|
|
|
|
writers = {
|
|
"PO": write_to_po,
|
|
"PY": write_to_py,
|
|
}
|