blender/release/scripts/modules/bl_i18n_utils/update_pot.py

#!/usr/bin/python3

# ***** BEGIN GPL LICENSE BLOCK *****
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ***** END GPL LICENSE BLOCK *****

# <pep8 compliant>

# Update blender.pot file from messages.txt

import subprocess
import os
import sys
import re
import tempfile
import argparse
import time
import pickle

try:
    import settings
    import utils
except:
    from . import (settings, utils)


LANGUAGES_CATEGORIES = settings.LANGUAGES_CATEGORIES
LANGUAGES = settings.LANGUAGES

COMMENT_PREFIX = settings.COMMENT_PREFIX
COMMENT_PREFIX_SOURCE = settings.COMMENT_PREFIX_SOURCE
CONTEXT_PREFIX = settings.CONTEXT_PREFIX
FILE_NAME_MESSAGES = settings.FILE_NAME_MESSAGES
FILE_NAME_POT = settings.FILE_NAME_POT
SOURCE_DIR = settings.SOURCE_DIR
POTFILES_DIR = settings.POTFILES_SOURCE_DIR
SRC_POTFILES = settings.FILE_NAME_SRC_POTFILES

CONTEXT_DEFAULT = settings.CONTEXT_DEFAULT
PYGETTEXT_ALLOWED_EXTS = settings.PYGETTEXT_ALLOWED_EXTS
PYGETTEXT_MAX_MULTI_CTXT = settings.PYGETTEXT_MAX_MULTI_CTXT

SVN_EXECUTABLE = settings.SVN_EXECUTABLE

WARN_NC = settings.WARN_MSGID_NOT_CAPITALIZED
NC_ALLOWED = settings.WARN_MSGID_NOT_CAPITALIZED_ALLOWED

SPELL_CACHE = settings.SPELL_CACHE


# Do this only once!
# Get contexts defined in blf.
CONTEXTS = {}
with open(os.path.join(SOURCE_DIR, settings.PYGETTEXT_CONTEXTS_DEFSRC)) as f:
    reg = re.compile(settings.PYGETTEXT_CONTEXTS)
    f = f.read()
    # This regex is supposed to yield tuples
    # (key=C_macro_name, value=C_string).
    CONTEXTS = dict(m.groups() for m in reg.finditer(f))

# Build regexes to extract messages (with optional contexts) from C source.
pygettexts = tuple(re.compile(r).search
                   for r in settings.PYGETTEXT_KEYWORDS)
_clean_str = re.compile(settings.str_clean_re).finditer
clean_str = lambda s: "".join(m.group("clean") for m in _clean_str(s))


def check_file(path, rel_path, messages):
    def process_entry(ctxt, msg):
        # Context.
        if ctxt:
            if ctxt in CONTEXTS:
                ctxt = CONTEXTS[ctxt]
            elif '"' in ctxt or "'" in ctxt:
                ctxt = clean_str(ctxt)
            else:
                print("WARNING: raw context “{}” couldn’t be resolved!"
                      "".format(ctxt))
                ctxt = CONTEXT_DEFAULT
        else:
            ctxt = CONTEXT_DEFAULT
        # Message.
        if msg:
            if '"' in msg or "'" in msg:
                msg = clean_str(msg)
            else:
                print("WARNING: raw message “{}” couldn’t be resolved!"
                      "".format(msg))
                msg = ""
        else:
            msg = ""
        return (ctxt, msg)

    with open(path, encoding="utf-8") as f:
        f = f.read()
        for srch in pygettexts:
            m = srch(f)
            line = pos = 0
            while m:
                d = m.groupdict()
                # Line.
                line += f[pos:m.start()].count('\n')
                msg = d.get("msg_raw")
                # First, try the "multi-contexts" stuff!
                ctxts = tuple(d.get("ctxt_raw{}".format(i)) for i in range(PYGETTEXT_MAX_MULTI_CTXT))
                if ctxts[0]:
                    for ctxt in ctxts:
                        if not ctxt:
                            break
                        ctxt, _msg = process_entry(ctxt, msg)
                        # And we are done for this item!
                        messages.setdefault((ctxt, _msg), []).append(":".join((rel_path, str(line))))
                else:
                    ctxt = d.get("ctxt_raw")
                    ctxt, msg = process_entry(ctxt, msg)
                    # And we are done for this item!
                    messages.setdefault((ctxt, msg), []).append(":".join((rel_path, str(line))))
                pos = m.end()
                line += f[m.start():pos].count('\n')
                m = srch(f, pos)


def py_xgettext(messages):
    forbidden = set()
    forced = set()
    with open(SRC_POTFILES) as src:
        for l in src:
            if l[0] == '-':
                forbidden.add(l[1:].rstrip('\n'))
            elif l[0] != '#':
                forced.add(l.rstrip('\n'))
    for root, dirs, files in os.walk(POTFILES_DIR):
        if "/.svn" in root:
            continue
        for fname in files:
            if os.path.splitext(fname)[1] not in PYGETTEXT_ALLOWED_EXTS:
                continue
            path = os.path.join(root, fname)
            rel_path = os.path.relpath(path, SOURCE_DIR)
            if rel_path in forbidden:
                continue
            elif rel_path not in forced:
                forced.add(rel_path)
    for rel_path in sorted(forced):
        path = os.path.join(SOURCE_DIR, rel_path)
        if os.path.exists(path):
            check_file(path, rel_path, messages)


# Spell checking!
import enchant
dict_spelling = enchant.Dict("en_US")

from spell_check_utils import (dict_uimsgs,
                               split_words,
                              )

_spell_checked = set()


def spell_check(txt, cache):
    ret = []

    if cache is not None and txt in cache:
        return ret

    for w in split_words(txt):
        w_lower = w.lower()
        if w_lower in dict_uimsgs | _spell_checked:
            continue
        if not dict_spelling.check(w):
            ret.append("{}: suggestions are ({})"
                       .format(w, "'" + "', '".join(dict_spelling.suggest(w))
                                  + "'"))
        else:
            _spell_checked.add(w_lower)

    if not ret:
        if cache is not None:
            cache.add(txt)

    return ret


def get_svnrev():
    cmd = [SVN_EXECUTABLE,
           "info",
           "--xml",
           SOURCE_DIR,
           ]
    xml = subprocess.check_output(cmd)
    return re.search(b'revision="(\d+)"', xml).group(1)


def gen_empty_pot():
    blender_rev = get_svnrev().decode()
    utctime = time.gmtime()
    time_str = time.strftime("%Y-%m-%d %H:%M+0000", utctime)
    year_str = time.strftime("%Y", utctime)

    return utils.gen_empty_messages(blender_rev, time_str, year_str)


escape_re = tuple(re.compile(r[0]) for r in settings.ESCAPE_RE)
escape = lambda s, n: escape_re[n].sub(settings.ESCAPE_RE[n][1], s)


def merge_messages(msgs, states, messages, do_checks, spell_cache):
    num_added = num_present = 0
    for (context, msgid), srcs in messages.items():
        if do_checks:
            err = spell_check(msgid, spell_cache)
            if err:
                print("WARNING: spell check failed on “" + msgid + "”:")
                print("\t\t" + "\n\t\t".join(err))
                print("\tFrom:\n\t\t" + "\n\t\t".join(srcs))

        # Escape some chars in msgid!
        for n in range(len(escape_re)):
            msgid = escape(msgid, n)

        srcs = [COMMENT_PREFIX_SOURCE + s for s in srcs]

        key = (context, msgid)
        if key not in msgs:
            msgs[key] = {"msgid_lines": [msgid],
                         "msgstr_lines": [""],
                         "comment_lines": srcs,
                         "msgctxt_lines": [context]}
            num_added += 1
        else:
            # We need to merge comments!
            msgs[key]["comment_lines"].extend(srcs)
            num_present += 1

    return num_added, num_present


def main():
    parser = argparse.ArgumentParser(description="Update blender.pot file from messages.txt and source code parsing, "
                                                 "and performs some checks over msgids.")
    parser.add_argument('-w', '--warning', action="store_true",
                        help="Show warnings.")
    parser.add_argument('-i', '--input', metavar="File",
                        help="Input messages file path.")
    parser.add_argument('-o', '--output', metavar="File",
                        help="Output pot file path.")

    args = parser.parse_args()
    if args.input:
        global FILE_NAME_MESSAGES
        FILE_NAME_MESSAGES = args.input
    if args.output:
        global FILE_NAME_POT
        FILE_NAME_POT = args.output

    print("Running fake py gettext…")
    # Not using any more xgettext, simpler to do it ourself!
    messages = utils.new_messages()
    py_xgettext(messages)
    print("Finished, found {} messages.".format(len(messages)))

    if SPELL_CACHE and os.path.exists(SPELL_CACHE):
        with open(SPELL_CACHE, 'rb') as f:
            spell_cache = pickle.load(f)
    else:
        spell_cache = set()

    print("Generating POT file {}…".format(FILE_NAME_POT))
    msgs, states = gen_empty_pot()
    tot_messages, _a = merge_messages(msgs, states, messages,
                                      True, spell_cache)

    # add messages collected automatically from RNA
    print("\tMerging RNA messages from {}…".format(FILE_NAME_MESSAGES))
    messages = utils.new_messages()
    with open(FILE_NAME_MESSAGES, encoding="utf-8") as f:
        srcs = []
        context = ""
        for line in f:
            line = utils.stripeol(line)

            if line.startswith(COMMENT_PREFIX):
                srcs.append(line[len(COMMENT_PREFIX):].strip())
            elif line.startswith(CONTEXT_PREFIX):
                context = line[len(CONTEXT_PREFIX):].strip()
            else:
                key = (context, line)
                messages[key] = srcs
                srcs = []
                context = ""
    num_added, num_present = merge_messages(msgs, states, messages,
                                            True, spell_cache)
    tot_messages += num_added
    print("\tMerged {} messages ({} were already present)."
          "".format(num_added, num_present))

    print("\tAdding languages labels...")
    messages = {(CONTEXT_DEFAULT, lng[1]):
                ("Languages’ labels from bl_i18n_utils/settings.py",)
                for lng in LANGUAGES}
    messages.update({(CONTEXT_DEFAULT, cat[1]):
                     ("Language categories’ labels from bl_i18n_utils/settings.py",)
                     for cat in LANGUAGES_CATEGORIES})
    num_added, num_present = merge_messages(msgs, states, messages,
                                            True, spell_cache)
    tot_messages += num_added
    print("\tAdded {} language messages.".format(num_added))

    # Write back all messages into blender.pot.
    utils.write_messages(FILE_NAME_POT, msgs, states["comm_msg"],
                         states["fuzzy_msg"])

    if SPELL_CACHE and spell_cache:
        with open(SPELL_CACHE, 'wb') as f:
            pickle.dump(spell_cache, f)

    print("Finished, total: {} messages!".format(tot_messages - 1))

    return 0


if __name__ == "__main__":
    print("\n\n *** Running {} *** \n".format(__file__))
    sys.exit(main())