From 8c5c5142d5e8fe88126aea674daa43d6a920863b Mon Sep 17 00:00:00 2001 From: Campbell Barton Date: Tue, 15 Aug 2023 15:18:01 +1000 Subject: [PATCH] Cleanup: use type checking for credits_git_gen.py & git_log.py --- tools/check_source/check_mypy_config.py | 2 - tools/utils/credits_git_gen.py | 63 +++++++++++++--------- tools/utils/git_log.py | 69 +++++++++++++++---------- 3 files changed, 80 insertions(+), 54 deletions(-) diff --git a/tools/check_source/check_mypy_config.py b/tools/check_source/check_mypy_config.py index e932f53cf20..ab7e2db08be 100644 --- a/tools/check_source/check_mypy_config.py +++ b/tools/check_source/check_mypy_config.py @@ -53,11 +53,9 @@ PATHS_EXCLUDE = set( "tools/utils/blender_keyconfig_export_permutations.py", "tools/utils/blender_merge_format_changes.py", "tools/utils/blender_theme_as_c.py", - "tools/utils/credits_git_gen.py", "tools/utils/cycles_commits_sync.py", "tools/utils/cycles_timeit.py", "tools/utils/gdb_struct_repr_c99.py", - "tools/utils/git_log.py", "tools/utils/git_log_review_commits.py", "tools/utils/git_log_review_commits_advanced.py", "tools/utils/gitea_inactive_developers.py", diff --git a/tools/utils/credits_git_gen.py b/tools/utils/credits_git_gen.py index 364fed90981..87790a565da 100755 --- a/tools/utils/credits_git_gen.py +++ b/tools/utils/credits_git_gen.py @@ -9,10 +9,22 @@ Example use: credits_git_gen.py --source=/src/blender --range=SHA1..HEAD """ -from git_log import GitCommitIter - -import re +import argparse import multiprocessing +import re +import unicodedata + +from git_log import ( + GitCommitIter, + GitCommit, +) + +from typing import ( + Dict, + Tuple, + Iterable, + List, +) # ----------------------------------------------------------------------------- @@ -100,7 +112,7 @@ author_table = { # Fully overwrite authors gathered from git commit info. # Intended usage: Correction of info stored in git commit itself. # Note that the names of the authors here are assumed fully valid and usable as-is. -commit_authors_overwrite = { +commit_authors_overwrite: Dict[bytes, Tuple[str, str]] = { # Format: {full_git_hash: (tuple, of, authors),}. # Example: # b"a60c1e5bb814078411ce105b7cf347afac6f2afd": ("Blender Foundation", "Suzanne", "Ton"), @@ -110,7 +122,7 @@ commit_authors_overwrite = { # ----------------------------------------------------------------------------- # Multi-Processing -def process_commits_for_map(commits): +def process_commits_for_map(commits: Iterable[GitCommit]) -> "Credits": result = Credits() for c in commits: result.process_commit(c) @@ -128,8 +140,10 @@ class CreditUser: "year_max", ) - def __init__(self): + def __init__(self) -> None: self.commit_total = 0 + self.year_min = 0 + self.year_max = 0 class Credits: @@ -143,15 +157,14 @@ class Credits: # `Co-authored-by: Blender Foundation ` GIT_COMMIT_COAUTHORS_RE = re.compile(r"^Co-authored-by:[ \t]*(?P[ \w\t]*\w)(?:$|[ \t]*<)", re.MULTILINE) - def __init__(self): - self.users = {} + def __init__(self) -> None: + self.users: Dict[str, CreditUser] = {} @classmethod - def commit_authors_get(cls, c): - authors = commit_authors_overwrite.get(c.sha1, None) - if authors is not None: + def commit_authors_get(cls, c: GitCommit) -> List[str]: + if (authors_overwrite := commit_authors_overwrite.get(c.sha1, None)) is not None: # Ignore git commit info for these having an entry in commit_authors_overwrite. - return [author_table.get(author, author) for author in authors] + return [author_table.get(author, author) for author in authors_overwrite] authors = [c.author] + cls.GIT_COMMIT_COAUTHORS_RE.findall(c.body) # Normalize author string into canonical form, prevents duplicate credit users @@ -159,7 +172,7 @@ class Credits: return [author_table.get(author, author) for author in authors] @classmethod - def is_credit_commit_valid(cls, c): + def is_credit_commit_valid(cls, c: GitCommit) -> bool: ignore_dir = ( b"blender/extern/", b"blender/intern/opennl/", @@ -170,7 +183,7 @@ class Credits: return True - def merge(self, other): + def merge(self, other: "Credits") -> None: """ Merge other Credits into this, clearing the other. """ @@ -185,7 +198,7 @@ class Credits: user.year_max = max(user.year_max, user_other.year_max) other.users.clear() - def process_commit(self, c): + def process_commit(self, c: GitCommit) -> None: if not self.is_credit_commit_valid(c): return @@ -202,7 +215,7 @@ class Credits: cu.year_min = min(cu.year_min, year) cu.year_max = max(cu.year_max, year) - def _process_multiprocessing(self, commit_iter, *, jobs): + def _process_multiprocessing(self, commit_iter: Iterable[GitCommit], *, jobs: int) -> None: print("Collecting commits...") # NOTE(@ideasman42): that the chunk size doesn't have as much impact on # performance as you might expect, values between 16 and 1024 seem reasonable. @@ -226,7 +239,7 @@ class Credits: print("{:d} of {:d}".format(i, len(chunk_list))) self.merge(result) - def process(self, commit_iter, *, jobs): + def process(self, commit_iter: Iterable[GitCommit], *, jobs: int) -> None: if jobs > 1: self._process_multiprocessing(commit_iter, jobs=jobs) return @@ -237,10 +250,13 @@ class Credits: if not (i % 100): print(i) - def write(self, filepath, - is_main_credits=True, - contrib_companies=(), - sort="name"): + def write( + self, + filepath: str, + is_main_credits: bool = True, + contrib_companies: Tuple[str, ...] = (), + sort: str = "name", + ) -> None: # patch_word = "patch", "patches" commit_word = "commit", "commits" @@ -280,8 +296,7 @@ class Credits: )) -def argparse_create(): - import argparse +def argparse_create() -> argparse.ArgumentParser: # When --help or no args are given, print this help usage_text = "Review revisions." @@ -325,7 +340,7 @@ def argparse_create(): return parser -def main(): +def main() -> None: # ---------- # Parse Args diff --git a/tools/utils/git_log.py b/tools/utils/git_log.py index e615d299c0a..5c4e6ddfb6c 100644 --- a/tools/utils/git_log.py +++ b/tools/utils/git_log.py @@ -6,6 +6,14 @@ import os import subprocess +import datetime + +from typing import ( + List, + Union, + Optional, + Tuple, +) class GitCommit: @@ -24,20 +32,19 @@ class GitCommit: "_diff", ) - def __init__(self, sha1, git_dir): + def __init__(self, sha1: bytes, git_dir: str): self.sha1 = sha1 self._git_dir = git_dir - self._author = \ - self._email = \ - self._date = \ - self._body = \ - self._files = \ - self._files_status = \ - self._diff = \ - None + self._author: Optional[str] = None + self._email: Optional[str] = None + self._date: Optional[datetime.datetime] = None + self._body: Optional[str] = None + self._files: Optional[List[bytes]] = None + self._files_status: Optional[List[List[bytes]]] = None + self._diff: Optional[str] = None - def cache(self): + def cache(self) -> None: """ Cache all properties (except for diff as it's significantly larger than other members). @@ -49,9 +56,9 @@ class GitCommit: self.files self.files_status - def _log_format(self, format, args=()): + def _log_format(self, format: str, args: Tuple[Union[str, bytes], ...] = ()) -> bytes: # sha1 = self.sha1.decode('ascii') - cmd = ( + cmd: Tuple[Union[str, bytes], ...] = ( "git", "--git-dir", self._git_dir, @@ -59,17 +66,20 @@ class GitCommit: "-1", # only this rev self.sha1, "--format=" + format, - ) + args + *args, + ) + # print(" ".join(cmd)) with subprocess.Popen( cmd, stdout=subprocess.PIPE, ) as p: + assert p is not None and p.stdout is not None return p.stdout.read() @property - def sha1_short(self): + def sha1_short(self) -> str: cmd = ( "git", "--git-dir", @@ -82,10 +92,11 @@ class GitCommit: cmd, stdout=subprocess.PIPE, ) as p: + assert p is not None and p.stdout is not None return p.stdout.read().strip().decode('ascii') @property - def author(self): + def author(self) -> str: ret = self._author if ret is None: content = self._log_format("%an")[:-1] @@ -94,7 +105,7 @@ class GitCommit: return ret @property - def email(self): + def email(self) -> str: ret = self._email if ret is None: content = self._log_format("%ae")[:-1] @@ -103,7 +114,7 @@ class GitCommit: return ret @property - def date(self): + def date(self) -> datetime.datetime: ret = self._date if ret is None: import datetime @@ -112,7 +123,7 @@ class GitCommit: return ret @property - def body(self): + def body(self) -> str: ret = self._body if ret is None: content = self._log_format("%B")[:-1] @@ -121,11 +132,11 @@ class GitCommit: return ret @property - def subject(self): + def subject(self) -> str: return self.body.lstrip().partition("\n")[0] @property - def files(self): + def files(self) -> List[bytes]: ret = self._files if ret is None: ret = [f for f in self._log_format("format:", args=("--name-only",)).split(b"\n") if f] @@ -133,7 +144,7 @@ class GitCommit: return ret @property - def files_status(self): + def files_status(self) -> List[List[bytes]]: ret = self._files_status if ret is None: ret = [f.split(None, 1) for f in self._log_format("format:", args=("--name-status",)).split(b"\n") if f] @@ -141,7 +152,7 @@ class GitCommit: return ret @property - def diff(self): + def diff(self) -> str: ret = self._diff if ret is None: content = self._log_format("", args=("-p",)) @@ -158,13 +169,13 @@ class GitCommitIter: "_process", ) - def __init__(self, path, sha1_range): + def __init__(self, path: str, sha1_range: str): self._path = path self._git_dir = os.path.join(path, ".git") self._sha1_range = sha1_range - self._process = None + self._process: Optional[subprocess.Popen[bytes]] = None - def __iter__(self): + def __iter__(self) -> "GitCommitIter": cmd = ( "git", "--git-dir", @@ -181,7 +192,8 @@ class GitCommitIter: ) return self - def __next__(self): + def __next__(self) -> GitCommit: + assert self._process is not None and self._process.stdout is not None sha1 = self._process.stdout.readline()[:-1] if sha1: return GitCommit(sha1, self._git_dir) @@ -195,12 +207,12 @@ class GitRepo: "_git_dir", ) - def __init__(self, path): + def __init__(self, path: str): self._path = path self._git_dir = os.path.join(path, ".git") @property - def branch(self): + def branch(self) -> bytes: cmd = ( "git", "--git-dir", @@ -215,4 +227,5 @@ class GitRepo: cmd, stdout=subprocess.PIPE, ) + assert p is not None and p.stdout is not None return p.stdout.read()