From c50d55dd347dc26099b25c82257cb9c63116dc2a Mon Sep 17 00:00:00 2001 From: Campbell Barton Date: Tue, 13 Apr 2021 21:12:17 +1000 Subject: [PATCH] Cleanup: use typing for checking utilities --- build_files/cmake/cmake_consistency_check.py | 1 + .../cmake/cmake_static_check_clang_array.py | 25 +++-- .../cmake/cmake_static_check_cppcheck.py | 25 +++-- build_files/cmake/project_source_info.py | 106 +++++++++++++----- 4 files changed, 110 insertions(+), 47 deletions(-) diff --git a/build_files/cmake/cmake_consistency_check.py b/build_files/cmake/cmake_consistency_check.py index f2212a5b002..c62e5fc0b59 100755 --- a/build_files/cmake/cmake_consistency_check.py +++ b/build_files/cmake/cmake_consistency_check.py @@ -49,6 +49,7 @@ from typing import ( Callable, Dict, Generator, + Iterator, List, Optional, Tuple, diff --git a/build_files/cmake/cmake_static_check_clang_array.py b/build_files/cmake/cmake_static_check_clang_array.py index 59743f0244b..76aeb9e194e 100644 --- a/build_files/cmake/cmake_static_check_clang_array.py +++ b/build_files/cmake/cmake_static_check_clang_array.py @@ -25,6 +25,14 @@ import subprocess import sys import os +from typing import ( + Any, + Callable, + List, + Tuple, +) + + USE_QUIET = (os.environ.get("QUIET", None) is not None) CHECKER_IGNORE_PREFIX = [ @@ -43,7 +51,7 @@ CHECKER_ARGS = [ ] -def main(): +def main() -> None: source_info = project_source_info.build_info(ignore_prefix_list=CHECKER_IGNORE_PREFIX) check_commands = [] @@ -52,18 +60,19 @@ def main(): # ~if "source/blender" not in c: # ~ continue - cmd = ([CHECKER_BIN] + - CHECKER_ARGS + - [c] + - [("-I%s" % i) for i in inc_dirs] + - [("-D%s" % d) for d in defs] - ) + cmd = ( + [CHECKER_BIN] + + CHECKER_ARGS + + [c] + + [("-I%s" % i) for i in inc_dirs] + + [("-D%s" % d) for d in defs] + ) check_commands.append((c, cmd)) process_functions = [] - def my_process(i, c, cmd): + def my_process(i: int, c: str, cmd: str) -> subprocess.Popen[Any]: if not USE_QUIET: percent = 100.0 * (i / (len(check_commands) - 1)) percent_str = "[" + ("%.2f]" % percent).rjust(7) + " %:" diff --git a/build_files/cmake/cmake_static_check_cppcheck.py b/build_files/cmake/cmake_static_check_cppcheck.py index c26ebb4cb84..1eef2efe2b5 100644 --- a/build_files/cmake/cmake_static_check_cppcheck.py +++ b/build_files/cmake/cmake_static_check_cppcheck.py @@ -25,6 +25,12 @@ import subprocess import sys import os +from typing import ( + Any, + List, +) + + USE_QUIET = (os.environ.get("QUIET", None) is not None) CHECKER_IGNORE_PREFIX = [ @@ -47,25 +53,26 @@ if USE_QUIET: CHECKER_ARGS.append("--quiet") -def main(): +def main() -> None: source_info = project_source_info.build_info(ignore_prefix_list=CHECKER_IGNORE_PREFIX) source_defines = project_source_info.build_defines_as_args() check_commands = [] for c, inc_dirs, defs in source_info: - cmd = ([CHECKER_BIN] + - CHECKER_ARGS + - [c] + - [("-I%s" % i) for i in inc_dirs] + - [("-D%s" % d) for d in defs] + - source_defines - ) + cmd = ( + [CHECKER_BIN] + + CHECKER_ARGS + + [c] + + [("-I%s" % i) for i in inc_dirs] + + [("-D%s" % d) for d in defs] + + source_defines + ) check_commands.append((c, cmd)) process_functions = [] - def my_process(i, c, cmd): + def my_process(i: int, c: str, cmd: List[str]) -> subprocess.Popen[Any]: if not USE_QUIET: percent = 100.0 * (i / len(check_commands)) percent_str = "[" + ("%.2f]" % percent).rjust(7) + " %:" diff --git a/build_files/cmake/project_source_info.py b/build_files/cmake/project_source_info.py index eb14eea18ef..d2ed80022ca 100644 --- a/build_files/cmake/project_source_info.py +++ b/build_files/cmake/project_source_info.py @@ -34,30 +34,45 @@ if sys.version_info.major < 3: import os from os.path import join, dirname, normpath, abspath +import subprocess + +from typing import ( + Any, + Callable, + Generator, + List, + Optional, + Sequence, + Tuple, + Union, + cast, +) + + SOURCE_DIR = join(dirname(__file__), "..", "..") SOURCE_DIR = normpath(SOURCE_DIR) SOURCE_DIR = abspath(SOURCE_DIR) -def is_c_header(filename): +def is_c_header(filename: str) -> bool: ext = os.path.splitext(filename)[1] return (ext in {".h", ".hpp", ".hxx", ".hh"}) -def is_c(filename): +def is_c(filename: str) -> bool: ext = os.path.splitext(filename)[1] return (ext in {".c", ".cpp", ".cxx", ".m", ".mm", ".rc", ".cc", ".inl", ".osl"}) -def is_c_any(filename): - return os.path.s_c(filename) or is_c_header(filename) +def is_c_any(filename: str) -> bool: + return is_c(filename) or is_c_header(filename) # copied from project_info.py CMAKE_DIR = "." -def cmake_cache_var_iter(): +def cmake_cache_var_iter() -> Generator[Tuple[str, str, str], None, None]: import re re_cache = re.compile(r'([A-Za-z0-9_\-]+)?:?([A-Za-z0-9_\-]+)?=(.*)$') with open(join(CMAKE_DIR, "CMakeCache.txt"), 'r', encoding='utf-8') as cache_file: @@ -68,14 +83,22 @@ def cmake_cache_var_iter(): yield (var, type_ or "", val) -def cmake_cache_var(var): +def cmake_cache_var(var: str) -> Optional[str]: for var_iter, type_iter, value_iter in cmake_cache_var_iter(): if var == var_iter: return value_iter return None -def do_ignore(filepath, ignore_prefix_list): +def cmake_cache_var_or_exit(var: str) -> str: + value = cmake_cache_var(var) + if value is None: + print("Unable to find %r exiting!" % value) + sys.exit(1) + return value + + +def do_ignore(filepath: str, ignore_prefix_list: Optional[Sequence[str]]) -> bool: if ignore_prefix_list is None: return False @@ -83,12 +106,13 @@ def do_ignore(filepath, ignore_prefix_list): return any([relpath.startswith(prefix) for prefix in ignore_prefix_list]) -def makefile_log(): +def makefile_log() -> List[str]: import subprocess import time # support both make and ninja - make_exe = cmake_cache_var("CMAKE_MAKE_PROGRAM") + make_exe = cmake_cache_var_or_exit("CMAKE_MAKE_PROGRAM") + make_exe_basename = os.path.basename(make_exe) if make_exe_basename.startswith(("make", "gmake")): @@ -102,26 +126,37 @@ def makefile_log(): stdout=subprocess.PIPE, ) + if process is None: + print("Can't execute process") + sys.exit(1) + + while process.poll(): time.sleep(1) - out = process.stdout.read() - process.stdout.close() + # We know this is always true based on the input arguments to `Popen`. + stdout: IO[bytes] = process.stdout # type: ignore + + out = stdout.read() + stdout.close() print("done!", len(out), "bytes") - return out.decode("utf-8", errors="ignore").split("\n") + return cast(List[str], out.decode("utf-8", errors="ignore").split("\n")) -def build_info(use_c=True, use_cxx=True, ignore_prefix_list=None): - +def build_info( + use_c: bool = True, + use_cxx: bool = True, + ignore_prefix_list: Optional[List[str]] = None, +) -> List[Tuple[str, List[str], List[str]]]: makelog = makefile_log() source = [] compilers = [] if use_c: - compilers.append(cmake_cache_var("CMAKE_C_COMPILER")) + compilers.append(cmake_cache_var_or_exit("CMAKE_C_COMPILER")) if use_cxx: - compilers.append(cmake_cache_var("CMAKE_CXX_COMPILER")) + compilers.append(cmake_cache_var_or_exit("CMAKE_CXX_COMPILER")) print("compilers:", " ".join(compilers)) @@ -131,7 +166,7 @@ def build_info(use_c=True, use_cxx=True, ignore_prefix_list=None): for line in makelog: - args = line.split() + args: Union[str, List[str]] = line.split() if not any([(c in args) for c in compilers]): continue @@ -176,29 +211,40 @@ def build_info(use_c=True, use_cxx=True, ignore_prefix_list=None): return source -def build_defines_as_source(): +def build_defines_as_source() -> str: """ Returns a string formatted as an include: '#defines A=B\n#define....' """ import subprocess # works for both gcc and clang - cmd = (cmake_cache_var("CMAKE_C_COMPILER"), "-dM", "-E", "-") - return subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stdin=subprocess.DEVNULL, - ).stdout.read().strip().decode('ascii') + cmd = (cmake_cache_var_or_exit("CMAKE_C_COMPILER"), "-dM", "-E", "-") + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stdin=subprocess.DEVNULL, + ) + + # We know this is always true based on the input arguments to `Popen`. + stdout: IO[bytes] = process.stdout # type: ignore + + return cast(str, stdout.read().strip().decode('ascii')) -def build_defines_as_args(): - return [("-D" + "=".join(l.split(maxsplit=2)[1:])) - for l in build_defines_as_source().split("\n") - if l.startswith('#define')] +def build_defines_as_args() -> List[str]: + return [ + ("-D" + "=".join(l.split(maxsplit=2)[1:])) + for l in build_defines_as_source().split("\n") + if l.startswith('#define') + ] # could be moved elsewhere!, this just happens to be used by scripts that also # use this module. -def queue_processes(process_funcs, job_total=-1): +def queue_processes( + process_funcs: Sequence[Tuple[Callable[..., subprocess.Popen[Any]], Tuple[Any, ...]]], + job_total: int =-1, +) -> None: """ Takes a list of function arg pairs, each function must return a process """ @@ -217,7 +263,7 @@ def queue_processes(process_funcs, job_total=-1): else: import time - processes = [] + processes: List[subprocess.Popen[Any]] = [] for func, args in process_funcs: # wait until a thread is free while 1: @@ -234,7 +280,7 @@ def queue_processes(process_funcs, job_total=-1): processes.append(func(*args)) -def main(): +def main() -> None: if not os.path.exists(join(CMAKE_DIR, "CMakeCache.txt")): print("This script must run from the cmake build dir") return