From c6df7266c718555cde5a729dae1c6023ef2b3530 Mon Sep 17 00:00:00 2001 From: Campbell Barton Date: Wed, 23 Feb 2022 12:30:59 +1100 Subject: [PATCH] CMake: move cmake_consistency_check.py into source/tools/ repo This isn't needed for building Blender so move this along side other source checking scripts. --- GNUmakefile | 2 +- build_files/cmake/cmake_consistency_check.py | 387 ------------------ .../cmake/cmake_consistency_check_config.py | 64 --- 3 files changed, 1 insertion(+), 452 deletions(-) delete mode 100755 build_files/cmake/cmake_consistency_check.py delete mode 100644 build_files/cmake/cmake_consistency_check_config.py diff --git a/GNUmakefile b/GNUmakefile index a410662647e..575f3e904df 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -489,7 +489,7 @@ check_pep8: .FORCE check_cmake: .FORCE @PYTHONIOENCODING=utf_8 $(PYTHON) \ - build_files/cmake/cmake_consistency_check.py + source/tools/check_source/check_cmake_consistency.py # ----------------------------------------------------------------------------- diff --git a/build_files/cmake/cmake_consistency_check.py b/build_files/cmake/cmake_consistency_check.py deleted file mode 100755 index ec31b4cdfe3..00000000000 --- a/build_files/cmake/cmake_consistency_check.py +++ /dev/null @@ -1,387 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: GPL-2.0-or-later - -# - -# Note: this code should be cleaned up / refactored. - -import sys -if sys.version_info.major < 3: - print("\nPython3.x needed, found %s.\nAborting!\n" % - sys.version.partition(" ")[0]) - sys.exit(1) - -import os -from os.path import ( - dirname, - join, - normpath, - splitext, -) - -from cmake_consistency_check_config import ( - IGNORE_SOURCE, - IGNORE_SOURCE_MISSING, - IGNORE_CMAKE, - UTF8_CHECK, - SOURCE_DIR, - BUILD_DIR, -) - -from typing import ( - Callable, - Dict, - Generator, - Iterator, - List, - Optional, - Tuple, -) - - -global_h = set() -global_c = set() -global_refs: Dict[str, List[Tuple[str, int]]] = {} - -# Flatten `IGNORE_SOURCE_MISSING` to avoid nested looping. -IGNORE_SOURCE_MISSING_FLAT = [ - (k, ignore_path) for k, ig_list in IGNORE_SOURCE_MISSING - for ignore_path in ig_list -] - -# Ignore cmake file, path pairs. -global_ignore_source_missing: Dict[str, List[str]] = {} -for k, v in IGNORE_SOURCE_MISSING_FLAT: - global_ignore_source_missing.setdefault(k, []).append(v) -del IGNORE_SOURCE_MISSING_FLAT - - -def replace_line(f: str, i: int, text: str, keep_indent: bool = True) -> None: - file_handle = open(f, 'r') - data = file_handle.readlines() - file_handle.close() - - l = data[i] - ws = l[:len(l) - len(l.lstrip())] - - data[i] = "%s%s\n" % (ws, text) - - file_handle = open(f, 'w') - file_handle.writelines(data) - file_handle.close() - - -def source_list( - path: str, - filename_check: Optional[Callable[[str], bool]] = None, -) -> Generator[str, None, None]: - for dirpath, dirnames, filenames in os.walk(path): - # skip '.git' - dirnames[:] = [d for d in dirnames if not d.startswith(".")] - - for filename in filenames: - if filename_check is None or filename_check(filename): - yield os.path.join(dirpath, filename) - - -# extension checking -def is_cmake(filename: str) -> bool: - ext = splitext(filename)[1] - return (ext == ".cmake") or (filename == "CMakeLists.txt") - - -def is_c_header(filename: str) -> bool: - ext = splitext(filename)[1] - return (ext in {".h", ".hpp", ".hxx", ".hh"}) - - -def is_c(filename: str) -> bool: - ext = splitext(filename)[1] - return (ext in {".c", ".cpp", ".cxx", ".m", ".mm", ".rc", ".cc", ".inl", ".metal"}) - - -def is_c_any(filename: str) -> bool: - return is_c(filename) or is_c_header(filename) - - -def cmake_get_src(f: str) -> None: - - sources_h = [] - sources_c = [] - - filen = open(f, "r", encoding="utf8") - it: Optional[Iterator[str]] = iter(filen) - found = False - i = 0 - # print(f) - - def is_definition(l: str, f: str, i: int, name: str) -> bool: - if l.startswith("unset("): - return False - - if ('set(%s' % name) in l or ('set(' in l and l.endswith(name)): - if len(l.split()) > 1: - raise Exception("strict formatting not kept 'set(%s*' %s:%d" % (name, f, i)) - return True - - if ("list(APPEND %s" % name) in l or ('list(APPEND ' in l and l.endswith(name)): - if l.endswith(")"): - raise Exception("strict formatting not kept 'list(APPEND %s...)' on 1 line %s:%d" % (name, f, i)) - return True - return False - - while it is not None: - context_name = "" - while it is not None: - i += 1 - try: - l = next(it) - except StopIteration: - it = None - break - l = l.strip() - if not l.startswith("#"): - found = is_definition(l, f, i, "SRC") - if found: - context_name = "SRC" - break - found = is_definition(l, f, i, "INC") - if found: - context_name = "INC" - break - - if found: - cmake_base = dirname(f) - cmake_base_bin = os.path.join(BUILD_DIR, os.path.relpath(cmake_base, SOURCE_DIR)) - - # Find known missing sources list (if we have one). - f_rel = os.path.relpath(f, SOURCE_DIR) - f_rel_key = f_rel - if os.sep != "/": - f_rel_key = f_rel_key.replace(os.sep, "/") - local_ignore_source_missing = global_ignore_source_missing.get(f_rel_key, []) - - while it is not None: - i += 1 - try: - l = next(it) - except StopIteration: - it = None - break - - l = l.strip() - - if not l.startswith("#"): - - # Remove in-line comments. - l = l.split(" # ")[0].rstrip() - - if ")" in l: - if l.strip() != ")": - raise Exception("strict formatting not kept '*)' %s:%d" % (f, i)) - break - - # replace dirs - l = l.replace("${CMAKE_SOURCE_DIR}", SOURCE_DIR) - l = l.replace("${CMAKE_CURRENT_SOURCE_DIR}", cmake_base) - l = l.replace("${CMAKE_CURRENT_BINARY_DIR}", cmake_base_bin) - l = l.strip('"') - - if not l: - pass - elif l in local_ignore_source_missing: - local_ignore_source_missing.remove(l) - elif l.startswith("$"): - if context_name == "SRC": - # assume if it ends with context_name we know about it - if not l.split("}")[0].endswith(context_name): - print("Can't use var '%s' %s:%d" % (l, f, i)) - elif len(l.split()) > 1: - raise Exception("Multi-line define '%s' %s:%d" % (l, f, i)) - else: - new_file = normpath(join(cmake_base, l)) - - if context_name == "SRC": - if is_c_header(new_file): - sources_h.append(new_file) - global_refs.setdefault(new_file, []).append((f, i)) - elif is_c(new_file): - sources_c.append(new_file) - global_refs.setdefault(new_file, []).append((f, i)) - elif l in {"PARENT_SCOPE", }: - # cmake var, ignore - pass - elif new_file.endswith(".list"): - pass - elif new_file.endswith(".def"): - pass - elif new_file.endswith(".cl"): # opencl - pass - elif new_file.endswith(".cu"): # cuda - pass - elif new_file.endswith(".osl"): # open shading language - pass - elif new_file.endswith(".glsl"): - pass - else: - raise Exception("unknown file type - not c or h %s -> %s" % (f, new_file)) - - elif context_name == "INC": - if new_file.startswith(BUILD_DIR): - # assume generated path - pass - elif os.path.isdir(new_file): - new_path_rel = os.path.relpath(new_file, cmake_base) - - if new_path_rel != l: - print("overly relative path:\n %s:%d\n %s\n %s" % (f, i, l, new_path_rel)) - - # # Save time. just replace the line - # replace_line(f, i - 1, new_path_rel) - - else: - raise Exception("non existent include %s:%d -> %s" % (f, i, new_file)) - - # print(new_file) - - global_h.update(set(sources_h)) - global_c.update(set(sources_c)) - ''' - if not sources_h and not sources_c: - raise Exception("No sources %s" % f) - - sources_h_fs = list(source_list(cmake_base, is_c_header)) - sources_c_fs = list(source_list(cmake_base, is_c)) - ''' - # find missing C files: - ''' - for ff in sources_c_fs: - if ff not in sources_c: - print(" missing: " + ff) - ''' - - # reset - del sources_h[:] - del sources_c[:] - - filen.close() - - -def is_ignore_source(f: str, ignore_used: List[bool]) -> bool: - for index, ignore_path in enumerate(IGNORE_SOURCE): - if ignore_path in f: - ignore_used[index] = True - return True - return False - - -def is_ignore_cmake(f: str, ignore_used: List[bool]) -> bool: - for index, ignore_path in enumerate(IGNORE_CMAKE): - if ignore_path in f: - ignore_used[index] = True - return True - return False - - -def main() -> None: - - print("Scanning:", SOURCE_DIR) - - ignore_used_source = [False] * len(IGNORE_SOURCE) - ignore_used_cmake = [False] * len(IGNORE_CMAKE) - - for cmake in source_list(SOURCE_DIR, is_cmake): - if not is_ignore_cmake(cmake, ignore_used_cmake): - cmake_get_src(cmake) - - # First do stupid check, do these files exist? - print("\nChecking for missing references:") - is_err = False - errs = [] - for f in (global_h | global_c): - if f.startswith(BUILD_DIR): - continue - - if not os.path.exists(f): - refs = global_refs[f] - if refs: - for cf, i in refs: - errs.append((cf, i)) - else: - raise Exception("CMake references missing, internal error, aborting!") - is_err = True - - errs.sort() - errs.reverse() - for cf, i in errs: - print("%s:%d" % (cf, i)) - # Write a 'sed' script, useful if we get a lot of these - # print("sed '%dd' '%s' > '%s.tmp' ; mv '%s.tmp' '%s'" % (i, cf, cf, cf, cf)) - - if is_err: - raise Exception("CMake references missing files, aborting!") - del is_err - del errs - - # now check on files not accounted for. - print("\nC/C++ Files CMake does not know about...") - for cf in sorted(source_list(SOURCE_DIR, is_c)): - if not is_ignore_source(cf, ignore_used_source): - if cf not in global_c: - print("missing_c: ", cf) - - # Check if automake builds a corresponding .o file. - ''' - if cf in global_c: - out1 = os.path.splitext(cf)[0] + ".o" - out2 = os.path.splitext(cf)[0] + ".Po" - out2_dir, out2_file = out2 = os.path.split(out2) - out2 = os.path.join(out2_dir, ".deps", out2_file) - if not os.path.exists(out1) and not os.path.exists(out2): - print("bad_c: ", cf) - ''' - - print("\nC/C++ Headers CMake does not know about...") - for hf in sorted(source_list(SOURCE_DIR, is_c_header)): - if not is_ignore_source(hf, ignore_used_source): - if hf not in global_h: - print("missing_h: ", hf) - - if UTF8_CHECK: - # test encoding - import traceback - for files in (global_c, global_h): - for f in sorted(files): - if os.path.exists(f): - # ignore outside of our source tree - if "extern" not in f: - i = 1 - try: - for _ in open(f, "r", encoding="utf8"): - i += 1 - except UnicodeDecodeError: - print("Non utf8: %s:%d" % (f, i)) - if i > 1: - traceback.print_exc() - - # Check ignores aren't stale - print("\nCheck for unused 'IGNORE_SOURCE' paths...") - for index, ignore_path in enumerate(IGNORE_SOURCE): - if not ignore_used_source[index]: - print("unused ignore: %r" % ignore_path) - - # Check ignores aren't stale - print("\nCheck for unused 'IGNORE_SOURCE_MISSING' paths...") - for k, v in sorted(global_ignore_source_missing.items()): - for ignore_path in v: - print("unused ignore: %r -> %r" % (ignore_path, k)) - - # Check ignores aren't stale - print("\nCheck for unused 'IGNORE_CMAKE' paths...") - for index, ignore_path in enumerate(IGNORE_CMAKE): - if not ignore_used_cmake[index]: - print("unused ignore: %r" % ignore_path) - - -if __name__ == "__main__": - main() diff --git a/build_files/cmake/cmake_consistency_check_config.py b/build_files/cmake/cmake_consistency_check_config.py deleted file mode 100644 index 94e5ddb4289..00000000000 --- a/build_files/cmake/cmake_consistency_check_config.py +++ /dev/null @@ -1,64 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-or-later -import os - -IGNORE_SOURCE = ( - "/test/", - "/tests/gtests/", - "/release/", - - # specific source files - "extern/audaspace/", - - # Use for `WIN32` only. - "source/creator/blender_launcher_win32.c", - - # specific source files - "extern/bullet2/src/BulletCollision/CollisionDispatch/btBox2dBox2dCollisionAlgorithm.cpp", - "extern/bullet2/src/BulletCollision/CollisionDispatch/btConvex2dConvex2dAlgorithm.cpp", - "extern/bullet2/src/BulletCollision/CollisionDispatch/btInternalEdgeUtility.cpp", - "extern/bullet2/src/BulletCollision/CollisionShapes/btBox2dShape.cpp", - "extern/bullet2/src/BulletCollision/CollisionShapes/btConvex2dShape.cpp", - "extern/bullet2/src/BulletDynamics/Character/btKinematicCharacterController.cpp", - "extern/bullet2/src/BulletDynamics/ConstraintSolver/btHinge2Constraint.cpp", - "extern/bullet2/src/BulletDynamics/ConstraintSolver/btUniversalConstraint.cpp", - - "doc/doxygen/doxygen.extern.h", - "doc/doxygen/doxygen.intern.h", - "doc/doxygen/doxygen.main.h", - "doc/doxygen/doxygen.source.h", - "extern/bullet2/src/BulletCollision/CollisionDispatch/btBox2dBox2dCollisionAlgorithm.h", - "extern/bullet2/src/BulletCollision/CollisionDispatch/btConvex2dConvex2dAlgorithm.h", - "extern/bullet2/src/BulletCollision/CollisionDispatch/btInternalEdgeUtility.h", - "extern/bullet2/src/BulletCollision/CollisionShapes/btBox2dShape.h", - "extern/bullet2/src/BulletCollision/CollisionShapes/btConvex2dShape.h", - "extern/bullet2/src/BulletDynamics/Character/btKinematicCharacterController.h", - "extern/bullet2/src/BulletDynamics/ConstraintSolver/btHinge2Constraint.h", - "extern/bullet2/src/BulletDynamics/ConstraintSolver/btUniversalConstraint.h", -) - -# Ignore cmake file, path pairs. -IGNORE_SOURCE_MISSING = ( - ( # Use for cycles stand-alone. - "intern/cycles/util/CMakeLists.txt", ( - "../../third_party/numaapi/include", - )), - ( # Use for `WITH_NANOVDB`. - "intern/cycles/kernel/CMakeLists.txt", ( - "nanovdb/util/CSampleFromVoxels.h", - "nanovdb/util/SampleFromVoxels.h", - "nanovdb/NanoVDB.h", - "nanovdb/CNanoVDB.h", - ), - ), -) - -IGNORE_CMAKE = ( - "extern/audaspace/CMakeLists.txt", -) - -UTF8_CHECK = True - -SOURCE_DIR = os.path.normpath(os.path.abspath(os.path.normpath(os.path.join(os.path.dirname(__file__), "..", "..")))) - -# doesn't have to exist, just use as reference -BUILD_DIR = os.path.normpath(os.path.abspath(os.path.normpath(os.path.join(SOURCE_DIR, "..", "build"))))