bartvdbraak/blender
1
0
Fork 3
Code Issues 3 Pull Requests 1 Actions 1 Packages Projects Releases Wiki Activity

Switch SVN to Git submodules using Git-LFS

Browse Source 
This change makes it so build system and update utilities for Blender builds
are using pre-compiled libraries and other resources attached as Git modules
instead of using checkout of SVN repositories in the parent folder.

The directory layout:
```
  * release/datafiles/
    * assets/        -> blender-assets.git
      * publish/
      * ...
      * README.txt
  * lib/
    * darwin_x64/    -> lib-darwin_x64.git
    * darwin_arm64/  -> lib-darwin_arm64.git
    * linux_x64/     -> lib-linux_x64.git
    * windows_x64/   -> lib-windows_x64.git
  * tests/
    * data/         -> blender-test-data.git
```

The changes about configuring the actual Git sub-modules are not included
into this patch, as those require repository to actually exist before it
can be used.

The assets submodule is enabled by default, and the rest of them are
disabled. This means that if someone runs `git submodule update --init`
they will not get heavy libraries. The platform-specific and tests
related submodules are enabled when using `make update` or `make test`.

All the submodules are tracked: this means that when new commits are
done to the submodule, the blender.git repository is to be updated to
point them to the new hash. This causes some extra manual work, but it
allows to more easily update Blender and its dependencies to known good
state when performing operations like bisect.

Ref #108978

Pull Request: https://projects.blender.org/blender/blender/pulls/117946
This commit is contained in:
Sergey Sharybin 2024-02-22 13:50:55 +01:00 committed by Sergey Sharybin
parent 0007c7a6b2
commit 3dc832a904
31 changed files with 378 additions and 361 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -56,6 +56,7 @@ waveletNoiseTile.bin
# External repositories.
/scripts/addons/
/scripts/addons_contrib/
/tests/benchmarks/
# Ignore old submodules directories.
# Eventually need to get rid of those, but for the first time of transition

29
.gitmodules vendored Normal file
View File

@ -0,0 +1,29 @@
[submodule "lib/linux_x64"]
update = none
path = lib/linux_x64
url = https://projects.blender.org/blender/lib-linux_x64.git
branch = blender-v4.1-release
[submodule "lib/macos_arm64"]
update = none
path = lib/macos_arm64
url = https://projects.blender.org/blender/lib-macos_arm64.git
branch = blender-v4.1-release
[submodule "lib/macos_x64"]
update = none
path = lib/macos_x64
url = https://projects.blender.org/blender/lib-macos_x64.git
branch = blender-v4.1-release
[submodule "lib/windows_x64"]
update = none
path = lib/windows_x64
url = https://projects.blender.org/blender/lib-windows_x64.git
branch = blender-v4.1-release
[submodule "release/datafiles/assets"]
path = release/datafiles/assets
url = https://projects.blender.org/blender/blender-assets.git
branch = blender-v4.1-release
[submodule "tests/data"]
update = none
path = tests/data
url = https://projects.blender.org/blender/blender-test-data.git
branch = blender-v4.1-release

13
build_files/cmake/platform/platform_apple.cmake
View File

@ -49,18 +49,17 @@ endif()
if(NOT DEFINED LIBDIR)
if("${CMAKE_OSX_ARCHITECTURES}" STREQUAL "x86_64")
set(LIBDIR ${CMAKE_SOURCE_DIR}/../lib/darwin)
set(LIBDIR ${CMAKE_SOURCE_DIR}/lib/macos_x64)
else()
set(LIBDIR ${CMAKE_SOURCE_DIR}/../lib/darwin_${CMAKE_OSX_ARCHITECTURES})
endif()
else()
if(FIRST_RUN)
message(STATUS "Using pre-compiled LIBDIR: ${LIBDIR}")
set(LIBDIR ${CMAKE_SOURCE_DIR}/lib/macos_${CMAKE_OSX_ARCHITECTURES})
endif()
endif()
if(NOT EXISTS "${LIBDIR}/")
if(NOT EXISTS "${LIBDIR}/.git")
message(FATAL_ERROR "Mac OSX requires pre-compiled libs at: '${LIBDIR}'")
endif()
if(FIRST_RUN)
message(STATUS "Using pre-compiled LIBDIR: ${LIBDIR}")
endif()
# Avoid searching for headers since this would otherwise override our lib
# directory as well as PYTHON_ROOT_DIR.

4
build_files/cmake/platform/platform_unix.cmake
View File

@ -16,13 +16,13 @@ else()
set(LIBDIR_NATIVE_ABI ${CMAKE_SOURCE_DIR}/../lib/${LIBDIR_NAME})
# Path to precompiled libraries with known glibc 2.28 ABI.
set(LIBDIR_GLIBC228_ABI ${CMAKE_SOURCE_DIR}/../lib/linux_x86_64_glibc_228)
set(LIBDIR_GLIBC228_ABI ${CMAKE_SOURCE_DIR}/lib/linux_x64)
# Choose the best suitable libraries.
if(EXISTS ${LIBDIR_NATIVE_ABI})
set(LIBDIR ${LIBDIR_NATIVE_ABI})
set(WITH_LIBC_MALLOC_HOOK_WORKAROUND TRUE)
elseif(EXISTS ${LIBDIR_GLIBC228_ABI})
elseif(EXISTS "${LIBDIR_GLIBC228_ABI}/.git")
set(LIBDIR ${LIBDIR_GLIBC228_ABI})
if(WITH_MEM_JEMALLOC)
# jemalloc provides malloc hooks.

8
build_files/cmake/platform/platform_win32.cmake
View File

@ -266,23 +266,23 @@ if(NOT DEFINED LIBDIR)
# Setup 64bit and 64bit windows systems
if(CMAKE_CL_64)
message(STATUS "64 bit compiler detected.")
set(LIBDIR_BASE "win64")
set(LIBDIR_BASE "windows_x64")
else()
message(FATAL_ERROR "32 bit compiler detected, blender no longer provides pre-build libraries for 32 bit windows, please set the LIBDIR cmake variable to your own library folder")
endif()
if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 19.30.30423)
message(STATUS "Visual Studio 2022 detected.")
set(LIBDIR ${CMAKE_SOURCE_DIR}/../lib/${LIBDIR_BASE}_vc15)
set(LIBDIR ${CMAKE_SOURCE_DIR}/lib/${LIBDIR_BASE})
elseif(MSVC_VERSION GREATER 1919)
message(STATUS "Visual Studio 2019 detected.")
set(LIBDIR ${CMAKE_SOURCE_DIR}/../lib/${LIBDIR_BASE}_vc15)
set(LIBDIR ${CMAKE_SOURCE_DIR}/lib/${LIBDIR_BASE})
endif()
else()
if(FIRST_RUN)
message(STATUS "Using pre-compiled LIBDIR: ${LIBDIR}")
endif()
endif()
if(NOT EXISTS "${LIBDIR}/")
if(NOT EXISTS "${LIBDIR}/.git")
message(FATAL_ERROR "\n\nWindows requires pre-compiled libs at: '${LIBDIR}'. Please run `make update` in the blender source folder to obtain them.")
endif()

19
build_files/utils/make_test.py
View File

@ -13,6 +13,7 @@ import sys
import make_utils
from make_utils import call
from pathlib import Path
# Parse arguments.
@ -21,7 +22,6 @@ def parse_arguments() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--ctest-command", default="ctest")
parser.add_argument("--cmake-command", default="cmake")
parser.add_argument("--svn-command", default="svn")
parser.add_argument("--git-command", default="git")
parser.add_argument("--config", default="")
parser.add_argument("build_directory")
@ -30,7 +30,6 @@ def parse_arguments() -> argparse.Namespace:
args = parse_arguments()
git_command = args.git_command
svn_command = args.svn_command
ctest_command = args.ctest_command
cmake_command = args.cmake_command
config = args.config
@ -45,24 +44,18 @@ if make_utils.command_missing(git_command):
sys.exit(1)
# Test if we are building a specific release version.
branch = make_utils.git_branch(git_command)
tag = make_utils.git_tag(git_command)
release_version = make_utils.git_branch_release_version(branch, tag)
lib_tests_dirpath = os.path.join('..', 'lib', "tests")
lib_tests_dirpath = Path("tests") / "data"
if not os.path.exists(lib_tests_dirpath):
if not (lib_tests_dirpath / ".git").exists():
print("Tests files not found, downloading...")
if make_utils.command_missing(svn_command):
sys.stderr.write("svn not found, can't checkout test files\n")
sys.exit(1)
if make_utils.command_missing(cmake_command):
sys.stderr.write("cmake not found, can't checkout test files\n")
sys.exit(1)
svn_url = make_utils.svn_libraries_base_url(release_version) + "/tests"
call([svn_command, "checkout", svn_url, lib_tests_dirpath])
# Ensure the test data files sub-module is configured and present.
make_utils.git_enable_submodule(git_command, "tests/data")
make_utils.git_update_submodule(args.git_command, lib_tests_dirpath)
# Run cmake again to detect tests files.
os.chdir(build_dir)

437
build_files/utils/make_update.py
View File

@ -4,11 +4,11 @@
# SPDX-License-Identifier: GPL-2.0-or-later
"""
"make update" for all platforms, updating svn libraries and tests and Blender
git repository and sub-modules.
"make update" for all platforms, updating Git LFS submodules for libraries and
tests, and Blender git repository.
For release branches, this will check out the appropriate branches of
sub-modules and libraries.
submodules and libraries.
"""
import argparse
@ -20,168 +20,163 @@ import sys
import make_utils
from pathlib import Path
from make_utils import call, check_output
from urllib.parse import urljoin
from urllib.parse import urljoin, urlsplit
from typing import (
Optional,
)
class Submodule:
path: str
branch: str
branch_fallback: str
def __init__(self, path: str, branch: str, branch_fallback: str) -> None:
self.path = path
self.branch = branch
self.branch_fallback = branch_fallback
from typing import Optional
def print_stage(text: str) -> None:
print("")
print(text)
print("=" * len(text))
print("")
# Parse arguments
def parse_arguments() -> argparse.Namespace:
"""
Parse command line line arguments.
Returns parsed object from which the command line arguments can be accessed
as properties. The name of the properties matches the command line argument,
but with the leading dashed omitted and all remaining dashes replaced with
underscore.
"""
parser = argparse.ArgumentParser()
parser.add_argument("--no-libraries", action="store_true")
parser.add_argument("--no-blender", action="store_true")
parser.add_argument("--no-submodules", action="store_true")
parser.add_argument("--use-tests", action="store_true")
parser.add_argument("--svn-command", default="svn")
parser.add_argument("--svn-branch", default=None)
parser.add_argument("--git-command", default="git")
parser.add_argument("--use-linux-libraries", action="store_true")
parser.add_argument("--architecture", type=str, choices=("x86_64", "amd64", "arm64",))
parser.add_argument("--architecture", type=str,
choices=("x86_64", "amd64", "arm64",))
return parser.parse_args()
def get_blender_git_root() -> str:
return check_output([args.git_command, "rev-parse", "--show-toplevel"])
def get_blender_git_root() -> Path:
"""
Get root directory of the current Git directory.
"""
return Path(
check_output([args.git_command, "rev-parse", "--show-toplevel"]))
# Setup for precompiled libraries and tests from svn.
def get_effective_platform(args: argparse.Namespace) -> str:
"""
Get platform of the host.
The result string is normalized to the name used by Blender releases and
library repository name prefixes: linux, macos, windows.
"""
if sys.platform == "darwin":
platform = "macos"
elif sys.platform == "win32":
platform = "windows"
else:
platform = sys.platform
assert (platform in ("linux", "macos", "windows"))
return platform
def get_effective_architecture(args: argparse.Namespace) -> str:
"""
Get architecture of the host.
The result string is normalized to the architecture name used by the Blender
releases and library repository name suffixes: x64, arm64.
NOTE: When cross-compiling the architecture is coming from the command line
argument.
"""
architecture = args.architecture
if architecture:
assert isinstance(architecture, str)
return architecture
# Check platform.version to detect arm64 with x86_64 python binary.
if "ARM64" in platform.version():
return "arm64"
return platform.machine().lower()
def svn_update(args: argparse.Namespace, release_version: Optional[str]) -> None:
svn_non_interactive = [args.svn_command, '--non-interactive']
lib_dirpath = os.path.join(get_blender_git_root(), '..', 'lib')
svn_url = make_utils.svn_libraries_base_url(release_version, args.svn_branch)
# Checkout precompiled libraries
architecture = get_effective_architecture(args)
if sys.platform == 'darwin':
if architecture == 'arm64':
lib_platform = "darwin_arm64"
elif architecture == 'x86_64':
lib_platform = "darwin"
else:
lib_platform = None
elif sys.platform == 'win32':
# Windows checkout is usually handled by bat scripts since python3 to run
# this script is bundled as part of the precompiled libraries. However it
# is used by the buildbot.
lib_platform = "win64_vc15"
elif args.use_linux_libraries:
lib_platform = "linux_x86_64_glibc_228"
elif "ARM64" in platform.version():
# Check platform.version to detect arm64 with x86_64 python binary.
architecture = "arm64"
else:
# No precompiled libraries for Linux.
lib_platform = None
architecture = platform.machine().lower()
if lib_platform:
lib_platform_dirpath = os.path.join(lib_dirpath, lib_platform)
# Normalize the architecture name.
if architecture == 'x86_64':
architecture = "x64"
if not os.path.exists(lib_platform_dirpath):
print_stage("Checking out Precompiled Libraries")
assert (architecture in ("x64", "arm64"))
if make_utils.command_missing(args.svn_command):
sys.stderr.write("svn not found, can't checkout libraries\n")
sys.exit(1)
svn_url_platform = svn_url + lib_platform
call(svn_non_interactive + ["checkout", svn_url_platform, lib_platform_dirpath])
if args.use_tests:
lib_tests = "tests"
lib_tests_dirpath = os.path.join(lib_dirpath, lib_tests)
if not os.path.exists(lib_tests_dirpath):
print_stage("Checking out Tests")
if make_utils.command_missing(args.svn_command):
sys.stderr.write("svn not found, can't checkout tests\n")
sys.exit(1)
svn_url_tests = svn_url + lib_tests
call(svn_non_interactive + ["checkout", svn_url_tests, lib_tests_dirpath])
lib_assets = "assets"
lib_assets_dirpath = os.path.join(lib_dirpath, lib_assets)
if not os.path.exists(lib_assets_dirpath):
print_stage("Checking out Assets")
if make_utils.command_missing(args.svn_command):
sys.stderr.write("svn not found, can't checkout assets\n")
sys.exit(1)
svn_url_assets = svn_url + lib_assets
call(svn_non_interactive + ["checkout", svn_url_assets, lib_assets_dirpath])
# Update precompiled libraries, assets and tests
if not os.path.isdir(lib_dirpath):
print("Library path: %r, not found, skipping" % lib_dirpath)
else:
paths_local_and_remote = []
if os.path.exists(os.path.join(lib_dirpath, ".svn")):
print_stage("Updating Precompiled Libraries, Assets and Tests (one repository)")
paths_local_and_remote.append((lib_dirpath, svn_url))
else:
print_stage("Updating Precompiled Libraries, Assets and Tests (multiple repositories)")
# Separate paths checked out.
for dirname in os.listdir(lib_dirpath):
if dirname.startswith("."):
# Temporary paths such as ".mypy_cache" will report a warning, skip hidden directories.
continue
dirpath = os.path.join(lib_dirpath, dirname)
if not (os.path.isdir(dirpath) and os.path.exists(os.path.join(dirpath, ".svn"))):
continue
paths_local_and_remote.append((dirpath, svn_url + dirname))
if paths_local_and_remote:
if make_utils.command_missing(args.svn_command):
sys.stderr.write("svn not found, can't update libraries\n")
sys.exit(1)
for dirpath, svn_url_full in paths_local_and_remote:
call(svn_non_interactive + ["cleanup", dirpath])
# Switch to appropriate branch and update.
call(svn_non_interactive + ["switch", svn_url_full, dirpath], exit_on_error=False)
call(svn_non_interactive + ["update", dirpath])
return architecture
def get_submodule_directories(args: argparse.Namespace):
"""
Get list of all configured submodule directories.
"""
blender_git_root = get_blender_git_root()
dot_modules = blender_git_root / ".gitmodules"
if not dot_modules.exists():
return ()
submodule_directories_output = check_output(
[args.git_command, "config", "--file", dot_modules, "--get-regexp", "path"])
return (Path(line.split(' ', 1)[1]) for line in submodule_directories_output.strip().splitlines())
def update_precompiled_libraries(args: argparse.Namespace) -> None:
"""
Configure and update submodule for precompiled libraries
This function detects the current host architecture and enables
corresponding submodule, and updates the submodule.
NOTE: When cross-compiling the architecture is coming from the command line
argument.
"""
print_stage("Configuring Precompiled Libraries")
platform = get_effective_platform(args)
arch = get_effective_architecture(args)
print(f"Detected platform : {platform}")
print(f"Detected architecture : {arch}")
print()
if sys.platform == "linux" and not args.use_linux_libraries:
print("Skipping Linux libraries configuration")
return
submodule_dir = f"lib/{platform}_{arch}"
submodule_directories = get_submodule_directories(args)
if Path(submodule_dir) not in submodule_directories:
print("Skipping libraries update: no configured submodule")
return
make_utils.git_enable_submodule(args.git_command, submodule_dir)
make_utils.git_update_submodule(args.git_command, submodule_dir)
def update_tests_data_files(args: argparse.Namespace) -> None:
"""
Configure and update submodule with files used by regression tests
"""
print_stage("Configuring Tests Data Files")
submodule_dir = "tests/data"
make_utils.git_enable_submodule(args.git_command, submodule_dir)
make_utils.git_update_submodule(args.git_command, submodule_dir)
# Test if git repo can be updated.
def git_update_skip(args: argparse.Namespace, check_remote_exists: bool = True) -> str:
"""Test if git repo can be updated."""
if make_utils.command_missing(args.git_command):
sys.stderr.write("git not found, can't update code\n")
sys.exit(1)
@ -274,23 +269,22 @@ def resolve_external_url(blender_url: str, repo_name: str) -> str:
return urljoin(blender_url + "/", "../" + repo_name)
def external_script_copy_old_submodule_over(args: argparse.Namespace, directory_name: str) -> None:
blender_git_root = Path(get_blender_git_root())
scripts_dir = blender_git_root / "scripts"
external_dir = scripts_dir / directory_name
def external_script_copy_old_submodule_over(
args: argparse.Namespace,
directory: Path,
old_submodules_dir: Path) -> None:
blender_git_root = get_blender_git_root()
external_dir = blender_git_root / directory
old_submodule_relative_dir = Path("release") / "scripts" / directory_name
print(f"Moving {old_submodule_relative_dir} to scripts/{directory_name} ...")
old_submodule_dir = blender_git_root / old_submodule_relative_dir
shutil.move(old_submodule_dir, external_dir)
print(f"Moving {old_submodules_dir} to {directory} ...")
shutil.move(blender_git_root / old_submodules_dir, external_dir)
# Remove old ".git" which is a file with path to a submodule bare repo inside of main
# repo .git/modules directory.
(external_dir / ".git").unlink()
bare_repo_relative_dir = Path(".git") / "modules" / "release" / "scripts" / directory_name
print(f"Copying {bare_repo_relative_dir} to scripts/{directory_name}/.git ...")
bare_repo_relative_dir = Path(".git") / "modules" / old_submodules_dir
print(f"Copying {bare_repo_relative_dir} to {directory}/.git ...")
bare_repo_dir = blender_git_root / bare_repo_relative_dir
shutil.copytree(bare_repo_dir, external_dir / ".git")
@ -298,25 +292,26 @@ def external_script_copy_old_submodule_over(args: argparse.Namespace, directory_
call((args.git_command, "config", "--file", str(git_config), "--unset", "core.worktree"))
def external_script_initialize_if_needed(args: argparse.Namespace,
repo_name: str,
directory_name: str) -> None:
"""Initialize checkout of an external repository scripts directory"""
def floating_checkout_initialize_if_needed(args: argparse.Namespace,
repo_name: str,
directory: Path,
old_submodules_dir: Path = None) -> None:
"""Initialize checkout of an external repository"""
blender_git_root = Path(get_blender_git_root())
blender_git_root = get_blender_git_root()
blender_dot_git = blender_git_root / ".git"
scripts_dir = blender_git_root / "scripts"
external_dir = scripts_dir / directory_name
external_dir = blender_git_root / directory
if external_dir.exists():
return
print(f"Initializing scripts/{directory_name} ...")
print(f"Initializing {directory} ...")
old_submodule_dot_git = blender_git_root / "release" / "scripts" / directory_name / ".git"
if old_submodule_dot_git.exists() and blender_dot_git.is_dir():
external_script_copy_old_submodule_over(args, directory_name)
return
if old_submodules_dir is not None:
old_submodule_dot_git = blender_git_root / old_submodules_dir / ".git"
if old_submodule_dot_git.exists() and blender_dot_git.is_dir():
external_script_copy_old_submodule_over(args, directory, old_submodules_dir)
return
origin_name = "upstream" if use_upstream_workflow(args) else "origin"
blender_url = make_utils.git_get_remote_url(args.git_command, origin_name)
@ -330,9 +325,9 @@ def external_script_initialize_if_needed(args: argparse.Namespace,
call((args.git_command, "clone", "--origin", origin_name, external_url, str(external_dir)))
def external_script_add_origin_if_needed(args: argparse.Namespace,
repo_name: str,
directory_name: str) -> None:
def floating_checkout_add_origin_if_needed(args: argparse.Namespace,
repo_name: str,
directory: Path) -> None:
"""
Add remote called 'origin' if there is a fork of the external repository available
@ -344,9 +339,8 @@ def external_script_add_origin_if_needed(args: argparse.Namespace,
cwd = os.getcwd()
blender_git_root = Path(get_blender_git_root())
scripts_dir = blender_git_root / "scripts"
external_dir = scripts_dir / directory_name
blender_git_root = get_blender_git_root()
external_dir = blender_git_root / directory
origin_blender_url = make_utils.git_get_remote_url(args.git_command, "origin")
origin_external_url = resolve_external_url(origin_blender_url, repo_name)
@ -361,7 +355,7 @@ def external_script_add_origin_if_needed(args: argparse.Namespace,
if not make_utils.git_is_remote_repository(args.git_command, origin_external_url):
return
print(f"Adding origin remote to {directory_name} pointing to fork ...")
print(f"Adding origin remote to {directory} pointing to fork ...")
# Non-obvious tricks to introduce the new remote called "origin" to the existing
# submodule configuration.
@ -390,23 +384,30 @@ def external_script_add_origin_if_needed(args: argparse.Namespace,
return
def external_scripts_update(args: argparse.Namespace,
repo_name: str,
directory_name: str,
branch: Optional[str]) -> str:
def floating_checkout_update(args: argparse.Namespace,
repo_name: str,
directory: Path,
branch: Optional[str],
old_submodules_dir: Path = None,
only_update=False) -> str:
"""Update a single external checkout with the given name in the scripts folder"""
external_script_initialize_if_needed(args, repo_name, directory_name)
external_script_add_origin_if_needed(args, repo_name, directory_name)
blender_git_root = get_blender_git_root()
external_dir = blender_git_root / directory
print(f"Updating scripts/{directory_name} ...")
if only_update and not external_dir.exists():
return ""
floating_checkout_initialize_if_needed(args, repo_name, directory, old_submodules_dir)
floating_checkout_add_origin_if_needed(args, repo_name, directory)
blender_git_root = get_blender_git_root()
external_dir = blender_git_root / directory
print(f"* Updating {directory} ...")
cwd = os.getcwd()
blender_git_root = Path(get_blender_git_root())
scripts_dir = blender_git_root / "scripts"
external_dir = scripts_dir / directory_name
# Update externals to appropriate given branch, falling back to main if none is given and/or
# found in a sub-repository.
branch_fallback = "main"
@ -419,7 +420,7 @@ def external_scripts_update(args: argparse.Namespace,
os.chdir(external_dir)
msg = git_update_skip(args, check_remote_exists=False)
if msg:
skip_msg += directory_name + " skipped: " + msg + "\n"
skip_msg += str(directory) + " skipped: " + msg + "\n"
else:
# Find a matching branch that exists.
for remote in ("origin", "upstream"):
@ -465,6 +466,17 @@ def external_scripts_update(args: argparse.Namespace,
return skip_msg
def external_scripts_update(args: argparse.Namespace,
repo_name: str,
directory_name: str,
branch: Optional[str]) -> str:
return floating_checkout_update(args,
repo_name,
Path("scripts") / directory_name,
branch,
old_submodules_dir=Path("release") / "scripts" / directory_name)
def scripts_submodules_update(args: argparse.Namespace, branch: Optional[str]) -> str:
"""Update working trees of addons and addons_contrib within the scripts/ directory"""
msg = ""
@ -475,12 +487,75 @@ def scripts_submodules_update(args: argparse.Namespace, branch: Optional[str]) -
return msg
def submodules_update(args: argparse.Namespace, branch: Optional[str]) -> str:
"""Update submodules or other externally tracked source trees"""
def floating_libraries_update(args: argparse.Namespace, branch: Optional[str]) -> str:
"""Update libraries checkouts which are floating (not attached as Git submodules)"""
msg = ""
msg += floating_checkout_update(args,
"benchmarks",
Path("tests") / "benchmarks",
branch,
only_update=True)
return msg
def add_submodule_push_url(args: argparse.Namespace):
"""
Add pushURL configuration for all locally activated submodules, pointing to SSH protocol.
"""
blender_git_root = get_blender_git_root()
modules = blender_git_root / ".git" / "modules"
submodule_directories = get_submodule_directories(args)
for submodule_path in submodule_directories:
module_path = modules / submodule_path
config = module_path / "config"
if not config.exists():
# Ignore modules which are not initialized
continue
push_url = check_output((args.git_command, "config", "--file", str(config),
"--get", "remote.origin.pushURL"), exit_on_error=False)
if push_url:
# Ignore modules which have pushURL configured.
continue
url = make_utils.git_get_config(args.git_command, "remote.origin.url", str(config))
url = "https://projects.blender.org/blender/lib-darwin_arm64.git" # XXX
if not url.startswith("https:"):
# Ignore non-URL URLs.
continue
url_parts = urlsplit(url)
push_url = f"git@{url_parts.netloc}:{url_parts.path[1:]}"
print(f"Setting pushURL to {push_url} for {submodule_path}")
make_utils.git_set_config(args.git_command, "remote.origin.pushURL", push_url, str(config))
def submodules_update(args: argparse.Namespace, branch: Optional[str]) -> str:
"""Update submodules or other externally tracked source trees"""
print_stage("Updating Submodules")
msg = ""
call((args.git_command, "lfs", "install"))
msg += scripts_submodules_update(args, branch)
msg += floating_libraries_update(args, branch)
print("* Updating Git submodules")
exitcode = call((args.git_command, "submodule", "update", "--init"), exit_on_error=False)
if exitcode != 0:
msg += "Error updating Git submodules\n"
add_submodule_push_url(args)
return msg
@ -494,26 +569,30 @@ if __name__ == "__main__":
major = blender_version.version // 100
minor = blender_version.version % 100
branch = f"blender-v{major}.{minor}-release"
release_version: Optional[str] = f"{major}.{minor}"
else:
branch = 'main'
release_version = None
if not args.no_libraries:
svn_update(args, release_version)
update_precompiled_libraries(args)
if args.use_tests:
update_tests_data_files(args)
if not args.no_blender:
blender_skip_msg = git_update_skip(args)
if not blender_skip_msg:
blender_skip_msg = blender_update(args)
if blender_skip_msg:
blender_skip_msg = "Blender repository skipped: " + blender_skip_msg + "\n"
if not args.no_submodules:
submodules_skip_msg = submodules_update(args, branch)
# Report any skipped repositories at the end, so it's not as easy to miss.
skip_msg = blender_skip_msg + submodules_skip_msg
if skip_msg:
print_stage(skip_msg.strip())
print()
print(skip_msg.strip())
print()
# For failed submodule update we throw an error, since not having correct
# submodules can make Blender throw errors.

68
build_files/utils/make_utils.py
View File

@ -11,9 +11,7 @@ import re
import shutil
import subprocess
import sys
import os
from pathlib import Path
from urllib.parse import urljoin
from typing import (
Sequence,
@ -48,7 +46,7 @@ def check_output(cmd: Sequence[str], exit_on_error: bool = True) -> str:
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, universal_newlines=True)
except subprocess.CalledProcessError as e:
if exit_on_error:
sys.stderr.write(" ".join(cmd))
sys.stderr.write(" ".join(cmd) + "\n")
sys.stderr.write(e.output + "\n")
sys.exit(e.returncode)
output = ""
@ -87,25 +85,6 @@ def git_remote_exist(git_command: str, remote_name: str) -> bool:
return remote_url != remote_name
def git_get_resolved_submodule_url(git_command: str, blender_url: str, submodule_path: str) -> str:
git_root = check_output([git_command, "rev-parse", "--show-toplevel"])
dot_gitmodules = os.path.join(git_root, ".gitmodules")
submodule_key_prefix = f"submodule.{submodule_path}"
submodule_key_url = f"{submodule_key_prefix}.url"
gitmodule_url = git_get_config(
git_command, submodule_key_url, file=dot_gitmodules)
# A bit of a trickery to construct final URL.
# Only works for the relative submodule URLs.
#
# Note that unless the LHS URL ends up with a slash urljoin treats the last component as a
# file.
assert gitmodule_url.startswith('..')
return urljoin(blender_url + "/", gitmodule_url)
def git_is_remote_repository(git_command: str, repo: str) -> bool:
"""Returns true if the given repository is a valid/clonable git repo"""
exit_code = call((git_command, "ls-remote", repo, "HEAD"), exit_on_error=False, silent=True)
@ -113,7 +92,8 @@ def git_is_remote_repository(git_command: str, repo: str) -> bool:
def git_branch(git_command: str) -> str:
# Get current branch name.
"""Get current branch name."""
try:
branch = subprocess.check_output([git_command, "rev-parse", "--abbrev-ref", "HEAD"])
except subprocess.CalledProcessError as e:
@ -137,44 +117,32 @@ def git_set_config(git_command: str, key: str, value: str, file: Optional[str] =
return check_output([git_command, "config", key, value])
def git_tag(git_command: str) -> Optional[str]:
# Get current tag name.
try:
tag = subprocess.check_output([git_command, "describe", "--exact-match"], stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
return None
def git_enable_submodule(git_command: str, submodule_dir: str):
"""Enable submodule denoted by its directory within the repository"""
return tag.strip().decode('utf8')
command = (git_command,
"config",
"--local",
f"submodule.{submodule_dir}.update", "checkout")
call(command, exit_on_error=True, silent=False)
def git_branch_release_version(branch: str, tag: Optional[str]) -> Optional[str]:
re_match = re.search("^blender-v(.*)-release$", branch)
release_version = None
if re_match:
release_version = re_match.group(1)
elif tag:
re_match = re.search(r"^v([0-9]*\.[0-9]*).*", tag)
if re_match:
release_version = re_match.group(1)
return release_version
def git_update_submodule(git_command: str, submodule_dir: str):
"""
Update the given submodule.
The submodule is denoted by its path within the repository.
This function will initialize the submodule if it has not been initialized.
"""
def svn_libraries_base_url(release_version: Optional[str], branch: Optional[str] = None) -> str:
if release_version:
svn_branch = "tags/blender-" + release_version + "-release"
elif branch:
svn_branch = "branches/" + branch
else:
svn_branch = "trunk"
return "https://svn.blender.org/svnroot/bf-blender/" + svn_branch + "/lib/"
call((git_command, "submodule", "update", "--init", submodule_dir))
def command_missing(command: str) -> bool:
# Support running with Python 2 for macOS
if sys.version_info >= (3, 0):
return shutil.which(command) is None
else:
return False
return False
class BlenderVersion:

33
build_files/windows/check_libraries.cmd
View File

@ -1,37 +1,30 @@
if "%BUILD_VS_YEAR%"=="2019" set BUILD_VS_LIBDIRPOST=vc15
if "%BUILD_VS_YEAR%"=="2022" set BUILD_VS_LIBDIRPOST=vc15
set BUILD_VS_SVNDIR=win64_%BUILD_VS_LIBDIRPOST%
set BUILD_VS_LIBDIR="%BLENDER_DIR%..\lib\%BUILD_VS_SVNDIR%"
set BUILD_VS_LIBDIR="lib\windows_x64"
if NOT "%verbose%" == "" (
echo Library Directory = "%BUILD_VS_LIBDIR%"
)
if NOT EXIST %BUILD_VS_LIBDIR% (
rem libs not found, but svn is on the system
if not "%SVN%"=="" (
if NOT EXIST "%BUILD_VS_LIBDIR%\.git" (
rem libs not found, but git is on the system
if not "%GIT%"=="" (
echo.
echo The required external libraries in %BUILD_VS_LIBDIR% are missing
echo.
set /p GetLibs= "Would you like to download them? (y/n)"
if /I "!GetLibs!"=="Y" (
echo.
echo Downloading %BUILD_VS_SVNDIR% libraries, please wait.
echo Downloading %BUILD_VS_LIBDIR% libraries, please wait.
echo.
:RETRY
"%SVN%" checkout https://svn.blender.org/svnroot/bf-blender/trunk/lib/%BUILD_VS_SVNDIR% %BUILD_VS_LIBDIR%
:RETRY
"%GIT%" -C "%BLENDER_DIR%" config --local "submodule.%BUILD_VS_LIBDIR%.update" "checkout"
"%GIT%" -C "%BLENDER_DIR%" submodule update --init "%BUILD_VS_LIBDIR%"
if errorlevel 1 (
set /p LibRetry= "Error during download, retry? y/n"
if /I "!LibRetry!"=="Y" (
cd %BUILD_VS_LIBDIR%
"%SVN%" cleanup
cd %BLENDER_DIR%
goto RETRY
)
echo.
echo Error: Download of external libraries failed.
echo This is needed for building, please manually run 'svn cleanup' and 'svn update' in
echo %BUILD_VS_LIBDIR% , until this is resolved you CANNOT make a successful blender build
echo Until this is resolved you CANNOT make a successful blender build.
echo.
exit /b 1
)
@ -39,11 +32,11 @@ if NOT EXIST %BUILD_VS_LIBDIR% (
)
) else (
if NOT EXIST %PYTHON% (
if not "%SVN%"=="" (
if not "%GIT%"=="" (
echo.
echo Python not found in external libraries, updating to latest version
echo.
"%SVN%" update %BUILD_VS_LIBDIR%
"%GIT%" -C "%BLENDER_DIR%" submodule update "%BUILD_VS_LIBDIR%"
)
)
)
@ -53,8 +46,8 @@ if NOT EXIST %BUILD_VS_LIBDIR% (
echo Error: Required libraries not found at "%BUILD_VS_LIBDIR%"
echo This is needed for building, aborting!
echo.
if "%SVN%"=="" (
echo This is most likely caused by svn.exe not being available.
if "%GIT%"=="" (
echo This is most likely caused by git.exe not being available.
)
exit /b 1
)

10
build_files/windows/find_dependencies.cmd
View File

@ -1,5 +1,4 @@
REM find all dependencies and set the corresponding environment variables.
for %%X in (svn.exe) do (set SVN=%%~$PATH:X)
for %%X in (cmake.exe) do (set CMAKE=%%~$PATH:X)
for %%X in (ctest.exe) do (set CTEST=%%~$PATH:X)
for %%X in (git.exe) do (set GIT=%%~$PATH:X)
@ -7,19 +6,19 @@ REM For python, default on 310 but if that does not exist also check
REM the 311, 312 and finally 39 folders to see if those are there, it checks
REM this far ahead to ensure good lib folder compatibility in the future
REM it falls back to 3.9 just incase it is a very old lib folder.
set PYTHON=%BLENDER_DIR%\..\lib\win64_vc15\python\310\bin\python.exe
set PYTHON=%BLENDER_DIR%\lib\windows_x64\python\310\bin\python.exe
if EXIST %PYTHON% (
goto detect_python_done
)
set PYTHON=%BLENDER_DIR%\..\lib\win64_vc15\python\311\bin\python.exe
set PYTHON=%BLENDER_DIR%\lib\windows_x64\python\311\bin\python.exe
if EXIST %PYTHON% (
goto detect_python_done
)
set PYTHON=%BLENDER_DIR%\..\lib\win64_vc15\python\312\bin\python.exe
set PYTHON=%BLENDER_DIR%\lib\windows_x64\python\312\bin\python.exe
if EXIST %PYTHON% (
goto detect_python_done
)
set PYTHON=%BLENDER_DIR%\..\lib\win64_vc15\python\39\bin\python.exe
set PYTHON=%BLENDER_DIR%\lib\windows_x64\python\39\bin\python.exe
if EXIST %PYTHON% (
goto detect_python_done
)
@ -31,7 +30,6 @@ if NOT EXIST %PYTHON% (
:detect_python_done
if NOT "%verbose%" == "" (
echo svn : "%SVN%"
echo cmake : "%CMAKE%"
echo ctest : "%CTEST%"
echo git : "%GIT%"

4
build_files/windows/format.cmd
View File

@ -1,5 +1,5 @@
if EXIST %BLENDER_DIR%\..\lib\win64_vc15\llvm\bin\clang-format.exe (
set CF_PATH=..\lib\win64_vc15\llvm\bin
if EXIST %BLENDER_DIR%\lib\windows_x64\llvm\bin\clang-format.exe (
set CF_PATH=lib\windows_x64\llvm\bin
goto detect_done
)

16
build_files/windows/lib_update.cmd Normal file
View File

@ -0,0 +1,16 @@
set BUILD_VS_LIBDIR="lib\windows_x64"
:RETRY
"%GIT%" -C "%BLENDER_DIR%" config --local "submodule.%BUILD_VS_LIBDIR%.update" "checkout"
"%GIT%" -C "%BLENDER_DIR%" submodule update --init "%BUILD_VS_LIBDIR%"
if errorlevel 1 (
set /p LibRetry= "Error during update, retry? y/n"
if /I "!LibRetry!"=="Y" (
goto RETRY
)
echo.
echo Error: Download of external libraries failed.
echo Until this is resolved you CANNOT make a successful blender build.
echo.
exit /b 1
)

3
build_files/windows/parse_arguments.cmd
View File

@ -110,9 +110,6 @@ if NOT "%1" == "" (
) else if "%1" == "doc_py" (
set DOC_PY=1
goto EOF
) else if "%1" == "svnfix" (
set SVN_FIX=1
goto EOF
) else (
echo Command "%1" unknown, aborting!
goto ERR

2
build_files/windows/reset_variables.cmd
View File

@ -4,9 +4,7 @@ set BUILD_CMAKE_ARGS=
set BUILD_ARCH=
set BUILD_VS_VER=
set BUILD_VS_YEAR=
set BUILD_VS_LIBDIRPOST=
set BUILD_VS_LIBDIR=
set BUILD_VS_SVNDIR=
set KEY_NAME=
set MSBUILD_PLATFORM=
set MUST_CLEAN=

13
build_files/windows/show_hashes.cmd
View File

@ -8,13 +8,10 @@ for /f "delims=" %%i in ('"%GIT%" rev-parse HEAD') do echo Branch_hash=%%i
cd "%BLENDER_DIR%/scripts/addons"
for /f "delims=" %%i in ('"%GIT%" rev-parse --abbrev-ref HEAD') do echo Addons_Branch_name=%%i
for /f "delims=" %%i in ('"%GIT%" rev-parse HEAD') do echo Addons_Branch_hash=%%i
if "%SVN%" == "" (
echo SVN not found, cannot library information.
goto EOF
)
set BUILD_VS_LIBDIR=%BLENDER_DIR%..\lib\win64_vc15
for /f "delims=" %%i in ('"%SVN%" info --show-item=url --no-newline %BUILD_VS_LIBDIR% ') do echo Libs_URL=%%i
for /f "delims=" %%i in ('"%SVN%" info --show-item=revision --no-newline %BUILD_VS_LIBDIR% ') do echo Libs_Revision=%%i
for /f "delims=" %%i in ('"%SVN%" info --show-item=last-changed-date --no-newline %BUILD_VS_LIBDIR% ') do echo Libs_LastChange=%%i
cd "%BLENDER_DIR%/lib/windows_x64"
for /f "delims=" %%i in ('"%GIT%" rev-parse --abbrev-ref HEAD') do echo Libs_Branch_name=%%i
for /f "delims=" %%i in ('"%GIT%" rev-parse HEAD') do echo Libs_Branch_hash=%%i
cd "%BLENDER_DIR%"
:EOF

25
build_files/windows/svn_fix.cmd
View File

@ -1,25 +0,0 @@
if "%BUILD_VS_YEAR%"=="2019" set BUILD_VS_LIBDIRPOST=vc15
if "%BUILD_VS_YEAR%"=="2022" set BUILD_VS_LIBDIRPOST=vc15
set BUILD_VS_SVNDIR=win64_%BUILD_VS_LIBDIRPOST%
set BUILD_VS_LIBDIR="%BLENDER_DIR%..\lib\%BUILD_VS_SVNDIR%"
echo Starting cleanup in %BUILD_VS_LIBDIR%.
cd %BUILD_VS_LIBDIR%
:RETRY
"%SVN%" cleanup
"%SVN%" update
if errorlevel 1 (
set /p LibRetry= "Error during update, retry? y/n"
if /I "!LibRetry!"=="Y" (
goto RETRY
)
echo.
echo Error: Download of external libraries failed.
echo This is needed for building, please manually run 'svn cleanup' and 'svn update' in
echo %BUILD_VS_LIBDIR% , until this is resolved you CANNOT make a successful blender build
echo.
exit /b 1
)
echo Cleanup complete

24
build_files/windows/svn_update.cmd
View File

@ -1,24 +0,0 @@
if "%BUILD_VS_YEAR%"=="2019" set BUILD_VS_LIBDIRPOST=vc15
if "%BUILD_VS_YEAR%"=="2022" set BUILD_VS_LIBDIRPOST=vc15
set BUILD_VS_SVNDIR=win64_%BUILD_VS_LIBDIRPOST%
set BUILD_VS_LIBDIR="%BLENDER_DIR%..\lib\%BUILD_VS_SVNDIR%"
cd %BUILD_VS_LIBDIR%
:RETRY
"%SVN%" update
if errorlevel 1 (
set /p LibRetry= "Error during update, retry? y/n"
if /I "!LibRetry!"=="Y" (
"%SVN%" cleanup
goto RETRY
)
echo.
echo Error: Download of external libraries failed.
echo This is needed for building, please manually run 'svn cleanup' and 'svn update' in
echo %BUILD_VS_LIBDIR% , until this is resolved you CANNOT make a successful blender build
echo.
exit /b 1
)
cd %BLENDER_DIR%

2
build_files/windows/test.cmd
View File

@ -8,6 +8,6 @@ exit /b 1
:detect_python_done
REM Use -B to avoid writing __pycache__ in lib directory and causing update conflicts.
%PYTHON% -B %BLENDER_DIR%\build_files\utils\make_test.py --git-command "%GIT%" --svn-command "%SVN%" --cmake-command="%CMAKE%" --ctest-command="%CTEST%" --config="%BUILD_TYPE%" %BUILD_DIR%
%PYTHON% -B %BLENDER_DIR%\build_files\utils\make_test.py --git-command "%GIT%" --cmake-command="%CMAKE%" --ctest-command="%CTEST%" --config="%BUILD_TYPE%" %BUILD_DIR%
:EOF

2
build_files/windows/update_sources.cmd
View File

@ -5,6 +5,6 @@ if NOT EXIST %PYTHON% (
:detect_python_done
REM Use -B to avoid writing __pycache__ in lib directory and causing update conflicts.
%PYTHON% -B %BLENDER_DIR%\build_files\utils\make_update.py --git-command "%GIT%" --svn-command "%SVN%" %BUILD_UPDATE_ARGS%
%PYTHON% -B %BLENDER_DIR%\build_files\utils\make_update.py --git-command "%GIT%" %BUILD_UPDATE_ARGS%
:EOF

1
lib/linux_x64 Submodule

@ -0,0 +1 @@
Subproject commit 591fce4abaa409b31482e515c6d1b95143f58114

1
lib/macos_arm64 Submodule

@ -0,0 +1 @@
Subproject commit 0b09ac9de1557c296cb483355fc9528db707dbd4

1
lib/macos_x64 Submodule

@ -0,0 +1 @@
Subproject commit 9af47e501151474c23afa1a6f5a8b1a419a122ba

1
lib/windows_x64 Submodule

@ -0,0 +1 @@
Subproject commit 58cb225fe4643c4879fb274078e5477c50c0429c

7
make.bat
View File

@ -56,11 +56,6 @@ if "%BUILD_VS_YEAR%" == "" (
)
)
if "%SVN_FIX%" == "1" (
call "%BLENDER_DIR%\build_files\windows\svn_fix.cmd"
goto EOF
)
if "%BUILD_UPDATE%" == "1" (
REM First see if the SVN libs are there and check them out if they are not.
call "%BLENDER_DIR%\build_files\windows\check_libraries.cmd"
@ -70,7 +65,7 @@ if "%BUILD_UPDATE%" == "1" (
REM running tends to be problematic. The python script that update_sources
REM calls later on may still try to switch branches and run into trouble,
REM but for *most* people this will side step the problem.
call "%BLENDER_DIR%\build_files\windows\svn_update.cmd"
call "%BLENDER_DIR%\build_files\windows\lib_update.cmd"
)
REM Finally call the python script shared between all platforms that updates git
REM and does any other SVN work like update the tests or branch switches

1
release/datafiles/assets Submodule

@ -0,0 +1 @@
Subproject commit a5b6ba8fe49679a8b1105c3e80b3e190a97942c0

4
source/creator/CMakeLists.txt
View File

@ -1672,10 +1672,6 @@ install(
set(ASSET_BUNDLE_DIR ${CMAKE_SOURCE_DIR}/release/datafiles/assets/publish/)
if(NOT EXISTS "${ASSET_BUNDLE_DIR}")
set(ASSET_BUNDLE_DIR ${CMAKE_SOURCE_DIR}/../lib/assets/publish/)
endif()
if(EXISTS "${ASSET_BUNDLE_DIR}")
install(
DIRECTORY ${ASSET_BUNDLE_DIR}

1
tests/data Submodule

@ -0,0 +1 @@
Subproject commit 44b6f5d4d180edb700db848be9208efe6f59c9a2

2
tests/performance/api/environment.py
View File

@ -26,7 +26,7 @@ class TestEnvironment:
self.build_dir = base_dir / 'build'
self.install_dir = self.build_dir / "bin"
self.lib_dir = base_dir / 'lib'
self.benchmarks_dir = self.blender_git_dir.parent / 'lib' / 'benchmarks'
self.benchmarks_dir = self.blender_git_dir / 'tests' / 'benchmarks'
self.git_executable = 'git'
self.cmake_executable = 'cmake'
self.cmake_options = ['-DWITH_INTERNATIONAL=OFF', '-DWITH_BUILDINFO=OFF']

2
tests/python/CMakeLists.txt
View File

@ -8,7 +8,7 @@
# and don't give deterministic results
set(USE_EXPERIMENTAL_TESTS FALSE)
set(TEST_SRC_DIR ${CMAKE_SOURCE_DIR}/../lib/tests)
set(TEST_SRC_DIR ${CMAKE_SOURCE_DIR}/tests/data)
set(TEST_PYTHON_DIR ${CMAKE_SOURCE_DIR}/tests/python)
set(TEST_OUT_DIR ${CMAKE_BINARY_DIR}/tests)

2
tests/python/collada/CMakeLists.txt
View File

@ -10,7 +10,7 @@
# and don't give deterministic results
set(USE_EXPERIMENTAL_TESTS FALSE)
set(TEST_SRC_DIR ${CMAKE_SOURCE_DIR}/../lib/tests)
set(TEST_SRC_DIR ${CMAKE_SOURCE_DIR}/tests/data)
set(TEST_OUT_DIR ${CMAKE_BINARY_DIR}/tests)
# ugh, any better way to do this on testing only?

3
tests/python/view_layer/CMakeLists.txt
View File

@ -1,3 +1,4 @@
tests/python/view_layer/CMakeLists.txt
# SPDX-FileCopyrightText: 2017-2023 Blender Authors
#
# SPDX-License-Identifier: GPL-2.0-or-later
@ -6,7 +7,7 @@
# Use '--write-blend=/tmp/test.blend' to view output