Add Chromium-only Blender WebEngine parity work

This commit is contained in:
mes123456
2026-08-12 04:47:48 -04:00
commit 9fd26010f6
18225 changed files with 11622124 additions and 0 deletions

View File

@@ -0,0 +1,8 @@
Make Utility Scripts
====================
Scripts used only by developers for now
Note: these scripts are assumed to be part of the deployment process, and thus
have to be able to run on older Python versions (3.6 at the moment of writing)
than the one bundled with Blender itself.

View File

@@ -0,0 +1,69 @@
#!/usr/bin/env python3
# SPDX-FileCopyrightText: 2026 Blender Authors
#
# SPDX-License-Identifier: GPL-2.0-or-later
"""
`make benchmark` helper
* Creates relevant directory if none exists and populates with BUILD_DIR binary
* Runs the default profile
"""
__all__ = {
"main"
}
import argparse
import sys
from pathlib import Path
from make_utils import call
from make_update import floating_checkout_update
def parse_arguments() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("build_directory")
parser.add_argument(
"--git-command",
default="git",
help="Path to the git binary. (Only useful if it is not in your PATH)")
return parser.parse_args()
def main() -> int:
args = parse_arguments()
msg = floating_checkout_update(
args,
"blender-benchmarks",
Path("tests") / "benchmarks",
"main",
)
if msg:
sys.stderr.write("Unable to initialize / update 'benchmark' repository: {}".format(msg))
return 1
benchmark_dir = Path(__file__).absolute().parent.joinpath("benchmark")
if not benchmark_dir.exists():
build_dir = Path(args.build_directory)
blender_bin = build_dir.joinpath("bin").absolute()
if not blender_bin.exists():
sys.stderr.write("blender `bin` directory not found, can't initialize benchmarks")
return 1
create_dir_command = ["./tests/performance/benchmark.py", "init", "--blender", blender_bin]
exitcode = call(create_dir_command)
if exitcode != 0:
return exitcode
run_command = ["./tests/performance/benchmark.py", "run", "default"]
return call(run_command)
if __name__ == "__main__":
sys.exit(main())

View File

@@ -0,0 +1,310 @@
#!/usr/bin/env python3
# SPDX-FileCopyrightText: 2022-2023 Blender Authors
#
# SPDX-License-Identifier: GPL-2.0-or-later
"""
Make Python wheel package (`*.whl`) file from Blender built with 'WITH_PYTHON_MODULE' enabled.
Example
=======
If the "bpy" module was build on Linux using the command:
make bpy lite
The command to package it as a wheel is:
./build_files/utils/make_bpy_wheel.py ../build_linux_bpy_lite/bin --output-dir=./
This will create a `*.whl` file in the current directory.
WARNING:
Python 3.9 is used on the built-bot.
Take care *not* to use features from the Python version used by Blender!
NOTE:
Some type annotations are quoted to avoid errors in Python 3.9.
These can be unquoted eventually.
"""
__all__ = (
"main",
)
import argparse
import make_utils
import os
import re
import platform
import string
import setuptools
import sys
from typing import (
Tuple,
# Proxies for `collections.abc`
Iterator,
Sequence,
)
# ------------------------------------------------------------------------------
# Long Description
long_description = """# Blender
[Blender](https://www.blender.org) is the free and open source 3D creation suite. It supports the entirety of the 3D pipeline: modeling, rigging, animation, simulation, rendering, compositing and motion tracking, even video editing.
This package provides Blender as a Python module for use in studio pipelines, web services, scientific research, and more.
### Archived Versions
Blender versions outside the current LTS window are removed from PyPI but are available at [https://download.blender.org/pypi/bpy/](https://download.blender.org/pypi/bpy/).
These versions can still be installed manually. For example, to install version 3.6.0:
```bash
pip install bpy==3.6.0 --extra-index-url https://download.blender.org/pypi/
```
## Documentation
* [Blender Python API](https://docs.blender.org/api/current/)
* [Blender as a Python Module](https://docs.blender.org/api/current/info_advanced_blender_as_bpy.html)
## Requirements
[System requirements](https://www.blender.org/download/requirements/) are the same as Blender.
Each Blender release supports one Python version, and the package is only compatible with that version.
## Source Code
* [Releases](https://download.blender.org/source/)
* Repository: [projects.blender.org/blender/blender.git](https://projects.blender.org/blender/blender)
## Credits
Created by the [Blender developer community](https://www.blender.org/about/credits/).
Thanks to Tyler Alden Gubala for maintaining the original version of this package."""
# ------------------------------------------------------------------------------
# Generic Functions
def find_dominating_file(
path: str,
search: Sequence[str],
) -> str:
while True:
for d in search:
if os.path.exists(os.path.join(path, d)):
return os.path.join(path, d)
path_next = os.path.normpath(os.path.join(path, ".."))
if path == path_next:
break
path = path_next
return ""
# ------------------------------------------------------------------------------
# CMake Cache Access
def cmake_cache_var_iter(filepath_cmake_cache: str) -> Iterator[Tuple[str, str, str]]:
re_cache = re.compile(r"([A-Za-z0-9_\-]+)?:?([A-Za-z0-9_\-]+)?=(.*)$")
with open(filepath_cmake_cache, "r", encoding="utf-8") as cache_file:
for l in cache_file:
match = re_cache.match(l.strip())
if match is not None:
var, type_, val = match.groups()
yield (var, type_ or "", val)
def cmake_cache_var(filepath_cmake_cache: str, var: str) -> "str | None":
for var_iter, _type_iter, value_iter in cmake_cache_var_iter(filepath_cmake_cache):
if var == var_iter:
return value_iter
return None
def cmake_cache_var_or_exit(filepath_cmake_cache: str, var: str) -> str:
value = cmake_cache_var(filepath_cmake_cache, var)
if value is None:
sys.stderr.write("Unable to find %r in %r, abort!\n" % (var, filepath_cmake_cache))
sys.exit(1)
return value
# ------------------------------------------------------------------------------
# Argument Parser
def argparse_create() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument(
"install_dir",
metavar='INSTALL_DIR',
type=str,
help="The installation directory containing the \"bpy\" package.",
)
parser.add_argument(
"--build-dir",
metavar='BUILD_DIR',
default=None,
help="The build directory containing 'CMakeCache.txt' (search parent directories of INSTALL_DIR when omitted).",
required=False,
)
parser.add_argument(
"--output-dir",
metavar='OUTPUT_DIR',
default=None,
help="The destination directory for the '*.whl' file (use INSTALL_DIR when omitted).",
required=False,
)
return parser
# ------------------------------------------------------------------------------
# Main Function
def main() -> None:
# NOTE: Import inline because the built-bot runs this with Python 3.6
# which fails to import `wheel`, the actual script should run with a newer Python.
# `wheel.bdist_wheel` is deprecated, `setuptools >= 70.1` includes it.
if tuple(int(x) for x in setuptools.__version__.split(".")[:2]) >= (70, 1):
from setuptools.command.bdist_wheel import bdist_wheel
else:
from wheel.bdist_wheel import bdist_wheel
# Parse arguments.
args = argparse_create().parse_args()
install_dir = os.path.abspath(args.install_dir)
output_dir = os.path.abspath(args.output_dir) if args.output_dir else install_dir
if args.build_dir:
build_dir = os.path.abspath(args.build_dir)
filepath_cmake_cache = os.path.join(build_dir, "CMakeCache.txt")
del build_dir
if not os.path.exists(filepath_cmake_cache):
sys.stderr.write("File not found %r, abort!\n" % filepath_cmake_cache)
sys.exit(1)
else:
filepath_cmake_cache = find_dominating_file(install_dir, ("CMakeCache.txt",))
if not filepath_cmake_cache:
# Should never fail.
sys.stderr.write("Unable to find CMakeCache.txt in or above %r, abort!\n" % install_dir)
sys.exit(1)
# Get the major and minor Python version.
python_version = cmake_cache_var_or_exit(filepath_cmake_cache, "PYTHON_VERSION")
python_version_number = (
tuple(int("".join(c for c in digit if c in string.digits)) for digit in python_version.split(".")) +
# Support version without a minor version "3" (add zero).
tuple((0, 0, 0))
)
# Get Blender version.
blender_version_str = str(make_utils.parse_blender_version())
# Set platform tag following conventions.
if sys.platform == "darwin":
target = cmake_cache_var_or_exit(filepath_cmake_cache, "CMAKE_OSX_DEPLOYMENT_TARGET").split(".")
# Minor version is expected to be always zero starting with macOS 11.
# https://github.com/pypa/packaging/issues/435
target_major = int(target[0])
target_minor = 0 # int(target[1])
machine = cmake_cache_var_or_exit(filepath_cmake_cache, "CMAKE_OSX_ARCHITECTURES")
platform_tag = "macosx_%d_%d_%s" % (target_major, target_minor, machine)
elif sys.platform == "win32":
# Workaround for Python process running in a virtualized environment on Windows-on-Arm:
# use the actual processor architecture instead of the virtualized one.
#
# The win_arm64 matches the behavior when native WoA Python is used, and also matches
# sysconfig.get_platform() from a native Python build (although it returns win-arm64 with a
# dash and not underscore).
if "ARM" in os.environ.get("PROCESSOR_IDENTIFIER", ""):
platform_tag = "win_arm64"
else:
platform_tag = "win_%s" % (platform.machine().lower())
elif sys.platform == "linux":
glibc = os.confstr("CS_GNU_LIBC_VERSION")
if glibc is None:
sys.stderr.write("Unable to find \"CS_GNU_LIBC_VERSION\", abort!\n")
sys.exit(1)
glibc = "%s_%s" % tuple(glibc.split()[1].split(".")[:2])
platform_tag = "manylinux_%s_%s" % (glibc, platform.machine().lower())
else:
sys.stderr.write("Unsupported platform: %s, abort!\n" % (sys.platform))
sys.exit(1)
# Manually specify, otherwise it uses the version of the executable used to run
# this script which may not match the Blender python version.
python_tag = "py%d%d" % (python_version_number[0], python_version_number[1])
cpython_tag = "cp%d%d" % (python_version_number[0], python_version_number[1])
os.chdir(install_dir)
# Include all files recursively.
def package_files(root_dir: str) -> list[str]:
paths = []
for path, dirs, files in os.walk(root_dir):
paths += [os.path.join("..", path, f) for f in files]
return paths
# Ensure this wheel is marked platform specific.
class BinaryDistribution(setuptools.dist.Distribution):
def has_ext_modules(self) -> bool:
return True
# NOTE: this class is needed because:
# - The Python used to build the wheel is the systems Python,
# so we can't rely on its "tag" matching Blender's.
# - There is no way to override the "tag" using options.
# If this is supported at some point, this class can be removed.
class TargetPythonBdistWheel(bdist_wheel):
def get_tag(self) -> Tuple[str, str, str]:
_python, _abi, plat = super().get_tag()
return cpython_tag, cpython_tag, plat
# Build wheel.
sys.argv = [sys.argv[0], "bdist_wheel"]
setuptools.setup(
name="bpy",
version=blender_version_str,
install_requires=["cattrs", "cython", "numpy>=2.2,<3.0", "requests", "zstandard"],
python_requires="==%d.%d.*" % (python_version_number[0], python_version_number[1]),
packages=["bpy"],
package_data={"": package_files("bpy")},
distclass=BinaryDistribution,
cmdclass={"bdist_wheel": TargetPythonBdistWheel},
options={"bdist_wheel": {"plat_name": platform_tag, "python_tag": python_tag}},
description="Blender as a Python module",
long_description=long_description,
long_description_content_type='text/markdown',
license="GPL-3.0",
author="Blender Foundation",
url="https://www.blender.org"
)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# Move wheel to output directory.
dist_dir = os.path.join(install_dir, "dist")
for f in os.listdir(dist_dir):
if f.endswith(".whl"):
# The wheel is already tagged correctly (cpXY-cpXY-plat) by TargetPythonBdistWheel,
# so only move it to the output directory.
os.rename(os.path.join(dist_dir, f), os.path.join(output_dir, f))
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,318 @@
#!/usr/bin/env python3
# SPDX-FileCopyrightText: 2021-2023 Blender Authors
#
# SPDX-License-Identifier: GPL-2.0-or-later
__all__ = (
"main",
)
import argparse
import make_utils
import os
import subprocess
import sys
from pathlib import Path
from typing import (
TextIO,
Any,
Tuple,
Union,
# Proxies for `collections.abc`
Iterable,
List,
)
# This script can run from any location,
# output is created in the $CWD
#
# NOTE: while the Python part of this script is portable,
# it relies on external commands typically found on GNU/Linux.
# Support for other platforms could be added by moving GNU `tar` & `md5sum` use to Python.
# This also relies on having a Unix shell (sh) to run some git commands.
SKIP_NAMES: Tuple[str, ...] = (
".gitignore",
".gitmodules",
".gitattributes",
".git-blame-ignore-revs",
".arcconfig",
".svn",
)
# Tuple with folder names relative to the main repository root that are to be excluded.
SKIP_FOLDERS: Tuple[str, ...] = (
)
# Generated list of Paths to be ignored based on the SKIP_FOLDERS and some rum-time rules like
# the type of package that is being created.
SKIP_PATHS: List[Path] = []
def main() -> None:
blender_srcdir = Path(__file__).absolute().parent.parent.parent
cli_parser = argparse.ArgumentParser(
description="Create a tarball of the Blender sources, optionally including sources of dependencies.",
epilog="This script is intended to be run by `make source_archive_complete`.",
)
group = cli_parser.add_mutually_exclusive_group()
group.add_argument(
"-p",
"--include-packages",
type=Path,
default=None,
metavar="PACKAGE_PATH",
help="Include all source files from the given package directory as well.",
)
group.add_argument(
"-t",
"--package-test-data",
action='store_true',
help="Package all test data into its own archive",
)
cli_args = cli_parser.parse_args()
print(f"Source dir: {blender_srcdir}")
curdir = blender_srcdir.parent
os.chdir(curdir)
blender_srcdir = blender_srcdir.relative_to(curdir)
# Update our SKIP_FOLDERS blacklist with the source directory name
global SKIP_PATHS
SKIP_PATHS = [blender_srcdir / entry for entry in SKIP_FOLDERS]
print(f"Output dir: {curdir}")
version = make_utils.parse_blender_version()
tarball = tarball_path(curdir, version, cli_args)
manifest = manifest_path(tarball)
packages_dir = packages_path(curdir, cli_args)
if cli_args.package_test_data:
print("Creating an archive of all test data.")
create_manifest(version, manifest, blender_srcdir / "tests/files", packages_dir)
else:
SKIP_PATHS.append(blender_srcdir / "tests/files")
create_manifest(version, manifest, blender_srcdir, packages_dir)
create_tarball(version, tarball, manifest, blender_srcdir, packages_dir)
create_checksum_file(tarball)
cleanup(manifest)
print("Done!")
def tarball_path(output_dir: Path, version: make_utils.BlenderVersion, cli_args: Any) -> Path:
extra = ""
if cli_args.include_packages:
extra = "-with-libraries"
elif cli_args.package_test_data:
extra = "-test-data"
return output_dir / f"blender{extra}-{version}.tar.xz"
def manifest_path(tarball: Path) -> Path:
"""Return the manifest path for the given tarball path.
>>> from pathlib import Path
>>> tarball = Path("/home/user/workspace/blender-git/blender-test.tar.gz")
>>> manifest_path(tarball).as_posix()
'/home/user/workspace/blender-git/blender-test-manifest.txt'
"""
# Note that `.tar.gz` is seen as two suffixes.
without_suffix = tarball.with_suffix("").with_suffix("")
name = without_suffix.name
return without_suffix.with_name(f"{name}-manifest.txt")
def packages_path(current_directory: Path, cli_args: Any) -> Union[Path, None]:
if not cli_args.include_packages:
return None
abspath = cli_args.include_packages.absolute()
# `os.path.relpath()` can return paths like "../../packages", where
# `Path.relative_to()` will not go up directories (so its return value never
# has "../" in there).
relpath = os.path.relpath(abspath, current_directory)
return Path(relpath)
# -----------------------------------------------------------------------------
# Manifest creation
def create_manifest(
version: make_utils.BlenderVersion,
outpath: Path,
blender_srcdir: Path,
packages_dir: Union[Path, None],
exclude: List[Path] = []
) -> None:
print(f'Building manifest of files: "{outpath}"...', end="", flush=True)
with outpath.open("w", encoding="utf-8") as outfile:
main_files_to_manifest(blender_srcdir, outfile)
if packages_dir:
packages_to_manifest(outfile, packages_dir)
print("OK")
def main_files_to_manifest(blender_srcdir: Path, outfile: TextIO) -> None:
assert not blender_srcdir.is_absolute()
for git_repo in git_gather_all_folders_to_package(blender_srcdir):
for path in git_ls_files(git_repo):
print(path, file=outfile)
def packages_to_manifest(outfile: TextIO, packages_dir: Path) -> None:
for path in packages_dir.glob("*"):
if not path.is_file():
continue
if path.name in SKIP_NAMES:
continue
print(path, file=outfile)
# -----------------------------------------------------------------------------
# Higher-level functions
def create_tarball(
version: make_utils.BlenderVersion,
tarball: Path,
manifest: Path,
blender_srcdir: Path,
packages_dir: Union[Path, None],
) -> None:
print(f'Creating archive: "{tarball}" ...', end="", flush=True)
# Requires GNU `tar`, since `--transform` is used.
if sys.platform == "darwin":
# Provided by `brew install gnu-tar`.
command = ["gtar"]
else:
command = ["tar"]
if packages_dir:
command += ["--transform", f"s,{packages_dir}/,packages/,g"]
command += [
"--transform",
f"s,^{blender_srcdir.name}/,blender-{version}/,g",
"--use-compress-program=xz -1",
"--create",
f"--file={tarball}",
f"--files-from={manifest}",
# Without owner/group args, extracting the files as root will
# use ownership from the tar archive:
"--owner=0",
"--group=0",
]
subprocess.run(command, check=True, timeout=3600)
print("OK")
def create_checksum_file(tarball: Path) -> None:
md5_path = tarball.with_name(tarball.name + ".md5sum")
print(f'Creating checksum: "{md5_path}" ...', end="", flush=True)
command = [
"md5sum",
# The name is enough, as the tarball resides in the same dir as the MD5
# file, and that's the current working directory.
tarball.name,
]
md5_cmd = subprocess.run(
command, stdout=subprocess.PIPE, check=True, text=True, timeout=300
)
with md5_path.open("w", encoding="utf-8") as outfile:
outfile.write(md5_cmd.stdout)
print("OK")
def cleanup(manifest: Path) -> None:
print("Cleaning up ...", end="", flush=True)
if manifest.exists():
manifest.unlink()
print("OK")
# -----------------------------------------------------------------------------
# Low-level commands
def git_gather_all_folders_to_package(directory: Path = Path(".")) -> Iterable[Path]:
"""Generator, yields lines which represents each directory to gather git files from.
Each directory represents either the top level git repository or a submodule.
All submodules that have the 'update = none' setting will be excluded from this list.
The directory path given to this function will be included in the yielded paths
"""
# For each submodule (recurse into submodules within submodules if they exist)
git_main_command = "submodule --quiet foreach --recursive"
# Return the path to the submodule and what the value is of their "update" setting
# If the "update" setting doesn't exist, only the path to the submodule is returned
git_command_args = "'echo $displaypath $(git config --file \"$toplevel/.gitmodules\" --get submodule.$name.update)'"
# Yield the root directory as this is our top level git repo
yield directory
for line in git_command(f"-C '{directory}' {git_main_command} {git_command_args}"):
# Check if we shouldn't include the directory on this line
split_line = line.rsplit(maxsplit=1)
if len(split_line) > 1 and split_line[-1] == "none":
continue
path = directory / split_line[0]
yield path
def is_path_ignored(file: Path) -> bool:
for skip_folder in SKIP_PATHS:
if file.is_relative_to(skip_folder):
return True
return False
def git_ls_files(directory: Path = Path(".")) -> Iterable[Path]:
"""Generator, yields lines of output from 'git ls-files'.
Only lines that are actually files (so no directories, sockets, etc.) are
returned, and never one from SKIP_NAMES.
"""
for line in git_command(f"-C '{directory}' ls-files -z", "\x00"):
path = directory / line
if not path.is_file() or path.name in SKIP_NAMES:
continue
if not is_path_ignored(path):
yield path
def git_command(cli_args: str, split_char: str = "\n") -> Iterable[str]:
"""Generator, yields lines of output from a Git command."""
command = "git " + cli_args
# import shlex
# print(">", " ".join(shlex.quote(arg) for arg in command))
git = subprocess.run(
command, stdout=subprocess.PIPE, shell=True, check=True, text=True, timeout=30
)
for line in git.stdout.split(split_char):
if line:
yield line
if __name__ == "__main__":
import doctest
if doctest.testmod().failed:
raise SystemExit("ERROR: Self-test failed, refusing to run")
main()

View File

@@ -0,0 +1,64 @@
#!/usr/bin/env python3
# SPDX-FileCopyrightText: 2019-2023 Blender Authors
#
# SPDX-License-Identifier: GPL-2.0-or-later
"""
"make test" for all platforms, running automated tests.
"""
__all__ = (
"main",
)
import argparse
import os
import sys
import make_utils
from make_utils import call
# Parse arguments.
def parse_arguments() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--ctest-command", default="ctest")
parser.add_argument("--git-command", default="git")
parser.add_argument("--config", default="")
parser.add_argument("build_directory")
return parser.parse_args()
def main() -> int:
args = parse_arguments()
git_command = args.git_command
ctest_command = args.ctest_command
config = args.config
build_dir = args.build_directory
if make_utils.command_missing(ctest_command):
sys.stderr.write("ctest not found, can't run tests\n")
return 1
if make_utils.command_missing(git_command):
sys.stderr.write("git not found, can't run tests\n")
return 1
# Run tests
tests_dir = os.path.join(build_dir, "tests")
os.makedirs(tests_dir, exist_ok=True)
os.chdir(build_dir)
command = [ctest_command, ".", "--output-on-failure"]
if len(config):
command += ["-C", config]
tests_log = "log_" + config + ".txt"
else:
tests_log = "log.txt"
command += ["-O", os.path.join(tests_dir, tests_log)]
call(command)
return 0
if __name__ == "__main__":
sys.exit(main())

View File

@@ -0,0 +1,733 @@
#!/usr/bin/env python3
# SPDX-FileCopyrightText: 2019-2023 Blender Authors
#
# SPDX-License-Identifier: GPL-2.0-or-later
"""
"make update" for all platforms, updating Git LFS submodules for libraries and
Blender git repository.
For release branches, this will check out the appropriate branches of
submodules and libraries.
WARNING:
- Python 3.6 is used on the Linux VM (Rocky8) to run "make update" to checkout LFS.
- Python 3.9 is used on the built-bot.
Take care *not* to use features from the Python version used by Blender!
NOTE:
Some type annotations are quoted to avoid errors in older Python versions.
These can be unquoted eventually.
"""
__all__ = (
"main",
"floating_checkout_update",
)
import argparse
import os
import platform
import shutil
import sys
import make_utils
from pathlib import Path
from make_utils import call, check_output
from urllib.parse import urljoin, urlsplit
def print_stage(text: str) -> None:
print("")
print(text)
print("=" * len(text))
print("")
def parse_arguments() -> argparse.Namespace:
"""
Parse command line arguments.
Returns parsed object from which the command line arguments can be accessed
as properties. The name of the properties matches the command line argument,
but with the leading dashed omitted and all remaining dashes replaced with
underscore.
"""
parser = argparse.ArgumentParser()
parser.add_argument("--no-libraries", action="store_true",
help="Don't fetch precompiled libraries for this system")
parser.add_argument("--no-blender", action="store_true", help="Don't update the Blender code repository")
parser.add_argument(
"--no-lfs-fallback",
action="store_true",
help="Don't set up fallback URLs for fetching LFS files from projects.blender.org. These are only used when cloning repositories hosted elsewhere.")
parser.add_argument(
"--git-command",
default="git",
help="Path to the git binary. (Only useful if it is not in your PATH)")
parser.add_argument("--architecture", type=str,
choices=("x86_64", "amd64", "arm64",))
parser.add_argument("--prune-destructive", action="store_true",
help="Destructive! Detect and remove stale files from older checkouts")
# Deprecated options, kept for compatibility with old configurations.
parser.add_argument("--use-tests", action="store_true", help=argparse.SUPPRESS)
parser.add_argument("--no-submodules", action="store_true", help=argparse.SUPPRESS)
parser.add_argument("--use-linux-libraries", action="store_true", help=argparse.SUPPRESS)
return parser.parse_args()
def get_blender_git_root(args: argparse.Namespace) -> Path:
"""
Get root directory of the current Git directory.
"""
return Path(
check_output([args.git_command, "rev-parse", "--show-toplevel"]))
def get_effective_platform(args: argparse.Namespace) -> str:
"""
Get platform of the host.
The result string is normalized to the name used by Blender releases and
library repository name prefixes: linux, macos, windows.
"""
if sys.platform == "darwin":
platform = "macos"
elif sys.platform == "win32":
platform = "windows"
else:
platform = sys.platform
assert (platform in ("linux", "macos", "windows"))
return platform
def get_effective_architecture(args: argparse.Namespace) -> str:
"""
Get architecture of the host.
The result string is normalized to the architecture name used by the Blender
releases and library repository name suffixes: x64, arm64.
NOTE: When cross-compiling the architecture is coming from the command line
argument.
"""
architecture: "str | None" = args.architecture
if architecture:
assert isinstance(architecture, str)
elif "ARM64" in platform.version():
# Check platform.version to detect arm64 with x86_64 python binary.
architecture = "arm64"
else:
architecture = platform.machine().lower()
# Normalize the architecture name.
if architecture in {"x86_64", "amd64"}:
architecture = "x64"
if architecture == "aarch64":
architecture = "arm64"
assert isinstance(architecture, str)
return architecture
# NOTE: unquote "tuple" once Python 3.6x is dropped.
def get_submodule_directories(args: argparse.Namespace) -> "tuple[Path, ...]":
"""
Get list of all configured submodule directories.
"""
blender_git_root = get_blender_git_root(args)
dot_modules = blender_git_root / ".gitmodules"
if not dot_modules.exists():
return ()
submodule_directories_output = check_output(
[args.git_command, "config", "--file", str(dot_modules), "--get-regexp", "path"])
return tuple([Path(line.split(' ', 1)[1]) for line in submodule_directories_output.strip().splitlines()])
def ensure_git_lfs(args: argparse.Namespace) -> None:
# Use `--skip-repo` to avoid creating git hooks.
# This is called from the `blender.git` checkout, so we don't need to install hooks there.
call((args.git_command, "lfs", "install", "--skip-repo"), exit_on_error=True)
def switch_blender_git_remotes(args: argparse.Namespace) -> None:
"""
Switch remote URLs from projects.blender.org to git.blender.org
"""
remotes = make_utils.git_get_remotes(args.git_command)
for remote in remotes:
url = make_utils.git_get_remote_url(args.git_command, remote)
new_url = url.replace("git@projects.blender.org", "git@git.blender.org")
if new_url == url:
continue
print(f"Replacing {remote} URL from {url} to {new_url}")
make_utils.git_set_config(args.git_command, f"remote.{remote}.url", new_url)
def prune_stale_files(args: argparse.Namespace) -> None:
"""
Ensure files from previous Git configurations do not exist anymore
"""
print_stage("Removing stale files")
blender_git_root = get_blender_git_root(args)
found_stale_files = False
for relative_dir_to_remove in (
Path("scripts") / "addons",
Path("scripts") / "addons_contrib",
):
dir_to_remove = blender_git_root / relative_dir_to_remove
if not dir_to_remove.exists():
continue
if not dir_to_remove.is_dir():
print(f"'{relative_dir_to_remove}' exists but is not a directory")
continue
print(f"Removing '{relative_dir_to_remove}'")
make_utils.remove_directory(dir_to_remove)
found_stale_files = True
if not found_stale_files:
print("Checkout looks pristine")
def initialize_precompiled_libraries(args: argparse.Namespace) -> str:
"""
Configure submodule for precompiled libraries
This function detects the current host architecture and enables
corresponding submodule, and updates the submodule.
NOTE: When cross-compiling the architecture is coming from the command line
argument.
"""
print_stage("Configuring Precompiled Libraries")
platform = get_effective_platform(args)
arch = get_effective_architecture(args)
print(f"Detected platform : {platform}")
print(f"Detected architecture : {arch}")
print()
submodule_dir = f"lib/{platform}_{arch}"
submodule_directories = get_submodule_directories(args)
if platform == "macos" and arch == "x64":
return ("WARNING: macOS x64/Intel support was dropped in Blender 5.0.\n"
" As such, pre-compiled dependencies are no longer provided.\n"
" You may build the dependencies yourself, or downgrade to Blender 4.5.\n"
" For more details, please see: https://devtalk.blender.org/t/38835")
if Path(submodule_dir) not in submodule_directories:
return "Skipping libraries update: no configured submodule\n"
print(f"* Enabling precompiled libraries at {submodule_dir}")
make_utils.git_enable_submodule(args.git_command, Path(submodule_dir))
return ""
def git_update_skip(args: argparse.Namespace, check_remote_exists: bool = True) -> str:
"""Test if git repo can be updated."""
if make_utils.command_missing(args.git_command):
sys.stderr.write("git not found, can't update code\n")
sys.exit(1)
# Abort if a rebase is still progress.
rebase_merge = check_output([args.git_command, 'rev-parse', '--git-path', 'rebase-merge'], exit_on_error=False)
rebase_apply = check_output([args.git_command, 'rev-parse', '--git-path', 'rebase-apply'], exit_on_error=False)
merge_head = check_output([args.git_command, 'rev-parse', '--git-path', 'MERGE_HEAD'], exit_on_error=False)
if (
os.path.exists(rebase_merge) or
os.path.exists(rebase_apply) or
os.path.exists(merge_head)
):
return "rebase or merge in progress, complete it first"
# Abort if uncommitted changes.
changes = check_output([args.git_command, 'status', '--porcelain', '--untracked-files=no', '--ignore-submodules'])
if len(changes) != 0:
return "you have unstaged changes"
# Test if there is an upstream branch configured
if check_remote_exists:
branch = check_output([args.git_command, "rev-parse", "--abbrev-ref", "HEAD"])
remote = check_output([args.git_command, "config", "branch." + branch + ".remote"], exit_on_error=False)
if len(remote) == 0:
return "no remote branch to pull from"
return ""
def use_upstream_workflow(args: argparse.Namespace) -> bool:
return make_utils.git_remote_exist(args.git_command, "upstream")
def work_tree_update_upstream_workflow(args: argparse.Namespace, use_fetch: bool = True) -> str:
"""
Update the Blender repository using the GitHub style of fork organization
Returns true if the current local branch has been updated to the upstream state.
Otherwise false is returned.
"""
branch_name = make_utils.git_branch(args.git_command)
if use_fetch:
call((args.git_command, "fetch", "upstream"))
upstream_branch = f"upstream/{branch_name}"
if not make_utils.git_branch_exists(args.git_command, upstream_branch):
return "no_branch"
retcode = call((args.git_command, "merge", "--ff-only", upstream_branch), exit_on_error=False)
if retcode != 0:
return "Unable to fast forward\n"
return ""
def work_tree_update(args: argparse.Namespace, use_fetch: bool = True) -> str:
"""
Update the Git working tree using the best strategy
This function detects whether it is a github style of fork remote organization is used, or
is it a repository which origin is an upstream.
"""
if use_upstream_workflow(args):
message = work_tree_update_upstream_workflow(args, use_fetch)
if message != "no_branch":
return message
# If there is upstream configured but the local branch is not in the upstream, try to
# update the branch from the fork.
update_command = [args.git_command, "pull", "--rebase"]
call(update_command)
return ""
# Update blender repository.
def blender_update(args: argparse.Namespace) -> str:
print_stage("Updating Blender Git Repository")
return work_tree_update(args)
# Extra LFS update for blender repository
def blender_lfs_update(args: argparse.Namespace) -> None:
print_stage("Updating Blender Git LFS")
# This seems to be required some times, e.g. on initial checkout from third party, non-lfs repository
# (like the github one). The fallback repository set by `lfs_fallback_setup` is fetched, but running the
# `update_command` above does not seem to do the actual checkout for these LFS-managed files.
update_lfs_command = [args.git_command, "lfs", "pull"]
call(update_lfs_command)
def resolve_external_url(blender_url: str, repo_name: str) -> str:
return urljoin(blender_url + "/", "../" + repo_name)
def external_script_copy_old_submodule_over(
args: argparse.Namespace,
directory: Path,
old_submodules_dir: Path,
) -> None:
blender_git_root = get_blender_git_root(args)
external_dir = blender_git_root / directory
print(f"Moving {old_submodules_dir} to {directory} ...")
shutil.move(blender_git_root / old_submodules_dir, external_dir)
# Remove old ".git" which is a file with path to a submodule bare repo inside of main
# repo .git/modules directory.
(external_dir / ".git").unlink()
bare_repo_relative_dir = Path(".git") / "modules" / old_submodules_dir
print(f"Copying {bare_repo_relative_dir} to {directory}/.git ...")
bare_repo_dir = blender_git_root / bare_repo_relative_dir
shutil.copytree(bare_repo_dir, external_dir / ".git")
git_config = external_dir / ".git" / "config"
call((args.git_command, "config", "--file", str(git_config), "--unset", "core.worktree"))
def floating_checkout_initialize_if_needed(
args: argparse.Namespace,
repo_name: str,
directory: Path,
old_submodules_dir: "Path | None" = None,
) -> None:
"""Initialize checkout of an external repository"""
blender_git_root = get_blender_git_root(args)
blender_dot_git = blender_git_root / ".git"
external_dir = blender_git_root / directory
if external_dir.exists():
return
print(f"Initializing {directory} ...")
if old_submodules_dir is not None:
old_submodule_dot_git = blender_git_root / old_submodules_dir / ".git"
if old_submodule_dot_git.exists() and blender_dot_git.is_dir():
external_script_copy_old_submodule_over(args, directory, old_submodules_dir)
return
origin_name = "upstream" if use_upstream_workflow(args) else "origin"
blender_url = make_utils.git_get_remote_url(args.git_command, origin_name)
external_url = resolve_external_url(blender_url, repo_name)
# When running `make update` from a freshly cloned fork check whether the fork of the submodule is
# available, If not, switch to the submodule relative to the main blender repository.
if origin_name == "origin" and not make_utils.git_is_remote_repository(args.git_command, external_url):
external_url = resolve_external_url("https://projects.blender.org/blender/blender", repo_name)
call((args.git_command, "clone", "--origin", origin_name, external_url, str(external_dir)))
def floating_checkout_add_origin_if_needed(
args: argparse.Namespace,
repo_name: str,
directory: Path,
) -> None:
"""
Add remote called 'origin' if there is a fork of the external repository available
This is only done when using Github style upstream workflow in the main repository.
"""
if not use_upstream_workflow(args):
return
cwd = os.getcwd()
blender_git_root = get_blender_git_root(args)
external_dir = blender_git_root / directory
origin_blender_url = make_utils.git_get_remote_url(args.git_command, "origin")
origin_external_url = resolve_external_url(origin_blender_url, repo_name)
try:
os.chdir(external_dir)
if (make_utils.git_remote_exist(args.git_command, "origin") or
not make_utils.git_remote_exist(args.git_command, "upstream")):
return
if not make_utils.git_is_remote_repository(args.git_command, origin_external_url):
return
print(f"Adding origin remote to {directory} pointing to fork ...")
# Non-obvious tricks to introduce the new remote called "origin" to the existing
# submodule configuration.
#
# This is all within the content of creating a fork of a submodule after `make update`
# has been run and possibly local branches tracking upstream were added.
#
# The idea here goes as following:
#
# - Rename remote "upstream" to "origin", which takes care of changing the names of
# remotes the local branches are tracking.
#
# - Change the URL to the "origin", which was still pointing to upstream.
#
# - Re-introduce the "upstream" remote, with the same URL as it had prior to rename.
upstream_url = make_utils.git_get_remote_url(args.git_command, "upstream")
call((args.git_command, "remote", "rename", "upstream", "origin"))
make_utils.git_set_config(args.git_command, "remote.origin.url", origin_external_url)
call((args.git_command, "remote", "add", "upstream", upstream_url))
finally:
os.chdir(cwd)
return
def floating_checkout_update(
args: argparse.Namespace,
repo_name: str,
directory: Path,
branch: "str | None",
old_submodules_dir: "Path | None" = None,
only_update: bool = False,
) -> str:
"""Update a single external checkout with the given name in the scripts folder"""
blender_git_root = get_blender_git_root(args)
external_dir = blender_git_root / directory
if only_update and not external_dir.exists():
return ""
floating_checkout_initialize_if_needed(args, repo_name, directory, old_submodules_dir)
floating_checkout_add_origin_if_needed(args, repo_name, directory)
blender_git_root = get_blender_git_root(args)
external_dir = blender_git_root / directory
print(f"* Updating {directory} ...")
cwd = os.getcwd()
# Update externals to appropriate given branch, falling back to main if none is given and/or
# found in a sub-repository.
branch_fallback = "main"
if not branch:
branch = branch_fallback
skip_msg = ""
try:
os.chdir(external_dir)
msg = git_update_skip(args, check_remote_exists=False)
if msg:
skip_msg += str(directory) + " skipped: " + msg + "\n"
else:
# Find a matching branch that exists.
for remote in ("origin", "upstream"):
if make_utils.git_remote_exist(args.git_command, remote):
call([args.git_command, "fetch", remote])
submodule_branch = branch
if make_utils.git_branch_exists(args.git_command, submodule_branch):
pass
elif make_utils.git_branch_exists(args.git_command, branch_fallback):
submodule_branch = branch_fallback
else:
# Skip.
submodule_branch = ""
# Switch to branch and pull.
if submodule_branch:
if make_utils.git_branch(args.git_command) != submodule_branch:
# If the local branch exists just check out to it.
# If there is no local branch but only remote specify an explicit remote.
# Without this explicit specification Git attempts to set-up tracking
# automatically and fails when the branch is available in multiple remotes.
if make_utils.git_local_branch_exists(args.git_command, submodule_branch):
call([args.git_command, "checkout", submodule_branch])
else:
if make_utils.git_remote_branch_exists(args.git_command, "origin", submodule_branch):
call([args.git_command, "checkout", "-t", f"origin/{submodule_branch}"])
elif make_utils.git_remote_exist(args.git_command, "upstream"):
# For the Github style of upstream workflow create a local branch from
# the upstream, but do not track it, so that we stick to the paradigm
# that no local branches are tracking upstream, preventing possible
# accidental commit to upstream.
call([args.git_command, "checkout", "-b", submodule_branch,
f"upstream/{submodule_branch}", "--no-track"])
# Don't use extra fetch since all remotes of interest have been already fetched
# some lines above.
skip_msg += work_tree_update(args, use_fetch=False)
finally:
os.chdir(cwd)
return skip_msg
def floating_libraries_update(args: argparse.Namespace, branch: "str | None") -> str:
"""Update libraries checkouts which are floating (not attached as Git submodules)"""
msg = ""
msg += floating_checkout_update(
args,
"benchmarks",
Path("tests") / "benchmarks",
branch,
only_update=True,
)
return msg
def add_submodule_push_url(args: argparse.Namespace) -> None:
"""
Add pushURL configuration for all locally activated submodules, pointing to SSH protocol.
"""
blender_git_root = get_blender_git_root(args)
modules = blender_git_root / ".git" / "modules"
submodule_directories = get_submodule_directories(args)
for submodule_path in submodule_directories:
module_path = modules / submodule_path
config = module_path / "config"
if not config.exists():
# Ignore modules which are not initialized
continue
push_url = check_output((args.git_command, "config", "--file", str(config),
"--get", "remote.origin.pushURL"), exit_on_error=False)
# Don't modify PushURL if it is set.
if push_url:
if "projects.blender.org" in push_url:
# Allow the code below to replace projects.blender.org with git.blender.org
pass
else:
continue
url = make_utils.git_get_config(args.git_command, "remote.origin.url", str(config))
if not url.startswith("https:"):
# Ignore non-URL URLs.
continue
url_parts = urlsplit(url)
host = url_parts.hostname
if host == "projects.blender.org":
host = "git.blender.org"
push_url = f"git@{host}:{url_parts.path[1:]}"
print(f"Setting pushURL to {push_url} for {submodule_path}")
make_utils.git_set_config(args.git_command, "remote.origin.pushURL", push_url, str(config))
def submodules_lib_update(args: argparse.Namespace, branch: "str | None") -> str:
print_stage("Updating Libraries")
msg = ""
msg += floating_libraries_update(args, branch)
submodule_directories = get_submodule_directories(args)
for submodule_path in submodule_directories:
if not make_utils.is_git_submodule_enabled(args.git_command, submodule_path):
print(f"* Skipping {submodule_path}")
continue
print(f"* Updating {submodule_path} ...")
if not make_utils.git_update_submodule(args.git_command, submodule_path):
msg += f"Error updating Git submodule {submodule_path}\n"
add_submodule_push_url(args)
return msg
def lfs_fallback_setup(args: argparse.Namespace) -> None:
"""
Set up an additional projects.blender.org remote, for LFS fetching fallback
in case the fork does not include LFS files.
"""
remotes = make_utils.git_get_remotes(args.git_command)
add_fallback_remote = True
fallback_remote = "lfs-fallback"
for remote in remotes:
url = make_utils.git_get_remote_url(args.git_command, remote)
if "projects.blender.org" not in url and "git.blender.org" not in url:
make_utils.git_set_config(args.git_command, "lfs.remote.searchall", "true")
else:
add_fallback_remote = False
if add_fallback_remote and not make_utils.git_remote_exist(args.git_command, fallback_remote):
print_stage("Adding Git LFS fallback remote")
print("Used to fetch files from projects.blender.org if missing.")
url = "https://projects.blender.org/blender/blender.git"
push_url = "no_push"
make_utils.git_add_remote(args.git_command, fallback_remote, url, push_url)
# Fetch potentially missing files.
call((args.git_command, "lfs", "fetch", fallback_remote))
def main() -> int:
args = parse_arguments()
blender_skip_msg = ""
libraries_skip_msg = ""
blender_version = make_utils. parse_blender_version()
if blender_version.cycle != 'alpha':
major = blender_version.version // 100
minor = blender_version.version % 100
branch = f"blender-v{major}.{minor}-release"
else:
branch = 'main'
# Submodules and precompiled libraries require Git LFS.
ensure_git_lfs(args)
switch_blender_git_remotes(args)
if args.prune_destructive:
prune_stale_files(args)
if not args.no_lfs_fallback:
lfs_fallback_setup(args)
if not args.no_blender:
blender_skip_msg = git_update_skip(args)
if not blender_skip_msg:
blender_skip_msg = blender_update(args)
if blender_skip_msg:
blender_skip_msg = "Blender repository skipped: " + blender_skip_msg + "\n"
blender_lfs_update(args)
if not args.no_libraries:
libraries_skip_msg += initialize_precompiled_libraries(args)
libraries_skip_msg += submodules_lib_update(args, branch)
# Report any skipped repositories at the end, so it's not as easy to miss.
skip_msg = blender_skip_msg + libraries_skip_msg
if skip_msg:
print_stage("Update finished with the following messages")
print(skip_msg.strip())
if args.use_tests:
print()
print('NOTE: --use-tests is a deprecated command line argument, kept for compatibility purposes.')
if args.no_submodules:
print()
print('NOTE: --no-submodules is a deprecated command line argument, kept for compatibility purposes.')
if args.use_linux_libraries:
print()
print('NOTE: --use-linux-libraries is a deprecated command line argument, kept for compatibility purposes.')
# For failed library update we throw an error, since not having correct
# libraries can make Blender throw errors.
# For Blender itself we don't and consider "make update" to be a command
# you can use while working on uncommitted code.
if libraries_skip_msg:
return 1
return 0
if __name__ == "__main__":
sys.exit(main())

View File

@@ -0,0 +1,350 @@
#!/usr/bin/env python3
# SPDX-FileCopyrightText: 2019-2023 Blender Authors
#
# SPDX-License-Identifier: GPL-2.0-or-later
"""
Utility functions for make update and make tests
WARNING:
- Python 3.6 is used on the Linux VM (Rocky8) to run "make update" to checkout LFS.
- Python 3.9 is used on the built-bot.
Take care *not* to use features from the Python version used by Blender!
NOTE:
Some type annotations are quoted to avoid errors in older Python versions.
These can be unquoted eventually.
"""
__all__ = (
"call",
"check_output",
"command_missing",
"git_branch",
"git_branch_exists",
"git_enable_submodule",
"git_get_remote_url",
"git_is_remote_repository",
"git_remote_exist",
"git_set_config",
"git_update_submodule",
"is_git_submodule_enabled",
"parse_blender_version",
"remove_directory",
)
import re
import os
import shutil
import stat
import subprocess
import sys
from pathlib import Path
from types import (
TracebackType,
)
from typing import (
Any,
)
if sys.version_info >= (3, 9):
from collections.abc import (
Callable,
Sequence,
)
else:
from typing import (
Callable,
Sequence,
)
def call(
cmd: Sequence[str],
exit_on_error: bool = True,
silent: bool = False,
env: "dict[str, str] | None" = None,
) -> int:
if not silent:
cmd_str = ""
if env:
cmd_str += " ".join([f"{item[0]}={item[1]}" for item in env.items()])
cmd_str += " "
cmd_str += " ".join([str(x) for x in cmd])
print(cmd_str)
env_full = None
if env:
env_full = os.environ.copy()
for key, value in env.items():
env_full[key] = value
# Flush to ensure correct order output on Windows.
sys.stdout.flush()
sys.stderr.flush()
if silent:
retcode = subprocess.call(
cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=env_full)
else:
retcode = subprocess.call(cmd, env=env_full)
if exit_on_error and retcode != 0:
sys.exit(retcode)
return retcode
def check_output(cmd: Sequence[str], exit_on_error: bool = True) -> str:
# Flush to ensure correct order output on Windows.
sys.stdout.flush()
sys.stderr.flush()
try:
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, universal_newlines=True)
except subprocess.CalledProcessError as e:
if exit_on_error:
sys.stderr.write(" ".join(cmd) + "\n")
sys.stderr.write(e.output + "\n")
sys.exit(e.returncode)
output = ""
return output.strip()
def git_local_branch_exists(git_command: str, branch: str) -> bool:
return (
call([git_command, "rev-parse", "--verify", branch], exit_on_error=False, silent=True) == 0
)
def git_remote_branch_exists(git_command: str, remote: str, branch: str) -> bool:
return call([git_command, "rev-parse", "--verify", f"remotes/{remote}/{branch}"],
exit_on_error=False, silent=True) == 0
def git_branch_exists(git_command: str, branch: str) -> bool:
return (
git_local_branch_exists(git_command, branch) or
git_remote_branch_exists(git_command, "upstream", branch) or
git_remote_branch_exists(git_command, "origin", branch)
)
def git_get_remote_url(git_command: str, remote_name: str) -> str:
return check_output((git_command, "ls-remote", "--get-url", remote_name))
def git_remote_exist(git_command: str, remote_name: str) -> bool:
"""Check whether there is a remote with the given name"""
# `git ls-remote --get-url upstream` will print an URL if there is such remote configured, and
# otherwise will print "upstream".
remote_url = check_output((git_command, "ls-remote", "--get-url", remote_name))
return remote_url != remote_name
def git_is_remote_repository(git_command: str, repo: str) -> bool:
"""Returns true if the given repository is a valid/clonable git repo"""
exit_code = call((git_command, "ls-remote", repo, "HEAD"), exit_on_error=False, silent=True)
return exit_code == 0
def git_get_remotes(git_command: str) -> Sequence[str]:
"""Get a list of git remotes"""
# Additional check if the remote exists, for safety in case the output of this command
# changes in the future.
remotes = check_output([git_command, "remote"]).split()
return [remote for remote in remotes if git_remote_exist(git_command, remote)]
def git_add_remote(git_command: str, name: str, url: str, push_url: str) -> None:
"""Add a git remote"""
call((git_command, "remote", "add", name, url), silent=True)
call((git_command, "remote", "set-url", "--push", name, push_url), silent=True)
def git_branch(git_command: str) -> str:
"""Get current branch name."""
try:
branch = subprocess.check_output([git_command, "rev-parse", "--abbrev-ref", "HEAD"])
except subprocess.CalledProcessError:
# No need to print the exception, error text is written to the output already.
sys.stderr.write("Failed to get Blender git branch\n")
sys.exit(1)
return branch.strip().decode('utf8')
def git_get_config(git_command: str, key: str, file: "str | None" = None) -> str:
if file:
return check_output([git_command, "config", "--file", file, "--get", key])
return check_output([git_command, "config", "--get", key])
def git_set_config(git_command: str, key: str, value: str, file: "str | None" = None) -> str:
if file:
return check_output([git_command, "config", "--file", file, key, value])
return check_output([git_command, "config", key, value])
def _git_submodule_config_key(submodule_dir: Path, key: str) -> str:
submodule_dir_str = submodule_dir.as_posix()
return f"submodule.{submodule_dir_str}.{key}"
def is_git_submodule_enabled(git_command: str, submodule_dir: Path) -> bool:
"""Check whether submodule denoted by its directory within the repository is enabled"""
git_root = Path(check_output([git_command, "rev-parse", "--show-toplevel"]))
gitmodules = git_root / ".gitmodules"
# Check whether the submodule actually exists.
# Request path of an unknown submodule will cause non-zero exit code.
path = git_get_config(
git_command, _git_submodule_config_key(submodule_dir, "path"), str(gitmodules))
if not path:
return False
# When the "update" strategy is not provided explicitly in the local configuration
# `git config` returns a non-zero exit code. For those assume the default "checkout"
# strategy.
update = check_output(
(git_command, "config", "--local", _git_submodule_config_key(submodule_dir, "update")),
exit_on_error=False)
if update == "":
# The repository is not in our local configuration.
# Check the default `.gitmodules` setting.
update = check_output(
(git_command, "config", "--file", str(gitmodules), _git_submodule_config_key(submodule_dir, "update")),
exit_on_error=False)
return update.lower() != "none"
def git_enable_submodule(git_command: str, submodule_dir: Path) -> None:
"""Enable submodule denoted by its directory within the repository"""
command = (git_command,
"config",
"--local",
_git_submodule_config_key(submodule_dir, "update"),
"checkout")
call(command, exit_on_error=True, silent=True)
def git_update_submodule(git_command: str, submodule_dir: Path) -> bool:
"""
Update the given submodule.
The submodule is denoted by its path within the repository.
This function will initialize the submodule if it has not been initialized.
Returns true if the update succeeded
"""
# Use the two stage update process:
# - Step 1: checkout the submodule to the desired (by the parent repository) hash, but
# skip the LFS smudging.
# - Step 2: Fetch LFS files, if needed.
#
# This allows to show download progress, potentially allowing resuming the download
# progress, and even recovering from partial/corrupted checkout of submodules.
#
# This bypasses the limitation of submodules which are configured as "update=checkout"
# with regular `git submodule update` which, depending on the Git version will not report
# any progress. This is because submodule--helper.c configures Git checkout process with
# the "quiet" flag, so that there is no detached head information printed after submodule
# update, and since Git 2.33 the LFS messages "Filtering contents..." is suppressed by
#
# https://github.com/git/git/commit/7a132c628e57b9bceeb88832ea051395c0637b16
#
# Doing `git lfs pull` after checkout with `GIT_LFS_SKIP_SMUDGE=true` seems to be the
# valid process. For example, https://www.mankier.com/7/git-lfs-faq
env = {"GIT_LFS_SKIP_SMUDGE": "1"}
if call((git_command, "submodule", "update", "--init", "--progress", str(submodule_dir)),
exit_on_error=False, env=env) != 0:
return False
return call((git_command, "-C", str(submodule_dir), "lfs", "pull"),
exit_on_error=False) == 0
def command_missing(command: str) -> bool:
# Support running with Python 2 for macOS
if sys.version_info >= (3, 0):
return shutil.which(command) is None
return False
class BlenderVersion:
def __init__(self, version: int, patch: int, cycle: str):
# 293 for 2.93.1
self.version = version
# 1 for 2.93.1
self.patch = patch
# 'alpha', 'beta', 'release', maybe others.
self.cycle = cycle
def is_release(self) -> bool:
return self.cycle == "release"
def __str__(self) -> str:
"""Convert to version string.
>>> str(BlenderVersion(293, 1, "alpha"))
'2.93.1-alpha'
>>> str(BlenderVersion(327, 0, "release"))
'3.27.0'
"""
version_major = self.version // 100
version_minor = self.version % 100
as_string = f"{version_major}.{version_minor}.{self.patch}"
if self.is_release():
return as_string
return f"{as_string}-{self.cycle}"
def parse_blender_version() -> BlenderVersion:
blender_srcdir = Path(__file__).absolute().parent.parent.parent
version_path = blender_srcdir / "source/blender/blenkernel/BKE_blender_version.h"
version_info = {}
line_re = re.compile(r"^#define (BLENDER_VERSION[A-Z_]*)\s+([0-9a-z]+)$")
with version_path.open(encoding="utf-8") as version_file:
for line in version_file:
match = line_re.match(line.strip())
if not match:
continue
version_info[match.group(1)] = match.group(2)
return BlenderVersion(
int(version_info["BLENDER_VERSION"]),
int(version_info["BLENDER_VERSION_PATCH"]),
version_info["BLENDER_VERSION_CYCLE"],
)
def remove_directory(directory: Path) -> None:
"""
Recursively remove the given directory
Takes care of clearing read-only attributes which might prevent deletion on
Windows.
"""
# NOTE: unquote typing once Python 3.6x is dropped.
def remove_readonly(
func: Callable[..., Any],
path: str,
_: "tuple[type[BaseException], BaseException, TracebackType]",
) -> None:
"Clear the read-only bit and reattempt the removal."
os.chmod(path, stat.S_IWRITE)
func(path)
shutil.rmtree(directory, onerror=remove_readonly)