diff --git a/GNUmakefile b/GNUmakefile index bcedd5ec342..244211af076 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -428,6 +428,8 @@ ifneq "$(findstring clean, $(MAKECMDGOALS))" "" DEPS_TARGET = clean endif +# Set the SOURCE_DATE_EPOCH to make builds reproducible (locks timestamps to the specified date). +deps: export SOURCE_DATE_EPOCH = 1745584760 deps: .FORCE @echo @echo Configuring dependencies in \"$(DEPS_BUILD_DIR)\", install to \"$(DEPS_INSTALL_DIR)\" diff --git a/build_files/build_environment/CMakeLists.txt b/build_files/build_environment/CMakeLists.txt index a937f451695..c7a8ee5bdf8 100644 --- a/build_files/build_environment/CMakeLists.txt +++ b/build_files/build_environment/CMakeLists.txt @@ -72,7 +72,9 @@ include(cmake/tbb.cmake) include(cmake/python.cmake) include(cmake/llvm.cmake) include(cmake/osl.cmake) +include(cmake/cython.cmake) include(cmake/numpy.cmake) +include(cmake/zstandard.cmake) include(cmake/python_site_packages.cmake) include(cmake/package_python.cmake) include(cmake/openimageio.cmake) @@ -179,6 +181,11 @@ include(cmake/vulkan.cmake) include(cmake/pybind11.cmake) include(cmake/nanobind.cmake) include(cmake/manifold.cmake) -# Keep these last +# Keep these last. include(cmake/deps_html.cmake) include(cmake/cve_check.cmake) + +if(UNIX) + # Strip all installed libraries. + harvest_strip_all_libraries() +endif() diff --git a/build_files/build_environment/cmake/cython.cmake b/build_files/build_environment/cmake/cython.cmake new file mode 100644 index 00000000000..c4666875f83 --- /dev/null +++ b/build_files/build_environment/cmake/cython.cmake @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: 2017-2022 Blender Authors +# +# SPDX-License-Identifier: GPL-2.0-or-later + +if(MSVC) + if(BUILD_MODE STREQUAL Debug) + set(CYTHON_DIR_POSTFIX -pydebug) + set(CYTHON_ARCHIVE_POSTFIX d) + set(CYTHON_BUILD_OPTION --debug) + else() + set(CYTHON_DIR_POSTFIX) + set(CYTHON_ARCHIVE_POSTFIX) + set(CYTHON_BUILD_OPTION) + endif() +endif() + +set(CYTHON_POSTFIX) + +ExternalProject_Add(external_cython + URL file://${PACKAGE_DIR}/${CYTHON_FILE} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + URL_HASH ${CYTHON_HASH_TYPE}=${CYTHON_HASH} + PREFIX ${BUILD_DIR}/cython + PATCH_COMMAND ${CYTHON_PATCH} + CONFIGURE_COMMAND "" + LOG_BUILD 1 + BUILD_IN_SOURCE 1 + + BUILD_COMMAND + ${PYTHON_BINARY} setup.py + build ${CYTHON_BUILD_OPTION} -j${MAKE_THREADS} + install + --old-and-unmanageable + + INSTALL_COMMAND "" +) + +add_dependencies( + external_cython + external_python +) diff --git a/build_files/build_environment/cmake/download.cmake b/build_files/build_environment/cmake/download.cmake index c90243623af..b0bf25bbbd7 100644 --- a/build_files/build_environment/cmake/download.cmake +++ b/build_files/build_environment/cmake/download.cmake @@ -99,6 +99,8 @@ download_source(OSL) download_source(PYTHON) download_source(TBB) download_source(OPENVDB) +download_source(CYTHON) +download_source(ZSTANDARD) download_source(NUMPY) download_source(LAME) download_source(OGG) diff --git a/build_files/build_environment/cmake/harvest.cmake b/build_files/build_environment/cmake/harvest.cmake index e67ad59c517..4fb11346b44 100644 --- a/build_files/build_environment/cmake/harvest.cmake +++ b/build_files/build_environment/cmake/harvest.cmake @@ -73,7 +73,7 @@ else() # # Also removes versioned symlinks, which give errors with macOS notarization. if(APPLE) - set(set_rpath_cmd python3 ${CMAKE_CURRENT_SOURCE_DIR}/darwin/set_rpath.py @loader_path) + set(set_rpath_cmd python3 ${CMAKE_CURRENT_SOURCE_DIR}/utils/set_rpath.py @loader_path) else() set(set_rpath_cmd patchelf --set-rpath $ORIGIN) endif() @@ -121,4 +121,9 @@ else() endif()\n endforeach()") endfunction() + + # Strip all shared/static libraries in the HARVEST_TARGET location. + function(harvest_strip_all_libraries) + install(CODE "execute_process(COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/utils/strip_libraries.py ${HARVEST_TARGET})") + endfunction() endif() diff --git a/build_files/build_environment/cmake/numpy.cmake b/build_files/build_environment/cmake/numpy.cmake index f0ed35daadc..caf78b28e83 100644 --- a/build_files/build_environment/cmake/numpy.cmake +++ b/build_files/build_environment/cmake/numpy.cmake @@ -28,8 +28,8 @@ ExternalProject_Add(external_numpy BUILD_IN_SOURCE 1 BUILD_COMMAND - ${PYTHON_BINARY} ${BUILD_DIR}/numpy/src/external_numpy/setup.py - build ${NUMPY_BUILD_OPTION} + ${PYTHON_BINARY} setup.py + build ${NUMPY_BUILD_OPTION} -j${MAKE_THREADS} install --old-and-unmanageable @@ -40,4 +40,5 @@ add_dependencies( external_numpy external_python external_python_site_packages + external_cython ) diff --git a/build_files/build_environment/cmake/python.cmake b/build_files/build_environment/cmake/python.cmake index 1fa29ce0159..0b4f03fcd6f 100644 --- a/build_files/build_environment/cmake/python.cmake +++ b/build_files/build_environment/cmake/python.cmake @@ -160,9 +160,8 @@ else() if(NOT APPLE) set(PYTHON_CONFIGURE_EXTRA_ARGS ${PYTHON_CONFIGURE_EXTRA_ARGS} - # Used on most release Linux builds (Fedora for e.g.), - # increases build times noticeably with the benefit of a modest speedup at runtime. - --enable-optimizations + # We disable optimzations as this flag turns on PGO which leads to non-reproducible builds. + --disable-optimizations # While LTO is OK when building on the same system, it's incompatible across GCC versions, # making it impractical for developers to build against, so keep it disabled. # `--with-lto` diff --git a/build_files/build_environment/cmake/python_site_packages.cmake b/build_files/build_environment/cmake/python_site_packages.cmake index 4329cb4e3f7..37f5d1d22b5 100644 --- a/build_files/build_environment/cmake/python_site_packages.cmake +++ b/build_files/build_environment/cmake/python_site_packages.cmake @@ -23,16 +23,15 @@ ExternalProject_Add(external_python_site_packages PREFIX ${BUILD_DIR}/site_packages # setuptools is downgraded to 63.2.0 (same as python 3.10.8) since numpy 1.23.x seemingly has - # issues building on windows with the newer versions that ships with python 3.10.9+ + # issues building on windows with the newer versions that ships with python 3.10.9+. + # We do not build numpy, cython, or zstandard here as the pip builds are not reproducible. INSTALL_COMMAND ${PYTHON_BINARY} -m pip install --no-cache-dir ${SITE_PACKAGES_EXTRA} setuptools==63.2.0 - cython==${CYTHON_VERSION} idna==${IDNA_VERSION} charset-normalizer==${CHARSET_NORMALIZER_VERSION} urllib3==${URLLIB3_VERSION} certifi==${CERTIFI_VERSION} requests==${REQUESTS_VERSION} - zstandard==${ZSTANDARD_VERSION} autopep8==${AUTOPEP8_VERSION} pycodestyle==${PYCODESTYLE_VERSION} meson==${MESON_VERSION} diff --git a/build_files/build_environment/cmake/versions.cmake b/build_files/build_environment/cmake/versions.cmake index c1f0fe72a9b..f3f9d85d70d 100644 --- a/build_files/build_environment/cmake/versions.cmake +++ b/build_files/build_environment/cmake/versions.cmake @@ -421,11 +421,25 @@ set(CERTIFI_VERSION 2021.10.8) set(REQUESTS_VERSION 2.27.1) # Needed by: Python's `numpy` module (used by some add-ons). set(CYTHON_VERSION 3.0.11) +set(CYTHON_URI +https://github.com/cython/cython/releases/download/${CYTHON_VERSION}-1/cython-${CYTHON_VERSION}.tar.gz) +set(CYTHON_HASH 388b85b7c23f501320d19d991b169f5d) +set(CYTHON_HASH_TYPE MD5) +set(CYTHON_FILE cython-${CYTHON_VERSION}.tar.gz) +set(CYTHON_HOMEPAGE https://cython.org/) +set(CYTHON_LICENSE SPDX:Apache-2.0) # Needed by: Python scripts that read `.blend` files, as files may use Z-standard compression. # The version of the ZSTD library used to build the Python package should match ZSTD_VERSION # defined below. At this time of writing, 0.17.0 was already released, # but built against ZSTD 1.5.1, while we use 1.5.0. set(ZSTANDARD_VERSION 0.16.0) +set(ZSTANDARD_URI +https://github.com/indygreg/python-zstandard/releases/download/${ZSTANDARD_VERSION}/zstandard-${ZSTANDARD_VERSION}.tar.gz) +set(ZSTANDARD_HASH 19b1b12edcd66165d86f25e2a2277517) +set(ZSTANDARD_HASH_TYPE MD5) +set(ZSTANDARD_FILE cython-${ZSTANDARD_VERSION}.tar.gz) +set(ZSTANDARD_HOMEPAGE https://cython.org/) +set(ZSTANDARD_LICENSE SPDX:BSD-3-Clause) # Auto-format Python source (developer tool, not used by Blender at run-time). set(AUTOPEP8_VERSION 2.3.1) # Needed by: `autopep8` (so the version doesn't change on rebuild). diff --git a/build_files/build_environment/cmake/zstandard.cmake b/build_files/build_environment/cmake/zstandard.cmake new file mode 100644 index 00000000000..9c45f7383fb --- /dev/null +++ b/build_files/build_environment/cmake/zstandard.cmake @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: 2017-2022 Blender Authors +# +# SPDX-License-Identifier: GPL-2.0-or-later + +if(MSVC) + if(BUILD_MODE STREQUAL Debug) + set(ZSTANDARD_DIR_POSTFIX -pydebug) + set(ZSTANDARD_ARCHIVE_POSTFIX d) + set(ZSTANDARD_BUILD_OPTION --debug) + else() + set(ZSTANDARD_DIR_POSTFIX) + set(ZSTANDARD_ARCHIVE_POSTFIX) + set(ZSTANDARD_BUILD_OPTION) + endif() +endif() + +set(ZSTANDARD_POSTFIX) + +ExternalProject_Add(external_zstandard + URL file://${PACKAGE_DIR}/${ZSTANDARD_FILE} + DOWNLOAD_DIR ${DOWNLOAD_DIR} + URL_HASH ${ZSTANDARD_HASH_TYPE}=${ZSTANDARD_HASH} + PREFIX ${BUILD_DIR}/zstandard + PATCH_COMMAND ${ZSTANDARD_PATCH} + CONFIGURE_COMMAND "" + LOG_BUILD 1 + BUILD_IN_SOURCE 1 + + BUILD_COMMAND + ${PYTHON_BINARY} setup.py + build ${ZSTANDARD_BUILD_OPTION} -j${MAKE_THREADS} + install + --old-and-unmanageable + + INSTALL_COMMAND "" +) + +add_dependencies( + external_zstandard + external_python +) diff --git a/build_files/build_environment/darwin/set_rpath.py b/build_files/build_environment/utils/set_rpath.py similarity index 100% rename from build_files/build_environment/darwin/set_rpath.py rename to build_files/build_environment/utils/set_rpath.py diff --git a/build_files/build_environment/utils/strip_libraries.py b/build_files/build_environment/utils/strip_libraries.py new file mode 100755 index 00000000000..0cb5a1951a9 --- /dev/null +++ b/build_files/build_environment/utils/strip_libraries.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2025 Blender Authors +# +# SPDX-License-Identifier: GPL-2.0-or-later + +""" +Script which strips all libraries in the given library directory. +This is so we don't keep any debug data or symbols that contains +random hashes that are not reproducible between builds. + +This will strip both static and shared libraries. + +Usage: + strip_libraries.py +""" +import argparse +import subprocess +import sys +from pathlib import Path + +def print_strip_lib(strip_lib: Path, prev_print_len: int) -> int: + print_str = f"Stripping: {strip_lib}" + if prev_print_len > 0: + print(f"\r{' ' * prev_print_len}\r", end="") + print(print_str, end="", flush=True) + return len(print_str) + +def strip_libs(strip_dir: Path) -> None: + print(f"Stripping libraries in: {strip_dir}") + prev_print_len = 0; + for shared_lib in strip_dir.rglob("*.so*"): + if shared_lib.suffix == ".py": + # Work around badly named sycl scripts. + continue + + if shared_lib.is_symlink(): + # Don't strip symlinks as we don't want to strip the same library multiple times. + continue + + prev_print_len = print_strip_lib(shared_lib, prev_print_len) + subprocess.check_call(["strip", "-s", "--enable-deterministic-archives", shared_lib]) + for static_lib in strip_dir.rglob("*.a"): + if static_lib.is_symlink(): + # Don't strip symlinks as we don't want to strip the same library multiple times. + continue + + prev_print_len = print_strip_lib(static_lib, prev_print_len) + subprocess.check_call(["objcopy", "--enable-deterministic-archives", static_lib]) + + print("\nDone stripping libraries!") + + +def main() -> None: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawTextHelpFormatter, + ) + parser.add_argument("directory", type=Path, help="Path to the library directory to strip") + args = parser.parse_args() + + if sys.platform == "linux": + strip_libs(args.directory) + return + + +if __name__ == "__main__": + main()