From bf969834b508fa03d31ea2bb2c671d61e612e92f Mon Sep 17 00:00:00 2001 From: Sebastian Parborg Date: Thu, 14 Nov 2024 17:59:21 +0100 Subject: [PATCH] Libs: Fix static libs not getting linked properly on linux Multiple of our libraries would pull in system libraries that we have static versions of. These where libxml2 and libz and also libzstd. I also noticed that the script that was supposed to check on this was not really usable to check for these kinds of things so I updated it. Now you can run it with `python3.11 tools/check_blender_release/check_release.py -- ../build_linux_release/bin/` and it will check all binaries and libraries that we ship for any system libraries that we don't expect to be linked. The libraries I'm aware of that pulled in system libs were: boost mesa osl llvm (The removed cmake flags for osl was because these were unused and cmake printed warnings) Pull Request: https://projects.blender.org/blender/blender/pulls/130236 --- .../build_environment/cmake/boost.cmake | 1 + .../build_environment/cmake/llvm.cmake | 9 +++- .../build_environment/cmake/mesa.cmake | 9 ++-- build_files/build_environment/cmake/osl.cmake | 7 +--- .../build_environment/cmake/zlib.cmake | 3 +- .../patches/boost.user.jam.in | 4 +- .../build_environment/patches/llvm.diff | 22 ++++++++++ .../check_static_binaries.py | 41 +++++++++++++++---- 8 files changed, 73 insertions(+), 23 deletions(-) diff --git a/build_files/build_environment/cmake/boost.cmake b/build_files/build_environment/cmake/boost.cmake index cebfec85fd2..6f139593d48 100644 --- a/build_files/build_environment/cmake/boost.cmake +++ b/build_files/build_environment/cmake/boost.cmake @@ -114,6 +114,7 @@ add_dependencies( external_boost external_python external_numpy + external_zlib ) if(NOT WIN32) diff --git a/build_files/build_environment/cmake/llvm.cmake b/build_files/build_environment/cmake/llvm.cmake index 2b8dd8c9479..0eb6ef5e331 100644 --- a/build_files/build_environment/cmake/llvm.cmake +++ b/build_files/build_environment/cmake/llvm.cmake @@ -8,6 +8,13 @@ else() set(LLVM_TARGETS X86) endif() +if(UNIX AND NOT APPLE) + # Make llvm's pkgconfig pick up our static xml2 lib + set(LLVM_XML2_ARGS + -DCMAKE_PREFIX_PATH=${LIBDIR}/xml2 + ) +endif() + if(APPLE) set(LLVM_XML2_ARGS -DLIBXML2_LIBRARY=${LIBDIR}/xml2/lib/libxml2.a @@ -114,7 +121,7 @@ else() endif() # We currently do not build libxml2 on Windows. -if(APPLE) +if(UNIX) add_dependencies( ll external_xml2 diff --git a/build_files/build_environment/cmake/mesa.cmake b/build_files/build_environment/cmake/mesa.cmake index 096e0671218..d5cd3a6c518 100644 --- a/build_files/build_environment/cmake/mesa.cmake +++ b/build_files/build_environment/cmake/mesa.cmake @@ -2,10 +2,6 @@ # # SPDX-License-Identifier: GPL-2.0-or-later -set(MESA_CFLAGS "-static-libgcc") -set(MESA_CXXFLAGS "-static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -l:libstdc++.a") -set(MESA_LDFLAGS "-L${LIBDIR}/zlib/lib -pthread -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -l:libstdc++.a -l:libz_pic.a") - # The 'native-file', used for overrides with the meson build system. # meson does not provide a way to do this using command line arguments. # @@ -35,8 +31,8 @@ set(MESA_EXTRA_FLAGS # At some point we will likely want to support Wayland. # Disable for now since it's not officially supported. -Dplatforms=x11 - # Needed to find the local expat. - --pkg-config-path=${LIBDIR}/expat/lib/pkgconfig + # Needed to find the local expat,zlib,zstd. + --pkg-config-path=${LIBDIR}/expat/lib/pkgconfig,${LIBDIR}/zstd/lib/pkgconfig,${LIBDIR}/zlib/share/pkgconfig --native-file ${BUILD_DIR}/mesa/tmp/native-file.ini ) @@ -68,6 +64,7 @@ add_dependencies( external_mesa ll external_zlib + external_zstd # Run-time dependency. external_expat # Needed for `MESON`. diff --git a/build_files/build_environment/cmake/osl.cmake b/build_files/build_environment/cmake/osl.cmake index 4cf9a9901cc..3ec8aeb65de 100644 --- a/build_files/build_environment/cmake/osl.cmake +++ b/build_files/build_environment/cmake/osl.cmake @@ -26,8 +26,6 @@ set(OSL_EXTRA_ARGS -DOpenEXR_ROOT=${LIBDIR}/openexr/ -DOpenImageIO_ROOT=${LIBDIR}/openimageio/ -DOSL_BUILD_TESTS=OFF - -DOSL_BUILD_MATERIALX=OFF - -DPNG_ROOT=${LIBDIR}/png -DZLIB_LIBRARY=${LIBDIR}/zlib/lib/${ZLIB_LIBRARY} -DZLIB_INCLUDE_DIR=${LIBDIR}/zlib/include/ ${OSL_FLEX_BISON} @@ -38,14 +36,11 @@ set(OSL_EXTRA_ARGS -DSTOP_ON_WARNING=OFF -DUSE_LLVM_BITCODE=OFF -DLLVM_ROOT=${LIBDIR}/llvm/ - -DLLVM_DIRECTORY=${LIBDIR}/llvm/ + -DLLVM_STATIC=ON -DUSE_PARTIO=OFF -DUSE_QT=OFF - -DUSE_Qt5=OFF -DINSTALL_DOCS=OFF -Dpugixml_ROOT=${LIBDIR}/pugixml - -DTIFF_ROOT=${LIBDIR}/tiff - -DJPEG_ROOT=${LIBDIR}/jpeg -DUSE_PYTHON=OFF -DImath_ROOT=${LIBDIR}/imath -DCMAKE_DEBUG_POSTFIX=_d diff --git a/build_files/build_environment/cmake/zlib.cmake b/build_files/build_environment/cmake/zlib.cmake index 410766c89a6..ba2d89d6a17 100644 --- a/build_files/build_environment/cmake/zlib.cmake +++ b/build_files/build_environment/cmake/zlib.cmake @@ -58,7 +58,8 @@ else() COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/zlib/lib/libz.a ${LIBDIR}/zlib/lib/libz_pic.a - + # Make sure that our libraries do not pick up the shared libraries by mistake + COMMAND sh -c "rm -f ${LIBDIR}/zlib/lib/*.so*" DEPENDEES install ) endif() diff --git a/build_files/build_environment/patches/boost.user.jam.in b/build_files/build_environment/patches/boost.user.jam.in index b20a90d522c..621d804e87a 100644 --- a/build_files/build_environment/patches/boost.user.jam.in +++ b/build_files/build_environment/patches/boost.user.jam.in @@ -1,4 +1,6 @@ using python : @PYTHON_SHORT_VERSION@ : @PYTHON_BINARY@ : @LIBDIR@/python/include @LIBDIR@/python/include/python@PYTHON_SHORT_VERSION@/ : @LIBDIR@/python/libs -; \ No newline at end of file +; + +using zlib : @ZLIB_VERSION@ : @LIBDIR@/zlib/lib @LIBDIR@/zlib/inlcude ; diff --git a/build_files/build_environment/patches/llvm.diff b/build_files/build_environment/patches/llvm.diff index 426fe173d37..00292a3d21c 100644 --- a/build_files/build_environment/patches/llvm.diff +++ b/build_files/build_environment/patches/llvm.diff @@ -12,3 +12,25 @@ diff -Naur ll.org/llvm/lib/Support/Unix/Path.inc ll/llvm/lib/Support/Unix/Path.i if (__builtin_available(macos 10.12, *)) { // Optimistically try to use clonefile() and handle errors, rather than // calling stat() to see if it'll work. +diff -Naur ll.org/llvm/lib/WindowsManifest/CMakeLists.txt ll/llvm/lib/WindowsManifest/CMakeLists.txt +--- ll.org/llvm/lib/WindowsManifest/CMakeLists.txt 2024-11-13 19:29:25.362863529 +0100 ++++ ll/llvm/lib/WindowsManifest/CMakeLists.txt 2024-11-13 19:29:43.376850350 +0100 +@@ -17,18 +17,3 @@ + LINK_COMPONENTS + Support + ) +- +-# This block is only needed for llvm-config. When we deprecate llvm-config and +-# move to using CMake export, this block can be removed. +-if(LLVM_ENABLE_LIBXML2) +- # CMAKE_BUILD_TYPE is only meaningful to single-configuration generators. +- if(CMAKE_BUILD_TYPE) +- string(TOUPPER ${CMAKE_BUILD_TYPE} build_type) +- get_property(libxml2_library TARGET LibXml2::LibXml2 PROPERTY LOCATION_${build_type}) +- endif() +- if(NOT libxml2_library) +- get_property(libxml2_library TARGET LibXml2::LibXml2 PROPERTY LOCATION) +- endif() +- get_library_name(${libxml2_library} libxml2_library) +- set_property(TARGET LLVMWindowsManifest PROPERTY LLVM_SYSTEM_LIBS ${libxml2_library}) +-endif() diff --git a/tools/check_blender_release/check_static_binaries.py b/tools/check_blender_release/check_static_binaries.py index cafc4229721..5b3685f20f3 100644 --- a/tools/check_blender_release/check_static_binaries.py +++ b/tools/check_blender_release/check_static_binaries.py @@ -8,6 +8,7 @@ from pathlib import Path import re import subprocess import unittest +import glob from check_utils import ( sliceCommandLineArguments, @@ -15,7 +16,7 @@ from check_utils import ( ) -ALLOWED_LIBS = ( +ALLOWED_LIBS = [ # Core C/C++ libraries: "ld-linux.so", "ld-linux-x86-64.so", @@ -30,16 +31,32 @@ ALLOWED_LIBS = ( # Libraries which are part of default install: "libcrypt.so", - "libnsl.so", - "libmvec.so.1", + "libuuid.so", + + # Bundled python ncurses deps + "libpanelw.so", + "libncursesw.so", + "libtinfo.so", # X11 libraries we don't link statically: + "libdrm.so", "libX11.so", "libXext.so", "libXrender.so", "libXxf86vm.so", "libXi.so", "libXfixes.so", + "libxkbcommon.so", + + # MaterialX X11 libs: + "libICE.so", + "libSM.so", + "libXt.so", + "libOpenGL.so", + "libGLX.so", + + # Level Zero (Intel GPU Render) + "libze_loader.so", # OpenGL libraries: "libGL.so", @@ -48,10 +65,7 @@ ALLOWED_LIBS = ( # Library the software-GL is linking against and distributes with it: 'libglapi.so', 'libxcb.so', - - # Own dependencies we don't link statically: - "libfreetype.so", -) +] IGNORE_FILES = ("blender-launcher", "blender-softwaregl", ) IGNORE_EXTENSION = (".sh", ".py", ) @@ -142,7 +156,8 @@ class UnitTesting(unittest.TestCase): libraries = getNeededLibraries(binary_filepath) for lib_name in libraries: lib_name_no_abi = stripLibraryABI(lib_name) - self.assertTrue(lib_name_no_abi in ALLOWED_LIBS, + with self.subTest(msg=os.path.basename(binary_filepath) + ' check'): + self.assertTrue(lib_name_no_abi in ALLOWED_LIBS, "Error detected in {}: library used {}" . format( binary_filepath, lib_name)) @@ -177,6 +192,16 @@ class UnitTesting(unittest.TestCase): self.assertTrue(os.path.isdir(args.directory), "Given path is not a directory: {}" . format(args.directory)) + # Add all libraries the we bundle to the allowed list + global ALLOWED_LIBS + ALLOWED_LIBS += glob.glob("*.so", root_dir=args.directory + "/lib") + # Add OIDN libs that do not have a .so symlink + for oidn_lib in glob.glob("libOpenImageDenoise_*.so*", root_dir=args.directory + "/lib"): + ALLOWED_LIBS.append(stripLibraryABI(oidn_lib)) + # Add all bundled python libs + for python_lib in glob.glob("[0-9].[0-9]/python/lib/**/*.so", root_dir=args.directory, recursive=True): + ALLOWED_LIBS.append(os.path.basename(python_lib)) + # Perform actual test, self.checkDirectory(args.directory)