deps_builder: 3.6 library update

- OIIO 2.4.11.0
- OpenEXR 3.1.7 with additional fix for threadpool shutdown
- FreeType 2.13.0
- Fmt 9.1.0
- Python 3.10.11
- Flac 1.4.2
- FFMpeg 6.0
- OpenSSL 1.1.1t
- SQLite 3.40.1
- USD 23.05 with MaterialX Support and Render module

Co-authored-by: Brecht Van Lommel <brecht@blender.org>
Co-authored-by: Campbell Barton <campbell@blender.org>
Pull Request: https://projects.blender.org/blender/blender/pulls/107783
This commit is contained in:
Ray Molenkamp
2023-05-10 16:20:16 +02:00
committed by Ray molenkamp
parent 8efd6d5f82
commit ab106607bd
18 changed files with 1267 additions and 136 deletions

View File

@@ -122,5 +122,6 @@ if(BUILD_MODE STREQUAL Release AND WIN32)
COMMAND ${CMAKE_COMMAND} -E rm -f ${HARVEST_TARGET}/dpcpp/bin/lld.exe
COMMAND ${CMAKE_COMMAND} -E rm -f ${HARVEST_TARGET}/dpcpp/bin/lld-link.exe
COMMAND ${CMAKE_COMMAND} -E rm -f ${HARVEST_TARGET}/dpcpp/bin/wasm-ld.exe
DEPENDEES install
)
endif()

View File

@@ -46,7 +46,7 @@ ${temp_LIBDIR}/vpx/lib/pkgconfig:\
${temp_LIBDIR}/theora/lib/pkgconfig:\
${temp_LIBDIR}/openjpeg/lib/pkgconfig:\
${temp_LIBDIR}/opus/lib/pkgconfig:\
${temp_LIBDIR}/aom/lib/pkgconfig"
${temp_LIBDIR}/aom/lib/pkgconfig:"
)
unset(temp_LIBDIR)

View File

@@ -2,35 +2,45 @@
set(FFTW_EXTRA_ARGS)
if(WIN32)
set(FFTW3_PATCH_COMMAND ${PATCH_CMD} --verbose -p 0 -N -d ${BUILD_DIR}/fftw3/src/external_fftw3 < ${PATCH_DIR}/fftw3.diff)
set(FFTW_EXTRA_ARGS --disable-static --enable-shared)
set(FFTW_INSTALL install-strip)
else()
set(FFTW_EXTRA_ARGS --enable-static)
set(FFTW_INSTALL install)
endif()
macro(fftw_build FFTW_POSTFIX)
if(WIN32)
set(FFTW3_PATCH_COMMAND ${PATCH_CMD} --verbose -p 0 -N -d ${BUILD_DIR}/fftw3/src/external_fftw3_${FFTW_POSTFIX} < ${PATCH_DIR}/fftw3.diff)
set(FFTW_EXTRA_ARGS --disable-static --enable-shared)
set(FFTW_INSTALL install-strip)
else()
set(FFTW_EXTRA_ARGS --enable-static)
set(FFTW_INSTALL install)
endif()
ExternalProject_Add(external_fftw3_${FFTW_POSTFIX}
URL file://${PACKAGE_DIR}/${FFTW_FILE}
DOWNLOAD_DIR ${DOWNLOAD_DIR}
URL_HASH ${FFTW_HASH_TYPE}=${FFTW_HASH}
PREFIX ${BUILD_DIR}/fftw3
CONFIGURE_COMMAND ${CONFIGURE_ENV} && cd ${BUILD_DIR}/fftw3/src/external_fftw3_${FFTW_POSTFIX}/ && ${CONFIGURE_COMMAND} ${FFTW_EXTRA_ARGS} ${ARGN} --prefix=${mingw_LIBDIR}/fftw3
PATCH_COMMAND ${FFTW3_PATCH_COMMAND}
BUILD_COMMAND ${CONFIGURE_ENV} && cd ${BUILD_DIR}/fftw3/src/external_fftw3_${FFTW_POSTFIX}/ && make -j${MAKE_THREADS}
INSTALL_COMMAND ${CONFIGURE_ENV} && cd ${BUILD_DIR}/fftw3/src/external_fftw3_${FFTW_POSTFIX}/ && make ${FFTW_INSTALL}
INSTALL_DIR ${LIBDIR}/fftw3
)
endmacro()
ExternalProject_Add(external_fftw3
URL file://${PACKAGE_DIR}/${FFTW_FILE}
DOWNLOAD_DIR ${DOWNLOAD_DIR}
URL_HASH ${FFTW_HASH_TYPE}=${FFTW_HASH}
PREFIX ${BUILD_DIR}/fftw3
CONFIGURE_COMMAND ${CONFIGURE_ENV} && cd ${BUILD_DIR}/fftw3/src/external_fftw3/ && ${CONFIGURE_COMMAND} ${FFTW_EXTRA_ARGS} --prefix=${mingw_LIBDIR}/fftw3
PATCH_COMMAND ${FFTW3_PATCH_COMMAND}
BUILD_COMMAND ${CONFIGURE_ENV} && cd ${BUILD_DIR}/fftw3/src/external_fftw3/ && make -j${MAKE_THREADS}
INSTALL_COMMAND ${CONFIGURE_ENV} && cd ${BUILD_DIR}/fftw3/src/external_fftw3/ && make ${FFTW_INSTALL}
INSTALL_DIR ${LIBDIR}/fftw3
)
fftw_build(double)
fftw_build(float --enable-float)
if(MSVC)
set_target_properties(external_fftw3 PROPERTIES FOLDER Mingw)
set_target_properties(external_fftw3_double PROPERTIES FOLDER Mingw)
if(BUILD_MODE STREQUAL Release)
ExternalProject_Add_Step(external_fftw3 after_install
COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/fftw3/lib/libfftw3.dll.a ${HARVEST_TARGET}/fftw3/lib/libfftw.lib
ExternalProject_Add_Step(external_fftw3_double after_install
COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/fftw3/lib/libfftw3.dll.a ${HARVEST_TARGET}/fftw3/lib/libfftw3-3.lib
COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/fftw3/bin/libfftw3-3.dll ${HARVEST_TARGET}/fftw3/lib/libfftw3-3.dll
COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/fftw3/include/fftw3.h ${HARVEST_TARGET}/fftw3/include/fftw3.h
DEPENDEES install
)
ExternalProject_Add_Step(external_fftw3_float after_install
COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/fftw3/lib/libfftw3f.dll.a ${HARVEST_TARGET}/fftw3/lib/libfftw3f.lib
COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/fftw3/bin/libfftw3f-3.dll ${HARVEST_TARGET}/fftw3/lib/libfftw3f-3.dll
DEPENDEES install
)
endif()
endif()

View File

@@ -218,7 +218,7 @@ else()
harvest(openimagedenoise/lib openimagedenoise/lib "*.a")
harvest(embree/include embree/include "*.h")
harvest(embree/lib embree/lib "*.a")
harvest(embree/lib embree/lib "*${SHAREDLIBEXT}*")
harvest_rpath_lib(embree/lib embree/lib "*${SHAREDLIBEXT}*")
harvest(openpgl/include openpgl/include "*.h")
harvest(openpgl/lib openpgl/lib "*.a")
harvest(openpgl/lib/cmake/openpgl-${OPENPGL_SHORT_VERSION} openpgl/lib/cmake/openpgl "*.cmake")

View File

@@ -11,7 +11,7 @@ if(WIN32)
elseif(APPLE)
# Use bison and flex installed via Homebrew.
# The ones that come with Xcode toolset are too old.
if("${CMAKE_HOST_SYSTEM_PROCESSOR}" STREQUAL "arm64")
if(BLENDER_PLATFORM_ARM)
set(ISPC_EXTRA_ARGS_APPLE
-DBISON_EXECUTABLE=/opt/homebrew/opt/bison/bin/bison
-DFLEX_EXECUTABLE=/opt/homebrew/opt/flex/bin/flex

View File

@@ -2,11 +2,12 @@
set(MATERIALX_EXTRA_ARGS
-DMATERIALX_BUILD_PYTHON=ON
-DMATERIALX_BUILD_RENDER=OFF
-DMATERIALX_BUILD_RENDER=ON
-DMATERIALX_INSTALL_PYTHON=OFF
-DMATERIALX_PYTHON_EXECUTABLE=${PYTHON_BINARY}
-DMATERIALX_PYTHON_VERSION=${PYTHON_SHORT_VERSION}
-DMATERIALX_BUILD_SHARED_LIBS=ON
-DMATERIALX_BUILD_TESTS=OFF
-DCMAKE_DEBUG_POSTFIX=_d
-Dpybind11_ROOT=${LIBDIR}/pybind11
-DPython_EXECUTABLE=${PYTHON_BINARY}

View File

@@ -27,6 +27,7 @@ ExternalProject_Add(external_openexr
URL file://${PACKAGE_DIR}/${OPENEXR_FILE}
DOWNLOAD_DIR ${DOWNLOAD_DIR}
URL_HASH ${OPENEXR_HASH_TYPE}=${OPENEXR_HASH}
PATCH_COMMAND ${PATCH_CMD} -p 1 -d ${BUILD_DIR}/openexr/src/external_openexr < ${PATCH_DIR}/openexr_b18905772e.diff
CMAKE_GENERATOR ${PLATFORM_ALT_GENERATOR}
PREFIX ${BUILD_DIR}/openexr
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${LIBDIR}/openexr ${DEFAULT_CMAKE_FLAGS} ${OPENEXR_EXTRA_ARGS}

View File

@@ -54,7 +54,7 @@ set(OPENIMAGEIO_EXTRA_ARGS
-DUSE_DCMTK=OFF
-DUSE_LIBHEIF=OFF
-DUSE_OPENGL=OFF
-DUSE_TBB=OFF
-DUSE_TBB=ON
-DUSE_QT=OFF
-DUSE_PYTHON=ON
-DUSE_GIF=OFF
@@ -96,6 +96,7 @@ set(OPENIMAGEIO_EXTRA_ARGS
-DImath_ROOT=${LIBDIR}/imath
-Dpybind11_ROOT=${LIBDIR}/pybind11
-DPython_EXECUTABLE=${PYTHON_BINARY}
-DTBB_ROOT=${LIBDIR}/tbb
)
ExternalProject_Add(external_openimageio
@@ -104,7 +105,8 @@ ExternalProject_Add(external_openimageio
URL_HASH ${OPENIMAGEIO_HASH_TYPE}=${OPENIMAGEIO_HASH}
CMAKE_GENERATOR ${PLATFORM_ALT_GENERATOR}
PREFIX ${BUILD_DIR}/openimageio
PATCH_COMMAND ${PATCH_CMD} -p 1 -N -d ${BUILD_DIR}/openimageio/src/external_openimageio/ < ${PATCH_DIR}/openimageio.diff
PATCH_COMMAND ${PATCH_CMD} -p 1 -N -d ${BUILD_DIR}/openimageio/src/external_openimageio/ < ${PATCH_DIR}/openimageio.diff &&
${PATCH_CMD} -p 1 -N -d ${BUILD_DIR}/openimageio/src/external_openimageio/ < ${PATCH_DIR}/oiio_3832.diff
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${LIBDIR}/openimageio ${DEFAULT_CMAKE_FLAGS} ${OPENIMAGEIO_EXTRA_ARGS}
INSTALL_DIR ${LIBDIR}/openimageio
)
@@ -125,6 +127,7 @@ add_dependencies(
external_webp
external_python
external_pybind11
external_tbb
)
if(WIN32)

View File

@@ -35,7 +35,7 @@ if(WIN32)
# regardless of the version actually in there.
PATCH_COMMAND mkdir ${PYTHON_EXTERNALS_FOLDER_DOS} &&
mklink /J ${PYTHON_EXTERNALS_FOLDER_DOS}\\zlib-1.2.13 ${ZLIB_SOURCE_FOLDER_DOS} &&
mklink /J ${PYTHON_EXTERNALS_FOLDER_DOS}\\openssl-1.1.1q ${SSL_SOURCE_FOLDER_DOS} &&
mklink /J ${PYTHON_EXTERNALS_FOLDER_DOS}\\openssl-1.1.1t ${SSL_SOURCE_FOLDER_DOS} &&
${CMAKE_COMMAND} -E copy ${ZLIB_SOURCE_FOLDER}/../external_zlib-build/zconf.h ${PYTHON_EXTERNALS_FOLDER}/zlib-1.2.13/zconf.h &&
${PATCH_CMD} --verbose -p1 -d ${BUILD_DIR}/python/src/external_python < ${PATCH_DIR}/python_windows.diff
CONFIGURE_COMMAND echo "."

View File

@@ -1,7 +1,6 @@
# SPDX-License-Identifier: GPL-2.0-or-later
set(SSL_CONFIGURE_COMMAND ./Configure)
set(SSL_PATCH_CMD echo .)
if(WIN32)
# Python will build this with its preferred build options and patches. We only need to unpack openssl
@@ -18,7 +17,6 @@ if(WIN32)
else()
if(APPLE)
set(SSL_OS_COMPILER "blender-darwin-${CMAKE_OSX_ARCHITECTURES}")
set(SSL_PATCH_CMD ${PATCH_CMD} --verbose -p 0 -d ${BUILD_DIR}/ssl/src/external_ssl < ${PATCH_DIR}/ssl.diff)
else()
if(BLENDER_PLATFORM_ARM)
set(SSL_OS_COMPILER "blender-linux-aarch64")
@@ -35,7 +33,6 @@ else()
DOWNLOAD_DIR ${DOWNLOAD_DIR}
URL_HASH ${SSL_HASH_TYPE}=${SSL_HASH}
PREFIX ${BUILD_DIR}/ssl
PATCH_COMMAND ${SSL_PATCH_CMD}
CONFIGURE_COMMAND ${CONFIGURE_ENV} && cd ${BUILD_DIR}/ssl/src/external_ssl/ && ${SSL_CONFIGURE_COMMAND} --prefix=${LIBDIR}/ssl
--openssldir=${LIBDIR}/ssl
no-shared

View File

@@ -15,8 +15,7 @@ if(WIN32)
-D_PXR_CXX_DEFINITIONS=/DBOOST_ALL_NO_LIB
-DCMAKE_SHARED_LINKER_FLAGS_INIT=/LIBPATH:${LIBDIR}/tbb/lib
-DPython_FIND_REGISTRY=NEVER
-DPYTHON_INCLUDE_DIRS=${LIBDIR}/python/include
-DPYTHON_LIBRARY=${LIBDIR}/python/libs/python${PYTHON_SHORT_VERSION_NO_DOTS}${PYTHON_POSTFIX}${LIBEXT}
-DPython3_EXECUTABLE=${PYTHON_BINARY}
)
if(BUILD_MODE STREQUAL Debug)
list(APPEND USD_PLATFORM_FLAGS -DPXR_USE_DEBUG_PYTHON=ON)
@@ -27,6 +26,12 @@ elseif(UNIX)
# part of the interpret in the USD library. Allow undefined Python symbols and replace
# Python library with TBB so it doesn't complain about missing library.
set(USD_PLATFORM_FLAGS
# NOTE(@ideasman42): Setting the root is needed, without this an older version of Python
# is detected from the system. Referencing the root-directory may remove the need
# to explicitly set the `PYTHON_INCLUDE_DIR` & `PYTHON_LIBRARY`.
# Keep them as it's known these are the libraries to use and it avoids any ambiguity.
-DPython3_ROOT_DIR=${LIBDIR}/python/
-DPYTHON_INCLUDE_DIR=${LIBDIR}/python/include/python${PYTHON_SHORT_VERSION}/
-DPYTHON_LIBRARY=${LIBDIR}/tbb/lib/${LIBPREFIX}${TBB_LIBRARY}${SHAREDLIBEXT}
)
@@ -44,6 +49,7 @@ set(USD_EXTRA_ARGS
${USD_PLATFORM_FLAGS}
-DOPENSUBDIV_ROOT_DIR=${LIBDIR}/opensubdiv
-DOpenImageIO_ROOT=${LIBDIR}/openimageio
-DMaterialX_ROOT=${LIBDIR}/materialx
-DOPENEXR_LIBRARIES=${LIBDIR}/imath/lib/${LIBPREFIX}Imath${OPENEXR_VERSION_POSTFIX}${SHAREDLIBEXT}
-DOPENEXR_INCLUDE_DIR=${LIBDIR}/imath/include
-DImath_DIR=${LIBDIR}/imath
@@ -56,9 +62,10 @@ set(USD_EXTRA_ARGS
-DPXR_BUILD_TUTORIALS=OFF
-DPXR_BUILD_USDVIEW=OFF
-DPXR_ENABLE_HDF5_SUPPORT=OFF
-DPXR_ENABLE_MATERIALX_SUPPORT=OFF
-DPXR_ENABLE_MATERIALX_SUPPORT=ON
-DPXR_ENABLE_OPENVDB_SUPPORT=ON
-DPYTHON_EXECUTABLE=${PYTHON_BINARY}
-DPython3_EXECUTABLE=${PYTHON_BINARY}
-DPXR_BUILD_MONOLITHIC=ON
# OSL is an optional dependency of the Imaging module. However, since that
# module was included for its support for converting primitive shapes (sphere,
@@ -95,7 +102,12 @@ ExternalProject_Add(external_usd
CMAKE_GENERATOR ${PLATFORM_ALT_GENERATOR}
PREFIX ${BUILD_DIR}/usd
LIST_SEPARATOR ^^
PATCH_COMMAND ${PATCH_CMD} -p 1 -d ${BUILD_DIR}/usd/src/external_usd < ${PATCH_DIR}/usd.diff
# usd_pull_1965.diff https://github.com/PixarAnimationStudios/USD/pull/1965
# usd_hydra.diff - https://github.com/bnagirniak/RPRHydraRenderBlenderAddon/blob/master/usd.diff
# usd_hydra.diff also included the blender changes and usd_pull_1965 and has been edited to remove those sections.
PATCH_COMMAND ${PATCH_CMD} -p 1 -d ${BUILD_DIR}/usd/src/external_usd < ${PATCH_DIR}/usd.diff &&
${PATCH_CMD} -p 1 -d ${BUILD_DIR}/usd/src/external_usd < ${PATCH_DIR}/usd_pull_1965.diff &&
${PATCH_CMD} -p 1 -d ${BUILD_DIR}/usd/src/external_usd < ${PATCH_DIR}/usd_hydra.diff
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${LIBDIR}/usd -Wno-dev ${DEFAULT_CMAKE_FLAGS} ${USD_EXTRA_ARGS}
INSTALL_DIR ${LIBDIR}/usd
)
@@ -107,6 +119,7 @@ add_dependencies(
external_opensubdiv
external_python
external_openimageio
external_materialx
openvdb
)

View File

@@ -58,16 +58,16 @@ set(PTHREADS_HASH f3bf81bb395840b3446197bcf4ecd653)
set(PTHREADS_HASH_TYPE MD5)
set(PTHREADS_FILE pthreads4w-code-${PTHREADS_VERSION}.zip)
set(OPENEXR_VERSION 3.1.5)
set(OPENEXR_VERSION 3.1.7)
set(OPENEXR_URI https://github.com/AcademySoftwareFoundation/openexr/archive/v${OPENEXR_VERSION}.tar.gz)
set(OPENEXR_HASH a92f38eedd43e56c0af56d4852506886)
set(OPENEXR_HASH ae68f0cb8b30a49c961fa87d31c60394)
set(OPENEXR_HASH_TYPE MD5)
set(OPENEXR_FILE openexr-${OPENEXR_VERSION}.tar.gz)
set(OPENEXR_CPE "cpe:2.3:a:openexr:openexr:${OPENEXR_VERSION}:*:*:*:*:*:*:*")
set(IMATH_VERSION 3.1.5)
set(IMATH_VERSION 3.1.7)
set(IMATH_URI https://github.com/AcademySoftwareFoundation/Imath/archive/v${OPENEXR_VERSION}.tar.gz)
set(IMATH_HASH dd375574276c54872b7b3d54053baff0)
set(IMATH_HASH 5cedab446ab296c080957c3037c6d097)
set(IMATH_HASH_TYPE MD5)
set(IMATH_FILE imath-${IMATH_VERSION}.tar.gz)
@@ -88,9 +88,9 @@ else()
set(OPENEXR_VERSION_POSTFIX)
endif()
set(FREETYPE_VERSION 2.12.1)
set(FREETYPE_VERSION 2.13.0)
set(FREETYPE_URI http://prdownloads.sourceforge.net/freetype/freetype-${FREETYPE_VERSION}.tar.gz)
set(FREETYPE_HASH 8bc5c9c9df7ac12c504f8918552a7cf2)
set(FREETYPE_HASH 98bc3cf234fe88ef3cf24569251fe0a4)
set(FREETYPE_HASH_TYPE MD5)
set(FREETYPE_FILE freetype-${FREETYPE_VERSION}.tar.gz)
SET(FREETYPE_CPE "cpe:2.3:a:freetype:freetype:${FREETYPE_VERSION}:*:*:*:*:*:*:*")
@@ -112,7 +112,6 @@ set(ALEMBIC_URI https://github.com/alembic/alembic/archive/${ALEMBIC_VERSION}.ta
set(ALEMBIC_HASH 2cd8d6e5a3ac4a014e24a4b04f4fadf9)
set(ALEMBIC_HASH_TYPE MD5)
set(ALEMBIC_FILE alembic-${ALEMBIC_VERSION}.tar.gz)
SET(FREETYPE_CPE "cpe:2.3:a:freetype:freetype:${FREETYPE_VERSION}:*:*:*:*:*:*:*")
set(OPENSUBDIV_VERSION v3_5_0)
set(OPENSUBDIV_URI https://github.com/PixarAnimationStudios/OpenSubdiv/archive/${OPENSUBDIV_VERSION}.tar.gz)
@@ -165,17 +164,17 @@ set(OPENMP_URI https://github.com/llvm/llvm-project/releases/download/llvmorg-${
set(OPENMP_HASH_TYPE MD5)
set(OPENMP_FILE openmp-${OPENMP_VERSION}.src.tar.xz)
set(OPENIMAGEIO_VERSION v2.4.9.0)
set(OPENIMAGEIO_VERSION v2.4.11.0)
set(OPENIMAGEIO_URI https://github.com/OpenImageIO/oiio/archive/refs/tags/${OPENIMAGEIO_VERSION}.tar.gz)
set(OPENIMAGEIO_HASH 7da92a7d6029921a8599a977ff1efa2a)
set(OPENIMAGEIO_HASH 7eb997479ecfe7d9fa59cc8ddd35d0ae)
set(OPENIMAGEIO_HASH_TYPE MD5)
set(OPENIMAGEIO_FILE OpenImageIO-${OPENIMAGEIO_VERSION}.tar.gz)
# 8.0.0 is currently oiio's preferred version although never versions may be available.
# 9.1.0 is currently oiio's preferred version although never versions may be available.
# the preferred version can be found in oiio's externalpackages.cmake
set(FMT_VERSION 8.0.0)
set(FMT_VERSION 9.1.0)
set(FMT_URI https://github.com/fmtlib/fmt/archive/refs/tags/${FMT_VERSION}.tar.gz)
set(FMT_HASH 7bce0e9e022e586b178b150002e7c2339994e3c2bbe44027e9abb0d60f9cce83)
set(FMT_HASH 5dea48d1fcddc3ec571ce2058e13910a0d4a6bab4cc09a809d8b1dd1c88ae6f2)
set(FMT_HASH_TYPE SHA256)
set(FMT_FILE fmt-${FMT_VERSION}.tar.gz)
set(FMT_CPE "cpe:2.3:a:fmt:fmt:${FMT_VERSION}:*:*:*:*:*:*:*")
@@ -209,11 +208,11 @@ set(OSL_FILE OpenShadingLanguage-${OSL_VERSION}.tar.gz)
# BZIP2, FFI, SQLITE and change the versions in this file as well. For compliance
# reasons there can be no exceptions to this.
set(PYTHON_VERSION 3.10.9)
set(PYTHON_VERSION 3.10.11)
set(PYTHON_SHORT_VERSION 3.10)
set(PYTHON_SHORT_VERSION_NO_DOTS 310)
set(PYTHON_URI https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tar.xz)
set(PYTHON_HASH dc8c0f274b28ee9e95923d20cfc364c9)
set(PYTHON_HASH 1bf8481a683e0881e14d52e0f23633a6)
set(PYTHON_HASH_TYPE MD5)
set(PYTHON_FILE Python-${PYTHON_VERSION}.tar.xz)
set(PYTHON_CPE "cpe:2.3:a:python:python:${PYTHON_VERSION}:-:*:*:*:*:*:*")
@@ -296,9 +295,9 @@ set(THEORA_HASH b6ae1ee2fa3d42ac489287d3ec34c5885730b1296f0801ae577a35193d3affbc
set(THEORA_HASH_TYPE SHA256)
set(THEORA_FILE libtheora-${THEORA_VERSION}.tar.bz2)
set(FLAC_VERSION 1.3.4)
set(FLAC_VERSION 1.4.2)
set(FLAC_URI http://downloads.xiph.org/releases/flac/flac-${FLAC_VERSION}.tar.xz)
set(FLAC_HASH 8ff0607e75a322dd7cd6ec48f4f225471404ae2730d0ea945127b1355155e737 )
set(FLAC_HASH e322d58a1f48d23d9dd38f432672865f6f79e73a6f9cc5a5f57fcaa83eb5a8e4 )
set(FLAC_HASH_TYPE SHA256)
set(FLAC_FILE flac-${FLAC_VERSION}.tar.xz)
set(FLAC_CPE "cpe:2.3:a:flac_project:flac:${FLAC_VERSION}:*:*:*:*:*:*:*")
@@ -336,9 +335,9 @@ set(OPENJPEG_HASH_TYPE SHA256)
set(OPENJPEG_FILE openjpeg-v${OPENJPEG_VERSION}.tar.gz)
set(OPENJPEG_CPE "cpe:2.3:a:uclouvain:openjpeg:${OPENJPEG_VERSION}:*:*:*:*:*:*:*")
set(FFMPEG_VERSION 5.1.2)
set(FFMPEG_VERSION 6.0)
set(FFMPEG_URI http://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.bz2)
set(FFMPEG_HASH 39a0bcc8d98549f16c570624678246a6ac736c066cebdb409f9502e915b22f2b)
set(FFMPEG_HASH 47d062731c9f66a78380e35a19aac77cebceccd1c7cc309b9c82343ffc430c3d)
set(FFMPEG_HASH_TYPE SHA256)
set(FFMPEG_FILE ffmpeg-${FFMPEG_VERSION}.tar.bz2)
set(FFMPEG_CPE "cpe:2.3:a:ffmpeg:ffmpeg:${FFMPEG_VERSION}:*:*:*:*:*:*:*")
@@ -460,9 +459,9 @@ set(LZMA_HASH_TYPE SHA256)
set(LZMA_FILE xz-${LZMA_VERSION}.tar.bz2)
# NOTE: Python's build has been modified to use our ssl version.
set(SSL_VERSION 1.1.1q)
set(SSL_VERSION 1.1.1t)
set(SSL_URI https://www.openssl.org/source/openssl-${SSL_VERSION}.tar.gz)
set(SSL_HASH d7939ce614029cdff0b6c20f0e2e5703158a489a72b2507b8bd51bf8c8fd10ca)
set(SSL_HASH 8dee9b24bdb1dcbf0c3d1e9b02fb8f6bf22165e807f45adeb7c9677536859d3b)
set(SSL_HASH_TYPE SHA256)
set(SSL_FILE openssl-${SSL_VERSION}.tar.gz)
set(SSL_CPE "cpe:2.3:a:openssl:openssl:${SSL_VERSION}:*:*:*:*:*:*:*")
@@ -470,10 +469,10 @@ set(SSL_CPE "cpe:2.3:a:openssl:openssl:${SSL_VERSION}:*:*:*:*:*:*:*")
# Note: This will *HAVE* to match the version python ships on windows which
# is hardcoded in pythons PCbuild/get_externals.bat for compliance reasons there
# can be no exceptions to this.
set(SQLITE_VERSION 3.39.4)
set(SQLLITE_LONG_VERSION 3390400)
set(SQLITE_VERSION 3.40.1)
set(SQLLITE_LONG_VERSION 3400100)
set(SQLITE_URI https://www.sqlite.org/2022/sqlite-autoconf-${SQLLITE_LONG_VERSION}.tar.gz)
set(SQLITE_HASH c4c5c39269d1b9bb1487cff580c1f583608229b2)
set(SQLITE_HASH b8c2d4bc0094f5c0ce985dc0e237dfcbaa1f6275)
set(SQLITE_HASH_TYPE SHA1)
set(SQLITE_FILE sqlite-autoconf-${SQLLITE_LONG_VERSION}.tar.gz)
set(SQLITE_CPE "cpe:2.3:a:sqlite:sqlite:${SQLITE_VERSION}:*:*:*:*:*:*:*")
@@ -484,9 +483,9 @@ set(EMBREE_HASH dd26617719a587e126b341d1b32f7fd0)
set(EMBREE_HASH_TYPE MD5)
set(EMBREE_FILE embree-v${EMBREE_VERSION}.zip)
set(USD_VERSION 22.11)
set(USD_VERSION 23.05)
set(USD_URI https://github.com/PixarAnimationStudios/USD/archive/v${USD_VERSION}.tar.gz)
set(USD_HASH 8c89459e48a2ef0e7ae9e7e490377507)
set(USD_HASH 56684f4fdd1a9209dabf03856be5eca6)
set(USD_HASH_TYPE MD5)
set(USD_FILE usd-v${USD_VERSION}.tar.gz)

View File

@@ -9,77 +9,3 @@
enabled libopenmpt && require_pkg_config libopenmpt "libopenmpt >= 0.2.6557" libopenmpt/libopenmpt.h openmpt_module_create -lstdc++ && append libopenmpt_extralibs "-lstdc++"
enabled libopus && {
enabled libopus_decoder && {
--- a/libavcodec/cfhddata.c
+++ b/libavcodec/cfhddata.c
@@ -276,10 +276,10 @@
av_cold int ff_cfhd_init_vlcs(CFHDContext *s)
{
int i, j, ret = 0;
- uint32_t new_cfhd_vlc_bits[NB_VLC_TABLE_18 * 2];
- uint8_t new_cfhd_vlc_len[NB_VLC_TABLE_18 * 2];
- uint16_t new_cfhd_vlc_run[NB_VLC_TABLE_18 * 2];
- int16_t new_cfhd_vlc_level[NB_VLC_TABLE_18 * 2];
+ uint32_t *new_cfhd_vlc_bits = av_calloc(sizeof(uint32_t), NB_VLC_TABLE_18 * 2);
+ uint8_t *new_cfhd_vlc_len = av_calloc(sizeof(uint8_t), NB_VLC_TABLE_18 * 2);
+ uint16_t *new_cfhd_vlc_run = av_calloc(sizeof(uint16_t), NB_VLC_TABLE_18 * 2);
+ int16_t *new_cfhd_vlc_level = av_calloc(sizeof(int16_t), NB_VLC_TABLE_18 * 2);
/** Similar to dv.c, generate signed VLC tables **/
@@ -305,8 +305,13 @@
ret = init_vlc(&s->vlc_9, VLC_BITS, j, new_cfhd_vlc_len,
1, 1, new_cfhd_vlc_bits, 4, 4, 0);
- if (ret < 0)
+ if (ret < 0) {
+ av_free(new_cfhd_vlc_bits);
+ av_free(new_cfhd_vlc_len);
+ av_free(new_cfhd_vlc_run);
+ av_free(new_cfhd_vlc_level);
return ret;
+ }
for (i = 0; i < s->vlc_9.table_size; i++) {
int code = s->vlc_9.table[i][0];
int len = s->vlc_9.table[i][1];
@@ -346,8 +351,14 @@
ret = init_vlc(&s->vlc_18, VLC_BITS, j, new_cfhd_vlc_len,
1, 1, new_cfhd_vlc_bits, 4, 4, 0);
- if (ret < 0)
+ if (ret < 0) {
+ av_free(new_cfhd_vlc_bits);
+ av_free(new_cfhd_vlc_len);
+ av_free(new_cfhd_vlc_run);
+ av_free(new_cfhd_vlc_level);
return ret;
+ }
+
av_assert0(s->vlc_18.table_size == 4572);
for (i = 0; i < s->vlc_18.table_size; i++) {
@@ -367,5 +378,10 @@
s->table_18_rl_vlc[i].run = run;
}
+ av_free(new_cfhd_vlc_bits);
+ av_free(new_cfhd_vlc_len);
+ av_free(new_cfhd_vlc_run);
+ av_free(new_cfhd_vlc_level);
+
return ret;
}
diff --git a/libavcodec/x86/simple_idct.asm b/libavcodec/x86/simple_idct.asm
index dcf0da6df121..982b2f0bbba1 100644
--- a/libavcodec/x86/simple_idct.asm
+++ b/libavcodec/x86/simple_idct.asm
@@ -25,9 +25,9 @@
%include "libavutil/x86/x86util.asm"
-%if ARCH_X86_32
SECTION_RODATA
+%if ARCH_X86_32
cextern pb_80
wm1010: dw 0, 0xffff, 0, 0xffff

View File

@@ -0,0 +1,13 @@
diff --git a/src/python/py_oiio.cpp b/src/python/py_oiio.cpp
index 6031d2c23..e71105da5 100644
--- a/src/python/py_oiio.cpp
+++ b/src/python/py_oiio.cpp
@@ -153,7 +153,7 @@ oiio_bufinfo::oiio_bufinfo(const py::buffer_info& pybuf, int nchans, int width,
format = TypeUnknown; // No idea what's going on -- error
error = Strutil::fmt::format(
"Python array shape is [{:,}] but expecting h={}, w={}, ch={}",
- cspan<ssize_t>(pybuf.shape), height, width, nchans);
+ cspan<py::ssize_t>(pybuf.shape), height, width, nchans);
}
} else if (pixeldims == 1) {
// Reading a 1D scanline span

View File

@@ -0,0 +1,1023 @@
From b18905772e9dd98658f8a37d16c6e53c7c17adaa Mon Sep 17 00:00:00 2001
From: Kimball Thurston <kdt3rd@gmail.com>
Date: Sat, 22 Apr 2023 15:58:43 +1200
Subject: [PATCH] Change setNumThreads to wait for thread start (#1291)
Fix Issue #890, issue with windows shutdown when exiting quickly prior to
threads actually starting. We do this by having a data block that is passed
by shared_ptr to the thread to avoid dereferencing a deleted object.
Further, greatly simplify the ThreadPool code by using atomic shared_ptr
functions instead of trying to manually implement something similar and
otherwise modernizing the thread code.
Fix a few potential null dereference locations and also fix an issue when
systems are overloaded enabling the TaskGroup destructor to destroy
the semaphore while code is still using it, causing undefined memory
corruption if some other thread immediately allocates that same block
Originally based on a proposed patch by Dieter De Baets @debaetsd
Signed-off-by: Kimball Thurston <kdt3rd@gmail.com>
---
src/lib/IlmThread/IlmThreadPool.cpp | 740 ++++++++++---------------
src/lib/IlmThread/IlmThreadSemaphore.h | 8 +-
2 files changed, 303 insertions(+), 445 deletions(-)
diff --git a/src/lib/IlmThread/IlmThreadPool.cpp b/src/lib/IlmThread/IlmThreadPool.cpp
index 0ddcf8d52..f4578a510 100644
--- a/src/lib/IlmThread/IlmThreadPool.cpp
+++ b/src/lib/IlmThread/IlmThreadPool.cpp
@@ -15,12 +15,17 @@
#include "Iex.h"
#include <atomic>
+#include <limits>
#include <memory>
#include <mutex>
#include <vector>
#include <thread>
-using namespace std;
+#if (defined(_WIN32) || defined(_WIN64))
+# include <windows.h>
+#else
+# include <unistd.h>
+#endif
ILMTHREAD_INTERNAL_NAMESPACE_SOURCE_ENTER
@@ -28,380 +33,301 @@ ILMTHREAD_INTERNAL_NAMESPACE_SOURCE_ENTER
# define ENABLE_THREADING
#endif
-#if defined(__GNU_LIBRARY__) && ( __GLIBC__ < 2 || ( __GLIBC__ == 2 && __GLIBC_MINOR__ < 21 ) )
-# define ENABLE_SEM_DTOR_WORKAROUND
-#endif
-
-#ifdef ENABLE_THREADING
-
-struct TaskGroup::Data
+namespace
{
- Data ();
- ~Data ();
-
- void addTask () ;
- void removeTask ();
- std::atomic<int> numPending;
- Semaphore isEmpty; // used to signal that the taskgroup is empty
-#if defined(ENABLE_SEM_DTOR_WORKAROUND)
- // this mutex is also used to lock numPending in the legacy c++ mode...
- std::mutex dtorMutex; // used to work around the glibc bug:
- // http://sources.redhat.com/bugzilla/show_bug.cgi?id=12674
-#endif
-};
-
-struct ThreadPool::Data
+static inline void
+handleProcessTask (Task* task)
{
- typedef ThreadPoolProvider *TPPointer;
-
- Data ();
- ~Data();
- Data (const Data&) = delete;
- Data &operator= (const Data&) = delete;
- Data (Data&&) = delete;
- Data &operator= (Data&&) = delete;
-
- struct SafeProvider
+ if (task)
{
- SafeProvider (Data *d, ThreadPoolProvider *p) : _data( d ), _ptr( p )
- {
- }
-
- ~SafeProvider()
- {
- if ( _data )
- _data->coalesceProviderUse();
- }
- SafeProvider (const SafeProvider &o)
- : _data( o._data ), _ptr( o._ptr )
- {
- if ( _data )
- _data->bumpProviderUse();
- }
- SafeProvider &operator= (const SafeProvider &o)
- {
- if ( this != &o )
- {
- if ( o._data )
- o._data->bumpProviderUse();
- if ( _data )
- _data->coalesceProviderUse();
- _data = o._data;
- _ptr = o._ptr;
- }
- return *this;
- }
- SafeProvider( SafeProvider &&o )
- : _data( o._data ), _ptr( o._ptr )
- {
- o._data = nullptr;
- }
- SafeProvider &operator=( SafeProvider &&o )
- {
- std::swap( _data, o._data );
- std::swap( _ptr, o._ptr );
- return *this;
- }
-
- inline ThreadPoolProvider *get () const
- {
- return _ptr;
- }
- ThreadPoolProvider *operator-> () const
- {
- return get();
- }
-
- Data *_data;
- ThreadPoolProvider *_ptr;
- };
-
- // NB: In C++20, there is full support for atomic shared_ptr, but that is not
- // yet in use or finalized. Once stabilized, add appropriate usage here
- inline SafeProvider getProvider ();
- inline void coalesceProviderUse ();
- inline void bumpProviderUse ();
- inline void setProvider (ThreadPoolProvider *p);
-
- std::atomic<int> provUsers;
- std::atomic<ThreadPoolProvider *> provider;
-};
-
+ TaskGroup* taskGroup = task->group ();
+ task->execute ();
-namespace {
+ // kill the task prior to notifying the group
+ // such that any internal reference-based
+ // semantics will be handled prior to
+ // the task group destructor letting it out
+ // of the scope of those references
+ delete task;
-class DefaultWorkerThread;
+ if (taskGroup) taskGroup->finishOneTask ();
+ }
+}
-struct DefaultWorkData
+#ifdef ENABLE_THREADING
+struct DefaultThreadPoolData
{
- Semaphore taskSemaphore; // threads wait on this for ready tasks
- mutable std::mutex taskMutex; // mutual exclusion for the tasks list
- vector<Task*> tasks; // the list of tasks to execute
+ Semaphore _taskSemaphore; // threads wait on this for ready tasks
+ mutable std::mutex _taskMutex; // mutual exclusion for the tasks list
+ std::vector<Task*> _tasks; // the list of tasks to execute
- Semaphore threadSemaphore; // signaled when a thread starts executing
- mutable std::mutex threadMutex; // mutual exclusion for threads list
- vector<DefaultWorkerThread*> threads; // the list of all threads
-
- std::atomic<bool> hasThreads;
- std::atomic<bool> stopping;
+ mutable std::mutex _threadMutex; // mutual exclusion for threads list
+ std::vector<std::thread> _threads; // the list of all threads
+
+ std::atomic<int> _threadCount;
+ std::atomic<bool> _stopping;
inline bool stopped () const
{
- return stopping.load( std::memory_order_relaxed );
+ return _stopping.load (std::memory_order_relaxed);
}
- inline void stop ()
+ inline void stop () { _stopping = true; }
+
+ inline void resetAtomics ()
{
- stopping = true;
+ _threadCount = 0;
+ _stopping = false;
}
};
+#endif
-//
-// class WorkerThread
-//
-class DefaultWorkerThread: public Thread
+} // namespace
+
+#ifdef ENABLE_THREADING
+
+struct TaskGroup::Data
{
- public:
+ Data ();
+ ~Data ();
+ Data (const Data&) = delete;
+ Data& operator= (const Data&) = delete;
+ Data (Data&&) = delete;
+ Data& operator= (Data&&) = delete;
- DefaultWorkerThread (DefaultWorkData* data);
+ void addTask ();
+ void removeTask ();
- virtual void run ();
-
- private:
+ void waitForEmpty ();
- DefaultWorkData * _data;
+ std::atomic<int> numPending;
+ std::atomic<int> inFlight;
+ Semaphore isEmpty; // used to signal that the taskgroup is empty
};
-DefaultWorkerThread::DefaultWorkerThread (DefaultWorkData* data):
- _data (data)
+struct ThreadPool::Data
{
- start();
-}
-
+ using ProviderPtr = std::shared_ptr<ThreadPoolProvider>;
-void
-DefaultWorkerThread::run ()
-{
- //
- // Signal that the thread has started executing
- //
+ Data ();
+ ~Data ();
+ Data (const Data&) = delete;
+ Data& operator= (const Data&) = delete;
+ Data (Data&&) = delete;
+ Data& operator= (Data&&) = delete;
- _data->threadSemaphore.post();
+ ProviderPtr getProvider () const { return std::atomic_load (&_provider); }
- while (true)
+ void setProvider (ProviderPtr provider)
{
- //
- // Wait for a task to become available
- //
-
- _data->taskSemaphore.wait();
-
- {
- std::unique_lock<std::mutex> taskLock (_data->taskMutex);
-
- //
- // If there is a task pending, pop off the next task in the FIFO
- //
+ ProviderPtr curp = std::atomic_exchange (&_provider, provider);
+ if (curp && curp != provider) curp->finish ();
+ }
- if (!_data->tasks.empty())
- {
- Task* task = _data->tasks.back();
- _data->tasks.pop_back();
- // release the mutex while we process
- taskLock.unlock();
+ std::shared_ptr<ThreadPoolProvider> _provider;
+};
- TaskGroup* taskGroup = task->group();
- task->execute();
- delete task;
-
- taskGroup->_data->removeTask ();
- }
- else if (_data->stopped())
- {
- break;
- }
- }
- }
-}
+namespace {
//
// class DefaultThreadPoolProvider
//
class DefaultThreadPoolProvider : public ThreadPoolProvider
{
- public:
- DefaultThreadPoolProvider(int count);
- virtual ~DefaultThreadPoolProvider();
+public:
+ DefaultThreadPoolProvider (int count);
+ DefaultThreadPoolProvider (const DefaultThreadPoolProvider&) = delete;
+ DefaultThreadPoolProvider&
+ operator= (const DefaultThreadPoolProvider&) = delete;
+ DefaultThreadPoolProvider (DefaultThreadPoolProvider&&) = delete;
+ DefaultThreadPoolProvider& operator= (DefaultThreadPoolProvider&&) = delete;
+ ~DefaultThreadPoolProvider () override;
+
+ int numThreads () const override;
+ void setNumThreads (int count) override;
+ void addTask (Task* task) override;
- virtual int numThreads() const;
- virtual void setNumThreads(int count);
- virtual void addTask(Task *task);
+ void finish () override;
- virtual void finish();
+private:
+ void lockedFinish ();
+ void threadLoop (std::shared_ptr<DefaultThreadPoolData> d);
- private:
- DefaultWorkData _data;
+ std::shared_ptr<DefaultThreadPoolData> _data;
};
DefaultThreadPoolProvider::DefaultThreadPoolProvider (int count)
+ : _data (std::make_shared<DefaultThreadPoolData> ())
{
- setNumThreads(count);
+ _data->resetAtomics ();
+ setNumThreads (count);
}
DefaultThreadPoolProvider::~DefaultThreadPoolProvider ()
-{
- finish();
-}
+{}
int
DefaultThreadPoolProvider::numThreads () const
{
- std::lock_guard<std::mutex> lock (_data.threadMutex);
- return static_cast<int> (_data.threads.size());
+ return _data->_threadCount.load ();
}
void
DefaultThreadPoolProvider::setNumThreads (int count)
{
- //
- // Lock access to thread list and size
- //
+ // since we're a private class, the thread pool won't call us if
+ // we aren't changing size so no need to check that...
- std::lock_guard<std::mutex> lock (_data.threadMutex);
+ std::lock_guard<std::mutex> lock (_data->_threadMutex);
- size_t desired = static_cast<size_t>(count);
- if (desired > _data.threads.size())
- {
- //
- // Add more threads
- //
+ size_t curThreads = _data->_threads.size ();
+ size_t nToAdd = static_cast<size_t> (count);
- while (_data.threads.size() < desired)
- _data.threads.push_back (new DefaultWorkerThread (&_data));
- }
- else if ((size_t)count < _data.threads.size())
+ if (nToAdd < curThreads)
{
- //
- // Wait until all existing threads are finished processing,
- // then delete all threads.
- //
- finish ();
-
- //
- // Add in new threads
- //
-
- while (_data.threads.size() < desired)
- _data.threads.push_back (new DefaultWorkerThread (&_data));
+ // no easy way to only shutdown the n threads at the end of
+ // the vector (well, really, guaranteeing they are the ones to
+ // be woken up), so just kill all of the threads
+ lockedFinish ();
+ curThreads = 0;
}
- _data.hasThreads = !(_data.threads.empty());
+ _data->_threads.resize (nToAdd);
+ for (size_t i = curThreads; i < nToAdd; ++i)
+ {
+ _data->_threads[i] =
+ std::thread (&DefaultThreadPoolProvider::threadLoop, this, _data);
+ }
+ _data->_threadCount = static_cast<int> (_data->_threads.size ());
}
void
DefaultThreadPoolProvider::addTask (Task *task)
{
- //
- // Lock the threads, needed to access numThreads
- //
- bool doPush = _data.hasThreads.load( std::memory_order_relaxed );
-
- if ( doPush )
+ // the thread pool will kill us and switch to a null provider
+ // if the thread count is set to 0, so we can always
+ // go ahead and lock and assume we have a thread to do the
+ // processing
{
- //
- // Get exclusive access to the tasks queue
- //
+ std::lock_guard<std::mutex> taskLock (_data->_taskMutex);
- {
- std::lock_guard<std::mutex> taskLock (_data.taskMutex);
-
- //
- // Push the new task into the FIFO
- //
- _data.tasks.push_back (task);
- }
-
//
- // Signal that we have a new task to process
+ // Push the new task into the FIFO
//
- _data.taskSemaphore.post ();
- }
- else
- {
- // this path shouldn't normally happen since we have the
- // NullThreadPoolProvider, but just in case...
- task->execute ();
- task->group()->_data->removeTask ();
- delete task;
+ _data->_tasks.push_back (task);
}
+
+ //
+ // Signal that we have a new task to process
+ //
+ _data->_taskSemaphore.post ();
}
void
DefaultThreadPoolProvider::finish ()
{
- _data.stop();
+ std::lock_guard<std::mutex> lock (_data->_threadMutex);
+
+ lockedFinish ();
+}
+
+void
+DefaultThreadPoolProvider::lockedFinish ()
+{
+ _data->stop ();
//
// Signal enough times to allow all threads to stop.
//
- // Wait until all threads have started their run functions.
- // If we do not wait before we destroy the threads then it's
- // possible that the threads have not yet called their run
- // functions.
- // If this happens then the run function will be called off
- // of an invalid object and we will crash, most likely with
- // an error like: "pure virtual method called"
+ // NB: we must do this as many times as we have threads.
//
-
- size_t curT = _data.threads.size();
+ // If there is still work in the queue, or this call happens "too
+ // quickly", threads will not be waiting on the semaphore, so we
+ // need to ensure the semaphore is at a count equal to the amount
+ // of work left plus the number of threads to ensure exit of a
+ // thread. There can be threads in a few states:
+ // - still starting up (successive calls to setNumThreads)
+ // - in the middle of processing a task / looping
+ // - waiting in the semaphore
+ size_t curT = _data->_threads.size ();
for (size_t i = 0; i != curT; ++i)
- {
- if (_data.threads[i]->joinable())
- {
- _data.taskSemaphore.post();
- _data.threadSemaphore.wait();
- }
- }
+ _data->_taskSemaphore.post ();
//
- // Join all the threads
+ // We should not need to check joinability, they should all, by
+ // definition, be joinable (assuming normal start)
//
for (size_t i = 0; i != curT; ++i)
{
- if (_data.threads[i]->joinable())
- _data.threads[i]->join();
- delete _data.threads[i];
+ // This isn't quite right in that the thread may have actually
+ // be in an exited / signalled state (needing the
+ // WaitForSingleObject call), and so already have an exit code
+ // (I think, but the docs are vague), but if we don't do the
+ // join, the stl thread seems to then throw an exception. The
+ // join should just return invalid handle and continue, and is
+ // more of a windows bug... except maybe someone needs to work
+ // around it...
+ //# ifdef TEST_FOR_WIN_THREAD_STATUS
+ //
+ // // per OIIO issue #2038, on exit / dll unload, windows may
+ // // kill the thread, double check that it is still active prior
+ // // to joining.
+ // DWORD tstatus;
+ // if (GetExitCodeThread (_threads[i].native_handle (), &tstatus))
+ // {
+ // if (tstatus != STILL_ACTIVE) { continue; }
+ // }
+ //# endif
+
+ _data->_threads[i].join ();
}
- std::lock_guard<std::mutex> lk( _data.taskMutex );
-
- _data.threads.clear();
- _data.tasks.clear();
+ _data->_threads.clear ();
- _data.stopping = false;
+ _data->resetAtomics ();
}
-
-class NullThreadPoolProvider : public ThreadPoolProvider
+void
+DefaultThreadPoolProvider::threadLoop (
+ std::shared_ptr<DefaultThreadPoolData> data)
{
- virtual ~NullThreadPoolProvider() {}
- virtual int numThreads () const { return 0; }
- virtual void setNumThreads (int count)
- {
- }
- virtual void addTask (Task *t)
+ while (true)
{
- t->execute ();
- t->group()->_data->removeTask ();
- delete t;
+ //
+ // Wait for a task to become available
+ //
+
+ data->_taskSemaphore.wait ();
+
+ {
+ std::unique_lock<std::mutex> taskLock (data->_taskMutex);
+
+ //
+ // If there is a task pending, pop off the next task in the FIFO
+ //
+
+ if (!data->_tasks.empty ())
+ {
+ Task* task = data->_tasks.back ();
+ data->_tasks.pop_back ();
+
+ // release the mutex while we process
+ taskLock.unlock ();
+
+ handleProcessTask (task);
+
+ // do not need to reacquire the lock at all since we
+ // will just loop around, pull any other task
+ }
+ else if (data->stopped ()) { break; }
+ }
}
- virtual void finish () {}
-};
+}
} //namespace
@@ -409,81 +335,69 @@ class NullThreadPoolProvider : public ThreadPoolProvider
// struct TaskGroup::Data
//
-TaskGroup::Data::Data () : numPending (0), isEmpty (1)
-{
- // empty
-}
+TaskGroup::Data::Data () : numPending (0), inFlight (0), isEmpty (1)
+{}
TaskGroup::Data::~Data ()
+{}
+
+void
+TaskGroup::Data::waitForEmpty ()
{
//
// A TaskGroup acts like an "inverted" semaphore: if the count
- // is above 0 then waiting on the taskgroup will block. This
+ // is above 0 then waiting on the taskgroup will block. The
// destructor waits until the taskgroup is empty before returning.
//
isEmpty.wait ();
-#ifdef ENABLE_SEM_DTOR_WORKAROUND
- // Update: this was fixed in v. 2.2.21, so this ifdef checks for that
- //
- // Alas, given the current bug in glibc we need a secondary
- // syncronisation primitive here to account for the fact that
- // destructing the isEmpty Semaphore in this thread can cause
- // an error for a separate thread that is issuing the post() call.
- // We are entitled to destruct the semaphore at this point, however,
- // that post() call attempts to access data out of the associated
- // memory *after* it has woken the waiting threads, including this one,
- // potentially leading to invalid memory reads.
- // http://sources.redhat.com/bugzilla/show_bug.cgi?id=12674
-
- std::lock_guard<std::mutex> lock (dtorMutex);
-#endif
+ // pseudo spin to wait for the notifying thread to finish the post
+ // to avoid a premature deletion of the semaphore
+ int count = 0;
+ while (inFlight.load () > 0)
+ {
+ ++count;
+ if (count > 100)
+ {
+ std::this_thread::yield ();
+ count = 0;
+ }
+ }
}
void
TaskGroup::Data::addTask ()
{
- //
- // in c++11, we use an atomic to protect numPending to avoid the
- // extra lock but for c++98, to add the ability for custom thread
- // pool we add the lock here
- //
- if (numPending++ == 0)
- isEmpty.wait ();
+ inFlight.fetch_add (1);
+
+ // if we are the first task off the rank, clear the
+ // isEmpty semaphore such that the group will actually pause
+ // until the task finishes
+ if (numPending.fetch_add (1) == 0) { isEmpty.wait (); }
}
void
TaskGroup::Data::removeTask ()
{
- // Alas, given the current bug in glibc we need a secondary
- // syncronisation primitive here to account for the fact that
- // destructing the isEmpty Semaphore in a separate thread can
- // cause an error. Issuing the post call here the current libc
- // implementation attempts to access memory *after* it has woken
- // waiting threads.
- // Since other threads are entitled to delete the semaphore the
- // access to the memory location can be invalid.
- // http://sources.redhat.com/bugzilla/show_bug.cgi?id=12674
- // Update: this bug has been fixed, but how do we know which
- // glibc version we're in?
-
- // Further update:
+ // if we are the last task, notify the group we're done
+ if (numPending.fetch_sub (1) == 1) { isEmpty.post (); }
+
+ // in theory, a background thread could actually finish a task
+ // prior to the next task being added. The fetch_add / fetch_sub
+ // logic between addTask and removeTask are fine to keep the
+ // inverted semaphore straight. All addTask must happen prior to
+ // the TaskGroup destructor.
//
- // we could remove this if it is a new enough glibc, however
- // we've changed the API to enable a custom override of a
- // thread pool. In order to provide safe access to the numPending,
- // we need the lock anyway, except for c++11 or newer
- if (--numPending == 0)
- {
-#ifdef ENABLE_SEM_DTOR_WORKAROUND
- std::lock_guard<std::mutex> lk (dtorMutex);
-#endif
- isEmpty.post ();
- }
+ // But to let the taskgroup thread waiting know we're actually
+ // finished with the last one and finished posting (the semaphore
+ // might wake up the other thread while in the middle of post) so
+ // we don't destroy the semaphore while posting to it, keep a
+ // separate counter that is modified pre / post semaphore
+ inFlight.fetch_sub (1);
}
@@ -491,8 +405,7 @@ TaskGroup::Data::removeTask ()
// struct ThreadPool::Data
//
-ThreadPool::Data::Data ():
- provUsers (0), provider (NULL)
+ThreadPool::Data::Data ()
{
// empty
}
@@ -500,82 +413,7 @@ ThreadPool::Data::Data ():
ThreadPool::Data::~Data()
{
- ThreadPoolProvider *p = provider.load( std::memory_order_relaxed );
- p->finish();
- delete p;
-}
-
-inline ThreadPool::Data::SafeProvider
-ThreadPool::Data::getProvider ()
-{
- provUsers.fetch_add( 1, std::memory_order_relaxed );
- return SafeProvider( this, provider.load( std::memory_order_relaxed ) );
-}
-
-
-inline void
-ThreadPool::Data::coalesceProviderUse ()
-{
- int ov = provUsers.fetch_sub( 1, std::memory_order_relaxed );
- // ov is the previous value, so one means that now it might be 0
- if ( ov == 1 )
- {
- // do we have anything to do here?
- }
-}
-
-
-inline void
-ThreadPool::Data::bumpProviderUse ()
-{
- provUsers.fetch_add( 1, std::memory_order_relaxed );
-}
-
-
-inline void
-ThreadPool::Data::setProvider (ThreadPoolProvider *p)
-{
- ThreadPoolProvider *old = provider.load( std::memory_order_relaxed );
- // work around older gcc bug just in case
- do
- {
- if ( ! provider.compare_exchange_weak( old, p, std::memory_order_release, std::memory_order_relaxed ) )
- continue;
- } while (false); // NOSONAR - suppress SonarCloud bug report.
-
- // wait for any other users to finish prior to deleting, given
- // that these are just mostly to query the thread count or push a
- // task to the queue (so fast), just spin...
- //
- // (well, and normally, people don't do this mid stream anyway, so
- // this will be 0 99.999% of the time, but just to be safe)
- //
- while ( provUsers.load( std::memory_order_relaxed ) > 0 )
- std::this_thread::yield();
-
- if ( old )
- {
- old->finish();
- delete old;
- }
-
- // NB: the shared_ptr mechanism is safer and means we don't have
- // to have the provUsers counter since the shared_ptr keeps that
- // for us. However, gcc 4.8/9 compilers which many people are
- // still using even though it is 2018 forgot to add the shared_ptr
- // functions... once that compiler is fully deprecated, switch to
- // using the below, change provider to a std::shared_ptr and remove
- // provUsers...
- //
-// std::shared_ptr<ThreadPoolProvider> newp( p );
-// std::shared_ptr<ThreadPoolProvider> curp = std::atomic_load_explicit( &provider, std::memory_order_relaxed );
-// do
-// {
-// if ( ! std::atomic_compare_exchange_weak_explicit( &provider, &curp, newp, std::memory_order_release, std::memory_order_relaxed ) )
-// continue;
-// } while ( false );
-// if ( curp )
-// curp->finish();
+ setProvider (nullptr);
}
#endif // ENABLE_THREADING
@@ -608,7 +446,7 @@ Task::group ()
TaskGroup::TaskGroup ():
#ifdef ENABLE_THREADING
- _data (new Data())
+ _data (new Data)
#else
_data (nullptr)
#endif
@@ -620,6 +458,7 @@ TaskGroup::TaskGroup ():
TaskGroup::~TaskGroup ()
{
#ifdef ENABLE_THREADING
+ _data->waitForEmpty ();
delete _data;
#endif
}
@@ -660,10 +499,7 @@ ThreadPool::ThreadPool (unsigned nthreads):
#endif
{
#ifdef ENABLE_THREADING
- if ( nthreads == 0 )
- _data->setProvider( new NullThreadPoolProvider );
- else
- _data->setProvider( new DefaultThreadPoolProvider( int(nthreads) ) );
+ setNumThreads (static_cast<int> (nthreads));
#endif
}
@@ -671,6 +507,8 @@ ThreadPool::ThreadPool (unsigned nthreads):
ThreadPool::~ThreadPool ()
{
#ifdef ENABLE_THREADING
+ // ensures any jobs / threads are finished & shutdown
+ _data->setProvider (nullptr);
delete _data;
#endif
}
@@ -680,7 +518,8 @@ int
ThreadPool::numThreads () const
{
#ifdef ENABLE_THREADING
- return _data->getProvider ()->numThreads ();
+ Data::ProviderPtr sp = _data->getProvider ();
+ return (sp) ? sp->numThreads () : 0;
#else
return 0;
#endif
@@ -695,36 +534,30 @@ ThreadPool::setNumThreads (int count)
throw IEX_INTERNAL_NAMESPACE::ArgExc ("Attempt to set the number of threads "
"in a thread pool to a negative value.");
- bool doReset = false;
{
- Data::SafeProvider sp = _data->getProvider ();
- int curT = sp->numThreads ();
- if ( curT == count )
- return;
-
- if ( curT == 0 )
+ Data::ProviderPtr sp = _data->getProvider ();
+ if (sp)
{
- NullThreadPoolProvider *npp = dynamic_cast<NullThreadPoolProvider *>( sp.get() );
- if ( npp )
- doReset = true;
- }
- else if ( count == 0 )
- {
- DefaultThreadPoolProvider *dpp = dynamic_cast<DefaultThreadPoolProvider *>( sp.get() );
- if ( dpp )
- doReset = true;
+ bool doReset = false;
+ int curT = sp->numThreads ();
+ if (curT == count) return;
+
+ if (count != 0)
+ {
+ sp->setNumThreads (count);
+ return;
+ }
}
- if ( ! doReset )
- sp->setNumThreads( count );
}
- if ( doReset )
- {
- if ( count == 0 )
- _data->setProvider( new NullThreadPoolProvider );
- else
- _data->setProvider( new DefaultThreadPoolProvider( count ) );
- }
+ // either a null provider or a case where we should switch from
+ // a default provider to a null one or vice-versa
+ if (count == 0)
+ _data->setProvider (nullptr);
+ else
+ _data->setProvider (
+ std::make_shared<DefaultThreadPoolProvider> (count));
+
#else
// just blindly ignore
(void)count;
@@ -736,7 +569,8 @@ void
ThreadPool::setThreadProvider (ThreadPoolProvider *provider)
{
#ifdef ENABLE_THREADING
- _data->setProvider (provider);
+ // contract is we take ownership and will free the provider
+ _data->setProvider (Data::ProviderPtr (provider));
#else
throw IEX_INTERNAL_NAMESPACE::ArgExc (
"Attempt to set a thread provider on a system with threads"
@@ -748,12 +582,19 @@ ThreadPool::setThreadProvider (ThreadPoolProvider *provider)
void
ThreadPool::addTask (Task* task)
{
+ if (task)
+ {
#ifdef ENABLE_THREADING
- _data->getProvider ()->addTask (task);
-#else
- task->execute ();
- delete task;
+ Data::ProviderPtr p = _data->getProvider ();
+ if (p)
+ {
+ p->addTask (task);
+ return;
+ }
#endif
+
+ handleProcessTask (task);
+ }
}
@@ -780,7 +621,24 @@ unsigned
ThreadPool::estimateThreadCountForFileIO ()
{
#ifdef ENABLE_THREADING
- return std::thread::hardware_concurrency ();
+ unsigned rv = std::thread::hardware_concurrency ();
+ // hardware concurrency is not required to work
+ if (rv == 0 ||
+ rv > static_cast<unsigned> (std::numeric_limits<int>::max ()))
+ {
+ rv = 1;
+# if (defined(_WIN32) || defined(_WIN64))
+ SYSTEM_INFO si;
+ GetNativeSystemInfo (&si);
+
+ rv = si.dwNumberOfProcessors;
+# else
+ // linux, bsd, and mac are fine with this
+ // other *nix should be too, right?
+ rv = sysconf (_SC_NPROCESSORS_ONLN);
+# endif
+ }
+ return rv;
#else
return 0;
#endif
diff --git a/src/lib/IlmThread/IlmThreadSemaphore.h b/src/lib/IlmThread/IlmThreadSemaphore.h
index f26e48a09..576968aa6 100644
--- a/src/lib/IlmThread/IlmThreadSemaphore.h
+++ b/src/lib/IlmThread/IlmThreadSemaphore.h
@@ -64,10 +64,10 @@ class ILMTHREAD_EXPORT_TYPE Semaphore
mutable HANDLE _semaphore;
#elif ILMTHREAD_THREADING_ENABLED
- //
- // If the platform has threads but no semapohores,
- // then we implement them ourselves using condition variables
- //
+ //
+ // If the platform has threads but no semaphores,
+ // then we implement them ourselves using condition variables
+ //
struct sema_t
{

View File

@@ -0,0 +1,37 @@
diff --git a/pxr/usd/usdMtlx/reader.cpp b/pxr/usd/usdMtlx/reader.cpp
index 29e901816..e6fc68b20 100644
--- a/pxr/usd/usdMtlx/reader.cpp
+++ b/pxr/usd/usdMtlx/reader.cpp
@@ -797,6 +797,15 @@ _NodeGraphBuilder::_CreateInterfaceInputs(
// We deliberately ignore tokens here.
}
+mx::StringSet _GetStdlibIncludes() {
+ mx::StringSet stdlibIncludes = UsdMtlxGetDocument("")->getReferencedSourceUris();
+ mx::StringSet normStdlibIncludes;
+ for (std::string const& entry : stdlibIncludes) {
+ normStdlibIncludes.insert(TfNormPath(entry));
+ }
+ return normStdlibIncludes;
+}
+
// Returns True if the mtlxNodeDef corresponds to a locally defined custom node
// with an associated nodegraph.
// XXX Locally defined custom nodes without nodegraphs are not supported
@@ -818,13 +827,14 @@ _NodeGraphBuilder::_IsLocalCustomNode(const mx::ConstNodeDefPtr &mtlxNodeDef)
}
// Combine with the nodeDef relative path
nodeDefUri = TfNormPath(fullMtlxPath + nodeDefUri);
+ } else {
+ nodeDefUri = TfNormPath(nodeDefUri);
}
// This is a locally defined custom node if the absolute path to the
// nodedef is not included in the stdlibDoc.
static mx::StringSet customNodeDefNames;
- static const mx::StringSet stdlibIncludes =
- UsdMtlxGetDocument("")->getReferencedSourceUris();
+ static const mx::StringSet stdlibIncludes = _GetStdlibIncludes();
if (stdlibIncludes.find(nodeDefUri) == stdlibIncludes.end()) {
// Check if we already used this custom node
if (std::find(customNodeDefNames.begin(), customNodeDefNames.end(),

View File

@@ -0,0 +1,106 @@
diff --git a/build_scripts/build_usd.py b/build_scripts/build_usd.py
index cfe243effb..a4bb94eee1 100644
--- a/build_scripts/build_usd.py
+++ b/build_scripts/build_usd.py
@@ -1415,7 +1415,7 @@ def InstallDraco(context, force, buildArgs):
############################################################
# MaterialX
-MATERIALX_URL = "https://github.com/materialx/MaterialX/archive/v1.38.4.zip"
+MATERIALX_URL = "https://github.com/materialx/MaterialX/archive/v1.38.5.zip"
def InstallMaterialX(context, force, buildArgs):
with CurrentWorkingDirectory(DownloadURL(MATERIALX_URL, context, force)):
diff --git a/pxr/imaging/hdSt/materialXShaderGen.cpp b/pxr/imaging/hdSt/materialXShaderGen.cpp
index df80ff119f..e4b5f04a73 100644
--- a/pxr/imaging/hdSt/materialXShaderGen.cpp
+++ b/pxr/imaging/hdSt/materialXShaderGen.cpp
@@ -136,8 +136,7 @@ HdStMaterialXShaderGen::HdStMaterialXShaderGen(
"st" : mxHdInfo.defaultTexcoordName;
// Register the customized version of the Surface node generator
- registerImplementation("IM_surface_" + GlslShaderGenerator::TARGET,
- HdStMaterialXSurfaceNodeGen::create);
+ registerImplementation("IM_surface_genglsl", HdStMaterialXSurfaceNodeGen::create);
}
// Based on GlslShaderGenerator::generate()
@@ -273,8 +272,7 @@ HdStMaterialXShaderGen::_EmitMxFunctions(
mx::ShaderStage& mxStage) const
{
// Add global constants and type definitions
- emitLibraryInclude("stdlib/" + mx::GlslShaderGenerator::TARGET
- + "/lib/mx_math.glsl", mxContext, mxStage);
+ emitLibraryInclude("stdlib/genglsl/lib/mx_math.glsl", mxContext, mxStage);
emitLine("#if NUM_LIGHTS > 0", mxStage, false);
emitLine("#define MAX_LIGHT_SOURCES NUM_LIGHTS", mxStage, false);
emitLine("#else", mxStage, false);
@@ -394,16 +392,24 @@ HdStMaterialXShaderGen::_EmitMxFunctions(
emitSpecularEnvironment(mxContext, mxStage);
}
if (shadowing) {
- emitLibraryInclude("pbrlib/" + mx::GlslShaderGenerator::TARGET
- + "/lib/mx_shadow.glsl", mxContext, mxStage);
+ emitLibraryInclude("pbrlib/genglsl/lib/mx_shadow.glsl", mxContext, mxStage);
}
+#if MATERIALX_MAJOR_VERSION > 1 || \
+ (MATERIALX_MAJOR_VERSION == 1 && MATERIALX_MINOR_VERSION > 38) || \
+ (MATERIALX_MAJOR_VERSION == 1 && MATERIALX_MINOR_VERSION == 38 && MATERIALX_BUILD_VERSION > 4)
+ // MaterialX 1.38.5 changes the default transmission method to "refraction".
+ mxContext.getOptions().hwTransmissionRenderMethod = mx::TRANSMISSION_OPACITY;
+
+ // Emit transmission code
+ emitTransmissionRender(mxContext, mxStage);
+#endif
+
// Emit directional albedo table code.
if (mxContext.getOptions().hwDirectionalAlbedoMethod ==
mx::HwDirectionalAlbedoMethod::DIRECTIONAL_ALBEDO_TABLE ||
mxContext.getOptions().hwWriteAlbedoTable) {
- emitLibraryInclude("pbrlib/" + mx::GlslShaderGenerator::TARGET
- + "/lib/mx_table.glsl", mxContext, mxStage);
+ emitLibraryInclude("pbrlib/genglsl/lib/mx_table.glsl", mxContext, mxStage);
emitLineBreak(mxStage);
}
@@ -421,7 +427,7 @@ HdStMaterialXShaderGen::_EmitMxFunctions(
// Emit uv transform code globally if needed.
if (mxContext.getOptions().hwAmbientOcclusion) {
emitLibraryInclude(
- "stdlib/" + mx::GlslShaderGenerator::TARGET + "/lib/" +
+ "stdlib/genglsl/lib/" +
_tokenSubstitutions[ShaderGenerator::T_FILE_TRANSFORM_UV],
mxContext, mxStage);
}
@@ -490,10 +496,30 @@ HdStMaterialXShaderGen::_EmitMxSurfaceShader(
// closure/shader nodes and need to be emitted first.
emitFunctionCalls(mxGraph, mxContext, mxStage, mx::ShaderNode::Classification::TEXTURE);
+#if MATERIALX_MAJOR_VERSION == 1 && \
+ MATERIALX_MINOR_VERSION == 38 && \
+ MATERIALX_BUILD_VERSION <= 4
// Emit function calls for all surface shader nodes.
// These will internally emit their closure function calls.
emitFunctionCalls(mxGraph, mxContext, mxStage, mx::ShaderNode::Classification::SHADER |
mx::ShaderNode::Classification::SURFACE);
+#else
+ // Emit function calls for "root" closure/shader nodes.
+ // These will internally emit function calls for any dependent closure nodes upstream.
+ for (mx::ShaderGraphOutputSocket* socket : mxGraph.getOutputSockets())
+ {
+ if (socket->getConnection())
+ {
+ const mx::ShaderNode* upstream = socket->getConnection()->getNode();
+ if (upstream->getParent() == &mxGraph &&
+ (upstream->hasClassification(mx::ShaderNode::Classification::CLOSURE) ||
+ upstream->hasClassification(mx::ShaderNode::Classification::SHADER)))
+ {
+ emitFunctionCall(*upstream, mxContext, mxStage);
+ }
+ }
+ }
+#endif
}
else
{

View File

@@ -108,7 +108,8 @@ set oiio_paths=%Staging%\%BuildDir%%ARCH%R\Release\openimageio\bin;%Staging%\%Bu
set boost_paths=%Staging%\%BuildDir%%ARCH%R\Release\boost\lib;%Staging%\%BuildDir%%ARCH%D\Debug\boost\lib
set openexr_paths=%Staging%\%BuildDir%%ARCH%R\Release\openexr\bin;%Staging%\%BuildDir%%ARCH%D\Debug\openexr\bin
set imath_paths=%Staging%\%BuildDir%%ARCH%R\Release\imath\bin;%Staging%\%BuildDir%%ARCH%D\Debug\imath\bin
set path=%BUILD_DIR%\downloads\mingw\mingw64\msys\1.0\bin\;%BUILD_DIR%\downloads\nasm-2.12.01\;%path%;%boost_paths%;%oiio_paths%;%openexr_paths%;%imath_paths%
set tbb_paths=%Staging%\%BuildDir%%ARCH%R\Release\tbb\bin;%Staging%\%BuildDir%%ARCH%D\Debug\tbb\bin
set path=%BUILD_DIR%\downloads\mingw\mingw64\msys\1.0\bin\;%BUILD_DIR%\downloads\nasm-2.12.01\;%path%;%boost_paths%;%oiio_paths%;%openexr_paths%;%imath_paths%;%tbb_paths%
mkdir %STAGING%\%BuildDir%%ARCH%R
cd %Staging%\%BuildDir%%ARCH%R
echo %DATE% %TIME% : Start > %StatusFile%