griefith/test
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Switch SVN to Git submodules using Git-LFS

Browse Source 
This change makes it so build system and update utilities for Blender builds
are using pre-compiled libraries and other resources attached as Git modules
instead of using checkout of SVN repositories in the parent folder.

The directory layout:
```
  * release/datafiles/
    * assets/        -> blender-assets.git
      * publish/
      * ...
      * README.txt
  * lib/
    * darwin_x64/    -> lib-darwin_x64.git
    * darwin_arm64/  -> lib-darwin_arm64.git
    * linux_x64/     -> lib-linux_x64.git
    * windows_x64/   -> lib-windows_x64.git
  * tests/
    * data/         -> blender-test-data.git
```

The changes about configuring the actual Git sub-modules are not included
into this patch, as those require repository to actually exist before it
can be used.

The assets submodule is enabled by default, and the rest of them are
disabled. This means that if someone runs `git submodule update --init`
they will not get heavy libraries. The platform-specific and tests
related submodules are enabled when using `make update` or `make test`.

All the submodules are tracked: this means that when new commits are
done to the submodule, the blender.git repository is to be updated to
point them to the new hash. This causes some extra manual work, but it
allows to more easily update Blender and its dependencies to known good
state when performing operations like bisect.

Ref #108978

Pull Request: https://projects.blender.org/blender/blender/pulls/117946
This commit is contained in:
Sergey Sharybin
2024-02-22 13:50:55 +01:00
committed by Sergey Sharybin
parent 0007c7a6b2
commit 3dc832a904
31 changed files with 378 additions and 361 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
build_files/windows/check_libraries.cmd
View File

@@ -1,37 +1,30 @@
if "%BUILD_VS_YEAR%"=="2019" set BUILD_VS_LIBDIRPOST=vc15
if "%BUILD_VS_YEAR%"=="2022" set BUILD_VS_LIBDIRPOST=vc15
set BUILD_VS_SVNDIR=win64_%BUILD_VS_LIBDIRPOST%
set BUILD_VS_LIBDIR="%BLENDER_DIR%..\lib\%BUILD_VS_SVNDIR%"
set BUILD_VS_LIBDIR="lib\windows_x64"
if NOT "%verbose%" == "" (
echo Library Directory = "%BUILD_VS_LIBDIR%"
)
if NOT EXIST %BUILD_VS_LIBDIR% (
rem libs not found, but svn is on the system
if not "%SVN%"=="" (
if NOT EXIST "%BUILD_VS_LIBDIR%\.git" (
rem libs not found, but git is on the system
if not "%GIT%"=="" (
echo.
echo The required external libraries in %BUILD_VS_LIBDIR% are missing
echo.
set /p GetLibs= "Would you like to download them? (y/n)"
if /I "!GetLibs!"=="Y" (
echo.
echo Downloading %BUILD_VS_SVNDIR% libraries, please wait.
echo Downloading %BUILD_VS_LIBDIR% libraries, please wait.
echo.
:RETRY
"%SVN%" checkout https://svn.blender.org/svnroot/bf-blender/trunk/lib/%BUILD_VS_SVNDIR% %BUILD_VS_LIBDIR%
:RETRY
"%GIT%" -C "%BLENDER_DIR%" config --local "submodule.%BUILD_VS_LIBDIR%.update" "checkout"
"%GIT%" -C "%BLENDER_DIR%" submodule update --init "%BUILD_VS_LIBDIR%"
if errorlevel 1 (
set /p LibRetry= "Error during download, retry? y/n"
if /I "!LibRetry!"=="Y" (
cd %BUILD_VS_LIBDIR%
"%SVN%" cleanup
cd %BLENDER_DIR%
goto RETRY
)
echo.
echo Error: Download of external libraries failed.
echo This is needed for building, please manually run 'svn cleanup' and 'svn update' in
echo %BUILD_VS_LIBDIR% , until this is resolved you CANNOT make a successful blender build
echo Until this is resolved you CANNOT make a successful blender build.
echo.
exit /b 1
)
@@ -39,11 +32,11 @@ if NOT EXIST %BUILD_VS_LIBDIR% (
)
) else (
if NOT EXIST %PYTHON% (
if not "%SVN%"=="" (
if not "%GIT%"=="" (
echo.
echo Python not found in external libraries, updating to latest version
echo.
"%SVN%" update %BUILD_VS_LIBDIR%
"%GIT%" -C "%BLENDER_DIR%" submodule update "%BUILD_VS_LIBDIR%"
)
)
)
@@ -53,8 +46,8 @@ if NOT EXIST %BUILD_VS_LIBDIR% (
echo Error: Required libraries not found at "%BUILD_VS_LIBDIR%"
echo This is needed for building, aborting!
echo.
if "%SVN%"=="" (
echo This is most likely caused by svn.exe not being available.
if "%GIT%"=="" (
echo This is most likely caused by git.exe not being available.
)
exit /b 1
)

10
build_files/windows/find_dependencies.cmd
View File

@@ -1,5 +1,4 @@
REM find all dependencies and set the corresponding environment variables.
for %%X in (svn.exe) do (set SVN=%%~$PATH:X)
for %%X in (cmake.exe) do (set CMAKE=%%~$PATH:X)
for %%X in (ctest.exe) do (set CTEST=%%~$PATH:X)
for %%X in (git.exe) do (set GIT=%%~$PATH:X)
@@ -7,19 +6,19 @@ REM For python, default on 310 but if that does not exist also check
REM the 311, 312 and finally 39 folders to see if those are there, it checks
REM this far ahead to ensure good lib folder compatibility in the future
REM it falls back to 3.9 just incase it is a very old lib folder.
set PYTHON=%BLENDER_DIR%\..\lib\win64_vc15\python\310\bin\python.exe
set PYTHON=%BLENDER_DIR%\lib\windows_x64\python\310\bin\python.exe
if EXIST %PYTHON% (
goto detect_python_done
)
set PYTHON=%BLENDER_DIR%\..\lib\win64_vc15\python\311\bin\python.exe
set PYTHON=%BLENDER_DIR%\lib\windows_x64\python\311\bin\python.exe
if EXIST %PYTHON% (
goto detect_python_done
)
set PYTHON=%BLENDER_DIR%\..\lib\win64_vc15\python\312\bin\python.exe
set PYTHON=%BLENDER_DIR%\lib\windows_x64\python\312\bin\python.exe
if EXIST %PYTHON% (
goto detect_python_done
)
set PYTHON=%BLENDER_DIR%\..\lib\win64_vc15\python\39\bin\python.exe
set PYTHON=%BLENDER_DIR%\lib\windows_x64\python\39\bin\python.exe
if EXIST %PYTHON% (
goto detect_python_done
)
@@ -31,7 +30,6 @@ if NOT EXIST %PYTHON% (
:detect_python_done
if NOT "%verbose%" == "" (
echo svn : "%SVN%"
echo cmake : "%CMAKE%"
echo ctest : "%CTEST%"
echo git : "%GIT%"

4
build_files/windows/format.cmd
View File

@@ -1,5 +1,5 @@
if EXIST %BLENDER_DIR%\..\lib\win64_vc15\llvm\bin\clang-format.exe (
set CF_PATH=..\lib\win64_vc15\llvm\bin
if EXIST %BLENDER_DIR%\lib\windows_x64\llvm\bin\clang-format.exe (
set CF_PATH=lib\windows_x64\llvm\bin
goto detect_done
)

16
build_files/windows/lib_update.cmd Normal file
View File

@@ -0,0 +1,16 @@
set BUILD_VS_LIBDIR="lib\windows_x64"
:RETRY
"%GIT%" -C "%BLENDER_DIR%" config --local "submodule.%BUILD_VS_LIBDIR%.update" "checkout"
"%GIT%" -C "%BLENDER_DIR%" submodule update --init "%BUILD_VS_LIBDIR%"
if errorlevel 1 (
set /p LibRetry= "Error during update, retry? y/n"
if /I "!LibRetry!"=="Y" (
goto RETRY
)
echo.
echo Error: Download of external libraries failed.
echo Until this is resolved you CANNOT make a successful blender build.
echo.
exit /b 1
)

3
build_files/windows/parse_arguments.cmd
View File

@@ -110,9 +110,6 @@ if NOT "%1" == "" (
) else if "%1" == "doc_py" (
set DOC_PY=1
goto EOF
) else if "%1" == "svnfix" (
set SVN_FIX=1
goto EOF
) else (
echo Command "%1" unknown, aborting!
goto ERR

2
build_files/windows/reset_variables.cmd
View File

@@ -4,9 +4,7 @@ set BUILD_CMAKE_ARGS=
set BUILD_ARCH=
set BUILD_VS_VER=
set BUILD_VS_YEAR=
set BUILD_VS_LIBDIRPOST=
set BUILD_VS_LIBDIR=
set BUILD_VS_SVNDIR=
set KEY_NAME=
set MSBUILD_PLATFORM=
set MUST_CLEAN=

13
build_files/windows/show_hashes.cmd
View File

@@ -8,13 +8,10 @@ for /f "delims=" %%i in ('"%GIT%" rev-parse HEAD') do echo Branch_hash=%%i
cd "%BLENDER_DIR%/scripts/addons"
for /f "delims=" %%i in ('"%GIT%" rev-parse --abbrev-ref HEAD') do echo Addons_Branch_name=%%i
for /f "delims=" %%i in ('"%GIT%" rev-parse HEAD') do echo Addons_Branch_hash=%%i
if "%SVN%" == "" (
echo SVN not found, cannot library information.
goto EOF
)
set BUILD_VS_LIBDIR=%BLENDER_DIR%..\lib\win64_vc15
for /f "delims=" %%i in ('"%SVN%" info --show-item=url --no-newline %BUILD_VS_LIBDIR% ') do echo Libs_URL=%%i
for /f "delims=" %%i in ('"%SVN%" info --show-item=revision --no-newline %BUILD_VS_LIBDIR% ') do echo Libs_Revision=%%i
for /f "delims=" %%i in ('"%SVN%" info --show-item=last-changed-date --no-newline %BUILD_VS_LIBDIR% ') do echo Libs_LastChange=%%i
cd "%BLENDER_DIR%/lib/windows_x64"
for /f "delims=" %%i in ('"%GIT%" rev-parse --abbrev-ref HEAD') do echo Libs_Branch_name=%%i
for /f "delims=" %%i in ('"%GIT%" rev-parse HEAD') do echo Libs_Branch_hash=%%i
cd "%BLENDER_DIR%"
:EOF

25
build_files/windows/svn_fix.cmd
View File

@@ -1,25 +0,0 @@
if "%BUILD_VS_YEAR%"=="2019" set BUILD_VS_LIBDIRPOST=vc15
if "%BUILD_VS_YEAR%"=="2022" set BUILD_VS_LIBDIRPOST=vc15
set BUILD_VS_SVNDIR=win64_%BUILD_VS_LIBDIRPOST%
set BUILD_VS_LIBDIR="%BLENDER_DIR%..\lib\%BUILD_VS_SVNDIR%"
echo Starting cleanup in %BUILD_VS_LIBDIR%.
cd %BUILD_VS_LIBDIR%
:RETRY
"%SVN%" cleanup
"%SVN%" update
if errorlevel 1 (
set /p LibRetry= "Error during update, retry? y/n"
if /I "!LibRetry!"=="Y" (
goto RETRY
)
echo.
echo Error: Download of external libraries failed.
echo This is needed for building, please manually run 'svn cleanup' and 'svn update' in
echo %BUILD_VS_LIBDIR% , until this is resolved you CANNOT make a successful blender build
echo.
exit /b 1
)
echo Cleanup complete

24
build_files/windows/svn_update.cmd
View File

@@ -1,24 +0,0 @@
if "%BUILD_VS_YEAR%"=="2019" set BUILD_VS_LIBDIRPOST=vc15
if "%BUILD_VS_YEAR%"=="2022" set BUILD_VS_LIBDIRPOST=vc15
set BUILD_VS_SVNDIR=win64_%BUILD_VS_LIBDIRPOST%
set BUILD_VS_LIBDIR="%BLENDER_DIR%..\lib\%BUILD_VS_SVNDIR%"
cd %BUILD_VS_LIBDIR%
:RETRY
"%SVN%" update
if errorlevel 1 (
set /p LibRetry= "Error during update, retry? y/n"
if /I "!LibRetry!"=="Y" (
"%SVN%" cleanup
goto RETRY
)
echo.
echo Error: Download of external libraries failed.
echo This is needed for building, please manually run 'svn cleanup' and 'svn update' in
echo %BUILD_VS_LIBDIR% , until this is resolved you CANNOT make a successful blender build
echo.
exit /b 1
)
cd %BLENDER_DIR%

2
build_files/windows/test.cmd
View File

@@ -8,6 +8,6 @@ exit /b 1
:detect_python_done
REM Use -B to avoid writing __pycache__ in lib directory and causing update conflicts.
%PYTHON% -B %BLENDER_DIR%\build_files\utils\make_test.py --git-command "%GIT%" --svn-command "%SVN%" --cmake-command="%CMAKE%" --ctest-command="%CTEST%" --config="%BUILD_TYPE%" %BUILD_DIR%
%PYTHON% -B %BLENDER_DIR%\build_files\utils\make_test.py --git-command "%GIT%" --cmake-command="%CMAKE%" --ctest-command="%CTEST%" --config="%BUILD_TYPE%" %BUILD_DIR%
:EOF

2
build_files/windows/update_sources.cmd
View File

@@ -5,6 +5,6 @@ if NOT EXIST %PYTHON% (
:detect_python_done
REM Use -B to avoid writing __pycache__ in lib directory and causing update conflicts.
%PYTHON% -B %BLENDER_DIR%\build_files\utils\make_update.py --git-command "%GIT%" --svn-command "%SVN%" %BUILD_UPDATE_ARGS%
%PYTHON% -B %BLENDER_DIR%\build_files\utils\make_update.py --git-command "%GIT%" %BUILD_UPDATE_ARGS%
:EOF