griefith/test2
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Build: Ninja: Use similar logic for default heavy and regular jobs amount.

Browse Source 
Update the regular jobs amount computation to follow the same logic as
for the heavy ones. the main difference is that it uses a '2Gb of RAM
per job' base value.

This change is mainly targetted at machines with a relatively low
RAM/cores ratio, since even regular compile jobs can end up using quite
a lot of RAM if many are running in parallel, previous defaults would
likely not work well on machines with e.g. 16Gb of RAM and 16 cores.

Also fix a typo in previous commit (6493d0233c), sorry about that.
This commit is contained in:
Bastien Montagne
2024-01-05 12:51:47 +01:00
parent 6493d0233c
commit 7de49d5f80
1 changed files with 19 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
CMakeLists.txt
View File

@@ -1669,11 +1669,10 @@ if("${CMAKE_GENERATOR}" MATCHES "Ninja" AND WITH_NINJA_POOL_JOBS)
math(EXPR _compile_heavy_jobs "${_TOT_MEM} / 8000") math(EXPR _compile_heavy_jobs "${_TOT_MEM} / 8000")
math(EXPR _compile_heavy_jobs_max "${_NUM_CORES} - 1") math(EXPR _compile_heavy_jobs_max "${_NUM_CORES} - 1")
if(${_compile_heavy_jobs} GREATER ${_compile_heavy_jobs_max}) if(${_compile_heavy_jobs} GREATER ${_compile_heavy_jobs_max})
set(_compile_heavy_jobs _compile_heavy_jobs_max) set(_compile_heavy_jobs ${_compile_heavy_jobs_max})
elseif(${_compile_heavy_jobs} LESS 1) elseif(${_compile_heavy_jobs} LESS 1)
set(_compile_heavy_jobs 1) set(_compile_heavy_jobs 1)
endif() endif()
set(NINJA_MAX_NUM_PARALLEL_COMPILE_HEAVY_JOBS "${_compile_heavy_jobs}" CACHE STRING "\ set(NINJA_MAX_NUM_PARALLEL_COMPILE_HEAVY_JOBS "${_compile_heavy_jobs}" CACHE STRING "\
Define the maximum number of concurrent heavy compilation jobs, for ninja build system \ Define the maximum number of concurrent heavy compilation jobs, for ninja build system \
(used for some targets which cpp files can take several GB each during compilation)." (used for some targets which cpp files can take several GB each during compilation)."
@@ -1681,20 +1680,33 @@ Define the maximum number of concurrent heavy compilation jobs, for ninja build
) )
mark_as_advanced(NINJA_MAX_NUM_PARALLEL_COMPILE_HEAVY_JOBS) mark_as_advanced(NINJA_MAX_NUM_PARALLEL_COMPILE_HEAVY_JOBS)
set(_compile_heavy_jobs) set(_compile_heavy_jobs)
set(_compile_heavy_jobs_max)
# Only set regular compile jobs if we set heavy jobs, and there are 'enough' cores available, # Heuristics: Assume 2Gb of RAM is needed per heavy compile job.
# otherwise default (using all cores) if fine. # Typical RAM peak usage of these is actually way less than 1GB usually,
# but this also accounts for the part of the physical RAM being used by other unrelated
# processes on the system, and the part being used by the 'heavy' compile and linking jobs.
#
# If there are 'enough' cores available, cap the maximum number of regular jobs to
# `number of cores - 1`, otherwise allow using all cores if there is enough RAM available.
# This allows to ensure that the heavy jobs won't get starved by too many normal jobs, # This allows to ensure that the heavy jobs won't get starved by too many normal jobs,
# since the former usually take a long time to process. # since the former usually take a long time to process.
if(NINJA_MAX_NUM_PARALLEL_COMPILE_HEAVY_JOBS AND ${_NUM_CORES} GREATER 3) math(EXPR _compile_jobs "${_TOT_MEM} / 2000")
math(EXPR _compile_jobs "${_NUM_CORES} - 1") if(${_NUM_CORES} GREATER 3)
math(EXPR _compile_jobs_max "${_NUM_CORES} - 1")
else() else()
set(_compile_jobs "") set(_compile_jobs_max ${_NUM_CORES})
endif()
if(${_compile_jobs} GREATER ${_compile_jobs_max})
set(_compile_jobs ${_compile_jobs_max})
elseif(${_compile_jobs} LESS 1)
set(_compile_jobs 1)
endif() endif()
set(NINJA_MAX_NUM_PARALLEL_COMPILE_JOBS "${_compile_jobs}" CACHE STRING set(NINJA_MAX_NUM_PARALLEL_COMPILE_JOBS "${_compile_jobs}" CACHE STRING
"Define the maximum number of concurrent compilation jobs, for ninja build system." FORCE) "Define the maximum number of concurrent compilation jobs, for ninja build system." FORCE)
mark_as_advanced(NINJA_MAX_NUM_PARALLEL_COMPILE_JOBS) mark_as_advanced(NINJA_MAX_NUM_PARALLEL_COMPILE_JOBS)
set(_compile_jobs) set(_compile_jobs)
set(_compile_jobs_max)
# In practice, even when there is RAM available, # In practice, even when there is RAM available,
# this proves to be quicker than running in parallel (due to slow disks accesses). # this proves to be quicker than running in parallel (due to slow disks accesses).