diff --git a/CMakeLists.txt b/CMakeLists.txt index a35d267513e..df9beb74bfe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -773,6 +773,12 @@ else() set(WITH_UI_TESTS OFF) endif() +# Enabled by default for typical use cases to speed up development cycles. However, when looking +# into threading or memory related issues (in dependency graph, out-of-bounds, etc) forcing single +# test per Blender instance could give much better clues about the root of the problem. +option(WITH_TESTS_BATCHED "Run multiple tests in a single Blender invocation, for faster test execution" ON) +mark_as_advanced(WITH_TESTS_BATCHED) + # NOTE: All callers of this must add `TEST_PYTHON_EXE_EXTRA_ARGS` before any other arguments. set(TEST_PYTHON_EXE "" CACHE PATH "Python executable to run unit tests") mark_as_advanced(TEST_PYTHON_EXE) diff --git a/tests/python/CMakeLists.txt b/tests/python/CMakeLists.txt index ae022b1462f..c1097008fbe 100644 --- a/tests/python/CMakeLists.txt +++ b/tests/python/CMakeLists.txt @@ -127,6 +127,15 @@ function(add_python_test testname testscript) endif() endfunction() +# Run Python render test. +function(add_render_test testname testscript) + set(_args ${ARGN} -blender "${TEST_BLENDER_EXE}" -idiff "${OPENIMAGEIO_IDIFF}") + if(WITH_TESTS_BATCHED) + list(APPEND _args --batch) + endif() + add_python_test(${testname} ${testscript} ${_args}) +endfunction() + # ------------------------------------------------------------------------------ # GENERAL PYTHON CORRECTNESS TESTS add_blender_test( @@ -629,12 +638,10 @@ if(TRUE) set(_svg_render_tests complex path) foreach(render_test ${_svg_render_tests}) - add_python_test( + add_render_test( io_curve_svg_${render_test} ${CMAKE_CURRENT_LIST_DIR}/bl_io_curve_svg_test.py - -blender "${TEST_BLENDER_EXE}" -testdir "${TEST_SRC_DIR}/io_tests/svg/${render_test}" - -idiff "${OPENIMAGEIO_IDIFF}" -outdir "${TEST_OUT_DIR}/io_curve_svg" ) endforeach() @@ -716,12 +723,10 @@ if(WITH_CYCLES OR WITH_OPENGL_RENDER_TESTS) foreach(render_test ${_cycles_render_tests}) set(_cycles_test_name "cycles_${render_test}_${_cycles_device_lower}") - add_python_test( + add_render_test( ${_cycles_test_name} ${CMAKE_CURRENT_LIST_DIR}/cycles_render_tests.py - -blender "${TEST_BLENDER_EXE}" -testdir "${TEST_SRC_DIR}/render/${render_test}" - -idiff "${OPENIMAGEIO_IDIFF}" -outdir "${TEST_OUT_DIR}/cycles" -device ${_cycles_device} -blacklist ${_cycles_blacklist} @@ -739,36 +744,30 @@ if(WITH_CYCLES OR WITH_OPENGL_RENDER_TESTS) if(WITH_OPENGL_RENDER_TESTS) # Eevee foreach(render_test ${render_tests}) - add_python_test( + add_render_test( eevee_${render_test}_test ${CMAKE_CURRENT_LIST_DIR}/eevee_render_tests.py - -blender "${TEST_BLENDER_EXE}" -testdir "${TEST_SRC_DIR}/render/${render_test}" - -idiff "${OPENIMAGEIO_IDIFF}" -outdir "${TEST_OUT_DIR}/eevee" ) endforeach() # Eevee Next foreach(render_test ${render_tests}) - add_python_test( + add_render_test( eevee_next_${render_test}_test ${CMAKE_CURRENT_LIST_DIR}/eevee_next_render_tests.py - -blender "${TEST_BLENDER_EXE}" -testdir "${TEST_SRC_DIR}/render/${render_test}" - -idiff "${OPENIMAGEIO_IDIFF}" -outdir "${TEST_OUT_DIR}/eevee_next" ) endforeach() foreach(render_test ${render_tests}) # Workbench - add_python_test( + add_render_test( workbench_${render_test}_test ${CMAKE_CURRENT_LIST_DIR}/workbench_render_tests.py - -blender "${TEST_BLENDER_EXE}" -testdir "${TEST_SRC_DIR}/render/${render_test}" - -idiff "${OPENIMAGEIO_IDIFF}" -outdir "${TEST_OUT_DIR}/workbench" ) endforeach() @@ -776,24 +775,20 @@ if(WITH_CYCLES OR WITH_OPENGL_RENDER_TESTS) if(WITH_HYDRA) # Hydra Storm foreach(render_test ${render_tests}) - add_python_test( + add_render_test( storm_hydra_${render_test}_test ${CMAKE_CURRENT_LIST_DIR}/storm_render_tests.py - -blender "${TEST_BLENDER_EXE}" -testdir "${TEST_SRC_DIR}/render/${render_test}" - -idiff "${OPENIMAGEIO_IDIFF}" -outdir "${TEST_OUT_DIR}/storm_hydra" -export_method "HYDRA" ) endforeach() foreach(render_test ${render_tests}) - add_python_test( + add_render_test( storm_usd_${render_test}_test ${CMAKE_CURRENT_LIST_DIR}/storm_render_tests.py - -blender "${TEST_BLENDER_EXE}" -testdir "${TEST_SRC_DIR}/render/${render_test}" - -idiff "${OPENIMAGEIO_IDIFF}" -outdir "${TEST_OUT_DIR}/storm_usd" -export_method "USD" ) @@ -823,12 +818,10 @@ if(WITH_COMPOSITOR_CPU) endif() foreach(comp_test ${compositor_tests}) - add_python_test( + add_render_test( compositor_${comp_test}_cpu_test ${CMAKE_CURRENT_LIST_DIR}/compositor_cpu_render_tests.py - -blender "${TEST_BLENDER_EXE}" -testdir "${TEST_SRC_DIR}/compositor/${comp_test}" - -idiff "${OPENIMAGEIO_IDIFF}" -outdir "${TEST_OUT_DIR}/compositor_cpu" ) endforeach() @@ -857,12 +850,10 @@ if(WITH_COMPOSITOR_REALTIME_TESTS AND WITH_COMPOSITOR_CPU) endif() foreach(comp_test ${compositor_tests}) - add_python_test( + add_render_test( compositor_${comp_test}_realtime_test ${CMAKE_CURRENT_LIST_DIR}/compositor_realtime_render_tests.py - -blender "${TEST_BLENDER_EXE}" -testdir "${TEST_SRC_DIR}/compositor/${comp_test}" - -idiff "${OPENIMAGEIO_IDIFF}" -outdir "${TEST_OUT_DIR}/compositor_realtime" ) endforeach() @@ -945,12 +936,10 @@ if(WITH_OPENGL_DRAW_TESTS) if(IS_DIRECTORY ${child_path}) file(GLOB_RECURSE blends "${child_path}/*.blend") if(blends) - add_python_test( + add_render_test( opengl_draw_${child} ${CMAKE_CURRENT_LIST_DIR}/opengl_draw_tests.py - -blender "${TEST_BLENDER_EXE}" -testdir "${child_path}" - -idiff "${OPENIMAGEIO_IDIFF}" -outdir "${TEST_OUT_DIR}/opengl_draw" ) endif() @@ -1057,12 +1046,10 @@ else() ) foreach(render_test ${render_tests}) - add_python_test( + add_render_test( sequencer_render_${render_test} ${CMAKE_CURRENT_LIST_DIR}/sequencer_render_tests.py - -blender "${TEST_BLENDER_EXE}" -testdir "${TEST_SRC_DIR}/sequence_editing/${render_test}" - -idiff "${OPENIMAGEIO_IDIFF}" -outdir "${TEST_OUT_DIR}/sequence_editing" ) endforeach() diff --git a/tests/python/bl_io_curve_svg_test.py b/tests/python/bl_io_curve_svg_test.py index e5e5feea9e3..bb722730d21 100644 --- a/tests/python/bl_io_curve_svg_test.py +++ b/tests/python/bl_io_curve_svg_test.py @@ -37,6 +37,7 @@ def create_argparse(): parser.add_argument("-testdir", nargs=1) parser.add_argument("-outdir", nargs=1) parser.add_argument("-idiff", nargs=1) + parser.add_argument('--batch', default=False, action='store_true') return parser @@ -57,7 +58,7 @@ def main(): if test_dir_name == 'complex': report.set_fail_percent(0.01) - ok = report.run(test_dir, blender, get_arguments, batch=True) + ok = report.run(test_dir, blender, get_arguments, batch=args.batch) sys.exit(not ok) diff --git a/tests/python/compositor_cpu_render_tests.py b/tests/python/compositor_cpu_render_tests.py index b488c102bc1..6f7b60d8939 100644 --- a/tests/python/compositor_cpu_render_tests.py +++ b/tests/python/compositor_cpu_render_tests.py @@ -38,6 +38,7 @@ def create_argparse(): parser.add_argument("-testdir", nargs=1) parser.add_argument("-outdir", nargs=1) parser.add_argument("-idiff", nargs=1) + parser.add_argument('--batch', default=False, action='store_true') return parser @@ -66,7 +67,7 @@ def main(): report.set_fail_threshold(0.06) report.set_fail_percent(2) - ok = report.run(test_dir, blender, get_arguments, batch=True) + ok = report.run(test_dir, blender, get_arguments, batch=args.batch) sys.exit(not ok) diff --git a/tests/python/compositor_realtime_render_tests.py b/tests/python/compositor_realtime_render_tests.py index 11d92e6bd84..0bd712f605e 100644 --- a/tests/python/compositor_realtime_render_tests.py +++ b/tests/python/compositor_realtime_render_tests.py @@ -49,6 +49,7 @@ def create_argparse(): parser.add_argument("-testdir", nargs=1) parser.add_argument("-outdir", nargs=1) parser.add_argument("-idiff", nargs=1) + parser.add_argument('--batch', default=False, action='store_true') return parser @@ -66,7 +67,7 @@ def main(): report = render_report.Report("Compositor Realtime", output_dir, idiff, blacklist=blacklist_all) report.set_reference_dir("compositor_realtime_renders") - ok = report.run(test_dir, blender, get_arguments, batch=True) + ok = report.run(test_dir, blender, get_arguments, batch=args.batch) sys.exit(not ok) diff --git a/tests/python/cycles_render_tests.py b/tests/python/cycles_render_tests.py index 7260e44450a..b258a90dfd7 100644 --- a/tests/python/cycles_render_tests.py +++ b/tests/python/cycles_render_tests.py @@ -107,6 +107,7 @@ def create_argparse(): parser.add_argument("-idiff", nargs=1) parser.add_argument("-device", nargs=1) parser.add_argument("-blacklist", nargs="*") + parser.add_argument('--batch', default=False, action='store_true') return parser @@ -147,7 +148,7 @@ def main(): if test_dir_name in ('motion_blur', 'integrator', ): report.set_fail_threshold(0.032) - ok = report.run(test_dir, blender, get_arguments, batch=True) + ok = report.run(test_dir, blender, get_arguments, batch=args.batch) sys.exit(not ok) diff --git a/tests/python/eevee_next_render_tests.py b/tests/python/eevee_next_render_tests.py index ca5d533495c..59f7be270ad 100644 --- a/tests/python/eevee_next_render_tests.py +++ b/tests/python/eevee_next_render_tests.py @@ -110,6 +110,7 @@ def create_argparse(): parser.add_argument("-testdir", nargs=1) parser.add_argument("-outdir", nargs=1) parser.add_argument("-idiff", nargs=1) + parser.add_argument('--batch', default=False, action='store_true') return parser @@ -138,7 +139,7 @@ def main(): if test_dir_name.startswith('image'): report.set_fail_threshold(0.051) - ok = report.run(test_dir, blender, get_arguments, batch=True) + ok = report.run(test_dir, blender, get_arguments, batch=args.batch) sys.exit(not ok) diff --git a/tests/python/eevee_render_tests.py b/tests/python/eevee_render_tests.py index 11327a78e9c..23d66c3f246 100644 --- a/tests/python/eevee_render_tests.py +++ b/tests/python/eevee_render_tests.py @@ -143,6 +143,7 @@ def create_argparse(): parser.add_argument("-testdir", nargs=1) parser.add_argument("-outdir", nargs=1) parser.add_argument("-idiff", nargs=1) + parser.add_argument('--batch', default=False, action='store_true') return parser @@ -171,7 +172,7 @@ def main(): if test_dir_name.startswith('image'): report.set_fail_threshold(0.051) - ok = report.run(test_dir, blender, get_arguments, batch=True) + ok = report.run(test_dir, blender, get_arguments, batch=args.batch) sys.exit(not ok) diff --git a/tests/python/sequencer_render_tests.py b/tests/python/sequencer_render_tests.py index bb95e97a27e..f94a43e6024 100644 --- a/tests/python/sequencer_render_tests.py +++ b/tests/python/sequencer_render_tests.py @@ -34,6 +34,7 @@ def create_argparse(): parser.add_argument("-testdir", nargs=1) parser.add_argument("-outdir", nargs=1) parser.add_argument("-idiff", nargs=1) + parser.add_argument('--batch', default=False, action='store_true') return parser @@ -52,7 +53,7 @@ def main(): report.set_reference_dir("reference") test_dir_name = Path(test_dir).name - ok = report.run(test_dir, blender, get_arguments, batch=True) + ok = report.run(test_dir, blender, get_arguments, batch=args.batch) sys.exit(not ok) diff --git a/tests/python/storm_render_tests.py b/tests/python/storm_render_tests.py index f5e9b7ec258..b7f64124422 100644 --- a/tests/python/storm_render_tests.py +++ b/tests/python/storm_render_tests.py @@ -62,6 +62,7 @@ def create_argparse(): parser.add_argument("-outdir", nargs=1) parser.add_argument("-idiff", nargs=1) parser.add_argument("-export_method", nargs=1) + parser.add_argument('--batch', default=False, action='store_true') return parser @@ -92,7 +93,7 @@ def main(): os.environ['BLENDER_HYDRA_EXPORT_METHOD'] = export_method - ok = report.run(test_dir, blender, get_arguments, batch=True) + ok = report.run(test_dir, blender, get_arguments, batch=args.batch) sys.exit(not ok) diff --git a/tests/python/workbench_render_tests.py b/tests/python/workbench_render_tests.py index 91a32ed78a1..e232b01adbf 100644 --- a/tests/python/workbench_render_tests.py +++ b/tests/python/workbench_render_tests.py @@ -57,6 +57,7 @@ def create_argparse(): parser.add_argument("-testdir", nargs=1) parser.add_argument("-outdir", nargs=1) parser.add_argument("-idiff", nargs=1) + parser.add_argument('--batch', default=False, action='store_true') return parser @@ -79,7 +80,7 @@ def main(): if test_dir_name.startswith('hair') and platform.system() == "Darwin": report.set_fail_threshold(0.050) - ok = report.run(test_dir, blender, get_arguments, batch=True) + ok = report.run(test_dir, blender, get_arguments, batch.args=batch) sys.exit(not ok)