diff --git a/tests/python/CMakeLists.txt b/tests/python/CMakeLists.txt index 07554987143..c1ae5aad16d 100644 --- a/tests/python/CMakeLists.txt +++ b/tests/python/CMakeLists.txt @@ -685,62 +685,65 @@ if((WITH_CYCLES OR WITH_GPU_RENDER_TESTS) AND TEST_SRC_DIR_EXISTS) # Cycles if(WITH_CYCLES) set(_cycles_blocklist "") - set(_cycles_known_test_devices CPU CUDA OPTIX HIP HIP-RT METAL METAL-RT ONEAPI ONEAPI-RT) - if((NOT WITH_CYCLES_OSL) OR (WITH_CYCLES_TEST_OSL AND WITH_CYCLES_OSL)) - # Disable OSL tests if built without OSL or - # Disable OSL tests during the "normal" test phase to avoid double - # testing during the OSL test phase. - set(_cycles_blocklist OSL) - endif() + set(_cycles_all_test_devices CPU CUDA OPTIX HIP HIP-RT METAL METAL-RT ONEAPI ONEAPI-RT) + set(_cycles_osl_test_devices CPU OPTIX) foreach(_cycles_device ${CYCLES_TEST_DEVICES}) - if(NOT ${_cycles_device} IN_LIST _cycles_known_test_devices) + if(NOT ${_cycles_device} IN_LIST _cycles_all_test_devices) message(FATAL_ERROR "Unknown Cycles test device ${_cycles_device}." - "Supported devices are: ${_cycles_known_test_devices}") + "Supported devices are: ${_cycles_all_test_devices}") endif() string(TOLOWER "${_cycles_device}" _cycles_device_lower) set(_cycles_render_tests bake;${render_tests};osl) + set(_cycles_osl_test_type none) + if(WITH_CYCLES_OSL) + # If the render device is CPU, or a GPU WITH_CYCLES_TEST_OSL enabled, + # then enable the limited OSL tests (Tests OSL camera with OSL shading turned off). + # This specific configuration is chosen to avoid long test times on the GPU due to + # OSL JIT compilation, unless the developer explicitly enables OSL tests. + if(("${_cycles_device_lower}" STREQUAL "cpu") OR (WITH_CYCLES_TEST_OSL AND ${_cycles_device} IN_LIST _cycles_osl_test_devices)) + set(_cycles_osl_test_type limited) + endif() + endif() + foreach(render_test ${_cycles_render_tests}) set(_cycles_test_name "cycles_${render_test}_${_cycles_device_lower}") if(NOT(WITH_CYCLES_TEST_OSL AND WITH_CYCLES_OSL AND ("${render_test}" STREQUAL "osl"))) - # Only run OSL basic tests during this phase if WITH_CYCLES_TEST_OSL isn't enabled + # Only run OSL specific tests during this phase if WITH_CYCLES_TEST_OSL isn't enabled add_render_test( ${_cycles_test_name} ${CMAKE_CURRENT_LIST_DIR}/cycles_render_tests.py --testdir "${TEST_SRC_DIR}/render/${render_test}" --outdir "${TEST_OUT_DIR}/cycles" --device ${_cycles_device} - --blocklist ${_cycles_blocklist} + --osl "${_cycles_osl_test_type}" ) if(NOT ("${_cycles_device_lower}" STREQUAL "cpu")) set_tests_properties(${_cycles_test_name} PROPERTIES RUN_SERIAL TRUE) endif() endif() - if(WITH_CYCLES_TEST_OSL AND WITH_CYCLES_OSL) - # OSL is only supported with CPU and OptiX - # TODO: Enable OptiX support once it's more stable - if(("${_cycles_device_lower}" STREQUAL "cpu") OR - ("${_cycles_device_lower}" STREQUAL "optix")) - add_render_test( - ${_cycles_test_name}_osl - ${CMAKE_CURRENT_LIST_DIR}/cycles_render_tests.py - --testdir "${TEST_SRC_DIR}/render/${render_test}" - --outdir "${TEST_OUT_DIR}/cycles_osl" - --device ${_cycles_device} - --osl - ) - if(NOT ("${_cycles_device_lower}" STREQUAL "cpu")) - set_tests_properties(${_cycles_test_name}_osl PROPERTIES RUN_SERIAL TRUE) - endif() + # OSL variations of every test if WITH_CYCLES_TEST_OSL is set. + if(WITH_CYCLES_TEST_OSL AND NOT "${_cycles_osl_test_type}" STREQUAL "none") + add_render_test( + ${_cycles_test_name}_osl + ${CMAKE_CURRENT_LIST_DIR}/cycles_render_tests.py + --testdir "${TEST_SRC_DIR}/render/${render_test}" + --outdir "${TEST_OUT_DIR}/cycles_osl" + --device ${_cycles_device} + --osl "all" + ) + if(NOT ("${_cycles_device_lower}" STREQUAL "cpu")) + set_tests_properties(${_cycles_test_name}_osl PROPERTIES RUN_SERIAL TRUE) endif() endif() unset(_cycles_test_name) endforeach() + unset(_cycles_osl_test_type) endforeach() - unset(_cycles_blocklist) - unset(_cycles_known_test_devices) + unset(_cycles_osl_test_devices) + unset(_cycles_all_test_devices) endif() if(WITH_GPU_RENDER_TESTS) diff --git a/tests/python/cycles_render_tests.py b/tests/python/cycles_render_tests.py index 3a66730dd11..52c221f963f 100644 --- a/tests/python/cycles_render_tests.py +++ b/tests/python/cycles_render_tests.py @@ -24,19 +24,20 @@ BLOCKLIST_ALL = [ "image_log_osl.blend", ] -# Blocklist that disables OSL specific tests for configurations that do not support OSL backend. -BLOCKLIST_EXPLICIT_OSL = [ +# Blocklist for device + build configuration that does not support OSL at all. +BLOCKLIST_OSL_NONE = [ '.*_osl.blend', 'osl_.*.blend', ] -# Blocklist for SVM tests that are forced to run with OSL to test consistency between the two backends. -BLOCKLIST_OSL = [ - # Block tests that fail with OSL due to differences from SVM. - # Note: Most of the tests below are expected to be different between OSL and SVM - # As such many of these tests have both a SVM and OSL file. Blocking the SVM - # tests here doesn't loose any test permutations. - # +# Blocklist for OSL with limited OSL tests for fast test execution. +BLOCKLIST_OSL_LIMITED = [] + +# Blocklist for tests that fail when running all tests with OSL backend. +# Most of these tests are blocked due to expected differences between SVM and OSL. +# Due to the expected differences there are usually a SVM and OSL version of the test. +# So blocking these tests doesn't lose any test permutations. +BLOCKLIST_OSL_ALL = BLOCKLIST_OSL_LIMITED + [ # AOVs are not supported. See 73266 'aov_position.blend', 'render_passes_aov.*.blend', @@ -53,7 +54,7 @@ BLOCKLIST_OSL = [ 'image_log.blend', 'image_non_color.blend', 'image_mapping_udim.blend', - # TODO: Tests that need investigating into why they're failing, and how to fix that. + # Tests that need investigating into why they're failing: # Noise differences due to Principled BSDF mixing/layering used in some of these scenes 'render_passes_.*.blend', ] @@ -64,14 +65,20 @@ BLOCKLIST_OPTIX = [ 'big_plane_43865.blend', ] -BLOCKLIST_OPTIX_OSL = [ +# Blocklist for OSL tests that fail with the OptiX OSL backend. +BLOCKLIST_OPTIX_OSL_LIMITED = [ + 'image_.*_osl.blend', + # OptiX OSL doesn't support the trace function + 'osl_trace_shader.blend', +] + +# Blocklist for SVM tests that fail when forced to run with OptiX OSL +BLOCKLIST_OPTIX_OSL_ALL = BLOCKLIST_OPTIX_OSL_LIMITED + [ # OptiX OSL does support AO or Bevel 'ambient_occlusion.*.blend', 'bake_bevel.blend', 'bevel.blend', 'principled_bsdf_bevel_emission_137420.blend', - # OptiX OSL doesn't support the trace function - 'osl_trace_shader.blend', # The 3D texture doesn't have the right mappings 'point_density_.*_object.blend', # Dicing tests use wireframe node which doesn't appear to be supported with OptiX OSL @@ -204,8 +211,7 @@ def create_argparse(): parser.add_argument("--outdir", required=True) parser.add_argument("--oiiotool", required=True) parser.add_argument("--device", required=True) - parser.add_argument("--blocklist", nargs="*", default=[]) - parser.add_argument("--osl", default=False, action='store_true') + parser.add_argument("--osl", default='none', type=str, choices=["none", "limited", "all"]) parser.add_argument('--batch', default=False, action='store_true') return parser @@ -217,20 +223,27 @@ def main(): device = args.device blocklist = BLOCKLIST_ALL + + if args.osl == 'none': + blocklist += BLOCKLIST_OSL_NONE + elif args.osl == "limited": + blocklist += BLOCKLIST_OSL_LIMITED + else: + blocklist += BLOCKLIST_OSL_ALL + if device != 'CPU': blocklist += BLOCKLIST_GPU - if device != 'CPU' or 'OSL' in args.blocklist: - blocklist += BLOCKLIST_EXPLICIT_OSL + if device == 'OPTIX': blocklist += BLOCKLIST_OPTIX - if args.osl: - blocklist += BLOCKLIST_OPTIX_OSL + if args.osl == 'limited': + blocklist += BLOCKLIST_OPTIX_OSL_LIMITED + elif args.osl == 'all': + blocklist += BLOCKLIST_OPTIX_OSL_ALL if device == 'METAL': blocklist += BLOCKLIST_METAL - if args.osl: - blocklist += BLOCKLIST_OSL - report = CyclesReport('Cycles', args.outdir, args.oiiotool, device, blocklist, args.osl) + report = CyclesReport('Cycles', args.outdir, args.oiiotool, device, blocklist, args.osl == 'all') report.set_pixelated(True) report.set_reference_dir("cycles_renders") if device == 'CPU': @@ -251,12 +264,12 @@ def main(): test_dir_name = Path(args.testdir).name if (test_dir_name in {'motion_blur', 'integrator', "displacement"}) or \ - ((args.osl) and (test_dir_name in {'shader', 'hair'})): + ((args.osl == 'all') and (test_dir_name in {'shader', 'hair'})): report.set_fail_threshold(0.032) # Layer mixing is different between SVM and OSL, so a few tests have # noticably different noise causing OSL Principled BSDF tests to fail. - if ((args.osl) and (test_dir_name == 'principled_bsdf')): + if ((args.osl == 'all') and (test_dir_name == 'principled_bsdf')): report.set_fail_threshold(0.06) ok = report.run(args.testdir, args.blender, get_arguments, batch=args.batch)