Cycles render tests: Add the option to test with OSL enabled

This commit adds a new cmake variable `WITH_CYCLES_TEST_OSL` that runs every Cycles test a second time with OSL enabled. At the moment only CPU OSL is enabled. There are plans to enable OptiX OSL in the future when stability issues with OptiX OSL have been resolved. Some render tests have been blocked from running until we can figure out a fix. The most notiable being all the Pricincipled BSDF tests as some of them are failing due to noise differences. Ref #123012 Pull Request: https://projects.blender.org/blender/blender/pulls/124601
2024-08-14 17:00:48 +02:00
parent 658b32093a
commit 1330d0ddda
7 changed files with 153 additions and 21 deletions
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -653,12 +653,17 @@ Build Cycles kernels with address sanitizer when WITH_COMPILER_ASAN is on, even
 set(CYCLES_TEST_DEVICES CPU CACHE STRING "\
 Run regression tests on the specified device types (CPU CUDA OPTIX HIP)"
 )
+option(WITH_CYCLES_TEST_OSL "\
+Run additional Cycles test with OSL enabled"
+  OFF
+)
 mark_as_advanced(WITH_CYCLES_KERNEL_ASAN)
 mark_as_advanced(WITH_CYCLES_LOGGING)
 mark_as_advanced(WITH_CYCLES_DEBUG_NAN)
 mark_as_advanced(WITH_CYCLES_NATIVE_ONLY)
 mark_as_advanced(WITH_CYCLES_PRECOMPUTE)
 mark_as_advanced(CYCLES_TEST_DEVICES)
+mark_as_advanced(WITH_CYCLES_TEST_OSL)

 # NVIDIA CUDA & OptiX
 if(NOT APPLE AND NOT (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "ARM64"))
@@ -1216,6 +1221,7 @@ set_and_warn_dependency(WITH_BOOST WITH_USD            OFF)
 if(WITH_CYCLES)
  set_and_warn_dependency(WITH_BOOST   WITH_CYCLES_OSL   OFF)
  set_and_warn_dependency(WITH_PUGIXML WITH_CYCLES_OSL   OFF)
+  set_and_warn_dependency(WITH_CYCLES_OSL WITH_CYCLES_TEST_OSL   OFF)
 endif()

 set_and_warn_dependency(WITH_TBB WITH_CYCLES            OFF)
--- a/build_files/buildbot/config/blender_linux.cmake
+++ b/build_files/buildbot/config/blender_linux.cmake
@@ -12,3 +12,4 @@ message(STATUS "Building in Rocky 8 Linux 64bit environment")
 # Options which are specific to Linux-only platforms

 set(WITH_DOC_MANPAGE         OFF CACHE BOOL "" FORCE)
+set(WITH_CYCLES_TEST_OSL     ON CACHE BOOL "" FORCE)
--- a/build_files/buildbot/config/blender_macos.cmake
+++ b/build_files/buildbot/config/blender_macos.cmake
@@ -3,3 +3,5 @@
 # SPDX-License-Identifier: GPL-2.0-or-later

 include("${CMAKE_CURRENT_LIST_DIR}/../../cmake/config/blender_release.cmake")
+
+set(WITH_CYCLES_TEST_OSL     ON CACHE BOOL "" FORCE)
--- a/build_files/buildbot/config/blender_windows.cmake
+++ b/build_files/buildbot/config/blender_windows.cmake
@@ -3,3 +3,5 @@
 # SPDX-License-Identifier: GPL-2.0-or-later

 include("${CMAKE_CURRENT_LIST_DIR}/../../cmake/config/blender_release.cmake")
+
+set(WITH_CYCLES_TEST_OSL     ON CACHE BOOL "" FORCE)
--- a/tests/python/CMakeLists.txt
+++ b/tests/python/CMakeLists.txt
@@ -689,7 +689,10 @@ if(WITH_CYCLES OR WITH_GPU_RENDER_TESTS)
    # Cycles
    if(WITH_CYCLES)
      set(_cycles_blocklist "")
-      if(NOT WITH_CYCLES_OSL)
+      if((NOT WITH_CYCLES_OSL) OR (WITH_CYCLES_TEST_OSL AND WITH_CYCLES_OSL))
+        # Disable OSL tests if built without OSL or
+        # Disable OSL tests during the "normal" test phase to avoid double
+        # testing during the OSL test phase.
        set(_cycles_blocklist OSL)
      endif()
      foreach(_cycles_device ${CYCLES_TEST_DEVICES})
@@ -698,18 +701,40 @@ if(WITH_CYCLES OR WITH_GPU_RENDER_TESTS)

        foreach(render_test ${_cycles_render_tests})
          set(_cycles_test_name "cycles_${render_test}_${_cycles_device_lower}")
-          add_render_test(
-            ${_cycles_test_name}
-            ${CMAKE_CURRENT_LIST_DIR}/cycles_render_tests.py
-            -testdir "${TEST_SRC_DIR}/render/${render_test}"
-            -outdir "${TEST_OUT_DIR}/cycles"
-            -device ${_cycles_device}
-            -blocklist ${_cycles_blocklist}
-          )
-
-          if(NOT ("${_cycles_device_lower}" STREQUAL "cpu"))
-            set_tests_properties(${_cycles_test_name} PROPERTIES RUN_SERIAL TRUE)
+          if (NOT(WITH_CYCLES_TEST_OSL AND WITH_CYCLES_OSL AND ("${render_test}" STREQUAL "osl")))
+            # Only run OSL basic tests during this phase if WITH_CYCLES_TEST_OSL isn't enabled
+            add_render_test(
+              ${_cycles_test_name}
+              ${CMAKE_CURRENT_LIST_DIR}/cycles_render_tests.py
+              -testdir "${TEST_SRC_DIR}/render/${render_test}"
+              -outdir "${TEST_OUT_DIR}/cycles"
+              -device ${_cycles_device}
+              -blocklist ${_cycles_blocklist}
+            )
+            if(NOT ("${_cycles_device_lower}" STREQUAL "cpu"))
+              set_tests_properties(${_cycles_test_name} PROPERTIES RUN_SERIAL TRUE)
+            endif()
          endif()
+
+          if (WITH_CYCLES_TEST_OSL AND WITH_CYCLES_OSL)
+            # OSL is only supported with CPU and OptiX
+            # TODO: Enable OptiX support once it's more stable
+            if("${_cycles_device_lower}" STREQUAL "cpu")
+              add_render_test(
+                ${_cycles_test_name}_osl
+                ${CMAKE_CURRENT_LIST_DIR}/cycles_render_tests.py
+                -testdir "${TEST_SRC_DIR}/render/${render_test}"
+                -outdir "${TEST_OUT_DIR}/cycles_osl"
+                -device ${_cycles_device}
+                -osl
+              )
+              # Doesn't do anything until OptiX is enabled
+              if(NOT ("${_cycles_device_lower}" STREQUAL "cpu"))
+                set_tests_properties(${_cycles_test_name}_osl PROPERTIES RUN_SERIAL TRUE)
+              endif()
+            endif()
+          endif()
+
          unset(_cycles_test_name)
        endforeach()
      endforeach()
--- a/tests/python/cycles_render_tests.py
+++ b/tests/python/cycles_render_tests.py
@@ -9,6 +9,7 @@ import os
 import shlex
 import sys
 from pathlib import Path
+from modules import render_report

 # List of .blend files that are known to be failing and are not ready to be
 # tested, or that only make sense on some devices. Accepts regular expressions.
@@ -21,18 +22,84 @@ BLOCKLIST_ALL = [
    "visibility_particles.blend",
 ]

-BLOCKLIST_OSL = [
-    # OSL only supported on CPU.
+# Blocklist that disables OSL specific tests for configurations that do not support OSL backend.
+BLOCKLIST_EXPLICIT_OSL = [
    '.*_osl.blend',
    'osl_.*.blend',
 ]

+# Blocklist for SVM tests that are forced to run with OSL to test consistency between the two backends.
+BLOCKLIST_OSL = [
+    # Block tests that fail with OSL due to differences from SVM.
+    # Note: Most of the tests below are expected to be different between OSL and SVM
+    # As such many of these tests have both a SVM and OSL file. Blocking the SVM
+    # tests here doesn't loose any test permutations.
+    #
+    # AOVs are not supported. See 73266
+    'aov_position.blend',
+    'render_passes_aov.*.blend',
+    # Image sampling is different from SVM. There are OSL variants of these tests
+    'image_byte.*.blend',
+    'image_float.*.blend',
+    'image_half.*.blend',
+    'image_mapping_.*_closest.blend',
+    'image_mapping_.*_cubic.blend',
+    'image_mapping_.*_linear.blend',
+    'image_alpha_blend.blend',
+    'image_alpha_channel_packed.blend',
+    'image_alpha_ignore.blend',
+    'image_log.blend',
+    'image_non_color.blend',
+    # Along with differences in image sampling, UDIM in OSL doesn't respect extrapolation settings
+    # This has been reported in 124847 for further investigation
+    'image_mapping_udim.blend',
+    # OSL handles bump + displacement differently from SVM. There are OSL variants of these tests
+    'both_displacement.blend',
+    'bump_with_displacement.blend',
+    # Ray portal test uses bump + displacement
+    'ray_portal.blend',
+    # TODO: Tests that need investigating into why they're failing, and how to fix that.
+    # Noise differences due to Principled BSDF mixing/layering used in some of these scenes
+    'render_passes_.*.blend',
+    # Noise differences in Principled BSDF mixing/layering
+    'principled_.*.blend',
+]
+
 BLOCKLIST_OPTIX = [
    # Ray intersection precision issues
    'T50164.blend',
    'T43865.blend',
 ]

+BLOCKLIST_OPTIX_OSL = [
+    # OPTIX OSL doesn't support trace function needed for AO and bevel
+    'bake_bevel.blend',
+    'ambient_occlusion.*.blend',
+    'bevel.blend',
+    'osl_trace_shader.blend',
+    # The Volumetric noise texture is different for some reason
+    'principled_absorption.blend',
+    # Dicing tests use wireframe node which doesn't appear to be supported in OptiX
+    'dicing_camera.blend',
+    'offscreen_dicing.blend',
+    'panorama_dicing.blend',
+    # Bump evaluation is not implemented yet. See 104276
+    'compare_bump.blend',
+    'both_displacement.blend',
+    'bump_with_displacement.blend',
+    'ray_portal.blend',
+    # TODO: Investigate every other failing case and add them here.
+    # Note: Many tests are failing due to CUDA errors. Some of these are driver issues that NVIDIA is currently looking into.
+    #
+    # Currently failing tests that aren't in this list are:
+    # ray_portal*.blend - CUDA error
+    # image_mapping_udim*.blend - Can't load UDIM from disk? But can load UDIM if it's packed, but doesn't seem to use it properly.
+    # points_volume.blend - CUDA error
+    # principled_emission_alpha.blend - CUDA error related to connected inputs. Probably the same as 122779
+    # point_density_*_object - Object scale doesn't appear to be appplied to texture
+    # All the other tests mentioned in BLOCKLIST_OSL (E.g. Principled BSDF tests having noise differences)
+]
+
 BLOCKLIST_METAL = []

 if platform.system() == "Darwin":
@@ -69,7 +136,18 @@ BLOCKLIST_GPU = [
 ]


-def get_arguments(filepath, output_filepath):
+class CyclesReport(render_report.Report):
+    def __init__(self, title, output_dir, oiiotool, device=None, blocklist=[], osl=False):
+        super().__init__(title, output_dir, oiiotool, device=device, blocklist=blocklist)
+        self.osl = osl
+        if osl:
+            self.title += " OSL"
+
+    def _get_render_arguments(self, arguments_cb, filepath, base_output_filepath):
+        return arguments_cb(filepath, base_output_filepath, self.osl)
+
+
+def get_arguments(filepath, output_filepath, osl=False):
    dirname = os.path.dirname(filepath)
    basedir = os.path.dirname(dirname)
    subject = os.path.basename(dirname)
@@ -96,6 +174,9 @@ def get_arguments(filepath, output_filepath):
    if spp_multiplier:
        args.extend(["--python-expr", f"import bpy; bpy.context.scene.cycles.samples *= {spp_multiplier}"])

+    if osl:
+        args.extend(["--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True"])
+
    if subject == 'bake':
        args.extend(['--python', os.path.join(basedir, "util", "render_bake.py")])
    elif subject == 'denoise_animation':
@@ -113,7 +194,8 @@ def create_argparse():
    parser.add_argument("-outdir", nargs=1)
    parser.add_argument("-oiiotool", nargs=1)
    parser.add_argument("-device", nargs=1)
-    parser.add_argument("-blocklist", nargs="*")
+    parser.add_argument("-blocklist", nargs="*", default=[])
+    parser.add_argument("-osl", default=False, action='store_true')
    parser.add_argument('--batch', default=False, action='store_true')
    return parser

@@ -132,14 +214,17 @@ def main():
    if device != 'CPU':
        blocklist += BLOCKLIST_GPU
    if device != 'CPU' or 'OSL' in args.blocklist:
-        blocklist += BLOCKLIST_OSL
+        blocklist += BLOCKLIST_EXPLICIT_OSL
    if device == 'OPTIX':
        blocklist += BLOCKLIST_OPTIX
+        if args.osl:
+            blocklist += BLOCKLIST_OPTIX_OSL
    if device == 'METAL':
        blocklist += BLOCKLIST_METAL
+    if args.osl:
+        blocklist += BLOCKLIST_OSL

-    from modules import render_report
-    report = render_report.Report('Cycles', output_dir, oiiotool, device, blocklist)
+    report = CyclesReport('Cycles', output_dir, oiiotool, device, blocklist, args.osl)
    report.set_pixelated(True)
    report.set_reference_dir("cycles_renders")
    if device == 'CPU':
@@ -151,8 +236,13 @@ def main():
    #
    # underwater_caustics.blend gives quite different results on Linux and Intel macOS compared to
    # Windows and Arm macOS.
+    #
+    # OSL tests:
+    # Blackbody is slightly different between SVM and OSL.
+    # Microfacet hair renders slightly differently, and fails on Windows and Linux with OSL
+
    test_dir_name = Path(test_dir).name
-    if test_dir_name in {'motion_blur', 'integrator'}:
+    if (test_dir_name in {'motion_blur', 'integrator'}) or ((args.osl) and (test_dir_name in {'shader', 'hair'})):
        report.set_fail_threshold(0.032)

    ok = report.run(test_dir, blender, get_arguments, batch=args.batch)
--- a/tests/python/modules/render_report.py
+++ b/tests/python/modules/render_report.py
@@ -458,6 +458,12 @@ class Report:

        return not failed

+    def _get_render_arguments(self, arguments_cb, filepath, base_output_filepath):
+        # Each render test can override this method to provide extra functionality.
+        # See Cycles render tests for an example.
+        # Do not delete.
+        return arguments_cb(filepath, base_output_filepath)
+
    def _run_tests(self, filepaths, blender, arguments_cb, batch):
        # Run multiple tests in a single Blender process since startup can be
        # a significant factor. In case of crashes, re-run the remaining tests.
@@ -482,7 +488,7 @@ class Report:
                if os.path.exists(output_filepath):
                    os.remove(output_filepath)

-                command.extend(arguments_cb(filepath, base_output_filepath))
+                command.extend(self._get_render_arguments(arguments_cb, filepath, base_output_filepath))

                # Only chain multiple commands for batch
                if not batch: