OSL Scons rules

Initial support of OSL builds using SCons build system. Only tested on Linux now. No changes to configuration files themselves -- for now check how it's configured for linux buildbot (it was already horror to make all this changes and verify them, changes to linux-config.py could easily be done later). Currently WITH_BF_STATICOSL and WITH_BF_STATICLLVM are more like rudiments because linking against oslexec requires special trick with --whole-archive. We woul either need to find a way dealing with this oslexec less hackish or drop STATICOSL and STATICLLVM flags. Will keep dropping this flags for until we have "final" build rules for OSL. Still can not make 32bit linux rendering with OSL -- blender simply crashes when starting rendering. So for time being this issues are solving disabled OSL for 32bit build slaves.
2012-11-15 13:11:12 +00:00
parent abeeea6a96
commit 9eccf40daa
10 changed files with 205 additions and 3 deletions
--- a/16
+++ b/16
@@ -725,6 +725,22 @@ if env['OURPLATFORM']!='darwin':
                    cubin_file = os.path.join(kernel_build_dir, "kernel_%s.cubin" % arch)
                    scriptinstall.append(env.Install(dir=dir,source=cubin_file))

+            # osl shaders
+            if env['WITH_BF_CYCLES_OSL']:
+                dir=os.path.join(env['BF_INSTALLDIR'], VERSION, 'scripts', 'addons','cycles', 'shader')
+
+                osl_source_dir = Dir('./intern/cycles/kernel/shaders').srcnode().path
+                oso_build_dir = os.path.join(B.root_build_dir, 'intern/cycles/kernel/shaders')
+
+                headers='node_color.h node_fresnel.h node_texture.h oslutil.h stdosl.h'.split()
+                source=['intern/cycles/kernel/shaders/'+s for s in headers]
+                scriptinstall.append(env.Install(dir=dir,source=source))
+
+                for f in os.listdir(osl_source_dir):
+                    if f.endswith('.osl'):
+                        oso_file = os.path.join(oso_build_dir, f.replace('.osl', '.oso'))
+                        scriptinstall.append(env.Install(dir=dir,source=oso_file))
+
    if env['WITH_BF_OCIO']:
        colormanagement = os.path.join('release', 'datafiles', 'colormanagement')

--- a/build_files/buildbot/config/user-config-glibc211-i686.py
+++ b/build_files/buildbot/config/user-config-glibc211-i686.py
@@ -120,6 +120,24 @@ BF_OIIO_INC = '${BF_OIIO}/include'
 BF_OIIO_LIB_STATIC = '${BF_OIIO_LIBPATH}/libOpenImageIO.a ${BF_OPENEXR}/lib/libIlmImf.a ${BF_JPEG}/lib/libjpeg.a'
 BF_OIIO_LIBPATH = '${BF_OIIO}/lib'

+WITH_BF_CYCLES_OSL = False
+WITH_BF_STATICOSL = False
+BF_OSL = '/opt/lib/osl'
+BF_OSL_INC = '${BF_OSL}/include'
+# note oslexec would passed via program linkflags, which is needed to
+# make llvm happy with osl_allocate_closure_component
+BF_OSL_LIB = 'oslcomp oslexec oslquery'
+BF_OSL_LIBPATH = '${BF_OSL}/lib'
+BF_OSL_COMPILER = '${BF_OSL}/bin/oslc'
+
+WITH_BF_LLVM = False
+WITH_BF_STATICLLVM = False
+BF_LLVM = '/opt/lib/llvm-3.1'
+BF_LLVM_LIB = 'LLVMBitReader LLVMJIT LLVMipo LLVMVectorize LLVMBitWriter LLVMX86CodeGen LLVMX86Desc LLVMX86Info LLVMX86AsmPrinter ' + \
+    'LLVMX86Utils LLVMSelectionDAG LLVMCodeGen LLVMScalarOpts LLVMInstCombine LLVMTransformUtils LLVMipa LLVMAnalysis LLVMExecutionEngine ' + \
+    'LLVMTarget LLVMMC LLVMCore LLVMSupport'
+BF_LLVM_LIBPATH = '${BF_LLVM}/lib'
+
 # Color management
 WITH_BF_OCIO = True
 WITH_BF_STATICOCIO = True
@@ -144,3 +162,4 @@ WITH_BF_OCEANSIM = True
 BF_DEBUG = False
 REL_CCFLAGS = ['-O2', '-msse', '-msse2']  # C & C++
 PLATFORM_LINKFLAGS = ['-lrt']
+BF_PROGRAM_LINKFLAGS = ['-Wl,--whole-archive', '-loslexec', '-Wl,--no-whole-archive']
--- a/build_files/buildbot/config/user-config-glibc211-x86_64.py
+++ b/build_files/buildbot/config/user-config-glibc211-x86_64.py
@@ -119,6 +119,24 @@ BF_OIIO_INC = '${BF_OIIO}/include'
 BF_OIIO_LIB_STATIC = '${BF_OIIO_LIBPATH}/libOpenImageIO.a ${BF_OPENEXR}/lib/libIlmImf.a ${BF_JPEG}/lib/libjpeg.a'
 BF_OIIO_LIBPATH = '${BF_OIIO}/lib'

+WITH_BF_CYCLES_OSL = True
+WITH_BF_STATICOSL = False
+BF_OSL = '/opt/lib/osl'
+BF_OSL_INC = '${BF_OSL}/include'
+# note oslexec would passed via program linkflags, which is needed to
+# make llvm happy with osl_allocate_closure_component
+BF_OSL_LIB = 'oslcomp oslexec oslquery'
+BF_OSL_LIBPATH = '${BF_OSL}/lib'
+BF_OSL_COMPILER = '${BF_OSL}/bin/oslc'
+
+WITH_BF_LLVM = True
+WITH_BF_STATICLLVM = False
+BF_LLVM = '/opt/lib/llvm-3.1'
+BF_LLVM_LIB = 'LLVMBitReader LLVMJIT LLVMipo LLVMVectorize LLVMBitWriter LLVMX86CodeGen LLVMX86Desc LLVMX86Info LLVMX86AsmPrinter ' + \
+    'LLVMX86Utils LLVMSelectionDAG LLVMCodeGen LLVMScalarOpts LLVMInstCombine LLVMTransformUtils LLVMipa LLVMAnalysis LLVMExecutionEngine ' + \
+    'LLVMTarget LLVMMC LLVMCore LLVMSupport'
+BF_LLVM_LIBPATH = '${BF_LLVM}/lib'
+
 # Color management
 WITH_BF_OCIO = True
 WITH_BF_STATICOCIO = True
@@ -143,3 +161,4 @@ WITH_BF_OCEANSIM = True
 BF_DEBUG = False
 REL_CCFLAGS = ['-O2', '-msse', '-msse2']  # C & C++
 PLATFORM_LINKFLAGS = ['-lrt']
+BF_PROGRAM_LINKFLAGS = ['-Wl,--whole-archive', '-loslexec', '-Wl,--no-whole-archive']
--- a/build_files/buildbot/config/user-config-glibc27-i686.py
+++ b/build_files/buildbot/config/user-config-glibc27-i686.py
@@ -107,6 +107,24 @@ BF_OIIO_INC = '${BF_OIIO}/include'
 BF_OIIO_LIB_STATIC = '${BF_OIIO_LIBPATH}/libOpenImageIO.a ${BF_OPENEXR}/lib/libIlmImf.a'
 BF_OIIO_LIBPATH = '${BF_OIIO}/lib'

+WITH_BF_CYCLES_OSL = False
+WITH_BF_STATICOSL = False
+BF_OSL = '/opt/osl'
+BF_OSL_INC = '${BF_OSL}/include'
+# note oslexec would passed via program linkflags, which is needed to
+# make llvm happy with osl_allocate_closure_component
+BF_OSL_LIB = 'oslcomp oslexec oslquery'
+BF_OSL_LIBPATH = '${BF_OSL}/lib'
+BF_OSL_COMPILER = '${BF_OSL}/bin/oslc'
+
+WITH_BF_LLVM = False
+WITH_BF_STATICLLVM = False
+BF_LLVM = '/opt/llvm-3.1'
+BF_LLVM_LIB = 'LLVMBitReader LLVMJIT LLVMipo LLVMVectorize LLVMBitWriter LLVMX86CodeGen LLVMX86Desc LLVMX86Info LLVMX86AsmPrinter ' + \
+    'LLVMX86Utils LLVMSelectionDAG LLVMCodeGen LLVMScalarOpts LLVMInstCombine LLVMTransformUtils LLVMipa LLVMAnalysis LLVMExecutionEngine ' + \
+    'LLVMTarget LLVMMC LLVMCore LLVMSupport'
+BF_LLVM_LIBPATH = '${BF_LLVM}/lib'
+
 # Color management
 WITH_BF_OCIO = True
 WITH_BF_STATICOCIO = True
@@ -129,3 +147,4 @@ WITH_BF_OCEANSIM = True
 BF_DEBUG = False
 REL_CCFLAGS = ['-O2']  # C & C++
 PLATFORM_LINKFLAGS = ['-L/home/sources/staticlibs/lib32']
+BF_PROGRAM_LINKFLAGS = ['-Wl,--whole-archive', '-loslexec', '-Wl,--no-whole-archive']
--- a/build_files/buildbot/config/user-config-glibc27-x86_64.py
+++ b/build_files/buildbot/config/user-config-glibc27-x86_64.py
@@ -106,6 +106,24 @@ BF_OIIO_INC = '${BF_OIIO}/include'
 BF_OIIO_LIB_STATIC = '${BF_OIIO_LIBPATH}/libOpenImageIO.a ${BF_OPENEXR}/lib/libIlmImf.a'
 BF_OIIO_LIBPATH = '${BF_OIIO}/lib'

+WITH_BF_CYCLES_OSL = True
+WITH_BF_STATICOSL = False
+BF_OSL = '/opt/osl'
+BF_OSL_INC = '${BF_OSL}/include'
+# note oslexec would passed via program linkflags, which is needed to
+# make llvm happy with osl_allocate_closure_component
+BF_OSL_LIB = 'oslcomp oslexec oslquery'
+BF_OSL_LIBPATH = '${BF_OSL}/lib'
+BF_OSL_COMPILER = '${BF_OSL}/bin/oslc'
+
+WITH_BF_LLVM = True
+WITH_BF_STATICLLVM = False
+BF_LLVM = '/opt/llvm-3.1'
+BF_LLVM_LIB = 'LLVMBitReader LLVMJIT LLVMipo LLVMVectorize LLVMBitWriter LLVMX86CodeGen LLVMX86Desc LLVMX86Info LLVMX86AsmPrinter ' + \
+    'LLVMX86Utils LLVMSelectionDAG LLVMCodeGen LLVMScalarOpts LLVMInstCombine LLVMTransformUtils LLVMipa LLVMAnalysis LLVMExecutionEngine ' + \
+    'LLVMTarget LLVMMC LLVMCore LLVMSupport'
+BF_LLVM_LIBPATH = '${BF_LLVM}/lib'
+
 # Color management
 WITH_BF_OCIO = True
 WITH_BF_STATICOCIO = True
@@ -128,3 +146,4 @@ WITH_BF_OCEANSIM = True
 BF_DEBUG = False
 REL_CCFLAGS = ['-O2', '-msse', '-msse2']  # C & C++
 PLATFORM_LINKFLAGS = ['-L/home/sources/staticlibs/lib64']
+BF_PROGRAM_LINKFLAGS = ['-Wl,--whole-archive', '-loslexec', '-Wl,--no-whole-archive']
--- a/build_files/scons/tools/Blender.py
+++ b/build_files/scons/tools/Blender.py
@@ -222,6 +222,16 @@ def setup_staticlibs(lenv):
        if lenv['WITH_BF_STATICBOOST']:
            statlibs += Split(lenv['BF_BOOST_LIB_STATIC'])

+    if lenv['WITH_BF_CYCLES_OSL']:
+        libincs += Split(lenv['BF_OSL_LIBPATH'])
+        if lenv['WITH_BF_STATICOSL']:
+            statlibs += Split(lenv['BF_OSL_LIB_STATIC'])
+
+    if lenv['WITH_BF_LLVM']:
+        libincs += Split(lenv['BF_LLVM_LIBPATH'])
+        if lenv['WITH_BF_STATICLLVM']:
+            statlibs += Split(lenv['BF_LLVM_LIB_STATIC'])
+
    # setting this last so any overriding of manually libs could be handled
    if lenv['OURPLATFORM'] not in ('win32-vc', 'win32-mingw', 'win64-vc', 'linuxcross', 'win64-mingw'):
        libincs.append('/usr/lib')
@@ -314,6 +324,12 @@ def setup_syslibs(lenv):
        if lenv['WITH_BF_INTERNATIONAL']:
            syslibs += Split(lenv['BF_BOOST_LIB_INTERNATIONAL'])

+    if lenv['WITH_BF_CYCLES_OSL'] and not lenv['WITH_BF_STATICOSL']:
+        syslibs += Split(lenv['BF_OSL_LIB'])
+
+    if lenv['WITH_BF_LLVM'] and not lenv['WITH_BF_STATICLLVM']:
+        syslibs += Split(lenv['BF_LLVM_LIB'])
+
    if not lenv['WITH_BF_STATICJPEG']:
        syslibs += Split(lenv['BF_JPEG_LIB'])

@@ -883,6 +899,7 @@ class BlenderEnvironment(SConsEnvironment):
        print bc.HEADER+'Configuring program '+bc.ENDC+bc.OKGREEN+progname+bc.ENDC
        lenv = self.Clone()
        lenv.Append(LINKFLAGS = lenv['PLATFORM_LINKFLAGS'])
+        lenv.Append(LINKFLAGS = lenv['BF_PROGRAM_LINKFLAGS'])
        if lenv['OURPLATFORM'] in ('win32-mingw', 'win64-mingw', 'linuxcross', 'cygwin', 'linux'):
            lenv.Replace(LINK = '$CXX')
        if lenv['OURPLATFORM'] in ('win32-vc', 'cygwin', 'win64-vc'):
--- a/build_files/scons/tools/btools.py
+++ b/build_files/scons/tools/btools.py
@@ -164,7 +164,9 @@ def validate_arguments(args, bc):
            'WITH_BF_OIIO', 'WITH_BF_STATICOIIO', 'BF_OIIO', 'BF_OIIO_INC', 'BF_OIIO_LIB', 'BF_OIIO_LIB_STATIC', 'BF_OIIO_LIBPATH',
            'WITH_BF_OCIO', 'WITH_BF_STATICOCIO', 'BF_OCIO', 'BF_OCIO_INC', 'BF_OCIO_LIB', 'BF_OCIO_LIB_STATIC', 'BF_OCIO_LIBPATH',
            'WITH_BF_BOOST', 'WITH_BF_STATICBOOST', 'BF_BOOST', 'BF_BOOST_INC', 'BF_BOOST_LIB', 'BF_BOOST_LIB_INTERNATIONAL', 'BF_BOOST_LIB_STATIC', 'BF_BOOST_LIBPATH',
-            'WITH_BF_LIBMV'
+            'WITH_BF_LIBMV',
+            'WITH_BF_CYCLES_OSL', 'WITH_BF_STATICOSL', 'BF_OSL', 'BF_OSL_INC', 'BF_OSL_LIB', 'BF_OSL_LIBPATH', 'BF_OSL_LIB_STATIC', 'BF_OSL_COMPILER',
+            'WITH_BF_LLVM', 'WITH_BF_STATICLLVM', 'BF_LLVM', 'BF_LLVM_LIB', 'BF_LLVM_LIBPATH', 'BF_LLVM_LIB_STATIC', 'BF_PROGRAM_LINKFLAGS'
            ]
    
    # Have options here that scons expects to be lists
@@ -178,7 +180,7 @@ def validate_arguments(args, bc):
            'BF_DEBUG_CFLAGS', 'BF_DEBUG_CCFLAGS', 'BF_DEBUG_CXXFLAGS',
            'C_WARN', 'CC_WARN', 'CXX_WARN',
            'LLIBS', 'PLATFORM_LINKFLAGS','MACOSX_ARCHITECTURE', 'MACOSX_SDK_CHECK', 'XCODE_CUR_VER',
-            'BF_CYCLES_CUDA_BINARIES_ARCH',
+            'BF_CYCLES_CUDA_BINARIES_ARCH', 'BF_PROGRAM_LINKFLAGS'
    ]
    
    
@@ -595,7 +597,25 @@ def read_opts(env, cfg, args):
        ('BF_BOOST_LIB_STATIC', 'Boost static library', ''),

        (BoolVariable('WITH_GHOST_XDND', 'Build with drag-n-drop support on Linux platforms using XDND protocol', True)),
-        (BoolVariable('WITH_BF_COMPOSITOR_LEGACY', 'Enable the legacy compositor', False))
+        (BoolVariable('WITH_BF_COMPOSITOR_LEGACY', 'Enable the legacy compositor', False)),
+
+        (BoolVariable('WITH_BF_CYCLES_OSL', 'Build with OSL sypport in Cycles', False)),
+        (BoolVariable('WITH_BF_STATICOSL', 'Staticly link to OSL', False)),
+        ('BF_OSL', 'OSL root path', ''),
+        ('BF_OSL_INC', 'OSL include path', ''),
+        ('BF_OSL_LIB', 'OSL library', ''),
+        ('BF_OSL_LIBPATH', 'OSL library path', ''),
+        ('BF_OSL_LIB_STATIC', 'OSL static library', ''),
+        ('BF_OSL_COMPILER', 'OSL compiler', ''),
+
+        (BoolVariable('WITH_BF_LLVM', 'Build with LLVM sypport (required for OSL)', False)),
+        (BoolVariable('WITH_BF_STATICLLVM', 'Staticly link to LLVM', False)),
+        ('BF_LLVM', 'LLVM root path', ''),
+        ('BF_LLVM_LIB', 'LLVM library', ''),
+        ('BF_LLVM_LIBPATH', 'LLVM library path', ''),
+        ('BF_LLVM_LIB_STATIC', 'LLVM static library', ''),
+
+        ('BF_PROGRAM_LINKFLAGS', 'Link flags applied only to final binaries (blender and blenderplayer, not makesrna/makesdna)', '')
    ) # end of opts.AddOptions()

    return localopts
--- a/intern/cycles/SConscript
+++ b/intern/cycles/SConscript
@@ -23,6 +23,10 @@ defs.append('WITH_OPENCL')
 defs.append('WITH_MULTI')
 defs.append('WITH_CUDA')

+if env['WITH_BF_CYCLES_OSL']:
+    defs.append('WITH_OSL')
+    incs.append(cycles['BF_OSL_INC'])
+
 if env['WITH_BF_CYCLES_CUDA_BINARIES']:
    defs.append('WITH_CUDA_BINARIES')

@@ -62,6 +66,12 @@ if env['WITH_BF_RAYOPTIMIZATION']:

 cycles.BlenderLib('bf_intern_cycles', sources, incs, defs, libtype=['intern'], priority=[0], cxx_compileflags=cxxflags)

+if env['WITH_BF_CYCLES_OSL']:
+    oso_files = SConscript(['kernel/shaders/SConscript'])
+    cycles.Depends("kernel/osl/osl_shader.o", oso_files)
+
+    SConscript(['kernel/osl/SConscript'])
+
 # cuda kernel binaries
 if env['WITH_BF_CYCLES_CUDA_BINARIES']:
    kernel_binaries = SConscript(['kernel/SConscript'])
--- a/intern/cycles/kernel/osl/SConscript
+++ b/intern/cycles/kernel/osl/SConscript
@@ -0,0 +1,21 @@
+#!/usr/bin/python
+
+Import('env')
+
+defs = []
+sources = env.Glob('*.cpp')
+
+incs = '. .. ../svm ../../render ../../util ../../device'
+
+incs += ' ' + env['BF_OIIO_INC']
+incs += ' ' + env['BF_BOOST_INC']
+incs += ' ' + env['BF_OSL_INC']
+
+defs.append('CCL_NAMESPACE_BEGIN=namespace ccl {')
+defs.append('CCL_NAMESPACE_END=}')
+defs.append('WITH_OSL')
+
+cxxflags = env['CXXFLAGS']
+cxxflags = "-fno-rtti -DBOOST_NO_RTTI -DBOOST_NO_TYPEID"
+
+env.BlenderLib ('cycles_kernel_osl', sources, Split(incs), defs, libtype=['intern'], priority=[10], cxx_compileflags=Split(cxxflags))
--- a/intern/cycles/kernel/shaders/SConscript
+++ b/intern/cycles/kernel/shaders/SConscript
@@ -0,0 +1,42 @@
+#!/usr/bin/python
+import sys
+import os
+import Blender as B
+
+def normpath(path):
+    return os.path.abspath(os.path.normpath(path))
+
+Import ('env')
+
+oso_files = []
+
+if env['WITH_BF_CYCLES_OSL']:
+    shaders = env.Clone()
+
+    # cuda info
+    osl_compiler = env['BF_OSL_COMPILER']
+
+    # build directory
+    root_build_dir = normpath(env['BF_BUILDDIR'])
+    build_dir = os.path.join(root_build_dir, 'intern/cycles/kernel/shaders')
+
+    # source directories and files
+    source_dir = Dir('.').srcnode().path
+
+    # dependencies
+    dependencies = shaders.Glob('*.h')
+
+    for f in os.listdir(source_dir):
+        if f.endswith('.osl'):
+            osl_file = os.path.join(source_dir, f)
+            oso_file = os.path.join(build_dir, f.replace('.osl', '.oso'))
+
+            command = "%s -O2 -I%s -o %s %s" % (osl_compiler, source_dir, oso_file, osl_file)
+
+            shaders.Command(oso_file, f, command)
+            shaders.Depends(oso_file, [f] + dependencies)
+
+            oso_files.append(oso_file)
+
+
+Return('oso_files')