griefith/test2
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Cycles: Make OSL shader compilation threadsafe and multi-threaded

Browse Source 
The original OSL Shading System API was stateful: You'd create a shader
group, configure it, and then end it. However, this means that only one
group can be created at a time. Further, since Cycles reuses the
Shading System for multiple instances (e.g. viewport render and
material preview), a process-wide mutex is needed.

However, for years now OSL has had a better interface, where you
explicitly provide the group you refer to. With this, we can not only
get rid of the mutex, but actually multi-thread the shader setup even
within one instance.

Realistically, most time is still spent in the JIT stage, but it's
still better than nothing.

Pull Request: https://projects.blender.org/blender/blender/pulls/130133
This commit is contained in:
Lukas Stockner
2024-12-09 14:36:35 +01:00
parent 745b93062c
commit d1796b8df0
2 changed files with 77 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

137
intern/cycles/scene/osl.cpp
View File

@@ -26,6 +26,7 @@
# include "util/path.h"
# include "util/progress.h"
# include "util/projection.h"
# include "util/task.h"
#endif
@@ -43,7 +44,6 @@ OSL::ErrorHandler OSLShaderManager::errhandler;
map<int, OSL::ShadingSystem *> OSLShaderManager::ss_shared;
int OSLShaderManager::ss_shared_users = 0;
thread_mutex OSLShaderManager::ss_shared_mutex;
thread_mutex OSLShaderManager::ss_mutex;
int OSLCompiler::texture_shared_unique_id = 0;
@@ -117,35 +117,49 @@ void OSLShaderManager::device_update_specific(Device *device,
/* create shaders */
Shader *background_shader = scene->background->get_shader(scene);
/* compile each shader to OSL shader groups */
TaskPool task_pool;
foreach (Shader *shader, scene->shaders) {
assert(shader->graph);
if (progress.get_cancel())
return;
auto compile = [this, scene, shader, background_shader](Device *sub_device) {
OSL::ShadingSystem *ss = ss_shared[sub_device->info.type];
/* we can only compile one shader at the time as the OSL ShadingSytem
* has a single state, but we put the lock here so different renders can
* compile shaders alternating */
thread_scoped_lock lock(ss_mutex);
OSLCompiler compiler(this, ss, scene);
compiler.background = (shader == background_shader);
compiler.compile(shader);
};
device->foreach_device(
[this, scene, shader, background = (shader == background_shader)](Device *sub_device) {
OSLGlobals *og = (OSLGlobals *)sub_device->get_cpu_osl_memory();
OSL::ShadingSystem *ss = ss_shared[sub_device->info.type];
task_pool.push(function_bind(&Device::foreach_device, device, compile));
}
task_pool.wait_work();
OSLCompiler compiler(this, ss, scene);
compiler.background = background;
compiler.compile(og, shader);
});
if (progress.get_cancel()) {
return;
}
/* collect shader groups from all shaders */
foreach (Shader *shader, scene->shaders) {
device->foreach_device([shader, background_shader](Device *sub_device) {
OSLGlobals *og = (OSLGlobals *)sub_device->get_cpu_osl_memory();
/* push state to array for lookup */
og->surface_state.push_back(shader->osl_surface_ref);
og->volume_state.push_back(shader->osl_volume_ref);
og->displacement_state.push_back(shader->osl_displacement_ref);
og->bump_state.push_back(shader->osl_surface_bump_ref);
if (shader == background_shader) {
og->background_state = shader->osl_surface_ref;
}
});
if (shader->emission_sampling != EMISSION_SAMPLING_NONE)
scene->light_manager->tag_update(scene, LightManager::SHADER_COMPILED);
}
/* setup shader engine */
int background_id = scene->shader_manager->get_shader_id(background_shader);
device->foreach_device([background_id](Device *sub_device) {
device->foreach_device([](Device *sub_device) {
OSLGlobals *og = (OSLGlobals *)sub_device->get_cpu_osl_memory();
OSL::ShadingSystem *ss = ss_shared[sub_device->info.type];
@@ -153,7 +167,6 @@ void OSLShaderManager::device_update_specific(Device *device,
og->ts = ts_shared;
og->services = static_cast<OSLRenderServices *>(ss->renderer());
og->background_state = og->surface_state[background_id & SHADER_MASK];
og->use = true;
});
@@ -848,13 +861,13 @@ void OSLCompiler::add(ShaderNode *node, const char *name, bool isfilepath)
/* Create shader of the appropriate type. OSL only distinguishes between "surface"
* and "displacement" at the moment. */
if (current_type == SHADER_TYPE_SURFACE)
ss->Shader("surface", name, id(node).c_str());
ss->Shader(*current_group, "surface", name, id(node).c_str());
else if (current_type == SHADER_TYPE_VOLUME)
ss->Shader("surface", name, id(node).c_str());
ss->Shader(*current_group, "surface", name, id(node).c_str());
else if (current_type == SHADER_TYPE_DISPLACEMENT)
ss->Shader("displacement", name, id(node).c_str());
ss->Shader(*current_group, "displacement", name, id(node).c_str());
else if (current_type == SHADER_TYPE_BUMP)
ss->Shader("displacement", name, id(node).c_str());
ss->Shader(*current_group, "displacement", name, id(node).c_str());
else
assert(0);
@@ -870,7 +883,8 @@ void OSLCompiler::add(ShaderNode *node, const char *name, bool isfilepath)
string param_from = compatible_name(input->link->parent, input->link);
string param_to = compatible_name(node, input);
ss->ConnectShaders(id_from.c_str(), param_from.c_str(), id_to.c_str(), param_to.c_str());
ss->ConnectShaders(
*current_group, id_from.c_str(), param_from.c_str(), id_to.c_str(), param_to.c_str());
}
}
@@ -922,59 +936,62 @@ void OSLCompiler::parameter(ShaderNode *node, const char *name)
switch (socket.type) {
case SocketType::BOOLEAN: {
int value = node->get_bool(socket);
ss->Parameter(name, TypeInt, &value);
ss->Parameter(*current_group, name, TypeInt, &value);
break;
}
case SocketType::FLOAT: {
float value = node->get_float(socket);
ss->Parameter(uname, TypeFloat, &value);
ss->Parameter(*current_group, uname, TypeFloat, &value);
break;
}
case SocketType::INT: {
int value = node->get_int(socket);
ss->Parameter(uname, TypeInt, &value);
ss->Parameter(*current_group, uname, TypeInt, &value);
break;
}
case SocketType::COLOR: {
float3 value = node->get_float3(socket);
ss->Parameter(uname, TypeColor, &value);
ss->Parameter(*current_group, uname, TypeColor, &value);
break;
}
case SocketType::VECTOR: {
float3 value = node->get_float3(socket);
ss->Parameter(uname, TypeVector, &value);
ss->Parameter(*current_group, uname, TypeVector, &value);
break;
}
case SocketType::POINT: {
float3 value = node->get_float3(socket);
ss->Parameter(uname, TypePoint, &value);
ss->Parameter(*current_group, uname, TypePoint, &value);
break;
}
case SocketType::NORMAL: {
float3 value = node->get_float3(socket);
ss->Parameter(uname, TypeNormal, &value);
ss->Parameter(*current_group, uname, TypeNormal, &value);
break;
}
case SocketType::POINT2: {
float2 value = node->get_float2(socket);
ss->Parameter(uname, TypeDesc(TypeDesc::FLOAT, TypeDesc::VEC2, TypeDesc::POINT), &value);
ss->Parameter(*current_group,
uname,
TypeDesc(TypeDesc::FLOAT, TypeDesc::VEC2, TypeDesc::POINT),
&value);
break;
}
case SocketType::STRING: {
ustring value = node->get_string(socket);
ss->Parameter(uname, TypeString, &value);
ss->Parameter(*current_group, uname, TypeString, &value);
break;
}
case SocketType::ENUM: {
ustring value = node->get_string(socket);
ss->Parameter(uname, TypeString, &value);
ss->Parameter(*current_group, uname, TypeString, &value);
break;
}
case SocketType::TRANSFORM: {
Transform value = node->get_transform(socket);
ProjectionTransform projection(value);
projection = projection_transpose(projection);
ss->Parameter(uname, TypeMatrix, &projection);
ss->Parameter(*current_group, uname, TypeMatrix, &projection);
break;
}
case SocketType::BOOLEAN_ARRAY: {
@@ -983,17 +1000,17 @@ void OSLCompiler::parameter(ShaderNode *node, const char *name)
array<int> intvalue(value.size());
for (size_t i = 0; i < value.size(); i++)
intvalue[i] = value[i];
ss->Parameter(uname, array_typedesc(TypeInt, value.size()), intvalue.data());
ss->Parameter(*current_group, uname, array_typedesc(TypeInt, value.size()), intvalue.data());
break;
}
case SocketType::FLOAT_ARRAY: {
const array<float> &value = node->get_float_array(socket);
ss->Parameter(uname, array_typedesc(TypeFloat, value.size()), value.data());
ss->Parameter(*current_group, uname, array_typedesc(TypeFloat, value.size()), value.data());
break;
}
case SocketType::INT_ARRAY: {
const array<int> &value = node->get_int_array(socket);
ss->Parameter(uname, array_typedesc(TypeInt, value.size()), value.data());
ss->Parameter(*current_group, uname, array_typedesc(TypeInt, value.size()), value.data());
break;
}
case SocketType::COLOR_ARRAY:
@@ -1029,12 +1046,13 @@ void OSLCompiler::parameter(ShaderNode *node, const char *name)
fvalue[j++] = value[i].z;
}
ss->Parameter(uname, array_typedesc(typedesc, value.size()), fvalue.data());
ss->Parameter(*current_group, uname, array_typedesc(typedesc, value.size()), fvalue.data());
break;
}
case SocketType::POINT2_ARRAY: {
const array<float2> &value = node->get_float2_array(socket);
ss->Parameter(
*current_group,
uname,
array_typedesc(TypeDesc(TypeDesc::FLOAT, TypeDesc::VEC2, TypeDesc::POINT), value.size()),
value.data());
@@ -1042,7 +1060,7 @@ void OSLCompiler::parameter(ShaderNode *node, const char *name)
}
case SocketType::STRING_ARRAY: {
const array<ustring> &value = node->get_string_array(socket);
ss->Parameter(uname, array_typedesc(TypeString, value.size()), value.data());
ss->Parameter(*current_group, uname, array_typedesc(TypeString, value.size()), value.data());
break;
}
case SocketType::TRANSFORM_ARRAY: {
@@ -1051,7 +1069,8 @@ void OSLCompiler::parameter(ShaderNode *node, const char *name)
for (size_t i = 0; i < value.size(); i++) {
fvalue[i] = projection_transpose(ProjectionTransform(value[i]));
}
ss->Parameter(uname, array_typedesc(TypeMatrix, fvalue.size()), fvalue.data());
ss->Parameter(
*current_group, uname, array_typedesc(TypeMatrix, fvalue.size()), fvalue.data());
break;
}
case SocketType::CLOSURE:
@@ -1069,57 +1088,57 @@ void OSLCompiler::parameter(ShaderNode *node, const char *name)
void OSLCompiler::parameter(const char *name, float f)
{
ss->Parameter(name, TypeFloat, &f);
ss->Parameter(*current_group, name, TypeFloat, &f);
}
void OSLCompiler::parameter_color(const char *name, float3 f)
{
ss->Parameter(name, TypeColor, &f);
ss->Parameter(*current_group, name, TypeColor, &f);
}
void OSLCompiler::parameter_point(const char *name, float3 f)
{
ss->Parameter(name, TypePoint, &f);
ss->Parameter(*current_group, name, TypePoint, &f);
}
void OSLCompiler::parameter_normal(const char *name, float3 f)
{
ss->Parameter(name, TypeNormal, &f);
ss->Parameter(*current_group, name, TypeNormal, &f);
}
void OSLCompiler::parameter_vector(const char *name, float3 f)
{
ss->Parameter(name, TypeVector, &f);
ss->Parameter(*current_group, name, TypeVector, &f);
}
void OSLCompiler::parameter(const char *name, int f)
{
ss->Parameter(name, TypeInt, &f);
ss->Parameter(*current_group, name, TypeInt, &f);
}
void OSLCompiler::parameter(const char *name, const char *s)
{
ss->Parameter(name, TypeString, &s);
ss->Parameter(*current_group, name, TypeString, &s);
}
void OSLCompiler::parameter(const char *name, ustring s)
{
const char *str = s.c_str();
ss->Parameter(name, TypeString, &str);
ss->Parameter(*current_group, name, TypeString, &str);
}
void OSLCompiler::parameter(const char *name, const Transform &tfm)
{
ProjectionTransform projection(tfm);
projection = projection_transpose(projection);
ss->Parameter(name, TypeMatrix, (float *)&projection);
ss->Parameter(*current_group, name, TypeMatrix, (float *)&projection);
}
void OSLCompiler::parameter_array(const char *name, const float f[], int arraylen)
{
TypeDesc type = TypeFloat;
type.arraylen = arraylen;
ss->Parameter(name, type, f);
ss->Parameter(*current_group, name, type, f);
}
void OSLCompiler::parameter_color_array(const char *name, const array<float3> &f)
@@ -1135,7 +1154,7 @@ void OSLCompiler::parameter_color_array(const char *name, const array<float3> &f
TypeDesc type = TypeColor;
type.arraylen = table.size();
ss->Parameter(name, type, table.data());
ss->Parameter(*current_group, name, type, table.data());
}
void OSLCompiler::parameter_attribute(const char *name, ustring s)
@@ -1217,7 +1236,7 @@ OSL::ShaderGroupRef OSLCompiler::compile_type(Shader *shader, ShaderGraph *graph
name.imbue(std::locale("C"));
name << "shader_" << shader->name.hash();
OSL::ShaderGroupRef group = ss->ShaderGroupBegin(name.str());
current_group = ss->ShaderGroupBegin(name.str());
ShaderNode *output = graph->output();
ShaderNodeSet dependencies;
@@ -1249,12 +1268,12 @@ OSL::ShaderGroupRef OSLCompiler::compile_type(Shader *shader, ShaderGraph *graph
else
assert(0);
ss->ShaderGroupEnd();
ss->ShaderGroupEnd(*current_group);
return group;
return std::move(current_group);
}
void OSLCompiler::compile(OSLGlobals *og, Shader *shader)
void OSLCompiler::compile(Shader *shader)
{
if (shader->is_modified()) {
ShaderGraph *graph = shader->graph;
@@ -1315,12 +1334,6 @@ void OSLCompiler::compile(OSLGlobals *og, Shader *shader)
/* Estimate emission for MIS. */
shader->estimate_emission();
}
/* push state to array for lookup */
og->surface_state.push_back(shader->osl_surface_ref);
og->volume_state.push_back(shader->osl_volume_ref);
og->displacement_state.push_back(shader->osl_displacement_ref);
og->bump_state.push_back(shader->osl_surface_bump_ref);
}
void OSLCompiler::parameter_texture(const char *name, ustring filename, ustring colorspace)

3
intern/cycles/scene/osl.h
View File

@@ -126,7 +126,7 @@ class OSLCompiler {
#ifdef WITH_OSL
OSLCompiler(OSLShaderManager *manager, OSL::ShadingSystem *shadingsys, Scene *scene);
#endif
void compile(OSLGlobals *og, Shader *shader);
void compile(Shader *shader);
void add(ShaderNode *node, const char *name, bool isfilepath = false);
@@ -177,6 +177,7 @@ class OSLCompiler {
ShaderType current_type;
Shader *current_shader;
OSL::ShaderGroupRef current_group;
static int texture_shared_unique_id;
};