griefith/test2
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

GPU: Remove runtime parsing of GLSL source

Browse Source 
See #129009 for context.

The preprocessor parses metadata and writes a header file containing
an inline function that inits the `GPUSource` with the metadata.

These header files are then included inside `gpu_shader_dependency.cc`.

This still keep the usage of the `metadata` enums and classes to avoid
pulling the whole blender module inside the preprocessor executable.

This speeds-up startup time in Debug build:
`gpu_shader_dependency_init`
- Before : 37ms
- After : 4ms

I didn't measure release, but it is unlikely to be noticeable (in the
order of 4ms > 1ms).

Pull Request: https://projects.blender.org/blender/blender/pulls/138070
This commit is contained in:
Clément Foucault
2025-04-29 15:32:36 +02:00
committed by Clément Foucault
parent 8b70166214
commit d24b0e4a4e
9 changed files with 334 additions and 303 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
build_files/cmake/macros.cmake
View File

@@ -1040,9 +1040,10 @@ function(glsl_to_c
)
# remove ../'s
get_filename_component(_file_from ${CMAKE_CURRENT_SOURCE_DIR}/${file_from} REALPATH)
get_filename_component(_file_tmp ${CMAKE_CURRENT_BINARY_DIR}/${file_from} REALPATH)
get_filename_component(_file_to ${CMAKE_CURRENT_BINARY_DIR}/${file_from}.c REALPATH)
get_filename_component(_file_from ${CMAKE_CURRENT_SOURCE_DIR}/${file_from} REALPATH)
get_filename_component(_file_tmp ${CMAKE_CURRENT_BINARY_DIR}/${file_from} REALPATH)
get_filename_component(_file_meta ${CMAKE_CURRENT_BINARY_DIR}/${file_from}.hh REALPATH)
get_filename_component(_file_to ${CMAKE_CURRENT_BINARY_DIR}/${file_from}.c REALPATH)
list(APPEND ${list_to_add} ${_file_to})
source_group(Generated FILES ${_file_to})
@@ -1052,14 +1053,15 @@ function(glsl_to_c
get_filename_component(_file_to_path ${_file_to} PATH)
add_custom_command(
OUTPUT ${_file_to}
OUTPUT ${_file_to} ${_file_meta}
COMMAND ${CMAKE_COMMAND} -E make_directory ${_file_to_path}
COMMAND "$<TARGET_FILE:glsl_preprocess>" ${_file_from} ${_file_tmp}
COMMAND "$<TARGET_FILE:glsl_preprocess>" ${_file_from} ${_file_tmp} ${_file_meta}
COMMAND "$<TARGET_FILE:datatoc>" ${_file_tmp} ${_file_to}
DEPENDS ${_file_from} datatoc glsl_preprocess)
set_source_files_properties(${_file_tmp} PROPERTIES GENERATED TRUE)
set_source_files_properties(${_file_to} PROPERTIES GENERATED TRUE)
set_source_files_properties(${_file_meta} PROPERTIES GENERATED TRUE)
endfunction()

7
intern/opencolorio/CMakeLists.txt
View File

@@ -67,15 +67,20 @@ if(WITH_OPENCOLORIO)
)
set(GLSL_SOURCE_CONTENT "")
set(GLSL_METADATA_CONTENT "")
foreach(GLSL_FILE ${GLSL_SRC})
get_filename_component(GLSL_FILE_NAME ${GLSL_FILE} NAME)
string(REPLACE "." "_" GLSL_FILE_NAME_UNDERSCORES ${GLSL_FILE_NAME})
string(APPEND GLSL_SOURCE_CONTENT "SHADER_SOURCE\(datatoc_${GLSL_FILE_NAME_UNDERSCORES}, \"${GLSL_FILE_NAME}\", \"${GLSL_FILE}\"\)\n")
string(APPEND GLSL_SOURCE_CONTENT "SHADER_SOURCE\(${GLSL_FILE_NAME_UNDERSCORES}, \"${GLSL_FILE_NAME}\", \"${GLSL_FILE}\"\)\n")
string(APPEND GLSL_METADATA_CONTENT "#include \"${GLSL_FILE}.hh\"\n")
endforeach()
set(glsl_source_list_file "${CMAKE_CURRENT_BINARY_DIR}/glsl_ocio_source_list.h")
file(GENERATE OUTPUT ${glsl_source_list_file} CONTENT "${GLSL_SOURCE_CONTENT}")
list(APPEND SRC ${glsl_source_list_file})
set(glsl_metadata_list_file "${CMAKE_CURRENT_BINARY_DIR}/glsl_ocio_metadata_list.hh")
file(GENERATE OUTPUT ${glsl_metadata_list_file} CONTENT "${GLSL_METADATA_CONTENT}")
list(APPEND SRC ${glsl_metadata_list_file})
list(APPEND INC ${CMAKE_CURRENT_BINARY_DIR})
target_include_directories(bf_ocio_shaders PUBLIC ${CMAKE_CURRENT_BINARY_DIR})

7
intern/opensubdiv/CMakeLists.txt
View File

@@ -102,15 +102,20 @@ if(WITH_OPENSUBDIV)
)
set(GLSL_SOURCE_CONTENT "")
set(GLSL_METADATA_CONTENT "")
foreach(GLSL_FILE ${GLSL_SRC})
get_filename_component(GLSL_FILE_NAME ${GLSL_FILE} NAME)
string(REPLACE "." "_" GLSL_FILE_NAME_UNDERSCORES ${GLSL_FILE_NAME})
string(APPEND GLSL_SOURCE_CONTENT "SHADER_SOURCE\(datatoc_${GLSL_FILE_NAME_UNDERSCORES}, \"${GLSL_FILE_NAME}\", \"${GLSL_FILE}\"\)\n")
string(APPEND GLSL_SOURCE_CONTENT "SHADER_SOURCE\(${GLSL_FILE_NAME_UNDERSCORES}, \"${GLSL_FILE_NAME}\", \"${GLSL_FILE}\"\)\n")
string(APPEND GLSL_METADATA_CONTENT "#include \"${GLSL_FILE}.hh\"\n")
endforeach()
set(glsl_source_list_file "${CMAKE_CURRENT_BINARY_DIR}/glsl_osd_source_list.h")
file(GENERATE OUTPUT ${glsl_source_list_file} CONTENT "${GLSL_SOURCE_CONTENT}")
list(APPEND SRC ${glsl_source_list_file})
set(glsl_metadata_list_file "${CMAKE_CURRENT_BINARY_DIR}/glsl_osd_metadata_list.hh")
file(GENERATE OUTPUT ${glsl_metadata_list_file} CONTENT "${GLSL_METADATA_CONTENT}")
list(APPEND SRC ${glsl_metadata_list_file})
list(APPEND INC ${CMAKE_CURRENT_BINARY_DIR})
target_include_directories(bf_osd_shaders PUBLIC ${CMAKE_CURRENT_BINARY_DIR})

7
source/blender/compositor/CMakeLists.txt
View File

@@ -315,15 +315,20 @@ list(APPEND LIB
)
set(GLSL_SOURCE_CONTENT "")
set(GLSL_METADATA_CONTENT "")
foreach(GLSL_FILE ${GLSL_SRC})
get_filename_component(GLSL_FILE_NAME ${GLSL_FILE} NAME)
string(REPLACE "." "_" GLSL_FILE_NAME_UNDERSCORES ${GLSL_FILE_NAME})
string(APPEND GLSL_SOURCE_CONTENT "SHADER_SOURCE\(datatoc_${GLSL_FILE_NAME_UNDERSCORES}, \"${GLSL_FILE_NAME}\", \"${GLSL_FILE}\"\)\n")
string(APPEND GLSL_SOURCE_CONTENT "SHADER_SOURCE\(${GLSL_FILE_NAME_UNDERSCORES}, \"${GLSL_FILE_NAME}\", \"${GLSL_FILE}\"\)\n")
string(APPEND GLSL_METADATA_CONTENT "#include \"${GLSL_FILE}.hh\"\n")
endforeach()
set(glsl_source_list_file "${CMAKE_CURRENT_BINARY_DIR}/glsl_compositor_source_list.h")
file(GENERATE OUTPUT ${glsl_source_list_file} CONTENT "${GLSL_SOURCE_CONTENT}")
list(APPEND SRC ${glsl_source_list_file})
set(glsl_metadata_list_file "${CMAKE_CURRENT_BINARY_DIR}/glsl_compositor_metadata_list.hh")
file(GENERATE OUTPUT ${glsl_metadata_list_file} CONTENT "${GLSL_METADATA_CONTENT}")
list(APPEND SRC ${glsl_metadata_list_file})
list(APPEND INC ${CMAKE_CURRENT_BINARY_DIR})
target_include_directories(bf_compositor_shaders PUBLIC ${CMAKE_CURRENT_BINARY_DIR})

8
source/blender/draw/CMakeLists.txt
View File

@@ -742,16 +742,20 @@ list(APPEND LIB
)
set(GLSL_SOURCE_CONTENT "")
set(GLSL_METADATA_CONTENT "")
foreach(GLSL_FILE ${GLSL_SRC})
get_filename_component(GLSL_FILE_NAME ${GLSL_FILE} NAME)
string(REPLACE "." "_" GLSL_FILE_NAME_UNDERSCORES ${GLSL_FILE_NAME})
string(APPEND GLSL_SOURCE_CONTENT "SHADER_SOURCE\(datatoc_${GLSL_FILE_NAME_UNDERSCORES}, \"${GLSL_FILE_NAME}\", \"${GLSL_FILE}\"\)\n")
string(APPEND GLSL_SOURCE_CONTENT "SHADER_SOURCE\(${GLSL_FILE_NAME_UNDERSCORES}, \"${GLSL_FILE_NAME}\", \"${GLSL_FILE}\"\)\n")
string(APPEND GLSL_METADATA_CONTENT "#include \"${GLSL_FILE}.hh\"\n")
endforeach()
set(glsl_source_list_file "${CMAKE_CURRENT_BINARY_DIR}/glsl_draw_source_list.h")
file(GENERATE OUTPUT ${glsl_source_list_file} CONTENT "${GLSL_SOURCE_CONTENT}")
list(APPEND SRC ${glsl_source_list_file})
set(glsl_metadata_list_file "${CMAKE_CURRENT_BINARY_DIR}/glsl_draw_metadata_list.hh")
file(GENERATE OUTPUT ${glsl_metadata_list_file} CONTENT "${GLSL_METADATA_CONTENT}")
list(APPEND SRC ${glsl_metadata_list_file})
list(APPEND INC ${CMAKE_CURRENT_BINARY_DIR})
target_include_directories(bf_draw_shaders PUBLIC ${CMAKE_CURRENT_BINARY_DIR})

7
source/blender/gpu/CMakeLists.txt
View File

@@ -716,15 +716,20 @@ list(APPEND LIB
)
set(GLSL_SOURCE_CONTENT "")
set(GLSL_METADATA_CONTENT "")
foreach(GLSL_FILE ${GLSL_SRC})
get_filename_component(GLSL_FILE_NAME ${GLSL_FILE} NAME)
string(REPLACE "." "_" GLSL_FILE_NAME_UNDERSCORES ${GLSL_FILE_NAME})
string(APPEND GLSL_SOURCE_CONTENT "SHADER_SOURCE\(datatoc_${GLSL_FILE_NAME_UNDERSCORES}, \"${GLSL_FILE_NAME}\", \"${GLSL_FILE}\"\)\n")
string(APPEND GLSL_SOURCE_CONTENT "SHADER_SOURCE\(${GLSL_FILE_NAME_UNDERSCORES}, \"${GLSL_FILE_NAME}\", \"${GLSL_FILE}\"\)\n")
string(APPEND GLSL_METADATA_CONTENT "#include \"${GLSL_FILE}.hh\"\n")
endforeach()
set(glsl_source_list_file "${CMAKE_CURRENT_BINARY_DIR}/glsl_gpu_source_list.h")
file(GENERATE OUTPUT ${glsl_source_list_file} CONTENT "${GLSL_SOURCE_CONTENT}")
list(APPEND SRC ${glsl_source_list_file})
set(glsl_metadata_list_file "${CMAKE_CURRENT_BINARY_DIR}/glsl_gpu_metadata_list.hh")
file(GENERATE OUTPUT ${glsl_metadata_list_file} CONTENT "${GLSL_METADATA_CONTENT}")
list(APPEND SRC ${glsl_metadata_list_file})
list(APPEND INC ${CMAKE_CURRENT_BINARY_DIR})
if(WITH_MOD_FLUID)

26
source/blender/gpu/glsl_preprocess/glsl_preprocess.cc
View File

@@ -15,13 +15,15 @@
int main(int argc, char **argv)
{
if (argc != 3) {
std::cerr << "Usage: glsl_preprocess <data_file_from> <data_file_to>" << std::endl;
if (argc != 4) {
std::cerr << "Usage: glsl_preprocess <data_file_from> <data_file_to> <metadata_file_to>"
<< std::endl;
exit(1);
}
const char *input_file_name = argv[1];
const char *output_file_name = argv[2];
const char *metadata_file_name = argv[3];
/* Open the input file for reading */
std::ifstream input_file(input_file_name);
@@ -38,6 +40,14 @@ int main(int argc, char **argv)
exit(1);
}
/* Open the output file for writing */
std::ofstream metadata_file(metadata_file_name, std::ofstream::out | std::ofstream::binary);
if (!output_file) {
std::cerr << "Error: Could not open output file " << metadata_file_name << std::endl;
input_file.close();
exit(1);
}
std::stringstream buffer;
buffer << input_file.rdbuf();
@@ -103,11 +113,21 @@ int main(int argc, char **argv)
language = Preprocessor::SourceLanguage::BLENDER_GLSL;
}
blender::gpu::shader::metadata::Source metadata;
output_file << processor.process(
language, buffer.str(), input_file_name, is_library, is_shared, report_error);
language, buffer.str(), input_file_name, is_library, is_shared, report_error, metadata);
/* TODO(fclem): Don't use regex for that. */
std::string metadata_function_name = "metadata_" +
std::regex_replace(
filename, std::regex(R"((?:.*)\/(.*))"), "$1");
std::replace(metadata_function_name.begin(), metadata_function_name.end(), '.', '_');
metadata_file << metadata.serialize(metadata_function_name);
input_file.close();
output_file.close();
metadata_file.close();
return error;
}

302
source/blender/gpu/glsl_preprocess/glsl_preprocess.hh
View File

@@ -18,6 +18,129 @@
namespace blender::gpu::shader {
/* Metadata extracted from shader source file.
* These are then converted to their GPU module equivalent. */
/* TODO(fclem): Make GPU enums standalone and directly use them instead of using separate enums
* and types. */
namespace metadata {
/* Compile-time hashing function which converts string to a 64bit hash. */
constexpr static uint64_t hash(const char *name)
{
uint64_t hash = 2166136261u;
while (*name) {
hash = hash * 16777619u;
hash = hash ^ *name;
++name;
}
return hash;
}
static uint64_t hash(const std::string &name)
{
return hash(name.c_str());
}
enum Builtin : uint64_t {
FragCoord = hash("gl_FragCoord"),
FrontFacing = hash("gl_FrontFacing"),
GlobalInvocationID = hash("gl_GlobalInvocationID"),
InstanceID = hash("gl_InstanceID"),
LocalInvocationID = hash("gl_LocalInvocationID"),
LocalInvocationIndex = hash("gl_LocalInvocationIndex"),
NumWorkGroup = hash("gl_NumWorkGroup"),
PointCoord = hash("gl_PointCoord"),
PointSize = hash("gl_PointSize"),
PrimitiveID = hash("gl_PrimitiveID"),
VertexID = hash("gl_VertexID"),
WorkGroupID = hash("gl_WorkGroupID"),
WorkGroupSize = hash("gl_WorkGroupSize"),
drw_debug = hash("drw_debug_"),
printf = hash("printf"),
assert = hash("assert"),
};
enum Qualifier : uint64_t {
in = hash("in"),
out = hash("out"),
inout = hash("inout"),
};
enum Type : uint64_t {
float1 = hash("float"),
float2 = hash("float2"),
float3 = hash("float3"),
float4 = hash("float4"),
float3x3 = hash("float3x3"),
float4x4 = hash("float4x4"),
sampler1DArray = hash("sampler1DArray"),
sampler2DArray = hash("sampler2DArray"),
sampler2D = hash("sampler2D"),
sampler3D = hash("sampler3D"),
Closure = hash("Closure"),
};
struct ArgumentFormat {
Qualifier qualifier;
Type type;
};
struct FunctionFormat {
std::string name;
std::vector<ArgumentFormat> arguments;
};
struct PrintfFormat {
uint32_t hash;
std::string format;
};
struct Source {
std::vector<Builtin> builtins;
/* Note: Could be a set, but for now the order matters. */
std::vector<std::string> dependencies;
std::vector<PrintfFormat> printf_formats;
std::vector<FunctionFormat> functions;
std::string serialize(const std::string &function_name) const
{
std::stringstream ss;
ss << "static void " << function_name
<< "(GPUSource &source, GPUFunctionDictionnary *g_functions, GPUPrintFormatMap *g_formats) "
"{\n";
for (auto function : functions) {
ss << " {\n";
ss << " Vector<metadata::ArgumentFormat> args = {\n";
for (auto arg : function.arguments) {
ss << " "
<< "metadata::ArgumentFormat{"
<< "metadata::Qualifier(" << std::to_string(uint64_t(arg.qualifier)) << "LLU), "
<< "metadata::Type(" << std::to_string(uint64_t(arg.type)) << "LLU)"
<< "},\n";
}
ss << " };\n";
ss << " source.add_function(\"" << function.name << "\", args, g_functions);\n";
ss << " }\n";
}
for (auto builtin : builtins) {
ss << " source.add_builtin(metadata::Builtin(" << std::to_string(builtin) << "LLU));\n";
}
for (auto dependency : dependencies) {
ss << " source.add_dependency(\"" << dependency << "\");\n";
}
for (auto format : printf_formats) {
ss << " source.add_printf_format(uint32_t(" << std::to_string(format.hash) << "), "
<< format.format << ", g_formats);\n";
}
/* Avoid warnings. */
ss << " UNUSED_VARS(source, g_functions, g_formats);\n";
ss << "}\n";
return ss.str();
}
};
} // namespace metadata
/**
* Shader source preprocessor that allow to mutate GLSL into cross API source that can be
* interpreted by the different GPU backends. Some syntax are mutated or reported as incompatible.
@@ -35,11 +158,8 @@ class Preprocessor {
};
std::vector<SharedVar> shared_vars_;
std::unordered_set<std::string> static_strings_;
std::unordered_set<std::string> gpu_builtins_;
/* Note: Could be a set, but for now the order matters. */
std::vector<std::string> dependencies_;
std::stringstream gpu_functions_;
metadata::Source metadata;
public:
enum SourceLanguage {
@@ -66,30 +186,14 @@ class Preprocessor {
return UNKNOWN;
}
/* Compile-time hashing function which converts string to a 64bit hash. */
constexpr static uint64_t hash(const char *name)
{
uint64_t hash = 2166136261u;
while (*name) {
hash = hash * 16777619u;
hash = hash ^ *name;
++name;
}
return hash;
}
static uint64_t hash(const std::string &name)
{
return hash(name.c_str());
}
/* Takes a whole source file and output processed source. */
std::string process(SourceLanguage language,
std::string str,
const std::string &filename,
bool do_parse_function,
bool do_small_type_linting,
report_callback report_error)
report_callback report_error,
metadata::Source &r_metadata)
{
if (language == UNKNOWN) {
report_error(std::smatch(), "Unknown file type");
@@ -97,7 +201,7 @@ class Preprocessor {
}
str = remove_comments(str, report_error);
threadgroup_variables_parsing(str);
parse_builtins(str);
parse_builtins(str, filename);
if (language == BLENDER_GLSL || language == CPP) {
if (do_parse_function) {
parse_library_functions(str);
@@ -125,16 +229,16 @@ class Preprocessor {
}
str = argument_decorator_macro_injection(str);
str = array_constructor_macro_injection(str);
return line_directive_prefix(filename) + str + threadgroup_variables_suffix() +
"//__blender_metadata_sta\n" + gpu_functions_.str() + static_strings_suffix() +
gpu_builtins_suffix(filename) + dependency_suffix() + "//__blender_metadata_end\n";
r_metadata = metadata;
return line_directive_prefix(filename) + str + threadgroup_variables_suffix();
}
/* Variant use for python shaders. */
std::string process(const std::string &str)
{
auto no_err_report = [](std::smatch, const char *) {};
return process(GLSL, str, "", false, false, no_err_report);
metadata::Source unused;
return process(GLSL, str, "", false, false, no_err_report, unused);
}
private:
@@ -221,7 +325,7 @@ class Preprocessor {
/* Skip info files. They are only for IDE linting. */
return;
}
dependencies_.emplace_back(dependency_name);
metadata.dependencies.emplace_back(dependency_name);
});
}
@@ -232,18 +336,6 @@ class Preprocessor {
return std::regex_replace(str, regex, "");
}
std::string dependency_suffix()
{
if (dependencies_.empty()) {
return "";
}
std::stringstream suffix;
for (const std::string &filename : dependencies_) {
suffix << "// " << std::to_string(hash("dependency")) << " " << filename << "\n";
}
return suffix.str();
}
void threadgroup_variables_parsing(const std::string &str)
{
std::regex regex(R"(shared\s+(\w+)\s+(\w+)([^;]*);)");
@@ -254,11 +346,14 @@ class Preprocessor {
void parse_library_functions(const std::string &str)
{
using namespace metadata;
std::regex regex_func(R"(void\s+(\w+)\s*\(([^)]+\))\s*\{)");
regex_global_search(str, regex_func, [&](const std::smatch &match) {
std::string name = match[1].str();
std::string args = match[2].str();
gpu_functions_ << "// " << hash("function") << " " << name;
FunctionFormat fn;
fn.name = name;
std::regex regex_arg(R"((?:(const|in|out|inout)\s)?(\w+)\s([\w\[\]]+)(?:,|\)))");
regex_global_search(args, regex_arg, [&](const std::smatch &arg) {
@@ -267,14 +362,19 @@ class Preprocessor {
if (qualifier.empty() || qualifier == "const") {
qualifier = "in";
}
gpu_functions_ << ' ' << hash(qualifier) << ' ' << hash(type);
fn.arguments.emplace_back(
ArgumentFormat{metadata::Qualifier(hash(qualifier)), metadata::Type(hash(type))});
});
gpu_functions_ << "\n";
metadata.functions.emplace_back(fn);
});
}
void parse_builtins(const std::string &str)
void parse_builtins(const std::string &str, const std::string &filename)
{
const bool skip_drw_debug = filename.find("draw_debug_draw_lib.glsl") != std::string::npos ||
filename.find("draw_debug_draw_display_vert.glsl") !=
std::string::npos;
using namespace metadata;
/* TODO: This can trigger false positive caused by disabled #if blocks. */
std::regex regex(
"("
@@ -297,28 +397,12 @@ class Preprocessor {
#endif
"printf"
")");
regex_global_search(
str, regex, [&](const std::smatch &match) { gpu_builtins_.insert(match[0].str()); });
}
std::string gpu_builtins_suffix(const std::string &filename)
{
if (gpu_builtins_.empty()) {
return "";
}
const bool skip_drw_debug = filename.find("draw_debug_draw_lib.glsl") != std::string::npos ||
filename.find("draw_debug_draw_display_vert.glsl") !=
std::string::npos;
std::stringstream suffix;
for (const std::string &str_var : gpu_builtins_) {
if (str_var == "drw_debug_" && skip_drw_debug) {
continue;
regex_global_search(str, regex, [&](const std::smatch &match) {
if (skip_drw_debug && match[0].str() == "drw_debug_") {
return;
}
suffix << "// " << hash("builtin") << " " << hash(str_var) << "\n";
}
return suffix.str();
metadata.builtins.emplace_back(Builtin(hash(match[0].str())));
});
}
template<typename ReportErrorF>
@@ -394,26 +478,30 @@ class Preprocessor {
return std::regex_replace(str, regex, replacement);
}
void static_strings_parsing(const std::string &str)
{
/* Matches any character inside a pair of un-escaped quote. */
std::regex regex(R"("(?:[^"])*")");
regex_global_search(
str, regex, [&](const std::smatch &match) { static_strings_.insert(match[0].str()); });
}
/* String hash are outputted inside GLSL and needs to fit 32 bits. */
static uint64_t hash_string(const std::string &str)
static uint32_t hash_string(const std::string &str)
{
uint64_t hash_64 = hash(str);
uint64_t hash_64 = metadata::hash(str);
uint32_t hash_32 = uint32_t(hash_64 ^ (hash_64 >> 32));
return hash_32;
}
void static_strings_parsing(const std::string &str)
{
using namespace metadata;
/* Matches any character inside a pair of un-escaped quote. */
std::regex regex(R"("(?:[^"])*")");
regex_global_search(str, regex, [&](const std::smatch &match) {
std::string format = match[0].str();
metadata.printf_formats.emplace_back(metadata::PrintfFormat{hash_string(format), format});
});
}
std::string static_strings_mutation(std::string str)
{
/* Replaces all matches by the respective string hash. */
for (const std::string &str_var : static_strings_) {
for (const metadata::PrintfFormat &format : metadata.printf_formats) {
const std::string &str_var = format.format;
std::regex escape_regex(R"([\\\.\^\$\+\(\)\[\]\{\}\|\?\*])");
std::string str_regex = std::regex_replace(str_var, escape_regex, "\\$&");
@@ -423,19 +511,6 @@ class Preprocessor {
return str;
}
std::string static_strings_suffix()
{
if (static_strings_.empty()) {
return "";
}
std::stringstream suffix;
for (const std::string &str_var : static_strings_) {
std::string no_quote = str_var.substr(1, str_var.size() - 2);
suffix << "// " << hash("string") << " " << hash_string(str_var) << " " << no_quote << "\n";
}
return suffix.str();
}
std::string enum_macro_injection(std::string str)
{
/**
@@ -677,7 +752,7 @@ class Preprocessor {
suffix << "\"" << filename << "\"";
}
#else
uint64_t hash_value = hash(filename);
uint64_t hash_value = metadata::hash(filename);
/* Fold the value so it fits the GLSL spec. */
hash_value = (hash_value ^ (hash_value >> 32)) & (~uint64_t(0) >> 33);
suffix << std::to_string(uint64_t(hash_value));
@@ -687,49 +762,4 @@ class Preprocessor {
}
};
/* Enum values of metadata that the preprocessor can append at the end of a source file.
* Eventually, remove the need for these and output the metadata inside header files. */
namespace metadata {
enum Builtin : uint64_t {
FragCoord = Preprocessor::hash("gl_FragCoord"),
FrontFacing = Preprocessor::hash("gl_FrontFacing"),
GlobalInvocationID = Preprocessor::hash("gl_GlobalInvocationID"),
InstanceID = Preprocessor::hash("gl_InstanceID"),
LocalInvocationID = Preprocessor::hash("gl_LocalInvocationID"),
LocalInvocationIndex = Preprocessor::hash("gl_LocalInvocationIndex"),
NumWorkGroup = Preprocessor::hash("gl_NumWorkGroup"),
PointCoord = Preprocessor::hash("gl_PointCoord"),
PointSize = Preprocessor::hash("gl_PointSize"),
PrimitiveID = Preprocessor::hash("gl_PrimitiveID"),
VertexID = Preprocessor::hash("gl_VertexID"),
WorkGroupID = Preprocessor::hash("gl_WorkGroupID"),
WorkGroupSize = Preprocessor::hash("gl_WorkGroupSize"),
drw_debug = Preprocessor::hash("drw_debug_"),
printf = Preprocessor::hash("printf"),
assert = Preprocessor::hash("assert"),
};
enum Qualifier : uint64_t {
in = Preprocessor::hash("in"),
out = Preprocessor::hash("out"),
inout = Preprocessor::hash("inout"),
};
enum Type : uint64_t {
float1 = Preprocessor::hash("float"),
float2 = Preprocessor::hash("float2"),
float3 = Preprocessor::hash("float3"),
float4 = Preprocessor::hash("float4"),
float3x3 = Preprocessor::hash("float3x3"),
float4x4 = Preprocessor::hash("float4x4"),
sampler1DArray = Preprocessor::hash("sampler1DArray"),
sampler2DArray = Preprocessor::hash("sampler2DArray"),
sampler2D = Preprocessor::hash("sampler2D"),
sampler3D = Preprocessor::hash("sampler3D"),
Closure = Preprocessor::hash("Closure"),
};
} // namespace metadata
} // namespace blender::gpu::shader

261
source/blender/gpu/intern/gpu_shader_dependency.cc
View File

@@ -32,7 +32,8 @@
#include "../glsl_preprocess/glsl_preprocess.hh"
extern "C" {
#define SHADER_SOURCE(datatoc, filename, filepath) extern char datatoc[];
#define SHADER_SOURCE(filename_underscore, filename, filepath) \
extern char datatoc_##filename_underscore[];
#include "glsl_compositor_source_list.h"
#include "glsl_draw_source_list.h"
#include "glsl_gpu_source_list.h"
@@ -61,11 +62,15 @@ struct GPUSource {
bool dependencies_init = false;
shader::BuiltinBits builtins = shader::BuiltinBits::NONE;
shader::BuiltinBits parse_builtin_bit(StringRef builtin)
/* NOTE: The next few functions are needed to keep isolation of the preprocessor.
* Eventually, this should be revisited and the preprocessor should output
* GPU structures. */
shader::BuiltinBits convert_builtin_bit(shader::metadata::Builtin builtin)
{
using namespace blender::gpu::shader;
using namespace blender::gpu::shader::metadata;
switch (Builtin(std::stoull(builtin))) {
switch (builtin) {
case Builtin::FragCoord:
return BuiltinBits::FRAG_COORD;
case Builtin::FrontFacing:
@@ -110,10 +115,10 @@ struct GPUSource {
return BuiltinBits::NONE;
}
GPUFunctionQual parse_qualifier(StringRef qualifier)
GPUFunctionQual convert_qualifier(shader::metadata::Qualifier qualifier)
{
using namespace blender::gpu::shader;
switch (metadata::Qualifier(std::stoull(qualifier))) {
switch (qualifier) {
case metadata::Qualifier::in:
return FUNCTION_QUAL_IN;
case metadata::Qualifier::out:
@@ -125,10 +130,10 @@ struct GPUSource {
return FUNCTION_QUAL_IN;
}
eGPUType parse_type(StringRef type)
eGPUType convert_type(shader::metadata::Type type)
{
using namespace blender::gpu::shader;
switch (metadata::Type(std::stoull(type))) {
switch (type) {
case metadata::Type::float1:
return GPU_FLOAT;
case metadata::Type::float2:
@@ -156,170 +161,99 @@ struct GPUSource {
return GPU_NONE;
}
StringRef split_on(StringRef &data, char token)
{
/* Assume lines are terminated by `\n`. */
int64_t pos = data.find(token);
if (pos == StringRef::not_found) {
StringRef line = data;
data = data.substr(0, 0);
return line;
}
StringRef line = data.substr(0, pos);
data = data.substr(pos + 1);
return line;
}
StringRef pop_line(StringRef &data)
{
/* Assume lines are terminated by `\n`. */
return split_on(data, '\n');
}
StringRef pop_token(StringRef &data)
{
/* Assumes tokens are split by spaces. */
return split_on(data, ' ');
}
GPUSource(const char *path,
const char *file,
const char *datatoc,
GPUFunctionDictionnary *g_functions,
GPUPrintFormatMap *g_formats)
GPUSource(
const char *path,
const char *file,
const char *datatoc,
GPUFunctionDictionnary *g_functions,
GPUPrintFormatMap *g_formats,
std::function<void(GPUSource &, GPUFunctionDictionnary *, GPUPrintFormatMap *)> metadata_fn)
: fullpath(path), filename(file), source(datatoc)
{
/* Extract metadata string. */
int64_t sta = source.rfind("//__blender_metadata_sta");
int64_t end = source.rfind("//__blender_metadata_end");
StringRef metadata = source.substr(sta, end - sta);
pop_line(metadata);
/* Non-library files contains functions with unsupported argument types.
* Also Non-library files are not supposed to be referenced for GPU node-tree. */
const bool do_parse_function = is_from_material_library();
StringRef line;
while ((line = pop_line(metadata)).is_empty() == false) {
using namespace blender::gpu::shader;
/* Skip comment start. */
pop_token(line);
StringRef identifier = pop_token(line);
switch (uint64_t(std::stoull(identifier))) {
case Preprocessor::hash("function"):
if (do_parse_function) {
parse_function(line, g_functions);
}
break;
case Preprocessor::hash("string"):
parse_string(line, g_formats);
break;
case Preprocessor::hash("builtin"):
parse_builtin(line);
break;
case Preprocessor::hash("dependency"):
parse_dependency(line);
break;
default:
BLI_assert_unreachable();
break;
}
}
metadata_fn(*this, g_functions, g_formats);
};
void parse_builtin(StringRef line)
void add_builtin(shader::metadata::Builtin builtin)
{
builtins |= parse_builtin_bit(pop_token(line));
builtins |= convert_builtin_bit(builtin);
}
void parse_dependency(StringRef line)
void add_dependency(StringRef line)
{
dependencies_names.append(line);
}
void parse_string(StringRef line, GPUPrintFormatMap *format_map)
void add_printf_format(uint32_t format_hash, std::string format, GPUPrintFormatMap *format_map)
{
/* TODO(fclem): Move this to gpu log. */
auto add_format = [&](uint32_t format_hash, std::string format) {
if (format_map->contains(format_hash)) {
if (format_map->lookup(format_hash).format_str != format) {
print_error(format, 0, "printf format hash collision.");
}
else {
/* The format map already have the same format. */
}
if (format_map->contains(format_hash)) {
if (format_map->lookup(format_hash).format_str != format) {
print_error(format, 0, "printf format hash collision.");
}
else {
shader::PrintfFormat fmt;
/* Save for hash collision comparison. */
fmt.format_str = format;
/* Escape characters replacement. Do the most common ones. */
format = std::regex_replace(format, std::regex(R"(\\n)"), "\n");
format = std::regex_replace(format, std::regex(R"(\\v)"), "\v");
format = std::regex_replace(format, std::regex(R"(\\t)"), "\t");
format = std::regex_replace(format, std::regex(R"(\\')"), "\'");
format = std::regex_replace(format, std::regex(R"(\\")"), "\"");
format = std::regex_replace(format, std::regex(R"(\\\\)"), "\\");
shader::PrintfFormat::Block::ArgumentType type =
shader::PrintfFormat::Block::ArgumentType::NONE;
int64_t start = 0, end = 0;
while ((end = format.find_first_of('%', start + 1)) != -1) {
/* Add the previous block without the newly found % character. */
fmt.format_blocks.append({type, format.substr(start, end - start)});
/* Format type of the next block. */
/* TODO(fclem): This doesn't support advance formats like `%3.2f`. */
switch (format[end + 1]) {
case 'x':
case 'u':
type = shader::PrintfFormat::Block::ArgumentType::UINT;
break;
case 'd':
type = shader::PrintfFormat::Block::ArgumentType::INT;
break;
case 'f':
type = shader::PrintfFormat::Block::ArgumentType::FLOAT;
break;
default:
BLI_assert_msg(0, "Printing format unsupported");
break;
}
/* Start of the next block. */
start = end;
}
fmt.format_blocks.append({type, format.substr(start, format.size() - start)});
format_map->add(format_hash, fmt);
/* The format map already have the same format. */
}
};
}
else {
shader::PrintfFormat fmt;
/* Save for hash collision comparison. */
fmt.format_str = format;
StringRef hash = pop_token(line);
StringRef string = line;
add_format(uint32_t(std::stoul(hash)), string);
/* Escape characters replacement. Do the most common ones. */
format = std::regex_replace(format, std::regex(R"(\\n)"), "\n");
format = std::regex_replace(format, std::regex(R"(\\v)"), "\v");
format = std::regex_replace(format, std::regex(R"(\\t)"), "\t");
format = std::regex_replace(format, std::regex(R"(\\')"), "\'");
format = std::regex_replace(format, std::regex(R"(\\")"), "\"");
format = std::regex_replace(format, std::regex(R"(\\\\)"), "\\");
shader::PrintfFormat::Block::ArgumentType type =
shader::PrintfFormat::Block::ArgumentType::NONE;
int64_t start = 0, end = 0;
while ((end = format.find_first_of('%', start + 1)) != -1) {
/* Add the previous block without the newly found % character. */
fmt.format_blocks.append({type, format.substr(start, end - start)});
/* Format type of the next block. */
/* TODO(fclem): This doesn't support advance formats like `%3.2f`. */
switch (format[end + 1]) {
case 'x':
case 'u':
type = shader::PrintfFormat::Block::ArgumentType::UINT;
break;
case 'd':
type = shader::PrintfFormat::Block::ArgumentType::INT;
break;
case 'f':
type = shader::PrintfFormat::Block::ArgumentType::FLOAT;
break;
default:
BLI_assert_msg(0, "Printing format unsupported");
break;
}
/* Start of the next block. */
start = end;
}
fmt.format_blocks.append({type, format.substr(start, format.size() - start)});
format_map->add(format_hash, fmt);
}
}
void parse_function(StringRef line, GPUFunctionDictionnary *g_functions)
void add_function(StringRefNull name,
Span<shader::metadata::ArgumentFormat> arguments,
GPUFunctionDictionnary *g_functions)
{
StringRef name = pop_token(line);
GPUFunction *func = MEM_new<GPUFunction>(__func__);
name.copy_utf8_truncated(func->name, sizeof(func->name));
func->source = reinterpret_cast<void *>(this);
func->totparam = 0;
while (true) {
StringRef arg_qual = pop_token(line);
StringRef arg_type = pop_token(line);
if (arg_qual.is_empty()) {
break;
}
for (auto arg : arguments) {
if (func->totparam >= ARRAY_SIZE(func->paramtype)) {
print_error(source, source.find(name), "Too many parameters in function");
break;
}
func->paramqual[func->totparam] = parse_qualifier(arg_qual);
func->paramtype[func->totparam] = parse_type(arg_type);
func->paramqual[func->totparam] = convert_qualifier(arg.qualifier);
func->paramtype[func->totparam] = convert_type(arg.type);
func->totparam++;
}
@@ -432,6 +366,20 @@ struct GPUSource {
}
};
namespace shader {
#include "glsl_compositor_metadata_list.hh"
#include "glsl_draw_metadata_list.hh"
#include "glsl_gpu_metadata_list.hh"
#ifdef WITH_OCIO
# include "glsl_ocio_metadata_list.hh"
#endif
#ifdef WITH_OPENSUBDIV
# include "glsl_osd_metadata_list.hh"
#endif
} // namespace shader
} // namespace blender::gpu
using namespace blender::gpu;
@@ -447,8 +395,15 @@ void gpu_shader_dependency_init()
g_sources = new GPUSourceDictionnary();
g_functions = new GPUFunctionDictionnary();
#define SHADER_SOURCE(datatoc, filename, filepath) \
g_sources->add_new(filename, new GPUSource(filepath, filename, datatoc, g_functions, g_formats));
#define SHADER_SOURCE(filename_underscore, filename, filepath) \
g_sources->add_new(filename, \
new GPUSource(filepath, \
filename, \
datatoc_##filename_underscore, \
g_functions, \
g_formats, \
blender::gpu::shader::metadata_##filename_underscore));
#include "glsl_compositor_source_list.h"
#include "glsl_draw_source_list.h"
#include "glsl_gpu_source_list.h"
@@ -461,14 +416,14 @@ void gpu_shader_dependency_init()
#undef SHADER_SOURCE
#ifdef WITH_OPENSUBDIV
const blender::StringRefNull patch_basis_source = openSubdiv_getGLSLPatchBasisSource();
static std::string osd_patch_basis_glsl =
"//__blender_metadata_sta\n//__blender_metadata_end\n" + patch_basis_source;
g_sources->add_new("osd_patch_basis.glsl",
new GPUSource("osd_patch_basis.glsl",
"osd_patch_basis.glsl",
osd_patch_basis_glsl.c_str(),
g_functions,
g_formats));
g_sources->add_new(
"osd_patch_basis.glsl",
new GPUSource("osd_patch_basis.glsl",
"osd_patch_basis.glsl",
patch_basis_source.c_str(),
g_functions,
g_formats,
[](GPUSource &, GPUFunctionDictionnary *, GPUPrintFormatMap *) {}));
#endif
int errors = 0;