From d24b0e4a4e87330532a6565284f5f1039dc6d3ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Foucault?= Date: Tue, 29 Apr 2025 15:32:36 +0200 Subject: [PATCH] GPU: Remove runtime parsing of GLSL source See #129009 for context. The preprocessor parses metadata and writes a header file containing an inline function that inits the `GPUSource` with the metadata. These header files are then included inside `gpu_shader_dependency.cc`. This still keep the usage of the `metadata` enums and classes to avoid pulling the whole blender module inside the preprocessor executable. This speeds-up startup time in Debug build: `gpu_shader_dependency_init` - Before : 37ms - After : 4ms I didn't measure release, but it is unlikely to be noticeable (in the order of 4ms > 1ms). Pull Request: https://projects.blender.org/blender/blender/pulls/138070 --- build_files/cmake/macros.cmake | 12 +- intern/opencolorio/CMakeLists.txt | 7 +- intern/opensubdiv/CMakeLists.txt | 7 +- source/blender/compositor/CMakeLists.txt | 7 +- source/blender/draw/CMakeLists.txt | 8 +- source/blender/gpu/CMakeLists.txt | 7 +- .../gpu/glsl_preprocess/glsl_preprocess.cc | 26 +- .../gpu/glsl_preprocess/glsl_preprocess.hh | 302 ++++++++++-------- .../gpu/intern/gpu_shader_dependency.cc | 261 +++++++-------- 9 files changed, 334 insertions(+), 303 deletions(-) diff --git a/build_files/cmake/macros.cmake b/build_files/cmake/macros.cmake index 1d4eb6e2a7a..64270ee94e3 100644 --- a/build_files/cmake/macros.cmake +++ b/build_files/cmake/macros.cmake @@ -1040,9 +1040,10 @@ function(glsl_to_c ) # remove ../'s - get_filename_component(_file_from ${CMAKE_CURRENT_SOURCE_DIR}/${file_from} REALPATH) - get_filename_component(_file_tmp ${CMAKE_CURRENT_BINARY_DIR}/${file_from} REALPATH) - get_filename_component(_file_to ${CMAKE_CURRENT_BINARY_DIR}/${file_from}.c REALPATH) + get_filename_component(_file_from ${CMAKE_CURRENT_SOURCE_DIR}/${file_from} REALPATH) + get_filename_component(_file_tmp ${CMAKE_CURRENT_BINARY_DIR}/${file_from} REALPATH) + get_filename_component(_file_meta ${CMAKE_CURRENT_BINARY_DIR}/${file_from}.hh REALPATH) + get_filename_component(_file_to ${CMAKE_CURRENT_BINARY_DIR}/${file_from}.c REALPATH) list(APPEND ${list_to_add} ${_file_to}) source_group(Generated FILES ${_file_to}) @@ -1052,14 +1053,15 @@ function(glsl_to_c get_filename_component(_file_to_path ${_file_to} PATH) add_custom_command( - OUTPUT ${_file_to} + OUTPUT ${_file_to} ${_file_meta} COMMAND ${CMAKE_COMMAND} -E make_directory ${_file_to_path} - COMMAND "$" ${_file_from} ${_file_tmp} + COMMAND "$" ${_file_from} ${_file_tmp} ${_file_meta} COMMAND "$" ${_file_tmp} ${_file_to} DEPENDS ${_file_from} datatoc glsl_preprocess) set_source_files_properties(${_file_tmp} PROPERTIES GENERATED TRUE) set_source_files_properties(${_file_to} PROPERTIES GENERATED TRUE) + set_source_files_properties(${_file_meta} PROPERTIES GENERATED TRUE) endfunction() diff --git a/intern/opencolorio/CMakeLists.txt b/intern/opencolorio/CMakeLists.txt index d3eada1ddc1..e7ae79c0e93 100644 --- a/intern/opencolorio/CMakeLists.txt +++ b/intern/opencolorio/CMakeLists.txt @@ -67,15 +67,20 @@ if(WITH_OPENCOLORIO) ) set(GLSL_SOURCE_CONTENT "") + set(GLSL_METADATA_CONTENT "") foreach(GLSL_FILE ${GLSL_SRC}) get_filename_component(GLSL_FILE_NAME ${GLSL_FILE} NAME) string(REPLACE "." "_" GLSL_FILE_NAME_UNDERSCORES ${GLSL_FILE_NAME}) - string(APPEND GLSL_SOURCE_CONTENT "SHADER_SOURCE\(datatoc_${GLSL_FILE_NAME_UNDERSCORES}, \"${GLSL_FILE_NAME}\", \"${GLSL_FILE}\"\)\n") + string(APPEND GLSL_SOURCE_CONTENT "SHADER_SOURCE\(${GLSL_FILE_NAME_UNDERSCORES}, \"${GLSL_FILE_NAME}\", \"${GLSL_FILE}\"\)\n") + string(APPEND GLSL_METADATA_CONTENT "#include \"${GLSL_FILE}.hh\"\n") endforeach() set(glsl_source_list_file "${CMAKE_CURRENT_BINARY_DIR}/glsl_ocio_source_list.h") file(GENERATE OUTPUT ${glsl_source_list_file} CONTENT "${GLSL_SOURCE_CONTENT}") list(APPEND SRC ${glsl_source_list_file}) + set(glsl_metadata_list_file "${CMAKE_CURRENT_BINARY_DIR}/glsl_ocio_metadata_list.hh") + file(GENERATE OUTPUT ${glsl_metadata_list_file} CONTENT "${GLSL_METADATA_CONTENT}") + list(APPEND SRC ${glsl_metadata_list_file}) list(APPEND INC ${CMAKE_CURRENT_BINARY_DIR}) target_include_directories(bf_ocio_shaders PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/intern/opensubdiv/CMakeLists.txt b/intern/opensubdiv/CMakeLists.txt index 736558174f8..f569262b701 100644 --- a/intern/opensubdiv/CMakeLists.txt +++ b/intern/opensubdiv/CMakeLists.txt @@ -102,15 +102,20 @@ if(WITH_OPENSUBDIV) ) set(GLSL_SOURCE_CONTENT "") + set(GLSL_METADATA_CONTENT "") foreach(GLSL_FILE ${GLSL_SRC}) get_filename_component(GLSL_FILE_NAME ${GLSL_FILE} NAME) string(REPLACE "." "_" GLSL_FILE_NAME_UNDERSCORES ${GLSL_FILE_NAME}) - string(APPEND GLSL_SOURCE_CONTENT "SHADER_SOURCE\(datatoc_${GLSL_FILE_NAME_UNDERSCORES}, \"${GLSL_FILE_NAME}\", \"${GLSL_FILE}\"\)\n") + string(APPEND GLSL_SOURCE_CONTENT "SHADER_SOURCE\(${GLSL_FILE_NAME_UNDERSCORES}, \"${GLSL_FILE_NAME}\", \"${GLSL_FILE}\"\)\n") + string(APPEND GLSL_METADATA_CONTENT "#include \"${GLSL_FILE}.hh\"\n") endforeach() set(glsl_source_list_file "${CMAKE_CURRENT_BINARY_DIR}/glsl_osd_source_list.h") file(GENERATE OUTPUT ${glsl_source_list_file} CONTENT "${GLSL_SOURCE_CONTENT}") list(APPEND SRC ${glsl_source_list_file}) + set(glsl_metadata_list_file "${CMAKE_CURRENT_BINARY_DIR}/glsl_osd_metadata_list.hh") + file(GENERATE OUTPUT ${glsl_metadata_list_file} CONTENT "${GLSL_METADATA_CONTENT}") + list(APPEND SRC ${glsl_metadata_list_file}) list(APPEND INC ${CMAKE_CURRENT_BINARY_DIR}) target_include_directories(bf_osd_shaders PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/source/blender/compositor/CMakeLists.txt b/source/blender/compositor/CMakeLists.txt index ec5b0e695fd..951643540fe 100644 --- a/source/blender/compositor/CMakeLists.txt +++ b/source/blender/compositor/CMakeLists.txt @@ -315,15 +315,20 @@ list(APPEND LIB ) set(GLSL_SOURCE_CONTENT "") +set(GLSL_METADATA_CONTENT "") foreach(GLSL_FILE ${GLSL_SRC}) get_filename_component(GLSL_FILE_NAME ${GLSL_FILE} NAME) string(REPLACE "." "_" GLSL_FILE_NAME_UNDERSCORES ${GLSL_FILE_NAME}) - string(APPEND GLSL_SOURCE_CONTENT "SHADER_SOURCE\(datatoc_${GLSL_FILE_NAME_UNDERSCORES}, \"${GLSL_FILE_NAME}\", \"${GLSL_FILE}\"\)\n") + string(APPEND GLSL_SOURCE_CONTENT "SHADER_SOURCE\(${GLSL_FILE_NAME_UNDERSCORES}, \"${GLSL_FILE_NAME}\", \"${GLSL_FILE}\"\)\n") + string(APPEND GLSL_METADATA_CONTENT "#include \"${GLSL_FILE}.hh\"\n") endforeach() set(glsl_source_list_file "${CMAKE_CURRENT_BINARY_DIR}/glsl_compositor_source_list.h") file(GENERATE OUTPUT ${glsl_source_list_file} CONTENT "${GLSL_SOURCE_CONTENT}") list(APPEND SRC ${glsl_source_list_file}) +set(glsl_metadata_list_file "${CMAKE_CURRENT_BINARY_DIR}/glsl_compositor_metadata_list.hh") +file(GENERATE OUTPUT ${glsl_metadata_list_file} CONTENT "${GLSL_METADATA_CONTENT}") +list(APPEND SRC ${glsl_metadata_list_file}) list(APPEND INC ${CMAKE_CURRENT_BINARY_DIR}) target_include_directories(bf_compositor_shaders PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/source/blender/draw/CMakeLists.txt b/source/blender/draw/CMakeLists.txt index 1d054d1e69c..3161a55ef53 100644 --- a/source/blender/draw/CMakeLists.txt +++ b/source/blender/draw/CMakeLists.txt @@ -742,16 +742,20 @@ list(APPEND LIB ) set(GLSL_SOURCE_CONTENT "") - +set(GLSL_METADATA_CONTENT "") foreach(GLSL_FILE ${GLSL_SRC}) get_filename_component(GLSL_FILE_NAME ${GLSL_FILE} NAME) string(REPLACE "." "_" GLSL_FILE_NAME_UNDERSCORES ${GLSL_FILE_NAME}) - string(APPEND GLSL_SOURCE_CONTENT "SHADER_SOURCE\(datatoc_${GLSL_FILE_NAME_UNDERSCORES}, \"${GLSL_FILE_NAME}\", \"${GLSL_FILE}\"\)\n") + string(APPEND GLSL_SOURCE_CONTENT "SHADER_SOURCE\(${GLSL_FILE_NAME_UNDERSCORES}, \"${GLSL_FILE_NAME}\", \"${GLSL_FILE}\"\)\n") + string(APPEND GLSL_METADATA_CONTENT "#include \"${GLSL_FILE}.hh\"\n") endforeach() set(glsl_source_list_file "${CMAKE_CURRENT_BINARY_DIR}/glsl_draw_source_list.h") file(GENERATE OUTPUT ${glsl_source_list_file} CONTENT "${GLSL_SOURCE_CONTENT}") list(APPEND SRC ${glsl_source_list_file}) +set(glsl_metadata_list_file "${CMAKE_CURRENT_BINARY_DIR}/glsl_draw_metadata_list.hh") +file(GENERATE OUTPUT ${glsl_metadata_list_file} CONTENT "${GLSL_METADATA_CONTENT}") +list(APPEND SRC ${glsl_metadata_list_file}) list(APPEND INC ${CMAKE_CURRENT_BINARY_DIR}) target_include_directories(bf_draw_shaders PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/source/blender/gpu/CMakeLists.txt b/source/blender/gpu/CMakeLists.txt index dd43738ffa1..869392dca13 100644 --- a/source/blender/gpu/CMakeLists.txt +++ b/source/blender/gpu/CMakeLists.txt @@ -716,15 +716,20 @@ list(APPEND LIB ) set(GLSL_SOURCE_CONTENT "") +set(GLSL_METADATA_CONTENT "") foreach(GLSL_FILE ${GLSL_SRC}) get_filename_component(GLSL_FILE_NAME ${GLSL_FILE} NAME) string(REPLACE "." "_" GLSL_FILE_NAME_UNDERSCORES ${GLSL_FILE_NAME}) - string(APPEND GLSL_SOURCE_CONTENT "SHADER_SOURCE\(datatoc_${GLSL_FILE_NAME_UNDERSCORES}, \"${GLSL_FILE_NAME}\", \"${GLSL_FILE}\"\)\n") + string(APPEND GLSL_SOURCE_CONTENT "SHADER_SOURCE\(${GLSL_FILE_NAME_UNDERSCORES}, \"${GLSL_FILE_NAME}\", \"${GLSL_FILE}\"\)\n") + string(APPEND GLSL_METADATA_CONTENT "#include \"${GLSL_FILE}.hh\"\n") endforeach() set(glsl_source_list_file "${CMAKE_CURRENT_BINARY_DIR}/glsl_gpu_source_list.h") file(GENERATE OUTPUT ${glsl_source_list_file} CONTENT "${GLSL_SOURCE_CONTENT}") list(APPEND SRC ${glsl_source_list_file}) +set(glsl_metadata_list_file "${CMAKE_CURRENT_BINARY_DIR}/glsl_gpu_metadata_list.hh") +file(GENERATE OUTPUT ${glsl_metadata_list_file} CONTENT "${GLSL_METADATA_CONTENT}") +list(APPEND SRC ${glsl_metadata_list_file}) list(APPEND INC ${CMAKE_CURRENT_BINARY_DIR}) if(WITH_MOD_FLUID) diff --git a/source/blender/gpu/glsl_preprocess/glsl_preprocess.cc b/source/blender/gpu/glsl_preprocess/glsl_preprocess.cc index fd2cded7fbb..d30f3bb18a8 100644 --- a/source/blender/gpu/glsl_preprocess/glsl_preprocess.cc +++ b/source/blender/gpu/glsl_preprocess/glsl_preprocess.cc @@ -15,13 +15,15 @@ int main(int argc, char **argv) { - if (argc != 3) { - std::cerr << "Usage: glsl_preprocess " << std::endl; + if (argc != 4) { + std::cerr << "Usage: glsl_preprocess " + << std::endl; exit(1); } const char *input_file_name = argv[1]; const char *output_file_name = argv[2]; + const char *metadata_file_name = argv[3]; /* Open the input file for reading */ std::ifstream input_file(input_file_name); @@ -38,6 +40,14 @@ int main(int argc, char **argv) exit(1); } + /* Open the output file for writing */ + std::ofstream metadata_file(metadata_file_name, std::ofstream::out | std::ofstream::binary); + if (!output_file) { + std::cerr << "Error: Could not open output file " << metadata_file_name << std::endl; + input_file.close(); + exit(1); + } + std::stringstream buffer; buffer << input_file.rdbuf(); @@ -103,11 +113,21 @@ int main(int argc, char **argv) language = Preprocessor::SourceLanguage::BLENDER_GLSL; } + blender::gpu::shader::metadata::Source metadata; output_file << processor.process( - language, buffer.str(), input_file_name, is_library, is_shared, report_error); + language, buffer.str(), input_file_name, is_library, is_shared, report_error, metadata); + + /* TODO(fclem): Don't use regex for that. */ + std::string metadata_function_name = "metadata_" + + std::regex_replace( + filename, std::regex(R"((?:.*)\/(.*))"), "$1"); + std::replace(metadata_function_name.begin(), metadata_function_name.end(), '.', '_'); + + metadata_file << metadata.serialize(metadata_function_name); input_file.close(); output_file.close(); + metadata_file.close(); return error; } diff --git a/source/blender/gpu/glsl_preprocess/glsl_preprocess.hh b/source/blender/gpu/glsl_preprocess/glsl_preprocess.hh index 2808dd408c4..b872c543e5e 100644 --- a/source/blender/gpu/glsl_preprocess/glsl_preprocess.hh +++ b/source/blender/gpu/glsl_preprocess/glsl_preprocess.hh @@ -18,6 +18,129 @@ namespace blender::gpu::shader { +/* Metadata extracted from shader source file. + * These are then converted to their GPU module equivalent. */ +/* TODO(fclem): Make GPU enums standalone and directly use them instead of using separate enums + * and types. */ +namespace metadata { + +/* Compile-time hashing function which converts string to a 64bit hash. */ +constexpr static uint64_t hash(const char *name) +{ + uint64_t hash = 2166136261u; + while (*name) { + hash = hash * 16777619u; + hash = hash ^ *name; + ++name; + } + return hash; +} + +static uint64_t hash(const std::string &name) +{ + return hash(name.c_str()); +} + +enum Builtin : uint64_t { + FragCoord = hash("gl_FragCoord"), + FrontFacing = hash("gl_FrontFacing"), + GlobalInvocationID = hash("gl_GlobalInvocationID"), + InstanceID = hash("gl_InstanceID"), + LocalInvocationID = hash("gl_LocalInvocationID"), + LocalInvocationIndex = hash("gl_LocalInvocationIndex"), + NumWorkGroup = hash("gl_NumWorkGroup"), + PointCoord = hash("gl_PointCoord"), + PointSize = hash("gl_PointSize"), + PrimitiveID = hash("gl_PrimitiveID"), + VertexID = hash("gl_VertexID"), + WorkGroupID = hash("gl_WorkGroupID"), + WorkGroupSize = hash("gl_WorkGroupSize"), + drw_debug = hash("drw_debug_"), + printf = hash("printf"), + assert = hash("assert"), +}; + +enum Qualifier : uint64_t { + in = hash("in"), + out = hash("out"), + inout = hash("inout"), +}; + +enum Type : uint64_t { + float1 = hash("float"), + float2 = hash("float2"), + float3 = hash("float3"), + float4 = hash("float4"), + float3x3 = hash("float3x3"), + float4x4 = hash("float4x4"), + sampler1DArray = hash("sampler1DArray"), + sampler2DArray = hash("sampler2DArray"), + sampler2D = hash("sampler2D"), + sampler3D = hash("sampler3D"), + Closure = hash("Closure"), +}; + +struct ArgumentFormat { + Qualifier qualifier; + Type type; +}; + +struct FunctionFormat { + std::string name; + std::vector arguments; +}; + +struct PrintfFormat { + uint32_t hash; + std::string format; +}; + +struct Source { + std::vector builtins; + /* Note: Could be a set, but for now the order matters. */ + std::vector dependencies; + std::vector printf_formats; + std::vector functions; + + std::string serialize(const std::string &function_name) const + { + std::stringstream ss; + ss << "static void " << function_name + << "(GPUSource &source, GPUFunctionDictionnary *g_functions, GPUPrintFormatMap *g_formats) " + "{\n"; + for (auto function : functions) { + ss << " {\n"; + ss << " Vector args = {\n"; + for (auto arg : function.arguments) { + ss << " " + << "metadata::ArgumentFormat{" + << "metadata::Qualifier(" << std::to_string(uint64_t(arg.qualifier)) << "LLU), " + << "metadata::Type(" << std::to_string(uint64_t(arg.type)) << "LLU)" + << "},\n"; + } + ss << " };\n"; + ss << " source.add_function(\"" << function.name << "\", args, g_functions);\n"; + ss << " }\n"; + } + for (auto builtin : builtins) { + ss << " source.add_builtin(metadata::Builtin(" << std::to_string(builtin) << "LLU));\n"; + } + for (auto dependency : dependencies) { + ss << " source.add_dependency(\"" << dependency << "\");\n"; + } + for (auto format : printf_formats) { + ss << " source.add_printf_format(uint32_t(" << std::to_string(format.hash) << "), " + << format.format << ", g_formats);\n"; + } + /* Avoid warnings. */ + ss << " UNUSED_VARS(source, g_functions, g_formats);\n"; + ss << "}\n"; + return ss.str(); + } +}; + +} // namespace metadata + /** * Shader source preprocessor that allow to mutate GLSL into cross API source that can be * interpreted by the different GPU backends. Some syntax are mutated or reported as incompatible. @@ -35,11 +158,8 @@ class Preprocessor { }; std::vector shared_vars_; - std::unordered_set static_strings_; - std::unordered_set gpu_builtins_; - /* Note: Could be a set, but for now the order matters. */ - std::vector dependencies_; - std::stringstream gpu_functions_; + + metadata::Source metadata; public: enum SourceLanguage { @@ -66,30 +186,14 @@ class Preprocessor { return UNKNOWN; } - /* Compile-time hashing function which converts string to a 64bit hash. */ - constexpr static uint64_t hash(const char *name) - { - uint64_t hash = 2166136261u; - while (*name) { - hash = hash * 16777619u; - hash = hash ^ *name; - ++name; - } - return hash; - } - - static uint64_t hash(const std::string &name) - { - return hash(name.c_str()); - } - /* Takes a whole source file and output processed source. */ std::string process(SourceLanguage language, std::string str, const std::string &filename, bool do_parse_function, bool do_small_type_linting, - report_callback report_error) + report_callback report_error, + metadata::Source &r_metadata) { if (language == UNKNOWN) { report_error(std::smatch(), "Unknown file type"); @@ -97,7 +201,7 @@ class Preprocessor { } str = remove_comments(str, report_error); threadgroup_variables_parsing(str); - parse_builtins(str); + parse_builtins(str, filename); if (language == BLENDER_GLSL || language == CPP) { if (do_parse_function) { parse_library_functions(str); @@ -125,16 +229,16 @@ class Preprocessor { } str = argument_decorator_macro_injection(str); str = array_constructor_macro_injection(str); - return line_directive_prefix(filename) + str + threadgroup_variables_suffix() + - "//__blender_metadata_sta\n" + gpu_functions_.str() + static_strings_suffix() + - gpu_builtins_suffix(filename) + dependency_suffix() + "//__blender_metadata_end\n"; + r_metadata = metadata; + return line_directive_prefix(filename) + str + threadgroup_variables_suffix(); } /* Variant use for python shaders. */ std::string process(const std::string &str) { auto no_err_report = [](std::smatch, const char *) {}; - return process(GLSL, str, "", false, false, no_err_report); + metadata::Source unused; + return process(GLSL, str, "", false, false, no_err_report, unused); } private: @@ -221,7 +325,7 @@ class Preprocessor { /* Skip info files. They are only for IDE linting. */ return; } - dependencies_.emplace_back(dependency_name); + metadata.dependencies.emplace_back(dependency_name); }); } @@ -232,18 +336,6 @@ class Preprocessor { return std::regex_replace(str, regex, ""); } - std::string dependency_suffix() - { - if (dependencies_.empty()) { - return ""; - } - std::stringstream suffix; - for (const std::string &filename : dependencies_) { - suffix << "// " << std::to_string(hash("dependency")) << " " << filename << "\n"; - } - return suffix.str(); - } - void threadgroup_variables_parsing(const std::string &str) { std::regex regex(R"(shared\s+(\w+)\s+(\w+)([^;]*);)"); @@ -254,11 +346,14 @@ class Preprocessor { void parse_library_functions(const std::string &str) { + using namespace metadata; std::regex regex_func(R"(void\s+(\w+)\s*\(([^)]+\))\s*\{)"); regex_global_search(str, regex_func, [&](const std::smatch &match) { std::string name = match[1].str(); std::string args = match[2].str(); - gpu_functions_ << "// " << hash("function") << " " << name; + + FunctionFormat fn; + fn.name = name; std::regex regex_arg(R"((?:(const|in|out|inout)\s)?(\w+)\s([\w\[\]]+)(?:,|\)))"); regex_global_search(args, regex_arg, [&](const std::smatch &arg) { @@ -267,14 +362,19 @@ class Preprocessor { if (qualifier.empty() || qualifier == "const") { qualifier = "in"; } - gpu_functions_ << ' ' << hash(qualifier) << ' ' << hash(type); + fn.arguments.emplace_back( + ArgumentFormat{metadata::Qualifier(hash(qualifier)), metadata::Type(hash(type))}); }); - gpu_functions_ << "\n"; + metadata.functions.emplace_back(fn); }); } - void parse_builtins(const std::string &str) + void parse_builtins(const std::string &str, const std::string &filename) { + const bool skip_drw_debug = filename.find("draw_debug_draw_lib.glsl") != std::string::npos || + filename.find("draw_debug_draw_display_vert.glsl") != + std::string::npos; + using namespace metadata; /* TODO: This can trigger false positive caused by disabled #if blocks. */ std::regex regex( "(" @@ -297,28 +397,12 @@ class Preprocessor { #endif "printf" ")"); - regex_global_search( - str, regex, [&](const std::smatch &match) { gpu_builtins_.insert(match[0].str()); }); - } - - std::string gpu_builtins_suffix(const std::string &filename) - { - if (gpu_builtins_.empty()) { - return ""; - } - - const bool skip_drw_debug = filename.find("draw_debug_draw_lib.glsl") != std::string::npos || - filename.find("draw_debug_draw_display_vert.glsl") != - std::string::npos; - - std::stringstream suffix; - for (const std::string &str_var : gpu_builtins_) { - if (str_var == "drw_debug_" && skip_drw_debug) { - continue; + regex_global_search(str, regex, [&](const std::smatch &match) { + if (skip_drw_debug && match[0].str() == "drw_debug_") { + return; } - suffix << "// " << hash("builtin") << " " << hash(str_var) << "\n"; - } - return suffix.str(); + metadata.builtins.emplace_back(Builtin(hash(match[0].str()))); + }); } template @@ -394,26 +478,30 @@ class Preprocessor { return std::regex_replace(str, regex, replacement); } - void static_strings_parsing(const std::string &str) - { - /* Matches any character inside a pair of un-escaped quote. */ - std::regex regex(R"("(?:[^"])*")"); - regex_global_search( - str, regex, [&](const std::smatch &match) { static_strings_.insert(match[0].str()); }); - } - /* String hash are outputted inside GLSL and needs to fit 32 bits. */ - static uint64_t hash_string(const std::string &str) + static uint32_t hash_string(const std::string &str) { - uint64_t hash_64 = hash(str); + uint64_t hash_64 = metadata::hash(str); uint32_t hash_32 = uint32_t(hash_64 ^ (hash_64 >> 32)); return hash_32; } + void static_strings_parsing(const std::string &str) + { + using namespace metadata; + /* Matches any character inside a pair of un-escaped quote. */ + std::regex regex(R"("(?:[^"])*")"); + regex_global_search(str, regex, [&](const std::smatch &match) { + std::string format = match[0].str(); + metadata.printf_formats.emplace_back(metadata::PrintfFormat{hash_string(format), format}); + }); + } + std::string static_strings_mutation(std::string str) { /* Replaces all matches by the respective string hash. */ - for (const std::string &str_var : static_strings_) { + for (const metadata::PrintfFormat &format : metadata.printf_formats) { + const std::string &str_var = format.format; std::regex escape_regex(R"([\\\.\^\$\+\(\)\[\]\{\}\|\?\*])"); std::string str_regex = std::regex_replace(str_var, escape_regex, "\\$&"); @@ -423,19 +511,6 @@ class Preprocessor { return str; } - std::string static_strings_suffix() - { - if (static_strings_.empty()) { - return ""; - } - std::stringstream suffix; - for (const std::string &str_var : static_strings_) { - std::string no_quote = str_var.substr(1, str_var.size() - 2); - suffix << "// " << hash("string") << " " << hash_string(str_var) << " " << no_quote << "\n"; - } - return suffix.str(); - } - std::string enum_macro_injection(std::string str) { /** @@ -677,7 +752,7 @@ class Preprocessor { suffix << "\"" << filename << "\""; } #else - uint64_t hash_value = hash(filename); + uint64_t hash_value = metadata::hash(filename); /* Fold the value so it fits the GLSL spec. */ hash_value = (hash_value ^ (hash_value >> 32)) & (~uint64_t(0) >> 33); suffix << std::to_string(uint64_t(hash_value)); @@ -687,49 +762,4 @@ class Preprocessor { } }; -/* Enum values of metadata that the preprocessor can append at the end of a source file. - * Eventually, remove the need for these and output the metadata inside header files. */ -namespace metadata { - -enum Builtin : uint64_t { - FragCoord = Preprocessor::hash("gl_FragCoord"), - FrontFacing = Preprocessor::hash("gl_FrontFacing"), - GlobalInvocationID = Preprocessor::hash("gl_GlobalInvocationID"), - InstanceID = Preprocessor::hash("gl_InstanceID"), - LocalInvocationID = Preprocessor::hash("gl_LocalInvocationID"), - LocalInvocationIndex = Preprocessor::hash("gl_LocalInvocationIndex"), - NumWorkGroup = Preprocessor::hash("gl_NumWorkGroup"), - PointCoord = Preprocessor::hash("gl_PointCoord"), - PointSize = Preprocessor::hash("gl_PointSize"), - PrimitiveID = Preprocessor::hash("gl_PrimitiveID"), - VertexID = Preprocessor::hash("gl_VertexID"), - WorkGroupID = Preprocessor::hash("gl_WorkGroupID"), - WorkGroupSize = Preprocessor::hash("gl_WorkGroupSize"), - drw_debug = Preprocessor::hash("drw_debug_"), - printf = Preprocessor::hash("printf"), - assert = Preprocessor::hash("assert"), -}; - -enum Qualifier : uint64_t { - in = Preprocessor::hash("in"), - out = Preprocessor::hash("out"), - inout = Preprocessor::hash("inout"), -}; - -enum Type : uint64_t { - float1 = Preprocessor::hash("float"), - float2 = Preprocessor::hash("float2"), - float3 = Preprocessor::hash("float3"), - float4 = Preprocessor::hash("float4"), - float3x3 = Preprocessor::hash("float3x3"), - float4x4 = Preprocessor::hash("float4x4"), - sampler1DArray = Preprocessor::hash("sampler1DArray"), - sampler2DArray = Preprocessor::hash("sampler2DArray"), - sampler2D = Preprocessor::hash("sampler2D"), - sampler3D = Preprocessor::hash("sampler3D"), - Closure = Preprocessor::hash("Closure"), -}; - -} // namespace metadata - } // namespace blender::gpu::shader diff --git a/source/blender/gpu/intern/gpu_shader_dependency.cc b/source/blender/gpu/intern/gpu_shader_dependency.cc index 2e524ef9874..67d648675d5 100644 --- a/source/blender/gpu/intern/gpu_shader_dependency.cc +++ b/source/blender/gpu/intern/gpu_shader_dependency.cc @@ -32,7 +32,8 @@ #include "../glsl_preprocess/glsl_preprocess.hh" extern "C" { -#define SHADER_SOURCE(datatoc, filename, filepath) extern char datatoc[]; +#define SHADER_SOURCE(filename_underscore, filename, filepath) \ + extern char datatoc_##filename_underscore[]; #include "glsl_compositor_source_list.h" #include "glsl_draw_source_list.h" #include "glsl_gpu_source_list.h" @@ -61,11 +62,15 @@ struct GPUSource { bool dependencies_init = false; shader::BuiltinBits builtins = shader::BuiltinBits::NONE; - shader::BuiltinBits parse_builtin_bit(StringRef builtin) + /* NOTE: The next few functions are needed to keep isolation of the preprocessor. + * Eventually, this should be revisited and the preprocessor should output + * GPU structures. */ + + shader::BuiltinBits convert_builtin_bit(shader::metadata::Builtin builtin) { using namespace blender::gpu::shader; using namespace blender::gpu::shader::metadata; - switch (Builtin(std::stoull(builtin))) { + switch (builtin) { case Builtin::FragCoord: return BuiltinBits::FRAG_COORD; case Builtin::FrontFacing: @@ -110,10 +115,10 @@ struct GPUSource { return BuiltinBits::NONE; } - GPUFunctionQual parse_qualifier(StringRef qualifier) + GPUFunctionQual convert_qualifier(shader::metadata::Qualifier qualifier) { using namespace blender::gpu::shader; - switch (metadata::Qualifier(std::stoull(qualifier))) { + switch (qualifier) { case metadata::Qualifier::in: return FUNCTION_QUAL_IN; case metadata::Qualifier::out: @@ -125,10 +130,10 @@ struct GPUSource { return FUNCTION_QUAL_IN; } - eGPUType parse_type(StringRef type) + eGPUType convert_type(shader::metadata::Type type) { using namespace blender::gpu::shader; - switch (metadata::Type(std::stoull(type))) { + switch (type) { case metadata::Type::float1: return GPU_FLOAT; case metadata::Type::float2: @@ -156,170 +161,99 @@ struct GPUSource { return GPU_NONE; } - StringRef split_on(StringRef &data, char token) - { - /* Assume lines are terminated by `\n`. */ - int64_t pos = data.find(token); - if (pos == StringRef::not_found) { - StringRef line = data; - data = data.substr(0, 0); - return line; - } - StringRef line = data.substr(0, pos); - data = data.substr(pos + 1); - return line; - } - - StringRef pop_line(StringRef &data) - { - /* Assume lines are terminated by `\n`. */ - return split_on(data, '\n'); - } - - StringRef pop_token(StringRef &data) - { - /* Assumes tokens are split by spaces. */ - return split_on(data, ' '); - } - - GPUSource(const char *path, - const char *file, - const char *datatoc, - GPUFunctionDictionnary *g_functions, - GPUPrintFormatMap *g_formats) + GPUSource( + const char *path, + const char *file, + const char *datatoc, + GPUFunctionDictionnary *g_functions, + GPUPrintFormatMap *g_formats, + std::function metadata_fn) : fullpath(path), filename(file), source(datatoc) { - /* Extract metadata string. */ - int64_t sta = source.rfind("//__blender_metadata_sta"); - int64_t end = source.rfind("//__blender_metadata_end"); - StringRef metadata = source.substr(sta, end - sta); - pop_line(metadata); - - /* Non-library files contains functions with unsupported argument types. - * Also Non-library files are not supposed to be referenced for GPU node-tree. */ - const bool do_parse_function = is_from_material_library(); - - StringRef line; - while ((line = pop_line(metadata)).is_empty() == false) { - using namespace blender::gpu::shader; - /* Skip comment start. */ - pop_token(line); - - StringRef identifier = pop_token(line); - switch (uint64_t(std::stoull(identifier))) { - case Preprocessor::hash("function"): - if (do_parse_function) { - parse_function(line, g_functions); - } - break; - case Preprocessor::hash("string"): - parse_string(line, g_formats); - break; - case Preprocessor::hash("builtin"): - parse_builtin(line); - break; - case Preprocessor::hash("dependency"): - parse_dependency(line); - break; - default: - BLI_assert_unreachable(); - break; - } - } + metadata_fn(*this, g_functions, g_formats); }; - void parse_builtin(StringRef line) + void add_builtin(shader::metadata::Builtin builtin) { - builtins |= parse_builtin_bit(pop_token(line)); + builtins |= convert_builtin_bit(builtin); } - void parse_dependency(StringRef line) + void add_dependency(StringRef line) { dependencies_names.append(line); } - void parse_string(StringRef line, GPUPrintFormatMap *format_map) + void add_printf_format(uint32_t format_hash, std::string format, GPUPrintFormatMap *format_map) { /* TODO(fclem): Move this to gpu log. */ - auto add_format = [&](uint32_t format_hash, std::string format) { - if (format_map->contains(format_hash)) { - if (format_map->lookup(format_hash).format_str != format) { - print_error(format, 0, "printf format hash collision."); - } - else { - /* The format map already have the same format. */ - } + if (format_map->contains(format_hash)) { + if (format_map->lookup(format_hash).format_str != format) { + print_error(format, 0, "printf format hash collision."); } else { - shader::PrintfFormat fmt; - /* Save for hash collision comparison. */ - fmt.format_str = format; - - /* Escape characters replacement. Do the most common ones. */ - format = std::regex_replace(format, std::regex(R"(\\n)"), "\n"); - format = std::regex_replace(format, std::regex(R"(\\v)"), "\v"); - format = std::regex_replace(format, std::regex(R"(\\t)"), "\t"); - format = std::regex_replace(format, std::regex(R"(\\')"), "\'"); - format = std::regex_replace(format, std::regex(R"(\\")"), "\""); - format = std::regex_replace(format, std::regex(R"(\\\\)"), "\\"); - - shader::PrintfFormat::Block::ArgumentType type = - shader::PrintfFormat::Block::ArgumentType::NONE; - int64_t start = 0, end = 0; - while ((end = format.find_first_of('%', start + 1)) != -1) { - /* Add the previous block without the newly found % character. */ - fmt.format_blocks.append({type, format.substr(start, end - start)}); - /* Format type of the next block. */ - /* TODO(fclem): This doesn't support advance formats like `%3.2f`. */ - switch (format[end + 1]) { - case 'x': - case 'u': - type = shader::PrintfFormat::Block::ArgumentType::UINT; - break; - case 'd': - type = shader::PrintfFormat::Block::ArgumentType::INT; - break; - case 'f': - type = shader::PrintfFormat::Block::ArgumentType::FLOAT; - break; - default: - BLI_assert_msg(0, "Printing format unsupported"); - break; - } - /* Start of the next block. */ - start = end; - } - fmt.format_blocks.append({type, format.substr(start, format.size() - start)}); - - format_map->add(format_hash, fmt); + /* The format map already have the same format. */ } - }; + } + else { + shader::PrintfFormat fmt; + /* Save for hash collision comparison. */ + fmt.format_str = format; - StringRef hash = pop_token(line); - StringRef string = line; - add_format(uint32_t(std::stoul(hash)), string); + /* Escape characters replacement. Do the most common ones. */ + format = std::regex_replace(format, std::regex(R"(\\n)"), "\n"); + format = std::regex_replace(format, std::regex(R"(\\v)"), "\v"); + format = std::regex_replace(format, std::regex(R"(\\t)"), "\t"); + format = std::regex_replace(format, std::regex(R"(\\')"), "\'"); + format = std::regex_replace(format, std::regex(R"(\\")"), "\""); + format = std::regex_replace(format, std::regex(R"(\\\\)"), "\\"); + + shader::PrintfFormat::Block::ArgumentType type = + shader::PrintfFormat::Block::ArgumentType::NONE; + int64_t start = 0, end = 0; + while ((end = format.find_first_of('%', start + 1)) != -1) { + /* Add the previous block without the newly found % character. */ + fmt.format_blocks.append({type, format.substr(start, end - start)}); + /* Format type of the next block. */ + /* TODO(fclem): This doesn't support advance formats like `%3.2f`. */ + switch (format[end + 1]) { + case 'x': + case 'u': + type = shader::PrintfFormat::Block::ArgumentType::UINT; + break; + case 'd': + type = shader::PrintfFormat::Block::ArgumentType::INT; + break; + case 'f': + type = shader::PrintfFormat::Block::ArgumentType::FLOAT; + break; + default: + BLI_assert_msg(0, "Printing format unsupported"); + break; + } + /* Start of the next block. */ + start = end; + } + fmt.format_blocks.append({type, format.substr(start, format.size() - start)}); + + format_map->add(format_hash, fmt); + } } - void parse_function(StringRef line, GPUFunctionDictionnary *g_functions) + void add_function(StringRefNull name, + Span arguments, + GPUFunctionDictionnary *g_functions) { - StringRef name = pop_token(line); - GPUFunction *func = MEM_new(__func__); name.copy_utf8_truncated(func->name, sizeof(func->name)); func->source = reinterpret_cast(this); func->totparam = 0; - while (true) { - StringRef arg_qual = pop_token(line); - StringRef arg_type = pop_token(line); - if (arg_qual.is_empty()) { - break; - } + for (auto arg : arguments) { if (func->totparam >= ARRAY_SIZE(func->paramtype)) { print_error(source, source.find(name), "Too many parameters in function"); break; } - func->paramqual[func->totparam] = parse_qualifier(arg_qual); - func->paramtype[func->totparam] = parse_type(arg_type); + func->paramqual[func->totparam] = convert_qualifier(arg.qualifier); + func->paramtype[func->totparam] = convert_type(arg.type); func->totparam++; } @@ -432,6 +366,20 @@ struct GPUSource { } }; +namespace shader { + +#include "glsl_compositor_metadata_list.hh" +#include "glsl_draw_metadata_list.hh" +#include "glsl_gpu_metadata_list.hh" +#ifdef WITH_OCIO +# include "glsl_ocio_metadata_list.hh" +#endif +#ifdef WITH_OPENSUBDIV +# include "glsl_osd_metadata_list.hh" +#endif + +} // namespace shader + } // namespace blender::gpu using namespace blender::gpu; @@ -447,8 +395,15 @@ void gpu_shader_dependency_init() g_sources = new GPUSourceDictionnary(); g_functions = new GPUFunctionDictionnary(); -#define SHADER_SOURCE(datatoc, filename, filepath) \ - g_sources->add_new(filename, new GPUSource(filepath, filename, datatoc, g_functions, g_formats)); +#define SHADER_SOURCE(filename_underscore, filename, filepath) \ + g_sources->add_new(filename, \ + new GPUSource(filepath, \ + filename, \ + datatoc_##filename_underscore, \ + g_functions, \ + g_formats, \ + blender::gpu::shader::metadata_##filename_underscore)); + #include "glsl_compositor_source_list.h" #include "glsl_draw_source_list.h" #include "glsl_gpu_source_list.h" @@ -461,14 +416,14 @@ void gpu_shader_dependency_init() #undef SHADER_SOURCE #ifdef WITH_OPENSUBDIV const blender::StringRefNull patch_basis_source = openSubdiv_getGLSLPatchBasisSource(); - static std::string osd_patch_basis_glsl = - "//__blender_metadata_sta\n//__blender_metadata_end\n" + patch_basis_source; - g_sources->add_new("osd_patch_basis.glsl", - new GPUSource("osd_patch_basis.glsl", - "osd_patch_basis.glsl", - osd_patch_basis_glsl.c_str(), - g_functions, - g_formats)); + g_sources->add_new( + "osd_patch_basis.glsl", + new GPUSource("osd_patch_basis.glsl", + "osd_patch_basis.glsl", + patch_basis_source.c_str(), + g_functions, + g_formats, + [](GPUSource &, GPUFunctionDictionnary *, GPUPrintFormatMap *) {})); #endif int errors = 0;