From 3c3f21ec00822f7559439202a02cb2b6ed96ab70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Foucault?= Date: Fri, 15 Aug 2025 15:13:54 +0200 Subject: [PATCH] GPU: Shader: Replace template macro implementation by copy paste Use the lately introduced parser for that. This allows to use preprocessor directive inside them. By extension, it alows having resource accessors inside templates. Also error report is less confusing on most shader compilers. The counterpart is that the shader files that are shipped with blender are inflated. Pull Request: https://projects.blender.org/blender/blender/pulls/144588 --- .../eevee/shaders/eevee_reverse_z_lib.glsl | 10 +- .../gpu/glsl_preprocess/glsl_preprocess.hh | 267 +++++++++++------- .../gpu/glsl_preprocess/shader_parser.hh | 52 +++- .../gpu/shaders/metal/mtl_shader_defines.msl | 5 - .../shaders/opengl/glsl_shader_defines.glsl | 5 - .../gpu/tests/shader_preprocess_test.cc | 99 ++++--- 6 files changed, 272 insertions(+), 166 deletions(-) diff --git a/source/blender/draw/engines/eevee/shaders/eevee_reverse_z_lib.glsl b/source/blender/draw/engines/eevee/shaders/eevee_reverse_z_lib.glsl index 3e17d2804e8..0372028b3d8 100644 --- a/source/blender/draw/engines/eevee/shaders/eevee_reverse_z_lib.glsl +++ b/source/blender/draw/engines/eevee/shaders/eevee_reverse_z_lib.glsl @@ -24,25 +24,19 @@ float4 transform(float4 hs_position) return hs_position; } -/* NOTE: Cannot put the ifdef inside the template because of template as macro. */ -#ifdef GPU_ARB_clip_control /* Needs to be called for every depth buffer read, but not for the HiZ. * The HiZ buffer is already reversed back before downsample. */ template T read(T depth_buffer_value) { +#ifdef GPU_ARB_clip_control /* Remapping from 0..1 to 1..0. The scaling to 0..1 is handled as normal by drw_screen_to_ndc. */ return 1.0f - depth_buffer_value; -} -template float read(float); -template float4 read(float4); #else -template T read(T depth_buffer_value) -{ /* Passthrough. */ return depth_buffer_value; +#endif } template float read(float); template float4 read(float4); -#endif } // namespace reverse_z diff --git a/source/blender/gpu/glsl_preprocess/glsl_preprocess.hh b/source/blender/gpu/glsl_preprocess/glsl_preprocess.hh index d8a26783ec1..d8f79b9dad7 100644 --- a/source/blender/gpu/glsl_preprocess/glsl_preprocess.hh +++ b/source/blender/gpu/glsl_preprocess/glsl_preprocess.hh @@ -242,11 +242,11 @@ class Preprocessor { str = namespace_separator_mutation(str); } str = argument_reference_mutation(str); - str = enum_macro_injection(str); str = default_argument_mutation(str); str = variable_reference_mutation(str, report_error); str = template_definition_mutation(str, report_error); str = template_call_mutation(str); + str = enum_macro_injection(str); } #ifdef __APPLE__ /* Limiting to Apple hardware since GLSL compilers might have issues. */ if (language == GLSL) { @@ -351,70 +351,12 @@ class Preprocessor { return str; } + using namespace std; + using namespace shader::parser; + std::string out_str = str; + { - /* Transform template definition into macro declaration. */ - std::regex regex(R"(template<([\w\d\n\,\ ]+)>(\s\w+\s)(\w+)\()"); - out_str = std::regex_replace(out_str, regex, "#define $3_TEMPLATE($1)$2$3@("); - } - { - /* Add backslash for each newline in template macro. */ - size_t start, end = 0; - while ((start = out_str.find("_TEMPLATE(", end)) != std::string::npos) { - /* Remove parameter type from macro argument list. */ - end = out_str.find(")", start); - std::string arg_list = out_str.substr(start, end - start); - arg_list = std::regex_replace(arg_list, std::regex(R"(\w+ (\w+))"), "$1"); - out_str.replace(start, end - start, arg_list); - - std::string template_body = get_content_between_balanced_pair( - out_str.substr(start), '{', '}'); - if (template_body.empty()) { - /* Empty body is unlikely to happen. This limitation can be worked-around by using a noop - * comment inside the function body. */ - report_error( - std::smatch(), - "Template function declaration is missing closing bracket or has empty body."); - break; - } - size_t body_end = out_str.find('{', start) + 1 + template_body.size(); - /* Contains "_TEMPLATE(macro_args) void fn@(fn_args) { body;". */ - std::string macro_body = out_str.substr(start, body_end - start); - - macro_body = std::regex_replace(macro_body, std::regex(R"(\n)"), " \\\n"); - - std::string macro_args = get_content_between_balanced_pair(macro_body, '(', ')'); - /* Find function argument list. - * Skip first 10 chars to skip "_TEMPLATE" and the argument list. */ - std::string fn_args = get_content_between_balanced_pair( - macro_body.substr(10 + macro_args.length() + 1), '(', ')'); - /* Remove white-spaces. */ - macro_args = std::regex_replace(macro_args, std::regex(R"(\s)"), ""); - std::vector macro_args_split = split_string(macro_args, ','); - /* Append arguments inside the function name. */ - std::string fn_name_suffix = "_"; - bool all_args_in_function_signature = true; - for (std::string macro_arg : macro_args_split) { - fn_name_suffix += "##" + macro_arg + "##_"; - /* Search macro arguments inside the function arguments types. */ - if (std::regex_search(fn_args, std::regex(R"(\b)" + macro_arg + R"(\b)")) == false) { - all_args_in_function_signature = false; - } - } - if (all_args_in_function_signature) { - /* No need for suffix. Use overload for type deduction. - * Otherwise, we require full explicit template call. */ - fn_name_suffix = ""; - } - size_t end_of_fn_name = macro_body.find("@"); - macro_body.replace(end_of_fn_name, 1, fn_name_suffix); - - out_str.replace(start, body_end - start, macro_body); - } - } - { - using namespace std; - using namespace shader::parser; Parser parser(out_str); parser.foreach_scope(ScopeType::Global, [&](Scope scope) { @@ -435,48 +377,153 @@ class Preprocessor { out_str = parser.result_get(); } { - /* Replace explicit instantiation by macro call. */ - /* Only `template ret_t fn(args);` syntax is supported. */ - std::regex regex_instance(R"(template \w+ (\w+)<([\w+\,\ \n]+)>\(([\w+\ \,\n]+)\);)"); - /* Notice the stupid way of keeping the number of lines the same by copying the argument list - * inside a multi-line comment. */ - out_str = std::regex_replace(out_str, regex_instance, "$1_TEMPLATE($2)/*$3*/"); + Parser parser(out_str); + + parser.foreach_scope(ScopeType::Template, [&](Scope temp) { + /* Parse template declaration. */ + Token fn_start = temp.end().next(); + Token fn_name = (fn_start == Static) ? fn_start.next().next() : fn_start.next(); + Scope fn_args = fn_name.next().scope(); + + bool error = false; + temp.foreach_match("=", [&](const std::vector & /*tokens*/) { + report_error(smatch(), + "Default arguments are not supported inside template declaration"); + error = true; + }); + if (error) { + return; + } + + string arg_pattern; + vector arg_list; + bool all_template_args_in_function_signature = true; + temp.foreach_scope(ScopeType::TemplateArg, [&](Scope arg) { + const Token type = arg.start(); + const Token name = type.next(); + const string name_str = name.str_no_whitespace(); + const string type_str = type.str_no_whitespace(); + + arg_list.emplace_back(name_str); + + if (type_str == "typename") { + arg_pattern += ",w"; + bool found = false; + /* Search argument list for typenames. If typename matches, the template argument is + * present inside the function signature. */ + fn_args.foreach_match("ww", [&](const std::vector &tokens) { + if (tokens[0].str_no_whitespace() == name_str) { + found = true; + } + }); + all_template_args_in_function_signature &= found; + } + else if (type_str == "enum" || type_str == "bool") { + arg_pattern += ",w"; + /* Values cannot be resolved using type deduction. */ + all_template_args_in_function_signature = false; + } + else if (type_str == "int" || type_str == "uint") { + arg_pattern += ",0"; + /* Values cannot be resolved using type deduction. */ + all_template_args_in_function_signature = false; + } + else { + report_error(smatch(), "Invalid template argument type"); + } + }); + + Token after_args = fn_name.next().scope().end().next(); + Scope fn_body = (after_args == Const) ? after_args.next().scope() : after_args.scope(); + Token fn_end = fn_body.end(); + const string fn_decl = parser.substr_range_inclusive(fn_start.str_index_start(), + fn_end.line_end()); + + /* Remove declaration. */ + Token template_keyword = temp.start().prev(); + parser.erase(template_keyword.str_index_start(), fn_end.line_end()); + + /* Replace instantiations. */ + Scope parent_scope = temp.scope(); + string specialization_pattern = "tww<" + arg_pattern.substr(1) + ">("; + parent_scope.foreach_match(specialization_pattern, [&](const std::vector &tokens) { + if (fn_name.str_no_whitespace() != tokens[2].str_no_whitespace()) { + return; + } + /* Parse template values. */ + vector> arg_name_value_pairs; + for (int i = 0; i < arg_list.size(); i++) { + arg_name_value_pairs.emplace_back(arg_list[i], tokens[4 + 2 * i].str_no_whitespace()); + } + /* Specialize template content. */ + Parser instance_parser(fn_decl, true); + instance_parser.foreach_match("w", [&](const std::vector &tokens) { + string token_str = tokens[0].str_no_whitespace(); + for (const auto &arg_name_value : arg_name_value_pairs) { + if (token_str == arg_name_value.first) { + instance_parser.replace(tokens[0], arg_name_value.second); + } + } + }); + + if (!all_template_args_in_function_signature) { + const string template_args = parser.substr_range_inclusive( + tokens[3], tokens[3 + arg_pattern.size()]); + size_t pos = fn_decl.find(" " + fn_name.str()); + instance_parser.insert_after(pos + fn_name.str().size(), template_args); + } + /* Paste template content in place of instantiation. */ + Token end_of_instantiation = tokens.back().scope().end().next(); + parser.insert_line_number(tokens.front().str_index_start() - 1, fn_start.line_number()); + parser.replace(tokens.front().str_index_start(), + end_of_instantiation.str_index_last_no_whitespace(), + instance_parser.result_get()); + parser.insert_line_number(end_of_instantiation.line_end() + 1, + end_of_instantiation.line_number() + 1); + }); + }); + + out_str = parser.result_get(); } { /* Check if there is no remaining declaration and instantiation that were not processed. */ if (out_str.find("template<") != std::string::npos) { - std::regex regex_declaration(R"(\btemplate<)"); - regex_global_search(out_str, regex_declaration, [&](const std::smatch &match) { - report_error(match, "Template declaration unsupported syntax"); - }); + report_error(smatch(), "Template declaration unsupported syntax"); } if (out_str.find("template ") != std::string::npos) { - std::regex regex_instance(R"(\btemplate )"); - regex_global_search(out_str, regex_instance, [&](const std::smatch &match) { - report_error(match, "Template instantiation unsupported syntax"); - }); + report_error(smatch(), "Template instantiation unsupported syntax"); } } return out_str; } - std::string template_call_mutation(std::string &str) + std::string template_call_mutation(const std::string &str) { - while (true) { - std::smatch match; - if (std::regex_search(str, match, std::regex(R"(([\w\d]+)<([\w\d\n, ]+)>)")) == false) { - break; + using namespace std; + using namespace shader::parser; + + Parser parser(str, true); + /* This rely on our codestyle that do not put spaces between template name and the opening + * angle bracket. */ + parser.foreach_match("w<", [&](const std::vector &tokens) { + Token token = tokens[1]; + parser.replace(token, "_"); + token = token.next(); + while (token != '>') { + if (token == ',') { + /* Also replace and skip the space after the comma. */ + Token next_token = token.next_not_whitespace(); + parser.replace(token, next_token.prev(), "_"); + token = next_token; + } + else { + token = token.next(); + } } - const std::string template_name = match[1].str(); - const std::string template_args = match[2].str(); - - std::string replacement = "TEMPLATE_GLUE" + - std::to_string(char_count(template_args, ',') + 1) + "(" + - template_name + ", " + template_args + ")"; - - replace_all(str, match[0].str(), replacement); - } - return str; + /* Replace closing angle bracket. */ + parser.replace(token, "_"); + }); + return parser.result_get(); } std::string remove_quotes(const std::string &str) @@ -899,10 +946,27 @@ class Preprocessor { std::string swizzle_function_mutation(const std::string &str) { - /* Change C++ swizzle functions into plain swizzle. */ - std::regex regex(R"((\.[rgbaxyzw]{2,4})\(\))"); - /* Keep character count the same. Replace parenthesis by spaces. */ - return std::regex_replace(str, regex, "$1 "); + using namespace std; + using namespace shader::parser; + + Parser parser(str); + + parser.foreach_scope(ScopeType::Global, [&](Scope scope) { + /* Change C++ swizzle functions into plain swizzle. */ + /** IMPORTANT: This prevent the usage of any method with a swizzle name. */ + scope.foreach_match(".w()", [&](const std::vector &tokens) { + string method_name = tokens[1].str_no_whitespace(); + if (method_name.length() > 1 && method_name.length() <= 4 && + (method_name.find_first_not_of("xyzw") == string::npos || + method_name.find_first_not_of("rgba") == string::npos)) + { + /* `.xyz()` -> `.xyz` */ + /* Keep character count the same. Replace parenthesis by spaces. */ + parser.replace(tokens[2], tokens[3], " "); + } + }); + }); + return parser.result_get(); } void threadgroup_variables_parsing(const std::string &str) @@ -1163,23 +1227,24 @@ class Preprocessor { if (is_const) { fn_parser.erase(args.end().next()); - fn_parser.insert_after( - args.start(), "const " + struct_name.str_no_whitespace() + " this" + suffix); + fn_parser.insert_after(args.start(), + "const " + struct_name.str_no_whitespace() + " this_" + + suffix); } else { fn_parser.insert_after(args.start(), - struct_name.str_no_whitespace() + " &this" + suffix); + struct_name.str_no_whitespace() + " &this_" + suffix); } }); } /* `*this` -> `this` */ scope.foreach_match("*T", [&](const std::vector &tokens) { - fn_parser.replace(tokens[0], tokens[1], tokens[1].str()); + fn_parser.replace(tokens[0], tokens[1], "this_"); }); /* `this->` -> `this.` */ scope.foreach_match("TD", [&](const std::vector &tokens) { - fn_parser.replace(tokens[0], tokens[1], tokens[0].str() + "."); + fn_parser.replace(tokens[0], tokens[1], "this_."); }); }); diff --git a/source/blender/gpu/glsl_preprocess/shader_parser.hh b/source/blender/gpu/glsl_preprocess/shader_parser.hh index aa2dd07d940..981c5fac70b 100644 --- a/source/blender/gpu/glsl_preprocess/shader_parser.hh +++ b/source/blender/gpu/glsl_preprocess/shader_parser.hh @@ -645,6 +645,16 @@ struct Token { return {data, index + 1}; } + /* Only usable when building with whitespace. */ + Token next_not_whitespace() const + { + Token next = this->next(); + while (next == ' ' || next == '\n') { + next = next.next(); + } + return next; + } + /* Returns the scope that contains this token. */ Scope scope() const; @@ -658,6 +668,11 @@ struct Token { return index_range().last(); } + size_t str_index_last_no_whitespace() const + { + return data->str.find_last_not_of(" \n", str_index_last()); + } + /* Index of the first character of the line this token is. */ size_t line_start() const { @@ -699,25 +714,29 @@ struct Token { } TokenType type() const - { - return TokenType(*this); - } - - operator TokenType() const { if (is_invalid()) { return Invalid; } return TokenType(data->token_types[index]); } + bool operator==(TokenType type) const { - return TokenType(*this) == type; + return this->type() == type; } bool operator!=(TokenType type) const { return !(*this == type); } + bool operator==(char type) const + { + return *this == TokenType(type); + } + bool operator!=(char type) const + { + return *this != TokenType(type); + } }; struct Scope { @@ -865,6 +884,13 @@ struct Parser { } } + void foreach_match(const std::string &pattern, + std::function)> callback) + { + foreach_scope(ScopeType::Global, + [&](const Scope scope) { scope.foreach_match(pattern, callback); }); + } + /* Run a callback for all existing function scopes. */ void foreach_function( std::functionparse(); return true; } + bool apply_mutations() + { + bool applied = only_apply_mutations(); + if (applied) { + this->parse(); + } + return applied; + } + /* Apply mutations if any and get resulting string. */ const std::string &result_get() { - apply_mutations(); + only_apply_mutations(); return data_.str; } diff --git a/source/blender/gpu/shaders/metal/mtl_shader_defines.msl b/source/blender/gpu/shaders/metal/mtl_shader_defines.msl index f813ab40edd..ae919c82f64 100644 --- a/source/blender/gpu/shaders/metal/mtl_shader_defines.msl +++ b/source/blender/gpu/shaders/metal/mtl_shader_defines.msl @@ -1231,11 +1231,6 @@ float4x4 __mat4x4(float3x3 a) { return to_float4x4(a); } } \ ; -#define TEMPLATE_GLUE1(name, arg1) name##_##arg1##_ -#define TEMPLATE_GLUE2(name, arg1, arg2) name##_##arg1##_##arg2##_ -#define TEMPLATE_GLUE3(name, arg1, arg2, arg3) name##_##arg1##_##arg2##_##arg3##_ -#define TEMPLATE_GLUE4(name, arg1, arg2, arg3, arg4) name##_##arg1##_##arg2##_##arg3##_##arg4##_ - /* Stage agnostic builtin function. * MSL allow mixing shader stages inside the same source file. * Leaving the calls untouched makes sure we catch invalid usage during CI testing. */ diff --git a/source/blender/gpu/shaders/opengl/glsl_shader_defines.glsl b/source/blender/gpu/shaders/opengl/glsl_shader_defines.glsl index 62d528e1f74..babb98a0416 100644 --- a/source/blender/gpu/shaders/opengl/glsl_shader_defines.glsl +++ b/source/blender/gpu/shaders/opengl/glsl_shader_defines.glsl @@ -164,11 +164,6 @@ RESHAPE(float3x3, mat3x3, mat3x4) #define _enum_decl(name) constexpr uint #define _enum_end _enum_dummy; -#define TEMPLATE_GLUE1(name, arg1) name##_##arg1##_ -#define TEMPLATE_GLUE2(name, arg1, arg2) name##_##arg1##_##arg2##_ -#define TEMPLATE_GLUE3(name, arg1, arg2, arg3) name##_##arg1##_##arg2##_##arg3##_ -#define TEMPLATE_GLUE4(name, arg1, arg2, arg3, arg4) name##_##arg1##_##arg2##_##arg3##_##arg4##_ - /* Stage agnostic builtin function. * GLSL doesn't allow mixing shader stages inside the same source file. * Make sure builtin functions are stubbed when used in an invalid stage. */ diff --git a/source/blender/gpu/tests/shader_preprocess_test.cc b/source/blender/gpu/tests/shader_preprocess_test.cc index e4677ce4023..6afefc4b38d 100644 --- a/source/blender/gpu/tests/shader_preprocess_test.cc +++ b/source/blender/gpu/tests/shader_preprocess_test.cc @@ -226,11 +226,15 @@ static void test_preprocess_template() string input = R"( template void func(T a) {a;} -template void func(float a);)"; +template void func(float a); +)"; string expect = R"( -#define func_TEMPLATE(T) \ -void func(T a) {a;} -func_TEMPLATE(float)/*float a*/)"; + + +#line 3 +void func(float a) {a;} +#line 5 +)"; string error; string output = process_test_string(input, error); EXPECT_EQ(output, expect); @@ -242,41 +246,55 @@ template void func(T a) { a; } -template void func(float a);)"; +template void func(float a); +)"; string expect = R"( -#define func_TEMPLATE(T, i) \ -void func_##T##_##i##_(T a) { \ - a; \ + + + + +#line 3 +void func_float_1_(float a) { + a; } -func_TEMPLATE(float, 1)/*float a*/)"; +#line 7 +)"; string error; string output = process_test_string(input, error); EXPECT_EQ(output, expect); EXPECT_EQ(error, ""); } { - string input = R"(template<> void func(T a) {a};)"; - string expect = R"( void func_T_Q_(T a) {a};)"; + string input = R"( +template<> void func(T a) {a} +)"; + string expect = R"( + void func_T_Q_(T a) {a} +)"; string error; string output = process_test_string(input, error); EXPECT_EQ(output, expect); EXPECT_EQ(error, ""); } { - string input = R"(template void func(T a) {a;})"; + string input = R"( +template void func(T a) {a;} +)"; string error; string output = process_test_string(input, error); - EXPECT_EQ(error, "Template declaration unsupported syntax"); + EXPECT_EQ(error, "Default arguments are not supported inside template declaration"); } { - string input = R"(template void func(float a);)"; + string input = R"( +template void func(float a); +)"; string error; string output = process_test_string(input, error); EXPECT_EQ(error, "Template instantiation unsupported syntax"); } { string input = R"(func(a);)"; - string expect = R"(TEMPLATE_GLUE2(func, float, 1)(a);)"; + string expect = R"(func_float_1_(a);)"; string error; string output = process_test_string(input, error); EXPECT_EQ(output, expect); @@ -710,11 +728,16 @@ float write(float a){ return a; } string expect = R"( -#define NS_read_TEMPLATE(T) T NS_read(T a) \ -{ \ - return a; \ + + + + +#line 3 +float NS_read(float a) +{ + return a; } -NS_read_TEMPLATE(float)/*float*/ +#line 8 float NS_write(float a){ return a; } )"; @@ -945,15 +968,15 @@ class S { void main() { S s = S::construct(); - a.b(); - a(0).b(); - a().b(); - a.b.c(); - a.b(0).c(); - a.b().c(); - a[0].b(); - a.b[0].c(); - a.b().c[0]; + f.f(); + f(0).f(); + f().f(); + l.o.t(); + l.o(0).t(); + l.o().t(); + l[0].o(); + l.o[0].t(); + l.o().t[0]; } )"; string expect = R"( @@ -1001,7 +1024,7 @@ struct S { return this_; } #line 25 - int size(const S this_) + int size(const S this_) { return this_.member; } @@ -1010,15 +1033,15 @@ struct S { void main() { S s = S_construct(); - b(a); - b(a(0)); - b(a()); - c(a.b); - c(b(a, 0)); - c(b(a)); - b(a[0]); - c(a.b[0]); - b(a).c[0]; + f(f); + f(f(0)); + f(f()); + t(l.o); + t(o(l, 0)); + t(o(l)); + o(l[0]); + t(l.o[0]); + o(l).t[0]; } )"; string error;