From 6f9c3b1bd547300d395b39d756f97dd47bcacc19 Mon Sep 17 00:00:00 2001
From: Jeroen Bakker <jeroen@blender.org>
Date: Thu, 17 Aug 2023 08:59:03 +0200
Subject: [PATCH] Vulkan: Report Incompatible Shaders

A difference was detected between stage interfaces between OpenGL and Vulkan
that are not compatible with our current API.

**OpenGL**

In OpenGL an stage interface struct can have different interpolation qualifiers
per attribute.

```glsl
struct MyStageInterface {
    smooth vec4 color;
    flat int face_flag;
};
layout(..) MyStageInterface interp;
```

**Vulkan**

In vulkan the interpolation qualifier isn't supported on attribute
level and needs to be added to the struct.

```glsl
struct MyStageInterface {
    vec4 color;
};
struct MyStageInterface_flat {
    int face_flag;
};
layout(..) smooth MyStageInterface interp;
layout(..) flat MyStageInterface_flat interp_flat;
```

This patch reports shaders that are incompatible with Vulkan so they can be
patched. Report is only done in debug mode and when using the vulkan backend.
After all shaders are patched an error will be raised so developers will
known immediately when incompatibility are created.

Making the shaders compatible and adding the error will be done in future
patches.

**Python**

Via Python gpu module (gpu.types.GPUShaderCreateInfo) it isn't possible
to construct an incompatible shader as instance names cannot be set
via the API. So this isn't a breaking change.

Pull Request: https://projects.blender.org/blender/blender/pulls/111138
---
 .../gpu/intern/gpu_shader_create_info.cc      | 52 +++++++++++++++++++
 .../gpu/intern/gpu_shader_create_info.hh      |  1 +
 source/blender/gpu/vulkan/vk_shader.cc        |  5 ++
 3 files changed, 58 insertions(+)

diff --git a/source/blender/gpu/intern/gpu_shader_create_info.cc b/source/blender/gpu/intern/gpu_shader_create_info.cc
index 91a46e956df..64e3fd6f1e7 100644
--- a/source/blender/gpu/intern/gpu_shader_create_info.cc
+++ b/source/blender/gpu/intern/gpu_shader_create_info.cc
@@ -33,6 +33,58 @@ using InterfaceDictionnary = Map<StringRef, StageInterfaceInfo *>;
 static CreateInfoDictionnary *g_create_infos = nullptr;
 static InterfaceDictionnary *g_interfaces = nullptr;
 
+/* -------------------------------------------------------------------- */
+/** \name Check Backend Support
+ *
+ * \{ */
+
+static bool is_vulkan_compatible_interface(const StageInterfaceInfo &iface)
+{
+  if (iface.instance_name.is_empty()) {
+    return true;
+  }
+
+  bool use_flat = false;
+  bool use_smooth = false;
+  bool use_noperspective = false;
+  for (const StageInterfaceInfo::InOut &attr : iface.inouts) {
+    switch (attr.interp) {
+      case Interpolation::FLAT:
+        use_flat = true;
+        break;
+      case Interpolation::SMOOTH:
+        use_smooth = true;
+        break;
+      case Interpolation::NO_PERSPECTIVE:
+        use_noperspective = true;
+        break;
+    }
+  }
+  int num_used_interpolation_types = (use_flat ? 1 : 0) + (use_smooth ? 1 : 0) +
+                                     (use_noperspective ? 1 : 0);
+
+  return num_used_interpolation_types <= 1;
+}
+
+bool ShaderCreateInfo::is_vulkan_compatible() const
+{
+  /* Vulkan doesn't support setting an interpolation mode per attribute in a struct. */
+  for (const StageInterfaceInfo *iface : vertex_out_interfaces_) {
+    if (!is_vulkan_compatible_interface(*iface)) {
+      return false;
+    }
+  }
+  for (const StageInterfaceInfo *iface : geometry_out_interfaces_) {
+    if (!is_vulkan_compatible_interface(*iface)) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+/** \} */
+
 void ShaderCreateInfo::finalize()
 {
   if (finalized_) {
diff --git a/source/blender/gpu/intern/gpu_shader_create_info.hh b/source/blender/gpu/intern/gpu_shader_create_info.hh
index 986d296da2c..59f378a0516 100644
--- a/source/blender/gpu/intern/gpu_shader_create_info.hh
+++ b/source/blender/gpu/intern/gpu_shader_create_info.hh
@@ -882,6 +882,7 @@ struct ShaderCreateInfo {
   void finalize();
 
   std::string check_error() const;
+  bool is_vulkan_compatible() const;
 
   /** Error detection that some backend compilers do not complain about. */
   void validate_merge(const ShaderCreateInfo &other_info);
diff --git a/source/blender/gpu/vulkan/vk_shader.cc b/source/blender/gpu/vulkan/vk_shader.cc
index a0d9399ba58..90702e98306 100644
--- a/source/blender/gpu/vulkan/vk_shader.cc
+++ b/source/blender/gpu/vulkan/vk_shader.cc
@@ -656,6 +656,11 @@ bool VKShader::finalize(const shader::ShaderCreateInfo *info)
   if (compilation_failed_) {
     return false;
   }
+#if DEBUG
+  if (!info->is_vulkan_compatible()) {
+    std::cout << "'" << info->name_ << "' stage interfaces are not compatible with Vulkan.\n";
+  }
+#endif
 
   VKShaderInterface *vk_interface = new VKShaderInterface();
   vk_interface->init(*info);