Multires Bake: Implement vector displacement baking

Supports baking to object and tangent space.

Compatible with Cycles Vector Displacement node which has the
(tangent, normal, bitangent) convention.

The viewport situation is a bit confusing: seems that Eevee
does not handle vector displacement properly and rips all faces
apart. Cycles renders the displaced object correctly.

Not entirely happy with the UI, as displacement space does not
really belong to the Output, but so doesn't Low Resolution Mesh.
Perhaps the best would be to have a separate pass to revisit the
settings, and also make it more clear what the Low Resolution Mesh
actually does.

Pull Request: https://projects.blender.org/blender/blender/pulls/145014

intern/cycles/blender/addon/ui.py

@@ -2147,6 +2147,8 @@ class CYCLES_RENDER_PT_bake_output(CyclesButtonsPanel, Panel):
             layout.prop(cbk, "use_clear", text="Clear Image")
             if cbk.type in {'DISPLACEMENT', 'VECTOR_DISPLACEMENT'}:
                 layout.prop(cbk, "use_lores_mesh")
+            if cbk.type == 'VECTOR_DISPLACEMENT':
+                layout.prop(cbk, "displacement_space", text="Space")
         else:
             layout.prop(cbk, "target")
             if cbk.target == 'IMAGE_TEXTURES':

source/blender/blenloader/intern/versioning_500.cc

@@ -2329,6 +2329,12 @@ void blo_do_versions_500(FileData * /*fd*/, Library * /*lib*/, Main *bmain)
     }
   }
 
+  if (!MAIN_VERSION_FILE_ATLEAST(bmain, 500, 62)) {
+    LISTBASE_FOREACH (Scene *, scene, &bmain->scenes) {
+      scene->r.bake.displacement_space = R_BAKE_SPACE_OBJECT;
+    }
+  }
+
   /**
    * Always bump subversion in BKE_blender_version.h when adding versioning
    * code here, and wrap it inside a MAIN_VERSION_FILE_ATLEAST check.

source/blender/editors/object/object_bake.cc

@@ -95,6 +95,7 @@ struct MultiresBakeJob {
   char bake_margin_type;
   /** mode of baking (displacement, normals, AO) */
   eBakeType type;
+  eBakeSpace displacement_space;
   /** Use low-resolution mesh when baking displacement maps */
   bool use_low_resolution_mesh;
 };
@@ -315,6 +316,7 @@ static wmOperatorStatus multiresbake_image_exec_locked(bContext *C, wmOperator *
       bake.bake_margin_type = eBakeMarginType(scene->r.bake.margin_type);
     }
     bake.type = eBakeType(scene->r.bake.type);
+    bake.displacement_space = eBakeSpace(scene->r.bake.displacement_space);
     bake.use_low_resolution_mesh = scene->r.bake.flag & R_BAKE_LORES_MESH;
 
     bake.ob_image = bake_object_image_get_array(object);
@@ -352,6 +354,7 @@ static void init_multiresbake_job(bContext *C, MultiresBakeJob *bkj)
     bkj->bake_margin_type = eBakeMarginType(scene->r.bake.margin_type);
   }
   bkj->type = eBakeType(scene->r.bake.type);
+  bkj->displacement_space = eBakeSpace(scene->r.bake.displacement_space);
   bkj->use_low_resolution_mesh = scene->r.bake.flag & R_BAKE_LORES_MESH;
   bkj->bake_clear = scene->r.bake.flag & R_BAKE_CLEAR;
 
@@ -402,6 +405,7 @@ static void multiresbake_startjob(void *bkv, wmJobWorkerStatus *worker_status)
     bake.bake_margin = bkj->bake_margin;
     bake.bake_margin_type = eBakeMarginType(bkj->bake_margin_type);
     bake.type = bkj->type;
+    bake.displacement_space = bkj->displacement_space;
     bake.use_low_resolution_mesh = bkj->use_low_resolution_mesh;
     bake.ob_image = data->ob_image;
 

source/blender/makesdna/DNA_scene_defaults.h

@@ -38,6 +38,7 @@
     .margin_type = R_BAKE_ADJACENT_FACES, \
     .normal_space = R_BAKE_SPACE_TANGENT, \
     .normal_swizzle = {R_BAKE_POSX, R_BAKE_POSY, R_BAKE_POSZ}, \
+    .displacement_space = R_BAKE_SPACE_OBJECT, \
   }
 
 #define _DNA_DEFAULT_FFMpegCodecData \

source/blender/makesdna/DNA_scene_types.h

@@ -665,11 +665,15 @@ typedef struct BakeData {
   char normal_swizzle[3];
   char normal_space;
 
+  char displacement_space;
+
   char target;
   char save_mode;
   char margin_type;
   char view_from;
 
+  char _pad[7];
+
   struct Object *cage_object;
 } BakeData;
 
@@ -744,13 +748,13 @@ typedef enum eBakePassFilter {
   R_BAKE_PASS_FILTER_COLOR = (1 << 8),
 } eBakePassFilter;
 
-/** #BakeData::normal_space */
-enum {
+/** #BakeData::normal_space and #BakeData::displacement_space */
+typedef enum eBakeSpace {
   R_BAKE_SPACE_CAMERA = 0,
   R_BAKE_SPACE_WORLD = 1,
   R_BAKE_SPACE_OBJECT = 2,
   R_BAKE_SPACE_TANGENT = 3,
-};
+} eBakeSpace;
 
 #define R_BAKE_PASS_FILTER_ALL (~0)
 

source/blender/makesrna/intern/rna_scene.cc

@@ -5844,15 +5844,17 @@ static void rna_def_bake_data(BlenderRNA *brna)
       //{R_BAKE_AO, "AO", 0, "Ambient Occlusion", "Bake ambient occlusion"},
       {R_BAKE_NORMALS, "NORMALS", 0, "Normals", "Bake normals"},
       {R_BAKE_DISPLACEMENT, "DISPLACEMENT", 0, "Displacement", "Bake displacement"},
+      {R_BAKE_VECTOR_DISPLACEMENT,
+       "VECTOR_DISPLACEMENT",
+       0,
+       "Vector Displacement",
+       "Bake vector displacement"},
+      {0, nullptr, 0, nullptr, nullptr},
+  };
 
-      /* TODO(sergey): Uncomment once tangent space displacement is supported. */
-      /* Use C++ style comment because #if 0 breaks indentation. */
-      // {RE_BAKE_VECTOR_DISPLACEMENT,
-      //  "VECTOR_DISPLACEMENT",
-      //  0,
-      //  "Vector Displacement",
-      //  "Bake vector displacement"},
-
+  static const EnumPropertyItem displacement_space_items[] = {
+      {R_BAKE_SPACE_OBJECT, "OBJECT", 0, "Object", "Bake the displacement in object space"},
+      {R_BAKE_SPACE_TANGENT, "TANGENT", 0, "Tangent", "Bake the displacement in tangent space"},
       {0, nullptr, 0, nullptr, nullptr},
   };
 
@@ -6054,6 +6056,12 @@ static void rna_def_bake_data(BlenderRNA *brna)
   RNA_def_property_ui_text(
       prop, "Low Resolution Mesh", "Calculate heights against unsubdivided low resolution mesh");
   RNA_def_property_update(prop, NC_SCENE | ND_RENDER_OPTIONS, nullptr);
+
+  prop = RNA_def_property(srna, "displacement_space", PROP_ENUM, PROP_NONE);
+  RNA_def_property_enum_sdna(prop, nullptr, "displacement_space");
+  RNA_def_property_enum_items(prop, displacement_space_items);
+  RNA_def_property_ui_text(prop, "Displacement Space", "Choose displacement space for baking");
+  RNA_def_property_update(prop, NC_SCENE | ND_RENDER_OPTIONS, nullptr);
 }
 
 static void rna_def_view_layers(BlenderRNA *brna, PropertyRNA *cprop)

source/blender/render/RE_multires_bake.h

@@ -29,6 +29,7 @@ struct MultiresBakeRender {
   int bake_margin = 0;
   eBakeMarginType bake_margin_type = R_BAKE_ADJACENT_FACES;
   eBakeType type = R_BAKE_NORMALS;
+  eBakeSpace displacement_space = R_BAKE_SPACE_OBJECT;
 
   /* Use low-resolution mesh when baking displacement maps.
    * When true displacement is calculated between the final position in the SubdivCCG and the

source/blender/render/intern/multires_bake.cc

@@ -473,7 +473,61 @@ static float2 get_tile_uv(Image &image, ImageTile &tile)
 
 static bool need_tangent(const MultiresBakeRender &bake)
 {
-  return bake.type == R_BAKE_NORMALS;
+  return (bake.type == R_BAKE_NORMALS) || (bake.type == R_BAKE_VECTOR_DISPLACEMENT &&
+                                           bake.displacement_space == R_BAKE_SPACE_TANGENT);
+}
+
+/* Get matrix which converts tangent space to object space in the (tangent, bitangent, normal)
+ * convention. */
+static float3x3 get_from_tangent_matrix_tbn(const RasterizeTriangle &triangle,
+                                            const float2 &bary_uv)
+{
+  if (!triangle.has_uv_tangents) {
+    return float3x3::identity();
+  }
+
+  const float u = bary_uv.x;
+  const float v = bary_uv.y;
+  const float w = 1 - u - v;
+
+  const float3 &no0 = triangle.normals[0];
+  const float3 &no1 = triangle.normals[1];
+  const float3 &no2 = triangle.normals[2];
+
+  const float4 &tang0 = triangle.uv_tangents[0];
+  const float4 &tang1 = triangle.uv_tangents[1];
+  const float4 &tang2 = triangle.uv_tangents[2];
+
+  /* The sign is the same at all face vertices for any non-degenerate face.
+   * Just in case we clamp the interpolated value though. */
+  const float sign = (tang0.w * u + tang1.w * v + tang2.w * w) < 0 ? (-1.0f) : 1.0f;
+
+  /* x - tangent
+   * y - bitangent (B = sign * cross(N, T))
+   * z - normal */
+  float3x3 from_tangent;
+  from_tangent.x = tang0.xyz() * u + tang1.xyz() * v + tang2.xyz() * w;
+  from_tangent.z = no0.xyz() * u + no1.xyz() * v + no2.xyz() * w;
+  from_tangent.y = sign * math::cross(from_tangent.z, from_tangent.x);
+
+  return from_tangent;
+}
+
+/* Get matrix which converts object space to tangent space in the (tangent, bitangent, normal)
+ * convention. */
+static float3x3 get_to_tangent_matrix_tbn(const RasterizeTriangle &triangle, const float2 &bary_uv)
+{
+  const float3x3 from_tangent = get_from_tangent_matrix_tbn(triangle, bary_uv);
+  return math::invert(from_tangent);
+}
+
+/* Get matrix which converts object space to tangent space in the (tangent, normal, bitangent)
+ * convention. */
+static float3x3 get_to_tangent_matrix_tnb(const RasterizeTriangle &triangle, const float2 &bary_uv)
+{
+  float3x3 from_tangent = get_from_tangent_matrix_tbn(triangle, bary_uv);
+  std::swap(from_tangent.y, from_tangent.z);
+  return math::invert(from_tangent);
 }
 
 /** \} */
@@ -704,10 +758,11 @@ class MultiresDisplacementBaker : public MultiresBaker {
 
 class MultiresVectorDisplacementBaker : public MultiresBaker {
   const SubdivCCG &high_subdiv_ccg_;
+  eBakeSpace space_;
 
  public:
-  explicit MultiresVectorDisplacementBaker(const SubdivCCG &subdiv_ccg)
-      : high_subdiv_ccg_(subdiv_ccg)
+  MultiresVectorDisplacementBaker(const SubdivCCG &subdiv_ccg, const eBakeSpace space)
+      : high_subdiv_ccg_(subdiv_ccg), space_(space)
   {
   }
 
@@ -721,7 +776,14 @@ class MultiresVectorDisplacementBaker : public MultiresBaker {
     const float3 high_level_position = sample_position_on_subdiv_ccg(
         high_subdiv_ccg_, triangle.grid_index, grid_uv);
 
-    return high_level_position - bake_level_position;
+    const float3 displacement = high_level_position - bake_level_position;
+
+    if (space_ == R_BAKE_SPACE_TANGENT) {
+      const float3x3 to_tangent = get_to_tangent_matrix_tnb(triangle, bary_uv);
+      return to_tangent * displacement;
+    }
+
+    return displacement;
   }
 
   void write_pixel(const RasterizeTile &tile,
@@ -749,7 +811,7 @@ class MultiresNormalsBaker : public MultiresBaker {
                     const float2 &grid_uv,
                     RasterizeResult & /*result*/) const override
   {
-    const float3x3 to_tangent = get_to_tangent_matrix(triangle, bary_uv);
+    const float3x3 to_tangent = get_to_tangent_matrix_tbn(triangle, bary_uv);
     const float3 normal = sample_normal_on_subdiv_ccg(subdiv_ccg_, triangle.grid_index, grid_uv);
     return math::normalize(to_tangent * normal) * 0.5f + float3(0.5f, 0.5f, 0.5f);
   }
@@ -760,39 +822,6 @@ class MultiresNormalsBaker : public MultiresBaker {
   {
     write_pixel_to_image_buffer(*tile.ibuf, coord, value);
   }
-
- private:
-  float3x3 get_to_tangent_matrix(const RasterizeTriangle &triangle, const float2 &bary_uv) const
-  {
-    if (!triangle.has_uv_tangents) {
-      return float3x3::identity();
-    }
-
-    const float u = bary_uv.x;
-    const float v = bary_uv.y;
-    const float w = 1 - u - v;
-
-    const float3 &no0 = triangle.normals[0];
-    const float3 &no1 = triangle.normals[1];
-    const float3 &no2 = triangle.normals[2];
-
-    const float4 &tang0 = triangle.uv_tangents[0];
-    const float4 &tang1 = triangle.uv_tangents[1];
-    const float4 &tang2 = triangle.uv_tangents[2];
-
-    /* The sign is the same at all face vertices for any non-degenerate face.
-     * Just in case we clamp the interpolated value though. */
-    const float sign = (tang0.w * u + tang1.w * v + tang2.w * w) < 0 ? (-1.0f) : 1.0f;
-
-    /* This sequence of math is designed specifically as is with great care to be compatible with
-     * our shader. Please don't change without good reason. */
-    float3x3 from_tang;
-    from_tang.x = tang0.xyz() * u + tang1.xyz() * v + tang2.xyz() * w;
-    from_tang.z = no0.xyz() * u + no1.xyz() * v + no2.xyz() * w;
-    from_tang.y = sign * math::cross(from_tang[2], from_tang[0]); /* `B = sign * cross(N, T)` */
-
-    return math::invert(from_tang);
-  }
 };
 
 /** \} */
@@ -829,7 +858,8 @@ static std::unique_ptr<MultiresBaker> create_baker(const MultiresBakeRender &bak
     case R_BAKE_DISPLACEMENT:
       return std::make_unique<MultiresDisplacementBaker>(subdiv_ccg, ibuf, extra_buffers);
     case R_BAKE_VECTOR_DISPLACEMENT:
-      return std::make_unique<MultiresVectorDisplacementBaker>(subdiv_ccg);
+      return std::make_unique<MultiresVectorDisplacementBaker>(subdiv_ccg,
+                                                               bake.displacement_space);
     case R_BAKE_AO:
       /* Not implemented, should not be used. */
       break;