Snap: Support occlusion of 'In Front' objects

This fixes #111767

`In Front` objects remain visible even if they are behind non
`In Front` objects.

It is to be expected then that the snap for them is not occluded as if
they were not `In Front`.

source/blender/blenlib/BLI_kdopbvh.h

@@ -307,7 +307,7 @@ int BLI_bvhtree_find_nearest_projected(const BVHTree *tree,
                                        float projmat[4][4],
                                        float winsize[2],
                                        float mval[2],
-                                       float clip_planes[6][4],
+                                       float (*clip_planes)[4],
                                        int clip_plane_len,
                                        BVHTreeNearest *nearest,
                                        BVHTree_NearestProjectedCallback callback,

source/blender/blenlib/intern/BLI_kdopbvh.c

@@ -145,7 +145,7 @@ typedef struct BVHNearestProjectedData {
   const BVHTree *tree;
   struct DistProjectedAABBPrecalc precalc;
   bool closest_axis[3];
-  float clip_plane[7][4];
+  float clip_plane[6][4];
   int clip_plane_len;
   BVHTree_NearestProjectedCallback callback;
   void *userdata;
@@ -2332,7 +2332,7 @@ int BLI_bvhtree_find_nearest_projected(const BVHTree *tree,
                                        float projmat[4][4],
                                        float winsize[2],
                                        float mval[2],
-                                       float clip_plane[6][4],
+                                       float (*clip_plane)[4],
                                        int clip_plane_len,
                                        BVHTreeNearest *nearest,
                                        BVHTree_NearestProjectedCallback callback,

source/blender/editors/transform/transform_snap_object.cc

@@ -40,6 +40,21 @@ static blender::timeit::Nanoseconds duration_;
 
 using namespace blender;
 
+static float4 occlusion_plane_create(float3 ray_dir, float3 ray_co, float3 ray_no)
+{
+  float4 plane;
+  plane_from_point_normal_v3(plane, ray_co, ray_no);
+  if (dot_v3v3(ray_dir, plane) > 0.0f) {
+    /* The plane is facing the wrong direction. */
+    negate_v4(plane);
+  }
+
+  /* Small offset to simulate a kind of volume for edges and vertices. */
+  plane[3] += 0.01f;
+
+  return plane;
+}
+
 static bool test_projected_vert_dist(const DistProjectedAABBPrecalc *precalc,
                                      const float (*clip_plane)[4],
                                      const int clip_plane_len,
@@ -118,23 +133,25 @@ SnapData::SnapData(SnapObjectContext *sctx, const float4x4 &obmat)
   copy_v3_fl3(this->nearest_point.no, 0.0f, 0.0f, 1.0f);
 }
 
-void SnapData::clip_planes_enable(SnapObjectContext *sctx, bool skip_occlusion_plane)
+void SnapData::clip_planes_enable(SnapObjectContext *sctx,
+                                  const Object *ob_eval,
+                                  bool skip_occlusion_plane)
 {
-  float(*clip_planes)[4] = reinterpret_cast<float(*)[4]>(sctx->runtime.clip_planes.data());
-  int clip_plane_len = sctx->runtime.clip_planes.size();
-
-  if (skip_occlusion_plane && sctx->runtime.has_occlusion_plane) {
-    /* We snap to vertices even if occluded. */
-    clip_planes++;
-    clip_plane_len--;
-  }
-
   float4x4 tobmat = math::transpose(this->obmat_);
-  for (int i : IndexRange(clip_plane_len)) {
+  if (!skip_occlusion_plane) {
-    this->clip_planes.append(tobmat * clip_planes[i]);
+    const bool is_in_front = sctx->runtime.params.use_occlusion_test &&
+                             (ob_eval->dtx & OB_DRAW_IN_FRONT) != 0;
+    if (!is_in_front && sctx->runtime.has_occlusion_plane) {
+      this->clip_planes.append(tobmat * sctx->runtime.occlusion_plane);
+    }
+    else if (sctx->runtime.has_occlusion_plane_in_front) {
+      this->clip_planes.append(tobmat * sctx->runtime.occlusion_plane_in_front);
+    }
   }
 
-  BLI_assert(this->clip_planes.size() == clip_plane_len);
+  for (float4 &plane : sctx->runtime.clip_planes) {
+    this->clip_planes.append(tobmat * plane);
+  }
 }
 
 bool SnapData::snap_boundbox(const float3 &min, const float3 &max)
@@ -304,6 +321,36 @@ void SnapData::register_result(SnapObjectContext *sctx, Object *ob_eval, const I
   this->register_result(sctx, ob_eval, id_eval, this->obmat_, &this->nearest_point);
 }
 
+void SnapData::register_result_raycast(SnapObjectContext *sctx,
+                                       Object *ob_eval,
+                                       const ID *id_eval,
+                                       const blender::float4x4 &obmat,
+                                       const BVHTreeRayHit *hit,
+                                       const bool is_in_front)
+{
+  const float depth_max = is_in_front ? sctx->ret.ray_depth_max_in_front : sctx->ret.ray_depth_max;
+  if (hit->dist <= depth_max) {
+    float3 co = math::transform_point(obmat, float3(hit->co));
+    float3 no = math::normalize(math::transform_direction(obmat, float3(hit->no)));
+
+    sctx->ret.loc = co;
+    sctx->ret.no = no;
+    sctx->ret.index = hit->index;
+    sctx->ret.obmat = obmat;
+    sctx->ret.ob = ob_eval;
+    sctx->ret.data = id_eval;
+    if (hit->dist <= sctx->ret.ray_depth_max) {
+      sctx->ret.ray_depth_max = hit->dist;
+    }
+
+    if (is_in_front) {
+      sctx->runtime.occlusion_plane_in_front = occlusion_plane_create(
+          sctx->runtime.ray_dir, co, no);
+      sctx->runtime.has_occlusion_plane_in_front = true;
+    }
+  }
+}
+
 /* -------------------------------------------------------------------- */
 /** \name Utilities
  * \{ */
@@ -584,9 +631,6 @@ static eSnapMode raycast_obj_fn(SnapObjectContext *sctx,
   }
 
   if (retval) {
-    sctx->ret.obmat = obmat;
-    sctx->ret.ob = ob_eval;
-    sctx->ret.data = ob_data;
     return SCE_SNAP_TO_FACE;
   }
   return SCE_SNAP_TO_NONE;
@@ -835,7 +879,7 @@ eSnapMode snap_object_center(SnapObjectContext *sctx,
 
   SnapData nearest2d(sctx, obmat);
 
-  nearest2d.clip_planes_enable(sctx);
+  nearest2d.clip_planes_enable(sctx, ob_eval);
 
   if (nearest2d.snap_point(float3(0.0f))) {
     nearest2d.register_result(sctx, ob_eval, static_cast<const ID *>(ob_eval->data));
@@ -1022,6 +1066,7 @@ static bool snap_object_context_runtime_init(SnapObjectContext *sctx,
   sctx->runtime.use_occlusion_test_edit = use_occlusion_test &&
                                           (snap_to_flag & SCE_SNAP_TO_FACE) == 0;
   sctx->runtime.has_occlusion_plane = false;
+  sctx->runtime.has_occlusion_plane_in_front = false;
   sctx->runtime.object_index = 0;
 
   copy_v3_v3(sctx->runtime.ray_start, ray_start);
@@ -1053,7 +1098,7 @@ static bool snap_object_context_runtime_init(SnapObjectContext *sctx,
     sctx->runtime.rv3d = rv3d;
   }
 
-  sctx->ret.ray_depth_max = ray_depth;
+  sctx->ret.ray_depth_max = sctx->ret.ray_depth_max_in_front = ray_depth;
   sctx->ret.index = -1;
   sctx->ret.hit_list = hit_list;
   sctx->ret.ob = nullptr;
@@ -1322,16 +1367,9 @@ eSnapMode ED_transform_snap_object_project_view3d_ex(SnapObjectContext *sctx,
         sctx->ret.ob->type != OB_CURVES_LEGACY)
     {
       /* Compute the new clip_pane but do not add it yet. */
-      float new_clipplane[4];
       BLI_ASSERT_UNIT_V3(sctx->ret.no);
-      plane_from_point_normal_v3(new_clipplane, sctx->ret.loc, sctx->ret.no);
+      sctx->runtime.occlusion_plane = occlusion_plane_create(
-      if (dot_v3v3(sctx->runtime.ray_dir, new_clipplane) > 0.0f) {
+          sctx->runtime.ray_dir, sctx->ret.loc, sctx->ret.no);
-        /* The plane is facing the wrong direction. */
-        negate_v4(new_clipplane);
-      }
-
-      /* Small offset to simulate a kind of volume for edges and vertices. */
-      new_clipplane[3] += 0.01f;
 
       /* Try to snap only to the face. */
       elem_test = snap_polygon(sctx, sctx->runtime.snap_to_flag);
@@ -1339,8 +1377,7 @@ eSnapMode ED_transform_snap_object_project_view3d_ex(SnapObjectContext *sctx,
         elem = elem_test;
       }
 
-      /* Add the new clip plane to the beginning of the list. */
+      /* Add the new clip plane. */
-      sctx->runtime.clip_planes.prepend(new_clipplane);
       sctx->runtime.has_occlusion_plane = true;
     }
 

source/blender/editors/transform/transform_snap_object.hh

@@ -10,7 +10,7 @@
 
 #include "BLI_math_geom.h"
 
-#define MAX_CLIPPLANE_LEN 7
+#define MAX_CLIPPLANE_LEN 6
 
 #define SNAP_TO_EDGE_ELEMENTS \
   (SCE_SNAP_TO_EDGE | SCE_SNAP_TO_EDGE_ENDPOINT | SCE_SNAP_TO_EDGE_MIDPOINT | \
@@ -52,11 +52,14 @@ struct SnapObjectContext {
     blender::float2 mval;
 
     blender::Vector<blender::float4, MAX_CLIPPLANE_LEN> clip_planes;
+    blender::float4 occlusion_plane;
+    blender::float4 occlusion_plane_in_front;
 
     /* read/write */
     uint object_index;
 
-    bool has_occlusion_plane; /* Ignore plane of occlusion in curves. */
+    bool has_occlusion_plane;
+    bool has_occlusion_plane_in_front;
     bool use_occlusion_test_edit;
   } runtime;
 
@@ -78,6 +81,7 @@ struct SnapObjectContext {
     const ID *data;
 
     float ray_depth_max;
+    float ray_depth_max_in_front;
     float dist_px_sq;
   } ret;
 };
@@ -103,7 +107,7 @@ class SnapData {
  public:
   /* Read-only. */
   DistProjectedAABBPrecalc nearest_precalc;
-  blender::Vector<blender::float4, MAX_CLIPPLANE_LEN> clip_planes;
+  blender::Vector<blender::float4, MAX_CLIPPLANE_LEN + 1> clip_planes;
   blender::float4x4 pmat_local;
   blender::float4x4 obmat_;
   const bool is_persp;
@@ -117,7 +121,9 @@ class SnapData {
   SnapData(SnapObjectContext *sctx,
            const blender::float4x4 &obmat = blender::float4x4::identity());
 
-  void clip_planes_enable(SnapObjectContext *sctx, bool skip_occlusion_plane = false);
+  void clip_planes_enable(SnapObjectContext *sctx,
+                          const Object *ob_eval,
+                          bool skip_occlusion_plane = false);
   bool snap_boundbox(const blender::float3 &min, const blender::float3 &max);
   bool snap_point(const blender::float3 &co, int index = -1);
   bool snap_edge(const blender::float3 &va, const blender::float3 &vb, int edge_index = -1);
@@ -128,6 +134,12 @@ class SnapData {
                               const blender::float4x4 &obmat,
                               BVHTreeNearest *r_nearest);
   void register_result(SnapObjectContext *sctx, Object *ob_eval, const ID *id_eval);
+  static void register_result_raycast(SnapObjectContext *sctx,
+                                      Object *ob_eval,
+                                      const ID *id_eval,
+                                      const blender::float4x4 &obmat,
+                                      const BVHTreeRayHit *hit,
+                                      const bool is_in_front);
 
   virtual void get_vert_co(const int /*index*/, const float ** /*r_co*/){};
   virtual void get_edge_verts_index(const int /*index*/, int /*r_v_index*/[2]){};

source/blender/editors/transform/transform_snap_object_armature.cc

@@ -46,7 +46,7 @@ eSnapMode snapArmature(SnapObjectContext *sctx,
     }
   }
 
-  nearest2d.clip_planes_enable(sctx);
+  nearest2d.clip_planes_enable(sctx, ob_eval);
 
   const float *head_vec = nullptr, *tail_vec = nullptr;
 

source/blender/editors/transform/transform_snap_object_camera.cc

@@ -45,7 +45,7 @@ eSnapMode snapCamera(SnapObjectContext *sctx,
   BKE_tracking_get_camera_object_matrix(object, orig_camera_mat.ptr());
 
   SnapData nearest2d(sctx);
-  nearest2d.clip_planes_enable(sctx);
+  nearest2d.clip_planes_enable(sctx, object);
 
   MovieTracking *tracking = &clip->tracking;
   LISTBASE_FOREACH (MovieTrackingObject *, tracking_object, &tracking->objects) {

source/blender/editors/transform/transform_snap_object_curve.cc

@@ -45,7 +45,7 @@ eSnapMode snapCurve(SnapObjectContext *sctx, Object *ob_eval, const float4x4 &ob
     }
   }
 
-  nearest2d.clip_planes_enable(sctx, true);
+  nearest2d.clip_planes_enable(sctx, ob_eval, true);
 
   bool skip_selected = (sctx->runtime.params.snap_target_select & SCE_SNAP_TARGET_NOT_SELECTED) !=
                        0;

source/blender/editors/transform/transform_snap_object_editmesh.cc

@@ -291,6 +291,7 @@ static void editmesh_looptri_raycast_backface_culling_cb(void *userdata,
 
 static bool raycastEditMesh(SnapCache_EditMesh *em_cache,
                             SnapObjectContext *sctx,
+                            Object *ob_eval,
                             BMEditMesh *em,
                             const float4x4 &obmat,
                             const uint ob_index)
@@ -305,7 +306,10 @@ static bool raycastEditMesh(SnapCache_EditMesh *em_cache,
   /* local scale in normal direction */
   ray_normal_local = math::normalize_and_get_length(ray_normal_local, local_scale);
 
-  local_depth = sctx->ret.ray_depth_max;
+  const bool is_in_front = sctx->runtime.params.use_occlusion_test &&
+                           (ob_eval->dtx & OB_DRAW_IN_FRONT) != 0;
+  const float depth_max = is_in_front ? sctx->ret.ray_depth_max_in_front : sctx->ret.ray_depth_max;
+  local_depth = depth_max;
   if (local_depth != BVH_RAYCAST_DIST_MAX) {
     local_depth *= local_scale;
   }
@@ -351,7 +355,7 @@ static bool raycastEditMesh(SnapCache_EditMesh *em_cache,
                              ray_start_local,
                              ray_normal_local,
                              0.0f,
-                             sctx->ret.ray_depth_max,
+                             depth_max,
                              raycast_all_cb,
                              &data);
 
@@ -374,16 +378,11 @@ static bool raycastEditMesh(SnapCache_EditMesh *em_cache,
     {
       hit.dist += len_diff;
       hit.dist /= local_scale;
-      if (hit.dist <= sctx->ret.ray_depth_max) {
+      if (hit.dist <= depth_max) {
-        sctx->ret.loc = math::transform_point(obmat, float3(hit.co));
+        hit.index = BM_elem_index_get(em->looptris[hit.index][0]->f);
-        sctx->ret.no = math::normalize(math::transform_direction(obmat, float3(hit.no)));
-
-        sctx->ret.ray_depth_max = hit.dist;
-
-        sctx->ret.index = BM_elem_index_get(em->looptris[hit.index][0]->f);
-
         retval = true;
       }
+      SnapData::register_result_raycast(sctx, ob_eval, nullptr, obmat, &hit, is_in_front);
     }
   }
 
@@ -472,7 +471,7 @@ eSnapMode snap_polygon_editmesh(SnapObjectContext *sctx,
 
   BMEditMesh *em = BKE_editmesh_from_object(ob_eval);
   SnapData_EditMesh nearest2d(sctx, em->bm, obmat);
-  nearest2d.clip_planes_enable(sctx);
+  nearest2d.clip_planes_enable(sctx, ob_eval);
 
   BVHTreeNearest nearest{};
   nearest.index = -1;
@@ -621,7 +620,7 @@ static eSnapMode snapEditMesh(SnapCache_EditMesh *em_cache,
    * alpha is 1.0 in this case. But even with the alpha being 1.0, the edit mesh is still not
    * occluded. */
   const bool skip_occlusion_plane = XRAY_FLAG_ENABLED(sctx->runtime.v3d);
-  nearest2d.clip_planes_enable(sctx, skip_occlusion_plane);
+  nearest2d.clip_planes_enable(sctx, ob_eval, skip_occlusion_plane);
 
   BVHTreeNearest nearest{};
   nearest.index = -1;
@@ -701,7 +700,7 @@ eSnapMode snap_object_editmesh(SnapObjectContext *sctx,
   }
 
   if (snap_mode_used & SCE_SNAP_TO_FACE) {
-    if (raycastEditMesh(em_cache, sctx, em, obmat, sctx->runtime.object_index++)) {
+    if (raycastEditMesh(em_cache, sctx, ob_eval, em, obmat, sctx->runtime.object_index++)) {
       return SCE_SNAP_TO_FACE;
     }
   }

source/blender/editors/transform/transform_snap_object_mesh.cc

@@ -108,7 +108,10 @@ static bool raycastMesh(SnapObjectContext *sctx,
   /* local scale in normal direction */
   ray_normal_local = math::normalize_and_get_length(ray_normal_local, local_scale);
 
-  local_depth = sctx->ret.ray_depth_max;
+  const bool is_in_front = sctx->runtime.params.use_occlusion_test &&
+                           (ob_eval->dtx & OB_DRAW_IN_FRONT) != 0;
+  const float depth_max = is_in_front ? sctx->ret.ray_depth_max_in_front : sctx->ret.ray_depth_max;
+  local_depth = depth_max;
   if (local_depth != BVH_RAYCAST_DIST_MAX) {
     local_depth *= local_scale;
   }
@@ -161,13 +164,8 @@ static bool raycastMesh(SnapObjectContext *sctx,
     data.hit_list = sctx->ret.hit_list;
 
     void *hit_last_prev = data.hit_list->last;
-    BLI_bvhtree_ray_cast_all(treedata.tree,
+    BLI_bvhtree_ray_cast_all(
-                             ray_start_local,
+        treedata.tree, ray_start_local, ray_normal_local, 0.0f, depth_max, raycast_all_cb, &data);
-                             ray_normal_local,
-                             0.0f,
-                             sctx->ret.ray_depth_max,
-                             raycast_all_cb,
-                             &data);
 
     retval = hit_last_prev != data.hit_list->last;
   }
@@ -188,14 +186,11 @@ static bool raycastMesh(SnapObjectContext *sctx,
     {
       hit.dist += len_diff;
       hit.dist /= local_scale;
-      if (hit.dist <= sctx->ret.ray_depth_max) {
+      if (hit.dist <= depth_max) {
-        sctx->ret.loc = math::transform_point(obmat, float3(hit.co));
+        hit.index = looptri_faces[hit.index];
-        sctx->ret.no = math::normalize(math::transform_direction(obmat, float3(hit.no)));
-
-        sctx->ret.ray_depth_max = hit.dist;
-        sctx->ret.index = looptri_faces[hit.index];
         retval = true;
       }
+      SnapData::register_result_raycast(sctx, ob_eval, &me_eval->id, obmat, &hit, is_in_front);
     }
   }
 
@@ -389,7 +384,7 @@ eSnapMode snap_polygon_mesh(SnapObjectContext *sctx,
   const Mesh *mesh_eval = reinterpret_cast<const Mesh *>(id);
 
   SnapData_Mesh nearest2d(sctx, mesh_eval, obmat);
-  nearest2d.clip_planes_enable(sctx);
+  nearest2d.clip_planes_enable(sctx, ob_eval);
 
   BVHTreeNearest nearest{};
   nearest.index = -1;
@@ -493,7 +488,7 @@ static eSnapMode snapMesh(SnapObjectContext *sctx,
     BLI_assert(treedata_dummy.cached);
   }
 
-  nearest2d.clip_planes_enable(sctx);
+  nearest2d.clip_planes_enable(sctx, ob_eval);
 
   BVHTreeNearest nearest{};
   nearest.index = -1;