Fix various issues with modifiers and edge display in edit mode

Consistently use edge draw flag instead of original index to determine if an
edge should be drawn or not.

In GPU subdivision the edge original index was used for both edge optimal
display and selection mapping to coarse edges, but they are not the same.
Now match the CPU subdivision logic and use a separate edge draw flag VBO.

For cage display, match Blender 3.3 behavior more in showing/hiding of edges
in wireframe mode. That is edges without a mapping to an original edge are
always hidden when there is no distinct cage, and drawn otherwise. This is
not ideal for e.g. the bevel modifier where it will always show some edges on
corners despite all edges being hidden by the user. But we currently have
no good information to decide if these should be hidden or not, so err on
the side of showing too much as it did before.

Fie #103706: bevel modifier edges not drawn correctly
Fix #103700: optimal display can't be turned of with GPU subdivision
Fix wrong edge display with GPU subdivision preceded by other modifiers

Pull Request #105384

source/blender/draw/intern/draw_cache_extract_mesh_render_data.cc

@@ -459,6 +459,9 @@ MeshRenderData *mesh_render_data_create(Object *object,
     mr->me = (do_final) ? editmesh_eval_final : editmesh_eval_cage;
     mr->edit_data = is_mode_active ? mr->me->runtime->edit_data : nullptr;
 
+    /* If there is no distinct cage, hide unmapped edges that can't be selected. */
+    mr->hide_unmapped_edges = !do_final || editmesh_eval_final == editmesh_eval_cage;
+
     if (mr->edit_data) {
       EditMeshData *emd = mr->edit_data;
       if (emd->vertexCos) {
@@ -521,6 +524,7 @@ MeshRenderData *mesh_render_data_create(Object *object,
     mr->me = me;
     mr->edit_bmesh = nullptr;
     mr->extract_type = MR_EXTRACT_MESH;
+    mr->hide_unmapped_edges = false;
 
     if (is_paint_mode && mr->me) {
       mr->v_origindex = static_cast<const int *>(

source/blender/draw/intern/draw_cache_impl_subdivision.cc

@@ -654,6 +654,7 @@ static void draw_subdiv_free_edit_mode_cache(DRWSubdivCache *cache)
 {
   GPU_VERTBUF_DISCARD_SAFE(cache->verts_orig_index);
   GPU_VERTBUF_DISCARD_SAFE(cache->edges_orig_index);
+  GPU_VERTBUF_DISCARD_SAFE(cache->edges_draw_flag);
   GPU_VERTBUF_DISCARD_SAFE(cache->fdots_patch_coords);
 }
 
@@ -877,11 +878,13 @@ struct DRWCacheBuildingContext {
   int *subdiv_loop_vert_index;
   int *subdiv_loop_subdiv_vert_index;
   int *subdiv_loop_edge_index;
+  int *subdiv_loop_edge_draw_flag;
   int *subdiv_loop_subdiv_edge_index;
   int *subdiv_loop_poly_index;
 
   /* Temporary buffers used during traversal. */
   int *vert_origindex_map;
+  int *edge_draw_flag_map;
   int *edge_origindex_map;
 
   /* #CD_ORIGINDEX layers from the mesh to directly look up during traversal the original-index
@@ -941,6 +944,11 @@ static bool draw_subdiv_topology_info_cb(const SubdivForeachContext *foreach_con
       cache->edges_orig_index, get_origindex_format(), GPU_USAGE_DYNAMIC);
   GPU_vertbuf_data_alloc(cache->edges_orig_index, cache->num_subdiv_loops);
 
+  cache->edges_draw_flag = GPU_vertbuf_calloc();
+  GPU_vertbuf_init_with_format_ex(
+      cache->edges_draw_flag, get_origindex_format(), GPU_USAGE_DYNAMIC);
+  GPU_vertbuf_data_alloc(cache->edges_draw_flag, cache->num_subdiv_loops);
+
   cache->subdiv_loop_subdiv_vert_index = static_cast<int *>(
       MEM_mallocN(cache->num_subdiv_loops * sizeof(int), "subdiv_loop_subdiv_vert_index"));
 
@@ -954,6 +962,7 @@ static bool draw_subdiv_topology_info_cb(const SubdivForeachContext *foreach_con
   ctx->patch_coords = (CompressedPatchCoord *)GPU_vertbuf_get_data(cache->patch_coords);
   ctx->subdiv_loop_vert_index = (int *)GPU_vertbuf_get_data(cache->verts_orig_index);
   ctx->subdiv_loop_edge_index = (int *)GPU_vertbuf_get_data(cache->edges_orig_index);
+  ctx->subdiv_loop_edge_draw_flag = (int *)GPU_vertbuf_get_data(cache->edges_draw_flag);
   ctx->subdiv_loop_subdiv_vert_index = cache->subdiv_loop_subdiv_vert_index;
   ctx->subdiv_loop_subdiv_edge_index = cache->subdiv_loop_subdiv_edge_index;
   ctx->subdiv_loop_poly_index = cache->subdiv_loop_poly_index;
@@ -978,6 +987,8 @@ static bool draw_subdiv_topology_info_cb(const SubdivForeachContext *foreach_con
     for (int i = 0; i < num_edges; i++) {
       ctx->edge_origindex_map[i] = -1;
     }
+    ctx->edge_draw_flag_map = static_cast<int *>(
+        MEM_callocN(cache->num_subdiv_edges * sizeof(int), "subdiv_edge_draw_flag_map"));
   }
 
   return true;
@@ -1025,15 +1036,27 @@ static void draw_subdiv_edge_cb(const SubdivForeachContext *foreach_context,
     return;
   }
 
-  int coarse_index = coarse_edge_index;
-
-  if (coarse_index != -1) {
-    if (ctx->e_origindex) {
-      coarse_index = ctx->e_origindex[coarse_index];
+  if (coarse_edge_index == ORIGINDEX_NONE) {
+    /* Not mapped to edge in the subdivision base mesh. */
+    ctx->edge_origindex_map[subdiv_edge_index] = ORIGINDEX_NONE;
+    if (!ctx->cache->optimal_display) {
+      ctx->edge_draw_flag_map[subdiv_edge_index] = 1;
+    }
+  }
+  else {
+    if (ctx->e_origindex) {
+      const int origindex = ctx->e_origindex[coarse_edge_index];
+      ctx->edge_origindex_map[subdiv_edge_index] = origindex;
+      if (!(origindex == ORIGINDEX_NONE && ctx->cache->hide_unmapped_edges)) {
+        /* Not mapped to edge in original mesh (generated by a preceding modifier). */
+        ctx->edge_draw_flag_map[subdiv_edge_index] = 1;
+      }
+    }
+    else {
+      ctx->edge_origindex_map[subdiv_edge_index] = coarse_edge_index;
+      ctx->edge_draw_flag_map[subdiv_edge_index] = 1;
     }
   }
-
-  ctx->edge_origindex_map[subdiv_edge_index] = coarse_index;
 }
 
 static void draw_subdiv_loop_cb(const SubdivForeachContext *foreach_context,
@@ -1084,9 +1107,11 @@ static void do_subdiv_traversal(DRWCacheBuildingContext *cache_building_context,
    * subdiv-loop-to-coarse-edge map.
    */
   for (int i = 0; i < cache_building_context->cache->num_subdiv_loops; i++) {
+    const int edge_index = cache_building_context->subdiv_loop_subdiv_edge_index[i];
     cache_building_context->subdiv_loop_edge_index[i] =
-        cache_building_context
-            ->edge_origindex_map[cache_building_context->subdiv_loop_subdiv_edge_index[i]];
+        cache_building_context->edge_origindex_map[edge_index];
+    cache_building_context->subdiv_loop_edge_draw_flag[i] =
+        cache_building_context->edge_draw_flag_map[edge_index];
   }
 }
 
@@ -1246,6 +1271,7 @@ static bool draw_subdiv_build_cache(DRWSubdivCache *cache,
   /* Cleanup. */
   MEM_SAFE_FREE(cache_building_context.vert_origindex_map);
   MEM_SAFE_FREE(cache_building_context.edge_origindex_map);
+  MEM_SAFE_FREE(cache_building_context.edge_draw_flag_map);
 
   return true;
 }
@@ -1276,11 +1302,6 @@ struct DRWSubdivUboStorage {
   /* Refined topology information. */
   uint num_subdiv_loops;
 
-  /* Subdivision settings, is int in C but bool in the GLSL code, as there, bools have the same
-   * size as ints, so we should use int in C to ensure that the size of the structure is what GLSL
-   * expects. */
-  int optimal_display;
-
   /* The sculpt mask data layer may be null. */
   int has_sculpt_mask;
 
@@ -1299,6 +1320,7 @@ struct DRWSubdivUboStorage {
   int is_edit_mode;
   int use_hide;
   int _pad3;
+  int _pad4;
 };
 
 static_assert((sizeof(DRWSubdivUboStorage) % 16) == 0,
@@ -1318,7 +1340,6 @@ static void draw_subdiv_init_ubo_storage(const DRWSubdivCache *cache,
   ubo->max_depth = cache->gpu_patch_map.max_depth;
   ubo->patches_are_triangular = cache->gpu_patch_map.patches_are_triangular;
   ubo->coarse_poly_count = cache->num_coarse_poly;
-  ubo->optimal_display = cache->optimal_display;
   ubo->num_subdiv_loops = cache->num_subdiv_loops;
   ubo->edge_loose_offset = cache->num_subdiv_loops * 2;
   ubo->has_sculpt_mask = has_sculpt_mask;
@@ -1819,7 +1840,7 @@ void draw_subdiv_build_lines_buffer(const DRWSubdivCache *cache, GPUIndexBuf *li
 
   int binding_point = 0;
   GPU_vertbuf_bind_as_ssbo(cache->subdiv_polygon_offset_buffer, binding_point++);
-  GPU_vertbuf_bind_as_ssbo(cache->edges_orig_index, binding_point++);
+  GPU_vertbuf_bind_as_ssbo(cache->edges_draw_flag, binding_point++);
   GPU_vertbuf_bind_as_ssbo(cache->extra_coarse_face_data, binding_point++);
   GPU_indexbuf_bind_as_ssbo(lines_indices, binding_point++);
   BLI_assert(binding_point <= MAX_GPU_SUBDIV_SSBOS);
@@ -1855,7 +1876,7 @@ void draw_subdiv_build_lines_loose_buffer(const DRWSubdivCache *cache,
 
 void draw_subdiv_build_edge_fac_buffer(const DRWSubdivCache *cache,
                                        GPUVertBuf *pos_nor,
-                                       GPUVertBuf *edge_idx,
+                                       GPUVertBuf *edge_draw_flag,
                                        GPUVertBuf *edge_fac)
 {
   GPUShader *shader = get_subdiv_shader(SHADER_BUFFER_EDGE_FAC);
@@ -1863,7 +1884,7 @@ void draw_subdiv_build_edge_fac_buffer(const DRWSubdivCache *cache,
 
   int binding_point = 0;
   GPU_vertbuf_bind_as_ssbo(pos_nor, binding_point++);
-  GPU_vertbuf_bind_as_ssbo(edge_idx, binding_point++);
+  GPU_vertbuf_bind_as_ssbo(edge_draw_flag, binding_point++);
   GPU_vertbuf_bind_as_ssbo(edge_fac, binding_point++);
   BLI_assert(binding_point <= MAX_GPU_SUBDIV_SSBOS);
 
@@ -2113,17 +2134,18 @@ static bool draw_subdiv_create_requested_buffers(Object *ob,
   }
 
   DRWSubdivCache *draw_cache = mesh_batch_cache_ensure_subdiv_cache(batch_cache);
+
+  draw_cache->optimal_display = runtime_data->use_optimal_display;
+  /* If there is no distinct cage, hide unmapped edges that can't be selected. */
+  draw_cache->hide_unmapped_edges = is_editmode && !do_cage;
+  draw_cache->bm = bm;
+  draw_cache->mesh = mesh_eval;
+  draw_cache->subdiv = subdiv;
+
   if (!draw_subdiv_build_cache(draw_cache, subdiv, mesh_eval, runtime_data)) {
     return false;
   }
 
-  /* Edges which do not come from coarse edges should not be drawn in edit cage mode. */
-  const bool optimal_display = runtime_data->use_optimal_display || (is_editmode && !do_cage);
-
-  draw_cache->bm = bm;
-  draw_cache->mesh = mesh_eval;
-  draw_cache->subdiv = subdiv;
-  draw_cache->optimal_display = optimal_display;
   draw_cache->num_subdiv_triangles = tris_count_from_number_of_loops(draw_cache->num_subdiv_loops);
 
   /* Copy topology information for stats display. */

source/blender/draw/intern/draw_subdivision.h

@@ -101,6 +101,7 @@ typedef struct DRWSubdivCache {
   struct BMesh *bm;
   struct Subdiv *subdiv;
   bool optimal_display;
+  bool hide_unmapped_edges;
   bool use_custom_loop_normals;
 
   /* Coordinates used to evaluate patches for positions and normals. */
@@ -148,6 +149,8 @@ typedef struct DRWSubdivCache {
   struct GPUVertBuf *verts_orig_index;
   /* Maps subdivision loop to original coarse edge index, only really useful for edit mode. */
   struct GPUVertBuf *edges_orig_index;
+  /* Indicates if edge should be drawn in optimal display mode. */
+  struct GPUVertBuf *edges_draw_flag;
 
   /* Owned by #Subdiv. Indexed by coarse polygon index, difference between value (i + 1) and (i)
    * gives the number of ptex faces for coarse polygon (i). */
@@ -259,7 +262,7 @@ void draw_subdiv_extract_uvs(const DRWSubdivCache *cache,
 
 void draw_subdiv_build_edge_fac_buffer(const DRWSubdivCache *cache,
                                        struct GPUVertBuf *pos_nor,
-                                       struct GPUVertBuf *edge_idx,
+                                       struct GPUVertBuf *edge_draw_flag,
                                        struct GPUVertBuf *edge_fac);
 
 void draw_subdiv_build_tris_buffer(const DRWSubdivCache *cache,

source/blender/draw/intern/mesh_extractors/extract_mesh.hh

@@ -49,6 +49,7 @@ struct MeshRenderData {
   bool use_hide;
   bool use_subsurf_fdots;
   bool use_final_mesh;
+  bool hide_unmapped_edges;
 
   /** Use for #MeshStatVis calculation which use world-space coords. */
   float obmat[4][4];

source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_lines.cc

@@ -58,7 +58,7 @@ static void extract_lines_iter_poly_mesh(const MeshRenderData *mr,
   GPUIndexBufBuilder *elb = static_cast<GPUIndexBufBuilder *>(data);
   /* Using poly & loop iterator would complicate accessing the adjacent loop. */
   const MLoop *mloop = mr->mloop;
-  const int *e_origindex = (mr->edit_bmesh) ? mr->e_origindex : nullptr;
+  const int *e_origindex = (mr->hide_unmapped_edges) ? mr->e_origindex : nullptr;
   if (mr->use_hide || (e_origindex != nullptr)) {
     const int ml_index_last = mp->loopstart + (mp->totloop - 1);
     int ml_index = ml_index_last, ml_index_next = mp->loopstart;
@@ -109,7 +109,7 @@ static void extract_lines_iter_ledge_mesh(const MeshRenderData *mr,
   GPUIndexBufBuilder *elb = static_cast<GPUIndexBufBuilder *>(data);
   const int l_index_offset = mr->edge_len + ledge_index;
   const int e_index = mr->ledges[ledge_index];
-  const int *e_origindex = (mr->edit_bmesh) ? mr->e_origindex : nullptr;
+  const int *e_origindex = (mr->hide_unmapped_edges) ? mr->e_origindex : nullptr;
   if (!((mr->use_hide && mr->hide_edge && mr->hide_edge[med - mr->medge]) ||
         ((e_origindex) && (e_origindex[e_index] == ORIGINDEX_NONE)))) {
     const int l_index = mr->loop_len + ledge_index * 2;
@@ -183,8 +183,8 @@ static void extract_lines_loose_geom_subdiv(const DRWSubdivCache *subdiv_cache,
 
   switch (mr->extract_type) {
     case MR_EXTRACT_MESH: {
-      const int *e_origindex = (mr->edit_bmesh) ? mr->e_origindex : nullptr;
-      if (mr->e_origindex == nullptr) {
+      const int *e_origindex = (mr->hide_unmapped_edges) ? mr->e_origindex : nullptr;
+      if (e_origindex == nullptr) {
         const bool *hide_edge = mr->hide_edge;
         if (hide_edge) {
           for (DRWSubdivLooseEdge edge : loose_edges) {

source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_edge_fac.cc

@@ -223,23 +223,23 @@ static void extract_edge_fac_init_subdiv(const DRWSubdivCache *subdiv_cache,
 
   /* Create a temporary buffer for the edge original indices if it was not requested. */
   const bool has_edge_idx = edge_idx != nullptr;
-  GPUVertBuf *loop_edge_idx = nullptr;
+  GPUVertBuf *loop_edge_draw_flag = nullptr;
   if (has_edge_idx) {
-    loop_edge_idx = edge_idx;
+    loop_edge_draw_flag = edge_idx;
   }
   else {
-    loop_edge_idx = GPU_vertbuf_calloc();
+    loop_edge_draw_flag = GPU_vertbuf_calloc();
     draw_subdiv_init_origindex_buffer(
-        loop_edge_idx,
-        static_cast<int *>(GPU_vertbuf_get_data(subdiv_cache->edges_orig_index)),
+        loop_edge_draw_flag,
+        static_cast<int *>(GPU_vertbuf_get_data(subdiv_cache->edges_draw_flag)),
         subdiv_cache->num_subdiv_loops,
         0);
   }
 
-  draw_subdiv_build_edge_fac_buffer(subdiv_cache, pos_nor, loop_edge_idx, vbo);
+  draw_subdiv_build_edge_fac_buffer(subdiv_cache, pos_nor, loop_edge_draw_flag, vbo);
 
   if (!has_edge_idx) {
-    GPU_vertbuf_discard(loop_edge_idx);
+    GPU_vertbuf_discard(loop_edge_draw_flag);
   }
 }
 

source/blender/draw/intern/shaders/common_subdiv_ibo_lines_comp.glsl

@@ -1,9 +1,9 @@
 
 /* To be compiled with common_subdiv_lib.glsl */
 
-layout(std430, binding = 1) readonly buffer inputEdgeOrigIndex
+layout(std430, binding = 1) readonly buffer inputEdgeDrawFlag
 {
-  int input_origindex[];
+  int input_edge_draw_flag[];
 };
 
 layout(std430, binding = 2) readonly restrict buffer extraCoarseFaceData
@@ -35,8 +35,7 @@ void emit_line(uint line_offset, uint quad_index, uint start_loop_index, uint co
   uint coarse_quad_index = coarse_polygon_index_from_subdiv_quad_index(quad_index,
                                                                        coarse_poly_count);
 
-  if (use_hide && is_face_hidden(coarse_quad_index) ||
-      (input_origindex[vertex_index] == ORIGINDEX_NONE && optimal_display)) {
+  if (use_hide && is_face_hidden(coarse_quad_index) || (input_edge_draw_flag[vertex_index] == 0)) {
     output_lines[line_offset + 0] = 0xffffffff;
     output_lines[line_offset + 1] = 0xffffffff;
   }

source/blender/draw/intern/shaders/common_subdiv_lib.glsl

@@ -21,9 +21,6 @@ layout(std140) uniform shader_data
   /* Subdiv topology information. */
   uint num_subdiv_loops;
 
-  /* Subdivision settings. */
-  bool optimal_display;
-
   /* Sculpt data. */
   bool has_sculpt_mask;
 

source/blender/draw/intern/shaders/common_subdiv_vbo_edge_fac_comp.glsl

@@ -6,9 +6,9 @@ layout(std430, binding = 0) readonly buffer inputVertexData
   PosNorLoop pos_nor[];
 };
 
-layout(std430, binding = 1) readonly buffer inputEdgeIndex
+layout(std430, binding = 1) readonly buffer inputEdgeDrawFlag
 {
-  uint input_edge_index[];
+  uint input_edge_draw_flag[];
 };
 
 layout(std430, binding = 2) writeonly buffer outputEdgeFactors
@@ -51,9 +51,9 @@ float loop_edge_factor_get(vec3 f_no, vec3 v_co, vec3 v_no, vec3 v_next_co)
 float compute_line_factor(uint start_loop_index, uint corner_index, vec3 face_normal)
 {
   uint vertex_index = start_loop_index + corner_index;
-  uint edge_index = input_edge_index[vertex_index];
+  uint edge_draw_flag = input_edge_draw_flag[vertex_index];
 
-  if (edge_index == -1 && optimal_display) {
+  if (edge_draw_flag == 0) {
     return 0.0;
   }