Fix #146836: Auto merge not working in edit mode

Regression in [0] removed checks for indices referencing themselves
which need to be kept but can still be used as targets.

Restore this logic as well as fixing another problem (#147022)
where auto-merge would not merge into the nearest vertex, this
was especially noticeable then the threshold was set to a large value
but would happen at smaller values too.

[0]: bdae3e28a2

source/blender/blenlib/BLI_kdtree_impl.h

@@ -107,6 +107,9 @@ int BLI_kdtree_nd_(calc_duplicates_fast)(const KDTree *tree,
  * which other indices are merged into.
  *
  * \param tree: A tree, all indices *must* be unique.
+ * \param has_self_index: When true, account for indices
+ * in the `duplicates` array that reference themselves,
+ * prioritizing them as targets before de-duplicating the remainder with each other.
  * \param deduplicate_cb: A function which receives duplicate indices,
  * it must choose the "target" index to keep which is returned.
  * The return value is an index in the `cluster` array (a value from `0..cluster_num`).
@@ -120,6 +123,7 @@ int BLI_kdtree_nd_(calc_duplicates_fast)(const KDTree *tree,
 int BLI_kdtree_nd_(calc_duplicates_cb)(const KDTree *tree,
                                        const float range,
                                        int *duplicates,
+                                       bool has_self_index,
                                        int (*deduplicate_cb)(void *user_data,
                                                              const int *cluster,
                                                              int cluster_num),
@@ -199,12 +203,14 @@ template<typename Fn>
 inline int BLI_kdtree_nd_(calc_duplicates_cb_cpp)(const KDTree *tree,
                                                   const float distance,
                                                   int *duplicates,
+                                                  const bool has_self_index,
                                                   const Fn &fn)
 {
   return BLI_kdtree_nd_(calc_duplicates_cb)(
       tree,
       distance,
       duplicates,
+      has_self_index,
       [](void *user_data, const int *cluster, int cluster_num) -> int {
         const Fn &fn = *static_cast<const Fn *>(user_data);
         return fn(cluster, cluster_num);

source/blender/blenlib/intern/kdtree_impl.h

@@ -9,7 +9,6 @@
 #include "MEM_guardedalloc.h"
 
 #include "BLI_array.hh"
-#include "BLI_bit_vector.hh"
 #include "BLI_kdtree_impl.h"
 #include "BLI_math_base.h"
 #include "BLI_utildefines.h"
@@ -899,6 +898,7 @@ int BLI_kdtree_nd_(calc_duplicates_fast)(const KDTree *tree,
 int BLI_kdtree_nd_(calc_duplicates_cb)(const KDTree *tree,
                                        const float range,
                                        int *duplicates,
+                                       const bool has_self_index,
                                        int (*duplicates_cb)(void *user_data,
                                                             const int *cluster,
                                                             int cluster_num),
@@ -916,26 +916,59 @@ int BLI_kdtree_nd_(calc_duplicates_cb)(const KDTree *tree,
     index_to_node_index[tree->nodes[i].index] = int(i);
   }
 
-  blender::BitVector<> visited(tree->max_node_index + 1, false);
+  int found = 0;
+
+  /* First pass, handle merging into self-index (if any exist). */
+  if (has_self_index) {
+    blender::Array<float> duplicates_dist_sq(tree->max_node_index + 1);
+    for (uint i = 0; i < nodes_len; i++) {
+      const int node_index = tree->nodes[i].index;
+      if (node_index != duplicates[node_index]) {
+        continue;
+      }
+      const float *search_co = tree->nodes[index_to_node_index[node_index]].co;
+      auto accumulate_neighbors_fn =
+          [&duplicates, &node_index, &duplicates_dist_sq, &found](
+              int neighbor_index, const float * /*co*/, const float dist_sq) -> bool {
+        const int target_index = duplicates[neighbor_index];
+        if (target_index == -1) {
+          duplicates[neighbor_index] = node_index;
+          duplicates_dist_sq[neighbor_index] = dist_sq;
+          found += 1;
+        }
+        /* Don't steal from self references. */
+        else if (target_index != neighbor_index) {
+          float &dist_sq_best = duplicates_dist_sq[neighbor_index];
+          /* Steal the target if it's closer. */
+          if (dist_sq < dist_sq_best) {
+            dist_sq_best = dist_sq;
+            duplicates[neighbor_index] = node_index;
+          }
+        }
+        return true;
+      };
+
+      BLI_kdtree_nd_(range_search_cb_cpp)(tree, search_co, range, accumulate_neighbors_fn);
+    }
+  }
+
+  /* Second pass, de-duplicate clusters that weren't handled in the first pass. */
+
   /* Could be inline, declare here to avoid re-allocation. */
   blender::Vector<int> cluster;
-
-  int found = 0;
   for (uint i = 0; i < nodes_len; i++) {
     const int node_index = tree->nodes[i].index;
-    if ((duplicates[node_index] != -1) || visited[node_index]) {
+    if (duplicates[node_index] != -1) {
       continue;
     }
 
     BLI_assert(cluster.is_empty());
     const float *search_co = tree->nodes[index_to_node_index[node_index]].co;
-    visited[node_index].set();
-    auto accumulate_neighbors_fn = [&duplicates, &visited, &cluster](int neighbor_index,
-                                                                     const float * /*co*/,
-                                                                     float /*dist_sq*/) -> bool {
-      if ((duplicates[neighbor_index] == -1) && !visited[neighbor_index]) {
+    auto accumulate_neighbors_fn = [&duplicates, &cluster](int neighbor_index,
+                                                           const float * /*co*/,
+                                                           const float /*dist_sq*/) -> bool {
+      if (duplicates[neighbor_index] == -1) {
         cluster.append(neighbor_index);
-        visited[neighbor_index].set();
       }
       return true;
     };

source/blender/bmesh/operators/bmo_removedoubles.cc

@@ -677,12 +677,14 @@ static int *bmesh_find_doubles_by_distance_impl(BMesh *bm,
 {
   int *duplicates = MEM_malloc_arrayN<int>(verts_len, __func__);
   bool found_duplicates = false;
+  bool has_self_index = false;
 
   KDTree_3d *tree = BLI_kdtree_3d_new(verts_len);
   for (int i = 0; i < verts_len; i++) {
     BLI_kdtree_3d_insert(tree, i, verts[i]->co);
     if (has_keep_vert && BMO_vert_flag_test(bm, verts[i], VERT_KEEP)) {
       duplicates[i] = i;
+      has_self_index = true;
     }
     else {
       duplicates[i] = -1;
@@ -730,7 +732,7 @@ static int *bmesh_find_doubles_by_distance_impl(BMesh *bm,
   };
 
   found_duplicates = BLI_kdtree_3d_calc_duplicates_cb_cpp(
-                         tree, dist, duplicates, deduplicate_target_calc_fn) != 0;
+                         tree, dist, duplicates, has_self_index, deduplicate_target_calc_fn) != 0;
 
   BLI_kdtree_3d_free(tree);