fix for mesh face filling when a valid edge-loop was selected but unselected connecting geometry existed inside the loop.

In this case edgenet_fill operator failed and it would fallback to filling as unordered vertices which was mostly fine but failed on some concave loops.

Add a new bmesh operator 'edgeloop_fill' fills in closed loops even if they don't make a valid edge-net.

source/blender/bmesh/CMakeLists.txt

@@ -45,6 +45,7 @@ set(SRC
 	operators/bmo_create.c
 	operators/bmo_dissolve.c
 	operators/bmo_dupe.c
+	operators/bmo_edgeloop_fill.c
 	operators/bmo_edgenet.c
 	operators/bmo_edgesplit.c
 	operators/bmo_extrude.c

source/blender/bmesh/intern/bmesh_opdefines.c

@@ -526,6 +526,30 @@ static BMOpDefine bmo_bridge_loops_def = {
 	0,
 };
 
+/*
+ * Edge Loop Fill.
+ *
+ * Create faces defined by one or more non overlapping edge loops.
+ */
+static BMOpDefine bmo_edgeloop_fill_def = {
+	"edgeloop_fill",
+	/* slots_in */
+	{{"edges", BMO_OP_SLOT_ELEMENT_BUF, {BM_EDGE}}, /* input edges */
+	/* restricts edges to groups.  maps edges to integer */
+	 {"mat_nr",         BMO_OP_SLOT_INT},      /* material to use */
+	 {"use_smooth",        BMO_OP_SLOT_BOOL},  /* smooth state to use */
+	 {{'\0'}},
+	},
+	/* slots_out */
+	/* maps new faces to the group numbers they came from */
+	{{"faces.out", BMO_OP_SLOT_ELEMENT_BUF, {BM_FACE}},     /* new faces */
+	 {{'\0'}},
+	},
+	bmo_edgeloop_fill_exec,
+	0,
+};
+
+
 /*
  * Edge Net Fill.
  *
@@ -541,7 +565,7 @@ static BMOpDefine bmo_edgenet_fill_def = {
 	 {"use_fill_check",        BMO_OP_SLOT_BOOL},
 	 {"exclude_faces", BMO_OP_SLOT_ELEMENT_BUF, {BM_FACE}}, /* list of faces to ignore for manifold check */
 	 {"mat_nr",         BMO_OP_SLOT_INT},      /* material to use */
-	 {"use_smooth",        BMO_OP_SLOT_BOOL},  /* material to use */
+	 {"use_smooth",        BMO_OP_SLOT_BOOL},  /* smooth state to use */
 	 {{'\0'}},
 	},
 	/* slots_out */
@@ -1615,6 +1639,7 @@ const BMOpDefine *bmo_opdefines[] = {
 	&bmo_dissolve_limit_def,
 	&bmo_dissolve_verts_def,
 	&bmo_duplicate_def,
+	&bmo_edgeloop_fill_def,
 	&bmo_edgenet_fill_def,
 	&bmo_edgenet_prepare_def,
 	&bmo_extrude_discrete_faces_def,

source/blender/bmesh/intern/bmesh_operators_private.h

@@ -57,6 +57,7 @@ void bmo_dissolve_faces_exec(BMesh *bm, BMOperator *op);
 void bmo_dissolve_limit_exec(BMesh *bm, BMOperator *op);
 void bmo_dissolve_verts_exec(BMesh *bm, BMOperator *op);
 void bmo_duplicate_exec(BMesh *bm, BMOperator *op);
+void bmo_edgeloop_fill_exec(BMesh *bm, BMOperator *op);
 void bmo_edgenet_fill_exec(BMesh *bm, BMOperator *op);
 void bmo_edgenet_prepare_exec(BMesh *bm, BMOperator *op);
 void bmo_extrude_discrete_faces_exec(BMesh *bm, BMOperator *op);

source/blender/bmesh/operators/bmo_create.c

@@ -132,7 +132,10 @@ void bmo_contextual_create_exec(BMesh *bm, BMOperator *op)
 	}
 	/* --- end special case support, continue as normal --- */
 
-	/* call edgenet create */
+
+	/* -------------------------------------------------------------------- */
+	/* EdgeNet Create */
+
 	/* call edgenet prepare op so additional face creation cases wore */
 	BMO_op_initf(bm, &op2, op->flag, "edgenet_prepare edges=%fe", ELE_NEW);
 	BMO_op_exec(bm, &op2);
@@ -154,8 +157,10 @@ void bmo_contextual_create_exec(BMesh *bm, BMOperator *op)
 	}
 
 	BMO_op_finish(bm, &op2);
-	
-	/* now call dissolve face */
+
+
+	/* -------------------------------------------------------------------- */
+	/* Dissolve Face */
 	BMO_op_initf(bm, &op2, op->flag, "dissolve_faces faces=%ff", ELE_NEW);
 	BMO_op_exec(bm, &op2);
 	
@@ -169,6 +174,32 @@ void bmo_contextual_create_exec(BMesh *bm, BMOperator *op)
 
 	BMO_op_finish(bm, &op2);
 
+
+	/* -------------------------------------------------------------------- */
+	/* Fill EdgeLoop's - fills isolated loops, different from edgenet */
+
+	/* note: in most cases 'edgenet_fill' will handle this case since in common cases
+	 * users fill in empty spaces, however its possible to have an edge selection around
+	 * existing geometry that makes 'edgenet_fill' fail. */
+	BMO_op_initf(bm, &op2, op->flag, "edgeloop_fill edges=%fe", ELE_NEW);
+	BMO_op_exec(bm, &op2);
+
+	/* return if edge loop fill did something */
+	if (BMO_slot_buffer_count(op2.slots_out, "faces.out")) {
+		BMO_slot_copy(&op2, slots_out, "faces.out",
+		              op,   slots_out, "faces.out");
+		BMO_op_finish(bm, &op2);
+		return;
+	}
+
+	BMO_op_finish(bm, &op2);
+
+
+
+	/* -------------------------------------------------------------------- */
+	/* Continue with ad-hoc fill methods since operators fail,
+	 * edge, vcloud... may add more */
+
 	/* now, count how many verts we have */
 	amount = 0;
 	BM_ITER_MESH (v, &iter, bm, BM_VERTS_OF_MESH) {
@@ -237,8 +268,9 @@ void bmo_contextual_create_exec(BMesh *bm, BMOperator *op)
 		/* done creating edges */
 	}
 	else if (amount > 2) {
-		/* TODO, all these verts may be connected by edges.
-		 * we should check on this before assuming they are a random set of verts */
+		/* TODO, some of these vertes may be connected by edges,
+		 * this connectivity could be used rather then treating
+		 * them as a bunch of isolated verts. */
 
 		BMVert **vert_arr = MEM_mallocN(sizeof(BMVert **) * totv, __func__);
 		int i = 0;

source/blender/bmesh/operators/bmo_edgeloop_fill.c

@@ -0,0 +1,157 @@
+/*
+ * ***** BEGIN GPL LICENSE BLOCK *****
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * Contributor(s): Campbell Barton.
+ *
+ * ***** END GPL LICENSE BLOCK *****
+ */
+
+/** \file blender/bmesh/operators/bmo_edgeloop_fill.c
+ *  \ingroup bmesh
+ */
+
+#include "MEM_guardedalloc.h"
+
+#include "BLI_listbase.h"
+
+#include "bmesh.h"
+
+#include "intern/bmesh_operators_private.h" /* own include */
+
+#define VERT_USED	1
+#define EDGE_MARK	2
+#define ELE_OUT		4
+
+void bmo_edgeloop_fill_exec(BMesh *bm, BMOperator *op)
+{
+	/* first collect an array of unique from the edges */
+	const int tote = BMO_slot_buffer_count(op->slots_in, "edges");
+	const int totv = tote;  /* these should be the same */
+	BMVert **verts = MEM_mallocN(sizeof(*verts) * totv, __func__);
+
+	BMVert *v;
+	BMEdge *e;
+	int i;
+	bool ok = true;
+
+	BMOIter oiter;
+
+	const short mat_nr     = BMO_slot_int_get(op->slots_in,  "mat_nr");
+	const bool use_smooth  = BMO_slot_bool_get(op->slots_in, "use_smooth");
+
+	/* 'VERT_USED' will be disabled, so enable and fill the array */
+	BMO_ITER (e, &oiter, op->slots_in, "edges", BM_EDGE) {
+		BMIter viter;
+		BMO_elem_flag_enable(bm, e, EDGE_MARK);
+		BM_ITER_ELEM (v, &viter, e, BM_VERTS_OF_EDGE) {
+			if (BMO_elem_flag_test(bm, v, VERT_USED) == false) {
+				BMO_elem_flag_enable(bm, v, VERT_USED);
+				verts[i++] = v;
+				if (i > tote) {
+					break;
+				}
+			}
+		}
+		if (i > tote) {
+			break;
+		}
+	}
+
+	/* we have a different number of verts to edges */
+	if (i != tote) {
+		MEM_freeN(verts);
+		return;
+	}
+
+	/* loop over connected flagged edges and fill in faces,  this is made slightly more
+	 * complicated because there may be multiple disconnected loops to fill. */
+
+	/* sanity check - that each vertex has 2 edge users */
+	for (i = 0; i < totv; i++) {
+		v = verts[i];
+		/* count how many flagged edges this vertex uses */
+		if (BMO_vert_edge_flags_count(bm, v, EDGE_MARK) != 2) {
+			ok = false;
+			break;
+		}
+	}
+
+	if (ok) {
+		/* note: in the case of multiple loops, this over-allocs (which is fine) */
+		BMVert **f_verts  = MEM_mallocN(sizeof(*verts) * totv, __func__);
+		BMIter eiter;
+
+		/* build array of connected verts and edges */
+		BMEdge *e_prev = NULL;
+		BMEdge *e_next = NULL;
+		int totv_used = 0;
+
+		while (totv_used < totv) {
+			for (i = 0; i < totv; i++) {
+				v = verts[i];
+				if (BMO_elem_flag_test(bm, v, VERT_USED)) {
+					break;
+				}
+			}
+
+			/* this should never fail, as long as (totv_used < totv)
+			 * we should have marked verts available */
+			BLI_assert(BMO_elem_flag_test(bm, v, VERT_USED));
+
+			/* watch it, 'i' is used for final face length */
+			i = 0;
+			do {
+				/* we know that there are 2 edges per vertex so no need to check */
+				BM_ITER_ELEM (e, &eiter, v, BM_EDGES_OF_VERT) {
+					if (BMO_elem_flag_test(bm, e, EDGE_MARK)) {
+						if (e != e_prev) {
+							e_next = e;
+							break;
+						}
+					}
+				}
+
+				/* fill in the array */
+				f_verts[i] = v;
+				BMO_elem_flag_disable(bm, v, VERT_USED);
+				totv_used++;
+
+				/* step over the edges */
+				v = BM_edge_other_vert(e_next, v);
+				e_prev = e_next;
+				i++;
+			} while ((v != f_verts[0]));
+
+			if (BM_face_exists(f_verts, i, NULL) == false) {
+				BMFace *f;
+
+				/* don't use calc_edges option because we already have the edges */
+				f = BM_face_create_ngon_verts(bm, f_verts, i, 0, true, false);
+				BMO_elem_flag_enable(bm, f, ELE_OUT);
+				f->mat_nr = mat_nr;
+				if (use_smooth) {
+					BM_elem_flag_enable(f, BM_ELEM_SMOOTH);
+				}
+			}
+		}
+		MEM_freeN(f_verts);
+
+		BMO_slot_buffer_from_enabled_flag(bm, op, op->slots_out, "faces.out", BM_FACE, ELE_OUT);
+	}
+
+	MEM_freeN(verts);
+}