fix for a bmesh glitch when making a face (Fkey).

On a place, Ctrl+T, Fkey would create a quad overlapping the 2 Tris.

Now this case is checked for in a general way - if the bounds of the face are already filled in with faces (that _only_ use these edges-verts), then dont create the face.

This is an option for the 'edgenet_fill' operator, since creating the face isnt incorrect, just not-what-you-want mostly.

added functions
* BM_edge_share_vert - returns shared vert between 2 edges.
* BM_face_exists_multi, BM_face_exists_multi_edge - check if existing faces fill the edge bounds.
* also add BM_ELEM_INTERNAL_TAG so low level functions can tag without conflicting with higher level functions that also rely on tagging elements.

source/blender/bmesh/bmesh.h

@@ -108,7 +108,10 @@ struct EditMesh;
  * only add new flags with care! - campbell */
 /* #define BM_ELEM_SPARE	 (1<<5) */
 /* #define BM_ELEM_SPARE	 (1<<6) */
-/* #define BM_ELEM_NONORMCALC (1<<7) */ /* UNUSED */
+
+#define BM_ELEM_INTERNAL_TAG (1<<7) /* for low level internal API tagging,
+                                     * since tools may want to tag verts and
+                                     * not have functions clobber them */
 
 /* Mesh Level Ops */
 extern int bm_mesh_allocsize_default[4];

source/blender/bmesh/bmesh_queries.h

@@ -97,6 +97,10 @@ float BM_vert_edge_angle(struct BMesh *bm, struct BMVert *v);
 /* checks overlapping of existing faces with the verts in varr. */
 int BM_face_exists_overlap(struct BMesh *bm, struct BMVert **varr, int len, struct BMFace **existface);
 
+/* checks if many existing faces overlap the faces defined by varr */
+int BM_face_exists_multi(BMesh *bm, struct BMVert **varr, struct BMEdge **earr, int len);
+int BM_face_exists_multi_edge(BMesh *bm, BMEdge **earr, int len);
+
 /* checks if a face defined by varr already exists. */
 int BM_face_exists(BMesh *bm, BMVert **varr, int len, BMFace **existface);
 
@@ -110,6 +114,8 @@ int BM_edge_share_face_count(struct BMEdge *e1, struct BMEdge *e2);
 /* returns bool 1/0 if the edges share a vertex */
 int BM_edge_share_vert_count(struct BMEdge *e1, struct BMEdge *e2);
 
+BMVert *BM_edge_share_vert(struct BMEdge *e1, struct BMEdge *e2);
+
 /* edge verts in winding order from face */
 void BM_edge_ordered_verts(struct BMEdge *edge, struct BMVert **r_v1, struct BMVert **r_v2);
 

source/blender/bmesh/intern/bmesh_opdefines.c

@@ -422,6 +422,7 @@ static BMOpDefine def_edgenet_fill = {
 	{{BMO_OP_SLOT_ELEMENT_BUF, "edges"}, /* input edge */
 	 {BMO_OP_SLOT_MAPPING,     "restrict"}, /* restricts edges to groups.  maps edges to integer */
 	 {BMO_OP_SLOT_BOOL,        "use_restrict"},
+	 {BMO_OP_SLOT_BOOL,        "use_fill_check"},
 	 {BMO_OP_SLOT_ELEMENT_BUF, "excludefaces"}, /* list of faces to ignore for manifold check */
 	 {BMO_OP_SLOT_MAPPING,     "faceout_groupmap"}, /* maps new faces to the group numbers they came fro */
 	 {BMO_OP_SLOT_ELEMENT_BUF, "faceout"}, /* new face */

source/blender/bmesh/intern/bmesh_queries.c

@@ -31,6 +31,9 @@
  * of inspecting the mesh structure directly.
  */
 
+#include "MEM_guardedalloc.h"
+
+#include "BLI_array.h"
 #include "BLI_math.h"
 
 #include "bmesh.h"
@@ -508,6 +511,27 @@ int BM_edge_share_vert_count(struct BMEdge *e1, struct BMEdge *e2)
 	        e1->v2 == e2->v2);
 }
 
+/**
+ *
+ *           BMESH EDGE SHARE A VERTEX
+ *
+ *	Return the shared vertex between the two edges or NULL
+ *
+ */
+
+BMVert *BM_edge_share_vert(struct BMEdge *e1, struct BMEdge *e2)
+{
+	if (BM_vert_in_edge(e2, e1->v1)) {
+		return e1->v1;
+	}
+	else if (BM_vert_in_edge(e2, e1->v2)) {
+		return e1->v2;
+	}
+	else {
+		return NULL;
+	}
+}
+
 /**
  *
  *           BMESH EDGE ORDERED VERTS
@@ -641,6 +665,163 @@ int BM_face_exists_overlap(BMesh *bm, BMVert **varr, int len, BMFace **overlapfa
 	return FALSE;
 }
 
+/*
+ * BMESH EXIST FACE MULTI
+ *
+ * Given a set of vertices and edges (varr, earr), find out if
+ * all those vertices are filled in by existing faces that _only_ use those vertices.
+ *
+ * This is for use in cases where creating a face is possible but would result in
+ * many overlapping faces.
+ *
+ * An example of how this is used: when 2 tri's are selected that share an edge,
+ * pressing Fkey would make a new overlapping quad (without a check like this)
+ *
+ * 'earr' and 'varr' can be in any order, however they _must_ form a closed loop.
+ *
+ * Returns:
+ * 0 for no overlap
+ * 1 for overlap
+ *
+ */
+
+int BM_face_exists_multi(BMesh *bm, BMVert **varr, BMEdge **earr, int len)
+{
+	BMFace *f;
+	BMEdge *e;
+	BMVert *v;
+	int ok;
+	int tot_tag;
+
+	BMIter fiter;
+	BMIter viter;
+
+	int i;
+
+	for (i = 0; i < len; i++) {
+		/* save some time by looping over edge faces rather then vert faces
+		 * will still loop over some faces twice but not as many */
+		BM_ITER(f, &fiter, bm, BM_FACES_OF_EDGE, earr[i]) {
+			BM_elem_flag_disable(f, BM_ELEM_INTERNAL_TAG);
+			BM_ITER(v, &viter, bm, BM_VERTS_OF_FACE, f) {
+				BM_elem_flag_disable(v, BM_ELEM_INTERNAL_TAG);
+			}
+		}
+
+		/* clear all edge tags */
+		BM_ITER(e, &fiter, bm, BM_EDGES_OF_VERT, varr[i]) {
+			BM_elem_flag_disable(e, BM_ELEM_INTERNAL_TAG);
+		}
+	}
+
+	/* now tag all verts and edges in the boundry array as true so
+	 * we can know if a face-vert is from our array */
+	for (i = 0; i < len; i++) {
+		BM_elem_flag_enable(varr[i], BM_ELEM_INTERNAL_TAG);
+		BM_elem_flag_enable(earr[i], BM_ELEM_INTERNAL_TAG);
+	}
+
+
+	/* so! boundry is tagged, everything else cleared */
+
+
+	/* 1) tag all faces connected to edges - if all their verts are boundry */
+	tot_tag = 0;
+	for (i = 0; i < len; i++) {
+		BM_ITER(f, &fiter, bm, BM_FACES_OF_EDGE, earr[i]) {
+			if (!BM_elem_flag_test(f, BM_ELEM_INTERNAL_TAG)) {
+				ok = TRUE;
+				BM_ITER(v, &viter, bm, BM_VERTS_OF_FACE, f) {
+					if (!BM_elem_flag_test(v, BM_ELEM_INTERNAL_TAG)) {
+						ok = FALSE;
+						break;
+					}
+				}
+
+				if (ok) {
+					/* we only use boundry verts */
+					BM_elem_flag_enable(f, BM_ELEM_INTERNAL_TAG);
+					tot_tag++;
+				}
+			}
+			else {
+				/* we already found! */
+			}
+		}
+	}
+
+	if (tot_tag == 0) {
+		/* no faces use only boundry verts, quit early */
+		return FALSE;
+	}
+
+	/* 2) loop over non-boundry edges that use boundry verts,
+	 *    check each have 2 tagges faces connected (faces that only use 'varr' verts) */
+	ok = TRUE;
+	for (i = 0; i < len; i++) {
+		BM_ITER(e, &fiter, bm, BM_EDGES_OF_VERT, varr[i]) {
+
+			if (/* non-boundry edge */
+			    BM_elem_flag_test(e, BM_ELEM_INTERNAL_TAG) == FALSE &&
+			    /* ...using boundry verts */
+			    BM_elem_flag_test(e->v1, BM_ELEM_INTERNAL_TAG) == TRUE &&
+			    BM_elem_flag_test(e->v2, BM_ELEM_INTERNAL_TAG) == TRUE)
+			{
+				int tot_face_tag = 0;
+				BM_ITER(f, &fiter, bm, BM_FACES_OF_EDGE, e) {
+					if (BM_elem_flag_test(f, BM_ELEM_INTERNAL_TAG)) {
+						tot_face_tag++;
+					}
+				}
+
+				if (tot_face_tag != 2) {
+					ok = FALSE;
+					break;
+				}
+
+			}
+		}
+
+		if (ok == FALSE) {
+			break;
+		}
+	}
+
+	return ok;
+}
+
+/* same as 'BM_face_exists_multi' but built vert array from edges */
+int BM_face_exists_multi_edge(BMesh *bm, BMEdge **earr, int len)
+{
+	BMVert **varr;
+	BLI_array_fixedstack_declare(varr, BM_NGON_STACK_SIZE, len, __func__);
+
+	int ok;
+	int i, i_next;
+
+	/* first check if verts have edges, if not we can bail out early */
+	ok = TRUE;
+	for (i = len - 1, i_next = 0; i_next < len; (i=i_next++)) {
+		if (!(varr[i] = BM_edge_share_vert(earr[i], earr[i_next]))) {
+			ok = FALSE;
+			break;
+		}
+	}
+
+	if (ok == FALSE) {
+		BMESH_ASSERT(0);
+		BLI_array_fixedstack_free(varr);
+		return FALSE;
+	}
+
+	ok = BM_face_exists_multi(bm, varr, earr, len);
+
+	BLI_array_fixedstack_free(varr);
+
+	return ok;
+}
+
+
 /*
  * BMESH FACE EXISTS
  *

source/blender/bmesh/operators/bmo_create.c

@@ -872,7 +872,8 @@ void bmesh_edgenet_fill_exec(BMesh *bm, BMOperator *op)
 	BMEdge **edges = NULL;
 	PathBase *pathbase = edge_pathbase_new();
 	BLI_array_declare(edges);
-	int use_restrict = BMO_slot_bool_get(op, "use_restrict");
+	int use_restrict   = BMO_slot_bool_get(op, "use_restrict");
+	int use_fill_check = BMO_slot_bool_get(op, "use_fill_check");
 	int i, j, group = 0;
 	unsigned int winding[2]; /* accumulte winding directions for each edge which has a face */
 
@@ -1014,13 +1015,19 @@ void bmesh_edgenet_fill_exec(BMesh *bm, BMOperator *op)
 				v2 = verts[0];
 			}
 
-			f = BM_face_create_ngon(bm, v1, v2, edges, i, TRUE);
-			if (f && !BMO_elem_flag_test(bm, f, ELE_ORIG)) {
-				BMO_elem_flag_enable(bm, f, FACE_NEW);
-			}
+			if ((use_fill_check == FALSE) ||
+			    /* fairly expensive check - see if there are already faces filling this area */
+			    (BM_face_exists_multi_edge(bm, edges, i) == FALSE))
+			{
+				f = BM_face_create_ngon(bm, v1, v2, edges, i, TRUE);
+				if (f && !BMO_elem_flag_test(bm, f, ELE_ORIG)) {
+					BMO_elem_flag_enable(bm, f, FACE_NEW);
+				}
 
-			if (use_restrict)
-				BMO_slot_map_int_insert(bm, op, "faceout_groupmap", f, path->group);
+				if (use_restrict) {
+					BMO_slot_map_int_insert(bm, op, "faceout_groupmap", f, path->group);
+				}
+			}
 		}
 		
 		edge_free_path(pathbase, path);
@@ -1336,7 +1343,7 @@ void bmesh_contextual_create_exec(BMesh *bm, BMOperator *op)
 	BMO_slot_buffer_flag_enable(bm, &op2, "edgeout", ELE_NEW, BM_EDGE);
 	BMO_op_finish(bm, &op2);
 
-	BMO_op_initf(bm, &op2, "edgenet_fill edges=%fe", ELE_NEW);
+	BMO_op_initf(bm, &op2, "edgenet_fill edges=%fe use_fill_check=%b", ELE_NEW, TRUE);
 	BMO_op_exec(bm, &op2);
 
 	/* return if edge net create did somethin */

source/blender/python/bmesh/bmesh_py_types.c

@@ -1274,7 +1274,7 @@ static PyObject *bpy_bmfaceseq_new(BPy_BMElemSeq *self, PyObject *args)
 		edge_array = (BMEdge **)PyMem_MALLOC(vert_seq_len * sizeof(BMEdge **));
 
 		/* ensure edges */
-		for (i_next = 0, i = vert_seq_len - 1; i_next < vert_seq_len; (i=i_next++)) {
+		for (i = vert_seq_len - 1, i_next = 0; i_next < vert_seq_len; (i=i_next++)) {
 			edge_array[i] = BM_edge_create(bm, vert_array[i], vert_array[i_next], NULL, TRUE);
 		}
 
@@ -2535,7 +2535,7 @@ void *BPy_BMElem_PySeq_As_Array(BMesh **r_bm, PyObject *seq, Py_ssize_t min, Py_
 			alloc[i] = item->ele;
 
 			if (do_unique_check) {
-				BM_elem_flag_enable(item->ele, BM_ELEM_TAG);
+				BM_elem_flag_enable(item->ele, BM_ELEM_INTERNAL_TAG);
 			}
 		}
 
@@ -2543,12 +2543,12 @@ void *BPy_BMElem_PySeq_As_Array(BMesh **r_bm, PyObject *seq, Py_ssize_t min, Py_
 			/* check for double verts! */
 			int ok = TRUE;
 			for (i = 0; i < seq_len; i++) {
-				if (UNLIKELY(BM_elem_flag_test(alloc[i], BM_ELEM_TAG) == FALSE)) {
+				if (UNLIKELY(BM_elem_flag_test(alloc[i], BM_ELEM_INTERNAL_TAG) == FALSE)) {
 					ok = FALSE;
 				}
 
 				/* ensure we dont leave this enabled */
-				BM_elem_flag_disable(alloc[i], BM_ELEM_TAG);
+				BM_elem_flag_disable(alloc[i], BM_ELEM_INTERNAL_TAG);
 			}
 
 			if (ok == FALSE) {