Fix T39196, Dynamic Topology Undo Applied to Wrong Mesh

Undoing nodes that do not belong to the current object will cause the
saved bmesh log entry to be reverted instead. This entry can belong to
another object though.
This is easy to fix by enforcing name matching (this was borrowed by
edit mode but can definitely be improved) between current object name
and undo node name and deleting older entries.

However there are complications. Deleting dyntopo entries in this way
can leave a brush stroke as first dyntopo log entry. This can present
issues if we attempt to delete that entry since it's deleted mesh
elements may now have had their ids (which would still be valid at the
time) cleaned up. This can result in crashing if we attempt to resculpt
on the mesh. To fix this I have disabled releasing the deleted entries.

This entanglement between bm_log and undo is quite volatile but I hope
the system works better now.

Also minor cleanup, fix unneeded check warning

source/blender/blenlib/intern/boxpack2d.c

@@ -91,7 +91,7 @@ typedef struct BoxVert {
 
 BLI_INLINE int quad_flag(unsigned int q)
 {
-	BLI_assert(q < 4 && q >= 0);
+	BLI_assert(q < 4);
 	return (1 << q);
 }
 

source/blender/bmesh/intern/bmesh_log.c

@@ -489,6 +489,29 @@ BMLog *BM_log_create(BMesh *bm)
 	return log;
 }
 
+void BM_log_cleanup_entry(BMLogEntry *entry)
+{
+	BMLog *log = entry->log;
+
+	if (log) {
+		/* Take all used IDs */
+		bm_log_id_ghash_retake(log->unused_ids, entry->deleted_verts);
+		bm_log_id_ghash_retake(log->unused_ids, entry->deleted_faces);
+		bm_log_id_ghash_retake(log->unused_ids, entry->added_verts);
+		bm_log_id_ghash_retake(log->unused_ids, entry->added_faces);
+		bm_log_id_ghash_retake(log->unused_ids, entry->modified_verts);
+		bm_log_id_ghash_retake(log->unused_ids, entry->modified_faces);
+
+		/* delete entries to avoid releasing ids in node cleanup */
+		BLI_ghash_clear(entry->deleted_verts, NULL, NULL);
+		BLI_ghash_clear(entry->deleted_faces, NULL, NULL);
+		BLI_ghash_clear(entry->added_verts, NULL, NULL);
+		BLI_ghash_clear(entry->added_faces, NULL, NULL);
+		BLI_ghash_clear(entry->modified_verts, NULL, NULL);
+	}
+}
+
+
 /* Allocate and initialize a new BMLog using existing BMLogEntries
  *
  * The 'entry' should be the last entry in the BMLog. Its prev pointer
@@ -684,8 +707,27 @@ void BM_log_entry_drop(BMLogEntry *entry)
 		 * entry. Since the entry is at the beginning of the undo
 		 * stack, and it's being deleted, those elements can never be
 		 * restored. Their IDs can go back into the pool. */
-		bm_log_id_ghash_release(log, entry->deleted_faces);
+
-		bm_log_id_ghash_release(log, entry->deleted_verts);
+		/* This would never happen usually since first entry of log is
+		 * usually dyntopo enable, which, when reverted will free the log
+		 * completely. However, it is possible have a stroke instead of
+		 * dyntopo enable as first entry if nodes have been cleaned up
+		 * after sculpting on a different object than A, B.
+		 *
+		 * The steps are:
+		 * A dyntopo enable - sculpt
+		 * B dyntopo enable - sculpt - undo (A objects operators get cleaned up)
+		 * A sculpt (now A's log has a sculpt operator as first entry)
+		 *
+		 * Causing a cleanup at this point will call the code below, however
+		 * this will invalidate the state of the log since the deleted vertices
+		 * have been reclaimed already on step 2 (see BM_log_cleanup_entry)
+		 *
+		 * Also, design wise, a first entry should not have any deleted vertices since it
+		 * should not have anything to delete them -from-
+		 */
+		//bm_log_id_ghash_release(log, entry->deleted_faces);
+		//bm_log_id_ghash_release(log, entry->deleted_verts);
 	}
 	else if (!entry->next) {
 		/* Release IDs of elements that are added by this entry. Since

source/blender/bmesh/intern/bmesh_log.h

@@ -51,6 +51,9 @@ void BM_log_mesh_elems_reorder(BMesh *bm, BMLog *log);
 /* Start a new log entry and update the log entry list */
 BMLogEntry *BM_log_entry_add(BMLog *log);
 
+/* Mark all used ids as unused for this node */
+void BM_log_cleanup_entry(BMLogEntry *entry);
+
 /* Remove an entry from the log */
 void BM_log_entry_drop(BMLogEntry *entry);
 

source/blender/editors/include/ED_sculpt.h

@@ -61,7 +61,7 @@ typedef void (*UndoFreeCb)(struct ListBase *lb);
 
 int ED_undo_paint_step(struct bContext *C, int type, int step, const char *name);
 void ED_undo_paint_step_num(struct bContext *C, int type, int num);
-const char *ED_undo_paint_get_name(int type, int nr, int *active);
+const char *ED_undo_paint_get_name(struct bContext *C, int type, int nr, int *active);
 void ED_undo_paint_free(void);
 int ED_undo_paint_valid(int type, const char *name);
 bool ED_undo_paint_empty(int type);

source/blender/editors/sculpt_paint/paint_intern.h

@@ -229,6 +229,7 @@ typedef enum BrushStrokeMode {
 /* paint_undo.c */
 struct ListBase *undo_paint_push_get_list(int type);
 void undo_paint_push_count_alloc(int type, int size);
+bool sculpt_undo_cleanup(struct bContext *C, struct ListBase *lb);
 
 /* paint_hide.c */
 

source/blender/editors/sculpt_paint/paint_undo.c

@@ -53,14 +53,17 @@ typedef struct UndoElem {
 	UndoFreeCb free;
 } UndoElem;
 
+typedef bool (*UndoCleanupCb)(struct bContext *C, ListBase *lb);
+
 typedef struct UndoStack {
 	int type;
 	ListBase elems;
 	UndoElem *current;
+	UndoCleanupCb cleanup;
 } UndoStack;
 
-static UndoStack ImageUndoStack = {UNDO_PAINT_IMAGE, {NULL, NULL}, NULL};
+static UndoStack ImageUndoStack = {UNDO_PAINT_IMAGE, {NULL, NULL}, NULL, NULL};
-static UndoStack MeshUndoStack = {UNDO_PAINT_MESH, {NULL, NULL}, NULL};
+static UndoStack MeshUndoStack = {UNDO_PAINT_MESH, {NULL, NULL}, NULL, sculpt_undo_cleanup};
 
 /* Generic */
 
@@ -171,10 +174,39 @@ static void undo_stack_push_end(UndoStack *stack)
 	}
 }
 
+static void undo_stack_cleanup(UndoStack *stack, bContext *C)
+{
+	UndoElem *uel = stack->elems.first;
+	bool stack_reset = false;
+
+	if (stack->cleanup) {
+		while (uel) {
+			if (stack->cleanup(C, &uel->elems)) {
+				UndoElem *uel_tmp = uel->next;
+				if (stack->current == uel) {
+					stack->current = NULL;
+					stack_reset = true;
+				}
+				undo_elem_free(stack, uel);
+				BLI_freelinkN(&stack->elems, uel);
+				uel = uel_tmp;
+			}
+			else
+				uel = uel->next;
+		}
+		if (stack_reset) {
+			stack->current = stack->elems.last;
+		}
+	}
+}
+
 static int undo_stack_step(bContext *C, UndoStack *stack, int step, const char *name)
 {
 	UndoElem *undo;
 
+	/* first cleanup any old undo steps that may belong to invalid data */
+	undo_stack_cleanup(stack, C);
+
 	if (step == 1) {
 		if (stack->current == NULL) {
 			/* pass */
@@ -315,12 +347,17 @@ static char *undo_stack_get_name(UndoStack *stack, int nr, int *active)
 	return NULL;
 }
 
-const char *ED_undo_paint_get_name(int type, int nr, int *active)
+const char *ED_undo_paint_get_name(bContext *C, int type, int nr, int *active)
 {
-	if (type == UNDO_PAINT_IMAGE)
+
+	if (type == UNDO_PAINT_IMAGE) {
+		undo_stack_cleanup(&ImageUndoStack, C);
 		return undo_stack_get_name(&ImageUndoStack, nr, active);
-	else if (type == UNDO_PAINT_MESH)
+	}
+	else if (type == UNDO_PAINT_MESH) {
+		undo_stack_cleanup(&MeshUndoStack, C);
 		return undo_stack_get_name(&MeshUndoStack, nr, active);
+	}
 	return NULL;
 }
 

source/blender/editors/sculpt_paint/sculpt_undo.c

@@ -526,9 +526,11 @@ static void sculpt_undo_free(ListBase *lb)
 		}
 		if (unode->mask)
 			MEM_freeN(unode->mask);
+
 		if (unode->bm_entry) {
 			BM_log_entry_drop(unode->bm_entry);
 		}
+
 		if (unode->bm_enter_totvert)
 			CustomData_free(&unode->bm_enter_vdata, unode->bm_enter_totvert);
 		if (unode->bm_enter_totedge)
@@ -540,6 +542,24 @@ static void sculpt_undo_free(ListBase *lb)
 	}
 }
 
+bool sculpt_undo_cleanup(bContext *C, ListBase *lb) {
+	Object *ob = CTX_data_active_object(C);
+	SculptUndoNode *unode;
+
+	unode = lb->first;
+
+	if (strcmp(unode->idname, ob->id.name) != 0) {
+		for (unode = lb->first; unode; unode = unode->next) {
+			if (unode->bm_entry)
+				BM_log_cleanup_entry(unode->bm_entry);
+		}
+
+		return true;
+	}
+
+	return false;
+}
+
 SculptUndoNode *sculpt_undo_get_node(PBVHNode *node)
 {
 	ListBase *lb = undo_paint_push_get_list(UNDO_PAINT_MESH);

source/blender/editors/util/undo.c

@@ -489,7 +489,7 @@ static EnumPropertyItem *rna_undo_itemf(bContext *C, int undosys, int *totitem)
 			name = undo_editmode_get_name(C, i, &active);
 		}
 		else if (undosys == UNDOSYSTEM_IMAPAINT) {
-			name = ED_undo_paint_get_name(UNDO_PAINT_IMAGE, i, &active);
+			name = ED_undo_paint_get_name(C, UNDO_PAINT_IMAGE, i, &active);
 		}
 		else {
 			name = BKE_undo_get_name(i, &active);