Fix #99887: Multires Unsubdivide freezes Blender with hidden geometry

Previously in
```
edge_y = edge_step(current_vertex_y, edge_y, &current_vertex_y);
while (!BM_elem_flag_test(current_vertex_y, BM_ELEM_TAG))
```
With hidden faces this code enters in an infinite loop by returning the same null
edge in edge_step() and the same current_vertex_y causing Blender to crash.

The current fix solves the infinite loop and provides a solution when faces are hidden.
This causes the hidden faces to be merged into one for unsubdivided per original dev's implementation.

At minimum the crash is identified and a solution proposed.
Attached is a video of the fix

Pull Request: https://projects.blender.org/blender/blender/pulls/121086
This commit is contained in:
Raul Fernandez 2024-05-07 18:04:52 +02:00 committed by Raul Fernandez Hernandez
parent 88b56a6ad2
commit 00e2f55239

@ -841,17 +841,27 @@ static void multires_unsubdivide_get_grid_corners_on_base_mesh(BMFace *f1,
/* Do an edge step until it finds a tagged vertex, which is part of the base mesh. */
/* x axis */
edge_x = edge_step(current_vertex_x, edge_x, &current_vertex_x);
while (!BM_elem_flag_test(current_vertex_x, BM_ELEM_TAG)) {
while (edge_x && !BM_elem_flag_test(current_vertex_x, BM_ELEM_TAG)) {
edge_x = edge_step(current_vertex_x, edge_x, &current_vertex_x);
}
(*r_corner_x) = current_vertex_x;
if (edge_x) {
*r_corner_x = current_vertex_x;
}
else {
*r_corner_x = nullptr;
}
/* Same for y axis */
edge_y = edge_step(current_vertex_y, edge_y, &current_vertex_y);
while (!BM_elem_flag_test(current_vertex_y, BM_ELEM_TAG)) {
while (edge_y && !BM_elem_flag_test(current_vertex_y, BM_ELEM_TAG)) {
edge_y = edge_step(current_vertex_y, edge_y, &current_vertex_y);
}
(*r_corner_y) = current_vertex_y;
if (edge_y) {
*r_corner_y = current_vertex_y;
}
else {
*r_corner_y = nullptr;
}
}
static BMesh *get_bmesh_from_mesh(Mesh *mesh)
@ -1043,6 +1053,9 @@ static void multires_unsubdivide_extract_grids(MultiresUnsubdivideContext *conte
* base mesh of the face of grid that is going to be extracted. */
BMVert *corner_x, *corner_y;
multires_unsubdivide_get_grid_corners_on_base_mesh(l->f, l->e, &corner_x, &corner_y);
if (!corner_x || !corner_y) {
continue;
}
/* Map the two obtained vertices to the base mesh. */
const int corner_x_index = orig_to_base_vmap[BM_elem_index_get(corner_x)];