Sculpt: fix 104174, clicking over empty space always pushes undo.

This was actually caused by two problems.  The first is that the
    code actually deliberately detects if nothing happened and pushed
    an undo step, a bugfix for a problem that no longer exists.

    The second was that SCULPT_test_location sometimes does a closest
    point search instead of ray casting and didn't check the result
    against the brush radius.
This commit is contained in:
Joseph Eagar 2023-02-24 00:01:41 -08:00
parent e9bb3510a1
commit 0534fff5ab
2 changed files with 57 additions and 36 deletions

@ -81,6 +81,19 @@ using blender::MutableSpan;
using blender::Set;
using blender::Vector;
static float sculpt_calc_radius(ViewContext *vc,
const Brush *brush,
const Scene *scene,
const float3 location)
{
if (!BKE_brush_use_locked_size(scene, brush)) {
return paint_calc_object_space_radius(vc, location, BKE_brush_size_get(scene, brush));
}
else {
return BKE_brush_unprojected_radius_get(scene, brush);
}
}
/* -------------------------------------------------------------------- */
/** \name Sculpt PBVH Abstraction API
*
@ -4925,14 +4938,11 @@ static void sculpt_update_cache_variants(bContext *C, Sculpt *sd, Object *ob, Po
/* Truly temporary data that isn't stored in properties. */
if (SCULPT_stroke_is_first_brush_step_of_symmetry_pass(ss->cache)) {
cache->initial_radius = sculpt_calc_radius(cache->vc, brush, scene, cache->true_location);
if (!BKE_brush_use_locked_size(scene, brush)) {
cache->initial_radius = paint_calc_object_space_radius(
cache->vc, cache->true_location, BKE_brush_size_get(scene, brush));
BKE_brush_unprojected_radius_set(scene, brush, cache->initial_radius);
}
else {
cache->initial_radius = BKE_brush_unprojected_radius_get(scene, brush);
}
}
/* Clay stabilized pressure. */
@ -5280,6 +5290,19 @@ bool SCULPT_stroke_get_location(bContext *C,
float out[3],
const float mval[2],
bool force_original)
{
const Brush *brush = BKE_paint_brush(BKE_paint_get_active_from_context(C));
bool check_closest = brush->falloff_shape == PAINT_FALLOFF_SHAPE_TUBE;
return SCULPT_stroke_get_location_ex(C, out, mval, force_original, check_closest, true);
}
bool SCULPT_stroke_get_location_ex(bContext *C,
float out[3],
const float mval[2],
bool force_original,
bool check_closest,
bool limit_closest_radius)
{
Depsgraph *depsgraph = CTX_data_depsgraph_pointer(C);
Object *ob;
@ -5329,11 +5352,7 @@ bool SCULPT_stroke_get_location(bContext *C,
}
}
if (hit) {
return hit;
}
if (!ELEM(brush->falloff_shape, PAINT_FALLOFF_SHAPE_TUBE)) {
if (hit || !check_closest) {
return hit;
}
@ -5348,14 +5367,20 @@ bool SCULPT_stroke_get_location(bContext *C,
BKE_pbvh_find_nearest_to_ray(
ss->pbvh, sculpt_find_nearest_to_ray_cb, &srd, ray_start, ray_normal, srd.original);
if (srd.hit) {
if (srd.hit && srd.dist_sq_to_ray) {
hit = true;
copy_v3_v3(out, ray_normal);
mul_v3_fl(out, srd.depth);
add_v3_v3(out, ray_start);
}
return hit;
float closest_radius_sq = FLT_MAX;
if (limit_closest_radius) {
closest_radius_sq = sculpt_calc_radius(&vc, brush, CTX_data_scene(C), out);
closest_radius_sq *= closest_radius_sq;
}
return hit && srd.dist_sq_to_ray < closest_radius_sq;
}
static void sculpt_brush_init_tex(Sculpt *sd, SculptSession *ss)
@ -5620,7 +5645,12 @@ void SCULPT_flush_update_done(const bContext *C, Object *ob, SculptUpdateType up
static bool over_mesh(bContext *C, wmOperator * /*op*/, const float mval[2])
{
float co_dummy[3];
return SCULPT_stroke_get_location(C, co_dummy, mval, false);
Sculpt *sd = CTX_data_tool_settings(C)->sculpt;
Brush *brush = BKE_paint_brush(&sd->paint);
bool check_closest = brush->falloff_shape == PAINT_FALLOFF_SHAPE_TUBE;
return SCULPT_stroke_get_location_ex(C, co_dummy, mval, false, check_closest, true);
}
static void sculpt_stroke_undo_begin(const bContext *C, wmOperator *op)
@ -5974,29 +6004,7 @@ static void sculpt_brush_stroke_cancel(bContext *C, wmOperator *op)
static int sculpt_brush_stroke_modal(bContext *C, wmOperator *op, const wmEvent *event)
{
bool started = op->customdata && paint_stroke_started((PaintStroke *)op->customdata);
int retval = paint_stroke_modal(C, op, event, (PaintStroke **)&op->customdata);
if (!started && ELEM(retval, OPERATOR_FINISHED, OPERATOR_CANCELLED)) {
/* Did the stroke never start? If so push a blank sculpt undo
* step to prevent a global undo step (which is triggered by the
* #OPTYPE_UNDO flag in #SCULPT_OT_brush_stroke).
*
* Having blank global undo steps interleaved with sculpt steps
* corrupts the DynTopo undo stack.
* See #101430.
*
* NOTE: simply returning #OPERATOR_CANCELLED was not
* sufficient to prevent this. */
Sculpt *sd = CTX_data_tool_settings(C)->sculpt;
Brush *brush = BKE_paint_brush(&sd->paint);
sculpt_stroke_undo_begin(C, op);
sculpt_stroke_undo_end(C, brush);
}
return retval;
return paint_stroke_modal(C, op, event, (PaintStroke **)&op->customdata);
}
static void sculpt_redo_empty_ui(bContext * /*C*/, wmOperator * /*op*/)

@ -24,6 +24,8 @@
#include "ED_view3d.h"
#include <functional>
struct AutomaskingCache;
struct AutomaskingNodeData;
struct Dial;
@ -890,7 +892,18 @@ void SCULPT_tag_update_overlays(bContext *C);
* Do a ray-cast in the tree to find the 3d brush location
* (This allows us to ignore the GL depth buffer)
* Returns 0 if the ray doesn't hit the mesh, non-zero otherwise.
*
* If check_closest is true and the ray test fails a point closest
* to the ray will be found. If limit_closest_radius is true then
* the closest point will be tested against the active brush radius.
*/
bool SCULPT_stroke_get_location_ex(bContext *C,
float out[3],
const float mval[2],
bool force_original,
bool check_closest,
bool limit_closest_radius);
bool SCULPT_stroke_get_location(bContext *C,
float out[3],
const float mouse[2],