fix for non Euler-XYZ rotations...

- Camera to 3D view didnt check for rotation order.
- Fly mode didnt check for rotation order.

added util functions.
- object_apply_mat4(ob, mat4); applies a 4x4 matrix to an objects loc,scale,rot (accounting for rotation modes)
- object_mat3_to_rot(ob, mat3, use_compat); apply a 3x3 matrix to the objects rotation, option to use a euler compatible with the existing euler.

source/blender/blenkernel/BKE_object.h

@@ -93,8 +93,10 @@ void disable_speed_curve(int val);
 float bsystem_time(struct Scene *scene, struct Object *ob, float cfra, float ofs);
 void object_scale_to_mat3(struct Object *ob, float mat[][3]);
 void object_rot_to_mat3(struct Object *ob, float mat[][3]);
+void object_mat3_to_rot(struct Object *ob, float mat[][3], int use_compat);
 void object_to_mat3(struct Object *ob, float mat[][3]);
 void object_to_mat4(struct Object *ob, float mat[][4]);
+void object_apply_mat4(struct Object *ob, float mat[][4]);
 
 void set_no_parent_ipo(int val);
 

source/blender/blenkernel/intern/object.c

@@ -1618,6 +1618,32 @@ void object_rot_to_mat3(Object *ob, float mat[][3])
 	mul_m3_m3m3(mat, dmat, rmat);
 }
 
+void object_mat3_to_rot(Object *ob, float mat[][3], int use_compat)
+{
+	if (ob->rotmode == ROT_MODE_QUAT)
+		mat3_to_quat(ob->quat, mat);
+	else if (ob->rotmode == ROT_MODE_AXISANGLE)
+		mat3_to_axis_angle(ob->rotAxis, &ob->rotAngle, mat);
+	else {
+		if(use_compat) {
+			float eul[3];
+			VECCOPY(eul, ob->rot);
+			mat3_to_compatible_eulO(ob->rot, eul, ob->rotmode, mat);
+		}
+		else
+			mat3_to_eulO(ob->rot, ob->rotmode, mat);
+	}
+}
+
+void object_apply_mat4(Object *ob, float mat[][4])
+{
+	float mat3[3][3];
+	VECCOPY(ob->loc, mat[3]);
+	mat4_to_size(ob->size, mat);
+	copy_m3_m4(mat3, mat);
+	object_mat3_to_rot(ob, mat3, 0);
+}
+
 void object_to_mat3(Object *ob, float mat[][3])	/* no parent */
 {
 	float smat[3][3];

source/blender/editors/object/object_transform.c

@@ -555,17 +555,7 @@ static int visual_transform_apply_exec(bContext *C, wmOperator *op)
 	
 	CTX_DATA_BEGIN(C, Object*, ob, selected_editable_objects) {
 		where_is_object(scene, ob);
-		
+		object_apply_mat4(ob, ob->obmat);
-		VECCOPY(ob->loc, ob->obmat[3]);
-		mat4_to_size(ob->size,ob->obmat);
-		
-		if (ob->rotmode == ROT_MODE_QUAT)
-			mat4_to_quat(ob->quat, ob->obmat);
-		else if (ob->rotmode == ROT_MODE_AXISANGLE)
-			mat4_to_axis_angle(ob->rotAxis, &ob->rotAngle, ob->obmat);
-		else
-			mat4_to_eul(ob->rot,ob->obmat);
-		
 		where_is_object(scene, ob);
 		
 		change = 1;

source/blender/editors/space_view3d/view3d_view.c

@@ -395,6 +395,7 @@ void VIEW3D_OT_smoothview(wmOperatorType *ot)
 static void setcameratoview3d(View3D *v3d, RegionView3D *rv3d, Object *ob)
 {
 	float dvec[3];
+	float mat3[3][3];
 	
 	dvec[0]= rv3d->dist*rv3d->viewinv[2][0];
 	dvec[1]= rv3d->dist*rv3d->viewinv[2][1];
@@ -404,7 +405,10 @@ static void setcameratoview3d(View3D *v3d, RegionView3D *rv3d, Object *ob)
 	sub_v3_v3v3(ob->loc, ob->loc, rv3d->ofs);
 	rv3d->viewquat[0]= -rv3d->viewquat[0];
 
-	quat_to_eul( ob->rot,rv3d->viewquat);
+	// quat_to_eul( ob->rot,rv3d->viewquat); // in 2.4x for xyz eulers only
+	quat_to_mat3(mat3, rv3d->viewquat);
+	object_mat3_to_rot(ob, mat3, 0);
+
 	rv3d->viewquat[0]= -rv3d->viewquat[0];
 	
 	ob->recalc= OB_RECALC_OB;
@@ -1914,7 +1918,7 @@ typedef struct FlyInfo {
 #define FLY_CANCEL		1
 #define FLY_CONFIRM		2
 
-int initFlyInfo (bContext *C, FlyInfo *fly, wmOperator *op, wmEvent *event)
+static int initFlyInfo (bContext *C, FlyInfo *fly, wmOperator *op, wmEvent *event)
 {
 	float upvec[3]; // tmp
 	float mat[3][3];
@@ -2041,7 +2045,7 @@ static int flyEnd(bContext *C, FlyInfo *fly)
 	else if (fly->persp_backup==RV3D_CAMOB) {	/* camera */
 		float mat3[3][3];
 		copy_m3_m4(mat3, v3d->camera->obmat);
-		mat3_to_compatible_eul( v3d->camera->rot, fly->rot_backup,mat3);
+		object_mat3_to_rot(v3d->camera, mat3, TRUE);
 
 		DAG_id_flush_update(&v3d->camera->id, OB_RECALC_OB);
 #if 0 //XXX2.5
@@ -2078,7 +2082,7 @@ static int flyEnd(bContext *C, FlyInfo *fly)
 	return OPERATOR_CANCELLED;
 }
 
-void flyEvent(FlyInfo *fly, wmEvent *event)
+static void flyEvent(FlyInfo *fly, wmEvent *event)
 {
 	if (event->type == TIMER && event->customdata == fly->timer) {
 		fly->redraw = 1;
@@ -2201,7 +2205,7 @@ void flyEvent(FlyInfo *fly, wmEvent *event)
 }
 
 //int fly_exec(bContext *C, wmOperator *op)
-int flyApply(FlyInfo *fly)
+static int flyApply(FlyInfo *fly)
 {
 	/*
 	fly mode - Shift+F