fix for fix, r33086.

- incorrect range check broke ZYX euler rotations, use MIN/MAX constants so this doesn't happen again.
- BGE Armature PyAPI also wasn't using correct min/max with rotation modes.
- clamp on file read rather then when calling the rotation functions, so developers don't use invalid args without realizing it.
- added assert() checks for debug builds so invalid axis constants don't slip through.

source/blender/blenlib/intern/math_matrix.c

@@ -25,7 +25,7 @@
  * ***** END GPL LICENSE BLOCK *****
  */
 
-
+#include <assert.h>
 #include "BLI_math.h"
 
 /********************************* Init **************************************/
@@ -1041,6 +1041,8 @@ void rotate_m4(float mat[][4], const char axis, const float angle)
 	float temp[4]= {0.0f, 0.0f, 0.0f, 0.0f};
 	float cosine, sine;
 
+	assert(axis >= 'X' && axis <= 'Z');
+
 	cosine = (float)cos(angle);
 	sine = (float)sin(angle);
 	switch(axis){

source/blender/blenlib/intern/math_rotation.c

@@ -26,6 +26,7 @@
  * */
 
 
+#include <assert.h>
 #include "BLI_math.h"
 
 /******************************** Quaternions ********************************/
@@ -365,6 +366,9 @@ void vec_to_quat(float q[4], const float vec[3], short axis, const short upflag)
 {
 	float q2[4], nor[3], *fp, mat[3][3], angle, si, co, x2, y2, z2, len1;
 	
+	assert(axis >= 0 && axis <= 5);
+	assert(upflag >= 0 && upflag <= 2);
+	
 	/* first rotate to axis */
 	if(axis>2) {	
 		x2= vec[0] ; y2= vec[1] ; z2= vec[2];
@@ -946,7 +950,9 @@ void eul_to_quat(float *quat, const float eul[3])
 void rotate_eul(float *beul, const char axis, const float ang)
 {
 	float eul[3], mat1[3][3], mat2[3][3], totmat[3][3];
-	
+
+	assert(axis >= 'X' && axis <= 'Z');
+
 	eul[0]= eul[1]= eul[2]= 0.0f;
 	if(axis=='X') eul[0]= ang;
 	else if(axis=='Y') eul[1]= ang;
@@ -1084,7 +1090,7 @@ static RotOrderInfo rotOrders[]= {
  * NOTE: since we start at 1 for the values, but arrays index from 0, 
  *		 there is -1 factor involved in this process...
  */
-#define GET_ROTATIONORDER_INFO(order) ( ((order)>5 || (order < 1)) ? &rotOrders[0] : &rotOrders[(order)-1] )
+#define GET_ROTATIONORDER_INFO(order) (assert(order>=0 && order<=6), (order < 1) ? &rotOrders[0] : &rotOrders[(order)-1])
 
 /* Construct quaternion from Euler angles (in radians). */
 void eulO_to_quat(float q[4], const float e[3], const short order)
@@ -1266,7 +1272,9 @@ void mat4_to_compatible_eulO(float eul[3], float oldrot[3], short order,float M[
 void rotate_eulO(float beul[3], short order, char axis, float ang)
 {
 	float eul[3], mat1[3][3], mat2[3][3], totmat[3][3];
-	
+
+	assert(axis >= 'X' && axis <= 'Z');
+
 	eul[0]= eul[1]= eul[2]= 0.0f;
 	if (axis=='X') 
 		eul[0]= ang;
@@ -1544,6 +1552,9 @@ void quat_apply_track(float quat[4], short axis, short upflag)
 	                              {0.5, -0.5, -0.5, 0.5}, /* Quaternion((1,0,0), radians(-90)) * Quaternion((0,1,0), radians(-90)) */ 
 	                              {-3.0908619663705394e-08, 0.70710676908493, 0.70710676908493, 3.0908619663705394e-08}}; /* no rotation */
 
+	assert(axis >= 0 && axis <= 5);
+	assert(upflag >= 0 && upflag <= 2);
+	
 	mul_qt_qtqt(quat, quat, quat_track[axis]);
 
 	if(axis>2)
@@ -1564,6 +1575,8 @@ void vec_apply_track(float vec[3], short axis)
 {
 	float tvec[3];
 
+	assert(axis >= 0 && axis <= 5);
+	
 	copy_v3_v3(tvec, vec);
 
 	switch(axis) {

source/blender/blenloader/intern/readfile.c

@@ -3705,6 +3705,9 @@ static void direct_link_pose(FileData *fd, bPose *pose)
 		
 		pchan->iktree.first= pchan->iktree.last= NULL;
 		pchan->path= NULL;
+		
+		/* incase this value changes in future, clamp else we get undefined behavior */
+		CLAMP(pchan->rotmode, ROT_MODE_MIN, ROT_MODE_MAX);
 	}
 	pose->ikdata = NULL;
 	if (pose->ikparam != NULL) {
@@ -4111,6 +4114,9 @@ static void direct_link_object(FileData *fd, Object *ob)
 	ob->gpulamp.first= ob->gpulamp.last= NULL;
 	link_list(fd, &ob->pc_ids);
 
+	/* incase this value changes in future, clamp else we get undefined behavior */
+	CLAMP(ob->rotmode, ROT_MODE_MIN, ROT_MODE_MAX);
+
 	if(ob->sculpt) {
 		ob->sculpt= MEM_callocN(sizeof(SculptSession), "reload sculpt session");
 		ob->sculpt->ob= ob;

source/blender/makesdna/DNA_action_types.h

@@ -309,9 +309,11 @@ typedef enum eRotationModes {
 	/* NOTE: space is reserved here for 18 other possible 
 	 * euler rotation orders not implemented 
 	 */
-	ROT_MODE_MAX,	/* sentinel for Py API */
 		/* axis angle rotations */
-	ROT_MODE_AXISANGLE = -1
+	ROT_MODE_AXISANGLE = -1,
+
+	ROT_MODE_MIN = ROT_MODE_AXISANGLE,	/* sentinel for Py API */
+	ROT_MODE_MAX = ROT_MODE_ZYX
 } eRotationModes;
 
 /* Pose ------------------------------------ */

source/gameengine/Converter/BL_ArmatureChannel.cpp

@@ -121,7 +121,7 @@ PyAttributeDef BL_ArmatureChannel::AttributesPtr[] = {
 	KX_PYATTRIBUTE_FLOAT_VECTOR_RW("scale",-FLT_MAX,FLT_MAX,bPoseChannel,size,3),
 	KX_PYATTRIBUTE_FLOAT_VECTOR_RW("rotation_quaternion",-1.0f,1.0f,bPoseChannel,quat,4),
 	KX_PYATTRIBUTE_FLOAT_VECTOR_RW("rotation_euler",-10.f,10.f,bPoseChannel,eul,3),
-	KX_PYATTRIBUTE_SHORT_RW("rotation_mode",0,ROT_MODE_MAX-1,false,bPoseChannel,rotmode),
+	KX_PYATTRIBUTE_SHORT_RW("rotation_mode",ROT_MODE_MIN,ROT_MODE_MAX,false,bPoseChannel,rotmode),
 	KX_PYATTRIBUTE_FLOAT_MATRIX_RO("channel_matrix",bPoseChannel,chan_mat,4),
 	KX_PYATTRIBUTE_FLOAT_MATRIX_RO("pose_matrix",bPoseChannel,pose_mat,4),
 	KX_PYATTRIBUTE_FLOAT_VECTOR_RO("pose_head",bPoseChannel,pose_head,3),