Fix T41219: Cycles backface detection doesn't work properly

Root of the issue goes back to the on-fly normals commit and the
latest fix for it wasn't actually correct. I've mixed two fixes
in there.

So the idea here goes back to storing negative scaled object flag
and flip runtime-calculated normal if this flag is set, which is
pretty much the same as the original fix for the issue from me.

The issue with motion blur wasn't caused by the rumtime normals
patch and it had issues before, because it already did runtime
normals calculation. Now made it so motion triangles takes the
negative scale flag into account.

This actually makes code more clean imo and avoids rather confusing
flipping code in mesh.cpp.

intern/cycles/kernel/geom/geom_motion_triangle.h

@@ -272,7 +272,11 @@ ccl_device_noinline void motion_triangle_shader_setup(KernelGlobals *kg, ShaderD
 #endif
 
 	/* compute face normal */
-	float3 Ng = normalize(cross(verts[1] - verts[0], verts[2] - verts[0]));
+	float3 Ng;
+	if(sd->flag & SD_NEGATIVE_SCALE_APPLIED)
+		Ng = normalize(cross(verts[2] - verts[0], verts[1] - verts[0]));
+	else
+		Ng = normalize(cross(verts[1] - verts[0], verts[2] - verts[0]));
 
 	sd->Ng = Ng;
 	sd->N = Ng;

intern/cycles/kernel/geom/geom_triangle.h

@@ -127,11 +127,14 @@ ccl_device_inline float3 triangle_normal(KernelGlobals *kg, ShaderData *sd)
 	float3 v2 = float4_to_float3(kernel_tex_fetch(__tri_verts, __float_as_int(tri_vindex.z)));
 	
 	/* return normal */
-	return normalize(cross(v1 - v0, v2 - v0));
+	if(sd->flag & SD_NEGATIVE_SCALE_APPLIED)
+		return normalize(cross(v2 - v0, v1 - v0));
+	else
+		return normalize(cross(v1 - v0, v2 - v0));
 }
 
 /* point and normal on triangle  */
-ccl_device_inline void triangle_point_normal(KernelGlobals *kg, int prim, float u, float v, float3 *P, float3 *Ng, int *shader)
+ccl_device_inline void triangle_point_normal(KernelGlobals *kg, int object, int prim, float u, float v, float3 *P, float3 *Ng, int *shader)
 {
 	/* load triangle vertices */
 	float3 tri_vindex = float4_to_float3(kernel_tex_fetch(__tri_vindex, prim));
@@ -145,7 +148,11 @@ ccl_device_inline void triangle_point_normal(KernelGlobals *kg, int prim, float
 	*P = (u*v0 + v*v1 + t*v2);
 
 	/* compute normal */
-	*Ng = normalize(cross(v1 - v0, v2 - v0));
+	int object_flag = kernel_tex_fetch(__object_flag, object);
+	if(object_flag & SD_NEGATIVE_SCALE_APPLIED)
+		*Ng = normalize(cross(v2 - v0, v1 - v0));
+	else
+		*Ng = normalize(cross(v1 - v0, v2 - v0));
 
 	/* shader`*/
 	*shader = __float_as_int(kernel_tex_fetch(__tri_shader, prim));

intern/cycles/kernel/kernel_bake.h

@@ -215,7 +215,7 @@ ccl_device void kernel_bake_evaluate(KernelGlobals *kg, ccl_global uint4 *input,
 	int shader;
 	float3 P, Ng;
 
-	triangle_point_normal(kg, prim, u, v, &P, &Ng, &shader);
+	triangle_point_normal(kg, object, prim, u, v, &P, &Ng, &shader);
 
 	/* dummy initilizations copied from SHADER_EVAL_DISPLACE */
 	float3 I = Ng;

intern/cycles/kernel/kernel_light.h

@@ -457,7 +457,7 @@ ccl_device void triangle_light_sample(KernelGlobals *kg, int prim, int object,
 	v = randv*randu;
 
 	/* triangle, so get position, normal, shader */
-	triangle_point_normal(kg, prim, u, v, &ls->P, &ls->Ng, &ls->shader);
+	triangle_point_normal(kg, object, prim, u, v, &ls->P, &ls->Ng, &ls->shader);
 	ls->object = object;
 	ls->prim = prim;
 	ls->lamp = LAMP_NONE;

intern/cycles/kernel/kernel_shader.h

@@ -340,7 +340,7 @@ ccl_device void shader_setup_from_displace(KernelGlobals *kg, ShaderData *sd,
 	float3 P, Ng, I = make_float3(0.0f, 0.0f, 0.0f);
 	int shader;
 
-	triangle_point_normal(kg, prim, u, v, &P, &Ng, &shader);
+	triangle_point_normal(kg, object, prim, u, v, &P, &Ng, &shader);
 
 	/* force smooth shading for displacement */
 	shader |= SHADER_SMOOTH_NORMAL;

intern/cycles/kernel/kernel_types.h

@@ -592,6 +592,7 @@ enum ShaderDataFlag {
 	SD_HOLDOUT_MASK = 524288,			/* holdout for camera rays */
 	SD_OBJECT_MOTION = 1048576,			/* has object motion blur */
 	SD_TRANSFORM_APPLIED = 2097152,		/* vertices have transform applied */
+	SD_NEGATIVE_SCALE_APPLIED = 4194304,	/* vertices have negative scale applied */
 
 	SD_OBJECT_FLAGS = (SD_HOLDOUT_MASK|SD_OBJECT_MOTION|SD_TRANSFORM_APPLIED)
 };

intern/cycles/render/mesh.cpp

@@ -341,13 +341,6 @@ void Mesh::add_vertex_normals()
 				vN[i] = -vN[i];
 		}
 	}
-	else if(flip) {
-		Attribute *attr_vN = attributes.find(ATTR_STD_VERTEX_NORMAL);
-		float3 *vN = attr_vN->data_float3();
-		for(size_t i = 0; i < verts_size; i++) {
-			vN[i] = -vN[i];
-		}
-	}
 
 	/* motion vertex normals */
 	Attribute *attr_mP = attributes.find(ATTR_STD_MOTION_VERTEX_POSITION);
@@ -382,14 +375,6 @@ void Mesh::add_vertex_normals()
 			}
 		}
 	}
-	else if(has_motion_blur() && attr_mN && flip) {
-		for(int step = 0; step < motion_steps - 1; step++) {
-			float3 *mN = attr_mN->data_float3() + step*verts.size();
-			for(size_t i = 0; i < verts_size; i++) {
-				mN[i] = -mN[i];
-			}
-		}
-	}
 }
 
 void Mesh::pack_normals(Scene *scene, float *tri_shader, float4 *vnormal)

intern/cycles/render/object.cpp

@@ -449,6 +449,8 @@ void ObjectManager::apply_static_transforms(DeviceScene *dscene, Scene *scene, u
 				}
 
 				object_flag[i] |= SD_TRANSFORM_APPLIED;
+				if(object->mesh->transform_negative_scaled)
+					object_flag[i] |= SD_NEGATIVE_SCALE_APPLIED;
 			}
 			else
 				have_instancing = true;