Attempt to fix #35041 and #34725: cycles crash with OSL and both a 3D viewport

and preview running at the same time.

It seems there's something in OSL/LLVM that's not thread safe, but I couldn't
figure out what exactly. Now all renders share the same OSL ShadingSystem which
should avoid the problem.

intern/cycles/kernel/kernel_types.h

@@ -469,6 +469,8 @@ enum ShaderDataFlag {
 	SD_TRANSFORM_APPLIED = 32768 		/* vertices have transform applied */
 };
 
+struct KernelGlobals;
+
 typedef struct ShaderData {
 	/* position */
 	float3 P;
@@ -536,6 +538,10 @@ typedef struct ShaderData {
 	/* Closure data, with a single sampled closure for low memory usage */
 	ShaderClosure closure;
 #endif
+
+#ifdef __OSL__
+	struct KernelGlobals *osl_globals;
+#endif
 } ShaderData;
 
 /* Constrant Kernel Data

intern/cycles/kernel/osl/osl_services.cpp

@@ -112,8 +112,8 @@ bool OSLRenderServices::get_matrix(OSL::Matrix44 &result, OSL::TransformationPtr
 	/* this is only used for shader and object space, we don't really have
 	 * a concept of shader space, so we just use object space for both. */
 	if (xform) {
-		KernelGlobals *kg = kernel_globals;
 		const ShaderData *sd = (const ShaderData *)xform;
+		KernelGlobals *kg = sd->osl_globals;
 		int object = sd->object;
 
 		if (object != ~0) {
@@ -142,8 +142,8 @@ bool OSLRenderServices::get_inverse_matrix(OSL::Matrix44 &result, OSL::Transform
 	/* this is only used for shader and object space, we don't really have
 	 * a concept of shader space, so we just use object space for both. */
 	if (xform) {
-		KernelGlobals *kg = kernel_globals;
 		const ShaderData *sd = (const ShaderData *)xform;
+		KernelGlobals *kg = sd->osl_globals;
 		int object = sd->object;
 
 		if (object != ~0) {
@@ -235,7 +235,7 @@ bool OSLRenderServices::get_matrix(OSL::Matrix44 &result, OSL::TransformationPtr
 #ifdef __OBJECT_MOTION__
 			Transform tfm = sd->ob_tfm;
 #else
-			KernelGlobals *kg = kernel_globals;
+			KernelGlobals *kg = sd->osl_globals;
 			Transform tfm = object_fetch_transform(kg, object, OBJECT_TRANSFORM);
 #endif
 			tfm = transform_transpose(tfm);
@@ -260,7 +260,7 @@ bool OSLRenderServices::get_inverse_matrix(OSL::Matrix44 &result, OSL::Transform
 #ifdef __OBJECT_MOTION__
 			Transform tfm = sd->ob_itfm;
 #else
-			KernelGlobals *kg = kernel_globals;
+			KernelGlobals *kg = sd->osl_globals;
 			Transform tfm = object_fetch_transform(kg, object, OBJECT_INVERSE_TRANSFORM);
 #endif
 			tfm = transform_transpose(tfm);
@@ -662,8 +662,8 @@ bool OSLRenderServices::get_background_attribute(KernelGlobals *kg, ShaderData *
 bool OSLRenderServices::get_attribute(void *renderstate, bool derivatives, ustring object_name,
                                       TypeDesc type, ustring name, void *val)
 {
-	KernelGlobals *kg = kernel_globals;
 	ShaderData *sd = (ShaderData *)renderstate;
+	KernelGlobals *kg = sd->osl_globals;
 	int object, prim, segment;
 
 	/* lookup of attribute on another object */
@@ -861,7 +861,7 @@ bool OSLRenderServices::trace(TraceOpt &options, OSL::ShaderGlobals *sg,
 	tracedata->init = true;
 
 	/* raytrace */
-	return scene_intersect(kernel_globals, &ray, ~0, &tracedata->isect);
+	return scene_intersect(sd->osl_globals, &ray, ~0, &tracedata->isect);
 }
 
 
@@ -880,8 +880,8 @@ bool OSLRenderServices::getmessage(OSL::ShaderGlobals *sg, ustring source, ustri
 				return set_attribute_float(f, type, derivatives, val);
 			}
 			else {
-				KernelGlobals *kg = kernel_globals;
 				ShaderData *sd = &tracedata->sd;
+				KernelGlobals *kg = sd->osl_globals;
 
 				if(!tracedata->setup) {
 					/* lazy shader data setup */

intern/cycles/kernel/osl/osl_shader.cpp

@@ -127,6 +127,9 @@ static void shaderdata_to_shaderglobals(KernelGlobals *kg, ShaderData *sd,
 
 	/* clear trace data */
 	tdata->tracedata.init = false;
+
+	/* used by renderservices */
+	sd->osl_globals = kg;
 }
 
 /* Surface */

intern/cycles/render/osl.cpp

@@ -41,46 +41,35 @@ CCL_NAMESPACE_BEGIN
 
 #ifdef WITH_OSL
 
-/* Shared Texture System */
+/* Shared Texture and Shading System */
 
 OSL::TextureSystem *OSLShaderManager::ts_shared = NULL;
 int OSLShaderManager::ts_shared_users = 0;
 thread_mutex OSLShaderManager::ts_shared_mutex;
 
+OSL::ShadingSystem *OSLShaderManager::ss_shared = NULL;
+OSLRenderServices *OSLShaderManager::services_shared = NULL;
+int OSLShaderManager::ss_shared_users = 0;
+thread_mutex OSLShaderManager::ss_shared_mutex;
+thread_mutex OSLShaderManager::ss_mutex;
+
 /* Shader Manager */
 
 OSLShaderManager::OSLShaderManager()
 {
-	services = new OSLRenderServices();
-
 	texture_system_init();
 	shading_system_init();
 }
 
 OSLShaderManager::~OSLShaderManager()
 {
-	OSL::ShadingSystem::destroy(ss);
+	shading_system_free();
-
+	texture_system_free();
-	/* shared texture system decrease users and destroy if no longer used */
-	{
-		thread_scoped_lock lock(ts_shared_mutex);
-		ts_shared_users--;
-
-		if(ts_shared_users == 0) {
-			OSL::TextureSystem::destroy(ts_shared);
-			ts_shared = NULL;
-		}
-	}
-
-	delete services;
 }
 
 void OSLShaderManager::reset(Scene *scene)
 {
-	OSL::ShadingSystem::destroy(ss);
+	shading_system_free();
-	delete services;
-
-	services = new OSLRenderServices();
 	shading_system_init();
 }
 
@@ -102,6 +91,11 @@ void OSLShaderManager::device_update(Device *device, DeviceScene *dscene, Scene
 
 		if(progress.get_cancel()) return;
 
+		/* we can only compile one shader at the time as the OSL ShadingSytem
+		 * has a single state, but we put the lock here so different renders can
+		 * compile shaders alternating */
+		thread_scoped_lock lock(ss_mutex);
+
 		OSLCompiler compiler((void*)this, (void*)ss, scene->image_manager);
 		compiler.background = (shader == scene->shaders[scene->default_background]);
 		compiler.compile(og, shader);
@@ -167,35 +161,78 @@ void OSLShaderManager::texture_system_init()
 	ts_shared_users++;
 }
 
+void OSLShaderManager::texture_system_free()
+{
+	/* shared texture system decrease users and destroy if no longer used */
+	thread_scoped_lock lock(ts_shared_mutex);
+	ts_shared_users--;
+
+	if(ts_shared_users == 0) {
+		OSL::TextureSystem::destroy(ts_shared);
+		ts_shared = NULL;
+	}
+
+	ts = NULL;
+}
+
 void OSLShaderManager::shading_system_init()
 {
-	ss = OSL::ShadingSystem::create(services, ts, &errhandler);
+	/* create shading system, shared between different renders to reduce memory usage */
-	ss->attribute("lockgeom", 1);
+	thread_scoped_lock lock(ss_shared_mutex);
-	ss->attribute("commonspace", "world");
-	ss->attribute("optimize", 2);
-	//ss->attribute("debug", 1);
-	//ss->attribute("statistics:level", 1);
-	ss->attribute("searchpath:shader", path_get("shader"));
 
-	/* our own ray types */
+	if(ss_shared_users == 0) {
-	static const char *raytypes[] = {
+		services_shared = new OSLRenderServices();
-		"camera",		/* PATH_RAY_CAMERA */
-		"reflection",	/* PATH_RAY_REFLECT */
-		"refraction",	/* PATH_RAY_TRANSMIT */
-		"diffuse",		/* PATH_RAY_DIFFUSE */
-		"glossy",		/* PATH_RAY_GLOSSY */
-		"singular",		/* PATH_RAY_SINGULAR */
-		"transparent",	/* PATH_RAY_TRANSPARENT */
-		"shadow",		/* PATH_RAY_SHADOW_OPAQUE */
-		"shadow",		/* PATH_RAY_SHADOW_TRANSPARENT */
-	};
 
-	const int nraytypes = sizeof(raytypes)/sizeof(raytypes[0]);
+		ss_shared = OSL::ShadingSystem::create(services_shared, ts_shared, &errhandler);
-	ss->attribute("raytypes", TypeDesc(TypeDesc::STRING, nraytypes), raytypes);
+		ss_shared->attribute("lockgeom", 1);
+		ss_shared->attribute("commonspace", "world");
+		ss_shared->attribute("optimize", 2);
+		//ss_shared->attribute("debug", 1);
+		//ss_shared->attribute("statistics:level", 1);
+		ss_shared->attribute("searchpath:shader", path_get("shader"));
 
-	OSLShader::register_closures((OSLShadingSystem*)ss);
+		/* our own ray types */
+		static const char *raytypes[] = {
+			"camera",		/* PATH_RAY_CAMERA */
+			"reflection",	/* PATH_RAY_REFLECT */
+			"refraction",	/* PATH_RAY_TRANSMIT */
+			"diffuse",		/* PATH_RAY_DIFFUSE */
+			"gloss_sharedy",		/* PATH_RAY_GLOSSY */
+			"singular",		/* PATH_RAY_SINGULAR */
+			"transparent",	/* PATH_RAY_TRANSPARENT */
+			"shadow",		/* PATH_RAY_SHADOW_OPAQUE */
+			"shadow",		/* PATH_RAY_SHADOW_TRANSPARENT */
+		};
 
-	loaded_shaders.clear();
+		const int nraytypes = sizeof(raytypes)/sizeof(raytypes[0]);
+		ss_shared->attribute("raytypes", TypeDesc(TypeDesc::STRING, nraytypes), raytypes);
+
+		OSLShader::register_closures((OSLShadingSystem*)ss_shared);
+
+		loaded_shaders.clear();
+	}
+
+	ss = ss_shared;
+	services = services_shared;
+	ss_shared_users++;
+}
+
+void OSLShaderManager::shading_system_free()
+{
+	/* shared shading system decrease users and destroy if no longer used */
+	thread_scoped_lock lock(ss_shared_mutex);
+	ss_shared_users--;
+
+	if(ss_shared_users == 0) {
+		OSL::ShadingSystem::destroy(ss_shared);
+		ss_shared = NULL;
+
+		delete services_shared;
+		services_shared = NULL;
+	}
+
+	ss = NULL;
+	services = NULL;
 }
 
 bool OSLShaderManager::osl_compile(const string& inputfile, const string& outputfile)

intern/cycles/render/osl.h

@@ -86,7 +86,10 @@ public:
 
 protected:
 	void texture_system_init();
+	void texture_system_free();
+
 	void shading_system_init();
+	void shading_system_free();
 
 	OSL::ShadingSystem *ss;
 	OSL::TextureSystem *ts;
@@ -97,6 +100,12 @@ protected:
 	static OSL::TextureSystem *ts_shared;
 	static thread_mutex ts_shared_mutex;
 	static int ts_shared_users;
+
+	static OSL::ShadingSystem *ss_shared;
+	static OSLRenderServices *services_shared;
+	static thread_mutex ss_shared_mutex;
+	static thread_mutex ss_mutex;
+	static int ss_shared_users;
 };
 
 #endif