Cycles: use reference count to detect used shaders

Shaders are only compiled if they are used by some other Node (Geometry, Light, etc.).
This usage detection is done before updating the Scene, however it fails at detecting
Shaders used by Procedurals not known to Cycles (e.g. ones defined by third party
applications), as Procedurals are only updated after the shaders are compiled.

To remedy this, we now use the Node reference counting mechanism to detect whether a
Shader is used and therefore should be compiled.

This removes `ShaderManager::update_shaders_used` as it is not needed anymore, however,
since it would also update the Shader ids, this is now performed in
`ShaderManager::device_update`, and a new virtual `device_update_specific` method was
added to handle device updates for SVM and OSL.

Reviewed By: brecht

Differential Revision: https://developer.blender.org/D10965

intern/cycles/render/osl.cpp

@@ -91,10 +91,10 @@ void OSLShaderManager::reset(Scene * /*scene*/)
   shading_system_init();
 }
 
-void OSLShaderManager::device_update(Device *device,
-                                     DeviceScene *dscene,
-                                     Scene *scene,
-                                     Progress &progress)
+void OSLShaderManager::device_update_specific(Device *device,
+                                              DeviceScene *dscene,
+                                              Scene *scene,
+                                              Progress &progress)
 {
   if (!need_update())
     return;
@@ -1149,7 +1149,7 @@ void OSLCompiler::compile(OSLGlobals *og, Shader *shader)
     shader->has_integrator_dependency = false;
 
     /* generate surface shader */
-    if (shader->used && graph && output->input("Surface")->link) {
+    if (shader->reference_count() && graph && output->input("Surface")->link) {
       shader->osl_surface_ref = compile_type(shader, shader->graph, SHADER_TYPE_SURFACE);
 
       if (has_bump)
@@ -1165,7 +1165,7 @@ void OSLCompiler::compile(OSLGlobals *og, Shader *shader)
     }
 
     /* generate volume shader */
-    if (shader->used && graph && output->input("Volume")->link) {
+    if (shader->reference_count() && graph && output->input("Volume")->link) {
       shader->osl_volume_ref = compile_type(shader, shader->graph, SHADER_TYPE_VOLUME);
       shader->has_volume = true;
     }
@@ -1173,7 +1173,7 @@ void OSLCompiler::compile(OSLGlobals *og, Shader *shader)
       shader->osl_volume_ref = OSL::ShaderGroupRef();
 
     /* generate displacement shader */
-    if (shader->used && graph && output->input("Displacement")->link) {
+    if (shader->reference_count() && graph && output->input("Displacement")->link) {
       shader->osl_displacement_ref = compile_type(shader, shader->graph, SHADER_TYPE_DISPLACEMENT);
       shader->has_displacement = true;
     }

intern/cycles/render/osl.h

@@ -79,7 +79,10 @@ class OSLShaderManager : public ShaderManager {
     return true;
   }
 
-  void device_update(Device *device, DeviceScene *dscene, Scene *scene, Progress &progress);
+  void device_update_specific(Device *device,
+                              DeviceScene *dscene,
+                              Scene *scene,
+                              Progress &progress) override;
   void device_free(Device *device, DeviceScene *dscene, Scene *scene);
 
   /* osl compile and query */

intern/cycles/render/scene.cpp

@@ -523,9 +523,6 @@ bool Scene::update(Progress &progress, bool &kernel_switch_needed)
 {
   /* update scene */
   if (need_update()) {
-    /* Updated used shader tag so we know which features are need for the kernel. */
-    shader_manager->update_shaders_used(this);
-
     /* Update max_closures. */
     KernelIntegrator *kintegrator = &dscene.data.integrator;
     if (params.background) {
@@ -606,7 +603,7 @@ int Scene::get_max_closure_count()
   int max_closures = 0;
   for (int i = 0; i < shaders.size(); i++) {
     Shader *shader = shaders[i];
-    if (shader->used) {
+    if (shader->reference_count()) {
       int num_closures = shader->graph->get_num_closures();
       max_closures = max(max_closures, num_closures);
     }
@@ -767,9 +764,10 @@ template<> void Scene::delete_node_impl(ParticleSystem *node)
   particle_system_manager->tag_update(this);
 }
 
-template<> void Scene::delete_node_impl(Shader * /*node*/)
+template<> void Scene::delete_node_impl(Shader *shader)
 {
   /* don't delete unused shaders, not supported */
+  shader->clear_reference_count();
 }
 
 template<> void Scene::delete_node_impl(Procedural *node)
@@ -836,9 +834,12 @@ template<> void Scene::delete_nodes(const set<ParticleSystem *> &nodes, const No
   particle_system_manager->tag_update(this);
 }
 
-template<> void Scene::delete_nodes(const set<Shader *> & /*nodes*/, const NodeOwner * /*owner*/)
+template<> void Scene::delete_nodes(const set<Shader *> &nodes, const NodeOwner * /*owner*/)
 {
   /* don't delete unused shaders, not supported */
+  for (Shader *shader : nodes) {
+    shader->clear_reference_count();
+  }
 }
 
 template<> void Scene::delete_nodes(const set<Procedural *> &nodes, const NodeOwner *owner)

intern/cycles/render/shader.cpp

@@ -16,7 +16,6 @@
 
 #include "device/device.h"
 
-#include "render/alembic.h"
 #include "render/background.h"
 #include "render/camera.h"
 #include "render/colorspace.h"
@@ -27,6 +26,7 @@
 #include "render/nodes.h"
 #include "render/object.h"
 #include "render/osl.h"
+#include "render/procedural.h"
 #include "render/scene.h"
 #include "render/shader.h"
 #include "render/svm.h"
@@ -218,7 +218,6 @@ Shader::Shader() : Node(get_node_type())
   displacement_method = DISPLACE_BUMP;
 
   id = -1;
-  used = false;
 
   need_update_uvs = true;
   need_update_attribute = true;
@@ -382,8 +381,9 @@ void Shader::tag_used(Scene *scene)
 {
   /* if an unused shader suddenly gets used somewhere, it needs to be
    * recompiled because it was skipped for compilation before */
-  if (!used) {
+  if (!reference_count()) {
     tag_modified();
+    /* We do not reference here as the shader will be referenced when added to a socket. */
     scene->shader_manager->tag_update(scene, ShaderManager::SHADER_MODIFIED);
   }
 }
@@ -461,52 +461,28 @@ int ShaderManager::get_shader_id(Shader *shader, bool smooth)
   return id;
 }
 
-void ShaderManager::update_shaders_used(Scene *scene)
+void ShaderManager::device_update(Device *device,
+                                  DeviceScene *dscene,
+                                  Scene *scene,
+                                  Progress &progress)
 {
   if (!need_update()) {
     return;
   }
 
-  /* figure out which shaders are in use, so SVM/OSL can skip compiling them
-   * for speed and avoid loading image textures into memory */
   uint id = 0;
   foreach (Shader *shader, scene->shaders) {
-    shader->used = false;
     shader->id = id++;
   }
 
-  scene->default_surface->used = true;
-  scene->default_light->used = true;
-  scene->default_background->used = true;
-  scene->default_empty->used = true;
+  /* Those shaders should always be compiled as they are used as fallback if a shader cannot be
+   * found, e.g. bad shader index for the triangle shaders on a Mesh. */
+  assert(scene->default_surface->reference_count() != 0);
+  assert(scene->default_light->reference_count() != 0);
+  assert(scene->default_background->reference_count() != 0);
+  assert(scene->default_empty->reference_count() != 0);
 
-  if (scene->background->get_shader())
-    scene->background->get_shader()->used = true;
-
-#ifdef WITH_ALEMBIC
-  foreach (Procedural *procedural, scene->procedurals) {
-    AlembicProcedural *abc_proc = static_cast<AlembicProcedural *>(procedural);
-
-    foreach (Node *abc_node, abc_proc->get_objects()) {
-      AlembicObject *abc_object = static_cast<AlembicObject *>(abc_node);
-
-      foreach (Node *node, abc_object->get_used_shaders()) {
-        Shader *shader = static_cast<Shader *>(node);
-        shader->used = true;
-      }
-    }
-  }
-#endif
-
-  foreach (Geometry *geom, scene->geometry)
-    foreach (Node *node, geom->get_used_shaders()) {
-      Shader *shader = static_cast<Shader *>(node);
-      shader->used = true;
-    }
-
-  foreach (Light *light, scene->lights)
-    if (light->get_shader())
-      const_cast<Shader *>(light->get_shader())->used = true;
+  device_update_specific(device, dscene, scene, progress);
 }
 
 void ShaderManager::device_update_common(Device *device,
@@ -639,6 +615,7 @@ void ShaderManager::add_default(Scene *scene)
     Shader *shader = scene->create_node<Shader>();
     shader->name = "default_surface";
     shader->set_graph(graph);
+    shader->reference();
     scene->default_surface = shader;
     shader->tag_update(scene);
   }
@@ -657,6 +634,8 @@ void ShaderManager::add_default(Scene *scene)
     shader->set_graph(graph);
     scene->default_volume = shader;
     shader->tag_update(scene);
+    /* No default reference for the volume to avoid compiling volume kernels if there are no actual
+     * volumes in the scene */
   }
 
   /* default light */
@@ -673,6 +652,7 @@ void ShaderManager::add_default(Scene *scene)
     Shader *shader = scene->create_node<Shader>();
     shader->name = "default_light";
     shader->set_graph(graph);
+    shader->reference();
     scene->default_light = shader;
     shader->tag_update(scene);
   }
@@ -684,6 +664,7 @@ void ShaderManager::add_default(Scene *scene)
     Shader *shader = scene->create_node<Shader>();
     shader->name = "default_background";
     shader->set_graph(graph);
+    shader->reference();
     scene->default_background = shader;
     shader->tag_update(scene);
   }
@@ -695,6 +676,7 @@ void ShaderManager::add_default(Scene *scene)
     Shader *shader = scene->create_node<Shader>();
     shader->name = "default_empty";
     shader->set_graph(graph);
+    shader->reference();
     scene->default_empty = shader;
     shader->tag_update(scene);
   }
@@ -735,7 +717,7 @@ void ShaderManager::get_requested_features(Scene *scene,
   requested_features->nodes_features = 0;
   for (int i = 0; i < scene->shaders.size(); i++) {
     Shader *shader = scene->shaders[i];
-    if (!shader->used) {
+    if (!shader->reference_count()) {
       continue;
     }
 

intern/cycles/render/shader.h

@@ -132,7 +132,6 @@ class Shader : public Node {
 
   /* determined before compiling */
   uint id;
-  bool used;
 
 #ifdef WITH_OSL
   /* osl shading state references */
@@ -187,10 +186,11 @@ class ShaderManager {
   }
 
   /* device update */
-  virtual void device_update(Device *device,
-                             DeviceScene *dscene,
-                             Scene *scene,
-                             Progress &progress) = 0;
+  void device_update(Device *device, DeviceScene *dscene, Scene *scene, Progress &progress);
+  virtual void device_update_specific(Device *device,
+                                      DeviceScene *dscene,
+                                      Scene *scene,
+                                      Progress &progress) = 0;
   virtual void device_free(Device *device, DeviceScene *dscene, Scene *scene) = 0;
 
   void device_update_common(Device *device, DeviceScene *dscene, Scene *scene, Progress &progress);
@@ -208,7 +208,6 @@ class ShaderManager {
   static void add_default(Scene *scene);
 
   /* Selective nodes compilation. */
-  void update_shaders_used(Scene *scene);
   void get_requested_features(Scene *scene, DeviceRequestedFeatures *requested_features);
 
   static void free_memory();

intern/cycles/render/svm.cpp

@@ -69,10 +69,10 @@ void SVMShaderManager::device_update_shader(Scene *scene,
           << summary.full_report();
 }
 
-void SVMShaderManager::device_update(Device *device,
-                                     DeviceScene *dscene,
-                                     Scene *scene,
-                                     Progress &progress)
+void SVMShaderManager::device_update_specific(Device *device,
+                                              DeviceScene *dscene,
+                                              Scene *scene,
+                                              Progress &progress)
 {
   if (!need_update())
     return;
@@ -776,7 +776,7 @@ void SVMCompiler::compile_type(Shader *shader, ShaderGraph *graph, ShaderType ty
     add_node(NODE_ENTER_BUMP_EVAL, bump_state_offset);
   }
 
-  if (shader->used) {
+  if (shader->reference_count()) {
     CompilerState state(graph);
     if (clin->link) {
       bool generate = false;

intern/cycles/render/svm.h

@@ -46,7 +46,10 @@ class SVMShaderManager : public ShaderManager {
 
   void reset(Scene *scene);
 
-  void device_update(Device *device, DeviceScene *dscene, Scene *scene, Progress &progress);
+  void device_update_specific(Device *device,
+                              DeviceScene *dscene,
+                              Scene *scene,
+                              Progress &progress) override;
   void device_free(Device *device, DeviceScene *dscene, Scene *scene);
 
  protected: