Cycles: internal support for the concept of procedurals

Procedurals are nodes in the scene that can generate an arbitrary number of
other nodes at render time. This will be used to implement an Alembic procedural
that can load an Alembic file into Cycles nodes. In the future we also expect to
have a USD procedural.

Direct loading of such files at render time is a standard feature in other
production renderers. Reasons to support this are memory usage and performance,
delayed loading of heavy scene data until rendering, Cycles standalone rendering
using standard file formats beyond our XML files, and shared functionality for
Cycles integration in multiple 3D apps.

Ref T79174, D3089

intern/cycles/blender/blender_sync.cpp

@@ -24,6 +24,7 @@
 #include "render/mesh.h"
 #include "render/nodes.h"
 #include "render/object.h"
+#include "render/procedural.h"
 #include "render/scene.h"
 #include "render/shader.h"
 
@@ -58,6 +59,7 @@ BlenderSync::BlenderSync(BL::RenderEngine &b_engine,
       b_scene(b_scene),
       shader_map(scene),
       object_map(scene),
+      procedural_map(scene),
       geometry_map(scene),
       light_map(scene),
       particle_system_map(scene),

intern/cycles/render/CMakeLists.txt

@@ -48,6 +48,7 @@ set(SRC
   mesh_displace.cpp
   mesh_subdivision.cpp
   nodes.cpp
+  procedural.cpp
   object.cpp
   osl.cpp
   particles.cpp
@@ -90,6 +91,7 @@ set(SRC_HEADERS
   object.h
   osl.h
   particles.h
+  procedural.h
   curves.h
   scene.h
   session.h

intern/cycles/render/procedural.cpp

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2011-2018 Blender Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "procedural.h"
+
+#include "render/scene.h"
+#include "render/stats.h"
+
+#include "util/util_foreach.h"
+#include "util/util_progress.h"
+
+CCL_NAMESPACE_BEGIN
+
+NODE_ABSTRACT_DEFINE(Procedural)
+{
+  NodeType *type = NodeType::add("procedural_base", NULL);
+  return type;
+}
+
+Procedural::Procedural(const NodeType *type) : Node(type)
+{
+}
+
+Procedural::~Procedural()
+{
+}
+
+ProceduralManager::ProceduralManager()
+{
+  need_update_ = true;
+}
+
+ProceduralManager::~ProceduralManager()
+{
+}
+
+void ProceduralManager::update(Scene *scene, Progress &progress)
+{
+  if (!need_update()) {
+    return;
+  }
+
+  progress.set_status("Updating Procedurals");
+
+  scoped_callback_timer timer([scene](double time) {
+    if (scene->update_stats) {
+      scene->update_stats->procedurals.times.add_entry({"update", time});
+    }
+  });
+
+  foreach (Procedural *procedural, scene->procedurals) {
+    if (progress.get_cancel()) {
+      return;
+    }
+
+    procedural->generate(scene, progress);
+  }
+
+  if (progress.get_cancel()) {
+    return;
+  }
+
+  need_update_ = false;
+}
+
+void ProceduralManager::tag_update()
+{
+  need_update_ = true;
+}
+
+bool ProceduralManager::need_update() const
+{
+  return need_update_;
+}
+
+CCL_NAMESPACE_END

intern/cycles/render/procedural.h

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2011-2018 Blender Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "graph/node.h"
+
+CCL_NAMESPACE_BEGIN
+
+class Progress;
+class Scene;
+
+/* A Procedural is a Node which can create other Nodes before rendering starts.
+ *
+ * The Procedural is supposed to be the owner of any nodes that it creates. It can also create
+ * Nodes directly in the Scene (through Scene.create_node), it should still be set as the owner of
+ * those Nodes.
+ */
+class Procedural : public Node, public NodeOwner {
+ public:
+  NODE_ABSTRACT_DECLARE
+
+  explicit Procedural(const NodeType *type);
+  virtual ~Procedural();
+
+  /* Called each time the ProceduralManager is tagged for an update, this function is the entry
+   * point for the data generated by this Procedural. */
+  virtual void generate(Scene *scene, Progress &progress) = 0;
+
+  /* Create a node and set this Procedural as the owner. */
+  template<typename T> T *create_node()
+  {
+    T *node = new T();
+    node->set_owner(this);
+    return node;
+  }
+
+  /* Delete a Node created and owned by this Procedural. */
+  template<typename T> void delete_node(T *node)
+  {
+    assert(node->get_owner() == this);
+    delete node;
+  }
+};
+
+class ProceduralManager {
+  bool need_update_;
+
+ public:
+  ProceduralManager();
+  ~ProceduralManager();
+
+  void update(Scene *scene, Progress &progress);
+
+  void tag_update();
+
+  bool need_update() const;
+};
+
+CCL_NAMESPACE_END

intern/cycles/render/scene.cpp

@@ -29,6 +29,7 @@
 #include "render/object.h"
 #include "render/osl.h"
 #include "render/particles.h"
+#include "render/procedural.h"
 #include "render/scene.h"
 #include "render/session.h"
 #include "render/shader.h"
@@ -114,6 +115,7 @@ Scene::Scene(const SceneParams &params_, Device *device)
   image_manager = new ImageManager(device->info);
   particle_system_manager = new ParticleSystemManager();
   bake_manager = new BakeManager();
+  procedural_manager = new ProceduralManager();
   kernels_loaded = false;
 
   /* TODO(sergey): Check if it's indeed optimal value for the split kernel. */
@@ -142,6 +144,9 @@ void Scene::free_memory(bool final)
 
   foreach (Shader *s, shaders)
     delete s;
+  /* delete procedurals before other types as they may hold pointers to those types */
+  foreach (Procedural *p, procedurals)
+    delete p;
   foreach (Geometry *g, geometry)
     delete g;
   foreach (Object *o, objects)
@@ -156,6 +161,7 @@ void Scene::free_memory(bool final)
   objects.clear();
   lights.clear();
   particle_systems.clear();
+  procedurals.clear();
 
   if (device) {
     camera->device_free(device, &dscene, this);
@@ -195,6 +201,7 @@ void Scene::free_memory(bool final)
     delete image_manager;
     delete bake_manager;
     delete update_stats;
+    delete procedural_manager;
   }
 }
 
@@ -236,6 +243,11 @@ void Scene::device_update(Device *device_, Progress &progress)
   if (progress.get_cancel() || device->have_error())
     return;
 
+  procedural_manager->update(this, progress);
+
+  if (progress.get_cancel())
+    return;
+
   progress.set_status("Updating Background");
   background->device_update(device, &dscene, this);
 
@@ -391,7 +403,7 @@ bool Scene::need_data_update()
           light_manager->need_update() || lookup_tables->need_update() ||
           integrator->is_modified() || shader_manager->need_update() ||
           particle_system_manager->need_update() || bake_manager->need_update() ||
-          film->is_modified());
+          film->is_modified() || procedural_manager->need_update());
 }
 
 bool Scene::need_reset()
@@ -416,6 +428,7 @@ void Scene::reset()
   geometry_manager->tag_update(this, GeometryManager::UPDATE_ALL);
   light_manager->tag_update(this, LightManager::UPDATE_ALL);
   particle_system_manager->tag_update(this);
+  procedural_manager->tag_update();
 }
 
 void Scene::device_free()
@@ -726,6 +739,12 @@ template<> void Scene::delete_node_impl(Shader * /*node*/)
   /* don't delete unused shaders, not supported */
 }
 
+template<> void Scene::delete_node_impl(Procedural *node)
+{
+  delete_node_from_array(procedurals, node);
+  procedural_manager->tag_update();
+}
+
 template<typename T>
 static void remove_nodes_in_set(const set<T *> &nodes_set,
                                 vector<T *> &nodes_array,
@@ -780,4 +799,10 @@ template<> void Scene::delete_nodes(const set<Shader *> & /*nodes*/, const NodeO
   /* don't delete unused shaders, not supported */
 }
 
+template<> void Scene::delete_nodes(const set<Procedural *> &nodes, const NodeOwner *owner)
+{
+  remove_nodes_in_set(nodes, procedurals, owner);
+  procedural_manager->tag_update();
+}
+
 CCL_NAMESPACE_END

intern/cycles/render/scene.h

@@ -53,6 +53,8 @@ class Object;
 class ObjectManager;
 class ParticleSystemManager;
 class ParticleSystem;
+class Procedural;
+class ProceduralManager;
 class CurveSystemManager;
 class Shader;
 class ShaderManager;
@@ -236,6 +238,7 @@ class Scene : public NodeOwner {
   vector<Light *> lights;
   vector<ParticleSystem *> particle_systems;
   vector<Pass> passes;
+  vector<Procedural *> procedurals;
 
   /* data managers */
   ImageManager *image_manager;
@@ -245,6 +248,7 @@ class Scene : public NodeOwner {
   ObjectManager *object_manager;
   ParticleSystemManager *particle_system_manager;
   BakeManager *bake_manager;
+  ProceduralManager *procedural_manager;
 
   /* default shaders */
   Shader *default_surface;
@@ -395,6 +399,8 @@ template<> void Scene::delete_node_impl(ParticleSystem *node);
 
 template<> void Scene::delete_node_impl(Shader *node);
 
+template<> void Scene::delete_node_impl(Procedural *node);
+
 template<> void Scene::delete_nodes(const set<Light *> &nodes, const NodeOwner *owner);
 
 template<> void Scene::delete_nodes(const set<Geometry *> &nodes, const NodeOwner *owner);
@@ -405,6 +411,8 @@ template<> void Scene::delete_nodes(const set<ParticleSystem *> &nodes, const No
 
 template<> void Scene::delete_nodes(const set<Shader *> &nodes, const NodeOwner *owner);
 
+template<> void Scene::delete_nodes(const set<Procedural *> &nodes, const NodeOwner *owner);
+
 CCL_NAMESPACE_END
 
 #endif /*  __SCENE_H__ */

intern/cycles/render/shader.cpp

@@ -366,6 +366,7 @@ void Shader::tag_update(Scene *scene)
   if (attributes.modified(prev_attributes)) {
     need_update_attribute = true;
     scene->geometry_manager->tag_update(scene, GeometryManager::SHADER_ATTRIBUTE_MODIFIED);
+    scene->procedural_manager->tag_update();
   }
 
   if (has_volume != prev_has_volume || volume_step_rate != prev_volume_step_rate) {

intern/cycles/render/stats.cpp

@@ -375,6 +375,7 @@ string SceneUpdateStats::full_report()
   result += "Particles:\n" + particles.full_report(1);
   result += "SVM:\n" + svm.full_report(1);
   result += "Tables:\n" + tables.full_report(1);
+  result += "Procedurals:\n" + procedurals.full_report(1);
   return result;
 }
 
@@ -394,6 +395,7 @@ void SceneUpdateStats::clear()
   scene.times.clear();
   svm.times.clear();
   tables.times.clear();
+  procedurals.times.clear();
 }
 
 CCL_NAMESPACE_END

intern/cycles/render/stats.h

@@ -219,6 +219,7 @@ class SceneUpdateStats {
   UpdateTimeStats scene;
   UpdateTimeStats svm;
   UpdateTimeStats tables;
+  UpdateTimeStats procedurals;
 
   string full_report();