From 5987c4bc579d865e96ccc6ee43789981fd4f106a Mon Sep 17 00:00:00 2001 From: Lukas Stockner Date: Thu, 8 Nov 2018 01:05:55 +0100 Subject: [PATCH 1/6] Render API: Support arbitrary length for custom image metadata fields --- source/blender/blenkernel/intern/image.c | 14 +++++++------- source/blender/makesrna/intern/rna_render.c | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/source/blender/blenkernel/intern/image.c b/source/blender/blenkernel/intern/image.c index d01650e3204..1d209dec935 100644 --- a/source/blender/blenkernel/intern/image.c +++ b/source/blender/blenkernel/intern/image.c @@ -1612,7 +1612,7 @@ typedef struct StampDataCustomField { struct StampDataCustomField *next, *prev; /* TODO(sergey): Think of better size here, maybe dynamically allocated even. */ char key[512]; - char value[512]; + char *value; /* TODO(sergey): Support non-string values. */ } StampDataCustomField; @@ -2177,12 +2177,9 @@ void BKE_stamp_info_callback(void *data, struct StampData *stamp_data, StampCall CALL(rendertime, "RenderTime"); CALL(memory, "Memory"); - for (StampDataCustomField *custom_field = stamp_data->custom_fields.first; - custom_field != NULL; - custom_field = custom_field->next) - { + LISTBASE_FOREACH(StampDataCustomField *, custom_field, &stamp_data->custom_fields) { if (noskip || custom_field->value[0]) { - callback(data, custom_field->key, custom_field->value, sizeof(custom_field->value)); + callback(data, custom_field->key, custom_field->value, strlen(custom_field->value) + 1); } } @@ -2199,7 +2196,7 @@ void BKE_render_result_stamp_data(RenderResult *rr, const char *key, const char StampDataCustomField *field = MEM_mallocN(sizeof(StampDataCustomField), "StampData Custom Field"); STRNCPY(field->key, key); - STRNCPY(field->value, value); + field->value = BLI_strdup(value); BLI_addtail(&stamp_data->custom_fields, field); } @@ -2208,6 +2205,9 @@ void BKE_stamp_data_free(struct StampData *stamp_data) if (stamp_data == NULL) { return; } + LISTBASE_FOREACH(StampDataCustomField *, custom_field, &stamp_data->custom_fields) { + MEM_freeN(custom_field->value); + } BLI_freelistN(&stamp_data->custom_fields); MEM_freeN(stamp_data); } diff --git a/source/blender/makesrna/intern/rna_render.c b/source/blender/makesrna/intern/rna_render.c index 513717a9e46..b516accbf1e 100644 --- a/source/blender/makesrna/intern/rna_render.c +++ b/source/blender/makesrna/intern/rna_render.c @@ -801,7 +801,7 @@ static void rna_def_render_result(BlenderRNA *brna) RNA_def_function_ui_description(func, "Add engine-specific stamp data to the result"); parm = RNA_def_string(func, "field", NULL, 1024, "Field", "Name of the stamp field to add"); RNA_def_parameter_flags(parm, 0, PARM_REQUIRED); - parm = RNA_def_string(func, "value", NULL, 1024, "Value", "Value of the stamp data"); + parm = RNA_def_string(func, "value", NULL, 0, "Value", "Value of the stamp data"); RNA_def_parameter_flags(parm, 0, PARM_REQUIRED); RNA_define_verify_sdna(0); From 47c77cd89c37595c5deeb00a9235d88c460ce356 Mon Sep 17 00:00:00 2001 From: Lukas Stockner Date: Wed, 7 Nov 2018 04:05:47 +0100 Subject: [PATCH 2/6] Cycles: Write Cryptomatte metadata according to the specification Reviewers: brecht, sergey, swerner Subscribers: creamsurfer, Tanguy, Noss, SteffenD Differential Revision: https://developer.blender.org/D3862 --- intern/cycles/blender/blender_session.cpp | 29 +++++++++++++++++-- intern/cycles/blender/blender_session.h | 3 ++ intern/cycles/blender/blender_util.h | 6 ++++ intern/cycles/render/object.cpp | 34 +++++++++++++++++++++++ intern/cycles/render/object.h | 3 ++ intern/cycles/render/shader.cpp | 16 +++++++++++ intern/cycles/render/shader.h | 2 ++ 7 files changed, 91 insertions(+), 2 deletions(-) diff --git a/intern/cycles/blender/blender_session.cpp b/intern/cycles/blender/blender_session.cpp index e05351eea40..75c7dcee05e 100644 --- a/intern/cycles/blender/blender_session.cpp +++ b/intern/cycles/blender/blender_session.cpp @@ -35,6 +35,7 @@ #include "util/util_function.h" #include "util/util_hash.h" #include "util/util_logging.h" +#include "util/util_murmurhash.h" #include "util/util_progress.h" #include "util/util_time.h" @@ -370,6 +371,17 @@ void BlenderSession::update_render_tile(RenderTile& rtile, bool highlight) do_write_update_render_tile(rtile, false, false); } +static void add_cryptomatte_layer(BL::RenderResult& b_rr, string name, string manifest) +{ + string identifier = string_printf("%08x", util_murmur_hash3(name.c_str(), name.length(), 0)); + string prefix = "cryptomatte/" + identifier.substr(0, 7) + "/"; + + render_add_metadata(b_rr, prefix+"name", name); + render_add_metadata(b_rr, prefix+"hash", "MurmurHash3_32"); + render_add_metadata(b_rr, prefix+"conversion", "uint32_to_float32"); + render_add_metadata(b_rr, prefix+"manifest", manifest); +} + void BlenderSession::render() { /* set callback to write out render results */ @@ -477,15 +489,28 @@ void BlenderSession::render() break; } + BL::RenderResult b_full_rr = b_engine.get_result(); if(is_single_layer) { - BL::RenderResult b_rr = b_engine.get_result(); string num_aa_samples = string_printf("%d", session->params.samples); - b_rr.stamp_data_add_field("Cycles Samples", num_aa_samples.c_str()); + render_add_metadata(b_full_rr, "Cycles Samples", num_aa_samples); /* TODO(sergey): Report whether we're doing resumable render * and also start/end sample if so. */ } + if(scene->film->cryptomatte_passes & CRYPT_OBJECT) { + add_cryptomatte_layer(b_full_rr, b_rlay_name+".CryptoObject", + scene->object_manager->get_cryptomatte_objects(scene)); + } + if(scene->film->cryptomatte_passes & CRYPT_MATERIAL) { + add_cryptomatte_layer(b_full_rr, b_rlay_name+".CryptoMaterial", + scene->shader_manager->get_cryptomatte_materials(scene)); + } + if(scene->film->cryptomatte_passes & CRYPT_ASSET) { + add_cryptomatte_layer(b_full_rr, b_rlay_name+".CryptoAsset", + scene->object_manager->get_cryptomatte_assets(scene)); + } + /* free result without merging */ end_render_result(b_engine, b_rr, true, true, false); diff --git a/intern/cycles/blender/blender_session.h b/intern/cycles/blender/blender_session.h index 08f5c873bef..540fa6a8a84 100644 --- a/intern/cycles/blender/blender_session.h +++ b/intern/cycles/blender/blender_session.h @@ -169,6 +169,9 @@ protected: /* Update tile manager to reflect resumable render settings. */ void update_resumable_tile_manager(int num_samples); + + /* Add metadata for cryptomatte layers to the render result. */ + void add_cryptomatte_metadata(string layer_prefix); }; CCL_NAMESPACE_END diff --git a/intern/cycles/blender/blender_util.h b/intern/cycles/blender/blender_util.h index 7e61888348b..09ae9bea313 100644 --- a/intern/cycles/blender/blender_util.h +++ b/intern/cycles/blender/blender_util.h @@ -243,6 +243,12 @@ static inline float *image_get_float_pixels_for_frame(BL::Image& image, return BKE_image_get_float_pixels_for_frame(image.ptr.data, frame); } +static inline void render_add_metadata(BL::RenderResult& b_rr, string name, string value) +{ + b_rr.stamp_data_add_field(name.c_str(), value.c_str()); +} + + /* Utilities */ static inline Transform get_transform(const BL::Array& array) diff --git a/intern/cycles/render/object.cpp b/intern/cycles/render/object.cpp index a56a8a6ec58..dc7a1043208 100644 --- a/intern/cycles/render/object.cpp +++ b/intern/cycles/render/object.cpp @@ -27,6 +27,7 @@ #include "util/util_logging.h" #include "util/util_map.h" #include "util/util_progress.h" +#include "util/util_set.h" #include "util/util_vector.h" #include "util/util_murmurhash.h" @@ -844,4 +845,37 @@ void ObjectManager::tag_update(Scene *scene) scene->light_manager->need_update = true; } +string ObjectManager::get_cryptomatte_objects(Scene *scene) +{ + string manifest = "{"; + + unordered_set objects; + foreach(Object *object, scene->objects) { + if(objects.count(object->name)) { + continue; + } + objects.insert(object->name); + uint32_t hash_name = util_murmur_hash3(object->name.c_str(), object->name.length(), 0); + manifest += string_printf("\"%s\":\"%08x\",", object->name.c_str(), hash_name); + } + manifest[manifest.size()-1] = '}'; + return manifest; +} + +string ObjectManager::get_cryptomatte_assets(Scene *scene) +{ + string manifest = "{"; + unordered_set assets; + foreach(Object *ob, scene->objects) { + if(assets.count(ob->asset_name)) { + continue; + } + assets.insert(ob->asset_name); + uint32_t hash_asset = util_murmur_hash3(ob->asset_name.c_str(), ob->asset_name.length(), 0); + manifest += string_printf("\"%s\":\"%08x\",", ob->asset_name.c_str(), hash_asset); + } + manifest[manifest.size()-1] = '}'; + return manifest; +} + CCL_NAMESPACE_END diff --git a/intern/cycles/render/object.h b/intern/cycles/render/object.h index bd44b35aba3..288c3f5457a 100644 --- a/intern/cycles/render/object.h +++ b/intern/cycles/render/object.h @@ -116,6 +116,9 @@ public: void apply_static_transforms(DeviceScene *dscene, Scene *scene, Progress& progress); + string get_cryptomatte_objects(Scene *scene); + string get_cryptomatte_assets(Scene *scene); + protected: void device_update_object_transform(UpdateObjectTransformState *state, Object *ob, diff --git a/intern/cycles/render/shader.cpp b/intern/cycles/render/shader.cpp index 8d0cec7b14e..a827f611583 100644 --- a/intern/cycles/render/shader.cpp +++ b/intern/cycles/render/shader.cpp @@ -699,4 +699,20 @@ float ShaderManager::linear_rgb_to_gray(float3 c) return dot(c, rgb_to_y); } +string ShaderManager::get_cryptomatte_materials(Scene *scene) +{ + string manifest = "{"; + unordered_set materials; + foreach(Shader *shader, scene->shaders) { + if(materials.count(shader->name)) { + continue; + } + materials.insert(shader->name); + uint32_t cryptomatte_id = util_murmur_hash3(shader->name.c_str(), shader->name.length(), 0); + manifest += string_printf("\"%s\":\"%08x\",", shader->name.c_str(), cryptomatte_id); + } + manifest[manifest.size()-1] = '}'; + return manifest; +} + CCL_NAMESPACE_END diff --git a/intern/cycles/render/shader.h b/intern/cycles/render/shader.h index 58314a1e310..80731384048 100644 --- a/intern/cycles/render/shader.h +++ b/intern/cycles/render/shader.h @@ -197,6 +197,8 @@ public: float linear_rgb_to_gray(float3 c); + string get_cryptomatte_materials(Scene *scene); + protected: ShaderManager(); From cd9f3019f4c0bbf00d83298be705cdcf86448dd0 Mon Sep 17 00:00:00 2001 From: Ray Molenkamp Date: Wed, 7 Nov 2018 19:18:53 -0700 Subject: [PATCH 3/6] build_deps_windows: add option to to only create project files. --- .../build_environment/windows/build_deps.cmd | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/build_files/build_environment/windows/build_deps.cmd b/build_files/build_environment/windows/build_deps.cmd index 500e689cdc8..bb9dda81435 100644 --- a/build_files/build_environment/windows/build_deps.cmd +++ b/build_files/build_environment/windows/build_deps.cmd @@ -68,6 +68,8 @@ goto usage setlocal ENABLEEXTENSIONS set CMAKE_DEBUG_OPTIONS=-DWITH_OPTIMIZED_DEBUG=On if "%3" == "debug" set CMAKE_DEBUG_OPTIONS=-DWITH_OPTIMIZED_DEBUG=Off +set dobuild=1 +if "%4" == "nobuild" set dobuild=0 set SOURCE_DIR=%~dp0\.. set BUILD_DIR=%cd%\build @@ -117,20 +119,24 @@ cd %Staging%\%BuildDir%%ARCH%R echo %DATE% %TIME% : Start > %StatusFile% cmake -G "%CMAKE_BUILDER%" %SOURCE_DIR% -DDOWNLOAD_DIR=%BUILD_DIR%/downloads -DBUILD_MODE=Release -DHARVEST_TARGET=%HARVEST_DIR%/%HARVESTROOT%%VSVER_SHORT%/ echo %DATE% %TIME% : Release Configuration done >> %StatusFile% -msbuild /m "ll.vcxproj" /p:Configuration=Release /fl /flp:logfile=BlenderDeps_llvm.log;Verbosity=normal -msbuild /m "BlenderDependencies.sln" /p:Configuration=Release /fl /flp:logfile=BlenderDeps.log;Verbosity=minimal /verbosity:minimal -echo %DATE% %TIME% : Release Build done >> %StatusFile% -cmake --build . --target Harvest_Release_Results > Harvest_Release.txt +if "%dobuild%" == "1" ( + msbuild /m "ll.vcxproj" /p:Configuration=Release /fl /flp:logfile=BlenderDeps_llvm.log;Verbosity=normal + msbuild /m "BlenderDependencies.sln" /p:Configuration=Release /fl /flp:logfile=BlenderDeps.log;Verbosity=minimal /verbosity:minimal + echo %DATE% %TIME% : Release Build done >> %StatusFile% + cmake --build . --target Harvest_Release_Results > Harvest_Release.txt +) echo %DATE% %TIME% : Release Harvest done >> %StatusFile% cd %BUILD_DIR% mkdir %STAGING%\%BuildDir%%ARCH%D cd %Staging%\%BuildDir%%ARCH%D cmake -G "%CMAKE_BUILDER%" %SOURCE_DIR% -DDOWNLOAD_DIR=%BUILD_DIR%/downloads -DCMAKE_BUILD_TYPE=Debug -DBUILD_MODE=Debug -DHARVEST_TARGET=%HARVEST_DIR%/%HARVESTROOT%%VSVER_SHORT%/ %CMAKE_DEBUG_OPTIONS% echo %DATE% %TIME% : Debug Configuration done >> %StatusFile% -msbuild /m "ll.vcxproj" /p:Configuration=Debug /fl /flp:logfile=BlenderDeps_llvm.log;;Verbosity=normal -msbuild /m "BlenderDependencies.sln" /p:Configuration=Debug /verbosity:n /fl /flp:logfile=BlenderDeps.log;;Verbosity=normal -echo %DATE% %TIME% : Debug Build done >> %StatusFile% -cmake --build . --target Harvest_Debug_Results> Harvest_Debug.txt +if "%dobuild%" == "1" ( + msbuild /m "ll.vcxproj" /p:Configuration=Debug /fl /flp:logfile=BlenderDeps_llvm.log;;Verbosity=normal + msbuild /m "BlenderDependencies.sln" /p:Configuration=Debug /verbosity:n /fl /flp:logfile=BlenderDeps.log;;Verbosity=normal + echo %DATE% %TIME% : Debug Build done >> %StatusFile% + cmake --build . --target Harvest_Debug_Results> Harvest_Debug.txt +) echo %DATE% %TIME% : Debug Harvest done >> %StatusFile% cd %BUILD_DIR% From d3b11de7d6d777ace9f5b8a3626dd2b1fde095aa Mon Sep 17 00:00:00 2001 From: Ray Molenkamp Date: Wed, 7 Nov 2018 19:33:07 -0700 Subject: [PATCH 4/6] build_environment: add debug libs for embree on windows --- .../build_environment/cmake/embree.cmake | 23 +++++++++++++++++++ .../build_environment/cmake/harvest.cmake | 2 -- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/build_files/build_environment/cmake/embree.cmake b/build_files/build_environment/cmake/embree.cmake index 34e5904db5e..8c523e25f0c 100644 --- a/build_files/build_environment/cmake/embree.cmake +++ b/build_files/build_environment/cmake/embree.cmake @@ -44,3 +44,26 @@ ExternalProject_Add(external_embree CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${LIBDIR}/embree ${DEFAULT_CMAKE_FLAGS} ${EMBREE_EXTRA_ARGS} INSTALL_DIR ${LIBDIR}/embree ) +if(WIN32) + + if(BUILD_MODE STREQUAL Release) + ExternalProject_Add_Step(external_embree after_install + COMMAND ${CMAKE_COMMAND} -E copy_directory ${LIBDIR}/embree ${HARVEST_TARGET}/embree + DEPENDEES install + ) + else() + ExternalProject_Add_Step(external_embree after_install + COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/embree/lib/embree3.lib ${HARVEST_TARGET}/embree/lib/embree3_d.lib + COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/embree/lib/embree_avx.lib ${HARVEST_TARGET}/embree/lib/embree_avx_d.lib + COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/embree/lib/embree_avx2.lib ${HARVEST_TARGET}/embree/lib/embree_avx2_d.lib + COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/embree/lib/embree_sse42.lib ${HARVEST_TARGET}/embree/lib/embree_sse42_d.lib + COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/embree/lib/lexers.lib ${HARVEST_TARGET}/embree/lib/lexers_d.lib + COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/embree/lib/math.lib ${HARVEST_TARGET}/embree/lib/math_d.lib + COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/embree/lib/simd.lib ${HARVEST_TARGET}/embree/lib/simd_d.lib + COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/embree/lib/sys.lib ${HARVEST_TARGET}/embree/lib/sys_d.lib + COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/embree/lib/tasking.lib ${HARVEST_TARGET}/embree/lib/tasking_d.lib + DEPENDEES install + ) + endif() + +endif() diff --git a/build_files/build_environment/cmake/harvest.cmake b/build_files/build_environment/cmake/harvest.cmake index d0d8b4cefb9..f2e9c2fa8db 100644 --- a/build_files/build_environment/cmake/harvest.cmake +++ b/build_files/build_environment/cmake/harvest.cmake @@ -58,8 +58,6 @@ if(BUILD_MODE STREQUAL Release) ${CMAKE_COMMAND} -E copy_directory ${LIBDIR}/hidapi/ ${HARVEST_TARGET}/hidapi/ && # webp, straight up copy ${CMAKE_COMMAND} -E copy_directory ${LIBDIR}/webp ${HARVEST_TARGET}/webp && - # embree - ${CMAKE_COMMAND} -E copy_directory ${LIBDIR}/embree ${HARVEST_TARGET}/embree DEPENDS ) endif() From 406f36f5742468fe57aed9ba5537c59faa6d079a Mon Sep 17 00:00:00 2001 From: Ray Molenkamp Date: Wed, 7 Nov 2018 19:34:49 -0700 Subject: [PATCH 5/6] build: add proper embree support to platform_win32.cmake made the use of findpackage optional and add support for debug builds --- .../cmake/platform/platform_win32.cmake | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/build_files/cmake/platform/platform_win32.cmake b/build_files/cmake/platform/platform_win32.cmake index 1bf04689971..d5d0b03d505 100644 --- a/build_files/cmake/platform/platform_win32.cmake +++ b/build_files/cmake/platform/platform_win32.cmake @@ -609,7 +609,30 @@ if(WITH_CYCLES_OSL) endif() if(WITH_CYCLES_EMBREE) - find_package(Embree 3.2.4 REQUIRED) + windows_find_package(Embree) + if(NOT EMBREE_FOUND) + set(EMBREE_INCLUDE_DIRS ${LIBDIR}/embree/include) + set(EMBREE_LIBRARIES + optimized ${LIBDIR}/embree/lib/embree3.lib + optimized ${LIBDIR}/embree/lib/embree_avx2.lib + optimized ${LIBDIR}/embree/lib/embree_avx.lib + optimized ${LIBDIR}/embree/lib/embree_sse42.lib + optimized ${LIBDIR}/embree/lib/lexers.lib + optimized ${LIBDIR}/embree/lib/math.lib + optimized ${LIBDIR}/embree/lib/simd.lib + optimized ${LIBDIR}/embree/lib/sys.lib + optimized ${LIBDIR}/embree/lib/tasking.lib + + debug ${LIBDIR}/embree/lib/embree3_d.lib + debug ${LIBDIR}/embree/lib/embree_avx2_d.lib + debug ${LIBDIR}/embree/lib/embree_avx_d.lib + debug ${LIBDIR}/embree/lib/embree_sse42_d.lib + debug ${LIBDIR}/embree/lib/lexers_d.lib + debug ${LIBDIR}/embree/lib/math_d.lib + debug ${LIBDIR}/embree/lib/simd_d.lib + debug ${LIBDIR}/embree/lib/sys_d.lib + debug ${LIBDIR}/embree/lib/tasking_d.lib) + endif() endif() if (WINDOWS_PYTHON_DEBUG) From 116be3deffc087a61cdab452cd17bcaaf049a508 Mon Sep 17 00:00:00 2001 From: Brecht Van Lommel Date: Thu, 8 Nov 2018 14:48:54 +0100 Subject: [PATCH 6/6] Fix build on 32bit after Embree changes. --- intern/cycles/kernel/kernel_types.h | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/intern/cycles/kernel/kernel_types.h b/intern/cycles/kernel/kernel_types.h index 7bc9855a807..14d5ff29bd8 100644 --- a/intern/cycles/kernel/kernel_types.h +++ b/intern/cycles/kernel/kernel_types.h @@ -1411,20 +1411,23 @@ typedef enum KernelBVHLayout { } KernelBVHLayout; typedef struct KernelBVH { - /* root node */ + /* Own BVH */ int root; int have_motion; int have_curves; int have_instancing; int bvh_layout; int use_bvh_steps; - int pad1; + + /* Embree */ #ifdef __EMBREE__ RTCScene scene; +# ifndef __KERNEL_64_BIT__ + int pad1; +# endif #else - void *unused; + int pad1, pad2; #endif - int pad2, pad3; } KernelBVH; static_assert_align(KernelBVH, 16);