Tests: run suites instead of individual test cases

Group all tests of a test suite into a single test command invocation.
This reduces the number of invocations by `ctest` by an order of
magnitude.

Since rB56aa5b0d8c6b663, `bin/tests/blender_test` was run for every
individual test. Having over a 1000 tests made testing slower than
necessary. Individual tests can still be run if desired by invocation of
`bin/tests/blender_test --gtest_filter=suitename.testname`.

NOTE: For this commit to have an immediate effect, it may be necessary
to remove the `tests` and `Testing` directories and some CMake files
from your build directory and rebuild. Run `ctest -N` to see the list of
tests; there should be less than 200.

Reviewed By: sergey, LazyDodo, sebbas

Maniphest Tasks: T83222

Differential Revision: https://developer.blender.org/D9649
This commit is contained in:
Sybren A. Stüvel 2021-01-11 11:29:30 +01:00
parent 63cea2b793
commit 54f8a5dd73
6 changed files with 82 additions and 406 deletions

@ -1773,6 +1773,10 @@ elseif(WITH_CYCLES_STANDALONE)
endif()
endif()
#-----------------------------------------------------------------------------
# Testing
add_subdirectory(tests)
#-----------------------------------------------------------------------------
# Blender Application
if(WITH_BLENDER)
@ -1780,11 +1784,6 @@ if(WITH_BLENDER)
endif()
#-----------------------------------------------------------------------------
# Testing
add_subdirectory(tests)
#-----------------------------------------------------------------------------
# Define 'heavy' submodules (for Ninja builder when using pools).
setup_heavy_lib_pool()

@ -330,6 +330,9 @@ function(gtest_add_tests)
set(gtest_case_name_regex ".*\\( *([A-Za-z_0-9]+) *, *([A-Za-z_0-9]+) *\\).*")
set(gtest_test_type_regex "(TYPED_TEST|TEST_?[FP]?)")
# This will get a filter for each test suite.
set(test_filters "")
foreach(source IN LISTS ARGS_SOURCES)
if(NOT ARGS_SKIP_DEPENDENCY)
set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${source})
@ -376,175 +379,32 @@ function(gtest_add_tests)
list(APPEND testList ${ctest_test_name})
endif()
else()
set(ctest_test_name ${ARGS_TEST_PREFIX}${gtest_test_name}${ARGS_TEST_SUFFIX})
add_test(NAME ${ctest_test_name}
${workDir}
COMMAND ${ARGS_TARGET}
--gtest_filter=${gtest_test_name}
${ARGS_EXTRA_ARGS}
)
list(APPEND testList ${ctest_test_name})
# BLENDER: collect tests named "suite.testcase" as list of "suite.*" filters.
string(REGEX REPLACE "\\..*$" "" gtest_suite_name ${gtest_test_name})
list(APPEND test_filters "${gtest_suite_name}.*")
endif()
endforeach()
endforeach()
# Join all found GTest suite names into one big filter.
list(REMOVE_DUPLICATES test_filters)
list(JOIN test_filters ":" gtest_filter)
add_test(NAME ${ARGS_TEST_PREFIX}
${workDir}
COMMAND ${ARGS_TARGET}
--gtest_filter=${gtest_filter}
${ARGS_EXTRA_ARGS}
)
list(APPEND testList ${ARGS_TEST_PREFIX})
if(ARGS_TEST_LIST)
set(${ARGS_TEST_LIST} ${testList} PARENT_SCOPE)
endif()
endfunction()
#------------------------------------------------------------------------------
function(gtest_discover_tests TARGET)
cmake_parse_arguments(
""
"NO_PRETTY_TYPES;NO_PRETTY_VALUES"
"TEST_PREFIX;TEST_SUFFIX;WORKING_DIRECTORY;TEST_LIST;DISCOVERY_TIMEOUT;XML_OUTPUT_DIR;DISCOVERY_MODE"
"EXTRA_ARGS;PROPERTIES"
${ARGN}
)
if(NOT _WORKING_DIRECTORY)
set(_WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
endif()
if(NOT _TEST_LIST)
set(_TEST_LIST ${TARGET}_TESTS)
endif()
if(NOT _DISCOVERY_TIMEOUT)
set(_DISCOVERY_TIMEOUT 5)
endif()
if(NOT _DISCOVERY_MODE)
if(NOT CMAKE_GTEST_DISCOVER_TESTS_DISCOVERY_MODE)
set(CMAKE_GTEST_DISCOVER_TESTS_DISCOVERY_MODE "POST_BUILD")
endif()
set(_DISCOVERY_MODE ${CMAKE_GTEST_DISCOVER_TESTS_DISCOVERY_MODE})
endif()
get_property(
has_counter
TARGET ${TARGET}
PROPERTY CTEST_DISCOVERED_TEST_COUNTER
SET
)
if(has_counter)
get_property(
counter
TARGET ${TARGET}
PROPERTY CTEST_DISCOVERED_TEST_COUNTER
)
math(EXPR counter "${counter} + 1")
else()
set(counter 1)
endif()
set_property(
TARGET ${TARGET}
PROPERTY CTEST_DISCOVERED_TEST_COUNTER
${counter}
)
# Define rule to generate test list for aforementioned test executable
# Blender: use _ instead of [] to avoid problems with zsh regex.
set(ctest_file_base "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}_${counter}_")
set(ctest_include_file "${ctest_file_base}_include.cmake")
set(ctest_tests_file "${ctest_file_base}_tests.cmake")
get_property(crosscompiling_emulator
TARGET ${TARGET}
PROPERTY CROSSCOMPILING_EMULATOR
)
if(_DISCOVERY_MODE STREQUAL "POST_BUILD")
add_custom_command(
TARGET ${TARGET} POST_BUILD
BYPRODUCTS "${ctest_tests_file}"
COMMAND "${CMAKE_COMMAND}"
-D "TEST_TARGET=${TARGET}"
-D "TEST_EXECUTABLE=$<TARGET_FILE:${TARGET}>"
-D "TEST_EXECUTOR=${crosscompiling_emulator}"
-D "TEST_WORKING_DIR=${_WORKING_DIRECTORY}"
-D "TEST_EXTRA_ARGS=${_EXTRA_ARGS}"
-D "TEST_PROPERTIES=${_PROPERTIES}"
-D "TEST_PREFIX=${_TEST_PREFIX}"
-D "TEST_SUFFIX=${_TEST_SUFFIX}"
-D "NO_PRETTY_TYPES=${_NO_PRETTY_TYPES}"
-D "NO_PRETTY_VALUES=${_NO_PRETTY_VALUES}"
-D "TEST_LIST=${_TEST_LIST}"
-D "CTEST_FILE=${ctest_tests_file}"
-D "TEST_DISCOVERY_TIMEOUT=${_DISCOVERY_TIMEOUT}"
-D "TEST_XML_OUTPUT_DIR=${_XML_OUTPUT_DIR}"
-P "${_GOOGLETEST_DISCOVER_TESTS_SCRIPT}"
VERBATIM
)
file(WRITE "${ctest_include_file}"
"if(EXISTS \"${ctest_tests_file}\")\n"
" include(\"${ctest_tests_file}\")\n"
"else()\n"
" add_test(${TARGET}_NOT_BUILT ${TARGET}_NOT_BUILT)\n"
"endif()\n"
)
elseif(_DISCOVERY_MODE STREQUAL "PRE_TEST")
get_property(GENERATOR_IS_MULTI_CONFIG GLOBAL
PROPERTY GENERATOR_IS_MULTI_CONFIG
)
if(GENERATOR_IS_MULTI_CONFIG)
set(ctest_tests_file "${ctest_file_base}_tests-$<CONFIG>.cmake")
endif()
string(CONCAT ctest_include_content
"if(EXISTS \"$<TARGET_FILE:${TARGET}>\")" "\n"
" if(\"$<TARGET_FILE:${TARGET}>\" IS_NEWER_THAN \"${ctest_tests_file}\")" "\n"
" include(\"${_GOOGLETEST_DISCOVER_TESTS_SCRIPT}\")" "\n"
" gtest_discover_tests_impl(" "\n"
" TEST_EXECUTABLE" " [==[" "$<TARGET_FILE:${TARGET}>" "]==]" "\n"
" TEST_EXECUTOR" " [==[" "${crosscompiling_emulator}" "]==]" "\n"
" TEST_WORKING_DIR" " [==[" "${_WORKING_DIRECTORY}" "]==]" "\n"
" TEST_EXTRA_ARGS" " [==[" "${_EXTRA_ARGS}" "]==]" "\n"
" TEST_PROPERTIES" " [==[" "${_PROPERTIES}" "]==]" "\n"
" TEST_PREFIX" " [==[" "${_TEST_PREFIX}" "]==]" "\n"
" TEST_SUFFIX" " [==[" "${_TEST_SUFFIX}" "]==]" "\n"
" NO_PRETTY_TYPES" " [==[" "${_NO_PRETTY_TYPES}" "]==]" "\n"
" NO_PRETTY_VALUES" " [==[" "${_NO_PRETTY_VALUES}" "]==]" "\n"
" TEST_LIST" " [==[" "${_TEST_LIST}" "]==]" "\n"
" CTEST_FILE" " [==[" "${ctest_tests_file}" "]==]" "\n"
" TEST_DISCOVERY_TIMEOUT" " [==[" "${_DISCOVERY_TIMEOUT}" "]==]" "\n"
" TEST_XML_OUTPUT_DIR" " [==[" "${_XML_OUTPUT_DIR}" "]==]" "\n"
" )" "\n"
" endif()" "\n"
" include(\"${ctest_tests_file}\")" "\n"
"else()" "\n"
" add_test(${TARGET}_NOT_BUILT ${TARGET}_NOT_BUILT)" "\n"
"endif()" "\n"
)
if(GENERATOR_IS_MULTI_CONFIG)
foreach(_config ${CMAKE_CONFIGURATION_TYPES})
file(GENERATE OUTPUT "${ctest_file_base}_include-${_config}.cmake" CONTENT "${ctest_include_content}" CONDITION $<CONFIG:${_config}>)
endforeach()
file(WRITE "${ctest_include_file}" "include(\"${ctest_file_base}_include-\${CTEST_CONFIGURATION_TYPE}.cmake\")")
else()
file(GENERATE OUTPUT "${ctest_file_base}_include.cmake" CONTENT "${ctest_include_content}")
file(WRITE "${ctest_include_file}" "include(\"${ctest_file_base}_include.cmake\")")
endif()
else()
message(FATAL_ERROR "Unknown DISCOVERY_MODE: ${_DISCOVERY_MODE}")
endif()
# Add discovered tests to directory TEST_INCLUDE_FILES
set_property(DIRECTORY
APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}"
)
endfunction()
###############################################################################
set(_GOOGLETEST_DISCOVER_TESTS_SCRIPT
${CMAKE_CURRENT_LIST_DIR}/GTestAddTests.cmake
)
# BLENDER: remove the discovery function gtest_discover_tests(). It's not used,
# as it generates too many test invocations.
# Restore project's policies
cmake_policy(POP)

@ -1,194 +0,0 @@
# Distributed under the OSI-approved BSD 3-Clause License,
# see accompanying file BSD-3-Clause-license.txt for details.
# Changes made to this script have been marked with "BLENDER".
# BLENDER: disable ASAN leak detection when trying to discover tests.
set(ENV{ASAN_OPTIONS} "detect_leaks=0")
cmake_minimum_required(VERSION ${CMAKE_VERSION})
# Overwrite possibly existing ${_CTEST_FILE} with empty file
set(flush_tests_MODE WRITE)
# Flushes script to ${_CTEST_FILE}
macro(flush_script)
file(${flush_tests_MODE} "${_CTEST_FILE}" "${script}")
set(flush_tests_MODE APPEND)
set(script "")
endmacro()
# Flushes tests_buffer to tests
macro(flush_tests_buffer)
list(APPEND tests "${tests_buffer}")
set(tests_buffer "")
endmacro()
macro(add_command NAME)
set(_args "")
foreach(_arg ${ARGN})
if(_arg MATCHES "[^-./:a-zA-Z0-9_]")
string(APPEND _args " [==[${_arg}]==]")
else()
string(APPEND _args " ${_arg}")
endif()
endforeach()
string(APPEND script "${NAME}(${_args})\n")
string(LENGTH "${script}" _script_len)
if(${_script_len} GREATER "50000")
flush_script()
endif()
# Unsets macro local variables to prevent leakage outside of this macro.
unset(_args)
unset(_script_len)
endmacro()
function(gtest_discover_tests_impl)
cmake_parse_arguments(
""
""
"NO_PRETTY_TYPES;NO_PRETTY_VALUES;TEST_EXECUTABLE;TEST_EXECUTOR;TEST_WORKING_DIR;TEST_PREFIX;TEST_SUFFIX;TEST_LIST;CTEST_FILE;TEST_DISCOVERY_TIMEOUT;TEST_XML_OUTPUT_DIR"
"TEST_EXTRA_ARGS;TEST_PROPERTIES"
${ARGN}
)
set(prefix "${_TEST_PREFIX}")
set(suffix "${_TEST_SUFFIX}")
set(extra_args ${_TEST_EXTRA_ARGS})
set(properties ${_TEST_PROPERTIES})
set(script)
set(suite)
set(tests)
set(tests_buffer)
# Run test executable to get list of available tests
if(NOT EXISTS "${_TEST_EXECUTABLE}")
message(FATAL_ERROR
"Specified test executable does not exist.\n"
" Path: '${_TEST_EXECUTABLE}'"
)
endif()
execute_process(
COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" --gtest_list_tests
WORKING_DIRECTORY "${_TEST_WORKING_DIR}"
TIMEOUT ${_TEST_DISCOVERY_TIMEOUT}
OUTPUT_VARIABLE output
RESULT_VARIABLE result
)
if(NOT ${result} EQUAL 0)
string(REPLACE "\n" "\n " output "${output}")
message(FATAL_ERROR
"Error running test executable.\n"
" Path: '${_TEST_EXECUTABLE}'\n"
" Result: ${result}\n"
" Output:\n"
" ${output}\n"
)
endif()
# Preserve semicolon in test-parameters
string(REPLACE [[;]] [[\;]] output "${output}")
string(REPLACE "\n" ";" output "${output}")
# Parse output
foreach(line ${output})
# Skip header
if(NOT line MATCHES "gtest_main\\.cc")
# Do we have a module name or a test name?
if(NOT line MATCHES "^ ")
# Module; remove trailing '.' to get just the name...
string(REGEX REPLACE "\\.( *#.*)?" "" suite "${line}")
if(line MATCHES "#" AND NOT _NO_PRETTY_TYPES)
string(REGEX REPLACE "/[0-9]\\.+ +#.*= +" "/" pretty_suite "${line}")
else()
set(pretty_suite "${suite}")
endif()
string(REGEX REPLACE "^DISABLED_" "" pretty_suite "${pretty_suite}")
else()
# Test name; strip spaces and comments to get just the name...
string(REGEX REPLACE " +" "" test "${line}")
if(test MATCHES "#" AND NOT _NO_PRETTY_VALUES)
string(REGEX REPLACE "/[0-9]+#GetParam..=" "/" pretty_test "${test}")
else()
string(REGEX REPLACE "#.*" "" pretty_test "${test}")
endif()
string(REGEX REPLACE "^DISABLED_" "" pretty_test "${pretty_test}")
string(REGEX REPLACE "#.*" "" test "${test}")
if(NOT "${_TEST_XML_OUTPUT_DIR}" STREQUAL "")
set(TEST_XML_OUTPUT_PARAM "--gtest_output=xml:${_TEST_XML_OUTPUT_DIR}/${prefix}${suite}.${test}${suffix}.xml")
else()
unset(TEST_XML_OUTPUT_PARAM)
endif()
# sanitize test name for further processing downstream
set(testname "${prefix}${pretty_suite}.${pretty_test}${suffix}")
# escape \
string(REPLACE [[\]] [[\\]] testname "${testname}")
# escape ;
string(REPLACE [[;]] [[\;]] testname "${testname}")
# escape $
string(REPLACE [[$]] [[\$]] testname "${testname}")
# ...and add to script
add_command(add_test
"${testname}"
${_TEST_EXECUTOR}
"${_TEST_EXECUTABLE}"
"--gtest_filter=${suite}.${test}"
"--gtest_also_run_disabled_tests"
${TEST_XML_OUTPUT_PARAM}
${extra_args}
)
if(suite MATCHES "^DISABLED" OR test MATCHES "^DISABLED")
add_command(set_tests_properties
"${testname}"
PROPERTIES DISABLED TRUE
)
endif()
add_command(set_tests_properties
"${testname}"
PROPERTIES
WORKING_DIRECTORY "${_TEST_WORKING_DIR}"
SKIP_REGULAR_EXPRESSION "\\\\[ SKIPPED \\\\]"
${properties}
)
list(APPEND tests_buffer "${testname}")
list(LENGTH tests_buffer tests_buffer_length)
if(${tests_buffer_length} GREATER "250")
flush_tests_buffer()
endif()
endif()
endif()
endforeach()
# Create a list of all discovered tests, which users may use to e.g. set
# properties on the tests
flush_tests_buffer()
add_command(set ${_TEST_LIST} ${tests})
# Write CTest script
flush_script()
endfunction()
if(CMAKE_SCRIPT_MODE_FILE)
gtest_discover_tests_impl(
NO_PRETTY_TYPES ${NO_PRETTY_TYPES}
NO_PRETTY_VALUES ${NO_PRETTY_VALUES}
TEST_EXECUTABLE ${TEST_EXECUTABLE}
TEST_EXECUTOR ${TEST_EXECUTOR}
TEST_WORKING_DIR ${TEST_WORKING_DIR}
TEST_PREFIX ${TEST_PREFIX}
TEST_SUFFIX ${TEST_SUFFIX}
TEST_LIST ${TEST_LIST}
CTEST_FILE ${CTEST_FILE}
TEST_DISCOVERY_TIMEOUT ${TEST_DISCOVERY_TIMEOUT}
TEST_XML_OUTPUT_DIR ${TEST_XML_OUTPUT_DIR}
TEST_EXTRA_ARGS ${TEST_EXTRA_ARGS}
TEST_PROPERTIES ${TEST_PROPERTIES}
)
endif()

@ -8,6 +8,17 @@
#
#=============================================================================
function(GET_BLENDER_TEST_INSTALL_DIR VARIABLE_NAME)
get_property(GENERATOR_IS_MULTI_CONFIG GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
if(GENERATOR_IS_MULTI_CONFIG)
string(REPLACE "\${BUILD_TYPE}" "$<CONFIG>" TEST_INSTALL_DIR ${CMAKE_INSTALL_PREFIX})
else()
string(REPLACE "\${BUILD_TYPE}" "" TEST_INSTALL_DIR ${CMAKE_INSTALL_PREFIX})
endif()
set(${VARIABLE_NAME} "${TEST_INSTALL_DIR}" PARENT_SCOPE)
endfunction()
macro(BLENDER_SRC_GTEST_EX)
if(WITH_GTESTS)
set(options SKIP_ADD_TEST)
@ -75,13 +86,7 @@ macro(BLENDER_SRC_GTEST_EX)
target_link_libraries(${TARGET_NAME} ${GMP_LIBRARIES})
endif()
get_property(GENERATOR_IS_MULTI_CONFIG GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
if(GENERATOR_IS_MULTI_CONFIG)
string(REPLACE "\${BUILD_TYPE}" "$<CONFIG>" TEST_INSTALL_DIR ${CMAKE_INSTALL_PREFIX})
else()
string(REPLACE "\${BUILD_TYPE}" "" TEST_INSTALL_DIR ${CMAKE_INSTALL_PREFIX})
endif()
GET_BLENDER_TEST_INSTALL_DIR(TEST_INSTALL_DIR)
set_target_properties(${TARGET_NAME} PROPERTIES
RUNTIME_OUTPUT_DIRECTORY "${TESTS_OUTPUT_DIR}"
RUNTIME_OUTPUT_DIRECTORY_RELEASE "${TESTS_OUTPUT_DIR}"

@ -388,6 +388,43 @@ function(blender_add_lib
set_property(GLOBAL APPEND PROPERTY BLENDER_LINK_LIBS ${name})
endfunction()
function(blender_add_test_suite)
if (ARGC LESS 1)
message(FATAL_ERROR "No arguments supplied to blender_add_test_suite()")
endif()
# Parse the arguments
set(oneValueArgs TARGET SUITE_NAME)
set(multiValueArgs SOURCES)
cmake_parse_arguments(ARGS "" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
# Figure out the release dir, as some tests need files from there.
GET_BLENDER_TEST_INSTALL_DIR(TEST_INSTALL_DIR)
if(APPLE)
set(_test_release_dir ${TEST_INSTALL_DIR}/Blender.app/Contents/Resources/${BLENDER_VERSION})
else()
if(WIN32 OR WITH_INSTALL_PORTABLE)
set(_test_release_dir ${TEST_INSTALL_DIR}/${BLENDER_VERSION})
else()
set(_test_release_dir ${TEST_INSTALL_DIR}/share/blender/${BLENDER_VERSION})
endif()
endif()
# Define a test case with our custom gtest_add_tests() command.
include(GTest)
gtest_add_tests(
TARGET ${ARGS_TARGET}
SOURCES "${ARGS_SOURCES}"
TEST_PREFIX ${ARGS_SUITE_NAME}
WORKING_DIRECTORY "${TEST_INSTALL_DIR}"
EXTRA_ARGS
--test-assets-dir "${CMAKE_SOURCE_DIR}/../lib/tests"
--test-release-dir "${_test_release_dir}"
)
unset(_test_release_dir)
endfunction()
# Add tests for a Blender library, to be called in tandem with blender_add_lib().
# The tests will be part of the blender_test executable (see tests/gtests/runner).
function(blender_add_test_lib
@ -421,6 +458,12 @@ function(blender_add_test_lib
blender_add_lib__impl(${name} "${sources}" "${includes}" "${includes_sys}" "${library_deps}")
set_property(GLOBAL APPEND PROPERTY BLENDER_TEST_LIBS ${name})
blender_add_test_suite(
TARGET blender_test
SUITE_NAME ${name}
SOURCES "${sources}"
)
endfunction()
@ -454,14 +497,10 @@ function(blender_add_test_executable
SKIP_ADD_TEST
)
include(GTest)
set(_GOOGLETEST_DISCOVER_TESTS_SCRIPT
${CMAKE_SOURCE_DIR}/build_files/cmake/Modules/GTestAddTests.cmake
)
gtest_discover_tests(${name}_test
DISCOVERY_MODE PRE_TEST
WORKING_DIRECTORY "${TEST_INSTALL_DIR}"
blender_add_test_suite(
TARGET ${name}_test
SUITE_NAME ${name}
SOURCES "${sources}"
)
endfunction()

@ -71,36 +71,3 @@ elseif(APPLE)
endif()
unset(_test_libs)
# This runs the blender_test executable with `--gtest_list_tests`, then
# exposes those tests individually to the ctest runner.
# See https://cmake.org/cmake/help/v3.18/module/GoogleTest.html
#
# We have our own modified copy of this CMake module.
include(GTest)
set(_GOOGLETEST_DISCOVER_TESTS_SCRIPT
${CMAKE_SOURCE_DIR}/build_files/cmake/Modules/GTestAddTests.cmake
)
if(APPLE)
set(_test_release_dir ${TEST_INSTALL_DIR}/Blender.app/Contents/Resources/${BLENDER_VERSION})
else()
if(WIN32 OR WITH_INSTALL_PORTABLE)
set(_test_release_dir ${TEST_INSTALL_DIR}/${BLENDER_VERSION})
else()
set(_test_release_dir ${TEST_INSTALL_DIR}/share/blender/${BLENDER_VERSION})
endif()
endif()
gtest_discover_tests(blender_test
WORKING_DIRECTORY "${TEST_INSTALL_DIR}"
# So that it will run after the install phase that will copy the required libraries
DISCOVERY_MODE PRE_TEST
# So that unit tests know where to find files:
EXTRA_ARGS
--test-assets-dir "${CMAKE_SOURCE_DIR}/../lib/tests"
--test-release-dir "${_test_release_dir}"
)
unset(_test_release_dir)