mirror of
https://gitlab.kitware.com/vtk/vtk-m
synced 2024-09-08 13:23:51 +00:00
159 lines
6.1 KiB
CMake
159 lines
6.1 KiB
CMake
##============================================================================
|
|
## Copyright (c) Kitware, Inc.
|
|
## All rights reserved.
|
|
## See LICENSE.txt for details.
|
|
##
|
|
## This software is distributed WITHOUT ANY WARRANTY; without even
|
|
## the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
|
|
## PURPOSE. See the above copyright notice for more information.
|
|
##============================================================================
|
|
|
|
#-----------------------------------------------------------------------------
|
|
# Adds a performance benchmark test
|
|
#
|
|
# Usage:
|
|
# add_benchmark_test(benchmark
|
|
# [ NAME <name> ]
|
|
# [ ARGS <args...> ]
|
|
# [ REGEX <benchmark_regex...> ]
|
|
# )
|
|
#
|
|
# benchmark: Target of an executable that uses Google Benchmark.
|
|
#
|
|
# NAME: The name given to the CMake tests. The benchmark target name is used
|
|
# if NAME is not specified.
|
|
#
|
|
# ARGS: Extra arguments passed to the benchmark executable when run.
|
|
#
|
|
# REGEX: Regular expressions that select the specific benchmarks within the binary
|
|
# to be used. It populates the Google Benchmark
|
|
# --benchmark_filter parameter. When multiple regexes are passed
|
|
# as independent positional arguments, they are joined using the "|"
|
|
# regex operator before populating the `--benchmark_filter` parameter.
|
|
#
|
|
function(add_benchmark_test benchmark)
|
|
|
|
# We need JSON support among other things for this to work
|
|
if (CMAKE_VERSION VERSION_LESS 3.19)
|
|
message(FATAL_ERROR "Performance regression testing needs CMAKE >= 3.19")
|
|
return()
|
|
endif()
|
|
|
|
###TEST VARIABLES############################################################
|
|
|
|
set(options)
|
|
set(one_value_keywords NAME)
|
|
set(multi_value_keywords ARGS REGEX)
|
|
cmake_parse_arguments(PARSE_ARGV 1 VTKm_PERF "${options}" "${one_value_keywords}" "${multi_value_keywords}")
|
|
if (VTKm_PERF_UNPARSED_ARGUMENTS)
|
|
message(FATAL_ERROR "Bad arguments to add_benchmark_test (${VTKm_PERF_UNPARSED_ARGUMENTS}).")
|
|
endif()
|
|
|
|
if (NOT VTKm_PERF_NAME)
|
|
set(VTKm_PERF_NAME ${benchmark})
|
|
endif()
|
|
|
|
if (VTKm_PERF_REGEX)
|
|
string(REPLACE ";" "|" VTKm_PERF_REGEX "${VTKm_PERF_REGEX}")
|
|
else()
|
|
set(VTKm_PERF_REGEX ".*")
|
|
endif()
|
|
|
|
set(VTKm_PERF_REMOTE_URL "https://gitlab.kitware.com/vbolea/vtk-m-benchmark-records.git")
|
|
|
|
# Parameters for the null hypothesis test
|
|
set(VTKm_PERF_ALPHA 0.05)
|
|
set(VTKm_PERF_REPETITIONS 10)
|
|
set(VTKm_PERF_MIN_TIME 1)
|
|
set(VTKm_PERF_DIST "normal")
|
|
|
|
set(VTKm_PERF_REPO "${CMAKE_BINARY_DIR}/vtk-m-benchmark-records")
|
|
set(VTKm_PERF_COMPARE_JSON "${CMAKE_BINARY_DIR}/nocommit_${VTKm_PERF_NAME}.json")
|
|
set(VTKm_PERF_STDOUT "${CMAKE_BINARY_DIR}/benchmark_${VTKm_PERF_NAME}.stdout")
|
|
set(VTKm_PERF_COMPARE_STDOUT "${CMAKE_BINARY_DIR}/compare_${VTKm_PERF_NAME}.stdout")
|
|
|
|
if (DEFINED ENV{CI_COMMIT_SHA})
|
|
set(VTKm_PERF_COMPARE_JSON "${CMAKE_BINARY_DIR}/$ENV{CI_COMMIT_SHA}_${VTKm_PERF_NAME}.json")
|
|
endif()
|
|
|
|
# Only upload when we are inside a CI build and in master. We need to check
|
|
# if VTKM_BENCH_RECORDS_TOKEN is either defined or non-empty, the reason is
|
|
# that in Gitlab CI Variables for protected branches are also defined in MR
|
|
# from forks, however, they are empty.
|
|
if (DEFINED ENV{VTKM_BENCH_RECORDS_TOKEN} AND ENV{VTKM_BENCH_RECORDS_TOKEN})
|
|
set(enable_upload TRUE)
|
|
endif()
|
|
|
|
set(test_name "PerformanceTest${VTKm_PERF_NAME}")
|
|
|
|
###TEST INVOKATIONS##########################################################
|
|
if (NOT TEST PerformanceTestFetch)
|
|
add_test(NAME "PerformanceTestFetch"
|
|
COMMAND ${CMAKE_COMMAND}
|
|
"-DVTKm_PERF_REPO=${VTKm_PERF_REPO}"
|
|
"-DVTKm_SOURCE_DIR=${VTKm_SOURCE_DIR}"
|
|
"-DVTKm_PERF_REMOTE_URL=${VTKm_PERF_REMOTE_URL}"
|
|
-P "${VTKm_SOURCE_DIR}/CMake/testing/VTKmPerformanceTestFetch.cmake"
|
|
)
|
|
set_property(TEST PerformanceTestFetch PROPERTY FIXTURES_SETUP "FixturePerformanceTestSetup")
|
|
endif()
|
|
|
|
add_test(NAME "${test_name}Run"
|
|
COMMAND ${CMAKE_COMMAND}
|
|
"-DVTKm_PERF_BENCH_DEVICE=Any"
|
|
"-DVTKm_PERF_BENCH_PATH=${CMAKE_BINARY_DIR}/bin/${benchmark}"
|
|
"-DVTKm_PERF_ARGS=${VTKm_PERF_ARGS}"
|
|
"-DVTKm_PERF_REGEX=${VTKm_PERF_REGEX}"
|
|
"-DVTKm_PERF_REPETITIONS=${VTKm_PERF_REPETITIONS}"
|
|
"-DVTKm_PERF_MIN_TIME=${VTKm_PERF_MIN_TIME}"
|
|
"-DVTKm_PERF_COMPARE_JSON=${VTKm_PERF_COMPARE_JSON}"
|
|
"-DVTKm_PERF_STDOUT=${VTKm_PERF_STDOUT}"
|
|
"-DVTKm_SOURCE_DIR=${VTKm_SOURCE_DIR}"
|
|
-P "${VTKm_SOURCE_DIR}/CMake/testing/VTKmPerformanceTestRun.cmake"
|
|
)
|
|
|
|
add_test(NAME "${test_name}Report"
|
|
COMMAND ${CMAKE_COMMAND}
|
|
"-DVTKm_BINARY_DIR=${VTKm_BINARY_DIR}"
|
|
"-DVTKm_PERF_ALPHA=${VTKm_PERF_ALPHA}"
|
|
"-DVTKm_PERF_COMPARE_JSON=${VTKm_PERF_COMPARE_JSON}"
|
|
"-DVTKm_PERF_COMPARE_STDOUT=${VTKm_PERF_COMPARE_STDOUT}"
|
|
"-DVTKm_PERF_DIST=${VTKm_PERF_DIST}"
|
|
"-DVTKm_PERF_NAME=${VTKm_PERF_NAME}"
|
|
"-DVTKm_PERF_REPO=${VTKm_PERF_REPO}"
|
|
"-DVTKm_SOURCE_DIR=${VTKm_SOURCE_DIR}"
|
|
-P "${VTKm_SOURCE_DIR}/CMake/testing/VTKmPerformanceTestReport.cmake"
|
|
)
|
|
|
|
if (enable_upload)
|
|
add_test(NAME "${test_name}Upload"
|
|
COMMAND ${CMAKE_COMMAND}
|
|
"-DVTKm_PERF_REPO=${VTKm_PERF_REPO}"
|
|
"-DVTKm_PERF_COMPARE_JSON=${VTKm_PERF_COMPARE_JSON}"
|
|
"-DVTKm_SOURCE_DIR=${VTKm_SOURCE_DIR}"
|
|
-P "${VTKm_SOURCE_DIR}/CMake/testing/VTKmPerformanceTestUpload.cmake"
|
|
)
|
|
|
|
set_tests_properties("${test_name}Upload" PROPERTIES
|
|
DEPENDS ${test_name}Report
|
|
FIXTURES_REQUIRED "FixturePerformanceTestCleanUp"
|
|
REQUIRED_FILES "${VTKm_PERF_COMPARE_JSON}"
|
|
RUN_SERIAL ON)
|
|
endif()
|
|
|
|
###TEST PROPERTIES###########################################################
|
|
set_property(TEST ${test_name}Report PROPERTY DEPENDS ${test_name}Run)
|
|
set_property(TEST ${test_name}Report PROPERTY FIXTURES_REQUIRED "FixturePerformanceTestSetup")
|
|
|
|
set_tests_properties("${test_name}Report"
|
|
PROPERTIES
|
|
REQUIRED_FILES "${VTKm_PERF_COMPARE_JSON}")
|
|
|
|
set_tests_properties("${test_name}Run"
|
|
"${test_name}Report"
|
|
"PerformanceTestFetch"
|
|
PROPERTIES RUN_SERIAL ON)
|
|
|
|
set_tests_properties(${test_name}Run PROPERTIES TIMEOUT 1800)
|
|
endfunction()
|