diff --git a/CMake/testing/VTKmPerformanceTest.cmake b/CMake/testing/VTKmPerformanceTest.cmake new file mode 100644 index 000000000..71f058862 --- /dev/null +++ b/CMake/testing/VTKmPerformanceTest.cmake @@ -0,0 +1,129 @@ +##============================================================================ +## Copyright (c) Kitware, Inc. +## All rights reserved. +## See LICENSE.txt for details. +## +## This software is distributed WITHOUT ANY WARRANTY; without even +## the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +## PURPOSE. See the above copyright notice for more information. +##============================================================================ + +#----------------------------------------------------------------------------- +# Adds a performance benchmark test +# +# add_benchmark_test(benchmark [ ]) +# +# Usage: +# add_benchmark_test(FiltersBenchmark BenchThreshold BenchTetrahedralize) +# +# benchmark: Target of an executable that uses Google Benchmark. +# +# filter_regex: CMake regexes that selects the specific benchmarks within the binary +# to be used. It populates the Google Benchmark +# --benchmark_filter parameter. When multiple regexes are passed +# as independent positional arguments, they are joined using the "|" +# regex operator before populating the `--benchmark_filter` parameter +# +function(add_benchmark_test benchmark) + + # We need JSON support among other things for this to work + if (CMAKE_VERSION VERSION_LESS 3.19) + message(FATAL_ERROR "Performance regression testing needs CMAKE >= 3.19") + return() + endif() + + ###TEST VARIABLES############################################################ + + # Optional positional parameters for filter_regex + set(VTKm_PERF_FILTER_NAME ".*") + if (${ARGC} GREATER_EQUAL 2) + string(REPLACE ";" "|" VTKm_PERF_FILTER_NAME "${ARGN}") + endif() + + set(VTKm_PERF_REMOTE_URL "https://gitlab.kitware.com/vbolea/vtk-m-benchmark-records.git") + + # Parameters for the null hypothesis test + set(VTKm_PERF_ALPHA 0.05) + set(VTKm_PERF_REPETITIONS 10) + set(VTKm_PERF_MIN_TIME 1) + set(VTKm_PERF_DIST "normal") + + set(VTKm_PERF_REPO "${CMAKE_BINARY_DIR}/vtk-m-benchmark-records") + set(VTKm_PERF_COMPARE_JSON "${CMAKE_BINARY_DIR}/nocommit_${benchmark}.json") + set(VTKm_PERF_STDOUT "${CMAKE_BINARY_DIR}/benchmark_${benchmark}.stdout") + set(VTKm_PERF_COMPARE_STDOUT "${CMAKE_BINARY_DIR}/compare_${benchmark}.stdout") + + if (DEFINED ENV{CI_COMMIT_SHA}) + set(VTKm_PERF_COMPARE_JSON "${CMAKE_BINARY_DIR}/$ENV{CI_COMMIT_SHA}_${benchmark}.json") + endif() + + set(test_name "PerformanceTest${benchmark}") + + ###TEST INVOKATIONS########################################################## + add_test(NAME "${test_name}Run" + COMMAND ${CMAKE_COMMAND} + "-DVTKm_PERF_BENCH_DEVICE=Any" + "-DVTKm_PERF_BENCH_PATH=${CMAKE_BINARY_DIR}/bin/${benchmark}" + "-DVTKm_PERF_FILTER_NAME=${VTKm_PERF_FILTER_NAME}" + "-DVTKm_PERF_REPETITIONS=${VTKm_PERF_REPETITIONS}" + "-DVTKm_PERF_MIN_TIME=${VTKm_PERF_MIN_TIME}" + "-DVTKm_PERF_COMPARE_JSON=${VTKm_PERF_COMPARE_JSON}" + "-DVTKm_PERF_STDOUT=${VTKm_PERF_STDOUT}" + "-DVTKm_SOURCE_DIR=${VTKm_SOURCE_DIR}" + -P "${VTKm_SOURCE_DIR}/CMake/testing/VTKmPerformanceTestRun.cmake" + ) + + add_test(NAME "${test_name}Fetch" + COMMAND ${CMAKE_COMMAND} + "-DVTKm_PERF_REPO=${VTKm_PERF_REPO}" + "-DVTKm_SOURCE_DIR=${VTKm_SOURCE_DIR}" + "-DVTKm_PERF_REMOTE_URL=${VTKm_PERF_REMOTE_URL}" + -P "${VTKm_SOURCE_DIR}/CMake/testing/VTKmPerformanceTestFetch.cmake" + ) + + add_test(NAME "${test_name}Upload" + COMMAND ${CMAKE_COMMAND} + "-DVTKm_PERF_REPO=${VTKm_PERF_REPO}" + "-DVTKm_PERF_COMPARE_JSON=${VTKm_PERF_COMPARE_JSON}" + "-DVTKm_SOURCE_DIR=${VTKm_SOURCE_DIR}" + -P "${VTKm_SOURCE_DIR}/CMake/testing/VTKmPerformanceTestUpload.cmake" + ) + + add_test(NAME "${test_name}Report" + COMMAND ${CMAKE_COMMAND} + "-DBENCHMARK_NAME=${benchmark}" + "-DVTKm_PERF_ALPHA=${VTKm_PERF_ALPHA}" + "-DVTKm_PERF_DIST=${VTKm_PERF_DIST}" + "-DVTKm_PERF_REPO=${VTKm_PERF_REPO}" + "-DVTKm_PERF_COMPARE_JSON=${VTKm_PERF_COMPARE_JSON}" + "-DVTKm_SOURCE_DIR=${VTKm_SOURCE_DIR}" + "-DVTKm_BINARY_DIR=${VTKm_BINARY_DIR}" + "-DVTKm_PERF_COMPARE_STDOUT=${VTKm_PERF_COMPARE_STDOUT}" + -P "${VTKm_SOURCE_DIR}/CMake/testing/VTKmPerformanceTestReport.cmake" + ) + + add_test(NAME "${test_name}CleanUp" + COMMAND ${CMAKE_COMMAND} -E rm -rf "${VTKm_PERF_REPO}" + ) + + ###TEST PROPERTIES########################################################### + set_tests_properties("${test_name}Report" "${test_name}Upload" + PROPERTIES + FIXTURE_REQUIRED "${test_name}Run;${test_name}Fetch" + FIXTURE_CLEANUP "${test_name}CleanUp" + REQUIRED_FILES "${VTKm_PERF_COMPARE_JSON}") + + set_tests_properties("${test_name}Run" + "${test_name}Report" + "${test_name}Upload" + "${test_name}Fetch" + "${test_name}CleanUp" + PROPERTIES RUN_SERIAL ON) + + set_tests_properties(${test_name}Run PROPERTIES TIMEOUT 1800) + + # Only upload when we are inside a CI build + if (NOT DEFINED ENV{CI_COMMIT_SHA} OR NOT DEFINED ENV{VTKM_BENCH_RECORDS_TOKEN}) + set_tests_properties(${test_name}Upload PROPERTIES DISABLED TRUE) + endif() +endfunction() diff --git a/CMake/testing/VTKmPerformanceTestFetch.cmake b/CMake/testing/VTKmPerformanceTestFetch.cmake new file mode 100644 index 000000000..9342c898b --- /dev/null +++ b/CMake/testing/VTKmPerformanceTestFetch.cmake @@ -0,0 +1,17 @@ +##============================================================================ +## Copyright (c) Kitware, Inc. +## All rights reserved. +## See LICENSE.txt for details. +## +## This software is distributed WITHOUT ANY WARRANTY; without even +## the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +## PURPOSE. See the above copyright notice for more information. +##============================================================================ + +include(${VTKm_SOURCE_DIR}/CMake/testing/VTKmPerformanceTestLib.cmake) + +REQUIRE_FLAG_MUTABLE("VTKm_PERF_REPO") +REQUIRE_FLAG_MUTABLE("VTKm_PERF_REMOTE_URL") + +file(REMOVE_RECURSE vtk-m-benchmark-records) +execute(COMMAND /usr/bin/git clone -b records ${VTKm_PERF_REMOTE_URL} ${VTKm_PERF_REPO}) diff --git a/CMake/testing/VTKmPerformanceTestLib.cmake b/CMake/testing/VTKmPerformanceTestLib.cmake new file mode 100644 index 000000000..ba5de8336 --- /dev/null +++ b/CMake/testing/VTKmPerformanceTestLib.cmake @@ -0,0 +1,36 @@ +##============================================================================ +## Copyright (c) Kitware, Inc. +## All rights reserved. +## See LICENSE.txt for details. +## +## This software is distributed WITHOUT ANY WARRANTY; without even +## the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +## PURPOSE. See the above copyright notice for more information. +##============================================================================ + +macro(REQUIRE_FLAG flag) + if (NOT DEFINED ${flag}) + message(FATAL_ERROR "Need to pass the ${flag}") + endif() +endmacro(REQUIRE_FLAG) + +macro(REQUIRE_FLAG_MUTABLE flag) + REQUIRE_FLAG(${flag}) + + # Env var overrides default value + if (DEFINED ENV{${flag}}) + set(${flag} "$ENV{${flag}}") + endif() +endmacro(REQUIRE_FLAG_MUTABLE) + +macro(execute) + execute_process( + ${ARGV} + COMMAND_ECHO STDOUT + ECHO_OUTPUT_VARIABLE + ECHO_ERROR_VARIABLE + COMMAND_ERROR_IS_FATAL ANY + ) +endmacro() + +message("CTEST_FULL_OUTPUT") diff --git a/CMake/testing/VTKmPerformanceTestReport.cmake b/CMake/testing/VTKmPerformanceTestReport.cmake new file mode 100644 index 000000000..17013d36b --- /dev/null +++ b/CMake/testing/VTKmPerformanceTestReport.cmake @@ -0,0 +1,77 @@ +##============================================================================ +## Copyright (c) Kitware, Inc. +## All rights reserved. +## See LICENSE.txt for details. +## +## This software is distributed WITHOUT ANY WARRANTY; without even +## the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +## PURPOSE. See the above copyright notice for more information. +##============================================================================ + +include("${VTKm_SOURCE_DIR}/CMake/testing/VTKmPerformanceTestLib.cmake") + +REQUIRE_FLAG("BENCHMARK_NAME") +REQUIRE_FLAG("VTKm_PERF_COMPARE_JSON") +REQUIRE_FLAG("VTKm_PERF_COMPARE_STDOUT") + +REQUIRE_FLAG_MUTABLE("VTKm_PERF_REPO") +REQUIRE_FLAG_MUTABLE("VTKm_PERF_ALPHA") +REQUIRE_FLAG_MUTABLE("VTKm_PERF_DIST") + +###FIND MOST RECENT BASELINE#################################################### +execute(COMMAND /usr/bin/git -C "${VTKm_SOURCE_DIR}" merge-base origin/master @ + OUTPUT_VARIABLE GIT_BASE_COMMIT) + +string(STRIP "${GIT_BASE_COMMIT}" GIT_BASE_COMMIT) + +execute_process(COMMAND /usr/bin/git -C "${VTKm_SOURCE_DIR}" log --format=%H --first-parent "${GIT_BASE_COMMIT}" + OUTPUT_VARIABLE GIT_ANCESTOR_COMMITS + COMMAND_ECHO STDOUT + ECHO_ERROR_VARIABLE + COMMAND_ERROR_IS_FATAL ANY + ) + +string(REPLACE "\n" ";" GIT_ANCESTOR_COMMITS ${GIT_ANCESTOR_COMMITS}) + +foreach(commit IN LISTS GIT_ANCESTOR_COMMITS) + if (EXISTS "${VTKm_PERF_REPO}/${commit}_${BENCHMARK_NAME}.json") + set(BASELINE_REPORT "${VTKm_PERF_REPO}/${commit}_${BENCHMARK_NAME}.json") + break() + endif() +endforeach() + +if (NOT DEFINED BASELINE_REPORT) + message(FATAL_ERROR "PerformanceTestReport: no ancestor benchmarks record found") +endif() + +###RUN COMPARE_PY SCRIPT######################################################## +execute(COMMAND /usr/bin/python3 + "${VTKm_SOURCE_DIR}/Utilities/Scripts/compare.py" + --alpha "${VTKm_PERF_ALPHA}" + --dist "${VTKm_PERF_DIST}" + benchmarks "${BASELINE_REPORT}" "${VTKm_PERF_COMPARE_JSON}" + OUTPUT_VARIABLE compare_py_output + ) + +# Write compare.py output to disk +file(WRITE "${VTKm_PERF_COMPARE_STDOUT}" "${compare_py_output}") + +###PERFORM NULL HYPHOTESIS###################################################### +string(REGEX MATCHALL "[^\n]*time_pvalue[^\n]*" pvalues_list ${compare_py_output}) + +foreach(pvalue_line IN LISTS pvalues_list) + # We only take into consideration the wall time of the test + string(REGEX MATCH "(.*)/manual_time_pvalue[ \t]+([0-9.]+)[ ]+" ignoreme ${pvalue_line}) + if (CMAKE_MATCH_0) + set(benchmark_name "${CMAKE_MATCH_1}") + set(benchmark_pvalue "${CMAKE_MATCH_2}") + if("${benchmark_pvalue}" LESS "${VTKm_PERF_ALPHA}") + list(APPEND time_failed_benchs ${benchmark_name}) + endif() + endif() +endforeach() + +if(time_failed_benchs) + string(REPLACE ";" "\n" time_failed_benchs "${time_failed_benchs}") + message(FATAL_ERROR "Time-failed Benchmarks:\n${time_failed_benchs}") +endif() diff --git a/CMake/testing/VTKmPerformanceTestRun.cmake b/CMake/testing/VTKmPerformanceTestRun.cmake new file mode 100644 index 000000000..06cf51b74 --- /dev/null +++ b/CMake/testing/VTKmPerformanceTestRun.cmake @@ -0,0 +1,35 @@ +##============================================================================ +## Copyright (c) Kitware, Inc. +## All rights reserved. +## See LICENSE.txt for details. +## +## This software is distributed WITHOUT ANY WARRANTY; without even +## the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +## PURPOSE. See the above copyright notice for more information. +##============================================================================ + +include("${VTKm_SOURCE_DIR}/CMake/testing/VTKmPerformanceTestLib.cmake") + +REQUIRE_FLAG("VTKm_PERF_BENCH_PATH") +REQUIRE_FLAG("VTKm_PERF_FILTER_NAME") +REQUIRE_FLAG("VTKm_PERF_COMPARE_JSON") +REQUIRE_FLAG("VTKm_PERF_STDOUT") + +REQUIRE_FLAG_MUTABLE("VTKm_PERF_BENCH_DEVICE") +REQUIRE_FLAG_MUTABLE("VTKm_PERF_REPETITIONS") +REQUIRE_FLAG_MUTABLE("VTKm_PERF_MIN_TIME") + +execute( + COMMAND "${VTKm_PERF_BENCH_PATH}" + --vtkm-device "${VTKm_PERF_BENCH_DEVICE}" + "--benchmark_filter=${VTKm_PERF_FILTER_NAME}" + "--benchmark_out=${VTKm_PERF_COMPARE_JSON}" + "--benchmark_repetitions=${VTKm_PERF_REPETITIONS}" + "--benchmark_min_time=${VTKm_PERF_MIN_TIME}" + --benchmark_out_format=json + --benchmark_counters_tabular=true + OUTPUT_VARIABLE report_output + ) + +# Write compare.py output to disk +file(WRITE "${VTKm_PERF_STDOUT}" "${report_output}") diff --git a/CMake/testing/VTKmPerformanceTestUpload.cmake b/CMake/testing/VTKmPerformanceTestUpload.cmake new file mode 100644 index 000000000..4c5567164 --- /dev/null +++ b/CMake/testing/VTKmPerformanceTestUpload.cmake @@ -0,0 +1,23 @@ +##============================================================================ +## Copyright (c) Kitware, Inc. +## All rights reserved. +## See LICENSE.txt for details. +## +## This software is distributed WITHOUT ANY WARRANTY; without even +## the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +## PURPOSE. See the above copyright notice for more information. +##============================================================================ + +include("${VTKm_SOURCE_DIR}/CMake/testing/VTKmPerformanceTestLib.cmake") + +REQUIRE_FLAG("VTKm_PERF_COMPARE_JSON") +REQUIRE_FLAG_MUTABLE("VTKm_PERF_REPO") + +file(COPY "${VTKm_PERF_COMPARE_JSON}" DESTINATION "${VTKm_PERF_REPO}/") +get_filename_component(perf_report_name "${VTKm_PERF_COMPARE_JSON}" NAME) + +execute(COMMAND /usr/bin/git -C "${VTKm_PERF_REPO}" config --local user.name vtk-m\ benchmark\ job) +execute(COMMAND /usr/bin/git -C "${VTKm_PERF_REPO}" config --local user.email do_not_email_the_robot@kitware.com) +execute(COMMAND /usr/bin/git -C "${VTKm_PERF_REPO}" add "${perf_report_name}") +execute(COMMAND /usr/bin/git -C "${VTKm_PERF_REPO}" commit -m "Added ${perf_report_name} record") +execute(COMMAND /usr/bin/git -C "${VTKm_PERF_REPO}" push origin records) diff --git a/benchmarking/CMakeLists.txt b/benchmarking/CMakeLists.txt index aa2946cce..ae8cfe509 100644 --- a/benchmarking/CMakeLists.txt +++ b/benchmarking/CMakeLists.txt @@ -63,3 +63,18 @@ if(TARGET vtkm_rendering) add_benchmark(NAME BenchmarkRayTracing FILE BenchmarkRayTracing.cxx LIBS vtkm_rendering vtkm_source) add_benchmark(NAME BenchmarkInSitu FILE BenchmarkInSitu.cxx LIBS vtkm_rendering vtkm_source vtkm_filter vtkm_io) endif() + +if(VTKm_ENABLE_PERFORMANCE_TESTING) + include("${VTKm_SOURCE_DIR}/CMake/testing/VTKmPerformanceTest.cmake") + add_benchmark_test(BenchmarkFilters + BenchThreshold + BenchContour/IsStructuredDataSet:1/NIsoVals:12/MergePts:1/GenNormals:0 + BenchContour/IsStructuredDataSet:1/NIsoVals:12/MergePts:0/GenNormals:1/FastNormals:1 + BenchContour/IsStructuredDataSet:0/NIsoVals:12/MergePts:1/GenNormals:0 + BenchContour/IsStructuredDataSet:0/NIsoVals:12/MergePts:0/GenNormals:/FastNormals:1 + BenchTetrahedralize + BenchVertexClustering/NumDivs:256) + if(TARGET vtkm_rendering) + add_benchmark_test(BenchmarkInSitu "BenchContour") + endif() +endif() diff --git a/docs/CI-README.md b/docs/CI-README.md index 1028ed82e..f63745125 100644 --- a/docs/CI-README.md +++ b/docs/CI-README.md @@ -19,6 +19,10 @@ Gitlab CI 4. ECP Continuous Integration - OLCF Ascent testing machine +5. Automated Performance Regression tests + - Overview + - Details + # Kitware Gitlab CI GitLab CI/CD allows for software development through continous integration, delivery, and deployment. @@ -308,4 +312,103 @@ GCC8. For a view of only ascent jobs refer to the following [link][cdash-ascent]. +# Automated Performance Regression tests + +## Overview + +The design of the performance regression test is composed of the following +components: + +1. The Kitware Gitlab instance which trigger the benchmark jobs when a git + commit is pushed. +2. Gitlab CI jobs for performing the benchmarks and for generating the + comparison with the historical results. +3. A Git repository that is used for storing the historical results. +4. The Kitware CDASH instance which files, displays the performance report and + inform the developer if a performance regression has occurred. + +The performance regression test is performed whenever a git commit is pushed. +The job _performancetest_ which invoke the benchmark suite in a Gitlab runner +job and later compare its results against the historical results, stored in +CDASH, of its most immediate master ancestor. The results of this comparison are +then displayed in a brief report in the form of a comment in its corresponding +Gitlab merge-request. + +![perftest_arch](perftest_arch.png) + +## Details + +### Selection of Benchmarks + +While we can possibly run all of the provided benchmarks in the continuous +build track to avoid potential performance and latency issues in the CI, I +have initially limited the benchmark suites to: + +- BenchmarkFilters +- BenchmarkInsitu + +### Benchmark ctest + +We provide a CMake function named `add_benchmark_test` which sets the +performance regression test for the given Google Benchmark suite. It admits one +argument to filter the number of benchmarks to be executed. If ran locally, it +will not upload the results to the online record repository. + +### Requirements + +- Python3 with the SciPy package +- Benchmark tests will be enabled in a CMAKE build that sets both + `VTKm_ENABLE_BENCHMARKS` and `VTKm_ENABLE_PERFORMANCE_TESTING` + +### New Gitlab Runner requirements + +- It must have disabled every type of CPU scaling option both at the BIOS and + Kernel level (`cpugovern`). +- It must provide a gitlab runner with a concurrency level 1 to avoid other + jobs being scheduled while the benchmark is being executed. + +### How to make sense of the results + +Results of both of the benchmark and the comparison against its most recent +commit ancestor can be accessed in the CDASH Notes for the performance +regression build. The CDASH Notes can be accessed by clicking the note-like +miniature image in the build name column. + +Performance Regressions test that report a performance failure are reported in +the form of a test failure of the test `PerformanceTest($TestName)Report`. The +results of the comparison can be seen by clicking this failed test. + +Performance regression test success is determined by the performance of a null +hypothesis test with the hypothesis that the given tests performs similarly +or better than the baseline test with a confidence level 1-alpha. If a pvalue is +small enough (less than alpha), we reject the null hypothesis and we report that +the current commit introduces a performance regression. By default we use a +t-distribution with an alpha value of 5%. The pvalues can be seen in the +uploaded reports. + +The following parameters can be modified by editing the corresponding +environmental variables: + +- Alpha value: `VTKm_PERF_ALPHA` +- Minimum number of repetitions for each benchmark: `VTKm_PERF_REPETITIONS` +- Minimum time to spend for each benchmark: `VTKm_PERF_MIN_TIME` +- Statistical distribution to use: `VTKm_PERF_DIST` + +Below is an example of this raw output of the comparison of the current commit +against the baseline results: + +``` +Benchmark Time CPU Time Old Time New CPU Old CPU New +------------------------------------------------------------------------------------------------------------ +BenchThreshold/manual_time +0.0043 +0.0036 73 73 92 92 +BenchThreshold/manual_time +0.0074 +0.0060 73 73 91 92 +BenchThreshold/manual_time -0.0003 -0.0007 73 73 92 92 +BenchThreshold/manual_time -0.0019 -0.0018 73 73 92 92 +BenchThreshold/manual_time -0.0021 -0.0017 73 73 92 92 +BenchThreshold/manual_time +0.0001 +0.0006 73 73 92 92 +BenchThreshold/manual_time +0.0033 +0.0031 73 73 92 92 +BenchThreshold/manual_time -0.0071 -0.0057 73 73 92 92 +BenchThreshold/manual_time -0.0050 -0.0041 73 73 92 92 +``` + [cdash-ascent]: https://open.cdash.org/index.php?project=VTKM&filtercount=1&showfilters=1&field1=site&compare1=63&value1=ascent diff --git a/docs/perftest_arch.png b/docs/perftest_arch.png new file mode 100644 index 000000000..f35520092 Binary files /dev/null and b/docs/perftest_arch.png differ