2015-07-24 21:22:10 +00:00
|
|
|
//============================================================================
|
|
|
|
// Copyright (c) Kitware, Inc.
|
|
|
|
// All rights reserved.
|
|
|
|
// See LICENSE.txt for details.
|
|
|
|
// This software is distributed WITHOUT ANY WARRANTY; without even
|
|
|
|
// the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
|
|
|
|
// PURPOSE. See the above copyright notice for more information.
|
|
|
|
//
|
|
|
|
// Copyright 2014 Sandia Corporation.
|
|
|
|
// Copyright 2014 UT-Battelle, LLC.
|
|
|
|
// Copyright 2014 Los Alamos National Security.
|
|
|
|
//
|
|
|
|
// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
|
|
|
|
// the U.S. Government retains certain rights in this software.
|
|
|
|
//
|
|
|
|
// Under the terms of Contract DE-AC52-06NA25396 with Los Alamos National
|
|
|
|
// Laboratory (LANL), the U.S. Government retains certain rights in
|
|
|
|
// this software.
|
|
|
|
//============================================================================
|
|
|
|
|
|
|
|
#ifndef vtk_m_benchmarking_Benchmarker_h
|
|
|
|
#define vtk_m_benchmarking_Benchmarker_h
|
|
|
|
|
2017-05-18 16:26:18 +00:00
|
|
|
#include <vtkm/ListTag.h>
|
2015-07-24 21:22:10 +00:00
|
|
|
#include <vtkm/Math.h>
|
2017-05-18 16:26:18 +00:00
|
|
|
#include <vtkm/cont/testing/Testing.h>
|
2015-07-24 21:22:10 +00:00
|
|
|
|
|
|
|
#include <algorithm>
|
|
|
|
#include <iostream>
|
2017-05-18 14:51:24 +00:00
|
|
|
#include <vector>
|
2015-07-24 21:22:10 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Writing a Benchmark
|
|
|
|
* -------------------
|
|
|
|
* To write a benchmark you must provide a functor that will run the operations
|
|
|
|
* you want to time and return the run time of those operations using the timer
|
|
|
|
* for the device. The benchmark should also be templated on the value type being
|
|
|
|
* operated on. Then use VTKM_MAKE_BENCHMARK to generate a maker functor and
|
|
|
|
* VTKM_RUN_BENCHMARK to run the benchmark on a list of types.
|
|
|
|
*
|
|
|
|
* For Example:
|
|
|
|
*
|
|
|
|
* template<typename Value>
|
|
|
|
* struct BenchSilly {
|
|
|
|
* // Setup anything that doesn't need to change per run in the constructor
|
2016-10-19 22:42:58 +00:00
|
|
|
* VTKM_CONT BenchSilly(){}
|
2015-07-24 21:22:10 +00:00
|
|
|
*
|
|
|
|
* // The overloaded call operator will run the operations being timed and
|
|
|
|
* // return the execution time
|
2016-10-19 22:42:58 +00:00
|
|
|
* VTKM_CONT
|
2015-07-24 21:22:10 +00:00
|
|
|
* vtkm::Float64 operator()(){
|
|
|
|
* return 0.05;
|
|
|
|
* }
|
|
|
|
*
|
|
|
|
* // The benchmark must also provide a method describing itself, this is
|
|
|
|
* // used when printing out run time statistics
|
2016-10-19 22:42:58 +00:00
|
|
|
* VTKM_CONT
|
2015-07-24 21:22:10 +00:00
|
|
|
* std::string Description() const {
|
|
|
|
* return "A silly benchmark";
|
|
|
|
* }
|
|
|
|
* };
|
|
|
|
*
|
|
|
|
* // Now use the VTKM_MAKE_BENCHMARK macro to generate a maker functor for
|
|
|
|
* // your benchmark. This lets us generate the benchmark functor for each type
|
|
|
|
* // we want to test
|
|
|
|
* VTKM_MAKE_BENCHMARK(Silly, BenchSilly);
|
|
|
|
*
|
|
|
|
* // You can also optionally pass arguments to the constructor like so:
|
|
|
|
* // VTKM_MAKE_BENCHMARK(Blah, BenchBlah, 1, 2, 3);
|
|
|
|
* // Note that benchmark names (the first argument) must be unique so different
|
|
|
|
* // parameters to the constructor should have different names
|
|
|
|
*
|
|
|
|
* // We can now run our benchmark using VTKM_RUN_BENCHMARK, passing the
|
|
|
|
* // benchmark name and type list to run on
|
|
|
|
* int main(int, char**){
|
|
|
|
* VTKM_RUN_BENCHMARK(Silly, vtkm::ListTagBase<vtkm::Float32>());
|
|
|
|
* return 0;
|
|
|
|
* }
|
|
|
|
*
|
|
|
|
* Check out vtkm/benchmarking/BenchmarkDeviceAdapter.h for some example usage
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Use the VTKM_MAKE_BENCHMARK macro to define a maker functor for your benchmark.
|
|
|
|
* This is used to allow you to template the benchmark functor on the type being benchmarked
|
|
|
|
* so you can write init code in the constructor. Then the maker will return a constructed
|
|
|
|
* instance of your benchmark for the type being benchmarked. The VA_ARGS are used to
|
|
|
|
* pass any extra arguments needed by your benchmark
|
|
|
|
*/
|
2017-05-18 14:29:41 +00:00
|
|
|
#define VTKM_MAKE_BENCHMARK(Name, Bench, ...) \
|
|
|
|
struct MakeBench##Name \
|
|
|
|
{ \
|
|
|
|
template <typename Value> \
|
|
|
|
VTKM_CONT Bench<Value> operator()(const Value vtkmNotUsed(v)) const \
|
|
|
|
{ \
|
|
|
|
return Bench<Value>(__VA_ARGS__); \
|
|
|
|
} \
|
2015-07-24 21:22:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Use the VTKM_RUN_BENCHMARK macro to run your benchmark on the type list passed.
|
|
|
|
* You must have previously defined a maker functor with VTKM_MAKE_BENCHMARK that this
|
|
|
|
* macro will look for and use
|
|
|
|
*/
|
2017-05-18 14:29:41 +00:00
|
|
|
#define VTKM_RUN_BENCHMARK(Name, Types) \
|
2015-07-24 21:22:10 +00:00
|
|
|
vtkm::benchmarking::BenchmarkTypes(MakeBench##Name(), (Types))
|
|
|
|
|
2017-05-18 14:29:41 +00:00
|
|
|
namespace vtkm
|
|
|
|
{
|
|
|
|
namespace benchmarking
|
|
|
|
{
|
|
|
|
namespace stats
|
|
|
|
{
|
2015-08-03 16:56:59 +00:00
|
|
|
// Checks that the sequence is sorted, returns true if it's sorted, false
|
|
|
|
// otherwise
|
2017-05-18 14:29:41 +00:00
|
|
|
template <typename ForwardIt>
|
|
|
|
bool is_sorted(ForwardIt first, ForwardIt last)
|
|
|
|
{
|
2015-08-03 16:56:59 +00:00
|
|
|
ForwardIt next = first;
|
|
|
|
++next;
|
2017-05-18 14:29:41 +00:00
|
|
|
for (; next != last; ++next, ++first)
|
|
|
|
{
|
|
|
|
if (*first > *next)
|
|
|
|
{
|
2015-08-03 16:56:59 +00:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
2015-07-24 21:22:10 +00:00
|
|
|
|
|
|
|
// Get the value representing the `percent` percentile of the
|
|
|
|
// sorted samples using linear interpolation
|
2017-05-18 14:29:41 +00:00
|
|
|
vtkm::Float64 PercentileValue(const std::vector<vtkm::Float64>& samples,
|
|
|
|
const vtkm::Float64 percent)
|
|
|
|
{
|
2016-04-20 21:41:14 +00:00
|
|
|
VTKM_ASSERT(!samples.empty());
|
2017-05-18 14:29:41 +00:00
|
|
|
if (samples.size() == 1)
|
|
|
|
{
|
2015-07-24 21:22:10 +00:00
|
|
|
return samples.front();
|
|
|
|
}
|
2016-04-20 21:41:14 +00:00
|
|
|
VTKM_ASSERT(percent >= 0.0);
|
|
|
|
VTKM_ASSERT(percent <= 100.0);
|
2017-05-18 14:29:41 +00:00
|
|
|
VTKM_ASSERT(vtkm::benchmarking::stats::is_sorted(samples.begin(), samples.end()));
|
|
|
|
if (percent == 100.0)
|
|
|
|
{
|
2015-07-24 21:22:10 +00:00
|
|
|
return samples.back();
|
|
|
|
}
|
|
|
|
// Find the two nearest percentile values and linearly
|
|
|
|
// interpolate between them
|
|
|
|
const vtkm::Float64 rank = percent / 100.0 * (static_cast<vtkm::Float64>(samples.size()) - 1.0);
|
|
|
|
const vtkm::Float64 low_rank = vtkm::Floor(rank);
|
|
|
|
const vtkm::Float64 dist = rank - low_rank;
|
|
|
|
const size_t k = static_cast<size_t>(low_rank);
|
|
|
|
const vtkm::Float64 low = samples[k];
|
|
|
|
const vtkm::Float64 high = samples[k + 1];
|
|
|
|
return low + (high - low) * dist;
|
|
|
|
}
|
|
|
|
// Winsorize the samples to clean up any very extreme outliers
|
|
|
|
// Will replace all samples below `percent` and above 100 - `percent` percentiles
|
|
|
|
// with the value at the percentile
|
|
|
|
// NOTE: Assumes the samples have been sorted, as we make use of PercentileValue
|
2017-05-18 14:29:41 +00:00
|
|
|
void Winsorize(std::vector<vtkm::Float64>& samples, const vtkm::Float64 percent)
|
|
|
|
{
|
2015-07-24 21:22:10 +00:00
|
|
|
const vtkm::Float64 low_percentile = PercentileValue(samples, percent);
|
|
|
|
const vtkm::Float64 high_percentile = PercentileValue(samples, 100.0 - percent);
|
2017-05-18 14:29:41 +00:00
|
|
|
for (std::vector<vtkm::Float64>::iterator it = samples.begin(); it != samples.end(); ++it)
|
|
|
|
{
|
|
|
|
if (*it < low_percentile)
|
|
|
|
{
|
2015-07-24 21:22:10 +00:00
|
|
|
*it = low_percentile;
|
|
|
|
}
|
2017-05-18 14:29:41 +00:00
|
|
|
else if (*it > high_percentile)
|
|
|
|
{
|
2015-07-24 21:22:10 +00:00
|
|
|
*it = high_percentile;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// Compute the mean value of the dataset
|
2017-05-18 14:29:41 +00:00
|
|
|
vtkm::Float64 Mean(const std::vector<vtkm::Float64>& samples)
|
|
|
|
{
|
2015-07-24 21:22:10 +00:00
|
|
|
vtkm::Float64 mean = 0;
|
2017-05-18 14:29:41 +00:00
|
|
|
for (std::vector<vtkm::Float64>::const_iterator it = samples.begin(); it != samples.end(); ++it)
|
|
|
|
{
|
2015-07-24 21:22:10 +00:00
|
|
|
mean += *it;
|
|
|
|
}
|
|
|
|
return mean / static_cast<vtkm::Float64>(samples.size());
|
|
|
|
}
|
|
|
|
// Compute the sample variance of the samples
|
2017-05-18 14:29:41 +00:00
|
|
|
vtkm::Float64 Variance(const std::vector<vtkm::Float64>& samples)
|
|
|
|
{
|
2015-07-24 21:22:10 +00:00
|
|
|
vtkm::Float64 mean = Mean(samples);
|
|
|
|
vtkm::Float64 square_deviations = 0;
|
2017-05-18 14:29:41 +00:00
|
|
|
for (std::vector<vtkm::Float64>::const_iterator it = samples.begin(); it != samples.end(); ++it)
|
|
|
|
{
|
2015-07-24 21:22:10 +00:00
|
|
|
square_deviations += vtkm::Pow(*it - mean, 2.0);
|
|
|
|
}
|
|
|
|
return square_deviations / (static_cast<vtkm::Float64>(samples.size()) - 1.0);
|
|
|
|
}
|
|
|
|
// Compute the standard deviation of the samples
|
2017-05-18 14:29:41 +00:00
|
|
|
vtkm::Float64 StandardDeviation(const std::vector<vtkm::Float64>& samples)
|
|
|
|
{
|
2015-07-24 21:22:10 +00:00
|
|
|
return vtkm::Sqrt(Variance(samples));
|
|
|
|
}
|
|
|
|
// Compute the median absolute deviation of the dataset
|
2017-05-18 14:29:41 +00:00
|
|
|
vtkm::Float64 MedianAbsDeviation(const std::vector<vtkm::Float64>& samples)
|
|
|
|
{
|
2015-07-24 21:22:10 +00:00
|
|
|
std::vector<vtkm::Float64> abs_deviations;
|
|
|
|
abs_deviations.reserve(samples.size());
|
|
|
|
const vtkm::Float64 median = PercentileValue(samples, 50.0);
|
2017-05-18 14:29:41 +00:00
|
|
|
for (std::vector<vtkm::Float64>::const_iterator it = samples.begin(); it != samples.end(); ++it)
|
|
|
|
{
|
2015-07-24 21:22:10 +00:00
|
|
|
abs_deviations.push_back(vtkm::Abs(*it - median));
|
|
|
|
}
|
2015-08-03 16:56:59 +00:00
|
|
|
std::sort(abs_deviations.begin(), abs_deviations.end());
|
2015-07-24 21:22:10 +00:00
|
|
|
return PercentileValue(abs_deviations, 50.0);
|
|
|
|
}
|
|
|
|
} // stats
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The benchmarker takes a functor to benchmark and runs it multiple times,
|
|
|
|
* printing out statistics of the run time at the end.
|
|
|
|
* The functor passed should return the run time of the thing being benchmarked
|
|
|
|
* in seconds, this lets us avoid including any per-run setup time in the benchmark.
|
|
|
|
* However any one-time setup should be done in the functor's constructor
|
|
|
|
*/
|
2017-05-18 14:29:41 +00:00
|
|
|
struct Benchmarker
|
|
|
|
{
|
2015-07-24 21:22:10 +00:00
|
|
|
const vtkm::Float64 MAX_RUNTIME;
|
|
|
|
const size_t MAX_ITERATIONS;
|
|
|
|
|
2017-05-18 14:29:41 +00:00
|
|
|
Benchmarker()
|
|
|
|
: MAX_RUNTIME(30)
|
|
|
|
, MAX_ITERATIONS(500)
|
|
|
|
{
|
|
|
|
}
|
2015-07-24 21:22:10 +00:00
|
|
|
|
2017-05-18 14:29:41 +00:00
|
|
|
template <typename Functor>
|
|
|
|
VTKM_CONT void operator()(Functor func) const
|
|
|
|
{
|
2015-07-24 21:22:10 +00:00
|
|
|
std::vector<vtkm::Float64> samples;
|
|
|
|
// Do a warm-up run. If the benchmark allocates any additional memory
|
|
|
|
// eg. storage for output results, this will let it do that and
|
|
|
|
// allow us to avoid measuring the allocation time in the actual benchmark run
|
|
|
|
func();
|
|
|
|
|
|
|
|
samples.reserve(MAX_ITERATIONS);
|
|
|
|
// Run each benchmark for MAX_RUNTIME seconds or MAX_ITERATIONS iterations, whichever
|
|
|
|
// takes less time. This kind of assumes that running for 500 iterations or 1.5s will give
|
|
|
|
// good statistics, but if median abs dev and/or std dev are too high both these limits
|
|
|
|
// could be increased
|
|
|
|
size_t iter = 0;
|
|
|
|
for (vtkm::Float64 elapsed = 0.0; elapsed < MAX_RUNTIME && iter < MAX_ITERATIONS;
|
2017-05-18 14:29:41 +00:00
|
|
|
elapsed += samples.back(), ++iter)
|
2015-07-24 21:22:10 +00:00
|
|
|
{
|
|
|
|
samples.push_back(func());
|
|
|
|
}
|
|
|
|
std::sort(samples.begin(), samples.end());
|
|
|
|
stats::Winsorize(samples, 5.0);
|
2017-05-18 14:29:41 +00:00
|
|
|
std::cout << "Benchmark \'" << func.Description() << "\' results:\n"
|
|
|
|
<< "\tmedian = " << stats::PercentileValue(samples, 50.0) << "s\n"
|
|
|
|
<< "\tmedian abs dev = " << stats::MedianAbsDeviation(samples) << "s\n"
|
|
|
|
<< "\tmean = " << stats::Mean(samples) << "s\n"
|
|
|
|
<< "\tstd dev = " << stats::StandardDeviation(samples) << "s\n"
|
|
|
|
<< "\tmin = " << samples.front() << "s\n"
|
|
|
|
<< "\tmax = " << samples.back() << "s\n";
|
2015-07-24 21:22:10 +00:00
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2017-05-18 14:29:41 +00:00
|
|
|
template <typename MakerFunctor>
|
|
|
|
class InternalPrintTypeAndBench
|
|
|
|
{
|
2015-07-24 21:22:10 +00:00
|
|
|
MakerFunctor Maker;
|
|
|
|
|
|
|
|
public:
|
2016-10-19 22:42:58 +00:00
|
|
|
VTKM_CONT
|
2017-05-18 14:29:41 +00:00
|
|
|
InternalPrintTypeAndBench(MakerFunctor maker)
|
|
|
|
: Maker(maker)
|
|
|
|
{
|
|
|
|
}
|
2015-07-24 21:22:10 +00:00
|
|
|
|
2017-05-18 14:29:41 +00:00
|
|
|
template <typename T>
|
|
|
|
VTKM_CONT void operator()(T t) const
|
|
|
|
{
|
|
|
|
std::cout << "*** " << vtkm::testing::TypeName<T>::Name() << " ***************" << std::endl;
|
2015-07-24 21:22:10 +00:00
|
|
|
Benchmarker bench;
|
|
|
|
bench(Maker(t));
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2017-05-18 14:29:41 +00:00
|
|
|
template <class MakerFunctor, class TypeList>
|
|
|
|
VTKM_CONT void BenchmarkTypes(const MakerFunctor& maker, TypeList)
|
|
|
|
{
|
2015-07-24 21:22:10 +00:00
|
|
|
vtkm::ListForEach(InternalPrintTypeAndBench<MakerFunctor>(maker), TypeList());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#endif
|