From 613b4d44b792fc2586e31062a8c32f196f703edf Mon Sep 17 00:00:00 2001 From: Mark Bolstad Date: Mon, 12 Sep 2022 09:24:47 -0600 Subject: [PATCH] Switch how InSitu benchmark iterates Previous version had a hard-coded loop for updating the dataset. This change relenquishes control over to google benchmarks. Running it from the command-line will iterate each sub-benchmark a different number of iterations depending on how long each individual test takes. The number of iterations can be increased (but not controlled) by increasing the minimum time for each test. Alternatively, using the "repetitions" argument, you can control exactly how many times the benchamrks are run. See the README for InSitu that throughly documents the arguments. --- benchmarking/BenchmarkInSitu.cxx | 364 ++++++++++++++++++------------- benchmarking/README_insitu.md | 20 ++ 2 files changed, 238 insertions(+), 146 deletions(-) create mode 100644 benchmarking/README_insitu.md diff --git a/benchmarking/BenchmarkInSitu.cxx b/benchmarking/BenchmarkInSitu.cxx index 721f53b6e..5c59b7d9c 100644 --- a/benchmarking/BenchmarkInSitu.cxx +++ b/benchmarking/BenchmarkInSitu.cxx @@ -10,8 +10,10 @@ #include "Benchmarker.h" +#include #include #include +#include #include #include @@ -48,7 +50,6 @@ namespace { -const uint32_t DEFAULT_NUM_CYCLES = 20; const vtkm::Id DEFAULT_DATASET_DIM = 128; const vtkm::Id DEFAULT_IMAGE_SIZE = 1024; @@ -66,6 +67,29 @@ std::string PointScalarsName; // The point vectors to use: std::string PointVectorsName; +// Global counters for number of cycles +// These are globals because google benchmarks restarts the test for every +// repetition when using --benchmark_repetitions + +// Additionlly, we need this global flag for when not doing repetitions, +// as benchmark will repeatedly drop in and out of the measured function +// and report the number of iterations for the last run of the function. +// Thus, we'll have way more output images than what the iteration number +// would lead you to believe (maybe fixed in > 1.7 with warmup time specifier) +bool benchmark_repetitions = false; + +inline void hash_combine(std::size_t& vtkmNotUsed(seed)) {} + +template +inline void hash_combine(std::size_t& seed, const T& v, Rest... rest) +{ + std::hash hasher; + seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); + hash_combine(seed, rest...); +} + +std::unordered_map bench_cycles; + enum class RenderingMode { None = 0, @@ -285,33 +309,43 @@ void BenchContour(::benchmark::State& state) { const vtkm::cont::DeviceAdapterId device = Config.Device; - const uint32_t cycle = static_cast(state.range(0)); - const vtkm::Id numIsoVals = static_cast(state.range(1)); - const bool isStructured = static_cast(state.range(2)); - const bool isMultiBlock = static_cast(state.range(3)); - const RenderingMode renderAlgo = static_cast(state.range(4)); - - vtkm::cont::Timer inputGenTimer{ device }; - inputGenTimer.Start(); - BuildInputDataSet(cycle, isStructured, isMultiBlock, DataSetDim); - inputGenTimer.Stop(); - - vtkm::filter::contour::Contour filter; - filter.SetActiveField(PointScalarsName, vtkm::cont::Field::Association::Points); - filter.SetMergeDuplicatePoints(true); - filter.SetGenerateNormals(true); - filter.SetComputeFastNormalsForStructured(true); - filter.SetComputeFastNormalsForUnstructured(true); + const vtkm::Id numIsoVals = static_cast(state.range(0)); + const bool isStructured = static_cast(state.range(1)); + const bool isMultiBlock = static_cast(state.range(2)); + const RenderingMode renderAlgo = static_cast(state.range(3)); vtkm::cont::Timer totalTimer{ device }; vtkm::cont::Timer filterTimer{ device }; vtkm::cont::Timer renderTimer{ device }; vtkm::cont::Timer writeTimer{ device }; + size_t hash_val = 0; + hash_combine(hash_val, std::string("BenchContour"), isStructured, isMultiBlock, renderAlgo); + + int* cycles = &(bench_cycles[hash_val]); + if (!benchmark_repetitions) + *cycles = 0; + for (auto _ : state) { (void)_; totalTimer.Start(); + + // Disable the benchmark timers for the updating/creation of the datasets + state.PauseTiming(); // Stop timers. + vtkm::cont::Timer inputGenTimer{ device }; + inputGenTimer.Start(); + BuildInputDataSet(*cycles, isStructured, isMultiBlock, DataSetDim); + inputGenTimer.Stop(); + + vtkm::filter::contour::Contour filter; + filter.SetActiveField(PointScalarsName, vtkm::cont::Field::Association::Points); + filter.SetMergeDuplicatePoints(true); + filter.SetGenerateNormals(true); + filter.SetComputeFastNormalsForStructured(true); + filter.SetComputeFastNormalsForUnstructured(true); + state.ResumeTiming(); // And resume timers. + filterTimer.Start(); std::vector dataSets; if (isMultiBlock) @@ -333,11 +367,13 @@ void BenchContour(::benchmark::State& state) renderTimer.Stop(); writeTimer.Start(); - WriteToDisk(*canvas, renderAlgo, "contour", isStructured, isMultiBlock, cycle); + WriteToDisk(*canvas, renderAlgo, "contour", isStructured, isMultiBlock, *cycles); writeTimer.Stop(); totalTimer.Stop(); + (*cycles)++; + state.SetIterationTime(totalTimer.GetElapsedTime()); state.counters.insert( { { "InputGenTime", static_cast(inputGenTimer.GetElapsedTime() * 1000) }, @@ -349,21 +385,24 @@ void BenchContour(::benchmark::State& state) void BenchContourGenerator(::benchmark::internal::Benchmark* bm) { - bm->ArgNames({ "Cycle", "NIsos", "IsStructured", "IsMultiBlock", "RenderingMode" }); + bm->ArgNames({ "NIsos", "IsStructured", "IsMultiBlock", "RenderingMode" }); std::vector isStructureds{ false, true }; std::vector isMultiBlocks{ false, true }; std::vector renderingModes{ RenderingMode::RayTrace }; - for (uint32_t cycle = 1; cycle <= DEFAULT_NUM_CYCLES; ++cycle) + for (auto& isStructured : isStructureds) { - for (auto& isStructured : isStructureds) + for (auto& isMultiBlock : isMultiBlocks) { - for (auto& isMultiBlock : isMultiBlocks) + for (auto& mode : renderingModes) { - for (auto& mode : renderingModes) - { - bm->Args({ cycle, 10, isStructured, isMultiBlock, static_cast(mode) }); - } + size_t hash_val = 0; + hash_combine(hash_val, std::string("BenchContour"), isStructured, isMultiBlock, mode); + auto search = bench_cycles.find(hash_val); + // If we can't find the hash, or we're not doing repetitions, reset to 0 + if (search == bench_cycles.end()) + bench_cycles[hash_val] = 0; + bm->Args({ 10, isStructured, isMultiBlock, static_cast(mode) }); } } } @@ -450,30 +489,40 @@ void BenchStreamlines(::benchmark::State& state) { const vtkm::cont::DeviceAdapterId device = Config.Device; - const uint32_t cycle = static_cast(state.range(0)); - const bool isStructured = static_cast(state.range(1)); - const bool isMultiBlock = static_cast(state.range(2)); - const RenderingMode renderAlgo = static_cast(state.range(3)); - - vtkm::cont::Timer inputGenTimer{ device }; - inputGenTimer.Start(); - BuildInputDataSet(cycle, isStructured, isMultiBlock, DataSetDim); - inputGenTimer.Stop(); - - vtkm::filter::flow::Streamline streamline; - streamline.SetStepSize(0.1f); - streamline.SetNumberOfSteps(1000); - streamline.SetActiveField(PointVectorsName); + const bool isStructured = static_cast(state.range(0)); + const bool isMultiBlock = static_cast(state.range(1)); + const RenderingMode renderAlgo = static_cast(state.range(2)); vtkm::cont::Timer totalTimer{ device }; vtkm::cont::Timer filterTimer{ device }; vtkm::cont::Timer renderTimer{ device }; vtkm::cont::Timer writeTimer{ device }; + size_t hash_val = 0; + hash_combine(hash_val, std::string("BenchStreamlines"), isStructured, isMultiBlock, renderAlgo); + + int* cycles = &(bench_cycles[hash_val]); + if (!benchmark_repetitions) + *cycles = 0; + for (auto _ : state) { (void)_; totalTimer.Start(); + + // Disable the benchmark timers for the updating/creation of the datasets + state.PauseTiming(); // Stop timers. + vtkm::cont::Timer inputGenTimer{ device }; + inputGenTimer.Start(); + BuildInputDataSet(*cycles, isStructured, isMultiBlock, DataSetDim); + inputGenTimer.Stop(); + + vtkm::filter::flow::Streamline streamline; + streamline.SetStepSize(0.1f); + streamline.SetNumberOfSteps(1000); + streamline.SetActiveField(PointVectorsName); + state.ResumeTiming(); // And resume timers. + filterTimer.Start(); std::vector dataSets; @@ -496,11 +545,13 @@ void BenchStreamlines(::benchmark::State& state) renderTimer.Stop(); writeTimer.Start(); - WriteToDisk(*canvas, renderAlgo, "streamlines", isStructured, isMultiBlock, cycle); + WriteToDisk(*canvas, renderAlgo, "streamlines", isStructured, isMultiBlock, *cycles); writeTimer.Stop(); totalTimer.Stop(); + (*cycles)++; + state.SetIterationTime(totalTimer.GetElapsedTime()); state.counters.insert( { { "InputGenTime", static_cast(inputGenTimer.GetElapsedTime() * 1000) }, @@ -512,21 +563,24 @@ void BenchStreamlines(::benchmark::State& state) void BenchStreamlinesGenerator(::benchmark::internal::Benchmark* bm) { - bm->ArgNames({ "Cycle", "IsStructured", "IsMultiBlock", "RenderingMode" }); + bm->ArgNames({ "IsStructured", "IsMultiBlock", "RenderingMode" }); std::vector isStructureds{ false, true }; std::vector isMultiBlocks{ false, true }; std::vector renderingModes{ RenderingMode::Mesh }; - for (uint32_t cycle = 1; cycle <= DEFAULT_NUM_CYCLES; ++cycle) + for (auto& isStructured : isStructureds) { - for (auto& isStructured : isStructureds) + for (auto& isMultiBlock : isMultiBlocks) { - for (auto& isMultiBlock : isMultiBlocks) + for (auto& mode : renderingModes) { - for (auto& mode : renderingModes) - { - bm->Args({ cycle, isStructured, isMultiBlock, static_cast(mode) }); - } + size_t hash_val = 0; + hash_combine(hash_val, std::string("BenchStreamlines"), isStructured, isMultiBlock, mode); + auto search = bench_cycles.find(hash_val); + // If we can't find the hash, or we're not doing repetitions, reset to 0 + if (search == bench_cycles.end()) + bench_cycles[hash_val] = 0; + bm->Args({ isStructured, isMultiBlock, static_cast(mode) }); } } } @@ -570,15 +624,9 @@ void BenchSlice(::benchmark::State& state) { const vtkm::cont::DeviceAdapterId device = Config.Device; - const uint32_t cycle = static_cast(state.range(0)); - const bool isStructured = static_cast(state.range(1)); - const bool isMultiBlock = static_cast(state.range(2)); - const RenderingMode renderAlgo = static_cast(state.range(3)); - - vtkm::cont::Timer inputGenTimer{ device }; - inputGenTimer.Start(); - BuildInputDataSet(cycle, isStructured, isMultiBlock, DataSetDim); - inputGenTimer.Stop(); + const bool isStructured = static_cast(state.range(0)); + const bool isMultiBlock = static_cast(state.range(1)); + const RenderingMode renderAlgo = static_cast(state.range(2)); vtkm::filter::contour::Slice filter; @@ -587,10 +635,26 @@ void BenchSlice(::benchmark::State& state) vtkm::cont::Timer renderTimer{ device }; vtkm::cont::Timer writeTimer{ device }; + size_t hash_val = 0; + hash_combine(hash_val, std::string("BenchSlice"), isStructured, isMultiBlock, renderAlgo); + + int* cycles = &(bench_cycles[hash_val]); + if (!benchmark_repetitions) + *cycles = 0; + for (auto _ : state) { (void)_; totalTimer.Start(); + + // Disable the benchmark timers for the updating/creation of the datasets + state.PauseTiming(); // Stop timers. + vtkm::cont::Timer inputGenTimer{ device }; + inputGenTimer.Start(); + BuildInputDataSet(*cycles, isStructured, isMultiBlock, DataSetDim); + inputGenTimer.Stop(); + state.ResumeTiming(); // And resume timers. + filterTimer.Start(); std::vector dataSets; if (isMultiBlock) @@ -620,11 +684,13 @@ void BenchSlice(::benchmark::State& state) renderTimer.Stop(); writeTimer.Start(); - WriteToDisk(*canvas, renderAlgo, "slice", isStructured, isMultiBlock, cycle); + WriteToDisk(*canvas, renderAlgo, "slice", isStructured, isMultiBlock, *cycles); writeTimer.Stop(); totalTimer.Stop(); + (*cycles)++; + state.SetIterationTime(totalTimer.GetElapsedTime()); state.counters.insert( { { "InputGenTime", static_cast(inputGenTimer.GetElapsedTime() * 1000) }, @@ -636,21 +702,24 @@ void BenchSlice(::benchmark::State& state) void BenchSliceGenerator(::benchmark::internal::Benchmark* bm) { - bm->ArgNames({ "Cycle", "IsStructured", "IsMultiBlock", "RenderingMode" }); + bm->ArgNames({ "IsStructured", "IsMultiBlock", "RenderingMode" }); std::vector isStructureds{ false, true }; std::vector isMultiBlocks{ false, true }; std::vector renderingModes{ RenderingMode::RayTrace }; - for (uint32_t cycle = 1; cycle <= DEFAULT_NUM_CYCLES; ++cycle) + for (auto& isStructured : isStructureds) { - for (auto& isStructured : isStructureds) + for (auto& isMultiBlock : isMultiBlocks) { - for (auto& isMultiBlock : isMultiBlocks) + for (auto& mode : renderingModes) { - for (auto& mode : renderingModes) - { - bm->Args({ cycle, isStructured, isMultiBlock, static_cast(mode) }); - } + size_t hash_val = 0; + hash_combine(hash_val, std::string("BenchSlice"), isStructured, isMultiBlock, mode); + auto search = bench_cycles.find(hash_val); + // If we can't find the hash, or we're not doing repetitions, reset to 0 + if (search == bench_cycles.end()) + bench_cycles[hash_val] = 0; + bm->Args({ isStructured, isMultiBlock, static_cast(mode) }); } } } @@ -662,26 +731,35 @@ void BenchMeshRendering(::benchmark::State& state) { const vtkm::cont::DeviceAdapterId device = Config.Device; - const uint32_t cycle = static_cast(state.range(0)); - const bool isStructured = static_cast(state.range(1)); - const bool isMultiBlock = static_cast(state.range(2)); + const bool isStructured = static_cast(state.range(0)); + const bool isMultiBlock = static_cast(state.range(1)); vtkm::cont::Timer inputGenTimer{ device }; vtkm::cont::Timer renderTimer{ device }; vtkm::cont::Timer writeTimer{ device }; - inputGenTimer.Start(); - BuildInputDataSet(cycle, isStructured, isMultiBlock, DataSetDim); - inputGenTimer.Stop(); - vtkm::cont::Timer totalTimer{ device }; + size_t hash_val = 0; + hash_combine(hash_val, std::string("BenchMeshRendering"), isStructured, isMultiBlock); + + int* cycles = &(bench_cycles[hash_val]); + if (!benchmark_repetitions) + *cycles = 0; + for (auto _ : state) { (void)_; totalTimer.Start(); + // Disable the benchmark timers for the updating/creation of the datasets + state.PauseTiming(); // Stop timers. + inputGenTimer.Start(); + BuildInputDataSet(*cycles, isStructured, isMultiBlock, DataSetDim); + inputGenTimer.Stop(); + state.ResumeTiming(); // And resume timers. + std::vector dataSets = isMultiBlock ? ExtractDataSets(PartitionedInputDataSet) : ExtractDataSets(InputDataSet); @@ -690,11 +768,13 @@ void BenchMeshRendering(::benchmark::State& state) renderTimer.Stop(); writeTimer.Start(); - WriteToDisk(*canvas, RenderingMode::Mesh, "mesh", isStructured, isMultiBlock, cycle); + WriteToDisk(*canvas, RenderingMode::Mesh, "mesh", isStructured, isMultiBlock, *cycles); writeTimer.Stop(); totalTimer.Stop(); + (*cycles)++; + state.SetIterationTime(totalTimer.GetElapsedTime()); state.counters.insert( { { "InputGenTime", static_cast(inputGenTimer.GetElapsedTime() * 1000) }, @@ -706,18 +786,21 @@ void BenchMeshRendering(::benchmark::State& state) void BenchMeshRenderingGenerator(::benchmark::internal::Benchmark* bm) { - bm->ArgNames({ "Cycle", "IsStructured", "IsMultiBlock" }); + bm->ArgNames({ "IsStructured", "IsMultiBlock" }); std::vector isStructureds{ false, true }; std::vector isMultiBlocks{ false, true }; - for (uint32_t cycle = 1; cycle <= DEFAULT_NUM_CYCLES; ++cycle) + for (auto& isStructured : isStructureds) { - for (auto& isStructured : isStructureds) + for (auto& isMultiBlock : isMultiBlocks) { - for (auto& isMultiBlock : isMultiBlocks) - { - bm->Args({ cycle, isStructured, isMultiBlock }); - } + size_t hash_val = 0; + hash_combine(hash_val, std::string("BenchMeshRendering"), isStructured, isMultiBlock); + auto search = bench_cycles.find(hash_val); + // If we can't find the hash, or we're not doing repetitions, reset to 0 + if (search == bench_cycles.end()) + bench_cycles[hash_val] = 0; + bm->Args({ isStructured, isMultiBlock }); } } } @@ -728,24 +811,33 @@ void BenchVolumeRendering(::benchmark::State& state) { const vtkm::cont::DeviceAdapterId device = Config.Device; - const uint32_t cycle = static_cast(state.range(0)); const bool isStructured = true; - const bool isMultiBlock = static_cast(state.range(1)); - - vtkm::cont::Timer inputGenTimer{ device }; - inputGenTimer.Start(); - BuildInputDataSet(cycle, isStructured, isMultiBlock, DataSetDim); - inputGenTimer.Stop(); + const bool isMultiBlock = static_cast(state.range(0)); vtkm::cont::Timer totalTimer{ device }; vtkm::cont::Timer renderTimer{ device }; vtkm::cont::Timer writeTimer{ device }; + size_t hash_val = 0; + hash_combine(hash_val, std::string("BenchVolumeRendering"), isMultiBlock); + + int* cycles = &(bench_cycles[hash_val]); + if (!benchmark_repetitions) + *cycles = 0; + for (auto _ : state) { (void)_; totalTimer.Start(); + // Disable the benchmark timers for the updating/creation of the datasets + state.PauseTiming(); // Stop timers. + vtkm::cont::Timer inputGenTimer{ device }; + inputGenTimer.Start(); + BuildInputDataSet(*cycles, isStructured, isMultiBlock, DataSetDim); + inputGenTimer.Stop(); + state.ResumeTiming(); // And resume timers. + renderTimer.Start(); std::vector dataSets = isMultiBlock ? ExtractDataSets(PartitionedInputDataSet) : ExtractDataSets(InputDataSet); @@ -753,11 +845,13 @@ void BenchVolumeRendering(::benchmark::State& state) renderTimer.Stop(); writeTimer.Start(); - WriteToDisk(*canvas, RenderingMode::Volume, "volume", isStructured, isMultiBlock, cycle); + WriteToDisk(*canvas, RenderingMode::Volume, "volume", isStructured, isMultiBlock, *cycles); writeTimer.Stop(); totalTimer.Stop(); + (*cycles)++; + state.SetIterationTime(totalTimer.GetElapsedTime()); state.counters.insert( { { "InputGenTime", static_cast(inputGenTimer.GetElapsedTime() * 1000) }, @@ -769,15 +863,18 @@ void BenchVolumeRendering(::benchmark::State& state) void BenchVolumeRenderingGenerator(::benchmark::internal::Benchmark* bm) { - bm->ArgNames({ "Cycle", "IsMultiBlock" }); + bm->ArgNames({ "IsMultiBlock" }); std::vector isMultiBlocks{ false }; - for (uint32_t cycle = 1; cycle <= DEFAULT_NUM_CYCLES; ++cycle) + for (auto& isMultiBlock : isMultiBlocks) { - for (auto& isMultiBlock : isMultiBlocks) - { - bm->Args({ cycle, isMultiBlock }); - } + size_t hash_val = 0; + hash_combine(hash_val, std::string("BenchVolumeRendering"), isMultiBlock); + auto search = bench_cycles.find(hash_val); + // If we can't find the hash, or we're not doing repetitions, reset to 0 + if (search == bench_cycles.end()) + bench_cycles[hash_val] = 0; + bm->Args({ isMultiBlock }); } } @@ -887,55 +984,6 @@ void ParseBenchmarkOptions(int& argc, char** argv) std::cerr << "Using data set dimensions = " << DataSetDim << std::endl; std::cerr << "Using image size = " << ImageSize << "x" << ImageSize << std::endl; - - // Now go back through the arg list and remove anything that is not in the list of - // unknown options or non-option arguments. - int destArg = 1; - // This is copy/pasted from vtkm::cont::Initialize(), should probably be abstracted eventually: - for (int srcArg = 1; srcArg < argc; ++srcArg) - { - std::string thisArg{ argv[srcArg] }; - bool copyArg = false; - - // Special case: "--" gets removed by optionparser but should be passed. - if (thisArg == "--") - { - copyArg = true; - } - for (const option::Option* opt = options[UNKNOWN]; !copyArg && opt != nullptr; - opt = opt->next()) - { - if (thisArg == opt->name) - { - copyArg = true; - } - if ((opt->arg != nullptr) && (thisArg == opt->arg)) - { - copyArg = true; - } - // Special case: optionparser sometimes removes a single "-" from an option - if (thisArg.substr(1) == opt->name) - { - copyArg = true; - } - } - for (int nonOpt = 0; !copyArg && nonOpt < commandLineParse.nonOptionsCount(); ++nonOpt) - { - if (thisArg == commandLineParse.nonOption(nonOpt)) - { - copyArg = true; - } - } - if (copyArg) - { - if (destArg != srcArg) - { - argv[destArg] = argv[srcArg]; - } - ++destArg; - } - } - argc = destArg; } } // end anon namespace @@ -957,5 +1005,29 @@ int main(int argc, char* argv[]) vtkm::cont::GetRuntimeDeviceTracker().ForceDevice(Config.Device); } + bool benchmark_min_time = false; + bool benchmark_report_aggregates_only = false; + for (auto i = 0; i <= argc; i++) + if (!strncmp(args[i], "--benchmark_repetitions", 23)) + benchmark_repetitions = true; + else if (!strncmp(args[i], "--benchmark_min_time", 20)) + benchmark_min_time = true; + else if (!strncmp(args[i], "--benchmark_report_aggregates_only", 34)) + benchmark_report_aggregates_only = true; + + // If repetitions are explicitly set without also specifying a minimum_time, + // force the minimum time to be fairly small so that in all likelyhood, benchmarks + // will only run 1 iteration for each test + // + // And, for good measure, only output the accumulated statistics + if (benchmark_repetitions) + { + if (!benchmark_min_time) + args[argc++] = strdup("--benchmark_min_time=0.00000001"); + + if (!benchmark_report_aggregates_only) + args[argc++] = strdup("--benchmark_report_aggregates_only=true"); + } + VTKM_EXECUTE_BENCHMARKS(argc, args.data()); } diff --git a/benchmarking/README_insitu.md b/benchmarking/README_insitu.md new file mode 100644 index 000000000..602d32268 --- /dev/null +++ b/benchmarking/README_insitu.md @@ -0,0 +1,20 @@ +This document describes how to use the command-line options for Google Benchmarks (GBench) to control the behavior of BenchmarkInSitu. + +Generally, "BenchmarkInSitu --help" will provide the list of standard benchmarks along with the associated ones from GBench. + +As a refresher, GBench iterates a test defined in the application, in this case, Contour, Streamlines, ..., a number of times until two criteria are met, a statistically stable set of samples have been generated, and the test ran for a specified minimum amount of time (by default, 0.5 seconds). + +There are three ways to run the InSitu benchmark that control the number of iterations run by GBench. These are independent of the "standard" arguments passed to the benchmark (we'll define the standard arguments as: --vtkm-device, --size, --image-size, plus other not defined by GBench). + +1. BenchmarkInSitu + - Under this scenario, the iterations are controlled completely by GBench. Generally, each test will be run between 1 and N iterations depending on how long each test runs. + +2. BenchmarkInSitu --benchmark_min_time= + - This will ensure that the test will run for at least seconds. You will set this option if you don't care about the actual number of iterations, but only that each test runs for at least a specified time. + +3. BenchmarkInSitu --benchmark_repetitions= + - The purpose of this option is to *exactly* control the number of iterations performed by GBench. Internally, this does two things: + - Sets the minimum time to a very small value ("--benchmark_min_time=0.00000001") + - Sets the output to only report aggregate statistics for each test, e.g., mean, median, standard deviation (--benchmark_report_aggregates_only=true) + Both of these arguments can be overridden by providing different values on the command-line. With the current setting, all test runs have resulted in only repetitions being executed. +