mirror of
https://gitlab.kitware.com/vtk/vtk-m
synced 2024-10-05 01:49:02 +00:00
Cleanup / expand the benchmark parser scripts.
This commit is contained in:
parent
39f4255263
commit
f16e1011ed
@ -3,14 +3,11 @@
|
||||
# Compares the output from BenchmarkDeviceAdapter from the serial
|
||||
# device to a parallel device and prints a table containing the results.
|
||||
#
|
||||
# While this was written for the device adapter algorithms, it could be used
|
||||
# to compare any VTKM benchmark output.
|
||||
#
|
||||
# Example usage:
|
||||
#
|
||||
# $ BenchmarkDeviceAdapter_SERIAL > serial.out
|
||||
# $ BenchmarkDeviceAdapter_TBB > tbb.out
|
||||
# $ devAlgoBenchSummary.py serial.out tbb.out
|
||||
# $ benchCompare.py serial.out tbb.out
|
||||
#
|
||||
#
|
||||
# The number of threads (optional -- only used to generate the "Warn" column)
|
111
Utilities/Scripts/benchSummary.py
Executable file
111
Utilities/Scripts/benchSummary.py
Executable file
@ -0,0 +1,111 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# Prints a concise summary of a benchmark output as a TSV blob.
|
||||
#
|
||||
# Example usage:
|
||||
#
|
||||
# $ BenchmarkXXX_DEVICE > bench.out
|
||||
# $ benchSummary.py bench.out
|
||||
#
|
||||
# Options SortByType, SortByName, or SortByMean may be passed after the
|
||||
# filename to sort the output by the indicated quantity. If no sort option
|
||||
# is provided, the output order matches the input. If multiple options are
|
||||
# specified, the list will be sorted repeatedly in the order requested.
|
||||
|
||||
import re
|
||||
import sys
|
||||
|
||||
assert(len(sys.argv) >= 2)
|
||||
|
||||
# Parses "*** vtkm::Float64 ***************" --> vtkm::Float64
|
||||
typeParser = re.compile("\\*{3} ([^*]+) \\*{15}")
|
||||
|
||||
# Parses "Benchmark 'Benchmark name' results:" --> Benchmark name
|
||||
nameParser = re.compile("Benchmark '([^-]+)' results:")
|
||||
|
||||
# Parses "mean = 0.0125s" --> 0.0125
|
||||
meanParser = re.compile("\\s+mean = ([0-9.Ee+-]+)s")
|
||||
|
||||
# Parses "std dev = 0.0125s" --> 0.0125
|
||||
stdDevParser = re.compile("\\s+std dev = ([naN0-9.Ee+-]+)s")
|
||||
|
||||
filename = sys.argv[1]
|
||||
benchFile = open(filename, 'r')
|
||||
|
||||
sortOpt = None
|
||||
if len(sys.argv) > 2:
|
||||
sortOpt = sys.argv[2:]
|
||||
|
||||
class BenchKey:
|
||||
def __init__(self, name_, type_):
|
||||
self.name = name_
|
||||
self.type = type_
|
||||
|
||||
def __eq__(self, other):
|
||||
return self.name == other.name and self.type == other.type
|
||||
|
||||
def __lt__(self, other):
|
||||
if self.name < other.name: return True
|
||||
elif self.name > other.name: return False
|
||||
else: return self.type < other.type
|
||||
|
||||
def __hash__(self):
|
||||
return (self.name + self.type).__hash__()
|
||||
|
||||
class BenchData:
|
||||
def __init__(self, mean_, stdDev_):
|
||||
self.mean = mean_
|
||||
self.stdDev = stdDev_
|
||||
|
||||
def parseFile(f, benchmarks):
|
||||
type = ""
|
||||
bench = ""
|
||||
mean = -1.
|
||||
stdDev = -1.
|
||||
for line in f:
|
||||
typeRes = typeParser.match(line)
|
||||
if typeRes:
|
||||
type = typeRes.group(1)
|
||||
continue
|
||||
|
||||
nameRes = nameParser.match(line)
|
||||
if nameRes:
|
||||
name = nameRes.group(1)
|
||||
continue
|
||||
|
||||
meanRes = meanParser.match(line)
|
||||
if meanRes:
|
||||
mean = float(meanRes.group(1))
|
||||
continue
|
||||
|
||||
stdDevRes = stdDevParser.match(line)
|
||||
if stdDevRes:
|
||||
stdDev = float(stdDevRes.group(1))
|
||||
|
||||
# stdDev is always the last parse for a given benchmark, add entry now
|
||||
benchmarks[BenchKey(name, type)] = BenchData(mean, stdDev)
|
||||
|
||||
mean = -1.
|
||||
stdDev = -1.
|
||||
|
||||
continue
|
||||
|
||||
benchmarks = {}
|
||||
parseFile(benchFile, benchmarks)
|
||||
|
||||
# Sort keys by type:
|
||||
keys = benchmarks.keys()
|
||||
if sortOpt:
|
||||
for opt in sortOpt:
|
||||
if opt.lower() == "sortbytype":
|
||||
keys = sorted(keys, key=lambda k: k.type)
|
||||
elif opt.lower() == "sortbyname":
|
||||
keys = sorted(keys, key=lambda k: k.name)
|
||||
elif opt.lower() == "sortbymean":
|
||||
keys = sorted(keys, key=lambda k: benchmarks[k].mean)
|
||||
|
||||
print("# Summary: (%s)"%filename)
|
||||
print("%-9s\t%-9s\t%-s"%("Mean", "Stdev", "Benchmark (type)"))
|
||||
for key in keys:
|
||||
data = benchmarks[key]
|
||||
print("%9.6f\t%9.6f\t%s (%s)"%(data.mean, data.stdDev, key.name, key.type))
|
156
Utilities/Scripts/benchSummaryWithBaselines.py
Executable file
156
Utilities/Scripts/benchSummaryWithBaselines.py
Executable file
@ -0,0 +1,156 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# Prints a concise summary of a benchmark output as a TSV blob. Benchmarks are
|
||||
# expected to have "Baseline" in the name, and a matching benchmark with the
|
||||
# same name but Baseline replaced with something else. For example,
|
||||
#
|
||||
# Baseline benchmark name: "Some benchmark: Baseline, Size=4"
|
||||
# Test benchmark name: "Some benchmark: Blahblah, Size=4"
|
||||
#
|
||||
# The output will print the baseline, test, and overhead times for the
|
||||
# benchmarks.
|
||||
#
|
||||
# Example usage:
|
||||
#
|
||||
# $ BenchmarkXXX_DEVICE > bench.out
|
||||
# $ benchSummaryWithBaselines.py bench.out
|
||||
#
|
||||
# Options SortByType, SortByName, SortByOverhead, or SortByRatio
|
||||
# (testtime/baseline) may be passed after the filename to sort the output by
|
||||
# the indicated quantity. If no sort option is provided, the output order
|
||||
# matches the input. If multiple options are specified, the list will be sorted
|
||||
# repeatedly in the order requested.
|
||||
|
||||
import re
|
||||
import sys
|
||||
|
||||
assert(len(sys.argv) >= 2)
|
||||
|
||||
# Parses "*** vtkm::Float64 ***************" --> vtkm::Float64
|
||||
typeParser = re.compile("\\*{3} ([^*]+) \\*{15}")
|
||||
|
||||
# Parses "Benchmark 'Benchmark name' results:" --> Benchmark name
|
||||
nameParser = re.compile("Benchmark '([^-]+)' results:")
|
||||
|
||||
# Parses "mean = 0.0125s" --> 0.0125
|
||||
meanParser = re.compile("\\s+mean = ([0-9.Ee+-]+)s")
|
||||
|
||||
# Parses "std dev = 0.0125s" --> 0.0125
|
||||
stdDevParser = re.compile("\\s+std dev = ([naN0-9.Ee+-]+)s")
|
||||
|
||||
# Parses "SomeText Baseline Other Text" --> ("SomeText ", " Other Text")
|
||||
baselineParser = re.compile("(.*)Baseline(.*)")
|
||||
|
||||
filename = sys.argv[1]
|
||||
benchFile = open(filename, 'r')
|
||||
|
||||
sortOpt = None
|
||||
if len(sys.argv) > 2:
|
||||
sortOpt = sys.argv[2:]
|
||||
|
||||
class BenchKey:
|
||||
def __init__(self, name_, type_):
|
||||
self.name = name_
|
||||
self.type = type_
|
||||
|
||||
def __eq__(self, other):
|
||||
return self.name == other.name and self.type == other.type
|
||||
|
||||
def __lt__(self, other):
|
||||
if self.name < other.name: return True
|
||||
elif self.name > other.name: return False
|
||||
else: return self.type < other.type
|
||||
|
||||
def __hash__(self):
|
||||
return (self.name + self.type).__hash__()
|
||||
|
||||
class BenchData:
|
||||
def __init__(self, mean_, stdDev_):
|
||||
self.mean = mean_
|
||||
self.stdDev = stdDev_
|
||||
|
||||
def parseFile(f, benchmarks):
|
||||
type = ""
|
||||
bench = ""
|
||||
mean = -1.
|
||||
stdDev = -1.
|
||||
for line in f:
|
||||
typeRes = typeParser.match(line)
|
||||
if typeRes:
|
||||
type = typeRes.group(1)
|
||||
continue
|
||||
|
||||
nameRes = nameParser.match(line)
|
||||
if nameRes:
|
||||
name = nameRes.group(1)
|
||||
continue
|
||||
|
||||
meanRes = meanParser.match(line)
|
||||
if meanRes:
|
||||
mean = float(meanRes.group(1))
|
||||
continue
|
||||
|
||||
stdDevRes = stdDevParser.match(line)
|
||||
if stdDevRes:
|
||||
stdDev = float(stdDevRes.group(1))
|
||||
|
||||
# stdDev is always the last parse for a given benchmark, add entry now
|
||||
benchmarks[BenchKey(name, type)] = BenchData(mean, stdDev)
|
||||
|
||||
mean = -1.
|
||||
stdDev = -1.
|
||||
|
||||
continue
|
||||
|
||||
class BaselinedBenchData:
|
||||
def __init__(self, baseline, test):
|
||||
self.baseline = baseline.mean
|
||||
self.test = test.mean
|
||||
self.overhead = test.mean - baseline.mean
|
||||
|
||||
def findBaselines(benchmarks):
|
||||
result = {}
|
||||
|
||||
for baseKey in benchmarks.keys():
|
||||
# Look for baseline entries
|
||||
baselineRes = baselineParser.match(baseKey.name)
|
||||
if baselineRes:
|
||||
prefix = baselineRes.group(1)
|
||||
suffix = baselineRes.group(2)
|
||||
|
||||
# Find the test entry matching the baseline:
|
||||
for testKey in benchmarks.keys():
|
||||
if baseKey.type != testKey.type: # Need same type
|
||||
continue
|
||||
if baseKey.name == testKey.name: # Skip the base key
|
||||
continue
|
||||
if testKey.name.startswith(prefix) and testKey.name.endswith(suffix):
|
||||
newName = (prefix + suffix).replace(", ,", ",")
|
||||
newKey = BenchKey(newName, testKey.type)
|
||||
newVal = BaselinedBenchData(benchmarks[baseKey], benchmarks[testKey])
|
||||
result[newKey] = newVal
|
||||
return result
|
||||
|
||||
benchmarks = {}
|
||||
parseFile(benchFile, benchmarks)
|
||||
benchmarks = findBaselines(benchmarks)
|
||||
|
||||
# Sort keys by type:
|
||||
keys = benchmarks.keys()
|
||||
if sortOpt:
|
||||
for opt in sortOpt:
|
||||
if opt.lower() == "sortbytype":
|
||||
keys = sorted(keys, key=lambda k: k.type)
|
||||
elif opt.lower() == "sortbyname":
|
||||
keys = sorted(keys, key=lambda k: k.name)
|
||||
elif opt.lower() == "sortbyoverhead":
|
||||
keys = sorted(keys, key=lambda k: benchmarks[k].overhead)
|
||||
elif opt.lower() == "sortbyratio":
|
||||
keys = sorted(keys, key=lambda k: benchmarks[k].overhead / benchmarks[k].baseline)
|
||||
|
||||
print("# Summary: (%s)"%filename)
|
||||
print("%-9s\t%-9s\t%-9s\t%-9s\t%-s"%("Baseline", "TestTime", "Overhead", "Test/Base", "Benchmark (type)"))
|
||||
for key in keys:
|
||||
data = benchmarks[key]
|
||||
print("%9.6f\t%9.6f\t%9.6f\t%9.6f\t%s (%s)"%(data.baseline, data.test,
|
||||
data.overhead, data.test / data.baseline, key.name, key.type))
|
Loading…
Reference in New Issue
Block a user