mirror of
https://gitlab.kitware.com/vtk/vtk-m
synced 2024-10-05 01:49:02 +00:00
Add python script that diffs VTKM benchmarks.
This commit is contained in:
parent
d4416031c4
commit
e22a410885
163
Utilities/Scripts/devAlgoBenchSummary.py
Executable file
163
Utilities/Scripts/devAlgoBenchSummary.py
Executable file
@ -0,0 +1,163 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# Compares the output from BenchmarkDeviceAdapter from the serial
|
||||
# device to a parallel device and prints a table containing the results.
|
||||
#
|
||||
# While this was written for the device adapter algorithms, it could be used
|
||||
# to compare any VTKM benchmark output.
|
||||
#
|
||||
# Example usage:
|
||||
#
|
||||
# $ BenchmarkDeviceAdapter_SERIAL > serial.out
|
||||
# $ BenchmarkDeviceAdapter_TBB > tbb.out
|
||||
# $ devAlgoBenchSummary.py serial.out tbb.out
|
||||
#
|
||||
#
|
||||
# The number of threads (optional -- only used to generate the "Warn" column)
|
||||
maxThreads = 4
|
||||
#
|
||||
# Print debugging output:
|
||||
doDebug = False
|
||||
#
|
||||
# End config options.
|
||||
|
||||
import re
|
||||
import sys
|
||||
|
||||
assert(len(sys.argv) == 3)
|
||||
|
||||
def debug(str):
|
||||
if (doDebug): print(str)
|
||||
|
||||
# Parses "*** vtkm::Float64 ***************" --> vtkm::Float64
|
||||
typeParser = re.compile("\\*{3} ([^*]+) \\*{15}")
|
||||
|
||||
# Parses "Benchmark 'Benchmark name' results:" --> Benchmark name
|
||||
nameParser = re.compile("Benchmark '([^-]+)' results:")
|
||||
|
||||
# Parses "mean = 0.0125s" --> 0.0125
|
||||
meanParser = re.compile("\\s+mean = ([0-9.Ee+-]+)s")
|
||||
|
||||
# Parses "std dev = 0.0125s" --> 0.0125
|
||||
stdDevParser = re.compile("\\s+std dev = ([0-9.Ee+-]+)s")
|
||||
|
||||
serialFilename = sys.argv[1]
|
||||
parallelFilename = sys.argv[2]
|
||||
|
||||
serialFile = open(serialFilename, 'r')
|
||||
parallelFile = open(parallelFilename, 'r')
|
||||
|
||||
class BenchKey:
|
||||
def __init__(self, name_, type_):
|
||||
self.name = name_
|
||||
self.type = type_
|
||||
|
||||
def __eq__(self, other):
|
||||
return self.name == other.name and self.type == other.type
|
||||
|
||||
def __lt__(self, other):
|
||||
if self.name < other.name: return True
|
||||
elif self.name > other.name: return False
|
||||
else: return self.type < other.type
|
||||
|
||||
def __hash__(self):
|
||||
return (self.name + self.type).__hash__()
|
||||
|
||||
class BenchData:
|
||||
def __init__(self, mean_, stdDev_):
|
||||
self.mean = mean_
|
||||
self.stdDev = stdDev_
|
||||
|
||||
def parseFile(f, benchmarks):
|
||||
type = ""
|
||||
bench = ""
|
||||
mean = -1.
|
||||
stdDev = -1.
|
||||
for line in f:
|
||||
debug("Line: {}".format(line))
|
||||
|
||||
typeRes = typeParser.match(line)
|
||||
if typeRes:
|
||||
type = typeRes.group(1)
|
||||
debug("Found type: {}".format(type))
|
||||
continue
|
||||
|
||||
nameRes = nameParser.match(line)
|
||||
if nameRes:
|
||||
name = nameRes.group(1)
|
||||
debug("Found name: {}".format(name))
|
||||
continue
|
||||
|
||||
meanRes = meanParser.match(line)
|
||||
if meanRes:
|
||||
mean = float(meanRes.group(1))
|
||||
debug("Found mean: {}".format(mean))
|
||||
continue
|
||||
|
||||
stdDevRes = stdDevParser.match(line)
|
||||
if stdDevRes:
|
||||
stdDev = float(stdDevRes.group(1))
|
||||
debug("Found stddev: {}".format(stdDev))
|
||||
|
||||
assert(mean >= 0.)
|
||||
assert(stdDev >= 0.)
|
||||
|
||||
# stdDev is always the last parse for a given benchmark, add entry now
|
||||
benchmarks[BenchKey(name, type)] = BenchData(mean, stdDev)
|
||||
debug("{} records found.".format(len(benchmarks)))
|
||||
|
||||
mean = -1.
|
||||
stdDev = -1.
|
||||
|
||||
continue
|
||||
|
||||
serialBenchmarks = {}
|
||||
parallelBenchmarks = {}
|
||||
|
||||
parseFile(serialFile, serialBenchmarks)
|
||||
parseFile(parallelFile, parallelBenchmarks)
|
||||
|
||||
serialKeys = set(serialBenchmarks.keys())
|
||||
parallelKeys = set(parallelBenchmarks.keys())
|
||||
|
||||
commonKeys = sorted(list(serialKeys.intersection(parallelKeys)))
|
||||
|
||||
serialOnlyKeys = sorted(list(serialKeys.difference(parallelKeys)))
|
||||
parallelOnlyKeys = sorted(list(parallelKeys.difference(serialKeys)))
|
||||
|
||||
debug("{} serial keys\n{} parallel keys\n{} common keys\n{} serialOnly keys\n{} parallelOnly keys.".format(
|
||||
len(serialKeys), len(parallelKeys), len(commonKeys), len(serialOnlyKeys), len(parallelOnlyKeys)))
|
||||
|
||||
if len(serialOnlyKeys) > 0:
|
||||
print("Keys found only in serial:")
|
||||
for k in serialOnlyKeys:
|
||||
print("%s (%s)"%(k.name, k.type))
|
||||
print("")
|
||||
|
||||
if len(parallelOnlyKeys) > 0:
|
||||
print("Keys found only in parallel:")
|
||||
for k in parallelOnlyKeys:
|
||||
print("%s (%s)"%(k.name, k.type))
|
||||
print("")
|
||||
|
||||
print("Comparison:")
|
||||
print("| %7s | %4s | %8s %8s | %8s %8s | %s (%s) |"%(
|
||||
"Speedup", "Warn", "serial", "", "parallel", "", "Benchmark", "Type"))
|
||||
print("|-%7s-|-%4s-|-%8s----%8s-|-%8s----%8s-|-%s--%s--|"%(
|
||||
"-"*7, "-"*4, "-"*8, "-"*8, "-"*8, "-"*8, "-"*9, "-"*4))
|
||||
for key in commonKeys:
|
||||
sData = serialBenchmarks[key]
|
||||
pData = parallelBenchmarks[key]
|
||||
speedup = sData.mean / pData.mean if pData.mean != 0. else 0.
|
||||
if speedup > maxThreads * .9:
|
||||
flag = " "
|
||||
elif speedup > maxThreads * .75:
|
||||
flag = "! "
|
||||
elif speedup > maxThreads * .5:
|
||||
flag = "!! "
|
||||
elif speedup > maxThreads * .25:
|
||||
flag = "!!! "
|
||||
else:
|
||||
flag = "!!!!"
|
||||
print("| %7.3f | %4s | %08.6f +- %08.6f | %08.6f +- %08.6f | %s (%s) |"%(
|
||||
speedup, flag, sData.mean, sData.stdDev, pData.mean, pData.stdDev, key.name, key.type))
|
Loading…
Reference in New Issue
Block a user