mirror of
https://gitlab.kitware.com/vtk/vtk-m
synced 2024-10-08 03:18:58 +00:00
compare.py: add normal and t-student utest
This commit is contained in:
parent
706bd8d2b7
commit
19f10b9a0b
@ -55,6 +55,15 @@ def create_parser():
|
|||||||
default=True,
|
default=True,
|
||||||
action="store_false",
|
action="store_false",
|
||||||
help="The tool can do a two-tailed Mann-Whitney U test with the null hypothesis that it is equally likely that a randomly selected value from one sample will be less than or greater than a randomly selected value from a second sample.\nWARNING: requires **LARGE** (no less than {}) number of repetitions to be meaningful!\nThe test is being done by default, if at least {} repetitions were done.\nThis option can disable the U Test.".format(report.UTEST_OPTIMAL_REPETITIONS, report.UTEST_MIN_REPETITIONS))
|
help="The tool can do a two-tailed Mann-Whitney U test with the null hypothesis that it is equally likely that a randomly selected value from one sample will be less than or greater than a randomly selected value from a second sample.\nWARNING: requires **LARGE** (no less than {}) number of repetitions to be meaningful!\nThe test is being done by default, if at least {} repetitions were done.\nThis option can disable the U Test.".format(report.UTEST_OPTIMAL_REPETITIONS, report.UTEST_MIN_REPETITIONS))
|
||||||
|
|
||||||
|
utest.add_argument(
|
||||||
|
'--dist',
|
||||||
|
dest='utest_dist',
|
||||||
|
choices=['mannwhitney', 'normal', 't'],
|
||||||
|
default='mannwhitney',
|
||||||
|
type=str,
|
||||||
|
help="Utest probabilistic distribution to use")
|
||||||
|
|
||||||
alpha_default = 0.05
|
alpha_default = 0.05
|
||||||
utest.add_argument(
|
utest.add_argument(
|
||||||
"--alpha",
|
"--alpha",
|
||||||
@ -239,7 +248,7 @@ def main():
|
|||||||
# Diff and output
|
# Diff and output
|
||||||
output_lines = gbench.report.generate_difference_report(
|
output_lines = gbench.report.generate_difference_report(
|
||||||
json1, json2, args.display_aggregates_only,
|
json1, json2, args.display_aggregates_only,
|
||||||
args.utest, args.utest_alpha, sys.stdout.isatty())
|
args.utest, args.utest_dist, args.utest_alpha, sys.stdout.isatty())
|
||||||
print(description)
|
print(description)
|
||||||
for ln in output_lines:
|
for ln in output_lines:
|
||||||
print(ln)
|
print(ln)
|
||||||
|
@ -6,7 +6,11 @@ import re
|
|||||||
import copy
|
import copy
|
||||||
|
|
||||||
from scipy.stats import mannwhitneyu
|
from scipy.stats import mannwhitneyu
|
||||||
|
from scipy.stats import norm
|
||||||
|
from scipy.stats import t
|
||||||
|
|
||||||
|
from statistics import mean
|
||||||
|
from statistics import stdev
|
||||||
|
|
||||||
class BenchmarkColor(object):
|
class BenchmarkColor(object):
|
||||||
def __init__(self, name, code):
|
def __init__(self, name, code):
|
||||||
@ -38,6 +42,7 @@ BC_UNDERLINE = BenchmarkColor('UNDERLINE', '\033[4m')
|
|||||||
UTEST_MIN_REPETITIONS = 2
|
UTEST_MIN_REPETITIONS = 2
|
||||||
UTEST_OPTIMAL_REPETITIONS = 9 # Lowest reasonable number, More is better.
|
UTEST_OPTIMAL_REPETITIONS = 9 # Lowest reasonable number, More is better.
|
||||||
UTEST_COL_NAME = "_pvalue"
|
UTEST_COL_NAME = "_pvalue"
|
||||||
|
UTEST_NORM_MIN_STDEV = 0.05
|
||||||
|
|
||||||
|
|
||||||
def color_format(use_color, fmt_str, *args, **kwargs):
|
def color_format(use_color, fmt_str, *args, **kwargs):
|
||||||
@ -154,7 +159,7 @@ def extract_field(partition, field_name):
|
|||||||
rhs = [x[field_name] for x in partition[1]]
|
rhs = [x[field_name] for x in partition[1]]
|
||||||
return [lhs, rhs]
|
return [lhs, rhs]
|
||||||
|
|
||||||
def calc_utest(timings_cpu, timings_time):
|
def calc_utest(utest_dist, timings_cpu, timings_time):
|
||||||
min_rep_cnt = min(len(timings_time[0]),
|
min_rep_cnt = min(len(timings_time[0]),
|
||||||
len(timings_time[1]),
|
len(timings_time[1]),
|
||||||
len(timings_cpu[0]),
|
len(timings_cpu[0]),
|
||||||
@ -164,20 +169,30 @@ def calc_utest(timings_cpu, timings_time):
|
|||||||
if min_rep_cnt < UTEST_MIN_REPETITIONS:
|
if min_rep_cnt < UTEST_MIN_REPETITIONS:
|
||||||
return False, None, None
|
return False, None, None
|
||||||
|
|
||||||
time_pvalue = mannwhitneyu(
|
pvalue_fn = str
|
||||||
timings_time[0], timings_time[1], alternative='two-sided').pvalue
|
if utest_dist == "normal":
|
||||||
cpu_pvalue = mannwhitneyu(
|
pvalue_fn = lambda x : norm.sf(mean(x[1]), mean(x[0]),
|
||||||
timings_cpu[0], timings_cpu[1], alternative='two-sided').pvalue
|
max(stdev(x[0]), mean(x[0])*UTEST_NORM_MIN_STDEV))
|
||||||
|
elif utest_dist == "t":
|
||||||
|
pvalue_fn = lambda x : t.sf(mean(x[1]), len(x[0]) - 1, mean(x[0]),
|
||||||
|
max(stdev(x[0]), mean(x[0])*UTEST_NORM_MIN_STDEV))
|
||||||
|
elif utest_dist == "mannwhitney":
|
||||||
|
pvalue_fn = lambda x : mannwhitneyu(x[0], x[1], alternative='two-sided').pvalue
|
||||||
|
else:
|
||||||
|
return False, None, None
|
||||||
|
|
||||||
|
time_pvalue = pvalue_fn(timings_time)
|
||||||
|
cpu_pvalue = pvalue_fn(timings_cpu)
|
||||||
|
|
||||||
return (min_rep_cnt >= UTEST_OPTIMAL_REPETITIONS), cpu_pvalue, time_pvalue
|
return (min_rep_cnt >= UTEST_OPTIMAL_REPETITIONS), cpu_pvalue, time_pvalue
|
||||||
|
|
||||||
def print_utest(partition, utest_alpha, first_col_width, use_color=True):
|
def print_utest(partition, utest_dist, utest_alpha, first_col_width, use_color=True):
|
||||||
def get_utest_color(pval):
|
def get_utest_color(pval):
|
||||||
return BC_FAIL if pval >= utest_alpha else BC_OKGREEN
|
return BC_FAIL if pval >= utest_alpha else BC_OKGREEN
|
||||||
|
|
||||||
timings_time = extract_field(partition, 'real_time')
|
timings_time = extract_field(partition, 'real_time')
|
||||||
timings_cpu = extract_field(partition, 'cpu_time')
|
timings_cpu = extract_field(partition, 'cpu_time')
|
||||||
have_optimal_repetitions, cpu_pvalue, time_pvalue = calc_utest(timings_cpu, timings_time)
|
have_optimal_repetitions, cpu_pvalue, time_pvalue = calc_utest(utest_dist, timings_cpu, timings_time)
|
||||||
|
|
||||||
# Check if we failed miserably with minimum required repetitions for utest
|
# Check if we failed miserably with minimum required repetitions for utest
|
||||||
if not have_optimal_repetitions and cpu_pvalue is None and time_pvalue is None:
|
if not have_optimal_repetitions and cpu_pvalue is None and time_pvalue is None:
|
||||||
@ -212,6 +227,7 @@ def generate_difference_report(
|
|||||||
json2,
|
json2,
|
||||||
display_aggregates_only=False,
|
display_aggregates_only=False,
|
||||||
utest=False,
|
utest=False,
|
||||||
|
utest_dist="mannwhitney",
|
||||||
utest_alpha=0.05,
|
utest_alpha=0.05,
|
||||||
use_color=True):
|
use_color=True):
|
||||||
"""
|
"""
|
||||||
@ -279,6 +295,7 @@ def generate_difference_report(
|
|||||||
# After processing the whole partition, if requested, do the U test.
|
# After processing the whole partition, if requested, do the U test.
|
||||||
if utest:
|
if utest:
|
||||||
output_strs += print_utest(partition,
|
output_strs += print_utest(partition,
|
||||||
|
utest_dist=utest_dist,
|
||||||
utest_alpha=utest_alpha,
|
utest_alpha=utest_alpha,
|
||||||
first_col_width=first_col_width,
|
first_col_width=first_col_width,
|
||||||
use_color=use_color)
|
use_color=use_color)
|
||||||
|
Loading…
Reference in New Issue
Block a user