diff --git a/test_synthetic.py b/test_synthetic.py index 2febd96..4ea3492 100644 --- a/test_synthetic.py +++ b/test_synthetic.py @@ -7,7 +7,7 @@ from functools import partial import random import sys -from timeit import Timer +import time import numpy as np from nonparametric_tests import friedman_test, nemenyi_multitest, bonferroni_dunn_test @@ -363,33 +363,58 @@ def test_accuracy_with_nonparam_tests(): print "%18s%10.2f%10.2f%10.2f" % (src, ncorrect_this / nsim, ncorrect_dc / nsim, ncorrect_dr / nsim) -def test_runtime(): - nrun = 5 - level = 0.05 - suppfX = range(-7, 8) - nvalues = 10 - sizes = [100000, 200000, 300000, 400000, 500000, - 600000, 700000, 800000, 900000, 1000000] - +def execute_algorithms(X, Y, nloop): + dcX, dcY = dc_compat(X), dc_compat(Y) + dc_t, dr_t, cisc_t = 0, 0, 0 + for i in xrange(nloop): + tstart = time.time() + dc(dcX, dcY) + tend = time.time() + dc_t += tend - tstart + + tstart = time.time() + dr(X, Y, 0.05) + tend = time.time() + dr_t += tend - tstart + + tstart = time.time() + cisc(X, Y) + tend = time.time() + cisc_t += tend - tstart + return dc_t / nloop, dr_t / nloop, cisc_t / nloop + + +def test_size_vs_runtime(): + nloop = 5 + suppX = range(20) + suppY = range(20) + sizes = range(1000000, 11000000, 1000000) + fp = open("results/size_vs_runtime.dat", "w") for size in sizes: - p_nums = [random.randint(1, 10) for i in xrange(nvalues)] - p_vals = [v / sum(p_nums) for v in p_nums] - X = np.random.multinomial(size, p_vals, size=1)[0].tolist() - X = [[i + 1] * f for i, f in enumerate(X)] - X = [j for sublist in X for j in sublist] - suppX = list(set(X)) - f = map_randomly(suppX, suppfX) - N = generate_additive_N(size) - Y = [f[X[i]] + N[i] for i in xrange(size)] - dcX, dcY = dc_compat(X), dc_compat(Y) - - dc_t = Timer(partial(dc, dcX, dcY), - setup="""from __main__ import *""").timeit(nrun) - dr_t = Timer(partial(dr, X, Y, level), - setup="""from __main__ import *""").timeit(nrun) - cisc_t = Timer(partial(cisc, X, Y), - setup="""from __main__ import *""").timeit(nrun) + X = [random.choice(suppX) for i in xrange(size)] + Y = [random.choice(suppY) for i in xrange(size)] + dc_t, dr_t, cisc_t = execute_algorithms(X, Y, nloop) print size, dc_t, dr_t, cisc_t + sys.stdout.flush() + fp.write("%d %f %f %f\n" % (size, dc_t, dr_t, cisc_t)) + fp.close() + + +def test_domain_vs_runtime(): + nloop = 5 + size = 10000 + domains = range(100, 1100, 100) + fp = open("results/domain_vs_runtime.dat", "w") + for domain in domains: + suppX = range(domain) + suppY = range(domain) + X = [random.choice(suppX) for i in xrange(size)] + Y = [random.choice(suppY) for i in xrange(size)] + dc_t, dr_t, cisc_t = execute_algorithms(X, Y, nloop) + print domain, dc_t, dr_t, cisc_t + sys.stdout.flush() + fp.write("%d %f %f %f\n" % (domain, dc_t, dr_t, cisc_t)) + fp.close() def test_power(): @@ -444,7 +469,5 @@ def test_power(): if __name__ == "__main__": - test_decision_rate() - # test_power() - # test_runtime() - # _decision_rate("multinomial") + # test_size_vs_runtime() + test_domain_vs_runtime()