diff --git a/test_synthetic.py b/test_synthetic.py index 8198ae8..8a332c0 100644 --- a/test_synthetic.py +++ b/test_synthetic.py @@ -263,7 +263,6 @@ def test_accuracy(): fp.close() -# todo: use crisp def test_accuracy_at_model_space(): nsim = 1000 size = 1000 @@ -622,6 +621,46 @@ def test_sample_size(): fp.close() +def test_accuracy_pvalue(): + print "testing various critical thresholds for DR" + nsim = 1000 + size = 1000 + level = 0.01 + suppfX = range(-7, 8) + srcX = "geometric" + levels = [0.05, 0.04, 0.03, 0.02, 0.01, 0.005, 0.001] + for level in levels: + nsamples = 0 + ncorrect = 0 + while nsamples < nsim: + X = generate_X(srcX, size) + suppX = list(set(X)) + f = map_randomly(suppX, suppfX) + N = generate_additive_N(size) + Y = [f[X[i]] + N[i] for i in xrange(size)] + + # check if f(x) + supp N are disjoint for x in domx + suppN = set(N) + decomps = [] + for x in suppX: + fx = f[x] + sum_fx_suppN = set([fx + n for n in suppN]) + decomps.append(sum_fx_suppN) + + non_overlapping_noise = are_disjoint(decomps) + if non_overlapping_noise: + continue + + nsamples += 1 + dr_score = dr(X, Y, level) + ncorrect += int(dr_score[0] > level and dr_score[1] < level) + + assert nsamples == nsim + acc = ncorrect * 100 / nsim + print level, acc + + if __name__ == "__main__": - test_sample_size() + # test_sample_size() # test_accuracy() + test_accuracy_pvalue()