06_baselines.py

import numpy as np
import matplotlib.pyplot as plt
from config import conf
import os, sys
import pandas as pns
from config import names as gs
import getopt
import matplotlib.gridspec as gridspec
from sklearn.metrics import f1_score, accuracy_score

import seaborn as sns
sns.set(style='whitegrid', color_codes=True)
sns.set_context('poster')

dark_color = sns.xkcd_rgb['charcoal grey']
light_color = sns.xkcd_rgb['cloudy blue']

max_n_feat = conf.max_n_feat
m_iter = conf.max_n_iter

featurelabels = gs.full_long_label_list
participant_ids = np.arange(0, conf.n_participants)


def plot_overview():
	all_baselines.groupby(by=['trait', 'clf_name'])['F1'].mean().to_csv(conf.figure_folder +
							'/figure1.csv')
	print 'Figure1.csv written'

	sns.set(font_scale=2.1)
	plt.figure(figsize=(20, 10))
	ax = plt.subplot(1,1,1)
	sns.barplot(x='trait', y='F1', hue='clf_name', data=all_baselines, capsize=.05, errwidth=3,
	            linewidth=3, estimator=np.mean, edgecolor=dark_color,
	            palette={'our classifier': sns.xkcd_rgb['windows blue'],
	                     'most frequent class': sns.xkcd_rgb['faded green'],
	                     'random guess':sns.xkcd_rgb['greyish brown'],
						 'label permutation':sns.xkcd_rgb['dusky pink']
                        }
	            )
	plt.plot([-0.5,6.5], [0.33, 0.33], c=dark_color, linestyle='--', linewidth=3, label='theoretical chance level')
	handles, labels = ax.get_legend_handles_labels()
	ax.legend([handles[1], handles[2], handles[3], handles[4], handles[0]], [labels[1], labels[2], labels[3], labels[4], labels[0]], fontsize=20)
	plt.xlabel('')
	plt.ylabel('F1 score', fontsize=20)
	plt.ylim((0, 0.55))
	filename = conf.figure_folder + '/figure1.pdf'
	plt.savefig(filename, bbox_inches='tight')
	plt.close()
	print 'wrote', filename.split('/')[-1]


if __name__ == "__main__":
	# collect F1 scores for classifiers on all data from a file that was written by evaluation_single_context.py
	datapath = conf.get_result_folder(conf.annotation_all) + '/f1s.csv'
	if not os.path.exists(datapath):
		print 'could not find', datapath
		print 'consider (re-)running evaluation_single_context.py'
		sys.exit(1)
	our_classifier = pns.read_csv(datapath)
	our_classifier['clf_name'] = 'our classifier'

	# baseline 1: guess the most frequent class from each training set that was written by train_baseline.py
	datapath = conf.result_folder + '/most_frequ_class_baseline.csv'
	if not os.path.exists(datapath):
		print 'could not find', datapath
		print 'consider (re-)running train_baseline.py'
		sys.exit(1)
	most_frequent_class_df = pns.read_csv(datapath)
	most_frequent_class_df['clf_name'] = 'most frequent class'

	# compute all other baselines ad hoc
	collection = []
	for trait in xrange(0, conf.n_traits):
		# baseline 2: random guess
		truth = np.genfromtxt(conf.binned_personality_file, skip_header=1, usecols=(trait+1,), delimiter=',')
		for i in xrange(0, 100):
			rand_guess = np.random.randint(1, 4, conf.n_participants)
			f1 = f1_score(truth, rand_guess, average='macro')
			collection.append([f1, conf.medium_traitlabels[trait], i, 'random guess'])

		# baseline 3: label permutation test
		#             was computed using label_permutation_test.sh and written into results. ie. is just loaded here
		for si in xrange(0, m_iter):
			filename_rand = conf.get_result_filename(conf.annotation_all, trait, True, si, add_suffix=True)
			if os.path.exists(filename_rand):
				data = np.load(filename_rand)
				pr = data['predictions']
				dt = truth[pr > 0]
				pr = pr[pr > 0]
				f1 = f1_score(dt, pr, average='macro')
				collection.append([f1, conf.medium_traitlabels[trait], si, 'label permutation'])
			else:
				print 'did not find', filename_rand
				print 'consider (re-)running label_permutation_test.sh'
				sys.exit(1)

	collectiondf = pns.DataFrame(data=collection,columns=['F1','trait','iteration','clf_name'])
	all_baselines = pns.concat([our_classifier, most_frequent_class_df, collectiondf])

	plot_overview()  # Figure 1