03_train_baseline.py

import sys
import numpy as np
from config import conf
import getopt
from sklearn.cross_validation import LabelKFold as LKF
from sklearn.cross_validation import StratifiedKFold as SKF
from sklearn.metrics import f1_score, accuracy_score
import pandas as pns

def load_data(ws, t):
	_, y_file, id_file = conf.get_merged_feature_files(ws)
	y_ws = np.genfromtxt(y_file, delimiter=',', skip_header=1).astype(int)[:,t]
	ids_ws = np.genfromtxt(id_file, delimiter=',', skip_header=1).astype(int)[:,0]
	return y_ws, ids_ws

def get_baseline_f1_score(t):
	"""
	train a baseline classifier and return the F1 score it achieves
	"""
	outer_cv = SKF(participant_scores, conf.n_outer_folds, shuffle=True)

	preds = np.zeros((conf.n_participants), dtype=int)
	truth = np.zeros((conf.n_participants), dtype=int)

	for outer_i, (outer_train_participants, outer_test_participants) in enumerate(outer_cv):
		inner_performance = np.zeros((conf.n_inner_folds, len(conf.all_window_sizes)))

		for ws_i in xrange(0, len(conf.all_window_sizes)):
			ws = conf.all_window_sizes[ws_i]

			# load data for this window size
			y_ws, ids_ws = load_data(ws, t)

			# cut out the outer train samples
			outer_train_samples = np.array([p in outer_train_participants for p in ids_ws])
			outer_train_y = y_ws[outer_train_samples]
			outer_train_y_ids = ids_ws[outer_train_samples]

			# build inner cross validation such that all samples of one person are either in training or testing
			inner_cv = LKF(outer_train_y_ids, n_folds=conf.n_inner_folds)
			for inner_i, (inner_train_indices, inner_test_indices) in enumerate(inner_cv):
				# create inner train and test samples. Note: both are taken from outer train samples!
				inner_y_train = outer_train_y[inner_train_indices]
				unique_inner_test_ids = np.unique(outer_train_y_ids[inner_test_indices])

				# predict the most frequent class from the training set
				hist,_ = np.histogram(inner_y_train, bins=[0.5,1.5,2.5,3.5])
				guess = np.argmax(hist) + 1
				innerpreds = np.full(len(unique_inner_test_ids), guess, dtype=int)
				innertruth = participant_scores[unique_inner_test_ids]

				inner_performance[inner_i, ws_i] = accuracy_score(np.array(innertruth), np.array(innerpreds))

		# evaluate classifier on outer cv using the best window size from inner cv
		chosen_ws_i = np.argmax(np.mean(inner_performance, axis=0))
		chosen_ws = conf.all_window_sizes[chosen_ws_i]
		y, ids = load_data(chosen_ws, t)

		outer_train_samples = np.array([p in outer_train_participants for p in ids])
		outer_test_samples = np.array([p in outer_test_participants for p in ids])

		if outer_train_samples.size > 0 and outer_test_samples.size > 0:
			y_train = y[outer_train_samples]

			# guess the most frequent class
			hist,_ = np.histogram(y_train, bins=[0.5, 1.5, 2.5, 3.5])
			guess = np.argmax(hist) + 1

			for testp in outer_test_participants:
				if testp in ids[outer_test_samples]:
					preds[testp] = guess
					truth[testp] = participant_scores[testp]
				else:
					# participant does not occour in outer test set, e.g. because their time in the shop was too short
					preds[testp] = -1
					truth[testp] = -1
					print 'not enough samples for participant', testp
			#print 'preds collected'
		else:
			for testp in outer_test_participants:
				preds[testp] = np.array([])
				truth[testp] = -1

	f1 = f1_score(truth, preds, average='macro')
	return f1

# If the program is run directly:
if __name__ == "__main__":
	df = []
	for trait in xrange(0, conf.n_traits):
		participant_scores = np.loadtxt(conf.binned_personality_file, delimiter=',', skiprows=1, usecols=(trait+1,))
		print conf.medium_traitlabels[trait]
		for si in xrange(0,conf.max_n_iter):
			f1 = get_baseline_f1_score(trait)
			print '\t'+str(si)+':', f1
			df.append([f1, conf.medium_traitlabels[trait], si])
	df_pns = pns.DataFrame(data=df, columns=['F1', 'trait', 'iteration'])
	df_pns.to_csv(conf.result_folder + '/most_frequ_class_baseline.csv')
	print conf.result_folder + '/most_frequ_class_baseline.csv written.'