diff --git a/.gitignore b/.gitignore index ae66896..56c42ce 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,7 @@ mxq_log.o mx_mysql.o mxqsub -/mxqsub.1 +mxqsub.1 mxqdump mxqkill mxqd @@ -23,6 +23,8 @@ test_mx_util test_mx_log test_mx_mysql +/web/pages/mxq/mxq + *~ gmon.out diff --git a/Makefile b/Makefile index 6e7e26f..03c853b 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ MXQ_VERSION_MAJOR = 0 -MXQ_VERSION_MINOR = 9 -MXQ_VERSION_PATCH = 1 +MXQ_VERSION_MINOR = 11 +MXQ_VERSION_PATCH = 0 MXQ_VERSION_EXTRA = "beta" MXQ_VERSION = ${MXQ_VERSION_MAJOR}.${MXQ_VERSION_MINOR}.${MXQ_VERSION_PATCH} @@ -39,6 +39,8 @@ ifeq ($(notdir ${LIBEXECDIR}),mxq) override LIBEXECDIR := $(patsubst %/,%,$(dir ${LIBEXECDIR})) endif +CGIDIR = ${LIBEXECDIR}/mxq/cgi + ############################################################################## MXQ_MYSQL_DEFAULT_FILE = ${SYSCONFDIR}/mxq/mysql.cnf @@ -101,6 +103,22 @@ quiet-install = $(call quiet-command,install -m ${1} ${2} ${3},"INSTALL ${3} [mo quiet-installdir = $(call quiet-command,install -m ${1} -d ${2}," MKDIR ${2} [mode=${1}]") quiet-installforuser = $(call quiet-command,install -m ${1} -o ${2} -g ${3} ${4} ${5},"INSTALL ${5} (user=${2} group=${3}) [mode=${1}]") +######################################################################## + +sed-rules = -e 's,@PREFIX@,${PREFIX},g' \ + -e 's,@EPREFIX@,${EPREFIX},g' \ + -e 's,@BINDIR@,${BINDIR},g' \ + -e 's,@SBINDIR@,${SBINDIR},g' \ + -e 's,@LIBDIR@,${LIBDIR},g' \ + -e 's,@SYSCONFDIR@,${SYSCONFDIR},g' \ + -e 's,@DEFCONFDIR@,${DEFCONFDIR},g' \ + -e 's,@LIBEXECDIR@,${LIBEXECDIR},g' \ + -e 's,@BEE_VERSION@,${BEE_VERSION},g' \ + -e 's,@DATADIR@,${DATADIR},g' \ + -e 's,@MXQ_VERSION@,${MXQ_VERSION},g' \ + -e 's,@MXQ_MYSQL_DEFAULT_FILE@,${MXQ_MYSQL_DEFAULT_FILE},g' \ + + ######################################################################## %.o: %.c Makefile @@ -111,6 +129,9 @@ quiet-installforuser = $(call quiet-command,install -m ${1} -o ${2} -g ${3} ${4} %: %.o $(call quiet-command,${CC} -o $@ $^ $(LDFLAGS) $(LDLIBS), " LINK $@") +%: %.in Makefile + $(call quiet-command,sed ${sed-rules} $< >$@, " GEN $@") + ######################################################################## .SECONDARY: @@ -121,7 +142,7 @@ manpages/%: manpages/%.xml $(call quiet-command,xmlto --stringparam man.output.quietly=1 man $^ -o manpages, " XMLTO $@") %: manpages/% Makefile - $(call quiet-command,sed -e "s/@MXQ_VERSION@/${MXQ_VERSION}/" $< >$@, " GEN $@") + $(call quiet-command,sed ${sed-rules} $< >$@, " GEN $@") ######################################################################## @@ -179,6 +200,7 @@ install:: $(call quiet-installdir,0755,${DESTDIR}${SBINDIR}) $(call quiet-installdir,0755,${DESTDIR}${SYSCONFDIR}/mxq) $(call quiet-installdir,0755,${DESTDIR}${MAN1DIR}) + $(call quiet-installdir,0755,${DESTDIR}${CGIDIR}) ######################################################################## @@ -419,6 +441,15 @@ clean: CLEAN += mxqsub.1 ######################################################################## +build: web/pages/mxq/mxq + +clean: CLEAN += web/pages/mxq/mxq + +install:: web/pages/mxq/mxq + $(call quiet-install,0755,$^,${DESTDIR}${CGIDIR}/mxq) + +######################################################################## + test_mx_util.o: $(mx_util.h) clean: CLEAN += test_mx_util.o diff --git a/README.md b/README.md new file mode 100644 index 0000000..6ec1688 --- /dev/null +++ b/README.md @@ -0,0 +1,14 @@ +# mxq +MXQ - mariux64 job scheduling system + +## Installation +### Install using `GNU make` +``` +make +make install [DESTDIR=...] +``` +### Install using `bee` +``` +bee init $(bee download git://github.molgen.mpg.de/mariux64/mxq.git) -e +bee install mxq +``` diff --git a/mxqsub.c b/mxqsub.c index cbd311c..7faabe1 100644 --- a/mxqsub.c +++ b/mxqsub.c @@ -551,8 +551,10 @@ int main(int argc, char *argv[]) case 'a': p = strchr(optctl.optarg, ' '); - if (p) - *p = 0; + if (p) { + mx_log_crit("--command-alias '%s': String contains whitespace characters.", optctl.optarg); + exit(EX_CONFIG); + } if (!(*optctl.optarg)) { mx_log_crit("--command-alias '%s': String is empty.", optctl.optarg); exit(EX_CONFIG); @@ -647,6 +649,15 @@ int main(int argc, char *argv[]) /* from this point values in argc,argv are the ones of the cluster job */ + if (!arg_program_name) { + p = strchr(argv[0], ' '); + if (p) { + mx_log_crit(" contains whitespace characters. Please set --command-alias if this is intended.", optctl.optarg); + exit(EX_CONFIG); + } + arg_program_name = argv[0]; + } + if (!arg_time) { arg_time = 15; mx_log_warning("option '--runtime' or '-t' not used. Your job will get killed if it runs longer than the default of %d minutes.", arg_time); @@ -656,9 +667,6 @@ int main(int argc, char *argv[]) mx_log_warning("option '--runtime' specifies a runtime longer than 24h. Your job may get killed. Be sure to implement some check pointing."); } - if (!arg_program_name) - arg_program_name = argv[0]; - if (!(*arg_program_name)) { mx_log_crit(" is empty. Please check usage with '%s --help'.", program_invocation_short_name); exit(EX_CONFIG); diff --git a/mysql/create_tables b/mysql/create_tables index 9afd5c3..8b6b99a 100644 --- a/mysql/create_tables +++ b/mysql/create_tables @@ -1,136 +1,4 @@ -ALTER TABLE mxq_job - MODIFY COLUMN stats_utime_sec INT8 UNSIGNED NOT NULL DEFAULT 0, - MODIFY COLUMN stats_stime_sec INT8 UNSIGNED NOT NULL DEFAULT 0, - MODIFY COLUMN stats_real_sec INT8 UNSIGNED NOT NULL DEFAULT 0, - MODIFY COLUMN stats_maxrss INT8 NOT NULL DEFAULT 0, - MODIFY COLUMN stats_minflt INT8 NOT NULL DEFAULT 0, - MODIFY COLUMN stats_majflt INT8 NOT NULL DEFAULT 0, - MODIFY COLUMN stats_nswap INT8 NOT NULL DEFAULT 0, - MODIFY COLUMN stats_inblock INT8 NOT NULL DEFAULT 0, - MODIFY COLUMN stats_oublock INT8 NOT NULL DEFAULT 0, - MODIFY COLUMN stats_nvcsw INT8 NOT NULL DEFAULT 0, - MODIFY COLUMN stats_nivcsw INT8 NOT NULL DEFAULT 0; - - -ALTER TABLE mxq_group - MODIFY COLUMN stats_max_maxrss INT8 UNSIGNED NOT NULL DEFAULT 0, - MODIFY COLUMN stats_max_utime_sec INT8 UNSIGNED NOT NULL DEFAULT 0, - MODIFY COLUMN stats_max_stime_sec INT8 UNSIGNED NOT NULL DEFAULT 0, - MODIFY COLUMN stats_max_real_sec INT8 UNSIGNED NOT NULL DEFAULT 0; - -UPDATE mxq_job - JOIN mxq_group - ON mxq_job.group_id = mxq_group.group_id - SET mxq_job.date_start = GREATEST(date_start, date_submit), - mxq_job.date_end = GREATEST(date_start, date_submit), - mxq_job.job_status = 755 - WHERE job_status IN (750, 400) - AND group_jobs_running > 0 - AND stats_real_sec = 0 - AND date_end = 0; - -UPDATE mxq_job - JOIN mxq_group - ON mxq_job.group_id = mxq_group.group_id - SET mxq_job.date_start = LEAST(GREATEST(date_start,date_submit), group_date_end), - mxq_job.date_end = GREATEST(date_start,date_submit,group_date_end), - mxq_job.job_status = 755 - WHERE job_status IN (750, 400) - AND group_jobs_running = 0 - AND stats_real_sec = 0 - AND date_end = 0; -UPDATE mxq_job - SET job_status = 999 - WHERE job_status = 755; - - -UPDATE mxq_job - JOIN mxq_group - ON mxq_job.group_id = mxq_group.group_id - SET mxq_job.date_end = mxq_group.group_date_end - WHERE job_status = 1000 - AND date_end = 0; - -UPDATE mxq_job - JOIN mxq_group - ON mxq_job.group_id = mxq_group.group_id - SET mxq_job.date_start = mxq_group.group_date_end, - mxq_job.date_end = mxq_group.group_date_end - WHERE job_status = 990 - AND (date_start = 0 OR date_end = 0); - -UPDATE mxq_group - SET group_date_end=group_mtime - WHERE group_date_end = 0 - AND group_jobs_inq = 0 - AND group_jobs_running = 0; - -ALTER TABLE mxq_group - ADD COLUMN group_date_end TIMESTAMP NOT NULL DEFAULT 0 - AFTER group_mtime; - -ALTER TABLE mxq_group - CHANGE COLUMN stats_total_idle_sec stats_idle_sec INT8 UNSIGNED NOT NULL DEFAULT 0, - ADD COLUMN stats_total_wait_sec_finished INT8 UNSIGNED NOT NULL DEFAULT 0 - AFTER stats_total_real_sec_finished, - ADD COLUMN stats_wait_sec INT8 UNSIGNED NOT NULL DEFAULT 0 - AFTER stats_total_wait_sec, - ADD COLUMN stats_run_sec INT8 UNSIGNED NOT NULL DEFAULT 0 - AFTER stats_wait_sec; - -ALTER TABLE mxq_job - ADD COLUMN job_id_first INT8 UNSIGNED NULL DEFAULT NULL - AFTER job_id_old; - -ALTER TABLE mxq_job - ADD COLUMN job_id_old INT8 UNSIGNED NULL DEFAULT NULL - AFTER job_id_new; - -ALTER TABLE mxq_group - ADD COLUMN group_jobs_restarted INT8 UNSIGNED NOT NULL DEFAULT 0 - AFTER group_jobs_unknown; - -ALTER TABLE mxq_group - ADD COLUMN group_flags INT8 UNSIGNED NOT NULL DEFAULT 0 - AFTER group_status; - -ALTER TABLE mxq_group - ADD COLUMN group_jobs_inq INT8 UNSIGNED NOT NULL DEFAULT 0 - AFTER group_jobs; - -ALTER TABLE mxq_group - ADD COLUMN stats_total_utime_sec INT8 UNSIGNED NOT NULL DEFAULT 0, - ADD COLUMN stats_total_stime_sec INT8 UNSIGNED NOT NULL DEFAULT 0, - ADD COLUMN stats_total_real_sec INT8 UNSIGNED NOT NULL DEFAULT 0, - ADD COLUMN stats_total_wait_sec INT8 UNSIGNED NOT NULL DEFAULT 0, - ADD COLUMN stats_total_idle_sec INT8 UNSIGNED NOT NULL DEFAULT 0, - - ADD COLUMN stats_total_utime_sec_finished INT8 UNSIGNED NOT NULL DEFAULT 0, - ADD COLUMN stats_total_stime_sec_finished INT8 UNSIGNED NOT NULL DEFAULT 0, - ADD COLUMN stats_total_real_sec_finished INT8 UNSIGNED NOT NULL DEFAULT 0; - -ALTER TABLE mxq_job - ADD INDEX (group_id); - -ALTER TABLE mxq_job - ADD COLUMN job_id_new INT8 UNSIGNED NULL DEFAULT NULL - AFTER date_end; - -ALTER TABLE mxq_job - ADD COLUMN job_flags INT8 UNSIGNED NOT NULL DEFAULT 0 - AFTER job_status; - -ALTER TABLE mxq_group - ADD COLUMN group_jobs_cancelled INT8 UNSIGNED NOT NULL DEFAULT 0 - AFTER group_jobs_failed; - -ALTER TABLE mxq_group - ADD COLUMN group_jobs_unknown INT8 UNSIGNED NOT NULL DEFAULT 0 - AFTER group_jobs_cancelled; - -DROP TRIGGER mxq_update_group; - DROP TABLE mxq_group; CREATE TABLE IF NOT EXISTS mxq_group ( group_id INT8 UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT, @@ -249,6 +117,27 @@ CREATE TABLE IF NOT EXISTS mxq_job ( INDEX (server_id(767)) ); +DROP TABLE mxq_server; +CREATE TABLE IF NOT EXISTS mxq_server ( + host_id INT4 UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT, + host_hostname VARCHAR(511) NOT NULL DEFAULT 'localhost', + + server_id VARCHAR(511) NOT NULL DEFAULT 'default', + + host_slots INT2 UNSIGNED NOT NULL DEFAULT 1, + host_memory INT8 UNSIGNED NOT NULL DEFAULT 1024, + host_time INT4 UNSIGNED NOT NULL DEFAULT 15, + + host_jobs_running INT2 UNSIGNED NOT NULL DEFAULT 0, + host_slots_running INT2 UNSIGNED NOT NULL DEFAULT 0, + + host_mtime TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + + server_start TIMESTAMP DEFAULT 0, + server_stop TIMESTAMP DEFAULT 0, +); + + LOCK TABLES mxq_job WRITE, mxq_group WRITE; DELIMITER | @@ -378,24 +267,3 @@ CREATE TRIGGER mxq_update_job BEFORE UPDATE ON mxq_job | DELIMITER ; UNLOCK TABLES; - -DROP TABLE mxq_server; -CREATE TABLE IF NOT EXISTS mxq_server ( - host_id INT4 UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT, - host_hostname VARCHAR(511) NOT NULL DEFAULT 'localhost', - - server_id VARCHAR(511) NOT NULL DEFAULT 'default', - - host_slots INT2 UNSIGNED NOT NULL DEFAULT 1, - host_memory INT8 UNSIGNED NOT NULL DEFAULT 1024, - host_time INT4 UNSIGNED NOT NULL DEFAULT 15, - - host_jobs_running INT2 UNSIGNED NOT NULL DEFAULT 0, - host_slots_running INT2 UNSIGNED NOT NULL DEFAULT 0, - - host_mtime TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - - server_start TIMESTAMP DEFAULT 0, - server_stop TIMESTAMP DEFAULT 0, -); - diff --git a/web/pages/mxq/mxq b/web/pages/mxq/mxq.in similarity index 95% rename from web/pages/mxq/mxq rename to web/pages/mxq/mxq.in index 9722dc6..fa4dad8 100755 --- a/web/pages/mxq/mxq +++ b/web/pages/mxq/mxq.in @@ -76,7 +76,7 @@ EOF sub db_init { - $dbh= DBI->connect('DBI:mysql:mysql_read_default_file=/etc/mxq/mysql.cnf',undef,undef, { RaiseError => 1, AutoCommit => 1 }); + $dbh= DBI->connect('DBI:mysql:mysql_read_default_file=@MXQ_MYSQL_DEFAULT_FILE@',undef,undef, { RaiseError => 1, AutoCommit => 1 }); # mxq_group.h register_group_status( OK => 0, @@ -445,6 +445,32 @@ sub group_table { $sth->execute(@bind_args); $out.=group_table_rows($sth,\@head); + @cols=qw( + group_id + group_name + user_name + group_date_emd + group_status + group_jobs + group_jobs_finished + group_jobs_failed + group_jobs_cancelled + group_jobs_unknown + ); + + @head=qw( + group_id + group_name + user_name + date_end + group_status + jobs + finished + failed + cancelled + unknown + ); + $out .= '

Finished Groups

'; my $sth=$dbh->prepare('SELECT '.join(',',@cols).' FROM mxq_group WHERE '.$sql_clause.' AND (group_jobs_running=0 AND group_jobs_inq=0) ORDER BY group_id DESC');