Merge branch 'master' into manpages

* master: mxqsub: Disallow whitespace in command or command-alias mysql: Cleanup create_tables MXQ bump version git: ignore mxsub.1 in all directories web/Makefile: Generate cgi script replacing make variables Makefile: Add generic sed rules to generate scripts and manpages web/Makefile: Install cgi to CGIDIR (defaults to LIBEXECDIR/mxq/cgi) web: Show date_end instead of mtime in finished groups view web: Remove columns running and inq in finished groups view github: Create README.md MXQ bump version
mariux · Aug 12, 2015 · d4d5eac · d4d5eac
2 parents cdfdb3a + b69a01c
commit d4d5eac
Show file tree

Hide file tree

Showing 6 changed files with 112 additions and 163 deletions.
diff --git a/.gitignore b/.gitignore
@@ -15,14 +15,16 @@ mxq_log.o
 mx_mysql.o
 
 mxqsub
-/mxqsub.1
+mxqsub.1
 mxqdump
 mxqkill
 mxqd
 test_mx_util
 test_mx_log
 test_mx_mysql
 
+/web/pages/mxq/mxq
+
 *~
 
 gmon.out

diff --git a/Makefile b/Makefile
@@ -1,6 +1,6 @@
 MXQ_VERSION_MAJOR = 0
-MXQ_VERSION_MINOR = 9
-MXQ_VERSION_PATCH = 1
+MXQ_VERSION_MINOR = 11
+MXQ_VERSION_PATCH = 0
 MXQ_VERSION_EXTRA = "beta"
 
 MXQ_VERSION = ${MXQ_VERSION_MAJOR}.${MXQ_VERSION_MINOR}.${MXQ_VERSION_PATCH}
@@ -39,6 +39,8 @@ ifeq ($(notdir ${LIBEXECDIR}),mxq)
     override LIBEXECDIR := $(patsubst %/,%,$(dir ${LIBEXECDIR}))
 endif
 
+CGIDIR = ${LIBEXECDIR}/mxq/cgi
+
 ##############################################################################
 
 MXQ_MYSQL_DEFAULT_FILE  = ${SYSCONFDIR}/mxq/mysql.cnf
@@ -101,6 +103,22 @@ quiet-install = $(call quiet-command,install -m ${1} ${2} ${3},"INSTALL ${3} [mo
 quiet-installdir = $(call quiet-command,install -m ${1} -d ${2},"  MKDIR ${2} [mode=${1}]")
 quiet-installforuser = $(call quiet-command,install -m ${1} -o ${2} -g ${3} ${4} ${5},"INSTALL ${5} (user=${2} group=${3}) [mode=${1}]")
 
+########################################################################
+
+sed-rules = -e 's,@PREFIX@,${PREFIX},g' \
+            -e 's,@EPREFIX@,${EPREFIX},g' \
+            -e 's,@BINDIR@,${BINDIR},g' \
+            -e 's,@SBINDIR@,${SBINDIR},g' \
+            -e 's,@LIBDIR@,${LIBDIR},g' \
+            -e 's,@SYSCONFDIR@,${SYSCONFDIR},g' \
+            -e 's,@DEFCONFDIR@,${DEFCONFDIR},g' \
+            -e 's,@LIBEXECDIR@,${LIBEXECDIR},g' \
+            -e 's,@BEE_VERSION@,${BEE_VERSION},g' \
+            -e 's,@DATADIR@,${DATADIR},g' \
+            -e 's,@MXQ_VERSION@,${MXQ_VERSION},g' \
+            -e 's,@MXQ_MYSQL_DEFAULT_FILE@,${MXQ_MYSQL_DEFAULT_FILE},g' \
+
+
 ########################################################################
 
 %.o: %.c Makefile
@@ -111,6 +129,9 @@ quiet-installforuser = $(call quiet-command,install -m ${1} -o ${2} -g ${3} ${4}
 %: %.o
 	$(call quiet-command,${CC} -o $@ $^ $(LDFLAGS) $(LDLIBS), "   LINK $@")
 
+%: %.in Makefile
+	$(call quiet-command,sed ${sed-rules} $< >$@, "    GEN $@")
+
 ########################################################################
 
 .SECONDARY:
@@ -121,7 +142,7 @@ manpages/%: manpages/%.xml
 	$(call quiet-command,xmlto --stringparam man.output.quietly=1 man $^ -o manpages, "  XMLTO $@")
 
 %: manpages/% Makefile
-	$(call quiet-command,sed -e "s/@MXQ_VERSION@/${MXQ_VERSION}/" $< >$@, "    GEN $@")
+	$(call quiet-command,sed ${sed-rules} $< >$@, "    GEN $@")
 
 ########################################################################
 
@@ -179,6 +200,7 @@ install::
 	$(call quiet-installdir,0755,${DESTDIR}${SBINDIR})
 	$(call quiet-installdir,0755,${DESTDIR}${SYSCONFDIR}/mxq)
 	$(call quiet-installdir,0755,${DESTDIR}${MAN1DIR})
+	$(call quiet-installdir,0755,${DESTDIR}${CGIDIR})
 
 ########################################################################
 
@@ -419,6 +441,15 @@ clean: CLEAN += mxqsub.1
 
 ########################################################################
 
+build: web/pages/mxq/mxq
+
+clean: CLEAN += web/pages/mxq/mxq
+
+install:: web/pages/mxq/mxq
+	$(call quiet-install,0755,$^,${DESTDIR}${CGIDIR}/mxq)
+
+########################################################################
+
 test_mx_util.o: $(mx_util.h)
 clean: CLEAN += test_mx_util.o
 

diff --git a/README.md b/README.md
@@ -0,0 +1,14 @@
+# mxq
+MXQ - mariux64 job scheduling system
+
+## Installation
+### Install using `GNU make`
+```
+make 
+make install [DESTDIR=...]
+```
+### Install using `bee`
+```
+bee init $(bee download git://github.molgen.mpg.de/mariux64/mxq.git) -e
+bee install mxq
+```
diff --git a/mxqsub.c b/mxqsub.c
@@ -551,8 +551,10 @@ int main(int argc, char *argv[])
 
             case 'a':
                 p = strchr(optctl.optarg, ' ');
-                if (p)
-                    *p = 0;
+                if (p) {
+                    mx_log_crit("--command-alias '%s': String contains whitespace characters.", optctl.optarg);
+                    exit(EX_CONFIG);
+                }
                 if (!(*optctl.optarg)) {
                     mx_log_crit("--command-alias '%s': String is empty.", optctl.optarg);
                     exit(EX_CONFIG);
@@ -647,6 +649,15 @@ int main(int argc, char *argv[])
 
     /* from this point values in argc,argv are the ones of the cluster job  */
 
+    if (!arg_program_name) {
+        p = strchr(argv[0], ' ');
+        if (p) {
+            mx_log_crit("<command> contains whitespace characters. Please set --command-alias if this is intended.", optctl.optarg);
+            exit(EX_CONFIG);
+        }
+        arg_program_name = argv[0];
+    }
+
     if (!arg_time) {
         arg_time = 15;
         mx_log_warning("option '--runtime' or '-t' not used. Your job will get killed if it runs longer than the default of %d minutes.", arg_time);
@@ -656,9 +667,6 @@ int main(int argc, char *argv[])
         mx_log_warning("option '--runtime' specifies a runtime longer than 24h. Your job may get killed. Be sure to implement some check pointing.");
     }
 
-    if (!arg_program_name)
-        arg_program_name = argv[0];
-
     if (!(*arg_program_name)) {
         mx_log_crit("<command> is empty. Please check usage with '%s --help'.", program_invocation_short_name);
         exit(EX_CONFIG);

diff --git a/mysql/create_tables b/mysql/create_tables
@@ -1,136 +1,4 @@
 
-ALTER TABLE mxq_job
-   MODIFY COLUMN stats_utime_sec  INT8 UNSIGNED NOT NULL DEFAULT 0,
-   MODIFY COLUMN stats_stime_sec  INT8 UNSIGNED NOT NULL DEFAULT 0,
-   MODIFY COLUMN stats_real_sec   INT8 UNSIGNED NOT NULL DEFAULT 0,
-   MODIFY COLUMN stats_maxrss   INT8 NOT NULL DEFAULT 0,
-   MODIFY COLUMN stats_minflt   INT8 NOT NULL DEFAULT 0,
-   MODIFY COLUMN stats_majflt   INT8 NOT NULL DEFAULT 0,
-   MODIFY COLUMN stats_nswap    INT8 NOT NULL DEFAULT 0,
-   MODIFY COLUMN stats_inblock  INT8 NOT NULL DEFAULT 0,
-   MODIFY COLUMN stats_oublock  INT8 NOT NULL DEFAULT 0,
-   MODIFY COLUMN stats_nvcsw    INT8 NOT NULL DEFAULT 0,
-   MODIFY COLUMN stats_nivcsw   INT8 NOT NULL DEFAULT 0;
-
-
-ALTER TABLE mxq_group
-   MODIFY COLUMN stats_max_maxrss    INT8 UNSIGNED NOT NULL DEFAULT 0,
-   MODIFY COLUMN stats_max_utime_sec INT8 UNSIGNED NOT NULL DEFAULT 0,
-   MODIFY COLUMN stats_max_stime_sec INT8 UNSIGNED NOT NULL DEFAULT 0,
-   MODIFY COLUMN stats_max_real_sec  INT8 UNSIGNED NOT NULL DEFAULT 0;
-
-UPDATE mxq_job
-    JOIN mxq_group
-        ON mxq_job.group_id = mxq_group.group_id
-    SET mxq_job.date_start = GREATEST(date_start, date_submit),
-        mxq_job.date_end   = GREATEST(date_start, date_submit),
-        mxq_job.job_status = 755
-    WHERE job_status IN (750, 400)
-        AND group_jobs_running > 0
-        AND stats_real_sec     = 0
-        AND date_end           = 0;
-
-UPDATE mxq_job
-    JOIN mxq_group
-        ON mxq_job.group_id = mxq_group.group_id
-    SET mxq_job.date_start = LEAST(GREATEST(date_start,date_submit), group_date_end),
-        mxq_job.date_end   = GREATEST(date_start,date_submit,group_date_end),
-        mxq_job.job_status = 755
-    WHERE job_status IN (750, 400)
-        AND group_jobs_running = 0
-        AND stats_real_sec     = 0
-        AND date_end           = 0;
-UPDATE mxq_job
-    SET job_status = 999
-    WHERE job_status = 755;
-
-
-UPDATE mxq_job
-    JOIN mxq_group
-        ON mxq_job.group_id = mxq_group.group_id
-    SET mxq_job.date_end = mxq_group.group_date_end
-    WHERE job_status = 1000
-        AND date_end = 0;
-
-UPDATE mxq_job
-    JOIN mxq_group
-        ON mxq_job.group_id = mxq_group.group_id
-    SET mxq_job.date_start = mxq_group.group_date_end,
-        mxq_job.date_end = mxq_group.group_date_end
-    WHERE job_status = 990
-        AND (date_start = 0 OR date_end = 0);
-
-UPDATE mxq_group
-    SET group_date_end=group_mtime
-    WHERE group_date_end = 0
-        AND group_jobs_inq = 0
-        AND group_jobs_running = 0;
-
-ALTER TABLE mxq_group
-    ADD COLUMN group_date_end TIMESTAMP NOT NULL DEFAULT 0
-        AFTER group_mtime;
-
-ALTER TABLE mxq_group
-   CHANGE COLUMN stats_total_idle_sec stats_idle_sec INT8 UNSIGNED NOT NULL DEFAULT 0,
-   ADD COLUMN stats_total_wait_sec_finished INT8 UNSIGNED NOT NULL DEFAULT 0
-      AFTER stats_total_real_sec_finished,
-   ADD COLUMN stats_wait_sec INT8 UNSIGNED NOT NULL DEFAULT 0
-      AFTER stats_total_wait_sec,
-   ADD COLUMN stats_run_sec INT8 UNSIGNED NOT NULL DEFAULT 0
-      AFTER stats_wait_sec;
-
-ALTER TABLE mxq_job
-   ADD COLUMN job_id_first INT8 UNSIGNED   NULL DEFAULT NULL
-   AFTER job_id_old;
-
-ALTER TABLE mxq_job
-   ADD COLUMN job_id_old INT8 UNSIGNED   NULL DEFAULT NULL
-   AFTER job_id_new;
-
-ALTER TABLE mxq_group
-   ADD COLUMN group_jobs_restarted INT8 UNSIGNED NOT NULL DEFAULT 0
-   AFTER group_jobs_unknown;
-
-ALTER TABLE mxq_group
-   ADD COLUMN group_flags INT8 UNSIGNED NOT NULL DEFAULT 0
-   AFTER group_status;
-
-ALTER TABLE mxq_group
-   ADD COLUMN group_jobs_inq INT8 UNSIGNED NOT NULL DEFAULT 0
-   AFTER group_jobs;
-
-ALTER TABLE mxq_group
-   ADD COLUMN stats_total_utime_sec INT8 UNSIGNED NOT NULL DEFAULT 0,
-   ADD COLUMN stats_total_stime_sec INT8 UNSIGNED NOT NULL DEFAULT 0,
-   ADD COLUMN stats_total_real_sec  INT8 UNSIGNED NOT NULL DEFAULT 0,
-   ADD COLUMN stats_total_wait_sec  INT8 UNSIGNED NOT NULL DEFAULT 0,
-   ADD COLUMN stats_total_idle_sec  INT8 UNSIGNED NOT NULL DEFAULT 0,
-
-   ADD COLUMN stats_total_utime_sec_finished    INT8 UNSIGNED NOT NULL DEFAULT 0,
-   ADD COLUMN stats_total_stime_sec_finished    INT8 UNSIGNED NOT NULL DEFAULT 0,
-   ADD COLUMN stats_total_real_sec_finished     INT8 UNSIGNED NOT NULL DEFAULT 0;
-
-ALTER TABLE mxq_job
-   ADD INDEX (group_id);
-
-ALTER TABLE mxq_job
-   ADD COLUMN job_id_new     INT8 UNSIGNED   NULL DEFAULT NULL
-   AFTER date_end;
-
-ALTER TABLE mxq_job
-   ADD COLUMN job_flags INT8 UNSIGNED NOT NULL DEFAULT 0
-   AFTER job_status;
-
-ALTER TABLE mxq_group
-   ADD COLUMN group_jobs_cancelled INT8 UNSIGNED NOT NULL DEFAULT 0
-   AFTER group_jobs_failed;
-
-ALTER TABLE mxq_group
-   ADD COLUMN group_jobs_unknown INT8 UNSIGNED NOT NULL DEFAULT 0
-   AFTER group_jobs_cancelled;
-
-DROP TRIGGER mxq_update_group;
-
 DROP TABLE mxq_group;
 CREATE TABLE IF NOT EXISTS mxq_group (
    group_id       INT8 UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT,
@@ -249,6 +117,27 @@ CREATE TABLE IF NOT EXISTS mxq_job (
    INDEX (server_id(767))
 );
 
+DROP TABLE mxq_server;
+CREATE TABLE IF NOT EXISTS mxq_server (
+   host_id          INT4 UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT,
+   host_hostname    VARCHAR(511)  NOT NULL DEFAULT 'localhost',
+
+   server_id        VARCHAR(511)  NOT NULL DEFAULT 'default',
+
+   host_slots       INT2 UNSIGNED NOT NULL DEFAULT 1,
+   host_memory      INT8 UNSIGNED NOT NULL DEFAULT 1024,
+   host_time        INT4 UNSIGNED NOT NULL DEFAULT 15,
+
+   host_jobs_running    INT2 UNSIGNED NOT NULL DEFAULT 0,
+   host_slots_running   INT2 UNSIGNED NOT NULL DEFAULT 0,
+
+   host_mtime       TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+
+   server_start     TIMESTAMP DEFAULT 0,
+   server_stop      TIMESTAMP DEFAULT 0,
+);
+
+
 
 LOCK TABLES mxq_job WRITE, mxq_group WRITE;
 DELIMITER |
@@ -378,24 +267,3 @@ CREATE TRIGGER mxq_update_job BEFORE UPDATE ON mxq_job
 |
 DELIMITER ;
 UNLOCK TABLES;
-
-DROP TABLE mxq_server;
-CREATE TABLE IF NOT EXISTS mxq_server (
-   host_id          INT4 UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT,
-   host_hostname    VARCHAR(511)  NOT NULL DEFAULT 'localhost',
-
-   server_id        VARCHAR(511)  NOT NULL DEFAULT 'default',
-
-   host_slots       INT2 UNSIGNED NOT NULL DEFAULT 1,
-   host_memory      INT8 UNSIGNED NOT NULL DEFAULT 1024,
-   host_time        INT4 UNSIGNED NOT NULL DEFAULT 15,
-
-   host_jobs_running    INT2 UNSIGNED NOT NULL DEFAULT 0,
-   host_slots_running   INT2 UNSIGNED NOT NULL DEFAULT 0,
-
-   host_mtime       TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-
-   server_start     TIMESTAMP DEFAULT 0,
-   server_stop      TIMESTAMP DEFAULT 0,
-);
-
diff --git a/web/pages/mxq/mxq → web/pages/mxq/mxq.in b/web/pages/mxq/mxq → web/pages/mxq/mxq.in
@@ -76,7 +76,7 @@ EOF
 
 sub db_init {
 
-	$dbh= DBI->connect('DBI:mysql:mysql_read_default_file=/etc/mxq/mysql.cnf',undef,undef, { RaiseError => 1, AutoCommit => 1 });
+	$dbh= DBI->connect('DBI:mysql:mysql_read_default_file=@MXQ_MYSQL_DEFAULT_FILE@',undef,undef, { RaiseError => 1, AutoCommit => 1 });
 	# mxq_group.h
 	register_group_status(
 		OK          =>     0,
@@ -445,6 +445,32 @@ sub group_table {
 	$sth->execute(@bind_args);
 	$out.=group_table_rows($sth,\@head);
 
+	@cols=qw(
+        group_id
+        group_name
+        user_name
+        group_date_emd
+        group_status
+        group_jobs
+        group_jobs_finished
+        group_jobs_failed
+        group_jobs_cancelled
+        group_jobs_unknown
+	);
+
+	@head=qw(
+        group_id
+        group_name
+        user_name
+        date_end
+        group_status
+        jobs
+        finished
+        failed
+        cancelled
+        unknown
+	);
+
 	$out .= '<h2>Finished Groups</h2>';
 
 	my $sth=$dbh->prepare('SELECT '.join(',',@cols).' FROM mxq_group WHERE '.$sql_clause.' AND (group_jobs_running=0 AND group_jobs_inq=0) ORDER BY group_id DESC');